sentences = ["Do you know?", "why it so popular.", "The quick brown fox runs over the lazy dog"]
with open("./data/week1_test.txt", "w", encoding="utf-8") as filePtr:
  for sentence in sentences:
    filePtr.write(sentence + "\n")

with open("./data/week1_test.txt", "r", encoding="utf-8") as filePtr:
  for line in filePtr:
    print(line)

import random
random.seed(42) #make results reproducible,

random.random() #return random number between [0.0 and 1.0)
>>> 0.35553263284394376
random.uniform(0, 10) #return a random floating point number N from uniform distribution such that a <= N <= b, where a=lower_end, b=higher_end
>>> 3.58
random.randint(0, 10) #generate a random integer between two endpoints in Python
>>> 7
random.gauss(mu, sigma) #return a random floating point number with gaussian distribution.

items = ['one', 'two', 'three', 'four', 'five']
random.choice(items) #choosing multiple elements from a sequence with replacement (duplicates are possible):
>>> 'four'
random.choices(items, k=2)
>>> ['three', 'three']
random.choices(items, k=3)
>>> ['three', 'five', 'four']


random.shuffle(items) #randomize a sequence in-place
>>> ['four', 'three', 'two', 'one', 'five']

import copy
a = [[0, 1], 2, 3]

#Shallow Copy: will only create a new object for the parent layer.
#it will NOT create a new object for any of the child layer.
b = copy.copy(a)

#only b has 4 appended, not a 
b.append(4) #a: [[0, 1], 2, 3]; b: [[0, 1], 2, 3, 4]
#both a[0][0] and b[0][0] will be updated to 9
b[0][0] = 9 #a: [[9, 1], 2, 3]; b: [[9, 1], 2, 3, 4]


c = copy.deepcopy(a)
c.append(5) #a: [[9, 1], 2, 3], c: [[9, 1], 2, 3, 5]
c[0][0] = 8 #a: [[9, 1], 2, 3], c: [[8, 1], 2, 3, 5]

from google.colab import drive
drive.mount('/content/drive')

!python --version #to check python version
!pip list #to list python modules

!apt-get install python3.8 #get the lastest python version on Colab
!python3.8 -m pip install transformers #to install new packages

#Run Shell Command
!python -c 'print("Hello Word")'
!python ./week1/hello_world.py # run hello_world.py

#Load models:
pickle.dump(model, open(path,"wb"))
#Load models:
pickle.load(open(path,"rb"))

from tqdm import tqdm

X_train = []
for row in tqdm(train_df['tokens']):
    model_vector = document_vector(row, sg_w2v_model)
    X_train.append(model_vector)
    
#100%|██████████| 1193/1193 [07:11<00:00,  2.77it/s]

#drop = True, to not retain the old index as a column
df = df.reset_index(drop=True)

train_bkp = train_df.copy(deep=True)
test_bkp = test_df.copy(deep=True)

#Save DataFrame
df.to_pickle('../data/cleaned_train_v2.pkl')
#Load DataFrame
df = pd.read_pickle('../data/cleaned_test_v2.pkl')

num_regex = re.compile(r'[0-9]+')
df["col"].apply(lambda x: len(num_regex.findall(x)))
 
#preprocess_data: is a separate function
df["col"].apply(lambda doc: preprocess_data(doc))  

arr1 = np.arange(5)
arr2 = np.array(range(5, 0, -1))
print(arr1 + arr2)
print(arr1 - arr2)
print(arr1 * arr2)
print(arr1 / arr2)

#arithmetic operations with scalars propagate the scalar argument to each element
print(arr1 + 1)
print(arr1 * 2)
print(1 / arr2)

#comparisons between arrays of the same size
print(arr1 > arr2) #[False False False  True  True]

from scipy.sparse import coo_matrix, hstack
A = coo_matrix([[1, 2], [3, 4]])
B = coo_matrix([[5], [6]])
# H-Stack Example
hstack([A,B]).toarray()
#array([[1, 2, 5],
#       [3, 4, 6]])

# V-Stack Example
vstack([A, B]).toarray()
#array([[1, 2],
#       [3, 4],
#       [5, 6]])

a = np.array([1, 2, 3])
b = np.array([4, 5, 6])

# H-Stack Example
np.hstack([a,b])
#array([1, 2, 3, 4, 5, 6])

# V-Stack Example
np.vstack([a,b])
#array([[1, 2, 3],
#       [4, 5, 6]])

# Get current yticks: An array of the values displayed on the y-axis (150, 175, 200, etc.)
ticks = ax.get_yticks()
# Format those values into strings beginning with dollar sign
new_labels = [f"${int(tick)}" for tick in ticks]
# Set the new labels
ax.set_yticklabels(new_labels)

fig = plt.figure()
# Any plot in between
fig.savefig(img_path.png)

train_df.duplicated().sum() #check if any duplication

#keeping only the first instance of the duplicate rows 
bool_series = train_df.duplicated(keep='first') 
train_df = train_df[~bool_series]

label_enocoder = sklearn.preprocessing.LabelEncoder()
# Encoding "Target" to number 
y_train = label_enocoder.fit_transform(labeled_train_df['label'].values)

my_tags = list(label_enocoder.classes_)
# my_tags = ['Addendum', 'Endorsement', 'Finance_Report', 'Insurance_Sheet']

# Label & Name mapping
le_name_mapping = dict(zip(label_enocoder.classes_, label_enocoder.transform(label_enocoder.classes_)))
#{'Addendum': 0,
#'Endorsement': 1,
#'Finance_Report': 2,
#'Insurance_Sheet': 3,

#Define Model
models = [RandomForestClassifier(n_estimators=100, max_depth=5, random_state=2022),
          LinearSVC(),
          MultinomialNB(),
          LogisticRegression(random_state=2022)]

#Define k-fold
cv = 5
kfold = StratifiedKFold(cv, shuffle=True, random_state = 2022)
cv_df = pd.DataFrame(index=range(cv * len(models))) #create a DataFrame with (k-fold * number of models) rows

entries = []
# Loop through models
for model in models:
    model_name = model.__class__.__name__
    accuracies = cross_val_score(model, X_train_vect, y_train, scoring='accuracy', cv=kfold)
    for fold_idx, accuracy in enumerate(accuracies):
        entries.append((model_name, fold_idx, accuracy))
#List down accuracy per fold for each model  
cv_df = pd.DataFrame(entries, columns=['model_name', 'fold_idx', 'accuracy'])

#Plot Base-Line Model Performance
plt.figure(figsize=(6,4))
sns.boxplot(x='model_name', y='accuracy', 
            data=cv_df, 
            color='lightblue', 
            showmeans=True)
plt.title("Boxplot of Base-line Accuracy")
plt.xticks(rotation = 90)
plt.show()

#Summary
mean_accuracy = cv_df.groupby('model_name').accuracy.mean()
std_accuracy = cv_df.groupby('model_name').accuracy.std()

acc = pd.concat([mean_accuracy, std_accuracy], axis= 1, ignore_index=True)
acc.columns = ['Mean Accuracy', 'Standard deviation']

# sort by accuracy
acc.sort_values(by=['Mean Accuracy'], axis=0 ,ascending=False, inplace=True)

# my_tags = ['Addendum', 'Endorsement', 'Finance_Report', 'Insurance_Sheet', 'NDA', 'Payslips', 'Term_Sheet']
# target_names=my_tags
print(classification_report(y_test, y_pred_nb, target_names=my_tags))

result_df = pd.DataFrame(zip(y_test, y_pred)), columns=["Label", "Pred"])
result_df = result_df[result_df.Label == "RandomPDFs"]
result_df["Pred"].value_counts()
#Finance_Report     77
#Others              9
#Endorsement         6