Upload 8 files

config.py

@@ -0,0 +1,55 @@
+import transformers
+import os
+import torch
+
+MAX_LEN = 150 #256
+TRAIN_BATCH_SIZE = 8
+VALID_BATCH_SIZE = 4
+EPOCHS = 5
+
+# Folder to contain all the datasets
+DATASET_LOCATION = "" # 
+MODEL_PATH = "/mnt/data/group3/gaurish/SentimentAnalyserLVTwitter/bert-sentiment/src/trained_models/mbert-7epoch-lower/model.bin"
+
+device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')
+
+# MBERT Raw Version
+# BERT_PATH = "bert-base-multilingual-cased"
+
+# 2 EPOCH Version
+# BERT_PATH = "bert-twitter-fine-tunning/LatvianTwittermBERT-v1"
+
+# 7 EPOCH Version
+BERT_PATH = "FFZG-cleopatra/bert-emoji-latvian-twitter"
+
+# 7 EPOCH Version + emoticons
+# BERT_PATH = "bert-twitter-language-pretraining/models/LatvianTwittermBERT-v2/checkpoint-106000"
+
+# TODO check if lower casing is required
+# BertTokenizer
+TOKENIZER = transformers.BertTokenizer.from_pretrained(
+    BERT_PATH,
+    do_lower_case=True
+)
+
+#####################################################################################################################################
+# Electra
+# Step 1: Model path
+# BERT_PATH = "lmtuners/experiments/disc_lm_small/electra-small/discriminator/final"
+# #"lmtuners/experiments/disc_lm_small/albert-small/final"
+
+# # Step 2: Vocab and Lowercase setting
+# TOKENIZER = transformers.BertTokenizer.from_pretrained(
+# 	"lmtuners/experiments/disc_lm_small/lvtwitterbwpt-vocab-lower_accent.txt",
+#     # "lmtuners/experiments/disc_lm_small/bert-base-multilingual-cased-vocab.txt",
+#     do_lower_case=True
+# )
+
+# ALBERT_CONFIG = transformers.AlbertConfig(
+#         vocab_size=len(TOKENIZER), #.get_vocab_size(),
+#         hidden_size=256,
+#         embedding_size=128,
+#         num_hidden_layers=12,
+#         num_attention_heads=4,
+#         intermediate_size=1024,
+#         max_position_embeddings=128)

dataset.py

@@ -0,0 +1,40 @@
+import config
+import torch
+
+
+class BERTDataset:
+    def __init__(self, review, target):
+        self.review = review
+        self.target = target
+        self.tokenizer = config.TOKENIZER
+        self.max_len = config.MAX_LEN
+    
+    def __len__(self):
+        return len(self.review)
+    
+    def __getitem__(self, item):
+        review = str(self.review[item])
+        review = " ".join(review.split())
+
+        inputs = self.tokenizer.encode_plus(
+            review,
+            None,
+            add_special_tokens=True,
+            max_length=self.max_len
+        )
+
+        ids = inputs["input_ids"]
+        mask = inputs["attention_mask"]
+        token_type_ids = inputs["token_type_ids"]
+
+        padding_length = self.max_len - len(ids)
+        ids = ids + ([0] * padding_length)
+        mask = mask + ([0] * padding_length)
+        token_type_ids = token_type_ids + ([0] * padding_length)
+
+        return {
+            'ids': torch.tensor(ids, dtype=torch.long),
+            'mask': torch.tensor(mask, dtype=torch.long),
+            'token_type_ids': torch.tensor(token_type_ids, dtype=torch.long),
+            'targets': torch.tensor(self.target[item], dtype=torch.float)
+        }

engine.py

@@ -0,0 +1,116 @@
+import torch
+import torch.nn as nn
+from tqdm import tqdm
+from utils import categorical_accuracy
+
+
+def loss_fn(outputs, targets):
+    return nn.CrossEntropyLoss()(outputs, targets)
+
+
+def train_fn(data_loader, model, optimizer, device, scheduler):
+    model.train()
+    train_loss, train_acc = 0.0, 0.0
+
+    for bi, d in tqdm(enumerate(data_loader), total=len(data_loader)):
+        ids = d["ids"]
+        token_type_ids = d["token_type_ids"]
+        mask = d["mask"]
+        targets = d["targets"]
+
+        ids = ids.to(device, dtype=torch.long)
+        token_type_ids = token_type_ids.to(device, dtype=torch.long)
+        mask = mask.to(device, dtype=torch.long)
+        targets = targets.to(device, dtype=torch.long)
+        
+        optimizer.zero_grad()
+        outputs = model(
+            ids=ids,
+            mask=mask,
+            token_type_ids=token_type_ids
+        )
+
+        loss = loss_fn(outputs, targets)
+        loss.backward()
+        
+        optimizer.step()
+        scheduler.step()
+        train_loss += loss.item()
+        pred_labels = torch.argmax(outputs, dim=1)
+        # (pred_labels == targets).sum().item()
+        train_acc += categorical_accuracy(outputs, targets).item()
+
+    train_loss /= len(data_loader)
+    train_acc /= len(data_loader)
+    return train_loss, train_acc
+
+
+def eval_fn(data_loader, model, device):
+    model.eval()
+    eval_loss, eval_acc = 0.0, 0.0
+    fin_targets = []
+    fin_outputs = []
+    with torch.no_grad():
+        for bi, d in tqdm(enumerate(data_loader), total=len(data_loader)):
+            ids = d["ids"]
+            token_type_ids = d["token_type_ids"]
+            mask = d["mask"]
+            targets = d["targets"]
+
+            ids = ids.to(device, dtype=torch.long)
+            token_type_ids = token_type_ids.to(device, dtype=torch.long)
+            mask = mask.to(device, dtype=torch.long)
+            targets = targets.to(device, dtype=torch.long)
+
+            outputs = model(
+                ids=ids,
+                mask=mask,
+                token_type_ids=token_type_ids
+            )
+            loss = loss_fn(outputs, targets)
+            eval_loss += loss.item()
+            pred_labels = torch.argmax(outputs, axis=1)
+            # (pred_labels == targets).sum().item()
+            eval_acc += categorical_accuracy(outputs, targets).item()
+            fin_targets.extend(targets.cpu().detach().numpy().tolist())
+            fin_outputs.extend(torch.argmax(
+                outputs, dim=1).cpu().detach().numpy().tolist())
+    eval_loss /= len(data_loader)
+    eval_acc /= len(data_loader)
+    return fin_outputs, fin_targets, eval_loss, eval_acc
+
+
+
+def predict_fn(data_loader, model, device, extract_features=False):
+    model.eval()
+   
+    fin_outputs = []
+    extracted_features =[]
+    with torch.no_grad():
+        for bi, d in tqdm(enumerate(data_loader), total=len(data_loader)):
+            ids = d["ids"]
+            token_type_ids = d["token_type_ids"]
+            mask = d["mask"]
+            # targets = d["targets"]
+
+            ids = ids.to(device, dtype=torch.long)
+            token_type_ids = token_type_ids.to(device, dtype=torch.long)
+            mask = mask.to(device, dtype=torch.long)
+            
+            outputs = model(
+                ids=ids,
+                mask=mask,
+                token_type_ids=token_type_ids
+            )
+            if extract_features:
+                extracted_features.extend( model.extract_features(
+                ids=ids,
+                mask=mask,
+                token_type_ids=token_type_ids
+            ).cpu().detach().numpy().tolist())  
+            
+            fin_outputs.extend(torch.argmax(
+                outputs, dim=1).cpu().detach().numpy().tolist())
+
+    return fin_outputs, extracted_features
+

main.py

@@ -0,0 +1,123 @@
+import flask
+import torch
+from flask import Flask, render_template, request
+from utils import label_full_decoder
+import sys
+import config
+import dataset
+import engine
+from model import BERTBaseUncased
+from tokenizer import tokenizer
+from werkzeug.serving import run_simple
+# from werkzeug.wsgi import DispatcherMiddleware
+
+
+T = tokenizer.TweetTokenizer(
+    preserve_handles=True, preserve_hashes=True, preserve_case=False, preserve_url=False)
+
+app = Flask(__name__,
+            static_folder='app_resources/static',
+            static_url_path='/sentimentanalyzer',
+            instance_relative_config=True,
+            template_folder='app_resources/templates/public')
+
+
+MODEL = None
+DEVICE = config.device
+
+
+def preprocess(text):
+    tokens = T.tokenize(text)
+    print(tokens, file=sys.stderr)
+    ptokens = []
+    for index, token in enumerate(tokens):
+        if "@" in token:
+            if index > 0:
+                # check if previous token was mention
+                if "@" in tokens[index-1]:
+                    pass
+                else:
+                    ptokens.append("mention_0")
+            else:
+                ptokens.append("mention_0")
+        else:
+            ptokens.append(token)
+
+    print(ptokens, file=sys.stderr)
+    return " ".join(ptokens)
+
+
+def sentence_prediction(sentence):
+    sentence = preprocess(sentence)
+    model_path = config.MODEL_PATH
+
+    test_dataset = dataset.BERTDataset(
+        review=[sentence],
+        target=[0]
+    )
+
+    test_data_loader = torch.utils.data.DataLoader(
+        test_dataset,
+        batch_size=config.VALID_BATCH_SIZE,
+        num_workers=3
+    )
+
+    device = config.device
+
+    model = BERTBaseUncased()
+    model.load_state_dict(torch.load(
+        model_path, map_location=torch.device(device)))
+    model.to(device)
+
+    outputs, [] = engine.predict_fn(test_data_loader, model, device)
+    print(outputs)
+    return outputs[0]
+
+
+@app.route("/sentimentanalyzer/predict", methods=['POST'])
+def predict():
+    print(request.form, file=sys.stderr)
+    # print([(x) for x in request.get_json()],file=sys.stderr)
+    # sentence = request.get_json().get("sentence","")
+    sentence = request.form['sentence']
+    if sentence:
+        print(sentence, file=sys.stderr)
+        prediction = sentence_prediction(sentence)
+        response = {}
+        response["response"] = {
+            'sentence': sentence,
+            'prediction': label_full_decoder(prediction),
+        }
+        return flask.jsonify(response)
+    else:
+        return flask.jsonify({"error": "empty text"})
+
+
+@app.route("/sentimentanalyzer/")
+def index():
+    return render_template("index.html")
+
+
+@app.route("/sentimentanalyzer/demo")
+def demo():
+    return render_template("demo.html")
+
+
+@app.route("/sentimentanalyzer/models")
+def models():
+    return render_template("models.html")
+
+
+@app.route("/sentimentanalyzer/about")
+def about():
+    return render_template("about.html")
+
+
+if __name__ == "__main__":
+    MODEL = BERTBaseUncased()
+    MODEL.load_state_dict(torch.load(
+        config.MODEL_PATH, map_location=torch.device(DEVICE)))
+    MODEL.eval()
+
+    app.run("127.0.0.1", port=1095, debug=True)
+# host="http://cleopatra.ijs.si/sentimentanalyzer"

metrics.py

@@ -0,0 +1,76 @@
+import numpy as np
+import pandas as pd
+
+import sys
+
+file_path = sys.argv[1] 
+
+metric = {}
+stastics = {}
+
+with open(file_path) as input_file:
+    current_epoch=None
+    for line in input_file:
+        line= line.strip()
+
+        if line.find("Bert Model") >-1:
+            stastics["Bert Model"] =line
+        if line.find("Current date and time") >-1:
+            stastics["Current date and time"] =line
+        if line.find("Train file") >-1:
+            stastics["Train file"] =line
+        if line.find("Valid file") >-1:
+            stastics["Valid file"] =line
+        if line.find("Test file") >-1:
+            stastics["Test file"] =line
+        if line.find("Train size") >-1:
+            stastics["Train size"] =line
+        if line.find("Valid size") >-1:
+            stastics["Valid size"] =line
+        if line.find("Test size") >-1:
+            stastics["Test size"] =line
+        tokens = line.split()
+        for token in tokens:
+            if  token.find("epoch")==0:
+                metric[token]=[]
+                current_epoch=token
+                continue
+            if  token.find("train_loss")>-1:
+                metric[current_epoch].append(token)
+            if  token.find("val_loss")>-1:
+                metric[current_epoch].append(token)
+            if  token.find("test_loss")>-1:
+                metric[current_epoch].append(token)
+            if  token.find("train_acc")>-1:
+                metric[current_epoch].append(token)
+            if  token.find("val_acc")>-1:
+                metric[current_epoch].append(token)
+            if  token.find("test_acc")>-1:
+                metric[current_epoch].append(token)
+results =[]
+for item in metric.items():
+    result=[]
+    result.append(item[0].replace('epoch=',""))
+    for fig in item[1]:
+        result.append(fig.split("=")[-1].replace(",",""))
+    results.append(result)
+
+for item in stastics.items():
+    print(item[0],item[1].split()[-1])
+
+#lets convert that to numpy array as np.array
+num = np.array(results)
+
+#now construct a beautiful table
+df = pd.DataFrame(num, columns=["EPOCH","Trn loss","Val Acc" ,"Tst loss","Trn Acc","Val loss","Tst Acc"]) #
+dash = 62
+print("-"*dash)
+print("| ".join(df.columns), "|")
+for index,row in df.iterrows():
+    print("-"*dash)
+    print("|",row["EPOCH"]," |", row["Trn loss"]," |", row["Val loss"]," |",row["Tst loss"], " |", row["Trn Acc"]," |",row["Val Acc"],"  |",row["Tst Acc"]," |")
+    
+print("-"*dash)
+
+
+# 

model.py

@@ -0,0 +1,36 @@
+import config
+import transformers
+import torch.nn as nn
+
+
+class BERTBaseUncased(nn.Module):
+    def __init__(self):
+        super(BERTBaseUncased, self).__init__()
+        self.bert = transformers.BertModel.from_pretrained(config.BERT_PATH)
+
+        self.bert_drop = nn.Dropout(0.3)
+
+        self.out = nn.Linear(768, 3)
+        # self.out = nn.Linear(256, 3)
+
+        nn.init.xavier_uniform_(self.out.weight)
+
+    def forward(self, ids, mask, token_type_ids):
+        _, o2 = self.bert(
+            ids, 
+            attention_mask=mask,
+            token_type_ids=token_type_ids
+        )
+        bo = self.bert_drop(o2)
+        # bo = self.tanh(self.fc(bo)) # to be commented if original
+        output = self.out(bo)
+        return output
+
+    def extract_features(self, ids, mask, token_type_ids):
+        _, o2 = self.bert(
+            ids, 
+            attention_mask=mask,
+            token_type_ids=token_type_ids
+        )
+        bo = self.bert_drop(o2)
+        return bo

predict.py

@@ -0,0 +1,145 @@
+import datetime
+import random
+
+import matplotlib
+import matplotlib.pyplot as plt
+import numpy as np
+import pandas as pd
+import seaborn as sns
+import torch
+import torch.nn as nn
+from absl import app, flags, logging
+from loguru import logger
+from scipy import stats
+from sklearn import metrics, model_selection
+from sklearn.decomposition import PCA
+from sklearn.manifold import TSNE
+from torch.utils.tensorboard import SummaryWriter
+
+import config
+import dataset
+import engine
+from model import BERTBaseUncased
+from utils import categorical_accuracy, label_decoder, label_encoder
+
+matplotlib.rcParams['interactive'] == True
+
+SEED = 42
+random.seed(SEED)
+np.random.seed(SEED)
+torch.manual_seed(SEED)
+torch.cuda.manual_seed(SEED)
+torch.backends.cudnn.deterministic = True
+
+writer = SummaryWriter()
+logger.add("experiment.log")
+
+flags.DEFINE_boolean('features', True, "")
+flags.DEFINE_string('test_file', None, "")
+flags.DEFINE_string('model_path', None, "")
+
+FLAGS = flags.FLAGS
+
+
+def main(_):
+    test_file = config.DATASET_LOCATION + "eval.prep.test.csv"
+    model_path = config.MODEL_PATH
+    if FLAGS.test_file:
+        test_file = FLAGS.test_file
+    if FLAGS.model_path:
+        model_path = FLAGS.model_path
+    df_test = pd.read_csv(test_file).fillna("none")
+
+    # Commenting as there are no labels
+    if FLAGS.features:
+        df_test.label = df_test.label.apply(label_encoder)
+
+    logger.info(f"Bert Model: {config.BERT_PATH}")
+    logger.info(
+        f"Current date and time :{datetime.datetime.now().strftime('%Y-%m-%d %H:%M:%S')} ")
+    logger.info(f"Test file: {test_file}")
+    logger.info(f"Test size : {len(df_test):.4f}")
+
+    test_dataset = dataset.BERTDataset(
+        review=df_test.text.values,
+        target=df_test.label.values
+    )
+
+    test_data_loader = torch.utils.data.DataLoader(
+        test_dataset,
+        batch_size=config.VALID_BATCH_SIZE,
+        num_workers=3
+    )
+
+    device = config.device
+    
+    model = BERTBaseUncased()
+    model.load_state_dict(torch.load(
+        model_path, map_location=torch.device(device)))
+    model.to(device)
+
+    outputs, extracted_features = engine.predict_fn(
+        test_data_loader, model, device, extract_features=FLAGS.features)
+    df_test["predicted"] = outputs
+    # save file
+    df_test.to_csv(model_path.split(
+        "/")[-2]+'.csv', header=None, index=False)
+
+    if FLAGS.features:
+        pca = PCA(n_components=50, random_state=7)
+        X1 = pca.fit_transform(extracted_features)
+        tsne = TSNE(n_components=2, perplexity=10, random_state=6,
+                    learning_rate=1000, n_iter=1500)
+        X1 = tsne.fit_transform(X1)
+        # if row == 0: print("Shape after t-SNE: ", X1.shape)
+
+        X = pd.DataFrame(np.concatenate([X1], axis=1),
+                         columns=["x1", "y1"])
+        X = X.astype({"x1": float, "y1": float})
+
+        # Plot for layer -1
+        plt.figure(figsize=(20, 15))
+        p1 = sns.scatterplot(x=X["x1"], y=X["y1"], palette="coolwarm")
+        # p1.set_title("development-"+str(row+1)+", layer -1")
+        x_texts = []
+        for output, value in zip(outputs, df_test.label.values):
+            if output == value:
+                x_texts.append("@"+label_decoder(output)
+                               [0] + label_decoder(output))
+            else:
+                x_texts.append(label_decoder(value) +
+                               "-" + label_decoder(output))
+
+        X["texts"] = x_texts
+        # X["texts"] = ["@G" + label_decoder(output) if output == value else "@R-" + label_decoder(value) + "-" + label_decoder(output)
+        #               for output, value in zip(outputs, df_test.label.values)]
+
+        # df_test.label.astype(str)
+        #([str(output)+"-" + str(value)] for output, value in zip(outputs, df_test.label.values))
+        # Label each datapoint with the word it corresponds to
+        for line in X.index:
+            text = X.loc[line, "texts"]+"-"+str(line)
+            if "@U" in text:
+                p1.text(X.loc[line, "x1"]+0.2, X.loc[line, "y1"], text[2:], horizontalalignment='left',
+                        size='medium', color='blue', weight='semibold')
+            elif "@P" in text:
+                p1.text(X.loc[line, "x1"]+0.2, X.loc[line, "y1"], text[2:], horizontalalignment='left',
+                        size='medium', color='green', weight='semibold')
+            elif "@N" in text:
+                p1.text(X.loc[line, "x1"]+0.2, X.loc[line, "y1"], text[2:], horizontalalignment='left',
+                        size='medium', color='red', weight='semibold')
+            else:
+                p1.text(X.loc[line, "x1"]+0.2, X.loc[line, "y1"], text, horizontalalignment='left',
+                        size='medium', color='black', weight='semibold')
+        plt.show()
+        plt.savefig(model_path.split(
+            "/")[-2]+'-figure.svg', format="svg")
+        # loocv = model_selection.LeaveOneOut()
+        # model = KNeighborsClassifier(n_neighbors=8)
+        # results = model_selection.cross_val_score(model, X, Y, cv=loocv)
+        # for i, j in outputs, extracted_features:
+        #     utils.write_embeddings_to_file(extracted_features, outputs)
+
+
+if __name__ == "__main__":
+    app.run(main)

utils.py

@@ -0,0 +1,25 @@
+import torch
+import config
+
+
+def categorical_accuracy(preds, y):
+    """
+    Returns accuracy per batch, i.e. if you get 8/10 right, this returns 0.8, NOT 8
+    """
+    max_preds = preds.argmax(
+        dim=1, keepdim=True)  # get the index of the max probability
+    correct = max_preds.squeeze(1).eq(y)
+    return correct.sum() / torch.FloatTensor([y.shape[0]])
+
+def label_encoder(x):
+    label_vec = {"0": 0, "1": 1, "-1": 2}
+    return label_vec[x.replace("__label__", "")]
+
+def label_decoder(x):
+    label_vec = { 0:"U",  1:"P",  2:"N"}
+    return label_vec[x]
+
+def label_full_decoder(x):
+    label_vec = { 0:"Neutral",  1:"Positive",  2:"Negative"}
+    return label_vec[x]
+