milestone-3

.gitignore

@@ -127,3 +127,4 @@ dmypy.json
 
 # Pyre type checker
 .pyre/
+./data

app.py

@@ -3,6 +3,7 @@ from transformers import AutoTokenizer, RobertaForSequenceClassification
 import numpy as np
 import torch
 
+# assignment 2
 st.title("CS482 Project Sentiment Analysis")
 
 text = st.text_area(label="Text to be analyzed", value="This sentiment analysis app is great!")
@@ -13,16 +14,17 @@ analyze_button = st.button(label="Analyze")
 
 st.markdown("**:red[Sentiment:]**")
 
-if analyze_button:
- if selected_model=="Model 1":
- tokenizer = AutoTokenizer.from_pretrained("cardiffnlp/twitter-roberta-base-emotion")
- model = RobertaForSequenceClassification.from_pretrained("cardiffnlp/twitter-roberta-base-emotion")
- else:
- tokenizer = AutoTokenizer.from_pretrained("cardiffnlp/twitter-roberta-base-sentiment-latest")
- model = RobertaForSequenceClassification.from_pretrained("cardiffnlp/twitter-roberta-base-sentiment-latest")
- inputs = tokenizer(text, return_tensors="pt")
- with torch.no_grad():
- logits = model(**inputs).logits
- prediction_id = logits.argmax().item()
- results = model.config.id2label[prediction_id]
- st.write(results)
+with st.spinner(text="Analyzing..."):
+ if analyze_button:
+ if selected_model=="Model 1":
+ tokenizer = AutoTokenizer.from_pretrained("cardiffnlp/twitter-roberta-base-emotion")
+ model = RobertaForSequenceClassification.from_pretrained("cardiffnlp/twitter-roberta-base-emotion")
+ else:
+ tokenizer = AutoTokenizer.from_pretrained("cardiffnlp/twitter-roberta-base-sentiment-latest")
+ model = RobertaForSequenceClassification.from_pretrained("cardiffnlp/twitter-roberta-base-sentiment-latest")
+ inputs = tokenizer(text, return_tensors="pt")
+ with torch.no_grad():
+ logits = model(**inputs).logits
+ prediction_id = logits.argmax().item()
+ results = model.config.id2label[prediction_id]
+ st.write(results)

milestone_3.py

@@ -0,0 +1,103 @@
+from transformers import DistilBertTokenizerFast, DistilBertModel, AdamW
+import torch
+from torch.utils.data import Dataset, DataLoader
+import pandas as pd
+
+
+# assignment 3
+model_name = "distilbert-base-uncased"
+tokenizer = DistilBertTokenizerFast.from_pretrained(model_name)
+
+print("Reading data...")
+data = pd.read_csv("./data/train.csv")
+toxic_data = pd.DataFrame()
+toxic_data["text"] = data["comment_text"]
+toxic_data["labels"] = data.iloc[:, 2:].values.tolist()
+print(toxic_data.head())
+
+class ToxicDataset(Dataset):
+
+ def __init__(self, dataframe, tokenizer):
+ self.tokenizer = tokenizer
+ self.data = dataframe
+ self.text = dataframe.text
+ self.labels = self.data.labels
+
+ def __len__(self):
+ return len(self.text)
+ 
+ def __getitem__(self, idx):
+ text = str(self.text[idx])
+ if len(text) > 12:
+ text = text[:12]
+
+ inputs = self.tokenizer.encode_plus(
+ text,
+ None,
+ max_length=12,
+ add_special_tokens=True,
+ pad_to_max_length=True,
+ return_token_type_ids=True
+ )
+
+ ids = inputs["input_ids"]
+ mask = inputs["attention_mask"]
+ token_type_ids = inputs["token_type_ids"]
+
+ return {
+ "ids": torch.tensor(ids, dtype=torch.long),
+ "mask": torch.tensor(mask, dtype=torch.long),
+ "token_type_ids": torch.tensor(token_type_ids, dtype=torch.long),
+ "targets": torch.tensor(self.labels[idx], dtype=torch.float)
+ }
+
+
+print("Data read. Splitting data...")
+train_data = toxic_data.sample(frac=.8)
+test_data = toxic_data.drop(train_data.index).reset_index(drop=True)
+train_data = train_data.reset_index(drop=True)
+
+print("Data split. Tokenizing data...")
+train_set = ToxicDataset(train_data, tokenizer)
+test_set = ToxicDataset(test_data, tokenizer)
+
+train_loader = DataLoader(train_set, batch_size=8, shuffle=True, num_workers=0)
+test_loader = DataLoader(test_set, batch_size=8, shuffle=True, num_workers=0)
+
+print("Data tokenized. Beginning training...")
+
+device = torch.device("cuda") if torch.cuda.is_available() else torch.device("cpu")
+
+model = DistilBertModel.from_pretrained(model_name)
+model.to(device)
+model.train()
+
+optim = AdamW(model.parameters(), lr=5e-5)
+
+num_train_epochs = 2
+
+for epoch in range(num_train_epochs):
+ for batch in train_loader:
+ optim.zero_grad()
+ input_ids = batch["ids"].to(device)
+ attention_mask = batch["mask"].to(device)
+ token_type_ids = batch["token_type_ids"].to(device, dtype = torch.long)
+ targets = batch["targets"].to(device)
+
+ outputs = model(input_ids, attention_mask, token_type_ids)
+ 
+ loss = torch.nn.BCEWithLogitsLoss()(outputs, targets)
+ loss.backward()
+ optim.step()
+
+model.eval()
+
+
+
+
+print("Training complete. Saving model...")
+
+save_directory = ".results/model"
+model.save_pretrained(save_directory)
+
+print("Model saved.")