Update README.md

README.md

@@ -1,42 +1,28 @@
 ## Evaluation Pipeline
-# Use the raw version of the text below to evaluate the model. Make sure to set the datapath.
-
-# -*- coding: utf-8 -*-
-"""CIS 5190 Transformer Model
-
-Automatically generated by Colab.
-
-Original file is located at
-    https://colab.research.google.com/drive/1Iy-nQcufaF7--hI9He7Dp9FsW1TomgrP
-"""
+# Use the raw version of the text below to evaluate the model. Make sure to set the dataset and model path.
 
 import pandas as pd
 from sklearn.model_selection import train_test_split
-from sklearn.metrics import accuracy_score, classification_report
+from google.colab import drive
 import torch
 from torch.utils.data import Dataset, DataLoader
 from transformers import BertTokenizer, BertForSequenceClassification, AdamW
-from transformers import get_scheduler
-from google.colab import drive
-
-drive.mount('/content/drive')
+from sklearn.metrics import accuracy_score, classification_report
 
-# SET DATASET PATH HERE
-dataset_path = '/content/drive/My Drive/24 Fall/CIS 5190/CIS 5190 Final Project/test_data_random_subset.csv'
+dataset_path = ""
+model_path = ""
 
 news_df = pd.read_csv(dataset_path)
 
 X = news_df['title']
 y = news_df['labels']
 
-# y = y.apply(lambda x: 1 if x == 'FoxNews' else 0)
-
-# Split the data into training and testing sets (80% train, 20% test)
-X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2, random_state=42, shuffle=True, stratify=y)
+X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2, random_state=1)
+X_train, X_val, y_train, y_val = train_test_split(X_train, y_train, test_size=0.25, random_state=1) # 0.25 x 0.8 = 0.2
 
-# Tokenize the text using a BERT tokenizer
 tokenizer = BertTokenizer.from_pretrained('bert-base-uncased')
 
+
 def tokenize_data(texts, tokenizer, max_len=128):
     return tokenizer(
         list(texts),
@@ -50,7 +36,7 @@ def tokenize_data(texts, tokenizer, max_len=128):
 train_encodings = tokenize_data(X_train, tokenizer)
 test_encodings = tokenize_data(X_test, tokenizer)
 
-# Create a custom dataset class
+# Create a custom Dataset class
 class NewsDataset(Dataset):
     def __init__(self, encodings, labels):
         self.encodings = encodings
@@ -71,32 +57,16 @@ test_dataset = NewsDataset(test_encodings, y_test.tolist())
 train_loader = DataLoader(train_dataset, batch_size=16, shuffle=True)
 test_loader = DataLoader(test_dataset, batch_size=16)
 
-# Define the model
 model = BertForSequenceClassification.from_pretrained('bert-base-uncased', num_labels=2)
+model.load_state_dict(torch.load(model_path))
 
 device = torch.device('cuda') if torch.cuda.is_available() else torch.device('cpu')
 model.to(device)
 
 # Define optimizer and scheduler
-optimizer = AdamW(model.parameters(), lr=5e-5)
-num_training_steps = len(train_loader) * 4  # Assume 4 epochs
-lr_scheduler = get_scheduler("linear", optimizer=optimizer, num_warmup_steps=0, num_training_steps=num_training_steps)
-
-# Train the model
-def train_model(model, train_loader, optimizer, scheduler, epochs=4):
-    model.train()
-    for epoch in range(epochs):
-        epoch_loss = 0
-        for batch in train_loader:
-            batch = {k: v.to(device) for k, v in batch.items()}
-            outputs = model(**batch)
-            loss = outputs.loss
-            loss.backward()
-            optimizer.step()
-            scheduler.step()
-            optimizer.zero_grad()
-            epoch_loss += loss.item()
-        print(f"Epoch {epoch+1}/{epochs}, Loss: {epoch_loss/len(train_loader):.4f}")
+# optimizer = AdamW(model.parameters(), lr=5e-5)
+# num_training_steps = len(train_loader) * 4  # Assume 4 epochs
+# lr_scheduler = get_scheduler("linear", optimizer=optimizer, num_warmup_steps=0, num_training_steps=num_training_steps)
 
 # Evaluate the model
 def evaluate_model(model, test_loader):
@@ -112,10 +82,8 @@ def evaluate_model(model, test_loader):
             y_pred.extend(predictions.tolist())
     return y_true, y_pred
 
-train_model(model, train_loader, optimizer, lr_scheduler)
-
 y_true, y_pred = evaluate_model(model, test_loader)
 
-# 11. Print evaluation metrics
+# Print evaluation metrics
 print(f"Accuracy: {accuracy_score(y_true, y_pred):.4f}")
 print("Classification Report:\n", classification_report(y_true, y_pred))