Upload comparison.py

lib/comparison.py

@@ -0,0 +1,94 @@
+from transformers import BertTokenizer, BertForSequenceClassification, pipeline, BertModel,T5ForConditionalGeneration, T5Tokenizer
+import torch
+from sklearn.feature_extraction.text import TfidfVectorizer
+from sklearn.metrics.pairwise import cosine_similarity
+import numpy as np
+# Load the pre-trained FinBERT model for sentiment analysis
+device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
+finbert_model_name = "yiyanghkust/finbert-tone"
+finbert_tokenizer = BertTokenizer.from_pretrained(finbert_model_name)
+finbert_model = BertForSequenceClassification.from_pretrained(finbert_model_name)
+finbert_model.to(device)
+
+finbert_pipeline = pipeline("sentiment-analysis", model=finbert_model, tokenizer=finbert_tokenizer, device=0 if device.type == "cuda" else -1)
+
+# Load the pre-trained T5 model for summarization
+t5_model_name = "t5-small"  # You can also use "t5-base" or "t5-large" for better summaries
+t5_model = T5ForConditionalGeneration.from_pretrained(t5_model_name).to(device)
+t5_tokenizer = T5Tokenizer.from_pretrained(t5_model_name)
+
+def analyze_and_summarize_paragraphs(paragraphs):
+    """Perform sentiment analysis and summarization on each paragraph."""
+    results = []
+
+    for paragraph in paragraphs:
+        # Perform sentiment analysis using FinBERT
+        sentiment_result = finbert_pipeline(paragraph)
+        
+        # Perform summarization using T5
+        t5_input = f"summarize: {paragraph}"
+        input_ids = t5_tokenizer.encode(t5_input, return_tensors="pt").to(device)
+        summary_ids = t5_model.generate(input_ids, max_length=80, num_beams=6, early_stopping=True)
+        summary = t5_tokenizer.decode(summary_ids[0], skip_special_tokens=True)
+        
+        # Store only positive or negative sentiment results
+        if sentiment_result and sentiment_result[0]['label'] in ['Positive', 'Negative', 'Neutral']:
+            results.append({
+                "paragraph_text": paragraph,
+                "summary": summary,
+                "sentiment": sentiment_result[0]
+            })
+
+    return results
+
+bert_model_name = "bert-base-uncased"
+tokenizer = BertTokenizer.from_pretrained(bert_model_name)
+model = BertModel.from_pretrained(bert_model_name)
+model.eval()  # Set to evaluation mode
+
+# Word embedding on summary text using BERT
+def get_bert_embeddings(texts):
+    """Obtain BERT embeddings for a list of texts."""
+    embeddings = []
+    with torch.no_grad():
+        for text in texts:
+            inputs = tokenizer(text, return_tensors='pt', truncation=True, padding=True)
+            outputs = model(**inputs)
+            # Take the mean of token embeddings as the sentence embedding
+            embedding = outputs.last_hidden_state.mean(dim=1).squeeze().numpy()
+            embeddings.append(embedding)
+    return np.array(embeddings)
+
+# Compute similirity matrices over embeddings
+def compute_similarity(embeddings1, embeddings2):
+    """Compute pairwise cosine similarity between two sets of embeddings."""
+    return cosine_similarity(embeddings1, embeddings2)
+
+# For each paragraphs summed up, get the most close summary from other year and compare contents
+def compare_summaries(results1, results2):
+    """Compare summaries from two documents and return similarity scores."""
+    # Get embeddings for each set of summaries
+    summaries1 = [result['summary'] for result in results1]
+    summaries2 = [result['summary'] for result in results2]
+    sentiment1 = [result['sentiment'] for result in results1]
+    sentiment2 = [result['sentiment'] for result in results2]
+    embeddings1 = get_bert_embeddings(summaries1)
+    embeddings2 = get_bert_embeddings(summaries2)
+
+    # Compute similarity
+    similarity_matrix = compute_similarity(embeddings1, embeddings2)
+
+    # Analyze matches
+    matches = []
+    for i, row in enumerate(similarity_matrix):
+        most_similar_index = np.argmax(row)
+        similarity_score = row[most_similar_index]
+        matches.append({
+            'summary_doc1': summaries1[i],
+            'summary_doc2': summaries2[most_similar_index],
+            'sentiment_doc1': sentiment1[i],
+            'sentiment_doc2': sentiment2[most_similar_index],
+            'similarity_score': similarity_score
+        })
+
+    return matches