Create function.py

function.py

@@ -0,0 +1,39 @@
+# ------------------ Sentiment Analysis Functions ------------------------#
+def split_text_by_token_limit(text, tokenizer, max_tokens):
+    tokens = tokenizer.encode(text, add_special_tokens=False)
+    chunks = []
+    for i in range(0, len(tokens), max_tokens):
+        chunk_tokens = tokens[i:i+max_tokens]
+        chunk_text = tokenizer.decode(chunk_tokens, skip_special_tokens=True)
+        chunks.append(chunk_text)
+    return chunks
+
+def safe_sentiment(text):
+    try:
+        result = sentiment_pipeline(text)[0]
+    except Exception as e:
+        result = None
+    return result
+
+def analyze_detail(text, tokenizer, sentiment_pipeline, max_tokens):
+    text = preprocess_text(text)
+    chunks = split_text_by_token_limit(text, tokenizer, max_tokens)
+    if not chunks:
+        return None
+    # Initialize accumulated scores for each sentiment category
+    scores = {"POSITIVE": 0, "NEGATIVE": 0, "NEUTRAL": 0}
+    for chunk in chunks:
+        result = safe_sentiment(chunk)
+        if result is not None:
+            label = result['label'].upper()
+            if label in scores:
+                scores[label] += result['score']
+    final_label = max(scores, key=lambda k: scores[k])
+    final_score = scores[final_label]
+    return {"label": final_label, "score": final_score}
+
+def preprocess_text(text):
+    # Replace URLs and user mentions
+    text = re.sub(r'http\S+', 'http', text)
+    text = re.sub(r'@\w+', '@user', text)
+    return text