Add a new evaluation function

eval_utils.py

@@ -77,3 +77,34 @@ def prompt_chat_LLM(model, tokenizer, text):
         if "نعم" in gen_text:
             predicted_dialects.append(dialect)
     return predicted_dialects
+
+
+def predict_binary_outcomes(model, tokenizer, texts, threshold=0.3):
+    """Predict the validity in each dialect, by indepenently applying a sigmoid activation to each dialect's logit.
+    Dialects with probabilities (sigmoid activations) above a threshold (set by defauly to 0.3) are considered predicted.
+    """
+    device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
+
+    encodings = tokenizer(
+        texts, truncation=True, padding=True, max_length=128, return_tensors="pt"
+    )
+
+    ## inputs
+    input_ids = encodings["input_ids"].to(device)
+    attention_mask = encodings["attention_mask"].to(device)
+
+    with torch.no_grad():
+        outputs = model(input_ids=input_ids, attention_mask=attention_mask)
+        logits = outputs.logits
+
+    probabilities = torch.sigmoid(logits).cpu().numpy().reshape(-1)
+    predictions = (probabilities >= threshold).astype(int)
+
+    # Map indices to actual labels
+    predicted_labels = [
+        dialect
+        for dialect, dialect_probability in zip(DIALECTS, predictions)
+        if dialect_probability == 1
+    ]
+
+    return predicted_labels