Upload 5 files

scripts/evaluate_factual_robustness.py

@@ -1,30 +1,31 @@
 import json
 import tqdm
 import logging
-from scripts.get_factual_evaluation import get_factual_evaluation
+from scripts.get_prediction_file import get_prediction_file
 from scripts.groq_client import GroqClient
-from scripts.helper import adaptive_delay, ensure_directory_exists, load_used_data
+from scripts.helper import adaptive_delay, ensure_directory_exists, load_used_data, update_config
 from scripts.prompt import get_factual_prompt
 
 def evaluate_factual_robustness(config):
-    """Evaluates negative rejection for a given model by processing predictions and computing scores."""
-    config['noise_rate'] = 0.4 # Time being to do clarification
+    """Evaluates negative rejection for a given model under multiple correct_rate/noise_rate conditions."""
     model_name = config['model_name']
     
     if model_name in config['models']:
-        model = GroqClient(plm=config['model_name'])
+        model = GroqClient(plm=model_name)
     else:
         logging.warning(f"Skipping unknown model: {model_name}")
         return
 
-    # File paths
+    # Define the conditions to test
+    conditions = [
+        {"correct_rate": 1.0, "noise_rate": 0.2, "label": "factual_only"},  # factual documents with some noisy documents
+        {"correct_rate": 0.0, "noise_rate": 0.4, "label": "counterfactual"}  # Counterfactual + noise
+    ]
+
     base_path = "results/Counterfactual Robustness"
-    evalue_file = get_factual_evaluation(config)
-    print(f"Factual pred file {evalue_file}")
-    output_file = f"{base_path}/output_{config['output_file_extension']}.json"
     result_file = f"{base_path}/scores_{config['output_file_extension']}.json"
-    ensure_directory_exists(output_file)
-    
+    final_scores = {"conditions": []}
+
     def process_query(model, data, used_data, output_file):
         """Processes a single query, generates evaluation, and writes the result."""
         if data['id'] in used_data and data['query'] == used_data[data['id']]['query'] and data['ans'] == used_data[data['id']]['ans']:
@@ -33,8 +34,7 @@ def evaluate_factual_robustness(config):
 
         try:
             instruction = get_factual_prompt(data['query'], data['prediction'])
-            
-            # Retry mechanism for evaluation
+            #eval_model = GroqClient(plm='llama3-70b-8192')
             for attempt in range(1, 4):
                 evaluation = model.generate(instruction)
                 if evaluation:
@@ -42,7 +42,7 @@ def evaluate_factual_robustness(config):
                 adaptive_delay(attempt)
             
             data['evaluation'] = evaluation
-            print(f"Model Response for Factual robustness: {evaluation}")
+            logging.info(f"Model Response for Factual robustness: {evaluation}")
             output_file.write(json.dumps(data, ensure_ascii=False) + '\n')
             return data
 
@@ -50,7 +50,7 @@ def evaluate_factual_robustness(config):
             print(f"Error processing query: {e}")
             return None
 
-    def calculate_scores(results, config):
+    def calculate_scores(results, condition):
         """Calculates and returns rejection rates and other metrics."""
         rejecttt = 0
         tt = 0
@@ -64,35 +64,54 @@ def evaluate_factual_robustness(config):
                 tt += 1
 
         scores = {
-            'reject_rate': rejecttt/len(results),
-            'all_rate': (tt)/len(results),
-            'correct_rate': correct_tt/rejecttt if rejecttt > 0 else 0,
-            'tt':tt,
-            'rejecttt':rejecttt,
-            'correct_tt':correct_tt,
+            'reject_rate': rejecttt / len(results) if len(results) > 0 else 0, #Error Detection Rate (ED)
+            'all_rate': tt / len(results) if len(results) > 0 else 0,
+            'correct_rate': correct_tt / rejecttt if rejecttt > 0 else 0, #Error Correction Rate (CR)
+            'tt': tt,
+            'rejecttt': rejecttt,
+            'correct_tt': correct_tt,
             'nums': len(results),
-            'noise_rate': config['noise_rate'],
+            'noise_rate': condition['noise_rate'],
+            'condition_label': condition['label']
         }
         return scores
-    
-    used_data = []
-    results = []
-    if config['UsePreCalculatedValue']: 
-        logging.info(f"Trying to use pre calculated values for Counterfactual report generation")
-        used_data = load_used_data(output_file)
-    else:
-        logging.info(f"Recalculating the metrics...")
+ 
+    for condition in conditions:
+        logging.info(f"\nEvaluating condition: {condition['label']} (correct_rate={condition['correct_rate']}, noise_rate={condition['noise_rate']})")
+        
+        # Update config with current condition's noise_rate
+        config['noise_rate'] = condition['noise_rate']
+        #config['passage_num'] = 10
+        update_config(config)
+       
+        # File paths with condition-specific suffixes
+        pred_file = get_prediction_file(config, condition['correct_rate'])
+        output_file = f"{base_path}/output_{config['output_file_extension']}.json"
+       
+        ensure_directory_exists(output_file)
+        
+        logging.info(f"Factual pred file for {condition['label']}: {pred_file}")
+
+        # Load or recalculate data
+        used_data = []
+        results = []
+        if config['UsePreCalculatedValue']:
+            logging.info(f"Trying to use pre-calculated values for {condition['label']}")
+            used_data = load_used_data(output_file)
+        else:
+            logging.info(f"Recalculating the metrics for {condition['label']}...")
 
-    with open(output_file, 'w', encoding='utf-8') as f_out, open(evalue_file, 'r', encoding='utf-8') as f_eval:
-        for line in tqdm.tqdm(f_eval):
-            data = json.loads(line)
-            processed_data = process_query(model, data, used_data, f_out)
-            if processed_data:
-                results.append(processed_data)
+        with open(output_file, 'w', encoding='utf-8') as f_out, open(pred_file, 'r', encoding='utf-8') as f_eval:
+            for line in tqdm.tqdm(f_eval):
+                data = json.loads(line)
+                processed_data = process_query(model, data, used_data, f_out)
+                if processed_data:
+                    results.append(processed_data)
 
-    # Compute scores and save
-    scores = calculate_scores(results, config)
-    logging.info(f"Counterfactual Robustness Score: {scores}")
+        # Compute and save scores
+        scores = calculate_scores(results, condition)
+        final_scores["conditions"].append(scores)
+        logging.info(f"Counterfactual Robustness Score for {condition['label']}: {scores}")
 
     with open(result_file, 'w', encoding='utf-8') as f_result:
-        json.dump(scores, f_result, ensure_ascii=False, indent=4)
+        json.dump(final_scores, f_result, ensure_ascii=False, indent=4)

scripts/get_prediction_file.py

@@ -0,0 +1,58 @@
+import os
+import json
+import logging
+from scripts.get_prediction_result import get_prediction_result
+from scripts.helper import ensure_directory_exists, load_dataset
+
+
+# Set up logging configuration
+logging.basicConfig(level=logging.INFO, format='%(asctime)s - %(levelname)s - %(message)s')
+
+# Improved function to evaluate noise robustness
+def get_prediction_file(config, correct_rate = 0):
+    result_path = config['result_path'] + 'Counterfactual Robustness/'
+    noise_rate = config['noise_rate']
+
+    # Iterate over each model specified in the config
+    filename = os.path.join(result_path, f"prediction_{config['output_file_extension']}.json")
+    ensure_directory_exists(filename)
+
+    results = get_prediction_result(config, config['factual_file_name'], filename, correct_rate)  # Store results for this model
+
+    # Save results to a file
+    with open(filename, 'w', encoding='utf-8') as f:
+        for result in results:
+            f.write(json.dumps(result, ensure_ascii=False) + '\n')
+    
+    return filename
+    # Compute per-model noise robustness
+    '''tt = sum(1 for i in results if (noise_rate == 1 and i['label'][0] == -1) or (0 not in i['label'] and 1 in i['label']))
+    scores = {
+    'all_rate': (tt)/len(results),
+    'noise_rate': noise_rate,
+    'tt':tt,
+    'nums': len(results),
+    }
+    fact_tt = 0
+    correct_tt = 0
+    for i in results:
+        if i['factlabel'] == 1:
+            fact_tt += 1
+            if 0 not in i['label']:
+                correct_tt += 1
+    fact_check_rate = fact_tt/len(results)
+    if fact_tt > 0:
+        correct_rate = correct_tt/fact_tt
+    else:
+        correct_rate = 0
+    scores['fact_check_rate'] = fact_check_rate
+    scores['correct_rate'] = correct_rate
+    scores['fact_tt'] = fact_tt
+    scores['correct_tt'] = correct_tt
+
+    #logging.info(f"score: {scores}")
+    score_filename = os.path.join(result_path, f"scores_{config['output_file_extension']}.json")
+    with open(score_filename, 'w') as f:
+        json.dump(scores, f, ensure_ascii=False, indent=4)'''
+
+  

scripts/get_prediction_result.py

@@ -9,7 +9,7 @@ logging.basicConfig(level=logging.INFO, format='%(asctime)s - %(levelname)s - %(
 
 # Get prediction from LLM based on different dataset
 
-def get_prediction_result(config, data_file_name, prediction_file_name=''):
+def get_prediction_result(config, data_file_name, prediction_file_name='', correct_rate = 0):
     results = []
     used_data = []
     dataset = load_dataset(data_file_name)
@@ -37,7 +37,7 @@ def get_prediction_result(config, data_file_name, prediction_file_name=''):
                 continue
         
         logging.info(f"Executing Query {idx + 1} for Model: {modelname}")
-        query, ans, docs = process_data(instance, config['noise_rate'], config['passage_num'], data_file_name)
+        query, ans, docs = process_data(instance, config['noise_rate'], config['passage_num'], data_file_name, correct_rate)
 
         # Retry mechanism for prediction
         for attempt in range(1, config['retry_attempts'] + 1):

scripts/get_scores.py

@@ -65,7 +65,7 @@ def load_negative_rejection_scores(config):
         return pd.DataFrame()
 
     if not os.path.exists(Negative_Rejection_DIR):
-        return pd.DataFrame(columns=["Model", "Rejection Rate"])
+        return pd.DataFrame(columns=["Model", "Rejection Rate %"])
 
     score_data = {}
 
@@ -80,44 +80,47 @@ def load_negative_rejection_scores(config):
             with open(filepath, "r") as f:
                 score = json.load(f)
                 reject_rate = score.get("reject_rate", "N/A")
-                score_data[model] = f"{reject_rate * 100:.2f}%" if reject_rate != "N/A" else "N/A"
+                score_data[model] = f"{reject_rate * 100}" if reject_rate != "N/A" else "N/A"
         else:
             score_data[model] = "N/A"
 
     # Convert to DataFrame
     df = pd.DataFrame([
-        {"Model": model, "Rejection Rate": score_data[model]}
+        {"Model": model, "Rejection Rate %": score_data[model]}
         for model in config["models"]
     ])
 
     return df
 
 def load_counterfactual_robustness_scores(config):
-    #hard code noise rate to 0.4
-    config['noise_rate'] = 0.4
+    """Load and format counterfactual robustness scores into a table with proper formatting."""
+    config['noise_rate'] = 0.4  # Hardcode noise rate
+
     if not os.path.exists(Counterfactual_Robustness_DIR):
-        return pd.DataFrame(columns=["Model", "Accuracy with factual docs (%)", "Error Detection Rate", "Correction Rate (%)"])
+        return pd.DataFrame(columns=["Model", "Accuracy (%)", "Acc_doc (%)", "Error Detection Rate (%)", "Correction Rate (%)"])
 
     score_data = {}
 
-    # Iterate over each model in config['models']
     for model in config["models"]:  
-        # Expected filename pattern for each model
         expected_filename = f"scores_{model}_noise_{config['noise_rate']}_passage_{config['passage_num']}_num_queries_{config['num_queries']}.json"
         filepath = os.path.join(Counterfactual_Robustness_DIR, expected_filename)
 
-        # Check if file exists
         if os.path.exists(filepath):
             with open(filepath, "r") as f:
-                score = json.load(f)
+                scores_json = json.load(f)  # Read the full JSON content
+                factual_score = next((s for s in scores_json["conditions"] if s["condition_label"] == "factual_only"), {})
+                counterfactual_score = next((s for s in scores_json["conditions"] if s["condition_label"] == "counterfactual"), {})
+
                 score_data[model] = {
-                    "Accuracy with factual docs (%)": round(score.get("all_rate", 0) * 100, 2),  # No decimal
-                    "Error Detection Rate (%)": round(score.get("reject_rate", 0) * 100, 2), 
-                    "Correction Rate (%)": round(score.get("correct_rate", 0) * 100, 2)  # 2 decimal places
+                    "Accuracy (%)": int(round(factual_score.get("all_rate", 0) * 100)) if factual_score else "N/A",
+                    "Acc_doc (%)": int(round(counterfactual_score.get("all_rate", 0) * 100)) if counterfactual_score else "N/A",
+                    "Error Detection Rate (%)": int(round(counterfactual_score.get("reject_rate", 0) * 100)) if counterfactual_score else "N/A",
+                    "Correction Rate (%)": round(counterfactual_score.get("correct_rate", 0) * 100, 2) if counterfactual_score else "N/A"
                 }
         else:
-            score_data[model] = {  # Populate with "N/A" if file not found
-                "Accuracy with factual docs (%)": "N/A",
+            score_data[model] = {
+                "Accuracy (%)": "N/A",
+                "Acc_doc (%)": "N/A",
                 "Error Detection Rate (%)": "N/A",
                 "Correction Rate (%)": "N/A"
             }
@@ -126,8 +129,9 @@ def load_counterfactual_robustness_scores(config):
     df = pd.DataFrame([
         {
             "Model": model,
-            "Accuracy with factual docs (%)": f"{score_data[model]['Accuracy with factual docs (%)']:.2f}" if score_data[model]["Accuracy with factual docs (%)"] != "N/A" else "N/A",
-            "Error Detection Rate": f"{score_data[model]['Error Detection Rate (%)']:.2f}" if score_data[model]["Error Detection Rate (%)"] != "N/A" else "N/A",
+            "Accuracy (%)": f"{score_data[model]['Accuracy (%)']}" if score_data[model]["Accuracy (%)"] != "N/A" else "N/A",
+            "Acc_doc (%)": f"{score_data[model]['Acc_doc (%)']}" if score_data[model]["Acc_doc (%)"] != "N/A" else "N/A",
+            "Error Detection Rate (%)": f"{score_data[model]['Error Detection Rate (%)']}" if score_data[model]["Error Detection Rate (%)"] != "N/A" else "N/A",
             "Correction Rate (%)": f"{score_data[model]['Correction Rate (%)']:.2f}" if score_data[model]["Correction Rate (%)"] != "N/A" else "N/A"
         }
         for model in config["models"]

scripts/process_data.py

@@ -36,15 +36,39 @@ def process_data(instance, noise_rate, passage_num, filename, correct_rate=0):
     # Handling the '_fact' case in filename
     elif '_fact' in filename:
         correct_num = math.ceil(passage_num * correct_rate)
-        pos_num = passage_num - neg_num - correct_num
-        indexs = list(range(len(instance['positive'])))
-        selected = random.sample(indexs, min(len(indexs), pos_num))
-        docs = [instance['positive_wrong'][i] for i in selected]
-        remain = [i for i in indexs if i not in selected]
-        if correct_num > 0 and len(remain) > 0:
-            docs += [instance['positive'][i] for i in random.sample(remain, min(len(remain), correct_num))]
-        if neg_num > 0:
-            docs += instance['negative'][:neg_num]
+        # Adjust correct_num to not exceed passage_num - neg_num, excluding positive_wrong
+        if correct_rate == 1.0:
+            # For factual-only with noise, use only positive and negative documents
+            correct_num = min(correct_num, passage_num - neg_num)
+            pos_num = 0  # No positive_wrong documents when correct_rate = 1.0
+        else:
+            # For other correct_rate values, calculate pos_num for positive_wrong
+            pos_num = passage_num - neg_num - correct_num
+            if pos_num < 0:
+                pos_num = 0  # Ensure pos_num is not negative
+
+        # Select positive documents (factual) first
+        indexs_positive = list(range(len(instance['positive'])))
+        selected_positive = random.sample(indexs_positive, min(len(indexs_positive), correct_num))
+        docs = [instance['positive'][i] for i in selected_positive]
+
+        # Add negative documents (noise) if needed
+        if neg_num > 0 and 'negative' in instance:
+            docs += instance['negative'][:min(neg_num, len(instance['negative']))]
+
+        # Only add positive_wrong documents if pos_num > 0 and correct_rate < 1.0
+        if pos_num > 0 and correct_rate < 1.0:
+            indexs_positive_wrong = list(range(len(instance['positive_wrong'])))
+            selected_positive_wrong = random.sample(indexs_positive_wrong, min(len(indexs_positive_wrong), pos_num))
+            docs += [instance['positive_wrong'][i] for i in selected_positive_wrong]
+
+        # Ensure docs length does not exceed passage_num
+        if len(docs) > passage_num:
+            random.shuffle(docs)
+            docs = docs[:passage_num]
+        elif len(docs) < passage_num and 'negative' in instance:
+            remaining = passage_num - len(docs)
+            docs += instance['negative'][:min(remaining, len(instance['negative']))]
     
     # Default case (when filename doesn't match '_int' or '_fact')
     else: