calculate.py

calculate.py

@@ -0,0 +1,57 @@
+# Read the CSV data into a DataFrame
+data = pd.read_csv('models.csv')
+
+# Define the score columns
+score_columns = ['Average', 'AGIEval', 'GPT4All', 'TruthfulQA', 'Bigbench']
+
+# Function to calculate the highest combined score for a given column
+def calculate_highest_combined_score(column):
+    start_time = time.time()
+    scores = data[column].tolist()
+    models = data['Model'].tolist()
+    top_combinations = {2: [], 3: [], 4: [], 5: [], 6: []}
+    calculations = {2: 0, 3: 0, 4: 0, 5: 0, 6: 0}
+
+    # Generate all unique combinations of two, three, four, and five models
+    for r in range(2, 7):  # r is the combination size (2, 3, 4, 5 or 6)
+        for combination in combinations(zip(scores, models), r):
+            combined_score = sum(score for score, _ in combination)
+            # Add the combination to the list along with its score
+            top_combinations[r].append((combined_score, tuple(model for _, model in combination)))
+            calculations[r] += 1
+        # Sort the list in descending order by score and keep only the top three
+        top_combinations[r] = sorted(top_combinations[r], key=lambda x: x[0], reverse=True)[:3]
+
+    elapsed_time = time.time() - start_time
+    return column, top_combinations, calculations, elapsed_time
+
+# Function to be executed in parallel
+def worker(column):
+    return calculate_highest_combined_score(column)
+
+if __name__ == '__main__':
+    with Pool() as pool:
+        results = pool.map(worker, score_columns)
+    # Sort results by max_score in descending order
+    sorted_results = sorted(results, key=lambda x: max(x[1][5])[0] if 5 in x[1] else 0, reverse=True)
+    # Print the sorted results
+    for column, top_combinations, calculations, elapsed_time in sorted_results:
+        for r in range(2, 7):
+            print(f"Column: {column}, Number of Models: {r}")
+            for score, combination in top_combinations[r]:
+                print(f"Combination: {combination}, Score: {score}")
+            print(f"Calculations required: {calculations[r]}")
+            print(f"Time taken: {elapsed_time:.4f} seconds")
+            print()  # Add an empty line for better readability
+
+    # Count how many times each model is mentioned
+    model_mentions = Counter()
+    for _, top_combinations, _, _ in sorted_results:
+        for r in range(2, 7):
+            for _, combination in top_combinations[r]:
+                model_mentions.update(combination)
+
+    # Print the top 5 most mentioned models
+    print("Top 5 most mentioned models:")
+    for model, count in model_mentions.most_common(5):
+        print(f"{model}: {count} times")