[
{"config": {
"model_name": "ChatGPT-4o-latest (2024-09-03)",
"organization": "OpenAI",
"license": "Proprietary",
"knowledge_cutoff": "2023/10"
},
"results": {
"math-algebra": {"Score": 99.19484702, "Avg Rank": 1.666666667, "Min Rank": 1, "Max Rank": 3},
"math-probability": {"Score": 100, "Avg Rank": 1, "Min Rank": 1, "Max Rank": 1},
"reasoning-logical": {"Avg Rank": 1, "Min Rank": 1, "Max Rank": 1},
"overall": {"Avg Rank": 2, "Min Rank": 2, "Max Rank": 2}
}},
{"config": {
"model_name": "gpt-4o-2024-08-06",
"organization": "OpenAI",
"license": "Proprietary",
"knowledge_cutoff": "2023/10"
},
"results": {
"math-algebra": {"Score": 98.38969404, "Avg Rank": 1.666666667, "Min Rank": 1, "Max Rank": 2},
"math-probability": {"Score": 96.49758454, "Avg Rank": 2.666666667, "Min Rank": 2, "Max Rank": 4},
"reasoning-logical": {"Avg Rank": 4.333333333, "Min Rank": 3, "Max Rank": 5},
"overall": {"Avg Rank": 7.33, "Min Rank": 4, "Max Rank": 9}
}},
{"config": {
"model_name": "gpt-4o-2024-05-13",
"organization": "OpenAI",
"license": "Proprietary",
"knowledge_cutoff": "2023/10"
},
"results": {
"math-algebra": {"Score": 98.15480333, "Avg Rank": 2.666666667, "Min Rank": 2, "Max Rank": 3},
"math-probability": {"Score": 94.83939431, "Avg Rank": 3.666666667, "Min Rank": 2, "Max Rank": 5},
"reasoning-logical": {"Avg Rank": 6.333333333, "Min Rank": 3, "Max Rank": 8},
"overall": {"Avg Rank": 7.67, "Min Rank": 7, "Max Rank": 9}
}},
{"config": {
"model_name": "gpt-4-turbo-2024-04-09",
"organization": "OpenAI",
"license": "Proprietary",
"knowledge_cutoff": "2023/12"
},
"results": {
"math-algebra": {"Score": 96.03195879, "Avg Rank": 4, "Min Rank": 4, "Max Rank": 4},
"math-probability": {"Score": 93.59903382, "Avg Rank": 6.666666667, "Min Rank": 6, "Max Rank": 8},
"reasoning-logical": {"Avg Rank": 4, "Min Rank": 2, "Max Rank": 7},
"overall": {"Avg Rank": 6, "Min Rank": 5, "Max Rank": 8}
}},
{"config": {
"model_name": "gemini-1.5-pro-001",
"organization": "Google",
"license": "Proprietary",
"knowledge_cutoff": "2024-01"
},
"results": {
"math-algebra": {"Score": 94.7572213, "Avg Rank": 5, "Min Rank": 5, "Max Rank": 5},
"math-probability": {"Score": 91.42512077, "Avg Rank": 9.666666667, "Min Rank": 9, "Max Rank": 10},
"reasoning-logical": {"Avg Rank": 9.666666667, "Min Rank": 9, "Max Rank": 11},
"overall": {"Avg Rank": 14, "Min Rank": 13, "Max Rank": 15}
}},
{"config": {
"model_name": "qwen2-72b-instruct",
"organization": "Alibaba",
"license": "Qianwen LICENSE",
"knowledge_cutoff": "2024-02"
},
"results": {
"math-algebra": {"Score": 93.88818605, "Avg Rank": 6, "Min Rank": 6, "Max Rank": 6},
"math-probability": {"Score": 91.54326174, "Avg Rank": 4, "Min Rank": 3, "Max Rank": 5},
"reasoning-logical": {"Avg Rank": 15.66666667, "Min Rank": 15, "Max Rank": 17},
"overall": {"Avg Rank": 17, "Min Rank": 17, "Max Rank": 17}
}},
{"config": {
"model_name": "gpt-4o-mini-2024-07-18",
"organization": "OpenAI",
"license": "Proprietary",
"knowledge_cutoff": "2024-07"
},
"results": {
"math-algebra": {"Score": 93.22073596, "Avg Rank": 7, "Min Rank": 7, "Max Rank": 7},
"math-probability": {"Score": 92.17351456, "Avg Rank": 3.666666667, "Min Rank": 3, "Max Rank": 5},
"reasoning-logical": {"Avg Rank": 9.666666667, "Min Rank": 9, "Max Rank": 10},
"overall": {"Avg Rank": 7, "Min Rank": 5, "Max Rank": 8}
}},
{"config": {
"model_name": "claude-3.5-sonnet",
"organization": "Anthropic",
"license": "Proprietary",
"knowledge_cutoff": "2024-03"
},
"results": {
"math-algebra": {"Score": 91.5823805, "Avg Rank": 8.333333333, "Min Rank": 8, "Max Rank": 9},
"math-probability": {"Score": 91.55011915, "Avg Rank": 8, "Min Rank": 7, "Max Rank": 9},
"reasoning-logical": {"Avg Rank": 5, "Min Rank": 2, "Max Rank": 7},
"overall": {"Avg Rank": 5, "Min Rank": 4, "Max Rank": 7}
}},
{"config": {
"model_name": "o1-mini",
"organization": "01 AI",
"license": "Proprietary",
"knowledge_cutoff": "2024-01"
},
"results": {
"math-algebra": None,
"math-probability": None,
"reasoning-logical": None,
"overall": {"Avg Rank": 1, "Min Rank": 1, "Max Rank": 1}
}},
{"config": {
"model_name": "o1-preview",
"organization": "01 AI",
"license": "Proprietary",
"knowledge_cutoff": "2024-01"
},
"results": {
"math-algebra": None,
"math-probability": None,
"reasoning-logical": None,
"overall": {"Avg Rank": 3, "Min Rank": 3, "Max Rank": 3}
}},
{"config": {
"model_name": "gemini-1.5-flash-001",
"organization": "Google",
"license": "Proprietary",
"knowledge_cutoff": "2024-02"
},
"results": {
"math-algebra": {"Score": 91.30211121, "Avg Rank": 11, "Min Rank": 11, "Max Rank": 11},
"math-probability": {"Score": 91.066099, "Avg Rank": 12, "Min Rank": 10, "Max Rank": 13},
"reasoning-logical": {"Avg Rank": 15.66666667, "Min Rank": 15, "Max Rank": 16},
"overall": {"Avg Rank": 14, "Min Rank": 13, "Max Rank": 15}
}},
{"config": {
"model_name": "gpt4-1106",
"organization": "OpenAI",
"license": "Proprietary",
"knowledge_cutoff": "2024-04"
},
"results": {
"math-algebra": {"Score": 91.2227739, "Avg Rank": 12, "Min Rank": 12, "Max Rank": 12},
"math-probability": {"Score": 91.09550085, "Avg Rank": 11.66666667, "Min Rank": 11, "Max Rank": 12},
"reasoning-logical": {"Avg Rank": 12, "Min Rank": 12, "Max Rank": 12},
"overall": {"Avg Rank": 12, "Min Rank": 11, "Max Rank": 12}
}},
{"config": {
"model_name": "gemma-2-27b-it",
"organization": "Google",
"license": "Gemma License",
"knowledge_cutoff": "2024-03"
},
"results": {
"math-algebra": {"Score": 91.08554346, "Avg Rank": 13.33333333, "Min Rank": 13, "Max Rank": 14},
"math-probability": {"Score": 91.09516215, "Avg Rank": 14, "Min Rank": 14, "Max Rank": 14},
"reasoning-logical": {"Avg Rank": 13, "Min Rank": 13, "Max Rank": 13},
"overall": {"Avg Rank": 13, "Min Rank": 12, "Max Rank": 14}
}},
{"config": {
"model_name": "claude-3-opus",
"organization": "Anthropic",
"license": "Proprietary",
"knowledge_cutoff": "2024-01"
},
"results": {
"math-algebra": {"Score": 89.75345785, "Avg Rank": 13.66666667, "Min Rank": 13, "Max Rank": 14},
"math-probability": {"Score": 91.06939607, "Avg Rank": 11.33333333, "Min Rank": 11, "Max Rank": 12},
"reasoning-logical": {"Avg Rank": 10.66666667, "Min Rank": 10, "Max Rank": 11},
"overall": {"Avg Rank": 12, "Min Rank": 10, "Max Rank": 15}
}},
{"config": {
"model_name": "gemma-2-9b-it-simpo",
"organization": "Google",
"license": "Gemma License",
"knowledge_cutoff": "2024-02"
},
"results": {
"math-algebra": {"Score": 87.66368227, "Avg Rank": 15, "Min Rank": 15, "Max Rank": 15},
"math-probability": {"Score": 73.64665336, "Avg Rank": 17, "Min Rank": 17, "Max Rank": 17},
"reasoning-logical": {"Avg Rank": 19, "Min Rank": 19, "Max Rank": 19},
"overall": {"Avg Rank": 17, "Min Rank": 15, "Max Rank": 19}
}},
{"config": {
"model_name": "qwen1.5-72b-chat",
"organization": "Alibaba",
"license": "Qianwen LICENSE",
"knowledge_cutoff": "2024-03"
},
"results": {
"math-algebra": {"Score": 86.56207015, "Avg Rank": 16, "Min Rank": 16, "Max Rank": 16},
"math-probability": {"Score": 72.7735874, "Avg Rank": 21, "Min Rank": 20, "Max Rank": 22},
"reasoning-logical": {"Avg Rank": 29.66666667, "Min Rank": 28, "Max Rank": 31},
"overall": {"Avg Rank": 23, "Min Rank": 16, "Max Rank": 31}
}},
{"config": {
"model_name": "qwen1.5-32b-chat",
"organization": "Alibaba",
"license": "Qianwen LICENSE",
"knowledge_cutoff": "2024-03"
},
"results": {
"math-algebra": {"Score": 84.59439036, "Avg Rank": 17.33333333, "Min Rank": 17, "Max Rank": 18},
"math-probability": {"Score": 76.61348265, "Avg Rank": 22.33333333, "Min Rank": 22, "Max Rank": 23},
"reasoning-logical": {"Avg Rank": 28.66666667, "Min Rank": 27, "Max Rank": 30},
"overall": {"Avg Rank": 22, "Min Rank": 17, "Max Rank": 30}
}},
{"config": {
"model_name": "google-gemma-2-9b-it",
"organization": "Google",
"license": "Proprietary",
"knowledge_cutoff": "2024-01"
},
"results": {
"math-algebra": {"Score": 84.18901776, "Avg Rank": 18, "Min Rank": 17, "Max Rank": 19},
"math-probability": {"Score": 74.46332504, "Avg Rank": 16, "Min Rank": 16, "Max Rank": 16},
"reasoning-logical": {"Avg Rank": 14, "Min Rank": 14, "Max Rank": 14},
"overall": {"Avg Rank": 16, "Min Rank": 14, "Max Rank": 19}
}},
{"config": {
"model_name": "yi-1.5-34b-chat",
"organization": "01 AI",
"license": "Proprietary",
"knowledge_cutoff": "2024-01"
},
"results": {
"math-algebra": {"Score": 81.82921677, "Avg Rank": 18.66666667, "Min Rank": 18, "Max Rank": 19},
"math-probability": {"Score": 77.41945842, "Avg Rank": 15, "Min Rank": 15, "Max Rank": 15},
"reasoning-logical": {"Avg Rank": 17.33333333, "Min Rank": 17, "Max Rank": 18},
"overall": {"Avg Rank": 18, "Min Rank": 15, "Max Rank": 19}
}},
{"config": {
"model_name": "meta-llama-3.1-8b-instruct",
"organization": "Meta",
"license": "Llama 3.1 Community",
"knowledge_cutoff": "2024-02"
},
"results": {
"math-algebra": {"Score": 75.57121963, "Avg Rank": 20.33333333, "Min Rank": 20, "Max Rank": 21},
"math-probability": {"Score": 75.46243493, "Avg Rank": 20.33333333, "Min Rank": 20, "Max Rank": 21},
"reasoning-logical": {"Avg Rank": 23.66666667, "Min Rank": 23, "Max Rank": 24},
"overall": {"Avg Rank": 21, "Min Rank": 20, "Max Rank": 24}
}},
{"config": {
"model_name": "gpt3.5-turbo-0125",
"organization": "OpenAI",
"license": "Proprietary",
"knowledge_cutoff": "2023-12"
},
"results": {
"math-algebra": {"Score": 73.29235048, "Avg Rank": 21.33333333, "Min Rank": 21, "Max Rank": 22},
"math-probability": {"Score": 66.27452275, "Avg Rank": 24, "Min Rank": 24, "Max Rank": 24},
"reasoning-logical": {"Avg Rank": 42.66666667, "Min Rank": 42, "Max Rank": 44},
"overall": {"Avg Rank": 29, "Min Rank": 21, "Max Rank": 44}
}},
{"config": {
"model_name": "llama-3-70b-instruct",
"organization": "Meta",
"license": "Llama 3 Community",
"knowledge_cutoff": "2024-03"
},
"results": {
"math-algebra": {"Score": 73.75419539, "Avg Rank": 21.33333333, "Min Rank": 20, "Max Rank": 22},
"math-probability": {"Score": 87.86358478, "Avg Rank": 18.33333333, "Min Rank": 18, "Max Rank": 19},
"reasoning-logical": {"Avg Rank": 3.333333333, "Min Rank": 2, "Max Rank": 4},
"overall": {"Avg Rank": 15, "Min Rank": 3, "Max Rank": 22}
}},
{"config": {
"model_name": "claude-3-sonnet",
"organization": "Anthropic",
"license": "Proprietary",
"knowledge_cutoff": "2024-02"
},
"results": {
"math-algebra": {"Score": 71.15353833, "Avg Rank": 23, "Min Rank": 23, "Max Rank": 23},
"math-probability": {"Score": 88.02362801, "Avg Rank": 18.66666667, "Min Rank": 18, "Max Rank": 19},
"reasoning-logical": {"Avg Rank": 17.33333333, "Min Rank": 16, "Max Rank": 18},
"overall": {"Avg Rank": 20, "Min Rank": 16, "Max Rank": 23}
}},
{"config": {
"model_name": "qwen1.5-14b-chat",
"organization": "Alibaba",
"license": "Qianwen LICENSE",
"knowledge_cutoff": "2024-01"
},
"results": {
"math-algebra": {"Score": 69.70470323, "Avg Rank": 24, "Min Rank": 24, "Max Rank": 24},
"math-probability": {"Score": 66.41420544, "Avg Rank": 28.66666667, "Min Rank": 28, "Max Rank": 29},
"reasoning-logical": {"Avg Rank": 34, "Min Rank": 34, "Max Rank": 34},
"overall": {"Avg Rank": 28, "Min Rank": 24, "Max Rank": 34}
}},
{"config": {
"model_name": "claude-3-haiku",
"organization": "Anthropic",
"license": "Proprietary",
"knowledge_cutoff": "2024-01"
},
"results": {
"math-algebra": {"Score": 68.44060149, "Avg Rank": 25, "Min Rank": 25, "Max Rank": 25},
"math-probability": {"Score": 76.46075239, "Avg Rank": 22.33333333, "Min Rank": 21, "Max Rank": 23},
"reasoning-logical": {"Avg Rank": 20, "Min Rank": 20, "Max Rank": 20},
"overall": {"Avg Rank": 22, "Min Rank": 20, "Max Rank": 25}
}},
{"config": {
"model_name": "claude-2.1",
"organization": "Anthropic",
"license": "Proprietary",
"knowledge_cutoff": "2023-12"
},
"results": {
"math-algebra": {"Score": 67.59939121, "Avg Rank": 26, "Min Rank": 26, "Max Rank": 26},
"math-probability": {"Score": 68.89772398, "Avg Rank": 27, "Min Rank": 27, "Max Rank": 27},
"reasoning-logical": {"Avg Rank": 21, "Min Rank": 21, "Max Rank": 21},
"overall": {"Avg Rank": 25, "Min Rank": 21, "Max Rank": 27}
}},
{"config": {
"model_name": "mistral-8x7b-instruct-v0.1",
"organization": "Mistral",
"license": "Apache 2.0",
"knowledge_cutoff": "2023-12"
},
"results": {
"math-algebra": {"Score": 64.71364004, "Avg Rank": 27, "Min Rank": 27, "Max Rank": 27},
"math-probability": {"Score": 67.67468595, "Avg Rank": 26, "Min Rank": 26, "Max Rank": 26},
"reasoning-logical": {"Avg Rank": 29, "Min Rank": 28, "Max Rank": 30},
"overall": {"Avg Rank": 27, "Min Rank": 26, "Max Rank": 30}
}},
{"config": {
"model_name": "claude-2.0",
"organization": "Anthropic",
"license": "Proprietary",
"knowledge_cutoff": "2023-10"
},
"results": {
"math-algebra": {"Score": 64.77311289, "Avg Rank": 28, "Min Rank": 28, "Max Rank": 28},
"math-probability": {"Score": 74.34063069, "Avg Rank": 25, "Min Rank": 25, "Max Rank": 25},
"reasoning-logical": {"Avg Rank": 23.33333333, "Min Rank": 23, "Max Rank": 24},
"overall": {"Avg Rank": 25, "Min Rank": 23, "Max Rank": 28}
}},
{"config": {
"model_name": "starling-lm-7b-beta",
"organization": "Nexusflow",
"license": "Apache-2.0",
"knowledge_cutoff": "2024-01"
},
"results": {
"math-algebra": {"Score": 64.01222884, "Avg Rank": 29.33333333, "Min Rank": 29, "Max Rank": 30},
"math-probability": {"Score": 70.42025806, "Avg Rank": 28.33333333, "Min Rank": 28, "Max Rank": 29},
"reasoning-logical": {"Avg Rank": 25, "Min Rank": 25, "Max Rank": 25},
"overall": {"Avg Rank": 27, "Min Rank": 25, "Max Rank": 30}
}},
{"config": {
"model_name": "gemini-1.0-pro-001",
"organization": "Google",
"license": "Proprietary",
"knowledge_cutoff": "2023-11"
},
"results": {
"math-algebra": {"Score": 63.93365247, "Avg Rank": 29.66666667, "Min Rank": 29, "Max Rank": 30},
"math-probability": {"Score": 62.13077748, "Avg Rank": 36.66666667, "Min Rank": 36, "Max Rank": 38},
"reasoning-logical": {"Avg Rank": 37.33333333, "Min Rank": 36, "Max Rank": 40},
"overall": {"Avg Rank": 34, "Min Rank": 29, "Max Rank": 40}
}},
{"config": {
"model_name": "openchat-3.5-0106",
"organization": "OpenChat",
"license": "Apache-2.0",
"knowledge_cutoff": "2024-01"
},
"results": {
"math-algebra": {"Score": 63.02959506, "Avg Rank": 31, "Min Rank": 31, "Max Rank": 31},
"math-probability": {"Score": 61.00599665, "Avg Rank": 30, "Min Rank": 30, "Max Rank": 30},
"reasoning-logical": {"Avg Rank": 27.66666667, "Min Rank": 27, "Max Rank": 29},
"overall": {"Avg Rank": 29, "Min Rank": 27, "Max Rank": 31}
}},
{"config": {
"model_name": "openchat-3.5",
"organization": "OpenChat",
"license": "Apache-2.0",
"knowledge_cutoff": "2023-12"
},
"results": {
"math-algebra": {"Score": 61.45954168, "Avg Rank": 32.33333333, "Min Rank": 32, "Max Rank": 33},
"math-probability": {"Score": 62.56195929, "Avg Rank": 32, "Min Rank": 32, "Max Rank": 32},
"reasoning-logical": {"Avg Rank": 32, "Min Rank": 31, "Max Rank": 33},
"overall": {"Avg Rank": 32, "Min Rank": 31, "Max Rank": 33}
}},
{"config": {
"model_name": "command-r-(08-2024)",
"organization": "Cohere",
"license": "CC-BY-NC-4.0",
"knowledge_cutoff": "2024-08"
},
"results": {
"math-algebra": {"Score": 61.0679475, "Avg Rank": 32.66666667, "Min Rank": 32, "Max Rank": 33},
"math-probability": {"Score": 66.00833826, "Avg Rank": 31, "Min Rank": 31, "Max Rank": 31},
"reasoning-logical": {"Avg Rank": 37.66666667, "Min Rank": 37, "Max Rank": 38},
"overall": {"Avg Rank": 34, "Min Rank": 31, "Max Rank": 38}
}},
{"config": {
"model_name": "gemma-1.1-7b-it",
"organization": "Google",
"license": "Gemma License",
"knowledge_cutoff": "2023-11"
},
"results": {
"math-algebra": {"Score": 60.92904194, "Avg Rank": 34.33333333, "Min Rank": 34, "Max Rank": 35},
"math-probability": {"Score": 62.17574935, "Avg Rank": 36.66666667, "Min Rank": 36, "Max Rank": 37},
"reasoning-logical": {"Avg Rank": 30.33333333, "Min Rank": 28, "Max Rank": 32},
"overall": {"Avg Rank": 34, "Min Rank": 28, "Max Rank": 37}
}},
{"config": {
"model_name": "llama3-8b-instruct",
"organization": "Meta",
"license": "Llama 3 Community",
"knowledge_cutoff": "2024-01"
},
"results": {
"math-algebra": {"Score": 61.06411319, "Avg Rank": 35, "Min Rank": 34, "Max Rank": 36},
"math-probability": {"Score": 62.13077748, "Avg Rank": 34.66666667, "Min Rank": 34, "Max Rank": 35},
"reasoning-logical": {"Avg Rank": 22, "Min Rank": 22, "Max Rank": 22},
"overall": {"Avg Rank": 30, "Min Rank": 22, "Max Rank": 36}
}},
{"config": {
"model_name": "gemma-2-2b-it",
"organization": "Google",
"license": "Gemma License",
"knowledge_cutoff": "2023-12"
},
"results": {
"math-algebra": {"Score": 59.70248014, "Avg Rank": 36, "Min Rank": 35, "Max Rank": 37},
"math-probability": {"Score": 61.08084527, "Avg Rank": 33.66666667, "Min Rank": 33, "Max Rank": 35},
"reasoning-logical": {"Avg Rank": 26, "Min Rank": 26, "Max Rank": 26},
"overall": {"Avg Rank": 32, "Min Rank": 26, "Max Rank": 37}
}},
{"config": {
"model_name": "starling-lm-7b-alpha",
"organization": "Nexusflow",
"license": "Apache-2.0",
"knowledge_cutoff": "2023-12"
},
"results": {
"math-algebra": {"Score": 59.574329, "Avg Rank": 36.66666667, "Min Rank": 36, "Max Rank": 37},
"math-probability": {"Score": 64.03683254, "Avg Rank": 33.66666667, "Min Rank": 33, "Max Rank": 34},
"reasoning-logical": {"Avg Rank": 35, "Min Rank": 35, "Max Rank": 35},
"overall": {"Avg Rank": 35, "Min Rank": 33, "Max Rank": 37}
}},
{"config": {
"model_name": "qwen1.5-4b-chat",
"organization": "Alibaba",
"license": "Qianwen LICENSE",
"knowledge_cutoff": "2024-02"
},
"results": {
"math-algebra": {"Score": 56.66282914, "Avg Rank": 38.33333333, "Min Rank": 38, "Max Rank": 39},
"math-probability": {"Score": 57.39032697, "Avg Rank": 43, "Min Rank": 43, "Max Rank": 43},
"reasoning-logical": {"Avg Rank": 46, "Min Rank": 46, "Max Rank": 46},
"overall": {"Avg Rank": 42, "Min Rank": 38, "Max Rank": 46}
}},
{"config": {
"model_name": "command-r-(04-2024)",
"organization": "Cohere",
"license": "CC-BY-NC-4.0",
"knowledge_cutoff": "2024-04"
},
"results": {
"math-algebra": {"Score": 56.19063413, "Avg Rank": 38.66666667, "Min Rank": 38, "Max Rank": 39},
"math-probability": {"Score": 54.37641509, "Avg Rank": 37.66666667, "Min Rank": 37, "Max Rank": 38},
"reasoning-logical": {"Avg Rank": 32.66666667, "Min Rank": 32, "Max Rank": 33},
"overall": {"Avg Rank": 36, "Min Rank": 32, "Max Rank": 39}
}},
{"config": {
"model_name": "vicuna-33b",
"organization": "LMSYS",
"license": "Non-commercial",
"knowledge_cutoff": "2023-12"
},
"results": {
"math-algebra": {"Score": 54.71037983, "Avg Rank": 40.66666667, "Min Rank": 40, "Max Rank": 42},
"math-probability": {"Score": 55.02214588, "Avg Rank": 41, "Min Rank": 41, "Max Rank": 41},
"reasoning-logical": {"Avg Rank": 41, "Min Rank": 41, "Max Rank": 41},
"overall": {"Avg Rank": 41, "Min Rank": 40, "Max Rank": 42}
}},
{"config": {
"model_name": "gemma-7b-it",
"organization": "Google",
"license": "Gemma License",
"knowledge_cutoff": "2023-12"
},
"results": {
"math-algebra": {"Score": 54.35817186, "Avg Rank": 40.66666667, "Min Rank": 40, "Max Rank": 41},
"math-probability": {"Score": 58.19573446, "Avg Rank": 42, "Min Rank": 42, "Max Rank": 42},
"reasoning-logical": {"Avg Rank": 39.33333333, "Min Rank": 39, "Max Rank": 40},
"overall": {"Avg Rank": 41, "Min Rank": 39, "Max Rank": 42}
}},
{"config": {
"model_name": "mistral-7b-instruct-2",
"organization": "Mistral",
"license": "Apache 2.0",
"knowledge_cutoff": "2023-12"
},
"results": {
"math-algebra": {"Score": 54.39240703, "Avg Rank": 41.66666667, "Min Rank": 41, "Max Rank": 42},
"math-probability": {"Score": 60.35257542, "Avg Rank": 39, "Min Rank": 39, "Max Rank": 39},
"reasoning-logical": {"Avg Rank": 36.66666667, "Min Rank": 36, "Max Rank": 37},
"overall": {"Avg Rank": 39, "Min Rank": 36, "Max Rank": 42}
}},
{"config": {
"model_name": "mistral-7b-instruct-1",
"organization": "Mistral",
"license": "Apache 2.0",
"knowledge_cutoff": "2023-12"
},
"results": {
"math-algebra": {"Score": 53.80157944, "Avg Rank": 43, "Min Rank": 43, "Max Rank": 43},
"math-probability": {"Score": 56.51960666, "Avg Rank": 40, "Min Rank": 40, "Max Rank": 40},
"reasoning-logical": {"Avg Rank": 45, "Min Rank": 45, "Max Rank": 45},
"overall": {"Avg Rank": 43, "Min Rank": 40, "Max Rank": 45}
}},
{"config": {
"model_name": "vicuna-13b",
"organization": "LMSYS",
"license": "Non-commercial",
"knowledge_cutoff": "2023-11"
},
"results": {
"math-algebra": {"Score": 53.5413765, "Avg Rank": 44, "Min Rank": 44, "Max Rank": 44},
"math-probability": {"Score": 53.53586693, "Avg Rank": 44, "Min Rank": 44, "Max Rank": 44},
"reasoning-logical": {"Avg Rank": 43.66666667, "Min Rank": 43, "Max Rank": 44},
"overall": {"Avg Rank": 44, "Min Rank": 43, "Max Rank": 44}
}},
{"config": {
"model_name": "zephyr-7b-beta",
"organization": "HuggingFace",
"license": "MIT",
"knowledge_cutoff": "2023-10"
},
"results": {
"math-algebra": {"Score": 52.23039742, "Avg Rank": 46, "Min Rank": 45, "Max Rank": 48},
"math-probability": {"Score": 51.67173535, "Avg Rank": 47.33333333, "Min Rank": 47, "Max Rank": 48},
"reasoning-logical": {"Avg Rank": 50, "Min Rank": 50, "Max Rank": 50},
"overall": {"Avg Rank": 48, "Min Rank": 45, "Max Rank": 50}
}},
{"config": {
"model_name": "gemma-1.1-2b-it",
"organization": "Google",
"license": "Gemma License",
"knowledge_cutoff": "2023-12"
},
"results": {
"math-algebra": {"Score": 52.22372428, "Avg Rank": 46, "Min Rank": 45, "Max Rank": 47},
"math-probability": {"Score": 51.74306688, "Avg Rank": 46.33333333, "Min Rank": 46, "Max Rank": 47},
"reasoning-logical": {"Avg Rank": 48, "Min Rank": 48, "Max Rank": 48},
"overall": {"Avg Rank": 47, "Min Rank": 45, "Max Rank": 48}
}},
{"config": {
"model_name": "llama2-7b-chat",
"organization": "Meta",
"license": "Llama 2 Community",
"knowledge_cutoff": "2023-10"
},
"results": {
"math-algebra": {"Score": 51.83025857, "Avg Rank": 46.33333333, "Min Rank": 46, "Max Rank": 47},
"math-probability": {"Score": 51.19585847, "Avg Rank": 47.33333333, "Min Rank": 46, "Max Rank": 48},
"reasoning-logical": {"Avg Rank": 42.66666667, "Min Rank": 42, "Max Rank": 43},
"overall": {"Avg Rank": 45, "Min Rank": 42, "Max Rank": 48}
}},
{"config": {
"model_name": "gemma-2b-it",
"organization": "Google",
"license": "Gemma License",
"knowledge_cutoff": "2023-11"
},
"results": {
"math-algebra": {"Score": 51.60281474, "Avg Rank": 47.66666667, "Min Rank": 47, "Max Rank": 48},
"math-probability": {"Score": 51.52250905, "Avg Rank": 50, "Min Rank": 50, "Max Rank": 50},
"reasoning-logical": {"Avg Rank": 51, "Min Rank": 51, "Max Rank": 51},
"overall": {"Avg Rank": 49, "Min Rank": 47, "Max Rank": 51}
}},
{"config": {
"model_name": "llama2-13b-chat",
"organization": "Meta",
"license": "Llama 2 Community",
"knowledge_cutoff": "2023-12"
},
"results": {
"math-algebra": {"Score": 51.21273132, "Avg Rank": 49, "Min Rank": 49, "Max Rank": 49},
"math-probability": {"Score": 51.72056522, "Avg Rank": 45, "Min Rank": 45, "Max Rank": 45},
"reasoning-logical": {"Avg Rank": 39, "Min Rank": 38, "Max Rank": 40},
"overall": {"Avg Rank": 44, "Min Rank": 38, "Max Rank": 49}
}},
{"config": {
"model_name": "vicuna-7b",
"organization": "LMSYS",
"license": "Non-commercial",
"knowledge_cutoff": "2023-11"
},
"results": {
"math-algebra": {"Score": 51.31450547, "Avg Rank": 50, "Min Rank": 50, "Max Rank": 50},
"math-probability": {"Score": 52.72504618, "Avg Rank": 49, "Min Rank": 49, "Max Rank": 49},
"reasoning-logical": {"Avg Rank": 47, "Min Rank": 47, "Max Rank": 47},
"overall": {"Avg Rank": 48, "Min Rank": 47, "Max Rank": 50}
}},
{"config": {
"model_name": "koala-13b",
"organization": "UC Berkeley",
"license": "Non-commercial",
"knowledge_cutoff": "2023-10"
},
"results": {
"math-algebra": {"Score": 50.19054677, "Avg Rank": 51, "Min Rank": 51, "Max Rank": 51},
"math-probability": {"Score": 50.741989, "Avg Rank": 51, "Min Rank": 51, "Max Rank": 51},
"reasoning-logical": {"Avg Rank": 49, "Min Rank": 49, "Max Rank": 49},
"overall": {"Avg Rank": 50, "Min Rank": 49, "Max Rank": 51}
}},
{"config": {
"model_name": "openassistant-pythia-12b",
"organization": "OpenAssistant",
"license": "Non-commercial",
"knowledge_cutoff": "2023-09"
},
"results": {
"math-algebra": {"Score": 50, "Avg Rank": 52, "Min Rank": 52, "Max Rank": 52},
"math-probability": {"Score": 50, "Avg Rank": 52, "Min Rank": 52, "Max Rank": 52},
"reasoning-logical": {"Avg Rank": 52, "Min Rank": 52, "Max Rank": 52},
"overall": {"Avg Rank": 52, "Min Rank": 52, "Max Rank": 52}
}}
]