| [ | |
| { | |
| "config": { | |
| "model_name": "ChatGPT-4o-latest (2024-09-03)", | |
| "organization": "OpenAI", | |
| "license": "Proprietary", | |
| "knowledge_cutoff": "2023/10" | |
| }, | |
| "results": { | |
| "OVERALL": { | |
| "Average Score": 0.974329609, | |
| "Standard Deviation": 0.005024959031, | |
| "Rank": 2 | |
| }, | |
| "Geometry": { | |
| "Average Score": 0.976028578, | |
| "Standard Deviation": 0.01507912373, | |
| "Rank": 3 | |
| }, | |
| "Algebra": { | |
| "Average Score": 0.951199453, | |
| "Standard Deviation": 0.08452452108, | |
| "Rank": 3 | |
| }, | |
| "Probability": { | |
| "Average Score": 0.842116641, | |
| "Standard Deviation": 0.006267759054, | |
| "Rank": 3 | |
| }, | |
| "Logical": { | |
| "Average Score": 0.828490728, | |
| "Standard Deviation": 0.009134213144, | |
| "Rank": 3 | |
| }, | |
| "Social": { | |
| "Average Score": 0.815902987, | |
| "Standard Deviation": 0.0196254222, | |
| "Rank": 4 | |
| } | |
| } | |
| }, | |
| { | |
| "config": { | |
| "model_name": "gpt-4o-2024-08-06", | |
| "organization": "OpenAI", | |
| "license": "Proprietary", | |
| "knowledge_cutoff": "2023/10" | |
| }, | |
| "results": { | |
| "OVERALL": { | |
| "Average Score": 0.846571548, | |
| "Standard Deviation": 0.03394056554, | |
| "Rank": 6 | |
| }, | |
| "Geometry": { | |
| "Average Score": 0.99773096, | |
| "Standard Deviation": 0.002835555172, | |
| "Rank": 1 | |
| }, | |
| "Algebra": { | |
| "Average Score": 1.0, | |
| "Standard Deviation": 0.0, | |
| "Rank": 1 | |
| }, | |
| "Probability": { | |
| "Average Score": 0.78855795, | |
| "Standard Deviation": 0.008188675452, | |
| "Rank": 6 | |
| }, | |
| "Logical": { | |
| "Average Score": 0.668635768, | |
| "Standard Deviation": 0.03466314094, | |
| "Rank": 11 | |
| }, | |
| "Social": { | |
| "Average Score": 0.680417314, | |
| "Standard Deviation": 0.00656867063, | |
| "Rank": 9 | |
| } | |
| } | |
| }, | |
| { | |
| "config": { | |
| "model_name": "gpt-4o-2024-05-13", | |
| "organization": "OpenAI", | |
| "license": "Proprietary", | |
| "knowledge_cutoff": "2023/10" | |
| }, | |
| "results": { | |
| "OVERALL": { | |
| "Average Score": 0.846334477, | |
| "Standard Deviation": 0.09377911572, | |
| "Rank": 7 | |
| }, | |
| "Geometry": { | |
| "Average Score": 0.972472377, | |
| "Standard Deviation": 0.01648274205, | |
| "Rank": 4 | |
| }, | |
| "Algebra": { | |
| "Average Score": 0.995511298, | |
| "Standard Deviation": 0.004097802515, | |
| "Rank": 2 | |
| }, | |
| "Probability": { | |
| "Average Score": 0.812149974, | |
| "Standard Deviation": 0.007669585485, | |
| "Rank": 4 | |
| }, | |
| "Logical": { | |
| "Average Score": 0.755019692, | |
| "Standard Deviation": 0.008149588572, | |
| "Rank": 6 | |
| }, | |
| "Social": { | |
| "Average Score": 0.609875087, | |
| "Standard Deviation": 0.038729239, | |
| "Rank": 14 | |
| } | |
| } | |
| }, | |
| { | |
| "config": { | |
| "model_name": "gpt-4-turbo-2024-04-09", | |
| "organization": "OpenAI", | |
| "license": "Proprietary", | |
| "knowledge_cutoff": "2023/12" | |
| }, | |
| "results": { | |
| "OVERALL": { | |
| "Average Score": 0.855357972, | |
| "Standard Deviation": 0.1016986368, | |
| "Rank": 4 | |
| }, | |
| "Geometry": { | |
| "Average Score": 0.95374588, | |
| "Standard Deviation": 0.03109307166, | |
| "Rank": 5 | |
| }, | |
| "Algebra": { | |
| "Average Score": 0.930945223, | |
| "Standard Deviation": 0.06705136813, | |
| "Rank": 4 | |
| }, | |
| "Probability": { | |
| "Average Score": 0.750705448, | |
| "Standard Deviation": 0.05944483103, | |
| "Rank": 8 | |
| }, | |
| "Logical": { | |
| "Average Score": 0.77906699, | |
| "Standard Deviation": 0.007406734161, | |
| "Rank": 4 | |
| }, | |
| "Social": { | |
| "Average Score": 0.715935163, | |
| "Standard Deviation": 0.1209141409, | |
| "Rank": 7 | |
| } | |
| } | |
| }, | |
| { | |
| "config": { | |
| "model_name": "gemini-1.5-pro-001", | |
| "organization": "Google", | |
| "license": "Proprietary", | |
| "knowledge_cutoff": "2024-01" | |
| }, | |
| "results": { | |
| "OVERALL": { | |
| "Average Score": 0.797187842, | |
| "Standard Deviation": 0.0272375249, | |
| "Rank": 10 | |
| }, | |
| "Geometry": { | |
| "Average Score": 0.9947169, | |
| "Standard Deviation": 0.009150597621, | |
| "Rank": 2 | |
| }, | |
| "Algebra": { | |
| "Average Score": 0.857464301, | |
| "Standard Deviation": 0.05014285338, | |
| "Rank": 5 | |
| }, | |
| "Probability": { | |
| "Average Score": 0.651781767, | |
| "Standard Deviation": 0.04156998547, | |
| "Rank": 12 | |
| }, | |
| "Logical": { | |
| "Average Score": 0.739745471, | |
| "Standard Deviation": 0.01631532019, | |
| "Rank": 7 | |
| }, | |
| "Social": { | |
| "Average Score": 0.649601885, | |
| "Standard Deviation": 0.104854889, | |
| "Rank": 12 | |
| } | |
| } | |
| }, | |
| { | |
| "config": { | |
| "model_name": "qwen2-72b-instruct", | |
| "organization": "Alibaba", | |
| "license": "Qianwen LICENSE", | |
| "knowledge_cutoff": "2024-02" | |
| }, | |
| "results": { | |
| "OVERALL": { | |
| "Average Score": 0.737918558, | |
| "Standard Deviation": 0.09069077339, | |
| "Rank": 11 | |
| }, | |
| "Geometry": { | |
| "Average Score": 0.796870305, | |
| "Standard Deviation": 0.0509025346, | |
| "Rank": 9 | |
| }, | |
| "Algebra": { | |
| "Average Score": 0.836194231, | |
| "Standard Deviation": 0.04517093028, | |
| "Rank": 6 | |
| }, | |
| "Probability": { | |
| "Average Score": 0.788068004, | |
| "Standard Deviation": 0.007288989044, | |
| "Rank": 7 | |
| }, | |
| "Logical": { | |
| "Average Score": 0.619300904, | |
| "Standard Deviation": 0.06377931612, | |
| "Rank": 15 | |
| }, | |
| "Social": { | |
| "Average Score": 0.652578786, | |
| "Standard Deviation": 0.04259293171, | |
| "Rank": 11 | |
| } | |
| } | |
| }, | |
| { | |
| "config": { | |
| "model_name": "gpt-4o-mini-2024-07-18", | |
| "organization": "OpenAI", | |
| "license": "Proprietary", | |
| "knowledge_cutoff": "2024-07" | |
| }, | |
| "results": { | |
| "OVERALL": { | |
| "Average Score": 0.847694133, | |
| "Standard Deviation": 0.02164304402, | |
| "Rank": 5 | |
| }, | |
| "Geometry": { | |
| "Average Score": 0.946650435, | |
| "Standard Deviation": 0.01831236482, | |
| "Rank": 7 | |
| }, | |
| "Algebra": { | |
| "Average Score": 0.796243022, | |
| "Standard Deviation": 0.05537539202, | |
| "Rank": 7 | |
| }, | |
| "Probability": { | |
| "Average Score": 0.798402685, | |
| "Standard Deviation": 0.009404491967, | |
| "Rank": 5 | |
| }, | |
| "Logical": { | |
| "Average Score": 0.727009735, | |
| "Standard Deviation": 0.02628110141, | |
| "Rank": 8 | |
| }, | |
| "Social": { | |
| "Average Score": 0.691949855, | |
| "Standard Deviation": 0.02072934333, | |
| "Rank": 8 | |
| } | |
| } | |
| }, | |
| { | |
| "config": { | |
| "model_name": "claude-3.5-sonnet", | |
| "organization": "Anthropic", | |
| "license": "Proprietary", | |
| "knowledge_cutoff": "2024-03" | |
| }, | |
| "results": { | |
| "OVERALL": { | |
| "Average Score": 0.839004422, | |
| "Standard Deviation": 0.1461079564, | |
| "Rank": 8 | |
| }, | |
| "Geometry": { | |
| "Average Score": 0.95316419, | |
| "Standard Deviation": 0.02081192856, | |
| "Rank": 6 | |
| }, | |
| "Algebra": { | |
| "Average Score": 0.759789952, | |
| "Standard Deviation": 0.02611765096, | |
| "Rank": 8 | |
| }, | |
| "Probability": { | |
| "Average Score": 0.707730127, | |
| "Standard Deviation": 0.0394436664, | |
| "Rank": 10 | |
| }, | |
| "Logical": { | |
| "Average Score": 0.77342666, | |
| "Standard Deviation": 0.002892426458, | |
| "Rank": 5 | |
| }, | |
| "Social": { | |
| "Average Score": 0.790002247, | |
| "Standard Deviation": 0.1007410022, | |
| "Rank": 5 | |
| } | |
| } | |
| }, | |
| { | |
| "config": { | |
| "model_name": "o1-mini", | |
| "organization": "OpenAI", | |
| "license": "Proprietary", | |
| "knowledge_cutoff": "2024-01" | |
| }, | |
| "results": { | |
| "OVERALL": { | |
| "Average Score": 1.0, | |
| "Standard Deviation": 0.0, | |
| "Rank": 1 | |
| }, | |
| "Geometry": { | |
| "Average Score": "N/A", | |
| "Standard Deviation": "N/A", | |
| "Rank": "N/A" | |
| }, | |
| "Algebra": { | |
| "Average Score": "N/A", | |
| "Standard Deviation": "N/A", | |
| "Rank": "N/A" | |
| }, | |
| "Probability": { | |
| "Average Score": 1.0, | |
| "Standard Deviation": 0.0, | |
| "Rank": 1 | |
| }, | |
| "Logical": { | |
| "Average Score": 1.0, | |
| "Standard Deviation": 0.0, | |
| "Rank": 1 | |
| }, | |
| "Social": { | |
| "Average Score": 0.993974241, | |
| "Standard Deviation": 0.001996882328, | |
| "Rank": 2 | |
| } | |
| } | |
| }, | |
| { | |
| "config": { | |
| "model_name": "o1-preview", | |
| "organization": "OpenAI", | |
| "license": "Proprietary", | |
| "knowledge_cutoff": "2024-01" | |
| }, | |
| "results": { | |
| "OVERALL": { | |
| "Average Score": 0.945884589, | |
| "Standard Deviation": 0.01059250762, | |
| "Rank": 3 | |
| }, | |
| "Geometry": { | |
| "Average Score": "N/A", | |
| "Standard Deviation": "N/A", | |
| "Rank": "N/A" | |
| }, | |
| "Algebra": { | |
| "Average Score": "N/A", | |
| "Standard Deviation": "N/A", | |
| "Rank": "N/A" | |
| }, | |
| "Probability": { | |
| "Average Score": 0.964666392, | |
| "Standard Deviation": 0.003139983398, | |
| "Rank": 2 | |
| }, | |
| "Logical": { | |
| "Average Score": 0.987950057, | |
| "Standard Deviation": 0.004881220327, | |
| "Rank": 2 | |
| }, | |
| "Social": { | |
| "Average Score": 1.0, | |
| "Standard Deviation": 0.0, | |
| "Rank": 1 | |
| } | |
| } | |
| }, | |
| { | |
| "config": { | |
| "model_name": "gemini-1.5-flash-001", | |
| "organization": "Google", | |
| "license": "Proprietary", | |
| "knowledge_cutoff": "2024-02" | |
| }, | |
| "results": { | |
| "OVERALL": { | |
| "Average Score": 0.726493401, | |
| "Standard Deviation": 0.01113913725, | |
| "Rank": 12 | |
| }, | |
| "Geometry": { | |
| "Average Score": 0.804144103, | |
| "Standard Deviation": 0.1327142178, | |
| "Rank": 8 | |
| }, | |
| "Algebra": { | |
| "Average Score": 0.731776765, | |
| "Standard Deviation": 0.02594657111, | |
| "Rank": 11 | |
| }, | |
| "Probability": { | |
| "Average Score": 0.614461891, | |
| "Standard Deviation": 0.04690131826, | |
| "Rank": 15 | |
| }, | |
| "Logical": { | |
| "Average Score": 0.630805991, | |
| "Standard Deviation": 0.04871350612, | |
| "Rank": 13 | |
| }, | |
| "Social": { | |
| "Average Score": 0.555933822, | |
| "Standard Deviation": 0.1029934524, | |
| "Rank": 16 | |
| } | |
| } | |
| }, | |
| { | |
| "config": { | |
| "model_name": "gpt4-1106", | |
| "organization": "OpenAI", | |
| "license": "Proprietary", | |
| "knowledge_cutoff": "2024-04" | |
| }, | |
| "results": { | |
| "OVERALL": { | |
| "Average Score": 0.816347784, | |
| "Standard Deviation": 0.1566815755, | |
| "Rank": 9 | |
| }, | |
| "Geometry": { | |
| "Average Score": 0.71843088, | |
| "Standard Deviation": 0.04778038294, | |
| "Rank": 13 | |
| }, | |
| "Algebra": { | |
| "Average Score": 0.712910417, | |
| "Standard Deviation": 0.02581828898, | |
| "Rank": 12 | |
| }, | |
| "Probability": { | |
| "Average Score": 0.623947619, | |
| "Standard Deviation": 0.03502982933, | |
| "Rank": 14 | |
| }, | |
| "Logical": { | |
| "Average Score": 0.637482274, | |
| "Standard Deviation": 0.04158809888, | |
| "Rank": 12 | |
| }, | |
| "Social": { | |
| "Average Score": 0.450609816, | |
| "Standard Deviation": 0.05208655446, | |
| "Rank": 23 | |
| } | |
| } | |
| }, | |
| { | |
| "config": { | |
| "model_name": "gemma-2-27b-it", | |
| "organization": "Google", | |
| "license": "Gemma License", | |
| "knowledge_cutoff": "2024-03" | |
| }, | |
| "results": { | |
| "OVERALL": { | |
| "Average Score": 0.624169623, | |
| "Standard Deviation": 0.1048365121, | |
| "Rank": 15 | |
| }, | |
| "Geometry": { | |
| "Average Score": 0.60112744, | |
| "Standard Deviation": 0.0469109952, | |
| "Rank": 19 | |
| }, | |
| "Algebra": { | |
| "Average Score": 0.687955914, | |
| "Standard Deviation": 0.01959958192, | |
| "Rank": 13 | |
| }, | |
| "Probability": { | |
| "Average Score": 0.589524771, | |
| "Standard Deviation": 0.03112689325, | |
| "Rank": 16 | |
| }, | |
| "Logical": { | |
| "Average Score": 0.614978944, | |
| "Standard Deviation": 0.05710657859, | |
| "Rank": 16 | |
| }, | |
| "Social": { | |
| "Average Score": 0.487844257, | |
| "Standard Deviation": 0.05857760809, | |
| "Rank": 20 | |
| } | |
| } | |
| }, | |
| { | |
| "config": { | |
| "model_name": "claude-3-opus", | |
| "organization": "Anthropic", | |
| "license": "Proprietary", | |
| "knowledge_cutoff": "2024-01" | |
| }, | |
| "results": { | |
| "OVERALL": { | |
| "Average Score": 0.650636271, | |
| "Standard Deviation": 0.1197773541, | |
| "Rank": 14 | |
| }, | |
| "Geometry": { | |
| "Average Score": 0.7215743, | |
| "Standard Deviation": 0.04712598358, | |
| "Rank": 12 | |
| }, | |
| "Algebra": { | |
| "Average Score": 0.68777327, | |
| "Standard Deviation": 0.02382683713, | |
| "Rank": 14 | |
| }, | |
| "Probability": { | |
| "Average Score": 0.626471421, | |
| "Standard Deviation": 0.02911817976, | |
| "Rank": 13 | |
| }, | |
| "Logical": { | |
| "Average Score": 0.692346381, | |
| "Standard Deviation": 0.03617185198, | |
| "Rank": 10 | |
| }, | |
| "Social": { | |
| "Average Score": 0.663410854, | |
| "Standard Deviation": 0.09540220876, | |
| "Rank": 10 | |
| } | |
| } | |
| }, | |
| { | |
| "config": { | |
| "model_name": "gemma-2-9b-it-simpo", | |
| "organization": "Google", | |
| "license": "Gemma License", | |
| "knowledge_cutoff": "2024-02" | |
| }, | |
| "results": { | |
| "OVERALL": { | |
| "Average Score": "N/A", | |
| "Standard Deviation": "N/A", | |
| "Rank": "N/A" | |
| }, | |
| "Geometry": { | |
| "Average Score": 0.582787508, | |
| "Standard Deviation": 0.03965204074, | |
| "Rank": 20 | |
| }, | |
| "Algebra": { | |
| "Average Score": 0.658648133, | |
| "Standard Deviation": 0.02565919856, | |
| "Rank": 15 | |
| }, | |
| "Probability": { | |
| "Average Score": 0.547861265, | |
| "Standard Deviation": 0.02885209131, | |
| "Rank": 19 | |
| }, | |
| "Logical": { | |
| "Average Score": 0.540720893, | |
| "Standard Deviation": 0.01970134508, | |
| "Rank": 20 | |
| }, | |
| "Social": { | |
| "Average Score": 0.635266187, | |
| "Standard Deviation": 0.03620021751, | |
| "Rank": 13 | |
| } | |
| } | |
| }, | |
| { | |
| "config": { | |
| "model_name": "qwen1.5-72b-chat", | |
| "organization": "Alibaba", | |
| "license": "Qianwen LICENSE", | |
| "knowledge_cutoff": "2024-03" | |
| }, | |
| "results": { | |
| "OVERALL": { | |
| "Average Score": 0.519549796, | |
| "Standard Deviation": 0.00903634343, | |
| "Rank": 18 | |
| }, | |
| "Geometry": { | |
| "Average Score": 0.543139301, | |
| "Standard Deviation": 0.03425202326, | |
| "Rank": 24 | |
| }, | |
| "Algebra": { | |
| "Average Score": 0.635228729, | |
| "Standard Deviation": 0.01944043425, | |
| "Rank": 16 | |
| }, | |
| "Probability": { | |
| "Average Score": 0.486948658, | |
| "Standard Deviation": 0.06064655315, | |
| "Rank": 23 | |
| }, | |
| "Logical": { | |
| "Average Score": 0.284069394, | |
| "Standard Deviation": 0.02686608506, | |
| "Rank": 33 | |
| }, | |
| "Social": { | |
| "Average Score": 0.415007627, | |
| "Standard Deviation": 0.03920053159, | |
| "Rank": 24 | |
| } | |
| } | |
| }, | |
| { | |
| "config": { | |
| "model_name": "qwen1.5-32b-chat", | |
| "organization": "Alibaba", | |
| "license": "Qianwen LICENSE", | |
| "knowledge_cutoff": "2024-03" | |
| }, | |
| "results": { | |
| "OVERALL": { | |
| "Average Score": 0.393789407, | |
| "Standard Deviation": 0.05413770095, | |
| "Rank": 29 | |
| }, | |
| "Geometry": { | |
| "Average Score": 0.51086835, | |
| "Standard Deviation": 0.04052471998, | |
| "Rank": 27 | |
| }, | |
| "Algebra": { | |
| "Average Score": 0.609003168, | |
| "Standard Deviation": 0.04874143541, | |
| "Rank": 17 | |
| }, | |
| "Probability": { | |
| "Average Score": 0.476300002, | |
| "Standard Deviation": 0.05322403912, | |
| "Rank": 24 | |
| }, | |
| "Logical": { | |
| "Average Score": 0.331781014, | |
| "Standard Deviation": 0.004938997686, | |
| "Rank": 30 | |
| }, | |
| "Social": { | |
| "Average Score": 0.380987334, | |
| "Standard Deviation": 0.03762251776, | |
| "Rank": 26 | |
| } | |
| } | |
| }, | |
| { | |
| "config": { | |
| "model_name": "google-gemma-2-9b-it", | |
| "organization": "Google", | |
| "license": "Proprietary", | |
| "knowledge_cutoff": "2024-01" | |
| }, | |
| "results": { | |
| "OVERALL": { | |
| "Average Score": 0.489663449, | |
| "Standard Deviation": 0.002595702019, | |
| "Rank": 21 | |
| }, | |
| "Geometry": { | |
| "Average Score": 0.575371308, | |
| "Standard Deviation": 0.03556220251, | |
| "Rank": 22 | |
| }, | |
| "Algebra": { | |
| "Average Score": 0.597045661, | |
| "Standard Deviation": 0.0313828123, | |
| "Rank": 18 | |
| }, | |
| "Probability": { | |
| "Average Score": 0.589221807, | |
| "Standard Deviation": 0.03110811656, | |
| "Rank": 18 | |
| }, | |
| "Logical": { | |
| "Average Score": 0.587579897, | |
| "Standard Deviation": 0.05512716783, | |
| "Rank": 18 | |
| }, | |
| "Social": { | |
| "Average Score": 0.768337958, | |
| "Standard Deviation": 0.04078610476, | |
| "Rank": 6 | |
| } | |
| } | |
| }, | |
| { | |
| "config": { | |
| "model_name": "yi-1.5-34b-chat", | |
| "organization": "01 AI", | |
| "license": "Proprietary", | |
| "knowledge_cutoff": "2024-01" | |
| }, | |
| "results": { | |
| "OVERALL": { | |
| "Average Score": 0.607812897, | |
| "Standard Deviation": 0.1440881293, | |
| "Rank": 16 | |
| }, | |
| "Geometry": { | |
| "Average Score": 0.566666724, | |
| "Standard Deviation": 0.04001381658, | |
| "Rank": 23 | |
| }, | |
| "Algebra": { | |
| "Average Score": 0.590997292, | |
| "Standard Deviation": 0.03594087315, | |
| "Rank": 19 | |
| }, | |
| "Probability": { | |
| "Average Score": 0.589524589, | |
| "Standard Deviation": 0.03112618772, | |
| "Rank": 17 | |
| }, | |
| "Logical": { | |
| "Average Score": 0.574105508, | |
| "Standard Deviation": 0.03441737941, | |
| "Rank": 19 | |
| }, | |
| "Social": { | |
| "Average Score": 0.516980832, | |
| "Standard Deviation": 0.03369347985, | |
| "Rank": 19 | |
| } | |
| } | |
| }, | |
| { | |
| "config": { | |
| "model_name": "meta-llama-3.1-8b-instruct", | |
| "organization": "Meta", | |
| "license": "Llama 3.1 Community", | |
| "knowledge_cutoff": "2024-02" | |
| }, | |
| "results": { | |
| "OVERALL": { | |
| "Average Score": 0.505936324, | |
| "Standard Deviation": 0.05286756493, | |
| "Rank": 19 | |
| }, | |
| "Geometry": { | |
| "Average Score": 0.522442162, | |
| "Standard Deviation": 0.03908236317, | |
| "Rank": 25 | |
| }, | |
| "Algebra": { | |
| "Average Score": 0.582702645, | |
| "Standard Deviation": 0.05002277711, | |
| "Rank": 20 | |
| }, | |
| "Probability": { | |
| "Average Score": 0.495001149, | |
| "Standard Deviation": 0.05244587037, | |
| "Rank": 22 | |
| }, | |
| "Logical": { | |
| "Average Score": 0.443030561, | |
| "Standard Deviation": 0.01343820628, | |
| "Rank": 25 | |
| }, | |
| "Social": { | |
| "Average Score": 0.329195941, | |
| "Standard Deviation": 0.03925019528, | |
| "Rank": 30 | |
| } | |
| } | |
| }, | |
| { | |
| "config": { | |
| "model_name": "gpt3.5-turbo-0125", | |
| "organization": "OpenAI", | |
| "license": "Proprietary", | |
| "knowledge_cutoff": "2023-12" | |
| }, | |
| "results": { | |
| "OVERALL": { | |
| "Average Score": 0.313398088, | |
| "Standard Deviation": 0.09322528606, | |
| "Rank": 40 | |
| }, | |
| "Geometry": { | |
| "Average Score": 0.678714519, | |
| "Standard Deviation": 0.05926546762, | |
| "Rank": 14 | |
| }, | |
| "Algebra": { | |
| "Average Score": 0.569296173, | |
| "Standard Deviation": 0.05277281097, | |
| "Rank": 21 | |
| }, | |
| "Probability": { | |
| "Average Score": 0.448460767, | |
| "Standard Deviation": 0.05768095196, | |
| "Rank": 26 | |
| }, | |
| "Logical": { | |
| "Average Score": 0.148521348, | |
| "Standard Deviation": 0.04033712907, | |
| "Rank": 45 | |
| }, | |
| "Social": { | |
| "Average Score": 0.235071541, | |
| "Standard Deviation": 0.02632892457, | |
| "Rank": 39 | |
| } | |
| } | |
| }, | |
| { | |
| "config": { | |
| "model_name": "llama-3-70b-instruct", | |
| "organization": "Meta", | |
| "license": "Llama 3 Community", | |
| "knowledge_cutoff": "2024-03" | |
| }, | |
| "results": { | |
| "OVERALL": { | |
| "Average Score": 0.456689885, | |
| "Standard Deviation": 0.01385989995, | |
| "Rank": 23 | |
| }, | |
| "Geometry": { | |
| "Average Score": 0.516865529, | |
| "Standard Deviation": 0.03858112564, | |
| "Rank": 26 | |
| }, | |
| "Algebra": { | |
| "Average Score": 0.566756531, | |
| "Standard Deviation": 0.03369826926, | |
| "Rank": 22 | |
| }, | |
| "Probability": { | |
| "Average Score": 0.513857306, | |
| "Standard Deviation": 0.05453699062, | |
| "Rank": 21 | |
| }, | |
| "Logical": { | |
| "Average Score": 0.713796415, | |
| "Standard Deviation": 0.02031215107, | |
| "Rank": 9 | |
| }, | |
| "Social": { | |
| "Average Score": 0.45872939, | |
| "Standard Deviation": 0.05347039576, | |
| "Rank": 22 | |
| } | |
| } | |
| }, | |
| { | |
| "config": { | |
| "model_name": "claude-3-sonnet", | |
| "organization": "Anthropic", | |
| "license": "Proprietary", | |
| "knowledge_cutoff": "2024-02" | |
| }, | |
| "results": { | |
| "OVERALL": { | |
| "Average Score": 0.520010833, | |
| "Standard Deviation": 0.005030563799, | |
| "Rank": 17 | |
| }, | |
| "Geometry": { | |
| "Average Score": 0.675613638, | |
| "Standard Deviation": 0.05275594408, | |
| "Rank": 15 | |
| }, | |
| "Algebra": { | |
| "Average Score": 0.552025728, | |
| "Standard Deviation": 0.04122192409, | |
| "Rank": 23 | |
| }, | |
| "Probability": { | |
| "Average Score": 0.516192848, | |
| "Standard Deviation": 0.04152293217, | |
| "Rank": 20 | |
| }, | |
| "Logical": { | |
| "Average Score": 0.588545747, | |
| "Standard Deviation": 0.06068211943, | |
| "Rank": 17 | |
| }, | |
| "Social": { | |
| "Average Score": 0.570437582, | |
| "Standard Deviation": 0.08607040862, | |
| "Rank": 15 | |
| } | |
| } | |
| }, | |
| { | |
| "config": { | |
| "model_name": "qwen1.5-14b-chat", | |
| "organization": "Alibaba", | |
| "license": "Qianwen LICENSE", | |
| "knowledge_cutoff": "2024-01" | |
| }, | |
| "results": { | |
| "OVERALL": { | |
| "Average Score": 0.415328996, | |
| "Standard Deviation": 0.0743938717, | |
| "Rank": 28 | |
| }, | |
| "Geometry": { | |
| "Average Score": 0.452504016, | |
| "Standard Deviation": 0.04225594393, | |
| "Rank": 28 | |
| }, | |
| "Algebra": { | |
| "Average Score": 0.538655725, | |
| "Standard Deviation": 0.03721542594, | |
| "Rank": 24 | |
| }, | |
| "Probability": { | |
| "Average Score": 0.397185975, | |
| "Standard Deviation": 0.05607695946, | |
| "Rank": 30 | |
| }, | |
| "Logical": { | |
| "Average Score": 0.264573129, | |
| "Standard Deviation": 0.03936133174, | |
| "Rank": 35 | |
| }, | |
| "Social": { | |
| "Average Score": 0.287370142, | |
| "Standard Deviation": 0.04264085315, | |
| "Rank": 32 | |
| } | |
| } | |
| }, | |
| { | |
| "config": { | |
| "model_name": "claude-3-haiku", | |
| "organization": "Anthropic", | |
| "license": "Proprietary", | |
| "knowledge_cutoff": "2024-01" | |
| }, | |
| "results": { | |
| "OVERALL": { | |
| "Average Score": 0.453901163, | |
| "Standard Deviation": 0.003604084261, | |
| "Rank": 24 | |
| }, | |
| "Geometry": { | |
| "Average Score": 0.607993912, | |
| "Standard Deviation": 0.05793460748, | |
| "Rank": 17 | |
| }, | |
| "Algebra": { | |
| "Average Score": 0.520054055, | |
| "Standard Deviation": 0.03333544511, | |
| "Rank": 25 | |
| }, | |
| "Probability": { | |
| "Average Score": 0.474460688, | |
| "Standard Deviation": 0.0446501933, | |
| "Rank": 25 | |
| }, | |
| "Logical": { | |
| "Average Score": 0.512815976, | |
| "Standard Deviation": 0.0163264281, | |
| "Rank": 21 | |
| }, | |
| "Social": { | |
| "Average Score": 0.551083976, | |
| "Standard Deviation": 0.05374722539, | |
| "Rank": 17 | |
| } | |
| } | |
| }, | |
| { | |
| "config": { | |
| "model_name": "claude-2.1", | |
| "organization": "Anthropic", | |
| "license": "Proprietary", | |
| "knowledge_cutoff": "2023-12" | |
| }, | |
| "results": { | |
| "OVERALL": { | |
| "Average Score": 0.35814708, | |
| "Standard Deviation": 0.09168134168, | |
| "Rank": 36 | |
| }, | |
| "Geometry": { | |
| "Average Score": 0.62752395, | |
| "Standard Deviation": 0.07232659398, | |
| "Rank": 16 | |
| }, | |
| "Algebra": { | |
| "Average Score": 0.508849609, | |
| "Standard Deviation": 0.0346897465, | |
| "Rank": 26 | |
| }, | |
| "Probability": { | |
| "Average Score": 0.41477086, | |
| "Standard Deviation": 0.05964060239, | |
| "Rank": 29 | |
| }, | |
| "Logical": { | |
| "Average Score": 0.482923674, | |
| "Standard Deviation": 0.01989147048, | |
| "Rank": 22 | |
| }, | |
| "Social": { | |
| "Average Score": 0.333804568, | |
| "Standard Deviation": 0.03775548253, | |
| "Rank": 29 | |
| } | |
| } | |
| }, | |
| { | |
| "config": { | |
| "model_name": "mistral-8x7b-instruct-v0.1", | |
| "organization": "Mistral", | |
| "license": "Apache 2.0", | |
| "knowledge_cutoff": "2023-12" | |
| }, | |
| "results": { | |
| "OVERALL": { | |
| "Average Score": 0.382659161, | |
| "Standard Deviation": 0.07594496929, | |
| "Rank": 31 | |
| }, | |
| "Geometry": { | |
| "Average Score": 0.432216097, | |
| "Standard Deviation": 0.04747949254, | |
| "Rank": 31 | |
| }, | |
| "Algebra": { | |
| "Average Score": 0.478314888, | |
| "Standard Deviation": 0.01998797419, | |
| "Rank": 27 | |
| }, | |
| "Probability": { | |
| "Average Score": 0.427144725, | |
| "Standard Deviation": 0.0590923329, | |
| "Rank": 28 | |
| }, | |
| "Logical": { | |
| "Average Score": 0.340041983, | |
| "Standard Deviation": 0.008397574592, | |
| "Rank": 28 | |
| }, | |
| "Social": { | |
| "Average Score": 0.251949622, | |
| "Standard Deviation": 0.03346674405, | |
| "Rank": 37 | |
| } | |
| } | |
| }, | |
| { | |
| "config": { | |
| "model_name": "claude-2.0", | |
| "organization": "Anthropic", | |
| "license": "Proprietary", | |
| "knowledge_cutoff": "2023-10" | |
| }, | |
| "results": { | |
| "OVERALL": { | |
| "Average Score": 0.322718057, | |
| "Standard Deviation": 0.08369883584, | |
| "Rank": 38 | |
| }, | |
| "Geometry": { | |
| "Average Score": 0.604141967, | |
| "Standard Deviation": 0.05116441826, | |
| "Rank": 18 | |
| }, | |
| "Algebra": { | |
| "Average Score": 0.474350734, | |
| "Standard Deviation": 0.01510393066, | |
| "Rank": 28 | |
| }, | |
| "Probability": { | |
| "Average Score": 0.437950412, | |
| "Standard Deviation": 0.05985594317, | |
| "Rank": 27 | |
| }, | |
| "Logical": { | |
| "Average Score": 0.445620646, | |
| "Standard Deviation": 0.01812614805, | |
| "Rank": 24 | |
| }, | |
| "Social": { | |
| "Average Score": 0.469422836, | |
| "Standard Deviation": 0.05999901796, | |
| "Rank": 21 | |
| } | |
| } | |
| }, | |
| { | |
| "config": { | |
| "model_name": "starling-lm-7b-beta", | |
| "organization": "Nexusflow", | |
| "license": "Apache-2.0", | |
| "knowledge_cutoff": "2024-01" | |
| }, | |
| "results": { | |
| "OVERALL": { | |
| "Average Score": 0.479391856, | |
| "Standard Deviation": 0.04199990887, | |
| "Rank": 22 | |
| }, | |
| "Geometry": { | |
| "Average Score": 0.446654388, | |
| "Standard Deviation": 0.05637864999, | |
| "Rank": 30 | |
| }, | |
| "Algebra": { | |
| "Average Score": 0.473952749, | |
| "Standard Deviation": 0.01584301288, | |
| "Rank": 29 | |
| }, | |
| "Probability": { | |
| "Average Score": 0.395197837, | |
| "Standard Deviation": 0.05814798892, | |
| "Rank": 31 | |
| }, | |
| "Logical": { | |
| "Average Score": 0.39927199, | |
| "Standard Deviation": 0.02125277518, | |
| "Rank": 26 | |
| }, | |
| "Social": { | |
| "Average Score": 0.380021662, | |
| "Standard Deviation": 0.04622452748, | |
| "Rank": 27 | |
| } | |
| } | |
| }, | |
| { | |
| "config": { | |
| "model_name": "gemini-1.0-pro-001", | |
| "organization": "Google", | |
| "license": "Proprietary", | |
| "knowledge_cutoff": "2023-11" | |
| }, | |
| "results": { | |
| "OVERALL": { | |
| "Average Score": 0.449040654, | |
| "Standard Deviation": 0.0450610177, | |
| "Rank": 25 | |
| }, | |
| "Geometry": { | |
| "Average Score": 0.578347959, | |
| "Standard Deviation": 0.04242873607, | |
| "Rank": 21 | |
| }, | |
| "Algebra": { | |
| "Average Score": 0.462417786, | |
| "Standard Deviation": 0.01668313635, | |
| "Rank": 30 | |
| }, | |
| "Probability": { | |
| "Average Score": 0.289836324, | |
| "Standard Deviation": 0.05739831115, | |
| "Rank": 39 | |
| }, | |
| "Logical": { | |
| "Average Score": 0.191140355, | |
| "Standard Deviation": 0.03394652499, | |
| "Rank": 41 | |
| }, | |
| "Social": { | |
| "Average Score": 0.130790863, | |
| "Standard Deviation": 0.02800188173, | |
| "Rank": 47 | |
| } | |
| } | |
| }, | |
| { | |
| "config": { | |
| "model_name": "openchat-3.5-0106", | |
| "organization": "OpenChat", | |
| "license": "Apache-2.0", | |
| "knowledge_cutoff": "2024-01" | |
| }, | |
| "results": { | |
| "OVERALL": { | |
| "Average Score": 0.363929888, | |
| "Standard Deviation": 0.08602347145, | |
| "Rank": 34 | |
| }, | |
| "Geometry": { | |
| "Average Score": 0.38715246, | |
| "Standard Deviation": 0.03701851946, | |
| "Rank": 34 | |
| }, | |
| "Algebra": { | |
| "Average Score": 0.441233712, | |
| "Standard Deviation": 0.01135753754, | |
| "Rank": 31 | |
| }, | |
| "Probability": { | |
| "Average Score": 0.38802618, | |
| "Standard Deviation": 0.05663879714, | |
| "Rank": 32 | |
| }, | |
| "Logical": { | |
| "Average Score": 0.336754383, | |
| "Standard Deviation": 0.01608478079, | |
| "Rank": 29 | |
| }, | |
| "Social": { | |
| "Average Score": 0.250891608, | |
| "Standard Deviation": 0.03253769914, | |
| "Rank": 38 | |
| } | |
| } | |
| }, | |
| { | |
| "config": { | |
| "model_name": "openchat-3.5", | |
| "organization": "OpenChat", | |
| "license": "Apache-2.0", | |
| "knowledge_cutoff": "2023-12" | |
| }, | |
| "results": { | |
| "OVERALL": { | |
| "Average Score": 0.361341296, | |
| "Standard Deviation": 0.09034869493, | |
| "Rank": 35 | |
| }, | |
| "Geometry": { | |
| "Average Score": 0.401699069, | |
| "Standard Deviation": 0.03410726557, | |
| "Rank": 32 | |
| }, | |
| "Algebra": { | |
| "Average Score": 0.414095336, | |
| "Standard Deviation": 0.01881964261, | |
| "Rank": 33 | |
| }, | |
| "Probability": { | |
| "Average Score": 0.349601002, | |
| "Standard Deviation": 0.05077455539, | |
| "Rank": 34 | |
| }, | |
| "Logical": { | |
| "Average Score": 0.331069242, | |
| "Standard Deviation": 0.02180827173, | |
| "Rank": 31 | |
| }, | |
| "Social": { | |
| "Average Score": 0.319991655, | |
| "Standard Deviation": 0.04502478724, | |
| "Rank": 31 | |
| } | |
| } | |
| }, | |
| { | |
| "config": { | |
| "model_name": "command-r-(08-2024)", | |
| "organization": "Cohere", | |
| "license": "CC-BY-NC-4.0", | |
| "knowledge_cutoff": "2024-08" | |
| }, | |
| "results": { | |
| "OVERALL": { | |
| "Average Score": 0.427605298, | |
| "Standard Deviation": 0.01747449163, | |
| "Rank": 26 | |
| }, | |
| "Geometry": { | |
| "Average Score": 0.448300727, | |
| "Standard Deviation": 0.04996362328, | |
| "Rank": 29 | |
| }, | |
| "Algebra": { | |
| "Average Score": 0.417519167, | |
| "Standard Deviation": 0.01822196902, | |
| "Rank": 32 | |
| }, | |
| "Probability": { | |
| "Average Score": 0.366336281, | |
| "Standard Deviation": 0.04716826942, | |
| "Rank": 33 | |
| }, | |
| "Logical": { | |
| "Average Score": 0.214657906, | |
| "Standard Deviation": 0.03003579835, | |
| "Rank": 38 | |
| }, | |
| "Social": { | |
| "Average Score": 0.276088379, | |
| "Standard Deviation": 0.03295234688, | |
| "Rank": 34 | |
| } | |
| } | |
| }, | |
| { | |
| "config": { | |
| "model_name": "gemma-1.1-7b-it", | |
| "organization": "Google", | |
| "license": "Gemma License", | |
| "knowledge_cutoff": "2023-11" | |
| }, | |
| "results": { | |
| "OVERALL": { | |
| "Average Score": 0.339506922, | |
| "Standard Deviation": 0.1066279108, | |
| "Rank": 37 | |
| }, | |
| "Geometry": { | |
| "Average Score": 0.324170977, | |
| "Standard Deviation": 0.04668553765, | |
| "Rank": 37 | |
| }, | |
| "Algebra": { | |
| "Average Score": 0.398684697, | |
| "Standard Deviation": 0.01982398259, | |
| "Rank": 34 | |
| }, | |
| "Probability": { | |
| "Average Score": 0.293253175, | |
| "Standard Deviation": 0.05126192191, | |
| "Rank": 38 | |
| }, | |
| "Logical": { | |
| "Average Score": 0.317750796, | |
| "Standard Deviation": 0.01101933543, | |
| "Rank": 32 | |
| }, | |
| "Social": { | |
| "Average Score": 0.179073276, | |
| "Standard Deviation": 0.02009658805, | |
| "Rank": 43 | |
| } | |
| } | |
| }, | |
| { | |
| "config": { | |
| "model_name": "llama3-8b-instruct", | |
| "organization": "Meta", | |
| "license": "Llama 3 Community", | |
| "knowledge_cutoff": "2024-01" | |
| }, | |
| "results": { | |
| "OVERALL": { | |
| "Average Score": 0.367722676, | |
| "Standard Deviation": 0.1071368221, | |
| "Rank": 32 | |
| }, | |
| "Geometry": { | |
| "Average Score": 0.367143758, | |
| "Standard Deviation": 0.04363680358, | |
| "Rank": 35 | |
| }, | |
| "Algebra": { | |
| "Average Score": 0.391480973, | |
| "Standard Deviation": 0.02757445266, | |
| "Rank": 35 | |
| }, | |
| "Probability": { | |
| "Average Score": 0.317616445, | |
| "Standard Deviation": 0.04300430361, | |
| "Rank": 37 | |
| }, | |
| "Logical": { | |
| "Average Score": 0.461607495, | |
| "Standard Deviation": 0.02185028842, | |
| "Rank": 23 | |
| }, | |
| "Social": { | |
| "Average Score": 0.336373622, | |
| "Standard Deviation": 0.05762408512, | |
| "Rank": 28 | |
| } | |
| } | |
| }, | |
| { | |
| "config": { | |
| "model_name": "gemma-2-2b-it", | |
| "organization": "Google", | |
| "license": "Gemma License", | |
| "knowledge_cutoff": "2023-12" | |
| }, | |
| "results": { | |
| "OVERALL": { | |
| "Average Score": 0.502167612, | |
| "Standard Deviation": 0.04389786763, | |
| "Rank": 20 | |
| }, | |
| "Geometry": { | |
| "Average Score": 0.395006676, | |
| "Standard Deviation": 0.05882607713, | |
| "Rank": 33 | |
| }, | |
| "Algebra": { | |
| "Average Score": 0.379391887, | |
| "Standard Deviation": 0.01722410785, | |
| "Rank": 36 | |
| }, | |
| "Probability": { | |
| "Average Score": 0.331231097, | |
| "Standard Deviation": 0.05392499987, | |
| "Rank": 36 | |
| }, | |
| "Logical": { | |
| "Average Score": 0.367687789, | |
| "Standard Deviation": 0.02547968808, | |
| "Rank": 27 | |
| }, | |
| "Social": { | |
| "Average Score": 0.393482094, | |
| "Standard Deviation": 0.06450214024, | |
| "Rank": 25 | |
| } | |
| } | |
| }, | |
| { | |
| "config": { | |
| "model_name": "starling-lm-7b-alpha", | |
| "organization": "Nexusflow", | |
| "license": "Apache-2.0", | |
| "knowledge_cutoff": "2023-12" | |
| }, | |
| "results": { | |
| "OVERALL": { | |
| "Average Score": 0.366628765, | |
| "Standard Deviation": 0.08405492929, | |
| "Rank": 33 | |
| }, | |
| "Geometry": { | |
| "Average Score": 0.336782578, | |
| "Standard Deviation": 0.04069449132, | |
| "Rank": 36 | |
| }, | |
| "Algebra": { | |
| "Average Score": 0.371551932, | |
| "Standard Deviation": 0.03367241745, | |
| "Rank": 37 | |
| }, | |
| "Probability": { | |
| "Average Score": 0.331472505, | |
| "Standard Deviation": 0.04833324282, | |
| "Rank": 35 | |
| }, | |
| "Logical": { | |
| "Average Score": 0.260869624, | |
| "Standard Deviation": 0.03562735237, | |
| "Rank": 36 | |
| }, | |
| "Social": { | |
| "Average Score": 0.271975534, | |
| "Standard Deviation": 0.04266753408, | |
| "Rank": 35 | |
| } | |
| } | |
| }, | |
| { | |
| "config": { | |
| "model_name": "qwen1.5-4b-chat", | |
| "organization": "Alibaba", | |
| "license": "Qianwen LICENSE", | |
| "knowledge_cutoff": "2024-02" | |
| }, | |
| "results": { | |
| "OVERALL": { | |
| "Average Score": 0.111876411, | |
| "Standard Deviation": 0.04241022785, | |
| "Rank": 49 | |
| }, | |
| "Geometry": { | |
| "Average Score": 0.215834522, | |
| "Standard Deviation": 0.0363766363, | |
| "Rank": 41 | |
| }, | |
| "Algebra": { | |
| "Average Score": 0.305589811, | |
| "Standard Deviation": 0.02354198912, | |
| "Rank": 38 | |
| }, | |
| "Probability": { | |
| "Average Score": 0.149365327, | |
| "Standard Deviation": 0.03489672675, | |
| "Rank": 45 | |
| }, | |
| "Logical": { | |
| "Average Score": 0.116210168, | |
| "Standard Deviation": 0.005927966496, | |
| "Rank": 47 | |
| }, | |
| "Social": { | |
| "Average Score": 0.18195615, | |
| "Standard Deviation": 0.02269805277, | |
| "Rank": 42 | |
| } | |
| } | |
| }, | |
| { | |
| "config": { | |
| "model_name": "command-r-(04-2024)", | |
| "organization": "Cohere", | |
| "license": "CC-BY-NC-4.0", | |
| "knowledge_cutoff": "2024-04" | |
| }, | |
| "results": { | |
| "OVERALL": { | |
| "Average Score": 0.388783887, | |
| "Standard Deviation": 0.07417186783, | |
| "Rank": 30 | |
| }, | |
| "Geometry": { | |
| "Average Score": 0.300416698, | |
| "Standard Deviation": 0.03485612736, | |
| "Rank": 38 | |
| }, | |
| "Algebra": { | |
| "Average Score": 0.293120231, | |
| "Standard Deviation": 0.032926484, | |
| "Rank": 39 | |
| }, | |
| "Probability": { | |
| "Average Score": 0.281271304, | |
| "Standard Deviation": 0.05697149867, | |
| "Rank": 40 | |
| }, | |
| "Logical": { | |
| "Average Score": 0.276189906, | |
| "Standard Deviation": 0.03562914754, | |
| "Rank": 34 | |
| }, | |
| "Social": { | |
| "Average Score": 0.283882949, | |
| "Standard Deviation": 0.03336901148, | |
| "Rank": 33 | |
| } | |
| } | |
| }, | |
| { | |
| "config": { | |
| "model_name": "vicuna-33b", | |
| "organization": "LMSYS", | |
| "license": "Non-commercial", | |
| "knowledge_cutoff": "2023-12" | |
| }, | |
| "results": { | |
| "OVERALL": { | |
| "Average Score": 0.316543555, | |
| "Standard Deviation": 0.08922095647, | |
| "Rank": 39 | |
| }, | |
| "Geometry": { | |
| "Average Score": 0.208284679, | |
| "Standard Deviation": 0.03937771461, | |
| "Rank": 42 | |
| }, | |
| "Algebra": { | |
| "Average Score": 0.248994048, | |
| "Standard Deviation": 0.02668175054, | |
| "Rank": 41 | |
| }, | |
| "Probability": { | |
| "Average Score": 0.222313995, | |
| "Standard Deviation": 0.03978859759, | |
| "Rank": 43 | |
| }, | |
| "Logical": { | |
| "Average Score": 0.180291222, | |
| "Standard Deviation": 0.021886267, | |
| "Rank": 42 | |
| }, | |
| "Social": { | |
| "Average Score": 0.257623798, | |
| "Standard Deviation": 0.02653724437, | |
| "Rank": 36 | |
| } | |
| } | |
| }, | |
| { | |
| "config": { | |
| "model_name": "gemma-7b-it", | |
| "organization": "Google", | |
| "license": "Gemma License", | |
| "knowledge_cutoff": "2023-12" | |
| }, | |
| "results": { | |
| "OVERALL": { | |
| "Average Score": 0.285077558, | |
| "Standard Deviation": 0.08871758453, | |
| "Rank": 41 | |
| }, | |
| "Geometry": { | |
| "Average Score": 0.244791417, | |
| "Standard Deviation": 0.0289612078, | |
| "Rank": 39 | |
| }, | |
| "Algebra": { | |
| "Average Score": 0.250614794, | |
| "Standard Deviation": 0.01991678295, | |
| "Rank": 40 | |
| }, | |
| "Probability": { | |
| "Average Score": 0.174313053, | |
| "Standard Deviation": 0.03765424728, | |
| "Rank": 44 | |
| }, | |
| "Logical": { | |
| "Average Score": 0.197505536, | |
| "Standard Deviation": 0.02050298885, | |
| "Rank": 39 | |
| }, | |
| "Social": { | |
| "Average Score": 0.202138025, | |
| "Standard Deviation": 0.02098346639, | |
| "Rank": 41 | |
| } | |
| } | |
| }, | |
| { | |
| "config": { | |
| "model_name": "mistral-7b-instruct-2", | |
| "organization": "Mistral", | |
| "license": "Apache 2.0", | |
| "knowledge_cutoff": "2023-12" | |
| }, | |
| "results": { | |
| "OVERALL": { | |
| "Average Score": 0.427513868, | |
| "Standard Deviation": 0.05553921135, | |
| "Rank": 27 | |
| }, | |
| "Geometry": { | |
| "Average Score": 0.216402626, | |
| "Standard Deviation": 0.03338414918, | |
| "Rank": 40 | |
| }, | |
| "Algebra": { | |
| "Average Score": 0.233777838, | |
| "Standard Deviation": 0.0155226054, | |
| "Rank": 42 | |
| }, | |
| "Probability": { | |
| "Average Score": 0.25118175, | |
| "Standard Deviation": 0.04065514593, | |
| "Rank": 41 | |
| }, | |
| "Logical": { | |
| "Average Score": 0.224469136, | |
| "Standard Deviation": 0.03404706752, | |
| "Rank": 37 | |
| }, | |
| "Social": { | |
| "Average Score": 0.209386782, | |
| "Standard Deviation": 0.02738569921, | |
| "Rank": 40 | |
| } | |
| } | |
| }, | |
| { | |
| "config": { | |
| "model_name": "mistral-7b-instruct-1", | |
| "organization": "Mistral", | |
| "license": "Apache 2.0", | |
| "knowledge_cutoff": "2023-12" | |
| }, | |
| "results": { | |
| "OVERALL": { | |
| "Average Score": 0.23016314, | |
| "Standard Deviation": 0.07137625271, | |
| "Rank": 46 | |
| }, | |
| "Geometry": { | |
| "Average Score": 0.161799938, | |
| "Standard Deviation": 0.03595278559, | |
| "Rank": 46 | |
| }, | |
| "Algebra": { | |
| "Average Score": 0.210341624, | |
| "Standard Deviation": 0.01736539119, | |
| "Rank": 43 | |
| }, | |
| "Probability": { | |
| "Average Score": 0.238417922, | |
| "Standard Deviation": 0.03744211933, | |
| "Rank": 42 | |
| }, | |
| "Logical": { | |
| "Average Score": 0.142636601, | |
| "Standard Deviation": 0.02080406365, | |
| "Rank": 46 | |
| }, | |
| "Social": { | |
| "Average Score": 0.117646827, | |
| "Standard Deviation": 0.009321202779, | |
| "Rank": 49 | |
| } | |
| } | |
| }, | |
| { | |
| "config": { | |
| "model_name": "vicuna-13b", | |
| "organization": "LMSYS", | |
| "license": "Non-commercial", | |
| "knowledge_cutoff": "2023-11" | |
| }, | |
| "results": { | |
| "OVERALL": { | |
| "Average Score": 0.201892849, | |
| "Standard Deviation": 0.06021749802, | |
| "Rank": 47 | |
| }, | |
| "Geometry": { | |
| "Average Score": 0.200941928, | |
| "Standard Deviation": 0.03366817781, | |
| "Rank": 43 | |
| }, | |
| "Algebra": { | |
| "Average Score": 0.196123323, | |
| "Standard Deviation": 0.0135715643, | |
| "Rank": 44 | |
| }, | |
| "Probability": { | |
| "Average Score": 0.141214079, | |
| "Standard Deviation": 0.02721328211, | |
| "Rank": 46 | |
| }, | |
| "Logical": { | |
| "Average Score": 0.148598631, | |
| "Standard Deviation": 0.02241523892, | |
| "Rank": 44 | |
| }, | |
| "Social": { | |
| "Average Score": 0.124655135, | |
| "Standard Deviation": 0.01122382671, | |
| "Rank": 48 | |
| } | |
| } | |
| }, | |
| { | |
| "config": { | |
| "model_name": "zephyr-7b-beta", | |
| "organization": "HuggingFace", | |
| "license": "MIT", | |
| "knowledge_cutoff": "2023-10" | |
| }, | |
| "results": { | |
| "OVERALL": { | |
| "Average Score": 0.102705119, | |
| "Standard Deviation": 0.03683757312, | |
| "Rank": 50 | |
| }, | |
| "Geometry": { | |
| "Average Score": 0.114005544, | |
| "Standard Deviation": 0.03144354365, | |
| "Rank": 47 | |
| }, | |
| "Algebra": { | |
| "Average Score": 0.141766633, | |
| "Standard Deviation": 0.03179520129, | |
| "Rank": 45 | |
| }, | |
| "Probability": { | |
| "Average Score": 0.089050714, | |
| "Standard Deviation": 0.002136754266, | |
| "Rank": 49 | |
| }, | |
| "Logical": { | |
| "Average Score": 0.069520789, | |
| "Standard Deviation": 0.004477840857, | |
| "Rank": 51 | |
| }, | |
| "Social": { | |
| "Average Score": 0.0, | |
| "Standard Deviation": 0.0, | |
| "Rank": 54 | |
| } | |
| } | |
| }, | |
| { | |
| "config": { | |
| "model_name": "gemma-1.1-2b-it", | |
| "organization": "Google", | |
| "license": "Gemma License", | |
| "knowledge_cutoff": "2023-12" | |
| }, | |
| "results": { | |
| "OVERALL": { | |
| "Average Score": 0.257700845, | |
| "Standard Deviation": 0.07369021445, | |
| "Rank": 44 | |
| }, | |
| "Geometry": { | |
| "Average Score": 0.183974034, | |
| "Standard Deviation": 0.0215548886, | |
| "Rank": 45 | |
| }, | |
| "Algebra": { | |
| "Average Score": 0.13422252, | |
| "Standard Deviation": 0.01922819511, | |
| "Rank": 46 | |
| }, | |
| "Probability": { | |
| "Average Score": 0.095628657, | |
| "Standard Deviation": 0.007536076456, | |
| "Rank": 48 | |
| }, | |
| "Logical": { | |
| "Average Score": 0.094965074, | |
| "Standard Deviation": 0.005019175487, | |
| "Rank": 49 | |
| }, | |
| "Social": { | |
| "Average Score": 0.167796727, | |
| "Standard Deviation": 0.01666541942, | |
| "Rank": 44 | |
| } | |
| } | |
| }, | |
| { | |
| "config": { | |
| "model_name": "llama2-7b-chat", | |
| "organization": "Meta", | |
| "license": "Llama 2 Community", | |
| "knowledge_cutoff": "2023-10" | |
| }, | |
| "results": { | |
| "OVERALL": { | |
| "Average Score": 0.260189428, | |
| "Standard Deviation": 0.08019299364, | |
| "Rank": 43 | |
| }, | |
| "Geometry": { | |
| "Average Score": 0.087067276, | |
| "Standard Deviation": 0.04274343402, | |
| "Rank": 48 | |
| }, | |
| "Algebra": { | |
| "Average Score": 0.12308805, | |
| "Standard Deviation": 0.01856053622, | |
| "Rank": 47 | |
| }, | |
| "Probability": { | |
| "Average Score": 0.087515438, | |
| "Standard Deviation": 0.006315053573, | |
| "Rank": 50 | |
| }, | |
| "Logical": { | |
| "Average Score": 0.17312827, | |
| "Standard Deviation": 0.01867044092, | |
| "Rank": 43 | |
| }, | |
| "Social": { | |
| "Average Score": 0.152905272, | |
| "Standard Deviation": 0.007166957097, | |
| "Rank": 45 | |
| } | |
| } | |
| }, | |
| { | |
| "config": { | |
| "model_name": "gemma-2b-it", | |
| "organization": "Google", | |
| "license": "Gemma License", | |
| "knowledge_cutoff": "2023-11" | |
| }, | |
| "results": { | |
| "OVERALL": { | |
| "Average Score": 0.234172069, | |
| "Standard Deviation": 0.06522685718, | |
| "Rank": 45 | |
| }, | |
| "Geometry": { | |
| "Average Score": 0.198571153, | |
| "Standard Deviation": 0.01699161031, | |
| "Rank": 44 | |
| }, | |
| "Algebra": { | |
| "Average Score": 0.109883009, | |
| "Standard Deviation": 0.01520005833, | |
| "Rank": 48 | |
| }, | |
| "Probability": { | |
| "Average Score": 0.06467432, | |
| "Standard Deviation": 0.002117497231, | |
| "Rank": 52 | |
| }, | |
| "Logical": { | |
| "Average Score": 0.039624492, | |
| "Standard Deviation": 0.007606972686, | |
| "Rank": 52 | |
| }, | |
| "Social": { | |
| "Average Score": 0.087452913, | |
| "Standard Deviation": 0.008170146562, | |
| "Rank": 52 | |
| } | |
| } | |
| }, | |
| { | |
| "config": { | |
| "model_name": "llama2-13b-chat", | |
| "organization": "Meta", | |
| "license": "Llama 2 Community", | |
| "knowledge_cutoff": "2023-12" | |
| }, | |
| "results": { | |
| "OVERALL": { | |
| "Average Score": 0.263305684, | |
| "Standard Deviation": 0.07283640689, | |
| "Rank": 42 | |
| }, | |
| "Geometry": { | |
| "Average Score": 0.072729954, | |
| "Standard Deviation": 0.02315988261, | |
| "Rank": 50 | |
| }, | |
| "Algebra": { | |
| "Average Score": 0.080371692, | |
| "Standard Deviation": 0.01277569453, | |
| "Rank": 49 | |
| }, | |
| "Probability": { | |
| "Average Score": 0.117757344, | |
| "Standard Deviation": 0.02418619619, | |
| "Rank": 47 | |
| }, | |
| "Logical": { | |
| "Average Score": 0.193149889, | |
| "Standard Deviation": 0.01776690764, | |
| "Rank": 40 | |
| }, | |
| "Social": { | |
| "Average Score": 0.149125922, | |
| "Standard Deviation": 0.01157416827, | |
| "Rank": 46 | |
| } | |
| } | |
| }, | |
| { | |
| "config": { | |
| "model_name": "vicuna-7b", | |
| "organization": "LMSYS", | |
| "license": "Non-commercial", | |
| "knowledge_cutoff": "2023-11" | |
| }, | |
| "results": { | |
| "OVERALL": { | |
| "Average Score": 0.198839786, | |
| "Standard Deviation": 0.05725381576, | |
| "Rank": 48 | |
| }, | |
| "Geometry": { | |
| "Average Score": 0.083457058, | |
| "Standard Deviation": 0.02520989111, | |
| "Rank": 49 | |
| }, | |
| "Algebra": { | |
| "Average Score": 0.070883882, | |
| "Standard Deviation": 0.007315853253, | |
| "Rank": 50 | |
| }, | |
| "Probability": { | |
| "Average Score": 0.080987673, | |
| "Standard Deviation": 0.005474288861, | |
| "Rank": 51 | |
| }, | |
| "Logical": { | |
| "Average Score": 0.100065588, | |
| "Standard Deviation": 0.003561886452, | |
| "Rank": 48 | |
| }, | |
| "Social": { | |
| "Average Score": 0.111076414, | |
| "Standard Deviation": 0.004805626512, | |
| "Rank": 50 | |
| } | |
| } | |
| }, | |
| { | |
| "config": { | |
| "model_name": "koala-13b", | |
| "organization": "UC Berkeley", | |
| "license": "Non-commercial", | |
| "knowledge_cutoff": "2023-10" | |
| }, | |
| "results": { | |
| "OVERALL": { | |
| "Average Score": 0.09387188, | |
| "Standard Deviation": 0.02642167489, | |
| "Rank": 51 | |
| }, | |
| "Geometry": { | |
| "Average Score": 0.017374001, | |
| "Standard Deviation": 0.01747053557, | |
| "Rank": 51 | |
| }, | |
| "Algebra": { | |
| "Average Score": 0.018129197, | |
| "Standard Deviation": 0.01054371383, | |
| "Rank": 51 | |
| }, | |
| "Probability": { | |
| "Average Score": 0.043654362, | |
| "Standard Deviation": 0.004288231886, | |
| "Rank": 53 | |
| }, | |
| "Logical": { | |
| "Average Score": 0.074694053, | |
| "Standard Deviation": 0.002674646998, | |
| "Rank": 50 | |
| }, | |
| "Social": { | |
| "Average Score": 0.096983835, | |
| "Standard Deviation": 0.007847059783, | |
| "Rank": 51 | |
| } | |
| } | |
| }, | |
| { | |
| "config": { | |
| "model_name": "openassistant-pythia-12b", | |
| "organization": "OpenAssistant", | |
| "license": "Non-commercial", | |
| "knowledge_cutoff": "2023-09" | |
| }, | |
| "results": { | |
| "OVERALL": { | |
| "Average Score": 0.0, | |
| "Standard Deviation": 0.0, | |
| "Rank": 52 | |
| }, | |
| "Geometry": { | |
| "Average Score": 0.0, | |
| "Standard Deviation": 0.0, | |
| "Rank": 52 | |
| }, | |
| "Algebra": { | |
| "Average Score": 0.0, | |
| "Standard Deviation": 0.0, | |
| "Rank": 52 | |
| }, | |
| "Probability": { | |
| "Average Score": 0.0, | |
| "Standard Deviation": 0.0, | |
| "Rank": 54 | |
| }, | |
| "Logical": { | |
| "Average Score": 0.0, | |
| "Standard Deviation": 0.0, | |
| "Rank": 53 | |
| }, | |
| "Social": { | |
| "Average Score": 0.030792528, | |
| "Standard Deviation": 0.007518796391, | |
| "Rank": 53 | |
| } | |
| } | |
| } | |
| ] |