StructEval_leaderboard / STRUCT_RES.json
Bowieee's picture
add models
1f868cb
raw
history blame
7.87 kB
{
"deepseek-v2-lite-chat": {
"BaseARC": 69.01287553648069,
"BaseOpenbook": 64.60000000000001,
"📝StructARC": 40.155538694992416,
"📝StructOpenbook": 41.98437130002368,
"📝StructMMLU": 40.95659182440028,
"📝StructAverage": 41.03216727313879,
"BaseMMLU": 55.7,
"BaseAverage": 63.10429184549357
},
"baichuan2-7b-chat": {
"BaseARC": 57.596566523605155,
"BaseOpenbook": 60.8,
"📝StructARC": 31.39226100151745,
"📝StructOpenbook": 33.838503433578026,
"📝StructMMLU": 32.019719285283976,
"📝StructAverage": 32.41682790679315,
"BaseMMLU": 52.49,
"BaseAverage": 56.962188841201716
},
"mistral-7b-instruct-v0.3": {
"BaseARC": 76.39484978540773,
"BaseOpenbook": 74.4,
"📝StructARC": 42.640364188163886,
"📝StructOpenbook": 43.6182808430026,
"📝StructMMLU": 41.756990621302315,
"📝StructAverage": 42.67187855082293,
"BaseMMLU": 61.63,
"BaseAverage": 70.80828326180257
},
"yi-6b-chat": {
"BaseARC": 59.227467811158796,
"BaseOpenbook": 55.60000000000001,
"📝StructARC": 38.39150227617603,
"📝StructOpenbook": 38.977030547004496,
"📝StructMMLU": 36.34025031348594,
"📝StructAverage": 37.90292771222215,
"BaseMMLU": 58.24,
"BaseAverage": 57.68915593705293
},
"yi-1.5-9b-chat": {
"BaseARC": 85.75107296137338,
"BaseOpenbook": 83.0,
"📝StructARC": 57.01820940819423,
"📝StructOpenbook": 54.53469097797774,
"📝StructMMLU": 53.431308242228596,
"📝StructAverage": 54.99473620946685,
"BaseMMLU": 69.5,
"BaseAverage": 79.4170243204578
},
"llama-2-7b-chat": {
"BaseARC": 53.648068669527895,
"BaseOpenbook": 57.99999999999999,
"📝StructARC": 29.429059180576633,
"📝StructOpenbook": 33.24650722235378,
"📝StructMMLU": 27.884038239302765,
"📝StructAverage": 30.186534880744393,
"BaseMMLU": 34.1,
"BaseAverage": 48.5826895565093
},
"llama-3-8b-instruct": {
"BaseARC": 80.68669527896995,
"BaseOpenbook": 79.60000000000001,
"📝StructARC": 56.74317147192717,
"📝StructOpenbook": 57.06843476201753,
"📝StructMMLU": 54.63728961686302,
"📝StructAverage": 56.149631950269246,
"BaseMMLU": 68.4,
"BaseAverage": 76.22889842632333
},
"qwen1.5-7b-chat": {
"BaseARC": 70.21459227467811,
"BaseOpenbook": 68.60000000000001,
"📝StructARC": 40.92374810318665,
"📝StructOpenbook": 41.439734785697375,
"📝StructMMLU": 39.75951206275273,
"📝StructAverage": 40.707664983878914,
"BaseMMLU": 61.70508772,
"BaseAverage": 66.83989333155938
},
"qwen2-7b-instruct": {
"BaseARC": 84.29184549356223,
"BaseOpenbook": 82.39999999999999,
"📝StructARC": 58.34597875569044,
"📝StructOpenbook": 57.28155339805825,
"📝StructMMLU": 56.78738087582798,
"📝StructAverage": 57.47163767652555,
"BaseMMLU": 70.5,
"BaseAverage": 79.06394849785407
},
"mistral-7b-v0.3": {
"BaseARC": 76.65236051502146,
"BaseOpenbook": 72.8,
"📝StructARC": 46.936646433990894,
"📝StructOpenbook": 48.23585129055174,
"📝StructMMLU": 46.05517926584278,
"📝StructAverage": 47.075892330128475,
"BaseMMLU": 60.1,
"BaseAverage": 69.85078683834048
},
"yi-1.5-9b": {
"BaseARC": 88.41201716738198,
"BaseOpenbook": 83.0,
"📝StructARC": 60.60318664643399,
"📝StructOpenbook": 59.2943405162207,
"📝StructMMLU": 57.41526232409641,
"📝StructAverage": 59.10426316225036,
"BaseMMLU": 69.5,
"BaseAverage": 80.30400572246066
},
"yi-6b": {
"BaseARC": 77.6824034334764,
"BaseOpenbook": 73.6,
"📝StructARC": 46.936646433990894,
"📝StructOpenbook": 49.51456310679612,
"📝StructMMLU": 46.21888085492341,
"📝StructAverage": 47.55669679857014,
"BaseMMLU": 63.2,
"BaseAverage": 71.49413447782545
},
"deepseek-v2-lite": {
"BaseARC": 68.24034334763948,
"BaseOpenbook": 66.4,
"📝StructARC": 40.73406676783004,
"📝StructOpenbook": 42.97892493488042,
"📝StructMMLU": 41.024161265238256,
"📝StructAverage": 41.579050989316244,
"BaseMMLU": 58.3,
"BaseAverage": 64.3134477825465
},
"llama-3-8b": {
"BaseARC": 76.22317596566523,
"BaseOpenbook": 75.0,
"📝StructARC": 52.78831562974203,
"📝StructOpenbook": 53.04286052569264,
"📝StructMMLU": 51.37044261097523,
"📝StructAverage": 52.4005395888033,
"BaseMMLU": 66.6,
"BaseAverage": 72.60772532188841
},
"llama-2-7b": {
"BaseARC": 48.66952789699571,
"BaseOpenbook": 43.6,
"📝StructARC": 29.040212443095598,
"📝StructOpenbook": 31.730996921619703,
"📝StructMMLU": 29.187423497033315,
"📝StructAverage": 29.986210953916203,
"BaseMMLU": 45.7,
"BaseAverage": 45.98984263233191
},
"qwen2-7b": {
"BaseARC": 86.18025751072962,
"BaseOpenbook": 84.2,
"📝StructARC": 66.17033383915023,
"📝StructOpenbook": 66.13781671797301,
"📝StructMMLU": 65.24702121505192,
"📝StructAverage": 65.85172392405839,
"BaseMMLU": 70.3,
"BaseAverage": 80.22675250357655
},
"qwen1.5-7b": {
"BaseARC": 78.54077253218884,
"BaseOpenbook": 78.8,
"📝StructARC": 48.501517450682854,
"📝StructOpenbook": 50.248638408714186,
"📝StructMMLU": 46.709001366105916,
"📝StructAverage": 48.48638574183432,
"BaseMMLU": 61.0,
"BaseAverage": 72.78025751072961
},
"baichuan2-7b-base": {
"BaseARC": 61.630901287553655,
"BaseOpenbook": 62.6,
"📝StructARC": 32.42602427921092,
"📝StructOpenbook": 37.08264267108691,
"📝StructMMLU": 34.59571358681649,
"📝StructAverage": 34.7014601790381,
"BaseMMLU": 54.16,
"BaseAverage": 59.46363376251788
},
"llama-3-70b": {
"BaseARC": 92.1030042918455,
"BaseOpenbook": 89.60000000000001,
"📝StructARC": 73.12215477996965,
"📝StructOpenbook": 72.6024153445418,
"📝StructMMLU": 70.85211858440155,
"📝StructAverage": 72.19222956963766,
"BaseMMLU": 79.5,
"BaseAverage": 87.06766809728184
},
"qwen2-72b": {
"BaseARC": 95.79399141630901,
"BaseOpenbook": 96.39999999999999,
"📝StructARC": 79.10660091047042,
"📝StructOpenbook": 78.4986976083353,
"📝StructMMLU": 78.57972263190929,
"📝StructAverage": 78.72834038357168,
"BaseMMLU": 84.2,
"BaseAverage": 92.131330472103
},
"yi-1.5-34b": {
"BaseARC": 92.1030042918455,
"BaseOpenbook": 83.6,
"📝StructARC": 64.37784522003035,
"📝StructOpenbook": 63.76983187307601,
"📝StructMMLU": 63.296635991674755,
"📝StructAverage": 63.81477102826037,
"BaseMMLU": 77.1,
"BaseAverage": 84.26766809728183
},
"mixtral-8x7b-v0.1": {
"BaseARC": 84.0343347639485,
"BaseOpenbook": 77.8,
"📝StructARC": 62.3103186646434,
"📝StructOpenbook": 62.32536111768885,
"📝StructMMLU": 60.46643846946602,
"📝StructAverage": 61.70070608393275,
"BaseMMLU": 70.6,
"BaseAverage": 77.47811158798282
}
}