| {"vidore/arxivqa_test_subsampled": {"ndcg_at_1": 0.24, "ndcg_at_3": 0.30264, "ndcg_at_5": 0.32226, "ndcg_at_10": 0.34454, "ndcg_at_20": 0.36508, "ndcg_at_50": 0.39549, "ndcg_at_100": 0.41211, "map_at_1": 0.24, "map_at_3": 0.287, "map_at_5": 0.2978, "map_at_10": 0.30678, "map_at_20": 0.31232, "map_at_50": 0.31713, "map_at_100": 0.31862, "recall_at_1": 0.24, "recall_at_3": 0.348, "recall_at_5": 0.396, "recall_at_10": 0.466, "recall_at_20": 0.548, "recall_at_50": 0.702, "recall_at_100": 0.804, "precision_at_1": 0.24, "precision_at_3": 0.116, "precision_at_5": 0.0792, "precision_at_10": 0.0466, "precision_at_20": 0.0274, "precision_at_50": 0.01404, "precision_at_100": 0.00804, "mrr_at_1": 0.24, "mrr_at_3": 0.28699999999999987, "mrr_at_5": 0.2977999999999999, "mrr_at_10": 0.3067817460317459, "mrr_at_20": 0.3123201421041976, "mrr_at_50": 0.3171343380495356, "mrr_at_100": 0.31862291104808416, "naucs_at_1_max": 0.18685897435897442, "naucs_at_1_std": -0.03619997603642465, "naucs_at_1_diff1": 0.5894919722022526, "naucs_at_3_max": 0.19260514493556943, "naucs_at_3_std": -0.0006994705140184183, "naucs_at_3_diff1": 0.48152078087304456, "naucs_at_5_max": 0.19514736912703196, "naucs_at_5_std": 0.0835977361955909, "naucs_at_5_diff1": 0.45112029427393674, "naucs_at_10_max": 0.15856905464801813, "naucs_at_10_std": 0.13505705504831184, "naucs_at_10_diff1": 0.4046128604949425, "naucs_at_20_max": 0.1260165563470669, "naucs_at_20_std": 0.1269777660701378, "naucs_at_20_diff1": 0.33776853621490766, "naucs_at_50_max": 0.2474828172157551, "naucs_at_50_std": 0.2220954120063917, "naucs_at_50_diff1": 0.37166262685550533, "naucs_at_100_max": 0.27522954457542, "naucs_at_100_std": 0.3287271727844628, "naucs_at_100_diff1": 0.36917486105173225}, "vidore/docvqa_test_subsampled": {"ndcg_at_1": 0.1663, "ndcg_at_3": 0.22986, "ndcg_at_5": 0.25352, "ndcg_at_10": 0.27125, "ndcg_at_20": 0.29253, "ndcg_at_50": 0.31695, "ndcg_at_100": 0.33526, "map_at_1": 0.1663, "map_at_3": 0.2136, "map_at_5": 0.22668, "map_at_10": 0.2339, "map_at_20": 0.23971, "map_at_50": 0.24372, "map_at_100": 0.24532, "recall_at_1": 0.1663, "recall_at_3": 0.27716, "recall_at_5": 0.33481, "recall_at_10": 0.39024, "recall_at_20": 0.4745, "recall_at_50": 0.59645, "recall_at_100": 0.70953, "precision_at_1": 0.1663, "precision_at_3": 0.09239, "precision_at_5": 0.06696, "precision_at_10": 0.03902, "precision_at_20": 0.02373, "precision_at_50": 0.01193, "precision_at_100": 0.0071, "mrr_at_1": 0.1662971175166297, "mrr_at_3": 0.21359940872135988, "mrr_at_5": 0.22668144863266812, "mrr_at_10": 0.2338973357266039, "mrr_at_20": 0.23971089894296316, "mrr_at_50": 0.2437175936453932, "mrr_at_100": 0.24532124584713808, "naucs_at_1_max": 0.2547631225692791, "naucs_at_1_std": 0.13049343645241313, "naucs_at_1_diff1": 0.538291120319166, "naucs_at_3_max": 0.2401158916208586, "naucs_at_3_std": 0.19988514213347763, "naucs_at_3_diff1": 0.4377698835403377, "naucs_at_5_max": 0.2493997320538922, "naucs_at_5_std": 0.21655685411704065, "naucs_at_5_diff1": 0.39081639455117123, "naucs_at_10_max": 0.21421016342215293, "naucs_at_10_std": 0.22331920645461106, "naucs_at_10_diff1": 0.35864044100130466, "naucs_at_20_max": 0.22430875035725428, "naucs_at_20_std": 0.22595058231333293, "naucs_at_20_diff1": 0.31094961052549774, "naucs_at_50_max": 0.1582606570694429, "naucs_at_50_std": 0.2521059555061037, "naucs_at_50_diff1": 0.273160388069171, "naucs_at_100_max": 0.1327654977581912, "naucs_at_100_std": 0.3168565883010314, "naucs_at_100_diff1": 0.281595139671596}, "vidore/infovqa_test_subsampled": {"ndcg_at_1": 0.56275, "ndcg_at_3": 0.65975, "ndcg_at_5": 0.68083, "ndcg_at_10": 0.70284, "ndcg_at_20": 0.71717, "ndcg_at_50": 0.72413, "ndcg_at_100": 0.72743, "map_at_1": 0.56275, "map_at_3": 0.63529, "map_at_5": 0.64713, "map_at_10": 0.65606, "map_at_20": 0.66, "map_at_50": 0.66118, "map_at_100": 0.66147, "recall_at_1": 0.56275, "recall_at_3": 0.73077, "recall_at_5": 0.78138, "recall_at_10": 0.8502, "recall_at_20": 0.90688, "recall_at_50": 0.9413, "recall_at_100": 0.96154, "precision_at_1": 0.56275, "precision_at_3": 0.24359, "precision_at_5": 0.15628, "precision_at_10": 0.08502, "precision_at_20": 0.04534, "precision_at_50": 0.01883, "precision_at_100": 0.00962, "mrr_at_1": 0.562753036437247, "mrr_at_3": 0.6352901484480431, "mrr_at_5": 0.647132253711201, "mrr_at_10": 0.6560640383008801, "mrr_at_20": 0.6599995976731192, "mrr_at_50": 0.6611775859328874, "mrr_at_100": 0.6614711981490079, "naucs_at_1_max": 0.4119424574346777, "naucs_at_1_std": -0.0003033005512287719, "naucs_at_1_diff1": 0.7262130152814296, "naucs_at_3_max": 0.34095211366176026, "naucs_at_3_std": 0.001497286592228783, "naucs_at_3_diff1": 0.6287695044119231, "naucs_at_5_max": 0.30095593564308837, "naucs_at_5_std": 0.04093377238359434, "naucs_at_5_diff1": 0.5992965272729808, "naucs_at_10_max": 0.3218669741560863, "naucs_at_10_std": 0.07964729330138849, "naucs_at_10_diff1": 0.5651943472652733, "naucs_at_20_max": 0.370091823338698, "naucs_at_20_std": 0.362922612760579, "naucs_at_20_diff1": 0.5145446928306314, "naucs_at_50_max": 0.38180297907997424, "naucs_at_50_std": 0.5021790461893741, "naucs_at_50_diff1": 0.48745018330010387, "naucs_at_100_max": 0.4312402149606384, "naucs_at_100_std": 0.6976206389014427, "naucs_at_100_diff1": 0.514062818533649}, "vidore/tabfquad_test_subsampled": {"ndcg_at_1": 0.30357, "ndcg_at_3": 0.37861, "ndcg_at_5": 0.41274, "ndcg_at_10": 0.4481, "ndcg_at_20": 0.48262, "ndcg_at_50": 0.51796, "ndcg_at_100": 0.52886, "map_at_1": 0.30357, "map_at_3": 0.36012, "map_at_5": 0.37923, "map_at_10": 0.39424, "map_at_20": 0.40382, "map_at_50": 0.40946, "map_at_100": 0.41056, "recall_at_1": 0.30357, "recall_at_3": 0.43214, "recall_at_5": 0.51429, "recall_at_10": 0.62143, "recall_at_20": 0.75714, "recall_at_50": 0.93571, "recall_at_100": 1.0, "precision_at_1": 0.30357, "precision_at_3": 0.14405, "precision_at_5": 0.10286, "precision_at_10": 0.06214, "precision_at_20": 0.03786, "precision_at_50": 0.01871, "precision_at_100": 0.01, "mrr_at_1": 0.30357142857142855, "mrr_at_3": 0.36011904761904756, "mrr_at_5": 0.37922619047619044, "mrr_at_10": 0.39423611111111106, "mrr_at_20": 0.40382279848818436, "mrr_at_50": 0.4094630508393822, "mrr_at_100": 0.41056344257884836, "naucs_at_1_max": 0.31497868445771837, "naucs_at_1_std": 0.27018473469977594, "naucs_at_1_diff1": 0.5545123919169535, "naucs_at_3_max": 0.34146613553224203, "naucs_at_3_std": 0.37422200737204664, "naucs_at_3_diff1": 0.45049433616718654, "naucs_at_5_max": 0.32754790604234263, "naucs_at_5_std": 0.30800397929222695, "naucs_at_5_diff1": 0.42043830165353135, "naucs_at_10_max": 0.3712364081039097, "naucs_at_10_std": 0.3108536867501738, "naucs_at_10_diff1": 0.3550142666710734, "naucs_at_20_max": 0.34740120603630337, "naucs_at_20_std": 0.38695705408797487, "naucs_at_20_diff1": 0.28598212372585646, "naucs_at_50_max": 0.48176678078639174, "naucs_at_50_std": 0.47294843863471453, "naucs_at_50_diff1": 0.3845834630148376, "naucs_at_100_max": 1.0, "naucs_at_100_std": 1.0, "naucs_at_100_diff1": 1.0}, "vidore/tatdqa_test": {"ndcg_at_1": 0.14459, "ndcg_at_3": 0.19269, "ndcg_at_5": 0.22555, "ndcg_at_10": 0.25651, "ndcg_at_20": 0.28608, "ndcg_at_50": 0.31593, "ndcg_at_100": 0.34081, "map_at_1": 0.14459, "map_at_3": 0.18003, "map_at_5": 0.19817, "map_at_10": 0.21113, "map_at_20": 0.2192, "map_at_50": 0.22398, "map_at_100": 0.22618, "recall_at_1": 0.14459, "recall_at_3": 0.22965, "recall_at_5": 0.30984, "recall_at_10": 0.40462, "recall_at_20": 0.52187, "recall_at_50": 0.67254, "recall_at_100": 0.82564, "precision_at_1": 0.14459, "precision_at_3": 0.07655, "precision_at_5": 0.06197, "precision_at_10": 0.04046, "precision_at_20": 0.02609, "precision_at_50": 0.01345, "precision_at_100": 0.00826, "mrr_at_1": 0.14398541919805588, "mrr_at_3": 0.18013365735115439, "mrr_at_5": 0.19781287970838374, "mrr_at_10": 0.21085917761191145, "mrr_at_20": 0.21892355560340898, "mrr_at_50": 0.22369740238695404, "mrr_at_100": 0.22590106500845392, "naucs_at_1_max": 0.06416812723480628, "naucs_at_1_std": 0.007997038592145738, "naucs_at_1_diff1": 0.4569826725709526, "naucs_at_3_max": 0.0369381830145481, "naucs_at_3_std": 0.024324018445125304, "naucs_at_3_diff1": 0.35740852142395474, "naucs_at_5_max": 0.008849586329722261, "naucs_at_5_std": 0.042359103282562253, "naucs_at_5_diff1": 0.28126420208735736, "naucs_at_10_max": 0.007945661952271768, "naucs_at_10_std": 0.04907570291997627, "naucs_at_10_diff1": 0.258344724984045, "naucs_at_20_max": 0.010718299080935887, "naucs_at_20_std": 0.05833262092233647, "naucs_at_20_diff1": 0.21182794469085606, "naucs_at_50_max": -0.019000768320708836, "naucs_at_50_std": 0.041618077315128774, "naucs_at_50_diff1": 0.2183398551651244, "naucs_at_100_max": 0.0411236371256684, "naucs_at_100_std": 0.11883066266391015, "naucs_at_100_diff1": 0.18329329730314198}, "vidore/shiftproject_test": {"ndcg_at_1": 0.18, "ndcg_at_3": 0.27047, "ndcg_at_5": 0.30361, "ndcg_at_10": 0.32038, "ndcg_at_20": 0.3501, "ndcg_at_50": 0.37316, "ndcg_at_100": 0.39927, "map_at_1": 0.18, "map_at_3": 0.24667, "map_at_5": 0.26517, "map_at_10": 0.27243, "map_at_20": 0.28026, "map_at_50": 0.28368, "map_at_100": 0.28602, "recall_at_1": 0.18, "recall_at_3": 0.34, "recall_at_5": 0.42, "recall_at_10": 0.47, "recall_at_20": 0.59, "recall_at_50": 0.71, "recall_at_100": 0.87, "precision_at_1": 0.18, "precision_at_3": 0.11333, "precision_at_5": 0.084, "precision_at_10": 0.047, "precision_at_20": 0.0295, "precision_at_50": 0.0142, "precision_at_100": 0.0087, "mrr_at_1": 0.18, "mrr_at_3": 0.24666666666666665, "mrr_at_5": 0.26516666666666666, "mrr_at_10": 0.27242857142857135, "mrr_at_20": 0.2802600763099989, "mrr_at_50": 0.2836808062356978, "mrr_at_100": 0.28602155740099383, "naucs_at_1_max": -0.049029926674385174, "naucs_at_1_std": -0.19275164017822682, "naucs_at_1_diff1": 0.1930147703750482, "naucs_at_3_max": 0.05887808368001252, "naucs_at_3_std": -0.07893690876811811, "naucs_at_3_diff1": 0.1289647314449242, "naucs_at_5_max": 0.14732871918500656, "naucs_at_5_std": 0.0508852425020092, "naucs_at_5_diff1": 0.05522720792181863, "naucs_at_10_max": 0.09290734318754254, "naucs_at_10_std": 0.06761692532865714, "naucs_at_10_diff1": -0.04397473595379663, "naucs_at_20_max": 0.12417274421471261, "naucs_at_20_std": 0.11425301187139605, "naucs_at_20_diff1": -0.06302551836471092, "naucs_at_50_max": 0.215351259196116, "naucs_at_50_std": 0.20271296999796162, "naucs_at_50_diff1": 0.06287641532160981, "naucs_at_100_max": 0.43323705843145927, "naucs_at_100_std": 0.4437532400207375, "naucs_at_100_diff1": 0.24742649781530016}, "vidore/syntheticDocQA_artificial_intelligence_test": {"ndcg_at_1": 0.54, "ndcg_at_3": 0.65202, "ndcg_at_5": 0.66363, "ndcg_at_10": 0.68366, "ndcg_at_20": 0.69876, "ndcg_at_50": 0.71146, "ndcg_at_100": 0.71313, "map_at_1": 0.54, "map_at_3": 0.625, "map_at_5": 0.631, "map_at_10": 0.63967, "map_at_20": 0.64378, "map_at_50": 0.64615, "map_at_100": 0.64631, "recall_at_1": 0.54, "recall_at_3": 0.73, "recall_at_5": 0.76, "recall_at_10": 0.82, "recall_at_20": 0.88, "recall_at_50": 0.94, "recall_at_100": 0.95, "precision_at_1": 0.54, "precision_at_3": 0.24333, "precision_at_5": 0.152, "precision_at_10": 0.082, "precision_at_20": 0.044, "precision_at_50": 0.0188, "precision_at_100": 0.0095, "mrr_at_1": 0.54, "mrr_at_3": 0.625, "mrr_at_5": 0.631, "mrr_at_10": 0.6396666666666667, "mrr_at_20": 0.6437826289207869, "mrr_at_50": 0.6461486686282057, "mrr_at_100": 0.6463073987869359, "naucs_at_1_max": 0.5435386624433889, "naucs_at_1_std": -0.1421466477730543, "naucs_at_1_diff1": 0.6817120946903393, "naucs_at_3_max": 0.5216143427234654, "naucs_at_3_std": -0.2992392223161449, "naucs_at_3_diff1": 0.7614264090113817, "naucs_at_5_max": 0.5886475728995417, "naucs_at_5_std": -0.2118845721207915, "naucs_at_5_diff1": 0.7462360474171499, "naucs_at_10_max": 0.5302070712633551, "naucs_at_10_std": -0.09805595329881955, "naucs_at_10_diff1": 0.6981495759444873, "naucs_at_20_max": 0.6711740375437496, "naucs_at_20_std": 0.10034202990773257, "naucs_at_20_diff1": 0.723075087496024, "naucs_at_50_max": 0.9101307189542484, "naucs_at_50_std": 0.41456582633053624, "naucs_at_50_diff1": 0.8358232181761611, "naucs_at_100_max": 0.9183006535947744, "naucs_at_100_std": 0.42586367880485737, "naucs_at_100_diff1": 0.802987861811396}, "vidore/syntheticDocQA_energy_test": {"ndcg_at_1": 0.61, "ndcg_at_3": 0.68917, "ndcg_at_5": 0.70938, "ndcg_at_10": 0.72782, "ndcg_at_20": 0.74319, "ndcg_at_50": 0.75456, "ndcg_at_100": 0.75456, "map_at_1": 0.61, "map_at_3": 0.66833, "map_at_5": 0.67933, "map_at_10": 0.68637, "map_at_20": 0.6907, "map_at_50": 0.69229, "map_at_100": 0.69229, "recall_at_1": 0.61, "recall_at_3": 0.75, "recall_at_5": 0.8, "recall_at_10": 0.86, "recall_at_20": 0.92, "recall_at_50": 0.98, "recall_at_100": 0.98, "precision_at_1": 0.61, "precision_at_3": 0.25, "precision_at_5": 0.16, "precision_at_10": 0.086, "precision_at_20": 0.046, "precision_at_50": 0.0196, "precision_at_100": 0.0098, "mrr_at_1": 0.61, "mrr_at_3": 0.6683333333333334, "mrr_at_5": 0.6793333333333333, "mrr_at_10": 0.686373015873016, "mrr_at_20": 0.6906960996519821, "mrr_at_50": 0.6922906137236967, "mrr_at_100": 0.6922906137236967, "naucs_at_1_max": 0.420965963168311, "naucs_at_1_std": -0.33118305561011013, "naucs_at_1_diff1": 0.775701358149653, "naucs_at_3_max": 0.4598011512297232, "naucs_at_3_std": -0.5087807430664556, "naucs_at_3_diff1": 0.8101936159079016, "naucs_at_5_max": 0.40549450549450505, "naucs_at_5_std": -0.708766233766233, "naucs_at_5_diff1": 0.7687812187812181, "naucs_at_10_max": 0.3752598032423449, "naucs_at_10_std": -0.6769086878204217, "naucs_at_10_diff1": 0.7617084661216568, "naucs_at_20_max": 0.31506769374416554, "naucs_at_20_std": -0.7546685340802911, "naucs_at_20_diff1": 0.8211367880485521, "naucs_at_50_max": 0.3384687208216665, "naucs_at_50_std": -0.3674136321195118, "naucs_at_50_diff1": 0.7770774976657274, "naucs_at_100_max": 0.3384687208216665, "naucs_at_100_std": -0.3674136321195118, "naucs_at_100_diff1": 0.7770774976657274}, "vidore/syntheticDocQA_government_reports_test": {"ndcg_at_1": 0.58, "ndcg_at_3": 0.70464, "ndcg_at_5": 0.73303, "ndcg_at_10": 0.74864, "ndcg_at_20": 0.75126, "ndcg_at_50": 0.76109, "ndcg_at_100": 0.7643, "map_at_1": 0.58, "map_at_3": 0.675, "map_at_5": 0.6905, "map_at_10": 0.69661, "map_at_20": 0.69738, "map_at_50": 0.69892, "map_at_100": 0.69919, "recall_at_1": 0.58, "recall_at_3": 0.79, "recall_at_5": 0.86, "recall_at_10": 0.91, "recall_at_20": 0.92, "recall_at_50": 0.97, "recall_at_100": 0.99, "precision_at_1": 0.58, "precision_at_3": 0.26333, "precision_at_5": 0.172, "precision_at_10": 0.091, "precision_at_20": 0.046, "precision_at_50": 0.0194, "precision_at_100": 0.0099, "mrr_at_1": 0.58, "mrr_at_3": 0.675, "mrr_at_5": 0.6905000000000001, "mrr_at_10": 0.6966111111111111, "mrr_at_20": 0.6973803418803419, "mrr_at_50": 0.6989176459985358, "mrr_at_100": 0.6991918395469229, "naucs_at_1_max": 0.3447775488289754, "naucs_at_1_std": -0.2717735148155674, "naucs_at_1_diff1": 0.5655454624604579, "naucs_at_3_max": 0.4775049895400006, "naucs_at_3_std": -0.1952052324043572, "naucs_at_3_diff1": 0.46127395580349545, "naucs_at_5_max": 0.592732437300817, "naucs_at_5_std": -0.12238464736039634, "naucs_at_5_diff1": 0.44121518636552576, "naucs_at_10_max": 0.6720095445585643, "naucs_at_10_std": -0.016184251478367565, "naucs_at_10_diff1": 0.4494760867309872, "naucs_at_20_max": 0.7112511671335223, "naucs_at_20_std": 0.19928804855275828, "naucs_at_20_diff1": 0.39700046685340834, "naucs_at_50_max": 0.5224089635854318, "naucs_at_50_std": -0.22362278244630937, "naucs_at_50_diff1": 0.6374105197634607, "naucs_at_100_max": 0.7222222222222041, "naucs_at_100_std": 0.35807656395892007, "naucs_at_100_diff1": 1.0}, "vidore/syntheticDocQA_healthcare_industry_test": {"ndcg_at_1": 0.69, "ndcg_at_3": 0.77547, "ndcg_at_5": 0.79182, "ndcg_at_10": 0.79895, "ndcg_at_20": 0.81178, "ndcg_at_50": 0.81804, "ndcg_at_100": 0.81804, "map_at_1": 0.69, "map_at_3": 0.75333, "map_at_5": 0.76233, "map_at_10": 0.76567, "map_at_20": 0.76927, "map_at_50": 0.7704, "map_at_100": 0.7704, "recall_at_1": 0.69, "recall_at_3": 0.84, "recall_at_5": 0.88, "recall_at_10": 0.9, "recall_at_20": 0.95, "recall_at_50": 0.98, "recall_at_100": 0.98, "precision_at_1": 0.69, "precision_at_3": 0.28, "precision_at_5": 0.176, "precision_at_10": 0.09, "precision_at_20": 0.0475, "precision_at_50": 0.0196, "precision_at_100": 0.0098, "mrr_at_1": 0.69, "mrr_at_3": 0.7533333333333333, "mrr_at_5": 0.7623333333333333, "mrr_at_10": 0.7656666666666667, "mrr_at_20": 0.7692692307692307, "mrr_at_50": 0.7703983974358976, "mrr_at_100": 0.7703983974358976, "naucs_at_1_max": 0.2543168068857264, "naucs_at_1_std": -0.33253964054535495, "naucs_at_1_diff1": 0.5925533978870132, "naucs_at_3_max": 0.38576951399116305, "naucs_at_3_std": -0.2952565046637219, "naucs_at_3_diff1": 0.640954835542463, "naucs_at_5_max": 0.45434298440980025, "naucs_at_5_std": -0.38613585746102075, "naucs_at_5_diff1": 0.5359926821508111, "naucs_at_10_max": 0.48968253968254033, "naucs_at_10_std": -0.4380952380952347, "naucs_at_10_diff1": 0.4553221288515409, "naucs_at_20_max": 0.47824463118580945, "naucs_at_20_std": -0.4903828197945701, "naucs_at_20_diff1": 0.46087768440709953, "naucs_at_50_max": 0.4960317460317565, "naucs_at_50_std": 0.795751633986937, "naucs_at_50_diff1": 0.0793650793650745, "naucs_at_100_max": 0.4960317460317565, "naucs_at_100_std": 0.795751633986937, "naucs_at_100_diff1": 0.0793650793650745}} |