| {"syntheticDocQA_energy": {"ndcg_at_1": 0.89, "ndcg_at_3": 0.93786, "ndcg_at_5": 0.93786, "ndcg_at_10": 0.93786, "ndcg_at_20": 0.94021, "ndcg_at_100": 0.94396, "ndcg_at_1000": 0.94396, "map_at_1": 0.89, "map_at_3": 0.92667, "map_at_5": 0.92667, "map_at_10": 0.92667, "map_at_20": 0.92722, "map_at_100": 0.92774, "map_at_1000": 0.92774, "recall_at_1": 0.89, "recall_at_3": 0.97, "recall_at_5": 0.97, "recall_at_10": 0.97, "recall_at_20": 0.98, "recall_at_100": 1.0, "recall_at_1000": 1.0, "precision_at_1": 0.89, "precision_at_3": 0.32333, "precision_at_5": 0.194, "precision_at_10": 0.097, "precision_at_20": 0.049, "precision_at_100": 0.01, "precision_at_1000": 0.001, "mrr_at_1": 0.9, "mrr_at_3": 0.9316666666666668, "mrr_at_5": 0.9316666666666668, "mrr_at_10": 0.9316666666666668, "mrr_at_20": 0.9322549019607842, "mrr_at_100": 0.9328175800234624, "mrr_at_1000": 0.9328175800234624, "naucs_at_1_max": 0.1416115453998798, "naucs_at_1_std": -0.3451163989347998, "naucs_at_1_diff1": 0.921398505283052, "naucs_at_3_max": 0.5659819483348902, "naucs_at_3_std": -0.4197012138188588, "naucs_at_3_diff1": 0.9564270152505466, "naucs_at_5_max": 0.5659819483348872, "naucs_at_5_std": -0.41970121381885955, "naucs_at_5_diff1": 0.9564270152505424, "naucs_at_10_max": 0.5659819483348872, "naucs_at_10_std": -0.41970121381885955, "naucs_at_10_diff1": 0.9564270152505424, "naucs_at_20_max": 0.9346405228758136, "naucs_at_20_std": 0.24042950513538955, "naucs_at_20_diff1": 0.9346405228758136, "naucs_at_100_max": NaN, "naucs_at_100_std": NaN, "naucs_at_100_diff1": NaN, "naucs_at_1000_max": NaN, "naucs_at_1000_std": NaN, "naucs_at_1000_diff1": NaN}, "syntheticDocQA_healthcare_industry": {"ndcg_at_1": 0.88, "ndcg_at_3": 0.92286, "ndcg_at_5": 0.93147, "ndcg_at_10": 0.9348, "ndcg_at_20": 0.9372, "ndcg_at_100": 0.93938, "ndcg_at_1000": 0.93938, "map_at_1": 0.88, "map_at_3": 0.91333, "map_at_5": 0.91833, "map_at_10": 0.91976, "map_at_20": 0.92035, "map_at_100": 0.92078, "map_at_1000": 0.92078, "recall_at_1": 0.88, "recall_at_3": 0.95, "recall_at_5": 0.97, "recall_at_10": 0.98, "recall_at_20": 0.99, "recall_at_100": 1.0, "recall_at_1000": 1.0, "precision_at_1": 0.88, "precision_at_3": 0.31667, "precision_at_5": 0.194, "precision_at_10": 0.098, "precision_at_20": 0.0495, "precision_at_100": 0.01, "precision_at_1000": 0.001, "mrr_at_1": 0.88, "mrr_at_3": 0.9133333333333334, "mrr_at_5": 0.9183333333333334, "mrr_at_10": 0.92, "mrr_at_20": 0.9205882352941177, "mrr_at_100": 0.9210427807486632, "mrr_at_1000": 0.9210427807486632, "naucs_at_1_max": 0.41314031180400845, "naucs_at_1_std": 0.09091632198536481, "naucs_at_1_diff1": 0.8812838052815782, "naucs_at_3_max": 0.23874883286648252, "naucs_at_3_std": -0.37497665732959407, "naucs_at_3_diff1": 0.8627450980392143, "naucs_at_5_max": 0.03781512605041818, "naucs_at_5_std": -0.7268907563025172, "naucs_at_5_diff1": 0.8638344226579531, "naucs_at_10_max": -0.004668534080301362, "naucs_at_10_std": -0.5144724556489195, "naucs_at_10_diff1": 0.7957516339869297, "naucs_at_20_max": -0.5634920634920767, "naucs_at_20_std": -1.1517273576097316, "naucs_at_20_diff1": 0.7222222222222276, "naucs_at_100_max": NaN, "naucs_at_100_std": NaN, "naucs_at_100_diff1": NaN, "naucs_at_1000_max": NaN, "naucs_at_1000_std": NaN, "naucs_at_1000_diff1": NaN}, "syntheticDocQA_artificial_intelligence_test": {"ndcg_at_1": 0.91, "ndcg_at_3": 0.95286, "ndcg_at_5": 0.95716, "ndcg_at_10": 0.96032, "ndcg_at_20": 0.96032, "ndcg_at_100": 0.96032, "ndcg_at_1000": 0.96032, "map_at_1": 0.91, "map_at_3": 0.94333, "map_at_5": 0.94583, "map_at_10": 0.94708, "map_at_20": 0.94708, "map_at_100": 0.94708, "map_at_1000": 0.94708, "recall_at_1": 0.91, "recall_at_3": 0.98, "recall_at_5": 0.99, "recall_at_10": 1.0, "recall_at_20": 1.0, "recall_at_100": 1.0, "recall_at_1000": 1.0, "precision_at_1": 0.91, "precision_at_3": 0.32667, "precision_at_5": 0.198, "precision_at_10": 0.1, "precision_at_20": 0.05, "precision_at_100": 0.01, "precision_at_1000": 0.001, "mrr_at_1": 0.92, "mrr_at_3": 0.9483333333333333, "mrr_at_5": 0.9508333333333333, "mrr_at_10": 0.9520833333333333, "mrr_at_20": 0.9520833333333333, "mrr_at_100": 0.9520833333333333, "mrr_at_1000": 0.9520833333333333, "naucs_at_1_max": 0.24592800082996116, "naucs_at_1_std": -0.49019607843137364, "naucs_at_1_diff1": 0.9273783587509066, "naucs_at_3_max": 0.7222222222222141, "naucs_at_3_std": -0.5144724556489392, "naucs_at_3_diff1": 0.9346405228758099, "naucs_at_5_max": 0.7222222222222276, "naucs_at_5_std": -1.1517273576097316, "naucs_at_5_diff1": 0.8692810457516413, "naucs_at_10_max": 1.0, "naucs_at_10_std": 1.0, "naucs_at_10_diff1": 1.0, "naucs_at_20_max": 1.0, "naucs_at_20_std": 1.0, "naucs_at_20_diff1": 1.0, "naucs_at_100_max": NaN, "naucs_at_100_std": NaN, "naucs_at_100_diff1": NaN, "naucs_at_1000_max": NaN, "naucs_at_1000_std": NaN, "naucs_at_1000_diff1": NaN}, "syntheticDocQA_government_reports": {"ndcg_at_1": 0.83, "ndcg_at_3": 0.92464, "ndcg_at_5": 0.92464, "ndcg_at_10": 0.92765, "ndcg_at_20": 0.93028, "ndcg_at_100": 0.93028, "ndcg_at_1000": 0.93028, "map_at_1": 0.83, "map_at_3": 0.905, "map_at_5": 0.905, "map_at_10": 0.90611, "map_at_20": 0.90688, "map_at_100": 0.90688, "map_at_1000": 0.90688, "recall_at_1": 0.83, "recall_at_3": 0.98, "recall_at_5": 0.98, "recall_at_10": 0.99, "recall_at_20": 1.0, "recall_at_100": 1.0, "recall_at_1000": 1.0, "precision_at_1": 0.83, "precision_at_3": 0.32667, "precision_at_5": 0.196, "precision_at_10": 0.099, "precision_at_20": 0.05, "precision_at_100": 0.01, "precision_at_1000": 0.001, "mrr_at_1": 0.86, "mrr_at_3": 0.92, "mrr_at_5": 0.92, "mrr_at_10": 0.9211111111111111, "mrr_at_20": 0.9218803418803418, "mrr_at_100": 0.9218803418803418, "mrr_at_1000": 0.9218803418803418, "naucs_at_1_max": 0.35191850002902464, "naucs_at_1_std": -0.0016543797527126534, "naucs_at_1_diff1": 0.9017820862599414, "naucs_at_3_max": 1.0, "naucs_at_3_std": 0.8611111111111119, "naucs_at_3_diff1": 0.8611111111111119, "naucs_at_5_max": 1.0, "naucs_at_5_std": 0.861111111111116, "naucs_at_5_diff1": 0.861111111111116, "naucs_at_10_max": 1.0, "naucs_at_10_std": 0.7222222222222276, "naucs_at_10_diff1": 0.7222222222222276, "naucs_at_20_max": 1.0, "naucs_at_20_std": 1.0, "naucs_at_20_diff1": 1.0, "naucs_at_100_max": NaN, "naucs_at_100_std": NaN, "naucs_at_100_diff1": NaN, "naucs_at_1000_max": NaN, "naucs_at_1000_std": NaN, "naucs_at_1000_diff1": NaN}, "infovqa_subsampled": {"ndcg_at_1": 0.716, "ndcg_at_3": 0.78674, "ndcg_at_5": 0.7999, "ndcg_at_10": 0.81139, "ndcg_at_20": 0.81628, "ndcg_at_100": 0.82424, "ndcg_at_1000": 0.82843, "map_at_1": 0.716, "map_at_3": 0.76967, "map_at_5": 0.77697, "map_at_10": 0.78161, "map_at_20": 0.78286, "map_at_100": 0.78402, "map_at_1000": 0.78415, "recall_at_1": 0.716, "recall_at_3": 0.836, "recall_at_5": 0.868, "recall_at_10": 0.904, "recall_at_20": 0.924, "recall_at_100": 0.966, "recall_at_1000": 1.0, "precision_at_1": 0.716, "precision_at_3": 0.27867, "precision_at_5": 0.1736, "precision_at_10": 0.0904, "precision_at_20": 0.0462, "precision_at_100": 0.00966, "precision_at_1000": 0.001, "mrr_at_1": 0.716, "mrr_at_3": 0.7706666666666663, "mrr_at_5": 0.7774666666666662, "mrr_at_10": 0.7822595238095235, "mrr_at_20": 0.7834290391754783, "mrr_at_100": 0.7846837692508419, "mrr_at_1000": 0.7848128972573761, "naucs_at_1_max": 0.5144395755239958, "naucs_at_1_std": 0.056706162745971116, "naucs_at_1_diff1": 0.8757608211966486, "naucs_at_3_max": 0.6206474526649232, "naucs_at_3_std": 0.2310119747546263, "naucs_at_3_diff1": 0.8256041371697351, "naucs_at_5_max": 0.6981123805627789, "naucs_at_5_std": 0.3026984776319997, "naucs_at_5_diff1": 0.8120489144727236, "naucs_at_10_max": 0.7312188764394637, "naucs_at_10_std": 0.39961679116090754, "naucs_at_10_diff1": 0.7691118113912242, "naucs_at_20_max": 0.7535259717922257, "naucs_at_20_std": 0.4052901862499385, "naucs_at_20_diff1": 0.758477075040541, "naucs_at_100_max": 0.8885044213763893, "naucs_at_100_std": 0.6311858076563935, "naucs_at_100_diff1": 0.7411984401603742, "naucs_at_1000_max": 1.0, "naucs_at_1000_std": 1.0, "naucs_at_1000_diff1": 1.0}, "docvqa_subsampled": {"ndcg_at_1": 0.394, "ndcg_at_3": 0.46421, "ndcg_at_5": 0.48048, "ndcg_at_10": 0.49561, "ndcg_at_20": 0.50938, "ndcg_at_100": 0.53597, "ndcg_at_1000": 0.56127, "map_at_1": 0.394, "map_at_3": 0.447, "map_at_5": 0.4559, "map_at_10": 0.46192, "map_at_20": 0.46576, "map_at_100": 0.46928, "map_at_1000": 0.47021, "recall_at_1": 0.394, "recall_at_3": 0.514, "recall_at_5": 0.554, "recall_at_10": 0.602, "recall_at_20": 0.656, "recall_at_100": 0.802, "recall_at_1000": 1.0, "precision_at_1": 0.394, "precision_at_3": 0.17133, "precision_at_5": 0.1108, "precision_at_10": 0.0602, "precision_at_20": 0.0328, "precision_at_100": 0.00802, "precision_at_1000": 0.001, "mrr_at_1": 0.394, "mrr_at_3": 0.44466666666666693, "mrr_at_5": 0.45446666666666674, "mrr_at_10": 0.4606452380952381, "mrr_at_20": 0.46431832950331403, "mrr_at_100": 0.46793416222345513, "mrr_at_1000": 0.4688637410389999, "naucs_at_1_max": 0.59049210041854, "naucs_at_1_std": 0.05796555116331319, "naucs_at_1_diff1": 0.8098445903552552, "naucs_at_3_max": 0.5727328519172382, "naucs_at_3_std": 0.06011122496154157, "naucs_at_3_diff1": 0.6923280730829647, "naucs_at_5_max": 0.5486981759514414, "naucs_at_5_std": 0.061820776722579115, "naucs_at_5_diff1": 0.6590126388767453, "naucs_at_10_max": 0.5135874988457401, "naucs_at_10_std": 0.08675863013943434, "naucs_at_10_diff1": 0.6397577692850738, "naucs_at_20_max": 0.512128536803836, "naucs_at_20_std": 0.14237775962906352, "naucs_at_20_diff1": 0.605205095105116, "naucs_at_100_max": 0.48413818834514916, "naucs_at_100_std": 0.37696760076233576, "naucs_at_100_diff1": 0.5448073490707792, "naucs_at_1000_max": 1.0, "naucs_at_1000_std": 1.0, "naucs_at_1000_diff1": 1.0}, "arxivqa_subsampled": {"ndcg_at_1": 0.64, "ndcg_at_3": 0.69686, "ndcg_at_5": 0.71312, "ndcg_at_10": 0.72568, "ndcg_at_20": 0.73619, "ndcg_at_100": 0.7531, "ndcg_at_1000": 0.7598, "map_at_1": 0.64, "map_at_3": 0.68267, "map_at_5": 0.69157, "map_at_10": 0.69692, "map_at_20": 0.69975, "map_at_100": 0.70204, "map_at_1000": 0.70234, "recall_at_1": 0.64, "recall_at_3": 0.738, "recall_at_5": 0.778, "recall_at_10": 0.816, "recall_at_20": 0.858, "recall_at_100": 0.95, "recall_at_1000": 1.0, "precision_at_1": 0.64, "precision_at_3": 0.246, "precision_at_5": 0.1556, "precision_at_10": 0.0816, "precision_at_20": 0.0429, "precision_at_100": 0.0095, "precision_at_1000": 0.001, "mrr_at_1": 0.646, "mrr_at_3": 0.6843333333333333, "mrr_at_5": 0.6939333333333332, "mrr_at_10": 0.6995880952380951, "mrr_at_20": 0.7024205660696371, "mrr_at_100": 0.7046153644586061, "mrr_at_1000": 0.7049344272544787, "naucs_at_1_max": 0.6400498147911611, "naucs_at_1_std": 0.30982245497509275, "naucs_at_1_diff1": 0.8982085834717077, "naucs_at_3_max": 0.6633502178056638, "naucs_at_3_std": 0.3770435651623774, "naucs_at_3_diff1": 0.8408064249648411, "naucs_at_5_max": 0.6786025034872963, "naucs_at_5_std": 0.3739256135569494, "naucs_at_5_diff1": 0.8232090490155005, "naucs_at_10_max": 0.7271186806544918, "naucs_at_10_std": 0.45356819064887993, "naucs_at_10_diff1": 0.8012077451107367, "naucs_at_20_max": 0.761528570060601, "naucs_at_20_std": 0.5156837799756484, "naucs_at_20_diff1": 0.7929713683807331, "naucs_at_100_max": 0.7033986928104585, "naucs_at_100_std": 0.42941176470588494, "naucs_at_100_diff1": 0.7332773109243655, "naucs_at_1000_max": 1.0, "naucs_at_1000_std": 1.0, "naucs_at_1000_diff1": 1.0}, "tabfquad_subsampled": {"ndcg_at_1": 0.77857, "ndcg_at_3": 0.8321, "ndcg_at_5": 0.83932, "ndcg_at_10": 0.85306, "ndcg_at_20": 0.86309, "ndcg_at_100": 0.8696, "ndcg_at_1000": 0.8696, "map_at_1": 0.77857, "map_at_3": 0.81964, "map_at_5": 0.82357, "map_at_10": 0.82918, "map_at_20": 0.83197, "map_at_100": 0.83306, "map_at_1000": 0.83306, "recall_at_1": 0.77857, "recall_at_3": 0.86786, "recall_at_5": 0.88571, "recall_at_10": 0.92857, "recall_at_20": 0.96786, "recall_at_100": 1.0, "recall_at_1000": 1.0, "precision_at_1": 0.77857, "precision_at_3": 0.28929, "precision_at_5": 0.17714, "precision_at_10": 0.09286, "precision_at_20": 0.04839, "precision_at_100": 0.01, "precision_at_1000": 0.001, "mrr_at_1": 0.775, "mrr_at_3": 0.8172619047619047, "mrr_at_5": 0.8213690476190477, "mrr_at_10": 0.8272264739229026, "mrr_at_20": 0.8296797314496894, "mrr_at_100": 0.8307939591209218, "mrr_at_1000": 0.8307939591209218, "naucs_at_1_max": 0.3645402573438554, "naucs_at_1_std": 0.19164868115392844, "naucs_at_1_diff1": 0.7632584806497849, "naucs_at_3_max": 0.37951885735136837, "naucs_at_3_std": 0.2538639977063613, "naucs_at_3_diff1": 0.7619308259702339, "naucs_at_5_max": 0.3969654722354073, "naucs_at_5_std": 0.30882178452776354, "naucs_at_5_diff1": 0.7626364499288074, "naucs_at_10_max": 0.40688608776844226, "naucs_at_10_std": 0.3362978524743217, "naucs_at_10_diff1": 0.7623949579831951, "naucs_at_20_max": 0.47883597883597717, "naucs_at_20_std": 0.5260400456478881, "naucs_at_20_diff1": 0.9255628177196795, "naucs_at_100_max": 1.0, "naucs_at_100_std": 1.0, "naucs_at_100_diff1": 1.0, "naucs_at_1000_max": NaN, "naucs_at_1000_std": NaN, "naucs_at_1000_diff1": NaN}, "tatdqa": {"ndcg_at_1": 0.46723, "ndcg_at_3": 0.56002, "ndcg_at_5": 0.59103, "ndcg_at_10": 0.62058, "ndcg_at_20": 0.63842, "ndcg_at_100": 0.65687, "ndcg_at_1000": 0.66202, "map_at_1": 0.46723, "map_at_3": 0.53668, "map_at_5": 0.55406, "map_at_10": 0.56648, "map_at_20": 0.57141, "map_at_100": 0.57398, "map_at_1000": 0.57422, "recall_at_1": 0.46723, "recall_at_3": 0.62778, "recall_at_5": 0.70235, "recall_at_10": 0.79254, "recall_at_20": 0.8629, "recall_at_100": 0.96212, "recall_at_1000": 1.0, "precision_at_1": 0.46723, "precision_at_3": 0.20926, "precision_at_5": 0.14047, "precision_at_10": 0.07925, "precision_at_20": 0.04314, "precision_at_100": 0.00962, "precision_at_1000": 0.001, "mrr_at_1": 0.4660252555622369, "mrr_at_3": 0.5375826819001811, "mrr_at_5": 0.5542092603728216, "mrr_at_10": 0.5662937032900962, "mrr_at_20": 0.571217019105013, "mrr_at_100": 0.5738015340316196, "mrr_at_1000": 0.5740512063675741, "naucs_at_1_max": 0.12316388049685562, "naucs_at_1_std": -0.03302911730144414, "naucs_at_1_diff1": 0.6367700909505911, "naucs_at_3_max": 0.1491828892683818, "naucs_at_3_std": -0.01776625584052866, "naucs_at_3_diff1": 0.5441671908451038, "naucs_at_5_max": 0.13509434719203903, "naucs_at_5_std": -0.008750994497691225, "naucs_at_5_diff1": 0.4852071271216003, "naucs_at_10_max": 0.16644379506149076, "naucs_at_10_std": 0.05915036291157827, "naucs_at_10_diff1": 0.46415728517159416, "naucs_at_20_max": 0.21050100386062404, "naucs_at_20_std": 0.12189428322749622, "naucs_at_20_diff1": 0.39296395537492884, "naucs_at_100_max": 0.34701914032223563, "naucs_at_100_std": 0.3284671046283788, "naucs_at_100_diff1": 0.3933540743785471, "naucs_at_1000_max": NaN, "naucs_at_1000_std": NaN, "naucs_at_1000_diff1": NaN}, "shift_project": {"ndcg_at_1": 0.59, "ndcg_at_3": 0.70964, "ndcg_at_5": 0.73847, "ndcg_at_10": 0.7544, "ndcg_at_20": 0.76229, "ndcg_at_100": 0.77165, "ndcg_at_1000": 0.77302, "map_at_1": 0.59, "map_at_3": 0.68167, "map_at_5": 0.69767, "map_at_10": 0.7041, "map_at_20": 0.70641, "map_at_100": 0.70774, "map_at_1000": 0.7078, "recall_at_1": 0.59, "recall_at_3": 0.79, "recall_at_5": 0.86, "recall_at_10": 0.91, "recall_at_20": 0.94, "recall_at_100": 0.99, "recall_at_1000": 1.0, "precision_at_1": 0.59, "precision_at_3": 0.26333, "precision_at_5": 0.172, "precision_at_10": 0.091, "precision_at_20": 0.047, "precision_at_100": 0.0099, "precision_at_1000": 0.001, "mrr_at_1": 0.59, "mrr_at_3": 0.6866666666666668, "mrr_at_5": 0.7001666666666668, "mrr_at_10": 0.7071507936507938, "mrr_at_20": 0.7095317460317462, "mrr_at_100": 0.7108961223961225, "mrr_at_1000": 0.7109623475616854, "naucs_at_1_max": -0.09331298516442392, "naucs_at_1_std": -0.20234199597928013, "naucs_at_1_diff1": 0.6018313352018425, "naucs_at_3_max": -0.061100824776973234, "naucs_at_3_std": -0.285545963882944, "naucs_at_3_diff1": 0.6008367999615265, "naucs_at_5_max": -0.25419149230982163, "naucs_at_5_std": -0.42237079118747267, "naucs_at_5_diff1": 0.5672717195510587, "naucs_at_10_max": -0.05612615416536863, "naucs_at_10_std": -0.4241622574955902, "naucs_at_10_diff1": 0.7744060587197855, "naucs_at_20_max": -0.05757858699035113, "naucs_at_20_std": -0.027699968876440207, "naucs_at_20_diff1": 0.8295985060690946, "naucs_at_100_max": -1.1517273576096694, "naucs_at_100_std": -1.7399626517273008, "naucs_at_100_diff1": 0.554154995331464, "naucs_at_1000_max": NaN, "naucs_at_1000_std": NaN, "naucs_at_1000_diff1": NaN}} |