results.json

{"../colpali/data_dir/eval_vidore/tatdqa_test": {"ndcg_at_1": 0.45504, "ndcg_at_3": 0.56872, "ndcg_at_5": 0.59711, "ndcg_at_10": 0.6331, "ndcg_at_20": 0.65064, "ndcg_at_50": 0.66009, "ndcg_at_100": 0.66538, "map_at_1": 0.45504, "map_at_3": 0.53989, "map_at_5": 0.55557, "map_at_10": 0.57078, "map_at_20": 0.57577, "map_at_50": 0.57736, "map_at_100": 0.57784, "recall_at_1": 0.45504, "recall_at_3": 0.65249, "recall_at_5": 0.72175, "recall_at_10": 0.83111, "recall_at_20": 0.89915, "recall_at_50": 0.94593, "recall_at_100": 0.97813, "precision_at_1": 0.45504, "precision_at_3": 0.2175, "precision_at_5": 0.14435, "precision_at_10": 0.08311, "precision_at_20": 0.04496, "precision_at_50": 0.01892, "precision_at_100": 0.00978, "mrr_at_1": 0.4526123936816525, "mrr_at_3": 0.5387808829485622, "mrr_at_5": 0.5541211016605914, "mrr_at_10": 0.5695155644274721, "mrr_at_20": 0.5744895949632595, "mrr_at_50": 0.5760732761650542, "mrr_at_100": 0.5765591764798109, "naucs_at_1_max": 0.4007616357070541, "naucs_at_1_std": -0.08607992151091595, "naucs_at_1_diff1": 0.6060499097590993, "naucs_at_3_max": 0.44870095096796053, "naucs_at_3_std": -0.008186482145121039, "naucs_at_3_diff1": 0.4878097052185952, "naucs_at_5_max": 0.43459142011431634, "naucs_at_5_std": -0.009447458561375376, "naucs_at_5_diff1": 0.4384749463893095, "naucs_at_10_max": 0.5620075425998711, "naucs_at_10_std": 0.16203801191956155, "naucs_at_10_diff1": 0.4322829810150347, "naucs_at_20_max": 0.5401898328924477, "naucs_at_20_std": 0.23026218741014237, "naucs_at_20_diff1": 0.4155676018698351, "naucs_at_50_max": 0.5491274432650216, "naucs_at_50_std": 0.30248595340618817, "naucs_at_50_diff1": 0.39369573233613886, "naucs_at_100_max": 0.7837692460321041, "naucs_at_100_std": 0.6807203771751222, "naucs_at_100_diff1": 0.3957565014112084}, "../colpali/data_dir/eval_vidore/shiftproject_test": {"ndcg_at_1": 0.52, "ndcg_at_3": 0.64833, "ndcg_at_5": 0.68446, "ndcg_at_10": 0.71008, "ndcg_at_20": 0.71264, "ndcg_at_50": 0.72455, "ndcg_at_100": 0.72624, "map_at_1": 0.52, "map_at_3": 0.61667, "map_at_5": 0.63617, "map_at_10": 0.64658, "map_at_20": 0.6473, "map_at_50": 0.64923, "map_at_100": 0.64939, "recall_at_1": 0.52, "recall_at_3": 0.74, "recall_at_5": 0.83, "recall_at_10": 0.91, "recall_at_20": 0.92, "recall_at_50": 0.98, "recall_at_100": 0.99, "precision_at_1": 0.52, "precision_at_3": 0.24667, "precision_at_5": 0.166, "precision_at_10": 0.091, "precision_at_20": 0.046, "precision_at_50": 0.0196, "precision_at_100": 0.0099, "mrr_at_1": 0.52, "mrr_at_3": 0.62, "mrr_at_5": 0.64, "mrr_at_10": 0.6504166666666668, "mrr_at_20": 0.6511309523809524, "mrr_at_50": 0.6530746753246752, "mrr_at_100": 0.6532413419913419, "naucs_at_1_max": 0.011072419609336812, "naucs_at_1_std": -0.28150334621034895, "naucs_at_1_diff1": 0.5199684422438654, "naucs_at_3_max": 0.30832083046105246, "naucs_at_3_std": -0.017132314180284387, "naucs_at_3_diff1": 0.3817768946920235, "naucs_at_5_max": 0.619753874731526, "naucs_at_5_std": 0.3159865327683294, "naucs_at_5_diff1": 0.32579671445985925, "naucs_at_10_max": 0.7622159975101153, "naucs_at_10_std": 0.4451187882560422, "naucs_at_10_diff1": 0.13990040460628678, "naucs_at_20_max": 0.7488328664799268, "naucs_at_20_std": 0.48541083099906845, "naucs_at_20_diff1": 0.03238795518207437, "naucs_at_50_max": 0.561391223155939, "naucs_at_50_std": 0.2754435107376223, "naucs_at_50_diff1": 0.4960317460317565, "naucs_at_100_max": 0.12278244631185525, "naucs_at_100_std": -0.17133520074697067, "naucs_at_100_diff1": 0.12278244631185525}, "../colpali/data_dir/eval_vidore/syntheticDocQA_artificial_intelligence_test": {"ndcg_at_1": 0.75, "ndcg_at_3": 0.82678, "ndcg_at_5": 0.84226, "ndcg_at_10": 0.84915, "ndcg_at_20": 0.85956, "ndcg_at_50": 0.86333, "ndcg_at_100": 0.86333, "map_at_1": 0.75, "map_at_3": 0.80833, "map_at_5": 0.81633, "map_at_10": 0.81943, "map_at_20": 0.82243, "map_at_50": 0.82296, "map_at_100": 0.82296, "recall_at_1": 0.75, "recall_at_3": 0.88, "recall_at_5": 0.92, "recall_at_10": 0.94, "recall_at_20": 0.98, "recall_at_50": 1.0, "recall_at_100": 1.0, "precision_at_1": 0.75, "precision_at_3": 0.29333, "precision_at_5": 0.184, "precision_at_10": 0.094, "precision_at_20": 0.049, "precision_at_50": 0.02, "precision_at_100": 0.01, "mrr_at_1": 0.75, "mrr_at_3": 0.8083333333333332, "mrr_at_5": 0.8163333333333334, "mrr_at_10": 0.8194285714285714, "mrr_at_20": 0.8224285714285715, "mrr_at_50": 0.8229552337063858, "mrr_at_100": 0.8229552337063858, "naucs_at_1_max": 0.40276295133437956, "naucs_at_1_std": -0.22528519099947694, "naucs_at_1_diff1": 0.8150915750915747, "naucs_at_3_max": 0.6308065542475333, "naucs_at_3_std": -0.2201718103722543, "naucs_at_3_diff1": 0.7729080496341069, "naucs_at_5_max": 0.701272175536881, "naucs_at_5_std": -0.12114845938375016, "naucs_at_5_diff1": 0.7619047619047611, "naucs_at_10_max": 0.9039060068471841, "naucs_at_10_std": 0.3204170557111718, "naucs_at_10_diff1": 0.7288359788359801, "naucs_at_20_max": 0.9346405228758136, "naucs_at_20_std": 0.6790382819794609, "naucs_at_20_diff1": 0.861111111111116, "naucs_at_50_max": NaN, "naucs_at_50_std": NaN, "naucs_at_50_diff1": NaN, "naucs_at_100_max": NaN, "naucs_at_100_std": NaN, "naucs_at_100_diff1": NaN}, "../colpali/data_dir/eval_vidore/syntheticDocQA_government_reports_test": {"ndcg_at_1": 0.73, "ndcg_at_3": 0.80678, "ndcg_at_5": 0.83175, "ndcg_at_10": 0.85149, "ndcg_at_20": 0.85647, "ndcg_at_50": 0.85647, "ndcg_at_100": 0.85647, "map_at_1": 0.73, "map_at_3": 0.78833, "map_at_5": 0.80233, "map_at_10": 0.81067, "map_at_20": 0.812, "map_at_50": 0.812, "map_at_100": 0.812, "recall_at_1": 0.73, "recall_at_3": 0.86, "recall_at_5": 0.92, "recall_at_10": 0.98, "recall_at_20": 1.0, "recall_at_50": 1.0, "recall_at_100": 1.0, "precision_at_1": 0.73, "precision_at_3": 0.28667, "precision_at_5": 0.184, "precision_at_10": 0.098, "precision_at_20": 0.05, "precision_at_50": 0.02, "precision_at_100": 0.01, "mrr_at_1": 0.73, "mrr_at_3": 0.79, "mrr_at_5": 0.8059999999999999, "mrr_at_10": 0.8126666666666666, "mrr_at_20": 0.8140000000000001, "mrr_at_50": 0.8140000000000001, "mrr_at_100": 0.8140000000000001, "naucs_at_1_max": 0.43661168884782514, "naucs_at_1_std": -0.10324559161768487, "naucs_at_1_diff1": 0.7004069275983408, "naucs_at_3_max": 0.5075169738118337, "naucs_at_3_std": -0.201711237356242, "naucs_at_3_diff1": 0.564361923236802, "naucs_at_5_max": 0.7782446311858078, "naucs_at_5_std": -0.36525443510737254, "naucs_at_5_diff1": 0.7284080298786184, "naucs_at_10_max": 0.7770774976657324, "naucs_at_10_std": -0.21475256769374024, "naucs_at_10_diff1": 0.27544351073763346, "naucs_at_20_max": 1.0, "naucs_at_20_std": 1.0, "naucs_at_20_diff1": 1.0, "naucs_at_50_max": NaN, "naucs_at_50_std": NaN, "naucs_at_50_diff1": NaN, "naucs_at_100_max": NaN, "naucs_at_100_std": NaN, "naucs_at_100_diff1": NaN}, "../colpali/data_dir/eval_vidore/arxivqa_test_subsampled": {"ndcg_at_1": 0.71, "ndcg_at_3": 0.76995, "ndcg_at_5": 0.78484, "ndcg_at_10": 0.79735, "ndcg_at_20": 0.80482, "ndcg_at_50": 0.81639, "ndcg_at_100": 0.81931, "map_at_1": 0.71, "map_at_3": 0.756, "map_at_5": 0.7643, "map_at_10": 0.76959, "map_at_20": 0.77158, "map_at_50": 0.77347, "map_at_100": 0.77372, "recall_at_1": 0.71, "recall_at_3": 0.81, "recall_at_5": 0.846, "recall_at_10": 0.884, "recall_at_20": 0.914, "recall_at_50": 0.972, "recall_at_100": 0.99, "precision_at_1": 0.71, "precision_at_3": 0.27, "precision_at_5": 0.1692, "precision_at_10": 0.0884, "precision_at_20": 0.0457, "precision_at_50": 0.01944, "precision_at_100": 0.0099, "mrr_at_1": 0.71, "mrr_at_3": 0.7573333333333334, "mrr_at_5": 0.7650333333333333, "mrr_at_10": 0.7700404761904762, "mrr_at_20": 0.7720276221714303, "mrr_at_50": 0.7738298194710532, "mrr_at_100": 0.7741667087707991, "naucs_at_1_max": 0.7299668291236584, "naucs_at_1_std": -0.3983062468728575, "naucs_at_1_diff1": 0.8380816485369605, "naucs_at_3_max": 0.7857509560479856, "naucs_at_3_std": -0.3172507726963166, "naucs_at_3_diff1": 0.7965215569175964, "naucs_at_5_max": 0.7825180779544266, "naucs_at_5_std": -0.35381320929759713, "naucs_at_5_diff1": 0.7722114199295939, "naucs_at_10_max": 0.7812469261287255, "naucs_at_10_std": -0.36191350536083366, "naucs_at_10_diff1": 0.7579428833732241, "naucs_at_20_max": 0.7745640892015706, "naucs_at_20_std": -0.35226803899854764, "naucs_at_20_diff1": 0.7307884394067706, "naucs_at_50_max": 0.8487728424703169, "naucs_at_50_std": -0.014705882352948679, "naucs_at_50_diff1": 0.764639189008937, "naucs_at_100_max": 1.0, "naucs_at_100_std": 0.8585434173669326, "naucs_at_100_diff1": 0.9215686274509768}, "../colpali/data_dir/eval_vidore/docvqa_test_subsampled": {"ndcg_at_1": 0.38803, "ndcg_at_3": 0.46109, "ndcg_at_5": 0.49029, "ndcg_at_10": 0.50319, "ndcg_at_20": 0.51979, "ndcg_at_50": 0.5325, "ndcg_at_100": 0.54495, "map_at_1": 0.38803, "map_at_3": 0.44272, "map_at_5": 0.45891, "map_at_10": 0.46423, "map_at_20": 0.46867, "map_at_50": 0.47069, "map_at_100": 0.47175, "recall_at_1": 0.38803, "recall_at_3": 0.51441, "recall_at_5": 0.58537, "recall_at_10": 0.62528, "recall_at_20": 0.6918, "recall_at_50": 0.7561, "recall_at_100": 0.8337, "precision_at_1": 0.38803, "precision_at_3": 0.17147, "precision_at_5": 0.11707, "precision_at_10": 0.06253, "precision_at_20": 0.03459, "precision_at_50": 0.01512, "precision_at_100": 0.00834, "mrr_at_1": 0.3858093126385809, "mrr_at_3": 0.4416112342941611, "mrr_at_5": 0.4577974870657798, "mrr_at_10": 0.46294125928272273, "mrr_at_20": 0.4675986755215576, "mrr_at_50": 0.4696136690706579, "mrr_at_100": 0.47067284991598984, "naucs_at_1_max": 0.6880912118081786, "naucs_at_1_std": 0.1227003172324307, "naucs_at_1_diff1": 0.7549324423028185, "naucs_at_3_max": 0.7563655154362492, "naucs_at_3_std": 0.2935640829770719, "naucs_at_3_diff1": 0.6672805174710129, "naucs_at_5_max": 0.7828606305856601, "naucs_at_5_std": 0.3598978437045625, "naucs_at_5_diff1": 0.6234834202744584, "naucs_at_10_max": 0.7933866938151406, "naucs_at_10_std": 0.3972366212280104, "naucs_at_10_diff1": 0.6243158334404361, "naucs_at_20_max": 0.8125506614708696, "naucs_at_20_std": 0.4573186675647723, "naucs_at_20_diff1": 0.6070464733977256, "naucs_at_50_max": 0.8001630369453867, "naucs_at_50_std": 0.5106491209009716, "naucs_at_50_diff1": 0.5700401539611133, "naucs_at_100_max": 0.8046502127635046, "naucs_at_100_std": 0.5681011886365621, "naucs_at_100_diff1": 0.5652815021362786}, "../colpali/data_dir/eval_vidore/syntheticDocQA_healthcare_industry_test": {"ndcg_at_1": 0.8, "ndcg_at_3": 0.88678, "ndcg_at_5": 0.89496, "ndcg_at_10": 0.90446, "ndcg_at_20": 0.90446, "ndcg_at_50": 0.90446, "ndcg_at_100": 0.90446, "map_at_1": 0.8, "map_at_3": 0.865, "map_at_5": 0.8695, "map_at_10": 0.87329, "map_at_20": 0.87329, "map_at_50": 0.87329, "map_at_100": 0.87329, "recall_at_1": 0.8, "recall_at_3": 0.95, "recall_at_5": 0.97, "recall_at_10": 1.0, "recall_at_20": 1.0, "recall_at_50": 1.0, "recall_at_100": 1.0, "precision_at_1": 0.8, "precision_at_3": 0.31667, "precision_at_5": 0.194, "precision_at_10": 0.1, "precision_at_20": 0.05, "precision_at_50": 0.02, "precision_at_100": 0.01, "mrr_at_1": 0.8, "mrr_at_3": 0.865, "mrr_at_5": 0.8695, "mrr_at_10": 0.8732896825396825, "mrr_at_20": 0.8732896825396825, "mrr_at_50": 0.8732896825396825, "mrr_at_100": 0.8732896825396825, "naucs_at_1_max": 0.3517732267732264, "naucs_at_1_std": -0.1562687312687319, "naucs_at_1_diff1": 0.6350899100899094, "naucs_at_3_max": 0.38347338935574254, "naucs_at_3_std": -0.5926237161531261, "naucs_at_3_diff1": 0.8627450980392143, "naucs_at_5_max": 0.7860255213196407, "naucs_at_5_std": -0.21988795518207335, "naucs_at_5_diff1": 0.8638344226579531, "naucs_at_10_max": 1.0, "naucs_at_10_std": 1.0, "naucs_at_10_diff1": 1.0, "naucs_at_20_max": 1.0, "naucs_at_20_std": 1.0, "naucs_at_20_diff1": 1.0, "naucs_at_50_max": NaN, "naucs_at_50_std": NaN, "naucs_at_50_diff1": NaN, "naucs_at_100_max": NaN, "naucs_at_100_std": NaN, "naucs_at_100_diff1": NaN}, "../colpali/data_dir/eval_vidore/infovqa_test_subsampled": {"ndcg_at_1": 0.70648, "ndcg_at_3": 0.77988, "ndcg_at_5": 0.79791, "ndcg_at_10": 0.81178, "ndcg_at_20": 0.81896, "ndcg_at_50": 0.82317, "ndcg_at_100": 0.82486, "map_at_1": 0.70648, "map_at_3": 0.76248, "map_at_5": 0.7723, "map_at_10": 0.7781, "map_at_20": 0.78007, "map_at_50": 0.78083, "map_at_100": 0.78099, "recall_at_1": 0.70648, "recall_at_3": 0.82996, "recall_at_5": 0.87449, "recall_at_10": 0.917, "recall_at_20": 0.94534, "recall_at_50": 0.96559, "recall_at_100": 0.97571, "precision_at_1": 0.70648, "precision_at_3": 0.27665, "precision_at_5": 0.1749, "precision_at_10": 0.0917, "precision_at_20": 0.04727, "precision_at_50": 0.01931, "precision_at_100": 0.00976, "mrr_at_1": 0.708502024291498, "mrr_at_3": 0.7641700404858299, "mrr_at_5": 0.7731781376518219, "mrr_at_10": 0.7793160786581839, "mrr_at_20": 0.7812885331505837, "mrr_at_50": 0.78204531049572, "mrr_at_100": 0.7822104132561041, "naucs_at_1_max": 0.35210693912642194, "naucs_at_1_std": -0.25981339755457183, "naucs_at_1_diff1": 0.8006069446320773, "naucs_at_3_max": 0.5563684521448528, "naucs_at_3_std": -0.010153976919273033, "naucs_at_3_diff1": 0.7128786278945445, "naucs_at_5_max": 0.6558564747123251, "naucs_at_5_std": 0.1663990479174898, "naucs_at_5_diff1": 0.6930629953864437, "naucs_at_10_max": 0.8006339864253019, "naucs_at_10_std": 0.4302090584095774, "naucs_at_10_diff1": 0.6775921451523413, "naucs_at_20_max": 0.8681033015554591, "naucs_at_20_std": 0.6931262081290481, "naucs_at_20_diff1": 0.6412583755689819, "naucs_at_50_max": 0.9606153470616298, "naucs_at_50_std": 0.8639551994341516, "naucs_at_50_diff1": 0.7053052116711681, "naucs_at_100_max": 0.9550885295093762, "naucs_at_100_std": 0.8934242475636182, "naucs_at_100_diff1": 0.6165437325193877}, "../colpali/data_dir/eval_vidore/syntheticDocQA_energy_test": {"ndcg_at_1": 0.76, "ndcg_at_3": 0.82547, "ndcg_at_5": 0.82934, "ndcg_at_10": 0.84907, "ndcg_at_20": 0.85426, "ndcg_at_50": 0.85988, "ndcg_at_100": 0.85988, "map_at_1": 0.76, "map_at_3": 0.81, "map_at_5": 0.812, "map_at_10": 0.82031, "map_at_20": 0.8218, "map_at_50": 0.82256, "map_at_100": 0.82256, "recall_at_1": 0.76, "recall_at_3": 0.87, "recall_at_5": 0.88, "recall_at_10": 0.94, "recall_at_20": 0.96, "recall_at_50": 0.99, "recall_at_100": 0.99, "precision_at_1": 0.76, "precision_at_3": 0.29, "precision_at_5": 0.176, "precision_at_10": 0.094, "precision_at_20": 0.048, "precision_at_50": 0.0198, "precision_at_100": 0.0099, "mrr_at_1": 0.77, "mrr_at_3": 0.8166666666666668, "mrr_at_5": 0.8186666666666667, "mrr_at_10": 0.8269801587301587, "mrr_at_20": 0.8284636752136753, "mrr_at_50": 0.8292760517772038, "mrr_at_100": 0.8292760517772038, "naucs_at_1_max": 0.4600242277407624, "naucs_at_1_std": -0.3603876438522109, "naucs_at_1_diff1": 0.8055723803755296, "naucs_at_3_max": 0.47600533214841056, "naucs_at_3_std": -0.3254832259497905, "naucs_at_3_diff1": 0.7038065615048492, "naucs_at_5_max": 0.4260658606426989, "naucs_at_5_std": -0.4347756920139959, "naucs_at_5_diff1": 0.681872414890232, "naucs_at_10_max": 0.6507158418923127, "naucs_at_10_std": -0.23996265172735637, "naucs_at_10_diff1": 0.8031434796140697, "naucs_at_20_max": 0.6365546218487388, "naucs_at_20_std": -0.406629318394016, "naucs_at_20_diff1": 0.7373949579831963, "naucs_at_50_max": 1.0, "naucs_at_50_std": 0.554154995331464, "naucs_at_50_diff1": 1.0, "naucs_at_100_max": 1.0, "naucs_at_100_std": 0.554154995331464, "naucs_at_100_diff1": 1.0}, "../colpali/data_dir/eval_vidore/tabfquad_test_subsampled": {"ndcg_at_1": 0.71429, "ndcg_at_3": 0.77185, "ndcg_at_5": 0.79075, "ndcg_at_10": 0.80106, "ndcg_at_20": 0.8125, "ndcg_at_50": 0.82107, "ndcg_at_100": 0.82468, "map_at_1": 0.71429, "map_at_3": 0.75833, "map_at_5": 0.76869, "map_at_10": 0.7729, "map_at_20": 0.77588, "map_at_50": 0.77729, "map_at_100": 0.77764, "recall_at_1": 0.71429, "recall_at_3": 0.81071, "recall_at_5": 0.85714, "recall_at_10": 0.88929, "recall_at_20": 0.93571, "recall_at_50": 0.97857, "recall_at_100": 1.0, "precision_at_1": 0.71429, "precision_at_3": 0.27024, "precision_at_5": 0.17143, "precision_at_10": 0.08893, "precision_at_20": 0.04679, "precision_at_50": 0.01957, "precision_at_100": 0.01, "mrr_at_1": 0.7142857142857143, "mrr_at_3": 0.7589285714285714, "mrr_at_5": 0.7696428571428572, "mrr_at_10": 0.7738577097505669, "mrr_at_20": 0.7768215657475112, "mrr_at_50": 0.7782313410735464, "mrr_at_100": 0.7785880748819538, "naucs_at_1_max": 0.7914299115518632, "naucs_at_1_std": 0.3343473599571155, "naucs_at_1_diff1": 0.8793621013133214, "naucs_at_3_max": 0.8434654450556667, "naucs_at_3_std": 0.43567767493381904, "naucs_at_3_diff1": 0.8138482623960349, "naucs_at_5_max": 0.857142857142856, "naucs_at_5_std": 0.5951409135082594, "naucs_at_5_diff1": 0.8132896015549081, "naucs_at_10_max": 0.8319910924010837, "naucs_at_10_std": 0.6162411152801917, "naucs_at_10_diff1": 0.7920289191909943, "naucs_at_20_max": 0.8185755783795007, "naucs_at_20_std": 0.6212522045855406, "naucs_at_20_diff1": 0.7624234879136864, "naucs_at_50_max": 0.7786336756925044, "naucs_at_50_std": 0.438297541238722, "naucs_at_50_diff1": 0.676314970432611, "naucs_at_100_max": 1.0, "naucs_at_100_std": 1.0, "naucs_at_100_diff1": 1.0}}