{ "vidore/arxivqa_test_subsampled_ocr_chunk": { "ndcg_at_1": 0.2986, "ndcg_at_3": 0.32832, "ndcg_at_5": 0.33583, "ndcg_at_10": 0.34361, "ndcg_at_20": 0.35364, "ndcg_at_100": 0.37566, "ndcg_at_1000": 0.43162, "map_at_1": 0.2986, "map_at_3": 0.32064, "map_at_5": 0.32485, "map_at_10": 0.32806, "map_at_20": 0.33076, "map_at_100": 0.33345, "map_at_1000": 0.3354, "recall_at_1": 0.2986, "recall_at_3": 0.3507, "recall_at_5": 0.36874, "recall_at_10": 0.39279, "recall_at_20": 0.43287, "recall_at_100": 0.55711, "recall_at_1000": 1.0, "precision_at_1": 0.2986, "precision_at_3": 0.1169, "precision_at_5": 0.07375, "precision_at_10": 0.03928, "precision_at_20": 0.02164, "precision_at_100": 0.00557, "precision_at_1000": 0.001, "mrr_at_1": 0.2965931863727455, "mrr_at_3": 0.31963927855711427, "mrr_at_5": 0.3234468937875752, "mrr_at_10": 0.3261880904666476, "mrr_at_20": 0.3285116859866204, "mrr_at_100": 0.3312035719015709, "mrr_at_1000": 0.33330186472763734, "naucs_at_1_max": 0.30904635770376127, "naucs_at_1_std": 0.08980665605658832, "naucs_at_1_diff1": 0.8183542882037567, "naucs_at_3_max": 0.31492911871623636, "naucs_at_3_std": 0.11499541343343214, "naucs_at_3_diff1": 0.7564148222064737, "naucs_at_5_max": 0.31203155402695176, "naucs_at_5_std": 0.11122642759739138, "naucs_at_5_diff1": 0.7348897516182112, "naucs_at_10_max": 0.31842231893538103, "naucs_at_10_std": 0.09831520728309058, "naucs_at_10_diff1": 0.694552438974271, "naucs_at_20_max": 0.31663139432163556, "naucs_at_20_std": 0.0902539670425055, "naucs_at_20_diff1": 0.6531413077725174, "naucs_at_100_max": 0.27848438926452895, "naucs_at_100_std": 0.06902233620201771, "naucs_at_100_diff1": 0.5818883206938849, "naucs_at_1000_max": 1.0, "naucs_at_1000_std": 1.0, "naucs_at_1000_diff1": 1.0 }, "vidore/docvqa_test_subsampled_ocr_chunk": { "ndcg_at_1": 0.28667, "ndcg_at_3": 0.33254, "ndcg_at_5": 0.3442, "ndcg_at_10": 0.36178, "ndcg_at_20": 0.36991, "ndcg_at_100": 0.39159, "ndcg_at_1000": 0.44347, "map_at_1": 0.28667, "map_at_3": 0.32148, "map_at_5": 0.32781, "map_at_10": 0.33484, "map_at_20": 0.33722, "map_at_100": 0.33972, "map_at_1000": 0.34188, "recall_at_1": 0.28667, "recall_at_3": 0.36444, "recall_at_5": 0.39333, "recall_at_10": 0.44889, "recall_at_20": 0.48, "recall_at_100": 0.60444, "recall_at_1000": 1.0, "precision_at_1": 0.28667, "precision_at_3": 0.12148, "precision_at_5": 0.07867, "precision_at_10": 0.04489, "precision_at_20": 0.024, "precision_at_100": 0.00604, "precision_at_1000": 0.001, "mrr_at_1": 0.28444444444444444, "mrr_at_3": 0.3199999999999999, "mrr_at_5": 0.32788888888888884, "mrr_at_10": 0.3348280423280422, "mrr_at_20": 0.33686708353375006, "mrr_at_100": 0.3393926825769153, "mrr_at_1000": 0.3415392967000492, "naucs_at_1_max": 0.483795422387148, "naucs_at_1_std": 0.3566081343634037, "naucs_at_1_diff1": 0.7680653646528732, "naucs_at_3_max": 0.4990281883568816, "naucs_at_3_std": 0.3485399548427641, "naucs_at_3_diff1": 0.7136559903167988, "naucs_at_5_max": 0.45057237723806814, "naucs_at_5_std": 0.31139674968934633, "naucs_at_5_diff1": 0.6757265560654193, "naucs_at_10_max": 0.426297509410252, "naucs_at_10_std": 0.26911866516955885, "naucs_at_10_diff1": 0.6434430759743316, "naucs_at_20_max": 0.4045758206602919, "naucs_at_20_std": 0.23759747362478373, "naucs_at_20_diff1": 0.6168183903179139, "naucs_at_100_max": 0.37411413527636356, "naucs_at_100_std": 0.1955550532516056, "naucs_at_100_diff1": 0.5594044595726576, "naucs_at_1000_max": 1.0, "naucs_at_1000_std": 1.0, "naucs_at_1000_diff1": 1.0 }, "vidore/infovqa_test_subsampled_ocr_chunk": { "ndcg_at_1": 0.62146, "ndcg_at_3": 0.67949, "ndcg_at_5": 0.69029, "ndcg_at_10": 0.7002, "ndcg_at_20": 0.70502, "ndcg_at_100": 0.71066, "ndcg_at_1000": 0.73195, "map_at_1": 0.62146, "map_at_3": 0.66464, "map_at_5": 0.67061, "map_at_10": 0.67476, "map_at_20": 0.67619, "map_at_100": 0.67698, "map_at_1000": 0.67766, "recall_at_1": 0.62146, "recall_at_3": 0.72267, "recall_at_5": 0.74899, "recall_at_10": 0.77935, "recall_at_20": 0.79757, "recall_at_100": 0.82794, "recall_at_1000": 1.0, "precision_at_1": 0.62146, "precision_at_3": 0.24089, "precision_at_5": 0.1498, "precision_at_10": 0.07794, "precision_at_20": 0.03988, "precision_at_100": 0.00828, "precision_at_1000": 0.001, "mrr_at_1": 0.6214574898785425, "mrr_at_3": 0.6649797570850202, "mrr_at_5": 0.6709514170040484, "mrr_at_10": 0.6749212775528564, "mrr_at_20": 0.6765598801783012, "mrr_at_100": 0.6773496139765206, "mrr_at_1000": 0.6779704928964845, "naucs_at_1_max": 0.4998263582080041, "naucs_at_1_std": 0.2569017395756454, "naucs_at_1_diff1": 0.8417425836037471, "naucs_at_3_max": 0.4860469821073518, "naucs_at_3_std": 0.2100170840198474, "naucs_at_3_diff1": 0.7790530771150608, "naucs_at_5_max": 0.4961713630202046, "naucs_at_5_std": 0.18924437943177114, "naucs_at_5_diff1": 0.7550867093153621, "naucs_at_10_max": 0.5354571732011593, "naucs_at_10_std": 0.22414513735170472, "naucs_at_10_diff1": 0.749972254388185, "naucs_at_20_max": 0.5824008570625927, "naucs_at_20_std": 0.22851761930324893, "naucs_at_20_diff1": 0.7311055690423088, "naucs_at_100_max": 0.6184215438597674, "naucs_at_100_std": 0.2390568151782533, "naucs_at_100_diff1": 0.7219124746182833, "naucs_at_1000_max": 1.0, "naucs_at_1000_std": 1.0, "naucs_at_1000_diff1": 1.0 }, "vidore/shiftproject_test_ocr_chunk": { "ndcg_at_1": 0.57, "ndcg_at_3": 0.69071, "ndcg_at_5": 0.73677, "ndcg_at_10": 0.75272, "ndcg_at_20": 0.76046, "ndcg_at_100": 0.76419, "ndcg_at_1000": 0.76567, "map_at_1": 0.57, "map_at_3": 0.66, "map_at_5": 0.686, "map_at_10": 0.69246, "map_at_20": 0.69466, "map_at_100": 0.69519, "map_at_1000": 0.69528, "recall_at_1": 0.57, "recall_at_3": 0.78, "recall_at_5": 0.89, "recall_at_10": 0.94, "recall_at_20": 0.97, "recall_at_100": 0.99, "recall_at_1000": 1.0, "precision_at_1": 0.57, "precision_at_3": 0.26, "precision_at_5": 0.178, "precision_at_10": 0.094, "precision_at_20": 0.0485, "precision_at_100": 0.0099, "precision_at_1000": 0.001, "mrr_at_1": 0.57, "mrr_at_3": 0.6600000000000001, "mrr_at_5": 0.686, "mrr_at_10": 0.6924563492063491, "mrr_at_20": 0.6946571067821068, "mrr_at_100": 0.6951922419172419, "mrr_at_1000": 0.695283985036508, "naucs_at_1_max": 0.06209727185315921, "naucs_at_1_std": -0.20209239247427543, "naucs_at_1_diff1": 0.7134369483790416, "naucs_at_3_max": 0.18727685816293413, "naucs_at_3_std": -0.08200027820280922, "naucs_at_3_diff1": 0.6528260768767103, "naucs_at_5_max": 0.5806631732669002, "naucs_at_5_std": 0.45082037625633586, "naucs_at_5_diff1": 0.7661712911261921, "naucs_at_10_max": 0.30843448490507197, "naucs_at_10_std": 0.2657173980703404, "naucs_at_10_diff1": 0.8102240896358563, "naucs_at_20_max": 0.08839091192031695, "naucs_at_20_std": 0.4609399315281665, "naucs_at_20_diff1": 0.7075941487706191, "naucs_at_100_max": 1.0, "naucs_at_100_std": 1.0, "naucs_at_100_diff1": 1.0, "naucs_at_1000_max": NaN, "naucs_at_1000_std": NaN, "naucs_at_1000_diff1": NaN }, "vidore/syntheticDocQA_artificial_intelligence_test_ocr_chunk": { "ndcg_at_1": 0.88, "ndcg_at_3": 0.93547, "ndcg_at_5": 0.93547, "ndcg_at_10": 0.93547, "ndcg_at_20": 0.93547, "ndcg_at_100": 0.93906, "ndcg_at_1000": 0.94027, "map_at_1": 0.88, "map_at_3": 0.92333, "map_at_5": 0.92333, "map_at_10": 0.92333, "map_at_20": 0.92333, "map_at_100": 0.9238, "map_at_1000": 0.92384, "recall_at_1": 0.88, "recall_at_3": 0.97, "recall_at_5": 0.97, "recall_at_10": 0.97, "recall_at_20": 0.97, "recall_at_100": 0.99, "recall_at_1000": 1.0, "precision_at_1": 0.88, "precision_at_3": 0.32333, "precision_at_5": 0.194, "precision_at_10": 0.097, "precision_at_20": 0.0485, "precision_at_100": 0.0099, "precision_at_1000": 0.001, "mrr_at_1": 0.88, "mrr_at_3": 0.9233333333333333, "mrr_at_5": 0.9233333333333333, "mrr_at_10": 0.9247619047619047, "mrr_at_20": 0.9247619047619047, "mrr_at_100": 0.9248629148629148, "mrr_at_1000": 0.924895069203751, "naucs_at_1_max": 0.3524498886414245, "naucs_at_1_std": -0.04589564110722234, "naucs_at_1_diff1": 0.9216512885777918, "naucs_at_3_max": 0.6498599439775861, "naucs_at_3_std": -0.2236227824463119, "naucs_at_3_diff1": 1.0, "naucs_at_5_max": 0.6498599439775937, "naucs_at_5_std": -0.22362278244631117, "naucs_at_5_diff1": 1.0, "naucs_at_10_max": 0.6498599439775937, "naucs_at_10_std": -0.22362278244631117, "naucs_at_10_diff1": 1.0, "naucs_at_20_max": 0.6498599439775937, "naucs_at_20_std": -0.22362278244631117, "naucs_at_20_diff1": 1.0, "naucs_at_100_max": 0.7222222222222041, "naucs_at_100_std": 0.12278244631185525, "naucs_at_100_diff1": 1.0, "naucs_at_1000_max": NaN, "naucs_at_1000_std": NaN, "naucs_at_1000_diff1": NaN }, "vidore/syntheticDocQA_energy_test_ocr_chunk": { "ndcg_at_1": 0.84, "ndcg_at_3": 0.86762, "ndcg_at_5": 0.87149, "ndcg_at_10": 0.88478, "ndcg_at_20": 0.89027, "ndcg_at_100": 0.89445, "ndcg_at_1000": 0.89697, "map_at_1": 0.84, "map_at_3": 0.86, "map_at_5": 0.862, "map_at_10": 0.86769, "map_at_20": 0.86944, "map_at_100": 0.87019, "map_at_1000": 0.87029, "recall_at_1": 0.84, "recall_at_3": 0.89, "recall_at_5": 0.9, "recall_at_10": 0.94, "recall_at_20": 0.96, "recall_at_100": 0.98, "recall_at_1000": 1.0, "precision_at_1": 0.84, "precision_at_3": 0.29667, "precision_at_5": 0.18, "precision_at_10": 0.094, "precision_at_20": 0.048, "precision_at_100": 0.0098, "precision_at_1000": 0.001, "mrr_at_1": 0.84, "mrr_at_3": 0.865, "mrr_at_5": 0.867, "mrr_at_10": 0.8710277777777778, "mrr_at_20": 0.872770202020202, "mrr_at_100": 0.8735251877751878, "mrr_at_1000": 0.8736228820308191, "naucs_at_1_max": 0.4240611192930788, "naucs_at_1_std": 0.07081492390770726, "naucs_at_1_diff1": 0.80780559646539, "naucs_at_3_max": 0.688428829138389, "naucs_at_3_std": 0.39983678378146165, "naucs_at_3_diff1": 0.9113048707155739, "naucs_at_5_max": 0.6613445378151257, "naucs_at_5_std": 0.3476657329598507, "naucs_at_5_diff1": 0.8905228758169933, "naucs_at_10_max": 0.564347961406786, "naucs_at_10_std": 0.4709772798008099, "naucs_at_10_diff1": 0.91013071895425, "naucs_at_20_max": 0.5764472455648917, "naucs_at_20_std": 0.42938842203548094, "naucs_at_20_diff1": 0.967320261437907, "naucs_at_100_max": 1.0, "naucs_at_100_std": 0.9346405228758174, "naucs_at_100_diff1": 1.0, "naucs_at_1000_max": NaN, "naucs_at_1000_std": NaN, "naucs_at_1000_diff1": NaN }, "vidore/syntheticDocQA_government_reports_test_ocr_chunk": { "ndcg_at_1": 0.79, "ndcg_at_3": 0.85417, "ndcg_at_5": 0.87052, "ndcg_at_10": 0.877, "ndcg_at_20": 0.877, "ndcg_at_100": 0.8802, "ndcg_at_1000": 0.88301, "map_at_1": 0.79, "map_at_3": 0.83833, "map_at_5": 0.84733, "map_at_10": 0.85001, "map_at_20": 0.85001, "map_at_100": 0.85028, "map_at_1000": 0.85043, "recall_at_1": 0.79, "recall_at_3": 0.9, "recall_at_5": 0.94, "recall_at_10": 0.96, "recall_at_20": 0.96, "recall_at_100": 0.98, "recall_at_1000": 1.0, "precision_at_1": 0.79, "precision_at_3": 0.3, "precision_at_5": 0.188, "precision_at_10": 0.096, "precision_at_20": 0.048, "precision_at_100": 0.0098, "precision_at_1000": 0.001, "mrr_at_1": 0.79, "mrr_at_3": 0.8383333333333334, "mrr_at_5": 0.8473333333333335, "mrr_at_10": 0.8500119047619049, "mrr_at_20": 0.8500119047619049, "mrr_at_100": 0.8505455642830755, "mrr_at_1000": 0.8506072926781373, "naucs_at_1_max": 0.391155890061798, "naucs_at_1_std": 0.16375309591939743, "naucs_at_1_diff1": 0.8795296607112802, "naucs_at_3_max": 0.4788515406162445, "naucs_at_3_std": 0.16269841269841115, "naucs_at_3_diff1": 0.7833800186741333, "naucs_at_5_max": 0.6713352007469663, "naucs_at_5_std": 0.27816682228446654, "naucs_at_5_diff1": 0.6825396825396852, "naucs_at_10_max": 0.5764472455648917, "naucs_at_10_std": 0.24276377217553516, "naucs_at_10_diff1": 0.625933706816059, "naucs_at_20_max": 0.5764472455648917, "naucs_at_20_std": 0.24276377217553516, "naucs_at_20_diff1": 0.625933706816059, "naucs_at_100_max": 0.9346405228758174, "naucs_at_100_std": 0.561391223155939, "naucs_at_100_diff1": 0.4561157796451911, "naucs_at_1000_max": NaN, "naucs_at_1000_std": NaN, "naucs_at_1000_diff1": NaN }, "vidore/syntheticDocQA_healthcare_industry_test_ocr_chunk": { "ndcg_at_1": 0.81, "ndcg_at_3": 0.87547, "ndcg_at_5": 0.87934, "ndcg_at_10": 0.89246, "ndcg_at_20": 0.89502, "ndcg_at_100": 0.89663, "ndcg_at_1000": 0.89792, "map_at_1": 0.81, "map_at_3": 0.86, "map_at_5": 0.862, "map_at_10": 0.86752, "map_at_20": 0.86824, "map_at_100": 0.86838, "map_at_1000": 0.86842, "recall_at_1": 0.81, "recall_at_3": 0.92, "recall_at_5": 0.93, "recall_at_10": 0.97, "recall_at_20": 0.98, "recall_at_100": 0.99, "recall_at_1000": 1.0, "precision_at_1": 0.81, "precision_at_3": 0.30667, "precision_at_5": 0.186, "precision_at_10": 0.097, "precision_at_20": 0.049, "precision_at_100": 0.0099, "precision_at_1000": 0.001, "mrr_at_1": 0.81, "mrr_at_3": 0.86, "mrr_at_5": 0.862, "mrr_at_10": 0.8687738095238096, "mrr_at_20": 0.8694880952380953, "mrr_at_100": 0.8694880952380953, "mrr_at_1000": 0.8695333441068734, "naucs_at_1_max": 0.43229084813243124, "naucs_at_1_std": 0.14869820315364835, "naucs_at_1_diff1": 0.9104458064854107, "naucs_at_3_max": 0.6793884220354839, "naucs_at_3_std": 0.17886321195144628, "naucs_at_3_diff1": 0.8304738562091516, "naucs_at_5_max": 0.6732693077230891, "naucs_at_5_std": 0.061557956515943035, "naucs_at_5_diff1": 0.8459383753501409, "naucs_at_10_max": 0.8638344226579531, "naucs_at_10_std": 0.1031746031746046, "naucs_at_10_diff1": 0.8148148148148106, "naucs_at_20_max": 0.9346405228758136, "naucs_at_20_std": 0.09337068160597826, "naucs_at_20_diff1": 0.722222222222224, "naucs_at_100_max": 1.0, "naucs_at_100_std": -0.17133520074697067, "naucs_at_100_diff1": 0.7222222222222041, "naucs_at_1000_max": NaN, "naucs_at_1000_std": NaN, "naucs_at_1000_diff1": NaN }, "vidore/tabfquad_test_subsampled_ocr_chunk": { "ndcg_at_1": 0.61071, "ndcg_at_3": 0.73478, "ndcg_at_5": 0.7549, "ndcg_at_10": 0.7715, "ndcg_at_20": 0.78226, "ndcg_at_100": 0.78902, "ndcg_at_1000": 0.78902, "map_at_1": 0.61071, "map_at_3": 0.70476, "map_at_5": 0.71565, "map_at_10": 0.72275, "map_at_20": 0.72567, "map_at_100": 0.72663, "map_at_1000": 0.72663, "recall_at_1": 0.61071, "recall_at_3": 0.82143, "recall_at_5": 0.87143, "recall_at_10": 0.92143, "recall_at_20": 0.96429, "recall_at_100": 1.0, "recall_at_1000": 1.0, "precision_at_1": 0.61071, "precision_at_3": 0.27381, "precision_at_5": 0.17429, "precision_at_10": 0.09214, "precision_at_20": 0.04821, "precision_at_100": 0.01, "precision_at_1000": 0.001, "mrr_at_1": 0.6107142857142858, "mrr_at_3": 0.7029761904761906, "mrr_at_5": 0.7151190476190477, "mrr_at_10": 0.7228741496598639, "mrr_at_20": 0.7258115655168949, "mrr_at_100": 0.7267819541007899, "mrr_at_1000": 0.7267819541007899, "naucs_at_1_max": 0.21190904523427462, "naucs_at_1_std": -0.08874518511315357, "naucs_at_1_diff1": 0.7049048367665566, "naucs_at_3_max": 0.3371705963938968, "naucs_at_3_std": 0.054695858926094264, "naucs_at_3_diff1": 0.6250049534376857, "naucs_at_5_max": 0.3066566940912499, "naucs_at_5_std": 0.016187626883213884, "naucs_at_5_diff1": 0.5970589806603274, "naucs_at_10_max": 0.165923945335711, "naucs_at_10_std": -0.08458534929123303, "naucs_at_10_diff1": 0.5313640607758263, "naucs_at_20_max": -0.119467787114844, "naucs_at_20_std": -0.43529411764705933, "naucs_at_20_diff1": 0.3754435107376275, "naucs_at_100_max": 1.0, "naucs_at_100_std": 1.0, "naucs_at_100_diff1": 1.0, "naucs_at_1000_max": NaN, "naucs_at_1000_std": NaN, "naucs_at_1000_diff1": NaN }, "vidore/tatdqa_test_ocr_chunk": { "ndcg_at_1": 0.43742, "ndcg_at_3": 0.53367, "ndcg_at_5": 0.55939, "ndcg_at_10": 0.58704, "ndcg_at_20": 0.60657, "ndcg_at_100": 0.63067, "ndcg_at_1000": 0.63671, "map_at_1": 0.43742, "map_at_3": 0.51002, "map_at_5": 0.52442, "map_at_10": 0.53591, "map_at_20": 0.54128, "map_at_100": 0.54462, "map_at_1000": 0.54492, "recall_at_1": 0.43742, "recall_at_3": 0.60207, "recall_at_5": 0.66403, "recall_at_10": 0.74909, "recall_at_20": 0.82625, "recall_at_100": 0.95626, "recall_at_1000": 1.0, "precision_at_1": 0.43742, "precision_at_3": 0.20069, "precision_at_5": 0.13281, "precision_at_10": 0.07491, "precision_at_20": 0.04131, "precision_at_100": 0.00956, "precision_at_1000": 0.001, "mrr_at_1": 0.43438639125151884, "mrr_at_3": 0.5090117456460109, "mrr_at_5": 0.5233799108950998, "mrr_at_10": 0.534759590348898, "mrr_at_20": 0.5401616883418155, "mrr_at_100": 0.5434718934891516, "mrr_at_1000": 0.5437690150069657, "naucs_at_1_max": 0.31026721156475245, "naucs_at_1_std": 0.17818955102750508, "naucs_at_1_diff1": 0.7064803750501206, "naucs_at_3_max": 0.3261289119541843, "naucs_at_3_std": 0.2066116158308213, "naucs_at_3_diff1": 0.6039682615859476, "naucs_at_5_max": 0.3495082942293466, "naucs_at_5_std": 0.24702763539342892, "naucs_at_5_diff1": 0.5944595048138549, "naucs_at_10_max": 0.3537468759854116, "naucs_at_10_std": 0.26880422433666223, "naucs_at_10_diff1": 0.5758303676132953, "naucs_at_20_max": 0.36419916364409105, "naucs_at_20_std": 0.308367420268763, "naucs_at_20_diff1": 0.5312382086481554, "naucs_at_100_max": 0.39447774079185205, "naucs_at_100_std": 0.3787028942579173, "naucs_at_100_diff1": 0.5770484457445563, "naucs_at_1000_max": NaN, "naucs_at_1000_std": NaN, "naucs_at_1000_diff1": NaN } }