| { | |
| "best_metric": null, | |
| "best_model_checkpoint": null, | |
| "epoch": 1.0, | |
| "eval_steps": 100, | |
| "global_step": 329, | |
| "is_hyper_param_search": false, | |
| "is_local_process_zero": true, | |
| "is_world_process_zero": true, | |
| "log_history": [ | |
| { | |
| "dpo_losses": 0.6931471824645996, | |
| "epoch": 0.0, | |
| "grad_norm": 2.094792348544482, | |
| "learning_rate": 1.5151515151515152e-07, | |
| "logits/chosen": -2.6820077896118164, | |
| "logits/rejected": -2.6930205821990967, | |
| "logps/chosen": -281.2528381347656, | |
| "logps/rejected": -258.0622253417969, | |
| "loss": 0.6931, | |
| "positive_losses": 0.0, | |
| "rewards/accuracies": 0.0, | |
| "rewards/chosen": 0.0, | |
| "rewards/margins": 0.0, | |
| "rewards/margins_max": 0.0, | |
| "rewards/margins_min": 0.0, | |
| "rewards/margins_std": 0.0, | |
| "rewards/rejected": 0.0, | |
| "step": 1 | |
| }, | |
| { | |
| "dpo_losses": 0.6931204795837402, | |
| "epoch": 0.03, | |
| "grad_norm": 7.907274986440123, | |
| "learning_rate": 1.5151515151515152e-06, | |
| "logits/chosen": -2.7687692642211914, | |
| "logits/rejected": -2.7543087005615234, | |
| "logps/chosen": -284.4098815917969, | |
| "logps/rejected": -249.6673126220703, | |
| "loss": 0.6967, | |
| "positive_losses": 0.03447924554347992, | |
| "rewards/accuracies": 0.4583333432674408, | |
| "rewards/chosen": 0.0019425028003752232, | |
| "rewards/margins": 5.710232289857231e-05, | |
| "rewards/margins_max": 0.003333130618557334, | |
| "rewards/margins_min": -0.0030696813482791185, | |
| "rewards/margins_std": 0.002927461639046669, | |
| "rewards/rejected": 0.0018854006193578243, | |
| "step": 10 | |
| }, | |
| { | |
| "dpo_losses": 0.6927222609519958, | |
| "epoch": 0.06, | |
| "grad_norm": 1.9785585238747578, | |
| "learning_rate": 3.0303030303030305e-06, | |
| "logits/chosen": -2.8341968059539795, | |
| "logits/rejected": -2.7815167903900146, | |
| "logps/chosen": -289.8644714355469, | |
| "logps/rejected": -268.81475830078125, | |
| "loss": 0.6935, | |
| "positive_losses": 0.00822601281106472, | |
| "rewards/accuracies": 0.5, | |
| "rewards/chosen": 0.016672641038894653, | |
| "rewards/margins": 0.0008590976940467954, | |
| "rewards/margins_max": 0.007572791539132595, | |
| "rewards/margins_min": -0.004768169019371271, | |
| "rewards/margins_std": 0.0055806804448366165, | |
| "rewards/rejected": 0.015813542529940605, | |
| "step": 20 | |
| }, | |
| { | |
| "dpo_losses": 0.6912546157836914, | |
| "epoch": 0.09, | |
| "grad_norm": 9.993422824831308, | |
| "learning_rate": 4.5454545454545455e-06, | |
| "logits/chosen": -2.86173152923584, | |
| "logits/rejected": -2.814188241958618, | |
| "logps/chosen": -255.656494140625, | |
| "logps/rejected": -224.1510467529297, | |
| "loss": 0.692, | |
| "positive_losses": 0.00018234252638649195, | |
| "rewards/accuracies": 0.5625, | |
| "rewards/chosen": 0.03588816151022911, | |
| "rewards/margins": 0.00383453955873847, | |
| "rewards/margins_max": 0.017018133774399757, | |
| "rewards/margins_min": -0.008637982420623302, | |
| "rewards/margins_std": 0.011648830957710743, | |
| "rewards/rejected": 0.0320536233484745, | |
| "step": 30 | |
| }, | |
| { | |
| "dpo_losses": 0.6852566003799438, | |
| "epoch": 0.12, | |
| "grad_norm": 7.551494566668885, | |
| "learning_rate": 4.993103596812269e-06, | |
| "logits/chosen": -2.8286097049713135, | |
| "logits/rejected": -2.7643656730651855, | |
| "logps/chosen": -311.00653076171875, | |
| "logps/rejected": -219.75357055664062, | |
| "loss": 0.6928, | |
| "positive_losses": 0.0, | |
| "rewards/accuracies": 0.6875, | |
| "rewards/chosen": 0.06535681337118149, | |
| "rewards/margins": 0.016044817864894867, | |
| "rewards/margins_max": 0.04139421135187149, | |
| "rewards/margins_min": -0.006711454130709171, | |
| "rewards/margins_std": 0.021322522312402725, | |
| "rewards/rejected": 0.049311988055706024, | |
| "step": 40 | |
| }, | |
| { | |
| "dpo_losses": 0.6857399940490723, | |
| "epoch": 0.15, | |
| "grad_norm": 7.613114395779342, | |
| "learning_rate": 4.95941685833271e-06, | |
| "logits/chosen": -2.7872977256774902, | |
| "logits/rejected": -2.8009393215179443, | |
| "logps/chosen": -235.7885284423828, | |
| "logps/rejected": -274.2627258300781, | |
| "loss": 0.6875, | |
| "positive_losses": 0.015755081549286842, | |
| "rewards/accuracies": 0.6875, | |
| "rewards/chosen": 0.07146342098712921, | |
| "rewards/margins": 0.0152506772428751, | |
| "rewards/margins_max": 0.058862246572971344, | |
| "rewards/margins_min": -0.022146200761198997, | |
| "rewards/margins_std": 0.03581809252500534, | |
| "rewards/rejected": 0.056212734431028366, | |
| "step": 50 | |
| }, | |
| { | |
| "dpo_losses": 0.680141806602478, | |
| "epoch": 0.18, | |
| "grad_norm": 7.505695469277918, | |
| "learning_rate": 4.898051734555676e-06, | |
| "logits/chosen": -2.8290228843688965, | |
| "logits/rejected": -2.839033603668213, | |
| "logps/chosen": -312.344482421875, | |
| "logps/rejected": -276.32183837890625, | |
| "loss": 0.6846, | |
| "positive_losses": 0.05971794202923775, | |
| "rewards/accuracies": 0.6625000238418579, | |
| "rewards/chosen": 0.0957661122083664, | |
| "rewards/margins": 0.027223944664001465, | |
| "rewards/margins_max": 0.09758423268795013, | |
| "rewards/margins_min": -0.04277171194553375, | |
| "rewards/margins_std": 0.061675649136304855, | |
| "rewards/rejected": 0.06854216754436493, | |
| "step": 60 | |
| }, | |
| { | |
| "dpo_losses": 0.6815682649612427, | |
| "epoch": 0.21, | |
| "grad_norm": 1.5360756176524522, | |
| "learning_rate": 4.809698831278217e-06, | |
| "logits/chosen": -2.736055374145508, | |
| "logits/rejected": -2.7235770225524902, | |
| "logps/chosen": -257.50408935546875, | |
| "logps/rejected": -239.8359375, | |
| "loss": 0.694, | |
| "positive_losses": 0.1467689573764801, | |
| "rewards/accuracies": 0.637499988079071, | |
| "rewards/chosen": 0.08973346650600433, | |
| "rewards/margins": 0.024347994476556778, | |
| "rewards/margins_max": 0.0899825394153595, | |
| "rewards/margins_min": -0.03823970630764961, | |
| "rewards/margins_std": 0.05780817195773125, | |
| "rewards/rejected": 0.06538547575473785, | |
| "step": 70 | |
| }, | |
| { | |
| "dpo_losses": 0.6777443885803223, | |
| "epoch": 0.24, | |
| "grad_norm": 7.6098783040686255, | |
| "learning_rate": 4.695352475952706e-06, | |
| "logits/chosen": -2.825540542602539, | |
| "logits/rejected": -2.8001182079315186, | |
| "logps/chosen": -272.2143859863281, | |
| "logps/rejected": -268.1064147949219, | |
| "loss": 0.6843, | |
| "positive_losses": 0.019883345812559128, | |
| "rewards/accuracies": 0.7749999761581421, | |
| "rewards/chosen": 0.10059104114770889, | |
| "rewards/margins": 0.03249071165919304, | |
| "rewards/margins_max": 0.10248656570911407, | |
| "rewards/margins_min": -0.04477204009890556, | |
| "rewards/margins_std": 0.06633640825748444, | |
| "rewards/rejected": 0.06810032576322556, | |
| "step": 80 | |
| }, | |
| { | |
| "dpo_losses": 0.6827337741851807, | |
| "epoch": 0.27, | |
| "grad_norm": 13.032468304298272, | |
| "learning_rate": 4.556299527482029e-06, | |
| "logits/chosen": -2.7769341468811035, | |
| "logits/rejected": -2.7250094413757324, | |
| "logps/chosen": -285.997314453125, | |
| "logps/rejected": -284.3042297363281, | |
| "loss": 0.6924, | |
| "positive_losses": 0.0755772590637207, | |
| "rewards/accuracies": 0.612500011920929, | |
| "rewards/chosen": 0.09554663300514221, | |
| "rewards/margins": 0.022927606478333473, | |
| "rewards/margins_max": 0.11027027666568756, | |
| "rewards/margins_min": -0.06920486688613892, | |
| "rewards/margins_std": 0.0804494172334671, | |
| "rewards/rejected": 0.07261903584003448, | |
| "step": 90 | |
| }, | |
| { | |
| "dpo_losses": 0.6703774333000183, | |
| "epoch": 0.3, | |
| "grad_norm": 1.6704382025698337, | |
| "learning_rate": 4.394104893853007e-06, | |
| "logits/chosen": -2.8705239295959473, | |
| "logits/rejected": -2.832496166229248, | |
| "logps/chosen": -262.93939208984375, | |
| "logps/rejected": -251.376953125, | |
| "loss": 0.6896, | |
| "positive_losses": 0.10124854743480682, | |
| "rewards/accuracies": 0.762499988079071, | |
| "rewards/chosen": 0.10522875934839249, | |
| "rewards/margins": 0.04796325042843819, | |
| "rewards/margins_max": 0.13638350367546082, | |
| "rewards/margins_min": -0.02367212623357773, | |
| "rewards/margins_std": 0.07211659848690033, | |
| "rewards/rejected": 0.0572655126452446, | |
| "step": 100 | |
| }, | |
| { | |
| "epoch": 0.3, | |
| "eval_dpo_losses": 0.6766990423202515, | |
| "eval_logits/chosen": -2.782886028289795, | |
| "eval_logits/rejected": -2.7451796531677246, | |
| "eval_logps/chosen": -271.81536865234375, | |
| "eval_logps/rejected": -249.28070068359375, | |
| "eval_loss": 0.692669689655304, | |
| "eval_positive_losses": 0.05891977623105049, | |
| "eval_rewards/accuracies": 0.6710000038146973, | |
| "eval_rewards/chosen": 0.12778052687644958, | |
| "eval_rewards/margins": 0.03479834645986557, | |
| "eval_rewards/margins_max": 0.15719929337501526, | |
| "eval_rewards/margins_min": -0.0766773670911789, | |
| "eval_rewards/margins_std": 0.07707200944423676, | |
| "eval_rewards/rejected": 0.09298218786716461, | |
| "eval_runtime": 429.6713, | |
| "eval_samples_per_second": 4.655, | |
| "eval_steps_per_second": 0.291, | |
| "step": 100 | |
| }, | |
| { | |
| "dpo_losses": 0.669114887714386, | |
| "epoch": 0.33, | |
| "grad_norm": 2.0588703005951716, | |
| "learning_rate": 4.210593920593201e-06, | |
| "logits/chosen": -2.732736110687256, | |
| "logits/rejected": -2.717144727706909, | |
| "logps/chosen": -297.5173645019531, | |
| "logps/rejected": -227.2909698486328, | |
| "loss": 0.6716, | |
| "positive_losses": 0.005372238345444202, | |
| "rewards/accuracies": 0.6875, | |
| "rewards/chosen": 0.14238855242729187, | |
| "rewards/margins": 0.06108074635267258, | |
| "rewards/margins_max": 0.21817930042743683, | |
| "rewards/margins_min": -0.03649354726076126, | |
| "rewards/margins_std": 0.11923160403966904, | |
| "rewards/rejected": 0.08130781352519989, | |
| "step": 110 | |
| }, | |
| { | |
| "dpo_losses": 0.6759502291679382, | |
| "epoch": 0.36, | |
| "grad_norm": 2.114750885236866, | |
| "learning_rate": 4.007831848252212e-06, | |
| "logits/chosen": -2.71893572807312, | |
| "logits/rejected": -2.7199361324310303, | |
| "logps/chosen": -312.4023132324219, | |
| "logps/rejected": -266.9503479003906, | |
| "loss": 0.6841, | |
| "positive_losses": 0.1982906311750412, | |
| "rewards/accuracies": 0.637499988079071, | |
| "rewards/chosen": 0.11162187904119492, | |
| "rewards/margins": 0.03713056445121765, | |
| "rewards/margins_max": 0.13481460511684418, | |
| "rewards/margins_min": -0.06108905002474785, | |
| "rewards/margins_std": 0.08896367251873016, | |
| "rewards/rejected": 0.07449130713939667, | |
| "step": 120 | |
| }, | |
| { | |
| "dpo_losses": 0.6827476024627686, | |
| "epoch": 0.4, | |
| "grad_norm": 1.6378871780311928, | |
| "learning_rate": 3.7881005700938635e-06, | |
| "logits/chosen": -2.7933266162872314, | |
| "logits/rejected": -2.803476333618164, | |
| "logps/chosen": -254.1282196044922, | |
| "logps/rejected": -224.1819610595703, | |
| "loss": 0.6872, | |
| "positive_losses": 0.06588058173656464, | |
| "rewards/accuracies": 0.612500011920929, | |
| "rewards/chosen": 0.12265358120203018, | |
| "rewards/margins": 0.023296961560845375, | |
| "rewards/margins_max": 0.1208866611123085, | |
| "rewards/margins_min": -0.07595702260732651, | |
| "rewards/margins_std": 0.08657418191432953, | |
| "rewards/rejected": 0.09935661405324936, | |
| "step": 130 | |
| }, | |
| { | |
| "dpo_losses": 0.6661409735679626, | |
| "epoch": 0.43, | |
| "grad_norm": 1.861594662390886, | |
| "learning_rate": 3.553872951569236e-06, | |
| "logits/chosen": -2.7491869926452637, | |
| "logits/rejected": -2.7360832691192627, | |
| "logps/chosen": -280.62664794921875, | |
| "logps/rejected": -262.3188781738281, | |
| "loss": 0.6745, | |
| "positive_losses": 0.11735143512487411, | |
| "rewards/accuracies": 0.737500011920929, | |
| "rewards/chosen": 0.13766564428806305, | |
| "rewards/margins": 0.05706212669610977, | |
| "rewards/margins_max": 0.1553352326154709, | |
| "rewards/margins_min": -0.027615731582045555, | |
| "rewards/margins_std": 0.08170460909605026, | |
| "rewards/rejected": 0.08060351014137268, | |
| "step": 140 | |
| }, | |
| { | |
| "dpo_losses": 0.6667992472648621, | |
| "epoch": 0.46, | |
| "grad_norm": 1.3867667176452936, | |
| "learning_rate": 3.307785000580313e-06, | |
| "logits/chosen": -2.8069396018981934, | |
| "logits/rejected": -2.788170099258423, | |
| "logps/chosen": -256.98443603515625, | |
| "logps/rejected": -237.07907104492188, | |
| "loss": 0.6927, | |
| "positive_losses": 0.10638203471899033, | |
| "rewards/accuracies": 0.762499988079071, | |
| "rewards/chosen": 0.1365698128938675, | |
| "rewards/margins": 0.05615685135126114, | |
| "rewards/margins_max": 0.1702376753091812, | |
| "rewards/margins_min": -0.057074159383773804, | |
| "rewards/margins_std": 0.099973663687706, | |
| "rewards/rejected": 0.08041295409202576, | |
| "step": 150 | |
| }, | |
| { | |
| "dpo_losses": 0.678627610206604, | |
| "epoch": 0.49, | |
| "grad_norm": 2.064113448248422, | |
| "learning_rate": 3.052606201731325e-06, | |
| "logits/chosen": -2.7706875801086426, | |
| "logits/rejected": -2.7537920475006104, | |
| "logps/chosen": -240.9077606201172, | |
| "logps/rejected": -229.16552734375, | |
| "loss": 0.6928, | |
| "positive_losses": 0.03613028675317764, | |
| "rewards/accuracies": 0.574999988079071, | |
| "rewards/chosen": 0.14596374332904816, | |
| "rewards/margins": 0.03164920210838318, | |
| "rewards/margins_max": 0.13097994029521942, | |
| "rewards/margins_min": -0.0587020106613636, | |
| "rewards/margins_std": 0.08477158099412918, | |
| "rewards/rejected": 0.11431453377008438, | |
| "step": 160 | |
| }, | |
| { | |
| "dpo_losses": 0.6667446494102478, | |
| "epoch": 0.52, | |
| "grad_norm": 1.6001303619683518, | |
| "learning_rate": 2.7912083484274266e-06, | |
| "logits/chosen": -2.7891182899475098, | |
| "logits/rejected": -2.7103054523468018, | |
| "logps/chosen": -287.3363342285156, | |
| "logps/rejected": -262.26116943359375, | |
| "loss": 0.67, | |
| "positive_losses": 0.011825943365693092, | |
| "rewards/accuracies": 0.7124999761581421, | |
| "rewards/chosen": 0.16382232308387756, | |
| "rewards/margins": 0.05722727254033089, | |
| "rewards/margins_max": 0.15611840784549713, | |
| "rewards/margins_min": -0.05285083130002022, | |
| "rewards/margins_std": 0.09372057765722275, | |
| "rewards/rejected": 0.10659502446651459, | |
| "step": 170 | |
| }, | |
| { | |
| "dpo_losses": 0.674170196056366, | |
| "epoch": 0.55, | |
| "grad_norm": 9.426032570856917, | |
| "learning_rate": 2.526533223585641e-06, | |
| "logits/chosen": -2.815913438796997, | |
| "logits/rejected": -2.7526583671569824, | |
| "logps/chosen": -241.7661590576172, | |
| "logps/rejected": -218.5923614501953, | |
| "loss": 0.6845, | |
| "positive_losses": 0.0364038459956646, | |
| "rewards/accuracies": 0.7124999761581421, | |
| "rewards/chosen": 0.1436757743358612, | |
| "rewards/margins": 0.040739208459854126, | |
| "rewards/margins_max": 0.14018966257572174, | |
| "rewards/margins_min": -0.03748547285795212, | |
| "rewards/margins_std": 0.07952649146318436, | |
| "rewards/rejected": 0.10293656587600708, | |
| "step": 180 | |
| }, | |
| { | |
| "dpo_losses": 0.6727156639099121, | |
| "epoch": 0.58, | |
| "grad_norm": 9.828820911992421, | |
| "learning_rate": 2.2615594926807554e-06, | |
| "logits/chosen": -2.754612922668457, | |
| "logits/rejected": -2.739302158355713, | |
| "logps/chosen": -284.7493896484375, | |
| "logps/rejected": -259.38885498046875, | |
| "loss": 0.7021, | |
| "positive_losses": 0.08002634346485138, | |
| "rewards/accuracies": 0.675000011920929, | |
| "rewards/chosen": 0.1583152711391449, | |
| "rewards/margins": 0.04616940766572952, | |
| "rewards/margins_max": 0.19369181990623474, | |
| "rewards/margins_min": -0.09055180847644806, | |
| "rewards/margins_std": 0.12469442188739777, | |
| "rewards/rejected": 0.11214585602283478, | |
| "step": 190 | |
| }, | |
| { | |
| "dpo_losses": 0.6642587184906006, | |
| "epoch": 0.61, | |
| "grad_norm": 12.372285795702703, | |
| "learning_rate": 1.9992691817133025e-06, | |
| "logits/chosen": -2.760648727416992, | |
| "logits/rejected": -2.7311196327209473, | |
| "logps/chosen": -267.6399230957031, | |
| "logps/rejected": -280.528076171875, | |
| "loss": 0.6919, | |
| "positive_losses": 0.2932780385017395, | |
| "rewards/accuracies": 0.6875, | |
| "rewards/chosen": 0.13960692286491394, | |
| "rewards/margins": 0.06209836155176163, | |
| "rewards/margins_max": 0.1877867430448532, | |
| "rewards/margins_min": -0.06410057097673416, | |
| "rewards/margins_std": 0.11066991090774536, | |
| "rewards/rejected": 0.07750856131315231, | |
| "step": 200 | |
| }, | |
| { | |
| "epoch": 0.61, | |
| "eval_dpo_losses": 0.6688500046730042, | |
| "eval_logits/chosen": -2.785562753677368, | |
| "eval_logits/rejected": -2.7480554580688477, | |
| "eval_logps/chosen": -270.2085876464844, | |
| "eval_logps/rejected": -249.3992919921875, | |
| "eval_loss": 0.6864475011825562, | |
| "eval_positive_losses": 0.09515316039323807, | |
| "eval_rewards/accuracies": 0.6830000281333923, | |
| "eval_rewards/chosen": 0.1438482403755188, | |
| "eval_rewards/margins": 0.052051883190870285, | |
| "eval_rewards/margins_max": 0.21146319806575775, | |
| "eval_rewards/margins_min": -0.09425032883882523, | |
| "eval_rewards/margins_std": 0.10122820734977722, | |
| "eval_rewards/rejected": 0.09179634600877762, | |
| "eval_runtime": 428.7542, | |
| "eval_samples_per_second": 4.665, | |
| "eval_steps_per_second": 0.292, | |
| "step": 200 | |
| }, | |
| { | |
| "dpo_losses": 0.6639125943183899, | |
| "epoch": 0.64, | |
| "grad_norm": 5.938345116150278, | |
| "learning_rate": 1.742614117358029e-06, | |
| "logits/chosen": -2.7786309719085693, | |
| "logits/rejected": -2.737117290496826, | |
| "logps/chosen": -291.00030517578125, | |
| "logps/rejected": -280.262451171875, | |
| "loss": 0.6787, | |
| "positive_losses": 0.15116862952709198, | |
| "rewards/accuracies": 0.6625000238418579, | |
| "rewards/chosen": 0.14316870272159576, | |
| "rewards/margins": 0.06285777688026428, | |
| "rewards/margins_max": 0.1971181035041809, | |
| "rewards/margins_min": -0.05821852758526802, | |
| "rewards/margins_std": 0.11372953653335571, | |
| "rewards/rejected": 0.08031092584133148, | |
| "step": 210 | |
| }, | |
| { | |
| "dpo_losses": 0.6715448498725891, | |
| "epoch": 0.67, | |
| "grad_norm": 1.9113410130667947, | |
| "learning_rate": 1.4944827069769125e-06, | |
| "logits/chosen": -2.8290793895721436, | |
| "logits/rejected": -2.8052849769592285, | |
| "logps/chosen": -296.83233642578125, | |
| "logps/rejected": -254.8493194580078, | |
| "loss": 0.6738, | |
| "positive_losses": 0.0, | |
| "rewards/accuracies": 0.675000011920929, | |
| "rewards/chosen": 0.16140839457511902, | |
| "rewards/margins": 0.04584597796201706, | |
| "rewards/margins_max": 0.14889702200889587, | |
| "rewards/margins_min": -0.04799387976527214, | |
| "rewards/margins_std": 0.08770614862442017, | |
| "rewards/rejected": 0.11556243896484375, | |
| "step": 220 | |
| }, | |
| { | |
| "dpo_losses": 0.6808522939682007, | |
| "epoch": 0.7, | |
| "grad_norm": 14.916786804283728, | |
| "learning_rate": 1.257667432355893e-06, | |
| "logits/chosen": -2.79703950881958, | |
| "logits/rejected": -2.8196463584899902, | |
| "logps/chosen": -274.06671142578125, | |
| "logps/rejected": -251.3921661376953, | |
| "loss": 0.6919, | |
| "positive_losses": 0.1536567509174347, | |
| "rewards/accuracies": 0.550000011920929, | |
| "rewards/chosen": 0.1457880586385727, | |
| "rewards/margins": 0.032118141651153564, | |
| "rewards/margins_max": 0.19181016087532043, | |
| "rewards/margins_min": -0.09802166372537613, | |
| "rewards/margins_std": 0.13066089153289795, | |
| "rewards/rejected": 0.11366990953683853, | |
| "step": 230 | |
| }, | |
| { | |
| "dpo_losses": 0.6678847074508667, | |
| "epoch": 0.73, | |
| "grad_norm": 2.0373215532852718, | |
| "learning_rate": 1.0348334229922677e-06, | |
| "logits/chosen": -2.854497194290161, | |
| "logits/rejected": -2.8086347579956055, | |
| "logps/chosen": -276.0729064941406, | |
| "logps/rejected": -265.6085510253906, | |
| "loss": 0.689, | |
| "positive_losses": 0.08907928317785263, | |
| "rewards/accuracies": 0.7250000238418579, | |
| "rewards/chosen": 0.14944827556610107, | |
| "rewards/margins": 0.05373426154255867, | |
| "rewards/margins_max": 0.16546599566936493, | |
| "rewards/margins_min": -0.03850485011935234, | |
| "rewards/margins_std": 0.093926340341568, | |
| "rewards/rejected": 0.0957140102982521, | |
| "step": 240 | |
| }, | |
| { | |
| "dpo_losses": 0.6652002334594727, | |
| "epoch": 0.76, | |
| "grad_norm": 2.0611778414132793, | |
| "learning_rate": 8.284884626103165e-07, | |
| "logits/chosen": -2.7938308715820312, | |
| "logits/rejected": -2.762302875518799, | |
| "logps/chosen": -285.65093994140625, | |
| "logps/rejected": -295.174560546875, | |
| "loss": 0.7577, | |
| "positive_losses": 0.03922748565673828, | |
| "rewards/accuracies": 0.637499988079071, | |
| "rewards/chosen": 0.15438929200172424, | |
| "rewards/margins": 0.06061772257089615, | |
| "rewards/margins_max": 0.1899864226579666, | |
| "rewards/margins_min": -0.060655929148197174, | |
| "rewards/margins_std": 0.11546076834201813, | |
| "rewards/rejected": 0.09377159178256989, | |
| "step": 250 | |
| }, | |
| { | |
| "dpo_losses": 0.6641849279403687, | |
| "epoch": 0.79, | |
| "grad_norm": 5.534968963232783, | |
| "learning_rate": 6.409547664531734e-07, | |
| "logits/chosen": -2.817701816558838, | |
| "logits/rejected": -2.7872586250305176, | |
| "logps/chosen": -316.77471923828125, | |
| "logps/rejected": -301.3828125, | |
| "loss": 0.6791, | |
| "positive_losses": 0.059366799890995026, | |
| "rewards/accuracies": 0.7250000238418579, | |
| "rewards/chosen": 0.17207232117652893, | |
| "rewards/margins": 0.060843367129564285, | |
| "rewards/margins_max": 0.1557174026966095, | |
| "rewards/margins_min": -0.024733999744057655, | |
| "rewards/margins_std": 0.0783897191286087, | |
| "rewards/rejected": 0.11122896522283554, | |
| "step": 260 | |
| }, | |
| { | |
| "dpo_losses": 0.6656020879745483, | |
| "epoch": 0.82, | |
| "grad_norm": 2.074871528049425, | |
| "learning_rate": 4.743428469705336e-07, | |
| "logits/chosen": -2.7688679695129395, | |
| "logits/rejected": -2.767603874206543, | |
| "logps/chosen": -289.3965759277344, | |
| "logps/rejected": -299.40924072265625, | |
| "loss": 0.6782, | |
| "positive_losses": 0.13356761634349823, | |
| "rewards/accuracies": 0.637499988079071, | |
| "rewards/chosen": 0.14411255717277527, | |
| "rewards/margins": 0.058389417827129364, | |
| "rewards/margins_max": 0.1643931269645691, | |
| "rewards/margins_min": -0.037889931350946426, | |
| "rewards/margins_std": 0.09290634840726852, | |
| "rewards/rejected": 0.0857231393456459, | |
| "step": 270 | |
| }, | |
| { | |
| "dpo_losses": 0.6564372181892395, | |
| "epoch": 0.85, | |
| "grad_norm": 2.142298246340979, | |
| "learning_rate": 3.3052776201888266e-07, | |
| "logits/chosen": -2.818432569503784, | |
| "logits/rejected": -2.8038887977600098, | |
| "logps/chosen": -308.4718933105469, | |
| "logps/rejected": -260.4856262207031, | |
| "loss": 0.6756, | |
| "positive_losses": 0.07436485588550568, | |
| "rewards/accuracies": 0.762499988079071, | |
| "rewards/chosen": 0.17093774676322937, | |
| "rewards/margins": 0.07805446535348892, | |
| "rewards/margins_max": 0.18430791795253754, | |
| "rewards/margins_min": -0.037783779203891754, | |
| "rewards/margins_std": 0.09794415533542633, | |
| "rewards/rejected": 0.09288326650857925, | |
| "step": 280 | |
| }, | |
| { | |
| "dpo_losses": 0.668868899345398, | |
| "epoch": 0.88, | |
| "grad_norm": 8.396243354371068, | |
| "learning_rate": 2.111280128780638e-07, | |
| "logits/chosen": -2.756545305252075, | |
| "logits/rejected": -2.7231669425964355, | |
| "logps/chosen": -259.9716491699219, | |
| "logps/rejected": -236.48770141601562, | |
| "loss": 0.684, | |
| "positive_losses": 0.07783536612987518, | |
| "rewards/accuracies": 0.675000011920929, | |
| "rewards/chosen": 0.1421826183795929, | |
| "rewards/margins": 0.051858507096767426, | |
| "rewards/margins_max": 0.16274937987327576, | |
| "rewards/margins_min": -0.04277097061276436, | |
| "rewards/margins_std": 0.09085315465927124, | |
| "rewards/rejected": 0.09032410383224487, | |
| "step": 290 | |
| }, | |
| { | |
| "dpo_losses": 0.6606593132019043, | |
| "epoch": 0.91, | |
| "grad_norm": 7.1470269484352835, | |
| "learning_rate": 1.1748732956682023e-07, | |
| "logits/chosen": -2.8557376861572266, | |
| "logits/rejected": -2.791503667831421, | |
| "logps/chosen": -309.07489013671875, | |
| "logps/rejected": -277.8647155761719, | |
| "loss": 0.6774, | |
| "positive_losses": 0.09160251915454865, | |
| "rewards/accuracies": 0.737500011920929, | |
| "rewards/chosen": 0.14638249576091766, | |
| "rewards/margins": 0.06913793087005615, | |
| "rewards/margins_max": 0.1924934983253479, | |
| "rewards/margins_min": -0.04603328928351402, | |
| "rewards/margins_std": 0.10587652772665024, | |
| "rewards/rejected": 0.07724456489086151, | |
| "step": 300 | |
| }, | |
| { | |
| "epoch": 0.91, | |
| "eval_dpo_losses": 0.6679198145866394, | |
| "eval_logits/chosen": -2.7836546897888184, | |
| "eval_logits/rejected": -2.746371030807495, | |
| "eval_logps/chosen": -269.767822265625, | |
| "eval_logps/rejected": -249.15957641601562, | |
| "eval_loss": 0.687869131565094, | |
| "eval_positive_losses": 0.09410960972309113, | |
| "eval_rewards/accuracies": 0.6899999976158142, | |
| "eval_rewards/chosen": 0.1482561081647873, | |
| "eval_rewards/margins": 0.05406271293759346, | |
| "eval_rewards/margins_max": 0.2146386057138443, | |
| "eval_rewards/margins_min": -0.0945044532418251, | |
| "eval_rewards/margins_std": 0.10281499475240707, | |
| "eval_rewards/rejected": 0.09419341385364532, | |
| "eval_runtime": 428.9213, | |
| "eval_samples_per_second": 4.663, | |
| "eval_steps_per_second": 0.291, | |
| "step": 300 | |
| }, | |
| { | |
| "dpo_losses": 0.6655529141426086, | |
| "epoch": 0.94, | |
| "grad_norm": 1.6083474443571015, | |
| "learning_rate": 5.065954844616722e-08, | |
| "logits/chosen": -2.7985780239105225, | |
| "logits/rejected": -2.756269931793213, | |
| "logps/chosen": -261.9189758300781, | |
| "logps/rejected": -271.715576171875, | |
| "loss": 0.6852, | |
| "positive_losses": 0.16939087212085724, | |
| "rewards/accuracies": 0.7124999761581421, | |
| "rewards/chosen": 0.1520276516675949, | |
| "rewards/margins": 0.05855011194944382, | |
| "rewards/margins_max": 0.17010439932346344, | |
| "rewards/margins_min": -0.046878743916749954, | |
| "rewards/margins_std": 0.09575958549976349, | |
| "rewards/rejected": 0.0934775322675705, | |
| "step": 310 | |
| }, | |
| { | |
| "dpo_losses": 0.6655773520469666, | |
| "epoch": 0.97, | |
| "grad_norm": 1.8914240259960515, | |
| "learning_rate": 1.1396752298723501e-08, | |
| "logits/chosen": -2.8393120765686035, | |
| "logits/rejected": -2.7888612747192383, | |
| "logps/chosen": -235.32876586914062, | |
| "logps/rejected": -249.7720184326172, | |
| "loss": 0.666, | |
| "positive_losses": 0.027046585455536842, | |
| "rewards/accuracies": 0.762499988079071, | |
| "rewards/chosen": 0.1361643224954605, | |
| "rewards/margins": 0.05848071724176407, | |
| "rewards/margins_max": 0.16818095743656158, | |
| "rewards/margins_min": -0.04760957881808281, | |
| "rewards/margins_std": 0.09690945595502853, | |
| "rewards/rejected": 0.07768361270427704, | |
| "step": 320 | |
| }, | |
| { | |
| "epoch": 1.0, | |
| "step": 329, | |
| "total_flos": 0.0, | |
| "train_loss": 0.6874516553791823, | |
| "train_runtime": 3895.4317, | |
| "train_samples_per_second": 1.351, | |
| "train_steps_per_second": 0.084 | |
| } | |
| ], | |
| "logging_steps": 10, | |
| "max_steps": 329, | |
| "num_input_tokens_seen": 0, | |
| "num_train_epochs": 1, | |
| "save_steps": 100, | |
| "total_flos": 0.0, | |
| "train_batch_size": 4, | |
| "trial_name": null, | |
| "trial_params": null | |
| } | |