{ "best_metric": null, "best_model_checkpoint": null, "epoch": 0.997638167217761, "eval_steps": 50, "global_step": 176, "is_hyper_param_search": false, "is_local_process_zero": true, "is_world_process_zero": true, "log_history": [ { "dpo_loss": 0.6931471824645996, "epoch": 0.005668398677373642, "grad_norm": 1341367.1350394282, "learning_rate": 2.7777777777777776e-07, "logits": -1.3147305250167847, "logps": -88.0877456665039, "loss": 0.6931, "objective": 0.6931471824645996, "ranking_idealized": 0.5208333134651184, "ranking_idealized_expo": 0.5208333134651184, "ranking_simple": 0.5208333134651184, "regularize": 0.6931471824645996, "step": 1 }, { "dpo_loss": 1280.619140625, "epoch": 0.02834199338686821, "grad_norm": 2623988.3634039685, "learning_rate": 1.3888888888888892e-06, "logits": -1.3680986166000366, "logps": -84.14160919189453, "loss": 1112.7681, "objective": 1280.619140625, "ranking_idealized": 0.546875, "ranking_idealized_expo": 0.546875, "ranking_simple": 0.5520833134651184, "regularize": 1280.619140625, "step": 5 }, { "dpo_loss": 2070.714599609375, "epoch": 0.05668398677373642, "grad_norm": 2330391.207726475, "learning_rate": 2.7777777777777783e-06, "logits": -1.45966374874115, "logps": -83.2505111694336, "loss": 2407.7941, "objective": 2070.714599609375, "ranking_idealized": 0.5083333253860474, "ranking_idealized_expo": 0.5083333253860474, "ranking_simple": 0.5166666507720947, "regularize": 2070.714599609375, "step": 10 }, { "dpo_loss": 5679.9853515625, "epoch": 0.08502598016060463, "grad_norm": 1644564.4852830889, "learning_rate": 4.166666666666667e-06, "logits": -1.4328060150146484, "logps": -82.17012023925781, "loss": 4264.0652, "objective": 5679.9853515625, "ranking_idealized": 0.5041666626930237, "ranking_idealized_expo": 0.5041666626930237, "ranking_simple": 0.5166666507720947, "regularize": 5679.9853515625, "step": 15 }, { "dpo_loss": 7525.20068359375, "epoch": 0.11336797354747284, "grad_norm": 1515613.7017780922, "learning_rate": 4.998023493068255e-06, "logits": -1.4038361310958862, "logps": -82.86460876464844, "loss": 6967.5602, "objective": 7525.20068359375, "ranking_idealized": 0.5375000238418579, "ranking_idealized_expo": 0.5375000238418579, "ranking_simple": 0.5583333373069763, "regularize": 7525.20068359375, "step": 20 }, { "dpo_loss": 8746.1123046875, "epoch": 0.14170996693434104, "grad_norm": 1687436.4259306653, "learning_rate": 4.975823666181256e-06, "logits": -1.371711254119873, "logps": -82.0951156616211, "loss": 9527.1836, "objective": 8746.1123046875, "ranking_idealized": 0.4958333373069763, "ranking_idealized_expo": 0.4958333373069763, "ranking_simple": 0.512499988079071, "regularize": 8746.1123046875, "step": 25 }, { "dpo_loss": 11099.513671875, "epoch": 0.17005196032120926, "grad_norm": 1294226.520732075, "learning_rate": 4.929173350101025e-06, "logits": -1.3348666429519653, "logps": -81.38406372070312, "loss": 10099.4609, "objective": 11099.513671875, "ranking_idealized": 0.5416666865348816, "ranking_idealized_expo": 0.5416666865348816, "ranking_simple": 0.5458333492279053, "regularize": 11099.513671875, "step": 30 }, { "dpo_loss": 12696.7705078125, "epoch": 0.19839395370807747, "grad_norm": 1391829.2894247118, "learning_rate": 4.858533249305337e-06, "logits": -1.411891222000122, "logps": -77.1667251586914, "loss": 9874.5609, "objective": 12696.7705078125, "ranking_idealized": 0.5041666626930237, "ranking_idealized_expo": 0.5041666626930237, "ranking_simple": 0.5375000238418579, "regularize": 12696.7705078125, "step": 35 }, { "dpo_loss": 8431.9521484375, "epoch": 0.22673594709494568, "grad_norm": 1186854.0601841968, "learning_rate": 4.764600984163809e-06, "logits": -1.3430523872375488, "logps": -77.41344451904297, "loss": 10212.3687, "objective": 8431.9521484375, "ranking_idealized": 0.5208333134651184, "ranking_idealized_expo": 0.5208333134651184, "ranking_simple": 0.5874999761581421, "regularize": 8431.9521484375, "step": 40 }, { "dpo_loss": 11319.8056640625, "epoch": 0.25507794048181387, "grad_norm": 1072714.7178744846, "learning_rate": 4.648304201449153e-06, "logits": -1.3935389518737793, "logps": -79.65689086914062, "loss": 10434.6648, "objective": 11319.8056640625, "ranking_idealized": 0.5375000238418579, "ranking_idealized_expo": 0.5375000238418579, "ranking_simple": 0.574999988079071, "regularize": 11319.8056640625, "step": 45 }, { "dpo_loss": 9569.685546875, "epoch": 0.2834199338686821, "grad_norm": 950921.657886262, "learning_rate": 4.510791413176912e-06, "logits": -1.3682667016983032, "logps": -77.96815490722656, "loss": 10470.1625, "objective": 9569.685546875, "ranking_idealized": 0.42500001192092896, "ranking_idealized_expo": 0.42500001192092896, "ranking_simple": 0.5, "regularize": 9569.685546875, "step": 50 }, { "epoch": 0.2834199338686821, "eval_dpo_loss": 11815.3388671875, "eval_logits": -1.4585951566696167, "eval_logps": -84.9886474609375, "eval_loss": 11322.9775390625, "eval_objective": 11815.3388671875, "eval_ranking_idealized": 0.5092975497245789, "eval_ranking_idealized_expo": 0.5092975497245789, "eval_ranking_simple": 0.5165289044380188, "eval_regularize": 11815.3388671875, "eval_runtime": 1211.6497, "eval_samples_per_second": 4.779, "eval_steps_per_second": 0.2, "step": 50 }, { "dpo_loss": 12371.0048828125, "epoch": 0.3117619272555503, "grad_norm": 1109923.0216191527, "learning_rate": 4.353420654246546e-06, "logits": -1.4771044254302979, "logps": -79.130126953125, "loss": 11113.0828, "objective": 12371.0048828125, "ranking_idealized": 0.5416666865348816, "ranking_idealized_expo": 0.5416666865348816, "ranking_simple": 0.5541666746139526, "regularize": 12371.0048828125, "step": 55 }, { "dpo_loss": 8734.4072265625, "epoch": 0.3401039206424185, "grad_norm": 1211359.9780672283, "learning_rate": 4.177746070897593e-06, "logits": -1.4155789613723755, "logps": -77.57923126220703, "loss": 11080.9961, "objective": 8734.4072265625, "ranking_idealized": 0.512499988079071, "ranking_idealized_expo": 0.512499988079071, "ranking_simple": 0.5708333253860474, "regularize": 8734.4072265625, "step": 60 }, { "dpo_loss": 9739.5908203125, "epoch": 0.3684459140292867, "grad_norm": 1057255.008796819, "learning_rate": 3.985502572429277e-06, "logits": -1.3343302011489868, "logps": -77.00284576416016, "loss": 11711.2734, "objective": 9739.5908203125, "ranking_idealized": 0.5083333253860474, "ranking_idealized_expo": 0.5083333253860474, "ranking_simple": 0.5541666746139526, "regularize": 9739.5908203125, "step": 65 }, { "dpo_loss": 9313.849609375, "epoch": 0.39678790741615494, "grad_norm": 1088725.4948180364, "learning_rate": 3.7785886977585562e-06, "logits": -1.3323314189910889, "logps": -76.60472106933594, "loss": 11694.0484, "objective": 9313.849609375, "ranking_idealized": 0.5166666507720947, "ranking_idealized_expo": 0.5166666507720947, "ranking_simple": 0.5416666865348816, "regularize": 9313.849609375, "step": 70 }, { "dpo_loss": 10158.4462890625, "epoch": 0.42512990080302315, "grad_norm": 999614.3040187941, "learning_rate": 3.5590478660213214e-06, "logits": -1.2671582698822021, "logps": -77.10884094238281, "loss": 11305.65, "objective": 10158.4462890625, "ranking_idealized": 0.5458333492279053, "ranking_idealized_expo": 0.5458333492279053, "ranking_simple": 0.5916666388511658, "regularize": 10158.4462890625, "step": 75 }, { "dpo_loss": 13367.1591796875, "epoch": 0.45347189418989137, "grad_norm": 1036547.2837553191, "learning_rate": 3.32904819638017e-06, "logits": -1.2884188890457153, "logps": -76.65129852294922, "loss": 11108.9961, "objective": 13367.1591796875, "ranking_idealized": 0.5166666507720947, "ranking_idealized_expo": 0.5166666507720947, "ranking_simple": 0.5583333373069763, "regularize": 13367.1591796875, "step": 80 }, { "dpo_loss": 9063.92578125, "epoch": 0.4818138875767596, "grad_norm": 878439.4100295339, "learning_rate": 3.090861096332263e-06, "logits": -1.2684974670410156, "logps": -75.37580108642578, "loss": 9492.8891, "objective": 9063.92578125, "ranking_idealized": 0.5249999761581421, "ranking_idealized_expo": 0.5249999761581421, "ranking_simple": 0.5916666388511658, "regularize": 9063.92578125, "step": 85 }, { "dpo_loss": 12140.421875, "epoch": 0.5101558809636277, "grad_norm": 839896.1614839052, "learning_rate": 2.8468388299726714e-06, "logits": -1.1826907396316528, "logps": -77.36532592773438, "loss": 8699.0336, "objective": 12140.421875, "ranking_idealized": 0.4791666567325592, "ranking_idealized_expo": 0.4791666567325592, "ranking_simple": 0.5166666507720947, "regularize": 12140.421875, "step": 90 }, { "dpo_loss": 7882.3046875, "epoch": 0.538497874350496, "grad_norm": 1169515.5696928622, "learning_rate": 2.599391287742315e-06, "logits": -1.2656235694885254, "logps": -76.93645477294922, "loss": 8873.2109, "objective": 7882.3046875, "ranking_idealized": 0.5416666865348816, "ranking_idealized_expo": 0.5416666865348816, "ranking_simple": 0.6166666746139526, "regularize": 7882.3046875, "step": 95 }, { "dpo_loss": 7251.060546875, "epoch": 0.5668398677373642, "grad_norm": 1006084.9040205956, "learning_rate": 2.3509621870754505e-06, "logits": -1.342369794845581, "logps": -78.5166244506836, "loss": 8558.2836, "objective": 7251.060546875, "ranking_idealized": 0.5416666865348816, "ranking_idealized_expo": 0.5416666865348816, "ranking_simple": 0.6041666865348816, "regularize": 7251.060546875, "step": 100 }, { "epoch": 0.5668398677373642, "eval_dpo_loss": 13248.5166015625, "eval_logits": -1.3720418214797974, "eval_logps": -82.3385238647461, "eval_loss": 12877.275390625, "eval_objective": 13248.5166015625, "eval_ranking_idealized": 0.5092975497245789, "eval_ranking_idealized_expo": 0.5092975497245789, "eval_ranking_simple": 0.5175619721412659, "eval_regularize": 13248.5166015625, "eval_runtime": 258.8572, "eval_samples_per_second": 22.368, "eval_steps_per_second": 0.935, "step": 100 }, { "dpo_loss": 9016.806640625, "epoch": 0.5951818611242324, "grad_norm": 861666.5853674684, "learning_rate": 2.1040049389819628e-06, "logits": -1.238232970237732, "logps": -78.26189422607422, "loss": 8298.018, "objective": 9016.806640625, "ranking_idealized": 0.5333333611488342, "ranking_idealized_expo": 0.5333333611488342, "ranking_simple": 0.612500011920929, "regularize": 9016.806640625, "step": 105 }, { "dpo_loss": 7970.41748046875, "epoch": 0.6235238545111006, "grad_norm": 865124.6869036607, "learning_rate": 1.8609584188988135e-06, "logits": -1.330941081047058, "logps": -76.76597595214844, "loss": 7414.1281, "objective": 7970.41748046875, "ranking_idealized": 0.44999998807907104, "ranking_idealized_expo": 0.44999998807907104, "ranking_simple": 0.5416666865348816, "regularize": 7970.41748046875, "step": 110 }, { "dpo_loss": 9534.12890625, "epoch": 0.6518658478979689, "grad_norm": 840576.3563797984, "learning_rate": 1.6242228810904393e-06, "logits": -1.3068194389343262, "logps": -76.78305053710938, "loss": 8769.2398, "objective": 9534.12890625, "ranking_idealized": 0.4833333194255829, "ranking_idealized_expo": 0.4833333194255829, "ranking_simple": 0.5458333492279053, "regularize": 9534.12890625, "step": 115 }, { "dpo_loss": 8552.21484375, "epoch": 0.680207841284837, "grad_norm": 794346.4700274817, "learning_rate": 1.3961362544602215e-06, "logits": -1.2461904287338257, "logps": -77.25305938720703, "loss": 7377.8172, "objective": 8552.21484375, "ranking_idealized": 0.5333333611488342, "ranking_idealized_expo": 0.5333333611488342, "ranking_simple": 0.5874999761581421, "regularize": 8552.21484375, "step": 120 }, { "dpo_loss": 9080.310546875, "epoch": 0.7085498346717053, "grad_norm": 866794.9526252571, "learning_rate": 1.1789510538684524e-06, "logits": -1.2470247745513916, "logps": -75.66565704345703, "loss": 7754.6125, "objective": 9080.310546875, "ranking_idealized": 0.550000011920929, "ranking_idealized_expo": 0.550000011920929, "ranking_simple": 0.6000000238418579, "regularize": 9080.310546875, "step": 125 }, { "dpo_loss": 7816.40869140625, "epoch": 0.7368918280585735, "grad_norm": 914484.5347620883, "learning_rate": 9.74812134973689e-07, "logits": -1.362328290939331, "logps": -74.44100952148438, "loss": 7071.1984, "objective": 7816.40869140625, "ranking_idealized": 0.44999998807907104, "ranking_idealized_expo": 0.44999998807907104, "ranking_simple": 0.5041666626930237, "regularize": 7816.40869140625, "step": 130 }, { "dpo_loss": 6297.37939453125, "epoch": 0.7652338214454416, "grad_norm": 908053.7687284376, "learning_rate": 7.857355122839674e-07, "logits": -1.3475323915481567, "logps": -76.51807403564453, "loss": 7334.7656, "objective": 6297.37939453125, "ranking_idealized": 0.5291666388511658, "ranking_idealized_expo": 0.5291666388511658, "ranking_simple": 0.5833333134651184, "regularize": 6297.37939453125, "step": 135 }, { "dpo_loss": 5713.158203125, "epoch": 0.7935758148323099, "grad_norm": 765346.0794896972, "learning_rate": 6.135884496044245e-07, "logits": -1.2585749626159668, "logps": -75.7166519165039, "loss": 6126.0992, "objective": 5713.158203125, "ranking_idealized": 0.5041666626930237, "ranking_idealized_expo": 0.5041666626930237, "ranking_simple": 0.5708333253860474, "regularize": 5713.158203125, "step": 140 }, { "dpo_loss": 6023.36669921875, "epoch": 0.821917808219178, "grad_norm": 864232.6098968883, "learning_rate": 4.6007101950209827e-07, "logits": -1.311624526977539, "logps": -75.54283905029297, "loss": 6440.6867, "objective": 6023.36669921875, "ranking_idealized": 0.4833333194255829, "ranking_idealized_expo": 0.4833333194255829, "ranking_simple": 0.5333333611488342, "regularize": 6023.36669921875, "step": 145 }, { "dpo_loss": 9931.4228515625, "epoch": 0.8502598016060463, "grad_norm": 789706.2833060493, "learning_rate": 3.266993139010438e-07, "logits": -1.3248921632766724, "logps": -73.5119857788086, "loss": 7521.95, "objective": 9931.4228515625, "ranking_idealized": 0.5375000238418579, "ranking_idealized_expo": 0.5375000238418579, "ranking_simple": 0.5833333134651184, "regularize": 9931.4228515625, "step": 150 }, { "epoch": 0.8502598016060463, "eval_dpo_loss": 12926.72265625, "eval_logits": -1.3555928468704224, "eval_logps": -80.5750961303711, "eval_loss": 12664.20703125, "eval_objective": 12926.72265625, "eval_ranking_idealized": 0.5092975497245789, "eval_ranking_idealized_expo": 0.5092975497245789, "eval_ranking_simple": 0.5227272510528564, "eval_regularize": 12926.72265625, "eval_runtime": 258.9258, "eval_samples_per_second": 22.362, "eval_steps_per_second": 0.935, "step": 150 }, { "dpo_loss": 5853.70361328125, "epoch": 0.8786017949929145, "grad_norm": 805936.4979433398, "learning_rate": 2.1479047161491351e-07, "logits": -1.3135905265808105, "logps": -73.38835144042969, "loss": 5683.3855, "objective": 5853.70361328125, "ranking_idealized": 0.512499988079071, "ranking_idealized_expo": 0.512499988079071, "ranking_simple": 0.612500011920929, "regularize": 5853.70361328125, "step": 155 }, { "dpo_loss": 6223.60693359375, "epoch": 0.9069437883797827, "grad_norm": 771613.5624330686, "learning_rate": 1.2544967068054332e-07, "logits": -1.2275891304016113, "logps": -74.46540069580078, "loss": 6799.2719, "objective": 6223.60693359375, "ranking_idealized": 0.4958333373069763, "ranking_idealized_expo": 0.4958333373069763, "ranking_simple": 0.574999988079071, "regularize": 6223.60693359375, "step": 160 }, { "dpo_loss": 4967.171875, "epoch": 0.9352857817666509, "grad_norm": 1004063.6300908878, "learning_rate": 5.955921395237318e-08, "logits": -1.2412575483322144, "logps": -75.6294174194336, "loss": 6197.7492, "objective": 4967.171875, "ranking_idealized": 0.5208333134651184, "ranking_idealized_expo": 0.5208333134651184, "ranking_simple": 0.5958333611488342, "regularize": 4967.171875, "step": 165 }, { "dpo_loss": 7402.99267578125, "epoch": 0.9636277751535192, "grad_norm": 778643.4373397075, "learning_rate": 1.7769815745066476e-08, "logits": -1.2604658603668213, "logps": -75.88407135009766, "loss": 6623.1367, "objective": 7402.99267578125, "ranking_idealized": 0.5083333253860474, "ranking_idealized_expo": 0.5083333253860474, "ranking_simple": 0.5666666626930237, "regularize": 7402.99267578125, "step": 170 }, { "dpo_loss": 5726.1806640625, "epoch": 0.9919697685403873, "grad_norm": 762884.4185589707, "learning_rate": 4.941755748361088e-10, "logits": -1.2217018604278564, "logps": -76.01050567626953, "loss": 5562.8855, "objective": 5726.1806640625, "ranking_idealized": 0.574999988079071, "ranking_idealized_expo": 0.574999988079071, "ranking_simple": 0.6333333253860474, "regularize": 5726.1806640625, "step": 175 }, { "epoch": 0.997638167217761, "step": 176, "total_flos": 0.0, "train_loss": 8072.862625997175, "train_runtime": 10876.2953, "train_samples_per_second": 4.671, "train_steps_per_second": 0.016 } ], "logging_steps": 5, "max_steps": 176, "num_input_tokens_seen": 0, "num_train_epochs": 1, "save_steps": 500, "stateful_callbacks": { "TrainerControl": { "args": { "should_epoch_stop": false, "should_evaluate": false, "should_log": false, "should_save": false, "should_training_stop": false }, "attributes": {} } }, "total_flos": 0.0, "train_batch_size": 4, "trial_name": null, "trial_params": null }