| { | |
| "best_metric": 0.017838381230831146, | |
| "best_model_checkpoint": "./Zephyr/27-03-24-Weni-WeniGPT-QA-Zephyr-7B-4.0.0-KTO_WeniGPT Experiment using KTO trainer with no collator-2_max_steps-786_batch_16_2024-03-27_ppid_9/checkpoint-600", | |
| "epoch": 2.2652194431335535, | |
| "eval_steps": 50, | |
| "global_step": 600, | |
| "is_hyper_param_search": false, | |
| "is_local_process_zero": true, | |
| "is_world_process_zero": true, | |
| "log_history": [ | |
| { | |
| "epoch": 0.08, | |
| "grad_norm": 2.2820355892181396, | |
| "kl": 0.7391773462295532, | |
| "learning_rate": 0.0001666666666666667, | |
| "logps/chosen": NaN, | |
| "logps/rejected": NaN, | |
| "loss": 785.5091, | |
| "rewards/chosen": NaN, | |
| "rewards/margins": NaN, | |
| "rewards/rejected": NaN, | |
| "step": 20 | |
| }, | |
| { | |
| "epoch": 0.15, | |
| "grad_norm": NaN, | |
| "kl": 1.364301085472107, | |
| "learning_rate": 0.00019606299212598428, | |
| "logps/chosen": NaN, | |
| "logps/rejected": NaN, | |
| "loss": 124.9389, | |
| "rewards/chosen": NaN, | |
| "rewards/margins": NaN, | |
| "rewards/rejected": NaN, | |
| "step": 40 | |
| }, | |
| { | |
| "epoch": 0.19, | |
| "eval_kl": 3.282975673675537, | |
| "eval_logps/chosen": -132.00738525390625, | |
| "eval_logps/rejected": -237.69239807128906, | |
| "eval_loss": 0.09803511947393417, | |
| "eval_rewards/chosen": 4.3711838722229, | |
| "eval_rewards/margins": 8.833364486694336, | |
| "eval_rewards/rejected": -4.462180137634277, | |
| "eval_runtime": 474.8108, | |
| "eval_samples_per_second": 1.07, | |
| "eval_steps_per_second": 0.535, | |
| "step": 50 | |
| }, | |
| { | |
| "epoch": 0.23, | |
| "grad_norm": 1.1073336601257324, | |
| "kl": 0.8054396510124207, | |
| "learning_rate": 0.00019107611548556432, | |
| "logps/chosen": NaN, | |
| "logps/rejected": NaN, | |
| "loss": 91.2134, | |
| "rewards/chosen": NaN, | |
| "rewards/margins": NaN, | |
| "rewards/rejected": NaN, | |
| "step": 60 | |
| }, | |
| { | |
| "epoch": 0.3, | |
| "grad_norm": 0.4274377226829529, | |
| "kl": 0.38704636693000793, | |
| "learning_rate": 0.00018582677165354333, | |
| "logps/chosen": NaN, | |
| "logps/rejected": NaN, | |
| "loss": 26.9569, | |
| "rewards/chosen": NaN, | |
| "rewards/margins": NaN, | |
| "rewards/rejected": NaN, | |
| "step": 80 | |
| }, | |
| { | |
| "epoch": 0.38, | |
| "grad_norm": 0.5270018577575684, | |
| "kl": 0.47709473967552185, | |
| "learning_rate": 0.0001805774278215223, | |
| "logps/chosen": NaN, | |
| "logps/rejected": NaN, | |
| "loss": 10.8269, | |
| "rewards/chosen": NaN, | |
| "rewards/margins": NaN, | |
| "rewards/rejected": NaN, | |
| "step": 100 | |
| }, | |
| { | |
| "epoch": 0.38, | |
| "eval_kl": 0.762260377407074, | |
| "eval_logps/chosen": -135.4526824951172, | |
| "eval_logps/rejected": -541.3764038085938, | |
| "eval_loss": 0.039932698011398315, | |
| "eval_rewards/chosen": 4.026655197143555, | |
| "eval_rewards/margins": 38.857234954833984, | |
| "eval_rewards/rejected": -34.83057403564453, | |
| "eval_runtime": 474.2877, | |
| "eval_samples_per_second": 1.071, | |
| "eval_steps_per_second": 0.536, | |
| "step": 100 | |
| }, | |
| { | |
| "epoch": 0.45, | |
| "grad_norm": 0.19858050346374512, | |
| "kl": 1.8940637111663818, | |
| "learning_rate": 0.00017532808398950132, | |
| "logps/chosen": NaN, | |
| "logps/rejected": NaN, | |
| "loss": 34.2088, | |
| "rewards/chosen": NaN, | |
| "rewards/margins": NaN, | |
| "rewards/rejected": NaN, | |
| "step": 120 | |
| }, | |
| { | |
| "epoch": 0.53, | |
| "grad_norm": 2.615041971206665, | |
| "kl": 1.40102219581604, | |
| "learning_rate": 0.00017007874015748033, | |
| "logps/chosen": NaN, | |
| "logps/rejected": NaN, | |
| "loss": 276.3512, | |
| "rewards/chosen": NaN, | |
| "rewards/margins": NaN, | |
| "rewards/rejected": NaN, | |
| "step": 140 | |
| }, | |
| { | |
| "epoch": 0.57, | |
| "eval_kl": 1.686139702796936, | |
| "eval_logps/chosen": -127.73213195800781, | |
| "eval_logps/rejected": -397.8934631347656, | |
| "eval_loss": 0.02803400717675686, | |
| "eval_rewards/chosen": 4.798709869384766, | |
| "eval_rewards/margins": 25.2810001373291, | |
| "eval_rewards/rejected": -20.482288360595703, | |
| "eval_runtime": 474.6717, | |
| "eval_samples_per_second": 1.07, | |
| "eval_steps_per_second": 0.535, | |
| "step": 150 | |
| }, | |
| { | |
| "epoch": 0.6, | |
| "grad_norm": 0.31603240966796875, | |
| "kl": 0.7387570738792419, | |
| "learning_rate": 0.00016482939632545934, | |
| "logps/chosen": NaN, | |
| "logps/rejected": NaN, | |
| "loss": 216.0894, | |
| "rewards/chosen": NaN, | |
| "rewards/margins": NaN, | |
| "rewards/rejected": NaN, | |
| "step": 160 | |
| }, | |
| { | |
| "epoch": 0.68, | |
| "grad_norm": 0.3982734978199005, | |
| "kl": 1.4841536283493042, | |
| "learning_rate": 0.00015958005249343832, | |
| "logps/chosen": NaN, | |
| "logps/rejected": NaN, | |
| "loss": 8.0994, | |
| "rewards/chosen": NaN, | |
| "rewards/margins": NaN, | |
| "rewards/rejected": NaN, | |
| "step": 180 | |
| }, | |
| { | |
| "epoch": 0.76, | |
| "grad_norm": 2.1777164936065674, | |
| "kl": 0.3081069588661194, | |
| "learning_rate": 0.00015433070866141733, | |
| "logps/chosen": NaN, | |
| "logps/rejected": NaN, | |
| "loss": 5.7214, | |
| "rewards/chosen": NaN, | |
| "rewards/margins": NaN, | |
| "rewards/rejected": NaN, | |
| "step": 200 | |
| }, | |
| { | |
| "epoch": 0.76, | |
| "eval_kl": 1.5452415943145752, | |
| "eval_logps/chosen": -125.70950317382812, | |
| "eval_logps/rejected": -412.7599182128906, | |
| "eval_loss": 0.029857149347662926, | |
| "eval_rewards/chosen": 5.000972270965576, | |
| "eval_rewards/margins": 26.969900131225586, | |
| "eval_rewards/rejected": -21.96892738342285, | |
| "eval_runtime": 474.8305, | |
| "eval_samples_per_second": 1.07, | |
| "eval_steps_per_second": 0.535, | |
| "step": 200 | |
| }, | |
| { | |
| "epoch": 0.83, | |
| "grad_norm": 0.08603761345148087, | |
| "kl": 1.142593264579773, | |
| "learning_rate": 0.00014908136482939634, | |
| "logps/chosen": NaN, | |
| "logps/rejected": NaN, | |
| "loss": 22.8407, | |
| "rewards/chosen": NaN, | |
| "rewards/margins": NaN, | |
| "rewards/rejected": NaN, | |
| "step": 220 | |
| }, | |
| { | |
| "epoch": 0.91, | |
| "grad_norm": 5.5862650871276855, | |
| "kl": 0.40408754348754883, | |
| "learning_rate": 0.00014383202099737535, | |
| "logps/chosen": NaN, | |
| "logps/rejected": NaN, | |
| "loss": 207.9747, | |
| "rewards/chosen": NaN, | |
| "rewards/margins": NaN, | |
| "rewards/rejected": NaN, | |
| "step": 240 | |
| }, | |
| { | |
| "epoch": 0.94, | |
| "eval_kl": 1.182385802268982, | |
| "eval_logps/chosen": -127.54723358154297, | |
| "eval_logps/rejected": -806.224853515625, | |
| "eval_loss": 0.026166189461946487, | |
| "eval_rewards/chosen": 4.817198276519775, | |
| "eval_rewards/margins": 66.13262176513672, | |
| "eval_rewards/rejected": -61.315425872802734, | |
| "eval_runtime": 474.7526, | |
| "eval_samples_per_second": 1.07, | |
| "eval_steps_per_second": 0.535, | |
| "step": 250 | |
| }, | |
| { | |
| "epoch": 0.98, | |
| "grad_norm": 0.39546045660972595, | |
| "kl": 0.7778356671333313, | |
| "learning_rate": 0.00013858267716535433, | |
| "logps/chosen": NaN, | |
| "logps/rejected": NaN, | |
| "loss": 720.8411, | |
| "rewards/chosen": NaN, | |
| "rewards/margins": NaN, | |
| "rewards/rejected": NaN, | |
| "step": 260 | |
| }, | |
| { | |
| "epoch": 1.06, | |
| "grad_norm": 0.1680106222629547, | |
| "kl": 1.798954963684082, | |
| "learning_rate": 0.00013333333333333334, | |
| "logps/chosen": NaN, | |
| "logps/rejected": NaN, | |
| "loss": 756.123, | |
| "rewards/chosen": NaN, | |
| "rewards/margins": NaN, | |
| "rewards/rejected": NaN, | |
| "step": 280 | |
| }, | |
| { | |
| "epoch": 1.13, | |
| "grad_norm": 0.13731369376182556, | |
| "kl": 1.9178543090820312, | |
| "learning_rate": 0.00012808398950131235, | |
| "logps/chosen": NaN, | |
| "logps/rejected": NaN, | |
| "loss": 25.0348, | |
| "rewards/chosen": NaN, | |
| "rewards/margins": NaN, | |
| "rewards/rejected": NaN, | |
| "step": 300 | |
| }, | |
| { | |
| "epoch": 1.13, | |
| "eval_kl": 1.4845337867736816, | |
| "eval_logps/chosen": -125.86082458496094, | |
| "eval_logps/rejected": -901.4517211914062, | |
| "eval_loss": 0.020615577697753906, | |
| "eval_rewards/chosen": 4.985838890075684, | |
| "eval_rewards/margins": 75.82396697998047, | |
| "eval_rewards/rejected": -70.83811950683594, | |
| "eval_runtime": 474.4234, | |
| "eval_samples_per_second": 1.071, | |
| "eval_steps_per_second": 0.535, | |
| "step": 300 | |
| }, | |
| { | |
| "epoch": 1.21, | |
| "grad_norm": 0.35939568281173706, | |
| "kl": 0.7613407373428345, | |
| "learning_rate": 0.00012283464566929136, | |
| "logps/chosen": NaN, | |
| "logps/rejected": NaN, | |
| "loss": 10.9288, | |
| "rewards/chosen": NaN, | |
| "rewards/margins": NaN, | |
| "rewards/rejected": NaN, | |
| "step": 320 | |
| }, | |
| { | |
| "epoch": 1.28, | |
| "grad_norm": 0.0034766767639666796, | |
| "kl": 0.3800249993801117, | |
| "learning_rate": 0.00011758530183727034, | |
| "logps/chosen": NaN, | |
| "logps/rejected": NaN, | |
| "loss": 3.1951, | |
| "rewards/chosen": NaN, | |
| "rewards/margins": NaN, | |
| "rewards/rejected": NaN, | |
| "step": 340 | |
| }, | |
| { | |
| "epoch": 1.32, | |
| "eval_kl": 0.6363816261291504, | |
| "eval_logps/chosen": -128.82322692871094, | |
| "eval_logps/rejected": -1020.8375244140625, | |
| "eval_loss": 0.026503143832087517, | |
| "eval_rewards/chosen": 4.689598560333252, | |
| "eval_rewards/margins": 87.4662857055664, | |
| "eval_rewards/rejected": -82.77667999267578, | |
| "eval_runtime": 474.6086, | |
| "eval_samples_per_second": 1.07, | |
| "eval_steps_per_second": 0.535, | |
| "step": 350 | |
| }, | |
| { | |
| "epoch": 1.36, | |
| "grad_norm": 1.1018553972244263, | |
| "kl": 0.1121591106057167, | |
| "learning_rate": 0.00011233595800524934, | |
| "logps/chosen": NaN, | |
| "logps/rejected": NaN, | |
| "loss": 3.5301, | |
| "rewards/chosen": NaN, | |
| "rewards/margins": NaN, | |
| "rewards/rejected": NaN, | |
| "step": 360 | |
| }, | |
| { | |
| "epoch": 1.43, | |
| "grad_norm": 0.6937451958656311, | |
| "kl": 1.0412858724594116, | |
| "learning_rate": 0.00010708661417322836, | |
| "logps/chosen": NaN, | |
| "logps/rejected": NaN, | |
| "loss": 2.2977, | |
| "rewards/chosen": NaN, | |
| "rewards/margins": NaN, | |
| "rewards/rejected": NaN, | |
| "step": 380 | |
| }, | |
| { | |
| "epoch": 1.51, | |
| "grad_norm": 0.25449952483177185, | |
| "kl": 0.832425594329834, | |
| "learning_rate": 0.00010183727034120735, | |
| "logps/chosen": NaN, | |
| "logps/rejected": NaN, | |
| "loss": 68.7248, | |
| "rewards/chosen": NaN, | |
| "rewards/margins": NaN, | |
| "rewards/rejected": NaN, | |
| "step": 400 | |
| }, | |
| { | |
| "epoch": 1.51, | |
| "eval_kl": 1.2176250219345093, | |
| "eval_logps/chosen": -125.15269470214844, | |
| "eval_logps/rejected": -730.7761840820312, | |
| "eval_loss": 0.020107893273234367, | |
| "eval_rewards/chosen": 5.056652545928955, | |
| "eval_rewards/margins": 58.827205657958984, | |
| "eval_rewards/rejected": -53.77055358886719, | |
| "eval_runtime": 474.7959, | |
| "eval_samples_per_second": 1.07, | |
| "eval_steps_per_second": 0.535, | |
| "step": 400 | |
| }, | |
| { | |
| "epoch": 1.59, | |
| "grad_norm": 0.14162199199199677, | |
| "kl": 0.6998993158340454, | |
| "learning_rate": 9.658792650918635e-05, | |
| "logps/chosen": NaN, | |
| "logps/rejected": NaN, | |
| "loss": 4.9788, | |
| "rewards/chosen": NaN, | |
| "rewards/margins": NaN, | |
| "rewards/rejected": NaN, | |
| "step": 420 | |
| }, | |
| { | |
| "epoch": 1.66, | |
| "grad_norm": 0.043907281011343, | |
| "kl": 0.17792066931724548, | |
| "learning_rate": 9.133858267716536e-05, | |
| "logps/chosen": NaN, | |
| "logps/rejected": NaN, | |
| "loss": 10.659, | |
| "rewards/chosen": NaN, | |
| "rewards/margins": NaN, | |
| "rewards/rejected": NaN, | |
| "step": 440 | |
| }, | |
| { | |
| "epoch": 1.7, | |
| "eval_kl": 0.8826082348823547, | |
| "eval_logps/chosen": -126.6418685913086, | |
| "eval_logps/rejected": -955.7069702148438, | |
| "eval_loss": 0.026313357055187225, | |
| "eval_rewards/chosen": 4.9077348709106445, | |
| "eval_rewards/margins": 81.17137908935547, | |
| "eval_rewards/rejected": -76.26364135742188, | |
| "eval_runtime": 474.6199, | |
| "eval_samples_per_second": 1.07, | |
| "eval_steps_per_second": 0.535, | |
| "step": 450 | |
| }, | |
| { | |
| "epoch": 1.74, | |
| "grad_norm": 0.03523483872413635, | |
| "kl": 0.21202507615089417, | |
| "learning_rate": 8.608923884514435e-05, | |
| "logps/chosen": NaN, | |
| "logps/rejected": NaN, | |
| "loss": 9.5266, | |
| "rewards/chosen": NaN, | |
| "rewards/margins": NaN, | |
| "rewards/rejected": NaN, | |
| "step": 460 | |
| }, | |
| { | |
| "epoch": 1.81, | |
| "grad_norm": 0.31107035279273987, | |
| "kl": 0.18568357825279236, | |
| "learning_rate": 8.083989501312336e-05, | |
| "logps/chosen": NaN, | |
| "logps/rejected": NaN, | |
| "loss": 4.6345, | |
| "rewards/chosen": NaN, | |
| "rewards/margins": NaN, | |
| "rewards/rejected": NaN, | |
| "step": 480 | |
| }, | |
| { | |
| "epoch": 1.89, | |
| "grad_norm": 0.05238344892859459, | |
| "kl": 0.0, | |
| "learning_rate": 7.559055118110236e-05, | |
| "logps/chosen": NaN, | |
| "logps/rejected": NaN, | |
| "loss": 177.5836, | |
| "rewards/chosen": NaN, | |
| "rewards/margins": NaN, | |
| "rewards/rejected": NaN, | |
| "step": 500 | |
| }, | |
| { | |
| "epoch": 1.89, | |
| "eval_kl": 0.47942328453063965, | |
| "eval_logps/chosen": -123.88298034667969, | |
| "eval_logps/rejected": -1018.103515625, | |
| "eval_loss": 0.01867891475558281, | |
| "eval_rewards/chosen": 5.183623790740967, | |
| "eval_rewards/margins": 87.6869125366211, | |
| "eval_rewards/rejected": -82.5032958984375, | |
| "eval_runtime": 474.5715, | |
| "eval_samples_per_second": 1.07, | |
| "eval_steps_per_second": 0.535, | |
| "step": 500 | |
| }, | |
| { | |
| "epoch": 1.96, | |
| "grad_norm": 0.01653907261788845, | |
| "kl": 0.29649871587753296, | |
| "learning_rate": 7.034120734908137e-05, | |
| "logps/chosen": NaN, | |
| "logps/rejected": NaN, | |
| "loss": 1.9292, | |
| "rewards/chosen": NaN, | |
| "rewards/margins": NaN, | |
| "rewards/rejected": NaN, | |
| "step": 520 | |
| }, | |
| { | |
| "epoch": 2.04, | |
| "grad_norm": 0.0456332191824913, | |
| "kl": 1.480500340461731, | |
| "learning_rate": 6.509186351706036e-05, | |
| "logps/chosen": NaN, | |
| "logps/rejected": NaN, | |
| "loss": 15.4933, | |
| "rewards/chosen": NaN, | |
| "rewards/margins": NaN, | |
| "rewards/rejected": NaN, | |
| "step": 540 | |
| }, | |
| { | |
| "epoch": 2.08, | |
| "eval_kl": 0.9202241897583008, | |
| "eval_logps/chosen": -127.73924255371094, | |
| "eval_logps/rejected": -1145.0382080078125, | |
| "eval_loss": 0.02805669605731964, | |
| "eval_rewards/chosen": 4.797997951507568, | |
| "eval_rewards/margins": 99.9947738647461, | |
| "eval_rewards/rejected": -95.19676971435547, | |
| "eval_runtime": 474.6365, | |
| "eval_samples_per_second": 1.07, | |
| "eval_steps_per_second": 0.535, | |
| "step": 550 | |
| }, | |
| { | |
| "epoch": 2.11, | |
| "grad_norm": 0.03160392865538597, | |
| "kl": 1.869972825050354, | |
| "learning_rate": 5.984251968503938e-05, | |
| "logps/chosen": NaN, | |
| "logps/rejected": NaN, | |
| "loss": 2.8522, | |
| "rewards/chosen": NaN, | |
| "rewards/margins": NaN, | |
| "rewards/rejected": NaN, | |
| "step": 560 | |
| }, | |
| { | |
| "epoch": 2.19, | |
| "grad_norm": 0.02318180911242962, | |
| "kl": 1.334816575050354, | |
| "learning_rate": 5.4593175853018376e-05, | |
| "logps/chosen": NaN, | |
| "logps/rejected": NaN, | |
| "loss": 23.0498, | |
| "rewards/chosen": NaN, | |
| "rewards/margins": NaN, | |
| "rewards/rejected": NaN, | |
| "step": 580 | |
| }, | |
| { | |
| "epoch": 2.27, | |
| "grad_norm": 0.015581952407956123, | |
| "kl": 0.0, | |
| "learning_rate": 4.934383202099738e-05, | |
| "logps/chosen": NaN, | |
| "logps/rejected": NaN, | |
| "loss": 3.827, | |
| "rewards/chosen": NaN, | |
| "rewards/margins": NaN, | |
| "rewards/rejected": NaN, | |
| "step": 600 | |
| }, | |
| { | |
| "epoch": 2.27, | |
| "eval_kl": 0.4924721121788025, | |
| "eval_logps/chosen": -125.38406372070312, | |
| "eval_logps/rejected": -1163.0284423828125, | |
| "eval_loss": 0.017838381230831146, | |
| "eval_rewards/chosen": 5.033515930175781, | |
| "eval_rewards/margins": 102.029296875, | |
| "eval_rewards/rejected": -96.99578094482422, | |
| "eval_runtime": 474.851, | |
| "eval_samples_per_second": 1.07, | |
| "eval_steps_per_second": 0.535, | |
| "step": 600 | |
| } | |
| ], | |
| "logging_steps": 20, | |
| "max_steps": 786, | |
| "num_input_tokens_seen": 0, | |
| "num_train_epochs": 3, | |
| "save_steps": 100, | |
| "total_flos": 0.0, | |
| "train_batch_size": 2, | |
| "trial_name": null, | |
| "trial_params": null | |
| } | |