| { | |
| "best_metric": null, | |
| "best_model_checkpoint": null, | |
| "epoch": 20.0, | |
| "eval_steps": 1000, | |
| "global_step": 1000, | |
| "is_hyper_param_search": false, | |
| "is_local_process_zero": true, | |
| "is_world_process_zero": true, | |
| "log_history": [ | |
| { | |
| "epoch": 0.02, | |
| "grad_norm": 0.35460313427884693, | |
| "learning_rate": 5.0000000000000004e-08, | |
| "logits/chosen": -1.8231227397918701, | |
| "logits/rejected": -1.5560479164123535, | |
| "logps/chosen": -85.02825927734375, | |
| "logps/rejected": -71.86117553710938, | |
| "loss": 0.6931, | |
| "rewards/accuracies": 0.0, | |
| "rewards/chosen": 0.0, | |
| "rewards/margins": 0.0, | |
| "rewards/rejected": 0.0, | |
| "step": 1 | |
| }, | |
| { | |
| "epoch": 0.2, | |
| "grad_norm": 0.30411563112440204, | |
| "learning_rate": 5.000000000000001e-07, | |
| "logits/chosen": -1.7207438945770264, | |
| "logits/rejected": -1.698093295097351, | |
| "logps/chosen": -75.15734100341797, | |
| "logps/rejected": -80.09143829345703, | |
| "loss": 0.6931, | |
| "rewards/accuracies": 0.5, | |
| "rewards/chosen": 9.971462714020163e-05, | |
| "rewards/margins": 0.0002858855586964637, | |
| "rewards/rejected": -0.00018617100431583822, | |
| "step": 10 | |
| }, | |
| { | |
| "epoch": 0.4, | |
| "grad_norm": 0.29583445812044185, | |
| "learning_rate": 1.0000000000000002e-06, | |
| "logits/chosen": -1.7815284729003906, | |
| "logits/rejected": -1.7640860080718994, | |
| "logps/chosen": -74.8746566772461, | |
| "logps/rejected": -77.6731185913086, | |
| "loss": 0.6927, | |
| "rewards/accuracies": 0.6499999761581421, | |
| "rewards/chosen": 0.00024838969693519175, | |
| "rewards/margins": 0.0008968595648184419, | |
| "rewards/rejected": -0.0006484698969870806, | |
| "step": 20 | |
| }, | |
| { | |
| "epoch": 0.6, | |
| "grad_norm": 0.2966957448432536, | |
| "learning_rate": 1.5e-06, | |
| "logits/chosen": -1.739800214767456, | |
| "logits/rejected": -1.7370967864990234, | |
| "logps/chosen": -75.44456481933594, | |
| "logps/rejected": -77.48068237304688, | |
| "loss": 0.6917, | |
| "rewards/accuracies": 0.875, | |
| "rewards/chosen": 0.0010354876285418868, | |
| "rewards/margins": 0.0030451309867203236, | |
| "rewards/rejected": -0.002009643241763115, | |
| "step": 30 | |
| }, | |
| { | |
| "epoch": 0.8, | |
| "grad_norm": 0.33102320941887775, | |
| "learning_rate": 2.0000000000000003e-06, | |
| "logits/chosen": -1.7791082859039307, | |
| "logits/rejected": -1.733616590499878, | |
| "logps/chosen": -76.32530212402344, | |
| "logps/rejected": -75.00889587402344, | |
| "loss": 0.689, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": 0.007160990033298731, | |
| "rewards/margins": 0.008440766483545303, | |
| "rewards/rejected": -0.0012797765666618943, | |
| "step": 40 | |
| }, | |
| { | |
| "epoch": 1.0, | |
| "grad_norm": 0.5247728899006847, | |
| "learning_rate": 2.5e-06, | |
| "logits/chosen": -1.7112547159194946, | |
| "logits/rejected": -1.7392151355743408, | |
| "logps/chosen": -73.26392364501953, | |
| "logps/rejected": -79.31649780273438, | |
| "loss": 0.6832, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": 0.013615945354104042, | |
| "rewards/margins": 0.019877230748534203, | |
| "rewards/rejected": -0.006261286325752735, | |
| "step": 50 | |
| }, | |
| { | |
| "epoch": 1.2, | |
| "grad_norm": 0.4135013743852781, | |
| "learning_rate": 3e-06, | |
| "logits/chosen": -1.732405424118042, | |
| "logits/rejected": -1.7449699640274048, | |
| "logps/chosen": -71.77066040039062, | |
| "logps/rejected": -80.100341796875, | |
| "loss": 0.6732, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": 0.024410251528024673, | |
| "rewards/margins": 0.03932947292923927, | |
| "rewards/rejected": -0.01491921953856945, | |
| "step": 60 | |
| }, | |
| { | |
| "epoch": 1.4, | |
| "grad_norm": 0.392415113956775, | |
| "learning_rate": 3.5e-06, | |
| "logits/chosen": -1.716195821762085, | |
| "logits/rejected": -1.731308937072754, | |
| "logps/chosen": -72.06253051757812, | |
| "logps/rejected": -80.39039611816406, | |
| "loss": 0.6542, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": 0.04634103178977966, | |
| "rewards/margins": 0.07862894982099533, | |
| "rewards/rejected": -0.032287921756505966, | |
| "step": 70 | |
| }, | |
| { | |
| "epoch": 1.6, | |
| "grad_norm": 0.5321176658546207, | |
| "learning_rate": 4.000000000000001e-06, | |
| "logits/chosen": -1.69076406955719, | |
| "logits/rejected": -1.6509063243865967, | |
| "logps/chosen": -68.82267761230469, | |
| "logps/rejected": -88.67816162109375, | |
| "loss": 0.605, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": 0.08192411065101624, | |
| "rewards/margins": 0.19158878922462463, | |
| "rewards/rejected": -0.1096646636724472, | |
| "step": 80 | |
| }, | |
| { | |
| "epoch": 1.8, | |
| "grad_norm": 0.5696532480733786, | |
| "learning_rate": 4.5e-06, | |
| "logits/chosen": -1.7061433792114258, | |
| "logits/rejected": -1.684195876121521, | |
| "logps/chosen": -59.4539680480957, | |
| "logps/rejected": -106.4749526977539, | |
| "loss": 0.4742, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": 0.18270966410636902, | |
| "rewards/margins": 0.5085535645484924, | |
| "rewards/rejected": -0.325843870639801, | |
| "step": 90 | |
| }, | |
| { | |
| "epoch": 2.0, | |
| "grad_norm": 0.3568428380556091, | |
| "learning_rate": 5e-06, | |
| "logits/chosen": -1.5610880851745605, | |
| "logits/rejected": -1.4533908367156982, | |
| "logps/chosen": -48.31792449951172, | |
| "logps/rejected": -164.77566528320312, | |
| "loss": 0.3175, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": 0.2694377899169922, | |
| "rewards/margins": 1.1592490673065186, | |
| "rewards/rejected": -0.8898113965988159, | |
| "step": 100 | |
| }, | |
| { | |
| "epoch": 2.2, | |
| "grad_norm": 0.28794719016802656, | |
| "learning_rate": 4.99847706754774e-06, | |
| "logits/chosen": -1.5363928079605103, | |
| "logits/rejected": -1.4212225675582886, | |
| "logps/chosen": -42.558448791503906, | |
| "logps/rejected": -187.63729858398438, | |
| "loss": 0.2626, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": 0.39137277007102966, | |
| "rewards/margins": 1.5502681732177734, | |
| "rewards/rejected": -1.1588953733444214, | |
| "step": 110 | |
| }, | |
| { | |
| "epoch": 2.4, | |
| "grad_norm": 0.15078063761313237, | |
| "learning_rate": 4.993910125649561e-06, | |
| "logits/chosen": -1.51887845993042, | |
| "logits/rejected": -1.437011957168579, | |
| "logps/chosen": -38.434608459472656, | |
| "logps/rejected": -232.57766723632812, | |
| "loss": 0.223, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": 0.42294931411743164, | |
| "rewards/margins": 2.0381760597229004, | |
| "rewards/rejected": -1.6152267456054688, | |
| "step": 120 | |
| }, | |
| { | |
| "epoch": 2.6, | |
| "grad_norm": 0.4347861863265109, | |
| "learning_rate": 4.986304738420684e-06, | |
| "logits/chosen": -1.3788427114486694, | |
| "logits/rejected": -1.37319016456604, | |
| "logps/chosen": -33.37788391113281, | |
| "logps/rejected": -288.50018310546875, | |
| "loss": 0.1776, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": 0.44394317269325256, | |
| "rewards/margins": 2.571049928665161, | |
| "rewards/rejected": -2.1271069049835205, | |
| "step": 130 | |
| }, | |
| { | |
| "epoch": 2.8, | |
| "grad_norm": 0.7461810630936248, | |
| "learning_rate": 4.975670171853926e-06, | |
| "logits/chosen": -1.190661072731018, | |
| "logits/rejected": -1.1670948266983032, | |
| "logps/chosen": -32.81340026855469, | |
| "logps/rejected": -336.36962890625, | |
| "loss": 0.1102, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": 0.4285716414451599, | |
| "rewards/margins": 3.032412528991699, | |
| "rewards/rejected": -2.6038408279418945, | |
| "step": 140 | |
| }, | |
| { | |
| "epoch": 3.0, | |
| "grad_norm": 0.053630215551159044, | |
| "learning_rate": 4.962019382530521e-06, | |
| "logits/chosen": -1.1552760601043701, | |
| "logits/rejected": -1.1048529148101807, | |
| "logps/chosen": -60.1913948059082, | |
| "logps/rejected": -567.5430908203125, | |
| "loss": 0.0207, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": 0.1675727218389511, | |
| "rewards/margins": 5.098362922668457, | |
| "rewards/rejected": -4.930790901184082, | |
| "step": 150 | |
| }, | |
| { | |
| "epoch": 3.2, | |
| "grad_norm": 0.031903287512855885, | |
| "learning_rate": 4.9453690018345144e-06, | |
| "logits/chosen": -1.0522866249084473, | |
| "logits/rejected": -1.0280696153640747, | |
| "logps/chosen": -102.0748062133789, | |
| "logps/rejected": -891.6737060546875, | |
| "loss": 0.0036, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": -0.2856892943382263, | |
| "rewards/margins": 7.848362922668457, | |
| "rewards/rejected": -8.134051322937012, | |
| "step": 160 | |
| }, | |
| { | |
| "epoch": 3.4, | |
| "grad_norm": 0.0197914954559647, | |
| "learning_rate": 4.925739315689991e-06, | |
| "logits/chosen": -1.1597774028778076, | |
| "logits/rejected": -1.1589906215667725, | |
| "logps/chosen": -61.60524368286133, | |
| "logps/rejected": -1039.020751953125, | |
| "loss": 0.0022, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": 0.12755107879638672, | |
| "rewards/margins": 9.746142387390137, | |
| "rewards/rejected": -9.61859130859375, | |
| "step": 170 | |
| }, | |
| { | |
| "epoch": 3.6, | |
| "grad_norm": 0.023011571746249113, | |
| "learning_rate": 4.903154239845798e-06, | |
| "logits/chosen": -1.2340686321258545, | |
| "logits/rejected": -1.0457987785339355, | |
| "logps/chosen": -59.74028778076172, | |
| "logps/rejected": -1156.054931640625, | |
| "loss": 0.0014, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": 0.25307127833366394, | |
| "rewards/margins": 11.132636070251465, | |
| "rewards/rejected": -10.879566192626953, | |
| "step": 180 | |
| }, | |
| { | |
| "epoch": 3.8, | |
| "grad_norm": 0.010025707600426755, | |
| "learning_rate": 4.8776412907378845e-06, | |
| "logits/chosen": -1.0119726657867432, | |
| "logits/rejected": -1.0299015045166016, | |
| "logps/chosen": -51.19286346435547, | |
| "logps/rejected": -1030.427490234375, | |
| "loss": 0.0015, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": 0.22199800610542297, | |
| "rewards/margins": 9.714365005493164, | |
| "rewards/rejected": -9.492366790771484, | |
| "step": 190 | |
| }, | |
| { | |
| "epoch": 4.0, | |
| "grad_norm": 0.009363825045997377, | |
| "learning_rate": 4.849231551964771e-06, | |
| "logits/chosen": -1.1452789306640625, | |
| "logits/rejected": -1.0538662672042847, | |
| "logps/chosen": -55.855430603027344, | |
| "logps/rejected": -1116.4219970703125, | |
| "loss": 0.0012, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": 0.2577177882194519, | |
| "rewards/margins": 10.68994140625, | |
| "rewards/rejected": -10.43222427368164, | |
| "step": 200 | |
| }, | |
| { | |
| "epoch": 4.2, | |
| "grad_norm": 0.007917186035092368, | |
| "learning_rate": 4.817959636416969e-06, | |
| "logits/chosen": -0.979863166809082, | |
| "logits/rejected": -0.954973042011261, | |
| "logps/chosen": -51.2191276550293, | |
| "logps/rejected": -1067.7679443359375, | |
| "loss": 0.0012, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": 0.21226799488067627, | |
| "rewards/margins": 10.104147911071777, | |
| "rewards/rejected": -9.89188003540039, | |
| "step": 210 | |
| }, | |
| { | |
| "epoch": 4.4, | |
| "grad_norm": 0.004467951143379201, | |
| "learning_rate": 4.783863644106502e-06, | |
| "logits/chosen": -1.0173983573913574, | |
| "logits/rejected": -0.9798928499221802, | |
| "logps/chosen": -52.295379638671875, | |
| "logps/rejected": -1050.406494140625, | |
| "loss": 0.001, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": 0.21525339782238007, | |
| "rewards/margins": 9.93811321258545, | |
| "rewards/rejected": -9.722859382629395, | |
| "step": 220 | |
| }, | |
| { | |
| "epoch": 4.6, | |
| "grad_norm": 0.0036885023764270065, | |
| "learning_rate": 4.746985115747918e-06, | |
| "logits/chosen": -1.1039237976074219, | |
| "logits/rejected": -0.9902089238166809, | |
| "logps/chosen": -56.54585647583008, | |
| "logps/rejected": -1109.8258056640625, | |
| "loss": 0.0008, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": 0.2429007589817047, | |
| "rewards/margins": 10.61030101776123, | |
| "rewards/rejected": -10.367398262023926, | |
| "step": 230 | |
| }, | |
| { | |
| "epoch": 4.8, | |
| "grad_norm": 0.0065247264850717725, | |
| "learning_rate": 4.707368982147318e-06, | |
| "logits/chosen": -1.1152589321136475, | |
| "logits/rejected": -1.0077005624771118, | |
| "logps/chosen": -54.775856018066406, | |
| "logps/rejected": -1131.518310546875, | |
| "loss": 0.0008, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": 0.22144417464733124, | |
| "rewards/margins": 10.786369323730469, | |
| "rewards/rejected": -10.564926147460938, | |
| "step": 240 | |
| }, | |
| { | |
| "epoch": 5.0, | |
| "grad_norm": 0.0032674373668132675, | |
| "learning_rate": 4.665063509461098e-06, | |
| "logits/chosen": -1.0255931615829468, | |
| "logits/rejected": -0.9475139379501343, | |
| "logps/chosen": -56.231163024902344, | |
| "logps/rejected": -1138.18115234375, | |
| "loss": 0.0008, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": 0.22440902888774872, | |
| "rewards/margins": 10.85453987121582, | |
| "rewards/rejected": -10.630131721496582, | |
| "step": 250 | |
| }, | |
| { | |
| "epoch": 5.2, | |
| "grad_norm": 0.004115527916331285, | |
| "learning_rate": 4.620120240391065e-06, | |
| "logits/chosen": -1.0400116443634033, | |
| "logits/rejected": -0.9953556060791016, | |
| "logps/chosen": -54.63758087158203, | |
| "logps/rejected": -1113.4554443359375, | |
| "loss": 0.0007, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": 0.21181125938892365, | |
| "rewards/margins": 10.56176471710205, | |
| "rewards/rejected": -10.34995174407959, | |
| "step": 260 | |
| }, | |
| { | |
| "epoch": 5.4, | |
| "grad_norm": 0.0030662391394572284, | |
| "learning_rate": 4.572593931387604e-06, | |
| "logits/chosen": -1.100754976272583, | |
| "logits/rejected": -0.9190059900283813, | |
| "logps/chosen": -55.8875846862793, | |
| "logps/rejected": -1182.8133544921875, | |
| "loss": 0.0007, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": 0.2545715868473053, | |
| "rewards/margins": 11.359747886657715, | |
| "rewards/rejected": -11.105175018310547, | |
| "step": 270 | |
| }, | |
| { | |
| "epoch": 5.6, | |
| "grad_norm": 0.004246284055742857, | |
| "learning_rate": 4.522542485937369e-06, | |
| "logits/chosen": -1.0873005390167236, | |
| "logits/rejected": -0.91912841796875, | |
| "logps/chosen": -57.15370559692383, | |
| "logps/rejected": -1198.202392578125, | |
| "loss": 0.0007, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": 0.24588795006275177, | |
| "rewards/margins": 11.502182960510254, | |
| "rewards/rejected": -11.256293296813965, | |
| "step": 280 | |
| }, | |
| { | |
| "epoch": 5.8, | |
| "grad_norm": 0.004139329336955742, | |
| "learning_rate": 4.470026884016805e-06, | |
| "logits/chosen": -0.9481426477432251, | |
| "logits/rejected": -0.9623401761054993, | |
| "logps/chosen": -52.13285446166992, | |
| "logps/rejected": -1074.572509765625, | |
| "loss": 0.0007, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": 0.20849105715751648, | |
| "rewards/margins": 10.169190406799316, | |
| "rewards/rejected": -9.960699081420898, | |
| "step": 290 | |
| }, | |
| { | |
| "epoch": 6.0, | |
| "grad_norm": 0.003034900790871541, | |
| "learning_rate": 4.415111107797445e-06, | |
| "logits/chosen": -1.0314205884933472, | |
| "logits/rejected": -1.0098388195037842, | |
| "logps/chosen": -54.42866134643555, | |
| "logps/rejected": -1097.184326171875, | |
| "loss": 0.0006, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": 0.20703566074371338, | |
| "rewards/margins": 10.398859024047852, | |
| "rewards/rejected": -10.19182300567627, | |
| "step": 300 | |
| }, | |
| { | |
| "epoch": 6.2, | |
| "grad_norm": 0.0038510374493456907, | |
| "learning_rate": 4.357862063693486e-06, | |
| "logits/chosen": -1.026374101638794, | |
| "logits/rejected": -0.9435133934020996, | |
| "logps/chosen": -53.9544792175293, | |
| "logps/rejected": -1142.4066162109375, | |
| "loss": 0.0006, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": 0.21439966559410095, | |
| "rewards/margins": 10.869270324707031, | |
| "rewards/rejected": -10.654870986938477, | |
| "step": 310 | |
| }, | |
| { | |
| "epoch": 6.4, | |
| "grad_norm": 0.0035014469049547427, | |
| "learning_rate": 4.2983495008466285e-06, | |
| "logits/chosen": -1.105359673500061, | |
| "logits/rejected": -0.9964410066604614, | |
| "logps/chosen": -57.14154815673828, | |
| "logps/rejected": -1162.7977294921875, | |
| "loss": 0.0006, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": 0.21488821506500244, | |
| "rewards/margins": 11.113906860351562, | |
| "rewards/rejected": -10.899017333984375, | |
| "step": 320 | |
| }, | |
| { | |
| "epoch": 6.6, | |
| "grad_norm": 0.002634385062559098, | |
| "learning_rate": 4.236645926147493e-06, | |
| "logits/chosen": -1.0640833377838135, | |
| "logits/rejected": -0.9242901802062988, | |
| "logps/chosen": -56.49944305419922, | |
| "logps/rejected": -1212.7708740234375, | |
| "loss": 0.0005, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": 0.23322686553001404, | |
| "rewards/margins": 11.629671096801758, | |
| "rewards/rejected": -11.396444320678711, | |
| "step": 330 | |
| }, | |
| { | |
| "epoch": 6.8, | |
| "grad_norm": 0.0030996294667543192, | |
| "learning_rate": 4.172826515897146e-06, | |
| "logits/chosen": -1.082220435142517, | |
| "logits/rejected": -0.9698995351791382, | |
| "logps/chosen": -59.28369903564453, | |
| "logps/rejected": -1214.587646484375, | |
| "loss": 0.0005, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": 0.19337359070777893, | |
| "rewards/margins": 11.616683959960938, | |
| "rewards/rejected": -11.423309326171875, | |
| "step": 340 | |
| }, | |
| { | |
| "epoch": 7.0, | |
| "grad_norm": 0.0015310049705482352, | |
| "learning_rate": 4.106969024216348e-06, | |
| "logits/chosen": -0.9850069284439087, | |
| "logits/rejected": -0.9987241625785828, | |
| "logps/chosen": -54.036949157714844, | |
| "logps/rejected": -1109.0908203125, | |
| "loss": 0.0006, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": 0.19889585673809052, | |
| "rewards/margins": 10.489348411560059, | |
| "rewards/rejected": -10.29045295715332, | |
| "step": 350 | |
| }, | |
| { | |
| "epoch": 7.2, | |
| "grad_norm": 0.001899366605942392, | |
| "learning_rate": 4.039153688314146e-06, | |
| "logits/chosen": -1.0366054773330688, | |
| "logits/rejected": -0.9512063264846802, | |
| "logps/chosen": -56.377410888671875, | |
| "logps/rejected": -1194.2122802734375, | |
| "loss": 0.0005, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": 0.2178737223148346, | |
| "rewards/margins": 11.424546241760254, | |
| "rewards/rejected": -11.206673622131348, | |
| "step": 360 | |
| }, | |
| { | |
| "epoch": 7.4, | |
| "grad_norm": 0.003085448477067617, | |
| "learning_rate": 3.969463130731183e-06, | |
| "logits/chosen": -1.058009147644043, | |
| "logits/rejected": -0.9505330324172974, | |
| "logps/chosen": -55.610870361328125, | |
| "logps/rejected": -1222.853515625, | |
| "loss": 0.0005, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": 0.2119036167860031, | |
| "rewards/margins": 11.69975471496582, | |
| "rewards/rejected": -11.4878511428833, | |
| "step": 370 | |
| }, | |
| { | |
| "epoch": 7.6, | |
| "grad_norm": 0.002631100351440524, | |
| "learning_rate": 3.897982258676867e-06, | |
| "logits/chosen": -0.9967459440231323, | |
| "logits/rejected": -1.021655797958374, | |
| "logps/chosen": -52.216705322265625, | |
| "logps/rejected": -1100.6519775390625, | |
| "loss": 0.0005, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": 0.19106148183345795, | |
| "rewards/margins": 10.369633674621582, | |
| "rewards/rejected": -10.178572654724121, | |
| "step": 380 | |
| }, | |
| { | |
| "epoch": 7.8, | |
| "grad_norm": 0.0031329070413845662, | |
| "learning_rate": 3.824798160583012e-06, | |
| "logits/chosen": -1.0442421436309814, | |
| "logits/rejected": -0.9009174108505249, | |
| "logps/chosen": -56.89820098876953, | |
| "logps/rejected": -1184.45263671875, | |
| "loss": 0.0005, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": 0.22130858898162842, | |
| "rewards/margins": 11.319818496704102, | |
| "rewards/rejected": -11.098509788513184, | |
| "step": 390 | |
| }, | |
| { | |
| "epoch": 8.0, | |
| "grad_norm": 0.003423988945465294, | |
| "learning_rate": 3.7500000000000005e-06, | |
| "logits/chosen": -1.035902500152588, | |
| "logits/rejected": -0.9636209607124329, | |
| "logps/chosen": -56.99961471557617, | |
| "logps/rejected": -1206.747802734375, | |
| "loss": 0.0005, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": 0.18911413848400116, | |
| "rewards/margins": 11.49039363861084, | |
| "rewards/rejected": -11.301280975341797, | |
| "step": 400 | |
| }, | |
| { | |
| "epoch": 8.2, | |
| "grad_norm": 0.003864669112701001, | |
| "learning_rate": 3.6736789069647273e-06, | |
| "logits/chosen": -0.9635273218154907, | |
| "logits/rejected": -0.8587255477905273, | |
| "logps/chosen": -58.7963981628418, | |
| "logps/rejected": -1161.4803466796875, | |
| "loss": 0.0005, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": 0.19340120255947113, | |
| "rewards/margins": 11.068787574768066, | |
| "rewards/rejected": -10.875386238098145, | |
| "step": 410 | |
| }, | |
| { | |
| "epoch": 8.4, | |
| "grad_norm": 0.003245040701864265, | |
| "learning_rate": 3.595927866972694e-06, | |
| "logits/chosen": -1.0136067867279053, | |
| "logits/rejected": -0.9350364804267883, | |
| "logps/chosen": -55.97736358642578, | |
| "logps/rejected": -1197.21484375, | |
| "loss": 0.0004, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": 0.20746412873268127, | |
| "rewards/margins": 11.428886413574219, | |
| "rewards/rejected": -11.221421241760254, | |
| "step": 420 | |
| }, | |
| { | |
| "epoch": 8.6, | |
| "grad_norm": 0.0023560411348610777, | |
| "learning_rate": 3.516841607689501e-06, | |
| "logits/chosen": -0.9947836995124817, | |
| "logits/rejected": -0.9126423597335815, | |
| "logps/chosen": -57.561622619628906, | |
| "logps/rejected": -1165.994384765625, | |
| "loss": 0.0004, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": 0.18999549746513367, | |
| "rewards/margins": 11.074243545532227, | |
| "rewards/rejected": -10.884247779846191, | |
| "step": 430 | |
| }, | |
| { | |
| "epoch": 8.8, | |
| "grad_norm": 0.0025971993823853895, | |
| "learning_rate": 3.436516483539781e-06, | |
| "logits/chosen": -1.0256311893463135, | |
| "logits/rejected": -0.8973512649536133, | |
| "logps/chosen": -59.285438537597656, | |
| "logps/rejected": -1248.3521728515625, | |
| "loss": 0.0004, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": 0.18996688723564148, | |
| "rewards/margins": 11.936816215515137, | |
| "rewards/rejected": -11.746848106384277, | |
| "step": 440 | |
| }, | |
| { | |
| "epoch": 9.0, | |
| "grad_norm": 0.0032315637570330567, | |
| "learning_rate": 3.3550503583141726e-06, | |
| "logits/chosen": -1.0244309902191162, | |
| "logits/rejected": -0.8780086636543274, | |
| "logps/chosen": -57.84431076049805, | |
| "logps/rejected": -1234.871337890625, | |
| "loss": 0.0004, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": 0.1998531073331833, | |
| "rewards/margins": 11.807432174682617, | |
| "rewards/rejected": -11.607580184936523, | |
| "step": 450 | |
| }, | |
| { | |
| "epoch": 9.2, | |
| "grad_norm": 0.0036226784518706475, | |
| "learning_rate": 3.272542485937369e-06, | |
| "logits/chosen": -0.9375013113021851, | |
| "logits/rejected": -0.8737777471542358, | |
| "logps/chosen": -54.31487274169922, | |
| "logps/rejected": -1172.191162109375, | |
| "loss": 0.0004, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": 0.19032403826713562, | |
| "rewards/margins": 11.127653121948242, | |
| "rewards/rejected": -10.937328338623047, | |
| "step": 460 | |
| }, | |
| { | |
| "epoch": 9.4, | |
| "grad_norm": 0.002997930586361418, | |
| "learning_rate": 3.189093389542498e-06, | |
| "logits/chosen": -1.0432255268096924, | |
| "logits/rejected": -0.9277008175849915, | |
| "logps/chosen": -58.5209846496582, | |
| "logps/rejected": -1200.554931640625, | |
| "loss": 0.0004, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": 0.19790109992027283, | |
| "rewards/margins": 11.468005180358887, | |
| "rewards/rejected": -11.270103454589844, | |
| "step": 470 | |
| }, | |
| { | |
| "epoch": 9.6, | |
| "grad_norm": 0.0022499622323506746, | |
| "learning_rate": 3.1048047389991693e-06, | |
| "logits/chosen": -1.0687272548675537, | |
| "logits/rejected": -0.9239826202392578, | |
| "logps/chosen": -60.52696990966797, | |
| "logps/rejected": -1241.221435546875, | |
| "loss": 0.0004, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": 0.18703293800354004, | |
| "rewards/margins": 11.873010635375977, | |
| "rewards/rejected": -11.685976028442383, | |
| "step": 480 | |
| }, | |
| { | |
| "epoch": 9.8, | |
| "grad_norm": 0.0019412808634179527, | |
| "learning_rate": 3.019779227044398e-06, | |
| "logits/chosen": -0.9789573550224304, | |
| "logits/rejected": -0.7663965225219727, | |
| "logps/chosen": -61.904319763183594, | |
| "logps/rejected": -1237.324951171875, | |
| "loss": 0.0003, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": 0.1967342346906662, | |
| "rewards/margins": 11.862494468688965, | |
| "rewards/rejected": -11.66576099395752, | |
| "step": 490 | |
| }, | |
| { | |
| "epoch": 10.0, | |
| "grad_norm": 0.002323126626066593, | |
| "learning_rate": 2.9341204441673267e-06, | |
| "logits/chosen": -1.0079948902130127, | |
| "logits/rejected": -0.9249541163444519, | |
| "logps/chosen": -58.3544807434082, | |
| "logps/rejected": -1210.3780517578125, | |
| "loss": 0.0004, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": 0.16591690480709076, | |
| "rewards/margins": 11.488636016845703, | |
| "rewards/rejected": -11.32271957397461, | |
| "step": 500 | |
| }, | |
| { | |
| "epoch": 10.2, | |
| "grad_norm": 0.0018387058134298486, | |
| "learning_rate": 2.847932752400164e-06, | |
| "logits/chosen": -0.9845121502876282, | |
| "logits/rejected": -0.8383527994155884, | |
| "logps/chosen": -59.52973556518555, | |
| "logps/rejected": -1259.748291015625, | |
| "loss": 0.0004, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": 0.17635658383369446, | |
| "rewards/margins": 12.032411575317383, | |
| "rewards/rejected": -11.856054306030273, | |
| "step": 510 | |
| }, | |
| { | |
| "epoch": 10.4, | |
| "grad_norm": 0.0013643138700466504, | |
| "learning_rate": 2.761321158169134e-06, | |
| "logits/chosen": -1.0216516256332397, | |
| "logits/rejected": -0.9077258110046387, | |
| "logps/chosen": -59.731300354003906, | |
| "logps/rejected": -1231.614501953125, | |
| "loss": 0.0004, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": 0.1921447217464447, | |
| "rewards/margins": 11.769120216369629, | |
| "rewards/rejected": -11.576974868774414, | |
| "step": 520 | |
| }, | |
| { | |
| "epoch": 10.6, | |
| "grad_norm": 0.002977922307910112, | |
| "learning_rate": 2.6743911843603134e-06, | |
| "logits/chosen": -1.0051695108413696, | |
| "logits/rejected": -0.8601359128952026, | |
| "logps/chosen": -62.12017822265625, | |
| "logps/rejected": -1268.072509765625, | |
| "loss": 0.0003, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": 0.1786160171031952, | |
| "rewards/margins": 12.141373634338379, | |
| "rewards/rejected": -11.962759017944336, | |
| "step": 530 | |
| }, | |
| { | |
| "epoch": 10.8, | |
| "grad_norm": 0.0019745371274113147, | |
| "learning_rate": 2.587248741756253e-06, | |
| "logits/chosen": -1.0047599077224731, | |
| "logits/rejected": -0.8642932176589966, | |
| "logps/chosen": -61.087615966796875, | |
| "logps/rejected": -1251.1107177734375, | |
| "loss": 0.0003, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": 0.18852102756500244, | |
| "rewards/margins": 11.956985473632812, | |
| "rewards/rejected": -11.768463134765625, | |
| "step": 540 | |
| }, | |
| { | |
| "epoch": 11.0, | |
| "grad_norm": 0.0018986393686723218, | |
| "learning_rate": 2.5e-06, | |
| "logits/chosen": -1.0309114456176758, | |
| "logits/rejected": -0.9270389676094055, | |
| "logps/chosen": -59.46849822998047, | |
| "logps/rejected": -1210.13720703125, | |
| "loss": 0.0003, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": 0.18577028810977936, | |
| "rewards/margins": 11.525289535522461, | |
| "rewards/rejected": -11.339518547058105, | |
| "step": 550 | |
| }, | |
| { | |
| "epoch": 11.2, | |
| "grad_norm": 0.0016611092816266825, | |
| "learning_rate": 2.4127512582437486e-06, | |
| "logits/chosen": -0.9609254598617554, | |
| "logits/rejected": -0.9043042063713074, | |
| "logps/chosen": -54.84800338745117, | |
| "logps/rejected": -1216.4305419921875, | |
| "loss": 0.0003, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": 0.20413394272327423, | |
| "rewards/margins": 11.576135635375977, | |
| "rewards/rejected": -11.372001647949219, | |
| "step": 560 | |
| }, | |
| { | |
| "epoch": 11.4, | |
| "grad_norm": 0.0019080916476901313, | |
| "learning_rate": 2.325608815639687e-06, | |
| "logits/chosen": -0.9941130876541138, | |
| "logits/rejected": -0.8515572547912598, | |
| "logps/chosen": -59.786705017089844, | |
| "logps/rejected": -1280.828857421875, | |
| "loss": 0.0003, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": 0.19204822182655334, | |
| "rewards/margins": 12.262724876403809, | |
| "rewards/rejected": -12.070676803588867, | |
| "step": 570 | |
| }, | |
| { | |
| "epoch": 11.6, | |
| "grad_norm": 0.001417529626290414, | |
| "learning_rate": 2.238678841830867e-06, | |
| "logits/chosen": -1.0167783498764038, | |
| "logits/rejected": -0.8408117294311523, | |
| "logps/chosen": -61.850563049316406, | |
| "logps/rejected": -1273.6541748046875, | |
| "loss": 0.0003, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": 0.17794856429100037, | |
| "rewards/margins": 12.197797775268555, | |
| "rewards/rejected": -12.019848823547363, | |
| "step": 580 | |
| }, | |
| { | |
| "epoch": 11.8, | |
| "grad_norm": 0.001985700625240649, | |
| "learning_rate": 2.1520672475998374e-06, | |
| "logits/chosen": -0.9488497972488403, | |
| "logits/rejected": -0.8910242319107056, | |
| "logps/chosen": -56.951904296875, | |
| "logps/rejected": -1185.1470947265625, | |
| "loss": 0.0003, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": 0.17158612608909607, | |
| "rewards/margins": 11.239965438842773, | |
| "rewards/rejected": -11.068379402160645, | |
| "step": 590 | |
| }, | |
| { | |
| "epoch": 12.0, | |
| "grad_norm": 0.0014036347657331724, | |
| "learning_rate": 2.0658795558326745e-06, | |
| "logits/chosen": -0.9536435008049011, | |
| "logits/rejected": -0.8297941088676453, | |
| "logps/chosen": -57.86296463012695, | |
| "logps/rejected": -1237.7889404296875, | |
| "loss": 0.0003, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": 0.1768932044506073, | |
| "rewards/margins": 11.793856620788574, | |
| "rewards/rejected": -11.616963386535645, | |
| "step": 600 | |
| }, | |
| { | |
| "epoch": 12.2, | |
| "grad_norm": 0.0014637474046611807, | |
| "learning_rate": 1.9802207729556023e-06, | |
| "logits/chosen": -0.9591706395149231, | |
| "logits/rejected": -0.8812329173088074, | |
| "logps/chosen": -56.78215789794922, | |
| "logps/rejected": -1231.258056640625, | |
| "loss": 0.0003, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": 0.1855459213256836, | |
| "rewards/margins": 11.722236633300781, | |
| "rewards/rejected": -11.536690711975098, | |
| "step": 610 | |
| }, | |
| { | |
| "epoch": 12.4, | |
| "grad_norm": 0.0017805033618398401, | |
| "learning_rate": 1.895195261000831e-06, | |
| "logits/chosen": -1.019749641418457, | |
| "logits/rejected": -0.9892827868461609, | |
| "logps/chosen": -58.21343994140625, | |
| "logps/rejected": -1209.8052978515625, | |
| "loss": 0.0003, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": 0.16973453760147095, | |
| "rewards/margins": 11.481805801391602, | |
| "rewards/rejected": -11.312070846557617, | |
| "step": 620 | |
| }, | |
| { | |
| "epoch": 12.6, | |
| "grad_norm": 0.0011184561683295657, | |
| "learning_rate": 1.8109066104575023e-06, | |
| "logits/chosen": -0.9216413497924805, | |
| "logits/rejected": -0.8319498300552368, | |
| "logps/chosen": -58.66545486450195, | |
| "logps/rejected": -1199.0224609375, | |
| "loss": 0.0003, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": 0.17631840705871582, | |
| "rewards/margins": 11.408339500427246, | |
| "rewards/rejected": -11.232022285461426, | |
| "step": 630 | |
| }, | |
| { | |
| "epoch": 12.8, | |
| "grad_norm": 0.0014408987665450045, | |
| "learning_rate": 1.7274575140626318e-06, | |
| "logits/chosen": -0.9979192614555359, | |
| "logits/rejected": -0.7878575325012207, | |
| "logps/chosen": -62.93827438354492, | |
| "logps/rejected": -1300.874755859375, | |
| "loss": 0.0003, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": 0.18857316672801971, | |
| "rewards/margins": 12.50341510772705, | |
| "rewards/rejected": -12.314840316772461, | |
| "step": 640 | |
| }, | |
| { | |
| "epoch": 13.0, | |
| "grad_norm": 0.0020645792333772377, | |
| "learning_rate": 1.6449496416858285e-06, | |
| "logits/chosen": -0.9246803522109985, | |
| "logits/rejected": -0.8380001783370972, | |
| "logps/chosen": -57.9194221496582, | |
| "logps/rejected": -1195.3837890625, | |
| "loss": 0.0003, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": 0.16603608429431915, | |
| "rewards/margins": 11.333303451538086, | |
| "rewards/rejected": -11.167266845703125, | |
| "step": 650 | |
| }, | |
| { | |
| "epoch": 13.2, | |
| "grad_norm": 0.0015139495857028652, | |
| "learning_rate": 1.56348351646022e-06, | |
| "logits/chosen": -1.0453752279281616, | |
| "logits/rejected": -0.8043217658996582, | |
| "logps/chosen": -63.8051872253418, | |
| "logps/rejected": -1392.04443359375, | |
| "loss": 0.0002, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": 0.19428420066833496, | |
| "rewards/margins": 13.428848266601562, | |
| "rewards/rejected": -13.234565734863281, | |
| "step": 660 | |
| }, | |
| { | |
| "epoch": 13.4, | |
| "grad_norm": 0.0013498029319408118, | |
| "learning_rate": 1.4831583923105e-06, | |
| "logits/chosen": -0.979832649230957, | |
| "logits/rejected": -0.8365589380264282, | |
| "logps/chosen": -60.9407844543457, | |
| "logps/rejected": -1260.807861328125, | |
| "loss": 0.0003, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": 0.16681666672229767, | |
| "rewards/margins": 12.033578872680664, | |
| "rewards/rejected": -11.866762161254883, | |
| "step": 670 | |
| }, | |
| { | |
| "epoch": 13.6, | |
| "grad_norm": 0.0018616946971343861, | |
| "learning_rate": 1.4040721330273063e-06, | |
| "logits/chosen": -0.9041559100151062, | |
| "logits/rejected": -0.8964747190475464, | |
| "logps/chosen": -54.83356857299805, | |
| "logps/rejected": -1143.5806884765625, | |
| "loss": 0.0003, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": 0.16955164074897766, | |
| "rewards/margins": 10.786935806274414, | |
| "rewards/rejected": -10.617385864257812, | |
| "step": 680 | |
| }, | |
| { | |
| "epoch": 13.8, | |
| "grad_norm": 0.0012150873329068216, | |
| "learning_rate": 1.3263210930352737e-06, | |
| "logits/chosen": -0.8522924184799194, | |
| "logits/rejected": -0.8536826968193054, | |
| "logps/chosen": -57.020355224609375, | |
| "logps/rejected": -1151.3582763671875, | |
| "loss": 0.0003, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": 0.16875942051410675, | |
| "rewards/margins": 10.881406784057617, | |
| "rewards/rejected": -10.712648391723633, | |
| "step": 690 | |
| }, | |
| { | |
| "epoch": 14.0, | |
| "grad_norm": 0.0015006535309411454, | |
| "learning_rate": 1.2500000000000007e-06, | |
| "logits/chosen": -0.9550180435180664, | |
| "logits/rejected": -0.8258699178695679, | |
| "logps/chosen": -58.025535583496094, | |
| "logps/rejected": -1249.10546875, | |
| "loss": 0.0003, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": 0.17841221392154694, | |
| "rewards/margins": 11.912469863891602, | |
| "rewards/rejected": -11.73405647277832, | |
| "step": 700 | |
| }, | |
| { | |
| "epoch": 14.2, | |
| "grad_norm": 0.0015715381830458688, | |
| "learning_rate": 1.1752018394169882e-06, | |
| "logits/chosen": -0.9631673693656921, | |
| "logits/rejected": -0.8420494794845581, | |
| "logps/chosen": -60.133888244628906, | |
| "logps/rejected": -1249.9853515625, | |
| "loss": 0.0002, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": 0.1643882542848587, | |
| "rewards/margins": 11.908964157104492, | |
| "rewards/rejected": -11.744573593139648, | |
| "step": 710 | |
| }, | |
| { | |
| "epoch": 14.4, | |
| "grad_norm": 0.0013660785683358696, | |
| "learning_rate": 1.1020177413231334e-06, | |
| "logits/chosen": -0.8709633946418762, | |
| "logits/rejected": -0.8834745287895203, | |
| "logps/chosen": -54.35590362548828, | |
| "logps/rejected": -1181.2337646484375, | |
| "loss": 0.0003, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": 0.17913100123405457, | |
| "rewards/margins": 11.173266410827637, | |
| "rewards/rejected": -10.994134902954102, | |
| "step": 720 | |
| }, | |
| { | |
| "epoch": 14.6, | |
| "grad_norm": 0.0016180177521581223, | |
| "learning_rate": 1.0305368692688175e-06, | |
| "logits/chosen": -0.9120365381240845, | |
| "logits/rejected": -0.7947243452072144, | |
| "logps/chosen": -59.50437545776367, | |
| "logps/rejected": -1239.4515380859375, | |
| "loss": 0.0003, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": 0.15415535867214203, | |
| "rewards/margins": 11.778593063354492, | |
| "rewards/rejected": -11.624438285827637, | |
| "step": 730 | |
| }, | |
| { | |
| "epoch": 14.8, | |
| "grad_norm": 0.0009131551082822374, | |
| "learning_rate": 9.608463116858544e-07, | |
| "logits/chosen": -0.9982080459594727, | |
| "logits/rejected": -0.8127807378768921, | |
| "logps/chosen": -62.322227478027344, | |
| "logps/rejected": -1302.121337890625, | |
| "loss": 0.0003, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": 0.17191007733345032, | |
| "rewards/margins": 12.468157768249512, | |
| "rewards/rejected": -12.296248435974121, | |
| "step": 740 | |
| }, | |
| { | |
| "epoch": 15.0, | |
| "grad_norm": 0.004586809943697903, | |
| "learning_rate": 8.930309757836517e-07, | |
| "logits/chosen": -0.9402343034744263, | |
| "logits/rejected": -0.8805861473083496, | |
| "logps/chosen": -56.05229568481445, | |
| "logps/rejected": -1228.699951171875, | |
| "loss": 0.0003, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": 0.19031982123851776, | |
| "rewards/margins": 11.688825607299805, | |
| "rewards/rejected": -11.498506546020508, | |
| "step": 750 | |
| }, | |
| { | |
| "epoch": 15.2, | |
| "grad_norm": 0.001855508098495543, | |
| "learning_rate": 8.271734841028553e-07, | |
| "logits/chosen": -0.9797531962394714, | |
| "logits/rejected": -0.8751746416091919, | |
| "logps/chosen": -59.685791015625, | |
| "logps/rejected": -1249.674560546875, | |
| "loss": 0.0003, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": 0.17554932832717896, | |
| "rewards/margins": 11.926473617553711, | |
| "rewards/rejected": -11.750923156738281, | |
| "step": 760 | |
| }, | |
| { | |
| "epoch": 15.4, | |
| "grad_norm": 0.0009559205861326566, | |
| "learning_rate": 7.633540738525066e-07, | |
| "logits/chosen": -0.9692495465278625, | |
| "logits/rejected": -0.8025411367416382, | |
| "logps/chosen": -60.701194763183594, | |
| "logps/rejected": -1309.8841552734375, | |
| "loss": 0.0002, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": 0.183763325214386, | |
| "rewards/margins": 12.551748275756836, | |
| "rewards/rejected": -12.367984771728516, | |
| "step": 770 | |
| }, | |
| { | |
| "epoch": 15.6, | |
| "grad_norm": 0.0016221612239979135, | |
| "learning_rate": 7.016504991533727e-07, | |
| "logits/chosen": -0.9435825347900391, | |
| "logits/rejected": -0.873455822467804, | |
| "logps/chosen": -58.68913650512695, | |
| "logps/rejected": -1226.319091796875, | |
| "loss": 0.0003, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": 0.16507244110107422, | |
| "rewards/margins": 11.652719497680664, | |
| "rewards/rejected": -11.487646102905273, | |
| "step": 780 | |
| }, | |
| { | |
| "epoch": 15.8, | |
| "grad_norm": 0.0011565013550164168, | |
| "learning_rate": 6.421379363065142e-07, | |
| "logits/chosen": -1.0095231533050537, | |
| "logits/rejected": -0.7704787850379944, | |
| "logps/chosen": -65.56999206542969, | |
| "logps/rejected": -1337.3360595703125, | |
| "loss": 0.0003, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": 0.18380019068717957, | |
| "rewards/margins": 12.87743091583252, | |
| "rewards/rejected": -12.693631172180176, | |
| "step": 790 | |
| }, | |
| { | |
| "epoch": 16.0, | |
| "grad_norm": 0.0012537423983021187, | |
| "learning_rate": 5.848888922025553e-07, | |
| "logits/chosen": -0.8937209248542786, | |
| "logits/rejected": -0.8747299313545227, | |
| "logps/chosen": -54.911643981933594, | |
| "logps/rejected": -1144.4791259765625, | |
| "loss": 0.0003, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": 0.16021445393562317, | |
| "rewards/margins": 10.787330627441406, | |
| "rewards/rejected": -10.627116203308105, | |
| "step": 800 | |
| }, | |
| { | |
| "epoch": 16.2, | |
| "grad_norm": 0.0014373038754308242, | |
| "learning_rate": 5.299731159831953e-07, | |
| "logits/chosen": -0.9762552976608276, | |
| "logits/rejected": -0.8652079701423645, | |
| "logps/chosen": -58.197059631347656, | |
| "logps/rejected": -1232.221435546875, | |
| "loss": 0.0003, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": 0.1831381916999817, | |
| "rewards/margins": 11.740102767944336, | |
| "rewards/rejected": -11.556963920593262, | |
| "step": 810 | |
| }, | |
| { | |
| "epoch": 16.4, | |
| "grad_norm": 0.000862440552553043, | |
| "learning_rate": 4.774575140626317e-07, | |
| "logits/chosen": -0.9985950589179993, | |
| "logits/rejected": -0.8936964273452759, | |
| "logps/chosen": -58.0911979675293, | |
| "logps/rejected": -1282.3681640625, | |
| "loss": 0.0003, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": 0.19094052910804749, | |
| "rewards/margins": 12.265246391296387, | |
| "rewards/rejected": -12.074307441711426, | |
| "step": 820 | |
| }, | |
| { | |
| "epoch": 16.6, | |
| "grad_norm": 0.0011309096722098083, | |
| "learning_rate": 4.27406068612396e-07, | |
| "logits/chosen": -0.9642303586006165, | |
| "logits/rejected": -0.8438752293586731, | |
| "logps/chosen": -59.80333709716797, | |
| "logps/rejected": -1246.0048828125, | |
| "loss": 0.0002, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": 0.16420656442642212, | |
| "rewards/margins": 11.867765426635742, | |
| "rewards/rejected": -11.703559875488281, | |
| "step": 830 | |
| }, | |
| { | |
| "epoch": 16.8, | |
| "grad_norm": 0.001634196377460084, | |
| "learning_rate": 3.798797596089351e-07, | |
| "logits/chosen": -0.9439367055892944, | |
| "logits/rejected": -0.8413814306259155, | |
| "logps/chosen": -59.905982971191406, | |
| "logps/rejected": -1256.216796875, | |
| "loss": 0.0003, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": 0.16103532910346985, | |
| "rewards/margins": 11.955886840820312, | |
| "rewards/rejected": -11.794851303100586, | |
| "step": 840 | |
| }, | |
| { | |
| "epoch": 17.0, | |
| "grad_norm": 0.0011128513110797832, | |
| "learning_rate": 3.3493649053890325e-07, | |
| "logits/chosen": -0.9567734003067017, | |
| "logits/rejected": -0.7875978350639343, | |
| "logps/chosen": -61.99500274658203, | |
| "logps/rejected": -1306.9703369140625, | |
| "loss": 0.0002, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": 0.1775272786617279, | |
| "rewards/margins": 12.505702018737793, | |
| "rewards/rejected": -12.32817554473877, | |
| "step": 850 | |
| }, | |
| { | |
| "epoch": 17.2, | |
| "grad_norm": 0.0013520172966032026, | |
| "learning_rate": 2.9263101785268253e-07, | |
| "logits/chosen": -0.9488283395767212, | |
| "logits/rejected": -0.8742234110832214, | |
| "logps/chosen": -56.80482864379883, | |
| "logps/rejected": -1268.548095703125, | |
| "loss": 0.0003, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": 0.17435792088508606, | |
| "rewards/margins": 12.08137321472168, | |
| "rewards/rejected": -11.907015800476074, | |
| "step": 860 | |
| }, | |
| { | |
| "epoch": 17.4, | |
| "grad_norm": 0.0012511118706732029, | |
| "learning_rate": 2.53014884252083e-07, | |
| "logits/chosen": -1.0541223287582397, | |
| "logits/rejected": -0.81927490234375, | |
| "logps/chosen": -63.943763732910156, | |
| "logps/rejected": -1374.1676025390625, | |
| "loss": 0.0002, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": 0.18310335278511047, | |
| "rewards/margins": 13.212549209594727, | |
| "rewards/rejected": -13.029446601867676, | |
| "step": 870 | |
| }, | |
| { | |
| "epoch": 17.6, | |
| "grad_norm": 0.0018809600968994445, | |
| "learning_rate": 2.1613635589349756e-07, | |
| "logits/chosen": -0.8839080929756165, | |
| "logits/rejected": -0.7969019412994385, | |
| "logps/chosen": -56.40065383911133, | |
| "logps/rejected": -1226.865478515625, | |
| "loss": 0.0003, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": 0.17095676064491272, | |
| "rewards/margins": 11.626653671264648, | |
| "rewards/rejected": -11.455695152282715, | |
| "step": 880 | |
| }, | |
| { | |
| "epoch": 17.8, | |
| "grad_norm": 0.0016373893137363934, | |
| "learning_rate": 1.8204036358303173e-07, | |
| "logits/chosen": -0.9771726727485657, | |
| "logits/rejected": -0.8529565930366516, | |
| "logps/chosen": -57.20879364013672, | |
| "logps/rejected": -1290.327880859375, | |
| "loss": 0.0002, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": 0.18934224545955658, | |
| "rewards/margins": 12.324792861938477, | |
| "rewards/rejected": -12.135449409484863, | |
| "step": 890 | |
| }, | |
| { | |
| "epoch": 18.0, | |
| "grad_norm": 0.000786600665947747, | |
| "learning_rate": 1.507684480352292e-07, | |
| "logits/chosen": -0.9094891548156738, | |
| "logits/rejected": -0.8147133588790894, | |
| "logps/chosen": -59.374755859375, | |
| "logps/rejected": -1234.203125, | |
| "loss": 0.0003, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": 0.1541765034198761, | |
| "rewards/margins": 11.714710235595703, | |
| "rewards/rejected": -11.560534477233887, | |
| "step": 900 | |
| }, | |
| { | |
| "epoch": 18.2, | |
| "grad_norm": 0.002709830744526596, | |
| "learning_rate": 1.223587092621162e-07, | |
| "logits/chosen": -1.00485098361969, | |
| "logits/rejected": -0.7558645606040955, | |
| "logps/chosen": -67.38685607910156, | |
| "logps/rejected": -1367.7635498046875, | |
| "loss": 0.0002, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": 0.17096880078315735, | |
| "rewards/margins": 13.181119918823242, | |
| "rewards/rejected": -13.010151863098145, | |
| "step": 910 | |
| }, | |
| { | |
| "epoch": 18.4, | |
| "grad_norm": 0.0010941683716015594, | |
| "learning_rate": 9.684576015420277e-08, | |
| "logits/chosen": -0.9139689207077026, | |
| "logits/rejected": -0.8125830888748169, | |
| "logps/chosen": -58.29936599731445, | |
| "logps/rejected": -1264.84033203125, | |
| "loss": 0.0003, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": 0.16245688498020172, | |
| "rewards/margins": 12.02514934539795, | |
| "rewards/rejected": -11.862692832946777, | |
| "step": 920 | |
| }, | |
| { | |
| "epoch": 18.6, | |
| "grad_norm": 0.001790071139550289, | |
| "learning_rate": 7.426068431000883e-08, | |
| "logits/chosen": -0.9928874969482422, | |
| "logits/rejected": -0.8945854306221008, | |
| "logps/chosen": -59.90386962890625, | |
| "logps/rejected": -1225.345458984375, | |
| "loss": 0.0002, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": 0.18468160927295685, | |
| "rewards/margins": 11.685681343078613, | |
| "rewards/rejected": -11.500999450683594, | |
| "step": 930 | |
| }, | |
| { | |
| "epoch": 18.8, | |
| "grad_norm": 0.001444924500021094, | |
| "learning_rate": 5.463099816548578e-08, | |
| "logits/chosen": -0.8352192640304565, | |
| "logits/rejected": -0.8455368876457214, | |
| "logps/chosen": -53.367652893066406, | |
| "logps/rejected": -1094.0107421875, | |
| "loss": 0.0003, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": 0.15318068861961365, | |
| "rewards/margins": 10.247284889221191, | |
| "rewards/rejected": -10.094103813171387, | |
| "step": 940 | |
| }, | |
| { | |
| "epoch": 19.0, | |
| "grad_norm": 0.0011127218090168816, | |
| "learning_rate": 3.798061746947995e-08, | |
| "logits/chosen": -0.9996277689933777, | |
| "logits/rejected": -0.7810768485069275, | |
| "logps/chosen": -62.9672737121582, | |
| "logps/rejected": -1361.350341796875, | |
| "loss": 0.0002, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": 0.1840919703245163, | |
| "rewards/margins": 13.088468551635742, | |
| "rewards/rejected": -12.904377937316895, | |
| "step": 950 | |
| }, | |
| { | |
| "epoch": 19.2, | |
| "grad_norm": 0.0011199039962919363, | |
| "learning_rate": 2.4329828146074096e-08, | |
| "logits/chosen": -0.9857956171035767, | |
| "logits/rejected": -0.8834479451179504, | |
| "logps/chosen": -61.652801513671875, | |
| "logps/rejected": -1269.288330078125, | |
| "loss": 0.0003, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": 0.14881816506385803, | |
| "rewards/margins": 12.092996597290039, | |
| "rewards/rejected": -11.944178581237793, | |
| "step": 960 | |
| }, | |
| { | |
| "epoch": 19.4, | |
| "grad_norm": 0.0013686833523596474, | |
| "learning_rate": 1.3695261579316776e-08, | |
| "logits/chosen": -0.9996697306632996, | |
| "logits/rejected": -0.8662905693054199, | |
| "logps/chosen": -60.64554977416992, | |
| "logps/rejected": -1260.70458984375, | |
| "loss": 0.0003, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": 0.18075986206531525, | |
| "rewards/margins": 12.048669815063477, | |
| "rewards/rejected": -11.867908477783203, | |
| "step": 970 | |
| }, | |
| { | |
| "epoch": 19.6, | |
| "grad_norm": 0.0017745592515844095, | |
| "learning_rate": 6.089874350439507e-09, | |
| "logits/chosen": -1.0439860820770264, | |
| "logits/rejected": -0.9185269474983215, | |
| "logps/chosen": -59.79936981201172, | |
| "logps/rejected": -1267.5032958984375, | |
| "loss": 0.0003, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": 0.18838214874267578, | |
| "rewards/margins": 12.108388900756836, | |
| "rewards/rejected": -11.92000675201416, | |
| "step": 980 | |
| }, | |
| { | |
| "epoch": 19.8, | |
| "grad_norm": 0.0013032690672856834, | |
| "learning_rate": 1.5229324522605949e-09, | |
| "logits/chosen": -0.9330822229385376, | |
| "logits/rejected": -0.8125349879264832, | |
| "logps/chosen": -58.721580505371094, | |
| "logps/rejected": -1287.365966796875, | |
| "loss": 0.0002, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": 0.17665794491767883, | |
| "rewards/margins": 12.307037353515625, | |
| "rewards/rejected": -12.130378723144531, | |
| "step": 990 | |
| }, | |
| { | |
| "epoch": 20.0, | |
| "grad_norm": 0.003965487525700407, | |
| "learning_rate": 0.0, | |
| "logits/chosen": -0.9679548144340515, | |
| "logits/rejected": -0.8124309778213501, | |
| "logps/chosen": -61.12044143676758, | |
| "logps/rejected": -1297.4541015625, | |
| "loss": 0.0002, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": 0.1760241687297821, | |
| "rewards/margins": 12.41258716583252, | |
| "rewards/rejected": -12.236563682556152, | |
| "step": 1000 | |
| }, | |
| { | |
| "epoch": 20.0, | |
| "step": 1000, | |
| "total_flos": 0.0, | |
| "train_loss": 0.07008945742505603, | |
| "train_runtime": 8233.3372, | |
| "train_samples_per_second": 1.943, | |
| "train_steps_per_second": 0.121 | |
| } | |
| ], | |
| "logging_steps": 10, | |
| "max_steps": 1000, | |
| "num_input_tokens_seen": 0, | |
| "num_train_epochs": 20, | |
| "save_steps": 1000, | |
| "stateful_callbacks": { | |
| "TrainerControl": { | |
| "args": { | |
| "should_epoch_stop": false, | |
| "should_evaluate": false, | |
| "should_log": false, | |
| "should_save": true, | |
| "should_training_stop": true | |
| }, | |
| "attributes": {} | |
| } | |
| }, | |
| "total_flos": 0.0, | |
| "train_batch_size": 2, | |
| "trial_name": null, | |
| "trial_params": null | |
| } | |