{
  "best_metric": null,
  "best_model_checkpoint": null,
  "epoch": 20.0,
  "eval_steps": 1000,
  "global_step": 1000,
  "is_hyper_param_search": false,
  "is_local_process_zero": true,
  "is_world_process_zero": true,
  "log_history": [
    {
      "epoch": 0.02,
      "grad_norm": 0.35460313427884693,
      "learning_rate": 5.0000000000000004e-08,
      "logits/chosen": -1.8231227397918701,
      "logits/rejected": -1.5560479164123535,
      "logps/chosen": -85.02825927734375,
      "logps/rejected": -71.86117553710938,
      "loss": 0.6931,
      "rewards/accuracies": 0.0,
      "rewards/chosen": 0.0,
      "rewards/margins": 0.0,
      "rewards/rejected": 0.0,
      "step": 1
    },
    {
      "epoch": 0.2,
      "grad_norm": 0.30411563112440204,
      "learning_rate": 5.000000000000001e-07,
      "logits/chosen": -1.7207438945770264,
      "logits/rejected": -1.698093295097351,
      "logps/chosen": -75.15734100341797,
      "logps/rejected": -80.09143829345703,
      "loss": 0.6931,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 9.971462714020163e-05,
      "rewards/margins": 0.0002858855586964637,
      "rewards/rejected": -0.00018617100431583822,
      "step": 10
    },
    {
      "epoch": 0.4,
      "grad_norm": 0.29583445812044185,
      "learning_rate": 1.0000000000000002e-06,
      "logits/chosen": -1.7815284729003906,
      "logits/rejected": -1.7640860080718994,
      "logps/chosen": -74.8746566772461,
      "logps/rejected": -77.6731185913086,
      "loss": 0.6927,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": 0.00024838969693519175,
      "rewards/margins": 0.0008968595648184419,
      "rewards/rejected": -0.0006484698969870806,
      "step": 20
    },
    {
      "epoch": 0.6,
      "grad_norm": 0.2966957448432536,
      "learning_rate": 1.5e-06,
      "logits/chosen": -1.739800214767456,
      "logits/rejected": -1.7370967864990234,
      "logps/chosen": -75.44456481933594,
      "logps/rejected": -77.48068237304688,
      "loss": 0.6917,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.0010354876285418868,
      "rewards/margins": 0.0030451309867203236,
      "rewards/rejected": -0.002009643241763115,
      "step": 30
    },
    {
      "epoch": 0.8,
      "grad_norm": 0.33102320941887775,
      "learning_rate": 2.0000000000000003e-06,
      "logits/chosen": -1.7791082859039307,
      "logits/rejected": -1.733616590499878,
      "logps/chosen": -76.32530212402344,
      "logps/rejected": -75.00889587402344,
      "loss": 0.689,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.007160990033298731,
      "rewards/margins": 0.008440766483545303,
      "rewards/rejected": -0.0012797765666618943,
      "step": 40
    },
    {
      "epoch": 1.0,
      "grad_norm": 0.5247728899006847,
      "learning_rate": 2.5e-06,
      "logits/chosen": -1.7112547159194946,
      "logits/rejected": -1.7392151355743408,
      "logps/chosen": -73.26392364501953,
      "logps/rejected": -79.31649780273438,
      "loss": 0.6832,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.013615945354104042,
      "rewards/margins": 0.019877230748534203,
      "rewards/rejected": -0.006261286325752735,
      "step": 50
    },
    {
      "epoch": 1.2,
      "grad_norm": 0.4135013743852781,
      "learning_rate": 3e-06,
      "logits/chosen": -1.732405424118042,
      "logits/rejected": -1.7449699640274048,
      "logps/chosen": -71.77066040039062,
      "logps/rejected": -80.100341796875,
      "loss": 0.6732,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.024410251528024673,
      "rewards/margins": 0.03932947292923927,
      "rewards/rejected": -0.01491921953856945,
      "step": 60
    },
    {
      "epoch": 1.4,
      "grad_norm": 0.392415113956775,
      "learning_rate": 3.5e-06,
      "logits/chosen": -1.716195821762085,
      "logits/rejected": -1.731308937072754,
      "logps/chosen": -72.06253051757812,
      "logps/rejected": -80.39039611816406,
      "loss": 0.6542,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.04634103178977966,
      "rewards/margins": 0.07862894982099533,
      "rewards/rejected": -0.032287921756505966,
      "step": 70
    },
    {
      "epoch": 1.6,
      "grad_norm": 0.5321176658546207,
      "learning_rate": 4.000000000000001e-06,
      "logits/chosen": -1.69076406955719,
      "logits/rejected": -1.6509063243865967,
      "logps/chosen": -68.82267761230469,
      "logps/rejected": -88.67816162109375,
      "loss": 0.605,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.08192411065101624,
      "rewards/margins": 0.19158878922462463,
      "rewards/rejected": -0.1096646636724472,
      "step": 80
    },
    {
      "epoch": 1.8,
      "grad_norm": 0.5696532480733786,
      "learning_rate": 4.5e-06,
      "logits/chosen": -1.7061433792114258,
      "logits/rejected": -1.684195876121521,
      "logps/chosen": -59.4539680480957,
      "logps/rejected": -106.4749526977539,
      "loss": 0.4742,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.18270966410636902,
      "rewards/margins": 0.5085535645484924,
      "rewards/rejected": -0.325843870639801,
      "step": 90
    },
    {
      "epoch": 2.0,
      "grad_norm": 0.3568428380556091,
      "learning_rate": 5e-06,
      "logits/chosen": -1.5610880851745605,
      "logits/rejected": -1.4533908367156982,
      "logps/chosen": -48.31792449951172,
      "logps/rejected": -164.77566528320312,
      "loss": 0.3175,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.2694377899169922,
      "rewards/margins": 1.1592490673065186,
      "rewards/rejected": -0.8898113965988159,
      "step": 100
    },
    {
      "epoch": 2.2,
      "grad_norm": 0.28794719016802656,
      "learning_rate": 4.99847706754774e-06,
      "logits/chosen": -1.5363928079605103,
      "logits/rejected": -1.4212225675582886,
      "logps/chosen": -42.558448791503906,
      "logps/rejected": -187.63729858398438,
      "loss": 0.2626,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.39137277007102966,
      "rewards/margins": 1.5502681732177734,
      "rewards/rejected": -1.1588953733444214,
      "step": 110
    },
    {
      "epoch": 2.4,
      "grad_norm": 0.15078063761313237,
      "learning_rate": 4.993910125649561e-06,
      "logits/chosen": -1.51887845993042,
      "logits/rejected": -1.437011957168579,
      "logps/chosen": -38.434608459472656,
      "logps/rejected": -232.57766723632812,
      "loss": 0.223,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.42294931411743164,
      "rewards/margins": 2.0381760597229004,
      "rewards/rejected": -1.6152267456054688,
      "step": 120
    },
    {
      "epoch": 2.6,
      "grad_norm": 0.4347861863265109,
      "learning_rate": 4.986304738420684e-06,
      "logits/chosen": -1.3788427114486694,
      "logits/rejected": -1.37319016456604,
      "logps/chosen": -33.37788391113281,
      "logps/rejected": -288.50018310546875,
      "loss": 0.1776,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.44394317269325256,
      "rewards/margins": 2.571049928665161,
      "rewards/rejected": -2.1271069049835205,
      "step": 130
    },
    {
      "epoch": 2.8,
      "grad_norm": 0.7461810630936248,
      "learning_rate": 4.975670171853926e-06,
      "logits/chosen": -1.190661072731018,
      "logits/rejected": -1.1670948266983032,
      "logps/chosen": -32.81340026855469,
      "logps/rejected": -336.36962890625,
      "loss": 0.1102,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.4285716414451599,
      "rewards/margins": 3.032412528991699,
      "rewards/rejected": -2.6038408279418945,
      "step": 140
    },
    {
      "epoch": 3.0,
      "grad_norm": 0.053630215551159044,
      "learning_rate": 4.962019382530521e-06,
      "logits/chosen": -1.1552760601043701,
      "logits/rejected": -1.1048529148101807,
      "logps/chosen": -60.1913948059082,
      "logps/rejected": -567.5430908203125,
      "loss": 0.0207,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.1675727218389511,
      "rewards/margins": 5.098362922668457,
      "rewards/rejected": -4.930790901184082,
      "step": 150
    },
    {
      "epoch": 3.2,
      "grad_norm": 0.031903287512855885,
      "learning_rate": 4.9453690018345144e-06,
      "logits/chosen": -1.0522866249084473,
      "logits/rejected": -1.0280696153640747,
      "logps/chosen": -102.0748062133789,
      "logps/rejected": -891.6737060546875,
      "loss": 0.0036,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.2856892943382263,
      "rewards/margins": 7.848362922668457,
      "rewards/rejected": -8.134051322937012,
      "step": 160
    },
    {
      "epoch": 3.4,
      "grad_norm": 0.0197914954559647,
      "learning_rate": 4.925739315689991e-06,
      "logits/chosen": -1.1597774028778076,
      "logits/rejected": -1.1589906215667725,
      "logps/chosen": -61.60524368286133,
      "logps/rejected": -1039.020751953125,
      "loss": 0.0022,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.12755107879638672,
      "rewards/margins": 9.746142387390137,
      "rewards/rejected": -9.61859130859375,
      "step": 170
    },
    {
      "epoch": 3.6,
      "grad_norm": 0.023011571746249113,
      "learning_rate": 4.903154239845798e-06,
      "logits/chosen": -1.2340686321258545,
      "logits/rejected": -1.0457987785339355,
      "logps/chosen": -59.74028778076172,
      "logps/rejected": -1156.054931640625,
      "loss": 0.0014,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.25307127833366394,
      "rewards/margins": 11.132636070251465,
      "rewards/rejected": -10.879566192626953,
      "step": 180
    },
    {
      "epoch": 3.8,
      "grad_norm": 0.010025707600426755,
      "learning_rate": 4.8776412907378845e-06,
      "logits/chosen": -1.0119726657867432,
      "logits/rejected": -1.0299015045166016,
      "logps/chosen": -51.19286346435547,
      "logps/rejected": -1030.427490234375,
      "loss": 0.0015,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.22199800610542297,
      "rewards/margins": 9.714365005493164,
      "rewards/rejected": -9.492366790771484,
      "step": 190
    },
    {
      "epoch": 4.0,
      "grad_norm": 0.009363825045997377,
      "learning_rate": 4.849231551964771e-06,
      "logits/chosen": -1.1452789306640625,
      "logits/rejected": -1.0538662672042847,
      "logps/chosen": -55.855430603027344,
      "logps/rejected": -1116.4219970703125,
      "loss": 0.0012,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.2577177882194519,
      "rewards/margins": 10.68994140625,
      "rewards/rejected": -10.43222427368164,
      "step": 200
    },
    {
      "epoch": 4.2,
      "grad_norm": 0.007917186035092368,
      "learning_rate": 4.817959636416969e-06,
      "logits/chosen": -0.979863166809082,
      "logits/rejected": -0.954973042011261,
      "logps/chosen": -51.2191276550293,
      "logps/rejected": -1067.7679443359375,
      "loss": 0.0012,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.21226799488067627,
      "rewards/margins": 10.104147911071777,
      "rewards/rejected": -9.89188003540039,
      "step": 210
    },
    {
      "epoch": 4.4,
      "grad_norm": 0.004467951143379201,
      "learning_rate": 4.783863644106502e-06,
      "logits/chosen": -1.0173983573913574,
      "logits/rejected": -0.9798928499221802,
      "logps/chosen": -52.295379638671875,
      "logps/rejected": -1050.406494140625,
      "loss": 0.001,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.21525339782238007,
      "rewards/margins": 9.93811321258545,
      "rewards/rejected": -9.722859382629395,
      "step": 220
    },
    {
      "epoch": 4.6,
      "grad_norm": 0.0036885023764270065,
      "learning_rate": 4.746985115747918e-06,
      "logits/chosen": -1.1039237976074219,
      "logits/rejected": -0.9902089238166809,
      "logps/chosen": -56.54585647583008,
      "logps/rejected": -1109.8258056640625,
      "loss": 0.0008,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.2429007589817047,
      "rewards/margins": 10.61030101776123,
      "rewards/rejected": -10.367398262023926,
      "step": 230
    },
    {
      "epoch": 4.8,
      "grad_norm": 0.0065247264850717725,
      "learning_rate": 4.707368982147318e-06,
      "logits/chosen": -1.1152589321136475,
      "logits/rejected": -1.0077005624771118,
      "logps/chosen": -54.775856018066406,
      "logps/rejected": -1131.518310546875,
      "loss": 0.0008,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.22144417464733124,
      "rewards/margins": 10.786369323730469,
      "rewards/rejected": -10.564926147460938,
      "step": 240
    },
    {
      "epoch": 5.0,
      "grad_norm": 0.0032674373668132675,
      "learning_rate": 4.665063509461098e-06,
      "logits/chosen": -1.0255931615829468,
      "logits/rejected": -0.9475139379501343,
      "logps/chosen": -56.231163024902344,
      "logps/rejected": -1138.18115234375,
      "loss": 0.0008,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.22440902888774872,
      "rewards/margins": 10.85453987121582,
      "rewards/rejected": -10.630131721496582,
      "step": 250
    },
    {
      "epoch": 5.2,
      "grad_norm": 0.004115527916331285,
      "learning_rate": 4.620120240391065e-06,
      "logits/chosen": -1.0400116443634033,
      "logits/rejected": -0.9953556060791016,
      "logps/chosen": -54.63758087158203,
      "logps/rejected": -1113.4554443359375,
      "loss": 0.0007,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.21181125938892365,
      "rewards/margins": 10.56176471710205,
      "rewards/rejected": -10.34995174407959,
      "step": 260
    },
    {
      "epoch": 5.4,
      "grad_norm": 0.0030662391394572284,
      "learning_rate": 4.572593931387604e-06,
      "logits/chosen": -1.100754976272583,
      "logits/rejected": -0.9190059900283813,
      "logps/chosen": -55.8875846862793,
      "logps/rejected": -1182.8133544921875,
      "loss": 0.0007,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.2545715868473053,
      "rewards/margins": 11.359747886657715,
      "rewards/rejected": -11.105175018310547,
      "step": 270
    },
    {
      "epoch": 5.6,
      "grad_norm": 0.004246284055742857,
      "learning_rate": 4.522542485937369e-06,
      "logits/chosen": -1.0873005390167236,
      "logits/rejected": -0.91912841796875,
      "logps/chosen": -57.15370559692383,
      "logps/rejected": -1198.202392578125,
      "loss": 0.0007,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.24588795006275177,
      "rewards/margins": 11.502182960510254,
      "rewards/rejected": -11.256293296813965,
      "step": 280
    },
    {
      "epoch": 5.8,
      "grad_norm": 0.004139329336955742,
      "learning_rate": 4.470026884016805e-06,
      "logits/chosen": -0.9481426477432251,
      "logits/rejected": -0.9623401761054993,
      "logps/chosen": -52.13285446166992,
      "logps/rejected": -1074.572509765625,
      "loss": 0.0007,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.20849105715751648,
      "rewards/margins": 10.169190406799316,
      "rewards/rejected": -9.960699081420898,
      "step": 290
    },
    {
      "epoch": 6.0,
      "grad_norm": 0.003034900790871541,
      "learning_rate": 4.415111107797445e-06,
      "logits/chosen": -1.0314205884933472,
      "logits/rejected": -1.0098388195037842,
      "logps/chosen": -54.42866134643555,
      "logps/rejected": -1097.184326171875,
      "loss": 0.0006,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.20703566074371338,
      "rewards/margins": 10.398859024047852,
      "rewards/rejected": -10.19182300567627,
      "step": 300
    },
    {
      "epoch": 6.2,
      "grad_norm": 0.0038510374493456907,
      "learning_rate": 4.357862063693486e-06,
      "logits/chosen": -1.026374101638794,
      "logits/rejected": -0.9435133934020996,
      "logps/chosen": -53.9544792175293,
      "logps/rejected": -1142.4066162109375,
      "loss": 0.0006,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.21439966559410095,
      "rewards/margins": 10.869270324707031,
      "rewards/rejected": -10.654870986938477,
      "step": 310
    },
    {
      "epoch": 6.4,
      "grad_norm": 0.0035014469049547427,
      "learning_rate": 4.2983495008466285e-06,
      "logits/chosen": -1.105359673500061,
      "logits/rejected": -0.9964410066604614,
      "logps/chosen": -57.14154815673828,
      "logps/rejected": -1162.7977294921875,
      "loss": 0.0006,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.21488821506500244,
      "rewards/margins": 11.113906860351562,
      "rewards/rejected": -10.899017333984375,
      "step": 320
    },
    {
      "epoch": 6.6,
      "grad_norm": 0.002634385062559098,
      "learning_rate": 4.236645926147493e-06,
      "logits/chosen": -1.0640833377838135,
      "logits/rejected": -0.9242901802062988,
      "logps/chosen": -56.49944305419922,
      "logps/rejected": -1212.7708740234375,
      "loss": 0.0005,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.23322686553001404,
      "rewards/margins": 11.629671096801758,
      "rewards/rejected": -11.396444320678711,
      "step": 330
    },
    {
      "epoch": 6.8,
      "grad_norm": 0.0030996294667543192,
      "learning_rate": 4.172826515897146e-06,
      "logits/chosen": -1.082220435142517,
      "logits/rejected": -0.9698995351791382,
      "logps/chosen": -59.28369903564453,
      "logps/rejected": -1214.587646484375,
      "loss": 0.0005,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.19337359070777893,
      "rewards/margins": 11.616683959960938,
      "rewards/rejected": -11.423309326171875,
      "step": 340
    },
    {
      "epoch": 7.0,
      "grad_norm": 0.0015310049705482352,
      "learning_rate": 4.106969024216348e-06,
      "logits/chosen": -0.9850069284439087,
      "logits/rejected": -0.9987241625785828,
      "logps/chosen": -54.036949157714844,
      "logps/rejected": -1109.0908203125,
      "loss": 0.0006,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.19889585673809052,
      "rewards/margins": 10.489348411560059,
      "rewards/rejected": -10.29045295715332,
      "step": 350
    },
    {
      "epoch": 7.2,
      "grad_norm": 0.001899366605942392,
      "learning_rate": 4.039153688314146e-06,
      "logits/chosen": -1.0366054773330688,
      "logits/rejected": -0.9512063264846802,
      "logps/chosen": -56.377410888671875,
      "logps/rejected": -1194.2122802734375,
      "loss": 0.0005,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.2178737223148346,
      "rewards/margins": 11.424546241760254,
      "rewards/rejected": -11.206673622131348,
      "step": 360
    },
    {
      "epoch": 7.4,
      "grad_norm": 0.003085448477067617,
      "learning_rate": 3.969463130731183e-06,
      "logits/chosen": -1.058009147644043,
      "logits/rejected": -0.9505330324172974,
      "logps/chosen": -55.610870361328125,
      "logps/rejected": -1222.853515625,
      "loss": 0.0005,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.2119036167860031,
      "rewards/margins": 11.69975471496582,
      "rewards/rejected": -11.4878511428833,
      "step": 370
    },
    {
      "epoch": 7.6,
      "grad_norm": 0.002631100351440524,
      "learning_rate": 3.897982258676867e-06,
      "logits/chosen": -0.9967459440231323,
      "logits/rejected": -1.021655797958374,
      "logps/chosen": -52.216705322265625,
      "logps/rejected": -1100.6519775390625,
      "loss": 0.0005,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.19106148183345795,
      "rewards/margins": 10.369633674621582,
      "rewards/rejected": -10.178572654724121,
      "step": 380
    },
    {
      "epoch": 7.8,
      "grad_norm": 0.0031329070413845662,
      "learning_rate": 3.824798160583012e-06,
      "logits/chosen": -1.0442421436309814,
      "logits/rejected": -0.9009174108505249,
      "logps/chosen": -56.89820098876953,
      "logps/rejected": -1184.45263671875,
      "loss": 0.0005,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.22130858898162842,
      "rewards/margins": 11.319818496704102,
      "rewards/rejected": -11.098509788513184,
      "step": 390
    },
    {
      "epoch": 8.0,
      "grad_norm": 0.003423988945465294,
      "learning_rate": 3.7500000000000005e-06,
      "logits/chosen": -1.035902500152588,
      "logits/rejected": -0.9636209607124329,
      "logps/chosen": -56.99961471557617,
      "logps/rejected": -1206.747802734375,
      "loss": 0.0005,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.18911413848400116,
      "rewards/margins": 11.49039363861084,
      "rewards/rejected": -11.301280975341797,
      "step": 400
    },
    {
      "epoch": 8.2,
      "grad_norm": 0.003864669112701001,
      "learning_rate": 3.6736789069647273e-06,
      "logits/chosen": -0.9635273218154907,
      "logits/rejected": -0.8587255477905273,
      "logps/chosen": -58.7963981628418,
      "logps/rejected": -1161.4803466796875,
      "loss": 0.0005,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.19340120255947113,
      "rewards/margins": 11.068787574768066,
      "rewards/rejected": -10.875386238098145,
      "step": 410
    },
    {
      "epoch": 8.4,
      "grad_norm": 0.003245040701864265,
      "learning_rate": 3.595927866972694e-06,
      "logits/chosen": -1.0136067867279053,
      "logits/rejected": -0.9350364804267883,
      "logps/chosen": -55.97736358642578,
      "logps/rejected": -1197.21484375,
      "loss": 0.0004,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.20746412873268127,
      "rewards/margins": 11.428886413574219,
      "rewards/rejected": -11.221421241760254,
      "step": 420
    },
    {
      "epoch": 8.6,
      "grad_norm": 0.0023560411348610777,
      "learning_rate": 3.516841607689501e-06,
      "logits/chosen": -0.9947836995124817,
      "logits/rejected": -0.9126423597335815,
      "logps/chosen": -57.561622619628906,
      "logps/rejected": -1165.994384765625,
      "loss": 0.0004,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.18999549746513367,
      "rewards/margins": 11.074243545532227,
      "rewards/rejected": -10.884247779846191,
      "step": 430
    },
    {
      "epoch": 8.8,
      "grad_norm": 0.0025971993823853895,
      "learning_rate": 3.436516483539781e-06,
      "logits/chosen": -1.0256311893463135,
      "logits/rejected": -0.8973512649536133,
      "logps/chosen": -59.285438537597656,
      "logps/rejected": -1248.3521728515625,
      "loss": 0.0004,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.18996688723564148,
      "rewards/margins": 11.936816215515137,
      "rewards/rejected": -11.746848106384277,
      "step": 440
    },
    {
      "epoch": 9.0,
      "grad_norm": 0.0032315637570330567,
      "learning_rate": 3.3550503583141726e-06,
      "logits/chosen": -1.0244309902191162,
      "logits/rejected": -0.8780086636543274,
      "logps/chosen": -57.84431076049805,
      "logps/rejected": -1234.871337890625,
      "loss": 0.0004,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.1998531073331833,
      "rewards/margins": 11.807432174682617,
      "rewards/rejected": -11.607580184936523,
      "step": 450
    },
    {
      "epoch": 9.2,
      "grad_norm": 0.0036226784518706475,
      "learning_rate": 3.272542485937369e-06,
      "logits/chosen": -0.9375013113021851,
      "logits/rejected": -0.8737777471542358,
      "logps/chosen": -54.31487274169922,
      "logps/rejected": -1172.191162109375,
      "loss": 0.0004,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.19032403826713562,
      "rewards/margins": 11.127653121948242,
      "rewards/rejected": -10.937328338623047,
      "step": 460
    },
    {
      "epoch": 9.4,
      "grad_norm": 0.002997930586361418,
      "learning_rate": 3.189093389542498e-06,
      "logits/chosen": -1.0432255268096924,
      "logits/rejected": -0.9277008175849915,
      "logps/chosen": -58.5209846496582,
      "logps/rejected": -1200.554931640625,
      "loss": 0.0004,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.19790109992027283,
      "rewards/margins": 11.468005180358887,
      "rewards/rejected": -11.270103454589844,
      "step": 470
    },
    {
      "epoch": 9.6,
      "grad_norm": 0.0022499622323506746,
      "learning_rate": 3.1048047389991693e-06,
      "logits/chosen": -1.0687272548675537,
      "logits/rejected": -0.9239826202392578,
      "logps/chosen": -60.52696990966797,
      "logps/rejected": -1241.221435546875,
      "loss": 0.0004,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.18703293800354004,
      "rewards/margins": 11.873010635375977,
      "rewards/rejected": -11.685976028442383,
      "step": 480
    },
    {
      "epoch": 9.8,
      "grad_norm": 0.0019412808634179527,
      "learning_rate": 3.019779227044398e-06,
      "logits/chosen": -0.9789573550224304,
      "logits/rejected": -0.7663965225219727,
      "logps/chosen": -61.904319763183594,
      "logps/rejected": -1237.324951171875,
      "loss": 0.0003,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.1967342346906662,
      "rewards/margins": 11.862494468688965,
      "rewards/rejected": -11.66576099395752,
      "step": 490
    },
    {
      "epoch": 10.0,
      "grad_norm": 0.002323126626066593,
      "learning_rate": 2.9341204441673267e-06,
      "logits/chosen": -1.0079948902130127,
      "logits/rejected": -0.9249541163444519,
      "logps/chosen": -58.3544807434082,
      "logps/rejected": -1210.3780517578125,
      "loss": 0.0004,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.16591690480709076,
      "rewards/margins": 11.488636016845703,
      "rewards/rejected": -11.32271957397461,
      "step": 500
    },
    {
      "epoch": 10.2,
      "grad_norm": 0.0018387058134298486,
      "learning_rate": 2.847932752400164e-06,
      "logits/chosen": -0.9845121502876282,
      "logits/rejected": -0.8383527994155884,
      "logps/chosen": -59.52973556518555,
      "logps/rejected": -1259.748291015625,
      "loss": 0.0004,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.17635658383369446,
      "rewards/margins": 12.032411575317383,
      "rewards/rejected": -11.856054306030273,
      "step": 510
    },
    {
      "epoch": 10.4,
      "grad_norm": 0.0013643138700466504,
      "learning_rate": 2.761321158169134e-06,
      "logits/chosen": -1.0216516256332397,
      "logits/rejected": -0.9077258110046387,
      "logps/chosen": -59.731300354003906,
      "logps/rejected": -1231.614501953125,
      "loss": 0.0004,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.1921447217464447,
      "rewards/margins": 11.769120216369629,
      "rewards/rejected": -11.576974868774414,
      "step": 520
    },
    {
      "epoch": 10.6,
      "grad_norm": 0.002977922307910112,
      "learning_rate": 2.6743911843603134e-06,
      "logits/chosen": -1.0051695108413696,
      "logits/rejected": -0.8601359128952026,
      "logps/chosen": -62.12017822265625,
      "logps/rejected": -1268.072509765625,
      "loss": 0.0003,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.1786160171031952,
      "rewards/margins": 12.141373634338379,
      "rewards/rejected": -11.962759017944336,
      "step": 530
    },
    {
      "epoch": 10.8,
      "grad_norm": 0.0019745371274113147,
      "learning_rate": 2.587248741756253e-06,
      "logits/chosen": -1.0047599077224731,
      "logits/rejected": -0.8642932176589966,
      "logps/chosen": -61.087615966796875,
      "logps/rejected": -1251.1107177734375,
      "loss": 0.0003,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.18852102756500244,
      "rewards/margins": 11.956985473632812,
      "rewards/rejected": -11.768463134765625,
      "step": 540
    },
    {
      "epoch": 11.0,
      "grad_norm": 0.0018986393686723218,
      "learning_rate": 2.5e-06,
      "logits/chosen": -1.0309114456176758,
      "logits/rejected": -0.9270389676094055,
      "logps/chosen": -59.46849822998047,
      "logps/rejected": -1210.13720703125,
      "loss": 0.0003,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.18577028810977936,
      "rewards/margins": 11.525289535522461,
      "rewards/rejected": -11.339518547058105,
      "step": 550
    },
    {
      "epoch": 11.2,
      "grad_norm": 0.0016611092816266825,
      "learning_rate": 2.4127512582437486e-06,
      "logits/chosen": -0.9609254598617554,
      "logits/rejected": -0.9043042063713074,
      "logps/chosen": -54.84800338745117,
      "logps/rejected": -1216.4305419921875,
      "loss": 0.0003,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.20413394272327423,
      "rewards/margins": 11.576135635375977,
      "rewards/rejected": -11.372001647949219,
      "step": 560
    },
    {
      "epoch": 11.4,
      "grad_norm": 0.0019080916476901313,
      "learning_rate": 2.325608815639687e-06,
      "logits/chosen": -0.9941130876541138,
      "logits/rejected": -0.8515572547912598,
      "logps/chosen": -59.786705017089844,
      "logps/rejected": -1280.828857421875,
      "loss": 0.0003,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.19204822182655334,
      "rewards/margins": 12.262724876403809,
      "rewards/rejected": -12.070676803588867,
      "step": 570
    },
    {
      "epoch": 11.6,
      "grad_norm": 0.001417529626290414,
      "learning_rate": 2.238678841830867e-06,
      "logits/chosen": -1.0167783498764038,
      "logits/rejected": -0.8408117294311523,
      "logps/chosen": -61.850563049316406,
      "logps/rejected": -1273.6541748046875,
      "loss": 0.0003,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.17794856429100037,
      "rewards/margins": 12.197797775268555,
      "rewards/rejected": -12.019848823547363,
      "step": 580
    },
    {
      "epoch": 11.8,
      "grad_norm": 0.001985700625240649,
      "learning_rate": 2.1520672475998374e-06,
      "logits/chosen": -0.9488497972488403,
      "logits/rejected": -0.8910242319107056,
      "logps/chosen": -56.951904296875,
      "logps/rejected": -1185.1470947265625,
      "loss": 0.0003,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.17158612608909607,
      "rewards/margins": 11.239965438842773,
      "rewards/rejected": -11.068379402160645,
      "step": 590
    },
    {
      "epoch": 12.0,
      "grad_norm": 0.0014036347657331724,
      "learning_rate": 2.0658795558326745e-06,
      "logits/chosen": -0.9536435008049011,
      "logits/rejected": -0.8297941088676453,
      "logps/chosen": -57.86296463012695,
      "logps/rejected": -1237.7889404296875,
      "loss": 0.0003,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.1768932044506073,
      "rewards/margins": 11.793856620788574,
      "rewards/rejected": -11.616963386535645,
      "step": 600
    },
    {
      "epoch": 12.2,
      "grad_norm": 0.0014637474046611807,
      "learning_rate": 1.9802207729556023e-06,
      "logits/chosen": -0.9591706395149231,
      "logits/rejected": -0.8812329173088074,
      "logps/chosen": -56.78215789794922,
      "logps/rejected": -1231.258056640625,
      "loss": 0.0003,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.1855459213256836,
      "rewards/margins": 11.722236633300781,
      "rewards/rejected": -11.536690711975098,
      "step": 610
    },
    {
      "epoch": 12.4,
      "grad_norm": 0.0017805033618398401,
      "learning_rate": 1.895195261000831e-06,
      "logits/chosen": -1.019749641418457,
      "logits/rejected": -0.9892827868461609,
      "logps/chosen": -58.21343994140625,
      "logps/rejected": -1209.8052978515625,
      "loss": 0.0003,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.16973453760147095,
      "rewards/margins": 11.481805801391602,
      "rewards/rejected": -11.312070846557617,
      "step": 620
    },
    {
      "epoch": 12.6,
      "grad_norm": 0.0011184561683295657,
      "learning_rate": 1.8109066104575023e-06,
      "logits/chosen": -0.9216413497924805,
      "logits/rejected": -0.8319498300552368,
      "logps/chosen": -58.66545486450195,
      "logps/rejected": -1199.0224609375,
      "loss": 0.0003,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.17631840705871582,
      "rewards/margins": 11.408339500427246,
      "rewards/rejected": -11.232022285461426,
      "step": 630
    },
    {
      "epoch": 12.8,
      "grad_norm": 0.0014408987665450045,
      "learning_rate": 1.7274575140626318e-06,
      "logits/chosen": -0.9979192614555359,
      "logits/rejected": -0.7878575325012207,
      "logps/chosen": -62.93827438354492,
      "logps/rejected": -1300.874755859375,
      "loss": 0.0003,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.18857316672801971,
      "rewards/margins": 12.50341510772705,
      "rewards/rejected": -12.314840316772461,
      "step": 640
    },
    {
      "epoch": 13.0,
      "grad_norm": 0.0020645792333772377,
      "learning_rate": 1.6449496416858285e-06,
      "logits/chosen": -0.9246803522109985,
      "logits/rejected": -0.8380001783370972,
      "logps/chosen": -57.9194221496582,
      "logps/rejected": -1195.3837890625,
      "loss": 0.0003,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.16603608429431915,
      "rewards/margins": 11.333303451538086,
      "rewards/rejected": -11.167266845703125,
      "step": 650
    },
    {
      "epoch": 13.2,
      "grad_norm": 0.0015139495857028652,
      "learning_rate": 1.56348351646022e-06,
      "logits/chosen": -1.0453752279281616,
      "logits/rejected": -0.8043217658996582,
      "logps/chosen": -63.8051872253418,
      "logps/rejected": -1392.04443359375,
      "loss": 0.0002,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.19428420066833496,
      "rewards/margins": 13.428848266601562,
      "rewards/rejected": -13.234565734863281,
      "step": 660
    },
    {
      "epoch": 13.4,
      "grad_norm": 0.0013498029319408118,
      "learning_rate": 1.4831583923105e-06,
      "logits/chosen": -0.979832649230957,
      "logits/rejected": -0.8365589380264282,
      "logps/chosen": -60.9407844543457,
      "logps/rejected": -1260.807861328125,
      "loss": 0.0003,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.16681666672229767,
      "rewards/margins": 12.033578872680664,
      "rewards/rejected": -11.866762161254883,
      "step": 670
    },
    {
      "epoch": 13.6,
      "grad_norm": 0.0018616946971343861,
      "learning_rate": 1.4040721330273063e-06,
      "logits/chosen": -0.9041559100151062,
      "logits/rejected": -0.8964747190475464,
      "logps/chosen": -54.83356857299805,
      "logps/rejected": -1143.5806884765625,
      "loss": 0.0003,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.16955164074897766,
      "rewards/margins": 10.786935806274414,
      "rewards/rejected": -10.617385864257812,
      "step": 680
    },
    {
      "epoch": 13.8,
      "grad_norm": 0.0012150873329068216,
      "learning_rate": 1.3263210930352737e-06,
      "logits/chosen": -0.8522924184799194,
      "logits/rejected": -0.8536826968193054,
      "logps/chosen": -57.020355224609375,
      "logps/rejected": -1151.3582763671875,
      "loss": 0.0003,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.16875942051410675,
      "rewards/margins": 10.881406784057617,
      "rewards/rejected": -10.712648391723633,
      "step": 690
    },
    {
      "epoch": 14.0,
      "grad_norm": 0.0015006535309411454,
      "learning_rate": 1.2500000000000007e-06,
      "logits/chosen": -0.9550180435180664,
      "logits/rejected": -0.8258699178695679,
      "logps/chosen": -58.025535583496094,
      "logps/rejected": -1249.10546875,
      "loss": 0.0003,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.17841221392154694,
      "rewards/margins": 11.912469863891602,
      "rewards/rejected": -11.73405647277832,
      "step": 700
    },
    {
      "epoch": 14.2,
      "grad_norm": 0.0015715381830458688,
      "learning_rate": 1.1752018394169882e-06,
      "logits/chosen": -0.9631673693656921,
      "logits/rejected": -0.8420494794845581,
      "logps/chosen": -60.133888244628906,
      "logps/rejected": -1249.9853515625,
      "loss": 0.0002,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.1643882542848587,
      "rewards/margins": 11.908964157104492,
      "rewards/rejected": -11.744573593139648,
      "step": 710
    },
    {
      "epoch": 14.4,
      "grad_norm": 0.0013660785683358696,
      "learning_rate": 1.1020177413231334e-06,
      "logits/chosen": -0.8709633946418762,
      "logits/rejected": -0.8834745287895203,
      "logps/chosen": -54.35590362548828,
      "logps/rejected": -1181.2337646484375,
      "loss": 0.0003,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.17913100123405457,
      "rewards/margins": 11.173266410827637,
      "rewards/rejected": -10.994134902954102,
      "step": 720
    },
    {
      "epoch": 14.6,
      "grad_norm": 0.0016180177521581223,
      "learning_rate": 1.0305368692688175e-06,
      "logits/chosen": -0.9120365381240845,
      "logits/rejected": -0.7947243452072144,
      "logps/chosen": -59.50437545776367,
      "logps/rejected": -1239.4515380859375,
      "loss": 0.0003,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.15415535867214203,
      "rewards/margins": 11.778593063354492,
      "rewards/rejected": -11.624438285827637,
      "step": 730
    },
    {
      "epoch": 14.8,
      "grad_norm": 0.0009131551082822374,
      "learning_rate": 9.608463116858544e-07,
      "logits/chosen": -0.9982080459594727,
      "logits/rejected": -0.8127807378768921,
      "logps/chosen": -62.322227478027344,
      "logps/rejected": -1302.121337890625,
      "loss": 0.0003,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.17191007733345032,
      "rewards/margins": 12.468157768249512,
      "rewards/rejected": -12.296248435974121,
      "step": 740
    },
    {
      "epoch": 15.0,
      "grad_norm": 0.004586809943697903,
      "learning_rate": 8.930309757836517e-07,
      "logits/chosen": -0.9402343034744263,
      "logits/rejected": -0.8805861473083496,
      "logps/chosen": -56.05229568481445,
      "logps/rejected": -1228.699951171875,
      "loss": 0.0003,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.19031982123851776,
      "rewards/margins": 11.688825607299805,
      "rewards/rejected": -11.498506546020508,
      "step": 750
    },
    {
      "epoch": 15.2,
      "grad_norm": 0.001855508098495543,
      "learning_rate": 8.271734841028553e-07,
      "logits/chosen": -0.9797531962394714,
      "logits/rejected": -0.8751746416091919,
      "logps/chosen": -59.685791015625,
      "logps/rejected": -1249.674560546875,
      "loss": 0.0003,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.17554932832717896,
      "rewards/margins": 11.926473617553711,
      "rewards/rejected": -11.750923156738281,
      "step": 760
    },
    {
      "epoch": 15.4,
      "grad_norm": 0.0009559205861326566,
      "learning_rate": 7.633540738525066e-07,
      "logits/chosen": -0.9692495465278625,
      "logits/rejected": -0.8025411367416382,
      "logps/chosen": -60.701194763183594,
      "logps/rejected": -1309.8841552734375,
      "loss": 0.0002,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.183763325214386,
      "rewards/margins": 12.551748275756836,
      "rewards/rejected": -12.367984771728516,
      "step": 770
    },
    {
      "epoch": 15.6,
      "grad_norm": 0.0016221612239979135,
      "learning_rate": 7.016504991533727e-07,
      "logits/chosen": -0.9435825347900391,
      "logits/rejected": -0.873455822467804,
      "logps/chosen": -58.68913650512695,
      "logps/rejected": -1226.319091796875,
      "loss": 0.0003,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.16507244110107422,
      "rewards/margins": 11.652719497680664,
      "rewards/rejected": -11.487646102905273,
      "step": 780
    },
    {
      "epoch": 15.8,
      "grad_norm": 0.0011565013550164168,
      "learning_rate": 6.421379363065142e-07,
      "logits/chosen": -1.0095231533050537,
      "logits/rejected": -0.7704787850379944,
      "logps/chosen": -65.56999206542969,
      "logps/rejected": -1337.3360595703125,
      "loss": 0.0003,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.18380019068717957,
      "rewards/margins": 12.87743091583252,
      "rewards/rejected": -12.693631172180176,
      "step": 790
    },
    {
      "epoch": 16.0,
      "grad_norm": 0.0012537423983021187,
      "learning_rate": 5.848888922025553e-07,
      "logits/chosen": -0.8937209248542786,
      "logits/rejected": -0.8747299313545227,
      "logps/chosen": -54.911643981933594,
      "logps/rejected": -1144.4791259765625,
      "loss": 0.0003,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.16021445393562317,
      "rewards/margins": 10.787330627441406,
      "rewards/rejected": -10.627116203308105,
      "step": 800
    },
    {
      "epoch": 16.2,
      "grad_norm": 0.0014373038754308242,
      "learning_rate": 5.299731159831953e-07,
      "logits/chosen": -0.9762552976608276,
      "logits/rejected": -0.8652079701423645,
      "logps/chosen": -58.197059631347656,
      "logps/rejected": -1232.221435546875,
      "loss": 0.0003,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.1831381916999817,
      "rewards/margins": 11.740102767944336,
      "rewards/rejected": -11.556963920593262,
      "step": 810
    },
    {
      "epoch": 16.4,
      "grad_norm": 0.000862440552553043,
      "learning_rate": 4.774575140626317e-07,
      "logits/chosen": -0.9985950589179993,
      "logits/rejected": -0.8936964273452759,
      "logps/chosen": -58.0911979675293,
      "logps/rejected": -1282.3681640625,
      "loss": 0.0003,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.19094052910804749,
      "rewards/margins": 12.265246391296387,
      "rewards/rejected": -12.074307441711426,
      "step": 820
    },
    {
      "epoch": 16.6,
      "grad_norm": 0.0011309096722098083,
      "learning_rate": 4.27406068612396e-07,
      "logits/chosen": -0.9642303586006165,
      "logits/rejected": -0.8438752293586731,
      "logps/chosen": -59.80333709716797,
      "logps/rejected": -1246.0048828125,
      "loss": 0.0002,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.16420656442642212,
      "rewards/margins": 11.867765426635742,
      "rewards/rejected": -11.703559875488281,
      "step": 830
    },
    {
      "epoch": 16.8,
      "grad_norm": 0.001634196377460084,
      "learning_rate": 3.798797596089351e-07,
      "logits/chosen": -0.9439367055892944,
      "logits/rejected": -0.8413814306259155,
      "logps/chosen": -59.905982971191406,
      "logps/rejected": -1256.216796875,
      "loss": 0.0003,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.16103532910346985,
      "rewards/margins": 11.955886840820312,
      "rewards/rejected": -11.794851303100586,
      "step": 840
    },
    {
      "epoch": 17.0,
      "grad_norm": 0.0011128513110797832,
      "learning_rate": 3.3493649053890325e-07,
      "logits/chosen": -0.9567734003067017,
      "logits/rejected": -0.7875978350639343,
      "logps/chosen": -61.99500274658203,
      "logps/rejected": -1306.9703369140625,
      "loss": 0.0002,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.1775272786617279,
      "rewards/margins": 12.505702018737793,
      "rewards/rejected": -12.32817554473877,
      "step": 850
    },
    {
      "epoch": 17.2,
      "grad_norm": 0.0013520172966032026,
      "learning_rate": 2.9263101785268253e-07,
      "logits/chosen": -0.9488283395767212,
      "logits/rejected": -0.8742234110832214,
      "logps/chosen": -56.80482864379883,
      "logps/rejected": -1268.548095703125,
      "loss": 0.0003,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.17435792088508606,
      "rewards/margins": 12.08137321472168,
      "rewards/rejected": -11.907015800476074,
      "step": 860
    },
    {
      "epoch": 17.4,
      "grad_norm": 0.0012511118706732029,
      "learning_rate": 2.53014884252083e-07,
      "logits/chosen": -1.0541223287582397,
      "logits/rejected": -0.81927490234375,
      "logps/chosen": -63.943763732910156,
      "logps/rejected": -1374.1676025390625,
      "loss": 0.0002,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.18310335278511047,
      "rewards/margins": 13.212549209594727,
      "rewards/rejected": -13.029446601867676,
      "step": 870
    },
    {
      "epoch": 17.6,
      "grad_norm": 0.0018809600968994445,
      "learning_rate": 2.1613635589349756e-07,
      "logits/chosen": -0.8839080929756165,
      "logits/rejected": -0.7969019412994385,
      "logps/chosen": -56.40065383911133,
      "logps/rejected": -1226.865478515625,
      "loss": 0.0003,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.17095676064491272,
      "rewards/margins": 11.626653671264648,
      "rewards/rejected": -11.455695152282715,
      "step": 880
    },
    {
      "epoch": 17.8,
      "grad_norm": 0.0016373893137363934,
      "learning_rate": 1.8204036358303173e-07,
      "logits/chosen": -0.9771726727485657,
      "logits/rejected": -0.8529565930366516,
      "logps/chosen": -57.20879364013672,
      "logps/rejected": -1290.327880859375,
      "loss": 0.0002,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.18934224545955658,
      "rewards/margins": 12.324792861938477,
      "rewards/rejected": -12.135449409484863,
      "step": 890
    },
    {
      "epoch": 18.0,
      "grad_norm": 0.000786600665947747,
      "learning_rate": 1.507684480352292e-07,
      "logits/chosen": -0.9094891548156738,
      "logits/rejected": -0.8147133588790894,
      "logps/chosen": -59.374755859375,
      "logps/rejected": -1234.203125,
      "loss": 0.0003,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.1541765034198761,
      "rewards/margins": 11.714710235595703,
      "rewards/rejected": -11.560534477233887,
      "step": 900
    },
    {
      "epoch": 18.2,
      "grad_norm": 0.002709830744526596,
      "learning_rate": 1.223587092621162e-07,
      "logits/chosen": -1.00485098361969,
      "logits/rejected": -0.7558645606040955,
      "logps/chosen": -67.38685607910156,
      "logps/rejected": -1367.7635498046875,
      "loss": 0.0002,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.17096880078315735,
      "rewards/margins": 13.181119918823242,
      "rewards/rejected": -13.010151863098145,
      "step": 910
    },
    {
      "epoch": 18.4,
      "grad_norm": 0.0010941683716015594,
      "learning_rate": 9.684576015420277e-08,
      "logits/chosen": -0.9139689207077026,
      "logits/rejected": -0.8125830888748169,
      "logps/chosen": -58.29936599731445,
      "logps/rejected": -1264.84033203125,
      "loss": 0.0003,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.16245688498020172,
      "rewards/margins": 12.02514934539795,
      "rewards/rejected": -11.862692832946777,
      "step": 920
    },
    {
      "epoch": 18.6,
      "grad_norm": 0.001790071139550289,
      "learning_rate": 7.426068431000883e-08,
      "logits/chosen": -0.9928874969482422,
      "logits/rejected": -0.8945854306221008,
      "logps/chosen": -59.90386962890625,
      "logps/rejected": -1225.345458984375,
      "loss": 0.0002,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.18468160927295685,
      "rewards/margins": 11.685681343078613,
      "rewards/rejected": -11.500999450683594,
      "step": 930
    },
    {
      "epoch": 18.8,
      "grad_norm": 0.001444924500021094,
      "learning_rate": 5.463099816548578e-08,
      "logits/chosen": -0.8352192640304565,
      "logits/rejected": -0.8455368876457214,
      "logps/chosen": -53.367652893066406,
      "logps/rejected": -1094.0107421875,
      "loss": 0.0003,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.15318068861961365,
      "rewards/margins": 10.247284889221191,
      "rewards/rejected": -10.094103813171387,
      "step": 940
    },
    {
      "epoch": 19.0,
      "grad_norm": 0.0011127218090168816,
      "learning_rate": 3.798061746947995e-08,
      "logits/chosen": -0.9996277689933777,
      "logits/rejected": -0.7810768485069275,
      "logps/chosen": -62.9672737121582,
      "logps/rejected": -1361.350341796875,
      "loss": 0.0002,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.1840919703245163,
      "rewards/margins": 13.088468551635742,
      "rewards/rejected": -12.904377937316895,
      "step": 950
    },
    {
      "epoch": 19.2,
      "grad_norm": 0.0011199039962919363,
      "learning_rate": 2.4329828146074096e-08,
      "logits/chosen": -0.9857956171035767,
      "logits/rejected": -0.8834479451179504,
      "logps/chosen": -61.652801513671875,
      "logps/rejected": -1269.288330078125,
      "loss": 0.0003,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.14881816506385803,
      "rewards/margins": 12.092996597290039,
      "rewards/rejected": -11.944178581237793,
      "step": 960
    },
    {
      "epoch": 19.4,
      "grad_norm": 0.0013686833523596474,
      "learning_rate": 1.3695261579316776e-08,
      "logits/chosen": -0.9996697306632996,
      "logits/rejected": -0.8662905693054199,
      "logps/chosen": -60.64554977416992,
      "logps/rejected": -1260.70458984375,
      "loss": 0.0003,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.18075986206531525,
      "rewards/margins": 12.048669815063477,
      "rewards/rejected": -11.867908477783203,
      "step": 970
    },
    {
      "epoch": 19.6,
      "grad_norm": 0.0017745592515844095,
      "learning_rate": 6.089874350439507e-09,
      "logits/chosen": -1.0439860820770264,
      "logits/rejected": -0.9185269474983215,
      "logps/chosen": -59.79936981201172,
      "logps/rejected": -1267.5032958984375,
      "loss": 0.0003,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.18838214874267578,
      "rewards/margins": 12.108388900756836,
      "rewards/rejected": -11.92000675201416,
      "step": 980
    },
    {
      "epoch": 19.8,
      "grad_norm": 0.0013032690672856834,
      "learning_rate": 1.5229324522605949e-09,
      "logits/chosen": -0.9330822229385376,
      "logits/rejected": -0.8125349879264832,
      "logps/chosen": -58.721580505371094,
      "logps/rejected": -1287.365966796875,
      "loss": 0.0002,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.17665794491767883,
      "rewards/margins": 12.307037353515625,
      "rewards/rejected": -12.130378723144531,
      "step": 990
    },
    {
      "epoch": 20.0,
      "grad_norm": 0.003965487525700407,
      "learning_rate": 0.0,
      "logits/chosen": -0.9679548144340515,
      "logits/rejected": -0.8124309778213501,
      "logps/chosen": -61.12044143676758,
      "logps/rejected": -1297.4541015625,
      "loss": 0.0002,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.1760241687297821,
      "rewards/margins": 12.41258716583252,
      "rewards/rejected": -12.236563682556152,
      "step": 1000
    },
    {
      "epoch": 20.0,
      "step": 1000,
      "total_flos": 0.0,
      "train_loss": 0.07008945742505603,
      "train_runtime": 8233.3372,
      "train_samples_per_second": 1.943,
      "train_steps_per_second": 0.121
    }
  ],
  "logging_steps": 10,
  "max_steps": 1000,
  "num_input_tokens_seen": 0,
  "num_train_epochs": 20,
  "save_steps": 1000,
  "stateful_callbacks": {
    "TrainerControl": {
      "args": {
        "should_epoch_stop": false,
        "should_evaluate": false,
        "should_log": false,
        "should_save": true,
        "should_training_stop": true
      },
      "attributes": {}
    }
  },
  "total_flos": 0.0,
  "train_batch_size": 2,
  "trial_name": null,
  "trial_params": null
}