{
  "best_metric": null,
  "best_model_checkpoint": null,
  "epoch": 1.9992126604204392,
  "eval_steps": 50,
  "global_step": 2116,
  "is_hyper_param_search": false,
  "is_local_process_zero": true,
  "is_world_process_zero": true,
  "log_history": [
    {
      "dpo_loss": 0.6931471824645996,
      "epoch": 0.0009448074954727974,
      "grad_norm": 66.46447760146653,
      "learning_rate": 4.716981132075471e-10,
      "logits": -2.096372604370117,
      "logps": -90.695556640625,
      "loss": 0.0053,
      "objective": 0.004870629869401455,
      "ranking_idealized": 0.5,
      "ranking_idealized_expo": 0.5,
      "ranking_simple": 0.5,
      "regularize": 0.004867264535278082,
      "step": 1
    },
    {
      "dpo_loss": 0.6931397914886475,
      "epoch": 0.004724037477363987,
      "grad_norm": 73.1061214549564,
      "learning_rate": 2.358490566037736e-09,
      "logits": -2.2148256301879883,
      "logps": -89.18419647216797,
      "loss": 0.0056,
      "objective": 0.006199519615620375,
      "ranking_idealized": 0.5,
      "ranking_idealized_expo": 0.3958333432674408,
      "ranking_simple": 0.3958333432674408,
      "regularize": 0.0061979773454368114,
      "step": 5
    },
    {
      "dpo_loss": 0.6931113600730896,
      "epoch": 0.009448074954727975,
      "grad_norm": 66.26130454557787,
      "learning_rate": 4.716981132075472e-09,
      "logits": -2.1163227558135986,
      "logps": -90.67784118652344,
      "loss": 0.0056,
      "objective": 0.0063161361031234264,
      "ranking_idealized": 0.5333333611488342,
      "ranking_idealized_expo": 0.4166666567325592,
      "ranking_simple": 0.4166666567325592,
      "regularize": 0.006314346566796303,
      "step": 10
    },
    {
      "dpo_loss": 0.6930894255638123,
      "epoch": 0.014172112432091962,
      "grad_norm": 65.86685369935576,
      "learning_rate": 7.075471698113207e-09,
      "logits": -2.0237159729003906,
      "logps": -88.98513793945312,
      "loss": 0.0052,
      "objective": 0.004170234780758619,
      "ranking_idealized": 0.6166666746139526,
      "ranking_idealized_expo": 0.5166666507720947,
      "ranking_simple": 0.5166666507720947,
      "regularize": 0.004166428931057453,
      "step": 15
    },
    {
      "dpo_loss": 0.6932118535041809,
      "epoch": 0.01889614990945595,
      "grad_norm": 67.08170030108543,
      "learning_rate": 9.433962264150943e-09,
      "logits": -2.220696449279785,
      "logps": -91.20802307128906,
      "loss": 0.0056,
      "objective": 0.0056175715290009975,
      "ranking_idealized": 0.6333333253860474,
      "ranking_idealized_expo": 0.46666666865348816,
      "ranking_simple": 0.46666666865348816,
      "regularize": 0.00561108160763979,
      "step": 20
    },
    {
      "dpo_loss": 0.6930363774299622,
      "epoch": 0.023620187386819935,
      "grad_norm": 64.75938875868191,
      "learning_rate": 1.1792452830188679e-08,
      "logits": -2.184110164642334,
      "logps": -93.84049224853516,
      "loss": 0.005,
      "objective": 0.005908097140491009,
      "ranking_idealized": 0.699999988079071,
      "ranking_idealized_expo": 0.6166666746139526,
      "ranking_simple": 0.6166666746139526,
      "regularize": 0.0059060631319880486,
      "step": 25
    },
    {
      "dpo_loss": 0.6930921673774719,
      "epoch": 0.028344224864183924,
      "grad_norm": 70.85746649835617,
      "learning_rate": 1.4150943396226414e-08,
      "logits": -2.048970937728882,
      "logps": -92.22063446044922,
      "loss": 0.0062,
      "objective": 0.006645068060606718,
      "ranking_idealized": 0.6166666746139526,
      "ranking_idealized_expo": 0.5,
      "ranking_simple": 0.5,
      "regularize": 0.0066424161195755005,
      "step": 30
    },
    {
      "dpo_loss": 0.6927813291549683,
      "epoch": 0.03306826234154791,
      "grad_norm": 69.39212454087999,
      "learning_rate": 1.6509433962264148e-08,
      "logits": -2.164494276046753,
      "logps": -90.56898498535156,
      "loss": 0.006,
      "objective": 0.0044576446525752544,
      "ranking_idealized": 0.6333333253860474,
      "ranking_idealized_expo": 0.5666666626930237,
      "ranking_simple": 0.5666666626930237,
      "regularize": 0.004453611560165882,
      "step": 35
    },
    {
      "dpo_loss": 0.6927517056465149,
      "epoch": 0.0377922998189119,
      "grad_norm": 79.75758352153623,
      "learning_rate": 1.8867924528301887e-08,
      "logits": -2.2105553150177,
      "logps": -94.57971954345703,
      "loss": 0.0058,
      "objective": 0.004820433910936117,
      "ranking_idealized": 0.6166666746139526,
      "ranking_idealized_expo": 0.5833333134651184,
      "ranking_simple": 0.5833333134651184,
      "regularize": 0.004817112348973751,
      "step": 40
    },
    {
      "dpo_loss": 0.6930319666862488,
      "epoch": 0.04251633729627589,
      "grad_norm": 64.62991156662962,
      "learning_rate": 2.1226415094339622e-08,
      "logits": -2.149827241897583,
      "logps": -90.14302062988281,
      "loss": 0.0057,
      "objective": 0.004925203043967485,
      "ranking_idealized": 0.550000011920929,
      "ranking_idealized_expo": 0.4833333194255829,
      "ranking_simple": 0.4833333194255829,
      "regularize": 0.004921570885926485,
      "step": 45
    },
    {
      "dpo_loss": 0.692544162273407,
      "epoch": 0.04724037477363987,
      "grad_norm": 68.06296242846598,
      "learning_rate": 2.3584905660377358e-08,
      "logits": -2.1563477516174316,
      "logps": -91.66796875,
      "loss": 0.006,
      "objective": 0.006787313614040613,
      "ranking_idealized": 0.6499999761581421,
      "ranking_idealized_expo": 0.550000011920929,
      "ranking_simple": 0.550000011920929,
      "regularize": 0.006785000674426556,
      "step": 50
    },
    {
      "epoch": 0.04724037477363987,
      "eval_dpo_loss": 0.6929724812507629,
      "eval_logits": -1.9958003759384155,
      "eval_logps": -98.61515808105469,
      "eval_loss": 0.0059888348914682865,
      "eval_objective": 0.006077947095036507,
      "eval_ranking_idealized": 0.6022099256515503,
      "eval_ranking_idealized_expo": 0.5207182168960571,
      "eval_ranking_simple": 0.5179557800292969,
      "eval_regularize": 0.006075054872781038,
      "eval_runtime": 445.1462,
      "eval_samples_per_second": 13.007,
      "eval_steps_per_second": 3.253,
      "step": 50
    },
    {
      "dpo_loss": 0.6929494738578796,
      "epoch": 0.05196441225100386,
      "grad_norm": 73.41432282855463,
      "learning_rate": 2.5943396226415093e-08,
      "logits": -2.1937828063964844,
      "logps": -93.48242950439453,
      "loss": 0.006,
      "objective": 0.005628067534416914,
      "ranking_idealized": 0.46666666865348816,
      "ranking_idealized_expo": 0.4166666567325592,
      "ranking_simple": 0.4166666567325592,
      "regularize": 0.005625530146062374,
      "step": 55
    },
    {
      "dpo_loss": 0.692658543586731,
      "epoch": 0.05668844972836785,
      "grad_norm": 65.33336896476966,
      "learning_rate": 2.830188679245283e-08,
      "logits": -2.2241933345794678,
      "logps": -89.14469146728516,
      "loss": 0.0058,
      "objective": 0.004998047836124897,
      "ranking_idealized": 0.6499999761581421,
      "ranking_idealized_expo": 0.6166666746139526,
      "ranking_simple": 0.6166666746139526,
      "regularize": 0.004994309972971678,
      "step": 60
    },
    {
      "dpo_loss": 0.692764163017273,
      "epoch": 0.06141248720573183,
      "grad_norm": 66.55102685698147,
      "learning_rate": 3.0660377358490564e-08,
      "logits": -2.2217800617218018,
      "logps": -89.03886413574219,
      "loss": 0.0074,
      "objective": 0.0076709226705133915,
      "ranking_idealized": 0.5,
      "ranking_idealized_expo": 0.38333332538604736,
      "ranking_simple": 0.38333332538604736,
      "regularize": 0.007668651174753904,
      "step": 65
    },
    {
      "dpo_loss": 0.6925709843635559,
      "epoch": 0.06613652468309582,
      "grad_norm": 75.22580153097067,
      "learning_rate": 3.3018867924528296e-08,
      "logits": -2.1472768783569336,
      "logps": -94.85504913330078,
      "loss": 0.0076,
      "objective": 0.007108477409929037,
      "ranking_idealized": 0.550000011920929,
      "ranking_idealized_expo": 0.5166666507720947,
      "ranking_simple": 0.5166666507720947,
      "regularize": 0.007105442229658365,
      "step": 70
    },
    {
      "dpo_loss": 0.6925690770149231,
      "epoch": 0.0708605621604598,
      "grad_norm": 79.40443090905357,
      "learning_rate": 3.5377358490566035e-08,
      "logits": -2.206367015838623,
      "logps": -92.62692260742188,
      "loss": 0.0071,
      "objective": 0.007145268842577934,
      "ranking_idealized": 0.5666666626930237,
      "ranking_idealized_expo": 0.5166666507720947,
      "ranking_simple": 0.5166666507720947,
      "regularize": 0.00714282738044858,
      "step": 75
    },
    {
      "dpo_loss": 0.6921057105064392,
      "epoch": 0.0755845996378238,
      "grad_norm": 76.34709727963714,
      "learning_rate": 3.7735849056603774e-08,
      "logits": -2.1775050163269043,
      "logps": -94.01787567138672,
      "loss": 0.0066,
      "objective": 0.006093442440032959,
      "ranking_idealized": 0.6499999761581421,
      "ranking_idealized_expo": 0.5666666626930237,
      "ranking_simple": 0.5666666626930237,
      "regularize": 0.006089916918426752,
      "step": 80
    },
    {
      "dpo_loss": 0.6927501559257507,
      "epoch": 0.08030863711518778,
      "grad_norm": 76.24460999547686,
      "learning_rate": 4.009433962264151e-08,
      "logits": -2.1956326961517334,
      "logps": -92.5903091430664,
      "loss": 0.0069,
      "objective": 0.0067410701885819435,
      "ranking_idealized": 0.699999988079071,
      "ranking_idealized_expo": 0.6000000238418579,
      "ranking_simple": 0.6000000238418579,
      "regularize": 0.006739132571965456,
      "step": 85
    },
    {
      "dpo_loss": 0.6902840733528137,
      "epoch": 0.08503267459255177,
      "grad_norm": 65.10729628861057,
      "learning_rate": 4.2452830188679244e-08,
      "logits": -2.2021093368530273,
      "logps": -96.31147766113281,
      "loss": 0.0075,
      "objective": 0.008030267432332039,
      "ranking_idealized": 0.6499999761581421,
      "ranking_idealized_expo": 0.5833333134651184,
      "ranking_simple": 0.5833333134651184,
      "regularize": 0.008029019460082054,
      "step": 90
    },
    {
      "dpo_loss": 0.6926410794258118,
      "epoch": 0.08975671206991576,
      "grad_norm": 71.92277477052832,
      "learning_rate": 4.481132075471698e-08,
      "logits": -2.1148061752319336,
      "logps": -89.49421691894531,
      "loss": 0.007,
      "objective": 0.0068625533021986485,
      "ranking_idealized": 0.5833333134651184,
      "ranking_idealized_expo": 0.550000011920929,
      "ranking_simple": 0.550000011920929,
      "regularize": 0.006859814748167992,
      "step": 95
    },
    {
      "dpo_loss": 0.6942407488822937,
      "epoch": 0.09448074954727974,
      "grad_norm": 65.63913294191785,
      "learning_rate": 4.7169811320754715e-08,
      "logits": -2.2743189334869385,
      "logps": -94.41613006591797,
      "loss": 0.0092,
      "objective": 0.012641828507184982,
      "ranking_idealized": 0.6499999761581421,
      "ranking_idealized_expo": 0.5666666626930237,
      "ranking_simple": 0.5333333611488342,
      "regularize": 0.012639058753848076,
      "step": 100
    },
    {
      "epoch": 0.09448074954727974,
      "eval_dpo_loss": 0.6929203867912292,
      "eval_logits": -1.995413899421692,
      "eval_logps": -98.7889404296875,
      "eval_loss": 0.00729329651221633,
      "eval_objective": 0.00731161143630743,
      "eval_ranking_idealized": 0.6022099256515503,
      "eval_ranking_idealized_expo": 0.5207182168960571,
      "eval_ranking_simple": 0.5186464190483093,
      "eval_regularize": 0.00730866938829422,
      "eval_runtime": 447.0397,
      "eval_samples_per_second": 12.952,
      "eval_steps_per_second": 3.239,
      "step": 100
    },
    {
      "dpo_loss": 0.6924384832382202,
      "epoch": 0.09920478702464373,
      "grad_norm": 67.55366715626052,
      "learning_rate": 4.9528301886792454e-08,
      "logits": -2.088653802871704,
      "logps": -92.43064880371094,
      "loss": 0.009,
      "objective": 0.00849145371466875,
      "ranking_idealized": 0.5833333134651184,
      "ranking_idealized_expo": 0.5166666507720947,
      "ranking_simple": 0.5166666507720947,
      "regularize": 0.008488119579851627,
      "step": 105
    },
    {
      "dpo_loss": 0.6923626661300659,
      "epoch": 0.10392882450200772,
      "grad_norm": 68.4026236372278,
      "learning_rate": 5.1886792452830186e-08,
      "logits": -2.1796536445617676,
      "logps": -91.34872436523438,
      "loss": 0.0083,
      "objective": 0.0080822529271245,
      "ranking_idealized": 0.5666666626930237,
      "ranking_idealized_expo": 0.4833333194255829,
      "ranking_simple": 0.4833333194255829,
      "regularize": 0.008080901578068733,
      "step": 110
    },
    {
      "dpo_loss": 0.6901246309280396,
      "epoch": 0.1086528619793717,
      "grad_norm": 64.06144820478669,
      "learning_rate": 5.4245283018867925e-08,
      "logits": -2.1858596801757812,
      "logps": -89.73294830322266,
      "loss": 0.0097,
      "objective": 0.013094036839902401,
      "ranking_idealized": 0.5333333611488342,
      "ranking_idealized_expo": 0.46666666865348816,
      "ranking_simple": 0.46666666865348816,
      "regularize": 0.013092008419334888,
      "step": 115
    },
    {
      "dpo_loss": 0.6923102736473083,
      "epoch": 0.1133768994567357,
      "grad_norm": 61.237648061668985,
      "learning_rate": 5.660377358490566e-08,
      "logits": -2.1982421875,
      "logps": -94.54019927978516,
      "loss": 0.0085,
      "objective": 0.008155700750648975,
      "ranking_idealized": 0.7166666388511658,
      "ranking_idealized_expo": 0.6666666865348816,
      "ranking_simple": 0.6666666865348816,
      "regularize": 0.00815290305763483,
      "step": 120
    },
    {
      "dpo_loss": 0.6917387247085571,
      "epoch": 0.11810093693409968,
      "grad_norm": 73.42929721502428,
      "learning_rate": 5.8962264150943396e-08,
      "logits": -2.1935439109802246,
      "logps": -95.18013000488281,
      "loss": 0.0103,
      "objective": 0.010648042894899845,
      "ranking_idealized": 0.5333333611488342,
      "ranking_idealized_expo": 0.4833333194255829,
      "ranking_simple": 0.4833333194255829,
      "regularize": 0.010645588859915733,
      "step": 125
    },
    {
      "dpo_loss": 0.6919466853141785,
      "epoch": 0.12282497441146366,
      "grad_norm": 68.99586341226502,
      "learning_rate": 6.132075471698113e-08,
      "logits": -2.1257071495056152,
      "logps": -97.63853454589844,
      "loss": 0.0103,
      "objective": 0.009905511513352394,
      "ranking_idealized": 0.6333333253860474,
      "ranking_idealized_expo": 0.6000000238418579,
      "ranking_simple": 0.6000000238418579,
      "regularize": 0.009904048405587673,
      "step": 130
    },
    {
      "dpo_loss": 0.6923593878746033,
      "epoch": 0.12754901188882764,
      "grad_norm": 56.22865998140236,
      "learning_rate": 6.367924528301887e-08,
      "logits": -2.1036369800567627,
      "logps": -91.52125549316406,
      "loss": 0.0098,
      "objective": 0.009057187475264072,
      "ranking_idealized": 0.5333333611488342,
      "ranking_idealized_expo": 0.46666666865348816,
      "ranking_simple": 0.46666666865348816,
      "regularize": 0.009054058231413364,
      "step": 135
    },
    {
      "dpo_loss": 0.6894025206565857,
      "epoch": 0.13227304936619164,
      "grad_norm": 64.3061621244485,
      "learning_rate": 6.603773584905659e-08,
      "logits": -2.246990442276001,
      "logps": -92.88211059570312,
      "loss": 0.0113,
      "objective": 0.011589973233640194,
      "ranking_idealized": 0.6000000238418579,
      "ranking_idealized_expo": 0.5333333611488342,
      "ranking_simple": 0.5333333611488342,
      "regularize": 0.011587386019527912,
      "step": 140
    },
    {
      "dpo_loss": 0.6934190988540649,
      "epoch": 0.13699708684355563,
      "grad_norm": 74.71610410718624,
      "learning_rate": 6.839622641509434e-08,
      "logits": -2.102886199951172,
      "logps": -90.7693099975586,
      "loss": 0.0123,
      "objective": 0.00984902959316969,
      "ranking_idealized": 0.5666666626930237,
      "ranking_idealized_expo": 0.5,
      "ranking_simple": 0.4833333194255829,
      "regularize": 0.009846380911767483,
      "step": 145
    },
    {
      "dpo_loss": 0.6908667087554932,
      "epoch": 0.1417211243209196,
      "grad_norm": 76.42928931969817,
      "learning_rate": 7.075471698113207e-08,
      "logits": -2.238095998764038,
      "logps": -94.04148864746094,
      "loss": 0.0142,
      "objective": 0.014535349793732166,
      "ranking_idealized": 0.6000000238418579,
      "ranking_idealized_expo": 0.5166666507720947,
      "ranking_simple": 0.5166666507720947,
      "regularize": 0.014532845467329025,
      "step": 150
    },
    {
      "epoch": 0.1417211243209196,
      "eval_dpo_loss": 0.693015992641449,
      "eval_logits": -1.9985584020614624,
      "eval_logps": -98.66200256347656,
      "eval_loss": 0.009166295640170574,
      "eval_objective": 0.00932803563773632,
      "eval_ranking_idealized": 0.6022099256515503,
      "eval_ranking_idealized_expo": 0.5207182168960571,
      "eval_ranking_simple": 0.5186464190483093,
      "eval_regularize": 0.009325726889073849,
      "eval_runtime": 446.6869,
      "eval_samples_per_second": 12.962,
      "eval_steps_per_second": 3.242,
      "step": 150
    },
    {
      "dpo_loss": 0.6918679475784302,
      "epoch": 0.1464451617982836,
      "grad_norm": 80.89548200084305,
      "learning_rate": 7.311320754716981e-08,
      "logits": -2.1368908882141113,
      "logps": -90.76949310302734,
      "loss": 0.0131,
      "objective": 0.011844370514154434,
      "ranking_idealized": 0.699999988079071,
      "ranking_idealized_expo": 0.5833333134651184,
      "ranking_simple": 0.5833333134651184,
      "regularize": 0.011842181906104088,
      "step": 155
    },
    {
      "dpo_loss": 0.6904442310333252,
      "epoch": 0.1511691992756476,
      "grad_norm": 70.5889289835053,
      "learning_rate": 7.547169811320755e-08,
      "logits": -2.216298818588257,
      "logps": -94.7828140258789,
      "loss": 0.0132,
      "objective": 0.012807334773242474,
      "ranking_idealized": 0.5666666626930237,
      "ranking_idealized_expo": 0.5,
      "ranking_simple": 0.5,
      "regularize": 0.012806250713765621,
      "step": 160
    },
    {
      "dpo_loss": 0.6893646121025085,
      "epoch": 0.15589323675301156,
      "grad_norm": 70.53443315770147,
      "learning_rate": 7.783018867924527e-08,
      "logits": -2.2371225357055664,
      "logps": -91.81126403808594,
      "loss": 0.0123,
      "objective": 0.014454828575253487,
      "ranking_idealized": 0.5666666626930237,
      "ranking_idealized_expo": 0.46666666865348816,
      "ranking_simple": 0.46666666865348816,
      "regularize": 0.01445354800671339,
      "step": 165
    },
    {
      "dpo_loss": 0.6928521990776062,
      "epoch": 0.16061727423037556,
      "grad_norm": 78.26059021460965,
      "learning_rate": 8.018867924528302e-08,
      "logits": -2.215101480484009,
      "logps": -92.55703735351562,
      "loss": 0.0141,
      "objective": 0.012299363501369953,
      "ranking_idealized": 0.699999988079071,
      "ranking_idealized_expo": 0.5833333134651184,
      "ranking_simple": 0.5833333134651184,
      "regularize": 0.012298095040023327,
      "step": 170
    },
    {
      "dpo_loss": 0.6916810274124146,
      "epoch": 0.16534131170773955,
      "grad_norm": 62.838329994801875,
      "learning_rate": 8.254716981132075e-08,
      "logits": -2.1370534896850586,
      "logps": -91.3881607055664,
      "loss": 0.014,
      "objective": 0.015395297668874264,
      "ranking_idealized": 0.6499999761581421,
      "ranking_idealized_expo": 0.5666666626930237,
      "ranking_simple": 0.5666666626930237,
      "regularize": 0.015393855981528759,
      "step": 175
    },
    {
      "dpo_loss": 0.6918050050735474,
      "epoch": 0.17006534918510355,
      "grad_norm": 73.17909568701033,
      "learning_rate": 8.490566037735849e-08,
      "logits": -2.281663417816162,
      "logps": -94.94965362548828,
      "loss": 0.0138,
      "objective": 0.017136668786406517,
      "ranking_idealized": 0.6000000238418579,
      "ranking_idealized_expo": 0.5166666507720947,
      "ranking_simple": 0.5166666507720947,
      "regularize": 0.017135154455900192,
      "step": 180
    },
    {
      "dpo_loss": 0.69007408618927,
      "epoch": 0.17478938666246752,
      "grad_norm": 62.37985427959411,
      "learning_rate": 8.726415094339621e-08,
      "logits": -2.1940929889678955,
      "logps": -93.47923278808594,
      "loss": 0.0159,
      "objective": 0.017906082794070244,
      "ranking_idealized": 0.550000011920929,
      "ranking_idealized_expo": 0.5,
      "ranking_simple": 0.5166666507720947,
      "regularize": 0.017902227118611336,
      "step": 185
    },
    {
      "dpo_loss": 0.6921120882034302,
      "epoch": 0.1795134241398315,
      "grad_norm": 70.11549174821204,
      "learning_rate": 8.962264150943397e-08,
      "logits": -2.1571404933929443,
      "logps": -87.7894287109375,
      "loss": 0.0144,
      "objective": 0.01052508968859911,
      "ranking_idealized": 0.6666666865348816,
      "ranking_idealized_expo": 0.5166666507720947,
      "ranking_simple": 0.5166666507720947,
      "regularize": 0.010523774661123753,
      "step": 190
    },
    {
      "dpo_loss": 0.6912659406661987,
      "epoch": 0.1842374616171955,
      "grad_norm": 78.18173910687094,
      "learning_rate": 9.198113207547169e-08,
      "logits": -2.1662251949310303,
      "logps": -90.77739715576172,
      "loss": 0.0139,
      "objective": 0.01253608800470829,
      "ranking_idealized": 0.7166666388511658,
      "ranking_idealized_expo": 0.6000000238418579,
      "ranking_simple": 0.6000000238418579,
      "regularize": 0.012534084729850292,
      "step": 195
    },
    {
      "dpo_loss": 0.6948674321174622,
      "epoch": 0.18896149909455948,
      "grad_norm": 74.9573189190215,
      "learning_rate": 9.433962264150943e-08,
      "logits": -2.1426119804382324,
      "logps": -93.58480072021484,
      "loss": 0.0173,
      "objective": 0.018598254770040512,
      "ranking_idealized": 0.5833333134651184,
      "ranking_idealized_expo": 0.5,
      "ranking_simple": 0.5,
      "regularize": 0.018596837297081947,
      "step": 200
    },
    {
      "epoch": 0.18896149909455948,
      "eval_dpo_loss": 0.6928583383560181,
      "eval_logits": -1.9957162141799927,
      "eval_logps": -98.79460144042969,
      "eval_loss": 0.009674900211393833,
      "eval_objective": 0.009766080416738987,
      "eval_ranking_idealized": 0.6022099256515503,
      "eval_ranking_idealized_expo": 0.5207182168960571,
      "eval_ranking_simple": 0.5172652006149292,
      "eval_regularize": 0.009763876907527447,
      "eval_runtime": 457.4524,
      "eval_samples_per_second": 12.657,
      "eval_steps_per_second": 3.165,
      "step": 200
    },
    {
      "dpo_loss": 0.6902305483818054,
      "epoch": 0.19368553657192347,
      "grad_norm": 69.58780380589624,
      "learning_rate": 9.669811320754716e-08,
      "logits": -2.137186288833618,
      "logps": -90.56549835205078,
      "loss": 0.0203,
      "objective": 0.016451861709356308,
      "ranking_idealized": 0.550000011920929,
      "ranking_idealized_expo": 0.5,
      "ranking_simple": 0.5,
      "regularize": 0.016451073810458183,
      "step": 205
    },
    {
      "dpo_loss": 0.693261444568634,
      "epoch": 0.19840957404928747,
      "grad_norm": 62.13478304132868,
      "learning_rate": 9.905660377358491e-08,
      "logits": -2.0685033798217773,
      "logps": -92.36515045166016,
      "loss": 0.0198,
      "objective": 0.02386774867773056,
      "ranking_idealized": 0.5166666507720947,
      "ranking_idealized_expo": 0.46666666865348816,
      "ranking_simple": 0.44999998807907104,
      "regularize": 0.023865588009357452,
      "step": 210
    },
    {
      "dpo_loss": 0.6884204149246216,
      "epoch": 0.20313361152665144,
      "grad_norm": 60.854335858242926,
      "learning_rate": 9.999938744161562e-08,
      "logits": -2.1262550354003906,
      "logps": -92.82433319091797,
      "loss": 0.0173,
      "objective": 0.020975453779101372,
      "ranking_idealized": 0.6333333253860474,
      "ranking_idealized_expo": 0.5833333134651184,
      "ranking_simple": 0.5833333134651184,
      "regularize": 0.020974494516849518,
      "step": 215
    },
    {
      "dpo_loss": 0.6918230652809143,
      "epoch": 0.20785764900401543,
      "grad_norm": 67.33088424835829,
      "learning_rate": 9.999564408362053e-08,
      "logits": -2.169888734817505,
      "logps": -88.9549331665039,
      "loss": 0.0187,
      "objective": 0.015909165143966675,
      "ranking_idealized": 0.5333333611488342,
      "ranking_idealized_expo": 0.4333333373069763,
      "ranking_simple": 0.4333333373069763,
      "regularize": 0.015907270833849907,
      "step": 220
    },
    {
      "dpo_loss": 0.689198911190033,
      "epoch": 0.21258168648137943,
      "grad_norm": 70.77403555695805,
      "learning_rate": 9.998849793231472e-08,
      "logits": -2.099949359893799,
      "logps": -91.25509643554688,
      "loss": 0.0218,
      "objective": 0.02295403741300106,
      "ranking_idealized": 0.6166666746139526,
      "ranking_idealized_expo": 0.46666666865348816,
      "ranking_simple": 0.46666666865348816,
      "regularize": 0.022952700033783913,
      "step": 225
    },
    {
      "dpo_loss": 0.6939520239830017,
      "epoch": 0.2173057239587434,
      "grad_norm": 80.6144373753939,
      "learning_rate": 9.997794947407808e-08,
      "logits": -2.1620826721191406,
      "logps": -94.76692962646484,
      "loss": 0.0239,
      "objective": 0.026342039927840233,
      "ranking_idealized": 0.6166666746139526,
      "ranking_idealized_expo": 0.5333333611488342,
      "ranking_simple": 0.5333333611488342,
      "regularize": 0.026341309770941734,
      "step": 230
    },
    {
      "dpo_loss": 0.688434362411499,
      "epoch": 0.2220297614361074,
      "grad_norm": 62.83216394514196,
      "learning_rate": 9.996399942685763e-08,
      "logits": -2.2001922130584717,
      "logps": -90.64398956298828,
      "loss": 0.0197,
      "objective": 0.02317703142762184,
      "ranking_idealized": 0.6666666865348816,
      "ranking_idealized_expo": 0.6000000238418579,
      "ranking_simple": 0.6000000238418579,
      "regularize": 0.023176301270723343,
      "step": 235
    },
    {
      "dpo_loss": 0.6894098520278931,
      "epoch": 0.2267537989134714,
      "grad_norm": 69.59337954721937,
      "learning_rate": 9.994664874011862e-08,
      "logits": -2.1389572620391846,
      "logps": -91.78279113769531,
      "loss": 0.0225,
      "objective": 0.02215094491839409,
      "ranking_idealized": 0.5333333611488342,
      "ranking_idealized_expo": 0.46666666865348816,
      "ranking_simple": 0.46666666865348816,
      "regularize": 0.02214915119111538,
      "step": 240
    },
    {
      "dpo_loss": 0.692762017250061,
      "epoch": 0.23147783639083536,
      "grad_norm": 70.45563918532531,
      "learning_rate": 9.992589859477995e-08,
      "logits": -2.0933754444122314,
      "logps": -92.82108306884766,
      "loss": 0.0215,
      "objective": 0.02130296640098095,
      "ranking_idealized": 0.6666666865348816,
      "ranking_idealized_expo": 0.550000011920929,
      "ranking_simple": 0.550000011920929,
      "regularize": 0.021301595494151115,
      "step": 245
    },
    {
      "dpo_loss": 0.6907398104667664,
      "epoch": 0.23620187386819935,
      "grad_norm": 68.46129704284265,
      "learning_rate": 9.990175040313376e-08,
      "logits": -2.0800933837890625,
      "logps": -88.17916870117188,
      "loss": 0.0245,
      "objective": 0.022766491398215294,
      "ranking_idealized": 0.6333333253860474,
      "ranking_idealized_expo": 0.5833333134651184,
      "ranking_simple": 0.5833333134651184,
      "regularize": 0.02276558242738247,
      "step": 250
    },
    {
      "epoch": 0.23620187386819935,
      "eval_dpo_loss": 0.6929402351379395,
      "eval_logits": -1.995142936706543,
      "eval_logps": -98.64155578613281,
      "eval_loss": 0.012135702185332775,
      "eval_objective": 0.012087649665772915,
      "eval_ranking_idealized": 0.6022099256515503,
      "eval_ranking_idealized_expo": 0.5207182168960571,
      "eval_ranking_simple": 0.5186464190483093,
      "eval_regularize": 0.012085951864719391,
      "eval_runtime": 446.4062,
      "eval_samples_per_second": 12.97,
      "eval_steps_per_second": 3.244,
      "step": 250
    },
    {
      "dpo_loss": 0.6918343901634216,
      "epoch": 0.24092591134556335,
      "grad_norm": 67.22846019191091,
      "learning_rate": 9.987420580874936e-08,
      "logits": -2.116420269012451,
      "logps": -90.83265686035156,
      "loss": 0.0239,
      "objective": 0.019956286996603012,
      "ranking_idealized": 0.6666666865348816,
      "ranking_idealized_expo": 0.5333333611488342,
      "ranking_simple": 0.5666666626930237,
      "regularize": 0.019955595955252647,
      "step": 255
    },
    {
      "dpo_loss": 0.688471257686615,
      "epoch": 0.24564994882292732,
      "grad_norm": 70.37729127367317,
      "learning_rate": 9.98432666863613e-08,
      "logits": -2.240382194519043,
      "logps": -94.62246704101562,
      "loss": 0.0228,
      "objective": 0.026484496891498566,
      "ranking_idealized": 0.5666666626930237,
      "ranking_idealized_expo": 0.5166666507720947,
      "ranking_simple": 0.5,
      "regularize": 0.026483872905373573,
      "step": 260
    },
    {
      "dpo_loss": 0.6923092603683472,
      "epoch": 0.25037398630029134,
      "grad_norm": 66.24088589632522,
      "learning_rate": 9.980893514174179e-08,
      "logits": -2.2002015113830566,
      "logps": -92.85660552978516,
      "loss": 0.0214,
      "objective": 0.016592005267739296,
      "ranking_idealized": 0.5833333134651184,
      "ranking_idealized_expo": 0.5166666507720947,
      "ranking_simple": 0.5333333611488342,
      "regularize": 0.01658981665968895,
      "step": 265
    },
    {
      "dpo_loss": 0.69212406873703,
      "epoch": 0.2550980237776553,
      "grad_norm": 67.74325606253207,
      "learning_rate": 9.97712135115574e-08,
      "logits": -2.1255643367767334,
      "logps": -93.65802764892578,
      "loss": 0.0234,
      "objective": 0.029443560168147087,
      "ranking_idealized": 0.6333333253860474,
      "ranking_idealized_expo": 0.5666666626930237,
      "ranking_simple": 0.5666666626930237,
      "regularize": 0.02944253757596016,
      "step": 270
    },
    {
      "dpo_loss": 0.6937362551689148,
      "epoch": 0.2598220612550193,
      "grad_norm": 62.77300536920228,
      "learning_rate": 9.973010436321003e-08,
      "logits": -2.1865429878234863,
      "logps": -94.39256286621094,
      "loss": 0.0231,
      "objective": 0.024354156106710434,
      "ranking_idealized": 0.6000000238418579,
      "ranking_idealized_expo": 0.5166666507720947,
      "ranking_simple": 0.5166666507720947,
      "regularize": 0.024351568892598152,
      "step": 275
    },
    {
      "dpo_loss": 0.6910390853881836,
      "epoch": 0.2645460987323833,
      "grad_norm": 62.67099824882817,
      "learning_rate": 9.968561049466213e-08,
      "logits": -2.1923298835754395,
      "logps": -90.44979095458984,
      "loss": 0.0231,
      "objective": 0.020880402997136116,
      "ranking_idealized": 0.6166666746139526,
      "ranking_idealized_expo": 0.44999998807907104,
      "ranking_simple": 0.44999998807907104,
      "regularize": 0.02087988331913948,
      "step": 280
    },
    {
      "dpo_loss": 0.6930695176124573,
      "epoch": 0.26927013620974727,
      "grad_norm": 87.62776016079746,
      "learning_rate": 9.963773493424628e-08,
      "logits": -2.2007596492767334,
      "logps": -95.48867797851562,
      "loss": 0.0265,
      "objective": 0.028789160773158073,
      "ranking_idealized": 0.5166666507720947,
      "ranking_idealized_expo": 0.4000000059604645,
      "ranking_simple": 0.4166666567325592,
      "regularize": 0.028786195442080498,
      "step": 285
    },
    {
      "dpo_loss": 0.6869128346443176,
      "epoch": 0.27399417368711126,
      "grad_norm": 72.49597645901943,
      "learning_rate": 9.95864809404591e-08,
      "logits": -2.1374075412750244,
      "logps": -93.5144271850586,
      "loss": 0.0275,
      "objective": 0.026990260928869247,
      "ranking_idealized": 0.5833333134651184,
      "ranking_idealized_expo": 0.4833333194255829,
      "ranking_simple": 0.5333333611488342,
      "regularize": 0.026989614591002464,
      "step": 290
    },
    {
      "dpo_loss": 0.6920251250267029,
      "epoch": 0.27871821116447526,
      "grad_norm": 77.90764756547169,
      "learning_rate": 9.953185200173945e-08,
      "logits": -2.111963987350464,
      "logps": -90.939453125,
      "loss": 0.0274,
      "objective": 0.022320417687296867,
      "ranking_idealized": 0.5166666507720947,
      "ranking_idealized_expo": 0.4833333194255829,
      "ranking_simple": 0.4833333194255829,
      "regularize": 0.02231987938284874,
      "step": 295
    },
    {
      "dpo_loss": 0.6920836567878723,
      "epoch": 0.2834422486418392,
      "grad_norm": 68.10918361524686,
      "learning_rate": 9.947385183623097e-08,
      "logits": -2.213435173034668,
      "logps": -91.73210906982422,
      "loss": 0.0234,
      "objective": 0.026671167463064194,
      "ranking_idealized": 0.6166666746139526,
      "ranking_idealized_expo": 0.5166666507720947,
      "ranking_simple": 0.5166666507720947,
      "regularize": 0.026670336723327637,
      "step": 300
    },
    {
      "epoch": 0.2834422486418392,
      "eval_dpo_loss": 0.6931979060173035,
      "eval_logits": -1.993965744972229,
      "eval_logps": -98.33208465576172,
      "eval_loss": 0.01361795049160719,
      "eval_objective": 0.01397615671157837,
      "eval_ranking_idealized": 0.6022099256515503,
      "eval_ranking_idealized_expo": 0.5207182168960571,
      "eval_ranking_simple": 0.5165745615959167,
      "eval_regularize": 0.013974088244140148,
      "eval_runtime": 447.4565,
      "eval_samples_per_second": 12.94,
      "eval_steps_per_second": 3.236,
      "step": 300
    },
    {
      "dpo_loss": 0.68857741355896,
      "epoch": 0.2881662861192032,
      "grad_norm": 68.87756454021408,
      "learning_rate": 9.94124843915291e-08,
      "logits": -2.1138675212860107,
      "logps": -93.91316986083984,
      "loss": 0.0266,
      "objective": 0.03335392475128174,
      "ranking_idealized": 0.5833333134651184,
      "ranking_idealized_expo": 0.5166666507720947,
      "ranking_simple": 0.5166666507720947,
      "regularize": 0.03335336595773697,
      "step": 305
    },
    {
      "dpo_loss": 0.6887925863265991,
      "epoch": 0.2928903235965672,
      "grad_norm": 125.82123779225154,
      "learning_rate": 9.934775384441227e-08,
      "logits": -2.138413190841675,
      "logps": -90.38550567626953,
      "loss": 0.0265,
      "objective": 0.021200962364673615,
      "ranking_idealized": 0.550000011920929,
      "ranking_idealized_expo": 0.4833333194255829,
      "ranking_simple": 0.4833333194255829,
      "regularize": 0.02119968645274639,
      "step": 310
    },
    {
      "dpo_loss": 0.6924371123313904,
      "epoch": 0.2976143610739312,
      "grad_norm": 64.16350361165954,
      "learning_rate": 9.92796646005578e-08,
      "logits": -2.1671297550201416,
      "logps": -93.71112823486328,
      "loss": 0.0255,
      "objective": 0.027043061330914497,
      "ranking_idealized": 0.6499999761581421,
      "ranking_idealized_expo": 0.6333333253860474,
      "ranking_simple": 0.6499999761581421,
      "regularize": 0.027042122557759285,
      "step": 315
    },
    {
      "dpo_loss": 0.6908622980117798,
      "epoch": 0.3023383985512952,
      "grad_norm": 73.45877351161931,
      "learning_rate": 9.920822129424189e-08,
      "logits": -2.0810954570770264,
      "logps": -93.48076629638672,
      "loss": 0.0264,
      "objective": 0.03161174803972244,
      "ranking_idealized": 0.6000000238418579,
      "ranking_idealized_expo": 0.4833333194255829,
      "ranking_simple": 0.4833333194255829,
      "regularize": 0.03161115199327469,
      "step": 320
    },
    {
      "dpo_loss": 0.6928619742393494,
      "epoch": 0.3070624360286592,
      "grad_norm": 73.35574157285905,
      "learning_rate": 9.913342878802423e-08,
      "logits": -2.102128505706787,
      "logps": -92.2950439453125,
      "loss": 0.0286,
      "objective": 0.037009891122579575,
      "ranking_idealized": 0.6499999761581421,
      "ranking_idealized_expo": 0.5833333134651184,
      "ranking_simple": 0.5666666626930237,
      "regularize": 0.03700947389006615,
      "step": 325
    },
    {
      "dpo_loss": 0.6908413767814636,
      "epoch": 0.3117864735060231,
      "grad_norm": 79.4503768102059,
      "learning_rate": 9.90552921724171e-08,
      "logits": -2.2076501846313477,
      "logps": -91.73526763916016,
      "loss": 0.0276,
      "objective": 0.020319262519478798,
      "ranking_idealized": 0.6000000238418579,
      "ranking_idealized_expo": 0.46666666865348816,
      "ranking_simple": 0.46666666865348816,
      "regularize": 0.020317750051617622,
      "step": 330
    },
    {
      "dpo_loss": 0.6880075335502625,
      "epoch": 0.3165105109833871,
      "grad_norm": 82.61039711327662,
      "learning_rate": 9.897381676553888e-08,
      "logits": -2.177678346633911,
      "logps": -91.5321044921875,
      "loss": 0.0261,
      "objective": 0.027510004118084908,
      "ranking_idealized": 0.5833333134651184,
      "ranking_idealized_expo": 0.550000011920929,
      "ranking_simple": 0.5333333611488342,
      "regularize": 0.027509503066539764,
      "step": 335
    },
    {
      "dpo_loss": 0.6954006552696228,
      "epoch": 0.3212345484607511,
      "grad_norm": 69.91545242214347,
      "learning_rate": 9.888900811275203e-08,
      "logits": -2.1840174198150635,
      "logps": -92.2385025024414,
      "loss": 0.0272,
      "objective": 0.037196170538663864,
      "ranking_idealized": 0.5666666626930237,
      "ranking_idealized_expo": 0.46666666865348816,
      "ranking_simple": 0.46666666865348816,
      "regularize": 0.037195660173892975,
      "step": 340
    },
    {
      "dpo_loss": 0.6906254887580872,
      "epoch": 0.3259585859381151,
      "grad_norm": 65.03933644888008,
      "learning_rate": 9.880087198628577e-08,
      "logits": -2.153885841369629,
      "logps": -91.60904693603516,
      "loss": 0.0246,
      "objective": 0.0248491782695055,
      "ranking_idealized": 0.5666666626930237,
      "ranking_idealized_expo": 0.5,
      "ranking_simple": 0.5,
      "regularize": 0.024848194792866707,
      "step": 345
    },
    {
      "dpo_loss": 0.6858618855476379,
      "epoch": 0.3306826234154791,
      "grad_norm": 65.04231679146841,
      "learning_rate": 9.870941438484314e-08,
      "logits": -2.1364102363586426,
      "logps": -89.71781921386719,
      "loss": 0.0262,
      "objective": 0.026014180853962898,
      "ranking_idealized": 0.550000011920929,
      "ranking_idealized_expo": 0.5,
      "ranking_simple": 0.5166666507720947,
      "regularize": 0.026013409718871117,
      "step": 350
    },
    {
      "epoch": 0.3306826234154791,
      "eval_dpo_loss": 0.6925798058509827,
      "eval_logits": -1.9947079420089722,
      "eval_logps": -98.34574890136719,
      "eval_loss": 0.017829304561018944,
      "eval_objective": 0.018099796026945114,
      "eval_ranking_idealized": 0.6022099256515503,
      "eval_ranking_idealized_expo": 0.5207182168960571,
      "eval_ranking_simple": 0.5200276374816895,
      "eval_regularize": 0.018098480999469757,
      "eval_runtime": 446.968,
      "eval_samples_per_second": 12.954,
      "eval_steps_per_second": 3.24,
      "step": 350
    },
    {
      "dpo_loss": 0.6954938173294067,
      "epoch": 0.3354066608928431,
      "grad_norm": 80.69779694329567,
      "learning_rate": 9.861464153319269e-08,
      "logits": -2.129030466079712,
      "logps": -93.99476623535156,
      "loss": 0.0296,
      "objective": 0.03383675962686539,
      "ranking_idealized": 0.6499999761581421,
      "ranking_idealized_expo": 0.5666666626930237,
      "ranking_simple": 0.550000011920929,
      "regularize": 0.03383495658636093,
      "step": 355
    },
    {
      "dpo_loss": 0.6874939799308777,
      "epoch": 0.3401306983702071,
      "grad_norm": 76.1212296270761,
      "learning_rate": 9.85165598817449e-08,
      "logits": -2.1060662269592285,
      "logps": -94.81258392333984,
      "loss": 0.0314,
      "objective": 0.03174449875950813,
      "ranking_idealized": 0.5166666507720947,
      "ranking_idealized_expo": 0.5166666507720947,
      "ranking_simple": 0.5166666507720947,
      "regularize": 0.031743988394737244,
      "step": 360
    },
    {
      "dpo_loss": 0.6876952648162842,
      "epoch": 0.34485473584757104,
      "grad_norm": 63.17330963799499,
      "learning_rate": 9.841517610611307e-08,
      "logits": -2.223184823989868,
      "logps": -94.34845733642578,
      "loss": 0.0271,
      "objective": 0.03227417171001434,
      "ranking_idealized": 0.550000011920929,
      "ranking_idealized_expo": 0.46666666865348816,
      "ranking_simple": 0.4833333194255829,
      "regularize": 0.03227332606911659,
      "step": 365
    },
    {
      "dpo_loss": 0.6924201846122742,
      "epoch": 0.34957877332493503,
      "grad_norm": 70.25439806775886,
      "learning_rate": 9.831049710665904e-08,
      "logits": -2.153981924057007,
      "logps": -92.24577331542969,
      "loss": 0.0267,
      "objective": 0.03014095313847065,
      "ranking_idealized": 0.550000011920929,
      "ranking_idealized_expo": 0.46666666865348816,
      "ranking_simple": 0.44999998807907104,
      "regularize": 0.030140016227960587,
      "step": 370
    },
    {
      "dpo_loss": 0.6881352066993713,
      "epoch": 0.35430281080229903,
      "grad_norm": 67.1301497619948,
      "learning_rate": 9.820253000802345e-08,
      "logits": -2.1567375659942627,
      "logps": -90.48641204833984,
      "loss": 0.029,
      "objective": 0.034893397241830826,
      "ranking_idealized": 0.5333333611488342,
      "ranking_idealized_expo": 0.46666666865348816,
      "ranking_simple": 0.4833333194255829,
      "regularize": 0.034892160445451736,
      "step": 375
    },
    {
      "dpo_loss": 0.6959127187728882,
      "epoch": 0.359026848279663,
      "grad_norm": 65.92204147729184,
      "learning_rate": 9.809128215864095e-08,
      "logits": -2.1123626232147217,
      "logps": -90.6343002319336,
      "loss": 0.0274,
      "objective": 0.02818784862756729,
      "ranking_idealized": 0.6166666746139526,
      "ranking_idealized_expo": 0.5833333134651184,
      "ranking_simple": 0.5833333134651184,
      "regularize": 0.028187256306409836,
      "step": 380
    },
    {
      "dpo_loss": 0.6918838620185852,
      "epoch": 0.363750885757027,
      "grad_norm": 78.02594257500782,
      "learning_rate": 9.797676113023989e-08,
      "logits": -2.1536900997161865,
      "logps": -91.86727905273438,
      "loss": 0.0271,
      "objective": 0.024834012612700462,
      "ranking_idealized": 0.6166666746139526,
      "ranking_idealized_expo": 0.5166666507720947,
      "ranking_simple": 0.5166666507720947,
      "regularize": 0.024833252653479576,
      "step": 385
    },
    {
      "dpo_loss": 0.6891117095947266,
      "epoch": 0.368474923234391,
      "grad_norm": 77.62275101385076,
      "learning_rate": 9.785897471732711e-08,
      "logits": -2.220367193222046,
      "logps": -93.69184875488281,
      "loss": 0.0315,
      "objective": 0.04405975714325905,
      "ranking_idealized": 0.6166666746139526,
      "ranking_idealized_expo": 0.550000011920929,
      "ranking_simple": 0.5333333611488342,
      "regularize": 0.04405748099088669,
      "step": 390
    },
    {
      "dpo_loss": 0.6931707859039307,
      "epoch": 0.37319896071175496,
      "grad_norm": 70.95977856029823,
      "learning_rate": 9.773793093665739e-08,
      "logits": -2.188248872756958,
      "logps": -90.29833984375,
      "loss": 0.0304,
      "objective": 0.0337492860853672,
      "ranking_idealized": 0.6000000238418579,
      "ranking_idealized_expo": 0.5,
      "ranking_simple": 0.5,
      "regularize": 0.03374841436743736,
      "step": 395
    },
    {
      "dpo_loss": 0.6871830224990845,
      "epoch": 0.37792299818911895,
      "grad_norm": 71.69739058358107,
      "learning_rate": 9.76136380266878e-08,
      "logits": -2.155177354812622,
      "logps": -92.64527130126953,
      "loss": 0.0315,
      "objective": 0.031953115016222,
      "ranking_idealized": 0.6000000238418579,
      "ranking_idealized_expo": 0.5,
      "ranking_simple": 0.5,
      "regularize": 0.03195144981145859,
      "step": 400
    },
    {
      "epoch": 0.37792299818911895,
      "eval_dpo_loss": 0.6926193833351135,
      "eval_logits": -1.9940829277038574,
      "eval_logps": -98.11278533935547,
      "eval_loss": 0.01653479039669037,
      "eval_objective": 0.01635783165693283,
      "eval_ranking_idealized": 0.6022099256515503,
      "eval_ranking_idealized_expo": 0.5207182168960571,
      "eval_ranking_simple": 0.5200276374816895,
      "eval_regularize": 0.01635659858584404,
      "eval_runtime": 446.4503,
      "eval_samples_per_second": 12.969,
      "eval_steps_per_second": 3.243,
      "step": 400
    },
    {
      "dpo_loss": 0.6941009759902954,
      "epoch": 0.38264703566648295,
      "grad_norm": 62.224154885036064,
      "learning_rate": 9.748610444701694e-08,
      "logits": -2.1617021560668945,
      "logps": -90.76243591308594,
      "loss": 0.0293,
      "objective": 0.02944065071642399,
      "ranking_idealized": 0.6333333253860474,
      "ranking_idealized_expo": 0.550000011920929,
      "ranking_simple": 0.550000011920929,
      "regularize": 0.02944008819758892,
      "step": 405
    },
    {
      "dpo_loss": 0.6934362649917603,
      "epoch": 0.38737107314384694,
      "grad_norm": 71.86989684775978,
      "learning_rate": 9.735533887780928e-08,
      "logits": -2.1968331336975098,
      "logps": -94.97209930419922,
      "loss": 0.0284,
      "objective": 0.02905886620283127,
      "ranking_idealized": 0.46666666865348816,
      "ranking_idealized_expo": 0.4000000059604645,
      "ranking_simple": 0.4000000059604645,
      "regularize": 0.029057972133159637,
      "step": 410
    },
    {
      "dpo_loss": 0.6933155059814453,
      "epoch": 0.39209511062121094,
      "grad_norm": 69.39777880403724,
      "learning_rate": 9.722135021920426e-08,
      "logits": -2.1606533527374268,
      "logps": -90.09014129638672,
      "loss": 0.0288,
      "objective": 0.025797124952077866,
      "ranking_idealized": 0.6333333253860474,
      "ranking_idealized_expo": 0.4833333194255829,
      "ranking_simple": 0.4833333194255829,
      "regularize": 0.025796448811888695,
      "step": 415
    },
    {
      "dpo_loss": 0.6879637837409973,
      "epoch": 0.39681914809857494,
      "grad_norm": 65.66563021960218,
      "learning_rate": 9.708414759071057e-08,
      "logits": -2.192812204360962,
      "logps": -90.60978698730469,
      "loss": 0.0293,
      "objective": 0.028554469347000122,
      "ranking_idealized": 0.5333333611488342,
      "ranking_idealized_expo": 0.4333333373069763,
      "ranking_simple": 0.4333333373069763,
      "regularize": 0.028553970158100128,
      "step": 420
    },
    {
      "dpo_loss": 0.6905270218849182,
      "epoch": 0.4015431855759389,
      "grad_norm": 73.81480159017963,
      "learning_rate": 9.694374033058549e-08,
      "logits": -2.1698479652404785,
      "logps": -92.80254364013672,
      "loss": 0.03,
      "objective": 0.032098546624183655,
      "ranking_idealized": 0.5666666626930237,
      "ranking_idealized_expo": 0.550000011920929,
      "ranking_simple": 0.550000011920929,
      "regularize": 0.0320979543030262,
      "step": 425
    },
    {
      "dpo_loss": 0.6938761472702026,
      "epoch": 0.4062672230533029,
      "grad_norm": 76.38130749325458,
      "learning_rate": 9.680013799519926e-08,
      "logits": -2.2929608821868896,
      "logps": -92.20008087158203,
      "loss": 0.0332,
      "objective": 0.03666527569293976,
      "ranking_idealized": 0.5833333134651184,
      "ranking_idealized_expo": 0.5333333611488342,
      "ranking_simple": 0.5333333611488342,
      "regularize": 0.036664899438619614,
      "step": 430
    },
    {
      "dpo_loss": 0.6846203207969666,
      "epoch": 0.41099126053066687,
      "grad_norm": 62.016252142417606,
      "learning_rate": 9.665335035838468e-08,
      "logits": -2.1209442615509033,
      "logps": -91.99024963378906,
      "loss": 0.0308,
      "objective": 0.04143450781702995,
      "ranking_idealized": 0.7333333492279053,
      "ranking_idealized_expo": 0.6166666746139526,
      "ranking_simple": 0.6166666746139526,
      "regularize": 0.04143408685922623,
      "step": 435
    },
    {
      "dpo_loss": 0.692919909954071,
      "epoch": 0.41571529800803086,
      "grad_norm": 83.01421114100998,
      "learning_rate": 9.650338741077189e-08,
      "logits": -2.1818275451660156,
      "logps": -88.64667510986328,
      "loss": 0.0282,
      "objective": 0.020624225959181786,
      "ranking_idealized": 0.5833333134651184,
      "ranking_idealized_expo": 0.550000011920929,
      "ranking_simple": 0.5666666626930237,
      "regularize": 0.02062271721661091,
      "step": 440
    },
    {
      "dpo_loss": 0.6882209777832031,
      "epoch": 0.42043933548539486,
      "grad_norm": 69.91926112096503,
      "learning_rate": 9.635025935910839e-08,
      "logits": -2.078962564468384,
      "logps": -94.8196792602539,
      "loss": 0.03,
      "objective": 0.029924126341938972,
      "ranking_idealized": 0.6499999761581421,
      "ranking_idealized_expo": 0.6000000238418579,
      "ranking_simple": 0.6000000238418579,
      "regularize": 0.029923679307103157,
      "step": 445
    },
    {
      "dpo_loss": 0.6905581951141357,
      "epoch": 0.42516337296275886,
      "grad_norm": 68.70387033094245,
      "learning_rate": 9.619397662556434e-08,
      "logits": -2.1093952655792236,
      "logps": -90.6502914428711,
      "loss": 0.0294,
      "objective": 0.028780171647667885,
      "ranking_idealized": 0.5833333134651184,
      "ranking_idealized_expo": 0.44999998807907104,
      "ranking_simple": 0.46666666865348816,
      "regularize": 0.028779106214642525,
      "step": 450
    },
    {
      "epoch": 0.42516337296275886,
      "eval_dpo_loss": 0.6923617124557495,
      "eval_logits": -1.994999885559082,
      "eval_logps": -98.37866973876953,
      "eval_loss": 0.014490882866084576,
      "eval_objective": 0.014800351113080978,
      "eval_ranking_idealized": 0.6022099256515503,
      "eval_ranking_idealized_expo": 0.5207182168960571,
      "eval_ranking_simple": 0.5186464190483093,
      "eval_regularize": 0.014799040742218494,
      "eval_runtime": 444.0476,
      "eval_samples_per_second": 13.039,
      "eval_steps_per_second": 3.261,
      "step": 450
    },
    {
      "dpo_loss": 0.6909880042076111,
      "epoch": 0.42988741044012285,
      "grad_norm": 68.38544757200873,
      "learning_rate": 9.60345498470232e-08,
      "logits": -2.158226728439331,
      "logps": -90.17542266845703,
      "loss": 0.0255,
      "objective": 0.02339431643486023,
      "ranking_idealized": 0.5833333134651184,
      "ranking_idealized_expo": 0.550000011920929,
      "ranking_simple": 0.550000011920929,
      "regularize": 0.023393774405121803,
      "step": 455
    },
    {
      "dpo_loss": 0.6882848739624023,
      "epoch": 0.4346114479174868,
      "grad_norm": 60.565560674058744,
      "learning_rate": 9.58719898743578e-08,
      "logits": -2.1875061988830566,
      "logps": -93.89222717285156,
      "loss": 0.0289,
      "objective": 0.03423256427049637,
      "ranking_idealized": 0.6166666746139526,
      "ranking_idealized_expo": 0.5333333611488342,
      "ranking_simple": 0.5333333611488342,
      "regularize": 0.034231893718242645,
      "step": 460
    },
    {
      "dpo_loss": 0.6911517977714539,
      "epoch": 0.4393354853948508,
      "grad_norm": 68.08241104925199,
      "learning_rate": 9.57063077716918e-08,
      "logits": -2.1419482231140137,
      "logps": -94.25173950195312,
      "loss": 0.0304,
      "objective": 0.02857878990471363,
      "ranking_idealized": 0.699999988079071,
      "ranking_idealized_expo": 0.6666666865348816,
      "ranking_simple": 0.6666666865348816,
      "regularize": 0.028578022494912148,
      "step": 465
    },
    {
      "dpo_loss": 0.6899906396865845,
      "epoch": 0.4440595228722148,
      "grad_norm": 75.2887054128907,
      "learning_rate": 9.553751481564658e-08,
      "logits": -2.0578720569610596,
      "logps": -88.11711120605469,
      "loss": 0.0318,
      "objective": 0.02341555431485176,
      "ranking_idealized": 0.6499999761581421,
      "ranking_idealized_expo": 0.6166666746139526,
      "ranking_simple": 0.6166666746139526,
      "regularize": 0.02341252751648426,
      "step": 470
    },
    {
      "dpo_loss": 0.6940017938613892,
      "epoch": 0.4487835603495788,
      "grad_norm": 61.894078510860375,
      "learning_rate": 9.536562249457386e-08,
      "logits": -2.1432507038116455,
      "logps": -91.78999328613281,
      "loss": 0.0276,
      "objective": 0.02927049808204174,
      "ranking_idealized": 0.5666666626930237,
      "ranking_idealized_expo": 0.5166666507720947,
      "ranking_simple": 0.5333333611488342,
      "regularize": 0.029269874095916748,
      "step": 475
    },
    {
      "dpo_loss": 0.6929585933685303,
      "epoch": 0.4535075978269428,
      "grad_norm": 69.75674817980894,
      "learning_rate": 9.51906425077736e-08,
      "logits": -2.1212713718414307,
      "logps": -91.61197662353516,
      "loss": 0.0303,
      "objective": 0.028644824400544167,
      "ranking_idealized": 0.5333333611488342,
      "ranking_idealized_expo": 0.4333333373069763,
      "ranking_simple": 0.44999998807907104,
      "regularize": 0.028642630204558372,
      "step": 480
    },
    {
      "dpo_loss": 0.6894620656967163,
      "epoch": 0.4582316353043068,
      "grad_norm": 68.06555771230744,
      "learning_rate": 9.501258676469798e-08,
      "logits": -2.2252414226531982,
      "logps": -92.49252319335938,
      "loss": 0.0273,
      "objective": 0.029991615563631058,
      "ranking_idealized": 0.6333333253860474,
      "ranking_idealized_expo": 0.6000000238418579,
      "ranking_simple": 0.6000000238418579,
      "regularize": 0.02999037504196167,
      "step": 485
    },
    {
      "dpo_loss": 0.6851038336753845,
      "epoch": 0.4629556727816707,
      "grad_norm": 68.81350143691881,
      "learning_rate": 9.483146738414056e-08,
      "logits": -2.1528584957122803,
      "logps": -93.01960754394531,
      "loss": 0.0298,
      "objective": 0.03543411195278168,
      "ranking_idealized": 0.5666666626930237,
      "ranking_idealized_expo": 0.4833333194255829,
      "ranking_simple": 0.5166666507720947,
      "regularize": 0.03543229401111603,
      "step": 490
    },
    {
      "dpo_loss": 0.6880256533622742,
      "epoch": 0.4676797102590347,
      "grad_norm": 63.11314302096046,
      "learning_rate": 9.46472966934116e-08,
      "logits": -2.1136722564697266,
      "logps": -90.93292999267578,
      "loss": 0.0346,
      "objective": 0.03396356850862503,
      "ranking_idealized": 0.6499999761581421,
      "ranking_idealized_expo": 0.5,
      "ranking_simple": 0.5166666507720947,
      "regularize": 0.03396301716566086,
      "step": 495
    },
    {
      "dpo_loss": 0.6870742440223694,
      "epoch": 0.4724037477363987,
      "grad_norm": 73.04815070979407,
      "learning_rate": 9.446008722749906e-08,
      "logits": -2.2441928386688232,
      "logps": -95.81112670898438,
      "loss": 0.032,
      "objective": 0.026291660964488983,
      "ranking_idealized": 0.6333333253860474,
      "ranking_idealized_expo": 0.5833333134651184,
      "ranking_simple": 0.6000000238418579,
      "regularize": 0.026291247457265854,
      "step": 500
    },
    {
      "epoch": 0.4724037477363987,
      "eval_dpo_loss": 0.6924601793289185,
      "eval_logits": -1.9919774532318115,
      "eval_logps": -98.64570617675781,
      "eval_loss": 0.013912476599216461,
      "eval_objective": 0.0139460489153862,
      "eval_ranking_idealized": 0.6022099256515503,
      "eval_ranking_idealized_expo": 0.5207182168960571,
      "eval_ranking_simple": 0.519336998462677,
      "eval_regularize": 0.013944561593234539,
      "eval_runtime": 444.8021,
      "eval_samples_per_second": 13.017,
      "eval_steps_per_second": 3.255,
      "step": 500
    },
    {
      "dpo_loss": 0.6906213164329529,
      "epoch": 0.4771277852137627,
      "grad_norm": 75.29236969807971,
      "learning_rate": 9.426985172821529e-08,
      "logits": -2.225041151046753,
      "logps": -90.76871490478516,
      "loss": 0.0313,
      "objective": 0.034964669495821,
      "ranking_idealized": 0.550000011920929,
      "ranking_idealized_expo": 0.5,
      "ranking_simple": 0.5,
      "regularize": 0.03496433049440384,
      "step": 505
    },
    {
      "dpo_loss": 0.6928242444992065,
      "epoch": 0.4818518226911267,
      "grad_norm": 71.72745918834562,
      "learning_rate": 9.407660314333001e-08,
      "logits": -2.0290334224700928,
      "logps": -92.85369110107422,
      "loss": 0.0322,
      "objective": 0.039766810834407806,
      "ranking_idealized": 0.7166666388511658,
      "ranking_idealized_expo": 0.5666666626930237,
      "ranking_simple": 0.5833333134651184,
      "regularize": 0.03976641967892647,
      "step": 510
    },
    {
      "dpo_loss": 0.6923685073852539,
      "epoch": 0.4865758601684907,
      "grad_norm": 61.131599755380016,
      "learning_rate": 9.388035462568891e-08,
      "logits": -2.147352933883667,
      "logps": -91.90682220458984,
      "loss": 0.0331,
      "objective": 0.035778481513261795,
      "ranking_idealized": 0.6666666865348816,
      "ranking_idealized_expo": 0.6000000238418579,
      "ranking_simple": 0.5666666626930237,
      "regularize": 0.035777974873781204,
      "step": 515
    },
    {
      "dpo_loss": 0.6962333917617798,
      "epoch": 0.49129989764585463,
      "grad_norm": 65.42531713674678,
      "learning_rate": 9.368111953231848e-08,
      "logits": -2.1052534580230713,
      "logps": -92.43571472167969,
      "loss": 0.0302,
      "objective": 0.030195049941539764,
      "ranking_idealized": 0.6499999761581421,
      "ranking_idealized_expo": 0.6166666746139526,
      "ranking_simple": 0.6166666746139526,
      "regularize": 0.030194593593478203,
      "step": 520
    },
    {
      "dpo_loss": 0.6958824396133423,
      "epoch": 0.49602393512321863,
      "grad_norm": 68.86518401985903,
      "learning_rate": 9.347891142351692e-08,
      "logits": -2.1327033042907715,
      "logps": -95.017578125,
      "loss": 0.0343,
      "objective": 0.033561404794454575,
      "ranking_idealized": 0.5166666507720947,
      "ranking_idealized_expo": 0.46666666865348816,
      "ranking_simple": 0.44999998807907104,
      "regularize": 0.033560872077941895,
      "step": 525
    },
    {
      "dpo_loss": 0.692583441734314,
      "epoch": 0.5007479726005827,
      "grad_norm": 68.03560111972747,
      "learning_rate": 9.327374406193124e-08,
      "logits": -2.1641759872436523,
      "logps": -92.0415267944336,
      "loss": 0.032,
      "objective": 0.03345762938261032,
      "ranking_idealized": 0.6666666865348816,
      "ranking_idealized_expo": 0.6000000238418579,
      "ranking_simple": 0.6000000238418579,
      "regularize": 0.03345634788274765,
      "step": 530
    },
    {
      "dpo_loss": 0.6896530389785767,
      "epoch": 0.5054720100779466,
      "grad_norm": 66.90019999237873,
      "learning_rate": 9.306563141162044e-08,
      "logits": -2.1231565475463867,
      "logps": -91.51903533935547,
      "loss": 0.0298,
      "objective": 0.03302415460348129,
      "ranking_idealized": 0.6666666865348816,
      "ranking_idealized_expo": 0.5166666507720947,
      "ranking_simple": 0.5166666507720947,
      "regularize": 0.03302332013845444,
      "step": 535
    },
    {
      "dpo_loss": 0.6914986371994019,
      "epoch": 0.5101960475553106,
      "grad_norm": 58.738988827079176,
      "learning_rate": 9.285458763710523e-08,
      "logits": -2.147346019744873,
      "logps": -93.07068634033203,
      "loss": 0.0341,
      "objective": 0.03528103977441788,
      "ranking_idealized": 0.5666666626930237,
      "ranking_idealized_expo": 0.4833333194255829,
      "ranking_simple": 0.4833333194255829,
      "regularize": 0.035279832780361176,
      "step": 540
    },
    {
      "dpo_loss": 0.6926673650741577,
      "epoch": 0.5149200850326746,
      "grad_norm": 65.83406826541673,
      "learning_rate": 9.264062710240386e-08,
      "logits": -2.1894426345825195,
      "logps": -97.09349822998047,
      "loss": 0.0276,
      "objective": 0.029730303213000298,
      "ranking_idealized": 0.5666666626930237,
      "ranking_idealized_expo": 0.46666666865348816,
      "ranking_simple": 0.46666666865348816,
      "regularize": 0.02972940169274807,
      "step": 545
    },
    {
      "dpo_loss": 0.6896089315414429,
      "epoch": 0.5196441225100386,
      "grad_norm": 80.74824222454775,
      "learning_rate": 9.242376437005448e-08,
      "logits": -2.1549692153930664,
      "logps": -93.35411834716797,
      "loss": 0.0314,
      "objective": 0.0288882777094841,
      "ranking_idealized": 0.6000000238418579,
      "ranking_idealized_expo": 0.5166666507720947,
      "ranking_simple": 0.5166666507720947,
      "regularize": 0.028887782245874405,
      "step": 550
    },
    {
      "epoch": 0.5196441225100386,
      "eval_dpo_loss": 0.6926965713500977,
      "eval_logits": -1.9942920207977295,
      "eval_logps": -98.96892547607422,
      "eval_loss": 0.013568516820669174,
      "eval_objective": 0.013540062122046947,
      "eval_ranking_idealized": 0.6022099256515503,
      "eval_ranking_idealized_expo": 0.5207182168960571,
      "eval_ranking_simple": 0.5186464190483093,
      "eval_regularize": 0.013538442552089691,
      "eval_runtime": 444.65,
      "eval_samples_per_second": 13.021,
      "eval_steps_per_second": 3.256,
      "step": 550
    },
    {
      "dpo_loss": 0.6818323135375977,
      "epoch": 0.5243681599874026,
      "grad_norm": 71.45675055284568,
      "learning_rate": 9.22040142001241e-08,
      "logits": -2.1764817237854004,
      "logps": -92.4581298828125,
      "loss": 0.0336,
      "objective": 0.04190651327371597,
      "ranking_idealized": 0.6166666746139526,
      "ranking_idealized_expo": 0.5333333611488342,
      "ranking_simple": 0.550000011920929,
      "regularize": 0.04190612956881523,
      "step": 555
    },
    {
      "dpo_loss": 0.6884815692901611,
      "epoch": 0.5290921974647665,
      "grad_norm": 72.10264430141908,
      "learning_rate": 9.198139154920388e-08,
      "logits": -2.2008354663848877,
      "logps": -90.6949234008789,
      "loss": 0.0344,
      "objective": 0.034483686089515686,
      "ranking_idealized": 0.5333333611488342,
      "ranking_idealized_expo": 0.46666666865348816,
      "ranking_simple": 0.46666666865348816,
      "regularize": 0.03448285162448883,
      "step": 560
    },
    {
      "dpo_loss": 0.6909436583518982,
      "epoch": 0.5338162349421306,
      "grad_norm": 60.56883204825771,
      "learning_rate": 9.175591156939118e-08,
      "logits": -2.1834826469421387,
      "logps": -94.38992309570312,
      "loss": 0.03,
      "objective": 0.02786482684314251,
      "ranking_idealized": 0.6333333253860474,
      "ranking_idealized_expo": 0.5666666626930237,
      "ranking_simple": 0.5666666626930237,
      "regularize": 0.02786322310566902,
      "step": 565
    },
    {
      "dpo_loss": 0.6896558403968811,
      "epoch": 0.5385402724194945,
      "grad_norm": 63.27876500292638,
      "learning_rate": 9.152758960725829e-08,
      "logits": -2.0850472450256348,
      "logps": -90.94063568115234,
      "loss": 0.0305,
      "objective": 0.03306278958916664,
      "ranking_idealized": 0.6000000238418579,
      "ranking_idealized_expo": 0.5333333611488342,
      "ranking_simple": 0.5166666507720947,
      "regularize": 0.03306075185537338,
      "step": 570
    },
    {
      "dpo_loss": 0.6910984516143799,
      "epoch": 0.5432643098968585,
      "grad_norm": 64.97441175201224,
      "learning_rate": 9.129644120280797e-08,
      "logits": -2.215700387954712,
      "logps": -93.2086181640625,
      "loss": 0.0363,
      "objective": 0.03730526939034462,
      "ranking_idealized": 0.6166666746139526,
      "ranking_idealized_expo": 0.46666666865348816,
      "ranking_simple": 0.46666666865348816,
      "regularize": 0.03730436787009239,
      "step": 575
    },
    {
      "dpo_loss": 0.6901772618293762,
      "epoch": 0.5479883473742225,
      "grad_norm": 60.84920518500226,
      "learning_rate": 9.106248208841567e-08,
      "logits": -2.077465534210205,
      "logps": -89.92863464355469,
      "loss": 0.0301,
      "objective": 0.029211556538939476,
      "ranking_idealized": 0.5166666507720947,
      "ranking_idealized_expo": 0.46666666865348816,
      "ranking_simple": 0.46666666865348816,
      "regularize": 0.02920910157263279,
      "step": 580
    },
    {
      "dpo_loss": 0.6912521123886108,
      "epoch": 0.5527123848515865,
      "grad_norm": 61.38401518242899,
      "learning_rate": 9.082572818775884e-08,
      "logits": -2.0964841842651367,
      "logps": -96.6317138671875,
      "loss": 0.0311,
      "objective": 0.0291235763579607,
      "ranking_idealized": 0.550000011920929,
      "ranking_idealized_expo": 0.5166666507720947,
      "ranking_simple": 0.5166666507720947,
      "regularize": 0.029122162610292435,
      "step": 585
    },
    {
      "dpo_loss": 0.6879535913467407,
      "epoch": 0.5574364223289505,
      "grad_norm": 70.97177434877489,
      "learning_rate": 9.058619561473306e-08,
      "logits": -2.1359400749206543,
      "logps": -91.66080474853516,
      "loss": 0.0309,
      "objective": 0.0273025743663311,
      "ranking_idealized": 0.550000011920929,
      "ranking_idealized_expo": 0.5166666507720947,
      "ranking_simple": 0.5166666507720947,
      "regularize": 0.02730202116072178,
      "step": 590
    },
    {
      "dpo_loss": 0.69169682264328,
      "epoch": 0.5621604598063145,
      "grad_norm": 66.67475961321956,
      "learning_rate": 9.034390067235538e-08,
      "logits": -2.122257947921753,
      "logps": -93.28813934326172,
      "loss": 0.0292,
      "objective": 0.03189357370138168,
      "ranking_idealized": 0.5833333134651184,
      "ranking_idealized_expo": 0.5,
      "ranking_simple": 0.5,
      "regularize": 0.03189240023493767,
      "step": 595
    },
    {
      "dpo_loss": 0.692051351070404,
      "epoch": 0.5668844972836784,
      "grad_norm": 66.82237885750858,
      "learning_rate": 9.009885985165465e-08,
      "logits": -2.1968979835510254,
      "logps": -91.0505599975586,
      "loss": 0.0311,
      "objective": 0.027313487604260445,
      "ranking_idealized": 0.6000000238418579,
      "ranking_idealized_expo": 0.5166666507720947,
      "ranking_simple": 0.5166666507720947,
      "regularize": 0.027312906458973885,
      "step": 600
    },
    {
      "epoch": 0.5668844972836784,
      "eval_dpo_loss": 0.6925215721130371,
      "eval_logits": -1.9967907667160034,
      "eval_logps": -98.12234497070312,
      "eval_loss": 0.014187943190336227,
      "eval_objective": 0.014395096339285374,
      "eval_ranking_idealized": 0.6022099256515503,
      "eval_ranking_idealized_expo": 0.5207182168960571,
      "eval_ranking_simple": 0.5186464190483093,
      "eval_regularize": 0.014393283054232597,
      "eval_runtime": 445.7504,
      "eval_samples_per_second": 12.989,
      "eval_steps_per_second": 3.248,
      "step": 600
    },
    {
      "dpo_loss": 0.6929703950881958,
      "epoch": 0.5716085347610425,
      "grad_norm": 64.74275314805836,
      "learning_rate": 8.985108983054912e-08,
      "logits": -2.0810482501983643,
      "logps": -92.1054916381836,
      "loss": 0.0343,
      "objective": 0.03585705906152725,
      "ranking_idealized": 0.5,
      "ranking_idealized_expo": 0.4166666567325592,
      "ranking_simple": 0.4166666567325592,
      "regularize": 0.03585506230592728,
      "step": 605
    },
    {
      "dpo_loss": 0.6893709897994995,
      "epoch": 0.5763325722384064,
      "grad_norm": 63.69006739310455,
      "learning_rate": 8.960060747271137e-08,
      "logits": -2.1485848426818848,
      "logps": -92.45893096923828,
      "loss": 0.0292,
      "objective": 0.03595684841275215,
      "ranking_idealized": 0.5666666626930237,
      "ranking_idealized_expo": 0.4833333194255829,
      "ranking_simple": 0.4833333194255829,
      "regularize": 0.03595583513379097,
      "step": 610
    },
    {
      "dpo_loss": 0.6894236207008362,
      "epoch": 0.5810566097157704,
      "grad_norm": 63.68171123239859,
      "learning_rate": 8.934742982642041e-08,
      "logits": -2.2213053703308105,
      "logps": -92.00927734375,
      "loss": 0.0289,
      "objective": 0.030653396621346474,
      "ranking_idealized": 0.5833333134651184,
      "ranking_idealized_expo": 0.4333333373069763,
      "ranking_simple": 0.44999998807907104,
      "regularize": 0.030652187764644623,
      "step": 615
    },
    {
      "dpo_loss": 0.6907939314842224,
      "epoch": 0.5857806471931344,
      "grad_norm": 64.38726305381529,
      "learning_rate": 8.90915741234015e-08,
      "logits": -2.22101092338562,
      "logps": -93.67798614501953,
      "loss": 0.0293,
      "objective": 0.028055744245648384,
      "ranking_idealized": 0.6499999761581421,
      "ranking_idealized_expo": 0.5666666626930237,
      "ranking_simple": 0.5666666626930237,
      "regularize": 0.028055019676685333,
      "step": 620
    },
    {
      "dpo_loss": 0.6883565783500671,
      "epoch": 0.5905046846704984,
      "grad_norm": 69.11436679749976,
      "learning_rate": 8.883305777765317e-08,
      "logits": -2.095867395401001,
      "logps": -95.01261138916016,
      "loss": 0.0311,
      "objective": 0.033847175538539886,
      "ranking_idealized": 0.5166666507720947,
      "ranking_idealized_expo": 0.4333333373069763,
      "ranking_simple": 0.4333333373069763,
      "regularize": 0.03384659066796303,
      "step": 625
    },
    {
      "dpo_loss": 0.6883829236030579,
      "epoch": 0.5952287221478624,
      "grad_norm": 74.04617527963917,
      "learning_rate": 8.857189838426216e-08,
      "logits": -2.183093547821045,
      "logps": -92.19212341308594,
      "loss": 0.0332,
      "objective": 0.03131110966205597,
      "ranking_idealized": 0.6333333253860474,
      "ranking_idealized_expo": 0.5666666626930237,
      "ranking_simple": 0.5666666626930237,
      "regularize": 0.03130975365638733,
      "step": 630
    },
    {
      "dpo_loss": 0.6901324987411499,
      "epoch": 0.5999527596252263,
      "grad_norm": 63.254018805089515,
      "learning_rate": 8.83081137182057e-08,
      "logits": -2.137653112411499,
      "logps": -92.25005340576172,
      "loss": 0.028,
      "objective": 0.02667616680264473,
      "ranking_idealized": 0.6166666746139526,
      "ranking_idealized_expo": 0.550000011920929,
      "ranking_simple": 0.550000011920929,
      "regularize": 0.026675010100007057,
      "step": 635
    },
    {
      "dpo_loss": 0.6854274868965149,
      "epoch": 0.6046767971025904,
      "grad_norm": 67.682861674831,
      "learning_rate": 8.804172173314183e-08,
      "logits": -2.1525957584381104,
      "logps": -96.51889038085938,
      "loss": 0.0305,
      "objective": 0.026471592485904694,
      "ranking_idealized": 0.6166666746139526,
      "ranking_idealized_expo": 0.4833333194255829,
      "ranking_simple": 0.46666666865348816,
      "regularize": 0.026470640674233437,
      "step": 640
    },
    {
      "dpo_loss": 0.6916467547416687,
      "epoch": 0.6094008345799543,
      "grad_norm": 74.11851413570284,
      "learning_rate": 8.777274056018745e-08,
      "logits": -2.0791733264923096,
      "logps": -90.76611328125,
      "loss": 0.0275,
      "objective": 0.025933992117643356,
      "ranking_idealized": 0.5333333611488342,
      "ranking_idealized_expo": 0.4333333373069763,
      "ranking_simple": 0.4333333373069763,
      "regularize": 0.025932662189006805,
      "step": 645
    },
    {
      "dpo_loss": 0.6861349940299988,
      "epoch": 0.6141248720573184,
      "grad_norm": 71.7083018593228,
      "learning_rate": 8.750118850668412e-08,
      "logits": -2.0774688720703125,
      "logps": -91.57192993164062,
      "loss": 0.0333,
      "objective": 0.03247459605336189,
      "ranking_idealized": 0.5833333134651184,
      "ranking_idealized_expo": 0.5166666507720947,
      "ranking_simple": 0.5,
      "regularize": 0.03247232735157013,
      "step": 650
    },
    {
      "epoch": 0.6141248720573184,
      "eval_dpo_loss": 0.6926119327545166,
      "eval_logits": -1.993467926979065,
      "eval_logps": -98.69168853759766,
      "eval_loss": 0.014501783065497875,
      "eval_objective": 0.014641453512012959,
      "eval_ranking_idealized": 0.6022099256515503,
      "eval_ranking_idealized_expo": 0.5207182168960571,
      "eval_ranking_simple": 0.5179557800292969,
      "eval_regularize": 0.014639632776379585,
      "eval_runtime": 445.8707,
      "eval_samples_per_second": 12.986,
      "eval_steps_per_second": 3.248,
      "step": 650
    },
    {
      "dpo_loss": 0.6902641654014587,
      "epoch": 0.6188489095346823,
      "grad_norm": 58.31607902985929,
      "learning_rate": 8.722708405495222e-08,
      "logits": -2.2487266063690186,
      "logps": -89.10828399658203,
      "loss": 0.0289,
      "objective": 0.02854420617222786,
      "ranking_idealized": 0.5666666626930237,
      "ranking_idealized_expo": 0.44999998807907104,
      "ranking_simple": 0.4333333373069763,
      "regularize": 0.028543464839458466,
      "step": 655
    },
    {
      "dpo_loss": 0.6865178942680359,
      "epoch": 0.6235729470120462,
      "grad_norm": 65.43564652911088,
      "learning_rate": 8.695044586103295e-08,
      "logits": -2.105522394180298,
      "logps": -94.11585998535156,
      "loss": 0.0288,
      "objective": 0.027172502130270004,
      "ranking_idealized": 0.4333333373069763,
      "ranking_idealized_expo": 0.4000000059604645,
      "ranking_simple": 0.4000000059604645,
      "regularize": 0.027170367538928986,
      "step": 660
    },
    {
      "dpo_loss": 0.6929230093955994,
      "epoch": 0.6282969844894103,
      "grad_norm": 75.08767896831156,
      "learning_rate": 8.667129275341853e-08,
      "logits": -2.261946201324463,
      "logps": -90.70641326904297,
      "loss": 0.036,
      "objective": 0.03217438980937004,
      "ranking_idealized": 0.6166666746139526,
      "ranking_idealized_expo": 0.5333333611488342,
      "ranking_simple": 0.550000011920929,
      "regularize": 0.03217388316988945,
      "step": 665
    },
    {
      "dpo_loss": 0.6909863948822021,
      "epoch": 0.6330210219667742,
      "grad_norm": 61.63435170443301,
      "learning_rate": 8.638964373177073e-08,
      "logits": -2.0806498527526855,
      "logps": -93.46875762939453,
      "loss": 0.0283,
      "objective": 0.03122856095433235,
      "ranking_idealized": 0.5833333134651184,
      "ranking_idealized_expo": 0.5333333611488342,
      "ranking_simple": 0.550000011920929,
      "regularize": 0.03122722916305065,
      "step": 670
    },
    {
      "dpo_loss": 0.6940844058990479,
      "epoch": 0.6377450594441383,
      "grad_norm": 65.12038155568429,
      "learning_rate": 8.610551796562768e-08,
      "logits": -2.2103471755981445,
      "logps": -92.73240661621094,
      "loss": 0.0319,
      "objective": 0.03719858080148697,
      "ranking_idealized": 0.6000000238418579,
      "ranking_idealized_expo": 0.5,
      "ranking_simple": 0.5,
      "regularize": 0.03719812259078026,
      "step": 675
    },
    {
      "dpo_loss": 0.6896507740020752,
      "epoch": 0.6424690969215022,
      "grad_norm": 65.07489196501477,
      "learning_rate": 8.581893479309924e-08,
      "logits": -2.2053842544555664,
      "logps": -93.24919128417969,
      "loss": 0.0265,
      "objective": 0.023518383502960205,
      "ranking_idealized": 0.5666666626930237,
      "ranking_idealized_expo": 0.4833333194255829,
      "ranking_simple": 0.4833333194255829,
      "regularize": 0.023516135290265083,
      "step": 680
    },
    {
      "dpo_loss": 0.6927950382232666,
      "epoch": 0.6471931343988663,
      "grad_norm": 70.32780445036671,
      "learning_rate": 8.552991371955072e-08,
      "logits": -2.296104907989502,
      "logps": -92.59764099121094,
      "loss": 0.0318,
      "objective": 0.034257274121046066,
      "ranking_idealized": 0.6833333373069763,
      "ranking_idealized_expo": 0.6166666746139526,
      "ranking_simple": 0.6166666746139526,
      "regularize": 0.03425610437989235,
      "step": 685
    },
    {
      "dpo_loss": 0.692520797252655,
      "epoch": 0.6519171718762302,
      "grad_norm": 70.14652096802335,
      "learning_rate": 8.523847441627536e-08,
      "logits": -2.193286657333374,
      "logps": -94.4785385131836,
      "loss": 0.0326,
      "objective": 0.03835910186171532,
      "ranking_idealized": 0.5333333611488342,
      "ranking_idealized_expo": 0.4166666567325592,
      "ranking_simple": 0.4000000059604645,
      "regularize": 0.03835882246494293,
      "step": 690
    },
    {
      "dpo_loss": 0.6892996430397034,
      "epoch": 0.6566412093535942,
      "grad_norm": 73.28164754238634,
      "learning_rate": 8.494463671915546e-08,
      "logits": -2.1629860401153564,
      "logps": -93.7652359008789,
      "loss": 0.0265,
      "objective": 0.026581525802612305,
      "ranking_idealized": 0.5666666626930237,
      "ranking_idealized_expo": 0.4166666567325592,
      "ranking_simple": 0.4000000059604645,
      "regularize": 0.026580767706036568,
      "step": 695
    },
    {
      "dpo_loss": 0.6907955408096313,
      "epoch": 0.6613652468309582,
      "grad_norm": 66.01685046319234,
      "learning_rate": 8.464842062731234e-08,
      "logits": -2.2634246349334717,
      "logps": -91.12454986572266,
      "loss": 0.028,
      "objective": 0.028724508360028267,
      "ranking_idealized": 0.6166666746139526,
      "ranking_idealized_expo": 0.5166666507720947,
      "ranking_simple": 0.5166666507720947,
      "regularize": 0.028722405433654785,
      "step": 700
    },
    {
      "epoch": 0.6613652468309582,
      "eval_dpo_loss": 0.6929543018341064,
      "eval_logits": -1.9953092336654663,
      "eval_logps": -98.67767333984375,
      "eval_loss": 0.013822407461702824,
      "eval_objective": 0.014025083743035793,
      "eval_ranking_idealized": 0.6022099256515503,
      "eval_ranking_idealized_expo": 0.5207182168960571,
      "eval_ranking_simple": 0.519336998462677,
      "eval_regularize": 0.01402334589511156,
      "eval_runtime": 446.1644,
      "eval_samples_per_second": 12.977,
      "eval_steps_per_second": 3.245,
      "step": 700
    },
    {
      "dpo_loss": 0.6904506683349609,
      "epoch": 0.6660892843083221,
      "grad_norm": 64.45584344371969,
      "learning_rate": 8.434984630174508e-08,
      "logits": -2.223440408706665,
      "logps": -94.05587005615234,
      "loss": 0.0302,
      "objective": 0.029637468978762627,
      "ranking_idealized": 0.6833333373069763,
      "ranking_idealized_expo": 0.5666666626930237,
      "ranking_simple": 0.5666666626930237,
      "regularize": 0.029636209830641747,
      "step": 705
    },
    {
      "dpo_loss": 0.6879330277442932,
      "epoch": 0.6708133217856862,
      "grad_norm": 67.77900053910153,
      "learning_rate": 8.404893406395842e-08,
      "logits": -2.1772301197052,
      "logps": -93.94538879394531,
      "loss": 0.0323,
      "objective": 0.030688025057315826,
      "ranking_idealized": 0.5166666507720947,
      "ranking_idealized_expo": 0.4166666567325592,
      "ranking_simple": 0.4166666567325592,
      "regularize": 0.030687240883708,
      "step": 710
    },
    {
      "dpo_loss": 0.6863754987716675,
      "epoch": 0.6755373592630501,
      "grad_norm": 78.62750388933773,
      "learning_rate": 8.37457043945796e-08,
      "logits": -2.1862614154815674,
      "logps": -88.71346282958984,
      "loss": 0.0319,
      "objective": 0.034725725650787354,
      "ranking_idealized": 0.6000000238418579,
      "ranking_idealized_expo": 0.4833333194255829,
      "ranking_simple": 0.4833333194255829,
      "regularize": 0.03472534194588661,
      "step": 715
    },
    {
      "dpo_loss": 0.6946022510528564,
      "epoch": 0.6802613967404142,
      "grad_norm": 63.01104309220956,
      "learning_rate": 8.344017793196442e-08,
      "logits": -2.1920392513275146,
      "logps": -90.14446258544922,
      "loss": 0.0265,
      "objective": 0.025683369487524033,
      "ranking_idealized": 0.6499999761581421,
      "ranking_idealized_expo": 0.5666666626930237,
      "ranking_simple": 0.5666666626930237,
      "regularize": 0.025682510808110237,
      "step": 720
    },
    {
      "dpo_loss": 0.6887207627296448,
      "epoch": 0.6849854342177781,
      "grad_norm": 75.04026894879733,
      "learning_rate": 8.313237547079252e-08,
      "logits": -2.10304594039917,
      "logps": -90.62553405761719,
      "loss": 0.0292,
      "objective": 0.029727067798376083,
      "ranking_idealized": 0.6833333373069763,
      "ranking_idealized_expo": 0.6166666746139526,
      "ranking_simple": 0.6166666746139526,
      "regularize": 0.0297266636043787,
      "step": 725
    },
    {
      "dpo_loss": 0.6926788687705994,
      "epoch": 0.6897094716951421,
      "grad_norm": 67.22966096550593,
      "learning_rate": 8.282231796065213e-08,
      "logits": -2.1637871265411377,
      "logps": -91.91923522949219,
      "loss": 0.0265,
      "objective": 0.02672416716814041,
      "ranking_idealized": 0.6333333253860474,
      "ranking_idealized_expo": 0.5833333134651184,
      "ranking_simple": 0.5833333134651184,
      "regularize": 0.026723742485046387,
      "step": 730
    },
    {
      "dpo_loss": 0.688262403011322,
      "epoch": 0.6944335091725061,
      "grad_norm": 64.56167756628024,
      "learning_rate": 8.251002650461411e-08,
      "logits": -2.1801397800445557,
      "logps": -93.63780212402344,
      "loss": 0.0294,
      "objective": 0.029570966958999634,
      "ranking_idealized": 0.5666666626930237,
      "ranking_idealized_expo": 0.4833333194255829,
      "ranking_simple": 0.4833333194255829,
      "regularize": 0.029570437967777252,
      "step": 735
    },
    {
      "dpo_loss": 0.6889380216598511,
      "epoch": 0.6991575466498701,
      "grad_norm": 76.20522666744934,
      "learning_rate": 8.219552235779577e-08,
      "logits": -2.1762733459472656,
      "logps": -93.22509765625,
      "loss": 0.0341,
      "objective": 0.03592396527528763,
      "ranking_idealized": 0.6333333253860474,
      "ranking_idealized_expo": 0.5666666626930237,
      "ranking_simple": 0.5833333134651184,
      "regularize": 0.035923395305871964,
      "step": 740
    },
    {
      "dpo_loss": 0.6904739141464233,
      "epoch": 0.7038815841272341,
      "grad_norm": 61.170974083082186,
      "learning_rate": 8.187882692591406e-08,
      "logits": -2.148138999938965,
      "logps": -91.92343139648438,
      "loss": 0.0298,
      "objective": 0.027687864378094673,
      "ranking_idealized": 0.5,
      "ranking_idealized_expo": 0.4333333373069763,
      "ranking_simple": 0.4166666567325592,
      "regularize": 0.027687139809131622,
      "step": 745
    },
    {
      "dpo_loss": 0.6924771666526794,
      "epoch": 0.7086056216045981,
      "grad_norm": 86.99376248944333,
      "learning_rate": 8.155996176382873e-08,
      "logits": -2.2314558029174805,
      "logps": -92.25162506103516,
      "loss": 0.0319,
      "objective": 0.033008575439453125,
      "ranking_idealized": 0.550000011920929,
      "ranking_idealized_expo": 0.46666666865348816,
      "ranking_simple": 0.4833333194255829,
      "regularize": 0.03300632908940315,
      "step": 750
    },
    {
      "epoch": 0.7086056216045981,
      "eval_dpo_loss": 0.6925805807113647,
      "eval_logits": -1.9951562881469727,
      "eval_logps": -98.77120208740234,
      "eval_loss": 0.014676159247756004,
      "eval_objective": 0.014523538760840893,
      "eval_ranking_idealized": 0.6022099256515503,
      "eval_ranking_idealized_expo": 0.5207182168960571,
      "eval_ranking_simple": 0.5179557800292969,
      "eval_regularize": 0.014521739445626736,
      "eval_runtime": 446.8328,
      "eval_samples_per_second": 12.958,
      "eval_steps_per_second": 3.241,
      "step": 750
    },
    {
      "dpo_loss": 0.6896133422851562,
      "epoch": 0.713329659081962,
      "grad_norm": 65.62950261910362,
      "learning_rate": 8.123894857407532e-08,
      "logits": -2.175105571746826,
      "logps": -92.83119201660156,
      "loss": 0.0297,
      "objective": 0.02945883385837078,
      "ranking_idealized": 0.5666666626930237,
      "ranking_idealized_expo": 0.4833333194255829,
      "ranking_simple": 0.4833333194255829,
      "regularize": 0.029456783086061478,
      "step": 755
    },
    {
      "dpo_loss": 0.6876399517059326,
      "epoch": 0.718053696559326,
      "grad_norm": 80.69728652803693,
      "learning_rate": 8.091580920538789e-08,
      "logits": -2.2073442935943604,
      "logps": -90.69680786132812,
      "loss": 0.0284,
      "objective": 0.029233213514089584,
      "ranking_idealized": 0.6000000238418579,
      "ranking_idealized_expo": 0.4333333373069763,
      "ranking_simple": 0.4333333373069763,
      "regularize": 0.029232406988739967,
      "step": 760
    },
    {
      "dpo_loss": 0.6934041380882263,
      "epoch": 0.72277773403669,
      "grad_norm": 59.09750353750027,
      "learning_rate": 8.059056565121216e-08,
      "logits": -2.2103536128997803,
      "logps": -91.05927276611328,
      "loss": 0.0275,
      "objective": 0.026471644639968872,
      "ranking_idealized": 0.5666666626930237,
      "ranking_idealized_expo": 0.38333332538604736,
      "ranking_simple": 0.38333332538604736,
      "regularize": 0.02647002600133419,
      "step": 765
    },
    {
      "dpo_loss": 0.6914010643959045,
      "epoch": 0.727501771514054,
      "grad_norm": 80.2131293813357,
      "learning_rate": 8.026324004820844e-08,
      "logits": -2.1993424892425537,
      "logps": -91.33180236816406,
      "loss": 0.0329,
      "objective": 0.03441242873668671,
      "ranking_idealized": 0.5333333611488342,
      "ranking_idealized_expo": 0.44999998807907104,
      "ranking_simple": 0.44999998807907104,
      "regularize": 0.03441200777888298,
      "step": 770
    },
    {
      "dpo_loss": 0.693133533000946,
      "epoch": 0.732225808991418,
      "grad_norm": 65.96085059361222,
      "learning_rate": 7.993385467474502e-08,
      "logits": -2.2453505992889404,
      "logps": -94.63382720947266,
      "loss": 0.0376,
      "objective": 0.032590463757514954,
      "ranking_idealized": 0.5333333611488342,
      "ranking_idealized_expo": 0.46666666865348816,
      "ranking_simple": 0.4833333194255829,
      "regularize": 0.032589759677648544,
      "step": 775
    },
    {
      "dpo_loss": 0.6881453394889832,
      "epoch": 0.736949846468782,
      "grad_norm": 66.36692143180971,
      "learning_rate": 7.960243194938191e-08,
      "logits": -2.1516549587249756,
      "logps": -94.29581451416016,
      "loss": 0.0322,
      "objective": 0.030854353681206703,
      "ranking_idealized": 0.5333333611488342,
      "ranking_idealized_expo": 0.44999998807907104,
      "ranking_simple": 0.44999998807907104,
      "regularize": 0.030853325501084328,
      "step": 780
    },
    {
      "dpo_loss": 0.6893908381462097,
      "epoch": 0.741673883946146,
      "grad_norm": 73.24038753014604,
      "learning_rate": 7.926899442934488e-08,
      "logits": -2.1456098556518555,
      "logps": -93.58820343017578,
      "loss": 0.0301,
      "objective": 0.030529705807566643,
      "ranking_idealized": 0.6666666865348816,
      "ranking_idealized_expo": 0.5833333134651184,
      "ranking_simple": 0.5833333134651184,
      "regularize": 0.030527813360095024,
      "step": 785
    },
    {
      "dpo_loss": 0.6873824000358582,
      "epoch": 0.7463979214235099,
      "grad_norm": 61.386663619909456,
      "learning_rate": 7.893356480899029e-08,
      "logits": -2.202815055847168,
      "logps": -89.73310089111328,
      "loss": 0.0284,
      "objective": 0.026303457096219063,
      "ranking_idealized": 0.7166666388511658,
      "ranking_idealized_expo": 0.6333333253860474,
      "ranking_simple": 0.6333333253860474,
      "regularize": 0.02630232647061348,
      "step": 790
    },
    {
      "dpo_loss": 0.6941797733306885,
      "epoch": 0.751121958900874,
      "grad_norm": 74.12319085244714,
      "learning_rate": 7.85961659182604e-08,
      "logits": -2.2534494400024414,
      "logps": -92.2616195678711,
      "loss": 0.0317,
      "objective": 0.028791796416044235,
      "ranking_idealized": 0.6499999761581421,
      "ranking_idealized_expo": 0.6166666746139526,
      "ranking_simple": 0.6166666746139526,
      "regularize": 0.028791317716240883,
      "step": 795
    },
    {
      "dpo_loss": 0.6879761815071106,
      "epoch": 0.7558459963782379,
      "grad_norm": 66.76187019896828,
      "learning_rate": 7.825682072112959e-08,
      "logits": -2.152491807937622,
      "logps": -90.43921661376953,
      "loss": 0.0297,
      "objective": 0.031770989298820496,
      "ranking_idealized": 0.6000000238418579,
      "ranking_idealized_expo": 0.5666666626930237,
      "ranking_simple": 0.5666666626930237,
      "regularize": 0.031770527362823486,
      "step": 800
    },
    {
      "epoch": 0.7558459963782379,
      "eval_dpo_loss": 0.6929048299789429,
      "eval_logits": -1.9949605464935303,
      "eval_logps": -98.13481903076172,
      "eval_loss": 0.015697013586759567,
      "eval_objective": 0.016285618767142296,
      "eval_ranking_idealized": 0.6022099256515503,
      "eval_ranking_idealized_expo": 0.5207182168960571,
      "eval_ranking_simple": 0.5186464190483093,
      "eval_regularize": 0.01628427766263485,
      "eval_runtime": 446.756,
      "eval_samples_per_second": 12.96,
      "eval_steps_per_second": 3.241,
      "step": 800
    },
    {
      "dpo_loss": 0.6914661526679993,
      "epoch": 0.760570033855602,
      "grad_norm": 61.85563229584601,
      "learning_rate": 7.79155523140413e-08,
      "logits": -2.1741960048675537,
      "logps": -94.34522247314453,
      "loss": 0.0341,
      "objective": 0.03409460559487343,
      "ranking_idealized": 0.46666666865348816,
      "ranking_idealized_expo": 0.38333332538604736,
      "ranking_simple": 0.38333332538604736,
      "regularize": 0.03409397229552269,
      "step": 805
    },
    {
      "dpo_loss": 0.6910237669944763,
      "epoch": 0.7652940713329659,
      "grad_norm": 66.02922525617878,
      "learning_rate": 7.757238392433613e-08,
      "logits": -2.218034267425537,
      "logps": -91.0445327758789,
      "loss": 0.0309,
      "objective": 0.02644220180809498,
      "ranking_idealized": 0.6000000238418579,
      "ranking_idealized_expo": 0.550000011920929,
      "ranking_simple": 0.550000011920929,
      "regularize": 0.026441721245646477,
      "step": 810
    },
    {
      "dpo_loss": 0.6903732419013977,
      "epoch": 0.77001810881033,
      "grad_norm": 71.25069648841824,
      "learning_rate": 7.722733890867088e-08,
      "logits": -2.13299298286438,
      "logps": -94.09717559814453,
      "loss": 0.0306,
      "objective": 0.027522355318069458,
      "ranking_idealized": 0.6000000238418579,
      "ranking_idealized_expo": 0.5166666507720947,
      "ranking_simple": 0.5166666507720947,
      "regularize": 0.027521483600139618,
      "step": 815
    },
    {
      "dpo_loss": 0.692348062992096,
      "epoch": 0.7747421462876939,
      "grad_norm": 62.71232207694529,
      "learning_rate": 7.688044075142886e-08,
      "logits": -2.2638330459594727,
      "logps": -89.2739486694336,
      "loss": 0.0265,
      "objective": 0.024408848956227303,
      "ranking_idealized": 0.5666666626930237,
      "ranking_idealized_expo": 0.5,
      "ranking_simple": 0.5,
      "regularize": 0.024408036842942238,
      "step": 820
    },
    {
      "dpo_loss": 0.6927106976509094,
      "epoch": 0.7794661837650578,
      "grad_norm": 68.11094417791882,
      "learning_rate": 7.653171306312161e-08,
      "logits": -2.155310869216919,
      "logps": -92.2811508178711,
      "loss": 0.0314,
      "objective": 0.029093213379383087,
      "ranking_idealized": 0.5333333611488342,
      "ranking_idealized_expo": 0.4833333194255829,
      "ranking_simple": 0.4833333194255829,
      "regularize": 0.029092345386743546,
      "step": 825
    },
    {
      "dpo_loss": 0.6911001801490784,
      "epoch": 0.7841902212424219,
      "grad_norm": 66.75059983982347,
      "learning_rate": 7.618117957878178e-08,
      "logits": -2.236713409423828,
      "logps": -93.50963592529297,
      "loss": 0.0363,
      "objective": 0.033676620572805405,
      "ranking_idealized": 0.5833333134651184,
      "ranking_idealized_expo": 0.46666666865348816,
      "ranking_simple": 0.4833333194255829,
      "regularize": 0.03367554768919945,
      "step": 830
    },
    {
      "dpo_loss": 0.6890848278999329,
      "epoch": 0.7889142587197858,
      "grad_norm": 59.842310291910785,
      "learning_rate": 7.582886415634773e-08,
      "logits": -2.1434099674224854,
      "logps": -89.62670135498047,
      "loss": 0.0261,
      "objective": 0.032363876700401306,
      "ranking_idealized": 0.6333333253860474,
      "ranking_idealized_expo": 0.5833333134651184,
      "ranking_simple": 0.5833333134651184,
      "regularize": 0.032363247126340866,
      "step": 835
    },
    {
      "dpo_loss": 0.6889583468437195,
      "epoch": 0.7936382961971499,
      "grad_norm": 60.528801427412084,
      "learning_rate": 7.547479077503975e-08,
      "logits": -2.0931692123413086,
      "logps": -90.27711486816406,
      "loss": 0.0286,
      "objective": 0.032331857830286026,
      "ranking_idealized": 0.5833333134651184,
      "ranking_idealized_expo": 0.550000011920929,
      "ranking_simple": 0.550000011920929,
      "regularize": 0.03233127295970917,
      "step": 840
    },
    {
      "dpo_loss": 0.6943262815475464,
      "epoch": 0.7983623336745138,
      "grad_norm": 66.0119748602127,
      "learning_rate": 7.511898353372797e-08,
      "logits": -2.21136212348938,
      "logps": -91.64664459228516,
      "loss": 0.0279,
      "objective": 0.03553476184606552,
      "ranking_idealized": 0.6333333253860474,
      "ranking_idealized_expo": 0.5666666626930237,
      "ranking_simple": 0.5666666626930237,
      "regularize": 0.03553430363535881,
      "step": 845
    },
    {
      "dpo_loss": 0.6893811821937561,
      "epoch": 0.8030863711518778,
      "grad_norm": 64.38574308982342,
      "learning_rate": 7.476146664929213e-08,
      "logits": -2.2435154914855957,
      "logps": -92.36274719238281,
      "loss": 0.0286,
      "objective": 0.0326698012650013,
      "ranking_idealized": 0.550000011920929,
      "ranking_idealized_expo": 0.44999998807907104,
      "ranking_simple": 0.46666666865348816,
      "regularize": 0.03266819566488266,
      "step": 850
    },
    {
      "epoch": 0.8030863711518778,
      "eval_dpo_loss": 0.6928204894065857,
      "eval_logits": -1.9953876733779907,
      "eval_logps": -98.59400939941406,
      "eval_loss": 0.012405806221067905,
      "eval_objective": 0.012502364814281464,
      "eval_ranking_idealized": 0.6022099256515503,
      "eval_ranking_idealized_expo": 0.5207182168960571,
      "eval_ranking_simple": 0.5172652006149292,
      "eval_regularize": 0.012500518001616001,
      "eval_runtime": 453.0739,
      "eval_samples_per_second": 12.779,
      "eval_steps_per_second": 3.196,
      "step": 850
    },
    {
      "dpo_loss": 0.6887614130973816,
      "epoch": 0.8078104086292418,
      "grad_norm": 68.55505260723746,
      "learning_rate": 7.440226445497333e-08,
      "logits": -2.201233386993408,
      "logps": -92.58832550048828,
      "loss": 0.0274,
      "objective": 0.026876337826251984,
      "ranking_idealized": 0.6166666746139526,
      "ranking_idealized_expo": 0.5166666507720947,
      "ranking_simple": 0.5333333611488342,
      "regularize": 0.0268756952136755,
      "step": 855
    },
    {
      "dpo_loss": 0.6904102563858032,
      "epoch": 0.8125344461066057,
      "grad_norm": 71.78389198214526,
      "learning_rate": 7.404140139871796e-08,
      "logits": -2.231065273284912,
      "logps": -95.48096466064453,
      "loss": 0.0317,
      "objective": 0.03271006420254707,
      "ranking_idealized": 0.6833333373069763,
      "ranking_idealized_expo": 0.550000011920929,
      "ranking_simple": 0.550000011920929,
      "regularize": 0.032709524035453796,
      "step": 860
    },
    {
      "dpo_loss": 0.6954269409179688,
      "epoch": 0.8172584835839698,
      "grad_norm": 66.09070940831865,
      "learning_rate": 7.36789020415136e-08,
      "logits": -2.1372532844543457,
      "logps": -91.0411605834961,
      "loss": 0.027,
      "objective": 0.03049122728407383,
      "ranking_idealized": 0.7166666388511658,
      "ranking_idealized_expo": 0.6000000238418579,
      "ranking_simple": 0.6000000238418579,
      "regularize": 0.03049064427614212,
      "step": 865
    },
    {
      "dpo_loss": 0.6953790187835693,
      "epoch": 0.8219825210613337,
      "grad_norm": 61.58311436161328,
      "learning_rate": 7.331479105571739e-08,
      "logits": -2.1517558097839355,
      "logps": -88.9326171875,
      "loss": 0.0281,
      "objective": 0.02535523846745491,
      "ranking_idealized": 0.5,
      "ranking_idealized_expo": 0.5,
      "ranking_simple": 0.5,
      "regularize": 0.025354215875267982,
      "step": 870
    },
    {
      "dpo_loss": 0.6913577318191528,
      "epoch": 0.8267065585386978,
      "grad_norm": 61.45457019245544,
      "learning_rate": 7.294909322337688e-08,
      "logits": -2.0830719470977783,
      "logps": -95.3221664428711,
      "loss": 0.0267,
      "objective": 0.02792440913617611,
      "ranking_idealized": 0.6499999761581421,
      "ranking_idealized_expo": 0.6000000238418579,
      "ranking_simple": 0.5833333134651184,
      "regularize": 0.0279233455657959,
      "step": 875
    },
    {
      "dpo_loss": 0.6916779279708862,
      "epoch": 0.8314305960160617,
      "grad_norm": 69.76363049930384,
      "learning_rate": 7.258183343454319e-08,
      "logits": -2.276218891143799,
      "logps": -91.77556610107422,
      "loss": 0.029,
      "objective": 0.02677006646990776,
      "ranking_idealized": 0.699999988079071,
      "ranking_idealized_expo": 0.5666666626930237,
      "ranking_simple": 0.5666666626930237,
      "regularize": 0.02676946483552456,
      "step": 880
    },
    {
      "dpo_loss": 0.6923359036445618,
      "epoch": 0.8361546334934257,
      "grad_norm": 65.0950944876437,
      "learning_rate": 7.221303668557696e-08,
      "logits": -2.1599981784820557,
      "logps": -90.72624969482422,
      "loss": 0.025,
      "objective": 0.026336384937167168,
      "ranking_idealized": 0.5166666507720947,
      "ranking_idealized_expo": 0.5,
      "ranking_simple": 0.5,
      "regularize": 0.026335686445236206,
      "step": 885
    },
    {
      "dpo_loss": 0.692168116569519,
      "epoch": 0.8408786709707897,
      "grad_norm": 63.609027029284135,
      "learning_rate": 7.184272807744725e-08,
      "logits": -2.1683857440948486,
      "logps": -92.93081665039062,
      "loss": 0.0278,
      "objective": 0.03211880847811699,
      "ranking_idealized": 0.5666666626930237,
      "ranking_idealized_expo": 0.5166666507720947,
      "ranking_simple": 0.5166666507720947,
      "regularize": 0.03211786970496178,
      "step": 890
    },
    {
      "dpo_loss": 0.6949544548988342,
      "epoch": 0.8456027084481537,
      "grad_norm": 71.19347753450685,
      "learning_rate": 7.147093281402281e-08,
      "logits": -2.2566373348236084,
      "logps": -91.25569915771484,
      "loss": 0.0287,
      "objective": 0.022946473211050034,
      "ranking_idealized": 0.5833333134651184,
      "ranking_idealized_expo": 0.4833333194255829,
      "ranking_simple": 0.4833333194255829,
      "regularize": 0.022945420816540718,
      "step": 895
    },
    {
      "dpo_loss": 0.6891117095947266,
      "epoch": 0.8503267459255177,
      "grad_norm": 73.13390434400947,
      "learning_rate": 7.109767620035688e-08,
      "logits": -2.1637258529663086,
      "logps": -95.48709869384766,
      "loss": 0.0285,
      "objective": 0.030570391565561295,
      "ranking_idealized": 0.7166666388511658,
      "ranking_idealized_expo": 0.6499999761581421,
      "ranking_simple": 0.6499999761581421,
      "regularize": 0.030569853261113167,
      "step": 900
    },
    {
      "epoch": 0.8503267459255177,
      "eval_dpo_loss": 0.69291752576828,
      "eval_logits": -1.993115782737732,
      "eval_logps": -98.94220733642578,
      "eval_loss": 0.011713932268321514,
      "eval_objective": 0.011828156188130379,
      "eval_ranking_idealized": 0.6022099256515503,
      "eval_ranking_idealized_expo": 0.5207182168960571,
      "eval_ranking_simple": 0.5165745615959167,
      "eval_regularize": 0.01182604394853115,
      "eval_runtime": 445.945,
      "eval_samples_per_second": 12.984,
      "eval_steps_per_second": 3.247,
      "step": 900
    },
    {
      "dpo_loss": 0.6910974383354187,
      "epoch": 0.8550507834028817,
      "grad_norm": 72.76821991685014,
      "learning_rate": 7.072298364096485e-08,
      "logits": -2.094447374343872,
      "logps": -89.45642852783203,
      "loss": 0.0266,
      "objective": 0.027411019429564476,
      "ranking_idealized": 0.5666666626930237,
      "ranking_idealized_expo": 0.550000011920929,
      "ranking_simple": 0.550000011920929,
      "regularize": 0.02741014026105404,
      "step": 905
    },
    {
      "dpo_loss": 0.6891763806343079,
      "epoch": 0.8597748208802457,
      "grad_norm": 71.26536161303537,
      "learning_rate": 7.034688063809511e-08,
      "logits": -2.1282496452331543,
      "logps": -91.80699157714844,
      "loss": 0.0298,
      "objective": 0.025737237185239792,
      "ranking_idealized": 0.6333333253860474,
      "ranking_idealized_expo": 0.4833333194255829,
      "ranking_simple": 0.5333333611488342,
      "regularize": 0.025736594572663307,
      "step": 910
    },
    {
      "dpo_loss": 0.6862377524375916,
      "epoch": 0.8644988583576096,
      "grad_norm": 69.04287293810181,
      "learning_rate": 6.996939278999337e-08,
      "logits": -2.152179479598999,
      "logps": -94.09297180175781,
      "loss": 0.0263,
      "objective": 0.027985723689198494,
      "ranking_idealized": 0.5833333134651184,
      "ranking_idealized_expo": 0.4833333194255829,
      "ranking_simple": 0.4833333194255829,
      "regularize": 0.02798387221992016,
      "step": 915
    },
    {
      "dpo_loss": 0.690563440322876,
      "epoch": 0.8692228958349736,
      "grad_norm": 60.9102612710031,
      "learning_rate": 6.959054578916042e-08,
      "logits": -2.1106715202331543,
      "logps": -90.7065658569336,
      "loss": 0.0281,
      "objective": 0.02792646363377571,
      "ranking_idealized": 0.6666666865348816,
      "ranking_idealized_expo": 0.5833333134651184,
      "ranking_simple": 0.6000000238418579,
      "regularize": 0.02792549505829811,
      "step": 920
    },
    {
      "dpo_loss": 0.6911750435829163,
      "epoch": 0.8739469333123376,
      "grad_norm": 62.2660390306647,
      "learning_rate": 6.921036542060343e-08,
      "logits": -1.9987537860870361,
      "logps": -89.90222930908203,
      "loss": 0.0251,
      "objective": 0.01876661367714405,
      "ranking_idealized": 0.6833333373069763,
      "ranking_idealized_expo": 0.6333333253860474,
      "ranking_simple": 0.6333333253860474,
      "regularize": 0.018765147775411606,
      "step": 925
    },
    {
      "dpo_loss": 0.6897457242012024,
      "epoch": 0.8786709707897016,
      "grad_norm": 59.1678071537183,
      "learning_rate": 6.882887756008093e-08,
      "logits": -2.111668825149536,
      "logps": -87.85643768310547,
      "loss": 0.0257,
      "objective": 0.02605438232421875,
      "ranking_idealized": 0.6000000238418579,
      "ranking_idealized_expo": 0.550000011920929,
      "ranking_simple": 0.550000011920929,
      "regularize": 0.026053914800286293,
      "step": 930
    },
    {
      "dpo_loss": 0.6957460045814514,
      "epoch": 0.8833950082670656,
      "grad_norm": 64.76262904725678,
      "learning_rate": 6.844610817234172e-08,
      "logits": -2.093857765197754,
      "logps": -92.8622055053711,
      "loss": 0.0273,
      "objective": 0.027011338621377945,
      "ranking_idealized": 0.5333333611488342,
      "ranking_idealized_expo": 0.4833333194255829,
      "ranking_simple": 0.46666666865348816,
      "regularize": 0.027010958641767502,
      "step": 935
    },
    {
      "dpo_loss": 0.6905789971351624,
      "epoch": 0.8881190457444296,
      "grad_norm": 70.7130564090627,
      "learning_rate": 6.806208330935765e-08,
      "logits": -2.2473738193511963,
      "logps": -91.84986114501953,
      "loss": 0.0268,
      "objective": 0.027645627036690712,
      "ranking_idealized": 0.6000000238418579,
      "ranking_idealized_expo": 0.5666666626930237,
      "ranking_simple": 0.5666666626930237,
      "regularize": 0.02764386683702469,
      "step": 940
    },
    {
      "dpo_loss": 0.6940723061561584,
      "epoch": 0.8928430832217935,
      "grad_norm": 60.88887279226543,
      "learning_rate": 6.767682910855045e-08,
      "logits": -2.287950038909912,
      "logps": -89.53514862060547,
      "loss": 0.0261,
      "objective": 0.028092078864574432,
      "ranking_idealized": 0.550000011920929,
      "ranking_idealized_expo": 0.46666666865348816,
      "ranking_simple": 0.46666666865348816,
      "regularize": 0.0280916728079319,
      "step": 945
    },
    {
      "dpo_loss": 0.694039523601532,
      "epoch": 0.8975671206991576,
      "grad_norm": 67.50050200668711,
      "learning_rate": 6.729037179101287e-08,
      "logits": -2.304736614227295,
      "logps": -93.9173812866211,
      "loss": 0.0248,
      "objective": 0.02201911062002182,
      "ranking_idealized": 0.5833333134651184,
      "ranking_idealized_expo": 0.46666666865348816,
      "ranking_simple": 0.4833333194255829,
      "regularize": 0.02201777510344982,
      "step": 950
    },
    {
      "epoch": 0.8975671206991576,
      "eval_dpo_loss": 0.6931836009025574,
      "eval_logits": -1.9902262687683105,
      "eval_logps": -98.64472198486328,
      "eval_loss": 0.015600171871483326,
      "eval_objective": 0.015454174019396305,
      "eval_ranking_idealized": 0.6022099256515503,
      "eval_ranking_idealized_expo": 0.5207182168960571,
      "eval_ranking_simple": 0.5172652006149292,
      "eval_regularize": 0.015452706255018711,
      "eval_runtime": 446.0966,
      "eval_samples_per_second": 12.979,
      "eval_steps_per_second": 3.246,
      "step": 950
    },
    {
      "dpo_loss": 0.6894887089729309,
      "epoch": 0.9022911581765215,
      "grad_norm": 62.475301521026935,
      "learning_rate": 6.690273765972383e-08,
      "logits": -2.1381261348724365,
      "logps": -90.50852966308594,
      "loss": 0.0247,
      "objective": 0.029864691197872162,
      "ranking_idealized": 0.7166666388511658,
      "ranking_idealized_expo": 0.5833333134651184,
      "ranking_simple": 0.6000000238418579,
      "regularize": 0.02986370399594307,
      "step": 955
    },
    {
      "dpo_loss": 0.6904736161231995,
      "epoch": 0.9070151956538856,
      "grad_norm": 71.36008054276485,
      "learning_rate": 6.651395309775836e-08,
      "logits": -2.161102294921875,
      "logps": -94.71805572509766,
      "loss": 0.0273,
      "objective": 0.03055974654853344,
      "ranking_idealized": 0.6166666746139526,
      "ranking_idealized_expo": 0.5833333134651184,
      "ranking_simple": 0.5833333134651184,
      "regularize": 0.03055933117866516,
      "step": 960
    },
    {
      "dpo_loss": 0.6911565661430359,
      "epoch": 0.9117392331312495,
      "grad_norm": 75.9289905899972,
      "learning_rate": 6.612404456649187e-08,
      "logits": -2.174187660217285,
      "logps": -90.80412292480469,
      "loss": 0.0255,
      "objective": 0.02420150302350521,
      "ranking_idealized": 0.550000011920929,
      "ranking_idealized_expo": 0.5166666507720947,
      "ranking_simple": 0.5166666507720947,
      "regularize": 0.024200933054089546,
      "step": 965
    },
    {
      "dpo_loss": 0.6908090114593506,
      "epoch": 0.9164632706086135,
      "grad_norm": 66.41988248188461,
      "learning_rate": 6.573303860379914e-08,
      "logits": -2.258518695831299,
      "logps": -91.59303283691406,
      "loss": 0.0271,
      "objective": 0.02120455540716648,
      "ranking_idealized": 0.5,
      "ranking_idealized_expo": 0.44999998807907104,
      "ranking_simple": 0.4333333373069763,
      "regularize": 0.02120377868413925,
      "step": 970
    },
    {
      "dpo_loss": 0.686578631401062,
      "epoch": 0.9211873080859775,
      "grad_norm": 68.57226169783617,
      "learning_rate": 6.534096182224808e-08,
      "logits": -2.0389044284820557,
      "logps": -94.76436614990234,
      "loss": 0.0299,
      "objective": 0.029232459142804146,
      "ranking_idealized": 0.4833333194255829,
      "ranking_idealized_expo": 0.4000000059604645,
      "ranking_simple": 0.4000000059604645,
      "regularize": 0.029231999069452286,
      "step": 975
    },
    {
      "dpo_loss": 0.6919539570808411,
      "epoch": 0.9259113455633414,
      "grad_norm": 69.41679502492849,
      "learning_rate": 6.494784090728851e-08,
      "logits": -2.2500946521759033,
      "logps": -96.09117126464844,
      "loss": 0.0262,
      "objective": 0.026704249903559685,
      "ranking_idealized": 0.6499999761581421,
      "ranking_idealized_expo": 0.5333333611488342,
      "ranking_simple": 0.5333333611488342,
      "regularize": 0.0267037320882082,
      "step": 980
    },
    {
      "dpo_loss": 0.6925919055938721,
      "epoch": 0.9306353830407055,
      "grad_norm": 64.18653487464074,
      "learning_rate": 6.455370261543578e-08,
      "logits": -2.1756606101989746,
      "logps": -93.87403106689453,
      "loss": 0.0251,
      "objective": 0.025608109310269356,
      "ranking_idealized": 0.6833333373069763,
      "ranking_idealized_expo": 0.5166666507720947,
      "ranking_simple": 0.5166666507720947,
      "regularize": 0.025606883689761162,
      "step": 985
    },
    {
      "dpo_loss": 0.6908634901046753,
      "epoch": 0.9353594205180694,
      "grad_norm": 61.06423988779827,
      "learning_rate": 6.415857377244979e-08,
      "logits": -2.1095356941223145,
      "logps": -88.83150482177734,
      "loss": 0.0251,
      "objective": 0.026077650487422943,
      "ranking_idealized": 0.5666666626930237,
      "ranking_idealized_expo": 0.5333333611488342,
      "ranking_simple": 0.5333333611488342,
      "regularize": 0.02607729658484459,
      "step": 990
    },
    {
      "dpo_loss": 0.689622700214386,
      "epoch": 0.9400834579954335,
      "grad_norm": 68.1910690637634,
      "learning_rate": 6.376248127150908e-08,
      "logits": -2.150278329849243,
      "logps": -91.8506088256836,
      "loss": 0.0269,
      "objective": 0.025722531601786613,
      "ranking_idealized": 0.6166666746139526,
      "ranking_idealized_expo": 0.5666666626930237,
      "ranking_simple": 0.5666666626930237,
      "regularize": 0.025721782818436623,
      "step": 995
    },
    {
      "dpo_loss": 0.6903151273727417,
      "epoch": 0.9448074954727974,
      "grad_norm": 64.10384868193276,
      "learning_rate": 6.33654520713805e-08,
      "logits": -2.1098899841308594,
      "logps": -93.50405883789062,
      "loss": 0.0272,
      "objective": 0.02512853965163231,
      "ranking_idealized": 0.6000000238418579,
      "ranking_idealized_expo": 0.5333333611488342,
      "ranking_simple": 0.5333333611488342,
      "regularize": 0.0251275934278965,
      "step": 1000
    },
    {
      "epoch": 0.9448074954727974,
      "eval_dpo_loss": 0.6931213736534119,
      "eval_logits": -1.9906424283981323,
      "eval_logps": -98.12418365478516,
      "eval_loss": 0.01257664430886507,
      "eval_objective": 0.012785565108060837,
      "eval_ranking_idealized": 0.6022099256515503,
      "eval_ranking_idealized_expo": 0.5207182168960571,
      "eval_ranking_simple": 0.5179557800292969,
      "eval_regularize": 0.012783819809556007,
      "eval_runtime": 444.658,
      "eval_samples_per_second": 13.021,
      "eval_steps_per_second": 3.256,
      "step": 1000
    },
    {
      "dpo_loss": 0.6847264170646667,
      "epoch": 0.9495315329501613,
      "grad_norm": 68.90168504337811,
      "learning_rate": 6.296751319458434e-08,
      "logits": -2.1259357929229736,
      "logps": -91.41114044189453,
      "loss": 0.0298,
      "objective": 0.03322311118245125,
      "ranking_idealized": 0.5666666626930237,
      "ranking_idealized_expo": 0.5,
      "ranking_simple": 0.5,
      "regularize": 0.03322221338748932,
      "step": 1005
    },
    {
      "dpo_loss": 0.6920087933540344,
      "epoch": 0.9542555704275254,
      "grad_norm": 64.89339434605618,
      "learning_rate": 6.256869172555513e-08,
      "logits": -2.1444926261901855,
      "logps": -91.69261932373047,
      "loss": 0.0264,
      "objective": 0.02593044377863407,
      "ranking_idealized": 0.4833333194255829,
      "ranking_idealized_expo": 0.44999998807907104,
      "ranking_simple": 0.4333333373069763,
      "regularize": 0.025929344817996025,
      "step": 1010
    },
    {
      "dpo_loss": 0.6885382533073425,
      "epoch": 0.9589796079048893,
      "grad_norm": 71.3368687595913,
      "learning_rate": 6.216901480879819e-08,
      "logits": -2.0881664752960205,
      "logps": -90.8614501953125,
      "loss": 0.0236,
      "objective": 0.021669141948223114,
      "ranking_idealized": 0.6000000238418579,
      "ranking_idealized_expo": 0.5,
      "ranking_simple": 0.5166666507720947,
      "regularize": 0.021668575704097748,
      "step": 1015
    },
    {
      "dpo_loss": 0.6919686794281006,
      "epoch": 0.9637036453822534,
      "grad_norm": 68.26105061311512,
      "learning_rate": 6.176850964704212e-08,
      "logits": -2.129828453063965,
      "logps": -95.18238830566406,
      "loss": 0.0226,
      "objective": 0.025728456676006317,
      "ranking_idealized": 0.6166666746139526,
      "ranking_idealized_expo": 0.5833333134651184,
      "ranking_simple": 0.5666666626930237,
      "regularize": 0.02572786808013916,
      "step": 1020
    },
    {
      "dpo_loss": 0.6866704225540161,
      "epoch": 0.9684276828596173,
      "grad_norm": 61.40903124064667,
      "learning_rate": 6.136720349938743e-08,
      "logits": -2.2576215267181396,
      "logps": -94.10470581054688,
      "loss": 0.0257,
      "objective": 0.022323768585920334,
      "ranking_idealized": 0.5666666626930237,
      "ranking_idealized_expo": 0.5166666507720947,
      "ranking_simple": 0.5166666507720947,
      "regularize": 0.022323111072182655,
      "step": 1025
    },
    {
      "dpo_loss": 0.69089674949646,
      "epoch": 0.9731517203369814,
      "grad_norm": 63.691297756435006,
      "learning_rate": 6.096512367945113e-08,
      "logits": -2.113276243209839,
      "logps": -90.31819152832031,
      "loss": 0.0244,
      "objective": 0.022346744313836098,
      "ranking_idealized": 0.5833333134651184,
      "ranking_idealized_expo": 0.5333333611488342,
      "ranking_simple": 0.5333333611488342,
      "regularize": 0.02234587073326111,
      "step": 1030
    },
    {
      "dpo_loss": 0.6910406351089478,
      "epoch": 0.9778757578143453,
      "grad_norm": 65.44632044043543,
      "learning_rate": 6.056229755350772e-08,
      "logits": -2.147958517074585,
      "logps": -93.92337036132812,
      "loss": 0.0234,
      "objective": 0.02298605814576149,
      "ranking_idealized": 0.6833333373069763,
      "ranking_idealized_expo": 0.6166666746139526,
      "ranking_simple": 0.6166666746139526,
      "regularize": 0.022985396906733513,
      "step": 1035
    },
    {
      "dpo_loss": 0.68792724609375,
      "epoch": 0.9825997952917093,
      "grad_norm": 76.3390109961412,
      "learning_rate": 6.01587525386267e-08,
      "logits": -2.1341538429260254,
      "logps": -90.12808227539062,
      "loss": 0.0253,
      "objective": 0.02918338030576706,
      "ranking_idealized": 0.5333333611488342,
      "ranking_idealized_expo": 0.46666666865348816,
      "ranking_simple": 0.46666666865348816,
      "regularize": 0.029182856902480125,
      "step": 1040
    },
    {
      "dpo_loss": 0.6883952021598816,
      "epoch": 0.9873238327690733,
      "grad_norm": 62.771275862582634,
      "learning_rate": 5.975451610080642e-08,
      "logits": -2.1016061305999756,
      "logps": -92.14013671875,
      "loss": 0.0243,
      "objective": 0.02377927675843239,
      "ranking_idealized": 0.5833333134651184,
      "ranking_idealized_expo": 0.4833333194255829,
      "ranking_simple": 0.4833333194255829,
      "regularize": 0.023778444156050682,
      "step": 1045
    },
    {
      "dpo_loss": 0.6908401846885681,
      "epoch": 0.9920478702464373,
      "grad_norm": 61.00213004445293,
      "learning_rate": 5.9349615753104655e-08,
      "logits": -2.1279587745666504,
      "logps": -97.24657440185547,
      "loss": 0.0215,
      "objective": 0.021991008892655373,
      "ranking_idealized": 0.6000000238418579,
      "ranking_idealized_expo": 0.5666666626930237,
      "ranking_simple": 0.5666666626930237,
      "regularize": 0.02199002355337143,
      "step": 1050
    },
    {
      "epoch": 0.9920478702464373,
      "eval_dpo_loss": 0.6927458047866821,
      "eval_logits": -1.991100788116455,
      "eval_logps": -98.33568572998047,
      "eval_loss": 0.01325704250484705,
      "eval_objective": 0.013452271930873394,
      "eval_ranking_idealized": 0.6022099256515503,
      "eval_ranking_idealized_expo": 0.5207182168960571,
      "eval_ranking_simple": 0.5179557800292969,
      "eval_regularize": 0.01345061045140028,
      "eval_runtime": 451.6429,
      "eval_samples_per_second": 12.82,
      "eval_steps_per_second": 3.206,
      "step": 1050
    },
    {
      "dpo_loss": 0.6905434131622314,
      "epoch": 0.9967719077238013,
      "grad_norm": 69.09205896680123,
      "learning_rate": 5.894407905376616e-08,
      "logits": -2.2125723361968994,
      "logps": -90.43359375,
      "loss": 0.0256,
      "objective": 0.022856025025248528,
      "ranking_idealized": 0.550000011920929,
      "ranking_idealized_expo": 0.46666666865348816,
      "ranking_simple": 0.46666666865348816,
      "regularize": 0.022855514660477638,
      "step": 1055
    },
    {
      "dpo_loss": 0.6896554231643677,
      "epoch": 1.0014959452011654,
      "grad_norm": 71.18743706384133,
      "learning_rate": 5.853793360434687e-08,
      "logits": -2.095319986343384,
      "logps": -92.04082489013672,
      "loss": 0.0261,
      "objective": 0.025890907272696495,
      "ranking_idealized": 0.6166666746139526,
      "ranking_idealized_expo": 0.550000011920929,
      "ranking_simple": 0.550000011920929,
      "regularize": 0.025890160351991653,
      "step": 1060
    },
    {
      "dpo_loss": 0.6926552057266235,
      "epoch": 1.0062199826785292,
      "grad_norm": 68.03463485699294,
      "learning_rate": 5.813120704783539e-08,
      "logits": -2.1974759101867676,
      "logps": -91.99174499511719,
      "loss": 0.0258,
      "objective": 0.026935292407870293,
      "ranking_idealized": 0.6000000238418579,
      "ranking_idealized_expo": 0.5666666626930237,
      "ranking_simple": 0.5666666626930237,
      "regularize": 0.026934677734971046,
      "step": 1065
    },
    {
      "dpo_loss": 0.6922659277915955,
      "epoch": 1.0109440201558932,
      "grad_norm": 84.22043191701646,
      "learning_rate": 5.772392706677148e-08,
      "logits": -2.0366082191467285,
      "logps": -93.32905578613281,
      "loss": 0.0257,
      "objective": 0.026561260223388672,
      "ranking_idealized": 0.6333333253860474,
      "ranking_idealized_expo": 0.5666666626930237,
      "ranking_simple": 0.5833333134651184,
      "regularize": 0.026560688391327858,
      "step": 1070
    },
    {
      "dpo_loss": 0.6917497515678406,
      "epoch": 1.0156680576332573,
      "grad_norm": 73.42469612086076,
      "learning_rate": 5.7316121381361984e-08,
      "logits": -2.204793691635132,
      "logps": -94.83844757080078,
      "loss": 0.0269,
      "objective": 0.029431190341711044,
      "ranking_idealized": 0.5666666626930237,
      "ranking_idealized_expo": 0.4333333373069763,
      "ranking_simple": 0.46666666865348816,
      "regularize": 0.029430601745843887,
      "step": 1075
    },
    {
      "dpo_loss": 0.683849573135376,
      "epoch": 1.0203920951106211,
      "grad_norm": 69.22367476602035,
      "learning_rate": 5.690781774759411e-08,
      "logits": -2.2117373943328857,
      "logps": -94.56703186035156,
      "loss": 0.0276,
      "objective": 0.030149787664413452,
      "ranking_idealized": 0.5333333611488342,
      "ranking_idealized_expo": 0.44999998807907104,
      "ranking_simple": 0.46666666865348816,
      "regularize": 0.030148986726999283,
      "step": 1080
    },
    {
      "dpo_loss": 0.6918656229972839,
      "epoch": 1.0251161325879852,
      "grad_norm": 67.88816058800367,
      "learning_rate": 5.649904395534636e-08,
      "logits": -2.1058478355407715,
      "logps": -94.81607055664062,
      "loss": 0.0249,
      "objective": 0.025079350918531418,
      "ranking_idealized": 0.550000011920929,
      "ranking_idealized_expo": 0.4833333194255829,
      "ranking_simple": 0.4833333194255829,
      "regularize": 0.025078853592276573,
      "step": 1085
    },
    {
      "dpo_loss": 0.6931339502334595,
      "epoch": 1.0298401700653492,
      "grad_norm": 71.9727166820056,
      "learning_rate": 5.6089827826497026e-08,
      "logits": -2.2008562088012695,
      "logps": -93.15959930419922,
      "loss": 0.0233,
      "objective": 0.024377651512622833,
      "ranking_idealized": 0.6166666746139526,
      "ranking_idealized_expo": 0.4333333373069763,
      "ranking_simple": 0.44999998807907104,
      "regularize": 0.024377118796110153,
      "step": 1090
    },
    {
      "dpo_loss": 0.6928921341896057,
      "epoch": 1.0345642075427133,
      "grad_norm": 64.563119648666,
      "learning_rate": 5.568019721303068e-08,
      "logits": -2.146667957305908,
      "logps": -95.26697540283203,
      "loss": 0.0232,
      "objective": 0.022780917584896088,
      "ranking_idealized": 0.5666666626930237,
      "ranking_idealized_expo": 0.5166666507720947,
      "ranking_simple": 0.5166666507720947,
      "regularize": 0.022780440747737885,
      "step": 1095
    },
    {
      "dpo_loss": 0.6914113759994507,
      "epoch": 1.039288245020077,
      "grad_norm": 70.34249089351219,
      "learning_rate": 5.527017999514239e-08,
      "logits": -2.1238293647766113,
      "logps": -90.81373596191406,
      "loss": 0.0242,
      "objective": 0.019299499690532684,
      "ranking_idealized": 0.550000011920929,
      "ranking_idealized_expo": 0.5,
      "ranking_simple": 0.5,
      "regularize": 0.019298112019896507,
      "step": 1100
    },
    {
      "epoch": 1.039288245020077,
      "eval_dpo_loss": 0.692744791507721,
      "eval_logits": -1.9881205558776855,
      "eval_logps": -98.5120849609375,
      "eval_loss": 0.012820076197385788,
      "eval_objective": 0.012685425579547882,
      "eval_ranking_idealized": 0.6022099256515503,
      "eval_ranking_idealized_expo": 0.5207182168960571,
      "eval_ranking_simple": 0.5179557800292969,
      "eval_regularize": 0.012683761306107044,
      "eval_runtime": 445.6772,
      "eval_samples_per_second": 12.991,
      "eval_steps_per_second": 3.249,
      "step": 1100
    },
    {
      "dpo_loss": 0.6895564198493958,
      "epoch": 1.0440122824974412,
      "grad_norm": 69.83976607143053,
      "learning_rate": 5.4859804079340266e-08,
      "logits": -2.158614158630371,
      "logps": -88.3459243774414,
      "loss": 0.0234,
      "objective": 0.024073513224720955,
      "ranking_idealized": 0.5333333611488342,
      "ranking_idealized_expo": 0.4833333194255829,
      "ranking_simple": 0.4833333194255829,
      "regularize": 0.024072829633951187,
      "step": 1105
    },
    {
      "dpo_loss": 0.6888210773468018,
      "epoch": 1.0487363199748052,
      "grad_norm": 78.2561714772354,
      "learning_rate": 5.444909739654602e-08,
      "logits": -2.2234978675842285,
      "logps": -92.2721939086914,
      "loss": 0.025,
      "objective": 0.025087477639317513,
      "ranking_idealized": 0.5833333134651184,
      "ranking_idealized_expo": 0.4333333373069763,
      "ranking_simple": 0.44999998807907104,
      "regularize": 0.025086527690291405,
      "step": 1110
    },
    {
      "dpo_loss": 0.6934700608253479,
      "epoch": 1.053460357452169,
      "grad_norm": 67.69947794878017,
      "learning_rate": 5.4038087900193974e-08,
      "logits": -2.0514726638793945,
      "logps": -92.03214263916016,
      "loss": 0.0243,
      "objective": 0.020962979644536972,
      "ranking_idealized": 0.5666666626930237,
      "ranking_idealized_expo": 0.44999998807907104,
      "ranking_simple": 0.44999998807907104,
      "regularize": 0.020961163565516472,
      "step": 1115
    },
    {
      "dpo_loss": 0.6921118497848511,
      "epoch": 1.058184394929533,
      "grad_norm": 60.76759737021519,
      "learning_rate": 5.362680356432846e-08,
      "logits": -2.082772731781006,
      "logps": -91.51958465576172,
      "loss": 0.0226,
      "objective": 0.02336716279387474,
      "ranking_idealized": 0.5333333611488342,
      "ranking_idealized_expo": 0.4833333194255829,
      "ranking_simple": 0.4833333194255829,
      "regularize": 0.023366322740912437,
      "step": 1120
    },
    {
      "dpo_loss": 0.6900876760482788,
      "epoch": 1.0629084324068971,
      "grad_norm": 89.61755393855444,
      "learning_rate": 5.321527238169992e-08,
      "logits": -2.137908935546875,
      "logps": -94.91239166259766,
      "loss": 0.0258,
      "objective": 0.0195107851177454,
      "ranking_idealized": 0.550000011920929,
      "ranking_idealized_expo": 0.46666666865348816,
      "ranking_simple": 0.46666666865348816,
      "regularize": 0.01950863003730774,
      "step": 1125
    },
    {
      "dpo_loss": 0.6918936371803284,
      "epoch": 1.067632469884261,
      "grad_norm": 60.32020595039731,
      "learning_rate": 5.280352236185959e-08,
      "logits": -2.223163604736328,
      "logps": -94.0035629272461,
      "loss": 0.0198,
      "objective": 0.020925112068653107,
      "ranking_idealized": 0.5666666626930237,
      "ranking_idealized_expo": 0.550000011920929,
      "ranking_simple": 0.550000011920929,
      "regularize": 0.020924491807818413,
      "step": 1130
    },
    {
      "dpo_loss": 0.6936992406845093,
      "epoch": 1.072356507361625,
      "grad_norm": 68.85146603912719,
      "learning_rate": 5.239158152925319e-08,
      "logits": -2.089085102081299,
      "logps": -89.82282257080078,
      "loss": 0.0232,
      "objective": 0.02565601095557213,
      "ranking_idealized": 0.5333333611488342,
      "ranking_idealized_expo": 0.46666666865348816,
      "ranking_simple": 0.46666666865348816,
      "regularize": 0.025654161348938942,
      "step": 1135
    },
    {
      "dpo_loss": 0.6891080737113953,
      "epoch": 1.077080544838989,
      "grad_norm": 63.283810030024625,
      "learning_rate": 5.197947792131348e-08,
      "logits": -2.201981782913208,
      "logps": -93.19425964355469,
      "loss": 0.0252,
      "objective": 0.02514718845486641,
      "ranking_idealized": 0.6666666865348816,
      "ranking_idealized_expo": 0.5833333134651184,
      "ranking_simple": 0.5833333134651184,
      "regularize": 0.025146154686808586,
      "step": 1140
    },
    {
      "dpo_loss": 0.6902083158493042,
      "epoch": 1.0818045823163531,
      "grad_norm": 65.2698459996647,
      "learning_rate": 5.1567239586552e-08,
      "logits": -2.185304880142212,
      "logps": -90.79157257080078,
      "loss": 0.0219,
      "objective": 0.021113820374011993,
      "ranking_idealized": 0.5833333134651184,
      "ranking_idealized_expo": 0.5166666507720947,
      "ranking_simple": 0.5166666507720947,
      "regularize": 0.021113011986017227,
      "step": 1145
    },
    {
      "dpo_loss": 0.6910417675971985,
      "epoch": 1.086528619793717,
      "grad_norm": 65.02233557080358,
      "learning_rate": 5.115489458265005e-08,
      "logits": -2.1189420223236084,
      "logps": -94.17777252197266,
      "loss": 0.0248,
      "objective": 0.023960812017321587,
      "ranking_idealized": 0.5666666626930237,
      "ranking_idealized_expo": 0.5333333611488342,
      "ranking_simple": 0.5333333611488342,
      "regularize": 0.02395929954946041,
      "step": 1150
    },
    {
      "epoch": 1.086528619793717,
      "eval_dpo_loss": 0.6928678154945374,
      "eval_logits": -1.9900000095367432,
      "eval_logps": -98.37399291992188,
      "eval_loss": 0.01211391482502222,
      "eval_objective": 0.012414646334946156,
      "eval_ranking_idealized": 0.6022099256515503,
      "eval_ranking_idealized_expo": 0.5207182168960571,
      "eval_ranking_simple": 0.5179557800292969,
      "eval_regularize": 0.0124129019677639,
      "eval_runtime": 446.4802,
      "eval_samples_per_second": 12.968,
      "eval_steps_per_second": 3.243,
      "step": 1150
    },
    {
      "dpo_loss": 0.6903362274169922,
      "epoch": 1.091252657271081,
      "grad_norm": 65.0696209993928,
      "learning_rate": 5.0742470974549036e-08,
      "logits": -2.100759744644165,
      "logps": -90.93352508544922,
      "loss": 0.0233,
      "objective": 0.020316295325756073,
      "ranking_idealized": 0.6000000238418579,
      "ranking_idealized_expo": 0.5333333611488342,
      "ranking_simple": 0.5333333611488342,
      "regularize": 0.020315730944275856,
      "step": 1155
    },
    {
      "dpo_loss": 0.6921348571777344,
      "epoch": 1.095976694748445,
      "grad_norm": 68.18545652779869,
      "learning_rate": 5.032999683254028e-08,
      "logits": -2.1389288902282715,
      "logps": -90.10498046875,
      "loss": 0.0247,
      "objective": 0.021737979725003242,
      "ranking_idealized": 0.6499999761581421,
      "ranking_idealized_expo": 0.5833333134651184,
      "ranking_simple": 0.5833333134651184,
      "regularize": 0.021737171337008476,
      "step": 1160
    },
    {
      "dpo_loss": 0.69456547498703,
      "epoch": 1.100700732225809,
      "grad_norm": 73.7471857905923,
      "learning_rate": 4.991750023035455e-08,
      "logits": -2.124562978744507,
      "logps": -90.93301391601562,
      "loss": 0.024,
      "objective": 0.02404080517590046,
      "ranking_idealized": 0.6666666865348816,
      "ranking_idealized_expo": 0.6000000238418579,
      "ranking_simple": 0.6000000238418579,
      "regularize": 0.02403992973268032,
      "step": 1165
    },
    {
      "dpo_loss": 0.6940571069717407,
      "epoch": 1.105424769703173,
      "grad_norm": 67.32632108670151,
      "learning_rate": 4.950500924325127e-08,
      "logits": -2.2072455883026123,
      "logps": -94.81490325927734,
      "loss": 0.0255,
      "objective": 0.026444217190146446,
      "ranking_idealized": 0.4833333194255829,
      "ranking_idealized_expo": 0.4166666567325592,
      "ranking_simple": 0.4333333373069763,
      "regularize": 0.026443663984537125,
      "step": 1170
    },
    {
      "dpo_loss": 0.692010223865509,
      "epoch": 1.110148807180537,
      "grad_norm": 60.40484735228916,
      "learning_rate": 4.909255194610773e-08,
      "logits": -2.1622235774993896,
      "logps": -90.50462341308594,
      "loss": 0.0217,
      "objective": 0.01786196231842041,
      "ranking_idealized": 0.5333333611488342,
      "ranking_idealized_expo": 0.4833333194255829,
      "ranking_simple": 0.5,
      "regularize": 0.01785971410572529,
      "step": 1175
    },
    {
      "dpo_loss": 0.6931031942367554,
      "epoch": 1.114872844657901,
      "grad_norm": 59.38906623535502,
      "learning_rate": 4.8680156411508193e-08,
      "logits": -2.16975474357605,
      "logps": -89.3101806640625,
      "loss": 0.0243,
      "objective": 0.01995784044265747,
      "ranking_idealized": 0.5833333134651184,
      "ranking_idealized_expo": 0.44999998807907104,
      "ranking_simple": 0.44999998807907104,
      "regularize": 0.019954947754740715,
      "step": 1180
    },
    {
      "dpo_loss": 0.690836489200592,
      "epoch": 1.1195968821352649,
      "grad_norm": 68.89482324651908,
      "learning_rate": 4.826785070783326e-08,
      "logits": -2.058103084564209,
      "logps": -91.24579620361328,
      "loss": 0.0233,
      "objective": 0.022483140230178833,
      "ranking_idealized": 0.6499999761581421,
      "ranking_idealized_expo": 0.6166666746139526,
      "ranking_simple": 0.6166666746139526,
      "regularize": 0.02248191460967064,
      "step": 1185
    },
    {
      "dpo_loss": 0.6914324164390564,
      "epoch": 1.124320919612629,
      "grad_norm": 61.796191905639844,
      "learning_rate": 4.7855662897349464e-08,
      "logits": -2.1661124229431152,
      "logps": -91.51298522949219,
      "loss": 0.0226,
      "objective": 0.018483061343431473,
      "ranking_idealized": 0.6166666746139526,
      "ranking_idealized_expo": 0.5333333611488342,
      "ranking_simple": 0.5166666507720947,
      "regularize": 0.018482128158211708,
      "step": 1190
    },
    {
      "dpo_loss": 0.6916797161102295,
      "epoch": 1.129044957089993,
      "grad_norm": 63.413323128276396,
      "learning_rate": 4.744362103429933e-08,
      "logits": -2.204550266265869,
      "logps": -90.19840240478516,
      "loss": 0.0244,
      "objective": 0.025325793772935867,
      "ranking_idealized": 0.6499999761581421,
      "ranking_idealized_expo": 0.6000000238418579,
      "ranking_simple": 0.6000000238418579,
      "regularize": 0.02532271295785904,
      "step": 1195
    },
    {
      "dpo_loss": 0.6876908540725708,
      "epoch": 1.1337689945673568,
      "grad_norm": 66.08019689679915,
      "learning_rate": 4.703175316299196e-08,
      "logits": -2.147653341293335,
      "logps": -94.6951904296875,
      "loss": 0.0238,
      "objective": 0.022918345406651497,
      "ranking_idealized": 0.699999988079071,
      "ranking_idealized_expo": 0.5666666626930237,
      "ranking_simple": 0.5666666626930237,
      "regularize": 0.022917365655303,
      "step": 1200
    },
    {
      "epoch": 1.1337689945673568,
      "eval_dpo_loss": 0.6931039094924927,
      "eval_logits": -1.988110899925232,
      "eval_logps": -98.65231323242188,
      "eval_loss": 0.013076459057629108,
      "eval_objective": 0.013204570859670639,
      "eval_ranking_idealized": 0.6022099256515503,
      "eval_ranking_idealized_expo": 0.5207182168960571,
      "eval_ranking_simple": 0.5186464190483093,
      "eval_regularize": 0.013202749192714691,
      "eval_runtime": 446.1352,
      "eval_samples_per_second": 12.978,
      "eval_steps_per_second": 3.246,
      "step": 1200
    },
    {
      "dpo_loss": 0.6895859837532043,
      "epoch": 1.1384930320447209,
      "grad_norm": 59.82159099357435,
      "learning_rate": 4.662008731589424e-08,
      "logits": -2.2835893630981445,
      "logps": -93.41173553466797,
      "loss": 0.0238,
      "objective": 0.01953883096575737,
      "ranking_idealized": 0.5666666626930237,
      "ranking_idealized_expo": 0.5166666507720947,
      "ranking_simple": 0.5166666507720947,
      "regularize": 0.019538111984729767,
      "step": 1205
    },
    {
      "dpo_loss": 0.6878269910812378,
      "epoch": 1.143217069522085,
      "grad_norm": 64.13908923913202,
      "learning_rate": 4.6208651511722916e-08,
      "logits": -2.107128381729126,
      "logps": -95.02501678466797,
      "loss": 0.0225,
      "objective": 0.02028297260403633,
      "ranking_idealized": 0.5833333134651184,
      "ranking_idealized_expo": 0.44999998807907104,
      "ranking_simple": 0.4833333194255829,
      "regularize": 0.020282411947846413,
      "step": 1210
    },
    {
      "dpo_loss": 0.6913109421730042,
      "epoch": 1.147941106999449,
      "grad_norm": 64.08267971583916,
      "learning_rate": 4.579747375353763e-08,
      "logits": -2.152080774307251,
      "logps": -93.3291244506836,
      "loss": 0.0225,
      "objective": 0.02415003441274166,
      "ranking_idealized": 0.550000011920929,
      "ranking_idealized_expo": 0.4833333194255829,
      "ranking_simple": 0.4833333194255829,
      "regularize": 0.024148976430296898,
      "step": 1215
    },
    {
      "dpo_loss": 0.6908139586448669,
      "epoch": 1.1526651444768128,
      "grad_norm": 61.90111498957097,
      "learning_rate": 4.5386582026834904e-08,
      "logits": -2.2181687355041504,
      "logps": -91.48564147949219,
      "loss": 0.0211,
      "objective": 0.01728188991546631,
      "ranking_idealized": 0.5333333611488342,
      "ranking_idealized_expo": 0.44999998807907104,
      "ranking_simple": 0.44999998807907104,
      "regularize": 0.017280517145991325,
      "step": 1220
    },
    {
      "dpo_loss": 0.6892186999320984,
      "epoch": 1.1573891819541768,
      "grad_norm": 67.90586520467559,
      "learning_rate": 4.497600429764349e-08,
      "logits": -2.0878336429595947,
      "logps": -92.51049041748047,
      "loss": 0.023,
      "objective": 0.021265888586640358,
      "ranking_idealized": 0.5833333134651184,
      "ranking_idealized_expo": 0.5166666507720947,
      "ranking_simple": 0.5,
      "regularize": 0.021264949813485146,
      "step": 1225
    },
    {
      "dpo_loss": 0.6902192831039429,
      "epoch": 1.162113219431541,
      "grad_norm": 69.62062658592635,
      "learning_rate": 4.456576851062089e-08,
      "logits": -2.1754400730133057,
      "logps": -89.9488296508789,
      "loss": 0.0202,
      "objective": 0.015848658978939056,
      "ranking_idealized": 0.6666666865348816,
      "ranking_idealized_expo": 0.550000011920929,
      "ranking_simple": 0.5666666626930237,
      "regularize": 0.015847818925976753,
      "step": 1230
    },
    {
      "dpo_loss": 0.6919657588005066,
      "epoch": 1.1668372569089047,
      "grad_norm": 76.5213820049961,
      "learning_rate": 4.4155902587151404e-08,
      "logits": -2.2707595825195312,
      "logps": -89.4555435180664,
      "loss": 0.0231,
      "objective": 0.02043619193136692,
      "ranking_idealized": 0.6000000238418579,
      "ranking_idealized_expo": 0.5333333611488342,
      "ranking_simple": 0.5333333611488342,
      "regularize": 0.020434273406863213,
      "step": 1235
    },
    {
      "dpo_loss": 0.6887553930282593,
      "epoch": 1.1715612943862688,
      "grad_norm": 68.85342051503754,
      "learning_rate": 4.374643442344576e-08,
      "logits": -2.1689367294311523,
      "logps": -91.3507080078125,
      "loss": 0.022,
      "objective": 0.02212885580956936,
      "ranking_idealized": 0.6000000238418579,
      "ranking_idealized_expo": 0.5833333134651184,
      "ranking_simple": 0.5833333134651184,
      "regularize": 0.02212790958583355,
      "step": 1240
    },
    {
      "dpo_loss": 0.6907090544700623,
      "epoch": 1.1762853318636328,
      "grad_norm": 63.9895529963574,
      "learning_rate": 4.333739188864243e-08,
      "logits": -2.116743564605713,
      "logps": -89.35691833496094,
      "loss": 0.0231,
      "objective": 0.02183300256729126,
      "ranking_idealized": 0.5333333611488342,
      "ranking_idealized_expo": 0.5,
      "ranking_simple": 0.5,
      "regularize": 0.021832352504134178,
      "step": 1245
    },
    {
      "dpo_loss": 0.6873170733451843,
      "epoch": 1.1810093693409969,
      "grad_norm": 62.000344928337206,
      "learning_rate": 4.292880282291083e-08,
      "logits": -2.0605881214141846,
      "logps": -89.00260925292969,
      "loss": 0.0213,
      "objective": 0.016554510220885277,
      "ranking_idealized": 0.550000011920929,
      "ranking_idealized_expo": 0.4833333194255829,
      "ranking_simple": 0.4833333194255829,
      "regularize": 0.016552967950701714,
      "step": 1250
    },
    {
      "epoch": 1.1810093693409969,
      "eval_dpo_loss": 0.6929011344909668,
      "eval_logits": -1.9892135858535767,
      "eval_logps": -98.38199615478516,
      "eval_loss": 0.011604293249547482,
      "eval_objective": 0.011785290203988552,
      "eval_ranking_idealized": 0.6022099256515503,
      "eval_ranking_idealized_expo": 0.5207182168960571,
      "eval_ranking_simple": 0.5186464190483093,
      "eval_regularize": 0.011783457361161709,
      "eval_runtime": 446.5262,
      "eval_samples_per_second": 12.967,
      "eval_steps_per_second": 3.243,
      "step": 1250
    },
    {
      "dpo_loss": 0.6883438229560852,
      "epoch": 1.1857334068183607,
      "grad_norm": 76.14193268278439,
      "learning_rate": 4.2520695035556444e-08,
      "logits": -2.238811731338501,
      "logps": -93.17744445800781,
      "loss": 0.0227,
      "objective": 0.02147439494729042,
      "ranking_idealized": 0.4166666567325592,
      "ranking_idealized_expo": 0.4000000059604645,
      "ranking_simple": 0.4000000059604645,
      "regularize": 0.021473314613103867,
      "step": 1255
    },
    {
      "dpo_loss": 0.6898403763771057,
      "epoch": 1.1904574442957248,
      "grad_norm": 72.14873570982253,
      "learning_rate": 4.211309630312812e-08,
      "logits": -2.186509847640991,
      "logps": -92.71757507324219,
      "loss": 0.0221,
      "objective": 0.024685295298695564,
      "ranking_idealized": 0.6666666865348816,
      "ranking_idealized_expo": 0.6166666746139526,
      "ranking_simple": 0.6000000238418579,
      "regularize": 0.02468358352780342,
      "step": 1260
    },
    {
      "dpo_loss": 0.6934939026832581,
      "epoch": 1.1951814817730888,
      "grad_norm": 76.43192783166353,
      "learning_rate": 4.1706034367527484e-08,
      "logits": -2.2296221256256104,
      "logps": -90.43429565429688,
      "loss": 0.0216,
      "objective": 0.017879430204629898,
      "ranking_idealized": 0.6000000238418579,
      "ranking_idealized_expo": 0.5,
      "ranking_simple": 0.5,
      "regularize": 0.017877496778964996,
      "step": 1265
    },
    {
      "dpo_loss": 0.6907635927200317,
      "epoch": 1.1999055192504526,
      "grad_norm": 63.68697185614875,
      "learning_rate": 4.12995369341208e-08,
      "logits": -2.2198116779327393,
      "logps": -89.92237854003906,
      "loss": 0.0186,
      "objective": 0.019899163395166397,
      "ranking_idealized": 0.6000000238418579,
      "ranking_idealized_expo": 0.5,
      "ranking_simple": 0.5,
      "regularize": 0.019898338243365288,
      "step": 1270
    },
    {
      "dpo_loss": 0.6885548233985901,
      "epoch": 1.2046295567278167,
      "grad_norm": 67.30508547665502,
      "learning_rate": 4.0893631669853315e-08,
      "logits": -2.2070553302764893,
      "logps": -91.00017547607422,
      "loss": 0.0213,
      "objective": 0.0249098539352417,
      "ranking_idealized": 0.6833333373069763,
      "ranking_idealized_expo": 0.5833333134651184,
      "ranking_simple": 0.5833333134651184,
      "regularize": 0.024909144267439842,
      "step": 1275
    },
    {
      "dpo_loss": 0.6864418387413025,
      "epoch": 1.2093535942051807,
      "grad_norm": 58.37239845591427,
      "learning_rate": 4.048834620136618e-08,
      "logits": -2.157111406326294,
      "logps": -90.54441833496094,
      "loss": 0.0216,
      "objective": 0.024475712329149246,
      "ranking_idealized": 0.6333333253860474,
      "ranking_idealized_expo": 0.5833333134651184,
      "ranking_simple": 0.5833333134651184,
      "regularize": 0.02447471395134926,
      "step": 1280
    },
    {
      "dpo_loss": 0.6923628449440002,
      "epoch": 1.2140776316825446,
      "grad_norm": 62.390419253758,
      "learning_rate": 4.0083708113116125e-08,
      "logits": -2.20066499710083,
      "logps": -91.0829849243164,
      "loss": 0.0194,
      "objective": 0.017484158277511597,
      "ranking_idealized": 0.699999988079071,
      "ranking_idealized_expo": 0.6499999761581421,
      "ranking_simple": 0.6499999761581421,
      "regularize": 0.017483441159129143,
      "step": 1285
    },
    {
      "dpo_loss": 0.6917316317558289,
      "epoch": 1.2188016691599086,
      "grad_norm": 62.080679222088676,
      "learning_rate": 3.9679744945498026e-08,
      "logits": -2.0995683670043945,
      "logps": -89.5858154296875,
      "loss": 0.0215,
      "objective": 0.019274141639471054,
      "ranking_idealized": 0.5333333611488342,
      "ranking_idealized_expo": 0.5,
      "ranking_simple": 0.5,
      "regularize": 0.01927088387310505,
      "step": 1290
    },
    {
      "dpo_loss": 0.6907222867012024,
      "epoch": 1.2235257066372727,
      "grad_norm": 69.35505760627515,
      "learning_rate": 3.9276484192970427e-08,
      "logits": -2.0752005577087402,
      "logps": -88.9092788696289,
      "loss": 0.0202,
      "objective": 0.01790427789092064,
      "ranking_idealized": 0.6499999761581421,
      "ranking_idealized_expo": 0.5833333134651184,
      "ranking_simple": 0.5833333134651184,
      "regularize": 0.017903130501508713,
      "step": 1295
    },
    {
      "dpo_loss": 0.6888744831085205,
      "epoch": 1.2282497441146367,
      "grad_norm": 64.13387937854051,
      "learning_rate": 3.887395330218428e-08,
      "logits": -2.088010549545288,
      "logps": -93.63009643554688,
      "loss": 0.0213,
      "objective": 0.01880194991827011,
      "ranking_idealized": 0.6333333253860474,
      "ranking_idealized_expo": 0.5666666626930237,
      "ranking_simple": 0.5666666626930237,
      "regularize": 0.018800783902406693,
      "step": 1300
    },
    {
      "epoch": 1.2282497441146367,
      "eval_dpo_loss": 0.6930280923843384,
      "eval_logits": -1.9901354312896729,
      "eval_logps": -98.35194396972656,
      "eval_loss": 0.010079173371195793,
      "eval_objective": 0.010304316878318787,
      "eval_ranking_idealized": 0.6022099256515503,
      "eval_ranking_idealized_expo": 0.5207182168960571,
      "eval_ranking_simple": 0.5179557800292969,
      "eval_regularize": 0.010302502661943436,
      "eval_runtime": 446.7461,
      "eval_samples_per_second": 12.96,
      "eval_steps_per_second": 3.241,
      "step": 1300
    },
    {
      "dpo_loss": 0.6898637413978577,
      "epoch": 1.2329737815920006,
      "grad_norm": 64.4984732207436,
      "learning_rate": 3.847217967011481e-08,
      "logits": -2.107663154602051,
      "logps": -90.70755004882812,
      "loss": 0.0188,
      "objective": 0.020963182672858238,
      "ranking_idealized": 0.4333333373069763,
      "ranking_idealized_expo": 0.4000000059604645,
      "ranking_simple": 0.4000000059604645,
      "regularize": 0.020961280912160873,
      "step": 1305
    },
    {
      "dpo_loss": 0.6896305084228516,
      "epoch": 1.2376978190693646,
      "grad_norm": 65.1483177905117,
      "learning_rate": 3.807119064219686e-08,
      "logits": -2.1721391677856445,
      "logps": -89.53897094726562,
      "loss": 0.0187,
      "objective": 0.022347215563058853,
      "ranking_idealized": 0.6833333373069763,
      "ranking_idealized_expo": 0.6000000238418579,
      "ranking_simple": 0.6000000238418579,
      "regularize": 0.02234494686126709,
      "step": 1310
    },
    {
      "dpo_loss": 0.6885426044464111,
      "epoch": 1.2424218565467287,
      "grad_norm": 67.6302362675184,
      "learning_rate": 3.7671013510463685e-08,
      "logits": -2.1977858543395996,
      "logps": -90.79574584960938,
      "loss": 0.0254,
      "objective": 0.021330129355192184,
      "ranking_idealized": 0.6000000238418579,
      "ranking_idealized_expo": 0.5333333611488342,
      "ranking_simple": 0.550000011920929,
      "regularize": 0.02132764458656311,
      "step": 1315
    },
    {
      "dpo_loss": 0.6933093667030334,
      "epoch": 1.2471458940240927,
      "grad_norm": 63.75946991656521,
      "learning_rate": 3.727167551168947e-08,
      "logits": -2.229327917098999,
      "logps": -91.51911163330078,
      "loss": 0.0225,
      "objective": 0.01876850612461567,
      "ranking_idealized": 0.5666666626930237,
      "ranking_idealized_expo": 0.46666666865348816,
      "ranking_simple": 0.46666666865348816,
      "regularize": 0.01876780204474926,
      "step": 1320
    },
    {
      "dpo_loss": 0.6886643171310425,
      "epoch": 1.2518699315014565,
      "grad_norm": 61.93022907625324,
      "learning_rate": 3.687320382553547e-08,
      "logits": -2.1852173805236816,
      "logps": -94.50476837158203,
      "loss": 0.0213,
      "objective": 0.019077714532613754,
      "ranking_idealized": 0.6666666865348816,
      "ranking_idealized_expo": 0.5666666626930237,
      "ranking_simple": 0.5666666626930237,
      "regularize": 0.01907687447965145,
      "step": 1325
    },
    {
      "dpo_loss": 0.690542459487915,
      "epoch": 1.2565939689788206,
      "grad_norm": 61.862265225762556,
      "learning_rate": 3.6475625572700156e-08,
      "logits": -2.1126949787139893,
      "logps": -93.08950805664062,
      "loss": 0.0201,
      "objective": 0.020278314128518105,
      "ranking_idealized": 0.550000011920929,
      "ranking_idealized_expo": 0.5166666507720947,
      "ranking_simple": 0.5166666507720947,
      "regularize": 0.020277447998523712,
      "step": 1330
    },
    {
      "dpo_loss": 0.6890642046928406,
      "epoch": 1.2613180064561846,
      "grad_norm": 65.89840804248666,
      "learning_rate": 3.607896781307333e-08,
      "logits": -2.0817463397979736,
      "logps": -93.81517791748047,
      "loss": 0.0186,
      "objective": 0.018369121477007866,
      "ranking_idealized": 0.6333333253860474,
      "ranking_idealized_expo": 0.44999998807907104,
      "ranking_simple": 0.44999998807907104,
      "regularize": 0.01836659200489521,
      "step": 1335
    },
    {
      "dpo_loss": 0.6935243606567383,
      "epoch": 1.2660420439335485,
      "grad_norm": 61.73867224830495,
      "learning_rate": 3.5683257543894376e-08,
      "logits": -2.158568859100342,
      "logps": -91.2880859375,
      "loss": 0.0199,
      "objective": 0.02297687530517578,
      "ranking_idealized": 0.6333333253860474,
      "ranking_idealized_expo": 0.5166666507720947,
      "ranking_simple": 0.5166666507720947,
      "regularize": 0.022976113483309746,
      "step": 1340
    },
    {
      "dpo_loss": 0.6870158910751343,
      "epoch": 1.2707660814109125,
      "grad_norm": 55.58597973847245,
      "learning_rate": 3.528852169791474e-08,
      "logits": -2.130025625228882,
      "logps": -92.3035888671875,
      "loss": 0.0191,
      "objective": 0.018635360524058342,
      "ranking_idealized": 0.5833333134651184,
      "ranking_idealized_expo": 0.5666666626930237,
      "ranking_simple": 0.5666666626930237,
      "regularize": 0.018634265288710594,
      "step": 1345
    },
    {
      "dpo_loss": 0.6925967335700989,
      "epoch": 1.2754901188882766,
      "grad_norm": 60.169587280433916,
      "learning_rate": 3.489478714156493e-08,
      "logits": -2.1538307666778564,
      "logps": -90.29386901855469,
      "loss": 0.0191,
      "objective": 0.01989407278597355,
      "ranking_idealized": 0.6499999761581421,
      "ranking_idealized_expo": 0.5833333134651184,
      "ranking_simple": 0.5833333134651184,
      "regularize": 0.01989228092133999,
      "step": 1350
    },
    {
      "epoch": 1.2754901188882766,
      "eval_dpo_loss": 0.6928868293762207,
      "eval_logits": -1.9894771575927734,
      "eval_logps": -98.17080688476562,
      "eval_loss": 0.010478594340384007,
      "eval_objective": 0.01073493529111147,
      "eval_ranking_idealized": 0.6022099256515503,
      "eval_ranking_idealized_expo": 0.5207182168960571,
      "eval_ranking_simple": 0.5186464190483093,
      "eval_regularize": 0.01073309313505888,
      "eval_runtime": 446.5009,
      "eval_samples_per_second": 12.967,
      "eval_steps_per_second": 3.243,
      "step": 1350
    },
    {
      "dpo_loss": 0.6914752721786499,
      "epoch": 1.2802141563656404,
      "grad_norm": 63.02270194963686,
      "learning_rate": 3.450208067312586e-08,
      "logits": -2.1492412090301514,
      "logps": -89.29891967773438,
      "loss": 0.019,
      "objective": 0.017367955297231674,
      "ranking_idealized": 0.5,
      "ranking_idealized_expo": 0.46666666865348816,
      "ranking_simple": 0.46666666865348816,
      "regularize": 0.01736696995794773,
      "step": 1355
    },
    {
      "dpo_loss": 0.6908875703811646,
      "epoch": 1.2849381938430045,
      "grad_norm": 66.84237286020337,
      "learning_rate": 3.411042902090492e-08,
      "logits": -2.2156636714935303,
      "logps": -91.21939849853516,
      "loss": 0.0183,
      "objective": 0.016017399728298187,
      "ranking_idealized": 0.6000000238418579,
      "ranking_idealized_expo": 0.5166666507720947,
      "ranking_simple": 0.5166666507720947,
      "regularize": 0.016014499589800835,
      "step": 1360
    },
    {
      "dpo_loss": 0.6913639307022095,
      "epoch": 1.2896622313203685,
      "grad_norm": 76.89545336747237,
      "learning_rate": 3.3719858841416836e-08,
      "logits": -2.1557438373565674,
      "logps": -92.71784210205078,
      "loss": 0.0196,
      "objective": 0.019386129453778267,
      "ranking_idealized": 0.5666666626930237,
      "ranking_idealized_expo": 0.5166666507720947,
      "ranking_simple": 0.5166666507720947,
      "regularize": 0.019384343177080154,
      "step": 1365
    },
    {
      "dpo_loss": 0.6889938712120056,
      "epoch": 1.2943862687977323,
      "grad_norm": 56.519950070251724,
      "learning_rate": 3.333039671756934e-08,
      "logits": -2.055145502090454,
      "logps": -90.29429626464844,
      "loss": 0.0191,
      "objective": 0.019297009333968163,
      "ranking_idealized": 0.5666666626930237,
      "ranking_idealized_expo": 0.4833333194255829,
      "ranking_simple": 0.5,
      "regularize": 0.01929611526429653,
      "step": 1370
    },
    {
      "dpo_loss": 0.689259946346283,
      "epoch": 1.2991103062750964,
      "grad_norm": 73.9021579793652,
      "learning_rate": 3.294206915685392e-08,
      "logits": -2.2613272666931152,
      "logps": -95.13359832763672,
      "loss": 0.0184,
      "objective": 0.015245441347360611,
      "ranking_idealized": 0.6666666865348816,
      "ranking_idealized_expo": 0.5833333134651184,
      "ranking_simple": 0.5833333134651184,
      "regularize": 0.01524441223591566,
      "step": 1375
    },
    {
      "dpo_loss": 0.6903609037399292,
      "epoch": 1.3038343437524604,
      "grad_norm": 63.39814646096018,
      "learning_rate": 3.2554902589541666e-08,
      "logits": -2.1530189514160156,
      "logps": -90.94368743896484,
      "loss": 0.0174,
      "objective": 0.017877008765935898,
      "ranking_idealized": 0.7166666388511658,
      "ranking_idealized_expo": 0.6499999761581421,
      "ranking_simple": 0.6499999761581421,
      "regularize": 0.017875246703624725,
      "step": 1380
    },
    {
      "dpo_loss": 0.6925813555717468,
      "epoch": 1.3085583812298245,
      "grad_norm": 67.03876383480757,
      "learning_rate": 3.216892336688435e-08,
      "logits": -2.162677526473999,
      "logps": -91.54695892333984,
      "loss": 0.0196,
      "objective": 0.019883565604686737,
      "ranking_idealized": 0.5666666626930237,
      "ranking_idealized_expo": 0.5,
      "ranking_simple": 0.5,
      "regularize": 0.019882572814822197,
      "step": 1385
    },
    {
      "dpo_loss": 0.690301775932312,
      "epoch": 1.3132824187071885,
      "grad_norm": 65.28692296573145,
      "learning_rate": 3.1784157759320954e-08,
      "logits": -2.116351842880249,
      "logps": -91.4957504272461,
      "loss": 0.018,
      "objective": 0.015376557596027851,
      "ranking_idealized": 0.6833333373069763,
      "ranking_idealized_expo": 0.6333333253860474,
      "ranking_simple": 0.6333333253860474,
      "regularize": 0.015375560149550438,
      "step": 1390
    },
    {
      "dpo_loss": 0.6893251538276672,
      "epoch": 1.3180064561845524,
      "grad_norm": 62.76399037621205,
      "learning_rate": 3.140063195468962e-08,
      "logits": -2.163536548614502,
      "logps": -92.03372192382812,
      "loss": 0.0177,
      "objective": 0.017055170610547066,
      "ranking_idealized": 0.5333333611488342,
      "ranking_idealized_expo": 0.5,
      "ranking_simple": 0.5,
      "regularize": 0.017053287476301193,
      "step": 1395
    },
    {
      "dpo_loss": 0.6904376745223999,
      "epoch": 1.3227304936619164,
      "grad_norm": 66.03868596298913,
      "learning_rate": 3.101837205644531e-08,
      "logits": -2.1526260375976562,
      "logps": -91.71768951416016,
      "loss": 0.0183,
      "objective": 0.017484767362475395,
      "ranking_idealized": 0.5666666626930237,
      "ranking_idealized_expo": 0.5,
      "ranking_simple": 0.5,
      "regularize": 0.017483625560998917,
      "step": 1400
    },
    {
      "epoch": 1.3227304936619164,
      "eval_dpo_loss": 0.6928439736366272,
      "eval_logits": -1.9895795583724976,
      "eval_logps": -98.29888916015625,
      "eval_loss": 0.009779366664588451,
      "eval_objective": 0.009947007521986961,
      "eval_ranking_idealized": 0.6022099256515503,
      "eval_ranking_idealized_expo": 0.5207182168960571,
      "eval_ranking_simple": 0.5179557800292969,
      "eval_regularize": 0.009945098310709,
      "eval_runtime": 445.875,
      "eval_samples_per_second": 12.986,
      "eval_steps_per_second": 3.248,
      "step": 1400
    },
    {
      "dpo_loss": 0.6910092234611511,
      "epoch": 1.3274545311392805,
      "grad_norm": 60.84233260746348,
      "learning_rate": 3.063740408188308e-08,
      "logits": -2.129271984100342,
      "logps": -88.37971496582031,
      "loss": 0.0189,
      "objective": 0.02033030241727829,
      "ranking_idealized": 0.6833333373069763,
      "ranking_idealized_expo": 0.5833333134651184,
      "ranking_simple": 0.5666666626930237,
      "regularize": 0.020329667255282402,
      "step": 1405
    },
    {
      "dpo_loss": 0.68757563829422,
      "epoch": 1.3321785686166443,
      "grad_norm": 66.02099749274366,
      "learning_rate": 3.0257753960367374e-08,
      "logits": -2.15506911277771,
      "logps": -94.14618682861328,
      "loss": 0.0203,
      "objective": 0.022749019786715508,
      "ranking_idealized": 0.6166666746139526,
      "ranking_idealized_expo": 0.5166666507720947,
      "ranking_simple": 0.5166666507720947,
      "regularize": 0.022747376933693886,
      "step": 1410
    },
    {
      "dpo_loss": 0.6922497153282166,
      "epoch": 1.3369026060940084,
      "grad_norm": 69.66802628356685,
      "learning_rate": 2.987944753156717e-08,
      "logits": -2.111666440963745,
      "logps": -91.04405975341797,
      "loss": 0.0182,
      "objective": 0.0166956577450037,
      "ranking_idealized": 0.6666666865348816,
      "ranking_idealized_expo": 0.6499999761581421,
      "ranking_simple": 0.6499999761581421,
      "regularize": 0.016694890335202217,
      "step": 1415
    },
    {
      "dpo_loss": 0.6890503168106079,
      "epoch": 1.3416266435713724,
      "grad_norm": 60.95975646160118,
      "learning_rate": 2.9502510543697322e-08,
      "logits": -2.153930902481079,
      "logps": -90.40367889404297,
      "loss": 0.0168,
      "objective": 0.01781788095831871,
      "ranking_idealized": 0.5666666626930237,
      "ranking_idealized_expo": 0.5,
      "ranking_simple": 0.4833333194255829,
      "regularize": 0.017816245555877686,
      "step": 1420
    },
    {
      "dpo_loss": 0.6874597668647766,
      "epoch": 1.3463506810487362,
      "grad_norm": 59.58581498665142,
      "learning_rate": 2.912696865176607e-08,
      "logits": -2.243446111679077,
      "logps": -96.16178894042969,
      "loss": 0.0177,
      "objective": 0.023015646263957024,
      "ranking_idealized": 0.6499999761581421,
      "ranking_idealized_expo": 0.6166666746139526,
      "ranking_simple": 0.6000000238418579,
      "regularize": 0.02301453799009323,
      "step": 1425
    },
    {
      "dpo_loss": 0.6897552013397217,
      "epoch": 1.3510747185261003,
      "grad_norm": 60.74669762947064,
      "learning_rate": 2.875284741582892e-08,
      "logits": -2.1864495277404785,
      "logps": -96.33065032958984,
      "loss": 0.017,
      "objective": 0.015690678730607033,
      "ranking_idealized": 0.5666666626930237,
      "ranking_idealized_expo": 0.4833333194255829,
      "ranking_simple": 0.4833333194255829,
      "regularize": 0.015688156709074974,
      "step": 1430
    },
    {
      "dpo_loss": 0.6917933821678162,
      "epoch": 1.3557987560034643,
      "grad_norm": 52.60951888043,
      "learning_rate": 2.838017229924894e-08,
      "logits": -2.142225742340088,
      "logps": -92.60111999511719,
      "loss": 0.0173,
      "objective": 0.01731746830046177,
      "ranking_idealized": 0.6166666746139526,
      "ranking_idealized_expo": 0.550000011920929,
      "ranking_simple": 0.550000011920929,
      "regularize": 0.01731485314667225,
      "step": 1435
    },
    {
      "dpo_loss": 0.6895142793655396,
      "epoch": 1.3605227934808282,
      "grad_norm": 72.2797304152481,
      "learning_rate": 2.8008968666963817e-08,
      "logits": -2.1638967990875244,
      "logps": -92.39698028564453,
      "loss": 0.0165,
      "objective": 0.017737431451678276,
      "ranking_idealized": 0.5166666507720947,
      "ranking_idealized_expo": 0.4833333194255829,
      "ranking_simple": 0.46666666865348816,
      "regularize": 0.017736157402396202,
      "step": 1440
    },
    {
      "dpo_loss": 0.689642608165741,
      "epoch": 1.3652468309581922,
      "grad_norm": 79.06644823842738,
      "learning_rate": 2.763926178375929e-08,
      "logits": -2.190371036529541,
      "logps": -91.85442352294922,
      "loss": 0.0148,
      "objective": 0.01207685936242342,
      "ranking_idealized": 0.5833333134651184,
      "ranking_idealized_expo": 0.550000011920929,
      "ranking_simple": 0.550000011920929,
      "regularize": 0.012074961327016354,
      "step": 1445
    },
    {
      "dpo_loss": 0.690252423286438,
      "epoch": 1.3699708684355563,
      "grad_norm": 61.578467113272715,
      "learning_rate": 2.7271076812549688e-08,
      "logits": -2.2324106693267822,
      "logps": -91.74723815917969,
      "loss": 0.0173,
      "objective": 0.020336376503109932,
      "ranking_idealized": 0.5666666626930237,
      "ranking_idealized_expo": 0.4333333373069763,
      "ranking_simple": 0.44999998807907104,
      "regularize": 0.02033485844731331,
      "step": 1450
    },
    {
      "epoch": 1.3699708684355563,
      "eval_dpo_loss": 0.6928586959838867,
      "eval_logits": -1.9887967109680176,
      "eval_logps": -98.44745635986328,
      "eval_loss": 0.01200713962316513,
      "eval_objective": 0.012023248709738255,
      "eval_ranking_idealized": 0.6022099256515503,
      "eval_ranking_idealized_expo": 0.5207182168960571,
      "eval_ranking_simple": 0.519336998462677,
      "eval_regularize": 0.012021254748106003,
      "eval_runtime": 446.5411,
      "eval_samples_per_second": 12.966,
      "eval_steps_per_second": 3.243,
      "step": 1450
    },
    {
      "dpo_loss": 0.6905301809310913,
      "epoch": 1.3746949059129203,
      "grad_norm": 63.36787972947676,
      "learning_rate": 2.6904438812665275e-08,
      "logits": -2.1534087657928467,
      "logps": -90.40296936035156,
      "loss": 0.0161,
      "objective": 0.01465876679867506,
      "ranking_idealized": 0.5666666626930237,
      "ranking_idealized_expo": 0.5333333611488342,
      "ranking_simple": 0.5333333611488342,
      "regularize": 0.01465718261897564,
      "step": 1455
    },
    {
      "dpo_loss": 0.6913805603981018,
      "epoch": 1.3794189433902841,
      "grad_norm": 82.33504933507443,
      "learning_rate": 2.6539372738146693e-08,
      "logits": -2.2307941913604736,
      "logps": -95.2313003540039,
      "loss": 0.0175,
      "objective": 0.01627987250685692,
      "ranking_idealized": 0.550000011920929,
      "ranking_idealized_expo": 0.5,
      "ranking_simple": 0.5166666507720947,
      "regularize": 0.01627855747938156,
      "step": 1460
    },
    {
      "dpo_loss": 0.6937485933303833,
      "epoch": 1.3841429808676482,
      "grad_norm": 77.80619617469834,
      "learning_rate": 2.6175903436046474e-08,
      "logits": -2.127424716949463,
      "logps": -93.49677276611328,
      "loss": 0.019,
      "objective": 0.018087979406118393,
      "ranking_idealized": 0.7333333492279053,
      "ranking_idealized_expo": 0.5833333134651184,
      "ranking_simple": 0.5666666626930237,
      "regularize": 0.018087027594447136,
      "step": 1465
    },
    {
      "dpo_loss": 0.6909429430961609,
      "epoch": 1.3888670183450122,
      "grad_norm": 65.449053741861,
      "learning_rate": 2.5814055644738007e-08,
      "logits": -2.13195538520813,
      "logps": -92.86261749267578,
      "loss": 0.018,
      "objective": 0.01916462555527687,
      "ranking_idealized": 0.6833333373069763,
      "ranking_idealized_expo": 0.5666666626930237,
      "ranking_simple": 0.5666666626930237,
      "regularize": 0.019163591787219048,
      "step": 1470
    },
    {
      "dpo_loss": 0.6916597485542297,
      "epoch": 1.3935910558223763,
      "grad_norm": 64.31813956475592,
      "learning_rate": 2.545385399223171e-08,
      "logits": -2.1243183612823486,
      "logps": -91.4704818725586,
      "loss": 0.016,
      "objective": 0.01847274787724018,
      "ranking_idealized": 0.5833333134651184,
      "ranking_idealized_expo": 0.46666666865348816,
      "ranking_simple": 0.46666666865348816,
      "regularize": 0.01847095414996147,
      "step": 1475
    },
    {
      "dpo_loss": 0.6896728277206421,
      "epoch": 1.3983150932997401,
      "grad_norm": 68.83830299489735,
      "learning_rate": 2.5095322994498846e-08,
      "logits": -2.1868510246276855,
      "logps": -88.3174819946289,
      "loss": 0.0148,
      "objective": 0.014686751179397106,
      "ranking_idealized": 0.6000000238418579,
      "ranking_idealized_expo": 0.5333333611488342,
      "ranking_simple": 0.5333333611488342,
      "regularize": 0.014684156514704227,
      "step": 1480
    },
    {
      "dpo_loss": 0.691864550113678,
      "epoch": 1.4030391307771042,
      "grad_norm": 64.34780184550137,
      "learning_rate": 2.4738487053802913e-08,
      "logits": -2.121894359588623,
      "logps": -90.51294708251953,
      "loss": 0.0178,
      "objective": 0.017730435356497765,
      "ranking_idealized": 0.699999988079071,
      "ranking_idealized_expo": 0.5833333134651184,
      "ranking_simple": 0.5666666626930237,
      "regularize": 0.017729543149471283,
      "step": 1485
    },
    {
      "dpo_loss": 0.6895692944526672,
      "epoch": 1.4077631682544682,
      "grad_norm": 76.56054457201986,
      "learning_rate": 2.4383370457038788e-08,
      "logits": -2.287666082382202,
      "logps": -91.52227020263672,
      "loss": 0.0178,
      "objective": 0.020892778411507607,
      "ranking_idealized": 0.6000000238418579,
      "ranking_idealized_expo": 0.4833333194255829,
      "ranking_simple": 0.46666666865348816,
      "regularize": 0.020891882479190826,
      "step": 1490
    },
    {
      "dpo_loss": 0.6884461641311646,
      "epoch": 1.412487205731832,
      "grad_norm": 66.46105530145252,
      "learning_rate": 2.4029997374079687e-08,
      "logits": -2.1489083766937256,
      "logps": -94.47865295410156,
      "loss": 0.0164,
      "objective": 0.017250513657927513,
      "ranking_idealized": 0.6333333253860474,
      "ranking_idealized_expo": 0.5833333134651184,
      "ranking_simple": 0.5833333134651184,
      "regularize": 0.01724918559193611,
      "step": 1495
    },
    {
      "dpo_loss": 0.6900500059127808,
      "epoch": 1.4172112432091961,
      "grad_norm": 65.6967093839243,
      "learning_rate": 2.3678391856132203e-08,
      "logits": -2.1476263999938965,
      "logps": -92.45713806152344,
      "loss": 0.0171,
      "objective": 0.017398254945874214,
      "ranking_idealized": 0.5666666626930237,
      "ranking_idealized_expo": 0.5,
      "ranking_simple": 0.5,
      "regularize": 0.01739557646214962,
      "step": 1500
    },
    {
      "epoch": 1.4172112432091961,
      "eval_dpo_loss": 0.6928529143333435,
      "eval_logits": -1.989194393157959,
      "eval_logps": -98.49781036376953,
      "eval_loss": 0.009285102598369122,
      "eval_objective": 0.009339064359664917,
      "eval_ranking_idealized": 0.6022099256515503,
      "eval_ranking_idealized_expo": 0.5207182168960571,
      "eval_ranking_simple": 0.5186464190483093,
      "eval_regularize": 0.009336920455098152,
      "eval_runtime": 454.5105,
      "eval_samples_per_second": 12.739,
      "eval_steps_per_second": 3.186,
      "step": 1500
    },
    {
      "dpo_loss": 0.6893900632858276,
      "epoch": 1.4219352806865602,
      "grad_norm": 66.7330764289215,
      "learning_rate": 2.3328577834099238e-08,
      "logits": -2.1472671031951904,
      "logps": -93.55850982666016,
      "loss": 0.0159,
      "objective": 0.01486156228929758,
      "ranking_idealized": 0.6666666865348816,
      "ranking_idealized_expo": 0.6000000238418579,
      "ranking_simple": 0.6000000238418579,
      "regularize": 0.014860122464597225,
      "step": 1505
    },
    {
      "dpo_loss": 0.6900672912597656,
      "epoch": 1.426659318163924,
      "grad_norm": 68.69500961540548,
      "learning_rate": 2.2980579116951266e-08,
      "logits": -2.205268144607544,
      "logps": -94.00220489501953,
      "loss": 0.0165,
      "objective": 0.015583300963044167,
      "ranking_idealized": 0.6333333253860474,
      "ranking_idealized_expo": 0.5666666626930237,
      "ranking_simple": 0.5666666626930237,
      "regularize": 0.015581325627863407,
      "step": 1510
    },
    {
      "dpo_loss": 0.6928814053535461,
      "epoch": 1.431383355641288,
      "grad_norm": 66.06160630988346,
      "learning_rate": 2.263441939010586e-08,
      "logits": -2.205580234527588,
      "logps": -94.78508758544922,
      "loss": 0.0176,
      "objective": 0.020189667120575905,
      "ranking_idealized": 0.6000000238418579,
      "ranking_idealized_expo": 0.5833333134651184,
      "ranking_simple": 0.5833333134651184,
      "regularize": 0.020186880603432655,
      "step": 1515
    },
    {
      "dpo_loss": 0.6897247433662415,
      "epoch": 1.436107393118652,
      "grad_norm": 64.2991321127621,
      "learning_rate": 2.2290122213815603e-08,
      "logits": -2.1570937633514404,
      "logps": -93.27649688720703,
      "loss": 0.0151,
      "objective": 0.015933455899357796,
      "ranking_idealized": 0.5666666626930237,
      "ranking_idealized_expo": 0.5333333611488342,
      "ranking_simple": 0.5333333611488342,
      "regularize": 0.01593177765607834,
      "step": 1520
    },
    {
      "dpo_loss": 0.6920062899589539,
      "epoch": 1.440831430596016,
      "grad_norm": 82.55367392818718,
      "learning_rate": 2.194771102156456e-08,
      "logits": -2.079338550567627,
      "logps": -90.78583526611328,
      "loss": 0.0161,
      "objective": 0.012961748987436295,
      "ranking_idealized": 0.6666666865348816,
      "ranking_idealized_expo": 0.6000000238418579,
      "ranking_simple": 0.6000000238418579,
      "regularize": 0.012958657927811146,
      "step": 1525
    },
    {
      "dpo_loss": 0.6941262483596802,
      "epoch": 1.44555546807338,
      "grad_norm": 64.16042224652975,
      "learning_rate": 2.1607209118473314e-08,
      "logits": -2.1198713779449463,
      "logps": -92.36068725585938,
      "loss": 0.0175,
      "objective": 0.013944637961685658,
      "ranking_idealized": 0.5666666626930237,
      "ranking_idealized_expo": 0.46666666865348816,
      "ranking_simple": 0.46666666865348816,
      "regularize": 0.013943412341177464,
      "step": 1530
    },
    {
      "dpo_loss": 0.6908197999000549,
      "epoch": 1.450279505550744,
      "grad_norm": 73.78678695774653,
      "learning_rate": 2.1268639679712813e-08,
      "logits": -2.197909355163574,
      "logps": -94.17871856689453,
      "loss": 0.0175,
      "objective": 0.020089728757739067,
      "ranking_idealized": 0.5666666626930237,
      "ranking_idealized_expo": 0.5,
      "ranking_simple": 0.5,
      "regularize": 0.020088987424969673,
      "step": 1535
    },
    {
      "dpo_loss": 0.6900218725204468,
      "epoch": 1.455003543028108,
      "grad_norm": 60.95051574524286,
      "learning_rate": 2.0932025748927014e-08,
      "logits": -2.0492825508117676,
      "logps": -90.19515228271484,
      "loss": 0.0156,
      "objective": 0.01438401360064745,
      "ranking_idealized": 0.6833333373069763,
      "ranking_idealized_expo": 0.5833333134651184,
      "ranking_simple": 0.5833333134651184,
      "regularize": 0.014382914640009403,
      "step": 1540
    },
    {
      "dpo_loss": 0.6942049264907837,
      "epoch": 1.4597275805054721,
      "grad_norm": 64.3944580161131,
      "learning_rate": 2.0597390236664474e-08,
      "logits": -2.1208300590515137,
      "logps": -92.43262481689453,
      "loss": 0.0188,
      "objective": 0.02223369851708412,
      "ranking_idealized": 0.5333333611488342,
      "ranking_idealized_expo": 0.4000000059604645,
      "ranking_simple": 0.4000000059604645,
      "regularize": 0.022232500836253166,
      "step": 1545
    },
    {
      "dpo_loss": 0.6913332343101501,
      "epoch": 1.464451617982836,
      "grad_norm": 65.89315791512014,
      "learning_rate": 2.026475591881906e-08,
      "logits": -2.080505609512329,
      "logps": -91.72013092041016,
      "loss": 0.0164,
      "objective": 0.015918172895908356,
      "ranking_idealized": 0.6333333253860474,
      "ranking_idealized_expo": 0.5666666626930237,
      "ranking_simple": 0.550000011920929,
      "regularize": 0.01591550186276436,
      "step": 1550
    },
    {
      "epoch": 1.464451617982836,
      "eval_dpo_loss": 0.6928287148475647,
      "eval_logits": -1.9898428916931152,
      "eval_logps": -98.48872375488281,
      "eval_loss": 0.009991789236664772,
      "eval_objective": 0.010131197050213814,
      "eval_ranking_idealized": 0.6022099256515503,
      "eval_ranking_idealized_expo": 0.5207182168960571,
      "eval_ranking_simple": 0.5179557800292969,
      "eval_regularize": 0.010128886438906193,
      "eval_runtime": 446.2677,
      "eval_samples_per_second": 12.974,
      "eval_steps_per_second": 3.245,
      "step": 1550
    },
    {
      "dpo_loss": 0.6896055936813354,
      "epoch": 1.4691756554602,
      "grad_norm": 72.03048326532462,
      "learning_rate": 1.9934145435079702e-08,
      "logits": -2.2838551998138428,
      "logps": -92.67180633544922,
      "loss": 0.0163,
      "objective": 0.01897108368575573,
      "ranking_idealized": 0.6333333253860474,
      "ranking_idealized_expo": 0.6000000238418579,
      "ranking_simple": 0.6000000238418579,
      "regularize": 0.018967188894748688,
      "step": 1555
    },
    {
      "dpo_loss": 0.6906958222389221,
      "epoch": 1.473899692937564,
      "grad_norm": 73.82372228393551,
      "learning_rate": 1.9605581287389632e-08,
      "logits": -2.126072645187378,
      "logps": -91.18004608154297,
      "loss": 0.017,
      "objective": 0.01644645445048809,
      "ranking_idealized": 0.5333333611488342,
      "ranking_idealized_expo": 0.4166666567325592,
      "ranking_simple": 0.4166666567325592,
      "regularize": 0.01644515059888363,
      "step": 1560
    },
    {
      "dpo_loss": 0.6889926791191101,
      "epoch": 1.478623730414928,
      "grad_norm": 64.09272915756699,
      "learning_rate": 1.92790858384147e-08,
      "logits": -2.13236927986145,
      "logps": -92.78182983398438,
      "loss": 0.0154,
      "objective": 0.016674092039465904,
      "ranking_idealized": 0.6833333373069763,
      "ranking_idealized_expo": 0.5166666507720947,
      "ranking_simple": 0.5166666507720947,
      "regularize": 0.01667255535721779,
      "step": 1565
    },
    {
      "dpo_loss": 0.6913832426071167,
      "epoch": 1.483347767892292,
      "grad_norm": 66.43479163914742,
      "learning_rate": 1.895468131002143e-08,
      "logits": -2.0496039390563965,
      "logps": -92.02823638916016,
      "loss": 0.0135,
      "objective": 0.014378294348716736,
      "ranking_idealized": 0.6333333253860474,
      "ranking_idealized_expo": 0.5333333611488342,
      "ranking_simple": 0.5333333611488342,
      "regularize": 0.014377345331013203,
      "step": 1570
    },
    {
      "dpo_loss": 0.6909292340278625,
      "epoch": 1.488071805369656,
      "grad_norm": 74.75415511259118,
      "learning_rate": 1.863238978176455e-08,
      "logits": -2.2580831050872803,
      "logps": -91.48379516601562,
      "loss": 0.014,
      "objective": 0.014301776885986328,
      "ranking_idealized": 0.6499999761581421,
      "ranking_idealized_expo": 0.5833333134651184,
      "ranking_simple": 0.5833333134651184,
      "regularize": 0.014300725422799587,
      "step": 1575
    },
    {
      "dpo_loss": 0.6893575191497803,
      "epoch": 1.4927958428470198,
      "grad_norm": 58.43183435208023,
      "learning_rate": 1.831223318938419e-08,
      "logits": -2.16597843170166,
      "logps": -93.69886016845703,
      "loss": 0.0152,
      "objective": 0.014309495687484741,
      "ranking_idealized": 0.550000011920929,
      "ranking_idealized_expo": 0.46666666865348816,
      "ranking_simple": 0.46666666865348816,
      "regularize": 0.014308045618236065,
      "step": 1580
    },
    {
      "dpo_loss": 0.6888726949691772,
      "epoch": 1.4975198803243839,
      "grad_norm": 73.21921475273949,
      "learning_rate": 1.7994233323312913e-08,
      "logits": -2.2151682376861572,
      "logps": -89.77079772949219,
      "loss": 0.0151,
      "objective": 0.017258943989872932,
      "ranking_idealized": 0.6000000238418579,
      "ranking_idealized_expo": 0.5833333134651184,
      "ranking_simple": 0.5833333134651184,
      "regularize": 0.01725638099014759,
      "step": 1585
    },
    {
      "dpo_loss": 0.6907024383544922,
      "epoch": 1.502243917801748,
      "grad_norm": 63.078892059144785,
      "learning_rate": 1.767841182719262e-08,
      "logits": -2.1972243785858154,
      "logps": -95.78870391845703,
      "loss": 0.0132,
      "objective": 0.011403498239815235,
      "ranking_idealized": 0.6166666746139526,
      "ranking_idealized_expo": 0.550000011920929,
      "ranking_simple": 0.5666666626930237,
      "regularize": 0.011402006261050701,
      "step": 1590
    },
    {
      "dpo_loss": 0.690780520439148,
      "epoch": 1.5069679552791118,
      "grad_norm": 72.75847192003988,
      "learning_rate": 1.7364790196401436e-08,
      "logits": -2.1997883319854736,
      "logps": -92.46896362304688,
      "loss": 0.0157,
      "objective": 0.017867466434836388,
      "ranking_idealized": 0.5833333134651184,
      "ranking_idealized_expo": 0.5333333611488342,
      "ranking_simple": 0.5333333611488342,
      "regularize": 0.017865275964140892,
      "step": 1595
    },
    {
      "dpo_loss": 0.6890024542808533,
      "epoch": 1.511691992756476,
      "grad_norm": 88.44042595685,
      "learning_rate": 1.705338977659071e-08,
      "logits": -2.1227810382843018,
      "logps": -93.08589172363281,
      "loss": 0.0165,
      "objective": 0.013732160441577435,
      "ranking_idealized": 0.6166666746139526,
      "ranking_idealized_expo": 0.550000011920929,
      "ranking_simple": 0.550000011920929,
      "regularize": 0.013729160651564598,
      "step": 1600
    },
    {
      "epoch": 1.511691992756476,
      "eval_dpo_loss": 0.6928747296333313,
      "eval_logits": -1.9891741275787354,
      "eval_logps": -98.44180297851562,
      "eval_loss": 0.009673803113400936,
      "eval_objective": 0.009624399244785309,
      "eval_ranking_idealized": 0.6022099256515503,
      "eval_ranking_idealized_expo": 0.5207182168960571,
      "eval_ranking_simple": 0.5186464190483093,
      "eval_regularize": 0.009622092358767986,
      "eval_runtime": 451.0127,
      "eval_samples_per_second": 12.838,
      "eval_steps_per_second": 3.211,
      "step": 1600
    },
    {
      "dpo_loss": 0.6902530193328857,
      "epoch": 1.5164160302338399,
      "grad_norm": 71.99972413373813,
      "learning_rate": 1.6744231762232176e-08,
      "logits": -2.190404176712036,
      "logps": -92.08161163330078,
      "loss": 0.0141,
      "objective": 0.013264096342027187,
      "ranking_idealized": 0.6000000238418579,
      "ranking_idealized_expo": 0.46666666865348816,
      "ranking_simple": 0.46666666865348816,
      "regularize": 0.013262915425002575,
      "step": 1605
    },
    {
      "dpo_loss": 0.6898870468139648,
      "epoch": 1.5211400677112037,
      "grad_norm": 74.10537546046199,
      "learning_rate": 1.6437337195175428e-08,
      "logits": -2.19018816947937,
      "logps": -92.24858856201172,
      "loss": 0.0144,
      "objective": 0.014542266726493835,
      "ranking_idealized": 0.6666666865348816,
      "ranking_idealized_expo": 0.550000011920929,
      "ranking_simple": 0.550000011920929,
      "regularize": 0.014540906064212322,
      "step": 1610
    },
    {
      "dpo_loss": 0.6916466951370239,
      "epoch": 1.525864105188568,
      "grad_norm": 58.38920752518266,
      "learning_rate": 1.613272696321576e-08,
      "logits": -2.136929750442505,
      "logps": -92.16728973388672,
      "loss": 0.0139,
      "objective": 0.015932830050587654,
      "ranking_idealized": 0.6666666865348816,
      "ranking_idealized_expo": 0.6000000238418579,
      "ranking_simple": 0.6000000238418579,
      "regularize": 0.015931693837046623,
      "step": 1615
    },
    {
      "dpo_loss": 0.6908102631568909,
      "epoch": 1.5305881426659318,
      "grad_norm": 63.00313504444051,
      "learning_rate": 1.5830421798672566e-08,
      "logits": -2.118234872817993,
      "logps": -90.81103515625,
      "loss": 0.0158,
      "objective": 0.013052166439592838,
      "ranking_idealized": 0.6000000238418579,
      "ranking_idealized_expo": 0.46666666865348816,
      "ranking_simple": 0.46666666865348816,
      "regularize": 0.013051275163888931,
      "step": 1620
    },
    {
      "dpo_loss": 0.6895550489425659,
      "epoch": 1.5353121801432958,
      "grad_norm": 58.07839320165127,
      "learning_rate": 1.5530442276978155e-08,
      "logits": -2.139536142349243,
      "logps": -90.25867462158203,
      "loss": 0.0148,
      "objective": 0.014852885156869888,
      "ranking_idealized": 0.5833333134651184,
      "ranking_idealized_expo": 0.5166666507720947,
      "ranking_simple": 0.5166666507720947,
      "regularize": 0.014851867221295834,
      "step": 1625
    },
    {
      "dpo_loss": 0.6881382465362549,
      "epoch": 1.54003621762066,
      "grad_norm": 68.39570526618726,
      "learning_rate": 1.523280881527743e-08,
      "logits": -2.058195114135742,
      "logps": -92.33999633789062,
      "loss": 0.0153,
      "objective": 0.018310803920030594,
      "ranking_idealized": 0.6333333253860474,
      "ranking_idealized_expo": 0.5666666626930237,
      "ranking_simple": 0.5666666626930237,
      "regularize": 0.018308250233530998,
      "step": 1630
    },
    {
      "dpo_loss": 0.6918675303459167,
      "epoch": 1.5447602550980237,
      "grad_norm": 71.28198542563501,
      "learning_rate": 1.4937541671038245e-08,
      "logits": -2.0989344120025635,
      "logps": -94.7214584350586,
      "loss": 0.0141,
      "objective": 0.016751030460000038,
      "ranking_idealized": 0.699999988079071,
      "ranking_idealized_expo": 0.6333333253860474,
      "ranking_simple": 0.6333333253860474,
      "regularize": 0.016748782247304916,
      "step": 1635
    },
    {
      "dpo_loss": 0.6916669011116028,
      "epoch": 1.5494842925753878,
      "grad_norm": 55.738628042601434,
      "learning_rate": 1.4644660940672625e-08,
      "logits": -2.154735803604126,
      "logps": -93.92298126220703,
      "loss": 0.0156,
      "objective": 0.012084761634469032,
      "ranking_idealized": 0.6333333253860474,
      "ranking_idealized_expo": 0.46666666865348816,
      "ranking_simple": 0.46666666865348816,
      "regularize": 0.01208337489515543,
      "step": 1640
    },
    {
      "dpo_loss": 0.6894612908363342,
      "epoch": 1.5542083300527518,
      "grad_norm": 62.99549337373731,
      "learning_rate": 1.435418655816899e-08,
      "logits": -2.169052839279175,
      "logps": -91.41073608398438,
      "loss": 0.0143,
      "objective": 0.014856117777526379,
      "ranking_idealized": 0.4833333194255829,
      "ranking_idealized_expo": 0.44999998807907104,
      "ranking_simple": 0.44999998807907104,
      "regularize": 0.014855272136628628,
      "step": 1645
    },
    {
      "dpo_loss": 0.6909356713294983,
      "epoch": 1.5589323675301157,
      "grad_norm": 67.0305140958766,
      "learning_rate": 1.4066138293735408e-08,
      "logits": -2.098741292953491,
      "logps": -92.3939208984375,
      "loss": 0.0128,
      "objective": 0.012405160814523697,
      "ranking_idealized": 0.6333333253860474,
      "ranking_idealized_expo": 0.550000011920929,
      "ranking_simple": 0.550000011920929,
      "regularize": 0.012403340078890324,
      "step": 1650
    },
    {
      "epoch": 1.5589323675301157,
      "eval_dpo_loss": 0.6927416324615479,
      "eval_logits": -1.9889006614685059,
      "eval_logps": -98.3604736328125,
      "eval_loss": 0.010028412565588951,
      "eval_objective": 0.010088582523167133,
      "eval_ranking_idealized": 0.6022099256515503,
      "eval_ranking_idealized_expo": 0.5207182168960571,
      "eval_ranking_simple": 0.5179557800292969,
      "eval_regularize": 0.010086326859891415,
      "eval_runtime": 446.4522,
      "eval_samples_per_second": 12.969,
      "eval_steps_per_second": 3.243,
      "step": 1650
    },
    {
      "dpo_loss": 0.6893053650856018,
      "epoch": 1.5636564050074797,
      "grad_norm": 63.18957665484938,
      "learning_rate": 1.3780535752453976e-08,
      "logits": -2.115309000015259,
      "logps": -92.01270294189453,
      "loss": 0.016,
      "objective": 0.01488524954766035,
      "ranking_idealized": 0.6166666746139526,
      "ranking_idealized_expo": 0.5666666626930237,
      "ranking_simple": 0.5666666626930237,
      "regularize": 0.014881560578942299,
      "step": 1655
    },
    {
      "dpo_loss": 0.6926845908164978,
      "epoch": 1.5683804424848438,
      "grad_norm": 58.824443635483924,
      "learning_rate": 1.34973983729465e-08,
      "logits": -2.0248100757598877,
      "logps": -91.5145263671875,
      "loss": 0.0155,
      "objective": 0.016196589916944504,
      "ranking_idealized": 0.5666666626930237,
      "ranking_idealized_expo": 0.4833333194255829,
      "ranking_simple": 0.4833333194255829,
      "regularize": 0.01619427278637886,
      "step": 1660
    },
    {
      "dpo_loss": 0.6904213428497314,
      "epoch": 1.5731044799622076,
      "grad_norm": 62.30456762761477,
      "learning_rate": 1.3216745426051451e-08,
      "logits": -2.12634015083313,
      "logps": -93.39350891113281,
      "loss": 0.0136,
      "objective": 0.012430812232196331,
      "ranking_idealized": 0.6166666746139526,
      "ranking_idealized_expo": 0.5333333611488342,
      "ranking_simple": 0.5333333611488342,
      "regularize": 0.012427231296896935,
      "step": 1665
    },
    {
      "dpo_loss": 0.6912954449653625,
      "epoch": 1.5778285174395716,
      "grad_norm": 69.75551627833396,
      "learning_rate": 1.293859601351232e-08,
      "logits": -2.253553867340088,
      "logps": -94.33558654785156,
      "loss": 0.0154,
      "objective": 0.015208045952022076,
      "ranking_idealized": 0.6166666746139526,
      "ranking_idealized_expo": 0.5166666507720947,
      "ranking_simple": 0.5166666507720947,
      "regularize": 0.015205765143036842,
      "step": 1670
    },
    {
      "dpo_loss": 0.6908196806907654,
      "epoch": 1.5825525549169357,
      "grad_norm": 66.72338681583422,
      "learning_rate": 1.266296906667762e-08,
      "logits": -2.232628107070923,
      "logps": -92.84732818603516,
      "loss": 0.0154,
      "objective": 0.014128237962722778,
      "ranking_idealized": 0.4166666567325592,
      "ranking_idealized_expo": 0.4166666567325592,
      "ranking_simple": 0.4166666567325592,
      "regularize": 0.014126955531537533,
      "step": 1675
    },
    {
      "dpo_loss": 0.6899545192718506,
      "epoch": 1.5872765923942995,
      "grad_norm": 63.2891144846116,
      "learning_rate": 1.238988334521226e-08,
      "logits": -2.1890323162078857,
      "logps": -95.2237777709961,
      "loss": 0.0152,
      "objective": 0.014830714091658592,
      "ranking_idealized": 0.550000011920929,
      "ranking_idealized_expo": 0.4833333194255829,
      "ranking_simple": 0.4833333194255829,
      "regularize": 0.014829723164439201,
      "step": 1680
    },
    {
      "dpo_loss": 0.6891339421272278,
      "epoch": 1.5920006298716638,
      "grad_norm": 59.6793164031309,
      "learning_rate": 1.2119357435820816e-08,
      "logits": -2.1683380603790283,
      "logps": -89.5069580078125,
      "loss": 0.0118,
      "objective": 0.01023172028362751,
      "ranking_idealized": 0.46666666865348816,
      "ranking_idealized_expo": 0.3166666626930237,
      "ranking_simple": 0.3166666626930237,
      "regularize": 0.010230082087218761,
      "step": 1685
    },
    {
      "dpo_loss": 0.6914657354354858,
      "epoch": 1.5967246673490276,
      "grad_norm": 61.16970720231615,
      "learning_rate": 1.1851409750982438e-08,
      "logits": -2.128115653991699,
      "logps": -92.26956176757812,
      "loss": 0.013,
      "objective": 0.013722400180995464,
      "ranking_idealized": 0.699999988079071,
      "ranking_idealized_expo": 0.5833333134651184,
      "ranking_simple": 0.6000000238418579,
      "regularize": 0.01371851097792387,
      "step": 1690
    },
    {
      "dpo_loss": 0.6910730600357056,
      "epoch": 1.6014487048263917,
      "grad_norm": 70.19949003863668,
      "learning_rate": 1.1586058527697707e-08,
      "logits": -2.1930956840515137,
      "logps": -91.90154266357422,
      "loss": 0.0136,
      "objective": 0.01538047008216381,
      "ranking_idealized": 0.6666666865348816,
      "ranking_idealized_expo": 0.6000000238418579,
      "ranking_simple": 0.6000000238418579,
      "regularize": 0.015379803255200386,
      "step": 1695
    },
    {
      "dpo_loss": 0.6901703476905823,
      "epoch": 1.6061727423037557,
      "grad_norm": 68.99239991969064,
      "learning_rate": 1.1323321826247346e-08,
      "logits": -2.1464996337890625,
      "logps": -93.59435272216797,
      "loss": 0.0132,
      "objective": 0.014362351037561893,
      "ranking_idealized": 0.6333333253860474,
      "ranking_idealized_expo": 0.5166666507720947,
      "ranking_simple": 0.5166666507720947,
      "regularize": 0.014360878616571426,
      "step": 1700
    },
    {
      "epoch": 1.6061727423037557,
      "eval_dpo_loss": 0.6927831768989563,
      "eval_logits": -1.9890520572662354,
      "eval_logps": -98.40552520751953,
      "eval_loss": 0.008960912004113197,
      "eval_objective": 0.008905092254281044,
      "eval_ranking_idealized": 0.6022099256515503,
      "eval_ranking_idealized_expo": 0.5207182168960571,
      "eval_ranking_simple": 0.5179557800292969,
      "eval_regularize": 0.008902397006750107,
      "eval_runtime": 446.3386,
      "eval_samples_per_second": 12.972,
      "eval_steps_per_second": 3.244,
      "step": 1700
    },
    {
      "dpo_loss": 0.6905626654624939,
      "epoch": 1.6108967797811196,
      "grad_norm": 61.071862990680025,
      "learning_rate": 1.1063217528963042e-08,
      "logits": -2.1819908618927,
      "logps": -95.61785125732422,
      "loss": 0.0138,
      "objective": 0.014233703725039959,
      "ranking_idealized": 0.5833333134651184,
      "ranking_idealized_expo": 0.5166666507720947,
      "ranking_simple": 0.5,
      "regularize": 0.014231013134121895,
      "step": 1705
    },
    {
      "dpo_loss": 0.6933135986328125,
      "epoch": 1.6156208172584836,
      "grad_norm": 65.29758946742726,
      "learning_rate": 1.0805763339010326e-08,
      "logits": -2.13582444190979,
      "logps": -92.55400848388672,
      "loss": 0.0135,
      "objective": 0.014589487574994564,
      "ranking_idealized": 0.6833333373069763,
      "ranking_idealized_expo": 0.6499999761581421,
      "ranking_simple": 0.6499999761581421,
      "regularize": 0.014586606994271278,
      "step": 1710
    },
    {
      "dpo_loss": 0.6906417608261108,
      "epoch": 1.6203448547358477,
      "grad_norm": 65.78620291334798,
      "learning_rate": 1.0550976779183651e-08,
      "logits": -2.1200928688049316,
      "logps": -95.07630920410156,
      "loss": 0.0136,
      "objective": 0.014620447531342506,
      "ranking_idealized": 0.5833333134651184,
      "ranking_idealized_expo": 0.5166666507720947,
      "ranking_simple": 0.5166666507720947,
      "regularize": 0.014618877321481705,
      "step": 1715
    },
    {
      "dpo_loss": 0.6903366446495056,
      "epoch": 1.6250688922132115,
      "grad_norm": 61.420985867018985,
      "learning_rate": 1.02988751907138e-08,
      "logits": -2.196997880935669,
      "logps": -89.81111145019531,
      "loss": 0.0124,
      "objective": 0.01165497861802578,
      "ranking_idealized": 0.6499999761581421,
      "ranking_idealized_expo": 0.5666666626930237,
      "ranking_simple": 0.5666666626930237,
      "regularize": 0.011652585119009018,
      "step": 1720
    },
    {
      "dpo_loss": 0.6910706758499146,
      "epoch": 1.6297929296905755,
      "grad_norm": 69.65328151335441,
      "learning_rate": 1.0049475732087559e-08,
      "logits": -2.1551551818847656,
      "logps": -92.8335952758789,
      "loss": 0.0137,
      "objective": 0.011478899046778679,
      "ranking_idealized": 0.5666666626930237,
      "ranking_idealized_expo": 0.5,
      "ranking_simple": 0.5,
      "regularize": 0.0114761833101511,
      "step": 1725
    },
    {
      "dpo_loss": 0.6897426247596741,
      "epoch": 1.6345169671679396,
      "grad_norm": 70.43018567128607,
      "learning_rate": 9.802795377879903e-09,
      "logits": -2.243594169616699,
      "logps": -91.26370239257812,
      "loss": 0.0128,
      "objective": 0.012597540393471718,
      "ranking_idealized": 0.5833333134651184,
      "ranking_idealized_expo": 0.5333333611488342,
      "ranking_simple": 0.5333333611488342,
      "regularize": 0.012595159001648426,
      "step": 1730
    },
    {
      "dpo_loss": 0.6892535090446472,
      "epoch": 1.6392410046453034,
      "grad_norm": 63.82988388608345,
      "learning_rate": 9.558850917598716e-09,
      "logits": -2.070333957672119,
      "logps": -92.88044738769531,
      "loss": 0.0123,
      "objective": 0.011261907406151295,
      "ranking_idealized": 0.5166666507720947,
      "ranking_idealized_expo": 0.4333333373069763,
      "ranking_simple": 0.44999998807907104,
      "regularize": 0.011259406805038452,
      "step": 1735
    },
    {
      "dpo_loss": 0.6904967427253723,
      "epoch": 1.6439650421226675,
      "grad_norm": 57.01783294970417,
      "learning_rate": 9.31765895454199e-09,
      "logits": -2.178701877593994,
      "logps": -92.82861328125,
      "loss": 0.0128,
      "objective": 0.013563827611505985,
      "ranking_idealized": 0.6000000238418579,
      "ranking_idealized_expo": 0.5666666626930237,
      "ranking_simple": 0.5666666626930237,
      "regularize": 0.013562088832259178,
      "step": 1740
    },
    {
      "dpo_loss": 0.6911302208900452,
      "epoch": 1.6486890796000315,
      "grad_norm": 67.14492805040307,
      "learning_rate": 9.079235904667825e-09,
      "logits": -2.160048484802246,
      "logps": -97.07341766357422,
      "loss": 0.0132,
      "objective": 0.011855502612888813,
      "ranking_idealized": 0.6499999761581421,
      "ranking_idealized_expo": 0.5833333134651184,
      "ranking_simple": 0.5833333134651184,
      "regularize": 0.011852098628878593,
      "step": 1745
    },
    {
      "dpo_loss": 0.6899723410606384,
      "epoch": 1.6534131170773954,
      "grad_norm": 61.803443023637456,
      "learning_rate": 8.84359799547712e-09,
      "logits": -2.1978349685668945,
      "logps": -92.97909545898438,
      "loss": 0.0133,
      "objective": 0.01489060465246439,
      "ranking_idealized": 0.5166666507720947,
      "ranking_idealized_expo": 0.44999998807907104,
      "ranking_simple": 0.44999998807907104,
      "regularize": 0.014889102429151535,
      "step": 1750
    },
    {
      "epoch": 1.6534131170773954,
      "eval_dpo_loss": 0.6928050518035889,
      "eval_logits": -1.988478183746338,
      "eval_logps": -98.41736602783203,
      "eval_loss": 0.009351465851068497,
      "eval_objective": 0.009363526478409767,
      "eval_ranking_idealized": 0.6022099256515503,
      "eval_ranking_idealized_expo": 0.5207182168960571,
      "eval_ranking_simple": 0.5179557800292969,
      "eval_regularize": 0.009361328557133675,
      "eval_runtime": 445.0354,
      "eval_samples_per_second": 13.01,
      "eval_steps_per_second": 3.254,
      "step": 1750
    },
    {
      "dpo_loss": 0.6889625787734985,
      "epoch": 1.6581371545547596,
      "grad_norm": 69.19823709150691,
      "learning_rate": 8.6107612649091e-09,
      "logits": -2.17798113822937,
      "logps": -90.93826293945312,
      "loss": 0.0136,
      "objective": 0.014197876676917076,
      "ranking_idealized": 0.5166666507720947,
      "ranking_idealized_expo": 0.44999998807907104,
      "ranking_simple": 0.4333333373069763,
      "regularize": 0.014196816831827164,
      "step": 1755
    },
    {
      "dpo_loss": 0.6894813776016235,
      "epoch": 1.6628611920321235,
      "grad_norm": 70.33904785688647,
      "learning_rate": 8.380741560249726e-09,
      "logits": -2.2228808403015137,
      "logps": -91.38529205322266,
      "loss": 0.0136,
      "objective": 0.0157302338629961,
      "ranking_idealized": 0.550000011920929,
      "ranking_idealized_expo": 0.5166666507720947,
      "ranking_simple": 0.5166666507720947,
      "regularize": 0.015728596597909927,
      "step": 1760
    },
    {
      "dpo_loss": 0.6904619336128235,
      "epoch": 1.6675852295094873,
      "grad_norm": 66.62367568306153,
      "learning_rate": 8.153554537053149e-09,
      "logits": -2.1843998432159424,
      "logps": -91.0223159790039,
      "loss": 0.0144,
      "objective": 0.012193134985864162,
      "ranking_idealized": 0.5833333134651184,
      "ranking_idealized_expo": 0.46666666865348816,
      "ranking_simple": 0.46666666865348816,
      "regularize": 0.01219152845442295,
      "step": 1765
    },
    {
      "dpo_loss": 0.691467821598053,
      "epoch": 1.6723092669868516,
      "grad_norm": 63.63343895864985,
      "learning_rate": 7.929215658076093e-09,
      "logits": -2.2059359550476074,
      "logps": -92.13400268554688,
      "loss": 0.0114,
      "objective": 0.01266545057296753,
      "ranking_idealized": 0.6166666746139526,
      "ranking_idealized_expo": 0.5,
      "ranking_simple": 0.5,
      "regularize": 0.012664331123232841,
      "step": 1770
    },
    {
      "dpo_loss": 0.6895557045936584,
      "epoch": 1.6770333044642154,
      "grad_norm": 61.16013319033806,
      "learning_rate": 7.707740192225515e-09,
      "logits": -2.257197380065918,
      "logps": -93.42992401123047,
      "loss": 0.0128,
      "objective": 0.014360646717250347,
      "ranking_idealized": 0.6666666865348816,
      "ranking_idealized_expo": 0.6666666865348816,
      "ranking_simple": 0.6666666865348816,
      "regularize": 0.014359161257743835,
      "step": 1775
    },
    {
      "dpo_loss": 0.691416323184967,
      "epoch": 1.6817573419415794,
      "grad_norm": 68.36063699136817,
      "learning_rate": 7.4891432135193e-09,
      "logits": -2.251347303390503,
      "logps": -91.27902221679688,
      "loss": 0.0141,
      "objective": 0.013033194467425346,
      "ranking_idealized": 0.6666666865348816,
      "ranking_idealized_expo": 0.5333333611488342,
      "ranking_simple": 0.5333333611488342,
      "regularize": 0.013032082468271255,
      "step": 1780
    },
    {
      "dpo_loss": 0.6915625929832458,
      "epoch": 1.6864813794189435,
      "grad_norm": 64.60918615534963,
      "learning_rate": 7.273439600060344e-09,
      "logits": -2.165839195251465,
      "logps": -93.43399810791016,
      "loss": 0.0158,
      "objective": 0.016496647149324417,
      "ranking_idealized": 0.6333333253860474,
      "ranking_idealized_expo": 0.5666666626930237,
      "ranking_simple": 0.550000011920929,
      "regularize": 0.016495179384946823,
      "step": 1785
    },
    {
      "dpo_loss": 0.6900497078895569,
      "epoch": 1.6912054168963073,
      "grad_norm": 59.4845336366133,
      "learning_rate": 7.060644033023894e-09,
      "logits": -2.1421496868133545,
      "logps": -89.32416534423828,
      "loss": 0.0132,
      "objective": 0.011856761761009693,
      "ranking_idealized": 0.5666666626930237,
      "ranking_idealized_expo": 0.5,
      "ranking_simple": 0.5,
      "regularize": 0.011854317970573902,
      "step": 1790
    },
    {
      "dpo_loss": 0.6918495297431946,
      "epoch": 1.6959294543736714,
      "grad_norm": 62.10968860121513,
      "learning_rate": 6.850770995658372e-09,
      "logits": -2.145258665084839,
      "logps": -91.16167449951172,
      "loss": 0.0135,
      "objective": 0.011840968392789364,
      "ranking_idealized": 0.6000000238418579,
      "ranking_idealized_expo": 0.5333333611488342,
      "ranking_simple": 0.5333333611488342,
      "regularize": 0.011839687824249268,
      "step": 1795
    },
    {
      "dpo_loss": 0.6910142302513123,
      "epoch": 1.7006534918510354,
      "grad_norm": 63.84635783884053,
      "learning_rate": 6.6438347722995445e-09,
      "logits": -2.2468461990356445,
      "logps": -91.39185333251953,
      "loss": 0.0138,
      "objective": 0.010580122470855713,
      "ranking_idealized": 0.5,
      "ranking_idealized_expo": 0.44999998807907104,
      "ranking_simple": 0.44999998807907104,
      "regularize": 0.010576292872428894,
      "step": 1800
    },
    {
      "epoch": 1.7006534918510354,
      "eval_dpo_loss": 0.6928184628486633,
      "eval_logits": -1.9885612726211548,
      "eval_logps": -98.35978698730469,
      "eval_loss": 0.009589685127139091,
      "eval_objective": 0.009668215177953243,
      "eval_ranking_idealized": 0.6022099256515503,
      "eval_ranking_idealized_expo": 0.5207182168960571,
      "eval_ranking_simple": 0.5179557800292969,
      "eval_regularize": 0.00966621097177267,
      "eval_runtime": 448.9142,
      "eval_samples_per_second": 12.898,
      "eval_steps_per_second": 3.226,
      "step": 1800
    },
    {
      "dpo_loss": 0.6905153393745422,
      "epoch": 1.7053775293283993,
      "grad_norm": 69.0367318972444,
      "learning_rate": 6.43984944739836e-09,
      "logits": -2.206124782562256,
      "logps": -93.37840270996094,
      "loss": 0.0133,
      "objective": 0.013736736960709095,
      "ranking_idealized": 0.5333333611488342,
      "ranking_idealized_expo": 0.4833333194255829,
      "ranking_simple": 0.4833333194255829,
      "regularize": 0.013735477812588215,
      "step": 1805
    },
    {
      "dpo_loss": 0.6886168718338013,
      "epoch": 1.7101015668057633,
      "grad_norm": 60.061121059879774,
      "learning_rate": 6.238828904562315e-09,
      "logits": -2.238112449645996,
      "logps": -93.0470199584961,
      "loss": 0.0146,
      "objective": 0.015197351574897766,
      "ranking_idealized": 0.6666666865348816,
      "ranking_idealized_expo": 0.6666666865348816,
      "ranking_simple": 0.6666666865348816,
      "regularize": 0.015195462852716446,
      "step": 1810
    },
    {
      "dpo_loss": 0.6895704865455627,
      "epoch": 1.7148256042831274,
      "grad_norm": 62.370319050099795,
      "learning_rate": 6.040786825610517e-09,
      "logits": -2.18115234375,
      "logps": -92.52540588378906,
      "loss": 0.0127,
      "objective": 0.015414653345942497,
      "ranking_idealized": 0.5833333134651184,
      "ranking_idealized_expo": 0.5,
      "ranking_simple": 0.5,
      "regularize": 0.015413584187626839,
      "step": 1815
    },
    {
      "dpo_loss": 0.6913415193557739,
      "epoch": 1.7195496417604912,
      "grad_norm": 68.7151840413099,
      "learning_rate": 5.845736689642472e-09,
      "logits": -2.2472267150878906,
      "logps": -93.7915267944336,
      "loss": 0.015,
      "objective": 0.01595698669552803,
      "ranking_idealized": 0.5833333134651184,
      "ranking_idealized_expo": 0.5333333611488342,
      "ranking_simple": 0.5166666507720947,
      "regularize": 0.01595553196966648,
      "step": 1820
    },
    {
      "dpo_loss": 0.6893234848976135,
      "epoch": 1.7242736792378552,
      "grad_norm": 61.62237599260527,
      "learning_rate": 5.653691772120672e-09,
      "logits": -2.2202160358428955,
      "logps": -91.71192932128906,
      "loss": 0.0125,
      "objective": 0.013099589385092258,
      "ranking_idealized": 0.699999988079071,
      "ranking_idealized_expo": 0.6333333253860474,
      "ranking_simple": 0.6333333253860474,
      "regularize": 0.013098122552037239,
      "step": 1825
    },
    {
      "dpo_loss": 0.6918686032295227,
      "epoch": 1.7289977167152193,
      "grad_norm": 73.26853232568251,
      "learning_rate": 5.464665143967051e-09,
      "logits": -2.095928907394409,
      "logps": -92.31403350830078,
      "loss": 0.0132,
      "objective": 0.014795850031077862,
      "ranking_idealized": 0.6000000238418579,
      "ranking_idealized_expo": 0.4833333194255829,
      "ranking_simple": 0.46666666865348816,
      "regularize": 0.01479416061192751,
      "step": 1830
    },
    {
      "dpo_loss": 0.6898643374443054,
      "epoch": 1.7337217541925831,
      "grad_norm": 58.513985312428574,
      "learning_rate": 5.278669670673347e-09,
      "logits": -2.1115658283233643,
      "logps": -91.03611755371094,
      "loss": 0.0117,
      "objective": 0.0112331947311759,
      "ranking_idealized": 0.699999988079071,
      "ranking_idealized_expo": 0.6000000238418579,
      "ranking_simple": 0.6000000238418579,
      "regularize": 0.011231665499508381,
      "step": 1835
    },
    {
      "dpo_loss": 0.6913903951644897,
      "epoch": 1.7384457916699474,
      "grad_norm": 63.11876863535166,
      "learning_rate": 5.095718011425454e-09,
      "logits": -2.168307304382324,
      "logps": -91.80027770996094,
      "loss": 0.0135,
      "objective": 0.014516009949147701,
      "ranking_idealized": 0.6499999761581421,
      "ranking_idealized_expo": 0.5666666626930237,
      "ranking_simple": 0.5666666626930237,
      "regularize": 0.01451488770544529,
      "step": 1840
    },
    {
      "dpo_loss": 0.689866304397583,
      "epoch": 1.7431698291473112,
      "grad_norm": 72.43548684342062,
      "learning_rate": 4.9158226182418104e-09,
      "logits": -2.1879196166992188,
      "logps": -88.85045623779297,
      "loss": 0.0151,
      "objective": 0.014930271543562412,
      "ranking_idealized": 0.550000011920929,
      "ranking_idealized_expo": 0.4833333194255829,
      "ranking_simple": 0.4833333194255829,
      "regularize": 0.014927973970770836,
      "step": 1845
    },
    {
      "dpo_loss": 0.689470648765564,
      "epoch": 1.7478938666246753,
      "grad_norm": 65.9250612110063,
      "learning_rate": 4.738995735125894e-09,
      "logits": -2.074735164642334,
      "logps": -94.42037963867188,
      "loss": 0.0122,
      "objective": 0.011366092599928379,
      "ranking_idealized": 0.6833333373069763,
      "ranking_idealized_expo": 0.5166666507720947,
      "ranking_simple": 0.5333333611488342,
      "regularize": 0.011363506317138672,
      "step": 1850
    },
    {
      "epoch": 1.7478938666246753,
      "eval_dpo_loss": 0.6928740739822388,
      "eval_logits": -1.9887943267822266,
      "eval_logps": -98.4156723022461,
      "eval_loss": 0.009023972786962986,
      "eval_objective": 0.00906344410032034,
      "eval_ranking_idealized": 0.6022099256515503,
      "eval_ranking_idealized_expo": 0.5207182168960571,
      "eval_ranking_simple": 0.5179557800292969,
      "eval_regularize": 0.009061108343303204,
      "eval_runtime": 445.128,
      "eval_samples_per_second": 13.007,
      "eval_steps_per_second": 3.253,
      "step": 1850
    },
    {
      "dpo_loss": 0.6925438046455383,
      "epoch": 1.7526179041020393,
      "grad_norm": 60.51205917887876,
      "learning_rate": 4.565249397232923e-09,
      "logits": -2.1302504539489746,
      "logps": -89.6479263305664,
      "loss": 0.0119,
      "objective": 0.012942561879754066,
      "ranking_idealized": 0.6666666865348816,
      "ranking_idealized_expo": 0.6333333253860474,
      "ranking_simple": 0.6333333253860474,
      "regularize": 0.012941225431859493,
      "step": 1855
    },
    {
      "dpo_loss": 0.6919233798980713,
      "epoch": 1.7573419415794032,
      "grad_norm": 73.74409349566294,
      "learning_rate": 4.394595430050613e-09,
      "logits": -2.1312899589538574,
      "logps": -92.23778533935547,
      "loss": 0.0134,
      "objective": 0.013376330956816673,
      "ranking_idealized": 0.6166666746139526,
      "ranking_idealized_expo": 0.5,
      "ranking_simple": 0.5,
      "regularize": 0.013374886475503445,
      "step": 1860
    },
    {
      "dpo_loss": 0.6894384026527405,
      "epoch": 1.7620659790567672,
      "grad_norm": 62.88647078101833,
      "learning_rate": 4.2270454485944125e-09,
      "logits": -2.0558435916900635,
      "logps": -91.38388061523438,
      "loss": 0.0107,
      "objective": 0.010976298712193966,
      "ranking_idealized": 0.7666666507720947,
      "ranking_idealized_expo": 0.6666666865348816,
      "ranking_simple": 0.6666666865348816,
      "regularize": 0.010971426963806152,
      "step": 1865
    },
    {
      "dpo_loss": 0.6887921094894409,
      "epoch": 1.7667900165341313,
      "grad_norm": 87.932062525531,
      "learning_rate": 4.062610856616922e-09,
      "logits": -2.1452383995056152,
      "logps": -92.77478790283203,
      "loss": 0.013,
      "objective": 0.017829876393079758,
      "ranking_idealized": 0.550000011920929,
      "ranking_idealized_expo": 0.44999998807907104,
      "ranking_simple": 0.44999998807907104,
      "regularize": 0.017827244475483894,
      "step": 1870
    },
    {
      "dpo_loss": 0.6901695132255554,
      "epoch": 1.771514054011495,
      "grad_norm": 59.56034569340847,
      "learning_rate": 3.901302845831728e-09,
      "logits": -2.1918883323669434,
      "logps": -91.66542053222656,
      "loss": 0.0103,
      "objective": 0.00952277984470129,
      "ranking_idealized": 0.6166666746139526,
      "ranking_idealized_expo": 0.5333333611488342,
      "ranking_simple": 0.550000011920929,
      "regularize": 0.009518155828118324,
      "step": 1875
    },
    {
      "dpo_loss": 0.6916844844818115,
      "epoch": 1.7762380914888591,
      "grad_norm": 59.31587765757665,
      "learning_rate": 3.743132395151705e-09,
      "logits": -2.1958460807800293,
      "logps": -92.2247085571289,
      "loss": 0.0132,
      "objective": 0.014002182520925999,
      "ranking_idealized": 0.5166666507720947,
      "ranking_idealized_expo": 0.44999998807907104,
      "ranking_simple": 0.44999998807907104,
      "regularize": 0.014000624418258667,
      "step": 1880
    },
    {
      "dpo_loss": 0.6910093426704407,
      "epoch": 1.7809621289662232,
      "grad_norm": 65.01445310326804,
      "learning_rate": 3.5881102699417463e-09,
      "logits": -2.15521240234375,
      "logps": -92.9655990600586,
      "loss": 0.0122,
      "objective": 0.013613136485219002,
      "ranking_idealized": 0.6166666746139526,
      "ranking_idealized_expo": 0.5,
      "ranking_simple": 0.5,
      "regularize": 0.013612000271677971,
      "step": 1885
    },
    {
      "dpo_loss": 0.688242495059967,
      "epoch": 1.785686166443587,
      "grad_norm": 68.21241149596634,
      "learning_rate": 3.4362470212860483e-09,
      "logits": -2.2397677898406982,
      "logps": -92.33454132080078,
      "loss": 0.0101,
      "objective": 0.011641601100564003,
      "ranking_idealized": 0.6833333373069763,
      "ranking_idealized_expo": 0.6000000238418579,
      "ranking_simple": 0.6000000238418579,
      "regularize": 0.011638239957392216,
      "step": 1890
    },
    {
      "dpo_loss": 0.691038966178894,
      "epoch": 1.790410203920951,
      "grad_norm": 68.20823651098704,
      "learning_rate": 3.2875529852700148e-09,
      "logits": -2.2314653396606445,
      "logps": -86.68601989746094,
      "loss": 0.0131,
      "objective": 0.01244689617305994,
      "ranking_idealized": 0.5,
      "ranking_idealized_expo": 0.4333333373069763,
      "ranking_simple": 0.4333333373069763,
      "regularize": 0.012444679625332355,
      "step": 1895
    },
    {
      "dpo_loss": 0.6882398128509521,
      "epoch": 1.7951342413983151,
      "grad_norm": 74.00426423645611,
      "learning_rate": 3.142038282276732e-09,
      "logits": -2.1504063606262207,
      "logps": -92.22313690185547,
      "loss": 0.0128,
      "objective": 0.013669600710272789,
      "ranking_idealized": 0.6333333253860474,
      "ranking_idealized_expo": 0.550000011920929,
      "ranking_simple": 0.550000011920929,
      "regularize": 0.013666595332324505,
      "step": 1900
    },
    {
      "epoch": 1.7951342413983151,
      "eval_dpo_loss": 0.6928610801696777,
      "eval_logits": -1.9891064167022705,
      "eval_logps": -98.42906951904297,
      "eval_loss": 0.008886425755918026,
      "eval_objective": 0.0089500043541193,
      "eval_ranking_idealized": 0.6022099256515503,
      "eval_ranking_idealized_expo": 0.5207182168960571,
      "eval_ranking_simple": 0.5179557800292969,
      "eval_regularize": 0.008947666734457016,
      "eval_runtime": 445.1268,
      "eval_samples_per_second": 13.008,
      "eval_steps_per_second": 3.253,
      "step": 1900
    },
    {
      "dpo_loss": 0.6886274218559265,
      "epoch": 1.799858278875679,
      "grad_norm": 70.48280427657417,
      "learning_rate": 2.9997128162981835e-09,
      "logits": -2.061340808868408,
      "logps": -92.88438415527344,
      "loss": 0.0124,
      "objective": 0.011484592221677303,
      "ranking_idealized": 0.6166666746139526,
      "ranking_idealized_expo": 0.5666666626930237,
      "ranking_simple": 0.5666666626930237,
      "regularize": 0.01148195844143629,
      "step": 1905
    },
    {
      "dpo_loss": 0.6910132169723511,
      "epoch": 1.8045823163530432,
      "grad_norm": 69.16766112778924,
      "learning_rate": 2.8605862742611453e-09,
      "logits": -2.2064709663391113,
      "logps": -93.21188354492188,
      "loss": 0.0131,
      "objective": 0.013887956738471985,
      "ranking_idealized": 0.6333333253860474,
      "ranking_idealized_expo": 0.5,
      "ranking_simple": 0.5,
      "regularize": 0.013886776752769947,
      "step": 1910
    },
    {
      "dpo_loss": 0.6885823011398315,
      "epoch": 1.809306353830407,
      "grad_norm": 60.392467741599056,
      "learning_rate": 2.724668125367896e-09,
      "logits": -2.063930034637451,
      "logps": -93.89399719238281,
      "loss": 0.0118,
      "objective": 0.012090322561562061,
      "ranking_idealized": 0.5,
      "ranking_idealized_expo": 0.46666666865348816,
      "ranking_simple": 0.46666666865348816,
      "regularize": 0.012088621035218239,
      "step": 1915
    },
    {
      "dpo_loss": 0.691336452960968,
      "epoch": 1.8140303913077709,
      "grad_norm": 64.14689736170588,
      "learning_rate": 2.591967620451707e-09,
      "logits": -2.2195615768432617,
      "logps": -95.23002624511719,
      "loss": 0.0126,
      "objective": 0.011388556100428104,
      "ranking_idealized": 0.6499999761581421,
      "ranking_idealized_expo": 0.550000011920929,
      "ranking_simple": 0.550000011920929,
      "regularize": 0.011385311372578144,
      "step": 1920
    },
    {
      "dpo_loss": 0.6888495683670044,
      "epoch": 1.8187544287851352,
      "grad_norm": 61.363679552810055,
      "learning_rate": 2.462493791347231e-09,
      "logits": -2.1906003952026367,
      "logps": -93.10894012451172,
      "loss": 0.0109,
      "objective": 0.012360634282231331,
      "ranking_idealized": 0.6666666865348816,
      "ranking_idealized_expo": 0.5666666626930237,
      "ranking_simple": 0.5666666626930237,
      "regularize": 0.012358280830085278,
      "step": 1925
    },
    {
      "dpo_loss": 0.6920725107192993,
      "epoch": 1.823478466262499,
      "grad_norm": 62.3361060804096,
      "learning_rate": 2.3362554502757536e-09,
      "logits": -2.174255132675171,
      "logps": -91.97770690917969,
      "loss": 0.0133,
      "objective": 0.0109206298366189,
      "ranking_idealized": 0.6499999761581421,
      "ranking_idealized_expo": 0.5333333611488342,
      "ranking_simple": 0.5166666507720947,
      "regularize": 0.010918223299086094,
      "step": 1930
    },
    {
      "dpo_loss": 0.6925593614578247,
      "epoch": 1.828202503739863,
      "grad_norm": 62.11529994850697,
      "learning_rate": 2.213261189245458e-09,
      "logits": -2.0892937183380127,
      "logps": -95.28164672851562,
      "loss": 0.0128,
      "objective": 0.0141153484582901,
      "ranking_idealized": 0.7166666388511658,
      "ranking_idealized_expo": 0.6166666746139526,
      "ranking_simple": 0.6333333253860474,
      "regularize": 0.01411362923681736,
      "step": 1935
    },
    {
      "dpo_loss": 0.6907196044921875,
      "epoch": 1.832926541217227,
      "grad_norm": 75.31073605655762,
      "learning_rate": 2.093519379466602e-09,
      "logits": -2.219423770904541,
      "logps": -93.87139129638672,
      "loss": 0.0136,
      "objective": 0.012250066734850407,
      "ranking_idealized": 0.6666666865348816,
      "ranking_idealized_expo": 0.6333333253860474,
      "ranking_simple": 0.6333333253860474,
      "regularize": 0.01224894542247057,
      "step": 1940
    },
    {
      "dpo_loss": 0.6914030909538269,
      "epoch": 1.837650578694591,
      "grad_norm": 59.36851585608135,
      "learning_rate": 1.9770381707817696e-09,
      "logits": -2.181976318359375,
      "logps": -89.93838500976562,
      "loss": 0.014,
      "objective": 0.01064326148480177,
      "ranking_idealized": 0.7333333492279053,
      "ranking_idealized_expo": 0.6000000238418579,
      "ranking_simple": 0.6000000238418579,
      "regularize": 0.010639534331858158,
      "step": 1945
    },
    {
      "dpo_loss": 0.6927531361579895,
      "epoch": 1.842374616171955,
      "grad_norm": 62.813994604552214,
      "learning_rate": 1.8638254911111816e-09,
      "logits": -2.1365416049957275,
      "logps": -93.81517028808594,
      "loss": 0.0133,
      "objective": 0.016254087910056114,
      "ranking_idealized": 0.6499999761581421,
      "ranking_idealized_expo": 0.5166666507720947,
      "ranking_simple": 0.5166666507720947,
      "regularize": 0.01625274494290352,
      "step": 1950
    },
    {
      "epoch": 1.842374616171955,
      "eval_dpo_loss": 0.6928887367248535,
      "eval_logits": -1.9892430305480957,
      "eval_logps": -98.45298767089844,
      "eval_loss": 0.008893881924450397,
      "eval_objective": 0.008952551521360874,
      "eval_ranking_idealized": 0.6022099256515503,
      "eval_ranking_idealized_expo": 0.5207182168960571,
      "eval_ranking_simple": 0.5179557800292969,
      "eval_regularize": 0.008950230665504932,
      "eval_runtime": 445.189,
      "eval_samples_per_second": 13.006,
      "eval_steps_per_second": 3.253,
      "step": 1950
    },
    {
      "dpo_loss": 0.6905800104141235,
      "epoch": 1.847098653649319,
      "grad_norm": 62.33667060485866,
      "learning_rate": 1.7538890459131094e-09,
      "logits": -2.0535969734191895,
      "logps": -92.17353057861328,
      "loss": 0.0125,
      "objective": 0.0103539377450943,
      "ranking_idealized": 0.6166666746139526,
      "ranking_idealized_expo": 0.5333333611488342,
      "ranking_simple": 0.5333333611488342,
      "regularize": 0.010349558666348457,
      "step": 1955
    },
    {
      "dpo_loss": 0.6896222233772278,
      "epoch": 1.8518226911266829,
      "grad_norm": 63.19798951059373,
      "learning_rate": 1.647236317659423e-09,
      "logits": -2.209256172180176,
      "logps": -90.65841674804688,
      "loss": 0.0117,
      "objective": 0.012652536854147911,
      "ranking_idealized": 0.699999988079071,
      "ranking_idealized_expo": 0.5833333134651184,
      "ranking_simple": 0.5833333134651184,
      "regularize": 0.012651127763092518,
      "step": 1960
    },
    {
      "dpo_loss": 0.6895377039909363,
      "epoch": 1.856546728604047,
      "grad_norm": 72.48024831454246,
      "learning_rate": 1.5438745653263086e-09,
      "logits": -2.0985636711120605,
      "logps": -91.15563201904297,
      "loss": 0.0112,
      "objective": 0.009469871409237385,
      "ranking_idealized": 0.6499999761581421,
      "ranking_idealized_expo": 0.5833333134651184,
      "ranking_simple": 0.5833333134651184,
      "regularize": 0.009467961266636848,
      "step": 1965
    },
    {
      "dpo_loss": 0.6904885172843933,
      "epoch": 1.861270766081411,
      "grad_norm": 58.60723650611681,
      "learning_rate": 1.4438108239002322e-09,
      "logits": -2.138218641281128,
      "logps": -93.22808074951172,
      "loss": 0.013,
      "objective": 0.013735007494688034,
      "ranking_idealized": 0.5666666626930237,
      "ranking_idealized_expo": 0.5166666507720947,
      "ranking_simple": 0.5166666507720947,
      "regularize": 0.01373241189867258,
      "step": 1970
    },
    {
      "dpo_loss": 0.6905199885368347,
      "epoch": 1.8659948035587748,
      "grad_norm": 75.78083265362716,
      "learning_rate": 1.3470519038991268e-09,
      "logits": -2.1427910327911377,
      "logps": -94.4820556640625,
      "loss": 0.0133,
      "objective": 0.016799870878458023,
      "ranking_idealized": 0.6166666746139526,
      "ranking_idealized_expo": 0.5666666626930237,
      "ranking_simple": 0.5666666626930237,
      "regularize": 0.016797440126538277,
      "step": 1975
    },
    {
      "dpo_loss": 0.692724347114563,
      "epoch": 1.8707188410361388,
      "grad_norm": 67.62639102220639,
      "learning_rate": 1.253604390908819e-09,
      "logits": -2.068427562713623,
      "logps": -91.92679595947266,
      "loss": 0.0118,
      "objective": 0.012069400399923325,
      "ranking_idealized": 0.6333333253860474,
      "ranking_idealized_expo": 0.5833333134651184,
      "ranking_simple": 0.5833333134651184,
      "regularize": 0.012066869996488094,
      "step": 1980
    },
    {
      "dpo_loss": 0.6905953288078308,
      "epoch": 1.8754428785135029,
      "grad_norm": 67.47097990851285,
      "learning_rate": 1.1634746451348487e-09,
      "logits": -2.1265487670898438,
      "logps": -91.03251647949219,
      "loss": 0.0117,
      "objective": 0.007730665151029825,
      "ranking_idealized": 0.5666666626930237,
      "ranking_idealized_expo": 0.4833333194255829,
      "ranking_simple": 0.4833333194255829,
      "regularize": 0.007728379685431719,
      "step": 1985
    },
    {
      "dpo_loss": 0.6893682479858398,
      "epoch": 1.8801669159908667,
      "grad_norm": 66.59499294449671,
      "learning_rate": 1.0766688009695545e-09,
      "logits": -2.200402021408081,
      "logps": -91.0616226196289,
      "loss": 0.0121,
      "objective": 0.011565645225346088,
      "ranking_idealized": 0.699999988079071,
      "ranking_idealized_expo": 0.6499999761581421,
      "ranking_simple": 0.6499999761581421,
      "regularize": 0.01156419888138771,
      "step": 1990
    },
    {
      "dpo_loss": 0.6916558742523193,
      "epoch": 1.884890953468231,
      "grad_norm": 59.74243706815504,
      "learning_rate": 9.931927665745521e-10,
      "logits": -2.1524574756622314,
      "logps": -90.13037109375,
      "loss": 0.0111,
      "objective": 0.012568699195981026,
      "ranking_idealized": 0.550000011920929,
      "ranking_idealized_expo": 0.4000000059604645,
      "ranking_simple": 0.4000000059604645,
      "regularize": 0.012566999532282352,
      "step": 1995
    },
    {
      "dpo_loss": 0.6858018636703491,
      "epoch": 1.8896149909455948,
      "grad_norm": 67.18519308744443,
      "learning_rate": 9.130522234786498e-10,
      "logits": -2.1586594581604004,
      "logps": -87.96702575683594,
      "loss": 0.012,
      "objective": 0.01719100959599018,
      "ranking_idealized": 0.6166666746139526,
      "ranking_idealized_expo": 0.5666666626930237,
      "ranking_simple": 0.5666666626930237,
      "regularize": 0.017189564183354378,
      "step": 2000
    },
    {
      "epoch": 1.8896149909455948,
      "eval_dpo_loss": 0.6928969025611877,
      "eval_logits": -1.9894063472747803,
      "eval_logps": -98.4583511352539,
      "eval_loss": 0.008732160553336143,
      "eval_objective": 0.008810975588858128,
      "eval_ranking_idealized": 0.6022099256515503,
      "eval_ranking_idealized_expo": 0.5207182168960571,
      "eval_ranking_simple": 0.5179557800292969,
      "eval_regularize": 0.008808653801679611,
      "eval_runtime": 446.6812,
      "eval_samples_per_second": 12.962,
      "eval_steps_per_second": 3.242,
      "step": 2000
    },
    {
      "dpo_loss": 0.6894116997718811,
      "epoch": 1.8943390284229589,
      "grad_norm": 63.88750093236265,
      "learning_rate": 8.36252626191103e-10,
      "logits": -2.1512064933776855,
      "logps": -90.55884552001953,
      "loss": 0.0112,
      "objective": 0.011560056358575821,
      "ranking_idealized": 0.5833333134651184,
      "ranking_idealized_expo": 0.4333333373069763,
      "ranking_simple": 0.4333333373069763,
      "regularize": 0.011556769721210003,
      "step": 2005
    },
    {
      "dpo_loss": 0.691102147102356,
      "epoch": 1.899063065900323,
      "grad_norm": 66.23415561126623,
      "learning_rate": 7.627992018304163e-10,
      "logits": -2.1225428581237793,
      "logps": -90.3371810913086,
      "loss": 0.012,
      "objective": 0.009777167811989784,
      "ranking_idealized": 0.5333333611488342,
      "ranking_idealized_expo": 0.38333332538604736,
      "ranking_simple": 0.38333332538604736,
      "regularize": 0.009775066748261452,
      "step": 2010
    },
    {
      "dpo_loss": 0.6881377100944519,
      "epoch": 1.9037871033776868,
      "grad_norm": 69.19769212044956,
      "learning_rate": 6.926969497685397e-10,
      "logits": -2.1484780311584473,
      "logps": -90.53960418701172,
      "loss": 0.0128,
      "objective": 0.011675039306282997,
      "ranking_idealized": 0.550000011920929,
      "ranking_idealized_expo": 0.5166666507720947,
      "ranking_simple": 0.5166666507720947,
      "regularize": 0.01167262066155672,
      "step": 2015
    },
    {
      "dpo_loss": 0.6911852955818176,
      "epoch": 1.9085111408550508,
      "grad_norm": 54.24867081294276,
      "learning_rate": 6.259506412906401e-10,
      "logits": -2.036067008972168,
      "logps": -94.92063903808594,
      "loss": 0.0114,
      "objective": 0.011328586377203465,
      "ranking_idealized": 0.4833333194255829,
      "ranking_idealized_expo": 0.44999998807907104,
      "ranking_simple": 0.44999998807907104,
      "regularize": 0.011327385902404785,
      "step": 2020
    },
    {
      "dpo_loss": 0.6897438168525696,
      "epoch": 1.9132351783324149,
      "grad_norm": 66.21589409339711,
      "learning_rate": 5.625648192703114e-10,
      "logits": -2.100722074508667,
      "logps": -88.59317779541016,
      "loss": 0.0111,
      "objective": 0.01007751189172268,
      "ranking_idealized": 0.6166666746139526,
      "ranking_idealized_expo": 0.550000011920929,
      "ranking_simple": 0.550000011920929,
      "regularize": 0.010075613856315613,
      "step": 2025
    },
    {
      "dpo_loss": 0.6879320740699768,
      "epoch": 1.9179592158097787,
      "grad_norm": 97.2021444500617,
      "learning_rate": 5.025437978604219e-10,
      "logits": -2.174182653427124,
      "logps": -98.01433563232422,
      "loss": 0.0128,
      "objective": 0.015440816059708595,
      "ranking_idealized": 0.699999988079071,
      "ranking_idealized_expo": 0.6333333253860474,
      "ranking_simple": 0.6333333253860474,
      "regularize": 0.01543727982789278,
      "step": 2030
    },
    {
      "dpo_loss": 0.6889265775680542,
      "epoch": 1.9226832532871427,
      "grad_norm": 60.36698105450304,
      "learning_rate": 4.458916621994713e-10,
      "logits": -2.1146934032440186,
      "logps": -96.32176208496094,
      "loss": 0.013,
      "objective": 0.012396620586514473,
      "ranking_idealized": 0.7833333611488342,
      "ranking_idealized_expo": 0.5833333134651184,
      "ranking_simple": 0.5833333134651184,
      "regularize": 0.012394459918141365,
      "step": 2035
    },
    {
      "dpo_loss": 0.6921891570091248,
      "epoch": 1.9274072907645068,
      "grad_norm": 62.70952327603329,
      "learning_rate": 3.9261226813353533e-10,
      "logits": -2.2713499069213867,
      "logps": -94.95305633544922,
      "loss": 0.0111,
      "objective": 0.012650835327804089,
      "ranking_idealized": 0.6000000238418579,
      "ranking_idealized_expo": 0.5166666507720947,
      "ranking_simple": 0.5166666507720947,
      "regularize": 0.012648210860788822,
      "step": 2040
    },
    {
      "dpo_loss": 0.6918179988861084,
      "epoch": 1.9321313282418706,
      "grad_norm": 60.71087366165103,
      "learning_rate": 3.4270924195384246e-10,
      "logits": -2.072065830230713,
      "logps": -93.6068344116211,
      "loss": 0.0112,
      "objective": 0.010315236635506153,
      "ranking_idealized": 0.5333333611488342,
      "ranking_idealized_expo": 0.46666666865348816,
      "ranking_simple": 0.46666666865348816,
      "regularize": 0.010312527418136597,
      "step": 2045
    },
    {
      "dpo_loss": 0.6912323236465454,
      "epoch": 1.9368553657192347,
      "grad_norm": 66.0225333978305,
      "learning_rate": 2.9618598014997107e-10,
      "logits": -2.1574909687042236,
      "logps": -93.39547729492188,
      "loss": 0.0119,
      "objective": 0.012261408381164074,
      "ranking_idealized": 0.5,
      "ranking_idealized_expo": 0.4166666567325592,
      "ranking_simple": 0.4166666567325592,
      "regularize": 0.012259239330887794,
      "step": 2050
    },
    {
      "epoch": 1.9368553657192347,
      "eval_dpo_loss": 0.6928916573524475,
      "eval_logits": -1.989404320716858,
      "eval_logps": -98.45708465576172,
      "eval_loss": 0.008758697658777237,
      "eval_objective": 0.008833469823002815,
      "eval_ranking_idealized": 0.6022099256515503,
      "eval_ranking_idealized_expo": 0.5207182168960571,
      "eval_ranking_simple": 0.5179557800292969,
      "eval_regularize": 0.008830932900309563,
      "eval_runtime": 446.0684,
      "eval_samples_per_second": 12.98,
      "eval_steps_per_second": 3.246,
      "step": 2050
    },
    {
      "dpo_loss": 0.6905261874198914,
      "epoch": 1.9415794031965987,
      "grad_norm": 72.43113470913532,
      "learning_rate": 2.5304564917865145e-10,
      "logits": -2.20497465133667,
      "logps": -93.12974548339844,
      "loss": 0.0106,
      "objective": 0.011251457966864109,
      "ranking_idealized": 0.7333333492279053,
      "ranking_idealized_expo": 0.6833333373069763,
      "ranking_simple": 0.6833333373069763,
      "regularize": 0.011249854229390621,
      "step": 2055
    },
    {
      "dpo_loss": 0.6920349597930908,
      "epoch": 1.9463034406739625,
      "grad_norm": 63.18535597705442,
      "learning_rate": 2.132911852482766e-10,
      "logits": -2.2412173748016357,
      "logps": -90.34603881835938,
      "loss": 0.0125,
      "objective": 0.013630078174173832,
      "ranking_idealized": 0.6000000238418579,
      "ranking_idealized_expo": 0.550000011920929,
      "ranking_simple": 0.550000011920929,
      "regularize": 0.013628335669636726,
      "step": 2060
    },
    {
      "dpo_loss": 0.691752016544342,
      "epoch": 1.9510274781513268,
      "grad_norm": 69.39489329530257,
      "learning_rate": 1.7692529411904578e-10,
      "logits": -2.248879909515381,
      "logps": -96.89351654052734,
      "loss": 0.012,
      "objective": 0.011724698357284069,
      "ranking_idealized": 0.6499999761581421,
      "ranking_idealized_expo": 0.5666666626930237,
      "ranking_simple": 0.5666666626930237,
      "regularize": 0.011722984723746777,
      "step": 2065
    },
    {
      "dpo_loss": 0.6898305416107178,
      "epoch": 1.9557515156286907,
      "grad_norm": 64.01320477510419,
      "learning_rate": 1.4395045091880608e-10,
      "logits": -2.133453845977783,
      "logps": -89.6153564453125,
      "loss": 0.0122,
      "objective": 0.013158326968550682,
      "ranking_idealized": 0.6166666746139526,
      "ranking_idealized_expo": 0.5333333611488342,
      "ranking_simple": 0.5333333611488342,
      "regularize": 0.013156161643564701,
      "step": 2070
    },
    {
      "dpo_loss": 0.6894416213035583,
      "epoch": 1.9604755531060545,
      "grad_norm": 70.45537055239048,
      "learning_rate": 1.1436889997460397e-10,
      "logits": -2.1818015575408936,
      "logps": -87.75623321533203,
      "loss": 0.0122,
      "objective": 0.01147166732698679,
      "ranking_idealized": 0.5666666626930237,
      "ranking_idealized_expo": 0.5333333611488342,
      "ranking_simple": 0.5333333611488342,
      "regularize": 0.0114695830270648,
      "step": 2075
    },
    {
      "dpo_loss": 0.689843475818634,
      "epoch": 1.9651995905834188,
      "grad_norm": 72.42995706576724,
      "learning_rate": 8.818265465991292e-11,
      "logits": -2.1221158504486084,
      "logps": -92.03446960449219,
      "loss": 0.0116,
      "objective": 0.01287260465323925,
      "ranking_idealized": 0.6666666865348816,
      "ranking_idealized_expo": 0.6000000238418579,
      "ranking_simple": 0.6000000238418579,
      "regularize": 0.012871033512055874,
      "step": 2080
    },
    {
      "dpo_loss": 0.6901150345802307,
      "epoch": 1.9699236280607826,
      "grad_norm": 67.99462621349231,
      "learning_rate": 6.539349725760423e-11,
      "logits": -2.2134578227996826,
      "logps": -94.01133728027344,
      "loss": 0.0138,
      "objective": 0.014971112832427025,
      "ranking_idealized": 0.6333333253860474,
      "ranking_idealized_expo": 0.5666666626930237,
      "ranking_simple": 0.5666666626930237,
      "regularize": 0.014969981275498867,
      "step": 2085
    },
    {
      "dpo_loss": 0.6904923915863037,
      "epoch": 1.9746476655381466,
      "grad_norm": 64.88891143293061,
      "learning_rate": 4.600297883866067e-11,
      "logits": -2.1357476711273193,
      "logps": -88.28942108154297,
      "loss": 0.0103,
      "objective": 0.008000458590686321,
      "ranking_idealized": 0.6166666746139526,
      "ranking_idealized_expo": 0.5,
      "ranking_simple": 0.5,
      "regularize": 0.00799502618610859,
      "step": 2090
    },
    {
      "dpo_loss": 0.691422700881958,
      "epoch": 1.9793717030155107,
      "grad_norm": 59.76257934662463,
      "learning_rate": 3.0012419156572044e-11,
      "logits": -2.106537103652954,
      "logps": -92.17442321777344,
      "loss": 0.0128,
      "objective": 0.012720795348286629,
      "ranking_idealized": 0.6499999761581421,
      "ranking_idealized_expo": 0.5666666626930237,
      "ranking_simple": 0.5666666626930237,
      "regularize": 0.0127179604023695,
      "step": 2095
    },
    {
      "dpo_loss": 0.6881771087646484,
      "epoch": 1.9840957404928745,
      "grad_norm": 65.98055833646369,
      "learning_rate": 1.7422906557557073e-11,
      "logits": -2.096862316131592,
      "logps": -96.3370132446289,
      "loss": 0.0116,
      "objective": 0.01271964143961668,
      "ranking_idealized": 0.550000011920929,
      "ranking_idealized_expo": 0.4833333194255829,
      "ranking_simple": 0.4833333194255829,
      "regularize": 0.01271754689514637,
      "step": 2100
    },
    {
      "epoch": 1.9840957404928745,
      "eval_dpo_loss": 0.6928920149803162,
      "eval_logits": -1.9894039630889893,
      "eval_logps": -98.45726013183594,
      "eval_loss": 0.008759252727031708,
      "eval_objective": 0.008833796717226505,
      "eval_ranking_idealized": 0.6022099256515503,
      "eval_ranking_idealized_expo": 0.5207182168960571,
      "eval_ranking_simple": 0.5179557800292969,
      "eval_regularize": 0.008831293322145939,
      "eval_runtime": 446.6316,
      "eval_samples_per_second": 12.964,
      "eval_steps_per_second": 3.242,
      "step": 2100
    },
    {
      "dpo_loss": 0.69146329164505,
      "epoch": 1.9888197779702386,
      "grad_norm": 64.49594040581191,
      "learning_rate": 8.235297906444837e-12,
      "logits": -2.149221181869507,
      "logps": -88.66574096679688,
      "loss": 0.0128,
      "objective": 0.01396742183715105,
      "ranking_idealized": 0.5666666626930237,
      "ranking_idealized_expo": 0.5,
      "ranking_simple": 0.5,
      "regularize": 0.013964297249913216,
      "step": 2105
    },
    {
      "dpo_loss": 0.6907679438591003,
      "epoch": 1.9935438154476026,
      "grad_norm": 70.66784622308231,
      "learning_rate": 2.450218528377013e-12,
      "logits": -2.137131452560425,
      "logps": -93.39989471435547,
      "loss": 0.0112,
      "objective": 0.010235412046313286,
      "ranking_idealized": 0.550000011920929,
      "ranking_idealized_expo": 0.4833333194255829,
      "ranking_simple": 0.46666666865348816,
      "regularize": 0.010233612731099129,
      "step": 2110
    },
    {
      "dpo_loss": 0.6910449266433716,
      "epoch": 1.9982678529249664,
      "grad_norm": 60.167547577945186,
      "learning_rate": 6.806216624188899e-14,
      "logits": -2.124387502670288,
      "logps": -93.0544204711914,
      "loss": 0.014,
      "objective": 0.009751829318702221,
      "ranking_idealized": 0.5666666626930237,
      "ranking_idealized_expo": 0.4833333194255829,
      "ranking_simple": 0.4833333194255829,
      "regularize": 0.009749443270266056,
      "step": 2115
    },
    {
      "epoch": 1.9992126604204392,
      "step": 2116,
      "total_flos": 0.0,
      "train_loss": 0.020640970417914562,
      "train_runtime": 38688.7685,
      "train_samples_per_second": 2.626,
      "train_steps_per_second": 0.055
    }
  ],
  "logging_steps": 5,
  "max_steps": 2116,
  "num_input_tokens_seen": 0,
  "num_train_epochs": 2,
  "save_steps": 500,
  "stateful_callbacks": {
    "TrainerControl": {
      "args": {
        "should_epoch_stop": false,
        "should_evaluate": false,
        "should_log": false,
        "should_save": false,
        "should_training_stop": false
      },
      "attributes": {}
    }
  },
  "total_flos": 0.0,
  "train_batch_size": 1,
  "trial_name": null,
  "trial_params": null
}