{
  "best_metric": null,
  "best_model_checkpoint": null,
  "epoch": 1.0,
  "eval_steps": 100,
  "global_step": 684,
  "is_hyper_param_search": false,
  "is_local_process_zero": true,
  "is_world_process_zero": true,
  "log_history": [
    {
      "dpo_losses": 0.6931471824645996,
      "epoch": 0.0,
      "grad_norm": 1.9721656718078973,
      "learning_rate": 7.246376811594203e-09,
      "logits/chosen": -2.8746490478515625,
      "logits/rejected": -2.840811252593994,
      "logps/chosen": -227.73272705078125,
      "logps/rejected": -174.71890258789062,
      "loss": 0.6931,
      "positive_losses": 0.0,
      "rewards/accuracies": 0.0,
      "rewards/chosen": 0.0,
      "rewards/margins": 0.0,
      "rewards/margins_max": 0.0,
      "rewards/margins_min": 0.0,
      "rewards/margins_std": 0.0,
      "rewards/rejected": 0.0,
      "step": 1
    },
    {
      "dpo_losses": 0.6933290958404541,
      "epoch": 0.01,
      "grad_norm": 25.64726309944567,
      "learning_rate": 7.246376811594203e-08,
      "logits/chosen": -2.7698583602905273,
      "logits/rejected": -2.726212501525879,
      "logps/chosen": -244.640380859375,
      "logps/rejected": -240.38246154785156,
      "loss": 0.6981,
      "positive_losses": 0.04526931792497635,
      "rewards/accuracies": 0.2638888955116272,
      "rewards/chosen": -7.686785829719156e-05,
      "rewards/margins": -0.0003627391706686467,
      "rewards/margins_max": 0.0012993874261155725,
      "rewards/margins_min": -0.0022027057129889727,
      "rewards/margins_std": 0.0015639414777979255,
      "rewards/rejected": 0.00028587126871570945,
      "step": 10
    },
    {
      "dpo_losses": 0.6930510401725769,
      "epoch": 0.03,
      "grad_norm": 20.592262133277757,
      "learning_rate": 1.4492753623188405e-07,
      "logits/chosen": -2.897423028945923,
      "logits/rejected": -2.8587498664855957,
      "logps/chosen": -317.21636962890625,
      "logps/rejected": -239.84121704101562,
      "loss": 0.6999,
      "positive_losses": 0.07797051966190338,
      "rewards/accuracies": 0.5249999761581421,
      "rewards/chosen": 0.0003467574715614319,
      "rewards/margins": 0.00019531897851265967,
      "rewards/margins_max": 0.0034703321289271116,
      "rewards/margins_min": -0.0027894906233996153,
      "rewards/margins_std": 0.002780328970402479,
      "rewards/rejected": 0.000151438478496857,
      "step": 20
    },
    {
      "dpo_losses": 0.6931239366531372,
      "epoch": 0.04,
      "grad_norm": 16.747192147468148,
      "learning_rate": 2.1739130434782607e-07,
      "logits/chosen": -2.811279535293579,
      "logits/rejected": -2.790393114089966,
      "logps/chosen": -264.90753173828125,
      "logps/rejected": -223.55331420898438,
      "loss": 0.6988,
      "positive_losses": 0.052257489413022995,
      "rewards/accuracies": 0.4625000059604645,
      "rewards/chosen": 0.0009221597574651241,
      "rewards/margins": 4.864180664299056e-05,
      "rewards/margins_max": 0.002964555751532316,
      "rewards/margins_min": -0.0025322353467345238,
      "rewards/margins_std": 0.0024663216900080442,
      "rewards/rejected": 0.0008735180017538369,
      "step": 30
    },
    {
      "dpo_losses": 0.6930278539657593,
      "epoch": 0.06,
      "grad_norm": 12.398083667958339,
      "learning_rate": 2.898550724637681e-07,
      "logits/chosen": -2.864264965057373,
      "logits/rejected": -2.834386110305786,
      "logps/chosen": -304.97265625,
      "logps/rejected": -266.8782043457031,
      "loss": 0.6961,
      "positive_losses": 0.031120896339416504,
      "rewards/accuracies": 0.4625000059604645,
      "rewards/chosen": 0.0022245876025408506,
      "rewards/margins": 0.0002419196825940162,
      "rewards/margins_max": 0.004220693372189999,
      "rewards/margins_min": -0.0033918179105967283,
      "rewards/margins_std": 0.0034401887096464634,
      "rewards/rejected": 0.001982667949050665,
      "step": 40
    },
    {
      "dpo_losses": 0.6931858658790588,
      "epoch": 0.07,
      "grad_norm": 8.878267358365656,
      "learning_rate": 3.6231884057971015e-07,
      "logits/chosen": -2.870177984237671,
      "logits/rejected": -2.8384270668029785,
      "logps/chosen": -260.7277526855469,
      "logps/rejected": -234.89218139648438,
      "loss": 0.6942,
      "positive_losses": 0.022727489471435547,
      "rewards/accuracies": 0.4124999940395355,
      "rewards/chosen": 0.003678619861602783,
      "rewards/margins": -7.445671508321539e-05,
      "rewards/margins_max": 0.0031701656989753246,
      "rewards/margins_min": -0.0034985982347279787,
      "rewards/margins_std": 0.0029860776849091053,
      "rewards/rejected": 0.003753076074644923,
      "step": 50
    },
    {
      "dpo_losses": 0.6925040483474731,
      "epoch": 0.09,
      "grad_norm": 1.6719366044432857,
      "learning_rate": 4.3478260869565214e-07,
      "logits/chosen": -2.891150951385498,
      "logits/rejected": -2.843611478805542,
      "logps/chosen": -280.95947265625,
      "logps/rejected": -248.34915161132812,
      "loss": 0.6931,
      "positive_losses": 0.0025458037853240967,
      "rewards/accuracies": 0.6625000238418579,
      "rewards/chosen": 0.0058168042451143265,
      "rewards/margins": 0.0012899416033178568,
      "rewards/margins_max": 0.005339757539331913,
      "rewards/margins_min": -0.002916422439739108,
      "rewards/margins_std": 0.003642101539298892,
      "rewards/rejected": 0.004526862408965826,
      "step": 60
    },
    {
      "dpo_losses": 0.6921008229255676,
      "epoch": 0.1,
      "grad_norm": 1.9262001556938266,
      "learning_rate": 4.999967381905813e-07,
      "logits/chosen": -2.7916743755340576,
      "logits/rejected": -2.7574045658111572,
      "logps/chosen": -290.14361572265625,
      "logps/rejected": -212.0419158935547,
      "loss": 0.6926,
      "positive_losses": 0.004826736636459827,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": 0.006598347332328558,
      "rewards/margins": 0.002098249038681388,
      "rewards/margins_max": 0.006628250237554312,
      "rewards/margins_min": -0.0018369384342804551,
      "rewards/margins_std": 0.00378109747543931,
      "rewards/rejected": 0.004500098060816526,
      "step": 70
    },
    {
      "dpo_losses": 0.6920569539070129,
      "epoch": 0.12,
      "grad_norm": 13.284239668101833,
      "learning_rate": 4.996054240392509e-07,
      "logits/chosen": -2.7785937786102295,
      "logits/rejected": -2.759201765060425,
      "logps/chosen": -264.8086242675781,
      "logps/rejected": -243.39212036132812,
      "loss": 0.6931,
      "positive_losses": 0.014845085330307484,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": 0.007849474437534809,
      "rewards/margins": 0.0021913917735219,
      "rewards/margins_max": 0.008834488689899445,
      "rewards/margins_min": -0.00322969863191247,
      "rewards/margins_std": 0.005265987012535334,
      "rewards/rejected": 0.0056580835953354836,
      "step": 80
    },
    {
      "dpo_losses": 0.6912720203399658,
      "epoch": 0.13,
      "grad_norm": 2.317442586572389,
      "learning_rate": 4.985629178361649e-07,
      "logits/chosen": -2.8908092975616455,
      "logits/rejected": -2.8579392433166504,
      "logps/chosen": -272.96112060546875,
      "logps/rejected": -245.47073364257812,
      "loss": 0.6918,
      "positive_losses": 0.008871316909790039,
      "rewards/accuracies": 0.737500011920929,
      "rewards/chosen": 0.009552950970828533,
      "rewards/margins": 0.0037618838250637054,
      "rewards/margins_max": 0.010290957987308502,
      "rewards/margins_min": -0.002331961877644062,
      "rewards/margins_std": 0.005662451032549143,
      "rewards/rejected": 0.00579106854274869,
      "step": 90
    },
    {
      "dpo_losses": 0.6900182962417603,
      "epoch": 0.15,
      "grad_norm": 12.043613650426611,
      "learning_rate": 4.968719393609756e-07,
      "logits/chosen": -2.8450093269348145,
      "logits/rejected": -2.7789740562438965,
      "logps/chosen": -337.09600830078125,
      "logps/rejected": -243.02517700195312,
      "loss": 0.6918,
      "positive_losses": 0.017842482775449753,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": 0.012230691500008106,
      "rewards/margins": 0.0062872739508748055,
      "rewards/margins_max": 0.01721397042274475,
      "rewards/margins_min": -0.002463764976710081,
      "rewards/margins_std": 0.008927479386329651,
      "rewards/rejected": 0.0059434170834720135,
      "step": 100
    },
    {
      "epoch": 0.15,
      "eval_dpo_losses": 0.6922104358673096,
      "eval_logits/chosen": -2.8104522228240967,
      "eval_logits/rejected": -2.7718498706817627,
      "eval_logps/chosen": -283.23919677734375,
      "eval_logps/rejected": -257.4134826660156,
      "eval_loss": 0.6930333375930786,
      "eval_positive_losses": 0.004661041311919689,
      "eval_rewards/accuracies": 0.6000000238418579,
      "eval_rewards/chosen": 0.013542202301323414,
      "eval_rewards/margins": 0.001887724851258099,
      "eval_rewards/margins_max": 0.013032433576881886,
      "eval_rewards/margins_min": -0.008088048547506332,
      "eval_rewards/margins_std": 0.006959096062928438,
      "eval_rewards/rejected": 0.011654475703835487,
      "eval_runtime": 439.0202,
      "eval_samples_per_second": 4.556,
      "eval_steps_per_second": 0.285,
      "step": 100
    },
    {
      "dpo_losses": 0.6904088258743286,
      "epoch": 0.16,
      "grad_norm": 5.183696409472315,
      "learning_rate": 4.945369001834514e-07,
      "logits/chosen": -2.8782482147216797,
      "logits/rejected": -2.8235890865325928,
      "logps/chosen": -287.38250732421875,
      "logps/rejected": -222.830078125,
      "loss": 0.6908,
      "positive_losses": 0.0031826018821448088,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": 0.01270085759460926,
      "rewards/margins": 0.00550065329298377,
      "rewards/margins_max": 0.014992681331932545,
      "rewards/margins_min": -0.0012518208241090178,
      "rewards/margins_std": 0.007309773005545139,
      "rewards/rejected": 0.0072002047672867775,
      "step": 110
    },
    {
      "dpo_losses": 0.6896742582321167,
      "epoch": 0.18,
      "grad_norm": 1.9553746809300425,
      "learning_rate": 4.915638921541951e-07,
      "logits/chosen": -2.8301899433135986,
      "logits/rejected": -2.8140978813171387,
      "logps/chosen": -256.82879638671875,
      "logps/rejected": -236.7211456298828,
      "loss": 0.6898,
      "positive_losses": 0.0022869110107421875,
      "rewards/accuracies": 0.7875000238418579,
      "rewards/chosen": 0.013588182628154755,
      "rewards/margins": 0.00698423758149147,
      "rewards/margins_max": 0.018451116979122162,
      "rewards/margins_min": -0.001020266441628337,
      "rewards/margins_std": 0.008837779983878136,
      "rewards/rejected": 0.006603945046663284,
      "step": 120
    },
    {
      "dpo_losses": 0.6898840665817261,
      "epoch": 0.19,
      "grad_norm": 1.6022519000216686,
      "learning_rate": 4.879606715117018e-07,
      "logits/chosen": -2.881978988647461,
      "logits/rejected": -2.82822847366333,
      "logps/chosen": -274.6767578125,
      "logps/rejected": -234.8880615234375,
      "loss": 0.6897,
      "positive_losses": 0.007077217102050781,
      "rewards/accuracies": 0.6625000238418579,
      "rewards/chosen": 0.014239470474421978,
      "rewards/margins": 0.006568121258169413,
      "rewards/margins_max": 0.017170870676636696,
      "rewards/margins_min": -0.0028213211335241795,
      "rewards/margins_std": 0.00886800978332758,
      "rewards/rejected": 0.00767135014757514,
      "step": 130
    },
    {
      "dpo_losses": 0.6880620718002319,
      "epoch": 0.2,
      "grad_norm": 3.638881806293095,
      "learning_rate": 4.837366386472174e-07,
      "logits/chosen": -2.8916850090026855,
      "logits/rejected": -2.8278439044952393,
      "logps/chosen": -296.95111083984375,
      "logps/rejected": -253.7423553466797,
      "loss": 0.6901,
      "positive_losses": 0.002955913543701172,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": 0.01859217882156372,
      "rewards/margins": 0.010244466364383698,
      "rewards/margins_max": 0.024400196969509125,
      "rewards/margins_min": -0.0004118576762266457,
      "rewards/margins_std": 0.011427570134401321,
      "rewards/rejected": 0.008347713388502598,
      "step": 140
    },
    {
      "dpo_losses": 0.6885315179824829,
      "epoch": 0.22,
      "grad_norm": 6.527916454528557,
      "learning_rate": 4.789028135801918e-07,
      "logits/chosen": -2.852397918701172,
      "logits/rejected": -2.8406243324279785,
      "logps/chosen": -297.78564453125,
      "logps/rejected": -271.2729187011719,
      "loss": 0.6914,
      "positive_losses": 0.054250240325927734,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": 0.01786804385483265,
      "rewards/margins": 0.009341379627585411,
      "rewards/margins_max": 0.03060457669198513,
      "rewards/margins_min": -0.008363897912204266,
      "rewards/margins_std": 0.01767081953585148,
      "rewards/rejected": 0.008526663295924664,
      "step": 150
    },
    {
      "dpo_losses": 0.6880606412887573,
      "epoch": 0.23,
      "grad_norm": 8.44151922500754,
      "learning_rate": 4.7347180720830627e-07,
      "logits/chosen": -2.8739285469055176,
      "logits/rejected": -2.7969882488250732,
      "logps/chosen": -300.0534973144531,
      "logps/rejected": -260.5088195800781,
      "loss": 0.6899,
      "positive_losses": 0.006076812744140625,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": 0.01983806863427162,
      "rewards/margins": 0.01025958452373743,
      "rewards/margins_max": 0.028022507205605507,
      "rewards/margins_min": -0.0024425422307103872,
      "rewards/margins_std": 0.013813845813274384,
      "rewards/rejected": 0.009578483179211617,
      "step": 160
    },
    {
      "dpo_losses": 0.684917151927948,
      "epoch": 0.25,
      "grad_norm": 1.8673882157090276,
      "learning_rate": 4.6745778840708107e-07,
      "logits/chosen": -2.8020739555358887,
      "logits/rejected": -2.7900993824005127,
      "logps/chosen": -295.7058410644531,
      "logps/rejected": -225.8823699951172,
      "loss": 0.6858,
      "positive_losses": 0.0014563084114342928,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": 0.026786331087350845,
      "rewards/margins": 0.01665376126766205,
      "rewards/margins_max": 0.03810211643576622,
      "rewards/margins_min": -0.0024380278773605824,
      "rewards/margins_std": 0.017996035516262054,
      "rewards/rejected": 0.010132570751011372,
      "step": 170
    },
    {
      "dpo_losses": 0.6871532201766968,
      "epoch": 0.26,
      "grad_norm": 2.603750597458806,
      "learning_rate": 4.6087644706489703e-07,
      "logits/chosen": -2.833789587020874,
      "logits/rejected": -2.7939248085021973,
      "logps/chosen": -294.24456787109375,
      "logps/rejected": -254.28988647460938,
      "loss": 0.6889,
      "positive_losses": 0.024321984499692917,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.022618165239691734,
      "rewards/margins": 0.01210461463779211,
      "rewards/margins_max": 0.030625376850366592,
      "rewards/margins_min": -0.004277624189853668,
      "rewards/margins_std": 0.015746023505926132,
      "rewards/rejected": 0.010513553395867348,
      "step": 180
    },
    {
      "dpo_losses": 0.686254620552063,
      "epoch": 0.28,
      "grad_norm": 1.7335945641686348,
      "learning_rate": 4.537449531498687e-07,
      "logits/chosen": -2.7748355865478516,
      "logits/rejected": -2.725888729095459,
      "logps/chosen": -280.95428466796875,
      "logps/rejected": -228.15097045898438,
      "loss": 0.6878,
      "positive_losses": 0.012805747799575329,
      "rewards/accuracies": 0.7875000238418579,
      "rewards/chosen": 0.028492039069533348,
      "rewards/margins": 0.013932852074503899,
      "rewards/margins_max": 0.03530807048082352,
      "rewards/margins_min": -0.002935736905783415,
      "rewards/margins_std": 0.017064960673451424,
      "rewards/rejected": 0.0145591851323843,
      "step": 190
    },
    {
      "dpo_losses": 0.6861851811408997,
      "epoch": 0.29,
      "grad_norm": 1.5625317379589145,
      "learning_rate": 4.4608191191535736e-07,
      "logits/chosen": -2.8794567584991455,
      "logits/rejected": -2.835148334503174,
      "logps/chosen": -274.84735107421875,
      "logps/rejected": -242.3157958984375,
      "loss": 0.6867,
      "positive_losses": 0.004458713345229626,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": 0.02746543660759926,
      "rewards/margins": 0.014083576388657093,
      "rewards/margins_max": 0.03864169120788574,
      "rewards/margins_min": -0.005826378706842661,
      "rewards/margins_std": 0.019664574414491653,
      "rewards/rejected": 0.01338185928761959,
      "step": 200
    },
    {
      "epoch": 0.29,
      "eval_dpo_losses": 0.6900327801704407,
      "eval_logits/chosen": -2.8075411319732666,
      "eval_logits/rejected": -2.7689805030822754,
      "eval_logps/chosen": -281.6053161621094,
      "eval_logps/rejected": -256.2250061035156,
      "eval_loss": 0.6917006373405457,
      "eval_positive_losses": 0.012367370538413525,
      "eval_rewards/accuracies": 0.6100000143051147,
      "eval_rewards/chosen": 0.02988099865615368,
      "eval_rewards/margins": 0.006341494619846344,
      "eval_rewards/margins_max": 0.03729286044836044,
      "eval_rewards/margins_min": -0.02144700102508068,
      "eval_rewards/margins_std": 0.01944614388048649,
      "eval_rewards/rejected": 0.023539502173662186,
      "eval_runtime": 457.2097,
      "eval_samples_per_second": 4.374,
      "eval_steps_per_second": 0.273,
      "step": 200
    },
    {
      "dpo_losses": 0.6849207878112793,
      "epoch": 0.31,
      "grad_norm": 6.776479704467401,
      "learning_rate": 4.379073153609896e-07,
      "logits/chosen": -2.85437273979187,
      "logits/rejected": -2.8159098625183105,
      "logps/chosen": -298.1737365722656,
      "logps/rejected": -258.9627685546875,
      "loss": 0.6892,
      "positive_losses": 0.024309396743774414,
      "rewards/accuracies": 0.737500011920929,
      "rewards/chosen": 0.031262680888175964,
      "rewards/margins": 0.016705606132745743,
      "rewards/margins_max": 0.0462987944483757,
      "rewards/margins_min": -0.009217771701514721,
      "rewards/margins_std": 0.024988915771245956,
      "rewards/rejected": 0.014557072892785072,
      "step": 210
    },
    {
      "dpo_losses": 0.6835730075836182,
      "epoch": 0.32,
      "grad_norm": 1.7559811016317117,
      "learning_rate": 4.292424900758128e-07,
      "logits/chosen": -2.8128745555877686,
      "logits/rejected": -2.7835748195648193,
      "logps/chosen": -278.7519836425781,
      "logps/rejected": -221.44998168945312,
      "loss": 0.687,
      "positive_losses": 0.01152725238353014,
      "rewards/accuracies": 0.7875000238418579,
      "rewards/chosen": 0.032679036259651184,
      "rewards/margins": 0.019430814310908318,
      "rewards/margins_max": 0.04703899472951889,
      "rewards/margins_min": -0.004503333941102028,
      "rewards/margins_std": 0.02318265102803707,
      "rewards/rejected": 0.013248221948742867,
      "step": 220
    },
    {
      "dpo_losses": 0.6845378279685974,
      "epoch": 0.34,
      "grad_norm": 5.2855521291360335,
      "learning_rate": 4.201100415996597e-07,
      "logits/chosen": -2.7867586612701416,
      "logits/rejected": -2.7540698051452637,
      "logps/chosen": -262.8429260253906,
      "logps/rejected": -259.90594482421875,
      "loss": 0.6873,
      "positive_losses": 0.010059595108032227,
      "rewards/accuracies": 0.737500011920929,
      "rewards/chosen": 0.033603329211473465,
      "rewards/margins": 0.017468122765421867,
      "rewards/margins_max": 0.05101371556520462,
      "rewards/margins_min": -0.005049402825534344,
      "rewards/margins_std": 0.025209274142980576,
      "rewards/rejected": 0.016135206446051598,
      "step": 230
    },
    {
      "dpo_losses": 0.6822143793106079,
      "epoch": 0.35,
      "grad_norm": 7.1123081067417635,
      "learning_rate": 4.1053379544787557e-07,
      "logits/chosen": -2.8566319942474365,
      "logits/rejected": -2.8126063346862793,
      "logps/chosen": -285.74066162109375,
      "logps/rejected": -246.8583221435547,
      "loss": 0.6875,
      "positive_losses": 0.0023063658736646175,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": 0.03864200785756111,
      "rewards/margins": 0.02219047024846077,
      "rewards/margins_max": 0.04993893578648567,
      "rewards/margins_min": -0.002027359325438738,
      "rewards/margins_std": 0.023801075294613838,
      "rewards/rejected": 0.01645154133439064,
      "step": 240
    },
    {
      "dpo_losses": 0.6844903230667114,
      "epoch": 0.37,
      "grad_norm": 2.225597078597608,
      "learning_rate": 4.0053873495326964e-07,
      "logits/chosen": -2.8150901794433594,
      "logits/rejected": -2.798675537109375,
      "logps/chosen": -243.178955078125,
      "logps/rejected": -226.9827117919922,
      "loss": 0.6875,
      "positive_losses": 0.0484701506793499,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": 0.033011239022016525,
      "rewards/margins": 0.017598429694771767,
      "rewards/margins_max": 0.047145940363407135,
      "rewards/margins_min": -0.0065514100715518,
      "rewards/margins_std": 0.02364528551697731,
      "rewards/rejected": 0.015412810258567333,
      "step": 250
    },
    {
      "dpo_losses": 0.6823347806930542,
      "epoch": 0.38,
      "grad_norm": 1.8954330972040565,
      "learning_rate": 3.9015093608745143e-07,
      "logits/chosen": -2.87410306930542,
      "logits/rejected": -2.808983087539673,
      "logps/chosen": -283.3040771484375,
      "logps/rejected": -240.4110565185547,
      "loss": 0.6824,
      "positive_losses": 0.011256242170929909,
      "rewards/accuracies": 0.824999988079071,
      "rewards/chosen": 0.03804592788219452,
      "rewards/margins": 0.021955588832497597,
      "rewards/margins_max": 0.05776814743876457,
      "rewards/margins_min": -0.004212898667901754,
      "rewards/margins_std": 0.028291797265410423,
      "rewards/rejected": 0.016090339049696922,
      "step": 260
    },
    {
      "dpo_losses": 0.681152880191803,
      "epoch": 0.39,
      "grad_norm": 2.3462040578320438,
      "learning_rate": 3.79397499431599e-07,
      "logits/chosen": -2.838275909423828,
      "logits/rejected": -2.7828450202941895,
      "logps/chosen": -291.1107177734375,
      "logps/rejected": -261.1417236328125,
      "loss": 0.6844,
      "positive_losses": 0.031092902645468712,
      "rewards/accuracies": 0.762499988079071,
      "rewards/chosen": 0.04311501979827881,
      "rewards/margins": 0.024433860555291176,
      "rewards/margins_max": 0.05960015207529068,
      "rewards/margins_min": -0.005230102688074112,
      "rewards/margins_std": 0.028690297156572342,
      "rewards/rejected": 0.018681161105632782,
      "step": 270
    },
    {
      "dpo_losses": 0.6834903955459595,
      "epoch": 0.41,
      "grad_norm": 1.678928576392419,
      "learning_rate": 3.6830647947413694e-07,
      "logits/chosen": -2.906564235687256,
      "logits/rejected": -2.8589329719543457,
      "logps/chosen": -273.8109436035156,
      "logps/rejected": -246.07925415039062,
      "loss": 0.6848,
      "positive_losses": 0.03764519840478897,
      "rewards/accuracies": 0.762499988079071,
      "rewards/chosen": 0.03491830825805664,
      "rewards/margins": 0.019722772762179375,
      "rewards/margins_max": 0.05492109805345535,
      "rewards/margins_min": -0.009231898002326488,
      "rewards/margins_std": 0.029070580378174782,
      "rewards/rejected": 0.015195533633232117,
      "step": 280
    },
    {
      "dpo_losses": 0.6819528937339783,
      "epoch": 0.42,
      "grad_norm": 1.5933364630845321,
      "learning_rate": 3.5690681141977837e-07,
      "logits/chosen": -2.8415896892547607,
      "logits/rejected": -2.795182704925537,
      "logps/chosen": -252.76455688476562,
      "logps/rejected": -203.42364501953125,
      "loss": 0.6836,
      "positive_losses": 0.024924278259277344,
      "rewards/accuracies": 0.762499988079071,
      "rewards/chosen": 0.039224088191986084,
      "rewards/margins": 0.022878160700201988,
      "rewards/margins_max": 0.06510117650032043,
      "rewards/margins_min": -0.010645577684044838,
      "rewards/margins_std": 0.034357957541942596,
      "rewards/rejected": 0.016345927491784096,
      "step": 290
    },
    {
      "dpo_losses": 0.6809358596801758,
      "epoch": 0.44,
      "grad_norm": 10.347091457783785,
      "learning_rate": 3.4522823570088067e-07,
      "logits/chosen": -2.840087413787842,
      "logits/rejected": -2.8272616863250732,
      "logps/chosen": -259.23687744140625,
      "logps/rejected": -251.6184844970703,
      "loss": 0.6825,
      "positive_losses": 0.014734840020537376,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": 0.03966939076781273,
      "rewards/margins": 0.02503836713731289,
      "rewards/margins_max": 0.07522507011890411,
      "rewards/margins_min": -0.010506678372621536,
      "rewards/margins_std": 0.03809111937880516,
      "rewards/rejected": 0.01463102363049984,
      "step": 300
    },
    {
      "epoch": 0.44,
      "eval_dpo_losses": 0.6878849864006042,
      "eval_logits/chosen": -2.8020598888397217,
      "eval_logits/rejected": -2.763356924057007,
      "eval_logps/chosen": -280.1855163574219,
      "eval_logps/rejected": -255.25390625,
      "eval_loss": 0.69089674949646,
      "eval_positive_losses": 0.022481659427285194,
      "eval_rewards/accuracies": 0.625,
      "eval_rewards/chosen": 0.04407881200313568,
      "eval_rewards/margins": 0.010828724130988121,
      "eval_rewards/margins_max": 0.061860937625169754,
      "eval_rewards/margins_min": -0.03410045802593231,
      "eval_rewards/margins_std": 0.03193770721554756,
      "eval_rewards/rejected": 0.03325008600950241,
      "eval_runtime": 439.4476,
      "eval_samples_per_second": 4.551,
      "eval_steps_per_second": 0.284,
      "step": 300
    },
    {
      "dpo_losses": 0.6795910596847534,
      "epoch": 0.45,
      "grad_norm": 1.8194274060306528,
      "learning_rate": 3.3330122038805277e-07,
      "logits/chosen": -2.861931562423706,
      "logits/rejected": -2.8135368824005127,
      "logps/chosen": -271.8392639160156,
      "logps/rejected": -225.01708984375,
      "loss": 0.6837,
      "positive_losses": 0.02801961824297905,
      "rewards/accuracies": 0.8500000238418579,
      "rewards/chosen": 0.04437270388007164,
      "rewards/margins": 0.027612268924713135,
      "rewards/margins_max": 0.07083487510681152,
      "rewards/margins_min": -0.0025140452198684216,
      "rewards/margins_std": 0.033803679049015045,
      "rewards/rejected": 0.016760434955358505,
      "step": 310
    },
    {
      "dpo_losses": 0.6798790693283081,
      "epoch": 0.47,
      "grad_norm": 1.8110670147086625,
      "learning_rate": 3.2115688170243734e-07,
      "logits/chosen": -2.8523640632629395,
      "logits/rejected": -2.82257080078125,
      "logps/chosen": -270.28253173828125,
      "logps/rejected": -248.6698455810547,
      "loss": 0.6802,
      "positive_losses": 0.009492230601608753,
      "rewards/accuracies": 0.762499988079071,
      "rewards/chosen": 0.04459889978170395,
      "rewards/margins": 0.027110621333122253,
      "rewards/margins_max": 0.06967566907405853,
      "rewards/margins_min": -0.005822637118399143,
      "rewards/margins_std": 0.03437542915344238,
      "rewards/rejected": 0.017488282173871994,
      "step": 320
    },
    {
      "dpo_losses": 0.6773503422737122,
      "epoch": 0.48,
      "grad_norm": 9.150401672552029,
      "learning_rate": 3.088269028370435e-07,
      "logits/chosen": -2.875394821166992,
      "logits/rejected": -2.8434793949127197,
      "logps/chosen": -295.0433349609375,
      "logps/rejected": -246.5893096923828,
      "loss": 0.6835,
      "positive_losses": 0.03743886947631836,
      "rewards/accuracies": 0.7875000238418579,
      "rewards/chosen": 0.04838422313332558,
      "rewards/margins": 0.03234843909740448,
      "rewards/margins_max": 0.08645691722631454,
      "rewards/margins_min": -0.006109377834945917,
      "rewards/margins_std": 0.04178870469331741,
      "rewards/rejected": 0.016035789623856544,
      "step": 330
    },
    {
      "dpo_losses": 0.6774032115936279,
      "epoch": 0.5,
      "grad_norm": 4.795115194677247,
      "learning_rate": 2.9634345129891294e-07,
      "logits/chosen": -2.8238093852996826,
      "logits/rejected": -2.785935163497925,
      "logps/chosen": -287.4772033691406,
      "logps/rejected": -254.5933074951172,
      "loss": 0.6804,
      "positive_losses": 0.03258161619305611,
      "rewards/accuracies": 0.762499988079071,
      "rewards/chosen": 0.04798700287938118,
      "rewards/margins": 0.032188571989536285,
      "rewards/margins_max": 0.0826764851808548,
      "rewards/margins_min": -0.006551785860210657,
      "rewards/margins_std": 0.04137192294001579,
      "rewards/rejected": 0.015798425301909447,
      "step": 340
    },
    {
      "dpo_losses": 0.6810469627380371,
      "epoch": 0.51,
      "grad_norm": 11.724114006745417,
      "learning_rate": 2.8373909498776744e-07,
      "logits/chosen": -2.8722732067108154,
      "logits/rejected": -2.854133129119873,
      "logps/chosen": -279.33917236328125,
      "logps/rejected": -276.32232666015625,
      "loss": 0.6845,
      "positive_losses": 0.04124107211828232,
      "rewards/accuracies": 0.762499988079071,
      "rewards/chosen": 0.04524795338511467,
      "rewards/margins": 0.024712715297937393,
      "rewards/margins_max": 0.06821637600660324,
      "rewards/margins_min": -0.0043562790378928185,
      "rewards/margins_std": 0.03368216007947922,
      "rewards/rejected": 0.020535241812467575,
      "step": 350
    },
    {
      "dpo_losses": 0.6761634945869446,
      "epoch": 0.53,
      "grad_norm": 5.401380341327095,
      "learning_rate": 2.710467172300768e-07,
      "logits/chosen": -2.7806248664855957,
      "logits/rejected": -2.74116587638855,
      "logps/chosen": -318.1079406738281,
      "logps/rejected": -248.9733428955078,
      "loss": 0.6774,
      "positive_losses": 0.012074279598891735,
      "rewards/accuracies": 0.7875000238418579,
      "rewards/chosen": 0.05469100549817085,
      "rewards/margins": 0.03459729254245758,
      "rewards/margins_max": 0.0798027291893959,
      "rewards/margins_min": -0.0013110924046486616,
      "rewards/margins_std": 0.03574884682893753,
      "rewards/rejected": 0.020093712955713272,
      "step": 360
    },
    {
      "dpo_losses": 0.6761563420295715,
      "epoch": 0.54,
      "grad_norm": 8.370611151073765,
      "learning_rate": 2.582994309902146e-07,
      "logits/chosen": -2.8219902515411377,
      "logits/rejected": -2.7716636657714844,
      "logps/chosen": -304.6541748046875,
      "logps/rejected": -254.8093719482422,
      "loss": 0.6842,
      "positive_losses": 0.04906577989459038,
      "rewards/accuracies": 0.737500011920929,
      "rewards/chosen": 0.054839521646499634,
      "rewards/margins": 0.03491091728210449,
      "rewards/margins_max": 0.09758211672306061,
      "rewards/margins_min": -0.014001704752445221,
      "rewards/margins_std": 0.05021950602531433,
      "rewards/rejected": 0.019928600639104843,
      "step": 370
    },
    {
      "dpo_losses": 0.6737654805183411,
      "epoch": 0.56,
      "grad_norm": 2.3064692447314554,
      "learning_rate": 2.455304924825151e-07,
      "logits/chosen": -2.837787389755249,
      "logits/rejected": -2.831657886505127,
      "logps/chosen": -275.2360534667969,
      "logps/rejected": -255.569580078125,
      "loss": 0.6738,
      "positive_losses": 0.01725444756448269,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": 0.05640103667974472,
      "rewards/margins": 0.0399165078997612,
      "rewards/margins_max": 0.09945692121982574,
      "rewards/margins_min": -0.001225404441356659,
      "rewards/margins_std": 0.045562874525785446,
      "rewards/rejected": 0.016484536230564117,
      "step": 380
    },
    {
      "dpo_losses": 0.6766735911369324,
      "epoch": 0.57,
      "grad_norm": 1.805419113024878,
      "learning_rate": 2.3277321440960732e-07,
      "logits/chosen": -2.838366985321045,
      "logits/rejected": -2.793659210205078,
      "logps/chosen": -266.8939208984375,
      "logps/rejected": -261.4732666015625,
      "loss": 0.6807,
      "positive_losses": 0.028429364785552025,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": 0.0527314729988575,
      "rewards/margins": 0.034005872905254364,
      "rewards/margins_max": 0.10273897647857666,
      "rewards/margins_min": -0.011970164254307747,
      "rewards/margins_std": 0.05079951137304306,
      "rewards/rejected": 0.018725596368312836,
      "step": 390
    },
    {
      "dpo_losses": 0.6763077974319458,
      "epoch": 0.58,
      "grad_norm": 6.733018778387388,
      "learning_rate": 2.2006087905337698e-07,
      "logits/chosen": -2.868297815322876,
      "logits/rejected": -2.8423171043395996,
      "logps/chosen": -238.1276092529297,
      "logps/rejected": -231.619384765625,
      "loss": 0.6808,
      "positive_losses": 0.045419443398714066,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": 0.05248277261853218,
      "rewards/margins": 0.03450237587094307,
      "rewards/margins_max": 0.09071175009012222,
      "rewards/margins_min": -0.006746978498995304,
      "rewards/margins_std": 0.04475879669189453,
      "rewards/rejected": 0.01798040233552456,
      "step": 400
    },
    {
      "epoch": 0.58,
      "eval_dpo_losses": 0.6860782504081726,
      "eval_logits/chosen": -2.7994227409362793,
      "eval_logits/rejected": -2.760826826095581,
      "eval_logps/chosen": -279.1606140136719,
      "eval_logps/rejected": -254.61215209960938,
      "eval_loss": 0.6905239224433899,
      "eval_positive_losses": 0.0314272977411747,
      "eval_rewards/accuracies": 0.6169999837875366,
      "eval_rewards/chosen": 0.05432803928852081,
      "eval_rewards/margins": 0.014660396613180637,
      "eval_rewards/margins_max": 0.08083328604698181,
      "eval_rewards/margins_min": -0.043623533099889755,
      "eval_rewards/margins_std": 0.04150667414069176,
      "eval_rewards/rejected": 0.03966763988137245,
      "eval_runtime": 428.0845,
      "eval_samples_per_second": 4.672,
      "eval_steps_per_second": 0.292,
      "step": 400
    },
    {
      "dpo_losses": 0.6729803085327148,
      "epoch": 0.6,
      "grad_norm": 6.255311463771333,
      "learning_rate": 2.0742665144529372e-07,
      "logits/chosen": -2.852710723876953,
      "logits/rejected": -2.812171459197998,
      "logps/chosen": -307.93133544921875,
      "logps/rejected": -281.7948303222656,
      "loss": 0.6784,
      "positive_losses": 0.0057888031005859375,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": 0.06238215044140816,
      "rewards/margins": 0.04161245375871658,
      "rewards/margins_max": 0.1156817227602005,
      "rewards/margins_min": -0.008250080049037933,
      "rewards/margins_std": 0.05661824345588684,
      "rewards/rejected": 0.020769696682691574,
      "step": 410
    },
    {
      "dpo_losses": 0.6805132627487183,
      "epoch": 0.61,
      "grad_norm": 2.3659410877777494,
      "learning_rate": 1.9490349284263033e-07,
      "logits/chosen": -2.788618564605713,
      "logits/rejected": -2.7586796283721924,
      "logps/chosen": -255.90536499023438,
      "logps/rejected": -242.9296417236328,
      "loss": 0.6804,
      "positive_losses": 0.0345400795340538,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": 0.051929228007793427,
      "rewards/margins": 0.026662463322281837,
      "rewards/margins_max": 0.08727220445871353,
      "rewards/margins_min": -0.02731608971953392,
      "rewards/margins_std": 0.053510747849941254,
      "rewards/rejected": 0.02526676654815674,
      "step": 420
    },
    {
      "dpo_losses": 0.6737715005874634,
      "epoch": 0.63,
      "grad_norm": 2.271597415804245,
      "learning_rate": 1.8252407473630605e-07,
      "logits/chosen": -2.8645436763763428,
      "logits/rejected": -2.8088572025299072,
      "logps/chosen": -295.20916748046875,
      "logps/rejected": -230.15567016601562,
      "loss": 0.6795,
      "positive_losses": 0.03842503949999809,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": 0.058985937386751175,
      "rewards/margins": 0.03972195088863373,
      "rewards/margins_max": 0.0958792120218277,
      "rewards/margins_min": -0.005095441360026598,
      "rewards/margins_std": 0.044632021337747574,
      "rewards/rejected": 0.01926398277282715,
      "step": 430
    },
    {
      "dpo_losses": 0.6767727136611938,
      "epoch": 0.64,
      "grad_norm": 10.115009795187461,
      "learning_rate": 1.7032069361469764e-07,
      "logits/chosen": -2.7652392387390137,
      "logits/rejected": -2.7494540214538574,
      "logps/chosen": -226.9882354736328,
      "logps/rejected": -269.02301025390625,
      "loss": 0.6792,
      "positive_losses": 0.009693240746855736,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": 0.05702495574951172,
      "rewards/margins": 0.03372834250330925,
      "rewards/margins_max": 0.09970168769359589,
      "rewards/margins_min": -0.013438252732157707,
      "rewards/margins_std": 0.051222048699855804,
      "rewards/rejected": 0.02329660952091217,
      "step": 440
    },
    {
      "dpo_losses": 0.6781430244445801,
      "epoch": 0.66,
      "grad_norm": 1.5440271538041583,
      "learning_rate": 1.58325186705788e-07,
      "logits/chosen": -2.8817787170410156,
      "logits/rejected": -2.8590903282165527,
      "logps/chosen": -267.78729248046875,
      "logps/rejected": -251.52542114257812,
      "loss": 0.6868,
      "positive_losses": 0.08534412086009979,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": 0.05013534426689148,
      "rewards/margins": 0.031011879444122314,
      "rewards/margins_max": 0.09007111191749573,
      "rewards/margins_min": -0.02042573131620884,
      "rewards/margins_std": 0.04950300231575966,
      "rewards/rejected": 0.019123462960124016,
      "step": 450
    },
    {
      "dpo_losses": 0.675542950630188,
      "epoch": 0.67,
      "grad_norm": 1.6924810012359601,
      "learning_rate": 1.4656884891747395e-07,
      "logits/chosen": -2.812002182006836,
      "logits/rejected": -2.765144109725952,
      "logps/chosen": -282.1943664550781,
      "logps/rejected": -234.0194854736328,
      "loss": 0.6808,
      "positive_losses": 0.03008117713034153,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": 0.055878765881061554,
      "rewards/margins": 0.03611396998167038,
      "rewards/margins_max": 0.09502451121807098,
      "rewards/margins_min": -0.008020835928618908,
      "rewards/margins_std": 0.04623667150735855,
      "rewards/rejected": 0.019764794036746025,
      "step": 460
    },
    {
      "dpo_losses": 0.6700197458267212,
      "epoch": 0.69,
      "grad_norm": 1.8078995560874291,
      "learning_rate": 1.3508235119272466e-07,
      "logits/chosen": -2.8690173625946045,
      "logits/rejected": -2.823881149291992,
      "logps/chosen": -322.7649230957031,
      "logps/rejected": -289.1722106933594,
      "loss": 0.6782,
      "positive_losses": 0.07821483910083771,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": 0.06522538512945175,
      "rewards/margins": 0.04820011183619499,
      "rewards/margins_max": 0.1358465701341629,
      "rewards/margins_min": -0.011710671707987785,
      "rewards/margins_std": 0.06825564056634903,
      "rewards/rejected": 0.017025265842676163,
      "step": 470
    },
    {
      "dpo_losses": 0.6712939143180847,
      "epoch": 0.7,
      "grad_norm": 1.451487777642733,
      "learning_rate": 1.2389566049259336e-07,
      "logits/chosen": -2.811398983001709,
      "logits/rejected": -2.7891058921813965,
      "logps/chosen": -238.219970703125,
      "logps/rejected": -230.36062622070312,
      "loss": 0.6767,
      "positive_losses": 0.054373931139707565,
      "rewards/accuracies": 0.7875000238418579,
      "rewards/chosen": 0.06273017823696136,
      "rewards/margins": 0.0451803021132946,
      "rewards/margins_max": 0.10969402641057968,
      "rewards/margins_min": -0.0036571747623384,
      "rewards/margins_std": 0.051760654896497726,
      "rewards/rejected": 0.017549879848957062,
      "step": 480
    },
    {
      "dpo_losses": 0.6667095422744751,
      "epoch": 0.72,
      "grad_norm": 2.410585831948769,
      "learning_rate": 1.1303796161583762e-07,
      "logits/chosen": -2.9242846965789795,
      "logits/rejected": -2.840134859085083,
      "logps/chosen": -316.83929443359375,
      "logps/rejected": -256.23675537109375,
      "loss": 0.6785,
      "positive_losses": 0.06970572471618652,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": 0.0768619254231453,
      "rewards/margins": 0.05491393804550171,
      "rewards/margins_max": 0.13214218616485596,
      "rewards/margins_min": -0.004780332557857037,
      "rewards/margins_std": 0.06225153058767319,
      "rewards/rejected": 0.02194797992706299,
      "step": 490
    },
    {
      "dpo_losses": 0.6667904853820801,
      "epoch": 0.73,
      "grad_norm": 1.9094755680052469,
      "learning_rate": 1.0253758105911167e-07,
      "logits/chosen": -2.886962890625,
      "logits/rejected": -2.840977430343628,
      "logps/chosen": -331.50872802734375,
      "logps/rejected": -285.4704895019531,
      "loss": 0.6769,
      "positive_losses": 0.03339347988367081,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.07617944478988647,
      "rewards/margins": 0.0544399730861187,
      "rewards/margins_max": 0.13042126595973969,
      "rewards/margins_min": -0.00018090903176926076,
      "rewards/margins_std": 0.05866100266575813,
      "rewards/rejected": 0.021739467978477478,
      "step": 500
    },
    {
      "epoch": 0.73,
      "eval_dpo_losses": 0.6854846477508545,
      "eval_logits/chosen": -2.797536611557007,
      "eval_logits/rejected": -2.7586684226989746,
      "eval_logps/chosen": -278.739990234375,
      "eval_logps/rejected": -254.3216094970703,
      "eval_loss": 0.6902235150337219,
      "eval_positive_losses": 0.032978080213069916,
      "eval_rewards/accuracies": 0.6119999885559082,
      "eval_rewards/chosen": 0.058533985167741776,
      "eval_rewards/margins": 0.015960821881890297,
      "eval_rewards/margins_max": 0.08850042521953583,
      "eval_rewards/margins_min": -0.048725374042987823,
      "eval_rewards/margins_std": 0.045832596719264984,
      "eval_rewards/rejected": 0.04257315769791603,
      "eval_runtime": 448.6936,
      "eval_samples_per_second": 4.457,
      "eval_steps_per_second": 0.279,
      "step": 500
    },
    {
      "dpo_losses": 0.6742907166481018,
      "epoch": 0.75,
      "grad_norm": 2.2304663983815316,
      "learning_rate": 9.242191311637049e-08,
      "logits/chosen": -2.8333396911621094,
      "logits/rejected": -2.8009631633758545,
      "logps/chosen": -277.5825500488281,
      "logps/rejected": -235.39108276367188,
      "loss": 0.6774,
      "positive_losses": 0.007343196775764227,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": 0.05940967798233032,
      "rewards/margins": 0.038841020315885544,
      "rewards/margins_max": 0.1059766411781311,
      "rewards/margins_min": -0.00883492175489664,
      "rewards/margins_std": 0.052775926887989044,
      "rewards/rejected": 0.02056865207850933,
      "step": 510
    },
    {
      "dpo_losses": 0.6684087514877319,
      "epoch": 0.76,
      "grad_norm": 1.9114464768987025,
      "learning_rate": 8.271734841028552e-08,
      "logits/chosen": -2.8523058891296387,
      "logits/rejected": -2.7778074741363525,
      "logps/chosen": -335.83966064453125,
      "logps/rejected": -264.36016845703125,
      "loss": 0.6743,
      "positive_losses": 0.0079803466796875,
      "rewards/accuracies": 0.887499988079071,
      "rewards/chosen": 0.06848612427711487,
      "rewards/margins": 0.05102965235710144,
      "rewards/margins_max": 0.12273881584405899,
      "rewards/margins_min": -0.0025417390279471874,
      "rewards/margins_std": 0.05730118602514267,
      "rewards/rejected": 0.017456475645303726,
      "step": 520
    },
    {
      "dpo_losses": 0.6773437261581421,
      "epoch": 0.77,
      "grad_norm": 1.8982933837442546,
      "learning_rate": 7.344920504212243e-08,
      "logits/chosen": -2.839207172393799,
      "logits/rejected": -2.7776246070861816,
      "logps/chosen": -260.9156799316406,
      "logps/rejected": -216.26376342773438,
      "loss": 0.6801,
      "positive_losses": 0.04813060909509659,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": 0.054224640130996704,
      "rewards/margins": 0.032528605312108994,
      "rewards/margins_max": 0.09149853885173798,
      "rewards/margins_min": -0.01699809357523918,
      "rewards/margins_std": 0.04981226474046707,
      "rewards/rejected": 0.02169603481888771,
      "step": 530
    },
    {
      "dpo_losses": 0.6705015301704407,
      "epoch": 0.79,
      "grad_norm": 1.723774449740064,
      "learning_rate": 6.46416625397067e-08,
      "logits/chosen": -2.834272861480713,
      "logits/rejected": -2.7763218879699707,
      "logps/chosen": -320.51885986328125,
      "logps/rejected": -295.53875732421875,
      "loss": 0.6751,
      "positive_losses": 0.018300628289580345,
      "rewards/accuracies": 0.8374999761581421,
      "rewards/chosen": 0.06741219758987427,
      "rewards/margins": 0.04666105657815933,
      "rewards/margins_max": 0.10739126056432724,
      "rewards/margins_min": -0.008476030081510544,
      "rewards/margins_std": 0.05081728845834732,
      "rewards/rejected": 0.020751142874360085,
      "step": 540
    },
    {
      "dpo_losses": 0.6752179861068726,
      "epoch": 0.8,
      "grad_norm": 1.6108967521538597,
      "learning_rate": 5.6317698775795344e-08,
      "logits/chosen": -2.8729920387268066,
      "logits/rejected": -2.833665370941162,
      "logps/chosen": -278.38275146484375,
      "logps/rejected": -270.0639953613281,
      "loss": 0.6723,
      "positive_losses": 0.005997705273330212,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.057433564215898514,
      "rewards/margins": 0.037055402994155884,
      "rewards/margins_max": 0.09951488673686981,
      "rewards/margins_min": -0.01219971850514412,
      "rewards/margins_std": 0.051766276359558105,
      "rewards/rejected": 0.02037816308438778,
      "step": 550
    },
    {
      "dpo_losses": 0.6663364171981812,
      "epoch": 0.82,
      "grad_norm": 1.9255604336867507,
      "learning_rate": 4.849903002143113e-08,
      "logits/chosen": -2.9117777347564697,
      "logits/rejected": -2.8610081672668457,
      "logps/chosen": -328.8406066894531,
      "logps/rejected": -272.1076965332031,
      "loss": 0.6731,
      "positive_losses": 0.038059137761592865,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": 0.07473967969417572,
      "rewards/margins": 0.055439990013837814,
      "rewards/margins_max": 0.12707814574241638,
      "rewards/margins_min": -0.002053080592304468,
      "rewards/margins_std": 0.05929755046963692,
      "rewards/rejected": 0.019299697130918503,
      "step": 560
    },
    {
      "dpo_losses": 0.6710665822029114,
      "epoch": 0.83,
      "grad_norm": 1.733507781542446,
      "learning_rate": 4.1206054290670535e-08,
      "logits/chosen": -2.8613619804382324,
      "logits/rejected": -2.82637357711792,
      "logps/chosen": -266.23773193359375,
      "logps/rejected": -258.0359802246094,
      "loss": 0.6756,
      "positive_losses": 0.013837432488799095,
      "rewards/accuracies": 0.7875000238418579,
      "rewards/chosen": 0.06855504214763641,
      "rewards/margins": 0.04551316797733307,
      "rewards/margins_max": 0.1095261201262474,
      "rewards/margins_min": -0.004014958627521992,
      "rewards/margins_std": 0.0510232150554657,
      "rewards/rejected": 0.02304188162088394,
      "step": 570
    },
    {
      "dpo_losses": 0.676629900932312,
      "epoch": 0.85,
      "grad_norm": 8.907703588890858,
      "learning_rate": 3.44577981244944e-08,
      "logits/chosen": -2.8228065967559814,
      "logits/rejected": -2.809900999069214,
      "logps/chosen": -257.9283752441406,
      "logps/rejected": -256.25799560546875,
      "loss": 0.6858,
      "positive_losses": 0.03807668760418892,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.05910138040781021,
      "rewards/margins": 0.034723442047834396,
      "rewards/margins_max": 0.12428979575634003,
      "rewards/margins_min": -0.03289930149912834,
      "rewards/margins_std": 0.07077684998512268,
      "rewards/rejected": 0.024377940222620964,
      "step": 580
    },
    {
      "dpo_losses": 0.6756445169448853,
      "epoch": 0.86,
      "grad_norm": 8.494851741114475,
      "learning_rate": 2.8271866952734814e-08,
      "logits/chosen": -2.8694353103637695,
      "logits/rejected": -2.844571113586426,
      "logps/chosen": -302.86883544921875,
      "logps/rejected": -284.4591369628906,
      "loss": 0.6829,
      "positive_losses": 0.07926483452320099,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": 0.05832204222679138,
      "rewards/margins": 0.03624555841088295,
      "rewards/margins_max": 0.10978803783655167,
      "rewards/margins_min": -0.017319655045866966,
      "rewards/margins_std": 0.05776820331811905,
      "rewards/rejected": 0.022076481953263283,
      "step": 590
    },
    {
      "dpo_losses": 0.6685220003128052,
      "epoch": 0.88,
      "grad_norm": 2.0828943807385567,
      "learning_rate": 2.2664399163518782e-08,
      "logits/chosen": -2.7986605167388916,
      "logits/rejected": -2.781226396560669,
      "logps/chosen": -289.7279968261719,
      "logps/rejected": -246.7861328125,
      "loss": 0.67,
      "positive_losses": 0.0,
      "rewards/accuracies": 0.762499988079071,
      "rewards/chosen": 0.07399223744869232,
      "rewards/margins": 0.051160674542188644,
      "rewards/margins_max": 0.12970498204231262,
      "rewards/margins_min": -0.008301710709929466,
      "rewards/margins_std": 0.06261205673217773,
      "rewards/rejected": 0.02283155545592308,
      "step": 600
    },
    {
      "epoch": 0.88,
      "eval_dpo_losses": 0.6851370930671692,
      "eval_logits/chosen": -2.798839569091797,
      "eval_logits/rejected": -2.7602357864379883,
      "eval_logps/chosen": -278.6050109863281,
      "eval_logps/rejected": -254.26231384277344,
      "eval_loss": 0.6902583241462708,
      "eval_positive_losses": 0.03728185221552849,
      "eval_rewards/accuracies": 0.6159999966621399,
      "eval_rewards/chosen": 0.05988401547074318,
      "eval_rewards/margins": 0.016717953607439995,
      "eval_rewards/margins_max": 0.09297847747802734,
      "eval_rewards/margins_min": -0.051056575030088425,
      "eval_rewards/margins_std": 0.0480518564581871,
      "eval_rewards/rejected": 0.043166060000658035,
      "eval_runtime": 428.2792,
      "eval_samples_per_second": 4.67,
      "eval_steps_per_second": 0.292,
      "step": 600
    },
    {
      "dpo_losses": 0.6745679974555969,
      "epoch": 0.89,
      "grad_norm": 1.3564104662072647,
      "learning_rate": 1.7650024000056414e-08,
      "logits/chosen": -2.7833163738250732,
      "logits/rejected": -2.7807528972625732,
      "logps/chosen": -241.8382568359375,
      "logps/rejected": -243.6163330078125,
      "loss": 0.6742,
      "positive_losses": 0.0,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.06093451380729675,
      "rewards/margins": 0.038374222815036774,
      "rewards/margins_max": 0.10369594395160675,
      "rewards/margins_min": -0.009305914863944054,
      "rewards/margins_std": 0.05068794637918472,
      "rewards/rejected": 0.022560294717550278,
      "step": 610
    },
    {
      "dpo_losses": 0.6677746772766113,
      "epoch": 0.91,
      "grad_norm": 7.979260503019614,
      "learning_rate": 1.3241823394615437e-08,
      "logits/chosen": -2.908095598220825,
      "logits/rejected": -2.8685083389282227,
      "logps/chosen": -304.2915954589844,
      "logps/rejected": -286.26873779296875,
      "loss": 0.675,
      "positive_losses": 0.05257830768823624,
      "rewards/accuracies": 0.8500000238418579,
      "rewards/chosen": 0.07303957641124725,
      "rewards/margins": 0.05244825407862663,
      "rewards/margins_max": 0.11377520859241486,
      "rewards/margins_min": -0.004015705548226833,
      "rewards/margins_std": 0.0548991933465004,
      "rewards/rejected": 0.020591318607330322,
      "step": 620
    },
    {
      "dpo_losses": 0.6700986623764038,
      "epoch": 0.92,
      "grad_norm": 8.477082267330537,
      "learning_rate": 9.451297839253913e-09,
      "logits/chosen": -2.7833220958709717,
      "logits/rejected": -2.7590456008911133,
      "logps/chosen": -276.22540283203125,
      "logps/rejected": -278.2480163574219,
      "loss": 0.6748,
      "positive_losses": 0.00507011404260993,
      "rewards/accuracies": 0.762499988079071,
      "rewards/chosen": 0.06989948451519012,
      "rewards/margins": 0.047721296548843384,
      "rewards/margins_max": 0.11920352280139923,
      "rewards/margins_min": -0.008280943147838116,
      "rewards/margins_std": 0.05656985193490982,
      "rewards/rejected": 0.02217818982899189,
      "step": 630
    },
    {
      "dpo_losses": 0.6701762080192566,
      "epoch": 0.94,
      "grad_norm": 1.4912730505697442,
      "learning_rate": 6.288336382349463e-09,
      "logits/chosen": -2.7986907958984375,
      "logits/rejected": -2.7656140327453613,
      "logps/chosen": -245.1352996826172,
      "logps/rejected": -207.5049285888672,
      "loss": 0.6724,
      "positive_losses": 0.004144692327827215,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": 0.06695154309272766,
      "rewards/margins": 0.04747752100229263,
      "rewards/margins_max": 0.12407982349395752,
      "rewards/margins_min": -0.003674466861411929,
      "rewards/margins_std": 0.05860821157693863,
      "rewards/rejected": 0.01947401836514473,
      "step": 640
    },
    {
      "dpo_losses": 0.6735118627548218,
      "epoch": 0.95,
      "grad_norm": 10.343240316228654,
      "learning_rate": 3.7611908292010665e-09,
      "logits/chosen": -2.859143018722534,
      "logits/rejected": -2.8227005004882812,
      "logps/chosen": -307.39508056640625,
      "logps/rejected": -258.0931091308594,
      "loss": 0.6774,
      "positive_losses": 0.10746382176876068,
      "rewards/accuracies": 0.7124999761581421,
      "rewards/chosen": 0.05885057523846626,
      "rewards/margins": 0.040867820382118225,
      "rewards/margins_max": 0.11666282266378403,
      "rewards/margins_min": -0.015247263014316559,
      "rewards/margins_std": 0.05968106538057327,
      "rewards/rejected": 0.017982754856348038,
      "step": 650
    },
    {
      "dpo_losses": 0.6789201498031616,
      "epoch": 0.96,
      "grad_norm": 12.216425135481341,
      "learning_rate": 1.8764542140112527e-09,
      "logits/chosen": -2.800410509109497,
      "logits/rejected": -2.8051278591156006,
      "logps/chosen": -219.88369750976562,
      "logps/rejected": -224.5011749267578,
      "loss": 0.6838,
      "positive_losses": 0.09240980446338654,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": 0.0470733642578125,
      "rewards/margins": 0.02947625145316124,
      "rewards/margins_max": 0.0936986654996872,
      "rewards/margins_min": -0.014550586231052876,
      "rewards/margins_std": 0.048393599689006805,
      "rewards/rejected": 0.01759711280465126,
      "step": 660
    },
    {
      "dpo_losses": 0.6718948483467102,
      "epoch": 0.98,
      "grad_norm": 7.347929588964824,
      "learning_rate": 6.390435994127752e-10,
      "logits/chosen": -2.8291993141174316,
      "logits/rejected": -2.7946581840515137,
      "logps/chosen": -279.2850646972656,
      "logps/rejected": -295.1185607910156,
      "loss": 0.6767,
      "positive_losses": 0.03623781353235245,
      "rewards/accuracies": 0.762499988079071,
      "rewards/chosen": 0.06117644160985947,
      "rewards/margins": 0.044048137962818146,
      "rewards/margins_max": 0.11197330802679062,
      "rewards/margins_min": -0.004027059301733971,
      "rewards/margins_std": 0.05286269634962082,
      "rewards/rejected": 0.01712830550968647,
      "step": 670
    },
    {
      "dpo_losses": 0.6698868870735168,
      "epoch": 0.99,
      "grad_norm": 9.194005207225429,
      "learning_rate": 5.2187248413465555e-11,
      "logits/chosen": -2.8184335231781006,
      "logits/rejected": -2.7909679412841797,
      "logps/chosen": -296.29473876953125,
      "logps/rejected": -258.4402160644531,
      "loss": 0.6757,
      "positive_losses": 0.03602256625890732,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.06865200400352478,
      "rewards/margins": 0.04830945283174515,
      "rewards/margins_max": 0.1308433711528778,
      "rewards/margins_min": -0.012822682037949562,
      "rewards/margins_std": 0.06532330065965652,
      "rewards/rejected": 0.020342541858553886,
      "step": 680
    },
    {
      "epoch": 1.0,
      "step": 684,
      "total_flos": 0.0,
      "train_loss": 0.6833745032026056,
      "train_runtime": 9125.7542,
      "train_samples_per_second": 1.199,
      "train_steps_per_second": 0.075
    }
  ],
  "logging_steps": 10,
  "max_steps": 684,
  "num_input_tokens_seen": 0,
  "num_train_epochs": 1,
  "save_steps": 100,
  "total_flos": 0.0,
  "train_batch_size": 4,
  "trial_name": null,
  "trial_params": null
}