{
  "best_metric": null,
  "best_model_checkpoint": null,
  "epoch": 0.9963674104826155,
  "eval_steps": 1000,
  "global_step": 240,
  "is_hyper_param_search": false,
  "is_local_process_zero": true,
  "is_world_process_zero": true,
  "log_history": [
    {
      "epoch": 0.0041515308770108976,
      "grad_norm": 6.2112270362028115,
      "learning_rate": 2.083333333333333e-08,
      "logits/chosen": -2.6958627700805664,
      "logits/rejected": -2.650068759918213,
      "logps/chosen": -403.2443542480469,
      "logps/rejected": -397.0637512207031,
      "loss": 0.6931,
      "rewards/accuracies": 0.0,
      "rewards/chosen": 0.0,
      "rewards/margins": 0.0,
      "rewards/rejected": 0.0,
      "step": 1
    },
    {
      "epoch": 0.04151530877010898,
      "grad_norm": 6.539288745411817,
      "learning_rate": 2.0833333333333333e-07,
      "logits/chosen": -2.690112352371216,
      "logits/rejected": -2.6779024600982666,
      "logps/chosen": -365.9743957519531,
      "logps/rejected": -393.799560546875,
      "loss": 0.6925,
      "rewards/accuracies": 0.5303819179534912,
      "rewards/chosen": 0.0005260682082735002,
      "rewards/margins": 0.0014649044023826718,
      "rewards/rejected": -0.0009388362523168325,
      "step": 10
    },
    {
      "epoch": 0.08303061754021795,
      "grad_norm": 6.876570939434779,
      "learning_rate": 4.1666666666666667e-07,
      "logits/chosen": -2.6719064712524414,
      "logits/rejected": -2.6511917114257812,
      "logps/chosen": -357.5186462402344,
      "logps/rejected": -380.5492858886719,
      "loss": 0.6749,
      "rewards/accuracies": 0.848437488079071,
      "rewards/chosen": 0.018601149320602417,
      "rewards/margins": 0.03796255216002464,
      "rewards/rejected": -0.019361400976777077,
      "step": 20
    },
    {
      "epoch": 0.12454592631032693,
      "grad_norm": 8.242981852125217,
      "learning_rate": 4.990486745229364e-07,
      "logits/chosen": -2.653783082962036,
      "logits/rejected": -2.639296293258667,
      "logps/chosen": -355.69207763671875,
      "logps/rejected": -415.68768310546875,
      "loss": 0.5772,
      "rewards/accuracies": 0.867968738079071,
      "rewards/chosen": 0.0034460537135601044,
      "rewards/margins": 0.2814808487892151,
      "rewards/rejected": -0.2780347764492035,
      "step": 30
    },
    {
      "epoch": 0.1660612350804359,
      "grad_norm": 14.409074768095572,
      "learning_rate": 4.932612176449559e-07,
      "logits/chosen": -2.6512644290924072,
      "logits/rejected": -2.613352060317993,
      "logps/chosen": -496.20770263671875,
      "logps/rejected": -619.66943359375,
      "loss": 0.4041,
      "rewards/accuracies": 0.848437488079071,
      "rewards/chosen": -1.1875219345092773,
      "rewards/margins": 1.0682841539382935,
      "rewards/rejected": -2.2558062076568604,
      "step": 40
    },
    {
      "epoch": 0.2075765438505449,
      "grad_norm": 14.911382681454745,
      "learning_rate": 4.823368810567056e-07,
      "logits/chosen": -2.64001202583313,
      "logits/rejected": -2.6261935234069824,
      "logps/chosen": -600.037109375,
      "logps/rejected": -856.2901611328125,
      "loss": 0.3072,
      "rewards/accuracies": 0.86328125,
      "rewards/chosen": -2.279493570327759,
      "rewards/margins": 2.3056600093841553,
      "rewards/rejected": -4.5851545333862305,
      "step": 50
    },
    {
      "epoch": 0.24909185262065386,
      "grad_norm": 18.472917689948186,
      "learning_rate": 4.6650635094610966e-07,
      "logits/chosen": -2.5924713611602783,
      "logits/rejected": -2.579529285430908,
      "logps/chosen": -587.2591552734375,
      "logps/rejected": -904.9329833984375,
      "loss": 0.2658,
      "rewards/accuracies": 0.88671875,
      "rewards/chosen": -2.1298751831054688,
      "rewards/margins": 2.945924758911133,
      "rewards/rejected": -5.07580041885376,
      "step": 60
    },
    {
      "epoch": 0.29060716139076287,
      "grad_norm": 12.678464521005063,
      "learning_rate": 4.461039162298939e-07,
      "logits/chosen": -2.2207727432250977,
      "logits/rejected": -1.98198664188385,
      "logps/chosen": -591.1898803710938,
      "logps/rejected": -989.9195556640625,
      "loss": 0.2276,
      "rewards/accuracies": 0.9046875238418579,
      "rewards/chosen": -2.155735969543457,
      "rewards/margins": 3.7345290184020996,
      "rewards/rejected": -5.890264987945557,
      "step": 70
    },
    {
      "epoch": 0.3321224701608718,
      "grad_norm": 14.772965954120863,
      "learning_rate": 4.2156040946718343e-07,
      "logits/chosen": -1.5045950412750244,
      "logits/rejected": -0.8392450213432312,
      "logps/chosen": -616.548828125,
      "logps/rejected": -1064.140869140625,
      "loss": 0.1988,
      "rewards/accuracies": 0.914843738079071,
      "rewards/chosen": -2.4090540409088135,
      "rewards/margins": 4.233697891235352,
      "rewards/rejected": -6.642751216888428,
      "step": 80
    },
    {
      "epoch": 0.3736377789309808,
      "grad_norm": 16.72422342704544,
      "learning_rate": 3.933941090877615e-07,
      "logits/chosen": -1.3608930110931396,
      "logits/rejected": -0.5576863288879395,
      "logps/chosen": -595.0042724609375,
      "logps/rejected": -1064.654052734375,
      "loss": 0.2026,
      "rewards/accuracies": 0.91015625,
      "rewards/chosen": -2.351076126098633,
      "rewards/margins": 4.467952728271484,
      "rewards/rejected": -6.819028377532959,
      "step": 90
    },
    {
      "epoch": 0.4151530877010898,
      "grad_norm": 9.743291843330928,
      "learning_rate": 3.6219979505011555e-07,
      "logits/chosen": -0.8342965245246887,
      "logits/rejected": -0.012925502844154835,
      "logps/chosen": -646.4923706054688,
      "logps/rejected": -1148.498779296875,
      "loss": 0.1867,
      "rewards/accuracies": 0.9195312261581421,
      "rewards/chosen": -2.8086471557617188,
      "rewards/margins": 4.717282295227051,
      "rewards/rejected": -7.525929927825928,
      "step": 100
    },
    {
      "epoch": 0.4566683964711988,
      "grad_norm": 11.81658995353539,
      "learning_rate": 3.286361890379034e-07,
      "logits/chosen": -0.1246568113565445,
      "logits/rejected": 0.6437393426895142,
      "logps/chosen": -648.837890625,
      "logps/rejected": -1166.3304443359375,
      "loss": 0.1785,
      "rewards/accuracies": 0.917187511920929,
      "rewards/chosen": -2.7460227012634277,
      "rewards/margins": 4.996233940124512,
      "rewards/rejected": -7.742256164550781,
      "step": 110
    },
    {
      "epoch": 0.49818370524130773,
      "grad_norm": 9.286297855632633,
      "learning_rate": 2.934120444167326e-07,
      "logits/chosen": -0.19480545818805695,
      "logits/rejected": 0.7439680695533752,
      "logps/chosen": -654.71875,
      "logps/rejected": -1212.785888671875,
      "loss": 0.1786,
      "rewards/accuracies": 0.91796875,
      "rewards/chosen": -2.892089366912842,
      "rewards/margins": 5.356635093688965,
      "rewards/rejected": -8.248723983764648,
      "step": 120
    },
    {
      "epoch": 0.5396990140114167,
      "grad_norm": 12.2921471616094,
      "learning_rate": 2.5727117968577785e-07,
      "logits/chosen": -0.4111465513706207,
      "logits/rejected": 0.8093876838684082,
      "logps/chosen": -645.12109375,
      "logps/rejected": -1192.375244140625,
      "loss": 0.1676,
      "rewards/accuracies": 0.9234374761581421,
      "rewards/chosen": -2.742527961730957,
      "rewards/margins": 5.263998031616211,
      "rewards/rejected": -8.006526947021484,
      "step": 130
    },
    {
      "epoch": 0.5812143227815257,
      "grad_norm": 17.430656596120464,
      "learning_rate": 2.209767714686924e-07,
      "logits/chosen": -0.029587041586637497,
      "logits/rejected": 1.3743274211883545,
      "logps/chosen": -648.7828979492188,
      "logps/rejected": -1265.0289306640625,
      "loss": 0.1632,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": -2.7970809936523438,
      "rewards/margins": 5.923464775085449,
      "rewards/rejected": -8.720545768737793,
      "step": 140
    },
    {
      "epoch": 0.6227296315516346,
      "grad_norm": 9.425394796778923,
      "learning_rate": 1.8529523872436977e-07,
      "logits/chosen": -0.18061885237693787,
      "logits/rejected": 1.3279445171356201,
      "logps/chosen": -643.9744873046875,
      "logps/rejected": -1251.9317626953125,
      "loss": 0.1642,
      "rewards/accuracies": 0.9273437261581421,
      "rewards/chosen": -2.7806482315063477,
      "rewards/margins": 5.8400492668151855,
      "rewards/rejected": -8.620697975158691,
      "step": 150
    },
    {
      "epoch": 0.6642449403217436,
      "grad_norm": 8.084677706157798,
      "learning_rate": 1.5098005849021078e-07,
      "logits/chosen": -0.20541512966156006,
      "logits/rejected": 1.3262333869934082,
      "logps/chosen": -645.2593994140625,
      "logps/rejected": -1234.405517578125,
      "loss": 0.1618,
      "rewards/accuracies": 0.907031238079071,
      "rewards/chosen": -2.804506301879883,
      "rewards/margins": 5.688388347625732,
      "rewards/rejected": -8.492895126342773,
      "step": 160
    },
    {
      "epoch": 0.7057602490918526,
      "grad_norm": 10.460249704683227,
      "learning_rate": 1.1875585491635998e-07,
      "logits/chosen": -0.6104969382286072,
      "logits/rejected": 1.1572777032852173,
      "logps/chosen": -623.3721923828125,
      "logps/rejected": -1259.294189453125,
      "loss": 0.1573,
      "rewards/accuracies": 0.9203125238418579,
      "rewards/chosen": -2.5536274909973145,
      "rewards/margins": 6.065129280090332,
      "rewards/rejected": -8.618757247924805,
      "step": 170
    },
    {
      "epoch": 0.7472755578619616,
      "grad_norm": 12.223705856186251,
      "learning_rate": 8.930309757836516e-08,
      "logits/chosen": -0.717302680015564,
      "logits/rejected": 1.1799119710922241,
      "logps/chosen": -645.1275634765625,
      "logps/rejected": -1283.7489013671875,
      "loss": 0.1616,
      "rewards/accuracies": 0.934374988079071,
      "rewards/chosen": -2.7811636924743652,
      "rewards/margins": 6.2163591384887695,
      "rewards/rejected": -8.997522354125977,
      "step": 180
    },
    {
      "epoch": 0.7887908666320705,
      "grad_norm": 9.676735610238229,
      "learning_rate": 6.324373218975104e-08,
      "logits/chosen": -0.8794542551040649,
      "logits/rejected": 0.99409419298172,
      "logps/chosen": -619.7286987304688,
      "logps/rejected": -1276.9617919921875,
      "loss": 0.1608,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": -2.615447521209717,
      "rewards/margins": 6.301668643951416,
      "rewards/rejected": -8.917116165161133,
      "step": 190
    },
    {
      "epoch": 0.8303061754021795,
      "grad_norm": 11.57113935464218,
      "learning_rate": 4.112804714676593e-08,
      "logits/chosen": -0.7840622663497925,
      "logits/rejected": 1.0056122541427612,
      "logps/chosen": -637.8081665039062,
      "logps/rejected": -1261.2838134765625,
      "loss": 0.162,
      "rewards/accuracies": 0.9281250238418579,
      "rewards/chosen": -2.6473495960235596,
      "rewards/margins": 6.003285884857178,
      "rewards/rejected": -8.650635719299316,
      "step": 200
    },
    {
      "epoch": 0.8718214841722886,
      "grad_norm": 13.665084975047886,
      "learning_rate": 2.3423053240837514e-08,
      "logits/chosen": -0.6102296710014343,
      "logits/rejected": 1.163267731666565,
      "logps/chosen": -638.1444091796875,
      "logps/rejected": -1243.947265625,
      "loss": 0.1576,
      "rewards/accuracies": 0.918749988079071,
      "rewards/chosen": -2.7065281867980957,
      "rewards/margins": 5.835700988769531,
      "rewards/rejected": -8.542229652404785,
      "step": 210
    },
    {
      "epoch": 0.9133367929423976,
      "grad_norm": 10.802886491160189,
      "learning_rate": 1.0502621921127774e-08,
      "logits/chosen": -0.5309673547744751,
      "logits/rejected": 1.326080560684204,
      "logps/chosen": -637.7506713867188,
      "logps/rejected": -1279.3565673828125,
      "loss": 0.1579,
      "rewards/accuracies": 0.922656238079071,
      "rewards/chosen": -2.6862006187438965,
      "rewards/margins": 6.1917243003845215,
      "rewards/rejected": -8.877924919128418,
      "step": 220
    },
    {
      "epoch": 0.9548521017125065,
      "grad_norm": 10.752955125049585,
      "learning_rate": 2.639590354763882e-09,
      "logits/chosen": -0.5724472403526306,
      "logits/rejected": 1.2889587879180908,
      "logps/chosen": -629.98779296875,
      "logps/rejected": -1255.991943359375,
      "loss": 0.1579,
      "rewards/accuracies": 0.94140625,
      "rewards/chosen": -2.661499500274658,
      "rewards/margins": 6.085452556610107,
      "rewards/rejected": -8.746953010559082,
      "step": 230
    },
    {
      "epoch": 0.9963674104826155,
      "grad_norm": 9.420585318893654,
      "learning_rate": 0.0,
      "logits/chosen": -0.5639629364013672,
      "logits/rejected": 1.3717336654663086,
      "logps/chosen": -631.6014404296875,
      "logps/rejected": -1273.350341796875,
      "loss": 0.1514,
      "rewards/accuracies": 0.934374988079071,
      "rewards/chosen": -2.7341790199279785,
      "rewards/margins": 6.176712989807129,
      "rewards/rejected": -8.91089153289795,
      "step": 240
    },
    {
      "epoch": 0.9963674104826155,
      "step": 240,
      "total_flos": 0.0,
      "train_loss": 0.250737202167511,
      "train_runtime": 29028.2374,
      "train_samples_per_second": 4.248,
      "train_steps_per_second": 0.008
    }
  ],
  "logging_steps": 10,
  "max_steps": 240,
  "num_input_tokens_seen": 0,
  "num_train_epochs": 1,
  "save_steps": 100,
  "stateful_callbacks": {
    "TrainerControl": {
      "args": {
        "should_epoch_stop": false,
        "should_evaluate": false,
        "should_log": false,
        "should_save": true,
        "should_training_stop": true
      },
      "attributes": {}
    }
  },
  "total_flos": 0.0,
  "train_batch_size": 16,
  "trial_name": null,
  "trial_params": null
}