{
  "best_metric": 0.8734214901924133,
  "best_model_checkpoint": "saves/Mistral-7B-Instruct-v0.3/lora/orpo/checkpoint-1500",
  "epoch": 2.997999555456768,
  "eval_steps": 500,
  "global_step": 1686,
  "is_hyper_param_search": false,
  "is_local_process_zero": true,
  "is_world_process_zero": true,
  "log_history": [
    {
      "epoch": 0.017781729273171815,
      "grad_norm": 2.492755651473999,
      "learning_rate": 4.9995745934141085e-06,
      "logits/chosen": -2.952331781387329,
      "logits/rejected": -2.973951816558838,
      "logps/chosen": -1.0092018842697144,
      "logps/rejected": -1.3774441480636597,
      "loss": 1.0773,
      "odds_ratio_loss": 0.6805658936500549,
      "rewards/accuracies": 0.518750011920929,
      "rewards/chosen": -0.1009201854467392,
      "rewards/margins": 0.03682423382997513,
      "rewards/rejected": -0.13774441182613373,
      "sft_loss": 1.0092018842697144,
      "step": 10
    },
    {
      "epoch": 0.03556345854634363,
      "grad_norm": 8.398221969604492,
      "learning_rate": 4.9982812903243405e-06,
      "logits/chosen": -2.924294948577881,
      "logits/rejected": -2.994157314300537,
      "logps/chosen": -1.0329482555389404,
      "logps/rejected": -1.2759336233139038,
      "loss": 1.1014,
      "odds_ratio_loss": 0.6848658323287964,
      "rewards/accuracies": 0.5687500238418579,
      "rewards/chosen": -0.10329482704401016,
      "rewards/margins": 0.024298548698425293,
      "rewards/rejected": -0.12759338319301605,
      "sft_loss": 1.0329482555389404,
      "step": 20
    },
    {
      "epoch": 0.05334518781951545,
      "grad_norm": 2.1793289184570312,
      "learning_rate": 4.996120496405222e-06,
      "logits/chosen": -2.9549760818481445,
      "logits/rejected": -2.9626007080078125,
      "logps/chosen": -1.0005769729614258,
      "logps/rejected": -1.488245964050293,
      "loss": 1.0616,
      "odds_ratio_loss": 0.6106585264205933,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -0.1000577062368393,
      "rewards/margins": 0.048766884952783585,
      "rewards/rejected": -0.14882458746433258,
      "sft_loss": 1.0005769729614258,
      "step": 30
    },
    {
      "epoch": 0.07112691709268726,
      "grad_norm": 2.8343796730041504,
      "learning_rate": 4.99309296196014e-06,
      "logits/chosen": -2.924588203430176,
      "logits/rejected": -2.9867076873779297,
      "logps/chosen": -1.0675694942474365,
      "logps/rejected": -1.2621403932571411,
      "loss": 1.1331,
      "odds_ratio_loss": 0.6557044982910156,
      "rewards/accuracies": 0.5375000238418579,
      "rewards/chosen": -0.10675694793462753,
      "rewards/margins": 0.019457101821899414,
      "rewards/rejected": -0.12621404230594635,
      "sft_loss": 1.0675694942474365,
      "step": 40
    },
    {
      "epoch": 0.08890864636585907,
      "grad_norm": 2.04829478263855,
      "learning_rate": 4.989199738255166e-06,
      "logits/chosen": -2.956892967224121,
      "logits/rejected": -2.9954347610473633,
      "logps/chosen": -0.9171065092086792,
      "logps/rejected": -1.2301478385925293,
      "loss": 0.9797,
      "odds_ratio_loss": 0.6256455183029175,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -0.0917106494307518,
      "rewards/margins": 0.03130412846803665,
      "rewards/rejected": -0.12301478534936905,
      "sft_loss": 0.9171065092086792,
      "step": 50
    },
    {
      "epoch": 0.1066903756390309,
      "grad_norm": 5.50786018371582,
      "learning_rate": 4.984442177154031e-06,
      "logits/chosen": -2.9277195930480957,
      "logits/rejected": -2.9476375579833984,
      "logps/chosen": -0.9940068125724792,
      "logps/rejected": -1.2362287044525146,
      "loss": 1.0632,
      "odds_ratio_loss": 0.6921108365058899,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": -0.09940069168806076,
      "rewards/margins": 0.02422218956053257,
      "rewards/rejected": -0.12362287193536758,
      "sft_loss": 0.9940068125724792,
      "step": 60
    },
    {
      "epoch": 0.12447210491220272,
      "grad_norm": 1.7581864595413208,
      "learning_rate": 4.978821930648704e-06,
      "logits/chosen": -2.9205572605133057,
      "logits/rejected": -2.973936080932617,
      "logps/chosen": -0.9317066073417664,
      "logps/rejected": -1.0809520483016968,
      "loss": 1.0027,
      "odds_ratio_loss": 0.7100493907928467,
      "rewards/accuracies": 0.5562499761581421,
      "rewards/chosen": -0.093170665204525,
      "rewards/margins": 0.014924542978405952,
      "rewards/rejected": -0.1080952063202858,
      "sft_loss": 0.9317066073417664,
      "step": 70
    },
    {
      "epoch": 0.14225383418537452,
      "grad_norm": 3.336517572402954,
      "learning_rate": 4.97234095028576e-06,
      "logits/chosen": -2.9701972007751465,
      "logits/rejected": -2.971057415008545,
      "logps/chosen": -0.9333993792533875,
      "logps/rejected": -1.1716864109039307,
      "loss": 0.9964,
      "odds_ratio_loss": 0.6299672722816467,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": -0.0933399349451065,
      "rewards/margins": 0.023828700184822083,
      "rewards/rejected": -0.11716864258050919,
      "sft_loss": 0.9333993792533875,
      "step": 80
    },
    {
      "epoch": 0.16003556345854633,
      "grad_norm": 1.333382248878479,
      "learning_rate": 4.965001486488743e-06,
      "logits/chosen": -2.9220926761627197,
      "logits/rejected": -2.951408863067627,
      "logps/chosen": -0.8873022198677063,
      "logps/rejected": -1.1284812688827515,
      "loss": 0.9472,
      "odds_ratio_loss": 0.5987495183944702,
      "rewards/accuracies": 0.606249988079071,
      "rewards/chosen": -0.08873023092746735,
      "rewards/margins": 0.02411791868507862,
      "rewards/rejected": -0.11284814029932022,
      "sft_loss": 0.8873022198677063,
      "step": 90
    },
    {
      "epoch": 0.17781729273171815,
      "grad_norm": 1.715163230895996,
      "learning_rate": 4.956806087776732e-06,
      "logits/chosen": -3.0303444862365723,
      "logits/rejected": -3.04186749458313,
      "logps/chosen": -0.9242479205131531,
      "logps/rejected": -1.3088445663452148,
      "loss": 0.9857,
      "odds_ratio_loss": 0.6147152185440063,
      "rewards/accuracies": 0.5874999761581421,
      "rewards/chosen": -0.09242479503154755,
      "rewards/margins": 0.038459669798612595,
      "rewards/rejected": -0.13088446855545044,
      "sft_loss": 0.9242479205131531,
      "step": 100
    },
    {
      "epoch": 0.19559902200489,
      "grad_norm": 2.949481248855591,
      "learning_rate": 4.947757599879411e-06,
      "logits/chosen": -3.0064456462860107,
      "logits/rejected": -3.0399320125579834,
      "logps/chosen": -0.9601238965988159,
      "logps/rejected": -1.2331488132476807,
      "loss": 1.0259,
      "odds_ratio_loss": 0.6574784517288208,
      "rewards/accuracies": 0.581250011920929,
      "rewards/chosen": -0.09601239114999771,
      "rewards/margins": 0.027302492409944534,
      "rewards/rejected": -0.12331489473581314,
      "sft_loss": 0.9601238965988159,
      "step": 110
    },
    {
      "epoch": 0.2133807512780618,
      "grad_norm": 1.2405259609222412,
      "learning_rate": 4.937859164748931e-06,
      "logits/chosen": -3.0256314277648926,
      "logits/rejected": -3.044879913330078,
      "logps/chosen": -0.8803631067276001,
      "logps/rejected": -1.0130887031555176,
      "loss": 0.9473,
      "odds_ratio_loss": 0.668988823890686,
      "rewards/accuracies": 0.5687500238418579,
      "rewards/chosen": -0.08803631365299225,
      "rewards/margins": 0.013272559270262718,
      "rewards/rejected": -0.10130886733531952,
      "sft_loss": 0.8803631067276001,
      "step": 120
    },
    {
      "epoch": 0.23116248055123362,
      "grad_norm": 2.040465831756592,
      "learning_rate": 4.92711421946891e-06,
      "logits/chosen": -3.0067856311798096,
      "logits/rejected": -2.970612049102783,
      "logps/chosen": -0.8932172060012817,
      "logps/rejected": -1.1789153814315796,
      "loss": 0.9558,
      "odds_ratio_loss": 0.6254903674125671,
      "rewards/accuracies": 0.5562499761581421,
      "rewards/chosen": -0.08932172507047653,
      "rewards/margins": 0.028569817543029785,
      "rewards/rejected": -0.11789155006408691,
      "sft_loss": 0.8932172060012817,
      "step": 130
    },
    {
      "epoch": 0.24894420982440543,
      "grad_norm": 1.586767554283142,
      "learning_rate": 4.915526495060961e-06,
      "logits/chosen": -3.0685572624206543,
      "logits/rejected": -3.0535078048706055,
      "logps/chosen": -0.8625435829162598,
      "logps/rejected": -1.1399943828582764,
      "loss": 0.9238,
      "odds_ratio_loss": 0.612372636795044,
      "rewards/accuracies": 0.5687500238418579,
      "rewards/chosen": -0.08625435084104538,
      "rewards/margins": 0.027745097875595093,
      "rewards/rejected": -0.11399944871664047,
      "sft_loss": 0.8625435829162598,
      "step": 140
    },
    {
      "epoch": 0.26672593909757725,
      "grad_norm": 1.953273057937622,
      "learning_rate": 4.903100015189153e-06,
      "logits/chosen": -3.0217204093933105,
      "logits/rejected": -3.059971570968628,
      "logps/chosen": -0.8424757719039917,
      "logps/rejected": -1.0430591106414795,
      "loss": 0.9062,
      "odds_ratio_loss": 0.6374109983444214,
      "rewards/accuracies": 0.606249988079071,
      "rewards/chosen": -0.08424757421016693,
      "rewards/margins": 0.020058346912264824,
      "rewards/rejected": -0.1043059229850769,
      "sft_loss": 0.8424757719039917,
      "step": 150
    },
    {
      "epoch": 0.28450766837074903,
      "grad_norm": 4.5785298347473145,
      "learning_rate": 4.889839094762848e-06,
      "logits/chosen": -3.001889705657959,
      "logits/rejected": -3.0023865699768066,
      "logps/chosen": -0.887285053730011,
      "logps/rejected": -1.1001445055007935,
      "loss": 0.952,
      "odds_ratio_loss": 0.6474493741989136,
      "rewards/accuracies": 0.5687500238418579,
      "rewards/chosen": -0.08872850239276886,
      "rewards/margins": 0.02128593623638153,
      "rewards/rejected": -0.11001445353031158,
      "sft_loss": 0.887285053730011,
      "step": 160
    },
    {
      "epoch": 0.3022893976439209,
      "grad_norm": 1.2961128950119019,
      "learning_rate": 4.875748338438416e-06,
      "logits/chosen": -3.055670738220215,
      "logits/rejected": -3.0634965896606445,
      "logps/chosen": -0.8919625282287598,
      "logps/rejected": -1.0326893329620361,
      "loss": 0.9602,
      "odds_ratio_loss": 0.6827921271324158,
      "rewards/accuracies": 0.543749988079071,
      "rewards/chosen": -0.08919624984264374,
      "rewards/margins": 0.014072686433792114,
      "rewards/rejected": -0.10326894372701645,
      "sft_loss": 0.8919625282287598,
      "step": 170
    },
    {
      "epoch": 0.32007112691709266,
      "grad_norm": 2.0726120471954346,
      "learning_rate": 4.8608326390203386e-06,
      "logits/chosen": -3.05631685256958,
      "logits/rejected": -3.0371289253234863,
      "logps/chosen": -0.8544119000434875,
      "logps/rejected": -1.0456076860427856,
      "loss": 0.9188,
      "odds_ratio_loss": 0.6443353891372681,
      "rewards/accuracies": 0.5687500238418579,
      "rewards/chosen": -0.08544120192527771,
      "rewards/margins": 0.01911957561969757,
      "rewards/rejected": -0.10456077009439468,
      "sft_loss": 0.8544119000434875,
      "step": 180
    },
    {
      "epoch": 0.3378528561902645,
      "grad_norm": 2.612196922302246,
      "learning_rate": 4.845097175762251e-06,
      "logits/chosen": -3.0822339057922363,
      "logits/rejected": -3.1029491424560547,
      "logps/chosen": -0.907193660736084,
      "logps/rejected": -1.0687024593353271,
      "loss": 0.9743,
      "odds_ratio_loss": 0.6714697480201721,
      "rewards/accuracies": 0.543749988079071,
      "rewards/chosen": -0.09071935713291168,
      "rewards/margins": 0.01615087315440178,
      "rewards/rejected": -0.10687023401260376,
      "sft_loss": 0.907193660736084,
      "step": 190
    },
    {
      "epoch": 0.3556345854634363,
      "grad_norm": 3.9038424491882324,
      "learning_rate": 4.8285474125685286e-06,
      "logits/chosen": -3.066904067993164,
      "logits/rejected": -3.086334705352783,
      "logps/chosen": -0.9056366086006165,
      "logps/rejected": -1.0252189636230469,
      "loss": 0.9757,
      "odds_ratio_loss": 0.7004884481430054,
      "rewards/accuracies": 0.53125,
      "rewards/chosen": -0.09056366980075836,
      "rewards/margins": 0.01195824146270752,
      "rewards/rejected": -0.10252189636230469,
      "sft_loss": 0.9056366086006165,
      "step": 200
    },
    {
      "epoch": 0.37341631473660813,
      "grad_norm": 1.57925283908844,
      "learning_rate": 4.811189096097025e-06,
      "logits/chosen": -3.044316530227661,
      "logits/rejected": -3.068372964859009,
      "logps/chosen": -0.882292628288269,
      "logps/rejected": -1.1092549562454224,
      "loss": 0.9473,
      "odds_ratio_loss": 0.6500683426856995,
      "rewards/accuracies": 0.5562499761581421,
      "rewards/chosen": -0.08822925388813019,
      "rewards/margins": 0.0226962361484766,
      "rewards/rejected": -0.11092549562454224,
      "sft_loss": 0.882292628288269,
      "step": 210
    },
    {
      "epoch": 0.39119804400978,
      "grad_norm": 3.1554384231567383,
      "learning_rate": 4.793028253763633e-06,
      "logits/chosen": -3.1082234382629395,
      "logits/rejected": -3.1198127269744873,
      "logps/chosen": -0.878674328327179,
      "logps/rejected": -1.0521525144577026,
      "loss": 0.9507,
      "odds_ratio_loss": 0.7204707860946655,
      "rewards/accuracies": 0.581250011920929,
      "rewards/chosen": -0.08786743879318237,
      "rewards/margins": 0.01734781637787819,
      "rewards/rejected": -0.10521525144577026,
      "sft_loss": 0.878674328327179,
      "step": 220
    },
    {
      "epoch": 0.40897977328295176,
      "grad_norm": 3.0515213012695312,
      "learning_rate": 4.774071191649352e-06,
      "logits/chosen": -3.0294933319091797,
      "logits/rejected": -3.036970615386963,
      "logps/chosen": -0.8506752252578735,
      "logps/rejected": -1.1133465766906738,
      "loss": 0.9107,
      "odds_ratio_loss": 0.6004607677459717,
      "rewards/accuracies": 0.581250011920929,
      "rewards/chosen": -0.08506752550601959,
      "rewards/margins": 0.026267144829034805,
      "rewards/rejected": -0.1113346666097641,
      "sft_loss": 0.8506752252578735,
      "step": 230
    },
    {
      "epoch": 0.4267615025561236,
      "grad_norm": 4.107941627502441,
      "learning_rate": 4.7543244923105975e-06,
      "logits/chosen": -3.052797794342041,
      "logits/rejected": -3.0861849784851074,
      "logps/chosen": -0.9088889956474304,
      "logps/rejected": -0.9762862324714661,
      "loss": 0.9829,
      "odds_ratio_loss": 0.7397087812423706,
      "rewards/accuracies": 0.518750011920929,
      "rewards/chosen": -0.09088890254497528,
      "rewards/margins": 0.006739714182913303,
      "rewards/rejected": -0.09762861579656601,
      "sft_loss": 0.9088889956474304,
      "step": 240
    },
    {
      "epoch": 0.4445432318292954,
      "grad_norm": 1.7963005304336548,
      "learning_rate": 4.733795012493506e-06,
      "logits/chosen": -3.077770471572876,
      "logits/rejected": -3.1305344104766846,
      "logps/chosen": -0.9044251441955566,
      "logps/rejected": -1.036949872970581,
      "loss": 0.9739,
      "odds_ratio_loss": 0.6946715712547302,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": -0.09044251590967178,
      "rewards/margins": 0.013252484612166882,
      "rewards/rejected": -0.10369499772787094,
      "sft_loss": 0.9044251441955566,
      "step": 250
    },
    {
      "epoch": 0.46232496110246724,
      "grad_norm": 1.4697704315185547,
      "learning_rate": 4.712489880753035e-06,
      "logits/chosen": -3.078249931335449,
      "logits/rejected": -3.072510004043579,
      "logps/chosen": -0.81315678358078,
      "logps/rejected": -0.9732586741447449,
      "loss": 0.8776,
      "odds_ratio_loss": 0.644811749458313,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -0.08131568133831024,
      "rewards/margins": 0.016010191291570663,
      "rewards/rejected": -0.0973258763551712,
      "sft_loss": 0.81315678358078,
      "step": 260
    },
    {
      "epoch": 0.480106690375639,
      "grad_norm": 5.489832878112793,
      "learning_rate": 4.690416494977673e-06,
      "logits/chosen": -3.067095994949341,
      "logits/rejected": -3.109727382659912,
      "logps/chosen": -0.8310638666152954,
      "logps/rejected": -1.1116752624511719,
      "loss": 0.8925,
      "odds_ratio_loss": 0.6144498586654663,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -0.0831063836812973,
      "rewards/margins": 0.028061147779226303,
      "rewards/rejected": -0.1111675351858139,
      "sft_loss": 0.8310638666152954,
      "step": 270
    },
    {
      "epoch": 0.49788841964881086,
      "grad_norm": 1.4339563846588135,
      "learning_rate": 4.667582519820639e-06,
      "logits/chosen": -3.068760395050049,
      "logits/rejected": -3.1055545806884766,
      "logps/chosen": -0.9461262822151184,
      "logps/rejected": -1.0382800102233887,
      "loss": 1.018,
      "odds_ratio_loss": 0.7186475992202759,
      "rewards/accuracies": 0.5687500238418579,
      "rewards/chosen": -0.09461262822151184,
      "rewards/margins": 0.009215375408530235,
      "rewards/rejected": -0.10382799804210663,
      "sft_loss": 0.9461262822151184,
      "step": 280
    },
    {
      "epoch": 0.5156701489219827,
      "grad_norm": 3.6930854320526123,
      "learning_rate": 4.643995884038443e-06,
      "logits/chosen": -3.0967042446136475,
      "logits/rejected": -3.1315600872039795,
      "logps/chosen": -0.8749726414680481,
      "logps/rejected": -1.058611273765564,
      "loss": 0.9404,
      "odds_ratio_loss": 0.6541867256164551,
      "rewards/accuracies": 0.5562499761581421,
      "rewards/chosen": -0.08749726414680481,
      "rewards/margins": 0.018363865092396736,
      "rewards/rejected": -0.1058611273765564,
      "sft_loss": 0.8749726414680481,
      "step": 290
    },
    {
      "epoch": 0.5334518781951545,
      "grad_norm": 1.7125145196914673,
      "learning_rate": 4.6196647777377475e-06,
      "logits/chosen": -3.0732457637786865,
      "logits/rejected": -3.093071222305298,
      "logps/chosen": -0.8488075137138367,
      "logps/rejected": -0.9796191453933716,
      "loss": 0.9176,
      "odds_ratio_loss": 0.6876064538955688,
      "rewards/accuracies": 0.543749988079071,
      "rewards/chosen": -0.0848807543516159,
      "rewards/margins": 0.013081158511340618,
      "rewards/rejected": -0.0979619026184082,
      "sft_loss": 0.8488075137138367,
      "step": 300
    },
    {
      "epoch": 0.5512336074683263,
      "grad_norm": 1.6855430603027344,
      "learning_rate": 4.59459764953147e-06,
      "logits/chosen": -3.115689754486084,
      "logits/rejected": -3.099546194076538,
      "logps/chosen": -0.8772395253181458,
      "logps/rejected": -1.0352530479431152,
      "loss": 0.9431,
      "odds_ratio_loss": 0.6584862470626831,
      "rewards/accuracies": 0.518750011920929,
      "rewards/chosen": -0.08772395551204681,
      "rewards/margins": 0.015801362693309784,
      "rewards/rejected": -0.1035253182053566,
      "sft_loss": 0.8772395253181458,
      "step": 310
    },
    {
      "epoch": 0.5690153367414981,
      "grad_norm": 3.039783239364624,
      "learning_rate": 4.568803203605133e-06,
      "logits/chosen": -3.1416523456573486,
      "logits/rejected": -3.1220498085021973,
      "logps/chosen": -0.8318166732788086,
      "logps/rejected": -1.035842776298523,
      "loss": 0.8969,
      "odds_ratio_loss": 0.650640606880188,
      "rewards/accuracies": 0.5687500238418579,
      "rewards/chosen": -0.08318166434764862,
      "rewards/margins": 0.020402604714035988,
      "rewards/rejected": -0.10358426719903946,
      "sft_loss": 0.8318166732788086,
      "step": 320
    },
    {
      "epoch": 0.58679706601467,
      "grad_norm": 1.5947670936584473,
      "learning_rate": 4.542290396694462e-06,
      "logits/chosen": -3.100538969039917,
      "logits/rejected": -3.1203720569610596,
      "logps/chosen": -0.855880081653595,
      "logps/rejected": -1.0065386295318604,
      "loss": 0.9255,
      "odds_ratio_loss": 0.6964801549911499,
      "rewards/accuracies": 0.512499988079071,
      "rewards/chosen": -0.0855880007147789,
      "rewards/margins": 0.01506584882736206,
      "rewards/rejected": -0.10065384954214096,
      "sft_loss": 0.855880081653595,
      "step": 330
    },
    {
      "epoch": 0.6045787952878418,
      "grad_norm": 3.104470729827881,
      "learning_rate": 4.515068434975298e-06,
      "logits/chosen": -3.0526375770568848,
      "logits/rejected": -3.0920848846435547,
      "logps/chosen": -0.8729322552680969,
      "logps/rejected": -1.0834085941314697,
      "loss": 0.9376,
      "odds_ratio_loss": 0.6469117403030396,
      "rewards/accuracies": 0.581250011920929,
      "rewards/chosen": -0.08729322254657745,
      "rewards/margins": 0.021047625690698624,
      "rewards/rejected": -0.10834084451198578,
      "sft_loss": 0.8729322552680969,
      "step": 340
    },
    {
      "epoch": 0.6223605245610135,
      "grad_norm": 1.5185527801513672,
      "learning_rate": 4.487146770866887e-06,
      "logits/chosen": -3.1082205772399902,
      "logits/rejected": -3.146754503250122,
      "logps/chosen": -0.866405189037323,
      "logps/rejected": -0.985508143901825,
      "loss": 0.9343,
      "odds_ratio_loss": 0.6789035797119141,
      "rewards/accuracies": 0.53125,
      "rewards/chosen": -0.08664052188396454,
      "rewards/margins": 0.011910290457308292,
      "rewards/rejected": -0.09855081886053085,
      "sft_loss": 0.866405189037323,
      "step": 350
    },
    {
      "epoch": 0.6401422538341853,
      "grad_norm": 2.0399420261383057,
      "learning_rate": 4.458535099749666e-06,
      "logits/chosen": -3.114278793334961,
      "logits/rejected": -3.1290249824523926,
      "logps/chosen": -0.9554277658462524,
      "logps/rejected": -1.0395957231521606,
      "loss": 1.0312,
      "odds_ratio_loss": 0.7574664950370789,
      "rewards/accuracies": 0.53125,
      "rewards/chosen": -0.095542773604393,
      "rewards/margins": 0.008416806347668171,
      "rewards/rejected": -0.1039595827460289,
      "sft_loss": 0.9554277658462524,
      "step": 360
    },
    {
      "epoch": 0.6579239831073572,
      "grad_norm": 2.1999988555908203,
      "learning_rate": 4.429243356598694e-06,
      "logits/chosen": -3.0874438285827637,
      "logits/rejected": -3.098285436630249,
      "logps/chosen": -0.8949627876281738,
      "logps/rejected": -1.1512229442596436,
      "loss": 0.9596,
      "odds_ratio_loss": 0.645936131477356,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": -0.08949627727270126,
      "rewards/margins": 0.02562602423131466,
      "rewards/rejected": -0.11512230336666107,
      "sft_loss": 0.8949627876281738,
      "step": 370
    },
    {
      "epoch": 0.675705712380529,
      "grad_norm": 5.087428092956543,
      "learning_rate": 4.399281712533875e-06,
      "logits/chosen": -3.118114709854126,
      "logits/rejected": -3.1233677864074707,
      "logps/chosen": -0.8167802095413208,
      "logps/rejected": -0.9615100026130676,
      "loss": 0.8862,
      "odds_ratio_loss": 0.6943861246109009,
      "rewards/accuracies": 0.512499988079071,
      "rewards/chosen": -0.08167801797389984,
      "rewards/margins": 0.01447297353297472,
      "rewards/rejected": -0.09615099430084229,
      "sft_loss": 0.8167802095413208,
      "step": 380
    },
    {
      "epoch": 0.6934874416537008,
      "grad_norm": 2.3240132331848145,
      "learning_rate": 4.368660571288192e-06,
      "logits/chosen": -3.1258320808410645,
      "logits/rejected": -3.1687591075897217,
      "logps/chosen": -0.8471567034721375,
      "logps/rejected": -0.9503539204597473,
      "loss": 0.917,
      "odds_ratio_loss": 0.6983430981636047,
      "rewards/accuracies": 0.4937500059604645,
      "rewards/chosen": -0.08471567928791046,
      "rewards/margins": 0.010319720953702927,
      "rewards/rejected": -0.09503538906574249,
      "sft_loss": 0.8471567034721375,
      "step": 390
    },
    {
      "epoch": 0.7112691709268726,
      "grad_norm": 1.7417421340942383,
      "learning_rate": 4.337390565595163e-06,
      "logits/chosen": -3.0782721042633057,
      "logits/rejected": -3.099292278289795,
      "logps/chosen": -0.9293394088745117,
      "logps/rejected": -0.9703164100646973,
      "loss": 1.0041,
      "odds_ratio_loss": 0.7478191256523132,
      "rewards/accuracies": 0.48750001192092896,
      "rewards/chosen": -0.09293393790721893,
      "rewards/margins": 0.004097697325050831,
      "rewards/rejected": -0.09703163802623749,
      "sft_loss": 0.9293394088745117,
      "step": 400
    },
    {
      "epoch": 0.7290509002000445,
      "grad_norm": 2.362359046936035,
      "learning_rate": 4.305482553496786e-06,
      "logits/chosen": -3.0271878242492676,
      "logits/rejected": -3.0372941493988037,
      "logps/chosen": -0.8028362989425659,
      "logps/rejected": -0.994833767414093,
      "loss": 0.868,
      "odds_ratio_loss": 0.6515198945999146,
      "rewards/accuracies": 0.5687500238418579,
      "rewards/chosen": -0.08028362691402435,
      "rewards/margins": 0.01919974386692047,
      "rewards/rejected": -0.09948337078094482,
      "sft_loss": 0.8028362989425659,
      "step": 410
    },
    {
      "epoch": 0.7468326294732163,
      "grad_norm": 2.797231674194336,
      "learning_rate": 4.272947614573244e-06,
      "logits/chosen": -3.0782933235168457,
      "logits/rejected": -3.116833448410034,
      "logps/chosen": -0.8883565068244934,
      "logps/rejected": -1.025665283203125,
      "loss": 0.9551,
      "odds_ratio_loss": 0.6675280332565308,
      "rewards/accuracies": 0.5874999761581421,
      "rewards/chosen": -0.08883564919233322,
      "rewards/margins": 0.013730885460972786,
      "rewards/rejected": -0.10256652534008026,
      "sft_loss": 0.8883565068244934,
      "step": 420
    },
    {
      "epoch": 0.7646143587463881,
      "grad_norm": 1.3282934427261353,
      "learning_rate": 4.23979704609569e-06,
      "logits/chosen": -3.10003399848938,
      "logits/rejected": -3.1360583305358887,
      "logps/chosen": -0.8379910588264465,
      "logps/rejected": -0.9617422819137573,
      "loss": 0.9024,
      "odds_ratio_loss": 0.6443312168121338,
      "rewards/accuracies": 0.5687500238418579,
      "rewards/chosen": -0.08379910886287689,
      "rewards/margins": 0.01237512193620205,
      "rewards/rejected": -0.0961742252111435,
      "sft_loss": 0.8379910588264465,
      "step": 430
    },
    {
      "epoch": 0.78239608801956,
      "grad_norm": 2.023909330368042,
      "learning_rate": 4.206042359103435e-06,
      "logits/chosen": -3.0400068759918213,
      "logits/rejected": -3.081937313079834,
      "logps/chosen": -0.8709232211112976,
      "logps/rejected": -1.0840847492218018,
      "loss": 0.9355,
      "odds_ratio_loss": 0.6454750299453735,
      "rewards/accuracies": 0.5375000238418579,
      "rewards/chosen": -0.0870923325419426,
      "rewards/margins": 0.021316152065992355,
      "rewards/rejected": -0.10840848833322525,
      "sft_loss": 0.8709232211112976,
      "step": 440
    },
    {
      "epoch": 0.8001778172927317,
      "grad_norm": 1.6410523653030396,
      "learning_rate": 4.17169527440691e-06,
      "logits/chosen": -3.09321928024292,
      "logits/rejected": -3.0952792167663574,
      "logps/chosen": -0.858233630657196,
      "logps/rejected": -0.9470105171203613,
      "loss": 0.9306,
      "odds_ratio_loss": 0.7236040830612183,
      "rewards/accuracies": 0.46875,
      "rewards/chosen": -0.08582336455583572,
      "rewards/margins": 0.008877689950168133,
      "rewards/rejected": -0.09470105171203613,
      "sft_loss": 0.858233630657196,
      "step": 450
    },
    {
      "epoch": 0.8179595465659035,
      "grad_norm": 3.3464248180389404,
      "learning_rate": 4.136767718517797e-06,
      "logits/chosen": -3.0975563526153564,
      "logits/rejected": -3.112638235092163,
      "logps/chosen": -0.7778853178024292,
      "logps/rejected": -0.9875160455703735,
      "loss": 0.8402,
      "odds_ratio_loss": 0.6234691143035889,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -0.07778853178024292,
      "rewards/margins": 0.020963061600923538,
      "rewards/rejected": -0.09875159710645676,
      "sft_loss": 0.7778853178024292,
      "step": 460
    },
    {
      "epoch": 0.8357412758390753,
      "grad_norm": 6.556829929351807,
      "learning_rate": 4.1012718195077196e-06,
      "logits/chosen": -3.1534528732299805,
      "logits/rejected": -3.208789348602295,
      "logps/chosen": -0.8605148196220398,
      "logps/rejected": -0.9714852571487427,
      "loss": 0.9289,
      "odds_ratio_loss": 0.6839339137077332,
      "rewards/accuracies": 0.4937500059604645,
      "rewards/chosen": -0.08605148643255234,
      "rewards/margins": 0.011097034439444542,
      "rewards/rejected": -0.09714852273464203,
      "sft_loss": 0.8605148196220398,
      "step": 470
    },
    {
      "epoch": 0.8535230051122472,
      "grad_norm": 1.461613655090332,
      "learning_rate": 4.065219902796953e-06,
      "logits/chosen": -3.090115785598755,
      "logits/rejected": -3.088887929916382,
      "logps/chosen": -0.8349069356918335,
      "logps/rejected": -1.0508782863616943,
      "loss": 0.8996,
      "odds_ratio_loss": 0.6472839713096619,
      "rewards/accuracies": 0.581250011920929,
      "rewards/chosen": -0.08349069207906723,
      "rewards/margins": 0.021597128361463547,
      "rewards/rejected": -0.10508781671524048,
      "sft_loss": 0.8349069356918335,
      "step": 480
    },
    {
      "epoch": 0.871304734385419,
      "grad_norm": 1.3518534898757935,
      "learning_rate": 4.028624486874608e-06,
      "logits/chosen": -3.1022493839263916,
      "logits/rejected": -3.1475415229797363,
      "logps/chosen": -0.8089026212692261,
      "logps/rejected": -1.0444796085357666,
      "loss": 0.8747,
      "odds_ratio_loss": 0.6580694317817688,
      "rewards/accuracies": 0.581250011920929,
      "rewards/chosen": -0.08089026063680649,
      "rewards/margins": 0.023557689040899277,
      "rewards/rejected": -0.10444796085357666,
      "sft_loss": 0.8089026212692261,
      "step": 490
    },
    {
      "epoch": 0.8890864636585908,
      "grad_norm": 1.6888097524642944,
      "learning_rate": 3.99149827895177e-06,
      "logits/chosen": -3.127162456512451,
      "logits/rejected": -3.143782138824463,
      "logps/chosen": -0.8767441511154175,
      "logps/rejected": -0.9676705598831177,
      "loss": 0.9464,
      "odds_ratio_loss": 0.6963816285133362,
      "rewards/accuracies": 0.5062500238418579,
      "rewards/chosen": -0.08767442405223846,
      "rewards/margins": 0.009092646650969982,
      "rewards/rejected": -0.09676706790924072,
      "sft_loss": 0.8767441511154175,
      "step": 500
    },
    {
      "epoch": 0.8890864636585908,
      "eval_logits/chosen": -3.114872932434082,
      "eval_logits/rejected": -3.143216371536255,
      "eval_logps/chosen": -0.828136146068573,
      "eval_logps/rejected": -1.0306241512298584,
      "eval_loss": 0.8918758630752563,
      "eval_odds_ratio_loss": 0.6373972296714783,
      "eval_rewards/accuracies": 0.5690000057220459,
      "eval_rewards/chosen": -0.08281362056732178,
      "eval_rewards/margins": 0.020248806104063988,
      "eval_rewards/rejected": -0.10306241363286972,
      "eval_runtime": 348.9195,
      "eval_samples_per_second": 2.866,
      "eval_sft_loss": 0.828136146068573,
      "eval_steps_per_second": 1.433,
      "step": 500
    },
    {
      "epoch": 0.9068681929317626,
      "grad_norm": 1.511196494102478,
      "learning_rate": 3.953854170549114e-06,
      "logits/chosen": -3.118255138397217,
      "logits/rejected": -3.1173043251037598,
      "logps/chosen": -0.8566571474075317,
      "logps/rejected": -0.9489420056343079,
      "loss": 0.9245,
      "odds_ratio_loss": 0.6780352592468262,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": -0.08566570281982422,
      "rewards/margins": 0.009228493086993694,
      "rewards/rejected": -0.09489420056343079,
      "sft_loss": 0.8566571474075317,
      "step": 510
    },
    {
      "epoch": 0.9246499222049345,
      "grad_norm": 2.5393214225769043,
      "learning_rate": 3.91570523302051e-06,
      "logits/chosen": -3.1395115852355957,
      "logits/rejected": -3.147805690765381,
      "logps/chosen": -0.7916607856750488,
      "logps/rejected": -0.9899943470954895,
      "loss": 0.8583,
      "odds_ratio_loss": 0.6660428047180176,
      "rewards/accuracies": 0.5249999761581421,
      "rewards/chosen": -0.07916607707738876,
      "rewards/margins": 0.01983334682881832,
      "rewards/rejected": -0.09899942576885223,
      "sft_loss": 0.7916607856750488,
      "step": 520
    },
    {
      "epoch": 0.9424316514781063,
      "grad_norm": 1.5944111347198486,
      "learning_rate": 3.8770647130141996e-06,
      "logits/chosen": -3.150245428085327,
      "logits/rejected": -3.141481876373291,
      "logps/chosen": -0.8228055834770203,
      "logps/rejected": -0.9976710081100464,
      "loss": 0.8888,
      "odds_ratio_loss": 0.6599084734916687,
      "rewards/accuracies": 0.512499988079071,
      "rewards/chosen": -0.08228056132793427,
      "rewards/margins": 0.017486536875367165,
      "rewards/rejected": -0.09976708889007568,
      "sft_loss": 0.8228055834770203,
      "step": 530
    },
    {
      "epoch": 0.960213380751278,
      "grad_norm": 2.3844027519226074,
      "learning_rate": 3.837946027873086e-06,
      "logits/chosen": -3.106717586517334,
      "logits/rejected": -3.109330177307129,
      "logps/chosen": -0.8973621129989624,
      "logps/rejected": -1.0649579763412476,
      "loss": 0.966,
      "odds_ratio_loss": 0.6861368417739868,
      "rewards/accuracies": 0.5874999761581421,
      "rewards/chosen": -0.089736208319664,
      "rewards/margins": 0.01675957441329956,
      "rewards/rejected": -0.10649579763412476,
      "sft_loss": 0.8973621129989624,
      "step": 540
    },
    {
      "epoch": 0.9779951100244498,
      "grad_norm": 5.164077281951904,
      "learning_rate": 3.7983627609757713e-06,
      "logits/chosen": -3.167064666748047,
      "logits/rejected": -3.16302490234375,
      "logps/chosen": -0.8979376554489136,
      "logps/rejected": -0.9942687153816223,
      "loss": 0.9667,
      "odds_ratio_loss": 0.6874598264694214,
      "rewards/accuracies": 0.5249999761581421,
      "rewards/chosen": -0.08979376405477524,
      "rewards/margins": 0.009633105248212814,
      "rewards/rejected": -0.09942687302827835,
      "sft_loss": 0.8979376554489136,
      "step": 550
    },
    {
      "epoch": 0.9957768392976217,
      "grad_norm": 1.5917680263519287,
      "learning_rate": 3.758328657019924e-06,
      "logits/chosen": -3.1346166133880615,
      "logits/rejected": -3.1376471519470215,
      "logps/chosen": -0.8218947649002075,
      "logps/rejected": -1.0310758352279663,
      "loss": 0.8891,
      "odds_ratio_loss": 0.6719549298286438,
      "rewards/accuracies": 0.543749988079071,
      "rewards/chosen": -0.08218947798013687,
      "rewards/margins": 0.020918114110827446,
      "rewards/rejected": -0.10310759395360947,
      "sft_loss": 0.8218947649002075,
      "step": 560
    },
    {
      "epoch": 1.0135585685707935,
      "grad_norm": 6.842823505401611,
      "learning_rate": 3.717857617249642e-06,
      "logits/chosen": -3.1036324501037598,
      "logits/rejected": -3.145653009414673,
      "logps/chosen": -0.8951196670532227,
      "logps/rejected": -1.0871955156326294,
      "loss": 0.9655,
      "odds_ratio_loss": 0.7041261792182922,
      "rewards/accuracies": 0.53125,
      "rewards/chosen": -0.08951196819543839,
      "rewards/margins": 0.01920759119093418,
      "rewards/rejected": -0.10871955007314682,
      "sft_loss": 0.8951196670532227,
      "step": 570
    },
    {
      "epoch": 1.0313402978439654,
      "grad_norm": 1.3233413696289062,
      "learning_rate": 3.6769636946284543e-06,
      "logits/chosen": -3.145310878753662,
      "logits/rejected": -3.1411328315734863,
      "logps/chosen": -0.8030536770820618,
      "logps/rejected": -0.9519485235214233,
      "loss": 0.8686,
      "odds_ratio_loss": 0.6551788449287415,
      "rewards/accuracies": 0.5375000238418579,
      "rewards/chosen": -0.08030536770820618,
      "rewards/margins": 0.01488947682082653,
      "rewards/rejected": -0.09519485384225845,
      "sft_loss": 0.8030536770820618,
      "step": 580
    },
    {
      "epoch": 1.049122027117137,
      "grad_norm": 1.561957597732544,
      "learning_rate": 3.6356610889596355e-06,
      "logits/chosen": -3.1137917041778564,
      "logits/rejected": -3.15521502494812,
      "logps/chosen": -0.8285630941390991,
      "logps/rejected": -0.9533591270446777,
      "loss": 0.895,
      "odds_ratio_loss": 0.6645855903625488,
      "rewards/accuracies": 0.4937500059604645,
      "rewards/chosen": -0.08285631239414215,
      "rewards/margins": 0.012479597702622414,
      "rewards/rejected": -0.09533590078353882,
      "sft_loss": 0.8285630941390991,
      "step": 590
    },
    {
      "epoch": 1.066903756390309,
      "grad_norm": 2.0521960258483887,
      "learning_rate": 3.593964141955541e-06,
      "logits/chosen": -3.0969531536102295,
      "logits/rejected": -3.0988070964813232,
      "logps/chosen": -0.8090001344680786,
      "logps/rejected": -0.9104982614517212,
      "loss": 0.8782,
      "odds_ratio_loss": 0.6919649839401245,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -0.08090001344680786,
      "rewards/margins": 0.01014980860054493,
      "rewards/rejected": -0.09104982018470764,
      "sft_loss": 0.8090001344680786,
      "step": 600
    },
    {
      "epoch": 1.0846854856634809,
      "grad_norm": 1.7750905752182007,
      "learning_rate": 3.5518873322576573e-06,
      "logits/chosen": -3.044728994369507,
      "logits/rejected": -3.0994975566864014,
      "logps/chosen": -0.8208731412887573,
      "logps/rejected": -0.9647499918937683,
      "loss": 0.8855,
      "odds_ratio_loss": 0.6465209126472473,
      "rewards/accuracies": 0.5687500238418579,
      "rewards/chosen": -0.08208731561899185,
      "rewards/margins": 0.014387677423655987,
      "rewards/rejected": -0.09647499024868011,
      "sft_loss": 0.8208731412887573,
      "step": 610
    },
    {
      "epoch": 1.1024672149366526,
      "grad_norm": 1.3415883779525757,
      "learning_rate": 3.5094452704091143e-06,
      "logits/chosen": -3.0923125743865967,
      "logits/rejected": -3.084038734436035,
      "logps/chosen": -0.7946149110794067,
      "logps/rejected": -0.9547470808029175,
      "loss": 0.8602,
      "odds_ratio_loss": 0.6559656858444214,
      "rewards/accuracies": 0.5375000238418579,
      "rewards/chosen": -0.0794614925980568,
      "rewards/margins": 0.016013216227293015,
      "rewards/rejected": -0.09547470510005951,
      "sft_loss": 0.7946149110794067,
      "step": 620
    },
    {
      "epoch": 1.1202489442098245,
      "grad_norm": 3.5497653484344482,
      "learning_rate": 3.46665269378139e-06,
      "logits/chosen": -3.059072971343994,
      "logits/rejected": -3.069256067276001,
      "logps/chosen": -0.8376399874687195,
      "logps/rejected": -0.9781678318977356,
      "loss": 0.9066,
      "odds_ratio_loss": 0.6894850134849548,
      "rewards/accuracies": 0.5375000238418579,
      "rewards/chosen": -0.08376399427652359,
      "rewards/margins": 0.01405278779566288,
      "rewards/rejected": -0.09781678020954132,
      "sft_loss": 0.8376399874687195,
      "step": 630
    },
    {
      "epoch": 1.1380306734829961,
      "grad_norm": 3.2695467472076416,
      "learning_rate": 3.4235244614569794e-06,
      "logits/chosen": -3.0830600261688232,
      "logits/rejected": -3.0843684673309326,
      "logps/chosen": -0.8990565538406372,
      "logps/rejected": -0.9821575880050659,
      "loss": 0.9716,
      "odds_ratio_loss": 0.7258378267288208,
      "rewards/accuracies": 0.59375,
      "rewards/chosen": -0.08990565687417984,
      "rewards/margins": 0.008310111239552498,
      "rewards/rejected": -0.09821576625108719,
      "sft_loss": 0.8990565538406372,
      "step": 640
    },
    {
      "epoch": 1.155812402756168,
      "grad_norm": 1.196513056755066,
      "learning_rate": 3.3800755490698008e-06,
      "logits/chosen": -3.126264810562134,
      "logits/rejected": -3.124204635620117,
      "logps/chosen": -0.8168405294418335,
      "logps/rejected": -1.0534611940383911,
      "loss": 0.8775,
      "odds_ratio_loss": 0.6068293452262878,
      "rewards/accuracies": 0.5874999761581421,
      "rewards/chosen": -0.08168406784534454,
      "rewards/margins": 0.023662051185965538,
      "rewards/rejected": -0.10534612834453583,
      "sft_loss": 0.8168405294418335,
      "step": 650
    },
    {
      "epoch": 1.17359413202934,
      "grad_norm": 1.7081139087677002,
      "learning_rate": 3.3363210436051287e-06,
      "logits/chosen": -3.130343198776245,
      "logits/rejected": -3.126983165740967,
      "logps/chosen": -0.8528251647949219,
      "logps/rejected": -1.019565224647522,
      "loss": 0.9228,
      "odds_ratio_loss": 0.6997644901275635,
      "rewards/accuracies": 0.518750011920929,
      "rewards/chosen": -0.08528250455856323,
      "rewards/margins": 0.01667400822043419,
      "rewards/rejected": -0.10195653140544891,
      "sft_loss": 0.8528251647949219,
      "step": 660
    },
    {
      "epoch": 1.1913758613025116,
      "grad_norm": 1.8305083513259888,
      "learning_rate": 3.292276138160867e-06,
      "logits/chosen": -3.109675645828247,
      "logits/rejected": -3.1157774925231934,
      "logps/chosen": -0.7888280153274536,
      "logps/rejected": -0.9577935338020325,
      "loss": 0.8538,
      "odds_ratio_loss": 0.6501932144165039,
      "rewards/accuracies": 0.512499988079071,
      "rewards/chosen": -0.07888280600309372,
      "rewards/margins": 0.016896549612283707,
      "rewards/rejected": -0.09577935189008713,
      "sft_loss": 0.7888280153274536,
      "step": 670
    },
    {
      "epoch": 1.2091575905756835,
      "grad_norm": 2.812506675720215,
      "learning_rate": 3.2479561266719694e-06,
      "logits/chosen": -3.1019396781921387,
      "logits/rejected": -3.107755184173584,
      "logps/chosen": -0.8298002481460571,
      "logps/rejected": -0.9901537895202637,
      "loss": 0.8946,
      "odds_ratio_loss": 0.6482952833175659,
      "rewards/accuracies": 0.5874999761581421,
      "rewards/chosen": -0.08298002183437347,
      "rewards/margins": 0.01603536494076252,
      "rewards/rejected": -0.09901538491249084,
      "sft_loss": 0.8298002481460571,
      "step": 680
    },
    {
      "epoch": 1.2269393198488552,
      "grad_norm": 3.2340750694274902,
      "learning_rate": 3.2033763985998533e-06,
      "logits/chosen": -3.121992588043213,
      "logits/rejected": -3.124979257583618,
      "logps/chosen": -0.7747536301612854,
      "logps/rejected": -1.1079862117767334,
      "loss": 0.8347,
      "odds_ratio_loss": 0.5990911722183228,
      "rewards/accuracies": 0.612500011920929,
      "rewards/chosen": -0.07747535407543182,
      "rewards/margins": 0.033323269337415695,
      "rewards/rejected": -0.11079863458871841,
      "sft_loss": 0.7747536301612854,
      "step": 690
    },
    {
      "epoch": 1.244721049122027,
      "grad_norm": 1.9602211713790894,
      "learning_rate": 3.1585524335886335e-06,
      "logits/chosen": -3.1363680362701416,
      "logits/rejected": -3.1302547454833984,
      "logps/chosen": -0.7745245695114136,
      "logps/rejected": -0.9697211980819702,
      "loss": 0.838,
      "odds_ratio_loss": 0.6345950365066528,
      "rewards/accuracies": 0.581250011920929,
      "rewards/chosen": -0.07745245844125748,
      "rewards/margins": 0.019519677385687828,
      "rewards/rejected": -0.09697212278842926,
      "sft_loss": 0.7745245695114136,
      "step": 700
    },
    {
      "epoch": 1.262502778395199,
      "grad_norm": 3.0812952518463135,
      "learning_rate": 3.1134997960900536e-06,
      "logits/chosen": -3.0771961212158203,
      "logits/rejected": -3.08510160446167,
      "logps/chosen": -0.7646561861038208,
      "logps/rejected": -1.0509836673736572,
      "loss": 0.8246,
      "odds_ratio_loss": 0.5993521809577942,
      "rewards/accuracies": 0.612500011920929,
      "rewards/chosen": -0.07646562159061432,
      "rewards/margins": 0.028632745146751404,
      "rewards/rejected": -0.10509836673736572,
      "sft_loss": 0.7646561861038208,
      "step": 710
    },
    {
      "epoch": 1.2802845076683709,
      "grad_norm": 1.5706931352615356,
      "learning_rate": 3.0682341299589583e-06,
      "logits/chosen": -3.096446990966797,
      "logits/rejected": -3.110931873321533,
      "logps/chosen": -0.8055674433708191,
      "logps/rejected": -0.9553298950195312,
      "loss": 0.8716,
      "odds_ratio_loss": 0.6605285406112671,
      "rewards/accuracies": 0.512499988079071,
      "rewards/chosen": -0.08055675029754639,
      "rewards/margins": 0.014976252801716328,
      "rewards/rejected": -0.09553299844264984,
      "sft_loss": 0.8055674433708191,
      "step": 720
    },
    {
      "epoch": 1.2980662369415426,
      "grad_norm": 1.670327067375183,
      "learning_rate": 3.022771153021201e-06,
      "logits/chosen": -3.127776622772217,
      "logits/rejected": -3.1598572731018066,
      "logps/chosen": -0.7699373960494995,
      "logps/rejected": -0.9526535272598267,
      "loss": 0.8354,
      "odds_ratio_loss": 0.654297947883606,
      "rewards/accuracies": 0.637499988079071,
      "rewards/chosen": -0.07699373364448547,
      "rewards/margins": 0.018271619454026222,
      "rewards/rejected": -0.09526535123586655,
      "sft_loss": 0.7699373960494995,
      "step": 730
    },
    {
      "epoch": 1.3158479662147144,
      "grad_norm": 1.666502833366394,
      "learning_rate": 2.9771266516158625e-06,
      "logits/chosen": -3.0938611030578613,
      "logits/rejected": -3.111356735229492,
      "logps/chosen": -0.795330822467804,
      "logps/rejected": -0.9487611055374146,
      "loss": 0.8641,
      "odds_ratio_loss": 0.687196671962738,
      "rewards/accuracies": 0.512499988079071,
      "rewards/chosen": -0.07953307777643204,
      "rewards/margins": 0.015343038365244865,
      "rewards/rejected": -0.09487612545490265,
      "sft_loss": 0.795330822467804,
      "step": 740
    },
    {
      "epoch": 1.3336296954878861,
      "grad_norm": 1.529642939567566,
      "learning_rate": 2.9313164751136802e-06,
      "logits/chosen": -3.082942485809326,
      "logits/rejected": -3.1158337593078613,
      "logps/chosen": -0.789255678653717,
      "logps/rejected": -0.9912340044975281,
      "loss": 0.8503,
      "odds_ratio_loss": 0.6099725961685181,
      "rewards/accuracies": 0.643750011920929,
      "rewards/chosen": -0.07892556488513947,
      "rewards/margins": 0.020197834819555283,
      "rewards/rejected": -0.09912340342998505,
      "sft_loss": 0.789255678653717,
      "step": 750
    },
    {
      "epoch": 1.351411424761058,
      "grad_norm": 2.9339799880981445,
      "learning_rate": 2.8853565304135956e-06,
      "logits/chosen": -3.1478281021118164,
      "logits/rejected": -3.144963264465332,
      "logps/chosen": -0.8711767196655273,
      "logps/rejected": -0.9750477075576782,
      "loss": 0.9427,
      "odds_ratio_loss": 0.7154635190963745,
      "rewards/accuracies": 0.512499988079071,
      "rewards/chosen": -0.08711767941713333,
      "rewards/margins": 0.010387107729911804,
      "rewards/rejected": -0.09750477969646454,
      "sft_loss": 0.8711767196655273,
      "step": 760
    },
    {
      "epoch": 1.36919315403423,
      "grad_norm": 3.5656025409698486,
      "learning_rate": 2.839262776419313e-06,
      "logits/chosen": -3.1182093620300293,
      "logits/rejected": -3.1154582500457764,
      "logps/chosen": -0.7866981029510498,
      "logps/rejected": -1.0985205173492432,
      "loss": 0.8467,
      "odds_ratio_loss": 0.5998324155807495,
      "rewards/accuracies": 0.643750011920929,
      "rewards/chosen": -0.07866980135440826,
      "rewards/margins": 0.031182238832116127,
      "rewards/rejected": -0.10985203832387924,
      "sft_loss": 0.7866981029510498,
      "step": 770
    },
    {
      "epoch": 1.3869748833074016,
      "grad_norm": 2.08962345123291,
      "learning_rate": 2.793051218497817e-06,
      "logits/chosen": -3.1209683418273926,
      "logits/rejected": -3.1391050815582275,
      "logps/chosen": -0.797200083732605,
      "logps/rejected": -0.8991384506225586,
      "loss": 0.8653,
      "odds_ratio_loss": 0.6806570887565613,
      "rewards/accuracies": 0.4937500059604645,
      "rewards/chosen": -0.07972002029418945,
      "rewards/margins": 0.010193833149969578,
      "rewards/rejected": -0.08991385996341705,
      "sft_loss": 0.797200083732605,
      "step": 780
    },
    {
      "epoch": 1.4047566125805735,
      "grad_norm": 1.7654404640197754,
      "learning_rate": 2.7467379029217437e-06,
      "logits/chosen": -3.092345714569092,
      "logits/rejected": -3.099000930786133,
      "logps/chosen": -0.7963561415672302,
      "logps/rejected": -0.9906966090202332,
      "loss": 0.8603,
      "odds_ratio_loss": 0.639264702796936,
      "rewards/accuracies": 0.581250011920929,
      "rewards/chosen": -0.0796356275677681,
      "rewards/margins": 0.019434038549661636,
      "rewards/rejected": -0.09906966984272003,
      "sft_loss": 0.7963561415672302,
      "step": 790
    },
    {
      "epoch": 1.4225383418537452,
      "grad_norm": 1.4254413843154907,
      "learning_rate": 2.7003389112975546e-06,
      "logits/chosen": -3.1396844387054443,
      "logits/rejected": -3.180053949356079,
      "logps/chosen": -0.844267725944519,
      "logps/rejected": -0.9890397191047668,
      "loss": 0.9108,
      "odds_ratio_loss": 0.6649594902992249,
      "rewards/accuracies": 0.518750011920929,
      "rewards/chosen": -0.08442677557468414,
      "rewards/margins": 0.01447719894349575,
      "rewards/rejected": -0.09890398383140564,
      "sft_loss": 0.844267725944519,
      "step": 800
    },
    {
      "epoch": 1.440320071126917,
      "grad_norm": 3.8261585235595703,
      "learning_rate": 2.653870354981437e-06,
      "logits/chosen": -3.123039722442627,
      "logits/rejected": -3.1270766258239746,
      "logps/chosen": -0.7622265219688416,
      "logps/rejected": -0.9670180082321167,
      "loss": 0.8268,
      "odds_ratio_loss": 0.6459091901779175,
      "rewards/accuracies": 0.543749988079071,
      "rewards/chosen": -0.07622265070676804,
      "rewards/margins": 0.020479146391153336,
      "rewards/rejected": -0.09670180082321167,
      "sft_loss": 0.7622265219688416,
      "step": 810
    },
    {
      "epoch": 1.458101800400089,
      "grad_norm": 6.478664875030518,
      "learning_rate": 2.6073483694848777e-06,
      "logits/chosen": -3.0914266109466553,
      "logits/rejected": -3.1468262672424316,
      "logps/chosen": -0.7940482497215271,
      "logps/rejected": -0.9618217349052429,
      "loss": 0.8608,
      "odds_ratio_loss": 0.6678633093833923,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -0.07940482348203659,
      "rewards/margins": 0.016777347773313522,
      "rewards/rejected": -0.09618218243122101,
      "sft_loss": 0.7940482497215271,
      "step": 820
    },
    {
      "epoch": 1.4758835296732609,
      "grad_norm": 1.7955982685089111,
      "learning_rate": 2.560789108871847e-06,
      "logits/chosen": -3.087249755859375,
      "logits/rejected": -3.099762439727783,
      "logps/chosen": -0.8293372392654419,
      "logps/rejected": -1.0816946029663086,
      "loss": 0.8942,
      "odds_ratio_loss": 0.6483136415481567,
      "rewards/accuracies": 0.512499988079071,
      "rewards/chosen": -0.08293372392654419,
      "rewards/margins": 0.025235742330551147,
      "rewards/rejected": -0.10816947370767593,
      "sft_loss": 0.8293372392654419,
      "step": 830
    },
    {
      "epoch": 1.4936652589464325,
      "grad_norm": 4.553436279296875,
      "learning_rate": 2.514208740149544e-06,
      "logits/chosen": -3.123802900314331,
      "logits/rejected": -3.1615843772888184,
      "logps/chosen": -0.8601408004760742,
      "logps/rejected": -1.0482033491134644,
      "loss": 0.9282,
      "odds_ratio_loss": 0.6806772947311401,
      "rewards/accuracies": 0.5562499761581421,
      "rewards/chosen": -0.08601407706737518,
      "rewards/margins": 0.018806258216500282,
      "rewards/rejected": -0.10482033342123032,
      "sft_loss": 0.8601408004760742,
      "step": 840
    },
    {
      "epoch": 1.5114469882196042,
      "grad_norm": 3.1794512271881104,
      "learning_rate": 2.46762343765464e-06,
      "logits/chosen": -3.1444077491760254,
      "logits/rejected": -3.1544933319091797,
      "logps/chosen": -0.8352905511856079,
      "logps/rejected": -1.0490363836288452,
      "loss": 0.898,
      "odds_ratio_loss": 0.6273452639579773,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": -0.0835290476679802,
      "rewards/margins": 0.021374579519033432,
      "rewards/rejected": -0.10490362346172333,
      "sft_loss": 0.8352905511856079,
      "step": 850
    },
    {
      "epoch": 1.5292287174927761,
      "grad_norm": 1.8062447309494019,
      "learning_rate": 2.4210493774369903e-06,
      "logits/chosen": -3.0938150882720947,
      "logits/rejected": -3.102355718612671,
      "logps/chosen": -0.8377913236618042,
      "logps/rejected": -0.9871052503585815,
      "loss": 0.9059,
      "odds_ratio_loss": 0.6812715530395508,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": -0.08377913385629654,
      "rewards/margins": 0.014931400306522846,
      "rewards/rejected": -0.09871052205562592,
      "sft_loss": 0.8377913236618042,
      "step": 860
    },
    {
      "epoch": 1.547010446765948,
      "grad_norm": 1.5386985540390015,
      "learning_rate": 2.374502731642732e-06,
      "logits/chosen": -3.1051342487335205,
      "logits/rejected": -3.1245017051696777,
      "logps/chosen": -0.8524861335754395,
      "logps/rejected": -1.017881155014038,
      "loss": 0.9179,
      "odds_ratio_loss": 0.6543157696723938,
      "rewards/accuracies": 0.59375,
      "rewards/chosen": -0.08524861931800842,
      "rewards/margins": 0.016539499163627625,
      "rewards/rejected": -0.10178811848163605,
      "sft_loss": 0.8524861335754395,
      "step": 870
    },
    {
      "epoch": 1.56479217603912,
      "grad_norm": 2.0160138607025146,
      "learning_rate": 2.3279996628987556e-06,
      "logits/chosen": -3.093174457550049,
      "logits/rejected": -3.1271913051605225,
      "logps/chosen": -0.8324817419052124,
      "logps/rejected": -0.9784995913505554,
      "loss": 0.8998,
      "odds_ratio_loss": 0.6732369661331177,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": -0.08324816823005676,
      "rewards/margins": 0.01460178941488266,
      "rewards/rejected": -0.09784995764493942,
      "sft_loss": 0.8324817419052124,
      "step": 880
    },
    {
      "epoch": 1.5825739053122916,
      "grad_norm": 1.5362610816955566,
      "learning_rate": 2.281556318700474e-06,
      "logits/chosen": -3.1063926219940186,
      "logits/rejected": -3.150496482849121,
      "logps/chosen": -0.7895249128341675,
      "logps/rejected": -0.9089393615722656,
      "loss": 0.8592,
      "odds_ratio_loss": 0.6968772411346436,
      "rewards/accuracies": 0.5375000238418579,
      "rewards/chosen": -0.07895249128341675,
      "rewards/margins": 0.011941445991396904,
      "rewards/rejected": -0.0908939391374588,
      "sft_loss": 0.7895249128341675,
      "step": 890
    },
    {
      "epoch": 1.6003556345854635,
      "grad_norm": 3.5220394134521484,
      "learning_rate": 2.2351888258048408e-06,
      "logits/chosen": -3.0469326972961426,
      "logits/rejected": -3.095856189727783,
      "logps/chosen": -0.798681914806366,
      "logps/rejected": -0.9773387908935547,
      "loss": 0.8628,
      "odds_ratio_loss": 0.641067385673523,
      "rewards/accuracies": 0.5687500238418579,
      "rewards/chosen": -0.07986819744110107,
      "rewards/margins": 0.017865682020783424,
      "rewards/rejected": -0.09773387759923935,
      "sft_loss": 0.798681914806366,
      "step": 900
    },
    {
      "epoch": 1.6181373638586352,
      "grad_norm": 2.17846941947937,
      "learning_rate": 2.188913284630584e-06,
      "logits/chosen": -3.1179308891296387,
      "logits/rejected": -3.14939284324646,
      "logps/chosen": -0.8766034841537476,
      "logps/rejected": -0.9665753245353699,
      "loss": 0.9481,
      "odds_ratio_loss": 0.7149003148078918,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.08766035735607147,
      "rewards/margins": 0.008997179567813873,
      "rewards/rejected": -0.09665753692388535,
      "sft_loss": 0.8766034841537476,
      "step": 910
    },
    {
      "epoch": 1.635919093131807,
      "grad_norm": 6.3163251876831055,
      "learning_rate": 2.1427457636675652e-06,
      "logits/chosen": -3.118419647216797,
      "logits/rejected": -3.1435821056365967,
      "logps/chosen": -0.8221105337142944,
      "logps/rejected": -0.9653439521789551,
      "loss": 0.8905,
      "odds_ratio_loss": 0.6840168833732605,
      "rewards/accuracies": 0.5687500238418579,
      "rewards/chosen": -0.08221106976270676,
      "rewards/margins": 0.014323326759040356,
      "rewards/rejected": -0.09653439372777939,
      "sft_loss": 0.8221105337142944,
      "step": 920
    },
    {
      "epoch": 1.653700822404979,
      "grad_norm": 2.3323636054992676,
      "learning_rate": 2.096702293897247e-06,
      "logits/chosen": -3.118809700012207,
      "logits/rejected": -3.1240172386169434,
      "logps/chosen": -0.7927727699279785,
      "logps/rejected": -1.0566480159759521,
      "loss": 0.856,
      "odds_ratio_loss": 0.6325381994247437,
      "rewards/accuracies": 0.5874999761581421,
      "rewards/chosen": -0.07927727699279785,
      "rewards/margins": 0.026387536898255348,
      "rewards/rejected": -0.10566481202840805,
      "sft_loss": 0.7927727699279785,
      "step": 930
    },
    {
      "epoch": 1.6714825516781509,
      "grad_norm": 2.5871617794036865,
      "learning_rate": 2.0507988632261672e-06,
      "logits/chosen": -3.0783491134643555,
      "logits/rejected": -3.142695188522339,
      "logps/chosen": -0.788642406463623,
      "logps/rejected": -0.9959260821342468,
      "loss": 0.8502,
      "odds_ratio_loss": 0.6157304048538208,
      "rewards/accuracies": 0.6312500238418579,
      "rewards/chosen": -0.07886423915624619,
      "rewards/margins": 0.020728373900055885,
      "rewards/rejected": -0.09959261119365692,
      "sft_loss": 0.788642406463623,
      "step": 940
    },
    {
      "epoch": 1.6892642809513225,
      "grad_norm": 6.09738302230835,
      "learning_rate": 2.005051410934382e-06,
      "logits/chosen": -3.1027965545654297,
      "logits/rejected": -3.1486849784851074,
      "logps/chosen": -0.8924347162246704,
      "logps/rejected": -1.025657057762146,
      "loss": 0.9605,
      "odds_ratio_loss": 0.6808988451957703,
      "rewards/accuracies": 0.5062500238418579,
      "rewards/chosen": -0.08924347162246704,
      "rewards/margins": 0.013322234153747559,
      "rewards/rejected": -0.1025657057762146,
      "sft_loss": 0.8924347162246704,
      "step": 950
    },
    {
      "epoch": 1.7070460102244942,
      "grad_norm": 2.1956799030303955,
      "learning_rate": 1.9594758221407843e-06,
      "logits/chosen": -3.1190600395202637,
      "logits/rejected": -3.1192500591278076,
      "logps/chosen": -0.7558837532997131,
      "logps/rejected": -0.9841734766960144,
      "loss": 0.8162,
      "odds_ratio_loss": 0.6032260060310364,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.07558837532997131,
      "rewards/margins": 0.0228289682418108,
      "rewards/rejected": -0.09841735661029816,
      "sft_loss": 0.7558837532997131,
      "step": 960
    },
    {
      "epoch": 1.724827739497666,
      "grad_norm": 3.2010116577148438,
      "learning_rate": 1.9140879222872408e-06,
      "logits/chosen": -3.119006872177124,
      "logits/rejected": -3.144542694091797,
      "logps/chosen": -0.7783070206642151,
      "logps/rejected": -0.8883264660835266,
      "loss": 0.8488,
      "odds_ratio_loss": 0.70525062084198,
      "rewards/accuracies": 0.518750011920929,
      "rewards/chosen": -0.07783070206642151,
      "rewards/margins": 0.011001949198544025,
      "rewards/rejected": -0.08883266150951385,
      "sft_loss": 0.7783070206642151,
      "step": 970
    },
    {
      "epoch": 1.742609468770838,
      "grad_norm": 1.4073106050491333,
      "learning_rate": 1.8689034716434346e-06,
      "logits/chosen": -3.143158197402954,
      "logits/rejected": -3.162026882171631,
      "logps/chosen": -0.849888026714325,
      "logps/rejected": -0.9449575543403625,
      "loss": 0.9213,
      "odds_ratio_loss": 0.7145692706108093,
      "rewards/accuracies": 0.5375000238418579,
      "rewards/chosen": -0.08498881012201309,
      "rewards/margins": 0.009506945498287678,
      "rewards/rejected": -0.0944957509636879,
      "sft_loss": 0.849888026714325,
      "step": 980
    },
    {
      "epoch": 1.76039119804401,
      "grad_norm": 1.643964171409607,
      "learning_rate": 1.8239381598343576e-06,
      "logits/chosen": -3.1246304512023926,
      "logits/rejected": -3.1464321613311768,
      "logps/chosen": -0.7999427914619446,
      "logps/rejected": -0.9506388902664185,
      "loss": 0.8683,
      "odds_ratio_loss": 0.6839095950126648,
      "rewards/accuracies": 0.53125,
      "rewards/chosen": -0.07999428361654282,
      "rewards/margins": 0.015069609507918358,
      "rewards/rejected": -0.09506388753652573,
      "sft_loss": 0.7999427914619446,
      "step": 990
    },
    {
      "epoch": 1.7781729273171816,
      "grad_norm": 4.767948150634766,
      "learning_rate": 1.779207600392312e-06,
      "logits/chosen": -3.123835325241089,
      "logits/rejected": -3.1298935413360596,
      "logps/chosen": -0.8067057728767395,
      "logps/rejected": -0.9445611238479614,
      "loss": 0.8737,
      "odds_ratio_loss": 0.6700451970100403,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -0.080670565366745,
      "rewards/margins": 0.013785535469651222,
      "rewards/rejected": -0.09445609152317047,
      "sft_loss": 0.8067057728767395,
      "step": 1000
    },
    {
      "epoch": 1.7781729273171816,
      "eval_logits/chosen": -3.1139109134674072,
      "eval_logits/rejected": -3.1430606842041016,
      "eval_logps/chosen": -0.8136406540870667,
      "eval_logps/rejected": -1.018557071685791,
      "eval_loss": 0.8773505687713623,
      "eval_odds_ratio_loss": 0.6370999217033386,
      "eval_rewards/accuracies": 0.5759999752044678,
      "eval_rewards/chosen": -0.08136406540870667,
      "eval_rewards/margins": 0.020491650328040123,
      "eval_rewards/rejected": -0.10185571014881134,
      "eval_runtime": 189.1267,
      "eval_samples_per_second": 5.287,
      "eval_sft_loss": 0.8136406540870667,
      "eval_steps_per_second": 2.644,
      "step": 1000
    },
    {
      "epoch": 1.7959546565903532,
      "grad_norm": 2.2980809211730957,
      "learning_rate": 1.7347273253353552e-06,
      "logits/chosen": -3.0896313190460205,
      "logits/rejected": -3.117469310760498,
      "logps/chosen": -0.8154736757278442,
      "logps/rejected": -0.9821268320083618,
      "loss": 0.8833,
      "odds_ratio_loss": 0.6783260107040405,
      "rewards/accuracies": 0.4937500059604645,
      "rewards/chosen": -0.08154736459255219,
      "rewards/margins": 0.016665320843458176,
      "rewards/rejected": -0.09821268171072006,
      "sft_loss": 0.8154736757278442,
      "step": 1010
    },
    {
      "epoch": 1.8137363858635251,
      "grad_norm": 4.3619232177734375,
      "learning_rate": 1.690512779774029e-06,
      "logits/chosen": -3.108875036239624,
      "logits/rejected": -3.119654655456543,
      "logps/chosen": -0.8301160931587219,
      "logps/rejected": -1.0722554922103882,
      "loss": 0.8927,
      "odds_ratio_loss": 0.6254863142967224,
      "rewards/accuracies": 0.581250011920929,
      "rewards/chosen": -0.08301161974668503,
      "rewards/margins": 0.024213943630456924,
      "rewards/rejected": -0.10722555965185165,
      "sft_loss": 0.8301160931587219,
      "step": 1020
    },
    {
      "epoch": 1.831518115136697,
      "grad_norm": 2.628239870071411,
      "learning_rate": 1.6465793165482838e-06,
      "logits/chosen": -3.098904609680176,
      "logits/rejected": -3.1030189990997314,
      "logps/chosen": -0.7733818888664246,
      "logps/rejected": -0.9600175619125366,
      "loss": 0.8352,
      "odds_ratio_loss": 0.6180769205093384,
      "rewards/accuracies": 0.612500011920929,
      "rewards/chosen": -0.07733818888664246,
      "rewards/margins": 0.018663574010133743,
      "rewards/rejected": -0.0960017591714859,
      "sft_loss": 0.7733818888664246,
      "step": 1030
    },
    {
      "epoch": 1.849299844409869,
      "grad_norm": 2.7811410427093506,
      "learning_rate": 1.6029421908964305e-06,
      "logits/chosen": -3.0989787578582764,
      "logits/rejected": -3.1128220558166504,
      "logps/chosen": -0.7662326693534851,
      "logps/rejected": -1.2116987705230713,
      "loss": 0.8252,
      "odds_ratio_loss": 0.5896168351173401,
      "rewards/accuracies": 0.612500011920929,
      "rewards/chosen": -0.07662326097488403,
      "rewards/margins": 0.044546615332365036,
      "rewards/rejected": -0.12116988748311996,
      "sft_loss": 0.7662326693534851,
      "step": 1040
    },
    {
      "epoch": 1.8670815736830408,
      "grad_norm": 2.588897466659546,
      "learning_rate": 1.559616555157985e-06,
      "logits/chosen": -3.1540348529815674,
      "logits/rejected": -3.1318535804748535,
      "logps/chosen": -0.8036566972732544,
      "logps/rejected": -0.9966574907302856,
      "loss": 0.8694,
      "odds_ratio_loss": 0.656964123249054,
      "rewards/accuracies": 0.543749988079071,
      "rewards/chosen": -0.08036566525697708,
      "rewards/margins": 0.019300078973174095,
      "rewards/rejected": -0.09966574609279633,
      "sft_loss": 0.8036566972732544,
      "step": 1050
    },
    {
      "epoch": 1.8848633029562125,
      "grad_norm": 3.195645332336426,
      "learning_rate": 1.516617453512252e-06,
      "logits/chosen": -3.133869171142578,
      "logits/rejected": -3.1599550247192383,
      "logps/chosen": -0.8567641377449036,
      "logps/rejected": -0.9691047668457031,
      "loss": 0.9289,
      "odds_ratio_loss": 0.7213839888572693,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.0856764167547226,
      "rewards/margins": 0.011234072968363762,
      "rewards/rejected": -0.0969104915857315,
      "sft_loss": 0.8567641377449036,
      "step": 1060
    },
    {
      "epoch": 1.9026450322293842,
      "grad_norm": 3.544257164001465,
      "learning_rate": 1.473959816754449e-06,
      "logits/chosen": -3.1071698665618896,
      "logits/rejected": -3.119621992111206,
      "logps/chosen": -0.8016077280044556,
      "logps/rejected": -0.9158931970596313,
      "loss": 0.8714,
      "odds_ratio_loss": 0.6980700492858887,
      "rewards/accuracies": 0.543749988079071,
      "rewards/chosen": -0.08016077429056168,
      "rewards/margins": 0.011428546160459518,
      "rewards/rejected": -0.09158932417631149,
      "sft_loss": 0.8016077280044556,
      "step": 1070
    },
    {
      "epoch": 1.920426761502556,
      "grad_norm": 2.2053537368774414,
      "learning_rate": 1.4316584571112213e-06,
      "logits/chosen": -3.1642978191375732,
      "logits/rejected": -3.1734910011291504,
      "logps/chosen": -0.8405769467353821,
      "logps/rejected": -0.9534690976142883,
      "loss": 0.9088,
      "odds_ratio_loss": 0.6824837327003479,
      "rewards/accuracies": 0.5375000238418579,
      "rewards/chosen": -0.08405770361423492,
      "rewards/margins": 0.011289209127426147,
      "rewards/rejected": -0.09534691274166107,
      "sft_loss": 0.8405769467353821,
      "step": 1080
    },
    {
      "epoch": 1.938208490775728,
      "grad_norm": 3.7732386589050293,
      "learning_rate": 1.389728063097306e-06,
      "logits/chosen": -3.134726047515869,
      "logits/rejected": -3.1553549766540527,
      "logps/chosen": -0.8409829139709473,
      "logps/rejected": -1.037058711051941,
      "loss": 0.9054,
      "odds_ratio_loss": 0.6446219682693481,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": -0.08409829437732697,
      "rewards/margins": 0.019607581198215485,
      "rewards/rejected": -0.10370586812496185,
      "sft_loss": 0.8409829139709473,
      "step": 1090
    },
    {
      "epoch": 1.9559902200488999,
      "grad_norm": 1.9941437244415283,
      "learning_rate": 1.348183194415179e-06,
      "logits/chosen": -3.12330961227417,
      "logits/rejected": -3.0894432067871094,
      "logps/chosen": -0.8183244466781616,
      "logps/rejected": -1.0717554092407227,
      "loss": 0.8794,
      "odds_ratio_loss": 0.6103757619857788,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.08183245360851288,
      "rewards/margins": 0.025343094021081924,
      "rewards/rejected": -0.1071755513548851,
      "sft_loss": 0.8183244466781616,
      "step": 1100
    },
    {
      "epoch": 1.9737719493220716,
      "grad_norm": 2.1723690032958984,
      "learning_rate": 1.3070382768994015e-06,
      "logits/chosen": -3.1375839710235596,
      "logits/rejected": -3.1476972103118896,
      "logps/chosen": -0.8107814788818359,
      "logps/rejected": -0.9439038038253784,
      "loss": 0.8756,
      "odds_ratio_loss": 0.6477454900741577,
      "rewards/accuracies": 0.512499988079071,
      "rewards/chosen": -0.08107815682888031,
      "rewards/margins": 0.01331222616136074,
      "rewards/rejected": -0.0943903774023056,
      "sft_loss": 0.8107814788818359,
      "step": 1110
    },
    {
      "epoch": 1.9915536785952432,
      "grad_norm": 2.5445873737335205,
      "learning_rate": 1.2663075975074746e-06,
      "logits/chosen": -3.1265861988067627,
      "logits/rejected": -3.1423563957214355,
      "logps/chosen": -0.79461270570755,
      "logps/rejected": -1.0579864978790283,
      "loss": 0.8606,
      "odds_ratio_loss": 0.6598424911499023,
      "rewards/accuracies": 0.59375,
      "rewards/chosen": -0.07946126163005829,
      "rewards/margins": 0.026337390765547752,
      "rewards/rejected": -0.10579866170883179,
      "sft_loss": 0.79461270570755,
      "step": 1120
    },
    {
      "epoch": 2.009335407868415,
      "grad_norm": 5.374589920043945,
      "learning_rate": 1.2260052993589034e-06,
      "logits/chosen": -3.117713212966919,
      "logits/rejected": -3.1392993927001953,
      "logps/chosen": -0.9073926210403442,
      "logps/rejected": -0.9984840154647827,
      "loss": 0.9829,
      "odds_ratio_loss": 0.7546229362487793,
      "rewards/accuracies": 0.46875,
      "rewards/chosen": -0.09073926508426666,
      "rewards/margins": 0.009109143167734146,
      "rewards/rejected": -0.09984840452671051,
      "sft_loss": 0.9073926210403442,
      "step": 1130
    },
    {
      "epoch": 2.027117137141587,
      "grad_norm": 1.554049015045166,
      "learning_rate": 1.1861453768242099e-06,
      "logits/chosen": -3.1674160957336426,
      "logits/rejected": -3.16347599029541,
      "logps/chosen": -0.730399489402771,
      "logps/rejected": -0.9565631747245789,
      "loss": 0.7897,
      "odds_ratio_loss": 0.5933364629745483,
      "rewards/accuracies": 0.65625,
      "rewards/chosen": -0.0730399638414383,
      "rewards/margins": 0.02261636219918728,
      "rewards/rejected": -0.09565632045269012,
      "sft_loss": 0.730399489402771,
      "step": 1140
    },
    {
      "epoch": 2.044898866414759,
      "grad_norm": 10.319910049438477,
      "learning_rate": 1.1467416706655982e-06,
      "logits/chosen": -3.141704559326172,
      "logits/rejected": -3.173985719680786,
      "logps/chosen": -0.8747559785842896,
      "logps/rejected": -1.045388102531433,
      "loss": 0.9448,
      "odds_ratio_loss": 0.7000676989555359,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.08747559785842896,
      "rewards/margins": 0.017063220962882042,
      "rewards/rejected": -0.10453881323337555,
      "sft_loss": 0.8747559785842896,
      "step": 1150
    },
    {
      "epoch": 2.062680595687931,
      "grad_norm": 2.64601993560791,
      "learning_rate": 1.1078078632309559e-06,
      "logits/chosen": -3.1251769065856934,
      "logits/rejected": -3.154083251953125,
      "logps/chosen": -0.7768465280532837,
      "logps/rejected": -0.9674509167671204,
      "loss": 0.8405,
      "odds_ratio_loss": 0.6370204091072083,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -0.0776846632361412,
      "rewards/margins": 0.01906043104827404,
      "rewards/rejected": -0.0967450961470604,
      "sft_loss": 0.7768465280532837,
      "step": 1160
    },
    {
      "epoch": 2.0804623249611023,
      "grad_norm": 8.88864517211914,
      "learning_rate": 1.0693574737028627e-06,
      "logits/chosen": -3.1327998638153076,
      "logits/rejected": -3.1513829231262207,
      "logps/chosen": -0.811104416847229,
      "logps/rejected": -0.959033191204071,
      "loss": 0.8792,
      "odds_ratio_loss": 0.6805382966995239,
      "rewards/accuracies": 0.543749988079071,
      "rewards/chosen": -0.08111042529344559,
      "rewards/margins": 0.01479288749396801,
      "rewards/rejected": -0.09590331465005875,
      "sft_loss": 0.811104416847229,
      "step": 1170
    },
    {
      "epoch": 2.098244054234274,
      "grad_norm": 2.3200793266296387,
      "learning_rate": 1.0314038534042586e-06,
      "logits/chosen": -3.154930830001831,
      "logits/rejected": -3.1501238346099854,
      "logps/chosen": -0.7636488676071167,
      "logps/rejected": -0.9957521557807922,
      "loss": 0.8285,
      "odds_ratio_loss": 0.6481651067733765,
      "rewards/accuracies": 0.5375000238418579,
      "rewards/chosen": -0.07636488974094391,
      "rewards/margins": 0.023210326209664345,
      "rewards/rejected": -0.0995752140879631,
      "sft_loss": 0.7636488676071167,
      "step": 1180
    },
    {
      "epoch": 2.116025783507446,
      "grad_norm": 3.470479965209961,
      "learning_rate": 9.939601811623946e-07,
      "logits/chosen": -3.1409804821014404,
      "logits/rejected": -3.14042329788208,
      "logps/chosen": -0.813196063041687,
      "logps/rejected": -0.9785780906677246,
      "loss": 0.8813,
      "odds_ratio_loss": 0.6813095808029175,
      "rewards/accuracies": 0.5375000238418579,
      "rewards/chosen": -0.08131961524486542,
      "rewards/margins": 0.01653820462524891,
      "rewards/rejected": -0.09785781800746918,
      "sft_loss": 0.813196063041687,
      "step": 1190
    },
    {
      "epoch": 2.133807512780618,
      "grad_norm": 2.298424005508423,
      "learning_rate": 9.570394587326825e-07,
      "logits/chosen": -3.1406095027923584,
      "logits/rejected": -3.138267993927002,
      "logps/chosen": -0.7988274693489075,
      "logps/rejected": -1.0399543046951294,
      "loss": 0.8608,
      "odds_ratio_loss": 0.6200910806655884,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -0.07988274842500687,
      "rewards/margins": 0.024112680926918983,
      "rewards/rejected": -0.1039954274892807,
      "sft_loss": 0.7988274693489075,
      "step": 1200
    },
    {
      "epoch": 2.15158924205379,
      "grad_norm": 1.9331620931625366,
      "learning_rate": 9.206545062840302e-07,
      "logits/chosen": -3.181776285171509,
      "logits/rejected": -3.1430013179779053,
      "logps/chosen": -0.7699235081672668,
      "logps/rejected": -1.0029253959655762,
      "loss": 0.831,
      "odds_ratio_loss": 0.6103402376174927,
      "rewards/accuracies": 0.581250011920929,
      "rewards/chosen": -0.07699234783649445,
      "rewards/margins": 0.02330019325017929,
      "rewards/rejected": -0.10029254108667374,
      "sft_loss": 0.7699235081672668,
      "step": 1210
    },
    {
      "epoch": 2.1693709713269618,
      "grad_norm": 1.9117600917816162,
      "learning_rate": 8.848179579472285e-07,
      "logits/chosen": -3.16937518119812,
      "logits/rejected": -3.171738862991333,
      "logps/chosen": -0.7665938138961792,
      "logps/rejected": -0.8684927821159363,
      "loss": 0.8349,
      "odds_ratio_loss": 0.6835185885429382,
      "rewards/accuracies": 0.4937500059604645,
      "rewards/chosen": -0.07665937393903732,
      "rewards/margins": 0.010189898312091827,
      "rewards/rejected": -0.08684928715229034,
      "sft_loss": 0.7665938138961792,
      "step": 1220
    },
    {
      "epoch": 2.1871527006001332,
      "grad_norm": 5.053982734680176,
      "learning_rate": 8.495422574279403e-07,
      "logits/chosen": -3.1011910438537598,
      "logits/rejected": -3.092721462249756,
      "logps/chosen": -0.7057160139083862,
      "logps/rejected": -0.9676464796066284,
      "loss": 0.7646,
      "odds_ratio_loss": 0.5885148048400879,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.07057160139083862,
      "rewards/margins": 0.026193052530288696,
      "rewards/rejected": -0.09676466137170792,
      "sft_loss": 0.7057160139083862,
      "step": 1230
    },
    {
      "epoch": 2.204934429873305,
      "grad_norm": 2.2379298210144043,
      "learning_rate": 8.148396536858063e-07,
      "logits/chosen": -3.1442742347717285,
      "logits/rejected": -3.1458396911621094,
      "logps/chosen": -0.8305691480636597,
      "logps/rejected": -1.0573723316192627,
      "loss": 0.8959,
      "odds_ratio_loss": 0.6531893610954285,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": -0.08305691182613373,
      "rewards/margins": 0.02268032357096672,
      "rewards/rejected": -0.10573724657297134,
      "sft_loss": 0.8305691480636597,
      "step": 1240
    },
    {
      "epoch": 2.222716159146477,
      "grad_norm": 2.2036707401275635,
      "learning_rate": 7.807221966811815e-07,
      "logits/chosen": -3.1296424865722656,
      "logits/rejected": -3.142879009246826,
      "logps/chosen": -0.815384566783905,
      "logps/rejected": -0.9788722991943359,
      "loss": 0.8822,
      "odds_ratio_loss": 0.6686090230941772,
      "rewards/accuracies": 0.5687500238418579,
      "rewards/chosen": -0.08153846114873886,
      "rewards/margins": 0.01634877361357212,
      "rewards/rejected": -0.09788723289966583,
      "sft_loss": 0.815384566783905,
      "step": 1250
    },
    {
      "epoch": 2.240497888419649,
      "grad_norm": 1.7958269119262695,
      "learning_rate": 7.47201733190962e-07,
      "logits/chosen": -3.1007513999938965,
      "logits/rejected": -3.1123993396759033,
      "logps/chosen": -0.7537363767623901,
      "logps/rejected": -0.9363679885864258,
      "loss": 0.8162,
      "odds_ratio_loss": 0.6245176792144775,
      "rewards/accuracies": 0.5062500238418579,
      "rewards/chosen": -0.07537363469600677,
      "rewards/margins": 0.018263157457113266,
      "rewards/rejected": -0.09363678842782974,
      "sft_loss": 0.7537363767623901,
      "step": 1260
    },
    {
      "epoch": 2.258279617692821,
      "grad_norm": 2.2002153396606445,
      "learning_rate": 7.142899026949721e-07,
      "logits/chosen": -3.181532382965088,
      "logits/rejected": -3.189258098602295,
      "logps/chosen": -0.7867833971977234,
      "logps/rejected": -0.9312666654586792,
      "loss": 0.8504,
      "odds_ratio_loss": 0.6360999941825867,
      "rewards/accuracies": 0.59375,
      "rewards/chosen": -0.07867833971977234,
      "rewards/margins": 0.01444832980632782,
      "rewards/rejected": -0.09312666952610016,
      "sft_loss": 0.7867833971977234,
      "step": 1270
    },
    {
      "epoch": 2.2760613469659923,
      "grad_norm": 5.216893672943115,
      "learning_rate": 6.819981333343273e-07,
      "logits/chosen": -3.0660347938537598,
      "logits/rejected": -3.095858097076416,
      "logps/chosen": -0.7724840641021729,
      "logps/rejected": -0.9914291501045227,
      "loss": 0.8347,
      "odds_ratio_loss": 0.6223303079605103,
      "rewards/accuracies": 0.581250011920929,
      "rewards/chosen": -0.07724839448928833,
      "rewards/margins": 0.021894508972764015,
      "rewards/rejected": -0.09914292395114899,
      "sft_loss": 0.7724840641021729,
      "step": 1280
    },
    {
      "epoch": 2.293843076239164,
      "grad_norm": 2.3061211109161377,
      "learning_rate": 6.503376379431839e-07,
      "logits/chosen": -3.1206648349761963,
      "logits/rejected": -3.1211998462677,
      "logps/chosen": -0.8609710931777954,
      "logps/rejected": -0.918415904045105,
      "loss": 0.9299,
      "odds_ratio_loss": 0.6894447803497314,
      "rewards/accuracies": 0.5375000238418579,
      "rewards/chosen": -0.0860971063375473,
      "rewards/margins": 0.005744467489421368,
      "rewards/rejected": -0.09184158593416214,
      "sft_loss": 0.8609710931777954,
      "step": 1290
    },
    {
      "epoch": 2.311624805512336,
      "grad_norm": 1.7814314365386963,
      "learning_rate": 6.193194101552502e-07,
      "logits/chosen": -3.126063346862793,
      "logits/rejected": -3.1108012199401855,
      "logps/chosen": -0.7555452585220337,
      "logps/rejected": -1.0133960247039795,
      "loss": 0.8139,
      "odds_ratio_loss": 0.5832154154777527,
      "rewards/accuracies": 0.6312500238418579,
      "rewards/chosen": -0.07555452734231949,
      "rewards/margins": 0.025785094127058983,
      "rewards/rejected": -0.10133961588144302,
      "sft_loss": 0.7555452585220337,
      "step": 1300
    },
    {
      "epoch": 2.329406534785508,
      "grad_norm": 3.5369389057159424,
      "learning_rate": 5.889542205864083e-07,
      "logits/chosen": -3.1118927001953125,
      "logits/rejected": -3.1322388648986816,
      "logps/chosen": -0.809761643409729,
      "logps/rejected": -1.0040373802185059,
      "loss": 0.873,
      "odds_ratio_loss": 0.6319769620895386,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -0.08097617328166962,
      "rewards/margins": 0.019427578896284103,
      "rewards/rejected": -0.10040374100208282,
      "sft_loss": 0.809761643409729,
      "step": 1310
    },
    {
      "epoch": 2.34718826405868,
      "grad_norm": 1.8906471729278564,
      "learning_rate": 5.592526130947862e-07,
      "logits/chosen": -3.094481945037842,
      "logits/rejected": -3.1228842735290527,
      "logps/chosen": -0.8294750452041626,
      "logps/rejected": -1.0103859901428223,
      "loss": 0.8988,
      "odds_ratio_loss": 0.6931812763214111,
      "rewards/accuracies": 0.48750001192092896,
      "rewards/chosen": -0.0829475075006485,
      "rewards/margins": 0.018091093748807907,
      "rewards/rejected": -0.10103859752416611,
      "sft_loss": 0.8294750452041626,
      "step": 1320
    },
    {
      "epoch": 2.3649699933318518,
      "grad_norm": 2.300255298614502,
      "learning_rate": 5.302249011195507e-07,
      "logits/chosen": -3.091862916946411,
      "logits/rejected": -3.1117231845855713,
      "logps/chosen": -0.7831630110740662,
      "logps/rejected": -0.9011325836181641,
      "loss": 0.8506,
      "odds_ratio_loss": 0.6743569374084473,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -0.07831630855798721,
      "rewards/margins": 0.01179695688188076,
      "rewards/rejected": -0.09011325985193253,
      "sft_loss": 0.7831630110740662,
      "step": 1330
    },
    {
      "epoch": 2.382751722605023,
      "grad_norm": 2.0519402027130127,
      "learning_rate": 5.018811640997307e-07,
      "logits/chosen": -3.1082584857940674,
      "logits/rejected": -3.143366813659668,
      "logps/chosen": -0.8331505656242371,
      "logps/rejected": -1.1331783533096313,
      "loss": 0.8955,
      "odds_ratio_loss": 0.6230587363243103,
      "rewards/accuracies": 0.612500011920929,
      "rewards/chosen": -0.08331505209207535,
      "rewards/margins": 0.030002791434526443,
      "rewards/rejected": -0.1133178323507309,
      "sft_loss": 0.8331505656242371,
      "step": 1340
    },
    {
      "epoch": 2.400533451878195,
      "grad_norm": 2.004222869873047,
      "learning_rate": 4.7423124397427105e-07,
      "logits/chosen": -3.0787928104400635,
      "logits/rejected": -3.1223533153533936,
      "logps/chosen": -0.8188081979751587,
      "logps/rejected": -0.9587352871894836,
      "loss": 0.8855,
      "odds_ratio_loss": 0.6670054197311401,
      "rewards/accuracies": 0.543749988079071,
      "rewards/chosen": -0.08188082277774811,
      "rewards/margins": 0.013992704451084137,
      "rewards/rejected": -0.09587351977825165,
      "sft_loss": 0.8188081979751587,
      "step": 1350
    },
    {
      "epoch": 2.418315181151367,
      "grad_norm": 2.007474422454834,
      "learning_rate": 4.472847417645787e-07,
      "logits/chosen": -3.1503520011901855,
      "logits/rejected": -3.1351065635681152,
      "logps/chosen": -0.8101593255996704,
      "logps/rejected": -1.086388349533081,
      "loss": 0.8703,
      "odds_ratio_loss": 0.6015632748603821,
      "rewards/accuracies": 0.6187499761581421,
      "rewards/chosen": -0.0810159370303154,
      "rewards/margins": 0.027622899040579796,
      "rewards/rejected": -0.10863884538412094,
      "sft_loss": 0.8101593255996704,
      "step": 1360
    },
    {
      "epoch": 2.436096910424539,
      "grad_norm": 1.4029066562652588,
      "learning_rate": 4.210510142406993e-07,
      "logits/chosen": -3.122816562652588,
      "logits/rejected": -3.095937490463257,
      "logps/chosen": -0.7846102714538574,
      "logps/rejected": -1.0122342109680176,
      "loss": 0.8472,
      "odds_ratio_loss": 0.626270055770874,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": -0.07846103608608246,
      "rewards/margins": 0.022762387990951538,
      "rewards/rejected": -0.101223424077034,
      "sft_loss": 0.7846102714538574,
      "step": 1370
    },
    {
      "epoch": 2.4538786396977104,
      "grad_norm": 1.7324745655059814,
      "learning_rate": 3.9553917067232966e-07,
      "logits/chosen": -3.122037172317505,
      "logits/rejected": -3.1394925117492676,
      "logps/chosen": -0.8041805028915405,
      "logps/rejected": -0.9916130900382996,
      "loss": 0.87,
      "odds_ratio_loss": 0.658187210559845,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -0.08041805773973465,
      "rewards/margins": 0.018743254244327545,
      "rewards/rejected": -0.0991613045334816,
      "sft_loss": 0.8041805028915405,
      "step": 1380
    },
    {
      "epoch": 2.4716603689708823,
      "grad_norm": 2.2863593101501465,
      "learning_rate": 3.707580696657509e-07,
      "logits/chosen": -3.118274450302124,
      "logits/rejected": -3.109182834625244,
      "logps/chosen": -0.7898752689361572,
      "logps/rejected": -0.945044219493866,
      "loss": 0.8546,
      "odds_ratio_loss": 0.6472212672233582,
      "rewards/accuracies": 0.5562499761581421,
      "rewards/chosen": -0.07898753136396408,
      "rewards/margins": 0.015516892075538635,
      "rewards/rejected": -0.09450441598892212,
      "sft_loss": 0.7898752689361572,
      "step": 1390
    },
    {
      "epoch": 2.489442098244054,
      "grad_norm": 2.1385016441345215,
      "learning_rate": 3.4671631608781815e-07,
      "logits/chosen": -3.125810384750366,
      "logits/rejected": -3.1365230083465576,
      "logps/chosen": -0.8170459866523743,
      "logps/rejected": -1.0128613710403442,
      "loss": 0.8858,
      "odds_ratio_loss": 0.6880038380622864,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -0.08170458674430847,
      "rewards/margins": 0.019581545144319534,
      "rewards/rejected": -0.1012861356139183,
      "sft_loss": 0.8170459866523743,
      "step": 1400
    },
    {
      "epoch": 2.507223827517226,
      "grad_norm": 2.561035633087158,
      "learning_rate": 3.234222580780405e-07,
      "logits/chosen": -3.1027114391326904,
      "logits/rejected": -3.124307632446289,
      "logps/chosen": -0.7941089868545532,
      "logps/rejected": -0.9503856897354126,
      "loss": 0.8579,
      "odds_ratio_loss": 0.6381289958953857,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -0.07941089570522308,
      "rewards/margins": 0.015627671033143997,
      "rewards/rejected": -0.09503857046365738,
      "sft_loss": 0.7941089868545532,
      "step": 1410
    },
    {
      "epoch": 2.525005556790398,
      "grad_norm": 2.1497950553894043,
      "learning_rate": 3.0088398414982375e-07,
      "logits/chosen": -3.0808122158050537,
      "logits/rejected": -3.118448257446289,
      "logps/chosen": -0.8251555562019348,
      "logps/rejected": -1.0561994314193726,
      "loss": 0.8918,
      "odds_ratio_loss": 0.6662226319313049,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": -0.08251555263996124,
      "rewards/margins": 0.023104388266801834,
      "rewards/rejected": -0.10561992973089218,
      "sft_loss": 0.8251555562019348,
      "step": 1420
    },
    {
      "epoch": 2.54278728606357,
      "grad_norm": 2.1733312606811523,
      "learning_rate": 2.7910932038184487e-07,
      "logits/chosen": -3.0798656940460205,
      "logits/rejected": -3.0586531162261963,
      "logps/chosen": -0.8029570579528809,
      "logps/rejected": -0.9888774156570435,
      "loss": 0.8665,
      "odds_ratio_loss": 0.6356409192085266,
      "rewards/accuracies": 0.6312500238418579,
      "rewards/chosen": -0.08029570430517197,
      "rewards/margins": 0.018592040985822678,
      "rewards/rejected": -0.09888775646686554,
      "sft_loss": 0.8029570579528809,
      "step": 1430
    },
    {
      "epoch": 2.5605690153367417,
      "grad_norm": 2.0504164695739746,
      "learning_rate": 2.5810582770057325e-07,
      "logits/chosen": -3.1239726543426514,
      "logits/rejected": -3.1643452644348145,
      "logps/chosen": -0.7773226499557495,
      "logps/rejected": -0.9956240653991699,
      "loss": 0.8421,
      "odds_ratio_loss": 0.6477575302124023,
      "rewards/accuracies": 0.5687500238418579,
      "rewards/chosen": -0.07773226499557495,
      "rewards/margins": 0.02183014526963234,
      "rewards/rejected": -0.09956242144107819,
      "sft_loss": 0.7773226499557495,
      "step": 1440
    },
    {
      "epoch": 2.578350744609913,
      "grad_norm": 2.4383292198181152,
      "learning_rate": 2.3788079925484402e-07,
      "logits/chosen": -3.1351797580718994,
      "logits/rejected": -3.1292059421539307,
      "logps/chosen": -0.8360335230827332,
      "logps/rejected": -0.9335094690322876,
      "loss": 0.9052,
      "odds_ratio_loss": 0.6920183300971985,
      "rewards/accuracies": 0.5249999761581421,
      "rewards/chosen": -0.08360335975885391,
      "rewards/margins": 0.009747589938342571,
      "rewards/rejected": -0.09335094690322876,
      "sft_loss": 0.8360335230827332,
      "step": 1450
    },
    {
      "epoch": 2.596132473883085,
      "grad_norm": 2.4566612243652344,
      "learning_rate": 2.1844125788342661e-07,
      "logits/chosen": -3.108156681060791,
      "logits/rejected": -3.1151247024536133,
      "logps/chosen": -0.7554203867912292,
      "logps/rejected": -1.1023544073104858,
      "loss": 0.8163,
      "odds_ratio_loss": 0.6091145277023315,
      "rewards/accuracies": 0.612500011920929,
      "rewards/chosen": -0.07554203271865845,
      "rewards/margins": 0.034693412482738495,
      "rewards/rejected": -0.11023545265197754,
      "sft_loss": 0.7554203867912292,
      "step": 1460
    },
    {
      "epoch": 2.613914203156257,
      "grad_norm": 2.3035502433776855,
      "learning_rate": 1.9979395367644428e-07,
      "logits/chosen": -3.143681049346924,
      "logits/rejected": -3.1587703227996826,
      "logps/chosen": -0.7682673931121826,
      "logps/rejected": -0.9972553253173828,
      "loss": 0.8278,
      "odds_ratio_loss": 0.5949780344963074,
      "rewards/accuracies": 0.59375,
      "rewards/chosen": -0.07682673633098602,
      "rewards/margins": 0.02289879135787487,
      "rewards/rejected": -0.09972552955150604,
      "sft_loss": 0.7682673931121826,
      "step": 1470
    },
    {
      "epoch": 2.631695932429429,
      "grad_norm": 3.8865010738372803,
      "learning_rate": 1.81945361631512e-07,
      "logits/chosen": -3.1585988998413086,
      "logits/rejected": -3.178792715072632,
      "logps/chosen": -0.8142994046211243,
      "logps/rejected": -0.951032817363739,
      "loss": 0.8828,
      "odds_ratio_loss": 0.6848469972610474,
      "rewards/accuracies": 0.5062500238418579,
      "rewards/chosen": -0.08142994344234467,
      "rewards/margins": 0.013673332519829273,
      "rewards/rejected": -0.09510327875614166,
      "sft_loss": 0.8142994046211243,
      "step": 1480
    },
    {
      "epoch": 2.6494776617026004,
      "grad_norm": 3.1666336059570312,
      "learning_rate": 1.6490167940538343e-07,
      "logits/chosen": -3.154137372970581,
      "logits/rejected": -3.1491308212280273,
      "logps/chosen": -0.7683095932006836,
      "logps/rejected": -1.0064373016357422,
      "loss": 0.8283,
      "odds_ratio_loss": 0.6001058220863342,
      "rewards/accuracies": 0.581250011920929,
      "rewards/chosen": -0.07683096826076508,
      "rewards/margins": 0.023812763392925262,
      "rewards/rejected": -0.10064373165369034,
      "sft_loss": 0.7683095932006836,
      "step": 1490
    },
    {
      "epoch": 2.6672593909757722,
      "grad_norm": 9.307645797729492,
      "learning_rate": 1.4866882516191339e-07,
      "logits/chosen": -3.0799524784088135,
      "logits/rejected": -3.1244568824768066,
      "logps/chosen": -0.8257862329483032,
      "logps/rejected": -1.011817216873169,
      "loss": 0.8923,
      "odds_ratio_loss": 0.6649435758590698,
      "rewards/accuracies": 0.5375000238418579,
      "rewards/chosen": -0.0825786143541336,
      "rewards/margins": 0.018603112548589706,
      "rewards/rejected": -0.10118173062801361,
      "sft_loss": 0.8257862329483032,
      "step": 1500
    },
    {
      "epoch": 2.6672593909757722,
      "eval_logits/chosen": -3.1171207427978516,
      "eval_logits/rejected": -3.145516872406006,
      "eval_logps/chosen": -0.8098240494728088,
      "eval_logps/rejected": -1.0174543857574463,
      "eval_loss": 0.8734214901924133,
      "eval_odds_ratio_loss": 0.6359757781028748,
      "eval_rewards/accuracies": 0.5720000267028809,
      "eval_rewards/chosen": -0.08098240196704865,
      "eval_rewards/margins": 0.02076304331421852,
      "eval_rewards/rejected": -0.10174543410539627,
      "eval_runtime": 237.6146,
      "eval_samples_per_second": 4.208,
      "eval_sft_loss": 0.8098240494728088,
      "eval_steps_per_second": 2.104,
      "step": 1500
    },
    {
      "epoch": 2.685041120248944,
      "grad_norm": 4.906961441040039,
      "learning_rate": 1.3325243551706057e-07,
      "logits/chosen": -3.0958564281463623,
      "logits/rejected": -3.1364972591400146,
      "logps/chosen": -0.7746607065200806,
      "logps/rejected": -1.0890486240386963,
      "loss": 0.834,
      "odds_ratio_loss": 0.5929327607154846,
      "rewards/accuracies": 0.612500011920929,
      "rewards/chosen": -0.07746607810258865,
      "rewards/margins": 0.03143879026174545,
      "rewards/rejected": -0.10890486091375351,
      "sft_loss": 0.7746607065200806,
      "step": 1510
    },
    {
      "epoch": 2.702822849522116,
      "grad_norm": 8.813859939575195,
      "learning_rate": 1.1865786358165737e-07,
      "logits/chosen": -3.1081910133361816,
      "logits/rejected": -3.160123586654663,
      "logps/chosen": -0.7778806686401367,
      "logps/rejected": -0.9513812065124512,
      "loss": 0.842,
      "odds_ratio_loss": 0.6413436532020569,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": -0.07778806984424591,
      "rewards/margins": 0.017350060865283012,
      "rewards/rejected": -0.09513812512159348,
      "sft_loss": 0.7778806686401367,
      "step": 1520
    },
    {
      "epoch": 2.720604578795288,
      "grad_norm": 5.624754428863525,
      "learning_rate": 1.0489017710262311e-07,
      "logits/chosen": -3.1469523906707764,
      "logits/rejected": -3.178358554840088,
      "logps/chosen": -0.8407548666000366,
      "logps/rejected": -1.1098471879959106,
      "loss": 0.9062,
      "odds_ratio_loss": 0.6548250317573547,
      "rewards/accuracies": 0.6312500238418579,
      "rewards/chosen": -0.08407549560070038,
      "rewards/margins": 0.026909226551651955,
      "rewards/rejected": -0.11098472028970718,
      "sft_loss": 0.8407548666000366,
      "step": 1530
    },
    {
      "epoch": 2.73838630806846,
      "grad_norm": 3.942481756210327,
      "learning_rate": 9.195415670326446e-08,
      "logits/chosen": -3.13153076171875,
      "logits/rejected": -3.1526730060577393,
      "logps/chosen": -0.8119581341743469,
      "logps/rejected": -1.0097941160202026,
      "loss": 0.8766,
      "odds_ratio_loss": 0.6465227007865906,
      "rewards/accuracies": 0.5375000238418579,
      "rewards/chosen": -0.08119582384824753,
      "rewards/margins": 0.019783606752753258,
      "rewards/rejected": -0.10097942501306534,
      "sft_loss": 0.8119581341743469,
      "step": 1540
    },
    {
      "epoch": 2.7561680373416317,
      "grad_norm": 3.0953104496002197,
      "learning_rate": 7.985429422327384e-08,
      "logits/chosen": -3.1453542709350586,
      "logits/rejected": -3.165792226791382,
      "logps/chosen": -0.8054162263870239,
      "logps/rejected": -0.9290239214897156,
      "loss": 0.875,
      "odds_ratio_loss": 0.6959558129310608,
      "rewards/accuracies": 0.48750001192092896,
      "rewards/chosen": -0.08054163306951523,
      "rewards/margins": 0.01236076932400465,
      "rewards/rejected": -0.09290239959955215,
      "sft_loss": 0.8054162263870239,
      "step": 1550
    },
    {
      "epoch": 2.773949766614803,
      "grad_norm": 1.9901708364486694,
      "learning_rate": 6.859479115900818e-08,
      "logits/chosen": -3.118248224258423,
      "logits/rejected": -3.158768892288208,
      "logps/chosen": -0.7784756422042847,
      "logps/rejected": -0.9923986196517944,
      "loss": 0.8408,
      "odds_ratio_loss": 0.6230874061584473,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -0.07784756273031235,
      "rewards/margins": 0.021392302587628365,
      "rewards/rejected": -0.09923987090587616,
      "sft_loss": 0.7784756422042847,
      "step": 1560
    },
    {
      "epoch": 2.791731495887975,
      "grad_norm": 10.119742393493652,
      "learning_rate": 5.817955720457902e-08,
      "logits/chosen": -3.107785701751709,
      "logits/rejected": -3.1253585815429688,
      "logps/chosen": -0.8034731149673462,
      "logps/rejected": -0.9698166847229004,
      "loss": 0.8686,
      "odds_ratio_loss": 0.6507803201675415,
      "rewards/accuracies": 0.5874999761581421,
      "rewards/chosen": -0.08034731447696686,
      "rewards/margins": 0.016634367406368256,
      "rewards/rejected": -0.09698168933391571,
      "sft_loss": 0.8034731149673462,
      "step": 1570
    },
    {
      "epoch": 2.809513225161147,
      "grad_norm": 3.319011926651001,
      "learning_rate": 4.861220889427199e-08,
      "logits/chosen": -3.1124069690704346,
      "logits/rejected": -3.131798505783081,
      "logps/chosen": -0.8197135925292969,
      "logps/rejected": -0.9885567426681519,
      "loss": 0.887,
      "odds_ratio_loss": 0.6726602911949158,
      "rewards/accuracies": 0.606249988079071,
      "rewards/chosen": -0.08197136968374252,
      "rewards/margins": 0.01688431203365326,
      "rewards/rejected": -0.09885567426681519,
      "sft_loss": 0.8197135925292969,
      "step": 1580
    },
    {
      "epoch": 2.827294954434319,
      "grad_norm": 2.032493829727173,
      "learning_rate": 3.9896068346758074e-08,
      "logits/chosen": -3.134514331817627,
      "logits/rejected": -3.1544039249420166,
      "logps/chosen": -0.8260439038276672,
      "logps/rejected": -0.9724828004837036,
      "loss": 0.8923,
      "odds_ratio_loss": 0.6625251770019531,
      "rewards/accuracies": 0.59375,
      "rewards/chosen": -0.08260440081357956,
      "rewards/margins": 0.01464388333261013,
      "rewards/rejected": -0.09724827855825424,
      "sft_loss": 0.8260439038276672,
      "step": 1590
    },
    {
      "epoch": 2.8450766837074903,
      "grad_norm": 4.936295986175537,
      "learning_rate": 3.203416211153832e-08,
      "logits/chosen": -3.1045830249786377,
      "logits/rejected": -3.163196086883545,
      "logps/chosen": -0.8155530691146851,
      "logps/rejected": -0.9749042391777039,
      "loss": 0.8845,
      "odds_ratio_loss": 0.6895264983177185,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.0815553218126297,
      "rewards/margins": 0.015935102477669716,
      "rewards/rejected": -0.09749042987823486,
      "sft_loss": 0.8155530691146851,
      "step": 1600
    },
    {
      "epoch": 2.8628584129806622,
      "grad_norm": 3.0522594451904297,
      "learning_rate": 2.5029220118019393e-08,
      "logits/chosen": -3.0816335678100586,
      "logits/rejected": -3.120738983154297,
      "logps/chosen": -0.8227775692939758,
      "logps/rejected": -0.9608209729194641,
      "loss": 0.8883,
      "odds_ratio_loss": 0.6554335355758667,
      "rewards/accuracies": 0.5562499761581421,
      "rewards/chosen": -0.08227775990962982,
      "rewards/margins": 0.013804334215819836,
      "rewards/rejected": -0.09608209133148193,
      "sft_loss": 0.8227775692939758,
      "step": 1610
    },
    {
      "epoch": 2.880640142253834,
      "grad_norm": 6.9008331298828125,
      "learning_rate": 1.8883674727586122e-08,
      "logits/chosen": -3.120664119720459,
      "logits/rejected": -3.1391146183013916,
      "logps/chosen": -0.7796843647956848,
      "logps/rejected": -1.048107385635376,
      "loss": 0.8417,
      "odds_ratio_loss": 0.620618999004364,
      "rewards/accuracies": 0.643750011920929,
      "rewards/chosen": -0.07796843349933624,
      "rewards/margins": 0.026842307299375534,
      "rewards/rejected": -0.10481073707342148,
      "sft_loss": 0.7796843647956848,
      "step": 1620
    },
    {
      "epoch": 2.898421871527006,
      "grad_norm": 1.4706188440322876,
      "learning_rate": 1.3599659889000639e-08,
      "logits/chosen": -3.1607601642608643,
      "logits/rejected": -3.1758437156677246,
      "logps/chosen": -0.7644230723381042,
      "logps/rejected": -0.8909217715263367,
      "loss": 0.8322,
      "odds_ratio_loss": 0.6777127981185913,
      "rewards/accuracies": 0.5062500238418579,
      "rewards/chosen": -0.07644230127334595,
      "rewards/margins": 0.01264987699687481,
      "rewards/rejected": -0.0890921801328659,
      "sft_loss": 0.7644230723381042,
      "step": 1630
    },
    {
      "epoch": 2.916203600800178,
      "grad_norm": 1.4733339548110962,
      "learning_rate": 9.179010397421528e-09,
      "logits/chosen": -3.1298046112060547,
      "logits/rejected": -3.1558828353881836,
      "logps/chosen": -0.7814117670059204,
      "logps/rejected": -0.9674129486083984,
      "loss": 0.8469,
      "odds_ratio_loss": 0.6549249887466431,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.07814116775989532,
      "rewards/margins": 0.01860012486577034,
      "rewards/rejected": -0.09674130380153656,
      "sft_loss": 0.7814117670059204,
      "step": 1640
    },
    {
      "epoch": 2.93398533007335,
      "grad_norm": 1.6659821271896362,
      "learning_rate": 5.623261257296509e-09,
      "logits/chosen": -3.100876569747925,
      "logits/rejected": -3.1546549797058105,
      "logps/chosen": -0.7405164241790771,
      "logps/rejected": -0.9196218252182007,
      "loss": 0.8037,
      "odds_ratio_loss": 0.6319458484649658,
      "rewards/accuracies": 0.543749988079071,
      "rewards/chosen": -0.0740516409277916,
      "rewards/margins": 0.017910538241267204,
      "rewards/rejected": -0.09196218103170395,
      "sft_loss": 0.7405164241790771,
      "step": 1650
    },
    {
      "epoch": 2.9517670593465217,
      "grad_norm": 2.577908754348755,
      "learning_rate": 2.933647149357122e-09,
      "logits/chosen": -3.1165366172790527,
      "logits/rejected": -3.137150764465332,
      "logps/chosen": -0.7795825600624084,
      "logps/rejected": -0.9781227111816406,
      "loss": 0.8445,
      "odds_ratio_loss": 0.648685097694397,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -0.07795824855566025,
      "rewards/margins": 0.01985403150320053,
      "rewards/rejected": -0.09781228005886078,
      "sft_loss": 0.7795825600624084,
      "step": 1660
    },
    {
      "epoch": 2.969548788619693,
      "grad_norm": 2.166626453399658,
      "learning_rate": 1.1111020018930717e-09,
      "logits/chosen": -3.156930446624756,
      "logits/rejected": -3.1462855339050293,
      "logps/chosen": -0.8264468908309937,
      "logps/rejected": -0.9435287714004517,
      "loss": 0.8929,
      "odds_ratio_loss": 0.6643285751342773,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.08264468610286713,
      "rewards/margins": 0.011708182282745838,
      "rewards/rejected": -0.09435287117958069,
      "sft_loss": 0.8264468908309937,
      "step": 1670
    },
    {
      "epoch": 2.987330517892865,
      "grad_norm": 2.1216466426849365,
      "learning_rate": 1.5625866646051813e-10,
      "logits/chosen": -3.1541049480438232,
      "logits/rejected": -3.1485564708709717,
      "logps/chosen": -0.779746413230896,
      "logps/rejected": -0.9904630780220032,
      "loss": 0.8399,
      "odds_ratio_loss": 0.6012987494468689,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.07797463238239288,
      "rewards/margins": 0.021071670576930046,
      "rewards/rejected": -0.09904631227254868,
      "sft_loss": 0.779746413230896,
      "step": 1680
    },
    {
      "epoch": 2.997999555456768,
      "step": 1686,
      "total_flos": 1.9814178520144282e+18,
      "train_loss": 0.8985705958283811,
      "train_runtime": 25618.6466,
      "train_samples_per_second": 1.054,
      "train_steps_per_second": 0.066
    }
  ],
  "logging_steps": 10,
  "max_steps": 1686,
  "num_input_tokens_seen": 0,
  "num_train_epochs": 3,
  "save_steps": 500,
  "total_flos": 1.9814178520144282e+18,
  "train_batch_size": 2,
  "trial_name": null,
  "trial_params": null
}