| { | |
| "best_metric": null, | |
| "best_model_checkpoint": null, | |
| "epoch": 0.9984, | |
| "eval_steps": 500, | |
| "global_step": 156, | |
| "is_hyper_param_search": false, | |
| "is_local_process_zero": true, | |
| "is_world_process_zero": true, | |
| "log_history": [ | |
| { | |
| "epoch": 0.01, | |
| "eta": 0.0010000000474974513, | |
| "grad_norm": 18.85951805804989, | |
| "learning_rate": 3.125e-08, | |
| "logits/chosen": -2.2437264919281006, | |
| "logits/rejected": -2.1319897174835205, | |
| "logps/chosen": -136.11781311035156, | |
| "logps/pi_response": -276.34149169921875, | |
| "logps/ref_response": -276.34149169921875, | |
| "logps/rejected": -134.32876586914062, | |
| "loss": 0.6931, | |
| "rewards/accuracies": 0.0, | |
| "rewards/chosen": 0.0, | |
| "rewards/margins": 0.0, | |
| "rewards/rejected": 0.0, | |
| "step": 1 | |
| }, | |
| { | |
| "epoch": 0.06, | |
| "eta": 0.0010000000474974513, | |
| "grad_norm": 15.166671167458636, | |
| "learning_rate": 3.1249999999999997e-07, | |
| "logits/chosen": -2.3832309246063232, | |
| "logits/rejected": -2.3636457920074463, | |
| "logps/chosen": -155.76785278320312, | |
| "logps/pi_response": -274.42333984375, | |
| "logps/ref_response": -272.425048828125, | |
| "logps/rejected": -158.64793395996094, | |
| "loss": 0.6926, | |
| "rewards/accuracies": 0.4270833432674408, | |
| "rewards/chosen": -0.00683738524094224, | |
| "rewards/margins": -0.00022508477559313178, | |
| "rewards/rejected": -0.006612300407141447, | |
| "step": 10 | |
| }, | |
| { | |
| "epoch": 0.13, | |
| "eta": 0.0010000000474974513, | |
| "grad_norm": 15.006144425101914, | |
| "learning_rate": 4.989935734988097e-07, | |
| "logits/chosen": -2.3142848014831543, | |
| "logits/rejected": -2.337123394012451, | |
| "logps/chosen": -169.91624450683594, | |
| "logps/pi_response": -305.30267333984375, | |
| "logps/ref_response": -275.4255065917969, | |
| "logps/rejected": -177.8936767578125, | |
| "loss": 0.692, | |
| "rewards/accuracies": 0.515625, | |
| "rewards/chosen": -0.154428631067276, | |
| "rewards/margins": 0.013679690659046173, | |
| "rewards/rejected": -0.16810832917690277, | |
| "step": 20 | |
| }, | |
| { | |
| "epoch": 0.19, | |
| "eta": 0.0010000000474974513, | |
| "grad_norm": 25.3042204309977, | |
| "learning_rate": 4.877641290737883e-07, | |
| "logits/chosen": -2.33975887298584, | |
| "logits/rejected": -2.3151369094848633, | |
| "logps/chosen": -189.64102172851562, | |
| "logps/pi_response": -321.45294189453125, | |
| "logps/ref_response": -261.0726013183594, | |
| "logps/rejected": -194.24017333984375, | |
| "loss": 0.6936, | |
| "rewards/accuracies": 0.528124988079071, | |
| "rewards/chosen": -0.33687421679496765, | |
| "rewards/margins": 0.005444393027573824, | |
| "rewards/rejected": -0.3423186242580414, | |
| "step": 30 | |
| }, | |
| { | |
| "epoch": 0.26, | |
| "eta": 0.0010000000474974513, | |
| "grad_norm": 14.197120755569808, | |
| "learning_rate": 4.646121984004665e-07, | |
| "logits/chosen": -2.4677834510803223, | |
| "logits/rejected": -2.4844822883605957, | |
| "logps/chosen": -176.27413940429688, | |
| "logps/pi_response": -294.74114990234375, | |
| "logps/ref_response": -256.48724365234375, | |
| "logps/rejected": -179.17926025390625, | |
| "loss": 0.6905, | |
| "rewards/accuracies": 0.578125, | |
| "rewards/chosen": -0.2434801161289215, | |
| "rewards/margins": -0.00023287050134968013, | |
| "rewards/rejected": -0.24324722588062286, | |
| "step": 40 | |
| }, | |
| { | |
| "epoch": 0.32, | |
| "eta": 0.0010000000474974513, | |
| "grad_norm": 15.243074464797877, | |
| "learning_rate": 4.3069871595684787e-07, | |
| "logits/chosen": -2.445664882659912, | |
| "logits/rejected": -2.4546258449554443, | |
| "logps/chosen": -189.74288940429688, | |
| "logps/pi_response": -307.9224548339844, | |
| "logps/ref_response": -267.48931884765625, | |
| "logps/rejected": -200.6833038330078, | |
| "loss": 0.6929, | |
| "rewards/accuracies": 0.5562499761581421, | |
| "rewards/chosen": -0.2953701615333557, | |
| "rewards/margins": 0.01801210641860962, | |
| "rewards/rejected": -0.3133822977542877, | |
| "step": 50 | |
| }, | |
| { | |
| "epoch": 0.38, | |
| "eta": 0.0010000000474974513, | |
| "grad_norm": 16.31963304577925, | |
| "learning_rate": 3.877242453630256e-07, | |
| "logits/chosen": -2.477487087249756, | |
| "logits/rejected": -2.4785385131835938, | |
| "logps/chosen": -185.6737518310547, | |
| "logps/pi_response": -291.3546447753906, | |
| "logps/ref_response": -254.33984375, | |
| "logps/rejected": -188.54415893554688, | |
| "loss": 0.6887, | |
| "rewards/accuracies": 0.5375000238418579, | |
| "rewards/chosen": -0.26406130194664, | |
| "rewards/margins": 0.008015439845621586, | |
| "rewards/rejected": -0.2720767557621002, | |
| "step": 60 | |
| }, | |
| { | |
| "epoch": 0.45, | |
| "eta": 0.0010000000474974513, | |
| "grad_norm": 16.071293511713314, | |
| "learning_rate": 3.378437060203357e-07, | |
| "logits/chosen": -2.359812021255493, | |
| "logits/rejected": -2.355583429336548, | |
| "logps/chosen": -209.6289520263672, | |
| "logps/pi_response": -323.3484802246094, | |
| "logps/ref_response": -260.3892517089844, | |
| "logps/rejected": -210.88949584960938, | |
| "loss": 0.6913, | |
| "rewards/accuracies": 0.515625, | |
| "rewards/chosen": -0.5319920778274536, | |
| "rewards/margins": 0.012827059254050255, | |
| "rewards/rejected": -0.5448191165924072, | |
| "step": 70 | |
| }, | |
| { | |
| "epoch": 0.51, | |
| "eta": 0.0010000000474974513, | |
| "grad_norm": 16.843259909148866, | |
| "learning_rate": 2.8355831645441387e-07, | |
| "logits/chosen": -2.4202404022216797, | |
| "logits/rejected": -2.362644910812378, | |
| "logps/chosen": -196.7368927001953, | |
| "logps/pi_response": -324.1815490722656, | |
| "logps/ref_response": -271.9532470703125, | |
| "logps/rejected": -200.54006958007812, | |
| "loss": 0.6866, | |
| "rewards/accuracies": 0.5687500238418579, | |
| "rewards/chosen": -0.402200847864151, | |
| "rewards/margins": 0.04481234401464462, | |
| "rewards/rejected": -0.4470131993293762, | |
| "step": 80 | |
| }, | |
| { | |
| "epoch": 0.58, | |
| "eta": 0.0010000000474974513, | |
| "grad_norm": 20.3973840448787, | |
| "learning_rate": 2.2759017277414164e-07, | |
| "logits/chosen": -2.416743278503418, | |
| "logits/rejected": -2.436403751373291, | |
| "logps/chosen": -202.49813842773438, | |
| "logps/pi_response": -311.49896240234375, | |
| "logps/ref_response": -254.697509765625, | |
| "logps/rejected": -198.43460083007812, | |
| "loss": 0.6881, | |
| "rewards/accuracies": 0.5249999761581421, | |
| "rewards/chosen": -0.3861594796180725, | |
| "rewards/margins": 0.029975295066833496, | |
| "rewards/rejected": -0.4161347448825836, | |
| "step": 90 | |
| }, | |
| { | |
| "epoch": 0.64, | |
| "eta": 0.0010000000474974513, | |
| "grad_norm": 19.031326931010696, | |
| "learning_rate": 1.7274575140626315e-07, | |
| "logits/chosen": -2.3818562030792236, | |
| "logits/rejected": -2.393977403640747, | |
| "logps/chosen": -200.36917114257812, | |
| "logps/pi_response": -338.3829040527344, | |
| "logps/ref_response": -261.9479064941406, | |
| "logps/rejected": -211.229736328125, | |
| "loss": 0.6832, | |
| "rewards/accuracies": 0.5562499761581421, | |
| "rewards/chosen": -0.5124812722206116, | |
| "rewards/margins": 0.037171028554439545, | |
| "rewards/rejected": -0.5496522188186646, | |
| "step": 100 | |
| }, | |
| { | |
| "epoch": 0.7, | |
| "eta": 0.0010000000474974513, | |
| "grad_norm": 15.411774484824894, | |
| "learning_rate": 1.2177518064852348e-07, | |
| "logits/chosen": -2.4000236988067627, | |
| "logits/rejected": -2.2974681854248047, | |
| "logps/chosen": -204.61512756347656, | |
| "logps/pi_response": -330.70831298828125, | |
| "logps/ref_response": -250.0836639404297, | |
| "logps/rejected": -208.4372100830078, | |
| "loss": 0.6858, | |
| "rewards/accuracies": 0.5562499761581421, | |
| "rewards/chosen": -0.5626230239868164, | |
| "rewards/margins": 0.039384625852108, | |
| "rewards/rejected": -0.6020076274871826, | |
| "step": 110 | |
| }, | |
| { | |
| "epoch": 0.77, | |
| "eta": 0.0010000000474974513, | |
| "grad_norm": 15.901638390226227, | |
| "learning_rate": 7.723433775328384e-08, | |
| "logits/chosen": -2.2554521560668945, | |
| "logits/rejected": -2.3232483863830566, | |
| "logps/chosen": -223.6305694580078, | |
| "logps/pi_response": -362.30291748046875, | |
| "logps/ref_response": -276.22747802734375, | |
| "logps/rejected": -226.0797119140625, | |
| "loss": 0.6871, | |
| "rewards/accuracies": 0.5218750238418579, | |
| "rewards/chosen": -0.6587863564491272, | |
| "rewards/margins": 0.01464476902037859, | |
| "rewards/rejected": -0.6734310984611511, | |
| "step": 120 | |
| }, | |
| { | |
| "epoch": 0.83, | |
| "eta": 0.0010000000474974513, | |
| "grad_norm": 15.415238088814565, | |
| "learning_rate": 4.1356686569674335e-08, | |
| "logits/chosen": -2.2723705768585205, | |
| "logits/rejected": -2.223027229309082, | |
| "logps/chosen": -215.4403839111328, | |
| "logps/pi_response": -351.7759094238281, | |
| "logps/ref_response": -266.7939453125, | |
| "logps/rejected": -221.5651397705078, | |
| "loss": 0.6829, | |
| "rewards/accuracies": 0.5562499761581421, | |
| "rewards/chosen": -0.6349462270736694, | |
| "rewards/margins": 0.02343112602829933, | |
| "rewards/rejected": -0.6583773493766785, | |
| "step": 130 | |
| }, | |
| { | |
| "epoch": 0.9, | |
| "eta": 0.0010000000474974513, | |
| "grad_norm": 17.415730450012774, | |
| "learning_rate": 1.5941282340065697e-08, | |
| "logits/chosen": -2.319291591644287, | |
| "logits/rejected": -2.372563600540161, | |
| "logps/chosen": -211.4011688232422, | |
| "logps/pi_response": -335.5188903808594, | |
| "logps/ref_response": -254.0056915283203, | |
| "logps/rejected": -212.92123413085938, | |
| "loss": 0.6817, | |
| "rewards/accuracies": 0.5218750238418579, | |
| "rewards/chosen": -0.5842172503471375, | |
| "rewards/margins": 0.024492263793945312, | |
| "rewards/rejected": -0.6087095141410828, | |
| "step": 140 | |
| }, | |
| { | |
| "epoch": 0.96, | |
| "eta": 0.0010000000474974513, | |
| "grad_norm": 16.173628365681967, | |
| "learning_rate": 2.2625595580163247e-09, | |
| "logits/chosen": -2.2677788734436035, | |
| "logits/rejected": -2.2873096466064453, | |
| "logps/chosen": -209.3054656982422, | |
| "logps/pi_response": -347.63079833984375, | |
| "logps/ref_response": -265.3609313964844, | |
| "logps/rejected": -218.402587890625, | |
| "loss": 0.6852, | |
| "rewards/accuracies": 0.550000011920929, | |
| "rewards/chosen": -0.5842490196228027, | |
| "rewards/margins": 0.023630866780877113, | |
| "rewards/rejected": -0.6078798770904541, | |
| "step": 150 | |
| }, | |
| { | |
| "epoch": 1.0, | |
| "step": 156, | |
| "total_flos": 0.0, | |
| "train_loss": 0.688222443828216, | |
| "train_runtime": 31827.1935, | |
| "train_samples_per_second": 0.628, | |
| "train_steps_per_second": 0.005 | |
| } | |
| ], | |
| "logging_steps": 10, | |
| "max_steps": 156, | |
| "num_input_tokens_seen": 0, | |
| "num_train_epochs": 1, | |
| "save_steps": 200, | |
| "total_flos": 0.0, | |
| "train_batch_size": 1, | |
| "trial_name": null, | |
| "trial_params": null | |
| } | |