Llama-2-7b-dpo-2k-hhrlhf / trainer_state.json
AmberYifan's picture
Model save
e19030c verified
{
"best_metric": null,
"best_model_checkpoint": null,
"epoch": 0.992,
"eval_steps": 200,
"global_step": 62,
"is_hyper_param_search": false,
"is_local_process_zero": true,
"is_world_process_zero": true,
"log_history": [
{
"epoch": 0.016,
"grad_norm": 11.613288464777222,
"learning_rate": 7.142857142857142e-08,
"logits/generated": -1.1354284286499023,
"logits/real": -0.6185171008110046,
"logps/generated": -118.00320434570312,
"logps/real": -200.14151000976562,
"loss": 0.8305,
"rewards/accuracies": 0.0,
"rewards/generated": 0.0,
"rewards/margins": 0.0,
"rewards/real": 0.0,
"step": 1
},
{
"epoch": 0.16,
"grad_norm": 9.733030230104115,
"learning_rate": 4.727272727272727e-07,
"logits/generated": -1.043811321258545,
"logits/real": -1.071096420288086,
"logps/generated": -150.50152587890625,
"logps/real": -150.53155517578125,
"loss": 0.8213,
"rewards/accuracies": 0.0833333358168602,
"rewards/generated": 0.08699116855859756,
"rewards/margins": -0.0006038660649210215,
"rewards/real": 0.08638730645179749,
"step": 10
},
{
"epoch": 0.32,
"grad_norm": 8.782709887193477,
"learning_rate": 3.818181818181818e-07,
"logits/generated": -1.3196420669555664,
"logits/real": -1.299203872680664,
"logps/generated": -141.1020050048828,
"logps/real": -135.54541015625,
"loss": 0.8026,
"rewards/accuracies": 0.1875,
"rewards/generated": 0.5164750814437866,
"rewards/margins": 0.023883480578660965,
"rewards/real": 0.5403585433959961,
"step": 20
},
{
"epoch": 0.48,
"grad_norm": 6.004735799495323,
"learning_rate": 2.909090909090909e-07,
"logits/generated": -1.1754536628723145,
"logits/real": -1.196388602256775,
"logps/generated": -128.12631225585938,
"logps/real": -131.55551147460938,
"loss": 0.8034,
"rewards/accuracies": 0.17499999701976776,
"rewards/generated": 0.9843085408210754,
"rewards/margins": 0.01862453855574131,
"rewards/real": 1.002933144569397,
"step": 30
},
{
"epoch": 0.64,
"grad_norm": 7.699712198334635,
"learning_rate": 2e-07,
"logits/generated": -1.1779353618621826,
"logits/real": -1.3112047910690308,
"logps/generated": -155.4976806640625,
"logps/real": -132.09780883789062,
"loss": 0.7895,
"rewards/accuracies": 0.20000000298023224,
"rewards/generated": 1.2571728229522705,
"rewards/margins": 0.08223126828670502,
"rewards/real": 1.3394043445587158,
"step": 40
},
{
"epoch": 0.8,
"grad_norm": 6.175849275981129,
"learning_rate": 1.0909090909090908e-07,
"logits/generated": -1.3791444301605225,
"logits/real": -1.37796151638031,
"logps/generated": -131.0447998046875,
"logps/real": -130.96212768554688,
"loss": 0.7898,
"rewards/accuracies": 0.13750000298023224,
"rewards/generated": 1.49892258644104,
"rewards/margins": 0.029646193608641624,
"rewards/real": 1.5285687446594238,
"step": 50
},
{
"epoch": 0.96,
"grad_norm": 7.696425445812134,
"learning_rate": 1.818181818181818e-08,
"logits/generated": -1.4152286052703857,
"logits/real": -1.4222019910812378,
"logps/generated": -129.74317932128906,
"logps/real": -120.62994384765625,
"loss": 0.7694,
"rewards/accuracies": 0.3125,
"rewards/generated": 1.5588010549545288,
"rewards/margins": 0.09574465453624725,
"rewards/real": 1.654545783996582,
"step": 60
},
{
"epoch": 0.992,
"step": 62,
"total_flos": 0.0,
"train_loss": 0.7963649149863951,
"train_runtime": 722.2719,
"train_samples_per_second": 2.768,
"train_steps_per_second": 0.086
}
],
"logging_steps": 10,
"max_steps": 62,
"num_input_tokens_seen": 0,
"num_train_epochs": 1,
"save_steps": 200,
"stateful_callbacks": {
"TrainerControl": {
"args": {
"should_epoch_stop": false,
"should_evaluate": false,
"should_log": false,
"should_save": true,
"should_training_stop": true
},
"attributes": {}
}
},
"total_flos": 0.0,
"train_batch_size": 4,
"trial_name": null,
"trial_params": null
}