sam1120's picture
Training in progress, step 80
147766d
{
"best_metric": 0.8392333984375,
"best_model_checkpoint": "/robodata/smodak/Projects/nspl/scripts/terrainseg/training/models/parking-utcustom-train-SF-RGB-b0_2/checkpoint-80",
"epoch": 80.0,
"global_step": 80,
"is_hyper_param_search": false,
"is_local_process_zero": true,
"is_world_process_zero": true,
"log_history": [
{
"epoch": 1.0,
"learning_rate": 4.9999999999999996e-06,
"loss": 1.2234,
"step": 1
},
{
"epoch": 2.0,
"learning_rate": 9.999999999999999e-06,
"loss": 1.2229,
"step": 2
},
{
"epoch": 3.0,
"learning_rate": 1.5e-05,
"loss": 1.2191,
"step": 3
},
{
"epoch": 4.0,
"learning_rate": 1.9999999999999998e-05,
"loss": 1.216,
"step": 4
},
{
"epoch": 5.0,
"learning_rate": 2.5e-05,
"loss": 1.2126,
"step": 5
},
{
"epoch": 6.0,
"learning_rate": 3e-05,
"loss": 1.2038,
"step": 6
},
{
"epoch": 7.0,
"learning_rate": 2.9736842105263157e-05,
"loss": 1.1946,
"step": 7
},
{
"epoch": 8.0,
"learning_rate": 2.9473684210526314e-05,
"loss": 1.1913,
"step": 8
},
{
"epoch": 9.0,
"learning_rate": 2.9210526315789474e-05,
"loss": 1.1799,
"step": 9
},
{
"epoch": 10.0,
"learning_rate": 2.8947368421052634e-05,
"loss": 1.174,
"step": 10
},
{
"epoch": 11.0,
"learning_rate": 2.868421052631579e-05,
"loss": 1.1617,
"step": 11
},
{
"epoch": 12.0,
"learning_rate": 2.8421052631578946e-05,
"loss": 1.1634,
"step": 12
},
{
"epoch": 13.0,
"learning_rate": 2.8157894736842106e-05,
"loss": 1.1502,
"step": 13
},
{
"epoch": 14.0,
"learning_rate": 2.7894736842105263e-05,
"loss": 1.1471,
"step": 14
},
{
"epoch": 15.0,
"learning_rate": 2.7631578947368423e-05,
"loss": 1.1306,
"step": 15
},
{
"epoch": 16.0,
"learning_rate": 2.736842105263158e-05,
"loss": 1.1303,
"step": 16
},
{
"epoch": 17.0,
"learning_rate": 2.710526315789474e-05,
"loss": 1.1261,
"step": 17
},
{
"epoch": 18.0,
"learning_rate": 2.6842105263157896e-05,
"loss": 1.1245,
"step": 18
},
{
"epoch": 19.0,
"learning_rate": 2.6578947368421052e-05,
"loss": 1.1075,
"step": 19
},
{
"epoch": 20.0,
"learning_rate": 2.631578947368421e-05,
"loss": 1.0965,
"step": 20
},
{
"epoch": 20.0,
"eval_accuracy_parking": NaN,
"eval_accuracy_unlabeled": NaN,
"eval_accuracy_unparking": 0.4298858642578125,
"eval_iou_parking": 0.0,
"eval_iou_unlabeled": 0.0,
"eval_iou_unparking": 0.4298858642578125,
"eval_loss": 1.0974003076553345,
"eval_mean_accuracy": 0.4298858642578125,
"eval_mean_iou": 0.1432952880859375,
"eval_overall_accuracy": 0.4298858642578125,
"eval_runtime": 0.4804,
"eval_samples_per_second": 2.082,
"eval_steps_per_second": 2.082,
"step": 20
},
{
"epoch": 21.0,
"learning_rate": 2.605263157894737e-05,
"loss": 1.0938,
"step": 21
},
{
"epoch": 22.0,
"learning_rate": 2.578947368421053e-05,
"loss": 1.0841,
"step": 22
},
{
"epoch": 23.0,
"learning_rate": 2.5526315789473685e-05,
"loss": 1.0771,
"step": 23
},
{
"epoch": 24.0,
"learning_rate": 2.526315789473684e-05,
"loss": 1.066,
"step": 24
},
{
"epoch": 25.0,
"learning_rate": 2.5e-05,
"loss": 1.0565,
"step": 25
},
{
"epoch": 26.0,
"learning_rate": 2.4736842105263158e-05,
"loss": 1.0522,
"step": 26
},
{
"epoch": 27.0,
"learning_rate": 2.4473684210526318e-05,
"loss": 1.0709,
"step": 27
},
{
"epoch": 28.0,
"learning_rate": 2.4210526315789474e-05,
"loss": 1.0567,
"step": 28
},
{
"epoch": 29.0,
"learning_rate": 2.3947368421052634e-05,
"loss": 1.0489,
"step": 29
},
{
"epoch": 30.0,
"learning_rate": 2.368421052631579e-05,
"loss": 1.0324,
"step": 30
},
{
"epoch": 31.0,
"learning_rate": 2.3421052631578947e-05,
"loss": 1.0285,
"step": 31
},
{
"epoch": 32.0,
"learning_rate": 2.3157894736842103e-05,
"loss": 1.0167,
"step": 32
},
{
"epoch": 33.0,
"learning_rate": 2.2894736842105263e-05,
"loss": 1.0062,
"step": 33
},
{
"epoch": 34.0,
"learning_rate": 2.2631578947368423e-05,
"loss": 1.004,
"step": 34
},
{
"epoch": 35.0,
"learning_rate": 2.236842105263158e-05,
"loss": 1.007,
"step": 35
},
{
"epoch": 36.0,
"learning_rate": 2.2105263157894736e-05,
"loss": 0.9915,
"step": 36
},
{
"epoch": 37.0,
"learning_rate": 2.1842105263157896e-05,
"loss": 0.9895,
"step": 37
},
{
"epoch": 38.0,
"learning_rate": 2.1578947368421053e-05,
"loss": 0.9813,
"step": 38
},
{
"epoch": 39.0,
"learning_rate": 2.1315789473684212e-05,
"loss": 0.9562,
"step": 39
},
{
"epoch": 40.0,
"learning_rate": 2.105263157894737e-05,
"loss": 0.9563,
"step": 40
},
{
"epoch": 40.0,
"eval_accuracy_parking": NaN,
"eval_accuracy_unlabeled": NaN,
"eval_accuracy_unparking": 0.7234535217285156,
"eval_iou_parking": 0.0,
"eval_iou_unlabeled": 0.0,
"eval_iou_unparking": 0.7234535217285156,
"eval_loss": 1.0286378860473633,
"eval_mean_accuracy": 0.7234535217285156,
"eval_mean_iou": 0.24115117390950522,
"eval_overall_accuracy": 0.7234535217285156,
"eval_runtime": 0.5302,
"eval_samples_per_second": 1.886,
"eval_steps_per_second": 1.886,
"step": 40
},
{
"epoch": 41.0,
"learning_rate": 2.078947368421053e-05,
"loss": 0.947,
"step": 41
},
{
"epoch": 42.0,
"learning_rate": 2.0526315789473685e-05,
"loss": 0.947,
"step": 42
},
{
"epoch": 43.0,
"learning_rate": 2.0263157894736842e-05,
"loss": 0.9411,
"step": 43
},
{
"epoch": 44.0,
"learning_rate": 1.9999999999999998e-05,
"loss": 0.9252,
"step": 44
},
{
"epoch": 45.0,
"learning_rate": 1.9736842105263158e-05,
"loss": 0.9327,
"step": 45
},
{
"epoch": 46.0,
"learning_rate": 1.9473684210526318e-05,
"loss": 0.9223,
"step": 46
},
{
"epoch": 47.0,
"learning_rate": 1.9210526315789474e-05,
"loss": 0.896,
"step": 47
},
{
"epoch": 48.0,
"learning_rate": 1.894736842105263e-05,
"loss": 0.9083,
"step": 48
},
{
"epoch": 49.0,
"learning_rate": 1.868421052631579e-05,
"loss": 0.9132,
"step": 49
},
{
"epoch": 50.0,
"learning_rate": 1.8421052631578947e-05,
"loss": 0.8891,
"step": 50
},
{
"epoch": 51.0,
"learning_rate": 1.8157894736842107e-05,
"loss": 0.8823,
"step": 51
},
{
"epoch": 52.0,
"learning_rate": 1.7894736842105264e-05,
"loss": 0.9112,
"step": 52
},
{
"epoch": 53.0,
"learning_rate": 1.7631578947368424e-05,
"loss": 0.8981,
"step": 53
},
{
"epoch": 54.0,
"learning_rate": 1.736842105263158e-05,
"loss": 0.8924,
"step": 54
},
{
"epoch": 55.0,
"learning_rate": 1.7105263157894737e-05,
"loss": 0.8735,
"step": 55
},
{
"epoch": 56.0,
"learning_rate": 1.6842105263157893e-05,
"loss": 0.8576,
"step": 56
},
{
"epoch": 57.0,
"learning_rate": 1.6578947368421053e-05,
"loss": 0.8609,
"step": 57
},
{
"epoch": 58.0,
"learning_rate": 1.6315789473684213e-05,
"loss": 0.841,
"step": 58
},
{
"epoch": 59.0,
"learning_rate": 1.605263157894737e-05,
"loss": 0.8494,
"step": 59
},
{
"epoch": 60.0,
"learning_rate": 1.5789473684210526e-05,
"loss": 0.8707,
"step": 60
},
{
"epoch": 60.0,
"eval_accuracy_parking": NaN,
"eval_accuracy_unlabeled": NaN,
"eval_accuracy_unparking": 0.8608512878417969,
"eval_iou_parking": 0.0,
"eval_iou_unlabeled": 0.0,
"eval_iou_unparking": 0.8608512878417969,
"eval_loss": 0.9260364174842834,
"eval_mean_accuracy": 0.8608512878417969,
"eval_mean_iou": 0.28695042928059894,
"eval_overall_accuracy": 0.8608512878417969,
"eval_runtime": 0.5026,
"eval_samples_per_second": 1.99,
"eval_steps_per_second": 1.99,
"step": 60
},
{
"epoch": 61.0,
"learning_rate": 1.5526315789473686e-05,
"loss": 0.8188,
"step": 61
},
{
"epoch": 62.0,
"learning_rate": 1.5263157894736842e-05,
"loss": 0.8495,
"step": 62
},
{
"epoch": 63.0,
"learning_rate": 1.5e-05,
"loss": 0.8173,
"step": 63
},
{
"epoch": 64.0,
"learning_rate": 1.4736842105263157e-05,
"loss": 0.8086,
"step": 64
},
{
"epoch": 65.0,
"learning_rate": 1.4473684210526317e-05,
"loss": 0.7986,
"step": 65
},
{
"epoch": 66.0,
"learning_rate": 1.4210526315789473e-05,
"loss": 0.8226,
"step": 66
},
{
"epoch": 67.0,
"learning_rate": 1.3947368421052631e-05,
"loss": 0.8185,
"step": 67
},
{
"epoch": 68.0,
"learning_rate": 1.368421052631579e-05,
"loss": 0.8063,
"step": 68
},
{
"epoch": 69.0,
"learning_rate": 1.3421052631578948e-05,
"loss": 0.8218,
"step": 69
},
{
"epoch": 70.0,
"learning_rate": 1.3157894736842104e-05,
"loss": 0.7961,
"step": 70
},
{
"epoch": 71.0,
"learning_rate": 1.2894736842105264e-05,
"loss": 0.832,
"step": 71
},
{
"epoch": 72.0,
"learning_rate": 1.263157894736842e-05,
"loss": 0.7845,
"step": 72
},
{
"epoch": 73.0,
"learning_rate": 1.2368421052631579e-05,
"loss": 0.8236,
"step": 73
},
{
"epoch": 74.0,
"learning_rate": 1.2105263157894737e-05,
"loss": 0.8004,
"step": 74
},
{
"epoch": 75.0,
"learning_rate": 1.1842105263157895e-05,
"loss": 0.7757,
"step": 75
},
{
"epoch": 76.0,
"learning_rate": 1.1578947368421052e-05,
"loss": 0.7931,
"step": 76
},
{
"epoch": 77.0,
"learning_rate": 1.1315789473684212e-05,
"loss": 0.7735,
"step": 77
},
{
"epoch": 78.0,
"learning_rate": 1.1052631578947368e-05,
"loss": 0.8168,
"step": 78
},
{
"epoch": 79.0,
"learning_rate": 1.0789473684210526e-05,
"loss": 0.7637,
"step": 79
},
{
"epoch": 80.0,
"learning_rate": 1.0526315789473684e-05,
"loss": 0.7662,
"step": 80
},
{
"epoch": 80.0,
"eval_accuracy_parking": NaN,
"eval_accuracy_unlabeled": NaN,
"eval_accuracy_unparking": 0.8852920532226562,
"eval_iou_parking": 0.0,
"eval_iou_unlabeled": 0.0,
"eval_iou_unparking": 0.8852920532226562,
"eval_loss": 0.8392333984375,
"eval_mean_accuracy": 0.8852920532226562,
"eval_mean_iou": 0.29509735107421875,
"eval_overall_accuracy": 0.8852920532226562,
"eval_runtime": 0.5065,
"eval_samples_per_second": 1.974,
"eval_steps_per_second": 1.974,
"step": 80
}
],
"max_steps": 120,
"num_train_epochs": 120,
"total_flos": 2.24373352955904e+16,
"trial_name": null,
"trial_params": null
}