sam1120's picture
Training in progress, step 80
2a4441f
{
"best_metric": 0.08725559711456299,
"best_model_checkpoint": "/robodata/smodak/Projects/nspl/scripts/terrainseg/training/models/parking-utcustom-train-SF-RGB-b5_5/checkpoint-80",
"epoch": 80.0,
"global_step": 80,
"is_hyper_param_search": false,
"is_local_process_zero": true,
"is_world_process_zero": true,
"log_history": [
{
"epoch": 1.0,
"learning_rate": 5.833333333333332e-06,
"loss": 1.1349,
"step": 1
},
{
"epoch": 2.0,
"learning_rate": 1.1666666666666665e-05,
"loss": 1.1345,
"step": 2
},
{
"epoch": 3.0,
"learning_rate": 1.75e-05,
"loss": 1.1236,
"step": 3
},
{
"epoch": 4.0,
"learning_rate": 2.333333333333333e-05,
"loss": 1.1094,
"step": 4
},
{
"epoch": 5.0,
"learning_rate": 2.9166666666666666e-05,
"loss": 1.0857,
"step": 5
},
{
"epoch": 6.0,
"learning_rate": 3.5e-05,
"loss": 1.0643,
"step": 6
},
{
"epoch": 7.0,
"learning_rate": 3.4692982456140346e-05,
"loss": 1.0201,
"step": 7
},
{
"epoch": 8.0,
"learning_rate": 3.4385964912280695e-05,
"loss": 0.982,
"step": 8
},
{
"epoch": 9.0,
"learning_rate": 3.407894736842105e-05,
"loss": 0.9439,
"step": 9
},
{
"epoch": 10.0,
"learning_rate": 3.37719298245614e-05,
"loss": 0.918,
"step": 10
},
{
"epoch": 11.0,
"learning_rate": 3.346491228070175e-05,
"loss": 0.8757,
"step": 11
},
{
"epoch": 12.0,
"learning_rate": 3.31578947368421e-05,
"loss": 0.8507,
"step": 12
},
{
"epoch": 13.0,
"learning_rate": 3.2850877192982455e-05,
"loss": 0.8198,
"step": 13
},
{
"epoch": 14.0,
"learning_rate": 3.2543859649122804e-05,
"loss": 0.7719,
"step": 14
},
{
"epoch": 15.0,
"learning_rate": 3.2236842105263154e-05,
"loss": 0.7421,
"step": 15
},
{
"epoch": 16.0,
"learning_rate": 3.19298245614035e-05,
"loss": 0.7219,
"step": 16
},
{
"epoch": 17.0,
"learning_rate": 3.162280701754386e-05,
"loss": 0.7028,
"step": 17
},
{
"epoch": 18.0,
"learning_rate": 3.131578947368421e-05,
"loss": 0.6675,
"step": 18
},
{
"epoch": 19.0,
"learning_rate": 3.100877192982456e-05,
"loss": 0.6362,
"step": 19
},
{
"epoch": 20.0,
"learning_rate": 3.070175438596491e-05,
"loss": 0.622,
"step": 20
},
{
"epoch": 20.0,
"eval_accuracy_parking": NaN,
"eval_accuracy_unlabeled": NaN,
"eval_accuracy_unparking": 0.9969444274902344,
"eval_iou_parking": 0.0,
"eval_iou_unlabeled": 0.0,
"eval_iou_unparking": 0.9969444274902344,
"eval_loss": 0.5550474524497986,
"eval_mean_accuracy": 0.9969444274902344,
"eval_mean_iou": 0.33231480916341144,
"eval_overall_accuracy": 0.9969444274902344,
"eval_runtime": 0.9808,
"eval_samples_per_second": 1.02,
"eval_steps_per_second": 1.02,
"step": 20
},
{
"epoch": 21.0,
"learning_rate": 3.0394736842105263e-05,
"loss": 0.5915,
"step": 21
},
{
"epoch": 22.0,
"learning_rate": 3.0087719298245612e-05,
"loss": 0.5758,
"step": 22
},
{
"epoch": 23.0,
"learning_rate": 2.978070175438596e-05,
"loss": 0.547,
"step": 23
},
{
"epoch": 24.0,
"learning_rate": 2.947368421052631e-05,
"loss": 0.5373,
"step": 24
},
{
"epoch": 25.0,
"learning_rate": 2.9166666666666666e-05,
"loss": 0.5113,
"step": 25
},
{
"epoch": 26.0,
"learning_rate": 2.8859649122807016e-05,
"loss": 0.4997,
"step": 26
},
{
"epoch": 27.0,
"learning_rate": 2.8552631578947365e-05,
"loss": 0.4804,
"step": 27
},
{
"epoch": 28.0,
"learning_rate": 2.8245614035087714e-05,
"loss": 0.4784,
"step": 28
},
{
"epoch": 29.0,
"learning_rate": 2.793859649122807e-05,
"loss": 0.4567,
"step": 29
},
{
"epoch": 30.0,
"learning_rate": 2.763157894736842e-05,
"loss": 0.4261,
"step": 30
},
{
"epoch": 31.0,
"learning_rate": 2.732456140350877e-05,
"loss": 0.4224,
"step": 31
},
{
"epoch": 32.0,
"learning_rate": 2.7017543859649118e-05,
"loss": 0.4097,
"step": 32
},
{
"epoch": 33.0,
"learning_rate": 2.6710526315789474e-05,
"loss": 0.398,
"step": 33
},
{
"epoch": 34.0,
"learning_rate": 2.6403508771929823e-05,
"loss": 0.3734,
"step": 34
},
{
"epoch": 35.0,
"learning_rate": 2.6096491228070172e-05,
"loss": 0.3728,
"step": 35
},
{
"epoch": 36.0,
"learning_rate": 2.578947368421052e-05,
"loss": 0.3638,
"step": 36
},
{
"epoch": 37.0,
"learning_rate": 2.5482456140350878e-05,
"loss": 0.3425,
"step": 37
},
{
"epoch": 38.0,
"learning_rate": 2.5175438596491227e-05,
"loss": 0.3259,
"step": 38
},
{
"epoch": 39.0,
"learning_rate": 2.4868421052631576e-05,
"loss": 0.3243,
"step": 39
},
{
"epoch": 40.0,
"learning_rate": 2.4561403508771925e-05,
"loss": 0.3143,
"step": 40
},
{
"epoch": 40.0,
"eval_accuracy_parking": NaN,
"eval_accuracy_unlabeled": NaN,
"eval_accuracy_unparking": 1.0,
"eval_iou_parking": NaN,
"eval_iou_unlabeled": NaN,
"eval_iou_unparking": 1.0,
"eval_loss": 0.22338461875915527,
"eval_mean_accuracy": 1.0,
"eval_mean_iou": 1.0,
"eval_overall_accuracy": 1.0,
"eval_runtime": 25.7138,
"eval_samples_per_second": 0.039,
"eval_steps_per_second": 0.039,
"step": 40
},
{
"epoch": 41.0,
"learning_rate": 2.425438596491228e-05,
"loss": 0.3134,
"step": 41
},
{
"epoch": 42.0,
"learning_rate": 2.394736842105263e-05,
"loss": 0.3006,
"step": 42
},
{
"epoch": 43.0,
"learning_rate": 2.364035087719298e-05,
"loss": 0.2879,
"step": 43
},
{
"epoch": 44.0,
"learning_rate": 2.333333333333333e-05,
"loss": 0.2722,
"step": 44
},
{
"epoch": 45.0,
"learning_rate": 2.3026315789473685e-05,
"loss": 0.2684,
"step": 45
},
{
"epoch": 46.0,
"learning_rate": 2.2719298245614034e-05,
"loss": 0.2608,
"step": 46
},
{
"epoch": 47.0,
"learning_rate": 2.2412280701754384e-05,
"loss": 0.2498,
"step": 47
},
{
"epoch": 48.0,
"learning_rate": 2.2105263157894733e-05,
"loss": 0.2531,
"step": 48
},
{
"epoch": 49.0,
"learning_rate": 2.179824561403509e-05,
"loss": 0.2414,
"step": 49
},
{
"epoch": 50.0,
"learning_rate": 2.1491228070175438e-05,
"loss": 0.2368,
"step": 50
},
{
"epoch": 51.0,
"learning_rate": 2.1184210526315787e-05,
"loss": 0.2219,
"step": 51
},
{
"epoch": 52.0,
"learning_rate": 2.0877192982456137e-05,
"loss": 0.2155,
"step": 52
},
{
"epoch": 53.0,
"learning_rate": 2.0570175438596493e-05,
"loss": 0.2106,
"step": 53
},
{
"epoch": 54.0,
"learning_rate": 2.0263157894736842e-05,
"loss": 0.2104,
"step": 54
},
{
"epoch": 55.0,
"learning_rate": 1.995614035087719e-05,
"loss": 0.1987,
"step": 55
},
{
"epoch": 56.0,
"learning_rate": 1.964912280701754e-05,
"loss": 0.1872,
"step": 56
},
{
"epoch": 57.0,
"learning_rate": 1.9342105263157896e-05,
"loss": 0.1859,
"step": 57
},
{
"epoch": 58.0,
"learning_rate": 1.9035087719298245e-05,
"loss": 0.1861,
"step": 58
},
{
"epoch": 59.0,
"learning_rate": 1.8728070175438595e-05,
"loss": 0.1767,
"step": 59
},
{
"epoch": 60.0,
"learning_rate": 1.8421052631578944e-05,
"loss": 0.1695,
"step": 60
},
{
"epoch": 60.0,
"eval_accuracy_parking": NaN,
"eval_accuracy_unlabeled": NaN,
"eval_accuracy_unparking": 1.0,
"eval_iou_parking": NaN,
"eval_iou_unlabeled": NaN,
"eval_iou_unparking": 1.0,
"eval_loss": 0.13935932517051697,
"eval_mean_accuracy": 1.0,
"eval_mean_iou": 1.0,
"eval_overall_accuracy": 1.0,
"eval_runtime": 0.8484,
"eval_samples_per_second": 1.179,
"eval_steps_per_second": 1.179,
"step": 60
},
{
"epoch": 61.0,
"learning_rate": 1.81140350877193e-05,
"loss": 0.1643,
"step": 61
},
{
"epoch": 62.0,
"learning_rate": 1.780701754385965e-05,
"loss": 0.1592,
"step": 62
},
{
"epoch": 63.0,
"learning_rate": 1.75e-05,
"loss": 0.1564,
"step": 63
},
{
"epoch": 64.0,
"learning_rate": 1.7192982456140348e-05,
"loss": 0.1635,
"step": 64
},
{
"epoch": 65.0,
"learning_rate": 1.68859649122807e-05,
"loss": 0.1588,
"step": 65
},
{
"epoch": 66.0,
"learning_rate": 1.657894736842105e-05,
"loss": 0.1471,
"step": 66
},
{
"epoch": 67.0,
"learning_rate": 1.6271929824561402e-05,
"loss": 0.1415,
"step": 67
},
{
"epoch": 68.0,
"learning_rate": 1.596491228070175e-05,
"loss": 0.1426,
"step": 68
},
{
"epoch": 69.0,
"learning_rate": 1.5657894736842104e-05,
"loss": 0.1394,
"step": 69
},
{
"epoch": 70.0,
"learning_rate": 1.5350877192982453e-05,
"loss": 0.1361,
"step": 70
},
{
"epoch": 71.0,
"learning_rate": 1.5043859649122806e-05,
"loss": 0.1367,
"step": 71
},
{
"epoch": 72.0,
"learning_rate": 1.4736842105263155e-05,
"loss": 0.1357,
"step": 72
},
{
"epoch": 73.0,
"learning_rate": 1.4429824561403508e-05,
"loss": 0.1273,
"step": 73
},
{
"epoch": 74.0,
"learning_rate": 1.4122807017543857e-05,
"loss": 0.1325,
"step": 74
},
{
"epoch": 75.0,
"learning_rate": 1.381578947368421e-05,
"loss": 0.1214,
"step": 75
},
{
"epoch": 76.0,
"learning_rate": 1.3508771929824559e-05,
"loss": 0.1282,
"step": 76
},
{
"epoch": 77.0,
"learning_rate": 1.3201754385964912e-05,
"loss": 0.1155,
"step": 77
},
{
"epoch": 78.0,
"learning_rate": 1.289473684210526e-05,
"loss": 0.1193,
"step": 78
},
{
"epoch": 79.0,
"learning_rate": 1.2587719298245613e-05,
"loss": 0.1178,
"step": 79
},
{
"epoch": 80.0,
"learning_rate": 1.2280701754385963e-05,
"loss": 0.1248,
"step": 80
},
{
"epoch": 80.0,
"eval_accuracy_parking": NaN,
"eval_accuracy_unlabeled": NaN,
"eval_accuracy_unparking": 1.0,
"eval_iou_parking": NaN,
"eval_iou_unlabeled": NaN,
"eval_iou_unparking": 1.0,
"eval_loss": 0.08725559711456299,
"eval_mean_accuracy": 1.0,
"eval_mean_iou": 1.0,
"eval_overall_accuracy": 1.0,
"eval_runtime": 1.4777,
"eval_samples_per_second": 0.677,
"eval_steps_per_second": 0.677,
"step": 80
}
],
"max_steps": 120,
"num_train_epochs": 120,
"total_flos": 5.1094062341554176e+17,
"trial_name": null,
"trial_params": null
}