sam1120's picture
Training in progress, step 80
cb43563
{
"best_metric": 0.05897674709558487,
"best_model_checkpoint": "/robodata/smodak/Projects/nspl/scripts/terrainseg/training/models/parking-utcustom-train-SF-RGBD-b5_1/checkpoint-80",
"epoch": 80.0,
"global_step": 80,
"is_hyper_param_search": false,
"is_local_process_zero": true,
"is_world_process_zero": true,
"log_history": [
{
"epoch": 1.0,
"learning_rate": 8.571428571428571e-06,
"loss": 1.1643,
"step": 1
},
{
"epoch": 2.0,
"learning_rate": 1.7142857142857142e-05,
"loss": 1.1586,
"step": 2
},
{
"epoch": 3.0,
"learning_rate": 2.5714285714285714e-05,
"loss": 1.1385,
"step": 3
},
{
"epoch": 4.0,
"learning_rate": 3.4285714285714284e-05,
"loss": 1.1084,
"step": 4
},
{
"epoch": 5.0,
"learning_rate": 4.2857142857142856e-05,
"loss": 1.0612,
"step": 5
},
{
"epoch": 6.0,
"learning_rate": 5.142857142857143e-05,
"loss": 1.0167,
"step": 6
},
{
"epoch": 7.0,
"learning_rate": 6e-05,
"loss": 0.9557,
"step": 7
},
{
"epoch": 8.0,
"learning_rate": 5.9548872180451125e-05,
"loss": 0.9142,
"step": 8
},
{
"epoch": 9.0,
"learning_rate": 5.9097744360902255e-05,
"loss": 0.8391,
"step": 9
},
{
"epoch": 10.0,
"learning_rate": 5.864661654135338e-05,
"loss": 0.7981,
"step": 10
},
{
"epoch": 11.0,
"learning_rate": 5.8195488721804516e-05,
"loss": 0.7414,
"step": 11
},
{
"epoch": 12.0,
"learning_rate": 5.774436090225564e-05,
"loss": 0.7038,
"step": 12
},
{
"epoch": 13.0,
"learning_rate": 5.729323308270677e-05,
"loss": 0.6573,
"step": 13
},
{
"epoch": 14.0,
"learning_rate": 5.684210526315789e-05,
"loss": 0.626,
"step": 14
},
{
"epoch": 15.0,
"learning_rate": 5.639097744360902e-05,
"loss": 0.6062,
"step": 15
},
{
"epoch": 16.0,
"learning_rate": 5.593984962406015e-05,
"loss": 0.5552,
"step": 16
},
{
"epoch": 17.0,
"learning_rate": 5.548872180451128e-05,
"loss": 0.5236,
"step": 17
},
{
"epoch": 18.0,
"learning_rate": 5.503759398496241e-05,
"loss": 0.4971,
"step": 18
},
{
"epoch": 19.0,
"learning_rate": 5.458646616541354e-05,
"loss": 0.4957,
"step": 19
},
{
"epoch": 20.0,
"learning_rate": 5.413533834586466e-05,
"loss": 0.4573,
"step": 20
},
{
"epoch": 20.0,
"eval_accuracy_parking": NaN,
"eval_accuracy_unlabeled": NaN,
"eval_accuracy_unparking": 0.9828910827636719,
"eval_iou_parking": 0.0,
"eval_iou_unlabeled": 0.0,
"eval_iou_unparking": 0.9828910827636719,
"eval_loss": 0.3023880124092102,
"eval_mean_accuracy": 0.9828910827636719,
"eval_mean_iou": 0.32763036092122394,
"eval_overall_accuracy": 0.9828910827636719,
"eval_runtime": 1.0267,
"eval_samples_per_second": 0.974,
"eval_steps_per_second": 0.974,
"step": 20
},
{
"epoch": 21.0,
"learning_rate": 5.368421052631579e-05,
"loss": 0.4436,
"step": 21
},
{
"epoch": 22.0,
"learning_rate": 5.3233082706766915e-05,
"loss": 0.4299,
"step": 22
},
{
"epoch": 23.0,
"learning_rate": 5.2781954887218045e-05,
"loss": 0.407,
"step": 23
},
{
"epoch": 24.0,
"learning_rate": 5.233082706766917e-05,
"loss": 0.3916,
"step": 24
},
{
"epoch": 25.0,
"learning_rate": 5.1879699248120305e-05,
"loss": 0.3743,
"step": 25
},
{
"epoch": 26.0,
"learning_rate": 5.142857142857143e-05,
"loss": 0.3573,
"step": 26
},
{
"epoch": 27.0,
"learning_rate": 5.097744360902256e-05,
"loss": 0.3571,
"step": 27
},
{
"epoch": 28.0,
"learning_rate": 5.052631578947368e-05,
"loss": 0.3352,
"step": 28
},
{
"epoch": 29.0,
"learning_rate": 5.007518796992481e-05,
"loss": 0.3284,
"step": 29
},
{
"epoch": 30.0,
"learning_rate": 4.9624060150375936e-05,
"loss": 0.3157,
"step": 30
},
{
"epoch": 31.0,
"learning_rate": 4.917293233082707e-05,
"loss": 0.303,
"step": 31
},
{
"epoch": 32.0,
"learning_rate": 4.87218045112782e-05,
"loss": 0.2972,
"step": 32
},
{
"epoch": 33.0,
"learning_rate": 4.827067669172933e-05,
"loss": 0.278,
"step": 33
},
{
"epoch": 34.0,
"learning_rate": 4.781954887218045e-05,
"loss": 0.2743,
"step": 34
},
{
"epoch": 35.0,
"learning_rate": 4.736842105263158e-05,
"loss": 0.2677,
"step": 35
},
{
"epoch": 36.0,
"learning_rate": 4.6917293233082704e-05,
"loss": 0.2473,
"step": 36
},
{
"epoch": 37.0,
"learning_rate": 4.6466165413533835e-05,
"loss": 0.2457,
"step": 37
},
{
"epoch": 38.0,
"learning_rate": 4.601503759398496e-05,
"loss": 0.2478,
"step": 38
},
{
"epoch": 39.0,
"learning_rate": 4.5563909774436095e-05,
"loss": 0.233,
"step": 39
},
{
"epoch": 40.0,
"learning_rate": 4.511278195488722e-05,
"loss": 0.2183,
"step": 40
},
{
"epoch": 40.0,
"eval_accuracy_parking": NaN,
"eval_accuracy_unlabeled": NaN,
"eval_accuracy_unparking": 0.9953346252441406,
"eval_iou_parking": 0.0,
"eval_iou_unlabeled": 0.0,
"eval_iou_unparking": 0.9953346252441406,
"eval_loss": 0.2365146428346634,
"eval_mean_accuracy": 0.9953346252441406,
"eval_mean_iou": 0.33177820841471356,
"eval_overall_accuracy": 0.9953346252441406,
"eval_runtime": 1.0332,
"eval_samples_per_second": 0.968,
"eval_steps_per_second": 0.968,
"step": 40
},
{
"epoch": 41.0,
"learning_rate": 4.466165413533835e-05,
"loss": 0.2116,
"step": 41
},
{
"epoch": 42.0,
"learning_rate": 4.421052631578947e-05,
"loss": 0.2048,
"step": 42
},
{
"epoch": 43.0,
"learning_rate": 4.37593984962406e-05,
"loss": 0.2003,
"step": 43
},
{
"epoch": 44.0,
"learning_rate": 4.3308270676691726e-05,
"loss": 0.1931,
"step": 44
},
{
"epoch": 45.0,
"learning_rate": 4.2857142857142856e-05,
"loss": 0.1859,
"step": 45
},
{
"epoch": 46.0,
"learning_rate": 4.2406015037593987e-05,
"loss": 0.1777,
"step": 46
},
{
"epoch": 47.0,
"learning_rate": 4.195488721804512e-05,
"loss": 0.179,
"step": 47
},
{
"epoch": 48.0,
"learning_rate": 4.150375939849624e-05,
"loss": 0.1746,
"step": 48
},
{
"epoch": 49.0,
"learning_rate": 4.105263157894737e-05,
"loss": 0.1659,
"step": 49
},
{
"epoch": 50.0,
"learning_rate": 4.0601503759398494e-05,
"loss": 0.1637,
"step": 50
},
{
"epoch": 51.0,
"learning_rate": 4.0150375939849624e-05,
"loss": 0.1604,
"step": 51
},
{
"epoch": 52.0,
"learning_rate": 3.969924812030075e-05,
"loss": 0.1565,
"step": 52
},
{
"epoch": 53.0,
"learning_rate": 3.9248120300751885e-05,
"loss": 0.1499,
"step": 53
},
{
"epoch": 54.0,
"learning_rate": 3.879699248120301e-05,
"loss": 0.1483,
"step": 54
},
{
"epoch": 55.0,
"learning_rate": 3.834586466165414e-05,
"loss": 0.1445,
"step": 55
},
{
"epoch": 56.0,
"learning_rate": 3.789473684210526e-05,
"loss": 0.1382,
"step": 56
},
{
"epoch": 57.0,
"learning_rate": 3.744360902255639e-05,
"loss": 0.1382,
"step": 57
},
{
"epoch": 58.0,
"learning_rate": 3.6992481203007516e-05,
"loss": 0.1317,
"step": 58
},
{
"epoch": 59.0,
"learning_rate": 3.6541353383458646e-05,
"loss": 0.1292,
"step": 59
},
{
"epoch": 60.0,
"learning_rate": 3.6090225563909776e-05,
"loss": 0.1266,
"step": 60
},
{
"epoch": 60.0,
"eval_accuracy_parking": NaN,
"eval_accuracy_unlabeled": NaN,
"eval_accuracy_unparking": 1.0,
"eval_iou_parking": NaN,
"eval_iou_unlabeled": NaN,
"eval_iou_unparking": 1.0,
"eval_loss": 0.09994122385978699,
"eval_mean_accuracy": 1.0,
"eval_mean_iou": 1.0,
"eval_overall_accuracy": 1.0,
"eval_runtime": 1.0058,
"eval_samples_per_second": 0.994,
"eval_steps_per_second": 0.994,
"step": 60
},
{
"epoch": 61.0,
"learning_rate": 3.5639097744360906e-05,
"loss": 0.1225,
"step": 61
},
{
"epoch": 62.0,
"learning_rate": 3.518796992481203e-05,
"loss": 0.1212,
"step": 62
},
{
"epoch": 63.0,
"learning_rate": 3.473684210526316e-05,
"loss": 0.1215,
"step": 63
},
{
"epoch": 64.0,
"learning_rate": 3.4285714285714284e-05,
"loss": 0.117,
"step": 64
},
{
"epoch": 65.0,
"learning_rate": 3.3834586466165414e-05,
"loss": 0.1158,
"step": 65
},
{
"epoch": 66.0,
"learning_rate": 3.338345864661654e-05,
"loss": 0.1139,
"step": 66
},
{
"epoch": 67.0,
"learning_rate": 3.2932330827067674e-05,
"loss": 0.1124,
"step": 67
},
{
"epoch": 68.0,
"learning_rate": 3.24812030075188e-05,
"loss": 0.1127,
"step": 68
},
{
"epoch": 69.0,
"learning_rate": 3.203007518796993e-05,
"loss": 0.1311,
"step": 69
},
{
"epoch": 70.0,
"learning_rate": 3.157894736842105e-05,
"loss": 0.1114,
"step": 70
},
{
"epoch": 71.0,
"learning_rate": 3.112781954887218e-05,
"loss": 0.1072,
"step": 71
},
{
"epoch": 72.0,
"learning_rate": 3.0676691729323305e-05,
"loss": 0.1122,
"step": 72
},
{
"epoch": 73.0,
"learning_rate": 3.022556390977444e-05,
"loss": 0.1026,
"step": 73
},
{
"epoch": 74.0,
"learning_rate": 2.9774436090225562e-05,
"loss": 0.106,
"step": 74
},
{
"epoch": 75.0,
"learning_rate": 2.932330827067669e-05,
"loss": 0.1034,
"step": 75
},
{
"epoch": 76.0,
"learning_rate": 2.887218045112782e-05,
"loss": 0.0995,
"step": 76
},
{
"epoch": 77.0,
"learning_rate": 2.8421052631578946e-05,
"loss": 0.0992,
"step": 77
},
{
"epoch": 78.0,
"learning_rate": 2.7969924812030073e-05,
"loss": 0.0971,
"step": 78
},
{
"epoch": 79.0,
"learning_rate": 2.7518796992481204e-05,
"loss": 0.0946,
"step": 79
},
{
"epoch": 80.0,
"learning_rate": 2.706766917293233e-05,
"loss": 0.0929,
"step": 80
},
{
"epoch": 80.0,
"eval_accuracy_parking": NaN,
"eval_accuracy_unlabeled": NaN,
"eval_accuracy_unparking": 0.9971656799316406,
"eval_iou_parking": 0.0,
"eval_iou_unlabeled": NaN,
"eval_iou_unparking": 0.9971656799316406,
"eval_loss": 0.05897674709558487,
"eval_mean_accuracy": 0.9971656799316406,
"eval_mean_iou": 0.4985828399658203,
"eval_overall_accuracy": 0.9971656799316406,
"eval_runtime": 1.0009,
"eval_samples_per_second": 0.999,
"eval_steps_per_second": 0.999,
"step": 80
}
],
"max_steps": 140,
"num_train_epochs": 140,
"total_flos": 1.0219948916657357e+18,
"trial_name": null,
"trial_params": null
}