sam1120's picture
Training in progress, step 80
47e03d1
raw
history blame
12 kB
{
"best_metric": 0.024854104965925217,
"best_model_checkpoint": "/robodata/smodak/Projects/nspl/scripts/terrainseg/training/models/parking-utcustom-train-SF-RGBD-b0_5/checkpoint-80",
"epoch": 80.0,
"global_step": 80,
"is_hyper_param_search": false,
"is_local_process_zero": true,
"is_world_process_zero": true,
"log_history": [
{
"epoch": 1.0,
"learning_rate": 4.375e-05,
"loss": 1.1864,
"step": 1
},
{
"epoch": 2.0,
"learning_rate": 8.75e-05,
"loss": 1.1854,
"step": 2
},
{
"epoch": 3.0,
"learning_rate": 0.00013125,
"loss": 1.1605,
"step": 3
},
{
"epoch": 4.0,
"learning_rate": 0.000175,
"loss": 1.1391,
"step": 4
},
{
"epoch": 5.0,
"learning_rate": 0.00021875,
"loss": 1.0912,
"step": 5
},
{
"epoch": 6.0,
"learning_rate": 0.0002625,
"loss": 1.0755,
"step": 6
},
{
"epoch": 7.0,
"learning_rate": 0.00030625,
"loss": 1.0185,
"step": 7
},
{
"epoch": 8.0,
"learning_rate": 0.00035,
"loss": 1.0217,
"step": 8
},
{
"epoch": 9.0,
"learning_rate": 0.0003475352112676056,
"loss": 0.9504,
"step": 9
},
{
"epoch": 10.0,
"learning_rate": 0.00034507042253521125,
"loss": 0.8592,
"step": 10
},
{
"epoch": 11.0,
"learning_rate": 0.00034260563380281693,
"loss": 0.7571,
"step": 11
},
{
"epoch": 12.0,
"learning_rate": 0.00034014084507042255,
"loss": 0.7478,
"step": 12
},
{
"epoch": 13.0,
"learning_rate": 0.0003376760563380282,
"loss": 0.6777,
"step": 13
},
{
"epoch": 14.0,
"learning_rate": 0.0003352112676056338,
"loss": 0.7145,
"step": 14
},
{
"epoch": 15.0,
"learning_rate": 0.00033274647887323943,
"loss": 0.646,
"step": 15
},
{
"epoch": 16.0,
"learning_rate": 0.00033028169014084506,
"loss": 0.5969,
"step": 16
},
{
"epoch": 17.0,
"learning_rate": 0.0003278169014084507,
"loss": 0.5562,
"step": 17
},
{
"epoch": 18.0,
"learning_rate": 0.0003253521126760563,
"loss": 0.5163,
"step": 18
},
{
"epoch": 19.0,
"learning_rate": 0.000322887323943662,
"loss": 0.485,
"step": 19
},
{
"epoch": 20.0,
"learning_rate": 0.0003204225352112676,
"loss": 0.4985,
"step": 20
},
{
"epoch": 20.0,
"eval_accuracy_parking": NaN,
"eval_accuracy_unlabeled": NaN,
"eval_accuracy_unparking": 0.9995574951171875,
"eval_iou_parking": 0.0,
"eval_iou_unlabeled": NaN,
"eval_iou_unparking": 0.9995574951171875,
"eval_loss": 0.48396068811416626,
"eval_mean_accuracy": 0.9995574951171875,
"eval_mean_iou": 0.49977874755859375,
"eval_overall_accuracy": 0.9995574951171875,
"eval_runtime": 0.6588,
"eval_samples_per_second": 1.518,
"eval_steps_per_second": 1.518,
"step": 20
},
{
"epoch": 21.0,
"learning_rate": 0.00031795774647887324,
"loss": 0.459,
"step": 21
},
{
"epoch": 22.0,
"learning_rate": 0.00031549295774647887,
"loss": 0.4079,
"step": 22
},
{
"epoch": 23.0,
"learning_rate": 0.0003130281690140845,
"loss": 0.3978,
"step": 23
},
{
"epoch": 24.0,
"learning_rate": 0.00031056338028169017,
"loss": 0.407,
"step": 24
},
{
"epoch": 25.0,
"learning_rate": 0.0003080985915492958,
"loss": 0.3762,
"step": 25
},
{
"epoch": 26.0,
"learning_rate": 0.00030563380281690137,
"loss": 0.3658,
"step": 26
},
{
"epoch": 27.0,
"learning_rate": 0.00030316901408450705,
"loss": 0.3377,
"step": 27
},
{
"epoch": 28.0,
"learning_rate": 0.0003007042253521127,
"loss": 0.3306,
"step": 28
},
{
"epoch": 29.0,
"learning_rate": 0.0002982394366197183,
"loss": 0.3057,
"step": 29
},
{
"epoch": 30.0,
"learning_rate": 0.0002957746478873239,
"loss": 0.2874,
"step": 30
},
{
"epoch": 31.0,
"learning_rate": 0.00029330985915492955,
"loss": 0.2998,
"step": 31
},
{
"epoch": 32.0,
"learning_rate": 0.00029084507042253523,
"loss": 0.2695,
"step": 32
},
{
"epoch": 33.0,
"learning_rate": 0.00028838028169014086,
"loss": 0.2571,
"step": 33
},
{
"epoch": 34.0,
"learning_rate": 0.0002859154929577465,
"loss": 0.2506,
"step": 34
},
{
"epoch": 35.0,
"learning_rate": 0.0002834507042253521,
"loss": 0.2423,
"step": 35
},
{
"epoch": 36.0,
"learning_rate": 0.00028098591549295774,
"loss": 0.2311,
"step": 36
},
{
"epoch": 37.0,
"learning_rate": 0.0002785211267605634,
"loss": 0.2209,
"step": 37
},
{
"epoch": 38.0,
"learning_rate": 0.000276056338028169,
"loss": 0.2123,
"step": 38
},
{
"epoch": 39.0,
"learning_rate": 0.0002735915492957746,
"loss": 0.1999,
"step": 39
},
{
"epoch": 40.0,
"learning_rate": 0.0002711267605633803,
"loss": 0.1979,
"step": 40
},
{
"epoch": 40.0,
"eval_accuracy_parking": NaN,
"eval_accuracy_unlabeled": NaN,
"eval_accuracy_unparking": 1.0,
"eval_iou_parking": NaN,
"eval_iou_unlabeled": NaN,
"eval_iou_unparking": 1.0,
"eval_loss": 0.13647860288619995,
"eval_mean_accuracy": 1.0,
"eval_mean_iou": 1.0,
"eval_overall_accuracy": 1.0,
"eval_runtime": 0.6254,
"eval_samples_per_second": 1.599,
"eval_steps_per_second": 1.599,
"step": 40
},
{
"epoch": 41.0,
"learning_rate": 0.0002686619718309859,
"loss": 0.1859,
"step": 41
},
{
"epoch": 42.0,
"learning_rate": 0.00026619718309859155,
"loss": 0.191,
"step": 42
},
{
"epoch": 43.0,
"learning_rate": 0.00026373239436619717,
"loss": 0.1762,
"step": 43
},
{
"epoch": 44.0,
"learning_rate": 0.0002612676056338028,
"loss": 0.1735,
"step": 44
},
{
"epoch": 45.0,
"learning_rate": 0.0002588028169014085,
"loss": 0.1719,
"step": 45
},
{
"epoch": 46.0,
"learning_rate": 0.0002563380281690141,
"loss": 0.1691,
"step": 46
},
{
"epoch": 47.0,
"learning_rate": 0.00025387323943661973,
"loss": 0.1599,
"step": 47
},
{
"epoch": 48.0,
"learning_rate": 0.00025140845070422535,
"loss": 0.1567,
"step": 48
},
{
"epoch": 49.0,
"learning_rate": 0.000248943661971831,
"loss": 0.1545,
"step": 49
},
{
"epoch": 50.0,
"learning_rate": 0.0002464788732394366,
"loss": 0.1499,
"step": 50
},
{
"epoch": 51.0,
"learning_rate": 0.00024401408450704223,
"loss": 0.1455,
"step": 51
},
{
"epoch": 52.0,
"learning_rate": 0.00024154929577464788,
"loss": 0.1481,
"step": 52
},
{
"epoch": 53.0,
"learning_rate": 0.0002390845070422535,
"loss": 0.1436,
"step": 53
},
{
"epoch": 54.0,
"learning_rate": 0.00023661971830985914,
"loss": 0.1381,
"step": 54
},
{
"epoch": 55.0,
"learning_rate": 0.0002341549295774648,
"loss": 0.1341,
"step": 55
},
{
"epoch": 56.0,
"learning_rate": 0.00023169014084507041,
"loss": 0.1319,
"step": 56
},
{
"epoch": 57.0,
"learning_rate": 0.00022922535211267607,
"loss": 0.1312,
"step": 57
},
{
"epoch": 58.0,
"learning_rate": 0.0002267605633802817,
"loss": 0.1276,
"step": 58
},
{
"epoch": 59.0,
"learning_rate": 0.00022429577464788735,
"loss": 0.1236,
"step": 59
},
{
"epoch": 60.0,
"learning_rate": 0.00022183098591549297,
"loss": 0.1257,
"step": 60
},
{
"epoch": 60.0,
"eval_accuracy_parking": NaN,
"eval_accuracy_unlabeled": NaN,
"eval_accuracy_unparking": 1.0,
"eval_iou_parking": NaN,
"eval_iou_unlabeled": NaN,
"eval_iou_unparking": 1.0,
"eval_loss": 0.04137684404850006,
"eval_mean_accuracy": 1.0,
"eval_mean_iou": 1.0,
"eval_overall_accuracy": 1.0,
"eval_runtime": 0.6701,
"eval_samples_per_second": 1.492,
"eval_steps_per_second": 1.492,
"step": 60
},
{
"epoch": 61.0,
"learning_rate": 0.0002193661971830986,
"loss": 0.1247,
"step": 61
},
{
"epoch": 62.0,
"learning_rate": 0.0002169014084507042,
"loss": 0.1194,
"step": 62
},
{
"epoch": 63.0,
"learning_rate": 0.00021443661971830985,
"loss": 0.1189,
"step": 63
},
{
"epoch": 64.0,
"learning_rate": 0.00021197183098591548,
"loss": 0.1139,
"step": 64
},
{
"epoch": 65.0,
"learning_rate": 0.00020950704225352113,
"loss": 0.1131,
"step": 65
},
{
"epoch": 66.0,
"learning_rate": 0.00020704225352112675,
"loss": 0.1136,
"step": 66
},
{
"epoch": 67.0,
"learning_rate": 0.00020457746478873238,
"loss": 0.1076,
"step": 67
},
{
"epoch": 68.0,
"learning_rate": 0.00020211267605633803,
"loss": 0.1095,
"step": 68
},
{
"epoch": 69.0,
"learning_rate": 0.00019964788732394366,
"loss": 0.1081,
"step": 69
},
{
"epoch": 70.0,
"learning_rate": 0.0001971830985915493,
"loss": 0.1043,
"step": 70
},
{
"epoch": 71.0,
"learning_rate": 0.00019471830985915494,
"loss": 0.1053,
"step": 71
},
{
"epoch": 72.0,
"learning_rate": 0.0001922535211267606,
"loss": 0.1066,
"step": 72
},
{
"epoch": 73.0,
"learning_rate": 0.00018978873239436622,
"loss": 0.1066,
"step": 73
},
{
"epoch": 74.0,
"learning_rate": 0.00018732394366197181,
"loss": 0.1066,
"step": 74
},
{
"epoch": 75.0,
"learning_rate": 0.00018485915492957744,
"loss": 0.1001,
"step": 75
},
{
"epoch": 76.0,
"learning_rate": 0.0001823943661971831,
"loss": 0.1014,
"step": 76
},
{
"epoch": 77.0,
"learning_rate": 0.00017992957746478872,
"loss": 0.0991,
"step": 77
},
{
"epoch": 78.0,
"learning_rate": 0.00017746478873239437,
"loss": 0.0993,
"step": 78
},
{
"epoch": 79.0,
"learning_rate": 0.000175,
"loss": 0.0949,
"step": 79
},
{
"epoch": 80.0,
"learning_rate": 0.00017253521126760562,
"loss": 0.0952,
"step": 80
},
{
"epoch": 80.0,
"eval_accuracy_parking": NaN,
"eval_accuracy_unlabeled": NaN,
"eval_accuracy_unparking": 1.0,
"eval_iou_parking": NaN,
"eval_iou_unlabeled": NaN,
"eval_iou_unparking": 1.0,
"eval_loss": 0.024854104965925217,
"eval_mean_accuracy": 1.0,
"eval_mean_iou": 1.0,
"eval_overall_accuracy": 1.0,
"eval_runtime": 0.6895,
"eval_samples_per_second": 1.45,
"eval_steps_per_second": 1.45,
"step": 80
}
],
"max_steps": 150,
"num_train_epochs": 150,
"total_flos": 4.493149300850688e+16,
"trial_name": null,
"trial_params": null
}