bpdevai's picture
Training in progress, epoch 7, checkpoint
dca167c verified
{
"best_metric": 0.1652,
"best_model_checkpoint": "test_detr_finetuned_cppe5\\checkpoint-749",
"epoch": 7.0,
"eval_steps": 500,
"global_step": 749,
"is_hyper_param_search": false,
"is_local_process_zero": true,
"is_world_process_zero": true,
"log_history": [
{
"epoch": 1.0,
"step": 107,
"train_loss": 2.531707286834717,
"train_map": 0.0033,
"train_map_50": 0.0117,
"train_map_75": 0.0011,
"train_map_Coverall": 0.0032,
"train_map_Face_Shield": 0.0001,
"train_map_Gloves": 0.0049,
"train_map_Goggles": 0.0,
"train_map_Mask": 0.0081,
"train_map_large": 0.0063,
"train_map_medium": 0.0037,
"train_map_small": 0.0017,
"train_mar_1": 0.0094,
"train_mar_10": 0.0727,
"train_mar_100": 0.1382,
"train_mar_100_Coverall": 0.2272,
"train_mar_100_Face_Shield": 0.0201,
"train_mar_100_Gloves": 0.199,
"train_mar_100_Goggles": 0.0,
"train_mar_100_Mask": 0.2445,
"train_mar_large": 0.0983,
"train_mar_medium": 0.1363,
"train_mar_small": 0.0346,
"train_runtime": 29.266,
"train_samples_per_second": 29.044,
"train_steps_per_second": 3.656
},
{
"epoch": 1.0,
"eval_loss": 2.618133068084717,
"eval_map": 0.0023,
"eval_map_50": 0.0084,
"eval_map_75": 0.0006,
"eval_map_Coverall": 0.0025,
"eval_map_Face_Shield": 0.0002,
"eval_map_Gloves": 0.0026,
"eval_map_Goggles": 0.0,
"eval_map_Mask": 0.0062,
"eval_map_large": 0.0075,
"eval_map_medium": 0.0029,
"eval_map_small": 0.0019,
"eval_mar_1": 0.0082,
"eval_mar_10": 0.0496,
"eval_mar_100": 0.1009,
"eval_mar_100_Coverall": 0.1667,
"eval_mar_100_Face_Shield": 0.0152,
"eval_mar_100_Gloves": 0.1442,
"eval_mar_100_Goggles": 0.0,
"eval_mar_100_Mask": 0.1782,
"eval_mar_large": 0.1089,
"eval_mar_medium": 0.1196,
"eval_mar_small": 0.0338,
"eval_runtime": 10.5496,
"eval_samples_per_second": 14.218,
"eval_steps_per_second": 1.801,
"step": 107
},
{
"epoch": 2.0,
"step": 214,
"train_loss": 2.1007583141326904,
"train_map": 0.0405,
"train_map_50": 0.0973,
"train_map_75": 0.0262,
"train_map_Coverall": 0.0862,
"train_map_Face_Shield": 0.0034,
"train_map_Gloves": 0.0161,
"train_map_Goggles": 0.0014,
"train_map_Mask": 0.0954,
"train_map_large": 0.0499,
"train_map_medium": 0.0351,
"train_map_small": 0.0156,
"train_mar_1": 0.1017,
"train_mar_10": 0.2491,
"train_mar_100": 0.2902,
"train_mar_100_Coverall": 0.5555,
"train_mar_100_Face_Shield": 0.1718,
"train_mar_100_Gloves": 0.2798,
"train_mar_100_Goggles": 0.0571,
"train_mar_100_Mask": 0.3865,
"train_mar_large": 0.2549,
"train_mar_medium": 0.2432,
"train_mar_small": 0.1161,
"train_runtime": 26.5597,
"train_samples_per_second": 32.003,
"train_steps_per_second": 4.029
},
{
"epoch": 2.0,
"eval_loss": 2.251506805419922,
"eval_map": 0.0386,
"eval_map_50": 0.0946,
"eval_map_75": 0.0261,
"eval_map_Coverall": 0.1234,
"eval_map_Face_Shield": 0.005,
"eval_map_Gloves": 0.0102,
"eval_map_Goggles": 0.0011,
"eval_map_Mask": 0.0531,
"eval_map_large": 0.0645,
"eval_map_medium": 0.0251,
"eval_map_small": 0.0109,
"eval_mar_1": 0.0744,
"eval_mar_10": 0.1938,
"eval_mar_100": 0.2324,
"eval_mar_100_Coverall": 0.4653,
"eval_mar_100_Face_Shield": 0.1722,
"eval_mar_100_Gloves": 0.2165,
"eval_mar_100_Goggles": 0.0277,
"eval_mar_100_Mask": 0.2804,
"eval_mar_large": 0.3155,
"eval_mar_medium": 0.2153,
"eval_mar_small": 0.0878,
"eval_runtime": 9.8776,
"eval_samples_per_second": 15.186,
"eval_steps_per_second": 1.924,
"step": 214
},
{
"epoch": 3.0,
"step": 321,
"train_loss": 1.8027496337890625,
"train_map": 0.0689,
"train_map_50": 0.1515,
"train_map_75": 0.053,
"train_map_Coverall": 0.1899,
"train_map_Face_Shield": 0.0098,
"train_map_Gloves": 0.0287,
"train_map_Goggles": 0.0123,
"train_map_Mask": 0.104,
"train_map_large": 0.0958,
"train_map_medium": 0.0492,
"train_map_small": 0.0201,
"train_mar_1": 0.1643,
"train_mar_10": 0.3427,
"train_mar_100": 0.3901,
"train_mar_100_Coverall": 0.6673,
"train_mar_100_Face_Shield": 0.3124,
"train_mar_100_Gloves": 0.3556,
"train_mar_100_Goggles": 0.2114,
"train_mar_100_Mask": 0.4038,
"train_mar_large": 0.4421,
"train_mar_medium": 0.3085,
"train_mar_small": 0.1399,
"train_runtime": 25.8607,
"train_samples_per_second": 32.868,
"train_steps_per_second": 4.138
},
{
"epoch": 3.0,
"eval_loss": 1.9757874011993408,
"eval_map": 0.0539,
"eval_map_50": 0.1302,
"eval_map_75": 0.0411,
"eval_map_Coverall": 0.1643,
"eval_map_Face_Shield": 0.0092,
"eval_map_Gloves": 0.0261,
"eval_map_Goggles": 0.0098,
"eval_map_Mask": 0.0601,
"eval_map_large": 0.1194,
"eval_map_medium": 0.0401,
"eval_map_small": 0.0231,
"eval_mar_1": 0.1144,
"eval_mar_10": 0.2597,
"eval_mar_100": 0.304,
"eval_mar_100_Coverall": 0.5284,
"eval_mar_100_Face_Shield": 0.243,
"eval_mar_100_Gloves": 0.2942,
"eval_mar_100_Goggles": 0.1446,
"eval_mar_100_Mask": 0.3098,
"eval_mar_large": 0.4992,
"eval_mar_medium": 0.259,
"eval_mar_small": 0.1355,
"eval_runtime": 9.6967,
"eval_samples_per_second": 15.469,
"eval_steps_per_second": 1.959,
"step": 321
},
{
"epoch": 4.0,
"step": 428,
"train_loss": 1.6371513605117798,
"train_map": 0.1256,
"train_map_50": 0.2579,
"train_map_75": 0.1074,
"train_map_Coverall": 0.3276,
"train_map_Face_Shield": 0.0195,
"train_map_Gloves": 0.0689,
"train_map_Goggles": 0.0266,
"train_map_Mask": 0.1852,
"train_map_large": 0.1425,
"train_map_medium": 0.094,
"train_map_small": 0.0312,
"train_mar_1": 0.2065,
"train_mar_10": 0.4048,
"train_mar_100": 0.4399,
"train_mar_100_Coverall": 0.7245,
"train_mar_100_Face_Shield": 0.3362,
"train_mar_100_Gloves": 0.3539,
"train_mar_100_Goggles": 0.3075,
"train_mar_100_Mask": 0.4775,
"train_mar_large": 0.5627,
"train_mar_medium": 0.3551,
"train_mar_small": 0.1666,
"train_runtime": 29.1332,
"train_samples_per_second": 29.176,
"train_steps_per_second": 3.673
},
{
"epoch": 4.0,
"eval_loss": 1.8584269285202026,
"eval_map": 0.0933,
"eval_map_50": 0.2047,
"eval_map_75": 0.0782,
"eval_map_Coverall": 0.2757,
"eval_map_Face_Shield": 0.0127,
"eval_map_Gloves": 0.04,
"eval_map_Goggles": 0.0208,
"eval_map_Mask": 0.1173,
"eval_map_large": 0.1536,
"eval_map_medium": 0.0659,
"eval_map_small": 0.0271,
"eval_mar_1": 0.1454,
"eval_mar_10": 0.3083,
"eval_mar_100": 0.3419,
"eval_mar_100_Coverall": 0.605,
"eval_mar_100_Face_Shield": 0.2519,
"eval_mar_100_Gloves": 0.2763,
"eval_mar_100_Goggles": 0.2354,
"eval_mar_100_Mask": 0.3409,
"eval_mar_large": 0.5593,
"eval_mar_medium": 0.2956,
"eval_mar_small": 0.1551,
"eval_runtime": 11.7158,
"eval_samples_per_second": 12.803,
"eval_steps_per_second": 1.622,
"step": 428
},
{
"epoch": 4.672897196261682,
"grad_norm": 17.656822204589844,
"learning_rate": 4.97310930008156e-05,
"loss": 4.0731,
"step": 500
},
{
"epoch": 5.0,
"step": 535,
"train_loss": 1.5120447874069214,
"train_map": 0.1712,
"train_map_50": 0.3563,
"train_map_75": 0.1466,
"train_map_Coverall": 0.4435,
"train_map_Face_Shield": 0.0746,
"train_map_Gloves": 0.0904,
"train_map_Goggles": 0.0174,
"train_map_Mask": 0.2302,
"train_map_large": 0.2072,
"train_map_medium": 0.1168,
"train_map_small": 0.0514,
"train_mar_1": 0.2393,
"train_mar_10": 0.4484,
"train_mar_100": 0.4809,
"train_mar_100_Coverall": 0.7121,
"train_mar_100_Face_Shield": 0.4876,
"train_mar_100_Gloves": 0.4058,
"train_mar_100_Goggles": 0.301,
"train_mar_100_Mask": 0.4979,
"train_mar_large": 0.6526,
"train_mar_medium": 0.394,
"train_mar_small": 0.2152,
"train_runtime": 25.2419,
"train_samples_per_second": 33.674,
"train_steps_per_second": 4.239
},
{
"epoch": 5.0,
"eval_loss": 1.7863460779190063,
"eval_map": 0.122,
"eval_map_50": 0.266,
"eval_map_75": 0.107,
"eval_map_Coverall": 0.3757,
"eval_map_Face_Shield": 0.047,
"eval_map_Gloves": 0.0488,
"eval_map_Goggles": 0.0105,
"eval_map_Mask": 0.1283,
"eval_map_large": 0.218,
"eval_map_medium": 0.0869,
"eval_map_small": 0.0417,
"eval_mar_1": 0.175,
"eval_mar_10": 0.3379,
"eval_mar_100": 0.3696,
"eval_mar_100_Coverall": 0.6068,
"eval_mar_100_Face_Shield": 0.4089,
"eval_mar_100_Gloves": 0.2759,
"eval_mar_100_Goggles": 0.2138,
"eval_mar_100_Mask": 0.3427,
"eval_mar_large": 0.6453,
"eval_mar_medium": 0.328,
"eval_mar_small": 0.1947,
"eval_runtime": 9.6261,
"eval_samples_per_second": 15.583,
"eval_steps_per_second": 1.974,
"step": 535
},
{
"epoch": 6.0,
"step": 642,
"train_loss": 1.3715598583221436,
"train_map": 0.2176,
"train_map_50": 0.443,
"train_map_75": 0.1854,
"train_map_Coverall": 0.5415,
"train_map_Face_Shield": 0.0693,
"train_map_Gloves": 0.143,
"train_map_Goggles": 0.0397,
"train_map_Mask": 0.2944,
"train_map_large": 0.2885,
"train_map_medium": 0.1558,
"train_map_small": 0.0616,
"train_mar_1": 0.2564,
"train_mar_10": 0.469,
"train_mar_100": 0.5014,
"train_mar_100_Coverall": 0.7413,
"train_mar_100_Face_Shield": 0.448,
"train_mar_100_Gloves": 0.4251,
"train_mar_100_Goggles": 0.3951,
"train_mar_100_Mask": 0.4975,
"train_mar_large": 0.666,
"train_mar_medium": 0.4181,
"train_mar_small": 0.2092,
"train_runtime": 47.3337,
"train_samples_per_second": 17.958,
"train_steps_per_second": 2.261
},
{
"epoch": 6.0,
"eval_loss": 1.6572372913360596,
"eval_map": 0.1544,
"eval_map_50": 0.3453,
"eval_map_75": 0.1228,
"eval_map_Coverall": 0.4258,
"eval_map_Face_Shield": 0.0479,
"eval_map_Gloves": 0.0671,
"eval_map_Goggles": 0.0421,
"eval_map_Mask": 0.1892,
"eval_map_large": 0.2912,
"eval_map_medium": 0.1215,
"eval_map_small": 0.0498,
"eval_mar_1": 0.1876,
"eval_mar_10": 0.3566,
"eval_mar_100": 0.3871,
"eval_mar_100_Coverall": 0.6063,
"eval_mar_100_Face_Shield": 0.3696,
"eval_mar_100_Gloves": 0.2946,
"eval_mar_100_Goggles": 0.3108,
"eval_mar_100_Mask": 0.3542,
"eval_mar_large": 0.6243,
"eval_mar_medium": 0.3671,
"eval_mar_small": 0.1839,
"eval_runtime": 13.4387,
"eval_samples_per_second": 11.162,
"eval_steps_per_second": 1.414,
"step": 642
},
{
"epoch": 7.0,
"step": 749,
"train_loss": 1.3229799270629883,
"train_map": 0.2439,
"train_map_50": 0.5149,
"train_map_75": 0.1888,
"train_map_Coverall": 0.5814,
"train_map_Face_Shield": 0.1397,
"train_map_Gloves": 0.1842,
"train_map_Goggles": 0.0718,
"train_map_Mask": 0.2422,
"train_map_large": 0.335,
"train_map_medium": 0.1703,
"train_map_small": 0.0787,
"train_mar_1": 0.2693,
"train_mar_10": 0.4628,
"train_mar_100": 0.4818,
"train_mar_100_Coverall": 0.743,
"train_mar_100_Face_Shield": 0.4908,
"train_mar_100_Gloves": 0.394,
"train_mar_100_Goggles": 0.3821,
"train_mar_100_Mask": 0.399,
"train_mar_large": 0.7072,
"train_mar_medium": 0.3997,
"train_mar_small": 0.176,
"train_runtime": 61.0765,
"train_samples_per_second": 13.917,
"train_steps_per_second": 1.752
},
{
"epoch": 7.0,
"eval_loss": 1.6533563137054443,
"eval_map": 0.1652,
"eval_map_50": 0.385,
"eval_map_75": 0.1207,
"eval_map_Coverall": 0.4605,
"eval_map_Face_Shield": 0.059,
"eval_map_Gloves": 0.0861,
"eval_map_Goggles": 0.0643,
"eval_map_Mask": 0.1559,
"eval_map_large": 0.3467,
"eval_map_medium": 0.1242,
"eval_map_small": 0.0537,
"eval_mar_1": 0.1888,
"eval_mar_10": 0.335,
"eval_mar_100": 0.3594,
"eval_mar_100_Coverall": 0.6054,
"eval_mar_100_Face_Shield": 0.3608,
"eval_mar_100_Gloves": 0.2728,
"eval_mar_100_Goggles": 0.2738,
"eval_mar_100_Mask": 0.2844,
"eval_mar_large": 0.6732,
"eval_mar_medium": 0.32,
"eval_mar_small": 0.1293,
"eval_runtime": 21.476,
"eval_samples_per_second": 6.985,
"eval_steps_per_second": 0.885,
"step": 749
}
],
"logging_steps": 500,
"max_steps": 10700,
"num_input_tokens_seen": 0,
"num_train_epochs": 100,
"save_steps": 500,
"stateful_callbacks": {
"TrainerControl": {
"args": {
"should_epoch_stop": false,
"should_evaluate": false,
"should_log": false,
"should_save": true,
"should_training_stop": false
},
"attributes": {}
}
},
"total_flos": 1.06894502620416e+18,
"train_batch_size": 8,
"trial_name": null,
"trial_params": null
}