Training in progress, step 30000, checkpoint

checkpoint-30000/model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:b6e5d12c3580498868f8cc52857ebe6d22706093c6833be065f348b1f47c116d
+oid sha256:c5fd4aaffba428501a0071e0b13cee0d5a6e446afd9020b2eba12b27b95e989a
 size 540279452

checkpoint-30000/optimizer.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:c070d4e4b6c3e7d2ed5166cc4fd06cd468b218d4d7e8b7af98ba21d94a928ba0
-size 1080630853
+oid sha256:f662a318511f99cd44914a627890ddaeb8a02cae9149cb788c2dd5fc67541f1a
+size 1080631290

checkpoint-30000/rng_state.pth

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:405bf0fde5f15101b4a46fb0fc9d6a1fcc5daee896ba44f804a8afc914801519
-size 14575
+oid sha256:024fcbc019164d406954cbc1020c58497a012b15543429ae667088338d503fad
+size 14244

checkpoint-30000/scheduler.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:9deca91d4ece8a1dec8855243746c88149339cbdaa61e416f10693dec224452c
-size 627
+oid sha256:d8c3364a669f770ad743aad5a0318284839312fa06119649e089f96029cf0663
+size 1064

checkpoint-30000/trainer_state.json

@@ -1,8 +1,8 @@
 {
-  "best_metric": 0.6703549228453575,
+  "best_metric": 0.6717793867023528,
   "best_model_checkpoint": "output/pretraining/vihealthbert-w_unsup-SynPD/lr3e-5_wr0.1_wd0.0/checkpoint-30000",
   "epoch": 5.1697397897639155,
-  "eval_steps": 5000,
+  "eval_steps": 2000,
   "global_step": 30000,
   "is_hyper_param_search": false,
   "is_local_process_zero": true,
@@ -10,113 +10,257 @@
   "log_history": [
     {
       "epoch": 0.00017232465965879716,
-      "grad_norm": 41.827266693115234,
+      "grad_norm": 35.608882904052734,
       "learning_rate": 5.169739789763915e-09,
-      "loss": 13.7242,
+      "loss": 10.7382,
       "step": 1
     },
     {
-      "epoch": 0.8616232982939859,
-      "grad_norm": 5.6151957511901855,
-      "learning_rate": 2.584869894881958e-05,
-      "loss": 7.0234,
-      "step": 5000
+      "epoch": 0.34464931931759435,
+      "grad_norm": 4.884454250335693,
+      "learning_rate": 1.033947957952783e-05,
+      "loss": 5.8543,
+      "step": 2000
     },
     {
-      "epoch": 0.8616232982939859,
-      "eval_accuracy": 0.5576033171446898,
-      "eval_loss": 2.590928077697754,
-      "eval_runtime": 37.589,
-      "eval_samples_per_second": 260.023,
-      "eval_steps_per_second": 16.255,
-      "step": 5000
+      "epoch": 0.34464931931759435,
+      "eval_accuracy": 0.3950195480446223,
+      "eval_loss": 3.896709680557251,
+      "eval_runtime": 24.6709,
+      "eval_samples_per_second": 396.175,
+      "eval_steps_per_second": 24.766,
+      "step": 2000
+    },
+    {
+      "epoch": 0.6892986386351887,
+      "grad_norm": 5.278345108032227,
+      "learning_rate": 2.067895915905566e-05,
+      "loss": 3.4544,
+      "step": 4000
+    },
+    {
+      "epoch": 0.6892986386351887,
+      "eval_accuracy": 0.530626322082222,
+      "eval_loss": 2.811899423599243,
+      "eval_runtime": 24.6443,
+      "eval_samples_per_second": 396.602,
+      "eval_steps_per_second": 24.793,
+      "step": 4000
+    },
+    {
+      "epoch": 1.033947957952783,
+      "grad_norm": 4.731122016906738,
+      "learning_rate": 2.988684014015739e-05,
+      "loss": 2.8312,
+      "step": 6000
+    },
+    {
+      "epoch": 1.033947957952783,
+      "eval_accuracy": 0.5771132321637026,
+      "eval_loss": 2.4039628505706787,
+      "eval_runtime": 24.6266,
+      "eval_samples_per_second": 396.889,
+      "eval_steps_per_second": 24.811,
+      "step": 6000
+    },
+    {
+      "epoch": 1.3785972772703774,
+      "grad_norm": 12.213573455810547,
+      "learning_rate": 2.873800907576541e-05,
+      "loss": 2.5914,
+      "step": 8000
+    },
+    {
+      "epoch": 1.3785972772703774,
+      "eval_accuracy": 0.5349752225383133,
+      "eval_loss": 2.6481692790985107,
+      "eval_runtime": 24.6033,
+      "eval_samples_per_second": 397.264,
+      "eval_steps_per_second": 24.834,
+      "step": 8000
     },
     {
       "epoch": 1.7232465965879717,
-      "grad_norm": 4.410743713378906,
+      "grad_norm": 5.172228813171387,
       "learning_rate": 2.758917801137343e-05,
-      "loss": 5.2736,
+      "loss": 2.5649,
       "step": 10000
     },
     {
       "epoch": 1.7232465965879717,
-      "eval_accuracy": 0.596153624823046,
-      "eval_loss": 2.188957452774048,
-      "eval_runtime": 37.6297,
-      "eval_samples_per_second": 259.741,
-      "eval_steps_per_second": 16.237,
+      "eval_accuracy": 0.6087076297937304,
+      "eval_loss": 2.13348388671875,
+      "eval_runtime": 24.6211,
+      "eval_samples_per_second": 396.976,
+      "eval_steps_per_second": 24.816,
       "step": 10000
     },
     {
-      "epoch": 2.5848698948819577,
-      "grad_norm": 4.754213809967041,
-      "learning_rate": 2.4717100350393475e-05,
-      "loss": 4.9126,
-      "step": 15000
+      "epoch": 2.067895915905566,
+      "grad_norm": 4.871993064880371,
+      "learning_rate": 2.6440346946981447e-05,
+      "loss": 2.2749,
+      "step": 12000
+    },
+    {
+      "epoch": 2.067895915905566,
+      "eval_accuracy": 0.6281529792701336,
+      "eval_loss": 1.9895449876785278,
+      "eval_runtime": 24.453,
+      "eval_samples_per_second": 399.705,
+      "eval_steps_per_second": 24.987,
+      "step": 12000
+    },
+    {
+      "epoch": 2.4125452352231607,
+      "grad_norm": 8.037242889404297,
+      "learning_rate": 2.5291515882589467e-05,
+      "loss": 2.1572,
+      "step": 14000
+    },
+    {
+      "epoch": 2.4125452352231607,
+      "eval_accuracy": 0.6353159252095423,
+      "eval_loss": 1.9312896728515625,
+      "eval_runtime": 24.4658,
+      "eval_samples_per_second": 399.496,
+      "eval_steps_per_second": 24.974,
+      "step": 14000
     },
     {
-      "epoch": 2.5848698948819577,
-      "eval_accuracy": 0.6381037526075694,
-      "eval_loss": 1.909491777420044,
-      "eval_runtime": 36.8556,
-      "eval_samples_per_second": 265.197,
-      "eval_steps_per_second": 16.578,
-      "step": 15000
+      "epoch": 2.757194554540755,
+      "grad_norm": 4.320974349975586,
+      "learning_rate": 2.4142684818197487e-05,
+      "loss": 2.1009,
+      "step": 16000
+    },
+    {
+      "epoch": 2.757194554540755,
+      "eval_accuracy": 0.6428579553856829,
+      "eval_loss": 1.864585280418396,
+      "eval_runtime": 24.4545,
+      "eval_samples_per_second": 399.68,
+      "eval_steps_per_second": 24.985,
+      "step": 16000
+    },
+    {
+      "epoch": 3.101843873858349,
+      "grad_norm": 4.6416168212890625,
+      "learning_rate": 2.2993853753805504e-05,
+      "loss": 2.0609,
+      "step": 18000
+    },
+    {
+      "epoch": 3.101843873858349,
+      "eval_accuracy": 0.6450457483325106,
+      "eval_loss": 1.8571828603744507,
+      "eval_runtime": 24.4634,
+      "eval_samples_per_second": 399.535,
+      "eval_steps_per_second": 24.976,
+      "step": 18000
     },
     {
       "epoch": 3.4464931931759435,
-      "grad_norm": 4.2047200202941895,
+      "grad_norm": 4.341440677642822,
       "learning_rate": 2.184502268941352e-05,
-      "loss": 4.791,
+      "loss": 2.0885,
       "step": 20000
     },
     {
       "epoch": 3.4464931931759435,
-      "eval_accuracy": 0.6468589272593681,
-      "eval_loss": 1.8286069631576538,
-      "eval_runtime": 35.9342,
-      "eval_samples_per_second": 271.997,
-      "eval_steps_per_second": 17.003,
+      "eval_accuracy": 0.6284792764252838,
+      "eval_loss": 1.9489394426345825,
+      "eval_runtime": 24.6433,
+      "eval_samples_per_second": 396.618,
+      "eval_steps_per_second": 24.794,
       "step": 20000
     },
     {
-      "epoch": 4.30811649146993,
-      "grad_norm": 4.629519462585449,
-      "learning_rate": 1.897294502843357e-05,
-      "loss": 4.6538,
-      "step": 25000
+      "epoch": 3.791142512493538,
+      "grad_norm": 4.660053253173828,
+      "learning_rate": 2.069619162502154e-05,
+      "loss": 1.9891,
+      "step": 22000
+    },
+    {
+      "epoch": 3.791142512493538,
+      "eval_accuracy": 0.6582960564701575,
+      "eval_loss": 1.7700080871582031,
+      "eval_runtime": 24.616,
+      "eval_samples_per_second": 397.059,
+      "eval_steps_per_second": 24.821,
+      "step": 22000
+    },
+    {
+      "epoch": 4.135791831811132,
+      "grad_norm": 3.9445292949676514,
+      "learning_rate": 1.954736056062956e-05,
+      "loss": 1.9368,
+      "step": 24000
+    },
+    {
+      "epoch": 4.135791831811132,
+      "eval_accuracy": 0.6608816779512006,
+      "eval_loss": 1.7397598028182983,
+      "eval_runtime": 24.6251,
+      "eval_samples_per_second": 396.912,
+      "eval_steps_per_second": 24.812,
+      "step": 24000
+    },
+    {
+      "epoch": 4.480441151128726,
+      "grad_norm": 4.561331748962402,
+      "learning_rate": 1.839852949623758e-05,
+      "loss": 1.9003,
+      "step": 26000
+    },
+    {
+      "epoch": 4.480441151128726,
+      "eval_accuracy": 0.666368433126864,
+      "eval_loss": 1.7164973020553589,
+      "eval_runtime": 24.4665,
+      "eval_samples_per_second": 399.485,
+      "eval_steps_per_second": 24.973,
+      "step": 26000
+    },
+    {
+      "epoch": 4.825090470446321,
+      "grad_norm": 4.358778476715088,
+      "learning_rate": 1.72496984318456e-05,
+      "loss": 1.9058,
+      "step": 28000
     },
     {
-      "epoch": 4.30811649146993,
-      "eval_accuracy": 0.6644416567883901,
-      "eval_loss": 1.714424967765808,
-      "eval_runtime": 35.6122,
-      "eval_samples_per_second": 274.456,
-      "eval_steps_per_second": 17.157,
-      "step": 25000
+      "epoch": 4.825090470446321,
+      "eval_accuracy": 0.666961039556835,
+      "eval_loss": 1.703224539756775,
+      "eval_runtime": 24.4455,
+      "eval_samples_per_second": 399.829,
+      "eval_steps_per_second": 24.994,
+      "step": 28000
     },
     {
       "epoch": 5.1697397897639155,
-      "grad_norm": 3.993448257446289,
+      "grad_norm": 4.522342205047607,
       "learning_rate": 1.6100867367453616e-05,
-      "loss": 4.5846,
+      "loss": 1.859,
       "step": 30000
     },
     {
       "epoch": 5.1697397897639155,
-      "eval_accuracy": 0.6703549228453575,
-      "eval_loss": 1.6779303550720215,
-      "eval_runtime": 36.1572,
-      "eval_samples_per_second": 270.32,
-      "eval_steps_per_second": 16.898,
+      "eval_accuracy": 0.6717793867023528,
+      "eval_loss": 1.677147626876831,
+      "eval_runtime": 24.4697,
+      "eval_samples_per_second": 399.432,
+      "eval_steps_per_second": 24.97,
       "step": 30000
     }
   ],
-  "logging_steps": 5000,
+  "logging_steps": 2000,
   "max_steps": 58030,
   "num_input_tokens_seen": 0,
   "num_train_epochs": 10,
-  "save_steps": 5000,
+  "save_steps": 2000,
   "total_flos": 0.0,
   "train_batch_size": 32,
   "trial_name": null,

checkpoint-30000/training_args.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:a2690ff9d34fa5fa3d550f183e187fe6edb89806dc8d5d196cb79437ad79b8f1
-size 4859
+oid sha256:828a11e9b75940b4120312d1450c3472eb18a2c7fa0179d8afafbcd0800beb75
+size 5304