Training in progress, step 50, checkpoint

last-checkpoint/optimizer.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:a421ecbb7a502e3d566f7d7879bcac0582f80af8f36b01e36d90b10cd4bfa045
+oid sha256:5f2583f7cea706cd5ea53e58038adce2ae4c5ba13ac0e381ea5f9242c2ae5183
 size 328468404

last-checkpoint/rng_state.pth

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:5526b8b7ff4c20d736e87c83b42b9d48f82bd219d9b3219996342c1bb2560fce
+oid sha256:285e1ae4ae4be7a31c8e1753fe690802b6e4d5bdbc22e242a4e9c46edc598be2
 size 14244

last-checkpoint/scheduler.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:8c931ece4da598a541d357f6c98f67481603252e193960022d37ddd49c584b1f
+oid sha256:b1df0528620c07325b8faa7567e59b0c1e86a1f1ee6af1245a69c6c0463fe4e2
 size 1064

last-checkpoint/trainer_state.json

@@ -1,9 +1,9 @@
 {
   "best_metric": NaN,
   "best_model_checkpoint": "miner_id_24/checkpoint-25",
-  "epoch": 0.9174311926605505,
+  "epoch": 1.834862385321101,
   "eval_steps": 25,
-  "global_step": 25,
+  "global_step": 50,
   "is_hyper_param_search": false,
   "is_local_process_zero": true,
   "is_world_process_zero": true,
@@ -198,6 +198,189 @@
       "eval_samples_per_second": 14.79,
       "eval_steps_per_second": 1.929,
       "step": 25
+    },
+    {
+      "epoch": 0.9541284403669725,
+      "grad_norm": NaN,
+      "learning_rate": 9.045084971874738e-05,
+      "loss": 0.0,
+      "step": 26
+    },
+    {
+      "epoch": 0.9908256880733946,
+      "grad_norm": NaN,
+      "learning_rate": 8.715724127386972e-05,
+      "loss": 0.0,
+      "step": 27
+    },
+    {
+      "epoch": 1.0275229357798166,
+      "grad_norm": NaN,
+      "learning_rate": 8.345653031794292e-05,
+      "loss": 0.0,
+      "step": 28
+    },
+    {
+      "epoch": 1.0642201834862386,
+      "grad_norm": NaN,
+      "learning_rate": 7.938926261462366e-05,
+      "loss": 0.0,
+      "step": 29
+    },
+    {
+      "epoch": 1.1009174311926606,
+      "grad_norm": NaN,
+      "learning_rate": 7.500000000000001e-05,
+      "loss": 0.0,
+      "step": 30
+    },
+    {
+      "epoch": 1.1376146788990826,
+      "grad_norm": NaN,
+      "learning_rate": 7.033683215379002e-05,
+      "loss": 0.0,
+      "step": 31
+    },
+    {
+      "epoch": 1.1743119266055047,
+      "grad_norm": NaN,
+      "learning_rate": 6.545084971874738e-05,
+      "loss": 0.0,
+      "step": 32
+    },
+    {
+      "epoch": 1.2110091743119267,
+      "grad_norm": NaN,
+      "learning_rate": 6.0395584540887963e-05,
+      "loss": 0.0,
+      "step": 33
+    },
+    {
+      "epoch": 1.2477064220183487,
+      "grad_norm": NaN,
+      "learning_rate": 5.522642316338268e-05,
+      "loss": 0.0,
+      "step": 34
+    },
+    {
+      "epoch": 1.2844036697247707,
+      "grad_norm": NaN,
+      "learning_rate": 5e-05,
+      "loss": 0.0,
+      "step": 35
+    },
+    {
+      "epoch": 1.3211009174311927,
+      "grad_norm": NaN,
+      "learning_rate": 4.477357683661734e-05,
+      "loss": 0.0,
+      "step": 36
+    },
+    {
+      "epoch": 1.3577981651376148,
+      "grad_norm": NaN,
+      "learning_rate": 3.960441545911204e-05,
+      "loss": 0.0,
+      "step": 37
+    },
+    {
+      "epoch": 1.3944954128440368,
+      "grad_norm": NaN,
+      "learning_rate": 3.4549150281252636e-05,
+      "loss": 0.0,
+      "step": 38
+    },
+    {
+      "epoch": 1.4311926605504588,
+      "grad_norm": NaN,
+      "learning_rate": 2.9663167846209998e-05,
+      "loss": 0.0,
+      "step": 39
+    },
+    {
+      "epoch": 1.4678899082568808,
+      "grad_norm": NaN,
+      "learning_rate": 2.500000000000001e-05,
+      "loss": 0.0,
+      "step": 40
+    },
+    {
+      "epoch": 1.5045871559633026,
+      "grad_norm": NaN,
+      "learning_rate": 2.061073738537635e-05,
+      "loss": 0.0,
+      "step": 41
+    },
+    {
+      "epoch": 1.5412844036697249,
+      "grad_norm": NaN,
+      "learning_rate": 1.6543469682057106e-05,
+      "loss": 0.0,
+      "step": 42
+    },
+    {
+      "epoch": 1.5779816513761467,
+      "grad_norm": NaN,
+      "learning_rate": 1.2842758726130283e-05,
+      "loss": 0.0,
+      "step": 43
+    },
+    {
+      "epoch": 1.614678899082569,
+      "grad_norm": NaN,
+      "learning_rate": 9.549150281252633e-06,
+      "loss": 0.0,
+      "step": 44
+    },
+    {
+      "epoch": 1.6513761467889907,
+      "grad_norm": NaN,
+      "learning_rate": 6.698729810778065e-06,
+      "loss": 0.0,
+      "step": 45
+    },
+    {
+      "epoch": 1.688073394495413,
+      "grad_norm": NaN,
+      "learning_rate": 4.322727117869951e-06,
+      "loss": 0.0,
+      "step": 46
+    },
+    {
+      "epoch": 1.7247706422018347,
+      "grad_norm": NaN,
+      "learning_rate": 2.4471741852423237e-06,
+      "loss": 0.0,
+      "step": 47
+    },
+    {
+      "epoch": 1.761467889908257,
+      "grad_norm": NaN,
+      "learning_rate": 1.0926199633097157e-06,
+      "loss": 0.0,
+      "step": 48
+    },
+    {
+      "epoch": 1.7981651376146788,
+      "grad_norm": NaN,
+      "learning_rate": 2.7390523158633554e-07,
+      "loss": 0.0,
+      "step": 49
+    },
+    {
+      "epoch": 1.834862385321101,
+      "grad_norm": NaN,
+      "learning_rate": 0.0,
+      "loss": 0.0,
+      "step": 50
+    },
+    {
+      "epoch": 1.834862385321101,
+      "eval_loss": NaN,
+      "eval_runtime": 3.1103,
+      "eval_samples_per_second": 14.789,
+      "eval_steps_per_second": 1.929,
+      "step": 50
     }
   ],
   "logging_steps": 1,
@@ -212,7 +395,7 @@
         "early_stopping_threshold": 0.0
       },
       "attributes": {
-        "early_stopping_patience_counter": 0
+        "early_stopping_patience_counter": 1
       }
     },
     "TrainerControl": {
@@ -221,12 +404,12 @@
         "should_evaluate": false,
         "should_log": false,
         "should_save": true,
-        "should_training_stop": false
+        "should_training_stop": true
       },
       "attributes": {}
     }
   },
-  "total_flos": 3.5547217133568e+16,
+  "total_flos": 7.078339611721728e+16,
   "train_batch_size": 8,
   "trial_name": null,
   "trial_params": null