End of training

README.md

@@ -2,11 +2,23 @@
 library_name: transformers
 tags:
 - generated_from_trainer
+datasets:
+- kanishka/babylm2-rewritten-clean-spacy_no-num-adj
 metrics:
 - accuracy
 model-index:
 - name: opt-babylm2-rewritten-clean-spacy_no-num-adj-earlystop-bpe_seed-42_1e-3
-  results: []
+  results:
+  - task:
+      name: Causal Language Modeling
+      type: text-generation
+    dataset:
+      name: kanishka/babylm2-rewritten-clean-spacy_no-num-adj
+      type: kanishka/babylm2-rewritten-clean-spacy_no-num-adj
+    metrics:
+    - name: Accuracy
+      type: accuracy
+      value: 0.4781093360218181
 ---
 
 <!-- This model card has been generated automatically according to the information the Trainer had access to. You
@@ -14,7 +26,7 @@ should probably proofread and complete it, then remove this comment. -->
 
 # opt-babylm2-rewritten-clean-spacy_no-num-adj-earlystop-bpe_seed-42_1e-3
 
-This model was trained from scratch on an unknown dataset.
+This model was trained from scratch on the kanishka/babylm2-rewritten-clean-spacy_no-num-adj dataset.
 It achieves the following results on the evaluation set:
 - Loss: 2.6900
 - Accuracy: 0.4781

all_results.json

@@ -1,16 +1,16 @@
 {
-    "epoch": 19.991404977248468,
-    "eval_accuracy": 0.4780018806422093,
-    "eval_loss": 2.688654899597168,
-    "eval_runtime": 71.3254,
-    "eval_samples": 60701,
-    "eval_samples_per_second": 851.044,
-    "eval_steps_per_second": 13.305,
-    "perplexity": 14.711873658228711,
-    "total_flos": 1.487763384827904e+18,
-    "train_loss": 2.800290222853757,
-    "train_runtime": 29774.6232,
-    "train_samples": 569632,
-    "train_samples_per_second": 382.629,
-    "train_steps_per_second": 1.495
+    "epoch": 19.99134539732494,
+    "eval_accuracy": 0.4781093360218181,
+    "eval_loss": 2.690006971359253,
+    "eval_runtime": 71.6993,
+    "eval_samples": 60680,
+    "eval_samples_per_second": 846.313,
+    "eval_steps_per_second": 13.236,
+    "perplexity": 14.73177862060579,
+    "total_flos": 1.487139158163456e+18,
+    "train_loss": 2.8016022716494775,
+    "train_runtime": 30047.0507,
+    "train_samples": 569394,
+    "train_samples_per_second": 379.002,
+    "train_steps_per_second": 1.48
 }

eval_results.json

@@ -1,10 +1,10 @@
 {
-    "epoch": 19.991404977248468,
-    "eval_accuracy": 0.4780018806422093,
-    "eval_loss": 2.688654899597168,
-    "eval_runtime": 71.3254,
-    "eval_samples": 60701,
-    "eval_samples_per_second": 851.044,
-    "eval_steps_per_second": 13.305,
-    "perplexity": 14.711873658228711
+    "epoch": 19.99134539732494,
+    "eval_accuracy": 0.4781093360218181,
+    "eval_loss": 2.690006971359253,
+    "eval_runtime": 71.6993,
+    "eval_samples": 60680,
+    "eval_samples_per_second": 846.313,
+    "eval_steps_per_second": 13.236,
+    "perplexity": 14.73177862060579
 }

train_results.json

@@ -1,9 +1,9 @@
 {
-    "epoch": 19.991404977248468,
-    "total_flos": 1.487763384827904e+18,
-    "train_loss": 2.800290222853757,
-    "train_runtime": 29774.6232,
-    "train_samples": 569632,
-    "train_samples_per_second": 382.629,
-    "train_steps_per_second": 1.495
+    "epoch": 19.99134539732494,
+    "total_flos": 1.487139158163456e+18,
+    "train_loss": 2.8016022716494775,
+    "train_runtime": 30047.0507,
+    "train_samples": 569394,
+    "train_samples_per_second": 379.002,
+    "train_steps_per_second": 1.48
 }

trainer_state.json

@@ -1,513 +1,513 @@
 {
-  "best_metric": 2.688654899597168,
-  "best_model_checkpoint": "models/opt-babylm2-rewritten-clean-spacy_no-num-adj-earlystop-bpe_seed-42_1e-3/checkpoint-44500",
-  "epoch": 19.991404977248468,
+  "best_metric": 2.690006971359253,
+  "best_model_checkpoint": "models/opt-babylm2-rewritten-clean-spacy_no-num-adj-earlystop-bpe_seed-42_1e-3/checkpoint-44480",
+  "epoch": 19.99134539732494,
   "eval_steps": 500,
-  "global_step": 44500,
+  "global_step": 44480,
   "is_hyper_param_search": false,
   "is_local_process_zero": true,
   "is_world_process_zero": true,
   "log_history": [
     {
-      "epoch": 0.4494129543284085,
-      "grad_norm": 0.41069796681404114,
+      "epoch": 0.44958974935371476,
+      "grad_norm": 0.5849416255950928,
       "learning_rate": 3.125e-05,
-      "loss": 5.5764,
+      "loss": 5.5771,
       "step": 1000
     },
     {
-      "epoch": 0.898825908656817,
-      "grad_norm": 0.5602710247039795,
+      "epoch": 0.8991794987074295,
+      "grad_norm": 0.6677811145782471,
       "learning_rate": 6.25e-05,
-      "loss": 4.0881,
+      "loss": 4.0896,
       "step": 2000
     },
     {
       "epoch": 1.0,
-      "eval_accuracy": 0.35879080714178885,
-      "eval_loss": 3.840114116668701,
-      "eval_runtime": 73.233,
-      "eval_samples_per_second": 828.875,
-      "eval_steps_per_second": 12.959,
-      "step": 2226
+      "eval_accuracy": 0.35931850789096126,
+      "eval_loss": 3.830660104751587,
+      "eval_runtime": 73.9335,
+      "eval_samples_per_second": 820.738,
+      "eval_steps_per_second": 12.836,
+      "step": 2225
     },
     {
-      "epoch": 1.3478456266501881,
-      "grad_norm": 0.5852290987968445,
+      "epoch": 1.348432055749129,
+      "grad_norm": 0.5427906513214111,
       "learning_rate": 9.375e-05,
-      "loss": 3.6865,
+      "loss": 3.6885,
       "step": 3000
     },
     {
-      "epoch": 1.7972585809785966,
-      "grad_norm": 0.54194575548172,
+      "epoch": 1.7980218051028438,
+      "grad_norm": 0.5098850727081299,
       "learning_rate": 0.000125,
-      "loss": 3.4347,
+      "loss": 3.4325,
       "step": 4000
     },
     {
       "epoch": 2.0,
-      "eval_accuracy": 0.40807745842608145,
-      "eval_loss": 3.31392502784729,
-      "eval_runtime": 73.0362,
-      "eval_samples_per_second": 831.109,
-      "eval_steps_per_second": 12.994,
-      "step": 4452
+      "eval_accuracy": 0.40814210192976336,
+      "eval_loss": 3.3144962787628174,
+      "eval_runtime": 73.7109,
+      "eval_samples_per_second": 823.216,
+      "eval_steps_per_second": 12.875,
+      "step": 4450
     },
     {
-      "epoch": 2.246278298971968,
-      "grad_norm": 0.4505850672721863,
+      "epoch": 2.2472743621445432,
+      "grad_norm": 0.47539061307907104,
       "learning_rate": 0.00015625,
-      "loss": 3.2332,
+      "loss": 3.2344,
       "step": 5000
     },
     {
-      "epoch": 2.6956912533003763,
-      "grad_norm": 0.5012966394424438,
+      "epoch": 2.696864111498258,
+      "grad_norm": 0.4557252824306488,
       "learning_rate": 0.0001875,
-      "loss": 3.1182,
+      "loss": 3.1208,
       "step": 6000
     },
     {
       "epoch": 3.0,
-      "eval_accuracy": 0.4283898801938528,
-      "eval_loss": 3.1022610664367676,
-      "eval_runtime": 72.6546,
-      "eval_samples_per_second": 835.474,
-      "eval_steps_per_second": 13.062,
-      "step": 6678
+      "eval_accuracy": 0.42951116108935333,
+      "eval_loss": 3.1043691635131836,
+      "eval_runtime": 73.4454,
+      "eval_samples_per_second": 826.192,
+      "eval_steps_per_second": 12.921,
+      "step": 6675
     },
     {
-      "epoch": 3.1447109712937475,
-      "grad_norm": 0.43612048029899597,
+      "epoch": 3.1461166685399573,
+      "grad_norm": 0.4797649085521698,
       "learning_rate": 0.00021875,
-      "loss": 3.0238,
+      "loss": 3.0251,
       "step": 7000
     },
     {
-      "epoch": 3.594123925622156,
-      "grad_norm": 0.38532325625419617,
+      "epoch": 3.595706417893672,
+      "grad_norm": 0.39974284172058105,
       "learning_rate": 0.00025,
-      "loss": 2.9562,
+      "loss": 2.957,
       "step": 8000
     },
     {
       "epoch": 4.0,
-      "eval_accuracy": 0.4398597303206879,
-      "eval_loss": 2.9949750900268555,
-      "eval_runtime": 72.7491,
-      "eval_samples_per_second": 834.388,
-      "eval_steps_per_second": 13.045,
-      "step": 8904
+      "eval_accuracy": 0.43958987682086675,
+      "eval_loss": 2.997297763824463,
+      "eval_runtime": 72.7828,
+      "eval_samples_per_second": 833.713,
+      "eval_steps_per_second": 13.039,
+      "step": 8900
     },
     {
-      "epoch": 4.043143643615527,
-      "grad_norm": 0.3981912136077881,
+      "epoch": 4.044958974935372,
+      "grad_norm": 0.37127092480659485,
       "learning_rate": 0.00028125000000000003,
-      "loss": 2.9072,
+      "loss": 2.91,
       "step": 9000
     },
     {
-      "epoch": 4.492556597943936,
-      "grad_norm": 0.3344660997390747,
+      "epoch": 4.4945487242890865,
+      "grad_norm": 0.35168108344078064,
       "learning_rate": 0.0003125,
-      "loss": 2.8545,
+      "loss": 2.8553,
       "step": 10000
     },
     {
-      "epoch": 4.941969552272345,
-      "grad_norm": 0.337446391582489,
+      "epoch": 4.944138473642801,
+      "grad_norm": 0.33759039640426636,
       "learning_rate": 0.00034375,
-      "loss": 2.8365,
+      "loss": 2.8381,
       "step": 11000
     },
     {
       "epoch": 5.0,
-      "eval_accuracy": 0.44592869387751144,
-      "eval_loss": 2.936906099319458,
-      "eval_runtime": 72.6273,
-      "eval_samples_per_second": 835.788,
-      "eval_steps_per_second": 13.067,
-      "step": 11130
+      "eval_accuracy": 0.4463711272247858,
+      "eval_loss": 2.9338483810424805,
+      "eval_runtime": 73.014,
+      "eval_samples_per_second": 831.073,
+      "eval_steps_per_second": 12.998,
+      "step": 11125
     },
     {
-      "epoch": 5.390989270265715,
-      "grad_norm": 0.3183291256427765,
+      "epoch": 5.3933910306845005,
+      "grad_norm": 0.3382190465927124,
       "learning_rate": 0.000375,
-      "loss": 2.7859,
+      "loss": 2.7866,
       "step": 12000
     },
     {
-      "epoch": 5.840402224594124,
-      "grad_norm": 0.2951858937740326,
+      "epoch": 5.842980780038215,
+      "grad_norm": 0.30114060640335083,
       "learning_rate": 0.00040625000000000004,
-      "loss": 2.7807,
+      "loss": 2.7819,
       "step": 13000
     },
     {
       "epoch": 6.0,
-      "eval_accuracy": 0.4497357830135563,
-      "eval_loss": 2.8994719982147217,
-      "eval_runtime": 72.6109,
-      "eval_samples_per_second": 835.976,
-      "eval_steps_per_second": 13.07,
-      "step": 13356
+      "eval_accuracy": 0.4507991133170473,
+      "eval_loss": 2.8903539180755615,
+      "eval_runtime": 72.6589,
+      "eval_samples_per_second": 835.135,
+      "eval_steps_per_second": 13.061,
+      "step": 13350
     },
     {
-      "epoch": 6.289421942587495,
-      "grad_norm": 0.2979063093662262,
+      "epoch": 6.292233337079915,
+      "grad_norm": 0.29889941215515137,
       "learning_rate": 0.0004375,
-      "loss": 2.7418,
+      "loss": 2.7435,
       "step": 14000
     },
     {
-      "epoch": 6.738834896915904,
-      "grad_norm": 0.28142639994621277,
-      "learning_rate": 0.00046875,
-      "loss": 2.7389,
+      "epoch": 6.741823086433629,
+      "grad_norm": 0.2726060450077057,
+      "learning_rate": 0.00046871875,
+      "loss": 2.7385,
       "step": 15000
     },
     {
       "epoch": 7.0,
-      "eval_accuracy": 0.4533856243606156,
-      "eval_loss": 2.8659732341766357,
-      "eval_runtime": 72.7067,
-      "eval_samples_per_second": 834.875,
-      "eval_steps_per_second": 13.052,
-      "step": 15582
+      "eval_accuracy": 0.4530601548463815,
+      "eval_loss": 2.8666210174560547,
+      "eval_runtime": 72.57,
+      "eval_samples_per_second": 836.158,
+      "eval_steps_per_second": 13.077,
+      "step": 15575
     },
     {
-      "epoch": 7.187854614909274,
-      "grad_norm": 0.2868495583534241,
+      "epoch": 7.191075643475329,
+      "grad_norm": 0.2712298333644867,
       "learning_rate": 0.00049996875,
-      "loss": 2.7131,
+      "loss": 2.7141,
       "step": 16000
     },
     {
-      "epoch": 7.637267569237683,
-      "grad_norm": 0.25408118963241577,
+      "epoch": 7.640665392829043,
+      "grad_norm": 0.2567969560623169,
       "learning_rate": 0.00053121875,
-      "loss": 2.7055,
+      "loss": 2.7061,
       "step": 17000
     },
     {
       "epoch": 8.0,
-      "eval_accuracy": 0.4554842427572502,
-      "eval_loss": 2.8459126949310303,
-      "eval_runtime": 72.7046,
-      "eval_samples_per_second": 834.898,
-      "eval_steps_per_second": 13.053,
-      "step": 17808
-    },
-    {
-      "epoch": 8.086287287231054,
-      "grad_norm": 0.25123631954193115,
-      "learning_rate": 0.0005624375,
-      "loss": 2.6925,
+      "eval_accuracy": 0.4559095609239081,
+      "eval_loss": 2.8456263542175293,
+      "eval_runtime": 72.544,
+      "eval_samples_per_second": 836.458,
+      "eval_steps_per_second": 13.082,
+      "step": 17800
+    },
+    {
+      "epoch": 8.089917949870744,
+      "grad_norm": 0.24305634200572968,
+      "learning_rate": 0.0005624687499999999,
+      "loss": 2.6951,
       "step": 18000
     },
     {
-      "epoch": 8.535700241559462,
-      "grad_norm": 0.24333898723125458,
+      "epoch": 8.539507699224458,
+      "grad_norm": 0.23454323410987854,
       "learning_rate": 0.0005936875,
-      "loss": 2.6726,
+      "loss": 2.6745,
       "step": 19000
     },
     {
-      "epoch": 8.985113195887871,
-      "grad_norm": 0.23374955356121063,
-      "learning_rate": 0.00062490625,
-      "loss": 2.6857,
+      "epoch": 8.989097448578173,
+      "grad_norm": 0.2262556552886963,
+      "learning_rate": 0.0006249375000000001,
+      "loss": 2.6855,
       "step": 20000
     },
     {
       "epoch": 9.0,
-      "eval_accuracy": 0.4575513986751518,
-      "eval_loss": 2.831002950668335,
-      "eval_runtime": 72.439,
-      "eval_samples_per_second": 837.96,
-      "eval_steps_per_second": 13.101,
-      "step": 20034
-    },
-    {
-      "epoch": 9.434132913881243,
-      "grad_norm": 0.22414237260818481,
-      "learning_rate": 0.000656125,
-      "loss": 2.6461,
+      "eval_accuracy": 0.4575118590613569,
+      "eval_loss": 2.833211898803711,
+      "eval_runtime": 72.5848,
+      "eval_samples_per_second": 835.988,
+      "eval_steps_per_second": 13.074,
+      "step": 20025
+    },
+    {
+      "epoch": 9.438350005619872,
+      "grad_norm": 0.2205825001001358,
+      "learning_rate": 0.0006561562500000001,
+      "loss": 2.6456,
       "step": 21000
     },
     {
-      "epoch": 9.883545868209652,
-      "grad_norm": 0.21065442264080048,
-      "learning_rate": 0.0006873749999999999,
-      "loss": 2.6638,
+      "epoch": 9.887939754973587,
+      "grad_norm": 0.21933791041374207,
+      "learning_rate": 0.00068740625,
+      "loss": 2.6669,
       "step": 22000
     },
     {
       "epoch": 10.0,
-      "eval_accuracy": 0.4584477885979848,
-      "eval_loss": 2.822634220123291,
-      "eval_runtime": 72.5174,
-      "eval_samples_per_second": 837.054,
-      "eval_steps_per_second": 13.087,
-      "step": 22260
+      "eval_accuracy": 0.45864309072343507,
+      "eval_loss": 2.819795608520508,
+      "eval_runtime": 72.9477,
+      "eval_samples_per_second": 831.828,
+      "eval_steps_per_second": 13.009,
+      "step": 22250
     },
     {
-      "epoch": 10.332565586203023,
-      "grad_norm": 0.21908308565616608,
+      "epoch": 10.337192312015286,
+      "grad_norm": 0.21199771761894226,
       "learning_rate": 0.000718625,
-      "loss": 2.6333,
+      "loss": 2.636,
       "step": 23000
     },
     {
-      "epoch": 10.78197854053143,
-      "grad_norm": 0.19101175665855408,
+      "epoch": 10.786782061369001,
+      "grad_norm": 0.20166124403476715,
       "learning_rate": 0.000749875,
-      "loss": 2.6495,
+      "loss": 2.6499,
       "step": 24000
     },
     {
       "epoch": 11.0,
-      "eval_accuracy": 0.45945684908120843,
-      "eval_loss": 2.81111741065979,
-      "eval_runtime": 72.5362,
-      "eval_samples_per_second": 836.837,
-      "eval_steps_per_second": 13.083,
-      "step": 24486
+      "eval_accuracy": 0.4597305698812155,
+      "eval_loss": 2.8118433952331543,
+      "eval_runtime": 73.1507,
+      "eval_samples_per_second": 829.521,
+      "eval_steps_per_second": 12.973,
+      "step": 24475
     },
     {
-      "epoch": 11.230998258524801,
-      "grad_norm": 0.19955122470855713,
+      "epoch": 11.2360346184107,
+      "grad_norm": 0.20014500617980957,
       "learning_rate": 0.000781125,
-      "loss": 2.6264,
+      "loss": 2.6298,
       "step": 25000
     },
     {
-      "epoch": 11.68041121285321,
-      "grad_norm": 0.18576420843601227,
+      "epoch": 11.685624367764415,
+      "grad_norm": 0.19325494766235352,
       "learning_rate": 0.00081234375,
-      "loss": 2.6341,
+      "loss": 2.6351,
       "step": 26000
     },
     {
       "epoch": 12.0,
-      "eval_accuracy": 0.46001277234506266,
-      "eval_loss": 2.8056912422180176,
-      "eval_runtime": 72.7843,
-      "eval_samples_per_second": 833.984,
-      "eval_steps_per_second": 13.039,
-      "step": 26712
+      "eval_accuracy": 0.4601314513940052,
+      "eval_loss": 2.807219982147217,
+      "eval_runtime": 72.6854,
+      "eval_samples_per_second": 834.831,
+      "eval_steps_per_second": 13.056,
+      "step": 26700
     },
     {
-      "epoch": 12.129430930846581,
-      "grad_norm": 0.21167126297950745,
+      "epoch": 12.134876924806115,
+      "grad_norm": 0.19165903329849243,
       "learning_rate": 0.00084359375,
-      "loss": 2.6248,
+      "loss": 2.6265,
       "step": 27000
     },
     {
-      "epoch": 12.57884388517499,
-      "grad_norm": 0.19634310901165009,
-      "learning_rate": 0.0008748125,
-      "loss": 2.6198,
+      "epoch": 12.58446667415983,
+      "grad_norm": 0.1863769292831421,
+      "learning_rate": 0.0008748437500000001,
+      "loss": 2.6204,
       "step": 28000
     },
     {
       "epoch": 13.0,
-      "eval_accuracy": 0.4610777158757277,
-      "eval_loss": 2.8013079166412354,
-      "eval_runtime": 72.8617,
-      "eval_samples_per_second": 833.099,
-      "eval_steps_per_second": 13.025,
-      "step": 28938
-    },
-    {
-      "epoch": 13.027863603168361,
-      "grad_norm": 0.1967993527650833,
-      "learning_rate": 0.0009060625,
-      "loss": 2.6292,
+      "eval_accuracy": 0.4611660010081818,
+      "eval_loss": 2.802619218826294,
+      "eval_runtime": 72.6835,
+      "eval_samples_per_second": 834.852,
+      "eval_steps_per_second": 13.057,
+      "step": 28925
+    },
+    {
+      "epoch": 13.033719231201529,
+      "grad_norm": 0.19991189241409302,
+      "learning_rate": 0.00090609375,
+      "loss": 2.6286,
       "step": 29000
     },
     {
-      "epoch": 13.47727655749677,
-      "grad_norm": 0.21736542880535126,
-      "learning_rate": 0.00093728125,
-      "loss": 2.6036,
+      "epoch": 13.483308980555243,
+      "grad_norm": 0.18545052409172058,
+      "learning_rate": 0.0009373125,
+      "loss": 2.6068,
       "step": 30000
     },
     {
-      "epoch": 13.926689511825177,
-      "grad_norm": 0.19267675280570984,
-      "learning_rate": 0.00096853125,
-      "loss": 2.6269,
+      "epoch": 13.932898729908958,
+      "grad_norm": 0.17478196322917938,
+      "learning_rate": 0.0009685625,
+      "loss": 2.6277,
       "step": 31000
     },
     {
       "epoch": 14.0,
-      "eval_accuracy": 0.4611890297378568,
-      "eval_loss": 2.794591188430786,
-      "eval_runtime": 72.6832,
-      "eval_samples_per_second": 835.145,
-      "eval_steps_per_second": 13.057,
-      "step": 31164
-    },
-    {
-      "epoch": 14.375709229818549,
-      "grad_norm": 0.19419512152671814,
-      "learning_rate": 0.00099975,
-      "loss": 2.5946,
+      "eval_accuracy": 0.4612847208758256,
+      "eval_loss": 2.801252841949463,
+      "eval_runtime": 72.4155,
+      "eval_samples_per_second": 837.942,
+      "eval_steps_per_second": 13.105,
+      "step": 31150
+    },
+    {
+      "epoch": 14.382151286950657,
+      "grad_norm": 0.17678463459014893,
+      "learning_rate": 0.00099978125,
+      "loss": 2.5975,
       "step": 32000
     },
     {
-      "epoch": 14.825122184146958,
-      "grad_norm": 0.1727887988090515,
-      "learning_rate": 0.00092064,
-      "loss": 2.6122,
+      "epoch": 14.831741036304372,
+      "grad_norm": 0.17033128440380096,
+      "learning_rate": 0.0009204326923076923,
+      "loss": 2.6136,
       "step": 33000
     },
     {
       "epoch": 15.0,
-      "eval_accuracy": 0.4638763905753402,
-      "eval_loss": 2.7776918411254883,
-      "eval_runtime": 72.7228,
-      "eval_samples_per_second": 834.69,
+      "eval_accuracy": 0.46383548541367764,
+      "eval_loss": 2.779118061065674,
+      "eval_runtime": 72.723,
+      "eval_samples_per_second": 834.399,
       "eval_steps_per_second": 13.05,
-      "step": 33390
+      "step": 33375
     },
     {
-      "epoch": 15.274141902140329,
-      "grad_norm": 0.1849382370710373,
-      "learning_rate": 0.00084072,
-      "loss": 2.5738,
+      "epoch": 15.280993593346071,
+      "grad_norm": 0.1788545548915863,
+      "learning_rate": 0.0008403044871794871,
+      "loss": 2.5726,
       "step": 34000
     },
     {
-      "epoch": 15.723554856468738,
-      "grad_norm": 0.16888663172721863,
-      "learning_rate": 0.00076072,
-      "loss": 2.565,
+      "epoch": 15.730583342699786,
+      "grad_norm": 0.17410264909267426,
+      "learning_rate": 0.0007602564102564103,
+      "loss": 2.5687,
       "step": 35000
     },
     {
       "epoch": 16.0,
-      "eval_accuracy": 0.4671347921715926,
-      "eval_loss": 2.7541751861572266,
-      "eval_runtime": 72.8608,
-      "eval_samples_per_second": 833.109,
-      "eval_steps_per_second": 13.025,
-      "step": 35616
-    },
-    {
-      "epoch": 16.172574574462107,
-      "grad_norm": 0.17893236875534058,
-      "learning_rate": 0.00068072,
-      "loss": 2.5371,
+      "eval_accuracy": 0.4676111908177905,
+      "eval_loss": 2.75136661529541,
+      "eval_runtime": 72.8808,
+      "eval_samples_per_second": 832.592,
+      "eval_steps_per_second": 13.021,
+      "step": 35600
+    },
+    {
+      "epoch": 16.179835899741487,
+      "grad_norm": 0.17518466711044312,
+      "learning_rate": 0.0006801282051282051,
+      "loss": 2.5356,
       "step": 36000
     },
     {
-      "epoch": 16.621987528790516,
-      "grad_norm": 0.1799129843711853,
-      "learning_rate": 0.0006008,
-      "loss": 2.5134,
+      "epoch": 16.6294256490952,
+      "grad_norm": 0.17421123385429382,
+      "learning_rate": 0.0006000801282051283,
+      "loss": 2.5184,
       "step": 37000
     },
     {
       "epoch": 17.0,
-      "eval_accuracy": 0.47074128377896024,
-      "eval_loss": 2.7257015705108643,
-      "eval_runtime": 72.9785,
-      "eval_samples_per_second": 831.766,
-      "eval_steps_per_second": 13.004,
-      "step": 37842
-    },
-    {
-      "epoch": 17.07100724678389,
-      "grad_norm": 0.17874906957149506,
-      "learning_rate": 0.0005208000000000001,
-      "loss": 2.5026,
+      "eval_accuracy": 0.4707787558002766,
+      "eval_loss": 2.728271245956421,
+      "eval_runtime": 72.8051,
+      "eval_samples_per_second": 833.458,
+      "eval_steps_per_second": 13.035,
+      "step": 37825
+    },
+    {
+      "epoch": 17.0786782061369,
+      "grad_norm": 0.18264968693256378,
+      "learning_rate": 0.0005199519230769231,
+      "loss": 2.4989,
       "step": 38000
     },
     {
-      "epoch": 17.520420201112298,
-      "grad_norm": 0.18105448782444,
-      "learning_rate": 0.00044088,
-      "loss": 2.4567,
+      "epoch": 17.528267955490616,
+      "grad_norm": 0.18594865500926971,
+      "learning_rate": 0.00043990384615384616,
+      "loss": 2.4571,
       "step": 39000
     },
     {
-      "epoch": 17.969833155440707,
-      "grad_norm": 0.17908377945423126,
-      "learning_rate": 0.00036088,
-      "loss": 2.4592,
+      "epoch": 17.97785770484433,
+      "grad_norm": 0.17989173531532288,
+      "learning_rate": 0.00035977564102564105,
+      "loss": 2.4613,
       "step": 40000
     },
     {
       "epoch": 18.0,
-      "eval_accuracy": 0.473861818989964,
-      "eval_loss": 2.7058815956115723,
-      "eval_runtime": 73.0112,
-      "eval_samples_per_second": 831.393,
-      "eval_steps_per_second": 12.998,
-      "step": 40068
-    },
-    {
-      "epoch": 18.418852873434076,
-      "grad_norm": 0.18544642627239227,
-      "learning_rate": 0.00028095999999999997,
-      "loss": 2.3906,
+      "eval_accuracy": 0.47402006023239884,
+      "eval_loss": 2.705965518951416,
+      "eval_runtime": 72.7008,
+      "eval_samples_per_second": 834.654,
+      "eval_steps_per_second": 13.054,
+      "step": 40050
+    },
+    {
+      "epoch": 18.42711026188603,
+      "grad_norm": 0.19071288406848907,
+      "learning_rate": 0.0002797275641025641,
+      "loss": 2.3913,
       "step": 41000
     },
     {
-      "epoch": 18.868265827762485,
-      "grad_norm": 0.1904035061597824,
-      "learning_rate": 0.00020096,
-      "loss": 2.3964,
+      "epoch": 18.876700011239745,
+      "grad_norm": 0.19211626052856445,
+      "learning_rate": 0.00019959935897435898,
+      "loss": 2.3966,
       "step": 42000
     },
     {
       "epoch": 19.0,
-      "eval_accuracy": 0.47651713590660233,
-      "eval_loss": 2.69079327583313,
-      "eval_runtime": 72.778,
-      "eval_samples_per_second": 834.057,
-      "eval_steps_per_second": 13.04,
-      "step": 42294
-    },
-    {
-      "epoch": 19.317285545755855,
-      "grad_norm": 0.1982097625732422,
-      "learning_rate": 0.00012103999999999999,
-      "loss": 2.3398,
+      "eval_accuracy": 0.4765565421949927,
+      "eval_loss": 2.6947293281555176,
+      "eval_runtime": 72.857,
+      "eval_samples_per_second": 832.864,
+      "eval_steps_per_second": 13.026,
+      "step": 42275
+    },
+    {
+      "epoch": 19.325952568281444,
+      "grad_norm": 0.19902721047401428,
+      "learning_rate": 0.00011947115384615386,
+      "loss": 2.3404,
       "step": 43000
     },
     {
-      "epoch": 19.766698500084264,
-      "grad_norm": 0.19803358614444733,
-      "learning_rate": 4.104e-05,
-      "loss": 2.3229,
+      "epoch": 19.775542317635157,
+      "grad_norm": 0.20530302822589874,
+      "learning_rate": 3.942307692307692e-05,
+      "loss": 2.3227,
       "step": 44000
     },
     {
-      "epoch": 19.991404977248468,
-      "eval_accuracy": 0.4780018806422093,
-      "eval_loss": 2.688654899597168,
-      "eval_runtime": 73.0151,
-      "eval_samples_per_second": 831.349,
-      "eval_steps_per_second": 12.997,
-      "step": 44500
+      "epoch": 19.99134539732494,
+      "eval_accuracy": 0.4781093360218181,
+      "eval_loss": 2.690006971359253,
+      "eval_runtime": 72.887,
+      "eval_samples_per_second": 832.521,
+      "eval_steps_per_second": 13.02,
+      "step": 44480
     },
     {
-      "epoch": 19.991404977248468,
-      "step": 44500,
-      "total_flos": 1.487763384827904e+18,
-      "train_loss": 2.800290222853757,
-      "train_runtime": 29774.6232,
-      "train_samples_per_second": 382.629,
-      "train_steps_per_second": 1.495
+      "epoch": 19.99134539732494,
+      "step": 44480,
+      "total_flos": 1.487139158163456e+18,
+      "train_loss": 2.8016022716494775,
+      "train_runtime": 30047.0507,
+      "train_samples_per_second": 379.002,
+      "train_steps_per_second": 1.48
     }
   ],
   "logging_steps": 1000,
-  "max_steps": 44500,
+  "max_steps": 44480,
   "num_input_tokens_seen": 0,
   "num_train_epochs": 20,
   "save_steps": 500,
@@ -532,7 +532,7 @@
       "attributes": {}
     }
   },
-  "total_flos": 1.487763384827904e+18,
+  "total_flos": 1.487139158163456e+18,
   "train_batch_size": 32,
   "trial_name": null,
   "trial_params": null