Model save

README.md

@@ -41,6 +41,7 @@ The following hyperparameters were used during training:
 - total_train_batch_size: 256
 - optimizer: Adam with betas=(0.9,0.999) and epsilon=1e-08
 - lr_scheduler_type: linear
+- lr_scheduler_warmup_steps: 2
 - num_epochs: 4
 
 ### Training results

adapter_config.json

@@ -16,17 +16,17 @@
   "megatron_core": "megatron.core",
   "modules_to_save": null,
   "peft_type": "LORA",
-  "r": 8,
+  "r": 16,
   "rank_pattern": {},
   "revision": null,
   "target_modules": [
+    "o_proj",
+    "v_proj",
+    "k_proj",
+    "q_proj",
     "down_proj",
-    "up_proj",
     "gate_proj",
-    "q_proj",
-    "k_proj",
-    "o_proj",
-    "v_proj"
+    "up_proj"
   ],
   "task_type": "CAUSAL_LM",
   "use_dora": false,

adapter_model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:c7f823d070fb4bc434289fe85e13dbd360fc27d0ba59d70e964da5d34863719a
-size 1080767344
+oid sha256:28d18e25ccb1e93e19ba60ec0285d931a9b17f4425c3ee0b0242cd14be802c51
+size 1103312056

trainer_state.json

@@ -10,68 +10,68 @@
   "log_history": [
     {
       "epoch": 0.4854368932038835,
-      "grad_norm": 1.1607048511505127,
-      "learning_rate": 8.786407766990292e-05,
-      "loss": 0.8852,
+      "grad_norm": 0.7365666627883911,
+      "learning_rate": 8.794946550048592e-05,
+      "loss": 0.9374,
       "step": 250
     },
     {
       "epoch": 0.970873786407767,
-      "grad_norm": 0.9901185631752014,
-      "learning_rate": 7.572815533980583e-05,
-      "loss": 0.4521,
+      "grad_norm": 0.701321005821228,
+      "learning_rate": 7.580174927113704e-05,
+      "loss": 0.4669,
       "step": 500
     },
     {
       "epoch": 1.4563106796116505,
-      "grad_norm": 0.8445346355438232,
-      "learning_rate": 6.359223300970875e-05,
-      "loss": 0.3889,
+      "grad_norm": 0.5943931937217712,
+      "learning_rate": 6.365403304178815e-05,
+      "loss": 0.3933,
       "step": 750
     },
     {
       "epoch": 1.941747572815534,
-      "grad_norm": 0.7807681560516357,
-      "learning_rate": 5.145631067961165e-05,
-      "loss": 0.3685,
+      "grad_norm": 0.5661283135414124,
+      "learning_rate": 5.150631681243926e-05,
+      "loss": 0.3706,
       "step": 1000
     },
     {
       "epoch": 2.4271844660194173,
-      "grad_norm": 0.7362964153289795,
-      "learning_rate": 3.9320388349514564e-05,
-      "loss": 0.3581,
+      "grad_norm": 0.5260675549507141,
+      "learning_rate": 3.9358600583090386e-05,
+      "loss": 0.3599,
       "step": 1250
     },
     {
       "epoch": 2.912621359223301,
-      "grad_norm": 0.785736620426178,
-      "learning_rate": 2.7184466019417475e-05,
-      "loss": 0.3525,
+      "grad_norm": 0.5369626879692078,
+      "learning_rate": 2.72108843537415e-05,
+      "loss": 0.3546,
       "step": 1500
     },
     {
       "epoch": 3.3980582524271843,
-      "grad_norm": 0.7113440632820129,
-      "learning_rate": 1.5048543689320387e-05,
-      "loss": 0.3446,
+      "grad_norm": 0.48315396904945374,
+      "learning_rate": 1.5063168124392615e-05,
+      "loss": 0.3473,
       "step": 1750
     },
     {
       "epoch": 3.883495145631068,
-      "grad_norm": 0.6975888013839722,
-      "learning_rate": 2.912621359223301e-06,
-      "loss": 0.3404,
+      "grad_norm": 0.46571284532546997,
+      "learning_rate": 2.915451895043732e-06,
+      "loss": 0.3437,
       "step": 2000
     },
     {
       "epoch": 4.0,
       "step": 2060,
-      "total_flos": 4.996840775351132e+17,
-      "train_loss": 0.4334296365386074,
-      "train_runtime": 6711.8297,
-      "train_samples_per_second": 78.57,
-      "train_steps_per_second": 0.307
+      "total_flos": 5.02561395295531e+17,
+      "train_loss": 0.4436599824035052,
+      "train_runtime": 6742.0133,
+      "train_samples_per_second": 78.218,
+      "train_steps_per_second": 0.306
     }
   ],
   "logging_steps": 250,
@@ -91,7 +91,7 @@
       "attributes": {}
     }
   },
-  "total_flos": 4.996840775351132e+17,
+  "total_flos": 5.02561395295531e+17,
   "train_batch_size": 16,
   "trial_name": null,
   "trial_params": null

training_args.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:4971a71404bdc4471db5496187a2a0962d0fcb9bda93b897a427de69a0807442
+oid sha256:6404f770a80f28004c986cae0c269c7894eeff9ed7b65250d652ac62b91db4e8
 size 5240