Model save

README.md

@@ -15,7 +15,7 @@ should probably proofread and complete it, then remove this comment. -->
 
 This model is a fine-tuned version of [mistralai/Mistral-7B-v0.1](https://huggingface.co/mistralai/Mistral-7B-v0.1) on an unknown dataset.
 It achieves the following results on the evaluation set:
-- Loss: 0.9900
+- Loss: 1.0240
 
 ## Model description
 
@@ -39,8 +39,10 @@ The following hyperparameters were used during training:
 - eval_batch_size: 8
 - seed: 42
 - distributed_type: multi-GPU
+- num_devices: 4
 - gradient_accumulation_steps: 128
-- total_train_batch_size: 512
+- total_train_batch_size: 2048
+- total_eval_batch_size: 32
 - optimizer: Adam with betas=(0.9,0.999) and epsilon=1e-08
 - lr_scheduler_type: cosine
 - num_epochs: 1
@@ -49,7 +51,7 @@ The following hyperparameters were used during training:
 
 | Training Loss | Epoch | Step | Validation Loss |
 |:-------------:|:-----:|:----:|:---------------:|
-| 0.9866        | 0.67  | 272  | 0.9900          |
+| 1.0225        | 0.67  | 68   | 1.0241          |
 
 
 ### Framework versions

adapter_config.json

@@ -18,8 +18,8 @@
   "target_modules": [
     "q_proj",
     "k_proj",
-    "v_proj",
-    "o_proj"
+    "o_proj",
+    "v_proj"
   ],
   "task_type": "CAUSAL_LM"
 }

adapter_model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:86153f980d9b196097ccc204da082dd9c6aeafc01ef1aa0bfcc4790e74b9d4a9
+oid sha256:37fa6035b5fe2e6e4738529281b40d10966134556c26cd800e3ca6b090d5b99e
 size 218138576

all_results.json

@@ -1,13 +1,13 @@
 {
     "epoch": 0.67,
-    "eval_loss": 0.9899672269821167,
-    "eval_runtime": 2925.3835,
+    "eval_loss": 1.024001121520996,
+    "eval_runtime": 742.2394,
     "eval_samples": 23110,
-    "eval_samples_per_second": 7.9,
-    "eval_steps_per_second": 0.988,
-    "train_loss": 1.0095806910711176,
-    "train_runtime": 87675.0459,
+    "eval_samples_per_second": 31.136,
+    "eval_steps_per_second": 0.974,
+    "train_loss": 1.0573345633114086,
+    "train_runtime": 21895.8144,
     "train_samples": 207865,
-    "train_samples_per_second": 2.371,
+    "train_samples_per_second": 9.493,
     "train_steps_per_second": 0.005
 }

eval_results.json

@@ -1,8 +1,8 @@
 {
     "epoch": 0.67,
-    "eval_loss": 0.9899672269821167,
-    "eval_runtime": 2925.3835,
+    "eval_loss": 1.024001121520996,
+    "eval_runtime": 742.2394,
     "eval_samples": 23110,
-    "eval_samples_per_second": 7.9,
-    "eval_steps_per_second": 0.988
+    "eval_samples_per_second": 31.136,
+    "eval_steps_per_second": 0.974
 }

runs/Jan02_10-16-20_node-0/events.out.tfevents.1704219571.node-0.3389.0

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:bda0cb9630f30cf61fc8494470798c9560143806235e5a3d5f04c8702a182d64
+size 5178

runs/Jan02_12-33-25_node-0/events.out.tfevents.1704227663.node-0.20373.0

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ffaab5b7bda7c62786d2b541ab81043011a9628585f118652dd05917b08b4f6c
+size 7024

runs/Jan02_12-33-25_node-0/events.out.tfevents.1704250300.node-0.20373.1

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:037962b59239d2ed51971e03c80a54cd2a15a44125e48e576a8f43525507b4cd
+size 354

train_results.json

@@ -1,8 +1,8 @@
 {
     "epoch": 0.67,
-    "train_loss": 1.0095806910711176,
-    "train_runtime": 87675.0459,
+    "train_loss": 1.0573345633114086,
+    "train_runtime": 21895.8144,
     "train_samples": 207865,
-    "train_samples_per_second": 2.371,
+    "train_samples_per_second": 9.493,
     "train_steps_per_second": 0.005
 }

trainer_state.json

@@ -1,366 +1,120 @@
 {
   "best_metric": null,
   "best_model_checkpoint": null,
-  "epoch": 0.6699636307656782,
+  "epoch": 0.6699507389162561,
   "eval_steps": 500,
-  "global_step": 272,
+  "global_step": 68,
   "is_hyper_param_search": false,
   "is_local_process_zero": true,
   "is_world_process_zero": true,
   "log_history": [
     {
-      "epoch": 0.0,
-      "learning_rate": 1.999969914479768e-05,
-      "loss": 1.1295,
+      "epoch": 0.01,
+      "learning_rate": 1.999516282291988e-05,
+      "loss": 1.1267,
       "step": 1
     },
     {
-      "epoch": 0.01,
-      "learning_rate": 1.9992479525042305e-05,
-      "loss": 1.1199,
+      "epoch": 0.05,
+      "learning_rate": 1.987930439740757e-05,
+      "loss": 1.1282,
       "step": 5
     },
     {
-      "epoch": 0.02,
-      "learning_rate": 1.996992941167792e-05,
-      "loss": 1.1063,
+      "epoch": 0.1,
+      "learning_rate": 1.95201310753273e-05,
+      "loss": 1.1091,
       "step": 10
     },
     {
-      "epoch": 0.04,
-      "learning_rate": 1.9932383577419432e-05,
-      "loss": 1.0914,
+      "epoch": 0.15,
+      "learning_rate": 1.8931150161867917e-05,
+      "loss": 1.0943,
       "step": 15
     },
     {
-      "epoch": 0.05,
-      "learning_rate": 1.9879898494768093e-05,
-      "loss": 1.094,
+      "epoch": 0.2,
+      "learning_rate": 1.8126579138282502e-05,
+      "loss": 1.0769,
       "step": 20
     },
     {
-      "epoch": 0.06,
-      "learning_rate": 1.9812553106273848e-05,
+      "epoch": 0.25,
+      "learning_rate": 1.7125839641475074e-05,
       "loss": 1.0658,
       "step": 25
     },
     {
-      "epoch": 0.07,
-      "learning_rate": 1.973044870579824e-05,
-      "loss": 1.0527,
+      "epoch": 0.3,
+      "learning_rate": 1.595308864276666e-05,
+      "loss": 1.0596,
       "step": 30
     },
     {
-      "epoch": 0.09,
-      "learning_rate": 1.9633708786158803e-05,
-      "loss": 1.0503,
+      "epoch": 0.34,
+      "learning_rate": 1.4636635319853274e-05,
+      "loss": 1.0499,
       "step": 35
     },
     {
-      "epoch": 0.1,
-      "learning_rate": 1.9522478853384154e-05,
-      "loss": 1.0305,
+      "epoch": 0.39,
+      "learning_rate": 1.3208257698153677e-05,
+      "loss": 1.0333,
       "step": 40
     },
     {
-      "epoch": 0.11,
-      "learning_rate": 1.9396926207859085e-05,
-      "loss": 1.0268,
+      "epoch": 0.44,
+      "learning_rate": 1.1702435557223988e-05,
+      "loss": 1.0313,
       "step": 45
     },
     {
-      "epoch": 0.12,
-      "learning_rate": 1.9257239692688907e-05,
+      "epoch": 0.49,
+      "learning_rate": 1.0155518119203511e-05,
       "loss": 1.0291,
       "step": 50
     },
-    {
-      "epoch": 0.14,
-      "learning_rate": 1.9103629409661468e-05,
-      "loss": 1.02,
-      "step": 55
-    },
-    {
-      "epoch": 0.15,
-      "learning_rate": 1.8936326403234125e-05,
-      "loss": 1.0127,
-      "step": 60
-    },
-    {
-      "epoch": 0.16,
-      "learning_rate": 1.8755582313020912e-05,
-      "loss": 1.0078,
-      "step": 65
-    },
-    {
-      "epoch": 0.17,
-      "learning_rate": 1.8561668995302668e-05,
-      "loss": 1.0003,
-      "step": 70
-    },
-    {
-      "epoch": 0.18,
-      "learning_rate": 1.8354878114129368e-05,
-      "loss": 1.0081,
-      "step": 75
-    },
-    {
-      "epoch": 0.2,
-      "learning_rate": 1.8135520702629677e-05,
-      "loss": 1.0157,
-      "step": 80
-    },
-    {
-      "epoch": 0.21,
-      "learning_rate": 1.7903926695187595e-05,
-      "loss": 1.0118,
-      "step": 85
-    },
-    {
-      "epoch": 0.22,
-      "learning_rate": 1.766044443118978e-05,
-      "loss": 1.0011,
-      "step": 90
-    },
-    {
-      "epoch": 0.23,
-      "learning_rate": 1.740544013109005e-05,
-      "loss": 0.9919,
-      "step": 95
-    },
-    {
-      "epoch": 0.25,
-      "learning_rate": 1.7139297345578992e-05,
-      "loss": 1.0025,
-      "step": 100
-    },
-    {
-      "epoch": 0.26,
-      "learning_rate": 1.686241637868734e-05,
-      "loss": 1.0187,
-      "step": 105
-    },
-    {
-      "epoch": 0.27,
-      "learning_rate": 1.657521368569064e-05,
-      "loss": 0.9959,
-      "step": 110
-    },
-    {
-      "epoch": 0.28,
-      "learning_rate": 1.627812124672099e-05,
-      "loss": 0.9936,
-      "step": 115
-    },
-    {
-      "epoch": 0.3,
-      "learning_rate": 1.5971585917027864e-05,
-      "loss": 0.9981,
-      "step": 120
-    },
-    {
-      "epoch": 0.31,
-      "learning_rate": 1.5656068754865388e-05,
-      "loss": 0.9954,
-      "step": 125
-    },
-    {
-      "epoch": 0.32,
-      "learning_rate": 1.5332044328016916e-05,
-      "loss": 1.0044,
-      "step": 130
-    },
-    {
-      "epoch": 0.33,
-      "learning_rate": 1.5000000000000002e-05,
-      "loss": 1.0032,
-      "step": 135
-    },
-    {
-      "epoch": 0.34,
-      "learning_rate": 1.4660435197025391e-05,
-      "loss": 0.9901,
-      "step": 140
-    },
-    {
-      "epoch": 0.36,
-      "learning_rate": 1.4313860656812537e-05,
-      "loss": 0.9965,
-      "step": 145
-    },
-    {
-      "epoch": 0.37,
-      "learning_rate": 1.396079766039157e-05,
-      "loss": 0.9915,
-      "step": 150
-    },
-    {
-      "epoch": 0.38,
-      "learning_rate": 1.3601777248047105e-05,
-      "loss": 0.9862,
-      "step": 155
-    },
-    {
-      "epoch": 0.39,
-      "learning_rate": 1.3237339420583213e-05,
-      "loss": 0.9757,
-      "step": 160
-    },
-    {
-      "epoch": 0.41,
-      "learning_rate": 1.2868032327110904e-05,
-      "loss": 0.9944,
-      "step": 165
-    },
-    {
-      "epoch": 0.42,
-      "learning_rate": 1.2494411440579814e-05,
-      "loss": 0.9963,
-      "step": 170
-    },
-    {
-      "epoch": 0.43,
-      "learning_rate": 1.211703872229411e-05,
-      "loss": 0.9817,
-      "step": 175
-    },
-    {
-      "epoch": 0.44,
-      "learning_rate": 1.1736481776669307e-05,
-      "loss": 0.9881,
-      "step": 180
-    },
-    {
-      "epoch": 0.46,
-      "learning_rate": 1.1353312997501313e-05,
-      "loss": 0.9965,
-      "step": 185
-    },
-    {
-      "epoch": 0.47,
-      "learning_rate": 1.0968108707031792e-05,
-      "loss": 0.9921,
-      "step": 190
-    },
-    {
-      "epoch": 0.48,
-      "learning_rate": 1.0581448289104759e-05,
-      "loss": 0.9934,
-      "step": 195
-    },
-    {
-      "epoch": 0.49,
-      "learning_rate": 1.0193913317718245e-05,
-      "loss": 0.9823,
-      "step": 200
-    },
-    {
-      "epoch": 0.5,
-      "learning_rate": 9.806086682281759e-06,
-      "loss": 0.988,
-      "step": 205
-    },
-    {
-      "epoch": 0.52,
-      "learning_rate": 9.418551710895243e-06,
-      "loss": 0.9867,
-      "step": 210
-    },
-    {
-      "epoch": 0.53,
-      "learning_rate": 9.03189129296821e-06,
-      "loss": 0.9983,
-      "step": 215
-    },
     {
       "epoch": 0.54,
-      "learning_rate": 8.646687002498692e-06,
-      "loss": 0.9866,
-      "step": 220
-    },
-    {
-      "epoch": 0.55,
-      "learning_rate": 8.263518223330698e-06,
-      "loss": 0.9918,
-      "step": 225
-    },
-    {
-      "epoch": 0.57,
-      "learning_rate": 7.882961277705897e-06,
-      "loss": 0.9902,
-      "step": 230
-    },
-    {
-      "epoch": 0.58,
-      "learning_rate": 7.505588559420188e-06,
-      "loss": 0.987,
-      "step": 235
+      "learning_rate": 8.604846610560771e-06,
+      "loss": 1.0259,
+      "step": 55
     },
     {
       "epoch": 0.59,
-      "learning_rate": 7.131967672889101e-06,
-      "loss": 0.9941,
-      "step": 240
-    },
-    {
-      "epoch": 0.6,
-      "learning_rate": 6.762660579416791e-06,
-      "loss": 0.9968,
-      "step": 245
-    },
-    {
-      "epoch": 0.62,
-      "learning_rate": 6.3982227519528986e-06,
-      "loss": 0.9842,
-      "step": 250
-    },
-    {
-      "epoch": 0.63,
-      "learning_rate": 6.039202339608432e-06,
-      "loss": 0.9788,
-      "step": 255
+      "learning_rate": 7.0878528777274814e-06,
+      "loss": 1.0259,
+      "step": 60
     },
     {
       "epoch": 0.64,
-      "learning_rate": 5.686139343187468e-06,
-      "loss": 0.9929,
-      "step": 260
-    },
-    {
-      "epoch": 0.65,
-      "learning_rate": 5.339564802974615e-06,
-      "loss": 0.9835,
-      "step": 265
-    },
-    {
-      "epoch": 0.67,
-      "learning_rate": 5.000000000000003e-06,
-      "loss": 0.9866,
-      "step": 270
+      "learning_rate": 5.64115581524629e-06,
+      "loss": 1.0225,
+      "step": 65
     },
     {
       "epoch": 0.67,
-      "eval_loss": 0.9899672269821167,
-      "eval_runtime": 2926.0545,
-      "eval_samples_per_second": 7.898,
-      "eval_steps_per_second": 0.987,
-      "step": 272
+      "eval_loss": 1.0241080522537231,
+      "eval_runtime": 741.6789,
+      "eval_samples_per_second": 31.159,
+      "eval_steps_per_second": 0.975,
+      "step": 68
     },
     {
       "epoch": 0.67,
-      "step": 272,
-      "total_flos": 1.2275875027210994e+19,
-      "train_loss": 1.0095806910711176,
-      "train_runtime": 87675.0459,
-      "train_samples_per_second": 2.371,
+      "step": 68,
+      "total_flos": 1.2276051069330522e+19,
+      "train_loss": 1.0573345633114086,
+      "train_runtime": 21895.8144,
+      "train_samples_per_second": 9.493,
       "train_steps_per_second": 0.005
     }
   ],
   "logging_steps": 5,
-  "max_steps": 405,
+  "max_steps": 101,
   "num_train_epochs": 1,
   "save_steps": 500,
-  "total_flos": 1.2275875027210994e+19,
+  "total_flos": 1.2276051069330522e+19,
   "trial_name": null,
   "trial_params": null
 }

training_args.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:48012117f359d93f6f11e37307131da164e15c7f0c3130b3e312cd4922e74142
+oid sha256:d348904f26b5457e762405e5c80d2f80128cc7ee749ce3fe7a9be01773480935
 size 4664