Model save

README.md

@@ -15,7 +15,7 @@ should probably proofread and complete it, then remove this comment. -->
 
 This model is a fine-tuned version of [mistralai/Mistral-7B-v0.1](https://huggingface.co/mistralai/Mistral-7B-v0.1) on an unknown dataset.
 It achieves the following results on the evaluation set:
-- Loss: 1.0240
+- Loss: 1.0052
 
 ## Model description
 
@@ -39,10 +39,10 @@ The following hyperparameters were used during training:
 - eval_batch_size: 8
 - seed: 42
 - distributed_type: multi-GPU
-- num_devices: 4
+- num_devices: 2
 - gradient_accumulation_steps: 128
-- total_train_batch_size: 2048
-- total_eval_batch_size: 32
+- total_train_batch_size: 1024
+- total_eval_batch_size: 16
 - optimizer: Adam with betas=(0.9,0.999) and epsilon=1e-08
 - lr_scheduler_type: cosine
 - num_epochs: 1
@@ -51,7 +51,7 @@ The following hyperparameters were used during training:
 
 | Training Loss | Epoch | Step | Validation Loss |
 |:-------------:|:-----:|:----:|:---------------:|
-| 1.0225        | 0.67  | 68   | 1.0241          |
+| 0.9999        | 0.67  | 136  | 1.0053          |
 
 
 ### Framework versions

adapter_config.json

@@ -16,10 +16,10 @@
   "rank_pattern": {},
   "revision": null,
   "target_modules": [
-    "q_proj",
     "k_proj",
-    "o_proj",
-    "v_proj"
+    "q_proj",
+    "v_proj",
+    "o_proj"
   ],
   "task_type": "CAUSAL_LM"
 }

adapter_model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:37fa6035b5fe2e6e4738529281b40d10966134556c26cd800e3ca6b090d5b99e
+oid sha256:b09751113cf7c2af966bb79ea4245bffd040c30778074f8dc191f85e62fb0eef
 size 218138576

all_results.json

@@ -1,13 +1,13 @@
 {
     "epoch": 0.67,
-    "eval_loss": 1.024001121520996,
-    "eval_runtime": 742.2394,
+    "eval_loss": 1.005220651626587,
+    "eval_runtime": 1465.2957,
     "eval_samples": 23110,
-    "eval_samples_per_second": 31.136,
-    "eval_steps_per_second": 0.974,
-    "train_loss": 1.0573345633114086,
-    "train_runtime": 21895.8144,
+    "eval_samples_per_second": 15.772,
+    "eval_steps_per_second": 0.986,
+    "train_loss": 1.0304046939401066,
+    "train_runtime": 43361.1536,
     "train_samples": 207865,
-    "train_samples_per_second": 9.493,
+    "train_samples_per_second": 4.794,
     "train_steps_per_second": 0.005
 }

eval_results.json

@@ -1,8 +1,8 @@
 {
     "epoch": 0.67,
-    "eval_loss": 1.024001121520996,
-    "eval_runtime": 742.2394,
+    "eval_loss": 1.005220651626587,
+    "eval_runtime": 1465.2957,
     "eval_samples": 23110,
-    "eval_samples_per_second": 31.136,
-    "eval_steps_per_second": 0.974
+    "eval_samples_per_second": 15.772,
+    "eval_steps_per_second": 0.986
 }

runs/Jan02_11-57-39_node-0/events.out.tfevents.1704225692.node-0.13725.0

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:edf71f497e347463814f8b6ac7be6713a08f5325a3f69f26c587f292a52fa098
+size 4254

runs/Jan02_12-03-43_node-0/events.out.tfevents.1704225880.node-0.14724.0

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:bd321e06cf251526961f7977eeacc03826304038932be4e8c3609abb7fe21e0b
+size 9197

runs/Jan02_12-03-43_node-0/events.out.tfevents.1704270706.node-0.14724.1

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f0faa54a869b7e90e681224eb2c35b8c3e17768eba2d2775f12e6f5545732fea
+size 359

train_results.json

@@ -1,8 +1,8 @@
 {
     "epoch": 0.67,
-    "train_loss": 1.0573345633114086,
-    "train_runtime": 21895.8144,
+    "train_loss": 1.0304046939401066,
+    "train_runtime": 43361.1536,
     "train_samples": 207865,
-    "train_samples_per_second": 9.493,
+    "train_samples_per_second": 4.794,
     "train_steps_per_second": 0.005
 }

trainer_state.json

@@ -3,118 +3,202 @@
   "best_model_checkpoint": null,
   "epoch": 0.6699507389162561,
   "eval_steps": 500,
-  "global_step": 68,
+  "global_step": 136,
   "is_hyper_param_search": false,
   "is_local_process_zero": true,
   "is_world_process_zero": true,
   "log_history": [
     {
-      "epoch": 0.01,
-      "learning_rate": 1.999516282291988e-05,
-      "loss": 1.1267,
+      "epoch": 0.0,
+      "learning_rate": 1.9998802517966852e-05,
+      "loss": 1.1243,
       "step": 1
     },
     {
-      "epoch": 0.05,
-      "learning_rate": 1.987930439740757e-05,
-      "loss": 1.1282,
+      "epoch": 0.02,
+      "learning_rate": 1.997007728639956e-05,
+      "loss": 1.122,
       "step": 5
     },
     {
-      "epoch": 0.1,
-      "learning_rate": 1.95201310753273e-05,
-      "loss": 1.1091,
+      "epoch": 0.05,
+      "learning_rate": 1.9880488219356086e-05,
+      "loss": 1.1164,
       "step": 10
     },
     {
-      "epoch": 0.15,
-      "learning_rate": 1.8931150161867917e-05,
-      "loss": 1.0943,
+      "epoch": 0.07,
+      "learning_rate": 1.973176894846855e-05,
+      "loss": 1.0903,
       "step": 15
     },
     {
-      "epoch": 0.2,
-      "learning_rate": 1.8126579138282502e-05,
-      "loss": 1.0769,
+      "epoch": 0.1,
+      "learning_rate": 1.9524809490566878e-05,
+      "loss": 1.0802,
       "step": 20
     },
     {
-      "epoch": 0.25,
-      "learning_rate": 1.7125839641475074e-05,
-      "loss": 1.0658,
+      "epoch": 0.12,
+      "learning_rate": 1.926084840336821e-05,
+      "loss": 1.0711,
       "step": 25
     },
     {
-      "epoch": 0.3,
-      "learning_rate": 1.595308864276666e-05,
-      "loss": 1.0596,
+      "epoch": 0.15,
+      "learning_rate": 1.894146537327533e-05,
+      "loss": 1.0548,
       "step": 30
     },
     {
-      "epoch": 0.34,
-      "learning_rate": 1.4636635319853274e-05,
-      "loss": 1.0499,
+      "epoch": 0.17,
+      "learning_rate": 1.8568571761675893e-05,
+      "loss": 1.0363,
       "step": 35
     },
     {
-      "epoch": 0.39,
-      "learning_rate": 1.3208257698153677e-05,
-      "loss": 1.0333,
+      "epoch": 0.2,
+      "learning_rate": 1.814439916631857e-05,
+      "loss": 1.038,
       "step": 40
     },
     {
-      "epoch": 0.44,
-      "learning_rate": 1.1702435557223988e-05,
-      "loss": 1.0313,
+      "epoch": 0.22,
+      "learning_rate": 1.7671486066220965e-05,
+      "loss": 1.0281,
       "step": 45
     },
     {
-      "epoch": 0.49,
-      "learning_rate": 1.0155518119203511e-05,
-      "loss": 1.0291,
+      "epoch": 0.25,
+      "learning_rate": 1.7152662630033506e-05,
+      "loss": 1.0167,
       "step": 50
     },
     {
-      "epoch": 0.54,
-      "learning_rate": 8.604846610560771e-06,
-      "loss": 1.0259,
+      "epoch": 0.27,
+      "learning_rate": 1.659103377877423e-05,
+      "loss": 1.0258,
       "step": 55
     },
     {
-      "epoch": 0.59,
-      "learning_rate": 7.0878528777274814e-06,
-      "loss": 1.0259,
+      "epoch": 0.3,
+      "learning_rate": 1.598996060429634e-05,
+      "loss": 1.0134,
       "step": 60
     },
     {
-      "epoch": 0.64,
-      "learning_rate": 5.64115581524629e-06,
-      "loss": 1.0225,
+      "epoch": 0.32,
+      "learning_rate": 1.5353040254690396e-05,
+      "loss": 1.0172,
       "step": 65
     },
+    {
+      "epoch": 0.34,
+      "learning_rate": 1.4684084406997903e-05,
+      "loss": 1.0136,
+      "step": 70
+    },
+    {
+      "epoch": 0.37,
+      "learning_rate": 1.3987096456067236e-05,
+      "loss": 1.0109,
+      "step": 75
+    },
+    {
+      "epoch": 0.39,
+      "learning_rate": 1.3266247556066122e-05,
+      "loss": 0.9972,
+      "step": 80
+    },
+    {
+      "epoch": 0.42,
+      "learning_rate": 1.252585165803135e-05,
+      "loss": 1.0118,
+      "step": 85
+    },
+    {
+      "epoch": 0.44,
+      "learning_rate": 1.1770339692844484e-05,
+      "loss": 1.0013,
+      "step": 90
+    },
+    {
+      "epoch": 0.47,
+      "learning_rate": 1.1004233054136726e-05,
+      "loss": 1.0107,
+      "step": 95
+    },
+    {
+      "epoch": 0.49,
+      "learning_rate": 1.0232116539815558e-05,
+      "loss": 1.0039,
+      "step": 100
+    },
+    {
+      "epoch": 0.52,
+      "learning_rate": 9.458610914145826e-06,
+      "loss": 1.0032,
+      "step": 105
+    },
+    {
+      "epoch": 0.54,
+      "learning_rate": 8.688345254588579e-06,
+      "loss": 1.008,
+      "step": 110
+    },
+    {
+      "epoch": 0.57,
+      "learning_rate": 7.92592924888925e-06,
+      "loss": 1.0066,
+      "step": 115
+    },
+    {
+      "epoch": 0.59,
+      "learning_rate": 7.175925608204428e-06,
+      "loss": 1.006,
+      "step": 120
+    },
+    {
+      "epoch": 0.62,
+      "learning_rate": 6.442822761362015e-06,
+      "loss": 1.0059,
+      "step": 125
+    },
+    {
+      "epoch": 0.64,
+      "learning_rate": 5.731007993667155e-06,
+      "loss": 1.001,
+      "step": 130
+    },
+    {
+      "epoch": 0.67,
+      "learning_rate": 5.044741191005908e-06,
+      "loss": 0.9999,
+      "step": 135
+    },
     {
       "epoch": 0.67,
-      "eval_loss": 1.0241080522537231,
-      "eval_runtime": 741.6789,
-      "eval_samples_per_second": 31.159,
-      "eval_steps_per_second": 0.975,
-      "step": 68
+      "eval_loss": 1.0052766799926758,
+      "eval_runtime": 1465.7345,
+      "eval_samples_per_second": 15.767,
+      "eval_steps_per_second": 0.986,
+      "step": 136
     },
     {
       "epoch": 0.67,
-      "step": 68,
-      "total_flos": 1.2276051069330522e+19,
-      "train_loss": 1.0573345633114086,
-      "train_runtime": 21895.8144,
-      "train_samples_per_second": 9.493,
+      "step": 136,
+      "total_flos": 1.227587501862106e+19,
+      "train_loss": 1.0304046939401066,
+      "train_runtime": 43361.1536,
+      "train_samples_per_second": 4.794,
       "train_steps_per_second": 0.005
     }
   ],
   "logging_steps": 5,
-  "max_steps": 101,
+  "max_steps": 203,
   "num_train_epochs": 1,
   "save_steps": 500,
-  "total_flos": 1.2276051069330522e+19,
+  "total_flos": 1.227587501862106e+19,
   "trial_name": null,
   "trial_params": null
 }

training_args.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:d348904f26b5457e762405e5c80d2f80128cc7ee749ce3fe7a9be01773480935
+oid sha256:48f7991ca3a8ab9ff8e692e6fc77b7cfbdf47cf847068d1e0bd7de23c7999616
 size 4664