Model save

README.md

@@ -1,16 +1,13 @@
 ---
+license: apache-2.0
 library_name: peft
 tags:
-- alignment-handbook
-- trl
-- sft
-- generated_from_trainer
 - trl
 - sft
 - alignment-handbook
 - generated_from_trainer
 datasets:
-- HuggingFaceH4/ultrachat_200k
+- generator
 base_model: mistralai/Mistral-7B-v0.1
 model-index:
 - name: zephyr-7b-pl-qlora
@@ -22,7 +19,7 @@ should probably proofread and complete it, then remove this comment. -->
 
 # zephyr-7b-pl-qlora
 
-This model is a fine-tuned version of [mistralai/Mistral-7B-v0.1](https://huggingface.co/mistralai/Mistral-7B-v0.1) on the HuggingFaceH4/ultrachat_200k dataset.
+This model is a fine-tuned version of [mistralai/Mistral-7B-v0.1](https://huggingface.co/mistralai/Mistral-7B-v0.1) on the generator dataset.
 
 ## Model description
 
@@ -46,9 +43,6 @@ The following hyperparameters were used during training:
 - eval_batch_size: 4
 - seed: 42
 - distributed_type: multi-GPU
-- num_devices: 4
-- total_train_batch_size: 8
-- total_eval_batch_size: 16
 - optimizer: Adam with betas=(0.9,0.999) and epsilon=1e-08
 - lr_scheduler_type: cosine
 - lr_scheduler_warmup_ratio: 0.1

all_results.json

@@ -1,8 +1,8 @@
 {
-    "epoch": 0.01,
-    "train_loss": 1.0116421318054198,
-    "train_runtime": 473.1338,
+    "epoch": 0.0,
+    "train_loss": 0.0,
+    "train_runtime": 501.823,
     "train_samples": 207865,
-    "train_samples_per_second": 1.691,
-    "train_steps_per_second": 0.211
+    "train_samples_per_second": 0.399,
+    "train_steps_per_second": 0.199
 }

lora_4/adapter_config.json

@@ -0,0 +1,37 @@
+{
+  "alpha_pattern": {},
+  "auto_mapping": {
+    "base_model_class": "MistralForCausalLM",
+    "parent_library": "transformers.models.mistral.modeling_mistral"
+  },
+  "base_model_name_or_path": "mistralai/Mistral-7B-v0.1",
+  "bias": "none",
+  "fan_in_fan_out": false,
+  "inference_mode": true,
+  "init_lora_weights": true,
+  "layer_replication": null,
+  "layers_pattern": null,
+  "layers_to_transform": null,
+  "loftq_config": {},
+  "lora_alpha": 16,
+  "lora_dropout": 0.05,
+  "megatron_config": null,
+  "megatron_core": "megatron.core",
+  "modules_to_save": null,
+  "peft_type": "LORA",
+  "r": 16,
+  "rank_pattern": {},
+  "revision": null,
+  "target_modules": [
+    "o_proj",
+    "v_proj",
+    "k_proj",
+    "up_proj",
+    "q_proj",
+    "down_proj",
+    "gate_proj"
+  ],
+  "task_type": null,
+  "use_dora": false,
+  "use_rslora": false
+}

lora_4/adapter_model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:264fb80c3a7830c9942425211af6bdc69e6bc596276a815cc422cd6c999f2d33
+size 167832240

train_results.json

@@ -1,8 +1,8 @@
 {
-    "epoch": 0.01,
-    "train_loss": 1.0116421318054198,
-    "train_runtime": 473.1338,
+    "epoch": 0.0,
+    "train_loss": 0.0,
+    "train_runtime": 501.823,
     "train_samples": 207865,
-    "train_samples_per_second": 1.691,
-    "train_steps_per_second": 0.211
+    "train_samples_per_second": 0.399,
+    "train_steps_per_second": 0.199
 }

trainer_state.json

@@ -1,7 +1,7 @@
 {
   "best_metric": null,
   "best_model_checkpoint": null,
-  "epoch": 0.005737563830397613,
+  "epoch": 0.0014344526845781992,
   "eval_steps": 500,
   "global_step": 100,
   "is_hyper_param_search": false,
@@ -11,47 +11,47 @@
     {
       "epoch": 0.0,
       "learning_rate": 2e-05,
-      "loss": 1.2234,
+      "loss": 0.0,
       "step": 1
     },
     {
       "epoch": 0.0,
       "learning_rate": 0.00019396926207859084,
-      "loss": 1.1179,
+      "loss": 0.0,
       "step": 20
     },
     {
       "epoch": 0.0,
       "learning_rate": 0.00015000000000000001,
-      "loss": 1.0056,
+      "loss": 0.0,
       "step": 40
     },
     {
       "epoch": 0.0,
       "learning_rate": 8.263518223330697e-05,
-      "loss": 0.9795,
+      "loss": 0.0,
       "step": 60
     },
     {
       "epoch": 0.0,
       "learning_rate": 2.339555568810221e-05,
-      "loss": 0.9954,
+      "loss": 0.0,
       "step": 80
     },
     {
-      "epoch": 0.01,
+      "epoch": 0.0,
       "learning_rate": 0.0,
-      "loss": 0.9545,
+      "loss": 0.0,
       "step": 100
     },
     {
-      "epoch": 0.01,
+      "epoch": 0.0,
       "step": 100,
-      "total_flos": 7.072526874181632e+16,
-      "train_loss": 1.0116421318054198,
-      "train_runtime": 473.1338,
-      "train_samples_per_second": 1.691,
-      "train_steps_per_second": 0.211
+      "total_flos": 1.768131718545408e+16,
+      "train_loss": 0.0,
+      "train_runtime": 501.823,
+      "train_samples_per_second": 0.399,
+      "train_steps_per_second": 0.199
     }
   ],
   "logging_steps": 20,
@@ -59,7 +59,7 @@
   "num_input_tokens_seen": 0,
   "num_train_epochs": 1,
   "save_steps": 100,
-  "total_flos": 7.072526874181632e+16,
+  "total_flos": 1.768131718545408e+16,
   "train_batch_size": 2,
   "trial_name": null,
   "trial_params": null