End of training

README.md

@@ -4,6 +4,7 @@ tags:
 model-index:
 - name: Kaggle-Science-LLM
   results: []
+library_name: peft
 ---
 
 <!-- This model card has been generated automatically according to the information the Trainer had access to. You
@@ -29,6 +30,17 @@ More information needed
 
 ## Training procedure
 
+
+The following `bitsandbytes` quantization config was used during training:
+- load_in_8bit: False
+- load_in_4bit: True
+- llm_int8_threshold: 6.0
+- llm_int8_skip_modules: None
+- llm_int8_enable_fp32_cpu_offload: False
+- llm_int8_has_fp16_weight: False
+- bnb_4bit_quant_type: nf4
+- bnb_4bit_use_double_quant: True
+- bnb_4bit_compute_dtype: bfloat16
 ### Training hyperparameters
 
 The following hyperparameters were used during training:
@@ -62,6 +74,7 @@ The following hyperparameters were used during training:
 
 ### Framework versions
 
+- PEFT 0.4.0
 - Transformers 4.30.2
 - Pytorch 2.0.0
 - Datasets 2.1.0

adapter_config.json

@@ -1,6 +1,6 @@
 {
   "auto_mapping": null,
-  "base_model_name_or_path": "meta-llama/Llama-2-7b-hf",
+  "base_model_name_or_path": null,
   "bias": "none",
   "fan_in_fan_out": false,
   "inference_mode": true,

adapter_model.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:554d9152e3fb5cfbb0656095bbeb1e9757e5e6dd5bea8e4ffa086b485a3d19c8
-size 134263757
+oid sha256:f13c037e9d5e77ed7381fbf545355fcdaa357489fe4fda768ad3b262980a490c
+size 134265933

all_results.json

@@ -1,8 +1,8 @@
 {
     "epoch": 0.08,
     "total_flos": 1782763267522560.0,
-    "train_loss": 5.71207763671875,
-    "train_runtime": 12536.5893,
+    "train_loss": 5.688729438781738,
+    "train_runtime": 12357.0887,
     "train_samples_per_second": 0.032,
     "train_steps_per_second": 0.004
 }

runs/Oct28_18-27-05_948ebb293dc9/events.out.tfevents.1698518147.948ebb293dc9.233.0

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:92c32358e8ceae5d1f8bef28288d8af98b7995ddb55f9070714d6b8673638d8a
+size 8967

train_results.json

@@ -1,8 +1,8 @@
 {
     "epoch": 0.08,
     "total_flos": 1782763267522560.0,
-    "train_loss": 5.71207763671875,
-    "train_runtime": 12536.5893,
+    "train_loss": 5.688729438781738,
+    "train_runtime": 12357.0887,
     "train_samples_per_second": 0.032,
     "train_steps_per_second": 0.004
 }

trainer_state.json

@@ -10,149 +10,149 @@
     {
       "epoch": 0.01,
       "learning_rate": 1.4960191082802548e-05,
-      "loss": 6.6677,
+      "loss": 6.6679,
       "step": 5
     },
     {
       "epoch": 0.01,
-      "eval_loss": 6.512014389038086,
-      "eval_runtime": 1183.8648,
-      "eval_samples_per_second": 1.414,
-      "eval_steps_per_second": 0.354,
+      "eval_loss": 6.511322021484375,
+      "eval_runtime": 1175.4522,
+      "eval_samples_per_second": 1.424,
+      "eval_steps_per_second": 0.356,
       "step": 5
     },
     {
       "epoch": 0.02,
       "learning_rate": 1.4920382165605097e-05,
-      "loss": 6.4854,
+      "loss": 6.4844,
       "step": 10
     },
     {
       "epoch": 0.02,
-      "eval_loss": 6.347909927368164,
-      "eval_runtime": 1183.7191,
-      "eval_samples_per_second": 1.414,
-      "eval_steps_per_second": 0.354,
+      "eval_loss": 6.34607458114624,
+      "eval_runtime": 1171.2756,
+      "eval_samples_per_second": 1.429,
+      "eval_steps_per_second": 0.358,
       "step": 10
     },
     {
       "epoch": 0.02,
       "learning_rate": 1.4880573248407642e-05,
-      "loss": 6.2537,
+      "loss": 6.2521,
       "step": 15
     },
     {
       "epoch": 0.02,
-      "eval_loss": 6.1641035079956055,
-      "eval_runtime": 1184.2747,
-      "eval_samples_per_second": 1.414,
-      "eval_steps_per_second": 0.354,
+      "eval_loss": 6.161616802215576,
+      "eval_runtime": 1175.2918,
+      "eval_samples_per_second": 1.424,
+      "eval_steps_per_second": 0.357,
       "step": 15
     },
     {
       "epoch": 0.03,
       "learning_rate": 1.4840764331210191e-05,
-      "loss": 6.0912,
+      "loss": 6.0889,
       "step": 20
     },
     {
       "epoch": 0.03,
-      "eval_loss": 5.954966068267822,
-      "eval_runtime": 1187.8538,
-      "eval_samples_per_second": 1.409,
-      "eval_steps_per_second": 0.353,
+      "eval_loss": 5.951450347900391,
+      "eval_runtime": 1163.7316,
+      "eval_samples_per_second": 1.438,
+      "eval_steps_per_second": 0.36,
       "step": 20
     },
     {
       "epoch": 0.04,
       "learning_rate": 1.4800955414012738e-05,
-      "loss": 5.8341,
+      "loss": 5.8295,
       "step": 25
     },
     {
       "epoch": 0.04,
-      "eval_loss": 5.724631309509277,
-      "eval_runtime": 1188.1346,
-      "eval_samples_per_second": 1.409,
-      "eval_steps_per_second": 0.353,
+      "eval_loss": 5.720163345336914,
+      "eval_runtime": 1170.5429,
+      "eval_samples_per_second": 1.43,
+      "eval_steps_per_second": 0.358,
       "step": 25
     },
     {
       "epoch": 0.05,
       "learning_rate": 1.4761146496815287e-05,
-      "loss": 5.6128,
+      "loss": 5.6072,
       "step": 30
     },
     {
       "epoch": 0.05,
-      "eval_loss": 5.47757625579834,
-      "eval_runtime": 1188.0744,
-      "eval_samples_per_second": 1.409,
-      "eval_steps_per_second": 0.353,
+      "eval_loss": 5.472379207611084,
+      "eval_runtime": 1167.985,
+      "eval_samples_per_second": 1.433,
+      "eval_steps_per_second": 0.359,
       "step": 30
     },
     {
       "epoch": 0.06,
-      "learning_rate": 1.4729299363057326e-05,
-      "loss": 5.3665,
+      "learning_rate": 1.4721337579617834e-05,
+      "loss": 5.339,
       "step": 35
     },
     {
       "epoch": 0.06,
-      "eval_loss": 5.272798538208008,
-      "eval_runtime": 1185.4582,
-      "eval_samples_per_second": 1.412,
-      "eval_steps_per_second": 0.353,
+      "eval_loss": 5.213606357574463,
+      "eval_runtime": 1168.8328,
+      "eval_samples_per_second": 1.432,
+      "eval_steps_per_second": 0.358,
       "step": 35
     },
     {
       "epoch": 0.06,
-      "learning_rate": 1.4689490445859873e-05,
-      "loss": 5.1581,
+      "learning_rate": 1.4681528662420383e-05,
+      "loss": 5.0985,
       "step": 40
     },
     {
       "epoch": 0.06,
-      "eval_loss": 5.01292610168457,
-      "eval_runtime": 1184.7537,
-      "eval_samples_per_second": 1.413,
-      "eval_steps_per_second": 0.354,
+      "eval_loss": 4.951411724090576,
+      "eval_runtime": 1160.2184,
+      "eval_samples_per_second": 1.443,
+      "eval_steps_per_second": 0.361,
       "step": 40
     },
     {
       "epoch": 0.07,
-      "learning_rate": 1.464968152866242e-05,
-      "loss": 4.9526,
+      "learning_rate": 1.4641719745222929e-05,
+      "loss": 4.8879,
       "step": 45
     },
     {
       "epoch": 0.07,
-      "eval_loss": 4.750097274780273,
-      "eval_runtime": 1184.6596,
-      "eval_samples_per_second": 1.413,
-      "eval_steps_per_second": 0.354,
+      "eval_loss": 4.6860737800598145,
+      "eval_runtime": 1166.5033,
+      "eval_samples_per_second": 1.435,
+      "eval_steps_per_second": 0.359,
       "step": 45
     },
     {
       "epoch": 0.08,
-      "learning_rate": 1.4609872611464967e-05,
-      "loss": 4.6988,
+      "learning_rate": 1.4601910828025478e-05,
+      "loss": 4.6319,
       "step": 50
     },
     {
       "epoch": 0.08,
-      "eval_loss": 4.482077598571777,
-      "eval_runtime": 1185.1466,
-      "eval_samples_per_second": 1.412,
-      "eval_steps_per_second": 0.354,
+      "eval_loss": 4.414491176605225,
+      "eval_runtime": 1165.2141,
+      "eval_samples_per_second": 1.437,
+      "eval_steps_per_second": 0.36,
       "step": 50
     },
     {
       "epoch": 0.08,
       "step": 50,
       "total_flos": 1782763267522560.0,
-      "train_loss": 5.71207763671875,
-      "train_runtime": 12536.5893,
+      "train_loss": 5.688729438781738,
+      "train_runtime": 12357.0887,
       "train_samples_per_second": 0.032,
       "train_steps_per_second": 0.004
     }

training_args.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:b63c609acbfe9fb282b8b5cd86b3af90ef32052a72c6a691f1bdff0059fe5a04
+oid sha256:5e7788309affb869aa11990ec5cffd9f89ef97293c0db6c8ea7c29509cb00fb3
 size 3963