Model save

README.md

@@ -2,15 +2,9 @@
 license: mit
 base_model: HuggingFaceH4/mistral-7b-sft-beta
 tags:
-- alignment-handbook
 - trl
 - dpo
 - generated_from_trainer
-- trl
-- dpo
-- generated_from_trainer
-datasets:
-- HuggingFaceH4/ultrafeedback_binarized
 model-index:
 - name: 0.001_ablation_4iters_bs256_only2third_iter_1
   results: []
@@ -21,7 +15,7 @@ should probably proofread and complete it, then remove this comment. -->
 
 # 0.001_ablation_4iters_bs256_only2third_iter_1
 
-This model is a fine-tuned version of [HuggingFaceH4/mistral-7b-sft-beta](https://huggingface.co/HuggingFaceH4/mistral-7b-sft-beta) on the HuggingFaceH4/ultrafeedback_binarized dataset.
+This model is a fine-tuned version of [HuggingFaceH4/mistral-7b-sft-beta](https://huggingface.co/HuggingFaceH4/mistral-7b-sft-beta) on the None dataset.
 
 ## Model description
 

all_results.json

@@ -1,9 +1,9 @@
 {
     "epoch": 1.0,
     "total_flos": 0.0,
-    "train_loss": 0.6658724308013916,
-    "train_runtime": 1113.1816,
+    "train_loss": 0.6658878326416016,
+    "train_runtime": 1111.8376,
     "train_samples": 10189,
-    "train_samples_per_second": 9.153,
+    "train_samples_per_second": 9.164,
     "train_steps_per_second": 0.036
 }

config.json

@@ -21,6 +21,6 @@
   "tie_word_embeddings": false,
   "torch_dtype": "bfloat16",
   "transformers_version": "4.40.0",
-  "use_cache": true,
+  "use_cache": false,
   "vocab_size": 32000
 }

model-00001-of-00003.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:9b9201be95a2305af8ac9a9549e00720ea3404303f1266cee03e54504afab8d1
+oid sha256:543cb65482446b412380174e3df863ed809d37793d88df5a4b5af5bbfe145a9a
 size 4943162336

model-00002-of-00003.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:6395558745a715df764ba37724bab66d642482bd185639b51b1ac50816180119
+oid sha256:10b980f1f59abc40d6a8197aa3886ffda6f1deb5e82d653b4bbd586b87a9ccac
 size 4999819336

model-00003-of-00003.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:64a584896eca7ba3f7f8d3f3dd78e9ae54b9229ca896101f03712dfc147ced98
+oid sha256:aaf11da0c53a34ae6f42dbb52abc3c3aa5ae1fb9acc5f3f9e2776121dbd9089a
 size 4540516344

train_results.json

@@ -1,9 +1,9 @@
 {
     "epoch": 1.0,
     "total_flos": 0.0,
-    "train_loss": 0.6658724308013916,
-    "train_runtime": 1113.1816,
+    "train_loss": 0.6658878326416016,
+    "train_runtime": 1111.8376,
     "train_samples": 10189,
-    "train_samples_per_second": 9.153,
+    "train_samples_per_second": 9.164,
     "train_steps_per_second": 0.036
 }

trainer_state.json

@@ -10,7 +10,7 @@
   "log_history": [
     {
       "epoch": 0.025,
-      "grad_norm": 6.698945088060824,
+      "grad_norm": 6.698416653714476,
       "learning_rate": 1.25e-07,
       "logits/chosen": -2.8582587242126465,
       "logits/rejected": -2.842068910598755,
@@ -25,71 +25,71 @@
     },
     {
       "epoch": 0.25,
-      "grad_norm": 6.888664410561039,
+      "grad_norm": 6.883361102730669,
       "learning_rate": 4.6650635094610966e-07,
-      "logits/chosen": -2.781330108642578,
-      "logits/rejected": -2.7729134559631348,
-      "logps/chosen": -254.4892578125,
-      "logps/rejected": -251.1736602783203,
+      "logits/chosen": -2.7813661098480225,
+      "logits/rejected": -2.772981643676758,
+      "logps/chosen": -254.5003204345703,
+      "logps/rejected": -251.1380157470703,
       "loss": 0.6908,
-      "rewards/accuracies": 0.5208333134651184,
-      "rewards/chosen": 0.005517133977264166,
-      "rewards/margins": 0.004050815477967262,
-      "rewards/rejected": 0.0014663181500509381,
+      "rewards/accuracies": 0.5381944179534912,
+      "rewards/chosen": 0.005406244192272425,
+      "rewards/margins": 0.0035838009789586067,
+      "rewards/rejected": 0.001822443911805749,
       "step": 10
     },
     {
       "epoch": 0.5,
-      "grad_norm": 6.422517638432347,
+      "grad_norm": 6.43798743744834,
       "learning_rate": 2.934120444167326e-07,
-      "logits/chosen": -2.7719380855560303,
-      "logits/rejected": -2.7447848320007324,
-      "logps/chosen": -265.4648132324219,
-      "logps/rejected": -252.11721801757812,
+      "logits/chosen": -2.7719171047210693,
+      "logits/rejected": -2.7447400093078613,
+      "logps/chosen": -265.3804931640625,
+      "logps/rejected": -252.05618286132812,
       "loss": 0.6742,
-      "rewards/accuracies": 0.6781250238418579,
-      "rewards/chosen": 0.03317389637231827,
-      "rewards/margins": 0.04384994134306908,
-      "rewards/rejected": -0.010676050558686256,
+      "rewards/accuracies": 0.6812499761581421,
+      "rewards/chosen": 0.03401705250144005,
+      "rewards/margins": 0.04408254474401474,
+      "rewards/rejected": -0.01006548572331667,
       "step": 20
     },
     {
       "epoch": 0.75,
-      "grad_norm": 7.490219062263049,
+      "grad_norm": 7.502439531203193,
       "learning_rate": 8.930309757836516e-08,
-      "logits/chosen": -2.7620229721069336,
-      "logits/rejected": -2.740633487701416,
-      "logps/chosen": -258.3665466308594,
-      "logps/rejected": -249.21975708007812,
-      "loss": 0.6514,
-      "rewards/accuracies": 0.671875,
-      "rewards/chosen": -0.02328680083155632,
-      "rewards/margins": 0.07368560880422592,
-      "rewards/rejected": -0.09697240591049194,
+      "logits/chosen": -2.761791467666626,
+      "logits/rejected": -2.740384578704834,
+      "logps/chosen": -258.3692626953125,
+      "logps/rejected": -249.21566772460938,
+      "loss": 0.6515,
+      "rewards/accuracies": 0.6781250238418579,
+      "rewards/chosen": -0.02331426367163658,
+      "rewards/margins": 0.07361713796854019,
+      "rewards/rejected": -0.09693139791488647,
       "step": 30
     },
     {
       "epoch": 1.0,
-      "grad_norm": 7.121984785862904,
+      "grad_norm": 7.122574066847829,
       "learning_rate": 0.0,
-      "logits/chosen": -2.758615016937256,
-      "logits/rejected": -2.7334706783294678,
-      "logps/chosen": -266.12847900390625,
-      "logps/rejected": -273.2154541015625,
+      "logits/chosen": -2.7583651542663574,
+      "logits/rejected": -2.733116626739502,
+      "logps/chosen": -266.1463317871094,
+      "logps/rejected": -273.2423095703125,
       "loss": 0.6468,
-      "rewards/accuracies": 0.590624988079071,
-      "rewards/chosen": -0.05707535147666931,
-      "rewards/margins": 0.062127478420734406,
-      "rewards/rejected": -0.11920282989740372,
+      "rewards/accuracies": 0.6000000238418579,
+      "rewards/chosen": -0.05725393444299698,
+      "rewards/margins": 0.06221733242273331,
+      "rewards/rejected": -0.11947127431631088,
       "step": 40
     },
     {
       "epoch": 1.0,
       "step": 40,
       "total_flos": 0.0,
-      "train_loss": 0.6658724308013916,
-      "train_runtime": 1113.1816,
-      "train_samples_per_second": 9.153,
+      "train_loss": 0.6658878326416016,
+      "train_runtime": 1111.8376,
+      "train_samples_per_second": 9.164,
       "train_steps_per_second": 0.036
     }
   ],

training_args.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:fa84316d3d3c5012ddc31a9fbc319ffde3eb823e9cd944ce6be549d29f9e757f
+oid sha256:d4d40b454190180c3f41b32904e3920440257a19d99dd810e99901ef41b7949b
 size 6328