Model save

README.md

@@ -2,15 +2,9 @@
 license: mit
 base_model: HuggingFaceH4/mistral-7b-sft-beta
 tags:
-- alignment-handbook
 - trl
 - dpo
 - generated_from_trainer
-- trl
-- dpo
-- generated_from_trainer
-datasets:
-- HuggingFaceH4/ultrafeedback_binarized
 model-index:
 - name: 0.001_ablation_4iters_bs256_only2third_iter_1
   results: []
@@ -21,7 +15,7 @@ should probably proofread and complete it, then remove this comment. -->
 
 # 0.001_ablation_4iters_bs256_only2third_iter_1
 
-This model is a fine-tuned version of [HuggingFaceH4/mistral-7b-sft-beta](https://huggingface.co/HuggingFaceH4/mistral-7b-sft-beta) on the HuggingFaceH4/ultrafeedback_binarized dataset.
+This model is a fine-tuned version of [HuggingFaceH4/mistral-7b-sft-beta](https://huggingface.co/HuggingFaceH4/mistral-7b-sft-beta) on the None dataset.
 
 ## Model description
 

all_results.json

@@ -1,9 +1,9 @@
 {
     "epoch": 1.0,
     "total_flos": 0.0,
-    "train_loss": 0.6658878326416016,
-    "train_runtime": 1111.8376,
+    "train_loss": 0.6658271312713623,
+    "train_runtime": 1112.3976,
     "train_samples": 10189,
-    "train_samples_per_second": 9.164,
+    "train_samples_per_second": 9.159,
     "train_steps_per_second": 0.036
 }

config.json

@@ -21,6 +21,6 @@
   "tie_word_embeddings": false,
   "torch_dtype": "bfloat16",
   "transformers_version": "4.40.0",
-  "use_cache": true,
+  "use_cache": false,
   "vocab_size": 32000
 }

model-00001-of-00003.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:543cb65482446b412380174e3df863ed809d37793d88df5a4b5af5bbfe145a9a
+oid sha256:a2906f2d96897abb9b67aabc3b231f5470f8729387395809eaf9d6e155fe326b
 size 4943162336

model-00002-of-00003.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:10b980f1f59abc40d6a8197aa3886ffda6f1deb5e82d653b4bbd586b87a9ccac
+oid sha256:033cebedf8a62438cc7bb74bd8aa9a99baf6142272c75a62a4526f44cea01853
 size 4999819336

model-00003-of-00003.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:aaf11da0c53a34ae6f42dbb52abc3c3aa5ae1fb9acc5f3f9e2776121dbd9089a
+oid sha256:476f924bf33d8384b00da00a4a6df8bde197fd07bcdad8cdc9ae01248dcb8c2d
 size 4540516344

train_results.json

@@ -1,9 +1,9 @@
 {
     "epoch": 1.0,
     "total_flos": 0.0,
-    "train_loss": 0.6658878326416016,
-    "train_runtime": 1111.8376,
+    "train_loss": 0.6658271312713623,
+    "train_runtime": 1112.3976,
     "train_samples": 10189,
-    "train_samples_per_second": 9.164,
+    "train_samples_per_second": 9.159,
     "train_steps_per_second": 0.036
 }

trainer_state.json

@@ -10,7 +10,7 @@
   "log_history": [
     {
       "epoch": 0.025,
-      "grad_norm": 6.698416653714476,
+      "grad_norm": 6.6984829306295985,
       "learning_rate": 1.25e-07,
       "logits/chosen": -2.8582587242126465,
       "logits/rejected": -2.842068910598755,
@@ -25,71 +25,71 @@
     },
     {
       "epoch": 0.25,
-      "grad_norm": 6.883361102730669,
+      "grad_norm": 6.880940565037714,
       "learning_rate": 4.6650635094610966e-07,
-      "logits/chosen": -2.7813661098480225,
-      "logits/rejected": -2.772981643676758,
-      "logps/chosen": -254.5003204345703,
-      "logps/rejected": -251.1380157470703,
+      "logits/chosen": -2.781381130218506,
+      "logits/rejected": -2.7730088233947754,
+      "logps/chosen": -254.4922637939453,
+      "logps/rejected": -251.15196228027344,
       "loss": 0.6908,
-      "rewards/accuracies": 0.5381944179534912,
-      "rewards/chosen": 0.005406244192272425,
-      "rewards/margins": 0.0035838009789586067,
-      "rewards/rejected": 0.001822443911805749,
+      "rewards/accuracies": 0.5277777910232544,
+      "rewards/chosen": 0.005487086251378059,
+      "rewards/margins": 0.0038041824009269476,
+      "rewards/rejected": 0.0016829040832817554,
       "step": 10
     },
     {
       "epoch": 0.5,
-      "grad_norm": 6.43798743744834,
+      "grad_norm": 6.415866483339294,
       "learning_rate": 2.934120444167326e-07,
-      "logits/chosen": -2.7719171047210693,
-      "logits/rejected": -2.7447400093078613,
-      "logps/chosen": -265.3804931640625,
-      "logps/rejected": -252.05618286132812,
-      "loss": 0.6742,
-      "rewards/accuracies": 0.6812499761581421,
-      "rewards/chosen": 0.03401705250144005,
-      "rewards/margins": 0.04408254474401474,
-      "rewards/rejected": -0.01006548572331667,
+      "logits/chosen": -2.771768569946289,
+      "logits/rejected": -2.744568347930908,
+      "logps/chosen": -265.42926025390625,
+      "logps/rejected": -252.08932495117188,
+      "loss": 0.6741,
+      "rewards/accuracies": 0.671875,
+      "rewards/chosen": 0.03352969139814377,
+      "rewards/margins": 0.04392678663134575,
+      "rewards/rejected": -0.010397094301879406,
       "step": 20
     },
     {
       "epoch": 0.75,
-      "grad_norm": 7.502439531203193,
+      "grad_norm": 7.499103181183479,
       "learning_rate": 8.930309757836516e-08,
-      "logits/chosen": -2.761791467666626,
-      "logits/rejected": -2.740384578704834,
-      "logps/chosen": -258.3692626953125,
-      "logps/rejected": -249.21566772460938,
-      "loss": 0.6515,
-      "rewards/accuracies": 0.6781250238418579,
-      "rewards/chosen": -0.02331426367163658,
-      "rewards/margins": 0.07361713796854019,
-      "rewards/rejected": -0.09693139791488647,
+      "logits/chosen": -2.7617874145507812,
+      "logits/rejected": -2.7403435707092285,
+      "logps/chosen": -258.38079833984375,
+      "logps/rejected": -249.26309204101562,
+      "loss": 0.6514,
+      "rewards/accuracies": 0.671875,
+      "rewards/chosen": -0.02342965081334114,
+      "rewards/margins": 0.07397619634866714,
+      "rewards/rejected": -0.09740584343671799,
       "step": 30
     },
     {
       "epoch": 1.0,
-      "grad_norm": 7.122574066847829,
+      "grad_norm": 7.140558621795842,
       "learning_rate": 0.0,
-      "logits/chosen": -2.7583651542663574,
-      "logits/rejected": -2.733116626739502,
-      "logps/chosen": -266.1463317871094,
-      "logps/rejected": -273.2423095703125,
-      "loss": 0.6468,
-      "rewards/accuracies": 0.6000000238418579,
-      "rewards/chosen": -0.05725393444299698,
-      "rewards/margins": 0.06221733242273331,
-      "rewards/rejected": -0.11947127431631088,
+      "logits/chosen": -2.7583961486816406,
+      "logits/rejected": -2.7331676483154297,
+      "logps/chosen": -266.16204833984375,
+      "logps/rejected": -273.2425537109375,
+      "loss": 0.6467,
+      "rewards/accuracies": 0.59375,
+      "rewards/chosen": -0.05741094797849655,
+      "rewards/margins": 0.062062572687864304,
+      "rewards/rejected": -0.11947351694107056,
       "step": 40
     },
     {
       "epoch": 1.0,
       "step": 40,
       "total_flos": 0.0,
-      "train_loss": 0.6658878326416016,
-      "train_runtime": 1111.8376,
-      "train_samples_per_second": 9.164,
+      "train_loss": 0.6658271312713623,
+      "train_runtime": 1112.3976,
+      "train_samples_per_second": 9.159,
       "train_steps_per_second": 0.036
     }
   ],

training_args.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:d4d40b454190180c3f41b32904e3920440257a19d99dd810e99901ef41b7949b
+oid sha256:25d0c445bcb8c314207cfd174ae581a7de9a0c99eaa33cc1c0abb147597b80a9
 size 6328