Model save

README.md

@@ -18,13 +18,13 @@ should probably proofread and complete it, then remove this comment. -->
 
 This model is a fine-tuned version of [HuggingFaceH4/zephyr-7b-beta](https://huggingface.co/HuggingFaceH4/zephyr-7b-beta) on the None dataset.
 It achieves the following results on the evaluation set:
-- Loss: 0.0052
-- Rewards/chosen: 6.8557
-- Rewards/rejected: -40.2341
-- Rewards/margins: 47.0897
-- Kl: 0.0
-- Logps/chosen: -138.8257
-- Logps/rejected: -603.3300
+- Loss: 0.0306
+- Rewards/chosen: 5.5134
+- Logps/chosen: -155.1431
+- Rewards/rejected: -19.7208
+- Logps/rejected: -384.9552
+- Kl: 0.3506
+- Rewards/margins: 25.2388
 
 ## Model description
 
@@ -45,35 +45,21 @@ More information needed
 The following hyperparameters were used during training:
 - learning_rate: 0.0002
 - train_batch_size: 4
-- eval_batch_size: 4
+- eval_batch_size: 2
 - seed: 42
 - gradient_accumulation_steps: 8
 - total_train_batch_size: 32
 - optimizer: Adam with betas=(0.9,0.999) and epsilon=1e-08
 - lr_scheduler_type: linear
 - lr_scheduler_warmup_ratio: 0.03
-- training_steps: 786
+- training_steps: 96
 - mixed_precision_training: Native AMP
 
 ### Training results
 
-| Training Loss | Epoch | Step | Validation Loss | Rewards/chosen | Rewards/rejected | Rewards/margins | Kl  | Logps/chosen | Logps/rejected |
-|:-------------:|:-----:|:----:|:---------------:|:--------------:|:----------------:|:---------------:|:---:|:------------:|:--------------:|
-| 0.1911        | 0.38  | 50   | 0.0198          | 5.0250         | -23.0117         | 28.0367         | 0.0 | -157.1321    | -431.1063      |
-| 0.0525        | 0.76  | 100  | 0.0155          | 5.6300         | -30.1143         | 35.7443         | 0.0 | -151.0826    | -502.1325      |
-| 0.0684        | 1.14  | 150  | 0.0104          | 6.1959         | -32.9347         | 39.1306         | 0.0 | -145.4235    | -530.3358      |
-| 0.0253        | 1.52  | 200  | 0.0126          | 5.9530         | -28.8204         | 34.7734         | 0.0 | -147.8525    | -489.1933      |
-| 0.0177        | 1.9   | 250  | 0.0163          | 6.4453         | -53.9510         | 60.3963         | 0.0 | -142.9294    | -740.4993      |
-| 0.0126        | 2.28  | 300  | 0.0206          | 6.2002         | -65.0484         | 71.2486         | 0.0 | -145.3807    | -851.4736      |
-| 0.007         | 2.66  | 350  | 0.0068          | 6.6872         | -41.5169         | 48.2041         | 0.0 | -140.5102    | -616.1578      |
-| 0.0085        | 3.04  | 400  | 0.0076          | 6.7677         | -52.6223         | 59.3899         | 0.0 | -139.7057    | -727.2117      |
-| 0.0057        | 3.43  | 450  | 0.0060          | 6.5419         | -38.1888         | 44.7308         | 0.0 | -141.9630    | -582.8775      |
-| 0.0029        | 3.81  | 500  | 0.0061          | 6.7190         | -37.0418         | 43.7608         | 0.0 | -140.1928    | -571.4076      |
-| 0.003         | 4.19  | 550  | 0.0062          | 6.8462         | -36.6584         | 43.5045         | 0.0 | -138.9207    | -567.5727      |
-| 0.0023        | 4.57  | 600  | 0.0058          | 6.8631         | -38.2151         | 45.0782         | 0.0 | -138.7514    | -583.1401      |
-| 0.0028        | 4.95  | 650  | 0.0050          | 6.8261         | -40.4010         | 47.2271         | 0.0 | -139.1210    | -604.9990      |
-| 0.0029        | 5.33  | 700  | 0.0052          | 6.8557         | -40.2341         | 47.0897         | 0.0 | -138.8257    | -603.3300      |
-| 0.0022        | 5.71  | 750  | 0.0052          | 6.8628         | -40.4610         | 47.3239         | 0.0 | -138.7541    | -605.5995      |
+| Training Loss | Epoch | Step | Validation Loss | Rewards/chosen | Logps/chosen | Rewards/rejected | Logps/rejected | Kl     | Rewards/margins |
+|:-------------:|:-----:|:----:|:---------------:|:--------------:|:------------:|:----------------:|:--------------:|:------:|:---------------:|
+| 0.0687        | 0.38  | 50   | 0.0442          | 4.9556         | -160.7203    | -13.7592         | -325.3391      | 0.6980 | 18.5049         |
 
 
 ### Framework versions
@@ -82,4 +68,4 @@ The following hyperparameters were used during training:
 - Transformers 4.39.1
 - Pytorch 2.1.0+cu118
 - Datasets 2.18.0
-- Tokenizers 0.15.2
+- Tokenizers 0.15.1

adapter_config.json

@@ -20,10 +20,10 @@
   "rank_pattern": {},
   "revision": null,
   "target_modules": [
-    "k_proj",
-    "q_proj",
     "o_proj",
-    "v_proj"
+    "q_proj",
+    "v_proj",
+    "k_proj"
   ],
   "task_type": "CAUSAL_LM",
   "use_dora": false,

adapter_model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:e9ff12f3f4d5befc156462ce388fd17f745898fa371492d3ca8fd33ef9dd7c58
-size 54560368
+oid sha256:099fac32cd1fbefde302f0c47045a7a1fe2d172ff0a9c19cabee49171a0d67d0
+size 1103203256

added_tokens.json

@@ -0,0 +1,4 @@
+{
+  "<|im_end|>": 32001,
+  "<|im_start|>": 32000
+}

special_tokens_map.json

@@ -1,24 +1,23 @@
 {
   "additional_special_tokens": [
-    "<unk>",
-    "<s>",
-    "</s>"
+    {
+      "content": "<|im_start|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false
+    },
+    {
+      "content": "<|im_end|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false
+    }
   ],
-  "bos_token": {
-    "content": "<s>",
-    "lstrip": false,
-    "normalized": false,
-    "rstrip": false,
-    "single_word": false
-  },
-  "eos_token": {
-    "content": "</s>",
-    "lstrip": false,
-    "normalized": false,
-    "rstrip": false,
-    "single_word": false
-  },
-  "pad_token": "<unk>",
+  "bos_token": "<|im_start|>",
+  "eos_token": "<|im_end|>",
+  "pad_token": "<|im_end|>",
   "unk_token": {
     "content": "<unk>",
     "lstrip": false,

tokenizer.json

@@ -29,6 +29,24 @@
       "rstrip": false,
       "normalized": false,
       "special": true
+    },
+    {
+      "id": 32000,
+      "content": "<|im_start|>",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    },
+    {
+      "id": 32001,
+      "content": "<|im_end|>",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
     }
   ],
   "normalizer": {

tokenizer_config.json

@@ -25,21 +25,36 @@
       "rstrip": false,
       "single_word": false,
       "special": true
+    },
+    "32000": {
+      "content": "<|im_start|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "32001": {
+      "content": "<|im_end|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
     }
   },
   "additional_special_tokens": [
-    "<unk>",
-    "<s>",
-    "</s>"
+    "<|im_start|>",
+    "<|im_end|>"
   ],
-  "bos_token": "<s>",
-  "chat_template": "{% for message in messages %}\n{% if message['role'] == 'user' %}\n{{ '<|user|>\n' + message['content'] + eos_token }}\n{% elif message['role'] == 'system' %}\n{{ '<|system|>\n' + message['content'] + eos_token }}\n{% elif message['role'] == 'assistant' %}\n{{ '<|assistant|>\n'  + message['content'] + eos_token }}\n{% endif %}\n{% if loop.last and add_generation_prompt %}\n{{ '<|assistant|>' }}\n{% endif %}\n{% endfor %}",
+  "bos_token": "<|im_start|>",
+  "chat_template": "{% if not add_generation_prompt is defined %}{% set add_generation_prompt = false %}{% endif %}{% for message in messages %}{{'<|im_start|>' + message['role'] + '\n' + message['content'] + '<|im_end|>' + '\n'}}{% endfor %}{% if add_generation_prompt %}{{ '<|im_start|>assistant\n' }}{% endif %}",
   "clean_up_tokenization_spaces": false,
-  "eos_token": "</s>",
+  "eos_token": "<|im_end|>",
   "legacy": true,
   "max_lenght": 8192,
   "model_max_length": 1000000000000000019884624838656,
-  "pad_token": "<unk>",
+  "pad_token": "<|im_end|>",
   "padding": true,
   "sp_model_kwargs": {},
   "spaces_between_special_tokens": false,

training_args.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:72ac543b1af4f9675ddc3d87c79e0f49fbd145783e3bc0097b4be05d98c85865
-size 5752
+oid sha256:9d7d2996f7c9647166fa0a6de85aa22f4274af7424b79d82eb5d5b498f157d56
+size 5688