Training in progress, step 100

adapter_config.json

@@ -8,18 +8,24 @@
   "init_lora_weights": true,
   "layers_pattern": null,
   "layers_to_transform": null,
+  "loftq_config": {},
   "lora_alpha": 16,
-  "lora_dropout": 0.1,
+  "lora_dropout": 0.05,
+  "megatron_config": null,
+  "megatron_core": "megatron.core",
   "modules_to_save": null,
   "peft_type": "LORA",
-  "r": 64,
+  "r": 16,
   "rank_pattern": {},
   "revision": null,
   "target_modules": [
+    "up_proj",
     "k_proj",
+    "q_proj",
     "v_proj",
+    "down_proj",
     "o_proj",
-    "q_proj"
+    "gate_proj"
   ],
   "task_type": "CAUSAL_LM"
 }

adapter_model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:264a3c053b11c90dcd9ceeeda0f7537f83dcc7a59125928c4f19031f03be635f
-size 218138576
+oid sha256:ddefb3ba460a5bd67e71d5df0b6dcee87862f5527e3a18273a8638ca090b90b2
+size 83945744

runs/Jan09_05-07-46_ip-26-0-175-170/events.out.tfevents.1704777003.ip-26-0-175-170.1799139.0

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:190bb2c0acf4940294ca8c350d7aa4bcbfa64ec3244ed29d82a50942b96dbc63
+size 12138

tokenizer_config.json

@@ -1,4 +1,6 @@
 {
+  "add_bos_token": true,
+  "add_eos_token": false,
   "added_tokens_decoder": {
     "0": {
       "content": "<unk>",
@@ -37,5 +39,5 @@
   "spaces_between_special_tokens": false,
   "tokenizer_class": "LlamaTokenizer",
   "unk_token": "<unk>",
-  "use_default_system_prompt": true
+  "use_default_system_prompt": false
 }

training_args.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:900c1632fa48b2074ba3b08024549d04fcc0354b9655feb068ee189e456064e2
-size 4728
+oid sha256:8ae0b3e95bcc3fed922ed4dd1c7c89d82e30a226cf3de077e9e69964bb3f02a6
+size 4856