Training in progress, step 500

adapter_config.json

@@ -4,7 +4,7 @@
     "base_model_class": "LlavaNextForConditionalGeneration",
     "parent_library": "transformers.models.llava_next.modeling_llava_next"
   },
-  "base_model_name_or_path": "llava-hf/llava-v1.6-vicuna-7b-hf",
+  "base_model_name_or_path": "llava-hf/llava-v1.6-mistral-7b-hf",
   "bias": "none",
   "fan_in_fan_out": false,
   "inference_mode": true,
@@ -24,11 +24,10 @@
   "revision": null,
   "target_modules": [
     "k_proj",
+    "v_projmm_projectorup_proj",
+    "down_proj",
     "gate_proj",
-    "v_projfc1",
-    "q_proj",
-    "fc2up_proj",
-    "down_proj"
+    "q_proj"
   ],
   "task_type": null,
   "use_dora": false,

adapter_model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:8dedd5497cd5d4febc1e396f4ad8901b2133eb7b38e2ebd726486488b902bf52
-size 406901688
+oid sha256:169d8604dc8a0fc430534667d370722c402792cc4fb04bb9dfbc0a01ef688821
+size 436261832

added_tokens.json

@@ -1,3 +1,4 @@
 {
-  "<image>": 32000
+  "<image>": 32000,
+  "<pad>": 32001
 }

runs/Jul26_15-16-35_ip-10-192-12-185/events.out.tfevents.1722006997.ip-10-192-12-185.22674.0

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:976601913cfc004bc21584200bd137aa25e643217694a2949dbbd8a9715f6ddb
-size 10538
+oid sha256:d705a192f658451ad529cfd9f5756bdda05eafc727814b9ceaf060090bfb380b
+size 10749

runs/Jul26_16-54-58_ip-10-192-12-185/events.out.tfevents.1722012900.ip-10-192-12-185.35787.0

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e937a0338b323a59e075fb62f72bc9dc694e070d878632c8f77ecccbddcbd828
+size 10676

special_tokens_map.json

@@ -14,7 +14,7 @@
     "single_word": false
   },
   "pad_token": {
-    "content": "<unk>",
+    "content": "<pad>",
     "lstrip": false,
     "normalized": false,
     "rstrip": false,

tokenizer.model

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:9e556afd44213b6bd1be2b850ebbbd98f5481437a8021afaf58ee7fb1818d347
-size 499723
+oid sha256:dadfd56d766715c61d2ef780a525ab43b8e6da4de6865bda3d95fdef5e134055
+size 493443

tokenizer_config.json

@@ -1,7 +1,7 @@
 {
   "add_bos_token": true,
   "add_eos_token": false,
-  "add_prefix_space": true,
+  "add_prefix_space": null,
   "added_tokens_decoder": {
     "0": {
       "content": "<unk>",
@@ -34,16 +34,28 @@
       "rstrip": false,
       "single_word": false,
       "special": true
+    },
+    "32001": {
+      "content": "<pad>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
     }
   },
+  "additional_special_tokens": [],
   "bos_token": "<s>",
   "chat_template": "A chat between an user and an artificial intelligence assistant about Science Question Answering. The assistant gives helpful, detailed, and polite answers to the user's questions.\nBased on the image, question and hint, please choose one of the given choices that answer the question.\nGive yourself room to think by extracting the image, question and hint before choosing the choice.\nDon't return the thinking, only return the highest accuracy choice.\nMake sure your answers are as correct as possible.\n{% for tag, content in messages.items() %}\n{% if tag == 'sample_question' %}\nUse the following examples as reference for the ideal answer style.\n{% for message in content %}\n{% if message['role'] == 'user' %} \nExample\nUSER: {% else %}ASSISTANT: {% endif %}\n{% for item in message['content'] %}\n{% if item['type'] == 'text_question' %}\nQuestion: {{ item['question'] }}\n{% elif item['type'] == 'text_hint' %}\nHint: {{ item['hint'] }}\n{% elif item['type'] == 'text_choice' %}\nChoices: {{ item['choice'] }}\n{% elif item['type'] == 'text_solution' %}\nSolution: {{ item['solution'] }}\n{% elif item['type'] == 'text_answer' %}\nAnswer: {{ item['answer'] }}{% elif item['type'] == 'image' %}<image>\n{% endif %}\n{% endfor %}\n{% if message['role'] == 'user' %}\n{% else %}\n{{eos_token}}\n{% endif %}{% endfor %}{% endif %}\n\n{% if tag == 'real_question' %}\nNow use the following image and question to choose the choice:\n{% for message in content %}\n{% if message['role'] == 'user' %}USER: {% else %}ASSISTANT: {% endif %}\n{% for item in message['content'] %}\n{% if item['type'] == 'text_question' %}\nQuestion: {{ item['question'] }}\n{% elif item['type'] == 'text_hint' %}\nHint: {{ item['hint'] }}\n{% elif item['type'] == 'text_choice' %}\nChoices: {{ item['choice'] }}\n{% elif item['type'] == 'text_solution' %}\nSolution: {{ item['solution'] }}\n{% elif item['type'] == 'text_answer' %}\nAnswer: {{ item['answer'] }}{% elif item['type'] == 'image' %}<image>\n{% endif %}\n{% endfor %}\n{% if message['role'] == 'user' %}\n{% else %}\n{{eos_token}}\n{% endif %}{% endfor %}{% endif %}\n{% endfor %}",
   "clean_up_tokenization_spaces": false,
   "eos_token": "</s>",
-  "legacy": false,
-  "model_max_length": 4096,
-  "pad_token": "<unk>",
-  "padding_side": "right",
+  "legacy": true,
+  "max_length": null,
+  "model_max_length": 1000000000000000019884624838656,
+  "pad_to_multiple_of": null,
+  "pad_token": "<pad>",
+  "pad_token_type_id": 0,
+  "padding_side": "left",
   "processor_class": "LlavaNextProcessor",
   "sp_model_kwargs": {},
   "spaces_between_special_tokens": false,

training_args.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:0865f4e9f64ef51679a056063f605c047cdfa2cba01be794dc2ec65f1563909a
+oid sha256:d07b90852007dee836f8471a303db9de6b17957abf8ae2c6ef661d759bf1b8cd
 size 5496