Training in progress, epoch 0

config.json

@@ -0,0 +1,27 @@
+{
+  "_name_or_path": "mistralai/Mistral-7B-v0.1",
+  "architectures": [
+    "MistralForCausalLM"
+  ],
+  "attention_dropout": 0.0,
+  "bos_token_id": 1,
+  "eos_token_id": 2,
+  "head_dim": 128,
+  "hidden_act": "silu",
+  "hidden_size": 4096,
+  "initializer_range": 0.02,
+  "intermediate_size": 14336,
+  "max_position_embeddings": 32768,
+  "model_type": "mistral",
+  "num_attention_heads": 32,
+  "num_hidden_layers": 32,
+  "num_key_value_heads": 8,
+  "rms_norm_eps": 1e-05,
+  "rope_theta": 10000.0,
+  "sliding_window": 4096,
+  "tie_word_embeddings": false,
+  "torch_dtype": "bfloat16",
+  "transformers_version": "4.46.1",
+  "use_cache": false,
+  "vocab_size": 32000
+}

model-00001-of-00003.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:cd8d06477aba2fe894767d0e8f5a043242815139febb37c396eceba16b4454ca
+size 4943162336

model-00002-of-00003.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:3f7661d5d96516aba00d3785d53258068228765b7ec9a317ba3eaeef7f3002bf
+size 4999819336

model-00003-of-00003.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:9d89193881d5c83bd75c79c7ba29a167237faac6f3cca441f2a20a940edcf592
+size 4540516344

model.safetensors.index.json

@@ -0,0 +1,298 @@
+{
+  "metadata": {
+    "total_size": 14483464192
+  },
+  "weight_map": {
+    "lm_head.weight": "model-00003-of-00003.safetensors",
+    "model.embed_tokens.weight": "model-00001-of-00003.safetensors",
+    "model.layers.0.input_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.0.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.0.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.0.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.0.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.0.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.0.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.0.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.0.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.1.input_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.1.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.1.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.1.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.1.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.1.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.1.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.1.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.1.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.10.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.10.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.10.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.10.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.10.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.10.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.10.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.10.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.10.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.11.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.11.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.11.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.11.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.11.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.11.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.11.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.11.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.11.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.12.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.12.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.12.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.12.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.12.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.12.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.12.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.12.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.12.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.13.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.13.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.13.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.13.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.13.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.13.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.13.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.13.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.13.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.14.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.14.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.14.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.14.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.14.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.14.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.14.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.14.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.14.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.15.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.15.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.15.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.15.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.15.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.15.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.15.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.15.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.15.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.16.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.16.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.16.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.16.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.16.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.16.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.16.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.16.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.16.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.17.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.17.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.17.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.17.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.17.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.17.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.17.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.17.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.17.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.18.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.18.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.18.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.18.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.18.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.18.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.18.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.18.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.18.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.19.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.19.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.19.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.19.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.19.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.19.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.19.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.19.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.19.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.2.input_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.2.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.2.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.2.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.2.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.2.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.2.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.2.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.2.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.20.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.20.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.20.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.20.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.20.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.20.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.20.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.20.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.20.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.21.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.21.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.21.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.21.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.21.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.21.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.21.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.21.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.21.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.22.input_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.22.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.22.mlp.gate_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.22.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.22.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.22.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.22.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.22.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.22.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.23.input_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.23.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.23.mlp.gate_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.23.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.23.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.23.self_attn.k_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.23.self_attn.o_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.23.self_attn.q_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.23.self_attn.v_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.24.input_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.24.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.24.mlp.gate_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.24.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.24.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.24.self_attn.k_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.24.self_attn.o_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.24.self_attn.q_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.24.self_attn.v_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.25.input_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.25.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.25.mlp.gate_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.25.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.25.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.25.self_attn.k_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.25.self_attn.o_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.25.self_attn.q_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.25.self_attn.v_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.26.input_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.26.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.26.mlp.gate_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.26.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.26.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.26.self_attn.k_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.26.self_attn.o_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.26.self_attn.q_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.26.self_attn.v_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.27.input_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.27.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.27.mlp.gate_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.27.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.27.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.27.self_attn.k_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.27.self_attn.o_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.27.self_attn.q_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.27.self_attn.v_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.28.input_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.28.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.28.mlp.gate_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.28.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.28.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.28.self_attn.k_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.28.self_attn.o_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.28.self_attn.q_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.28.self_attn.v_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.29.input_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.29.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.29.mlp.gate_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.29.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.29.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.29.self_attn.k_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.29.self_attn.o_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.29.self_attn.q_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.29.self_attn.v_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.3.input_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.3.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.3.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.3.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.3.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.3.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.3.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.3.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.3.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.30.input_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.30.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.30.mlp.gate_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.30.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.30.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.30.self_attn.k_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.30.self_attn.o_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.30.self_attn.q_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.30.self_attn.v_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.31.input_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.31.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.31.mlp.gate_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.31.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.31.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.31.self_attn.k_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.31.self_attn.o_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.31.self_attn.q_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.31.self_attn.v_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.4.input_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.4.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.4.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.4.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.4.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.4.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.4.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.4.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.4.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.5.input_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.5.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.5.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.5.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.5.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.5.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.5.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.5.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.5.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.6.input_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.6.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.6.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.6.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.6.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.6.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.6.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.6.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.6.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.7.input_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.7.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.7.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.7.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.7.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.7.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.7.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.7.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.7.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.8.input_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.8.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.8.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.8.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.8.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.8.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.8.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.8.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.8.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.9.input_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.9.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.9.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.9.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.9.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.9.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.9.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.9.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.9.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.norm.weight": "model-00003-of-00003.safetensors"
+  }
+}

special_tokens_map.json

@@ -0,0 +1,24 @@
+{
+  "bos_token": {
+    "content": "<s>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "eos_token": {
+    "content": "</s>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "pad_token": "</s>",
+  "unk_token": {
+    "content": "<unk>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  }
+}

tokenizer.model

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:dadfd56d766715c61d2ef780a525ab43b8e6da4de6865bda3d95fdef5e134055
+size 493443

tokenizer_config.json

@@ -0,0 +1,46 @@
+{
+  "add_bos_token": true,
+  "add_eos_token": false,
+  "add_prefix_space": null,
+  "added_tokens_decoder": {
+    "0": {
+      "content": "<unk>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "1": {
+      "content": "<s>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "2": {
+      "content": "</s>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    }
+  },
+  "additional_special_tokens": [],
+  "bos_token": "<s>",
+  "chat_template": "{{ '<s>' }}{% if messages[0]['role'] == 'system' %}{% set loop_messages = messages[1:] %}{% set system_message = messages[0]['content'] %}{% else %}{% set loop_messages = messages %}{% endif %}{% if system_message is defined %}{{ system_message }}{% endif %}{% for message in loop_messages %}{% set content = message['content'] %}{% if message['role'] == 'user' %}{{ '[INST] ' + content + ' [/INST]' }}{% elif message['role'] == 'assistant' %}{{ content + '</s>' }}{% endif %}{% endfor %}",
+  "clean_up_tokenization_spaces": false,
+  "eos_token": "</s>",
+  "legacy": false,
+  "model_max_length": 1000000000000000019884624838656,
+  "pad_token": "</s>",
+  "padding_side": "right",
+  "sp_model_kwargs": {},
+  "spaces_between_special_tokens": false,
+  "split_special_tokens": false,
+  "tokenizer_class": "LlamaTokenizer",
+  "unk_token": "<unk>",
+  "use_default_system_prompt": false
+}

trainer_log.jsonl

@@ -0,0 +1,68 @@
+{"current_steps": 10, "total_steps": 2013, "loss": 0.8797, "lr": 4.950495049504951e-07, "epoch": 0.014883720930232559, "percentage": 0.5, "elapsed_time": "0:05:26", "remaining_time": "18:10:40"}
+{"current_steps": 20, "total_steps": 2013, "loss": 0.774, "lr": 9.900990099009902e-07, "epoch": 0.029767441860465118, "percentage": 0.99, "elapsed_time": "0:10:46", "remaining_time": "17:53:58"}
+{"current_steps": 30, "total_steps": 2013, "loss": 0.6921, "lr": 1.4851485148514852e-06, "epoch": 0.044651162790697675, "percentage": 1.49, "elapsed_time": "0:16:08", "remaining_time": "17:46:58"}
+{"current_steps": 40, "total_steps": 2013, "loss": 0.6479, "lr": 1.9801980198019803e-06, "epoch": 0.059534883720930236, "percentage": 1.99, "elapsed_time": "0:21:30", "remaining_time": "17:40:38"}
+{"current_steps": 50, "total_steps": 2013, "loss": 0.6181, "lr": 2.4752475247524753e-06, "epoch": 0.07441860465116279, "percentage": 2.48, "elapsed_time": "0:26:50", "remaining_time": "17:33:41"}
+{"current_steps": 60, "total_steps": 2013, "loss": 0.598, "lr": 2.9702970297029703e-06, "epoch": 0.08930232558139535, "percentage": 2.98, "elapsed_time": "0:32:12", "remaining_time": "17:28:28"}
+{"current_steps": 70, "total_steps": 2013, "loss": 0.5869, "lr": 3.4653465346534653e-06, "epoch": 0.10418604651162791, "percentage": 3.48, "elapsed_time": "0:37:35", "remaining_time": "17:23:16"}
+{"current_steps": 80, "total_steps": 2013, "loss": 0.5731, "lr": 3.960396039603961e-06, "epoch": 0.11906976744186047, "percentage": 3.97, "elapsed_time": "0:42:58", "remaining_time": "17:18:16"}
+{"current_steps": 90, "total_steps": 2013, "loss": 0.5715, "lr": 4.455445544554456e-06, "epoch": 0.13395348837209303, "percentage": 4.47, "elapsed_time": "0:48:21", "remaining_time": "17:13:24"}
+{"current_steps": 100, "total_steps": 2013, "loss": 0.5589, "lr": 4.950495049504951e-06, "epoch": 0.14883720930232558, "percentage": 4.97, "elapsed_time": "0:53:44", "remaining_time": "17:07:55"}
+{"current_steps": 110, "total_steps": 2013, "loss": 0.5591, "lr": 4.999753989526703e-06, "epoch": 0.16372093023255813, "percentage": 5.46, "elapsed_time": "0:59:07", "remaining_time": "17:02:56"}
+{"current_steps": 120, "total_steps": 2013, "loss": 0.5473, "lr": 4.998903652018798e-06, "epoch": 0.1786046511627907, "percentage": 5.96, "elapsed_time": "1:04:30", "remaining_time": "16:57:36"}
+{"current_steps": 130, "total_steps": 2013, "loss": 0.5418, "lr": 4.997446179820209e-06, "epoch": 0.19348837209302325, "percentage": 6.46, "elapsed_time": "1:09:52", "remaining_time": "16:52:03"}
+{"current_steps": 140, "total_steps": 2013, "loss": 0.5392, "lr": 4.995381966403521e-06, "epoch": 0.20837209302325582, "percentage": 6.95, "elapsed_time": "1:15:14", "remaining_time": "16:46:32"}
+{"current_steps": 150, "total_steps": 2013, "loss": 0.538, "lr": 4.9927115690427536e-06, "epoch": 0.22325581395348837, "percentage": 7.45, "elapsed_time": "1:20:36", "remaining_time": "16:41:09"}
+{"current_steps": 160, "total_steps": 2013, "loss": 0.5337, "lr": 4.989435708662909e-06, "epoch": 0.23813953488372094, "percentage": 7.95, "elapsed_time": "1:25:59", "remaining_time": "16:35:53"}
+{"current_steps": 170, "total_steps": 2013, "loss": 0.5236, "lr": 4.985555269645351e-06, "epoch": 0.25302325581395346, "percentage": 8.45, "elapsed_time": "1:31:20", "remaining_time": "16:30:17"}
+{"current_steps": 180, "total_steps": 2013, "loss": 0.5245, "lr": 4.981071299589047e-06, "epoch": 0.26790697674418606, "percentage": 8.94, "elapsed_time": "1:36:42", "remaining_time": "16:24:52"}
+{"current_steps": 190, "total_steps": 2013, "loss": 0.5218, "lr": 4.975985009027748e-06, "epoch": 0.2827906976744186, "percentage": 9.44, "elapsed_time": "1:42:06", "remaining_time": "16:19:38"}
+{"current_steps": 200, "total_steps": 2013, "loss": 0.5225, "lr": 4.970297771103183e-06, "epoch": 0.29767441860465116, "percentage": 9.94, "elapsed_time": "1:47:28", "remaining_time": "16:14:17"}
+{"current_steps": 210, "total_steps": 2013, "loss": 0.5125, "lr": 4.964011121194349e-06, "epoch": 0.3125581395348837, "percentage": 10.43, "elapsed_time": "1:52:51", "remaining_time": "16:08:59"}
+{"current_steps": 220, "total_steps": 2013, "loss": 0.5119, "lr": 4.957126756503014e-06, "epoch": 0.32744186046511625, "percentage": 10.93, "elapsed_time": "1:58:11", "remaining_time": "16:03:13"}
+{"current_steps": 230, "total_steps": 2013, "loss": 0.5089, "lr": 4.949646535595514e-06, "epoch": 0.34232558139534885, "percentage": 11.43, "elapsed_time": "2:03:32", "remaining_time": "15:57:45"}
+{"current_steps": 240, "total_steps": 2013, "loss": 0.5028, "lr": 4.941572477901008e-06, "epoch": 0.3572093023255814, "percentage": 11.92, "elapsed_time": "2:08:54", "remaining_time": "15:52:20"}
+{"current_steps": 250, "total_steps": 2013, "loss": 0.5036, "lr": 4.932906763166286e-06, "epoch": 0.37209302325581395, "percentage": 12.42, "elapsed_time": "2:14:15", "remaining_time": "15:46:47"}
+{"current_steps": 260, "total_steps": 2013, "loss": 0.5051, "lr": 4.9236517308673135e-06, "epoch": 0.3869767441860465, "percentage": 12.92, "elapsed_time": "2:19:37", "remaining_time": "15:41:24"}
+{"current_steps": 270, "total_steps": 2013, "loss": 0.4996, "lr": 4.9138098795776335e-06, "epoch": 0.4018604651162791, "percentage": 13.41, "elapsed_time": "2:24:59", "remaining_time": "15:36:02"}
+{"current_steps": 280, "total_steps": 2013, "loss": 0.5003, "lr": 4.903383866293839e-06, "epoch": 0.41674418604651164, "percentage": 13.91, "elapsed_time": "2:30:19", "remaining_time": "15:30:26"}
+{"current_steps": 290, "total_steps": 2013, "loss": 0.5009, "lr": 4.89237650571826e-06, "epoch": 0.4316279069767442, "percentage": 14.41, "elapsed_time": "2:35:42", "remaining_time": "15:25:06"}
+{"current_steps": 300, "total_steps": 2013, "loss": 0.4983, "lr": 4.880790769499083e-06, "epoch": 0.44651162790697674, "percentage": 14.9, "elapsed_time": "2:41:04", "remaining_time": "15:19:46"}
+{"current_steps": 310, "total_steps": 2013, "loss": 0.497, "lr": 4.868629785428096e-06, "epoch": 0.4613953488372093, "percentage": 15.4, "elapsed_time": "2:46:28", "remaining_time": "15:14:32"}
+{"current_steps": 320, "total_steps": 2013, "loss": 0.4929, "lr": 4.855896836596282e-06, "epoch": 0.4762790697674419, "percentage": 15.9, "elapsed_time": "2:51:51", "remaining_time": "15:09:11"}
+{"current_steps": 330, "total_steps": 2013, "loss": 0.4966, "lr": 4.842595360507486e-06, "epoch": 0.49116279069767443, "percentage": 16.39, "elapsed_time": "2:57:11", "remaining_time": "15:03:41"}
+{"current_steps": 340, "total_steps": 2013, "loss": 0.4948, "lr": 4.828728948150395e-06, "epoch": 0.5060465116279069, "percentage": 16.89, "elapsed_time": "3:02:33", "remaining_time": "14:58:18"}
+{"current_steps": 350, "total_steps": 2013, "loss": 0.4918, "lr": 4.8143013430290805e-06, "epoch": 0.5209302325581395, "percentage": 17.39, "elapsed_time": "3:07:57", "remaining_time": "14:53:04"}
+{"current_steps": 360, "total_steps": 2013, "loss": 0.4899, "lr": 4.799316440152367e-06, "epoch": 0.5358139534883721, "percentage": 17.88, "elapsed_time": "3:13:20", "remaining_time": "14:47:44"}
+{"current_steps": 370, "total_steps": 2013, "loss": 0.4859, "lr": 4.783778284982303e-06, "epoch": 0.5506976744186046, "percentage": 18.38, "elapsed_time": "3:18:43", "remaining_time": "14:42:25"}
+{"current_steps": 380, "total_steps": 2013, "loss": 0.4884, "lr": 4.767691072342006e-06, "epoch": 0.5655813953488372, "percentage": 18.88, "elapsed_time": "3:24:05", "remaining_time": "14:37:05"}
+{"current_steps": 390, "total_steps": 2013, "loss": 0.4809, "lr": 4.7510591452831975e-06, "epoch": 0.5804651162790697, "percentage": 19.37, "elapsed_time": "3:29:26", "remaining_time": "14:31:37"}
+{"current_steps": 400, "total_steps": 2013, "loss": 0.4857, "lr": 4.733886993913704e-06, "epoch": 0.5953488372093023, "percentage": 19.87, "elapsed_time": "3:34:49", "remaining_time": "14:26:17"}
+{"current_steps": 410, "total_steps": 2013, "loss": 0.4872, "lr": 4.7161792541852675e-06, "epoch": 0.6102325581395349, "percentage": 20.37, "elapsed_time": "3:40:11", "remaining_time": "14:20:54"}
+{"current_steps": 420, "total_steps": 2013, "loss": 0.4829, "lr": 4.69794070664199e-06, "epoch": 0.6251162790697674, "percentage": 20.86, "elapsed_time": "3:45:34", "remaining_time": "14:15:34"}
+{"current_steps": 430, "total_steps": 2013, "loss": 0.481, "lr": 4.6791762751297236e-06, "epoch": 0.64, "percentage": 21.36, "elapsed_time": "3:50:57", "remaining_time": "14:10:13"}
+{"current_steps": 440, "total_steps": 2013, "loss": 0.4821, "lr": 4.65989102546679e-06, "epoch": 0.6548837209302325, "percentage": 21.86, "elapsed_time": "3:56:20", "remaining_time": "14:04:55"}
+{"current_steps": 450, "total_steps": 2013, "loss": 0.4749, "lr": 4.640090164076361e-06, "epoch": 0.6697674418604651, "percentage": 22.35, "elapsed_time": "4:01:43", "remaining_time": "13:59:36"}
+{"current_steps": 460, "total_steps": 2013, "loss": 0.4804, "lr": 4.61977903658089e-06, "epoch": 0.6846511627906977, "percentage": 22.85, "elapsed_time": "4:07:07", "remaining_time": "13:54:17"}
+{"current_steps": 470, "total_steps": 2013, "loss": 0.4743, "lr": 4.5989631263589546e-06, "epoch": 0.6995348837209302, "percentage": 23.35, "elapsed_time": "4:12:31", "remaining_time": "13:49:00"}
+{"current_steps": 480, "total_steps": 2013, "loss": 0.4726, "lr": 4.5776480530649155e-06, "epoch": 0.7144186046511628, "percentage": 23.85, "elapsed_time": "4:17:53", "remaining_time": "13:43:38"}
+{"current_steps": 490, "total_steps": 2013, "loss": 0.4728, "lr": 4.555839571111782e-06, "epoch": 0.7293023255813953, "percentage": 24.34, "elapsed_time": "4:23:15", "remaining_time": "13:38:14"}
+{"current_steps": 500, "total_steps": 2013, "loss": 0.4725, "lr": 4.533543568117697e-06, "epoch": 0.7441860465116279, "percentage": 24.84, "elapsed_time": "4:28:37", "remaining_time": "13:32:51"}
+{"current_steps": 510, "total_steps": 2013, "loss": 0.475, "lr": 4.5107660633164645e-06, "epoch": 0.7590697674418605, "percentage": 25.34, "elapsed_time": "4:33:59", "remaining_time": "13:27:28"}
+{"current_steps": 520, "total_steps": 2013, "loss": 0.4758, "lr": 4.487513205932537e-06, "epoch": 0.773953488372093, "percentage": 25.83, "elapsed_time": "4:39:21", "remaining_time": "13:22:06"}
+{"current_steps": 530, "total_steps": 2013, "loss": 0.4651, "lr": 4.46379127352092e-06, "epoch": 0.7888372093023256, "percentage": 26.33, "elapsed_time": "4:44:41", "remaining_time": "13:16:35"}
+{"current_steps": 540, "total_steps": 2013, "loss": 0.4717, "lr": 4.439606670272421e-06, "epoch": 0.8037209302325582, "percentage": 26.83, "elapsed_time": "4:50:03", "remaining_time": "13:11:12"}
+{"current_steps": 550, "total_steps": 2013, "loss": 0.4683, "lr": 4.414965925284719e-06, "epoch": 0.8186046511627907, "percentage": 27.32, "elapsed_time": "4:55:25", "remaining_time": "13:05:48"}
+{"current_steps": 560, "total_steps": 2013, "loss": 0.4705, "lr": 4.389875690799706e-06, "epoch": 0.8334883720930233, "percentage": 27.82, "elapsed_time": "5:00:45", "remaining_time": "13:00:21"}
+{"current_steps": 570, "total_steps": 2013, "loss": 0.4684, "lr": 4.364342740407589e-06, "epoch": 0.8483720930232558, "percentage": 28.32, "elapsed_time": "5:06:06", "remaining_time": "12:54:57"}
+{"current_steps": 580, "total_steps": 2013, "loss": 0.4629, "lr": 4.338373967218229e-06, "epoch": 0.8632558139534884, "percentage": 28.81, "elapsed_time": "5:11:29", "remaining_time": "12:49:35"}
+{"current_steps": 590, "total_steps": 2013, "loss": 0.4643, "lr": 4.3119763820002105e-06, "epoch": 0.878139534883721, "percentage": 29.31, "elapsed_time": "5:16:50", "remaining_time": "12:44:11"}
+{"current_steps": 600, "total_steps": 2013, "loss": 0.4642, "lr": 4.285157111288156e-06, "epoch": 0.8930232558139535, "percentage": 29.81, "elapsed_time": "5:22:11", "remaining_time": "12:38:44"}
+{"current_steps": 610, "total_steps": 2013, "loss": 0.4606, "lr": 4.257923395458778e-06, "epoch": 0.9079069767441861, "percentage": 30.3, "elapsed_time": "5:27:34", "remaining_time": "12:33:24"}
+{"current_steps": 620, "total_steps": 2013, "loss": 0.4584, "lr": 4.230282586776198e-06, "epoch": 0.9227906976744186, "percentage": 30.8, "elapsed_time": "5:32:55", "remaining_time": "12:28:00"}
+{"current_steps": 630, "total_steps": 2013, "loss": 0.4621, "lr": 4.202242147407065e-06, "epoch": 0.9376744186046512, "percentage": 31.3, "elapsed_time": "5:38:17", "remaining_time": "12:22:36"}
+{"current_steps": 640, "total_steps": 2013, "loss": 0.4601, "lr": 4.173809647406001e-06, "epoch": 0.9525581395348838, "percentage": 31.79, "elapsed_time": "5:43:41", "remaining_time": "12:17:20"}
+{"current_steps": 650, "total_steps": 2013, "loss": 0.456, "lr": 4.1449927626719164e-06, "epoch": 0.9674418604651163, "percentage": 32.29, "elapsed_time": "5:49:03", "remaining_time": "12:11:57"}
+{"current_steps": 660, "total_steps": 2013, "loss": 0.4548, "lr": 4.115799272875756e-06, "epoch": 0.9823255813953489, "percentage": 32.79, "elapsed_time": "5:54:26", "remaining_time": "12:06:37"}
+{"current_steps": 670, "total_steps": 2013, "loss": 0.4624, "lr": 4.086237059360228e-06, "epoch": 0.9972093023255814, "percentage": 33.28, "elapsed_time": "5:59:48", "remaining_time": "12:01:13"}
+{"current_steps": 671, "total_steps": 2013, "eval_loss": 0.05731714889407158, "epoch": 0.9986976744186047, "percentage": 33.33, "elapsed_time": "6:08:03", "remaining_time": "12:16:07"}

training_args.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:9a42c97128b144835cea798cdb330d305033d18847359f8e27a986d8249890ba
+size 7352