Upload model

README.md

@@ -14,6 +14,7 @@ model-index:
 <!-- This model card has been generated automatically according to the information the Trainer had access to. You
 should probably proofread and complete it, then remove this comment. -->
 
+[<img src="https://raw.githubusercontent.com/wandb/assets/main/wandb-github-badge-28.svg" alt="Visualize in Weights & Biases" width="200" height="32"/>](https://wandb.ai/codi_dankook/huggingface/runs/5f3sbz8h)
 # finetune_starcoder2
 
 This model is a fine-tuned version of [bigcode/starcoder2-3b](https://huggingface.co/bigcode/starcoder2-3b) on an unknown dataset.
@@ -48,7 +49,6 @@ The following hyperparameters were used during training:
 - lr_scheduler_type: cosine
 - lr_scheduler_warmup_steps: 20
 - training_steps: 10000
-- mixed_precision_training: Native AMP
 
 ### Training results
 
@@ -57,7 +57,7 @@ The following hyperparameters were used during training:
 ### Framework versions
 
 - PEFT 0.8.2
-- Transformers 4.39.3
-- Pytorch 2.2.2
+- Transformers 4.41.0.dev0
+- Pytorch 2.3.0
 - Datasets 2.18.0
-- Tokenizers 0.15.2
+- Tokenizers 0.19.1

adapter_config.json

@@ -21,9 +21,9 @@
   "target_modules": [
     "k_proj",
     "v_proj",
+    "gate_proj",
     "o_proj",
     "q_proj",
-    "gate_proj",
     "down_proj",
     "up_proj"
   ],

adapter_model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:cdd306a373798ab1a5666e822208cd0cfa345a64ab2eea10f72ac40a338b6112
-size 18218008
+oid sha256:f9dccf8efc2fbefc3fa3c535218d7f9d94d2266cf26981d8d9cd4a43d5eae62c
+size 9124880

final_checkpoint/config.json

@@ -19,11 +19,26 @@
   "num_attention_heads": 24,
   "num_hidden_layers": 30,
   "num_key_value_heads": 2,
+  "quantization_config": {
+    "_load_in_4bit": true,
+    "_load_in_8bit": false,
+    "bnb_4bit_compute_dtype": "bfloat16",
+    "bnb_4bit_quant_storage": "uint8",
+    "bnb_4bit_quant_type": "nf4",
+    "bnb_4bit_use_double_quant": false,
+    "llm_int8_enable_fp32_cpu_offload": false,
+    "llm_int8_has_fp16_weight": false,
+    "llm_int8_skip_modules": null,
+    "llm_int8_threshold": 6.0,
+    "load_in_4bit": true,
+    "load_in_8bit": false,
+    "quant_method": "bitsandbytes"
+  },
   "residual_dropout": 0.1,
   "rope_theta": 999999.4420358813,
   "sliding_window": 4096,
-  "torch_dtype": "float32",
-  "transformers_version": "4.39.3",
+  "torch_dtype": "bfloat16",
+  "transformers_version": "4.41.0.dev0",
   "use_bias": true,
   "use_cache": true,
   "vocab_size": 49152

final_checkpoint/generation_config.json

@@ -2,5 +2,5 @@
   "_from_model_config": true,
   "bos_token_id": 0,
   "eos_token_id": 0,
-  "transformers_version": "4.39.3"
+  "transformers_version": "4.41.0.dev0"
 }

final_checkpoint/model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:11a53025c306c48adf224af26dc64b4d48f7a26f6b6363c086e4c7a4424012f7
+size 1934288228

tokenizer.json

@@ -1,11 +1,6 @@
 {
   "version": "1.0",
-  "truncation": {
-    "direction": "Right",
-    "max_length": 1024,
-    "strategy": "LongestFirst",
-    "stride": 0
-  },
+  "truncation": null,
   "padding": null,
   "added_tokens": [
     {
@@ -382,6 +377,7 @@
     "end_of_word_suffix": null,
     "fuse_unk": false,
     "byte_fallback": false,
+    "ignore_merges": false,
     "vocab": {
       "<|endoftext|>": 0,
       "<fim_prefix>": 1,

training_args.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:e797fda69961c9263b7afd1004d35aa09ead71ef1aaa45c4bc0d8ca1401cc031
-size 4920
+oid sha256:57b8fe122be92cf5df03a1b23adb6c226b6b076a5b1d6d2626c633c2d92fccf6
+size 5112