Upload 17 files

README.md

@@ -0,0 +1,62 @@
+---
+base_model: microsoft/Phi-3-medium-128k-instruct
+library_name: peft
+license: other
+tags:
+- llama-factory
+- lora
+- generated_from_trainer
+model-index:
+- name: train_2024-07-24-15-00-21
+  results: []
+---
+
+<!-- This model card has been generated automatically according to the information the Trainer had access to. You
+should probably proofread and complete it, then remove this comment. -->
+
+# train_2024-07-24-15-00-21
+
+This model is a fine-tuned version of [microsoft/Phi-3-medium-128k-instruct](https://huggingface.co/microsoft/Phi-3-medium-128k-instruct) on the ALS dataset.
+
+## Model description
+
+More information needed
+
+## Intended uses & limitations
+
+More information needed
+
+## Training and evaluation data
+
+More information needed
+
+## Training procedure
+
+### Training hyperparameters
+
+The following hyperparameters were used during training:
+- learning_rate: 5e-05
+- train_batch_size: 2
+- eval_batch_size: 8
+- seed: 42
+- distributed_type: multi-GPU
+- num_devices: 2
+- gradient_accumulation_steps: 8
+- total_train_batch_size: 32
+- total_eval_batch_size: 16
+- optimizer: Adam with betas=(0.9,0.999) and epsilon=1e-08
+- lr_scheduler_type: cosine
+- lr_scheduler_warmup_steps: 50
+- num_epochs: 3.0
+
+### Training results
+
+
+
+### Framework versions
+
+- PEFT 0.11.1
+- Transformers 4.43.1
+- Pytorch 2.3.1+cu121
+- Datasets 2.19.1
+- Tokenizers 0.19.1

adapter_config.json

@@ -0,0 +1,31 @@
+{
+  "alpha_pattern": {},
+  "auto_mapping": null,
+  "base_model_name_or_path": "microsoft/Phi-3-medium-128k-instruct",
+  "bias": "none",
+  "fan_in_fan_out": false,
+  "inference_mode": true,
+  "init_lora_weights": true,
+  "layer_replication": null,
+  "layers_pattern": null,
+  "layers_to_transform": null,
+  "loftq_config": {},
+  "lora_alpha": 16,
+  "lora_dropout": 0,
+  "megatron_config": null,
+  "megatron_core": "megatron.core",
+  "modules_to_save": null,
+  "peft_type": "LORA",
+  "r": 8,
+  "rank_pattern": {},
+  "revision": null,
+  "target_modules": [
+    "o_proj",
+    "qkv_proj",
+    "down_proj",
+    "gate_up_proj"
+  ],
+  "task_type": "CAUSAL_LM",
+  "use_dora": false,
+  "use_rslora": false
+}

adapter_model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:cb2de7ec0fe1de023e38bd91807dfe90489c24cfc01b61f7a997ca30cba48b64
+size 55748640

added_tokens.json

@@ -0,0 +1,13 @@
+{
+  "<|assistant|>": 32001,
+  "<|endoftext|>": 32000,
+  "<|end|>": 32007,
+  "<|placeholder1|>": 32002,
+  "<|placeholder2|>": 32003,
+  "<|placeholder3|>": 32004,
+  "<|placeholder4|>": 32005,
+  "<|placeholder5|>": 32008,
+  "<|placeholder6|>": 32009,
+  "<|system|>": 32006,
+  "<|user|>": 32010
+}

all_results.json

@@ -0,0 +1,9 @@
+{
+    "epoch": 2.9902912621359223,
+    "num_input_tokens_seen": 61072080,
+    "total_flos": 5.065534456681464e+18,
+    "train_loss": 0.1973544524990635,
+    "train_runtime": 23852.8939,
+    "train_samples_per_second": 0.622,
+    "train_steps_per_second": 0.019
+}

llamaboard_config.yaml

@@ -0,0 +1,67 @@
+top.booster: flashattn2
+top.checkpoint_path: []
+top.finetuning_type: lora
+top.model_name: Custom
+top.quantization_bit: '4'
+top.quantization_method: bitsandbytes
+top.rope_scaling: none
+top.template: default
+top.visual_inputs: false
+train.additional_target: ''
+train.badam_mode: layer
+train.badam_switch_interval: 50
+train.badam_switch_mode: ascending
+train.badam_update_ratio: 0.05
+train.batch_size: 2
+train.compute_type: bf16
+train.create_new_adapter: false
+train.cutoff_len: 20000
+train.dataset:
+- ALS
+train.dataset_dir: data
+train.ds_offload: true
+train.ds_stage: '2'
+train.freeze_extra_modules: ''
+train.freeze_trainable_layers: 2
+train.freeze_trainable_modules: all
+train.galore_rank: 16
+train.galore_scale: 0.25
+train.galore_target: all
+train.galore_update_interval: 200
+train.gradient_accumulation_steps: 8
+train.learning_rate: 5e-5
+train.logging_steps: 10
+train.lora_alpha: 16
+train.lora_dropout: 0
+train.lora_rank: 8
+train.lora_target: ''
+train.loraplus_lr_ratio: 0
+train.lr_scheduler_type: cosine
+train.mask_history: false
+train.max_grad_norm: '1.0'
+train.max_samples: '100000'
+train.neat_packing: false
+train.neftune_alpha: 0
+train.num_train_epochs: '3.0'
+train.optim: adamw_torch
+train.packing: false
+train.ppo_score_norm: false
+train.ppo_whiten_rewards: false
+train.pref_beta: 0.1
+train.pref_ftx: 0
+train.pref_loss: sigmoid
+train.report_to: false
+train.resize_vocab: false
+train.reward_model: null
+train.save_steps: 500
+train.shift_attn: false
+train.train_on_prompt: false
+train.training_stage: Supervised Fine-Tuning
+train.use_badam: false
+train.use_dora: false
+train.use_galore: false
+train.use_llama_pro: false
+train.use_pissa: false
+train.use_rslora: false
+train.val_size: 0
+train.warmup_steps: 50

running_log.txt

@@ -0,0 +1,746 @@
+[WARNING|parser.py:272] 2024-07-24 15:04:58,287 >> We recommend enable `upcast_layernorm` in quantized training.
+
+[WARNING|parser.py:292] 2024-07-24 15:04:58,287 >> `ddp_find_unused_parameters` needs to be set as False for LoRA in DDP training.
+
+[INFO|parser.py:344] 2024-07-24 15:04:58,288 >> Process rank: 0, device: cuda:0, n_gpu: 1, distributed training: True, compute dtype: torch.bfloat16
+
+07/24/2024 15:04:58 - WARNING - llamafactory.hparams.parser - We recommend enable `upcast_layernorm` in quantized training.
+
+07/24/2024 15:04:58 - WARNING - llamafactory.hparams.parser - `ddp_find_unused_parameters` needs to be set as False for LoRA in DDP training.
+
+07/24/2024 15:04:58 - INFO - llamafactory.hparams.parser - Process rank: 1, device: cuda:1, n_gpu: 1, distributed training: True, compute dtype: torch.bfloat16
+
+[INFO|tokenization_utils_base.py:2289] 2024-07-24 15:05:00,642 >> loading file tokenizer.model from cache at /workspace/data/huggingface-cache/hub/models--microsoft--Phi-3-medium-128k-instruct/snapshots/cae1d42b5577398fd1be9f0746052562ae552886/tokenizer.model
+
+[INFO|tokenization_utils_base.py:2289] 2024-07-24 15:05:00,643 >> loading file tokenizer.json from cache at /workspace/data/huggingface-cache/hub/models--microsoft--Phi-3-medium-128k-instruct/snapshots/cae1d42b5577398fd1be9f0746052562ae552886/tokenizer.json
+
+[INFO|tokenization_utils_base.py:2289] 2024-07-24 15:05:00,643 >> loading file added_tokens.json from cache at /workspace/data/huggingface-cache/hub/models--microsoft--Phi-3-medium-128k-instruct/snapshots/cae1d42b5577398fd1be9f0746052562ae552886/added_tokens.json
+
+[INFO|tokenization_utils_base.py:2289] 2024-07-24 15:05:00,643 >> loading file special_tokens_map.json from cache at /workspace/data/huggingface-cache/hub/models--microsoft--Phi-3-medium-128k-instruct/snapshots/cae1d42b5577398fd1be9f0746052562ae552886/special_tokens_map.json
+
+[INFO|tokenization_utils_base.py:2289] 2024-07-24 15:05:00,643 >> loading file tokenizer_config.json from cache at /workspace/data/huggingface-cache/hub/models--microsoft--Phi-3-medium-128k-instruct/snapshots/cae1d42b5577398fd1be9f0746052562ae552886/tokenizer_config.json
+
+[INFO|tokenization_utils_base.py:2533] 2024-07-24 15:05:00,693 >> Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.
+
+[INFO|loader.py:52] 2024-07-24 15:05:00,694 >> Loading dataset dataset_alpaca_IT_train_and_eval_25K.json...
+
+07/24/2024 15:05:08 - INFO - llamafactory.data.loader - Loading dataset dataset_alpaca_IT_train_and_eval_25K.json...
+
+[INFO|configuration_utils.py:733] 2024-07-24 15:05:11,124 >> loading configuration file config.json from cache at /workspace/data/huggingface-cache/hub/models--microsoft--Phi-3-medium-128k-instruct/snapshots/cae1d42b5577398fd1be9f0746052562ae552886/config.json
+
+[INFO|configuration_utils.py:733] 2024-07-24 15:05:11,485 >> loading configuration file config.json from cache at /workspace/data/huggingface-cache/hub/models--microsoft--Phi-3-medium-128k-instruct/snapshots/cae1d42b5577398fd1be9f0746052562ae552886/config.json
+
+[INFO|configuration_utils.py:800] 2024-07-24 15:05:11,488 >> Model config Phi3Config {
+  "_name_or_path": "microsoft/Phi-3-medium-128k-instruct",
+  "architectures": [
+    "Phi3ForCausalLM"
+  ],
+  "attention_bias": false,
+  "attention_dropout": 0.0,
+  "auto_map": {
+    "AutoConfig": "microsoft/Phi-3-medium-128k-instruct--configuration_phi3.Phi3Config",
+    "AutoModelForCausalLM": "microsoft/Phi-3-medium-128k-instruct--modeling_phi3.Phi3ForCausalLM"
+  },
+  "bos_token_id": 1,
+  "embd_pdrop": 0.0,
+  "eos_token_id": 32000,
+  "hidden_act": "silu",
+  "hidden_size": 5120,
+  "initializer_range": 0.02,
+  "intermediate_size": 17920,
+  "max_position_embeddings": 131072,
+  "model_type": "phi3",
+  "num_attention_heads": 40,
+  "num_hidden_layers": 40,
+  "num_key_value_heads": 10,
+  "original_max_position_embeddings": 4096,
+  "pad_token_id": null,
+  "resid_pdrop": 0.0,
+  "rms_norm_eps": 1e-05,
+  "rope_scaling": {
+    "long_factor": [
+      1.0,
+      1.0,
+      1.0,
+      1.0,
+      1.0,
+      1.0,
+      1.0,
+      1.0,
+      1.0,
+      1.0,
+      1.0,
+      1.0,
+      1.0,
+      1.25,
+      1.25,
+      1.5,
+      2.0,
+      2.75,
+      5.75,
+      5.75,
+      6.5,
+      9.25,
+      11.0,
+      13.25,
+      19.25,
+      19.75,
+      19.75,
+      21.25,
+      21.5,
+      26.5,
+      30.0,
+      33.75,
+      35.25,
+      38.5,
+      42.0,
+      42.25,
+      46.0,
+      47.0,
+      50.0,
+      50.5,
+      51.0,
+      52.0,
+      52.75,
+      53.75,
+      54.75,
+      57.0,
+      57.25,
+      58.5,
+      59.25,
+      59.5,
+      62.0,
+      62.5,
+      62.75,
+      63.25,
+      63.25,
+      63.25,
+      63.75,
+      64.0,
+      64.0,
+      64.25,
+      64.5,
+      64.5,
+      65.0,
+      65.0
+    ],
+    "short_factor": [
+      1.0,
+      1.0,
+      1.0,
+      1.0,
+      1.0,
+      1.0,
+      1.01,
+      1.02,
+      1.02,
+      1.04,
+      1.04,
+      1.07,
+      1.07,
+      1.1,
+      1.3000000000000003,
+      1.3000000000000003,
+      1.5000000000000004,
+      1.5700000000000005,
+      1.9000000000000008,
+      2.3100000000000014,
+      2.759999999999992,
+      3.3899999999999784,
+      3.9399999999999666,
+      4.009999999999965,
+      4.289999999999959,
+      4.349999999999958,
+      5.349999999999937,
+      6.659999999999909,
+      7.029999999999901,
+      7.51999999999989,
+      8.00999999999988,
+      8.249999999999876,
+      8.279999999999875,
+      9.629999999999846,
+      9.89999999999984,
+      10.589999999999826,
+      11.049999999999816,
+      11.7899999999998,
+      12.189999999999792,
+      12.889999999999777,
+      13.129999999999772,
+      13.16999999999977,
+      13.20999999999977,
+      13.479999999999764,
+      13.539999999999763,
+      13.779999999999758,
+      13.929999999999755,
+      14.429999999999744,
+      14.759999999999737,
+      15.149999999999729,
+      15.419999999999723,
+      15.53999999999972,
+      15.659999999999718,
+      15.749999999999716,
+      15.759999999999716,
+      15.799999999999715,
+      16.05999999999971,
+      16.079999999999714,
+      16.11999999999972,
+      16.11999999999972,
+      16.18999999999973,
+      16.31999999999975,
+      16.539999999999786,
+      16.799999999999827
+    ],
+    "type": "su"
+  },
+  "rope_theta": 10000.0,
+  "sliding_window": 131072,
+  "tie_word_embeddings": false,
+  "torch_dtype": "bfloat16",
+  "transformers_version": "4.43.1",
+  "use_cache": true,
+  "vocab_size": 32064
+}
+
+
+[INFO|quantization.py:182] 2024-07-24 15:05:11,496 >> Quantizing model to 4 bit with bitsandbytes.
+
+[INFO|modeling_utils.py:3621] 2024-07-24 15:05:12,104 >> loading weights file model.safetensors from cache at /workspace/data/huggingface-cache/hub/models--microsoft--Phi-3-medium-128k-instruct/snapshots/cae1d42b5577398fd1be9f0746052562ae552886/model.safetensors.index.json
+
+07/24/2024 15:05:12 - INFO - llamafactory.model.model_utils.quantization - Quantizing model to 4 bit with bitsandbytes.
+
+[INFO|modeling_utils.py:1569] 2024-07-24 15:10:52,981 >> Instantiating Phi3ForCausalLM model under default dtype torch.bfloat16.
+
+[INFO|configuration_utils.py:1038] 2024-07-24 15:10:52,989 >> Generate config GenerationConfig {
+  "bos_token_id": 1,
+  "eos_token_id": 32000
+}
+
+
+[INFO|modeling_utils.py:4450] 2024-07-24 15:11:18,809 >> All model checkpoint weights were used when initializing Phi3ForCausalLM.
+
+
+[INFO|modeling_utils.py:4458] 2024-07-24 15:11:18,810 >> All the weights of Phi3ForCausalLM were initialized from the model checkpoint at microsoft/Phi-3-medium-128k-instruct.
+If your task is similar to the task the model of the checkpoint was trained on, you can already use Phi3ForCausalLM for predictions without further training.
+
+[INFO|configuration_utils.py:993] 2024-07-24 15:11:18,895 >> loading configuration file generation_config.json from cache at /workspace/data/huggingface-cache/hub/models--microsoft--Phi-3-medium-128k-instruct/snapshots/cae1d42b5577398fd1be9f0746052562ae552886/generation_config.json
+
+[INFO|configuration_utils.py:1038] 2024-07-24 15:11:18,896 >> Generate config GenerationConfig {
+  "bos_token_id": 1,
+  "eos_token_id": [
+    32000,
+    32001,
+    32007
+  ],
+  "pad_token_id": 32000
+}
+
+
+07/24/2024 15:12:39 - INFO - llamafactory.model.model_utils.checkpointing - Gradient checkpointing enabled.
+
+07/24/2024 15:12:39 - INFO - llamafactory.model.model_utils.attention - Using FlashAttention-2 for faster training and inference.
+
+07/24/2024 15:12:39 - INFO - llamafactory.model.adapter - Upcasting trainable params to float32.
+
+07/24/2024 15:12:39 - INFO - llamafactory.model.adapter - Fine-tuning method: LoRA
+
+07/24/2024 15:12:39 - INFO - llamafactory.model.model_utils.misc - Found linear modules: o_proj,gate_up_proj,qkv_proj,down_proj
+
+07/24/2024 15:12:39 - INFO - llamafactory.model.loader - trainable params: 27,852,800 || all params: 13,988,090,880 || trainable%: 0.1991
+
+[INFO|checkpointing.py:103] 2024-07-24 15:12:42,061 >> Gradient checkpointing enabled.
+
+[INFO|attention.py:82] 2024-07-24 15:12:42,061 >> Using FlashAttention-2 for faster training and inference.
+
+[INFO|adapter.py:302] 2024-07-24 15:12:42,061 >> Upcasting trainable params to float32.
+
+[INFO|adapter.py:158] 2024-07-24 15:12:42,061 >> Fine-tuning method: LoRA
+
+[INFO|misc.py:51] 2024-07-24 15:12:42,062 >> Found linear modules: o_proj,qkv_proj,down_proj,gate_up_proj
+
+[INFO|loader.py:196] 2024-07-24 15:12:42,467 >> trainable params: 27,852,800 || all params: 13,988,090,880 || trainable%: 0.1991
+
+[INFO|trainer.py:648] 2024-07-24 15:12:42,473 >> Using auto half precision backend
+
+[INFO|deepspeed.py:329] 2024-07-24 15:12:42,673 >> Detected ZeRO Offload and non-DeepSpeed optimizers: This combination should work as long as the custom optimizer has both CPU and GPU implementation (except LAMB)
+
+[INFO|trainer.py:2134] 2024-07-24 15:13:06,954 >> ***** Running training *****
+
+[INFO|trainer.py:2135] 2024-07-24 15:13:06,954 >>   Num examples = 4,944
+
+[INFO|trainer.py:2136] 2024-07-24 15:13:06,954 >>   Num Epochs = 3
+
+[INFO|trainer.py:2137] 2024-07-24 15:13:06,954 >>   Instantaneous batch size per device = 2
+
+[INFO|trainer.py:2140] 2024-07-24 15:13:06,954 >>   Total train batch size (w. parallel, distributed & accumulation) = 32
+
+[INFO|trainer.py:2141] 2024-07-24 15:13:06,954 >>   Gradient Accumulation steps = 8
+
+[INFO|trainer.py:2142] 2024-07-24 15:13:06,954 >>   Total optimization steps = 462
+
+[INFO|trainer.py:2143] 2024-07-24 15:13:06,958 >>   Number of trainable parameters = 27,852,800
+
+[INFO|callbacks.py:310] 2024-07-24 15:21:31,280 >> {'loss': 0.5099, 'learning_rate': 1.0000e-05, 'epoch': 0.06, 'throughput': 2571.26}
+
+[INFO|callbacks.py:310] 2024-07-24 15:30:13,855 >> {'loss': 0.5115, 'learning_rate': 2.0000e-05, 'epoch': 0.13, 'throughput': 2534.92}
+
+[INFO|callbacks.py:310] 2024-07-24 15:38:39,051 >> {'loss': 0.4846, 'learning_rate': 3.0000e-05, 'epoch': 0.19, 'throughput': 2535.35}
+
+[INFO|callbacks.py:310] 2024-07-24 15:47:32,541 >> {'loss': 0.4076, 'learning_rate': 4.0000e-05, 'epoch': 0.26, 'throughput': 2553.97}
+
+[INFO|callbacks.py:310] 2024-07-24 15:56:37,450 >> {'loss': 0.3073, 'learning_rate': 5.0000e-05, 'epoch': 0.32, 'throughput': 2556.06}
+
+[INFO|callbacks.py:310] 2024-07-24 16:04:50,564 >> {'loss': 0.2516, 'learning_rate': 4.9927e-05, 'epoch': 0.39, 'throughput': 2559.12}
+
+[INFO|callbacks.py:310] 2024-07-24 16:13:54,185 >> {'loss': 0.2256, 'learning_rate': 4.9710e-05, 'epoch': 0.45, 'throughput': 2549.95}
+
+[INFO|callbacks.py:310] 2024-07-24 16:22:56,812 >> {'loss': 0.2146, 'learning_rate': 4.9349e-05, 'epoch': 0.52, 'throughput': 2547.79}
+
+[INFO|callbacks.py:310] 2024-07-24 16:31:48,796 >> {'loss': 0.2018, 'learning_rate': 4.8846e-05, 'epoch': 0.58, 'throughput': 2552.59}
+
+[INFO|callbacks.py:310] 2024-07-24 16:40:22,764 >> {'loss': 0.1958, 'learning_rate': 4.8205e-05, 'epoch': 0.65, 'throughput': 2556.94}
+
+[INFO|callbacks.py:310] 2024-07-24 16:48:46,503 >> {'loss': 0.1912, 'learning_rate': 4.7429e-05, 'epoch': 0.71, 'throughput': 2557.17}
+
+[INFO|callbacks.py:310] 2024-07-24 16:57:16,752 >> {'loss': 0.1876, 'learning_rate': 4.6522e-05, 'epoch': 0.78, 'throughput': 2558.65}
+
+[INFO|callbacks.py:310] 2024-07-24 17:05:34,669 >> {'loss': 0.1802, 'learning_rate': 4.5491e-05, 'epoch': 0.84, 'throughput': 2561.14}
+
+[INFO|callbacks.py:310] 2024-07-24 17:14:13,797 >> {'loss': 0.1793, 'learning_rate': 4.4340e-05, 'epoch': 0.91, 'throughput': 2560.01}
+
+[INFO|callbacks.py:310] 2024-07-24 17:22:00,855 >> {'loss': 0.1759, 'learning_rate': 4.3077e-05, 'epoch': 0.97, 'throughput': 2565.87}
+
+[INFO|callbacks.py:310] 2024-07-24 17:30:17,877 >> {'loss': 0.1746, 'learning_rate': 4.1709e-05, 'epoch': 1.04, 'throughput': 2564.86}
+
+[INFO|callbacks.py:310] 2024-07-24 17:38:59,853 >> {'loss': 0.1699, 'learning_rate': 4.0244e-05, 'epoch': 1.10, 'throughput': 2564.04}
+
+[INFO|callbacks.py:310] 2024-07-24 17:48:09,599 >> {'loss': 0.1680, 'learning_rate': 3.8690e-05, 'epoch': 1.17, 'throughput': 2560.03}
+
+[INFO|callbacks.py:310] 2024-07-24 17:56:04,957 >> {'loss': 0.1646, 'learning_rate': 3.7057e-05, 'epoch': 1.23, 'throughput': 2563.38}
+
+[INFO|callbacks.py:310] 2024-07-24 18:04:29,792 >> {'loss': 0.1667, 'learning_rate': 3.5354e-05, 'epoch': 1.29, 'throughput': 2565.53}
+
+[INFO|callbacks.py:310] 2024-07-24 18:13:16,644 >> {'loss': 0.1664, 'learning_rate': 3.3590e-05, 'epoch': 1.36, 'throughput': 2565.84}
+
+[INFO|callbacks.py:310] 2024-07-24 18:21:27,030 >> {'loss': 0.1622, 'learning_rate': 3.1777e-05, 'epoch': 1.42, 'throughput': 2565.58}
+
+[INFO|callbacks.py:310] 2024-07-24 18:30:03,234 >> {'loss': 0.1623, 'learning_rate': 2.9924e-05, 'epoch': 1.49, 'throughput': 2565.43}
+
+[INFO|callbacks.py:310] 2024-07-24 18:38:50,374 >> {'loss': 0.1616, 'learning_rate': 2.8043e-05, 'epoch': 1.55, 'throughput': 2565.51}
+
+[INFO|callbacks.py:310] 2024-07-24 18:47:03,606 >> {'loss': 0.1590, 'learning_rate': 2.6143e-05, 'epoch': 1.62, 'throughput': 2566.31}
+
+[INFO|callbacks.py:310] 2024-07-24 18:55:49,268 >> {'loss': 0.1619, 'learning_rate': 2.4238e-05, 'epoch': 1.68, 'throughput': 2564.17}
+
+[INFO|callbacks.py:310] 2024-07-24 19:04:24,775 >> {'loss': 0.1616, 'learning_rate': 2.2336e-05, 'epoch': 1.75, 'throughput': 2565.53}
+
+[INFO|callbacks.py:310] 2024-07-24 19:12:40,325 >> {'loss': 0.1604, 'learning_rate': 2.0450e-05, 'epoch': 1.81, 'throughput': 2566.61}
+
+[INFO|callbacks.py:310] 2024-07-24 19:21:50,945 >> {'loss': 0.1563, 'learning_rate': 1.8591e-05, 'epoch': 1.88, 'throughput': 2564.70}
+
+[INFO|callbacks.py:310] 2024-07-24 19:30:42,384 >> {'loss': 0.1548, 'learning_rate': 1.6769e-05, 'epoch': 1.94, 'throughput': 2565.36}
+
+[INFO|callbacks.py:310] 2024-07-24 19:39:26,495 >> {'loss': 0.1555, 'learning_rate': 1.4994e-05, 'epoch': 2.01, 'throughput': 2565.22}
+
+[INFO|callbacks.py:310] 2024-07-24 19:48:16,049 >> {'loss': 0.1526, 'learning_rate': 1.3278e-05, 'epoch': 2.07, 'throughput': 2564.82}
+
+[INFO|callbacks.py:310] 2024-07-24 19:56:55,676 >> {'loss': 0.1526, 'learning_rate': 1.1630e-05, 'epoch': 2.14, 'throughput': 2563.60}
+
+[INFO|callbacks.py:310] 2024-07-24 20:05:48,055 >> {'loss': 0.1516, 'learning_rate': 1.0060e-05, 'epoch': 2.20, 'throughput': 2564.75}
+
+[INFO|callbacks.py:310] 2024-07-24 20:14:05,975 >> {'loss': 0.1524, 'learning_rate': 8.5762e-06, 'epoch': 2.27, 'throughput': 2565.56}
+
+[INFO|callbacks.py:310] 2024-07-24 20:23:04,597 >> {'loss': 0.1502, 'learning_rate': 7.1880e-06, 'epoch': 2.33, 'throughput': 2564.08}
+
+[INFO|callbacks.py:310] 2024-07-24 20:31:47,755 >> {'loss': 0.1506, 'learning_rate': 5.9035e-06, 'epoch': 2.39, 'throughput': 2563.27}
+
+[INFO|callbacks.py:310] 2024-07-24 20:40:43,735 >> {'loss': 0.1479, 'learning_rate': 4.7298e-06, 'epoch': 2.46, 'throughput': 2560.31}
+
+[INFO|callbacks.py:310] 2024-07-24 20:49:04,924 >> {'loss': 0.1501, 'learning_rate': 3.6740e-06, 'epoch': 2.52, 'throughput': 2560.79}
+
+[INFO|callbacks.py:310] 2024-07-24 20:57:37,960 >> {'loss': 0.1504, 'learning_rate': 2.7422e-06, 'epoch': 2.59, 'throughput': 2561.70}
+
+[INFO|callbacks.py:310] 2024-07-24 21:05:54,158 >> {'loss': 0.1524, 'learning_rate': 1.9397e-06, 'epoch': 2.65, 'throughput': 2561.78}
+
+[INFO|callbacks.py:310] 2024-07-24 21:14:51,826 >> {'loss': 0.1494, 'learning_rate': 1.2712e-06, 'epoch': 2.72, 'throughput': 2559.88}
+
+[INFO|callbacks.py:310] 2024-07-24 21:23:35,877 >> {'loss': 0.1515, 'learning_rate': 7.4056e-07, 'epoch': 2.78, 'throughput': 2559.75}
+
+[INFO|callbacks.py:310] 2024-07-24 21:32:00,699 >> {'loss': 0.1504, 'learning_rate': 3.5095e-07, 'epoch': 2.85, 'throughput': 2559.97}
+
+[INFO|callbacks.py:310] 2024-07-24 21:40:43,444 >> {'loss': 0.1498, 'learning_rate': 1.0459e-07, 'epoch': 2.91, 'throughput': 2559.45}
+
+[INFO|callbacks.py:310] 2024-07-24 21:48:52,834 >> {'loss': 0.1469, 'learning_rate': 2.9071e-09, 'epoch': 2.98, 'throughput': 2559.91}
+
+[INFO|trainer.py:3503] 2024-07-24 21:50:38,895 >> Saving model checkpoint to saves/Custom/lora/train_2024-07-24-15-00-21/checkpoint-462
+
+[INFO|configuration_utils.py:733] 2024-07-24 21:50:39,083 >> loading configuration file config.json from cache at /workspace/data/huggingface-cache/hub/models--microsoft--Phi-3-medium-128k-instruct/snapshots/cae1d42b5577398fd1be9f0746052562ae552886/config.json
+
+[INFO|configuration_utils.py:800] 2024-07-24 21:50:39,084 >> Model config Phi3Config {
+  "_name_or_path": "Phi-3-medium-128k-instruct",
+  "architectures": [
+    "Phi3ForCausalLM"
+  ],
+  "attention_bias": false,
+  "attention_dropout": 0.0,
+  "auto_map": {
+    "AutoConfig": "microsoft/Phi-3-medium-128k-instruct--configuration_phi3.Phi3Config",
+    "AutoModelForCausalLM": "microsoft/Phi-3-medium-128k-instruct--modeling_phi3.Phi3ForCausalLM"
+  },
+  "bos_token_id": 1,
+  "embd_pdrop": 0.0,
+  "eos_token_id": 32000,
+  "hidden_act": "silu",
+  "hidden_size": 5120,
+  "initializer_range": 0.02,
+  "intermediate_size": 17920,
+  "max_position_embeddings": 131072,
+  "model_type": "phi3",
+  "num_attention_heads": 40,
+  "num_hidden_layers": 40,
+  "num_key_value_heads": 10,
+  "original_max_position_embeddings": 4096,
+  "pad_token_id": null,
+  "resid_pdrop": 0.0,
+  "rms_norm_eps": 1e-05,
+  "rope_scaling": {
+    "long_factor": [
+      1.0,
+      1.0,
+      1.0,
+      1.0,
+      1.0,
+      1.0,
+      1.0,
+      1.0,
+      1.0,
+      1.0,
+      1.0,
+      1.0,
+      1.0,
+      1.25,
+      1.25,
+      1.5,
+      2.0,
+      2.75,
+      5.75,
+      5.75,
+      6.5,
+      9.25,
+      11.0,
+      13.25,
+      19.25,
+      19.75,
+      19.75,
+      21.25,
+      21.5,
+      26.5,
+      30.0,
+      33.75,
+      35.25,
+      38.5,
+      42.0,
+      42.25,
+      46.0,
+      47.0,
+      50.0,
+      50.5,
+      51.0,
+      52.0,
+      52.75,
+      53.75,
+      54.75,
+      57.0,
+      57.25,
+      58.5,
+      59.25,
+      59.5,
+      62.0,
+      62.5,
+      62.75,
+      63.25,
+      63.25,
+      63.25,
+      63.75,
+      64.0,
+      64.0,
+      64.25,
+      64.5,
+      64.5,
+      65.0,
+      65.0
+    ],
+    "short_factor": [
+      1.0,
+      1.0,
+      1.0,
+      1.0,
+      1.0,
+      1.0,
+      1.01,
+      1.02,
+      1.02,
+      1.04,
+      1.04,
+      1.07,
+      1.07,
+      1.1,
+      1.3000000000000003,
+      1.3000000000000003,
+      1.5000000000000004,
+      1.5700000000000005,
+      1.9000000000000008,
+      2.3100000000000014,
+      2.759999999999992,
+      3.3899999999999784,
+      3.9399999999999666,
+      4.009999999999965,
+      4.289999999999959,
+      4.349999999999958,
+      5.349999999999937,
+      6.659999999999909,
+      7.029999999999901,
+      7.51999999999989,
+      8.00999999999988,
+      8.249999999999876,
+      8.279999999999875,
+      9.629999999999846,
+      9.89999999999984,
+      10.589999999999826,
+      11.049999999999816,
+      11.7899999999998,
+      12.189999999999792,
+      12.889999999999777,
+      13.129999999999772,
+      13.16999999999977,
+      13.20999999999977,
+      13.479999999999764,
+      13.539999999999763,
+      13.779999999999758,
+      13.929999999999755,
+      14.429999999999744,
+      14.759999999999737,
+      15.149999999999729,
+      15.419999999999723,
+      15.53999999999972,
+      15.659999999999718,
+      15.749999999999716,
+      15.759999999999716,
+      15.799999999999715,
+      16.05999999999971,
+      16.079999999999714,
+      16.11999999999972,
+      16.11999999999972,
+      16.18999999999973,
+      16.31999999999975,
+      16.539999999999786,
+      16.799999999999827
+    ],
+    "type": "su"
+  },
+  "rope_theta": 10000.0,
+  "sliding_window": 131072,
+  "tie_word_embeddings": false,
+  "torch_dtype": "bfloat16",
+  "transformers_version": "4.43.1",
+  "use_cache": true,
+  "vocab_size": 32064
+}
+
+
+[INFO|tokenization_utils_base.py:2702] 2024-07-24 21:50:39,135 >> tokenizer config file saved in saves/Custom/lora/train_2024-07-24-15-00-21/checkpoint-462/tokenizer_config.json
+
+[INFO|tokenization_utils_base.py:2711] 2024-07-24 21:50:39,135 >> Special tokens file saved in saves/Custom/lora/train_2024-07-24-15-00-21/checkpoint-462/special_tokens_map.json
+
+[INFO|trainer.py:2394] 2024-07-24 21:50:39,852 >> 
+
+Training completed. Do not forget to share your model on huggingface.co/models =)
+
+
+
+[INFO|trainer.py:3503] 2024-07-24 21:50:42,012 >> Saving model checkpoint to saves/Custom/lora/train_2024-07-24-15-00-21
+
+[INFO|configuration_utils.py:733] 2024-07-24 21:50:42,204 >> loading configuration file config.json from cache at /workspace/data/huggingface-cache/hub/models--microsoft--Phi-3-medium-128k-instruct/snapshots/cae1d42b5577398fd1be9f0746052562ae552886/config.json
+
+[INFO|configuration_utils.py:800] 2024-07-24 21:50:42,205 >> Model config Phi3Config {
+  "_name_or_path": "Phi-3-medium-128k-instruct",
+  "architectures": [
+    "Phi3ForCausalLM"
+  ],
+  "attention_bias": false,
+  "attention_dropout": 0.0,
+  "auto_map": {
+    "AutoConfig": "microsoft/Phi-3-medium-128k-instruct--configuration_phi3.Phi3Config",
+    "AutoModelForCausalLM": "microsoft/Phi-3-medium-128k-instruct--modeling_phi3.Phi3ForCausalLM"
+  },
+  "bos_token_id": 1,
+  "embd_pdrop": 0.0,
+  "eos_token_id": 32000,
+  "hidden_act": "silu",
+  "hidden_size": 5120,
+  "initializer_range": 0.02,
+  "intermediate_size": 17920,
+  "max_position_embeddings": 131072,
+  "model_type": "phi3",
+  "num_attention_heads": 40,
+  "num_hidden_layers": 40,
+  "num_key_value_heads": 10,
+  "original_max_position_embeddings": 4096,
+  "pad_token_id": null,
+  "resid_pdrop": 0.0,
+  "rms_norm_eps": 1e-05,
+  "rope_scaling": {
+    "long_factor": [
+      1.0,
+      1.0,
+      1.0,
+      1.0,
+      1.0,
+      1.0,
+      1.0,
+      1.0,
+      1.0,
+      1.0,
+      1.0,
+      1.0,
+      1.0,
+      1.25,
+      1.25,
+      1.5,
+      2.0,
+      2.75,
+      5.75,
+      5.75,
+      6.5,
+      9.25,
+      11.0,
+      13.25,
+      19.25,
+      19.75,
+      19.75,
+      21.25,
+      21.5,
+      26.5,
+      30.0,
+      33.75,
+      35.25,
+      38.5,
+      42.0,
+      42.25,
+      46.0,
+      47.0,
+      50.0,
+      50.5,
+      51.0,
+      52.0,
+      52.75,
+      53.75,
+      54.75,
+      57.0,
+      57.25,
+      58.5,
+      59.25,
+      59.5,
+      62.0,
+      62.5,
+      62.75,
+      63.25,
+      63.25,
+      63.25,
+      63.75,
+      64.0,
+      64.0,
+      64.25,
+      64.5,
+      64.5,
+      65.0,
+      65.0
+    ],
+    "short_factor": [
+      1.0,
+      1.0,
+      1.0,
+      1.0,
+      1.0,
+      1.0,
+      1.01,
+      1.02,
+      1.02,
+      1.04,
+      1.04,
+      1.07,
+      1.07,
+      1.1,
+      1.3000000000000003,
+      1.3000000000000003,
+      1.5000000000000004,
+      1.5700000000000005,
+      1.9000000000000008,
+      2.3100000000000014,
+      2.759999999999992,
+      3.3899999999999784,
+      3.9399999999999666,
+      4.009999999999965,
+      4.289999999999959,
+      4.349999999999958,
+      5.349999999999937,
+      6.659999999999909,
+      7.029999999999901,
+      7.51999999999989,
+      8.00999999999988,
+      8.249999999999876,
+      8.279999999999875,
+      9.629999999999846,
+      9.89999999999984,
+      10.589999999999826,
+      11.049999999999816,
+      11.7899999999998,
+      12.189999999999792,
+      12.889999999999777,
+      13.129999999999772,
+      13.16999999999977,
+      13.20999999999977,
+      13.479999999999764,
+      13.539999999999763,
+      13.779999999999758,
+      13.929999999999755,
+      14.429999999999744,
+      14.759999999999737,
+      15.149999999999729,
+      15.419999999999723,
+      15.53999999999972,
+      15.659999999999718,
+      15.749999999999716,
+      15.759999999999716,
+      15.799999999999715,
+      16.05999999999971,
+      16.079999999999714,
+      16.11999999999972,
+      16.11999999999972,
+      16.18999999999973,
+      16.31999999999975,
+      16.539999999999786,
+      16.799999999999827
+    ],
+    "type": "su"
+  },
+  "rope_theta": 10000.0,
+  "sliding_window": 131072,
+  "tie_word_embeddings": false,
+  "torch_dtype": "bfloat16",
+  "transformers_version": "4.43.1",
+  "use_cache": true,
+  "vocab_size": 32064
+}
+
+
+[INFO|tokenization_utils_base.py:2702] 2024-07-24 21:50:42,263 >> tokenizer config file saved in saves/Custom/lora/train_2024-07-24-15-00-21/tokenizer_config.json
+
+[INFO|tokenization_utils_base.py:2711] 2024-07-24 21:50:42,264 >> Special tokens file saved in saves/Custom/lora/train_2024-07-24-15-00-21/special_tokens_map.json
+
+[WARNING|ploting.py:89] 2024-07-24 21:50:42,678 >> No metric eval_loss to plot.
+
+[WARNING|ploting.py:89] 2024-07-24 21:50:42,678 >> No metric eval_accuracy to plot.
+
+[INFO|modelcard.py:449] 2024-07-24 21:50:42,679 >> Dropping the following result as it does not have all the necessary fields:
+{'task': {'name': 'Causal Language Modeling', 'type': 'text-generation'}}
+

special_tokens_map.json

@@ -0,0 +1,30 @@
+{
+  "bos_token": {
+    "content": "<s>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "eos_token": {
+    "content": "<|endoftext|>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "pad_token": {
+    "content": "<|endoftext|>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "unk_token": {
+    "content": "<unk>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  }
+}

tokenizer.model

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:9e556afd44213b6bd1be2b850ebbbd98f5481437a8021afaf58ee7fb1818d347
+size 499723

tokenizer_config.json

@@ -0,0 +1,132 @@
+{
+  "add_bos_token": false,
+  "add_eos_token": false,
+  "add_prefix_space": null,
+  "added_tokens_decoder": {
+    "0": {
+      "content": "<unk>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "1": {
+      "content": "<s>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "2": {
+      "content": "</s>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": true,
+      "single_word": false,
+      "special": false
+    },
+    "32000": {
+      "content": "<|endoftext|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "32001": {
+      "content": "<|assistant|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": true,
+      "single_word": false,
+      "special": true
+    },
+    "32002": {
+      "content": "<|placeholder1|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": true,
+      "single_word": false,
+      "special": true
+    },
+    "32003": {
+      "content": "<|placeholder2|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": true,
+      "single_word": false,
+      "special": true
+    },
+    "32004": {
+      "content": "<|placeholder3|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": true,
+      "single_word": false,
+      "special": true
+    },
+    "32005": {
+      "content": "<|placeholder4|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": true,
+      "single_word": false,
+      "special": true
+    },
+    "32006": {
+      "content": "<|system|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": true,
+      "single_word": false,
+      "special": true
+    },
+    "32007": {
+      "content": "<|end|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": true,
+      "single_word": false,
+      "special": true
+    },
+    "32008": {
+      "content": "<|placeholder5|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": true,
+      "single_word": false,
+      "special": true
+    },
+    "32009": {
+      "content": "<|placeholder6|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": true,
+      "single_word": false,
+      "special": true
+    },
+    "32010": {
+      "content": "<|user|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": true,
+      "single_word": false,
+      "special": true
+    }
+  },
+  "bos_token": "<s>",
+  "chat_template": "{% if messages[0]['role'] == 'system' %}{% set system_message = messages[0]['content'] %}{% endif %}{% if system_message is defined %}{{ system_message + '\n' }}{% endif %}{% for message in messages %}{% set content = message['content'] %}{% if message['role'] == 'user' %}{{ 'Human: ' + content + '\nAssistant:' }}{% elif message['role'] == 'assistant' %}{{ content + '<|endoftext|>' + '\n' }}{% endif %}{% endfor %}",
+  "clean_up_tokenization_spaces": false,
+  "eos_token": "<|endoftext|>",
+  "legacy": false,
+  "model_max_length": 131072,
+  "pad_token": "<|endoftext|>",
+  "padding_side": "right",
+  "sp_model_kwargs": {},
+  "split_special_tokens": false,
+  "tokenizer_class": "LlamaTokenizer",
+  "unk_token": "<unk>",
+  "use_default_system_prompt": false
+}

train_results.json

@@ -0,0 +1,9 @@
+{
+    "epoch": 2.9902912621359223,
+    "num_input_tokens_seen": 61072080,
+    "total_flos": 5.065534456681464e+18,
+    "train_loss": 0.1973544524990635,
+    "train_runtime": 23852.8939,
+    "train_samples_per_second": 0.622,
+    "train_steps_per_second": 0.019
+}

trainer_log.jsonl

@@ -0,0 +1,47 @@
+{"current_steps": 10, "total_steps": 462, "loss": 0.5099, "learning_rate": 1e-05, "epoch": 0.06472491909385113, "percentage": 2.16, "elapsed_time": "0:08:24", "remaining_time": "6:19:55", "throughput": "2571.26", "total_tokens": 1296736}
+{"current_steps": 20, "total_steps": 462, "loss": 0.5115, "learning_rate": 2e-05, "epoch": 0.12944983818770225, "percentage": 4.33, "elapsed_time": "0:17:06", "remaining_time": "6:18:14", "throughput": "2534.92", "total_tokens": 2603088}
+{"current_steps": 30, "total_steps": 462, "loss": 0.4846, "learning_rate": 3e-05, "epoch": 0.1941747572815534, "percentage": 6.49, "elapsed_time": "0:25:32", "remaining_time": "6:07:42", "throughput": "2535.35", "total_tokens": 3884384}
+{"current_steps": 40, "total_steps": 462, "loss": 0.4076, "learning_rate": 4e-05, "epoch": 0.2588996763754045, "percentage": 8.66, "elapsed_time": "0:34:25", "remaining_time": "6:03:11", "throughput": "2553.97", "total_tokens": 5275424}
+{"current_steps": 50, "total_steps": 462, "loss": 0.3073, "learning_rate": 5e-05, "epoch": 0.32362459546925565, "percentage": 10.82, "elapsed_time": "0:43:30", "remaining_time": "5:58:30", "throughput": "2556.06", "total_tokens": 6672576}
+{"current_steps": 60, "total_steps": 462, "loss": 0.2516, "learning_rate": 4.992735514089577e-05, "epoch": 0.3883495145631068, "percentage": 12.99, "elapsed_time": "0:51:43", "remaining_time": "5:46:34", "throughput": "2559.12", "total_tokens": 7942496}
+{"current_steps": 70, "total_steps": 462, "loss": 0.2256, "learning_rate": 4.970984274562741e-05, "epoch": 0.45307443365695793, "percentage": 15.15, "elapsed_time": "1:00:47", "remaining_time": "5:40:24", "throughput": "2549.95", "total_tokens": 9300240}
+{"current_steps": 80, "total_steps": 462, "loss": 0.2146, "learning_rate": 4.934872690677953e-05, "epoch": 0.517799352750809, "percentage": 17.32, "elapsed_time": "1:09:49", "remaining_time": "5:33:26", "throughput": "2547.79", "total_tokens": 10674880}
+{"current_steps": 90, "total_steps": 462, "loss": 0.2018, "learning_rate": 4.884610628109082e-05, "epoch": 0.5825242718446602, "percentage": 19.48, "elapsed_time": "1:18:41", "remaining_time": "5:25:16", "throughput": "2552.59", "total_tokens": 12052928}
+{"current_steps": 100, "total_steps": 462, "loss": 0.1958, "learning_rate": 4.820490189292415e-05, "epoch": 0.6472491909385113, "percentage": 21.65, "elapsed_time": "1:27:15", "remaining_time": "5:15:53", "throughput": "2556.94", "total_tokens": 13387616}
+{"current_steps": 110, "total_steps": 462, "loss": 0.1912, "learning_rate": 4.742884015847436e-05, "epoch": 0.7119741100323624, "percentage": 23.81, "elapsed_time": "1:35:39", "remaining_time": "5:06:06", "throughput": "2557.17", "total_tokens": 14676976}
+{"current_steps": 120, "total_steps": 462, "loss": 0.1876, "learning_rate": 4.652243122936986e-05, "epoch": 0.7766990291262136, "percentage": 25.97, "elapsed_time": "1:44:09", "remaining_time": "4:56:51", "throughput": "2558.65", "total_tokens": 15991008}
+{"current_steps": 130, "total_steps": 462, "loss": 0.1802, "learning_rate": 4.5490942781526316e-05, "epoch": 0.8414239482200647, "percentage": 28.14, "elapsed_time": "1:52:27", "remaining_time": "4:47:12", "throughput": "2561.14", "total_tokens": 17281840}
+{"current_steps": 140, "total_steps": 462, "loss": 0.1793, "learning_rate": 4.434036940158062e-05, "epoch": 0.9061488673139159, "percentage": 30.3, "elapsed_time": "2:01:06", "remaining_time": "4:38:33", "throughput": "2560.01", "total_tokens": 18603200}
+{"current_steps": 150, "total_steps": 462, "loss": 0.1759, "learning_rate": 4.307739774881878e-05, "epoch": 0.970873786407767, "percentage": 32.47, "elapsed_time": "2:08:53", "remaining_time": "4:28:06", "throughput": "2565.87", "total_tokens": 19844160}
+{"current_steps": 160, "total_steps": 462, "loss": 0.1746, "learning_rate": 4.170936769506222e-05, "epoch": 1.035598705501618, "percentage": 34.63, "elapsed_time": "2:17:10", "remaining_time": "4:18:55", "throughput": "2564.86", "total_tokens": 21111168}
+{"current_steps": 170, "total_steps": 462, "loss": 0.1699, "learning_rate": 4.024422966835136e-05, "epoch": 1.1003236245954693, "percentage": 36.8, "elapsed_time": "2:25:52", "remaining_time": "4:10:34", "throughput": "2564.04", "total_tokens": 22442800}
+{"current_steps": 180, "total_steps": 462, "loss": 0.168, "learning_rate": 3.86904984483277e-05, "epoch": 1.1650485436893203, "percentage": 38.96, "elapsed_time": "2:35:02", "remaining_time": "4:02:54", "throughput": "2560.03", "total_tokens": 23815072}
+{"current_steps": 190, "total_steps": 462, "loss": 0.1646, "learning_rate": 3.7057203681836406e-05, "epoch": 1.2297734627831716, "percentage": 41.13, "elapsed_time": "2:42:57", "remaining_time": "3:53:17", "throughput": "2563.38", "total_tokens": 25064672}
+{"current_steps": 200, "total_steps": 462, "loss": 0.1667, "learning_rate": 3.535383740633246e-05, "epoch": 1.2944983818770226, "percentage": 43.29, "elapsed_time": "2:51:22", "remaining_time": "3:44:30", "throughput": "2565.53", "total_tokens": 26380864}
+{"current_steps": 210, "total_steps": 462, "loss": 0.1664, "learning_rate": 3.3590298886062833e-05, "epoch": 1.3592233009708738, "percentage": 45.45, "elapsed_time": "3:00:09", "remaining_time": "3:36:11", "throughput": "2565.84", "total_tokens": 27735952}
+{"current_steps": 220, "total_steps": 462, "loss": 0.1622, "learning_rate": 3.177683708161389e-05, "epoch": 1.4239482200647249, "percentage": 47.62, "elapsed_time": "3:08:20", "remaining_time": "3:27:10", "throughput": "2565.58", "total_tokens": 28991248}
+{"current_steps": 230, "total_steps": 462, "loss": 0.1623, "learning_rate": 2.9923991087167658e-05, "epoch": 1.4886731391585761, "percentage": 49.78, "elapsed_time": "3:16:56", "remaining_time": "3:18:39", "throughput": "2565.43", "total_tokens": 30313776}
+{"current_steps": 240, "total_steps": 462, "loss": 0.1616, "learning_rate": 2.804252888162079e-05, "epoch": 1.5533980582524272, "percentage": 51.95, "elapsed_time": "3:25:43", "remaining_time": "3:10:17", "throughput": "2565.51", "total_tokens": 31667088}
+{"current_steps": 250, "total_steps": 462, "loss": 0.159, "learning_rate": 2.6143384749519866e-05, "epoch": 1.6181229773462782, "percentage": 54.11, "elapsed_time": "3:33:56", "remaining_time": "3:01:25", "throughput": "2566.31", "total_tokens": 32942832}
+{"current_steps": 260, "total_steps": 462, "loss": 0.1619, "learning_rate": 2.423759573549647e-05, "epoch": 1.6828478964401294, "percentage": 56.28, "elapsed_time": "3:42:42", "remaining_time": "2:53:01", "throughput": "2564.17", "total_tokens": 34263264}
+{"current_steps": 270, "total_steps": 462, "loss": 0.1616, "learning_rate": 2.23362375015031e-05, "epoch": 1.7475728155339807, "percentage": 58.44, "elapsed_time": "3:51:17", "remaining_time": "2:44:28", "throughput": "2565.53", "total_tokens": 35603952}
+{"current_steps": 280, "total_steps": 462, "loss": 0.1604, "learning_rate": 2.0450359959620967e-05, "epoch": 1.8122977346278317, "percentage": 60.61, "elapsed_time": "3:59:33", "remaining_time": "2:35:42", "throughput": "2566.61", "total_tokens": 36890768}
+{"current_steps": 290, "total_steps": 462, "loss": 0.1563, "learning_rate": 1.8590923054515503e-05, "epoch": 1.8770226537216828, "percentage": 62.77, "elapsed_time": "4:08:43", "remaining_time": "2:27:31", "throughput": "2564.70", "total_tokens": 38275536}
+{"current_steps": 300, "total_steps": 462, "loss": 0.1548, "learning_rate": 1.676873306874547e-05, "epoch": 1.941747572815534, "percentage": 64.94, "elapsed_time": "4:17:35", "remaining_time": "2:19:05", "throughput": "2565.36", "total_tokens": 39648800}
+{"current_steps": 310, "total_steps": 462, "loss": 0.1555, "learning_rate": 1.4994379821093049e-05, "epoch": 2.0064724919093853, "percentage": 67.1, "elapsed_time": "4:26:19", "remaining_time": "2:10:35", "throughput": "2565.22", "total_tokens": 40991008}
+{"current_steps": 320, "total_steps": 462, "loss": 0.1526, "learning_rate": 1.3278175122892416e-05, "epoch": 2.071197411003236, "percentage": 69.26, "elapsed_time": "4:35:09", "remaining_time": "2:02:05", "throughput": "2564.82", "total_tokens": 42342880}
+{"current_steps": 330, "total_steps": 462, "loss": 0.1526, "learning_rate": 1.1630092850023147e-05, "epoch": 2.1359223300970873, "percentage": 71.43, "elapsed_time": "4:43:48", "remaining_time": "1:53:31", "throughput": "2563.60", "total_tokens": 43654784}
+{"current_steps": 340, "total_steps": 462, "loss": 0.1516, "learning_rate": 1.005971097884561e-05, "epoch": 2.2006472491909386, "percentage": 73.59, "elapsed_time": "4:52:41", "remaining_time": "1:45:01", "throughput": "2564.75", "total_tokens": 45039744}
+{"current_steps": 350, "total_steps": 462, "loss": 0.1524, "learning_rate": 8.576155922941548e-06, "epoch": 2.26537216828479, "percentage": 75.76, "elapsed_time": "5:00:59", "remaining_time": "1:36:18", "throughput": "2565.56", "total_tokens": 46331568}
+{"current_steps": 360, "total_steps": 462, "loss": 0.1502, "learning_rate": 7.1880494941517026e-06, "epoch": 2.3300970873786406, "percentage": 77.92, "elapsed_time": "5:09:57", "remaining_time": "1:27:49", "throughput": "2564.08", "total_tokens": 47685808}
+{"current_steps": 370, "total_steps": 462, "loss": 0.1506, "learning_rate": 5.903458796151381e-06, "epoch": 2.394822006472492, "percentage": 80.09, "elapsed_time": "5:18:40", "remaining_time": "1:19:14", "throughput": "2563.27", "total_tokens": 49011728}
+{"current_steps": 380, "total_steps": 462, "loss": 0.1479, "learning_rate": 4.729849341761602e-06, "epoch": 2.459546925566343, "percentage": 82.25, "elapsed_time": "5:27:36", "remaining_time": "1:10:41", "throughput": "2560.31", "total_tokens": 50327504}
+{"current_steps": 390, "total_steps": 462, "loss": 0.1501, "learning_rate": 3.674041666458963e-06, "epoch": 2.524271844660194, "percentage": 84.42, "elapsed_time": "5:35:57", "remaining_time": "1:02:01", "throughput": "2560.79", "total_tokens": 51620256}
+{"current_steps": 400, "total_steps": 462, "loss": 0.1504, "learning_rate": 2.7421716902285623e-06, "epoch": 2.588996763754045, "percentage": 86.58, "elapsed_time": "5:44:30", "remaining_time": "0:53:24", "throughput": "2561.70", "total_tokens": 52952880}
+{"current_steps": 410, "total_steps": 462, "loss": 0.1524, "learning_rate": 1.939655058120521e-06, "epoch": 2.6537216828478964, "percentage": 88.74, "elapsed_time": "5:52:47", "remaining_time": "0:44:44", "throughput": "2561.78", "total_tokens": 54225632}
+{"current_steps": 420, "total_steps": 462, "loss": 0.1494, "learning_rate": 1.271155666748311e-06, "epoch": 2.7184466019417477, "percentage": 90.91, "elapsed_time": "6:01:44", "remaining_time": "0:36:10", "throughput": "2559.88", "total_tokens": 55561888}
+{"current_steps": 430, "total_steps": 462, "loss": 0.1515, "learning_rate": 7.405585596397313e-07, "epoch": 2.783171521035599, "percentage": 93.07, "elapsed_time": "6:10:28", "remaining_time": "0:27:34", "throughput": "2559.75", "total_tokens": 56900512}
+{"current_steps": 440, "total_steps": 462, "loss": 0.1504, "learning_rate": 3.5094734896174987e-07, "epoch": 2.8478964401294498, "percentage": 95.24, "elapsed_time": "6:18:53", "remaining_time": "0:18:56", "throughput": "2559.97", "total_tokens": 58197632}
+{"current_steps": 450, "total_steps": 462, "loss": 0.1498, "learning_rate": 1.0458629483476867e-07, "epoch": 2.912621359223301, "percentage": 97.4, "elapsed_time": "6:27:36", "remaining_time": "0:10:20", "throughput": "2559.45", "total_tokens": 59523808}
+{"current_steps": 460, "total_steps": 462, "loss": 0.1469, "learning_rate": 2.9071463840540935e-09, "epoch": 2.9773462783171523, "percentage": 99.57, "elapsed_time": "6:35:45", "remaining_time": "0:01:43", "throughput": "2559.91", "total_tokens": 60787344}
+{"current_steps": 462, "total_steps": 462, "epoch": 2.9902912621359223, "percentage": 100.0, "elapsed_time": "6:37:32", "remaining_time": "0:00:00", "throughput": "2560.36", "total_tokens": 61072080}

trainer_state.json

@@ -0,0 +1,411 @@
+{
+  "best_metric": null,
+  "best_model_checkpoint": null,
+  "epoch": 2.9902912621359223,
+  "eval_steps": 500,
+  "global_step": 462,
+  "is_hyper_param_search": false,
+  "is_local_process_zero": true,
+  "is_world_process_zero": true,
+  "log_history": [
+    {
+      "epoch": 0.06472491909385113,
+      "grad_norm": 0.09439529478549957,
+      "learning_rate": 1e-05,
+      "loss": 0.5099,
+      "num_input_tokens_seen": 1296736,
+      "step": 10
+    },
+    {
+      "epoch": 0.12944983818770225,
+      "grad_norm": 0.11287413537502289,
+      "learning_rate": 2e-05,
+      "loss": 0.5115,
+      "num_input_tokens_seen": 2603088,
+      "step": 20
+    },
+    {
+      "epoch": 0.1941747572815534,
+      "grad_norm": 0.13694792985916138,
+      "learning_rate": 3e-05,
+      "loss": 0.4846,
+      "num_input_tokens_seen": 3884384,
+      "step": 30
+    },
+    {
+      "epoch": 0.2588996763754045,
+      "grad_norm": 0.14105218648910522,
+      "learning_rate": 4e-05,
+      "loss": 0.4076,
+      "num_input_tokens_seen": 5275424,
+      "step": 40
+    },
+    {
+      "epoch": 0.32362459546925565,
+      "grad_norm": 0.09946475923061371,
+      "learning_rate": 5e-05,
+      "loss": 0.3073,
+      "num_input_tokens_seen": 6672576,
+      "step": 50
+    },
+    {
+      "epoch": 0.3883495145631068,
+      "grad_norm": 0.11918677389621735,
+      "learning_rate": 4.992735514089577e-05,
+      "loss": 0.2516,
+      "num_input_tokens_seen": 7942496,
+      "step": 60
+    },
+    {
+      "epoch": 0.45307443365695793,
+      "grad_norm": 0.07622148096561432,
+      "learning_rate": 4.970984274562741e-05,
+      "loss": 0.2256,
+      "num_input_tokens_seen": 9300240,
+      "step": 70
+    },
+    {
+      "epoch": 0.517799352750809,
+      "grad_norm": 0.07995602488517761,
+      "learning_rate": 4.934872690677953e-05,
+      "loss": 0.2146,
+      "num_input_tokens_seen": 10674880,
+      "step": 80
+    },
+    {
+      "epoch": 0.5825242718446602,
+      "grad_norm": 0.07474230229854584,
+      "learning_rate": 4.884610628109082e-05,
+      "loss": 0.2018,
+      "num_input_tokens_seen": 12052928,
+      "step": 90
+    },
+    {
+      "epoch": 0.6472491909385113,
+      "grad_norm": 0.07976042479276657,
+      "learning_rate": 4.820490189292415e-05,
+      "loss": 0.1958,
+      "num_input_tokens_seen": 13387616,
+      "step": 100
+    },
+    {
+      "epoch": 0.7119741100323624,
+      "grad_norm": 0.08398651331663132,
+      "learning_rate": 4.742884015847436e-05,
+      "loss": 0.1912,
+      "num_input_tokens_seen": 14676976,
+      "step": 110
+    },
+    {
+      "epoch": 0.7766990291262136,
+      "grad_norm": 0.088481605052948,
+      "learning_rate": 4.652243122936986e-05,
+      "loss": 0.1876,
+      "num_input_tokens_seen": 15991008,
+      "step": 120
+    },
+    {
+      "epoch": 0.8414239482200647,
+      "grad_norm": 0.09539451450109482,
+      "learning_rate": 4.5490942781526316e-05,
+      "loss": 0.1802,
+      "num_input_tokens_seen": 17281840,
+      "step": 130
+    },
+    {
+      "epoch": 0.9061488673139159,
+      "grad_norm": 0.08230065554380417,
+      "learning_rate": 4.434036940158062e-05,
+      "loss": 0.1793,
+      "num_input_tokens_seen": 18603200,
+      "step": 140
+    },
+    {
+      "epoch": 0.970873786407767,
+      "grad_norm": 0.09070578217506409,
+      "learning_rate": 4.307739774881878e-05,
+      "loss": 0.1759,
+      "num_input_tokens_seen": 19844160,
+      "step": 150
+    },
+    {
+      "epoch": 1.035598705501618,
+      "grad_norm": 0.09057565778493881,
+      "learning_rate": 4.170936769506222e-05,
+      "loss": 0.1746,
+      "num_input_tokens_seen": 21111168,
+      "step": 160
+    },
+    {
+      "epoch": 1.1003236245954693,
+      "grad_norm": 0.08840323239564896,
+      "learning_rate": 4.024422966835136e-05,
+      "loss": 0.1699,
+      "num_input_tokens_seen": 22442800,
+      "step": 170
+    },
+    {
+      "epoch": 1.1650485436893203,
+      "grad_norm": 0.10438356548547745,
+      "learning_rate": 3.86904984483277e-05,
+      "loss": 0.168,
+      "num_input_tokens_seen": 23815072,
+      "step": 180
+    },
+    {
+      "epoch": 1.2297734627831716,
+      "grad_norm": 0.09192164242267609,
+      "learning_rate": 3.7057203681836406e-05,
+      "loss": 0.1646,
+      "num_input_tokens_seen": 25064672,
+      "step": 190
+    },
+    {
+      "epoch": 1.2944983818770226,
+      "grad_norm": 0.08962567150592804,
+      "learning_rate": 3.535383740633246e-05,
+      "loss": 0.1667,
+      "num_input_tokens_seen": 26380864,
+      "step": 200
+    },
+    {
+      "epoch": 1.3592233009708738,
+      "grad_norm": 0.1008475124835968,
+      "learning_rate": 3.3590298886062833e-05,
+      "loss": 0.1664,
+      "num_input_tokens_seen": 27735952,
+      "step": 210
+    },
+    {
+      "epoch": 1.4239482200647249,
+      "grad_norm": 0.09535906463861465,
+      "learning_rate": 3.177683708161389e-05,
+      "loss": 0.1622,
+      "num_input_tokens_seen": 28991248,
+      "step": 220
+    },
+    {
+      "epoch": 1.4886731391585761,
+      "grad_norm": 0.09163232892751694,
+      "learning_rate": 2.9923991087167658e-05,
+      "loss": 0.1623,
+      "num_input_tokens_seen": 30313776,
+      "step": 230
+    },
+    {
+      "epoch": 1.5533980582524272,
+      "grad_norm": 0.09542625397443771,
+      "learning_rate": 2.804252888162079e-05,
+      "loss": 0.1616,
+      "num_input_tokens_seen": 31667088,
+      "step": 240
+    },
+    {
+      "epoch": 1.6181229773462782,
+      "grad_norm": 0.09714806824922562,
+      "learning_rate": 2.6143384749519866e-05,
+      "loss": 0.159,
+      "num_input_tokens_seen": 32942832,
+      "step": 250
+    },
+    {
+      "epoch": 1.6828478964401294,
+      "grad_norm": 0.10313564538955688,
+      "learning_rate": 2.423759573549647e-05,
+      "loss": 0.1619,
+      "num_input_tokens_seen": 34263264,
+      "step": 260
+    },
+    {
+      "epoch": 1.7475728155339807,
+      "grad_norm": 0.09954512119293213,
+      "learning_rate": 2.23362375015031e-05,
+      "loss": 0.1616,
+      "num_input_tokens_seen": 35603952,
+      "step": 270
+    },
+    {
+      "epoch": 1.8122977346278317,
+      "grad_norm": 0.1067601814866066,
+      "learning_rate": 2.0450359959620967e-05,
+      "loss": 0.1604,
+      "num_input_tokens_seen": 36890768,
+      "step": 280
+    },
+    {
+      "epoch": 1.8770226537216828,
+      "grad_norm": 0.09286054223775864,
+      "learning_rate": 1.8590923054515503e-05,
+      "loss": 0.1563,
+      "num_input_tokens_seen": 38275536,
+      "step": 290
+    },
+    {
+      "epoch": 1.941747572815534,
+      "grad_norm": 0.09992893040180206,
+      "learning_rate": 1.676873306874547e-05,
+      "loss": 0.1548,
+      "num_input_tokens_seen": 39648800,
+      "step": 300
+    },
+    {
+      "epoch": 2.0064724919093853,
+      "grad_norm": 0.09988085925579071,
+      "learning_rate": 1.4994379821093049e-05,
+      "loss": 0.1555,
+      "num_input_tokens_seen": 40991008,
+      "step": 310
+    },
+    {
+      "epoch": 2.071197411003236,
+      "grad_norm": 0.09905947744846344,
+      "learning_rate": 1.3278175122892416e-05,
+      "loss": 0.1526,
+      "num_input_tokens_seen": 42342880,
+      "step": 320
+    },
+    {
+      "epoch": 2.1359223300970873,
+      "grad_norm": 0.10081404447555542,
+      "learning_rate": 1.1630092850023147e-05,
+      "loss": 0.1526,
+      "num_input_tokens_seen": 43654784,
+      "step": 330
+    },
+    {
+      "epoch": 2.2006472491909386,
+      "grad_norm": 0.10631163418292999,
+      "learning_rate": 1.005971097884561e-05,
+      "loss": 0.1516,
+      "num_input_tokens_seen": 45039744,
+      "step": 340
+    },
+    {
+      "epoch": 2.26537216828479,
+      "grad_norm": 0.10914473235607147,
+      "learning_rate": 8.576155922941548e-06,
+      "loss": 0.1524,
+      "num_input_tokens_seen": 46331568,
+      "step": 350
+    },
+    {
+      "epoch": 2.3300970873786406,
+      "grad_norm": 0.10382701456546783,
+      "learning_rate": 7.1880494941517026e-06,
+      "loss": 0.1502,
+      "num_input_tokens_seen": 47685808,
+      "step": 360
+    },
+    {
+      "epoch": 2.394822006472492,
+      "grad_norm": 0.10681454837322235,
+      "learning_rate": 5.903458796151381e-06,
+      "loss": 0.1506,
+      "num_input_tokens_seen": 49011728,
+      "step": 370
+    },
+    {
+      "epoch": 2.459546925566343,
+      "grad_norm": 0.10908275097608566,
+      "learning_rate": 4.729849341761602e-06,
+      "loss": 0.1479,
+      "num_input_tokens_seen": 50327504,
+      "step": 380
+    },
+    {
+      "epoch": 2.524271844660194,
+      "grad_norm": 0.1181163340806961,
+      "learning_rate": 3.674041666458963e-06,
+      "loss": 0.1501,
+      "num_input_tokens_seen": 51620256,
+      "step": 390
+    },
+    {
+      "epoch": 2.588996763754045,
+      "grad_norm": 0.09960592538118362,
+      "learning_rate": 2.7421716902285623e-06,
+      "loss": 0.1504,
+      "num_input_tokens_seen": 52952880,
+      "step": 400
+    },
+    {
+      "epoch": 2.6537216828478964,
+      "grad_norm": 0.10874225944280624,
+      "learning_rate": 1.939655058120521e-06,
+      "loss": 0.1524,
+      "num_input_tokens_seen": 54225632,
+      "step": 410
+    },
+    {
+      "epoch": 2.7184466019417477,
+      "grad_norm": 0.10990295559167862,
+      "learning_rate": 1.271155666748311e-06,
+      "loss": 0.1494,
+      "num_input_tokens_seen": 55561888,
+      "step": 420
+    },
+    {
+      "epoch": 2.783171521035599,
+      "grad_norm": 0.10411816090345383,
+      "learning_rate": 7.405585596397313e-07,
+      "loss": 0.1515,
+      "num_input_tokens_seen": 56900512,
+      "step": 430
+    },
+    {
+      "epoch": 2.8478964401294498,
+      "grad_norm": 0.10403899103403091,
+      "learning_rate": 3.5094734896174987e-07,
+      "loss": 0.1504,
+      "num_input_tokens_seen": 58197632,
+      "step": 440
+    },
+    {
+      "epoch": 2.912621359223301,
+      "grad_norm": 0.1029229462146759,
+      "learning_rate": 1.0458629483476867e-07,
+      "loss": 0.1498,
+      "num_input_tokens_seen": 59523808,
+      "step": 450
+    },
+    {
+      "epoch": 2.9773462783171523,
+      "grad_norm": 0.10788564383983612,
+      "learning_rate": 2.9071463840540935e-09,
+      "loss": 0.1469,
+      "num_input_tokens_seen": 60787344,
+      "step": 460
+    },
+    {
+      "epoch": 2.9902912621359223,
+      "num_input_tokens_seen": 61072080,
+      "step": 462,
+      "total_flos": 5.065534456681464e+18,
+      "train_loss": 0.1973544524990635,
+      "train_runtime": 23852.8939,
+      "train_samples_per_second": 0.622,
+      "train_steps_per_second": 0.019
+    }
+  ],
+  "logging_steps": 10,
+  "max_steps": 462,
+  "num_input_tokens_seen": 61072080,
+  "num_train_epochs": 3,
+  "save_steps": 500,
+  "stateful_callbacks": {
+    "TrainerControl": {
+      "args": {
+        "should_epoch_stop": false,
+        "should_evaluate": false,
+        "should_log": false,
+        "should_save": true,
+        "should_training_stop": true
+      },
+      "attributes": {}
+    }
+  },
+  "total_flos": 5.065534456681464e+18,
+  "train_batch_size": 2,
+  "trial_name": null,
+  "trial_params": null
+}

training_args.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:346b49d9842ae5cd7e514f29b370e03b5ce38cb58f691419ff1d443ec5ed1ed3
+size 6712

training_args.yaml

@@ -0,0 +1,35 @@
+bf16: true
+cutoff_len: 20000
+dataset: ALS
+dataset_dir: data
+ddp_timeout: 180000000
+deepspeed: cache/ds_z2_offload_config.json
+do_train: true
+finetuning_type: lora
+flash_attn: fa2
+gradient_accumulation_steps: 8
+include_num_input_tokens_seen: true
+learning_rate: 5.0e-05
+logging_steps: 10
+lora_alpha: 16
+lora_dropout: 0
+lora_rank: 8
+lora_target: all
+lr_scheduler_type: cosine
+max_grad_norm: 1.0
+max_samples: 100000
+model_name_or_path: microsoft/Phi-3-medium-128k-instruct
+num_train_epochs: 3.0
+optim: adamw_torch
+output_dir: saves/Custom/lora/train_2024-07-24-15-00-21
+packing: false
+per_device_train_batch_size: 2
+plot_loss: true
+preprocessing_num_workers: 16
+quantization_bit: 4
+quantization_method: bitsandbytes
+report_to: none
+save_steps: 500
+stage: sft
+template: default
+warmup_steps: 50