Upload folder using huggingface_hub

Qwen2_1.5B_merged/config.json

@@ -0,0 +1,28 @@
+{
+ "_name_or_path": "/home/scratch.yuekaiz_wwfo_1/Qwen2-1.5B-Instruct",
+ "architectures": [
+ "Qwen2ForCausalLM"
+ ],
+ "attention_dropout": 0.0,
+ "bos_token_id": 151643,
+ "eos_token_id": 151645,
+ "hidden_act": "silu",
+ "hidden_size": 1536,
+ "initializer_range": 0.02,
+ "intermediate_size": 8960,
+ "max_position_embeddings": 32768,
+ "max_window_layers": 28,
+ "model_type": "qwen2",
+ "num_attention_heads": 12,
+ "num_hidden_layers": 28,
+ "num_key_value_heads": 2,
+ "rms_norm_eps": 1e-06,
+ "rope_theta": 1000000.0,
+ "sliding_window": 32768,
+ "tie_word_embeddings": true,
+ "torch_dtype": "float16",
+ "transformers_version": "4.42.4",
+ "use_cache": true,
+ "use_sliding_window": false,
+ "vocab_size": 151936
+}

Qwen2_1.5B_merged/generation_config.json

@@ -0,0 +1,14 @@
+{
+ "bos_token_id": 151643,
+ "do_sample": true,
+ "eos_token_id": [
+ 151645,
+ 151643
+ ],
+ "pad_token_id": 151643,
+ "repetition_penalty": 1.1,
+ "temperature": 0.7,
+ "top_k": 20,
+ "top_p": 0.8,
+ "transformers_version": "4.42.4"
+}

Qwen2_1.5B_merged/model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:cd9913bcefe5f587335ad14d182a3d624abd87db893389398493c3a5cee3d30d
+size 3087466808

Qwen2_1.5B_merged/tokenizer_config.json

@@ -0,0 +1,40 @@
+{
+ "add_prefix_space": false,
+ "added_tokens_decoder": {
+ "151643": {
+ "content": "<|endoftext|>",
+ "lstrip": false,
+ "normalized": false,
+ "rstrip": false,
+ "single_word": false,
+ "special": true
+ },
+ "151644": {
+ "content": "<|im_start|>",
+ "lstrip": false,
+ "normalized": false,
+ "rstrip": false,
+ "single_word": false,
+ "special": true
+ },
+ "151645": {
+ "content": "<|im_end|>",
+ "lstrip": false,
+ "normalized": false,
+ "rstrip": false,
+ "single_word": false,
+ "special": true
+ }
+ },
+ "additional_special_tokens": ["<|im_start|>", "<|im_end|>"],
+ "bos_token": null,
+ "chat_template": "{% for message in messages %}{% if loop.first and messages[0]['role'] != 'system' %}{{ '<|im_start|>system\nYou are a helpful assistant.<|im_end|>\n' }}{% endif %}{{'<|im_start|>' + message['role'] + '\n' + message['content'] + '<|im_end|>' + '\n'}}{% endfor %}{% if add_generation_prompt %}{{ '<|im_start|>assistant\n' }}{% endif %}",
+ "clean_up_tokenization_spaces": false,
+ "eos_token": "<|im_end|>",
+ "errors": "replace",
+ "model_max_length": 32768,
+ "pad_token": "<|endoftext|>",
+ "split_special_tokens": false,
+ "tokenizer_class": "Qwen2Tokenizer",
+ "unk_token": null
+}

build_qwen.sh

@@ -0,0 +1,16 @@
+# You need to merge Qwen2-1.5B-Instruct with fine-tuned LLM lora first, see merge_lora.py.
+# Then you can build the engine with the merged model.
+# python3 convert_checkpoint.py --model_dir ${model_dir} \
+# --output_dir ${checkpoint_dir} \
+# --dtype float16
+# We have merged the two models and convert it for trt-llm into the below checkpoint:
+checkpoint_dir=tllm_checkpoint_1gpu_fp16_qwen2_1.5B_instruct_merged
+# output engine directory
+engine_dir=qwen2_1.5B_instruct_fp16_merged
+
+# max_prompt_embedding_table_size should >= max_batch_size * speech_embedding_seq_length
+trtllm-build --checkpoint_dir ${checkpoint_dir} \
+ --output_dir $engine_dir \
+ --max_prompt_embedding_table_size 4096 \
+ --max_batch_size 16 \
+ --gemm_plugin float16

build_whisper_encoder.sh

@@ -0,0 +1,21 @@
+
+INFERENCE_PRECISION=float16
+MAX_BEAM_WIDTH=4
+MAX_BATCH_SIZE=64
+checkpoint_dir=whisper_multi_zh_tllm_checkpoint
+output_dir=whisper_multi_zh
+
+# Download the fine-tuned model https://huggingface.co/yuekai/icefall_asr_multi-hans-zh_whisper/blob/main/v1.1/whisper-large-v2-multi-hans-zh-epoch-3-avg-10.pt.
+# python3 convert_checkpoint.py \
+# --output_dir whisper_multi_zh_tllm_checkpoint \
+# --model_name large-v2
+
+trtllm-build --checkpoint_dir ${checkpoint_dir}/encoder \
+ --output_dir ${output_dir}/encoder \
+ --moe_plugin disable \
+ --enable_xqa disable \
+ --max_batch_size ${MAX_BATCH_SIZE} \
+ --gemm_plugin disable \
+ --bert_attention_plugin ${INFERENCE_PRECISION} \
+ --max_input_len 3000 --max_seq_len=3000 \
+ --remove_input_padding disable --paged_kv_cache disable

icefall_asr_multi-hans_whisper_qwen2_1.5B/epoch-2-avg-6.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c20e8745dace75339447806d5f9177e0b719a9eb796512542c268868f8d8bebb
+size 375542023

merge_lora.py

@@ -0,0 +1,76 @@
+import torch
+from torch import nn
+from transformers.trainer_pt_utils import LabelSmoother
+from transformers import AutoTokenizer, AutoModelForCausalLM
+from peft import LoraConfig, get_peft_model, prepare_model_for_kbit_training
+IGNORE_TOKEN_ID = LabelSmoother.ignore_index
+DEFAULT_SPEECH_TOKEN = "<speech>"
+
+
+class SPEECH_LLM(nn.Module):
+ """
+ The Speech-to-Text model. It consists of an encoder, a language model and an encoder projector.
+ The encoder is used to extract speech features from the input speech signal.
+ The encoder projector is used to project the encoder outputs to the same dimension as the language model.
+ The language model is used to generate the text from the speech features.
+ Args:
+ encoder (:obj:`nn.Module`): The encoder module.
+ llm (:obj:`nn.Module`): The language model module.
+ encoder_projector (:obj:`nn.Module`): The encoder projector module.
+ """
+
+ def __init__(
+ self,
+ encoder: nn.Module = None,
+ llm: nn.Module = None,
+ encoder_projector: nn.Module = None,
+ ):
+ super().__init__()
+ self.encoder = encoder
+ self.llm = llm
+ self.encoder_projector = encoder_projector
+
+
+
+if __name__ == "__main__":
+ speech_encoder_dim = 1280
+ encoder_projector_ds_rate = 8
+ llm_config_hidden_size = 1536
+
+ adapter_dir="/home/scratch.yuekaiz_wwfo_1/icefall_asr_multi-hans_whisper_qwen2_1.5B/epoch-2-avg-6.pt"
+ llm_dir="/home/scratch.yuekaiz_wwfo_1/Qwen2-1.5B-Instruct"
+ target_dir="/home/scratch.yuekaiz_wwfo_1/Qwen2_1.5B_merged"
+
+ llm = AutoModelForCausalLM.from_pretrained(
+ llm_dir,
+ torch_dtype=torch.float16,
+ )
+ lora_config = LoraConfig(
+ r=64,
+ lora_alpha=16,
+ target_modules=[
+ "q_proj",
+ "k_proj",
+ "v_proj",
+ "o_proj",
+ "up_proj",
+ "gate_proj",
+ "down_proj",
+ ],
+ task_type="CAUSAL_LM",
+ )
+ llm = get_peft_model(llm, lora_config)
+ model = SPEECH_LLM(
+ llm = llm,
+ )
+ 
+ checkpoint = torch.load(
+ adapter_dir, map_location="cpu"
+ )
+ missing_keys, unexpected_keys = model.load_state_dict(checkpoint, strict=False)
+
+ print(missing_keys, unexpected_keys)
+
+ llm_merged = model.llm.merge_and_unload()
+
+ llm_merged.save_pretrained(target_dir)

tllm_checkpoint_1gpu_fp16_qwen2_1.5B_instruct_merged/config.json

@@ -0,0 +1,53 @@
+{
+ "mlp_bias": false,
+ "attn_bias": true,
+ "rotary_base": 1000000.0,
+ "rotary_scaling": null,
+ "disable_weight_only_quant_plugin": false,
+ "moe": {
+ "num_experts": 0,
+ "top_k": 0,
+ "normalization_mode": 0,
+ "tp_mode": 0
+ },
+ "architecture": "Qwen2ForCausalLM",
+ "dtype": "float16",
+ "vocab_size": 151936,
+ "hidden_size": 1536,
+ "num_hidden_layers": 28,
+ "num_attention_heads": 12,
+ "hidden_act": "silu",
+ "logits_dtype": "float32",
+ "norm_epsilon": 1e-06,
+ "position_embedding_type": "rope_gpt_neox",
+ "max_position_embeddings": 32768,
+ "num_key_value_heads": 2,
+ "intermediate_size": 8960,
+ "mapping": {
+ "world_size": 1,
+ "gpus_per_node": 8,
+ "cp_size": 1,
+ "tp_size": 1,
+ "pp_size": 1,
+ "moe_tp_size": 1,
+ "moe_ep_size": 1
+ },
+ "quantization": {
+ "quant_algo": null,
+ "kv_cache_quant_algo": null,
+ "group_size": 128,
+ "smoothquant_val": 0.5,
+ "clamp_val": null,
+ "has_zero_point": false,
+ "pre_quant_scale": false,
+ "exclude_modules": null
+ },
+ "use_parallel_embedding": false,
+ "embedding_sharding_dim": 0,
+ "share_embedding_table": false,
+ "head_size": 128,
+ "qk_layernorm": false,
+ "qwen_type": "qwen2",
+ "moe_intermediate_size": 0,
+ "moe_shared_expert_intermediate_size": 0
+}

tllm_checkpoint_1gpu_fp16_qwen2_1.5B_instruct_merged/rank0.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:2d7c0035fb7e28fffd625e180be6238dccb44f763a763fe8843f0f0db2c0aada
+size 3587757176

whisper_multi_zh_tllm_checkpoint/encoder/config.json

@@ -0,0 +1,15 @@
+{
+ "architecture": "WhisperEncoder",
+ "dtype": "float16",
+ "num_hidden_layers": 32,
+ "num_attention_heads": 20,
+ "hidden_size": 1280,
+ "n_mels": 80,
+ "n_audio_ctx": 1500,
+ "vocab_size": 51865,
+ "hidden_act": "gelu",
+ "num_languages": 99,
+ "quantization": {
+ "quant_algo": null
+ }
+}

whisper_multi_zh_tllm_checkpoint/encoder/rank0.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:34c8b445a098fcd90eda7bfba8b9ff2fe7c34fb0b3589f80bc048b7411a6248d
+size 1277533656