build.sh

INFERENCE_PRECISION=float16
MAX_BEAM_WIDTH=4
MAX_BATCH_SIZE=64

checkpoint_dir=multi_zh_distil_tllm_checkpoint_pos_emb_true
output_dir=distil_whisper_multi_zh_remove_padding
# python3 convert_checkpoint.py \
# --output_dir $checkpoint_dir \
# --model_name distil-large-v2

# checkpoint_dir=multi_zh_distil_tllm_int8_checkpoint_pos_emb_true
# output_dir=distil_whisper_multi_zh_int8_remove_padding
# python3 convert_checkpoint.py --use_weight_only \
# --weight_only_precision int8 \
# --output_dir $checkpoint_dir --model_name distil-large-v2

trtllm-build --checkpoint_dir ${checkpoint_dir}/encoder \
 --output_dir ${output_dir}/encoder \
 --moe_plugin disable \
 --enable_xqa disable \
 --max_batch_size ${MAX_BATCH_SIZE} \
 --gemm_plugin disable \
 --bert_attention_plugin ${INFERENCE_PRECISION} \
 --max_input_len 3000 --max_seq_len=3000


trtllm-build --checkpoint_dir ${checkpoint_dir}/decoder \
 --output_dir ${output_dir}/decoder \
 --moe_plugin disable \
 --enable_xqa disable \
 --max_beam_width ${MAX_BEAM_WIDTH} \
 --max_batch_size ${MAX_BATCH_SIZE} \
 --max_seq_len 114 \
 --max_input_len 14 \
 --max_encoder_input_len 3000 \
 --gemm_plugin ${INFERENCE_PRECISION} \
 --bert_attention_plugin ${INFERENCE_PRECISION} \
 --gpt_attention_plugin ${INFERENCE_PRECISION}


# batch_size=32
# padding_strategy=zero
# dataset=wenet-e2e/wenetspeech
# dataset_name=TEST_MEETING
# python3 run.py --engine_dir $output_dir \
# --enable_warmup \
# --dataset $dataset \
# --dataset_name $dataset_name \
# --dataset_split test \
# --compute_cer \
# --text_prefix "<|startoftranscript|><|zh|><|transcribe|><|notimestamps|>" \
# --name aishell_${dataset_name}_${output_dir}_padding_${padding_strategy}_batch_${batch_size}_cppsession \
# --batch_size $batch_size --padding_strategy $padding_strategy