lucyknada commited on
Commit
d6d49c4
·
verified ·
1 Parent(s): 283f496

Upload folder using huggingface_hub

Browse files
.gitattributes CHANGED
@@ -33,3 +33,5 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
33
  *.zip filter=lfs diff=lfs merge=lfs -text
34
  *.zst filter=lfs diff=lfs merge=lfs -text
35
  *tfevents* filter=lfs diff=lfs merge=lfs -text
 
 
 
33
  *.zip filter=lfs diff=lfs merge=lfs -text
34
  *.zst filter=lfs diff=lfs merge=lfs -text
35
  *tfevents* filter=lfs diff=lfs merge=lfs -text
36
+ flow.decoder.estimator.fp16.A10.plan filter=lfs diff=lfs merge=lfs -text
37
+ flow.decoder.estimator.fp16.Volta.plan filter=lfs diff=lfs merge=lfs -text
.msc ADDED
Binary file (1.5 kB). View file
 
.mv ADDED
@@ -0,0 +1 @@
 
 
1
+ Revision:master,CreatedAt:1734333501
1.zip ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:3c5a4049e1e995cc800dabfa33c7ddc6c37189bb3b685869287b841733f949e0
3
+ size 1088281
CosyVoice-BlankEN/config.json ADDED
@@ -0,0 +1,27 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "architectures": [
3
+ "Qwen2ForCausalLM"
4
+ ],
5
+ "attention_dropout": 0.0,
6
+ "bos_token_id": 151643,
7
+ "eos_token_id": 151645,
8
+ "hidden_act": "silu",
9
+ "hidden_size": 896,
10
+ "initializer_range": 0.02,
11
+ "intermediate_size": 4864,
12
+ "max_position_embeddings": 32768,
13
+ "max_window_layers": 24,
14
+ "model_type": "qwen2",
15
+ "num_attention_heads": 14,
16
+ "num_hidden_layers": 24,
17
+ "num_key_value_heads": 2,
18
+ "rms_norm_eps": 1e-06,
19
+ "rope_theta": 1000000.0,
20
+ "sliding_window": 32768,
21
+ "tie_word_embeddings": true,
22
+ "torch_dtype": "bfloat16",
23
+ "transformers_version": "4.40.1",
24
+ "use_cache": true,
25
+ "use_sliding_window": false,
26
+ "vocab_size": 151936
27
+ }
CosyVoice-BlankEN/generation_config.json ADDED
@@ -0,0 +1,14 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "bos_token_id": 151643,
3
+ "pad_token_id": 151643,
4
+ "do_sample": true,
5
+ "eos_token_id": [
6
+ 151645,
7
+ 151643
8
+ ],
9
+ "repetition_penalty": 1.1,
10
+ "temperature": 0.7,
11
+ "top_p": 0.8,
12
+ "top_k": 20,
13
+ "transformers_version": "4.37.0"
14
+ }
CosyVoice-BlankEN/merges.txt ADDED
The diff for this file is too large to render. See raw diff
 
CosyVoice-BlankEN/model.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:130282af0dfa9fe5840737cc49a0d339d06075f83c5a315c3372c9a0740d0b96
3
+ size 988097824
CosyVoice-BlankEN/tokenizer_config.json ADDED
@@ -0,0 +1,40 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "add_prefix_space": false,
3
+ "added_tokens_decoder": {
4
+ "151643": {
5
+ "content": "<|endoftext|>",
6
+ "lstrip": false,
7
+ "normalized": false,
8
+ "rstrip": false,
9
+ "single_word": false,
10
+ "special": true
11
+ },
12
+ "151644": {
13
+ "content": "<|im_start|>",
14
+ "lstrip": false,
15
+ "normalized": false,
16
+ "rstrip": false,
17
+ "single_word": false,
18
+ "special": true
19
+ },
20
+ "151645": {
21
+ "content": "<|im_end|>",
22
+ "lstrip": false,
23
+ "normalized": false,
24
+ "rstrip": false,
25
+ "single_word": false,
26
+ "special": true
27
+ }
28
+ },
29
+ "additional_special_tokens": ["<|im_start|>", "<|im_end|>"],
30
+ "bos_token": null,
31
+ "chat_template": "{% for message in messages %}{% if loop.first and messages[0]['role'] != 'system' %}{{ '<|im_start|>system\nYou are a helpful assistant.<|im_end|>\n' }}{% endif %}{{'<|im_start|>' + message['role'] + '\n' + message['content'] + '<|im_end|>' + '\n'}}{% endfor %}{% if add_generation_prompt %}{{ '<|im_start|>assistant\n' }}{% endif %}",
32
+ "clean_up_tokenization_spaces": false,
33
+ "eos_token": "<|im_end|>",
34
+ "errors": "replace",
35
+ "model_max_length": 32768,
36
+ "pad_token": "<|endoftext|>",
37
+ "split_special_tokens": false,
38
+ "tokenizer_class": "Qwen2Tokenizer",
39
+ "unk_token": null
40
+ }
CosyVoice-BlankEN/vocab.json ADDED
The diff for this file is too large to render. See raw diff
 
README.md ADDED
@@ -0,0 +1,224 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # CosyVoice
2
+
3
+ ## 👉🏻 [CosyVoice2 Demos](https://funaudiollm.github.io/cosyvoice2/) 👈🏻
4
+ [[CosyVoice2 Paper](https://fun-audio-llm.github.io/pdf/CosyVoice_v1.pdf)][[CosyVoice2 Studio](https://www.modelscope.cn/studios/iic/CosyVoice2-0.5B)]
5
+
6
+ ## 👉🏻 [CosyVoice Demos](https://fun-audio-llm.github.io/) 👈🏻
7
+ [[CosyVoice Paper](https://fun-audio-llm.github.io/pdf/CosyVoice_v1.pdf)][[CosyVoice Studio](https://www.modelscope.cn/studios/iic/CosyVoice-300M)][[CosyVoice Code](https://github.com/FunAudioLLM/CosyVoice)]
8
+
9
+ For `SenseVoice`, visit [SenseVoice repo](https://github.com/FunAudioLLM/SenseVoice) and [SenseVoice space](https://www.modelscope.cn/studios/iic/SenseVoice).
10
+
11
+ ## Roadmap
12
+
13
+ - [x] 2024/12
14
+
15
+ - [x] CosyVoice2-0.5B model release
16
+ - [x] CosyVoice2-0.5B streaming inference with no quality degradation
17
+
18
+ - [x] 2024/07
19
+
20
+ - [x] Flow matching training support
21
+ - [x] WeTextProcessing support when ttsfrd is not avaliable
22
+ - [x] Fastapi server and client
23
+
24
+ - [x] 2024/08
25
+
26
+ - [x] Repetition Aware Sampling(RAS) inference for llm stability
27
+ - [x] Streaming inference mode support, including kv cache and sdpa for rtf optimization
28
+
29
+ - [x] 2024/09
30
+
31
+ - [x] 25hz cosyvoice base model
32
+ - [x] 25hz cosyvoice voice conversion model
33
+
34
+ - [ ] TBD
35
+
36
+ - [ ] CosyVoice2-0.5B bistream inference support
37
+ - [ ] CosyVoice2-0.5B training and finetune recipie
38
+ - [ ] CosyVoice-500M trained with more multi-lingual data
39
+ - [ ] More...
40
+
41
+ ## Install
42
+
43
+ **Clone and install**
44
+
45
+ - Clone the repo
46
+ ``` sh
47
+ git clone --recursive https://github.com/FunAudioLLM/CosyVoice.git
48
+ # If you failed to clone submodule due to network failures, please run following command until success
49
+ cd CosyVoice
50
+ git submodule update --init --recursive
51
+ ```
52
+
53
+ - Install Conda: please see https://docs.conda.io/en/latest/miniconda.html
54
+ - Create Conda env:
55
+
56
+ ``` sh
57
+ conda create -n cosyvoice python=3.10
58
+ conda activate cosyvoice
59
+ # pynini is required by WeTextProcessing, use conda to install it as it can be executed on all platform.
60
+ conda install -y -c conda-forge pynini==2.1.5
61
+ pip install -r requirements.txt -i https://mirrors.aliyun.com/pypi/simple/ --trusted-host=mirrors.aliyun.com
62
+
63
+ # If you encounter sox compatibility issues
64
+ # ubuntu
65
+ sudo apt-get install sox libsox-dev
66
+ # centos
67
+ sudo yum install sox sox-devel
68
+ ```
69
+
70
+ **Model download**
71
+
72
+ We strongly recommend that you download our pretrained `CosyVoice-300M` `CosyVoice-300M-SFT` `CosyVoice-300M-Instruct` model and `CosyVoice-ttsfrd` resource.
73
+
74
+ If you are expert in this field, and you are only interested in training your own CosyVoice model from scratch, you can skip this step.
75
+
76
+ ``` python
77
+ # SDK模型下载
78
+ from modelscope import snapshot_download
79
+ snapshot_download('iic/CosyVoice2-0.5B', local_dir='pretrained_models/CosyVoice2-0.5B')
80
+ snapshot_download('iic/CosyVoice-300M', local_dir='pretrained_models/CosyVoice-300M')
81
+ snapshot_download('iic/CosyVoice-300M-25Hz', local_dir='pretrained_models/CosyVoice-300M-25Hz')
82
+ snapshot_download('iic/CosyVoice-300M-SFT', local_dir='pretrained_models/CosyVoice-300M-SFT')
83
+ snapshot_download('iic/CosyVoice-300M-Instruct', local_dir='pretrained_models/CosyVoice-300M-Instruct')
84
+ snapshot_download('iic/CosyVoice-ttsfrd', local_dir='pretrained_models/CosyVoice-ttsfrd')
85
+ ```
86
+
87
+ ``` sh
88
+ # git模型下载,请确保已安装git lfs
89
+ mkdir -p pretrained_models
90
+ git clone https://www.modelscope.cn/iic/CosyVoice2-0.5B.git pretrained_models/CosyVoice2-0.5B
91
+ git clone https://www.modelscope.cn/iic/CosyVoice-300M.git pretrained_models/CosyVoice-300M
92
+ git clone https://www.modelscope.cn/iic/CosyVoice-300M-25Hz.git pretrained_models/CosyVoice-300M-25Hz
93
+ git clone https://www.modelscope.cn/iic/CosyVoice-300M-SFT.git pretrained_models/CosyVoice-300M-SFT
94
+ git clone https://www.modelscope.cn/iic/CosyVoice-300M-Instruct.git pretrained_models/CosyVoice-300M-Instruct
95
+ git clone https://www.modelscope.cn/iic/CosyVoice-ttsfrd.git pretrained_models/CosyVoice-ttsfrd
96
+ ```
97
+
98
+ Optionaly, you can unzip `ttsfrd` resouce and install `ttsfrd` package for better text normalization performance.
99
+
100
+ Notice that this step is not necessary. If you do not install `ttsfrd` package, we will use WeTextProcessing by default.
101
+
102
+ ``` sh
103
+ cd pretrained_models/CosyVoice-ttsfrd/
104
+ unzip resource.zip -d .
105
+ pip install ttsfrd-0.3.6-cp38-cp38-linux_x86_64.whl
106
+ ```
107
+
108
+ **Basic Usage**
109
+
110
+ For zero_shot/cross_lingual inference, please use `CosyVoice2-0.5B` or `CosyVoice-300M` model.
111
+ For sft inference, please use `CosyVoice-300M-SFT` model.
112
+ For instruct inference, please use `CosyVoice-300M-Instruct` model.
113
+ We strongly recommend using `CosyVoice2-0.5B` model for better streaming performance.
114
+
115
+ First, add `third_party/Matcha-TTS` to your `PYTHONPATH`.
116
+
117
+ ``` sh
118
+ export PYTHONPATH=third_party/Matcha-TTS
119
+ ```
120
+
121
+ ``` python
122
+ from cosyvoice.cli.cosyvoice import CosyVoice, CosyVoice2
123
+ from cosyvoice.utils.file_utils import load_wav
124
+ import torchaudio
125
+
126
+ ## cosyvoice2 usage
127
+ cosyvoice2 = CosyVoice('pretrained_models/CosyVoice-300M-SFT', load_jit=False, load_onnx=False, load_trt=False)
128
+ # sft usage
129
+ prompt_speech_16k = load_wav('zero_shot_prompt.wav', 16000)
130
+ for i, j in enumerate(cosyvoice2.inference_zero_shot('收到好友从远方寄来的生日礼物,那份意外的惊喜与深深的祝福让我心中充满了甜蜜的快乐,笑容如花儿般绽放。', '希望你以后能够做的比我还好呦。', prompt_speech_16k, stream=True)):
131
+ torchaudio.save('zero_shot_{}.wav'.format(i), j['tts_speech'], cosyvoice2.sample_rate)
132
+
133
+ ## cosyvoice usage
134
+ cosyvoice = CosyVoice('pretrained_models/CosyVoice-300M-SFT', load_jit=True, load_onnx=False, fp16=True)
135
+ # sft usage
136
+ print(cosyvoice.list_avaliable_spks())
137
+ # change stream=True for chunk stream inference
138
+ for i, j in enumerate(cosyvoice.inference_sft('你好,我是通义生成式语音大模型,请问有什么可以帮您的吗?', '中文女', stream=False)):
139
+ torchaudio.save('sft_{}.wav'.format(i), j['tts_speech'], cosyvoice.sample_rate)
140
+
141
+ cosyvoice = CosyVoice('pretrained_models/CosyVoice-300M-25Hz') # or change to pretrained_models/CosyVoice-300M for 50Hz inference
142
+ # zero_shot usage, <|zh|><|en|><|jp|><|yue|><|ko|> for Chinese/English/Japanese/Cantonese/Korean
143
+ prompt_speech_16k = load_wav('zero_shot_prompt.wav', 16000)
144
+ for i, j in enumerate(cosyvoice.inference_zero_shot('收到好友从远方寄来的生日礼物,那份意外的惊喜与深深的祝福让我心中充满了甜蜜的快乐,笑容如花儿般绽放。', '希望你以后能够做的比我还好呦。', prompt_speech_16k, stream=False)):
145
+ torchaudio.save('zero_shot_{}.wav'.format(i), j['tts_speech'], cosyvoice.sample_rate)
146
+ # cross_lingual usage
147
+ prompt_speech_16k = load_wav('cross_lingual_prompt.wav', 16000)
148
+ for i, j in enumerate(cosyvoice.inference_cross_lingual('<|en|>And then later on, fully acquiring that company. So keeping management in line, interest in line with the asset that\'s coming into the family is a reason why sometimes we don\'t buy the whole thing.', prompt_speech_16k, stream=False)):
149
+ torchaudio.save('cross_lingual_{}.wav'.format(i), j['tts_speech'], cosyvoice.sample_rate)
150
+ # vc usage
151
+ prompt_speech_16k = load_wav('zero_shot_prompt.wav', 16000)
152
+ source_speech_16k = load_wav('cross_lingual_prompt.wav', 16000)
153
+ for i, j in enumerate(cosyvoice.inference_vc(source_speech_16k, prompt_speech_16k, stream=False)):
154
+ torchaudio.save('vc_{}.wav'.format(i), j['tts_speech'], cosyvoice.sample_rate)
155
+
156
+ cosyvoice = CosyVoice('pretrained_models/CosyVoice-300M-Instruct')
157
+ # instruct usage, support <laughter></laughter><strong></strong>[laughter][breath]
158
+ for i, j in enumerate(cosyvoice.inference_instruct('在面对挑战时,他展现了非凡的<strong>勇气</strong>与<strong>智慧</strong>。', '中文男', 'Theo \'Crimson\', is a fiery, passionate rebel leader. Fights with fervor for justice, but struggles with impulsiveness.', stream=False)):
159
+ torchaudio.save('instruct_{}.wav'.format(i), j['tts_speech'], cosyvoice.sample_rate)
160
+ ```
161
+
162
+ **Start web demo**
163
+
164
+ You can use our web demo page to get familiar with CosyVoice quickly.
165
+ We support sft/zero_shot/cross_lingual/instruct inference in web demo.
166
+
167
+ Please see the demo website for details.
168
+
169
+ ``` python
170
+ # change iic/CosyVoice-300M-SFT for sft inference, or iic/CosyVoice-300M-Instruct for instruct inference
171
+ python3 webui.py --port 50000 --model_dir pretrained_models/CosyVoice-300M
172
+ ```
173
+
174
+ **Advanced Usage**
175
+
176
+ For advanced user, we have provided train and inference scripts in `examples/libritts/cosyvoice/run.sh`.
177
+ You can get familiar with CosyVoice following this recipie.
178
+
179
+ **Build for deployment**
180
+
181
+ Optionally, if you want to use grpc for service deployment,
182
+ you can run following steps. Otherwise, you can just ignore this step.
183
+
184
+ ``` sh
185
+ cd runtime/python
186
+ docker build -t cosyvoice:v1.0 .
187
+ # change iic/CosyVoice-300M to iic/CosyVoice-300M-Instruct if you want to use instruct inference
188
+ # for grpc usage
189
+ docker run -d --runtime=nvidia -p 50000:50000 cosyvoice:v1.0 /bin/bash -c "cd /opt/CosyVoice/CosyVoice/runtime/python/grpc && python3 server.py --port 50000 --max_conc 4 --model_dir iic/CosyVoice-300M && sleep infinity"
190
+ cd grpc && python3 client.py --port 50000 --mode <sft|zero_shot|cross_lingual|instruct>
191
+ # for fastapi usage
192
+ docker run -d --runtime=nvidia -p 50000:50000 cosyvoice:v1.0 /bin/bash -c "cd /opt/CosyVoice/CosyVoice/runtime/python/fastapi && python3 server.py --port 50000 --model_dir iic/CosyVoice-300M && sleep infinity"
193
+ cd fastapi && python3 client.py --port 50000 --mode <sft|zero_shot|cross_lingual|instruct>
194
+ ```
195
+
196
+ ## Discussion & Communication
197
+
198
+ You can directly discuss on [Github Issues](https://github.com/FunAudioLLM/CosyVoice/issues).
199
+
200
+ You can also scan the QR code to join our official Dingding chat group.
201
+
202
+ <img src="./asset/dingding.png" width="250px">
203
+
204
+ ## Acknowledge
205
+
206
+ 1. We borrowed a lot of code from [FunASR](https://github.com/modelscope/FunASR).
207
+ 2. We borrowed a lot of code from [FunCodec](https://github.com/modelscope/FunCodec).
208
+ 3. We borrowed a lot of code from [Matcha-TTS](https://github.com/shivammehta25/Matcha-TTS).
209
+ 4. We borrowed a lot of code from [AcademiCodec](https://github.com/yangdongchao/AcademiCodec).
210
+ 5. We borrowed a lot of code from [WeNet](https://github.com/wenet-e2e/wenet).
211
+
212
+ ## Citations
213
+
214
+ ``` bibtex
215
+ @article{du2024cosyvoice,
216
+ title={Cosyvoice: A scalable multilingual zero-shot text-to-speech synthesizer based on supervised semantic tokens},
217
+ author={Du, Zhihao and Chen, Qian and Zhang, Shiliang and Hu, Kai and Lu, Heng and Yang, Yexin and Hu, Hangrui and Zheng, Siqi and Gu, Yue and Ma, Ziyang and others},
218
+ journal={arXiv preprint arXiv:2407.05407},
219
+ year={2024}
220
+ }
221
+ ```
222
+
223
+ ## Disclaimer
224
+ The content provided above is for academic purposes only and is intended to demonstrate technical capabilities. Some examples are sourced from the internet. If any content infringes on your rights, please contact us to request its removal.
asset/dingding.png ADDED
campplus.onnx ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:a6ac6a63997761ae2997373e2ee1c47040854b4b759ea41ec48e4e42df0f4d73
3
+ size 28303423
configuration.json ADDED
@@ -0,0 +1 @@
 
 
1
+ {"framework":"Pytorch","task":"text-to-speech"}
cosyvoice.yaml ADDED
@@ -0,0 +1,140 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # set random seed, so that you may reproduce your result.
2
+ __set_seed1: !apply:random.seed [1986]
3
+ __set_seed2: !apply:numpy.random.seed [1986]
4
+ __set_seed3: !apply:torch.manual_seed [1986]
5
+ __set_seed4: !apply:torch.cuda.manual_seed_all [1986]
6
+
7
+ # fixed params
8
+ sample_rate: 24000
9
+ llm_input_size: 896
10
+ llm_output_size: 896
11
+ spk_embed_dim: 192
12
+ qwen_pretrain_path: ''
13
+
14
+ # model params
15
+ # for all class/function included in this repo, we use !<name> or !<new> for intialization, so that user may find all corresponding class/function according to one single yaml.
16
+ # for system/third_party class/function, we do not require this.
17
+ llm: !new:cosyvoice.llm.llm.Qwen2LM
18
+ llm_input_size: !ref <llm_input_size>
19
+ llm_output_size: !ref <llm_output_size>
20
+ speech_token_size: 6561
21
+ length_normalized_loss: True
22
+ lsm_weight: 0
23
+ llm: !new:cosyvoice.llm.llm.Qwen2Encoder
24
+ pretrain_path: !ref <qwen_pretrain_path>
25
+ sampling: !name:cosyvoice.utils.common.ras_sampling
26
+ top_p: 0.8
27
+ top_k: 25
28
+ win_size: 10
29
+ tau_r: 0.1
30
+
31
+ flow: !new:cosyvoice.flow.flow.CausalMaskedDiffWithXvec
32
+ input_size: 512
33
+ output_size: 80
34
+ spk_embed_dim: !ref <spk_embed_dim>
35
+ output_type: 'mel'
36
+ vocab_size: 6561
37
+ input_frame_rate: 25
38
+ only_mask_loss: True
39
+ token_mel_ratio: 2
40
+ pre_lookahead_len: 3
41
+ encoder: !new:cosyvoice.transformer.upsample_encoder.UpsampleConformerEncoder
42
+ output_size: 512
43
+ attention_heads: 8
44
+ linear_units: 2048
45
+ num_blocks: 6
46
+ dropout_rate: 0.1
47
+ positional_dropout_rate: 0.1
48
+ attention_dropout_rate: 0.1
49
+ normalize_before: True
50
+ input_layer: 'linear'
51
+ pos_enc_layer_type: 'rel_pos_espnet'
52
+ selfattention_layer_type: 'rel_selfattn'
53
+ input_size: 512
54
+ use_cnn_module: False
55
+ macaron_style: False
56
+ decoder: !new:cosyvoice.flow.flow_matching.CausalConditionalCFM
57
+ in_channels: 240
58
+ n_spks: 1
59
+ spk_emb_dim: 80
60
+ cfm_params: !new:omegaconf.DictConfig
61
+ content:
62
+ sigma_min: 1e-06
63
+ solver: 'euler'
64
+ t_scheduler: 'cosine'
65
+ training_cfg_rate: 0.2
66
+ inference_cfg_rate: 0.7
67
+ reg_loss_type: 'l1'
68
+ estimator: !new:cosyvoice.flow.decoder.ConditionalDecoder
69
+ in_channels: 320
70
+ out_channels: 80
71
+ causal: True
72
+ channels: [256]
73
+ dropout: 0.0
74
+ attention_head_dim: 64
75
+ n_blocks: 4
76
+ num_mid_blocks: 12
77
+ num_heads: 8
78
+ act_fn: 'gelu'
79
+
80
+ hift: !new:cosyvoice.hifigan.generator.HiFTGenerator
81
+ in_channels: 80
82
+ base_channels: 512
83
+ nb_harmonics: 8
84
+ sampling_rate: !ref <sample_rate>
85
+ nsf_alpha: 0.1
86
+ nsf_sigma: 0.003
87
+ nsf_voiced_threshold: 10
88
+ upsample_rates: [8, 5, 3]
89
+ upsample_kernel_sizes: [16, 11, 7]
90
+ istft_params:
91
+ n_fft: 16
92
+ hop_len: 4
93
+ resblock_kernel_sizes: [3, 7, 11]
94
+ resblock_dilation_sizes: [[1, 3, 5], [1, 3, 5], [1, 3, 5]]
95
+ source_resblock_kernel_sizes: [7, 7, 11]
96
+ source_resblock_dilation_sizes: [[1, 3, 5], [1, 3, 5], [1, 3, 5]]
97
+ lrelu_slope: 0.1
98
+ audio_limit: 0.99
99
+ f0_predictor: !new:cosyvoice.hifigan.f0_predictor.ConvRNNF0Predictor
100
+ num_class: 1
101
+ in_channels: 80
102
+ cond_channels: 512
103
+
104
+ # processor functions
105
+ parquet_opener: !name:cosyvoice.dataset.processor.parquet_opener
106
+ get_tokenizer: !name:cosyvoice.tokenizer.tokenizer.get_qwen_tokenizer
107
+ token_path: !ref <qwen_pretrain_path>
108
+ skip_special_tokens: True
109
+ allowed_special: 'all'
110
+ tokenize: !name:cosyvoice.dataset.processor.tokenize
111
+ get_tokenizer: !ref <get_tokenizer>
112
+ allowed_special: !ref <allowed_special>
113
+ filter: !name:cosyvoice.dataset.processor.filter
114
+ max_length: 40960
115
+ min_length: 0
116
+ token_max_length: 200
117
+ token_min_length: 1
118
+ resample: !name:cosyvoice.dataset.processor.resample
119
+ resample_rate: !ref <sample_rate>
120
+ feat_extractor: !name:matcha.utils.audio.mel_spectrogram
121
+ n_fft: 1920
122
+ num_mels: 80
123
+ sampling_rate: !ref <sample_rate>
124
+ hop_size: 480
125
+ win_size: 1920
126
+ fmin: 0
127
+ fmax: 8000
128
+ center: False
129
+ compute_fbank: !name:cosyvoice.dataset.processor.compute_fbank
130
+ feat_extractor: !ref <feat_extractor>
131
+ parse_embedding: !name:cosyvoice.dataset.processor.parse_embedding
132
+ normalize: True
133
+ shuffle: !name:cosyvoice.dataset.processor.shuffle
134
+ shuffle_size: 1000
135
+ sort: !name:cosyvoice.dataset.processor.sort
136
+ sort_size: 500 # sort_size should be less than shuffle_size
137
+ batch: !name:cosyvoice.dataset.processor.batch
138
+ batch_type: 'dynamic'
139
+ max_frames_in_batch: 2000
140
+ padding: !name:cosyvoice.dataset.processor.padding
flow.decoder.estimator.fp16.A10.plan ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:6f6b9073bd9e7b8ac5bef0a21431391cbc32376b9265ec73935d6f28a0d32d01
3
+ size 168597292
flow.decoder.estimator.fp16.Volta.plan ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:f231edf01451fafbc3dc0498a51feb3a264afad43275536c8151fff954ef3c56
3
+ size 161799540
flow.decoder.estimator.fp32.onnx ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:51aed3efa2c153898ea53a780893c920e968dab1d7aec25402bd6c9815d94702
3
+ size 286521895
flow.encoder.fp32.zip ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:7cf2fdb6cc83952d20a4faade946ab867068b1f5531c95f7b61db932c44c5a43
3
+ size 192366262
flow.pt ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:ff4c2f867674411e0a08cee702996df13fa67c1cd864c06108da88d16d088541
3
+ size 450575567
hift.pt ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:1d4af0d661a416c69544eec83ff9c070dc80c37ee53ef44af3a37d910c95bc21
3
+ size 83364158
llm.pt ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:b144ef55b51ce8cfb79a73c90dbba0bdaba4e451c0ebcfab20f769264f84a608
3
+ size 2023316821
speech_tokenizer_v2.onnx ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:8ce1918400101d13593d64dd35368f0254cf25ea36dee2254e55303142e52d5e
3
+ size 496127134
spk2info.pt ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:fb491609da25a3514b76d76d55dbf26686394c378d596128fb3e15f2b74cdf44
3
+ size 1317821