NaturalSpeech2 / utils /prompt_preparer.py
yuancwang
init
b725c5a
raw
history blame
2.88 kB
# Copyright (c) 2023 Amphion.
#
# This source code is licensed under the MIT license found in the
# LICENSE file in the root directory of this source tree.
import torch
class PromptPreparer:
def prepare_prompts(self, y, y_lens, codes, nar_stage, y_prompts_codes):
if self.prefix_mode == 0:
y_emb, prefix_len = self._handle_prefix_mode_0(y, codes, nar_stage)
elif self.prefix_mode == 1:
y_emb, prefix_len = self._handle_prefix_mode_1(y, y_lens, codes, nar_stage)
elif self.prefix_mode in [2, 4]:
y_emb, prefix_len = self._handle_prefix_mode_2_4(
y, y_lens, codes, nar_stage, y_prompts_codes
)
else:
raise ValueError("Invalid prefix mode")
return y_emb, prefix_len
def _handle_prefix_mode_0(self, y, codes, nar_stage):
prefix_len = 0
y_emb = self.nar_audio_embeddings[0](y)
for j in range(1, nar_stage):
y_emb = y_emb + self.nar_audio_embeddings[j](codes[..., j])
return y_emb, 0
def _handle_prefix_mode_1(self, y, y_lens, codes, nar_stage):
int_low = (0.25 * y_lens.min()).type(torch.int64).item()
prefix_len = torch.randint(int_low, int_low * 2, size=()).item()
prefix_len = min(prefix_len, 225)
y_prompts = self.nar_audio_embeddings[0](y[:, :prefix_len])
y_emb = self.nar_audio_embeddings[0](y[:, prefix_len:])
for j in range(1, self.num_quantizers):
y_prompts += self.nar_audio_embeddings[j](codes[:, :prefix_len, j])
if j < nar_stage:
y_emb += self.nar_audio_embeddings[j](codes[:, prefix_len:, j])
y_emb = torch.concat([y_prompts, y_emb], axis=1)
return y_emb, prefix_len
def _handle_prefix_mode_2_4(self, y, y_lens, codes, nar_stage, y_prompts_codes):
if self.prefix_mode == 2:
prefix_len = min(225, int(0.25 * y_lens.min().item()))
y_prompts_codes = []
for b in range(codes.shape[0]):
start = self.rng.randint(0, y_lens[b].item() - prefix_len)
y_prompts_codes.append(
torch.clone(codes[b, start : start + prefix_len])
)
codes[b, start : start + prefix_len, nar_stage] = self.audio_token_num
y_prompts_codes = torch.stack(y_prompts_codes, dim=0)
else:
prefix_len = y_prompts_codes.shape[1]
y_prompts = self.nar_audio_embeddings[0](y_prompts_codes[..., 0])
y_emb = self.nar_audio_embeddings[0](y)
for j in range(1, self.num_quantizers):
y_prompts += self.nar_audio_embeddings[j](y_prompts_codes[..., j])
if j < nar_stage:
y_emb += self.nar_audio_embeddings[j](codes[..., j])
y_emb = torch.concat([y_prompts, y_emb], axis=1)
return y_emb, prefix_len