SOTA Open Source TTS
Implement optional delay pattern
Lengyue073d2266e448
diff --git a/fish_speech/datasets/text.py b/fish_speech/datasets/text.pyindex 32853c9..98c2b20 100644--- a/fish_speech/datasets/text.py+++ b/fish_speech/datasets/text.py@@ -199,6 +199,7 @@ class AutoAugTextDataset(IterableDataset):mix_text_phone_prob: float = 0.5,use_negative_samples: bool = False,num_codebooks: Optional[int] = None,+ use_delay_pattern: bool = True,):"""Args:@@ -216,6 +217,7 @@ class AutoAugTextDataset(IterableDataset):mix_text_phone_prob: probability to mix text and phones, if this is 0, then it will be pure text or pure phonesuse_negative_samples: generate negative samplesnum_codebooks: number of codebooks, if None, it will be automatically detected+ use_delay_pattern: use delay pattern for codebooks"""super().__init__()@@ -238,6 +240,7 @@ class AutoAugTextDataset(IterableDataset):self.mix_text_phone_prob = mix_text_phone_probself.use_negative_samples = use_negative_samplesself.num_codebooks = num_codebooks+ self.use_delay_pattern = use_delay_patternif use_data_server is True:self.channel = grpc.insecure_channel(server)@@ -494,9 +497,12 @@ class AutoAugTextDataset(IterableDataset):bos_bias = 1 if add_bos else 0# Pack the tokens and semantics (add <s> and </s> to semantic tokens)+ pad_token_length = semantic_length + (+ num_codebooks - 1 if self.use_delay_pattern else 0+ )tokens = (encoded- + [self.tokenizer.pad_token_id] * (semantic_length + num_codebooks - 1)+ + [self.tokenizer.pad_token_id] * pad_token_length+ [self.tokenizer.eos_token_id])@@ -506,7 +512,8 @@ class AutoAugTextDataset(IterableDataset):# Codebook bos/padding: 0, eos: 1# Implement delay patterncodes = [- [CODEBOOK_PAD_TOKEN_ID] * (prompt_length + bos_bias + i)+ [CODEBOOK_PAD_TOKEN_ID]+ * (prompt_length + bos_bias + (i if self.use_delay_pattern else 0))for i in range(num_codebooks)]for segment in semantics:@@ -515,7 +522,8 @@ class AutoAugTextDataset(IterableDataset):codes[book_idx].append(int(j) + 2)for idx, book in enumerate(codes):- book.extend([CODEBOOK_PAD_TOKEN_ID] * (len(codes) - idx - 1))+ if self.use_delay_pattern:+ book.extend([CODEBOOK_PAD_TOKEN_ID] * (len(codes) - idx - 1))book.append(CODEBOOK_EOS_TOKEN_ID)tokens = [tokens] + codes