SOTA Open Source TTS
Optimize text dataset
Lengyued5bb7f526a13
diff --git a/fish_speech/datasets/text.py b/fish_speech/datasets/text.pyindex 15abc6f..21add2e 100644--- a/fish_speech/datasets/text.py+++ b/fish_speech/datasets/text.py@@ -26,6 +26,9 @@ from fish_speech.utils.braceexpand import braceexpandlog = RankedLogger(__name__, rank_zero_only=True)+CODEBOOK_BOS_TOKEN_ID = 0+CODEBOOK_EOS_TOKEN_ID = 1+def split_by_rank_worker(files):# We need to know the total number of devices@@ -171,6 +174,12 @@ class AutoAugTextDataset(IterableDataset):1. Random concatenate multiple sentences from the same speaker to form a longer sentence2. Automatically normalize the text3. Mix text and phones++ For interactive mode, we use the following format (multiple sequences):+ <s> [INST] [SPK: speaker] text [/INST] ... [INST] text [/INST] </s>++ For non-interactive mode, we use the following format (one long sequence):+ <s> [INST] text [/INST] ... </s>"""def __init__(@@ -179,6 +188,7 @@ class AutoAugTextDataset(IterableDataset):seed: int = 42,phones_prob: float = 0.3,repetition_prob: float = 0.0,+ interactive_prob: float = 0.5,max_length: int = 1024,tokenizer: AutoTokenizer = None,use_speaker: bool = True,@@ -189,6 +199,7 @@ class AutoAugTextDataset(IterableDataset):seed: random seedphones_prob: probability to use phonesrepetition_prob: probability to repeat the same sentence+ interactive_prob: probability to use interactive modemax_length: max length of the texttokenizer: tokenizer"""@@ -200,6 +211,7 @@ class AutoAugTextDataset(IterableDataset):self.max_length = max_lengthself.tokenizer = tokenizerself.repetition_prob = repetition_prob+ self.interactive_prob = interactive_probself.use_speaker = use_speaker# Read all lines, and group by speaker@@ -258,6 +270,10 @@ class AutoAugTextDataset(IterableDataset):return Nonesamples = list(response.samples)+ idx = 0+ use_interactive = random.random() < self.interactive_prob++ all_tokens, all_labels = [], []while remaining_tokens > 0 and len(samples) > 0:if random.random() < self.repetition_prob:# Repeat the same sentence@@ -269,70 +285,107 @@ class AutoAugTextDataset(IterableDataset):sentence.text, sentence.phones, mode=mode)remaining_tokens -= length + len(sentence.semantics[0].values)- final_text.append(text)- final_semantic.append(sentence.semantics)- if self.use_speaker:- final_text = [f"[SPK: {response.name}]"] + final_text+ if use_interactive is False:+ final_text.append(text)+ final_semantic.append(sentence.semantics)+ else:+ # For interactive mode, we only apply speaker for the first sentence+ # [INST] [SPK: speaker] text [/INST] ... [INST] text [/INST]+ tokens, labels = self.pack_sentences(+ sentences=[text],+ semantics=[sentence.semantics],+ speaker=response.name if (self.use_speaker and idx == 0) else None,+ add_bos=idx == 0,+ )++ all_tokens.append(tokens)+ all_labels.append(labels)++ idx += 1++ if use_interactive is False:+ tokens, labels = self.pack_sentences(+ final_text,+ semantics=final_semantic,+ speaker=None if self.use_speaker else sentence.speaker,+ add_bos=True,+ )+ else:+ print(all_tokens[0].shape)+ tokens = torch.cat(all_tokens, dim=1)+ labels = torch.cat(all_labels, dim=1)++ # Verify that the length is correct+ assert tokens.size(1) == labels.size(1), f"{tokens.size(1)} != {labels.size(1)}"++ # Verify only one <s> token+ assert (tokens[:, 0] == self.tokenizer.bos_token_id).sum() == 1++ return {"tokens": tokens, "labels": labels}- final_text = "[INST] " + " ".join(final_text) + " [/INST]"+ def pack_sentences(+ self,+ sentences: list[str],+ semantics=list,+ speaker: Optional[str] = None,+ add_bos: bool = True,+ ):+ if speaker is not None:+ sentences = [f"[SPK: {speaker}]"] + sentences++ final_text = "[INST] " + " ".join(sentences) + " [/INST]"encoded = self.tokenizer.encode(final_text,add_special_tokens=False,truncation=False,max_length=10**6,)- semantic_length = sum([len(i[0].values) for i in final_semantic])+ semantic_length = sum([len(i[0].values) for i in semantics])+ bos_bias = 1 if add_bos else 0++ # Pack the tokens and semantics (add <s> and </s> to semantic tokens)+ tokens = (+ encoded+ + [self.tokenizer.pad_token_id] * semantic_length+ + [self.tokenizer.eos_token_id]+ )- # Single codebook- if len(final_semantic[0]) == 1:- semantic_tokens = [f"<s:{j}>" for i in final_semantic for j in i[0].values]- tokenized = self.tokenizer.encode(- f" ".join(semantic_tokens),- add_special_tokens=False,- truncation=False,- max_length=10**6,- )+ if add_bos:+ tokens = [self.tokenizer.bos_token_id] + tokens- # Pack the tokens and semantics (add <s> and </s> to semantic tokens)- tokens = (- [self.tokenizer.bos_token_id]- + encoded- + tokenized- + [self.tokenizer.eos_token_id]- )- tokens = torch.tensor([tokens], dtype=torch.long)- labels = tokens.clone()- labels[0, : len(encoded) + 1] = -100 # Mask out the <s> and query tokens- else:- # Pack the tokens and semantics (add <s> and </s> to semantic tokens)- tokens = (- [self.tokenizer.bos_token_id]- + encoded- + [self.tokenizer.pad_token_id] * semantic_length- + [self.tokenizer.eos_token_id]- )- codes = [[0] * (len(encoded) + 1) for _ in range(len(final_semantic[0]))]- for segment in final_semantic:- for book_idx, book in enumerate(segment):- for j in book.values:- codes[book_idx].append(int(j) + 2)+ # Codebook bos/padding: 0, eos: 1+ codes = [+ [CODEBOOK_BOS_TOKEN_ID] * (len(encoded) + bos_bias)+ for _ in range(len(semantics[0]))+ ]+ for segment in semantics:+ for book_idx, book in enumerate(segment):+ for j in book.values:+ codes[book_idx].append(int(j) + 2)- for book in codes:- book.append(1)+ for book in codes:+ book.append(CODEBOOK_EOS_TOKEN_ID)- tokens = [tokens] + codes+ tokens = [tokens] + codes- tokens = torch.tensor(tokens, dtype=torch.long)- labels = tokens.clone()- labels[- 1:, : len(encoded) + 1- ] = -100 # Mask out the <s> tokens for semantic+ tokens = torch.tensor(tokens, dtype=torch.long)+ labels = tokens.clone()- return {- "tokens": tokens[:, :-1],- "labels": labels[:, 1:],- }+ # Mask out the <s> tokens for semantic, predict semantic tokens only+ # Since we don't mask out the input tokens, the language modeling still works+ labels[1:, : (len(encoded) + bos_bias)] = -100++ tokens = tokens[:, :-1]+ labels = labels[:, 1:]++ # Verify the padding is correct, and the last token is eos+ assert add_bos is False or tokens[0, 0] == self.tokenizer.bos_token_id+ assert (tokens[1:, : len(encoded) + bos_bias] == CODEBOOK_BOS_TOKEN_ID).all()+ assert labels[0, -1] == self.tokenizer.eos_token_id+ assert (labels[1:, -1] == CODEBOOK_EOS_TOKEN_ID).all()++ return tokens, labels@dataclass@@ -346,18 +399,20 @@ class TextDataCollator:_tokens = example["tokens"][:, : self.max_length]_labels = example["labels"][:, : self.max_length]_attention_mask = torch.ones((self.max_length,), dtype=torch.bool)- _attention_mask[: _tokens.size(1)] = False+ tokens_length = _tokens.size(1)+ _attention_mask[:tokens_length] = False- assert _tokens.size(1) == _labels.size(+ assert tokens_length == _labels.size(1- ), f"{_tokens.size(1)} != {_labels.size(1)}"+ ), f"{tokens_length} != {_labels.size(1)}"- if _tokens.size(1) < self.max_length:+ if tokens_length < self.max_length:_tokens = F.pad(_tokens,- (0, self.max_length - _tokens.size(1)),+ (0, self.max_length - tokens_length),value=self.tokenizer.eos_token_id,)+ _tokens[1:, tokens_length:] = CODEBOOK_EOS_TOKEN_ID_labels = F.pad(_labels, (0, self.max_length - _labels.size(1)), value=-100)@@ -444,20 +499,14 @@ class TextDataModule(LightningDataModule):if __name__ == "__main__":- import json-from tqdm import tqdmds = AutoAugTextDataset(tokenizer=AutoTokenizer.from_pretrained("fishaudio/speech-lm-v1"),use_speaker=True,+ interactive_prob=1.0,)- # ds = StreamTextDataset(- # prefix="en/",- # tokenizer=AutoTokenizer.from_pretrained("fishaudio/speech-lm-v1"),- # )-dm = TextDataModule(train_dataset=ds,val_dataset=ds,diff --git a/tools/llama/build_dataset.py b/tools/llama/build_dataset.pyindex ca75eed..08d9576 100644--- a/tools/llama/build_dataset.py+++ b/tools/llama/build_dataset.py@@ -28,17 +28,25 @@ def task_generator_yaml(config):row["group_parent_level"],)+ if isinstance(parent_level, int):+ parent_level = [parent_level]+# Load the filesfiles = list_files(root, AUDIO_EXTENSIONS, recursive=True, sort=True)grouped_files = defaultdict(list)for file in files:- if parent_level == 1:- p = file.parent.name- elif parent_level == 2:- p = file.parent.parent.name- else:- raise ValueError(f"Invalid parent level {parent_level}")+ all_parents = []+ pointer = file+ while pointer.parent.name:+ all_parents.append(pointer.parent.name)+ pointer = pointer.parent++ ps = []+ for level in parent_level:+ ps.append(all_parents[level - 1])++ p = "-".join(ps)grouped_files[p].append(file)logger.info(f"Found {len(grouped_files)} groups in {root}")