SOTA Open Source TTS
Support stream reading
Lengyue712908e6738e
diff --git a/speech_lm/datasets/cultura_x.py b/speech_lm/datasets/cultura_x.pyindex 93099d4..fa564ee 100644--- a/speech_lm/datasets/cultura_x.py+++ b/speech_lm/datasets/cultura_x.py@@ -1,15 +1,17 @@-from dataclasses import dataclassimport random+from dataclasses import dataclass+from logging import getLogger+from random import Random++import numpy as npimport pandas as pd-from speech_lm.utils.braceexpand import braceexpand-from torch.utils.data import IterableDataset, get_worker_info+import pyarrow.parquet as pq+from datasets.download.streaming_download_manager import xopenfrom torch.distributed import get_rank, get_world_size, is_initialized-from random import Random-from logging import getLogger-from huggingface_hub import hf_hub_download+from torch.utils.data import IterableDataset, get_worker_infofrom transformers import AutoTokenizer-import numpy as np+from speech_lm.utils.braceexpand import braceexpandSUBSETS = {"en": "en_part_{00000..03071}",@@ -21,11 +23,12 @@ log = getLogger(__name__)class CulturaXDataset(IterableDataset):- def __init__(self, lang: str, seed: int = 42):+ def __init__(self, lang: str, seed: int = 42, parquet_batch_size: int = 10000):super().__init__()self.lang = langself.seed = seed+ self.parquet_batch_size = parquet_batch_size# Get sharded filesself.files = sorted(list(braceexpand(f"{lang}/{SUBSETS[lang]}.parquet")))@@ -68,21 +71,18 @@ class CulturaXDataset(IterableDataset):log.exception(f"Failed to parse {filename}")def parse_data(self, filename: str):- fname = hf_hub_download(- "uonlp/CulturaX",- filename,- repo_type="dataset",- )-- # Read the file- df = pd.read_parquet(fname)-- # Shuffle the data- df = df.sample(frac=1.0)-- # Yield the data- for text in df["text"]:- yield {"text": text}+ url = f"https://huggingface.co/datasets/uonlp/CulturaX/resolve/main/{filename}"++ with xopen(url, mode="rb") as stream:+ parquet_file = pq.ParquetFile(stream)++ for batch in parquet_file.iter_batches(+ batch_size=self.parquet_batch_size, columns=["text"]+ ):+ # In-batch shuffling+ texts = [{"text": text.as_py()} for text in batch["text"]]+ random.shuffle(texts)+ yield from texts@dataclass