Skip to content

SOTA Open Source TTS

Add data splitter & GPU kmeans

Lengyue7f4b9631abcf
diff --git a/tools/vqgan/calculate_kmeans_init.py b/tools/vqgan/calculate_kmeans_init.py
new file mode 100644
index 0000000..d60ab14
--- /dev/null
+++ b/tools/vqgan/calculate_kmeans_init.py
@@ -0,0 +1,100 @@
+from pathlib import Path
+
+import click
+import numpy as np
+import torch
+from einops import rearrange, repeat
+from torch.utils.data import DataLoader, Dataset
+from tqdm import tqdm
+from vector_quantize_pytorch.vector_quantize_pytorch import (
+ batched_bincount,
+ batched_sample_vectors,
+ cdist,
+)
+
+
+class KMeansDataset(Dataset):
+ def __init__(self, filelist):
+ root = Path(filelist).parent
+
+ with open(filelist) as f:
+ self.files = f.read().splitlines()
+
+ self.files = [root / file.strip() for file in self.files]
+
+ def __len__(self):
+ return len(self.files)
+
+ def __getitem__(self, idx):
+ file = self.files[idx]
+ feature = np.load(file.with_suffix(".npy"))
+ return torch.from_numpy(feature).float()
+
+ @staticmethod
+ def collate_fn(features):
+ features = torch.concat(features, dim=0)
+ return features
+
+
+ "--filelist",
+ type=click.Path(exists=True, path_type=Path),
+ default="data/test.filelist",
+)
[email protected]("--output", type=click.Path(path_type=Path), default="kmeans.pt")
[email protected]("--num-clusters", type=int, default=2048)
[email protected]("--epochs", type=int, default=10)
+def main(filelist: Path, output: Path, num_clusters: int, epochs: int):
+ loader = DataLoader(
+ KMeansDataset(filelist),
+ batch_size=1024,
+ shuffle=True,
+ num_workers=2,
+ collate_fn=KMeansDataset.collate_fn,
+ )
+
+ means = None
+ for _ in tqdm(range(epochs), desc="Epochs", position=0):
+ total_bins = torch.zeros(1, num_clusters, dtype=torch.int64, device="cuda")
+
+ for samples in tqdm(loader, desc="Batches", position=1):
+ samples = samples.cuda()[None]
+ num_codebooks, dim, dtype = (
+ samples.shape[0],
+ samples.shape[-1],
+ samples.dtype,
+ )
+
+ if means is None:
+ means = batched_sample_vectors(samples, num_clusters)
+
+ dists = -cdist(samples, means)
+
+ buckets = torch.argmax(dists, dim=-1)
+ bins = batched_bincount(buckets, minlength=num_clusters)
+
+ zero_mask = bins == 0
+ bins_min_clamped = bins.masked_fill(zero_mask, 1)
+
+ new_means = buckets.new_zeros(num_codebooks, num_clusters, dim, dtype=dtype)
+
+ new_means.scatter_add_(1, repeat(buckets, "h n -> h n d", d=dim), samples)
+ new_means = new_means / rearrange(bins_min_clamped, "... -> ... 1")
+
+ means = torch.where(rearrange(zero_mask, "... -> ... 1"), means, new_means)
+
+ total_bins += bins
+
+ torch.save(
+ {
+ "centroids": means,
+ "bins": bins,
+ },
+ output,
+ )
+ print(f"Saved to {output}")
+
+
+if __name__ == "__main__":
+ main()
diff --git a/tools/vqgan/create_train_split.py b/tools/vqgan/create_train_split.py
new file mode 100644
index 0000000..c063f86
--- /dev/null
+++ b/tools/vqgan/create_train_split.py
@@ -0,0 +1,31 @@
+from pathlib import Path
+from random import Random
+
+import click
+from tqdm import tqdm
+
+from fish_speech.utils.file import AUDIO_EXTENSIONS, list_files
+
+
[email protected]("root", type=click.Path(exists=True, path_type=Path))
+def main(root):
+ files = list_files(root, AUDIO_EXTENSIONS, recursive=True, show_progress=True)
+ print(f"Found {len(files)} files")
+
+ files = [str(file) for file in tqdm(files) if file.with_suffix(".npy").exists()]
+ print(f"Found {len(files)} files with features")
+
+ Random(42).shuffle(files)
+
+ with open(root / "vq_train_filelist.txt", "w") as f:
+ f.write("\n".join(files[:-100]))
+
+ with open(root / "vq_val_filelist.txt", "w") as f:
+ f.write("\n".join(files[-100:]))
+
+ print("Done")
+
+
+if __name__ == "__main__":
+ main()