Open-source State-of-the-art TTS model which runs on a CPU 😻
Python
Audio decoder and voice cloning: less GPU memory, and AMD GPUs
haaziq-sla6936f51167f
diff --git a/kittenml/kittentts2/s3gen.py b/kittenml/kittentts2/s3gen.pyindex fbcb382..79beda5 100644--- a/kittenml/kittentts2/s3gen.py+++ b/kittenml/kittentts2/s3gen.py@@ -3914,7 +3914,7 @@ class S3Token2Mel(torch.nn.Module):TODO: make these modules configurable?"""- def __init__(self, meanflow=False):+ def __init__(self, meanflow=False, flow=True):super().__init__()self.tokenizer = S3Tokenizer("speech_tokenizer_v2_25hz")self.mel_extractor = mel_spectrogram # TODO: make it a torch module?@@ -3924,6 +3924,13 @@ class S3Token2Mel(torch.nn.Module):memory_efficient=False,)self.meanflow = meanflow+ self.resamplers = {}+ # KittenTTS: flow=False leaves out the token-to-mel flow, for a caller that+ # decodes with a replacement flow and only needs the tokenizer, speaker+ # encoder and (in S3Token2Wav) the vocoder from here.+ if not flow:+ self.flow = None+ returnencoder = UpsampleConformerEncoder(output_size=512,@@ -3967,8 +3974,6 @@ class S3Token2Mel(torch.nn.Module):decoder=decoder)- self.resamplers = {}-@propertydef device(self):params = self.tokenizer.parameters()@@ -3976,7 +3981,9 @@ class S3Token2Mel(torch.nn.Module):@propertydef dtype(self):- params = self.flow.parameters()+ # Without the flow (flow=False), the speaker encoder sets the dtype; both+ # are float32 as loaded.+ params = (self.flow if self.flow is not None else self.speaker_encoder).parameters()return next(params).dtypedef embed_ref(@@ -4101,8 +4108,8 @@ class S3Token2Wav(S3Token2Mel):ignore_state_dict_missing = ("tokenizer._mel_filters", "tokenizer.window")- def __init__(self, meanflow=False):- super().__init__(meanflow)+ def __init__(self, meanflow=False, flow=True):+ super().__init__(meanflow, flow=flow)f0_predictor = ConvRNNF0Predictor()self.mel2wav = HiFTGenerator(diff --git a/kittenml/kittentts2/speaker.py b/kittenml/kittentts2/speaker.pyindex dff8756..8f68569 100644--- a/kittenml/kittentts2/speaker.py+++ b/kittenml/kittentts2/speaker.py@@ -11,6 +11,7 @@ loaded when cloning from a new reference clip. Whisper is likewise only loadedwhen a caller clones without supplying the reference transcript."""+import loggingimport pathlibimport numpy as np@@ -22,6 +23,18 @@ from .speaker_embedding import EMBEDDING_DIM as SPEAKER_EMBEDDING_DIMfrom .speaker_embedding import SAMPLE_RATEWHISPER_MODEL = "openai/whisper-large-v3"+# Whisper large-v3 in float16 on the GPU: 2.9 GiB of weights plus working memory.+# With less GPU memory free than this, it transcribes on the CPU instead.+WHISPER_GPU_BYTES = 3584 << 20+# The model for that CPU fallback: large-v3 turbo (large-v3's encoder, 4 decoder+# layers instead of 32). Measured in float32 on 10 LibriSpeech reference clips: the+# same words as large-v3 (WER 2.4% for both), 2.7x faster (26 s instead of 72 s per+# 10-second clip on a 6-core CPU), and 3.1 GB of host memory instead of 5.9 GB. In+# a cloning run on an 8-core laptop: 12 s instead of 43 s, and a host-memory peak+# 4.1 GB above the process instead of 8.5 GB.+WHISPER_CPU_FALLBACK_MODEL = "openai/whisper-large-v3-turbo"++logger = logging.getLogger(__name__)def load_speaker_head(hidden, checkpoint_dir, device, dtype,@@ -147,15 +160,78 @@ class ReferenceTranscriber:self.model_name = model_nameself.cache_dir = cache_dirself._pipeline = None+ # Where Whisper runs once loaded: "cuda", or "cpu" (see _choose_device),+ # and which model that is.+ self.resolved_device = None+ self.resolved_model = None++ def _choose_device(self):+ """The GPU, unless it has too little free memory for Whisper; then the CPU.++ Memory this process has cached but not used counts as free, since Whisper+ can reuse it.+ """+ if self.device != "cuda":+ return "cpu"+ free, _ = torch.cuda.mem_get_info()+ free += torch.cuda.memory_reserved() - torch.cuda.memory_allocated()+ self._free_bytes = free+ return "cuda" if free >= WHISPER_GPU_BYTES else "cpu"++ def _label_cpu_fallback(self, missing=None):+ """Warn about the CPU fallback once the model is known: its cost depends on it."""+ free = getattr(self, "_free_bytes", None)+ why = (f"{free >> 20:,} MiB of GPU memory free, too little for Whisper "+ f"(about {WHISPER_GPU_BYTES >> 20:,} MiB)" if free is not None+ else "too little GPU memory free for Whisper")+ if self.resolved_model == WHISPER_CPU_FALLBACK_MODEL:+ how = ("with Whisper large-v3 turbo. This is supported but not the optimal path: it "+ "is slower (measured 12 s instead of 1.4 s on a GPU for a 10-second clip, on "+ "an 8-core laptop CPU), uses about 4 GB of host memory, and its transcript, "+ "and so the cloned voice, can differ from a GPU run, which uses large-v3")+ elif missing and self.resolved_model == WHISPER_MODEL:+ how = (f"with Whisper large-v3, because {missing} is not available (offline or not "+ "downloaded). This is supported but not the optimal path: measured 43 s per "+ "10-second clip on an 8-core laptop CPU and about 8.5 GB of host memory; "+ f"downloading {missing} makes it about 3.5x faster")+ else:+ how = (f"with {self.resolved_model}. This is supported but not the optimal path: "+ "it is slower than on a GPU")+ logger.warning("KittenTTS: %s, so the reference clip is transcribed on the CPU %s. "+ "Pass reference_text to skip transcription.", why, how)++ def _model_for(self, device):+ """The checkpoint to load: the smaller fallback model when the GPU is too full."""+ if device == "cpu" and self.device == "cuda" and self.model_name == WHISPER_MODEL:+ return WHISPER_CPU_FALLBACK_MODEL+ return self.model_namedef _load(self):if self._pipeline is None:from transformers import pipeline- self._pipeline = pipeline(- "automatic-speech-recognition", model=self.model_name,- chunk_length_s=30, device=0 if self.device == "cuda" else -1,- dtype=torch.float16 if self.device == "cuda" else torch.float32,- model_kwargs={"cache_dir": self.cache_dir} if self.cache_dir else {})+ self.resolved_device = self._choose_device()+ self.resolved_model = self._model_for(self.resolved_device)+ on_gpu = self.resolved_device == "cuda"++ def load(model):+ return pipeline(+ "automatic-speech-recognition", model=model,+ chunk_length_s=30, device=0 if on_gpu else -1,+ dtype=torch.float16 if on_gpu else torch.float32,+ model_kwargs={"cache_dir": self.cache_dir} if self.cache_dir else {})++ missing = None+ try:+ self._pipeline = load(self.resolved_model)+ except OSError:+ # The fallback model is a separate download; offline, use the one+ # that is already there.+ if self.resolved_model == self.model_name:+ raise+ missing, self.resolved_model = self.resolved_model, self.model_name+ self._pipeline = load(self.model_name)+ if self.resolved_device == "cpu" and self.device == "cuda":+ self._label_cpu_fallback(missing)return self._pipelinedef __call__(self, wav_path):diff --git a/kittenml/kittentts2/vocoder.py b/kittenml/kittentts2/vocoder.pyindex 1e22690..0ab6670 100644--- a/kittenml/kittentts2/vocoder.py+++ b/kittenml/kittentts2/vocoder.py@@ -12,6 +12,7 @@ own text-to-token model is unused, since that is precisely what KittenTTS 2replaces."""+import contextlibimport pathlibimport librosa@@ -40,6 +41,8 @@ CFM_TIMESTEPS = 2# it was trained on.STUDENT_REF_SECONDS = 4.0STUDENT_MAX_PROMPT_TOKENS = 112+# The stock flow's tensors in the s3gen checkpoint; unused when the student decodes.+STOCK_FLOW_PREFIX = "flow."class S3Codec:@@ -57,9 +60,18 @@ class S3Codec:self.device = deviceweights_path = hf_hub_download(repo_id=CHATTERBOX_REPO, filename=S3GEN_WEIGHTS,cache_dir=cache_dir)- self.s3gen = S3Gen(meanflow=True)- self.s3gen.load_state_dict(load_file(weights_path), strict=True)+ # The student replaces the whole stock flow, so with it the stock flow is+ # neither built nor loaded: about 110M parameters (437 MiB in float32) that+ # would otherwise sit unused on the GPU next to the student's.+ stock_flow = student_weights is None+ self.s3gen = S3Gen(meanflow=True, flow=stock_flow)+ state = load_file(weights_path)+ if not stock_flow:+ state = {k: v for k, v in state.items() if not k.startswith(STOCK_FLOW_PREFIX)}+ self.s3gen.load_state_dict(state, strict=True)+ del stateself.s3gen.to(device).eval()+ _mix_source_without_blas(self.s3gen, device)self._ref_cache = {}self.student = None@@ -69,7 +81,7 @@ class S3Codec:def encode_reference(self, wav_path):"""Reference clip -> S3 codec token ids, for the in-context prompt."""wav16, _ = librosa.load(str(wav_path), sr=S3_SR, mono=True)- with torch.inference_mode():+ with torch.inference_mode(), _convolutions():tokens, lengths = self.s3gen.tokenizer(torch.from_numpy(wav16).float().unsqueeze(0).to(self.device))return tokens[0, :int(lengths[0].item())].long().cpu().tolist()@@ -85,7 +97,7 @@ class S3Codec:if key not in self._ref_cache:seconds = STUDENT_REF_SECONDS if self.student is not None else DECODER_COND_SECONDSwav24 = _load_decoder_reference(str(wav_path))[:int(seconds * S3GEN_SR)]- with torch.inference_mode():+ with torch.inference_mode(), _convolutions():ref = self.s3gen.embed_ref(wav24, S3GEN_SR, device=self.device)self._ref_cache[key] = self._truncate_prompt(ref) if self.student else refreturn self._ref_cache[key]@@ -119,7 +131,7 @@ class S3Codec:# to keep the final syllable of the generation.tokens = torch.cat([tokens, torch.full((3,), int(S3GEN_SIL),dtype=torch.long, device=self.device)])- with torch.inference_mode():+ with torch.inference_mode(), _convolutions():if self.student is not None:wav = self._decode_with_student(ref_conditioning, tokens)else:@@ -145,6 +157,45 @@ class S3Codec:return wav+def _mix_source_without_blas(s3gen, device):+ """On AMD GPUs (ROCm builds of PyTorch), compute the vocoder's harmonic-source mix without a GEMM.++ The mix is an nn.Linear(9 -> 1) applied to every output sample: about 585,000 rows for a+ 24-second sentence chunk. On an RX 9070 XT (ROCm 7.2, PyTorch 2.13) the GEMM behind F.linear+ returns wrong values for inputs this long: 121,658 of 585,120 rows in an isolated test, and in+ the decoder, depending on where PyTorch placed the tensors, every sample after 2^19 (21.8 s),+ which garbled the end of long chunks (missing words). Below 2^18 rows, or as a multiply and sum+ over the 9 inputs, the result is right. Unchanged on NVIDIA GPUs and on the CPU.+ """+ if getattr(torch.version, "hip", None) and torch.device(device).type == "cuda":+ source = s3gen.mel2wav.m_source+ source.l_linear = _SmallLinear(source.l_linear)+++class _SmallLinear(torch.nn.Module):+ """nn.Linear as an elementwise multiply and sum; same parameters, so the same state dict."""++ def __init__(self, linear):+ super().__init__()+ self.weight = linear.weight+ self.bias = linear.bias++ def forward(self, x):+ y = (x.unsqueeze(-2) * self.weight).sum(-1)+ return y if self.bias is None else y + self.bias+++def _convolutions():+ """How the decoder's convolutions run: unchanged, except on AMD GPUs (ROCm builds of+ PyTorch), where they bypass MIOpen. MIOpen searches and compiles kernels for every new input+ length, and every utterance has a new length: on an RX 7900 XTX the first decode of a new+ length took about 4x as long as a repeat (real-time factor 0.9 vs 0.22), and PyTorch's own+ kernels were faster even after that (0.17-0.19)."""+ if getattr(torch.version, "hip", None):+ return torch.backends.cudnn.flags(enabled=False)+ return contextlib.nullcontext()++def _load_decoder_reference(wav_path, min_seconds=DECODER_MIN_SECONDS):"""Load a reference at 24 kHz, looping it if it is too short for s3gen.diff --git a/tests/test_small_gpu.py b/tests/test_small_gpu.pynew file mode 100644index 0000000..42f3d65--- /dev/null+++ b/tests/test_small_gpu.py@@ -0,0 +1,132 @@+"""Memory on small GPUs: the stock flow is not loaded under the student decoder, and+Whisper moves to the CPU when the GPU is short of memory. No GPU or download needed."""++from types import SimpleNamespace+from unittest import mock++import pytest+import torch+++def test_s3gen_without_its_flow_has_every_other_weight_and_keeps_its_dtype():+ from kittenml.kittentts2.s3gen import S3Gen++ with torch.device("meta"):+ full = S3Gen(meanflow=True)+ lean = S3Gen(meanflow=True, flow=False)+ assert lean.flow is None+ full_keys, lean_keys = set(full.state_dict()), set(lean.state_dict())+ assert lean_keys == {k for k in full_keys if not k.startswith("flow.")}+ assert any(k.startswith("flow.") for k in full_keys)+ assert lean.dtype == full.dtype == torch.float32+ flow_params = sum(p.numel() for p in full.flow.parameters())+ assert flow_params > 100_000_000 # the ~110M parameters the student path no longer holds++[email protected]("student", [False, True])+def test_codec_loads_the_stock_flow_only_without_a_student(student):+ from kittenml.kittentts2 import vocoder++ checkpoint = {"flow.decoder.w": torch.zeros(1), "mel2wav.w": torch.zeros(1),+ "tokenizer.w": torch.zeros(1)}+ with mock.patch.object(vocoder, "hf_hub_download", return_value="s3gen.safetensors"), \+ mock.patch.object(vocoder, "load_file", return_value=dict(checkpoint)), \+ mock.patch.object(vocoder, "S3Gen") as s3gen, \+ mock.patch.object(vocoder, "load_student_flow") as load_student:+ codec = vocoder.S3Codec(device="cpu", student_weights="student.pt" if student else None)+ s3gen.assert_called_once_with(meanflow=True, flow=not student)+ (state,), kwargs = s3gen.return_value.load_state_dict.call_args+ assert kwargs == {"strict": True}+ assert set(state) == (set(checkpoint) - {"flow.decoder.w"} if student else set(checkpoint))+ assert (codec.student is load_student.return_value) == student+++def test_whisper_runs_on_the_gpu_with_room_and_on_the_cpu_without(caplog):+ from kittenml.kittentts2.speaker import WHISPER_GPU_BYTES, ReferenceTranscriber++ assert ReferenceTranscriber(device="cpu")._choose_device() == "cpu"+ gpu = ReferenceTranscriber(device="cuda")+ with mock.patch("torch.cuda.mem_get_info", return_value=(WHISPER_GPU_BYTES, 8 << 30)), \+ mock.patch("torch.cuda.memory_reserved", return_value=0), \+ mock.patch("torch.cuda.memory_allocated", return_value=0):+ assert gpu._choose_device() == "cuda"+ # Memory this process has cached but not used counts as free.+ with mock.patch("torch.cuda.mem_get_info", return_value=(WHISPER_GPU_BYTES - (1 << 30), 0)), \+ mock.patch("torch.cuda.memory_reserved", return_value=3 << 30), \+ mock.patch("torch.cuda.memory_allocated", return_value=2 << 30):+ assert gpu._choose_device() == "cuda"+ with mock.patch("torch.cuda.mem_get_info", return_value=(2 << 30, 8 << 30)), \+ mock.patch("torch.cuda.memory_reserved", return_value=0), \+ mock.patch("torch.cuda.memory_allocated", return_value=0), \+ caplog.at_level("WARNING"):+ assert gpu._choose_device() == "cpu"+ # The cost is reported when the model is loaded, since it depends on which model that is.+ assert caplog.text == ""+++def test_whisper_pipeline_follows_the_chosen_device(caplog):+ from kittenml.kittentts2.speaker import (WHISPER_CPU_FALLBACK_MODEL, WHISPER_MODEL,+ ReferenceTranscriber)++ # (device asked for, device chosen) -> pipeline device, dtype and model. Only the+ # fallback from a full GPU uses the smaller model; the GPU path and an explicit+ # device="cpu" keep large-v3.+ for asked, chosen, index, dtype, model in (+ ("cuda", "cuda", 0, torch.float16, WHISPER_MODEL),+ ("cuda", "cpu", -1, torch.float32, WHISPER_CPU_FALLBACK_MODEL),+ ("cpu", "cpu", -1, torch.float32, WHISPER_MODEL)):+ transcriber = ReferenceTranscriber(device=asked)+ pipeline = mock.Mock()+ caplog.clear()+ with mock.patch.object(ReferenceTranscriber, "_choose_device", return_value=chosen), \+ mock.patch.dict("sys.modules", {"transformers": SimpleNamespace(pipeline=pipeline)}), \+ caplog.at_level("WARNING"):+ transcriber._load()+ kwargs = pipeline.call_args.kwargs+ assert kwargs["device"] == index and kwargs["dtype"] is dtype and kwargs["model"] == model+ assert transcriber.resolved_device == chosen and transcriber.resolved_model == model+ if asked == "cuda" and chosen == "cpu":+ # Only the fallback from a full GPU is labelled, with the turbo model's cost and the+ # note that the transcript (and so the cloned voice) can differ from a GPU run.+ for text in ("supported but not the optimal path", "large-v3 turbo",+ "4 GB of host memory", "can differ", "reference_text"):+ assert text in caplog.text+ else:+ assert caplog.text == ""+ # A configured transcriber other than the default is kept on the CPU too.+ assert ReferenceTranscriber(device="cuda", model_name="my/whisper")._model_for("cpu") == \+ "my/whisper"+++def test_whisper_fallback_model_missing_offline_uses_the_configured_one(caplog):+ from kittenml.kittentts2.speaker import WHISPER_MODEL, ReferenceTranscriber++ def pipeline(task, model, **kwargs):+ if model != WHISPER_MODEL:+ raise OSError("not in the cache and offline")+ return mock.sentinel.pipeline++ transcriber = ReferenceTranscriber(device="cuda")+ with mock.patch.object(ReferenceTranscriber, "_choose_device", return_value="cpu"), \+ mock.patch.dict("sys.modules", {"transformers": SimpleNamespace(pipeline=pipeline)}), \+ caplog.at_level("WARNING"):+ assert transcriber._load() is mock.sentinel.pipeline+ assert transcriber.resolved_model == WHISPER_MODEL and "not available" in caplog.text+ # The slower second fallback is labelled with its own cost.+ for text in ("supported but not the optimal path", "43 s", "8.5 GB",+ "openai/whisper-large-v3-turbo", "reference_text"):+ assert text in caplog.text+++def test_cloning_with_a_transcript_never_loads_whisper(tmp_path):+ from kittenml.kittentts2.model import KittenTTS2++ clip = tmp_path / "clip.wav"+ clip.write_bytes(b"")+ owner = SimpleNamespace(_reference_cache={}, transcriber=mock.Mock(), speaker_encoder=mock.Mock(),+ codec=mock.Mock(), _build_reference=mock.Mock(return_value="ref"))+ with mock.patch("kittenml.kittentts2.model.resolve_reference_path", return_value=str(clip)):+ assert KittenTTS2._clone_reference(owner, clip, "What the clip says.") == "ref"+ owner.transcriber.assert_not_called()+ KittenTTS2._clone_reference(owner, clip, None)+ owner.transcriber.assert_called_once_with(str(clip))