Skip to content

Open-source State-of-the-art TTS model which runs on a CPU 😻

Python

Audio decoder and voice cloning: less GPU memory, and AMD GPUs

haaziq-sla6936f51167f
diff --git a/kittenml/kittentts2/s3gen.py b/kittenml/kittentts2/s3gen.py
index fbcb382..79beda5 100644
--- a/kittenml/kittentts2/s3gen.py
+++ b/kittenml/kittentts2/s3gen.py
@@ -3914,7 +3914,7 @@ class S3Token2Mel(torch.nn.Module):
TODO: make these modules configurable?
"""
- def __init__(self, meanflow=False):
+ def __init__(self, meanflow=False, flow=True):
super().__init__()
self.tokenizer = S3Tokenizer("speech_tokenizer_v2_25hz")
self.mel_extractor = mel_spectrogram # TODO: make it a torch module?
@@ -3924,6 +3924,13 @@ class S3Token2Mel(torch.nn.Module):
memory_efficient=False,
)
self.meanflow = meanflow
+ self.resamplers = {}
+ # KittenTTS: flow=False leaves out the token-to-mel flow, for a caller that
+ # decodes with a replacement flow and only needs the tokenizer, speaker
+ # encoder and (in S3Token2Wav) the vocoder from here.
+ if not flow:
+ self.flow = None
+ return
encoder = UpsampleConformerEncoder(
output_size=512,
@@ -3967,8 +3974,6 @@ class S3Token2Mel(torch.nn.Module):
decoder=decoder
)
- self.resamplers = {}
-
@property
def device(self):
params = self.tokenizer.parameters()
@@ -3976,7 +3981,9 @@ class S3Token2Mel(torch.nn.Module):
@property
def dtype(self):
- params = self.flow.parameters()
+ # Without the flow (flow=False), the speaker encoder sets the dtype; both
+ # are float32 as loaded.
+ params = (self.flow if self.flow is not None else self.speaker_encoder).parameters()
return next(params).dtype
def embed_ref(
@@ -4101,8 +4108,8 @@ class S3Token2Wav(S3Token2Mel):
ignore_state_dict_missing = ("tokenizer._mel_filters", "tokenizer.window")
- def __init__(self, meanflow=False):
- super().__init__(meanflow)
+ def __init__(self, meanflow=False, flow=True):
+ super().__init__(meanflow, flow=flow)
f0_predictor = ConvRNNF0Predictor()
self.mel2wav = HiFTGenerator(
diff --git a/kittenml/kittentts2/speaker.py b/kittenml/kittentts2/speaker.py
index dff8756..8f68569 100644
--- a/kittenml/kittentts2/speaker.py
+++ b/kittenml/kittentts2/speaker.py
@@ -11,6 +11,7 @@ loaded when cloning from a new reference clip. Whisper is likewise only loaded
when a caller clones without supplying the reference transcript.
"""
+import logging
import pathlib
import numpy as np
@@ -22,6 +23,18 @@ from .speaker_embedding import EMBEDDING_DIM as SPEAKER_EMBEDDING_DIM
from .speaker_embedding import SAMPLE_RATE
WHISPER_MODEL = "openai/whisper-large-v3"
+# Whisper large-v3 in float16 on the GPU: 2.9 GiB of weights plus working memory.
+# With less GPU memory free than this, it transcribes on the CPU instead.
+WHISPER_GPU_BYTES = 3584 << 20
+# The model for that CPU fallback: large-v3 turbo (large-v3's encoder, 4 decoder
+# layers instead of 32). Measured in float32 on 10 LibriSpeech reference clips: the
+# same words as large-v3 (WER 2.4% for both), 2.7x faster (26 s instead of 72 s per
+# 10-second clip on a 6-core CPU), and 3.1 GB of host memory instead of 5.9 GB. In
+# a cloning run on an 8-core laptop: 12 s instead of 43 s, and a host-memory peak
+# 4.1 GB above the process instead of 8.5 GB.
+WHISPER_CPU_FALLBACK_MODEL = "openai/whisper-large-v3-turbo"
+
+logger = logging.getLogger(__name__)
def load_speaker_head(hidden, checkpoint_dir, device, dtype,
@@ -147,15 +160,78 @@ class ReferenceTranscriber:
self.model_name = model_name
self.cache_dir = cache_dir
self._pipeline = None
+ # Where Whisper runs once loaded: "cuda", or "cpu" (see _choose_device),
+ # and which model that is.
+ self.resolved_device = None
+ self.resolved_model = None
+
+ def _choose_device(self):
+ """The GPU, unless it has too little free memory for Whisper; then the CPU.
+
+ Memory this process has cached but not used counts as free, since Whisper
+ can reuse it.
+ """
+ if self.device != "cuda":
+ return "cpu"
+ free, _ = torch.cuda.mem_get_info()
+ free += torch.cuda.memory_reserved() - torch.cuda.memory_allocated()
+ self._free_bytes = free
+ return "cuda" if free >= WHISPER_GPU_BYTES else "cpu"
+
+ def _label_cpu_fallback(self, missing=None):
+ """Warn about the CPU fallback once the model is known: its cost depends on it."""
+ free = getattr(self, "_free_bytes", None)
+ why = (f"{free >> 20:,} MiB of GPU memory free, too little for Whisper "
+ f"(about {WHISPER_GPU_BYTES >> 20:,} MiB)" if free is not None
+ else "too little GPU memory free for Whisper")
+ if self.resolved_model == WHISPER_CPU_FALLBACK_MODEL:
+ how = ("with Whisper large-v3 turbo. This is supported but not the optimal path: it "
+ "is slower (measured 12 s instead of 1.4 s on a GPU for a 10-second clip, on "
+ "an 8-core laptop CPU), uses about 4 GB of host memory, and its transcript, "
+ "and so the cloned voice, can differ from a GPU run, which uses large-v3")
+ elif missing and self.resolved_model == WHISPER_MODEL:
+ how = (f"with Whisper large-v3, because {missing} is not available (offline or not "
+ "downloaded). This is supported but not the optimal path: measured 43 s per "
+ "10-second clip on an 8-core laptop CPU and about 8.5 GB of host memory; "
+ f"downloading {missing} makes it about 3.5x faster")
+ else:
+ how = (f"with {self.resolved_model}. This is supported but not the optimal path: "
+ "it is slower than on a GPU")
+ logger.warning("KittenTTS: %s, so the reference clip is transcribed on the CPU %s. "
+ "Pass reference_text to skip transcription.", why, how)
+
+ def _model_for(self, device):
+ """The checkpoint to load: the smaller fallback model when the GPU is too full."""
+ if device == "cpu" and self.device == "cuda" and self.model_name == WHISPER_MODEL:
+ return WHISPER_CPU_FALLBACK_MODEL
+ return self.model_name
def _load(self):
if self._pipeline is None:
from transformers import pipeline
- self._pipeline = pipeline(
- "automatic-speech-recognition", model=self.model_name,
- chunk_length_s=30, device=0 if self.device == "cuda" else -1,
- dtype=torch.float16 if self.device == "cuda" else torch.float32,
- model_kwargs={"cache_dir": self.cache_dir} if self.cache_dir else {})
+ self.resolved_device = self._choose_device()
+ self.resolved_model = self._model_for(self.resolved_device)
+ on_gpu = self.resolved_device == "cuda"
+
+ def load(model):
+ return pipeline(
+ "automatic-speech-recognition", model=model,
+ chunk_length_s=30, device=0 if on_gpu else -1,
+ dtype=torch.float16 if on_gpu else torch.float32,
+ model_kwargs={"cache_dir": self.cache_dir} if self.cache_dir else {})
+
+ missing = None
+ try:
+ self._pipeline = load(self.resolved_model)
+ except OSError:
+ # The fallback model is a separate download; offline, use the one
+ # that is already there.
+ if self.resolved_model == self.model_name:
+ raise
+ missing, self.resolved_model = self.resolved_model, self.model_name
+ self._pipeline = load(self.model_name)
+ if self.resolved_device == "cpu" and self.device == "cuda":
+ self._label_cpu_fallback(missing)
return self._pipeline
def __call__(self, wav_path):
diff --git a/kittenml/kittentts2/vocoder.py b/kittenml/kittentts2/vocoder.py
index 1e22690..0ab6670 100644
--- a/kittenml/kittentts2/vocoder.py
+++ b/kittenml/kittentts2/vocoder.py
@@ -12,6 +12,7 @@ own text-to-token model is unused, since that is precisely what KittenTTS 2
replaces.
"""
+import contextlib
import pathlib
import librosa
@@ -40,6 +41,8 @@ CFM_TIMESTEPS = 2
# it was trained on.
STUDENT_REF_SECONDS = 4.0
STUDENT_MAX_PROMPT_TOKENS = 112
+# The stock flow's tensors in the s3gen checkpoint; unused when the student decodes.
+STOCK_FLOW_PREFIX = "flow."
class S3Codec:
@@ -57,9 +60,18 @@ class S3Codec:
self.device = device
weights_path = hf_hub_download(repo_id=CHATTERBOX_REPO, filename=S3GEN_WEIGHTS,
cache_dir=cache_dir)
- self.s3gen = S3Gen(meanflow=True)
- self.s3gen.load_state_dict(load_file(weights_path), strict=True)
+ # The student replaces the whole stock flow, so with it the stock flow is
+ # neither built nor loaded: about 110M parameters (437 MiB in float32) that
+ # would otherwise sit unused on the GPU next to the student's.
+ stock_flow = student_weights is None
+ self.s3gen = S3Gen(meanflow=True, flow=stock_flow)
+ state = load_file(weights_path)
+ if not stock_flow:
+ state = {k: v for k, v in state.items() if not k.startswith(STOCK_FLOW_PREFIX)}
+ self.s3gen.load_state_dict(state, strict=True)
+ del state
self.s3gen.to(device).eval()
+ _mix_source_without_blas(self.s3gen, device)
self._ref_cache = {}
self.student = None
@@ -69,7 +81,7 @@ class S3Codec:
def encode_reference(self, wav_path):
"""Reference clip -> S3 codec token ids, for the in-context prompt."""
wav16, _ = librosa.load(str(wav_path), sr=S3_SR, mono=True)
- with torch.inference_mode():
+ with torch.inference_mode(), _convolutions():
tokens, lengths = self.s3gen.tokenizer(
torch.from_numpy(wav16).float().unsqueeze(0).to(self.device))
return tokens[0, :int(lengths[0].item())].long().cpu().tolist()
@@ -85,7 +97,7 @@ class S3Codec:
if key not in self._ref_cache:
seconds = STUDENT_REF_SECONDS if self.student is not None else DECODER_COND_SECONDS
wav24 = _load_decoder_reference(str(wav_path))[:int(seconds * S3GEN_SR)]
- with torch.inference_mode():
+ with torch.inference_mode(), _convolutions():
ref = self.s3gen.embed_ref(wav24, S3GEN_SR, device=self.device)
self._ref_cache[key] = self._truncate_prompt(ref) if self.student else ref
return self._ref_cache[key]
@@ -119,7 +131,7 @@ class S3Codec:
# to keep the final syllable of the generation.
tokens = torch.cat([tokens, torch.full((3,), int(S3GEN_SIL),
dtype=torch.long, device=self.device)])
- with torch.inference_mode():
+ with torch.inference_mode(), _convolutions():
if self.student is not None:
wav = self._decode_with_student(ref_conditioning, tokens)
else:
@@ -145,6 +157,45 @@ class S3Codec:
return wav
+def _mix_source_without_blas(s3gen, device):
+ """On AMD GPUs (ROCm builds of PyTorch), compute the vocoder's harmonic-source mix without a GEMM.
+
+ The mix is an nn.Linear(9 -> 1) applied to every output sample: about 585,000 rows for a
+ 24-second sentence chunk. On an RX 9070 XT (ROCm 7.2, PyTorch 2.13) the GEMM behind F.linear
+ returns wrong values for inputs this long: 121,658 of 585,120 rows in an isolated test, and in
+ the decoder, depending on where PyTorch placed the tensors, every sample after 2^19 (21.8 s),
+ which garbled the end of long chunks (missing words). Below 2^18 rows, or as a multiply and sum
+ over the 9 inputs, the result is right. Unchanged on NVIDIA GPUs and on the CPU.
+ """
+ if getattr(torch.version, "hip", None) and torch.device(device).type == "cuda":
+ source = s3gen.mel2wav.m_source
+ source.l_linear = _SmallLinear(source.l_linear)
+
+
+class _SmallLinear(torch.nn.Module):
+ """nn.Linear as an elementwise multiply and sum; same parameters, so the same state dict."""
+
+ def __init__(self, linear):
+ super().__init__()
+ self.weight = linear.weight
+ self.bias = linear.bias
+
+ def forward(self, x):
+ y = (x.unsqueeze(-2) * self.weight).sum(-1)
+ return y if self.bias is None else y + self.bias
+
+
+def _convolutions():
+ """How the decoder's convolutions run: unchanged, except on AMD GPUs (ROCm builds of
+ PyTorch), where they bypass MIOpen. MIOpen searches and compiles kernels for every new input
+ length, and every utterance has a new length: on an RX 7900 XTX the first decode of a new
+ length took about 4x as long as a repeat (real-time factor 0.9 vs 0.22), and PyTorch's own
+ kernels were faster even after that (0.17-0.19)."""
+ if getattr(torch.version, "hip", None):
+ return torch.backends.cudnn.flags(enabled=False)
+ return contextlib.nullcontext()
+
+
def _load_decoder_reference(wav_path, min_seconds=DECODER_MIN_SECONDS):
"""Load a reference at 24 kHz, looping it if it is too short for s3gen.
diff --git a/tests/test_small_gpu.py b/tests/test_small_gpu.py
new file mode 100644
index 0000000..42f3d65
--- /dev/null
+++ b/tests/test_small_gpu.py
@@ -0,0 +1,132 @@
+"""Memory on small GPUs: the stock flow is not loaded under the student decoder, and
+Whisper moves to the CPU when the GPU is short of memory. No GPU or download needed."""
+
+from types import SimpleNamespace
+from unittest import mock
+
+import pytest
+import torch
+
+
+def test_s3gen_without_its_flow_has_every_other_weight_and_keeps_its_dtype():
+ from kittenml.kittentts2.s3gen import S3Gen
+
+ with torch.device("meta"):
+ full = S3Gen(meanflow=True)
+ lean = S3Gen(meanflow=True, flow=False)
+ assert lean.flow is None
+ full_keys, lean_keys = set(full.state_dict()), set(lean.state_dict())
+ assert lean_keys == {k for k in full_keys if not k.startswith("flow.")}
+ assert any(k.startswith("flow.") for k in full_keys)
+ assert lean.dtype == full.dtype == torch.float32
+ flow_params = sum(p.numel() for p in full.flow.parameters())
+ assert flow_params > 100_000_000 # the ~110M parameters the student path no longer holds
+
+
[email protected]("student", [False, True])
+def test_codec_loads_the_stock_flow_only_without_a_student(student):
+ from kittenml.kittentts2 import vocoder
+
+ checkpoint = {"flow.decoder.w": torch.zeros(1), "mel2wav.w": torch.zeros(1),
+ "tokenizer.w": torch.zeros(1)}
+ with mock.patch.object(vocoder, "hf_hub_download", return_value="s3gen.safetensors"), \
+ mock.patch.object(vocoder, "load_file", return_value=dict(checkpoint)), \
+ mock.patch.object(vocoder, "S3Gen") as s3gen, \
+ mock.patch.object(vocoder, "load_student_flow") as load_student:
+ codec = vocoder.S3Codec(device="cpu", student_weights="student.pt" if student else None)
+ s3gen.assert_called_once_with(meanflow=True, flow=not student)
+ (state,), kwargs = s3gen.return_value.load_state_dict.call_args
+ assert kwargs == {"strict": True}
+ assert set(state) == (set(checkpoint) - {"flow.decoder.w"} if student else set(checkpoint))
+ assert (codec.student is load_student.return_value) == student
+
+
+def test_whisper_runs_on_the_gpu_with_room_and_on_the_cpu_without(caplog):
+ from kittenml.kittentts2.speaker import WHISPER_GPU_BYTES, ReferenceTranscriber
+
+ assert ReferenceTranscriber(device="cpu")._choose_device() == "cpu"
+ gpu = ReferenceTranscriber(device="cuda")
+ with mock.patch("torch.cuda.mem_get_info", return_value=(WHISPER_GPU_BYTES, 8 << 30)), \
+ mock.patch("torch.cuda.memory_reserved", return_value=0), \
+ mock.patch("torch.cuda.memory_allocated", return_value=0):
+ assert gpu._choose_device() == "cuda"
+ # Memory this process has cached but not used counts as free.
+ with mock.patch("torch.cuda.mem_get_info", return_value=(WHISPER_GPU_BYTES - (1 << 30), 0)), \
+ mock.patch("torch.cuda.memory_reserved", return_value=3 << 30), \
+ mock.patch("torch.cuda.memory_allocated", return_value=2 << 30):
+ assert gpu._choose_device() == "cuda"
+ with mock.patch("torch.cuda.mem_get_info", return_value=(2 << 30, 8 << 30)), \
+ mock.patch("torch.cuda.memory_reserved", return_value=0), \
+ mock.patch("torch.cuda.memory_allocated", return_value=0), \
+ caplog.at_level("WARNING"):
+ assert gpu._choose_device() == "cpu"
+ # The cost is reported when the model is loaded, since it depends on which model that is.
+ assert caplog.text == ""
+
+
+def test_whisper_pipeline_follows_the_chosen_device(caplog):
+ from kittenml.kittentts2.speaker import (WHISPER_CPU_FALLBACK_MODEL, WHISPER_MODEL,
+ ReferenceTranscriber)
+
+ # (device asked for, device chosen) -> pipeline device, dtype and model. Only the
+ # fallback from a full GPU uses the smaller model; the GPU path and an explicit
+ # device="cpu" keep large-v3.
+ for asked, chosen, index, dtype, model in (
+ ("cuda", "cuda", 0, torch.float16, WHISPER_MODEL),
+ ("cuda", "cpu", -1, torch.float32, WHISPER_CPU_FALLBACK_MODEL),
+ ("cpu", "cpu", -1, torch.float32, WHISPER_MODEL)):
+ transcriber = ReferenceTranscriber(device=asked)
+ pipeline = mock.Mock()
+ caplog.clear()
+ with mock.patch.object(ReferenceTranscriber, "_choose_device", return_value=chosen), \
+ mock.patch.dict("sys.modules", {"transformers": SimpleNamespace(pipeline=pipeline)}), \
+ caplog.at_level("WARNING"):
+ transcriber._load()
+ kwargs = pipeline.call_args.kwargs
+ assert kwargs["device"] == index and kwargs["dtype"] is dtype and kwargs["model"] == model
+ assert transcriber.resolved_device == chosen and transcriber.resolved_model == model
+ if asked == "cuda" and chosen == "cpu":
+ # Only the fallback from a full GPU is labelled, with the turbo model's cost and the
+ # note that the transcript (and so the cloned voice) can differ from a GPU run.
+ for text in ("supported but not the optimal path", "large-v3 turbo",
+ "4 GB of host memory", "can differ", "reference_text"):
+ assert text in caplog.text
+ else:
+ assert caplog.text == ""
+ # A configured transcriber other than the default is kept on the CPU too.
+ assert ReferenceTranscriber(device="cuda", model_name="my/whisper")._model_for("cpu") == \
+ "my/whisper"
+
+
+def test_whisper_fallback_model_missing_offline_uses_the_configured_one(caplog):
+ from kittenml.kittentts2.speaker import WHISPER_MODEL, ReferenceTranscriber
+
+ def pipeline(task, model, **kwargs):
+ if model != WHISPER_MODEL:
+ raise OSError("not in the cache and offline")
+ return mock.sentinel.pipeline
+
+ transcriber = ReferenceTranscriber(device="cuda")
+ with mock.patch.object(ReferenceTranscriber, "_choose_device", return_value="cpu"), \
+ mock.patch.dict("sys.modules", {"transformers": SimpleNamespace(pipeline=pipeline)}), \
+ caplog.at_level("WARNING"):
+ assert transcriber._load() is mock.sentinel.pipeline
+ assert transcriber.resolved_model == WHISPER_MODEL and "not available" in caplog.text
+ # The slower second fallback is labelled with its own cost.
+ for text in ("supported but not the optimal path", "43 s", "8.5 GB",
+ "openai/whisper-large-v3-turbo", "reference_text"):
+ assert text in caplog.text
+
+
+def test_cloning_with_a_transcript_never_loads_whisper(tmp_path):
+ from kittenml.kittentts2.model import KittenTTS2
+
+ clip = tmp_path / "clip.wav"
+ clip.write_bytes(b"")
+ owner = SimpleNamespace(_reference_cache={}, transcriber=mock.Mock(), speaker_encoder=mock.Mock(),
+ codec=mock.Mock(), _build_reference=mock.Mock(return_value="ref"))
+ with mock.patch("kittenml.kittentts2.model.resolve_reference_path", return_value=str(clip)):
+ assert KittenTTS2._clone_reference(owner, clip, "What the clip says.") == "ref"
+ owner.transcriber.assert_not_called()
+ KittenTTS2._clone_reference(owner, clip, None)
+ owner.transcriber.assert_called_once_with(str(clip))