Open-source State-of-the-art TTS model which runs on a CPU π»
Python
"""AMD (ROCm) paths of the vLLM backend and the audio decoder; no GPU, model download or ROCm
build required."""
from types import SimpleNamespace
from unittest import mock
import pytest
import torch
from kittenml.kittentts2.vllm import rocm
from kittenml.kittentts2.vllm.w4a16 import GROUP_SIZE, pack_int4
def _amd(arch="gfx1100", built=("gfx90a", "gfx942", "gfx1100", "gfx1201")):
"""Patches that make this PyTorch look like a ROCm build on an AMD GPU."""
return [mock.patch.object(torch.version, "hip", "7.2.53211"),
mock.patch("torch.cuda.is_available", return_value=True),
mock.patch("torch.cuda.get_device_properties",
return_value=SimpleNamespace(gcnArchName=f"{arch}:sramecc-:xnack-")),
mock.patch("torch.cuda.get_arch_list", return_value=list(built)),
mock.patch("torch.cuda.get_device_capability",
side_effect=AssertionError("NVIDIA capability checked on AMD"))]
def _apply(patches):
for patch in patches:
patch.start()
def test_amd_gpus_are_checked_by_architecture_not_nvidia_capability():
from kittenml.kittentts2.vllm.backend import validate_environment
find = mock.patch("kittenml.kittentts2.vllm.backend.importlib.util.find_spec",
return_value=object())
for arch in ("gfx1100", "gfx1201"):
with find, mock.patch("kittenml.kittentts2.vllm.backend.check_torchcodec") as torchcodec:
_apply(_amd(arch))
try:
validate_environment()
# vLLM imports torchcodec on AMD too, so a broken one is refused there as well.
torchcodec.assert_called_once_with()
with pytest.raises(RuntimeError, match="'float16' is not supported on AMD GPUs"):
validate_environment(dtype="float16")
finally:
mock.patch.stopall()
with find:
_apply(_amd("gfx1030"))
try:
with pytest.raises(RuntimeError, match=r"AMD GPU \(gfx1030\) is not supported.*"
r"gfx1100, gfx1201"):
validate_environment()
finally:
mock.patch.stopall()
def test_w4a16_on_amd_disables_the_rdna3_kernel_and_keeps_the_hybrid_one():
from kittenml.kittentts2.vllm.backend import (check_w4a16_environment,
w4a16_engine_environment)
with mock.patch.object(torch.version, "hip", "7.2.53211"):
assert w4a16_engine_environment({}) == {"VLLM_DISABLED_KERNELS": "RDNA3W4A16LinearKernel"}
assert w4a16_engine_environment({"VLLM_DISABLED_KERNELS": "ExllamaLinearKernel"}) == {
"VLLM_DISABLED_KERNELS": "ExllamaLinearKernel,RDNA3W4A16LinearKernel"}
check_w4a16_environment({"VLLM_DISABLED_KERNELS": "MarlinLinearKernel"})
with pytest.raises(RuntimeError, match="RDNAHybridW4A16LinearKernel"):
check_w4a16_environment({"VLLM_DISABLED_KERNELS": "RDNAHybridW4A16LinearKernel"})
# NVIDIA GPUs are unchanged.
with mock.patch.object(torch.version, "hip", None):
assert w4a16_engine_environment({}) == {"VLLM_DISABLED_KERNELS": "MacheteLinearKernel"}
check_w4a16_environment({"VLLM_DISABLED_KERNELS": "RDNAHybridW4A16LinearKernel"})
def _kernel(name):
kernel = object.__new__(type(name, (), {}))
kernel.config = SimpleNamespace(act_type=torch.bfloat16)
module = torch.nn.Module()
module.scheme = SimpleNamespace(kernel=kernel)
return module
def test_loader_allows_only_the_hybrid_kernel_on_amd_and_asks_for_the_probe():
pytest.importorskip("vllm")
from kittenml.kittentts2.vllm.loader import _check_w4a16_kernels
bf16 = SimpleNamespace(dtype=torch.bfloat16)
model = torch.nn.Module()
model.a = _kernel("RDNAHybridW4A16LinearKernel")
with mock.patch.object(torch.version, "hip", "7.2.53211"):
assert _check_w4a16_kernels(model, bf16) is True
for other in ("RDNA3W4A16LinearKernel", "TritonW4A16LinearKernel", "MarlinLinearKernel"):
model.b = _kernel(other)
with pytest.raises(RuntimeError, match=rf"\['{other}'\] for 1 linear layers.*AMD"):
_check_w4a16_kernels(model, bf16)
with mock.patch.object(torch.version, "hip", None):
model.b = _kernel("RDNAHybridW4A16LinearKernel")
with pytest.raises(RuntimeError, match="only with MarlinLinearKernel"):
_check_w4a16_kernels(model, bf16)
# ββ The probe, with stand-in kernels on the CPU ββββββββββββββββββββββββββββββββββββββββββ
def _layer(out_features, in_features, generator):
signs = torch.randint(-1, 2, (out_features, in_features), generator=generator,
dtype=torch.int8)
scale = (torch.rand(out_features, in_features // GROUP_SIZE, generator=generator) * 0.05
+ 0.01).to(torch.bfloat16)
return pack_int4(signs), scale
def _model(compute, generator):
"""Two probed layers whose kernels compute y = compute(x, w) with w the exact weight."""
model, references = torch.nn.Module(), {}
modules = {}
for layer, in_features in zip(rocm.PROBE_LAYERS, (2048, 6144)):
packed, scale = _layer(64, in_features, generator)
references[f"{layer}.weight_packed"], references[f"{layer}.weight_scale"] = packed, scale
weight = rocm.dequantize(packed, scale)
module = torch.nn.Module()
module.register_parameter("w", torch.nn.Parameter(torch.zeros(1), requires_grad=False))
kernel = object.__new__(type("RDNAHybridW4A16LinearKernel", (), {}))
kernel.config = SimpleNamespace(act_type=torch.bfloat16)
module.scheme = SimpleNamespace(kernel=kernel,
apply_weights=lambda _, x, bias, w=weight: compute(x, w))
modules[layer] = module
model.named_modules = lambda *args, **kwargs: iter(modules.items())
return model, references
def _float32(x, w):
return (x.float() @ w.T).to(x.dtype)
def _bfloat16_split_k(x, w, shuffle=False):
"""Like vLLM's RDNA3 kernel: float32 sums over 256-wide slices, added into bfloat16
(with `shuffle`, in a different order on every call, as atomic adds can be)."""
out = torch.zeros(x.shape[0], w.shape[0], dtype=x.dtype)
starts = list(range(0, x.shape[1], 256))
if shuffle:
starts = [starts[i] for i in torch.randperm(len(starts))]
for start in starts:
part = x[:, start:start + 256].float() @ w[:, start:start + 256].T
out = (out.float() + part).to(x.dtype)
return out
def test_probe_passes_a_float32_kernel_with_the_exact_weights():
model, references = _model(_float32, torch.Generator().manual_seed(1))
report = rocm.probe_w4a16(model, references, torch.bfloat16)
assert set(report) == set(rocm.PROBE_LAYERS)
assert all(r["worst_error_vs_allowed"] <= 1 for r in report.values())
@pytest.mark.parametrize("compute, message", [
(_bfloat16_split_k, "does not sum in float32"),
# Codes read in the wrong nibble order.
(lambda x, w: _float32(x, w.reshape(w.shape[0], -1, 8).flip(-1).reshape(w.shape)),
"does not reproduce the 4-bit weights"),
# Activations rounded to 8 bits (W4A8-like).
(lambda x, w: _float32(x.float().to(torch.float8_e4m3fn).to(x.dtype), w),
"does not reproduce the 4-bit weights"),
# A different summation order on every call.
(lambda x, w: _bfloat16_split_k(x, w, shuffle=True), "different results for the same input"),
])
def test_probe_refuses_kernels_that_change_the_result(compute, message):
model, references = _model(compute, torch.Generator().manual_seed(2))
with pytest.raises(RuntimeError, match=message) as refused:
rocm.probe_w4a16(model, references, torch.bfloat16)
# Advice that also works when small-GPU fitting chose 4-bit by itself.
assert "vllm_options={'quantization': None}" in str(refused.value)
def test_references_keep_only_the_probed_tensors_and_pass_everything_through():
tensors = [(f"{layer}.{part}", torch.ones(2)) for layer in rocm.PROBE_LAYERS
for part in ("weight_packed", "weight_scale", "weight_shape")]
tensors.append(("model.layers.1.mlp.down_proj.weight_packed", torch.ones(2)))
references = {}
assert [n for n, _ in rocm.keep_references(iter(tensors), references)] == [n for n, _ in tensors]
assert sorted(references) == sorted(f"{layer}.{part}" for layer in rocm.PROBE_LAYERS
for part in ("weight_packed", "weight_scale"))
def test_decoder_convolutions_bypass_miopen_only_on_amd():
from kittenml.kittentts2.vocoder import _convolutions
before = torch.backends.cudnn.enabled
with mock.patch.object(torch.version, "hip", None):
with _convolutions():
assert torch.backends.cudnn.enabled == before
with mock.patch.object(torch.version, "hip", "7.2.53211"):
with _convolutions():
assert torch.backends.cudnn.enabled is False
assert torch.backends.cudnn.enabled == before
def test_vocoder_source_mix_avoids_the_gemm_only_on_amd_gpus():
from kittenml.kittentts2.vocoder import _SmallLinear, _mix_source_without_blas
def s3gen():
torch.manual_seed(0)
source = SimpleNamespace(l_linear=torch.nn.Linear(9, 1))
return SimpleNamespace(mel2wav=SimpleNamespace(m_source=source))
for hip, device in ((None, "cuda"), ("7.2.53211", "cpu")):
model = s3gen()
linear = model.mel2wav.m_source.l_linear
with mock.patch.object(torch.version, "hip", hip):
_mix_source_without_blas(model, device)
assert model.mel2wav.m_source.l_linear is linear # NVIDIA and CPU: unchanged
model = s3gen()
linear = model.mel2wav.m_source.l_linear
with mock.patch.object(torch.version, "hip", "7.2.53211"):
_mix_source_without_blas(model, "cuda")
mix = model.mel2wav.m_source.l_linear
assert isinstance(mix, _SmallLinear)
# Same parameters (same state dict keys and tensors), same result as the linear layer.
assert [name for name, _ in mix.named_parameters()] == ["weight", "bias"]
assert mix.weight is linear.weight and mix.bias is linear.bias
x = torch.randn(1, 4096, 9, dtype=torch.float64)
with torch.no_grad():
expected = torch.nn.functional.linear(x, linear.weight.double(), linear.bias.double())
mix.double()
torch.testing.assert_close(mix(x), expected, rtol=0, atol=1e-12)
def test_auto_fit_chooses_4bit_on_radeon_rx_7000_and_9000_only():
from kittenml.kittentts2.vllm.backend import VLLMBackend
gib = 1 << 30
lm = SimpleNamespace(hidden_size=2048, num_hidden_layers=28, num_attention_heads=16,
num_key_value_heads=8, head_dim=128, intermediate_size=6144,
vocab_size=158241, tie_word_embeddings=True)
# gfx1250 is a gfx12 name, but vLLM treats it as a data-center GPU and its RDNA kernel refuses it.
for arch, expected in (("gfx1100", "w4a16"), ("gfx1201", "w4a16"), ("gfx942", None),
("gfx1250", None)):
with mock.patch.object(torch.version, "hip", "7.2.53211"), \
mock.patch("torch.cuda.get_device_properties",
return_value=SimpleNamespace(gcnArchName=f"{arch}:sramecc-:xnack-")), \
mock.patch("kittenml.kittentts2.vllm.backend._free_gpu_memory",
return_value=(int(7.39 * gib), int(7.62 * gib))):
fitted, report = VLLMBackend._fit({}, lm)
assert report["fitted"] and fitted.get("quantization") == expected, arch
@pytest.mark.parametrize("compute, passes", [(_float32, True), (_bfloat16_split_k, False)])
def test_loader_keeps_the_probed_weights_and_checks_the_kernel_after_loading(compute, passes):
# The wiring between load_weights (keeps CPU copies of the probed layers' 4-bit tensors)
# and load_model (runs the probe on the repacked layers, then drops the copies).
pytest.importorskip("vllm")
from vllm.model_executor.model_loader.base_loader import BaseModelLoader
from kittenml.kittentts2.vllm.loader import KittenModelLoader
model, references = _model(compute, torch.Generator().manual_seed(3))
weights = list(references.items()) + [("model.norm.weight", torch.ones(4))]
def load_weights(tensors):
for _ in tensors:
pass
return set(dict(model.named_parameters()))
model.load_weights = load_weights
loader = object.__new__(KittenModelLoader)
loader.load_config = SimpleNamespace(model_loader_extra_config={
"repo_dir": "unused", "config": {}, "weights": "packed", "quantization": "w4a16"})
bf16 = SimpleNamespace(dtype=torch.bfloat16)
def base_load_model(self, vllm_config, model_config, prefix=""):
self.load_weights(model, model_config)
return model
probe = mock.Mock(wraps=rocm.probe_w4a16)
with mock.patch.object(torch.version, "hip", "7.2.53211"), \
mock.patch.object(BaseModelLoader, "load_model", base_load_model), \
mock.patch("kittenml.kittentts2.vllm.w4a16.iter_w4a16", return_value=iter(weights)), \
mock.patch.object(rocm, "probe_w4a16", probe):
if passes:
assert loader.load_model(None, bf16) is model
else:
with pytest.raises(RuntimeError, match="does not sum in float32"):
loader.load_model(None, bf16)
probe.assert_called_once()
seen = probe.call_args.args[1]
assert sorted(seen) == sorted(references)
assert all(torch.equal(seen[name], references[name]) for name in references)
assert loader._probe_references is None
def test_auto_fit_does_not_choose_4bit_on_amd_when_the_environment_refuses_it(monkeypatch):
from kittenml.kittentts2.vllm.backend import VLLMBackend
gib = 1 << 30
lm = SimpleNamespace(hidden_size=2048, num_hidden_layers=28, num_attention_heads=16,
num_key_value_heads=8, head_dim=128, intermediate_size=6144,
vocab_size=158241, tie_word_embeddings=True)
monkeypatch.setenv("VLLM_DISABLED_KERNELS", "RDNAHybridW4A16LinearKernel")
with mock.patch.object(torch.version, "hip", "7.2.53211"), \
mock.patch("torch.cuda.get_device_properties",
return_value=SimpleNamespace(gcnArchName="gfx1201")), \
mock.patch("kittenml.kittentts2.vllm.backend._free_gpu_memory",
return_value=(int(7.39 * gib), int(7.62 * gib))):
fitted, report = VLLMBackend._fit({}, lm)
assert report["fitted"] and "quantization" not in fitted