Open-source State-of-the-art TTS model which runs on a CPU 😻
Python
"""Small checkpoints and sampler parity; no GPU or model download required."""
import json
from dataclasses import asdict
from types import SimpleNamespace
from unittest import mock
import numpy as np
import pytest
import torch
from safetensors.torch import save_file
from kittenml import KittenTTS
from kittenml.kittentts2.ternary import pack_state_dict
from kittenml.kittentts2.tokens import TokenMap
from kittenml.kittentts2.weights import load_lm_weights
def test_default_backend_keeps_native_loading(tmp_path):
from kittenml.kittentts2.model import KittenTTS2
config = {"token_map": asdict(TokenMap(5, 10, 1, 2, 3, 4, 0, 15))}
with mock.patch.object(KittenTTS2, "_load_language_model") as load, \
mock.patch("kittenml.kittentts2.model.S3Codec"), \
mock.patch("kittenml.kittentts2.model.SpeakerEncoder"), \
mock.patch("kittenml.kittentts2.model.ReferenceTranscriber"):
model = KittenTTS2(tmp_path, config, device="cpu")
load.assert_called_once_with()
assert model.backend == "torch" and model.device == "cpu"
def test_selected_packed_weights_are_exact_and_full_ignores_packed_files(tmp_path):
lm = tmp_path / "lm"
lm.mkdir()
state = {"model.layer.weight": torch.arange(256).remainder(3).reshape(2, 128)
.sub(1).to(torch.bfloat16) * 0.125,
"spk_proj.0.bias": torch.tensor([0.7, -0.2], dtype=torch.bfloat16)}
save_file(state, lm / "model.safetensors")
tensors, metadata = pack_state_dict(state)
save_file(tensors, lm / "model-ternary.safetensors", metadata=metadata)
config = {"lm_packed": "lm/model-ternary.safetensors"}
for variant in ("packed", "full"):
result = load_lm_weights(tmp_path, config, variant)
assert result.keys() == state.keys()
assert all(torch.equal(result[name], tensor) for name, tensor in state.items())
with pytest.raises(ValueError, match="Unknown weights"):
load_lm_weights(tmp_path, config, "invalid")
with pytest.raises(ValueError, match="no 'emb4'"):
load_lm_weights(tmp_path, config, "emb4")
def test_native_full_loader_preserves_explicit_checkpoint_directory(tmp_path):
from kittenml.kittentts2.model import KittenTTS2
lm_dir = tmp_path / "custom-lm"
lm_dir.mkdir()
expected = {"model.weight": torch.tensor([0.25, -0.5], dtype=torch.bfloat16)}
save_file(expected, lm_dir / "model.safetensors")
owner = SimpleNamespace(repo_dir=tmp_path, config={}, weights="full")
loaded = KittenTTS2._load_lm_weights(owner, lm_dir)
assert loaded.keys() == expected.keys()
torch.testing.assert_close(loaded["model.weight"], expected["model.weight"], rtol=0, atol=0)
def test_shared_speaker_head_matches_native_prefill_and_leaves_decode_unchanged():
from kittenml.kittentts2.speaker import attach_speaker_head, load_speaker_head
class TinyLM(torch.nn.Module):
def __init__(self):
super().__init__()
self.config = SimpleNamespace(hidden_size=4)
self.embedding = torch.nn.Embedding(8, 4)
def get_input_embeddings(self):
return self.embedding
def forward(self, input_ids=None, inputs_embeds=None, cache_position=None):
return inputs_embeds
projection = torch.nn.Sequential(torch.nn.Linear(2, 4), torch.nn.LayerNorm(4))
state = {"spk_proj." + k: v for k, v in projection.state_dict().items()}
model = attach_speaker_head(TinyLM(), "unused", "cpu", torch.float32,
spk_dim=2, state=state)
head = load_speaker_head(4, "unused", "cpu", torch.float32, spk_dim=2, state=state)
speaker = torch.tensor([[0.5, -0.3]])
model._pending_speaker_embedding = speaker
ids = torch.tensor([[0, 3, 1]])
original = model.embedding(ids)
prefill = model(input_ids=ids, cache_position=torch.tensor([0]))
torch.testing.assert_close(prefill[:, 0], head(speaker))
torch.testing.assert_close(prefill[:, 1:], original[:, 1:])
decode = model(input_ids=ids, cache_position=torch.tensor([3]))
torch.testing.assert_close(decode, original)
with pytest.raises(ValueError, match="no spk_proj"):
load_speaker_head(4, "unused", "cpu", torch.float32, spk_dim=2, state={})
def test_emb4_unpacking_is_independent_of_vllm_default_dtype():
from kittenml.kittentts2.tl2_emb4 import dequant_embedding
tensors = {
"emb4/tier_map": torch.zeros(2, dtype=torch.uint8),
"emb4/t0_codes": torch.zeros(1, dtype=torch.uint8),
"emb4/t0_centers": torch.tensor([[1., 2.], [3., 4.]], dtype=torch.float32),
"emb4/t0_s": torch.ones(2, dtype=torch.float32),
"emb4/t0_t": torch.ones(2, dtype=torch.float32),
}
metadata = {"V": "2", "D": "2", "Ks": "[2]"}
previous = torch.get_default_dtype()
try:
torch.set_default_dtype(torch.float32)
expected = dequant_embedding(tensors.__getitem__, metadata)
torch.set_default_dtype(torch.bfloat16)
actual = dequant_embedding(tensors.__getitem__, metadata)
finally:
torch.set_default_dtype(previous)
assert actual.dtype == torch.bfloat16 and torch.equal(actual, expected)
def _emb4_tensors(vocab=37, dim=16, seed=0):
"""A three-tier emb4 table (8-bit and two 4-bit books) and its full-matrix decode.
Small integer centres, power-of-two scales and dim=16 (Hadamard entries ±1/4)
keep every product and sum exact, so the expected matrix does not depend on
how a matrix multiply orders its additions.
"""
import math
generator = torch.Generator().manual_seed(seed)
sizes = [256, 16, 16]
tier_map = torch.randint(0, len(sizes), (vocab,), generator=generator).to(torch.uint8)
tensors = {"emb4/tier_map": tier_map}
hadamard = torch.tensor([[1.0]])
for _ in range(int(math.log2(dim))):
hadamard = torch.cat([torch.cat([hadamard, hadamard], 1),
torch.cat([hadamard, -hadamard], 1)], 0)
hadamard /= math.sqrt(dim)
expected = torch.empty(vocab, dim)
for tier, size in enumerate(sizes):
rows = torch.nonzero(tier_map == tier).flatten()
codes = torch.randint(0, size, (rows.numel(), dim), generator=generator)
flat = codes.flatten().to(torch.uint8)
if size == 16:
flat = flat[0::2] | (flat[1::2] << 4)
centres = torch.randint(-8, 9, (size, dim), generator=generator).to(torch.float16)
row_scale = 2.0 ** torch.randint(-3, 3, (rows.numel(),), generator=generator)
col_scale = 2.0 ** torch.randint(-3, 3, (dim,), generator=generator)
tensors.update({f"emb4/t{tier}_codes": flat, f"emb4/t{tier}_centers": centres,
f"emb4/t{tier}_s": row_scale.to(torch.float16),
f"emb4/t{tier}_t": col_scale.to(torch.float16)})
rotated = torch.gather(centres.float().T, 1, codes.T).T
expected[rows] = (rotated @ hadamard.T) * col_scale[None, :] * row_scale[:, None]
metadata = {"V": str(vocab), "D": str(dim), "Ks": json.dumps(sizes)}
return tensors, metadata, expected.to(torch.bfloat16)
@pytest.mark.parametrize("chunk, lookup_chunk", [(1, 1), (5, 2), (8, 3), (1 << 15, 1 << 12)])
def test_emb4_embedding_is_the_same_decoded_in_chunks(chunk, lookup_chunk):
from kittenml.kittentts2.tl2_emb4 import dequant_embedding
tensors, metadata, expected = _emb4_tensors()
actual = dequant_embedding(tensors.__getitem__, metadata, chunk=chunk,
lookup_chunk=lookup_chunk)
assert actual.dtype == torch.bfloat16 and torch.equal(actual, expected)
def _write_emb4_checkpoint(path):
tensors, metadata, embedding = _emb4_tensors()
# One TL2 block (32 output rows x 128 inputs): random trits, finite group scales.
tl2 = torch.randint(0, 256, (1, 1, 928), dtype=torch.uint8)
tl2[..., :64] = (torch.rand(32) + 0.5).to(torch.float16).view(torch.uint8)
tensors = {**tensors,
"tl2/model.layers.0.self_attn.o_proj.weight": tl2,
"fp/model.norm.weight": torch.rand(16, dtype=torch.bfloat16),
"fp/spk_proj.0.weight": torch.rand(16, 4, dtype=torch.bfloat16)}
path.parent.mkdir(parents=True, exist_ok=True)
save_file(tensors, path, metadata={"schema": "kitten_ternary_tl2_emb4_v1",
"embedding_metadata": json.dumps(metadata)})
return embedding
def test_emb4_reader_unpacks_only_the_tensors_asked_for(tmp_path):
from kittenml.kittentts2 import tl2_emb4
from kittenml.kittentts2.weights import iter_lm_weights
config = {"lm_emb4": "lm/model-tl2-emb4.safetensors"}
embedding = _write_emb4_checkpoint(tmp_path / config["lm_emb4"])
full = load_lm_weights(tmp_path, config, "emb4")
assert set(full) == {"model.layers.0.self_attn.o_proj.weight", "model.norm.weight",
"spk_proj.0.weight", "model.embed_tokens.weight"}
assert full["model.layers.0.self_attn.o_proj.weight"].shape == (32, 128)
assert torch.equal(full["model.embed_tokens.weight"], embedding)
# The main process asks for the speaker head only (and the embedding): nothing
# else may be unpacked.
with mock.patch.object(tl2_emb4, "unpack_tl2", side_effect=AssertionError), \
mock.patch.object(tl2_emb4, "dequant_embedding", side_effect=AssertionError):
head = load_lm_weights(tmp_path, config, "emb4",
include=lambda name: name.startswith("spk_proj."))
assert list(head) == ["spk_proj.0.weight"]
assert torch.equal(head["spk_proj.0.weight"], full["spk_proj.0.weight"])
# The engine side gets the same tensors one at a time, not a dict.
streamed = iter_lm_weights(tmp_path, config, "emb4")
assert not isinstance(streamed, dict)
streamed = dict(streamed)
assert set(streamed) == set(full)
assert all(torch.equal(streamed[name], full[name]) for name in full)
with pytest.raises(FileNotFoundError):
iter_lm_weights(tmp_path, {"lm_emb4": "lm/missing.safetensors"}, "emb4")
def test_optional_backend_errors_before_model_download():
resolved = (None, "example/repo", {"type": "KITTEN2"})
with mock.patch("kittenml.get_model.resolve_repo", return_value=resolved), \
mock.patch("kittenml.kittentts2.vllm.backend.importlib.util.find_spec", return_value=None), \
mock.patch("kittenml.get_model.snapshot_download") as download:
with pytest.raises(ImportError, match=r"kittenml\[vllm\]"):
KittenTTS("example/repo", backend="vllm")
download.assert_not_called()
with mock.patch("kittenml.get_model.resolve_repo", return_value=(None, "legacy", {"type": "ONNX1"})):
with pytest.raises(ValueError, match="KittenTTS 2"):
KittenTTS("legacy", backend="vllm")
def test_vllm_options_require_backend():
with mock.patch("kittenml.get_model.resolve_repo", return_value=(None, "model", {"type": "KITTEN2"})):
with pytest.raises(ValueError, match="requires backend"):
KittenTTS("model", vllm_options={})
def test_explicit_default_options_do_not_break_legacy_models():
resolved = (None, "legacy", {"type": "ONNX1"})
with mock.patch("kittenml.get_model.resolve_repo", return_value=resolved), \
mock.patch("kittenml.kittentts_legacy.model.build_onnx_model") as build:
model = KittenTTS("legacy", vllm_options=None)
assert model.model is build.return_value
build.assert_called_once_with(resolved[2], None, "legacy", None, None)
def test_environment_rejects_unsupported_devices_before_engine_start():
from kittenml.kittentts2.vllm.backend import validate_environment
with mock.patch("kittenml.kittentts2.vllm.backend.sys.platform", "win32"):
with pytest.raises(RuntimeError, match="Linux"):
validate_environment()
with mock.patch("kittenml.kittentts2.vllm.backend.importlib.util.find_spec", return_value=object()), \
mock.patch("torch.cuda.is_available", return_value=False):
with pytest.raises(RuntimeError, match="CUDA GPU"):
validate_environment()
with mock.patch("kittenml.kittentts2.vllm.backend.importlib.util.find_spec", return_value=object()), \
mock.patch("torch.cuda.is_available", return_value=True):
with pytest.raises(RuntimeError, match="CUDA GPU"):
validate_environment("cpu")
with pytest.raises(ValueError, match="CUDA_VISIBLE_DEVICES"):
validate_environment("cuda:1")
@pytest.mark.parametrize("key, default, override", [
("VLLM_USE_FLASHINFER_SAMPLER", "0", "1"),
("VLLM_LOGGING_LEVEL", "WARNING", "INFO"),
("TQDM_DISABLE", "1", "0"),
])
def test_engine_environment_preserves_user_choice_and_restores_after_failure(
monkeypatch, key, default, override):
import os
from kittenml.kittentts2.vllm.backend import _engine_environment
monkeypatch.delenv(key, raising=False)
with pytest.raises(RuntimeError):
with _engine_environment():
assert os.environ[key] == default
raise RuntimeError("engine failed")
assert key not in os.environ
monkeypatch.setenv(key, override)
with _engine_environment():
assert os.environ[key] == override
assert os.environ[key] == override
def test_engine_receives_quiet_logging_config_without_changing_native_options(monkeypatch, tmp_path):
pytest.importorskip("vllm")
import os
from kittenml.kittentts2.vllm.backend import VLLMBackend
monkeypatch.delenv("VLLM_LOGGING_LEVEL", raising=False)
owner = SimpleNamespace(device="cuda", repo_dir=tmp_path, config={}, weights="packed",
token_map=None, _load_lm_weights=mock.Mock(return_value={
"model.embed_tokens.weight": torch.zeros(2, 4)}))
def create_engine(**kwargs):
assert kwargs["logging_config"].log_level == "WARNING"
assert os.environ["VLLM_LOGGING_LEVEL"] == "WARNING"
assert os.environ["TQDM_DISABLE"] == "1"
assert kwargs["enable_prompt_embeds"] and not kwargs["enable_prefix_caching"]
assert kwargs["model_loader_extra_config"]["weights"] == "packed"
return mock.sentinel.engine
with mock.patch("kittenml.kittentts2.vllm.backend.validate_environment"), \
mock.patch("torch.cuda.get_device_capability", return_value=(8, 6)), \
mock.patch("transformers.AutoConfig.from_pretrained", return_value=SimpleNamespace(
model_type="qwen3", hidden_size=4)), \
mock.patch("transformers.AutoTokenizer.from_pretrained"), \
mock.patch("kittenml.kittentts2.speaker.load_speaker_head"), \
mock.patch("vllm.LLM", side_effect=create_engine):
backend = VLLMBackend(owner)
assert backend.engine is mock.sentinel.engine
assert "VLLM_LOGGING_LEVEL" not in os.environ
def test_vllm_loader_filters_speaker_weights_and_rejects_missing_parameters():
pytest.importorskip("vllm")
from kittenml.kittentts2.vllm.loader import KittenModelLoader
loader = object.__new__(KittenModelLoader)
loader.load_config = SimpleNamespace(model_loader_extra_config={
"repo_dir": "unused", "config": {}, "weights": "packed"})
class TinyModel:
def load_weights(self, weights):
self.loaded = dict(weights)
return set(self.loaded)
def named_parameters(self):
return [("model.weight", None)]
def named_modules(self):
return []
model = TinyModel()
state = {"model.weight": torch.ones(2), "spk_proj.0.bias": torch.zeros(2)}
with mock.patch("kittenml.kittentts2.vllm.loader.iter_lm_weights", return_value=state.items()):
loader.load_weights(model, None)
assert set(model.loaded) == {"model.weight"}
with mock.patch("kittenml.kittentts2.vllm.loader.iter_lm_weights", return_value={}.items()):
with pytest.raises(ValueError, match="missing from the checkpoint"):
loader.load_weights(model, None)
@pytest.mark.parametrize("window", [None, 3])
def test_sampling_matches_native_when_requests_reorder_and_slots_are_reused(window):
pytest.importorskip("vllm")
from kittenml.kittentts2.logits import build_logits_processors
from kittenml.kittentts2.vllm.logits import KittenLogitsProcessor
tm = TokenMap(audio_id_base=5, num_audio_tokens=5000, speech_start_id=1,
speech_end_id=2, text_start_id=3, text_end_id=4, start_id=0, stop_id=5005)
sampling = dict(temperature=0.8, top_k=60, top_p=0.9, min_p=0.05,
repetition_penalty=1.1, repetition_window=window,
token_run_penalty=1.3, token_run_grace=3)
original = {0: [0, 3, 7, 1], 1: [0, 3, 8, 9, 1]}
history = {0: original[0] + [7, 7, 7, 7],
1: original[1] + [tm.audio_id_base + 4299] * 4}
buf = torch.zeros(2, 16, dtype=torch.long)
# Embedding prompts may have placeholder IDs inside vLLM. The processor must
# restore the real prompt before applying full-history repetition penalties.
for slot in history:
buf[slot, len(original[slot]):len(history[slot])] = torch.tensor(history[slot][len(original[slot]):])
states = SimpleNamespace(device=torch.device("cpu"), all_token_ids=SimpleNamespace(gpu=buf))
processor = KittenLogitsProcessor(None, states)
for slot in original:
params = SimpleNamespace(extra_args={"kittenml": {"prompt": original[slot],
"token_map": asdict(tm), "sampling": sampling}})
assert processor.add_request(slot, params)
ctx = SimpleNamespace(idx_mapping_np=np.array([1, 0]),
seq_lens_upper_bound_np=np.array([len(history[1]), len(history[0])]))
scores = torch.randn(2, 5006, generator=torch.Generator().manual_seed(7))
expected = scores.clone()
for row, slot in enumerate([1, 0]):
native = build_logits_processors(tm, **sampling, prompt_length=len(original[slot]))
expected[row:row + 1] = native(torch.tensor([history[slot]]), expected[row:row + 1])
torch.testing.assert_close(processor.apply(scores, ctx), expected)
assert not processor.add_request(0, SimpleNamespace(extra_args=None))
assert 0 not in processor.requests
def test_sampler_is_neutral_and_speaker_replaces_only_first_embedding():
pytest.importorskip("vllm")
from kittenml.kittentts2.vllm.backend import VLLMBackend
backend = object.__new__(VLLMBackend)
backend.max_model_len = 32
backend.token_map = TokenMap(5, 10, 1, 2, 3, 4, 0, 15)
backend.embedding = torch.arange(64, dtype=torch.float32).reshape(16, 4)
backend.spk_proj = torch.nn.Identity()
settings = dict(temperature=0.7, top_k=5, top_p=0.8, min_p=0.02)
advanced = dict(seed=42, repetition_penalty=1.2, repetition_window=3,
token_run_penalty=1.5, token_run_grace=4)
request, params = backend.prepare_request([0, 3, 1], torch.ones(1, 4), settings, 8, advanced)
torch.testing.assert_close(request["prompt_embeds"][0], torch.ones(4))
torch.testing.assert_close(request["prompt_embeds"][1:], backend.embedding[[3, 1]])
assert params.temperature == params.top_p == params.repetition_penalty == 1.0
assert params.top_k == -1 and params.min_p == 0.0 and params.seed == 42
assert params.stop_token_ids == [2, 15] and params.ignore_eos
assert params.extra_args["kittenml"]["sampling"]["temperature"] == 0.7
with pytest.raises(ValueError, match="token budget"):
backend.prepare_request([0, 3, 1], torch.ones(1, 4), settings, 30, advanced)
# ── Exact W4A16 (vllm_options={"quantization": "w4a16"}) ───────────────────────
_PROJECTIONS = {"self_attn.q_proj": (256, 256), "self_attn.k_proj": (128, 256),
"self_attn.v_proj": (128, 256), "self_attn.o_proj": (256, 256),
"mlp.gate_proj": (384, 256), "mlp.up_proj": (384, 256),
"mlp.down_proj": (256, 384)}
def _ternary(rows, cols, generator, low=1e-4):
signs = torch.randint(-1, 2, (rows, cols), generator=generator)
scale = (torch.rand(rows, cols // 128, generator=generator) * 0.5 + low).to(torch.bfloat16)
scale[0, 0] = 0 # an all-zero group has a zero scale
signs[0, :128] = 0
return signs.to(torch.bfloat16) * scale.repeat_interleave(128, dim=1)
def _w4a16_checkpoint(tmp_path, layers=2, perturb=None, low=1e-4):
"""A small Qwen3-shaped LM written as both full bf16 and packed ternary."""
generator = torch.Generator().manual_seed(0)
state = {"model.embed_tokens.weight": torch.randn(32, 256, generator=generator).to(torch.bfloat16),
"model.norm.weight": torch.rand(256, generator=generator).to(torch.bfloat16),
"spk_proj.0.weight": torch.randn(256, 8, generator=generator).to(torch.bfloat16),
"spk_proj.0.bias": torch.zeros(256, dtype=torch.bfloat16),
"spk_proj.1.weight": torch.ones(256, dtype=torch.bfloat16),
"spk_proj.1.bias": torch.zeros(256, dtype=torch.bfloat16)}
for layer in range(layers):
prefix = f"model.layers.{layer}."
state[prefix + "input_layernorm.weight"] = torch.ones(256, dtype=torch.bfloat16)
state[prefix + "self_attn.q_norm.weight"] = torch.ones(128, dtype=torch.bfloat16)
for name, shape in _PROJECTIONS.items():
state[prefix + name + ".weight"] = _ternary(*shape, generator, low)
if perturb is not None:
weight = state[perturb]
weight[1, 5] = weight[1, :128].abs().max() / 2 # no longer ternary in that group
lm = tmp_path / "lm"
lm.mkdir()
save_file(state, lm / "model.safetensors")
tensors, metadata = pack_state_dict(state)
save_file(tensors, lm / "model-ternary.safetensors", metadata=metadata)
return state, {"lm_packed": "lm/model-ternary.safetensors"}
def _unpack_int4(packed, shape):
"""Inverse of w4a16.pack_int4: int32 words -> int8 values of `shape`."""
from kittenml.kittentts2.vllm.w4a16 import NUM_BITS, OFFSET, PACK_FACTOR
words = packed.to(torch.int64) & 0xFFFFFFFF
shifts = torch.arange(PACK_FACTOR, dtype=torch.int64) * NUM_BITS
codes = (words.unsqueeze(2) >> shifts) & ((1 << NUM_BITS) - 1)
return (codes - OFFSET).reshape(packed.shape[0], -1)[:, :int(shape[1])].to(torch.int8)
def _reference_unpack():
"""compressed-tensors' own unpacker when installed (it ships with vLLM)."""
try:
from compressed_tensors.compressors.pack_quantized.helpers import unpack_from_int32
except ImportError:
return _unpack_int4
return lambda packed, shape: unpack_from_int32(packed, 4, torch.Size(shape))
@pytest.mark.parametrize("variant", ["packed", "full"])
def test_w4a16_dequantizes_bit_exactly_to_the_selected_weights(tmp_path, variant):
from kittenml.kittentts2.vllm.w4a16 import bit_equal, is_projection, iter_w4a16
state, config = _w4a16_checkpoint(tmp_path)
expected = load_lm_weights(tmp_path, config, variant)
produced = dict(iter_w4a16(tmp_path, config, variant))
unpack = _reference_unpack()
projections = [name for name in state if is_projection(name)]
assert len(projections) == 14
dense = {name for name in state if not is_projection(name) and not name.startswith("spk_proj.")}
assert set(produced) == dense | {name.removesuffix(".weight") + suffix
for name in projections
for suffix in (".weight_packed", ".weight_scale",
".weight_shape")}
for name in dense:
assert bit_equal(produced[name], expected[name])
for name in projections:
base = name.removesuffix(".weight")
rows, cols = expected[name].shape
packed = produced[base + ".weight_packed"]
scale = produced[base + ".weight_scale"]
shape = produced[base + ".weight_shape"]
assert packed.dtype == torch.int32 and packed.shape == (rows, cols // 8)
assert scale.dtype == torch.bfloat16 and scale.shape == (rows, cols // 128)
assert shape.dtype == torch.int64 and shape.tolist() == [rows, cols]
signs = unpack(packed, shape.tolist())
assert set((signs.to(torch.int16) + 8).unique().tolist()) <= {7, 8, 9}
weight = signs.to(torch.bfloat16) * scale.repeat_interleave(128, dim=1)
assert bit_equal(weight, expected[name]), name
def test_w4a16_packed_and_full_sources_give_identical_tensors(tmp_path):
from kittenml.kittentts2.vllm.w4a16 import bit_equal, iter_w4a16
_, config = _w4a16_checkpoint(tmp_path)
packed = dict(iter_w4a16(tmp_path, config, "packed"))
full = dict(iter_w4a16(tmp_path, config, "full"))
assert packed.keys() == full.keys()
assert all(bit_equal(packed[k], full[k]) for k in packed)
def test_w4a16_int32_bit_order_matches_compressed_tensors():
from kittenml.kittentts2.vllm.w4a16 import pack_int4
# Codes 7,8,9,8,8,8,8,9: column 0 in bits 0-3 ... column 7 in bits 28-31.
golden = pack_int4(torch.tensor([[-1, 0, 1, 0, 0, 0, 0, 1]], dtype=torch.int8))
assert golden.dtype == torch.int32 and golden.tolist() == [[-1735882361]] # 0x98888987
assert pack_int4(torch.zeros(1, 8, dtype=torch.int8)).tolist() == [[0x88888888 - (1 << 32)]]
values = torch.randint(-8, 8, (6, 256), generator=torch.Generator().manual_seed(1)).to(torch.int8)
assert torch.equal(_unpack_int4(pack_int4(values), values.shape), values)
with pytest.raises(ValueError, match="4 bits"):
pack_int4(torch.full((1, 8), 8, dtype=torch.int8))
helpers = pytest.importorskip("compressed_tensors.compressors.pack_quantized.helpers")
assert torch.equal(pack_int4(values), helpers.pack_to_int32(values, 4))
assert torch.equal(helpers.unpack_from_int32(pack_int4(values), 4, values.shape), values)
@pytest.mark.parametrize("variant", ["packed", "full"])
def test_w4a16_refuses_weights_that_are_not_exactly_ternary(tmp_path, variant):
from kittenml.kittentts2.vllm.w4a16 import iter_w4a16
_, config = _w4a16_checkpoint(tmp_path, perturb="model.layers.1.mlp.up_proj.weight")
with pytest.raises(ValueError, match="model.layers.1.mlp.up_proj.weight is not exactly ternary"):
list(iter_w4a16(tmp_path, config, variant))
def test_w4a16_refuses_scales_that_are_not_bfloat16(tmp_path):
# Marlin reads the group scales in the activation dtype, which is bfloat16.
from kittenml.kittentts2.vllm.w4a16 import iter_w4a16
weight = torch.zeros(2, 128, dtype=torch.float32)
weight[0, :2] = torch.tensor([0.5, -0.5])
(tmp_path / "lm").mkdir()
save_file({"model.layers.0.self_attn.o_proj.weight": weight}, tmp_path / "lm" / "model.safetensors")
with pytest.raises(ValueError, match="o_proj.weight: group scales are torch.float32, not bfloat16"):
list(iter_w4a16(tmp_path, {}, "full"))
def test_w4a16_accepts_emb4_and_packs_its_ternary_body_exactly(tmp_path):
from kittenml.kittentts2.vllm import w4a16
from kittenml.kittentts2.vllm.backend import check_options
signs = torch.randint(-1, 2, (16, 256), dtype=torch.int8)
scale = torch.rand(16, 2, dtype=torch.bfloat16) + 0.5
weight = (signs.reshape(16, 2, 128).to(torch.float32)
* scale.to(torch.float32).unsqueeze(-1)).reshape(16, 256).to(torch.bfloat16)
embedding = torch.randn(8, 4, dtype=torch.bfloat16)
state = {"model.layers.0.self_attn.o_proj.weight": weight,
"model.embed_tokens.weight": embedding,
"spk_proj.weight": torch.zeros(2, 2, dtype=torch.bfloat16)}
with mock.patch("kittenml.kittentts2.weights.iter_lm_weights",
return_value=iter(state.items())) as load:
out = dict(w4a16.iter_w4a16(tmp_path, {"lm_emb4": "x"}, "emb4"))
load.assert_called_once_with(tmp_path, {"lm_emb4": "x"}, "emb4")
assert "spk_proj.weight" not in out
assert torch.equal(out["model.embed_tokens.weight"], embedding)
expected = dict(w4a16._from_dense("model.layers.0.self_attn.o_proj.weight", weight))
for key, value in expected.items():
assert torch.equal(out[key], value)
check_options({"quantization": "w4a16"}, "emb4")
with pytest.raises(ValueError, match="float16"):
check_options({"dtype": "float16"}, "emb4")
def test_w4a16_targets_112_fused_modules_for_196_projections():
# Qwen3-1.7B shape: 28 layers x 7 ternary projections. vLLM's own Qwen3 model
# loads q/k/v into qkv_proj and gate/up into gate_up_proj (its
# packed_modules_mapping), so 4 quantized modules per layer.
pytest.importorskip("vllm")
from vllm.model_executor.models.qwen3 import Qwen3ForCausalLM
from kittenml.kittentts2.vllm.w4a16 import is_projection
fused = {part: module for module, parts in Qwen3ForCausalLM.packed_modules_mapping.items()
for part in parts}
names = ["model.embed_tokens.weight", "model.norm.weight", "spk_proj.0.weight"]
for layer in range(28):
names += [f"model.layers.{layer}.{p}.weight" for p in _PROJECTIONS]
names += [f"model.layers.{layer}.input_layernorm.weight",
f"model.layers.{layer}.self_attn.k_norm.weight"]
projections = [name.removesuffix(".weight") for name in names if is_projection(name)]
assert len(projections) == 196
modules = {name.rsplit(".", 1)[0] + "." + fused.get(name.rsplit(".", 1)[1], name.rsplit(".", 1)[1])
for name in projections}
assert len(modules) == 112
assert {m.rsplit(".", 1)[1] for m in modules} == {"qkv_proj", "o_proj", "gate_up_proj",
"down_proj"}
def test_vllm_options_validate_quantization_dtype_and_cache_size():
from kittenml.kittentts2.vllm.backend import resolve_options
engine, kitten = resolve_options(None)
assert engine == dict(gpu_memory_utilization=0.5, max_model_len=4096, max_num_seqs=8,
enforce_eager=False)
assert kitten == dict(quantization=None, dtype=None, kv_cache_memory_bytes=None,
max_num_batched_tokens=None)
engine, kitten = resolve_options({"quantization": "w4a16", "dtype": "bfloat16",
"kv_cache_memory_bytes": 1 << 30, "max_num_seqs": 4})
assert engine["kv_cache_memory_bytes"] == 1 << 30 and engine["max_num_seqs"] == 4
assert "quantization" not in engine and "dtype" not in engine
assert kitten["quantization"] == "w4a16" and kitten["dtype"] == "bfloat16"
engine, _ = resolve_options({"max_num_batched_tokens": 2048})
assert engine["max_num_batched_tokens"] == 2048
assert "max_num_batched_tokens" not in resolve_options({"max_num_batched_tokens": None})[0]
for bad, match in [({"quantization": "w8a8"}, "quantization"),
({"quantization": "W4A16"}, "quantization"),
({"dtype": "fp16"}, "dtype"),
({"dtype": "float64"}, "dtype"),
({"kv_cache_memory_bytes": 0}, "kv_cache_memory_bytes"),
({"kv_cache_memory_bytes": 1.5e9}, "kv_cache_memory_bytes"),
({"kv_cache_memory_bytes": True}, "kv_cache_memory_bytes"),
({"max_num_batched_tokens": 0}, "max_num_batched_tokens"),
({"max_num_batched_tokens": 2048.0}, "max_num_batched_tokens"),
({"kv_cache_dtype": "fp8"}, "unknown vllm_options")]:
with pytest.raises(ValueError, match=match):
resolve_options(bad)
def test_float16_is_an_explicit_option():
# Explicit dtype overrides are kept separate from vLLM's engine options.
from kittenml.kittentts2.vllm.backend import resolve_options
for options in ({"dtype": "float16"}, {"dtype": "float16", "quantization": "w4a16"}):
engine, kitten = resolve_options(options)
assert kitten["dtype"] == "float16" and "dtype" not in engine
assert resolve_options(None)[1]["dtype"] is None
assert resolve_options({"dtype": "float32"})[1]["dtype"] == "float32"
def test_gpu_capability_and_explicit_dtype_are_checked_before_download():
from kittenml.kittentts2.vllm.backend import validate_environment
with mock.patch("kittenml.kittentts2.vllm.backend.importlib.util.find_spec", return_value=object()), \
mock.patch("torch.cuda.is_available", return_value=True), \
mock.patch("torch.cuda.get_arch_list", return_value=["sm_70", "sm_75", "sm_80"]):
for capability in ((8, 0), (8, 6), (12, 0)):
with mock.patch("torch.cuda.get_device_capability", return_value=capability):
for dtype in (None, "bfloat16", "float16", "float32"):
assert validate_environment(dtype=dtype) == capability
for capability in ((7, 5), (7, 0)):
with mock.patch("torch.cuda.get_device_capability", return_value=capability):
for dtype in (None, "float16", "float32"):
assert validate_environment(dtype=dtype) == capability
with pytest.raises(ValueError, match="no native bfloat16"):
validate_environment(dtype="bfloat16")
with mock.patch("torch.cuda.get_device_capability", return_value=(6, 1)):
for dtype in (None, "float16", "float32"):
with pytest.raises(RuntimeError, match="SM61.*default backend"):
validate_environment(dtype=dtype)
def test_v100_requires_a_pytorch_build_that_can_execute_sm70():
from kittenml.kittentts2.vllm.backend import validate_environment
with mock.patch("kittenml.kittentts2.vllm.backend.importlib.util.find_spec", return_value=object()), \
mock.patch("torch.cuda.is_available", return_value=True), \
mock.patch("torch.cuda.get_device_capability", return_value=(7, 0)):
with mock.patch("torch.cuda.get_arch_list", return_value=["sm_75", "sm_80", "compute_80"]):
with pytest.raises(RuntimeError, match="does not support SM70.*CUDA 12.6"):
validate_environment()
with mock.patch("torch.cuda.get_arch_list", return_value=["sm_75", "compute_70"]):
assert validate_environment() == (7, 0)
def test_legacy_gpu_options_choose_the_measured_mode_and_respect_overrides(monkeypatch):
from kittenml.kittentts2.vllm.backend import options_for_gpu
for capability in ((8, 0), (8, 6), (12, 0), None):
for given in (None, {}, {"quantization": None}, {"dtype": "bfloat16"}):
assert options_for_gpu(given, capability) is given
assert options_for_gpu(None, (7, 5)) == {"dtype": "float16", "quantization": "w4a16"}
assert options_for_gpu(None, (7, 0)) == {"dtype": "float16"}
assert options_for_gpu({"quantization": None}, (7, 5)) == {"dtype": "float16", "quantization": None}
assert options_for_gpu({"dtype": "float32"}, (7, 5)) == {"dtype": "float32"}
with pytest.raises(ValueError, match="W4A16 requires compute capability 7.5"):
options_for_gpu({"quantization": "w4a16"}, (7, 0))
monkeypatch.setenv("VLLM_MARLIN_INPUT_DTYPE", "int8")
assert options_for_gpu(None, (7, 5)) == {"dtype": "float16"}
def test_vllm_options_are_checked_before_model_download(monkeypatch):
resolved = (None, "example/repo", {"type": "KITTEN2"})
with mock.patch("kittenml.get_model.resolve_repo", return_value=resolved), \
mock.patch("kittenml.kittentts2.vllm.backend.validate_environment"), \
mock.patch("kittenml.get_model.snapshot_download") as download:
for options, weights, match in [({"quantisation": "w4a16"}, None, "unknown vllm_options"),
({"dtype": "float16"}, "emb4", "float16.*emb4"),
({"dtype": "fp16"}, None, "'float16'")]:
with pytest.raises(ValueError, match=match):
KittenTTS("example/repo", backend="vllm", weights=weights, vllm_options=options)
monkeypatch.setenv("VLLM_MARLIN_INPUT_DTYPE", "int8")
with pytest.raises(RuntimeError, match="W4A8"):
KittenTTS("example/repo", backend="vllm", vllm_options={"quantization": "w4a16"})
download.assert_not_called()
def test_old_gpu_and_automatic_float16_emb4_errors_precede_model_download():
resolved = (None, "example/repo", {"type": "KITTEN2"})
with mock.patch("kittenml.get_model.resolve_repo", return_value=resolved), \
mock.patch("kittenml.kittentts2.vllm.backend.importlib.util.find_spec", return_value=object()), \
mock.patch("torch.cuda.is_available", return_value=True), \
mock.patch("kittenml.get_model.snapshot_download") as download:
with mock.patch("torch.cuda.get_device_capability", return_value=(6, 1)):
with pytest.raises(RuntimeError, match="SM61.*default backend"):
KittenTTS("example/repo", backend="vllm")
with mock.patch("torch.cuda.get_device_capability", return_value=(7, 5)):
with pytest.raises(ValueError, match="float16.*emb4"):
KittenTTS("example/repo", backend="vllm", weights="emb4")
download.assert_not_called()
def test_w4a16_refuses_marlin_environment_overrides(monkeypatch, tmp_path):
from kittenml.kittentts2.vllm.backend import VLLMBackend, check_w4a16_environment
check_w4a16_environment({})
# vLLM reads only VLLM_MARLIN_USE_ATOMIC_ADD=1 as on; other kernels may be disabled.
check_w4a16_environment({"VLLM_MARLIN_USE_ATOMIC_ADD": "0",
"VLLM_DISABLED_KERNELS": "ExllamaLinearKernel"})
for key, value in (("VLLM_MARLIN_INPUT_DTYPE", "int8"), ("VLLM_MARLIN_INPUT_DTYPE", "fp8"),
("VLLM_MARLIN_USE_ATOMIC_ADD", "1"),
("VLLM_DISABLED_KERNELS", "ConchLinearKernel,MarlinLinearKernel")):
with pytest.raises(RuntimeError, match=key):
check_w4a16_environment({key: value})
monkeypatch.setenv("VLLM_MARLIN_INPUT_DTYPE", "int8")
owner = SimpleNamespace(device="cuda", repo_dir=tmp_path, config={}, weights="packed",
_load_lm_weights=mock.Mock())
with mock.patch("kittenml.kittentts2.vllm.backend.validate_environment"):
with pytest.raises(RuntimeError, match="W4A8"):
VLLMBackend(owner, {"quantization": "w4a16"})
owner._load_lm_weights.assert_not_called()
def test_w4a16_hf_overrides_payload():
from kittenml.kittentts2.vllm.backend import quantization_args
from kittenml.kittentts2.vllm.w4a16 import QUANTIZATION_CONFIG
assert quantization_args(None) == {}
args = quantization_args("w4a16")
assert args["quantization"] == "compressed-tensors"
payload = args["hf_overrides"]["quantization_config"]
assert payload == {
"quant_method": "compressed-tensors", "format": "pack-quantized",
"config_groups": {"group_0": {
"targets": ["Linear"],
"weights": {"num_bits": 4, "type": "int", "symmetric": True, "strategy": "group",
"group_size": 128, "dynamic": False, "actorder": None},
"input_activations": None}},
"ignore": ["lm_head"]}
payload["total_num_heads"] = 16 # vLLM writes into the dict it receives
assert "total_num_heads" not in QUANTIZATION_CONFIG
def test_w4a16_payload_selects_vllm_uint4b8_weight_only_scheme():
pytest.importorskip("vllm")
from vllm.model_executor.layers.quantization.compressed_tensors.compressed_tensors import (
CompressedTensorsConfig)
from vllm.model_executor.layers.quantization.compressed_tensors.schemes.compressed_tensors_wNa16 import (
WNA16_SUPPORTED_TYPES_MAP)
from kittenml.kittentts2.vllm.w4a16 import OFFSET, quantization_config
config = CompressedTensorsConfig.from_config(quantization_config())
assert config.ignore == ["lm_head"] and config.quant_format == "pack-quantized"
scheme = config.target_scheme_map["Linear"]
assert scheme["input_activations"] is None
assert CompressedTensorsConfig._is_wNa16_group_channel(scheme["weights"], None)
weights = scheme["weights"]
assert (weights.num_bits, weights.group_size, weights.symmetric) == (4, 128, True)
assert WNA16_SUPPORTED_TYPES_MAP[4].bias == OFFSET == 8 # uint4b8: code = sign + 8
def test_engine_receives_w4a16_arguments_and_main_process_loads_only_conditioning(tmp_path):
pytest.importorskip("vllm")
from kittenml.kittentts2.vllm.backend import VLLMBackend
import os
seen = []
def create_engine(**kwargs):
seen.append(dict(kwargs, disabled_kernels=os.environ.get("VLLM_DISABLED_KERNELS")))
def run(options):
owner = SimpleNamespace(device="cuda", repo_dir=tmp_path, config={}, weights="packed",
token_map=None, _load_lm_weights=mock.Mock(return_value={
"model.embed_tokens.weight": torch.zeros(2, 4)}))
with mock.patch("kittenml.kittentts2.vllm.backend.validate_environment"), \
mock.patch("torch.cuda.get_device_capability", return_value=(8, 6)), \
mock.patch("transformers.AutoConfig.from_pretrained", return_value=SimpleNamespace(
model_type="qwen3", hidden_size=4)), \
mock.patch("transformers.AutoTokenizer.from_pretrained"), \
mock.patch("kittenml.kittentts2.speaker.load_speaker_head"), \
mock.patch("vllm.LLM", side_effect=create_engine):
VLLMBackend(owner, options)
include = owner._load_lm_weights.call_args.kwargs["include"]
assert include("model.embed_tokens.weight") and include("spk_proj.0.weight")
assert not include("model.layers.0.self_attn.q_proj.weight")
return owner, seen[-1]
os.environ.pop("VLLM_DISABLED_KERNELS", None)
owner, kwargs = run(None)
assert {"quantization", "hf_overrides", "kv_cache_memory_bytes"}.isdisjoint(kwargs)
assert set(kwargs["model_loader_extra_config"]) == {"repo_dir", "config", "weights"}
assert kwargs["dtype"] is torch.bfloat16 and owner.dtype is torch.bfloat16
assert kwargs["disabled_kernels"] is None
owner, kwargs = run({"quantization": "w4a16"})
# Machete, which vLLM prefers on Hopper, is disabled for the engine only.
assert kwargs["disabled_kernels"] == "MacheteLinearKernel"
assert "VLLM_DISABLED_KERNELS" not in os.environ
owner, kwargs = run({"quantization": "w4a16", "kv_cache_memory_bytes": 1 << 30,
"dtype": "bfloat16"})
assert kwargs["quantization"] == "compressed-tensors"
# bfloat16 W4A16: only the quantization config, the head stays tied.
assert set(kwargs["hf_overrides"]) == {"quantization_config"}
assert kwargs["hf_overrides"]["quantization_config"]["ignore"] == ["lm_head"]
assert kwargs["model_loader_extra_config"]["quantization"] == "w4a16"
assert "float16_rescale" not in kwargs["model_loader_extra_config"]
assert kwargs["kv_cache_memory_bytes"] == 1 << 30
assert kwargs["dtype"] is torch.bfloat16 and owner.dtype is torch.bfloat16
def _linear_with_kernel(kind, act_type=torch.bfloat16):
"""A module carrying a compressed-tensors scheme whose kernel is `kind`."""
kernel = object.__new__(kind)
kernel.config = SimpleNamespace(act_type=act_type)
module = torch.nn.Module()
module.scheme = SimpleNamespace(kernel=kernel)
return module
def test_vllm_loader_streams_w4a16_tensors_without_speaker_head():
pytest.importorskip("vllm")
from vllm.model_executor.kernels.linear import MarlinLinearKernel
from kittenml.kittentts2.vllm.loader import KittenModelLoader
loader = object.__new__(KittenModelLoader)
loader.load_config = SimpleNamespace(model_loader_extra_config={
"repo_dir": "unused", "config": {}, "weights": "packed", "quantization": "w4a16"})
class TinyModel:
def load_weights(self, weights):
self.loaded = dict(weights)
return set(self.loaded)
def named_parameters(self):
return [("model.layers.0.mlp.down_proj.weight_packed", None)]
def named_modules(self):
return [("model.layers.0.mlp.down_proj", _linear_with_kernel(MarlinLinearKernel))]
model = TinyModel()
tensors = [("model.layers.0.mlp.down_proj.weight_packed", torch.zeros(1, 1, dtype=torch.int32))]
with mock.patch("kittenml.kittentts2.vllm.w4a16.iter_w4a16",
return_value=iter(tensors)) as convert:
loader.load_weights(model, SimpleNamespace(dtype=torch.bfloat16))
convert.assert_called_once_with("unused", {}, "packed")
assert set(model.loaded) == {"model.layers.0.mlp.down_proj.weight_packed"}
loader.load_config.model_loader_extra_config["quantization"] = "int8"
with pytest.raises(ValueError, match="unknown KittenTTS quantization"):
loader.load_weights(model, None)
def test_vllm_loader_allows_only_kv_cache_scales_to_be_missing():
# compressed-tensors adds KV-cache scale parameters to every attention layer;
# the checkpoint has none (the KV cache stays in the model dtype). On a GPU,
# the strict missing-weights check rejected the real model because of them.
pytest.importorskip("vllm")
from vllm.model_executor.kernels.linear import MarlinLinearKernel
from vllm.model_executor.layers.quantization.kv_cache import BaseKVCacheMethod
from kittenml.kittentts2.vllm.loader import KittenModelLoader
loader = object.__new__(KittenModelLoader)
loader.load_config = SimpleNamespace(model_loader_extra_config={
"repo_dir": "unused", "config": {}, "weights": "packed", "quantization": "w4a16"})
attn = torch.nn.Module()
attn.quant_method = object.__new__(BaseKVCacheMethod)
attn.k_scale = torch.nn.Parameter(torch.tensor(-1.0), requires_grad=False)
attn.k_zero_point = torch.nn.Parameter(torch.tensor(0), requires_grad=False)
model = torch.nn.Module()
model.attn = attn
model.proj = torch.nn.Linear(2, 2, bias=False)
model.proj.scheme = _linear_with_kernel(MarlinLinearKernel).scheme
model.load_weights = lambda weights: {name for name, _ in weights}
with mock.patch("kittenml.kittentts2.vllm.w4a16.iter_w4a16",
return_value=iter([("proj.weight", torch.zeros(2, 2))])):
loader.load_weights(model, SimpleNamespace(dtype=torch.bfloat16))
with mock.patch("kittenml.kittentts2.vllm.w4a16.iter_w4a16", return_value=iter([])):
with pytest.raises(ValueError, match=r"missing from the checkpoint: \['proj.weight'\]"):
loader.load_weights(model, SimpleNamespace(dtype=torch.bfloat16))
@pytest.mark.parametrize("variant", ["packed", "full"])
def test_selected_tensor_loading_matches_full_load(tmp_path, variant):
from kittenml.kittentts2.vllm.backend import _main_process_tensor
_, config = _w4a16_checkpoint(tmp_path, layers=1)
everything = load_lm_weights(tmp_path, config, variant)
selected = load_lm_weights(tmp_path, config, variant, include=_main_process_tensor)
assert set(selected) == {"model.embed_tokens.weight", "spk_proj.0.weight", "spk_proj.0.bias",
"spk_proj.1.weight", "spk_proj.1.bias"}
assert all(torch.equal(selected[k], everything[k]) for k in selected)
def test_vllm_loader_refuses_w4a16_layers_on_any_kernel_but_marlin():
# On Hopper vLLM would pick Machete, and with VLLM_DISABLED_KERNELS it can fall
# back to Triton or others; exact output was verified on Marlin only.
pytest.importorskip("vllm")
from vllm.model_executor.kernels.linear import MarlinLinearKernel
from vllm.model_executor.kernels.linear.mixed_precision.machete import MacheteLinearKernel
from kittenml.kittentts2.vllm.loader import KittenModelLoader
loader = object.__new__(KittenModelLoader)
loader.load_config = SimpleNamespace(model_loader_extra_config={
"repo_dir": "unused", "config": {}, "weights": "packed", "quantization": "w4a16"})
model = torch.nn.Module()
model.a = _linear_with_kernel(MarlinLinearKernel)
model.b = _linear_with_kernel(MacheteLinearKernel)
model.load_weights = mock.Mock(return_value=set())
with mock.patch("kittenml.kittentts2.vllm.w4a16.iter_w4a16") as convert:
with pytest.raises(RuntimeError, match=r"\['MacheteLinearKernel'\] for 1 linear layers"):
loader.load_weights(model, SimpleNamespace(dtype=torch.bfloat16))
with pytest.raises(RuntimeError, match="no quantized linear layers"):
loader.load_weights(torch.nn.Module(), SimpleNamespace(dtype=torch.bfloat16))
convert.assert_not_called()
model.load_weights.assert_not_called()
def test_vllm_loader_refuses_w4a8_activations():
# VLLM_MARLIN_INPUT_DTYPE keeps MarlinLinearKernel but sets its activation type
# to int8/fp8 (W4A8). The loader runs in the engine process and checks the
# activation type vLLM gave each layer.
pytest.importorskip("vllm")
from vllm.model_executor.kernels.linear import MarlinLinearKernel
from kittenml.kittentts2.vllm.loader import KittenModelLoader
loader = object.__new__(KittenModelLoader)
loader.load_config = SimpleNamespace(model_loader_extra_config={
"repo_dir": "unused", "config": {}, "weights": "packed", "quantization": "w4a16"})
bf16 = SimpleNamespace(dtype=torch.bfloat16)
model = torch.nn.Module()
model.a = _linear_with_kernel(MarlinLinearKernel)
model.b = _linear_with_kernel(MarlinLinearKernel, act_type=torch.int8)
model.load_weights = mock.Mock(return_value=set())
with mock.patch("kittenml.kittentts2.vllm.w4a16.iter_w4a16") as convert:
with pytest.raises(RuntimeError, match=r"1 linear layers would quantize their "
r"activations to \['torch.int8'\].*"
r"VLLM_MARLIN_INPUT_DTYPE"):
loader.load_weights(model, bf16)
model.b = _linear_with_kernel(MarlinLinearKernel)
with pytest.raises(RuntimeError, match="engine dtype is torch.float16"):
loader.load_weights(model, SimpleNamespace(dtype=torch.float16))
convert.assert_not_called()
model.load_weights.assert_not_called()
def test_engine_sees_the_checked_environment_after_an_earlier_engine(monkeypatch, tmp_path):
# In single-process mode vLLM freezes its view of the environment once an
# engine has started (enable_envs_cache). A later W4A16 engine must see the
# environment KittenTTS checked, not the frozen one.
pytest.importorskip("vllm")
import vllm.envs
from kittenml.kittentts2.vllm.backend import VLLMBackend
seen = []
def create_engine(**kwargs):
seen.append((vllm.envs.VLLM_MARLIN_INPUT_DTYPE, vllm.envs.VLLM_DISABLED_KERNELS))
owner = SimpleNamespace(device="cuda", repo_dir=tmp_path, config={}, weights="packed",
token_map=None, _load_lm_weights=mock.Mock(return_value={
"model.embed_tokens.weight": torch.zeros(2, 4)}))
monkeypatch.delenv("VLLM_DISABLED_KERNELS", raising=False)
monkeypatch.setenv("VLLM_MARLIN_INPUT_DTYPE", "int8")
try:
vllm.envs.enable_envs_cache() # as after a first engine in this process
monkeypatch.delenv("VLLM_MARLIN_INPUT_DTYPE")
assert vllm.envs.VLLM_MARLIN_INPUT_DTYPE == "int8"
with mock.patch("kittenml.kittentts2.vllm.backend.validate_environment"), \
mock.patch("torch.cuda.get_device_capability", return_value=(8, 6)), \
mock.patch("transformers.AutoConfig.from_pretrained", return_value=SimpleNamespace(
model_type="qwen3", hidden_size=4)), \
mock.patch("transformers.AutoTokenizer.from_pretrained"), \
mock.patch("kittenml.kittentts2.speaker.load_speaker_head"), \
mock.patch("vllm.LLM", side_effect=create_engine):
VLLMBackend(owner, {"quantization": "w4a16"})
finally:
vllm.envs.disable_envs_cache()
assert seen == [(None, ["MacheteLinearKernel"])]
def test_w4a16_exactness_checks_compare_bits_not_values(tmp_path):
# torch.equal treats -0.0 and +0.0 as equal; the W4A16 checks must not.
from kittenml.kittentts2.vllm.w4a16 import bit_equal, iter_w4a16
zero = torch.zeros(3, dtype=torch.bfloat16)
assert torch.equal(-zero, zero) and not bit_equal(-zero, zero)
assert bit_equal(zero, zero.clone()) and not bit_equal(zero, zero.float())
weight = torch.zeros(2, 128, dtype=torch.bfloat16)
weight[0, :2] = torch.tensor([0.5, -0.5])
weight[1, 3] = -0.0 # a ternary rebuild gives +0.0 here
(tmp_path / "lm").mkdir()
save_file({"model.layers.0.self_attn.o_proj.weight": weight},
tmp_path / "lm" / "model.safetensors")
with pytest.raises(ValueError, match="not exactly ternary"):
list(iter_w4a16(tmp_path, {}, "full"))
weight[1, 3] = 0.0
save_file({"model.layers.0.self_attn.o_proj.weight": weight},
tmp_path / "lm" / "model.safetensors")
assert len(list(iter_w4a16(tmp_path, {}, "full"))) == 3
# ── Opt-in float16 (vllm_options={"dtype": "float16"}), fp16.py ─────────────────
def _qwen3(layers=2, tied=True):
from transformers import Qwen3Config, Qwen3ForCausalLM
config = Qwen3Config(vocab_size=32, hidden_size=256, intermediate_size=384,
num_hidden_layers=layers, num_attention_heads=2, num_key_value_heads=1,
head_dim=128, rms_norm_eps=1e-6, tie_word_embeddings=tied,
max_position_embeddings=64)
config._attn_implementation = "eager"
return Qwen3ForCausalLM(config).double().eval()
def _realistic_state(model, generator):
"""Random weights at the published model's scales: small embeddings (mean(x^2)
around 1e-3, so an unscaled epsilon would matter) and a residual stream that grows."""
state = {}
for name, tensor in model.state_dict().items():
if name == "lm_head.weight":
continue
if name.endswith("norm.weight"):
value = torch.rand(tensor.shape, generator=generator, dtype=torch.float64) + 0.5
elif name == "model.embed_tokens.weight":
value = torch.randn(tensor.shape, generator=generator, dtype=torch.float64) * 0.033
else:
value = torch.randn(tensor.shape, generator=generator, dtype=torch.float64) * 0.5
state[name] = value
return state
def test_float16_rescale_is_the_same_model_in_float64():
# The rescale is exact: with the stream's inputs at 2^-9, every branch at 2^-9 and
# the residual norms' epsilon at 2^-18, the logits equal the original model's.
# Checked on transformers' Qwen3 in float64, with the plan's own functions.
from kittenml.kittentts2.vllm.fp16 import (EMBEDDING, LM_HEAD, RESIDUAL_SHIFT,
residual_norms, scale_norm_epsilons, shift_of)
generator = torch.Generator().manual_seed(3)
reference = _qwen3(tied=True)
state = _realistic_state(reference, generator)
reference.load_state_dict(state, strict=False)
reference.tie_weights()
rescaled = _qwen3(tied=False)
rescaled.load_state_dict({**{name: tensor * 2.0 ** -shift_of(name)
for name, tensor in state.items()},
LM_HEAD: state[EMBEDDING]}, strict=True)
assert rescaled.lm_head.weight.data_ptr() != rescaled.model.embed_tokens.weight.data_ptr()
assert scale_norm_epsilons(rescaled, layers=2, eps=1e-6) == 5
assert [name for name, _ in residual_norms(rescaled)] == [
"model.layers.0.input_layernorm", "model.layers.0.post_attention_layernorm",
"model.layers.1.input_layernorm", "model.layers.1.post_attention_layernorm",
"model.norm"]
assert rescaled.model.layers[0].self_attn.q_norm.variance_epsilon == 1e-6
ids = torch.tensor([[0, 5, 9, 2, 31, 7]])
embeds = state[EMBEDDING][ids].clone()
embeds[0, 0] = torch.randn(256, generator=generator, dtype=torch.float64) # spk_proj slot
with torch.no_grad():
expected = reference(inputs_embeds=embeds).logits
actual = rescaled(inputs_embeds=embeds * 2.0 ** -RESIDUAL_SHIFT).logits
# transformers' RMSNorm computes in float32, so agreement is to float32 rounding.
torch.testing.assert_close(actual, expected, rtol=1e-5, atol=1e-5)
# Negative control: without the epsilon scale the first norm (embeddings at
# 2^-9, mean(x^2) ~ 4e-9 << 1e-6) is visibly wrong.
for _, module in residual_norms(rescaled):
module.variance_epsilon = 1e-6
wrong = rescaled(inputs_embeds=embeds * 2.0 ** -RESIDUAL_SHIFT).logits
assert (wrong - expected).abs().max() > 1e-2
def test_float16_plan_shifts_each_branch_by_the_residual_scale():
from kittenml.kittentts2.vllm import fp16
p = "model.layers.3."
shifts = {name: fp16.shift_of(p + name + ".weight") for name in _PROJECTIONS}
assert shifts == {"self_attn.q_proj": 0, "self_attn.k_proj": 0, "self_attn.v_proj": 3,
"self_attn.o_proj": 6, "mlp.gate_proj": 0, "mlp.up_proj": 5,
"mlp.down_proj": 4}
assert fp16.shift_of(p + "mlp.down_proj.weight_scale") == 4
assert fp16.shift_of(p + "mlp.down_proj.weight_packed") == 0
assert fp16.shift_of("model.embed_tokens.weight") == fp16.RESIDUAL_SHIFT == 9
assert fp16.shift_of(p + "input_layernorm.weight") == 0
assert fp16.EPS_SHIFT == 18
assert fp16.hf_overrides() == {"tie_word_embeddings": False,
"kittenml_float16_rescale": fp16.PLAN}
assert fp16.PLAN == ("residual 2^-9; v_proj 2^-3, o_proj 2^-6, up_proj 2^-5, down_proj 2^-4; "
"residual-norm eps x 2^-18; untied unscaled lm_head")
def test_float16_plan_reaches_the_config_vllm_hashes_for_its_compile_cache():
# vLLM's ModelConfig.compute_hash ignores hf_overrides itself but hashes
# hf_config.to_json_string() after the overrides are applied with setattr.
from transformers import Qwen3Config
from kittenml.kittentts2.vllm.fp16 import PLAN, hf_overrides
config = Qwen3Config(tie_word_embeddings=True)
before = config.to_json_string()
for key, value in hf_overrides().items():
setattr(config, key, value)
after = config.to_json_string()
assert after != before and PLAN in after and config.tie_word_embeddings is False
@pytest.mark.parametrize("variant", ["packed", "full"])
def test_float16_rescale_dense_and_w4a16_give_the_same_exact_float16_weights(tmp_path, variant):
from kittenml.kittentts2.vllm.fp16 import rescale_weights, scaled, shift_of
from kittenml.kittentts2.vllm.w4a16 import bit_equal, is_projection, iter_w4a16
# Group scales >= 2^-6, as in the published checkpoint: every shifted weight
# stays a normal float16 number.
state, config = _w4a16_checkpoint(tmp_path, low=2.0 ** -6)
weights = load_lm_weights(tmp_path, config, variant)
dense = dict(rescale_weights((n, t) for n, t in weights.items()
if not n.startswith("spk_proj.")))
original = dict(iter_w4a16(tmp_path, config, variant))
quantized = dict(rescale_weights(original.items()))
unpack = _reference_unpack()
embedding = state["model.embed_tokens.weight"]
for out in (dense, quantized):
assert bit_equal(out["model.embed_tokens.weight"], scaled(embedding, 9))
assert bit_equal(out["lm_head.weight"], embedding.to(torch.float16)) # untied, unscaled
assert bit_equal(out["model.norm.weight"], state["model.norm.weight"]) # unchanged
for name in (n for n in state if is_projection(n)):
exact = state[name].to(torch.float32) * 2.0 ** -shift_of(name)
assert dense[name].dtype == torch.float16
assert torch.equal(dense[name].to(torch.float32), exact), name
base = name.removesuffix(".weight")
scale = quantized[base + ".weight_scale"]
assert scale.dtype == torch.float16
assert quantized[base + ".weight_packed"] is original[base + ".weight_packed"]
assert torch.equal(scale.to(torch.float32), original[base + ".weight_scale"].float()
* 2.0 ** -shift_of(name))
signs = unpack(quantized[base + ".weight_packed"], state[name].shape)
rebuilt = signs.to(torch.float16) * scale.repeat_interleave(128, dim=1)
assert bit_equal(rebuilt, dense[name]), name
def test_float16_rescale_rounds_small_weights_and_reports_the_error(caplog):
from kittenml.kittentts2.vllm.fp16 import is_exact, rescale_weights
weight = torch.full((2, 128), 0.25, dtype=torch.bfloat16)
weight[1, 7] = 2.0 ** -20 # x 2^-6 = 2^-26: below float16's smallest subnormal
pairs = [("model.layers.0.self_attn.o_proj.weight", weight)]
report = {}
with caplog.at_level("INFO", logger="vllm.kittenml"):
converted = dict(rescale_weights(pairs, report))
assert converted[pairs[0][0]][1, 7] == 0
assert converted[pairs[0][0]][0, 0] == 2.0 ** -8
assert report["projections"] == {
"elements": 256, "rounded": 1, "max_absolute_error": 2.0 ** -26,
"worst_tensor": pairs[0][0]}
assert sum("float16 weight conversion" in r.message for r in caplog.records) == 1
# The same value is exact without a shift (q_proj) ... and -0.0 must stay -0.0.
assert len(dict(rescale_weights([("model.layers.0.self_attn.q_proj.weight", weight)]))) == 1
zero = torch.tensor([-0.0], dtype=torch.bfloat16)
assert is_exact(zero, 6) and is_exact(torch.tensor([65504.0]), 0)
assert not is_exact(torch.tensor([65536.0]), 0) # overflow is not exact
with pytest.raises(ValueError, match="tied"):
dict(rescale_weights([("lm_head.weight", weight)]))
@pytest.mark.parametrize("name", ["model.embed_tokens.weight", "model.norm.weight",
"model.layers.0.self_attn.q_proj.weight",
"model.layers.0.mlp.down_proj.weight_scale"])
@pytest.mark.parametrize("value", [float("nan"), float("inf"), 2.0 ** 24])
def test_float16_rescale_still_refuses_nonfinite_and_overflowing_weights(name, value):
from kittenml.kittentts2.vllm.fp16 import rescale_weights
with pytest.raises(ValueError, match="nonfinite values or overflows float16"):
dict(rescale_weights([(name, torch.tensor([value]))]))
def test_float16_rescale_tracks_embedding_rounding_separately_and_keeps_codes():
from kittenml.kittentts2.vllm.fp16 import rescale_weights
embedding = torch.tensor([[2.0 ** -20, -0.0]], dtype=torch.bfloat16)
codes = torch.tensor([1, 2], dtype=torch.int32)
report = {}
converted = dict(rescale_weights([("model.embed_tokens.weight", embedding),
("model.layers.0.mlp.up_proj.weight_packed", codes)], report))
assert report["embedding"]["rounded"] == 1
assert report["output_head"]["rounded"] == 0
assert converted["model.layers.0.mlp.up_proj.weight_packed"] is codes
assert torch.signbit(converted["model.embed_tokens.weight"][0, 1])
assert converted["lm_head.weight"][0, 0] == 2.0 ** -20
def test_float16_prompt_embeddings_match_the_engine_scale():
pytest.importorskip("vllm")
from kittenml.kittentts2.vllm.backend import VLLMBackend
from kittenml.kittentts2.vllm.fp16 import ScaledSpeakerHead, scale_embedding, scaled
generator = torch.Generator().manual_seed(5)
table = (torch.randn(16, 4, generator=generator) * 0.03).to(torch.bfloat16)
head = torch.nn.Linear(4, 4).to(torch.bfloat16)
backend = object.__new__(VLLMBackend)
backend.max_model_len = 32
backend.token_map = TokenMap(5, 10, 1, 2, 3, 4, 0, 15)
backend.embedding = scale_embedding(table)
backend.spk_proj = ScaledSpeakerHead(head)
speaker = torch.randn(1, 4, generator=generator).to(torch.float16)
settings = dict(temperature=0.7, top_k=5, top_p=0.8, min_p=0.02)
advanced = dict(seed=1, repetition_penalty=1.2, repetition_window=3,
token_run_penalty=1.5, token_run_grace=4)
request, _ = backend.prepare_request([0, 3, 1], speaker, settings, 8, advanced)
embeds = request["prompt_embeds"]
assert embeds.dtype == torch.float16
assert torch.equal(embeds[1:], scaled(table[[3, 1]], 9))
expected = scaled(torch.nn.functional.linear(speaker.float(), head.weight.float(),
head.bias.float()), 9).squeeze(0)
assert torch.equal(embeds[0], expected)
def _norm(eps=1e-6):
module = torch.nn.Module()
module.variance_epsilon = eps
return module
def _float16_engine_model(layers=2, tied=False):
"""Just enough of vLLM's Qwen3ForCausalLM for the loader's float16 checks."""
model = torch.nn.Module()
model.model = torch.nn.Module()
model.model.embed_tokens = torch.nn.Embedding(4, 2)
model.model.layers = torch.nn.ModuleList()
for _ in range(layers):
layer = torch.nn.Module()
layer.input_layernorm, layer.post_attention_layernorm = _norm(), _norm()
layer.self_attn = torch.nn.Module()
layer.self_attn.q_norm, layer.self_attn.k_norm = _norm(), _norm()
model.model.layers.append(layer)
model.model.norm = _norm()
model.lm_head = model.model.embed_tokens if tied else torch.nn.Embedding(4, 2)
model.loaded = {}
def load_weights(weights):
model.loaded.update(weights)
return set(dict(model.named_parameters()))
model.load_weights = load_weights
return model
def test_vllm_loader_float16_rescales_weights_and_residual_norm_epsilon_only():
pytest.importorskip("vllm")
from kittenml.kittentts2.vllm.loader import KittenModelLoader
loader = object.__new__(KittenModelLoader)
loader.load_config = SimpleNamespace(model_loader_extra_config={
"repo_dir": "unused", "config": {}, "weights": "packed", "float16_rescale": True})
hf_config = SimpleNamespace(num_hidden_layers=2, rms_norm_eps=1e-6, tie_word_embeddings=False)
fp16_config = SimpleNamespace(dtype=torch.float16, hf_config=hf_config)
embedding = torch.full((4, 2), 0.03, dtype=torch.bfloat16)
state = {"model.embed_tokens.weight": embedding, "spk_proj.0.bias": torch.zeros(2),
"model.layers.0.mlp.up_proj.weight": torch.full((2, 2), 0.5, dtype=torch.bfloat16)}
model = _float16_engine_model()
with mock.patch("kittenml.kittentts2.vllm.loader.iter_lm_weights", return_value=state.items()):
loader.load_weights(model, fp16_config)
assert set(model.loaded) == {"model.embed_tokens.weight", "lm_head.weight",
"model.layers.0.mlp.up_proj.weight"}
assert torch.equal(model.loaded["lm_head.weight"], embedding.to(torch.float16))
assert model.loaded["model.layers.0.mlp.up_proj.weight"].unique().tolist() == [2.0 ** -6]
for layer in model.model.layers:
assert layer.input_layernorm.variance_epsilon == 1e-6 * 2.0 ** -18
assert layer.post_attention_layernorm.variance_epsilon == 1e-6 * 2.0 ** -18
assert layer.self_attn.q_norm.variance_epsilon == 1e-6 # reads a normed input
assert layer.self_attn.k_norm.variance_epsilon == 1e-6
assert model.model.norm.variance_epsilon == 1e-6 * 2.0 ** -18
loader.load_weights(model, fp16_config) # a second load does not scale twice
assert model.model.norm.variance_epsilon == 1e-6 * 2.0 ** -18
with pytest.raises(RuntimeError, match="engine dtype is torch.bfloat16, not float16"):
loader.load_weights(_float16_engine_model(),
SimpleNamespace(dtype=torch.bfloat16, hf_config=hf_config))
with pytest.raises(RuntimeError, match="tied to the scaled embedding"):
loader.load_weights(_float16_engine_model(tied=True), fp16_config)
with pytest.raises(RuntimeError, match="expected 5 residual-stream norms, found 7"):
loader.load_weights(_float16_engine_model(layers=3), fp16_config)
def test_vllm_loader_bfloat16_paths_hand_over_the_checkpoint_tensors_untouched():
# Without the float16 opt-in the loader passes the very same tensors and never
# touches an epsilon, for the dense and the W4A16 path.
pytest.importorskip("vllm")
from vllm.model_executor.kernels.linear import MarlinLinearKernel
from kittenml.kittentts2.vllm.loader import KittenModelLoader
bf16 = SimpleNamespace(dtype=torch.bfloat16)
for quantization in (None, "w4a16"):
loader = object.__new__(KittenModelLoader)
extra = {"repo_dir": "unused", "config": {}, "weights": "packed"}
if quantization:
extra["quantization"] = quantization
loader.load_config = SimpleNamespace(model_loader_extra_config=extra)
model = _float16_engine_model(tied=True)
model.proj = _linear_with_kernel(MarlinLinearKernel)
tensors = {"model.embed_tokens.weight": torch.full((4, 2), 0.03, dtype=torch.bfloat16),
"model.layers.0.mlp.up_proj.weight": torch.ones(2, 2, dtype=torch.bfloat16)}
with mock.patch("kittenml.kittentts2.vllm.loader.iter_lm_weights",
return_value=tensors.items()), \
mock.patch("kittenml.kittentts2.vllm.w4a16.iter_w4a16",
return_value=iter(tensors.items())):
loader.load_weights(model, bf16)
assert set(model.loaded) == set(tensors)
assert all(model.loaded[name] is tensors[name] for name in tensors)
assert all(module.variance_epsilon == 1e-6 for name, module in model.named_modules()
if hasattr(module, "variance_epsilon"))
def test_engine_receives_float16_when_selected_and_warns_once_on_legacy_gpu(tmp_path, caplog):
pytest.importorskip("vllm")
from kittenml.kittentts2.vllm.backend import VLLMBackend
from kittenml.kittentts2.vllm.fp16 import PLAN, ScaledSpeakerHead, scale_embedding
seen = []
embedding = torch.full((2, 4), 0.03, dtype=torch.bfloat16)
def run(options, capability=None):
owner = SimpleNamespace(device="cpu", repo_dir=tmp_path, config={}, weights="packed",
token_map=None, _load_lm_weights=mock.Mock(return_value={
"model.embed_tokens.weight": embedding}))
with mock.patch("kittenml.kittentts2.vllm.backend.validate_environment", return_value=capability) as validate, \
mock.patch("transformers.AutoConfig.from_pretrained", return_value=SimpleNamespace(
model_type="qwen3", hidden_size=4)), \
mock.patch("transformers.AutoTokenizer.from_pretrained"), \
mock.patch("kittenml.kittentts2.speaker.load_speaker_head",
return_value=torch.nn.Linear(4, 4)) as head, \
mock.patch("vllm.LLM", side_effect=lambda **kwargs: seen.append(kwargs)):
backend = VLLMBackend(owner, options)
validate.assert_called_once_with("cpu", (options or {}).get("dtype"))
return owner, backend, head, seen[-1]
for options in ({"dtype": "float16"}, {"dtype": "float16", "quantization": "w4a16"}):
owner, backend, head, kwargs = run(options)
assert kwargs["dtype"] is torch.float16 and owner.dtype is torch.float16
assert kwargs["model_loader_extra_config"]["float16_rescale"] is True
overrides = kwargs["hf_overrides"]
assert overrides["tie_word_embeddings"] is False
assert overrides["kittenml_float16_rescale"] == PLAN
assert ("quantization_config" in overrides) == ("quantization" in options)
assert torch.equal(backend.embedding, scale_embedding(embedding))
assert isinstance(backend.spk_proj, ScaledSpeakerHead)
assert head.call_args.args[3] is torch.float32 # the speaker head runs in float32
for options in (None, {"dtype": "bfloat16"}, {"quantization": "w4a16"}):
owner, backend, head, kwargs = run(options)
assert kwargs["dtype"] is torch.bfloat16 and owner.dtype is torch.bfloat16
assert "float16_rescale" not in kwargs["model_loader_extra_config"]
assert "tie_word_embeddings" not in kwargs.get("hf_overrides", {})
assert backend.embedding.dtype is torch.bfloat16 and torch.equal(backend.embedding,
embedding)
assert backend.spk_proj is head.return_value
assert head.call_args.args[3] is torch.bfloat16
caplog.clear()
with caplog.at_level("WARNING"):
owner, backend, head, kwargs = run(None, (7, 5))
assert kwargs["dtype"] is torch.float16 and kwargs["quantization"] == "compressed-tensors"
assert sum("has no native bfloat16" in record.message for record in caplog.records) == 1
assert "rounded float16 with 4-bit weights" in caplog.text
assert "supported, not optimal" in caplog.text
# ── Fitting vLLM to the GPU's free memory ─────────────────────────────────────
GIB = 1 << 30
# KittenTTS 2's language model (lm/config.json).
_LM = SimpleNamespace(hidden_size=2048, num_hidden_layers=28, num_attention_heads=16,
num_key_value_heads=8, head_dim=128, intermediate_size=6144,
vocab_size=158241, tie_word_embeddings=True)
def test_lm_memory_matches_what_vllm_reports_for_the_weights():
from kittenml.kittentts2.vllm.backend import lm_memory
# vLLM 0.31.0 logs "Model loading took 3.24 GiB" (bf16) and "1.29 GiB" (w4a16).
bf16, kv_token = lm_memory(_LM)
assert round(bf16 / GIB, 2) == 3.23 and kv_token == 112 * 1024
assert round(lm_memory(_LM, "w4a16")[0] / GIB, 2) == 1.28
# float16 keeps a separate output head (618 MiB, docs/vllm.md).
assert (lm_memory(_LM, None, float16=True)[0] - bf16) >> 20 == 618
assert lm_memory(_LM, float32=True) == (bf16 * 2, kv_token * 2)
def test_float32_fits_weights_and_double_width_cache_without_choosing_w4a16():
from kittenml.kittentts2.vllm.backend import check_options, fit_to_gpu, lm_memory
free, total = 11600 << 20, 12288 << 20
options, report = fit_to_gpu({"dtype": "float32"}, free, total, _LM, force=True)
assert report["fitted"]
assert options["dtype"] == "float32" and "quantization" not in options
assert options["max_num_batched_tokens"] == 2048
assert options.get("max_model_len", 4096) == 4096
assert report["planned_kv_cache_tokens"] >= 4096
assert options["gpu_memory_utilization"] * total + (2816 + 256) * (1 << 20) <= free
with pytest.raises(ValueError, match="W4A16.*not float32"):
check_options({"dtype": "float32", "quantization": "w4a16"}, "packed")
with pytest.raises(RuntimeError, match="not enough free GPU memory"):
fit_to_gpu({"dtype": "float32"}, 5600 << 20, 6000 << 20, _LM)
@pytest.mark.parametrize("total_gib, free_gib", [(95.0, 94.0), (31.4, 30.8), (23.5, 23.0),
(15.6, 15.0), (11.6, 11.2), (9.6, 9.3)])
def test_defaults_are_kept_on_gpus_with_room_for_them(total_gib, free_gib):
from kittenml.kittentts2.vllm.backend import fit_to_gpu
for options in ({}, {"quantization": "w4a16"}, {"max_num_seqs": 2}):
fitted, report = fit_to_gpu(options, int(free_gib * GIB), int(total_gib * GIB), _LM)
assert fitted == options and report["fitted"] is False and "suggested" not in report
@pytest.mark.parametrize("total_gib, free_gib, options", [
(31.4, 30.8, {"gpu_memory_utilization": 0.9}), # vLLM's own default share
(23.5, 23.0, {"gpu_memory_utilization": 0.9}),
(79.15, 41.5, {}), # a shared 80 GB GPU
(9.77, 7.8, {}), # a 10 GB card with a busy desktop
(11.6, 5.0, {}), # a 12 GB card that is mostly in use
])
def test_larger_gpus_and_short_settings_are_never_changed_only_given_a_suggestion(
total_gib, free_gib, options):
from kittenml.kittentts2.vllm.backend import fit_to_gpu
fitted, report = fit_to_gpu(options, int(free_gib * GIB), int(total_gib * GIB), _LM)
assert fitted == options and report["fitted"] is False
# The settings leave too little room for audio decoding, so a fitting set is suggested.
assert report["decoding_room_mib"] < 2816
assert report["suggested"] is None or "gpu_memory_utilization" in report["suggested"]
def test_small_gpus_get_4bit_and_a_share_that_leaves_room_for_decoding():
from kittenml.kittentts2.vllm.backend import (_ENGINE_CONTEXT_BYTES, _MAIN_PROCESS_BYTES,
fit_to_gpu)
total, free = int(7.62 * GIB), int(7.39 * GIB) # RTX 4060 8 GB
fitted, report = fit_to_gpu({}, free, total, _LM)
assert report["fitted"] and fitted["quantization"] == "w4a16"
# vLLM's default prefill step still fits here, so it is kept.
assert "max_num_batched_tokens" not in fitted and "max_model_len" not in fitted
share = fitted["gpu_memory_utilization"] * total
assert share <= free - _ENGINE_CONTEXT_BYTES - _MAIN_PROCESS_BYTES
# The planned cache holds four full requests (4,096 + 32 tokens each).
assert report["planned_kv_cache_tokens"] == 4 * 4128 and report["full_requests"] == 4
assert set(report["chosen"]) == {"quantization", "gpu_memory_utilization"}
# With 5.25 GiB free a full request fits only with 2,048-token prefill steps.
fitted, _ = fit_to_gpu({}, int(5.25 * GIB), total, _LM)
assert fitted["max_num_batched_tokens"] == 2048 and "max_model_len" not in fitted
def test_prefill_step_choice_keeps_the_same_rounding_margin_as_the_context_check():
from kittenml.kittentts2.vllm.backend import (_ENGINE_CONTEXT_BYTES, _KV_MARGIN_TOKENS,
_MAIN_PROCESS_BYTES, _VLLM_BATCHED_TOKENS,
_engine_bytes, fit_to_gpu, lm_memory)
# Free memory inside the window where a full request fits with vLLM's 8,192-token prefill
# step, but not once the share's rounding margin (total / 10^6) is taken off: the 2,048
# step must be chosen then, keeping the full 4,096-token context instead of cutting it.
total = int(7.62 * GIB)
weights, kv_token = lm_memory(_LM, "w4a16")
one_request = (4096 + _KV_MARGIN_TOKENS) * kv_token
free = int(_ENGINE_CONTEXT_BYTES + _MAIN_PROCESS_BYTES
+ _engine_bytes(weights, _VLLM_BATCHED_TOKENS) + one_request + total / 2e6)
fitted, _ = fit_to_gpu({}, free, total, _LM)
assert fitted["max_num_batched_tokens"] == 2048 and "max_model_len" not in fitted
def test_options_the_caller_gives_are_never_changed():
from kittenml.kittentts2.vllm.backend import fit_to_gpu
total, free = int(7.62 * GIB), int(7.39 * GIB)
given = {"quantization": None, "max_num_batched_tokens": None, "max_model_len": 4096}
fitted, report = fit_to_gpu(given, free, total, _LM)
assert {k: fitted[k] for k in given} == given and set(report["chosen"]) == {
"gpu_memory_utilization"}
# A memory share the caller set is theirs: nothing else is chosen around it, even when
# it leaves too little room (then only a suggestion is made).
for options in ({"gpu_memory_utilization": 0.45}, {"gpu_memory_utilization": 0.7},
{"kv_cache_memory_bytes": 1 << 29}):
fitted, report = fit_to_gpu(options, free, total, _LM)
assert fitted == options and report["fitted"] is False
assert report["suggested"]["quantization"] == "w4a16"
# 4-bit is not chosen where it is not allowed (non-NVIDIA, or a refusing environment).
fitted, _ = fit_to_gpu({}, free, total, _LM, w4a16=False)
assert "quantization" not in fitted
def test_tight_gpus_shorten_the_context_and_hopeless_ones_are_refused_with_a_size():
from kittenml.kittentts2.vllm.backend import fit_to_gpu
total = int(7.62 * GIB)
fitted, report = fit_to_gpu({}, int(4.95 * GIB), total, _LM)
assert 2048 <= fitted["max_model_len"] < 4096 and report["planned_kv_cache_tokens"] > 2048
with pytest.raises(RuntimeError, match=r"about [\d,]+ MiB free is needed and 4,096 MiB"):
fit_to_gpu({}, 4 * GIB, total, _LM)
# An explicit max_model_len is not shortened: it is refused instead.
with pytest.raises(RuntimeError, match="not enough free GPU memory"):
fit_to_gpu({"max_model_len": 4096}, int(4.95 * GIB), total, _LM)
def test_rounded_share_still_holds_one_full_request():
"""vLLM's cache is ceil(total * utilization) - the rest; rounding must not cut below a request."""
import math
from kittenml.kittentts2.vllm.backend import _engine_bytes, fit_to_gpu, lm_memory
total = int(7.62 * GIB)
for free_mib in range(4800, 8000, 7):
try:
fitted, _ = fit_to_gpu({}, free_mib << 20, total, _LM)
except RuntimeError:
continue
weights, kv_token = lm_memory(_LM, fitted.get("quantization"))
fixed = _engine_bytes(weights, fitted.get("max_num_batched_tokens", 8192))
cache = math.ceil(total * fitted["gpu_memory_utilization"]) - fixed
assert cache >= (fitted.get("max_model_len", 4096) + 32) * kv_token, free_mib
def test_fit_never_picks_4bit_on_amd_or_when_the_environment_refuses_it(monkeypatch):
from kittenml.kittentts2.vllm.backend import VLLMBackend
small = (int(7.39 * GIB), int(7.62 * GIB))
with mock.patch("kittenml.kittentts2.vllm.backend._free_gpu_memory", return_value=small):
assert VLLMBackend._fit({}, _LM)[0]["quantization"] == "w4a16"
# An AMD GPU without the checked RDNA 4-bit kernel (Radeon RX 7000/9000 have it,
# tests/test_vllm_rocm.py).
with mock.patch("torch.version.hip", "6.3"), \
mock.patch("torch.cuda.get_device_properties",
return_value=SimpleNamespace(gcnArchName="gfx942:sramecc+:xnack-")):
fitted, report = VLLMBackend._fit({}, _LM)
assert report["fitted"] and "quantization" not in fitted
monkeypatch.setenv("VLLM_MARLIN_USE_ATOMIC_ADD", "1")
fitted, report = VLLMBackend._fit({}, _LM)
assert report["fitted"] and "quantization" not in fitted
# A model that is not 16-bit is left alone (the byte counts assume 2-byte weights).
assert VLLMBackend._fit({}, _LM, half=False) == ({}, {"fitted": False,
"skipped": "not a 16-bit model"})
def test_fit_warnings_label_the_fallback_and_say_when_nothing_was_changed(caplog):
from kittenml.kittentts2.vllm.backend import VLLMBackend
with mock.patch("kittenml.kittentts2.vllm.backend._free_gpu_memory",
return_value=(int(7.39 * GIB), int(7.62 * GIB))), caplog.at_level("WARNING"):
VLLMBackend._fit({}, _LM)
assert "supported, but not the optimal path" in caplog.text
assert "16,512 tokens (4 full-length requests at a time)" in caplog.text
caplog.clear()
with mock.patch("kittenml.kittentts2.vllm.backend._free_gpu_memory",
return_value=(5691 << 20, 5798 << 20)), caplog.at_level("WARNING"):
VLLMBackend._fit({}, _LM)
assert "(1 full-length request at a time)" in caplog.text
caplog.clear()
with mock.patch("kittenml.kittentts2.vllm.backend._free_gpu_memory",
return_value=(int(30.8 * GIB), int(31.4 * GIB))), caplog.at_level("WARNING"):
fitted, _ = VLLMBackend._fit({"gpu_memory_utilization": 0.9}, _LM)
assert fitted == {"gpu_memory_utilization": 0.9}
assert "nothing was changed" in caplog.text and "left for audio decoding" in caplog.text
caplog.clear()
with mock.patch("kittenml.kittentts2.vllm.backend._free_gpu_memory",
return_value=(int(30.8 * GIB), int(31.4 * GIB))), caplog.at_level("WARNING"):
VLLMBackend._fit({}, _LM)
assert caplog.text == ""
def test_backend_passes_the_unchanged_defaults_on_a_large_gpu_and_fitted_ones_on_a_small_one(
tmp_path):
pytest.importorskip("vllm")
from kittenml.kittentts2.vllm.backend import VLLMBackend
def run(free, total, options=None):
seen = {}
owner = SimpleNamespace(device="cuda", repo_dir=tmp_path, config={}, weights="packed",
token_map=None, _load_lm_weights=mock.Mock(return_value={
"model.embed_tokens.weight": torch.zeros(2, 4)}))
with mock.patch("kittenml.kittentts2.vllm.backend.validate_environment"), \
mock.patch("kittenml.kittentts2.vllm.backend._free_gpu_memory",
return_value=(free, total)), \
mock.patch("transformers.AutoConfig.from_pretrained",
return_value=SimpleNamespace(model_type="qwen3", **vars(_LM))), \
mock.patch("transformers.AutoTokenizer.from_pretrained"), \
mock.patch("kittenml.kittentts2.speaker.load_speaker_head"), \
mock.patch("vllm.LLM", side_effect=lambda **kw: seen.update(kw)):
backend = VLLMBackend(owner, options)
return backend, seen
backend, large = run(int(30.8 * GIB), int(31.4 * GIB))
_, explicit = run(int(30.8 * GIB), int(31.4 * GIB), {"gpu_memory_utilization": 0.5,
"max_model_len": 4096})
assert large == explicit and backend.auto_fit["fitted"] is False
assert {"quantization", "max_num_batched_tokens", "kv_cache_memory_bytes"}.isdisjoint(large)
assert large["gpu_memory_utilization"] == 0.5 and large["max_model_len"] == 4096
backend, small = run(int(7.39 * GIB), int(7.62 * GIB))
assert backend.auto_fit["fitted"] and small["quantization"] == "compressed-tensors"
assert "max_num_batched_tokens" not in small and small["gpu_memory_utilization"] < 0.5
_, smaller = run(int(5.25 * GIB), int(7.62 * GIB))
assert smaller["max_num_batched_tokens"] == 2048
assert small["model_loader_extra_config"]["quantization"] == "w4a16"
def test_a_torchcodec_that_cannot_load_is_refused_with_what_to_do():
import sys
import types
from kittenml.kittentts2.vllm.backend import check_torchcodec
class Broken(types.ModuleType):
def __init__(self, error):
super().__init__("torchcodec.decoders")
self.error = error
def __getattr__(self, name):
raise self.error
def run(error):
modules = {"torchcodec": types.ModuleType("torchcodec"),
"torchcodec.decoders": Broken(error)}
with mock.patch("kittenml.kittentts2.vllm.backend.importlib.util.find_spec",
return_value=object()), mock.patch.dict(sys.modules, modules):
check_torchcodec()
# What vLLM copes with passes: no FFmpeg (RuntimeError), an incomplete install.
run(RuntimeError("Could not load libtorchcodec. Likely causes: FFmpeg is not installed"))
run(ImportError("cannot import name AudioDecoder"))
# A build for another CUDA version, with FFmpeg present, raises OSError: refused.
with pytest.raises(RuntimeError, match=r"torchcodec is installed but cannot load \(Could "
r"not load this library.*pip uninstall torchcodec"):
run(OSError("Could not load this library: .../libtorchcodec_core6.so\nmore detail"))
# Not installed: nothing to check.
with mock.patch("kittenml.kittentts2.vllm.backend.importlib.util.find_spec",
return_value=None):
check_torchcodec()
def test_engine_is_stopped_at_exit_before_multiprocessing_waits_for_it():
import weakref
from kittenml.kittentts2.vllm.backend import _stop_engine
class Backend: # VLLMBackend stand-in; SimpleNamespace takes no weak references
pass
calls = []
backend = Backend()
backend.engine = SimpleNamespace(llm_engine=SimpleNamespace(
engine_core=SimpleNamespace(shutdown=lambda: calls.append("shutdown"))))
ref = weakref.ref(backend)
_stop_engine(ref)
assert calls == ["shutdown"]
def refuse():
raise RuntimeError("already stopped")
backend.engine.llm_engine.engine_core.shutdown = refuse
_stop_engine(ref) # errors at exit are swallowed
del backend
_stop_engine(ref) # a backend already collected is skipped
def test_backend_registers_the_engine_stop_at_exit(tmp_path):
pytest.importorskip("vllm")
import weakref
from kittenml.kittentts2.vllm.backend import VLLMBackend, _stop_engine
owner = SimpleNamespace(device="cuda", repo_dir=tmp_path, config={}, weights="packed",
token_map=None, _load_lm_weights=mock.Mock(return_value={
"model.embed_tokens.weight": torch.zeros(2, 4)}))
with mock.patch("kittenml.kittentts2.vllm.backend.validate_environment"), \
mock.patch("torch.cuda.get_device_capability", return_value=(8, 6)), \
mock.patch("transformers.AutoConfig.from_pretrained", return_value=SimpleNamespace(
model_type="qwen3", hidden_size=4)), \
mock.patch("transformers.AutoTokenizer.from_pretrained"), \
mock.patch("kittenml.kittentts2.speaker.load_speaker_head"), \
mock.patch("vllm.LLM", return_value=mock.sentinel.engine), \
mock.patch("kittenml.kittentts2.vllm.backend.atexit") as fake_atexit:
backend = VLLMBackend(owner)
# Registered once, after the engine exists, with a weak reference so the
# handler does not keep the backend (and its engine) alive.
fake_atexit.register.assert_called_once()
handler, ref = fake_atexit.register.call_args.args
assert handler is _stop_engine
assert isinstance(ref, weakref.ref) and ref() is backend
assert backend.engine is mock.sentinel.engine