Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
# SPDX-License-Identifier: AGPL-3.0-only
# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
"""Small-M activation padding: real CPU torch; the granularity gate reads fake torchao attributes."""
from __future__ import annotations
import types
import pytest
torch = pytest.importorskip("torch")
nn = torch.nn
from core.inference.diffusion_quant_pad import ( # noqa: E402
DEFAULT_PAD_TO,
INT_MM_MIN_M,
PadToMinM,
ZeroRowSafeLinear,
activation_granularity_is_per_row,
is_quantized_linear,
matching_linear_fqns,
padding_is_bitwise_exact,
wrap_small_m_linears,
wrap_zero_row_linears,
)
class _RecordingLinear(nn.Linear):
"""A Linear that remembers the activation it was handed, so the pad rows are inspectable."""
def forward(self, x):
self.seen = x.detach().clone()
return super().forward(x)
def _fake_quant_tensor_type(**class_attrs):
"""Fake torchao weight type; granularity is set on the class, as nn.Parameter loses instance attrs."""
def __tensor_flatten__(self): # pragma: no cover - presence is the whole point
return ["qdata"], None
return type(
"_FakeQuantTensor",
(torch.Tensor,),
{"__tensor_flatten__": __tensor_flatten__, **class_attrs},
)
def _fake_quantized_linear(
in_f = 8,
out_f = 6,
*,
per_row = True,
unknown = False,
):
"""A Linear whose weight advertises a torchao layout with a chosen activation granularity."""
if unknown:
attrs = {}
elif per_row:
attrs = {
"input_quant_func": types.SimpleNamespace(
__name__ = "_int8_symm_per_token_reduced_range_quant"
)
}
else:
attrs = {"input_quant_func": types.SimpleNamespace(__name__ = "_int8_symm_per_tensor_quant")}
lin = nn.Linear(in_f, out_f)
lin.weight = nn.Parameter(
lin.weight.data.as_subclass(_fake_quant_tensor_type(**attrs)), requires_grad = False
)
return lin
@pytest.mark.parametrize("m", [1, 5, 10, 16, 17, 19, 64])
@pytest.mark.parametrize("lead", [(), (2,), (2, 3)])
def test_padding_returns_the_unpadded_result(m, lead):
"""Padded output must match the unpadded result; dense float tolerates BLAS reordering."""
torch.manual_seed(0)
inner = nn.Linear(8, 6)
wrapped = PadToMinM(inner, min_m = INT_MM_MIN_M, pad_to = DEFAULT_PAD_TO)
x = torch.randn(*lead, m, 8)
with torch.no_grad():
got, want = wrapped(x), inner(x)
assert got.shape == (*lead, m, 6) == want.shape
assert torch.allclose(got, want, rtol = 0, atol = 1e-6)
def test_pad_rows_replicate_row_zero_rather_than_being_zeros():
"""Zero pad rows give amax 0 and NaN in the activation quantizer; replicating row 0 stays
finite, adds no new element values, and keeps the per-tensor AMAX invariant."""
inner = _RecordingLinear(8, 6)
wrapped = PadToMinM(inner, min_m = 17, pad_to = 32)
x = torch.randn(5, 8)
wrapped(x)
seen = inner.seen
assert seen.shape == (32, 8), "activation must reach the GEMM at exactly pad_to rows"
assert torch.equal(seen[:5], x), "the caller's rows must be untouched"
for row in range(5, 32):
assert torch.equal(seen[row], x[0]), f"pad row {row} is not a copy of row 0"
assert not (seen[5:] == 0).all(), "pad rows must not be zeros"
def test_every_small_activation_normalises_to_one_row_count():
"""Small activations pad to one row count, so one inductor graph covers every prompt length."""
inner = _RecordingLinear(8, 6)
wrapped = PadToMinM(inner, min_m = 17, pad_to = 32)
for m in (1, 10, 13, 14, 16, 17, 19, 31):
wrapped(torch.randn(m, 8))
assert inner.seen.shape == (32, 8), f"M = {m} did not normalise to 32 rows"
def test_no_padding_at_or_above_pad_to():
"""A module that is small on one call and large on the next must pay nothing on the large
one: at or above ``pad_to`` the activation reaches the GEMM at its own row count."""
inner = _RecordingLinear(8, 6)
wrapped = PadToMinM(inner, min_m = 17, pad_to = 32)
wrapped(torch.randn(32, 8))
assert inner.seen.shape == (32, 8)
wrapped(torch.randn(4096, 8))
assert inner.seen.shape == (4096, 8)
def test_zero_rows_return_the_projected_width():
"""torchao hands a zero-row activation back UNPROJECTED, so a downstream width-sensitive add
crashes. The wrapper has no row 0 to replicate either, so it synthesises the empty result."""
inner = _RecordingLinear(8, 6)
wrapped = PadToMinM(inner)
out = wrapped(torch.randn(0, 8))
assert out.shape == (0, 6), "an empty activation must still come back 6 wide, not 8"
assert not hasattr(inner, "seen"), "the inner GEMM must not be called with zero rows"
def test_pad_to_can_never_sit_below_min_m():
"""``pad_to`` exists to buy tiling and shape stability above the floor, never to undercut
it: a 'padded' activation below ``min_m`` would still trip the assert it exists to clear."""
inner = nn.Linear(8, 6)
assert PadToMinM(inner, min_m = 17, pad_to = 8).pad_to == 17
assert PadToMinM(inner, min_m = 17, pad_to = None).pad_to == 17
assert PadToMinM(inner, min_m = 17, pad_to = 32).pad_to == 32
def test_forward_holds_no_mutable_integer_state():
"""Dynamo guards on module int attributes: a counter bumped in forward recompiles each call
until the limit drops to eager; wrapper ints must be exactly the two configured constants."""
def int_attrs(module):
return {
k: v for k, v in vars(module).items() if isinstance(v, int) and not isinstance(v, bool)
}
wrapped = PadToMinM(nn.Linear(8, 6), min_m = 17, pad_to = 32)
before = int_attrs(wrapped)
for m in (3, 3, 40, 3):
wrapped(torch.randn(m, 8))
assert before == int_attrs(wrapped) == {"min_m": 17, "pad_to": 32}
def test_attributes_pass_through_to_the_inner_linear():
"""diffusers' attention processors read ``to_q.weight.dtype`` and H3's blocks read
``context_embedder.weight``; without the passthrough the wrapper is a drop-in only until the
first such access, which fails at render time rather than at wrap time."""
inner = nn.Linear(8, 6)
wrapped = PadToMinM(inner)
assert wrapped.in_features == 8
assert wrapped.out_features == 6
assert wrapped.weight is inner.weight
assert wrapped.bias is inner.bias
with pytest.raises(AttributeError):
wrapped.definitely_not_a_linear_attribute
class _Tiny(nn.Module):
def __init__(self):
super().__init__()
self.context_embedder = nn.Linear(8, 6)
self.other = nn.Linear(4, 4)
def test_state_dict_hides_the_wrapper_in_both_directions():
"""The prequant builder saves a state dict and the loader loads one into a fresh tree, so a
wrapper that renamed ``context_embedder.weight`` to ``context_embedder.inner.weight`` would
split the two halves of the checkpoint contract. It must be invisible either way round."""
plain, wrapped_model = _Tiny(), _Tiny()
wrapped_model.context_embedder = PadToMinM(wrapped_model.context_embedder)
assert sorted(wrapped_model.state_dict()) == sorted(plain.state_dict())
assert "context_embedder.inner.weight" not in wrapped_model.state_dict()
assert plain.load_state_dict(dict(wrapped_model.state_dict()), strict = True)
assert wrapped_model.load_state_dict(dict(plain.state_dict()), strict = True)
assert torch.equal(wrapped_model.context_embedder.weight, plain.context_embedder.weight)
def test_activation_granularity_probe_reads_both_torchao_layouts():
"""v1 tensors expose the activation quantizer as ``input_quant_func`` and v2 ones as
``act_quant_kwargs.granularity``. An unrecognised layout answers None, not True: the caller
treats unproven as unsafe."""
assert activation_granularity_is_per_row(_fake_quantized_linear(per_row = True)) is True
assert activation_granularity_is_per_row(_fake_quantized_linear(per_row = False)) is False
assert activation_granularity_is_per_row(_fake_quantized_linear(unknown = True)) is None
def _v2(granularity_cls_name):
lin = nn.Linear(8, 6)
kwargs = types.SimpleNamespace(granularity = type(granularity_cls_name, (), {})())
lin.weight = nn.Parameter(
lin.weight.data.as_subclass(_fake_quant_tensor_type(act_quant_kwargs = kwargs)),
requires_grad = False,
)
return lin
assert activation_granularity_is_per_row(_v2("PerRow")) is True
assert activation_granularity_is_per_row(_v2("PerToken")) is True
assert activation_granularity_is_per_row(_v2("PerTensor")) is False
assert activation_granularity_is_per_row(_v2("PerGroup")) is False
assert activation_granularity_is_per_row(nn.Linear(8, 6)) is None
def test_wrap_raises_rather_than_skipping_an_unprovable_linear():
"""Silence is the worst outcome: a half-padded transformer compiles on the modules that were
wrapped and crashes inside ``_int_mm`` on the ones that were not."""
model = _Tiny()
model.context_embedder = _fake_quantized_linear(per_row = False)
with pytest.raises(RuntimeError, match = "provably per row"):
wrap_small_m_linears(model, ["context_embedder"])
assert isinstance(model.context_embedder, nn.Linear), "must not leave a partial wrap behind"
model.context_embedder = _fake_quantized_linear(unknown = True)
with pytest.raises(RuntimeError, match = "provably per row"):
wrap_small_m_linears(model, ["context_embedder"])
def test_wrap_skips_dense_and_already_wrapped_linears():
"""Dense F.linear has no row floor and a PadToMinM is not an nn.Linear, so one gate skips both
and a second pass cannot double the padding."""
model = _Tiny()
assert wrap_small_m_linears(model, ["context_embedder"]) == ()
assert isinstance(model.context_embedder, nn.Linear)
model.context_embedder = _fake_quantized_linear()
assert wrap_small_m_linears(model, ["context_embedder"]) == ("context_embedder",)
assert isinstance(model.context_embedder, PadToMinM)
assert wrap_small_m_linears(model, ["context_embedder"]) == ()
assert not isinstance(model.context_embedder.inner, PadToMinM)
# the wrapper forwards `weight` to its inner Linear, so the gate cannot rely on it alone
assert model.context_embedder.weight is model.context_embedder.inner.weight
assert is_quantized_linear(model.context_embedder) is False
def test_wrap_ignores_names_absent_from_this_checkpoint_variant():
"""Family token lists are name lists, not promises: pruned and dense H3 trees differ."""
assert wrap_small_m_linears(_Tiny(), ["token_refiner.refiner_blocks.0.attn.to_q"]) == ()
def test_matching_fqns_selects_quantized_linears_by_substring():
"""Same substring rule ``make_filter_fn`` applies to exclusions, so the pad list and the
exclude list are read the same way -- and a dense Linear never enters the pad list."""
model = _Tiny()
model.context_embedder = _fake_quantized_linear()
assert matching_linear_fqns(model, ("context_embedder",)) == ("context_embedder",)
assert matching_linear_fqns(model, ("CONTEXT_EMBEDDER",)) == ("context_embedder",)
assert matching_linear_fqns(model, ("other",)) == (), "dense Linears are not pad candidates"
assert matching_linear_fqns(model, ()) == ()
def test_is_quantized_linear_only_accepts_a_torchao_weight():
assert is_quantized_linear(_fake_quantized_linear()) is True
assert is_quantized_linear(nn.Linear(8, 6)) is False
assert is_quantized_linear(nn.LayerNorm(8)) is False
@pytest.mark.parametrize("bias", [True, False])
@pytest.mark.parametrize("shape", [(1, 0, 8), (0, 8), (2, 0, 8)])
def test_an_empty_activation_comes_back_at_the_projected_width(bias, shape):
inner = _RecordingLinear(8, 6, bias = bias).to(torch.bfloat16)
wrapped = ZeroRowSafeLinear(inner)
x = torch.zeros(*shape, dtype = torch.bfloat16)
with torch.no_grad():
got, want = wrapped(x), torch.nn.functional.linear(x, inner.weight, inner.bias)
assert got.shape == (*shape[:-1], 6) == want.shape
assert got.dtype == want.dtype == torch.bfloat16
assert torch.equal(got, want)
assert not hasattr(inner, "seen"), "the inner GEMM must not be called with an empty input"
def test_the_guard_is_inert_on_a_non_empty_activation():
torch.manual_seed(0)
inner = _RecordingLinear(8, 6)
wrapped = ZeroRowSafeLinear(inner)
for shape in ((1, 8), (5, 8), (2, 7, 8)):
x = torch.randn(*shape)
with torch.no_grad():
assert torch.equal(wrapped(x), inner(x))
assert inner.seen.shape == x.shape
def test_the_guard_passes_attributes_and_keys_through():
plain, guarded = _Tiny(), _Tiny()
guarded.context_embedder = ZeroRowSafeLinear(guarded.context_embedder)
assert guarded.context_embedder.in_features == 8
assert guarded.context_embedder.out_features == 6
assert guarded.context_embedder.weight is guarded.context_embedder.inner.weight
with pytest.raises(AttributeError):
guarded.context_embedder.definitely_not_a_linear_attribute
assert sorted(guarded.state_dict()) == sorted(plain.state_dict())
assert "context_embedder.inner.weight" not in guarded.state_dict()
assert plain.load_state_dict(dict(guarded.state_dict()), strict = True)
assert guarded.load_state_dict(dict(plain.state_dict()), strict = True)
assert torch.equal(guarded.context_embedder.weight, plain.context_embedder.weight)
def test_wrapping_for_zero_rows_is_idempotent_and_skips_dense_linears():
model = _Tiny()
assert wrap_zero_row_linears(model, ["context_embedder"]) == ()
assert isinstance(model.context_embedder, nn.Linear)
model.context_embedder = _fake_quantized_linear()
assert wrap_zero_row_linears(model, ["context_embedder"]) == ("context_embedder",)
assert isinstance(model.context_embedder, ZeroRowSafeLinear)
assert wrap_zero_row_linears(model, ["context_embedder"]) == ()
assert not isinstance(model.context_embedder.inner, ZeroRowSafeLinear)
def test_the_two_wrappers_never_stack():
model = _Tiny()
model.context_embedder = _fake_quantized_linear()
assert wrap_small_m_linears(model, ["context_embedder"]) == ("context_embedder",)
assert wrap_zero_row_linears(model, ["context_embedder"]) == ()
assert isinstance(model.context_embedder, PadToMinM)
assert model.context_embedder(torch.randn(0, 8)).shape == (0, 6)
def test_the_guard_list_is_read_with_the_same_substring_rule():
class _Projection(nn.Module):
def __init__(self):
super().__init__()
self.linear_1 = _fake_quantized_linear(8, 8)
self.linear_2 = _fake_quantized_linear(8, 6)
def forward(self, x):
return self.linear_2(self.linear_1(x))
model = _Tiny()
model.image_embedder = _Projection()
fqns = matching_linear_fqns(model, ("image_embedder",))
assert fqns == ("image_embedder.linear_1", "image_embedder.linear_2")
assert wrap_zero_row_linears(model, fqns) == fqns
assert model.image_embedder(torch.zeros(1, 0, 8)).shape == (1, 0, 6)
@pytest.mark.skipif(not torch.cuda.is_available(), reason = "int8 dynamic quant needs CUDA")
def test_int8_padding_is_bitwise_exact_on_a_real_quantized_linear():
"""Closes the loop the fakes leave open: a genuinely torchao-quantized Linear, at the row
counts H3's seven eval prompts produce (M = 10..19, straddling ``_int_mm``'s floor of 16)."""
pytest.importorskip("torchao")
from torchao.quantization import quantize_
from core.inference.diffusion_transformer_quant import _make_quant_config, make_filter_fn
lin = nn.Linear(1024, 768, bias = False).cuda().bfloat16().eval()
quantize_(lin, _make_quant_config("int8"), filter_fn = make_filter_fn(0))
assert is_quantized_linear(lin)
assert activation_granularity_is_per_row(lin) is True
for m in (1, 10, 13, 16, 17, 19, 64):
assert padding_is_bitwise_exact(lin, m), f"padding changed the kept rows at M = {m}"
def test_a_dynamic_row_count_compiles_one_graph_for_short_and_long_captions():
from torch._dynamo.utils import counters
torch.manual_seed(0)
wrapped = PadToMinM(_RecordingLinear(64, 48), min_m = 17, pad_to = 32).eval()
torch._dynamo.reset()
counters.clear()
compiled = torch.compile(wrapped, backend = "eager", dynamic = None)
for m in (19, 21, 540, 7, 32):
x = torch.randn(1, m, 64)
torch._dynamo.mark_dynamic(x, 1)
with torch.no_grad():
got = compiled(x)
want = wrapped(x)
assert got.shape == (1, m, 48)
assert torch.equal(got, want), m
if tuple(int(v) for v in torch.__version__.split(".")[:2]) >= (2, 8):
assert counters["stats"]["unique_graphs"] == 1
torch._dynamo.reset()
@pytest.mark.parametrize("dynamic", [None, False])
def test_a_static_row_count_still_compiles_fullgraph(dynamic):
torch.manual_seed(0)
wrapped = PadToMinM(nn.Linear(64, 48), min_m = 17, pad_to = 32).eval()
torch._dynamo.reset()
compiled = torch.compile(wrapped, backend = "eager", dynamic = dynamic, fullgraph = True)
for m in (1, 19, 32, 540):
x = torch.randn(2, m, 64)
with torch.no_grad():
assert torch.equal(compiled(x), wrapped(x)), m
torch._dynamo.reset()
@pytest.mark.skipif(not torch.cuda.is_available(), reason = "nvfp4 dynamic quant needs Blackwell")
def test_the_zero_row_guard_on_a_real_nvfp4_linear():
"""A real nvfp4 Linear survives a zero-row call once wrapped."""
pytest.importorskip("torchao")
if torch.cuda.get_device_capability() < (10, 0):
pytest.skip("nvfp4 needs sm_100+")
from torchao.quantization import quantize_
from core.inference.diffusion_transformer_quant import _make_quant_config, make_filter_fn
lin = nn.Linear(1152, 3072, dtype = torch.bfloat16).cuda().eval()
quantize_(lin, _make_quant_config("nvfp4"), filter_fn = make_filter_fn(0))
assert is_quantized_linear(lin)
empty = torch.zeros(1, 0, 1152, dtype = torch.bfloat16, device = "cuda")
with pytest.raises(RuntimeError, match = "numel"):
lin(empty)
guarded = ZeroRowSafeLinear(lin)
out = guarded(empty)
assert out.shape == (1, 0, 3072) and out.dtype == torch.bfloat16
torch.manual_seed(0)
for m in (1, 5, 7):
x = torch.randn(1, m, 1152, dtype = torch.bfloat16, device = "cuda")
with torch.no_grad():
got = guarded(x)
assert torch.equal(got, lin(x))
assert torch.isfinite(got).all()