Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

18.2 KB
Raw Blame
# SPDX-License-Identifier: AGPL-3.0-only
# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0

"""Small-M activation padding: real CPU torch; the granularity gate reads fake torchao attributes."""

from __future__ import annotations

import types

import pytest

torch = pytest.importorskip("torch")
nn = torch.nn

from core.inference.diffusion_quant_pad import (  # noqa: E402
    DEFAULT_PAD_TO,
    INT_MM_MIN_M,
    PadToMinM,
    ZeroRowSafeLinear,
    activation_granularity_is_per_row,
    is_quantized_linear,
    matching_linear_fqns,
    padding_is_bitwise_exact,
    wrap_small_m_linears,
    wrap_zero_row_linears,
)


class _RecordingLinear(nn.Linear):
    """A Linear that remembers the activation it was handed, so the pad rows are inspectable."""

    def forward(self, x):
        self.seen = x.detach().clone()
        return super().forward(x)


def _fake_quant_tensor_type(**class_attrs):
    """Fake torchao weight type; granularity is set on the class, as nn.Parameter loses instance attrs."""

    def __tensor_flatten__(self):  # pragma: no cover - presence is the whole point
        return ["qdata"], None

    return type(
        "_FakeQuantTensor",
        (torch.Tensor,),
        {"__tensor_flatten__": __tensor_flatten__, **class_attrs},
    )


def _fake_quantized_linear(
    in_f = 8,
    out_f = 6,
    *,
    per_row = True,
    unknown = False,
):
    """A Linear whose weight advertises a torchao layout with a chosen activation granularity."""
    if unknown:
        attrs = {}
    elif per_row:
        attrs = {
            "input_quant_func": types.SimpleNamespace(
                __name__ = "_int8_symm_per_token_reduced_range_quant"
            )
        }
    else:
        attrs = {"input_quant_func": types.SimpleNamespace(__name__ = "_int8_symm_per_tensor_quant")}
    lin = nn.Linear(in_f, out_f)
    lin.weight = nn.Parameter(
        lin.weight.data.as_subclass(_fake_quant_tensor_type(**attrs)), requires_grad = False
    )
    return lin


@pytest.mark.parametrize("m", [1, 5, 10, 16, 17, 19, 64])
@pytest.mark.parametrize("lead", [(), (2,), (2, 3)])
def test_padding_returns_the_unpadded_result(m, lead):
    """Padded output must match the unpadded result; dense float tolerates BLAS reordering."""
    torch.manual_seed(0)
    inner = nn.Linear(8, 6)
    wrapped = PadToMinM(inner, min_m = INT_MM_MIN_M, pad_to = DEFAULT_PAD_TO)
    x = torch.randn(*lead, m, 8)
    with torch.no_grad():
        got, want = wrapped(x), inner(x)
    assert got.shape == (*lead, m, 6) == want.shape
    assert torch.allclose(got, want, rtol = 0, atol = 1e-6)


def test_pad_rows_replicate_row_zero_rather_than_being_zeros():
    """Zero pad rows give amax 0 and NaN in the activation quantizer; replicating row 0 stays
    finite, adds no new element values, and keeps the per-tensor AMAX invariant."""
    inner = _RecordingLinear(8, 6)
    wrapped = PadToMinM(inner, min_m = 17, pad_to = 32)
    x = torch.randn(5, 8)
    wrapped(x)
    seen = inner.seen
    assert seen.shape == (32, 8), "activation must reach the GEMM at exactly pad_to rows"
    assert torch.equal(seen[:5], x), "the caller's rows must be untouched"
    for row in range(5, 32):
        assert torch.equal(seen[row], x[0]), f"pad row {row} is not a copy of row 0"
    assert not (seen[5:] == 0).all(), "pad rows must not be zeros"


def test_every_small_activation_normalises_to_one_row_count():
    """Small activations pad to one row count, so one inductor graph covers every prompt length."""
    inner = _RecordingLinear(8, 6)
    wrapped = PadToMinM(inner, min_m = 17, pad_to = 32)
    for m in (1, 10, 13, 14, 16, 17, 19, 31):
        wrapped(torch.randn(m, 8))
        assert inner.seen.shape == (32, 8), f"M = {m} did not normalise to 32 rows"


def test_no_padding_at_or_above_pad_to():
    """A module that is small on one call and large on the next must pay nothing on the large
    one: at or above ``pad_to`` the activation reaches the GEMM at its own row count."""
    inner = _RecordingLinear(8, 6)
    wrapped = PadToMinM(inner, min_m = 17, pad_to = 32)
    wrapped(torch.randn(32, 8))
    assert inner.seen.shape == (32, 8)
    wrapped(torch.randn(4096, 8))
    assert inner.seen.shape == (4096, 8)


def test_zero_rows_return_the_projected_width():
    """torchao hands a zero-row activation back UNPROJECTED, so a downstream width-sensitive add
    crashes. The wrapper has no row 0 to replicate either, so it synthesises the empty result."""
    inner = _RecordingLinear(8, 6)
    wrapped = PadToMinM(inner)
    out = wrapped(torch.randn(0, 8))
    assert out.shape == (0, 6), "an empty activation must still come back 6 wide, not 8"
    assert not hasattr(inner, "seen"), "the inner GEMM must not be called with zero rows"


def test_pad_to_can_never_sit_below_min_m():
    """``pad_to`` exists to buy tiling and shape stability above the floor, never to undercut
    it: a 'padded' activation below ``min_m`` would still trip the assert it exists to clear."""
    inner = nn.Linear(8, 6)
    assert PadToMinM(inner, min_m = 17, pad_to = 8).pad_to == 17
    assert PadToMinM(inner, min_m = 17, pad_to = None).pad_to == 17
    assert PadToMinM(inner, min_m = 17, pad_to = 32).pad_to == 32


def test_forward_holds_no_mutable_integer_state():
    """Dynamo guards on module int attributes: a counter bumped in forward recompiles each call
    until the limit drops to eager; wrapper ints must be exactly the two configured constants."""

    def int_attrs(module):
        return {
            k: v for k, v in vars(module).items() if isinstance(v, int) and not isinstance(v, bool)
        }

    wrapped = PadToMinM(nn.Linear(8, 6), min_m = 17, pad_to = 32)
    before = int_attrs(wrapped)
    for m in (3, 3, 40, 3):
        wrapped(torch.randn(m, 8))
    assert before == int_attrs(wrapped) == {"min_m": 17, "pad_to": 32}


def test_attributes_pass_through_to_the_inner_linear():
    """diffusers' attention processors read ``to_q.weight.dtype`` and H3's blocks read
    ``context_embedder.weight``; without the passthrough the wrapper is a drop-in only until the
    first such access, which fails at render time rather than at wrap time."""
    inner = nn.Linear(8, 6)
    wrapped = PadToMinM(inner)
    assert wrapped.in_features == 8
    assert wrapped.out_features == 6
    assert wrapped.weight is inner.weight
    assert wrapped.bias is inner.bias
    with pytest.raises(AttributeError):
        wrapped.definitely_not_a_linear_attribute


class _Tiny(nn.Module):
    def __init__(self):
        super().__init__()
        self.context_embedder = nn.Linear(8, 6)
        self.other = nn.Linear(4, 4)


def test_state_dict_hides_the_wrapper_in_both_directions():
    """The prequant builder saves a state dict and the loader loads one into a fresh tree, so a
    wrapper that renamed ``context_embedder.weight`` to ``context_embedder.inner.weight`` would
    split the two halves of the checkpoint contract. It must be invisible either way round."""
    plain, wrapped_model = _Tiny(), _Tiny()
    wrapped_model.context_embedder = PadToMinM(wrapped_model.context_embedder)

    assert sorted(wrapped_model.state_dict()) == sorted(plain.state_dict())
    assert "context_embedder.inner.weight" not in wrapped_model.state_dict()

    assert plain.load_state_dict(dict(wrapped_model.state_dict()), strict = True)
    assert wrapped_model.load_state_dict(dict(plain.state_dict()), strict = True)
    assert torch.equal(wrapped_model.context_embedder.weight, plain.context_embedder.weight)


def test_activation_granularity_probe_reads_both_torchao_layouts():
    """v1 tensors expose the activation quantizer as ``input_quant_func`` and v2 ones as
    ``act_quant_kwargs.granularity``. An unrecognised layout answers None, not True: the caller
    treats unproven as unsafe."""
    assert activation_granularity_is_per_row(_fake_quantized_linear(per_row = True)) is True
    assert activation_granularity_is_per_row(_fake_quantized_linear(per_row = False)) is False
    assert activation_granularity_is_per_row(_fake_quantized_linear(unknown = True)) is None

    def _v2(granularity_cls_name):
        lin = nn.Linear(8, 6)
        kwargs = types.SimpleNamespace(granularity = type(granularity_cls_name, (), {})())
        lin.weight = nn.Parameter(
            lin.weight.data.as_subclass(_fake_quant_tensor_type(act_quant_kwargs = kwargs)),
            requires_grad = False,
        )
        return lin

    assert activation_granularity_is_per_row(_v2("PerRow")) is True
    assert activation_granularity_is_per_row(_v2("PerToken")) is True
    assert activation_granularity_is_per_row(_v2("PerTensor")) is False
    assert activation_granularity_is_per_row(_v2("PerGroup")) is False

    assert activation_granularity_is_per_row(nn.Linear(8, 6)) is None


def test_wrap_raises_rather_than_skipping_an_unprovable_linear():
    """Silence is the worst outcome: a half-padded transformer compiles on the modules that were
    wrapped and crashes inside ``_int_mm`` on the ones that were not."""
    model = _Tiny()
    model.context_embedder = _fake_quantized_linear(per_row = False)
    with pytest.raises(RuntimeError, match = "provably per row"):
        wrap_small_m_linears(model, ["context_embedder"])
    assert isinstance(model.context_embedder, nn.Linear), "must not leave a partial wrap behind"

    model.context_embedder = _fake_quantized_linear(unknown = True)
    with pytest.raises(RuntimeError, match = "provably per row"):
        wrap_small_m_linears(model, ["context_embedder"])


def test_wrap_skips_dense_and_already_wrapped_linears():
    """Dense F.linear has no row floor and a PadToMinM is not an nn.Linear, so one gate skips both
    and a second pass cannot double the padding."""
    model = _Tiny()
    assert wrap_small_m_linears(model, ["context_embedder"]) == ()
    assert isinstance(model.context_embedder, nn.Linear)

    model.context_embedder = _fake_quantized_linear()
    assert wrap_small_m_linears(model, ["context_embedder"]) == ("context_embedder",)
    assert isinstance(model.context_embedder, PadToMinM)
    assert wrap_small_m_linears(model, ["context_embedder"]) == ()
    assert not isinstance(model.context_embedder.inner, PadToMinM)
    # the wrapper forwards `weight` to its inner Linear, so the gate cannot rely on it alone
    assert model.context_embedder.weight is model.context_embedder.inner.weight
    assert is_quantized_linear(model.context_embedder) is False


def test_wrap_ignores_names_absent_from_this_checkpoint_variant():
    """Family token lists are name lists, not promises: pruned and dense H3 trees differ."""
    assert wrap_small_m_linears(_Tiny(), ["token_refiner.refiner_blocks.0.attn.to_q"]) == ()


def test_matching_fqns_selects_quantized_linears_by_substring():
    """Same substring rule ``make_filter_fn`` applies to exclusions, so the pad list and the
    exclude list are read the same way -- and a dense Linear never enters the pad list."""
    model = _Tiny()
    model.context_embedder = _fake_quantized_linear()
    assert matching_linear_fqns(model, ("context_embedder",)) == ("context_embedder",)
    assert matching_linear_fqns(model, ("CONTEXT_EMBEDDER",)) == ("context_embedder",)
    assert matching_linear_fqns(model, ("other",)) == (), "dense Linears are not pad candidates"
    assert matching_linear_fqns(model, ()) == ()


def test_is_quantized_linear_only_accepts_a_torchao_weight():
    assert is_quantized_linear(_fake_quantized_linear()) is True
    assert is_quantized_linear(nn.Linear(8, 6)) is False
    assert is_quantized_linear(nn.LayerNorm(8)) is False


@pytest.mark.parametrize("bias", [True, False])
@pytest.mark.parametrize("shape", [(1, 0, 8), (0, 8), (2, 0, 8)])
def test_an_empty_activation_comes_back_at_the_projected_width(bias, shape):
    inner = _RecordingLinear(8, 6, bias = bias).to(torch.bfloat16)
    wrapped = ZeroRowSafeLinear(inner)
    x = torch.zeros(*shape, dtype = torch.bfloat16)
    with torch.no_grad():
        got, want = wrapped(x), torch.nn.functional.linear(x, inner.weight, inner.bias)
    assert got.shape == (*shape[:-1], 6) == want.shape
    assert got.dtype == want.dtype == torch.bfloat16
    assert torch.equal(got, want)
    assert not hasattr(inner, "seen"), "the inner GEMM must not be called with an empty input"


def test_the_guard_is_inert_on_a_non_empty_activation():
    torch.manual_seed(0)
    inner = _RecordingLinear(8, 6)
    wrapped = ZeroRowSafeLinear(inner)
    for shape in ((1, 8), (5, 8), (2, 7, 8)):
        x = torch.randn(*shape)
        with torch.no_grad():
            assert torch.equal(wrapped(x), inner(x))
        assert inner.seen.shape == x.shape


def test_the_guard_passes_attributes_and_keys_through():
    plain, guarded = _Tiny(), _Tiny()
    guarded.context_embedder = ZeroRowSafeLinear(guarded.context_embedder)
    assert guarded.context_embedder.in_features == 8
    assert guarded.context_embedder.out_features == 6
    assert guarded.context_embedder.weight is guarded.context_embedder.inner.weight
    with pytest.raises(AttributeError):
        guarded.context_embedder.definitely_not_a_linear_attribute

    assert sorted(guarded.state_dict()) == sorted(plain.state_dict())
    assert "context_embedder.inner.weight" not in guarded.state_dict()
    assert plain.load_state_dict(dict(guarded.state_dict()), strict = True)
    assert guarded.load_state_dict(dict(plain.state_dict()), strict = True)
    assert torch.equal(guarded.context_embedder.weight, plain.context_embedder.weight)


def test_wrapping_for_zero_rows_is_idempotent_and_skips_dense_linears():
    model = _Tiny()
    assert wrap_zero_row_linears(model, ["context_embedder"]) == ()
    assert isinstance(model.context_embedder, nn.Linear)

    model.context_embedder = _fake_quantized_linear()
    assert wrap_zero_row_linears(model, ["context_embedder"]) == ("context_embedder",)
    assert isinstance(model.context_embedder, ZeroRowSafeLinear)
    assert wrap_zero_row_linears(model, ["context_embedder"]) == ()
    assert not isinstance(model.context_embedder.inner, ZeroRowSafeLinear)


def test_the_two_wrappers_never_stack():
    model = _Tiny()
    model.context_embedder = _fake_quantized_linear()
    assert wrap_small_m_linears(model, ["context_embedder"]) == ("context_embedder",)
    assert wrap_zero_row_linears(model, ["context_embedder"]) == ()
    assert isinstance(model.context_embedder, PadToMinM)
    assert model.context_embedder(torch.randn(0, 8)).shape == (0, 6)


def test_the_guard_list_is_read_with_the_same_substring_rule():
    class _Projection(nn.Module):
        def __init__(self):
            super().__init__()
            self.linear_1 = _fake_quantized_linear(8, 8)
            self.linear_2 = _fake_quantized_linear(8, 6)

        def forward(self, x):
            return self.linear_2(self.linear_1(x))

    model = _Tiny()
    model.image_embedder = _Projection()
    fqns = matching_linear_fqns(model, ("image_embedder",))
    assert fqns == ("image_embedder.linear_1", "image_embedder.linear_2")
    assert wrap_zero_row_linears(model, fqns) == fqns
    assert model.image_embedder(torch.zeros(1, 0, 8)).shape == (1, 0, 6)


@pytest.mark.skipif(not torch.cuda.is_available(), reason = "int8 dynamic quant needs CUDA")
def test_int8_padding_is_bitwise_exact_on_a_real_quantized_linear():
    """Closes the loop the fakes leave open: a genuinely torchao-quantized Linear, at the row
    counts H3's seven eval prompts produce (M = 10..19, straddling ``_int_mm``'s floor of 16)."""
    pytest.importorskip("torchao")
    from torchao.quantization import quantize_

    from core.inference.diffusion_transformer_quant import _make_quant_config, make_filter_fn

    lin = nn.Linear(1024, 768, bias = False).cuda().bfloat16().eval()
    quantize_(lin, _make_quant_config("int8"), filter_fn = make_filter_fn(0))
    assert is_quantized_linear(lin)
    assert activation_granularity_is_per_row(lin) is True
    for m in (1, 10, 13, 16, 17, 19, 64):
        assert padding_is_bitwise_exact(lin, m), f"padding changed the kept rows at M = {m}"


def test_a_dynamic_row_count_compiles_one_graph_for_short_and_long_captions():
    from torch._dynamo.utils import counters

    torch.manual_seed(0)
    wrapped = PadToMinM(_RecordingLinear(64, 48), min_m = 17, pad_to = 32).eval()
    torch._dynamo.reset()
    counters.clear()
    compiled = torch.compile(wrapped, backend = "eager", dynamic = None)
    for m in (19, 21, 540, 7, 32):
        x = torch.randn(1, m, 64)
        torch._dynamo.mark_dynamic(x, 1)
        with torch.no_grad():
            got = compiled(x)
            want = wrapped(x)
        assert got.shape == (1, m, 48)
        assert torch.equal(got, want), m
    if tuple(int(v) for v in torch.__version__.split(".")[:2]) >= (2, 8):
        assert counters["stats"]["unique_graphs"] == 1
    torch._dynamo.reset()


@pytest.mark.parametrize("dynamic", [None, False])
def test_a_static_row_count_still_compiles_fullgraph(dynamic):
    torch.manual_seed(0)
    wrapped = PadToMinM(nn.Linear(64, 48), min_m = 17, pad_to = 32).eval()
    torch._dynamo.reset()
    compiled = torch.compile(wrapped, backend = "eager", dynamic = dynamic, fullgraph = True)
    for m in (1, 19, 32, 540):
        x = torch.randn(2, m, 64)
        with torch.no_grad():
            assert torch.equal(compiled(x), wrapped(x)), m
    torch._dynamo.reset()


@pytest.mark.skipif(not torch.cuda.is_available(), reason = "nvfp4 dynamic quant needs Blackwell")
def test_the_zero_row_guard_on_a_real_nvfp4_linear():
    """A real nvfp4 Linear survives a zero-row call once wrapped."""
    pytest.importorskip("torchao")
    if torch.cuda.get_device_capability() < (10, 0):
        pytest.skip("nvfp4 needs sm_100+")
    from torchao.quantization import quantize_

    from core.inference.diffusion_transformer_quant import _make_quant_config, make_filter_fn

    lin = nn.Linear(1152, 3072, dtype = torch.bfloat16).cuda().eval()
    quantize_(lin, _make_quant_config("nvfp4"), filter_fn = make_filter_fn(0))
    assert is_quantized_linear(lin)

    empty = torch.zeros(1, 0, 1152, dtype = torch.bfloat16, device = "cuda")
    with pytest.raises(RuntimeError, match = "numel"):
        lin(empty)

    guarded = ZeroRowSafeLinear(lin)
    out = guarded(empty)
    assert out.shape == (1, 0, 3072) and out.dtype == torch.bfloat16

    torch.manual_seed(0)
    for m in (1, 5, 7):
        x = torch.randn(1, m, 1152, dtype = torch.bfloat16, device = "cuda")
        with torch.no_grad():
            got = guarded(x)
            assert torch.equal(got, lin(x))
        assert torch.isfinite(got).all()