Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
# SPDX-License-Identifier: AGPL-3.0-only
# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
"""Transformer quantisation tests with torch and torchao stubbed in sys.modules; CPU-only."""
from __future__ import annotations
import os
import sys
import types
import pytest
import core.inference.diffusion_transformer_quant as tq
from core.inference.diffusion_transformer_quant import (
TQ_FP8,
TQ_INT8,
TQ_MXFP8,
TQ_NVFP4,
dense_quant_supported_kind,
dense_quant_unsupported_kind_reason,
dense_transformer_supported,
make_filter_fn,
normalize_transformer_quant,
quantize_transformer,
select_transformer_quant_scheme,
)
def _target(*, device = "cuda", dtype = "bfloat16"):
return types.SimpleNamespace(device = device, dtype = dtype)
def _stub_torch(
monkeypatch,
*,
cc = (10, 0),
with_fp8 = True,
cuda_available = True,
device_name = "NVIDIA B200",
):
torch = types.ModuleType("torch")
torch.bfloat16 = "bfloat16"
torch.float16 = "float16"
if with_fp8:
torch.float8_e4m3fn = "float8_e4m3fn"
torch.cuda = types.SimpleNamespace(
is_available = lambda: cuda_available,
get_device_capability = lambda *a: cc,
# Data-center name by default; consumer tests pass a GeForce name.
get_device_name = lambda *a: device_name,
)
monkeypatch.setitem(sys.modules, "torch", torch)
return torch
def test_normalize_transformer_quant():
assert normalize_transformer_quant(None) is None
assert normalize_transformer_quant("") is None
assert normalize_transformer_quant("none") is None
assert normalize_transformer_quant("off") is None
assert normalize_transformer_quant("AUTO") == "auto"
assert normalize_transformer_quant("INT8") == TQ_INT8
assert normalize_transformer_quant("fp8") == TQ_FP8
with pytest.raises(ValueError):
normalize_transformer_quant("int2")
def test_dense_transformer_supported_requires_cuda_bf16(monkeypatch):
_stub_torch(monkeypatch)
assert dense_transformer_supported(_target()) is True
assert dense_transformer_supported(_target(device = "cpu")) is False
assert dense_transformer_supported(_target(dtype = "float16")) is False
def _allow(monkeypatch, allowed):
"""Force ``_scheme_supported`` to accept only ``allowed`` (simulates smoke results)."""
monkeypatch.setattr(tq, "_scheme_supported", lambda scheme, device, **kw: scheme in allowed)
def test_auto_blackwell_prefers_int8_then_walks_the_ladder(monkeypatch):
_stub_torch(monkeypatch, cc = (10, 0))
_allow(monkeypatch, {TQ_NVFP4, TQ_MXFP8, TQ_FP8, TQ_INT8})
assert select_transformer_quant_scheme(_target(), "auto") == TQ_INT8
_allow(monkeypatch, {TQ_NVFP4, TQ_MXFP8, TQ_FP8})
assert select_transformer_quant_scheme(_target(), "auto") == TQ_FP8
# auto skips nvfp4 even where the hardware runs it: opt-in only.
_allow(monkeypatch, {TQ_NVFP4, TQ_MXFP8})
assert select_transformer_quant_scheme(_target(), "auto") == TQ_MXFP8
_allow(monkeypatch, {TQ_NVFP4})
assert select_transformer_quant_scheme(_target(), "auto") is None
def test_auto_consumer_blackwell_prefers_int8(monkeypatch):
# Consumer Blackwell (RTX 50xx): fp8 FP32 accumulate is halved, int8 is full-rate.
_stub_torch(monkeypatch, cc = (10, 0), device_name = "NVIDIA GeForce RTX 5090")
_allow(monkeypatch, {TQ_NVFP4, TQ_MXFP8, TQ_FP8, TQ_INT8})
assert select_transformer_quant_scheme(_target(), "auto") == TQ_INT8
_allow(monkeypatch, {TQ_NVFP4, TQ_MXFP8, TQ_FP8})
assert select_transformer_quant_scheme(_target(), "auto") == TQ_FP8
def test_auto_consumer_ada_prefers_int8(monkeypatch):
# Consumer Ada (RTX 4090): int8 runs ~2x fp8's nerfed FP32-accumulate rate.
_stub_torch(monkeypatch, cc = (8, 9), device_name = "NVIDIA GeForce RTX 4090")
_allow(monkeypatch, {TQ_FP8, TQ_INT8})
assert select_transformer_quant_scheme(_target(), "auto") == TQ_INT8
def test_auto_workstation_unknown_prefers_int8(monkeypatch):
_stub_torch(monkeypatch, cc = (8, 9), device_name = "NVIDIA RTX A5000")
_allow(monkeypatch, {TQ_FP8, TQ_INT8})
assert select_transformer_quant_scheme(_target(), "auto") == TQ_INT8
def test_auto_professional_rtx_prefers_int8(monkeypatch):
for device_name, cc in (
("NVIDIA RTX PRO 6000 Blackwell Server Edition", (10, 0)),
("NVIDIA RTX 6000 Ada Generation", (8, 9)),
):
_stub_torch(monkeypatch, cc = cc, device_name = device_name)
_allow(monkeypatch, {TQ_FP8, TQ_INT8})
assert select_transformer_quant_scheme(_target(), "auto") == TQ_INT8
def test_auto_ada_hopper_prefers_int8_then_fp8(monkeypatch):
_stub_torch(monkeypatch, cc = (8, 9), device_name = "NVIDIA L40S")
_allow(monkeypatch, {TQ_NVFP4, TQ_MXFP8, TQ_FP8, TQ_INT8})
assert select_transformer_quant_scheme(_target(), "auto") == TQ_INT8
_stub_torch(monkeypatch, cc = (9, 0), device_name = "NVIDIA H100 80GB HBM3")
assert select_transformer_quant_scheme(_target(), "auto") == TQ_INT8
_allow(monkeypatch, {TQ_FP8})
assert select_transformer_quant_scheme(_target(), "auto") == TQ_FP8
def test_auto_ampere_prefers_int8(monkeypatch):
_stub_torch(monkeypatch, cc = (8, 0))
_allow(monkeypatch, {TQ_FP8, TQ_INT8}) # no fp8 cores on Ampere
assert select_transformer_quant_scheme(_target(), "auto") == TQ_INT8
_stub_torch(monkeypatch, cc = (8, 6))
assert select_transformer_quant_scheme(_target(), "auto") == TQ_INT8
def test_auto_pre_ampere_unsupported(monkeypatch):
_stub_torch(monkeypatch, cc = (7, 5)) # Turing: below the int8-dynamic floor
_allow(monkeypatch, {TQ_INT8, TQ_FP8})
assert select_transformer_quant_scheme(_target(), "auto") is None
def test_explicit_scheme_honored_or_none(monkeypatch):
_stub_torch(monkeypatch, cc = (8, 0))
_allow(monkeypatch, {TQ_INT8})
assert select_transformer_quant_scheme(_target(), "int8") == TQ_INT8
# An explicit unsupported scheme is not downgraded: None means the GGUF fallback.
assert select_transformer_quant_scheme(_target(), "fp8") is None
assert select_transformer_quant_scheme(_target(), "nvfp4") is None
def test_select_none_when_disabled_or_non_cuda(monkeypatch):
_stub_torch(monkeypatch)
_allow(monkeypatch, {TQ_INT8, TQ_FP8, TQ_NVFP4})
assert select_transformer_quant_scheme(_target(), None) is None
assert select_transformer_quant_scheme(_target(device = "cpu"), "auto") is None
def test_scheme_supported_shortcircuits(monkeypatch):
_stub_torch(monkeypatch, cuda_available = False)
monkeypatch.setattr(tq, "_smoke_probe", lambda *a: pytest.fail("probe should not run"))
assert tq._scheme_supported(TQ_INT8, "cuda") is False
_stub_torch(monkeypatch, with_fp8 = False)
monkeypatch.setattr(tq, "_smoke_probe", lambda *a: pytest.fail("probe should not run"))
assert tq._scheme_supported(TQ_FP8, "cuda") is False
class _FakeTensor:
"""Records the probe's zero-row write so the assertion below can check it happened."""
def __init__(self):
self.zeroed = []
def __setitem__(self, key, value):
self.zeroed.append((key, value))
class _FakeBool:
def __init__(self, value):
self.value = value
def all(self):
return self
def item(self):
return self.value
def test_smoke_probe_caches_and_tolerates_failure(monkeypatch):
tq._SMOKE_CACHE.clear()
calls = {"n": 0}
finite = {"ok": True}
class _Lin:
def __init__(self, *a, **k):
pass
def to(self, **k):
return self
torch = types.ModuleType("torch")
torch.bfloat16 = "bfloat16"
torch.nn = types.SimpleNamespace(Linear = _Lin)
torch.randn = lambda *a, **k: _FakeTensor()
torch.isfinite = lambda t: _FakeBool(finite["ok"])
torch.no_grad = lambda: __import__("contextlib").nullcontext()
torch.cuda = types.SimpleNamespace(is_available = lambda: True, synchronize = lambda: None)
monkeypatch.setitem(sys.modules, "torch", torch)
tqz = types.ModuleType("torchao.quantization")
def _quantize_ok(
module,
config,
filter_fn = None,
):
calls["n"] += 1
tqz.quantize_ = _quantize_ok
tqz.Int8DynamicActivationInt8WeightConfig = lambda: "int8cfg"
tqz.Float8DynamicActivationFloat8WeightConfig = lambda: "fp8cfg"
monkeypatch.setitem(sys.modules, "torchao.quantization", tqz)
_Lin.__call__ = lambda self, x: x
assert tq._smoke_probe(TQ_INT8, "cuda") is True
assert tq._smoke_probe(TQ_INT8, "cuda") is True
assert calls["n"] == 1
tq._SMOKE_CACHE.clear()
def _quantize_boom(
module,
config,
filter_fn = None,
):
raise RuntimeError("kernel unavailable")
tqz.quantize_ = _quantize_boom
assert tq._smoke_probe(TQ_FP8, "cuda") is False
# torchao's fp8 scale has no eps clamp, so a zero row gives NaN unless floored: probe must check finiteness.
# int8, not fp8: the fp8 config imports PerRow, which this stub lacks.
tq._SMOKE_CACHE.clear()
tqz.quantize_ = _quantize_ok
finite["ok"] = False
assert tq._smoke_probe(TQ_INT8, "cuda") is False
def test_the_smoke_probe_does_not_cache_an_out_of_memory(monkeypatch):
# The probe runs before eviction, so OOM is expected; caching it would refuse explicit requests forever.
class _OOM(RuntimeError):
pass
class _Lin:
def __init__(self, *a, **k):
pass
def to(self, **k):
return self
def __call__(self, x):
return x
torch = types.ModuleType("torch")
torch.bfloat16 = "bfloat16"
torch.nn = types.SimpleNamespace(Linear = _Lin)
torch.randn = lambda *a, **k: _FakeTensor()
torch.isfinite = lambda t: _FakeBool(True)
torch.no_grad = lambda: __import__("contextlib").nullcontext()
torch.cuda = types.SimpleNamespace(
is_available = lambda: True, synchronize = lambda: None, OutOfMemoryError = _OOM
)
torch.OutOfMemoryError = _OOM
monkeypatch.setitem(sys.modules, "torch", torch)
tqz = types.ModuleType("torchao.quantization")
calls = {"n": 0}
def _quantize(
module,
config,
filter_fn = None,
):
calls["n"] += 1
tqz.quantize_ = _quantize
monkeypatch.setitem(sys.modules, "torchao.quantization", tqz)
# Raising in the config builder keeps this test off the real torchao.
fault: list = [_OOM("CUDA out of memory. Tried to allocate 2.00 GiB")]
def _config(scheme, fast_accum = None):
if fault[0] is not None:
raise fault[0]
return "cfg"
monkeypatch.setattr(tq, "_make_quant_config", _config)
tq._SMOKE_CACHE.clear()
assert tq._smoke_probe(TQ_INT8, "cuda") is False
assert tq._SMOKE_CACHE == {}, "an OOM must not be remembered as 'this scheme cannot run'"
fault[0] = None
assert tq._smoke_probe(TQ_INT8, "cuda") is True
assert calls["n"] == 1
tq._SMOKE_CACHE.clear()
fault[0] = RuntimeError("kernel unavailable")
assert tq._smoke_probe(TQ_INT8, "cuda") is False
assert tq._SMOKE_CACHE == {(TQ_INT8, "cuda"): False}
# Pre-eviction callers get "could not tell" on OOM, since eviction may free the VRAM needed.
tq._SMOKE_CACHE.clear()
fault[0] = _OOM("CUDA out of memory. Tried to allocate 2.00 GiB")
assert tq._smoke_probe(TQ_INT8, "cuda", unproven_ok = True) is True
assert tq._SMOKE_CACHE == {}
fault[0] = RuntimeError("kernel unavailable")
assert tq._smoke_probe(TQ_INT8, "cuda", unproven_ok = True) is False
# The probe runs in a child because a CUDA context is never released (~800 MiB per backend).
# Verdict parity with the in-process probe is the contract.
@pytest.fixture(autouse = True)
def _reset_child_probe_state():
tq._SMOKE_CACHE.clear()
tq._CHILD_PROBE_UNAVAILABLE = False
tq._CHILD_PROBE_SPAWN_ERRORS = 0
yield
tq._SMOKE_CACHE.clear()
tq._CHILD_PROBE_UNAVAILABLE = False
tq._CHILD_PROBE_SPAWN_ERRORS = 0
def test_the_child_answers_for_every_scheme_in_one_go(monkeypatch):
# One child, not one per ladder step: spawning costs ~4 s (importing torch).
_stub_torch(monkeypatch)
spawns = {"n": 0}
def _table(device):
spawns["n"] += 1
return {TQ_INT8: True, TQ_FP8: True, TQ_NVFP4: False, TQ_MXFP8: False}
monkeypatch.setattr(tq, "_child_probe_table", _table)
monkeypatch.setattr(
tq, "_run_smoke_probe", lambda *a: pytest.fail("probed in-process after the child answered")
)
assert [tq._scheme_supported(s, "cuda") for s in tq.TQ_SCHEMES] == [True, True, False, False]
assert spawns["n"] == 1
assert tq._SMOKE_CACHE == {
(TQ_INT8, "cuda"): True,
(TQ_FP8, "cuda"): True,
(TQ_NVFP4, "cuda"): False,
(TQ_MXFP8, "cuda"): False,
}
def test_a_child_out_of_memory_is_not_cached_and_is_not_a_verdict(monkeypatch):
# OOM says nothing about the scheme; re-probing in-process would hit the same full GPU.
_stub_torch(monkeypatch)
monkeypatch.setattr(tq, "_child_probe_table", lambda device: {TQ_INT8: None, TQ_FP8: True})
monkeypatch.setattr(
tq, "_run_smoke_probe", lambda *a: pytest.fail("re-probed in-process after a child OOM")
)
assert tq._scheme_supported(TQ_INT8, "cuda") is False
assert tq._scheme_supported(TQ_INT8, "cuda", unproven_ok = True) is True
assert (TQ_INT8, "cuda") not in tq._SMOKE_CACHE
assert tq._SMOKE_CACHE == {(TQ_FP8, "cuda"): True}
def test_no_child_falls_back_to_the_in_process_probe(monkeypatch):
# A frozen build or sandbox that refuses to spawn must still load a model.
_stub_torch(monkeypatch)
monkeypatch.setattr(tq, "_child_probe_table", lambda device: None)
monkeypatch.setattr(tq, "_smoke_probe", lambda *a, **k: True)
assert tq._scheme_supported(TQ_INT8, "cuda") is True
def test_a_host_without_cuda_never_spawns_a_child(monkeypatch):
_stub_torch(monkeypatch, cuda_available = False)
monkeypatch.setattr(
tq, "_child_probe_table", lambda device: pytest.fail("spawned on a CUDA-less host")
)
assert tq._scheme_supported(TQ_INT8, "cuda") is False
_stub_torch(monkeypatch, with_fp8 = False)
assert tq._scheme_supported(TQ_FP8, "cuda") is False
def test_a_cached_scheme_does_not_spawn_a_child(monkeypatch):
_stub_torch(monkeypatch)
tq._SMOKE_CACHE[(TQ_INT8, "cuda")] = True
monkeypatch.setattr(tq, "_child_probe_table", lambda device: pytest.fail("spawned on a hit"))
assert tq._scheme_supported(TQ_INT8, "cuda") is True
def test_the_child_entry_posts_one_table_covering_every_scheme(monkeypatch):
# Every scheme is attempted after a failure: verdicts are independent and the CUDA context is paid once.
posted = []
monkeypatch.setattr(
tq,
"_run_smoke_probe",
lambda scheme, device: {TQ_INT8: True, TQ_FP8: None}.get(scheme, False),
)
tq._child_probe_entry("cuda", tq.TQ_SCHEMES, types.SimpleNamespace(put = posted.append))
assert posted == [{TQ_INT8: True, TQ_FP8: None, TQ_NVFP4: False, TQ_MXFP8: False}]
def test_a_spawn_failure_is_remembered_so_it_is_paid_once(monkeypatch):
import multiprocessing
def _no_spawn(name):
raise ValueError(f"no {name} start method here")
monkeypatch.setattr(multiprocessing, "get_context", _no_spawn)
assert tq._child_probe_table("cuda") is None
assert tq._CHILD_PROBE_UNAVAILABLE is True
monkeypatch.setattr(
multiprocessing, "get_context", lambda name: pytest.fail("retried a spawn known to fail")
)
assert tq._child_probe_table("cuda") is None
class _FakeProbeChild:
"""dies_on None is a wedged child that survives SIGKILL; its lifetime record must survive too."""
def __init__(self, dies_on = "terminate"):
self.pid = 4321
self.alive = False
self.signals = []
self._dies_on = dies_on
def start(self):
self.alive = self._dies_on != "start"
def is_alive(self):
return self.alive
def join(self, timeout = None):
pass
def terminate(self):
self.signals.append("terminate")
if self._dies_on == "terminate":
self.alive = False
def kill(self):
self.signals.append("kill")
if self._dies_on == "kill":
self.alive = False
class _FakeProbeQueue:
def __init__(self, table = None):
self._table = table
self.closed = False
def get(self, timeout = None):
if self._table is None:
raise ValueError("empty")
table, self._table = self._table, None
return table
def close(self):
self.closed = True
def join_thread(self):
pass
class _FakeSpawnContext:
def __init__(self, child, queue):
self._child = child
self._queue = queue
def Queue(self):
return self._queue
def Process(
self,
target = None,
args = (),
daemon = None,
):
return self._child
@pytest.fixture
def _probe_lifetime_records(monkeypatch):
"""Capture what the probe adopts and forgets, without touching the real record."""
import utils.process_lifetime as pl
records = {"adopted": [], "forgotten": []}
monkeypatch.setattr(pl, "adopt_pid", records["adopted"].append)
monkeypatch.setattr(pl, "forget_pid", records["forgotten"].append)
return records
def test_the_probe_child_is_adopted_so_a_shutdown_sweep_can_reach_it(
monkeypatch, _probe_lifetime_records
):
# PDEATHSIG is Linux only and the Windows job object can fail, so this record is the fallback reaper.
import multiprocessing
monkeypatch.setattr(tq, "_CHILD_PROBE_TIMEOUT", 0.0)
child = _FakeProbeChild(dies_on = "start")
monkeypatch.setattr(
multiprocessing,
"get_context",
lambda name: _FakeSpawnContext(child, _FakeProbeQueue({TQ_INT8: True})),
)
assert tq._child_probe_table("cuda") == {TQ_INT8: True}
assert _probe_lifetime_records["adopted"] == [child.pid]
def test_the_queue_is_built_with_the_lease_secret_already_scrubbed(
monkeypatch, _probe_lifetime_records
):
# The first spawn-context queue starts the resource tracker with this env, so build it after the scrub.
import multiprocessing
from utils.native_path_leases import LEASE_SECRET_ENV
secret = "x" * 64
monkeypatch.setenv(LEASE_SECRET_ENV, secret)
monkeypatch.setattr(tq, "_CHILD_PROBE_TIMEOUT", 0.0)
seen = {}
class _WatchingContext(_FakeSpawnContext):
def Queue(self):
seen["at_queue"] = os.environ.get(LEASE_SECRET_ENV)
return super().Queue()
monkeypatch.setattr(
multiprocessing,
"get_context",
lambda name: _WatchingContext(
_FakeProbeChild(dies_on = "start"), _FakeProbeQueue({TQ_INT8: True})
),
)
assert tq._child_probe_table("cuda") == {TQ_INT8: True}
assert seen["at_queue"] is None
assert os.environ.get(LEASE_SECRET_ENV) == secret
def test_a_transient_spawn_oserror_is_retried_rather_than_latched(
monkeypatch, _probe_lifetime_records
):
# Spawn OSErrors are often transient; latching would pin the ~800 MiB in-process probe until restart.
import multiprocessing
calls = {"n": 0}
def _get_context(name):
calls["n"] += 1
if calls["n"] == 1:
raise OSError(24, "Too many open files")
return _FakeSpawnContext(_FakeProbeChild(dies_on = "start"), _FakeProbeQueue({TQ_INT8: True}))
monkeypatch.setattr(multiprocessing, "get_context", _get_context)
monkeypatch.setattr(tq, "_CHILD_PROBE_TIMEOUT", 0.0)
assert tq._child_probe_table("cuda") is None
assert tq._CHILD_PROBE_UNAVAILABLE is False
assert tq._child_probe_table("cuda") == {TQ_INT8: True}
assert calls["n"] == 2
assert tq._CHILD_PROBE_SPAWN_ERRORS == 0
def test_a_host_that_refuses_every_spawn_stops_being_asked(monkeypatch):
# Retry is bounded: repeated OSErrors look like a sandbox that will never spawn.
import multiprocessing
calls = {"n": 0}
def _always_refuses(name):
calls["n"] += 1
raise PermissionError(1, "Operation not permitted")
monkeypatch.setattr(multiprocessing, "get_context", _always_refuses)
for _ in range(tq._CHILD_PROBE_SPAWN_ERROR_LIMIT):
assert tq._child_probe_table("cuda") is None
assert tq._CHILD_PROBE_UNAVAILABLE is True
settled = calls["n"]
assert tq._child_probe_table("cuda") is None
assert calls["n"] == settled
def test_a_child_that_survives_terminate_is_killed(_probe_lifetime_records):
# Giving up after terminate would hold the VRAM for the 180 s timeout, or forever if wedged.
child = _FakeProbeChild(dies_on = "kill")
child.start()
assert tq._close_probe_child(child, _FakeProbeQueue()) is True
assert child.signals == ["terminate", "kill"]
assert _probe_lifetime_records["forgotten"] == [child.pid]
def test_a_child_that_survives_kill_keeps_its_breadcrumb_and_is_reported(_probe_lifetime_records):
# A survivor keeps its handle so the sweep can retry it.
child = _FakeProbeChild(dies_on = None)
child.start()
assert tq._close_probe_child(child, _FakeProbeQueue()) is False
assert child.signals == ["terminate", "kill"]
assert _probe_lifetime_records["forgotten"] == []
def test_a_clean_exit_forgets_the_pid(_probe_lifetime_records):
child = _FakeProbeChild(dies_on = "start")
child.start()
queue = _FakeProbeQueue()
assert tq._close_probe_child(child, queue) is True
assert child.signals == []
assert queue.closed is True
assert _probe_lifetime_records["forgotten"] == [child.pid]
def test_the_probe_body_separates_an_oom_from_a_verdict(monkeypatch):
# Both the child and the in-process fallback call _run_smoke_probe, so this pins them together.
class _OOM(RuntimeError):
pass
fault = [None]
class _Lin:
def __init__(self, *a, **k):
pass
def to(self, **k):
return self
def __call__(self, x):
if fault[0] is not None:
raise fault[0]
return x
torch = types.ModuleType("torch")
torch.bfloat16 = "bfloat16"
torch.nn = types.SimpleNamespace(Linear = _Lin)
torch.randn = lambda *a, **k: _FakeTensor()
torch.isfinite = lambda t: _FakeBool(True)
torch.no_grad = lambda: __import__("contextlib").nullcontext()
torch.cuda = types.SimpleNamespace(is_available = lambda: True, synchronize = lambda: None)
monkeypatch.setitem(sys.modules, "torch", torch)
tqz = types.ModuleType("torchao.quantization")
tqz.quantize_ = lambda module, config, filter_fn = None: None
tqz.Int8DynamicActivationInt8WeightConfig = lambda: "int8cfg"
tqz.Float8DynamicActivationFloat8WeightConfig = lambda: "fp8cfg"
monkeypatch.setitem(sys.modules, "torchao.quantization", tqz)
assert tq._run_smoke_probe(TQ_INT8, "cuda") is True
fault[0] = _OOM("CUDA out of memory. Tried to allocate 2.00 GiB")
assert tq._run_smoke_probe(TQ_INT8, "cuda") is None
fault[0] = RuntimeError("kernel unavailable")
assert tq._run_smoke_probe(TQ_INT8, "cuda") is False
def test_an_oom_is_recognised_however_it_is_spelled():
# torch.OutOfMemoryError subclasses RuntimeError and moved between torch.cuda and torch.
assert tq._is_out_of_memory(MemoryError("no room")) is True
assert tq._is_out_of_memory(RuntimeError("CUDA out of memory. Tried to allocate 2 GiB")) is True
assert tq._is_out_of_memory(RuntimeError("kernel unavailable")) is False
assert tq._is_out_of_memory(ImportError("cannot import name 'ScalingType'")) is False
def test_an_unusable_scheme_names_the_fault_the_user_can_actually_fix(monkeypatch):
# The selector folds three faults into one None, and explicit schemes fail CLOSED, so 409 must say which.
monkeypatch.setattr(tq, "_TORCHAO_UNAVAILABLE", (None,))
assert tq.explain_unusable_scheme("z-image-turbo", "fp8") == (
"'fp8' is not usable for family 'z-image-turbo' on this GPU"
)
# The deny list holds on every GPU. mxfp8, not fp8: fp8 on qwen-image is no longer denied.
denied = tq.explain_unusable_scheme("qwen-image", "mxfp8")
assert "measured accuracy gate" in denied and "whatever the GPU" in denied
assert "on this GPU" not in denied.replace("whatever the GPU", "")
monkeypatch.setattr(
tq, "_TORCHAO_UNAVAILABLE", ("ImportError: cannot import name 'ScalingType'",)
)
broken = tq.explain_unusable_scheme("z-image-turbo", "fp8")
assert "cannot import name 'ScalingType'" in broken
assert "not a limit of the GPU" in broken
assert "measured accuracy gate" in tq.explain_unusable_scheme("qwen-image", "mxfp8")
def test_torchao_unavailable_reason_is_resolved_once_and_covers_the_stub(monkeypatch):
monkeypatch.setattr(tq, "_TORCHAO_UNAVAILABLE", None)
monkeypatch.setattr(tq, "is_stubbed", lambda pkg: True)
reason = tq.torchao_unavailable_reason()
assert reason is not None and "stub" in reason
monkeypatch.setattr(tq, "is_stubbed", lambda pkg: False)
assert tq.torchao_unavailable_reason() == reason
monkeypatch.setattr(tq, "_TORCHAO_UNAVAILABLE", None)
monkeypatch.setitem(
sys.modules, "torchao.quantization", types.ModuleType("torchao.quantization")
)
sys.modules["torchao.quantization"].quantize_ = lambda *a, **k: None
assert tq.torchao_unavailable_reason() is None
def test_the_smoke_probe_feeds_zero_rows_not_only_noise(monkeypatch):
# torch.randn never yields a zero row, so the finiteness check needs one explicitly.
tq._SMOKE_CACHE.clear()
seen = {}
class _Lin:
def __init__(self, *a, **k):
pass
def to(self, **k):
return self
def __call__(self, x):
seen["x"] = x
return x
torch = types.ModuleType("torch")
torch.bfloat16 = "bfloat16"
torch.nn = types.SimpleNamespace(Linear = _Lin)
torch.randn = lambda *a, **k: _FakeTensor()
torch.isfinite = lambda t: _FakeBool(True)
torch.no_grad = lambda: __import__("contextlib").nullcontext()
torch.cuda = types.SimpleNamespace(is_available = lambda: True, synchronize = lambda: None)
monkeypatch.setitem(sys.modules, "torch", torch)
tqz = types.ModuleType("torchao.quantization")
tqz.quantize_ = lambda module, config, filter_fn = None: None
tqz.Int8DynamicActivationInt8WeightConfig = lambda: "int8cfg"
tqz.Float8DynamicActivationFloat8WeightConfig = lambda: "fp8cfg"
monkeypatch.setitem(sys.modules, "torchao.quantization", tqz)
assert tq._smoke_probe(TQ_INT8, "cuda") is True
assert seen["x"].zeroed, "probe input was never zeroed anywhere"
assert all(value == 0 for _, value in seen["x"].zeroed)
def _stub_device_name(monkeypatch, name):
torch = types.ModuleType("torch")
torch.cuda = types.SimpleNamespace(get_device_name = lambda device = None: name)
monkeypatch.setitem(sys.modules, "torch", torch)
@pytest.mark.parametrize(
"name",
[
"NVIDIA GeForce RTX 5090",
"NVIDIA GeForce RTX 4090",
"NVIDIA RTX A4000", # workstation: A4000 token, NOT the data-center A40
"NVIDIA RTX A5000",
"NVIDIA Some Future Card 9000", # unknown -> consumer (fast accum is free on DC)
],
)
def test_is_consumer_gpu_true(monkeypatch, name):
_stub_device_name(monkeypatch, name)
assert tq._is_consumer_gpu() is True
@pytest.mark.parametrize(
"name",
[
"NVIDIA B200",
"NVIDIA B300",
"NVIDIA GH200 480GB",
"NVIDIA H100 80GB HBM3",
"NVIDIA A100-SXM4-80GB",
"NVIDIA A40",
"NVIDIA L40S",
"NVIDIA L4",
"Tesla V100-SXM2-16GB",
"NVIDIA RTX PRO 6000 Blackwell Server Edition",
"NVIDIA RTX 6000 Ada Generation",
],
)
def test_is_consumer_gpu_false_for_datacenter(monkeypatch, name):
_stub_device_name(monkeypatch, name)
assert tq._is_consumer_gpu() is False
def test_is_consumer_gpu_defaults_true_on_probe_failure(monkeypatch):
torch = types.ModuleType("torch")
torch.cuda = types.SimpleNamespace()
monkeypatch.setitem(sys.modules, "torch", torch)
assert tq._is_consumer_gpu() is True
def test_make_filter_fn(monkeypatch):
class _Lin:
def __init__(self, i, o):
self.in_features, self.out_features = i, o
torch = types.ModuleType("torch")
torch.nn = types.SimpleNamespace(Linear = _Lin)
monkeypatch.setitem(sys.modules, "torch", torch)
keep = make_filter_fn(512)
assert keep(_Lin(1024, 4096), "blocks.0.attn.to_q") is True
assert keep(_Lin(256, 4096), "time_proj") is False
assert keep(_Lin(4096, 256), "out_proj") is False
assert keep(object(), "not_linear") is False
assert keep(types.SimpleNamespace(), "no_attrs") is False
def test_require_bf16_schemes_excludes_nvfp4():
# fp8/mxfp8 assert a bf16 weight (torchao 0.17), so they gate on it; nvfp4 quantises fp32 fine.
from core.inference.diffusion_transformer_quant import (
_REQUIRE_BF16_SCHEMES,
TQ_FP8,
TQ_MXFP8,
TQ_NVFP4,
TQ_INT8,
)
assert TQ_FP8 in _REQUIRE_BF16_SCHEMES
assert TQ_MXFP8 in _REQUIRE_BF16_SCHEMES
assert TQ_NVFP4 not in _REQUIRE_BF16_SCHEMES
assert TQ_INT8 not in _REQUIRE_BF16_SCHEMES
def test_make_filter_fn_require_bf16_skips_non_bf16(monkeypatch):
# One fp32 Linear raising inside quantize_ would no-op the whole pass.
torch = types.ModuleType("torch")
torch.bfloat16, torch.float32 = "bf16", "fp32"
class _Lin:
def __init__(self, i, o, dtype):
self.in_features, self.out_features = i, o
self.weight = types.SimpleNamespace(dtype = dtype)
torch.nn = types.SimpleNamespace(Linear = _Lin)
monkeypatch.setitem(sys.modules, "torch", torch)
gated = make_filter_fn(512, require_bf16 = True)
assert gated(_Lin(1024, 4096, torch.bfloat16), "blocks.0.attn.to_q") is True
assert gated(_Lin(1024, 4096, torch.float32), "blocks.0.attn.to_q") is False
assert gated(types.SimpleNamespace(in_features = 1024, out_features = 4096), "no_weight") is False
assert make_filter_fn(512)(_Lin(1024, 4096, torch.float32), "blocks.0.attn.to_q") is True
def test_make_filter_fn_int8_excludes_modulation_and_embedders(monkeypatch):
# int8 skips M=1 modulation/embedder linears (torch._int_mm needs M >= 16); fp8 keeps everything.
from core.inference.diffusion_transformer_quant import _INT8_EXCLUDE_NAME_TOKENS
class _Lin:
def __init__(self, i, o):
self.in_features, self.out_features = i, o
torch = types.ModuleType("torch")
torch.nn = types.SimpleNamespace(Linear = _Lin)
monkeypatch.setitem(sys.modules, "torch", torch)
keep = make_filter_fn(512, exclude_name_tokens = _INT8_EXCLUDE_NAME_TOKENS)
big = lambda: _Lin(3072, 18432) # noqa: E731 - large enough to pass min_features
for fqn in (
"transformer_blocks.0.norm1.linear",
"transformer_blocks.0.norm1_context.linear",
"single_transformer_blocks.0.norm.linear",
"norm_out.linear",
"transformer_blocks.0.img_mod.1",
"transformer_blocks.0.txt_mod.1",
"double_stream_modulation_img.linear",
"time_text_embed.timestep_embedder.linear_2",
"time_text_embed.guidance_embedder.linear_2",
"time_guidance_embed.timestep_embedder.linear_2",
):
assert keep(big(), fqn) is False, fqn
for fqn in (
"transformer_blocks.0.attn.to_q",
"transformer_blocks.0.ff.net.0.proj",
"single_transformer_blocks.0.proj_mlp",
"single_transformer_blocks.0.attn.to_qkv_mlp_proj",
"context_embedder", # "context" contains "text": must NOT be excluded
"txt_in",
):
assert keep(big(), fqn) is True, fqn
assert make_filter_fn(512)(big(), "transformer_blocks.0.norm1.linear") is True
assert keep(big(), None) is True
assert keep(big(), "") is True
def test_exclude_tokens_for_scheme_shared_by_runtime_and_builder():
# Runtime and offline prequant must share the int8 exclusion, else artifacts bake the M=1 linears.
from core.inference.diffusion_transformer_quant import (
_INT8_EXCLUDE_NAME_TOKENS,
exclude_tokens_for_scheme,
)
assert exclude_tokens_for_scheme(TQ_INT8) == _INT8_EXCLUDE_NAME_TOKENS
for scheme in (TQ_FP8, TQ_NVFP4, TQ_MXFP8):
assert exclude_tokens_for_scheme(scheme) == ()
def test_exclude_tokens_for_scheme():
from core.inference.diffusion_transformer_quant import (
_INT8_EXCLUDE_NAME_TOKENS,
exclude_tokens_for_scheme,
)
assert exclude_tokens_for_scheme(TQ_INT8) == _INT8_EXCLUDE_NAME_TOKENS
assert exclude_tokens_for_scheme(TQ_FP8) == ()
assert exclude_tokens_for_scheme(TQ_NVFP4) == ()
assert exclude_tokens_for_scheme(TQ_MXFP8) == ()
def test_exclude_tokens_for_scheme_family(monkeypatch):
from core.inference.diffusion_transformer_quant import (
INT8_TEXT_STREAM_ENV,
_INT8_EXCLUDE_NAME_TOKENS,
_QWENIMAGE_INT8_EXCLUDES,
exclude_tokens_for_scheme,
)
monkeypatch.delenv(INT8_TEXT_STREAM_ENV, raising = False)
for fam in ("qwen-image", "qwen-image-edit"):
assert (
exclude_tokens_for_scheme(TQ_INT8, fam, baked = True)
== _INT8_EXCLUDE_NAME_TOKENS + _QWENIMAGE_INT8_EXCLUDES
)
assert (
exclude_tokens_for_scheme(TQ_INT8, "qwen-image-edit")
== _INT8_EXCLUDE_NAME_TOKENS + _QWENIMAGE_INT8_EXCLUDES
)
assert (
exclude_tokens_for_scheme(TQ_INT8, "qwen-image")
== _INT8_EXCLUDE_NAME_TOKENS + _QWENIMAGE_INT8_EXCLUDES
)
from core.inference.diffusion_transformer_quant import int8_text_stream_scope
with int8_text_stream_scope(True):
assert exclude_tokens_for_scheme(TQ_INT8, "qwen-image") == _INT8_EXCLUDE_NAME_TOKENS + (
"txt_in",
)
for token in ("txt_in", "add_q_proj", "to_add_out", "txt_mlp"):
assert token in _QWENIMAGE_INT8_EXCLUDES
assert exclude_tokens_for_scheme(TQ_INT8, "z-image") == _INT8_EXCLUDE_NAME_TOKENS
assert exclude_tokens_for_scheme(TQ_FP8, "qwen-image") == ()
def test_resolve_fast_accum(monkeypatch):
# GPU-class-derived accumulate made fp8 2x slower than int8 on RTX 6000 Ada; a no-op on B200.
for consumer in (True, False):
monkeypatch.setattr(tq, "_is_consumer_gpu", lambda *a, _c = consumer: _c)
assert tq._resolve_fast_accum(None) is True
assert tq._resolve_fast_accum(True) is True
assert tq._resolve_fast_accum(False) is False
def test_fp8_config_uses_per_row_granularity():
"""FP8 must use PerRow: the per-tensor default collapses the denoise on outlier activations."""
torchao_quant = pytest.importorskip("torchao.quantization")
per_row = getattr(torchao_quant, "PerRow", None)
if per_row is None:
pytest.skip("torchao build without PerRow granularity")
cfg = tq._make_quant_config(TQ_FP8)
gran = getattr(cfg, "granularity", None)
assert gran is not None, "fp8 config must set an explicit granularity, not torchao's default"
grans = gran if isinstance(gran, (list, tuple)) else [gran]
assert grans and all(isinstance(g, per_row) for g in grans), f"expected all PerRow, got {gran}"
def test_fp8_config_pins_torch_kernel_preference():
"""Pin KernelPreference.TORCH: AUTO would pick MSLK and change fp8 rounding bitwise."""
pytest.importorskip("torchao.quantization")
try:
from torchao.quantization.quantize_.common.kernel_preference import KernelPreference
except Exception:
pytest.skip("torchao build without KernelPreference")
cfg = tq._make_quant_config(TQ_FP8)
if not hasattr(cfg, "kernel_preference"):
pytest.skip("torchao config without kernel_preference")
assert cfg.kernel_preference == KernelPreference.TORCH
def test_quantize_transformer_applies_and_marks(monkeypatch):
monkeypatch.setattr(
tq, "select_transformer_quant_scheme", lambda target, mode, family = None, **_kw: TQ_FP8
)
seen: dict = {}
def _mk(scheme, fast_accum = None):
seen["scheme"], seen["fast_accum"] = scheme, fast_accum
return f"{scheme}cfg"
monkeypatch.setattr(tq, "_make_quant_config", _mk)
recorder: list = []
tqz = types.ModuleType("torchao.quantization")
tqz.quantize_ = lambda module, config, filter_fn = None: recorder.append(
(module, config, filter_fn)
)
monkeypatch.setitem(sys.modules, "torchao.quantization", tqz)
transformer = types.SimpleNamespace()
pipe = types.SimpleNamespace(transformer = transformer)
assert quantize_transformer(pipe, _target(), mode = "fp8", fast_accum = False) == TQ_FP8
assert len(recorder) == 1 and recorder[0][0] is transformer and recorder[0][1] == "fp8cfg"
assert callable(recorder[0][2])
assert transformer._unsloth_runtime_quant == TQ_FP8
assert seen["fast_accum"] is False
def test_quantize_transformer_none_when_unsupported(monkeypatch):
monkeypatch.setattr(
tq, "select_transformer_quant_scheme", lambda target, mode, family = None, **_kw: None
)
pipe = types.SimpleNamespace(transformer = types.SimpleNamespace())
assert quantize_transformer(pipe, _target(), mode = "auto") is None
def test_quantize_transformer_tolerates_failure(monkeypatch):
monkeypatch.setattr(
tq, "select_transformer_quant_scheme", lambda target, mode, family = None, **_kw: TQ_INT8
)
monkeypatch.setattr(tq, "_make_quant_config", lambda scheme: "cfg")
tqz = types.ModuleType("torchao.quantization")
def _boom(
module,
config,
filter_fn = None,
):
raise RuntimeError("partial quant failure")
tqz.quantize_ = _boom
monkeypatch.setitem(sys.modules, "torchao.quantization", tqz)
pipe = types.SimpleNamespace(transformer = types.SimpleNamespace())
assert quantize_transformer(pipe, _target(), mode = "int8") is None
def test_family_deny_auto_skips_mx_and_nvfp4_for_qwen(monkeypatch):
_stub_torch(monkeypatch, cc = (10, 0))
_allow(monkeypatch, {TQ_FP8, TQ_NVFP4, TQ_MXFP8, TQ_INT8})
assert select_transformer_quant_scheme(_target(), "auto", family = "qwen-image") == TQ_INT8
assert select_transformer_quant_scheme(_target(), "auto", family = "qwen-image-edit") == TQ_INT8
# fp8 is not denied: activation_value_lb fixed the black frames.
_allow(monkeypatch, {TQ_FP8, TQ_NVFP4, TQ_MXFP8})
assert select_transformer_quant_scheme(_target(), "auto", family = "qwen-image") == TQ_FP8
_allow(monkeypatch, {TQ_NVFP4, TQ_MXFP8})
assert select_transformer_quant_scheme(_target(), "auto", family = "qwen-image") is None
def test_family_deny_refuses_explicit_mxfp8_and_nvfp4_for_qwen(monkeypatch):
_stub_torch(monkeypatch, cc = (10, 0))
_allow(monkeypatch, {TQ_FP8, TQ_MXFP8, TQ_NVFP4, TQ_INT8})
assert select_transformer_quant_scheme(_target(), "mxfp8", family = "qwen-image") is None
assert select_transformer_quant_scheme(_target(), "nvfp4", family = "qwen-image") is None
assert select_transformer_quant_scheme(_target(), "fp8", family = "qwen-image") == TQ_FP8
assert select_transformer_quant_scheme(_target(), "int8", family = "qwen-image") == TQ_INT8
assert select_transformer_quant_scheme(_target(), "fp8", family = "z-image") == TQ_FP8
def test_family_deny_no_family_keeps_ladder(monkeypatch):
_stub_torch(monkeypatch, cc = (10, 0))
_allow(monkeypatch, {TQ_FP8, TQ_INT8})
assert select_transformer_quant_scheme(_target(), "auto") == TQ_INT8
assert select_transformer_quant_scheme(_target(), "auto", family = "sdxl") == TQ_INT8
def test_quantize_transformer_threads_family(monkeypatch):
# mxfp8, not fp8: fp8 on qwen-image is not denied, so it would reach torchao and prove nothing.
_stub_torch(monkeypatch, cc = (10, 0))
_allow(monkeypatch, {TQ_FP8, TQ_INT8})
pipe = types.SimpleNamespace(transformer = types.SimpleNamespace())
called = {}
tqz = types.ModuleType("torchao.quantization")
def _quantize(
module,
config,
filter_fn = None,
):
called["scheme"] = True
tqz.quantize_ = _quantize
tqz.Int8DynamicActivationInt8WeightConfig = lambda: "int8-cfg"
tqz.Float8DynamicActivationFloat8WeightConfig = lambda **kw: "fp8-cfg"
tqz.PerRow = lambda: "per-row"
monkeypatch.setitem(sys.modules, "torchao.quantization", tqz)
assert quantize_transformer(pipe, _target(), mode = "mxfp8", family = "qwen-image") is None
assert called == {}
def test_quantize_transformer_threads_base_repo(monkeypatch):
import core.inference.diffusion_transformer_quant as dtq
seen = {}
def _select(
target,
mode,
family = None,
**kwargs,
):
seen.update(kwargs, family = family)
return None
monkeypatch.setattr(dtq, "select_transformer_quant_scheme", _select)
pipe = types.SimpleNamespace(transformer = types.SimpleNamespace())
base = "Qwen/Qwen-Image-2512"
assert (
quantize_transformer(pipe, _target(), mode = "nvfp4", family = "qwen-image", base_repo = base)
is None
)
assert seen == {"family": "qwen-image", "base_repo": base}
def test_the_attention_trim_families_exclude_their_small_m_text_streams():
"""Small-M text streams must be excluded: M = 0 is unprojected and M <= 16 trips torch._int_mm."""
from core.inference.diffusion_transformer_quant import TQ_INT8, exclude_tokens_for_scheme
for family in ("hunyuanvideo-1.5", "hunyuanvideo-1.5-720p"):
tokens = exclude_tokens_for_scheme(TQ_INT8, family)
for name in (
"context_embedder", # also matches context_embedder_2, by substring
"image_embedder",
"add_q_proj",
"add_k_proj",
"add_v_proj",
"to_add_out",
"ff_context",
):
assert name in tokens, f"{family} must exclude {name}"
from core.inference.diffusion_transformer_quant import _INT8_EXCLUDE_NAME_TOKENS
assert exclude_tokens_for_scheme("fp8", "hunyuanvideo-1.5") == ()
# Not ltx-2: a video-only run feeds a one-token audio stream that hits the M floor.
assert exclude_tokens_for_scheme(TQ_INT8, "flux.1") == _INT8_EXCLUDE_NAME_TOKENS
assert exclude_tokens_for_scheme(TQ_INT8, None) == _INT8_EXCLUDE_NAME_TOKENS
def test_minimax_h3_int8_excludes_its_adaln_projection():
"""adaln_proj needs an explicit exclude; generic tokens miss it and int8 then fails at M = 1."""
from core.inference.diffusion_transformer_quant import TQ_INT8, exclude_tokens_for_scheme
assert "adaln_proj" in exclude_tokens_for_scheme(TQ_INT8, "minimax-h3")
# Fails if a generic token starts matching adaln_proj, so the family entry can be reconsidered.
from core.inference.diffusion_transformer_quant import _INT8_EXCLUDE_NAME_TOKENS
assert not any(t in "adaln_proj" for t in _INT8_EXCLUDE_NAME_TOKENS)
assert exclude_tokens_for_scheme("fp8", "minimax-h3") == ()
def test_minimax_h3_pads_its_text_stream_instead_of_excluding_it():
"""Text-stream layers are padded to 32 rows, not excluded; per-row padding is bitwise exact."""
from core.inference.diffusion_transformer_quant import (
TQ_INT8,
exclude_tokens_for_scheme,
pad_tokens_for_scheme,
)
pad = pad_tokens_for_scheme(TQ_INT8, "minimax-h3")
exclude = exclude_tokens_for_scheme(TQ_INT8, "minimax-h3")
for name in ("context_embedder", "token_refiner"):
assert name in pad, f"minimax-h3 int8 must pad {name}"
assert name not in exclude, f"{name} is padded, so excluding it would quantise nothing"
def test_pad_and_exclude_sets_never_overlap():
"""An excluded Linear is never quantized, so there would be nothing to pad. A name in both
lists means one of them is dead, and which one is dead is not visible at the call site."""
from core.inference.diffusion_transformer_quant import (
_INT8_FAMILY_PAD_NAME_TOKENS,
TQ_INT8,
exclude_tokens_for_scheme,
pad_tokens_for_scheme,
)
for family in _INT8_FAMILY_PAD_NAME_TOKENS:
exclude = exclude_tokens_for_scheme(TQ_INT8, family)
for pad_token in pad_tokens_for_scheme(TQ_INT8, family):
assert not any(
e in pad_token or pad_token in e for e in exclude
), f"{family}: {pad_token!r} is both padded and excluded"
def test_only_minimax_h3_pads_today(monkeypatch):
"""Only MiniMax-H3 pads its own tokens; streamed qwen-image builds pad the text stream they
quantise, while resident builds and the kill switch pad nothing."""
from core.inference.diffusion_transformer_quant import (
INT8_TEXT_STREAM_ENV,
TQ_INT8,
pad_tokens_for_scheme,
runtime_quant_tokens_for_scheme,
)
from core.inference.diffusion_transformer_quant import int8_text_stream_scope
monkeypatch.delenv(INT8_TEXT_STREAM_ENV, raising = False)
for family in ("qwen-image", "qwen-image-edit", "hunyuanvideo-1.5", "hunyuanvideo-1.5-720p"):
assert pad_tokens_for_scheme(TQ_INT8, family) == ()
with int8_text_stream_scope(True):
assert pad_tokens_for_scheme(TQ_INT8, "qwen-image") == runtime_quant_tokens_for_scheme(
TQ_INT8, "qwen-image"
)
assert pad_tokens_for_scheme(TQ_INT8, "qwen-image")
monkeypatch.setenv(INT8_TEXT_STREAM_ENV, "0")
assert pad_tokens_for_scheme(TQ_INT8, "qwen-image") == ()
monkeypatch.delenv(INT8_TEXT_STREAM_ENV)
assert pad_tokens_for_scheme(TQ_INT8, "z-image") == ()
assert pad_tokens_for_scheme(TQ_INT8, None) == ()
def test_only_int8_pads():
"""``_int_mm``'s row floor is int8's alone: scaled_mm and the MX/FP4 kernels have no
equivalent, so no other scheme should be paying for a pad-and-slice."""
from core.inference.diffusion_transformer_quant import pad_tokens_for_scheme
for scheme in ("fp8", "nvfp4", "mxfp8", "auto"):
assert pad_tokens_for_scheme(scheme, "minimax-h3") == ()
def test_quantize_transformer_pads_after_quantising(monkeypatch):
"""Padding also runs on the runtime dense-quantise path, after quantize_, since it reparents
Linears that must already hold quantized weights."""
_stub_torch(monkeypatch, cc = (10, 0))
_allow(monkeypatch, {TQ_INT8})
order = []
tqz = types.ModuleType("torchao.quantization")
tqz.quantize_ = lambda module, config, filter_fn = None: order.append("quantize_")
tqz.Int8DynamicActivationInt8WeightConfig = lambda: "int8-cfg"
tqz.Float8DynamicActivationFloat8WeightConfig = lambda **kw: "fp8-cfg"
tqz.PerRow = lambda: "per-row"
monkeypatch.setitem(sys.modules, "torchao.quantization", tqz)
monkeypatch.setattr(
tq,
"apply_small_m_padding",
lambda transformer, scheme, family = None, logger = None: (
order.append(("pad", scheme, family)) or ()
),
)
pipe = types.SimpleNamespace(transformer = types.SimpleNamespace())
assert quantize_transformer(pipe, _target(), mode = "int8", family = "minimax-h3") == TQ_INT8
assert order == ["quantize_", ("pad", TQ_INT8, "minimax-h3")]
def test_quantize_transformer_refuses_when_padding_cannot_be_proven(monkeypatch):
"""A half-padded transformer crashes inside _int_mm, so a padding failure must fail the whole
quantise and fall back to GGUF, not be swallowed."""
_stub_torch(monkeypatch, cc = (10, 0))
_allow(monkeypatch, {TQ_INT8})
tqz = types.ModuleType("torchao.quantization")
tqz.quantize_ = lambda module, config, filter_fn = None: None
tqz.Int8DynamicActivationInt8WeightConfig = lambda: "int8-cfg"
tqz.Float8DynamicActivationFloat8WeightConfig = lambda **kw: "fp8-cfg"
tqz.PerRow = lambda: "per-row"
monkeypatch.setitem(sys.modules, "torchao.quantization", tqz)
def _boom(
transformer,
scheme,
family = None,
logger = None,
):
raise RuntimeError("cannot prove per-row granularity")
monkeypatch.setattr(tq, "apply_small_m_padding", _boom)
pipe = types.SimpleNamespace(transformer = types.SimpleNamespace())
assert quantize_transformer(pipe, _target(), mode = "int8", family = "minimax-h3") is None
assert not hasattr(pipe.transformer, "_unsloth_runtime_quant")
def test_apply_small_m_padding_is_inert_without_a_pad_list(monkeypatch):
"""Without a pad list diffusion_quant_pad is never imported, since it imports torch eagerly."""
from core.inference.diffusion_transformer_quant import TQ_INT8, apply_small_m_padding
stub = types.ModuleType("core.inference.diffusion_quant_pad")
monkeypatch.setitem(sys.modules, "core.inference.diffusion_quant_pad", stub)
assert apply_small_m_padding(object(), TQ_INT8, "z-image") == ()
assert apply_small_m_padding(object(), "fp8", "minimax-h3") == ()
assert apply_small_m_padding(object(), TQ_INT8, None) == ()
with pytest.raises(ImportError):
apply_small_m_padding(object(), TQ_INT8, "minimax-h3")
def test_the_zero_row_guard_is_nvfp4_only():
from core.inference.diffusion_transformer_quant import zero_row_tokens_for_scheme
for scheme in (TQ_FP8, TQ_INT8, TQ_MXFP8, "auto"):
for family in ("hunyuanvideo-1.5", "hunyuanvideo-1.5-720p"):
assert zero_row_tokens_for_scheme(scheme, family) == (), (scheme, family)
def test_both_hunyuan_tiers_guard_their_trimmable_streams():
from core.inference.diffusion_transformer_quant import zero_row_tokens_for_scheme
for family in ("hunyuanvideo-1.5", "hunyuanvideo-1.5-720p"):
tokens = zero_row_tokens_for_scheme(TQ_NVFP4, family)
assert "image_embedder" in tokens
assert "context_embedder_2" in tokens
def test_an_unlisted_family_has_no_zero_row_guard():
"""Scoped to the families whose attention path is measured to produce an empty activation."""
from core.inference.diffusion_transformer_quant import zero_row_tokens_for_scheme
for family in ("z-image", "qwen-image", "wan2.2-ti2v-5b", "minimax-h3", None):
assert zero_row_tokens_for_scheme(TQ_NVFP4, family) == ()
def test_the_guard_list_does_not_double_up_with_the_pad_or_exclude_lists():
from core.inference.diffusion_transformer_quant import (
_NVFP4_FAMILY_ZERO_ROW_NAME_TOKENS,
exclude_tokens_for_scheme,
pad_tokens_for_scheme,
zero_row_tokens_for_scheme,
)
for family in _NVFP4_FAMILY_ZERO_ROW_NAME_TOKENS:
guarded = zero_row_tokens_for_scheme(TQ_NVFP4, family)
assert guarded
assert exclude_tokens_for_scheme(TQ_NVFP4, family) == ()
assert pad_tokens_for_scheme(TQ_NVFP4, family) == ()
def test_quantize_transformer_guards_after_padding(monkeypatch):
_stub_torch(monkeypatch, cc = (10, 0))
_allow(monkeypatch, {TQ_NVFP4})
# The ordering under test, not the family's measured nvfp4 deny (HunyuanVideo-1.5 failed the gate).
monkeypatch.setattr(tq, "_FAMILY_SCHEME_DENY", {})
order = []
tqz = types.ModuleType("torchao.quantization")
tqz.quantize_ = lambda module, config, filter_fn = None: order.append("quantize_")
monkeypatch.setitem(sys.modules, "torchao.quantization", tqz)
monkeypatch.setattr(tq, "_make_quant_config", lambda scheme, fast_accum = None: "cfg")
monkeypatch.setattr(
tq,
"apply_small_m_padding",
lambda transformer, scheme, family = None, logger = None: (order.append("pad") or ()),
)
monkeypatch.setattr(
tq,
"apply_zero_row_guard",
lambda transformer, scheme, family = None, logger = None: (
order.append(("guard", scheme, family)) or ()
),
)
transformer = types.SimpleNamespace()
pipe = types.SimpleNamespace(transformer = transformer)
assert (
quantize_transformer(pipe, _target(), mode = "nvfp4", family = "hunyuanvideo-1.5") == TQ_NVFP4
)
assert order == ["quantize_", "pad", ("guard", TQ_NVFP4, "hunyuanvideo-1.5")]
assert transformer._unsloth_runtime_quant == TQ_NVFP4
def test_a_guard_failure_fails_the_whole_quantise(monkeypatch):
_stub_torch(monkeypatch, cc = (10, 0))
_allow(monkeypatch, {TQ_NVFP4})
tqz = types.ModuleType("torchao.quantization")
tqz.quantize_ = lambda module, config, filter_fn = None: None
monkeypatch.setitem(sys.modules, "torchao.quantization", tqz)
monkeypatch.setattr(tq, "_make_quant_config", lambda scheme, fast_accum = None: "cfg")
def _boom(
transformer,
scheme,
family = None,
logger = None,
):
raise RuntimeError("cannot resolve the guard list")
monkeypatch.setattr(tq, "apply_zero_row_guard", _boom)
pipe = types.SimpleNamespace(transformer = types.SimpleNamespace())
assert quantize_transformer(pipe, _target(), mode = "nvfp4", family = "hunyuanvideo-1.5") is None
assert not hasattr(pipe.transformer, "_unsloth_runtime_quant")
def test_apply_zero_row_guard_is_inert_without_a_guard_list(monkeypatch):
"""apply_zero_row_guard is inert, and imports nothing, without a guard list."""
from core.inference.diffusion_transformer_quant import apply_zero_row_guard
stub = types.ModuleType("core.inference.diffusion_quant_pad")
monkeypatch.setitem(sys.modules, "core.inference.diffusion_quant_pad", stub)
assert apply_zero_row_guard(object(), TQ_NVFP4, "z-image") == ()
assert apply_zero_row_guard(object(), TQ_FP8, "hunyuanvideo-1.5") == ()
assert apply_zero_row_guard(object(), TQ_NVFP4, None) == ()
with pytest.raises(ImportError):
apply_zero_row_guard(object(), TQ_NVFP4, "hunyuanvideo-1.5")
def test_the_training_deny_is_a_superset_of_the_inference_deny():
# Anything inference refuses, training must refuse too; never the other way round.
from core.inference.diffusion_transformer_quant import (
_FAMILY_SCHEME_DENY,
TQ_SCHEMES,
_family_denied,
_family_train_denied,
)
families = set(_FAMILY_SCHEME_DENY) | {"qwen-image", "qwen-image-edit", "z-image", "sdxl", ""}
for fam in families:
for scheme in TQ_SCHEMES:
if _family_denied(fam, scheme):
assert _family_train_denied(fam, scheme), (fam, scheme)
# qwen-image fp8 renders but is not cleared for training.
for fam in ("qwen-image", "qwen-image-edit"):
assert not _family_denied(fam, TQ_FP8)
assert _family_train_denied(fam, TQ_FP8)
assert not _family_train_denied(fam, TQ_INT8)
def test_auto_scheme_candidates_lists_the_whole_ladder_not_just_the_winner(monkeypatch):
# If the winner has no prequant and cannot fit dense, the loader needs the next rung, not GGUF.
from core.inference.diffusion_transformer_quant import auto_scheme_candidates
_stub_torch(monkeypatch, cc = (10, 0))
_allow(monkeypatch, {TQ_FP8, TQ_MXFP8, TQ_INT8})
assert auto_scheme_candidates(_target()) == (TQ_INT8, TQ_FP8, TQ_MXFP8)
assert auto_scheme_candidates(_target(), "qwen-image") == (TQ_INT8, TQ_FP8)
_allow(monkeypatch, {TQ_INT8})
assert auto_scheme_candidates(_target(), "qwen-image") == (TQ_INT8,)
assert auto_scheme_candidates(_target(device = "cpu")) == ()
def test_the_candidate_list_agrees_with_the_selector_on_the_winner(monkeypatch):
# The selector's answer must head the candidate list, or retry could propose a scheme auto refuses.
from core.inference.diffusion_transformer_quant import auto_scheme_candidates
for cc, allowed, family in (
((10, 0), {TQ_FP8, TQ_MXFP8, TQ_INT8}, None),
((10, 0), {TQ_FP8, TQ_MXFP8, TQ_INT8}, "qwen-image"),
((8, 9), {TQ_FP8, TQ_INT8}, "qwen-image-edit"),
((8, 0), {TQ_INT8}, None),
):
_stub_torch(monkeypatch, cc = cc)
_allow(monkeypatch, allowed)
chosen = select_transformer_quant_scheme(_target(), "auto", family = family)
candidates = auto_scheme_candidates(_target(), family)
assert (candidates[0] if candidates else None) == chosen, (cc, family)
def _prefer(
monkeypatch,
row,
family = "fake-video",
):
"""Install one ``_FAMILY_AUTO_PREFER`` row."""
monkeypatch.setattr(tq, "_FAMILY_AUTO_PREFER", {family: row})
def _nvfp4_head(**kw):
return tq._AutoPrefer(floor = (10, 0), schemes = (TQ_NVFP4,), **kw)
_HAS_PREQUANT = {"has_prequant": lambda scheme: True}
def test_a_prefer_row_leads_the_tier_on_datacenter_blackwell(monkeypatch):
_stub_torch(monkeypatch, cc = (10, 0))
_allow(monkeypatch, {TQ_NVFP4, TQ_FP8, TQ_MXFP8, TQ_INT8})
_prefer(monkeypatch, _nvfp4_head())
assert (
select_transformer_quant_scheme(_target(), "auto", family = "fake-video", **_HAS_PREQUANT)
== TQ_NVFP4
)
assert tq.auto_scheme_candidates(_target(), "fake-video", **_HAS_PREQUANT) == (
TQ_NVFP4,
TQ_INT8,
TQ_FP8,
TQ_MXFP8,
)
assert select_transformer_quant_scheme(_target(), "auto", family = "z-image") == TQ_INT8
assert tq.auto_scheme_candidates(_target()) == (TQ_INT8, TQ_FP8, TQ_MXFP8)
def test_a_prefer_row_is_dropped_below_its_capability_floor(monkeypatch):
_stub_torch(monkeypatch, cc = (8, 9), device_name = "NVIDIA L40S")
_allow(monkeypatch, {TQ_NVFP4, TQ_FP8, TQ_INT8})
_prefer(monkeypatch, _nvfp4_head())
assert select_transformer_quant_scheme(_target(), "auto", family = "fake-video") == TQ_INT8
assert tq.auto_scheme_candidates(_target(), "fake-video") == (TQ_INT8, TQ_FP8)
def test_a_head_the_probe_rejects_falls_through_to_the_tier(monkeypatch):
_stub_torch(monkeypatch, cc = (10, 0))
_allow(monkeypatch, {TQ_FP8, TQ_MXFP8, TQ_INT8})
_prefer(monkeypatch, _nvfp4_head())
assert select_transformer_quant_scheme(_target(), "auto", family = "fake-video") == TQ_INT8
assert tq.auto_scheme_candidates(_target(), "fake-video") == (TQ_INT8, TQ_FP8, TQ_MXFP8)
def test_the_deny_list_outranks_a_prefer_row(monkeypatch):
_stub_torch(monkeypatch, cc = (10, 0))
_allow(monkeypatch, {TQ_NVFP4, TQ_FP8, TQ_MXFP8, TQ_INT8})
_prefer(monkeypatch, _nvfp4_head(), family = "qwen-image")
assert select_transformer_quant_scheme(_target(), "auto", family = "qwen-image") == TQ_INT8
assert TQ_NVFP4 not in tq.auto_scheme_candidates(_target(), "qwen-image")
def test_a_prefer_row_needs_consumer_ok_to_apply_to_a_consumer_gpu(monkeypatch):
_stub_torch(monkeypatch, cc = (12, 0), device_name = "NVIDIA GeForce RTX 5090")
_allow(monkeypatch, {TQ_NVFP4, TQ_FP8, TQ_MXFP8, TQ_INT8})
_prefer(monkeypatch, _nvfp4_head())
assert (
select_transformer_quant_scheme(_target(), "auto", family = "fake-video", **_HAS_PREQUANT)
== TQ_INT8
)
_prefer(monkeypatch, _nvfp4_head(consumer_ok = True))
assert (
select_transformer_quant_scheme(_target(), "auto", family = "fake-video", **_HAS_PREQUANT)
== TQ_NVFP4
)
assert tq.auto_scheme_candidates(_target(), "fake-video", **_HAS_PREQUANT) == (
TQ_NVFP4,
TQ_INT8,
TQ_FP8,
TQ_MXFP8,
)
def test_a_head_already_in_the_tier_is_not_listed_twice(monkeypatch):
_stub_torch(monkeypatch, cc = (10, 0))
_allow(monkeypatch, {TQ_FP8, TQ_MXFP8, TQ_INT8})
_prefer(monkeypatch, tq._AutoPrefer(floor = (10, 0), schemes = (TQ_INT8,)))
assert tq.auto_scheme_candidates(_target(), "fake-video") == (TQ_INT8, TQ_FP8, TQ_MXFP8)
def test_a_capability_below_every_tier_has_no_order_even_with_a_head(monkeypatch):
_stub_torch(monkeypatch, cc = (7, 5))
_allow(monkeypatch, {TQ_NVFP4, TQ_FP8, TQ_INT8})
_prefer(monkeypatch, tq._AutoPrefer(floor = (7, 0), schemes = (TQ_NVFP4,)))
assert tq._auto_scheme_order("fake-video", "cuda", (7, 5)) == ()
assert select_transformer_quant_scheme(_target(), "auto", family = "fake-video") is None
assert tq.auto_scheme_candidates(_target(), "fake-video") == ()
@pytest.mark.parametrize("family", [None, "fake-video", "z-image", "qwen-image"])
@pytest.mark.parametrize(
"allowed",
[
{TQ_NVFP4, TQ_FP8, TQ_MXFP8, TQ_INT8},
{TQ_FP8, TQ_MXFP8, TQ_INT8},
{TQ_NVFP4, TQ_INT8},
{TQ_INT8},
set(),
],
)
@pytest.mark.parametrize("row_backend", [None, "flashinfer", "torchao"])
@pytest.mark.parametrize("probe_backend", ["flashinfer", "torchao", "raises"])
def test_the_candidate_head_stays_the_selector_winner_with_a_prefer_row(
monkeypatch, family, allowed, row_backend, probe_backend
):
_stub_torch(monkeypatch, cc = (10, 0))
_allow(monkeypatch, allowed)
_stub_nvfp4_backend(monkeypatch, probe_backend)
_prefer(monkeypatch, _nvfp4_head(backend = row_backend))
candidates = tq.auto_scheme_candidates(_target(), family, **_HAS_PREQUANT)
chosen = select_transformer_quant_scheme(_target(), "auto", family = family, **_HAS_PREQUANT)
assert (candidates[0] if candidates else None) == chosen, (family, allowed)
def test_the_shipped_prefer_table_keeps_the_ladder_as_it_was(monkeypatch):
_stub_torch(monkeypatch, cc = (10, 0))
_allow(monkeypatch, {TQ_NVFP4, TQ_FP8, TQ_MXFP8, TQ_INT8})
assert set(tq._FAMILY_AUTO_PREFER) == {
"z-image",
"flux.1",
"qwen-image",
"qwen-image-2.1",
"wan2.2-t2v-a14b",
}
# An nvfp4-bearing auto head needs a passing gate record, the image rule; none ships ungated.
ungated = {
name
for name, row in tq._FAMILY_AUTO_PREFER.items()
if not row.gated and TQ_NVFP4 in row.schemes
}
assert ungated == set()
assert tq._FAMILY_AUTO_PREFER["wan2.2-t2v-a14b"].backend == "flashinfer"
families = (
None,
"hunyuanvideo-1.5",
"hunyuanvideo-1.5-720p",
"wan2.2-ti2v-5b",
"z-image",
"flux.1",
)
for family in families:
assert (
select_transformer_quant_scheme(
_target(),
"auto",
family = family,
base_repo = "Tongyi-MAI/Z-Image-Turbo",
**_HAS_PREQUANT,
)
== TQ_INT8
)
assert select_transformer_quant_scheme(_target(), "auto", family = family) == TQ_INT8
assert tq.auto_scheme_candidates(
_target(), "qwen-image", base_repo = "Qwen/Qwen-Image", **_HAS_PREQUANT
) == (TQ_INT8, TQ_FP8)
def _stub_nvfp4_backend(monkeypatch, answer):
"""Make ``select_nvfp4_backend`` answer ``answer``, or raise when it is "raises"."""
from core.inference import diffusion_nvfp4_ops as ops
def _answer(device = None):
if answer == "raises":
raise RuntimeError("probe blew up")
return answer
monkeypatch.setattr(ops, "select_nvfp4_backend", _answer)
def test_a_backend_conditional_head_applies_on_the_backend_it_was_measured_on(monkeypatch):
_stub_torch(monkeypatch, cc = (10, 0))
_allow(monkeypatch, {TQ_NVFP4, TQ_FP8, TQ_MXFP8, TQ_INT8})
_stub_nvfp4_backend(monkeypatch, "flashinfer")
_prefer(monkeypatch, _nvfp4_head(backend = "flashinfer"))
assert (
select_transformer_quant_scheme(_target(), "auto", family = "fake-video", **_HAS_PREQUANT)
== TQ_NVFP4
)
assert tq.auto_scheme_candidates(_target(), "fake-video", **_HAS_PREQUANT) == (
TQ_NVFP4,
TQ_INT8,
TQ_FP8,
TQ_MXFP8,
)
def test_a_backend_conditional_head_is_dropped_on_the_other_backend(monkeypatch):
_stub_torch(monkeypatch, cc = (10, 0))
_allow(monkeypatch, {TQ_NVFP4, TQ_FP8, TQ_MXFP8, TQ_INT8})
_stub_nvfp4_backend(monkeypatch, "torchao")
_prefer(monkeypatch, _nvfp4_head(backend = "flashinfer"))
assert (
select_transformer_quant_scheme(_target(), "auto", family = "fake-video", **_HAS_PREQUANT)
== TQ_INT8
)
assert tq.auto_scheme_candidates(_target(), "fake-video", **_HAS_PREQUANT) == (
TQ_INT8,
TQ_FP8,
TQ_MXFP8,
)
def test_a_backend_probe_that_raises_drops_the_head_rather_than_the_load(monkeypatch):
_stub_torch(monkeypatch, cc = (10, 0))
_allow(monkeypatch, {TQ_NVFP4, TQ_FP8, TQ_MXFP8, TQ_INT8})
_stub_nvfp4_backend(monkeypatch, "raises")
_prefer(monkeypatch, _nvfp4_head(backend = "flashinfer"))
assert tq._nvfp4_backend_is("cuda", "flashinfer") is False
assert (
select_transformer_quant_scheme(_target(), "auto", family = "fake-video", **_HAS_PREQUANT)
== TQ_INT8
)
assert tq.auto_scheme_candidates(_target(), "fake-video", **_HAS_PREQUANT) == (
TQ_INT8,
TQ_FP8,
TQ_MXFP8,
)
def test_the_backend_helper_answers_false_without_torch(monkeypatch):
monkeypatch.setitem(sys.modules, "torch", None)
assert tq._nvfp4_backend_is("cuda", "flashinfer") is False
def test_a_row_with_no_backend_requirement_is_unconditional(monkeypatch):
_stub_torch(monkeypatch, cc = (10, 0))
_allow(monkeypatch, {TQ_NVFP4, TQ_FP8, TQ_MXFP8, TQ_INT8})
_stub_nvfp4_backend(monkeypatch, "torchao")
_prefer(monkeypatch, _nvfp4_head())
assert (
select_transformer_quant_scheme(_target(), "auto", family = "fake-video", **_HAS_PREQUANT)
== TQ_NVFP4
)
def test_the_floor_and_the_consumer_rule_still_outrank_the_backend(monkeypatch):
_allow(monkeypatch, {TQ_NVFP4, TQ_FP8, TQ_MXFP8, TQ_INT8})
_stub_nvfp4_backend(monkeypatch, "flashinfer")
_prefer(monkeypatch, _nvfp4_head(backend = "flashinfer"))
_stub_torch(monkeypatch, cc = (8, 9), device_name = "NVIDIA L40S")
assert (
select_transformer_quant_scheme(_target(), "auto", family = "fake-video", **_HAS_PREQUANT)
== TQ_INT8
)
_stub_torch(monkeypatch, cc = (12, 0), device_name = "NVIDIA GeForce RTX 5090")
assert (
select_transformer_quant_scheme(_target(), "auto", family = "fake-video", **_HAS_PREQUANT)
== TQ_INT8
)
def _gate_lifted(monkeypatch, passed = True):
"""A passing gate record on every backend (or none): what a measured A14B record would read as."""
monkeypatch.setattr(tq, "_nvfp4_gate_passed", lambda family, base_repo: passed)
monkeypatch.setattr(tq, "_nvfp4_gate_backend_ok", lambda family, base_repo, device: passed)
def test_the_a14b_row_does_not_lead_with_nvfp4_without_a_gate_record(monkeypatch):
_stub_torch(monkeypatch, cc = (10, 0))
_allow(monkeypatch, {TQ_NVFP4, TQ_FP8, TQ_MXFP8, TQ_INT8})
base = "Wan-AI/Wan2.2-T2V-A14B-Diffusers"
_stub_nvfp4_backend(monkeypatch, "flashinfer")
assert (
select_transformer_quant_scheme(
_target(), "auto", family = "wan2.2-t2v-a14b", base_repo = base, **_HAS_PREQUANT
)
== TQ_INT8
)
assert tq.auto_scheme_candidates(
_target(), "wan2.2-t2v-a14b", base_repo = base, **_HAS_PREQUANT
) == (TQ_INT8, TQ_FP8, TQ_MXFP8)
def test_the_a14b_row_leads_with_nvfp4_only_where_a_gate_record_and_flashinfer_serve_it(
monkeypatch,
):
_stub_torch(monkeypatch, cc = (10, 0))
_allow(monkeypatch, {TQ_NVFP4, TQ_FP8, TQ_MXFP8, TQ_INT8})
_gate_lifted(monkeypatch)
base = "Wan-AI/Wan2.2-T2V-A14B-Diffusers"
_stub_nvfp4_backend(monkeypatch, "flashinfer")
assert (
select_transformer_quant_scheme(
_target(), "auto", family = "wan2.2-t2v-a14b", base_repo = base, **_HAS_PREQUANT
)
== TQ_NVFP4
)
assert tq.auto_scheme_candidates(
_target(), "wan2.2-t2v-a14b", base_repo = base, **_HAS_PREQUANT
) == (TQ_NVFP4, TQ_INT8, TQ_FP8, TQ_MXFP8)
assert (
select_transformer_quant_scheme(
_target(),
"auto",
family = "wan2.2-t2v-a14b",
base_repo = base,
has_prequant = lambda scheme: False,
)
== TQ_INT8
)
_stub_nvfp4_backend(monkeypatch, "torchao")
assert (
select_transformer_quant_scheme(
_target(), "auto", family = "wan2.2-t2v-a14b", base_repo = base, **_HAS_PREQUANT
)
== TQ_INT8
)
def test_the_a14b_promotion_does_not_touch_the_explicit_request_path(monkeypatch):
_stub_torch(monkeypatch, cc = (10, 0))
base = "Wan-AI/Wan2.2-T2V-A14B-Diffusers"
for backend in ("flashinfer", "torchao", "raises"):
_stub_nvfp4_backend(monkeypatch, backend)
_allow(monkeypatch, {TQ_NVFP4, TQ_FP8, TQ_MXFP8, TQ_INT8})
for scheme in (TQ_NVFP4, TQ_FP8, TQ_INT8):
assert (
select_transformer_quant_scheme(
_target(),
scheme,
family = "wan2.2-t2v-a14b",
base_repo = base,
has_prequant = lambda candidate: False,
)
== scheme
), (backend, scheme)
_allow(monkeypatch, {TQ_FP8})
assert (
select_transformer_quant_scheme(
_target(), TQ_NVFP4, family = "wan2.2-t2v-a14b", base_repo = base
)
is None
)
def test_the_other_three_measured_video_families_were_not_promoted(monkeypatch):
_stub_torch(monkeypatch, cc = (10, 0))
_allow(monkeypatch, {TQ_NVFP4, TQ_FP8, TQ_MXFP8, TQ_INT8})
for family in ("wan2.2-ti2v-5b", "hunyuanvideo-1.5", "hunyuanvideo-1.5-720p"):
assert family not in tq._FAMILY_AUTO_PREFER
for family in (None, "hunyuanvideo-1.5", "hunyuanvideo-1.5-720p", "wan2.2-ti2v-5b"):
assert select_transformer_quant_scheme(_target(), "auto", family = family) == TQ_INT8
def test_divisible_for_scheme_matches_each_gemm():
"""scaled_mm (fp8, and torchao's nvfp4 path) needs 16-aligned dims and MX block scaling 32."""
from core.inference.diffusion_transformer_quant import divisible_for_scheme
assert divisible_for_scheme(TQ_FP8) == 16
assert divisible_for_scheme(TQ_NVFP4) == 16
assert divisible_for_scheme(TQ_MXFP8) == 32
assert divisible_for_scheme(TQ_INT8) == 0
assert divisible_for_scheme("auto") == 0
def test_quantize_transformer_filters_on_the_scheme_alignment(monkeypatch):
torch_stub = _stub_torch(monkeypatch, cc = (10, 0))
class _Linear:
def __init__(self, in_features, out_features):
self.in_features, self.out_features = in_features, out_features
self.weight = types.SimpleNamespace(dtype = torch_stub.bfloat16)
torch_stub.nn = types.SimpleNamespace(Linear = _Linear)
monkeypatch.setattr(tq, "_make_quant_config", lambda scheme, fast_accum = None: "cfg")
tqz = types.ModuleType("torchao.quantization")
seen: list = []
tqz.quantize_ = lambda module, config, filter_fn = None: seen.append(filter_fn)
monkeypatch.setitem(sys.modules, "torchao.quantization", tqz)
filters = {}
for scheme in (TQ_NVFP4, TQ_MXFP8):
_allow(monkeypatch, {scheme})
pipe = types.SimpleNamespace(transformer = types.SimpleNamespace())
assert quantize_transformer(pipe, _target(), mode = scheme) == scheme
filters[scheme] = seen[-1]
aligned_16 = _Linear(528, 2048) # 528 % 16 == 0, 528 % 32 == 16
assert filters[TQ_NVFP4](aligned_16, "image_embedder.linear_2") is True
assert filters[TQ_MXFP8](aligned_16, "image_embedder.linear_2") is False
assert filters[TQ_NVFP4](_Linear(520, 2048), "blocks.0.ff.net.0") is False
def test_the_pre_eviction_gate_does_not_refuse_on_an_indeterminate_probe(monkeypatch):
# The route gate asks the selector, not the probe, so unproven_ok must reach through it.
monkeypatch.setattr(tq, "dense_transformer_supported", lambda target: True)
seen: list = []
def _supported(
scheme,
device,
*,
unproven_ok = False,
):
seen.append(unproven_ok)
return unproven_ok
monkeypatch.setattr(tq, "_scheme_supported", _supported)
target = types.SimpleNamespace(device = "cuda")
assert tq.select_transformer_quant_scheme(target, "fp8") is None
assert tq.select_transformer_quant_scheme(target, "fp8", unproven_ok = True) == "fp8"
assert seen == [False, True]
def test_a_refusal_reason_does_not_carry_server_paths(monkeypatch):
# The import error becomes the 409 detail and may name absolute paths; strip it, log it whole.
monkeypatch.setattr(tq, "_TORCHAO_UNAVAILABLE", None)
monkeypatch.setattr(tq, "is_stubbed", lambda pkg: False)
broken = types.ModuleType("torchao.quantization")
def _raise():
raise ImportError(
"cannot import name 'ScalingType' from 'torch.nn.functional' "
"(/srv/unsloth/.venv/lib/python3.12/site-packages/torch/nn/functional.py)"
)
broken.__getattr__ = lambda name: _raise()
monkeypatch.setitem(sys.modules, "torchao.quantization", broken)
reason = tq.torchao_unavailable_reason()
assert reason is not None
assert "/srv/unsloth" not in reason and "site-packages" not in reason
assert "ScalingType" in reason and "torch.nn.functional" in reason
def test_paths_are_stripped_without_eating_dotted_module_names():
assert tq._strip_paths("ImportError: no module 'torchao.quantization'") == (
"ImportError: no module 'torchao.quantization'"
)
assert "C:\\Users" not in tq._strip_paths(
r"ImportError: DLL load failed: C:\Users\me\.venv\Lib\site-packages\torchao\_C.pyd"
)
class _RecordingConfig:
"""Stands in for a torchao config dataclass: records every kwarg it was built with."""
built: list = []
def __init__(
self,
*,
set_inductor_config = True,
**kw,
):
self.set_inductor_config = set_inductor_config
self.kw = kw
type(self).built.append(self)
def _stub_torchao_configs(
monkeypatch,
*,
int8 = None,
fp8 = None,
):
tqz = types.ModuleType("torchao.quantization")
tqz.quantize_ = lambda *a, **k: None
tqz.PerRow = lambda: "per_row"
tqz.Int8DynamicActivationInt8WeightConfig = int8 or _RecordingConfig
tqz.Float8DynamicActivationFloat8WeightConfig = fp8 or _RecordingConfig
monkeypatch.setitem(sys.modules, "torchao.quantization", tqz)
monkeypatch.setitem(sys.modules, "torchao.float8", None)
monkeypatch.setitem(sys.modules, "torchao.quantization.quantize_", None)
return tqz
def test_int8_config_disables_torchao_inductor_config(monkeypatch):
_RecordingConfig.built = []
_stub_torchao_configs(monkeypatch)
cfg = tq._make_quant_config(TQ_INT8)
assert isinstance(cfg, _RecordingConfig)
assert cfg.set_inductor_config is False
assert cfg.kw == {}
def test_fp8_config_disables_torchao_inductor_config_and_keeps_its_kwargs(monkeypatch):
_RecordingConfig.built = []
_stub_torchao_configs(monkeypatch)
cfg = tq._make_quant_config(TQ_FP8)
assert isinstance(cfg, _RecordingConfig)
assert cfg.set_inductor_config is False
assert cfg.kw["granularity"] == "per_row"
def test_quiet_config_omits_the_kwarg_when_the_class_does_not_accept_it():
"""The mx_formats configs have no set_inductor_config, so the helper must not raise TypeError on them."""
class _NoKnob:
def __init__(self, use_triton_kernel = True):
self.use_triton_kernel = use_triton_kernel
cfg = tq._quiet_config(_NoKnob, use_triton_kernel = False)
assert cfg.use_triton_kernel is False and not hasattr(cfg, "set_inductor_config")
assert tq._quiet_config(lambda: "cfg") == "cfg"
def test_quiet_config_is_reenabled_by_env(monkeypatch):
"""UNSLOTH_TORCHAO_INDUCTOR_CONFIG=1 restores torchao's default so the two behaviours can be benchmarked A/B."""
monkeypatch.setenv("UNSLOTH_TORCHAO_INDUCTOR_CONFIG", "1")
assert tq._quiet_config(_RecordingConfig).set_inductor_config is True
monkeypatch.setenv("UNSLOTH_TORCHAO_INDUCTOR_CONFIG", "0")
assert tq._quiet_config(_RecordingConfig).set_inductor_config is False
monkeypatch.delenv("UNSLOTH_TORCHAO_INDUCTOR_CONFIG")
assert tq._quiet_config(_RecordingConfig).set_inductor_config is False
def test_quiet_config_tolerates_an_unintrospectable_class(monkeypatch):
import inspect
real = inspect.signature
def _boom(obj):
if obj is _RecordingConfig:
raise ValueError("no signature found for builtin")
return real(obj)
monkeypatch.setattr(tq._inspect, "signature", _boom)
cfg = tq._quiet_config(_RecordingConfig)
assert cfg.set_inductor_config is True
def test_real_torchao_configs_carry_set_inductor_config_false():
pytest.importorskip("torchao.quantization")
torch = pytest.importorskip("torch")
ic = getattr(getattr(torch, "_inductor", None), "config", None)
before = getattr(ic, "coordinate_descent_tuning", None) if ic is not None else None
for scheme in (TQ_INT8, TQ_FP8):
cfg = tq._make_quant_config(scheme)
if not hasattr(cfg, "set_inductor_config"):
pytest.skip("torchao config without set_inductor_config")
assert cfg.set_inductor_config is False, scheme
if ic is not None:
assert getattr(ic, "coordinate_descent_tuning", None) == before
def _policy_stub(monkeypatch, *, policy, applied, resolved):
from core.inference import diffusion_nvfp4_policy as np
def _resolve(family, base_repo = None):
resolved.append((family, base_repo))
return policy
def _apply(
transformer,
policy_arg,
*,
min_features = None,
fast_accum = None,
logger = None,
):
applied.append((transformer, policy_arg, min_features, fast_accum))
return {"blocks.0.attention.to_q": "nvfp4"}
monkeypatch.setattr(np, "resolve_policy", _resolve)
monkeypatch.setattr(np, "quantize_with_policy", _apply)
def _nvfp4_runtime(monkeypatch, order):
_stub_torch(monkeypatch, cc = (10, 0))
_allow(monkeypatch, {TQ_NVFP4})
tqz = types.ModuleType("torchao.quantization")
tqz.quantize_ = lambda module, config, filter_fn = None: order.append("quantize_")
monkeypatch.setitem(sys.modules, "torchao.quantization", tqz)
monkeypatch.setattr(tq, "_make_quant_config", lambda scheme, fast_accum = None: "cfg")
monkeypatch.setattr(
tq,
"apply_small_m_padding",
lambda transformer, scheme, family = None, logger = None: (order.append("pad") or ()),
)
monkeypatch.setattr(
tq,
"apply_zero_row_guard",
lambda transformer, scheme, family = None, logger = None: (order.append("guard") or ()),
)
def test_quantize_transformer_applies_the_policy_when_one_resolves(monkeypatch):
"""An explicit nvfp4 on a gated base quantises the policy's layers, never the whole model."""
order: list = []
_nvfp4_runtime(monkeypatch, order)
policy = types.SimpleNamespace(policy_id = "zimg_rg76_v1", version = 1)
applied: list = []
resolved: list = []
_policy_stub(monkeypatch, policy = policy, applied = applied, resolved = resolved)
transformer = types.SimpleNamespace()
pipe = types.SimpleNamespace(transformer = transformer)
engaged = quantize_transformer(
pipe,
_target(),
mode = "nvfp4",
family = "z-image",
base_repo = "Tongyi-MAI/Z-Image-Turbo",
min_features = 512,
)
assert engaged == TQ_NVFP4
assert resolved == [("z-image", "Tongyi-MAI/Z-Image-Turbo")]
assert len(applied) == 1 and applied[0][0] is transformer and applied[0][1] is policy
assert applied[0][2] == 512
assert order == ["pad", "guard"]
assert transformer._unsloth_runtime_quant == TQ_NVFP4
assert transformer._unsloth_nvfp4_policy == "zimg_rg76_v1"
def test_quantize_transformer_quantises_the_whole_model_without_a_policy(monkeypatch):
order: list = []
_nvfp4_runtime(monkeypatch, order)
applied: list = []
resolved: list = []
_policy_stub(monkeypatch, policy = None, applied = applied, resolved = resolved)
pipe = types.SimpleNamespace(transformer = types.SimpleNamespace())
assert quantize_transformer(pipe, _target(), mode = "nvfp4", family = "z-image") == TQ_NVFP4
assert resolved == [("z-image", None)]
assert applied == []
assert order == ["quantize_", "pad", "guard"]
assert not hasattr(pipe.transformer, "_unsloth_nvfp4_policy")
def test_quantize_transformer_only_asks_about_a_policy_for_nvfp4(monkeypatch):
"""fp8 on a base that HAS an nvfp4 policy is still whole-model fp8."""
order: list = []
_stub_torch(monkeypatch, cc = (10, 0))
_allow(monkeypatch, {TQ_FP8})
tqz = types.ModuleType("torchao.quantization")
tqz.quantize_ = lambda module, config, filter_fn = None: order.append("quantize_")
monkeypatch.setitem(sys.modules, "torchao.quantization", tqz)
monkeypatch.setattr(tq, "_make_quant_config", lambda scheme, fast_accum = None: "cfg")
resolved: list = []
_policy_stub(
monkeypatch,
policy = types.SimpleNamespace(policy_id = "zimg_rg76_v1", version = 1),
applied = [],
resolved = resolved,
)
pipe = types.SimpleNamespace(transformer = types.SimpleNamespace())
engaged = quantize_transformer(
pipe,
_target(),
mode = "fp8",
family = "z-image",
base_repo = "Tongyi-MAI/Z-Image-Turbo",
)
assert engaged == TQ_FP8
assert resolved == []
assert order == ["quantize_"]
def test_a_policy_mismatch_fails_the_whole_quantise(monkeypatch):
order: list = []
_nvfp4_runtime(monkeypatch, order)
from core.inference import diffusion_nvfp4_policy as np
monkeypatch.setattr(
np,
"resolve_policy",
lambda family, base_repo = None: types.SimpleNamespace(policy_id = "zimg_rg76_v1", version = 1),
)
def _boom(transformer, policy, **kwargs):
raise np.PolicyMismatch("rule 'attention.to_q' expects 34 layers, found 30")
monkeypatch.setattr(np, "quantize_with_policy", _boom)
pipe = types.SimpleNamespace(transformer = types.SimpleNamespace())
assert (
quantize_transformer(
pipe,
_target(),
mode = "nvfp4",
family = "z-image",
base_repo = "Tongyi-MAI/Z-Image-Turbo",
)
is None
)
assert order == []
assert not hasattr(pipe.transformer, "_unsloth_runtime_quant")
assert not hasattr(pipe.transformer, "_unsloth_nvfp4_policy")
_ZIMAGE_BASE = "Tongyi-MAI/Z-Image-Turbo"
_QWEN_BASE = "Qwen/Qwen-Image"
def _gate_row(family, base_repo, policy_id, **overrides):
row = {
"family": family,
"base_repo": base_repo,
"policy_id": policy_id,
"policy_version": 1,
"checkpoint_sha256": "b" * 64,
"all_pass": True,
"backend": "flashinfer",
}
row.update(overrides)
return row
def _gate(monkeypatch, tmp_path, *rows):
"""Point the gate module at a record file holding ``rows`` (none = the shipped, empty state)."""
import json
import core.inference.diffusion_nvfp4_gate as gate
path = tmp_path / "nvfp4_gate_record.json"
path.write_text(json.dumps({"version": 1, "records": list(rows)}), encoding = "utf-8")
monkeypatch.setattr(gate, "GATE_RECORD_PATH", path)
return path
def _zimage_candidates(
monkeypatch,
*,
has_prequant = True,
base_repo = _ZIMAGE_BASE,
):
return tq.auto_scheme_candidates(
_target(),
"z-image",
base_repo = base_repo,
has_prequant = (lambda scheme: True) if has_prequant else (lambda scheme: False),
)
def test_a_gated_row_is_inert_without_a_record_and_leads_with_one(monkeypatch, tmp_path):
_stub_torch(monkeypatch, cc = (10, 0))
_allow(monkeypatch, {TQ_NVFP4, TQ_FP8, TQ_MXFP8, TQ_INT8})
_stub_nvfp4_backend(monkeypatch, "flashinfer")
_gate(monkeypatch, tmp_path)
assert _zimage_candidates(monkeypatch) == (TQ_INT8, TQ_FP8, TQ_MXFP8)
_gate(monkeypatch, tmp_path, _gate_row("z-image", _ZIMAGE_BASE, "zimg_rg76_v1"))
assert _zimage_candidates(monkeypatch) == (TQ_INT8, TQ_FP8, TQ_NVFP4, TQ_MXFP8)
assert (
select_transformer_quant_scheme(
_target(),
"auto",
family = "z-image",
base_repo = _ZIMAGE_BASE,
has_prequant = lambda scheme: True,
)
== TQ_INT8
)
_allow(monkeypatch, {TQ_NVFP4, TQ_MXFP8})
assert (
select_transformer_quant_scheme(
_target(),
"auto",
family = "z-image",
base_repo = _ZIMAGE_BASE,
has_prequant = lambda scheme: True,
)
== TQ_NVFP4
)
def test_the_gated_head_stands_on_any_backend_a_passing_record_names(monkeypatch, tmp_path):
"""One policy can hold artifacts gated on either backend; neither may lose nvfp4 to row order."""
_stub_torch(monkeypatch, cc = (10, 0))
_allow(monkeypatch, {TQ_NVFP4, TQ_FP8, TQ_MXFP8, TQ_INT8})
_gate(
monkeypatch,
tmp_path,
_gate_row("z-image", _ZIMAGE_BASE, "zimg_rg76_v1"),
_gate_row(
"z-image",
_ZIMAGE_BASE,
"zimg_rg76_v1",
checkpoint_sha256 = "c" * 64,
gptq = True,
backend = "torchao",
),
)
for backend in ("flashinfer", "torchao"):
_stub_nvfp4_backend(monkeypatch, backend)
assert _zimage_candidates(monkeypatch) == (TQ_INT8, TQ_FP8, TQ_NVFP4, TQ_MXFP8), backend
_stub_nvfp4_backend(monkeypatch, "something-else")
assert _zimage_candidates(monkeypatch) == (TQ_INT8, TQ_FP8, TQ_MXFP8)
@pytest.mark.parametrize(
"row",
[
None,
_gate_row("z-image", _ZIMAGE_BASE, "zimg_rg76_v1", all_pass = False),
_gate_row("z-image", _ZIMAGE_BASE, "zimg_rg76_v1", policy_version = 2),
_gate_row("z-image", "some-org/Z-Image-Fork", "zimg_rg76_v1"),
_gate_row("flux.1", "black-forest-labs/FLUX.1-schnell", "flux_r420_v1"),
],
ids = ["absent", "failed", "version-bump", "another-base", "another-family"],
)
def test_nvfp4_stays_out_of_auto_unless_the_record_matches_exactly(monkeypatch, tmp_path, row):
_stub_torch(monkeypatch, cc = (10, 0))
_allow(monkeypatch, {TQ_NVFP4, TQ_FP8, TQ_MXFP8, TQ_INT8})
_gate(monkeypatch, tmp_path, *([] if row is None else [row]))
assert _zimage_candidates(monkeypatch) == (TQ_INT8, TQ_FP8, TQ_MXFP8)
def test_auto_will_not_offer_nvfp4_without_a_checkpoint_to_run(monkeypatch, tmp_path):
_stub_torch(monkeypatch, cc = (10, 0))
_allow(monkeypatch, {TQ_NVFP4, TQ_FP8, TQ_MXFP8, TQ_INT8})
_gate(monkeypatch, tmp_path, _gate_row("z-image", _ZIMAGE_BASE, "zimg_rg76_v1"))
assert _zimage_candidates(monkeypatch, has_prequant = False) == (TQ_INT8, TQ_FP8, TQ_MXFP8)
assert tq.auto_scheme_candidates(_target(), "z-image", base_repo = _ZIMAGE_BASE) == (
TQ_INT8,
TQ_FP8,
TQ_MXFP8,
)
def _boom(scheme):
raise RuntimeError("hub unreachable")
assert tq.auto_scheme_candidates(
_target(), "z-image", base_repo = _ZIMAGE_BASE, has_prequant = _boom
) == (TQ_INT8, TQ_FP8, TQ_MXFP8)
assert (
select_transformer_quant_scheme(
_target(), TQ_NVFP4, family = "z-image", base_repo = _ZIMAGE_BASE
)
== TQ_NVFP4
)
def test_a_record_lifts_the_qwen_nvfp4_deny_for_the_gated_base_only(monkeypatch, tmp_path):
_stub_torch(monkeypatch, cc = (10, 0))
_allow(monkeypatch, {TQ_NVFP4, TQ_FP8, TQ_MXFP8, TQ_INT8})
_stub_nvfp4_backend(monkeypatch, "flashinfer")
_gate(monkeypatch, tmp_path)
assert tq._family_denied("qwen-image", TQ_NVFP4, _QWEN_BASE) is True
assert (
select_transformer_quant_scheme(
_target(), TQ_NVFP4, family = "qwen-image", base_repo = _QWEN_BASE
)
is None
)
_gate(monkeypatch, tmp_path, _gate_row("qwen-image", _QWEN_BASE, "qwen_p02_v1"))
assert tq._family_denied("qwen-image", TQ_NVFP4, _QWEN_BASE) is False
assert (
select_transformer_quant_scheme(
_target(), TQ_NVFP4, family = "qwen-image", base_repo = _QWEN_BASE
)
== TQ_NVFP4
)
for family, base in (
("qwen-image", "Qwen/Qwen-Image-2509"),
("qwen-image-edit", "Qwen/Qwen-Image-Edit"),
("qwen-image", None),
):
assert tq._family_denied(family, TQ_NVFP4, base) is True, (family, base)
assert tq._family_denied("qwen-image", TQ_MXFP8, _QWEN_BASE) is True
assert tq.auto_scheme_candidates(
_target(), "qwen-image", base_repo = _QWEN_BASE, has_prequant = lambda scheme: True
) == (TQ_INT8, TQ_FP8, TQ_NVFP4)
def test_the_refusal_message_names_the_missing_gate_record(monkeypatch, tmp_path):
_stub_torch(monkeypatch, cc = (10, 0))
_gate(monkeypatch, tmp_path)
message = tq.explain_unusable_scheme("qwen-image", TQ_NVFP4, _QWEN_BASE)
assert "nvfp4_gate_record.json" in message and _QWEN_BASE in message
assert "measured accuracy gate" in tq.explain_unusable_scheme("qwen-image", TQ_MXFP8)
def test_training_is_denied_nvfp4_for_every_family_gated_or_not(monkeypatch, tmp_path):
from core.inference.diffusion_families import supported_family_names
_gate(monkeypatch, tmp_path, _gate_row("z-image", _ZIMAGE_BASE, "zimg_rg76_v1"))
assert tq._family_denied("z-image", TQ_NVFP4, _ZIMAGE_BASE) is False
for family in (*supported_family_names(), "wan2.2-ti2v-5b", "minimax-h3", None, ""):
assert tq._family_train_denied(family, TQ_NVFP4) is True, family
assert tq._family_train_denied(family, TQ_NVFP4, _ZIMAGE_BASE) is True, family
assert tq._family_train_denied("z-image", TQ_INT8) is False
assert tq._family_train_denied("z-image", TQ_FP8) is False
@pytest.mark.parametrize("has_prequant", [True, False])
@pytest.mark.parametrize("gated", [True, False])
@pytest.mark.parametrize(
"allowed",
[
{TQ_NVFP4, TQ_FP8, TQ_MXFP8, TQ_INT8},
{TQ_NVFP4, TQ_MXFP8, TQ_INT8},
{TQ_NVFP4},
{TQ_INT8},
set(),
],
)
def test_the_candidate_head_stays_the_selector_winner_under_the_gate(
monkeypatch, tmp_path, allowed, gated, has_prequant
):
_stub_torch(monkeypatch, cc = (10, 0))
_allow(monkeypatch, allowed)
_stub_nvfp4_backend(monkeypatch, "flashinfer")
rows = [_gate_row("z-image", _ZIMAGE_BASE, "zimg_rg76_v1")] if gated else []
_gate(monkeypatch, tmp_path, *rows)
kwargs = {"base_repo": _ZIMAGE_BASE, "has_prequant": lambda scheme: has_prequant}
candidates = tq.auto_scheme_candidates(_target(), "z-image", **kwargs)
chosen = select_transformer_quant_scheme(_target(), "auto", family = "z-image", **kwargs)
assert (candidates[0] if candidates else None) == chosen, (allowed, gated, has_prequant)
assert (TQ_NVFP4 in candidates) == (gated and has_prequant and TQ_NVFP4 in allowed)
@pytest.mark.parametrize(
("probe", "recorded", "offered"),
[
("flashinfer", "flashinfer", True),
("torchao", "flashinfer", False),
("flashinfer", "torchao", False),
("torchao", "torchao", True),
("raises", "flashinfer", False),
("flashinfer", None, False),
],
ids = [
"measured",
"unmeasured-torchao",
"unmeasured-flashinfer",
"torchao-record",
"unprobed",
"no-backend",
],
)
def test_a_gated_row_stands_only_on_the_backend_its_record_was_measured_on(
monkeypatch, tmp_path, probe, recorded, offered
):
"""The two NVFP4 backends quantise activations differently, so one verdict cannot enable both."""
_stub_torch(monkeypatch, cc = (10, 0))
_allow(monkeypatch, {TQ_NVFP4, TQ_FP8, TQ_MXFP8, TQ_INT8})
_stub_nvfp4_backend(monkeypatch, probe)
_gate(
monkeypatch,
tmp_path,
_gate_row("z-image", _ZIMAGE_BASE, "zimg_rg76_v1", backend = recorded),
)
candidates = _zimage_candidates(monkeypatch)
assert (TQ_NVFP4 in candidates) is offered
assert candidates[0] == TQ_INT8
_allow(monkeypatch, {TQ_NVFP4, TQ_MXFP8})
chosen = select_transformer_quant_scheme(
_target(),
"auto",
family = "z-image",
base_repo = _ZIMAGE_BASE,
has_prequant = lambda scheme: True,
)
assert chosen == (TQ_NVFP4 if offered else TQ_MXFP8)
def test_the_dense_quant_kinds_are_gguf_and_pipeline():
assert tq.DENSE_QUANT_KINDS == ("gguf", "pipeline")
assert dense_quant_supported_kind("gguf") is True
assert dense_quant_supported_kind("pipeline") is True
assert dense_quant_supported_kind("single_file") is False
assert dense_quant_supported_kind(" PIPELINE ") is True
assert dense_quant_supported_kind(None) is False
assert dense_quant_supported_kind("") is False
def test_the_unsupported_kind_reason_names_the_kind_and_the_two_that_work():
reason = dense_quant_unsupported_kind_reason("single_file")
assert "single_file" in reason
assert "GGUF and pipeline" in reason
assert "the precision its checkpoint carries" in reason
class _Denoiser:
"""A module as the gate reads one: parameters that report a dtype."""
def __init__(
self,
dtype = "torch.bfloat16",
**attrs,
) -> None:
self._params = [types.SimpleNamespace(dtype = dtype)]
for key, value in attrs.items():
setattr(self, key, value)
def parameters(self, recurse = True):
return iter(self._params)
def test_a_dense_bf16_pipeline_is_the_one_shape_that_passes():
pipe = types.SimpleNamespace(transformer = _Denoiser())
assert tq.dense_quant_blocker(pipe) is None
assert [attr for attr, _m in tq.denoiser_modules(pipe)] == ["transformer"]
def test_a_unet_pipeline_is_blocked_by_having_no_transformer():
blocker = tq.dense_quant_blocker(types.SimpleNamespace(unet = _Denoiser()))
assert blocker is not None and "UNet" in blocker
@pytest.mark.parametrize(
"dtype", ["torch.uint8", "torch.float8_e4m3fn", "torch.float16", "torch.int8"]
)
def test_a_pre_quantised_pipeline_is_blocked_by_its_parameter_dtypes(dtype):
blocker = tq.dense_quant_blocker(types.SimpleNamespace(transformer = _Denoiser(dtype)))
assert blocker is not None and dtype.split(".")[-1] in blocker
@pytest.mark.parametrize(
("attrs", "expected"),
[
({"_unsloth_runtime_quant": "int8"}, "already quantised"),
({"is_loaded_in_4bit": True}, "4-bit"),
({"is_loaded_in_8bit": True}, "8-bit"),
({"config": types.SimpleNamespace(quantization_config = object())}, "quantization_config"),
],
)
def test_a_declared_quantisation_blocks_it_too(attrs, expected):
blocker = tq.dense_quant_blocker(types.SimpleNamespace(transformer = _Denoiser(**attrs)))
assert blocker is not None and expected in blocker
def test_the_second_denoiser_is_enumerated_and_judged():
pipe = types.SimpleNamespace(
transformer = _Denoiser(), unconditional_transformer = _Denoiser("torch.float8_e4m3fn")
)
assert [attr for attr, _m in tq.denoiser_modules(pipe)] == [
"transformer",
"unconditional_transformer",
]
blocker = tq.dense_quant_blocker(pipe)
assert blocker is not None and "unconditional_transformer" in blocker
def test_the_denoiser_view_presents_an_arbitrary_attribute_as_the_transformer():
second = _Denoiser()
pipe = types.SimpleNamespace(transformer = _Denoiser(), unconditional_transformer = second, vae = "v")
view = tq.DenoiserView(pipe, "unconditional_transformer")
assert view.transformer is second
assert view.vae == "v"
def test_a_pipeline_that_cannot_be_walked_is_not_called_quantised():
class _Unwalkable:
def parameters(self, recurse = True):
raise RuntimeError("no")
assert tq.dense_quant_blocker(types.SimpleNamespace(transformer = _Unwalkable())) is None
def test_a_dequantised_source_blocks_the_quant_even_though_its_tensors_are_bf16():
widened = _Denoiser()
assert tq.dense_quant_blocker(types.SimpleNamespace(transformer = widened)) is None
tq.mark_source_precision(widened, "fp8")
blocker = tq.dense_quant_blocker(types.SimpleNamespace(transformer = widened))
assert blocker is not None
assert "fp8" in blocker and "widened to bf16" in blocker
def test_the_source_marker_is_best_effort_and_returns_the_module():
module = _Denoiser()
assert tq.mark_source_precision(module, "fp8") is module
class _Frozen:
__slots__ = ()
frozen = _Frozen()
assert tq.mark_source_precision(frozen, "fp8") is frozen
assert getattr(frozen, tq.SOURCE_PRECISION_ATTR, None) is None
def test_the_ideogram_fp8_loader_stamps_what_it_widened():
import pathlib
import core.inference.diffusion_ideogram4 as ideo
source = pathlib.Path(ideo.__file__).read_text(encoding = "utf-8")
assert 'mark_source_precision(model, "fp8")' in source
def _capable_host(
monkeypatch,
*,
torchao_reason = None,
cap = (8, 9),
):
"""Fake CUDA host past the arch floor; compile_eligible is forced true for namespace targets."""
from core.inference import diffusion_speed
monkeypatch.setattr(diffusion_speed, "compile_eligible", lambda target, **kw: True)
monkeypatch.setattr(tq, "dense_transformer_supported", lambda target: True)
monkeypatch.setattr(tq, "_capability", lambda ordinal = None: cap)
monkeypatch.setattr(tq, "_TORCHAO_UNAVAILABLE", (torchao_reason,))
monkeypatch.setattr(tq, "_smoke_cache_device_key", lambda device, ordinal = None: "cuda:0")
monkeypatch.setattr(tq, "_SMOKE_CACHE", {})
def test_a_capable_host_advertises_dense_quant(monkeypatch):
_capable_host(monkeypatch)
assert tq.dense_quant_host_capable(_target()) is True
def test_a_host_whose_torchao_cannot_import_advertises_nothing(monkeypatch):
"""Unimportable torchao must report no capability, else the picker advertises rows that fall back."""
_capable_host(monkeypatch, torchao_reason = "ImportError: cannot import name 'ScalingType'")
assert tq.dense_quant_host_capable(_target()) is False
def test_a_host_below_the_arch_floor_advertises_nothing(monkeypatch):
_capable_host(monkeypatch, cap = (7, 0))
assert tq.dense_quant_host_capable(_target()) is False
def test_an_unsupported_device_advertises_nothing(monkeypatch):
_capable_host(monkeypatch)
monkeypatch.setattr(tq, "dense_transformer_supported", lambda target: False)
assert tq.dense_quant_host_capable(_target(device = "cpu")) is False
def test_a_probed_failure_withdraws_the_capability(monkeypatch):
"""The arch floor is necessary, not sufficient: a probed failure must not stay advertised."""
_capable_host(monkeypatch)
assert tq.dense_quant_host_capable(_target()) is True
tq._SMOKE_CACHE.update({(TQ_FP8, "cuda:0"): False, (TQ_INT8, "cuda:0"): False})
assert tq.dense_quant_host_capable(_target()) is False
tq._SMOKE_CACHE[(TQ_INT8, "cuda:0")] = True
assert tq.dense_quant_host_capable(_target()) is True
def test_an_explicit_only_scheme_does_not_make_auto_capable(monkeypatch):
"""`auto` cannot pick nvfp4, so a host that runs only nvfp4 has nothing automatic to offer."""
_capable_host(monkeypatch, cap = (10, 0))
tq._SMOKE_CACHE.update(
{
(TQ_FP8, "cuda:0"): False,
(TQ_MXFP8, "cuda:0"): False,
(TQ_INT8, "cuda:0"): False,
(TQ_NVFP4, "cuda:0"): True,
}
)
assert tq.dense_quant_host_capable(_target()) is False
assert not any(TQ_NVFP4 in schemes for _floor, schemes in tq._AUTO_LADDER)
def test_the_capability_never_probes(monkeypatch):
"""A polled status route must not buy the CUDA context the in-process probe leaks."""
_capable_host(monkeypatch)
monkeypatch.setattr(
tq, "_smoke_probe", lambda *a, **k: pytest.fail("the status path probed in-process")
)
monkeypatch.setattr(
tq, "_child_probe_table", lambda device: pytest.fail("the status path spawned a child")
)
assert tq.dense_quant_host_capable(_target()) is True
def _write_shard(tmp_path, attr, dtype):
import torch
from safetensors.torch import save_file
sub = tmp_path / attr
sub.mkdir(parents = True, exist_ok = True)
save_file(
{"proj.weight": torch.zeros(4, 4, dtype = dtype)},
str(sub / "diffusion_pytorch_model.safetensors"),
)
def test_a_raw_fp8_checkpoint_is_recognised_from_its_header(tmp_path):
"""from_pretrained widens fp8 to bf16, so the header is the only surviving evidence."""
import torch
_write_shard(tmp_path, "transformer", torch.float8_e4m3fn)
assert tq.stored_denoiser_precision(str(tmp_path)) == "fp8"
def test_a_bf16_checkpoint_reports_nothing(tmp_path):
import torch
_write_shard(tmp_path, "transformer", torch.bfloat16)
assert tq.stored_denoiser_precision(str(tmp_path)) is None
def test_the_second_denoiser_is_read_too(tmp_path):
import torch
_write_shard(tmp_path, "transformer", torch.bfloat16)
_write_shard(tmp_path, "unconditional_transformer", torch.float8_e5m2)
assert tq.stored_denoiser_precision(str(tmp_path)) == "fp8"
def test_an_int8_checkpoint_is_recognised(tmp_path):
import torch
_write_shard(tmp_path, "transformer", torch.int8)
assert tq.stored_denoiser_precision(str(tmp_path)) == "int8"
@pytest.mark.parametrize("local_dir", [None, "", "/nonexistent/path"])
def test_no_snapshot_is_no_verdict(local_dir):
"""Without a local snapshot this must not guess, and must never reach the network."""
assert tq.stored_denoiser_precision(local_dir) is None
def _cached_snapshot(cache, repo_id, dtype):
snap = cache / f"models--{repo_id.replace('/', '--')}" / "snapshots" / "abc123"
_write_shard(snap, "transformer", dtype)
(cache / f"models--{repo_id.replace('/', '--')}" / "refs").mkdir()
(cache / f"models--{repo_id.replace('/', '--')}" / "refs" / "main").write_text("abc123")
def test_an_offline_hub_id_reads_its_cached_snapshot(tmp_path, monkeypatch):
"""An offline load passes a repo id (nothing staged); the cached snapshot still carries the headers."""
import torch
from huggingface_hub import constants
monkeypatch.setattr(constants, "HF_HUB_CACHE", str(tmp_path))
_cached_snapshot(tmp_path, "org/narrow-dit", torch.float8_e4m3fn)
_cached_snapshot(tmp_path, "org/wide-dit", torch.bfloat16)
assert tq.stored_denoiser_precision("org/narrow-dit") == "fp8"
assert tq.stored_denoiser_precision("org/wide-dit") is None
assert tq.stored_denoiser_precision("org/not-cached") is None
def test_an_offline_hub_id_reads_studios_live_cache(tmp_path, monkeypatch):
"""Studio can move its cache at runtime; the loaders read from there, so the probe must too."""
import torch
from huggingface_hub import constants
from utils import hf_cache_settings
live = tmp_path / "live"
monkeypatch.setattr(constants, "HF_HUB_CACHE", str(tmp_path / "import_time_default"))
monkeypatch.setattr(hf_cache_settings, "active_hf_hub_cache", lambda: str(live))
_cached_snapshot(live, "org/narrow-dit", torch.float8_e4m3fn)
assert tq.stored_denoiser_precision("org/narrow-dit") == "fp8"
def test_a_recovered_source_precision_blocks_the_quant(tmp_path):
"""The recovered marker feeds the same refusal the Ideogram loader's marker does."""
import torch
_write_shard(tmp_path, "transformer", torch.float8_e4m3fn)
widened = _Denoiser()
pipe = types.SimpleNamespace(transformer = widened)
assert tq.dense_quant_blocker(pipe) is None
tq.mark_source_precision(widened, tq.stored_denoiser_precision(str(tmp_path)))
blocker = tq.dense_quant_blocker(pipe)
assert blocker is not None and "fp8" in blocker and "widened to bf16" in blocker
def test_a_host_that_cannot_compile_advertises_nothing(monkeypatch):
"""The loader keeps a pipeline dense when nothing can compile it, so there is no path to sell.
Reachable on a Windows CUDA install with no Triton wheel, and under TORCHDYNAMO_DISABLE."""
from core.inference import diffusion_speed
_capable_host(monkeypatch)
monkeypatch.setattr(diffusion_speed, "compile_eligible", lambda target, **kw: False)
assert tq.dense_quant_host_capable(_target()) is False
monkeypatch.setattr(diffusion_speed, "compile_eligible", lambda target, **kw: True)
assert tq.dense_quant_host_capable(_target()) is True
def test_a_later_shard_still_reveals_a_narrow_source(tmp_path):
"""Narrow checkpoints keep norms and embeddings wide, so shard one can be entirely bf16."""
import torch
from safetensors.torch import save_file
sub = tmp_path / "transformer"
sub.mkdir(parents = True)
save_file(
{"norm.weight": torch.zeros(4, dtype = torch.bfloat16)},
str(sub / "diffusion_pytorch_model-00001-of-00002.safetensors"),
)
save_file(
{"proj.weight": torch.zeros(4, 4, dtype = torch.float8_e4m3fn)},
str(sub / "diffusion_pytorch_model-00002-of-00002.safetensors"),
)
assert tq.stored_denoiser_precision(str(tmp_path)) == "fp8"
def test_a_missing_nvfp4_prequant_is_named_instead_of_the_gpu(monkeypatch):
monkeypatch.setattr(tq, "_TORCHAO_UNAVAILABLE", (None,))
monkeypatch.setattr(tq, "family_denies_scheme", lambda *a, **k: False)
message = tq.explain_unusable_scheme("z-image", TQ_NVFP4, prequant_missing = True)
assert "needs a pre-quantized checkpoint" in message and "on this GPU" not in message
monkeypatch.setattr(tq, "family_denies_scheme", lambda *a, **k: True)
assert "accuracy-gate record" in tq.explain_unusable_scheme(
"qwen-image", TQ_NVFP4, prequant_missing = True
)
def test_the_loader_decline_asks_explain_unusable_scheme():
import inspect
from core.inference import diffusion
src = inspect.getsource(diffusion.DiffusionBackend)
assert 'is not usable for family "\n' not in src
assert "transformer_quant_decline = explain_unusable_scheme(" in src