Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

Studio: run INT8 on T4 and other Turing GPUs instead of refusing it (#17167)

Daniel Han3f5d01297be7
diff --git a/studio/backend/core/inference/diffusion.py b/studio/backend/core/inference/diffusion.py
index bb6e7b1c590..a0b070dde73 100644
--- a/studio/backend/core/inference/diffusion.py
+++ b/studio/backend/core/inference/diffusion.py
@@ -183,6 +183,7 @@ from .diffusion_flux3 import REFERENCE_AREA as FLUX3_REFERENCE_AREA
from .diffusion_small_host import (
INT8_DENOISER_NAMES,
decide_small_host,
+ enable_int8_act_ as small_host_enable_int8_act,
download_first_enabled as small_host_download_first,
engaged_on as small_host_engaged_on,
host_commit_mib,
@@ -196,6 +197,7 @@ from .diffusion_small_host import (
from .model_ids import hf_cache_repo_id
from .safetensors_pread import pread_available
from .diffusion_memory import (
+ pipe_small_host_int8,
commit_load_budget_fraction,
release_load_budget_fraction,
vram_budget_scope,
@@ -524,6 +526,8 @@ from .diffusion_transformer_quant import (
denoiser_modules,
explain_unusable_scheme,
family_denies_scheme,
+ fp16_card_int8_host,
+ fp16_card_int8_scheme,
mark_source_precision,
nvfp4_gated_base,
native_int8_act,
@@ -4501,6 +4505,9 @@ class DiffusionBackend:
):
if native_quant_scheme(target, pinned, family = getattr(fam, "name", None)) is None:
reason = explain_unusable_scheme(getattr(fam, "name", None), pinned)
+ elif model_kind == "pipeline" and pinned == TQ_INT8 and fp16_card_int8_host(target):
+ if fp16_card_int8_scheme(target, pinned, family = getattr(fam, "name", None)) is None:
+ reason = explain_unusable_scheme(getattr(fam, "name", None), pinned)
elif (
model_kind == "pipeline"
and pinned in NATIVE_OFFLOAD_SCHEMES
@@ -8640,6 +8647,9 @@ class DiffusionBackend:
TQ_INT8 if rdna_auto_w8a8 is not None else transformer_quant_pinned,
family = getattr(fam, "name", None),
)
+ or fp16_card_int8_scheme(
+ target, transformer_quant_pinned, family = getattr(fam, "name", None)
+ )
if kind == "pipeline"
else None
)
@@ -10226,6 +10236,26 @@ class DiffusionBackend:
if not hasattr(pending_cache_write, "start"):
pending_cache_write = None
self._raise_if_load_cancelled(_load_token)
+ if (
+ pipe is not None
+ and kind == "pipeline"
+ and transformer_quant_engaged is None
+ and native_scheme == TQ_INT8
+ and fp16_card_int8_host(target)
+ and pipe_small_host_int8(pipe)
+ ):
+ # Already per-row int8 (small-host route): re-quantising would only re-round the codes.
+ transformer_quant_engaged = TQ_INT8
+ act_linears = sum(
+ small_host_enable_int8_act(module)
+ for _attr, module in denoiser_modules(pipe)
+ )
+ logger.info(
+ "diffusion.transformer_quant: int8 runs on the small-host int8 weights (fp16-only "
+ "GPU, %s compute; W8A8 armed on %d linears for float32 compute)",
+ str(getattr(target, "dtype", None)).replace("torch.", ""),
+ act_linears,
+ )
# Quantise dense bf16 pipeline denoisers in place. The blocker excludes UNet and pre-quantised
# pipelines; offloaded plans stay dense unless whole-module.
if (
@@ -10256,7 +10286,10 @@ class DiffusionBackend:
None
if native_scheme is not None or native_offload_scheme is not None
else pipeline_quant_uncompilable
- ) or dense_quant_blocker(pipe)
+ ) or dense_quant_blocker(
+ pipe,
+ allow_float16 = native_scheme is not None and fp16_card_int8_host(target),
+ )
if pipeline_quant_blocker is not None:
logger.info(
"diffusion.transformer_quant: skipped (%s)", pipeline_quant_blocker
diff --git a/studio/backend/core/inference/diffusion_small_host.py b/studio/backend/core/inference/diffusion_small_host.py
index 5cad62161cf..25dc54b6baf 100644
--- a/studio/backend/core/inference/diffusion_small_host.py
+++ b/studio/backend/core/inference/diffusion_small_host.py
@@ -659,6 +659,20 @@ def int8_act_device_ok(device: Any) -> bool:
return ok
+def enable_int8_act_(module: Any) -> int:
+ """Let every int8-stored Linear of ``module`` run W8A8 (an explicit int8 request on an fp16-only card); returns
+ how many. The per-call gate is unchanged: float32 CUDA input on sm_75 with an exact ``_int_mm``, else dequantise.
+ ``UNSLOTH_DIFFUSION_SMALL_HOST_INT8_ACT=0`` keeps them dequantised."""
+ if int8_act_disabled() or _INT8_CLS is None:
+ return 0
+ count = 0
+ for sub in module.modules():
+ if isinstance(sub, _INT8_CLS):
+ sub.act_int8 = True
+ count += 1
+ return count
+
+
def _int8_linear_class():
import torch
import torch.nn.functional as F
diff --git a/studio/backend/core/inference/diffusion_transformer_quant.py b/studio/backend/core/inference/diffusion_transformer_quant.py
index b8409145a06..5156afa46a2 100644
--- a/studio/backend/core/inference/diffusion_transformer_quant.py
+++ b/studio/backend/core/inference/diffusion_transformer_quant.py
@@ -1052,8 +1052,9 @@ def stored_denoiser_precision(local_dir: Optional[str]) -> Optional[str]:
return None
-def _module_quantised_marker(module: Any) -> Optional[str]:
- """Explain why ``module`` is not a dense bf16 source, or return None."""
+def _module_quantised_marker(module: Any, *, allow_float16: bool = False) -> Optional[str]:
+ """Explain why ``module`` is not a dense bf16 source, or return None. ``allow_float16``: an fp16 pipeline on an
+ fp16-only card (``fp16_card_int8_host``), whose dense weights were cast to float16 on load."""
if getattr(module, "_unsloth_runtime_quant", None):
return "it is already quantised by this loader"
source = getattr(module, SOURCE_PRECISION_ATTR, None)
@@ -1071,14 +1072,14 @@ def _module_quantised_marker(module: Any) -> Optional[str]:
try:
for param in module.parameters(recurse = True):
name = str(getattr(param, "dtype", "")).rsplit(".", 1)[-1]
- if name not in _DENSE_PARAM_DTYPE_NAMES:
+ if name not in _DENSE_PARAM_DTYPE_NAMES and not (allow_float16 and name == "float16"):
return f"its weights are stored as {name}, not dense bfloat16"
except Exception: # noqa: BLE001 -- an unwalkable module is not evidence of anything
return None
return None
-def dense_quant_blocker(pipe: Any) -> Optional[str]:
+def dense_quant_blocker(pipe: Any, *, allow_float16: bool = False) -> Optional[str]:
"""Explain why a pipeline cannot be quantised in place, or return None."""
denoisers = denoiser_modules(pipe)
if not denoisers:
@@ -1087,7 +1088,7 @@ def dense_quant_blocker(pipe: Any) -> Optional[str]:
"dense torchao schemes do not cover)"
)
for attr, module in denoisers:
- marker = _module_quantised_marker(module)
+ marker = _module_quantised_marker(module, allow_float16 = allow_float16)
if marker is not None:
where = "its transformer" if attr == "transformer" else f"its {attr}"
return f"{where} is not a dense bf16 source: {marker}"
@@ -1216,6 +1217,52 @@ def native_quant_scheme(
return scheme
+FP16_CARD_INT8_ENV = "UNSLOTH_DIFFUSION_FP16_CARD_INT8"
+# sm70 has no int8 tensor cores; Ampere+ runs bf16 through torchao.
+_FP16_CARD_INT8_CAPS = ((7, 5), (8, 0))
+
+
+def fp16_card_int8_enabled() -> bool:
+ raw = (_os.environ.get(FP16_CARD_INT8_ENV) or "").strip().lower()
+ return raw not in ("0", "false", "no", "off")
+
+
+def fp16_card_int8_host(target: Any) -> bool:
+ """A Turing NVIDIA card running the pipeline in fp16 or fp32 (no bf16): torchao refuses it, but the torchao-free
+ int8 linear runs there (``torch._int_mm`` is cuBLASLt int8 with no capability gate). False on every bf16 GPU."""
+ if getattr(target, "device", None) != "cuda" or not fp16_card_int8_enabled():
+ return False
+ if torch_is_rocm() or is_stubbed("torchao"):
+ return False
+ try:
+ import torch
+ if getattr(target, "dtype", None) not in (torch.float16, torch.float32):
+ return False
+ except Exception:
+ return False
+ cap = _capability(getattr(target, "ordinal", None))
+ return cap is not None and _FP16_CARD_INT8_CAPS[0] <= cap < _FP16_CARD_INT8_CAPS[1]
+
+
+def fp16_card_int8_scheme(
+ target: Any,
+ requested: Optional[str],
+ family: Optional[str] = None,
+) -> Optional[str]:
+ """``int8`` for an explicit int8 request on an ``fp16_card_int8_host``, else None (AUTO is unchanged)."""
+ if normalize_transformer_quant(requested) != TQ_INT8 or not fp16_card_int8_host(target):
+ return None
+ if _family_denied(family, TQ_INT8):
+ return None
+ return TQ_INT8
+
+
+def fp16_card_int8_act(target: Any) -> bool:
+ """W8A8 for the explicit-int8 Turing build; ``int8_storage_host`` (also true there) answers only for checkpoint
+ int8 storage, which stays weight-only."""
+ return fp16_card_int8_host(target) and not int8_act_disabled()
+
+
# Checkpoint int8 codes on a GPU with no Studio int8 runtime (pre-Ampere fp16, XPU) stay int8, widened per layer.
# Unset: offloaded plans only (resident, the expanded DiT is faster); 1: resident too; 0: always expand.
INT8_STORAGE_ENV = "UNSLOTH_DIFFUSION_INT8_STORAGE"
@@ -2038,8 +2085,10 @@ def make_filter_fn(
*,
require_bf16: bool = False,
require_divisible: int = 0,
+ require_dtype: Any = None,
):
- """A non-bf16 Linear aborts the whole quantise pass; an unaligned one crashes the first matmul later."""
+ """A non-bf16 Linear aborts the whole quantise pass; an unaligned one crashes the first matmul later.
+ ``require_dtype``: keep only Linears of that dtype (the fp16-card int8 path's compute dtype)."""
def filter_fn(module: Any, fqn: str = "") -> bool:
try:
@@ -2067,6 +2116,10 @@ def make_filter_fn(
weight = getattr(module, "weight", None)
if weight is None or weight.dtype != torch.bfloat16:
return False
+ if require_dtype is not None:
+ weight = getattr(module, "weight", None)
+ if weight is None or weight.dtype != require_dtype:
+ return False
return True
return filter_fn
@@ -2087,6 +2140,11 @@ def quantize_transformer(
) -> Optional[str]:
"""Quantise ``pipe.transformer`` in place, returning the scheme or None (GGUF); ``base_repo`` picks the NVFP4 policy."""
native = native_quant_scheme(target, mode, family = family, offload = offload)
+ fp16_card = native is None and fp16_card_int8_scheme(target, mode, family) is not None
+ if fp16_card:
+ native = TQ_INT8
+ # native_int8_act is False on any fp16 CUDA target (int8_storage_host); this build is the W8A8 one.
+ act_int8 = fp16_card_int8_act(target)
if native is not None:
mps = _mps_bf16(target)
if act_int8 is None or mps:
@@ -2099,6 +2157,7 @@ def quantize_transformer(
act_int8 = bool(act_int8) and native == TQ_INT8,
logger = logger,
scale_dtype = getattr(target, "dtype", None) if mps else None,
+ compute_dtype = getattr(target, "dtype", None) if fp16_card else None,
)
scheme = select_transformer_quant_scheme(target, mode, family = family, base_repo = base_repo)
if scheme is None:
@@ -2192,7 +2251,9 @@ def _quantize_native(
act_int8: bool,
logger: Any,
scale_dtype: Any = None,
+ compute_dtype: Any = None,
) -> Optional[str]:
+ """``compute_dtype``: the pipeline dtype on an fp16-only card (fp16 / fp32); None keeps the bf16 filter."""
transformer = getattr(pipe, "transformer", None)
if transformer is None:
return None
@@ -2203,7 +2264,8 @@ def _quantize_native(
filter_fn = make_filter_fn(
min_features,
exclude_name_tokens = exclude_tokens_for_scheme(scheme, family, baked = True) + ("lora_",),
- require_bf16 = True,
+ require_bf16 = compute_dtype is None,
+ require_dtype = compute_dtype,
)
if not apply_native_weight_quant(
transformer,
diff --git a/studio/backend/tests/test_diffusion_fp16_card_int8.py b/studio/backend/tests/test_diffusion_fp16_card_int8.py
new file mode 100644
index 00000000000..841cf08695b
--- /dev/null
+++ b/studio/backend/tests/test_diffusion_fp16_card_int8.py
@@ -0,0 +1,361 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
+
+"""INT8 on fp16-only NVIDIA cards (Turing, T4): the torchao-free int8 linear instead of a refusal. Capability and
+ROCm are monkeypatched, so this runs without a GPU."""
+
+from __future__ import annotations
+
+import types
+
+import pytest
+
+torch = pytest.importorskip("torch")
+
+import core.inference.diffusion_native_quant as nq
+import core.inference.diffusion_transformer_quant as tq
+
+# Literal, not tq.FP16_CARD_INT8_ENV: the behavioural tests below must also run against a tree without the name.
+KILL_SWITCH = "UNSLOTH_DIFFUSION_FP16_CARD_INT8"
+
+
+def _target(dtype, device = "cuda"):
+ return types.SimpleNamespace(device = device, dtype = dtype)
+
+
+def nvidia(monkeypatch):
+ monkeypatch.setattr(tq, "torch_is_rocm", lambda: False)
+ monkeypatch.setattr(tq, "is_stubbed", lambda name: False)
+ monkeypatch.delenv(KILL_SWITCH, raising = False)
+
+
+def _cap(monkeypatch, cap):
+ monkeypatch.setattr(tq, "_capability", lambda ordinal = None: cap)
+
+
+class _Toy(torch.nn.Module):
+ def __init__(self, dim = 128):
+ super().__init__()
+ self.to_q = torch.nn.Linear(dim, dim)
+ self.ff = torch.nn.Linear(dim, 2 * dim)
+ self.out = torch.nn.Linear(2 * dim, dim)
+
+ def forward(self, x):
+ return self.out(torch.nn.functional.gelu(self.ff(self.to_q(x))))
+
+
[email protected]("dtype", [torch.float16, torch.float32])
+def test_turing_fp16_and_fp32_pipelines_take_int8(nvidia, monkeypatch, dtype):
+ _cap(monkeypatch, (7, 5))
+ target = _target(dtype)
+ assert tq.fp16_card_int8_host(target)
+ assert tq.fp16_card_int8_scheme(target, "int8") == "int8"
+ assert tq.fp16_card_int8_scheme(target, " INT8 ") == "int8"
+
+
[email protected]("scheme", [None, "auto", "none", "fp8", "mxfp8"])
+def test_only_an_explicit_int8_request_is_taken(nvidia, monkeypatch, scheme):
+ _cap(monkeypatch, (7, 5))
+ assert tq.fp16_card_int8_scheme(_target(torch.float16), scheme) is None
+
+
+ "cap", [(7, 0), (6, 1), (8, 0), (8, 6), (8, 9), (9, 0), (10, 0), (12, 0), None]
+)
[email protected]("dtype", [torch.float16, torch.float32, torch.bfloat16])
+def test_every_other_nvidia_card_is_unchanged(nvidia, monkeypatch, cap, dtype):
+ # sm70 has no int8 tensor cores; Ampere+ runs bf16 through torchao; None = no CUDA answer.
+ _cap(monkeypatch, cap)
+ target = _target(dtype)
+ assert not tq.fp16_card_int8_host(target)
+ assert tq.fp16_card_int8_scheme(target, "int8") is None
+
+
+def test_a_bf16_pipeline_on_turing_is_unchanged(nvidia, monkeypatch):
+ _cap(monkeypatch, (7, 5))
+ assert not tq.fp16_card_int8_host(_target(torch.bfloat16))
+
+
[email protected]("device", ["cpu", "mps", "xpu"])
+def test_non_cuda_devices_are_unchanged(nvidia, monkeypatch, device):
+ _cap(monkeypatch, (7, 5))
+ assert not tq.fp16_card_int8_host(_target(torch.float16, device = device))
+
+
+def test_rocm_and_the_torchao_stub_keep_their_own_path(monkeypatch):
+ _cap(monkeypatch, (7, 5))
+ monkeypatch.setattr(tq, "is_stubbed", lambda name: False)
+ monkeypatch.setattr(tq, "torch_is_rocm", lambda: True)
+ assert not tq.fp16_card_int8_host(_target(torch.float16))
+ monkeypatch.setattr(tq, "torch_is_rocm", lambda: False)
+ monkeypatch.setattr(tq, "is_stubbed", lambda name: name == "torchao")
+ assert not tq.fp16_card_int8_host(_target(torch.float16))
+
+
[email protected]("raw", ["0", "false", "off", "no"])
+def test_kill_switch_restores_the_refusal(nvidia, monkeypatch, raw):
+ _cap(monkeypatch, (7, 5))
+ monkeypatch.setenv(KILL_SWITCH, raw)
+ assert not tq.fp16_card_int8_host(_target(torch.float16))
+ assert tq.fp16_card_int8_scheme(_target(torch.float16), "int8") is None
+
+
+def test_the_family_deny_list_still_applies(nvidia, monkeypatch):
+ _cap(monkeypatch, (7, 5))
+ monkeypatch.setitem(tq._FAMILY_SCHEME_DENY, "toy-family", frozenset({tq.TQ_INT8}))
+ assert tq.fp16_card_int8_scheme(_target(torch.float16), "int8", family = "toy-family") is None
+
+
+def test_the_shared_native_route_is_not_widened(nvidia, monkeypatch):
+ # The single-file int8 loader and video read native_quant_scheme: Turing must not change their answer.
+ _cap(monkeypatch, (7, 5))
+ for dtype in (torch.float16, torch.float32):
+ assert not tq.native_quant_host(_target(dtype))
+ for offload in (False, True):
+ assert tq.native_quant_scheme(_target(dtype), "int8", offload = offload) is None
+
+
[email protected]("dtype", [torch.float16, torch.float32])
+def test_quantize_transformer_builds_native_int8_linears_in_the_pipeline_dtype(
+ nvidia, monkeypatch, dtype
+):
+ _cap(monkeypatch, (7, 5))
+ torch.manual_seed(0)
+ model = _Toy().to(dtype)
+ x = torch.randn(4, 128, dtype = dtype)
+ with torch.no_grad():
+ ref = model(x).float()
+ got = tq.quantize_transformer(
+ types.SimpleNamespace(transformer = model), _target(dtype), mode = "int8", min_features = 64
+ )
+ assert got == "int8"
+ for name in ("to_q", "ff", "out"):
+ layer = getattr(model, name)
+ assert nq.is_native_linear(layer), name
+ assert layer.compute_dtype == dtype
+ assert layer.weight_q.dtype == torch.int8
+ with torch.no_grad():
+ y = model(x)
+ assert y.dtype == dtype and torch.isfinite(y).all()
+ rel = ((y.float() - ref).norm() / ref.norm()).item()
+ assert rel < 0.05, rel
+
+
+def test_mixed_dtype_linears_keep_their_deliberate_fp32_layers(nvidia, monkeypatch):
+ _cap(monkeypatch, (7, 5))
+ model = _Toy().to(torch.float16)
+ model.out.float()
+ got = tq.quantize_transformer(
+ types.SimpleNamespace(transformer = model),
+ _target(torch.float16),
+ mode = "int8",
+ min_features = 64,
+ )
+ assert got == "int8"
+ assert nq.is_native_linear(model.to_q) and nq.is_native_linear(model.ff)
+ assert isinstance(model.out, torch.nn.Linear) and model.out.weight.dtype == torch.float32
+
+
+def test_quantize_transformer_on_ampere_fp16_still_declines(nvidia, monkeypatch):
+ _cap(monkeypatch, (8, 6))
+ model = _Toy().to(torch.float16)
+ got = tq.quantize_transformer(
+ types.SimpleNamespace(transformer = model),
+ _target(torch.float16),
+ mode = "int8",
+ min_features = 64,
+ )
+ assert got is None
+ assert isinstance(model.to_q, torch.nn.Linear) and not nq.is_native_linear(model.to_q)
+
+
+def _image_family(name):
+ from core.inference.diffusion_families import _FAMILIES
+ return next(f for f in _FAMILIES if f.name == name)
+
+
+def _image_gate(
+ monkeypatch,
+ fam,
+ pinned,
+ dtype,
+ *,
+ memory_mode = None,
+):
+ import core.inference.diffusion as d
+ monkeypatch.setattr(d, "effective_te_quant", lambda *a, **k: None)
+ d.DiffusionBackend._assert_precision_for_target(
+ None,
+ fam,
+ _target(dtype),
+ model_kind = "pipeline",
+ pinned = pinned,
+ te_mode = None,
+ memory_mode = memory_mode,
+ cpu_offload = False,
+ )
+
+
+ "family,dtype",
+ [
+ ("qwen-image-2.1", torch.float16),
+ ("flux.3-klein", torch.float32),
+ ("flux.3-klein", torch.float16),
+ ],
+)
[email protected]("memory_mode", [None, "low_vram"])
+def test_image_gate_admits_int8_on_turing(nvidia, monkeypatch, family, dtype, memory_mode):
+ _cap(monkeypatch, (7, 5))
+ _image_gate(monkeypatch, _image_family(family), "int8", dtype, memory_mode = memory_mode)
+
+
[email protected]("pinned", ["fp8", "mxfp8"])
+def test_image_gate_still_refuses_other_schemes_on_turing(nvidia, monkeypatch, pinned):
+ _cap(monkeypatch, (7, 5))
+ with pytest.raises(RuntimeError, match = "transformer_quant"):
+ _image_gate(monkeypatch, _image_family("qwen-image-2.1"), pinned, torch.float16)
+
+
+def test_image_gate_kill_switch_refuses_int8_on_turing(nvidia, monkeypatch):
+ _cap(monkeypatch, (7, 5))
+ monkeypatch.setenv(KILL_SWITCH, "0")
+ with pytest.raises(RuntimeError, match = "needs a CUDA GPU in bf16"):
+ _image_gate(monkeypatch, _image_family("qwen-image-2.1"), "int8", torch.float16)
+
+
[email protected]("cap", [(7, 0), (8, 6)])
+def test_image_gate_on_other_fp16_cards_is_unchanged(nvidia, monkeypatch, cap):
+ _cap(monkeypatch, cap)
+ with pytest.raises(RuntimeError, match = "needs a CUDA GPU in bf16"):
+ _image_gate(monkeypatch, _image_family("qwen-image-2.1"), "int8", torch.float16)
+
+
+def test_dense_blocker_accepts_float16_only_when_asked():
+ pipe = types.SimpleNamespace(transformer = _Toy().to(torch.float16))
+ assert "float16" in (tq.dense_quant_blocker(pipe) or "")
+ assert tq.dense_quant_blocker(pipe, allow_float16 = True) is None
+ pipe.transformer._unsloth_runtime_quant = "int8"
+ assert "already quantised" in (tq.dense_quant_blocker(pipe, allow_float16 = True) or "")
+
+
+def test_dense_blocker_still_refuses_fp8_storage_with_float16_allowed():
+ model = _Toy().to(torch.float16)
+ model.to_q.weight.data = model.to_q.weight.data.to(torch.float8_e4m3fn)
+ pipe = types.SimpleNamespace(transformer = model)
+ assert "float8_e4m3fn" in (tq.dense_quant_blocker(pipe, allow_float16 = True) or "")
+
+
+def test_small_host_int8_linears_arm_w8a8_on_request(monkeypatch):
+ import core.inference.diffusion_small_host as sh
+
+ monkeypatch.delenv(sh.INT8_ACT_ENV, raising = False)
+ monkeypatch.setattr(sh, "_INT8_MIN_ELEMENTS", 1)
+ model = _Toy().to(torch.float32)
+ sh.quantize_int8_weight_(model, compute_dtype = torch.float32, work_device = "cpu", act_int8 = False)
+ layers = [m for m in model.modules() if isinstance(m, sh.int8_linear_class())]
+ assert layers and not any(m.act_int8 for m in layers)
+ assert sh.enable_int8_act_(model) == len(layers)
+ assert all(m.act_int8 for m in layers)
+ with torch.no_grad():
+ assert torch.isfinite(model(torch.randn(32, 128))).all()
+
+
+def test_small_host_w8a8_kill_switch(monkeypatch):
+ import core.inference.diffusion_small_host as sh
+
+ monkeypatch.setattr(sh, "_INT8_MIN_ELEMENTS", 1)
+ model = _Toy().to(torch.float32)
+ sh.quantize_int8_weight_(model, compute_dtype = torch.float32, work_device = "cpu")
+ assert any(isinstance(m, sh.int8_linear_class()) for m in model.modules())
+ monkeypatch.setenv(sh.INT8_ACT_ENV, "0")
+ assert sh.enable_int8_act_(model) == 0
+ assert not any(getattr(m, "act_int8", False) for m in model.modules())
+
+
+def test_the_kill_switch_name_matches_the_module():
+ assert tq.FP16_CARD_INT8_ENV == KILL_SWITCH
+
+
+# Explicit int8 on Turing (W8A8) vs #17079's checkpoint int8 storage under offload (weight-only): same fp16 target.
+STORAGE_SWITCH = "UNSLOTH_DIFFUSION_INT8_STORAGE"
+ACT_SWITCH = "UNSLOTH_NATIVE_INT8_ACT"
+
+
+def _quantised_layers(dtype, target):
+ model = _Toy().to(dtype)
+ got = tq.quantize_transformer(
+ types.SimpleNamespace(transformer = model), target, mode = "int8", min_features = 64
+ )
+ assert got == "int8"
+ return [m for m in model.modules() if nq.is_native_linear(m)]
+
+
+def test_explicit_int8_on_a_t4_fp16_target_builds_w8a8_although_it_is_a_storage_host(
+ nvidia, monkeypatch
+):
+ _cap(monkeypatch, (7, 5))
+ monkeypatch.delenv(STORAGE_SWITCH, raising = False)
+ monkeypatch.delenv(ACT_SWITCH, raising = False)
+ target = _target(torch.float16)
+ assert tq.int8_storage_host(target)
+ layers = _quantised_layers(torch.float16, target)
+ assert layers and all(m.act_int8 for m in layers)
+ # A caller that passes the storage answer (native_int8_act) still gets the W8A8 build.
+ model = _Toy().to(torch.float16)
+ tq.quantize_transformer(
+ types.SimpleNamespace(transformer = model),
+ target,
+ mode = "int8",
+ min_features = 64,
+ act_int8 = tq.native_int8_act(target),
+ )
+ assert all(m.act_int8 for m in model.modules() if nq.is_native_linear(m))
+
+
+def test_explicit_int8_w8a8_honours_the_native_act_kill_switch(nvidia, monkeypatch):
+ _cap(monkeypatch, (7, 5))
+ monkeypatch.setenv(ACT_SWITCH, "0")
+ layers = _quantised_layers(torch.float16, _target(torch.float16))
+ assert layers and not any(m.act_int8 for m in layers)
+
+
[email protected]("offload", [False, True])
+def test_the_storage_scheme_on_a_t4_keeps_its_merged_answers(nvidia, monkeypatch, offload):
+ import core.inference.diffusion_comfy_quant as cq
+
+ _cap(monkeypatch, (7, 5))
+ monkeypatch.delenv(STORAGE_SWITCH, raising = False)
+ target = _target(torch.float16)
+ # Storage stays weight-only: native_int8_act is False whatever the Turing switch says.
+ for raw in (None, "0"):
+ if raw is None:
+ monkeypatch.delenv(KILL_SWITCH, raising = False)
+ else:
+ monkeypatch.setenv(KILL_SWITCH, raw)
+ assert tq.native_int8_act(target) is False
+ assert tq.int8_storage_scheme(target, offload = offload) == ("int8" if offload else None)
+ assert cq.comfy_int8_backend(target, "qwen-image-2.1", offload = offload) == (
+ "native" if offload else None
+ )
+
+
[email protected]("cap,device", [((7, 0), "cuda"), ((6, 1), "cuda"), (None, "xpu")])
+def test_sm70_and_xpu_keep_storage_and_never_get_w8a8(nvidia, monkeypatch, cap, device):
+ _cap(monkeypatch, cap)
+ target = _target(torch.float16, device = device)
+ assert tq.int8_storage_host(target)
+ assert not tq.fp16_card_int8_act(target)
+ assert tq.fp16_card_int8_scheme(target, "int8") is None
+ assert tq.native_int8_act(target) is False
+
+
+def test_storage_kill_switch_does_not_touch_the_turing_w8a8_build(nvidia, monkeypatch):
+ _cap(monkeypatch, (7, 5))
+ monkeypatch.setenv(STORAGE_SWITCH, "0")
+ target = _target(torch.float16)
+ assert not tq.int8_storage_host(target)
+ assert tq.int8_storage_scheme(target, offload = True) is None
+ layers = _quantised_layers(torch.float16, target)
+ assert layers and all(m.act_int8 for m in layers)
diff --git a/studio/backend/tests/test_diffusion_pipeline_prequant.py b/studio/backend/tests/test_diffusion_pipeline_prequant.py
index 27872b71de0..8526e328439 100644
--- a/studio/backend/tests/test_diffusion_pipeline_prequant.py
+++ b/studio/backend/tests/test_diffusion_pipeline_prequant.py
@@ -865,7 +865,7 @@ def _load_backend(
monkeypatch.setattr(
dmod, "select_transformer_quant_scheme", lambda target, mode, family = None, **_k: "fp8"
)
- monkeypatch.setattr(dmod, "dense_quant_blocker", lambda _pipe: None)
+ monkeypatch.setattr(dmod, "dense_quant_blocker", lambda _pipe, **_kw: None)
monkeypatch.setattr(dmod, "_pipeline_quant_uncompilable_reason", lambda *_a, **_k: None)
monkeypatch.setattr(dmod, "stored_denoiser_precision", lambda *_a, **_k: None)
monkeypatch.setattr(dmod, "denoiser_modules", lambda pipe: [("transformer", object())])