Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
Studio: run INT8 on T4 and other Turing GPUs instead of refusing it (#17167)
Daniel Han3f5d01297be7
diff --git a/studio/backend/core/inference/diffusion.py b/studio/backend/core/inference/diffusion.pyindex bb6e7b1c590..a0b070dde73 100644--- a/studio/backend/core/inference/diffusion.py+++ b/studio/backend/core/inference/diffusion.py@@ -183,6 +183,7 @@ from .diffusion_flux3 import REFERENCE_AREA as FLUX3_REFERENCE_AREAfrom .diffusion_small_host import (INT8_DENOISER_NAMES,decide_small_host,+ enable_int8_act_ as small_host_enable_int8_act,download_first_enabled as small_host_download_first,engaged_on as small_host_engaged_on,host_commit_mib,@@ -196,6 +197,7 @@ from .diffusion_small_host import (from .model_ids import hf_cache_repo_idfrom .safetensors_pread import pread_availablefrom .diffusion_memory import (+ pipe_small_host_int8,commit_load_budget_fraction,release_load_budget_fraction,vram_budget_scope,@@ -524,6 +526,8 @@ from .diffusion_transformer_quant import (denoiser_modules,explain_unusable_scheme,family_denies_scheme,+ fp16_card_int8_host,+ fp16_card_int8_scheme,mark_source_precision,nvfp4_gated_base,native_int8_act,@@ -4501,6 +4505,9 @@ class DiffusionBackend:):if native_quant_scheme(target, pinned, family = getattr(fam, "name", None)) is None:reason = explain_unusable_scheme(getattr(fam, "name", None), pinned)+ elif model_kind == "pipeline" and pinned == TQ_INT8 and fp16_card_int8_host(target):+ if fp16_card_int8_scheme(target, pinned, family = getattr(fam, "name", None)) is None:+ reason = explain_unusable_scheme(getattr(fam, "name", None), pinned)elif (model_kind == "pipeline"and pinned in NATIVE_OFFLOAD_SCHEMES@@ -8640,6 +8647,9 @@ class DiffusionBackend:TQ_INT8 if rdna_auto_w8a8 is not None else transformer_quant_pinned,family = getattr(fam, "name", None),)+ or fp16_card_int8_scheme(+ target, transformer_quant_pinned, family = getattr(fam, "name", None)+ )if kind == "pipeline"else None)@@ -10226,6 +10236,26 @@ class DiffusionBackend:if not hasattr(pending_cache_write, "start"):pending_cache_write = Noneself._raise_if_load_cancelled(_load_token)+ if (+ pipe is not None+ and kind == "pipeline"+ and transformer_quant_engaged is None+ and native_scheme == TQ_INT8+ and fp16_card_int8_host(target)+ and pipe_small_host_int8(pipe)+ ):+ # Already per-row int8 (small-host route): re-quantising would only re-round the codes.+ transformer_quant_engaged = TQ_INT8+ act_linears = sum(+ small_host_enable_int8_act(module)+ for _attr, module in denoiser_modules(pipe)+ )+ logger.info(+ "diffusion.transformer_quant: int8 runs on the small-host int8 weights (fp16-only "+ "GPU, %s compute; W8A8 armed on %d linears for float32 compute)",+ str(getattr(target, "dtype", None)).replace("torch.", ""),+ act_linears,+ )# Quantise dense bf16 pipeline denoisers in place. The blocker excludes UNet and pre-quantised# pipelines; offloaded plans stay dense unless whole-module.if (@@ -10256,7 +10286,10 @@ class DiffusionBackend:Noneif native_scheme is not None or native_offload_scheme is not Noneelse pipeline_quant_uncompilable- ) or dense_quant_blocker(pipe)+ ) or dense_quant_blocker(+ pipe,+ allow_float16 = native_scheme is not None and fp16_card_int8_host(target),+ )if pipeline_quant_blocker is not None:logger.info("diffusion.transformer_quant: skipped (%s)", pipeline_quant_blockerdiff --git a/studio/backend/core/inference/diffusion_small_host.py b/studio/backend/core/inference/diffusion_small_host.pyindex 5cad62161cf..25dc54b6baf 100644--- a/studio/backend/core/inference/diffusion_small_host.py+++ b/studio/backend/core/inference/diffusion_small_host.py@@ -659,6 +659,20 @@ def int8_act_device_ok(device: Any) -> bool:return ok+def enable_int8_act_(module: Any) -> int:+ """Let every int8-stored Linear of ``module`` run W8A8 (an explicit int8 request on an fp16-only card); returns+ how many. The per-call gate is unchanged: float32 CUDA input on sm_75 with an exact ``_int_mm``, else dequantise.+ ``UNSLOTH_DIFFUSION_SMALL_HOST_INT8_ACT=0`` keeps them dequantised."""+ if int8_act_disabled() or _INT8_CLS is None:+ return 0+ count = 0+ for sub in module.modules():+ if isinstance(sub, _INT8_CLS):+ sub.act_int8 = True+ count += 1+ return count++def _int8_linear_class():import torchimport torch.nn.functional as Fdiff --git a/studio/backend/core/inference/diffusion_transformer_quant.py b/studio/backend/core/inference/diffusion_transformer_quant.pyindex b8409145a06..5156afa46a2 100644--- a/studio/backend/core/inference/diffusion_transformer_quant.py+++ b/studio/backend/core/inference/diffusion_transformer_quant.py@@ -1052,8 +1052,9 @@ def stored_denoiser_precision(local_dir: Optional[str]) -> Optional[str]:return None-def _module_quantised_marker(module: Any) -> Optional[str]:- """Explain why ``module`` is not a dense bf16 source, or return None."""+def _module_quantised_marker(module: Any, *, allow_float16: bool = False) -> Optional[str]:+ """Explain why ``module`` is not a dense bf16 source, or return None. ``allow_float16``: an fp16 pipeline on an+ fp16-only card (``fp16_card_int8_host``), whose dense weights were cast to float16 on load."""if getattr(module, "_unsloth_runtime_quant", None):return "it is already quantised by this loader"source = getattr(module, SOURCE_PRECISION_ATTR, None)@@ -1071,14 +1072,14 @@ def _module_quantised_marker(module: Any) -> Optional[str]:try:for param in module.parameters(recurse = True):name = str(getattr(param, "dtype", "")).rsplit(".", 1)[-1]- if name not in _DENSE_PARAM_DTYPE_NAMES:+ if name not in _DENSE_PARAM_DTYPE_NAMES and not (allow_float16 and name == "float16"):return f"its weights are stored as {name}, not dense bfloat16"except Exception: # noqa: BLE001 -- an unwalkable module is not evidence of anythingreturn Nonereturn None-def dense_quant_blocker(pipe: Any) -> Optional[str]:+def dense_quant_blocker(pipe: Any, *, allow_float16: bool = False) -> Optional[str]:"""Explain why a pipeline cannot be quantised in place, or return None."""denoisers = denoiser_modules(pipe)if not denoisers:@@ -1087,7 +1088,7 @@ def dense_quant_blocker(pipe: Any) -> Optional[str]:"dense torchao schemes do not cover)")for attr, module in denoisers:- marker = _module_quantised_marker(module)+ marker = _module_quantised_marker(module, allow_float16 = allow_float16)if marker is not None:where = "its transformer" if attr == "transformer" else f"its {attr}"return f"{where} is not a dense bf16 source: {marker}"@@ -1216,6 +1217,52 @@ def native_quant_scheme(return scheme+FP16_CARD_INT8_ENV = "UNSLOTH_DIFFUSION_FP16_CARD_INT8"+# sm70 has no int8 tensor cores; Ampere+ runs bf16 through torchao.+_FP16_CARD_INT8_CAPS = ((7, 5), (8, 0))+++def fp16_card_int8_enabled() -> bool:+ raw = (_os.environ.get(FP16_CARD_INT8_ENV) or "").strip().lower()+ return raw not in ("0", "false", "no", "off")+++def fp16_card_int8_host(target: Any) -> bool:+ """A Turing NVIDIA card running the pipeline in fp16 or fp32 (no bf16): torchao refuses it, but the torchao-free+ int8 linear runs there (``torch._int_mm`` is cuBLASLt int8 with no capability gate). False on every bf16 GPU."""+ if getattr(target, "device", None) != "cuda" or not fp16_card_int8_enabled():+ return False+ if torch_is_rocm() or is_stubbed("torchao"):+ return False+ try:+ import torch+ if getattr(target, "dtype", None) not in (torch.float16, torch.float32):+ return False+ except Exception:+ return False+ cap = _capability(getattr(target, "ordinal", None))+ return cap is not None and _FP16_CARD_INT8_CAPS[0] <= cap < _FP16_CARD_INT8_CAPS[1]+++def fp16_card_int8_scheme(+ target: Any,+ requested: Optional[str],+ family: Optional[str] = None,+) -> Optional[str]:+ """``int8`` for an explicit int8 request on an ``fp16_card_int8_host``, else None (AUTO is unchanged)."""+ if normalize_transformer_quant(requested) != TQ_INT8 or not fp16_card_int8_host(target):+ return None+ if _family_denied(family, TQ_INT8):+ return None+ return TQ_INT8+++def fp16_card_int8_act(target: Any) -> bool:+ """W8A8 for the explicit-int8 Turing build; ``int8_storage_host`` (also true there) answers only for checkpoint+ int8 storage, which stays weight-only."""+ return fp16_card_int8_host(target) and not int8_act_disabled()++# Checkpoint int8 codes on a GPU with no Studio int8 runtime (pre-Ampere fp16, XPU) stay int8, widened per layer.# Unset: offloaded plans only (resident, the expanded DiT is faster); 1: resident too; 0: always expand.INT8_STORAGE_ENV = "UNSLOTH_DIFFUSION_INT8_STORAGE"@@ -2038,8 +2085,10 @@ def make_filter_fn(*,require_bf16: bool = False,require_divisible: int = 0,+ require_dtype: Any = None,):- """A non-bf16 Linear aborts the whole quantise pass; an unaligned one crashes the first matmul later."""+ """A non-bf16 Linear aborts the whole quantise pass; an unaligned one crashes the first matmul later.+ ``require_dtype``: keep only Linears of that dtype (the fp16-card int8 path's compute dtype)."""def filter_fn(module: Any, fqn: str = "") -> bool:try:@@ -2067,6 +2116,10 @@ def make_filter_fn(weight = getattr(module, "weight", None)if weight is None or weight.dtype != torch.bfloat16:return False+ if require_dtype is not None:+ weight = getattr(module, "weight", None)+ if weight is None or weight.dtype != require_dtype:+ return Falsereturn Truereturn filter_fn@@ -2087,6 +2140,11 @@ def quantize_transformer() -> Optional[str]:"""Quantise ``pipe.transformer`` in place, returning the scheme or None (GGUF); ``base_repo`` picks the NVFP4 policy."""native = native_quant_scheme(target, mode, family = family, offload = offload)+ fp16_card = native is None and fp16_card_int8_scheme(target, mode, family) is not None+ if fp16_card:+ native = TQ_INT8+ # native_int8_act is False on any fp16 CUDA target (int8_storage_host); this build is the W8A8 one.+ act_int8 = fp16_card_int8_act(target)if native is not None:mps = _mps_bf16(target)if act_int8 is None or mps:@@ -2099,6 +2157,7 @@ def quantize_transformer(act_int8 = bool(act_int8) and native == TQ_INT8,logger = logger,scale_dtype = getattr(target, "dtype", None) if mps else None,+ compute_dtype = getattr(target, "dtype", None) if fp16_card else None,)scheme = select_transformer_quant_scheme(target, mode, family = family, base_repo = base_repo)if scheme is None:@@ -2192,7 +2251,9 @@ def _quantize_native(act_int8: bool,logger: Any,scale_dtype: Any = None,+ compute_dtype: Any = None,) -> Optional[str]:+ """``compute_dtype``: the pipeline dtype on an fp16-only card (fp16 / fp32); None keeps the bf16 filter."""transformer = getattr(pipe, "transformer", None)if transformer is None:return None@@ -2203,7 +2264,8 @@ def _quantize_native(filter_fn = make_filter_fn(min_features,exclude_name_tokens = exclude_tokens_for_scheme(scheme, family, baked = True) + ("lora_",),- require_bf16 = True,+ require_bf16 = compute_dtype is None,+ require_dtype = compute_dtype,)if not apply_native_weight_quant(transformer,diff --git a/studio/backend/tests/test_diffusion_fp16_card_int8.py b/studio/backend/tests/test_diffusion_fp16_card_int8.pynew file mode 100644index 00000000000..841cf08695b--- /dev/null+++ b/studio/backend/tests/test_diffusion_fp16_card_int8.py@@ -0,0 +1,361 @@+# SPDX-License-Identifier: AGPL-3.0-only+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0++"""INT8 on fp16-only NVIDIA cards (Turing, T4): the torchao-free int8 linear instead of a refusal. Capability and+ROCm are monkeypatched, so this runs without a GPU."""++from __future__ import annotations++import types++import pytest++torch = pytest.importorskip("torch")++import core.inference.diffusion_native_quant as nq+import core.inference.diffusion_transformer_quant as tq++# Literal, not tq.FP16_CARD_INT8_ENV: the behavioural tests below must also run against a tree without the name.+KILL_SWITCH = "UNSLOTH_DIFFUSION_FP16_CARD_INT8"+++def _target(dtype, device = "cuda"):+ return types.SimpleNamespace(device = device, dtype = dtype)+++def nvidia(monkeypatch):+ monkeypatch.setattr(tq, "torch_is_rocm", lambda: False)+ monkeypatch.setattr(tq, "is_stubbed", lambda name: False)+ monkeypatch.delenv(KILL_SWITCH, raising = False)+++def _cap(monkeypatch, cap):+ monkeypatch.setattr(tq, "_capability", lambda ordinal = None: cap)+++class _Toy(torch.nn.Module):+ def __init__(self, dim = 128):+ super().__init__()+ self.to_q = torch.nn.Linear(dim, dim)+ self.ff = torch.nn.Linear(dim, 2 * dim)+ self.out = torch.nn.Linear(2 * dim, dim)++ def forward(self, x):+ return self.out(torch.nn.functional.gelu(self.ff(self.to_q(x))))++[email protected]("dtype", [torch.float16, torch.float32])+def test_turing_fp16_and_fp32_pipelines_take_int8(nvidia, monkeypatch, dtype):+ _cap(monkeypatch, (7, 5))+ target = _target(dtype)+ assert tq.fp16_card_int8_host(target)+ assert tq.fp16_card_int8_scheme(target, "int8") == "int8"+ assert tq.fp16_card_int8_scheme(target, " INT8 ") == "int8"++[email protected]("scheme", [None, "auto", "none", "fp8", "mxfp8"])+def test_only_an_explicit_int8_request_is_taken(nvidia, monkeypatch, scheme):+ _cap(monkeypatch, (7, 5))+ assert tq.fp16_card_int8_scheme(_target(torch.float16), scheme) is None+++ "cap", [(7, 0), (6, 1), (8, 0), (8, 6), (8, 9), (9, 0), (10, 0), (12, 0), None]+)[email protected]("dtype", [torch.float16, torch.float32, torch.bfloat16])+def test_every_other_nvidia_card_is_unchanged(nvidia, monkeypatch, cap, dtype):+ # sm70 has no int8 tensor cores; Ampere+ runs bf16 through torchao; None = no CUDA answer.+ _cap(monkeypatch, cap)+ target = _target(dtype)+ assert not tq.fp16_card_int8_host(target)+ assert tq.fp16_card_int8_scheme(target, "int8") is None+++def test_a_bf16_pipeline_on_turing_is_unchanged(nvidia, monkeypatch):+ _cap(monkeypatch, (7, 5))+ assert not tq.fp16_card_int8_host(_target(torch.bfloat16))++[email protected]("device", ["cpu", "mps", "xpu"])+def test_non_cuda_devices_are_unchanged(nvidia, monkeypatch, device):+ _cap(monkeypatch, (7, 5))+ assert not tq.fp16_card_int8_host(_target(torch.float16, device = device))+++def test_rocm_and_the_torchao_stub_keep_their_own_path(monkeypatch):+ _cap(monkeypatch, (7, 5))+ monkeypatch.setattr(tq, "is_stubbed", lambda name: False)+ monkeypatch.setattr(tq, "torch_is_rocm", lambda: True)+ assert not tq.fp16_card_int8_host(_target(torch.float16))+ monkeypatch.setattr(tq, "torch_is_rocm", lambda: False)+ monkeypatch.setattr(tq, "is_stubbed", lambda name: name == "torchao")+ assert not tq.fp16_card_int8_host(_target(torch.float16))++[email protected]("raw", ["0", "false", "off", "no"])+def test_kill_switch_restores_the_refusal(nvidia, monkeypatch, raw):+ _cap(monkeypatch, (7, 5))+ monkeypatch.setenv(KILL_SWITCH, raw)+ assert not tq.fp16_card_int8_host(_target(torch.float16))+ assert tq.fp16_card_int8_scheme(_target(torch.float16), "int8") is None+++def test_the_family_deny_list_still_applies(nvidia, monkeypatch):+ _cap(monkeypatch, (7, 5))+ monkeypatch.setitem(tq._FAMILY_SCHEME_DENY, "toy-family", frozenset({tq.TQ_INT8}))+ assert tq.fp16_card_int8_scheme(_target(torch.float16), "int8", family = "toy-family") is None+++def test_the_shared_native_route_is_not_widened(nvidia, monkeypatch):+ # The single-file int8 loader and video read native_quant_scheme: Turing must not change their answer.+ _cap(monkeypatch, (7, 5))+ for dtype in (torch.float16, torch.float32):+ assert not tq.native_quant_host(_target(dtype))+ for offload in (False, True):+ assert tq.native_quant_scheme(_target(dtype), "int8", offload = offload) is None++[email protected]("dtype", [torch.float16, torch.float32])+def test_quantize_transformer_builds_native_int8_linears_in_the_pipeline_dtype(+ nvidia, monkeypatch, dtype+):+ _cap(monkeypatch, (7, 5))+ torch.manual_seed(0)+ model = _Toy().to(dtype)+ x = torch.randn(4, 128, dtype = dtype)+ with torch.no_grad():+ ref = model(x).float()+ got = tq.quantize_transformer(+ types.SimpleNamespace(transformer = model), _target(dtype), mode = "int8", min_features = 64+ )+ assert got == "int8"+ for name in ("to_q", "ff", "out"):+ layer = getattr(model, name)+ assert nq.is_native_linear(layer), name+ assert layer.compute_dtype == dtype+ assert layer.weight_q.dtype == torch.int8+ with torch.no_grad():+ y = model(x)+ assert y.dtype == dtype and torch.isfinite(y).all()+ rel = ((y.float() - ref).norm() / ref.norm()).item()+ assert rel < 0.05, rel+++def test_mixed_dtype_linears_keep_their_deliberate_fp32_layers(nvidia, monkeypatch):+ _cap(monkeypatch, (7, 5))+ model = _Toy().to(torch.float16)+ model.out.float()+ got = tq.quantize_transformer(+ types.SimpleNamespace(transformer = model),+ _target(torch.float16),+ mode = "int8",+ min_features = 64,+ )+ assert got == "int8"+ assert nq.is_native_linear(model.to_q) and nq.is_native_linear(model.ff)+ assert isinstance(model.out, torch.nn.Linear) and model.out.weight.dtype == torch.float32+++def test_quantize_transformer_on_ampere_fp16_still_declines(nvidia, monkeypatch):+ _cap(monkeypatch, (8, 6))+ model = _Toy().to(torch.float16)+ got = tq.quantize_transformer(+ types.SimpleNamespace(transformer = model),+ _target(torch.float16),+ mode = "int8",+ min_features = 64,+ )+ assert got is None+ assert isinstance(model.to_q, torch.nn.Linear) and not nq.is_native_linear(model.to_q)+++def _image_family(name):+ from core.inference.diffusion_families import _FAMILIES+ return next(f for f in _FAMILIES if f.name == name)+++def _image_gate(+ monkeypatch,+ fam,+ pinned,+ dtype,+ *,+ memory_mode = None,+):+ import core.inference.diffusion as d+ monkeypatch.setattr(d, "effective_te_quant", lambda *a, **k: None)+ d.DiffusionBackend._assert_precision_for_target(+ None,+ fam,+ _target(dtype),+ model_kind = "pipeline",+ pinned = pinned,+ te_mode = None,+ memory_mode = memory_mode,+ cpu_offload = False,+ )+++ "family,dtype",+ [+ ("qwen-image-2.1", torch.float16),+ ("flux.3-klein", torch.float32),+ ("flux.3-klein", torch.float16),+ ],+)[email protected]("memory_mode", [None, "low_vram"])+def test_image_gate_admits_int8_on_turing(nvidia, monkeypatch, family, dtype, memory_mode):+ _cap(monkeypatch, (7, 5))+ _image_gate(monkeypatch, _image_family(family), "int8", dtype, memory_mode = memory_mode)++[email protected]("pinned", ["fp8", "mxfp8"])+def test_image_gate_still_refuses_other_schemes_on_turing(nvidia, monkeypatch, pinned):+ _cap(monkeypatch, (7, 5))+ with pytest.raises(RuntimeError, match = "transformer_quant"):+ _image_gate(monkeypatch, _image_family("qwen-image-2.1"), pinned, torch.float16)+++def test_image_gate_kill_switch_refuses_int8_on_turing(nvidia, monkeypatch):+ _cap(monkeypatch, (7, 5))+ monkeypatch.setenv(KILL_SWITCH, "0")+ with pytest.raises(RuntimeError, match = "needs a CUDA GPU in bf16"):+ _image_gate(monkeypatch, _image_family("qwen-image-2.1"), "int8", torch.float16)++[email protected]("cap", [(7, 0), (8, 6)])+def test_image_gate_on_other_fp16_cards_is_unchanged(nvidia, monkeypatch, cap):+ _cap(monkeypatch, cap)+ with pytest.raises(RuntimeError, match = "needs a CUDA GPU in bf16"):+ _image_gate(monkeypatch, _image_family("qwen-image-2.1"), "int8", torch.float16)+++def test_dense_blocker_accepts_float16_only_when_asked():+ pipe = types.SimpleNamespace(transformer = _Toy().to(torch.float16))+ assert "float16" in (tq.dense_quant_blocker(pipe) or "")+ assert tq.dense_quant_blocker(pipe, allow_float16 = True) is None+ pipe.transformer._unsloth_runtime_quant = "int8"+ assert "already quantised" in (tq.dense_quant_blocker(pipe, allow_float16 = True) or "")+++def test_dense_blocker_still_refuses_fp8_storage_with_float16_allowed():+ model = _Toy().to(torch.float16)+ model.to_q.weight.data = model.to_q.weight.data.to(torch.float8_e4m3fn)+ pipe = types.SimpleNamespace(transformer = model)+ assert "float8_e4m3fn" in (tq.dense_quant_blocker(pipe, allow_float16 = True) or "")+++def test_small_host_int8_linears_arm_w8a8_on_request(monkeypatch):+ import core.inference.diffusion_small_host as sh++ monkeypatch.delenv(sh.INT8_ACT_ENV, raising = False)+ monkeypatch.setattr(sh, "_INT8_MIN_ELEMENTS", 1)+ model = _Toy().to(torch.float32)+ sh.quantize_int8_weight_(model, compute_dtype = torch.float32, work_device = "cpu", act_int8 = False)+ layers = [m for m in model.modules() if isinstance(m, sh.int8_linear_class())]+ assert layers and not any(m.act_int8 for m in layers)+ assert sh.enable_int8_act_(model) == len(layers)+ assert all(m.act_int8 for m in layers)+ with torch.no_grad():+ assert torch.isfinite(model(torch.randn(32, 128))).all()+++def test_small_host_w8a8_kill_switch(monkeypatch):+ import core.inference.diffusion_small_host as sh++ monkeypatch.setattr(sh, "_INT8_MIN_ELEMENTS", 1)+ model = _Toy().to(torch.float32)+ sh.quantize_int8_weight_(model, compute_dtype = torch.float32, work_device = "cpu")+ assert any(isinstance(m, sh.int8_linear_class()) for m in model.modules())+ monkeypatch.setenv(sh.INT8_ACT_ENV, "0")+ assert sh.enable_int8_act_(model) == 0+ assert not any(getattr(m, "act_int8", False) for m in model.modules())+++def test_the_kill_switch_name_matches_the_module():+ assert tq.FP16_CARD_INT8_ENV == KILL_SWITCH+++# Explicit int8 on Turing (W8A8) vs #17079's checkpoint int8 storage under offload (weight-only): same fp16 target.+STORAGE_SWITCH = "UNSLOTH_DIFFUSION_INT8_STORAGE"+ACT_SWITCH = "UNSLOTH_NATIVE_INT8_ACT"+++def _quantised_layers(dtype, target):+ model = _Toy().to(dtype)+ got = tq.quantize_transformer(+ types.SimpleNamespace(transformer = model), target, mode = "int8", min_features = 64+ )+ assert got == "int8"+ return [m for m in model.modules() if nq.is_native_linear(m)]+++def test_explicit_int8_on_a_t4_fp16_target_builds_w8a8_although_it_is_a_storage_host(+ nvidia, monkeypatch+):+ _cap(monkeypatch, (7, 5))+ monkeypatch.delenv(STORAGE_SWITCH, raising = False)+ monkeypatch.delenv(ACT_SWITCH, raising = False)+ target = _target(torch.float16)+ assert tq.int8_storage_host(target)+ layers = _quantised_layers(torch.float16, target)+ assert layers and all(m.act_int8 for m in layers)+ # A caller that passes the storage answer (native_int8_act) still gets the W8A8 build.+ model = _Toy().to(torch.float16)+ tq.quantize_transformer(+ types.SimpleNamespace(transformer = model),+ target,+ mode = "int8",+ min_features = 64,+ act_int8 = tq.native_int8_act(target),+ )+ assert all(m.act_int8 for m in model.modules() if nq.is_native_linear(m))+++def test_explicit_int8_w8a8_honours_the_native_act_kill_switch(nvidia, monkeypatch):+ _cap(monkeypatch, (7, 5))+ monkeypatch.setenv(ACT_SWITCH, "0")+ layers = _quantised_layers(torch.float16, _target(torch.float16))+ assert layers and not any(m.act_int8 for m in layers)++[email protected]("offload", [False, True])+def test_the_storage_scheme_on_a_t4_keeps_its_merged_answers(nvidia, monkeypatch, offload):+ import core.inference.diffusion_comfy_quant as cq++ _cap(monkeypatch, (7, 5))+ monkeypatch.delenv(STORAGE_SWITCH, raising = False)+ target = _target(torch.float16)+ # Storage stays weight-only: native_int8_act is False whatever the Turing switch says.+ for raw in (None, "0"):+ if raw is None:+ monkeypatch.delenv(KILL_SWITCH, raising = False)+ else:+ monkeypatch.setenv(KILL_SWITCH, raw)+ assert tq.native_int8_act(target) is False+ assert tq.int8_storage_scheme(target, offload = offload) == ("int8" if offload else None)+ assert cq.comfy_int8_backend(target, "qwen-image-2.1", offload = offload) == (+ "native" if offload else None+ )++[email protected]("cap,device", [((7, 0), "cuda"), ((6, 1), "cuda"), (None, "xpu")])+def test_sm70_and_xpu_keep_storage_and_never_get_w8a8(nvidia, monkeypatch, cap, device):+ _cap(monkeypatch, cap)+ target = _target(torch.float16, device = device)+ assert tq.int8_storage_host(target)+ assert not tq.fp16_card_int8_act(target)+ assert tq.fp16_card_int8_scheme(target, "int8") is None+ assert tq.native_int8_act(target) is False+++def test_storage_kill_switch_does_not_touch_the_turing_w8a8_build(nvidia, monkeypatch):+ _cap(monkeypatch, (7, 5))+ monkeypatch.setenv(STORAGE_SWITCH, "0")+ target = _target(torch.float16)+ assert not tq.int8_storage_host(target)+ assert tq.int8_storage_scheme(target, offload = True) is None+ layers = _quantised_layers(torch.float16, target)+ assert layers and all(m.act_int8 for m in layers)diff --git a/studio/backend/tests/test_diffusion_pipeline_prequant.py b/studio/backend/tests/test_diffusion_pipeline_prequant.pyindex 27872b71de0..8526e328439 100644--- a/studio/backend/tests/test_diffusion_pipeline_prequant.py+++ b/studio/backend/tests/test_diffusion_pipeline_prequant.py@@ -865,7 +865,7 @@ def _load_backend(monkeypatch.setattr(dmod, "select_transformer_quant_scheme", lambda target, mode, family = None, **_k: "fp8")- monkeypatch.setattr(dmod, "dense_quant_blocker", lambda _pipe: None)+ monkeypatch.setattr(dmod, "dense_quant_blocker", lambda _pipe, **_kw: None)monkeypatch.setattr(dmod, "_pipeline_quant_uncompilable_reason", lambda *_a, **_k: None)monkeypatch.setattr(dmod, "stored_denoiser_precision", lambda *_a, **_k: None)monkeypatch.setattr(dmod, "denoiser_modules", lambda pipe: [("transformer", object())])