Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
Studio: run an explicit int8 under offload on NVIDIA as torchao-free W8A8 (#11712)
Daniel Hande7c4fdfd0c8
diff --git a/studio/backend/core/inference/diffusion.py b/studio/backend/core/inference/diffusion.pyindex 50f836666fe..f3732435e07 100644--- a/studio/backend/core/inference/diffusion.py+++ b/studio/backend/core/inference/diffusion.py@@ -212,6 +212,7 @@ from .diffusion_auto_policy import ()from .diffusion_transformer_quant import (TQ_AUTO,+ TQ_INT8,DEFAULT_MIN_LINEAR_FEATURES,DenoiserView as _DenoiserView,dense_quant_blocker,@@ -222,8 +223,11 @@ from .diffusion_transformer_quant import (denoiser_modules,explain_unusable_scheme,mark_source_precision,+ native_int8_act,+ native_offload_host,native_quant_host,native_quant_scheme,+ NATIVE_OFFLOAD_SCHEMES,NATIVE_QUANT_SCHEMES,normalize_transformer_quant,quantize_transformer,@@ -231,6 +235,7 @@ from .diffusion_transformer_quant import (stored_denoiser_precision,transformer_is_quantised,)+from .diffusion_native_quant import native_quant_reasonfrom utils.paths.path_utils import (any_not_appledouble_metadata,is_appledouble_metadata,@@ -1828,6 +1833,19 @@ class DiffusionBackend:):if native_quant_scheme(target, pinned, family = getattr(fam, "name", None)) is None:reason = explain_unusable_scheme(getattr(fam, "name", None), pinned)+ elif (+ model_kind == "pipeline"+ and pinned in NATIVE_OFFLOAD_SCHEMES+ and _memory_request_forces_offload(memory_mode, cpu_offload)+ and native_offload_host(target)+ ):+ if (+ native_quant_scheme(+ target, pinned, family = getattr(fam, "name", None), offload = True+ )+ is None+ ):+ reason = explain_unusable_scheme(getattr(fam, "name", None), pinned)elif _memory_request_forces_offload(memory_mode, cpu_offload):# Not a measurement: balanced and low_vram name their policy outright, and the legacy flag forces# model offload. Offload hooks move modules with Module.to(), which torchao tensors do not survive, so@@ -2524,6 +2542,8 @@ class DiffusionBackend:model_kind = resolve_model_kind(gguf_filename, model_kind),transformer_quant = transformer_quant,text_encoder_quant = text_encoder_quant,+ memory_mode = memory_mode,+ cpu_offload = cpu_offload,# An uncompiled torchao transformer loses to the bf16 it replaces, so 'eager' stays dense. Refusing# an explicit scheme here says so before the download.speed_mode = speed_mode,@@ -4611,6 +4631,19 @@ class DiffusionBackend:if kind == "pipeline"else None)+ native_offload_scheme = (+ native_quant_scheme(+ target,+ transformer_quant_pinned,+ family = getattr(fam, "name", None),+ offload = True,+ )+ if kind == "pipeline" and native_scheme is None+ else None+ )+ native_offload_forced = native_offload_scheme is not None and (+ _memory_request_forces_offload(memory_mode, cpu_offload)+ )if transformer_quant_pinned is not None and not dense_quant_supported_kind(kind):transformer_quant_decline = dense_quant_unsupported_kind_reason(kind)transformer_quant_decline_status = RESOLVED_UNSUPPORTED@@ -4627,6 +4660,7 @@ class DiffusionBackend:elif (transformer_quant_pinned is not Noneand native_scheme is None+ and not native_offload_forcedand select_transformer_quant_scheme(target, transformer_quant_pinned, family = getattr(fam, "name", None))@@ -5510,9 +5544,10 @@ class DiffusionBackend:if source_precision is not None:for _attr, denoiser in denoiser_modules(pipe):mark_source_precision(denoiser, source_precision)- # Weight-only runs bf16 arithmetic, so it has no compile to require.pipeline_quant_blocker = (- None if native_scheme is not None else pipeline_quant_uncompilable+ None+ if native_scheme is not None or native_offload_scheme is not None+ else pipeline_quant_uncompilable) or dense_quant_blocker(pipe)if pipeline_quant_blocker is not None:logger.info(@@ -5561,6 +5596,18 @@ class DiffusionBackend:plan.offload_policy,)plan = replanned+ if (+ plan.offload_policy != OFFLOAD_NONE+ and native_scheme is None+ and native_offload_scheme is not None+ ):+ native_scheme = native_offload_scheme+ logger.info(+ "diffusion.transformer_quant: %s runs torchao-free under the plan's '%s' "+ "offload (torchao tensors reject the hooks' Module.to())",+ native_scheme,+ plan.offload_policy,+ )if plan.offload_policy != OFFLOAD_NONE and native_scheme is None:logger.info("diffusion.transformer_quant: skipped (the memory plan picked '%s' "@@ -5572,6 +5619,14 @@ class DiffusionBackend:"the transformer via Module.to(); torchao quantised tensors reject ""that. Pin a resident memory mode to combine the two")+ elif native_scheme is None and pipeline_quant_uncompilable is not None:+ # Resident plan: torchao still needs the compile check skipped above.+ logger.info(+ "diffusion.transformer_quant: skipped (%s)",+ pipeline_quant_uncompilable,+ )+ transformer_quant_decline = pipeline_quant_uncompilable+ transformer_quant_decline_status = RESOLVED_UNSUPPORTEDelse:if _has_active_lora(loras):# PEFT must wrap dense Linears before torchao converts their base layers.@@ -5595,6 +5650,15 @@ class DiffusionBackend:)# Convert every denoiser so multi-branch pipelines use one precision.denoisers = denoiser_modules(pipe)+ native_kwargs = (+ {+ "offload": plan.offload_policy != OFFLOAD_NONE,+ "act_int8": native_scheme == TQ_INT8+ and native_int8_act(target),+ }+ if native_scheme is not None+ else {}+ )engaged: list[str] = []for attr, _module in denoisers:scheme = quantize_transformer(@@ -5606,6 +5670,7 @@ class DiffusionBackend:family = getattr(fam, "name", None),fast_accum = transformer_quant_fast_accum,logger = logger,+ **native_kwargs,)if scheme is None:break@@ -5960,8 +6025,9 @@ class DiffusionBackend:))if transformer_quant_engaged is None- else f"weight-only: {transformer_quant_engaged} weights, bf16 compute "- "(torchao-free, a memory saving rather than a speed-up)"+ else native_quant_reason(+ getattr(pipe, "transformer", None), transformer_quant_engaged+ )if native_scheme is not Noneelse f"seeded from the hosted checkpoint "f"{transformer_quant_artifact.split(':', 1)[1]}"diff --git a/studio/backend/core/inference/diffusion_compile_cache.py b/studio/backend/core/inference/diffusion_compile_cache.pyindex c77c23678e5..16969082eba 100644--- a/studio/backend/core/inference/diffusion_compile_cache.py+++ b/studio/backend/core/inference/diffusion_compile_cache.py@@ -298,6 +298,10 @@ def model_fingerprint(what gets compiled."""blocks = list(getattr(transformer, "_repeated_blocks", []) or [])+ if quant is not None and transformer is not None:+ # Native layers compile to a different graph than torchao under the same scheme name.+ from .diffusion_native_quant import native_quant_signature+ quant = native_quant_signature(transformer) or quantreturn {"family": str(family),"transformer_cls": type(transformer).__name__ if transformer is not None else None,diff --git a/studio/backend/core/inference/diffusion_native_quant.py b/studio/backend/core/inference/diffusion_native_quant.pyindex 1b78d0e4903..74ccfbd98b8 100644--- a/studio/backend/core/inference/diffusion_native_quant.py+++ b/studio/backend/core/inference/diffusion_native_quant.py@@ -14,6 +14,31 @@ from typing import Any, Callable, OptionalNATIVE_INT8 = "int8"NATIVE_FP8 = "fp8"_QMAX = {NATIVE_INT8: 127.0, NATIVE_FP8: 448.0}+NATIVE_INT8_ACT_ENV = "UNSLOTH_NATIVE_INT8_ACT"+NATIVE_INT8_ROT_ENV = "UNSLOTH_NATIVE_INT8_ROT"+# torch._int_mm needs M > 16 and K, N multiples of 8.+_INT_MM_MIN_ROWS = 17+_INT_MM_CHUNK_ELEMS = 1 << 25+++def int8_act_requested() -> bool:+ import os+ return os.environ.get(NATIVE_INT8_ACT_ENV, "").strip() == "1"+++def int8_act_disabled() -> bool:+ import os+ return os.environ.get(NATIVE_INT8_ACT_ENV, "").strip() == "0"+++def int8_rotation_group() -> int:+ import os++ if os.environ.get(NATIVE_INT8_ROT_ENV, "").strip() == "0":+ return 0+ from .diffusion_convrot import DEFAULT_CONVROT_GROUPSIZE++ return DEFAULT_CONVROT_GROUPSIZE@lru_cache(maxsize = 1)@@ -26,7 +51,13 @@ def native_linear_class():class NativeWeightOnlyLinear(nn.Module):"""A Linear whose weight is stored as int8 / fp8 with a per-output-row scale."""- def __init__(self, linear: Any, scheme: str):+ def __init__(+ self,+ linear: Any,+ scheme: str,+ act_int8: bool = False,+ rot_group: int = 0,+ ):super().__init__()if scheme not in _QMAX:raise ValueError(f"unsupported native scheme {scheme!r}")@@ -34,9 +65,22 @@ def native_linear_class():self.out_features = int(linear.out_features)self.scheme = schemeself.compute_dtype = linear.weight.dtype+ self.act_int8 = bool(act_int8) and scheme == NATIVE_INT8+ self.rot_group = (+ int(rot_group)+ if self.act_int8 and rot_group and self.in_features % int(rot_group) == 0+ else 0+ )weight = linear.weight.detach()with torch.no_grad():w = weight.float()+ if self.rot_group:+ from .diffusion_convrot import build_convrot_hadamard++ h = build_convrot_hadamard(self.rot_group, device = w.device, dtype = torch.float32)+ g = self.rot_group+ w = (w.reshape(self.out_features, -1, g) @ h.T).reshape(self.out_features, -1)+ self.register_buffer("rot_h", h.to(self.compute_dtype), persistent = False)scale = w.abs().amax(dim = 1, keepdim = True).clamp(min = 1e-12) / _QMAX[scheme]if scheme == NATIVE_INT8:wq = (w / scale).round_().clamp_(-127, 127).to(torch.int8)@@ -50,7 +94,7 @@ def native_linear_class():)self.bias = linear.bias- def dequantized_weight(self, dtype: Any) -> Any:+ def _stored_weight(self, dtype: Any) -> Any:wq = (self.weight_q.view(torch.float8_e4m3fn)if self.scheme == NATIVE_FP8@@ -59,19 +103,82 @@ def native_linear_class():scale = self.weight_scale.view(torch.float32)return (wq.to(torch.float32) * scale[:, None]).to(dtype)+ def _rotate(self, x: Any) -> Any:+ g = self.rot_group+ # Leading dims, not -1: a zero-row input has no inferable -1 on older torch.+ grouped = x.reshape(*x.shape[:-1], self.in_features // g, g)+ return (grouped @ self.rot_h.to(x.dtype)).reshape(x.shape)++ def dequantized_weight(self, dtype: Any) -> Any:+ w = self._stored_weight(torch.float32 if self.rot_group else dtype)+ if not self.rot_group:+ return w+ g = self.rot_group+ h = self.rot_h.to(torch.float32)+ return (+ (w.reshape(self.out_features, -1, g) @ h).reshape(self.out_features, -1).to(dtype)+ )+@propertydef weight(self) -> Any:"""The dense weight, rebuilt on each read. Code that reads a wrapped Linear's weight directly(PEFT's DoRA forward reads ``base_layer.weight``) keeps working; nothing holds it."""return self.dequantized_weight(self.compute_dtype)+ def _int_mm_ok(self, rows: int) -> bool:+ return (+ rows >= _INT_MM_MIN_ROWS+ and self.in_features % 8 == 0+ and self.out_features % 8 == 0+ )++ def _int_mm_rows(self, x2: Any) -> Any:+ # Must stay fast in eager (Windows ROCm has no Triton): upcasts ride mixed-dtype ops, no .float() copies.+ x_scale = torch.linalg.vector_norm(+ x2, ord = float("inf"), dim = 1, keepdim = True, dtype = torch.float32+ )+ x_scale = x_scale.clamp_(min = 1e-12).div_(127.0)+ xq = torch.mul(x2, x_scale.reciprocal()).round_().clamp_(-127, 127).to(torch.int8)+ acc = torch._int_mm(xq, self.weight_q.t())+ w_scale = self.weight_scale.view(torch.float32)+ out = torch.mul(acc, x_scale)+ if self.bias is not None:+ out = torch.addcmul(self.bias.float(), out, w_scale)+ else:+ out = out.mul_(w_scale)+ return out++ def _forward_int_mm(self, x: Any) -> Any:+ x2 = x.reshape(-1, self.in_features)+ rows = x2.shape[0]+ # Eager materialises fp32 [rows, features] temporaries (inductor fuses them): cap them per chunk.+ chunk = _INT_MM_CHUNK_ELEMS // max(self.in_features, self.out_features)+ if torch.compiler.is_compiling() or rows <= chunk or chunk < 2 * _INT_MM_MIN_ROWS:+ out = self._int_mm_rows(x2).to(x.dtype)+ else:+ out = torch.cat(+ [+ self._int_mm_rows(part).to(x.dtype)+ for part in torch.tensor_split(x2, -(-rows // chunk))+ ]+ )+ return out.reshape(*x.shape[:-1], self.out_features)+def forward(self, x: Any) -> Any:- return F.linear(x, self.dequantized_weight(x.dtype), self.bias)+ if self.rot_group:+ x = self._rotate(x)+ if (+ self.act_int8+ and x.is_cuda+ and self._int_mm_ok(x.numel() // max(1, self.in_features))+ ):+ return self._forward_int_mm(x)+ return F.linear(x, self._stored_weight(x.dtype), self.bias)def extra_repr(self) -> str:return (f"in_features={self.in_features}, out_features={self.out_features}, "- f"scheme={self.scheme}, bias={self.bias is not None}"+ f"scheme={self.scheme}, act_int8={self.act_int8}, rot_group={self.rot_group}, bias={self.bias is not None}")return NativeWeightOnlyLinear@@ -94,25 +201,78 @@ def apply_native_weight_quant(scheme: str,*,filter_fn: Callable[[Any, str], bool],+ act_int8: bool,logger: Any = None,) -> int:"""Swap every Linear ``filter_fn`` keeps for a native weight-only twin; returns layers swapped. One layerat a time; a failure part-way leaves a partial conversion the caller must refuse."""cls = native_linear_class()+ act_int8 = scheme == NATIVE_INT8 and bool(act_int8)+ rot_group = int8_rotation_group() if act_int8 else 0+ # Names only: holding the old modules would keep every bf16 weight alive until the pass ends.names = [name for name, mod in transformer.named_modules() if name and filter_fn(mod, name)]for name in names:parent_name, _, leaf = name.rpartition(".")parent = transformer.get_submodule(parent_name) if parent_name else transformer- setattr(parent, leaf, cls(getattr(parent, leaf), scheme))+ setattr(+ parent, leaf, cls(getattr(parent, leaf), scheme, act_int8 = act_int8, rot_group = rot_group)+ )if logger is not None:logger.info(- "diffusion.transformer_quant: %s weight-only (torchao-free) on %d linears",+ "diffusion.transformer_quant: %s %s (torchao-free) on %d linears",scheme,+ ("W8A8 torch._int_mm" + (f" ConvRot g{rot_group}" if rot_group else ""))+ if act_int8+ else "weight-only",len(names),)return len(names)+def _first_native_linear(module: Any) -> Any:+ try:+ return next((sub for sub in module.modules() if is_native_linear(sub)), None)+ except Exception: # noqa: BLE001 -- a probe must never raise+ return None+++def _rotation_group(module: Any, layer: Any) -> int:+ # Every layer: rotation skips widths the group does not divide, so the first layer alone can read unrotated.+ try:+ return max(+ int(getattr(sub, "rot_group", 0) or 0)+ for sub in module.modules()+ if is_native_linear(sub)+ )+ except Exception: # noqa: BLE001 -- a probe must never raise+ return int(getattr(layer, "rot_group", 0) or 0)+++def native_quant_signature(module: Any) -> Optional[str]:+ layer = _first_native_linear(module)+ if layer is None:+ return None+ if not getattr(layer, "act_int8", False):+ return f"{layer.scheme}-wo"+ rot = _rotation_group(module, layer)+ return f"{layer.scheme}-w8a8" + (f"-rot{rot}" if rot else "")+++def native_quant_reason(module: Any, scheme: str) -> str:+ layer = _first_native_linear(module)+ if layer is not None and getattr(layer, "act_int8", False):+ rot = _rotation_group(module, layer)+ return (+ f"W8A8: {scheme} weights and activations through torch._int_mm"+ + (f" with a g{rot} ConvRot rotation" if rot else "")+ + " (torchao-free; plain buffers, so the offload hooks can move it)"+ )+ return (+ f"weight-only: {scheme} weights, bf16 compute "+ "(torchao-free, a memory saving rather than a speed-up)"+ )++def native_weight_error(linear: Any, scheme: str) -> Optional[float]:"""Relative Frobenius error of one Linear's native reconstruction; for tests and probes."""try:diff --git a/studio/backend/core/inference/diffusion_transformer_quant.py b/studio/backend/core/inference/diffusion_transformer_quant.pyindex 9fa351f9ab6..ee0a65f5511 100644--- a/studio/backend/core/inference/diffusion_transformer_quant.py+++ b/studio/backend/core/inference/diffusion_transformer_quant.py@@ -30,7 +30,7 @@ from typing import Any, Optional# stdlib-only module (no torch), so this stays inside the "imported lazily" promise above.from core._torchao_stub import is_stubbed, torch_is_rocm-from .diffusion_native_quant import is_native_linear+from .diffusion_native_quant import int8_act_disabled, int8_act_requested, is_native_linearfrom .diffusion_torchao_patches import install_torchao_int_mm_patch# Also runs in the spawned smoke-probe child, which imports this module and nothing else of the backend.@@ -686,21 +686,47 @@ def native_quant_host(target: Any) -> bool:return False+NATIVE_OFFLOAD_SCHEMES = (TQ_INT8,)+++def native_offload_host(target: Any) -> bool:+ if getattr(target, "device", None) != "cuda":+ return False+ if torch_is_rocm() or is_stubbed("torchao"):+ return False+ try:+ import torch+ return getattr(target, "dtype", None) is torch.bfloat16+ except Exception:+ return False++def native_quant_scheme(target: Any,requested: Optional[str],family: Optional[str] = None,+ *,+ offload: bool = False,) -> Optional[str]:- """The weight-only scheme an EXPLICIT ``requested`` runs as on a ``native_quant_host``, or None.- Not in ``select_transformer_quant_scheme``: that also feeds hosted torchao prequant planners."""+ """Not ``select_transformer_quant_scheme``: that also feeds hosted prequant planners (torchao checkpoints)."""scheme = normalize_transformer_quant(requested)- if scheme not in NATIVE_QUANT_SCHEMES or not native_quant_host(target):+ if scheme not in NATIVE_QUANT_SCHEMES:+ return None+ if not native_quant_host(target) and not (+ offload and scheme in NATIVE_OFFLOAD_SCHEMES and native_offload_host(target)+ ):return Noneif _family_denied(family, scheme):return Nonereturn scheme+def native_int8_act(target: Any) -> bool:+ if native_quant_host(target):+ return int8_act_requested()+ return not int8_act_disabled()++def select_transformer_quant_scheme(target: Any,requested: Optional[str],@@ -1404,16 +1430,25 @@ def quantize_transformer(min_features: int = DEFAULT_MIN_LINEAR_FEATURES,fast_accum: Optional[bool] = None,logger: Any = None,+ offload: bool = False,+ act_int8: Optional[bool] = None,) -> Optional[str]:"""Quantise ``pipe.transformer``'s FLOP-heavy linears in place with the arch-chosen scheme.Returns the scheme engaged, or None when disabled / unsupported / failed (caller loads GGUF).Best-effort: never raises for an unsupported environment (failure leaves it dense).``fast_accum`` (fp8 only) overrides the per-GPU-class accumulate choice: None auto-detects,- True/False force it. On a ``native_quant_host`` an explicit int8 / fp8 runs weight-only without torchao."""- native = native_quant_scheme(target, mode, family = family)+ True/False force it."""+ native = native_quant_scheme(target, mode, family = family, offload = offload)if native is not None:+ if act_int8 is None:+ act_int8 = native_int8_act(target)return _quantize_native(- pipe, native, family = family, min_features = min_features, logger = logger+ pipe,+ native,+ family = family,+ min_features = min_features,+ act_int8 = bool(act_int8) and native == TQ_INT8,+ logger = logger,)scheme = select_transformer_quant_scheme(target, mode, family = family)if scheme is None:@@ -1456,9 +1491,8 @@ def quantize_transformer(def _quantize_native(- pipe: Any, scheme: str, *, family: Optional[str], min_features: int, logger: Any+ pipe: Any, scheme: str, *, family: Optional[str], min_features: int, act_int8: bool, logger: Any) -> Optional[str]:- """The torchao-free branch of ``quantize_transformer``: the same layer filter, weight-only."""transformer = getattr(pipe, "transformer", None)if transformer is None:return None@@ -1471,7 +1505,9 @@ def _quantize_native(exclude_name_tokens = exclude_tokens_for_scheme(scheme, family) + ("lora_",),require_bf16 = True,)- if not apply_native_weight_quant(transformer, scheme, filter_fn = filter_fn, logger = logger):+ if not apply_native_weight_quant(+ transformer, scheme, filter_fn = filter_fn, act_int8 = act_int8, logger = logger+ ):return Nonetry:transformer._unsloth_runtime_quant = schemediff --git a/studio/backend/core/inference/video.py b/studio/backend/core/inference/video.pyindex 3af575b20ad..615d3a58c84 100644--- a/studio/backend/core/inference/video.py+++ b/studio/backend/core/inference/video.py@@ -110,11 +110,15 @@ from .diffusion_auto_policy import ()from .diffusion_transformer_quant import (TQ_AUTO,+ TQ_INT8,dense_transformer_supported,dense_transformer_unsupported_reason,explain_unusable_scheme,+ native_int8_act,+ native_offload_host,native_quant_host,native_quant_scheme,+ NATIVE_OFFLOAD_SCHEMES,NATIVE_QUANT_SCHEMES,normalize_transformer_quant,quantize_transformer,@@ -124,6 +128,7 @@ from .diffusion_transformer_quant import (transformer_is_quantised,)from .diffusion import _memory_request_forces_offload+from .diffusion_native_quant import native_quant_reasonfrom .diffusion_batched import is_oom_errorfrom .diffusion_precision import (effective_te_quant,@@ -306,6 +311,13 @@ def _assert_video_precision_for_target(reason = explain_unusable_scheme(getattr(fam, "name", None), pinned)elif not dense_transformer_supported(target):reason = dense_transformer_unsupported_reason(target)+ elif forces_offload and pinned in NATIVE_OFFLOAD_SCHEMES and native_offload_host(target):+ # forces_offload is already False for the modular workflow.+ if (+ native_quant_scheme(target, pinned, family = getattr(fam, "name", None), offload = True)+ is None+ ):+ reason = explain_unusable_scheme(getattr(fam, "name", None), pinned)elif forces_offload:# balanced and low_vram name their offload policy without measuring anything, and offload hooks move modules# with Module.to(), which torchao tensors do not survive. load_pipeline therefore skips the dense build and@@ -1687,6 +1699,7 @@ class VideoBackend:model_kind = resolve_video_model_kind(gguf_filename, model_kind),transformer_quant = transformer_quant,text_encoder_quant = text_encoder_quant,+ memory_mode = memory_mode,gpu_ordinal = gpu_ordinal,)# Resolved out here so the companion claim is published in the SAME locked section as _loading. begin_load@@ -4309,11 +4322,22 @@ class VideoBackend:# Why the quant did not engage, in the caller's terms; threaded into `resolved`.transformer_quant_decline: Optional[str] = Nonetransformer_quant_decline_status = RESOLVED_FELL_BACK+ video_offload = plan.offload_policy != "none"native_scheme = (- native_quant_scheme(target, transformer_quant_pinned, family = fam.name)+ native_quant_scheme(+ target, transformer_quant_pinned, family = fam.name, offload = video_offload+ )if kind == "pipeline"else None)+ native_kwargs = (+ {+ "offload": video_offload,+ "act_int8": native_scheme == TQ_INT8 and native_int8_act(target),+ }+ if native_scheme is not None+ else {}+ )# Auto quantises a video DiT only to keep it resident: with bf16 resident, int8 fails the default LPIPS bar.if (kind == "pipeline"@@ -4351,6 +4375,7 @@ class VideoBackend:and normalize_transformer_quant(transformer_quant) is not Noneand dense_transformer_supported(target)and plan.offload_policy != "none"+ and native_scheme is None):# Offload hooks move modules with Module.to(), which torchao quantized tensors reject. Skip quant# (dense-under-offload beats a crash); forceable via a resident mode.@@ -4397,6 +4422,7 @@ class VideoBackend:mode = transformer_quant,family = fam.name,logger = logger,+ **native_kwargs,)if scheme is not None:engaged.append(scheme)@@ -4657,8 +4683,9 @@ class VideoBackend:transformer_quant_engaged or "off",# Honest framing: the shipped torchao schemes cut load time and resident memory ~2x, but# per-step GEMMs are at best bf16 parity.- f"weight-only: {transformer_quant_engaged} weights, bf16 compute "- "(torchao-free, a memory saving rather than a speed-up)"+ native_quant_reason(+ getattr(pipe, "transformer", None), transformer_quant_engaged+ )if transformer_quant_engaged is not None and native_scheme is not Noneelse "DiT(s) quantised (halves resident weights; hosted checkpoints cut ""load time; per-step speed is roughly bf16 parity)"diff --git a/studio/backend/tests/test_diffusion_backend.py b/studio/backend/tests/test_diffusion_backend.pyindex 9806f601433..a5ad1fee84c 100644--- a/studio/backend/tests/test_diffusion_backend.py+++ b/studio/backend/tests/test_diffusion_backend.py@@ -12475,6 +12475,178 @@ def test_an_explicit_scheme_on_nvidia_keeps_the_torchao_path_and_its_wording(backend.unload()+def _stub_nvidia_offload_host(+ backend,+ monkeypatch,+ *,+ engages = "int8",+):+ from core.inference import diffusion as dmod+ from core.inference import diffusion_transformer_quant as tq++ calls = _stub_pipeline_dense_quant(backend, monkeypatch, engages = engages)+ monkeypatch.setattr(+ dmod, "select_transformer_quant_scheme", lambda target, mode, family = None: mode+ )+ monkeypatch.setattr(tq, "native_quant_host", lambda target: False)+ monkeypatch.setattr(tq, "native_offload_host", lambda target: True)+ monkeypatch.setattr(dmod, "native_offload_host", lambda target: True)+ monkeypatch.delenv("UNSLOTH_NATIVE_INT8_ACT", raising = False)+ reasons: list = []++ def _reason(module, scheme):+ reasons.append(scheme)+ return f"W8A8: {scheme} (stub)"++ monkeypatch.setattr(dmod, "native_quant_reason", _reason)+ return calls, reasons+++ "memory",+ [{"memory_mode": "balanced"}, {"memory_mode": "low_vram"}, {"cpu_offload": True}],+)+def test_an_explicit_int8_under_offload_on_nvidia_runs_native_w8a8(+ fake_runtime, tmp_path, monkeypatch, memory+):+ backend = DiffusionBackend()+ calls, reasons = _stub_nvidia_offload_host(backend, monkeypatch)+ status = backend.load_pipeline(+ "Qwen/Qwen-Image-2512",+ model_kind = "pipeline",+ transformer_quant = "int8",+ _base_local_dir = str(tmp_path),+ **memory,+ )+ assert len(calls) == 1+ assert calls[0]["offload"] is True and calls[0]["act_int8"] is True+ assert status["offload_policy"] != "none"+ assert status["transformer_quant"] == "int8"+ resolved = status["resolved"]["transformer_quant"]+ assert resolved["value"] == "int8" and resolved["status"] == "applied"+ assert resolved["reason"].startswith("W8A8") and reasons == ["int8"]+ backend.unload()+++def test_the_act_kill_switch_keeps_nvidia_offload_native_but_weight_only(+ fake_runtime, tmp_path, monkeypatch+):+ backend = DiffusionBackend()+ calls, _ = _stub_nvidia_offload_host(backend, monkeypatch)+ monkeypatch.setenv("UNSLOTH_NATIVE_INT8_ACT", "0")+ status = backend.load_pipeline(+ "Qwen/Qwen-Image-2512",+ model_kind = "pipeline",+ transformer_quant = "int8",+ memory_mode = "balanced",+ _base_local_dir = str(tmp_path),+ )+ assert calls[0]["offload"] is True and calls[0]["act_int8"] is False+ assert status["transformer_quant"] == "int8"+ backend.unload()++[email protected]("fallback", ["0", "1"])+def test_an_explicit_fp8_under_offload_on_nvidia_is_still_declined(+ fake_runtime, tmp_path, monkeypatch, fallback+):+ monkeypatch.setenv("UNSLOTH_DIFFUSION_ALLOW_PRECISION_FALLBACK", fallback)+ backend = DiffusionBackend()+ calls, reasons = _stub_nvidia_offload_host(backend, monkeypatch, engages = "fp8")+ kwargs = dict(+ model_kind = "pipeline",+ transformer_quant = "fp8",+ memory_mode = "balanced",+ _base_local_dir = str(tmp_path),+ )+ if fallback == "0":+ with pytest.raises(RuntimeError, match = "Module.to"):+ backend.load_pipeline("Qwen/Qwen-Image-2512", **kwargs)+ else:+ status = backend.load_pipeline("Qwen/Qwen-Image-2512", **kwargs)+ assert status["transformer_quant"] is None+ assert "Module.to()" in status["resolved"]["transformer_quant"]["reason"]+ backend.unload()+ assert calls == [] and reasons == []+++def test_an_explicit_int8_resident_on_nvidia_keeps_torchao(fake_runtime, tmp_path, monkeypatch):+ backend = DiffusionBackend()+ calls, reasons = _stub_nvidia_offload_host(backend, monkeypatch)+ status = backend.load_pipeline(+ "Qwen/Qwen-Image-2512",+ model_kind = "pipeline",+ transformer_quant = "int8",+ memory_mode = "fast",+ _base_local_dir = str(tmp_path),+ )+ assert status["offload_policy"] == "none"+ assert len(calls) == 1 and "offload" not in calls[0] and "act_int8" not in calls[0]+ assert status["transformer_quant"] == "int8"+ assert reasons == []+ assert "W8A8" not in status["resolved"]["transformer_quant"]["reason"]+ backend.unload()+++def test_auto_under_offload_on_nvidia_never_goes_native(fake_runtime, tmp_path, monkeypatch):+ backend = DiffusionBackend()+ calls, reasons = _stub_nvidia_offload_host(backend, monkeypatch)+ status = backend.load_pipeline(+ "Qwen/Qwen-Image-2512",+ model_kind = "pipeline",+ memory_mode = "balanced",+ _base_local_dir = str(tmp_path),+ )+ assert calls == [] and reasons == []+ assert status["transformer_quant"] is None+ backend.unload()+++def test_a_resident_nvidia_int8_that_cannot_compile_still_declines(+ fake_runtime, tmp_path, monkeypatch, allow_precision_fallback+):+ from core.inference import diffusion as dmod++ backend = DiffusionBackend()+ calls, _ = _stub_nvidia_offload_host(backend, monkeypatch)+ monkeypatch.setattr(+ dmod, "_pipeline_quant_uncompilable_reason", lambda *a, **k: "no compile here (stub)"+ )+ status = backend.load_pipeline(+ "Qwen/Qwen-Image-2512",+ model_kind = "pipeline",+ transformer_quant = "int8",+ memory_mode = "fast",+ _base_local_dir = str(tmp_path),+ )+ assert calls == []+ assert status["transformer_quant"] is None+ assert status["resolved"]["transformer_quant"]["reason"] == "no compile here (stub)"+ backend.unload()+++def test_an_offloaded_nvidia_int8_ignores_the_compile_requirement(+ fake_runtime, tmp_path, monkeypatch+):+ from core.inference import diffusion as dmod++ backend = DiffusionBackend()+ calls, _ = _stub_nvidia_offload_host(backend, monkeypatch)+ monkeypatch.setattr(+ dmod, "_pipeline_quant_uncompilable_reason", lambda *a, **k: "no compile here (stub)"+ )+ status = backend.load_pipeline(+ "Qwen/Qwen-Image-2512",+ model_kind = "pipeline",+ transformer_quant = "int8",+ memory_mode = "low_vram",+ _base_local_dir = str(tmp_path),+ )+ assert len(calls) == 1 and calls[0]["offload"] is True+ assert status["transformer_quant"] == "int8"+ backend.unload()++def _record_step_cache(monkeypatch,*,diff --git a/studio/backend/tests/test_diffusion_native_quant.py b/studio/backend/tests/test_diffusion_native_quant.pyindex 013f768715e..28c52d83595 100644--- a/studio/backend/tests/test_diffusion_native_quant.py+++ b/studio/backend/tests/test_diffusion_native_quant.py@@ -1,7 +1,7 @@# SPDX-License-Identifier: AGPL-3.0-only# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0-"""Torchao-free weight-only int8 / fp8; ROCm simulated by patching ``torch_is_rocm`` / ``is_stubbed``."""+"""Torchao-free int8 / fp8 (AMD, ROCm stub) and NVIDIA int8 W8A8 under offload; ROCm is monkeypatched."""from __future__ import annotations@@ -104,10 +104,87 @@ def test_non_cuda_or_non_bf16_is_never_native_even_on_rocm(rocm, target):@pytest.mark.parametrize("device", ["cuda", "cpu", "mps", "xpu"])-def test_nvidia_and_other_hosts_are_never_native(nvidia, device):+def test_nvidia_and_other_hosts_are_never_native_without_offload(nvidia, device):assert not tq.native_quant_host(_target(device = device))for scheme in ("int8", "fp8", "nvfp4", "mxfp8", "auto"):assert tq.native_quant_scheme(_target(device = device), scheme) is None+ assert tq.native_quant_scheme(_target(device = device), scheme, offload = False) is None+++_NVIDIA_TABLE = [+ ("int8", True, "int8"),+ ("INT8", True, "int8"),+ ("int8", False, None),+ ("fp8", True, None),+ ("fp8", False, None),+ ("nvfp4", True, None),+ ("mxfp8", True, None),+ ("auto", True, None),+ (None, True, None),+ ("none", True, None),+]++[email protected]("scheme,offload,expect", _NVIDIA_TABLE)+def test_nvidia_takes_native_int8_only_under_offload(nvidia, scheme, offload, expect):+ assert tq.native_offload_host(_target())+ assert tq.native_quant_scheme(_target(), scheme, offload = offload) == expect+++ "target",+ [+ _target(device = "cpu"),+ _target(device = "mps"),+ _target(device = "xpu"),+ _target(dtype = torch.float16),+ _target(dtype = torch.float32),+ ],+)+def test_the_offload_route_needs_a_bf16_cuda_target(nvidia, target):+ assert not tq.native_offload_host(target)+ assert tq.native_quant_scheme(target, "int8", offload = True) is None+++def test_amd_hosts_are_not_the_nvidia_offload_host(rocm):+ assert not tq.native_offload_host(_target())+ for scheme in ("int8", "fp8"):+ assert tq.native_quant_scheme(_target(), scheme, offload = True) == scheme+ assert tq.native_quant_scheme(_target(), scheme, offload = False) == scheme+++def test_windows_stub_is_not_the_nvidia_offload_host(win_stub):+ assert not tq.native_offload_host(_target())+ assert tq.native_quant_scheme(_target(), "fp8", offload = True) == "fp8"+++def test_nvidia_offload_route_respects_the_family_deny_list(nvidia, monkeypatch):+ monkeypatch.setitem(tq._FAMILY_SCHEME_DENY, "toy-family", frozenset({tq.TQ_INT8}))+ assert tq.native_quant_scheme(_target(), "int8", family = "toy-family", offload = True) is None+ assert (+ tq.native_quant_scheme(_target(), "int8", family = "qwen-image-2.1", offload = True) == "int8"+ )+++ "env,expect_amd,expect_nvidia",+ [+ (None, False, True),+ ("1", True, True),+ ("0", False, False),+ ("yes", False, True),+ ],+)+def test_w8a8_default_differs_by_route(monkeypatch, env, expect_amd, expect_nvidia):+ if env is None:+ monkeypatch.delenv(nq.NATIVE_INT8_ACT_ENV, raising = False)+ else:+ monkeypatch.setenv(nq.NATIVE_INT8_ACT_ENV, env)+ monkeypatch.setattr(tq, "is_stubbed", lambda name: False)+ monkeypatch.setattr(tq, "torch_is_rocm", lambda: True)+ assert tq.native_int8_act(_target()) is expect_amd+ monkeypatch.setattr(tq, "torch_is_rocm", lambda: False)+ assert tq.native_int8_act(_target()) is expect_nvidiadef test_family_deny_list_still_applies(rocm, monkeypatch):@@ -184,6 +261,130 @@ def test_a_module_wide_dtype_cast_leaves_the_stored_weights_alone(scheme):assert torch.allclose(layer(x.to(torch.float16)).float(), before.float(), rtol = 1e-2, atol = 1e-2)+def _int_mm_on_cpu():+ try:+ torch._int_mm(torch.ones(32, 8, dtype = torch.int8), torch.ones(8, 8, dtype = torch.int8))+ return True+ except (RuntimeError, AttributeError, NotImplementedError):+ return False+++def test_int8_activation_path_is_off_by_default(monkeypatch):+ monkeypatch.delenv(nq.NATIVE_INT8_ACT_ENV, raising = False)+ assert not nq.int8_act_requested()+ monkeypatch.setenv(nq.NATIVE_INT8_ACT_ENV, "0")+ assert not nq.int8_act_requested()+ monkeypatch.setenv(nq.NATIVE_INT8_ACT_ENV, "1")+ assert nq.int8_act_requested()+++def test_int8_activation_flag_is_ignored_for_fp8():+ lin = torch.nn.Linear(64, 64).to(torch.bfloat16)+ assert nq.native_linear_class()(lin, "int8", act_int8 = True).act_int8+ assert not nq.native_linear_class()(lin, "fp8", act_int8 = True).act_int8+ assert not nq.native_linear_class()(lin, "int8").act_int8+++def test_int_mm_shape_gate():+ cls = nq.native_linear_class()+ layer = cls(torch.nn.Linear(64, 64).to(torch.bfloat16), "int8", act_int8 = True)+ assert not layer._int_mm_ok(16) and layer._int_mm_ok(17)+ odd = cls(torch.nn.Linear(64, 12).to(torch.bfloat16), "int8", act_int8 = True)+ assert not odd._int_mm_ok(4096)+++def test_cpu_inputs_keep_the_weight_only_path(monkeypatch):+ torch.manual_seed(0)+ lin = torch.nn.Linear(64, 64).to(torch.bfloat16)+ layer = nq.native_linear_class()(lin, "int8", act_int8 = True)+ monkeypatch.setattr(+ layer, "_forward_int_mm", lambda x: pytest.fail("int_mm used on a CPU input")+ )+ x = torch.randn(32, 64, dtype = torch.bfloat16)+ assert torch.equal(+ layer(x), torch.nn.functional.linear(x, layer.dequantized_weight(x.dtype), lin.bias)+ )++[email protected](not _int_mm_on_cpu(), reason = "torch._int_mm has no CPU kernel in this build")+def test_int_mm_matches_the_dense_layer():+ torch.manual_seed(0)+ lin = torch.nn.Linear(256, 128).to(torch.bfloat16)+ layer = nq.native_linear_class()(lin, "int8", act_int8 = True)+ x = torch.randn(2, 24, 256, dtype = torch.bfloat16)+ ref = lin(x).float()+ got = layer._forward_int_mm(x)+ assert got.shape == ref.shape and got.dtype == torch.bfloat16+ assert ((got.float() - ref).norm() / ref.norm()).item() < 0.03+++def test_rotation_only_applies_to_w8a8_and_divisible_inputs():+ cls = nq.native_linear_class()+ lin = torch.nn.Linear(512, 64).to(torch.bfloat16)+ assert cls(lin, "int8", act_int8 = True, rot_group = 256).rot_group == 256+ assert cls(lin, "int8", rot_group = 256).rot_group == 0+ assert cls(lin, "fp8", act_int8 = True, rot_group = 256).rot_group == 0+ odd = torch.nn.Linear(320, 64).to(torch.bfloat16)+ assert cls(odd, "int8", act_int8 = True, rot_group = 256).rot_group == 0+++def test_rotated_layer_keeps_the_model_basis():+ torch.manual_seed(0)+ lin = torch.nn.Linear(512, 128).to(torch.bfloat16)+ layer = nq.native_linear_class()(lin, "int8", act_int8 = True, rot_group = 256)+ assert "rot_h" not in layer.state_dict()+ w = lin.weight.float()+ assert ((layer.weight.float() - w).norm() / w.norm()).item() < 0.01+ x = torch.randn(4, 512, dtype = torch.bfloat16)+ ref = lin(x).float()+ got = layer(x).float()+ assert ((got - ref).norm() / ref.norm()).item() < 0.02++[email protected](not _int_mm_on_cpu(), reason = "torch._int_mm has no CPU kernel in this build")+def test_rotation_cuts_the_int8_activation_error_on_outlier_channels():+ torch.manual_seed(0)+ lin = torch.nn.Linear(512, 256).to(torch.bfloat16)+ cls = nq.native_linear_class()+ plain = cls(lin, "int8", act_int8 = True)+ rot = cls(lin, "int8", act_int8 = True, rot_group = 256)+ x = torch.randn(64, 512, dtype = torch.bfloat16)+ x[:, :5] *= 30.0+ ref = lin(x).float()+ err = lambda y: ((y.float() - ref).norm() / ref.norm()).item() # noqa: E731+ assert err(rot._forward_int_mm(rot._rotate(x))) < 0.5 * err(plain._forward_int_mm(x))+++def test_rotation_env_switch(monkeypatch):+ monkeypatch.delenv(nq.NATIVE_INT8_ROT_ENV, raising = False)+ assert nq.int8_rotation_group() == 256+ monkeypatch.setenv(nq.NATIVE_INT8_ROT_ENV, "0")+ assert nq.int8_rotation_group() == 0+++def test_apply_native_weight_quant_takes_act_from_the_caller_not_the_env(monkeypatch):+ for env, act, scheme, expect in (+ ("", True, "int8", True),+ ("1", False, "int8", False),+ ("0", True, "int8", True),+ ("1", True, "fp8", False),+ ):+ monkeypatch.setenv(nq.NATIVE_INT8_ACT_ENV, env)+ model = _toy()+ n = nq.apply_native_weight_quant(+ model, scheme, filter_fn = lambda m, name: isinstance(m, torch.nn.Linear), act_int8 = act+ )+ assert n > 0+ flags = {m.act_int8 for m in model.modules() if hasattr(m, "act_int8")}+ assert flags == {expect}+++def test_int8_act_kill_switch_reads_only_zero(monkeypatch):+ for value, disabled in (("", False), ("1", False), ("0", True), (" 0 ", True), ("off", False)):+ monkeypatch.setenv(nq.NATIVE_INT8_ACT_ENV, value)+ assert nq.int8_act_disabled() is disabled++def _block_torchao(monkeypatch):# A None entry makes any `import torchao.quantization` raise, proving the branch never reaches it.monkeypatch.setitem(sys.modules, "torchao", None)@@ -251,11 +452,11 @@ def test_quantize_transformer_failure_is_reported_as_dirty(rocm, monkeypatch):real = nq.native_linear_class()calls = []- def _boom(linear, scheme):+ def _boom(linear, scheme, **kw):calls.append(1)if len(calls) == 3:raise RuntimeError("out of memory")- return real(linear, scheme)+ return real(linear, scheme, **kw)monkeypatch.setattr(nq, "native_linear_class", lambda: _boom)warned = []@@ -302,6 +503,149 @@ def test_quantize_transformer_on_nvidia_never_reaches_the_native_branch(nvidia,assert len(seen) == 3+def _logger(logs):+ return types.SimpleNamespace(+ info = lambda *a: logs.append(a[0] % a[1:]), warning = lambda *a: pytest.fail(f"warned: {a}")+ )+++def test_quantize_transformer_on_nvidia_under_offload_runs_w8a8_convrot(nvidia, monkeypatch):+ _block_torchao(monkeypatch)+ monkeypatch.delenv(nq.NATIVE_INT8_ACT_ENV, raising = False)+ monkeypatch.delenv(nq.NATIVE_INT8_ROT_ENV, raising = False)+ monkeypatch.setattr(+ tq, "select_transformer_quant_scheme", lambda *a, **k: pytest.fail("torchao selector asked")+ )+ torch.manual_seed(0)+ model = torch.nn.Sequential(+ torch.nn.Linear(512, 512), torch.nn.GELU(), torch.nn.Linear(512, 512)+ )+ model = model.to(torch.bfloat16)+ x = torch.randn(4, 512, dtype = torch.bfloat16)+ with torch.no_grad():+ ref = model(x).float()+ logs = []+ got = tq.quantize_transformer(+ types.SimpleNamespace(transformer = model),+ _target(),+ mode = "int8",+ min_features = 64,+ logger = _logger(logs),+ offload = True,+ )+ assert got == "int8"+ layers = [m for m in model.modules() if nq.is_native_linear(m)]+ assert len(layers) == 2+ assert all(m.act_int8 and m.rot_group == 256 for m in layers)+ assert any("W8A8 torch._int_mm ConvRot g256" in line for line in logs)+ assert nq.native_quant_signature(model) == "int8-w8a8-rot256"+ assert nq.native_quant_reason(model, "int8").startswith("W8A8")+ with torch.no_grad():+ out = model(x).float()+ assert ((out - ref).norm() / ref.norm()).item() < 0.05+++def test_quantize_transformer_honours_the_kill_switches_on_nvidia(nvidia, monkeypatch):+ _block_torchao(monkeypatch)+ monkeypatch.setenv(nq.NATIVE_INT8_ACT_ENV, "0")+ model = _toy()+ assert (+ tq.quantize_transformer(+ types.SimpleNamespace(transformer = model),+ _target(),+ mode = "int8",+ min_features = 64,+ offload = True,+ )+ == "int8"+ )+ assert {m.act_int8 for m in model.modules() if nq.is_native_linear(m)} == {False}+ assert nq.native_quant_signature(model) == "int8-wo"+ assert nq.native_quant_reason(model, "int8").startswith("weight-only")++ monkeypatch.delenv(nq.NATIVE_INT8_ACT_ENV)+ monkeypatch.setenv(nq.NATIVE_INT8_ROT_ENV, "0")+ model = _toy()+ tq.quantize_transformer(+ types.SimpleNamespace(transformer = model),+ _target(),+ mode = "int8",+ min_features = 64,+ offload = True,+ )+ layers = [m for m in model.modules() if nq.is_native_linear(m)]+ assert layers and all(m.act_int8 and m.rot_group == 0 for m in layers)+ assert nq.native_quant_signature(model) == "int8-w8a8"+++def test_an_explicit_act_argument_overrides_the_host_default(nvidia, monkeypatch):+ _block_torchao(monkeypatch)+ monkeypatch.delenv(nq.NATIVE_INT8_ACT_ENV, raising = False)+ model = _toy()+ tq.quantize_transformer(+ types.SimpleNamespace(transformer = model),+ _target(),+ mode = "int8",+ min_features = 64,+ offload = True,+ act_int8 = False,+ )+ assert {m.act_int8 for m in model.modules() if nq.is_native_linear(m)} == {False}++[email protected]("scheme", ["fp8", "auto", "nvfp4"])+def test_other_schemes_under_offload_on_nvidia_stay_on_torchao(nvidia, monkeypatch, scheme):+ monkeypatch.setattr(+ tq, "_quantize_native", lambda *a, **k: pytest.fail("native branch for a non-int8 scheme")+ )+ seen = []+ monkeypatch.setattr(+ tq, "select_transformer_quant_scheme", lambda *a, **k: seen.append(a) or None+ )+ assert (+ tq.quantize_transformer(+ types.SimpleNamespace(transformer = _toy()), _target(), mode = scheme, offload = True+ )+ is None+ )+ assert len(seen) == 1+++def test_amd_quantize_keeps_its_opt_in_w8a8(rocm, monkeypatch):+ _block_torchao(monkeypatch)+ for env, expect in (("", False), ("1", True)):+ monkeypatch.setenv(nq.NATIVE_INT8_ACT_ENV, env)+ model = _toy()+ tq.quantize_transformer(+ types.SimpleNamespace(transformer = model), _target(), mode = "int8", min_features = 64+ )+ assert {m.act_int8 for m in model.modules() if nq.is_native_linear(m)} == {expect}+++def test_compile_cache_keys_native_apart_from_torchao():+ from core.inference.diffusion_compile_cache import model_fingerprint++ dense = _toy()+ native = _toy()+ nq.apply_native_weight_quant(+ native, "int8", filter_fn = lambda m, name: isinstance(m, torch.nn.Linear), act_int8 = True+ )++ def fp(module, quant):+ return model_fingerprint(+ family = "toy",+ transformer = module,+ dtype = torch.bfloat16,+ quant = quant,+ attention_backend = None,+ compile_kwargs = {},+ )["quant"]++ assert fp(dense, "int8") == "int8"+ assert fp(dense, None) == "none"+ assert fp(native, "int8").startswith("int8-w8a8")++@pytest.mark.parametrize("scheme", ["int8", "fp8"])def test_reading_weight_gives_the_dense_weight_without_storing_it(scheme):"""PEFT DoRA reads ``base_layer.weight``: dense, rebuilt per read, not in the state dict."""@@ -369,6 +713,7 @@ def _image_gate(*,model_kind = "pipeline",memory_mode = None,+ cpu_offload = False,):import core.inference.diffusion as dmonkeypatch.setattr(d, "effective_te_quant", lambda *a, **k: None)@@ -380,7 +725,7 @@ def _image_gate(pinned = pinned,te_mode = None,memory_mode = memory_mode,- cpu_offload = False,+ cpu_offload = cpu_offload,)@@ -460,9 +805,144 @@ def test_video_gate_refuses_native_on_gguf_loads(rocm, monkeypatch):_video_gate(monkeypatch, _video_family("wan2.2-ti2v-5b"), "int8", model_kind = "gguf")+def _torchao_gate_answers(+ monkeypatch,+ module,+ *,+ supported = True,+):+ monkeypatch.setattr(module, "dense_transformer_supported", lambda target: supported)+ monkeypatch.setattr(module, "select_transformer_quant_scheme", lambda target, mode, **k: mode)+ if hasattr(module, "_pipeline_quant_uncompilable_reason"):+ monkeypatch.setattr(module, "_pipeline_quant_uncompilable_reason", lambda *a, **k: None)+++ "memory",+ [{"memory_mode": "balanced"}, {"memory_mode": "low_vram"}, {"cpu_offload": True}],+)+def test_image_gate_admits_int8_under_offload_on_nvidia(nvidia, monkeypatch, memory):+ import core.inference.diffusion as d+ _torchao_gate_answers(monkeypatch, d)+ _image_gate(monkeypatch, _image_family("qwen-image-2.1"), "int8", **memory)++[email protected]("pinned", ["fp8", "nvfp4", "mxfp8"])[email protected]("memory", [{"memory_mode": "balanced"}, {"cpu_offload": True}])+def test_image_gate_still_refuses_other_schemes_under_offload_on_nvidia(+ nvidia, monkeypatch, pinned, memory+):+ import core.inference.diffusion as d+ _torchao_gate_answers(monkeypatch, d)+ with pytest.raises(RuntimeError, match = "cannot be moved by the offload hooks"):+ _image_gate(monkeypatch, _image_family("qwen-image-2.1"), pinned, **memory)++[email protected]("kind", ["gguf", "single_file"])+def test_image_gate_keeps_non_pipeline_int8_under_offload_refused_on_nvidia(+ nvidia, monkeypatch, kind+):+ import core.inference.diffusion as d+ _torchao_gate_answers(monkeypatch, d)+ with pytest.raises(RuntimeError, match = "transformer_quant"):+ _image_gate(+ monkeypatch,+ _image_family("qwen-image-2.1"),+ "int8",+ model_kind = kind,+ memory_mode = "balanced",+ )+++def test_image_gate_refuses_a_denied_family_under_offload_on_nvidia(nvidia, monkeypatch):+ import core.inference.diffusion as d++ _torchao_gate_answers(monkeypatch, d)+ monkeypatch.setitem(tq._FAMILY_SCHEME_DENY, "qwen-image-2.1", frozenset({tq.TQ_INT8}))+ with pytest.raises(RuntimeError, match = "ruled out"):+ _image_gate(monkeypatch, _image_family("qwen-image-2.1"), "int8", memory_mode = "balanced")++[email protected]("memory_mode", [None, "fast", "auto"])+def test_image_gate_on_nvidia_without_a_named_offload_never_asks_native(+ nvidia, monkeypatch, memory_mode+):+ import core.inference.diffusion as d++ _torchao_gate_answers(monkeypatch, d)+ monkeypatch.setattr(+ d, "native_quant_scheme", lambda *a, **k: pytest.fail("native on a resident request")+ )+ for pinned in ("int8", "fp8"):+ _image_gate(monkeypatch, _image_family("qwen-image-2.1"), pinned, memory_mode = memory_mode)+++def test_image_gate_offload_route_needs_no_compile(nvidia, monkeypatch):+ import core.inference.diffusion as d++ _torchao_gate_answers(monkeypatch, d)+ monkeypatch.setattr(+ d, "_pipeline_quant_uncompilable_reason", lambda *a, **k: "no compile (stub)"+ )+ _image_gate(monkeypatch, _image_family("qwen-image-2.1"), "int8", memory_mode = "low_vram")+ with pytest.raises(RuntimeError, match = "no compile"):+ _image_gate(monkeypatch, _image_family("qwen-image-2.1"), "int8", memory_mode = "fast")++[email protected]("memory_mode", ["balanced", "low_vram"])+def test_video_gate_admits_int8_under_offload_on_nvidia(nvidia, monkeypatch, memory_mode):+ import core.inference.video as v+ _torchao_gate_answers(monkeypatch, v)+ _video_gate(monkeypatch, _video_family("wan2.2-ti2v-5b"), "int8", memory_mode = memory_mode)++[email protected]("pinned", ["fp8", "nvfp4", "mxfp8"])+def test_video_gate_still_refuses_other_schemes_under_offload_on_nvidia(+ nvidia, monkeypatch, pinned+):+ import core.inference.video as v+ _torchao_gate_answers(monkeypatch, v)+ with pytest.raises(RuntimeError, match = "cannot be moved by the offload hooks"):+ _video_gate(monkeypatch, _video_family("wan2.2-ti2v-5b"), pinned, memory_mode = "balanced")+++def test_video_gate_refuses_a_denied_family_under_offload_on_nvidia(nvidia, monkeypatch):+ import core.inference.video as v++ _torchao_gate_answers(monkeypatch, v)+ monkeypatch.setitem(tq._FAMILY_SCHEME_DENY, "wan2.2-ti2v-5b", frozenset({tq.TQ_INT8}))+ with pytest.raises(RuntimeError, match = "ruled out"):+ _video_gate(monkeypatch, _video_family("wan2.2-ti2v-5b"), "int8", memory_mode = "balanced")+++def test_video_gate_keeps_gguf_int8_under_offload_refused_on_nvidia(nvidia, monkeypatch):+ import core.inference.video as v+ _torchao_gate_answers(monkeypatch, v)+ with pytest.raises(RuntimeError, match = "transformer_quant"):+ _video_gate(+ monkeypatch,+ _video_family("wan2.2-ti2v-5b"),+ "int8",+ model_kind = "gguf",+ memory_mode = "balanced",+ )++[email protected]("memory_mode", [None, "fast"])+def test_video_gate_on_nvidia_resident_keeps_the_torchao_answer(nvidia, monkeypatch, memory_mode):+ import core.inference.video as v++ _torchao_gate_answers(monkeypatch, v)+ monkeypatch.setattr(+ v, "native_quant_scheme", lambda *a, **k: pytest.fail("native on a resident request")+ )+ for pinned in ("int8", "fp8"):+ _video_gate(monkeypatch, _video_family("wan2.2-ti2v-5b"), pinned, memory_mode = memory_mode)++def _native_block(features = 64):lin = torch.nn.Linear(features, features, dtype = torch.bfloat16)- return nq.native_linear_class()(lin, "int8")+ return nq.native_linear_class()(lin, "int8", act_int8 = True, rot_group = 0)def test_stream_group_offload_puts_native_weight_buffers_back_on_the_host():@@ -534,6 +1014,49 @@ def test_native_int8_under_real_stream_group_offload_leaves_nothing_resident():assert {b.device.type for b in model.buffers()} == {"cpu"}+class _GateSeen(Exception):+ pass+++ "memory", [{"memory_mode": "balanced"}, {"memory_mode": "low_vram"}, {"cpu_offload": True}]+)+def test_image_begin_load_rechecks_with_the_offload_request(monkeypatch, memory):+ import core.inference.diffusion as d++ seen = {}++ def gate(self, fam, **kwargs):+ seen.update(kwargs)+ raise _GateSeen++ monkeypatch.setattr(d.DiffusionBackend, "validate_load_request", lambda self, *a, **k: None)+ monkeypatch.setattr(d.DiffusionBackend, "assert_precision_available", gate)+ with pytest.raises(_GateSeen):+ d.DiffusionBackend().begin_load(+ "some/repo", transformer_quant = "int8", speed_mode = "eager", **memory+ )+ assert seen.get("memory_mode") == memory.get("memory_mode")+ assert seen.get("cpu_offload", False) == memory.get("cpu_offload", False)++[email protected]("memory_mode", ["balanced", "low_vram"])+def test_video_begin_load_rechecks_with_the_memory_request(monkeypatch, memory_mode):+ import core.inference.video as v++ seen = {}++ def gate(fam, **kwargs):+ seen.update(kwargs)+ raise _GateSeen++ monkeypatch.setattr(v.VideoBackend, "validate_load_request", lambda self, *a, **k: None)+ monkeypatch.setattr(v, "assert_video_precision_available", gate)+ with pytest.raises(_GateSeen):+ v.VideoBackend().begin_load("some/repo", transformer_quant = "int8", memory_mode = memory_mode)+ assert seen.get("memory_mode") == memory_mode++def test_qkv_fusion_skips_a_native_quantised_denoiser():from core.inference import diffusion_speed@@ -552,3 +1075,40 @@ def test_qkv_fusion_skips_a_native_quantised_denoiser():assert diffusion_speed._fuse_qkv(native, None) is False and not callsdense = types.SimpleNamespace(transformer = _Dit(native = False))assert diffusion_speed._fuse_qkv(dense, None) is True and len(calls) == 1++[email protected](not torch.cuda.is_available(), reason = "the W8A8 path runs on CUDA inputs only")+def test_eager_w8a8_chunks_long_inputs_without_changing_the_result(monkeypatch):+ torch.manual_seed(0)+ layer = nq.native_linear_class()(+ torch.nn.Linear(64, 32).to(torch.bfloat16), "int8", act_int8 = True+ ).cuda()+ x = torch.randn(300, 64, dtype = torch.bfloat16, device = "cuda")+ whole = layer(x)+ seen = []+ rows = layer._int_mm_rows+ monkeypatch.setattr(+ layer, "_int_mm_rows", lambda part: seen.append(part.shape[0]) or rows(part)+ )+ monkeypatch.setattr(nq, "_INT_MM_CHUNK_ELEMS", 64 * 40)+ assert torch.equal(layer(x), whole)+ assert len(seen) > 1 and sum(seen) == 300 and min(seen) >= 17+++def test_signature_sees_rotation_past_an_unrotated_first_layer():+ cls = nq.native_linear_class()+ model = torch.nn.Sequential(+ cls(torch.nn.Linear(320, 256).to(torch.bfloat16), "int8", act_int8 = True, rot_group = 256),+ cls(torch.nn.Linear(256, 64).to(torch.bfloat16), "int8", act_int8 = True, rot_group = 256),+ )+ assert model[0].rot_group == 0 and model[1].rot_group == 256+ assert nq.native_quant_signature(model) == "int8-w8a8-rot256"+ assert "g256 ConvRot" in nq.native_quant_reason(model, "int8")+++def test_a_rotated_layer_passes_an_empty_input_through():+ layer = nq.native_linear_class()(+ torch.nn.Linear(256, 64).to(torch.bfloat16), "int8", act_int8 = True, rot_group = 256+ )+ out = layer(torch.zeros(2, 0, 256, dtype = torch.bfloat16))+ assert out.shape == (2, 0, 64)diff --git a/studio/backend/tests/test_video_backend.py b/studio/backend/tests/test_video_backend.pyindex 91d819879ba..7642e0db82d 100644--- a/studio/backend/tests/test_video_backend.py+++ b/studio/backend/tests/test_video_backend.py@@ -2211,12 +2211,13 @@ def test_wan_a14b_dense_quant_applies_to_both_dits(fake_runtime, monkeypatch):def test_dense_quant_skipped_under_offload(fake_runtime, monkeypatch):# Offload hooks move modules with Module.to(), which torchao tensors reject, so any offload- # policy must SKIP quant. With the legacy escape hatch set the load still succeeds dense and the- # record explains why; the strict default refuses instead (see the test below).+ # policy must SKIP a torchao quant (escape hatch: dense plus a record).import core.inference.video as video_mod+ from core.inference import diffusion_transformer_quant as tqmonkeypatch.setenv("UNSLOTH_DIFFUSION_ALLOW_PRECISION_FALLBACK", "1")monkeypatch.setattr(video_mod, "dense_transformer_supported", lambda target: True)+ monkeypatch.setattr(tq, "native_offload_host", lambda target: True)quantised = []def _fake_quant(@@ -2241,7 +2242,7 @@ def test_dense_quant_skipped_under_offload(fake_runtime, monkeypatch):status = backend.load_pipeline("Wan-AI/Wan2.2-T2V-A14B-Diffusers",model_kind = "pipeline",- transformer_quant = "int8",+ transformer_quant = "fp8",)_assert_placement_follows_the_target(placements, video_mod)assert status["offload_policy"] == "model"@@ -2250,7 +2251,7 @@ def test_dense_quant_skipped_under_offload(fake_runtime, monkeypatch):resolved = status["resolved"]["transformer_quant"]assert "moves the DiT" in resolved["reason"]# BOTH sides of the story survive: the ask, the outcome, and that they disagree.- assert resolved["requested"] == "int8"+ assert resolved["requested"] == "fp8"assert resolved["value"] == "off"assert resolved["status"] == "fell_back"@@ -2281,13 +2282,13 @@ def test_the_video_load_places_on_the_selected_card_not_a_bare_device(fake_runtidef test_explicit_dense_quant_refuses_under_offload(fake_runtime, monkeypatch):- # Strict default (no escape hatch): an explicit int8 the offload plan cannot honor stops the- # load, rather than denoising at bf16 while the Precision dropdown still reads INT8.import dataclassesimport core.inference.video as video_mod+ from core.inference import diffusion_transformer_quant as tqmonkeypatch.setattr(video_mod, "dense_transformer_supported", lambda target: True)+ monkeypatch.setattr(tq, "native_offload_host", lambda target: True)monkeypatch.setattr(video_mod,"quantize_transformer",@@ -2299,9 +2300,9 @@ def test_explicit_dense_quant_refuses_under_offload(fake_runtime, monkeypatch):backend.load_pipeline("Wan-AI/Wan2.2-T2V-A14B-Diffusers",model_kind = "pipeline",- transformer_quant = "int8",+ transformer_quant = "fp8",)- assert "transformer_quant='int8' could not be used" in str(excinfo.value)+ assert "transformer_quant='fp8' could not be used" in str(excinfo.value)assert "resident memory mode" in str(excinfo.value)assert backend.status()["loaded"] is False@@ -10059,6 +10060,97 @@ def test_video_status_reports_cuda_graph_off_once_every_armed_step_ran_eager(fakbackend.unload()+def _stub_nvidia_video_offload(monkeypatch, *, offload = True):+ import dataclasses++ import core.inference.video as video_mod+ from core.inference import diffusion_transformer_quant as tq++ monkeypatch.setattr(video_mod, "dense_transformer_supported", lambda target: True)+ monkeypatch.setattr(tq, "native_quant_host", lambda target: False)+ monkeypatch.setattr(tq, "native_offload_host", lambda target: True)+ monkeypatch.setattr(+ video_mod, "select_transformer_quant_scheme", lambda target, mode, family = None: mode+ )+ monkeypatch.delenv("UNSLOTH_NATIVE_INT8_ACT", raising = False)+ if offload:+ real_plan = video_mod.plan_diffusion_memory+ monkeypatch.setattr(+ video_mod,+ "plan_diffusion_memory",+ lambda **kwargs: dataclasses.replace(real_plan(**kwargs), offload_policy = "model"),+ )+ _stub_apply_memory_plan(monkeypatch, video_mod, policy = "model" if offload else "none")+ calls: list = []++ def _quantize(+ view,+ target,+ *,+ mode,+ family = None,+ **kw,+ ):+ calls.append({"view": view, "mode": mode, **kw})+ return mode++ monkeypatch.setattr(video_mod, "quantize_transformer", _quantize)+ monkeypatch.setattr(+ video_mod, "native_quant_reason", lambda module, scheme: f"W8A8: {scheme} (stub)"+ )+ return calls+++def test_an_explicit_int8_under_offload_on_nvidia_runs_native_on_both_experts(+ fake_runtime, monkeypatch+):+ calls = _stub_nvidia_video_offload(monkeypatch)+ backend = VideoBackend()+ status = backend.load_pipeline(+ "Wan-AI/Wan2.2-T2V-A14B-Diffusers", model_kind = "pipeline", transformer_quant = "int8"+ )+ assert status["offload_policy"] == "model"+ assert len(calls) == 2+ assert all(c["offload"] is True and c["act_int8"] is True for c in calls)+ assert status["transformer_quant"] == "int8"+ resolved = status["resolved"]["transformer_quant"]+ assert resolved["status"] == "applied" and resolved["reason"].startswith("W8A8")+ backend.unload()+++def test_the_video_act_kill_switch_keeps_offload_native_weight_only(fake_runtime, monkeypatch):+ calls = _stub_nvidia_video_offload(monkeypatch)+ monkeypatch.setenv("UNSLOTH_NATIVE_INT8_ACT", "0")+ backend = VideoBackend()+ status = backend.load_pipeline(+ "Wan-AI/Wan2.2-TI2V-5B-Diffusers", model_kind = "pipeline", transformer_quant = "int8"+ )+ assert calls and all(c["offload"] is True and c["act_int8"] is False for c in calls)+ assert status["transformer_quant"] == "int8"+ backend.unload()+++def test_a_resident_video_int8_on_nvidia_keeps_torchao(fake_runtime, monkeypatch):+ calls = _stub_nvidia_video_offload(monkeypatch, offload = False)+ backend = VideoBackend()+ status = backend.load_pipeline(+ "Wan-AI/Wan2.2-TI2V-5B-Diffusers", model_kind = "pipeline", transformer_quant = "int8"+ )+ assert status["offload_policy"] == "none"+ assert calls and all("offload" not in c and "act_int8" not in c for c in calls)+ assert "W8A8" not in status["resolved"]["transformer_quant"]["reason"]+ backend.unload()+++def test_video_auto_under_offload_on_nvidia_is_still_skipped(fake_runtime, monkeypatch):+ calls = _stub_nvidia_video_offload(monkeypatch)+ backend = VideoBackend()+ status = backend.load_pipeline("Wan-AI/Wan2.2-TI2V-5B-Diffusers", model_kind = "pipeline")+ assert calls == []+ assert status["transformer_quant"] is None+ backend.unload()++def test_video_auto_quant_on_a_host_without_dense_quant_reports_as_before(fake_runtime):status = VideoBackend().load_pipeline("Wan-AI/Wan2.2-TI2V-5B-Diffusers", model_kind = "pipeline", speed_mode = "default"