Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

Studio: run an explicit int8 under offload on NVIDIA as torchao-free W8A8 (#11712)

Daniel Hande7c4fdfd0c8
diff --git a/studio/backend/core/inference/diffusion.py b/studio/backend/core/inference/diffusion.py
index 50f836666fe..f3732435e07 100644
--- a/studio/backend/core/inference/diffusion.py
+++ b/studio/backend/core/inference/diffusion.py
@@ -212,6 +212,7 @@ from .diffusion_auto_policy import (
)
from .diffusion_transformer_quant import (
TQ_AUTO,
+ TQ_INT8,
DEFAULT_MIN_LINEAR_FEATURES,
DenoiserView as _DenoiserView,
dense_quant_blocker,
@@ -222,8 +223,11 @@ from .diffusion_transformer_quant import (
denoiser_modules,
explain_unusable_scheme,
mark_source_precision,
+ native_int8_act,
+ native_offload_host,
native_quant_host,
native_quant_scheme,
+ NATIVE_OFFLOAD_SCHEMES,
NATIVE_QUANT_SCHEMES,
normalize_transformer_quant,
quantize_transformer,
@@ -231,6 +235,7 @@ from .diffusion_transformer_quant import (
stored_denoiser_precision,
transformer_is_quantised,
)
+from .diffusion_native_quant import native_quant_reason
from utils.paths.path_utils import (
any_not_appledouble_metadata,
is_appledouble_metadata,
@@ -1828,6 +1833,19 @@ class DiffusionBackend:
):
if native_quant_scheme(target, pinned, family = getattr(fam, "name", None)) is None:
reason = explain_unusable_scheme(getattr(fam, "name", None), pinned)
+ elif (
+ model_kind == "pipeline"
+ and pinned in NATIVE_OFFLOAD_SCHEMES
+ and _memory_request_forces_offload(memory_mode, cpu_offload)
+ and native_offload_host(target)
+ ):
+ if (
+ native_quant_scheme(
+ target, pinned, family = getattr(fam, "name", None), offload = True
+ )
+ is None
+ ):
+ reason = explain_unusable_scheme(getattr(fam, "name", None), pinned)
elif _memory_request_forces_offload(memory_mode, cpu_offload):
# Not a measurement: balanced and low_vram name their policy outright, and the legacy flag forces
# model offload. Offload hooks move modules with Module.to(), which torchao tensors do not survive, so
@@ -2524,6 +2542,8 @@ class DiffusionBackend:
model_kind = resolve_model_kind(gguf_filename, model_kind),
transformer_quant = transformer_quant,
text_encoder_quant = text_encoder_quant,
+ memory_mode = memory_mode,
+ cpu_offload = cpu_offload,
# An uncompiled torchao transformer loses to the bf16 it replaces, so 'eager' stays dense. Refusing
# an explicit scheme here says so before the download.
speed_mode = speed_mode,
@@ -4611,6 +4631,19 @@ class DiffusionBackend:
if kind == "pipeline"
else None
)
+ native_offload_scheme = (
+ native_quant_scheme(
+ target,
+ transformer_quant_pinned,
+ family = getattr(fam, "name", None),
+ offload = True,
+ )
+ if kind == "pipeline" and native_scheme is None
+ else None
+ )
+ native_offload_forced = native_offload_scheme is not None and (
+ _memory_request_forces_offload(memory_mode, cpu_offload)
+ )
if transformer_quant_pinned is not None and not dense_quant_supported_kind(kind):
transformer_quant_decline = dense_quant_unsupported_kind_reason(kind)
transformer_quant_decline_status = RESOLVED_UNSUPPORTED
@@ -4627,6 +4660,7 @@ class DiffusionBackend:
elif (
transformer_quant_pinned is not None
and native_scheme is None
+ and not native_offload_forced
and select_transformer_quant_scheme(
target, transformer_quant_pinned, family = getattr(fam, "name", None)
)
@@ -5510,9 +5544,10 @@ class DiffusionBackend:
if source_precision is not None:
for _attr, denoiser in denoiser_modules(pipe):
mark_source_precision(denoiser, source_precision)
- # Weight-only runs bf16 arithmetic, so it has no compile to require.
pipeline_quant_blocker = (
- None if native_scheme is not None else pipeline_quant_uncompilable
+ None
+ if native_scheme is not None or native_offload_scheme is not None
+ else pipeline_quant_uncompilable
) or dense_quant_blocker(pipe)
if pipeline_quant_blocker is not None:
logger.info(
@@ -5561,6 +5596,18 @@ class DiffusionBackend:
plan.offload_policy,
)
plan = replanned
+ if (
+ plan.offload_policy != OFFLOAD_NONE
+ and native_scheme is None
+ and native_offload_scheme is not None
+ ):
+ native_scheme = native_offload_scheme
+ logger.info(
+ "diffusion.transformer_quant: %s runs torchao-free under the plan's '%s' "
+ "offload (torchao tensors reject the hooks' Module.to())",
+ native_scheme,
+ plan.offload_policy,
+ )
if plan.offload_policy != OFFLOAD_NONE and native_scheme is None:
logger.info(
"diffusion.transformer_quant: skipped (the memory plan picked '%s' "
@@ -5572,6 +5619,14 @@ class DiffusionBackend:
"the transformer via Module.to(); torchao quantised tensors reject "
"that. Pin a resident memory mode to combine the two"
)
+ elif native_scheme is None and pipeline_quant_uncompilable is not None:
+ # Resident plan: torchao still needs the compile check skipped above.
+ logger.info(
+ "diffusion.transformer_quant: skipped (%s)",
+ pipeline_quant_uncompilable,
+ )
+ transformer_quant_decline = pipeline_quant_uncompilable
+ transformer_quant_decline_status = RESOLVED_UNSUPPORTED
else:
if _has_active_lora(loras):
# PEFT must wrap dense Linears before torchao converts their base layers.
@@ -5595,6 +5650,15 @@ class DiffusionBackend:
)
# Convert every denoiser so multi-branch pipelines use one precision.
denoisers = denoiser_modules(pipe)
+ native_kwargs = (
+ {
+ "offload": plan.offload_policy != OFFLOAD_NONE,
+ "act_int8": native_scheme == TQ_INT8
+ and native_int8_act(target),
+ }
+ if native_scheme is not None
+ else {}
+ )
engaged: list[str] = []
for attr, _module in denoisers:
scheme = quantize_transformer(
@@ -5606,6 +5670,7 @@ class DiffusionBackend:
family = getattr(fam, "name", None),
fast_accum = transformer_quant_fast_accum,
logger = logger,
+ **native_kwargs,
)
if scheme is None:
break
@@ -5960,8 +6025,9 @@ class DiffusionBackend:
)
)
if transformer_quant_engaged is None
- else f"weight-only: {transformer_quant_engaged} weights, bf16 compute "
- "(torchao-free, a memory saving rather than a speed-up)"
+ else native_quant_reason(
+ getattr(pipe, "transformer", None), transformer_quant_engaged
+ )
if native_scheme is not None
else f"seeded from the hosted checkpoint "
f"{transformer_quant_artifact.split(':', 1)[1]}"
diff --git a/studio/backend/core/inference/diffusion_compile_cache.py b/studio/backend/core/inference/diffusion_compile_cache.py
index c77c23678e5..16969082eba 100644
--- a/studio/backend/core/inference/diffusion_compile_cache.py
+++ b/studio/backend/core/inference/diffusion_compile_cache.py
@@ -298,6 +298,10 @@ def model_fingerprint(
what gets compiled.
"""
blocks = list(getattr(transformer, "_repeated_blocks", []) or [])
+ if quant is not None and transformer is not None:
+ # Native layers compile to a different graph than torchao under the same scheme name.
+ from .diffusion_native_quant import native_quant_signature
+ quant = native_quant_signature(transformer) or quant
return {
"family": str(family),
"transformer_cls": type(transformer).__name__ if transformer is not None else None,
diff --git a/studio/backend/core/inference/diffusion_native_quant.py b/studio/backend/core/inference/diffusion_native_quant.py
index 1b78d0e4903..74ccfbd98b8 100644
--- a/studio/backend/core/inference/diffusion_native_quant.py
+++ b/studio/backend/core/inference/diffusion_native_quant.py
@@ -14,6 +14,31 @@ from typing import Any, Callable, Optional
NATIVE_INT8 = "int8"
NATIVE_FP8 = "fp8"
_QMAX = {NATIVE_INT8: 127.0, NATIVE_FP8: 448.0}
+NATIVE_INT8_ACT_ENV = "UNSLOTH_NATIVE_INT8_ACT"
+NATIVE_INT8_ROT_ENV = "UNSLOTH_NATIVE_INT8_ROT"
+# torch._int_mm needs M > 16 and K, N multiples of 8.
+_INT_MM_MIN_ROWS = 17
+_INT_MM_CHUNK_ELEMS = 1 << 25
+
+
+def int8_act_requested() -> bool:
+ import os
+ return os.environ.get(NATIVE_INT8_ACT_ENV, "").strip() == "1"
+
+
+def int8_act_disabled() -> bool:
+ import os
+ return os.environ.get(NATIVE_INT8_ACT_ENV, "").strip() == "0"
+
+
+def int8_rotation_group() -> int:
+ import os
+
+ if os.environ.get(NATIVE_INT8_ROT_ENV, "").strip() == "0":
+ return 0
+ from .diffusion_convrot import DEFAULT_CONVROT_GROUPSIZE
+
+ return DEFAULT_CONVROT_GROUPSIZE
@lru_cache(maxsize = 1)
@@ -26,7 +51,13 @@ def native_linear_class():
class NativeWeightOnlyLinear(nn.Module):
"""A Linear whose weight is stored as int8 / fp8 with a per-output-row scale."""
- def __init__(self, linear: Any, scheme: str):
+ def __init__(
+ self,
+ linear: Any,
+ scheme: str,
+ act_int8: bool = False,
+ rot_group: int = 0,
+ ):
super().__init__()
if scheme not in _QMAX:
raise ValueError(f"unsupported native scheme {scheme!r}")
@@ -34,9 +65,22 @@ def native_linear_class():
self.out_features = int(linear.out_features)
self.scheme = scheme
self.compute_dtype = linear.weight.dtype
+ self.act_int8 = bool(act_int8) and scheme == NATIVE_INT8
+ self.rot_group = (
+ int(rot_group)
+ if self.act_int8 and rot_group and self.in_features % int(rot_group) == 0
+ else 0
+ )
weight = linear.weight.detach()
with torch.no_grad():
w = weight.float()
+ if self.rot_group:
+ from .diffusion_convrot import build_convrot_hadamard
+
+ h = build_convrot_hadamard(self.rot_group, device = w.device, dtype = torch.float32)
+ g = self.rot_group
+ w = (w.reshape(self.out_features, -1, g) @ h.T).reshape(self.out_features, -1)
+ self.register_buffer("rot_h", h.to(self.compute_dtype), persistent = False)
scale = w.abs().amax(dim = 1, keepdim = True).clamp(min = 1e-12) / _QMAX[scheme]
if scheme == NATIVE_INT8:
wq = (w / scale).round_().clamp_(-127, 127).to(torch.int8)
@@ -50,7 +94,7 @@ def native_linear_class():
)
self.bias = linear.bias
- def dequantized_weight(self, dtype: Any) -> Any:
+ def _stored_weight(self, dtype: Any) -> Any:
wq = (
self.weight_q.view(torch.float8_e4m3fn)
if self.scheme == NATIVE_FP8
@@ -59,19 +103,82 @@ def native_linear_class():
scale = self.weight_scale.view(torch.float32)
return (wq.to(torch.float32) * scale[:, None]).to(dtype)
+ def _rotate(self, x: Any) -> Any:
+ g = self.rot_group
+ # Leading dims, not -1: a zero-row input has no inferable -1 on older torch.
+ grouped = x.reshape(*x.shape[:-1], self.in_features // g, g)
+ return (grouped @ self.rot_h.to(x.dtype)).reshape(x.shape)
+
+ def dequantized_weight(self, dtype: Any) -> Any:
+ w = self._stored_weight(torch.float32 if self.rot_group else dtype)
+ if not self.rot_group:
+ return w
+ g = self.rot_group
+ h = self.rot_h.to(torch.float32)
+ return (
+ (w.reshape(self.out_features, -1, g) @ h).reshape(self.out_features, -1).to(dtype)
+ )
+
@property
def weight(self) -> Any:
"""The dense weight, rebuilt on each read. Code that reads a wrapped Linear's weight directly
(PEFT's DoRA forward reads ``base_layer.weight``) keeps working; nothing holds it."""
return self.dequantized_weight(self.compute_dtype)
+ def _int_mm_ok(self, rows: int) -> bool:
+ return (
+ rows >= _INT_MM_MIN_ROWS
+ and self.in_features % 8 == 0
+ and self.out_features % 8 == 0
+ )
+
+ def _int_mm_rows(self, x2: Any) -> Any:
+ # Must stay fast in eager (Windows ROCm has no Triton): upcasts ride mixed-dtype ops, no .float() copies.
+ x_scale = torch.linalg.vector_norm(
+ x2, ord = float("inf"), dim = 1, keepdim = True, dtype = torch.float32
+ )
+ x_scale = x_scale.clamp_(min = 1e-12).div_(127.0)
+ xq = torch.mul(x2, x_scale.reciprocal()).round_().clamp_(-127, 127).to(torch.int8)
+ acc = torch._int_mm(xq, self.weight_q.t())
+ w_scale = self.weight_scale.view(torch.float32)
+ out = torch.mul(acc, x_scale)
+ if self.bias is not None:
+ out = torch.addcmul(self.bias.float(), out, w_scale)
+ else:
+ out = out.mul_(w_scale)
+ return out
+
+ def _forward_int_mm(self, x: Any) -> Any:
+ x2 = x.reshape(-1, self.in_features)
+ rows = x2.shape[0]
+ # Eager materialises fp32 [rows, features] temporaries (inductor fuses them): cap them per chunk.
+ chunk = _INT_MM_CHUNK_ELEMS // max(self.in_features, self.out_features)
+ if torch.compiler.is_compiling() or rows <= chunk or chunk < 2 * _INT_MM_MIN_ROWS:
+ out = self._int_mm_rows(x2).to(x.dtype)
+ else:
+ out = torch.cat(
+ [
+ self._int_mm_rows(part).to(x.dtype)
+ for part in torch.tensor_split(x2, -(-rows // chunk))
+ ]
+ )
+ return out.reshape(*x.shape[:-1], self.out_features)
+
def forward(self, x: Any) -> Any:
- return F.linear(x, self.dequantized_weight(x.dtype), self.bias)
+ if self.rot_group:
+ x = self._rotate(x)
+ if (
+ self.act_int8
+ and x.is_cuda
+ and self._int_mm_ok(x.numel() // max(1, self.in_features))
+ ):
+ return self._forward_int_mm(x)
+ return F.linear(x, self._stored_weight(x.dtype), self.bias)
def extra_repr(self) -> str:
return (
f"in_features={self.in_features}, out_features={self.out_features}, "
- f"scheme={self.scheme}, bias={self.bias is not None}"
+ f"scheme={self.scheme}, act_int8={self.act_int8}, rot_group={self.rot_group}, bias={self.bias is not None}"
)
return NativeWeightOnlyLinear
@@ -94,25 +201,78 @@ def apply_native_weight_quant(
scheme: str,
*,
filter_fn: Callable[[Any, str], bool],
+ act_int8: bool,
logger: Any = None,
) -> int:
"""Swap every Linear ``filter_fn`` keeps for a native weight-only twin; returns layers swapped. One layer
at a time; a failure part-way leaves a partial conversion the caller must refuse."""
cls = native_linear_class()
+ act_int8 = scheme == NATIVE_INT8 and bool(act_int8)
+ rot_group = int8_rotation_group() if act_int8 else 0
+ # Names only: holding the old modules would keep every bf16 weight alive until the pass ends.
names = [name for name, mod in transformer.named_modules() if name and filter_fn(mod, name)]
for name in names:
parent_name, _, leaf = name.rpartition(".")
parent = transformer.get_submodule(parent_name) if parent_name else transformer
- setattr(parent, leaf, cls(getattr(parent, leaf), scheme))
+ setattr(
+ parent, leaf, cls(getattr(parent, leaf), scheme, act_int8 = act_int8, rot_group = rot_group)
+ )
if logger is not None:
logger.info(
- "diffusion.transformer_quant: %s weight-only (torchao-free) on %d linears",
+ "diffusion.transformer_quant: %s %s (torchao-free) on %d linears",
scheme,
+ ("W8A8 torch._int_mm" + (f" ConvRot g{rot_group}" if rot_group else ""))
+ if act_int8
+ else "weight-only",
len(names),
)
return len(names)
+def _first_native_linear(module: Any) -> Any:
+ try:
+ return next((sub for sub in module.modules() if is_native_linear(sub)), None)
+ except Exception: # noqa: BLE001 -- a probe must never raise
+ return None
+
+
+def _rotation_group(module: Any, layer: Any) -> int:
+ # Every layer: rotation skips widths the group does not divide, so the first layer alone can read unrotated.
+ try:
+ return max(
+ int(getattr(sub, "rot_group", 0) or 0)
+ for sub in module.modules()
+ if is_native_linear(sub)
+ )
+ except Exception: # noqa: BLE001 -- a probe must never raise
+ return int(getattr(layer, "rot_group", 0) or 0)
+
+
+def native_quant_signature(module: Any) -> Optional[str]:
+ layer = _first_native_linear(module)
+ if layer is None:
+ return None
+ if not getattr(layer, "act_int8", False):
+ return f"{layer.scheme}-wo"
+ rot = _rotation_group(module, layer)
+ return f"{layer.scheme}-w8a8" + (f"-rot{rot}" if rot else "")
+
+
+def native_quant_reason(module: Any, scheme: str) -> str:
+ layer = _first_native_linear(module)
+ if layer is not None and getattr(layer, "act_int8", False):
+ rot = _rotation_group(module, layer)
+ return (
+ f"W8A8: {scheme} weights and activations through torch._int_mm"
+ + (f" with a g{rot} ConvRot rotation" if rot else "")
+ + " (torchao-free; plain buffers, so the offload hooks can move it)"
+ )
+ return (
+ f"weight-only: {scheme} weights, bf16 compute "
+ "(torchao-free, a memory saving rather than a speed-up)"
+ )
+
+
def native_weight_error(linear: Any, scheme: str) -> Optional[float]:
"""Relative Frobenius error of one Linear's native reconstruction; for tests and probes."""
try:
diff --git a/studio/backend/core/inference/diffusion_transformer_quant.py b/studio/backend/core/inference/diffusion_transformer_quant.py
index 9fa351f9ab6..ee0a65f5511 100644
--- a/studio/backend/core/inference/diffusion_transformer_quant.py
+++ b/studio/backend/core/inference/diffusion_transformer_quant.py
@@ -30,7 +30,7 @@ from typing import Any, Optional
# stdlib-only module (no torch), so this stays inside the "imported lazily" promise above.
from core._torchao_stub import is_stubbed, torch_is_rocm
-from .diffusion_native_quant import is_native_linear
+from .diffusion_native_quant import int8_act_disabled, int8_act_requested, is_native_linear
from .diffusion_torchao_patches import install_torchao_int_mm_patch
# Also runs in the spawned smoke-probe child, which imports this module and nothing else of the backend.
@@ -686,21 +686,47 @@ def native_quant_host(target: Any) -> bool:
return False
+NATIVE_OFFLOAD_SCHEMES = (TQ_INT8,)
+
+
+def native_offload_host(target: Any) -> bool:
+ if getattr(target, "device", None) != "cuda":
+ return False
+ if torch_is_rocm() or is_stubbed("torchao"):
+ return False
+ try:
+ import torch
+ return getattr(target, "dtype", None) is torch.bfloat16
+ except Exception:
+ return False
+
+
def native_quant_scheme(
target: Any,
requested: Optional[str],
family: Optional[str] = None,
+ *,
+ offload: bool = False,
) -> Optional[str]:
- """The weight-only scheme an EXPLICIT ``requested`` runs as on a ``native_quant_host``, or None.
- Not in ``select_transformer_quant_scheme``: that also feeds hosted torchao prequant planners."""
+ """Not ``select_transformer_quant_scheme``: that also feeds hosted prequant planners (torchao checkpoints)."""
scheme = normalize_transformer_quant(requested)
- if scheme not in NATIVE_QUANT_SCHEMES or not native_quant_host(target):
+ if scheme not in NATIVE_QUANT_SCHEMES:
+ return None
+ if not native_quant_host(target) and not (
+ offload and scheme in NATIVE_OFFLOAD_SCHEMES and native_offload_host(target)
+ ):
return None
if _family_denied(family, scheme):
return None
return scheme
+def native_int8_act(target: Any) -> bool:
+ if native_quant_host(target):
+ return int8_act_requested()
+ return not int8_act_disabled()
+
+
def select_transformer_quant_scheme(
target: Any,
requested: Optional[str],
@@ -1404,16 +1430,25 @@ def quantize_transformer(
min_features: int = DEFAULT_MIN_LINEAR_FEATURES,
fast_accum: Optional[bool] = None,
logger: Any = None,
+ offload: bool = False,
+ act_int8: Optional[bool] = None,
) -> Optional[str]:
"""Quantise ``pipe.transformer``'s FLOP-heavy linears in place with the arch-chosen scheme.
Returns the scheme engaged, or None when disabled / unsupported / failed (caller loads GGUF).
Best-effort: never raises for an unsupported environment (failure leaves it dense).
``fast_accum`` (fp8 only) overrides the per-GPU-class accumulate choice: None auto-detects,
- True/False force it. On a ``native_quant_host`` an explicit int8 / fp8 runs weight-only without torchao."""
- native = native_quant_scheme(target, mode, family = family)
+ True/False force it."""
+ native = native_quant_scheme(target, mode, family = family, offload = offload)
if native is not None:
+ if act_int8 is None:
+ act_int8 = native_int8_act(target)
return _quantize_native(
- pipe, native, family = family, min_features = min_features, logger = logger
+ pipe,
+ native,
+ family = family,
+ min_features = min_features,
+ act_int8 = bool(act_int8) and native == TQ_INT8,
+ logger = logger,
)
scheme = select_transformer_quant_scheme(target, mode, family = family)
if scheme is None:
@@ -1456,9 +1491,8 @@ def quantize_transformer(
def _quantize_native(
- pipe: Any, scheme: str, *, family: Optional[str], min_features: int, logger: Any
+ pipe: Any, scheme: str, *, family: Optional[str], min_features: int, act_int8: bool, logger: Any
) -> Optional[str]:
- """The torchao-free branch of ``quantize_transformer``: the same layer filter, weight-only."""
transformer = getattr(pipe, "transformer", None)
if transformer is None:
return None
@@ -1471,7 +1505,9 @@ def _quantize_native(
exclude_name_tokens = exclude_tokens_for_scheme(scheme, family) + ("lora_",),
require_bf16 = True,
)
- if not apply_native_weight_quant(transformer, scheme, filter_fn = filter_fn, logger = logger):
+ if not apply_native_weight_quant(
+ transformer, scheme, filter_fn = filter_fn, act_int8 = act_int8, logger = logger
+ ):
return None
try:
transformer._unsloth_runtime_quant = scheme
diff --git a/studio/backend/core/inference/video.py b/studio/backend/core/inference/video.py
index 3af575b20ad..615d3a58c84 100644
--- a/studio/backend/core/inference/video.py
+++ b/studio/backend/core/inference/video.py
@@ -110,11 +110,15 @@ from .diffusion_auto_policy import (
)
from .diffusion_transformer_quant import (
TQ_AUTO,
+ TQ_INT8,
dense_transformer_supported,
dense_transformer_unsupported_reason,
explain_unusable_scheme,
+ native_int8_act,
+ native_offload_host,
native_quant_host,
native_quant_scheme,
+ NATIVE_OFFLOAD_SCHEMES,
NATIVE_QUANT_SCHEMES,
normalize_transformer_quant,
quantize_transformer,
@@ -124,6 +128,7 @@ from .diffusion_transformer_quant import (
transformer_is_quantised,
)
from .diffusion import _memory_request_forces_offload
+from .diffusion_native_quant import native_quant_reason
from .diffusion_batched import is_oom_error
from .diffusion_precision import (
effective_te_quant,
@@ -306,6 +311,13 @@ def _assert_video_precision_for_target(
reason = explain_unusable_scheme(getattr(fam, "name", None), pinned)
elif not dense_transformer_supported(target):
reason = dense_transformer_unsupported_reason(target)
+ elif forces_offload and pinned in NATIVE_OFFLOAD_SCHEMES and native_offload_host(target):
+ # forces_offload is already False for the modular workflow.
+ if (
+ native_quant_scheme(target, pinned, family = getattr(fam, "name", None), offload = True)
+ is None
+ ):
+ reason = explain_unusable_scheme(getattr(fam, "name", None), pinned)
elif forces_offload:
# balanced and low_vram name their offload policy without measuring anything, and offload hooks move modules
# with Module.to(), which torchao tensors do not survive. load_pipeline therefore skips the dense build and
@@ -1687,6 +1699,7 @@ class VideoBackend:
model_kind = resolve_video_model_kind(gguf_filename, model_kind),
transformer_quant = transformer_quant,
text_encoder_quant = text_encoder_quant,
+ memory_mode = memory_mode,
gpu_ordinal = gpu_ordinal,
)
# Resolved out here so the companion claim is published in the SAME locked section as _loading. begin_load
@@ -4309,11 +4322,22 @@ class VideoBackend:
# Why the quant did not engage, in the caller's terms; threaded into `resolved`.
transformer_quant_decline: Optional[str] = None
transformer_quant_decline_status = RESOLVED_FELL_BACK
+ video_offload = plan.offload_policy != "none"
native_scheme = (
- native_quant_scheme(target, transformer_quant_pinned, family = fam.name)
+ native_quant_scheme(
+ target, transformer_quant_pinned, family = fam.name, offload = video_offload
+ )
if kind == "pipeline"
else None
)
+ native_kwargs = (
+ {
+ "offload": video_offload,
+ "act_int8": native_scheme == TQ_INT8 and native_int8_act(target),
+ }
+ if native_scheme is not None
+ else {}
+ )
# Auto quantises a video DiT only to keep it resident: with bf16 resident, int8 fails the default LPIPS bar.
if (
kind == "pipeline"
@@ -4351,6 +4375,7 @@ class VideoBackend:
and normalize_transformer_quant(transformer_quant) is not None
and dense_transformer_supported(target)
and plan.offload_policy != "none"
+ and native_scheme is None
):
# Offload hooks move modules with Module.to(), which torchao quantized tensors reject. Skip quant
# (dense-under-offload beats a crash); forceable via a resident mode.
@@ -4397,6 +4422,7 @@ class VideoBackend:
mode = transformer_quant,
family = fam.name,
logger = logger,
+ **native_kwargs,
)
if scheme is not None:
engaged.append(scheme)
@@ -4657,8 +4683,9 @@ class VideoBackend:
transformer_quant_engaged or "off",
# Honest framing: the shipped torchao schemes cut load time and resident memory ~2x, but
# per-step GEMMs are at best bf16 parity.
- f"weight-only: {transformer_quant_engaged} weights, bf16 compute "
- "(torchao-free, a memory saving rather than a speed-up)"
+ native_quant_reason(
+ getattr(pipe, "transformer", None), transformer_quant_engaged
+ )
if transformer_quant_engaged is not None and native_scheme is not None
else "DiT(s) quantised (halves resident weights; hosted checkpoints cut "
"load time; per-step speed is roughly bf16 parity)"
diff --git a/studio/backend/tests/test_diffusion_backend.py b/studio/backend/tests/test_diffusion_backend.py
index 9806f601433..a5ad1fee84c 100644
--- a/studio/backend/tests/test_diffusion_backend.py
+++ b/studio/backend/tests/test_diffusion_backend.py
@@ -12475,6 +12475,178 @@ def test_an_explicit_scheme_on_nvidia_keeps_the_torchao_path_and_its_wording(
backend.unload()
+def _stub_nvidia_offload_host(
+ backend,
+ monkeypatch,
+ *,
+ engages = "int8",
+):
+ from core.inference import diffusion as dmod
+ from core.inference import diffusion_transformer_quant as tq
+
+ calls = _stub_pipeline_dense_quant(backend, monkeypatch, engages = engages)
+ monkeypatch.setattr(
+ dmod, "select_transformer_quant_scheme", lambda target, mode, family = None: mode
+ )
+ monkeypatch.setattr(tq, "native_quant_host", lambda target: False)
+ monkeypatch.setattr(tq, "native_offload_host", lambda target: True)
+ monkeypatch.setattr(dmod, "native_offload_host", lambda target: True)
+ monkeypatch.delenv("UNSLOTH_NATIVE_INT8_ACT", raising = False)
+ reasons: list = []
+
+ def _reason(module, scheme):
+ reasons.append(scheme)
+ return f"W8A8: {scheme} (stub)"
+
+ monkeypatch.setattr(dmod, "native_quant_reason", _reason)
+ return calls, reasons
+
+
+ "memory",
+ [{"memory_mode": "balanced"}, {"memory_mode": "low_vram"}, {"cpu_offload": True}],
+)
+def test_an_explicit_int8_under_offload_on_nvidia_runs_native_w8a8(
+ fake_runtime, tmp_path, monkeypatch, memory
+):
+ backend = DiffusionBackend()
+ calls, reasons = _stub_nvidia_offload_host(backend, monkeypatch)
+ status = backend.load_pipeline(
+ "Qwen/Qwen-Image-2512",
+ model_kind = "pipeline",
+ transformer_quant = "int8",
+ _base_local_dir = str(tmp_path),
+ **memory,
+ )
+ assert len(calls) == 1
+ assert calls[0]["offload"] is True and calls[0]["act_int8"] is True
+ assert status["offload_policy"] != "none"
+ assert status["transformer_quant"] == "int8"
+ resolved = status["resolved"]["transformer_quant"]
+ assert resolved["value"] == "int8" and resolved["status"] == "applied"
+ assert resolved["reason"].startswith("W8A8") and reasons == ["int8"]
+ backend.unload()
+
+
+def test_the_act_kill_switch_keeps_nvidia_offload_native_but_weight_only(
+ fake_runtime, tmp_path, monkeypatch
+):
+ backend = DiffusionBackend()
+ calls, _ = _stub_nvidia_offload_host(backend, monkeypatch)
+ monkeypatch.setenv("UNSLOTH_NATIVE_INT8_ACT", "0")
+ status = backend.load_pipeline(
+ "Qwen/Qwen-Image-2512",
+ model_kind = "pipeline",
+ transformer_quant = "int8",
+ memory_mode = "balanced",
+ _base_local_dir = str(tmp_path),
+ )
+ assert calls[0]["offload"] is True and calls[0]["act_int8"] is False
+ assert status["transformer_quant"] == "int8"
+ backend.unload()
+
+
[email protected]("fallback", ["0", "1"])
+def test_an_explicit_fp8_under_offload_on_nvidia_is_still_declined(
+ fake_runtime, tmp_path, monkeypatch, fallback
+):
+ monkeypatch.setenv("UNSLOTH_DIFFUSION_ALLOW_PRECISION_FALLBACK", fallback)
+ backend = DiffusionBackend()
+ calls, reasons = _stub_nvidia_offload_host(backend, monkeypatch, engages = "fp8")
+ kwargs = dict(
+ model_kind = "pipeline",
+ transformer_quant = "fp8",
+ memory_mode = "balanced",
+ _base_local_dir = str(tmp_path),
+ )
+ if fallback == "0":
+ with pytest.raises(RuntimeError, match = "Module.to"):
+ backend.load_pipeline("Qwen/Qwen-Image-2512", **kwargs)
+ else:
+ status = backend.load_pipeline("Qwen/Qwen-Image-2512", **kwargs)
+ assert status["transformer_quant"] is None
+ assert "Module.to()" in status["resolved"]["transformer_quant"]["reason"]
+ backend.unload()
+ assert calls == [] and reasons == []
+
+
+def test_an_explicit_int8_resident_on_nvidia_keeps_torchao(fake_runtime, tmp_path, monkeypatch):
+ backend = DiffusionBackend()
+ calls, reasons = _stub_nvidia_offload_host(backend, monkeypatch)
+ status = backend.load_pipeline(
+ "Qwen/Qwen-Image-2512",
+ model_kind = "pipeline",
+ transformer_quant = "int8",
+ memory_mode = "fast",
+ _base_local_dir = str(tmp_path),
+ )
+ assert status["offload_policy"] == "none"
+ assert len(calls) == 1 and "offload" not in calls[0] and "act_int8" not in calls[0]
+ assert status["transformer_quant"] == "int8"
+ assert reasons == []
+ assert "W8A8" not in status["resolved"]["transformer_quant"]["reason"]
+ backend.unload()
+
+
+def test_auto_under_offload_on_nvidia_never_goes_native(fake_runtime, tmp_path, monkeypatch):
+ backend = DiffusionBackend()
+ calls, reasons = _stub_nvidia_offload_host(backend, monkeypatch)
+ status = backend.load_pipeline(
+ "Qwen/Qwen-Image-2512",
+ model_kind = "pipeline",
+ memory_mode = "balanced",
+ _base_local_dir = str(tmp_path),
+ )
+ assert calls == [] and reasons == []
+ assert status["transformer_quant"] is None
+ backend.unload()
+
+
+def test_a_resident_nvidia_int8_that_cannot_compile_still_declines(
+ fake_runtime, tmp_path, monkeypatch, allow_precision_fallback
+):
+ from core.inference import diffusion as dmod
+
+ backend = DiffusionBackend()
+ calls, _ = _stub_nvidia_offload_host(backend, monkeypatch)
+ monkeypatch.setattr(
+ dmod, "_pipeline_quant_uncompilable_reason", lambda *a, **k: "no compile here (stub)"
+ )
+ status = backend.load_pipeline(
+ "Qwen/Qwen-Image-2512",
+ model_kind = "pipeline",
+ transformer_quant = "int8",
+ memory_mode = "fast",
+ _base_local_dir = str(tmp_path),
+ )
+ assert calls == []
+ assert status["transformer_quant"] is None
+ assert status["resolved"]["transformer_quant"]["reason"] == "no compile here (stub)"
+ backend.unload()
+
+
+def test_an_offloaded_nvidia_int8_ignores_the_compile_requirement(
+ fake_runtime, tmp_path, monkeypatch
+):
+ from core.inference import diffusion as dmod
+
+ backend = DiffusionBackend()
+ calls, _ = _stub_nvidia_offload_host(backend, monkeypatch)
+ monkeypatch.setattr(
+ dmod, "_pipeline_quant_uncompilable_reason", lambda *a, **k: "no compile here (stub)"
+ )
+ status = backend.load_pipeline(
+ "Qwen/Qwen-Image-2512",
+ model_kind = "pipeline",
+ transformer_quant = "int8",
+ memory_mode = "low_vram",
+ _base_local_dir = str(tmp_path),
+ )
+ assert len(calls) == 1 and calls[0]["offload"] is True
+ assert status["transformer_quant"] == "int8"
+ backend.unload()
+
+
def _record_step_cache(
monkeypatch,
*,
diff --git a/studio/backend/tests/test_diffusion_native_quant.py b/studio/backend/tests/test_diffusion_native_quant.py
index 013f768715e..28c52d83595 100644
--- a/studio/backend/tests/test_diffusion_native_quant.py
+++ b/studio/backend/tests/test_diffusion_native_quant.py
@@ -1,7 +1,7 @@
# SPDX-License-Identifier: AGPL-3.0-only
# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
-"""Torchao-free weight-only int8 / fp8; ROCm simulated by patching ``torch_is_rocm`` / ``is_stubbed``."""
+"""Torchao-free int8 / fp8 (AMD, ROCm stub) and NVIDIA int8 W8A8 under offload; ROCm is monkeypatched."""
from __future__ import annotations
@@ -104,10 +104,87 @@ def test_non_cuda_or_non_bf16_is_never_native_even_on_rocm(rocm, target):
@pytest.mark.parametrize("device", ["cuda", "cpu", "mps", "xpu"])
-def test_nvidia_and_other_hosts_are_never_native(nvidia, device):
+def test_nvidia_and_other_hosts_are_never_native_without_offload(nvidia, device):
assert not tq.native_quant_host(_target(device = device))
for scheme in ("int8", "fp8", "nvfp4", "mxfp8", "auto"):
assert tq.native_quant_scheme(_target(device = device), scheme) is None
+ assert tq.native_quant_scheme(_target(device = device), scheme, offload = False) is None
+
+
+_NVIDIA_TABLE = [
+ ("int8", True, "int8"),
+ ("INT8", True, "int8"),
+ ("int8", False, None),
+ ("fp8", True, None),
+ ("fp8", False, None),
+ ("nvfp4", True, None),
+ ("mxfp8", True, None),
+ ("auto", True, None),
+ (None, True, None),
+ ("none", True, None),
+]
+
+
[email protected]("scheme,offload,expect", _NVIDIA_TABLE)
+def test_nvidia_takes_native_int8_only_under_offload(nvidia, scheme, offload, expect):
+ assert tq.native_offload_host(_target())
+ assert tq.native_quant_scheme(_target(), scheme, offload = offload) == expect
+
+
+ "target",
+ [
+ _target(device = "cpu"),
+ _target(device = "mps"),
+ _target(device = "xpu"),
+ _target(dtype = torch.float16),
+ _target(dtype = torch.float32),
+ ],
+)
+def test_the_offload_route_needs_a_bf16_cuda_target(nvidia, target):
+ assert not tq.native_offload_host(target)
+ assert tq.native_quant_scheme(target, "int8", offload = True) is None
+
+
+def test_amd_hosts_are_not_the_nvidia_offload_host(rocm):
+ assert not tq.native_offload_host(_target())
+ for scheme in ("int8", "fp8"):
+ assert tq.native_quant_scheme(_target(), scheme, offload = True) == scheme
+ assert tq.native_quant_scheme(_target(), scheme, offload = False) == scheme
+
+
+def test_windows_stub_is_not_the_nvidia_offload_host(win_stub):
+ assert not tq.native_offload_host(_target())
+ assert tq.native_quant_scheme(_target(), "fp8", offload = True) == "fp8"
+
+
+def test_nvidia_offload_route_respects_the_family_deny_list(nvidia, monkeypatch):
+ monkeypatch.setitem(tq._FAMILY_SCHEME_DENY, "toy-family", frozenset({tq.TQ_INT8}))
+ assert tq.native_quant_scheme(_target(), "int8", family = "toy-family", offload = True) is None
+ assert (
+ tq.native_quant_scheme(_target(), "int8", family = "qwen-image-2.1", offload = True) == "int8"
+ )
+
+
+ "env,expect_amd,expect_nvidia",
+ [
+ (None, False, True),
+ ("1", True, True),
+ ("0", False, False),
+ ("yes", False, True),
+ ],
+)
+def test_w8a8_default_differs_by_route(monkeypatch, env, expect_amd, expect_nvidia):
+ if env is None:
+ monkeypatch.delenv(nq.NATIVE_INT8_ACT_ENV, raising = False)
+ else:
+ monkeypatch.setenv(nq.NATIVE_INT8_ACT_ENV, env)
+ monkeypatch.setattr(tq, "is_stubbed", lambda name: False)
+ monkeypatch.setattr(tq, "torch_is_rocm", lambda: True)
+ assert tq.native_int8_act(_target()) is expect_amd
+ monkeypatch.setattr(tq, "torch_is_rocm", lambda: False)
+ assert tq.native_int8_act(_target()) is expect_nvidia
def test_family_deny_list_still_applies(rocm, monkeypatch):
@@ -184,6 +261,130 @@ def test_a_module_wide_dtype_cast_leaves_the_stored_weights_alone(scheme):
assert torch.allclose(layer(x.to(torch.float16)).float(), before.float(), rtol = 1e-2, atol = 1e-2)
+def _int_mm_on_cpu():
+ try:
+ torch._int_mm(torch.ones(32, 8, dtype = torch.int8), torch.ones(8, 8, dtype = torch.int8))
+ return True
+ except (RuntimeError, AttributeError, NotImplementedError):
+ return False
+
+
+def test_int8_activation_path_is_off_by_default(monkeypatch):
+ monkeypatch.delenv(nq.NATIVE_INT8_ACT_ENV, raising = False)
+ assert not nq.int8_act_requested()
+ monkeypatch.setenv(nq.NATIVE_INT8_ACT_ENV, "0")
+ assert not nq.int8_act_requested()
+ monkeypatch.setenv(nq.NATIVE_INT8_ACT_ENV, "1")
+ assert nq.int8_act_requested()
+
+
+def test_int8_activation_flag_is_ignored_for_fp8():
+ lin = torch.nn.Linear(64, 64).to(torch.bfloat16)
+ assert nq.native_linear_class()(lin, "int8", act_int8 = True).act_int8
+ assert not nq.native_linear_class()(lin, "fp8", act_int8 = True).act_int8
+ assert not nq.native_linear_class()(lin, "int8").act_int8
+
+
+def test_int_mm_shape_gate():
+ cls = nq.native_linear_class()
+ layer = cls(torch.nn.Linear(64, 64).to(torch.bfloat16), "int8", act_int8 = True)
+ assert not layer._int_mm_ok(16) and layer._int_mm_ok(17)
+ odd = cls(torch.nn.Linear(64, 12).to(torch.bfloat16), "int8", act_int8 = True)
+ assert not odd._int_mm_ok(4096)
+
+
+def test_cpu_inputs_keep_the_weight_only_path(monkeypatch):
+ torch.manual_seed(0)
+ lin = torch.nn.Linear(64, 64).to(torch.bfloat16)
+ layer = nq.native_linear_class()(lin, "int8", act_int8 = True)
+ monkeypatch.setattr(
+ layer, "_forward_int_mm", lambda x: pytest.fail("int_mm used on a CPU input")
+ )
+ x = torch.randn(32, 64, dtype = torch.bfloat16)
+ assert torch.equal(
+ layer(x), torch.nn.functional.linear(x, layer.dequantized_weight(x.dtype), lin.bias)
+ )
+
+
[email protected](not _int_mm_on_cpu(), reason = "torch._int_mm has no CPU kernel in this build")
+def test_int_mm_matches_the_dense_layer():
+ torch.manual_seed(0)
+ lin = torch.nn.Linear(256, 128).to(torch.bfloat16)
+ layer = nq.native_linear_class()(lin, "int8", act_int8 = True)
+ x = torch.randn(2, 24, 256, dtype = torch.bfloat16)
+ ref = lin(x).float()
+ got = layer._forward_int_mm(x)
+ assert got.shape == ref.shape and got.dtype == torch.bfloat16
+ assert ((got.float() - ref).norm() / ref.norm()).item() < 0.03
+
+
+def test_rotation_only_applies_to_w8a8_and_divisible_inputs():
+ cls = nq.native_linear_class()
+ lin = torch.nn.Linear(512, 64).to(torch.bfloat16)
+ assert cls(lin, "int8", act_int8 = True, rot_group = 256).rot_group == 256
+ assert cls(lin, "int8", rot_group = 256).rot_group == 0
+ assert cls(lin, "fp8", act_int8 = True, rot_group = 256).rot_group == 0
+ odd = torch.nn.Linear(320, 64).to(torch.bfloat16)
+ assert cls(odd, "int8", act_int8 = True, rot_group = 256).rot_group == 0
+
+
+def test_rotated_layer_keeps_the_model_basis():
+ torch.manual_seed(0)
+ lin = torch.nn.Linear(512, 128).to(torch.bfloat16)
+ layer = nq.native_linear_class()(lin, "int8", act_int8 = True, rot_group = 256)
+ assert "rot_h" not in layer.state_dict()
+ w = lin.weight.float()
+ assert ((layer.weight.float() - w).norm() / w.norm()).item() < 0.01
+ x = torch.randn(4, 512, dtype = torch.bfloat16)
+ ref = lin(x).float()
+ got = layer(x).float()
+ assert ((got - ref).norm() / ref.norm()).item() < 0.02
+
+
[email protected](not _int_mm_on_cpu(), reason = "torch._int_mm has no CPU kernel in this build")
+def test_rotation_cuts_the_int8_activation_error_on_outlier_channels():
+ torch.manual_seed(0)
+ lin = torch.nn.Linear(512, 256).to(torch.bfloat16)
+ cls = nq.native_linear_class()
+ plain = cls(lin, "int8", act_int8 = True)
+ rot = cls(lin, "int8", act_int8 = True, rot_group = 256)
+ x = torch.randn(64, 512, dtype = torch.bfloat16)
+ x[:, :5] *= 30.0
+ ref = lin(x).float()
+ err = lambda y: ((y.float() - ref).norm() / ref.norm()).item() # noqa: E731
+ assert err(rot._forward_int_mm(rot._rotate(x))) < 0.5 * err(plain._forward_int_mm(x))
+
+
+def test_rotation_env_switch(monkeypatch):
+ monkeypatch.delenv(nq.NATIVE_INT8_ROT_ENV, raising = False)
+ assert nq.int8_rotation_group() == 256
+ monkeypatch.setenv(nq.NATIVE_INT8_ROT_ENV, "0")
+ assert nq.int8_rotation_group() == 0
+
+
+def test_apply_native_weight_quant_takes_act_from_the_caller_not_the_env(monkeypatch):
+ for env, act, scheme, expect in (
+ ("", True, "int8", True),
+ ("1", False, "int8", False),
+ ("0", True, "int8", True),
+ ("1", True, "fp8", False),
+ ):
+ monkeypatch.setenv(nq.NATIVE_INT8_ACT_ENV, env)
+ model = _toy()
+ n = nq.apply_native_weight_quant(
+ model, scheme, filter_fn = lambda m, name: isinstance(m, torch.nn.Linear), act_int8 = act
+ )
+ assert n > 0
+ flags = {m.act_int8 for m in model.modules() if hasattr(m, "act_int8")}
+ assert flags == {expect}
+
+
+def test_int8_act_kill_switch_reads_only_zero(monkeypatch):
+ for value, disabled in (("", False), ("1", False), ("0", True), (" 0 ", True), ("off", False)):
+ monkeypatch.setenv(nq.NATIVE_INT8_ACT_ENV, value)
+ assert nq.int8_act_disabled() is disabled
+
+
def _block_torchao(monkeypatch):
# A None entry makes any `import torchao.quantization` raise, proving the branch never reaches it.
monkeypatch.setitem(sys.modules, "torchao", None)
@@ -251,11 +452,11 @@ def test_quantize_transformer_failure_is_reported_as_dirty(rocm, monkeypatch):
real = nq.native_linear_class()
calls = []
- def _boom(linear, scheme):
+ def _boom(linear, scheme, **kw):
calls.append(1)
if len(calls) == 3:
raise RuntimeError("out of memory")
- return real(linear, scheme)
+ return real(linear, scheme, **kw)
monkeypatch.setattr(nq, "native_linear_class", lambda: _boom)
warned = []
@@ -302,6 +503,149 @@ def test_quantize_transformer_on_nvidia_never_reaches_the_native_branch(nvidia,
assert len(seen) == 3
+def _logger(logs):
+ return types.SimpleNamespace(
+ info = lambda *a: logs.append(a[0] % a[1:]), warning = lambda *a: pytest.fail(f"warned: {a}")
+ )
+
+
+def test_quantize_transformer_on_nvidia_under_offload_runs_w8a8_convrot(nvidia, monkeypatch):
+ _block_torchao(monkeypatch)
+ monkeypatch.delenv(nq.NATIVE_INT8_ACT_ENV, raising = False)
+ monkeypatch.delenv(nq.NATIVE_INT8_ROT_ENV, raising = False)
+ monkeypatch.setattr(
+ tq, "select_transformer_quant_scheme", lambda *a, **k: pytest.fail("torchao selector asked")
+ )
+ torch.manual_seed(0)
+ model = torch.nn.Sequential(
+ torch.nn.Linear(512, 512), torch.nn.GELU(), torch.nn.Linear(512, 512)
+ )
+ model = model.to(torch.bfloat16)
+ x = torch.randn(4, 512, dtype = torch.bfloat16)
+ with torch.no_grad():
+ ref = model(x).float()
+ logs = []
+ got = tq.quantize_transformer(
+ types.SimpleNamespace(transformer = model),
+ _target(),
+ mode = "int8",
+ min_features = 64,
+ logger = _logger(logs),
+ offload = True,
+ )
+ assert got == "int8"
+ layers = [m for m in model.modules() if nq.is_native_linear(m)]
+ assert len(layers) == 2
+ assert all(m.act_int8 and m.rot_group == 256 for m in layers)
+ assert any("W8A8 torch._int_mm ConvRot g256" in line for line in logs)
+ assert nq.native_quant_signature(model) == "int8-w8a8-rot256"
+ assert nq.native_quant_reason(model, "int8").startswith("W8A8")
+ with torch.no_grad():
+ out = model(x).float()
+ assert ((out - ref).norm() / ref.norm()).item() < 0.05
+
+
+def test_quantize_transformer_honours_the_kill_switches_on_nvidia(nvidia, monkeypatch):
+ _block_torchao(monkeypatch)
+ monkeypatch.setenv(nq.NATIVE_INT8_ACT_ENV, "0")
+ model = _toy()
+ assert (
+ tq.quantize_transformer(
+ types.SimpleNamespace(transformer = model),
+ _target(),
+ mode = "int8",
+ min_features = 64,
+ offload = True,
+ )
+ == "int8"
+ )
+ assert {m.act_int8 for m in model.modules() if nq.is_native_linear(m)} == {False}
+ assert nq.native_quant_signature(model) == "int8-wo"
+ assert nq.native_quant_reason(model, "int8").startswith("weight-only")
+
+ monkeypatch.delenv(nq.NATIVE_INT8_ACT_ENV)
+ monkeypatch.setenv(nq.NATIVE_INT8_ROT_ENV, "0")
+ model = _toy()
+ tq.quantize_transformer(
+ types.SimpleNamespace(transformer = model),
+ _target(),
+ mode = "int8",
+ min_features = 64,
+ offload = True,
+ )
+ layers = [m for m in model.modules() if nq.is_native_linear(m)]
+ assert layers and all(m.act_int8 and m.rot_group == 0 for m in layers)
+ assert nq.native_quant_signature(model) == "int8-w8a8"
+
+
+def test_an_explicit_act_argument_overrides_the_host_default(nvidia, monkeypatch):
+ _block_torchao(monkeypatch)
+ monkeypatch.delenv(nq.NATIVE_INT8_ACT_ENV, raising = False)
+ model = _toy()
+ tq.quantize_transformer(
+ types.SimpleNamespace(transformer = model),
+ _target(),
+ mode = "int8",
+ min_features = 64,
+ offload = True,
+ act_int8 = False,
+ )
+ assert {m.act_int8 for m in model.modules() if nq.is_native_linear(m)} == {False}
+
+
[email protected]("scheme", ["fp8", "auto", "nvfp4"])
+def test_other_schemes_under_offload_on_nvidia_stay_on_torchao(nvidia, monkeypatch, scheme):
+ monkeypatch.setattr(
+ tq, "_quantize_native", lambda *a, **k: pytest.fail("native branch for a non-int8 scheme")
+ )
+ seen = []
+ monkeypatch.setattr(
+ tq, "select_transformer_quant_scheme", lambda *a, **k: seen.append(a) or None
+ )
+ assert (
+ tq.quantize_transformer(
+ types.SimpleNamespace(transformer = _toy()), _target(), mode = scheme, offload = True
+ )
+ is None
+ )
+ assert len(seen) == 1
+
+
+def test_amd_quantize_keeps_its_opt_in_w8a8(rocm, monkeypatch):
+ _block_torchao(monkeypatch)
+ for env, expect in (("", False), ("1", True)):
+ monkeypatch.setenv(nq.NATIVE_INT8_ACT_ENV, env)
+ model = _toy()
+ tq.quantize_transformer(
+ types.SimpleNamespace(transformer = model), _target(), mode = "int8", min_features = 64
+ )
+ assert {m.act_int8 for m in model.modules() if nq.is_native_linear(m)} == {expect}
+
+
+def test_compile_cache_keys_native_apart_from_torchao():
+ from core.inference.diffusion_compile_cache import model_fingerprint
+
+ dense = _toy()
+ native = _toy()
+ nq.apply_native_weight_quant(
+ native, "int8", filter_fn = lambda m, name: isinstance(m, torch.nn.Linear), act_int8 = True
+ )
+
+ def fp(module, quant):
+ return model_fingerprint(
+ family = "toy",
+ transformer = module,
+ dtype = torch.bfloat16,
+ quant = quant,
+ attention_backend = None,
+ compile_kwargs = {},
+ )["quant"]
+
+ assert fp(dense, "int8") == "int8"
+ assert fp(dense, None) == "none"
+ assert fp(native, "int8").startswith("int8-w8a8")
+
+
@pytest.mark.parametrize("scheme", ["int8", "fp8"])
def test_reading_weight_gives_the_dense_weight_without_storing_it(scheme):
"""PEFT DoRA reads ``base_layer.weight``: dense, rebuilt per read, not in the state dict."""
@@ -369,6 +713,7 @@ def _image_gate(
*,
model_kind = "pipeline",
memory_mode = None,
+ cpu_offload = False,
):
import core.inference.diffusion as d
monkeypatch.setattr(d, "effective_te_quant", lambda *a, **k: None)
@@ -380,7 +725,7 @@ def _image_gate(
pinned = pinned,
te_mode = None,
memory_mode = memory_mode,
- cpu_offload = False,
+ cpu_offload = cpu_offload,
)
@@ -460,9 +805,144 @@ def test_video_gate_refuses_native_on_gguf_loads(rocm, monkeypatch):
_video_gate(monkeypatch, _video_family("wan2.2-ti2v-5b"), "int8", model_kind = "gguf")
+def _torchao_gate_answers(
+ monkeypatch,
+ module,
+ *,
+ supported = True,
+):
+ monkeypatch.setattr(module, "dense_transformer_supported", lambda target: supported)
+ monkeypatch.setattr(module, "select_transformer_quant_scheme", lambda target, mode, **k: mode)
+ if hasattr(module, "_pipeline_quant_uncompilable_reason"):
+ monkeypatch.setattr(module, "_pipeline_quant_uncompilable_reason", lambda *a, **k: None)
+
+
+ "memory",
+ [{"memory_mode": "balanced"}, {"memory_mode": "low_vram"}, {"cpu_offload": True}],
+)
+def test_image_gate_admits_int8_under_offload_on_nvidia(nvidia, monkeypatch, memory):
+ import core.inference.diffusion as d
+ _torchao_gate_answers(monkeypatch, d)
+ _image_gate(monkeypatch, _image_family("qwen-image-2.1"), "int8", **memory)
+
+
[email protected]("pinned", ["fp8", "nvfp4", "mxfp8"])
[email protected]("memory", [{"memory_mode": "balanced"}, {"cpu_offload": True}])
+def test_image_gate_still_refuses_other_schemes_under_offload_on_nvidia(
+ nvidia, monkeypatch, pinned, memory
+):
+ import core.inference.diffusion as d
+ _torchao_gate_answers(monkeypatch, d)
+ with pytest.raises(RuntimeError, match = "cannot be moved by the offload hooks"):
+ _image_gate(monkeypatch, _image_family("qwen-image-2.1"), pinned, **memory)
+
+
[email protected]("kind", ["gguf", "single_file"])
+def test_image_gate_keeps_non_pipeline_int8_under_offload_refused_on_nvidia(
+ nvidia, monkeypatch, kind
+):
+ import core.inference.diffusion as d
+ _torchao_gate_answers(monkeypatch, d)
+ with pytest.raises(RuntimeError, match = "transformer_quant"):
+ _image_gate(
+ monkeypatch,
+ _image_family("qwen-image-2.1"),
+ "int8",
+ model_kind = kind,
+ memory_mode = "balanced",
+ )
+
+
+def test_image_gate_refuses_a_denied_family_under_offload_on_nvidia(nvidia, monkeypatch):
+ import core.inference.diffusion as d
+
+ _torchao_gate_answers(monkeypatch, d)
+ monkeypatch.setitem(tq._FAMILY_SCHEME_DENY, "qwen-image-2.1", frozenset({tq.TQ_INT8}))
+ with pytest.raises(RuntimeError, match = "ruled out"):
+ _image_gate(monkeypatch, _image_family("qwen-image-2.1"), "int8", memory_mode = "balanced")
+
+
[email protected]("memory_mode", [None, "fast", "auto"])
+def test_image_gate_on_nvidia_without_a_named_offload_never_asks_native(
+ nvidia, monkeypatch, memory_mode
+):
+ import core.inference.diffusion as d
+
+ _torchao_gate_answers(monkeypatch, d)
+ monkeypatch.setattr(
+ d, "native_quant_scheme", lambda *a, **k: pytest.fail("native on a resident request")
+ )
+ for pinned in ("int8", "fp8"):
+ _image_gate(monkeypatch, _image_family("qwen-image-2.1"), pinned, memory_mode = memory_mode)
+
+
+def test_image_gate_offload_route_needs_no_compile(nvidia, monkeypatch):
+ import core.inference.diffusion as d
+
+ _torchao_gate_answers(monkeypatch, d)
+ monkeypatch.setattr(
+ d, "_pipeline_quant_uncompilable_reason", lambda *a, **k: "no compile (stub)"
+ )
+ _image_gate(monkeypatch, _image_family("qwen-image-2.1"), "int8", memory_mode = "low_vram")
+ with pytest.raises(RuntimeError, match = "no compile"):
+ _image_gate(monkeypatch, _image_family("qwen-image-2.1"), "int8", memory_mode = "fast")
+
+
[email protected]("memory_mode", ["balanced", "low_vram"])
+def test_video_gate_admits_int8_under_offload_on_nvidia(nvidia, monkeypatch, memory_mode):
+ import core.inference.video as v
+ _torchao_gate_answers(monkeypatch, v)
+ _video_gate(monkeypatch, _video_family("wan2.2-ti2v-5b"), "int8", memory_mode = memory_mode)
+
+
[email protected]("pinned", ["fp8", "nvfp4", "mxfp8"])
+def test_video_gate_still_refuses_other_schemes_under_offload_on_nvidia(
+ nvidia, monkeypatch, pinned
+):
+ import core.inference.video as v
+ _torchao_gate_answers(monkeypatch, v)
+ with pytest.raises(RuntimeError, match = "cannot be moved by the offload hooks"):
+ _video_gate(monkeypatch, _video_family("wan2.2-ti2v-5b"), pinned, memory_mode = "balanced")
+
+
+def test_video_gate_refuses_a_denied_family_under_offload_on_nvidia(nvidia, monkeypatch):
+ import core.inference.video as v
+
+ _torchao_gate_answers(monkeypatch, v)
+ monkeypatch.setitem(tq._FAMILY_SCHEME_DENY, "wan2.2-ti2v-5b", frozenset({tq.TQ_INT8}))
+ with pytest.raises(RuntimeError, match = "ruled out"):
+ _video_gate(monkeypatch, _video_family("wan2.2-ti2v-5b"), "int8", memory_mode = "balanced")
+
+
+def test_video_gate_keeps_gguf_int8_under_offload_refused_on_nvidia(nvidia, monkeypatch):
+ import core.inference.video as v
+ _torchao_gate_answers(monkeypatch, v)
+ with pytest.raises(RuntimeError, match = "transformer_quant"):
+ _video_gate(
+ monkeypatch,
+ _video_family("wan2.2-ti2v-5b"),
+ "int8",
+ model_kind = "gguf",
+ memory_mode = "balanced",
+ )
+
+
[email protected]("memory_mode", [None, "fast"])
+def test_video_gate_on_nvidia_resident_keeps_the_torchao_answer(nvidia, monkeypatch, memory_mode):
+ import core.inference.video as v
+
+ _torchao_gate_answers(monkeypatch, v)
+ monkeypatch.setattr(
+ v, "native_quant_scheme", lambda *a, **k: pytest.fail("native on a resident request")
+ )
+ for pinned in ("int8", "fp8"):
+ _video_gate(monkeypatch, _video_family("wan2.2-ti2v-5b"), pinned, memory_mode = memory_mode)
+
+
def _native_block(features = 64):
lin = torch.nn.Linear(features, features, dtype = torch.bfloat16)
- return nq.native_linear_class()(lin, "int8")
+ return nq.native_linear_class()(lin, "int8", act_int8 = True, rot_group = 0)
def test_stream_group_offload_puts_native_weight_buffers_back_on_the_host():
@@ -534,6 +1014,49 @@ def test_native_int8_under_real_stream_group_offload_leaves_nothing_resident():
assert {b.device.type for b in model.buffers()} == {"cpu"}
+class _GateSeen(Exception):
+ pass
+
+
+ "memory", [{"memory_mode": "balanced"}, {"memory_mode": "low_vram"}, {"cpu_offload": True}]
+)
+def test_image_begin_load_rechecks_with_the_offload_request(monkeypatch, memory):
+ import core.inference.diffusion as d
+
+ seen = {}
+
+ def gate(self, fam, **kwargs):
+ seen.update(kwargs)
+ raise _GateSeen
+
+ monkeypatch.setattr(d.DiffusionBackend, "validate_load_request", lambda self, *a, **k: None)
+ monkeypatch.setattr(d.DiffusionBackend, "assert_precision_available", gate)
+ with pytest.raises(_GateSeen):
+ d.DiffusionBackend().begin_load(
+ "some/repo", transformer_quant = "int8", speed_mode = "eager", **memory
+ )
+ assert seen.get("memory_mode") == memory.get("memory_mode")
+ assert seen.get("cpu_offload", False) == memory.get("cpu_offload", False)
+
+
[email protected]("memory_mode", ["balanced", "low_vram"])
+def test_video_begin_load_rechecks_with_the_memory_request(monkeypatch, memory_mode):
+ import core.inference.video as v
+
+ seen = {}
+
+ def gate(fam, **kwargs):
+ seen.update(kwargs)
+ raise _GateSeen
+
+ monkeypatch.setattr(v.VideoBackend, "validate_load_request", lambda self, *a, **k: None)
+ monkeypatch.setattr(v, "assert_video_precision_available", gate)
+ with pytest.raises(_GateSeen):
+ v.VideoBackend().begin_load("some/repo", transformer_quant = "int8", memory_mode = memory_mode)
+ assert seen.get("memory_mode") == memory_mode
+
+
def test_qkv_fusion_skips_a_native_quantised_denoiser():
from core.inference import diffusion_speed
@@ -552,3 +1075,40 @@ def test_qkv_fusion_skips_a_native_quantised_denoiser():
assert diffusion_speed._fuse_qkv(native, None) is False and not calls
dense = types.SimpleNamespace(transformer = _Dit(native = False))
assert diffusion_speed._fuse_qkv(dense, None) is True and len(calls) == 1
+
+
[email protected](not torch.cuda.is_available(), reason = "the W8A8 path runs on CUDA inputs only")
+def test_eager_w8a8_chunks_long_inputs_without_changing_the_result(monkeypatch):
+ torch.manual_seed(0)
+ layer = nq.native_linear_class()(
+ torch.nn.Linear(64, 32).to(torch.bfloat16), "int8", act_int8 = True
+ ).cuda()
+ x = torch.randn(300, 64, dtype = torch.bfloat16, device = "cuda")
+ whole = layer(x)
+ seen = []
+ rows = layer._int_mm_rows
+ monkeypatch.setattr(
+ layer, "_int_mm_rows", lambda part: seen.append(part.shape[0]) or rows(part)
+ )
+ monkeypatch.setattr(nq, "_INT_MM_CHUNK_ELEMS", 64 * 40)
+ assert torch.equal(layer(x), whole)
+ assert len(seen) > 1 and sum(seen) == 300 and min(seen) >= 17
+
+
+def test_signature_sees_rotation_past_an_unrotated_first_layer():
+ cls = nq.native_linear_class()
+ model = torch.nn.Sequential(
+ cls(torch.nn.Linear(320, 256).to(torch.bfloat16), "int8", act_int8 = True, rot_group = 256),
+ cls(torch.nn.Linear(256, 64).to(torch.bfloat16), "int8", act_int8 = True, rot_group = 256),
+ )
+ assert model[0].rot_group == 0 and model[1].rot_group == 256
+ assert nq.native_quant_signature(model) == "int8-w8a8-rot256"
+ assert "g256 ConvRot" in nq.native_quant_reason(model, "int8")
+
+
+def test_a_rotated_layer_passes_an_empty_input_through():
+ layer = nq.native_linear_class()(
+ torch.nn.Linear(256, 64).to(torch.bfloat16), "int8", act_int8 = True, rot_group = 256
+ )
+ out = layer(torch.zeros(2, 0, 256, dtype = torch.bfloat16))
+ assert out.shape == (2, 0, 64)
diff --git a/studio/backend/tests/test_video_backend.py b/studio/backend/tests/test_video_backend.py
index 91d819879ba..7642e0db82d 100644
--- a/studio/backend/tests/test_video_backend.py
+++ b/studio/backend/tests/test_video_backend.py
@@ -2211,12 +2211,13 @@ def test_wan_a14b_dense_quant_applies_to_both_dits(fake_runtime, monkeypatch):
def test_dense_quant_skipped_under_offload(fake_runtime, monkeypatch):
# Offload hooks move modules with Module.to(), which torchao tensors reject, so any offload
- # policy must SKIP quant. With the legacy escape hatch set the load still succeeds dense and the
- # record explains why; the strict default refuses instead (see the test below).
+ # policy must SKIP a torchao quant (escape hatch: dense plus a record).
import core.inference.video as video_mod
+ from core.inference import diffusion_transformer_quant as tq
monkeypatch.setenv("UNSLOTH_DIFFUSION_ALLOW_PRECISION_FALLBACK", "1")
monkeypatch.setattr(video_mod, "dense_transformer_supported", lambda target: True)
+ monkeypatch.setattr(tq, "native_offload_host", lambda target: True)
quantised = []
def _fake_quant(
@@ -2241,7 +2242,7 @@ def test_dense_quant_skipped_under_offload(fake_runtime, monkeypatch):
status = backend.load_pipeline(
"Wan-AI/Wan2.2-T2V-A14B-Diffusers",
model_kind = "pipeline",
- transformer_quant = "int8",
+ transformer_quant = "fp8",
)
_assert_placement_follows_the_target(placements, video_mod)
assert status["offload_policy"] == "model"
@@ -2250,7 +2251,7 @@ def test_dense_quant_skipped_under_offload(fake_runtime, monkeypatch):
resolved = status["resolved"]["transformer_quant"]
assert "moves the DiT" in resolved["reason"]
# BOTH sides of the story survive: the ask, the outcome, and that they disagree.
- assert resolved["requested"] == "int8"
+ assert resolved["requested"] == "fp8"
assert resolved["value"] == "off"
assert resolved["status"] == "fell_back"
@@ -2281,13 +2282,13 @@ def test_the_video_load_places_on_the_selected_card_not_a_bare_device(fake_runti
def test_explicit_dense_quant_refuses_under_offload(fake_runtime, monkeypatch):
- # Strict default (no escape hatch): an explicit int8 the offload plan cannot honor stops the
- # load, rather than denoising at bf16 while the Precision dropdown still reads INT8.
import dataclasses
import core.inference.video as video_mod
+ from core.inference import diffusion_transformer_quant as tq
monkeypatch.setattr(video_mod, "dense_transformer_supported", lambda target: True)
+ monkeypatch.setattr(tq, "native_offload_host", lambda target: True)
monkeypatch.setattr(
video_mod,
"quantize_transformer",
@@ -2299,9 +2300,9 @@ def test_explicit_dense_quant_refuses_under_offload(fake_runtime, monkeypatch):
backend.load_pipeline(
"Wan-AI/Wan2.2-T2V-A14B-Diffusers",
model_kind = "pipeline",
- transformer_quant = "int8",
+ transformer_quant = "fp8",
)
- assert "transformer_quant='int8' could not be used" in str(excinfo.value)
+ assert "transformer_quant='fp8' could not be used" in str(excinfo.value)
assert "resident memory mode" in str(excinfo.value)
assert backend.status()["loaded"] is False
@@ -10059,6 +10060,97 @@ def test_video_status_reports_cuda_graph_off_once_every_armed_step_ran_eager(fak
backend.unload()
+def _stub_nvidia_video_offload(monkeypatch, *, offload = True):
+ import dataclasses
+
+ import core.inference.video as video_mod
+ from core.inference import diffusion_transformer_quant as tq
+
+ monkeypatch.setattr(video_mod, "dense_transformer_supported", lambda target: True)
+ monkeypatch.setattr(tq, "native_quant_host", lambda target: False)
+ monkeypatch.setattr(tq, "native_offload_host", lambda target: True)
+ monkeypatch.setattr(
+ video_mod, "select_transformer_quant_scheme", lambda target, mode, family = None: mode
+ )
+ monkeypatch.delenv("UNSLOTH_NATIVE_INT8_ACT", raising = False)
+ if offload:
+ real_plan = video_mod.plan_diffusion_memory
+ monkeypatch.setattr(
+ video_mod,
+ "plan_diffusion_memory",
+ lambda **kwargs: dataclasses.replace(real_plan(**kwargs), offload_policy = "model"),
+ )
+ _stub_apply_memory_plan(monkeypatch, video_mod, policy = "model" if offload else "none")
+ calls: list = []
+
+ def _quantize(
+ view,
+ target,
+ *,
+ mode,
+ family = None,
+ **kw,
+ ):
+ calls.append({"view": view, "mode": mode, **kw})
+ return mode
+
+ monkeypatch.setattr(video_mod, "quantize_transformer", _quantize)
+ monkeypatch.setattr(
+ video_mod, "native_quant_reason", lambda module, scheme: f"W8A8: {scheme} (stub)"
+ )
+ return calls
+
+
+def test_an_explicit_int8_under_offload_on_nvidia_runs_native_on_both_experts(
+ fake_runtime, monkeypatch
+):
+ calls = _stub_nvidia_video_offload(monkeypatch)
+ backend = VideoBackend()
+ status = backend.load_pipeline(
+ "Wan-AI/Wan2.2-T2V-A14B-Diffusers", model_kind = "pipeline", transformer_quant = "int8"
+ )
+ assert status["offload_policy"] == "model"
+ assert len(calls) == 2
+ assert all(c["offload"] is True and c["act_int8"] is True for c in calls)
+ assert status["transformer_quant"] == "int8"
+ resolved = status["resolved"]["transformer_quant"]
+ assert resolved["status"] == "applied" and resolved["reason"].startswith("W8A8")
+ backend.unload()
+
+
+def test_the_video_act_kill_switch_keeps_offload_native_weight_only(fake_runtime, monkeypatch):
+ calls = _stub_nvidia_video_offload(monkeypatch)
+ monkeypatch.setenv("UNSLOTH_NATIVE_INT8_ACT", "0")
+ backend = VideoBackend()
+ status = backend.load_pipeline(
+ "Wan-AI/Wan2.2-TI2V-5B-Diffusers", model_kind = "pipeline", transformer_quant = "int8"
+ )
+ assert calls and all(c["offload"] is True and c["act_int8"] is False for c in calls)
+ assert status["transformer_quant"] == "int8"
+ backend.unload()
+
+
+def test_a_resident_video_int8_on_nvidia_keeps_torchao(fake_runtime, monkeypatch):
+ calls = _stub_nvidia_video_offload(monkeypatch, offload = False)
+ backend = VideoBackend()
+ status = backend.load_pipeline(
+ "Wan-AI/Wan2.2-TI2V-5B-Diffusers", model_kind = "pipeline", transformer_quant = "int8"
+ )
+ assert status["offload_policy"] == "none"
+ assert calls and all("offload" not in c and "act_int8" not in c for c in calls)
+ assert "W8A8" not in status["resolved"]["transformer_quant"]["reason"]
+ backend.unload()
+
+
+def test_video_auto_under_offload_on_nvidia_is_still_skipped(fake_runtime, monkeypatch):
+ calls = _stub_nvidia_video_offload(monkeypatch)
+ backend = VideoBackend()
+ status = backend.load_pipeline("Wan-AI/Wan2.2-TI2V-5B-Diffusers", model_kind = "pipeline")
+ assert calls == []
+ assert status["transformer_quant"] is None
+ backend.unload()
+
+
def test_video_auto_quant_on_a_host_without_dense_quant_reports_as_before(fake_runtime):
status = VideoBackend().load_pipeline(
"Wan-AI/Wan2.2-TI2V-5B-Diffusers", model_kind = "pipeline", speed_mode = "default"