Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

Unsloth Studio (AMD): allow INT8 / FP8 image and video precision without torchao (#11631)

Daniel Hanf0b73b2fed4f
diff --git a/studio/backend/core/inference/diffusion.py b/studio/backend/core/inference/diffusion.py
index 28cecbb4b9f..da2e82adc86 100644
--- a/studio/backend/core/inference/diffusion.py
+++ b/studio/backend/core/inference/diffusion.py
@@ -222,6 +222,9 @@ from .diffusion_transformer_quant import (
denoiser_modules,
explain_unusable_scheme,
mark_source_precision,
+ native_quant_host,
+ native_quant_scheme,
+ NATIVE_QUANT_SCHEMES,
normalize_transformer_quant,
quantize_transformer,
select_transformer_quant_scheme,
@@ -1818,6 +1821,13 @@ class DiffusionBackend:
f"{getattr(fam, 'denoiser_attr', 'unet')}, not a transformer, and the dense torchao schemes "
"do not cover it"
)
+ elif (
+ model_kind == "pipeline"
+ and pinned in NATIVE_QUANT_SCHEMES
+ and native_quant_host(target)
+ ):
+ if native_quant_scheme(target, pinned, family = getattr(fam, "name", None)) is None:
+ reason = explain_unusable_scheme(getattr(fam, "name", None), pinned)
elif _memory_request_forces_offload(memory_mode, cpu_offload):
# Not a measurement: balanced and low_vram name their policy outright, and the legacy flag forces
# model offload. Offload hooks move modules with Module.to(), which torchao tensors do not survive, so
@@ -4517,19 +4527,30 @@ class DiffusionBackend:
# visible, and into the refusal so it is actionable.
transformer_quant_decline: Optional[str] = None
transformer_quant_decline_status = RESOLVED_FELL_BACK
+ native_scheme = (
+ native_quant_scheme(
+ target, transformer_quant_pinned, family = getattr(fam, "name", None)
+ )
+ if kind == "pipeline"
+ else None
+ )
if transformer_quant_pinned is not None and not dense_quant_supported_kind(kind):
transformer_quant_decline = dense_quant_unsupported_kind_reason(kind)
transformer_quant_decline_status = RESOLVED_UNSUPPORTED
- elif transformer_quant_pinned is not None and not dense_transformer_supported(
- target
+ elif (
+ transformer_quant_pinned is not None
+ and native_scheme is None
+ and not dense_transformer_supported(target)
):
# Ask the helper rather than repeating its fallback: on ROCm and on the Windows
# torchao stub it knows a truer reason, and an AMD owner reading "needs a CUDA
# GPU" while holding a working GPU learns nothing about why it declined.
transformer_quant_decline = dense_transformer_unsupported_reason(target)
transformer_quant_decline_status = RESOLVED_UNSUPPORTED
- elif transformer_quant_pinned is not None and (
- select_transformer_quant_scheme(
+ elif (
+ transformer_quant_pinned is not None
+ and native_scheme is None
+ and select_transformer_quant_scheme(
target, transformer_quant_pinned, family = getattr(fam, "name", None)
)
is None
@@ -5399,7 +5420,7 @@ class DiffusionBackend:
and kind == "pipeline"
and transformer_quant_engaged is None
and normalize_transformer_quant(transformer_quant) is not None
- and dense_transformer_supported(target)
+ and (dense_transformer_supported(target) or native_scheme is not None)
):
# from_pretrained widens a raw fp8/int8 checkpoint to bf16, erasing the one thing the blocker
# below reads, so recover it from the shard header (Ideogram's loader stamps its own).
@@ -5409,9 +5430,10 @@ class DiffusionBackend:
if source_precision is not None:
for _attr, denoiser in denoiser_modules(pipe):
mark_source_precision(denoiser, source_precision)
- pipeline_quant_blocker = pipeline_quant_uncompilable or dense_quant_blocker(
- pipe
- )
+ # Weight-only runs bf16 arithmetic, so it has no compile to require.
+ pipeline_quant_blocker = (
+ None if native_scheme is not None else pipeline_quant_uncompilable
+ ) or dense_quant_blocker(pipe)
if pipeline_quant_blocker is not None:
logger.info(
"diffusion.transformer_quant: skipped (%s)", pipeline_quant_blocker
@@ -5422,7 +5444,7 @@ class DiffusionBackend:
# Re-plan against the quantised steady size. The build peak remains bf16.
bf16_plan = plan
if plan.offload_policy != OFFLOAD_NONE:
- preview_scheme = select_transformer_quant_scheme(
+ preview_scheme = native_scheme or select_transformer_quant_scheme(
target, transformer_quant, family = getattr(fam, "name", None)
)
# This in-memory rewrite needs no cache-space or hosted-checkpoint checks.
@@ -5458,7 +5480,7 @@ class DiffusionBackend:
plan.offload_policy,
)
plan = replanned
- if plan.offload_policy != OFFLOAD_NONE:
+ if plan.offload_policy != OFFLOAD_NONE and native_scheme is None:
logger.info(
"diffusion.transformer_quant: skipped (the memory plan picked '%s' "
"offload, which moves the transformer via Module.to())",
@@ -5567,8 +5589,12 @@ class DiffusionBackend:
# Effective speed: GGUF defaults to `default` (~2.2x, below the quant noise floor); dense stays
# bit-identical `off`.
effective_speed = resolve_speed_mode(speed_mode, is_gguf = kind == "gguf")
- # A torchao-quantized dense transformer must be compiled (eager is ~30x slower, losing to GGUF)
- if transformer_quant_engaged is not None and effective_speed == SPEED_OFF:
+ # A torchao-quantized dense transformer must be compiled (eager is ~30x slower, losing to GGUF).
+ if (
+ transformer_quant_engaged is not None
+ and native_scheme is None
+ and effective_speed == SPEED_OFF
+ ):
logger.info(
"diffusion.transformer_quant: forcing speed_mode=default "
"(quantized transformer must be compiled; eager is ~30x slower)"
@@ -5719,7 +5745,11 @@ class DiffusionBackend:
logger = logger,
)
self._raise_if_load_cancelled(_load_token)
- if transformer_quant_engaged is not None and not speed_applied.get("compiled"):
+ if (
+ transformer_quant_engaged is not None
+ and native_scheme is None
+ and not speed_applied.get("compiled")
+ ):
# Compile could not engage: the quantized transformer runs eager, far slower than the GGUF it
# replaced
logger.warning(
@@ -5823,6 +5853,7 @@ class DiffusionBackend:
"deferred" if speed_deferred else effective_speed,
"quantized transformer requires compile"
if transformer_quant_engaged is not None
+ and native_scheme is None
and normalize_speed_mode(speed_mode) in (None, SPEED_OFF)
else "auto: exact eager for the first two images; "
"the compile profile engages on the 3rd"
@@ -5848,6 +5879,9 @@ class DiffusionBackend:
)
)
if transformer_quant_engaged is None
+ else f"weight-only: {transformer_quant_engaged} weights, bf16 compute "
+ "(torchao-free, a memory saving rather than a speed-up)"
+ if native_scheme is not None
else f"seeded from the hosted checkpoint "
f"{transformer_quant_artifact.split(':', 1)[1]}"
if transformer_quant_artifact is not None
diff --git a/studio/backend/core/inference/diffusion_memory.py b/studio/backend/core/inference/diffusion_memory.py
index 4c0e2a806ac..598f60139e5 100644
--- a/studio/backend/core/inference/diffusion_memory.py
+++ b/studio/backend/core/inference/diffusion_memory.py
@@ -1389,6 +1389,41 @@ def _pin_vision_embedding_device(module: Any) -> int:
return patched
+def install_group_offload_buffer_restore() -> bool:
+ """diffusers stream group offload restores only parameters, leaving buffers (native int8 weights) on the GPU."""
+ try:
+ from diffusers.hooks import group_offloading as go
+ except Exception: # noqa: BLE001 - no group offload in this diffusers
+ return False
+ group_cls = getattr(go, "ModuleGroup", None)
+ original = getattr(group_cls, "_offload_to_memory", None)
+ if original is None or getattr(original, "_unsloth_buffer_restore", False):
+ return False
+
+ @functools.wraps(original)
+ def _offload_to_memory(self, *args: Any, **kwargs: Any) -> Any:
+ out = original(self, *args, **kwargs)
+ cpu_copies = getattr(self, "cpu_param_dict", None)
+ if getattr(self, "stream", None) is None or not cpu_copies:
+ return out
+ restore_torchao = getattr(go, "_restore_torchao_tensor", None)
+ is_torchao = getattr(go, "_is_torchao_tensor", None)
+ for group_module in getattr(self, "modules", None) or ():
+ for buffer in group_module.buffers():
+ cpu = cpu_copies.get(buffer)
+ if cpu is None:
+ continue
+ if callable(is_torchao) and callable(restore_torchao) and is_torchao(buffer):
+ restore_torchao(buffer, cpu)
+ else:
+ buffer.data = cpu
+ return out
+
+ _offload_to_memory._unsloth_buffer_restore = True
+ group_cls._offload_to_memory = _offload_to_memory
+ return True
+
+
def _apply_group_offload(
pipe: Any,
device: str,
@@ -1412,6 +1447,8 @@ def _apply_group_offload(
import torch
from diffusers.hooks import apply_group_offloading
+ install_group_offload_buffer_restore()
+
# A dual-DiT pipeline (Ideogram 4) carries a second denoiser as large as the first, so stream every DiT and keep
# only smaller companions resident.
streamed: dict[str, Any] = {"transformer": transformer}
@@ -1704,6 +1741,8 @@ def _apply_streaming_offload(pipe: Any, device: str, logger: Any) -> None:
import torch
from diffusers.hooks import apply_group_offloading
+ install_group_offload_buffer_restore()
+
components = getattr(pipe, "components", {})
if not isinstance(components, dict):
raise RuntimeError("pipeline does not expose its components")
diff --git a/studio/backend/core/inference/diffusion_native_quant.py b/studio/backend/core/inference/diffusion_native_quant.py
new file mode 100644
index 00000000000..1b78d0e4903
--- /dev/null
+++ b/studio/backend/core/inference/diffusion_native_quant.py
@@ -0,0 +1,126 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
+
+"""Torchao-free weight-only int8 / fp8 for the diffusion transformer (ROCm, Windows torchao stub).
+
+Plain buffers, unlike torchao tensors, survive the offload hooks' ``Module.to()``; torch imported lazily.
+"""
+
+from __future__ import annotations
+
+from functools import lru_cache
+from typing import Any, Callable, Optional
+
+NATIVE_INT8 = "int8"
+NATIVE_FP8 = "fp8"
+_QMAX = {NATIVE_INT8: 127.0, NATIVE_FP8: 448.0}
+
+
+@lru_cache(maxsize = 1)
+def native_linear_class():
+ """``NativeWeightOnlyLinear``, defined on first use so this module imports torch-free."""
+ import torch
+ from torch import nn
+ from torch.nn import functional as F
+
+ class NativeWeightOnlyLinear(nn.Module):
+ """A Linear whose weight is stored as int8 / fp8 with a per-output-row scale."""
+
+ def __init__(self, linear: Any, scheme: str):
+ super().__init__()
+ if scheme not in _QMAX:
+ raise ValueError(f"unsupported native scheme {scheme!r}")
+ self.in_features = int(linear.in_features)
+ self.out_features = int(linear.out_features)
+ self.scheme = scheme
+ self.compute_dtype = linear.weight.dtype
+ weight = linear.weight.detach()
+ with torch.no_grad():
+ w = weight.float()
+ scale = w.abs().amax(dim = 1, keepdim = True).clamp(min = 1e-12) / _QMAX[scheme]
+ if scheme == NATIVE_INT8:
+ wq = (w / scale).round_().clamp_(-127, 127).to(torch.int8)
+ else:
+ wq = (w / scale).to(torch.float8_e4m3fn)
+ del w
+ # Integer views: a module-wide ``.to(dtype)`` casts floating buffers, which would widen fp8 or round the scales.
+ self.register_buffer("weight_q", wq.view(torch.uint8) if scheme == NATIVE_FP8 else wq)
+ self.register_buffer(
+ "weight_scale", scale.squeeze(1).to(torch.float32).contiguous().view(torch.int32)
+ )
+ self.bias = linear.bias
+
+ def dequantized_weight(self, dtype: Any) -> Any:
+ wq = (
+ self.weight_q.view(torch.float8_e4m3fn)
+ if self.scheme == NATIVE_FP8
+ else self.weight_q
+ )
+ scale = self.weight_scale.view(torch.float32)
+ return (wq.to(torch.float32) * scale[:, None]).to(dtype)
+
+ @property
+ def weight(self) -> Any:
+ """The dense weight, rebuilt on each read. Code that reads a wrapped Linear's weight directly
+ (PEFT's DoRA forward reads ``base_layer.weight``) keeps working; nothing holds it."""
+ return self.dequantized_weight(self.compute_dtype)
+
+ def forward(self, x: Any) -> Any:
+ return F.linear(x, self.dequantized_weight(x.dtype), self.bias)
+
+ def extra_repr(self) -> str:
+ return (
+ f"in_features={self.in_features}, out_features={self.out_features}, "
+ f"scheme={self.scheme}, bias={self.bias is not None}"
+ )
+
+ return NativeWeightOnlyLinear
+
+
+def is_native_linear(module: Any) -> bool:
+ return type(module).__name__ == "NativeWeightOnlyLinear" and hasattr(module, "weight_q")
+
+
+def is_native_quantised(module: Any) -> bool:
+ """Whether any layer of ``module`` has been swapped for a native weight-only Linear."""
+ try:
+ return any(is_native_linear(sub) for sub in module.modules())
+ except Exception: # noqa: BLE001 -- a probe must never replace the failure it describes
+ return False
+
+
+def apply_native_weight_quant(
+ transformer: Any,
+ scheme: str,
+ *,
+ filter_fn: Callable[[Any, str], bool],
+ logger: Any = None,
+) -> int:
+ """Swap every Linear ``filter_fn`` keeps for a native weight-only twin; returns layers swapped. One layer
+ at a time; a failure part-way leaves a partial conversion the caller must refuse."""
+ cls = native_linear_class()
+ names = [name for name, mod in transformer.named_modules() if name and filter_fn(mod, name)]
+ for name in names:
+ parent_name, _, leaf = name.rpartition(".")
+ parent = transformer.get_submodule(parent_name) if parent_name else transformer
+ setattr(parent, leaf, cls(getattr(parent, leaf), scheme))
+ if logger is not None:
+ logger.info(
+ "diffusion.transformer_quant: %s weight-only (torchao-free) on %d linears",
+ scheme,
+ len(names),
+ )
+ return len(names)
+
+
+def native_weight_error(linear: Any, scheme: str) -> Optional[float]:
+ """Relative Frobenius error of one Linear's native reconstruction; for tests and probes."""
+ try:
+ import torch
+
+ layer = native_linear_class()(linear, scheme)
+ ref = linear.weight.detach().float()
+ rec = layer.dequantized_weight(torch.float32)
+ return float((rec - ref).norm() / ref.norm().clamp(min = 1e-12))
+ except Exception: # noqa: BLE001
+ return None
diff --git a/studio/backend/core/inference/diffusion_speed.py b/studio/backend/core/inference/diffusion_speed.py
index 97becc5c7ec..3af547b117a 100644
--- a/studio/backend/core/inference/diffusion_speed.py
+++ b/studio/backend/core/inference/diffusion_speed.py
@@ -925,6 +925,15 @@ def _enable_fp16_accumulation(
def _fuse_qkv(pipe: Any, logger: Any) -> bool:
+ from .diffusion_native_quant import is_native_quantised
+
+ # Fusing reads each projection's .weight, which a native layer dequantises: a dense to_qkv beside the int8 buffers.
+ if any(is_native_quantised(dit) for dit in _denoiser_dits(pipe)):
+ if logger is not None:
+ logger.info(
+ "diffusion.speed: fuse_qkv skipped (native quantised projections stay unfused)"
+ )
+ return False
# Prefer the pipe-level fuse (covers every component); else fuse each denoiser DiT so a dual-DiT family fuses BOTH
# experts.
fn = getattr(pipe, "fuse_qkv_projections", None)
diff --git a/studio/backend/core/inference/diffusion_transformer_quant.py b/studio/backend/core/inference/diffusion_transformer_quant.py
index a68ebeb6fe1..9fa351f9ab6 100644
--- a/studio/backend/core/inference/diffusion_transformer_quant.py
+++ b/studio/backend/core/inference/diffusion_transformer_quant.py
@@ -30,6 +30,7 @@ from typing import Any, Optional
# stdlib-only module (no torch), so this stays inside the "imported lazily" promise above.
from core._torchao_stub import is_stubbed, torch_is_rocm
+from .diffusion_native_quant import is_native_linear
from .diffusion_torchao_patches import install_torchao_int_mm_patch
# Also runs in the spawned smoke-probe child, which imports this module and nothing else of the backend.
@@ -508,6 +509,25 @@ _NARROW_STORED_DTYPES: dict[str, str] = {
}
+def _cached_snapshot_dir(repo_id: str) -> Any:
+ """``repo_id``'s already-cached snapshot, Studio's live cache first (the loaders pass it as ``cache_dir``)."""
+ from pathlib import Path
+
+ from huggingface_hub import snapshot_download
+
+ try:
+ from utils.hf_cache_settings import active_hf_hub_cache
+ roots = (active_hf_hub_cache(), None)
+ except Exception: # noqa: BLE001 -- the smoke-probe child has no Studio settings
+ roots = (None,)
+ for cache_dir in roots:
+ try:
+ return Path(snapshot_download(repo_id, cache_dir = cache_dir, local_files_only = True))
+ except Exception: # noqa: BLE001 -- not cached there
+ continue
+ return None
+
+
def stored_denoiser_precision(local_dir: Optional[str]) -> Optional[str]:
"""The narrow precision a LOCAL snapshot's denoiser shards are stored at, or None.
@@ -518,8 +538,8 @@ def stored_denoiser_precision(local_dir: Optional[str]) -> Optional[str]:
is the only place left to look.
Headers only, and only under a directory we already have: no download, no tensor read, and no
- verdict without a local snapshot, which is the behaviour without this check. A hub id is not a
- directory and answers None, so the caller may pass the staged snapshot or the load's own base."""
+ verdict without a local snapshot, which is the behaviour without this check. A hub id resolves to
+ its already-cached snapshot (``local_files_only``: an offline load stages nothing), else None."""
if not local_dir:
return None
try:
@@ -528,6 +548,10 @@ def stored_denoiser_precision(local_dir: Optional[str]) -> Optional[str]:
import safetensors
root = Path(local_dir).expanduser()
+ if not root.is_dir():
+ root = _cached_snapshot_dir(str(local_dir))
+ if root is None:
+ return None
for attr in DENOISER_ATTRS:
sub = root / attr
if not sub.is_dir():
@@ -590,10 +614,13 @@ def dense_quant_blocker(pipe: Any) -> Optional[str]:
def transformer_is_quantised(module: Any) -> bool:
- """Whether any Linear weight has been replaced by a torchao tensor subclass."""
+ """Whether any Linear weight has been replaced by a torchao tensor subclass, or any Linear by a
+ torchao-free weight-only twin (``diffusion_native_quant``)."""
try:
import torch
for sub in module.modules():
+ if is_native_linear(sub):
+ return True
if not isinstance(sub, torch.nn.Linear):
continue
weight = getattr(sub, "weight", None)
@@ -640,6 +667,40 @@ def dense_transformer_unsupported_reason(target: Any) -> str:
return "this device cannot run a dense torchao quant (it needs a CUDA GPU in bf16)"
+# ``auto`` never lands here: weight-only measured no speed win on gfx1151, only memory.
+NATIVE_QUANT_SCHEMES = (TQ_INT8, TQ_FP8)
+
+
+def native_quant_host(target: Any) -> bool:
+ """Whether ``target`` is a bf16 GPU whose torchao path is closed for a platform reason: a ROCm torch
+ (the NVIDIA-shaped capability ladder misclassifies it, and gfx1151 refuses fp8 ``_scaled_mm``) or the
+ Windows-ROCm torchao stub. False on every NVIDIA, MPS, XPU and CPU host, which keep the torchao path."""
+ if getattr(target, "device", None) != "cuda":
+ return False
+ if not (torch_is_rocm() or is_stubbed("torchao")):
+ return False
+ try:
+ import torch
+ return getattr(target, "dtype", None) is torch.bfloat16
+ except Exception:
+ return False
+
+
+def native_quant_scheme(
+ target: Any,
+ requested: Optional[str],
+ family: Optional[str] = None,
+) -> Optional[str]:
+ """The weight-only scheme an EXPLICIT ``requested`` runs as on a ``native_quant_host``, or None.
+ Not in ``select_transformer_quant_scheme``: that also feeds hosted torchao prequant planners."""
+ scheme = normalize_transformer_quant(requested)
+ if scheme not in NATIVE_QUANT_SCHEMES or not native_quant_host(target):
+ return None
+ if _family_denied(family, scheme):
+ return None
+ return scheme
+
+
def select_transformer_quant_scheme(
target: Any,
requested: Optional[str],
@@ -1348,7 +1409,12 @@ def quantize_transformer(
Returns the scheme engaged, or None when disabled / unsupported / failed (caller loads GGUF).
Best-effort: never raises for an unsupported environment (failure leaves it dense).
``fast_accum`` (fp8 only) overrides the per-GPU-class accumulate choice: None auto-detects,
- True/False force it."""
+ True/False force it. On a ``native_quant_host`` an explicit int8 / fp8 runs weight-only without torchao."""
+ native = native_quant_scheme(target, mode, family = family)
+ if native is not None:
+ return _quantize_native(
+ pipe, native, family = family, min_features = min_features, logger = logger
+ )
scheme = select_transformer_quant_scheme(target, mode, family = family)
if scheme is None:
return None
@@ -1389,6 +1455,34 @@ def quantize_transformer(
return None
+def _quantize_native(
+ pipe: Any, scheme: str, *, family: Optional[str], min_features: int, logger: Any
+) -> Optional[str]:
+ """The torchao-free branch of ``quantize_transformer``: the same layer filter, weight-only."""
+ transformer = getattr(pipe, "transformer", None)
+ if transformer is None:
+ return None
+ try:
+ from .diffusion_native_quant import apply_native_weight_quant
+
+ # require_bf16 keeps the fp32 linears some DiTs deliberately hold (Wan, Hunyuan) at full precision.
+ filter_fn = make_filter_fn(
+ min_features,
+ exclude_name_tokens = exclude_tokens_for_scheme(scheme, family) + ("lora_",),
+ require_bf16 = True,
+ )
+ if not apply_native_weight_quant(transformer, scheme, filter_fn = filter_fn, logger = logger):
+ return None
+ try:
+ transformer._unsloth_runtime_quant = scheme
+ except Exception: # noqa: BLE001 - marker is best-effort
+ pass
+ return scheme
+ except Exception as exc: # noqa: BLE001 - a partial swap is reported by transformer_is_quantised
+ _warn(logger, f"{scheme} (weight-only)", exc)
+ return None
+
+
def _warn(logger: Any, what: str, exc: Exception) -> None:
if logger is not None:
logger.warning("diffusion.transformer_quant: %s failed: %s", what, exc)
diff --git a/studio/backend/core/inference/video.py b/studio/backend/core/inference/video.py
index 79045d828e1..3af575b20ad 100644
--- a/studio/backend/core/inference/video.py
+++ b/studio/backend/core/inference/video.py
@@ -113,9 +113,15 @@ from .diffusion_transformer_quant import (
dense_transformer_supported,
dense_transformer_unsupported_reason,
explain_unusable_scheme,
+ native_quant_host,
+ native_quant_scheme,
+ NATIVE_QUANT_SCHEMES,
normalize_transformer_quant,
quantize_transformer,
+ mark_source_precision,
select_transformer_quant_scheme,
+ stored_denoiser_precision,
+ transformer_is_quantised,
)
from .diffusion import _memory_request_forces_offload
from .diffusion_batched import is_oom_error
@@ -290,6 +296,14 @@ def _assert_video_precision_for_target(
f"the dense DiT quant applies to full-pipeline loads only, and this is a "
f"'{model_kind}' load, which runs the precision its checkpoint carries"
)
+ elif (
+ pinned in NATIVE_QUANT_SCHEMES
+ and not getattr(fam, "modular_workflow", None)
+ and native_quant_host(target)
+ ):
+ # The modular workflow seeds torchao checkpoints, so it stays off the native path.
+ if native_quant_scheme(target, pinned, family = getattr(fam, "name", None)) is None:
+ reason = explain_unusable_scheme(getattr(fam, "name", None), pinned)
elif not dense_transformer_supported(target):
reason = dense_transformer_unsupported_reason(target)
elif forces_offload:
@@ -4110,12 +4124,15 @@ class VideoBackend:
kind == "pipeline"
and planned.offload_policy != "none"
and normalize_transformer_quant(transformer_quant) is not None
- and dense_transformer_supported(target)
+ and (
+ dense_transformer_supported(target)
+ or native_quant_scheme(target, transformer_quant, family = fam.name) is not None
+ )
and components is not None
):
- scheme_preview = select_transformer_quant_scheme(
+ scheme_preview = native_quant_scheme(
target, transformer_quant, family = fam.name
- )
+ ) or select_transformer_quant_scheme(target, transformer_quant, family = fam.name)
factor = _QUANT_STEADY_FACTOR.get(scheme_preview) if scheme_preview else None
if factor is not None:
quant_mib = int((components[0] * factor + companions_gb) * mib_per_gb)
@@ -4292,6 +4309,11 @@ class VideoBackend:
# Why the quant did not engage, in the caller's terms; threaded into `resolved`.
transformer_quant_decline: Optional[str] = None
transformer_quant_decline_status = RESOLVED_FELL_BACK
+ native_scheme = (
+ native_quant_scheme(target, transformer_quant_pinned, family = fam.name)
+ if kind == "pipeline"
+ else None
+ )
# Auto quantises a video DiT only to keep it resident: with bf16 resident, int8 fails the default LPIPS bar.
if (
kind == "pipeline"
@@ -4314,7 +4336,11 @@ class VideoBackend:
f"'{kind}' load, which runs the precision its checkpoint carries"
)
transformer_quant_decline_status = RESOLVED_UNSUPPORTED
- elif transformer_quant_pinned is not None and not dense_transformer_supported(target):
+ elif (
+ transformer_quant_pinned is not None
+ and native_scheme is None
+ and not dense_transformer_supported(target)
+ ):
# Ask the helper rather than repeating its fallback: on ROCm and on the Windows torchao
# stub it knows a truer reason, and an AMD owner reading "needs a CUDA GPU" while
# holding a working GPU learns nothing about why it declined.
@@ -4343,7 +4369,23 @@ class VideoBackend:
elif (
kind == "pipeline"
and normalize_transformer_quant(transformer_quant) is not None
- and dense_transformer_supported(target)
+ and (dense_transformer_supported(target) or native_scheme is not None)
+ and (source_precision := stored_denoiser_precision(_base_local_dir or repo_id))
+ is not None
+ ):
+ # from_pretrained widened a narrow checkpoint to bf16; requantising compounds the loss.
+ for view in views:
+ mark_source_precision(view, source_precision)
+ transformer_quant_decline = (
+ f"its published weights are {source_precision} and were widened to bf16 on load, so "
+ "quantising them again would compound that loss"
+ )
+ transformer_quant_decline_status = RESOLVED_UNSUPPORTED
+ logger.info("video.transformer_quant: skipped (%s)", transformer_quant_decline)
+ elif (
+ kind == "pipeline"
+ and normalize_transformer_quant(transformer_quant) is not None
+ and (dense_transformer_supported(target) or native_scheme is not None)
):
engaged = []
for view in views:
@@ -4366,6 +4408,20 @@ class VideoBackend:
f"transformer_quant={engaged[0]} engaged on only "
f"{len(engaged)}/{len(views)} experts; retry without quant."
)
+ # A part-converted view is unusable even when the precision fallback is allowed.
+ dirty = (
+ []
+ if engaged
+ else [i for i, view in enumerate(views) if transformer_is_quantised(view)]
+ )
+ if dirty:
+ del pipe
+ clear_gpu_cache()
+ raise RuntimeError(
+ f"transformer_quant='{transformer_quant}' converted part of the denoiser and then "
+ "failed, leaving it neither dense nor usable. Reload with Precision set to Off to "
+ "run the checkpoint as-is."
+ )
if engaged:
transformer_quant_engaged = engaged[0]
elif transformer_quant_pinned is not None:
@@ -4437,7 +4493,11 @@ class VideoBackend:
)
# A torchao-quantised DiT must be compiled (eager is ~30x slower), so force the regional profile when quant
# engaged but speed was off.
- if transformer_quant_engaged is not None and effective_speed == SPEED_OFF:
+ if (
+ transformer_quant_engaged is not None
+ and native_scheme is None
+ and effective_speed == SPEED_OFF
+ ):
logger.info(
"video.transformer_quant: forcing speed_mode=default "
"(quantized transformer must be compiled; eager is ~30x slower)"
@@ -4577,7 +4637,7 @@ class VideoBackend:
speed_mode,
effective_speed,
"quantized transformer requires compile"
- if transformer_quant_engaged is not None
+ if transformer_quant_engaged is not None and native_scheme is None
else "clip denoises amortise the one-time compile within a single run"
if speed_mode is None
else "requested",
@@ -4597,7 +4657,10 @@ class VideoBackend:
transformer_quant_engaged or "off",
# Honest framing: the shipped torchao schemes cut load time and resident memory ~2x, but
# per-step GEMMs are at best bf16 parity.
- "DiT(s) quantised (halves resident weights; hosted checkpoints cut "
+ f"weight-only: {transformer_quant_engaged} weights, bf16 compute "
+ "(torchao-free, a memory saving rather than a speed-up)"
+ if transformer_quant_engaged is not None and native_scheme is not None
+ else "DiT(s) quantised (halves resident weights; hosted checkpoints cut "
"load time; per-step speed is roughly bf16 parity)"
if transformer_quant_engaged is not None
else (
diff --git a/studio/backend/tests/test_diffusion_backend.py b/studio/backend/tests/test_diffusion_backend.py
index 0dd91819f9f..4b5a2590a26 100644
--- a/studio/backend/tests/test_diffusion_backend.py
+++ b/studio/backend/tests/test_diffusion_backend.py
@@ -12067,6 +12067,94 @@ def test_a_prequant_repo_missing_its_artifact_marks_the_plan_incomplete(monkeypa
assert "prequant artifact missing" in str(failures[0])
+def _stub_amd_weight_only_host(backend, monkeypatch):
+ """ROCm / torchao-stub host; records the quantise calls."""
+ from core.inference import diffusion as dmod
+ from core.inference import diffusion_transformer_quant as tq
+
+ calls: list = []
+ _force_cuda_target(backend, monkeypatch)
+ monkeypatch.setattr(dmod, "dense_transformer_supported", lambda target: False)
+ monkeypatch.setattr(tq, "dense_transformer_supported", lambda target: False)
+ monkeypatch.setattr(dmod, "native_quant_host", lambda target: True)
+ monkeypatch.setattr(tq, "native_quant_host", lambda target: True)
+ real_init = _FakePipe.__init__
+
+ def _init(self):
+ real_init(self)
+ self.transformer = _FakeDenoiser()
+
+ monkeypatch.setattr(_FakePipe, "__init__", _init)
+
+ def _quantize(
+ pipe,
+ target,
+ *,
+ mode,
+ family = None,
+ **kwargs,
+ ):
+ scheme = tq.native_quant_scheme(target, mode, family = family)
+ if scheme is None:
+ raise AssertionError("the torchao path must not run on an AMD weight-only host")
+ calls.append({"module": pipe.transformer, "scheme": scheme})
+ return scheme
+
+ monkeypatch.setattr(dmod, "quantize_transformer", _quantize)
+ monkeypatch.setattr(dmod, "transformer_is_quantised", lambda module: bool(calls))
+ return calls
+
+
[email protected]("scheme", ["int8", "fp8"])
+def test_an_explicit_scheme_on_amd_runs_weight_only(fake_runtime, tmp_path, monkeypatch, scheme):
+ backend = DiffusionBackend()
+ calls = _stub_amd_weight_only_host(backend, monkeypatch)
+ status = backend.load_pipeline(
+ "Qwen/Qwen-Image-2512",
+ model_kind = "pipeline",
+ transformer_quant = scheme,
+ _base_local_dir = str(tmp_path),
+ )
+ assert [c["scheme"] for c in calls] == [scheme]
+ assert calls[0]["module"] is backend._state.pipe.transformer
+ assert status["transformer_quant"] == scheme
+ resolved = status["resolved"]["transformer_quant"]
+ assert resolved["value"] == scheme
+ assert resolved["source"] == "explicit" and resolved["status"] == "applied"
+ assert "weight-only" in resolved["reason"] and "bf16 compute" in resolved["reason"]
+ assert "requires compile" not in status["resolved"]["speed_mode"]["reason"]
+ backend.unload()
+
+
+def test_auto_on_amd_stays_bf16(fake_runtime, tmp_path, monkeypatch):
+ backend = DiffusionBackend()
+ calls = _stub_amd_weight_only_host(backend, monkeypatch)
+ status = backend.load_pipeline(
+ "Qwen/Qwen-Image-2512", model_kind = "pipeline", _base_local_dir = str(tmp_path)
+ )
+ assert calls == []
+ assert status["transformer_quant"] is None
+ assert status["resolved"]["transformer_quant"]["value"] == "off"
+ backend.unload()
+
+
+def test_an_explicit_scheme_on_nvidia_keeps_the_torchao_path_and_its_wording(
+ fake_runtime, tmp_path, monkeypatch
+):
+ backend = DiffusionBackend()
+ calls = _stub_pipeline_dense_quant(backend, monkeypatch)
+ status = backend.load_pipeline(
+ "Qwen/Qwen-Image-2512",
+ model_kind = "pipeline",
+ transformer_quant = "fp8",
+ _base_local_dir = str(tmp_path),
+ )
+ assert len(calls) == 1
+ assert status["transformer_quant"] == "fp8"
+ assert "weight-only" not in status["resolved"]["transformer_quant"]["reason"]
+ backend.unload()
+
+
def _record_step_cache(
monkeypatch,
*,
diff --git a/studio/backend/tests/test_diffusion_native_quant.py b/studio/backend/tests/test_diffusion_native_quant.py
new file mode 100644
index 00000000000..013f768715e
--- /dev/null
+++ b/studio/backend/tests/test_diffusion_native_quant.py
@@ -0,0 +1,554 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
+
+"""Torchao-free weight-only int8 / fp8; ROCm simulated by patching ``torch_is_rocm`` / ``is_stubbed``."""
+
+from __future__ import annotations
+
+import sys
+import types
+
+import pytest
+
+torch = pytest.importorskip("torch")
+
+import core.inference.diffusion_native_quant as nq
+import core.inference.diffusion_transformer_quant as tq
+
+
+def _target(device = "cuda", dtype = None):
+ return types.SimpleNamespace(device = device, dtype = torch.bfloat16 if dtype is None else dtype)
+
+
+def rocm(monkeypatch):
+ monkeypatch.setattr(tq, "torch_is_rocm", lambda: True)
+ monkeypatch.setattr(tq, "is_stubbed", lambda name: False)
+
+
+def win_stub(monkeypatch):
+ monkeypatch.setattr(tq, "torch_is_rocm", lambda: False)
+ monkeypatch.setattr(tq, "is_stubbed", lambda name: name == "torchao")
+
+
+def nvidia(monkeypatch):
+ monkeypatch.setattr(tq, "torch_is_rocm", lambda: False)
+ monkeypatch.setattr(tq, "is_stubbed", lambda name: False)
+
+
+class _Block(torch.nn.Module):
+ def __init__(self, dim):
+ super().__init__()
+ self.to_q = torch.nn.Linear(dim, dim)
+ self.ff = torch.nn.Linear(dim, 2 * dim, bias = False)
+ self.out = torch.nn.Linear(2 * dim, dim)
+ self.small = torch.nn.Linear(dim, 8)
+
+ def forward(self, x):
+ return self.out(torch.nn.functional.gelu(self.ff(self.to_q(x)))) + self.small(x).sum(
+ -1, keepdim = True
+ )
+
+
+class _Toy(torch.nn.Module):
+ def __init__(
+ self,
+ dim = 128,
+ depth = 2,
+ ):
+ super().__init__()
+ self.blocks = torch.nn.ModuleList(_Block(dim) for _ in range(depth))
+
+ def forward(self, x):
+ for b in self.blocks:
+ x = x + b(x)
+ return x
+
+
+def _toy(dtype = torch.bfloat16):
+ torch.manual_seed(0)
+ return _Toy().to(dtype)
+
+
[email protected]("scheme", ["int8", "fp8", "INT8", " fp8 "])
+def test_rocm_and_stub_hosts_take_explicit_int8_and_fp8(rocm, scheme):
+ assert tq.native_quant_host(_target())
+ assert tq.native_quant_scheme(_target(), scheme) == scheme.strip().lower()
+
+
+def test_windows_stub_host_is_native(win_stub):
+ assert tq.native_quant_host(_target())
+ assert tq.native_quant_scheme(_target(), "int8") == "int8"
+
+
[email protected]("scheme", [None, "auto", "none", "off", "nvfp4", "mxfp8"])
+def test_auto_and_other_schemes_never_go_native(rocm, scheme):
+ assert tq.native_quant_scheme(_target(), scheme) is None
+
+
+ "target",
+ [
+ _target(device = "cpu"),
+ _target(device = "mps"),
+ _target(device = "xpu"),
+ _target(dtype = torch.float16),
+ _target(dtype = torch.float32),
+ ],
+)
+def test_non_cuda_or_non_bf16_is_never_native_even_on_rocm(rocm, target):
+ assert not tq.native_quant_host(target)
+ assert tq.native_quant_scheme(target, "int8") is None
+
+
[email protected]("device", ["cuda", "cpu", "mps", "xpu"])
+def test_nvidia_and_other_hosts_are_never_native(nvidia, device):
+ assert not tq.native_quant_host(_target(device = device))
+ for scheme in ("int8", "fp8", "nvfp4", "mxfp8", "auto"):
+ assert tq.native_quant_scheme(_target(device = device), scheme) is None
+
+
+def test_family_deny_list_still_applies(rocm, monkeypatch):
+ monkeypatch.setitem(tq._FAMILY_SCHEME_DENY, "toy-family", frozenset({tq.TQ_INT8}))
+ assert tq.native_quant_scheme(_target(), "int8", family = "toy-family") is None
+ assert tq.native_quant_scheme(_target(), "fp8", family = "toy-family") == "fp8"
+ # The shipped deny list only fences nvfp4 / mxfp8 off qwen-image, so int8 / fp8 run there.
+ assert tq.native_quant_scheme(_target(), "int8", family = "qwen-image") == "int8"
+ assert tq.native_quant_scheme(_target(), "fp8", family = "qwen-image-2.1") == "fp8"
+
+
+def test_the_shared_selector_is_untouched_on_rocm(rocm):
+ # Hosted prequant planners read this selector; AMD must never be handed a torchao checkpoint.
+ for scheme in ("int8", "fp8", "auto"):
+ assert tq.select_transformer_quant_scheme(_target(), scheme, unproven_ok = True) is None
+ assert not tq.dense_transformer_supported(_target())
+
+
[email protected]("scheme,bound", [("int8", 0.01), ("fp8", 0.04)])
+def test_native_layer_reconstructs_the_weight(scheme, bound):
+ torch.manual_seed(0)
+ lin = torch.nn.Linear(256, 512).to(torch.bfloat16)
+ err = nq.native_weight_error(lin, scheme)
+ assert err is not None and err < bound
+ layer = nq.native_linear_class()(lin, scheme)
+ assert layer.weight_q.dtype == (torch.int8 if scheme == "int8" else torch.uint8)
+ assert layer.weight_scale.dtype == torch.int32 and layer.weight_scale.shape == (512,)
+ assert layer.weight_q.element_size() == 1
+ assert layer.bias is lin.bias
+ x = torch.randn(4, 256, dtype = torch.bfloat16)
+ ref = lin(x).float()
+ got = layer(x).float()
+ assert layer(x).dtype == torch.bfloat16
+ assert ((got - ref).norm() / ref.norm()).item() < bound * 2
+
+
+def test_native_layer_rejects_an_unknown_scheme():
+ with pytest.raises(ValueError):
+ nq.native_linear_class()(torch.nn.Linear(8, 8), "nvfp4")
+
+
+def test_zero_rows_do_not_divide_by_zero():
+ lin = torch.nn.Linear(16, 4, bias = False).to(torch.bfloat16)
+ with torch.no_grad():
+ lin.weight.zero_()
+ for scheme in ("int8", "fp8"):
+ layer = nq.native_linear_class()(lin, scheme)
+ assert torch.isfinite(layer.dequantized_weight(torch.float32)).all()
+ assert layer(torch.ones(2, 16, dtype = torch.bfloat16)).abs().max().item() == 0
+
+
+def test_native_buffers_move_with_module_to_and_state_dict():
+ lin = torch.nn.Linear(64, 64).to(torch.bfloat16)
+ layer = nq.native_linear_class()(lin, "int8")
+ keys = set(layer.state_dict())
+ assert {"weight_q", "weight_scale", "bias"} <= keys and "weight" not in keys
+ layer.to("meta")
+ assert layer.weight_q.device.type == "meta" and layer.weight_q.dtype == torch.int8
+ assert layer.weight_scale.dtype == torch.int32
+
+
[email protected]("scheme", ["int8", "fp8"])
+def test_a_module_wide_dtype_cast_leaves_the_stored_weights_alone(scheme):
+ """A module ``.to(dtype)`` must not cast the integer-view payload or scales."""
+ torch.manual_seed(0)
+ lin = torch.nn.Linear(128, 64).to(torch.bfloat16)
+ layer = nq.native_linear_class()(lin, scheme)
+ x = torch.randn(3, 128, dtype = torch.bfloat16)
+ before = layer(x).clone()
+ q_before = layer.weight_q.clone()
+ layer.to(torch.float16)
+ assert layer.weight_q.dtype == q_before.dtype and torch.equal(layer.weight_q, q_before)
+ assert layer.weight_scale.dtype == torch.int32
+ assert torch.allclose(layer(x.to(torch.float16)).float(), before.float(), rtol = 1e-2, atol = 1e-2)
+
+
+def _block_torchao(monkeypatch):
+ # A None entry makes any `import torchao.quantization` raise, proving the branch never reaches it.
+ monkeypatch.setitem(sys.modules, "torchao", None)
+ monkeypatch.setitem(sys.modules, "torchao.quantization", None)
+
+
[email protected]("scheme", ["int8", "fp8"])
+def test_quantize_transformer_goes_native_on_rocm(rocm, monkeypatch, scheme):
+ _block_torchao(monkeypatch)
+ model = _toy()
+ x = torch.randn(3, 128, dtype = torch.bfloat16)
+ with torch.no_grad():
+ ref = model(x).float()
+ pipe = types.SimpleNamespace(transformer = model)
+ logs = []
+ logger = types.SimpleNamespace(
+ info = lambda *a: logs.append(a), warning = lambda *a: pytest.fail(f"warned: {a}")
+ )
+ got = tq.quantize_transformer(pipe, _target(), mode = scheme, min_features = 64, logger = logger)
+ assert got == scheme
+ assert model._unsloth_runtime_quant == scheme
+ swapped = {n for n, m in model.named_modules() if nq.is_native_linear(m)}
+ assert swapped == {f"blocks.{i}.{n}" for i in range(2) for n in ("to_q", "ff", "out")}
+ assert isinstance(model.blocks[0].small, torch.nn.Linear)
+ assert tq.transformer_is_quantised(model) and nq.is_native_quantised(model)
+ assert tq.dense_quant_blocker(pipe) is not None # a second pass would compound the loss
+ assert any("torchao-free" in str(a[0]) for a in logs)
+ with torch.no_grad():
+ out = model(x).float()
+ assert ((out - ref).norm() / ref.norm()).item() < 0.05
+
+
+def test_quantize_transformer_leaves_non_bf16_linears_dense(rocm, monkeypatch):
+ _block_torchao(monkeypatch)
+ model = _toy()
+ model.blocks[1].ff.float() # a deliberately fp32 layer, as Wan and Hunyuan keep
+ tq.quantize_transformer(
+ types.SimpleNamespace(transformer = model), _target(), mode = "int8", min_features = 64
+ )
+ assert isinstance(model.blocks[1].ff, torch.nn.Linear)
+ assert nq.is_native_linear(model.blocks[0].ff)
+
+
+def test_quantize_transformer_skips_lora_side_paths(rocm, monkeypatch):
+ _block_torchao(monkeypatch)
+ model = _toy()
+ model.blocks[0].lora_A = torch.nn.Linear(128, 128).to(torch.bfloat16)
+ tq.quantize_transformer(
+ types.SimpleNamespace(transformer = model), _target(), mode = "fp8", min_features = 64
+ )
+ assert isinstance(model.blocks[0].lora_A, torch.nn.Linear)
+
+
+def test_quantize_transformer_auto_on_rocm_stays_dense(rocm, monkeypatch):
+ _block_torchao(monkeypatch)
+ model = _toy()
+ got = tq.quantize_transformer(
+ types.SimpleNamespace(transformer = model), _target(), mode = "auto", min_features = 64
+ )
+ assert got is None and not tq.transformer_is_quantised(model)
+
+
+def test_quantize_transformer_failure_is_reported_as_dirty(rocm, monkeypatch):
+ model = _toy()
+ real = nq.native_linear_class()
+ calls = []
+
+ def _boom(linear, scheme):
+ calls.append(1)
+ if len(calls) == 3:
+ raise RuntimeError("out of memory")
+ return real(linear, scheme)
+
+ monkeypatch.setattr(nq, "native_linear_class", lambda: _boom)
+ warned = []
+ logger = types.SimpleNamespace(info = lambda *a: None, warning = lambda *a: warned.append(a))
+ got = tq.quantize_transformer(
+ types.SimpleNamespace(transformer = model),
+ _target(),
+ mode = "int8",
+ min_features = 64,
+ logger = logger,
+ )
+ assert got is None and warned
+ assert tq.transformer_is_quantised(model)
+ assert not getattr(model, "_unsloth_runtime_quant", None)
+
+
+def test_the_pass_does_not_hold_every_dense_layer_alive(rocm, monkeypatch):
+ import weakref
+
+ model = _toy()
+ dense_ff = weakref.ref(model.blocks[0].ff)
+ tq.quantize_transformer(
+ types.SimpleNamespace(transformer = model), _target(), mode = "int8", min_features = 64
+ )
+ assert dense_ff() is None
+
+
+def test_quantize_transformer_on_nvidia_never_reaches_the_native_branch(nvidia, monkeypatch):
+ monkeypatch.setattr(
+ tq, "_quantize_native", lambda *a, **k: pytest.fail("native branch on an NVIDIA host")
+ )
+ seen = []
+ monkeypatch.setattr(
+ tq, "select_transformer_quant_scheme", lambda *a, **k: seen.append(a) or None
+ )
+ model = _toy()
+ for scheme in ("int8", "fp8", "auto"):
+ assert (
+ tq.quantize_transformer(
+ types.SimpleNamespace(transformer = model), _target(), mode = scheme
+ )
+ is None
+ )
+ assert len(seen) == 3
+
+
[email protected]("scheme", ["int8", "fp8"])
+def test_reading_weight_gives_the_dense_weight_without_storing_it(scheme):
+ """PEFT DoRA reads ``base_layer.weight``: dense, rebuilt per read, not in the state dict."""
+ torch.manual_seed(0)
+ lin = torch.nn.Linear(128, 64).to(torch.bfloat16)
+ layer = nq.native_linear_class()(lin, scheme)
+ weight = layer.weight
+ assert weight.dtype == torch.bfloat16 and weight.shape == (64, 128)
+ assert torch.equal(weight, layer.dequantized_weight(torch.bfloat16))
+ assert "weight" not in set(layer.state_dict())
+ assert all(p is not weight for p in layer.parameters())
+
+
+def test_a_dora_adapter_runs_on_a_native_base_layer(rocm, monkeypatch):
+ peft = pytest.importorskip("peft")
+ _block_torchao(monkeypatch)
+ torch.manual_seed(0)
+ model = _toy()
+ cfg = peft.LoraConfig(r = 4, target_modules = ["to_q"], use_dora = True, init_lora_weights = False)
+ wrapped = peft.inject_adapter_in_model(cfg, model)
+ x = torch.randn(2, 128, dtype = torch.bfloat16)
+ with torch.no_grad():
+ ref = wrapped(x).float()
+ tq.quantize_transformer(
+ types.SimpleNamespace(transformer = wrapped), _target(), mode = "int8", min_features = 64
+ )
+ assert nq.is_native_linear(wrapped.blocks[0].to_q.base_layer)
+ with torch.no_grad():
+ out = wrapped(x).float()
+ assert torch.isfinite(out).all()
+ assert float((out - ref).norm() / ref.norm()) < 0.05
+
+
+def test_peft_wrapped_base_layer_runs_native(rocm, monkeypatch):
+ peft = pytest.importorskip("peft")
+ _block_torchao(monkeypatch)
+ model = _toy()
+ cfg = peft.LoraConfig(r = 4, target_modules = ["to_q"], lora_alpha = 4)
+ wrapped = peft.inject_adapter_in_model(cfg, model)
+ x = torch.randn(2, 128, dtype = torch.bfloat16)
+ tq.quantize_transformer(
+ types.SimpleNamespace(transformer = wrapped), _target(), mode = "int8", min_features = 64
+ )
+ to_q = wrapped.blocks[0].to_q
+ assert nq.is_native_linear(to_q.base_layer)
+ assert isinstance(to_q.lora_A["default"], torch.nn.Linear)
+ with torch.no_grad():
+ assert torch.isfinite(wrapped(x)).all()
+
+
+def _image_family(name):
+ from core.inference.diffusion_families import _FAMILIES
+ return next(f for f in _FAMILIES if f.name == name)
+
+
+def _video_family(name):
+ from core.inference.video_families import _FAMILIES
+ return next(f for f in _FAMILIES if f.name == name)
+
+
+def _image_gate(
+ monkeypatch,
+ fam,
+ pinned,
+ *,
+ model_kind = "pipeline",
+ memory_mode = None,
+):
+ import core.inference.diffusion as d
+ monkeypatch.setattr(d, "effective_te_quant", lambda *a, **k: None)
+ d.DiffusionBackend._assert_precision_for_target(
+ None,
+ fam,
+ _target(),
+ model_kind = model_kind,
+ pinned = pinned,
+ te_mode = None,
+ memory_mode = memory_mode,
+ cpu_offload = False,
+ )
+
+
+def _video_gate(
+ monkeypatch,
+ fam,
+ pinned,
+ *,
+ model_kind = "pipeline",
+ memory_mode = None,
+):
+ import core.inference.video as v
+ monkeypatch.setattr(v, "effective_te_quant", lambda *a, **k: None)
+ v._assert_video_precision_for_target(
+ fam, _target(), model_kind = model_kind, transformer_quant = pinned, memory_mode = memory_mode
+ )
+
+
[email protected]("pinned", ["int8", "fp8"])
[email protected]("memory_mode", [None, "balanced", "low_vram"])
+def test_image_gate_admits_native_int8_fp8_on_rocm(rocm, monkeypatch, pinned, memory_mode):
+ _image_gate(monkeypatch, _image_family("qwen-image-2.1"), pinned, memory_mode = memory_mode)
+
+
[email protected]("pinned", ["nvfp4", "mxfp8"])
+def test_image_gate_still_refuses_torchao_only_schemes_on_rocm(rocm, monkeypatch, pinned):
+ with pytest.raises(RuntimeError, match = "transformer_quant"):
+ _image_gate(monkeypatch, _image_family("qwen-image-2.1"), pinned)
+
+
[email protected]("kind", ["gguf", "single_file"])
+def test_image_gate_still_refuses_native_on_non_pipeline_loads(rocm, monkeypatch, kind):
+ with pytest.raises(RuntimeError, match = "transformer_quant"):
+ _image_gate(monkeypatch, _image_family("qwen-image-2.1"), "int8", model_kind = kind)
+
+
+def test_image_gate_refuses_a_unet_family_on_rocm(rocm, monkeypatch):
+ with pytest.raises(RuntimeError, match = "transformer_quant"):
+ _image_gate(monkeypatch, _image_family("sdxl"), "int8")
+
+
+def test_image_gate_refuses_a_denied_family_on_rocm(rocm, monkeypatch):
+ monkeypatch.setitem(tq._FAMILY_SCHEME_DENY, "qwen-image-2.1", frozenset({tq.TQ_INT8}))
+ with pytest.raises(RuntimeError, match = "transformer_quant"):
+ _image_gate(monkeypatch, _image_family("qwen-image-2.1"), "int8")
+ _image_gate(monkeypatch, _image_family("qwen-image-2.1"), "fp8")
+
+
+def test_image_gate_on_nvidia_never_consults_the_native_scheme(nvidia, monkeypatch):
+ import core.inference.diffusion as d
+
+ monkeypatch.setattr(d, "native_quant_scheme", lambda *a, **k: pytest.fail("native on NVIDIA"))
+ monkeypatch.setattr(d, "dense_transformer_supported", lambda target: False)
+ with pytest.raises(RuntimeError, match = "transformer_quant"):
+ _image_gate(monkeypatch, _image_family("qwen-image-2.1"), "int8")
+
+
[email protected]("pinned", ["int8", "fp8"])
[email protected]("memory_mode", [None, "balanced"])
+def test_video_gate_admits_native_int8_fp8_on_rocm(rocm, monkeypatch, pinned, memory_mode):
+ _video_gate(monkeypatch, _video_family("wan2.2-ti2v-5b"), pinned, memory_mode = memory_mode)
+
+
+def test_video_gate_keeps_the_modular_workflow_off_the_native_path(rocm, monkeypatch):
+ with pytest.raises(RuntimeError, match = "transformer_quant"):
+ _video_gate(monkeypatch, _video_family("minimax-h3"), "int8")
+
+
[email protected]("pinned", ["nvfp4", "mxfp8"])
+def test_video_gate_still_refuses_torchao_only_schemes_on_rocm(rocm, monkeypatch, pinned):
+ with pytest.raises(RuntimeError, match = "transformer_quant"):
+ _video_gate(monkeypatch, _video_family("wan2.2-ti2v-5b"), pinned)
+
+
+def test_video_gate_refuses_native_on_gguf_loads(rocm, monkeypatch):
+ with pytest.raises(RuntimeError, match = "transformer_quant"):
+ _video_gate(monkeypatch, _video_family("wan2.2-ti2v-5b"), "int8", model_kind = "gguf")
+
+
+def _native_block(features = 64):
+ lin = torch.nn.Linear(features, features, dtype = torch.bfloat16)
+ return nq.native_linear_class()(lin, "int8")
+
+
+def test_stream_group_offload_puts_native_weight_buffers_back_on_the_host():
+ go = pytest.importorskip("diffusers.hooks.group_offloading")
+ from core.inference.diffusion_memory import install_group_offload_buffer_restore
+
+ install_group_offload_buffer_restore()
+ layer = _native_block()
+ host = {b: b.data for b in layer.buffers()}
+ host.update({p: p.data for p in layer.parameters()})
+ group = object.__new__(go.ModuleGroup)
+ group.modules, group.parameters, group.buffers = [layer], [], []
+ group.stream, group.record_stream = object(), True
+ group.cpu_param_dict = dict(host)
+ for tensor in list(layer.buffers()) + list(layer.parameters()):
+ tensor.data = tensor.data.clone() # stands in for the onloaded device copy
+ group._offload_to_memory()
+ assert all(b.data_ptr() == host[b].data_ptr() for b in layer.buffers())
+ assert all(p.data_ptr() == host[p].data_ptr() for p in layer.parameters())
+
+
+def test_buffer_restore_install_is_idempotent():
+ go = pytest.importorskip("diffusers.hooks.group_offloading")
+ from core.inference.diffusion_memory import install_group_offload_buffer_restore
+
+ install_group_offload_buffer_restore()
+ patched = go.ModuleGroup._offload_to_memory
+ assert install_group_offload_buffer_restore() is False
+ assert go.ModuleGroup._offload_to_memory is patched
+ assert getattr(patched, "__wrapped__", None) is not None
+
+
+ not torch.cuda.is_available(), reason = "stream group offload needs a CUDA device"
+)
+def test_native_int8_under_real_stream_group_offload_leaves_nothing_resident():
+ pytest.importorskip("diffusers.hooks.group_offloading")
+ from diffusers.hooks import apply_group_offloading
+
+ from core.inference.diffusion_memory import install_group_offload_buffer_restore
+
+ install_group_offload_buffer_restore()
+
+ class Model(torch.nn.Module):
+ def __init__(self):
+ super().__init__()
+ self.blocks = torch.nn.ModuleList([_native_block(), _native_block()])
+
+ def forward(self, x):
+ for block in self.blocks:
+ x = block(x)
+ return x
+
+ model = Model()
+ apply_group_offloading(
+ model,
+ onload_device = torch.device("cuda"),
+ offload_device = torch.device("cpu"),
+ offload_type = "block_level",
+ num_blocks_per_group = 1,
+ use_stream = True,
+ non_blocking = True,
+ record_stream = True,
+ )
+ with torch.inference_mode():
+ out = model(torch.randn(32, 64, device = "cuda", dtype = torch.bfloat16))
+ torch.cuda.synchronize()
+ assert out.device.type == "cuda"
+ assert {b.device.type for b in model.buffers()} == {"cpu"}
+
+
+def test_qkv_fusion_skips_a_native_quantised_denoiser():
+ from core.inference import diffusion_speed
+
+ calls = []
+
+ class _Dit(torch.nn.Module):
+ def __init__(self, native):
+ super().__init__()
+ lin = torch.nn.Linear(64, 64).to(torch.bfloat16)
+ self.to_q = nq.native_linear_class()(lin, "int8") if native else lin
+
+ def fuse_qkv_projections(self):
+ calls.append(self)
+
+ native = types.SimpleNamespace(transformer = _Dit(native = True))
+ assert diffusion_speed._fuse_qkv(native, None) is False and not calls
+ dense = types.SimpleNamespace(transformer = _Dit(native = False))
+ assert diffusion_speed._fuse_qkv(dense, None) is True and len(calls) == 1
diff --git a/studio/backend/tests/test_diffusion_transformer_quant.py b/studio/backend/tests/test_diffusion_transformer_quant.py
index 0c9524a0814..6308a8aae9a 100644
--- a/studio/backend/tests/test_diffusion_transformer_quant.py
+++ b/studio/backend/tests/test_diffusion_transformer_quant.py
@@ -1862,6 +1862,39 @@ def test_no_snapshot_is_no_verdict(local_dir):
assert tq.stored_denoiser_precision(local_dir) is None
+def _cached_snapshot(cache, repo_id, dtype):
+ snap = cache / f"models--{repo_id.replace('/', '--')}" / "snapshots" / "abc123"
+ _write_shard(snap, "transformer", dtype)
+ (cache / f"models--{repo_id.replace('/', '--')}" / "refs").mkdir()
+ (cache / f"models--{repo_id.replace('/', '--')}" / "refs" / "main").write_text("abc123")
+
+
+def test_an_offline_hub_id_reads_its_cached_snapshot(tmp_path, monkeypatch):
+ """An offline load passes a repo id (nothing staged); the cached snapshot still carries the headers."""
+ import torch
+ from huggingface_hub import constants
+
+ monkeypatch.setattr(constants, "HF_HUB_CACHE", str(tmp_path))
+ _cached_snapshot(tmp_path, "org/narrow-dit", torch.float8_e4m3fn)
+ _cached_snapshot(tmp_path, "org/wide-dit", torch.bfloat16)
+ assert tq.stored_denoiser_precision("org/narrow-dit") == "fp8"
+ assert tq.stored_denoiser_precision("org/wide-dit") is None
+ assert tq.stored_denoiser_precision("org/not-cached") is None
+
+
+def test_an_offline_hub_id_reads_studios_live_cache(tmp_path, monkeypatch):
+ """Studio can move its cache at runtime; the loaders read from there, so the probe must too."""
+ import torch
+ from huggingface_hub import constants
+ from utils import hf_cache_settings
+
+ live = tmp_path / "live"
+ monkeypatch.setattr(constants, "HF_HUB_CACHE", str(tmp_path / "import_time_default"))
+ monkeypatch.setattr(hf_cache_settings, "active_hf_hub_cache", lambda: str(live))
+ _cached_snapshot(live, "org/narrow-dit", torch.float8_e4m3fn)
+ assert tq.stored_denoiser_precision("org/narrow-dit") == "fp8"
+
+
def test_a_recovered_source_precision_blocks_the_quant(tmp_path):
"""The recovered marker feeds the same refusal the Ideogram loader's marker does."""
import torch
diff --git a/studio/backend/tests/test_video_backend.py b/studio/backend/tests/test_video_backend.py
index 648dc72baba..91d819879ba 100644
--- a/studio/backend/tests/test_video_backend.py
+++ b/studio/backend/tests/test_video_backend.py
@@ -9864,6 +9864,123 @@ def test_the_boundary_marker_waits_out_a_busy_capture_lock(fake_runtime, monkeyp
assert at_decode.get("phase") == "decode"
[email protected]("scheme", ["int8", "fp8"])
+def test_an_explicit_video_scheme_on_amd_runs_weight_only_without_forcing_compile(
+ fake_runtime, monkeypatch, scheme
+):
+ """An explicit int8 / fp8 video DiT engages the native branch on AMD, honouring speed=off."""
+ import core.inference.video as video_mod
+ from core.inference import diffusion_transformer_quant as tq
+
+ monkeypatch.setattr(video_mod, "dense_transformer_supported", lambda target: False)
+ monkeypatch.setattr(tq, "dense_transformer_supported", lambda target: False)
+ monkeypatch.setattr(video_mod, "native_quant_host", lambda target: True)
+ monkeypatch.setattr(tq, "native_quant_host", lambda target: True)
+ calls: list = []
+
+ def _quantize(
+ view,
+ target,
+ *,
+ mode,
+ family = None,
+ **kw,
+ ):
+ assert tq.native_quant_scheme(target, mode, family = family) == scheme
+ calls.append(mode)
+ return scheme
+
+ monkeypatch.setattr(video_mod, "quantize_transformer", _quantize)
+ backend = VideoBackend()
+ status = backend.load_pipeline(
+ "Wan-AI/Wan2.2-TI2V-5B-Diffusers",
+ model_kind = "pipeline",
+ transformer_quant = scheme,
+ speed_mode = "off",
+ )
+ assert calls and set(calls) == {scheme}
+ assert status["transformer_quant"] == scheme
+ assert status["speed_mode"] == "off"
+ assert "requires compile" not in status["resolved"]["speed_mode"]["reason"]
+ assert "weight-only" in status["resolved"]["transformer_quant"]["reason"]
+ backend.unload()
+
+
[email protected]("fallback", ["0", "1"])
+def test_a_video_checkpoint_stored_narrow_is_not_quantised_again(
+ fake_runtime, monkeypatch, fallback
+):
+ """A bf16-widened fp8 checkpoint is declined without calling the quantiser."""
+ import core.inference.video as video_mod
+ from core.inference import diffusion_transformer_quant as tq
+
+ monkeypatch.setenv("UNSLOTH_DIFFUSION_ALLOW_PRECISION_FALLBACK", fallback)
+ monkeypatch.setattr(video_mod, "dense_transformer_supported", lambda target: False)
+ monkeypatch.setattr(video_mod, "native_quant_host", lambda target: True)
+ monkeypatch.setattr(tq, "dense_transformer_supported", lambda target: False)
+ monkeypatch.setattr(tq, "native_quant_host", lambda target: True)
+ monkeypatch.setattr(video_mod, "stored_denoiser_precision", lambda local_dir: "fp8")
+
+ def _quantize(view, target, **kw):
+ raise AssertionError("a narrow-stored DiT must not be quantised again")
+
+ monkeypatch.setattr(video_mod, "quantize_transformer", _quantize)
+ backend = VideoBackend()
+ if fallback == "0":
+ with pytest.raises(RuntimeError, match = "widened to bf16"):
+ backend.load_pipeline(
+ "Wan-AI/Wan2.2-TI2V-5B-Diffusers", model_kind = "pipeline", transformer_quant = "int8"
+ )
+ return
+ status = backend.load_pipeline(
+ "Wan-AI/Wan2.2-TI2V-5B-Diffusers", model_kind = "pipeline", transformer_quant = "int8"
+ )
+ resolved = status["resolved"]["transformer_quant"]
+ assert resolved["status"] == "unsupported" and "widened to bf16" in resolved["reason"]
+ backend.unload()
+
+
[email protected]("fallback", ["0", "1"])
+def test_a_partly_converted_video_denoiser_is_refused_even_with_the_fallback_allowed(
+ fake_runtime, monkeypatch, fallback
+):
+ """A part-converted weight-only DiT is refused even with the precision fallback."""
+ import core.inference.video as video_mod
+ from core.inference import diffusion_transformer_quant as tq
+
+ monkeypatch.setenv("UNSLOTH_DIFFUSION_ALLOW_PRECISION_FALLBACK", fallback)
+ monkeypatch.setattr(video_mod, "dense_transformer_supported", lambda target: False)
+ monkeypatch.setattr(video_mod, "native_quant_host", lambda target: True)
+ monkeypatch.setattr(tq, "dense_transformer_supported", lambda target: False)
+ monkeypatch.setattr(tq, "native_quant_host", lambda target: True)
+ monkeypatch.setattr(video_mod, "quantize_transformer", lambda view, target, **kw: None)
+ monkeypatch.setattr(video_mod, "transformer_is_quantised", lambda view: True)
+ backend = VideoBackend()
+ with pytest.raises(RuntimeError, match = "neither dense nor usable"):
+ backend.load_pipeline(
+ "Wan-AI/Wan2.2-TI2V-5B-Diffusers", model_kind = "pipeline", transformer_quant = "int8"
+ )
+ assert backend._state is None
+
+
+def test_a_clean_video_decline_under_the_fallback_still_loads_dense(fake_runtime, monkeypatch):
+ import core.inference.video as video_mod
+ from core.inference import diffusion_transformer_quant as tq
+
+ monkeypatch.setenv("UNSLOTH_DIFFUSION_ALLOW_PRECISION_FALLBACK", "1")
+ monkeypatch.setattr(video_mod, "dense_transformer_supported", lambda target: False)
+ monkeypatch.setattr(video_mod, "native_quant_host", lambda target: True)
+ monkeypatch.setattr(tq, "dense_transformer_supported", lambda target: False)
+ monkeypatch.setattr(tq, "native_quant_host", lambda target: True)
+ monkeypatch.setattr(video_mod, "quantize_transformer", lambda view, target, **kw: None)
+ monkeypatch.setattr(video_mod, "transformer_is_quantised", lambda view: False)
+ backend = VideoBackend()
+ status = backend.load_pipeline(
+ "Wan-AI/Wan2.2-TI2V-5B-Diffusers", model_kind = "pipeline", transformer_quant = "int8"
+ )
+ assert status["transformer_quant"] in (None, "none", "off")
+
+
def test_video_auto_below_max_names_an_uncacheable_dit(fake_runtime, monkeypatch):
monkeypatch.setattr(
"core.inference.video.step_cache_supported", lambda pipe, logger = None: False