Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
Unsloth Studio (AMD): allow INT8 / FP8 image and video precision without torchao (#11631)
Daniel Hanf0b73b2fed4f
diff --git a/studio/backend/core/inference/diffusion.py b/studio/backend/core/inference/diffusion.pyindex 28cecbb4b9f..da2e82adc86 100644--- a/studio/backend/core/inference/diffusion.py+++ b/studio/backend/core/inference/diffusion.py@@ -222,6 +222,9 @@ from .diffusion_transformer_quant import (denoiser_modules,explain_unusable_scheme,mark_source_precision,+ native_quant_host,+ native_quant_scheme,+ NATIVE_QUANT_SCHEMES,normalize_transformer_quant,quantize_transformer,select_transformer_quant_scheme,@@ -1818,6 +1821,13 @@ class DiffusionBackend:f"{getattr(fam, 'denoiser_attr', 'unet')}, not a transformer, and the dense torchao schemes ""do not cover it")+ elif (+ model_kind == "pipeline"+ and pinned in NATIVE_QUANT_SCHEMES+ and native_quant_host(target)+ ):+ if native_quant_scheme(target, pinned, family = getattr(fam, "name", None)) is None:+ reason = explain_unusable_scheme(getattr(fam, "name", None), pinned)elif _memory_request_forces_offload(memory_mode, cpu_offload):# Not a measurement: balanced and low_vram name their policy outright, and the legacy flag forces# model offload. Offload hooks move modules with Module.to(), which torchao tensors do not survive, so@@ -4517,19 +4527,30 @@ class DiffusionBackend:# visible, and into the refusal so it is actionable.transformer_quant_decline: Optional[str] = Nonetransformer_quant_decline_status = RESOLVED_FELL_BACK+ native_scheme = (+ native_quant_scheme(+ target, transformer_quant_pinned, family = getattr(fam, "name", None)+ )+ if kind == "pipeline"+ else None+ )if transformer_quant_pinned is not None and not dense_quant_supported_kind(kind):transformer_quant_decline = dense_quant_unsupported_kind_reason(kind)transformer_quant_decline_status = RESOLVED_UNSUPPORTED- elif transformer_quant_pinned is not None and not dense_transformer_supported(- target+ elif (+ transformer_quant_pinned is not None+ and native_scheme is None+ and not dense_transformer_supported(target)):# Ask the helper rather than repeating its fallback: on ROCm and on the Windows# torchao stub it knows a truer reason, and an AMD owner reading "needs a CUDA# GPU" while holding a working GPU learns nothing about why it declined.transformer_quant_decline = dense_transformer_unsupported_reason(target)transformer_quant_decline_status = RESOLVED_UNSUPPORTED- elif transformer_quant_pinned is not None and (- select_transformer_quant_scheme(+ elif (+ transformer_quant_pinned is not None+ and native_scheme is None+ and select_transformer_quant_scheme(target, transformer_quant_pinned, family = getattr(fam, "name", None))is None@@ -5399,7 +5420,7 @@ class DiffusionBackend:and kind == "pipeline"and transformer_quant_engaged is Noneand normalize_transformer_quant(transformer_quant) is not None- and dense_transformer_supported(target)+ and (dense_transformer_supported(target) or native_scheme is not None)):# from_pretrained widens a raw fp8/int8 checkpoint to bf16, erasing the one thing the blocker# below reads, so recover it from the shard header (Ideogram's loader stamps its own).@@ -5409,9 +5430,10 @@ class DiffusionBackend:if source_precision is not None:for _attr, denoiser in denoiser_modules(pipe):mark_source_precision(denoiser, source_precision)- pipeline_quant_blocker = pipeline_quant_uncompilable or dense_quant_blocker(- pipe- )+ # Weight-only runs bf16 arithmetic, so it has no compile to require.+ pipeline_quant_blocker = (+ None if native_scheme is not None else pipeline_quant_uncompilable+ ) or dense_quant_blocker(pipe)if pipeline_quant_blocker is not None:logger.info("diffusion.transformer_quant: skipped (%s)", pipeline_quant_blocker@@ -5422,7 +5444,7 @@ class DiffusionBackend:# Re-plan against the quantised steady size. The build peak remains bf16.bf16_plan = planif plan.offload_policy != OFFLOAD_NONE:- preview_scheme = select_transformer_quant_scheme(+ preview_scheme = native_scheme or select_transformer_quant_scheme(target, transformer_quant, family = getattr(fam, "name", None))# This in-memory rewrite needs no cache-space or hosted-checkpoint checks.@@ -5458,7 +5480,7 @@ class DiffusionBackend:plan.offload_policy,)plan = replanned- if plan.offload_policy != OFFLOAD_NONE:+ if plan.offload_policy != OFFLOAD_NONE and native_scheme is None:logger.info("diffusion.transformer_quant: skipped (the memory plan picked '%s' ""offload, which moves the transformer via Module.to())",@@ -5567,8 +5589,12 @@ class DiffusionBackend:# Effective speed: GGUF defaults to `default` (~2.2x, below the quant noise floor); dense stays# bit-identical `off`.effective_speed = resolve_speed_mode(speed_mode, is_gguf = kind == "gguf")- # A torchao-quantized dense transformer must be compiled (eager is ~30x slower, losing to GGUF)- if transformer_quant_engaged is not None and effective_speed == SPEED_OFF:+ # A torchao-quantized dense transformer must be compiled (eager is ~30x slower, losing to GGUF).+ if (+ transformer_quant_engaged is not None+ and native_scheme is None+ and effective_speed == SPEED_OFF+ ):logger.info("diffusion.transformer_quant: forcing speed_mode=default ""(quantized transformer must be compiled; eager is ~30x slower)"@@ -5719,7 +5745,11 @@ class DiffusionBackend:logger = logger,)self._raise_if_load_cancelled(_load_token)- if transformer_quant_engaged is not None and not speed_applied.get("compiled"):+ if (+ transformer_quant_engaged is not None+ and native_scheme is None+ and not speed_applied.get("compiled")+ ):# Compile could not engage: the quantized transformer runs eager, far slower than the GGUF it# replacedlogger.warning(@@ -5823,6 +5853,7 @@ class DiffusionBackend:"deferred" if speed_deferred else effective_speed,"quantized transformer requires compile"if transformer_quant_engaged is not None+ and native_scheme is Noneand normalize_speed_mode(speed_mode) in (None, SPEED_OFF)else "auto: exact eager for the first two images; ""the compile profile engages on the 3rd"@@ -5848,6 +5879,9 @@ class DiffusionBackend:))if transformer_quant_engaged is None+ else f"weight-only: {transformer_quant_engaged} weights, bf16 compute "+ "(torchao-free, a memory saving rather than a speed-up)"+ if native_scheme is not Noneelse f"seeded from the hosted checkpoint "f"{transformer_quant_artifact.split(':', 1)[1]}"if transformer_quant_artifact is not Nonediff --git a/studio/backend/core/inference/diffusion_memory.py b/studio/backend/core/inference/diffusion_memory.pyindex 4c0e2a806ac..598f60139e5 100644--- a/studio/backend/core/inference/diffusion_memory.py+++ b/studio/backend/core/inference/diffusion_memory.py@@ -1389,6 +1389,41 @@ def _pin_vision_embedding_device(module: Any) -> int:return patched+def install_group_offload_buffer_restore() -> bool:+ """diffusers stream group offload restores only parameters, leaving buffers (native int8 weights) on the GPU."""+ try:+ from diffusers.hooks import group_offloading as go+ except Exception: # noqa: BLE001 - no group offload in this diffusers+ return False+ group_cls = getattr(go, "ModuleGroup", None)+ original = getattr(group_cls, "_offload_to_memory", None)+ if original is None or getattr(original, "_unsloth_buffer_restore", False):+ return False++ @functools.wraps(original)+ def _offload_to_memory(self, *args: Any, **kwargs: Any) -> Any:+ out = original(self, *args, **kwargs)+ cpu_copies = getattr(self, "cpu_param_dict", None)+ if getattr(self, "stream", None) is None or not cpu_copies:+ return out+ restore_torchao = getattr(go, "_restore_torchao_tensor", None)+ is_torchao = getattr(go, "_is_torchao_tensor", None)+ for group_module in getattr(self, "modules", None) or ():+ for buffer in group_module.buffers():+ cpu = cpu_copies.get(buffer)+ if cpu is None:+ continue+ if callable(is_torchao) and callable(restore_torchao) and is_torchao(buffer):+ restore_torchao(buffer, cpu)+ else:+ buffer.data = cpu+ return out++ _offload_to_memory._unsloth_buffer_restore = True+ group_cls._offload_to_memory = _offload_to_memory+ return True++def _apply_group_offload(pipe: Any,device: str,@@ -1412,6 +1447,8 @@ def _apply_group_offload(import torchfrom diffusers.hooks import apply_group_offloading+ install_group_offload_buffer_restore()+# A dual-DiT pipeline (Ideogram 4) carries a second denoiser as large as the first, so stream every DiT and keep# only smaller companions resident.streamed: dict[str, Any] = {"transformer": transformer}@@ -1704,6 +1741,8 @@ def _apply_streaming_offload(pipe: Any, device: str, logger: Any) -> None:import torchfrom diffusers.hooks import apply_group_offloading+ install_group_offload_buffer_restore()+components = getattr(pipe, "components", {})if not isinstance(components, dict):raise RuntimeError("pipeline does not expose its components")diff --git a/studio/backend/core/inference/diffusion_native_quant.py b/studio/backend/core/inference/diffusion_native_quant.pynew file mode 100644index 00000000000..1b78d0e4903--- /dev/null+++ b/studio/backend/core/inference/diffusion_native_quant.py@@ -0,0 +1,126 @@+# SPDX-License-Identifier: AGPL-3.0-only+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0++"""Torchao-free weight-only int8 / fp8 for the diffusion transformer (ROCm, Windows torchao stub).++Plain buffers, unlike torchao tensors, survive the offload hooks' ``Module.to()``; torch imported lazily.+"""++from __future__ import annotations++from functools import lru_cache+from typing import Any, Callable, Optional++NATIVE_INT8 = "int8"+NATIVE_FP8 = "fp8"+_QMAX = {NATIVE_INT8: 127.0, NATIVE_FP8: 448.0}+++@lru_cache(maxsize = 1)+def native_linear_class():+ """``NativeWeightOnlyLinear``, defined on first use so this module imports torch-free."""+ import torch+ from torch import nn+ from torch.nn import functional as F++ class NativeWeightOnlyLinear(nn.Module):+ """A Linear whose weight is stored as int8 / fp8 with a per-output-row scale."""++ def __init__(self, linear: Any, scheme: str):+ super().__init__()+ if scheme not in _QMAX:+ raise ValueError(f"unsupported native scheme {scheme!r}")+ self.in_features = int(linear.in_features)+ self.out_features = int(linear.out_features)+ self.scheme = scheme+ self.compute_dtype = linear.weight.dtype+ weight = linear.weight.detach()+ with torch.no_grad():+ w = weight.float()+ scale = w.abs().amax(dim = 1, keepdim = True).clamp(min = 1e-12) / _QMAX[scheme]+ if scheme == NATIVE_INT8:+ wq = (w / scale).round_().clamp_(-127, 127).to(torch.int8)+ else:+ wq = (w / scale).to(torch.float8_e4m3fn)+ del w+ # Integer views: a module-wide ``.to(dtype)`` casts floating buffers, which would widen fp8 or round the scales.+ self.register_buffer("weight_q", wq.view(torch.uint8) if scheme == NATIVE_FP8 else wq)+ self.register_buffer(+ "weight_scale", scale.squeeze(1).to(torch.float32).contiguous().view(torch.int32)+ )+ self.bias = linear.bias++ def dequantized_weight(self, dtype: Any) -> Any:+ wq = (+ self.weight_q.view(torch.float8_e4m3fn)+ if self.scheme == NATIVE_FP8+ else self.weight_q+ )+ scale = self.weight_scale.view(torch.float32)+ return (wq.to(torch.float32) * scale[:, None]).to(dtype)++ @property+ def weight(self) -> Any:+ """The dense weight, rebuilt on each read. Code that reads a wrapped Linear's weight directly+ (PEFT's DoRA forward reads ``base_layer.weight``) keeps working; nothing holds it."""+ return self.dequantized_weight(self.compute_dtype)++ def forward(self, x: Any) -> Any:+ return F.linear(x, self.dequantized_weight(x.dtype), self.bias)++ def extra_repr(self) -> str:+ return (+ f"in_features={self.in_features}, out_features={self.out_features}, "+ f"scheme={self.scheme}, bias={self.bias is not None}"+ )++ return NativeWeightOnlyLinear+++def is_native_linear(module: Any) -> bool:+ return type(module).__name__ == "NativeWeightOnlyLinear" and hasattr(module, "weight_q")+++def is_native_quantised(module: Any) -> bool:+ """Whether any layer of ``module`` has been swapped for a native weight-only Linear."""+ try:+ return any(is_native_linear(sub) for sub in module.modules())+ except Exception: # noqa: BLE001 -- a probe must never replace the failure it describes+ return False+++def apply_native_weight_quant(+ transformer: Any,+ scheme: str,+ *,+ filter_fn: Callable[[Any, str], bool],+ logger: Any = None,+) -> int:+ """Swap every Linear ``filter_fn`` keeps for a native weight-only twin; returns layers swapped. One layer+ at a time; a failure part-way leaves a partial conversion the caller must refuse."""+ cls = native_linear_class()+ names = [name for name, mod in transformer.named_modules() if name and filter_fn(mod, name)]+ for name in names:+ parent_name, _, leaf = name.rpartition(".")+ parent = transformer.get_submodule(parent_name) if parent_name else transformer+ setattr(parent, leaf, cls(getattr(parent, leaf), scheme))+ if logger is not None:+ logger.info(+ "diffusion.transformer_quant: %s weight-only (torchao-free) on %d linears",+ scheme,+ len(names),+ )+ return len(names)+++def native_weight_error(linear: Any, scheme: str) -> Optional[float]:+ """Relative Frobenius error of one Linear's native reconstruction; for tests and probes."""+ try:+ import torch++ layer = native_linear_class()(linear, scheme)+ ref = linear.weight.detach().float()+ rec = layer.dequantized_weight(torch.float32)+ return float((rec - ref).norm() / ref.norm().clamp(min = 1e-12))+ except Exception: # noqa: BLE001+ return Nonediff --git a/studio/backend/core/inference/diffusion_speed.py b/studio/backend/core/inference/diffusion_speed.pyindex 97becc5c7ec..3af547b117a 100644--- a/studio/backend/core/inference/diffusion_speed.py+++ b/studio/backend/core/inference/diffusion_speed.py@@ -925,6 +925,15 @@ def _enable_fp16_accumulation(def _fuse_qkv(pipe: Any, logger: Any) -> bool:+ from .diffusion_native_quant import is_native_quantised++ # Fusing reads each projection's .weight, which a native layer dequantises: a dense to_qkv beside the int8 buffers.+ if any(is_native_quantised(dit) for dit in _denoiser_dits(pipe)):+ if logger is not None:+ logger.info(+ "diffusion.speed: fuse_qkv skipped (native quantised projections stay unfused)"+ )+ return False# Prefer the pipe-level fuse (covers every component); else fuse each denoiser DiT so a dual-DiT family fuses BOTH# experts.fn = getattr(pipe, "fuse_qkv_projections", None)diff --git a/studio/backend/core/inference/diffusion_transformer_quant.py b/studio/backend/core/inference/diffusion_transformer_quant.pyindex a68ebeb6fe1..9fa351f9ab6 100644--- a/studio/backend/core/inference/diffusion_transformer_quant.py+++ b/studio/backend/core/inference/diffusion_transformer_quant.py@@ -30,6 +30,7 @@ from typing import Any, Optional# stdlib-only module (no torch), so this stays inside the "imported lazily" promise above.from core._torchao_stub import is_stubbed, torch_is_rocm+from .diffusion_native_quant import is_native_linearfrom .diffusion_torchao_patches import install_torchao_int_mm_patch# Also runs in the spawned smoke-probe child, which imports this module and nothing else of the backend.@@ -508,6 +509,25 @@ _NARROW_STORED_DTYPES: dict[str, str] = {}+def _cached_snapshot_dir(repo_id: str) -> Any:+ """``repo_id``'s already-cached snapshot, Studio's live cache first (the loaders pass it as ``cache_dir``)."""+ from pathlib import Path++ from huggingface_hub import snapshot_download++ try:+ from utils.hf_cache_settings import active_hf_hub_cache+ roots = (active_hf_hub_cache(), None)+ except Exception: # noqa: BLE001 -- the smoke-probe child has no Studio settings+ roots = (None,)+ for cache_dir in roots:+ try:+ return Path(snapshot_download(repo_id, cache_dir = cache_dir, local_files_only = True))+ except Exception: # noqa: BLE001 -- not cached there+ continue+ return None++def stored_denoiser_precision(local_dir: Optional[str]) -> Optional[str]:"""The narrow precision a LOCAL snapshot's denoiser shards are stored at, or None.@@ -518,8 +538,8 @@ def stored_denoiser_precision(local_dir: Optional[str]) -> Optional[str]:is the only place left to look.Headers only, and only under a directory we already have: no download, no tensor read, and no- verdict without a local snapshot, which is the behaviour without this check. A hub id is not a- directory and answers None, so the caller may pass the staged snapshot or the load's own base."""+ verdict without a local snapshot, which is the behaviour without this check. A hub id resolves to+ its already-cached snapshot (``local_files_only``: an offline load stages nothing), else None."""if not local_dir:return Nonetry:@@ -528,6 +548,10 @@ def stored_denoiser_precision(local_dir: Optional[str]) -> Optional[str]:import safetensorsroot = Path(local_dir).expanduser()+ if not root.is_dir():+ root = _cached_snapshot_dir(str(local_dir))+ if root is None:+ return Nonefor attr in DENOISER_ATTRS:sub = root / attrif not sub.is_dir():@@ -590,10 +614,13 @@ def dense_quant_blocker(pipe: Any) -> Optional[str]:def transformer_is_quantised(module: Any) -> bool:- """Whether any Linear weight has been replaced by a torchao tensor subclass."""+ """Whether any Linear weight has been replaced by a torchao tensor subclass, or any Linear by a+ torchao-free weight-only twin (``diffusion_native_quant``)."""try:import torchfor sub in module.modules():+ if is_native_linear(sub):+ return Trueif not isinstance(sub, torch.nn.Linear):continueweight = getattr(sub, "weight", None)@@ -640,6 +667,40 @@ def dense_transformer_unsupported_reason(target: Any) -> str:return "this device cannot run a dense torchao quant (it needs a CUDA GPU in bf16)"+# ``auto`` never lands here: weight-only measured no speed win on gfx1151, only memory.+NATIVE_QUANT_SCHEMES = (TQ_INT8, TQ_FP8)+++def native_quant_host(target: Any) -> bool:+ """Whether ``target`` is a bf16 GPU whose torchao path is closed for a platform reason: a ROCm torch+ (the NVIDIA-shaped capability ladder misclassifies it, and gfx1151 refuses fp8 ``_scaled_mm``) or the+ Windows-ROCm torchao stub. False on every NVIDIA, MPS, XPU and CPU host, which keep the torchao path."""+ if getattr(target, "device", None) != "cuda":+ return False+ if not (torch_is_rocm() or is_stubbed("torchao")):+ return False+ try:+ import torch+ return getattr(target, "dtype", None) is torch.bfloat16+ except Exception:+ return False+++def native_quant_scheme(+ target: Any,+ requested: Optional[str],+ family: Optional[str] = None,+) -> Optional[str]:+ """The weight-only scheme an EXPLICIT ``requested`` runs as on a ``native_quant_host``, or None.+ Not in ``select_transformer_quant_scheme``: that also feeds hosted torchao prequant planners."""+ scheme = normalize_transformer_quant(requested)+ if scheme not in NATIVE_QUANT_SCHEMES or not native_quant_host(target):+ return None+ if _family_denied(family, scheme):+ return None+ return scheme++def select_transformer_quant_scheme(target: Any,requested: Optional[str],@@ -1348,7 +1409,12 @@ def quantize_transformer(Returns the scheme engaged, or None when disabled / unsupported / failed (caller loads GGUF).Best-effort: never raises for an unsupported environment (failure leaves it dense).``fast_accum`` (fp8 only) overrides the per-GPU-class accumulate choice: None auto-detects,- True/False force it."""+ True/False force it. On a ``native_quant_host`` an explicit int8 / fp8 runs weight-only without torchao."""+ native = native_quant_scheme(target, mode, family = family)+ if native is not None:+ return _quantize_native(+ pipe, native, family = family, min_features = min_features, logger = logger+ )scheme = select_transformer_quant_scheme(target, mode, family = family)if scheme is None:return None@@ -1389,6 +1455,34 @@ def quantize_transformer(return None+def _quantize_native(+ pipe: Any, scheme: str, *, family: Optional[str], min_features: int, logger: Any+) -> Optional[str]:+ """The torchao-free branch of ``quantize_transformer``: the same layer filter, weight-only."""+ transformer = getattr(pipe, "transformer", None)+ if transformer is None:+ return None+ try:+ from .diffusion_native_quant import apply_native_weight_quant++ # require_bf16 keeps the fp32 linears some DiTs deliberately hold (Wan, Hunyuan) at full precision.+ filter_fn = make_filter_fn(+ min_features,+ exclude_name_tokens = exclude_tokens_for_scheme(scheme, family) + ("lora_",),+ require_bf16 = True,+ )+ if not apply_native_weight_quant(transformer, scheme, filter_fn = filter_fn, logger = logger):+ return None+ try:+ transformer._unsloth_runtime_quant = scheme+ except Exception: # noqa: BLE001 - marker is best-effort+ pass+ return scheme+ except Exception as exc: # noqa: BLE001 - a partial swap is reported by transformer_is_quantised+ _warn(logger, f"{scheme} (weight-only)", exc)+ return None++def _warn(logger: Any, what: str, exc: Exception) -> None:if logger is not None:logger.warning("diffusion.transformer_quant: %s failed: %s", what, exc)diff --git a/studio/backend/core/inference/video.py b/studio/backend/core/inference/video.pyindex 79045d828e1..3af575b20ad 100644--- a/studio/backend/core/inference/video.py+++ b/studio/backend/core/inference/video.py@@ -113,9 +113,15 @@ from .diffusion_transformer_quant import (dense_transformer_supported,dense_transformer_unsupported_reason,explain_unusable_scheme,+ native_quant_host,+ native_quant_scheme,+ NATIVE_QUANT_SCHEMES,normalize_transformer_quant,quantize_transformer,+ mark_source_precision,select_transformer_quant_scheme,+ stored_denoiser_precision,+ transformer_is_quantised,)from .diffusion import _memory_request_forces_offloadfrom .diffusion_batched import is_oom_error@@ -290,6 +296,14 @@ def _assert_video_precision_for_target(f"the dense DiT quant applies to full-pipeline loads only, and this is a "f"'{model_kind}' load, which runs the precision its checkpoint carries")+ elif (+ pinned in NATIVE_QUANT_SCHEMES+ and not getattr(fam, "modular_workflow", None)+ and native_quant_host(target)+ ):+ # The modular workflow seeds torchao checkpoints, so it stays off the native path.+ if native_quant_scheme(target, pinned, family = getattr(fam, "name", None)) is None:+ reason = explain_unusable_scheme(getattr(fam, "name", None), pinned)elif not dense_transformer_supported(target):reason = dense_transformer_unsupported_reason(target)elif forces_offload:@@ -4110,12 +4124,15 @@ class VideoBackend:kind == "pipeline"and planned.offload_policy != "none"and normalize_transformer_quant(transformer_quant) is not None- and dense_transformer_supported(target)+ and (+ dense_transformer_supported(target)+ or native_quant_scheme(target, transformer_quant, family = fam.name) is not None+ )and components is not None):- scheme_preview = select_transformer_quant_scheme(+ scheme_preview = native_quant_scheme(target, transformer_quant, family = fam.name- )+ ) or select_transformer_quant_scheme(target, transformer_quant, family = fam.name)factor = _QUANT_STEADY_FACTOR.get(scheme_preview) if scheme_preview else Noneif factor is not None:quant_mib = int((components[0] * factor + companions_gb) * mib_per_gb)@@ -4292,6 +4309,11 @@ class VideoBackend:# Why the quant did not engage, in the caller's terms; threaded into `resolved`.transformer_quant_decline: Optional[str] = Nonetransformer_quant_decline_status = RESOLVED_FELL_BACK+ native_scheme = (+ native_quant_scheme(target, transformer_quant_pinned, family = fam.name)+ if kind == "pipeline"+ else None+ )# Auto quantises a video DiT only to keep it resident: with bf16 resident, int8 fails the default LPIPS bar.if (kind == "pipeline"@@ -4314,7 +4336,11 @@ class VideoBackend:f"'{kind}' load, which runs the precision its checkpoint carries")transformer_quant_decline_status = RESOLVED_UNSUPPORTED- elif transformer_quant_pinned is not None and not dense_transformer_supported(target):+ elif (+ transformer_quant_pinned is not None+ and native_scheme is None+ and not dense_transformer_supported(target)+ ):# Ask the helper rather than repeating its fallback: on ROCm and on the Windows torchao# stub it knows a truer reason, and an AMD owner reading "needs a CUDA GPU" while# holding a working GPU learns nothing about why it declined.@@ -4343,7 +4369,23 @@ class VideoBackend:elif (kind == "pipeline"and normalize_transformer_quant(transformer_quant) is not None- and dense_transformer_supported(target)+ and (dense_transformer_supported(target) or native_scheme is not None)+ and (source_precision := stored_denoiser_precision(_base_local_dir or repo_id))+ is not None+ ):+ # from_pretrained widened a narrow checkpoint to bf16; requantising compounds the loss.+ for view in views:+ mark_source_precision(view, source_precision)+ transformer_quant_decline = (+ f"its published weights are {source_precision} and were widened to bf16 on load, so "+ "quantising them again would compound that loss"+ )+ transformer_quant_decline_status = RESOLVED_UNSUPPORTED+ logger.info("video.transformer_quant: skipped (%s)", transformer_quant_decline)+ elif (+ kind == "pipeline"+ and normalize_transformer_quant(transformer_quant) is not None+ and (dense_transformer_supported(target) or native_scheme is not None)):engaged = []for view in views:@@ -4366,6 +4408,20 @@ class VideoBackend:f"transformer_quant={engaged[0]} engaged on only "f"{len(engaged)}/{len(views)} experts; retry without quant.")+ # A part-converted view is unusable even when the precision fallback is allowed.+ dirty = (+ []+ if engaged+ else [i for i, view in enumerate(views) if transformer_is_quantised(view)]+ )+ if dirty:+ del pipe+ clear_gpu_cache()+ raise RuntimeError(+ f"transformer_quant='{transformer_quant}' converted part of the denoiser and then "+ "failed, leaving it neither dense nor usable. Reload with Precision set to Off to "+ "run the checkpoint as-is."+ )if engaged:transformer_quant_engaged = engaged[0]elif transformer_quant_pinned is not None:@@ -4437,7 +4493,11 @@ class VideoBackend:)# A torchao-quantised DiT must be compiled (eager is ~30x slower), so force the regional profile when quant# engaged but speed was off.- if transformer_quant_engaged is not None and effective_speed == SPEED_OFF:+ if (+ transformer_quant_engaged is not None+ and native_scheme is None+ and effective_speed == SPEED_OFF+ ):logger.info("video.transformer_quant: forcing speed_mode=default ""(quantized transformer must be compiled; eager is ~30x slower)"@@ -4577,7 +4637,7 @@ class VideoBackend:speed_mode,effective_speed,"quantized transformer requires compile"- if transformer_quant_engaged is not None+ if transformer_quant_engaged is not None and native_scheme is Noneelse "clip denoises amortise the one-time compile within a single run"if speed_mode is Noneelse "requested",@@ -4597,7 +4657,10 @@ class VideoBackend:transformer_quant_engaged or "off",# Honest framing: the shipped torchao schemes cut load time and resident memory ~2x, but# per-step GEMMs are at best bf16 parity.- "DiT(s) quantised (halves resident weights; hosted checkpoints cut "+ f"weight-only: {transformer_quant_engaged} weights, bf16 compute "+ "(torchao-free, a memory saving rather than a speed-up)"+ if transformer_quant_engaged is not None and native_scheme is not None+ else "DiT(s) quantised (halves resident weights; hosted checkpoints cut ""load time; per-step speed is roughly bf16 parity)"if transformer_quant_engaged is not Noneelse (diff --git a/studio/backend/tests/test_diffusion_backend.py b/studio/backend/tests/test_diffusion_backend.pyindex 0dd91819f9f..4b5a2590a26 100644--- a/studio/backend/tests/test_diffusion_backend.py+++ b/studio/backend/tests/test_diffusion_backend.py@@ -12067,6 +12067,94 @@ def test_a_prequant_repo_missing_its_artifact_marks_the_plan_incomplete(monkeypaassert "prequant artifact missing" in str(failures[0])+def _stub_amd_weight_only_host(backend, monkeypatch):+ """ROCm / torchao-stub host; records the quantise calls."""+ from core.inference import diffusion as dmod+ from core.inference import diffusion_transformer_quant as tq++ calls: list = []+ _force_cuda_target(backend, monkeypatch)+ monkeypatch.setattr(dmod, "dense_transformer_supported", lambda target: False)+ monkeypatch.setattr(tq, "dense_transformer_supported", lambda target: False)+ monkeypatch.setattr(dmod, "native_quant_host", lambda target: True)+ monkeypatch.setattr(tq, "native_quant_host", lambda target: True)+ real_init = _FakePipe.__init__++ def _init(self):+ real_init(self)+ self.transformer = _FakeDenoiser()++ monkeypatch.setattr(_FakePipe, "__init__", _init)++ def _quantize(+ pipe,+ target,+ *,+ mode,+ family = None,+ **kwargs,+ ):+ scheme = tq.native_quant_scheme(target, mode, family = family)+ if scheme is None:+ raise AssertionError("the torchao path must not run on an AMD weight-only host")+ calls.append({"module": pipe.transformer, "scheme": scheme})+ return scheme++ monkeypatch.setattr(dmod, "quantize_transformer", _quantize)+ monkeypatch.setattr(dmod, "transformer_is_quantised", lambda module: bool(calls))+ return calls++[email protected]("scheme", ["int8", "fp8"])+def test_an_explicit_scheme_on_amd_runs_weight_only(fake_runtime, tmp_path, monkeypatch, scheme):+ backend = DiffusionBackend()+ calls = _stub_amd_weight_only_host(backend, monkeypatch)+ status = backend.load_pipeline(+ "Qwen/Qwen-Image-2512",+ model_kind = "pipeline",+ transformer_quant = scheme,+ _base_local_dir = str(tmp_path),+ )+ assert [c["scheme"] for c in calls] == [scheme]+ assert calls[0]["module"] is backend._state.pipe.transformer+ assert status["transformer_quant"] == scheme+ resolved = status["resolved"]["transformer_quant"]+ assert resolved["value"] == scheme+ assert resolved["source"] == "explicit" and resolved["status"] == "applied"+ assert "weight-only" in resolved["reason"] and "bf16 compute" in resolved["reason"]+ assert "requires compile" not in status["resolved"]["speed_mode"]["reason"]+ backend.unload()+++def test_auto_on_amd_stays_bf16(fake_runtime, tmp_path, monkeypatch):+ backend = DiffusionBackend()+ calls = _stub_amd_weight_only_host(backend, monkeypatch)+ status = backend.load_pipeline(+ "Qwen/Qwen-Image-2512", model_kind = "pipeline", _base_local_dir = str(tmp_path)+ )+ assert calls == []+ assert status["transformer_quant"] is None+ assert status["resolved"]["transformer_quant"]["value"] == "off"+ backend.unload()+++def test_an_explicit_scheme_on_nvidia_keeps_the_torchao_path_and_its_wording(+ fake_runtime, tmp_path, monkeypatch+):+ backend = DiffusionBackend()+ calls = _stub_pipeline_dense_quant(backend, monkeypatch)+ status = backend.load_pipeline(+ "Qwen/Qwen-Image-2512",+ model_kind = "pipeline",+ transformer_quant = "fp8",+ _base_local_dir = str(tmp_path),+ )+ assert len(calls) == 1+ assert status["transformer_quant"] == "fp8"+ assert "weight-only" not in status["resolved"]["transformer_quant"]["reason"]+ backend.unload()++def _record_step_cache(monkeypatch,*,diff --git a/studio/backend/tests/test_diffusion_native_quant.py b/studio/backend/tests/test_diffusion_native_quant.pynew file mode 100644index 00000000000..013f768715e--- /dev/null+++ b/studio/backend/tests/test_diffusion_native_quant.py@@ -0,0 +1,554 @@+# SPDX-License-Identifier: AGPL-3.0-only+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0++"""Torchao-free weight-only int8 / fp8; ROCm simulated by patching ``torch_is_rocm`` / ``is_stubbed``."""++from __future__ import annotations++import sys+import types++import pytest++torch = pytest.importorskip("torch")++import core.inference.diffusion_native_quant as nq+import core.inference.diffusion_transformer_quant as tq+++def _target(device = "cuda", dtype = None):+ return types.SimpleNamespace(device = device, dtype = torch.bfloat16 if dtype is None else dtype)+++def rocm(monkeypatch):+ monkeypatch.setattr(tq, "torch_is_rocm", lambda: True)+ monkeypatch.setattr(tq, "is_stubbed", lambda name: False)+++def win_stub(monkeypatch):+ monkeypatch.setattr(tq, "torch_is_rocm", lambda: False)+ monkeypatch.setattr(tq, "is_stubbed", lambda name: name == "torchao")+++def nvidia(monkeypatch):+ monkeypatch.setattr(tq, "torch_is_rocm", lambda: False)+ monkeypatch.setattr(tq, "is_stubbed", lambda name: False)+++class _Block(torch.nn.Module):+ def __init__(self, dim):+ super().__init__()+ self.to_q = torch.nn.Linear(dim, dim)+ self.ff = torch.nn.Linear(dim, 2 * dim, bias = False)+ self.out = torch.nn.Linear(2 * dim, dim)+ self.small = torch.nn.Linear(dim, 8)++ def forward(self, x):+ return self.out(torch.nn.functional.gelu(self.ff(self.to_q(x)))) + self.small(x).sum(+ -1, keepdim = True+ )+++class _Toy(torch.nn.Module):+ def __init__(+ self,+ dim = 128,+ depth = 2,+ ):+ super().__init__()+ self.blocks = torch.nn.ModuleList(_Block(dim) for _ in range(depth))++ def forward(self, x):+ for b in self.blocks:+ x = x + b(x)+ return x+++def _toy(dtype = torch.bfloat16):+ torch.manual_seed(0)+ return _Toy().to(dtype)++[email protected]("scheme", ["int8", "fp8", "INT8", " fp8 "])+def test_rocm_and_stub_hosts_take_explicit_int8_and_fp8(rocm, scheme):+ assert tq.native_quant_host(_target())+ assert tq.native_quant_scheme(_target(), scheme) == scheme.strip().lower()+++def test_windows_stub_host_is_native(win_stub):+ assert tq.native_quant_host(_target())+ assert tq.native_quant_scheme(_target(), "int8") == "int8"++[email protected]("scheme", [None, "auto", "none", "off", "nvfp4", "mxfp8"])+def test_auto_and_other_schemes_never_go_native(rocm, scheme):+ assert tq.native_quant_scheme(_target(), scheme) is None+++ "target",+ [+ _target(device = "cpu"),+ _target(device = "mps"),+ _target(device = "xpu"),+ _target(dtype = torch.float16),+ _target(dtype = torch.float32),+ ],+)+def test_non_cuda_or_non_bf16_is_never_native_even_on_rocm(rocm, target):+ assert not tq.native_quant_host(target)+ assert tq.native_quant_scheme(target, "int8") is None++[email protected]("device", ["cuda", "cpu", "mps", "xpu"])+def test_nvidia_and_other_hosts_are_never_native(nvidia, device):+ assert not tq.native_quant_host(_target(device = device))+ for scheme in ("int8", "fp8", "nvfp4", "mxfp8", "auto"):+ assert tq.native_quant_scheme(_target(device = device), scheme) is None+++def test_family_deny_list_still_applies(rocm, monkeypatch):+ monkeypatch.setitem(tq._FAMILY_SCHEME_DENY, "toy-family", frozenset({tq.TQ_INT8}))+ assert tq.native_quant_scheme(_target(), "int8", family = "toy-family") is None+ assert tq.native_quant_scheme(_target(), "fp8", family = "toy-family") == "fp8"+ # The shipped deny list only fences nvfp4 / mxfp8 off qwen-image, so int8 / fp8 run there.+ assert tq.native_quant_scheme(_target(), "int8", family = "qwen-image") == "int8"+ assert tq.native_quant_scheme(_target(), "fp8", family = "qwen-image-2.1") == "fp8"+++def test_the_shared_selector_is_untouched_on_rocm(rocm):+ # Hosted prequant planners read this selector; AMD must never be handed a torchao checkpoint.+ for scheme in ("int8", "fp8", "auto"):+ assert tq.select_transformer_quant_scheme(_target(), scheme, unproven_ok = True) is None+ assert not tq.dense_transformer_supported(_target())++[email protected]("scheme,bound", [("int8", 0.01), ("fp8", 0.04)])+def test_native_layer_reconstructs_the_weight(scheme, bound):+ torch.manual_seed(0)+ lin = torch.nn.Linear(256, 512).to(torch.bfloat16)+ err = nq.native_weight_error(lin, scheme)+ assert err is not None and err < bound+ layer = nq.native_linear_class()(lin, scheme)+ assert layer.weight_q.dtype == (torch.int8 if scheme == "int8" else torch.uint8)+ assert layer.weight_scale.dtype == torch.int32 and layer.weight_scale.shape == (512,)+ assert layer.weight_q.element_size() == 1+ assert layer.bias is lin.bias+ x = torch.randn(4, 256, dtype = torch.bfloat16)+ ref = lin(x).float()+ got = layer(x).float()+ assert layer(x).dtype == torch.bfloat16+ assert ((got - ref).norm() / ref.norm()).item() < bound * 2+++def test_native_layer_rejects_an_unknown_scheme():+ with pytest.raises(ValueError):+ nq.native_linear_class()(torch.nn.Linear(8, 8), "nvfp4")+++def test_zero_rows_do_not_divide_by_zero():+ lin = torch.nn.Linear(16, 4, bias = False).to(torch.bfloat16)+ with torch.no_grad():+ lin.weight.zero_()+ for scheme in ("int8", "fp8"):+ layer = nq.native_linear_class()(lin, scheme)+ assert torch.isfinite(layer.dequantized_weight(torch.float32)).all()+ assert layer(torch.ones(2, 16, dtype = torch.bfloat16)).abs().max().item() == 0+++def test_native_buffers_move_with_module_to_and_state_dict():+ lin = torch.nn.Linear(64, 64).to(torch.bfloat16)+ layer = nq.native_linear_class()(lin, "int8")+ keys = set(layer.state_dict())+ assert {"weight_q", "weight_scale", "bias"} <= keys and "weight" not in keys+ layer.to("meta")+ assert layer.weight_q.device.type == "meta" and layer.weight_q.dtype == torch.int8+ assert layer.weight_scale.dtype == torch.int32++[email protected]("scheme", ["int8", "fp8"])+def test_a_module_wide_dtype_cast_leaves_the_stored_weights_alone(scheme):+ """A module ``.to(dtype)`` must not cast the integer-view payload or scales."""+ torch.manual_seed(0)+ lin = torch.nn.Linear(128, 64).to(torch.bfloat16)+ layer = nq.native_linear_class()(lin, scheme)+ x = torch.randn(3, 128, dtype = torch.bfloat16)+ before = layer(x).clone()+ q_before = layer.weight_q.clone()+ layer.to(torch.float16)+ assert layer.weight_q.dtype == q_before.dtype and torch.equal(layer.weight_q, q_before)+ assert layer.weight_scale.dtype == torch.int32+ assert torch.allclose(layer(x.to(torch.float16)).float(), before.float(), rtol = 1e-2, atol = 1e-2)+++def _block_torchao(monkeypatch):+ # A None entry makes any `import torchao.quantization` raise, proving the branch never reaches it.+ monkeypatch.setitem(sys.modules, "torchao", None)+ monkeypatch.setitem(sys.modules, "torchao.quantization", None)++[email protected]("scheme", ["int8", "fp8"])+def test_quantize_transformer_goes_native_on_rocm(rocm, monkeypatch, scheme):+ _block_torchao(monkeypatch)+ model = _toy()+ x = torch.randn(3, 128, dtype = torch.bfloat16)+ with torch.no_grad():+ ref = model(x).float()+ pipe = types.SimpleNamespace(transformer = model)+ logs = []+ logger = types.SimpleNamespace(+ info = lambda *a: logs.append(a), warning = lambda *a: pytest.fail(f"warned: {a}")+ )+ got = tq.quantize_transformer(pipe, _target(), mode = scheme, min_features = 64, logger = logger)+ assert got == scheme+ assert model._unsloth_runtime_quant == scheme+ swapped = {n for n, m in model.named_modules() if nq.is_native_linear(m)}+ assert swapped == {f"blocks.{i}.{n}" for i in range(2) for n in ("to_q", "ff", "out")}+ assert isinstance(model.blocks[0].small, torch.nn.Linear)+ assert tq.transformer_is_quantised(model) and nq.is_native_quantised(model)+ assert tq.dense_quant_blocker(pipe) is not None # a second pass would compound the loss+ assert any("torchao-free" in str(a[0]) for a in logs)+ with torch.no_grad():+ out = model(x).float()+ assert ((out - ref).norm() / ref.norm()).item() < 0.05+++def test_quantize_transformer_leaves_non_bf16_linears_dense(rocm, monkeypatch):+ _block_torchao(monkeypatch)+ model = _toy()+ model.blocks[1].ff.float() # a deliberately fp32 layer, as Wan and Hunyuan keep+ tq.quantize_transformer(+ types.SimpleNamespace(transformer = model), _target(), mode = "int8", min_features = 64+ )+ assert isinstance(model.blocks[1].ff, torch.nn.Linear)+ assert nq.is_native_linear(model.blocks[0].ff)+++def test_quantize_transformer_skips_lora_side_paths(rocm, monkeypatch):+ _block_torchao(monkeypatch)+ model = _toy()+ model.blocks[0].lora_A = torch.nn.Linear(128, 128).to(torch.bfloat16)+ tq.quantize_transformer(+ types.SimpleNamespace(transformer = model), _target(), mode = "fp8", min_features = 64+ )+ assert isinstance(model.blocks[0].lora_A, torch.nn.Linear)+++def test_quantize_transformer_auto_on_rocm_stays_dense(rocm, monkeypatch):+ _block_torchao(monkeypatch)+ model = _toy()+ got = tq.quantize_transformer(+ types.SimpleNamespace(transformer = model), _target(), mode = "auto", min_features = 64+ )+ assert got is None and not tq.transformer_is_quantised(model)+++def test_quantize_transformer_failure_is_reported_as_dirty(rocm, monkeypatch):+ model = _toy()+ real = nq.native_linear_class()+ calls = []++ def _boom(linear, scheme):+ calls.append(1)+ if len(calls) == 3:+ raise RuntimeError("out of memory")+ return real(linear, scheme)++ monkeypatch.setattr(nq, "native_linear_class", lambda: _boom)+ warned = []+ logger = types.SimpleNamespace(info = lambda *a: None, warning = lambda *a: warned.append(a))+ got = tq.quantize_transformer(+ types.SimpleNamespace(transformer = model),+ _target(),+ mode = "int8",+ min_features = 64,+ logger = logger,+ )+ assert got is None and warned+ assert tq.transformer_is_quantised(model)+ assert not getattr(model, "_unsloth_runtime_quant", None)+++def test_the_pass_does_not_hold_every_dense_layer_alive(rocm, monkeypatch):+ import weakref++ model = _toy()+ dense_ff = weakref.ref(model.blocks[0].ff)+ tq.quantize_transformer(+ types.SimpleNamespace(transformer = model), _target(), mode = "int8", min_features = 64+ )+ assert dense_ff() is None+++def test_quantize_transformer_on_nvidia_never_reaches_the_native_branch(nvidia, monkeypatch):+ monkeypatch.setattr(+ tq, "_quantize_native", lambda *a, **k: pytest.fail("native branch on an NVIDIA host")+ )+ seen = []+ monkeypatch.setattr(+ tq, "select_transformer_quant_scheme", lambda *a, **k: seen.append(a) or None+ )+ model = _toy()+ for scheme in ("int8", "fp8", "auto"):+ assert (+ tq.quantize_transformer(+ types.SimpleNamespace(transformer = model), _target(), mode = scheme+ )+ is None+ )+ assert len(seen) == 3++[email protected]("scheme", ["int8", "fp8"])+def test_reading_weight_gives_the_dense_weight_without_storing_it(scheme):+ """PEFT DoRA reads ``base_layer.weight``: dense, rebuilt per read, not in the state dict."""+ torch.manual_seed(0)+ lin = torch.nn.Linear(128, 64).to(torch.bfloat16)+ layer = nq.native_linear_class()(lin, scheme)+ weight = layer.weight+ assert weight.dtype == torch.bfloat16 and weight.shape == (64, 128)+ assert torch.equal(weight, layer.dequantized_weight(torch.bfloat16))+ assert "weight" not in set(layer.state_dict())+ assert all(p is not weight for p in layer.parameters())+++def test_a_dora_adapter_runs_on_a_native_base_layer(rocm, monkeypatch):+ peft = pytest.importorskip("peft")+ _block_torchao(monkeypatch)+ torch.manual_seed(0)+ model = _toy()+ cfg = peft.LoraConfig(r = 4, target_modules = ["to_q"], use_dora = True, init_lora_weights = False)+ wrapped = peft.inject_adapter_in_model(cfg, model)+ x = torch.randn(2, 128, dtype = torch.bfloat16)+ with torch.no_grad():+ ref = wrapped(x).float()+ tq.quantize_transformer(+ types.SimpleNamespace(transformer = wrapped), _target(), mode = "int8", min_features = 64+ )+ assert nq.is_native_linear(wrapped.blocks[0].to_q.base_layer)+ with torch.no_grad():+ out = wrapped(x).float()+ assert torch.isfinite(out).all()+ assert float((out - ref).norm() / ref.norm()) < 0.05+++def test_peft_wrapped_base_layer_runs_native(rocm, monkeypatch):+ peft = pytest.importorskip("peft")+ _block_torchao(monkeypatch)+ model = _toy()+ cfg = peft.LoraConfig(r = 4, target_modules = ["to_q"], lora_alpha = 4)+ wrapped = peft.inject_adapter_in_model(cfg, model)+ x = torch.randn(2, 128, dtype = torch.bfloat16)+ tq.quantize_transformer(+ types.SimpleNamespace(transformer = wrapped), _target(), mode = "int8", min_features = 64+ )+ to_q = wrapped.blocks[0].to_q+ assert nq.is_native_linear(to_q.base_layer)+ assert isinstance(to_q.lora_A["default"], torch.nn.Linear)+ with torch.no_grad():+ assert torch.isfinite(wrapped(x)).all()+++def _image_family(name):+ from core.inference.diffusion_families import _FAMILIES+ return next(f for f in _FAMILIES if f.name == name)+++def _video_family(name):+ from core.inference.video_families import _FAMILIES+ return next(f for f in _FAMILIES if f.name == name)+++def _image_gate(+ monkeypatch,+ fam,+ pinned,+ *,+ model_kind = "pipeline",+ memory_mode = None,+):+ import core.inference.diffusion as d+ monkeypatch.setattr(d, "effective_te_quant", lambda *a, **k: None)+ d.DiffusionBackend._assert_precision_for_target(+ None,+ fam,+ _target(),+ model_kind = model_kind,+ pinned = pinned,+ te_mode = None,+ memory_mode = memory_mode,+ cpu_offload = False,+ )+++def _video_gate(+ monkeypatch,+ fam,+ pinned,+ *,+ model_kind = "pipeline",+ memory_mode = None,+):+ import core.inference.video as v+ monkeypatch.setattr(v, "effective_te_quant", lambda *a, **k: None)+ v._assert_video_precision_for_target(+ fam, _target(), model_kind = model_kind, transformer_quant = pinned, memory_mode = memory_mode+ )++[email protected]("pinned", ["int8", "fp8"])[email protected]("memory_mode", [None, "balanced", "low_vram"])+def test_image_gate_admits_native_int8_fp8_on_rocm(rocm, monkeypatch, pinned, memory_mode):+ _image_gate(monkeypatch, _image_family("qwen-image-2.1"), pinned, memory_mode = memory_mode)++[email protected]("pinned", ["nvfp4", "mxfp8"])+def test_image_gate_still_refuses_torchao_only_schemes_on_rocm(rocm, monkeypatch, pinned):+ with pytest.raises(RuntimeError, match = "transformer_quant"):+ _image_gate(monkeypatch, _image_family("qwen-image-2.1"), pinned)++[email protected]("kind", ["gguf", "single_file"])+def test_image_gate_still_refuses_native_on_non_pipeline_loads(rocm, monkeypatch, kind):+ with pytest.raises(RuntimeError, match = "transformer_quant"):+ _image_gate(monkeypatch, _image_family("qwen-image-2.1"), "int8", model_kind = kind)+++def test_image_gate_refuses_a_unet_family_on_rocm(rocm, monkeypatch):+ with pytest.raises(RuntimeError, match = "transformer_quant"):+ _image_gate(monkeypatch, _image_family("sdxl"), "int8")+++def test_image_gate_refuses_a_denied_family_on_rocm(rocm, monkeypatch):+ monkeypatch.setitem(tq._FAMILY_SCHEME_DENY, "qwen-image-2.1", frozenset({tq.TQ_INT8}))+ with pytest.raises(RuntimeError, match = "transformer_quant"):+ _image_gate(monkeypatch, _image_family("qwen-image-2.1"), "int8")+ _image_gate(monkeypatch, _image_family("qwen-image-2.1"), "fp8")+++def test_image_gate_on_nvidia_never_consults_the_native_scheme(nvidia, monkeypatch):+ import core.inference.diffusion as d++ monkeypatch.setattr(d, "native_quant_scheme", lambda *a, **k: pytest.fail("native on NVIDIA"))+ monkeypatch.setattr(d, "dense_transformer_supported", lambda target: False)+ with pytest.raises(RuntimeError, match = "transformer_quant"):+ _image_gate(monkeypatch, _image_family("qwen-image-2.1"), "int8")++[email protected]("pinned", ["int8", "fp8"])[email protected]("memory_mode", [None, "balanced"])+def test_video_gate_admits_native_int8_fp8_on_rocm(rocm, monkeypatch, pinned, memory_mode):+ _video_gate(monkeypatch, _video_family("wan2.2-ti2v-5b"), pinned, memory_mode = memory_mode)+++def test_video_gate_keeps_the_modular_workflow_off_the_native_path(rocm, monkeypatch):+ with pytest.raises(RuntimeError, match = "transformer_quant"):+ _video_gate(monkeypatch, _video_family("minimax-h3"), "int8")++[email protected]("pinned", ["nvfp4", "mxfp8"])+def test_video_gate_still_refuses_torchao_only_schemes_on_rocm(rocm, monkeypatch, pinned):+ with pytest.raises(RuntimeError, match = "transformer_quant"):+ _video_gate(monkeypatch, _video_family("wan2.2-ti2v-5b"), pinned)+++def test_video_gate_refuses_native_on_gguf_loads(rocm, monkeypatch):+ with pytest.raises(RuntimeError, match = "transformer_quant"):+ _video_gate(monkeypatch, _video_family("wan2.2-ti2v-5b"), "int8", model_kind = "gguf")+++def _native_block(features = 64):+ lin = torch.nn.Linear(features, features, dtype = torch.bfloat16)+ return nq.native_linear_class()(lin, "int8")+++def test_stream_group_offload_puts_native_weight_buffers_back_on_the_host():+ go = pytest.importorskip("diffusers.hooks.group_offloading")+ from core.inference.diffusion_memory import install_group_offload_buffer_restore++ install_group_offload_buffer_restore()+ layer = _native_block()+ host = {b: b.data for b in layer.buffers()}+ host.update({p: p.data for p in layer.parameters()})+ group = object.__new__(go.ModuleGroup)+ group.modules, group.parameters, group.buffers = [layer], [], []+ group.stream, group.record_stream = object(), True+ group.cpu_param_dict = dict(host)+ for tensor in list(layer.buffers()) + list(layer.parameters()):+ tensor.data = tensor.data.clone() # stands in for the onloaded device copy+ group._offload_to_memory()+ assert all(b.data_ptr() == host[b].data_ptr() for b in layer.buffers())+ assert all(p.data_ptr() == host[p].data_ptr() for p in layer.parameters())+++def test_buffer_restore_install_is_idempotent():+ go = pytest.importorskip("diffusers.hooks.group_offloading")+ from core.inference.diffusion_memory import install_group_offload_buffer_restore++ install_group_offload_buffer_restore()+ patched = go.ModuleGroup._offload_to_memory+ assert install_group_offload_buffer_restore() is False+ assert go.ModuleGroup._offload_to_memory is patched+ assert getattr(patched, "__wrapped__", None) is not None+++ not torch.cuda.is_available(), reason = "stream group offload needs a CUDA device"+)+def test_native_int8_under_real_stream_group_offload_leaves_nothing_resident():+ pytest.importorskip("diffusers.hooks.group_offloading")+ from diffusers.hooks import apply_group_offloading++ from core.inference.diffusion_memory import install_group_offload_buffer_restore++ install_group_offload_buffer_restore()++ class Model(torch.nn.Module):+ def __init__(self):+ super().__init__()+ self.blocks = torch.nn.ModuleList([_native_block(), _native_block()])++ def forward(self, x):+ for block in self.blocks:+ x = block(x)+ return x++ model = Model()+ apply_group_offloading(+ model,+ onload_device = torch.device("cuda"),+ offload_device = torch.device("cpu"),+ offload_type = "block_level",+ num_blocks_per_group = 1,+ use_stream = True,+ non_blocking = True,+ record_stream = True,+ )+ with torch.inference_mode():+ out = model(torch.randn(32, 64, device = "cuda", dtype = torch.bfloat16))+ torch.cuda.synchronize()+ assert out.device.type == "cuda"+ assert {b.device.type for b in model.buffers()} == {"cpu"}+++def test_qkv_fusion_skips_a_native_quantised_denoiser():+ from core.inference import diffusion_speed++ calls = []++ class _Dit(torch.nn.Module):+ def __init__(self, native):+ super().__init__()+ lin = torch.nn.Linear(64, 64).to(torch.bfloat16)+ self.to_q = nq.native_linear_class()(lin, "int8") if native else lin++ def fuse_qkv_projections(self):+ calls.append(self)++ native = types.SimpleNamespace(transformer = _Dit(native = True))+ assert diffusion_speed._fuse_qkv(native, None) is False and not calls+ dense = types.SimpleNamespace(transformer = _Dit(native = False))+ assert diffusion_speed._fuse_qkv(dense, None) is True and len(calls) == 1diff --git a/studio/backend/tests/test_diffusion_transformer_quant.py b/studio/backend/tests/test_diffusion_transformer_quant.pyindex 0c9524a0814..6308a8aae9a 100644--- a/studio/backend/tests/test_diffusion_transformer_quant.py+++ b/studio/backend/tests/test_diffusion_transformer_quant.py@@ -1862,6 +1862,39 @@ def test_no_snapshot_is_no_verdict(local_dir):assert tq.stored_denoiser_precision(local_dir) is None+def _cached_snapshot(cache, repo_id, dtype):+ snap = cache / f"models--{repo_id.replace('/', '--')}" / "snapshots" / "abc123"+ _write_shard(snap, "transformer", dtype)+ (cache / f"models--{repo_id.replace('/', '--')}" / "refs").mkdir()+ (cache / f"models--{repo_id.replace('/', '--')}" / "refs" / "main").write_text("abc123")+++def test_an_offline_hub_id_reads_its_cached_snapshot(tmp_path, monkeypatch):+ """An offline load passes a repo id (nothing staged); the cached snapshot still carries the headers."""+ import torch+ from huggingface_hub import constants++ monkeypatch.setattr(constants, "HF_HUB_CACHE", str(tmp_path))+ _cached_snapshot(tmp_path, "org/narrow-dit", torch.float8_e4m3fn)+ _cached_snapshot(tmp_path, "org/wide-dit", torch.bfloat16)+ assert tq.stored_denoiser_precision("org/narrow-dit") == "fp8"+ assert tq.stored_denoiser_precision("org/wide-dit") is None+ assert tq.stored_denoiser_precision("org/not-cached") is None+++def test_an_offline_hub_id_reads_studios_live_cache(tmp_path, monkeypatch):+ """Studio can move its cache at runtime; the loaders read from there, so the probe must too."""+ import torch+ from huggingface_hub import constants+ from utils import hf_cache_settings++ live = tmp_path / "live"+ monkeypatch.setattr(constants, "HF_HUB_CACHE", str(tmp_path / "import_time_default"))+ monkeypatch.setattr(hf_cache_settings, "active_hf_hub_cache", lambda: str(live))+ _cached_snapshot(live, "org/narrow-dit", torch.float8_e4m3fn)+ assert tq.stored_denoiser_precision("org/narrow-dit") == "fp8"++def test_a_recovered_source_precision_blocks_the_quant(tmp_path):"""The recovered marker feeds the same refusal the Ideogram loader's marker does."""import torchdiff --git a/studio/backend/tests/test_video_backend.py b/studio/backend/tests/test_video_backend.pyindex 648dc72baba..91d819879ba 100644--- a/studio/backend/tests/test_video_backend.py+++ b/studio/backend/tests/test_video_backend.py@@ -9864,6 +9864,123 @@ def test_the_boundary_marker_waits_out_a_busy_capture_lock(fake_runtime, monkeypassert at_decode.get("phase") == "decode"[email protected]("scheme", ["int8", "fp8"])+def test_an_explicit_video_scheme_on_amd_runs_weight_only_without_forcing_compile(+ fake_runtime, monkeypatch, scheme+):+ """An explicit int8 / fp8 video DiT engages the native branch on AMD, honouring speed=off."""+ import core.inference.video as video_mod+ from core.inference import diffusion_transformer_quant as tq++ monkeypatch.setattr(video_mod, "dense_transformer_supported", lambda target: False)+ monkeypatch.setattr(tq, "dense_transformer_supported", lambda target: False)+ monkeypatch.setattr(video_mod, "native_quant_host", lambda target: True)+ monkeypatch.setattr(tq, "native_quant_host", lambda target: True)+ calls: list = []++ def _quantize(+ view,+ target,+ *,+ mode,+ family = None,+ **kw,+ ):+ assert tq.native_quant_scheme(target, mode, family = family) == scheme+ calls.append(mode)+ return scheme++ monkeypatch.setattr(video_mod, "quantize_transformer", _quantize)+ backend = VideoBackend()+ status = backend.load_pipeline(+ "Wan-AI/Wan2.2-TI2V-5B-Diffusers",+ model_kind = "pipeline",+ transformer_quant = scheme,+ speed_mode = "off",+ )+ assert calls and set(calls) == {scheme}+ assert status["transformer_quant"] == scheme+ assert status["speed_mode"] == "off"+ assert "requires compile" not in status["resolved"]["speed_mode"]["reason"]+ assert "weight-only" in status["resolved"]["transformer_quant"]["reason"]+ backend.unload()++[email protected]("fallback", ["0", "1"])+def test_a_video_checkpoint_stored_narrow_is_not_quantised_again(+ fake_runtime, monkeypatch, fallback+):+ """A bf16-widened fp8 checkpoint is declined without calling the quantiser."""+ import core.inference.video as video_mod+ from core.inference import diffusion_transformer_quant as tq++ monkeypatch.setenv("UNSLOTH_DIFFUSION_ALLOW_PRECISION_FALLBACK", fallback)+ monkeypatch.setattr(video_mod, "dense_transformer_supported", lambda target: False)+ monkeypatch.setattr(video_mod, "native_quant_host", lambda target: True)+ monkeypatch.setattr(tq, "dense_transformer_supported", lambda target: False)+ monkeypatch.setattr(tq, "native_quant_host", lambda target: True)+ monkeypatch.setattr(video_mod, "stored_denoiser_precision", lambda local_dir: "fp8")++ def _quantize(view, target, **kw):+ raise AssertionError("a narrow-stored DiT must not be quantised again")++ monkeypatch.setattr(video_mod, "quantize_transformer", _quantize)+ backend = VideoBackend()+ if fallback == "0":+ with pytest.raises(RuntimeError, match = "widened to bf16"):+ backend.load_pipeline(+ "Wan-AI/Wan2.2-TI2V-5B-Diffusers", model_kind = "pipeline", transformer_quant = "int8"+ )+ return+ status = backend.load_pipeline(+ "Wan-AI/Wan2.2-TI2V-5B-Diffusers", model_kind = "pipeline", transformer_quant = "int8"+ )+ resolved = status["resolved"]["transformer_quant"]+ assert resolved["status"] == "unsupported" and "widened to bf16" in resolved["reason"]+ backend.unload()++[email protected]("fallback", ["0", "1"])+def test_a_partly_converted_video_denoiser_is_refused_even_with_the_fallback_allowed(+ fake_runtime, monkeypatch, fallback+):+ """A part-converted weight-only DiT is refused even with the precision fallback."""+ import core.inference.video as video_mod+ from core.inference import diffusion_transformer_quant as tq++ monkeypatch.setenv("UNSLOTH_DIFFUSION_ALLOW_PRECISION_FALLBACK", fallback)+ monkeypatch.setattr(video_mod, "dense_transformer_supported", lambda target: False)+ monkeypatch.setattr(video_mod, "native_quant_host", lambda target: True)+ monkeypatch.setattr(tq, "dense_transformer_supported", lambda target: False)+ monkeypatch.setattr(tq, "native_quant_host", lambda target: True)+ monkeypatch.setattr(video_mod, "quantize_transformer", lambda view, target, **kw: None)+ monkeypatch.setattr(video_mod, "transformer_is_quantised", lambda view: True)+ backend = VideoBackend()+ with pytest.raises(RuntimeError, match = "neither dense nor usable"):+ backend.load_pipeline(+ "Wan-AI/Wan2.2-TI2V-5B-Diffusers", model_kind = "pipeline", transformer_quant = "int8"+ )+ assert backend._state is None+++def test_a_clean_video_decline_under_the_fallback_still_loads_dense(fake_runtime, monkeypatch):+ import core.inference.video as video_mod+ from core.inference import diffusion_transformer_quant as tq++ monkeypatch.setenv("UNSLOTH_DIFFUSION_ALLOW_PRECISION_FALLBACK", "1")+ monkeypatch.setattr(video_mod, "dense_transformer_supported", lambda target: False)+ monkeypatch.setattr(video_mod, "native_quant_host", lambda target: True)+ monkeypatch.setattr(tq, "dense_transformer_supported", lambda target: False)+ monkeypatch.setattr(tq, "native_quant_host", lambda target: True)+ monkeypatch.setattr(video_mod, "quantize_transformer", lambda view, target, **kw: None)+ monkeypatch.setattr(video_mod, "transformer_is_quantised", lambda view: False)+ backend = VideoBackend()+ status = backend.load_pipeline(+ "Wan-AI/Wan2.2-TI2V-5B-Diffusers", model_kind = "pipeline", transformer_quant = "int8"+ )+ assert status["transformer_quant"] in (None, "none", "off")++def test_video_auto_below_max_names_an_uncacheable_dit(fake_runtime, monkeypatch):monkeypatch.setattr("core.inference.video.step_cache_supported", lambda pipe, logger = None: False