Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
Studio: int8 Qwen-Image streamed modulations through a fused GEMV (1.23x under a 16 GiB budget) (#16087)
Daniel Hanfca19f5f0a9c
diff --git a/studio/backend/core/inference/diffusion_compile_cache.py b/studio/backend/core/inference/diffusion_compile_cache.pyindex c7eb8022472..49550aa8efb 100644--- a/studio/backend/core/inference/diffusion_compile_cache.py+++ b/studio/backend/core/inference/diffusion_compile_cache.py@@ -226,6 +226,10 @@ _GRAPH_SWITCHES = {"UNSLOTH_DIFFUSION_INT8_ROTQUANT_UNLISTED": "auto","UNSLOTH_DIFFUSION_INT8_FUSED": "on","UNSLOTH_DIFFUSION_INT8_TEXT_STREAM": "on",+ "UNSLOTH_DIFFUSION_INT8_MODULATION": "on",+ # Keyed on the clamped count the reader uses (0 = none).+ "UNSLOTH_DIFFUSION_INT8_MOD_OUTLIERS": "count:8",+ "UNSLOTH_DIFFUSION_INT8_GEMV": "on","UNSLOTH_DIFFUSION_ZIMAGE_FUSED": "on","UNSLOTH_DIFFUSION_Q21_REAL_ROPE": "on","UNSLOTH_DIFFUSION_QWEN_REAL_ROPE": "on",@@ -258,6 +262,12 @@ def _switch_value(name: str, default: str) -> Optional[str]:except ValueError:return Nonereturn None if rows == default.partition(":")[2] else rows+ if default.startswith("count:"):+ try:+ count = str(max(0, min(16, int(raw))))+ except ValueError:+ return None+ return None if count == default.partition(":")[2] else countif default.startswith("raw"):return None if raw in _OFF_WORDS or raw == default.partition(":")[2] else rawvalue = "off" if raw in _OFF_WORDS else "on" if raw in _ON_WORDS else Nonediff --git a/studio/backend/core/inference/diffusion_families.py b/studio/backend/core/inference/diffusion_families.pyindex 139e1c70c8c..8b02cd2afce 100644--- a/studio/backend/core/inference/diffusion_families.py+++ b/studio/backend/core/inference/diffusion_families.py@@ -281,6 +281,8 @@ _QWENIMAGE_TEXT_STREAM_INT8 = ("to_add_out","txt_mlp",)+# Streamed-only int8 weight-only; W8A8 would also round the timestep row per row.+_QWENIMAGE_MOD_WEIGHT_ONLY = ("img_mod", "txt_mod")# Families whose activation ranges break specific schemes at the MODEL level (the smoke probe only proves the GEMM# runs). Measured with the 28-pair prequant accuracy gate on B200: qwen-image + mxfp8 does semantic damage at 1024px,# + nvfp4 is unusable (LPIPS 0.51). Neither has a known fix, so both stay denied and auto falls through to int8@@ -873,6 +875,7 @@ _FAMILIES: tuple[DiffusionFamily, ...] = (profile = FamilyProfile(int8_exclude_tokens = _QWENIMAGE_INT8_EXCLUDES,int8_runtime_quant_tokens = _QWENIMAGE_TEXT_STREAM_INT8,+ int8_streamed_weight_only_tokens = _QWENIMAGE_MOD_WEIGHT_ONLY,scheme_deny = _QWENIMAGE_SCHEME_DENY,train_scheme_deny = _QWENIMAGE_TRAIN_SCHEME_DENY,auto_prefer = _GATED_AUTO_PREFER,diff --git a/studio/backend/core/inference/diffusion_int8_gemv.py b/studio/backend/core/inference/diffusion_int8_gemv.pynew file mode 100644index 00000000000..985d84ce663--- /dev/null+++ b/studio/backend/core/inference/diffusion_int8_gemv.py@@ -0,0 +1,460 @@+# SPDX-License-Identifier: AGPL-3.0-only+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0++"""Fused int8 weight-only GEMV for small-M Linears (AdaLN modulations, M = batch).++torchao's weight-only linear materialises a bf16 copy of the whole weight per call; this reads the int8 codes and+scales directly with one float32 accumulate. Per-row and per-group symmetric scales only; anything else keeps+torchao's forward. Opaque to Inductor and M is the batch, so no recompiles or CUDA-graph re-captures.+"""++from __future__ import annotations++import os+import threading+from functools import lru_cache+from typing import Any, Optional++INT8_GEMV_ENV = "UNSLOTH_DIFFUSION_INT8_GEMV"+# Above this many rows the stock dequant + GEMM is faster.+MAX_ROWS = 4+MAX_OUTLIERS = 16+OUTLIER_IDX = "outlier_idx"+OUTLIER_WEIGHT = "outlier_weight"+_OP_NAMESPACE = "unsloth_studio"+_OP_NAME = "int8_wo_gemv"+# (BLOCK_N, BLOCK_K, warps, stages), first that launches wins; the first needs ~40 KB smem, the second fits any card.+_CONFIGS = ((128, 128, 8, 3), (64, 64, 4, 2))+_DEVICE_CFG: dict = {}+_LOCK = threading.Lock()+# Engagement evidence for benches.+COUNTS = {"gemv": 0, "fallback": 0}+# Resolved once by ``install``, outside any trace.+_OP_HANDLE: Any = None+++def gemv_enabled() -> bool:+ raw = (os.environ.get(INT8_GEMV_ENV) or "").strip().lower()+ return raw not in ("0", "off", "false", "no")+++@lru_cache(maxsize = 1)+def _kernel() -> Any:+ try:+ import triton+ import triton.language as tl+ except Exception: # noqa: BLE001 - no Triton: stock path+ return None++ @triton.jit+ def i8_wo_gemv(+ x_ptr,+ q_ptr,+ s_ptr,+ b_ptr,+ oi_ptr,+ ow_ptr,+ y_ptr,+ M,+ N,+ K,+ KO,+ stride_xm,+ stride_qn,+ stride_sn,+ stride_on,+ stride_ym,+ HAS_BIAS: tl.constexpr,+ HAS_OUT: tl.constexpr,+ GROUPED: tl.constexpr,+ BLOCK_M: tl.constexpr,+ BLOCK_N: tl.constexpr,+ BLOCK_K: tl.constexpr,+ ):+ pid = tl.program_id(0)+ rn = pid * BLOCK_N + tl.arange(0, BLOCK_N)+ rm = tl.arange(0, BLOCK_M)+ n_ok = rn < N+ m_ok = rm < M+ acc = tl.zeros((BLOCK_M, BLOCK_N), dtype = tl.float32)+ for k0 in range(0, K, BLOCK_K):+ rk = k0 + tl.arange(0, BLOCK_K)+ q = tl.load(q_ptr + rn[:, None] * stride_qn + rk[None, :], mask = n_ok[:, None], other = 0)+ x = tl.load(+ x_ptr + rm[:, None] * stride_xm + rk[None, :], mask = m_ok[:, None], other = 0.0+ )+ # int8 -> bf16 is exact and bf16 x bf16 products are exact in the float32 accumulator+ part = tl.dot(x, tl.trans(q.to(tl.bfloat16)), out_dtype = tl.float32)+ if GROUPED:+ s = tl.load(s_ptr + rn * stride_sn + k0 // BLOCK_K, mask = n_ok, other = 0.0)+ acc += part * s.to(tl.float32)[None, :]+ else:+ acc += part+ if not GROUPED:+ s = tl.load(s_ptr + rn * stride_sn, mask = n_ok, other = 0.0)+ acc = acc * s.to(tl.float32)[None, :]+ if HAS_OUT:+ # bf16 outlier columns (their int8 codes are zero): x[:, idx] @ W_out.T, same accumulator+ ro = tl.arange(0, 16)+ o_ok = ro < KO+ idx = tl.load(oi_ptr + ro, mask = o_ok, other = 0)+ xo = tl.load(+ x_ptr + rm[:, None] * stride_xm + idx[None, :],+ mask = m_ok[:, None] & o_ok[None, :],+ other = 0.0,+ )+ wo = tl.load(+ ow_ptr + rn[:, None] * stride_on + ro[None, :],+ mask = n_ok[:, None] & o_ok[None, :],+ other = 0.0,+ )+ acc += tl.dot(xo, tl.trans(wo), out_dtype = tl.float32)+ if HAS_BIAS:+ b = tl.load(b_ptr + rn, mask = n_ok, other = 0.0)+ acc = acc + b.to(tl.float32)[None, :]+ tl.store(+ y_ptr + rm[:, None] * stride_ym + rn[None, :],+ acc.to(tl.bfloat16),+ mask = m_ok[:, None] & n_ok[None, :],+ )++ return triton, i8_wo_gemv+++def reference(+ x: Any,+ q: Any,+ s: Any,+ bias: Any,+ oidx: Any = None,+ ow: Any = None,+) -> Any:+ """Same math in torch (float32 accumulate, one bf16 rounding): the CPU / no-Triton body of the op."""+ import torch++ n, k = q.shape+ groups = s.numel() // n+ xf = x.float()+ if groups == 1:+ y = (xf @ q.float().t()) * s.reshape(1, n).float()+ else:+ g = k // groups+ parts = torch.einsum(+ "mgk,ngk->mng", xf.reshape(-1, groups, g), q.float().reshape(n, groups, g)+ )+ y = (parts * s.reshape(1, n, groups).float()).sum(-1)+ if oidx is not None:+ y = y + xf.index_select(-1, oidx.long()) @ ow.float().t()+ if bias is not None:+ y = y + bias.float()+ return y.to(torch.bfloat16)+++def _launch(x: Any, q: Any, s2: Any, bias: Any, oidx: Any, ow: Any, cfg: tuple) -> Any:+ import torch++ triton, fn = _kernel()+ block_n, block_k, warps, stages = cfg+ m, k = x.shape+ n, groups = s2.shape+ if groups > 1:+ block_k = k // groups+ ko = 0 if oidx is None else int(oidx.numel())+ y = torch.empty((m, n), device = x.device, dtype = torch.bfloat16)+ fn[(triton.cdiv(n, block_n),)](+ x,+ q,+ s2,+ bias if bias is not None else y,+ oidx if ko else y,+ ow if ko else y,+ y,+ m,+ n,+ k,+ ko,+ x.stride(0),+ q.stride(0),+ s2.stride(0),+ ow.stride(0) if ko else 0,+ y.stride(0),+ HAS_BIAS = bias is not None,+ HAS_OUT = ko > 0,+ GROUPED = groups > 1,+ # tl.dot wants >= 16 rows: the masked pad rows load zeros and are never stored+ BLOCK_M = 16,+ BLOCK_N = block_n,+ BLOCK_K = block_k,+ num_warps = warps,+ num_stages = stages,+ )+ return y+++def _run(+ x: Any,+ q: Any,+ s: Any,+ bias: Any,+ oidx: Any = None,+ ow: Any = None,+) -> Any:+ m, k = x.shape+ n = q.shape[0]+ groups = s.numel() // n+ block_k = k // groups if groups > 1 else _CONFIGS[-1][1]+ ko = 0 if oidx is None else int(oidx.numel())+ index = x.device.index if x.is_cuda else None+ start = _DEVICE_CFG.get(index, 0)+ if (+ index is None+ or _kernel() is None+ or start >= len(_CONFIGS)+ or k % block_k+ or (groups == 1 and k % _CONFIGS[start][1])+ or q.stride(-1) != 1+ or block_k & (block_k - 1)+ or block_k < 16+ or ko > MAX_OUTLIERS+ or (ko and ow.stride(-1) != 1)+ ):+ COUNTS["fallback"] += 1+ return reference(x, q, s, bias, oidx, ow)+ x = x if x.stride(-1) == 1 else x.contiguous()+ s2 = s.reshape(n, groups)+ if s2.stride(-1) != 1:+ s2 = s2.contiguous()+ for pos in range(start, len(_CONFIGS)):+ try:+ y = _launch(x, q, s2, bias, oidx, ow, _CONFIGS[pos])+ except Exception: # noqa: BLE001 - a tile this card cannot hold (shared memory): the next, smaller one+ continue+ _DEVICE_CFG[index] = pos+ COUNTS["gemv"] += 1+ return y+ _DEVICE_CFG[index] = len(_CONFIGS)+ COUNTS["fallback"] += 1+ return reference(x, q, s, bias, oidx, ow)+++@lru_cache(maxsize = 1)+def _op() -> Any:+ """``unsloth_studio::int8_wo_gemv`` (opaque to Inductor), or None."""+ try:+ import torch+ except Exception: # noqa: BLE001+ return None+ ns = getattr(torch.ops, _OP_NAMESPACE, None)+ if ns is not None and hasattr(ns, _OP_NAME):+ return getattr(ns, _OP_NAME)+ custom_op = getattr(getattr(torch, "library", None), "custom_op", None)+ if custom_op is None:+ return None+ try:++ @custom_op(+ f"{_OP_NAMESPACE}::{_OP_NAME}",+ mutates_args = (),+ schema = "(Tensor x, Tensor q, Tensor s, Tensor? bias, Tensor? oidx, Tensor? ow) -> Tensor",+ )+ def _int8_wo_gemv(x, q, s, bias, oidx, ow):+ return _run(x, q, s, bias, oidx, ow)++ @_int8_wo_gemv.register_fake+ def _(x, q, s, bias, oidx, ow):+ return x.new_empty((x.shape[0], q.shape[0]), dtype = torch.bfloat16)+ except Exception: # noqa: BLE001+ return None+ return getattr(torch.ops, _OP_NAMESPACE).int8_wo_gemv+++def weight_only_parts(w: Any) -> Optional[tuple]:+ """(int8 [N, K], scale [N, K / G]) of a symmetric weight-only torchao ``Int8Tensor`` (per row or per group)."""+ if type(w).__name__ != "Int8Tensor":+ return None+ if (+ getattr(w, "act_quant_kwargs", None) is not None+ or getattr(w, "act_pre_scale", None) is not None+ ):+ return None+ q, s = getattr(w, "qdata", None), getattr(w, "scale", None)+ block = list(getattr(w, "block_size", ()) or ())+ if q is None or s is None or q.dim() != 2 or len(block) != 2 or block[0] != 1:+ return None+ n, k = q.shape+ if k % block[1] or s.numel() != n * (k // block[1]):+ return None+ return q, s+++def _outliers(module: Any) -> tuple:+ return getattr(module, OUTLIER_IDX, None), getattr(module, OUTLIER_WEIGHT, None)+++def _forward(self: Any, x: Any) -> Any:+ import torch+ import torch.nn.functional as F++ w = self.weight+ oidx, ow = _outliers(self)+ parts = weight_only_parts(w)+ flat_rows = x.numel() // x.shape[-1] if x.shape[-1] else 0+ op = _OP_HANDLE+ if (+ op is None+ or parts is None+ or x.dtype != torch.bfloat16+ or not x.is_cuda+ or flat_rows == 0+ or flat_rows > MAX_ROWS+ or parts[0].device != x.device+ or (oidx is not None and (oidx.device != x.device or ow.device != x.device))+ ):+ # stock torchao weight-only forward, plus the dense outlier columns (exact: their codes are zero)+ y = F.linear(x, w, self.bias)+ if oidx is not None:+ y = y + F.linear(+ x.index_select(-1, oidx.to(x.device)), ow.to(device = x.device, dtype = x.dtype)+ )+ return y+ lead = x.shape[:-1]+ y = op(x.reshape(-1, x.shape[-1]), parts[0], parts[1], self.bias, oidx, ow)+ return y.reshape(*lead, y.shape[-1])+++@lru_cache(maxsize = None)+def gemv_linear_class() -> Any:+ """Built once: a fresh class per call would make torch.compile retrace every block."""+ from torch import nn++ class Int8GemvLinear(nn.Linear):+ forward = _forward++ return Int8GemvLinear+++def is_gemv_linear(module: Any) -> bool:+ return type(module).__name__ == "Int8GemvLinear"+++def split_outliers_(module: Any, channels: Any) -> int:+ """Before quantize_: move the weight columns of ``channels`` into a dense ``outlier_weight`` buffer and zero them+ in the weight, so ``x @ W.T == x @ W_rest.T + x[:, idx] @ W_out.T`` exactly while the int8 scales only see the+ rest. Returns how many columns moved (0 leaves the module untouched)."""+ import torch++ if channels is None or not len(channels) or len(channels) > MAX_OUTLIERS:+ return 0+ w = module.weight.data+ idx = torch.as_tensor(sorted(int(c) for c in channels), dtype = torch.int64, device = w.device)+ if int(idx.max()) >= w.shape[1]:+ return 0+ module.register_buffer(OUTLIER_WEIGHT, w.index_select(1, idx).contiguous())+ module.register_buffer(OUTLIER_IDX, idx)+ rest = w.clone()+ rest[:, idx] = 0+ module.weight.data = rest+ # swapped now, not at install: from here on only this forward adds the columns back (dense or quantized weight)+ module.__class__ = gemv_linear_class()+ return int(idx.numel())+++def prepare_outlier_buffers(transformer: Any, state_dict: Any) -> int:+ """Before ``load_state_dict(strict = True, assign = True)`` of an artifact built with outlier columns: give each+ named Linear placeholder buffers of the stored shapes and the GEMV forward, so the strict load assigns them and the+ forward adds them back. A key naming no Linear is left alone (the strict load then refuses the artifact)."""+ import torch+ from torch import nn++ suffix = "." + OUTLIER_WEIGHT+ count = 0+ for key in [k for k in state_dict if isinstance(k, str) and k.endswith(suffix)]:+ fqn = key[: -len(suffix)]+ idx = state_dict.get(f"{fqn}.{OUTLIER_IDX}")+ try:+ module = transformer.get_submodule(fqn)+ except AttributeError:+ continue+ if idx is None or not isinstance(module, nn.Linear):+ continue+ weight = state_dict[key]+ module.register_buffer(+ OUTLIER_WEIGHT, torch.empty(tuple(weight.shape), dtype = weight.dtype, device = "meta")+ )+ module.register_buffer(+ OUTLIER_IDX, torch.empty(tuple(idx.shape), dtype = idx.dtype, device = "meta")+ )+ module.__class__ = gemv_linear_class()+ count += 1+ return count+++def timestep_outlier_channels(transformer: Any, count: int) -> tuple:+ """The ``count`` input channels with the largest |SiLU(temb)| over the whole timestep range, from the model's own+ ``time_text_embed`` (Qwen-Image: the modulations' only input is SiLU of it, a function of the timestep alone, so+ this needs no prompts). () when the transformer has no such embedder or anything fails."""+ if count <= 0:+ return ()+ try:+ import torch++ tte = getattr(transformer, "time_text_embed", None)+ proj = getattr(tte, "time_proj", None)+ embedder = getattr(tte, "timestep_embedder", None)+ if proj is None or embedder is None:+ return ()+ weight = embedder.linear_1.weight+ if getattr(weight, "is_meta", False):+ return ()+ with torch.no_grad():+ sigma = torch.linspace(0.0, 1.0, 1001, device = weight.device)+ emb = embedder(proj(sigma).to(weight.dtype))+ extra = getattr(tte, "addition_t_embedding", None)+ if extra is not None:+ rows = extra.weight.to(emb.dtype)+ emb = torch.cat([emb + r for r in rows], dim = 0)+ score = torch.nn.functional.silu(emb.float()).abs().amax(dim = 0)+ if not bool(torch.isfinite(score).all()) or count > score.numel():+ return ()+ return tuple(sorted(int(i) for i in torch.topk(score, count).indices.tolist()))+ except Exception: # noqa: BLE001 - calibration is optional: no outliers, plain int8 codes+ return ()+++def install(+ transformer: Any,+ fqns: Any,+ *,+ logger: Any = None,+) -> int:+ """Swap the named weight-only Linears onto the GEMV forward (class swap, no hook); returns how many. A Linear+ carrying outlier columns is swapped even with the kernel off: its forward is what adds them back."""+ from torch import nn++ global _OP_HANDLE+ kernel = gemv_enabled() and _kernel() is not None and _op() is not None+ _OP_HANDLE = _op() if kernel else None+ modules = dict(transformer.named_modules())+ count = 0+ with _LOCK:+ for fqn in fqns:+ module = modules.get(fqn)+ if type(module) is not nn.Linear and not is_gemv_linear(module):+ continue+ if weight_only_parts(module.weight) is None or (+ not kernel and _outliers(module)[0] is None+ ):+ continue+ module.__class__ = gemv_linear_class()+ count += 1+ if logger is not None and count:+ logger.info(+ "diffusion.int8_gemv: %s on %d weight-only Linear(s)",+ "fused GEMV" if kernel else "outlier columns (stock GEMM)",+ count,+ )+ return count+++def call_count() -> int:+ return COUNTS["gemv"]diff --git a/studio/backend/core/inference/diffusion_prequant.py b/studio/backend/core/inference/diffusion_prequant.pyindex 3f59234ca54..a4c153d85df 100644--- a/studio/backend/core/inference/diffusion_prequant.py+++ b/studio/backend/core/inference/diffusion_prequant.py@@ -2344,6 +2344,7 @@ def load_prequantized_transformer(transformer = transformer_cls.from_config(config)if prepare_model is not None:prepare_model(transformer, metadata)+ _prepare_outlier_buffers(transformer, state_dict)transformer.load_state_dict(state_dict, strict = True, assign = True)if into is None and _has_meta_tensors(transformer):# Non-persistent buffers (built in __init__, absent from the state dict) stay on meta. Rebuild on CPU so@@ -2353,6 +2354,7 @@ def load_prequantized_transformer(# mismatches.if prepare_model is not None:prepare_model(transformer, metadata)+ _prepare_outlier_buffers(transformer, state_dict)transformer.load_state_dict(state_dict, strict = True, assign = True)# Online half of the rotation, applied here so no route skips it: after load_state_dict (meta retry rebuilds), before small-M padding reparents Linears.@@ -2406,6 +2408,7 @@ def load_prequantized_transformer(apply_small_m_padding,apply_zero_row_guard,quantize_runtime_leftovers,+ quantize_streamed_weight_only,)# Must run before padding (it reparents Linears); a failure keeps the artifact, leftovers stay dense.@@ -2421,6 +2424,14 @@ def load_prequantized_transformer()except Exception as exc: # noqa: BLE001 - optimisation only_warn(logger, f"{scheme}:runtime leftovers", exc)+ # Streamed scope only; a no-op on an artifact whose modulations are already weight-only. Never raises.+ quantize_streamed_weight_only(+ transformer,+ scheme,+ metadata.get("family") or family,+ min_features = int(metadata.get("min_features") or min_features or 512),+ logger = logger,+ )apply_small_m_padding(transformer, scheme, metadata.get("family"), logger = logger)apply_zero_row_guard(transformer, scheme, metadata.get("family"), logger = logger)@@ -2975,6 +2986,20 @@ def _validate_checkpoint(),)return False+ # Weight-only modulations are a streamed-build layout: refuse them where this runtime keeps them dense.+ ckpt_weight_only = meta.get("weight_only_name_tokens")+ if ckpt_weight_only:+ from .diffusion_transformer_quant import streamed_weight_only_tokens_for_scheme+ allowed = set(streamed_weight_only_tokens_for_scheme(scheme, meta.get("family")))+ if not isinstance(ckpt_weight_only, (list, tuple)) or not set(ckpt_weight_only) <= allowed:+ _warn(+ logger,+ scheme,+ ValueError(+ f"checkpoint stores {ckpt_weight_only!r} weight-only; this load keeps them dense"+ ),+ )+ return False# Verify recorded require_bf16 so a _REQUIRE_BF16_SCHEMES change can't load old-filter checkpoints.ckpt_require_bf16 = meta.get("require_bf16")if ckpt_require_bf16 is not None:@@ -3299,6 +3324,11 @@ class PrequantIntoError(RuntimeError):"""The module is neither dense nor fully loaded."""+def _prepare_outlier_buffers(transformer: Any, state_dict: dict) -> int:+ from .diffusion_int8_gemv import prepare_outlier_buffers+ return prepare_outlier_buffers(transformer, state_dict)++def _load_into_dense(transformer: Any, state_dict: dict, prepare_model: Optional[Any]) -> None:"""ValueError leaves the module untouched; PrequantIntoError means it was partly assigned."""if prepare_model is not None:@@ -3306,6 +3336,16 @@ def _load_into_dense(transformer: Any, state_dict: dict, prepare_model: Optional"prepare_model reshapes a fresh skeleton; it cannot run on an existing transformer")own = transformer.state_dict()+ from .diffusion_int8_gemv import OUTLIER_IDX, OUTLIER_WEIGHT++ # A streamed weight-only artifact also stores each modulation's dense outlier columns; the skeleton gets them below.+ outlier_keys = {+ k+ for k in state_dict+ if k.endswith((f".{OUTLIER_WEIGHT}", f".{OUTLIER_IDX}")) and k not in own+ }+ for name in outlier_keys:+ own[name] = state_dict[name]if set(own) != set(state_dict):missing = sorted(set(own) - set(state_dict))[:3]extra = sorted(set(state_dict) - set(own))[:3]@@ -3320,6 +3360,8 @@ def _load_into_dense(transformer: Any, state_dict: dict, prepare_model: Optional)del owntry:+ if outlier_keys:+ _prepare_outlier_buffers(transformer, state_dict)transformer.load_state_dict(state_dict, strict = True, assign = True)except Exception as exc: # noqa: BLE001raise PrequantIntoError(f"{type(exc).__name__}: {exc}") from excdiff --git a/studio/backend/core/inference/diffusion_prequant_cache.py b/studio/backend/core/inference/diffusion_prequant_cache.pyindex 9cd4bccf69f..f2c00116077 100644--- a/studio/backend/core/inference/diffusion_prequant_cache.py+++ b/studio/backend/core/inference/diffusion_prequant_cache.py@@ -354,9 +354,12 @@ def key_parts(divisible_for_scheme,exclude_tokens_for_scheme,int8_convrot_enabled,+ streamed_weight_only_layout,)group, suffixes = convrot_spec_for_scheme(scheme, family)+ # Only when set (streamed qwen-image int8), so every other build keeps its existing key.+ extra = streamed_weight_only_layout(scheme, family)return {"policy": POLICY_VERSION,"family": str(family or ""),@@ -373,6 +376,7 @@ def key_parts(if group and int8_convrot_enabled(family)else None,"fast_accum": _resolve_fast_accum(None) if scheme == TQ_FP8 else None,+ **extra,**_toolchain(),}@@ -1189,6 +1193,14 @@ def save_runtime_quant("skipped (fp8 quantized with a forced non-default accumulate mode)",)return None+ wo_keys = ("weight_only_name_tokens", "weight_only_mode", "weight_only_outliers")+ wanted_wo = {k: parts[k] for k in wo_keys if k in parts}+ if {k: info[k] for k in wo_keys if k in info} != wanted_wo:+ _log(logger, "info", "skipped (weight-only modulation state differs from the key)")+ return None+ if wanted_wo and wanted_wo["weight_only_mode"] != "wo":+ _log(logger, "info", "skipped (diagnostic W8A8 modulation mode)")+ return Noneif bool(info.get("convrot_fqns")) != bool(parts["convrot"]):_log(logger, "info", "skipped (ConvRot state differs from the key)")return None@@ -1266,6 +1278,9 @@ def save_runtime_quant(if scheme == "fp8":from .diffusion_transformer_quant import FP8_GRANULARITYmetadata["fp8_granularity"] = FP8_GRANULARITY+ if wanted_wo:+ metadata["weight_only_name_tokens"] = list(wanted_wo["weight_only_name_tokens"])+ metadata["weight_only_outliers"] = int(wanted_wo["weight_only_outliers"])metadata.update(_rotation(transformer, info))header[UNSLOTH_FORMAT_KEY] = prequant_format_for(metadata)header[UNSLOTH_METADATA_KEY] = json.dumps(metadata, default = str)diff --git a/studio/backend/core/inference/diffusion_transformer_quant.py b/studio/backend/core/inference/diffusion_transformer_quant.pyindex 39e9d2493d6..221a1630348 100644--- a/studio/backend/core/inference/diffusion_transformer_quant.py+++ b/studio/backend/core/inference/diffusion_transformer_quant.py@@ -109,6 +109,181 @@ def runtime_quant_tokens_for_scheme(scheme: str, family: Optional[str] = None) -return _INT8_FAMILY_RUNTIME_QUANT_TOKENS.get(str(family or "").strip().lower(), ())+_INT8_FAMILY_STREAMED_WO_TOKENS: dict[str, tuple[str, ...]] = _profile_table(+ "int8_streamed_weight_only_tokens", quant = True+)+# Kill switches (=0): UNSLOTH_DIFFUSION_INT8_MODULATION keeps modulations bf16, UNSLOTH_DIFFUSION_INT8_GEMV uses+# torchao's forward, UNSLOTH_DIFFUSION_INT8_MOD_OUTLIERS keeps no dense outlier columns.+INT8_MODULATION_ENV = "UNSLOTH_DIFFUSION_INT8_MODULATION"+++INT8_MOD_OUTLIERS_ENV = "UNSLOTH_DIFFUSION_INT8_MOD_OUTLIERS"+DEFAULT_MOD_OUTLIERS = 8+++def int8_modulation_mode() -> str:+ """``wo`` (default) int8 weight-only; ``w8a8`` dynamic int8 activations, row-padded; ``w8a8rot`` the same behind a+ ConvRot rotation (both diagnostic); ``off``."""+ raw = (_os.environ.get(INT8_MODULATION_ENV) or "").strip().lower()+ if raw in ("0", "off", "false", "no", "none"):+ return "off"+ return raw if raw in ("w8a8", "w8a8rot") else "wo"+++def int8_mod_outliers() -> int:+ """Timestep-embedding outlier channels the weight-only modulations keep dense (0 = none)."""+ raw = (_os.environ.get(INT8_MOD_OUTLIERS_ENV) or "").strip()+ try:+ return max(0, min(16, int(raw))) if raw else DEFAULT_MOD_OUTLIERS+ except ValueError:+ return DEFAULT_MOD_OUTLIERS+++def streamed_weight_only_tokens_for_scheme(+ scheme: str, family: Optional[str] = None+) -> tuple[str, ...]:+ """Name tokens a streamed int8 build stores weight-only; () outside ``int8_text_stream_scope(True)``."""+ if scheme != TQ_INT8 or not _TEXT_STREAM_SCOPE.get() or int8_modulation_mode() == "off":+ return ()+ return _INT8_FAMILY_STREAMED_WO_TOKENS.get(str(family or "").strip().lower(), ())+++def _weight_only_config() -> Any:+ """torchao int8 weight-only (Int8Tensor, per-row weight scale, no activation quant) or None on an old torchao."""+ from torchao.quantization import Int8WeightOnlyConfig++ if "version" not in _inspect.signature(Int8WeightOnlyConfig).parameters:+ return None+ from torchao.quantization import Int8Tensor # noqa: F401 - 0.14 has the config but no Int8Tensor++ return _quiet_config(Int8WeightOnlyConfig, version = 2)+++def streamed_weight_only_layout(scheme: str, family: Optional[str] = None) -> dict:+ """What a streamed build stores for the weight-only modulations (the prequant-cache key carries it): {} outside the+ scope, else the name tokens, the mode and the dense outlier-channel count."""+ tokens = streamed_weight_only_tokens_for_scheme(scheme, family)+ if not tokens:+ return {}+ mode = int8_modulation_mode()+ return {+ "weight_only_name_tokens": sorted(tokens),+ "weight_only_mode": mode,+ "weight_only_outliers": int8_mod_outliers() if mode == "wo" else 0,+ }+++def _modulation_fqns(+ transformer: Any, tokens: tuple, min_features: int, *, quantized: bool+) -> list:+ import torch++ from .diffusion_int8_gemv import weight_only_parts+ from .diffusion_quant_pad import is_quantized_linear++ out = []+ for name, module in transformer.named_modules():+ lname = name.lower()+ if (+ not isinstance(module, torch.nn.Linear)+ or "lora_" in lname+ or not any(t in lname for t in tokens)+ ):+ continue+ if module.in_features < min_features or module.out_features < min_features:+ continue+ if quantized:+ if weight_only_parts(module.weight) is not None:+ out.append(name)+ elif getattr(module.weight, "dtype", None) == torch.bfloat16 and not is_quantized_linear(+ module+ ):+ out.append(name)+ return out+++def quantize_streamed_weight_only(+ transformer: Any,+ scheme: str,+ family: Optional[str],+ *,+ min_features: int = DEFAULT_MIN_LINEAR_FEATURES,+ logger: Any = None,+) -> tuple[str, ...]:+ """Quantise the still-dense Linears named by ``streamed_weight_only_tokens_for_scheme``; returns their fqns.+ Weight-only (default): the timestep-embedding outlier channels stay dense beside the int8 codes and the fused GEMV+ runs them (``diffusion_int8_gemv``). An artifact loaded with them already weight-only only gets the GEMV forward.+ Best effort: any failure leaves them dense (the build is still valid, only bigger)."""+ layout = streamed_weight_only_layout(scheme, family)+ if not layout:+ return ()+ tokens = tuple(layout["weight_only_name_tokens"])+ mode = layout["weight_only_mode"]+ used = 0+ try:+ from torchao.quantization import quantize_++ from .diffusion_int8_gemv import install, split_outliers_, timestep_outlier_channels+ from .diffusion_quant_pad import wrap_small_m_linears++ names = _modulation_fqns(transformer, tokens, min_features, quantized = False)+ if not names:+ if mode == "wo":+ install(+ transformer,+ _modulation_fqns(transformer, tokens, min_features, quantized = True),+ logger = logger,+ )+ return ()+ config = _weight_only_config() if mode == "wo" else _make_quant_config(scheme)+ if config is None:+ return ()+ wanted = frozenset(names)+ outliers = (+ timestep_outlier_channels(transformer, layout["weight_only_outliers"])+ if mode == "wo"+ else ()+ )+ if outliers:+ modules = dict(transformer.named_modules())+ used = min(split_outliers_(modules[name], outliers) for name in names)+ if mode == "w8a8rot":+ from .diffusion_convrot import DEFAULT_CONVROT_GROUPSIZE, rotate_linears_+ rotate_linears_(+ transformer,+ [+ n+ for n in names+ if transformer.get_submodule(n).in_features % DEFAULT_CONVROT_GROUPSIZE == 0+ ],+ )+ quantize_(transformer, config, filter_fn = lambda _m, fqn = "": fqn in wanted)+ if mode == "wo":+ install(transformer, names, logger = logger)+ else:+ wrap_small_m_linears(transformer, names)+ except Exception as exc: # noqa: BLE001 - optimisation only: the rest of the build stands+ _warn(logger, f"{scheme}:streamed weight-only", exc)+ return ()+ try:+ transformer._unsloth_weight_only_layout = {**layout, "weight_only_outliers": used}+ except Exception: # noqa: BLE001 - marker is best-effort+ pass+ if logger is not None:+ logger.info(+ "diffusion.transformer_quant: %s %d streamed modulation Linear(s) (%s; %d dense outlier channels)",+ {+ "wo": "int8 weight-only",+ "w8a8": "int8 W8A8 padded",+ "w8a8rot": "int8 W8A8 ConvRot padded",+ }[mode],+ len(names),+ ", ".join(tokens),+ used,+ )+ return tuple(names)++# ALL-scheme PER-FAMILY exclusions (``FamilyProfile.quant_exclude_tokens``): unlike the int8 table above, these also# apply to fp8 / mxfp8 / nvfp4, whose scaled_mm paths otherwise quantise every eligible Linear._FAMILY_QUANT_EXCLUDE_NAME_TOKENS: dict[str, tuple[str, ...]] = _profile_table(@@ -1813,6 +1988,9 @@ def quantize_transformer(_make_quant_config(scheme, fast_accum = fast_accum),filter_fn = filter_fn,)+ weight_only = quantize_streamed_weight_only(+ transformer, scheme, family, min_features = min_features, logger = logger+ )# Not best-effort: a raise means quantized but not compilable, so the caller loads GGUF.apply_small_m_padding(transformer, scheme, family, logger = logger)apply_zero_row_guard(transformer, scheme, family, logger = logger)@@ -1831,6 +2009,10 @@ def quantize_transformer("convrot_fqns": list(rotated or ()),"dtype": str(getattr(target, "dtype", None) or _first_weight_dtype(transformer)),}+ if weight_only:+ transformer._unsloth_runtime_quant_info.update(+ getattr(transformer, "_unsloth_weight_only_layout", None) or {}+ )except Exception: # noqa: BLE001 - marker is best-effortpassreturn schemediff --git a/studio/backend/core/inference/family_profile.py b/studio/backend/core/inference/family_profile.pyindex 826eeb6204c..e4bb01aa369 100644--- a/studio/backend/core/inference/family_profile.py+++ b/studio/backend/core/inference/family_profile.py@@ -59,6 +59,8 @@ class FamilyProfile:int8_exclude_tokens: tuple[str, ...] = ()# Subset of ``int8_exclude_tokens`` a STREAMED build quantises anyway (``int8_text_stream_scope``).int8_runtime_quant_tokens: tuple[str, ...] = ()+ # Generic-excluded Linears a STREAMED build stores int8 weight-only.+ int8_streamed_weight_only_tokens: tuple[str, ...] = ()# Name tokens left dense for EVERY transformer-quant scheme (int8 / fp8 / mxfp8 / nvfp4), on top of the int8# tokens above. For layers whose quantisation error is measured to dominate the image delta regardless of the# scheme (FLUX.3: per-token modulations, the 20480-wide text projection, the text-stream per-mode blocks).diff --git a/studio/backend/tests/fixtures/diffusion_family_data_freeze.json b/studio/backend/tests/fixtures/diffusion_family_data_freeze.jsonindex a8bf5321baf..7f0bf92adef 100644--- a/studio/backend/tests/fixtures/diffusion_family_data_freeze.json+++ b/studio/backend/tests/fixtures/diffusion_family_data_freeze.json@@ -593,6 +593,7 @@"exclude": [],"pad": [],"runtime_quant": [],+ "streamed_weight_only": [],"zero_row": []},"auto_bf16_when_resident": null,@@ -607,6 +608,7 @@"exclude": [],"pad": [],"runtime_quant": [],+ "streamed_weight_only": [],"zero_row": []},"convrot_default_on": false,@@ -621,6 +623,7 @@"exclude": [],"pad": [],"runtime_quant": [],+ "streamed_weight_only": [],"zero_row": []},"int8": {@@ -643,6 +646,7 @@],"pad": [],"runtime_quant": [],+ "streamed_weight_only": [],"zero_row": []},"mxfp8": {@@ -656,6 +660,7 @@"exclude": [],"pad": [],"runtime_quant": [],+ "streamed_weight_only": [],"zero_row": []},"nvfp4": {@@ -669,6 +674,7 @@"exclude": [],"pad": [],"runtime_quant": [],+ "streamed_weight_only": [],"zero_row": []},"page": null,@@ -687,6 +693,7 @@"exclude": [],"pad": [],"runtime_quant": [],+ "streamed_weight_only": [],"zero_row": []},"auto_bf16_when_resident": null,@@ -701,6 +708,7 @@"exclude": [],"pad": [],"runtime_quant": [],+ "streamed_weight_only": [],"zero_row": []},"convrot_default_on": false,@@ -715,6 +723,7 @@"exclude": [],"pad": [],"runtime_quant": [],+ "streamed_weight_only": [],"zero_row": []},"int8": {@@ -749,6 +758,10 @@"to_add_out","txt_mlp"],+ "streamed_weight_only": [+ "img_mod",+ "txt_mod"+ ],"zero_row": []},"mxfp8": {@@ -762,6 +775,7 @@"exclude": [],"pad": [],"runtime_quant": [],+ "streamed_weight_only": [],"zero_row": []},"nvfp4": {@@ -775,6 +789,7 @@"exclude": [],"pad": [],"runtime_quant": [],+ "streamed_weight_only": [],"zero_row": []},"page": "image",@@ -858,6 +873,7 @@"exclude": [],"pad": [],"runtime_quant": [],+ "streamed_weight_only": [],"zero_row": []},"auto_bf16_when_resident": null,@@ -872,6 +888,7 @@"exclude": [],"pad": [],"runtime_quant": [],+ "streamed_weight_only": [],"zero_row": []},"convrot_default_on": false,@@ -886,6 +903,7 @@"exclude": [],"pad": [],"runtime_quant": [],+ "streamed_weight_only": [],"zero_row": []},"int8": {@@ -920,6 +938,10 @@"to_add_out","txt_mlp"],+ "streamed_weight_only": [+ "img_mod",+ "txt_mod"+ ],"zero_row": []},"mxfp8": {@@ -933,6 +955,7 @@"exclude": [],"pad": [],"runtime_quant": [],+ "streamed_weight_only": [],"zero_row": []},"nvfp4": {@@ -946,6 +969,7 @@"exclude": [],"pad": [],"runtime_quant": [],+ "streamed_weight_only": [],"zero_row": []},"page": "image",@@ -964,6 +988,7 @@"exclude": [],"pad": [],"runtime_quant": [],+ "streamed_weight_only": [],"zero_row": []},"auto_bf16_when_resident": null,@@ -978,6 +1003,7 @@"exclude": [],"pad": [],"runtime_quant": [],+ "streamed_weight_only": [],"zero_row": []},"convrot_default_on": false,@@ -992,6 +1018,7 @@"exclude": [],"pad": [],"runtime_quant": [],+ "streamed_weight_only": [],"zero_row": []},"int8": {@@ -1014,6 +1041,7 @@],"pad": [],"runtime_quant": [],+ "streamed_weight_only": [],"zero_row": []},"mxfp8": {@@ -1027,6 +1055,7 @@"exclude": [],"pad": [],"runtime_quant": [],+ "streamed_weight_only": [],"zero_row": []},"nvfp4": {@@ -1040,6 +1069,7 @@"exclude": [],"pad": [],"runtime_quant": [],+ "streamed_weight_only": [],"zero_row": []},"page": "video",@@ -1058,6 +1088,7 @@"exclude": [],"pad": [],"runtime_quant": [],+ "streamed_weight_only": [],"zero_row": []},"auto_bf16_when_resident": null,@@ -1072,6 +1103,7 @@"exclude": [],"pad": [],"runtime_quant": [],+ "streamed_weight_only": [],"zero_row": []},"convrot_default_on": false,@@ -1086,6 +1118,7 @@"exclude": [],"pad": [],"runtime_quant": [],+ "streamed_weight_only": [],"zero_row": []},"int8": {@@ -1108,6 +1141,7 @@],"pad": [],"runtime_quant": [],+ "streamed_weight_only": [],"zero_row": []},"mxfp8": {@@ -1121,6 +1155,7 @@"exclude": [],"pad": [],"runtime_quant": [],+ "streamed_weight_only": [],"zero_row": []},"nvfp4": {@@ -1134,6 +1169,7 @@"exclude": [],"pad": [],"runtime_quant": [],+ "streamed_weight_only": [],"zero_row": []},"page": "video",@@ -1152,6 +1188,7 @@"exclude": [],"pad": [],"runtime_quant": [],+ "streamed_weight_only": [],"zero_row": []},"auto_bf16_when_resident": null,@@ -1166,6 +1203,7 @@"exclude": [],"pad": [],"runtime_quant": [],+ "streamed_weight_only": [],"zero_row": []},"convrot_default_on": false,@@ -1180,6 +1218,7 @@"exclude": [],"pad": [],"runtime_quant": [],+ "streamed_weight_only": [],"zero_row": []},"int8": {@@ -1202,6 +1241,7 @@],"pad": [],"runtime_quant": [],+ "streamed_weight_only": [],"zero_row": []},"mxfp8": {@@ -1215,6 +1255,7 @@"exclude": [],"pad": [],"runtime_quant": [],+ "streamed_weight_only": [],"zero_row": []},"nvfp4": {@@ -1228,6 +1269,7 @@"exclude": [],"pad": [],"runtime_quant": [],+ "streamed_weight_only": [],"zero_row": []},"page": "image",@@ -1246,6 +1288,7 @@"exclude": [],"pad": [],"runtime_quant": [],+ "streamed_weight_only": [],"zero_row": []},"auto_bf16_when_resident": null,@@ -1260,6 +1303,7 @@"exclude": [],"pad": [],"runtime_quant": [],+ "streamed_weight_only": [],"zero_row": []},"convrot_default_on": false,@@ -1274,6 +1318,7 @@"exclude": [],"pad": [],"runtime_quant": [],+ "streamed_weight_only": [],"zero_row": []},"int8": {@@ -1296,6 +1341,7 @@],"pad": [],"runtime_quant": [],+ "streamed_weight_only": [],"zero_row": []},"mxfp8": {@@ -1309,6 +1355,7 @@"exclude": [],"pad": [],"runtime_quant": [],+ "streamed_weight_only": [],"zero_row": []},"nvfp4": {@@ -1322,6 +1369,7 @@"exclude": [],"pad": [],"runtime_quant": [],+ "streamed_weight_only": [],"zero_row": []},"page": "image",@@ -1340,6 +1388,7 @@"exclude": [],"pad": [],"runtime_quant": [],+ "streamed_weight_only": [],"zero_row": []},"auto_bf16_when_resident": null,@@ -1354,6 +1403,7 @@"exclude": [],"pad": [],"runtime_quant": [],+ "streamed_weight_only": [],"zero_row": []},"convrot_default_on": false,@@ -1368,6 +1418,7 @@"exclude": [],"pad": [],"runtime_quant": [],+ "streamed_weight_only": [],"zero_row": []},"int8": {@@ -1390,6 +1441,7 @@],"pad": [],"runtime_quant": [],+ "streamed_weight_only": [],"zero_row": []},"mxfp8": {@@ -1403,6 +1455,7 @@"exclude": [],"pad": [],"runtime_quant": [],+ "streamed_weight_only": [],"zero_row": []},"nvfp4": {@@ -1416,6 +1469,7 @@"exclude": [],"pad": [],"runtime_quant": [],+ "streamed_weight_only": [],"zero_row": []},"page": "image",@@ -1434,6 +1488,7 @@"exclude": [],"pad": [],"runtime_quant": [],+ "streamed_weight_only": [],"zero_row": []},"auto_bf16_when_resident": "a small denoiser stays bf16 while it fits (INT8 / FP8 change image detail: FP8 LPIPS 0.065 on FLUX.2-klein-4B, 0.103 on Z-Image-Turbo)",@@ -1448,6 +1503,7 @@"exclude": [],"pad": [],"runtime_quant": [],+ "streamed_weight_only": [],"zero_row": []},"convrot_default_on": false,@@ -1462,6 +1518,7 @@"exclude": [],"pad": [],"runtime_quant": [],+ "streamed_weight_only": [],"zero_row": []},"int8": {@@ -1484,6 +1541,7 @@],"pad": [],"runtime_quant": [],+ "streamed_weight_only": [],"zero_row": []},"mxfp8": {@@ -1497,6 +1555,7 @@"exclude": [],"pad": [],"runtime_quant": [],+ "streamed_weight_only": [],"zero_row": []},"nvfp4": {@@ -1510,6 +1569,7 @@"exclude": [],"pad": [],"runtime_quant": [],+ "streamed_weight_only": [],"zero_row": []},"page": "image",@@ -1534,6 +1594,7 @@],"pad": [],"runtime_quant": [],+ "streamed_weight_only": [],"zero_row": []},"auto_bf16_when_resident": "a small denoiser stays bf16 while it fits (INT8 / FP8 change image detail: FP8 LPIPS 0.065 on FLUX.2-klein-4B, 0.103 on Z-Image-Turbo)",@@ -1554,6 +1615,7 @@],"pad": [],"runtime_quant": [],+ "streamed_weight_only": [],"zero_row": []},"convrot_default_on": false,@@ -1574,6 +1636,7 @@],"pad": [],"runtime_quant": [],+ "streamed_weight_only": [],"zero_row": []},"int8": {@@ -1601,6 +1664,7 @@],"pad": [],"runtime_quant": [],+ "streamed_weight_only": [],"zero_row": []},"mxfp8": {@@ -1620,6 +1684,7 @@],"pad": [],"runtime_quant": [],+ "streamed_weight_only": [],"zero_row": []},"nvfp4": {@@ -1639,6 +1704,7 @@],"pad": [],"runtime_quant": [],+ "streamed_weight_only": [],"zero_row": []},"page": "image",@@ -1663,6 +1729,7 @@],"pad": [],"runtime_quant": [],+ "streamed_weight_only": [],"zero_row": []},"auto_bf16_when_resident": "a small denoiser stays bf16 while it fits (INT8 / FP8 change image detail: FP8 LPIPS 0.065 on FLUX.2-klein-4B, 0.103 on Z-Image-Turbo)",@@ -1683,6 +1750,7 @@],"pad": [],"runtime_quant": [],+ "streamed_weight_only": [],"zero_row": []},"convrot_default_on": false,@@ -1703,6 +1771,7 @@],"pad": [],"runtime_quant": [],+ "streamed_weight_only": [],"zero_row": []},"int8": {@@ -1730,6 +1799,7 @@],"pad": [],"runtime_quant": [],+ "streamed_weight_only": [],"zero_row": []},"mxfp8": {@@ -1749,6 +1819,7 @@],"pad": [],"runtime_quant": [],+ "streamed_weight_only": [],"zero_row": []},"nvfp4": {@@ -1768,6 +1839,7 @@],"pad": [],"runtime_quant": [],+ "streamed_weight_only": [],"zero_row": []},"page": "image",@@ -1786,6 +1858,7 @@"exclude": [],"pad": [],"runtime_quant": [],+ "streamed_weight_only": [],"zero_row": []},"auto_bf16_when_resident": "INT8 is barely faster than compiled bf16 on a card that holds bf16 and changes image detail",@@ -1800,6 +1873,7 @@"exclude": [],"pad": [],"runtime_quant": [],+ "streamed_weight_only": [],"zero_row": []},"convrot_default_on": false,@@ -1814,6 +1888,7 @@"exclude": [],"pad": [],"runtime_quant": [],+ "streamed_weight_only": [],"zero_row": []},"int8": {@@ -1836,6 +1911,7 @@],"pad": [],"runtime_quant": [],+ "streamed_weight_only": [],"zero_row": []},"mxfp8": {@@ -1849,6 +1925,7 @@"exclude": [],"pad": [],"runtime_quant": [],+ "streamed_weight_only": [],"zero_row": []},"nvfp4": {@@ -1862,6 +1939,7 @@"exclude": [],"pad": [],"runtime_quant": [],+ "streamed_weight_only": [],"zero_row": []},"page": "image",@@ -1880,6 +1958,7 @@"exclude": [],"pad": [],"runtime_quant": [],+ "streamed_weight_only": [],"zero_row": []},"auto_bf16_when_resident": null,@@ -1894,6 +1973,7 @@"exclude": [],"pad": [],"runtime_quant": [],+ "streamed_weight_only": [],"zero_row": []},"convrot_default_on": false,@@ -1908,6 +1988,7 @@"exclude": [],"pad": [],"runtime_quant": [],+ "streamed_weight_only": [],"zero_row": []},"int8": {@@ -1930,6 +2011,7 @@],"pad": [],"runtime_quant": [],+ "streamed_weight_only": [],"zero_row": []},"mxfp8": {@@ -1943,6 +2025,7 @@"exclude": [],"pad": [],"runtime_quant": [],+ "streamed_weight_only": [],"zero_row": []},"nvfp4": {@@ -1956,6 +2039,7 @@"exclude": [],"pad": [],"runtime_quant": [],+ "streamed_weight_only": [],"zero_row": []},"page": "image",@@ -1974,6 +2058,7 @@"exclude": [],"pad": [],"runtime_quant": [],+ "streamed_weight_only": [],"zero_row": []},"auto_bf16_when_resident": null,@@ -1988,6 +2073,7 @@"exclude": [],"pad": [],"runtime_quant": [],+ "streamed_weight_only": [],"zero_row": []},"convrot_default_on": false,@@ -2002,6 +2088,7 @@"exclude": [],"pad": [],"runtime_quant": [],+ "streamed_weight_only": [],"zero_row": []},"int8": {@@ -2031,6 +2118,7 @@],"pad": [],"runtime_quant": [],+ "streamed_weight_only": [],"zero_row": []},"mxfp8": {@@ -2044,6 +2132,7 @@"exclude": [],"pad": [],"runtime_quant": [],+ "streamed_weight_only": [],"zero_row": []},"nvfp4": {@@ -2057,6 +2146,7 @@"exclude": [],"pad": [],"runtime_quant": [],+ "streamed_weight_only": [],"zero_row": ["image_embedder","context_embedder_2"@@ -2078,6 +2168,7 @@"exclude": [],"pad": [],"runtime_quant": [],+ "streamed_weight_only": [],"zero_row": []},"auto_bf16_when_resident": null,@@ -2092,6 +2183,7 @@"exclude": [],"pad": [],"runtime_quant": [],+ "streamed_weight_only": [],"zero_row": []},"convrot_default_on": false,@@ -2106,6 +2198,7 @@"exclude": [],"pad": [],"runtime_quant": [],+ "streamed_weight_only": [],"zero_row": []},"int8": {@@ -2135,6 +2228,7 @@],"pad": [],"runtime_quant": [],+ "streamed_weight_only": [],"zero_row": []},"mxfp8": {@@ -2148,6 +2242,7 @@"exclude": [],"pad": [],"runtime_quant": [],+ "streamed_weight_only": [],"zero_row": []},"nvfp4": {@@ -2161,6 +2256,7 @@"exclude": [],"pad": [],"runtime_quant": [],+ "streamed_weight_only": [],"zero_row": ["image_embedder","context_embedder_2"@@ -2182,6 +2278,7 @@"exclude": [],"pad": [],"runtime_quant": [],+ "streamed_weight_only": [],"zero_row": []},"auto_bf16_when_resident": null,@@ -2196,6 +2293,7 @@"exclude": [],"pad": [],"runtime_quant": [],+ "streamed_weight_only": [],"zero_row": []},"convrot_default_on": false,@@ -2210,6 +2308,7 @@"exclude": [],"pad": [],"runtime_quant": [],+ "streamed_weight_only": [],"zero_row": []},"int8": {@@ -2232,6 +2331,7 @@],"pad": [],"runtime_quant": [],+ "streamed_weight_only": [],"zero_row": []},"mxfp8": {@@ -2245,6 +2345,7 @@"exclude": [],"pad": [],"runtime_quant": [],+ "streamed_weight_only": [],"zero_row": []},"nvfp4": {@@ -2258,6 +2359,7 @@"exclude": [],"pad": [],"runtime_quant": [],+ "streamed_weight_only": [],"zero_row": []},"page": "image",@@ -2276,6 +2378,7 @@"exclude": [],"pad": [],"runtime_quant": [],+ "streamed_weight_only": [],"zero_row": []},"auto_bf16_when_resident": null,@@ -2290,6 +2393,7 @@"exclude": [],"pad": [],"runtime_quant": [],+ "streamed_weight_only": [],"zero_row": []},"convrot_default_on": false,@@ -2304,6 +2408,7 @@"exclude": [],"pad": [],"runtime_quant": [],+ "streamed_weight_only": [],"zero_row": []},"int8": {@@ -2326,6 +2431,7 @@],"pad": [],"runtime_quant": [],+ "streamed_weight_only": [],"zero_row": []},"mxfp8": {@@ -2339,6 +2445,7 @@"exclude": [],"pad": [],"runtime_quant": [],+ "streamed_weight_only": [],"zero_row": []},"nvfp4": {@@ -2352,6 +2459,7 @@"exclude": [],"pad": [],"runtime_quant": [],+ "streamed_weight_only": [],"zero_row": []},"page": "image",@@ -2370,6 +2478,7 @@"exclude": [],"pad": [],"runtime_quant": [],+ "streamed_weight_only": [],"zero_row": []},"auto_bf16_when_resident": null,@@ -2384,6 +2493,7 @@"exclude": [],"pad": [],"runtime_quant": [],+ "streamed_weight_only": [],"zero_row": []},"convrot_default_on": false,@@ -2398,6 +2508,7 @@"exclude": [],"pad": [],"runtime_quant": [],+ "streamed_weight_only": [],"zero_row": []},"int8": {@@ -2423,6 +2534,7 @@],"pad": [],"runtime_quant": [],+ "streamed_weight_only": [],"zero_row": []},"mxfp8": {@@ -2436,6 +2548,7 @@"exclude": [],"pad": [],"runtime_quant": [],+ "streamed_weight_only": [],"zero_row": []},"nvfp4": {@@ -2449,6 +2562,7 @@"exclude": [],"pad": [],"runtime_quant": [],+ "streamed_weight_only": [],"zero_row": []},"page": "video",@@ -2467,6 +2581,7 @@"exclude": [],"pad": [],"runtime_quant": [],+ "streamed_weight_only": [],"zero_row": []},"auto_bf16_when_resident": null,@@ -2481,6 +2596,7 @@"exclude": [],"pad": [],"runtime_quant": [],+ "streamed_weight_only": [],"zero_row": []},"convrot_default_on": true,@@ -2495,6 +2611,7 @@"exclude": [],"pad": [],"runtime_quant": [],+ "streamed_weight_only": [],"zero_row": []},"int8": {@@ -2522,6 +2639,7 @@],"pad": [],"runtime_quant": [],+ "streamed_weight_only": [],"zero_row": []},"mxfp8": {@@ -2535,6 +2653,7 @@"exclude": [],"pad": [],"runtime_quant": [],+ "streamed_weight_only": [],"zero_row": []},"nvfp4": {@@ -2548,6 +2667,7 @@"exclude": [],"pad": [],"runtime_quant": [],+ "streamed_weight_only": [],"zero_row": []},"page": "video",@@ -2566,6 +2686,7 @@"exclude": [],"pad": [],"runtime_quant": [],+ "streamed_weight_only": [],"zero_row": []},"auto_bf16_when_resident": "INT8 is no faster than compiled bf16 on a card that holds bf16 and changes image detail",@@ -2580,6 +2701,7 @@"exclude": [],"pad": [],"runtime_quant": [],+ "streamed_weight_only": [],"zero_row": []},"convrot_default_on": false,@@ -2594,6 +2716,7 @@"exclude": [],"pad": [],"runtime_quant": [],+ "streamed_weight_only": [],"zero_row": []},"int8": {@@ -2616,6 +2739,7 @@],"pad": [],"runtime_quant": [],+ "streamed_weight_only": [],"zero_row": []},"mxfp8": {@@ -2629,6 +2753,7 @@"exclude": [],"pad": [],"runtime_quant": [],+ "streamed_weight_only": [],"zero_row": []},"nvfp4": {@@ -2642,6 +2767,7 @@"exclude": [],"pad": [],"runtime_quant": [],+ "streamed_weight_only": [],"zero_row": []},"page": "image",@@ -2660,6 +2786,7 @@"exclude": [],"pad": [],"runtime_quant": [],+ "streamed_weight_only": [],"zero_row": []},"auto_bf16_when_resident": null,@@ -2674,6 +2801,7 @@"exclude": [],"pad": [],"runtime_quant": [],+ "streamed_weight_only": [],"zero_row": []},"convrot_default_on": false,@@ -2688,6 +2816,7 @@"exclude": [],"pad": [],"runtime_quant": [],+ "streamed_weight_only": [],"zero_row": []},"int8": {@@ -2714,6 +2843,7 @@"token_refiner"],"runtime_quant": [],+ "streamed_weight_only": [],"zero_row": []},"mxfp8": {@@ -2727,6 +2857,7 @@"exclude": [],"pad": [],"runtime_quant": [],+ "streamed_weight_only": [],"zero_row": []},"nvfp4": {@@ -2740,6 +2871,7 @@"exclude": [],"pad": [],"runtime_quant": [],+ "streamed_weight_only": [],"zero_row": []},"page": "video",@@ -2762,6 +2894,7 @@],"pad": [],"runtime_quant": [],+ "streamed_weight_only": [],"zero_row": []},"auto_bf16_when_resident": "a small denoiser stays bf16 while it fits (INT8 / FP8 change image detail: FP8 LPIPS 0.065 on FLUX.2-klein-4B, 0.103 on Z-Image-Turbo)",@@ -2780,6 +2913,7 @@],"pad": [],"runtime_quant": [],+ "streamed_weight_only": [],"zero_row": []},"convrot_default_on": true,@@ -2798,6 +2932,7 @@],"pad": [],"runtime_quant": [],+ "streamed_weight_only": [],"zero_row": []},"int8": {@@ -2832,6 +2967,7 @@],"pad": [],"runtime_quant": [],+ "streamed_weight_only": [],"zero_row": []},"mxfp8": {@@ -2849,6 +2985,7 @@],"pad": [],"runtime_quant": [],+ "streamed_weight_only": [],"zero_row": []},"nvfp4": {@@ -2866,6 +3003,7 @@],"pad": [],"runtime_quant": [],+ "streamed_weight_only": [],"zero_row": []},"page": "image",@@ -2884,6 +3022,7 @@"exclude": [],"pad": [],"runtime_quant": [],+ "streamed_weight_only": [],"zero_row": []},"auto_bf16_when_resident": null,@@ -2898,6 +3037,7 @@"exclude": [],"pad": [],"runtime_quant": [],+ "streamed_weight_only": [],"zero_row": []},"convrot_default_on": false,@@ -2912,6 +3052,7 @@"exclude": [],"pad": [],"runtime_quant": [],+ "streamed_weight_only": [],"zero_row": []},"int8": {@@ -2946,6 +3087,10 @@"to_add_out","txt_mlp"],+ "streamed_weight_only": [+ "img_mod",+ "txt_mod"+ ],"zero_row": []},"mxfp8": {@@ -2959,6 +3104,7 @@"exclude": [],"pad": [],"runtime_quant": [],+ "streamed_weight_only": [],"zero_row": []},"nvfp4": {@@ -2972,6 +3118,7 @@"exclude": [],"pad": [],"runtime_quant": [],+ "streamed_weight_only": [],"zero_row": []},"page": "image",@@ -2990,6 +3137,7 @@"exclude": [],"pad": [],"runtime_quant": [],+ "streamed_weight_only": [],"zero_row": []},"auto_bf16_when_resident": null,@@ -3004,6 +3152,7 @@"exclude": [],"pad": [],"runtime_quant": [],+ "streamed_weight_only": [],"zero_row": []},"convrot_default_on": true,@@ -3018,6 +3167,7 @@"exclude": [],"pad": [],"runtime_quant": [],+ "streamed_weight_only": [],"zero_row": []},"int8": {@@ -3052,6 +3202,7 @@],"pad": [],"runtime_quant": [],+ "streamed_weight_only": [],"zero_row": []},"mxfp8": {@@ -3065,6 +3216,7 @@"exclude": [],"pad": [],"runtime_quant": [],+ "streamed_weight_only": [],"zero_row": []},"nvfp4": {@@ -3078,6 +3230,7 @@"exclude": [],"pad": [],"runtime_quant": [],+ "streamed_weight_only": [],"zero_row": []},"page": "image",@@ -3096,6 +3249,7 @@"exclude": [],"pad": [],"runtime_quant": [],+ "streamed_weight_only": [],"zero_row": []},"auto_bf16_when_resident": null,@@ -3110,6 +3264,7 @@"exclude": [],"pad": [],"runtime_quant": [],+ "streamed_weight_only": [],"zero_row": []},"convrot_default_on": false,@@ -3124,6 +3279,7 @@"exclude": [],"pad": [],"runtime_quant": [],+ "streamed_weight_only": [],"zero_row": []},"int8": {@@ -3152,6 +3308,7 @@],"pad": [],"runtime_quant": [],+ "streamed_weight_only": [],"zero_row": []},"mxfp8": {@@ -3165,6 +3322,7 @@"exclude": [],"pad": [],"runtime_quant": [],+ "streamed_weight_only": [],"zero_row": []},"nvfp4": {@@ -3178,6 +3336,7 @@"exclude": [],"pad": [],"runtime_quant": [],+ "streamed_weight_only": [],"zero_row": []},"page": "image",@@ -3196,6 +3355,7 @@"exclude": [],"pad": [],"runtime_quant": [],+ "streamed_weight_only": [],"zero_row": []},"auto_bf16_when_resident": null,@@ -3210,6 +3370,7 @@"exclude": [],"pad": [],"runtime_quant": [],+ "streamed_weight_only": [],"zero_row": []},"convrot_default_on": false,@@ -3224,6 +3385,7 @@"exclude": [],"pad": [],"runtime_quant": [],+ "streamed_weight_only": [],"zero_row": []},"int8": {@@ -3252,6 +3414,7 @@],"pad": [],"runtime_quant": [],+ "streamed_weight_only": [],"zero_row": []},"mxfp8": {@@ -3265,6 +3428,7 @@"exclude": [],"pad": [],"runtime_quant": [],+ "streamed_weight_only": [],"zero_row": []},"nvfp4": {@@ -3278,6 +3442,7 @@"exclude": [],"pad": [],"runtime_quant": [],+ "streamed_weight_only": [],"zero_row": []},"page": "image",@@ -3296,6 +3461,7 @@"exclude": [],"pad": [],"runtime_quant": [],+ "streamed_weight_only": [],"zero_row": []},"auto_bf16_when_resident": null,@@ -3310,6 +3476,7 @@"exclude": [],"pad": [],"runtime_quant": [],+ "streamed_weight_only": [],"zero_row": []},"convrot_default_on": false,@@ -3324,6 +3491,7 @@"exclude": [],"pad": [],"runtime_quant": [],+ "streamed_weight_only": [],"zero_row": []},"int8": {@@ -3346,6 +3514,7 @@],"pad": [],"runtime_quant": [],+ "streamed_weight_only": [],"zero_row": []},"mxfp8": {@@ -3359,6 +3528,7 @@"exclude": [],"pad": [],"runtime_quant": [],+ "streamed_weight_only": [],"zero_row": []},"nvfp4": {@@ -3372,6 +3542,7 @@"exclude": [],"pad": [],"runtime_quant": [],+ "streamed_weight_only": [],"zero_row": []},"page": "image",@@ -3390,6 +3561,7 @@"exclude": [],"pad": [],"runtime_quant": [],+ "streamed_weight_only": [],"zero_row": []},"auto_bf16_when_resident": null,@@ -3404,6 +3576,7 @@"exclude": [],"pad": [],"runtime_quant": [],+ "streamed_weight_only": [],"zero_row": []},"convrot_default_on": false,@@ -3418,6 +3591,7 @@"exclude": [],"pad": [],"runtime_quant": [],+ "streamed_weight_only": [],"zero_row": []},"int8": {@@ -3440,6 +3614,7 @@],"pad": [],"runtime_quant": [],+ "streamed_weight_only": [],"zero_row": []},"mxfp8": {@@ -3453,6 +3628,7 @@"exclude": [],"pad": [],"runtime_quant": [],+ "streamed_weight_only": [],"zero_row": []},"nvfp4": {@@ -3466,6 +3642,7 @@"exclude": [],"pad": [],"runtime_quant": [],+ "streamed_weight_only": [],"zero_row": []},"page": "image",@@ -3484,6 +3661,7 @@"exclude": [],"pad": [],"runtime_quant": [],+ "streamed_weight_only": [],"zero_row": []},"auto_bf16_when_resident": null,@@ -3498,6 +3676,7 @@"exclude": [],"pad": [],"runtime_quant": [],+ "streamed_weight_only": [],"zero_row": []},"convrot_default_on": false,@@ -3512,6 +3691,7 @@"exclude": [],"pad": [],"runtime_quant": [],+ "streamed_weight_only": [],"zero_row": []},"int8": {@@ -3534,6 +3714,7 @@],"pad": [],"runtime_quant": [],+ "streamed_weight_only": [],"zero_row": []},"mxfp8": {@@ -3547,6 +3728,7 @@"exclude": [],"pad": [],"runtime_quant": [],+ "streamed_weight_only": [],"zero_row": []},"nvfp4": {@@ -3560,6 +3742,7 @@"exclude": [],"pad": [],"runtime_quant": [],+ "streamed_weight_only": [],"zero_row": []},"page": "video",@@ -3578,6 +3761,7 @@"exclude": [],"pad": [],"runtime_quant": [],+ "streamed_weight_only": [],"zero_row": []},"auto_bf16_when_resident": null,@@ -3592,6 +3776,7 @@"exclude": [],"pad": [],"runtime_quant": [],+ "streamed_weight_only": [],"zero_row": []},"convrot_default_on": false,@@ -3606,6 +3791,7 @@"exclude": [],"pad": [],"runtime_quant": [],+ "streamed_weight_only": [],"zero_row": []},"int8": {@@ -3628,6 +3814,7 @@],"pad": [],"runtime_quant": [],+ "streamed_weight_only": [],"zero_row": []},"mxfp8": {@@ -3641,6 +3828,7 @@"exclude": [],"pad": [],"runtime_quant": [],+ "streamed_weight_only": [],"zero_row": []},"nvfp4": {@@ -3654,6 +3842,7 @@"exclude": [],"pad": [],"runtime_quant": [],+ "streamed_weight_only": [],"zero_row": []},"page": "video",@@ -3672,6 +3861,7 @@"exclude": [],"pad": [],"runtime_quant": [],+ "streamed_weight_only": [],"zero_row": []},"auto_bf16_when_resident": null,@@ -3686,6 +3876,7 @@"exclude": [],"pad": [],"runtime_quant": [],+ "streamed_weight_only": [],"zero_row": []},"convrot_default_on": false,@@ -3700,6 +3891,7 @@"exclude": [],"pad": [],"runtime_quant": [],+ "streamed_weight_only": [],"zero_row": []},"int8": {@@ -3722,6 +3914,7 @@],"pad": [],"runtime_quant": [],+ "streamed_weight_only": [],"zero_row": []},"mxfp8": {@@ -3735,6 +3928,7 @@"exclude": [],"pad": [],"runtime_quant": [],+ "streamed_weight_only": [],"zero_row": []},"nvfp4": {@@ -3748,6 +3942,7 @@"exclude": [],"pad": [],"runtime_quant": [],+ "streamed_weight_only": [],"zero_row": []},"page": "video",@@ -3766,6 +3961,7 @@"exclude": [],"pad": [],"runtime_quant": [],+ "streamed_weight_only": [],"zero_row": []},"auto_bf16_when_resident": null,@@ -3780,6 +3976,7 @@"exclude": [],"pad": [],"runtime_quant": [],+ "streamed_weight_only": [],"zero_row": []},"convrot_default_on": false,@@ -3794,6 +3991,7 @@"exclude": [],"pad": [],"runtime_quant": [],+ "streamed_weight_only": [],"zero_row": []},"int8": {@@ -3816,6 +4014,7 @@],"pad": [],"runtime_quant": [],+ "streamed_weight_only": [],"zero_row": []},"mxfp8": {@@ -3829,6 +4028,7 @@"exclude": [],"pad": [],"runtime_quant": [],+ "streamed_weight_only": [],"zero_row": []},"nvfp4": {@@ -3842,6 +4042,7 @@"exclude": [],"pad": [],"runtime_quant": [],+ "streamed_weight_only": [],"zero_row": []},"page": "video",@@ -3860,6 +4061,7 @@"exclude": [],"pad": [],"runtime_quant": [],+ "streamed_weight_only": [],"zero_row": []},"auto_bf16_when_resident": "a small denoiser stays bf16 while it fits (INT8 / FP8 change image detail: FP8 LPIPS 0.065 on FLUX.2-klein-4B, 0.103 on Z-Image-Turbo)",@@ -3874,6 +4076,7 @@"exclude": [],"pad": [],"runtime_quant": [],+ "streamed_weight_only": [],"zero_row": []},"convrot_default_on": true,@@ -3888,6 +4091,7 @@"exclude": [],"pad": [],"runtime_quant": [],+ "streamed_weight_only": [],"zero_row": []},"int8": {@@ -3919,6 +4123,7 @@],"pad": [],"runtime_quant": [],+ "streamed_weight_only": [],"zero_row": []},"mxfp8": {@@ -3932,6 +4137,7 @@"exclude": [],"pad": [],"runtime_quant": [],+ "streamed_weight_only": [],"zero_row": []},"nvfp4": {@@ -3945,6 +4151,7 @@"exclude": [],"pad": [],"runtime_quant": [],+ "streamed_weight_only": [],"zero_row": []},"page": "image",diff --git a/studio/backend/tests/test_diffusion_compile_cache_variant.py b/studio/backend/tests/test_diffusion_compile_cache_variant.pyindex 2cb278c74e3..86cebcc75be 100644--- a/studio/backend/tests/test_diffusion_compile_cache_variant.py+++ b/studio/backend/tests/test_diffusion_compile_cache_variant.py@@ -125,7 +125,7 @@ def test_an_instance_forward_swap_keys_apart():def test_graph_switches_key_apart_only_off_their_default(monkeypatch, name, default, quant):m = _int8() if quant else _model()k0 = _key(m, quant)- rows = default.partition(":")[2] if default.startswith("rows:") else None+ rows = default.partition(":")[2] if default.startswith(("rows:", "count:")) else Nonefor same in (("",)+ (() if default == "raw" else ("auto",))@@ -134,7 +134,10 @@ def test_graph_switches_key_apart_only_off_their_default(monkeypatch, name, defa):monkeypatch.setenv(name, same)assert _key(m, quant) == k0, same- if rows:+ if default.startswith("count:"):+ # 0 is a real value here (no dense channels), not an off word.+ flipped = "0"+ elif rows:flipped = "1"else:flipped = {"on": "0", "off": "1", "auto": "0", "raw": "torchao", "raw:auto": "torchao"}[diff --git a/studio/backend/tests/test_diffusion_family_data_freeze.py b/studio/backend/tests/test_diffusion_family_data_freeze.pyindex d27ac6594fc..9fe10928e7d 100644--- a/studio/backend/tests/test_diffusion_family_data_freeze.py+++ b/studio/backend/tests/test_diffusion_family_data_freeze.py@@ -84,6 +84,9 @@ def _snapshot() -> dict:row[scheme] = {"exclude": list(tq.exclude_tokens_for_scheme(scheme, name, baked = True)),"runtime_quant": _streamed(tq.runtime_quant_tokens_for_scheme, scheme, name),+ "streamed_weight_only": _streamed(+ tq.streamed_weight_only_tokens_for_scheme, scheme, name+ ),"pad": list(tq.pad_tokens_for_scheme(scheme, name)),"zero_row": list(tq.zero_row_tokens_for_scheme(scheme, name)),"convrot_spec": _jsonable(tq.convrot_spec_for_scheme(scheme, name)),@@ -190,6 +193,7 @@ def _clean_env(monkeypatch):for var in ("UNSLOTH_DIFFUSION_INT8_CONVROT","UNSLOTH_DIFFUSION_INT8_TEXT_STREAM",+ "UNSLOTH_DIFFUSION_INT8_MODULATION","UNSLOTH_DIFFUSION_NVFP4","UNSLOTH_DIFFUSION_AUTO_STEP_SKIP","UNSLOTH_VIDEO_VAE_UNTILED",diff --git a/studio/backend/tests/test_diffusion_qwenimage_mod_int8.py b/studio/backend/tests/test_diffusion_qwenimage_mod_int8.pynew file mode 100644index 00000000000..cb048a5b8b4--- /dev/null+++ b/studio/backend/tests/test_diffusion_qwenimage_mod_int8.py@@ -0,0 +1,298 @@+# SPDX-License-Identifier: AGPL-3.0-only+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0++"""Qwen-Image streamed int8 weight-only modulations: scope, cache key, the outlier-column split and the GEMV+module's torch reference path. CPU-only (the Triton kernel itself is checked on a GPU by the bench scripts)."""++from __future__ import annotations++import pytest+import torch+from torch import nn++import core.inference.diffusion_int8_gemv as gv+import core.inference.diffusion_transformer_quant as tq+from core.inference.diffusion_transformer_quant import TQ_FP8, TQ_INT8++_MODS = ("img_mod", "txt_mod")++[email protected](autouse = True)+def _clean_env(monkeypatch):+ for var in (+ tq.INT8_MODULATION_ENV,+ tq.INT8_MOD_OUTLIERS_ENV,+ tq.INT8_TEXT_STREAM_ENV,+ gv.INT8_GEMV_ENV,+ ):+ monkeypatch.delenv(var, raising = False)+++def test_tokens_only_inside_the_streamed_scope():+ assert tq.streamed_weight_only_tokens_for_scheme(TQ_INT8, "qwen-image") == ()+ assert tq.streamed_weight_only_layout(TQ_INT8, "qwen-image") == {}+ with tq.int8_text_stream_scope(True):+ assert tq.streamed_weight_only_tokens_for_scheme(TQ_INT8, "qwen-image") == _MODS+ assert tq.streamed_weight_only_tokens_for_scheme(TQ_FP8, "qwen-image") == ()+ assert tq.streamed_weight_only_tokens_for_scheme(TQ_INT8, "flux.1") == ()+ assert tq.streamed_weight_only_layout(TQ_INT8, "qwen-image") == {+ "weight_only_name_tokens": sorted(_MODS),+ "weight_only_mode": "wo",+ "weight_only_outliers": tq.DEFAULT_MOD_OUTLIERS,+ }+ with tq.int8_text_stream_scope(False):+ assert tq.streamed_weight_only_tokens_for_scheme(TQ_INT8, "qwen-image") == ()+++ "raw, mode",+ [+ ("", "wo"),+ ("1", "wo"),+ ("on", "wo"),+ ("0", "off"),+ ("off", "off"),+ ("w8a8", "w8a8"),+ ("w8a8rot", "w8a8rot"),+ ],+)+def test_modulation_switch(monkeypatch, raw, mode):+ monkeypatch.setenv(tq.INT8_MODULATION_ENV, raw)+ assert tq.int8_modulation_mode() == mode+ with tq.int8_text_stream_scope(True):+ tokens = tq.streamed_weight_only_tokens_for_scheme(TQ_INT8, "qwen-image")+ assert tokens == (() if mode == "off" else _MODS)++[email protected]("raw, n", [("", 8), ("0", 0), ("4", 4), ("99", 16), ("junk", 8)])+def test_outlier_switch(monkeypatch, raw, n):+ monkeypatch.setenv(tq.INT8_MOD_OUTLIERS_ENV, raw)+ assert tq.int8_mod_outliers() == n+++ "raw, keyed", [("", None), ("8", None), ("junk", None), ("0", "0"), ("4", "4"), ("99", "16")]+)+def test_compile_cache_switch_keys_on_the_outlier_count_the_reader_uses(monkeypatch, raw, keyed):+ from core.inference import diffusion_compile_cache as cc++ monkeypatch.setenv(tq.INT8_MOD_OUTLIERS_ENV, raw)+ assert cc.graph_switches().get(tq.INT8_MOD_OUTLIERS_ENV) == keyed+ assert keyed is None or int(keyed) == tq.int8_mod_outliers()+++def test_cache_key_carries_the_layout(monkeypatch):+ from core.inference.diffusion_prequant_cache import key_parts++ def parts():+ return key_parts(+ family = "qwen-image",+ transformer_class = "QwenImageTransformer2DModel",+ scheme = TQ_INT8,+ component = "transformer",+ source_fp = "fp",+ )++ resident = parts()+ assert "weight_only_name_tokens" not in resident and "weight_only_outliers" not in resident+ with tq.int8_text_stream_scope(True):+ default = parts()+ monkeypatch.setenv(tq.INT8_MOD_OUTLIERS_ENV, "0")+ no_outliers = parts()+ monkeypatch.setenv(tq.INT8_MODULATION_ENV, "w8a8rot")+ rot = parts()+ monkeypatch.setenv(tq.INT8_MODULATION_ENV, "0")+ off = parts()+ assert default["weight_only_outliers"] == 8 and no_outliers["weight_only_outliers"] == 0+ assert rot["weight_only_mode"] == "w8a8rot"+ assert "weight_only_name_tokens" not in off+ keys = [repr(sorted(p.items())) for p in (resident, default, no_outliers, rot, off)]+ assert len(set(keys)) == len(keys)+++def _linear(+ n = 48,+ k = 256,+ seed = 0,+):+ g = torch.Generator().manual_seed(seed)+ lin = nn.Linear(k, n, dtype = torch.bfloat16)+ lin.weight.data = (torch.randn(n, k, generator = g) * 0.02).to(torch.bfloat16)+ lin.bias.data = torch.randn(n, generator = g).to(torch.bfloat16)+ return lin+++def _outlier_x(+ m = 2,+ k = 256,+ seed = 1,+ channels = (3, 77),+):+ g = torch.Generator().manual_seed(seed)+ x = torch.randn(m, k, generator = g)+ for c in channels:+ x[:, c] = 300.0 + c+ return x.to(torch.bfloat16)+++def test_split_is_exact_on_the_dense_weight():+ lin = _linear()+ x = _outlier_x()+ want = torch.nn.functional.linear(x.float(), lin.weight.float(), lin.bias.float())+ moved = gv.split_outliers_(lin, (77, 3))+ assert moved == 2 and gv.is_gemv_linear(lin)+ assert lin.outlier_idx.tolist() == [3, 77]+ assert bool((lin.weight[:, [3, 77]] == 0).all())+ got = lin(x.float().to(torch.bfloat16)).float()+ assert torch.allclose(got, want, rtol = 2e-2, atol = 2e-2)++[email protected]("bad", [(), None, tuple(range(17)), (999,)])+def test_split_refuses_bad_channel_sets(bad):+ lin = _linear()+ before = lin.weight.clone()+ assert gv.split_outliers_(lin, bad) == 0+ assert type(lin) is nn.Linear and torch.equal(lin.weight, before)+++def _quantized(lin):+ pytest.importorskip("torchao")+ from torchao.quantization import quantize_++ config = tq._weight_only_config()+ if config is None:+ pytest.skip("torchao without Int8Tensor weight-only")+ quantize_(lin, config, filter_fn = lambda _m, fqn = "": True)+ assert gv.weight_only_parts(lin.weight) is not None+ return lin+++def test_outlier_columns_cut_the_weight_only_error():+ x = _outlier_x()+ plain, split = _linear(), _linear()+ truth = x.double() @ plain.weight.double().t() + plain.bias.double()+ gv.split_outliers_(split, (3, 77))+ _quantized(plain)+ _quantized(split)+ gv.install(nn.ModuleDict({"a": plain}), ["a"])++ def err(y):+ return float((y.double() - truth).norm() / truth.norm())++ q, s = gv.weight_only_parts(plain.weight)+ e_plain = err(gv.reference(x, q, s, plain.bias))+ q2, s2 = gv.weight_only_parts(split.weight)+ e_split = err(gv.reference(x, q2, s2, split.bias, split.outlier_idx, split.outlier_weight))+ assert e_split < e_plain / 3+ # the module forward (CPU: torchao weight-only + the dense columns) agrees with the fused reference+ y_mod = split(x)+ y_ref = gv.reference(x, q2, s2, split.bias, split.outlier_idx, split.outlier_weight)+ assert torch.allclose(y_mod.float(), y_ref.float(), rtol = 2e-2, atol = 5e-2)+++def test_reference_grouped_scales_match_dequant():+ g = torch.Generator().manual_seed(3)+ q = torch.randint(-127, 128, (32, 128), generator = g, dtype = torch.int8)+ s = torch.rand(32, 4, generator = g) * 1e-2+ x = torch.randn(3, 128, generator = g).to(torch.bfloat16)+ deq = (q.float().reshape(32, 4, 32) * s.reshape(32, 4, 1)).reshape(32, 128)+ want = (x.float() @ deq.t()).to(torch.bfloat16)+ assert torch.allclose(gv.reference(x, q, s, None).float(), want.float(), rtol = 1e-2, atol = 1e-3)+++class _Tte(nn.Module):+ def __init__(self, hot = (5, 40)):+ super().__init__()+ from diffusers.models.embeddings import TimestepEmbedding, Timesteps++ self.time_proj = Timesteps(256, flip_sin_to_cos = True, downscale_freq_shift = 0, scale = 1000)+ self.timestep_embedder = TimestepEmbedding(256, 64)+ with torch.no_grad():+ self.timestep_embedder.linear_2.bias[list(hot)] = 50.0+++class _Toy(nn.Module):+ def __init__(self):+ super().__init__()+ self.time_text_embed = _Tte()+ self.img_mod = nn.Sequential(nn.SiLU(), nn.Linear(64, 96))+++def test_timestep_outlier_channels_finds_the_hot_ones():+ pytest.importorskip("diffusers")+ toy = _Toy()+ assert set(gv.timestep_outlier_channels(toy, 2)) == {5, 40}+ assert gv.timestep_outlier_channels(toy, 0) == ()+ assert gv.timestep_outlier_channels(nn.Linear(4, 4), 4) == ()+++def test_strict_load_round_trip_restores_the_columns():+ src = nn.ModuleDict({"mod": _linear()})+ x = _outlier_x()+ gv.split_outliers_(src["mod"], (3, 77))+ _quantized(src["mod"])+ want = src["mod"](x)+ state = src.state_dict()+ assert "mod.outlier_weight" in state and "mod.outlier_idx" in state+ dst = nn.ModuleDict({"mod": _linear(seed = 9)})+ _quantized(dst["mod"])+ assert gv.prepare_outlier_buffers(dst, state) == 1+ dst.load_state_dict(state, strict = True, assign = True)+ assert gv.is_gemv_linear(dst["mod"])+ assert torch.equal(dst["mod"](x), want)+++def test_install_skips_plain_linears_without_the_kernel(monkeypatch):+ monkeypatch.setenv(gv.INT8_GEMV_ENV, "0")+ plain = _quantized(_linear())+ split = _linear()+ gv.split_outliers_(split, (3,))+ _quantized(split)+ tree = nn.ModuleDict({"plain": plain, "split": split, "dense": _linear()})+ assert gv.install(tree, ["plain", "split", "dense"]) == 1+ assert type(tree["plain"]) is nn.Linear and gv.is_gemv_linear(tree["split"])+ assert type(tree["dense"]) is nn.Linear+ assert gv._OP_HANDLE is None+++def test_streamed_quantize_builds_the_layout(monkeypatch):+ pytest.importorskip("diffusers")+ if tq._weight_only_config() is None:+ pytest.skip("torchao without Int8Tensor weight-only")++ class Block(nn.Module):+ def __init__(self):+ super().__init__()+ self.img_mod = nn.Sequential(nn.SiLU(), nn.Linear(64, 512, dtype = torch.bfloat16))+ self.txt_mod = nn.Sequential(nn.SiLU(), nn.Linear(64, 512, dtype = torch.bfloat16))+ self.other = nn.Linear(64, 512, dtype = torch.bfloat16)++ toy = _Toy()+ toy.time_text_embed.to(torch.bfloat16)+ toy.block = Block()+ temb = torch.nn.functional.silu(+ toy.time_text_embed.timestep_embedder(+ toy.time_text_embed.time_proj(torch.tensor([0.5])).to(torch.bfloat16)+ )+ )+ want = toy.block.img_mod(temb.float().to(torch.bfloat16)).float()+ with tq.int8_text_stream_scope(True):+ names = tq.quantize_streamed_weight_only(toy, TQ_INT8, "qwen-image", min_features = 32)+ assert set(names) == {+ "block.img_mod.1",+ "block.txt_mod.1",+ } # the float32 toy img_mod is not bf16: left alone+ layout = toy._unsloth_weight_only_layout+ assert (+ layout["weight_only_outliers"] == tq.DEFAULT_MOD_OUTLIERS+ and layout["weight_only_mode"] == "wo"+ )+ assert gv.is_gemv_linear(toy.block.img_mod[1]) and type(toy.block.other) is nn.Linear+ assert {5, 40} <= set(toy.block.img_mod[1].outlier_idx.tolist())+ got = toy.block.img_mod(temb).float()+ assert float((got - want).norm() / want.norm()) < 1e-2+ # resident scope: nothing changes+ fresh = _Toy()+ with tq.int8_text_stream_scope(False):+ assert tq.quantize_streamed_weight_only(fresh, TQ_INT8, "qwen-image", min_features = 32) == ()+ assert type(fresh.img_mod[1]) is nn.Lineardiff --git a/studio/backend/tests/test_diffusion_qwenimage_text_int8.py b/studio/backend/tests/test_diffusion_qwenimage_text_int8.pyindex 1d2a48a8fa1..b87cccc1a6a 100644--- a/studio/backend/tests/test_diffusion_qwenimage_text_int8.py+++ b/studio/backend/tests/test_diffusion_qwenimage_text_int8.py@@ -25,6 +25,7 @@ _UNSCOPED = {@pytest.fixture(autouse = True)def _default_switch(monkeypatch, request):monkeypatch.delenv(tq.INT8_TEXT_STREAM_ENV, raising = False)+ monkeypatch.delenv(tq.INT8_MODULATION_ENV, raising = False)if request.node.name in _UNSCOPED:yieldreturn@@ -272,6 +273,8 @@ def test_local_prequant_cache_key_follows_the_switch(monkeypatch):with tq.int8_text_stream_scope(False):resident, resident_tokens = key()monkeypatch.setenv(tq.INT8_TEXT_STREAM_ENV, "0")+ # The streamed weight-only modulations key separately (test_diffusion_qwenimage_mod_int8.py).+ monkeypatch.setenv(tq.INT8_MODULATION_ENV, "0")off, off_tokens = key()assert on != offassert "add_q_proj" in off_tokens and "add_q_proj" not in on_tokens