Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

Studio: int8 Qwen-Image streamed modulations through a fused GEMV (1.23x under a 16 GiB budget) (#16087)

Daniel Hanfca19f5f0a9c
diff --git a/studio/backend/core/inference/diffusion_compile_cache.py b/studio/backend/core/inference/diffusion_compile_cache.py
index c7eb8022472..49550aa8efb 100644
--- a/studio/backend/core/inference/diffusion_compile_cache.py
+++ b/studio/backend/core/inference/diffusion_compile_cache.py
@@ -226,6 +226,10 @@ _GRAPH_SWITCHES = {
"UNSLOTH_DIFFUSION_INT8_ROTQUANT_UNLISTED": "auto",
"UNSLOTH_DIFFUSION_INT8_FUSED": "on",
"UNSLOTH_DIFFUSION_INT8_TEXT_STREAM": "on",
+ "UNSLOTH_DIFFUSION_INT8_MODULATION": "on",
+ # Keyed on the clamped count the reader uses (0 = none).
+ "UNSLOTH_DIFFUSION_INT8_MOD_OUTLIERS": "count:8",
+ "UNSLOTH_DIFFUSION_INT8_GEMV": "on",
"UNSLOTH_DIFFUSION_ZIMAGE_FUSED": "on",
"UNSLOTH_DIFFUSION_Q21_REAL_ROPE": "on",
"UNSLOTH_DIFFUSION_QWEN_REAL_ROPE": "on",
@@ -258,6 +262,12 @@ def _switch_value(name: str, default: str) -> Optional[str]:
except ValueError:
return None
return None if rows == default.partition(":")[2] else rows
+ if default.startswith("count:"):
+ try:
+ count = str(max(0, min(16, int(raw))))
+ except ValueError:
+ return None
+ return None if count == default.partition(":")[2] else count
if default.startswith("raw"):
return None if raw in _OFF_WORDS or raw == default.partition(":")[2] else raw
value = "off" if raw in _OFF_WORDS else "on" if raw in _ON_WORDS else None
diff --git a/studio/backend/core/inference/diffusion_families.py b/studio/backend/core/inference/diffusion_families.py
index 139e1c70c8c..8b02cd2afce 100644
--- a/studio/backend/core/inference/diffusion_families.py
+++ b/studio/backend/core/inference/diffusion_families.py
@@ -281,6 +281,8 @@ _QWENIMAGE_TEXT_STREAM_INT8 = (
"to_add_out",
"txt_mlp",
)
+# Streamed-only int8 weight-only; W8A8 would also round the timestep row per row.
+_QWENIMAGE_MOD_WEIGHT_ONLY = ("img_mod", "txt_mod")
# Families whose activation ranges break specific schemes at the MODEL level (the smoke probe only proves the GEMM
# runs). Measured with the 28-pair prequant accuracy gate on B200: qwen-image + mxfp8 does semantic damage at 1024px,
# + nvfp4 is unusable (LPIPS 0.51). Neither has a known fix, so both stay denied and auto falls through to int8
@@ -873,6 +875,7 @@ _FAMILIES: tuple[DiffusionFamily, ...] = (
profile = FamilyProfile(
int8_exclude_tokens = _QWENIMAGE_INT8_EXCLUDES,
int8_runtime_quant_tokens = _QWENIMAGE_TEXT_STREAM_INT8,
+ int8_streamed_weight_only_tokens = _QWENIMAGE_MOD_WEIGHT_ONLY,
scheme_deny = _QWENIMAGE_SCHEME_DENY,
train_scheme_deny = _QWENIMAGE_TRAIN_SCHEME_DENY,
auto_prefer = _GATED_AUTO_PREFER,
diff --git a/studio/backend/core/inference/diffusion_int8_gemv.py b/studio/backend/core/inference/diffusion_int8_gemv.py
new file mode 100644
index 00000000000..985d84ce663
--- /dev/null
+++ b/studio/backend/core/inference/diffusion_int8_gemv.py
@@ -0,0 +1,460 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
+
+"""Fused int8 weight-only GEMV for small-M Linears (AdaLN modulations, M = batch).
+
+torchao's weight-only linear materialises a bf16 copy of the whole weight per call; this reads the int8 codes and
+scales directly with one float32 accumulate. Per-row and per-group symmetric scales only; anything else keeps
+torchao's forward. Opaque to Inductor and M is the batch, so no recompiles or CUDA-graph re-captures.
+"""
+
+from __future__ import annotations
+
+import os
+import threading
+from functools import lru_cache
+from typing import Any, Optional
+
+INT8_GEMV_ENV = "UNSLOTH_DIFFUSION_INT8_GEMV"
+# Above this many rows the stock dequant + GEMM is faster.
+MAX_ROWS = 4
+MAX_OUTLIERS = 16
+OUTLIER_IDX = "outlier_idx"
+OUTLIER_WEIGHT = "outlier_weight"
+_OP_NAMESPACE = "unsloth_studio"
+_OP_NAME = "int8_wo_gemv"
+# (BLOCK_N, BLOCK_K, warps, stages), first that launches wins; the first needs ~40 KB smem, the second fits any card.
+_CONFIGS = ((128, 128, 8, 3), (64, 64, 4, 2))
+_DEVICE_CFG: dict = {}
+_LOCK = threading.Lock()
+# Engagement evidence for benches.
+COUNTS = {"gemv": 0, "fallback": 0}
+# Resolved once by ``install``, outside any trace.
+_OP_HANDLE: Any = None
+
+
+def gemv_enabled() -> bool:
+ raw = (os.environ.get(INT8_GEMV_ENV) or "").strip().lower()
+ return raw not in ("0", "off", "false", "no")
+
+
+@lru_cache(maxsize = 1)
+def _kernel() -> Any:
+ try:
+ import triton
+ import triton.language as tl
+ except Exception: # noqa: BLE001 - no Triton: stock path
+ return None
+
+ @triton.jit
+ def i8_wo_gemv(
+ x_ptr,
+ q_ptr,
+ s_ptr,
+ b_ptr,
+ oi_ptr,
+ ow_ptr,
+ y_ptr,
+ M,
+ N,
+ K,
+ KO,
+ stride_xm,
+ stride_qn,
+ stride_sn,
+ stride_on,
+ stride_ym,
+ HAS_BIAS: tl.constexpr,
+ HAS_OUT: tl.constexpr,
+ GROUPED: tl.constexpr,
+ BLOCK_M: tl.constexpr,
+ BLOCK_N: tl.constexpr,
+ BLOCK_K: tl.constexpr,
+ ):
+ pid = tl.program_id(0)
+ rn = pid * BLOCK_N + tl.arange(0, BLOCK_N)
+ rm = tl.arange(0, BLOCK_M)
+ n_ok = rn < N
+ m_ok = rm < M
+ acc = tl.zeros((BLOCK_M, BLOCK_N), dtype = tl.float32)
+ for k0 in range(0, K, BLOCK_K):
+ rk = k0 + tl.arange(0, BLOCK_K)
+ q = tl.load(q_ptr + rn[:, None] * stride_qn + rk[None, :], mask = n_ok[:, None], other = 0)
+ x = tl.load(
+ x_ptr + rm[:, None] * stride_xm + rk[None, :], mask = m_ok[:, None], other = 0.0
+ )
+ # int8 -> bf16 is exact and bf16 x bf16 products are exact in the float32 accumulator
+ part = tl.dot(x, tl.trans(q.to(tl.bfloat16)), out_dtype = tl.float32)
+ if GROUPED:
+ s = tl.load(s_ptr + rn * stride_sn + k0 // BLOCK_K, mask = n_ok, other = 0.0)
+ acc += part * s.to(tl.float32)[None, :]
+ else:
+ acc += part
+ if not GROUPED:
+ s = tl.load(s_ptr + rn * stride_sn, mask = n_ok, other = 0.0)
+ acc = acc * s.to(tl.float32)[None, :]
+ if HAS_OUT:
+ # bf16 outlier columns (their int8 codes are zero): x[:, idx] @ W_out.T, same accumulator
+ ro = tl.arange(0, 16)
+ o_ok = ro < KO
+ idx = tl.load(oi_ptr + ro, mask = o_ok, other = 0)
+ xo = tl.load(
+ x_ptr + rm[:, None] * stride_xm + idx[None, :],
+ mask = m_ok[:, None] & o_ok[None, :],
+ other = 0.0,
+ )
+ wo = tl.load(
+ ow_ptr + rn[:, None] * stride_on + ro[None, :],
+ mask = n_ok[:, None] & o_ok[None, :],
+ other = 0.0,
+ )
+ acc += tl.dot(xo, tl.trans(wo), out_dtype = tl.float32)
+ if HAS_BIAS:
+ b = tl.load(b_ptr + rn, mask = n_ok, other = 0.0)
+ acc = acc + b.to(tl.float32)[None, :]
+ tl.store(
+ y_ptr + rm[:, None] * stride_ym + rn[None, :],
+ acc.to(tl.bfloat16),
+ mask = m_ok[:, None] & n_ok[None, :],
+ )
+
+ return triton, i8_wo_gemv
+
+
+def reference(
+ x: Any,
+ q: Any,
+ s: Any,
+ bias: Any,
+ oidx: Any = None,
+ ow: Any = None,
+) -> Any:
+ """Same math in torch (float32 accumulate, one bf16 rounding): the CPU / no-Triton body of the op."""
+ import torch
+
+ n, k = q.shape
+ groups = s.numel() // n
+ xf = x.float()
+ if groups == 1:
+ y = (xf @ q.float().t()) * s.reshape(1, n).float()
+ else:
+ g = k // groups
+ parts = torch.einsum(
+ "mgk,ngk->mng", xf.reshape(-1, groups, g), q.float().reshape(n, groups, g)
+ )
+ y = (parts * s.reshape(1, n, groups).float()).sum(-1)
+ if oidx is not None:
+ y = y + xf.index_select(-1, oidx.long()) @ ow.float().t()
+ if bias is not None:
+ y = y + bias.float()
+ return y.to(torch.bfloat16)
+
+
+def _launch(x: Any, q: Any, s2: Any, bias: Any, oidx: Any, ow: Any, cfg: tuple) -> Any:
+ import torch
+
+ triton, fn = _kernel()
+ block_n, block_k, warps, stages = cfg
+ m, k = x.shape
+ n, groups = s2.shape
+ if groups > 1:
+ block_k = k // groups
+ ko = 0 if oidx is None else int(oidx.numel())
+ y = torch.empty((m, n), device = x.device, dtype = torch.bfloat16)
+ fn[(triton.cdiv(n, block_n),)](
+ x,
+ q,
+ s2,
+ bias if bias is not None else y,
+ oidx if ko else y,
+ ow if ko else y,
+ y,
+ m,
+ n,
+ k,
+ ko,
+ x.stride(0),
+ q.stride(0),
+ s2.stride(0),
+ ow.stride(0) if ko else 0,
+ y.stride(0),
+ HAS_BIAS = bias is not None,
+ HAS_OUT = ko > 0,
+ GROUPED = groups > 1,
+ # tl.dot wants >= 16 rows: the masked pad rows load zeros and are never stored
+ BLOCK_M = 16,
+ BLOCK_N = block_n,
+ BLOCK_K = block_k,
+ num_warps = warps,
+ num_stages = stages,
+ )
+ return y
+
+
+def _run(
+ x: Any,
+ q: Any,
+ s: Any,
+ bias: Any,
+ oidx: Any = None,
+ ow: Any = None,
+) -> Any:
+ m, k = x.shape
+ n = q.shape[0]
+ groups = s.numel() // n
+ block_k = k // groups if groups > 1 else _CONFIGS[-1][1]
+ ko = 0 if oidx is None else int(oidx.numel())
+ index = x.device.index if x.is_cuda else None
+ start = _DEVICE_CFG.get(index, 0)
+ if (
+ index is None
+ or _kernel() is None
+ or start >= len(_CONFIGS)
+ or k % block_k
+ or (groups == 1 and k % _CONFIGS[start][1])
+ or q.stride(-1) != 1
+ or block_k & (block_k - 1)
+ or block_k < 16
+ or ko > MAX_OUTLIERS
+ or (ko and ow.stride(-1) != 1)
+ ):
+ COUNTS["fallback"] += 1
+ return reference(x, q, s, bias, oidx, ow)
+ x = x if x.stride(-1) == 1 else x.contiguous()
+ s2 = s.reshape(n, groups)
+ if s2.stride(-1) != 1:
+ s2 = s2.contiguous()
+ for pos in range(start, len(_CONFIGS)):
+ try:
+ y = _launch(x, q, s2, bias, oidx, ow, _CONFIGS[pos])
+ except Exception: # noqa: BLE001 - a tile this card cannot hold (shared memory): the next, smaller one
+ continue
+ _DEVICE_CFG[index] = pos
+ COUNTS["gemv"] += 1
+ return y
+ _DEVICE_CFG[index] = len(_CONFIGS)
+ COUNTS["fallback"] += 1
+ return reference(x, q, s, bias, oidx, ow)
+
+
+@lru_cache(maxsize = 1)
+def _op() -> Any:
+ """``unsloth_studio::int8_wo_gemv`` (opaque to Inductor), or None."""
+ try:
+ import torch
+ except Exception: # noqa: BLE001
+ return None
+ ns = getattr(torch.ops, _OP_NAMESPACE, None)
+ if ns is not None and hasattr(ns, _OP_NAME):
+ return getattr(ns, _OP_NAME)
+ custom_op = getattr(getattr(torch, "library", None), "custom_op", None)
+ if custom_op is None:
+ return None
+ try:
+
+ @custom_op(
+ f"{_OP_NAMESPACE}::{_OP_NAME}",
+ mutates_args = (),
+ schema = "(Tensor x, Tensor q, Tensor s, Tensor? bias, Tensor? oidx, Tensor? ow) -> Tensor",
+ )
+ def _int8_wo_gemv(x, q, s, bias, oidx, ow):
+ return _run(x, q, s, bias, oidx, ow)
+
+ @_int8_wo_gemv.register_fake
+ def _(x, q, s, bias, oidx, ow):
+ return x.new_empty((x.shape[0], q.shape[0]), dtype = torch.bfloat16)
+ except Exception: # noqa: BLE001
+ return None
+ return getattr(torch.ops, _OP_NAMESPACE).int8_wo_gemv
+
+
+def weight_only_parts(w: Any) -> Optional[tuple]:
+ """(int8 [N, K], scale [N, K / G]) of a symmetric weight-only torchao ``Int8Tensor`` (per row or per group)."""
+ if type(w).__name__ != "Int8Tensor":
+ return None
+ if (
+ getattr(w, "act_quant_kwargs", None) is not None
+ or getattr(w, "act_pre_scale", None) is not None
+ ):
+ return None
+ q, s = getattr(w, "qdata", None), getattr(w, "scale", None)
+ block = list(getattr(w, "block_size", ()) or ())
+ if q is None or s is None or q.dim() != 2 or len(block) != 2 or block[0] != 1:
+ return None
+ n, k = q.shape
+ if k % block[1] or s.numel() != n * (k // block[1]):
+ return None
+ return q, s
+
+
+def _outliers(module: Any) -> tuple:
+ return getattr(module, OUTLIER_IDX, None), getattr(module, OUTLIER_WEIGHT, None)
+
+
+def _forward(self: Any, x: Any) -> Any:
+ import torch
+ import torch.nn.functional as F
+
+ w = self.weight
+ oidx, ow = _outliers(self)
+ parts = weight_only_parts(w)
+ flat_rows = x.numel() // x.shape[-1] if x.shape[-1] else 0
+ op = _OP_HANDLE
+ if (
+ op is None
+ or parts is None
+ or x.dtype != torch.bfloat16
+ or not x.is_cuda
+ or flat_rows == 0
+ or flat_rows > MAX_ROWS
+ or parts[0].device != x.device
+ or (oidx is not None and (oidx.device != x.device or ow.device != x.device))
+ ):
+ # stock torchao weight-only forward, plus the dense outlier columns (exact: their codes are zero)
+ y = F.linear(x, w, self.bias)
+ if oidx is not None:
+ y = y + F.linear(
+ x.index_select(-1, oidx.to(x.device)), ow.to(device = x.device, dtype = x.dtype)
+ )
+ return y
+ lead = x.shape[:-1]
+ y = op(x.reshape(-1, x.shape[-1]), parts[0], parts[1], self.bias, oidx, ow)
+ return y.reshape(*lead, y.shape[-1])
+
+
+@lru_cache(maxsize = None)
+def gemv_linear_class() -> Any:
+ """Built once: a fresh class per call would make torch.compile retrace every block."""
+ from torch import nn
+
+ class Int8GemvLinear(nn.Linear):
+ forward = _forward
+
+ return Int8GemvLinear
+
+
+def is_gemv_linear(module: Any) -> bool:
+ return type(module).__name__ == "Int8GemvLinear"
+
+
+def split_outliers_(module: Any, channels: Any) -> int:
+ """Before quantize_: move the weight columns of ``channels`` into a dense ``outlier_weight`` buffer and zero them
+ in the weight, so ``x @ W.T == x @ W_rest.T + x[:, idx] @ W_out.T`` exactly while the int8 scales only see the
+ rest. Returns how many columns moved (0 leaves the module untouched)."""
+ import torch
+
+ if channels is None or not len(channels) or len(channels) > MAX_OUTLIERS:
+ return 0
+ w = module.weight.data
+ idx = torch.as_tensor(sorted(int(c) for c in channels), dtype = torch.int64, device = w.device)
+ if int(idx.max()) >= w.shape[1]:
+ return 0
+ module.register_buffer(OUTLIER_WEIGHT, w.index_select(1, idx).contiguous())
+ module.register_buffer(OUTLIER_IDX, idx)
+ rest = w.clone()
+ rest[:, idx] = 0
+ module.weight.data = rest
+ # swapped now, not at install: from here on only this forward adds the columns back (dense or quantized weight)
+ module.__class__ = gemv_linear_class()
+ return int(idx.numel())
+
+
+def prepare_outlier_buffers(transformer: Any, state_dict: Any) -> int:
+ """Before ``load_state_dict(strict = True, assign = True)`` of an artifact built with outlier columns: give each
+ named Linear placeholder buffers of the stored shapes and the GEMV forward, so the strict load assigns them and the
+ forward adds them back. A key naming no Linear is left alone (the strict load then refuses the artifact)."""
+ import torch
+ from torch import nn
+
+ suffix = "." + OUTLIER_WEIGHT
+ count = 0
+ for key in [k for k in state_dict if isinstance(k, str) and k.endswith(suffix)]:
+ fqn = key[: -len(suffix)]
+ idx = state_dict.get(f"{fqn}.{OUTLIER_IDX}")
+ try:
+ module = transformer.get_submodule(fqn)
+ except AttributeError:
+ continue
+ if idx is None or not isinstance(module, nn.Linear):
+ continue
+ weight = state_dict[key]
+ module.register_buffer(
+ OUTLIER_WEIGHT, torch.empty(tuple(weight.shape), dtype = weight.dtype, device = "meta")
+ )
+ module.register_buffer(
+ OUTLIER_IDX, torch.empty(tuple(idx.shape), dtype = idx.dtype, device = "meta")
+ )
+ module.__class__ = gemv_linear_class()
+ count += 1
+ return count
+
+
+def timestep_outlier_channels(transformer: Any, count: int) -> tuple:
+ """The ``count`` input channels with the largest |SiLU(temb)| over the whole timestep range, from the model's own
+ ``time_text_embed`` (Qwen-Image: the modulations' only input is SiLU of it, a function of the timestep alone, so
+ this needs no prompts). () when the transformer has no such embedder or anything fails."""
+ if count <= 0:
+ return ()
+ try:
+ import torch
+
+ tte = getattr(transformer, "time_text_embed", None)
+ proj = getattr(tte, "time_proj", None)
+ embedder = getattr(tte, "timestep_embedder", None)
+ if proj is None or embedder is None:
+ return ()
+ weight = embedder.linear_1.weight
+ if getattr(weight, "is_meta", False):
+ return ()
+ with torch.no_grad():
+ sigma = torch.linspace(0.0, 1.0, 1001, device = weight.device)
+ emb = embedder(proj(sigma).to(weight.dtype))
+ extra = getattr(tte, "addition_t_embedding", None)
+ if extra is not None:
+ rows = extra.weight.to(emb.dtype)
+ emb = torch.cat([emb + r for r in rows], dim = 0)
+ score = torch.nn.functional.silu(emb.float()).abs().amax(dim = 0)
+ if not bool(torch.isfinite(score).all()) or count > score.numel():
+ return ()
+ return tuple(sorted(int(i) for i in torch.topk(score, count).indices.tolist()))
+ except Exception: # noqa: BLE001 - calibration is optional: no outliers, plain int8 codes
+ return ()
+
+
+def install(
+ transformer: Any,
+ fqns: Any,
+ *,
+ logger: Any = None,
+) -> int:
+ """Swap the named weight-only Linears onto the GEMV forward (class swap, no hook); returns how many. A Linear
+ carrying outlier columns is swapped even with the kernel off: its forward is what adds them back."""
+ from torch import nn
+
+ global _OP_HANDLE
+ kernel = gemv_enabled() and _kernel() is not None and _op() is not None
+ _OP_HANDLE = _op() if kernel else None
+ modules = dict(transformer.named_modules())
+ count = 0
+ with _LOCK:
+ for fqn in fqns:
+ module = modules.get(fqn)
+ if type(module) is not nn.Linear and not is_gemv_linear(module):
+ continue
+ if weight_only_parts(module.weight) is None or (
+ not kernel and _outliers(module)[0] is None
+ ):
+ continue
+ module.__class__ = gemv_linear_class()
+ count += 1
+ if logger is not None and count:
+ logger.info(
+ "diffusion.int8_gemv: %s on %d weight-only Linear(s)",
+ "fused GEMV" if kernel else "outlier columns (stock GEMM)",
+ count,
+ )
+ return count
+
+
+def call_count() -> int:
+ return COUNTS["gemv"]
diff --git a/studio/backend/core/inference/diffusion_prequant.py b/studio/backend/core/inference/diffusion_prequant.py
index 3f59234ca54..a4c153d85df 100644
--- a/studio/backend/core/inference/diffusion_prequant.py
+++ b/studio/backend/core/inference/diffusion_prequant.py
@@ -2344,6 +2344,7 @@ def load_prequantized_transformer(
transformer = transformer_cls.from_config(config)
if prepare_model is not None:
prepare_model(transformer, metadata)
+ _prepare_outlier_buffers(transformer, state_dict)
transformer.load_state_dict(state_dict, strict = True, assign = True)
if into is None and _has_meta_tensors(transformer):
# Non-persistent buffers (built in __init__, absent from the state dict) stay on meta. Rebuild on CPU so
@@ -2353,6 +2354,7 @@ def load_prequantized_transformer(
# mismatches.
if prepare_model is not None:
prepare_model(transformer, metadata)
+ _prepare_outlier_buffers(transformer, state_dict)
transformer.load_state_dict(state_dict, strict = True, assign = True)
# Online half of the rotation, applied here so no route skips it: after load_state_dict (meta retry rebuilds), before small-M padding reparents Linears.
@@ -2406,6 +2408,7 @@ def load_prequantized_transformer(
apply_small_m_padding,
apply_zero_row_guard,
quantize_runtime_leftovers,
+ quantize_streamed_weight_only,
)
# Must run before padding (it reparents Linears); a failure keeps the artifact, leftovers stay dense.
@@ -2421,6 +2424,14 @@ def load_prequantized_transformer(
)
except Exception as exc: # noqa: BLE001 - optimisation only
_warn(logger, f"{scheme}:runtime leftovers", exc)
+ # Streamed scope only; a no-op on an artifact whose modulations are already weight-only. Never raises.
+ quantize_streamed_weight_only(
+ transformer,
+ scheme,
+ metadata.get("family") or family,
+ min_features = int(metadata.get("min_features") or min_features or 512),
+ logger = logger,
+ )
apply_small_m_padding(transformer, scheme, metadata.get("family"), logger = logger)
apply_zero_row_guard(transformer, scheme, metadata.get("family"), logger = logger)
@@ -2975,6 +2986,20 @@ def _validate_checkpoint(
),
)
return False
+ # Weight-only modulations are a streamed-build layout: refuse them where this runtime keeps them dense.
+ ckpt_weight_only = meta.get("weight_only_name_tokens")
+ if ckpt_weight_only:
+ from .diffusion_transformer_quant import streamed_weight_only_tokens_for_scheme
+ allowed = set(streamed_weight_only_tokens_for_scheme(scheme, meta.get("family")))
+ if not isinstance(ckpt_weight_only, (list, tuple)) or not set(ckpt_weight_only) <= allowed:
+ _warn(
+ logger,
+ scheme,
+ ValueError(
+ f"checkpoint stores {ckpt_weight_only!r} weight-only; this load keeps them dense"
+ ),
+ )
+ return False
# Verify recorded require_bf16 so a _REQUIRE_BF16_SCHEMES change can't load old-filter checkpoints.
ckpt_require_bf16 = meta.get("require_bf16")
if ckpt_require_bf16 is not None:
@@ -3299,6 +3324,11 @@ class PrequantIntoError(RuntimeError):
"""The module is neither dense nor fully loaded."""
+def _prepare_outlier_buffers(transformer: Any, state_dict: dict) -> int:
+ from .diffusion_int8_gemv import prepare_outlier_buffers
+ return prepare_outlier_buffers(transformer, state_dict)
+
+
def _load_into_dense(transformer: Any, state_dict: dict, prepare_model: Optional[Any]) -> None:
"""ValueError leaves the module untouched; PrequantIntoError means it was partly assigned."""
if prepare_model is not None:
@@ -3306,6 +3336,16 @@ def _load_into_dense(transformer: Any, state_dict: dict, prepare_model: Optional
"prepare_model reshapes a fresh skeleton; it cannot run on an existing transformer"
)
own = transformer.state_dict()
+ from .diffusion_int8_gemv import OUTLIER_IDX, OUTLIER_WEIGHT
+
+ # A streamed weight-only artifact also stores each modulation's dense outlier columns; the skeleton gets them below.
+ outlier_keys = {
+ k
+ for k in state_dict
+ if k.endswith((f".{OUTLIER_WEIGHT}", f".{OUTLIER_IDX}")) and k not in own
+ }
+ for name in outlier_keys:
+ own[name] = state_dict[name]
if set(own) != set(state_dict):
missing = sorted(set(own) - set(state_dict))[:3]
extra = sorted(set(state_dict) - set(own))[:3]
@@ -3320,6 +3360,8 @@ def _load_into_dense(transformer: Any, state_dict: dict, prepare_model: Optional
)
del own
try:
+ if outlier_keys:
+ _prepare_outlier_buffers(transformer, state_dict)
transformer.load_state_dict(state_dict, strict = True, assign = True)
except Exception as exc: # noqa: BLE001
raise PrequantIntoError(f"{type(exc).__name__}: {exc}") from exc
diff --git a/studio/backend/core/inference/diffusion_prequant_cache.py b/studio/backend/core/inference/diffusion_prequant_cache.py
index 9cd4bccf69f..f2c00116077 100644
--- a/studio/backend/core/inference/diffusion_prequant_cache.py
+++ b/studio/backend/core/inference/diffusion_prequant_cache.py
@@ -354,9 +354,12 @@ def key_parts(
divisible_for_scheme,
exclude_tokens_for_scheme,
int8_convrot_enabled,
+ streamed_weight_only_layout,
)
group, suffixes = convrot_spec_for_scheme(scheme, family)
+ # Only when set (streamed qwen-image int8), so every other build keeps its existing key.
+ extra = streamed_weight_only_layout(scheme, family)
return {
"policy": POLICY_VERSION,
"family": str(family or ""),
@@ -373,6 +376,7 @@ def key_parts(
if group and int8_convrot_enabled(family)
else None,
"fast_accum": _resolve_fast_accum(None) if scheme == TQ_FP8 else None,
+ **extra,
**_toolchain(),
}
@@ -1189,6 +1193,14 @@ def save_runtime_quant(
"skipped (fp8 quantized with a forced non-default accumulate mode)",
)
return None
+ wo_keys = ("weight_only_name_tokens", "weight_only_mode", "weight_only_outliers")
+ wanted_wo = {k: parts[k] for k in wo_keys if k in parts}
+ if {k: info[k] for k in wo_keys if k in info} != wanted_wo:
+ _log(logger, "info", "skipped (weight-only modulation state differs from the key)")
+ return None
+ if wanted_wo and wanted_wo["weight_only_mode"] != "wo":
+ _log(logger, "info", "skipped (diagnostic W8A8 modulation mode)")
+ return None
if bool(info.get("convrot_fqns")) != bool(parts["convrot"]):
_log(logger, "info", "skipped (ConvRot state differs from the key)")
return None
@@ -1266,6 +1278,9 @@ def save_runtime_quant(
if scheme == "fp8":
from .diffusion_transformer_quant import FP8_GRANULARITY
metadata["fp8_granularity"] = FP8_GRANULARITY
+ if wanted_wo:
+ metadata["weight_only_name_tokens"] = list(wanted_wo["weight_only_name_tokens"])
+ metadata["weight_only_outliers"] = int(wanted_wo["weight_only_outliers"])
metadata.update(_rotation(transformer, info))
header[UNSLOTH_FORMAT_KEY] = prequant_format_for(metadata)
header[UNSLOTH_METADATA_KEY] = json.dumps(metadata, default = str)
diff --git a/studio/backend/core/inference/diffusion_transformer_quant.py b/studio/backend/core/inference/diffusion_transformer_quant.py
index 39e9d2493d6..221a1630348 100644
--- a/studio/backend/core/inference/diffusion_transformer_quant.py
+++ b/studio/backend/core/inference/diffusion_transformer_quant.py
@@ -109,6 +109,181 @@ def runtime_quant_tokens_for_scheme(scheme: str, family: Optional[str] = None) -
return _INT8_FAMILY_RUNTIME_QUANT_TOKENS.get(str(family or "").strip().lower(), ())
+_INT8_FAMILY_STREAMED_WO_TOKENS: dict[str, tuple[str, ...]] = _profile_table(
+ "int8_streamed_weight_only_tokens", quant = True
+)
+# Kill switches (=0): UNSLOTH_DIFFUSION_INT8_MODULATION keeps modulations bf16, UNSLOTH_DIFFUSION_INT8_GEMV uses
+# torchao's forward, UNSLOTH_DIFFUSION_INT8_MOD_OUTLIERS keeps no dense outlier columns.
+INT8_MODULATION_ENV = "UNSLOTH_DIFFUSION_INT8_MODULATION"
+
+
+INT8_MOD_OUTLIERS_ENV = "UNSLOTH_DIFFUSION_INT8_MOD_OUTLIERS"
+DEFAULT_MOD_OUTLIERS = 8
+
+
+def int8_modulation_mode() -> str:
+ """``wo`` (default) int8 weight-only; ``w8a8`` dynamic int8 activations, row-padded; ``w8a8rot`` the same behind a
+ ConvRot rotation (both diagnostic); ``off``."""
+ raw = (_os.environ.get(INT8_MODULATION_ENV) or "").strip().lower()
+ if raw in ("0", "off", "false", "no", "none"):
+ return "off"
+ return raw if raw in ("w8a8", "w8a8rot") else "wo"
+
+
+def int8_mod_outliers() -> int:
+ """Timestep-embedding outlier channels the weight-only modulations keep dense (0 = none)."""
+ raw = (_os.environ.get(INT8_MOD_OUTLIERS_ENV) or "").strip()
+ try:
+ return max(0, min(16, int(raw))) if raw else DEFAULT_MOD_OUTLIERS
+ except ValueError:
+ return DEFAULT_MOD_OUTLIERS
+
+
+def streamed_weight_only_tokens_for_scheme(
+ scheme: str, family: Optional[str] = None
+) -> tuple[str, ...]:
+ """Name tokens a streamed int8 build stores weight-only; () outside ``int8_text_stream_scope(True)``."""
+ if scheme != TQ_INT8 or not _TEXT_STREAM_SCOPE.get() or int8_modulation_mode() == "off":
+ return ()
+ return _INT8_FAMILY_STREAMED_WO_TOKENS.get(str(family or "").strip().lower(), ())
+
+
+def _weight_only_config() -> Any:
+ """torchao int8 weight-only (Int8Tensor, per-row weight scale, no activation quant) or None on an old torchao."""
+ from torchao.quantization import Int8WeightOnlyConfig
+
+ if "version" not in _inspect.signature(Int8WeightOnlyConfig).parameters:
+ return None
+ from torchao.quantization import Int8Tensor # noqa: F401 - 0.14 has the config but no Int8Tensor
+
+ return _quiet_config(Int8WeightOnlyConfig, version = 2)
+
+
+def streamed_weight_only_layout(scheme: str, family: Optional[str] = None) -> dict:
+ """What a streamed build stores for the weight-only modulations (the prequant-cache key carries it): {} outside the
+ scope, else the name tokens, the mode and the dense outlier-channel count."""
+ tokens = streamed_weight_only_tokens_for_scheme(scheme, family)
+ if not tokens:
+ return {}
+ mode = int8_modulation_mode()
+ return {
+ "weight_only_name_tokens": sorted(tokens),
+ "weight_only_mode": mode,
+ "weight_only_outliers": int8_mod_outliers() if mode == "wo" else 0,
+ }
+
+
+def _modulation_fqns(
+ transformer: Any, tokens: tuple, min_features: int, *, quantized: bool
+) -> list:
+ import torch
+
+ from .diffusion_int8_gemv import weight_only_parts
+ from .diffusion_quant_pad import is_quantized_linear
+
+ out = []
+ for name, module in transformer.named_modules():
+ lname = name.lower()
+ if (
+ not isinstance(module, torch.nn.Linear)
+ or "lora_" in lname
+ or not any(t in lname for t in tokens)
+ ):
+ continue
+ if module.in_features < min_features or module.out_features < min_features:
+ continue
+ if quantized:
+ if weight_only_parts(module.weight) is not None:
+ out.append(name)
+ elif getattr(module.weight, "dtype", None) == torch.bfloat16 and not is_quantized_linear(
+ module
+ ):
+ out.append(name)
+ return out
+
+
+def quantize_streamed_weight_only(
+ transformer: Any,
+ scheme: str,
+ family: Optional[str],
+ *,
+ min_features: int = DEFAULT_MIN_LINEAR_FEATURES,
+ logger: Any = None,
+) -> tuple[str, ...]:
+ """Quantise the still-dense Linears named by ``streamed_weight_only_tokens_for_scheme``; returns their fqns.
+ Weight-only (default): the timestep-embedding outlier channels stay dense beside the int8 codes and the fused GEMV
+ runs them (``diffusion_int8_gemv``). An artifact loaded with them already weight-only only gets the GEMV forward.
+ Best effort: any failure leaves them dense (the build is still valid, only bigger)."""
+ layout = streamed_weight_only_layout(scheme, family)
+ if not layout:
+ return ()
+ tokens = tuple(layout["weight_only_name_tokens"])
+ mode = layout["weight_only_mode"]
+ used = 0
+ try:
+ from torchao.quantization import quantize_
+
+ from .diffusion_int8_gemv import install, split_outliers_, timestep_outlier_channels
+ from .diffusion_quant_pad import wrap_small_m_linears
+
+ names = _modulation_fqns(transformer, tokens, min_features, quantized = False)
+ if not names:
+ if mode == "wo":
+ install(
+ transformer,
+ _modulation_fqns(transformer, tokens, min_features, quantized = True),
+ logger = logger,
+ )
+ return ()
+ config = _weight_only_config() if mode == "wo" else _make_quant_config(scheme)
+ if config is None:
+ return ()
+ wanted = frozenset(names)
+ outliers = (
+ timestep_outlier_channels(transformer, layout["weight_only_outliers"])
+ if mode == "wo"
+ else ()
+ )
+ if outliers:
+ modules = dict(transformer.named_modules())
+ used = min(split_outliers_(modules[name], outliers) for name in names)
+ if mode == "w8a8rot":
+ from .diffusion_convrot import DEFAULT_CONVROT_GROUPSIZE, rotate_linears_
+ rotate_linears_(
+ transformer,
+ [
+ n
+ for n in names
+ if transformer.get_submodule(n).in_features % DEFAULT_CONVROT_GROUPSIZE == 0
+ ],
+ )
+ quantize_(transformer, config, filter_fn = lambda _m, fqn = "": fqn in wanted)
+ if mode == "wo":
+ install(transformer, names, logger = logger)
+ else:
+ wrap_small_m_linears(transformer, names)
+ except Exception as exc: # noqa: BLE001 - optimisation only: the rest of the build stands
+ _warn(logger, f"{scheme}:streamed weight-only", exc)
+ return ()
+ try:
+ transformer._unsloth_weight_only_layout = {**layout, "weight_only_outliers": used}
+ except Exception: # noqa: BLE001 - marker is best-effort
+ pass
+ if logger is not None:
+ logger.info(
+ "diffusion.transformer_quant: %s %d streamed modulation Linear(s) (%s; %d dense outlier channels)",
+ {
+ "wo": "int8 weight-only",
+ "w8a8": "int8 W8A8 padded",
+ "w8a8rot": "int8 W8A8 ConvRot padded",
+ }[mode],
+ len(names),
+ ", ".join(tokens),
+ used,
+ )
+ return tuple(names)
+
+
# ALL-scheme PER-FAMILY exclusions (``FamilyProfile.quant_exclude_tokens``): unlike the int8 table above, these also
# apply to fp8 / mxfp8 / nvfp4, whose scaled_mm paths otherwise quantise every eligible Linear.
_FAMILY_QUANT_EXCLUDE_NAME_TOKENS: dict[str, tuple[str, ...]] = _profile_table(
@@ -1813,6 +1988,9 @@ def quantize_transformer(
_make_quant_config(scheme, fast_accum = fast_accum),
filter_fn = filter_fn,
)
+ weight_only = quantize_streamed_weight_only(
+ transformer, scheme, family, min_features = min_features, logger = logger
+ )
# Not best-effort: a raise means quantized but not compilable, so the caller loads GGUF.
apply_small_m_padding(transformer, scheme, family, logger = logger)
apply_zero_row_guard(transformer, scheme, family, logger = logger)
@@ -1831,6 +2009,10 @@ def quantize_transformer(
"convrot_fqns": list(rotated or ()),
"dtype": str(getattr(target, "dtype", None) or _first_weight_dtype(transformer)),
}
+ if weight_only:
+ transformer._unsloth_runtime_quant_info.update(
+ getattr(transformer, "_unsloth_weight_only_layout", None) or {}
+ )
except Exception: # noqa: BLE001 - marker is best-effort
pass
return scheme
diff --git a/studio/backend/core/inference/family_profile.py b/studio/backend/core/inference/family_profile.py
index 826eeb6204c..e4bb01aa369 100644
--- a/studio/backend/core/inference/family_profile.py
+++ b/studio/backend/core/inference/family_profile.py
@@ -59,6 +59,8 @@ class FamilyProfile:
int8_exclude_tokens: tuple[str, ...] = ()
# Subset of ``int8_exclude_tokens`` a STREAMED build quantises anyway (``int8_text_stream_scope``).
int8_runtime_quant_tokens: tuple[str, ...] = ()
+ # Generic-excluded Linears a STREAMED build stores int8 weight-only.
+ int8_streamed_weight_only_tokens: tuple[str, ...] = ()
# Name tokens left dense for EVERY transformer-quant scheme (int8 / fp8 / mxfp8 / nvfp4), on top of the int8
# tokens above. For layers whose quantisation error is measured to dominate the image delta regardless of the
# scheme (FLUX.3: per-token modulations, the 20480-wide text projection, the text-stream per-mode blocks).
diff --git a/studio/backend/tests/fixtures/diffusion_family_data_freeze.json b/studio/backend/tests/fixtures/diffusion_family_data_freeze.json
index a8bf5321baf..7f0bf92adef 100644
--- a/studio/backend/tests/fixtures/diffusion_family_data_freeze.json
+++ b/studio/backend/tests/fixtures/diffusion_family_data_freeze.json
@@ -593,6 +593,7 @@
"exclude": [],
"pad": [],
"runtime_quant": [],
+ "streamed_weight_only": [],
"zero_row": []
},
"auto_bf16_when_resident": null,
@@ -607,6 +608,7 @@
"exclude": [],
"pad": [],
"runtime_quant": [],
+ "streamed_weight_only": [],
"zero_row": []
},
"convrot_default_on": false,
@@ -621,6 +623,7 @@
"exclude": [],
"pad": [],
"runtime_quant": [],
+ "streamed_weight_only": [],
"zero_row": []
},
"int8": {
@@ -643,6 +646,7 @@
],
"pad": [],
"runtime_quant": [],
+ "streamed_weight_only": [],
"zero_row": []
},
"mxfp8": {
@@ -656,6 +660,7 @@
"exclude": [],
"pad": [],
"runtime_quant": [],
+ "streamed_weight_only": [],
"zero_row": []
},
"nvfp4": {
@@ -669,6 +674,7 @@
"exclude": [],
"pad": [],
"runtime_quant": [],
+ "streamed_weight_only": [],
"zero_row": []
},
"page": null,
@@ -687,6 +693,7 @@
"exclude": [],
"pad": [],
"runtime_quant": [],
+ "streamed_weight_only": [],
"zero_row": []
},
"auto_bf16_when_resident": null,
@@ -701,6 +708,7 @@
"exclude": [],
"pad": [],
"runtime_quant": [],
+ "streamed_weight_only": [],
"zero_row": []
},
"convrot_default_on": false,
@@ -715,6 +723,7 @@
"exclude": [],
"pad": [],
"runtime_quant": [],
+ "streamed_weight_only": [],
"zero_row": []
},
"int8": {
@@ -749,6 +758,10 @@
"to_add_out",
"txt_mlp"
],
+ "streamed_weight_only": [
+ "img_mod",
+ "txt_mod"
+ ],
"zero_row": []
},
"mxfp8": {
@@ -762,6 +775,7 @@
"exclude": [],
"pad": [],
"runtime_quant": [],
+ "streamed_weight_only": [],
"zero_row": []
},
"nvfp4": {
@@ -775,6 +789,7 @@
"exclude": [],
"pad": [],
"runtime_quant": [],
+ "streamed_weight_only": [],
"zero_row": []
},
"page": "image",
@@ -858,6 +873,7 @@
"exclude": [],
"pad": [],
"runtime_quant": [],
+ "streamed_weight_only": [],
"zero_row": []
},
"auto_bf16_when_resident": null,
@@ -872,6 +888,7 @@
"exclude": [],
"pad": [],
"runtime_quant": [],
+ "streamed_weight_only": [],
"zero_row": []
},
"convrot_default_on": false,
@@ -886,6 +903,7 @@
"exclude": [],
"pad": [],
"runtime_quant": [],
+ "streamed_weight_only": [],
"zero_row": []
},
"int8": {
@@ -920,6 +938,10 @@
"to_add_out",
"txt_mlp"
],
+ "streamed_weight_only": [
+ "img_mod",
+ "txt_mod"
+ ],
"zero_row": []
},
"mxfp8": {
@@ -933,6 +955,7 @@
"exclude": [],
"pad": [],
"runtime_quant": [],
+ "streamed_weight_only": [],
"zero_row": []
},
"nvfp4": {
@@ -946,6 +969,7 @@
"exclude": [],
"pad": [],
"runtime_quant": [],
+ "streamed_weight_only": [],
"zero_row": []
},
"page": "image",
@@ -964,6 +988,7 @@
"exclude": [],
"pad": [],
"runtime_quant": [],
+ "streamed_weight_only": [],
"zero_row": []
},
"auto_bf16_when_resident": null,
@@ -978,6 +1003,7 @@
"exclude": [],
"pad": [],
"runtime_quant": [],
+ "streamed_weight_only": [],
"zero_row": []
},
"convrot_default_on": false,
@@ -992,6 +1018,7 @@
"exclude": [],
"pad": [],
"runtime_quant": [],
+ "streamed_weight_only": [],
"zero_row": []
},
"int8": {
@@ -1014,6 +1041,7 @@
],
"pad": [],
"runtime_quant": [],
+ "streamed_weight_only": [],
"zero_row": []
},
"mxfp8": {
@@ -1027,6 +1055,7 @@
"exclude": [],
"pad": [],
"runtime_quant": [],
+ "streamed_weight_only": [],
"zero_row": []
},
"nvfp4": {
@@ -1040,6 +1069,7 @@
"exclude": [],
"pad": [],
"runtime_quant": [],
+ "streamed_weight_only": [],
"zero_row": []
},
"page": "video",
@@ -1058,6 +1088,7 @@
"exclude": [],
"pad": [],
"runtime_quant": [],
+ "streamed_weight_only": [],
"zero_row": []
},
"auto_bf16_when_resident": null,
@@ -1072,6 +1103,7 @@
"exclude": [],
"pad": [],
"runtime_quant": [],
+ "streamed_weight_only": [],
"zero_row": []
},
"convrot_default_on": false,
@@ -1086,6 +1118,7 @@
"exclude": [],
"pad": [],
"runtime_quant": [],
+ "streamed_weight_only": [],
"zero_row": []
},
"int8": {
@@ -1108,6 +1141,7 @@
],
"pad": [],
"runtime_quant": [],
+ "streamed_weight_only": [],
"zero_row": []
},
"mxfp8": {
@@ -1121,6 +1155,7 @@
"exclude": [],
"pad": [],
"runtime_quant": [],
+ "streamed_weight_only": [],
"zero_row": []
},
"nvfp4": {
@@ -1134,6 +1169,7 @@
"exclude": [],
"pad": [],
"runtime_quant": [],
+ "streamed_weight_only": [],
"zero_row": []
},
"page": "video",
@@ -1152,6 +1188,7 @@
"exclude": [],
"pad": [],
"runtime_quant": [],
+ "streamed_weight_only": [],
"zero_row": []
},
"auto_bf16_when_resident": null,
@@ -1166,6 +1203,7 @@
"exclude": [],
"pad": [],
"runtime_quant": [],
+ "streamed_weight_only": [],
"zero_row": []
},
"convrot_default_on": false,
@@ -1180,6 +1218,7 @@
"exclude": [],
"pad": [],
"runtime_quant": [],
+ "streamed_weight_only": [],
"zero_row": []
},
"int8": {
@@ -1202,6 +1241,7 @@
],
"pad": [],
"runtime_quant": [],
+ "streamed_weight_only": [],
"zero_row": []
},
"mxfp8": {
@@ -1215,6 +1255,7 @@
"exclude": [],
"pad": [],
"runtime_quant": [],
+ "streamed_weight_only": [],
"zero_row": []
},
"nvfp4": {
@@ -1228,6 +1269,7 @@
"exclude": [],
"pad": [],
"runtime_quant": [],
+ "streamed_weight_only": [],
"zero_row": []
},
"page": "image",
@@ -1246,6 +1288,7 @@
"exclude": [],
"pad": [],
"runtime_quant": [],
+ "streamed_weight_only": [],
"zero_row": []
},
"auto_bf16_when_resident": null,
@@ -1260,6 +1303,7 @@
"exclude": [],
"pad": [],
"runtime_quant": [],
+ "streamed_weight_only": [],
"zero_row": []
},
"convrot_default_on": false,
@@ -1274,6 +1318,7 @@
"exclude": [],
"pad": [],
"runtime_quant": [],
+ "streamed_weight_only": [],
"zero_row": []
},
"int8": {
@@ -1296,6 +1341,7 @@
],
"pad": [],
"runtime_quant": [],
+ "streamed_weight_only": [],
"zero_row": []
},
"mxfp8": {
@@ -1309,6 +1355,7 @@
"exclude": [],
"pad": [],
"runtime_quant": [],
+ "streamed_weight_only": [],
"zero_row": []
},
"nvfp4": {
@@ -1322,6 +1369,7 @@
"exclude": [],
"pad": [],
"runtime_quant": [],
+ "streamed_weight_only": [],
"zero_row": []
},
"page": "image",
@@ -1340,6 +1388,7 @@
"exclude": [],
"pad": [],
"runtime_quant": [],
+ "streamed_weight_only": [],
"zero_row": []
},
"auto_bf16_when_resident": null,
@@ -1354,6 +1403,7 @@
"exclude": [],
"pad": [],
"runtime_quant": [],
+ "streamed_weight_only": [],
"zero_row": []
},
"convrot_default_on": false,
@@ -1368,6 +1418,7 @@
"exclude": [],
"pad": [],
"runtime_quant": [],
+ "streamed_weight_only": [],
"zero_row": []
},
"int8": {
@@ -1390,6 +1441,7 @@
],
"pad": [],
"runtime_quant": [],
+ "streamed_weight_only": [],
"zero_row": []
},
"mxfp8": {
@@ -1403,6 +1455,7 @@
"exclude": [],
"pad": [],
"runtime_quant": [],
+ "streamed_weight_only": [],
"zero_row": []
},
"nvfp4": {
@@ -1416,6 +1469,7 @@
"exclude": [],
"pad": [],
"runtime_quant": [],
+ "streamed_weight_only": [],
"zero_row": []
},
"page": "image",
@@ -1434,6 +1488,7 @@
"exclude": [],
"pad": [],
"runtime_quant": [],
+ "streamed_weight_only": [],
"zero_row": []
},
"auto_bf16_when_resident": "a small denoiser stays bf16 while it fits (INT8 / FP8 change image detail: FP8 LPIPS 0.065 on FLUX.2-klein-4B, 0.103 on Z-Image-Turbo)",
@@ -1448,6 +1503,7 @@
"exclude": [],
"pad": [],
"runtime_quant": [],
+ "streamed_weight_only": [],
"zero_row": []
},
"convrot_default_on": false,
@@ -1462,6 +1518,7 @@
"exclude": [],
"pad": [],
"runtime_quant": [],
+ "streamed_weight_only": [],
"zero_row": []
},
"int8": {
@@ -1484,6 +1541,7 @@
],
"pad": [],
"runtime_quant": [],
+ "streamed_weight_only": [],
"zero_row": []
},
"mxfp8": {
@@ -1497,6 +1555,7 @@
"exclude": [],
"pad": [],
"runtime_quant": [],
+ "streamed_weight_only": [],
"zero_row": []
},
"nvfp4": {
@@ -1510,6 +1569,7 @@
"exclude": [],
"pad": [],
"runtime_quant": [],
+ "streamed_weight_only": [],
"zero_row": []
},
"page": "image",
@@ -1534,6 +1594,7 @@
],
"pad": [],
"runtime_quant": [],
+ "streamed_weight_only": [],
"zero_row": []
},
"auto_bf16_when_resident": "a small denoiser stays bf16 while it fits (INT8 / FP8 change image detail: FP8 LPIPS 0.065 on FLUX.2-klein-4B, 0.103 on Z-Image-Turbo)",
@@ -1554,6 +1615,7 @@
],
"pad": [],
"runtime_quant": [],
+ "streamed_weight_only": [],
"zero_row": []
},
"convrot_default_on": false,
@@ -1574,6 +1636,7 @@
],
"pad": [],
"runtime_quant": [],
+ "streamed_weight_only": [],
"zero_row": []
},
"int8": {
@@ -1601,6 +1664,7 @@
],
"pad": [],
"runtime_quant": [],
+ "streamed_weight_only": [],
"zero_row": []
},
"mxfp8": {
@@ -1620,6 +1684,7 @@
],
"pad": [],
"runtime_quant": [],
+ "streamed_weight_only": [],
"zero_row": []
},
"nvfp4": {
@@ -1639,6 +1704,7 @@
],
"pad": [],
"runtime_quant": [],
+ "streamed_weight_only": [],
"zero_row": []
},
"page": "image",
@@ -1663,6 +1729,7 @@
],
"pad": [],
"runtime_quant": [],
+ "streamed_weight_only": [],
"zero_row": []
},
"auto_bf16_when_resident": "a small denoiser stays bf16 while it fits (INT8 / FP8 change image detail: FP8 LPIPS 0.065 on FLUX.2-klein-4B, 0.103 on Z-Image-Turbo)",
@@ -1683,6 +1750,7 @@
],
"pad": [],
"runtime_quant": [],
+ "streamed_weight_only": [],
"zero_row": []
},
"convrot_default_on": false,
@@ -1703,6 +1771,7 @@
],
"pad": [],
"runtime_quant": [],
+ "streamed_weight_only": [],
"zero_row": []
},
"int8": {
@@ -1730,6 +1799,7 @@
],
"pad": [],
"runtime_quant": [],
+ "streamed_weight_only": [],
"zero_row": []
},
"mxfp8": {
@@ -1749,6 +1819,7 @@
],
"pad": [],
"runtime_quant": [],
+ "streamed_weight_only": [],
"zero_row": []
},
"nvfp4": {
@@ -1768,6 +1839,7 @@
],
"pad": [],
"runtime_quant": [],
+ "streamed_weight_only": [],
"zero_row": []
},
"page": "image",
@@ -1786,6 +1858,7 @@
"exclude": [],
"pad": [],
"runtime_quant": [],
+ "streamed_weight_only": [],
"zero_row": []
},
"auto_bf16_when_resident": "INT8 is barely faster than compiled bf16 on a card that holds bf16 and changes image detail",
@@ -1800,6 +1873,7 @@
"exclude": [],
"pad": [],
"runtime_quant": [],
+ "streamed_weight_only": [],
"zero_row": []
},
"convrot_default_on": false,
@@ -1814,6 +1888,7 @@
"exclude": [],
"pad": [],
"runtime_quant": [],
+ "streamed_weight_only": [],
"zero_row": []
},
"int8": {
@@ -1836,6 +1911,7 @@
],
"pad": [],
"runtime_quant": [],
+ "streamed_weight_only": [],
"zero_row": []
},
"mxfp8": {
@@ -1849,6 +1925,7 @@
"exclude": [],
"pad": [],
"runtime_quant": [],
+ "streamed_weight_only": [],
"zero_row": []
},
"nvfp4": {
@@ -1862,6 +1939,7 @@
"exclude": [],
"pad": [],
"runtime_quant": [],
+ "streamed_weight_only": [],
"zero_row": []
},
"page": "image",
@@ -1880,6 +1958,7 @@
"exclude": [],
"pad": [],
"runtime_quant": [],
+ "streamed_weight_only": [],
"zero_row": []
},
"auto_bf16_when_resident": null,
@@ -1894,6 +1973,7 @@
"exclude": [],
"pad": [],
"runtime_quant": [],
+ "streamed_weight_only": [],
"zero_row": []
},
"convrot_default_on": false,
@@ -1908,6 +1988,7 @@
"exclude": [],
"pad": [],
"runtime_quant": [],
+ "streamed_weight_only": [],
"zero_row": []
},
"int8": {
@@ -1930,6 +2011,7 @@
],
"pad": [],
"runtime_quant": [],
+ "streamed_weight_only": [],
"zero_row": []
},
"mxfp8": {
@@ -1943,6 +2025,7 @@
"exclude": [],
"pad": [],
"runtime_quant": [],
+ "streamed_weight_only": [],
"zero_row": []
},
"nvfp4": {
@@ -1956,6 +2039,7 @@
"exclude": [],
"pad": [],
"runtime_quant": [],
+ "streamed_weight_only": [],
"zero_row": []
},
"page": "image",
@@ -1974,6 +2058,7 @@
"exclude": [],
"pad": [],
"runtime_quant": [],
+ "streamed_weight_only": [],
"zero_row": []
},
"auto_bf16_when_resident": null,
@@ -1988,6 +2073,7 @@
"exclude": [],
"pad": [],
"runtime_quant": [],
+ "streamed_weight_only": [],
"zero_row": []
},
"convrot_default_on": false,
@@ -2002,6 +2088,7 @@
"exclude": [],
"pad": [],
"runtime_quant": [],
+ "streamed_weight_only": [],
"zero_row": []
},
"int8": {
@@ -2031,6 +2118,7 @@
],
"pad": [],
"runtime_quant": [],
+ "streamed_weight_only": [],
"zero_row": []
},
"mxfp8": {
@@ -2044,6 +2132,7 @@
"exclude": [],
"pad": [],
"runtime_quant": [],
+ "streamed_weight_only": [],
"zero_row": []
},
"nvfp4": {
@@ -2057,6 +2146,7 @@
"exclude": [],
"pad": [],
"runtime_quant": [],
+ "streamed_weight_only": [],
"zero_row": [
"image_embedder",
"context_embedder_2"
@@ -2078,6 +2168,7 @@
"exclude": [],
"pad": [],
"runtime_quant": [],
+ "streamed_weight_only": [],
"zero_row": []
},
"auto_bf16_when_resident": null,
@@ -2092,6 +2183,7 @@
"exclude": [],
"pad": [],
"runtime_quant": [],
+ "streamed_weight_only": [],
"zero_row": []
},
"convrot_default_on": false,
@@ -2106,6 +2198,7 @@
"exclude": [],
"pad": [],
"runtime_quant": [],
+ "streamed_weight_only": [],
"zero_row": []
},
"int8": {
@@ -2135,6 +2228,7 @@
],
"pad": [],
"runtime_quant": [],
+ "streamed_weight_only": [],
"zero_row": []
},
"mxfp8": {
@@ -2148,6 +2242,7 @@
"exclude": [],
"pad": [],
"runtime_quant": [],
+ "streamed_weight_only": [],
"zero_row": []
},
"nvfp4": {
@@ -2161,6 +2256,7 @@
"exclude": [],
"pad": [],
"runtime_quant": [],
+ "streamed_weight_only": [],
"zero_row": [
"image_embedder",
"context_embedder_2"
@@ -2182,6 +2278,7 @@
"exclude": [],
"pad": [],
"runtime_quant": [],
+ "streamed_weight_only": [],
"zero_row": []
},
"auto_bf16_when_resident": null,
@@ -2196,6 +2293,7 @@
"exclude": [],
"pad": [],
"runtime_quant": [],
+ "streamed_weight_only": [],
"zero_row": []
},
"convrot_default_on": false,
@@ -2210,6 +2308,7 @@
"exclude": [],
"pad": [],
"runtime_quant": [],
+ "streamed_weight_only": [],
"zero_row": []
},
"int8": {
@@ -2232,6 +2331,7 @@
],
"pad": [],
"runtime_quant": [],
+ "streamed_weight_only": [],
"zero_row": []
},
"mxfp8": {
@@ -2245,6 +2345,7 @@
"exclude": [],
"pad": [],
"runtime_quant": [],
+ "streamed_weight_only": [],
"zero_row": []
},
"nvfp4": {
@@ -2258,6 +2359,7 @@
"exclude": [],
"pad": [],
"runtime_quant": [],
+ "streamed_weight_only": [],
"zero_row": []
},
"page": "image",
@@ -2276,6 +2378,7 @@
"exclude": [],
"pad": [],
"runtime_quant": [],
+ "streamed_weight_only": [],
"zero_row": []
},
"auto_bf16_when_resident": null,
@@ -2290,6 +2393,7 @@
"exclude": [],
"pad": [],
"runtime_quant": [],
+ "streamed_weight_only": [],
"zero_row": []
},
"convrot_default_on": false,
@@ -2304,6 +2408,7 @@
"exclude": [],
"pad": [],
"runtime_quant": [],
+ "streamed_weight_only": [],
"zero_row": []
},
"int8": {
@@ -2326,6 +2431,7 @@
],
"pad": [],
"runtime_quant": [],
+ "streamed_weight_only": [],
"zero_row": []
},
"mxfp8": {
@@ -2339,6 +2445,7 @@
"exclude": [],
"pad": [],
"runtime_quant": [],
+ "streamed_weight_only": [],
"zero_row": []
},
"nvfp4": {
@@ -2352,6 +2459,7 @@
"exclude": [],
"pad": [],
"runtime_quant": [],
+ "streamed_weight_only": [],
"zero_row": []
},
"page": "image",
@@ -2370,6 +2478,7 @@
"exclude": [],
"pad": [],
"runtime_quant": [],
+ "streamed_weight_only": [],
"zero_row": []
},
"auto_bf16_when_resident": null,
@@ -2384,6 +2493,7 @@
"exclude": [],
"pad": [],
"runtime_quant": [],
+ "streamed_weight_only": [],
"zero_row": []
},
"convrot_default_on": false,
@@ -2398,6 +2508,7 @@
"exclude": [],
"pad": [],
"runtime_quant": [],
+ "streamed_weight_only": [],
"zero_row": []
},
"int8": {
@@ -2423,6 +2534,7 @@
],
"pad": [],
"runtime_quant": [],
+ "streamed_weight_only": [],
"zero_row": []
},
"mxfp8": {
@@ -2436,6 +2548,7 @@
"exclude": [],
"pad": [],
"runtime_quant": [],
+ "streamed_weight_only": [],
"zero_row": []
},
"nvfp4": {
@@ -2449,6 +2562,7 @@
"exclude": [],
"pad": [],
"runtime_quant": [],
+ "streamed_weight_only": [],
"zero_row": []
},
"page": "video",
@@ -2467,6 +2581,7 @@
"exclude": [],
"pad": [],
"runtime_quant": [],
+ "streamed_weight_only": [],
"zero_row": []
},
"auto_bf16_when_resident": null,
@@ -2481,6 +2596,7 @@
"exclude": [],
"pad": [],
"runtime_quant": [],
+ "streamed_weight_only": [],
"zero_row": []
},
"convrot_default_on": true,
@@ -2495,6 +2611,7 @@
"exclude": [],
"pad": [],
"runtime_quant": [],
+ "streamed_weight_only": [],
"zero_row": []
},
"int8": {
@@ -2522,6 +2639,7 @@
],
"pad": [],
"runtime_quant": [],
+ "streamed_weight_only": [],
"zero_row": []
},
"mxfp8": {
@@ -2535,6 +2653,7 @@
"exclude": [],
"pad": [],
"runtime_quant": [],
+ "streamed_weight_only": [],
"zero_row": []
},
"nvfp4": {
@@ -2548,6 +2667,7 @@
"exclude": [],
"pad": [],
"runtime_quant": [],
+ "streamed_weight_only": [],
"zero_row": []
},
"page": "video",
@@ -2566,6 +2686,7 @@
"exclude": [],
"pad": [],
"runtime_quant": [],
+ "streamed_weight_only": [],
"zero_row": []
},
"auto_bf16_when_resident": "INT8 is no faster than compiled bf16 on a card that holds bf16 and changes image detail",
@@ -2580,6 +2701,7 @@
"exclude": [],
"pad": [],
"runtime_quant": [],
+ "streamed_weight_only": [],
"zero_row": []
},
"convrot_default_on": false,
@@ -2594,6 +2716,7 @@
"exclude": [],
"pad": [],
"runtime_quant": [],
+ "streamed_weight_only": [],
"zero_row": []
},
"int8": {
@@ -2616,6 +2739,7 @@
],
"pad": [],
"runtime_quant": [],
+ "streamed_weight_only": [],
"zero_row": []
},
"mxfp8": {
@@ -2629,6 +2753,7 @@
"exclude": [],
"pad": [],
"runtime_quant": [],
+ "streamed_weight_only": [],
"zero_row": []
},
"nvfp4": {
@@ -2642,6 +2767,7 @@
"exclude": [],
"pad": [],
"runtime_quant": [],
+ "streamed_weight_only": [],
"zero_row": []
},
"page": "image",
@@ -2660,6 +2786,7 @@
"exclude": [],
"pad": [],
"runtime_quant": [],
+ "streamed_weight_only": [],
"zero_row": []
},
"auto_bf16_when_resident": null,
@@ -2674,6 +2801,7 @@
"exclude": [],
"pad": [],
"runtime_quant": [],
+ "streamed_weight_only": [],
"zero_row": []
},
"convrot_default_on": false,
@@ -2688,6 +2816,7 @@
"exclude": [],
"pad": [],
"runtime_quant": [],
+ "streamed_weight_only": [],
"zero_row": []
},
"int8": {
@@ -2714,6 +2843,7 @@
"token_refiner"
],
"runtime_quant": [],
+ "streamed_weight_only": [],
"zero_row": []
},
"mxfp8": {
@@ -2727,6 +2857,7 @@
"exclude": [],
"pad": [],
"runtime_quant": [],
+ "streamed_weight_only": [],
"zero_row": []
},
"nvfp4": {
@@ -2740,6 +2871,7 @@
"exclude": [],
"pad": [],
"runtime_quant": [],
+ "streamed_weight_only": [],
"zero_row": []
},
"page": "video",
@@ -2762,6 +2894,7 @@
],
"pad": [],
"runtime_quant": [],
+ "streamed_weight_only": [],
"zero_row": []
},
"auto_bf16_when_resident": "a small denoiser stays bf16 while it fits (INT8 / FP8 change image detail: FP8 LPIPS 0.065 on FLUX.2-klein-4B, 0.103 on Z-Image-Turbo)",
@@ -2780,6 +2913,7 @@
],
"pad": [],
"runtime_quant": [],
+ "streamed_weight_only": [],
"zero_row": []
},
"convrot_default_on": true,
@@ -2798,6 +2932,7 @@
],
"pad": [],
"runtime_quant": [],
+ "streamed_weight_only": [],
"zero_row": []
},
"int8": {
@@ -2832,6 +2967,7 @@
],
"pad": [],
"runtime_quant": [],
+ "streamed_weight_only": [],
"zero_row": []
},
"mxfp8": {
@@ -2849,6 +2985,7 @@
],
"pad": [],
"runtime_quant": [],
+ "streamed_weight_only": [],
"zero_row": []
},
"nvfp4": {
@@ -2866,6 +3003,7 @@
],
"pad": [],
"runtime_quant": [],
+ "streamed_weight_only": [],
"zero_row": []
},
"page": "image",
@@ -2884,6 +3022,7 @@
"exclude": [],
"pad": [],
"runtime_quant": [],
+ "streamed_weight_only": [],
"zero_row": []
},
"auto_bf16_when_resident": null,
@@ -2898,6 +3037,7 @@
"exclude": [],
"pad": [],
"runtime_quant": [],
+ "streamed_weight_only": [],
"zero_row": []
},
"convrot_default_on": false,
@@ -2912,6 +3052,7 @@
"exclude": [],
"pad": [],
"runtime_quant": [],
+ "streamed_weight_only": [],
"zero_row": []
},
"int8": {
@@ -2946,6 +3087,10 @@
"to_add_out",
"txt_mlp"
],
+ "streamed_weight_only": [
+ "img_mod",
+ "txt_mod"
+ ],
"zero_row": []
},
"mxfp8": {
@@ -2959,6 +3104,7 @@
"exclude": [],
"pad": [],
"runtime_quant": [],
+ "streamed_weight_only": [],
"zero_row": []
},
"nvfp4": {
@@ -2972,6 +3118,7 @@
"exclude": [],
"pad": [],
"runtime_quant": [],
+ "streamed_weight_only": [],
"zero_row": []
},
"page": "image",
@@ -2990,6 +3137,7 @@
"exclude": [],
"pad": [],
"runtime_quant": [],
+ "streamed_weight_only": [],
"zero_row": []
},
"auto_bf16_when_resident": null,
@@ -3004,6 +3152,7 @@
"exclude": [],
"pad": [],
"runtime_quant": [],
+ "streamed_weight_only": [],
"zero_row": []
},
"convrot_default_on": true,
@@ -3018,6 +3167,7 @@
"exclude": [],
"pad": [],
"runtime_quant": [],
+ "streamed_weight_only": [],
"zero_row": []
},
"int8": {
@@ -3052,6 +3202,7 @@
],
"pad": [],
"runtime_quant": [],
+ "streamed_weight_only": [],
"zero_row": []
},
"mxfp8": {
@@ -3065,6 +3216,7 @@
"exclude": [],
"pad": [],
"runtime_quant": [],
+ "streamed_weight_only": [],
"zero_row": []
},
"nvfp4": {
@@ -3078,6 +3230,7 @@
"exclude": [],
"pad": [],
"runtime_quant": [],
+ "streamed_weight_only": [],
"zero_row": []
},
"page": "image",
@@ -3096,6 +3249,7 @@
"exclude": [],
"pad": [],
"runtime_quant": [],
+ "streamed_weight_only": [],
"zero_row": []
},
"auto_bf16_when_resident": null,
@@ -3110,6 +3264,7 @@
"exclude": [],
"pad": [],
"runtime_quant": [],
+ "streamed_weight_only": [],
"zero_row": []
},
"convrot_default_on": false,
@@ -3124,6 +3279,7 @@
"exclude": [],
"pad": [],
"runtime_quant": [],
+ "streamed_weight_only": [],
"zero_row": []
},
"int8": {
@@ -3152,6 +3308,7 @@
],
"pad": [],
"runtime_quant": [],
+ "streamed_weight_only": [],
"zero_row": []
},
"mxfp8": {
@@ -3165,6 +3322,7 @@
"exclude": [],
"pad": [],
"runtime_quant": [],
+ "streamed_weight_only": [],
"zero_row": []
},
"nvfp4": {
@@ -3178,6 +3336,7 @@
"exclude": [],
"pad": [],
"runtime_quant": [],
+ "streamed_weight_only": [],
"zero_row": []
},
"page": "image",
@@ -3196,6 +3355,7 @@
"exclude": [],
"pad": [],
"runtime_quant": [],
+ "streamed_weight_only": [],
"zero_row": []
},
"auto_bf16_when_resident": null,
@@ -3210,6 +3370,7 @@
"exclude": [],
"pad": [],
"runtime_quant": [],
+ "streamed_weight_only": [],
"zero_row": []
},
"convrot_default_on": false,
@@ -3224,6 +3385,7 @@
"exclude": [],
"pad": [],
"runtime_quant": [],
+ "streamed_weight_only": [],
"zero_row": []
},
"int8": {
@@ -3252,6 +3414,7 @@
],
"pad": [],
"runtime_quant": [],
+ "streamed_weight_only": [],
"zero_row": []
},
"mxfp8": {
@@ -3265,6 +3428,7 @@
"exclude": [],
"pad": [],
"runtime_quant": [],
+ "streamed_weight_only": [],
"zero_row": []
},
"nvfp4": {
@@ -3278,6 +3442,7 @@
"exclude": [],
"pad": [],
"runtime_quant": [],
+ "streamed_weight_only": [],
"zero_row": []
},
"page": "image",
@@ -3296,6 +3461,7 @@
"exclude": [],
"pad": [],
"runtime_quant": [],
+ "streamed_weight_only": [],
"zero_row": []
},
"auto_bf16_when_resident": null,
@@ -3310,6 +3476,7 @@
"exclude": [],
"pad": [],
"runtime_quant": [],
+ "streamed_weight_only": [],
"zero_row": []
},
"convrot_default_on": false,
@@ -3324,6 +3491,7 @@
"exclude": [],
"pad": [],
"runtime_quant": [],
+ "streamed_weight_only": [],
"zero_row": []
},
"int8": {
@@ -3346,6 +3514,7 @@
],
"pad": [],
"runtime_quant": [],
+ "streamed_weight_only": [],
"zero_row": []
},
"mxfp8": {
@@ -3359,6 +3528,7 @@
"exclude": [],
"pad": [],
"runtime_quant": [],
+ "streamed_weight_only": [],
"zero_row": []
},
"nvfp4": {
@@ -3372,6 +3542,7 @@
"exclude": [],
"pad": [],
"runtime_quant": [],
+ "streamed_weight_only": [],
"zero_row": []
},
"page": "image",
@@ -3390,6 +3561,7 @@
"exclude": [],
"pad": [],
"runtime_quant": [],
+ "streamed_weight_only": [],
"zero_row": []
},
"auto_bf16_when_resident": null,
@@ -3404,6 +3576,7 @@
"exclude": [],
"pad": [],
"runtime_quant": [],
+ "streamed_weight_only": [],
"zero_row": []
},
"convrot_default_on": false,
@@ -3418,6 +3591,7 @@
"exclude": [],
"pad": [],
"runtime_quant": [],
+ "streamed_weight_only": [],
"zero_row": []
},
"int8": {
@@ -3440,6 +3614,7 @@
],
"pad": [],
"runtime_quant": [],
+ "streamed_weight_only": [],
"zero_row": []
},
"mxfp8": {
@@ -3453,6 +3628,7 @@
"exclude": [],
"pad": [],
"runtime_quant": [],
+ "streamed_weight_only": [],
"zero_row": []
},
"nvfp4": {
@@ -3466,6 +3642,7 @@
"exclude": [],
"pad": [],
"runtime_quant": [],
+ "streamed_weight_only": [],
"zero_row": []
},
"page": "image",
@@ -3484,6 +3661,7 @@
"exclude": [],
"pad": [],
"runtime_quant": [],
+ "streamed_weight_only": [],
"zero_row": []
},
"auto_bf16_when_resident": null,
@@ -3498,6 +3676,7 @@
"exclude": [],
"pad": [],
"runtime_quant": [],
+ "streamed_weight_only": [],
"zero_row": []
},
"convrot_default_on": false,
@@ -3512,6 +3691,7 @@
"exclude": [],
"pad": [],
"runtime_quant": [],
+ "streamed_weight_only": [],
"zero_row": []
},
"int8": {
@@ -3534,6 +3714,7 @@
],
"pad": [],
"runtime_quant": [],
+ "streamed_weight_only": [],
"zero_row": []
},
"mxfp8": {
@@ -3547,6 +3728,7 @@
"exclude": [],
"pad": [],
"runtime_quant": [],
+ "streamed_weight_only": [],
"zero_row": []
},
"nvfp4": {
@@ -3560,6 +3742,7 @@
"exclude": [],
"pad": [],
"runtime_quant": [],
+ "streamed_weight_only": [],
"zero_row": []
},
"page": "video",
@@ -3578,6 +3761,7 @@
"exclude": [],
"pad": [],
"runtime_quant": [],
+ "streamed_weight_only": [],
"zero_row": []
},
"auto_bf16_when_resident": null,
@@ -3592,6 +3776,7 @@
"exclude": [],
"pad": [],
"runtime_quant": [],
+ "streamed_weight_only": [],
"zero_row": []
},
"convrot_default_on": false,
@@ -3606,6 +3791,7 @@
"exclude": [],
"pad": [],
"runtime_quant": [],
+ "streamed_weight_only": [],
"zero_row": []
},
"int8": {
@@ -3628,6 +3814,7 @@
],
"pad": [],
"runtime_quant": [],
+ "streamed_weight_only": [],
"zero_row": []
},
"mxfp8": {
@@ -3641,6 +3828,7 @@
"exclude": [],
"pad": [],
"runtime_quant": [],
+ "streamed_weight_only": [],
"zero_row": []
},
"nvfp4": {
@@ -3654,6 +3842,7 @@
"exclude": [],
"pad": [],
"runtime_quant": [],
+ "streamed_weight_only": [],
"zero_row": []
},
"page": "video",
@@ -3672,6 +3861,7 @@
"exclude": [],
"pad": [],
"runtime_quant": [],
+ "streamed_weight_only": [],
"zero_row": []
},
"auto_bf16_when_resident": null,
@@ -3686,6 +3876,7 @@
"exclude": [],
"pad": [],
"runtime_quant": [],
+ "streamed_weight_only": [],
"zero_row": []
},
"convrot_default_on": false,
@@ -3700,6 +3891,7 @@
"exclude": [],
"pad": [],
"runtime_quant": [],
+ "streamed_weight_only": [],
"zero_row": []
},
"int8": {
@@ -3722,6 +3914,7 @@
],
"pad": [],
"runtime_quant": [],
+ "streamed_weight_only": [],
"zero_row": []
},
"mxfp8": {
@@ -3735,6 +3928,7 @@
"exclude": [],
"pad": [],
"runtime_quant": [],
+ "streamed_weight_only": [],
"zero_row": []
},
"nvfp4": {
@@ -3748,6 +3942,7 @@
"exclude": [],
"pad": [],
"runtime_quant": [],
+ "streamed_weight_only": [],
"zero_row": []
},
"page": "video",
@@ -3766,6 +3961,7 @@
"exclude": [],
"pad": [],
"runtime_quant": [],
+ "streamed_weight_only": [],
"zero_row": []
},
"auto_bf16_when_resident": null,
@@ -3780,6 +3976,7 @@
"exclude": [],
"pad": [],
"runtime_quant": [],
+ "streamed_weight_only": [],
"zero_row": []
},
"convrot_default_on": false,
@@ -3794,6 +3991,7 @@
"exclude": [],
"pad": [],
"runtime_quant": [],
+ "streamed_weight_only": [],
"zero_row": []
},
"int8": {
@@ -3816,6 +4014,7 @@
],
"pad": [],
"runtime_quant": [],
+ "streamed_weight_only": [],
"zero_row": []
},
"mxfp8": {
@@ -3829,6 +4028,7 @@
"exclude": [],
"pad": [],
"runtime_quant": [],
+ "streamed_weight_only": [],
"zero_row": []
},
"nvfp4": {
@@ -3842,6 +4042,7 @@
"exclude": [],
"pad": [],
"runtime_quant": [],
+ "streamed_weight_only": [],
"zero_row": []
},
"page": "video",
@@ -3860,6 +4061,7 @@
"exclude": [],
"pad": [],
"runtime_quant": [],
+ "streamed_weight_only": [],
"zero_row": []
},
"auto_bf16_when_resident": "a small denoiser stays bf16 while it fits (INT8 / FP8 change image detail: FP8 LPIPS 0.065 on FLUX.2-klein-4B, 0.103 on Z-Image-Turbo)",
@@ -3874,6 +4076,7 @@
"exclude": [],
"pad": [],
"runtime_quant": [],
+ "streamed_weight_only": [],
"zero_row": []
},
"convrot_default_on": true,
@@ -3888,6 +4091,7 @@
"exclude": [],
"pad": [],
"runtime_quant": [],
+ "streamed_weight_only": [],
"zero_row": []
},
"int8": {
@@ -3919,6 +4123,7 @@
],
"pad": [],
"runtime_quant": [],
+ "streamed_weight_only": [],
"zero_row": []
},
"mxfp8": {
@@ -3932,6 +4137,7 @@
"exclude": [],
"pad": [],
"runtime_quant": [],
+ "streamed_weight_only": [],
"zero_row": []
},
"nvfp4": {
@@ -3945,6 +4151,7 @@
"exclude": [],
"pad": [],
"runtime_quant": [],
+ "streamed_weight_only": [],
"zero_row": []
},
"page": "image",
diff --git a/studio/backend/tests/test_diffusion_compile_cache_variant.py b/studio/backend/tests/test_diffusion_compile_cache_variant.py
index 2cb278c74e3..86cebcc75be 100644
--- a/studio/backend/tests/test_diffusion_compile_cache_variant.py
+++ b/studio/backend/tests/test_diffusion_compile_cache_variant.py
@@ -125,7 +125,7 @@ def test_an_instance_forward_swap_keys_apart():
def test_graph_switches_key_apart_only_off_their_default(monkeypatch, name, default, quant):
m = _int8() if quant else _model()
k0 = _key(m, quant)
- rows = default.partition(":")[2] if default.startswith("rows:") else None
+ rows = default.partition(":")[2] if default.startswith(("rows:", "count:")) else None
for same in (
("",)
+ (() if default == "raw" else ("auto",))
@@ -134,7 +134,10 @@ def test_graph_switches_key_apart_only_off_their_default(monkeypatch, name, defa
):
monkeypatch.setenv(name, same)
assert _key(m, quant) == k0, same
- if rows:
+ if default.startswith("count:"):
+ # 0 is a real value here (no dense channels), not an off word.
+ flipped = "0"
+ elif rows:
flipped = "1"
else:
flipped = {"on": "0", "off": "1", "auto": "0", "raw": "torchao", "raw:auto": "torchao"}[
diff --git a/studio/backend/tests/test_diffusion_family_data_freeze.py b/studio/backend/tests/test_diffusion_family_data_freeze.py
index d27ac6594fc..9fe10928e7d 100644
--- a/studio/backend/tests/test_diffusion_family_data_freeze.py
+++ b/studio/backend/tests/test_diffusion_family_data_freeze.py
@@ -84,6 +84,9 @@ def _snapshot() -> dict:
row[scheme] = {
"exclude": list(tq.exclude_tokens_for_scheme(scheme, name, baked = True)),
"runtime_quant": _streamed(tq.runtime_quant_tokens_for_scheme, scheme, name),
+ "streamed_weight_only": _streamed(
+ tq.streamed_weight_only_tokens_for_scheme, scheme, name
+ ),
"pad": list(tq.pad_tokens_for_scheme(scheme, name)),
"zero_row": list(tq.zero_row_tokens_for_scheme(scheme, name)),
"convrot_spec": _jsonable(tq.convrot_spec_for_scheme(scheme, name)),
@@ -190,6 +193,7 @@ def _clean_env(monkeypatch):
for var in (
"UNSLOTH_DIFFUSION_INT8_CONVROT",
"UNSLOTH_DIFFUSION_INT8_TEXT_STREAM",
+ "UNSLOTH_DIFFUSION_INT8_MODULATION",
"UNSLOTH_DIFFUSION_NVFP4",
"UNSLOTH_DIFFUSION_AUTO_STEP_SKIP",
"UNSLOTH_VIDEO_VAE_UNTILED",
diff --git a/studio/backend/tests/test_diffusion_qwenimage_mod_int8.py b/studio/backend/tests/test_diffusion_qwenimage_mod_int8.py
new file mode 100644
index 00000000000..cb048a5b8b4
--- /dev/null
+++ b/studio/backend/tests/test_diffusion_qwenimage_mod_int8.py
@@ -0,0 +1,298 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
+
+"""Qwen-Image streamed int8 weight-only modulations: scope, cache key, the outlier-column split and the GEMV
+module's torch reference path. CPU-only (the Triton kernel itself is checked on a GPU by the bench scripts)."""
+
+from __future__ import annotations
+
+import pytest
+import torch
+from torch import nn
+
+import core.inference.diffusion_int8_gemv as gv
+import core.inference.diffusion_transformer_quant as tq
+from core.inference.diffusion_transformer_quant import TQ_FP8, TQ_INT8
+
+_MODS = ("img_mod", "txt_mod")
+
+
[email protected](autouse = True)
+def _clean_env(monkeypatch):
+ for var in (
+ tq.INT8_MODULATION_ENV,
+ tq.INT8_MOD_OUTLIERS_ENV,
+ tq.INT8_TEXT_STREAM_ENV,
+ gv.INT8_GEMV_ENV,
+ ):
+ monkeypatch.delenv(var, raising = False)
+
+
+def test_tokens_only_inside_the_streamed_scope():
+ assert tq.streamed_weight_only_tokens_for_scheme(TQ_INT8, "qwen-image") == ()
+ assert tq.streamed_weight_only_layout(TQ_INT8, "qwen-image") == {}
+ with tq.int8_text_stream_scope(True):
+ assert tq.streamed_weight_only_tokens_for_scheme(TQ_INT8, "qwen-image") == _MODS
+ assert tq.streamed_weight_only_tokens_for_scheme(TQ_FP8, "qwen-image") == ()
+ assert tq.streamed_weight_only_tokens_for_scheme(TQ_INT8, "flux.1") == ()
+ assert tq.streamed_weight_only_layout(TQ_INT8, "qwen-image") == {
+ "weight_only_name_tokens": sorted(_MODS),
+ "weight_only_mode": "wo",
+ "weight_only_outliers": tq.DEFAULT_MOD_OUTLIERS,
+ }
+ with tq.int8_text_stream_scope(False):
+ assert tq.streamed_weight_only_tokens_for_scheme(TQ_INT8, "qwen-image") == ()
+
+
+ "raw, mode",
+ [
+ ("", "wo"),
+ ("1", "wo"),
+ ("on", "wo"),
+ ("0", "off"),
+ ("off", "off"),
+ ("w8a8", "w8a8"),
+ ("w8a8rot", "w8a8rot"),
+ ],
+)
+def test_modulation_switch(monkeypatch, raw, mode):
+ monkeypatch.setenv(tq.INT8_MODULATION_ENV, raw)
+ assert tq.int8_modulation_mode() == mode
+ with tq.int8_text_stream_scope(True):
+ tokens = tq.streamed_weight_only_tokens_for_scheme(TQ_INT8, "qwen-image")
+ assert tokens == (() if mode == "off" else _MODS)
+
+
[email protected]("raw, n", [("", 8), ("0", 0), ("4", 4), ("99", 16), ("junk", 8)])
+def test_outlier_switch(monkeypatch, raw, n):
+ monkeypatch.setenv(tq.INT8_MOD_OUTLIERS_ENV, raw)
+ assert tq.int8_mod_outliers() == n
+
+
+ "raw, keyed", [("", None), ("8", None), ("junk", None), ("0", "0"), ("4", "4"), ("99", "16")]
+)
+def test_compile_cache_switch_keys_on_the_outlier_count_the_reader_uses(monkeypatch, raw, keyed):
+ from core.inference import diffusion_compile_cache as cc
+
+ monkeypatch.setenv(tq.INT8_MOD_OUTLIERS_ENV, raw)
+ assert cc.graph_switches().get(tq.INT8_MOD_OUTLIERS_ENV) == keyed
+ assert keyed is None or int(keyed) == tq.int8_mod_outliers()
+
+
+def test_cache_key_carries_the_layout(monkeypatch):
+ from core.inference.diffusion_prequant_cache import key_parts
+
+ def parts():
+ return key_parts(
+ family = "qwen-image",
+ transformer_class = "QwenImageTransformer2DModel",
+ scheme = TQ_INT8,
+ component = "transformer",
+ source_fp = "fp",
+ )
+
+ resident = parts()
+ assert "weight_only_name_tokens" not in resident and "weight_only_outliers" not in resident
+ with tq.int8_text_stream_scope(True):
+ default = parts()
+ monkeypatch.setenv(tq.INT8_MOD_OUTLIERS_ENV, "0")
+ no_outliers = parts()
+ monkeypatch.setenv(tq.INT8_MODULATION_ENV, "w8a8rot")
+ rot = parts()
+ monkeypatch.setenv(tq.INT8_MODULATION_ENV, "0")
+ off = parts()
+ assert default["weight_only_outliers"] == 8 and no_outliers["weight_only_outliers"] == 0
+ assert rot["weight_only_mode"] == "w8a8rot"
+ assert "weight_only_name_tokens" not in off
+ keys = [repr(sorted(p.items())) for p in (resident, default, no_outliers, rot, off)]
+ assert len(set(keys)) == len(keys)
+
+
+def _linear(
+ n = 48,
+ k = 256,
+ seed = 0,
+):
+ g = torch.Generator().manual_seed(seed)
+ lin = nn.Linear(k, n, dtype = torch.bfloat16)
+ lin.weight.data = (torch.randn(n, k, generator = g) * 0.02).to(torch.bfloat16)
+ lin.bias.data = torch.randn(n, generator = g).to(torch.bfloat16)
+ return lin
+
+
+def _outlier_x(
+ m = 2,
+ k = 256,
+ seed = 1,
+ channels = (3, 77),
+):
+ g = torch.Generator().manual_seed(seed)
+ x = torch.randn(m, k, generator = g)
+ for c in channels:
+ x[:, c] = 300.0 + c
+ return x.to(torch.bfloat16)
+
+
+def test_split_is_exact_on_the_dense_weight():
+ lin = _linear()
+ x = _outlier_x()
+ want = torch.nn.functional.linear(x.float(), lin.weight.float(), lin.bias.float())
+ moved = gv.split_outliers_(lin, (77, 3))
+ assert moved == 2 and gv.is_gemv_linear(lin)
+ assert lin.outlier_idx.tolist() == [3, 77]
+ assert bool((lin.weight[:, [3, 77]] == 0).all())
+ got = lin(x.float().to(torch.bfloat16)).float()
+ assert torch.allclose(got, want, rtol = 2e-2, atol = 2e-2)
+
+
[email protected]("bad", [(), None, tuple(range(17)), (999,)])
+def test_split_refuses_bad_channel_sets(bad):
+ lin = _linear()
+ before = lin.weight.clone()
+ assert gv.split_outliers_(lin, bad) == 0
+ assert type(lin) is nn.Linear and torch.equal(lin.weight, before)
+
+
+def _quantized(lin):
+ pytest.importorskip("torchao")
+ from torchao.quantization import quantize_
+
+ config = tq._weight_only_config()
+ if config is None:
+ pytest.skip("torchao without Int8Tensor weight-only")
+ quantize_(lin, config, filter_fn = lambda _m, fqn = "": True)
+ assert gv.weight_only_parts(lin.weight) is not None
+ return lin
+
+
+def test_outlier_columns_cut_the_weight_only_error():
+ x = _outlier_x()
+ plain, split = _linear(), _linear()
+ truth = x.double() @ plain.weight.double().t() + plain.bias.double()
+ gv.split_outliers_(split, (3, 77))
+ _quantized(plain)
+ _quantized(split)
+ gv.install(nn.ModuleDict({"a": plain}), ["a"])
+
+ def err(y):
+ return float((y.double() - truth).norm() / truth.norm())
+
+ q, s = gv.weight_only_parts(plain.weight)
+ e_plain = err(gv.reference(x, q, s, plain.bias))
+ q2, s2 = gv.weight_only_parts(split.weight)
+ e_split = err(gv.reference(x, q2, s2, split.bias, split.outlier_idx, split.outlier_weight))
+ assert e_split < e_plain / 3
+ # the module forward (CPU: torchao weight-only + the dense columns) agrees with the fused reference
+ y_mod = split(x)
+ y_ref = gv.reference(x, q2, s2, split.bias, split.outlier_idx, split.outlier_weight)
+ assert torch.allclose(y_mod.float(), y_ref.float(), rtol = 2e-2, atol = 5e-2)
+
+
+def test_reference_grouped_scales_match_dequant():
+ g = torch.Generator().manual_seed(3)
+ q = torch.randint(-127, 128, (32, 128), generator = g, dtype = torch.int8)
+ s = torch.rand(32, 4, generator = g) * 1e-2
+ x = torch.randn(3, 128, generator = g).to(torch.bfloat16)
+ deq = (q.float().reshape(32, 4, 32) * s.reshape(32, 4, 1)).reshape(32, 128)
+ want = (x.float() @ deq.t()).to(torch.bfloat16)
+ assert torch.allclose(gv.reference(x, q, s, None).float(), want.float(), rtol = 1e-2, atol = 1e-3)
+
+
+class _Tte(nn.Module):
+ def __init__(self, hot = (5, 40)):
+ super().__init__()
+ from diffusers.models.embeddings import TimestepEmbedding, Timesteps
+
+ self.time_proj = Timesteps(256, flip_sin_to_cos = True, downscale_freq_shift = 0, scale = 1000)
+ self.timestep_embedder = TimestepEmbedding(256, 64)
+ with torch.no_grad():
+ self.timestep_embedder.linear_2.bias[list(hot)] = 50.0
+
+
+class _Toy(nn.Module):
+ def __init__(self):
+ super().__init__()
+ self.time_text_embed = _Tte()
+ self.img_mod = nn.Sequential(nn.SiLU(), nn.Linear(64, 96))
+
+
+def test_timestep_outlier_channels_finds_the_hot_ones():
+ pytest.importorskip("diffusers")
+ toy = _Toy()
+ assert set(gv.timestep_outlier_channels(toy, 2)) == {5, 40}
+ assert gv.timestep_outlier_channels(toy, 0) == ()
+ assert gv.timestep_outlier_channels(nn.Linear(4, 4), 4) == ()
+
+
+def test_strict_load_round_trip_restores_the_columns():
+ src = nn.ModuleDict({"mod": _linear()})
+ x = _outlier_x()
+ gv.split_outliers_(src["mod"], (3, 77))
+ _quantized(src["mod"])
+ want = src["mod"](x)
+ state = src.state_dict()
+ assert "mod.outlier_weight" in state and "mod.outlier_idx" in state
+ dst = nn.ModuleDict({"mod": _linear(seed = 9)})
+ _quantized(dst["mod"])
+ assert gv.prepare_outlier_buffers(dst, state) == 1
+ dst.load_state_dict(state, strict = True, assign = True)
+ assert gv.is_gemv_linear(dst["mod"])
+ assert torch.equal(dst["mod"](x), want)
+
+
+def test_install_skips_plain_linears_without_the_kernel(monkeypatch):
+ monkeypatch.setenv(gv.INT8_GEMV_ENV, "0")
+ plain = _quantized(_linear())
+ split = _linear()
+ gv.split_outliers_(split, (3,))
+ _quantized(split)
+ tree = nn.ModuleDict({"plain": plain, "split": split, "dense": _linear()})
+ assert gv.install(tree, ["plain", "split", "dense"]) == 1
+ assert type(tree["plain"]) is nn.Linear and gv.is_gemv_linear(tree["split"])
+ assert type(tree["dense"]) is nn.Linear
+ assert gv._OP_HANDLE is None
+
+
+def test_streamed_quantize_builds_the_layout(monkeypatch):
+ pytest.importorskip("diffusers")
+ if tq._weight_only_config() is None:
+ pytest.skip("torchao without Int8Tensor weight-only")
+
+ class Block(nn.Module):
+ def __init__(self):
+ super().__init__()
+ self.img_mod = nn.Sequential(nn.SiLU(), nn.Linear(64, 512, dtype = torch.bfloat16))
+ self.txt_mod = nn.Sequential(nn.SiLU(), nn.Linear(64, 512, dtype = torch.bfloat16))
+ self.other = nn.Linear(64, 512, dtype = torch.bfloat16)
+
+ toy = _Toy()
+ toy.time_text_embed.to(torch.bfloat16)
+ toy.block = Block()
+ temb = torch.nn.functional.silu(
+ toy.time_text_embed.timestep_embedder(
+ toy.time_text_embed.time_proj(torch.tensor([0.5])).to(torch.bfloat16)
+ )
+ )
+ want = toy.block.img_mod(temb.float().to(torch.bfloat16)).float()
+ with tq.int8_text_stream_scope(True):
+ names = tq.quantize_streamed_weight_only(toy, TQ_INT8, "qwen-image", min_features = 32)
+ assert set(names) == {
+ "block.img_mod.1",
+ "block.txt_mod.1",
+ } # the float32 toy img_mod is not bf16: left alone
+ layout = toy._unsloth_weight_only_layout
+ assert (
+ layout["weight_only_outliers"] == tq.DEFAULT_MOD_OUTLIERS
+ and layout["weight_only_mode"] == "wo"
+ )
+ assert gv.is_gemv_linear(toy.block.img_mod[1]) and type(toy.block.other) is nn.Linear
+ assert {5, 40} <= set(toy.block.img_mod[1].outlier_idx.tolist())
+ got = toy.block.img_mod(temb).float()
+ assert float((got - want).norm() / want.norm()) < 1e-2
+ # resident scope: nothing changes
+ fresh = _Toy()
+ with tq.int8_text_stream_scope(False):
+ assert tq.quantize_streamed_weight_only(fresh, TQ_INT8, "qwen-image", min_features = 32) == ()
+ assert type(fresh.img_mod[1]) is nn.Linear
diff --git a/studio/backend/tests/test_diffusion_qwenimage_text_int8.py b/studio/backend/tests/test_diffusion_qwenimage_text_int8.py
index 1d2a48a8fa1..b87cccc1a6a 100644
--- a/studio/backend/tests/test_diffusion_qwenimage_text_int8.py
+++ b/studio/backend/tests/test_diffusion_qwenimage_text_int8.py
@@ -25,6 +25,7 @@ _UNSCOPED = {
@pytest.fixture(autouse = True)
def _default_switch(monkeypatch, request):
monkeypatch.delenv(tq.INT8_TEXT_STREAM_ENV, raising = False)
+ monkeypatch.delenv(tq.INT8_MODULATION_ENV, raising = False)
if request.node.name in _UNSCOPED:
yield
return
@@ -272,6 +273,8 @@ def test_local_prequant_cache_key_follows_the_switch(monkeypatch):
with tq.int8_text_stream_scope(False):
resident, resident_tokens = key()
monkeypatch.setenv(tq.INT8_TEXT_STREAM_ENV, "0")
+ # The streamed weight-only modulations key separately (test_diffusion_qwenimage_mod_int8.py).
+ monkeypatch.setenv(tq.INT8_MODULATION_ENV, "0")
off, off_tokens = key()
assert on != off
assert "add_q_proj" in off_tokens and "add_q_proj" not in on_tokens