Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

Studio: int8 Qwen-Image text stream when the denoiser is streamed (1.24x under a 16 GiB budget) (#16027)

Daniel Hanaca386ff2790
diff --git a/studio/backend/core/inference/diffusion.py b/studio/backend/core/inference/diffusion.py
index e3e41ad5bac..7cbc2af66ec 100644
--- a/studio/backend/core/inference/diffusion.py
+++ b/studio/backend/core/inference/diffusion.py
@@ -505,6 +505,7 @@ from .diffusion_transformer_quant import (
NATIVE_OFFLOAD_SCHEMES,
NATIVE_QUANT_SCHEMES,
normalize_transformer_quant,
+ int8_text_stream_scope,
quantize_transformer,
select_transformer_quant_scheme,
stored_denoiser_precision,
@@ -9816,38 +9817,41 @@ class DiffusionBackend:
else {}
)
engaged: list[str] = []
- cached_scheme = (
- self._pipeline_quant_from_cache(
- pipe,
- fam,
- denoisers,
- target = target,
- transformer_quant = transformer_quant,
- fast_accum = transformer_quant_fast_accum,
- loras = loras,
- base = base,
- fetch_base = fetch_base,
- source_base = _base_local_dir or fetch_base,
- hf_token = hf_token,
+ # Text-stream int8 only when streamed; the scope is part of the prequant-cache key.
+ with int8_text_stream_scope(not keeps_resident):
+ cached_scheme = (
+ self._pipeline_quant_from_cache(
+ pipe,
+ fam,
+ denoisers,
+ target = target,
+ transformer_quant = transformer_quant,
+ fast_accum = transformer_quant_fast_accum,
+ loras = loras,
+ base = base,
+ fetch_base = fetch_base,
+ source_base = _base_local_dir or fetch_base,
+ hf_token = hf_token,
+ )
+ if native_scheme is None
+ else None
)
- if native_scheme is None
- else None
- )
if cached_scheme is not None:
engaged.append(cached_scheme)
for attr, _module in denoisers if cached_scheme is None else ():
- scheme = quantize_transformer(
- pipe
- if attr == "transformer"
- else _DenoiserView(pipe, attr),
- target,
- mode = transformer_quant,
- family = getattr(fam, "name", None),
- base_repo = base,
- fast_accum = transformer_quant_fast_accum,
- logger = logger,
- **native_kwargs,
- )
+ with int8_text_stream_scope(not keeps_resident):
+ scheme = quantize_transformer(
+ pipe
+ if attr == "transformer"
+ else _DenoiserView(pipe, attr),
+ target,
+ mode = transformer_quant,
+ family = getattr(fam, "name", None),
+ base_repo = base,
+ fast_accum = transformer_quant_fast_accum,
+ logger = logger,
+ **native_kwargs,
+ )
if scheme is None:
break
engaged.append(scheme)
@@ -9859,19 +9863,20 @@ class DiffusionBackend:
transformer_quant_engaged = cached_scheme
elif engaged and len(engaged) == len(denoisers):
transformer_quant_engaged = engaged[0]
- pending_cache_write = self._save_pipeline_quant_to_cache(
- pipe,
- fam,
- denoisers,
- loras = loras,
- base = base,
- source_base = _base_local_dir or fetch_base,
- cancel_check = lambda: self._raise_if_load_cancelled(
- _load_token
- ),
- defer = True,
- native_cache = native_cache_ok,
- )
+ with int8_text_stream_scope(not keeps_resident):
+ pending_cache_write = self._save_pipeline_quant_to_cache(
+ pipe,
+ fam,
+ denoisers,
+ loras = loras,
+ base = base,
+ source_base = _base_local_dir or fetch_base,
+ cancel_check = lambda: self._raise_if_load_cancelled(
+ _load_token
+ ),
+ defer = True,
+ native_cache = native_cache_ok,
+ )
if not hasattr(pending_cache_write, "start"):
pending_cache_write = None
self._raise_if_load_cancelled(_load_token)
diff --git a/studio/backend/core/inference/diffusion_comfy_quant.py b/studio/backend/core/inference/diffusion_comfy_quant.py
index c98d7cbe3ff..ce93b6a1751 100644
--- a/studio/backend/core/inference/diffusion_comfy_quant.py
+++ b/studio/backend/core/inference/diffusion_comfy_quant.py
@@ -1437,7 +1437,7 @@ def load_comfy_quant_transformer(
filters = {
fmt: make_filter_fn(
min_features,
- exclude_name_tokens = exclude_tokens_for_scheme(scheme, family),
+ exclude_name_tokens = exclude_tokens_for_scheme(scheme, family, baked = True),
require_divisible = divisible_for_scheme(scheme),
)
for fmt, scheme in schemes.items()
diff --git a/studio/backend/core/inference/diffusion_compile_cache.py b/studio/backend/core/inference/diffusion_compile_cache.py
index 66277e50c47..c7eb8022472 100644
--- a/studio/backend/core/inference/diffusion_compile_cache.py
+++ b/studio/backend/core/inference/diffusion_compile_cache.py
@@ -225,6 +225,7 @@ _GRAPH_SWITCHES = {
"UNSLOTH_DIFFUSION_INT8_ROTQUANT": "auto",
"UNSLOTH_DIFFUSION_INT8_ROTQUANT_UNLISTED": "auto",
"UNSLOTH_DIFFUSION_INT8_FUSED": "on",
+ "UNSLOTH_DIFFUSION_INT8_TEXT_STREAM": "on",
"UNSLOTH_DIFFUSION_ZIMAGE_FUSED": "on",
"UNSLOTH_DIFFUSION_Q21_REAL_ROPE": "on",
"UNSLOTH_DIFFUSION_QWEN_REAL_ROPE": "on",
diff --git a/studio/backend/core/inference/diffusion_families.py b/studio/backend/core/inference/diffusion_families.py
index 3d4b0b25556..139e1c70c8c 100644
--- a/studio/backend/core/inference/diffusion_families.py
+++ b/studio/backend/core/inference/diffusion_families.py
@@ -262,10 +262,8 @@ _QWEN_REDUCTION_RACE_ARCHS: tuple[tuple[int, int], ...] = _REDUCTION_RACE_ARCHS
# --- Shared FamilyProfile data (see family_profile.FamilyProfile) ---
-# int8 PER-FAMILY exclusions, on top of diffusion_transformer_quant._INT8_EXCLUDE_NAME_TOKENS. Qwen-Image MMDiT runs
-# every TEXT-stream Linear at M = actual prompt tokens (unpadded, unlike FLUX's 512-token T5), so a short prompt falls
-# under _int_mm's M floor of 16 and the denoise crashes. bf16 there costs ~nothing. Baked into the published int8
-# artifacts of qwen-image / qwen-image-edit, so it moves only with a rebuild and republish.
+# Qwen-Image text-stream Linears run at M = prompt tokens (~6 for the empty CFG negative), under _int_mm's floor of 16
+# where a compiled torchao int8 Linear raises. Baked into published int8 artifacts; streamed builds pad part (below).
_QWENIMAGE_INT8_EXCLUDES = (
"txt_in",
"add_q_proj",
@@ -274,6 +272,15 @@ _QWENIMAGE_INT8_EXCLUDES = (
"to_add_out",
"txt_mlp",
)
+# Quantised (row-padded to 32) by STREAMED qwen-image t2i int8 builds only: 12.90 -> 10.43 s/image on 2512 at a 16 GiB
+# budget. Resident int8 measured slower and LPIPS-worse; txt_in int8 scored worse too. Edit / layered unmeasured.
+_QWENIMAGE_TEXT_STREAM_INT8 = (
+ "add_q_proj",
+ "add_k_proj",
+ "add_v_proj",
+ "to_add_out",
+ "txt_mlp",
+)
# Families whose activation ranges break specific schemes at the MODEL level (the smoke probe only proves the GEMM
# runs). Measured with the 28-pair prequant accuracy gate on B200: qwen-image + mxfp8 does semantic damage at 1024px,
# + nvfp4 is unusable (LPIPS 0.51). Neither has a known fix, so both stay denied and auto falls through to int8
@@ -865,6 +872,7 @@ _FAMILIES: tuple[DiffusionFamily, ...] = (
label = "Qwen-Image",
profile = FamilyProfile(
int8_exclude_tokens = _QWENIMAGE_INT8_EXCLUDES,
+ int8_runtime_quant_tokens = _QWENIMAGE_TEXT_STREAM_INT8,
scheme_deny = _QWENIMAGE_SCHEME_DENY,
train_scheme_deny = _QWENIMAGE_TRAIN_SCHEME_DENY,
auto_prefer = _GATED_AUTO_PREFER,
diff --git a/studio/backend/core/inference/diffusion_native_cache.py b/studio/backend/core/inference/diffusion_native_cache.py
index eb88a01c3a4..7c354efccfd 100644
--- a/studio/backend/core/inference/diffusion_native_cache.py
+++ b/studio/backend/core/inference/diffusion_native_cache.py
@@ -38,7 +38,9 @@ def native_spec(
"rot_group": int(int8_rotation_group()) if act else 0,
"scale_dtype": str(scale_dtype) if scale_dtype is not None and not act else None,
"min_features": DEFAULT_MIN_LINEAR_FEATURES,
- "exclude_name_tokens": sorted(set(exclude_tokens_for_scheme(scheme, family)) | {"lora_"}),
+ "exclude_name_tokens": sorted(
+ set(exclude_tokens_for_scheme(scheme, family, baked = True)) | {"lora_"}
+ ),
"require_bf16": True,
}
@@ -687,7 +689,7 @@ def build_from_shards(
)
filter_fn = make_filter_fn(
int(spec["min_features"]),
- exclude_name_tokens = exclude_tokens_for_scheme(scheme, family) + ("lora_",),
+ exclude_name_tokens = exclude_tokens_for_scheme(scheme, family, baked = True) + ("lora_",),
require_bf16 = True,
)
quant = {name for name, mod in model.named_modules() if name and filter_fn(mod, name)}
diff --git a/studio/backend/core/inference/diffusion_prequant.py b/studio/backend/core/inference/diffusion_prequant.py
index 2de1fc0b2cc..3f59234ca54 100644
--- a/studio/backend/core/inference/diffusion_prequant.py
+++ b/studio/backend/core/inference/diffusion_prequant.py
@@ -2402,7 +2402,25 @@ def load_prequantized_transformer(
except Exception: # noqa: BLE001
pass
# Small-M padding, else quantised small-M linears raise in _int_mm. After load and .to().
- from .diffusion_transformer_quant import apply_small_m_padding, apply_zero_row_guard
+ from .diffusion_transformer_quant import (
+ apply_small_m_padding,
+ apply_zero_row_guard,
+ quantize_runtime_leftovers,
+ )
+
+ # Must run before padding (it reparents Linears); a failure keeps the artifact, leftovers stay dense.
+ try:
+ quantize_runtime_leftovers(
+ transformer,
+ scheme,
+ metadata.get("family") or family,
+ metadata.get("exclude_name_tokens"),
+ min_features = int(metadata.get("min_features") or min_features or 512),
+ device = device,
+ logger = logger,
+ )
+ except Exception as exc: # noqa: BLE001 - optimisation only
+ _warn(logger, f"{scheme}:runtime leftovers", exc)
apply_small_m_padding(transformer, scheme, metadata.get("family"), logger = logger)
apply_zero_row_guard(transformer, scheme, metadata.get("family"), logger = logger)
diff --git a/studio/backend/core/inference/diffusion_transformer_quant.py b/studio/backend/core/inference/diffusion_transformer_quant.py
index de00fa9ead0..39e9d2493d6 100644
--- a/studio/backend/core/inference/diffusion_transformer_quant.py
+++ b/studio/backend/core/inference/diffusion_transformer_quant.py
@@ -6,6 +6,8 @@ torchao imported lazily; probes best-effort, GGUF fallback"""
from __future__ import annotations
+import contextlib as _contextlib
+import contextvars as _contextvars
import inspect as _inspect
import os as _os
import re as _re
@@ -71,6 +73,41 @@ _INT8_FAMILY_EXCLUDE_NAME_TOKENS: dict[str, tuple[str, ...]] = _profile_table(
)
_QWENIMAGE_INT8_EXCLUDES = _INT8_FAMILY_EXCLUDE_NAME_TOKENS["qwen-image"]
+# Applied ONLY inside ``int8_text_stream_scope(True)`` (streamed builds); hosted artifacts still validate since their
+# exclusions are a superset. The scope is part of the prequant-cache key.
+_INT8_FAMILY_RUNTIME_QUANT_TOKENS: dict[str, tuple[str, ...]] = _profile_table(
+ "int8_runtime_quant_tokens", quant = True
+)
+INT8_TEXT_STREAM_ENV = "UNSLOTH_DIFFUSION_INT8_TEXT_STREAM"
+_TEXT_STREAM_SCOPE: "_contextvars.ContextVar[bool]" = _contextvars.ContextVar(
+ "unsloth_int8_text_stream", default = False
+)
+
+
+@_contextlib.contextmanager
+def int8_text_stream_scope(streamed: bool):
+ token = _TEXT_STREAM_SCOPE.set(bool(streamed))
+ try:
+ yield
+ finally:
+ _TEXT_STREAM_SCOPE.reset(token)
+
+
+def int8_text_stream_enabled() -> bool:
+ return (_os.environ.get(INT8_TEXT_STREAM_ENV) or "").strip().lower() not in (
+ "0",
+ "off",
+ "false",
+ "no",
+ )
+
+
+def runtime_quant_tokens_for_scheme(scheme: str, family: Optional[str] = None) -> tuple[str, ...]:
+ """Baked int8 exclusions quantised anyway; () outside ``int8_text_stream_scope(True)``."""
+ if scheme != TQ_INT8 or not _TEXT_STREAM_SCOPE.get() or not int8_text_stream_enabled():
+ return ()
+ return _INT8_FAMILY_RUNTIME_QUANT_TOKENS.get(str(family or "").strip().lower(), ())
+
# ALL-scheme PER-FAMILY exclusions (``FamilyProfile.quant_exclude_tokens``): unlike the int8 table above, these also
# apply to fp8 / mxfp8 / nvfp4, whose scaled_mm paths otherwise quantise every eligible Linear.
@@ -91,10 +128,13 @@ _INT8_FAMILY_PAD_NAME_TOKENS: dict[str, tuple[str, ...]] = _profile_table(
def pad_tokens_for_scheme(scheme: str, family: Optional[str] = None) -> tuple[str, ...]:
- """int8 only, and disjoint from the exclude tokens since excluded Linears are never quantised."""
+ """int8 only, disjoint from the exclude tokens; includes a streamed build's runtime-quantised text stream."""
if scheme != TQ_INT8:
return ()
- return _INT8_FAMILY_PAD_NAME_TOKENS.get(str(family or "").strip().lower(), ())
+ key = str(family or "").strip().lower()
+ tokens = _INT8_FAMILY_PAD_NAME_TOKENS.get(key, ())
+ runtime = tuple(t for t in runtime_quant_tokens_for_scheme(scheme, key) if t not in tokens)
+ return tokens + runtime
def apply_small_m_padding(
@@ -335,7 +375,12 @@ def apply_zero_row_guard(
return wrapped
-def exclude_tokens_for_scheme(scheme: str, family: Optional[str] = None) -> tuple[str, ...]:
+def exclude_tokens_for_scheme(
+ scheme: str,
+ family: Optional[str] = None,
+ *,
+ baked: bool = False,
+) -> tuple[str, ...]:
"""Name tokens to exclude from quantisation for ``scheme`` (optionally family-specific). int8
(M>16) skips the M=1 modulation / conditioning-embedder projections
(_INT8_EXCLUDE_NAME_TOKENS) plus per-family small-M text streams
@@ -343,13 +388,15 @@ def exclude_tokens_for_scheme(scheme: str, family: Optional[str] = None) -> tupl
``family=None`` preserves the historical behaviour. Shared by the runtime path and the
offline prequant builder so they never drift (else an int8 checkpoint bakes the small-M
projections and crashes at the first denoise on Flux / Qwen). ``FamilyProfile.quant_exclude_tokens`` applies
- to every scheme."""
+ to every scheme. Streamed builds drop runtime tokens; ``baked`` keeps the published-artifact list."""
key = str(family or "").strip().lower()
every_scheme = _FAMILY_QUANT_EXCLUDE_NAME_TOKENS.get(key, ())
if scheme == TQ_INT8:
- return (
- _INT8_EXCLUDE_NAME_TOKENS + _INT8_FAMILY_EXCLUDE_NAME_TOKENS.get(key, ()) + every_scheme
- )
+ family_tokens = _INT8_FAMILY_EXCLUDE_NAME_TOKENS.get(key, ())
+ runtime = () if baked else runtime_quant_tokens_for_scheme(scheme, key)
+ if runtime:
+ family_tokens = tuple(t for t in family_tokens if t not in runtime)
+ return _INT8_EXCLUDE_NAME_TOKENS + family_tokens + every_scheme
return every_scheme
@@ -1808,10 +1855,10 @@ def _quantize_native(
try:
from .diffusion_native_quant import apply_native_weight_quant
- # require_bf16 keeps the fp32 linears some DiTs deliberately hold (Wan, Hunyuan) at full precision.
+ # require_bf16 keeps deliberate fp32 linears (Wan, Hunyuan). Baked list: W8A8's small-M branch would recompile.
filter_fn = make_filter_fn(
min_features,
- exclude_name_tokens = exclude_tokens_for_scheme(scheme, family) + ("lora_",),
+ exclude_name_tokens = exclude_tokens_for_scheme(scheme, family, baked = True) + ("lora_",),
require_bf16 = True,
)
if not apply_native_weight_quant(
@@ -1839,6 +1886,75 @@ def _quantize_native(
return None
+def quantize_runtime_leftovers(
+ transformer: Any,
+ scheme: str,
+ family: Optional[str],
+ baked_excludes: Any,
+ *,
+ min_features: int = DEFAULT_MIN_LINEAR_FEATURES,
+ device: Any = None,
+ logger: Any = None,
+) -> int:
+ """Quantise the dense Linears an artifact baked under ``baked_excludes`` that this runtime quantises
+ (``runtime_quant_tokens_for_scheme``); host-placed Linears are quantised on ``device`` and moved back. Returns the
+ count."""
+ runtime = runtime_quant_tokens_for_scheme(scheme, family)
+ if not runtime or not isinstance(baked_excludes, (list, tuple)):
+ return 0
+ tokens = tuple(t for t in runtime if t in baked_excludes)
+ if not tokens:
+ return 0
+ import time
+
+ import torch
+ from torchao.quantization import quantize_
+
+ from .diffusion_quant_pad import is_quantized_linear
+
+ keep = make_filter_fn(
+ min_features,
+ exclude_name_tokens = exclude_tokens_for_scheme(scheme, family) + ("lora_",),
+ require_bf16 = True,
+ )
+ targets = [
+ (name, module)
+ for name, module in transformer.named_modules()
+ if isinstance(module, torch.nn.Linear)
+ and any(tok in name.lower() for tok in tokens)
+ and not is_quantized_linear(module)
+ and keep(module, name)
+ ]
+ if not targets:
+ return 0
+ work = None
+ try:
+ work = torch.device(device) if device is not None else None
+ except Exception: # noqa: BLE001 - an unparsable device quantises in place
+ work = None
+ if work is not None and work.type != "cuda":
+ work = None
+ config = _make_quant_config(scheme)
+ t0 = time.perf_counter()
+ for _name, module in targets:
+ home = module.weight.device
+ hop = work is not None and home.type == "cpu"
+ if hop:
+ module.to(work)
+ quantize_(module, config)
+ if hop:
+ module.to(home)
+ if logger is not None:
+ logger.info(
+ "diffusion.transformer_quant: quantised %d %s Linear(s) the artifact kept dense (%s) in %.1f s",
+ len(targets),
+ scheme,
+ ", ".join(tokens),
+ time.perf_counter() - t0,
+ )
+ return len(targets)
+
+
def _first_weight_dtype(module: Any) -> Any:
"""The dtype the dense model was loaded in: a parameter left high precision by the quant filter (norms, embeds)."""
try:
diff --git a/studio/backend/core/inference/family_profile.py b/studio/backend/core/inference/family_profile.py
index f49e750d244..826eeb6204c 100644
--- a/studio/backend/core/inference/family_profile.py
+++ b/studio/backend/core/inference/family_profile.py
@@ -57,6 +57,8 @@ class FamilyProfile:
# int8 name tokens left dense on top of the generic M = 1 tokens (``_int_mm`` needs M > 16, or a measured quality
# lever). Baked into published int8 artifacts.
int8_exclude_tokens: tuple[str, ...] = ()
+ # Subset of ``int8_exclude_tokens`` a STREAMED build quantises anyway (``int8_text_stream_scope``).
+ int8_runtime_quant_tokens: tuple[str, ...] = ()
# Name tokens left dense for EVERY transformer-quant scheme (int8 / fp8 / mxfp8 / nvfp4), on top of the int8
# tokens above. For layers whose quantisation error is measured to dominate the image delta regardless of the
# scheme (FLUX.3: per-token modulations, the 20480-wide text projection, the text-stream per-mode blocks).
diff --git a/studio/backend/core/training/diffusion_dit_trainer.py b/studio/backend/core/training/diffusion_dit_trainer.py
index c964414cdb4..f5794a61baf 100644
--- a/studio/backend/core/training/diffusion_dit_trainer.py
+++ b/studio/backend/core/training/diffusion_dit_trainer.py
@@ -392,7 +392,7 @@ def _int8_quantize_base(transformer, family: Optional[str] = None) -> None:
transformer,
_quiet_config(Int8WeightOnlyConfig),
filter_fn = make_filter_fn(
- 512, exclude_name_tokens = exclude_tokens_for_scheme("int8", family)
+ 512, exclude_name_tokens = exclude_tokens_for_scheme("int8", family, baked = True)
),
)
# After quantize_: the helper reparents the Linears it wraps. Not best-effort: a raise leaves the base quantized but not safely runnable.
@@ -406,7 +406,7 @@ _LOWP_EXTRA_EXCLUDE_TOKENS = ("final_layer",)
def _lowp_exclude_tokens(family: Optional[str] = None) -> tuple[str, ...]:
from core.inference.diffusion_transformer_quant import exclude_tokens_for_scheme
- return tuple(exclude_tokens_for_scheme("int8", family)) + _LOWP_EXTRA_EXCLUDE_TOKENS
+ return tuple(exclude_tokens_for_scheme("int8", family, baked = True)) + _LOWP_EXTRA_EXCLUDE_TOKENS
def _lowp_module_filter(
diff --git a/studio/backend/tests/fixtures/diffusion_family_data_freeze.json b/studio/backend/tests/fixtures/diffusion_family_data_freeze.json
index b32a9f8e829..a8bf5321baf 100644
--- a/studio/backend/tests/fixtures/diffusion_family_data_freeze.json
+++ b/studio/backend/tests/fixtures/diffusion_family_data_freeze.json
@@ -592,6 +592,7 @@
"convrot_variant_repos": [],
"exclude": [],
"pad": [],
+ "runtime_quant": [],
"zero_row": []
},
"auto_bf16_when_resident": null,
@@ -605,6 +606,7 @@
"convrot_variant_repos": [],
"exclude": [],
"pad": [],
+ "runtime_quant": [],
"zero_row": []
},
"convrot_default_on": false,
@@ -618,6 +620,7 @@
"convrot_variant_repos": [],
"exclude": [],
"pad": [],
+ "runtime_quant": [],
"zero_row": []
},
"int8": {
@@ -639,6 +642,7 @@
"time_embed"
],
"pad": [],
+ "runtime_quant": [],
"zero_row": []
},
"mxfp8": {
@@ -651,6 +655,7 @@
"convrot_variant_repos": [],
"exclude": [],
"pad": [],
+ "runtime_quant": [],
"zero_row": []
},
"nvfp4": {
@@ -663,6 +668,7 @@
"convrot_variant_repos": [],
"exclude": [],
"pad": [],
+ "runtime_quant": [],
"zero_row": []
},
"page": null,
@@ -680,6 +686,7 @@
"convrot_variant_repos": [],
"exclude": [],
"pad": [],
+ "runtime_quant": [],
"zero_row": []
},
"auto_bf16_when_resident": null,
@@ -693,6 +700,7 @@
"convrot_variant_repos": [],
"exclude": [],
"pad": [],
+ "runtime_quant": [],
"zero_row": []
},
"convrot_default_on": false,
@@ -706,6 +714,7 @@
"convrot_variant_repos": [],
"exclude": [],
"pad": [],
+ "runtime_quant": [],
"zero_row": []
},
"int8": {
@@ -733,6 +742,13 @@
"txt_mlp"
],
"pad": [],
+ "runtime_quant": [
+ "add_q_proj",
+ "add_k_proj",
+ "add_v_proj",
+ "to_add_out",
+ "txt_mlp"
+ ],
"zero_row": []
},
"mxfp8": {
@@ -745,6 +761,7 @@
"convrot_variant_repos": [],
"exclude": [],
"pad": [],
+ "runtime_quant": [],
"zero_row": []
},
"nvfp4": {
@@ -757,6 +774,7 @@
"convrot_variant_repos": [],
"exclude": [],
"pad": [],
+ "runtime_quant": [],
"zero_row": []
},
"page": "image",
@@ -839,6 +857,7 @@
"convrot_variant_repos": [],
"exclude": [],
"pad": [],
+ "runtime_quant": [],
"zero_row": []
},
"auto_bf16_when_resident": null,
@@ -852,6 +871,7 @@
"convrot_variant_repos": [],
"exclude": [],
"pad": [],
+ "runtime_quant": [],
"zero_row": []
},
"convrot_default_on": false,
@@ -865,6 +885,7 @@
"convrot_variant_repos": [],
"exclude": [],
"pad": [],
+ "runtime_quant": [],
"zero_row": []
},
"int8": {
@@ -892,6 +913,13 @@
"txt_mlp"
],
"pad": [],
+ "runtime_quant": [
+ "add_q_proj",
+ "add_k_proj",
+ "add_v_proj",
+ "to_add_out",
+ "txt_mlp"
+ ],
"zero_row": []
},
"mxfp8": {
@@ -904,6 +932,7 @@
"convrot_variant_repos": [],
"exclude": [],
"pad": [],
+ "runtime_quant": [],
"zero_row": []
},
"nvfp4": {
@@ -916,6 +945,7 @@
"convrot_variant_repos": [],
"exclude": [],
"pad": [],
+ "runtime_quant": [],
"zero_row": []
},
"page": "image",
@@ -933,6 +963,7 @@
"convrot_variant_repos": [],
"exclude": [],
"pad": [],
+ "runtime_quant": [],
"zero_row": []
},
"auto_bf16_when_resident": null,
@@ -946,6 +977,7 @@
"convrot_variant_repos": [],
"exclude": [],
"pad": [],
+ "runtime_quant": [],
"zero_row": []
},
"convrot_default_on": false,
@@ -959,6 +991,7 @@
"convrot_variant_repos": [],
"exclude": [],
"pad": [],
+ "runtime_quant": [],
"zero_row": []
},
"int8": {
@@ -980,6 +1013,7 @@
"time_embed"
],
"pad": [],
+ "runtime_quant": [],
"zero_row": []
},
"mxfp8": {
@@ -992,6 +1026,7 @@
"convrot_variant_repos": [],
"exclude": [],
"pad": [],
+ "runtime_quant": [],
"zero_row": []
},
"nvfp4": {
@@ -1004,6 +1039,7 @@
"convrot_variant_repos": [],
"exclude": [],
"pad": [],
+ "runtime_quant": [],
"zero_row": []
},
"page": "video",
@@ -1021,6 +1057,7 @@
"convrot_variant_repos": [],
"exclude": [],
"pad": [],
+ "runtime_quant": [],
"zero_row": []
},
"auto_bf16_when_resident": null,
@@ -1034,6 +1071,7 @@
"convrot_variant_repos": [],
"exclude": [],
"pad": [],
+ "runtime_quant": [],
"zero_row": []
},
"convrot_default_on": false,
@@ -1047,6 +1085,7 @@
"convrot_variant_repos": [],
"exclude": [],
"pad": [],
+ "runtime_quant": [],
"zero_row": []
},
"int8": {
@@ -1068,6 +1107,7 @@
"time_embed"
],
"pad": [],
+ "runtime_quant": [],
"zero_row": []
},
"mxfp8": {
@@ -1080,6 +1120,7 @@
"convrot_variant_repos": [],
"exclude": [],
"pad": [],
+ "runtime_quant": [],
"zero_row": []
},
"nvfp4": {
@@ -1092,6 +1133,7 @@
"convrot_variant_repos": [],
"exclude": [],
"pad": [],
+ "runtime_quant": [],
"zero_row": []
},
"page": "video",
@@ -1109,6 +1151,7 @@
"convrot_variant_repos": [],
"exclude": [],
"pad": [],
+ "runtime_quant": [],
"zero_row": []
},
"auto_bf16_when_resident": null,
@@ -1122,6 +1165,7 @@
"convrot_variant_repos": [],
"exclude": [],
"pad": [],
+ "runtime_quant": [],
"zero_row": []
},
"convrot_default_on": false,
@@ -1135,6 +1179,7 @@
"convrot_variant_repos": [],
"exclude": [],
"pad": [],
+ "runtime_quant": [],
"zero_row": []
},
"int8": {
@@ -1156,6 +1201,7 @@
"time_embed"
],
"pad": [],
+ "runtime_quant": [],
"zero_row": []
},
"mxfp8": {
@@ -1168,6 +1214,7 @@
"convrot_variant_repos": [],
"exclude": [],
"pad": [],
+ "runtime_quant": [],
"zero_row": []
},
"nvfp4": {
@@ -1180,6 +1227,7 @@
"convrot_variant_repos": [],
"exclude": [],
"pad": [],
+ "runtime_quant": [],
"zero_row": []
},
"page": "image",
@@ -1197,6 +1245,7 @@
"convrot_variant_repos": [],
"exclude": [],
"pad": [],
+ "runtime_quant": [],
"zero_row": []
},
"auto_bf16_when_resident": null,
@@ -1210,6 +1259,7 @@
"convrot_variant_repos": [],
"exclude": [],
"pad": [],
+ "runtime_quant": [],
"zero_row": []
},
"convrot_default_on": false,
@@ -1223,6 +1273,7 @@
"convrot_variant_repos": [],
"exclude": [],
"pad": [],
+ "runtime_quant": [],
"zero_row": []
},
"int8": {
@@ -1244,6 +1295,7 @@
"time_embed"
],
"pad": [],
+ "runtime_quant": [],
"zero_row": []
},
"mxfp8": {
@@ -1256,6 +1308,7 @@
"convrot_variant_repos": [],
"exclude": [],
"pad": [],
+ "runtime_quant": [],
"zero_row": []
},
"nvfp4": {
@@ -1268,6 +1321,7 @@
"convrot_variant_repos": [],
"exclude": [],
"pad": [],
+ "runtime_quant": [],
"zero_row": []
},
"page": "image",
@@ -1285,6 +1339,7 @@
"convrot_variant_repos": [],
"exclude": [],
"pad": [],
+ "runtime_quant": [],
"zero_row": []
},
"auto_bf16_when_resident": null,
@@ -1298,6 +1353,7 @@
"convrot_variant_repos": [],
"exclude": [],
"pad": [],
+ "runtime_quant": [],
"zero_row": []
},
"convrot_default_on": false,
@@ -1311,6 +1367,7 @@
"convrot_variant_repos": [],
"exclude": [],
"pad": [],
+ "runtime_quant": [],
"zero_row": []
},
"int8": {
@@ -1332,6 +1389,7 @@
"time_embed"
],
"pad": [],
+ "runtime_quant": [],
"zero_row": []
},
"mxfp8": {
@@ -1344,6 +1402,7 @@
"convrot_variant_repos": [],
"exclude": [],
"pad": [],
+ "runtime_quant": [],
"zero_row": []
},
"nvfp4": {
@@ -1356,6 +1415,7 @@
"convrot_variant_repos": [],
"exclude": [],
"pad": [],
+ "runtime_quant": [],
"zero_row": []
},
"page": "image",
@@ -1373,6 +1433,7 @@
"convrot_variant_repos": [],
"exclude": [],
"pad": [],
+ "runtime_quant": [],
"zero_row": []
},
"auto_bf16_when_resident": "a small denoiser stays bf16 while it fits (INT8 / FP8 change image detail: FP8 LPIPS 0.065 on FLUX.2-klein-4B, 0.103 on Z-Image-Turbo)",
@@ -1386,6 +1447,7 @@
"convrot_variant_repos": [],
"exclude": [],
"pad": [],
+ "runtime_quant": [],
"zero_row": []
},
"convrot_default_on": false,
@@ -1399,6 +1461,7 @@
"convrot_variant_repos": [],
"exclude": [],
"pad": [],
+ "runtime_quant": [],
"zero_row": []
},
"int8": {
@@ -1420,6 +1483,7 @@
"time_embed"
],
"pad": [],
+ "runtime_quant": [],
"zero_row": []
},
"mxfp8": {
@@ -1432,6 +1496,7 @@
"convrot_variant_repos": [],
"exclude": [],
"pad": [],
+ "runtime_quant": [],
"zero_row": []
},
"nvfp4": {
@@ -1444,6 +1509,7 @@
"convrot_variant_repos": [],
"exclude": [],
"pad": [],
+ "runtime_quant": [],
"zero_row": []
},
"page": "image",
@@ -1467,6 +1533,7 @@
"context_embedder"
],
"pad": [],
+ "runtime_quant": [],
"zero_row": []
},
"auto_bf16_when_resident": "a small denoiser stays bf16 while it fits (INT8 / FP8 change image detail: FP8 LPIPS 0.065 on FLUX.2-klein-4B, 0.103 on Z-Image-Turbo)",
@@ -1486,6 +1553,7 @@
"context_embedder"
],
"pad": [],
+ "runtime_quant": [],
"zero_row": []
},
"convrot_default_on": false,
@@ -1505,6 +1573,7 @@
"context_embedder"
],
"pad": [],
+ "runtime_quant": [],
"zero_row": []
},
"int8": {
@@ -1531,6 +1600,7 @@
"context_embedder"
],
"pad": [],
+ "runtime_quant": [],
"zero_row": []
},
"mxfp8": {
@@ -1549,6 +1619,7 @@
"context_embedder"
],
"pad": [],
+ "runtime_quant": [],
"zero_row": []
},
"nvfp4": {
@@ -1567,6 +1638,7 @@
"context_embedder"
],
"pad": [],
+ "runtime_quant": [],
"zero_row": []
},
"page": "image",
@@ -1590,6 +1662,7 @@
"context_embedder"
],
"pad": [],
+ "runtime_quant": [],
"zero_row": []
},
"auto_bf16_when_resident": "a small denoiser stays bf16 while it fits (INT8 / FP8 change image detail: FP8 LPIPS 0.065 on FLUX.2-klein-4B, 0.103 on Z-Image-Turbo)",
@@ -1609,6 +1682,7 @@
"context_embedder"
],
"pad": [],
+ "runtime_quant": [],
"zero_row": []
},
"convrot_default_on": false,
@@ -1628,6 +1702,7 @@
"context_embedder"
],
"pad": [],
+ "runtime_quant": [],
"zero_row": []
},
"int8": {
@@ -1654,6 +1729,7 @@
"context_embedder"
],
"pad": [],
+ "runtime_quant": [],
"zero_row": []
},
"mxfp8": {
@@ -1672,6 +1748,7 @@
"context_embedder"
],
"pad": [],
+ "runtime_quant": [],
"zero_row": []
},
"nvfp4": {
@@ -1690,6 +1767,7 @@
"context_embedder"
],
"pad": [],
+ "runtime_quant": [],
"zero_row": []
},
"page": "image",
@@ -1707,6 +1785,7 @@
"convrot_variant_repos": [],
"exclude": [],
"pad": [],
+ "runtime_quant": [],
"zero_row": []
},
"auto_bf16_when_resident": "INT8 is barely faster than compiled bf16 on a card that holds bf16 and changes image detail",
@@ -1720,6 +1799,7 @@
"convrot_variant_repos": [],
"exclude": [],
"pad": [],
+ "runtime_quant": [],
"zero_row": []
},
"convrot_default_on": false,
@@ -1733,6 +1813,7 @@
"convrot_variant_repos": [],
"exclude": [],
"pad": [],
+ "runtime_quant": [],
"zero_row": []
},
"int8": {
@@ -1754,6 +1835,7 @@
"time_embed"
],
"pad": [],
+ "runtime_quant": [],
"zero_row": []
},
"mxfp8": {
@@ -1766,6 +1848,7 @@
"convrot_variant_repos": [],
"exclude": [],
"pad": [],
+ "runtime_quant": [],
"zero_row": []
},
"nvfp4": {
@@ -1778,6 +1861,7 @@
"convrot_variant_repos": [],
"exclude": [],
"pad": [],
+ "runtime_quant": [],
"zero_row": []
},
"page": "image",
@@ -1795,6 +1879,7 @@
"convrot_variant_repos": [],
"exclude": [],
"pad": [],
+ "runtime_quant": [],
"zero_row": []
},
"auto_bf16_when_resident": null,
@@ -1808,6 +1893,7 @@
"convrot_variant_repos": [],
"exclude": [],
"pad": [],
+ "runtime_quant": [],
"zero_row": []
},
"convrot_default_on": false,
@@ -1821,6 +1907,7 @@
"convrot_variant_repos": [],
"exclude": [],
"pad": [],
+ "runtime_quant": [],
"zero_row": []
},
"int8": {
@@ -1842,6 +1929,7 @@
"time_embed"
],
"pad": [],
+ "runtime_quant": [],
"zero_row": []
},
"mxfp8": {
@@ -1854,6 +1942,7 @@
"convrot_variant_repos": [],
"exclude": [],
"pad": [],
+ "runtime_quant": [],
"zero_row": []
},
"nvfp4": {
@@ -1866,6 +1955,7 @@
"convrot_variant_repos": [],
"exclude": [],
"pad": [],
+ "runtime_quant": [],
"zero_row": []
},
"page": "image",
@@ -1883,6 +1973,7 @@
"convrot_variant_repos": [],
"exclude": [],
"pad": [],
+ "runtime_quant": [],
"zero_row": []
},
"auto_bf16_when_resident": null,
@@ -1896,6 +1987,7 @@
"convrot_variant_repos": [],
"exclude": [],
"pad": [],
+ "runtime_quant": [],
"zero_row": []
},
"convrot_default_on": false,
@@ -1909,6 +2001,7 @@
"convrot_variant_repos": [],
"exclude": [],
"pad": [],
+ "runtime_quant": [],
"zero_row": []
},
"int8": {
@@ -1937,6 +2030,7 @@
"ff_context"
],
"pad": [],
+ "runtime_quant": [],
"zero_row": []
},
"mxfp8": {
@@ -1949,6 +2043,7 @@
"convrot_variant_repos": [],
"exclude": [],
"pad": [],
+ "runtime_quant": [],
"zero_row": []
},
"nvfp4": {
@@ -1961,6 +2056,7 @@
"convrot_variant_repos": [],
"exclude": [],
"pad": [],
+ "runtime_quant": [],
"zero_row": [
"image_embedder",
"context_embedder_2"
@@ -1981,6 +2077,7 @@
"convrot_variant_repos": [],
"exclude": [],
"pad": [],
+ "runtime_quant": [],
"zero_row": []
},
"auto_bf16_when_resident": null,
@@ -1994,6 +2091,7 @@
"convrot_variant_repos": [],
"exclude": [],
"pad": [],
+ "runtime_quant": [],
"zero_row": []
},
"convrot_default_on": false,
@@ -2007,6 +2105,7 @@
"convrot_variant_repos": [],
"exclude": [],
"pad": [],
+ "runtime_quant": [],
"zero_row": []
},
"int8": {
@@ -2035,6 +2134,7 @@
"ff_context"
],
"pad": [],
+ "runtime_quant": [],
"zero_row": []
},
"mxfp8": {
@@ -2047,6 +2147,7 @@
"convrot_variant_repos": [],
"exclude": [],
"pad": [],
+ "runtime_quant": [],
"zero_row": []
},
"nvfp4": {
@@ -2059,6 +2160,7 @@
"convrot_variant_repos": [],
"exclude": [],
"pad": [],
+ "runtime_quant": [],
"zero_row": [
"image_embedder",
"context_embedder_2"
@@ -2079,6 +2181,7 @@
"convrot_variant_repos": [],
"exclude": [],
"pad": [],
+ "runtime_quant": [],
"zero_row": []
},
"auto_bf16_when_resident": null,
@@ -2092,6 +2195,7 @@
"convrot_variant_repos": [],
"exclude": [],
"pad": [],
+ "runtime_quant": [],
"zero_row": []
},
"convrot_default_on": false,
@@ -2105,6 +2209,7 @@
"convrot_variant_repos": [],
"exclude": [],
"pad": [],
+ "runtime_quant": [],
"zero_row": []
},
"int8": {
@@ -2126,6 +2231,7 @@
"time_embed"
],
"pad": [],
+ "runtime_quant": [],
"zero_row": []
},
"mxfp8": {
@@ -2138,6 +2244,7 @@
"convrot_variant_repos": [],
"exclude": [],
"pad": [],
+ "runtime_quant": [],
"zero_row": []
},
"nvfp4": {
@@ -2150,6 +2257,7 @@
"convrot_variant_repos": [],
"exclude": [],
"pad": [],
+ "runtime_quant": [],
"zero_row": []
},
"page": "image",
@@ -2167,6 +2275,7 @@
"convrot_variant_repos": [],
"exclude": [],
"pad": [],
+ "runtime_quant": [],
"zero_row": []
},
"auto_bf16_when_resident": null,
@@ -2180,6 +2289,7 @@
"convrot_variant_repos": [],
"exclude": [],
"pad": [],
+ "runtime_quant": [],
"zero_row": []
},
"convrot_default_on": false,
@@ -2193,6 +2303,7 @@
"convrot_variant_repos": [],
"exclude": [],
"pad": [],
+ "runtime_quant": [],
"zero_row": []
},
"int8": {
@@ -2214,6 +2325,7 @@
"time_embed"
],
"pad": [],
+ "runtime_quant": [],
"zero_row": []
},
"mxfp8": {
@@ -2226,6 +2338,7 @@
"convrot_variant_repos": [],
"exclude": [],
"pad": [],
+ "runtime_quant": [],
"zero_row": []
},
"nvfp4": {
@@ -2238,6 +2351,7 @@
"convrot_variant_repos": [],
"exclude": [],
"pad": [],
+ "runtime_quant": [],
"zero_row": []
},
"page": "image",
@@ -2255,6 +2369,7 @@
"convrot_variant_repos": [],
"exclude": [],
"pad": [],
+ "runtime_quant": [],
"zero_row": []
},
"auto_bf16_when_resident": null,
@@ -2268,6 +2383,7 @@
"convrot_variant_repos": [],
"exclude": [],
"pad": [],
+ "runtime_quant": [],
"zero_row": []
},
"convrot_default_on": false,
@@ -2281,6 +2397,7 @@
"convrot_variant_repos": [],
"exclude": [],
"pad": [],
+ "runtime_quant": [],
"zero_row": []
},
"int8": {
@@ -2305,6 +2422,7 @@
"adaln"
],
"pad": [],
+ "runtime_quant": [],
"zero_row": []
},
"mxfp8": {
@@ -2317,6 +2435,7 @@
"convrot_variant_repos": [],
"exclude": [],
"pad": [],
+ "runtime_quant": [],
"zero_row": []
},
"nvfp4": {
@@ -2329,6 +2448,7 @@
"convrot_variant_repos": [],
"exclude": [],
"pad": [],
+ "runtime_quant": [],
"zero_row": []
},
"page": "video",
@@ -2346,6 +2466,7 @@
"convrot_variant_repos": [],
"exclude": [],
"pad": [],
+ "runtime_quant": [],
"zero_row": []
},
"auto_bf16_when_resident": null,
@@ -2359,6 +2480,7 @@
"convrot_variant_repos": [],
"exclude": [],
"pad": [],
+ "runtime_quant": [],
"zero_row": []
},
"convrot_default_on": true,
@@ -2372,6 +2494,7 @@
"convrot_variant_repos": [],
"exclude": [],
"pad": [],
+ "runtime_quant": [],
"zero_row": []
},
"int8": {
@@ -2398,6 +2521,7 @@
"adaln"
],
"pad": [],
+ "runtime_quant": [],
"zero_row": []
},
"mxfp8": {
@@ -2410,6 +2534,7 @@
"convrot_variant_repos": [],
"exclude": [],
"pad": [],
+ "runtime_quant": [],
"zero_row": []
},
"nvfp4": {
@@ -2422,6 +2547,7 @@
"convrot_variant_repos": [],
"exclude": [],
"pad": [],
+ "runtime_quant": [],
"zero_row": []
},
"page": "video",
@@ -2439,6 +2565,7 @@
"convrot_variant_repos": [],
"exclude": [],
"pad": [],
+ "runtime_quant": [],
"zero_row": []
},
"auto_bf16_when_resident": "INT8 is no faster than compiled bf16 on a card that holds bf16 and changes image detail",
@@ -2452,6 +2579,7 @@
"convrot_variant_repos": [],
"exclude": [],
"pad": [],
+ "runtime_quant": [],
"zero_row": []
},
"convrot_default_on": false,
@@ -2465,6 +2593,7 @@
"convrot_variant_repos": [],
"exclude": [],
"pad": [],
+ "runtime_quant": [],
"zero_row": []
},
"int8": {
@@ -2486,6 +2615,7 @@
"time_embed"
],
"pad": [],
+ "runtime_quant": [],
"zero_row": []
},
"mxfp8": {
@@ -2498,6 +2628,7 @@
"convrot_variant_repos": [],
"exclude": [],
"pad": [],
+ "runtime_quant": [],
"zero_row": []
},
"nvfp4": {
@@ -2510,6 +2641,7 @@
"convrot_variant_repos": [],
"exclude": [],
"pad": [],
+ "runtime_quant": [],
"zero_row": []
},
"page": "image",
@@ -2527,6 +2659,7 @@
"convrot_variant_repos": [],
"exclude": [],
"pad": [],
+ "runtime_quant": [],
"zero_row": []
},
"auto_bf16_when_resident": null,
@@ -2540,6 +2673,7 @@
"convrot_variant_repos": [],
"exclude": [],
"pad": [],
+ "runtime_quant": [],
"zero_row": []
},
"convrot_default_on": false,
@@ -2553,6 +2687,7 @@
"convrot_variant_repos": [],
"exclude": [],
"pad": [],
+ "runtime_quant": [],
"zero_row": []
},
"int8": {
@@ -2578,6 +2713,7 @@
"context_embedder",
"token_refiner"
],
+ "runtime_quant": [],
"zero_row": []
},
"mxfp8": {
@@ -2590,6 +2726,7 @@
"convrot_variant_repos": [],
"exclude": [],
"pad": [],
+ "runtime_quant": [],
"zero_row": []
},
"nvfp4": {
@@ -2602,6 +2739,7 @@
"convrot_variant_repos": [],
"exclude": [],
"pad": [],
+ "runtime_quant": [],
"zero_row": []
},
"page": "video",
@@ -2623,6 +2761,7 @@
"modulation"
],
"pad": [],
+ "runtime_quant": [],
"zero_row": []
},
"auto_bf16_when_resident": "a small denoiser stays bf16 while it fits (INT8 / FP8 change image detail: FP8 LPIPS 0.065 on FLUX.2-klein-4B, 0.103 on Z-Image-Turbo)",
@@ -2640,6 +2779,7 @@
"modulation"
],
"pad": [],
+ "runtime_quant": [],
"zero_row": []
},
"convrot_default_on": true,
@@ -2657,6 +2797,7 @@
"modulation"
],
"pad": [],
+ "runtime_quant": [],
"zero_row": []
},
"int8": {
@@ -2690,6 +2831,7 @@
"modulation"
],
"pad": [],
+ "runtime_quant": [],
"zero_row": []
},
"mxfp8": {
@@ -2706,6 +2848,7 @@
"modulation"
],
"pad": [],
+ "runtime_quant": [],
"zero_row": []
},
"nvfp4": {
@@ -2722,6 +2865,7 @@
"modulation"
],
"pad": [],
+ "runtime_quant": [],
"zero_row": []
},
"page": "image",
@@ -2739,6 +2883,7 @@
"convrot_variant_repos": [],
"exclude": [],
"pad": [],
+ "runtime_quant": [],
"zero_row": []
},
"auto_bf16_when_resident": null,
@@ -2752,6 +2897,7 @@
"convrot_variant_repos": [],
"exclude": [],
"pad": [],
+ "runtime_quant": [],
"zero_row": []
},
"convrot_default_on": false,
@@ -2765,6 +2911,7 @@
"convrot_variant_repos": [],
"exclude": [],
"pad": [],
+ "runtime_quant": [],
"zero_row": []
},
"int8": {
@@ -2792,6 +2939,13 @@
"txt_mlp"
],
"pad": [],
+ "runtime_quant": [
+ "add_q_proj",
+ "add_k_proj",
+ "add_v_proj",
+ "to_add_out",
+ "txt_mlp"
+ ],
"zero_row": []
},
"mxfp8": {
@@ -2804,6 +2958,7 @@
"convrot_variant_repos": [],
"exclude": [],
"pad": [],
+ "runtime_quant": [],
"zero_row": []
},
"nvfp4": {
@@ -2816,6 +2971,7 @@
"convrot_variant_repos": [],
"exclude": [],
"pad": [],
+ "runtime_quant": [],
"zero_row": []
},
"page": "image",
@@ -2833,6 +2989,7 @@
"convrot_variant_repos": [],
"exclude": [],
"pad": [],
+ "runtime_quant": [],
"zero_row": []
},
"auto_bf16_when_resident": null,
@@ -2846,6 +3003,7 @@
"convrot_variant_repos": [],
"exclude": [],
"pad": [],
+ "runtime_quant": [],
"zero_row": []
},
"convrot_default_on": true,
@@ -2859,6 +3017,7 @@
"convrot_variant_repos": [],
"exclude": [],
"pad": [],
+ "runtime_quant": [],
"zero_row": []
},
"int8": {
@@ -2892,6 +3051,7 @@
"txt_in"
],
"pad": [],
+ "runtime_quant": [],
"zero_row": []
},
"mxfp8": {
@@ -2904,6 +3064,7 @@
"convrot_variant_repos": [],
"exclude": [],
"pad": [],
+ "runtime_quant": [],
"zero_row": []
},
"nvfp4": {
@@ -2916,6 +3077,7 @@
"convrot_variant_repos": [],
"exclude": [],
"pad": [],
+ "runtime_quant": [],
"zero_row": []
},
"page": "image",
@@ -2933,6 +3095,7 @@
"convrot_variant_repos": [],
"exclude": [],
"pad": [],
+ "runtime_quant": [],
"zero_row": []
},
"auto_bf16_when_resident": null,
@@ -2946,6 +3109,7 @@
"convrot_variant_repos": [],
"exclude": [],
"pad": [],
+ "runtime_quant": [],
"zero_row": []
},
"convrot_default_on": false,
@@ -2959,6 +3123,7 @@
"convrot_variant_repos": [],
"exclude": [],
"pad": [],
+ "runtime_quant": [],
"zero_row": []
},
"int8": {
@@ -2986,6 +3151,7 @@
"txt_mlp"
],
"pad": [],
+ "runtime_quant": [],
"zero_row": []
},
"mxfp8": {
@@ -2998,6 +3164,7 @@
"convrot_variant_repos": [],
"exclude": [],
"pad": [],
+ "runtime_quant": [],
"zero_row": []
},
"nvfp4": {
@@ -3010,6 +3177,7 @@
"convrot_variant_repos": [],
"exclude": [],
"pad": [],
+ "runtime_quant": [],
"zero_row": []
},
"page": "image",
@@ -3027,6 +3195,7 @@
"convrot_variant_repos": [],
"exclude": [],
"pad": [],
+ "runtime_quant": [],
"zero_row": []
},
"auto_bf16_when_resident": null,
@@ -3040,6 +3209,7 @@
"convrot_variant_repos": [],
"exclude": [],
"pad": [],
+ "runtime_quant": [],
"zero_row": []
},
"convrot_default_on": false,
@@ -3053,6 +3223,7 @@
"convrot_variant_repos": [],
"exclude": [],
"pad": [],
+ "runtime_quant": [],
"zero_row": []
},
"int8": {
@@ -3080,6 +3251,7 @@
"txt_mlp"
],
"pad": [],
+ "runtime_quant": [],
"zero_row": []
},
"mxfp8": {
@@ -3092,6 +3264,7 @@
"convrot_variant_repos": [],
"exclude": [],
"pad": [],
+ "runtime_quant": [],
"zero_row": []
},
"nvfp4": {
@@ -3104,6 +3277,7 @@
"convrot_variant_repos": [],
"exclude": [],
"pad": [],
+ "runtime_quant": [],
"zero_row": []
},
"page": "image",
@@ -3121,6 +3295,7 @@
"convrot_variant_repos": [],
"exclude": [],
"pad": [],
+ "runtime_quant": [],
"zero_row": []
},
"auto_bf16_when_resident": null,
@@ -3134,6 +3309,7 @@
"convrot_variant_repos": [],
"exclude": [],
"pad": [],
+ "runtime_quant": [],
"zero_row": []
},
"convrot_default_on": false,
@@ -3147,6 +3323,7 @@
"convrot_variant_repos": [],
"exclude": [],
"pad": [],
+ "runtime_quant": [],
"zero_row": []
},
"int8": {
@@ -3168,6 +3345,7 @@
"time_embed"
],
"pad": [],
+ "runtime_quant": [],
"zero_row": []
},
"mxfp8": {
@@ -3180,6 +3358,7 @@
"convrot_variant_repos": [],
"exclude": [],
"pad": [],
+ "runtime_quant": [],
"zero_row": []
},
"nvfp4": {
@@ -3192,6 +3371,7 @@
"convrot_variant_repos": [],
"exclude": [],
"pad": [],
+ "runtime_quant": [],
"zero_row": []
},
"page": "image",
@@ -3209,6 +3389,7 @@
"convrot_variant_repos": [],
"exclude": [],
"pad": [],
+ "runtime_quant": [],
"zero_row": []
},
"auto_bf16_when_resident": null,
@@ -3222,6 +3403,7 @@
"convrot_variant_repos": [],
"exclude": [],
"pad": [],
+ "runtime_quant": [],
"zero_row": []
},
"convrot_default_on": false,
@@ -3235,6 +3417,7 @@
"convrot_variant_repos": [],
"exclude": [],
"pad": [],
+ "runtime_quant": [],
"zero_row": []
},
"int8": {
@@ -3256,6 +3439,7 @@
"time_embed"
],
"pad": [],
+ "runtime_quant": [],
"zero_row": []
},
"mxfp8": {
@@ -3268,6 +3452,7 @@
"convrot_variant_repos": [],
"exclude": [],
"pad": [],
+ "runtime_quant": [],
"zero_row": []
},
"nvfp4": {
@@ -3280,6 +3465,7 @@
"convrot_variant_repos": [],
"exclude": [],
"pad": [],
+ "runtime_quant": [],
"zero_row": []
},
"page": "image",
@@ -3297,6 +3483,7 @@
"convrot_variant_repos": [],
"exclude": [],
"pad": [],
+ "runtime_quant": [],
"zero_row": []
},
"auto_bf16_when_resident": null,
@@ -3310,6 +3497,7 @@
"convrot_variant_repos": [],
"exclude": [],
"pad": [],
+ "runtime_quant": [],
"zero_row": []
},
"convrot_default_on": false,
@@ -3323,6 +3511,7 @@
"convrot_variant_repos": [],
"exclude": [],
"pad": [],
+ "runtime_quant": [],
"zero_row": []
},
"int8": {
@@ -3344,6 +3533,7 @@
"time_embed"
],
"pad": [],
+ "runtime_quant": [],
"zero_row": []
},
"mxfp8": {
@@ -3356,6 +3546,7 @@
"convrot_variant_repos": [],
"exclude": [],
"pad": [],
+ "runtime_quant": [],
"zero_row": []
},
"nvfp4": {
@@ -3368,6 +3559,7 @@
"convrot_variant_repos": [],
"exclude": [],
"pad": [],
+ "runtime_quant": [],
"zero_row": []
},
"page": "video",
@@ -3385,6 +3577,7 @@
"convrot_variant_repos": [],
"exclude": [],
"pad": [],
+ "runtime_quant": [],
"zero_row": []
},
"auto_bf16_when_resident": null,
@@ -3398,6 +3591,7 @@
"convrot_variant_repos": [],
"exclude": [],
"pad": [],
+ "runtime_quant": [],
"zero_row": []
},
"convrot_default_on": false,
@@ -3411,6 +3605,7 @@
"convrot_variant_repos": [],
"exclude": [],
"pad": [],
+ "runtime_quant": [],
"zero_row": []
},
"int8": {
@@ -3432,6 +3627,7 @@
"time_embed"
],
"pad": [],
+ "runtime_quant": [],
"zero_row": []
},
"mxfp8": {
@@ -3444,6 +3640,7 @@
"convrot_variant_repos": [],
"exclude": [],
"pad": [],
+ "runtime_quant": [],
"zero_row": []
},
"nvfp4": {
@@ -3456,6 +3653,7 @@
"convrot_variant_repos": [],
"exclude": [],
"pad": [],
+ "runtime_quant": [],
"zero_row": []
},
"page": "video",
@@ -3473,6 +3671,7 @@
"convrot_variant_repos": [],
"exclude": [],
"pad": [],
+ "runtime_quant": [],
"zero_row": []
},
"auto_bf16_when_resident": null,
@@ -3486,6 +3685,7 @@
"convrot_variant_repos": [],
"exclude": [],
"pad": [],
+ "runtime_quant": [],
"zero_row": []
},
"convrot_default_on": false,
@@ -3499,6 +3699,7 @@
"convrot_variant_repos": [],
"exclude": [],
"pad": [],
+ "runtime_quant": [],
"zero_row": []
},
"int8": {
@@ -3520,6 +3721,7 @@
"time_embed"
],
"pad": [],
+ "runtime_quant": [],
"zero_row": []
},
"mxfp8": {
@@ -3532,6 +3734,7 @@
"convrot_variant_repos": [],
"exclude": [],
"pad": [],
+ "runtime_quant": [],
"zero_row": []
},
"nvfp4": {
@@ -3544,6 +3747,7 @@
"convrot_variant_repos": [],
"exclude": [],
"pad": [],
+ "runtime_quant": [],
"zero_row": []
},
"page": "video",
@@ -3561,6 +3765,7 @@
"convrot_variant_repos": [],
"exclude": [],
"pad": [],
+ "runtime_quant": [],
"zero_row": []
},
"auto_bf16_when_resident": null,
@@ -3574,6 +3779,7 @@
"convrot_variant_repos": [],
"exclude": [],
"pad": [],
+ "runtime_quant": [],
"zero_row": []
},
"convrot_default_on": false,
@@ -3587,6 +3793,7 @@
"convrot_variant_repos": [],
"exclude": [],
"pad": [],
+ "runtime_quant": [],
"zero_row": []
},
"int8": {
@@ -3608,6 +3815,7 @@
"time_embed"
],
"pad": [],
+ "runtime_quant": [],
"zero_row": []
},
"mxfp8": {
@@ -3620,6 +3828,7 @@
"convrot_variant_repos": [],
"exclude": [],
"pad": [],
+ "runtime_quant": [],
"zero_row": []
},
"nvfp4": {
@@ -3632,6 +3841,7 @@
"convrot_variant_repos": [],
"exclude": [],
"pad": [],
+ "runtime_quant": [],
"zero_row": []
},
"page": "video",
@@ -3649,6 +3859,7 @@
"convrot_variant_repos": [],
"exclude": [],
"pad": [],
+ "runtime_quant": [],
"zero_row": []
},
"auto_bf16_when_resident": "a small denoiser stays bf16 while it fits (INT8 / FP8 change image detail: FP8 LPIPS 0.065 on FLUX.2-klein-4B, 0.103 on Z-Image-Turbo)",
@@ -3662,6 +3873,7 @@
"convrot_variant_repos": [],
"exclude": [],
"pad": [],
+ "runtime_quant": [],
"zero_row": []
},
"convrot_default_on": true,
@@ -3675,6 +3887,7 @@
"convrot_variant_repos": [],
"exclude": [],
"pad": [],
+ "runtime_quant": [],
"zero_row": []
},
"int8": {
@@ -3705,6 +3918,7 @@
"time_embed"
],
"pad": [],
+ "runtime_quant": [],
"zero_row": []
},
"mxfp8": {
@@ -3717,6 +3931,7 @@
"convrot_variant_repos": [],
"exclude": [],
"pad": [],
+ "runtime_quant": [],
"zero_row": []
},
"nvfp4": {
@@ -3729,6 +3944,7 @@
"convrot_variant_repos": [],
"exclude": [],
"pad": [],
+ "runtime_quant": [],
"zero_row": []
},
"page": "image",
diff --git a/studio/backend/tests/test_diffusion_family_data_freeze.py b/studio/backend/tests/test_diffusion_family_data_freeze.py
index 128f0156281..d27ac6594fc 100644
--- a/studio/backend/tests/test_diffusion_family_data_freeze.py
+++ b/studio/backend/tests/test_diffusion_family_data_freeze.py
@@ -38,6 +38,13 @@ def _jsonable(value):
return value
+def _streamed(fn, *args):
+ """``fn`` inside a streamed build's scope, where the runtime-quantised tokens apply."""
+ from core.inference import diffusion_transformer_quant as tq
+ with tq.int8_text_stream_scope(True):
+ return list(fn(*args))
+
+
def _snapshot() -> dict:
from core.inference import diffusion_transformer_quant as tq
from core.inference import diffusion_precision as prec
@@ -75,7 +82,8 @@ def _snapshot() -> dict:
for scheme in schemes:
variant_repos = [r for r in repos if tq.convrot_prequant_variant_repo(scheme, name, r)]
row[scheme] = {
- "exclude": list(tq.exclude_tokens_for_scheme(scheme, name)),
+ "exclude": list(tq.exclude_tokens_for_scheme(scheme, name, baked = True)),
+ "runtime_quant": _streamed(tq.runtime_quant_tokens_for_scheme, scheme, name),
"pad": list(tq.pad_tokens_for_scheme(scheme, name)),
"zero_row": list(tq.zero_row_tokens_for_scheme(scheme, name)),
"convrot_spec": _jsonable(tq.convrot_spec_for_scheme(scheme, name)),
@@ -181,6 +189,7 @@ def _snapshot() -> dict:
def _clean_env(monkeypatch):
for var in (
"UNSLOTH_DIFFUSION_INT8_CONVROT",
+ "UNSLOTH_DIFFUSION_INT8_TEXT_STREAM",
"UNSLOTH_DIFFUSION_NVFP4",
"UNSLOTH_DIFFUSION_AUTO_STEP_SKIP",
"UNSLOTH_VIDEO_VAE_UNTILED",
diff --git a/studio/backend/tests/test_diffusion_h3_trainer.py b/studio/backend/tests/test_diffusion_h3_trainer.py
index 0078d60c0d1..67e0cd78dc9 100644
--- a/studio/backend/tests/test_diffusion_h3_trainer.py
+++ b/studio/backend/tests/test_diffusion_h3_trainer.py
@@ -799,7 +799,8 @@ def test_int8_training_applies_h3s_small_m_guards():
assert "_int8_quantize_base(transformer, cfg.resolved_family)" in src
dit_src = Path(dit.__file__).read_text()
body = dit_src.split("def _int8_quantize_base", 1)[1].split("\ndef ", 1)[0]
- assert 'exclude_tokens_for_scheme("int8", family)' in body
+ # training keeps the baked list: the streamed-inference text-stream scope never reaches it
+ assert 'exclude_tokens_for_scheme("int8", family, baked = True)' in body
assert 'apply_small_m_padding(transformer, "int8", family)' in body
diff --git a/studio/backend/tests/test_diffusion_prequant.py b/studio/backend/tests/test_diffusion_prequant.py
index 860e2d5ef71..b522ded1e29 100644
--- a/studio/backend/tests/test_diffusion_prequant.py
+++ b/studio/backend/tests/test_diffusion_prequant.py
@@ -2885,3 +2885,71 @@ def test_task_specific_artifacts_get_no_twin():
)
src = resolve_prequant_source(fam, "int8", task = "keyframe")
assert src.candidate_filenames == ("Model-KF-INT8.safetensors",)
+
+
+def _qwen_int8_ckpt():
+ from core.inference.diffusion_transformer_quant import exclude_tokens_for_scheme
+
+ ckpt = _good_ckpt(scheme = "int8")
+ ckpt["metadata"].update(
+ {
+ "family": "qwen-image",
+ "exclude_name_tokens": list(
+ exclude_tokens_for_scheme("int8", "qwen-image", baked = True)
+ ),
+ "min_features": 512,
+ }
+ )
+ return ckpt
+
+
+def test_load_tops_up_the_dense_text_stream_of_a_baked_artifact(monkeypatch, tmp_path):
+ import core.inference.diffusion_transformer_quant as tq
+
+ seen = []
+ monkeypatch.setattr(
+ tq,
+ "quantize_runtime_leftovers",
+ lambda transformer, scheme, family, baked, **kw: seen.append(
+ (scheme, family, list(baked), kw)
+ )
+ or 0,
+ )
+ ckpt = _qwen_int8_ckpt()
+ t = _load(monkeypatch, tmp_path, ckpt, scheme = "int8")
+ assert t is not None
+ assert len(seen) == 1
+ scheme, family, baked, kw = seen[0]
+ assert (scheme, family) == ("int8", "qwen-image")
+ assert baked == ckpt["metadata"]["exclude_name_tokens"]
+ assert kw["min_features"] == 512 and kw["device"] == "cuda"
+
+
+def test_load_survives_a_failed_top_up(monkeypatch, tmp_path):
+ import core.inference.diffusion_transformer_quant as tq
+
+ def boom(*a, **k):
+ raise RuntimeError("no torchao")
+
+ monkeypatch.setattr(tq, "quantize_runtime_leftovers", boom)
+ assert _load(monkeypatch, tmp_path, _qwen_int8_ckpt(), scheme = "int8") is not None
+
+
+def test_kill_switch_refuses_an_artifact_built_with_the_int8_text_stream(monkeypatch, tmp_path):
+ from core.inference.diffusion_transformer_quant import (
+ INT8_TEXT_STREAM_ENV,
+ exclude_tokens_for_scheme,
+ )
+
+ ckpt = _qwen_int8_ckpt()
+ from core.inference.diffusion_transformer_quant import int8_text_stream_scope
+
+ with int8_text_stream_scope(True):
+ ckpt["metadata"]["exclude_name_tokens"] = list(
+ exclude_tokens_for_scheme("int8", "qwen-image")
+ )
+ assert _load(monkeypatch, tmp_path, ckpt, scheme = "int8") is not None
+ assert _load(monkeypatch, tmp_path, ckpt, scheme = "int8") is None
+ monkeypatch.setenv(INT8_TEXT_STREAM_ENV, "0")
+ with int8_text_stream_scope(True):
+ assert _load(monkeypatch, tmp_path, ckpt, scheme = "int8") is None
diff --git a/studio/backend/tests/test_diffusion_qwenimage_text_int8.py b/studio/backend/tests/test_diffusion_qwenimage_text_int8.py
new file mode 100644
index 00000000000..1d2a48a8fa1
--- /dev/null
+++ b/studio/backend/tests/test_diffusion_qwenimage_text_int8.py
@@ -0,0 +1,278 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
+
+"""Qwen-Image streamed int8 text stream (``int8_text_stream_scope``). CPU-only."""
+
+from __future__ import annotations
+
+import pytest
+
+import core.inference.diffusion_transformer_quant as tq
+from core.inference.diffusion_transformer_quant import TQ_FP8, TQ_INT8, exclude_tokens_for_scheme
+
+_TEXT_STREAM = ("add_q_proj", "add_k_proj", "add_v_proj", "to_add_out", "txt_mlp")
+
+
+_UNSCOPED = {
+ "test_resident_build_is_main",
+ "test_streamed_scope_is_scoped_and_nests",
+ "test_streamed_scope_resets_on_error",
+ "test_streamed_scope_does_not_leak_into_other_threads",
+ "test_load_wraps_lookup_build_and_write_in_the_placement_scope",
+}
+
+
[email protected](autouse = True)
+def _default_switch(monkeypatch, request):
+ monkeypatch.delenv(tq.INT8_TEXT_STREAM_ENV, raising = False)
+ if request.node.name in _UNSCOPED:
+ yield
+ return
+ with tq.int8_text_stream_scope(True):
+ yield
+
+
+def test_resident_build_is_main():
+ assert tq.runtime_quant_tokens_for_scheme(TQ_INT8, "qwen-image") == ()
+ assert exclude_tokens_for_scheme(TQ_INT8, "qwen-image") == exclude_tokens_for_scheme(
+ TQ_INT8, "qwen-image", baked = True
+ )
+ assert tq.pad_tokens_for_scheme(TQ_INT8, "qwen-image") == ()
+ with tq.int8_text_stream_scope(False):
+ assert tq.runtime_quant_tokens_for_scheme(TQ_INT8, "qwen-image") == ()
+
+
+def test_streamed_scope_is_scoped_and_nests():
+ with tq.int8_text_stream_scope(True):
+ assert tq.runtime_quant_tokens_for_scheme(TQ_INT8, "qwen-image") == _TEXT_STREAM
+ assert tq.pad_tokens_for_scheme(TQ_INT8, "qwen-image") == _TEXT_STREAM
+ with tq.int8_text_stream_scope(False):
+ assert tq.runtime_quant_tokens_for_scheme(TQ_INT8, "qwen-image") == ()
+ assert tq.runtime_quant_tokens_for_scheme(TQ_INT8, "qwen-image") == _TEXT_STREAM
+ assert tq.runtime_quant_tokens_for_scheme(TQ_INT8, "qwen-image") == ()
+
+
+def test_streamed_scope_resets_on_error():
+ with pytest.raises(RuntimeError):
+ with tq.int8_text_stream_scope(True):
+ raise RuntimeError("load failed")
+ assert tq.runtime_quant_tokens_for_scheme(TQ_INT8, "qwen-image") == ()
+
+
+def test_streamed_scope_does_not_leak_into_other_threads():
+ import threading
+
+ seen = []
+ with tq.int8_text_stream_scope(True):
+ t = threading.Thread(
+ target = lambda: seen.append(tq.runtime_quant_tokens_for_scheme(TQ_INT8, "qwen-image"))
+ )
+ t.start()
+ t.join()
+ assert seen == [()]
+
+
+def test_load_wraps_lookup_build_and_write_in_the_placement_scope():
+ import inspect
+
+ from core.inference import diffusion
+
+ src = inspect.getsource(diffusion.DiffusionBackend)
+ assert src.count("with int8_text_stream_scope(not keeps_resident):") == 3
+ for call in (
+ "self._pipeline_quant_from_cache(",
+ "quantize_transformer(",
+ "self._save_pipeline_quant_to_cache(",
+ ):
+ i = src.index(call, src.index("with int8_text_stream_scope(not keeps_resident):"))
+ assert "with int8_text_stream_scope(not keeps_resident):" in src[max(0, i - 400) : i], call
+
+
+def test_runtime_exclusions_drop_the_text_stream():
+ tokens = exclude_tokens_for_scheme(TQ_INT8, "qwen-image")
+ for token in _TEXT_STREAM:
+ assert token not in tokens
+ assert "txt_in" in tokens
+ assert set(tq._INT8_EXCLUDE_NAME_TOKENS) <= set(tokens)
+ assert tq.runtime_quant_tokens_for_scheme(TQ_INT8, "qwen-image") == _TEXT_STREAM
+ assert exclude_tokens_for_scheme(TQ_INT8, " Qwen-Image ") == tokens
+
+
+def test_baked_list_is_unchanged():
+ assert (
+ exclude_tokens_for_scheme(TQ_INT8, "qwen-image", baked = True)
+ == tq._INT8_EXCLUDE_NAME_TOKENS + tq._QWENIMAGE_INT8_EXCLUDES
+ )
+ for token in ("txt_in",) + _TEXT_STREAM:
+ assert token in tq._QWENIMAGE_INT8_EXCLUDES
+
+
[email protected]("value", ["0", "off", "false", "no", " OFF "])
+def test_kill_switch_restores_the_dense_text_stream(monkeypatch, value):
+ monkeypatch.setenv(tq.INT8_TEXT_STREAM_ENV, value)
+ assert tq.runtime_quant_tokens_for_scheme(TQ_INT8, "qwen-image") == ()
+ assert exclude_tokens_for_scheme(TQ_INT8, "qwen-image") == exclude_tokens_for_scheme(
+ TQ_INT8, "qwen-image", baked = True
+ )
+
+
+ "family", ["qwen-image-edit", "qwen-image-layered", "qwen-image-2.1", "z-image", None]
+)
+def test_other_families_are_untouched(family):
+ # only qwen-image t2i was measured; the edit / layered siblings share the DiT but not the evidence
+ assert tq.runtime_quant_tokens_for_scheme(TQ_INT8, family) == ()
+ assert exclude_tokens_for_scheme(TQ_INT8, family) == exclude_tokens_for_scheme(
+ TQ_INT8, family, baked = True
+ )
+
+
+def test_other_schemes_are_untouched():
+ assert tq.runtime_quant_tokens_for_scheme(TQ_FP8, "qwen-image") == ()
+ assert exclude_tokens_for_scheme(TQ_FP8, "qwen-image") == ()
+
+
+def test_runtime_tokens_are_a_subset_of_the_baked_tokens():
+ # _validate_checkpoint accepts an artifact whose exclusions cover the runtime list: a hosted artifact built under
+ # the baked list must keep validating, so a runtime token can only ever REMOVE a baked one
+ for family, runtime in tq._INT8_FAMILY_RUNTIME_QUANT_TOKENS.items():
+ baked = tq._INT8_FAMILY_EXCLUDE_NAME_TOKENS.get(family, ())
+ assert set(runtime) <= set(baked), family
+ assert set(exclude_tokens_for_scheme(TQ_INT8, family)) <= set(
+ exclude_tokens_for_scheme(TQ_INT8, family, baked = True)
+ )
+
+
+def test_training_keeps_the_baked_list():
+ from core.training.diffusion_dit_trainer import _lowp_exclude_tokens
+ tokens = _lowp_exclude_tokens("qwen-image")
+ for token in _TEXT_STREAM:
+ assert token in tokens
+
+
+def test_compile_switch_follows_the_kill_switch(monkeypatch):
+ from core.inference import diffusion_compile_cache as cc
+
+ assert "UNSLOTH_DIFFUSION_INT8_TEXT_STREAM" in cc._GRAPH_SWITCHES
+ on = exclude_tokens_for_scheme(TQ_INT8, "qwen-image")
+ monkeypatch.setenv(tq.INT8_TEXT_STREAM_ENV, "0")
+ assert exclude_tokens_for_scheme(TQ_INT8, "qwen-image") != on
+ assert cc.graph_switches() == {"UNSLOTH_DIFFUSION_INT8_TEXT_STREAM": "off"}
+
+
+def _tiny_qwen_like(torch):
+ nn = torch.nn
+
+ class Attn(nn.Module):
+ def __init__(self, d):
+ super().__init__()
+ for name in ("to_q", "add_q_proj", "add_k_proj", "add_v_proj", "to_add_out"):
+ setattr(self, name, nn.Linear(d, d))
+
+ class FF(nn.Module):
+ def __init__(self, d):
+ super().__init__()
+ self.net = nn.ModuleList([nn.Linear(d, 2 * d), nn.GELU(), nn.Linear(2 * d, d)])
+
+ class Block(nn.Module):
+ def __init__(self, d):
+ super().__init__()
+ self.attn = Attn(d)
+ self.txt_mlp = FF(d)
+ self.img_mlp = FF(d)
+ self.txt_mod = nn.Sequential(nn.SiLU(), nn.Linear(d, 6 * d))
+
+ class Model(nn.Module):
+ def __init__(self, d = 512):
+ super().__init__()
+ self.txt_in = nn.Linear(d, d)
+ self.transformer_blocks = nn.ModuleList([Block(d), Block(d)])
+ self.small = nn.Linear(64, 64)
+
+ return Model().to(torch.bfloat16)
+
+
+def _kinds(model, torch):
+ return {
+ name: type(module.weight).__name__
+ for name, module in model.named_modules()
+ if isinstance(module, torch.nn.Linear)
+ }
+
+
+def test_leftovers_quantise_exactly_the_text_stream():
+ torch = pytest.importorskip("torch")
+ pytest.importorskip("torchao")
+ model = _tiny_qwen_like(torch)
+ baked = list(exclude_tokens_for_scheme(TQ_INT8, "qwen-image", baked = True))
+ x = torch.randn(3, 512, dtype = torch.bfloat16)
+ before = model.transformer_blocks[0].attn.add_q_proj(x)
+ n = tq.quantize_runtime_leftovers(model, TQ_INT8, "qwen-image", baked)
+ kinds = _kinds(model, torch)
+ text = [k for k in kinds if any(t in k for t in _TEXT_STREAM)]
+ assert n == len(text) == 2 * (4 + 2)
+ for name, kind in kinds.items():
+ quantised = kind not in ("Parameter", "Tensor")
+ assert quantised == (name in text), name
+ assert tq.quantize_runtime_leftovers(model, TQ_INT8, "qwen-image", baked) == 0
+ after = model.transformer_blocks[0].attn.add_q_proj(x)
+ assert after.shape == before.shape
+ assert torch.isfinite(after.float()).all()
+ rel = (after.float() - before.float()).norm() / before.float().norm()
+ assert rel < 0.05
+
+
+def test_leftovers_noop_cases(monkeypatch):
+ torch = pytest.importorskip("torch")
+ pytest.importorskip("torchao")
+ baked = list(exclude_tokens_for_scheme(TQ_INT8, "qwen-image", baked = True))
+ assert (
+ tq.quantize_runtime_leftovers(
+ _tiny_qwen_like(torch), TQ_INT8, "qwen-image", ["norm", "txt_in"]
+ )
+ == 0
+ )
+ assert tq.quantize_runtime_leftovers(_tiny_qwen_like(torch), TQ_INT8, "qwen-image", None) == 0
+ assert tq.quantize_runtime_leftovers(_tiny_qwen_like(torch), TQ_FP8, "qwen-image", baked) == 0
+ assert (
+ tq.quantize_runtime_leftovers(_tiny_qwen_like(torch), TQ_INT8, "qwen-image-edit", baked)
+ == 0
+ )
+ monkeypatch.setenv(tq.INT8_TEXT_STREAM_ENV, "0")
+ model = _tiny_qwen_like(torch)
+ assert tq.quantize_runtime_leftovers(model, TQ_INT8, "qwen-image", baked) == 0
+ assert set(_kinds(model, torch).values()) == {"Parameter"}
+
+
+def test_leftovers_ignore_a_non_cuda_work_device():
+ torch = pytest.importorskip("torch")
+ pytest.importorskip("torchao")
+ model = _tiny_qwen_like(torch)
+ baked = list(exclude_tokens_for_scheme(TQ_INT8, "qwen-image", baked = True))
+ assert tq.quantize_runtime_leftovers(model, TQ_INT8, "qwen-image", baked, device = "cpu") == 12
+ assert all(p.device.type == "cpu" for p in model.parameters())
+
+
+def test_local_prequant_cache_key_follows_the_switch(monkeypatch):
+ pytest.importorskip("torch")
+ from core.inference import diffusion_prequant_cache as pc
+
+ def key():
+ parts = pc.key_parts(
+ family = "qwen-image",
+ transformer_class = "QwenImageTransformer2DModel",
+ scheme = TQ_INT8,
+ component = "transformer",
+ source_fp = "fp",
+ base = "Qwen/Qwen-Image-2512",
+ )
+ return pc.cache_key(parts), parts["exclude_name_tokens"]
+
+ on, on_tokens = key()
+ with tq.int8_text_stream_scope(False):
+ resident, resident_tokens = key()
+ monkeypatch.setenv(tq.INT8_TEXT_STREAM_ENV, "0")
+ off, off_tokens = key()
+ assert on != off
+ assert "add_q_proj" in off_tokens and "add_q_proj" not in on_tokens
+ assert resident != on and resident == off and resident_tokens == off_tokens
diff --git a/studio/backend/tests/test_diffusion_transformer_quant.py b/studio/backend/tests/test_diffusion_transformer_quant.py
index c5a40155016..5ad57c00558 100644
--- a/studio/backend/tests/test_diffusion_transformer_quant.py
+++ b/studio/backend/tests/test_diffusion_transformer_quant.py
@@ -904,19 +904,34 @@ def test_exclude_tokens_for_scheme():
assert exclude_tokens_for_scheme(TQ_MXFP8) == ()
-def test_exclude_tokens_for_scheme_family():
- # Qwen-Image never pads its text stream, so short prompts hit M < 16 in torch._int_mm.
+def test_exclude_tokens_for_scheme_family(monkeypatch):
from core.inference.diffusion_transformer_quant import (
+ INT8_TEXT_STREAM_ENV,
_INT8_EXCLUDE_NAME_TOKENS,
_QWENIMAGE_INT8_EXCLUDES,
exclude_tokens_for_scheme,
)
+ monkeypatch.delenv(INT8_TEXT_STREAM_ENV, raising = False)
for fam in ("qwen-image", "qwen-image-edit"):
assert (
- exclude_tokens_for_scheme(TQ_INT8, fam)
+ exclude_tokens_for_scheme(TQ_INT8, fam, baked = True)
== _INT8_EXCLUDE_NAME_TOKENS + _QWENIMAGE_INT8_EXCLUDES
)
+ assert (
+ exclude_tokens_for_scheme(TQ_INT8, "qwen-image-edit")
+ == _INT8_EXCLUDE_NAME_TOKENS + _QWENIMAGE_INT8_EXCLUDES
+ )
+ assert (
+ exclude_tokens_for_scheme(TQ_INT8, "qwen-image")
+ == _INT8_EXCLUDE_NAME_TOKENS + _QWENIMAGE_INT8_EXCLUDES
+ )
+ from core.inference.diffusion_transformer_quant import int8_text_stream_scope
+
+ with int8_text_stream_scope(True):
+ assert exclude_tokens_for_scheme(TQ_INT8, "qwen-image") == _INT8_EXCLUDE_NAME_TOKENS + (
+ "txt_in",
+ )
for token in ("txt_in", "add_q_proj", "to_add_out", "txt_mlp"):
assert token in _QWENIMAGE_INT8_EXCLUDES
assert exclude_tokens_for_scheme(TQ_INT8, "z-image") == _INT8_EXCLUDE_NAME_TOKENS
@@ -1160,12 +1175,30 @@ def test_pad_and_exclude_sets_never_overlap():
), f"{family}: {pad_token!r} is both padded and excluded"
-def test_only_minimax_h3_pads_today():
- """Only MiniMax-H3 pads: other published int8 checkpoints bake the exclusion set into metadata."""
- from core.inference.diffusion_transformer_quant import TQ_INT8, pad_tokens_for_scheme
+def test_only_minimax_h3_pads_today(monkeypatch):
+ """Only MiniMax-H3 pads its own tokens: other published int8 checkpoints bake the exclusion set into metadata.
+ A streamed qwen-image build pads exactly the text stream it quantises on top of its artifacts; resident builds and
+ the kill switch pad nothing new."""
+ from core.inference.diffusion_transformer_quant import (
+ INT8_TEXT_STREAM_ENV,
+ TQ_INT8,
+ pad_tokens_for_scheme,
+ runtime_quant_tokens_for_scheme,
+ )
+
+ from core.inference.diffusion_transformer_quant import int8_text_stream_scope
+ monkeypatch.delenv(INT8_TEXT_STREAM_ENV, raising = False)
for family in ("qwen-image", "qwen-image-edit", "hunyuanvideo-1.5", "hunyuanvideo-1.5-720p"):
assert pad_tokens_for_scheme(TQ_INT8, family) == ()
+ with int8_text_stream_scope(True):
+ assert pad_tokens_for_scheme(TQ_INT8, "qwen-image") == runtime_quant_tokens_for_scheme(
+ TQ_INT8, "qwen-image"
+ )
+ assert pad_tokens_for_scheme(TQ_INT8, "qwen-image")
+ monkeypatch.setenv(INT8_TEXT_STREAM_ENV, "0")
+ assert pad_tokens_for_scheme(TQ_INT8, "qwen-image") == ()
+ monkeypatch.delenv(INT8_TEXT_STREAM_ENV)
assert pad_tokens_for_scheme(TQ_INT8, "z-image") == ()
assert pad_tokens_for_scheme(TQ_INT8, None) == ()