Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

Studio: key the diffusion compile cache by the loaded quant variant (#12887)

Daniel Hand407fb92eb54
diff --git a/studio/backend/core/inference/diffusion_compile_cache.py b/studio/backend/core/inference/diffusion_compile_cache.py
index 3be9c6dfb91..22f06a3258d 100644
--- a/studio/backend/core/inference/diffusion_compile_cache.py
+++ b/studio/backend/core/inference/diffusion_compile_cache.py
@@ -10,9 +10,10 @@ Mega-cache (``save_cache_artifacts`` / ``load_cache_artifacts``, torch >= 2.7).
PORTABILITY IS NOT UNIVERSAL: an artifact is only valid for the SAME torch/Triton/CUDA
build, GPU arch, and model graph (family, dtype, quant, attention backend, compile
-kwargs, shape bucket). torch validates these on load; a mismatch yields no hit, not an
-error. Hence an EXACT-MATCH fingerprint with SILENT FALLBACK to local compile (a miss is
-normal), and per-arch bundles keyed by the full fingerprint. See
+kwargs, shape bucket, quant variant, graph kill switches). torch validates these on load;
+a mismatch yields no hit, not an error. Hence
+an EXACT-MATCH fingerprint with SILENT FALLBACK to local compile (a miss is normal), and
+per-arch bundles keyed by the full fingerprint. See
``outputs/compile_cache/DISTRIBUTION.md``.
GGUF loads participate too (fingerprinted ``quant="gguf"``, a different compiled graph
@@ -60,6 +61,7 @@ import hashlib
import json
import math
import os
+import re
import shutil
import tempfile
import threading
@@ -282,6 +284,174 @@ def environment_fingerprint() -> dict[str, Any]:
return fp
+# Switches that change the traced graph after begin() (swaps in apply_speed_optims / first forward), so the tree can't
+# show them. name -> default ("raw" = value-carrying). Keyed only off default; runtime-only knobs are excluded.
+_GRAPH_SWITCHES = {
+ "UNSLOTH_DIFFUSION_INT8_GEMM": "auto",
+ "UNSLOTH_DIFFUSION_INT8_GEMM_CONVROT": "on",
+ "UNSLOTH_DIFFUSION_INT8_GEMM_STREAMED": "on",
+ "UNSLOTH_DIFFUSION_INT8_ROTQUANT": "auto",
+ "UNSLOTH_DIFFUSION_INT8_FUSED": "on",
+ "UNSLOTH_DIFFUSION_ZIMAGE_FUSED": "on",
+ "UNSLOTH_DIFFUSION_Q21_REAL_ROPE": "on",
+ "UNSLOTH_DIFFUSION_QWEN_REAL_ROPE": "on",
+ "UNSLOTH_DIFFUSION_EAGER_PATCHES": "on",
+ "UNSLOTH_DIFFUSION_ARCH_PATCHES": "on",
+ "UNSLOTH_DIFFUSION_FLUX2_FUSED_ROPE": "on",
+ "UNSLOTH_DIFFUSION_BLOCK_RESTRIDE": "on",
+ "UNSLOTH_DIFFUSION_INDUCTOR_BACKPORTS": "on",
+ "UNSLOTH_STATIC_STREAM_MERGE_DETECT": "off",
+ "UNSLOTH_TORCHAO_INDUCTOR_CONFIG": "off",
+ "UNSLOTH_NVFP4_BACKEND": "raw:auto",
+ # Unset is off; "auto" protects the head + last step, so "auto" is NOT the default here.
+ "UNSLOTH_NVFP4_PROTECT_STEPS": "raw",
+}
+_OFF_WORDS = ("0", "off", "false", "no", "none")
+_ON_WORDS = ("1", "on", "true", "yes", "force")
+_ADDRESS_RE = re.compile(r" at 0x[0-9a-fA-F]+")
+
+
+def _switch_value(name: str, default: str) -> Optional[str]:
+ raw = (os.environ.get(name) or "").strip().lower()
+ if not raw:
+ return None
+ if default.startswith("raw"):
+ return None if raw in _OFF_WORDS or raw == default.partition(":")[2] else raw
+ value = "off" if raw in _OFF_WORDS else "on" if raw in _ON_WORDS else None
+ if value is None:
+ return None
+ return None if value == default else value
+
+
+def graph_switches() -> dict[str, str]:
+ out: dict[str, str] = {}
+ for name, default in sorted(_GRAPH_SWITCHES.items()):
+ value = _switch_value(name, default)
+ if value is not None:
+ out[name] = value
+ return out
+
+
+def _tensor_signature(tensor: Any, depth: int = 0) -> str:
+ """``Cls/dtype(inner...)``: what a traced op dispatches on, never shapes or values."""
+ head = f"{type(tensor).__name__}/{getattr(tensor, 'dtype', None)}"
+ if depth > 2:
+ return head
+ inner: list[str] = []
+ cls = type(tensor)
+ data_names = [
+ *(getattr(cls, "tensor_data_names", None) or ()),
+ *(getattr(cls, "optional_tensor_data_names", None) or ()),
+ ]
+ if data_names:
+ for name in data_names:
+ value = getattr(tensor, name, None)
+ inner.append(
+ f"{name}={_tensor_signature(value, depth + 1) if value is not None else None}"
+ )
+ for attr in ("tensor_attribute_names", "optional_tensor_attribute_names"):
+ for name in getattr(cls, attr, None) or ():
+ if name == "shape":
+ continue
+ if name == "block_size":
+ # Relative to the shape: per-row [1, k] vs per-tensor [n, k] trace different scale broadcasts.
+ shape = tuple(getattr(tensor, "shape", ()) or ())
+ block = tuple(getattr(tensor, name, None) or ())
+ rel = ",".join(
+ "full" if i < len(shape) and b == shape[i] else str(b)
+ for i, b in enumerate(block)
+ )
+ inner.append(f"block_size={rel}")
+ continue
+ inner.append(f"{name}={_ADDRESS_RE.sub('', repr(getattr(tensor, name, None)))}")
+ elif cls.__name__ not in ("Tensor", "Parameter") and hasattr(tensor, "__tensor_flatten__"):
+ # Older wrappers (AffineQuantizedTensor): inner tensors only, the flatten context carries the shape.
+ try:
+ names, _ctx = tensor.__tensor_flatten__()
+ for name in names:
+ inner.append(f"{name}={_tensor_signature(getattr(tensor, name, None), depth + 1)}")
+ except Exception: # noqa: BLE001
+ inner.append("flatten=unavailable")
+ return head + (f"({'; '.join(inner)})" if inner else "")
+
+
+def _weight_signature(module: Any, weight: Any) -> str:
+ """Scale dtype (fp32 hosted vs bf16 runtime) and W8A8 vs weight-only both land here."""
+ parts = [type(module).__name__, _tensor_signature(weight)]
+ group = getattr(module, "convrot_groupsize", None)
+ if group is not None:
+ parts.append(f"convrot{group}")
+ rot = getattr(module, "rot_group", None)
+ if rot:
+ parts.append(f"rot{rot}")
+ if getattr(module, "act_int8", False):
+ parts.append("act_int8")
+ fwd = module.__dict__.get("forward") if hasattr(module, "__dict__") else None
+ if fwd is not None:
+ func = getattr(fwd, "__func__", fwd)
+ parts.append(
+ f"fwd={getattr(func, '__module__', '')}.{getattr(func, '__qualname__', type(func).__name__)}"
+ )
+ return "/".join(parts)
+
+
+def _stored_weight(module: Any) -> tuple[Any, bool]:
+ """The stored weight, never a ``weight`` property: native layers dequantize the whole matrix on each read."""
+ d = getattr(module, "__dict__", None) or {}
+ for store in ("_parameters", "_buffers"):
+ weight = (d.get(store) or {}).get("weight")
+ if weight is not None:
+ return weight, False
+ if d.get("weight") is not None:
+ return d["weight"], False
+ packed = (d.get("_buffers") or {}).get("weight_q")
+ return packed, packed is not None
+
+
+def graph_variant(transformer: Any) -> dict[str, Any]:
+ """Structural descriptor of the loaded quant artifact (no weight hashing): stable across processes, distinct for
+ plain vs ConvRot, hosted vs ComfyUI vs runtime quantise, torchao vs native. Never raises."""
+ variant: dict[str, Any] = {}
+ try:
+ import torch # noqa: PLC0415
+
+ counts: dict[str, int] = {}
+ for module in transformer.modules():
+ weight, packed = _stored_weight(module)
+ if not isinstance(weight, torch.Tensor):
+ continue
+ sig = _weight_signature(module, weight) + ("/packed" if packed else "")
+ counts[sig] = counts.get(sig, 0) + 1
+ variant["weights"] = dict(sorted(counts.items()))
+ except Exception as exc: # noqa: BLE001 - an unreadable tree keys apart rather than sharing a bundle
+ variant["weights"] = f"unavailable: {type(exc).__name__}"
+ try:
+ from .diffusion_convrot import CONVROT_ATTR # noqa: PLC0415
+ rotation = getattr(transformer, CONVROT_ATTR, None)
+ if isinstance(rotation, dict):
+ variant["rotation"] = {k: rotation[k] for k in sorted(rotation)}
+ except Exception: # noqa: BLE001
+ pass
+ return variant
+
+
+def _has_quantized_layout(variant: dict[str, Any]) -> bool:
+ """A ComfyUI single file keys quant as none, so the tree decides."""
+ if variant.get("rotation"):
+ return True
+ weights = variant.get("weights")
+ if not isinstance(weights, dict):
+ return False
+ for sig in weights:
+ parts = sig.split("/")
+ if len(parts) < 2 or parts[1] not in ("Parameter", "Tensor") or "(" in sig:
+ return True
+ # An instance forward alone is not quantized: offload hooks install one on dense loads too.
+ if any(p.startswith(("convrot", "rot", "act_int8", "packed")) for p in parts[3:]):
+ return True
+ return False
+
+
def model_fingerprint(
*,
family: Any,
@@ -321,6 +491,14 @@ def model_fingerprint(
},
"shape_bucket": shape_bucket,
}
+ # INT8 and INT8-ConvRot share "int8" but compile different graphs. Quantized layouts only: dense / GGUF keys hold.
+ if str(quant) != "gguf" and transformer is not None:
+ variant = graph_variant(transformer)
+ if quant is not None or _has_quantized_layout(variant):
+ fp["variant"] = variant
+ switches = graph_switches()
+ if switches:
+ fp["switches"] = switches
# Added only when armed, so other bundles keep their key.
try:
from .diffusion_dynamic_text import fingerprint as _dynamic_text_fp
diff --git a/studio/backend/tests/test_diffusion_compile_cache_variant.py b/studio/backend/tests/test_diffusion_compile_cache_variant.py
new file mode 100644
index 00000000000..2b1f8070bba
--- /dev/null
+++ b/studio/backend/tests/test_diffusion_compile_cache_variant.py
@@ -0,0 +1,253 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
+
+"""INT8 and INT8-ConvRot share ``transformer_quant="int8"``; a bundle from one reported a hit for the other, then
+recompiled in full. The key now carries ``graph_variant`` for quantized loads; dense and GGUF keys are unchanged."""
+
+from __future__ import annotations
+
+import json
+import types
+
+import pytest
+
+torch = pytest.importorskip("torch")
+from torch import nn # noqa: E402
+
+from core.inference import diffusion_compile_cache as cc # noqa: E402
+from core.inference.diffusion_convrot import ( # noqa: E402
+ apply_activation_rotation,
+ rotation_metadata,
+)
+
+_FP_KW = dict(
+ family = "z-image",
+ dtype = "torch.bfloat16",
+ attention_backend = "_native_cudnn",
+ compile_kwargs = {"fullgraph": True, "dynamic": True, "mode": "default"},
+)
+
+
+def _model():
+ torch.manual_seed(0)
+ m = nn.Sequential(nn.Linear(256, 256, bias = False), nn.LayerNorm(256), nn.Linear(256, 64))
+ m._repeated_blocks = ["Linear"]
+ return m.to(torch.bfloat16)
+
+
+def _int8(act = True):
+ tq = pytest.importorskip("torchao.quantization")
+ m = _model()
+ cfg = (
+ tq.Int8DynamicActivationInt8WeightConfig(version = 2)
+ if act
+ else tq.Int8WeightOnlyConfig(version = 2)
+ )
+ tq.quantize_(m, cfg)
+ return m
+
+
+def _convrot(m):
+ apply_activation_rotation(m, rotation_metadata(256, ["0"]))
+ return m
+
+
+def _key(transformer, quant = "int8"):
+ return cc.cache_key(
+ cc.environment_fingerprint(),
+ cc.model_fingerprint(transformer = transformer, quant = quant, **_FP_KW),
+ )
+
+
[email protected](autouse = True)
+def _clean_switches(monkeypatch):
+ for name in cc._GRAPH_SWITCHES:
+ monkeypatch.delenv(name, raising = False)
+
+
+def test_identical_config_keys_the_same_across_builds():
+ a, b = _int8(), _int8()
+ assert _key(a) == _key(b)
+ # Survives a JSON round trip (the manifest's exact-match guard compares the parsed dict).
+ fp = cc.model_fingerprint(transformer = a, quant = "int8", **_FP_KW)
+ assert json.loads(json.dumps(fp, sort_keys = True, default = str)) == fp
+
+
+def test_plain_and_convrot_int8_key_apart():
+ plain = _int8()
+ rotated = _convrot(_int8())
+ assert _key(plain) != _key(rotated)
+ variant = cc.graph_variant(rotated)
+ assert variant["rotation"] == {"group": 256, "kind": "convrot_hadamard_v1", "linears": 1}
+ assert any(sig.startswith("ConvRotLinear/") for sig in variant["weights"])
+ assert "rotation" not in cc.graph_variant(plain)
+
+
+def test_w8a8_and_weight_only_key_apart():
+ assert _key(_int8(act = True)) != _key(_int8(act = False))
+
+
+def test_runtime_quantise_and_prequant_scale_dtype_key_apart():
+ # A v2 hosted checkpoint stores fp32 scales; a runtime quantise or a v1 file carries bf16 ones. Different
+ # epilogue graph, so different key.
+ a, b = _int8(), _int8()
+ w = b[0].weight
+ if type(w).__name__ != "Int8Tensor":
+ pytest.skip("this torchao predates Int8Tensor")
+ other = torch.float32 if w.scale.dtype == torch.bfloat16 else torch.bfloat16
+ w.scale = w.scale.to(other)
+ if b[0].weight.scale.dtype != other:
+ pytest.skip("this torchao does not take a scale reassignment")
+ assert _key(a) != _key(b)
+
+
+def test_a_native_twin_keys_apart_from_torchao():
+ # A ComfyUI file served by native layers instead of torchao: different module class, different key.
+ a = _int8()
+ b = _int8()
+ native_cls = type("NativeInt8Linear", (nn.Linear,), {})
+ b[0].__class__ = native_cls
+ assert _key(a) != _key(b)
+
+
+def test_an_instance_forward_swap_keys_apart():
+ a, b = _int8(), _int8()
+
+ def fused_forward(self, x): # stands in for a fused-kernel swap installed at load
+ return x
+
+ b[0].forward = types.MethodType(fused_forward, b[0])
+ assert _key(a) != _key(b)
+
+
[email protected]("name,default", sorted(cc._GRAPH_SWITCHES.items()))
[email protected]("quant", ["int8", None])
+def test_graph_switches_key_apart_only_off_their_default(monkeypatch, name, default, quant):
+ m = _int8() if quant else _model()
+ k0 = _key(m, quant)
+ for same in (
+ ("",)
+ + (() if default == "raw" else ("auto",))
+ + ((default,) if default in ("on", "off") else ())
+ ):
+ monkeypatch.setenv(name, same)
+ assert _key(m, quant) == k0, same
+ flipped = {"on": "0", "off": "1", "auto": "0", "raw": "torchao", "raw:auto": "torchao"}[default]
+ monkeypatch.setenv(name, flipped)
+ assert _key(m, quant) != k0
+ assert name in cc.model_fingerprint(transformer = m, quant = quant, **_FP_KW)["switches"]
+
+
+def test_a_default_environment_adds_no_switches():
+ assert cc.graph_switches() == {}
+ assert "switches" not in cc.model_fingerprint(transformer = _model(), quant = None, **_FP_KW)
+
+
+def test_a_comfy_single_file_with_no_engaged_scheme_still_keys_its_layout():
+ # ComfyUI single-file loads engage no scheme (quant reads none), so the tree itself has to key them apart.
+ dense = _model()
+ comfy = _int8()
+ fp_dense = cc.model_fingerprint(transformer = dense, quant = None, **_FP_KW)
+ fp_comfy = cc.model_fingerprint(transformer = comfy, quant = None, **_FP_KW)
+ assert "variant" not in fp_dense
+ assert "variant" in fp_comfy
+ assert _key(dense, None) != _key(comfy, None)
+ assert _key(comfy, None) != _key(_convrot(_int8()), None)
+
+
+def test_an_offload_hook_forward_alone_keeps_a_dense_key():
+ m = _model()
+ k0 = _key(m, None)
+ m[0].forward = types.MethodType(lambda self, x: x, m[0])
+ assert "variant" not in cc.model_fingerprint(transformer = m, quant = None, **_FP_KW)
+ assert _key(m, None) == k0
+
+
+def test_dense_and_gguf_keys_are_unchanged():
+ # No variant on a dense or GGUF load, so their existing bundles keep hitting.
+ m = _model()
+ for quant in (None, "gguf"):
+ fp = cc.model_fingerprint(transformer = m, quant = quant, **_FP_KW)
+ assert "variant" not in fp
+
+
+def test_unreadable_tree_never_raises():
+ variant = cc.graph_variant(types.SimpleNamespace())
+ assert variant["weights"].startswith("unavailable")
+
+
+def test_a_bundle_saved_for_one_variant_is_a_miss_for_the_other(monkeypatch, tmp_path):
+ store: dict = {}
+ monkeypatch.setattr(
+ torch.compiler, "save_cache_artifacts", lambda: (b"artifacts", object()), raising = False
+ )
+
+ def fake_load(data):
+ store["loaded"] = data
+ return object()
+
+ monkeypatch.setattr(torch.compiler, "load_cache_artifacts", fake_load, raising = False)
+ monkeypatch.setenv(cc._ENV_DIR, str(tmp_path))
+ monkeypatch.setenv(cc._ENV_MODE, "1")
+ rotated = cc.begin(transformer = _convrot(_int8()), quant = "int8", **_FP_KW)
+ assert rotated is not None and not rotated.hit
+ assert cc.save(rotated) is True
+ cc.restore(rotated)
+ plain = cc.begin(transformer = _int8(), quant = "int8", **_FP_KW)
+ assert plain is not None and not plain.hit and plain.key != rotated.key
+ assert "loaded" not in store
+ cc.restore(plain)
+ again = cc.begin(transformer = _convrot(_int8()), quant = "int8", **_FP_KW)
+ assert again is not None and again.hit and again.key == rotated.key
+ cc.restore(again)
+
+
+def test_protect_steps_auto_is_not_its_default(monkeypatch):
+ m = _int8()
+ k0 = _key(m)
+ monkeypatch.setenv("UNSLOTH_NVFP4_PROTECT_STEPS", "auto")
+ assert _key(m) != k0
+ monkeypatch.setenv("UNSLOTH_NVFP4_BACKEND", "auto")
+ assert cc.graph_switches() == {"UNSLOTH_NVFP4_PROTECT_STEPS": "auto"}
+
+
+def test_a_dequantizing_weight_property_is_never_read():
+ reads = []
+
+ class PackedLinear(nn.Module):
+ def __init__(self, scheme):
+ super().__init__()
+ self.register_buffer("weight_q", torch.zeros(4, 4, dtype = scheme))
+
+ @property
+ def weight(self):
+ reads.append(1)
+ return self.weight_q.float()
+
+ m = _model()
+ m[0] = PackedLinear(torch.int8)
+ other = _model()
+ other[0] = PackedLinear(torch.uint8)
+ fp = cc.model_fingerprint(transformer = m, quant = None, **_FP_KW)
+ assert not reads
+ assert "variant" in fp and any(sig.endswith("/packed") for sig in fp["variant"]["weights"])
+ assert _key(m, None) != _key(other, None)
+ assert not reads
+
+
+def test_per_row_and_per_tensor_scales_key_apart():
+ class FakeFp8:
+ tensor_data_names = ("qdata", "scale")
+ tensor_attribute_names = ("block_size",)
+ dtype = torch.bfloat16
+
+ def __init__(self, block, scale_shape):
+ self.shape = (64, 256)
+ self.block_size = block
+ self.qdata = torch.zeros(64, 256, dtype = torch.float8_e4m3fn)
+ self.scale = torch.ones(scale_shape)
+
+ per_row = cc._tensor_signature(FakeFp8([1, 256], (64, 1)))
+ per_tensor = cc._tensor_signature(FakeFp8([64, 256], (1,)))
+ assert per_row != per_tensor
+ assert "block_size=1,full" in per_row and "block_size=full,full" in per_tensor