Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
Studio: key the diffusion compile cache by the loaded quant variant (#12887)
Daniel Hand407fb92eb54
diff --git a/studio/backend/core/inference/diffusion_compile_cache.py b/studio/backend/core/inference/diffusion_compile_cache.pyindex 3be9c6dfb91..22f06a3258d 100644--- a/studio/backend/core/inference/diffusion_compile_cache.py+++ b/studio/backend/core/inference/diffusion_compile_cache.py@@ -10,9 +10,10 @@ Mega-cache (``save_cache_artifacts`` / ``load_cache_artifacts``, torch >= 2.7).PORTABILITY IS NOT UNIVERSAL: an artifact is only valid for the SAME torch/Triton/CUDAbuild, GPU arch, and model graph (family, dtype, quant, attention backend, compile-kwargs, shape bucket). torch validates these on load; a mismatch yields no hit, not an-error. Hence an EXACT-MATCH fingerprint with SILENT FALLBACK to local compile (a miss is-normal), and per-arch bundles keyed by the full fingerprint. See+kwargs, shape bucket, quant variant, graph kill switches). torch validates these on load;+a mismatch yields no hit, not an error. Hence+an EXACT-MATCH fingerprint with SILENT FALLBACK to local compile (a miss is normal), and+per-arch bundles keyed by the full fingerprint. See``outputs/compile_cache/DISTRIBUTION.md``.GGUF loads participate too (fingerprinted ``quant="gguf"``, a different compiled graph@@ -60,6 +61,7 @@ import hashlibimport jsonimport mathimport os+import reimport shutilimport tempfileimport threading@@ -282,6 +284,174 @@ def environment_fingerprint() -> dict[str, Any]:return fp+# Switches that change the traced graph after begin() (swaps in apply_speed_optims / first forward), so the tree can't+# show them. name -> default ("raw" = value-carrying). Keyed only off default; runtime-only knobs are excluded.+_GRAPH_SWITCHES = {+ "UNSLOTH_DIFFUSION_INT8_GEMM": "auto",+ "UNSLOTH_DIFFUSION_INT8_GEMM_CONVROT": "on",+ "UNSLOTH_DIFFUSION_INT8_GEMM_STREAMED": "on",+ "UNSLOTH_DIFFUSION_INT8_ROTQUANT": "auto",+ "UNSLOTH_DIFFUSION_INT8_FUSED": "on",+ "UNSLOTH_DIFFUSION_ZIMAGE_FUSED": "on",+ "UNSLOTH_DIFFUSION_Q21_REAL_ROPE": "on",+ "UNSLOTH_DIFFUSION_QWEN_REAL_ROPE": "on",+ "UNSLOTH_DIFFUSION_EAGER_PATCHES": "on",+ "UNSLOTH_DIFFUSION_ARCH_PATCHES": "on",+ "UNSLOTH_DIFFUSION_FLUX2_FUSED_ROPE": "on",+ "UNSLOTH_DIFFUSION_BLOCK_RESTRIDE": "on",+ "UNSLOTH_DIFFUSION_INDUCTOR_BACKPORTS": "on",+ "UNSLOTH_STATIC_STREAM_MERGE_DETECT": "off",+ "UNSLOTH_TORCHAO_INDUCTOR_CONFIG": "off",+ "UNSLOTH_NVFP4_BACKEND": "raw:auto",+ # Unset is off; "auto" protects the head + last step, so "auto" is NOT the default here.+ "UNSLOTH_NVFP4_PROTECT_STEPS": "raw",+}+_OFF_WORDS = ("0", "off", "false", "no", "none")+_ON_WORDS = ("1", "on", "true", "yes", "force")+_ADDRESS_RE = re.compile(r" at 0x[0-9a-fA-F]+")+++def _switch_value(name: str, default: str) -> Optional[str]:+ raw = (os.environ.get(name) or "").strip().lower()+ if not raw:+ return None+ if default.startswith("raw"):+ return None if raw in _OFF_WORDS or raw == default.partition(":")[2] else raw+ value = "off" if raw in _OFF_WORDS else "on" if raw in _ON_WORDS else None+ if value is None:+ return None+ return None if value == default else value+++def graph_switches() -> dict[str, str]:+ out: dict[str, str] = {}+ for name, default in sorted(_GRAPH_SWITCHES.items()):+ value = _switch_value(name, default)+ if value is not None:+ out[name] = value+ return out+++def _tensor_signature(tensor: Any, depth: int = 0) -> str:+ """``Cls/dtype(inner...)``: what a traced op dispatches on, never shapes or values."""+ head = f"{type(tensor).__name__}/{getattr(tensor, 'dtype', None)}"+ if depth > 2:+ return head+ inner: list[str] = []+ cls = type(tensor)+ data_names = [+ *(getattr(cls, "tensor_data_names", None) or ()),+ *(getattr(cls, "optional_tensor_data_names", None) or ()),+ ]+ if data_names:+ for name in data_names:+ value = getattr(tensor, name, None)+ inner.append(+ f"{name}={_tensor_signature(value, depth + 1) if value is not None else None}"+ )+ for attr in ("tensor_attribute_names", "optional_tensor_attribute_names"):+ for name in getattr(cls, attr, None) or ():+ if name == "shape":+ continue+ if name == "block_size":+ # Relative to the shape: per-row [1, k] vs per-tensor [n, k] trace different scale broadcasts.+ shape = tuple(getattr(tensor, "shape", ()) or ())+ block = tuple(getattr(tensor, name, None) or ())+ rel = ",".join(+ "full" if i < len(shape) and b == shape[i] else str(b)+ for i, b in enumerate(block)+ )+ inner.append(f"block_size={rel}")+ continue+ inner.append(f"{name}={_ADDRESS_RE.sub('', repr(getattr(tensor, name, None)))}")+ elif cls.__name__ not in ("Tensor", "Parameter") and hasattr(tensor, "__tensor_flatten__"):+ # Older wrappers (AffineQuantizedTensor): inner tensors only, the flatten context carries the shape.+ try:+ names, _ctx = tensor.__tensor_flatten__()+ for name in names:+ inner.append(f"{name}={_tensor_signature(getattr(tensor, name, None), depth + 1)}")+ except Exception: # noqa: BLE001+ inner.append("flatten=unavailable")+ return head + (f"({'; '.join(inner)})" if inner else "")+++def _weight_signature(module: Any, weight: Any) -> str:+ """Scale dtype (fp32 hosted vs bf16 runtime) and W8A8 vs weight-only both land here."""+ parts = [type(module).__name__, _tensor_signature(weight)]+ group = getattr(module, "convrot_groupsize", None)+ if group is not None:+ parts.append(f"convrot{group}")+ rot = getattr(module, "rot_group", None)+ if rot:+ parts.append(f"rot{rot}")+ if getattr(module, "act_int8", False):+ parts.append("act_int8")+ fwd = module.__dict__.get("forward") if hasattr(module, "__dict__") else None+ if fwd is not None:+ func = getattr(fwd, "__func__", fwd)+ parts.append(+ f"fwd={getattr(func, '__module__', '')}.{getattr(func, '__qualname__', type(func).__name__)}"+ )+ return "/".join(parts)+++def _stored_weight(module: Any) -> tuple[Any, bool]:+ """The stored weight, never a ``weight`` property: native layers dequantize the whole matrix on each read."""+ d = getattr(module, "__dict__", None) or {}+ for store in ("_parameters", "_buffers"):+ weight = (d.get(store) or {}).get("weight")+ if weight is not None:+ return weight, False+ if d.get("weight") is not None:+ return d["weight"], False+ packed = (d.get("_buffers") or {}).get("weight_q")+ return packed, packed is not None+++def graph_variant(transformer: Any) -> dict[str, Any]:+ """Structural descriptor of the loaded quant artifact (no weight hashing): stable across processes, distinct for+ plain vs ConvRot, hosted vs ComfyUI vs runtime quantise, torchao vs native. Never raises."""+ variant: dict[str, Any] = {}+ try:+ import torch # noqa: PLC0415++ counts: dict[str, int] = {}+ for module in transformer.modules():+ weight, packed = _stored_weight(module)+ if not isinstance(weight, torch.Tensor):+ continue+ sig = _weight_signature(module, weight) + ("/packed" if packed else "")+ counts[sig] = counts.get(sig, 0) + 1+ variant["weights"] = dict(sorted(counts.items()))+ except Exception as exc: # noqa: BLE001 - an unreadable tree keys apart rather than sharing a bundle+ variant["weights"] = f"unavailable: {type(exc).__name__}"+ try:+ from .diffusion_convrot import CONVROT_ATTR # noqa: PLC0415+ rotation = getattr(transformer, CONVROT_ATTR, None)+ if isinstance(rotation, dict):+ variant["rotation"] = {k: rotation[k] for k in sorted(rotation)}+ except Exception: # noqa: BLE001+ pass+ return variant+++def _has_quantized_layout(variant: dict[str, Any]) -> bool:+ """A ComfyUI single file keys quant as none, so the tree decides."""+ if variant.get("rotation"):+ return True+ weights = variant.get("weights")+ if not isinstance(weights, dict):+ return False+ for sig in weights:+ parts = sig.split("/")+ if len(parts) < 2 or parts[1] not in ("Parameter", "Tensor") or "(" in sig:+ return True+ # An instance forward alone is not quantized: offload hooks install one on dense loads too.+ if any(p.startswith(("convrot", "rot", "act_int8", "packed")) for p in parts[3:]):+ return True+ return False++def model_fingerprint(*,family: Any,@@ -321,6 +491,14 @@ def model_fingerprint(},"shape_bucket": shape_bucket,}+ # INT8 and INT8-ConvRot share "int8" but compile different graphs. Quantized layouts only: dense / GGUF keys hold.+ if str(quant) != "gguf" and transformer is not None:+ variant = graph_variant(transformer)+ if quant is not None or _has_quantized_layout(variant):+ fp["variant"] = variant+ switches = graph_switches()+ if switches:+ fp["switches"] = switches# Added only when armed, so other bundles keep their key.try:from .diffusion_dynamic_text import fingerprint as _dynamic_text_fpdiff --git a/studio/backend/tests/test_diffusion_compile_cache_variant.py b/studio/backend/tests/test_diffusion_compile_cache_variant.pynew file mode 100644index 00000000000..2b1f8070bba--- /dev/null+++ b/studio/backend/tests/test_diffusion_compile_cache_variant.py@@ -0,0 +1,253 @@+# SPDX-License-Identifier: AGPL-3.0-only+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0++"""INT8 and INT8-ConvRot share ``transformer_quant="int8"``; a bundle from one reported a hit for the other, then+recompiled in full. The key now carries ``graph_variant`` for quantized loads; dense and GGUF keys are unchanged."""++from __future__ import annotations++import json+import types++import pytest++torch = pytest.importorskip("torch")+from torch import nn # noqa: E402++from core.inference import diffusion_compile_cache as cc # noqa: E402+from core.inference.diffusion_convrot import ( # noqa: E402+ apply_activation_rotation,+ rotation_metadata,+)++_FP_KW = dict(+ family = "z-image",+ dtype = "torch.bfloat16",+ attention_backend = "_native_cudnn",+ compile_kwargs = {"fullgraph": True, "dynamic": True, "mode": "default"},+)+++def _model():+ torch.manual_seed(0)+ m = nn.Sequential(nn.Linear(256, 256, bias = False), nn.LayerNorm(256), nn.Linear(256, 64))+ m._repeated_blocks = ["Linear"]+ return m.to(torch.bfloat16)+++def _int8(act = True):+ tq = pytest.importorskip("torchao.quantization")+ m = _model()+ cfg = (+ tq.Int8DynamicActivationInt8WeightConfig(version = 2)+ if act+ else tq.Int8WeightOnlyConfig(version = 2)+ )+ tq.quantize_(m, cfg)+ return m+++def _convrot(m):+ apply_activation_rotation(m, rotation_metadata(256, ["0"]))+ return m+++def _key(transformer, quant = "int8"):+ return cc.cache_key(+ cc.environment_fingerprint(),+ cc.model_fingerprint(transformer = transformer, quant = quant, **_FP_KW),+ )++[email protected](autouse = True)+def _clean_switches(monkeypatch):+ for name in cc._GRAPH_SWITCHES:+ monkeypatch.delenv(name, raising = False)+++def test_identical_config_keys_the_same_across_builds():+ a, b = _int8(), _int8()+ assert _key(a) == _key(b)+ # Survives a JSON round trip (the manifest's exact-match guard compares the parsed dict).+ fp = cc.model_fingerprint(transformer = a, quant = "int8", **_FP_KW)+ assert json.loads(json.dumps(fp, sort_keys = True, default = str)) == fp+++def test_plain_and_convrot_int8_key_apart():+ plain = _int8()+ rotated = _convrot(_int8())+ assert _key(plain) != _key(rotated)+ variant = cc.graph_variant(rotated)+ assert variant["rotation"] == {"group": 256, "kind": "convrot_hadamard_v1", "linears": 1}+ assert any(sig.startswith("ConvRotLinear/") for sig in variant["weights"])+ assert "rotation" not in cc.graph_variant(plain)+++def test_w8a8_and_weight_only_key_apart():+ assert _key(_int8(act = True)) != _key(_int8(act = False))+++def test_runtime_quantise_and_prequant_scale_dtype_key_apart():+ # A v2 hosted checkpoint stores fp32 scales; a runtime quantise or a v1 file carries bf16 ones. Different+ # epilogue graph, so different key.+ a, b = _int8(), _int8()+ w = b[0].weight+ if type(w).__name__ != "Int8Tensor":+ pytest.skip("this torchao predates Int8Tensor")+ other = torch.float32 if w.scale.dtype == torch.bfloat16 else torch.bfloat16+ w.scale = w.scale.to(other)+ if b[0].weight.scale.dtype != other:+ pytest.skip("this torchao does not take a scale reassignment")+ assert _key(a) != _key(b)+++def test_a_native_twin_keys_apart_from_torchao():+ # A ComfyUI file served by native layers instead of torchao: different module class, different key.+ a = _int8()+ b = _int8()+ native_cls = type("NativeInt8Linear", (nn.Linear,), {})+ b[0].__class__ = native_cls+ assert _key(a) != _key(b)+++def test_an_instance_forward_swap_keys_apart():+ a, b = _int8(), _int8()++ def fused_forward(self, x): # stands in for a fused-kernel swap installed at load+ return x++ b[0].forward = types.MethodType(fused_forward, b[0])+ assert _key(a) != _key(b)++[email protected]("name,default", sorted(cc._GRAPH_SWITCHES.items()))[email protected]("quant", ["int8", None])+def test_graph_switches_key_apart_only_off_their_default(monkeypatch, name, default, quant):+ m = _int8() if quant else _model()+ k0 = _key(m, quant)+ for same in (+ ("",)+ + (() if default == "raw" else ("auto",))+ + ((default,) if default in ("on", "off") else ())+ ):+ monkeypatch.setenv(name, same)+ assert _key(m, quant) == k0, same+ flipped = {"on": "0", "off": "1", "auto": "0", "raw": "torchao", "raw:auto": "torchao"}[default]+ monkeypatch.setenv(name, flipped)+ assert _key(m, quant) != k0+ assert name in cc.model_fingerprint(transformer = m, quant = quant, **_FP_KW)["switches"]+++def test_a_default_environment_adds_no_switches():+ assert cc.graph_switches() == {}+ assert "switches" not in cc.model_fingerprint(transformer = _model(), quant = None, **_FP_KW)+++def test_a_comfy_single_file_with_no_engaged_scheme_still_keys_its_layout():+ # ComfyUI single-file loads engage no scheme (quant reads none), so the tree itself has to key them apart.+ dense = _model()+ comfy = _int8()+ fp_dense = cc.model_fingerprint(transformer = dense, quant = None, **_FP_KW)+ fp_comfy = cc.model_fingerprint(transformer = comfy, quant = None, **_FP_KW)+ assert "variant" not in fp_dense+ assert "variant" in fp_comfy+ assert _key(dense, None) != _key(comfy, None)+ assert _key(comfy, None) != _key(_convrot(_int8()), None)+++def test_an_offload_hook_forward_alone_keeps_a_dense_key():+ m = _model()+ k0 = _key(m, None)+ m[0].forward = types.MethodType(lambda self, x: x, m[0])+ assert "variant" not in cc.model_fingerprint(transformer = m, quant = None, **_FP_KW)+ assert _key(m, None) == k0+++def test_dense_and_gguf_keys_are_unchanged():+ # No variant on a dense or GGUF load, so their existing bundles keep hitting.+ m = _model()+ for quant in (None, "gguf"):+ fp = cc.model_fingerprint(transformer = m, quant = quant, **_FP_KW)+ assert "variant" not in fp+++def test_unreadable_tree_never_raises():+ variant = cc.graph_variant(types.SimpleNamespace())+ assert variant["weights"].startswith("unavailable")+++def test_a_bundle_saved_for_one_variant_is_a_miss_for_the_other(monkeypatch, tmp_path):+ store: dict = {}+ monkeypatch.setattr(+ torch.compiler, "save_cache_artifacts", lambda: (b"artifacts", object()), raising = False+ )++ def fake_load(data):+ store["loaded"] = data+ return object()++ monkeypatch.setattr(torch.compiler, "load_cache_artifacts", fake_load, raising = False)+ monkeypatch.setenv(cc._ENV_DIR, str(tmp_path))+ monkeypatch.setenv(cc._ENV_MODE, "1")+ rotated = cc.begin(transformer = _convrot(_int8()), quant = "int8", **_FP_KW)+ assert rotated is not None and not rotated.hit+ assert cc.save(rotated) is True+ cc.restore(rotated)+ plain = cc.begin(transformer = _int8(), quant = "int8", **_FP_KW)+ assert plain is not None and not plain.hit and plain.key != rotated.key+ assert "loaded" not in store+ cc.restore(plain)+ again = cc.begin(transformer = _convrot(_int8()), quant = "int8", **_FP_KW)+ assert again is not None and again.hit and again.key == rotated.key+ cc.restore(again)+++def test_protect_steps_auto_is_not_its_default(monkeypatch):+ m = _int8()+ k0 = _key(m)+ monkeypatch.setenv("UNSLOTH_NVFP4_PROTECT_STEPS", "auto")+ assert _key(m) != k0+ monkeypatch.setenv("UNSLOTH_NVFP4_BACKEND", "auto")+ assert cc.graph_switches() == {"UNSLOTH_NVFP4_PROTECT_STEPS": "auto"}+++def test_a_dequantizing_weight_property_is_never_read():+ reads = []++ class PackedLinear(nn.Module):+ def __init__(self, scheme):+ super().__init__()+ self.register_buffer("weight_q", torch.zeros(4, 4, dtype = scheme))++ @property+ def weight(self):+ reads.append(1)+ return self.weight_q.float()++ m = _model()+ m[0] = PackedLinear(torch.int8)+ other = _model()+ other[0] = PackedLinear(torch.uint8)+ fp = cc.model_fingerprint(transformer = m, quant = None, **_FP_KW)+ assert not reads+ assert "variant" in fp and any(sig.endswith("/packed") for sig in fp["variant"]["weights"])+ assert _key(m, None) != _key(other, None)+ assert not reads+++def test_per_row_and_per_tensor_scales_key_apart():+ class FakeFp8:+ tensor_data_names = ("qdata", "scale")+ tensor_attribute_names = ("block_size",)+ dtype = torch.bfloat16++ def __init__(self, block, scale_shape):+ self.shape = (64, 256)+ self.block_size = block+ self.qdata = torch.zeros(64, 256, dtype = torch.float8_e4m3fn)+ self.scale = torch.ones(scale_shape)++ per_row = cc._tensor_signature(FakeFp8([1, 256], (64, 1)))+ per_tensor = cc._tensor_signature(FakeFp8([64, 256], (1,)))+ assert per_row != per_tensor+ assert "block_size=1,full" in per_row and "block_size=full,full" in per_tensor