Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
Studio: regionally compile Lumina-2 and HiDream-I1, and re-decide their auto precision by measurement (#12039)
Daniel Haneec898cec794
diff --git a/studio/backend/core/inference/diffusion.py b/studio/backend/core/inference/diffusion.pyindex 703e12ab6aa..7d7c90b2673 100644--- a/studio/backend/core/inference/diffusion.py+++ b/studio/backend/core/inference/diffusion.py@@ -245,6 +245,7 @@ from .diffusion_nvfp4_flag import nvfp4_diffusion_enabledfrom .diffusion_nvfp4_install import nvfp4_backend_fields as _nvfp4_backend_fieldsfrom .diffusion_transformer_quant import (TQ_AUTO,+ auto_bf16_when_resident_reason,TQ_NVFP4,TQ_INT8,DEFAULT_MIN_LINEAR_FEATURES,@@ -1701,21 +1702,39 @@ def _plan_proves_resident(plan: Any) -> bool:)+def _auto_keeps_bf16_reason(fam: Any, kind: Optional[str] = "pipeline") -> Optional[str]:+ """Why AUTO keeps bf16 when it fits (measured rule: pipeline loads only), or None; both deciders read this."""+ if not family_compiles_regionally(fam):+ return (+ f"'{getattr(fam, 'name', None)}' cannot be regionally compiled (its transformer declares no "+ "repeated blocks), and an uncompiled quantised transformer runs far slower than the weights "+ "it replaces; they fit on this GPU as they are"+ )+ if kind != "pipeline":+ return None+ measured = auto_bf16_when_resident_reason(getattr(fam, "name", None))+ if measured is None:+ return None+ return (+ f"'{getattr(fam, 'name', None)}': {measured}; the bf16 weights fit on this GPU as they are"+ )++def _auto_quant_eager_reason(fam: Any,plan: Any,prequant_path: Optional[str] = None,+ kind: Optional[str] = "pipeline",) -> Optional[str]:- """Why AUTO keeps bf16 for an uncompilable family, or None; an operator's own checkpoint wins."""- if not _plan_proves_resident(plan) or family_compiles_regionally(fam):+ """Why AUTO keeps bf16 for this load, or None; an operator's own checkpoint wins."""+ if not _plan_proves_resident(plan):+ return None+ reason = _auto_keeps_bf16_reason(fam, kind)+ if reason is None:return Noneif prequant_path and local_prequant_path_ready(prequant_path):return None- return (- f"'{getattr(fam, 'name', None)}' cannot be regionally compiled (its transformer declares no "- "repeated blocks), and an uncompiled quantised transformer runs far slower than the weights "- "it replaces; they fit on this GPU as they are"- )+ return reasondef _clear_exception_frames(exc: BaseException) -> None:@@ -3334,7 +3353,7 @@ class DiffusionBackend:is not None):return None- if auto and not family_compiles_regionally(fam):+ if auto and _auto_keeps_bf16_reason(fam) is not None:bf16_memory = snapshot_device_memory(target)bf16_plan = self._bf16_table_plan(target,@@ -5208,7 +5227,7 @@ class DiffusionBackend:and dense_quant_supported_kind(kind)and dense_transformer_supported(target)and not (kind == "gguf" and _has_active_lora(loras))- and not family_compiles_regionally(fam)+ and _auto_keeps_bf16_reason(fam, kind) is not Noneand (eager_reason := _auto_quant_eager_reason(fam,@@ -5228,6 +5247,7 @@ class DiffusionBackend:else None,),transformer_prequant_path,+ kind,))is not Nonediff --git a/studio/backend/core/inference/diffusion_compile_cache.py b/studio/backend/core/inference/diffusion_compile_cache.pyindex b50e34871ac..a1881941c21 100644--- a/studio/backend/core/inference/diffusion_compile_cache.py+++ b/studio/backend/core/inference/diffusion_compile_cache.py@@ -298,6 +298,9 @@ def model_fingerprint(what gets compiled."""blocks = list(getattr(transformer, "_repeated_blocks", []) or [])+ if not blocks and transformer is not None:+ from .diffusion_regional_compile import verified_repeated_blocks+ blocks = list(verified_repeated_blocks(type(transformer).__name__))if quant is not None and transformer is not None:# Native layers compile to a different graph than torchao under the same scheme name.from .diffusion_native_quant import native_quant_signaturediff --git a/studio/backend/core/inference/diffusion_regional_compile.py b/studio/backend/core/inference/diffusion_regional_compile.pynew file mode 100644index 00000000000..ce4bd903428--- /dev/null+++ b/studio/backend/core/inference/diffusion_regional_compile.py@@ -0,0 +1,82 @@+# SPDX-License-Identifier: AGPL-3.0-only+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0++"""Supplies ``_repeated_blocks`` (on the instance) for verified Diffusers transformers that declare none."""++from __future__ import annotations++from typing import Any, Optional++_VERIFIED_BLOCKS: dict[str, tuple[str, ...]] = {+ "Lumina2Transformer2DModel": ("Lumina2TransformerBlock",),+ "HiDreamImageTransformer2DModel": (+ "HiDreamImageTransformerBlock",+ "HiDreamImageSingleTransformerBlock",+ ),+}+++def discover_repeated_blocks(model: Any) -> tuple[str, ...]:+ names = tuple(getattr(model, "_no_split_modules", None) or ())+ if not names:+ return ()+ try:+ import torch.nn as nn+ counts: dict[str, int] = {}+ for sub in model.modules():+ if isinstance(sub, nn.ModuleList):+ for child in sub.modules():+ name = type(child).__name__+ if name in names:+ counts[name] = counts.get(name, 0) + 1+ except Exception: # noqa: BLE001 - a probe, never a failed load+ return ()+ return tuple(n for n in names if counts.get(n, 0) >= 2)+++def verified_repeated_blocks(class_name: Optional[str]) -> tuple[str, ...]:+ return _VERIFIED_BLOCKS.get(class_name or "", ())+++def ensure_repeated_blocks(model: Any) -> tuple[str, ...]:+ blocks = tuple(getattr(model, "_repeated_blocks", None) or ())+ if not blocks:+ verified = verified_repeated_blocks(type(model).__name__)+ found = discover_repeated_blocks(model) if verified else ()+ blocks = tuple(n for n in verified if n in found)+ if not blocks:+ return ()+ model._repeated_blocks = list(blocks)+ # Also when Diffusers declares the blocks itself: its released moe_infer still host-syncs.+ if type(model).__name__ == "HiDreamImageTransformer2DModel":+ install_traceable_moe(model)+ return blocks+++def _moe_infer_dense(self: Any, x: Any, flat_expert_indices: Any, flat_expert_weights: Any) -> Any:+ # `where`, not a zero weight: an unpicked expert that overflows in fp16 must add 0, not 0 * inf = NaN.+ import torch++ k = self.num_activated_experts+ idx = flat_expert_indices.view(-1, k)+ weights = flat_expert_weights.view(-1, k).to(x.dtype)+ zero = x.new_zeros(())+ out = torch.zeros_like(x)+ for i, expert in enumerate(self.experts):+ hit = idx == i+ weight = (weights * hit).sum(-1, keepdim = True)+ out = out + torch.where(hit.any(-1, keepdim = True), expert(x) * weight, zero)+ return out+++def install_traceable_moe(model: Any) -> int:+ """Swap HiDream's ``moe_infer`` (host ``bincount().cpu()`` breaks every block's graph) for dense experts."""+ patched = 0+ for sub in model.modules():+ if type(sub).__name__ != "MOEFeedForwardSwiGLU":+ continue+ if not callable(getattr(sub, "moe_infer", None)) or not hasattr(sub, "experts"):+ continue+ sub.moe_infer = _moe_infer_dense.__get__(sub)+ patched += 1+ return patcheddiff --git a/studio/backend/core/inference/diffusion_speed.py b/studio/backend/core/inference/diffusion_speed.pyindex fbd5f0d9093..b9cf81ae026 100644--- a/studio/backend/core/inference/diffusion_speed.py+++ b/studio/backend/core/inference/diffusion_speed.py@@ -334,7 +334,7 @@ def fp16_compile_explicit_only(target: Any) -> bool:def family_compiles_regionally(family: Any) -> bool:- """False only for an EMPTY ``_repeated_blocks`` (``compile_repeated_blocks`` raises); unknown reads True."""+ """False only for an EMPTY ``_repeated_blocks`` Studio cannot supply; unknown reads True."""if getattr(family, "denoiser_attr", "transformer") != "transformer":return Truename = getattr(family, "transformer_class", None)@@ -354,7 +354,11 @@ def family_compiles_regionally(family: Any) -> bool:return Trueif cls is None or not hasattr(cls, "_repeated_blocks"):return True- return bool(cls._repeated_blocks)+ if cls._repeated_blocks:+ return True+ from .diffusion_regional_compile import verified_repeated_blocks++ return bool(verified_repeated_blocks(name))def _is_bfloat16(dtype: Any) -> bool:@@ -734,6 +738,13 @@ def _compile_repeated_blocks(unet = _denoiser_unet(pipe) if not dits else Noneif not dits and unet is None:return False+ # Before the stream-merge probe below, which reads the same block names.+ for transformer in dits:+ try:+ from .diffusion_regional_compile import ensure_repeated_blocks+ ensure_repeated_blocks(transformer)+ except Exception as exc: # noqa: BLE001 - optimisation only+ _warn(logger, "repeated block discovery", exc)# default: dynamic=True, fast cold start, no recompile on resolution change. max: max-autotune-no-cudagraphs +# automatic dynamic (None): the first shape compiles static and autotuned, and a dimension that then changes is# generalised once. dynamic=False recompiled on every new prompt length for DiTs whose blocks see the text tokensdiff --git a/studio/backend/core/inference/diffusion_transformer_quant.py b/studio/backend/core/inference/diffusion_transformer_quant.pyindex 61db8a0f083..347c094db43 100644--- a/studio/backend/core/inference/diffusion_transformer_quant.py+++ b/studio/backend/core/inference/diffusion_transformer_quant.py@@ -429,6 +429,18 @@ _FAMILY_AUTO_PREFER: dict[str, _AutoPrefer] = {}+# AUTO keeps bf16 while it fits for these (measured with both sides compiled: INT8 was at most ~5% faster and failed the+# default-on LPIPS bar); when bf16 would offload, INT8 is still picked.+_FAMILY_AUTO_BF16_WHEN_RESIDENT: dict[str, str] = {+ "lumina-2": "INT8 is no faster than compiled bf16 on a card that holds bf16 and changes image detail",+ "hidream-i1": "INT8 is barely faster than compiled bf16 on a card that holds bf16 and changes image detail",+}+++def auto_bf16_when_resident_reason(family: Optional[str]) -> Optional[str]:+ return _FAMILY_AUTO_BF16_WHEN_RESIDENT.get(str(family or "").strip().lower())++# Schemes denied for TRAINING on top of the inference table. Training holds a stricter bar because the evidence above# is rendering evidence: it says a frozen fp8 forward reconstructs the bf16 image, not that a LoRA converges when its# frozen linears are fp8. Nobody has run that, so qwen fp8 stays out of the Train UI until someone does. Delete thediff --git a/studio/backend/tests/test_diffusion_pipeline_prequant.py b/studio/backend/tests/test_diffusion_pipeline_prequant.pyindex aba787026a1..11898f4dedc 100644--- a/studio/backend/tests/test_diffusion_pipeline_prequant.py+++ b/studio/backend/tests/test_diffusion_pipeline_prequant.py@@ -1222,6 +1222,75 @@ def test_an_uncompilable_family_quantises_auto_when_the_budget_is_unmeasured(assert spy.quantised == ["auto"]+def _measured_bf16_family(monkeypatch, *, measured = True):+ _uncompilable(monkeypatch, measured = measured)+ monkeypatch.setattr(dmod, "family_compiles_regionally", lambda _fam: True)+ monkeypatch.setattr(dmod, "auto_bf16_when_resident_reason", lambda _name: "measured: no faster")+++def test_a_measured_bf16_family_keeps_the_released_weights_that_fit(monkeypatch):+ backend = _settle_backend(monkeypatch)+ _offload_when_bf16_sized(monkeypatch, "none")+ _measured_bf16_family(monkeypatch)+ assert _settle(backend) is None+++def test_a_measured_bf16_family_seeds_when_the_released_weights_would_offload(monkeypatch):+ backend = _settle_backend(monkeypatch)+ _offload_when_bf16_sized(monkeypatch, "sequential")+ _measured_bf16_family(monkeypatch)+ assert _settle(backend) == "fp8"+++def test_a_measured_bf16_family_loads_auto_unquantised_with_the_measured_reason(+ fake_runtime, monkeypatch+):+ backend, spy = _load_backend(monkeypatch)+ _measured_bf16_family(monkeypatch)+ status = _load(backend, _pipeline_prequant_planned = None, _pipeline_prequant_skipped = ())++ assert spy.quantised == [] and spy.seeds == []+ resolved = status["resolved"]["transformer_quant"]+ assert (resolved["value"], resolved["source"]) == ("off", "auto")+ assert "measured: no faster" in resolved["reason"]+ assert "cannot be regionally compiled" not in resolved["reason"]+++def test_a_measured_bf16_family_quantises_auto_when_the_budget_is_unmeasured(+ fake_runtime, monkeypatch+):+ backend, spy = _load_backend(monkeypatch)+ _measured_bf16_family(monkeypatch, measured = False)+ _load(backend, _pipeline_prequant_planned = None, _pipeline_prequant_skipped = ())++ assert spy.quantised == ["auto"]+++ "repo, kept",+ [+ ("Alpha-VLLM/Lumina-Image-2.0", True),+ ("HiDream-ai/HiDream-I1-Full", True),+ ("black-forest-labs/FLUX.1-dev", False),+ ("Tongyi-MAI/Z-Image-Turbo", False),+ ("Qwen/Qwen-Image", False),+ ],+)+def test_the_bf16_rule_names_only_the_measured_families(repo, kept):+ fam = detect_family_for_pick(repo, None, None)+ assert fam is not None+ reason = dmod._auto_keeps_bf16_reason(fam)+ assert (reason is not None) is kept+ assert reason is None or "cannot be regionally compiled" not in reason+ assert dmod._auto_keeps_bf16_reason(fam, "gguf") is None+++def test_an_uncompilable_family_keeps_its_reason_for_a_gguf_load(monkeypatch):+ fam = detect_family_for_pick("Alpha-VLLM/Lumina-Image-2.0", None, None)+ monkeypatch.setattr(dmod, "family_compiles_regionally", lambda _fam: False)+ assert "cannot be regionally compiled" in dmod._auto_keeps_bf16_reason(fam, "gguf")++def test_a_resident_plan_whose_requirement_exceeds_the_budget_proves_no_fit():fits = types.SimpleNamespace(offload_policy = "none",diff --git a/studio/backend/tests/test_diffusion_regional_compile.py b/studio/backend/tests/test_diffusion_regional_compile.pynew file mode 100644index 00000000000..5aec9940d2e--- /dev/null+++ b/studio/backend/tests/test_diffusion_regional_compile.py@@ -0,0 +1,231 @@+# SPDX-License-Identifier: AGPL-3.0-only+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0++"""Lumina-2 / HiDream-I1 declare no ``_repeated_blocks``; Studio supplies them so the denoiser compiles regionally."""++import sys+import types++import pytest++torch = pytest.importorskip("torch")+diffusers = pytest.importorskip("diffusers")++from core.inference import diffusion_regional_compile as rc+from core.inference import diffusion_speed as ds+++def _tiny_lumina():+ return diffusers.Lumina2Transformer2DModel(+ sample_size = 8,+ patch_size = 2,+ in_channels = 4,+ hidden_size = 32,+ num_layers = 2,+ num_refiner_layers = 1,+ num_attention_heads = 2,+ num_kv_heads = 1,+ multiple_of = 16,+ norm_eps = 1e-5,+ scaling_factor = 1.0,+ axes_dim_rope = (4, 4, 8),+ axes_lens = (32, 32, 32),+ cap_feat_dim = 16,+ ).eval()+++def _tiny_hidream():+ return diffusers.HiDreamImageTransformer2DModel(+ patch_size = 2,+ in_channels = 4,+ out_channels = 4,+ num_layers = 2,+ num_single_layers = 2,+ attention_head_dim = 8,+ num_attention_heads = 2,+ caption_channels = [8, 8],+ text_emb_dim = 8,+ num_routed_experts = 4,+ num_activated_experts = 2,+ axes_dims_rope = (4, 2, 2),+ max_resolution = (8, 8),+ llama_layers = [0, 1, 2, 3],+ ).eval()+++def _hidream_inputs(batch = 2):+ g = torch.Generator().manual_seed(0)+ return dict(+ hidden_states = torch.randn(batch, 4, 8, 8, generator = g),+ timesteps = torch.tensor([500] * batch),+ encoder_hidden_states_t5 = torch.randn(batch, 5, 8, generator = g),+ encoder_hidden_states_llama3 = torch.randn(4, batch, 7, 8, generator = g),+ pooled_embeds = torch.randn(batch, 8, generator = g),+ return_dict = False,+ )++[email protected]("build", [_tiny_lumina, _tiny_hidream])+def test_released_class_still_declares_no_repeated_blocks(build):+ model = build()+ declared = list(type(model)._repeated_blocks or [])+ if declared:+ assert list(rc.ensure_repeated_blocks(model)) == declared+ return+ with pytest.raises(ValueError):+ model.compile_repeated_blocks()+++ "build, blocks",+ [+ (_tiny_lumina, ["Lumina2TransformerBlock"]),+ (_tiny_hidream, ["HiDreamImageTransformerBlock", "HiDreamImageSingleTransformerBlock"]),+ ],+)+def test_supplied_blocks_make_compile_repeated_blocks_run(build, blocks):+ model = build()+ assert list(rc.ensure_repeated_blocks(model)) == blocks+ assert model._repeated_blocks == blocks and not type(model)._repeated_blocks+ model.compile_repeated_blocks(fullgraph = True, dynamic = True)+ compiled = [m for m in model.modules() if type(m).__name__ in blocks]+ assert compiled and all(getattr(m, "_compiled_call_impl", None) is not None for m in compiled)+++def test_unverified_class_with_empty_blocks_is_left_alone():+ model = _tiny_lumina()+ model.__class__ = type("SomeOtherTransformer", (type(model),), {})+ assert rc.ensure_repeated_blocks(model) == ()+ assert "_repeated_blocks" not in vars(model)+++def test_declared_blocks_are_returned_unchanged():+ model = _tiny_lumina()+ model._repeated_blocks = ["Custom"]+ assert rc.ensure_repeated_blocks(model) == ("Custom",)+++def test_hidream_declaring_its_own_blocks_still_gets_traceable_experts():+ model = _tiny_hidream()+ model._repeated_blocks = ["HiDreamImageTransformerBlock", "HiDreamImageSingleTransformerBlock"]+ assert rc.ensure_repeated_blocks(model) == tuple(model._repeated_blocks)+ moes = [m for m in model.modules() if type(m).__name__ == "MOEFeedForwardSwiGLU"]+ assert moes and all(m.moe_infer.__func__ is rc._moe_infer_dense for m in moes)+++def test_discovery_matches_the_verified_names():+ assert rc.discover_repeated_blocks(_tiny_lumina()) == rc.verified_repeated_blocks(+ "Lumina2Transformer2DModel"+ )+ assert rc.discover_repeated_blocks(_tiny_hidream()) == rc.verified_repeated_blocks(+ "HiDreamImageTransformer2DModel"+ )++[email protected]("dtype, tol", [(torch.float32, 1e-6), (torch.bfloat16, 1e-2)])+def test_hidream_dense_experts_match_the_released_loop(dtype, tol):+ torch.manual_seed(0)+ model = _tiny_hidream().to(dtype)+ inputs = {+ k: (v.to(dtype) if v.is_floating_point() else v)+ for k, v in _hidream_inputs().items()+ if k != "return_dict"+ }+ with torch.no_grad():+ ref = model(**inputs, return_dict = False)[0]+ assert rc.install_traceable_moe(model) == 4+ out = model(**inputs, return_dict = False)[0]+ torch.testing.assert_close(out, ref, rtol = tol, atol = tol)+++def test_hidream_unpicked_expert_overflow_adds_zero_not_nan():+ model = _tiny_hidream()+ moe = next(m for m in model.modules() if type(m).__name__ == "MOEFeedForwardSwiGLU")+ rc.install_traceable_moe(model)+ x = torch.randn(6, 16)+ idx = torch.tensor([[0, 1]] * 6).view(-1)+ weights = torch.full((12, 1), 0.5)+ moe.experts[3].forward = lambda t: torch.full_like(t, float("inf")) # never picked+ with torch.no_grad():+ out = moe.moe_infer(x, idx, weights)+ assert torch.isfinite(out).all()+++def _block_calls(model, inputs, name):+ captured = {}++ def grab(mod, args, kwargs):+ captured.setdefault(type(mod).__name__, (mod, args, kwargs))++ hooks = [+ m.register_forward_pre_hook(grab, with_kwargs = True)+ for m in model.modules()+ if type(m).__name__ == name+ ]+ with torch.no_grad():+ model(**inputs)+ for h in hooks:+ h.remove()+ return captured[name]+++def _assert_one_graph(mod, args, kwargs):+ torch._dynamo.reset()+ explained = torch._dynamo.explain(mod)(*args, **kwargs)+ assert explained.graph_break_count == 0, [str(r.reason) for r in explained.break_reasons]+ assert explained.graph_count == 1+++ "name", ["HiDreamImageTransformerBlock", "HiDreamImageSingleTransformerBlock"]+)+def test_hidream_blocks_trace_without_graph_breaks(name):+ model = _tiny_hidream()+ rc.ensure_repeated_blocks(model)+ _assert_one_graph(*_block_calls(model, _hidream_inputs(), name))+++def test_hidream_released_moe_loop_breaks_the_graph():+ model = _tiny_hidream()+ mod, args, kwargs = _block_calls(model, _hidream_inputs(), "HiDreamImageTransformerBlock")+ torch._dynamo.reset()+ explained = torch._dynamo.explain(mod)(*args, **kwargs)+ assert explained.graph_break_count > 0+++def test_lumina_block_traces_without_graph_breaks():+ model = _tiny_lumina()+ rc.ensure_repeated_blocks(model)+ g = torch.Generator().manual_seed(0)+ inputs = dict(+ hidden_states = torch.randn(1, 4, 8, 8, generator = g),+ timestep = torch.tensor([0.5]),+ encoder_hidden_states = torch.randn(1, 6, 16, generator = g),+ encoder_attention_mask = torch.ones(1, 6, dtype = torch.bool),+ return_dict = False,+ )+ _assert_one_graph(*_block_calls(model, inputs, "Lumina2TransformerBlock"))+++def test_family_compiles_regionally_trusts_verified_classes(monkeypatch):+ fake = types.ModuleType("diffusers")+ for name in ("Lumina2Transformer2DModel", "HiDreamImageTransformer2DModel", "OtherEmpty"):+ setattr(fake, name, type(name, (), {"_repeated_blocks": []}))+ monkeypatch.setitem(sys.modules, "diffusers", fake)++ def fam(cls):+ return types.SimpleNamespace(transformer_class = cls, denoiser_attr = "transformer")++ assert ds.family_compiles_regionally(fam("Lumina2Transformer2DModel")) is True+ assert ds.family_compiles_regionally(fam("HiDreamImageTransformer2DModel")) is True+ assert ds.family_compiles_regionally(fam("OtherEmpty")) is False+++def test_studio_compile_path_engages_on_lumina(monkeypatch):+ monkeypatch.setattr(ds, "guard_compiled_blocks", lambda *a, **k: None)+ pipe = types.SimpleNamespace(transformer = _tiny_lumina())+ assert ds._compile_repeated_blocks(pipe, None) is True+ blocks = [+ m for m in pipe.transformer.modules() if type(m).__name__ == "Lumina2TransformerBlock"+ ]+ assert blocks and all(getattr(m, "_compiled_call_impl", None) is not None for m in blocks)