Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
Studio: fix inductor CantSplit on torch 2.12/2.13 and speed up the Qwen-Image-2.1 VAE (#12059)
Daniel Han03d3cc4c4e12
diff --git a/studio/backend/core/inference/diffusion_inductor_backports.py b/studio/backend/core/inference/diffusion_inductor_backports.pynew file mode 100644index 00000000000..e4c3c060ae1--- /dev/null+++ b/studio/backend/core/inference/diffusion_inductor_backports.py@@ -0,0 +1,148 @@+# SPDX-License-Identifier: AGPL-3.0-only+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0++"""Backport of torch 2.14's polynomial-gcd proof in ``SizeVarAllocator.statically_known_multiple_of``.++torch 2.12 / 2.13 cannot prove ``(k*a - k*b) % (a - b) == 0`` and raise inductor ``CantSplit`` under+``torch.compile(dynamic=True)`` (Qwen-Image-2.1 torchao fp8 / int8, FLUX.1 single-stream blocks). Probe-gated, not+version-gated; only turns False into True when the gcd equals the denominator up to sign.+Kill switch: ``UNSLOTH_DIFFUSION_INDUCTOR_BACKPORTS=0``."""++from __future__ import annotations++import os+import threading+from typing import Any++BACKPORTS_ENV = "UNSLOTH_DIFFUSION_INDUCTOR_BACKPORTS"++# Mirrors torch 2.14 ``_MAX_SYMBOLS_FOR_EXPENSIVE_SYMPY_OPS`` / ``_MAX_ADD_TERMS_FOR_POLY_GCD``.+_MAX_SYMBOLS = 20+_MAX_ADD_TERMS = 20++_LOCK = threading.Lock()+_STATE: dict[str, Any] = {}+++def backports_disabled() -> bool:+ return (os.environ.get(BACKPORTS_ENV) or "").strip().lower() in ("0", "off", "false", "no")+++def _gcd_proves_multiple(allocator: Any, numerator: Any, denominator: Any) -> bool:+ import sympy++ if numerator == 0:+ return True+ try:+ # sympy gcd is over the rationals: gcd((a + b) / 2, a + b) == a + b, so require integer polynomials.+ for expr in (numerator, denominator):+ if any(not number.is_integer for number in expr.atoms(sympy.Number)):+ return False+ except Exception: # noqa: BLE001+ return False+ try:+ from torch.utils._sympy.functions import simple_floordiv_gcd+ except Exception: # noqa: BLE001 - very old torch: only the polynomial gcd below+ simple_floordiv_gcd = None+ try:+ symbols = set(getattr(numerator, "free_symbols", ())) | set(+ getattr(denominator, "free_symbols", ())+ )+ except Exception: # noqa: BLE001+ return False+ if len(symbols) > _MAX_SYMBOLS:+ return False++ def covers(gcd: Any) -> bool:+ if gcd == 1 or gcd == -1:+ return False+ if gcd == denominator or gcd == -denominator:+ return True+ simplify = getattr(allocator, "simplify", sympy.expand)+ return simplify(gcd - denominator) == 0 or simplify(gcd + denominator) == 0++ wide = any(+ isinstance(e, sympy.Add) and len(e.args) > _MAX_ADD_TERMS for e in (numerator, denominator)+ )+ try:+ if simple_floordiv_gcd is not None and covers(simple_floordiv_gcd(numerator, denominator)):+ return True+ if not wide and covers(sympy.gcd(numerator, denominator)):+ return True+ except (sympy.PolynomialError, TypeError, ValueError):+ return False+ return False+++def _stock_proves(allocator_cls: Any) -> bool:+ """Whether this torch's own check already proves the canonical failing case (torch 2.14+)."""+ import sympy++ a, b = sympy.symbols("s0 s1", integer = True, positive = True)+ allocator = allocator_cls()+ return bool(allocator.statically_known_multiple_of(4096 * a - 4096 * b, a - b))+++def install(logger: Any = None) -> bool:+ """Idempotently patch ``SizeVarAllocator.statically_known_multiple_of``. True when the backport is active."""+ if backports_disabled():+ return False+ with _LOCK:+ if "original" in _STATE:+ return True+ if _STATE.get("not_needed"):+ return False+ try:+ from torch._inductor.sizevars import SizeVarAllocator+ except Exception: # noqa: BLE001 - no inductor, nothing to fix+ return False+ original = getattr(SizeVarAllocator, "statically_known_multiple_of", None)+ if original is None:+ return False+ try:+ if _stock_proves(SizeVarAllocator):+ _STATE["not_needed"] = True+ return False+ except Exception: # noqa: BLE001 - a probe failure is not proof the stock check works; still patch+ pass++ def statically_known_multiple_of(self: Any, numerator: Any, denominator: Any) -> bool:+ if original(self, numerator, denominator):+ return True+ try:+ import sympy+ if isinstance(denominator, (int, sympy.Integer)):+ return False+ return _gcd_proves_multiple(self, numerator, denominator)+ except Exception: # noqa: BLE001 - a failed proof is the stock answer+ return False++ statically_known_multiple_of._unsloth_backport = True # type: ignore[attr-defined]+ statically_known_multiple_of.__wrapped__ = original # type: ignore[attr-defined]+ SizeVarAllocator.statically_known_multiple_of = statically_known_multiple_of+ _STATE["original"] = original+ _STATE["cls"] = SizeVarAllocator+ if logger is not None:+ logger.info(+ "diffusion.speed: backported torch 2.14 symbolic divisibility proof (inductor CantSplit fix)"+ )+ return True+++def uninstall() -> None:+ with _LOCK:+ original = _STATE.pop("original", None)+ cls = _STATE.pop("cls", None)+ _STATE.pop("not_needed", None)+ if original is not None and cls is not None:+ cls.statically_known_multiple_of = original+++def is_installed() -> bool:+ return "original" in _STATE+++def proof_available() -> bool:+ """Whether inductor can prove the stream-merge divisibility: stock torch 2.14+ or this backport installed."""+ install()+ return is_installed() or bool(_STATE.get("not_needed"))diff --git a/studio/backend/core/inference/diffusion_speed.py b/studio/backend/core/inference/diffusion_speed.pyindex ff3b5c8f6c2..c77ade0266c 100644--- a/studio/backend/core/inference/diffusion_speed.py+++ b/studio/backend/core/inference/diffusion_speed.py@@ -457,13 +457,15 @@ def apply_speed_optims(family_allows_compile = bool(getattr(family, "supports_torch_compile", True))# Lossless: a channels-last VAE speeds up its convs with no numeric change.- applied["channels_last"] = _vae_channels_last(pipe, logger)+ applied["channels_last"] = _vae_channels_last(+ pipe, logger, fused = on_cuda and _fused_vae_planned(pipe)+ )# Near-lossless, not bit-identical, so never on "off" (returned above).applied["vae_fp16_decode"] = _video_vae_half_decode(pipe, target, family, logger)if on_cuda:applied["vae_fused"] = _install_fused_vae(pipe, logger)- if on_cuda:+ if on_cuda and not _cudnn_benchmark_pointless(pipe):applied["cudnn_benchmark"] = _enable_cudnn_benchmark(logger)if on_cuda:@@ -579,10 +581,31 @@ def fp16_unet_offloaded(target: Any, pipe: Any, *, offload_active: bool) -> bool)-def _vae_channels_last(pipe: Any, logger: Any) -> bool:+# channels_last: slower on the stock path (72.7 vs 89 ms), faster once the fused norms install (104.7 vs 121.0 ms).+_VAE_CHANNELS_LAST_DENY: frozenset[str] = frozenset({"AutoencoderKLQwenImage21"})+++# DiT-only VAEs: cudnn.benchmark saves ~3 ms per decode (fused, 1024) but re-tunes 0.7-2 s at every new resolution.+_CUDNN_BENCHMARK_DENY_VAES: frozenset[str] = frozenset({"AutoencoderKLQwenImage21"})+++def _cudnn_benchmark_pointless(pipe: Any) -> bool:+ if _denoiser_unet(pipe) is not None:+ return False+ return type(getattr(pipe, "vae", None)).__name__ in _CUDNN_BENCHMARK_DENY_VAES+++def _vae_channels_last(+ pipe: Any,+ logger: Any,+ *,+ fused: bool = False,+) -> bool:vae = getattr(pipe, "vae", None)if vae is None or not hasattr(vae, "to"):return False+ if type(vae).__name__ in _VAE_CHANNELS_LAST_DENY and not fused:+ return Falsetry:import torchvae.to(memory_format = torch.channels_last)@@ -759,6 +782,7 @@ def _compile_repeated_blocks(unet = _denoiser_unet(pipe) if not dits else Noneif not dits and unet is None:return False+ _install_inductor_backports(logger)# Before the stream-merge probe below, which reads the same block names.for transformer in dits:try:@@ -891,13 +915,23 @@ def _compile_repeated_blocks(return engaged+def _install_inductor_backports(logger: Any) -> bool:+ """Probe-gated backport of torch 2.14's CantSplit divisibility proof for 2.12 / 2.13. Never fails a load."""+ try:+ from . import diffusion_inductor_backports+ return diffusion_inductor_backports.install(logger)+ except Exception as exc: # noqa: BLE001 - optimisation only+ _warn(logger, "inductor backports", exc)+ return False++def compile_dynamic(transformer: Any, dynamic: Optional[bool]) -> Optional[bool]:"""The ``dynamic`` a DiT is actually compiled with, so compile-cache fingerprints key on the same value.dynamic=True makes even the constant segment starts symbolic, and on Qwen-Image-2.1 the attention output cat- (text + target, length s87 - s89) then fuses into torchao's per-row activation-quant reduction, which inductor- cannot split (CantSplit, every render failed). Automatic dynamic (None) compiles the first shapes static and only- generalises what actually varies: stable after ~3 recompiles, same numerics."""+ (text + target, length s87 - s89) then fuses into torchao's per-row activation-quant reduction (CantSplit on torch+ 2.12 / 2.13 without ``diffusion_inductor_backports``). Even with the backport it is slower than automatic dynamic+ (None), which does not recompile across prompt lengths or resolutions."""# Static kernels for video DiTs: dynamic shapes made LTX-2.3's QK-norm + RoPE kernels ~3x slower.if transformer is not None and getattr(transformer, "_unsloth_compile_static", False):return False@@ -1299,6 +1333,7 @@ def _compile_vae_decode(return Trueif getattr(vae, "_unsloth_compile_decode_error", None):return False+ _install_inductor_backports(logger)try:import torchdiff --git a/studio/backend/tests/test_diffusion_inductor_backports.py b/studio/backend/tests/test_diffusion_inductor_backports.pynew file mode 100644index 00000000000..555a76dd06e--- /dev/null+++ b/studio/backend/tests/test_diffusion_inductor_backports.py@@ -0,0 +1,163 @@+# SPDX-License-Identifier: AGPL-3.0-only+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0++"""Tests for ``diffusion_inductor_backports.py`` (the torch 2.14 symbolic divisibility backport). CPU only."""++from __future__ import annotations++import pytest++from core.inference import diffusion_inductor_backports as bp++torch = pytest.importorskip("torch")+sympy = pytest.importorskip("sympy")+sizevars = pytest.importorskip("torch._inductor.sizevars")++[email protected](autouse = True)+def _restore(monkeypatch):+ monkeypatch.delenv(bp.BACKPORTS_ENV, raising = False)+ bp.uninstall()+ # Exercise the patched method on every torch, including ones whose stock check already proves the probe.+ monkeypatch.setattr(bp, "_stock_proves", lambda _cls: False)+ yield+ bp.uninstall()+++def _syms():+ return sympy.symbols("s87 s89", integer = True, positive = True)+++def _proves(num, den) -> bool:+ return bool(sizevars.SizeVarAllocator().statically_known_multiple_of(num, den))+++def test_proves_the_qwen21_and_flux_cantsplit_expressions():+ a, b = _syms()+ bp.install()+ # Qwen-Image-2.1 torchao (text + target attention output) and FLUX.1 single-block (attn + mlp cat) shapes.+ assert _proves(4096 * a - 4096 * b, a - b)+ assert _proves(15360 * a + 15360 * b, a + b)+ assert _proves(4096 * a - 4096 * b, b - a) # sign of the group does not matter for divisibility+++def test_never_proves_a_non_multiple():+ a, b = _syms()+ bp.install()+ assert not _proves(4096 * a - 4095 * b, a - b)+ assert not _proves(4096 * a, a + b)+ assert not _proves(a * b + 1, a)+ # sympy.gcd is taken over the rationals: gcd((a + b) / 2, a + b) == a + b, yet (a + b) / 2 is no multiple.+ assert not _proves((a + b) / 2, a + b)+ assert not _proves(sympy.Rational(8, 3) * a * b, a * b)+++def test_integer_denominators_keep_the_stock_answer():+ a, _b = _syms()+ bp.install()+ assert _proves(4096 * a, 64)+ assert not _proves(4096 * a + 3, 64)+++def test_install_is_idempotent_and_uninstall_restores():+ stock = sizevars.SizeVarAllocator.statically_known_multiple_of+ first = bp.install()+ assert first is True+ patched = sizevars.SizeVarAllocator.statically_known_multiple_of+ assert bp.install() is first+ assert sizevars.SizeVarAllocator.statically_known_multiple_of is patched+ bp.uninstall()+ assert sizevars.SizeVarAllocator.statically_known_multiple_of is stock+ assert not bp.is_installed()+++def test_kill_switch(monkeypatch):+ monkeypatch.setenv(bp.BACKPORTS_ENV, "0")+ stock = sizevars.SizeVarAllocator.statically_known_multiple_of+ assert bp.install() is False+ assert sizevars.SizeVarAllocator.statically_known_multiple_of is stock+++def test_torch_that_already_proves_it_is_left_alone(monkeypatch):+ monkeypatch.setattr(bp, "_stock_proves", lambda _cls: True)+ stock = sizevars.SizeVarAllocator.statically_known_multiple_of+ assert bp.install() is False+ assert sizevars.SizeVarAllocator.statically_known_multiple_of is stock+ assert not bp.is_installed()+++def test_stock_probe_matches_torch_version(monkeypatch):+ """Regressed in 2.12.0, fixed in 2.14.0; the probe, not the version, decides."""+ monkeypatch.undo()+ major_minor = tuple(int(x) for x in torch.__version__.split("+")[0].split(".")[:2])+ proves = bp._stock_proves(sizevars.SizeVarAllocator)+ if (2, 12) <= major_minor < (2, 14):+ assert not proves+ else:+ assert proves+++def test_a_failing_proof_returns_the_stock_answer(monkeypatch):+ a, b = _syms()+ stock = _proves(4096 * a - 4096 * b, a - b)+ bp.install()++ def boom(*_a, **_k):+ raise RuntimeError("sympy exploded")++ monkeypatch.setattr(bp, "_gcd_proves_multiple", boom)+ assert _proves(4096 * a - 4096 * b, a - b) == stock+++def test_regional_compile_and_vae_compile_install_the_backport(monkeypatch):+ import types++ from core.inference import diffusion_speed as ds_mod++ calls: list = []+ monkeypatch.setattr(bp, "install", lambda logger = None: calls.append("install") or True)++ class _Dit(torch.nn.Module):+ _repeated_blocks: list = []++ def compile_repeated_blocks(self, **kwargs):+ calls.append(("compile", kwargs.get("dynamic")))++ pipe = types.SimpleNamespace(transformer = _Dit())+ monkeypatch.setattr(ds_mod, "_inductor_config", lambda: None)+ monkeypatch.setattr(ds_mod, "guard_compiled_blocks", lambda transformer, logger = None: 0)+ assert ds_mod._compile_repeated_blocks(pipe, None) is True+ # Installed BEFORE the (lazy) compile is requested, so the first forward already lowers with it.+ compiles = [c for c in calls if c != "install"]+ assert calls[0] == "install" and len(compiles) == 1 and compiles[0][0] == "compile"++ calls.clear()+ monkeypatch.setattr(torch, "compile", lambda fn, **kw: fn)+ vae = types.SimpleNamespace(decode = lambda x: x)+ assert ds_mod._compile_vae_decode(types.SimpleNamespace(vae = vae), None) is True+ assert calls and set(calls) == {"install"}+++def test_a_broken_backport_never_fails_the_compile(monkeypatch):+ import types++ from core.inference import diffusion_speed as ds_mod++ def boom(logger = None):+ raise ImportError("no inductor")++ monkeypatch.setattr(bp, "install", boom)+ assert ds_mod._install_inductor_backports(None) is False+ monkeypatch.setattr(torch, "compile", lambda fn, **kw: fn)+ vae = types.SimpleNamespace(decode = lambda x: x)+ assert ds_mod._compile_vae_decode(types.SimpleNamespace(vae = vae), None) is True+++def test_proof_available_on_a_patched_or_fixed_torch(monkeypatch):+ assert bp.proof_available() is True+ bp.uninstall()+ monkeypatch.setattr(bp, "_stock_proves", lambda _cls: True)+ assert bp.proof_available() is True+ bp.uninstall()+ monkeypatch.setenv(bp.BACKPORTS_ENV, "0")+ assert bp.proof_available() is Falsediff --git a/studio/backend/tests/test_diffusion_qwenimage21_vae_layout.py b/studio/backend/tests/test_diffusion_qwenimage21_vae_layout.pynew file mode 100644index 00000000000..2c261d107f7--- /dev/null+++ b/studio/backend/tests/test_diffusion_qwenimage21_vae_layout.py@@ -0,0 +1,100 @@+# SPDX-License-Identifier: AGPL-3.0-only+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0++"""AutoencoderKLQwenImage21 stays contiguous unless the fused VAE norms install; other VAEs unchanged."""++from __future__ import annotations++import types++import pytest++from core.inference import diffusion_speed as ds_mod++torch = pytest.importorskip("torch")+++def _vae(name: str):+ calls: list = []++ def to(self, memory_format = None):+ calls.append(memory_format)+ return self++ cls = type(name, (), {"to": to})+ return cls(), calls+++def test_qwen_image_21_vae_stays_contiguous():+ vae, calls = _vae("AutoencoderKLQwenImage21")+ assert ds_mod._vae_channels_last(types.SimpleNamespace(vae = vae), None) is False+ assert calls == []+++def test_qwen_image_21_vae_goes_channels_last_when_the_fused_norms_install():+ vae, calls = _vae("AutoencoderKLQwenImage21")+ assert ds_mod._vae_channels_last(types.SimpleNamespace(vae = vae), None, fused = True) is True+ assert calls == [torch.channels_last]++[email protected]("planned", [True, False])+def test_apply_speed_optims_layout_follows_the_fused_planner(monkeypatch, planned):+ vae, calls = _vae("AutoencoderKLQwenImage21")+ monkeypatch.setattr(ds_mod, "_fused_vae_planned", lambda pipe: planned)+ monkeypatch.setattr(ds_mod, "_install_fused_vae", lambda pipe, logger: planned)+ monkeypatch.setattr(ds_mod, "_enable_cudnn_benchmark", lambda logger: True)+ monkeypatch.setattr(ds_mod, "_compile_repeated_blocks", lambda *a, **k: False)+ monkeypatch.setattr(ds_mod, "compile_eligible", lambda *a, **k: False)+ target = types.SimpleNamespace(device = "cuda", dtype = torch.bfloat16)+ family = types.SimpleNamespace(name = "qwen-image-2.1", supports_torch_compile = True)+ applied = ds_mod.apply_speed_optims(+ types.SimpleNamespace(vae = vae), target, is_gguf = False, family = family, speed_mode = "default"+ )+ assert applied["channels_last"] is planned+ assert calls == ([torch.channels_last] if planned else [])++[email protected]("name", ["AutoencoderKL", "AutoencoderKLQwenImage", "AutoencoderKLWan"])+def test_other_vaes_still_go_channels_last(name):+ vae, calls = _vae(name)+ assert ds_mod._vae_channels_last(types.SimpleNamespace(vae = vae), None) is True+ assert calls == [torch.channels_last]+++def test_real_qwen_image_21_vae_class_name_matches_the_deny_list():+ mod = pytest.importorskip("diffusers.models.autoencoders.autoencoder_kl_qwenimage21")+ assert mod.AutoencoderKLQwenImage21.__name__ in ds_mod._VAE_CHANNELS_LAST_DENY+++def test_cudnn_benchmark_is_skipped_for_a_qwen_image_21_pipeline():+ vae, _calls = _vae("AutoencoderKLQwenImage21")+ assert ds_mod._cudnn_benchmark_pointless(types.SimpleNamespace(vae = vae)) is True++[email protected]("name", ["AutoencoderKL", "AutoencoderKLQwenImage", "AutoencoderKLWan"])+def test_cudnn_benchmark_kept_for_other_vaes(name):+ vae, _calls = _vae(name)+ assert ds_mod._cudnn_benchmark_pointless(types.SimpleNamespace(vae = vae)) is False+++def test_cudnn_benchmark_kept_when_a_unet_denoises(monkeypatch):+ vae, _calls = _vae("AutoencoderKLQwenImage21")+ monkeypatch.setattr(ds_mod, "_denoiser_unet", lambda pipe: object())+ assert ds_mod._cudnn_benchmark_pointless(types.SimpleNamespace(vae = vae)) is False+++def test_apply_speed_optims_leaves_cudnn_benchmark_alone_for_qwen_image_21(monkeypatch):+ vae, _calls = _vae("AutoencoderKLQwenImage21")+ pipe = types.SimpleNamespace(vae = vae)+ enabled: list = []+ monkeypatch.setattr(ds_mod, "_enable_cudnn_benchmark", lambda logger: enabled.append(1) or True)+ monkeypatch.setattr(ds_mod, "_compile_repeated_blocks", lambda *a, **k: False)+ monkeypatch.setattr(ds_mod, "compile_eligible", lambda *a, **k: False)+ monkeypatch.setattr(ds_mod, "_fused_vae_planned", lambda pipe: False)+ target = types.SimpleNamespace(device = "cuda", dtype = torch.bfloat16)+ family = types.SimpleNamespace(name = "qwen-image-2.1", supports_torch_compile = True)+ applied = ds_mod.apply_speed_optims(+ pipe, target, is_gguf = False, family = family, speed_mode = "default"+ )+ assert applied["cudnn_benchmark"] is False and enabled == []+ assert applied["channels_last"] is Falsediff --git a/studio/backend/tests/test_middleware.py b/studio/backend/tests/test_middleware.pyindex 32c992492da..9f5e877146e 100644--- a/studio/backend/tests/test_middleware.py+++ b/studio/backend/tests/test_middleware.py@@ -1467,7 +1467,6 @@ class TestRemoteAccessCORS:exposed = response.headers["access-control-expose-headers"].lower().split(",")assert "x-typesafe-request-id" in {header.strip() for header in exposed}-TUNNEL = "https://demo-abc.trycloudflare.com"@staticmethod