Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

Studio: fix inductor CantSplit on torch 2.12/2.13 and speed up the Qwen-Image-2.1 VAE (#12059)

Daniel Han03d3cc4c4e12
diff --git a/studio/backend/core/inference/diffusion_inductor_backports.py b/studio/backend/core/inference/diffusion_inductor_backports.py
new file mode 100644
index 00000000000..e4c3c060ae1
--- /dev/null
+++ b/studio/backend/core/inference/diffusion_inductor_backports.py
@@ -0,0 +1,148 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
+
+"""Backport of torch 2.14's polynomial-gcd proof in ``SizeVarAllocator.statically_known_multiple_of``.
+
+torch 2.12 / 2.13 cannot prove ``(k*a - k*b) % (a - b) == 0`` and raise inductor ``CantSplit`` under
+``torch.compile(dynamic=True)`` (Qwen-Image-2.1 torchao fp8 / int8, FLUX.1 single-stream blocks). Probe-gated, not
+version-gated; only turns False into True when the gcd equals the denominator up to sign.
+Kill switch: ``UNSLOTH_DIFFUSION_INDUCTOR_BACKPORTS=0``."""
+
+from __future__ import annotations
+
+import os
+import threading
+from typing import Any
+
+BACKPORTS_ENV = "UNSLOTH_DIFFUSION_INDUCTOR_BACKPORTS"
+
+# Mirrors torch 2.14 ``_MAX_SYMBOLS_FOR_EXPENSIVE_SYMPY_OPS`` / ``_MAX_ADD_TERMS_FOR_POLY_GCD``.
+_MAX_SYMBOLS = 20
+_MAX_ADD_TERMS = 20
+
+_LOCK = threading.Lock()
+_STATE: dict[str, Any] = {}
+
+
+def backports_disabled() -> bool:
+ return (os.environ.get(BACKPORTS_ENV) or "").strip().lower() in ("0", "off", "false", "no")
+
+
+def _gcd_proves_multiple(allocator: Any, numerator: Any, denominator: Any) -> bool:
+ import sympy
+
+ if numerator == 0:
+ return True
+ try:
+ # sympy gcd is over the rationals: gcd((a + b) / 2, a + b) == a + b, so require integer polynomials.
+ for expr in (numerator, denominator):
+ if any(not number.is_integer for number in expr.atoms(sympy.Number)):
+ return False
+ except Exception: # noqa: BLE001
+ return False
+ try:
+ from torch.utils._sympy.functions import simple_floordiv_gcd
+ except Exception: # noqa: BLE001 - very old torch: only the polynomial gcd below
+ simple_floordiv_gcd = None
+ try:
+ symbols = set(getattr(numerator, "free_symbols", ())) | set(
+ getattr(denominator, "free_symbols", ())
+ )
+ except Exception: # noqa: BLE001
+ return False
+ if len(symbols) > _MAX_SYMBOLS:
+ return False
+
+ def covers(gcd: Any) -> bool:
+ if gcd == 1 or gcd == -1:
+ return False
+ if gcd == denominator or gcd == -denominator:
+ return True
+ simplify = getattr(allocator, "simplify", sympy.expand)
+ return simplify(gcd - denominator) == 0 or simplify(gcd + denominator) == 0
+
+ wide = any(
+ isinstance(e, sympy.Add) and len(e.args) > _MAX_ADD_TERMS for e in (numerator, denominator)
+ )
+ try:
+ if simple_floordiv_gcd is not None and covers(simple_floordiv_gcd(numerator, denominator)):
+ return True
+ if not wide and covers(sympy.gcd(numerator, denominator)):
+ return True
+ except (sympy.PolynomialError, TypeError, ValueError):
+ return False
+ return False
+
+
+def _stock_proves(allocator_cls: Any) -> bool:
+ """Whether this torch's own check already proves the canonical failing case (torch 2.14+)."""
+ import sympy
+
+ a, b = sympy.symbols("s0 s1", integer = True, positive = True)
+ allocator = allocator_cls()
+ return bool(allocator.statically_known_multiple_of(4096 * a - 4096 * b, a - b))
+
+
+def install(logger: Any = None) -> bool:
+ """Idempotently patch ``SizeVarAllocator.statically_known_multiple_of``. True when the backport is active."""
+ if backports_disabled():
+ return False
+ with _LOCK:
+ if "original" in _STATE:
+ return True
+ if _STATE.get("not_needed"):
+ return False
+ try:
+ from torch._inductor.sizevars import SizeVarAllocator
+ except Exception: # noqa: BLE001 - no inductor, nothing to fix
+ return False
+ original = getattr(SizeVarAllocator, "statically_known_multiple_of", None)
+ if original is None:
+ return False
+ try:
+ if _stock_proves(SizeVarAllocator):
+ _STATE["not_needed"] = True
+ return False
+ except Exception: # noqa: BLE001 - a probe failure is not proof the stock check works; still patch
+ pass
+
+ def statically_known_multiple_of(self: Any, numerator: Any, denominator: Any) -> bool:
+ if original(self, numerator, denominator):
+ return True
+ try:
+ import sympy
+ if isinstance(denominator, (int, sympy.Integer)):
+ return False
+ return _gcd_proves_multiple(self, numerator, denominator)
+ except Exception: # noqa: BLE001 - a failed proof is the stock answer
+ return False
+
+ statically_known_multiple_of._unsloth_backport = True # type: ignore[attr-defined]
+ statically_known_multiple_of.__wrapped__ = original # type: ignore[attr-defined]
+ SizeVarAllocator.statically_known_multiple_of = statically_known_multiple_of
+ _STATE["original"] = original
+ _STATE["cls"] = SizeVarAllocator
+ if logger is not None:
+ logger.info(
+ "diffusion.speed: backported torch 2.14 symbolic divisibility proof (inductor CantSplit fix)"
+ )
+ return True
+
+
+def uninstall() -> None:
+ with _LOCK:
+ original = _STATE.pop("original", None)
+ cls = _STATE.pop("cls", None)
+ _STATE.pop("not_needed", None)
+ if original is not None and cls is not None:
+ cls.statically_known_multiple_of = original
+
+
+def is_installed() -> bool:
+ return "original" in _STATE
+
+
+def proof_available() -> bool:
+ """Whether inductor can prove the stream-merge divisibility: stock torch 2.14+ or this backport installed."""
+ install()
+ return is_installed() or bool(_STATE.get("not_needed"))
diff --git a/studio/backend/core/inference/diffusion_speed.py b/studio/backend/core/inference/diffusion_speed.py
index ff3b5c8f6c2..c77ade0266c 100644
--- a/studio/backend/core/inference/diffusion_speed.py
+++ b/studio/backend/core/inference/diffusion_speed.py
@@ -457,13 +457,15 @@ def apply_speed_optims(
family_allows_compile = bool(getattr(family, "supports_torch_compile", True))
# Lossless: a channels-last VAE speeds up its convs with no numeric change.
- applied["channels_last"] = _vae_channels_last(pipe, logger)
+ applied["channels_last"] = _vae_channels_last(
+ pipe, logger, fused = on_cuda and _fused_vae_planned(pipe)
+ )
# Near-lossless, not bit-identical, so never on "off" (returned above).
applied["vae_fp16_decode"] = _video_vae_half_decode(pipe, target, family, logger)
if on_cuda:
applied["vae_fused"] = _install_fused_vae(pipe, logger)
- if on_cuda:
+ if on_cuda and not _cudnn_benchmark_pointless(pipe):
applied["cudnn_benchmark"] = _enable_cudnn_benchmark(logger)
if on_cuda:
@@ -579,10 +581,31 @@ def fp16_unet_offloaded(target: Any, pipe: Any, *, offload_active: bool) -> bool
)
-def _vae_channels_last(pipe: Any, logger: Any) -> bool:
+# channels_last: slower on the stock path (72.7 vs 89 ms), faster once the fused norms install (104.7 vs 121.0 ms).
+_VAE_CHANNELS_LAST_DENY: frozenset[str] = frozenset({"AutoencoderKLQwenImage21"})
+
+
+# DiT-only VAEs: cudnn.benchmark saves ~3 ms per decode (fused, 1024) but re-tunes 0.7-2 s at every new resolution.
+_CUDNN_BENCHMARK_DENY_VAES: frozenset[str] = frozenset({"AutoencoderKLQwenImage21"})
+
+
+def _cudnn_benchmark_pointless(pipe: Any) -> bool:
+ if _denoiser_unet(pipe) is not None:
+ return False
+ return type(getattr(pipe, "vae", None)).__name__ in _CUDNN_BENCHMARK_DENY_VAES
+
+
+def _vae_channels_last(
+ pipe: Any,
+ logger: Any,
+ *,
+ fused: bool = False,
+) -> bool:
vae = getattr(pipe, "vae", None)
if vae is None or not hasattr(vae, "to"):
return False
+ if type(vae).__name__ in _VAE_CHANNELS_LAST_DENY and not fused:
+ return False
try:
import torch
vae.to(memory_format = torch.channels_last)
@@ -759,6 +782,7 @@ def _compile_repeated_blocks(
unet = _denoiser_unet(pipe) if not dits else None
if not dits and unet is None:
return False
+ _install_inductor_backports(logger)
# Before the stream-merge probe below, which reads the same block names.
for transformer in dits:
try:
@@ -891,13 +915,23 @@ def _compile_repeated_blocks(
return engaged
+def _install_inductor_backports(logger: Any) -> bool:
+ """Probe-gated backport of torch 2.14's CantSplit divisibility proof for 2.12 / 2.13. Never fails a load."""
+ try:
+ from . import diffusion_inductor_backports
+ return diffusion_inductor_backports.install(logger)
+ except Exception as exc: # noqa: BLE001 - optimisation only
+ _warn(logger, "inductor backports", exc)
+ return False
+
+
def compile_dynamic(transformer: Any, dynamic: Optional[bool]) -> Optional[bool]:
"""The ``dynamic`` a DiT is actually compiled with, so compile-cache fingerprints key on the same value.
dynamic=True makes even the constant segment starts symbolic, and on Qwen-Image-2.1 the attention output cat
- (text + target, length s87 - s89) then fuses into torchao's per-row activation-quant reduction, which inductor
- cannot split (CantSplit, every render failed). Automatic dynamic (None) compiles the first shapes static and only
- generalises what actually varies: stable after ~3 recompiles, same numerics."""
+ (text + target, length s87 - s89) then fuses into torchao's per-row activation-quant reduction (CantSplit on torch
+ 2.12 / 2.13 without ``diffusion_inductor_backports``). Even with the backport it is slower than automatic dynamic
+ (None), which does not recompile across prompt lengths or resolutions."""
# Static kernels for video DiTs: dynamic shapes made LTX-2.3's QK-norm + RoPE kernels ~3x slower.
if transformer is not None and getattr(transformer, "_unsloth_compile_static", False):
return False
@@ -1299,6 +1333,7 @@ def _compile_vae_decode(
return True
if getattr(vae, "_unsloth_compile_decode_error", None):
return False
+ _install_inductor_backports(logger)
try:
import torch
diff --git a/studio/backend/tests/test_diffusion_inductor_backports.py b/studio/backend/tests/test_diffusion_inductor_backports.py
new file mode 100644
index 00000000000..555a76dd06e
--- /dev/null
+++ b/studio/backend/tests/test_diffusion_inductor_backports.py
@@ -0,0 +1,163 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
+
+"""Tests for ``diffusion_inductor_backports.py`` (the torch 2.14 symbolic divisibility backport). CPU only."""
+
+from __future__ import annotations
+
+import pytest
+
+from core.inference import diffusion_inductor_backports as bp
+
+torch = pytest.importorskip("torch")
+sympy = pytest.importorskip("sympy")
+sizevars = pytest.importorskip("torch._inductor.sizevars")
+
+
[email protected](autouse = True)
+def _restore(monkeypatch):
+ monkeypatch.delenv(bp.BACKPORTS_ENV, raising = False)
+ bp.uninstall()
+ # Exercise the patched method on every torch, including ones whose stock check already proves the probe.
+ monkeypatch.setattr(bp, "_stock_proves", lambda _cls: False)
+ yield
+ bp.uninstall()
+
+
+def _syms():
+ return sympy.symbols("s87 s89", integer = True, positive = True)
+
+
+def _proves(num, den) -> bool:
+ return bool(sizevars.SizeVarAllocator().statically_known_multiple_of(num, den))
+
+
+def test_proves_the_qwen21_and_flux_cantsplit_expressions():
+ a, b = _syms()
+ bp.install()
+ # Qwen-Image-2.1 torchao (text + target attention output) and FLUX.1 single-block (attn + mlp cat) shapes.
+ assert _proves(4096 * a - 4096 * b, a - b)
+ assert _proves(15360 * a + 15360 * b, a + b)
+ assert _proves(4096 * a - 4096 * b, b - a) # sign of the group does not matter for divisibility
+
+
+def test_never_proves_a_non_multiple():
+ a, b = _syms()
+ bp.install()
+ assert not _proves(4096 * a - 4095 * b, a - b)
+ assert not _proves(4096 * a, a + b)
+ assert not _proves(a * b + 1, a)
+ # sympy.gcd is taken over the rationals: gcd((a + b) / 2, a + b) == a + b, yet (a + b) / 2 is no multiple.
+ assert not _proves((a + b) / 2, a + b)
+ assert not _proves(sympy.Rational(8, 3) * a * b, a * b)
+
+
+def test_integer_denominators_keep_the_stock_answer():
+ a, _b = _syms()
+ bp.install()
+ assert _proves(4096 * a, 64)
+ assert not _proves(4096 * a + 3, 64)
+
+
+def test_install_is_idempotent_and_uninstall_restores():
+ stock = sizevars.SizeVarAllocator.statically_known_multiple_of
+ first = bp.install()
+ assert first is True
+ patched = sizevars.SizeVarAllocator.statically_known_multiple_of
+ assert bp.install() is first
+ assert sizevars.SizeVarAllocator.statically_known_multiple_of is patched
+ bp.uninstall()
+ assert sizevars.SizeVarAllocator.statically_known_multiple_of is stock
+ assert not bp.is_installed()
+
+
+def test_kill_switch(monkeypatch):
+ monkeypatch.setenv(bp.BACKPORTS_ENV, "0")
+ stock = sizevars.SizeVarAllocator.statically_known_multiple_of
+ assert bp.install() is False
+ assert sizevars.SizeVarAllocator.statically_known_multiple_of is stock
+
+
+def test_torch_that_already_proves_it_is_left_alone(monkeypatch):
+ monkeypatch.setattr(bp, "_stock_proves", lambda _cls: True)
+ stock = sizevars.SizeVarAllocator.statically_known_multiple_of
+ assert bp.install() is False
+ assert sizevars.SizeVarAllocator.statically_known_multiple_of is stock
+ assert not bp.is_installed()
+
+
+def test_stock_probe_matches_torch_version(monkeypatch):
+ """Regressed in 2.12.0, fixed in 2.14.0; the probe, not the version, decides."""
+ monkeypatch.undo()
+ major_minor = tuple(int(x) for x in torch.__version__.split("+")[0].split(".")[:2])
+ proves = bp._stock_proves(sizevars.SizeVarAllocator)
+ if (2, 12) <= major_minor < (2, 14):
+ assert not proves
+ else:
+ assert proves
+
+
+def test_a_failing_proof_returns_the_stock_answer(monkeypatch):
+ a, b = _syms()
+ stock = _proves(4096 * a - 4096 * b, a - b)
+ bp.install()
+
+ def boom(*_a, **_k):
+ raise RuntimeError("sympy exploded")
+
+ monkeypatch.setattr(bp, "_gcd_proves_multiple", boom)
+ assert _proves(4096 * a - 4096 * b, a - b) == stock
+
+
+def test_regional_compile_and_vae_compile_install_the_backport(monkeypatch):
+ import types
+
+ from core.inference import diffusion_speed as ds_mod
+
+ calls: list = []
+ monkeypatch.setattr(bp, "install", lambda logger = None: calls.append("install") or True)
+
+ class _Dit(torch.nn.Module):
+ _repeated_blocks: list = []
+
+ def compile_repeated_blocks(self, **kwargs):
+ calls.append(("compile", kwargs.get("dynamic")))
+
+ pipe = types.SimpleNamespace(transformer = _Dit())
+ monkeypatch.setattr(ds_mod, "_inductor_config", lambda: None)
+ monkeypatch.setattr(ds_mod, "guard_compiled_blocks", lambda transformer, logger = None: 0)
+ assert ds_mod._compile_repeated_blocks(pipe, None) is True
+ # Installed BEFORE the (lazy) compile is requested, so the first forward already lowers with it.
+ compiles = [c for c in calls if c != "install"]
+ assert calls[0] == "install" and len(compiles) == 1 and compiles[0][0] == "compile"
+
+ calls.clear()
+ monkeypatch.setattr(torch, "compile", lambda fn, **kw: fn)
+ vae = types.SimpleNamespace(decode = lambda x: x)
+ assert ds_mod._compile_vae_decode(types.SimpleNamespace(vae = vae), None) is True
+ assert calls and set(calls) == {"install"}
+
+
+def test_a_broken_backport_never_fails_the_compile(monkeypatch):
+ import types
+
+ from core.inference import diffusion_speed as ds_mod
+
+ def boom(logger = None):
+ raise ImportError("no inductor")
+
+ monkeypatch.setattr(bp, "install", boom)
+ assert ds_mod._install_inductor_backports(None) is False
+ monkeypatch.setattr(torch, "compile", lambda fn, **kw: fn)
+ vae = types.SimpleNamespace(decode = lambda x: x)
+ assert ds_mod._compile_vae_decode(types.SimpleNamespace(vae = vae), None) is True
+
+
+def test_proof_available_on_a_patched_or_fixed_torch(monkeypatch):
+ assert bp.proof_available() is True
+ bp.uninstall()
+ monkeypatch.setattr(bp, "_stock_proves", lambda _cls: True)
+ assert bp.proof_available() is True
+ bp.uninstall()
+ monkeypatch.setenv(bp.BACKPORTS_ENV, "0")
+ assert bp.proof_available() is False
diff --git a/studio/backend/tests/test_diffusion_qwenimage21_vae_layout.py b/studio/backend/tests/test_diffusion_qwenimage21_vae_layout.py
new file mode 100644
index 00000000000..2c261d107f7
--- /dev/null
+++ b/studio/backend/tests/test_diffusion_qwenimage21_vae_layout.py
@@ -0,0 +1,100 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
+
+"""AutoencoderKLQwenImage21 stays contiguous unless the fused VAE norms install; other VAEs unchanged."""
+
+from __future__ import annotations
+
+import types
+
+import pytest
+
+from core.inference import diffusion_speed as ds_mod
+
+torch = pytest.importorskip("torch")
+
+
+def _vae(name: str):
+ calls: list = []
+
+ def to(self, memory_format = None):
+ calls.append(memory_format)
+ return self
+
+ cls = type(name, (), {"to": to})
+ return cls(), calls
+
+
+def test_qwen_image_21_vae_stays_contiguous():
+ vae, calls = _vae("AutoencoderKLQwenImage21")
+ assert ds_mod._vae_channels_last(types.SimpleNamespace(vae = vae), None) is False
+ assert calls == []
+
+
+def test_qwen_image_21_vae_goes_channels_last_when_the_fused_norms_install():
+ vae, calls = _vae("AutoencoderKLQwenImage21")
+ assert ds_mod._vae_channels_last(types.SimpleNamespace(vae = vae), None, fused = True) is True
+ assert calls == [torch.channels_last]
+
+
[email protected]("planned", [True, False])
+def test_apply_speed_optims_layout_follows_the_fused_planner(monkeypatch, planned):
+ vae, calls = _vae("AutoencoderKLQwenImage21")
+ monkeypatch.setattr(ds_mod, "_fused_vae_planned", lambda pipe: planned)
+ monkeypatch.setattr(ds_mod, "_install_fused_vae", lambda pipe, logger: planned)
+ monkeypatch.setattr(ds_mod, "_enable_cudnn_benchmark", lambda logger: True)
+ monkeypatch.setattr(ds_mod, "_compile_repeated_blocks", lambda *a, **k: False)
+ monkeypatch.setattr(ds_mod, "compile_eligible", lambda *a, **k: False)
+ target = types.SimpleNamespace(device = "cuda", dtype = torch.bfloat16)
+ family = types.SimpleNamespace(name = "qwen-image-2.1", supports_torch_compile = True)
+ applied = ds_mod.apply_speed_optims(
+ types.SimpleNamespace(vae = vae), target, is_gguf = False, family = family, speed_mode = "default"
+ )
+ assert applied["channels_last"] is planned
+ assert calls == ([torch.channels_last] if planned else [])
+
+
[email protected]("name", ["AutoencoderKL", "AutoencoderKLQwenImage", "AutoencoderKLWan"])
+def test_other_vaes_still_go_channels_last(name):
+ vae, calls = _vae(name)
+ assert ds_mod._vae_channels_last(types.SimpleNamespace(vae = vae), None) is True
+ assert calls == [torch.channels_last]
+
+
+def test_real_qwen_image_21_vae_class_name_matches_the_deny_list():
+ mod = pytest.importorskip("diffusers.models.autoencoders.autoencoder_kl_qwenimage21")
+ assert mod.AutoencoderKLQwenImage21.__name__ in ds_mod._VAE_CHANNELS_LAST_DENY
+
+
+def test_cudnn_benchmark_is_skipped_for_a_qwen_image_21_pipeline():
+ vae, _calls = _vae("AutoencoderKLQwenImage21")
+ assert ds_mod._cudnn_benchmark_pointless(types.SimpleNamespace(vae = vae)) is True
+
+
[email protected]("name", ["AutoencoderKL", "AutoencoderKLQwenImage", "AutoencoderKLWan"])
+def test_cudnn_benchmark_kept_for_other_vaes(name):
+ vae, _calls = _vae(name)
+ assert ds_mod._cudnn_benchmark_pointless(types.SimpleNamespace(vae = vae)) is False
+
+
+def test_cudnn_benchmark_kept_when_a_unet_denoises(monkeypatch):
+ vae, _calls = _vae("AutoencoderKLQwenImage21")
+ monkeypatch.setattr(ds_mod, "_denoiser_unet", lambda pipe: object())
+ assert ds_mod._cudnn_benchmark_pointless(types.SimpleNamespace(vae = vae)) is False
+
+
+def test_apply_speed_optims_leaves_cudnn_benchmark_alone_for_qwen_image_21(monkeypatch):
+ vae, _calls = _vae("AutoencoderKLQwenImage21")
+ pipe = types.SimpleNamespace(vae = vae)
+ enabled: list = []
+ monkeypatch.setattr(ds_mod, "_enable_cudnn_benchmark", lambda logger: enabled.append(1) or True)
+ monkeypatch.setattr(ds_mod, "_compile_repeated_blocks", lambda *a, **k: False)
+ monkeypatch.setattr(ds_mod, "compile_eligible", lambda *a, **k: False)
+ monkeypatch.setattr(ds_mod, "_fused_vae_planned", lambda pipe: False)
+ target = types.SimpleNamespace(device = "cuda", dtype = torch.bfloat16)
+ family = types.SimpleNamespace(name = "qwen-image-2.1", supports_torch_compile = True)
+ applied = ds_mod.apply_speed_optims(
+ pipe, target, is_gguf = False, family = family, speed_mode = "default"
+ )
+ assert applied["cudnn_benchmark"] is False and enabled == []
+ assert applied["channels_last"] is False
diff --git a/studio/backend/tests/test_middleware.py b/studio/backend/tests/test_middleware.py
index 32c992492da..9f5e877146e 100644
--- a/studio/backend/tests/test_middleware.py
+++ b/studio/backend/tests/test_middleware.py
@@ -1467,7 +1467,6 @@ class TestRemoteAccessCORS:
exposed = response.headers["access-control-expose-headers"].lower().split(",")
assert "x-typesafe-request-id" in {header.strip() for header in exposed}
-
TUNNEL = "https://demo-abc.trycloudflare.com"
@staticmethod