Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

Studio: make compile knobs reach the render thread on torch 2.12+ (#12075)

Daniel Hana735bccb538e
diff --git a/studio/backend/core/inference/diffusion_compile_config.py b/studio/backend/core/inference/diffusion_compile_config.py
new file mode 100644
index 00000000000..4c22497ac4a
--- /dev/null
+++ b/studio/backend/core/inference/diffusion_compile_config.py
@@ -0,0 +1,95 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
+
+"""torch 2.12+ keeps dynamo / inductor config writes in a ContextVar, so knobs set on the load thread are invisible on
+the render thread that compiles. Knobs are recorded here and ``apply()`` re-writes them into the current context."""
+
+from __future__ import annotations
+
+import threading
+from contextvars import ContextVar
+from typing import Any
+
+_LOCK = threading.Lock()
+_KNOBS: dict[tuple[str, str], Any] = {}
+_generation = 0
+# ContextVar, not thread-local: a copy_context() run on an already-applied thread must re-apply.
+_applied: ContextVar[int] = ContextVar("unsloth_compile_config_applied", default = -1)
+
+
+def _module(name: str) -> Any:
+ try:
+ import torch
+ except Exception: # noqa: BLE001 - no torch -> nothing to configure
+ return None
+ obj: Any = torch
+ for part in name.split(".")[1:]:
+ obj = getattr(obj, part, None)
+ if obj is None:
+ return None
+ return obj
+
+
+def _write(cfg: Any, attr: str, value: Any) -> None:
+ # Write only on change: every write dirties the config and forces a rehash.
+ try:
+ if getattr(cfg, attr) != value:
+ setattr(cfg, attr, value)
+ except Exception: # noqa: BLE001 - a missing or read-only knob on this torch build is skipped
+ pass
+
+
+def set_knob(module_name: str, attr: str, value: Any) -> bool:
+ global _generation
+ cfg = _module(module_name)
+ if cfg is None or not hasattr(cfg, attr):
+ return False
+ with _LOCK:
+ _KNOBS.pop((module_name, attr), None)
+ _KNOBS[(module_name, attr)] = value
+ _generation += 1
+ _write(cfg, attr, value)
+ return True
+
+
+def get_knob(
+ module_name: str,
+ attr: str,
+ default: Any = None,
+) -> Any:
+ with _LOCK:
+ if (module_name, attr) in _KNOBS:
+ return _KNOBS[(module_name, attr)]
+ cfg = _module(module_name)
+ return getattr(cfg, attr, default) if cfg is not None else default
+
+
+def apply() -> None:
+ generation = _generation
+ if _applied.get() == generation:
+ return
+ with _LOCK:
+ generation = _generation
+ knobs = list(_KNOBS.items())
+ for (module_name, attr), value in knobs:
+ cfg = _module(module_name)
+ if cfg is not None:
+ _write(cfg, attr, value)
+ _applied.set(generation)
+
+
+def is_recorded(module_name: str, attr: str) -> bool:
+ with _LOCK:
+ return (module_name, attr) in _KNOBS
+
+
+def recorded() -> dict[tuple[str, str], Any]:
+ with _LOCK:
+ return dict(_KNOBS)
+
+
+def _reset_for_tests() -> None:
+ global _generation
+ with _LOCK:
+ _KNOBS.clear()
+ _generation += 1
diff --git a/studio/backend/core/inference/diffusion_render_thread.py b/studio/backend/core/inference/diffusion_render_thread.py
index 6de37c30890..1f0f3e736c2 100644
--- a/studio/backend/core/inference/diffusion_render_thread.py
+++ b/studio/backend/core/inference/diffusion_render_thread.py
@@ -44,8 +44,17 @@ def _executor(name: str) -> ThreadPoolExecutor:
return ex
+def _apply_compile_config() -> None:
+ try:
+ from . import diffusion_compile_config # noqa: PLC0415
+ diffusion_compile_config.apply()
+ except Exception: # noqa: BLE001 - never fail a render over a config write
+ pass
+
+
def run(name: str, fn: Callable[[], Any]) -> Any:
if threading.get_ident() in _RENDER_THREAD_IDS or not enabled():
+ _apply_compile_config()
return fn()
import torch # noqa: PLC0415
@@ -56,6 +65,7 @@ def run(name: str, fn: Callable[[], Any]) -> Any:
def call() -> Any:
torch.cuda.set_device(device)
+ _apply_compile_config()
if inference:
with torch.inference_mode():
return fn()
diff --git a/studio/backend/core/inference/diffusion_speed.py b/studio/backend/core/inference/diffusion_speed.py
index b9cf81ae026..09ab6645b25 100644
--- a/studio/backend/core/inference/diffusion_speed.py
+++ b/studio/backend/core/inference/diffusion_speed.py
@@ -42,6 +42,7 @@ from contextlib import contextmanager
from functools import lru_cache
from typing import Any, Iterator, Optional
+from . import diffusion_compile_config as compile_config
from . import diffusion_gguf_compile as gguf_compile
SPEED_OFF = "off"
@@ -60,6 +61,9 @@ _INDUCTOR_FLAGS = (
("fx_graph_cache", "inductor_fx_graph_cache"),
)
_INDUCTOR_TRITON_FLAGS = (("unique_kernel_names", "inductor_triton_unique_kernel_names"),)
+_DYNAMO_MODULE = "torch._dynamo.config"
+_INDUCTOR_MODULE = "torch._inductor.config"
+_INDUCTOR_TRITON_MODULE = "torch._inductor.config.triton"
# allow_fp16_accumulation has one owner: writes during an open scope go to the recorded process value, so closing
@@ -173,14 +177,15 @@ def snapshot_backend_flags() -> Optional[dict]:
state["cudnn_benchmark"] = _read_cudnn_benchmark(cudnn)
inductor_cfg = _inductor_config()
if inductor_cfg is not None:
+ # Process-wide value: on torch 2.12+ a knob set on another thread is invisible here.
for attr, key in _INDUCTOR_FLAGS:
if hasattr(inductor_cfg, attr):
- state[key] = bool(getattr(inductor_cfg, attr))
+ state[key] = bool(compile_config.get_knob(_INDUCTOR_MODULE, attr))
triton_cfg = getattr(inductor_cfg, "triton", None)
if triton_cfg is not None:
for attr, key in _INDUCTOR_TRITON_FLAGS:
if hasattr(triton_cfg, attr):
- state[key] = bool(getattr(triton_cfg, attr))
+ state[key] = bool(compile_config.get_knob(_INDUCTOR_TRITON_MODULE, attr))
getter = getattr(torch, "get_float32_matmul_precision", None)
if callable(getter):
try:
@@ -232,12 +237,19 @@ def restore_backend_flags(state: Optional[dict]) -> None:
_write_cudnn_benchmark(cudnn, state["cudnn_benchmark"])
except Exception: # noqa: BLE001 - best-effort per-flag restore
pass
+
+ def _set_knob(obj: Any, module_name: str, attr: str, key: str) -> None:
+ if key in state and compile_config.is_recorded(module_name, attr):
+ compile_config.set_knob(module_name, attr, state[key])
+ else:
+ _set(obj, attr, key)
+
inductor_cfg = _inductor_config()
for attr, key in _INDUCTOR_FLAGS:
- _set(inductor_cfg, attr, key)
+ _set_knob(inductor_cfg, _INDUCTOR_MODULE, attr, key)
triton_cfg = getattr(inductor_cfg, "triton", None) if inductor_cfg is not None else None
for attr, key in _INDUCTOR_TRITON_FLAGS:
- _set(triton_cfg, attr, key)
+ _set_knob(triton_cfg, _INDUCTOR_TRITON_MODULE, attr, key)
def _inductor_config() -> Any:
@@ -766,6 +778,7 @@ def _compile_repeated_blocks(
try:
import torch
+ # Via compile_config: the lazy compile runs on the render thread, which on torch 2.12+ cannot see these writes.
# Heterogeneous-block DiTs (Z-Image needs ~11 graphs) exceed dynamo's default recompile_limit of 8, where a
# resident load hard-errors under fullgraph, so raise it to 64. NOT force_parameter_static_shapes=False: no win
# and ~6x slower.
@@ -773,14 +786,15 @@ def _compile_repeated_blocks(
if dynamo_cfg is not None:
for _limit_attr in ("recompile_limit", "cache_size_limit"): # name varies by torch ver
if hasattr(dynamo_cfg, _limit_attr):
- setattr(dynamo_cfg, _limit_attr, max(getattr(dynamo_cfg, _limit_attr) or 0, 64))
+ current = compile_config.get_knob(_DYNAMO_MODULE, _limit_attr) or 0
+ compile_config.set_knob(_DYNAMO_MODULE, _limit_attr, max(current, 64))
# Match eager intermediate rounding in inductor's fused pointwise kernels: they keep chains in fp32 where eager
# materialises bf16 between ops, a per-forward delta a multi-step denoise amplifies. Measured LPIPS vs eager:
# Qwen-Image 0.019 to 0.006, HunyuanVideo-1.5-720p 0.221 to 0.052, at ~zero cost. Process-global, so
# snapshot_backend_flags restores it on unload.
inductor_cfg = _inductor_config()
if inductor_cfg is not None and hasattr(inductor_cfg, "emulate_precision_casts"):
- inductor_cfg.emulate_precision_casts = True
+ compile_config.set_knob(_INDUCTOR_MODULE, "emulate_precision_casts", True)
except Exception as exc: # noqa: BLE001 - optimisation only
_warn(logger, "compile_repeated_blocks", exc)
return False
@@ -944,6 +958,7 @@ class _CompileGuard:
def guarded(*args: Any, **kwargs: Any) -> Any:
if guard.error is None:
+ compile_config.apply()
try:
return compiled(*args, **kwargs)
except Exception as exc: # noqa: BLE001 - reraised unless a compile-time failure
diff --git a/studio/backend/tests/test_diffusion_compile_config.py b/studio/backend/tests/test_diffusion_compile_config.py
new file mode 100644
index 00000000000..1df7b7b1f3b
--- /dev/null
+++ b/studio/backend/tests/test_diffusion_compile_config.py
@@ -0,0 +1,160 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
+
+"""Compile knobs set on the load thread must reach the thread that compiles (torch 2.12+ per-context config)."""
+
+from __future__ import annotations
+
+import sys
+import threading
+import types
+
+import pytest
+
+torch = pytest.importorskip("torch")
+import torch._dynamo.config # noqa: E402
+import torch._inductor.config # noqa: E402
+
+from core.inference import diffusion_compile_config as cc # noqa: E402
+from core.inference import diffusion_render_thread as rt # noqa: E402
+from core.inference import diffusion_speed as ds # noqa: E402
+
+_DYNAMO = torch._dynamo.config
+_INDUCTOR = torch._inductor.config
+_LIMIT = "recompile_limit" if hasattr(_DYNAMO, "recompile_limit") else "cache_size_limit"
+
+
[email protected](autouse = True)
+def _clean_knobs():
+ before = (getattr(_DYNAMO, _LIMIT), _INDUCTOR.emulate_precision_casts)
+ cc._reset_for_tests()
+ yield
+ cc._reset_for_tests()
+ setattr(_DYNAMO, _LIMIT, before[0])
+ _INDUCTOR.emulate_precision_casts = before[1]
+
+
+def armed(monkeypatch):
+ monkeypatch.setattr(rt, "enabled", lambda: True)
+ monkeypatch.setattr(torch.cuda, "current_device", lambda: 0)
+ monkeypatch.setattr(torch.cuda, "set_device", lambda d: None)
+
+
+def _on_thread(fn):
+ out = {}
+
+ def body():
+ try:
+ out["value"] = fn()
+ except BaseException as exc: # noqa: BLE001 - surfaced to the test
+ out["error"] = exc
+
+ t = threading.Thread(target = body)
+ t.start()
+ t.join()
+ if "error" in out:
+ raise out["error"]
+ return out["value"]
+
+
+def _read():
+ return getattr(_DYNAMO, _LIMIT), bool(_INDUCTOR.emulate_precision_casts)
+
+
+class _Transformer:
+ _repeated_blocks = ()
+
+ def compile_repeated_blocks(self, **kwargs):
+ self.kwargs = kwargs
+
+ def named_modules(self):
+ return iter(())
+
+ def modules(self):
+ return iter(())
+
+ def parameters(self):
+ return iter(())
+
+
+def _load_compile():
+ pipe = types.SimpleNamespace(transformer = _Transformer())
+ assert ds._compile_repeated_blocks(pipe, None) is True
+
+
+def _baseline():
+ return _on_thread(_read)
+
+
+def test_load_thread_knobs_reach_the_render_thread(armed):
+ default_limit, _ = _baseline()
+ _on_thread(_load_compile)
+ seen = _on_thread(lambda: rt.run("cfg1", _read))
+ assert seen == (max(default_limit, 64), True)
+
+
+def test_render_inline_path_applies_too(monkeypatch):
+ monkeypatch.setattr(rt, "enabled", lambda: False)
+ default_limit, _ = _baseline()
+ _on_thread(_load_compile)
+ assert _on_thread(lambda: rt.run("cfg2", _read)) == (max(default_limit, 64), True)
+
+
+def test_guarded_compiled_block_applies_the_knobs():
+ default_limit, _ = _baseline()
+ _on_thread(_load_compile)
+ owner = types.SimpleNamespace()
+ guard = ds._CompileGuard(None)
+ guarded = guard.wrap(lambda: _read(), lambda: ("eager",), owner)
+ assert _on_thread(guarded) == (max(default_limit, 64), True)
+
+
+def test_unload_restore_reaches_the_render_thread(armed):
+ snap = _on_thread(ds.snapshot_backend_flags)
+ assert snap["inductor_emulate_precision_casts"] is False
+ _on_thread(_load_compile)
+ assert _on_thread(lambda: rt.run("cfg3", _read))[1] is True
+ _on_thread(lambda: ds.restore_backend_flags(snap))
+ limit, emulate = _on_thread(lambda: rt.run("cfg3", _read))
+ assert emulate is False
+ # recompile_limit is not in the unload snapshot, so it stays raised.
+ assert limit >= 64
+
+
+def test_snapshot_reads_the_process_value_not_this_threads_view():
+ _on_thread(_load_compile)
+ assert _on_thread(ds.snapshot_backend_flags)["inductor_emulate_precision_casts"] is True
+
+
+def test_apply_is_idempotent_per_context(monkeypatch):
+ cc.set_knob("torch._inductor.config", "emulate_precision_casts", True)
+ writes = []
+ real_write = cc._write
+ monkeypatch.setattr(cc, "_write", lambda *a: (writes.append(a), real_write(*a)))
+
+ def twice():
+ cc.apply()
+ cc.apply()
+ return bool(_INDUCTOR.emulate_precision_casts)
+
+ assert _on_thread(twice) is True
+ assert len(writes) == 1
+
+
+def test_missing_knob_is_not_recorded():
+ assert cc.set_knob("torch._inductor.config", "no_such_knob_anywhere", True) is False
+ assert not cc.is_recorded("torch._inductor.config", "no_such_knob_anywhere")
+ assert cc.set_knob("torch._no_such_module.config", "x", 1) is False
+ cc.apply()
+
+
+def test_process_global_config_without_contextvar(monkeypatch):
+ fake_cfg = types.SimpleNamespace(emulate_precision_casts = False)
+ fake_torch = types.SimpleNamespace(_inductor = types.SimpleNamespace(config = fake_cfg))
+ monkeypatch.setitem(sys.modules, "torch", fake_torch)
+ assert cc.set_knob("torch._inductor.config", "emulate_precision_casts", True) is True
+ assert fake_cfg.emulate_precision_casts is True
+ assert _on_thread(lambda: (cc.apply(), fake_cfg.emulate_precision_casts)[1]) is True
+ fake_cfg.emulate_precision_casts = False
+ assert _on_thread(lambda: (cc.apply(), fake_cfg.emulate_precision_casts)[1]) is True
diff --git a/studio/backend/tests/test_diffusion_speed.py b/studio/backend/tests/test_diffusion_speed.py
index 4784cd6e9d6..67048d863f9 100644
--- a/studio/backend/tests/test_diffusion_speed.py
+++ b/studio/backend/tests/test_diffusion_speed.py
@@ -14,6 +14,7 @@ import types
import pytest
+from core.inference import diffusion_compile_config as compile_config
from core.inference import diffusion_speed as ds_mod
from core.inference.diffusion_speed import (
SPEED_DEFAULT,
@@ -60,6 +61,13 @@ def _family(*, compile_ok = True):
return types.SimpleNamespace(supports_torch_compile = compile_ok)
[email protected](autouse = True)
+def _fresh_compile_knobs():
+ compile_config._reset_for_tests()
+ yield
+ compile_config._reset_for_tests()
+
+
@pytest.fixture(autouse = True)
def _compile_runtime_independent_of_the_host(monkeypatch):
"""Keep these tests off the HOST's toolchain, which is what "hermetic" above claims.