Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
Studio: make compile knobs reach the render thread on torch 2.12+ (#12075)
Daniel Hana735bccb538e
diff --git a/studio/backend/core/inference/diffusion_compile_config.py b/studio/backend/core/inference/diffusion_compile_config.pynew file mode 100644index 00000000000..4c22497ac4a--- /dev/null+++ b/studio/backend/core/inference/diffusion_compile_config.py@@ -0,0 +1,95 @@+# SPDX-License-Identifier: AGPL-3.0-only+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0++"""torch 2.12+ keeps dynamo / inductor config writes in a ContextVar, so knobs set on the load thread are invisible on+the render thread that compiles. Knobs are recorded here and ``apply()`` re-writes them into the current context."""++from __future__ import annotations++import threading+from contextvars import ContextVar+from typing import Any++_LOCK = threading.Lock()+_KNOBS: dict[tuple[str, str], Any] = {}+_generation = 0+# ContextVar, not thread-local: a copy_context() run on an already-applied thread must re-apply.+_applied: ContextVar[int] = ContextVar("unsloth_compile_config_applied", default = -1)+++def _module(name: str) -> Any:+ try:+ import torch+ except Exception: # noqa: BLE001 - no torch -> nothing to configure+ return None+ obj: Any = torch+ for part in name.split(".")[1:]:+ obj = getattr(obj, part, None)+ if obj is None:+ return None+ return obj+++def _write(cfg: Any, attr: str, value: Any) -> None:+ # Write only on change: every write dirties the config and forces a rehash.+ try:+ if getattr(cfg, attr) != value:+ setattr(cfg, attr, value)+ except Exception: # noqa: BLE001 - a missing or read-only knob on this torch build is skipped+ pass+++def set_knob(module_name: str, attr: str, value: Any) -> bool:+ global _generation+ cfg = _module(module_name)+ if cfg is None or not hasattr(cfg, attr):+ return False+ with _LOCK:+ _KNOBS.pop((module_name, attr), None)+ _KNOBS[(module_name, attr)] = value+ _generation += 1+ _write(cfg, attr, value)+ return True+++def get_knob(+ module_name: str,+ attr: str,+ default: Any = None,+) -> Any:+ with _LOCK:+ if (module_name, attr) in _KNOBS:+ return _KNOBS[(module_name, attr)]+ cfg = _module(module_name)+ return getattr(cfg, attr, default) if cfg is not None else default+++def apply() -> None:+ generation = _generation+ if _applied.get() == generation:+ return+ with _LOCK:+ generation = _generation+ knobs = list(_KNOBS.items())+ for (module_name, attr), value in knobs:+ cfg = _module(module_name)+ if cfg is not None:+ _write(cfg, attr, value)+ _applied.set(generation)+++def is_recorded(module_name: str, attr: str) -> bool:+ with _LOCK:+ return (module_name, attr) in _KNOBS+++def recorded() -> dict[tuple[str, str], Any]:+ with _LOCK:+ return dict(_KNOBS)+++def _reset_for_tests() -> None:+ global _generation+ with _LOCK:+ _KNOBS.clear()+ _generation += 1diff --git a/studio/backend/core/inference/diffusion_render_thread.py b/studio/backend/core/inference/diffusion_render_thread.pyindex 6de37c30890..1f0f3e736c2 100644--- a/studio/backend/core/inference/diffusion_render_thread.py+++ b/studio/backend/core/inference/diffusion_render_thread.py@@ -44,8 +44,17 @@ def _executor(name: str) -> ThreadPoolExecutor:return ex+def _apply_compile_config() -> None:+ try:+ from . import diffusion_compile_config # noqa: PLC0415+ diffusion_compile_config.apply()+ except Exception: # noqa: BLE001 - never fail a render over a config write+ pass++def run(name: str, fn: Callable[[], Any]) -> Any:if threading.get_ident() in _RENDER_THREAD_IDS or not enabled():+ _apply_compile_config()return fn()import torch # noqa: PLC0415@@ -56,6 +65,7 @@ def run(name: str, fn: Callable[[], Any]) -> Any:def call() -> Any:torch.cuda.set_device(device)+ _apply_compile_config()if inference:with torch.inference_mode():return fn()diff --git a/studio/backend/core/inference/diffusion_speed.py b/studio/backend/core/inference/diffusion_speed.pyindex b9cf81ae026..09ab6645b25 100644--- a/studio/backend/core/inference/diffusion_speed.py+++ b/studio/backend/core/inference/diffusion_speed.py@@ -42,6 +42,7 @@ from contextlib import contextmanagerfrom functools import lru_cachefrom typing import Any, Iterator, Optional+from . import diffusion_compile_config as compile_configfrom . import diffusion_gguf_compile as gguf_compileSPEED_OFF = "off"@@ -60,6 +61,9 @@ _INDUCTOR_FLAGS = (("fx_graph_cache", "inductor_fx_graph_cache"),)_INDUCTOR_TRITON_FLAGS = (("unique_kernel_names", "inductor_triton_unique_kernel_names"),)+_DYNAMO_MODULE = "torch._dynamo.config"+_INDUCTOR_MODULE = "torch._inductor.config"+_INDUCTOR_TRITON_MODULE = "torch._inductor.config.triton"# allow_fp16_accumulation has one owner: writes during an open scope go to the recorded process value, so closing@@ -173,14 +177,15 @@ def snapshot_backend_flags() -> Optional[dict]:state["cudnn_benchmark"] = _read_cudnn_benchmark(cudnn)inductor_cfg = _inductor_config()if inductor_cfg is not None:+ # Process-wide value: on torch 2.12+ a knob set on another thread is invisible here.for attr, key in _INDUCTOR_FLAGS:if hasattr(inductor_cfg, attr):- state[key] = bool(getattr(inductor_cfg, attr))+ state[key] = bool(compile_config.get_knob(_INDUCTOR_MODULE, attr))triton_cfg = getattr(inductor_cfg, "triton", None)if triton_cfg is not None:for attr, key in _INDUCTOR_TRITON_FLAGS:if hasattr(triton_cfg, attr):- state[key] = bool(getattr(triton_cfg, attr))+ state[key] = bool(compile_config.get_knob(_INDUCTOR_TRITON_MODULE, attr))getter = getattr(torch, "get_float32_matmul_precision", None)if callable(getter):try:@@ -232,12 +237,19 @@ def restore_backend_flags(state: Optional[dict]) -> None:_write_cudnn_benchmark(cudnn, state["cudnn_benchmark"])except Exception: # noqa: BLE001 - best-effort per-flag restorepass++ def _set_knob(obj: Any, module_name: str, attr: str, key: str) -> None:+ if key in state and compile_config.is_recorded(module_name, attr):+ compile_config.set_knob(module_name, attr, state[key])+ else:+ _set(obj, attr, key)+inductor_cfg = _inductor_config()for attr, key in _INDUCTOR_FLAGS:- _set(inductor_cfg, attr, key)+ _set_knob(inductor_cfg, _INDUCTOR_MODULE, attr, key)triton_cfg = getattr(inductor_cfg, "triton", None) if inductor_cfg is not None else Nonefor attr, key in _INDUCTOR_TRITON_FLAGS:- _set(triton_cfg, attr, key)+ _set_knob(triton_cfg, _INDUCTOR_TRITON_MODULE, attr, key)def _inductor_config() -> Any:@@ -766,6 +778,7 @@ def _compile_repeated_blocks(try:import torch+ # Via compile_config: the lazy compile runs on the render thread, which on torch 2.12+ cannot see these writes.# Heterogeneous-block DiTs (Z-Image needs ~11 graphs) exceed dynamo's default recompile_limit of 8, where a# resident load hard-errors under fullgraph, so raise it to 64. NOT force_parameter_static_shapes=False: no win# and ~6x slower.@@ -773,14 +786,15 @@ def _compile_repeated_blocks(if dynamo_cfg is not None:for _limit_attr in ("recompile_limit", "cache_size_limit"): # name varies by torch verif hasattr(dynamo_cfg, _limit_attr):- setattr(dynamo_cfg, _limit_attr, max(getattr(dynamo_cfg, _limit_attr) or 0, 64))+ current = compile_config.get_knob(_DYNAMO_MODULE, _limit_attr) or 0+ compile_config.set_knob(_DYNAMO_MODULE, _limit_attr, max(current, 64))# Match eager intermediate rounding in inductor's fused pointwise kernels: they keep chains in fp32 where eager# materialises bf16 between ops, a per-forward delta a multi-step denoise amplifies. Measured LPIPS vs eager:# Qwen-Image 0.019 to 0.006, HunyuanVideo-1.5-720p 0.221 to 0.052, at ~zero cost. Process-global, so# snapshot_backend_flags restores it on unload.inductor_cfg = _inductor_config()if inductor_cfg is not None and hasattr(inductor_cfg, "emulate_precision_casts"):- inductor_cfg.emulate_precision_casts = True+ compile_config.set_knob(_INDUCTOR_MODULE, "emulate_precision_casts", True)except Exception as exc: # noqa: BLE001 - optimisation only_warn(logger, "compile_repeated_blocks", exc)return False@@ -944,6 +958,7 @@ class _CompileGuard:def guarded(*args: Any, **kwargs: Any) -> Any:if guard.error is None:+ compile_config.apply()try:return compiled(*args, **kwargs)except Exception as exc: # noqa: BLE001 - reraised unless a compile-time failurediff --git a/studio/backend/tests/test_diffusion_compile_config.py b/studio/backend/tests/test_diffusion_compile_config.pynew file mode 100644index 00000000000..1df7b7b1f3b--- /dev/null+++ b/studio/backend/tests/test_diffusion_compile_config.py@@ -0,0 +1,160 @@+# SPDX-License-Identifier: AGPL-3.0-only+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0++"""Compile knobs set on the load thread must reach the thread that compiles (torch 2.12+ per-context config)."""++from __future__ import annotations++import sys+import threading+import types++import pytest++torch = pytest.importorskip("torch")+import torch._dynamo.config # noqa: E402+import torch._inductor.config # noqa: E402++from core.inference import diffusion_compile_config as cc # noqa: E402+from core.inference import diffusion_render_thread as rt # noqa: E402+from core.inference import diffusion_speed as ds # noqa: E402++_DYNAMO = torch._dynamo.config+_INDUCTOR = torch._inductor.config+_LIMIT = "recompile_limit" if hasattr(_DYNAMO, "recompile_limit") else "cache_size_limit"++[email protected](autouse = True)+def _clean_knobs():+ before = (getattr(_DYNAMO, _LIMIT), _INDUCTOR.emulate_precision_casts)+ cc._reset_for_tests()+ yield+ cc._reset_for_tests()+ setattr(_DYNAMO, _LIMIT, before[0])+ _INDUCTOR.emulate_precision_casts = before[1]+++def armed(monkeypatch):+ monkeypatch.setattr(rt, "enabled", lambda: True)+ monkeypatch.setattr(torch.cuda, "current_device", lambda: 0)+ monkeypatch.setattr(torch.cuda, "set_device", lambda d: None)+++def _on_thread(fn):+ out = {}++ def body():+ try:+ out["value"] = fn()+ except BaseException as exc: # noqa: BLE001 - surfaced to the test+ out["error"] = exc++ t = threading.Thread(target = body)+ t.start()+ t.join()+ if "error" in out:+ raise out["error"]+ return out["value"]+++def _read():+ return getattr(_DYNAMO, _LIMIT), bool(_INDUCTOR.emulate_precision_casts)+++class _Transformer:+ _repeated_blocks = ()++ def compile_repeated_blocks(self, **kwargs):+ self.kwargs = kwargs++ def named_modules(self):+ return iter(())++ def modules(self):+ return iter(())++ def parameters(self):+ return iter(())+++def _load_compile():+ pipe = types.SimpleNamespace(transformer = _Transformer())+ assert ds._compile_repeated_blocks(pipe, None) is True+++def _baseline():+ return _on_thread(_read)+++def test_load_thread_knobs_reach_the_render_thread(armed):+ default_limit, _ = _baseline()+ _on_thread(_load_compile)+ seen = _on_thread(lambda: rt.run("cfg1", _read))+ assert seen == (max(default_limit, 64), True)+++def test_render_inline_path_applies_too(monkeypatch):+ monkeypatch.setattr(rt, "enabled", lambda: False)+ default_limit, _ = _baseline()+ _on_thread(_load_compile)+ assert _on_thread(lambda: rt.run("cfg2", _read)) == (max(default_limit, 64), True)+++def test_guarded_compiled_block_applies_the_knobs():+ default_limit, _ = _baseline()+ _on_thread(_load_compile)+ owner = types.SimpleNamespace()+ guard = ds._CompileGuard(None)+ guarded = guard.wrap(lambda: _read(), lambda: ("eager",), owner)+ assert _on_thread(guarded) == (max(default_limit, 64), True)+++def test_unload_restore_reaches_the_render_thread(armed):+ snap = _on_thread(ds.snapshot_backend_flags)+ assert snap["inductor_emulate_precision_casts"] is False+ _on_thread(_load_compile)+ assert _on_thread(lambda: rt.run("cfg3", _read))[1] is True+ _on_thread(lambda: ds.restore_backend_flags(snap))+ limit, emulate = _on_thread(lambda: rt.run("cfg3", _read))+ assert emulate is False+ # recompile_limit is not in the unload snapshot, so it stays raised.+ assert limit >= 64+++def test_snapshot_reads_the_process_value_not_this_threads_view():+ _on_thread(_load_compile)+ assert _on_thread(ds.snapshot_backend_flags)["inductor_emulate_precision_casts"] is True+++def test_apply_is_idempotent_per_context(monkeypatch):+ cc.set_knob("torch._inductor.config", "emulate_precision_casts", True)+ writes = []+ real_write = cc._write+ monkeypatch.setattr(cc, "_write", lambda *a: (writes.append(a), real_write(*a)))++ def twice():+ cc.apply()+ cc.apply()+ return bool(_INDUCTOR.emulate_precision_casts)++ assert _on_thread(twice) is True+ assert len(writes) == 1+++def test_missing_knob_is_not_recorded():+ assert cc.set_knob("torch._inductor.config", "no_such_knob_anywhere", True) is False+ assert not cc.is_recorded("torch._inductor.config", "no_such_knob_anywhere")+ assert cc.set_knob("torch._no_such_module.config", "x", 1) is False+ cc.apply()+++def test_process_global_config_without_contextvar(monkeypatch):+ fake_cfg = types.SimpleNamespace(emulate_precision_casts = False)+ fake_torch = types.SimpleNamespace(_inductor = types.SimpleNamespace(config = fake_cfg))+ monkeypatch.setitem(sys.modules, "torch", fake_torch)+ assert cc.set_knob("torch._inductor.config", "emulate_precision_casts", True) is True+ assert fake_cfg.emulate_precision_casts is True+ assert _on_thread(lambda: (cc.apply(), fake_cfg.emulate_precision_casts)[1]) is True+ fake_cfg.emulate_precision_casts = False+ assert _on_thread(lambda: (cc.apply(), fake_cfg.emulate_precision_casts)[1]) is Truediff --git a/studio/backend/tests/test_diffusion_speed.py b/studio/backend/tests/test_diffusion_speed.pyindex 4784cd6e9d6..67048d863f9 100644--- a/studio/backend/tests/test_diffusion_speed.py+++ b/studio/backend/tests/test_diffusion_speed.py@@ -14,6 +14,7 @@ import typesimport pytest+from core.inference import diffusion_compile_config as compile_configfrom core.inference import diffusion_speed as ds_modfrom core.inference.diffusion_speed import (SPEED_DEFAULT,@@ -60,6 +61,13 @@ def _family(*, compile_ok = True):return types.SimpleNamespace(supports_torch_compile = compile_ok)[email protected](autouse = True)+def _fresh_compile_knobs():+ compile_config._reset_for_tests()+ yield+ compile_config._reset_for_tests()++@pytest.fixture(autouse = True)def _compile_runtime_independent_of_the_host(monkeypatch):"""Keep these tests off the HOST's toolchain, which is what "hermetic" above claims.