Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

Studio: the SDXL background compile runs only while no render runs (#16312)

Daniel Hand6950ad83510
diff --git a/studio/backend/core/inference/diffusion.py b/studio/backend/core/inference/diffusion.py
index 183266390dd..9ae98e92264 100644
--- a/studio/backend/core/inference/diffusion.py
+++ b/studio/backend/core/inference/diffusion.py
@@ -3756,6 +3756,25 @@ def _wait_or_pause_bg_compile(bg: Any, state: Any, loras: Any, cancel: Any) -> A
return None
pipe = getattr(state, "pipe", None)
if bg.yields() and not _has_active_lora(loras) and not getattr(pipe, "_unsloth_loras", ()):
+ if getattr(bg, "idle", False):
+ # Freeze only a resident denoiser: its eager forward moves no weight
+ freeze = getattr(state, "offload_policy", OFFLOAD_NONE) == OFFLOAD_NONE
+ waited, paused = bg.pause(cancel, budget_s = 0.0, freeze = freeze)
+ if paused:
+ logger.info(
+ "diffusion.bg_compile: render stopped the background compile in %.3f s (%s); it resumes "
+ "once no render runs",
+ waited,
+ "frozen mid-trace"
+ if getattr(bg, "_frozen", False)
+ else "parked at a block boundary"
+ if getattr(bg, "_parked", False)
+ else "stops at its next Python call"
+ if getattr(bg, "_freeze_ok", False)
+ else "stops at the next block boundary",
+ )
+ return bg
+ return None
budget = bg_compile.slice_seconds(getattr(pipe, "_unsloth_eager_render_s", None))
waited, paused = bg.pause(cancel, budget_s = budget)
if paused:
diff --git a/studio/backend/core/inference/diffusion_bg_compile.py b/studio/backend/core/inference/diffusion_bg_compile.py
index 962b3477a55..9b094e96abe 100644
--- a/studio/backend/core/inference/diffusion_bg_compile.py
+++ b/studio/backend/core/inference/diffusion_bg_compile.py
@@ -13,8 +13,10 @@ import hashlib
import json
import os
import re
+import sys
import threading
import time
+from _thread import get_ident as _thread_ident
from pathlib import Path
from typing import Any, Callable, Optional
@@ -114,6 +116,362 @@ def slice_seconds(eager_render_s: Optional[float]) -> float:
return max(0.0, min(_SLICE_MAX_S, float(value)))
+IDLE_ENV = "UNSLOTH_DIFFUSION_BG_COMPILE_IDLE_ONLY"
+IDLE_GRACE_ENV = "UNSLOTH_DIFFUSION_BG_COMPILE_IDLE_S"
+# Resume delay after a render: covers sending the response and a back-to-back client's next request.
+_IDLE_GRACE_S = 1.0
+_IDLE_GRACE_MAX_S = 60.0
+# The compile thread's innermost frame unchanged this long = blocked without the GIL: the render stops waiting.
+_FREEZE_QUIET_S = 0.02
+_FREEZE_POLL_S = 0.002
+
+
+def _innermost(ident: Optional[int]) -> Optional[tuple]:
+ frame = sys._current_frames().get(ident) if ident is not None else None
+ if frame is None:
+ return None
+ return (id(frame.f_code), frame.f_lasti, id(frame))
+
+
+def idle_only_enabled() -> bool:
+ """Per-block compiles run only while no render runs (default). ``=0`` restores the slice a render let run."""
+ raw = (os.environ.get(IDLE_ENV) or "").strip().lower()
+ return raw not in ("0", "false", "no", "off")
+
+
+def idle_grace_seconds() -> float:
+ raw = (os.environ.get(IDLE_GRACE_ENV) or "").strip()
+ try:
+ value = float(raw) if raw else _IDLE_GRACE_S
+ except ValueError:
+ value = _IDLE_GRACE_S
+ if value != value: # NaN
+ value = _IDLE_GRACE_S
+ return max(0.0, min(_IDLE_GRACE_MAX_S, value))
+
+
+# Mid-trace freeze via sys.monitoring PY_START (3.12+): park only the compile thread, only inside torch's compile,
+# never with a lock-taking module frame (or a Studio frame inside a lock block) on its stack.
+_FREEZE_TOOL_IDS = (4, 3)
+_FREEZE_TOOL: list = [None] # None = not tried, False = unavailable, else the tool id
+_FREEZE_TARGET: list = [None]
+# Job frozen mid-trace now (_FREEZE_TARGET is cleared by then): what _CompileLockProxy thaws.
+_FROZEN_JOB: list = [None]
+_FREEZE_LOCK = threading.Lock()
+_FREEZE_MAX_DEPTH = 5000
+# Frames here may hold a lock a render (or the compile pool) takes: never park there.
+_FREEZE_DENY_ROOTS = frozenset(
+ {
+ "logging",
+ "importlib",
+ "_frozen_importlib",
+ "_frozen_importlib_external",
+ "zipimport",
+ "threading",
+ "_threading_local",
+ "concurrent",
+ "multiprocessing",
+ "queue",
+ "subprocess",
+ "selectors",
+ "asyncio",
+ "socket",
+ "ssl",
+ "warnings",
+ "tqdm",
+ "structlog",
+ "rich",
+ "filelock",
+ "weakref",
+ "_weakrefset",
+ "gc",
+ }
+)
+# (torch._inductor.async_compile takes no lock: it only hands kernels to the pool, whose locks compile_worker holds.)
+_FREEZE_DENY_MODULES = (
+ "torch._logging",
+ "torch._inductor.compile_worker",
+ "torch._inductor.autotune_process",
+ "torch.utils._filelock",
+ "core.inference.diffusion_compile_config",
+ __name__,
+)
+# Studio frames are refused only inside their own lock-holding code, decided once per code object from its source.
+_STUDIO_ROOTS = ("core.", "studio.")
+_LOCKISH = re.compile(r"lock|LOCK|_IDLE\b|_cv\b|hold_off_capture|held_work|capture_hold")
+# code -> True (bare acquire: whole function) or (first, last) line spans of lock-holding ``with`` blocks
+_LOCK_CODE: dict = {}
+
+
+def _lock_spans(code: Any) -> Any:
+ hit = _LOCK_CODE.get(code)
+ if hit is not None:
+ return hit
+ try:
+ import ast
+ import inspect
+ import textwrap
+
+ lines, first = inspect.getsourcelines(code)
+ tree = ast.parse(textwrap.dedent("".join(lines)))
+ spans = []
+ bare = False
+ for node in ast.walk(tree):
+ if isinstance(node, (ast.With, ast.AsyncWith)):
+ if any(_LOCKISH.search(ast.unparse(item.context_expr)) for item in node.items):
+ spans.append(
+ (node.lineno + first - 1, (node.end_lineno or node.lineno) + first - 1)
+ )
+ elif (
+ isinstance(node, ast.Call)
+ and isinstance(node.func, ast.Attribute)
+ and node.func.attr in ("acquire", "capture_hold")
+ ):
+ bare = True
+ elif (
+ isinstance(node, ast.Call)
+ and isinstance(node.func, ast.Name)
+ and node.func.id == "capture_hold"
+ ):
+ bare = True
+ hit = True if bare else tuple(spans)
+ except Exception: # noqa: BLE001 - no source: assume it may hold one
+ hit = True
+ _LOCK_CODE[code] = hit
+ return hit
+
+
+def _holds_lock(frame: Any) -> bool:
+ spans = _lock_spans(frame.f_code)
+ if spans is True:
+ return True
+ if not spans:
+ return False
+ line = frame.f_lineno or 0
+ return any(a <= line <= b for a, b in spans)
+
+
+# Park only inside the compile: outside it runs eager Studio / diffusers code whose locks no list can name.
+_FREEZE_INSIDE = ("torch._dynamo.", "torch._inductor.", "torch._functorch.")
+
+
+FREEZE_ENV = "UNSLOTH_DIFFUSION_BG_COMPILE_FREEZE"
+
+
+def freeze_enabled() -> bool:
+ """Idle-only and ``FREEZE_ENV`` not off and the interpreter can (``=0``: stop only at block boundaries)."""
+ raw = (os.environ.get(FREEZE_ENV) or "").strip().lower()
+ if raw in ("0", "false", "no", "off"):
+ return False
+ return idle_only_enabled() and _freeze_tool() is not None
+
+
+def _freeze_tool() -> Optional[int]:
+ tool = _FREEZE_TOOL[0]
+ if tool is not None:
+ return tool or None
+ with _FREEZE_LOCK:
+ if _FREEZE_TOOL[0] is not None:
+ return _FREEZE_TOOL[0] or None
+ _FREEZE_TOOL[0] = False
+ mon = getattr(sys, "monitoring", None)
+ if mon is None:
+ return None
+ try:
+ # dynamo would try to trace the callback when it fires between two compiled frames on the compile thread
+ from torch._dynamo.eval_frame import skip_code
+ for fn in (
+ _on_py_start,
+ _freeze_safe,
+ _freeze_refusal,
+ _frame_refusal,
+ _holds_lock,
+ _lock_spans,
+ _compile_lock_owned,
+ BackgroundCompile._maybe_freeze,
+ BackgroundCompile._park,
+ ):
+ skip_code(fn.__code__)
+ except Exception: # noqa: BLE001 - no skip hook: park at block boundaries only
+ return None
+ for tool_id in _FREEZE_TOOL_IDS:
+ try:
+ mon.use_tool_id(tool_id, "unsloth-diffusion-bg-compile")
+ except Exception: # noqa: BLE001 - id held by a profiler / debugger
+ continue
+ try:
+ mon.register_callback(tool_id, mon.events.PY_START, _on_py_start)
+ except Exception: # noqa: BLE001
+ with contextlib.suppress(Exception):
+ mon.free_tool_id(tool_id)
+ continue
+ _FREEZE_TOOL[0] = tool_id
+ return tool_id
+ return None
+
+
+def _set_freeze_events(on: bool) -> None:
+ tool = _FREEZE_TOOL[0]
+ if not tool:
+ return
+ mon = sys.monitoring
+ with contextlib.suppress(Exception):
+ mon.set_events(tool, mon.events.PY_START if on else 0)
+
+
+def _on_py_start(code: Any, offset: int) -> Any:
+ bg = _FREEZE_TARGET[0]
+ if bg is None or _thread_ident() != bg._tid:
+ return None
+ bg._maybe_freeze()
+ return None
+
+
+def _frame_refusal(frame: Any) -> Optional[str]:
+ """Why ``frame`` alone forbids a park (a module or function that may hold a lock a render takes), else None."""
+ name = frame.f_globals.get("__name__") or ""
+ if name.partition(".")[0] in _FREEZE_DENY_ROOTS:
+ return name
+ for deny in _FREEZE_DENY_MODULES:
+ if name == deny or name.startswith(deny + "."):
+ return name
+ if name.startswith(_STUDIO_ROOTS) and _holds_lock(frame):
+ return f"{name}:{frame.f_code.co_name}"
+ return None
+
+
+def _freeze_refusal(frame: Any, memo: Optional[dict] = None) -> Optional[str]:
+ """None when the compile thread may park at ``frame``, else why not. ``memo`` caches per-frame verdicts."""
+ chain = []
+ depth = 0
+ run_code = BackgroundCompile._run.__code__
+ refusal: Optional[str] = None
+ inside = False
+ f = frame
+ while True:
+ if f is None:
+ refusal = "not the compile thread"
+ break
+ if depth >= _FREEZE_MAX_DEPTH:
+ refusal = "stack depth"
+ break
+ if memo is not None:
+ hit = memo.get(id(f))
+ if hit is not None and hit[0] is f:
+ refusal, inside = hit[1], hit[2]
+ break
+ if f.f_code is run_code:
+ break
+ chain.append(f)
+ f = f.f_back
+ depth += 1
+ for g in reversed(chain):
+ if refusal is None:
+ refusal = _frame_refusal(g)
+ if not inside:
+ inside = (g.f_globals.get("__name__") or "").startswith(_FREEZE_INSIDE)
+ if memo is not None:
+ memo[id(g)] = (g, refusal, inside)
+ if refusal is None and not inside:
+ return "outside the compile"
+ return refusal
+
+
+def _stack_summary(frame: Any, limit: int) -> str:
+ where = []
+ while frame is not None and len(where) < limit:
+ where.append(f"{frame.f_globals.get('__name__', '?')}:{frame.f_code.co_name}")
+ frame = frame.f_back
+ return " < ".join(where)
+
+
+def _freeze_safe(frame: Any) -> bool:
+ return _freeze_refusal(frame) is None
+
+
+class _CompileLockProxy:
+ """dynamo's ``compile_lock``, unchanged, except that a thread blocked on it while the background compile is frozen
+ mid-trace (holding it) thaws that compile, so the wait ends when its block graph does instead of never."""
+
+ _unsloth_bg_proxy = True
+
+ def __init__(self, lock: Any) -> None:
+ self._lock = lock
+
+ def acquire(
+ self,
+ blocking: bool = True,
+ timeout: float = -1,
+ ) -> bool:
+ if self._lock.acquire(False):
+ return True
+ if not blocking:
+ return False
+ deadline = None if timeout is None or timeout < 0 else time.perf_counter() + timeout
+ with _WAITERS_LOCK:
+ _LOCK_WAITERS[0] += 1
+ try:
+ while True:
+ for bg in {
+ id(j): j for j in (_FROZEN_JOB[0], _FREEZE_TARGET[0]) if j is not None
+ }.values():
+ bg._thaw_for_lock()
+ if self._lock.acquire(True, 0.05):
+ return True
+ if deadline is not None and time.perf_counter() >= deadline:
+ return False
+ finally:
+ with _WAITERS_LOCK:
+ _LOCK_WAITERS[0] -= 1
+
+ def release(self) -> None:
+ self._lock.release()
+
+ def __enter__(self) -> bool:
+ return self.acquire()
+
+ def __exit__(self, *exc: Any) -> None:
+ self._lock.release()
+
+ def __getattr__(self, name: str) -> Any:
+ return getattr(self._lock, name)
+
+
+# Threads blocked on dynamo's compile_lock: the compile thread never freezes holding it while one waits.
+_LOCK_WAITERS: list = [0]
+_WAITERS_LOCK = threading.Lock()
+_COMPILE_LOCK: list = [None]
+
+
+def _compile_lock_owned() -> bool:
+ lock = _COMPILE_LOCK[0]
+ if lock is None:
+ return False
+ try:
+ return bool(lock._is_owned())
+ except Exception: # noqa: BLE001 - assume held: never freeze on it while a thread waits
+ return True
+
+
+def _install_compile_lock_proxy() -> bool:
+ try:
+ from torch._dynamo import convert_frame
+ except Exception: # noqa: BLE001
+ return False
+ lock = getattr(convert_frame, "compile_lock", None)
+ if lock is None:
+ return False
+ if getattr(lock, "_unsloth_bg_proxy", False):
+ _COMPILE_LOCK[0] = lock._lock
+ return True
+ try:
+ _COMPILE_LOCK[0] = lock
+ convert_frame.compile_lock = _CompileLockProxy(lock)
+ except Exception: # noqa: BLE001
+ return False
+ return True
+
+
def yield_point() -> None:
"""Called before a guarded compiled block runs. On a background compile thread with a pause requested, parks
here (between two block graphs, never inside a trace) until the render resumes it. Elsewhere a no-op."""
@@ -562,6 +920,27 @@ class BackgroundCompile:
self.graph_s: Optional[float] = None
self._last_yield: Optional[float] = None
self._owe_graph = False
+ # Idle-only: a pause stops the compile at once; it resumes idle_grace_seconds() after the last render.
+ self.idle = idle_only_enabled()
+ self._idle_grace_s = idle_grace_seconds()
+ self._release_at = 0.0
+ self._tid: Optional[int] = None
+ self._freeze_ok = False
+ self._frozen = False
+ self._thaw = False
+ self.freezes = 0
+ self.pause_wait_s = 0.0
+ self.thaws = 0
+ self._thaw_where: Optional[str] = None
+ self._refusals: dict = {}
+ # frame verdicts for _freeze_refusal; holds frames, so cleared on every resume
+ self._frame_memo: dict = {}
+ self._freezes_at_pause = 0
+ self._pause_t0 = 0.0
+ self._stopped_after: Optional[float] = None
+ self._stopped_how = ""
+ # A frozen compile holds dynamo's compile_lock: freeze only once the proxy that thaws it is in place.
+ self._lock_proxy = False
def yields(self) -> bool:
return bool(getattr(self.module, "_unsloth_bg_yield", False)) and yield_enabled()
@@ -571,14 +950,18 @@ class BackgroundCompile:
cancel: Any = None,
budget_s: float = 0.0,
poll_s: float = 0.05,
+ freeze: bool = False,
) -> tuple[float, bool]:
"""Let the compile run ``budget_s`` more seconds, then park it at its next yield point. Returns
``(seconds waited, paused)``; ``paused`` means a pause is held and the caller MUST ``resume()``. A compile
- that ends inside the budget returns ``paused=False`` (the render then runs compiled)."""
+ that ends inside the budget returns ``paused=False`` (the render then runs compiled). Idle-only ignores the
+ budget; ``freeze`` (resident denoiser) stops it at its next Python call inside the trace."""
t0 = time.perf_counter()
thread = self._thread
if thread is None or thread is threading.current_thread() or not thread.is_alive():
return 0.0, False
+ if self.idle:
+ return self._pause_now(thread, cancel, freeze, t0), True
# Ask one graph_s early: the pause lands after the block graph in flight.
budget = max(0.0, float(budget_s or 0.0))
lead = self.graph_s if self.graph_s is not None else budget
@@ -600,15 +983,185 @@ class BackgroundCompile:
self._cv.wait(poll_s)
return time.perf_counter() - t0, True
+ def _pause_now(self, thread: Any, cancel: Any, freeze: bool, t0: float) -> float:
+ with self._cv:
+ self._pause_depth += 1
+ self.pauses += 1
+ nested = self._pause_depth > 1
+ if not nested:
+ self._freeze_ok = (
+ bool(freeze) and self._lock_proxy and self._tid is not None and freeze_enabled()
+ )
+ self._freezes_at_pause = self.freezes
+ self._refusals = {}
+ self._pause_t0 = t0
+ self._stopped_after = None
+ if self._freeze_ok and not self._parked:
+ _FREEZE_TARGET[0] = self
+ _set_freeze_events(True)
+ self._cv.notify_all()
+ if not nested:
+ # Frozen and blocked without the GIL: costs the render nothing, the armed freeze stops it on wake.
+ # Unfrozen (offloaded): it would move the render's weights, so wait for the block-boundary park.
+ last = _innermost(thread.ident)
+ quiet_since = time.perf_counter()
+ while thread.is_alive() and not self._parked:
+ if cancel is not None and cancel.is_set():
+ break
+ self._cv.wait(_FREEZE_POLL_S)
+ now = time.perf_counter()
+ here = _innermost(thread.ident)
+ if here != last:
+ last, quiet_since = here, now
+ elif self._freeze_ok and now - quiet_since >= _FREEZE_QUIET_S:
+ break
+ waited = time.perf_counter() - t0
+ self.pause_wait_s += waited
+ return waited
+
def resume(self) -> None:
with self._cv:
self._pause_depth = max(0, self._pause_depth - 1)
if not self._pause_depth:
self._release_epoch += 1
- # the next pause() waits for a fresh park, after the block graph this release owes
- self._parked = False
+ if self.idle:
+ self._frame_memo = {}
+ self._release_at = time.perf_counter() + self._idle_grace_s
+ if (
+ self._freeze_ok
+ and self.freezes == self._freezes_at_pause
+ and not self._parked
+ ):
+ self._log_unfrozen()
+ elif self.logger is not None and (self._stopped_after or 0.0) > 0.1:
+ self.logger.info(
+ "diffusion.bg_compile: the compile stopped %.2f s into the render (%s); refusals %s",
+ self._stopped_after,
+ self._stopped_how,
+ sorted(self._refusals.items(), key = lambda kv: -kv[1])[:6],
+ )
+ if self._thaw_where is not None and self.logger is not None:
+ self.logger.info(
+ "diffusion.bg_compile: a render needed dynamo's compile lock while the compile held it "
+ "(%d thaw(s)); first from %s",
+ self.thaws,
+ self._thaw_where,
+ )
+ self._thaw_where = None
+ self._thaw = False
+ if not self._frozen:
+ # never stopped: leave no event on for later renders
+ self._disarm_freeze()
+ else:
+ # the next pause() waits for a fresh park, after the block graph this release owes
+ self._parked = False
self._cv.notify_all()
+ def _log_unfrozen(self) -> None:
+ if self.logger is None:
+ return
+ try:
+ frame = sys._current_frames().get(self._tid)
+ where = []
+ while frame is not None and len(where) < 12:
+ where.append(f"{frame.f_globals.get('__name__', '?')}:{frame.f_code.co_name}")
+ frame = frame.f_back
+ top = sorted(self._refusals.items(), key = lambda kv: -kv[1])[:6]
+ others = []
+ names = {t.ident: t.name for t in threading.enumerate()}
+ for ident, f in sys._current_frames().items():
+ if ident in (self._tid, _thread_ident()):
+ continue
+ chain = []
+ while f is not None and len(chain) < 40:
+ chain.append(f"{f.f_globals.get('__name__', '?')}:{f.f_code.co_name}")
+ f = f.f_back
+ if any(c.startswith(_FREEZE_INSIDE) for c in chain):
+ others.append(f"[{names.get(ident, ident)}] " + " < ".join(chain[:8]))
+ self.logger.info(
+ "diffusion.bg_compile: the compile did not stop during this render (thawed %d, refusals %s); at %s; "
+ "other compile threads: %s",
+ self.thaws,
+ top,
+ " < ".join(where),
+ " || ".join(others) or "none",
+ )
+ except Exception: # noqa: BLE001 - diagnostic only
+ pass
+
+ def _disarm_freeze(self) -> None:
+ if _FREEZE_TARGET[0] is self:
+ _set_freeze_events(False)
+ _FREEZE_TARGET[0] = None
+
+ def _thaw_for_lock(self) -> None:
+ """Another thread waits on dynamo's compile_lock: a compile frozen holding it runs on to the end of its frame
+ (it releases the lock there) and freezes again at its next safe call."""
+ if _thread_ident() == self._tid:
+ return
+ with self._cv:
+ if self._frozen and not self._thaw:
+ self.thaws += 1
+ if self._thaw_where is None:
+ self._thaw_where = _stack_summary(sys._getframe(2), 14)
+ self._thaw = True
+ if self._pause_depth and self._freeze_ok:
+ _FREEZE_TARGET[0] = self
+ _set_freeze_events(True)
+ self._cv.notify_all()
+
+ def _maybe_freeze(self) -> None:
+ if not self._pause_depth or self._parked or self._closed.is_set() or not self._freeze_ok:
+ return
+ if _LOCK_WAITERS[0] and _compile_lock_owned():
+ return # a thread waits on compile_lock: finish the frame that holds it first
+ self._thaw = False
+ frame = sys._getframe(2) # _maybe_freeze <- _on_py_start <- the function that just started
+ memo = self._frame_memo
+ if len(memo) > 200_000:
+ memo.clear()
+ why = _freeze_refusal(frame, memo)
+ if why is not None:
+ self._refusals[why] = self._refusals.get(why, 0) + 1
+ return
+ self._park(frozen = True)
+
+ def _park(self, frozen: bool) -> None:
+ """Idle-only park until unpaused and the grace elapsed, close, or (frozen) a thaw."""
+ now = time.perf_counter()
+ with self._cv:
+ if not self._pause_depth and now >= self._release_at:
+ return
+ if frozen:
+ self._frozen = True
+ self.freezes += 1
+ _FROZEN_JOB[0] = self
+ self._disarm_freeze()
+ self._parked = True
+ if self._pause_depth and self._stopped_after is None:
+ self._stopped_after = now - self._pause_t0
+ self._stopped_how = "frozen" if frozen else "block boundary"
+ self._cv.notify_all()
+ try:
+ while not self._closed.is_set() and not (frozen and self._thaw):
+ if self._pause_depth:
+ self._cv.wait(0.5)
+ continue
+ left = self._release_at - time.perf_counter()
+ if left <= 0:
+ break
+ self._cv.wait(left)
+ finally:
+ self._parked = False
+ self._frozen = False
+ if _FROZEN_JOB[0] is self:
+ _FROZEN_JOB[0] = None
+ if not frozen:
+ self._thaw = False
+ end = time.perf_counter()
+ self.paused_s += end - now
+ self._last_yield = end
+
def _yield_point(self) -> None:
self.yield_points += 1
if self._closed.is_set():
@@ -619,6 +1172,12 @@ class BackgroundCompile:
self.graph_s = seg if self.graph_s is None else 0.5 * (self.graph_s + seg)
self._owe_graph = False
self._last_yield = now
+ if self.idle:
+ if self._pause_depth or time.perf_counter() < self._release_at:
+ self._park(frozen = False)
+ if self._closed.is_set():
+ raise _Closed()
+ return
if not self._pause_depth or self._owe_graph:
return
with self._cv:
@@ -784,6 +1343,12 @@ class BackgroundCompile:
# Before the thread starts: the compiled trace must not contain the hook.
self._remove_hook()
self.state = "compiling"
+ if self.idle:
+ # kicked from the render's finally: start once its response is out (the first block boundary waits)
+ self._release_at = time.perf_counter() + self._idle_grace_s
+ self._lock_proxy = (
+ self.yields() and freeze_enabled() and _install_compile_lock_proxy()
+ )
self._thread = threading.Thread(
target = self._run, name = "unsloth-diffusion-bg-compile", daemon = True
)
@@ -812,6 +1377,7 @@ class BackgroundCompile:
t0 = time.perf_counter()
token = _NO_CAPTURE.set(True)
_ACTIVE.set(self if self.yields() else None)
+ self._tid = threading.get_ident()
try:
import torch
@@ -872,6 +1438,7 @@ class BackgroundCompile:
_NO_CAPTURE.reset(token)
_ACTIVE.set(None)
with self._cv:
+ self._disarm_freeze()
self._cv.notify_all()
def _seed_floats(self) -> Optional[tuple]:
@@ -910,6 +1477,10 @@ class BackgroundCompile:
"pauses": self.pauses,
"paused_s": round(self.paused_s, 2),
"float_seeds": self.float_seeds,
+ "idle_only": self.idle,
+ "freezes": self.freezes,
+ "thaws": self.thaws,
+ "pause_wait_s": round(self.pause_wait_s, 3),
"error": self.error,
}
diff --git a/studio/backend/tests/test_diffusion_bg_compile_idle.py b/studio/backend/tests/test_diffusion_bg_compile_idle.py
new file mode 100644
index 00000000000..43c60ae4361
--- /dev/null
+++ b/studio/backend/tests/test_diffusion_bg_compile_idle.py
@@ -0,0 +1,451 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
+
+"""Idle-only background compile: a render never waits for the per-block compile. The pause stops the compile thread
+at once (mid-trace through sys.monitoring for a resident denoiser), back-to-back renders owe it nothing, and it resumes
+only after the last render ended. CPU only: real dynamo traces (backend "eager") stand in for a U-Net block graph."""
+
+from __future__ import annotations
+
+import logging
+import sys
+import threading
+import time
+from types import SimpleNamespace
+
+import pytest
+
+torch = pytest.importorskip("torch")
+
+from core.inference import diffusion_bg_compile as bg # noqa: E402
+from core.inference import diffusion_speed as ds # noqa: E402
+
+LOG = logging.getLogger("test_bg_compile_idle")
+
+needs_monitoring = pytest.mark.skipif(
+ not hasattr(sys, "monitoring"), reason = "mid-trace freeze needs sys.monitoring (3.12+)"
+)
+
+
[email protected](autouse = True)
+def _idle_env(monkeypatch):
+ for env in (bg.IDLE_ENV, bg.IDLE_GRACE_ENV, bg.FREEZE_ENV, bg.YIELD_ENV):
+ monkeypatch.delenv(env, raising = False)
+ monkeypatch.setenv(bg.IDLE_GRACE_ENV, "0.3")
+ yield
+ torch._dynamo.reset()
+
+
+def _wait_for(pred, timeout = 30.0):
+ t0 = time.time()
+ while not pred():
+ if time.time() - t0 > timeout:
+ raise AssertionError("timed out")
+ time.sleep(0.005)
+
+
+class _LongBlock(torch.nn.Module):
+ """A block whose dynamo trace takes about a second on CPU (many small ops), like a U-Net block's graph."""
+
+ def __init__(self, ops: int) -> None:
+ super().__init__()
+ self.ops = ops
+ self.lin = torch.nn.Linear(8, 8)
+ self.traced = 0
+
+ def forward(self, x):
+ for i in range(self.ops):
+ x = torch.tanh(self.lin(x) * (1.0 + i * 1e-4)) + x * 0.5
+ return x
+
+
+class _Net(torch.nn.Module):
+ def __init__(
+ self,
+ n: int = 3,
+ ops: int = 400,
+ ) -> None:
+ super().__init__()
+ self.blocks = torch.nn.ModuleList(_LongBlock(ops) for _ in range(n))
+ for blk in self.blocks:
+ blk.compile(backend = "eager", fullgraph = True, dynamic = False)
+ ds.guard_compiled_blocks(self, LOG)
+ self._unsloth_bg_yield = True
+
+ def forward(self, x):
+ for blk in self.blocks:
+ x = blk(x)
+ return x
+
+
+def _compiled_frames() -> int:
+ from torch._dynamo.utils import counters
+ return int(counters["stats"]["unique_graphs"])
+
+
+def _armed(net):
+ job = bg.arm(net, logger = LOG)
+ assert job is not None and job.idle
+ with torch.no_grad(), bg.force_eager():
+ net(torch.randn(2, 8)) # records the sample; the guard routes the blocks eager
+ assert len(job.samples) == 1
+ return job
+
+
+def _render(net):
+ with torch.no_grad(), bg.force_eager():
+ return net(torch.randn(2, 8))
+
+
+def _in_dynamo(job) -> bool:
+ frame = sys._current_frames().get(job._thread.ident)
+ while frame is not None:
+ if (frame.f_globals.get("__name__") or "").startswith("torch._dynamo."):
+ return True
+ frame = frame.f_back
+ return False
+
+
+def test_switches(monkeypatch):
+ assert bg.idle_only_enabled() is True
+ assert bg.idle_grace_seconds() == pytest.approx(0.3)
+ monkeypatch.setenv(bg.IDLE_GRACE_ENV, "junk")
+ assert bg.idle_grace_seconds() == pytest.approx(1.0)
+ monkeypatch.setenv(bg.IDLE_GRACE_ENV, "1e9")
+ assert bg.idle_grace_seconds() == pytest.approx(60.0)
+ monkeypatch.setenv(bg.IDLE_ENV, "0")
+ assert bg.idle_only_enabled() is False
+ assert bg.freeze_enabled() is False
+ monkeypatch.delenv(bg.IDLE_ENV)
+ monkeypatch.setenv(bg.FREEZE_ENV, "0")
+ assert bg.freeze_enabled() is False
+ assert bg.BackgroundCompile(torch.nn.Linear(2, 2)).idle is True
+
+
+@needs_monitoring
+def test_a_render_freezes_the_compile_mid_trace_without_waiting():
+ net = _Net(n = 3, ops = 500)
+ job = _armed(net)
+ assert job.kick()
+ # past the kick's grace, inside the first block graph's trace
+ _wait_for(lambda: _in_dynamo(job))
+ time.sleep(0.05)
+ graphs = _compiled_frames()
+ waited, paused = job.pause(budget_s = 5.0, freeze = True)
+ assert paused
+ # typically a few ms; the bound only absorbs a CPU-starved runner. _in_dynamo below proves it stopped mid-trace,
+ # not at a block boundary. A starved compile thread also looks quiet, so pause may return before it runs again;
+ # the armed freeze then stops it at its next Python call
+ assert waited < 2.0, waited
+ _wait_for(lambda: job._frozen, timeout = 5.0)
+ assert job.freezes == 1
+ assert _in_dynamo(job) # parked inside the trace, not at a block boundary
+ assert sys.monitoring.get_events(bg._FREEZE_TOOL[0]) == 0
+ t0 = time.perf_counter()
+ _render(net)
+ eager_s = time.perf_counter() - t0
+ time.sleep(0.5)
+ assert (
+ _compiled_frames() == graphs and job.compiling()
+ ) # no progress while the render holds the pause
+ job.resume()
+ _wait_for(lambda: job.state == "done")
+ assert job.error is None
+ assert _compiled_frames() >= graphs + 1
+ assert eager_s < 5.0
+ x = torch.randn(2, 8)
+ with torch.no_grad():
+ compiled = net(x)
+ with bg.force_eager():
+ eager = net(x)
+ torch.testing.assert_close(compiled, eager)
+
+
+@needs_monitoring
+def test_resume_waits_for_the_idle_grace(monkeypatch):
+ monkeypatch.setenv(bg.IDLE_GRACE_ENV, "0.8")
+ net = _Net(n = 2, ops = 500)
+ job = _armed(net)
+ job.kick()
+ _wait_for(lambda: _in_dynamo(job))
+ _, paused = job.pause(freeze = True)
+ assert paused
+ _wait_for(lambda: job._frozen, timeout = 2.0)
+ job.resume()
+ t0 = time.perf_counter()
+ time.sleep(0.4)
+ assert job._frozen # still inside the grace: a back-to-back client's next request arrives here
+ _, paused = job.pause(freeze = True) # the next render: nothing to wait for
+ assert paused and time.perf_counter() - t0 < 0.6
+ job.resume()
+ _wait_for(lambda: job.state == "done")
+
+
+def test_back_to_back_renders_never_wait_and_owe_nothing():
+ """main: each resume owed one block graph, so a render waited for it (0.3-5 s on L4). Idle-only: never."""
+ net = _FakeBlocks(n = 5, compile_s = 0.3)
+ job = bg.arm(net, logger = LOG)
+ with torch.no_grad(), bg.force_eager():
+ net(torch.randn(2, 4))
+ job.kick()
+ time.sleep(0.4) # past the kick's grace: first block graph in flight
+ waits = []
+ for _ in range(12):
+ waited, paused = job.pause(freeze = False)
+ assert paused
+ waits.append(waited)
+ time.sleep(0.05)
+ job.resume()
+ time.sleep(0.02) # the client's turnaround, inside the grace
+ # without freeze the first pause waits out the block graph in flight; after that nothing is owed
+ assert waits[0] < 0.4 and max(waits[1:]) < 0.05, waits
+ done_during_renders = len(net.compiled_blocks)
+ assert done_during_renders <= 2 # the graph in flight at the first pause, then nothing
+ _wait_for(lambda: job.state == "done")
+ assert len(net.compiled_blocks) == 5
+
+
+def test_without_freeze_the_compile_parks_at_the_next_block_boundary():
+ net = _FakeBlocks(n = 6, compile_s = 0.2)
+ job = bg.arm(net, logger = LOG)
+ with torch.no_grad(), bg.force_eager():
+ net(torch.randn(2, 4))
+ job.kick()
+ _wait_for(lambda: len(net.compiled_blocks) >= 1)
+ waited, paused = job.pause(freeze = False)
+ # unfrozen: pause returns only once parked, an offloaded block must not run beside the render
+ assert paused and job._parked and waited < 1.0
+ held = len(net.compiled_blocks)
+ time.sleep(0.5)
+ assert len(net.compiled_blocks) == held and not job._frozen
+ job.resume()
+ _wait_for(lambda: job.state == "done")
+
+
+def test_close_ends_an_idle_parked_compile():
+ net = _FakeBlocks(n = 6, compile_s = 0.15)
+ job = bg.arm(net, logger = LOG)
+ with torch.no_grad(), bg.force_eager():
+ net(torch.randn(2, 4))
+ job.kick()
+ job.pause(freeze = False)
+ _wait_for(lambda: job._parked, timeout = 2.0)
+ thread = job._thread
+ job.close(timeout = 5.0)
+ assert not thread.is_alive()
+ assert job.state == "failed" and "closed" in (job.error or "")
+
+
+@needs_monitoring
+def test_close_ends_a_frozen_compile():
+ net = _Net(n = 3, ops = 500)
+ job = _armed(net)
+ job.kick()
+ _wait_for(lambda: _in_dynamo(job))
+ job.pause(freeze = True)
+ _wait_for(lambda: job._frozen, timeout = 2.0)
+ thread = job._thread
+ job.close(timeout = 30.0)
+ assert not thread.is_alive()
+ assert job.state == "failed"
+
+
+@needs_monitoring
+def test_a_compile_elsewhere_thaws_a_frozen_compile_instead_of_deadlocking():
+ """The frozen trace holds dynamo's compile_lock; a render that compiles something must not wait forever."""
+ net = _Net(n = 2, ops = 500)
+ job = _armed(net)
+ job.kick()
+ _wait_for(lambda: _in_dynamo(job))
+ job.pause(freeze = True)
+ _wait_for(lambda: job._frozen, timeout = 2.0)
+
+ other = torch.compile(lambda t: t * 2 + 1, backend = "eager")
+ out: list = []
+ worker = threading.Thread(target = lambda: out.append(other(torch.ones(3))), daemon = True)
+ worker.start()
+ worker.join(30.0)
+ assert not worker.is_alive(), "deadlock: the render's compile waited on the frozen trace"
+ torch.testing.assert_close(out[0], torch.full((3,), 3.0))
+ assert job.thaws == 1
+ # it ran on only to the end of the frame that held the lock, then froze again while the render still runs
+ _wait_for(lambda: job._frozen, timeout = 30.0)
+ assert job.freezes >= 2
+ job.resume()
+ _wait_for(lambda: job.state == "done")
+ assert job.error is None
+
+
+def test_freeze_safe_refuses_a_lock_taking_frame():
+ def inner():
+ return sys._getframe()
+
+ # no compile-thread entry below: never safe
+ assert bg._freeze_safe(inner()) is False
+
+ code = bg.BackgroundCompile._run.__code__
+ assert code.co_name == "_run"
+
+ class _F:
+ def __init__(
+ self,
+ name,
+ back,
+ code = None,
+ ):
+ self.f_globals = {"__name__": name}
+ self.f_back = back
+ self.f_code = code if code is not None else inner.__code__
+
+ run = _F(bg.__name__, None, code)
+ unet = _F("diffusers.models.unets.unet_2d_condition", run)
+ trace = _F("torch._dynamo.symbolic_convert", _F("torch._dynamo.convert_frame", unet))
+ assert bg._freeze_safe(trace) is True
+ assert bg._freeze_safe(unet) is False # outside the compile: Studio / diffusers eager code
+ assert bg._freeze_safe(_F("logging", trace)) is False
+ assert bg._freeze_safe(_F("logging.handlers", trace)) is False
+ assert bg._freeze_safe(_F("torch._inductor.compile_worker.subproc_pool", trace)) is False
+ # handing kernels to the pool holds no lock: a refusal there kept the compile beside the render for 0.7-1.6 s (L4)
+ assert bg._freeze_safe(_F("torch._inductor.async_compile", trace)) is True
+ assert bg._freeze_safe(_F("core.inference.diffusion_compile_config", trace)) is False
+
+
+_HELD_LOCK = threading.Lock()
+
+
+def _studio_like(record):
+ record.append(sys._getframe().f_lineno)
+ with _HELD_LOCK:
+ record.append(sys._getframe().f_lineno)
+ record.append(sys._getframe().f_lineno)
+
+
+def test_a_studio_frame_is_refused_only_inside_its_lock_block():
+ """Refusing every Studio frame kept the compile beside the render (A100: 30-290 s); only a frame in a lock is."""
+ lines: list = []
+ _studio_like(lines)
+ before, held, after = lines
+ code = _studio_like.__code__
+
+ class _F:
+ def __init__(self, line):
+ self.f_globals = {"__name__": "core.inference.some_module"}
+ self.f_code = code
+ self.f_lineno = line
+ self.f_back = None
+
+ assert bg._frame_refusal(_F(before)) is None
+ assert bg._frame_refusal(_F(after)) is None
+ assert bg._frame_refusal(_F(held)) == "core.inference.some_module:_studio_like"
+ # the real wrapper on the replay path: eager call outside its capture locks
+ from core.inference import diffusion_cuda_graph as cg
+
+ assert bg._lock_spans(cg.GraphedForward._eager.__code__) == ()
+ assert bg._lock_spans(cg.run_if_capture_idle.__code__) is True
+
+
+def test_freeze_refusal_memo_matches_a_fresh_walk():
+ def leaf(memo):
+ return bg._freeze_refusal(sys._getframe(), memo), bg._freeze_refusal(sys._getframe())
+
+ memo: dict = {}
+ first = leaf(memo)
+ again = leaf(memo)
+ assert first[0] == first[1] == again[0] == again[1] == "not the compile thread"
+ assert memo
+
+
+def test_compile_lock_proxy_is_the_same_lock():
+ from torch._dynamo import convert_frame
+
+ lock = convert_frame.compile_lock
+ if not getattr(lock, "_unsloth_bg_proxy", False):
+ assert bg._install_compile_lock_proxy()
+ proxy = convert_frame.compile_lock
+ assert proxy._unsloth_bg_proxy and bg._install_compile_lock_proxy()
+ with proxy:
+ with proxy: # reentrant, like the RLock it wraps
+ assert proxy._is_owned()
+ got = []
+ with proxy:
+ t = threading.Thread(target = lambda: got.append(proxy.acquire(False)))
+ t.start()
+ t.join()
+ assert got == [False]
+
+
+class _FakeBg:
+ def __init__(self) -> None:
+ self.idle = True
+ self.calls: list = []
+ self._freeze_ok = False
+ self._frozen = False
+ self._parked = False
+
+ def compiling(self):
+ return True
+
+ def yields(self):
+ return True
+
+ def pause(
+ self,
+ cancel = None,
+ budget_s = 0.0,
+ freeze = False,
+ ):
+ self.calls.append(("pause", budget_s, freeze))
+ return 0.0, True
+
+ def wait(self, cancel = None):
+ self.calls.append(("wait",))
+ return 9.0
+
+
[email protected]("policy, freeze", [("none", True), ("group", False), ("model", False)])
+def test_render_stops_an_idle_compile_without_a_slice(policy, freeze):
+ from core.inference import diffusion as d
+
+ fake = _FakeBg()
+ pipe = SimpleNamespace(_unsloth_loras = (), _unsloth_eager_render_s = 9.9)
+ state = SimpleNamespace(pipe = pipe, speed_optims = (), offload_policy = policy)
+ assert d.OFFLOAD_NONE == "none"
+ assert d._wait_or_pause_bg_compile(fake, state, None, threading.Event()) is fake
+ assert fake.calls == [("pause", 0.0, freeze)]
+
+
+def test_lora_render_still_waits_for_an_idle_compile():
+ from core.inference import diffusion as d
+
+ fake = _FakeBg()
+ pipe = SimpleNamespace(_unsloth_loras = (), _unsloth_eager_render_s = 9.9)
+ state = SimpleNamespace(pipe = pipe, speed_optims = (), offload_policy = "none")
+ assert d._wait_or_pause_bg_compile(fake, state, [("a/lora", 1.0)], threading.Event()) is None
+ assert fake.calls == [("wait",)]
+
+
+class _FakeBlocks(torch.nn.Module):
+ """Guarded blocks whose first compiled call sleeps ``compile_s`` (a block graph's compile)."""
+
+ def __init__(self, n: int, compile_s: float) -> None:
+ super().__init__()
+ self.blocks = torch.nn.ModuleList(torch.nn.Linear(4, 4) for _ in range(n))
+ self.compiled_blocks: set = set()
+ for i, blk in enumerate(self.blocks):
+ blk._compiled_call_impl = self._fake(i, blk, compile_s)
+ ds.guard_compiled_blocks(self, LOG)
+ self._unsloth_bg_yield = True
+
+ def _fake(self, index, blk, compile_s):
+ def compiled(*args, **kwargs):
+ if index not in self.compiled_blocks:
+ time.sleep(compile_s)
+ self.compiled_blocks.add(index)
+ return blk.forward(*args, **kwargs)
+
+ return compiled
+
+ def forward(self, x):
+ for blk in self.blocks:
+ x = blk(x)
+ return x
diff --git a/studio/backend/tests/test_diffusion_unet_block_compile.py b/studio/backend/tests/test_diffusion_unet_block_compile.py
index 499c5635df1..36dda14c71f 100644
--- a/studio/backend/tests/test_diffusion_unet_block_compile.py
+++ b/studio/backend/tests/test_diffusion_unet_block_compile.py
@@ -214,6 +214,7 @@ def _wait_for(pred, timeout = 10.0):
def test_a_render_pauses_the_compile_between_blocks_and_it_resumes(monkeypatch):
+ monkeypatch.setenv(bg.IDLE_ENV, "0")
monkeypatch.delenv(bg.YIELD_ENV, raising = False)
net = _Blocks(n = 6, compile_s = 0.15)
job = _armed(net)
@@ -241,6 +242,7 @@ def test_a_render_pauses_the_compile_between_blocks_and_it_resumes(monkeypatch):
def test_a_compile_that_ends_inside_the_slice_is_not_paused(monkeypatch):
+ monkeypatch.setenv(bg.IDLE_ENV, "0")
monkeypatch.delenv(bg.YIELD_ENV, raising = False)
net = _Blocks(n = 4, compile_s = 0.3)
job = _armed(net)
@@ -254,6 +256,7 @@ def test_a_compile_that_ends_inside_the_slice_is_not_paused(monkeypatch):
def test_the_pause_is_asked_one_block_graph_before_the_slice_ends(monkeypatch):
+ monkeypatch.setenv(bg.IDLE_ENV, "0")
monkeypatch.delenv(bg.YIELD_ENV, raising = False)
net = _Blocks(n = 12, compile_s = 0.4)
job = _armed(net)
@@ -268,6 +271,7 @@ def test_the_pause_is_asked_one_block_graph_before_the_slice_ends(monkeypatch):
def test_back_to_back_renders_cannot_starve_the_compile(monkeypatch):
+ monkeypatch.setenv(bg.IDLE_ENV, "0")
monkeypatch.delenv(bg.YIELD_ENV, raising = False)
net = _Blocks(n = 5, compile_s = 0.3)
job = _armed(net)
@@ -285,6 +289,7 @@ def test_back_to_back_renders_cannot_starve_the_compile(monkeypatch):
def test_nested_pause_runs_no_second_slice(monkeypatch):
+ monkeypatch.setenv(bg.IDLE_ENV, "0")
monkeypatch.delenv(bg.YIELD_ENV, raising = False)
net = _Blocks(n = 6, compile_s = 0.2)
job = _armed(net)
@@ -302,6 +307,7 @@ def test_nested_pause_runs_no_second_slice(monkeypatch):
def test_close_ends_a_parked_compile(monkeypatch):
+ monkeypatch.setenv(bg.IDLE_ENV, "0")
monkeypatch.delenv(bg.YIELD_ENV, raising = False)
net = _Blocks(n = 6, compile_s = 0.15)
job = _armed(net)
@@ -316,6 +322,7 @@ def test_close_ends_a_parked_compile(monkeypatch):
@pytest.mark.parametrize("env, flag", [("0", True), (None, False)])
def test_without_yield_a_pause_waits_for_the_whole_compile(monkeypatch, env, flag):
+ monkeypatch.setenv(bg.IDLE_ENV, "0")
if env is None:
monkeypatch.delenv(bg.YIELD_ENV, raising = False)
else: