Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

Studio: compile the SDXL U-Net per block so the 4th image no longer waits for the whole compile (#16085)

Daniel Han8a021506bd91
diff --git a/studio/backend/core/inference/diffusion.py b/studio/backend/core/inference/diffusion.py
index a3d9b1cb24e..99afcfee4bc 100644
--- a/studio/backend/core/inference/diffusion.py
+++ b/studio/backend/core/inference/diffusion.py
@@ -3733,6 +3733,52 @@ def _progress_latch_enabled() -> bool:
)
+def _wait_or_pause_bg_compile(bg: Any, state: Any, loras: Any, cancel: Any) -> Any:
+ """Wait for an in-flight bg compile, or (per-block) let it run one slice then park it; returns the paused
+ compile for the caller to resume, else None. LoRA renders always wait (they would change the module mid-trace)."""
+ if bg is None or not bg.compiling():
+ return None
+ pipe = getattr(state, "pipe", None)
+ if bg.yields() and not _has_active_lora(loras) and not getattr(pipe, "_unsloth_loras", ()):
+ budget = bg_compile.slice_seconds(getattr(pipe, "_unsloth_eager_render_s", None))
+ waited, paused = bg.pause(cancel, budget_s = budget)
+ if paused:
+ logger.info(
+ "diffusion.bg_compile: render paused the background compile after %.1f s (slice %.1f s); this "
+ "render runs eager and the compile resumes after it",
+ waited,
+ budget,
+ )
+ return bg
+ logger.info(
+ "diffusion.bg_compile: render waited %.1f s for the background compile (slice %.1f s)",
+ waited,
+ budget,
+ )
+ return None
+ logger.info("diffusion.bg_compile: render waits for the background compile to finish")
+ waited = bg.wait(cancel)
+ logger.info("diffusion.bg_compile: render waited %.1f s for the background compile", waited)
+ return None
+
+
+def _note_eager_render(state: Any, bg_eager: bool, seconds: float) -> None:
+ """Keeps this load's fastest eager render (the background compile's slice): renders before the deferred
+ profile, and those that ran eager beside a pending compile."""
+ pipe = getattr(state, "pipe", None)
+ if pipe is None or not (seconds > 0):
+ return
+ eager = bg_eager or "compiled" not in tuple(getattr(state, "speed_optims", None) or ())
+ if not eager:
+ return
+ try:
+ best = getattr(pipe, "_unsloth_eager_render_s", None)
+ if not isinstance(best, (int, float)) or seconds < best:
+ pipe._unsloth_eager_render_s = float(seconds)
+ except Exception: # noqa: BLE001 - a heuristic only
+ pass
+
+
def _bg_compile_module(
pipe: Any,
*,
@@ -13216,29 +13262,28 @@ class DiffusionBackend:
if _hires_pass is None:
bind_generation_cancel(cancel)
with nullcontext() if _hires_pass is not None else self._generation_slot(cancel):
- # Never render beside a background compile; wait outside the state lock so status polls keep answering.
+ # Wait outside the state lock so status polls keep answering.
pending_bg = getattr(self._state, "bg_compile", None)
- if pending_bg is not None and pending_bg.compiling():
- logger.info(
- "diffusion.bg_compile: render waits for the background compile to finish"
- )
- waited = pending_bg.wait(cancel)
- logger.info(
- "diffusion.bg_compile: render waited %.1f s for the background compile", waited
- )
- with self._lock:
- state = self._state
- if state is None:
- raise RuntimeError(DIFFUSION_NOT_LOADED_MSG)
- if cancel.is_set():
- raise RuntimeError(DIFFUSION_CANCELLED_MSG)
- # The slot admits on a zero fence, which a COMMITTED replacement also satisfies (#9448)
- loaded_id = load_identity(state.repo_id, state.base_repo, state.family.name)
- if expected_load is not None and expected_load != loaded_id:
- raise DiffusionModelReplacedError(expected_load, loaded_id)
- # Publish active step-0 state before slow setup, so a reload's mount probe does not read idle.
- self._gen = _GenState(total_steps = steps)
- begin_attention_run(state)
+ paused_bg = _wait_or_pause_bg_compile(pending_bg, self._state, loras, cancel)
+ try:
+ with self._lock:
+ state = self._state
+ if state is None:
+ raise RuntimeError(DIFFUSION_NOT_LOADED_MSG)
+ if cancel.is_set():
+ raise RuntimeError(DIFFUSION_CANCELLED_MSG)
+ # The slot admits on a zero fence, which a COMMITTED replacement also satisfies (#9448)
+ loaded_id = load_identity(state.repo_id, state.base_repo, state.family.name)
+ if expected_load is not None and expected_load != loaded_id:
+ raise DiffusionModelReplacedError(expected_load, loaded_id)
+ # Publish active step-0 state before slow setup, so a reload's mount probe does not read idle.
+ self._gen = _GenState(total_steps = steps)
+ begin_attention_run(state)
+ except BaseException:
+ if paused_bg is not None:
+ paused_bg.resume()
+ raise
+ render_t0 = time.perf_counter()
# Reset in the finally, so a failed or cancelled generation frees its reused outputs.
static_skip_pipe = None
restore_scheduler: Optional[tuple[Any, Any]] = None
@@ -13281,6 +13326,14 @@ class DiffusionBackend:
lora_requested = _has_active_lora(loras)
# Stay eager while prior adapters are attached: compiling would bake them in.
loras_attached = bool(getattr(state.pipe, "_unsloth_loras", ()))
+ if (
+ state.speed_deferred
+ and state.generation_count == 1
+ and not aux_pass
+ and str(getattr(state, "device", "")).startswith("cuda")
+ ):
+ # 2nd image: warm inductor's workers so the deferred compile does not wait on them.
+ bg_compile.warm_compile_pool(logger)
if (
state.speed_deferred
and state.generation_count >= 2
@@ -14433,6 +14486,8 @@ class DiffusionBackend:
if applied_embeddings is not None:
result["embeddings"] = list(applied_embeddings.used)
result["embeddings_skipped"] = list(applied_embeddings.skipped)
+ if not aux_pass and _hires_pass is None:
+ _note_eager_render(state, bg_eager, time.perf_counter() - render_t0)
logger.info(
"diffusion.generated: %s",
format_generation_for_log(
@@ -14472,6 +14527,8 @@ class DiffusionBackend:
bg.kick()
except Exception as exc: # noqa: BLE001 - the next render compiles inline
logger.warning("diffusion.bg_compile: kick failed: %s", exc)
+ if paused_bg is not None:
+ paused_bg.resume()
with self._generation_cancel_lock:
if self._active_generate_cancel is cancel:
self._active_generate_cancel = None
diff --git a/studio/backend/core/inference/diffusion_autoopt/plan.py b/studio/backend/core/inference/diffusion_autoopt/plan.py
index 9ead0d04ef7..5f754aaa0cf 100644
--- a/studio/backend/core/inference/diffusion_autoopt/plan.py
+++ b/studio/backend/core/inference/diffusion_autoopt/plan.py
@@ -272,4 +272,7 @@ def compile_cache_kwargs(
if offload_below_hooks:
# An offloaded denoiser compiles its blocks below the hooks, a different graph than a traced-hook bundle.
kwargs["below_hooks"] = True
+ if ds.unet_compiles_regionally(ds._denoiser_unet(pipe), max_autotune = mode == SPEED_MAX):
+ # Per-block U-Net graphs: never served a whole-module bundle (nor the other way round).
+ kwargs["unet_graph"] = "blocks"
return kwargs
diff --git a/studio/backend/core/inference/diffusion_autoopt/targets.py b/studio/backend/core/inference/diffusion_autoopt/targets.py
index b38e08e008a..3f0dc54b50c 100644
--- a/studio/backend/core/inference/diffusion_autoopt/targets.py
+++ b/studio/backend/core/inference/diffusion_autoopt/targets.py
@@ -15,9 +15,24 @@ HOOKED_ATTRS: tuple[str, ...] = DIT_ATTRS + ("unet",)
# Named partitions a modular workflow compiles through a view (MiniMax-H3's reference DiT).
PARTITION_ATTRS: tuple[str, ...] = ("transformer_ref",)
-# U-Net denoisers ship no ``_repeated_blocks``, so the regional compile cannot reach them; these classes get a
-# WHOLE-module static compile (diffusion_speed).
+# Whole-module U-Net compile only under max-autotune or UNSLOTH_DIFFUSION_UNET_REGIONAL=0.
UNET_WHOLE_COMPILE: frozenset[str] = frozenset({"UNet2DConditionModel"})
+# SDXL 1024^2 on A100: 19 graphs, 55 s cold vs 154 s whole. Resnets/samplers are needed: step 97.3 ms eager,
+# transformer blocks only 91.2, all four 83.1, whole module 82.5.
+UNET_REGIONAL_BLOCKS: tuple[str, ...] = (
+ "BasicTransformerBlock",
+ "ResnetBlock2D",
+ "Downsample2D",
+ "Upsample2D",
+)
+UNET_REGIONAL_ENV = "UNSLOTH_DIFFUSION_UNET_REGIONAL"
+
+
+def unet_regional_enabled() -> bool:
+ import os
+ raw = (os.environ.get(UNET_REGIONAL_ENV) or "").strip().lower()
+ return raw not in ("0", "false", "no", "off")
+
# Component names that are never a denoiser even when they are a ModelMixin with repeated blocks.
_NOT_DENOISERS = (
diff --git a/studio/backend/core/inference/diffusion_bg_compile.py b/studio/backend/core/inference/diffusion_bg_compile.py
index 74d0f26af91..962b3477a55 100644
--- a/studio/backend/core/inference/diffusion_bg_compile.py
+++ b/studio/backend/core/inference/diffusion_bg_compile.py
@@ -20,6 +20,12 @@ from typing import Any, Callable, Optional
_ENV = "UNSLOTH_DIFFUSION_BG_COMPILE"
OFFLOADED_ENV = "UNSLOTH_DIFFUSION_BG_COMPILE_OFFLOADED"
+YIELD_ENV = "UNSLOTH_DIFFUSION_BG_COMPILE_YIELD"
+# Seconds of compile a render lets run before it pauses it; unset = this load's fastest eager render.
+SLICE_ENV = "UNSLOTH_DIFFUSION_BG_COMPILE_SLICE_S"
+_SLICE_MAX_S = 30.0
+# A gap between two yield points longer than this compiled a block graph (a cached block call takes milliseconds).
+_GRAPH_SEGMENT_S = 0.25
_MAX_SAMPLES = 4
# ContextVars, not thread-locals: diffusion_render_thread.run copies the caller's context onto the render thread.
@@ -35,6 +41,14 @@ _NO_RECORD: contextvars.ContextVar[bool] = contextvars.ContextVar(
_AUXILIARY: contextvars.ContextVar[bool] = contextvars.ContextVar(
"unsloth_diffusion_auxiliary_pass", default = False
)
+# The BackgroundCompile whose thread this is (set only on that thread): yield_point's target.
+_ACTIVE: contextvars.ContextVar[Optional["BackgroundCompile"]] = contextvars.ContextVar(
+ "unsloth_diffusion_bg_active", default = None
+)
+
+
+class _Closed(BaseException):
+ """Raised at a yield point once the compile is closed (unload): ends the paused replay."""
def enabled() -> bool:
@@ -47,6 +61,67 @@ def offloaded_enabled() -> bool:
return raw not in ("0", "false", "no", "off")
+def yield_enabled() -> bool:
+ raw = (os.environ.get(YIELD_ENV) or "").strip().lower()
+ return raw not in ("0", "false", "no", "off")
+
+
+WARM_POOL_ENV = "UNSLOTH_DIFFUSION_BG_COMPILE_WARM_POOL"
+_POOL_WARMED = False
+
+
+def warm_compile_pool(logger: Any = None) -> bool:
+ """Start inductor's compile worker pool ahead of the deferred compile."""
+ global _POOL_WARMED
+ if _POOL_WARMED or not enabled():
+ return False
+ raw = (os.environ.get(WARM_POOL_ENV) or "").strip().lower()
+ if raw in ("0", "false", "no", "off"):
+ return False
+ _POOL_WARMED = True
+ try:
+ from . import diffusion_compile_config
+ from torch._inductor.async_compile import AsyncCompile
+
+ diffusion_compile_config.apply()
+ AsyncCompile.warm_pool()
+ AsyncCompile.wakeup()
+ except Exception as exc: # noqa: BLE001 - the compile warms its own pool as before
+ if logger is not None:
+ logger.debug("diffusion.bg_compile: compile pool warm-up skipped: %s", exc)
+ return False
+ if logger is not None:
+ logger.info(
+ "diffusion.bg_compile: inductor compile workers starting ahead of the deferred compile"
+ )
+ return True
+
+
+def slice_seconds(eager_render_s: Optional[float]) -> float:
+ """Compile seconds a render lets run before it pauses the compile: ``SLICE_ENV`` when set, else the load's
+ fastest eager render (0 when none was timed), clamped to [0, 30]."""
+ raw = (os.environ.get(SLICE_ENV) or "").strip()
+ value: Optional[float] = None
+ if raw:
+ try:
+ value = float(raw)
+ except ValueError:
+ value = None
+ if value is None:
+ value = eager_render_s if isinstance(eager_render_s, (int, float)) else 0.0
+ if value != value: # NaN
+ value = 0.0
+ return max(0.0, min(_SLICE_MAX_S, float(value)))
+
+
+def yield_point() -> None:
+ """Called before a guarded compiled block runs. On a background compile thread with a pause requested, parks
+ here (between two block graphs, never inside a trace) until the render resumes it. Elsewhere a no-op."""
+ bg = _ACTIVE.get()
+ if bg is not None:
+ bg._yield_point()
+
+
FLOAT_SEEDS_ENV = "UNSLOTH_DIFFUSION_BG_COMPILE_FLOAT_SEEDS"
FLOAT_SEEDS_FILE = "bg_compile_float_specializations.json"
_FLOAT_SEEDS_VERSION = 1
@@ -475,6 +550,94 @@ class BackgroundCompile:
self.error: Optional[str] = None
self.compile_s: Optional[float] = None
self.eager_generations = 0
+ self._cv = threading.Condition(threading.Lock())
+ self._pause_depth = 0
+ self._parked = False
+ # Bumped when the last pause is released: a parked thread leaves on it even if a new pause is already held.
+ self._release_epoch = 0
+ self.pauses = 0
+ self.paused_s = 0.0
+ self.yield_points = 0
+ # _owe_graph: one graph per resume, so back-to-back renders cannot starve the compile.
+ self.graph_s: Optional[float] = None
+ self._last_yield: Optional[float] = None
+ self._owe_graph = False
+
+ def yields(self) -> bool:
+ return bool(getattr(self.module, "_unsloth_bg_yield", False)) and yield_enabled()
+
+ def pause(
+ self,
+ cancel: Any = None,
+ budget_s: float = 0.0,
+ poll_s: float = 0.05,
+ ) -> tuple[float, bool]:
+ """Let the compile run ``budget_s`` more seconds, then park it at its next yield point. Returns
+ ``(seconds waited, paused)``; ``paused`` means a pause is held and the caller MUST ``resume()``. A compile
+ that ends inside the budget returns ``paused=False`` (the render then runs compiled)."""
+ t0 = time.perf_counter()
+ thread = self._thread
+ if thread is None or thread is threading.current_thread() or not thread.is_alive():
+ return 0.0, False
+ # Ask one graph_s early: the pause lands after the block graph in flight.
+ budget = max(0.0, float(budget_s or 0.0))
+ lead = self.graph_s if self.graph_s is not None else budget
+ deadline = t0 + (0.0 if self._pause_depth else max(0.0, budget - lead))
+ while thread.is_alive() and not (cancel is not None and cancel.is_set()):
+ left = deadline - time.perf_counter()
+ if left <= 0:
+ break
+ thread.join(min(poll_s * 5, left))
+ if not thread.is_alive():
+ return time.perf_counter() - t0, False
+ with self._cv:
+ self._pause_depth += 1
+ self.pauses += 1
+ self._cv.notify_all()
+ while thread.is_alive() and not self._parked:
+ if cancel is not None and cancel.is_set():
+ break
+ self._cv.wait(poll_s)
+ return time.perf_counter() - t0, True
+
+ def resume(self) -> None:
+ with self._cv:
+ self._pause_depth = max(0, self._pause_depth - 1)
+ if not self._pause_depth:
+ self._release_epoch += 1
+ # the next pause() waits for a fresh park, after the block graph this release owes
+ self._parked = False
+ self._cv.notify_all()
+
+ def _yield_point(self) -> None:
+ self.yield_points += 1
+ if self._closed.is_set():
+ raise _Closed()
+ now = time.perf_counter()
+ if self._last_yield is not None and now - self._last_yield > _GRAPH_SEGMENT_S:
+ seg = now - self._last_yield
+ self.graph_s = seg if self.graph_s is None else 0.5 * (self.graph_s + seg)
+ self._owe_graph = False
+ self._last_yield = now
+ if not self._pause_depth or self._owe_graph:
+ return
+ with self._cv:
+ if not self._pause_depth:
+ return
+ self._parked = True
+ self._cv.notify_all()
+ epoch = self._release_epoch
+ try:
+ while self._release_epoch == epoch and not self._closed.is_set():
+ self._cv.wait(0.5)
+ finally:
+ self._parked = False
+ end = time.perf_counter()
+ self.paused_s += end - now
+ self._last_yield = end
+ self._owe_graph = True
+ if self._closed.is_set():
+ raise _Closed()
def install(self) -> bool:
try:
@@ -648,6 +811,7 @@ class BackgroundCompile:
def _run(self) -> None:
t0 = time.perf_counter()
token = _NO_CAPTURE.set(True)
+ _ACTIVE.set(self if self.yields() else None)
try:
import torch
@@ -675,13 +839,20 @@ class BackgroundCompile:
self._finish("done")
if self.logger is not None:
self.logger.info(
- "diffusion.bg_compile: %s compiled in the background in %.1f s%s; the next render runs compiled",
+ "diffusion.bg_compile: %s compiled in the background in %.1f s%s%s; the next render runs compiled",
type(self.module).__name__,
self.compile_s,
f" ({self.float_seeds} float specialisation(s) pre-seeded)"
if self.float_seeds
else "",
+ f" (paused {self.pauses}x for renders, {self.paused_s:.1f} s, compiling "
+ f"{self.compile_s - self.paused_s:.1f} s)"
+ if self.pauses
+ else "",
)
+ except _Closed:
+ self.compile_s = time.perf_counter() - t0
+ self._finish("failed", "closed before the compile finished")
except BaseException as exc: # noqa: BLE001 - never kills the process; the next render compiles inline
self.compile_s = time.perf_counter() - t0
self._finish(
@@ -699,6 +870,9 @@ class BackgroundCompile:
exc.__traceback__ = None
finally:
_NO_CAPTURE.reset(token)
+ _ACTIVE.set(None)
+ with self._cv:
+ self._cv.notify_all()
def _seed_floats(self) -> Optional[tuple]:
"""Pre-seed the SymFloat specialisations this key's last compile learned; returns what ``_keep_floats`` needs."""
@@ -711,6 +885,8 @@ class BackgroundCompile:
def close(self, timeout: Optional[float] = None) -> None:
"""Stop recording and wait for an in-flight compile (it holds the module); used on unload."""
self._closed.set()
+ with self._cv:
+ self._cv.notify_all() # a parked compile wakes, sees closed and ends
self._remove_hook()
thread = self._thread
if thread is not None and thread.is_alive() and thread is not threading.current_thread():
@@ -731,6 +907,8 @@ class BackgroundCompile:
"samples": len(self.samples),
"compile_s": None if self.compile_s is None else round(self.compile_s, 2),
"eager_generations": self.eager_generations,
+ "pauses": self.pauses,
+ "paused_s": round(self.paused_s, 2),
"float_seeds": self.float_seeds,
"error": self.error,
}
diff --git a/studio/backend/core/inference/diffusion_speed.py b/studio/backend/core/inference/diffusion_speed.py
index 991a07d694f..c5a6ad432b0 100644
--- a/studio/backend/core/inference/diffusion_speed.py
+++ b/studio/backend/core/inference/diffusion_speed.py
@@ -16,6 +16,7 @@ from typing import Any, Callable, Iterator, Optional
from . import diffusion_compile_config as compile_config
from .diffusion_bg_compile import eager_forced as _bg_eager_forced
+from .diffusion_bg_compile import yield_point as _bg_yield_point
from . import diffusion_gguf_compile as gguf_compile
from .diffusion_autoopt import targets as _targets
@@ -1166,12 +1167,14 @@ def _compile_repeated_blocks(
except Exception as exc: # noqa: BLE001 - optimisation only
_warn(logger, "sdpa aot cache", exc)
if unet is not None:
- # Whole-module U-Net compile: dynamic always False, so each new (height, width, batch) recompiles.
+ # dynamic always False, so each new (height, width, batch) recompiles.
unet_kwargs: dict[str, Any] = {"fullgraph": kwargs["fullgraph"], "dynamic": False}
if max_autotune:
unet_kwargs["mode"] = "max-autotune-no-cudagraphs"
if filter_reductions:
pin_reduction_configs(unet_kwargs, logger)
+ if unet_compiles_regionally(unet, max_autotune = max_autotune):
+ return _compile_unet_blocks(unet, unet_kwargs, logger)
try:
unet.compile(**unet_kwargs)
return True
@@ -1272,6 +1275,70 @@ def _compile_repeated_blocks(
return engaged
+def _unet_blocks(unet: Any) -> list:
+ names = set(_targets.UNET_REGIONAL_BLOCKS)
+ try:
+ return [m for m in unet.modules() if m is not unet and type(m).__name__ in names]
+ except Exception: # noqa: BLE001 - not a torch module
+ return []
+
+
+def unet_compiles_regionally(unet: Any, *, max_autotune: bool = False) -> bool:
+ return (
+ unet is not None
+ and not max_autotune
+ and _targets.unet_regional_enabled()
+ and bool(_unet_blocks(unet))
+ )
+
+
+_UNET_RECOMPILE_LIMIT = 256
+
+
+def _compile_unet_blocks(unet: Any, unet_kwargs: dict[str, Any], logger: Any) -> bool:
+ """Compile each repeated U-Net block (guarded; the outer forward stays eager for the CUDA graph)."""
+ count = 0
+ try:
+ # One ResnetBlock2D forward serves ~14 graphs per (height, width, batch): 64 would stop at the 5th size.
+ import torch
+ dynamo_cfg = getattr(getattr(torch, "_dynamo", None), "config", None)
+ for _limit_attr in ("recompile_limit", "cache_size_limit"):
+ if dynamo_cfg is not None and hasattr(dynamo_cfg, _limit_attr):
+ current = compile_config.get_knob(_DYNAMO_MODULE, _limit_attr) or 0
+ compile_config.set_knob(
+ _DYNAMO_MODULE, _limit_attr, max(current, _UNET_RECOMPILE_LIMIT)
+ )
+ except Exception as exc: # noqa: BLE001 - optimisation only
+ _warn(logger, "unet recompile limit", exc)
+ try:
+ for module in _unet_blocks(unet):
+ module.compile(**unet_kwargs)
+ count += 1
+ except Exception as exc: # noqa: BLE001 - optimisation only
+ _warn(logger, "unet regional compile", exc)
+ for module in unet.modules():
+ if module is not unet and getattr(module, "_compiled_call_impl", None) is not None:
+ module._compiled_call_impl = None
+ return False
+ if not count:
+ return False
+ guard_compiled_blocks(unet, logger)
+ try:
+ unet._unsloth_regional_compile_kwargs = dict(unet_kwargs)
+ unet._unsloth_bg_yield = True
+ except Exception: # noqa: BLE001 - not a settable module (tests/fakes)
+ pass
+ if logger is not None:
+ logger.info(
+ "diffusion.speed: %s compiles per block (%d blocks: %s); set %s=0 for the whole-module compile",
+ type(unet).__name__,
+ count,
+ ", ".join(_targets.UNET_REGIONAL_BLOCKS),
+ _targets.UNET_REGIONAL_ENV,
+ )
+ return True
+
+
def _install_compile_patches(
stage: str,
transformer: Any,
@@ -1429,6 +1496,7 @@ class _CompileGuard:
def guarded(*args: Any, **kwargs: Any) -> Any:
# Compile still in flight in the background: entering it here would compile inline.
if guard.error is None and not _bg_eager_forced():
+ _bg_yield_point()
compile_config.apply()
try:
return compiled(*args, **kwargs)
@@ -1485,6 +1553,13 @@ def _guarded_dits(pipe: Any) -> list:
extra = getattr(pipe, attr, None)
if extra is not None and extra not in dits:
dits.append(extra)
+ unet = _denoiser_unet(pipe)
+ if (
+ unet is not None
+ and unet not in dits
+ and getattr(unet, "_unsloth_compile_guard", None) is not None
+ ):
+ dits.append(unet)
return dits
diff --git a/studio/backend/tests/test_diffusion_unet_block_compile.py b/studio/backend/tests/test_diffusion_unet_block_compile.py
new file mode 100644
index 00000000000..499c5635df1
--- /dev/null
+++ b/studio/backend/tests/test_diffusion_unet_block_compile.py
@@ -0,0 +1,445 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
+
+"""Per-block U-Net compile (SDXL) and the background compile that a render pauses between blocks instead of waiting
+for the whole compile. CPU only: ``Module.compile`` is lazy, and the "compiled" block calls below are stand-ins
+that take time like a block graph's first compile."""
+
+from __future__ import annotations
+
+import logging
+import threading
+import time
+from types import SimpleNamespace
+
+import pytest
+
+torch = pytest.importorskip("torch")
+
+from core.inference import diffusion_bg_compile as bg # noqa: E402
+from core.inference import diffusion_speed as ds # noqa: E402
+from core.inference.diffusion_autoopt import targets # noqa: E402
+
+LOG = logging.getLogger("test_unet_block_compile")
+
+
+ "raw, on",
+ [
+ (None, True),
+ ("", True),
+ ("1", True),
+ ("0", False),
+ ("off", False),
+ ("False", False),
+ ("no", False),
+ ],
+)
+def test_unet_regional_switch(monkeypatch, raw, on):
+ if raw is None:
+ monkeypatch.delenv(targets.UNET_REGIONAL_ENV, raising = False)
+ else:
+ monkeypatch.setenv(targets.UNET_REGIONAL_ENV, raw)
+ assert targets.unet_regional_enabled() is on
+ unet = _tiny_unet()
+ assert ds.unet_compiles_regionally(unet) is on
+ assert ds.unet_compiles_regionally(unet, max_autotune = True) is False
+ assert ds.unet_compiles_regionally(None) is False
+ # nothing to compile per block (not a torch module, or no repeated block): the whole-module compile
+ assert ds.unet_compiles_regionally(object()) is False
+ assert ds.unet_compiles_regionally(torch.nn.Linear(2, 2)) is False
+
+
+ "raw, eager_s, want",
+ [
+ (None, None, 0.0),
+ (None, 2.8, 2.8),
+ (None, 99.0, 30.0),
+ (None, -1.0, 0.0),
+ (None, float("nan"), 0.0),
+ ("5", 2.8, 5.0),
+ ("0", 2.8, 0.0),
+ ("junk", 2.8, 2.8),
+ ],
+)
+def test_slice_seconds(monkeypatch, raw, eager_s, want):
+ if raw is None:
+ monkeypatch.delenv(bg.SLICE_ENV, raising = False)
+ else:
+ monkeypatch.setenv(bg.SLICE_ENV, raw)
+ assert bg.slice_seconds(eager_s) == pytest.approx(want)
+
+
+def _tiny_unet():
+ diffusers = pytest.importorskip("diffusers")
+ return diffusers.UNet2DConditionModel(
+ sample_size = 8,
+ in_channels = 4,
+ out_channels = 4,
+ layers_per_block = 1,
+ block_out_channels = (8, 16),
+ down_block_types = ("DownBlock2D", "CrossAttnDownBlock2D"),
+ up_block_types = ("CrossAttnUpBlock2D", "UpBlock2D"),
+ cross_attention_dim = 8,
+ attention_head_dim = 2,
+ norm_num_groups = 4,
+ ).eval()
+
+
+def _compiled_blocks(unet):
+ return [
+ m
+ for m in unet.modules()
+ if m is not unet and getattr(m, "_compiled_call_impl", None) is not None
+ ]
+
+
+def _restore_dynamo_limits():
+ import torch._dynamo
+
+ cfg = torch._dynamo.config
+ saved = {a: getattr(cfg, a) for a in ("recompile_limit", "cache_size_limit") if hasattr(cfg, a)}
+ yield
+ for a, v in saved.items():
+ setattr(cfg, a, v)
+
+
+def test_unet_compiles_its_repeated_blocks_not_the_whole_module(
+ monkeypatch, _restore_dynamo_limits
+):
+ monkeypatch.delenv(targets.UNET_REGIONAL_ENV, raising = False)
+ unet = _tiny_unet()
+ pipe = SimpleNamespace(unet = unet)
+ assert ds._compile_repeated_blocks(pipe, LOG) is True
+ assert getattr(unet, "_compiled_call_impl", None) is None
+ names = {type(m).__name__ for m in _compiled_blocks(unet)}
+ assert names == set(targets.UNET_REGIONAL_BLOCKS)
+ # every compiled block sits behind the compile guard (fallback to eager, background-compile gating)
+ assert all(
+ getattr(m._compiled_call_impl, "_unsloth_compile_guard", None)
+ for m in _compiled_blocks(unet)
+ )
+ assert unet._unsloth_bg_yield is True
+ assert unet._unsloth_regional_compile_kwargs == {"fullgraph": True, "dynamic": False}
+ # the guard is found like a DiT's, so a compile failure is reported
+ assert unet in ds._guarded_dits(pipe)
+ assert ds.compile_fallback_error(pipe) is None
+ unet._unsloth_compile_guard.error = "InductorError: boom"
+ assert ds.compile_fallback_error(pipe) == "InductorError: boom"
+
+
+def test_unet_kill_switch_keeps_the_whole_module_compile(monkeypatch, _restore_dynamo_limits):
+ monkeypatch.setenv(targets.UNET_REGIONAL_ENV, "0")
+ unet = _tiny_unet()
+ assert ds._compile_repeated_blocks(SimpleNamespace(unet = unet), LOG) is True
+ assert getattr(unet, "_compiled_call_impl", None) is not None
+ assert _compiled_blocks(unet) == [] # no block compiled on its own
+ assert not getattr(unet, "_unsloth_bg_yield", False)
+
+
+def test_max_autotune_keeps_the_whole_module_compile(monkeypatch, _restore_dynamo_limits):
+ monkeypatch.delenv(targets.UNET_REGIONAL_ENV, raising = False)
+ unet = _tiny_unet()
+ assert ds._compile_repeated_blocks(SimpleNamespace(unet = unet), LOG, max_autotune = True) is True
+ assert getattr(unet, "_compiled_call_impl", None) is not None
+
+
+def test_compile_cache_key_names_the_block_graphs(monkeypatch):
+ from core.inference.diffusion_autoopt import plan
+
+ unet = _tiny_unet()
+ pipe = SimpleNamespace(unet = unet, vae = None)
+ kw = dict(speed_mode = "default", fullgraph = True, offload_below_hooks = False, deferred = True)
+ monkeypatch.delenv(targets.UNET_REGIONAL_ENV, raising = False)
+ assert plan.compile_cache_kwargs(pipe, **kw)["unet_graph"] == "blocks"
+ monkeypatch.setenv(targets.UNET_REGIONAL_ENV, "0")
+ assert "unet_graph" not in plan.compile_cache_kwargs(pipe, **kw)
+ # a DiT pipe's key is unchanged
+ monkeypatch.delenv(targets.UNET_REGIONAL_ENV, raising = False)
+ assert "unet_graph" not in plan.compile_cache_kwargs(SimpleNamespace(vae = None), **kw)
+
+
+class _Blocks(torch.nn.Module):
+ """A denoiser of N guarded blocks whose first compiled call takes ``compile_s`` (a block graph compile)."""
+
+ def __init__(
+ self,
+ n: int = 6,
+ compile_s: float = 0.15,
+ yields: bool = True,
+ ) -> None:
+ super().__init__()
+ self.blocks = torch.nn.ModuleList(torch.nn.Linear(4, 4) for _ in range(n))
+ self.compiled_calls = 0
+ self.compiled_blocks: set = set()
+ for i, blk in enumerate(self.blocks):
+ blk._compiled_call_impl = self._fake_compiled(i, blk, compile_s)
+ ds.guard_compiled_blocks(self, LOG)
+ if yields:
+ self._unsloth_bg_yield = True
+
+ def _fake_compiled(self, index, blk, compile_s):
+ def compiled(*args, **kwargs):
+ if index not in self.compiled_blocks:
+ time.sleep(compile_s)
+ self.compiled_blocks.add(index)
+ self.compiled_calls += 1
+ return blk.forward(*args, **kwargs)
+
+ return compiled
+
+ def forward(self, x):
+ for blk in self.blocks:
+ x = blk(x)
+ return x
+
+
+def _armed(net):
+ job = bg.arm(net, logger = LOG)
+ assert job is not None
+ with torch.no_grad(), bg.force_eager():
+ net(torch.randn(2, 4)) # records the sample; the guard routes the blocks eager
+ assert net.compiled_calls == 0 and len(job.samples) == 1
+ return job
+
+
+def _wait_for(pred, timeout = 10.0):
+ t0 = time.time()
+ while not pred():
+ if time.time() - t0 > timeout:
+ raise AssertionError("timed out")
+ time.sleep(0.01)
+
+
+def test_a_render_pauses_the_compile_between_blocks_and_it_resumes(monkeypatch):
+ monkeypatch.delenv(bg.YIELD_ENV, raising = False)
+ net = _Blocks(n = 6, compile_s = 0.15)
+ job = _armed(net)
+ assert job.yields()
+ assert job.kick() and job.compiling()
+ _wait_for(lambda: len(net.compiled_blocks) >= 1)
+ waited, paused = job.pause(budget_s = 0.0)
+ assert paused
+ assert waited < 0.15 + 0.5 # at most the block graph in flight
+ done_at_pause = len(net.compiled_blocks)
+ assert done_at_pause < 6
+ time.sleep(0.5) # parked: no progress while the render runs
+ assert len(net.compiled_blocks) == done_at_pause
+ assert job.compiling()
+ # the render runs eager beside the parked compile
+ with torch.no_grad(), bg.force_eager():
+ calls = net.compiled_calls
+ net(torch.randn(2, 4))
+ assert net.compiled_calls == calls
+ job.resume()
+ _wait_for(lambda: job.state == "done")
+ assert len(net.compiled_blocks) == 6
+ assert job.pauses == 1 and job.paused_s >= 0.4
+ assert job.describe()["pauses"] == 1
+
+
+def test_a_compile_that_ends_inside_the_slice_is_not_paused(monkeypatch):
+ monkeypatch.delenv(bg.YIELD_ENV, raising = False)
+ net = _Blocks(n = 4, compile_s = 0.3)
+ job = _armed(net)
+ job.kick()
+ _wait_for(lambda: job.graph_s is not None) # one block graph timed
+ assert job.graph_s == pytest.approx(0.3, abs = 0.15)
+ waited, paused = job.pause(budget_s = 5.0)
+ assert not paused and job.state == "done"
+ assert waited < 5.0
+ job.resume() # harmless without a pause held
+
+
+def test_the_pause_is_asked_one_block_graph_before_the_slice_ends(monkeypatch):
+ monkeypatch.delenv(bg.YIELD_ENV, raising = False)
+ net = _Blocks(n = 12, compile_s = 0.4)
+ job = _armed(net)
+ job.kick()
+ _wait_for(lambda: job.graph_s is not None)
+ waited, paused = job.pause(budget_s = 1.0)
+ assert paused
+ # asked at ~0.6 s, the graph in flight ends by ~1.0 s (+ its own jitter); never slice + a whole graph
+ assert 0.5 <= waited < 1.0 + 0.3
+ job.resume()
+ _wait_for(lambda: job.state == "done")
+
+
+def test_back_to_back_renders_cannot_starve_the_compile(monkeypatch):
+ monkeypatch.delenv(bg.YIELD_ENV, raising = False)
+ net = _Blocks(n = 5, compile_s = 0.3)
+ job = _armed(net)
+ job.kick()
+ renders = 0
+ while job.compiling() and renders < 50:
+ _, paused = job.pause(budget_s = 0.0) # every render asks at once
+ if paused:
+ job.resume()
+ renders += 1
+ _wait_for(lambda: job.state == "done")
+ # each resume owes one block graph before the next park: at most one render per graph (+ the first)
+ assert renders <= 5 + 1
+ assert len(net.compiled_blocks) == 5
+
+
+def test_nested_pause_runs_no_second_slice(monkeypatch):
+ monkeypatch.delenv(bg.YIELD_ENV, raising = False)
+ net = _Blocks(n = 6, compile_s = 0.2)
+ job = _armed(net)
+ job.kick()
+ _, paused = job.pause(budget_s = 0.0)
+ assert paused
+ t0 = time.time()
+ _, paused2 = job.pause(budget_s = 5.0) # a hires pass inside the paused render
+ assert paused2 and time.time() - t0 < 1.0
+ job.resume()
+ time.sleep(0.4)
+ assert job.compiling() # still held by the outer render
+ job.resume()
+ _wait_for(lambda: job.state == "done")
+
+
+def test_close_ends_a_parked_compile(monkeypatch):
+ monkeypatch.delenv(bg.YIELD_ENV, raising = False)
+ net = _Blocks(n = 6, compile_s = 0.15)
+ job = _armed(net)
+ job.kick()
+ _, paused = job.pause(budget_s = 0.0)
+ assert paused
+ thread = job._thread
+ job.close(timeout = 5.0)
+ assert not thread.is_alive()
+ assert job.state == "failed" and "closed" in (job.error or "")
+
+
[email protected]("env, flag", [("0", True), (None, False)])
+def test_without_yield_a_pause_waits_for_the_whole_compile(monkeypatch, env, flag):
+ if env is None:
+ monkeypatch.delenv(bg.YIELD_ENV, raising = False)
+ else:
+ monkeypatch.setenv(bg.YIELD_ENV, env)
+ net = _Blocks(n = 4, compile_s = 0.1, yields = flag)
+ job = _armed(net)
+ assert not job.yields()
+ job.kick()
+ _wait_for(lambda: len(net.compiled_blocks) >= 1)
+ _, paused = job.pause(budget_s = 0.0)
+ # no yield point is armed: the thread never parks, the pause returns when the compile ends
+ assert job.state == "done" and len(net.compiled_blocks) == 4
+ job.resume()
+
+
+def test_yield_point_is_a_no_op_off_the_compile_thread():
+ bg.yield_point() # no active compile on this thread
+
+
+class _FakeBg:
+ def __init__(
+ self,
+ yields: bool,
+ compiling: bool = True,
+ ) -> None:
+ self._yields, self._compiling = yields, compiling
+ self.calls: list = []
+
+ def compiling(self):
+ return self._compiling
+
+ def yields(self):
+ return self._yields
+
+ def pause(
+ self,
+ cancel = None,
+ budget_s = 0.0,
+ ):
+ self.calls.append(("pause", budget_s))
+ return 0.1, True
+
+ def wait(self, cancel = None):
+ self.calls.append(("wait",))
+ return 0.2
+
+
+def _state(loras = ()):
+ pipe = SimpleNamespace(_unsloth_loras = loras, _unsloth_eager_render_s = 2.5)
+ return SimpleNamespace(pipe = pipe, speed_optims = ())
+
+
+def test_render_pauses_a_yielding_compile_for_one_slice(monkeypatch):
+ from core.inference import diffusion as d
+
+ monkeypatch.delenv(bg.SLICE_ENV, raising = False)
+ fake = _FakeBg(yields = True)
+ assert d._wait_or_pause_bg_compile(fake, _state(), None, threading.Event()) is fake
+ assert fake.calls == [("pause", 2.5)]
+
+
+ "yields, loras, attached",
+ [(False, None, ()), (True, [("some/lora", 1.0)], ()), (True, None, (("a", "p", 1.0),))],
+)
+def test_render_waits_for_the_whole_compile_otherwise(yields, loras, attached):
+ from core.inference import diffusion as d
+
+ fake = _FakeBg(yields = yields)
+ assert d._wait_or_pause_bg_compile(fake, _state(attached), loras, threading.Event()) is None
+ assert fake.calls == [("wait",)]
+
+
+def test_no_compile_in_flight_neither_waits_nor_pauses():
+ from core.inference import diffusion as d
+
+ fake = _FakeBg(yields = True, compiling = False)
+ assert d._wait_or_pause_bg_compile(fake, _state(), None, threading.Event()) is None
+ assert d._wait_or_pause_bg_compile(None, _state(), None, threading.Event()) is None
+ assert fake.calls == []
+
+
+def test_note_eager_render_keeps_the_fastest_eager_render():
+ from core.inference import diffusion as d
+
+ state = SimpleNamespace(pipe = SimpleNamespace(), speed_optims = ())
+ d._note_eager_render(state, False, 4.1)
+ d._note_eager_render(state, False, 2.8)
+ d._note_eager_render(state, True, 10.1)
+ assert state.pipe._unsloth_eager_render_s == pytest.approx(2.8)
+ compiled = SimpleNamespace(pipe = state.pipe, speed_optims = ("compiled",))
+ d._note_eager_render(compiled, False, 1.0) # a compiled render is not an eager one
+ assert state.pipe._unsloth_eager_render_s == pytest.approx(2.8)
+ d._note_eager_render(compiled, True, 2.0) # eager beside a pending compile
+ assert state.pipe._unsloth_eager_render_s == pytest.approx(2.0)
+
+
+def _fake_pool(monkeypatch):
+ from torch._inductor import async_compile
+
+ calls: list = []
+ monkeypatch.setattr(
+ async_compile.AsyncCompile, "warm_pool", classmethod(lambda cls: calls.append("warm"))
+ )
+ monkeypatch.setattr(
+ async_compile.AsyncCompile, "wakeup", classmethod(lambda cls: calls.append("wakeup"))
+ )
+ monkeypatch.setattr(bg, "_POOL_WARMED", False)
+ monkeypatch.delenv(bg.WARM_POOL_ENV, raising = False)
+ monkeypatch.delenv("UNSLOTH_DIFFUSION_BG_COMPILE", raising = False)
+ return calls
+
+
+def test_warm_compile_pool_starts_the_workers_once(_fake_pool):
+ assert bg.warm_compile_pool(LOG) is True
+ assert bg.warm_compile_pool(LOG) is False
+ assert _fake_pool == ["warm", "wakeup"]
+
+
+ "env", ["UNSLOTH_DIFFUSION_BG_COMPILE_WARM_POOL", "UNSLOTH_DIFFUSION_BG_COMPILE"]
+)
+def test_warm_compile_pool_switches(monkeypatch, _fake_pool, env):
+ monkeypatch.setenv(env, "0")
+ assert bg.warm_compile_pool(LOG) is False
+ assert _fake_pool == []