Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
Studio: compile the SDXL U-Net per block so the 4th image no longer waits for the whole compile (#16085)
Daniel Han8a021506bd91
diff --git a/studio/backend/core/inference/diffusion.py b/studio/backend/core/inference/diffusion.pyindex a3d9b1cb24e..99afcfee4bc 100644--- a/studio/backend/core/inference/diffusion.py+++ b/studio/backend/core/inference/diffusion.py@@ -3733,6 +3733,52 @@ def _progress_latch_enabled() -> bool:)+def _wait_or_pause_bg_compile(bg: Any, state: Any, loras: Any, cancel: Any) -> Any:+ """Wait for an in-flight bg compile, or (per-block) let it run one slice then park it; returns the paused+ compile for the caller to resume, else None. LoRA renders always wait (they would change the module mid-trace)."""+ if bg is None or not bg.compiling():+ return None+ pipe = getattr(state, "pipe", None)+ if bg.yields() and not _has_active_lora(loras) and not getattr(pipe, "_unsloth_loras", ()):+ budget = bg_compile.slice_seconds(getattr(pipe, "_unsloth_eager_render_s", None))+ waited, paused = bg.pause(cancel, budget_s = budget)+ if paused:+ logger.info(+ "diffusion.bg_compile: render paused the background compile after %.1f s (slice %.1f s); this "+ "render runs eager and the compile resumes after it",+ waited,+ budget,+ )+ return bg+ logger.info(+ "diffusion.bg_compile: render waited %.1f s for the background compile (slice %.1f s)",+ waited,+ budget,+ )+ return None+ logger.info("diffusion.bg_compile: render waits for the background compile to finish")+ waited = bg.wait(cancel)+ logger.info("diffusion.bg_compile: render waited %.1f s for the background compile", waited)+ return None+++def _note_eager_render(state: Any, bg_eager: bool, seconds: float) -> None:+ """Keeps this load's fastest eager render (the background compile's slice): renders before the deferred+ profile, and those that ran eager beside a pending compile."""+ pipe = getattr(state, "pipe", None)+ if pipe is None or not (seconds > 0):+ return+ eager = bg_eager or "compiled" not in tuple(getattr(state, "speed_optims", None) or ())+ if not eager:+ return+ try:+ best = getattr(pipe, "_unsloth_eager_render_s", None)+ if not isinstance(best, (int, float)) or seconds < best:+ pipe._unsloth_eager_render_s = float(seconds)+ except Exception: # noqa: BLE001 - a heuristic only+ pass++def _bg_compile_module(pipe: Any,*,@@ -13216,29 +13262,28 @@ class DiffusionBackend:if _hires_pass is None:bind_generation_cancel(cancel)with nullcontext() if _hires_pass is not None else self._generation_slot(cancel):- # Never render beside a background compile; wait outside the state lock so status polls keep answering.+ # Wait outside the state lock so status polls keep answering.pending_bg = getattr(self._state, "bg_compile", None)- if pending_bg is not None and pending_bg.compiling():- logger.info(- "diffusion.bg_compile: render waits for the background compile to finish"- )- waited = pending_bg.wait(cancel)- logger.info(- "diffusion.bg_compile: render waited %.1f s for the background compile", waited- )- with self._lock:- state = self._state- if state is None:- raise RuntimeError(DIFFUSION_NOT_LOADED_MSG)- if cancel.is_set():- raise RuntimeError(DIFFUSION_CANCELLED_MSG)- # The slot admits on a zero fence, which a COMMITTED replacement also satisfies (#9448)- loaded_id = load_identity(state.repo_id, state.base_repo, state.family.name)- if expected_load is not None and expected_load != loaded_id:- raise DiffusionModelReplacedError(expected_load, loaded_id)- # Publish active step-0 state before slow setup, so a reload's mount probe does not read idle.- self._gen = _GenState(total_steps = steps)- begin_attention_run(state)+ paused_bg = _wait_or_pause_bg_compile(pending_bg, self._state, loras, cancel)+ try:+ with self._lock:+ state = self._state+ if state is None:+ raise RuntimeError(DIFFUSION_NOT_LOADED_MSG)+ if cancel.is_set():+ raise RuntimeError(DIFFUSION_CANCELLED_MSG)+ # The slot admits on a zero fence, which a COMMITTED replacement also satisfies (#9448)+ loaded_id = load_identity(state.repo_id, state.base_repo, state.family.name)+ if expected_load is not None and expected_load != loaded_id:+ raise DiffusionModelReplacedError(expected_load, loaded_id)+ # Publish active step-0 state before slow setup, so a reload's mount probe does not read idle.+ self._gen = _GenState(total_steps = steps)+ begin_attention_run(state)+ except BaseException:+ if paused_bg is not None:+ paused_bg.resume()+ raise+ render_t0 = time.perf_counter()# Reset in the finally, so a failed or cancelled generation frees its reused outputs.static_skip_pipe = Nonerestore_scheduler: Optional[tuple[Any, Any]] = None@@ -13281,6 +13326,14 @@ class DiffusionBackend:lora_requested = _has_active_lora(loras)# Stay eager while prior adapters are attached: compiling would bake them in.loras_attached = bool(getattr(state.pipe, "_unsloth_loras", ()))+ if (+ state.speed_deferred+ and state.generation_count == 1+ and not aux_pass+ and str(getattr(state, "device", "")).startswith("cuda")+ ):+ # 2nd image: warm inductor's workers so the deferred compile does not wait on them.+ bg_compile.warm_compile_pool(logger)if (state.speed_deferredand state.generation_count >= 2@@ -14433,6 +14486,8 @@ class DiffusionBackend:if applied_embeddings is not None:result["embeddings"] = list(applied_embeddings.used)result["embeddings_skipped"] = list(applied_embeddings.skipped)+ if not aux_pass and _hires_pass is None:+ _note_eager_render(state, bg_eager, time.perf_counter() - render_t0)logger.info("diffusion.generated: %s",format_generation_for_log(@@ -14472,6 +14527,8 @@ class DiffusionBackend:bg.kick()except Exception as exc: # noqa: BLE001 - the next render compiles inlinelogger.warning("diffusion.bg_compile: kick failed: %s", exc)+ if paused_bg is not None:+ paused_bg.resume()with self._generation_cancel_lock:if self._active_generate_cancel is cancel:self._active_generate_cancel = Nonediff --git a/studio/backend/core/inference/diffusion_autoopt/plan.py b/studio/backend/core/inference/diffusion_autoopt/plan.pyindex 9ead0d04ef7..5f754aaa0cf 100644--- a/studio/backend/core/inference/diffusion_autoopt/plan.py+++ b/studio/backend/core/inference/diffusion_autoopt/plan.py@@ -272,4 +272,7 @@ def compile_cache_kwargs(if offload_below_hooks:# An offloaded denoiser compiles its blocks below the hooks, a different graph than a traced-hook bundle.kwargs["below_hooks"] = True+ if ds.unet_compiles_regionally(ds._denoiser_unet(pipe), max_autotune = mode == SPEED_MAX):+ # Per-block U-Net graphs: never served a whole-module bundle (nor the other way round).+ kwargs["unet_graph"] = "blocks"return kwargsdiff --git a/studio/backend/core/inference/diffusion_autoopt/targets.py b/studio/backend/core/inference/diffusion_autoopt/targets.pyindex b38e08e008a..3f0dc54b50c 100644--- a/studio/backend/core/inference/diffusion_autoopt/targets.py+++ b/studio/backend/core/inference/diffusion_autoopt/targets.py@@ -15,9 +15,24 @@ HOOKED_ATTRS: tuple[str, ...] = DIT_ATTRS + ("unet",)# Named partitions a modular workflow compiles through a view (MiniMax-H3's reference DiT).PARTITION_ATTRS: tuple[str, ...] = ("transformer_ref",)-# U-Net denoisers ship no ``_repeated_blocks``, so the regional compile cannot reach them; these classes get a-# WHOLE-module static compile (diffusion_speed).+# Whole-module U-Net compile only under max-autotune or UNSLOTH_DIFFUSION_UNET_REGIONAL=0.UNET_WHOLE_COMPILE: frozenset[str] = frozenset({"UNet2DConditionModel"})+# SDXL 1024^2 on A100: 19 graphs, 55 s cold vs 154 s whole. Resnets/samplers are needed: step 97.3 ms eager,+# transformer blocks only 91.2, all four 83.1, whole module 82.5.+UNET_REGIONAL_BLOCKS: tuple[str, ...] = (+ "BasicTransformerBlock",+ "ResnetBlock2D",+ "Downsample2D",+ "Upsample2D",+)+UNET_REGIONAL_ENV = "UNSLOTH_DIFFUSION_UNET_REGIONAL"+++def unet_regional_enabled() -> bool:+ import os+ raw = (os.environ.get(UNET_REGIONAL_ENV) or "").strip().lower()+ return raw not in ("0", "false", "no", "off")+# Component names that are never a denoiser even when they are a ModelMixin with repeated blocks._NOT_DENOISERS = (diff --git a/studio/backend/core/inference/diffusion_bg_compile.py b/studio/backend/core/inference/diffusion_bg_compile.pyindex 74d0f26af91..962b3477a55 100644--- a/studio/backend/core/inference/diffusion_bg_compile.py+++ b/studio/backend/core/inference/diffusion_bg_compile.py@@ -20,6 +20,12 @@ from typing import Any, Callable, Optional_ENV = "UNSLOTH_DIFFUSION_BG_COMPILE"OFFLOADED_ENV = "UNSLOTH_DIFFUSION_BG_COMPILE_OFFLOADED"+YIELD_ENV = "UNSLOTH_DIFFUSION_BG_COMPILE_YIELD"+# Seconds of compile a render lets run before it pauses it; unset = this load's fastest eager render.+SLICE_ENV = "UNSLOTH_DIFFUSION_BG_COMPILE_SLICE_S"+_SLICE_MAX_S = 30.0+# A gap between two yield points longer than this compiled a block graph (a cached block call takes milliseconds).+_GRAPH_SEGMENT_S = 0.25_MAX_SAMPLES = 4# ContextVars, not thread-locals: diffusion_render_thread.run copies the caller's context onto the render thread.@@ -35,6 +41,14 @@ _NO_RECORD: contextvars.ContextVar[bool] = contextvars.ContextVar(_AUXILIARY: contextvars.ContextVar[bool] = contextvars.ContextVar("unsloth_diffusion_auxiliary_pass", default = False)+# The BackgroundCompile whose thread this is (set only on that thread): yield_point's target.+_ACTIVE: contextvars.ContextVar[Optional["BackgroundCompile"]] = contextvars.ContextVar(+ "unsloth_diffusion_bg_active", default = None+)+++class _Closed(BaseException):+ """Raised at a yield point once the compile is closed (unload): ends the paused replay."""def enabled() -> bool:@@ -47,6 +61,67 @@ def offloaded_enabled() -> bool:return raw not in ("0", "false", "no", "off")+def yield_enabled() -> bool:+ raw = (os.environ.get(YIELD_ENV) or "").strip().lower()+ return raw not in ("0", "false", "no", "off")+++WARM_POOL_ENV = "UNSLOTH_DIFFUSION_BG_COMPILE_WARM_POOL"+_POOL_WARMED = False+++def warm_compile_pool(logger: Any = None) -> bool:+ """Start inductor's compile worker pool ahead of the deferred compile."""+ global _POOL_WARMED+ if _POOL_WARMED or not enabled():+ return False+ raw = (os.environ.get(WARM_POOL_ENV) or "").strip().lower()+ if raw in ("0", "false", "no", "off"):+ return False+ _POOL_WARMED = True+ try:+ from . import diffusion_compile_config+ from torch._inductor.async_compile import AsyncCompile++ diffusion_compile_config.apply()+ AsyncCompile.warm_pool()+ AsyncCompile.wakeup()+ except Exception as exc: # noqa: BLE001 - the compile warms its own pool as before+ if logger is not None:+ logger.debug("diffusion.bg_compile: compile pool warm-up skipped: %s", exc)+ return False+ if logger is not None:+ logger.info(+ "diffusion.bg_compile: inductor compile workers starting ahead of the deferred compile"+ )+ return True+++def slice_seconds(eager_render_s: Optional[float]) -> float:+ """Compile seconds a render lets run before it pauses the compile: ``SLICE_ENV`` when set, else the load's+ fastest eager render (0 when none was timed), clamped to [0, 30]."""+ raw = (os.environ.get(SLICE_ENV) or "").strip()+ value: Optional[float] = None+ if raw:+ try:+ value = float(raw)+ except ValueError:+ value = None+ if value is None:+ value = eager_render_s if isinstance(eager_render_s, (int, float)) else 0.0+ if value != value: # NaN+ value = 0.0+ return max(0.0, min(_SLICE_MAX_S, float(value)))+++def yield_point() -> None:+ """Called before a guarded compiled block runs. On a background compile thread with a pause requested, parks+ here (between two block graphs, never inside a trace) until the render resumes it. Elsewhere a no-op."""+ bg = _ACTIVE.get()+ if bg is not None:+ bg._yield_point()++FLOAT_SEEDS_ENV = "UNSLOTH_DIFFUSION_BG_COMPILE_FLOAT_SEEDS"FLOAT_SEEDS_FILE = "bg_compile_float_specializations.json"_FLOAT_SEEDS_VERSION = 1@@ -475,6 +550,94 @@ class BackgroundCompile:self.error: Optional[str] = Noneself.compile_s: Optional[float] = Noneself.eager_generations = 0+ self._cv = threading.Condition(threading.Lock())+ self._pause_depth = 0+ self._parked = False+ # Bumped when the last pause is released: a parked thread leaves on it even if a new pause is already held.+ self._release_epoch = 0+ self.pauses = 0+ self.paused_s = 0.0+ self.yield_points = 0+ # _owe_graph: one graph per resume, so back-to-back renders cannot starve the compile.+ self.graph_s: Optional[float] = None+ self._last_yield: Optional[float] = None+ self._owe_graph = False++ def yields(self) -> bool:+ return bool(getattr(self.module, "_unsloth_bg_yield", False)) and yield_enabled()++ def pause(+ self,+ cancel: Any = None,+ budget_s: float = 0.0,+ poll_s: float = 0.05,+ ) -> tuple[float, bool]:+ """Let the compile run ``budget_s`` more seconds, then park it at its next yield point. Returns+ ``(seconds waited, paused)``; ``paused`` means a pause is held and the caller MUST ``resume()``. A compile+ that ends inside the budget returns ``paused=False`` (the render then runs compiled)."""+ t0 = time.perf_counter()+ thread = self._thread+ if thread is None or thread is threading.current_thread() or not thread.is_alive():+ return 0.0, False+ # Ask one graph_s early: the pause lands after the block graph in flight.+ budget = max(0.0, float(budget_s or 0.0))+ lead = self.graph_s if self.graph_s is not None else budget+ deadline = t0 + (0.0 if self._pause_depth else max(0.0, budget - lead))+ while thread.is_alive() and not (cancel is not None and cancel.is_set()):+ left = deadline - time.perf_counter()+ if left <= 0:+ break+ thread.join(min(poll_s * 5, left))+ if not thread.is_alive():+ return time.perf_counter() - t0, False+ with self._cv:+ self._pause_depth += 1+ self.pauses += 1+ self._cv.notify_all()+ while thread.is_alive() and not self._parked:+ if cancel is not None and cancel.is_set():+ break+ self._cv.wait(poll_s)+ return time.perf_counter() - t0, True++ def resume(self) -> None:+ with self._cv:+ self._pause_depth = max(0, self._pause_depth - 1)+ if not self._pause_depth:+ self._release_epoch += 1+ # the next pause() waits for a fresh park, after the block graph this release owes+ self._parked = False+ self._cv.notify_all()++ def _yield_point(self) -> None:+ self.yield_points += 1+ if self._closed.is_set():+ raise _Closed()+ now = time.perf_counter()+ if self._last_yield is not None and now - self._last_yield > _GRAPH_SEGMENT_S:+ seg = now - self._last_yield+ self.graph_s = seg if self.graph_s is None else 0.5 * (self.graph_s + seg)+ self._owe_graph = False+ self._last_yield = now+ if not self._pause_depth or self._owe_graph:+ return+ with self._cv:+ if not self._pause_depth:+ return+ self._parked = True+ self._cv.notify_all()+ epoch = self._release_epoch+ try:+ while self._release_epoch == epoch and not self._closed.is_set():+ self._cv.wait(0.5)+ finally:+ self._parked = False+ end = time.perf_counter()+ self.paused_s += end - now+ self._last_yield = end+ self._owe_graph = True+ if self._closed.is_set():+ raise _Closed()def install(self) -> bool:try:@@ -648,6 +811,7 @@ class BackgroundCompile:def _run(self) -> None:t0 = time.perf_counter()token = _NO_CAPTURE.set(True)+ _ACTIVE.set(self if self.yields() else None)try:import torch@@ -675,13 +839,20 @@ class BackgroundCompile:self._finish("done")if self.logger is not None:self.logger.info(- "diffusion.bg_compile: %s compiled in the background in %.1f s%s; the next render runs compiled",+ "diffusion.bg_compile: %s compiled in the background in %.1f s%s%s; the next render runs compiled",type(self.module).__name__,self.compile_s,f" ({self.float_seeds} float specialisation(s) pre-seeded)"if self.float_seedselse "",+ f" (paused {self.pauses}x for renders, {self.paused_s:.1f} s, compiling "+ f"{self.compile_s - self.paused_s:.1f} s)"+ if self.pauses+ else "",)+ except _Closed:+ self.compile_s = time.perf_counter() - t0+ self._finish("failed", "closed before the compile finished")except BaseException as exc: # noqa: BLE001 - never kills the process; the next render compiles inlineself.compile_s = time.perf_counter() - t0self._finish(@@ -699,6 +870,9 @@ class BackgroundCompile:exc.__traceback__ = Nonefinally:_NO_CAPTURE.reset(token)+ _ACTIVE.set(None)+ with self._cv:+ self._cv.notify_all()def _seed_floats(self) -> Optional[tuple]:"""Pre-seed the SymFloat specialisations this key's last compile learned; returns what ``_keep_floats`` needs."""@@ -711,6 +885,8 @@ class BackgroundCompile:def close(self, timeout: Optional[float] = None) -> None:"""Stop recording and wait for an in-flight compile (it holds the module); used on unload."""self._closed.set()+ with self._cv:+ self._cv.notify_all() # a parked compile wakes, sees closed and endsself._remove_hook()thread = self._threadif thread is not None and thread.is_alive() and thread is not threading.current_thread():@@ -731,6 +907,8 @@ class BackgroundCompile:"samples": len(self.samples),"compile_s": None if self.compile_s is None else round(self.compile_s, 2),"eager_generations": self.eager_generations,+ "pauses": self.pauses,+ "paused_s": round(self.paused_s, 2),"float_seeds": self.float_seeds,"error": self.error,}diff --git a/studio/backend/core/inference/diffusion_speed.py b/studio/backend/core/inference/diffusion_speed.pyindex 991a07d694f..c5a6ad432b0 100644--- a/studio/backend/core/inference/diffusion_speed.py+++ b/studio/backend/core/inference/diffusion_speed.py@@ -16,6 +16,7 @@ from typing import Any, Callable, Iterator, Optionalfrom . import diffusion_compile_config as compile_configfrom .diffusion_bg_compile import eager_forced as _bg_eager_forced+from .diffusion_bg_compile import yield_point as _bg_yield_pointfrom . import diffusion_gguf_compile as gguf_compilefrom .diffusion_autoopt import targets as _targets@@ -1166,12 +1167,14 @@ def _compile_repeated_blocks(except Exception as exc: # noqa: BLE001 - optimisation only_warn(logger, "sdpa aot cache", exc)if unet is not None:- # Whole-module U-Net compile: dynamic always False, so each new (height, width, batch) recompiles.+ # dynamic always False, so each new (height, width, batch) recompiles.unet_kwargs: dict[str, Any] = {"fullgraph": kwargs["fullgraph"], "dynamic": False}if max_autotune:unet_kwargs["mode"] = "max-autotune-no-cudagraphs"if filter_reductions:pin_reduction_configs(unet_kwargs, logger)+ if unet_compiles_regionally(unet, max_autotune = max_autotune):+ return _compile_unet_blocks(unet, unet_kwargs, logger)try:unet.compile(**unet_kwargs)return True@@ -1272,6 +1275,70 @@ def _compile_repeated_blocks(return engaged+def _unet_blocks(unet: Any) -> list:+ names = set(_targets.UNET_REGIONAL_BLOCKS)+ try:+ return [m for m in unet.modules() if m is not unet and type(m).__name__ in names]+ except Exception: # noqa: BLE001 - not a torch module+ return []+++def unet_compiles_regionally(unet: Any, *, max_autotune: bool = False) -> bool:+ return (+ unet is not None+ and not max_autotune+ and _targets.unet_regional_enabled()+ and bool(_unet_blocks(unet))+ )+++_UNET_RECOMPILE_LIMIT = 256+++def _compile_unet_blocks(unet: Any, unet_kwargs: dict[str, Any], logger: Any) -> bool:+ """Compile each repeated U-Net block (guarded; the outer forward stays eager for the CUDA graph)."""+ count = 0+ try:+ # One ResnetBlock2D forward serves ~14 graphs per (height, width, batch): 64 would stop at the 5th size.+ import torch+ dynamo_cfg = getattr(getattr(torch, "_dynamo", None), "config", None)+ for _limit_attr in ("recompile_limit", "cache_size_limit"):+ if dynamo_cfg is not None and hasattr(dynamo_cfg, _limit_attr):+ current = compile_config.get_knob(_DYNAMO_MODULE, _limit_attr) or 0+ compile_config.set_knob(+ _DYNAMO_MODULE, _limit_attr, max(current, _UNET_RECOMPILE_LIMIT)+ )+ except Exception as exc: # noqa: BLE001 - optimisation only+ _warn(logger, "unet recompile limit", exc)+ try:+ for module in _unet_blocks(unet):+ module.compile(**unet_kwargs)+ count += 1+ except Exception as exc: # noqa: BLE001 - optimisation only+ _warn(logger, "unet regional compile", exc)+ for module in unet.modules():+ if module is not unet and getattr(module, "_compiled_call_impl", None) is not None:+ module._compiled_call_impl = None+ return False+ if not count:+ return False+ guard_compiled_blocks(unet, logger)+ try:+ unet._unsloth_regional_compile_kwargs = dict(unet_kwargs)+ unet._unsloth_bg_yield = True+ except Exception: # noqa: BLE001 - not a settable module (tests/fakes)+ pass+ if logger is not None:+ logger.info(+ "diffusion.speed: %s compiles per block (%d blocks: %s); set %s=0 for the whole-module compile",+ type(unet).__name__,+ count,+ ", ".join(_targets.UNET_REGIONAL_BLOCKS),+ _targets.UNET_REGIONAL_ENV,+ )+ return True++def _install_compile_patches(stage: str,transformer: Any,@@ -1429,6 +1496,7 @@ class _CompileGuard:def guarded(*args: Any, **kwargs: Any) -> Any:# Compile still in flight in the background: entering it here would compile inline.if guard.error is None and not _bg_eager_forced():+ _bg_yield_point()compile_config.apply()try:return compiled(*args, **kwargs)@@ -1485,6 +1553,13 @@ def _guarded_dits(pipe: Any) -> list:extra = getattr(pipe, attr, None)if extra is not None and extra not in dits:dits.append(extra)+ unet = _denoiser_unet(pipe)+ if (+ unet is not None+ and unet not in dits+ and getattr(unet, "_unsloth_compile_guard", None) is not None+ ):+ dits.append(unet)return ditsdiff --git a/studio/backend/tests/test_diffusion_unet_block_compile.py b/studio/backend/tests/test_diffusion_unet_block_compile.pynew file mode 100644index 00000000000..499c5635df1--- /dev/null+++ b/studio/backend/tests/test_diffusion_unet_block_compile.py@@ -0,0 +1,445 @@+# SPDX-License-Identifier: AGPL-3.0-only+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0++"""Per-block U-Net compile (SDXL) and the background compile that a render pauses between blocks instead of waiting+for the whole compile. CPU only: ``Module.compile`` is lazy, and the "compiled" block calls below are stand-ins+that take time like a block graph's first compile."""++from __future__ import annotations++import logging+import threading+import time+from types import SimpleNamespace++import pytest++torch = pytest.importorskip("torch")++from core.inference import diffusion_bg_compile as bg # noqa: E402+from core.inference import diffusion_speed as ds # noqa: E402+from core.inference.diffusion_autoopt import targets # noqa: E402++LOG = logging.getLogger("test_unet_block_compile")+++ "raw, on",+ [+ (None, True),+ ("", True),+ ("1", True),+ ("0", False),+ ("off", False),+ ("False", False),+ ("no", False),+ ],+)+def test_unet_regional_switch(monkeypatch, raw, on):+ if raw is None:+ monkeypatch.delenv(targets.UNET_REGIONAL_ENV, raising = False)+ else:+ monkeypatch.setenv(targets.UNET_REGIONAL_ENV, raw)+ assert targets.unet_regional_enabled() is on+ unet = _tiny_unet()+ assert ds.unet_compiles_regionally(unet) is on+ assert ds.unet_compiles_regionally(unet, max_autotune = True) is False+ assert ds.unet_compiles_regionally(None) is False+ # nothing to compile per block (not a torch module, or no repeated block): the whole-module compile+ assert ds.unet_compiles_regionally(object()) is False+ assert ds.unet_compiles_regionally(torch.nn.Linear(2, 2)) is False+++ "raw, eager_s, want",+ [+ (None, None, 0.0),+ (None, 2.8, 2.8),+ (None, 99.0, 30.0),+ (None, -1.0, 0.0),+ (None, float("nan"), 0.0),+ ("5", 2.8, 5.0),+ ("0", 2.8, 0.0),+ ("junk", 2.8, 2.8),+ ],+)+def test_slice_seconds(monkeypatch, raw, eager_s, want):+ if raw is None:+ monkeypatch.delenv(bg.SLICE_ENV, raising = False)+ else:+ monkeypatch.setenv(bg.SLICE_ENV, raw)+ assert bg.slice_seconds(eager_s) == pytest.approx(want)+++def _tiny_unet():+ diffusers = pytest.importorskip("diffusers")+ return diffusers.UNet2DConditionModel(+ sample_size = 8,+ in_channels = 4,+ out_channels = 4,+ layers_per_block = 1,+ block_out_channels = (8, 16),+ down_block_types = ("DownBlock2D", "CrossAttnDownBlock2D"),+ up_block_types = ("CrossAttnUpBlock2D", "UpBlock2D"),+ cross_attention_dim = 8,+ attention_head_dim = 2,+ norm_num_groups = 4,+ ).eval()+++def _compiled_blocks(unet):+ return [+ m+ for m in unet.modules()+ if m is not unet and getattr(m, "_compiled_call_impl", None) is not None+ ]+++def _restore_dynamo_limits():+ import torch._dynamo++ cfg = torch._dynamo.config+ saved = {a: getattr(cfg, a) for a in ("recompile_limit", "cache_size_limit") if hasattr(cfg, a)}+ yield+ for a, v in saved.items():+ setattr(cfg, a, v)+++def test_unet_compiles_its_repeated_blocks_not_the_whole_module(+ monkeypatch, _restore_dynamo_limits+):+ monkeypatch.delenv(targets.UNET_REGIONAL_ENV, raising = False)+ unet = _tiny_unet()+ pipe = SimpleNamespace(unet = unet)+ assert ds._compile_repeated_blocks(pipe, LOG) is True+ assert getattr(unet, "_compiled_call_impl", None) is None+ names = {type(m).__name__ for m in _compiled_blocks(unet)}+ assert names == set(targets.UNET_REGIONAL_BLOCKS)+ # every compiled block sits behind the compile guard (fallback to eager, background-compile gating)+ assert all(+ getattr(m._compiled_call_impl, "_unsloth_compile_guard", None)+ for m in _compiled_blocks(unet)+ )+ assert unet._unsloth_bg_yield is True+ assert unet._unsloth_regional_compile_kwargs == {"fullgraph": True, "dynamic": False}+ # the guard is found like a DiT's, so a compile failure is reported+ assert unet in ds._guarded_dits(pipe)+ assert ds.compile_fallback_error(pipe) is None+ unet._unsloth_compile_guard.error = "InductorError: boom"+ assert ds.compile_fallback_error(pipe) == "InductorError: boom"+++def test_unet_kill_switch_keeps_the_whole_module_compile(monkeypatch, _restore_dynamo_limits):+ monkeypatch.setenv(targets.UNET_REGIONAL_ENV, "0")+ unet = _tiny_unet()+ assert ds._compile_repeated_blocks(SimpleNamespace(unet = unet), LOG) is True+ assert getattr(unet, "_compiled_call_impl", None) is not None+ assert _compiled_blocks(unet) == [] # no block compiled on its own+ assert not getattr(unet, "_unsloth_bg_yield", False)+++def test_max_autotune_keeps_the_whole_module_compile(monkeypatch, _restore_dynamo_limits):+ monkeypatch.delenv(targets.UNET_REGIONAL_ENV, raising = False)+ unet = _tiny_unet()+ assert ds._compile_repeated_blocks(SimpleNamespace(unet = unet), LOG, max_autotune = True) is True+ assert getattr(unet, "_compiled_call_impl", None) is not None+++def test_compile_cache_key_names_the_block_graphs(monkeypatch):+ from core.inference.diffusion_autoopt import plan++ unet = _tiny_unet()+ pipe = SimpleNamespace(unet = unet, vae = None)+ kw = dict(speed_mode = "default", fullgraph = True, offload_below_hooks = False, deferred = True)+ monkeypatch.delenv(targets.UNET_REGIONAL_ENV, raising = False)+ assert plan.compile_cache_kwargs(pipe, **kw)["unet_graph"] == "blocks"+ monkeypatch.setenv(targets.UNET_REGIONAL_ENV, "0")+ assert "unet_graph" not in plan.compile_cache_kwargs(pipe, **kw)+ # a DiT pipe's key is unchanged+ monkeypatch.delenv(targets.UNET_REGIONAL_ENV, raising = False)+ assert "unet_graph" not in plan.compile_cache_kwargs(SimpleNamespace(vae = None), **kw)+++class _Blocks(torch.nn.Module):+ """A denoiser of N guarded blocks whose first compiled call takes ``compile_s`` (a block graph compile)."""++ def __init__(+ self,+ n: int = 6,+ compile_s: float = 0.15,+ yields: bool = True,+ ) -> None:+ super().__init__()+ self.blocks = torch.nn.ModuleList(torch.nn.Linear(4, 4) for _ in range(n))+ self.compiled_calls = 0+ self.compiled_blocks: set = set()+ for i, blk in enumerate(self.blocks):+ blk._compiled_call_impl = self._fake_compiled(i, blk, compile_s)+ ds.guard_compiled_blocks(self, LOG)+ if yields:+ self._unsloth_bg_yield = True++ def _fake_compiled(self, index, blk, compile_s):+ def compiled(*args, **kwargs):+ if index not in self.compiled_blocks:+ time.sleep(compile_s)+ self.compiled_blocks.add(index)+ self.compiled_calls += 1+ return blk.forward(*args, **kwargs)++ return compiled++ def forward(self, x):+ for blk in self.blocks:+ x = blk(x)+ return x+++def _armed(net):+ job = bg.arm(net, logger = LOG)+ assert job is not None+ with torch.no_grad(), bg.force_eager():+ net(torch.randn(2, 4)) # records the sample; the guard routes the blocks eager+ assert net.compiled_calls == 0 and len(job.samples) == 1+ return job+++def _wait_for(pred, timeout = 10.0):+ t0 = time.time()+ while not pred():+ if time.time() - t0 > timeout:+ raise AssertionError("timed out")+ time.sleep(0.01)+++def test_a_render_pauses_the_compile_between_blocks_and_it_resumes(monkeypatch):+ monkeypatch.delenv(bg.YIELD_ENV, raising = False)+ net = _Blocks(n = 6, compile_s = 0.15)+ job = _armed(net)+ assert job.yields()+ assert job.kick() and job.compiling()+ _wait_for(lambda: len(net.compiled_blocks) >= 1)+ waited, paused = job.pause(budget_s = 0.0)+ assert paused+ assert waited < 0.15 + 0.5 # at most the block graph in flight+ done_at_pause = len(net.compiled_blocks)+ assert done_at_pause < 6+ time.sleep(0.5) # parked: no progress while the render runs+ assert len(net.compiled_blocks) == done_at_pause+ assert job.compiling()+ # the render runs eager beside the parked compile+ with torch.no_grad(), bg.force_eager():+ calls = net.compiled_calls+ net(torch.randn(2, 4))+ assert net.compiled_calls == calls+ job.resume()+ _wait_for(lambda: job.state == "done")+ assert len(net.compiled_blocks) == 6+ assert job.pauses == 1 and job.paused_s >= 0.4+ assert job.describe()["pauses"] == 1+++def test_a_compile_that_ends_inside_the_slice_is_not_paused(monkeypatch):+ monkeypatch.delenv(bg.YIELD_ENV, raising = False)+ net = _Blocks(n = 4, compile_s = 0.3)+ job = _armed(net)+ job.kick()+ _wait_for(lambda: job.graph_s is not None) # one block graph timed+ assert job.graph_s == pytest.approx(0.3, abs = 0.15)+ waited, paused = job.pause(budget_s = 5.0)+ assert not paused and job.state == "done"+ assert waited < 5.0+ job.resume() # harmless without a pause held+++def test_the_pause_is_asked_one_block_graph_before_the_slice_ends(monkeypatch):+ monkeypatch.delenv(bg.YIELD_ENV, raising = False)+ net = _Blocks(n = 12, compile_s = 0.4)+ job = _armed(net)+ job.kick()+ _wait_for(lambda: job.graph_s is not None)+ waited, paused = job.pause(budget_s = 1.0)+ assert paused+ # asked at ~0.6 s, the graph in flight ends by ~1.0 s (+ its own jitter); never slice + a whole graph+ assert 0.5 <= waited < 1.0 + 0.3+ job.resume()+ _wait_for(lambda: job.state == "done")+++def test_back_to_back_renders_cannot_starve_the_compile(monkeypatch):+ monkeypatch.delenv(bg.YIELD_ENV, raising = False)+ net = _Blocks(n = 5, compile_s = 0.3)+ job = _armed(net)+ job.kick()+ renders = 0+ while job.compiling() and renders < 50:+ _, paused = job.pause(budget_s = 0.0) # every render asks at once+ if paused:+ job.resume()+ renders += 1+ _wait_for(lambda: job.state == "done")+ # each resume owes one block graph before the next park: at most one render per graph (+ the first)+ assert renders <= 5 + 1+ assert len(net.compiled_blocks) == 5+++def test_nested_pause_runs_no_second_slice(monkeypatch):+ monkeypatch.delenv(bg.YIELD_ENV, raising = False)+ net = _Blocks(n = 6, compile_s = 0.2)+ job = _armed(net)+ job.kick()+ _, paused = job.pause(budget_s = 0.0)+ assert paused+ t0 = time.time()+ _, paused2 = job.pause(budget_s = 5.0) # a hires pass inside the paused render+ assert paused2 and time.time() - t0 < 1.0+ job.resume()+ time.sleep(0.4)+ assert job.compiling() # still held by the outer render+ job.resume()+ _wait_for(lambda: job.state == "done")+++def test_close_ends_a_parked_compile(monkeypatch):+ monkeypatch.delenv(bg.YIELD_ENV, raising = False)+ net = _Blocks(n = 6, compile_s = 0.15)+ job = _armed(net)+ job.kick()+ _, paused = job.pause(budget_s = 0.0)+ assert paused+ thread = job._thread+ job.close(timeout = 5.0)+ assert not thread.is_alive()+ assert job.state == "failed" and "closed" in (job.error or "")++[email protected]("env, flag", [("0", True), (None, False)])+def test_without_yield_a_pause_waits_for_the_whole_compile(monkeypatch, env, flag):+ if env is None:+ monkeypatch.delenv(bg.YIELD_ENV, raising = False)+ else:+ monkeypatch.setenv(bg.YIELD_ENV, env)+ net = _Blocks(n = 4, compile_s = 0.1, yields = flag)+ job = _armed(net)+ assert not job.yields()+ job.kick()+ _wait_for(lambda: len(net.compiled_blocks) >= 1)+ _, paused = job.pause(budget_s = 0.0)+ # no yield point is armed: the thread never parks, the pause returns when the compile ends+ assert job.state == "done" and len(net.compiled_blocks) == 4+ job.resume()+++def test_yield_point_is_a_no_op_off_the_compile_thread():+ bg.yield_point() # no active compile on this thread+++class _FakeBg:+ def __init__(+ self,+ yields: bool,+ compiling: bool = True,+ ) -> None:+ self._yields, self._compiling = yields, compiling+ self.calls: list = []++ def compiling(self):+ return self._compiling++ def yields(self):+ return self._yields++ def pause(+ self,+ cancel = None,+ budget_s = 0.0,+ ):+ self.calls.append(("pause", budget_s))+ return 0.1, True++ def wait(self, cancel = None):+ self.calls.append(("wait",))+ return 0.2+++def _state(loras = ()):+ pipe = SimpleNamespace(_unsloth_loras = loras, _unsloth_eager_render_s = 2.5)+ return SimpleNamespace(pipe = pipe, speed_optims = ())+++def test_render_pauses_a_yielding_compile_for_one_slice(monkeypatch):+ from core.inference import diffusion as d++ monkeypatch.delenv(bg.SLICE_ENV, raising = False)+ fake = _FakeBg(yields = True)+ assert d._wait_or_pause_bg_compile(fake, _state(), None, threading.Event()) is fake+ assert fake.calls == [("pause", 2.5)]+++ "yields, loras, attached",+ [(False, None, ()), (True, [("some/lora", 1.0)], ()), (True, None, (("a", "p", 1.0),))],+)+def test_render_waits_for_the_whole_compile_otherwise(yields, loras, attached):+ from core.inference import diffusion as d++ fake = _FakeBg(yields = yields)+ assert d._wait_or_pause_bg_compile(fake, _state(attached), loras, threading.Event()) is None+ assert fake.calls == [("wait",)]+++def test_no_compile_in_flight_neither_waits_nor_pauses():+ from core.inference import diffusion as d++ fake = _FakeBg(yields = True, compiling = False)+ assert d._wait_or_pause_bg_compile(fake, _state(), None, threading.Event()) is None+ assert d._wait_or_pause_bg_compile(None, _state(), None, threading.Event()) is None+ assert fake.calls == []+++def test_note_eager_render_keeps_the_fastest_eager_render():+ from core.inference import diffusion as d++ state = SimpleNamespace(pipe = SimpleNamespace(), speed_optims = ())+ d._note_eager_render(state, False, 4.1)+ d._note_eager_render(state, False, 2.8)+ d._note_eager_render(state, True, 10.1)+ assert state.pipe._unsloth_eager_render_s == pytest.approx(2.8)+ compiled = SimpleNamespace(pipe = state.pipe, speed_optims = ("compiled",))+ d._note_eager_render(compiled, False, 1.0) # a compiled render is not an eager one+ assert state.pipe._unsloth_eager_render_s == pytest.approx(2.8)+ d._note_eager_render(compiled, True, 2.0) # eager beside a pending compile+ assert state.pipe._unsloth_eager_render_s == pytest.approx(2.0)+++def _fake_pool(monkeypatch):+ from torch._inductor import async_compile++ calls: list = []+ monkeypatch.setattr(+ async_compile.AsyncCompile, "warm_pool", classmethod(lambda cls: calls.append("warm"))+ )+ monkeypatch.setattr(+ async_compile.AsyncCompile, "wakeup", classmethod(lambda cls: calls.append("wakeup"))+ )+ monkeypatch.setattr(bg, "_POOL_WARMED", False)+ monkeypatch.delenv(bg.WARM_POOL_ENV, raising = False)+ monkeypatch.delenv("UNSLOTH_DIFFUSION_BG_COMPILE", raising = False)+ return calls+++def test_warm_compile_pool_starts_the_workers_once(_fake_pool):+ assert bg.warm_compile_pool(LOG) is True+ assert bg.warm_compile_pool(LOG) is False+ assert _fake_pool == ["warm", "wakeup"]+++ "env", ["UNSLOTH_DIFFUSION_BG_COMPILE_WARM_POOL", "UNSLOTH_DIFFUSION_BG_COMPILE"]+)+def test_warm_compile_pool_switches(monkeypatch, _fake_pool, env):+ monkeypatch.setenv(env, "0")+ assert bg.warm_compile_pool(LOG) is False+ assert _fake_pool == []