Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

Studio diffusion: move the deferred compile off the render for offloaded and Qwen-Image-2.1 denoisers (#15150)

Daniel Hanc2c7d345892e
diff --git a/studio/backend/core/inference/diffusion.py b/studio/backend/core/inference/diffusion.py
index 7b408e61b5a..1b188d0bda7 100644
--- a/studio/backend/core/inference/diffusion.py
+++ b/studio/backend/core/inference/diffusion.py
@@ -13239,6 +13239,8 @@ class DiffusionBackend:
restore_vae: Optional[Callable[[], None]] = None
restore_resident: Optional[Callable[[], None]] = None
bg = None
+ # Kicked from the finally below: an offloaded denoiser's placement (restore_resident) must be back first.
+ kick_bg = False
bg_eager = False
bg_token = None
# A Detailer crop inpaint (diffusion_bg_compile.auxiliary_pass): eager, uncounted, nothing recorded.
@@ -14293,7 +14295,7 @@ class DiffusionBackend:
# Nothing compiled ran, so nothing to register or save yet.
bg.note_eager_generation()
if _hires_pass is None or not _hires_pass.defer_bg_kick:
- bg.kick()
+ kick_bg = True
else:
try:
# Register the dims the forward ACTUALLY compiled with, and every distinct chunk size (a static
@@ -14429,6 +14431,11 @@ class DiffusionBackend:
restore_resident()
except Exception as exc: # noqa: BLE001 - the groups keep streaming
logger.warning("diffusion.memory: re-residency failed: %s", exc)
+ if kick_bg and bg is not None:
+ try:
+ bg.kick()
+ except Exception as exc: # noqa: BLE001 - the next render compiles inline
+ logger.warning("diffusion.bg_compile: kick failed: %s", exc)
with self._generation_cancel_lock:
if self._active_generate_cancel is cancel:
self._active_generate_cancel = None
diff --git a/studio/backend/core/inference/diffusion_bg_compile.py b/studio/backend/core/inference/diffusion_bg_compile.py
index 09645f49f54..808eefba778 100644
--- a/studio/backend/core/inference/diffusion_bg_compile.py
+++ b/studio/backend/core/inference/diffusion_bg_compile.py
@@ -8,9 +8,10 @@ guard and CUDA-graph wrapper); ``kick`` replays them through the compiled callab
capture suppressed, so the next render pays only the capture. A render arriving mid-compile waits for it: a render
beside the compile was starved to 49-62 s on a B200. Any failure ends the attempt and the next render compiles inline.
-Dense, non-offloaded denoisers only: torchao ones are ~30x slower eager, and offload hooks would fight two forwards.
-By default only the deferred profile (generation 3, already eager -> compiled) uses it; UNSLOTH_DIFFUSION_BG_COMPILE=1
-also moves a load's first-render compile (render 1 then differs from render 2 for one seed), =0 disables it.
+Dense denoisers only: torchao ones are ~30x slower eager. Offloaded ones too: no render runs beside the compile,
+so offload hooks see one forward at a time (UNSLOTH_DIFFUSION_BG_COMPILE_OFFLOADED=0 opts out). By default only the deferred
+profile (generation 3, already eager -> compiled) uses it; UNSLOTH_DIFFUSION_BG_COMPILE=1 also moves a load's
+first-render compile (render 1 then differs from render 2 for one seed), =0 disables it.
"""
from __future__ import annotations
@@ -23,6 +24,7 @@ import time
from typing import Any, Callable, Optional
_ENV = "UNSLOTH_DIFFUSION_BG_COMPILE"
+OFFLOADED_ENV = "UNSLOTH_DIFFUSION_BG_COMPILE_OFFLOADED"
_MAX_SAMPLES = 4
# ContextVars, not thread-locals: diffusion_render_thread.run copies the caller's context onto the render thread.
@@ -45,6 +47,11 @@ def enabled() -> bool:
return raw not in ("0", "false", "no", "off")
+def offloaded_enabled() -> bool:
+ raw = (os.environ.get(OFFLOADED_ENV) or "").strip().lower()
+ return raw not in ("0", "false", "no", "off")
+
+
def load_time_enabled() -> bool:
"""Opt-in (``=1``): render 1 eager and render 2 compiled would break "same seed twice repeats"."""
raw = (os.environ.get(_ENV) or "").strip().lower()
@@ -153,8 +160,90 @@ def recording(module: Any) -> bool:
def _cuda_graph_helpers() -> tuple[Callable, Callable, Callable]:
- from .diffusion_cuda_graph import _flatten, _rebuild, graph_key
- return _flatten, _rebuild, graph_key
+ from .diffusion_cuda_graph import graph_key
+ return _flatten_sample, _rebuild_sample, graph_key
+
+
+# Plain state objects in a call (Qwen-Image-2.1's ``QwenImage21KVCache``) are recorded as fresh instances of cloned tensors.
+OBJECTS_ENV = "UNSLOTH_DIFFUSION_BG_COMPILE_OBJECTS"
+_OBJECT_DEPTH = 3
+# A prefix holding a condition image is ~2.2 GiB: such a call keeps compiling on the render.
+_OBJECT_MAX_BYTES = 1 << 30
+
+
+def _objects_enabled() -> bool:
+ raw = (os.environ.get(OBJECTS_ENV) or "").strip().lower()
+ return raw not in ("0", "false", "no", "off")
+
+
+def _plain_state(obj: Any) -> Optional[dict]:
+ import torch
+
+ if isinstance(obj, (torch.nn.Module, torch.Tensor, type)) or callable(obj):
+ return None
+ cls = type(obj)
+ if str(cls.__module__).split(".")[0] in ("builtins", "torch") or hasattr(cls, "__slots__"):
+ return None
+ state = getattr(obj, "__dict__", None)
+ return state if isinstance(state, dict) else None
+
+
+def _flatten_sample(
+ obj: Any,
+ out: list,
+ depth: int = 0,
+ budget: Optional[list] = None,
+) -> tuple:
+ import torch
+
+ if budget is None:
+ budget = [0]
+ if torch.is_tensor(obj):
+ out.append(obj)
+ return ("t", len(out) - 1)
+ if isinstance(obj, (list, tuple)):
+ return ("l", isinstance(obj, tuple), [_flatten_sample(o, out, depth, budget) for o in obj])
+ if isinstance(obj, dict):
+ return ("d", [(k, _flatten_sample(v, out, depth, budget)) for k, v in obj.items()])
+ if isinstance(obj, (int, float, bool, str, bytes)) or obj is None:
+ return ("v", obj)
+ state = _plain_state(obj) if _objects_enabled() and depth < _OBJECT_DEPTH else None
+ if state is None:
+ raise TypeError(f"cannot make a static buffer for a {type(obj).__name__} in the call tree")
+ first = len(out)
+ spec = (
+ "o",
+ type(obj),
+ [(k, _flatten_sample(v, out, depth + 1, budget)) for k, v in state.items()],
+ )
+ held = budget[1] if len(budget) > 1 else None
+ if held is not None:
+ held.update(range(first, len(out)))
+ if depth == 0:
+ # The outermost object's slice holds every nested one's: charging each level would count a tensor twice.
+ budget[0] += sum(t.numel() * t.element_size() for t in out[first:])
+ if budget[0] > _OBJECT_MAX_BYTES:
+ raise TypeError(
+ f"{type(obj).__name__} holds {budget[0] >> 20} MiB, above the {_OBJECT_MAX_BYTES >> 20} MiB a sample copies"
+ )
+ return spec
+
+
+def _rebuild_sample(spec: tuple, tensors: list) -> Any:
+ kind = spec[0]
+ if kind == "t":
+ return tensors[spec[1]]
+ if kind == "l":
+ values = [_rebuild_sample(s, tensors) for s in spec[2]]
+ return tuple(values) if spec[1] else values
+ if kind == "d":
+ return {k: _rebuild_sample(s, tensors) for k, s in spec[1]}
+ if kind == "o":
+ obj = object.__new__(spec[1])
+ for k, s in spec[2]:
+ obj.__dict__[k] = _rebuild_sample(s, tensors)
+ return obj
+ return spec[1]
def _capture_armed(module: Any) -> bool:
@@ -260,11 +349,15 @@ class BackgroundCompile:
if key in self._keys:
return None
live: list = []
- spec = _flatten((args, kwargs), live)
+ held: set = set()
+ spec = _flatten((args, kwargs), live, 0, [0, held])
if _capture_armed(module):
# Like the capture's static buffers (made outside inference mode): dynamo guards on inference-ness.
- with torch.inference_mode(False):
- clones = [torch.empty_like(t) for t in live]
+ # Tensors inside a state object are not graph inputs: they keep the inference-ness they arrived with.
+ clones = []
+ for index, t in enumerate(live):
+ with torch.inference_mode(index in held and t.is_inference()):
+ clones.append(torch.empty_like(t))
for dst, src in zip(clones, live):
dst.copy_(src)
else:
@@ -282,6 +375,14 @@ class BackgroundCompile:
except Exception as exc: # noqa: BLE001 - an uncloneable tree just means no background compile
self.state = "failed"
self.error = f"record: {type(exc).__name__}: {exc}"
+ # Not _finish: the hook is running, and removing it mid-dispatch would mutate the dict torch iterates.
+ self.samples = []
+ if self.logger is not None:
+ self.logger.info(
+ "diffusion.bg_compile: %s cannot be recorded (%s); the next render compiles inline",
+ type(module).__name__,
+ self.error,
+ )
return None
def wants_samples(self) -> bool:
@@ -452,15 +553,15 @@ def select_module(
backend: Any,
denoiser_hooked: bool,
) -> Any:
- """The one dense, resident, default-tier CUDA denoiser whose compile may move off the render, or None.
+ """The one dense default-tier CUDA denoiser whose compile may move off the render, or None.
- Not torchao (~30x slower eager), GGUF (compiles only its dequant), a step cache (toggles graphs per step),
- max-autotune (an explicit request to pay the compile) or an offloaded denoiser."""
+ Not torchao (~30x slower eager), GGUF (compiles only its dequant), a step cache (toggles graphs per step) or
+ max-autotune (an explicit request to pay the compile). An offloaded denoiser only with ``offloaded_enabled``."""
if not enabled():
return None
if "compiled" not in tuple(speed_optims or ()) or not default_tier:
return None
- if quantized or gguf or step_cache or denoiser_hooked:
+ if quantized or gguf or step_cache or (denoiser_hooked and not offloaded_enabled()):
return None
if device != "cuda" or backend == "rocm":
return None
diff --git a/studio/backend/core/inference/diffusion_cuda_graph.py b/studio/backend/core/inference/diffusion_cuda_graph.py
index 605651d4973..880b3981478 100644
--- a/studio/backend/core/inference/diffusion_cuda_graph.py
+++ b/studio/backend/core/inference/diffusion_cuda_graph.py
@@ -26,6 +26,18 @@ from typing import Any, Optional
from .diffusion_bg_compile import capture_suppressed as _bg_capture_suppressed
from .diffusion_bg_compile import eager_forced as _bg_eager_forced
+BG_PLANNED_REPLAY_ENV = "UNSLOTH_DIFFUSION_BG_COMPILE_PLANNED"
+
+
+def _bg_planned_replay_enabled() -> bool:
+ return (os.environ.get(BG_PLANNED_REPLAY_ENV) or "").strip().lower() not in (
+ "0",
+ "false",
+ "no",
+ "off",
+ )
+
+
CUDA_GRAPH_DISABLE_ENV = "UNSLOTH_DISABLE_CUDA_GRAPH"
# Master switch for every diffusion CUDA graph (whole-forward and per-block): "0" turns them all off.
CUDA_GRAPHS_ENV = "UNSLOTH_DIFFUSION_CUDA_GRAPHS"
@@ -1546,6 +1558,27 @@ class GraphedForward:
args, kwargs = _rebuild(spec, live)
return self._eager(args, kwargs)
+ def _background_call(self, args: tuple, kwargs: dict) -> Any:
+ """Replay through the call a render captures: otherwise the render's compiled code guards on other layouts and recompiles."""
+ if (
+ self.plan is not None
+ and self.enabled
+ and not self.poisoned
+ and not self.bypassed
+ and kwargs.get("return_dict", True) is False
+ and _bg_planned_replay_enabled()
+ ):
+ planned = self.plan(self.module, args, kwargs)
+ if planned is not None:
+ call, planned_kwargs = planned
+ if self.placement is not None:
+ call = self._placed_step
+ live: list = []
+ spec = _flatten(((), planned_kwargs), live)
+ warm_args, warm_kwargs = _rebuild(spec, self._statics(live))
+ return call(*warm_args, **warm_kwargs)
+ return self.orig(*args, **kwargs)
+
def _placement_declined(self) -> bool:
# A declined placement refuses every later recording for the load (a merely dropped key does not).
return getattr(getattr(self, "placement", None), "_declined", None) is not None
@@ -1570,7 +1603,7 @@ class GraphedForward:
def __call__(self, *args, **kwargs):
if _bg_capture_suppressed():
# Background compile thread: never capture off the render thread.
- return self.orig(*args, **kwargs)
+ return self._background_call(args, kwargs)
if getattr(self, "fallback", None) is not None and (self.poisoned or self._dropped):
self._engage_fallback()
if (
diff --git a/studio/backend/tests/test_diffusion_bg_compile_state_objects.py b/studio/backend/tests/test_diffusion_bg_compile_state_objects.py
new file mode 100644
index 00000000000..415e146c959
--- /dev/null
+++ b/studio/backend/tests/test_diffusion_bg_compile_state_objects.py
@@ -0,0 +1,215 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
+
+"""Background compile of a denoiser whose call carries a state object (Qwen-Image-2.1's prefix KV cache) or is
+offloaded: the recording keeps such calls, and a planned CUDA-graph step replays the way the render captures it.
+CPU only."""
+
+from __future__ import annotations
+
+import logging
+
+import pytest
+
+torch = pytest.importorskip("torch")
+
+from core.inference import diffusion_bg_compile as bg # noqa: E402
+from core.inference import diffusion_cuda_graph as cg # noqa: E402
+
+
+class _LayerCache:
+ def __init__(self) -> None:
+ self.k = None
+ self.v = None
+
+
+class _KVCache:
+ """Shaped like diffusers' ``QwenImage21KVCache``: a list of per-layer holders, filled by the prefill."""
+
+ def __init__(self, layers: int) -> None:
+ self.layer_caches = [_LayerCache() for _ in range(layers)]
+
+ def get_layer(self, index: int) -> _LayerCache:
+ return self.layer_caches[index]
+
+
+class _CachedNet(torch.nn.Module):
+ def __init__(self) -> None:
+ super().__init__()
+ self.lin = torch.nn.Linear(8, 8)
+
+ def forward(
+ self,
+ hidden_states,
+ *,
+ kv_cache = None,
+ kv_cache_mode = "extract",
+ return_dict = True,
+ ):
+ out = self.lin(hidden_states)
+ layer = kv_cache.get_layer(0)
+ if kv_cache_mode == "extract":
+ layer.k, layer.v = out.detach().clone(), (out * 2).detach().clone()
+ return out
+ return out + layer.k
+
+
+def _prefill_and_decode(net):
+ cache = _KVCache(1)
+ x = torch.randn(2, 8)
+ with torch.no_grad(), bg.force_eager():
+ net(x, kv_cache = cache, kv_cache_mode = "extract", return_dict = False)
+ net(x, kv_cache = cache, kv_cache_mode = "cached", return_dict = False)
+ net(x, kv_cache = cache, kv_cache_mode = "cached", return_dict = False) # same key: deduped
+ return cache
+
+
+def test_a_state_object_in_the_call_is_recorded_as_a_fresh_copy(monkeypatch):
+ monkeypatch.delenv(bg.OBJECTS_ENV, raising = False)
+ net = _CachedNet()
+ job = bg.arm(net)
+ cache = _prefill_and_decode(net)
+ assert job.state == "recording" and len(job.samples) == 2
+ spec, clones, *_ = job.samples[1]
+ args, kwargs = bg._rebuild_sample(spec, clones)
+ rebuilt = kwargs["kv_cache"]
+ assert type(rebuilt) is _KVCache and rebuilt is not cache
+ assert rebuilt.layer_caches[0] is not cache.layer_caches[0]
+ assert torch.equal(rebuilt.get_layer(0).k, cache.get_layer(0).k)
+ assert rebuilt.get_layer(0).k.data_ptr() != cache.get_layer(0).k.data_ptr()
+ # the prefill sample holds the cache as the prefill saw it: empty, so its replay fills its own copy
+ spec0, clones0, *_ = job.samples[0]
+ assert bg._rebuild_sample(spec0, clones0)[1]["kv_cache"].get_layer(0).k is None
+ assert job.kick()
+ job._thread.join(60)
+ assert job.state == "done", job.error
+ job.close()
+
+
+def test_the_object_kill_switch_restores_the_old_refusal(monkeypatch, caplog):
+ monkeypatch.setenv(bg.OBJECTS_ENV, "0")
+ net = _CachedNet()
+ job = bg.arm(net, logger = logging.getLogger("bgtest"))
+ with caplog.at_level(logging.INFO, logger = "bgtest"):
+ _prefill_and_decode(net)
+ assert job.state == "failed" and "_KVCache" in (job.error or "")
+ assert not job.samples and not job.pending()
+ # the silent failure hid a 47-77 s inline compile: it is logged now
+ assert any("cannot be recorded" in r.getMessage() for r in caplog.records)
+ job.close()
+
+
+def test_an_oversized_state_object_is_not_copied(monkeypatch):
+ monkeypatch.delenv(bg.OBJECTS_ENV, raising = False)
+ monkeypatch.setattr(bg, "_OBJECT_MAX_BYTES", 16)
+ net = _CachedNet()
+ job = bg.arm(net)
+ _prefill_and_decode(net)
+ assert job.state == "failed" and "above" in (job.error or "")
+ job.close()
+
+
+def test_modules_and_callables_are_never_copied():
+ for leaf in (torch.nn.Linear(2, 2), lambda: None, _KVCache):
+ with pytest.raises(TypeError):
+ bg._flatten_sample({"x": leaf}, [])
+
+
+def test_select_module_takes_an_offloaded_denoiser_unless_switched_off(monkeypatch):
+ import types
+
+ monkeypatch.delenv("UNSLOTH_DIFFUSION_BG_COMPILE", raising = False)
+ monkeypatch.setattr(cg, "_denoiser_modules", lambda pipe: [pipe.transformer])
+ pipe = types.SimpleNamespace(transformer = torch.nn.Linear(2, 2))
+ kwargs = dict(
+ speed_optims = ("compiled",),
+ default_tier = True,
+ quantized = False,
+ gguf = False,
+ step_cache = False,
+ device = "cuda",
+ backend = "cuda",
+ denoiser_hooked = True,
+ )
+ monkeypatch.delenv(bg.OFFLOADED_ENV, raising = False)
+ assert bg.select_module(pipe, **kwargs) is pipe.transformer
+ monkeypatch.setenv(bg.OFFLOADED_ENV, "0")
+ assert bg.select_module(pipe, **kwargs) is None
+ assert bg.select_module(pipe, **{**kwargs, "denoiser_hooked": False}) is pipe.transformer
+
+
+class _PlainNet(torch.nn.Module):
+ def __init__(self) -> None:
+ super().__init__()
+ self.lin = torch.nn.Linear(8, 8)
+
+ def forward(
+ self,
+ x,
+ *,
+ return_dict = True,
+ ):
+ return self.lin(x)
+
+
[email protected]("planned_replay", [True, False])
+def test_a_background_replay_takes_the_planned_step_on_fresh_statics(monkeypatch, planned_replay):
+ if not planned_replay:
+ monkeypatch.setenv(cg.BG_PLANNED_REPLAY_ENV, "0")
+ else:
+ monkeypatch.delenv(cg.BG_PLANNED_REPLAY_ENV, raising = False)
+ net = _PlainNet()
+ graphed = cg.GraphedForward(net).enable()
+ seen = []
+
+ def step(*, x):
+ seen.append(x)
+ return (x,)
+
+ graphed.plan = lambda module, args, kwargs: (step, {"x": args[0][:, :4]})
+ try:
+ x = torch.randn(2, 8)
+ token = bg._NO_CAPTURE.set(True)
+ try:
+ with torch.no_grad():
+ net(x, return_dict = False)
+ finally:
+ bg._NO_CAPTURE.reset(token)
+ assert graphed.stats["captures"] == 0
+ if planned_replay:
+ # a fresh offset-0 copy, as the capture's warm-up steps on: not the caller's strided view
+ assert len(seen) == 1 and seen[0].storage_offset() == 0 and seen[0].is_contiguous()
+ assert torch.equal(seen[0], x[:, :4])
+ else:
+ assert seen == []
+ finally:
+ graphed.free()
+
+
+def test_nested_state_objects_charge_each_tensor_once(monkeypatch):
+ monkeypatch.delenv(bg.OBJECTS_ENV, raising = False)
+ cache = _KVCache(2)
+ for layer in cache.layer_caches:
+ layer.k = torch.zeros(4, dtype = torch.float32)
+ layer.v = torch.zeros(4, dtype = torch.float32)
+ total = 2 * 2 * 4 * 4
+ monkeypatch.setattr(bg, "_OBJECT_MAX_BYTES", total)
+ out: list = []
+ budget = [0, set()]
+ bg._flatten_sample(((), {"cache": cache}), out, 0, budget)
+ assert budget[0] == total and len(out) == 4
+ monkeypatch.setattr(bg, "_OBJECT_MAX_BYTES", total - 1)
+ with pytest.raises(TypeError, match = "above"):
+ bg._flatten_sample(((), {"cache": cache}), [], 0, [0, set()])
+
+
+def test_the_background_kick_waits_for_the_resident_restore():
+ """An offloaded denoiser's groups go back on the card before its background compile can start."""
+ import inspect
+
+ from core.inference import diffusion as D
+
+ src = inspect.getsource(D)
+ assert "bg.kick()" not in src.split("kick_bg = True")[0].split("kick_bg = False")[-1]
+ tail = src.split("restore_resident()", 2)[-1]
+ assert tail.index("if kick_bg and bg is not None:") < tail.index("bg.kick()")
diff --git a/studio/backend/tests/test_diffusion_cold_start.py b/studio/backend/tests/test_diffusion_cold_start.py
index ec21fc8a802..7b9d8176713 100644
--- a/studio/backend/tests/test_diffusion_cold_start.py
+++ b/studio/backend/tests/test_diffusion_cold_start.py
@@ -358,7 +358,9 @@ class _UNet2DConditionModel(torch.nn.Module):
({}, True),
({"transformer_quant": "int8"}, False),
({"gguf_transformer": True}, False),
- ({"offload_policy": "group", "_hooked": True}, False),
+ # An offloaded denoiser compiles in the background too, unless UNSLOTH_DIFFUSION_BG_COMPILE_OFFLOADED=0.
+ ({"offload_policy": "group", "_hooked": True}, True),
+ ({"offload_policy": "group", "_hooked": True, "_offloaded_off": True}, False),
({"offload_policy": "group"}, True),
({"speed_mode": "max"}, False),
({"speed_optims": ("cuda_graph",)}, False),
@@ -368,12 +370,15 @@ class _UNet2DConditionModel(torch.nn.Module):
({"target": types.SimpleNamespace(device = "mps", backend = "mps")}, False),
],
)
-def test_only_dense_resident_default_tier_cuda_loads_compile_in_the_background(
+def test_only_dense_default_tier_cuda_loads_compile_in_the_background(
monkeypatch, override, expect
):
from core.inference import diffusion as D
monkeypatch.delenv("UNSLOTH_DIFFUSION_BG_COMPILE", raising = False)
+ monkeypatch.delenv("UNSLOTH_DIFFUSION_BG_COMPILE_OFFLOADED", raising = False)
+ if override.pop("_offloaded_off", False):
+ monkeypatch.setenv("UNSLOTH_DIFFUSION_BG_COMPILE_OFFLOADED", "0")
monkeypatch.setattr(
speed, "_UNET_WHOLE_COMPILE", frozenset({"_UNet2DConditionModel"}), raising = False
)