Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
Studio diffusion: move the deferred compile off the render for offloaded and Qwen-Image-2.1 denoisers (#15150)
Daniel Hanc2c7d345892e
diff --git a/studio/backend/core/inference/diffusion.py b/studio/backend/core/inference/diffusion.pyindex 7b408e61b5a..1b188d0bda7 100644--- a/studio/backend/core/inference/diffusion.py+++ b/studio/backend/core/inference/diffusion.py@@ -13239,6 +13239,8 @@ class DiffusionBackend:restore_vae: Optional[Callable[[], None]] = Nonerestore_resident: Optional[Callable[[], None]] = Nonebg = None+ # Kicked from the finally below: an offloaded denoiser's placement (restore_resident) must be back first.+ kick_bg = Falsebg_eager = Falsebg_token = None# A Detailer crop inpaint (diffusion_bg_compile.auxiliary_pass): eager, uncounted, nothing recorded.@@ -14293,7 +14295,7 @@ class DiffusionBackend:# Nothing compiled ran, so nothing to register or save yet.bg.note_eager_generation()if _hires_pass is None or not _hires_pass.defer_bg_kick:- bg.kick()+ kick_bg = Trueelse:try:# Register the dims the forward ACTUALLY compiled with, and every distinct chunk size (a static@@ -14429,6 +14431,11 @@ class DiffusionBackend:restore_resident()except Exception as exc: # noqa: BLE001 - the groups keep streaminglogger.warning("diffusion.memory: re-residency failed: %s", exc)+ if kick_bg and bg is not None:+ try:+ bg.kick()+ except Exception as exc: # noqa: BLE001 - the next render compiles inline+ logger.warning("diffusion.bg_compile: kick failed: %s", exc)with self._generation_cancel_lock:if self._active_generate_cancel is cancel:self._active_generate_cancel = Nonediff --git a/studio/backend/core/inference/diffusion_bg_compile.py b/studio/backend/core/inference/diffusion_bg_compile.pyindex 09645f49f54..808eefba778 100644--- a/studio/backend/core/inference/diffusion_bg_compile.py+++ b/studio/backend/core/inference/diffusion_bg_compile.py@@ -8,9 +8,10 @@ guard and CUDA-graph wrapper); ``kick`` replays them through the compiled callabcapture suppressed, so the next render pays only the capture. A render arriving mid-compile waits for it: a renderbeside the compile was starved to 49-62 s on a B200. Any failure ends the attempt and the next render compiles inline.-Dense, non-offloaded denoisers only: torchao ones are ~30x slower eager, and offload hooks would fight two forwards.-By default only the deferred profile (generation 3, already eager -> compiled) uses it; UNSLOTH_DIFFUSION_BG_COMPILE=1-also moves a load's first-render compile (render 1 then differs from render 2 for one seed), =0 disables it.+Dense denoisers only: torchao ones are ~30x slower eager. Offloaded ones too: no render runs beside the compile,+so offload hooks see one forward at a time (UNSLOTH_DIFFUSION_BG_COMPILE_OFFLOADED=0 opts out). By default only the deferred+profile (generation 3, already eager -> compiled) uses it; UNSLOTH_DIFFUSION_BG_COMPILE=1 also moves a load's+first-render compile (render 1 then differs from render 2 for one seed), =0 disables it."""from __future__ import annotations@@ -23,6 +24,7 @@ import timefrom typing import Any, Callable, Optional_ENV = "UNSLOTH_DIFFUSION_BG_COMPILE"+OFFLOADED_ENV = "UNSLOTH_DIFFUSION_BG_COMPILE_OFFLOADED"_MAX_SAMPLES = 4# ContextVars, not thread-locals: diffusion_render_thread.run copies the caller's context onto the render thread.@@ -45,6 +47,11 @@ def enabled() -> bool:return raw not in ("0", "false", "no", "off")+def offloaded_enabled() -> bool:+ raw = (os.environ.get(OFFLOADED_ENV) or "").strip().lower()+ return raw not in ("0", "false", "no", "off")++def load_time_enabled() -> bool:"""Opt-in (``=1``): render 1 eager and render 2 compiled would break "same seed twice repeats"."""raw = (os.environ.get(_ENV) or "").strip().lower()@@ -153,8 +160,90 @@ def recording(module: Any) -> bool:def _cuda_graph_helpers() -> tuple[Callable, Callable, Callable]:- from .diffusion_cuda_graph import _flatten, _rebuild, graph_key- return _flatten, _rebuild, graph_key+ from .diffusion_cuda_graph import graph_key+ return _flatten_sample, _rebuild_sample, graph_key+++# Plain state objects in a call (Qwen-Image-2.1's ``QwenImage21KVCache``) are recorded as fresh instances of cloned tensors.+OBJECTS_ENV = "UNSLOTH_DIFFUSION_BG_COMPILE_OBJECTS"+_OBJECT_DEPTH = 3+# A prefix holding a condition image is ~2.2 GiB: such a call keeps compiling on the render.+_OBJECT_MAX_BYTES = 1 << 30+++def _objects_enabled() -> bool:+ raw = (os.environ.get(OBJECTS_ENV) or "").strip().lower()+ return raw not in ("0", "false", "no", "off")+++def _plain_state(obj: Any) -> Optional[dict]:+ import torch++ if isinstance(obj, (torch.nn.Module, torch.Tensor, type)) or callable(obj):+ return None+ cls = type(obj)+ if str(cls.__module__).split(".")[0] in ("builtins", "torch") or hasattr(cls, "__slots__"):+ return None+ state = getattr(obj, "__dict__", None)+ return state if isinstance(state, dict) else None+++def _flatten_sample(+ obj: Any,+ out: list,+ depth: int = 0,+ budget: Optional[list] = None,+) -> tuple:+ import torch++ if budget is None:+ budget = [0]+ if torch.is_tensor(obj):+ out.append(obj)+ return ("t", len(out) - 1)+ if isinstance(obj, (list, tuple)):+ return ("l", isinstance(obj, tuple), [_flatten_sample(o, out, depth, budget) for o in obj])+ if isinstance(obj, dict):+ return ("d", [(k, _flatten_sample(v, out, depth, budget)) for k, v in obj.items()])+ if isinstance(obj, (int, float, bool, str, bytes)) or obj is None:+ return ("v", obj)+ state = _plain_state(obj) if _objects_enabled() and depth < _OBJECT_DEPTH else None+ if state is None:+ raise TypeError(f"cannot make a static buffer for a {type(obj).__name__} in the call tree")+ first = len(out)+ spec = (+ "o",+ type(obj),+ [(k, _flatten_sample(v, out, depth + 1, budget)) for k, v in state.items()],+ )+ held = budget[1] if len(budget) > 1 else None+ if held is not None:+ held.update(range(first, len(out)))+ if depth == 0:+ # The outermost object's slice holds every nested one's: charging each level would count a tensor twice.+ budget[0] += sum(t.numel() * t.element_size() for t in out[first:])+ if budget[0] > _OBJECT_MAX_BYTES:+ raise TypeError(+ f"{type(obj).__name__} holds {budget[0] >> 20} MiB, above the {_OBJECT_MAX_BYTES >> 20} MiB a sample copies"+ )+ return spec+++def _rebuild_sample(spec: tuple, tensors: list) -> Any:+ kind = spec[0]+ if kind == "t":+ return tensors[spec[1]]+ if kind == "l":+ values = [_rebuild_sample(s, tensors) for s in spec[2]]+ return tuple(values) if spec[1] else values+ if kind == "d":+ return {k: _rebuild_sample(s, tensors) for k, s in spec[1]}+ if kind == "o":+ obj = object.__new__(spec[1])+ for k, s in spec[2]:+ obj.__dict__[k] = _rebuild_sample(s, tensors)+ return obj+ return spec[1]def _capture_armed(module: Any) -> bool:@@ -260,11 +349,15 @@ class BackgroundCompile:if key in self._keys:return Nonelive: list = []- spec = _flatten((args, kwargs), live)+ held: set = set()+ spec = _flatten((args, kwargs), live, 0, [0, held])if _capture_armed(module):# Like the capture's static buffers (made outside inference mode): dynamo guards on inference-ness.- with torch.inference_mode(False):- clones = [torch.empty_like(t) for t in live]+ # Tensors inside a state object are not graph inputs: they keep the inference-ness they arrived with.+ clones = []+ for index, t in enumerate(live):+ with torch.inference_mode(index in held and t.is_inference()):+ clones.append(torch.empty_like(t))for dst, src in zip(clones, live):dst.copy_(src)else:@@ -282,6 +375,14 @@ class BackgroundCompile:except Exception as exc: # noqa: BLE001 - an uncloneable tree just means no background compileself.state = "failed"self.error = f"record: {type(exc).__name__}: {exc}"+ # Not _finish: the hook is running, and removing it mid-dispatch would mutate the dict torch iterates.+ self.samples = []+ if self.logger is not None:+ self.logger.info(+ "diffusion.bg_compile: %s cannot be recorded (%s); the next render compiles inline",+ type(module).__name__,+ self.error,+ )return Nonedef wants_samples(self) -> bool:@@ -452,15 +553,15 @@ def select_module(backend: Any,denoiser_hooked: bool,) -> Any:- """The one dense, resident, default-tier CUDA denoiser whose compile may move off the render, or None.+ """The one dense default-tier CUDA denoiser whose compile may move off the render, or None.- Not torchao (~30x slower eager), GGUF (compiles only its dequant), a step cache (toggles graphs per step),- max-autotune (an explicit request to pay the compile) or an offloaded denoiser."""+ Not torchao (~30x slower eager), GGUF (compiles only its dequant), a step cache (toggles graphs per step) or+ max-autotune (an explicit request to pay the compile). An offloaded denoiser only with ``offloaded_enabled``."""if not enabled():return Noneif "compiled" not in tuple(speed_optims or ()) or not default_tier:return None- if quantized or gguf or step_cache or denoiser_hooked:+ if quantized or gguf or step_cache or (denoiser_hooked and not offloaded_enabled()):return Noneif device != "cuda" or backend == "rocm":return Nonediff --git a/studio/backend/core/inference/diffusion_cuda_graph.py b/studio/backend/core/inference/diffusion_cuda_graph.pyindex 605651d4973..880b3981478 100644--- a/studio/backend/core/inference/diffusion_cuda_graph.py+++ b/studio/backend/core/inference/diffusion_cuda_graph.py@@ -26,6 +26,18 @@ from typing import Any, Optionalfrom .diffusion_bg_compile import capture_suppressed as _bg_capture_suppressedfrom .diffusion_bg_compile import eager_forced as _bg_eager_forced+BG_PLANNED_REPLAY_ENV = "UNSLOTH_DIFFUSION_BG_COMPILE_PLANNED"+++def _bg_planned_replay_enabled() -> bool:+ return (os.environ.get(BG_PLANNED_REPLAY_ENV) or "").strip().lower() not in (+ "0",+ "false",+ "no",+ "off",+ )++CUDA_GRAPH_DISABLE_ENV = "UNSLOTH_DISABLE_CUDA_GRAPH"# Master switch for every diffusion CUDA graph (whole-forward and per-block): "0" turns them all off.CUDA_GRAPHS_ENV = "UNSLOTH_DIFFUSION_CUDA_GRAPHS"@@ -1546,6 +1558,27 @@ class GraphedForward:args, kwargs = _rebuild(spec, live)return self._eager(args, kwargs)+ def _background_call(self, args: tuple, kwargs: dict) -> Any:+ """Replay through the call a render captures: otherwise the render's compiled code guards on other layouts and recompiles."""+ if (+ self.plan is not None+ and self.enabled+ and not self.poisoned+ and not self.bypassed+ and kwargs.get("return_dict", True) is False+ and _bg_planned_replay_enabled()+ ):+ planned = self.plan(self.module, args, kwargs)+ if planned is not None:+ call, planned_kwargs = planned+ if self.placement is not None:+ call = self._placed_step+ live: list = []+ spec = _flatten(((), planned_kwargs), live)+ warm_args, warm_kwargs = _rebuild(spec, self._statics(live))+ return call(*warm_args, **warm_kwargs)+ return self.orig(*args, **kwargs)+def _placement_declined(self) -> bool:# A declined placement refuses every later recording for the load (a merely dropped key does not).return getattr(getattr(self, "placement", None), "_declined", None) is not None@@ -1570,7 +1603,7 @@ class GraphedForward:def __call__(self, *args, **kwargs):if _bg_capture_suppressed():# Background compile thread: never capture off the render thread.- return self.orig(*args, **kwargs)+ return self._background_call(args, kwargs)if getattr(self, "fallback", None) is not None and (self.poisoned or self._dropped):self._engage_fallback()if (diff --git a/studio/backend/tests/test_diffusion_bg_compile_state_objects.py b/studio/backend/tests/test_diffusion_bg_compile_state_objects.pynew file mode 100644index 00000000000..415e146c959--- /dev/null+++ b/studio/backend/tests/test_diffusion_bg_compile_state_objects.py@@ -0,0 +1,215 @@+# SPDX-License-Identifier: AGPL-3.0-only+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0++"""Background compile of a denoiser whose call carries a state object (Qwen-Image-2.1's prefix KV cache) or is+offloaded: the recording keeps such calls, and a planned CUDA-graph step replays the way the render captures it.+CPU only."""++from __future__ import annotations++import logging++import pytest++torch = pytest.importorskip("torch")++from core.inference import diffusion_bg_compile as bg # noqa: E402+from core.inference import diffusion_cuda_graph as cg # noqa: E402+++class _LayerCache:+ def __init__(self) -> None:+ self.k = None+ self.v = None+++class _KVCache:+ """Shaped like diffusers' ``QwenImage21KVCache``: a list of per-layer holders, filled by the prefill."""++ def __init__(self, layers: int) -> None:+ self.layer_caches = [_LayerCache() for _ in range(layers)]++ def get_layer(self, index: int) -> _LayerCache:+ return self.layer_caches[index]+++class _CachedNet(torch.nn.Module):+ def __init__(self) -> None:+ super().__init__()+ self.lin = torch.nn.Linear(8, 8)++ def forward(+ self,+ hidden_states,+ *,+ kv_cache = None,+ kv_cache_mode = "extract",+ return_dict = True,+ ):+ out = self.lin(hidden_states)+ layer = kv_cache.get_layer(0)+ if kv_cache_mode == "extract":+ layer.k, layer.v = out.detach().clone(), (out * 2).detach().clone()+ return out+ return out + layer.k+++def _prefill_and_decode(net):+ cache = _KVCache(1)+ x = torch.randn(2, 8)+ with torch.no_grad(), bg.force_eager():+ net(x, kv_cache = cache, kv_cache_mode = "extract", return_dict = False)+ net(x, kv_cache = cache, kv_cache_mode = "cached", return_dict = False)+ net(x, kv_cache = cache, kv_cache_mode = "cached", return_dict = False) # same key: deduped+ return cache+++def test_a_state_object_in_the_call_is_recorded_as_a_fresh_copy(monkeypatch):+ monkeypatch.delenv(bg.OBJECTS_ENV, raising = False)+ net = _CachedNet()+ job = bg.arm(net)+ cache = _prefill_and_decode(net)+ assert job.state == "recording" and len(job.samples) == 2+ spec, clones, *_ = job.samples[1]+ args, kwargs = bg._rebuild_sample(spec, clones)+ rebuilt = kwargs["kv_cache"]+ assert type(rebuilt) is _KVCache and rebuilt is not cache+ assert rebuilt.layer_caches[0] is not cache.layer_caches[0]+ assert torch.equal(rebuilt.get_layer(0).k, cache.get_layer(0).k)+ assert rebuilt.get_layer(0).k.data_ptr() != cache.get_layer(0).k.data_ptr()+ # the prefill sample holds the cache as the prefill saw it: empty, so its replay fills its own copy+ spec0, clones0, *_ = job.samples[0]+ assert bg._rebuild_sample(spec0, clones0)[1]["kv_cache"].get_layer(0).k is None+ assert job.kick()+ job._thread.join(60)+ assert job.state == "done", job.error+ job.close()+++def test_the_object_kill_switch_restores_the_old_refusal(monkeypatch, caplog):+ monkeypatch.setenv(bg.OBJECTS_ENV, "0")+ net = _CachedNet()+ job = bg.arm(net, logger = logging.getLogger("bgtest"))+ with caplog.at_level(logging.INFO, logger = "bgtest"):+ _prefill_and_decode(net)+ assert job.state == "failed" and "_KVCache" in (job.error or "")+ assert not job.samples and not job.pending()+ # the silent failure hid a 47-77 s inline compile: it is logged now+ assert any("cannot be recorded" in r.getMessage() for r in caplog.records)+ job.close()+++def test_an_oversized_state_object_is_not_copied(monkeypatch):+ monkeypatch.delenv(bg.OBJECTS_ENV, raising = False)+ monkeypatch.setattr(bg, "_OBJECT_MAX_BYTES", 16)+ net = _CachedNet()+ job = bg.arm(net)+ _prefill_and_decode(net)+ assert job.state == "failed" and "above" in (job.error or "")+ job.close()+++def test_modules_and_callables_are_never_copied():+ for leaf in (torch.nn.Linear(2, 2), lambda: None, _KVCache):+ with pytest.raises(TypeError):+ bg._flatten_sample({"x": leaf}, [])+++def test_select_module_takes_an_offloaded_denoiser_unless_switched_off(monkeypatch):+ import types++ monkeypatch.delenv("UNSLOTH_DIFFUSION_BG_COMPILE", raising = False)+ monkeypatch.setattr(cg, "_denoiser_modules", lambda pipe: [pipe.transformer])+ pipe = types.SimpleNamespace(transformer = torch.nn.Linear(2, 2))+ kwargs = dict(+ speed_optims = ("compiled",),+ default_tier = True,+ quantized = False,+ gguf = False,+ step_cache = False,+ device = "cuda",+ backend = "cuda",+ denoiser_hooked = True,+ )+ monkeypatch.delenv(bg.OFFLOADED_ENV, raising = False)+ assert bg.select_module(pipe, **kwargs) is pipe.transformer+ monkeypatch.setenv(bg.OFFLOADED_ENV, "0")+ assert bg.select_module(pipe, **kwargs) is None+ assert bg.select_module(pipe, **{**kwargs, "denoiser_hooked": False}) is pipe.transformer+++class _PlainNet(torch.nn.Module):+ def __init__(self) -> None:+ super().__init__()+ self.lin = torch.nn.Linear(8, 8)++ def forward(+ self,+ x,+ *,+ return_dict = True,+ ):+ return self.lin(x)++[email protected]("planned_replay", [True, False])+def test_a_background_replay_takes_the_planned_step_on_fresh_statics(monkeypatch, planned_replay):+ if not planned_replay:+ monkeypatch.setenv(cg.BG_PLANNED_REPLAY_ENV, "0")+ else:+ monkeypatch.delenv(cg.BG_PLANNED_REPLAY_ENV, raising = False)+ net = _PlainNet()+ graphed = cg.GraphedForward(net).enable()+ seen = []++ def step(*, x):+ seen.append(x)+ return (x,)++ graphed.plan = lambda module, args, kwargs: (step, {"x": args[0][:, :4]})+ try:+ x = torch.randn(2, 8)+ token = bg._NO_CAPTURE.set(True)+ try:+ with torch.no_grad():+ net(x, return_dict = False)+ finally:+ bg._NO_CAPTURE.reset(token)+ assert graphed.stats["captures"] == 0+ if planned_replay:+ # a fresh offset-0 copy, as the capture's warm-up steps on: not the caller's strided view+ assert len(seen) == 1 and seen[0].storage_offset() == 0 and seen[0].is_contiguous()+ assert torch.equal(seen[0], x[:, :4])+ else:+ assert seen == []+ finally:+ graphed.free()+++def test_nested_state_objects_charge_each_tensor_once(monkeypatch):+ monkeypatch.delenv(bg.OBJECTS_ENV, raising = False)+ cache = _KVCache(2)+ for layer in cache.layer_caches:+ layer.k = torch.zeros(4, dtype = torch.float32)+ layer.v = torch.zeros(4, dtype = torch.float32)+ total = 2 * 2 * 4 * 4+ monkeypatch.setattr(bg, "_OBJECT_MAX_BYTES", total)+ out: list = []+ budget = [0, set()]+ bg._flatten_sample(((), {"cache": cache}), out, 0, budget)+ assert budget[0] == total and len(out) == 4+ monkeypatch.setattr(bg, "_OBJECT_MAX_BYTES", total - 1)+ with pytest.raises(TypeError, match = "above"):+ bg._flatten_sample(((), {"cache": cache}), [], 0, [0, set()])+++def test_the_background_kick_waits_for_the_resident_restore():+ """An offloaded denoiser's groups go back on the card before its background compile can start."""+ import inspect++ from core.inference import diffusion as D++ src = inspect.getsource(D)+ assert "bg.kick()" not in src.split("kick_bg = True")[0].split("kick_bg = False")[-1]+ tail = src.split("restore_resident()", 2)[-1]+ assert tail.index("if kick_bg and bg is not None:") < tail.index("bg.kick()")diff --git a/studio/backend/tests/test_diffusion_cold_start.py b/studio/backend/tests/test_diffusion_cold_start.pyindex ec21fc8a802..7b9d8176713 100644--- a/studio/backend/tests/test_diffusion_cold_start.py+++ b/studio/backend/tests/test_diffusion_cold_start.py@@ -358,7 +358,9 @@ class _UNet2DConditionModel(torch.nn.Module):({}, True),({"transformer_quant": "int8"}, False),({"gguf_transformer": True}, False),- ({"offload_policy": "group", "_hooked": True}, False),+ # An offloaded denoiser compiles in the background too, unless UNSLOTH_DIFFUSION_BG_COMPILE_OFFLOADED=0.+ ({"offload_policy": "group", "_hooked": True}, True),+ ({"offload_policy": "group", "_hooked": True, "_offloaded_off": True}, False),({"offload_policy": "group"}, True),({"speed_mode": "max"}, False),({"speed_optims": ("cuda_graph",)}, False),@@ -368,12 +370,15 @@ class _UNet2DConditionModel(torch.nn.Module):({"target": types.SimpleNamespace(device = "mps", backend = "mps")}, False),],)-def test_only_dense_resident_default_tier_cuda_loads_compile_in_the_background(+def test_only_dense_default_tier_cuda_loads_compile_in_the_background(monkeypatch, override, expect):from core.inference import diffusion as Dmonkeypatch.delenv("UNSLOTH_DIFFUSION_BG_COMPILE", raising = False)+ monkeypatch.delenv("UNSLOTH_DIFFUSION_BG_COMPILE_OFFLOADED", raising = False)+ if override.pop("_offloaded_off", False):+ monkeypatch.setenv("UNSLOTH_DIFFUSION_BG_COMPILE_OFFLOADED", "0")monkeypatch.setattr(speed, "_UNET_WHOLE_COMPILE", frozenset({"_UNet2DConditionModel"}), raising = False)