Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

Studio diffusion: keep a model-offloaded denoiser's step eager instead of re-recording its CUDA graph every render (#14497)

Daniel Han4d7ebf5ceaf4
diff --git a/studio/backend/core/inference/diffusion_cuda_graph.py b/studio/backend/core/inference/diffusion_cuda_graph.py
index a5e3859fced..8148e4ecb7e 100644
--- a/studio/backend/core/inference/diffusion_cuda_graph.py
+++ b/studio/backend/core/inference/diffusion_cuda_graph.py
@@ -31,6 +31,12 @@ CUDA_GRAPHS_ENV = "UNSLOTH_DIFFUSION_CUDA_GRAPHS"
OFFLOAD_CUDA_GRAPH_ENV = "UNSLOTH_DIFFUSION_OFFLOAD_CUDA_GRAPH"
# =0: a block-streamed whole-step graph records its copies into the graph pool instead of the prefetcher's slot ring.
STEP_SLOTS_ENV = "UNSLOTH_DIFFUSION_STEP_GRAPH_SLOTS"
+MODEL_OFFLOAD_CUDA_GRAPH_ENV = "UNSLOTH_DIFFUSION_MODEL_OFFLOAD_CUDA_GRAPH"
+# Default off: replay saved ~1% of a step (FLUX.2-klein 4B, 1024^2) but the per-render re-record + ~0.6 GB pool cost more.
+MODEL_OFFLOAD_REASON = (
+ "model offload re-uploads the denoiser every render at new addresses, so its step graph would record again on "
+ "every render; set " + MODEL_OFFLOAD_CUDA_GRAPH_ENV + "=1 to record it anyway"
+)
# Per module. Legitimate second keys exist, but each costs a pool-sized slice of VRAM.
MAX_GRAPHS_PER_MODULE = 4
@@ -100,6 +106,10 @@ def offload_graphs_enabled() -> bool:
)
+def model_offload_graphs_requested() -> bool:
+ return (os.environ.get(MODEL_OFFLOAD_CUDA_GRAPH_ENV) or "").strip().lower() in _TRUE_TOKENS
+
+
def cuda_graph_disabled() -> bool:
"""Whether an env kill switch is set. Read before any torch or pipe inspection."""
if os.environ.get(CUDA_GRAPHS_ENV, "").strip().lower() in ("0", "off", "false", "no"):
@@ -1570,8 +1580,9 @@ class OffloadPlacement:
and the resident groups read in place. The pool holds the in-flight window at fixed addresses, so it plays the
prefetch ring. Recorded again when a residency change, a release / restore or a host-copy swap moves an address.
- ``model``: whole-module CPU offload (accelerate). The hook moves the module on the eager path first and the replay
- sits under it, in ``_old_forward``. Recorded again when the onload placed the weights at new addresses; after
+ ``model``: whole-module CPU offload (accelerate), only with ``MODEL_OFFLOAD_CUDA_GRAPH_ENV=1`` (see
+ ``MODEL_OFFLOAD_REASON``). The hook moves the module on the eager path first and the replay sits under it, in
+ ``_old_forward``. Recorded again when the onload placed the weights at new addresses; after
``MODEL_MAX_INVALIDATIONS`` re-records the module runs eager (each re-record costs a forward)."""
MODEL_MAX_INVALIDATIONS = 2
@@ -1843,6 +1854,8 @@ def offload_placement(module: Any) -> tuple:
None,
f"offload hook {type(hf_hook).__name__} moves the weights inside the forward",
)
+ if not model_offload_graphs_requested():
+ return None, MODEL_OFFLOAD_REASON
why = _forward_refusal(module, "model")
if why is not None:
return None, why
@@ -2430,7 +2443,7 @@ def arm_block_graphs(
return ()
try:
from .diffusion_block_graph import (
- MODEL_OFFLOAD_REASON,
+ BLOCK_GRAPHS_ENV,
OPT_IN_REASON,
block_graphs_disabled,
block_graphs_requested,
@@ -2459,7 +2472,12 @@ def arm_block_graphs(
if block_graphs_disabled():
_set_reason(pipe, "disabled by UNSLOTH_DIFFUSION_BLOCK_GRAPHS=0")
else:
- _set_reason(pipe, MODEL_OFFLOAD_REASON if model_offload else OPT_IN_REASON)
+ # Keep the whole step's own refusal visible (its opt-in, or why the opt-in still could not record).
+ if model_offload:
+ whole_why = prior if prior and prior != "offload active" else MODEL_OFFLOAD_REASON
+ _set_reason(pipe, whole_why + "; per block: set " + BLOCK_GRAPHS_ENV + "=1")
+ else:
+ _set_reason(pipe, OPT_IN_REASON)
return ()
armed: list = []
reasons: list = []
diff --git a/studio/backend/tests/test_diffusion_block_graph.py b/studio/backend/tests/test_diffusion_block_graph.py
index 31d366f8225..7ce7686a254 100644
--- a/studio/backend/tests/test_diffusion_block_graph.py
+++ b/studio/backend/tests/test_diffusion_block_graph.py
@@ -940,3 +940,22 @@ def test_a_variable_length_denoiser_records_no_block_graphs(monkeypatch):
assert handles == () and not applied["cuda_graph"]
assert cg.status_reason(pipe, False).startswith(cg.PROMPT_LENGTH_REASON)
assert not any(isinstance(b.__dict__.get("forward"), bg.BlockGraph) for b in net.blocks)
+
+
+def test_a_model_offloaded_denoiser_reason_names_both_opt_ins(monkeypatch):
+ pipe = _pipe_with(Net(blocks = 3))
+ pipe.transformer._hf_hook = types.SimpleNamespace()
+ handles, applied = _arm(pipe, monkeypatch, hooked = True, opt_in = False)
+ reason = pipe._unsloth_cuda_graph_reason
+ assert handles == () and applied["cuda_graph"] is False
+ assert cg.MODEL_OFFLOAD_CUDA_GRAPH_ENV in reason and bg.BLOCK_GRAPHS_ENV in reason
+
+
+def test_a_model_offloaded_denoiser_keeps_a_specific_whole_step_refusal(monkeypatch):
+ pipe = _pipe_with(Net(blocks = 3))
+ pipe.transformer._hf_hook = types.SimpleNamespace()
+ pipe._unsloth_cuda_graph_reason = "forward is not capture-safe: its rewrite is disabled"
+ handles, applied = _arm(pipe, monkeypatch, hooked = True, opt_in = False)
+ reason = pipe._unsloth_cuda_graph_reason
+ assert handles == () and reason.startswith("forward is not capture-safe")
+ assert cg.MODEL_OFFLOAD_CUDA_GRAPH_ENV not in reason and bg.BLOCK_GRAPHS_ENV in reason
diff --git a/studio/backend/tests/test_diffusion_offload_cuda_graph.py b/studio/backend/tests/test_diffusion_offload_cuda_graph.py
index aa92798480a..46a6bc4af0c 100644
--- a/studio/backend/tests/test_diffusion_offload_cuda_graph.py
+++ b/studio/backend/tests/test_diffusion_offload_cuda_graph.py
@@ -32,6 +32,7 @@ def _clean_env(monkeypatch):
op.PREFETCH_DEPTH_ENV,
cg.OFFLOAD_CUDA_GRAPH_ENV,
cg.CUDA_GRAPH_DISABLE_ENV,
+ cg.MODEL_OFFLOAD_CUDA_GRAPH_ENV,
cg.SPEED_CHECK_ENV,
"UNSLOTH_DIFFUSION_PARTIAL_RESIDENT",
"UNSLOTH_DIFFUSION_GROUP_OFFLOAD_PIN",
@@ -203,7 +204,8 @@ def test_sequential_offload_is_refused_with_the_reason():
assert placement is None and "sequential" in why
-def test_model_offload_replays_under_the_hook_and_records_again_when_weights_move():
+def test_model_offload_replays_under_the_hook_and_records_again_when_weights_move(monkeypatch):
+ monkeypatch.setenv(cg.MODEL_OFFLOAD_CUDA_GRAPH_ENV, "1")
_cuda()
accelerate = pytest.importorskip("accelerate")
from accelerate import cpu_offload_with_hook
@@ -235,9 +237,10 @@ def test_model_offload_replays_under_the_hook_and_records_again_when_weights_mov
del accelerate, other_hook
-def test_model_offload_re_records_a_moved_module_without_a_warm_up():
+def test_model_offload_re_records_a_moved_module_without_a_warm_up(monkeypatch):
"""Every onload of a model-offloaded denoiser may land at new addresses: the same key records again with no
warm-up step (its kernels exist), and replays stay exact across many moves."""
+ monkeypatch.setenv(cg.MODEL_OFFLOAD_CUDA_GRAPH_ENV, "1")
_cuda()
pytest.importorskip("accelerate")
from accelerate import cpu_offload_with_hook
@@ -279,6 +282,57 @@ def test_model_offload_re_records_a_moved_module_without_a_warm_up():
del _hold
+def _model_offloaded():
+ from accelerate import cpu_offload_with_hook
+
+ torch.manual_seed(0)
+ ref = _Net().cuda()
+ torch.manual_seed(0)
+ net, hook = cpu_offload_with_hook(_Net(), execution_device = "cuda")
+ return ref, net, hook
+
+
+def test_model_offload_keeps_the_step_eager_by_default_with_the_reason():
+ _cuda()
+ pytest.importorskip("accelerate")
+ _ref, net, _hook = _model_offloaded()
+ placement, why = cg.offload_placement(net)
+ assert placement is None and why == cg.MODEL_OFFLOAD_REASON
+ assert cg.MODEL_OFFLOAD_CUDA_GRAPH_ENV in why
+ handles, reason = cg.arm_after_placement(_pipe(net))
+ assert handles == () and "re-uploads the denoiser every render" in reason
+
+
[email protected]("opt_in", [False, True])
+def test_model_offload_records_nothing_after_the_first_render(monkeypatch, opt_in):
+ """Studio low_vram: renders 2+ record nothing by default; the opt-in records one per moved render."""
+ _cuda()
+ pytest.importorskip("accelerate")
+ if opt_in:
+ monkeypatch.setenv(cg.MODEL_OFFLOAD_CUDA_GRAPH_ENV, "1")
+ ref, net, hook = _model_offloaded()
+ handles, reason = cg.arm_after_placement(_pipe(net))
+ assert len(handles) == (1 if opt_in else 0), reason
+ dev = torch.cuda.current_device()
+ later, hold = 0, []
+ for render in range(4):
+ seeds = [10 * render + i for i in range(5)]
+ before = cg.capture_generation(dev)
+ got = _run(net, seeds)
+ if render:
+ later += cg.capture_generation(dev) - before
+ for a, b in zip(got, _run(ref, seeds)):
+ assert torch.equal(a, b)
+ hook.offload()
+ torch.cuda.empty_cache()
+ # the next stage takes the freed VRAM, so the next onload lands elsewhere
+ hold.append(torch.empty(4 << 20, device = "cuda"))
+ if opt_in:
+ assert later >= 1 and handles[0].stats["invalidations"] >= 1
+ else:
+ assert later == 0
+
+
def test_model_offload_stops_recording_only_when_moves_outpace_replays():
placement = cg.OffloadPlacement.__new__(cg.OffloadPlacement)
placement.mode = "model"