Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
Studio diffusion: keep a model-offloaded denoiser's step eager instead of re-recording its CUDA graph every render (#14497)
Daniel Han4d7ebf5ceaf4
diff --git a/studio/backend/core/inference/diffusion_cuda_graph.py b/studio/backend/core/inference/diffusion_cuda_graph.pyindex a5e3859fced..8148e4ecb7e 100644--- a/studio/backend/core/inference/diffusion_cuda_graph.py+++ b/studio/backend/core/inference/diffusion_cuda_graph.py@@ -31,6 +31,12 @@ CUDA_GRAPHS_ENV = "UNSLOTH_DIFFUSION_CUDA_GRAPHS"OFFLOAD_CUDA_GRAPH_ENV = "UNSLOTH_DIFFUSION_OFFLOAD_CUDA_GRAPH"# =0: a block-streamed whole-step graph records its copies into the graph pool instead of the prefetcher's slot ring.STEP_SLOTS_ENV = "UNSLOTH_DIFFUSION_STEP_GRAPH_SLOTS"+MODEL_OFFLOAD_CUDA_GRAPH_ENV = "UNSLOTH_DIFFUSION_MODEL_OFFLOAD_CUDA_GRAPH"+# Default off: replay saved ~1% of a step (FLUX.2-klein 4B, 1024^2) but the per-render re-record + ~0.6 GB pool cost more.+MODEL_OFFLOAD_REASON = (+ "model offload re-uploads the denoiser every render at new addresses, so its step graph would record again on "+ "every render; set " + MODEL_OFFLOAD_CUDA_GRAPH_ENV + "=1 to record it anyway"+)# Per module. Legitimate second keys exist, but each costs a pool-sized slice of VRAM.MAX_GRAPHS_PER_MODULE = 4@@ -100,6 +106,10 @@ def offload_graphs_enabled() -> bool:)+def model_offload_graphs_requested() -> bool:+ return (os.environ.get(MODEL_OFFLOAD_CUDA_GRAPH_ENV) or "").strip().lower() in _TRUE_TOKENS++def cuda_graph_disabled() -> bool:"""Whether an env kill switch is set. Read before any torch or pipe inspection."""if os.environ.get(CUDA_GRAPHS_ENV, "").strip().lower() in ("0", "off", "false", "no"):@@ -1570,8 +1580,9 @@ class OffloadPlacement:and the resident groups read in place. The pool holds the in-flight window at fixed addresses, so it plays theprefetch ring. Recorded again when a residency change, a release / restore or a host-copy swap moves an address.- ``model``: whole-module CPU offload (accelerate). The hook moves the module on the eager path first and the replay- sits under it, in ``_old_forward``. Recorded again when the onload placed the weights at new addresses; after+ ``model``: whole-module CPU offload (accelerate), only with ``MODEL_OFFLOAD_CUDA_GRAPH_ENV=1`` (see+ ``MODEL_OFFLOAD_REASON``). The hook moves the module on the eager path first and the replay sits under it, in+ ``_old_forward``. Recorded again when the onload placed the weights at new addresses; after``MODEL_MAX_INVALIDATIONS`` re-records the module runs eager (each re-record costs a forward)."""MODEL_MAX_INVALIDATIONS = 2@@ -1843,6 +1854,8 @@ def offload_placement(module: Any) -> tuple:None,f"offload hook {type(hf_hook).__name__} moves the weights inside the forward",)+ if not model_offload_graphs_requested():+ return None, MODEL_OFFLOAD_REASONwhy = _forward_refusal(module, "model")if why is not None:return None, why@@ -2430,7 +2443,7 @@ def arm_block_graphs(return ()try:from .diffusion_block_graph import (- MODEL_OFFLOAD_REASON,+ BLOCK_GRAPHS_ENV,OPT_IN_REASON,block_graphs_disabled,block_graphs_requested,@@ -2459,7 +2472,12 @@ def arm_block_graphs(if block_graphs_disabled():_set_reason(pipe, "disabled by UNSLOTH_DIFFUSION_BLOCK_GRAPHS=0")else:- _set_reason(pipe, MODEL_OFFLOAD_REASON if model_offload else OPT_IN_REASON)+ # Keep the whole step's own refusal visible (its opt-in, or why the opt-in still could not record).+ if model_offload:+ whole_why = prior if prior and prior != "offload active" else MODEL_OFFLOAD_REASON+ _set_reason(pipe, whole_why + "; per block: set " + BLOCK_GRAPHS_ENV + "=1")+ else:+ _set_reason(pipe, OPT_IN_REASON)return ()armed: list = []reasons: list = []diff --git a/studio/backend/tests/test_diffusion_block_graph.py b/studio/backend/tests/test_diffusion_block_graph.pyindex 31d366f8225..7ce7686a254 100644--- a/studio/backend/tests/test_diffusion_block_graph.py+++ b/studio/backend/tests/test_diffusion_block_graph.py@@ -940,3 +940,22 @@ def test_a_variable_length_denoiser_records_no_block_graphs(monkeypatch):assert handles == () and not applied["cuda_graph"]assert cg.status_reason(pipe, False).startswith(cg.PROMPT_LENGTH_REASON)assert not any(isinstance(b.__dict__.get("forward"), bg.BlockGraph) for b in net.blocks)+++def test_a_model_offloaded_denoiser_reason_names_both_opt_ins(monkeypatch):+ pipe = _pipe_with(Net(blocks = 3))+ pipe.transformer._hf_hook = types.SimpleNamespace()+ handles, applied = _arm(pipe, monkeypatch, hooked = True, opt_in = False)+ reason = pipe._unsloth_cuda_graph_reason+ assert handles == () and applied["cuda_graph"] is False+ assert cg.MODEL_OFFLOAD_CUDA_GRAPH_ENV in reason and bg.BLOCK_GRAPHS_ENV in reason+++def test_a_model_offloaded_denoiser_keeps_a_specific_whole_step_refusal(monkeypatch):+ pipe = _pipe_with(Net(blocks = 3))+ pipe.transformer._hf_hook = types.SimpleNamespace()+ pipe._unsloth_cuda_graph_reason = "forward is not capture-safe: its rewrite is disabled"+ handles, applied = _arm(pipe, monkeypatch, hooked = True, opt_in = False)+ reason = pipe._unsloth_cuda_graph_reason+ assert handles == () and reason.startswith("forward is not capture-safe")+ assert cg.MODEL_OFFLOAD_CUDA_GRAPH_ENV not in reason and bg.BLOCK_GRAPHS_ENV in reasondiff --git a/studio/backend/tests/test_diffusion_offload_cuda_graph.py b/studio/backend/tests/test_diffusion_offload_cuda_graph.pyindex aa92798480a..46a6bc4af0c 100644--- a/studio/backend/tests/test_diffusion_offload_cuda_graph.py+++ b/studio/backend/tests/test_diffusion_offload_cuda_graph.py@@ -32,6 +32,7 @@ def _clean_env(monkeypatch):op.PREFETCH_DEPTH_ENV,cg.OFFLOAD_CUDA_GRAPH_ENV,cg.CUDA_GRAPH_DISABLE_ENV,+ cg.MODEL_OFFLOAD_CUDA_GRAPH_ENV,cg.SPEED_CHECK_ENV,"UNSLOTH_DIFFUSION_PARTIAL_RESIDENT","UNSLOTH_DIFFUSION_GROUP_OFFLOAD_PIN",@@ -203,7 +204,8 @@ def test_sequential_offload_is_refused_with_the_reason():assert placement is None and "sequential" in why-def test_model_offload_replays_under_the_hook_and_records_again_when_weights_move():+def test_model_offload_replays_under_the_hook_and_records_again_when_weights_move(monkeypatch):+ monkeypatch.setenv(cg.MODEL_OFFLOAD_CUDA_GRAPH_ENV, "1")_cuda()accelerate = pytest.importorskip("accelerate")from accelerate import cpu_offload_with_hook@@ -235,9 +237,10 @@ def test_model_offload_replays_under_the_hook_and_records_again_when_weights_movdel accelerate, other_hook-def test_model_offload_re_records_a_moved_module_without_a_warm_up():+def test_model_offload_re_records_a_moved_module_without_a_warm_up(monkeypatch):"""Every onload of a model-offloaded denoiser may land at new addresses: the same key records again with nowarm-up step (its kernels exist), and replays stay exact across many moves."""+ monkeypatch.setenv(cg.MODEL_OFFLOAD_CUDA_GRAPH_ENV, "1")_cuda()pytest.importorskip("accelerate")from accelerate import cpu_offload_with_hook@@ -279,6 +282,57 @@ def test_model_offload_re_records_a_moved_module_without_a_warm_up():del _hold+def _model_offloaded():+ from accelerate import cpu_offload_with_hook++ torch.manual_seed(0)+ ref = _Net().cuda()+ torch.manual_seed(0)+ net, hook = cpu_offload_with_hook(_Net(), execution_device = "cuda")+ return ref, net, hook+++def test_model_offload_keeps_the_step_eager_by_default_with_the_reason():+ _cuda()+ pytest.importorskip("accelerate")+ _ref, net, _hook = _model_offloaded()+ placement, why = cg.offload_placement(net)+ assert placement is None and why == cg.MODEL_OFFLOAD_REASON+ assert cg.MODEL_OFFLOAD_CUDA_GRAPH_ENV in why+ handles, reason = cg.arm_after_placement(_pipe(net))+ assert handles == () and "re-uploads the denoiser every render" in reason++[email protected]("opt_in", [False, True])+def test_model_offload_records_nothing_after_the_first_render(monkeypatch, opt_in):+ """Studio low_vram: renders 2+ record nothing by default; the opt-in records one per moved render."""+ _cuda()+ pytest.importorskip("accelerate")+ if opt_in:+ monkeypatch.setenv(cg.MODEL_OFFLOAD_CUDA_GRAPH_ENV, "1")+ ref, net, hook = _model_offloaded()+ handles, reason = cg.arm_after_placement(_pipe(net))+ assert len(handles) == (1 if opt_in else 0), reason+ dev = torch.cuda.current_device()+ later, hold = 0, []+ for render in range(4):+ seeds = [10 * render + i for i in range(5)]+ before = cg.capture_generation(dev)+ got = _run(net, seeds)+ if render:+ later += cg.capture_generation(dev) - before+ for a, b in zip(got, _run(ref, seeds)):+ assert torch.equal(a, b)+ hook.offload()+ torch.cuda.empty_cache()+ # the next stage takes the freed VRAM, so the next onload lands elsewhere+ hold.append(torch.empty(4 << 20, device = "cuda"))+ if opt_in:+ assert later >= 1 and handles[0].stats["invalidations"] >= 1+ else:+ assert later == 0++def test_model_offload_stops_recording_only_when_moves_outpace_replays():placement = cg.OffloadPlacement.__new__(cg.OffloadPlacement)placement.mode = "model"