Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
Studio diffusion: the prompt encode streams only the free-memory shortfall (#15348)
Daniel Han2b48056ccd39
diff --git a/studio/backend/core/inference/diffusion_memory.py b/studio/backend/core/inference/diffusion_memory.pyindex d46ca48c622..51278639b0f 100644--- a/studio/backend/core/inference/diffusion_memory.py+++ b/studio/backend/core/inference/diffusion_memory.py@@ -5338,6 +5338,136 @@ def _is_text_encoder_module(pipe: Any, module: Any) -> bool:# "0": pin the denoiser back after every text encoder instead of once per encode_prompt call.ENCODE_RELEASE_ONCE_ENV = "UNSLOTH_DIFFUSION_ENCODE_RELEASE_ONCE"+# "0": the prompt encode always streams the whole surplus, even when free memory holds the encoder's need.+ENCODE_SHORTFALL_ENV = "UNSLOTH_DIFFUSION_ENCODE_SHORTFALL"+_ENCODE_SLACK_MIB = 512+# Per token, in hidden_size x 2 bytes (plus one per layer: output_hidden_states keeps every layer).+_ENCODE_TOKEN_HIDDEN_FACTOR = 32+# Vision inputs are not priced.+_ENCODE_VISUAL_KWARGS = (+ "pixel_values",+ "pixel_values_videos",+ "image_grid_thw",+ "video_grid_thw",+ "images",+)+++def _encoder_width(module: Any) -> tuple[Optional[int], int, int]:+ config = getattr(module, "config", None)+ for cfg in (config, getattr(config, "text_config", None)):+ hidden = getattr(cfg, "hidden_size", None) or getattr(cfg, "d_model", None)+ if hidden:+ heads = (+ getattr(cfg, "num_attention_heads", None) or getattr(cfg, "num_heads", None) or 1+ )+ layers = (+ getattr(cfg, "num_hidden_layers", None) or getattr(cfg, "num_layers", None) or 0+ )+ return int(hidden), int(heads), int(layers)+ return None, 1, 0+++def encode_need_mib(module: Any, args: Any, kwargs: Any) -> Optional[int]:+ """MiB one encoder call needs: the ``depth + 1`` largest streamed groups plus prompt activations, with margin.+ None when unpriceable (visual inputs, no token ids, unknown width)."""+ try:+ import torch++ from .diffusion_offload_prefetch import prefetch_depth++ kwargs = kwargs or {}+ if any(kwargs.get(name) is not None for name in _ENCODE_VISUAL_KWARGS):+ return None+ ids = kwargs.get("input_ids")+ if ids is None:+ ids = next(+ (+ a+ for a in (args or ())+ if isinstance(a, torch.Tensor)+ and a.dim() == 2+ and not torch.is_floating_point(a)+ ),+ None,+ )+ if ids is None:+ embeds = kwargs.get("inputs_embeds")+ if not isinstance(embeds, torch.Tensor) or embeds.dim() != 3:+ return None+ batch, tokens = int(embeds.shape[0]), int(embeds.shape[1])+ elif isinstance(ids, torch.Tensor) and ids.dim() == 2:+ batch, tokens = int(ids.shape[0]), int(ids.shape[1])+ else:+ return None+ hidden, heads, layers = _encoder_width(module)+ if not hidden:+ return None+ streamed = sorted(+ (+ _group_nbytes(g)+ for g in (_offload_groups(module) or ())+ if not getattr(g, "_unsloth_resident", False)+ ),+ reverse = True,+ )+ window = sum(streamed[: prefetch_depth() + 1])+ acts = (+ batch+ * tokens+ * (hidden * 2 * (_ENCODE_TOKEN_HIDDEN_FACTOR + layers) + heads * tokens * 4)+ )+ return (+ int(math.ceil((window + acts) / float(1 << 20) * _MEASURED_PEAK_MARGIN))+ + _ENCODE_SLACK_MIB+ )+ except Exception: # noqa: BLE001 - unpriced: the whole surplus streams+ return None+++def encode_available_mib(device: Any = None) -> Optional[int]:+ """Driver free less the VRAM Budget reserve, plus allocator cache outside CUDA graph pools, in MiB."""+ try:+ import torch++ dev = torch.device(device if device is not None else "cuda")+ if dev.type != "cuda":+ return None+ if dev.index is None:+ dev = torch.device("cuda", torch.cuda.current_device())+ free, total = (int(x) for x in torch.cuda.mem_get_info(dev))+ free -= user_budget_reserve_bytes(dev, total)+ cached = max(+ 0,+ int(torch.cuda.memory_reserved(dev))+ - int(torch.cuda.memory_allocated(dev))+ - _graph_held_cache_bytes(dev.index),+ )+ # clamp after adding the cache: a reserve larger than driver free still claims part of it+ return max(free + cached, 0) >> 20+ except Exception: # noqa: BLE001+ return None+++def _encode_device(pipe: Any) -> Any:+ for module in (getattr(pipe, "components", {}) or {}).values():+ device = getattr(module, "_unsloth_resident_device", None)+ if device is not None and getattr(module, "_unsloth_resident_room", None):+ return device+ return None+++def encode_release_mib(pipe: Any, module: Any, args: Any, kwargs: Any, surplus: int) -> int:+ """Only the shortfall (ComfyUI's ``free_memory(memory_required - free)``), at most ``surplus``; unpriced: all."""+ if _env_off(ENCODE_SHORTFALL_ENV):+ return surplus+ need = encode_need_mib(module, args, kwargs)+ if need is None:+ return surplus+ available = encode_available_mib(_encode_device(pipe))+ if available is None:+ return surplus+ return max(0, min(int(surplus), int(need) - int(available)))def install_encode_release(@@ -5367,29 +5497,44 @@ def install_encode_release(pipe_ref = lambda: pipe # noqa: E731 - not weak-referenceabledisable = getattr(getattr(torch, "compiler", None), "disable", None)pending: list = []-- def _before(module: Any, args: Any) -> None:+ asked = [0]+ active = [0]++ def _before(+ module: Any,+ args: Any,+ kwargs: Any = None,+ ) -> None:+ active[0] += 1owner = pipe_ref()- if owner is None or pending:+ if owner is None or active[0] > 1:return # nested encoder call: the outer one already released+ # Each encoder prices its own shortfall; a prompt never streams more than the surplus in total.+ need = min(+ encode_release_mib(owner, module, args, kwargs, surplus),+ surplus - asked[0],+ )+ if need <= 0:+ return+ asked[0] += needrestore = release_resident_groups(- owner, surplus, logger, denoisers_only = True, reason = "the prompt encode"+ owner, need, logger, denoisers_only = True, reason = "the prompt encode")# No empty_cache: the allocator frees cached segments and retries on OOM; flushing per prompt cost a sync.- pending.append(restore)+ if restore is not None:+ pending.append(restore)# encode_prompt calls in flight: their encoders keep one release and the outermost return pins backscope = [0]def _restore_pending() -> None:- if not pending:- return- restore = pending.pop()- if restore is not None:- restore()+ asked[0] = 0+ while pending:+ pending.pop()()def _after(module: Any, args: Any, output: Any) -> None:- if scope[0] > 0:+ active[0] = max(0, active[0] - 1)+ if active[0] > 0 or scope[0] > 0:return # the next encoder of this prompt reuses the room; encode_prompt's return pins back_restore_pending()@@ -5400,7 +5545,7 @@ def install_encode_release(if not str(name).startswith("text_encoder") or not isinstance(module, torch.nn.Module):continuehandles = (- module.register_forward_pre_hook(before),+ module.register_forward_pre_hook(before, with_kwargs = True),# always_call: an encode that raises (cancel, OOM) still pins the denoiser backmodule.register_forward_hook(after, always_call = True),)diff --git a/studio/backend/tests/test_diffusion_measured_placement.py b/studio/backend/tests/test_diffusion_measured_placement.pyindex 2a5027549cd..d99f0a764b1 100644--- a/studio/backend/tests/test_diffusion_measured_placement.py+++ b/studio/backend/tests/test_diffusion_measured_placement.py@@ -932,6 +932,147 @@ def test_encode_release_once_kill_switch(monkeypatch):assert calls == {"release": 2, "restore": 2} # once per encoder+def _token_encoder_pipe(monkeypatch, available_mib):+ torch, net, enc, pipe, x, ref, on_device = _encode_release_pipe(monkeypatch)++ class TokenEncoder(torch.nn.Module):+ def __init__(self):+ super().__init__()+ self.embed = torch.nn.Embedding(16, 8).cuda()+ self.config = types.SimpleNamespace(+ text_config = types.SimpleNamespace(+ hidden_size = 8, num_attention_heads = 1, num_hidden_layers = 1+ )+ )+ self.seen: list = []++ def forward(+ self,+ input_ids = None,+ attention_mask = None,+ pixel_values = None,+ ):+ self.seen.append(on_device())+ return self.embed(input_ids)++ tok = TokenEncoder()+ pipe.components["text_encoder"] = tok+ monkeypatch.setattr(+ dm, "encode_available_mib", lambda device = None: available_mib, raising = False+ )+ ids = torch.zeros(1, 4, dtype = torch.long, device = "cuda")+ return torch, net, tok, pipe, x, ref, on_device, ids+++def test_encode_keeps_the_denoiser_whole_when_free_memory_holds_the_encoder(monkeypatch):+ """Qwen-Image-2.1 at 12 GB: main streamed the whole surplus out and back on every new prompt."""+ torch, net, tok, pipe, x, ref, on_device, ids = _token_encoder_pipe(monkeypatch, 1 << 20)+ assert dm.install_encode_release(pipe, _whole_plan(1024, 1024 - 8), None) == 1+ for _ in range(3):+ tok(input_ids = ids)+ assert tok.seen[-1] == ["cuda"] * 6+ assert on_device() == ["cuda"] * 6+ assert torch.equal(net(x.cuda()).cpu(), ref)+++def test_encode_streams_only_the_shortfall(monkeypatch):+ torch, net, tok, pipe, x, ref, on_device, ids = _token_encoder_pipe(monkeypatch, 0)+ need = dm.encode_need_mib(tok, (), {"input_ids": ids})+ assert need is not None and need > 0+ monkeypatch.setattr(dm, "encode_available_mib", lambda device = None: need - 3)+ assert dm.install_encode_release(pipe, _whole_plan(1024, 1024 - 8), None) == 1+ tok(input_ids = ids)+ assert tok.seen[-1] == ["cuda"] * 5 + ["cpu"]+ assert on_device() == ["cuda"] * 6+ assert torch.equal(net(x.cuda()).cpu(), ref)+++def test_encode_shortfall_prices_each_encoder_of_a_prompt(monkeypatch):+ torch, net, tok, pipe, x, ref, on_device, ids = _token_encoder_pipe(monkeypatch, 0)+ big = type(tok)()+ big.config = types.SimpleNamespace(+ hidden_size = 1 << 20, num_attention_heads = 1, num_hidden_layers = 1+ )+ pipe.components["text_encoder_2"] = big+ small_need = dm.encode_need_mib(tok, (), {"input_ids": ids})+ big_need = dm.encode_need_mib(big, (), {"input_ids": ids})+ assert small_need < big_need - 3+ monkeypatch.setattr(dm, "encode_available_mib", lambda device = None: big_need - 3)++ def encode_prompt(prompt):+ tok(input_ids = ids)+ return big(input_ids = ids)++ pipe.encode_prompt = encode_prompt+ assert dm.install_encode_release(pipe, _whole_plan(1024, 1024 - 8), None) == 2+ for prompt in range(2):+ pipe.encode_prompt(f"p{prompt}")+ assert tok.seen[-1] == ["cuda"] * 6+ assert big.seen[-1] == ["cuda"] * 5 + ["cpu"]+ assert on_device() == ["cuda"] * 6+ assert torch.equal(net(x.cuda()).cpu(), ref)++[email protected]("why", ["kill_switch", "visual_input", "unreadable"])+def test_encode_streams_the_whole_surplus_when_unpriced(monkeypatch, why):+ torch, net, tok, pipe, x, ref, on_device, ids = _token_encoder_pipe(monkeypatch, 1 << 20)+ kwargs = {"input_ids": ids}+ if why == "kill_switch":+ monkeypatch.setenv(dm.ENCODE_SHORTFALL_ENV, "0")+ elif why == "visual_input":+ kwargs["pixel_values"] = torch.zeros(4, 3, device = "cuda")+ else:+ monkeypatch.setattr(dm, "encode_available_mib", lambda device = None: None)+ assert dm.install_encode_release(pipe, _whole_plan(1024, 1024 - 8), None) == 1+ tok(**kwargs)+ assert tok.seen[-1] == ["cuda"] * 4 + ["cpu"] * 2+ assert on_device() == ["cuda"] * 6+ assert torch.equal(net(x.cuda()).cpu(), ref)+++def test_encode_need_prices_the_largest_streamed_groups_and_the_tokens(monkeypatch):+ from core.inference import diffusion_offload_prefetch as pf++ torch = pytest.importorskip("torch")+ monkeypatch.setattr(pf, "prefetch_depth", lambda: 1)+ mib = 1 << 20+ groups = [+ types.SimpleNamespace(_unsloth_resident = False, n = 100 * mib),+ types.SimpleNamespace(_unsloth_resident = True, n = 900 * mib),+ types.SimpleNamespace(_unsloth_resident = False, n = 300 * mib),+ types.SimpleNamespace(_unsloth_resident = False, n = 200 * mib),+ ]+ monkeypatch.setattr(dm, "_offload_groups", lambda module: groups)+ monkeypatch.setattr(dm, "_group_nbytes", lambda g: g.n)+ enc = types.SimpleNamespace(+ config = types.SimpleNamespace(hidden_size = 1024, num_attention_heads = 8, num_hidden_layers = 4)+ )+ ids = torch.zeros(2, 64, dtype = torch.long)+ acts = 2 * 64 * (1024 * 2 * (dm._ENCODE_TOKEN_HIDDEN_FACTOR + 4) + 8 * 64 * 4)+ want = (+ int(__import__("math").ceil((500 * mib + acts) / float(mib) * dm._MEASURED_PEAK_MARGIN))+ + dm._ENCODE_SLACK_MIB+ )+ assert dm.encode_need_mib(enc, (ids,), {}) == want+ assert dm.encode_need_mib(enc, (), {"input_ids": ids}) == want+ assert dm.encode_need_mib(enc, (), {"input_ids": ids, "image_grid_thw": ids}) is None+ assert dm.encode_need_mib(types.SimpleNamespace(), (), {"input_ids": ids}) is None+++def test_encode_available_keeps_a_reserve_larger_than_driver_free(monkeypatch):+ torch = pytest.importorskip("torch")+ mib = 1 << 20+ monkeypatch.setattr(torch.cuda, "mem_get_info", lambda dev: (500 * mib, 8000 * mib))+ monkeypatch.setattr(dm, "user_budget_reserve_bytes", lambda dev, total: 2000 * mib)+ monkeypatch.setattr(torch.cuda, "memory_reserved", lambda dev: 3000 * mib)+ monkeypatch.setattr(torch.cuda, "memory_allocated", lambda dev: 1000 * mib)+ monkeypatch.setattr(dm, "_graph_held_cache_bytes", lambda index: 0)+ # 500 free + 2000 reusable cache - 2000 reserve: the reserve still claims the cache+ assert dm.encode_available_mib(torch.device("cuda", 0)) == 500+ monkeypatch.setattr(dm, "user_budget_reserve_bytes", lambda dev, total: 3000 * mib)+ assert dm.encode_available_mib(torch.device("cuda", 0)) == 0++def test_encode_release_only_for_the_whole_tier(monkeypatch):torch, net, enc, pipe, x, ref, on_device = _encode_release_pipe(monkeypatch)assert (