Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

Studio diffusion: the prompt encode streams only the free-memory shortfall (#15348)

Daniel Han2b48056ccd39
diff --git a/studio/backend/core/inference/diffusion_memory.py b/studio/backend/core/inference/diffusion_memory.py
index d46ca48c622..51278639b0f 100644
--- a/studio/backend/core/inference/diffusion_memory.py
+++ b/studio/backend/core/inference/diffusion_memory.py
@@ -5338,6 +5338,136 @@ def _is_text_encoder_module(pipe: Any, module: Any) -> bool:
# "0": pin the denoiser back after every text encoder instead of once per encode_prompt call.
ENCODE_RELEASE_ONCE_ENV = "UNSLOTH_DIFFUSION_ENCODE_RELEASE_ONCE"
+# "0": the prompt encode always streams the whole surplus, even when free memory holds the encoder's need.
+ENCODE_SHORTFALL_ENV = "UNSLOTH_DIFFUSION_ENCODE_SHORTFALL"
+_ENCODE_SLACK_MIB = 512
+# Per token, in hidden_size x 2 bytes (plus one per layer: output_hidden_states keeps every layer).
+_ENCODE_TOKEN_HIDDEN_FACTOR = 32
+# Vision inputs are not priced.
+_ENCODE_VISUAL_KWARGS = (
+ "pixel_values",
+ "pixel_values_videos",
+ "image_grid_thw",
+ "video_grid_thw",
+ "images",
+)
+
+
+def _encoder_width(module: Any) -> tuple[Optional[int], int, int]:
+ config = getattr(module, "config", None)
+ for cfg in (config, getattr(config, "text_config", None)):
+ hidden = getattr(cfg, "hidden_size", None) or getattr(cfg, "d_model", None)
+ if hidden:
+ heads = (
+ getattr(cfg, "num_attention_heads", None) or getattr(cfg, "num_heads", None) or 1
+ )
+ layers = (
+ getattr(cfg, "num_hidden_layers", None) or getattr(cfg, "num_layers", None) or 0
+ )
+ return int(hidden), int(heads), int(layers)
+ return None, 1, 0
+
+
+def encode_need_mib(module: Any, args: Any, kwargs: Any) -> Optional[int]:
+ """MiB one encoder call needs: the ``depth + 1`` largest streamed groups plus prompt activations, with margin.
+ None when unpriceable (visual inputs, no token ids, unknown width)."""
+ try:
+ import torch
+
+ from .diffusion_offload_prefetch import prefetch_depth
+
+ kwargs = kwargs or {}
+ if any(kwargs.get(name) is not None for name in _ENCODE_VISUAL_KWARGS):
+ return None
+ ids = kwargs.get("input_ids")
+ if ids is None:
+ ids = next(
+ (
+ a
+ for a in (args or ())
+ if isinstance(a, torch.Tensor)
+ and a.dim() == 2
+ and not torch.is_floating_point(a)
+ ),
+ None,
+ )
+ if ids is None:
+ embeds = kwargs.get("inputs_embeds")
+ if not isinstance(embeds, torch.Tensor) or embeds.dim() != 3:
+ return None
+ batch, tokens = int(embeds.shape[0]), int(embeds.shape[1])
+ elif isinstance(ids, torch.Tensor) and ids.dim() == 2:
+ batch, tokens = int(ids.shape[0]), int(ids.shape[1])
+ else:
+ return None
+ hidden, heads, layers = _encoder_width(module)
+ if not hidden:
+ return None
+ streamed = sorted(
+ (
+ _group_nbytes(g)
+ for g in (_offload_groups(module) or ())
+ if not getattr(g, "_unsloth_resident", False)
+ ),
+ reverse = True,
+ )
+ window = sum(streamed[: prefetch_depth() + 1])
+ acts = (
+ batch
+ * tokens
+ * (hidden * 2 * (_ENCODE_TOKEN_HIDDEN_FACTOR + layers) + heads * tokens * 4)
+ )
+ return (
+ int(math.ceil((window + acts) / float(1 << 20) * _MEASURED_PEAK_MARGIN))
+ + _ENCODE_SLACK_MIB
+ )
+ except Exception: # noqa: BLE001 - unpriced: the whole surplus streams
+ return None
+
+
+def encode_available_mib(device: Any = None) -> Optional[int]:
+ """Driver free less the VRAM Budget reserve, plus allocator cache outside CUDA graph pools, in MiB."""
+ try:
+ import torch
+
+ dev = torch.device(device if device is not None else "cuda")
+ if dev.type != "cuda":
+ return None
+ if dev.index is None:
+ dev = torch.device("cuda", torch.cuda.current_device())
+ free, total = (int(x) for x in torch.cuda.mem_get_info(dev))
+ free -= user_budget_reserve_bytes(dev, total)
+ cached = max(
+ 0,
+ int(torch.cuda.memory_reserved(dev))
+ - int(torch.cuda.memory_allocated(dev))
+ - _graph_held_cache_bytes(dev.index),
+ )
+ # clamp after adding the cache: a reserve larger than driver free still claims part of it
+ return max(free + cached, 0) >> 20
+ except Exception: # noqa: BLE001
+ return None
+
+
+def _encode_device(pipe: Any) -> Any:
+ for module in (getattr(pipe, "components", {}) or {}).values():
+ device = getattr(module, "_unsloth_resident_device", None)
+ if device is not None and getattr(module, "_unsloth_resident_room", None):
+ return device
+ return None
+
+
+def encode_release_mib(pipe: Any, module: Any, args: Any, kwargs: Any, surplus: int) -> int:
+ """Only the shortfall (ComfyUI's ``free_memory(memory_required - free)``), at most ``surplus``; unpriced: all."""
+ if _env_off(ENCODE_SHORTFALL_ENV):
+ return surplus
+ need = encode_need_mib(module, args, kwargs)
+ if need is None:
+ return surplus
+ available = encode_available_mib(_encode_device(pipe))
+ if available is None:
+ return surplus
+ return max(0, min(int(surplus), int(need) - int(available)))
def install_encode_release(
@@ -5367,29 +5497,44 @@ def install_encode_release(
pipe_ref = lambda: pipe # noqa: E731 - not weak-referenceable
disable = getattr(getattr(torch, "compiler", None), "disable", None)
pending: list = []
-
- def _before(module: Any, args: Any) -> None:
+ asked = [0]
+ active = [0]
+
+ def _before(
+ module: Any,
+ args: Any,
+ kwargs: Any = None,
+ ) -> None:
+ active[0] += 1
owner = pipe_ref()
- if owner is None or pending:
+ if owner is None or active[0] > 1:
return # nested encoder call: the outer one already released
+ # Each encoder prices its own shortfall; a prompt never streams more than the surplus in total.
+ need = min(
+ encode_release_mib(owner, module, args, kwargs, surplus),
+ surplus - asked[0],
+ )
+ if need <= 0:
+ return
+ asked[0] += need
restore = release_resident_groups(
- owner, surplus, logger, denoisers_only = True, reason = "the prompt encode"
+ owner, need, logger, denoisers_only = True, reason = "the prompt encode"
)
# No empty_cache: the allocator frees cached segments and retries on OOM; flushing per prompt cost a sync.
- pending.append(restore)
+ if restore is not None:
+ pending.append(restore)
# encode_prompt calls in flight: their encoders keep one release and the outermost return pins back
scope = [0]
def _restore_pending() -> None:
- if not pending:
- return
- restore = pending.pop()
- if restore is not None:
- restore()
+ asked[0] = 0
+ while pending:
+ pending.pop()()
def _after(module: Any, args: Any, output: Any) -> None:
- if scope[0] > 0:
+ active[0] = max(0, active[0] - 1)
+ if active[0] > 0 or scope[0] > 0:
return # the next encoder of this prompt reuses the room; encode_prompt's return pins back
_restore_pending()
@@ -5400,7 +5545,7 @@ def install_encode_release(
if not str(name).startswith("text_encoder") or not isinstance(module, torch.nn.Module):
continue
handles = (
- module.register_forward_pre_hook(before),
+ module.register_forward_pre_hook(before, with_kwargs = True),
# always_call: an encode that raises (cancel, OOM) still pins the denoiser back
module.register_forward_hook(after, always_call = True),
)
diff --git a/studio/backend/tests/test_diffusion_measured_placement.py b/studio/backend/tests/test_diffusion_measured_placement.py
index 2a5027549cd..d99f0a764b1 100644
--- a/studio/backend/tests/test_diffusion_measured_placement.py
+++ b/studio/backend/tests/test_diffusion_measured_placement.py
@@ -932,6 +932,147 @@ def test_encode_release_once_kill_switch(monkeypatch):
assert calls == {"release": 2, "restore": 2} # once per encoder
+def _token_encoder_pipe(monkeypatch, available_mib):
+ torch, net, enc, pipe, x, ref, on_device = _encode_release_pipe(monkeypatch)
+
+ class TokenEncoder(torch.nn.Module):
+ def __init__(self):
+ super().__init__()
+ self.embed = torch.nn.Embedding(16, 8).cuda()
+ self.config = types.SimpleNamespace(
+ text_config = types.SimpleNamespace(
+ hidden_size = 8, num_attention_heads = 1, num_hidden_layers = 1
+ )
+ )
+ self.seen: list = []
+
+ def forward(
+ self,
+ input_ids = None,
+ attention_mask = None,
+ pixel_values = None,
+ ):
+ self.seen.append(on_device())
+ return self.embed(input_ids)
+
+ tok = TokenEncoder()
+ pipe.components["text_encoder"] = tok
+ monkeypatch.setattr(
+ dm, "encode_available_mib", lambda device = None: available_mib, raising = False
+ )
+ ids = torch.zeros(1, 4, dtype = torch.long, device = "cuda")
+ return torch, net, tok, pipe, x, ref, on_device, ids
+
+
+def test_encode_keeps_the_denoiser_whole_when_free_memory_holds_the_encoder(monkeypatch):
+ """Qwen-Image-2.1 at 12 GB: main streamed the whole surplus out and back on every new prompt."""
+ torch, net, tok, pipe, x, ref, on_device, ids = _token_encoder_pipe(monkeypatch, 1 << 20)
+ assert dm.install_encode_release(pipe, _whole_plan(1024, 1024 - 8), None) == 1
+ for _ in range(3):
+ tok(input_ids = ids)
+ assert tok.seen[-1] == ["cuda"] * 6
+ assert on_device() == ["cuda"] * 6
+ assert torch.equal(net(x.cuda()).cpu(), ref)
+
+
+def test_encode_streams_only_the_shortfall(monkeypatch):
+ torch, net, tok, pipe, x, ref, on_device, ids = _token_encoder_pipe(monkeypatch, 0)
+ need = dm.encode_need_mib(tok, (), {"input_ids": ids})
+ assert need is not None and need > 0
+ monkeypatch.setattr(dm, "encode_available_mib", lambda device = None: need - 3)
+ assert dm.install_encode_release(pipe, _whole_plan(1024, 1024 - 8), None) == 1
+ tok(input_ids = ids)
+ assert tok.seen[-1] == ["cuda"] * 5 + ["cpu"]
+ assert on_device() == ["cuda"] * 6
+ assert torch.equal(net(x.cuda()).cpu(), ref)
+
+
+def test_encode_shortfall_prices_each_encoder_of_a_prompt(monkeypatch):
+ torch, net, tok, pipe, x, ref, on_device, ids = _token_encoder_pipe(monkeypatch, 0)
+ big = type(tok)()
+ big.config = types.SimpleNamespace(
+ hidden_size = 1 << 20, num_attention_heads = 1, num_hidden_layers = 1
+ )
+ pipe.components["text_encoder_2"] = big
+ small_need = dm.encode_need_mib(tok, (), {"input_ids": ids})
+ big_need = dm.encode_need_mib(big, (), {"input_ids": ids})
+ assert small_need < big_need - 3
+ monkeypatch.setattr(dm, "encode_available_mib", lambda device = None: big_need - 3)
+
+ def encode_prompt(prompt):
+ tok(input_ids = ids)
+ return big(input_ids = ids)
+
+ pipe.encode_prompt = encode_prompt
+ assert dm.install_encode_release(pipe, _whole_plan(1024, 1024 - 8), None) == 2
+ for prompt in range(2):
+ pipe.encode_prompt(f"p{prompt}")
+ assert tok.seen[-1] == ["cuda"] * 6
+ assert big.seen[-1] == ["cuda"] * 5 + ["cpu"]
+ assert on_device() == ["cuda"] * 6
+ assert torch.equal(net(x.cuda()).cpu(), ref)
+
+
[email protected]("why", ["kill_switch", "visual_input", "unreadable"])
+def test_encode_streams_the_whole_surplus_when_unpriced(monkeypatch, why):
+ torch, net, tok, pipe, x, ref, on_device, ids = _token_encoder_pipe(monkeypatch, 1 << 20)
+ kwargs = {"input_ids": ids}
+ if why == "kill_switch":
+ monkeypatch.setenv(dm.ENCODE_SHORTFALL_ENV, "0")
+ elif why == "visual_input":
+ kwargs["pixel_values"] = torch.zeros(4, 3, device = "cuda")
+ else:
+ monkeypatch.setattr(dm, "encode_available_mib", lambda device = None: None)
+ assert dm.install_encode_release(pipe, _whole_plan(1024, 1024 - 8), None) == 1
+ tok(**kwargs)
+ assert tok.seen[-1] == ["cuda"] * 4 + ["cpu"] * 2
+ assert on_device() == ["cuda"] * 6
+ assert torch.equal(net(x.cuda()).cpu(), ref)
+
+
+def test_encode_need_prices_the_largest_streamed_groups_and_the_tokens(monkeypatch):
+ from core.inference import diffusion_offload_prefetch as pf
+
+ torch = pytest.importorskip("torch")
+ monkeypatch.setattr(pf, "prefetch_depth", lambda: 1)
+ mib = 1 << 20
+ groups = [
+ types.SimpleNamespace(_unsloth_resident = False, n = 100 * mib),
+ types.SimpleNamespace(_unsloth_resident = True, n = 900 * mib),
+ types.SimpleNamespace(_unsloth_resident = False, n = 300 * mib),
+ types.SimpleNamespace(_unsloth_resident = False, n = 200 * mib),
+ ]
+ monkeypatch.setattr(dm, "_offload_groups", lambda module: groups)
+ monkeypatch.setattr(dm, "_group_nbytes", lambda g: g.n)
+ enc = types.SimpleNamespace(
+ config = types.SimpleNamespace(hidden_size = 1024, num_attention_heads = 8, num_hidden_layers = 4)
+ )
+ ids = torch.zeros(2, 64, dtype = torch.long)
+ acts = 2 * 64 * (1024 * 2 * (dm._ENCODE_TOKEN_HIDDEN_FACTOR + 4) + 8 * 64 * 4)
+ want = (
+ int(__import__("math").ceil((500 * mib + acts) / float(mib) * dm._MEASURED_PEAK_MARGIN))
+ + dm._ENCODE_SLACK_MIB
+ )
+ assert dm.encode_need_mib(enc, (ids,), {}) == want
+ assert dm.encode_need_mib(enc, (), {"input_ids": ids}) == want
+ assert dm.encode_need_mib(enc, (), {"input_ids": ids, "image_grid_thw": ids}) is None
+ assert dm.encode_need_mib(types.SimpleNamespace(), (), {"input_ids": ids}) is None
+
+
+def test_encode_available_keeps_a_reserve_larger_than_driver_free(monkeypatch):
+ torch = pytest.importorskip("torch")
+ mib = 1 << 20
+ monkeypatch.setattr(torch.cuda, "mem_get_info", lambda dev: (500 * mib, 8000 * mib))
+ monkeypatch.setattr(dm, "user_budget_reserve_bytes", lambda dev, total: 2000 * mib)
+ monkeypatch.setattr(torch.cuda, "memory_reserved", lambda dev: 3000 * mib)
+ monkeypatch.setattr(torch.cuda, "memory_allocated", lambda dev: 1000 * mib)
+ monkeypatch.setattr(dm, "_graph_held_cache_bytes", lambda index: 0)
+ # 500 free + 2000 reusable cache - 2000 reserve: the reserve still claims the cache
+ assert dm.encode_available_mib(torch.device("cuda", 0)) == 500
+ monkeypatch.setattr(dm, "user_budget_reserve_bytes", lambda dev, total: 3000 * mib)
+ assert dm.encode_available_mib(torch.device("cuda", 0)) == 0
+
+
def test_encode_release_only_for_the_whole_tier(monkeypatch):
torch, net, enc, pipe, x, ref, on_device = _encode_release_pipe(monkeypatch)
assert (