Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

Studio: keep the whole int8 Qwen-Image-2.1 denoiser resident at 12 GB, released only while the encoders run (298 to 212 ms per step) (#12536)

Daniel Han80c92a6ef974
diff --git a/studio/backend/core/inference/diffusion.py b/studio/backend/core/inference/diffusion.py
index 28f4f711120..1f1ff3445ec 100644
--- a/studio/backend/core/inference/diffusion.py
+++ b/studio/backend/core/inference/diffusion.py
@@ -133,6 +133,7 @@ from .diffusion_memory import (
plan_diffusion_memory,
plan_fits_total_capacity,
denoisers_pinned_resident,
+ install_encode_release,
plan_keeps_transformer_resident,
prequant_seed_device,
raise_on_image_activation_shortfall,
@@ -7155,6 +7156,8 @@ class DiffusionBackend:
pipe._unsloth_cuda_graphs = ()
pipe._unsloth_cuda_graph_reason = "offload active"
speed_applied["cuda_graph"] = False
+ # streams the whole-resident denoiser back to the flat room while the encoders run
+ install_encode_release(pipe, plan, logger)
# the speed layer saw only the plan; placement may have pinned every denoiser group since
if denoisers_pinned_resident(pipe):
engage_pinned_denoisers(pipe, speed_applied, logger)
diff --git a/studio/backend/core/inference/diffusion_memory.py b/studio/backend/core/inference/diffusion_memory.py
index b1b14dbf24c..df2996ccb35 100644
--- a/studio/backend/core/inference/diffusion_memory.py
+++ b/studio/backend/core/inference/diffusion_memory.py
@@ -2283,6 +2283,31 @@ def _loaded_component_mib(pipe: Any) -> Optional[dict[str, tuple[int, str]]]:
return None
+RESIDENT_DIT_ENV = "UNSLOTH_DIFFUSION_RESIDENT_DIT"
+# Whole-resident tier slack: 10% of the card, min 1 GiB, instead of the flat reserve + base overhead sized for an
+# unmeasured activation. Free memory is read after CUDA init and the measured peak covers encode, steps and decode, so
+# the slack only covers fragmentation and lazily loaded kernels.
+_RESIDENT_DIT_SLACK_FRACTION = 0.10
+_RESIDENT_DIT_SLACK_MIN_MIB = 1024
+
+
+def _resident_dit_slack_mib(memory: Any) -> int:
+ base = getattr(memory, "total_mib", None) or getattr(memory, "free_mib", None) or 0
+ return max(_RESIDENT_DIT_SLACK_MIN_MIB, int(int(base) * _RESIDENT_DIT_SLACK_FRACTION))
+
+
+def _resident_dit_fits(memory: Any, dit_mib: int, headroom_mib: int, other_mib: int) -> bool:
+ """Whole denoiser + non-encoder companions + measured peak x margin + slack within free memory, encoders streamed."""
+ if _env_off(RESIDENT_DIT_ENV):
+ return False
+ free = getattr(memory, "free_mib", None)
+ if free is None:
+ return False
+ return int(dit_mib) + int(other_mib) + int(headroom_mib) + _resident_dit_slack_mib(
+ memory
+ ) <= int(free)
+
+
def _denoiser_compute_bytes(pipe: Any) -> Optional[int]:
"""Denoiser compute element size for the dense table: 2 (fp16), 4 (fp32); None for bf16 / unreadable."""
try:
@@ -2360,14 +2385,28 @@ def refine_plan_from_loaded_weights(
room = budget - floor
if policy == OFFLOAD_GROUP and not stream_te:
room -= encoders # resident companions
+ whole_dit = False
if not bool(getattr(plan, "stream_transformer", True)):
room -= dit
dit_room = 0
else:
dit_room = min(max(room, 0), dit)
+ if (
+ dit_room < dit
+ # torchao denoisers only: the slack was measured on the int8 route, not the dense eager table
+ and dense_mib is None
+ and (policy == OFFLOAD_STREAMING or stream_te)
+ and _resident_dit_fits(memory, dit, headroom, other)
+ ):
+ # pin it whole; during the encode it drops back to the flat room (install_encode_release)
+ encode_room = int(dit_room)
+ dit_room, room, whole_dit = dit, dit, True
te_room = max(0, room - dit_room) if stream_te and policy == OFFLOAD_GROUP else 0
if dit_room <= 0 and te_room <= 0:
return plan
+ if whole_dit:
+ estimates["resident_dit_slack_mib"] = _resident_dit_slack_mib(memory)
+ estimates["encode_resident_transformer_mib"] = encode_room
new = replace(
plan,
resident_transformer_mib = int(dit_room) if dit_room > 0 else None,
@@ -2380,7 +2419,13 @@ def refine_plan_from_loaded_weights(
reasons = plan.reasons
+ (
f"{int(dit_room)} MiB of the {dit} MiB transformer and {int(te_room)} MiB of the {encoders} MiB "
- f"encoders stay resident (measured peak {headroom} MiB); the rest streams",
+ f"encoders stay resident (measured peak {headroom} MiB); the rest streams"
+ + (
+ f" (whole transformer within free memory less a {_resident_dit_slack_mib(memory)} MiB slack;"
+ f" {encode_room} MiB of it while the encoders run)"
+ if whole_dit
+ else ""
+ ),
),
)
if logger is not None:
@@ -2414,9 +2459,11 @@ def _keep_groups_resident(
room_mib: int,
device: Any,
logger: Any = None,
+ only: Optional[set] = None,
) -> int:
"""Make whole offload groups of a streamed ``module`` resident within ``room_mib`` (top-level group first, then
- blocks in order); their onload / offload become no-ops and the rest keeps streaming. Returns the MiB kept."""
+ blocks in order); their onload / offload become no-ops and the rest keeps streaming. Returns the MiB kept.
+ ``only`` (group ids): a restore pins back what its own release streamed, not an enclosing release's groups."""
if room_mib is None or int(room_mib) <= 0 or _env_off(PARTIAL_RESIDENT_ENV):
return 0
try:
@@ -2484,8 +2531,8 @@ def _keep_groups_resident(
if getattr(group, "_unsloth_resident", False):
left -= need
continue
- if need > left:
- # too large: it streams; a later, smaller group may still fit
+ if need > left or (only is not None and id(group) not in only):
+ # too large (a later, smaller group may still fit), or another release's group
continue
cpu = getattr(group, "cpu_param_dict", None) or {}
for t in tensors:
@@ -2548,10 +2595,13 @@ def release_resident_groups(
pipe: Any,
need_mib: int,
logger: Any = None,
+ denoisers_only: bool = False,
+ reason: str = "an oversized request",
) -> Optional[Callable[[], None]]:
"""Stream resident offload groups again until ``need_mib`` is freed (text encoders first, then the denoiser's
- blocks from the last); returns a callable restoring them, or None when nothing was resident. A request larger
- than the measured placement reserved (reference images, a bigger canvas, a batch) then runs as the flat plan."""
+ blocks from the last); returns a callable pinning exactly those groups again, or None when nothing was resident.
+ A request larger than the measured placement reserved (reference images, a bigger canvas, a batch) then runs as
+ the flat plan. ``denoisers_only`` leaves the text encoders' groups alone (the prompt encode needs them)."""
try:
import torch
@@ -2559,12 +2609,14 @@ def release_resident_groups(
m
for m in (getattr(pipe, "components", {}) or {}).values()
if getattr(m, "_unsloth_resident_room", None)
+ and not (denoisers_only and _is_text_encoder_module(pipe, m))
]
if not modules or int(need_mib) <= 0:
return None
modules.sort(key = lambda m: 0 if _is_text_encoder_module(pipe, m) else 1)
left = int(need_mib) * 1024 * 1024
released: list = []
+ streamed: set = set()
for module in modules:
for group in reversed(_offload_groups(module) or []):
if left <= 0:
@@ -2576,6 +2628,7 @@ def release_resident_groups(
if isinstance(state, dict):
state["streamed"] = 1
_release_group(group)
+ streamed.add(id(group))
left -= int(getattr(group, "_unsloth_resident_bytes", 0))
if module not in released:
released.append(module)
@@ -2590,8 +2643,9 @@ def release_resident_groups(
torch.cuda.synchronize(device)
if logger is not None:
logger.info(
- "diffusion.memory: streaming %d MiB of resident groups for an oversized request",
+ "diffusion.memory: streaming %d MiB of resident groups for %s",
(int(need_mib) * 1024 * 1024 - max(left, 0)) >> 20,
+ reason,
)
def restore() -> None:
@@ -2601,6 +2655,7 @@ def release_resident_groups(
module._unsloth_resident_room,
module._unsloth_resident_device,
logger,
+ only = streamed,
)
return restore
@@ -2617,6 +2672,85 @@ def _is_text_encoder_module(pipe: Any, module: Any) -> bool:
return False
+def install_encode_release(
+ pipe: Any,
+ plan: Any,
+ logger: Any = None,
+) -> int:
+ """While a text encoder runs, stream the whole-resident denoiser back to the flat room (the partial placement's
+ encode state); pin it back on return. Returns the number of encoders hooked."""
+ estimates = getattr(plan, "estimates", None) or {}
+ encode_room = estimates.get("encode_resident_transformer_mib")
+ whole = getattr(plan, "resident_transformer_mib", None)
+ if encode_room is None or not whole:
+ return 0
+ surplus = int(whole) - max(int(encode_room), 0)
+ if surplus <= 0:
+ return 0
+ try:
+ import weakref
+
+ import torch
+
+ try:
+ pipe_ref = weakref.ref(pipe) # the hooks live on the encoder, which the pipe owns
+ except TypeError:
+ pipe_ref = lambda: pipe # noqa: E731 - not weak-referenceable
+ disable = getattr(getattr(torch, "compiler", None), "disable", None)
+ pending: list = []
+
+ def _before(module: Any, args: Any) -> None:
+ owner = pipe_ref()
+ if owner is None or pending:
+ return # nested encoder call: the outer one already released
+ restore = release_resident_groups(
+ owner, surplus, logger, denoisers_only = True, reason = "the prompt encode"
+ )
+ pending.append(restore)
+ if restore is not None and torch.cuda.is_available():
+ # freed blocks sit in the default stream's pool; the encoder onloads on the copy stream
+ torch.cuda.empty_cache()
+
+ def _after(module: Any, args: Any, output: Any) -> None:
+ if not pending:
+ return
+ restore = pending.pop()
+ if restore is not None:
+ restore()
+
+ before = disable(_before) if callable(disable) else _before
+ after = disable(_after) if callable(disable) else _after
+ hooked = 0
+ for name, module in (getattr(pipe, "components", {}) or {}).items():
+ if not str(name).startswith("text_encoder") or not isinstance(module, torch.nn.Module):
+ continue
+ handles = (
+ module.register_forward_pre_hook(before),
+ # always_call: an encode that raises (cancel, OOM) still pins the denoiser back
+ module.register_forward_hook(after, always_call = True),
+ )
+ module._unsloth_encode_release = handles
+ hooked += 1
+ if hooked and logger is not None:
+ logger.info(
+ "diffusion.memory: the prompt encode streams %d MiB of the resident transformer (%d MiB stay), "
+ "pinned again before step 0",
+ surplus,
+ max(int(encode_room), 0),
+ )
+ return hooked
+ except Exception as exc: # noqa: BLE001 - fall back to the partial placement rather than risk the encode
+ if logger is not None:
+ logger.warning(
+ "diffusion.memory: encode release not installed (%s); the transformer streams past the flat room",
+ exc,
+ )
+ release_resident_groups(
+ pipe, surplus, logger, denoisers_only = True, reason = "the partial placement"
+ )
+ return 0
+
+
def measured_request_extra_mib(
pipe: Any,
*,
diff --git a/studio/backend/tests/test_diffusion_measured_placement.py b/studio/backend/tests/test_diffusion_measured_placement.py
index 5e2ce8d365b..1d07711baf3 100644
--- a/studio/backend/tests/test_diffusion_measured_placement.py
+++ b/studio/backend/tests/test_diffusion_measured_placement.py
@@ -150,15 +150,52 @@ def test_16gb_encoder_room_and_kill_switch(q21_pipe, monkeypatch):
assert _refine(q21_pipe, plan) is plan
-def test_12gb_partial_residency_sized_from_budget(q21_pipe):
+def test_12gb_keeps_the_whole_transformer_resident_encoders_streamed(q21_pipe, monkeypatch):
+ """12 GB: the flat margins fit only part of the int8 DiT; with encoders streamed the whole DiT fits the slack."""
+ monkeypatch.delenv(dm.RESIDENT_DIT_ENV, raising = False)
plan = _flat_plan(9550, 12288)
assert plan.offload_policy == dm.OFFLOAD_STREAMING
+ free = plan.device_memory.free_mib
new = _refine(q21_pipe, plan)
- assert new.offload_policy == dm.OFFLOAD_STREAMING
- room = 9550 - 2304 - dm.DEFAULT_BASE_OVERHEAD_MIB - 644
+ assert new.offload_policy == dm.OFFLOAD_STREAMING and new.stream_transformer
+ assert new.resident_transformer_mib == 6922
+ assert new.resident_text_encoder_mib is None
+ assert new.estimates["resident_dit_slack_mib"] == 1228
+ # while the encoders run, the transformer drops back to the flat room the partial placement kept
+ assert (
+ new.estimates["encode_resident_transformer_mib"]
+ == 9550 - 2304 - dm.DEFAULT_BASE_OVERHEAD_MIB - 644
+ )
+ assert 6922 + 644 + 2304 + 1228 <= free
+ assert new.as_public_dict()["resident_transformer_mib"] == 6922
+
+
+def test_12gb_partial_residency_when_the_whole_transformer_does_not_fit(q21_pipe, monkeypatch):
+ monkeypatch.delenv(dm.RESIDENT_DIT_ENV, raising = False)
+ # 500 MiB less free memory (a desktop session on the card): the whole DiT misses the slack, the flat room holds
+ plan = _flat_plan(9000, 12288)
+ assert 6922 + 644 + 2304 + 1228 > plan.device_memory.free_mib
+ new = _refine(q21_pipe, plan)
+ room = 9000 - 2304 - dm.DEFAULT_BASE_OVERHEAD_MIB - 644
assert new.resident_transformer_mib == room
assert 0 < room < 6922
- assert new.as_public_dict()["resident_transformer_mib"] == room
+ assert "resident_dit_slack_mib" not in new.estimates
+
+
+def test_whole_transformer_tier_kill_switch(q21_pipe, monkeypatch):
+ monkeypatch.setenv(dm.RESIDENT_DIT_ENV, "0")
+ new = _refine(q21_pipe, _flat_plan(9550, 12288))
+ assert new.resident_transformer_mib == 9550 - 2304 - dm.DEFAULT_BASE_OVERHEAD_MIB - 644
+
+
+def test_whole_transformer_tier_needs_streamed_encoders(q21_pipe, monkeypatch):
+ """A group plan whose encoders stay resident cannot trade them for the DiT: the tier leaves it alone."""
+ monkeypatch.delenv(dm.RESIDENT_DIT_ENV, raising = False)
+ plan = dm.replace(
+ _flat_plan(9550, 12288), offload_policy = dm.OFFLOAD_GROUP, stream_text_encoders = False
+ )
+ new = _refine(q21_pipe, plan)
+ assert int(new.resident_transformer_mib or 0) < 6922
def test_no_room_keeps_flat_plan(q21_pipe):
@@ -237,6 +274,13 @@ def test_every_tier_fits_measured_need(q21_pipe, budget):
+ (0 if new.stream_text_encoders or new.offload_policy == dm.OFFLOAD_STREAMING else 8959)
)
assert int(new.resident_transformer_mib or 0) <= 6922
+ if "resident_dit_slack_mib" in new.estimates:
+ # the whole-DiT tier: every encoder streams, and weights + peak x margin + slack fit free memory
+ assert new.resident_transformer_mib == 6922 and not new.resident_text_encoder_mib
+ slack = new.estimates["resident_dit_slack_mib"]
+ assert slack >= max(1024, total // 10)
+ assert kept + 2304 + slack <= plan.device_memory.free_mib
+ return
assert kept + head <= budget
@@ -596,6 +640,126 @@ def test_torchao_groups_stay_on_device_after_release_and_restore(monkeypatch):
assert torch.equal(net(x), ref)
+def _encode_release_pipe(monkeypatch):
+ """A real group-offloaded denoiser pinned whole; the encoder records which blocks are on the device."""
+ torch, net = _cuda_offload_model()
+ from diffusers.hooks import apply_group_offloading
+
+ monkeypatch.delenv("UNSLOTH_DIFFUSION_PARTIAL_RESIDENT", raising = False)
+ x = torch.randn(3, 64)
+ ref = net.to("cuda")(x.cuda()).cpu()
+ net.to("cpu")
+ apply_group_offloading(
+ net,
+ onload_device = torch.device("cuda"),
+ offload_device = torch.device("cpu"),
+ offload_type = "block_level",
+ num_blocks_per_group = 1,
+ use_stream = True,
+ record_stream = True,
+ non_blocking = True,
+ )
+
+ def on_device():
+ return [next(b.parameters()).device.type for b in net.blocks]
+
+ class Encoder(torch.nn.Module):
+ def __init__(self):
+ super().__init__()
+ self.proj = torch.nn.Linear(8, 8).cuda()
+ self.seen: list = []
+ self.fail = False
+
+ def forward(self, t):
+ self.seen.append(on_device())
+ if self.fail:
+ raise RuntimeError("encode failed")
+ return self.proj(t)
+
+ enc = Encoder()
+ pipe = types.SimpleNamespace(components = {"transformer": net, "text_encoder": enc})
+ assert dm._keep_groups_resident(net, 1024, "cuda") > 0
+ assert on_device() == ["cuda"] * 6
+ return torch, net, enc, pipe, x, ref, on_device
+
+
+def _whole_plan(whole_mib, encode_mib):
+ return types.SimpleNamespace(
+ resident_transformer_mib = whole_mib,
+ estimates = {"encode_resident_transformer_mib": encode_mib},
+ )
+
+
+def test_encode_streams_the_whole_resident_denoiser_back_to_the_flat_room(monkeypatch):
+ """12 GB whole-resident tier: while the text encoder runs, the denoiser groups past the flat room stream again
+ (the device state the partial placement had during the encode); the denoiser is whole again before step 0 and
+ stays bit-identical."""
+ torch, net, enc, pipe, x, ref, on_device = _encode_release_pipe(monkeypatch)
+ block_mib = 4 # each 1024x1024 fp32 block is 4 MiB + bias
+ whole = 1024
+ assert dm.install_encode_release(pipe, _whole_plan(whole, whole - 2 * block_mib), None) == 1
+ for _ in range(2): # per request; nothing accumulates
+ enc(torch.randn(2, 8, device = "cuda"))
+ assert enc.seen[-1] == ["cuda"] * 4 + ["cpu"] * 2
+ assert on_device() == ["cuda"] * 6
+ assert dm.denoisers_pinned_resident(types.SimpleNamespace(transformer = net))
+ assert torch.equal(net(x.cuda()).cpu(), ref)
+
+
+def test_encode_release_restores_after_a_failed_encode(monkeypatch):
+ torch, net, enc, pipe, x, ref, on_device = _encode_release_pipe(monkeypatch)
+ dm.install_encode_release(pipe, _whole_plan(1024, 0), None)
+ enc.fail = True
+ with pytest.raises(RuntimeError):
+ enc(torch.randn(2, 8, device = "cuda"))
+ assert enc.seen[-1] == ["cpu"] * 6 # encode room 0: every block streamed during the encode
+ assert on_device() == ["cuda"] * 6
+ enc.fail = False
+ enc(torch.randn(2, 8, device = "cuda"))
+ assert on_device() == ["cuda"] * 6
+ assert torch.equal(net(x.cuda()).cpu(), ref)
+
+
+def test_encode_release_keeps_an_oversized_release_streamed(monkeypatch):
+ """Inside an oversized request (groups already streamed for a bigger canvas), the encode streams the same surplus
+ on top and pins back only its own groups, so the big denoise still runs with the oversized release in place."""
+ torch, net, enc, pipe, x, ref, on_device = _encode_release_pipe(monkeypatch)
+ dm.install_encode_release(pipe, _whole_plan(1024, 1024 - 4), None) # surplus covers one block
+ outer = dm.release_resident_groups(pipe, 4) # the oversized request: one block
+ assert on_device() == ["cuda"] * 5 + ["cpu"]
+ enc(torch.randn(2, 8, device = "cuda"))
+ assert enc.seen[-1] == ["cuda"] * 4 + ["cpu"] * 2
+ assert on_device() == ["cuda"] * 5 + ["cpu"] # the oversized release is still in force
+ assert torch.equal(net(x.cuda()).cpu(), ref)
+ outer()
+ assert on_device() == ["cuda"] * 6
+ assert torch.equal(net(x.cuda()).cpu(), ref)
+
+
+def test_encode_release_only_for_the_whole_tier(monkeypatch):
+ torch, net, enc, pipe, x, ref, on_device = _encode_release_pipe(monkeypatch)
+ # the partial / 16 GB placements carry no encode room: nothing is hooked, the encode sees the placement as is
+ assert (
+ dm.install_encode_release(
+ pipe, types.SimpleNamespace(resident_transformer_mib = 1024, estimates = {}), None
+ )
+ == 0
+ )
+ assert dm.install_encode_release(pipe, _whole_plan(None, 0), None) == 0
+ enc(torch.randn(2, 8, device = "cuda"))
+ assert enc.seen[-1] == ["cuda"] * 6
+
+
+def test_generate_load_installs_the_encode_release():
+ src = (__import__("pathlib").Path(dm.__file__).parent / "diffusion.py").read_text(
+ encoding = "utf-8"
+ )
+ at = src.index("install_encode_release(pipe, plan, logger)")
+ # after the placement is applied and before the int8 GEMM reads the final residency
+ assert src.index("effective_policy, effective_tiling = apply_memory_plan(") < at
+ assert at < src.index("if denoisers_pinned_resident(pipe):", at)
+
+
def _inner_devices(tensor):
names, _ = tensor.__tensor_flatten__()
out = []
diff --git a/studio/backend/tests/test_diffusion_measured_placement_dense.py b/studio/backend/tests/test_diffusion_measured_placement_dense.py
index 3b2e54ba955..0c2a3320aa3 100644
--- a/studio/backend/tests/test_diffusion_measured_placement_dense.py
+++ b/studio/backend/tests/test_diffusion_measured_placement_dense.py
@@ -163,6 +163,14 @@ def test_every_budget_fits_measured_need(klein_pipe, budget):
assert kept + KLEIN_HEADROOM + dm.DEFAULT_BASE_OVERHEAD_MIB <= budget
+def test_dense_dit_never_takes_the_whole_resident_tier(klein_pipe):
+ """The whole-resident tier's slack was measured on the int8 route; a dense DiT keeps the partial room."""
+ new = _refine(klein_pipe, _flat_plan(11000, 13400))
+ assert "resident_dit_slack_mib" not in new.estimates
+ assert "encode_resident_transformer_mib" not in new.estimates
+ assert int(new.resident_transformer_mib or 0) < 7393
+
+
def test_dense_request_extra_releases_for_oversized(monkeypatch):
for name in ENVS:
monkeypatch.delenv(name, raising = False)