Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
Studio: keep the whole int8 Qwen-Image-2.1 denoiser resident at 12 GB, released only while the encoders run (298 to 212 ms per step) (#12536)
Daniel Han80c92a6ef974
diff --git a/studio/backend/core/inference/diffusion.py b/studio/backend/core/inference/diffusion.pyindex 28f4f711120..1f1ff3445ec 100644--- a/studio/backend/core/inference/diffusion.py+++ b/studio/backend/core/inference/diffusion.py@@ -133,6 +133,7 @@ from .diffusion_memory import (plan_diffusion_memory,plan_fits_total_capacity,denoisers_pinned_resident,+ install_encode_release,plan_keeps_transformer_resident,prequant_seed_device,raise_on_image_activation_shortfall,@@ -7155,6 +7156,8 @@ class DiffusionBackend:pipe._unsloth_cuda_graphs = ()pipe._unsloth_cuda_graph_reason = "offload active"speed_applied["cuda_graph"] = False+ # streams the whole-resident denoiser back to the flat room while the encoders run+ install_encode_release(pipe, plan, logger)# the speed layer saw only the plan; placement may have pinned every denoiser group sinceif denoisers_pinned_resident(pipe):engage_pinned_denoisers(pipe, speed_applied, logger)diff --git a/studio/backend/core/inference/diffusion_memory.py b/studio/backend/core/inference/diffusion_memory.pyindex b1b14dbf24c..df2996ccb35 100644--- a/studio/backend/core/inference/diffusion_memory.py+++ b/studio/backend/core/inference/diffusion_memory.py@@ -2283,6 +2283,31 @@ def _loaded_component_mib(pipe: Any) -> Optional[dict[str, tuple[int, str]]]:return None+RESIDENT_DIT_ENV = "UNSLOTH_DIFFUSION_RESIDENT_DIT"+# Whole-resident tier slack: 10% of the card, min 1 GiB, instead of the flat reserve + base overhead sized for an+# unmeasured activation. Free memory is read after CUDA init and the measured peak covers encode, steps and decode, so+# the slack only covers fragmentation and lazily loaded kernels.+_RESIDENT_DIT_SLACK_FRACTION = 0.10+_RESIDENT_DIT_SLACK_MIN_MIB = 1024+++def _resident_dit_slack_mib(memory: Any) -> int:+ base = getattr(memory, "total_mib", None) or getattr(memory, "free_mib", None) or 0+ return max(_RESIDENT_DIT_SLACK_MIN_MIB, int(int(base) * _RESIDENT_DIT_SLACK_FRACTION))+++def _resident_dit_fits(memory: Any, dit_mib: int, headroom_mib: int, other_mib: int) -> bool:+ """Whole denoiser + non-encoder companions + measured peak x margin + slack within free memory, encoders streamed."""+ if _env_off(RESIDENT_DIT_ENV):+ return False+ free = getattr(memory, "free_mib", None)+ if free is None:+ return False+ return int(dit_mib) + int(other_mib) + int(headroom_mib) + _resident_dit_slack_mib(+ memory+ ) <= int(free)++def _denoiser_compute_bytes(pipe: Any) -> Optional[int]:"""Denoiser compute element size for the dense table: 2 (fp16), 4 (fp32); None for bf16 / unreadable."""try:@@ -2360,14 +2385,28 @@ def refine_plan_from_loaded_weights(room = budget - floorif policy == OFFLOAD_GROUP and not stream_te:room -= encoders # resident companions+ whole_dit = Falseif not bool(getattr(plan, "stream_transformer", True)):room -= ditdit_room = 0else:dit_room = min(max(room, 0), dit)+ if (+ dit_room < dit+ # torchao denoisers only: the slack was measured on the int8 route, not the dense eager table+ and dense_mib is None+ and (policy == OFFLOAD_STREAMING or stream_te)+ and _resident_dit_fits(memory, dit, headroom, other)+ ):+ # pin it whole; during the encode it drops back to the flat room (install_encode_release)+ encode_room = int(dit_room)+ dit_room, room, whole_dit = dit, dit, Truete_room = max(0, room - dit_room) if stream_te and policy == OFFLOAD_GROUP else 0if dit_room <= 0 and te_room <= 0:return plan+ if whole_dit:+ estimates["resident_dit_slack_mib"] = _resident_dit_slack_mib(memory)+ estimates["encode_resident_transformer_mib"] = encode_roomnew = replace(plan,resident_transformer_mib = int(dit_room) if dit_room > 0 else None,@@ -2380,7 +2419,13 @@ def refine_plan_from_loaded_weights(reasons = plan.reasons+ (f"{int(dit_room)} MiB of the {dit} MiB transformer and {int(te_room)} MiB of the {encoders} MiB "- f"encoders stay resident (measured peak {headroom} MiB); the rest streams",+ f"encoders stay resident (measured peak {headroom} MiB); the rest streams"+ + (+ f" (whole transformer within free memory less a {_resident_dit_slack_mib(memory)} MiB slack;"+ f" {encode_room} MiB of it while the encoders run)"+ if whole_dit+ else ""+ ),),)if logger is not None:@@ -2414,9 +2459,11 @@ def _keep_groups_resident(room_mib: int,device: Any,logger: Any = None,+ only: Optional[set] = None,) -> int:"""Make whole offload groups of a streamed ``module`` resident within ``room_mib`` (top-level group first, then- blocks in order); their onload / offload become no-ops and the rest keeps streaming. Returns the MiB kept."""+ blocks in order); their onload / offload become no-ops and the rest keeps streaming. Returns the MiB kept.+ ``only`` (group ids): a restore pins back what its own release streamed, not an enclosing release's groups."""if room_mib is None or int(room_mib) <= 0 or _env_off(PARTIAL_RESIDENT_ENV):return 0try:@@ -2484,8 +2531,8 @@ def _keep_groups_resident(if getattr(group, "_unsloth_resident", False):left -= needcontinue- if need > left:- # too large: it streams; a later, smaller group may still fit+ if need > left or (only is not None and id(group) not in only):+ # too large (a later, smaller group may still fit), or another release's groupcontinuecpu = getattr(group, "cpu_param_dict", None) or {}for t in tensors:@@ -2548,10 +2595,13 @@ def release_resident_groups(pipe: Any,need_mib: int,logger: Any = None,+ denoisers_only: bool = False,+ reason: str = "an oversized request",) -> Optional[Callable[[], None]]:"""Stream resident offload groups again until ``need_mib`` is freed (text encoders first, then the denoiser's- blocks from the last); returns a callable restoring them, or None when nothing was resident. A request larger- than the measured placement reserved (reference images, a bigger canvas, a batch) then runs as the flat plan."""+ blocks from the last); returns a callable pinning exactly those groups again, or None when nothing was resident.+ A request larger than the measured placement reserved (reference images, a bigger canvas, a batch) then runs as+ the flat plan. ``denoisers_only`` leaves the text encoders' groups alone (the prompt encode needs them)."""try:import torch@@ -2559,12 +2609,14 @@ def release_resident_groups(mfor m in (getattr(pipe, "components", {}) or {}).values()if getattr(m, "_unsloth_resident_room", None)+ and not (denoisers_only and _is_text_encoder_module(pipe, m))]if not modules or int(need_mib) <= 0:return Nonemodules.sort(key = lambda m: 0 if _is_text_encoder_module(pipe, m) else 1)left = int(need_mib) * 1024 * 1024released: list = []+ streamed: set = set()for module in modules:for group in reversed(_offload_groups(module) or []):if left <= 0:@@ -2576,6 +2628,7 @@ def release_resident_groups(if isinstance(state, dict):state["streamed"] = 1_release_group(group)+ streamed.add(id(group))left -= int(getattr(group, "_unsloth_resident_bytes", 0))if module not in released:released.append(module)@@ -2590,8 +2643,9 @@ def release_resident_groups(torch.cuda.synchronize(device)if logger is not None:logger.info(- "diffusion.memory: streaming %d MiB of resident groups for an oversized request",+ "diffusion.memory: streaming %d MiB of resident groups for %s",(int(need_mib) * 1024 * 1024 - max(left, 0)) >> 20,+ reason,)def restore() -> None:@@ -2601,6 +2655,7 @@ def release_resident_groups(module._unsloth_resident_room,module._unsloth_resident_device,logger,+ only = streamed,)return restore@@ -2617,6 +2672,85 @@ def _is_text_encoder_module(pipe: Any, module: Any) -> bool:return False+def install_encode_release(+ pipe: Any,+ plan: Any,+ logger: Any = None,+) -> int:+ """While a text encoder runs, stream the whole-resident denoiser back to the flat room (the partial placement's+ encode state); pin it back on return. Returns the number of encoders hooked."""+ estimates = getattr(plan, "estimates", None) or {}+ encode_room = estimates.get("encode_resident_transformer_mib")+ whole = getattr(plan, "resident_transformer_mib", None)+ if encode_room is None or not whole:+ return 0+ surplus = int(whole) - max(int(encode_room), 0)+ if surplus <= 0:+ return 0+ try:+ import weakref++ import torch++ try:+ pipe_ref = weakref.ref(pipe) # the hooks live on the encoder, which the pipe owns+ except TypeError:+ pipe_ref = lambda: pipe # noqa: E731 - not weak-referenceable+ disable = getattr(getattr(torch, "compiler", None), "disable", None)+ pending: list = []++ def _before(module: Any, args: Any) -> None:+ owner = pipe_ref()+ if owner is None or pending:+ return # nested encoder call: the outer one already released+ restore = release_resident_groups(+ owner, surplus, logger, denoisers_only = True, reason = "the prompt encode"+ )+ pending.append(restore)+ if restore is not None and torch.cuda.is_available():+ # freed blocks sit in the default stream's pool; the encoder onloads on the copy stream+ torch.cuda.empty_cache()++ def _after(module: Any, args: Any, output: Any) -> None:+ if not pending:+ return+ restore = pending.pop()+ if restore is not None:+ restore()++ before = disable(_before) if callable(disable) else _before+ after = disable(_after) if callable(disable) else _after+ hooked = 0+ for name, module in (getattr(pipe, "components", {}) or {}).items():+ if not str(name).startswith("text_encoder") or not isinstance(module, torch.nn.Module):+ continue+ handles = (+ module.register_forward_pre_hook(before),+ # always_call: an encode that raises (cancel, OOM) still pins the denoiser back+ module.register_forward_hook(after, always_call = True),+ )+ module._unsloth_encode_release = handles+ hooked += 1+ if hooked and logger is not None:+ logger.info(+ "diffusion.memory: the prompt encode streams %d MiB of the resident transformer (%d MiB stay), "+ "pinned again before step 0",+ surplus,+ max(int(encode_room), 0),+ )+ return hooked+ except Exception as exc: # noqa: BLE001 - fall back to the partial placement rather than risk the encode+ if logger is not None:+ logger.warning(+ "diffusion.memory: encode release not installed (%s); the transformer streams past the flat room",+ exc,+ )+ release_resident_groups(+ pipe, surplus, logger, denoisers_only = True, reason = "the partial placement"+ )+ return 0++def measured_request_extra_mib(pipe: Any,*,diff --git a/studio/backend/tests/test_diffusion_measured_placement.py b/studio/backend/tests/test_diffusion_measured_placement.pyindex 5e2ce8d365b..1d07711baf3 100644--- a/studio/backend/tests/test_diffusion_measured_placement.py+++ b/studio/backend/tests/test_diffusion_measured_placement.py@@ -150,15 +150,52 @@ def test_16gb_encoder_room_and_kill_switch(q21_pipe, monkeypatch):assert _refine(q21_pipe, plan) is plan-def test_12gb_partial_residency_sized_from_budget(q21_pipe):+def test_12gb_keeps_the_whole_transformer_resident_encoders_streamed(q21_pipe, monkeypatch):+ """12 GB: the flat margins fit only part of the int8 DiT; with encoders streamed the whole DiT fits the slack."""+ monkeypatch.delenv(dm.RESIDENT_DIT_ENV, raising = False)plan = _flat_plan(9550, 12288)assert plan.offload_policy == dm.OFFLOAD_STREAMING+ free = plan.device_memory.free_mibnew = _refine(q21_pipe, plan)- assert new.offload_policy == dm.OFFLOAD_STREAMING- room = 9550 - 2304 - dm.DEFAULT_BASE_OVERHEAD_MIB - 644+ assert new.offload_policy == dm.OFFLOAD_STREAMING and new.stream_transformer+ assert new.resident_transformer_mib == 6922+ assert new.resident_text_encoder_mib is None+ assert new.estimates["resident_dit_slack_mib"] == 1228+ # while the encoders run, the transformer drops back to the flat room the partial placement kept+ assert (+ new.estimates["encode_resident_transformer_mib"]+ == 9550 - 2304 - dm.DEFAULT_BASE_OVERHEAD_MIB - 644+ )+ assert 6922 + 644 + 2304 + 1228 <= free+ assert new.as_public_dict()["resident_transformer_mib"] == 6922+++def test_12gb_partial_residency_when_the_whole_transformer_does_not_fit(q21_pipe, monkeypatch):+ monkeypatch.delenv(dm.RESIDENT_DIT_ENV, raising = False)+ # 500 MiB less free memory (a desktop session on the card): the whole DiT misses the slack, the flat room holds+ plan = _flat_plan(9000, 12288)+ assert 6922 + 644 + 2304 + 1228 > plan.device_memory.free_mib+ new = _refine(q21_pipe, plan)+ room = 9000 - 2304 - dm.DEFAULT_BASE_OVERHEAD_MIB - 644assert new.resident_transformer_mib == roomassert 0 < room < 6922- assert new.as_public_dict()["resident_transformer_mib"] == room+ assert "resident_dit_slack_mib" not in new.estimates+++def test_whole_transformer_tier_kill_switch(q21_pipe, monkeypatch):+ monkeypatch.setenv(dm.RESIDENT_DIT_ENV, "0")+ new = _refine(q21_pipe, _flat_plan(9550, 12288))+ assert new.resident_transformer_mib == 9550 - 2304 - dm.DEFAULT_BASE_OVERHEAD_MIB - 644+++def test_whole_transformer_tier_needs_streamed_encoders(q21_pipe, monkeypatch):+ """A group plan whose encoders stay resident cannot trade them for the DiT: the tier leaves it alone."""+ monkeypatch.delenv(dm.RESIDENT_DIT_ENV, raising = False)+ plan = dm.replace(+ _flat_plan(9550, 12288), offload_policy = dm.OFFLOAD_GROUP, stream_text_encoders = False+ )+ new = _refine(q21_pipe, plan)+ assert int(new.resident_transformer_mib or 0) < 6922def test_no_room_keeps_flat_plan(q21_pipe):@@ -237,6 +274,13 @@ def test_every_tier_fits_measured_need(q21_pipe, budget):+ (0 if new.stream_text_encoders or new.offload_policy == dm.OFFLOAD_STREAMING else 8959))assert int(new.resident_transformer_mib or 0) <= 6922+ if "resident_dit_slack_mib" in new.estimates:+ # the whole-DiT tier: every encoder streams, and weights + peak x margin + slack fit free memory+ assert new.resident_transformer_mib == 6922 and not new.resident_text_encoder_mib+ slack = new.estimates["resident_dit_slack_mib"]+ assert slack >= max(1024, total // 10)+ assert kept + 2304 + slack <= plan.device_memory.free_mib+ returnassert kept + head <= budget@@ -596,6 +640,126 @@ def test_torchao_groups_stay_on_device_after_release_and_restore(monkeypatch):assert torch.equal(net(x), ref)+def _encode_release_pipe(monkeypatch):+ """A real group-offloaded denoiser pinned whole; the encoder records which blocks are on the device."""+ torch, net = _cuda_offload_model()+ from diffusers.hooks import apply_group_offloading++ monkeypatch.delenv("UNSLOTH_DIFFUSION_PARTIAL_RESIDENT", raising = False)+ x = torch.randn(3, 64)+ ref = net.to("cuda")(x.cuda()).cpu()+ net.to("cpu")+ apply_group_offloading(+ net,+ onload_device = torch.device("cuda"),+ offload_device = torch.device("cpu"),+ offload_type = "block_level",+ num_blocks_per_group = 1,+ use_stream = True,+ record_stream = True,+ non_blocking = True,+ )++ def on_device():+ return [next(b.parameters()).device.type for b in net.blocks]++ class Encoder(torch.nn.Module):+ def __init__(self):+ super().__init__()+ self.proj = torch.nn.Linear(8, 8).cuda()+ self.seen: list = []+ self.fail = False++ def forward(self, t):+ self.seen.append(on_device())+ if self.fail:+ raise RuntimeError("encode failed")+ return self.proj(t)++ enc = Encoder()+ pipe = types.SimpleNamespace(components = {"transformer": net, "text_encoder": enc})+ assert dm._keep_groups_resident(net, 1024, "cuda") > 0+ assert on_device() == ["cuda"] * 6+ return torch, net, enc, pipe, x, ref, on_device+++def _whole_plan(whole_mib, encode_mib):+ return types.SimpleNamespace(+ resident_transformer_mib = whole_mib,+ estimates = {"encode_resident_transformer_mib": encode_mib},+ )+++def test_encode_streams_the_whole_resident_denoiser_back_to_the_flat_room(monkeypatch):+ """12 GB whole-resident tier: while the text encoder runs, the denoiser groups past the flat room stream again+ (the device state the partial placement had during the encode); the denoiser is whole again before step 0 and+ stays bit-identical."""+ torch, net, enc, pipe, x, ref, on_device = _encode_release_pipe(monkeypatch)+ block_mib = 4 # each 1024x1024 fp32 block is 4 MiB + bias+ whole = 1024+ assert dm.install_encode_release(pipe, _whole_plan(whole, whole - 2 * block_mib), None) == 1+ for _ in range(2): # per request; nothing accumulates+ enc(torch.randn(2, 8, device = "cuda"))+ assert enc.seen[-1] == ["cuda"] * 4 + ["cpu"] * 2+ assert on_device() == ["cuda"] * 6+ assert dm.denoisers_pinned_resident(types.SimpleNamespace(transformer = net))+ assert torch.equal(net(x.cuda()).cpu(), ref)+++def test_encode_release_restores_after_a_failed_encode(monkeypatch):+ torch, net, enc, pipe, x, ref, on_device = _encode_release_pipe(monkeypatch)+ dm.install_encode_release(pipe, _whole_plan(1024, 0), None)+ enc.fail = True+ with pytest.raises(RuntimeError):+ enc(torch.randn(2, 8, device = "cuda"))+ assert enc.seen[-1] == ["cpu"] * 6 # encode room 0: every block streamed during the encode+ assert on_device() == ["cuda"] * 6+ enc.fail = False+ enc(torch.randn(2, 8, device = "cuda"))+ assert on_device() == ["cuda"] * 6+ assert torch.equal(net(x.cuda()).cpu(), ref)+++def test_encode_release_keeps_an_oversized_release_streamed(monkeypatch):+ """Inside an oversized request (groups already streamed for a bigger canvas), the encode streams the same surplus+ on top and pins back only its own groups, so the big denoise still runs with the oversized release in place."""+ torch, net, enc, pipe, x, ref, on_device = _encode_release_pipe(monkeypatch)+ dm.install_encode_release(pipe, _whole_plan(1024, 1024 - 4), None) # surplus covers one block+ outer = dm.release_resident_groups(pipe, 4) # the oversized request: one block+ assert on_device() == ["cuda"] * 5 + ["cpu"]+ enc(torch.randn(2, 8, device = "cuda"))+ assert enc.seen[-1] == ["cuda"] * 4 + ["cpu"] * 2+ assert on_device() == ["cuda"] * 5 + ["cpu"] # the oversized release is still in force+ assert torch.equal(net(x.cuda()).cpu(), ref)+ outer()+ assert on_device() == ["cuda"] * 6+ assert torch.equal(net(x.cuda()).cpu(), ref)+++def test_encode_release_only_for_the_whole_tier(monkeypatch):+ torch, net, enc, pipe, x, ref, on_device = _encode_release_pipe(monkeypatch)+ # the partial / 16 GB placements carry no encode room: nothing is hooked, the encode sees the placement as is+ assert (+ dm.install_encode_release(+ pipe, types.SimpleNamespace(resident_transformer_mib = 1024, estimates = {}), None+ )+ == 0+ )+ assert dm.install_encode_release(pipe, _whole_plan(None, 0), None) == 0+ enc(torch.randn(2, 8, device = "cuda"))+ assert enc.seen[-1] == ["cuda"] * 6+++def test_generate_load_installs_the_encode_release():+ src = (__import__("pathlib").Path(dm.__file__).parent / "diffusion.py").read_text(+ encoding = "utf-8"+ )+ at = src.index("install_encode_release(pipe, plan, logger)")+ # after the placement is applied and before the int8 GEMM reads the final residency+ assert src.index("effective_policy, effective_tiling = apply_memory_plan(") < at+ assert at < src.index("if denoisers_pinned_resident(pipe):", at)++def _inner_devices(tensor):names, _ = tensor.__tensor_flatten__()out = []diff --git a/studio/backend/tests/test_diffusion_measured_placement_dense.py b/studio/backend/tests/test_diffusion_measured_placement_dense.pyindex 3b2e54ba955..0c2a3320aa3 100644--- a/studio/backend/tests/test_diffusion_measured_placement_dense.py+++ b/studio/backend/tests/test_diffusion_measured_placement_dense.py@@ -163,6 +163,14 @@ def test_every_budget_fits_measured_need(klein_pipe, budget):assert kept + KLEIN_HEADROOM + dm.DEFAULT_BASE_OVERHEAD_MIB <= budget+def test_dense_dit_never_takes_the_whole_resident_tier(klein_pipe):+ """The whole-resident tier's slack was measured on the int8 route; a dense DiT keeps the partial room."""+ new = _refine(klein_pipe, _flat_plan(11000, 13400))+ assert "resident_dit_slack_mib" not in new.estimates+ assert "encode_resident_transformer_mib" not in new.estimates+ assert int(new.resident_transformer_mib or 0) < 7393++def test_dense_request_extra_releases_for_oversized(monkeypatch):for name in ENVS:monkeypatch.delenv(name, raising = False)