Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

Studio diffusion: cap the prefetcher's pinned staging so guided Z-Image-Turbo renders on a 31 GB host (Kaggle T4x2) (#14975)

Daniel Han8dff63cddec2
diff --git a/studio/backend/core/inference/diffusion_offload_prefetch.py b/studio/backend/core/inference/diffusion_offload_prefetch.py
index 1774c2396f4..b2860e6c21a 100644
--- a/studio/backend/core/inference/diffusion_offload_prefetch.py
+++ b/studio/backend/core/inference/diffusion_offload_prefetch.py
@@ -12,12 +12,14 @@ group uploads on the compute stream): queued earlier they hold that upload back
Memory stays bounded without a host wait: each offload records a compute-stream event the copy stream waits on, so a
freed block goes to the next prefetch only after the compute that read it. Bytes in flight never pass ``window``.
+Unpinned copies' temporary pinned staging is capped at ``window`` (else T4 Z-Image pinned the whole denoiser).
Hooks are per-group / per-module instance attributes; diffusers' classes are untouched.
"""
from __future__ import annotations
import os
+from collections import deque
from typing import Any, Optional
ASYNC_PREFETCH_ENV = "UNSLOTH_DIFFUSION_ASYNC_PREFETCH"
@@ -321,6 +323,9 @@ class GroupPrefetcher:
self.ready: dict = {}
self.inflight_bytes = 0
self.peak_inflight_bytes = 0
+ self.staged: deque = deque()
+ self.staged_bytes = 0
+ self.peak_staged_bytes = 0
self.order: list = []
self.seen: list = []
self.pos = 0
@@ -396,13 +401,20 @@ class GroupPrefetcher:
cpu = group.cpu_param_dict
stream = group.stream
where = self.slot_of.get(id(group)) if slot else None
+ tensors = _group_tensors(group)
+ pinned = [cpu[t].is_pinned() for t in tensors]
+ staged = 0
+ if not all(pinned) and not getattr(self, "capturing", False):
+ from .diffusion_memory import _storage_nbytes
+ staged = sum(sum(_storage_nbytes(cpu[t])) for t, p in zip(tensors, pinned) if not p)
+ self._bound_staging(staged)
with torch.cuda.stream(stream):
# made on the copy stream: a block the compute stream freed but has not finished reading is never handed
# to these copies (the allocator orders reuse per stream)
buffers = self._views_for(group, where) if where is not None else None
- for i, t in enumerate(_group_tensors(group)):
+ for i, t in enumerate(tensors):
src = cpu[t]
- if not src.is_pinned():
+ if not pinned[i]:
if getattr(self, "capturing", False):
# a graph would replay this copy from a temporary pinned buffer reused after the call
raise RuntimeError("a streamed group's host copy is not pinned")
@@ -427,9 +439,30 @@ class GroupPrefetcher:
self.ready[gid] = ready
self.inflight_bytes += self.nbytes.get(gid, 0)
self.peak_inflight_bytes = max(self.peak_inflight_bytes, self.inflight_bytes)
+ if staged:
+ self._staging().append((ready, staged))
+ self.staged_bytes += staged
+ self.peak_staged_bytes = max(getattr(self, "peak_staged_bytes", 0), self.staged_bytes)
self.stats["copies"] += 1
return True
+ def _staging(self) -> deque:
+ # lazy: prefetchers built without __init__ (tests) still stage
+ if "staged" not in self.__dict__:
+ self.staged, self.staged_bytes = deque(), 0
+ return self.staged
+
+ def _bound_staging(self, nbytes: int) -> None:
+ """Drop landed copies (no wait), then wait on the oldest until ``nbytes`` fits; each is queued, so waits end."""
+ staged = self._staging()
+ while staged and staged[0][0].query():
+ self.staged_bytes -= staged.popleft()[1]
+ while staged and self.staged_bytes + int(nbytes) > self.window:
+ event, n = staged.popleft()
+ event.synchronize()
+ self.staged_bytes -= n
+ self.stats["staging_waits"] = self.stats.get("staging_waits", 0) + 1
+
def onload(self, group: Any) -> None:
gid = id(group)
self.kick() # first block of the forward: this group is order[pos], so it is queued first
@@ -709,6 +742,8 @@ class GroupPrefetcher:
self.active = self.on_order = self.pending = self.capturing = False
self.ready.clear()
self.inflight_bytes = 0
+ self._staging().clear() # nothing is staged under capture; the synchronize below lands the rest
+ self.staged_bytes = 0
try:
torch.cuda.synchronize(self.device)
except Exception: # noqa: BLE001 - the copies below are what matters
diff --git a/studio/backend/tests/test_diffusion_offload_prefetch.py b/studio/backend/tests/test_diffusion_offload_prefetch.py
index 56ff13cfb87..d22db9ebd61 100644
--- a/studio/backend/tests/test_diffusion_offload_prefetch.py
+++ b/studio/backend/tests/test_diffusion_offload_prefetch.py
@@ -135,6 +135,129 @@ def test_one_group_larger_than_the_window_still_progresses():
assert pf.stats["forwards"] == 2 and pf.inflight_bytes == 0
+class _FrozenGpu:
+ """Card whose events complete only when the host waits; tracks pinned staging buffers whose copy has not run."""
+
+ def __init__(self):
+ self.outstanding = 0
+ self.peak = 0
+ self.unattributed = 0
+ self.waits = 0
+ gpu = self
+
+ class Event:
+ def __init__(self):
+ self.staged = 0
+ self.done = False
+
+ def record(self, stream = None):
+ self.staged, gpu.unattributed = gpu.unattributed, 0
+
+ def query(self):
+ return self.done
+
+ def synchronize(self):
+ gpu.waits += 1
+ if not self.done:
+ self.done = True
+ gpu.outstanding -= self.staged
+
+ class HostTensor:
+ def __init__(
+ self,
+ nbytes: int,
+ pinned: bool = False,
+ ):
+ self.nbytes = nbytes
+ self.pinned = pinned
+
+ def is_pinned(self):
+ return self.pinned
+
+ def numel(self):
+ return self.nbytes
+
+ def element_size(self):
+ return 1
+
+ def pin_memory(self):
+ gpu.outstanding += self.nbytes
+ gpu.unattributed += self.nbytes
+ gpu.peak = max(gpu.peak, gpu.outstanding)
+ return HostTensor(self.nbytes, pinned = True)
+
+ def to(
+ self,
+ device,
+ non_blocking = False,
+ ):
+ return self
+
+ self.Event = Event
+ self.HostTensor = HostTensor
+
+
+class _Param:
+ data = None
+
+
+def _frozen_gpu_prefetcher(
+ monkeypatch,
+ pinned: bool,
+ groups_n: int = 10,
+ tensors: int = 3,
+):
+ import contextlib
+
+ gpu = _FrozenGpu()
+ monkeypatch.setattr(torch.cuda, "Event", gpu.Event)
+ monkeypatch.setattr(torch.cuda, "stream", lambda s: contextlib.nullcontext())
+ monkeypatch.setattr(op, "_go", lambda: types.SimpleNamespace())
+ groups = []
+ for _ in range(groups_n):
+ params = [_Param() for _ in range(tensors)]
+ groups.append(
+ types.SimpleNamespace(
+ modules = [],
+ parameters = params,
+ buffers = [],
+ cpu_param_dict = {p: gpu.HostTensor(100, pinned) for p in params},
+ stream = object(),
+ )
+ )
+ pf = op.GroupPrefetcher(None, groups, "cpu", depth = 1)
+ return pf, groups, gpu
+
+
+def test_unpinned_staging_stays_in_the_window_on_a_slow_card(monkeypatch):
+ """Host a whole forward ahead of the card: unbounded, staging held the whole denoiser in pinned RAM."""
+ pf, groups, gpu = _frozen_gpu_prefetcher(monkeypatch, pinned = False)
+ group_bytes = 300
+ assert pf.window == 2 * group_bytes
+ for g in groups:
+ pf._issue(g)
+ assert gpu.peak <= pf.window, (gpu.peak, pf.window)
+ assert gpu.waits > 0
+ assert pf.stats["copies"] == len(groups)
+ assert all(p.data is not None for g in groups for p in g.parameters)
+
+
+def test_landed_copies_free_staging_without_a_wait(monkeypatch):
+ pf, groups, gpu = _frozen_gpu_prefetcher(monkeypatch, pinned = False)
+ for g in groups:
+ pf._issue(g)
+ for event, _n in list(pf.staged):
+ event.done = True
+ assert gpu.waits == 0
+
+
+def test_pinned_host_copies_never_wait(monkeypatch):
+ pf, groups, gpu = _frozen_gpu_prefetcher(monkeypatch, pinned = True)
+ for g in groups:
+ pf._issue(g)
+ assert gpu.waits == 0 and gpu.peak == 0
+
+
def test_resident_groups_are_skipped_and_not_counted():
pf, groups, log = _cpu_prefetcher("abcdef", depth = 2)
_forward(pf, groups)
@@ -496,6 +619,35 @@ def test_unpinned_host_copies_stream_bit_identical(monkeypatch):
assert op.module_prefetcher(net).stats["prefetched"] > 0
+def test_slow_gpu_unpinned_staging_stays_in_the_window(monkeypatch):
+ """Unpinned host copies on a card slower than the host thread: pinned staging is capped, outputs bit-identical."""
+ _cuda()
+ monkeypatch.setenv("UNSLOTH_DIFFUSION_GROUP_OFFLOAD_PIN", "0")
+ monkeypatch.setenv(op.PREFETCH_DEPTH_ENV, "1")
+ torch.manual_seed(0)
+ # ~0.1 s per block on any card: the host queues the whole forward before the first blocks have run
+ net = _Net(width = 2048, blocks = 12, sleep_cycles = 200_000_000)
+ ref = copy.deepcopy(net).cuda()
+ _streamed(net, stream_text_encoders = True)
+ pf = op.module_prefetcher(net)
+ x = torch.randn(4, 64, device = "cuda")
+ with torch.no_grad():
+ want = ref(x)
+ net(x)
+ torch.cuda.synchronize()
+ torch.cuda.reset_peak_host_memory_stats()
+ before = torch.cuda.host_memory_stats().get("allocated_bytes.current", 0)
+ got = net(x)
+ host_ahead = not torch.cuda.current_stream().query()
+ torch.cuda.synchronize()
+ peak = torch.cuda.host_memory_stats().get("allocated_bytes.peak", 0) - before
+ assert torch.equal(got, want)
+ assert host_ahead
+ block_bytes = 2048 * 2048 * 4 + 2048 * 4
+ # window (2 groups) + the group being staged; unbounded it was every block of the forward
+ assert peak <= pf.window + 2 * block_bytes, (peak, pf.window)
+
+
@pytest.mark.parametrize("version", [None, 2])
def test_torchao_int8_weights_stream_bit_identical(version):
"""torchao's default int8 weight (v1 on 0.17) and ``Int8Tensor`` (version 2), whose 0.17 ``to`` drops