Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
Studio diffusion: cap the prefetcher's pinned staging so guided Z-Image-Turbo renders on a 31 GB host (Kaggle T4x2) (#14975)
Daniel Han8dff63cddec2
diff --git a/studio/backend/core/inference/diffusion_offload_prefetch.py b/studio/backend/core/inference/diffusion_offload_prefetch.pyindex 1774c2396f4..b2860e6c21a 100644--- a/studio/backend/core/inference/diffusion_offload_prefetch.py+++ b/studio/backend/core/inference/diffusion_offload_prefetch.py@@ -12,12 +12,14 @@ group uploads on the compute stream): queued earlier they hold that upload backMemory stays bounded without a host wait: each offload records a compute-stream event the copy stream waits on, so afreed block goes to the next prefetch only after the compute that read it. Bytes in flight never pass ``window``.+Unpinned copies' temporary pinned staging is capped at ``window`` (else T4 Z-Image pinned the whole denoiser).Hooks are per-group / per-module instance attributes; diffusers' classes are untouched."""from __future__ import annotationsimport os+from collections import dequefrom typing import Any, OptionalASYNC_PREFETCH_ENV = "UNSLOTH_DIFFUSION_ASYNC_PREFETCH"@@ -321,6 +323,9 @@ class GroupPrefetcher:self.ready: dict = {}self.inflight_bytes = 0self.peak_inflight_bytes = 0+ self.staged: deque = deque()+ self.staged_bytes = 0+ self.peak_staged_bytes = 0self.order: list = []self.seen: list = []self.pos = 0@@ -396,13 +401,20 @@ class GroupPrefetcher:cpu = group.cpu_param_dictstream = group.streamwhere = self.slot_of.get(id(group)) if slot else None+ tensors = _group_tensors(group)+ pinned = [cpu[t].is_pinned() for t in tensors]+ staged = 0+ if not all(pinned) and not getattr(self, "capturing", False):+ from .diffusion_memory import _storage_nbytes+ staged = sum(sum(_storage_nbytes(cpu[t])) for t, p in zip(tensors, pinned) if not p)+ self._bound_staging(staged)with torch.cuda.stream(stream):# made on the copy stream: a block the compute stream freed but has not finished reading is never handed# to these copies (the allocator orders reuse per stream)buffers = self._views_for(group, where) if where is not None else None- for i, t in enumerate(_group_tensors(group)):+ for i, t in enumerate(tensors):src = cpu[t]- if not src.is_pinned():+ if not pinned[i]:if getattr(self, "capturing", False):# a graph would replay this copy from a temporary pinned buffer reused after the callraise RuntimeError("a streamed group's host copy is not pinned")@@ -427,9 +439,30 @@ class GroupPrefetcher:self.ready[gid] = readyself.inflight_bytes += self.nbytes.get(gid, 0)self.peak_inflight_bytes = max(self.peak_inflight_bytes, self.inflight_bytes)+ if staged:+ self._staging().append((ready, staged))+ self.staged_bytes += staged+ self.peak_staged_bytes = max(getattr(self, "peak_staged_bytes", 0), self.staged_bytes)self.stats["copies"] += 1return True+ def _staging(self) -> deque:+ # lazy: prefetchers built without __init__ (tests) still stage+ if "staged" not in self.__dict__:+ self.staged, self.staged_bytes = deque(), 0+ return self.staged++ def _bound_staging(self, nbytes: int) -> None:+ """Drop landed copies (no wait), then wait on the oldest until ``nbytes`` fits; each is queued, so waits end."""+ staged = self._staging()+ while staged and staged[0][0].query():+ self.staged_bytes -= staged.popleft()[1]+ while staged and self.staged_bytes + int(nbytes) > self.window:+ event, n = staged.popleft()+ event.synchronize()+ self.staged_bytes -= n+ self.stats["staging_waits"] = self.stats.get("staging_waits", 0) + 1+def onload(self, group: Any) -> None:gid = id(group)self.kick() # first block of the forward: this group is order[pos], so it is queued first@@ -709,6 +742,8 @@ class GroupPrefetcher:self.active = self.on_order = self.pending = self.capturing = Falseself.ready.clear()self.inflight_bytes = 0+ self._staging().clear() # nothing is staged under capture; the synchronize below lands the rest+ self.staged_bytes = 0try:torch.cuda.synchronize(self.device)except Exception: # noqa: BLE001 - the copies below are what mattersdiff --git a/studio/backend/tests/test_diffusion_offload_prefetch.py b/studio/backend/tests/test_diffusion_offload_prefetch.pyindex 56ff13cfb87..d22db9ebd61 100644--- a/studio/backend/tests/test_diffusion_offload_prefetch.py+++ b/studio/backend/tests/test_diffusion_offload_prefetch.py@@ -135,6 +135,129 @@ def test_one_group_larger_than_the_window_still_progresses():assert pf.stats["forwards"] == 2 and pf.inflight_bytes == 0+class _FrozenGpu:+ """Card whose events complete only when the host waits; tracks pinned staging buffers whose copy has not run."""++ def __init__(self):+ self.outstanding = 0+ self.peak = 0+ self.unattributed = 0+ self.waits = 0+ gpu = self++ class Event:+ def __init__(self):+ self.staged = 0+ self.done = False++ def record(self, stream = None):+ self.staged, gpu.unattributed = gpu.unattributed, 0++ def query(self):+ return self.done++ def synchronize(self):+ gpu.waits += 1+ if not self.done:+ self.done = True+ gpu.outstanding -= self.staged++ class HostTensor:+ def __init__(+ self,+ nbytes: int,+ pinned: bool = False,+ ):+ self.nbytes = nbytes+ self.pinned = pinned++ def is_pinned(self):+ return self.pinned++ def numel(self):+ return self.nbytes++ def element_size(self):+ return 1++ def pin_memory(self):+ gpu.outstanding += self.nbytes+ gpu.unattributed += self.nbytes+ gpu.peak = max(gpu.peak, gpu.outstanding)+ return HostTensor(self.nbytes, pinned = True)++ def to(+ self,+ device,+ non_blocking = False,+ ):+ return self++ self.Event = Event+ self.HostTensor = HostTensor+++class _Param:+ data = None+++def _frozen_gpu_prefetcher(+ monkeypatch,+ pinned: bool,+ groups_n: int = 10,+ tensors: int = 3,+):+ import contextlib++ gpu = _FrozenGpu()+ monkeypatch.setattr(torch.cuda, "Event", gpu.Event)+ monkeypatch.setattr(torch.cuda, "stream", lambda s: contextlib.nullcontext())+ monkeypatch.setattr(op, "_go", lambda: types.SimpleNamespace())+ groups = []+ for _ in range(groups_n):+ params = [_Param() for _ in range(tensors)]+ groups.append(+ types.SimpleNamespace(+ modules = [],+ parameters = params,+ buffers = [],+ cpu_param_dict = {p: gpu.HostTensor(100, pinned) for p in params},+ stream = object(),+ )+ )+ pf = op.GroupPrefetcher(None, groups, "cpu", depth = 1)+ return pf, groups, gpu+++def test_unpinned_staging_stays_in_the_window_on_a_slow_card(monkeypatch):+ """Host a whole forward ahead of the card: unbounded, staging held the whole denoiser in pinned RAM."""+ pf, groups, gpu = _frozen_gpu_prefetcher(monkeypatch, pinned = False)+ group_bytes = 300+ assert pf.window == 2 * group_bytes+ for g in groups:+ pf._issue(g)+ assert gpu.peak <= pf.window, (gpu.peak, pf.window)+ assert gpu.waits > 0+ assert pf.stats["copies"] == len(groups)+ assert all(p.data is not None for g in groups for p in g.parameters)+++def test_landed_copies_free_staging_without_a_wait(monkeypatch):+ pf, groups, gpu = _frozen_gpu_prefetcher(monkeypatch, pinned = False)+ for g in groups:+ pf._issue(g)+ for event, _n in list(pf.staged):+ event.done = True+ assert gpu.waits == 0+++def test_pinned_host_copies_never_wait(monkeypatch):+ pf, groups, gpu = _frozen_gpu_prefetcher(monkeypatch, pinned = True)+ for g in groups:+ pf._issue(g)+ assert gpu.waits == 0 and gpu.peak == 0++def test_resident_groups_are_skipped_and_not_counted():pf, groups, log = _cpu_prefetcher("abcdef", depth = 2)_forward(pf, groups)@@ -496,6 +619,35 @@ def test_unpinned_host_copies_stream_bit_identical(monkeypatch):assert op.module_prefetcher(net).stats["prefetched"] > 0+def test_slow_gpu_unpinned_staging_stays_in_the_window(monkeypatch):+ """Unpinned host copies on a card slower than the host thread: pinned staging is capped, outputs bit-identical."""+ _cuda()+ monkeypatch.setenv("UNSLOTH_DIFFUSION_GROUP_OFFLOAD_PIN", "0")+ monkeypatch.setenv(op.PREFETCH_DEPTH_ENV, "1")+ torch.manual_seed(0)+ # ~0.1 s per block on any card: the host queues the whole forward before the first blocks have run+ net = _Net(width = 2048, blocks = 12, sleep_cycles = 200_000_000)+ ref = copy.deepcopy(net).cuda()+ _streamed(net, stream_text_encoders = True)+ pf = op.module_prefetcher(net)+ x = torch.randn(4, 64, device = "cuda")+ with torch.no_grad():+ want = ref(x)+ net(x)+ torch.cuda.synchronize()+ torch.cuda.reset_peak_host_memory_stats()+ before = torch.cuda.host_memory_stats().get("allocated_bytes.current", 0)+ got = net(x)+ host_ahead = not torch.cuda.current_stream().query()+ torch.cuda.synchronize()+ peak = torch.cuda.host_memory_stats().get("allocated_bytes.peak", 0) - before+ assert torch.equal(got, want)+ assert host_ahead+ block_bytes = 2048 * 2048 * 4 + 2048 * 4+ # window (2 groups) + the group being staged; unbounded it was every block of the forward+ assert peak <= pf.window + 2 * block_bytes, (peak, pf.window)++@pytest.mark.parametrize("version", [None, 2])def test_torchao_int8_weights_stream_bit_identical(version):"""torchao's default int8 weight (v1 on 0.17) and ``Int8Tensor`` (version 2), whose 0.17 ``to`` drops