Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
Studio: async block prefetch for MiniMax-H3's streamed denoiser (#12683)
Daniel Han8e11ba15ef59
diff --git a/studio/backend/core/inference/video.py b/studio/backend/core/inference/video.pyindex a10b49ed831..e96462eedd6 100644--- a/studio/backend/core/inference/video.py+++ b/studio/backend/core/inference/video.py@@ -7347,10 +7347,18 @@ class VideoBackend:if denoiser_streamed in ("stream", "stream_lazy"):from .video_minimax_h3_residency import pin_streamed_top_level_group+try:pin_streamed_top_level_group(denoiser, logger = logger)except Exception as exc: # noqa: BLE001 -- a speed-up onlylogger.warning("video.h3_top_group: %s", exc)+ # after the top-level pin (it becomes one of the prefetched groups), before the residency fit+ from .video_minimax_h3_residency import install_h3_stream_prefetch++ try:+ install_h3_stream_prefetch(denoiser, device, logger = logger)+ except Exception as exc: # noqa: BLE001 -- a speed-up only+ logger.warning("video.h3_stream_prefetch: %s", exc)if h3_dit_resident_enabled() and denoiser_streamed in ("stream","stream_lazy",diff --git a/studio/backend/core/inference/video_minimax_h3_residency.py b/studio/backend/core/inference/video_minimax_h3_residency.pyindex bc8d855cad4..189a9f58ad0 100644--- a/studio/backend/core/inference/video_minimax_h3_residency.py+++ b/studio/backend/core/inference/video_minimax_h3_residency.py@@ -112,6 +112,19 @@ def _noop() -> None:return None+def _resident_onload(group: Any) -> Any:+ """A resident group's onload kicks the prefetch, else the first streamed block behind it is never prefetched."""+ prefetcher = getattr(group, "_unsloth_prefetcher", None)+ kick = getattr(prefetcher, "kick", None)+ if not callable(kick):+ return _noop++ def onload_(*_a: Any, **_k: Any) -> None:+ kick()++ return _outside_inference_mode(onload_)++def is_resident(group: Any) -> bool:return bool(getattr(group, "_unsloth_resident", False))@@ -123,14 +136,20 @@ def make_resident(group: Any) -> None:if is_resident(group):return# The instance attribute, not the class method: an instance override (pinned top group) is an onload path too.- group.onload_()+ # Except the prefetcher's own: a group leaving the stream must not count against its in-flight window.+ prefetcher = getattr(group, "_unsloth_prefetcher", None)+ owns = getattr(prefetcher, "owns", None)+ if callable(owns) and owns(group):+ _outside_inference_mode(type(group).onload_)(group)+ else:+ group.onload_()stream = getattr(group, "stream", None)if stream is not None:stream.synchronize()if torch.cuda.is_available():torch.cuda.current_stream().synchronize()group._unsloth_saved_io = (group.__dict__.get("onload_"), group.__dict__.get("offload_"))- group.onload_ = _noop+ group.onload_ = _resident_onload(group)group.offload_ = _noopgroup._unsloth_resident = True@@ -444,6 +463,117 @@ def pin_streamed_top_level_group(transformer: Any, logger: Any = None) -> bool:return True+H3_STREAM_PREFETCH_ENV = "UNSLOTH_H3_STREAM_PREFETCH"+++def h3_stream_prefetch_depth(block_bytes: list, default_depth: int) -> int:+ """Groups copied ahead: as many as ``H3_STREAM_WINDOW_GB`` (reserved by the residency plan) holds next to the+ running block, capped by ``default_depth``, at least 1."""+ largest = max(block_bytes) if block_bytes else 0+ if largest <= 0:+ return max(1, int(default_depth))+ fits = int(H3_STREAM_WINDOW_GB * 1e9 // largest) - 1+ return max(1, min(int(default_depth), fits))+++def h3_stream_prefetch_window(top_bytes: int, block_bytes: list) -> int:+ """Bytes the prefetcher may hold on the device: the top-level group plus two blocks, diffusers' own footprint."""+ largest = max(block_bytes) if block_bytes else 0+ return int(max(0, top_bytes) + 2 * largest)+++def install_h3_stream_prefetch(+ transformer: Any,+ device: Any,+ logger: Any = None,+) -> int:+ """Drive a block-streamed H3 denoiser's offload groups with the event-fenced prefetch (no host sync per onload).+ Run after the top-level pin, before the residency fit; returns groups covered. ``UNSLOTH_H3_STREAM_PREFETCH=0`` off."""+ if str(os.environ.get(H3_STREAM_PREFETCH_ENV, "")).strip().lower() in (+ "0",+ "off",+ "false",+ "no",+ ):+ return 0+ try:+ from .diffusion_offload_prefetch import install_group_prefetch, prefetch_depth+ except ImportError:+ return 0+ top, blocks = h3_offload_groups(transformer)+ if not blocks:+ return 0+ depth = h3_stream_prefetch_depth([group_payload_bytes(g) for g in blocks], prefetch_depth())+ from .diffusion_memory import _pinned_memory_capped++ if _pinned_memory_capped():+ depth = 1 # each in-flight group is pinned on the fly; diffusers' own path holds two near a ~1 GiB cap+ # The prefetcher refuses groups whose onload_ is replaced: lift the outside-inference_mode wrappers (torchao v1+ # int8 cannot be re-pointed inside it), install, then wrap its moves the same way.+ groups = _all_offload_groups(transformer)+ # A top-level group the H3 pin left unstreamed (kill switch, host refusal) keeps its wrapper, so the generic+ # top-group adoption cannot pin it.+ kept = top if top is not None and getattr(top, "stream", None) is None else None+ lifted = [g for g in groups if g is not kept and _lift_outside_inference_wrappers(g)]+ try:+ covered = install_group_prefetch(transformer, device, logger, depth = depth)+ except Exception: # noqa: BLE001 -- install_group_prefetch never raises today; keep the wrappers either way+ covered = 0+ for group in lifted:+ for name in ("onload_", "offload_"):+ setattr(group, name, _outside_inference_mode(getattr(group, name)))+ if covered:+ from .diffusion_offload_prefetch import module_prefetcher++ prefetcher = module_prefetcher(transformer)+ for group in groups:+ if getattr(group, "_unsloth_prefetcher", None) is prefetcher:+ # owns() compares the instance onload_ with this+ group._unsloth_prefetch_onload = group.__dict__.get("onload_")+ if prefetcher is not None:+ # (depth + 1) x the 0.8 GB top-level group would leave no block ahead once that group streams (12 GB).+ prefetcher.window = h3_stream_prefetch_window(+ group_payload_bytes(top) if top is not None else 0,+ [group_payload_bytes(g) for g in blocks],+ )+ # end() puts groups copied ahead but never run back on the host copy; kick() queues copies+ for name in ("end", "kick"):+ setattr(prefetcher, name, _outside_inference_mode(getattr(prefetcher, name)))+ return covered+++def _all_offload_groups(transformer: Any) -> list:+ out: list = []+ seen: set = set()+ for sub in transformer.modules():+ group = getattr(_group_hook(sub), "group", None)+ if group is not None and id(group) not in seen:+ seen.add(id(group))+ out.append(group)+ return out+++def _lift_outside_inference_wrappers(group: Any) -> bool:+ """Drop the instance onload_ / offload_ that only re-enter the class methods outside inference_mode."""+ attrs = getattr(group, "__dict__", {})+ if is_resident(group) or not any(name in attrs for name in ("onload_", "offload_")):+ return False+ for name in ("onload_", "offload_"):+ attrs.pop(name, None)+ return True+++def _outside_inference_mode(fn: Any) -> Any:+ import torch++ def call(*args: Any, **kwargs: Any) -> Any:+ with torch.inference_mode(False), torch.no_grad():+ return fn(*args, **kwargs)++ disable = getattr(getattr(torch, "compiler", None), "disable", None)+ return disable(call) if callable(disable) else call++H3_VAE_PINNED_SWAP_ENV = "UNSLOTH_H3_VAE_PINNED_SWAP"diff --git a/studio/backend/tests/test_h3_stream_prefetch.py b/studio/backend/tests/test_h3_stream_prefetch.pynew file mode 100644index 00000000000..ca562c0d42f--- /dev/null+++ b/studio/backend/tests/test_h3_stream_prefetch.py@@ -0,0 +1,336 @@+# SPDX-License-Identifier: AGPL-3.0-only+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0++"""MiniMax-H3's block-streamed denoiser runs on the event-fenced prefetch (``diffusion_offload_prefetch``).++H3 streams through ``stream_prequantized_module`` (diffusers block-level stream groups), pins its top-level group onto+the blocks' copy stream and keeps a resident prefix (``H3Residency``), none of which reached the prefetcher, so every+streamed group was fenced on the host. The CUDA cases build that exact layout and check: no host synchronization per+forward, bit-identical output to the resident model across residency re-fits, the first streamed block prefetched+behind a resident prefix, and the kill switch keeping diffusers' path.+"""++from __future__ import annotations++import copy+import warnings++import pytest++torch = pytest.importorskip("torch")+pytest.importorskip("diffusers.hooks")++import core.inference.video_minimax_h3_residency as res # noqa: E402++[email protected](autouse = True)+def _clean_env(monkeypatch):+ for name in (+ res.H3_STREAM_PREFETCH_ENV,+ "UNSLOTH_DIFFUSION_ASYNC_PREFETCH",+ "UNSLOTH_DIFFUSION_PREFETCH_DEPTH",+ "UNSLOTH_H3_TOP_GROUP_PIN",+ ):+ monkeypatch.delenv(name, raising = False)+++def test_depth_keeps_the_prefetch_inside_the_reserved_stream_window():+ gb = int(1e9)+ # H3 int8 blocks are ~0.39 GB: running + 2 ahead = 1.17 GB fits the 1.5 GB window+ assert res.h3_stream_prefetch_depth([int(0.39 * gb)] * 50, 2) == 2+ # a block too big for two ahead drops to diffusers' own depth (the next group only)+ assert res.h3_stream_prefetch_depth([int(0.6 * gb)], 2) == 1+ assert res.h3_stream_prefetch_depth([int(2 * gb)], 4) == 1+ # never deeper than the prefetcher's own setting+ assert res.h3_stream_prefetch_depth([int(0.1 * gb)], 2) == 2+ assert res.h3_stream_prefetch_depth([], 3) == 3+++def test_window_is_diffusers_footprint_top_plus_two_blocks():+ assert res.h3_stream_prefetch_window(800, [390, 380]) == 800 + 2 * 390+ assert res.h3_stream_prefetch_window(0, [390]) == 780+ assert res.h3_stream_prefetch_window(0, []) == 0+++def test_resident_onload_kicks_the_prefetcher_and_stays_a_noop_without_one():+ calls = []++ class _PF:+ def kick(self):+ calls.append("kick")++ class _G:+ pass++ g = _G()+ res._resident_onload(g)() # no prefetcher: plain no-op+ assert calls == []+ g._unsloth_prefetcher = _PF()+ res._resident_onload(g)()+ assert calls == ["kick"]+++def test_kill_switch_and_unstreamed_module_leave_it_alone(monkeypatch):+ monkeypatch.setenv(res.H3_STREAM_PREFETCH_ENV, "0")+ assert res.install_h3_stream_prefetch(torch.nn.Linear(2, 2), "cuda") == 0+ monkeypatch.delenv(res.H3_STREAM_PREFETCH_ENV)+ assert res.install_h3_stream_prefetch(torch.nn.Linear(2, 2), "cuda") == 0 # no offload groups+++def _cuda():+ if not torch.cuda.is_available():+ pytest.skip("needs CUDA: diffusers stream group offload")+++class _Block(torch.nn.Module):+ def __init__(self, width):+ super().__init__()+ self.lin = torch.nn.Linear(width, width)++ def forward(self, x):+ return torch.nn.functional.gelu(self.lin(x))+++class _DiT(torch.nn.Module):+ """H3's layout: top-level embedders / output and ``transformer_blocks``; the top-level group is about twice a+ block, like H3's 0.8 GB top-level group against 0.39 GB blocks."""++ def __init__(+ self,+ width = 1024,+ blocks = 8,+ ):+ super().__init__()+ self.proj_in = torch.nn.Linear(64, width)+ self.embed = torch.nn.Linear(width, 2 * width)+ self.transformer_blocks = torch.nn.ModuleList(_Block(width) for _ in range(blocks))+ self.proj_out = torch.nn.Linear(width, 64)++ def forward(self, x):+ x = self.proj_in(x)+ x = x + self.embed(x)[..., : x.shape[-1]]+ for block in self.transformer_blocks:+ x = block(x)+ return self.proj_out(x)+++def _h3_streamed(+ net,+ monkeypatch,+ prefetch = True,+ outside_inference = True,+ pin_top = True,+):+ """What the H3 load does at a capped tier: stream groups with record_stream, pinned top group, prefetch."""+ import inspect++ from diffusers.hooks import apply_group_offloading++ import core.inference.video_minimax_h3_te as te++ monkeypatch.setattr(te, "h3_te_pin_allowed", lambda *_a, **_k: True)+ kwargs = {+ "onload_device": torch.device("cuda"),+ "offload_device": torch.device("cpu"),+ "offload_type": "block_level",+ "num_blocks_per_group": 1,+ "use_stream": True,+ }+ params = inspect.signature(apply_group_offloading).parameters+ if "record_stream" not in params:+ pytest.skip("diffusers without record_stream")+ kwargs["record_stream"] = True+ if "non_blocking" in params:+ kwargs["non_blocking"] = True+ apply_group_offloading(net, **kwargs)+ if outside_inference:+ # what stream_prequantized_module does for torchao v1 int8: every group move outside inference_mode+ from core.inference.diffusion_prequant import _move_groups_outside_inference_mode+ _move_groups_outside_inference_mode(net)+ assert res.pin_streamed_top_level_group(net) is pin_top+ n = res.install_h3_stream_prefetch(net, "cuda") if prefetch else 0+ return n+++def _count_late_copies(pf):+ """Count block copies issued after their predecessor block was offloaded (serialized behind its compute)."""+ late = {"n": 0, "offloaded": set()}+ issue, offload, begin = pf._issue, pf.offload, pf.begin++ def _begin():+ late["offloaded"] = set()+ return begin()++ def _offload(group):+ late["offloaded"].add(id(group))+ return offload(group)++ def _issue(group):+ gid = id(group)+ if pf.active and pf.on_order and gid in pf.order:+ k = pf.order.index(gid)+ if k > 0 and pf.order[k - 1] in late["offloaded"]:+ late["n"] += 1+ return issue(group)++ pf.begin, pf.offload, pf._issue = _begin, _offload, _issue+ return late+++def _count_syncs(fn):+ prev = torch.cuda.get_sync_debug_mode()+ torch.cuda.set_sync_debug_mode("warn")+ try:+ with warnings.catch_warnings(record = True) as caught:+ warnings.simplefilter("always")+ out = fn()+ n = sum(1 for w in caught if "synchroniz" in str(w.message).lower())+ finally:+ torch.cuda.set_sync_debug_mode(prev)+ return out, n++[email protected]("resident", [0, 1, 3])+def test_streamed_h3_forward_has_no_host_sync_and_is_bit_identical(resident, monkeypatch):+ _cuda()+ from core.inference.diffusion_offload_prefetch import module_prefetcher++ torch.manual_seed(0)+ net = _DiT().eval()+ ref = copy.deepcopy(net).cuda()+ block = sum(p.numel() * p.element_size() for p in net.transformer_blocks[0].parameters())+ # the 1.5 GB H3 reserve, scaled to these blocks: 3.85 blocks, as 1.5 GB is to H3's 0.39 GB blocks+ monkeypatch.setattr(res, "H3_STREAM_WINDOW_GB", 3.85 * block / 1e9)+ covered = _h3_streamed(net, monkeypatch)+ assert covered == 1 + len(net.transformer_blocks) # the pinned top group is one of them+ pf = module_prefetcher(net)+ assert pf is not None+ residency = res.H3Residency(net, "cuda")+ residency.apply(resident > 0, resident)+ late = _count_late_copies(pf)+ x = torch.randn(4, 64, device = "cuda")+ with torch.no_grad():+ want = ref(x)+ assert torch.equal(net(x), want) # records the order+ late["n"] = 0+ for _ in range(3):+ got, syncs = _count_syncs(lambda: net(x))+ assert syncs == 0+ torch.cuda.synchronize()+ assert torch.equal(got, want)+ assert pf.stats["prefetched"] > 0 and pf.stats["missed"] == 0+ # every block's copy is queued before its predecessor's offload fences the copy stream on that predecessor's+ # compute; otherwise the copy runs after it and the GPU alternates copy and compute (no overlap)+ assert late["n"] == 0, late+ assert pf.peak_inflight_bytes <= pf.window+ if resident:+ # the resident prefix kicks the prefetch, so the first streamed block is already in flight+ assert all(res.is_resident(g) for g in residency.blocks[:resident])+ assert net.transformer_blocks[-1].lin.weight.device.type == "cpu"+++def test_streamed_h3_forward_under_inference_mode(monkeypatch):+ """The inference_mode wrappers stream_prequantized_module installs are lifted and re-applied around the prefetch."""+ _cuda()+ from core.inference.diffusion_offload_prefetch import module_prefetcher++ torch.manual_seed(0)+ net = _DiT().eval()+ ref = copy.deepcopy(net).cuda()+ assert _h3_streamed(net, monkeypatch) == 1 + len(net.transformer_blocks)+ pf = module_prefetcher(net)+ residency = res.H3Residency(net, "cuda")+ residency.apply(True, 2)+ x = torch.randn(4, 64, device = "cuda")+ with torch.inference_mode():+ want = ref(x)+ for _ in range(3):+ got, syncs = _count_syncs(lambda: net(x))+ torch.cuda.synchronize()+ assert torch.equal(got, want)+ assert syncs == 0+ assert pf.stats["prefetched"] > 0+ for group in residency.blocks[2:]:+ assert pf.owns(group)+++def test_refit_between_requests_stays_bit_identical(monkeypatch):+ _cuda()+ torch.manual_seed(0)+ net = _DiT().eval()+ ref = copy.deepcopy(net).cuda()+ _h3_streamed(net, monkeypatch)+ residency = res.H3Residency(net, "cuda")+ x = torch.randn(4, 64, device = "cuda")+ with torch.no_grad():+ want = ref(x)+ for top, n in ((True, 4), (True, 1), (False, 0), (True, 6), (False, 2)):+ residency.apply(top, n)+ for _ in range(2):+ got, syncs = _count_syncs(lambda: net(x))+ torch.cuda.synchronize()+ assert torch.equal(got, want), (top, n)+ assert syncs == 0, (top, n)+ res.release_all(residency)+ assert torch.equal(net(x), want)+++def test_without_the_prefetch_every_streamed_group_waits_on_the_host(monkeypatch):+ """Control: the H3 layout as main builds it synchronizes the host for every streamed group."""+ _cuda()+ torch.manual_seed(0)+ net = _DiT().eval()+ monkeypatch.setenv(res.H3_STREAM_PREFETCH_ENV, "0")+ assert _h3_streamed(net, monkeypatch) == 0+ x = torch.randn(4, 64, device = "cuda")+ with torch.no_grad():+ net(x)+ _, syncs = _count_syncs(lambda: net(x))+ assert syncs >= len(net.transformer_blocks)+++def test_top_pin_kill_switch_keeps_diffusers_top_group(monkeypatch):+ """UNSLOTH_H3_TOP_GROUP_PIN=0: the generic top-group adoption must not pin it behind the H3 switch's back."""+ _cuda()+ from core.inference.diffusion_offload_prefetch import module_prefetcher++ import core.inference.diffusion_offload_prefetch as op++ monkeypatch.setenv(res.H3_TOP_GROUP_PIN_ENV, "0")+ seen = []+ adopt = op._adopt_top_group++ def _spy(module, *a, **k):+ top_group, _ = res.h3_offload_groups(module)+ # H3's top group is torchao: adoption takes it whenever it is unstreamed and its onload_ is not replaced+ seen.append(top_group.stream is None and "onload_" not in top_group.__dict__)+ return adopt(module, *a, **k)++ monkeypatch.setattr(op, "_adopt_top_group", _spy)+ torch.manual_seed(0)+ net = _DiT().eval()+ ref = copy.deepcopy(net).cuda()+ assert _h3_streamed(net, monkeypatch, pin_top = False) == len(net.transformer_blocks)+ top, _ = res.h3_offload_groups(net)+ pf = module_prefetcher(net)+ assert seen == [False]+ assert top.stream is None and not pf.owns(top)+ x = torch.randn(4, 64, device = "cuda")+ with torch.inference_mode():+ want = ref(x)+ for _ in range(2):+ got = net(x)+ torch.cuda.synchronize()+ assert torch.equal(got, want)+++def test_capped_pinned_host_prefetches_one_ahead(monkeypatch):+ """Windows / WSL2 pin each in-flight group on the fly under a ~1 GiB cap: one ahead, as diffusers holds."""+ _cuda()+ import core.inference.diffusion_memory as dm+ from core.inference.diffusion_offload_prefetch import module_prefetcher++ monkeypatch.setattr(dm, "_pinned_memory_capped", lambda: True)+ net = _DiT().eval()+ assert _h3_streamed(net, monkeypatch) == 1 + len(net.transformer_blocks)+ assert module_prefetcher(net).depth == 1