Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

Studio: async block prefetch for MiniMax-H3's streamed denoiser (#12683)

Daniel Han8e11ba15ef59
diff --git a/studio/backend/core/inference/video.py b/studio/backend/core/inference/video.py
index a10b49ed831..e96462eedd6 100644
--- a/studio/backend/core/inference/video.py
+++ b/studio/backend/core/inference/video.py
@@ -7347,10 +7347,18 @@ class VideoBackend:
if denoiser_streamed in ("stream", "stream_lazy"):
from .video_minimax_h3_residency import pin_streamed_top_level_group
+
try:
pin_streamed_top_level_group(denoiser, logger = logger)
except Exception as exc: # noqa: BLE001 -- a speed-up only
logger.warning("video.h3_top_group: %s", exc)
+ # after the top-level pin (it becomes one of the prefetched groups), before the residency fit
+ from .video_minimax_h3_residency import install_h3_stream_prefetch
+
+ try:
+ install_h3_stream_prefetch(denoiser, device, logger = logger)
+ except Exception as exc: # noqa: BLE001 -- a speed-up only
+ logger.warning("video.h3_stream_prefetch: %s", exc)
if h3_dit_resident_enabled() and denoiser_streamed in (
"stream",
"stream_lazy",
diff --git a/studio/backend/core/inference/video_minimax_h3_residency.py b/studio/backend/core/inference/video_minimax_h3_residency.py
index bc8d855cad4..189a9f58ad0 100644
--- a/studio/backend/core/inference/video_minimax_h3_residency.py
+++ b/studio/backend/core/inference/video_minimax_h3_residency.py
@@ -112,6 +112,19 @@ def _noop() -> None:
return None
+def _resident_onload(group: Any) -> Any:
+ """A resident group's onload kicks the prefetch, else the first streamed block behind it is never prefetched."""
+ prefetcher = getattr(group, "_unsloth_prefetcher", None)
+ kick = getattr(prefetcher, "kick", None)
+ if not callable(kick):
+ return _noop
+
+ def onload_(*_a: Any, **_k: Any) -> None:
+ kick()
+
+ return _outside_inference_mode(onload_)
+
+
def is_resident(group: Any) -> bool:
return bool(getattr(group, "_unsloth_resident", False))
@@ -123,14 +136,20 @@ def make_resident(group: Any) -> None:
if is_resident(group):
return
# The instance attribute, not the class method: an instance override (pinned top group) is an onload path too.
- group.onload_()
+ # Except the prefetcher's own: a group leaving the stream must not count against its in-flight window.
+ prefetcher = getattr(group, "_unsloth_prefetcher", None)
+ owns = getattr(prefetcher, "owns", None)
+ if callable(owns) and owns(group):
+ _outside_inference_mode(type(group).onload_)(group)
+ else:
+ group.onload_()
stream = getattr(group, "stream", None)
if stream is not None:
stream.synchronize()
if torch.cuda.is_available():
torch.cuda.current_stream().synchronize()
group._unsloth_saved_io = (group.__dict__.get("onload_"), group.__dict__.get("offload_"))
- group.onload_ = _noop
+ group.onload_ = _resident_onload(group)
group.offload_ = _noop
group._unsloth_resident = True
@@ -444,6 +463,117 @@ def pin_streamed_top_level_group(transformer: Any, logger: Any = None) -> bool:
return True
+H3_STREAM_PREFETCH_ENV = "UNSLOTH_H3_STREAM_PREFETCH"
+
+
+def h3_stream_prefetch_depth(block_bytes: list, default_depth: int) -> int:
+ """Groups copied ahead: as many as ``H3_STREAM_WINDOW_GB`` (reserved by the residency plan) holds next to the
+ running block, capped by ``default_depth``, at least 1."""
+ largest = max(block_bytes) if block_bytes else 0
+ if largest <= 0:
+ return max(1, int(default_depth))
+ fits = int(H3_STREAM_WINDOW_GB * 1e9 // largest) - 1
+ return max(1, min(int(default_depth), fits))
+
+
+def h3_stream_prefetch_window(top_bytes: int, block_bytes: list) -> int:
+ """Bytes the prefetcher may hold on the device: the top-level group plus two blocks, diffusers' own footprint."""
+ largest = max(block_bytes) if block_bytes else 0
+ return int(max(0, top_bytes) + 2 * largest)
+
+
+def install_h3_stream_prefetch(
+ transformer: Any,
+ device: Any,
+ logger: Any = None,
+) -> int:
+ """Drive a block-streamed H3 denoiser's offload groups with the event-fenced prefetch (no host sync per onload).
+ Run after the top-level pin, before the residency fit; returns groups covered. ``UNSLOTH_H3_STREAM_PREFETCH=0`` off."""
+ if str(os.environ.get(H3_STREAM_PREFETCH_ENV, "")).strip().lower() in (
+ "0",
+ "off",
+ "false",
+ "no",
+ ):
+ return 0
+ try:
+ from .diffusion_offload_prefetch import install_group_prefetch, prefetch_depth
+ except ImportError:
+ return 0
+ top, blocks = h3_offload_groups(transformer)
+ if not blocks:
+ return 0
+ depth = h3_stream_prefetch_depth([group_payload_bytes(g) for g in blocks], prefetch_depth())
+ from .diffusion_memory import _pinned_memory_capped
+
+ if _pinned_memory_capped():
+ depth = 1 # each in-flight group is pinned on the fly; diffusers' own path holds two near a ~1 GiB cap
+ # The prefetcher refuses groups whose onload_ is replaced: lift the outside-inference_mode wrappers (torchao v1
+ # int8 cannot be re-pointed inside it), install, then wrap its moves the same way.
+ groups = _all_offload_groups(transformer)
+ # A top-level group the H3 pin left unstreamed (kill switch, host refusal) keeps its wrapper, so the generic
+ # top-group adoption cannot pin it.
+ kept = top if top is not None and getattr(top, "stream", None) is None else None
+ lifted = [g for g in groups if g is not kept and _lift_outside_inference_wrappers(g)]
+ try:
+ covered = install_group_prefetch(transformer, device, logger, depth = depth)
+ except Exception: # noqa: BLE001 -- install_group_prefetch never raises today; keep the wrappers either way
+ covered = 0
+ for group in lifted:
+ for name in ("onload_", "offload_"):
+ setattr(group, name, _outside_inference_mode(getattr(group, name)))
+ if covered:
+ from .diffusion_offload_prefetch import module_prefetcher
+
+ prefetcher = module_prefetcher(transformer)
+ for group in groups:
+ if getattr(group, "_unsloth_prefetcher", None) is prefetcher:
+ # owns() compares the instance onload_ with this
+ group._unsloth_prefetch_onload = group.__dict__.get("onload_")
+ if prefetcher is not None:
+ # (depth + 1) x the 0.8 GB top-level group would leave no block ahead once that group streams (12 GB).
+ prefetcher.window = h3_stream_prefetch_window(
+ group_payload_bytes(top) if top is not None else 0,
+ [group_payload_bytes(g) for g in blocks],
+ )
+ # end() puts groups copied ahead but never run back on the host copy; kick() queues copies
+ for name in ("end", "kick"):
+ setattr(prefetcher, name, _outside_inference_mode(getattr(prefetcher, name)))
+ return covered
+
+
+def _all_offload_groups(transformer: Any) -> list:
+ out: list = []
+ seen: set = set()
+ for sub in transformer.modules():
+ group = getattr(_group_hook(sub), "group", None)
+ if group is not None and id(group) not in seen:
+ seen.add(id(group))
+ out.append(group)
+ return out
+
+
+def _lift_outside_inference_wrappers(group: Any) -> bool:
+ """Drop the instance onload_ / offload_ that only re-enter the class methods outside inference_mode."""
+ attrs = getattr(group, "__dict__", {})
+ if is_resident(group) or not any(name in attrs for name in ("onload_", "offload_")):
+ return False
+ for name in ("onload_", "offload_"):
+ attrs.pop(name, None)
+ return True
+
+
+def _outside_inference_mode(fn: Any) -> Any:
+ import torch
+
+ def call(*args: Any, **kwargs: Any) -> Any:
+ with torch.inference_mode(False), torch.no_grad():
+ return fn(*args, **kwargs)
+
+ disable = getattr(getattr(torch, "compiler", None), "disable", None)
+ return disable(call) if callable(disable) else call
+
+
H3_VAE_PINNED_SWAP_ENV = "UNSLOTH_H3_VAE_PINNED_SWAP"
diff --git a/studio/backend/tests/test_h3_stream_prefetch.py b/studio/backend/tests/test_h3_stream_prefetch.py
new file mode 100644
index 00000000000..ca562c0d42f
--- /dev/null
+++ b/studio/backend/tests/test_h3_stream_prefetch.py
@@ -0,0 +1,336 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
+
+"""MiniMax-H3's block-streamed denoiser runs on the event-fenced prefetch (``diffusion_offload_prefetch``).
+
+H3 streams through ``stream_prequantized_module`` (diffusers block-level stream groups), pins its top-level group onto
+the blocks' copy stream and keeps a resident prefix (``H3Residency``), none of which reached the prefetcher, so every
+streamed group was fenced on the host. The CUDA cases build that exact layout and check: no host synchronization per
+forward, bit-identical output to the resident model across residency re-fits, the first streamed block prefetched
+behind a resident prefix, and the kill switch keeping diffusers' path.
+"""
+
+from __future__ import annotations
+
+import copy
+import warnings
+
+import pytest
+
+torch = pytest.importorskip("torch")
+pytest.importorskip("diffusers.hooks")
+
+import core.inference.video_minimax_h3_residency as res # noqa: E402
+
+
[email protected](autouse = True)
+def _clean_env(monkeypatch):
+ for name in (
+ res.H3_STREAM_PREFETCH_ENV,
+ "UNSLOTH_DIFFUSION_ASYNC_PREFETCH",
+ "UNSLOTH_DIFFUSION_PREFETCH_DEPTH",
+ "UNSLOTH_H3_TOP_GROUP_PIN",
+ ):
+ monkeypatch.delenv(name, raising = False)
+
+
+def test_depth_keeps_the_prefetch_inside_the_reserved_stream_window():
+ gb = int(1e9)
+ # H3 int8 blocks are ~0.39 GB: running + 2 ahead = 1.17 GB fits the 1.5 GB window
+ assert res.h3_stream_prefetch_depth([int(0.39 * gb)] * 50, 2) == 2
+ # a block too big for two ahead drops to diffusers' own depth (the next group only)
+ assert res.h3_stream_prefetch_depth([int(0.6 * gb)], 2) == 1
+ assert res.h3_stream_prefetch_depth([int(2 * gb)], 4) == 1
+ # never deeper than the prefetcher's own setting
+ assert res.h3_stream_prefetch_depth([int(0.1 * gb)], 2) == 2
+ assert res.h3_stream_prefetch_depth([], 3) == 3
+
+
+def test_window_is_diffusers_footprint_top_plus_two_blocks():
+ assert res.h3_stream_prefetch_window(800, [390, 380]) == 800 + 2 * 390
+ assert res.h3_stream_prefetch_window(0, [390]) == 780
+ assert res.h3_stream_prefetch_window(0, []) == 0
+
+
+def test_resident_onload_kicks_the_prefetcher_and_stays_a_noop_without_one():
+ calls = []
+
+ class _PF:
+ def kick(self):
+ calls.append("kick")
+
+ class _G:
+ pass
+
+ g = _G()
+ res._resident_onload(g)() # no prefetcher: plain no-op
+ assert calls == []
+ g._unsloth_prefetcher = _PF()
+ res._resident_onload(g)()
+ assert calls == ["kick"]
+
+
+def test_kill_switch_and_unstreamed_module_leave_it_alone(monkeypatch):
+ monkeypatch.setenv(res.H3_STREAM_PREFETCH_ENV, "0")
+ assert res.install_h3_stream_prefetch(torch.nn.Linear(2, 2), "cuda") == 0
+ monkeypatch.delenv(res.H3_STREAM_PREFETCH_ENV)
+ assert res.install_h3_stream_prefetch(torch.nn.Linear(2, 2), "cuda") == 0 # no offload groups
+
+
+def _cuda():
+ if not torch.cuda.is_available():
+ pytest.skip("needs CUDA: diffusers stream group offload")
+
+
+class _Block(torch.nn.Module):
+ def __init__(self, width):
+ super().__init__()
+ self.lin = torch.nn.Linear(width, width)
+
+ def forward(self, x):
+ return torch.nn.functional.gelu(self.lin(x))
+
+
+class _DiT(torch.nn.Module):
+ """H3's layout: top-level embedders / output and ``transformer_blocks``; the top-level group is about twice a
+ block, like H3's 0.8 GB top-level group against 0.39 GB blocks."""
+
+ def __init__(
+ self,
+ width = 1024,
+ blocks = 8,
+ ):
+ super().__init__()
+ self.proj_in = torch.nn.Linear(64, width)
+ self.embed = torch.nn.Linear(width, 2 * width)
+ self.transformer_blocks = torch.nn.ModuleList(_Block(width) for _ in range(blocks))
+ self.proj_out = torch.nn.Linear(width, 64)
+
+ def forward(self, x):
+ x = self.proj_in(x)
+ x = x + self.embed(x)[..., : x.shape[-1]]
+ for block in self.transformer_blocks:
+ x = block(x)
+ return self.proj_out(x)
+
+
+def _h3_streamed(
+ net,
+ monkeypatch,
+ prefetch = True,
+ outside_inference = True,
+ pin_top = True,
+):
+ """What the H3 load does at a capped tier: stream groups with record_stream, pinned top group, prefetch."""
+ import inspect
+
+ from diffusers.hooks import apply_group_offloading
+
+ import core.inference.video_minimax_h3_te as te
+
+ monkeypatch.setattr(te, "h3_te_pin_allowed", lambda *_a, **_k: True)
+ kwargs = {
+ "onload_device": torch.device("cuda"),
+ "offload_device": torch.device("cpu"),
+ "offload_type": "block_level",
+ "num_blocks_per_group": 1,
+ "use_stream": True,
+ }
+ params = inspect.signature(apply_group_offloading).parameters
+ if "record_stream" not in params:
+ pytest.skip("diffusers without record_stream")
+ kwargs["record_stream"] = True
+ if "non_blocking" in params:
+ kwargs["non_blocking"] = True
+ apply_group_offloading(net, **kwargs)
+ if outside_inference:
+ # what stream_prequantized_module does for torchao v1 int8: every group move outside inference_mode
+ from core.inference.diffusion_prequant import _move_groups_outside_inference_mode
+ _move_groups_outside_inference_mode(net)
+ assert res.pin_streamed_top_level_group(net) is pin_top
+ n = res.install_h3_stream_prefetch(net, "cuda") if prefetch else 0
+ return n
+
+
+def _count_late_copies(pf):
+ """Count block copies issued after their predecessor block was offloaded (serialized behind its compute)."""
+ late = {"n": 0, "offloaded": set()}
+ issue, offload, begin = pf._issue, pf.offload, pf.begin
+
+ def _begin():
+ late["offloaded"] = set()
+ return begin()
+
+ def _offload(group):
+ late["offloaded"].add(id(group))
+ return offload(group)
+
+ def _issue(group):
+ gid = id(group)
+ if pf.active and pf.on_order and gid in pf.order:
+ k = pf.order.index(gid)
+ if k > 0 and pf.order[k - 1] in late["offloaded"]:
+ late["n"] += 1
+ return issue(group)
+
+ pf.begin, pf.offload, pf._issue = _begin, _offload, _issue
+ return late
+
+
+def _count_syncs(fn):
+ prev = torch.cuda.get_sync_debug_mode()
+ torch.cuda.set_sync_debug_mode("warn")
+ try:
+ with warnings.catch_warnings(record = True) as caught:
+ warnings.simplefilter("always")
+ out = fn()
+ n = sum(1 for w in caught if "synchroniz" in str(w.message).lower())
+ finally:
+ torch.cuda.set_sync_debug_mode(prev)
+ return out, n
+
+
[email protected]("resident", [0, 1, 3])
+def test_streamed_h3_forward_has_no_host_sync_and_is_bit_identical(resident, monkeypatch):
+ _cuda()
+ from core.inference.diffusion_offload_prefetch import module_prefetcher
+
+ torch.manual_seed(0)
+ net = _DiT().eval()
+ ref = copy.deepcopy(net).cuda()
+ block = sum(p.numel() * p.element_size() for p in net.transformer_blocks[0].parameters())
+ # the 1.5 GB H3 reserve, scaled to these blocks: 3.85 blocks, as 1.5 GB is to H3's 0.39 GB blocks
+ monkeypatch.setattr(res, "H3_STREAM_WINDOW_GB", 3.85 * block / 1e9)
+ covered = _h3_streamed(net, monkeypatch)
+ assert covered == 1 + len(net.transformer_blocks) # the pinned top group is one of them
+ pf = module_prefetcher(net)
+ assert pf is not None
+ residency = res.H3Residency(net, "cuda")
+ residency.apply(resident > 0, resident)
+ late = _count_late_copies(pf)
+ x = torch.randn(4, 64, device = "cuda")
+ with torch.no_grad():
+ want = ref(x)
+ assert torch.equal(net(x), want) # records the order
+ late["n"] = 0
+ for _ in range(3):
+ got, syncs = _count_syncs(lambda: net(x))
+ assert syncs == 0
+ torch.cuda.synchronize()
+ assert torch.equal(got, want)
+ assert pf.stats["prefetched"] > 0 and pf.stats["missed"] == 0
+ # every block's copy is queued before its predecessor's offload fences the copy stream on that predecessor's
+ # compute; otherwise the copy runs after it and the GPU alternates copy and compute (no overlap)
+ assert late["n"] == 0, late
+ assert pf.peak_inflight_bytes <= pf.window
+ if resident:
+ # the resident prefix kicks the prefetch, so the first streamed block is already in flight
+ assert all(res.is_resident(g) for g in residency.blocks[:resident])
+ assert net.transformer_blocks[-1].lin.weight.device.type == "cpu"
+
+
+def test_streamed_h3_forward_under_inference_mode(monkeypatch):
+ """The inference_mode wrappers stream_prequantized_module installs are lifted and re-applied around the prefetch."""
+ _cuda()
+ from core.inference.diffusion_offload_prefetch import module_prefetcher
+
+ torch.manual_seed(0)
+ net = _DiT().eval()
+ ref = copy.deepcopy(net).cuda()
+ assert _h3_streamed(net, monkeypatch) == 1 + len(net.transformer_blocks)
+ pf = module_prefetcher(net)
+ residency = res.H3Residency(net, "cuda")
+ residency.apply(True, 2)
+ x = torch.randn(4, 64, device = "cuda")
+ with torch.inference_mode():
+ want = ref(x)
+ for _ in range(3):
+ got, syncs = _count_syncs(lambda: net(x))
+ torch.cuda.synchronize()
+ assert torch.equal(got, want)
+ assert syncs == 0
+ assert pf.stats["prefetched"] > 0
+ for group in residency.blocks[2:]:
+ assert pf.owns(group)
+
+
+def test_refit_between_requests_stays_bit_identical(monkeypatch):
+ _cuda()
+ torch.manual_seed(0)
+ net = _DiT().eval()
+ ref = copy.deepcopy(net).cuda()
+ _h3_streamed(net, monkeypatch)
+ residency = res.H3Residency(net, "cuda")
+ x = torch.randn(4, 64, device = "cuda")
+ with torch.no_grad():
+ want = ref(x)
+ for top, n in ((True, 4), (True, 1), (False, 0), (True, 6), (False, 2)):
+ residency.apply(top, n)
+ for _ in range(2):
+ got, syncs = _count_syncs(lambda: net(x))
+ torch.cuda.synchronize()
+ assert torch.equal(got, want), (top, n)
+ assert syncs == 0, (top, n)
+ res.release_all(residency)
+ assert torch.equal(net(x), want)
+
+
+def test_without_the_prefetch_every_streamed_group_waits_on_the_host(monkeypatch):
+ """Control: the H3 layout as main builds it synchronizes the host for every streamed group."""
+ _cuda()
+ torch.manual_seed(0)
+ net = _DiT().eval()
+ monkeypatch.setenv(res.H3_STREAM_PREFETCH_ENV, "0")
+ assert _h3_streamed(net, monkeypatch) == 0
+ x = torch.randn(4, 64, device = "cuda")
+ with torch.no_grad():
+ net(x)
+ _, syncs = _count_syncs(lambda: net(x))
+ assert syncs >= len(net.transformer_blocks)
+
+
+def test_top_pin_kill_switch_keeps_diffusers_top_group(monkeypatch):
+ """UNSLOTH_H3_TOP_GROUP_PIN=0: the generic top-group adoption must not pin it behind the H3 switch's back."""
+ _cuda()
+ from core.inference.diffusion_offload_prefetch import module_prefetcher
+
+ import core.inference.diffusion_offload_prefetch as op
+
+ monkeypatch.setenv(res.H3_TOP_GROUP_PIN_ENV, "0")
+ seen = []
+ adopt = op._adopt_top_group
+
+ def _spy(module, *a, **k):
+ top_group, _ = res.h3_offload_groups(module)
+ # H3's top group is torchao: adoption takes it whenever it is unstreamed and its onload_ is not replaced
+ seen.append(top_group.stream is None and "onload_" not in top_group.__dict__)
+ return adopt(module, *a, **k)
+
+ monkeypatch.setattr(op, "_adopt_top_group", _spy)
+ torch.manual_seed(0)
+ net = _DiT().eval()
+ ref = copy.deepcopy(net).cuda()
+ assert _h3_streamed(net, monkeypatch, pin_top = False) == len(net.transformer_blocks)
+ top, _ = res.h3_offload_groups(net)
+ pf = module_prefetcher(net)
+ assert seen == [False]
+ assert top.stream is None and not pf.owns(top)
+ x = torch.randn(4, 64, device = "cuda")
+ with torch.inference_mode():
+ want = ref(x)
+ for _ in range(2):
+ got = net(x)
+ torch.cuda.synchronize()
+ assert torch.equal(got, want)
+
+
+def test_capped_pinned_host_prefetches_one_ahead(monkeypatch):
+ """Windows / WSL2 pin each in-flight group on the fly under a ~1 GiB cap: one ahead, as diffusers holds."""
+ _cuda()
+ import core.inference.diffusion_memory as dm
+ from core.inference.diffusion_offload_prefetch import module_prefetcher
+
+ monkeypatch.setattr(dm, "_pinned_memory_capped", lambda: True)
+ net = _DiT().eval()
+ assert _h3_streamed(net, monkeypatch) == 1 + len(net.transformer_blocks)
+ assert module_prefetcher(net).depth == 1