Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

Studio video: pin MiniMax-H3's lazily streamed blocks in place so the stream prefetch engages (#15118)

Daniel Hand32a267f6439
diff --git a/studio/backend/core/inference/video.py b/studio/backend/core/inference/video.py
index f76cc5a1365..f982a6f92ad 100644
--- a/studio/backend/core/inference/video.py
+++ b/studio/backend/core/inference/video.py
@@ -10867,6 +10867,17 @@ class VideoBackend:
pin_streamed_top_level_group(denoiser, logger = logger)
except Exception as exc: # noqa: BLE001 -- a speed-up only
logger.warning("video.h3_top_group: %s", exc)
+ if denoiser_streamed == "stream_lazy":
+ from .video_minimax_h3_residency import pin_streamed_block_groups
+ try:
+ done, total = pin_streamed_block_groups(denoiser, logger = logger)
+ except Exception as exc: # noqa: BLE001 -- a speed-up only
+ done, total = 0, 0
+ logger.warning("video.h3_block_pin: %s", exc)
+ if total and done == total:
+ # every group now holds one pinned copy: the prefetch below engages
+ denoiser_streamed = "stream"
+ denoiser_single_host_copy = True
# after the top-level pin (it becomes one of the prefetched groups), before the residency fit
from .video_minimax_h3_residency import install_h3_stream_prefetch
diff --git a/studio/backend/core/inference/video_minimax_h3_residency.py b/studio/backend/core/inference/video_minimax_h3_residency.py
index 7b74bf9764f..aa00a3334be 100644
--- a/studio/backend/core/inference/video_minimax_h3_residency.py
+++ b/studio/backend/core/inference/video_minimax_h3_residency.py
@@ -463,6 +463,71 @@ def pin_streamed_top_level_group(transformer: Any, logger: Any = None) -> bool:
return True
+H3_BLOCK_PIN_ENV = "UNSLOTH_H3_STREAM_BLOCK_PIN"
+
+
+def pin_streamed_block_groups(transformer: Any, logger: Any = None) -> tuple[int, int]:
+ """Pin a lazily streamed denoiser's block groups in place, group by group, while the pin budget holds.
+
+ The up-front plan prices a pin as extra host RAM, so 12 / 16 GB hosts stream H3 lazily: every onload re-pins
+ behind a host sync and the stream prefetch stays off. The arena re-points each weight at its slab, freeing the
+ pageable copy; the budget is re-read per group, so a surviving pageable copy stops the pin.
+ Returns (groups pinned, block groups). ``UNSLOTH_H3_STREAM_BLOCK_PIN=0`` off."""
+ if str(os.environ.get(H3_BLOCK_PIN_ENV, "")).strip().lower() in ("0", "off", "false", "no"):
+ return 0, 0
+ _top, blocks = h3_offload_groups(transformer)
+ lazy = [
+ g
+ for g in blocks
+ if getattr(g, "low_cpu_mem_usage", False)
+ and getattr(g, "stream", None) is not None
+ and not getattr(g, "offload_to_disk_path", None)
+ and callable(getattr(g, "_init_cpu_param_dict", None))
+ and not is_resident(g)
+ ]
+ if not lazy:
+ return 0, len(blocks)
+ from .diffusion_memory import _pinned_memory_capped
+ from .diffusion_pinned_arena import pin_arena_enabled, pinned_arena_for_group_offload
+ from .video_minimax_h3_te import h3_te_pin_allowed
+
+ # Per-tensor pin_memory() keeps the pageable copy until the first offload; capped hosts pin each onload anyway.
+ if not pin_arena_enabled() or _pinned_memory_capped():
+ return 0, len(blocks)
+ pinned = 0
+ with pinned_arena_for_group_offload() as arena:
+ if arena is None:
+ return 0, len(blocks)
+ for group in lazy:
+ tensors = _group_tensors(group)
+ if any(getattr(t, "device", None) is None or t.device.type != "cpu" for t in tensors):
+ break
+ if not h3_te_pin_allowed(group_payload_bytes(group)):
+ break
+ saved = (group.low_cpu_mem_usage, group.cpu_param_dict)
+ try:
+ group.low_cpu_mem_usage = False
+ group.cpu_param_dict = group._init_cpu_param_dict()
+ except Exception as exc: # noqa: BLE001 -- this group and the rest stay lazily pinned
+ group.low_cpu_mem_usage, group.cpu_param_dict = saved
+ if logger is not None:
+ logger.warning("video.h3_block_pin: stopped at a group: %s", exc)
+ break
+ pinned += 1
+ payload, reserved = arena.payload_bytes, arena.reserved_bytes
+ if pinned == len(blocks) and len(lazy) == len(blocks):
+ transformer._unsloth_pin_arena_bytes = (payload, reserved)
+ if logger is not None:
+ logger.info(
+ "video.h3_block_pin: %d of %d streamed block groups pinned in place (%.2f GB in %.2f GB of slabs)",
+ pinned,
+ len(blocks),
+ payload / 1e9,
+ reserved / 1e9,
+ )
+ return pinned, len(blocks)
+
+
H3_STREAM_PREFETCH_ENV = "UNSLOTH_H3_STREAM_PREFETCH"
diff --git a/studio/backend/tests/test_h3_stream_block_pin.py b/studio/backend/tests/test_h3_stream_block_pin.py
new file mode 100644
index 00000000000..3ac98cf5b84
--- /dev/null
+++ b/studio/backend/tests/test_h3_stream_block_pin.py
@@ -0,0 +1,193 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
+
+"""A lazily streamed H3 denoiser (pin budget refused the up-front pin) gets its block groups pinned in place, group by
+group, while the budget holds; the rest keep diffusers' per-onload pin."""
+
+from __future__ import annotations
+
+import copy
+
+import pytest
+
+torch = pytest.importorskip("torch")
+pytest.importorskip("diffusers.hooks")
+
+import core.inference.diffusion_memory as dm # noqa: E402
+import core.inference.video_minimax_h3_residency as res # noqa: E402
+import core.inference.video_minimax_h3_te as te # noqa: E402
+
+
[email protected](autouse = True)
+def _clean_env(monkeypatch):
+ for name in (
+ res.H3_BLOCK_PIN_ENV,
+ res.H3_STREAM_PREFETCH_ENV,
+ "UNSLOTH_DIFFUSION_PIN_ARENA",
+ "UNSLOTH_DIFFUSION_GROUP_OFFLOAD_PIN",
+ "UNSLOTH_H3_TOP_GROUP_PIN",
+ ):
+ monkeypatch.delenv(name, raising = False)
+ monkeypatch.setattr(dm, "_pinned_memory_capped", lambda: False)
+
+
+class _FakeGroup:
+ def __init__(self, fail = False):
+ self.low_cpu_mem_usage = True
+ self.stream = object()
+ self.offload_to_disk_path = None
+ self.modules = [torch.nn.Linear(4, 4)]
+ self.parameters = []
+ self.buffers = []
+ self.cpu_param_dict = {"lazy": True}
+ self.fail = fail
+ self.inits = 0
+
+ def _init_cpu_param_dict(self):
+ self.inits += 1
+ if self.fail:
+ raise RuntimeError("boom")
+ return {"pinned": True}
+
+
+def _fake(monkeypatch, groups, allow):
+ net = torch.nn.Module()
+ monkeypatch.setattr(res, "h3_offload_groups", lambda _t: (None, groups))
+ calls = iter(allow)
+ monkeypatch.setattr(te, "h3_te_pin_allowed", lambda *_a, **_k: next(calls, False))
+ return net
+
+
+def test_pins_every_group_the_budget_allows_and_marks_one_host_copy(monkeypatch):
+ groups = [_FakeGroup() for _ in range(4)]
+ net = _fake(monkeypatch, groups, [True] * 4)
+ assert res.pin_streamed_block_groups(net) == (4, 4)
+ assert all(not g.low_cpu_mem_usage and g.cpu_param_dict == {"pinned": True} for g in groups)
+ assert getattr(net, "_unsloth_pin_arena_bytes", None) is not None
+
+
+def test_budget_reread_per_group_stops_partway(monkeypatch):
+ groups = [_FakeGroup() for _ in range(4)]
+ net = _fake(monkeypatch, groups, [True, True, False])
+ assert res.pin_streamed_block_groups(net) == (2, 4)
+ assert [g.low_cpu_mem_usage for g in groups] == [False, False, True, True]
+ assert groups[2].inits == 0 and groups[2].cpu_param_dict == {"lazy": True}
+ # partly pinned: still lazy, so the caller keeps stream_lazy and the prefetch stays off
+ assert getattr(net, "_unsloth_pin_arena_bytes", None) is None
+
+
+def test_a_failing_group_is_restored_and_stops_the_pin(monkeypatch):
+ groups = [_FakeGroup(), _FakeGroup(fail = True), _FakeGroup()]
+ net = _fake(monkeypatch, groups, [True] * 3)
+ assert res.pin_streamed_block_groups(net) == (1, 3)
+ assert groups[1].low_cpu_mem_usage is True and groups[1].cpu_param_dict == {"lazy": True}
+ assert groups[2].inits == 0
+
+
+ "env, capped",
+ [
+ ({"UNSLOTH_H3_STREAM_BLOCK_PIN": "0"}, False),
+ ({"UNSLOTH_DIFFUSION_PIN_ARENA": "0"}, False),
+ ({}, True),
+ ],
+)
+def test_kill_switch_no_arena_and_capped_hosts_leave_it_lazy(monkeypatch, env, capped):
+ for key, value in env.items():
+ monkeypatch.setenv(key, value)
+ monkeypatch.setattr(dm, "_pinned_memory_capped", lambda: capped)
+ groups = [_FakeGroup() for _ in range(3)]
+ net = _fake(monkeypatch, groups, [True] * 3)
+ assert res.pin_streamed_block_groups(net)[0] == 0
+ assert all(g.low_cpu_mem_usage and g.inits == 0 for g in groups)
+
+
+def test_pin_refused_by_the_host_gate_pins_nothing(monkeypatch):
+ """h3_te_pin_allowed is the gate: integrated GPUs (host_pinning_allowed) and the group-offload pin switch say no."""
+ import core.inference.host_pinning as hp
+
+ monkeypatch.setattr(hp, "host_pinning_allowed", lambda *_a, **_k: False)
+ groups = [_FakeGroup() for _ in range(2)]
+ net = torch.nn.Module()
+ monkeypatch.setattr(res, "h3_offload_groups", lambda _t: (None, groups))
+ assert res.pin_streamed_block_groups(net) == (0, 2)
+ assert all(g.low_cpu_mem_usage for g in groups)
+
+
+def test_already_pinned_groups_are_left_alone(monkeypatch):
+ groups = [_FakeGroup() for _ in range(2)]
+ for g in groups:
+ g.low_cpu_mem_usage = False
+ net = _fake(monkeypatch, groups, [True] * 2)
+ assert res.pin_streamed_block_groups(net) == (0, 2)
+ assert all(g.inits == 0 for g in groups)
+
+
+class _Block(torch.nn.Module):
+ def __init__(self, width):
+ super().__init__()
+ self.lin = torch.nn.Linear(width, width)
+
+ def forward(self, x):
+ return torch.nn.functional.gelu(self.lin(x))
+
+
+class _DiT(torch.nn.Module):
+ def __init__(
+ self,
+ width = 512,
+ blocks = 6,
+ ):
+ super().__init__()
+ self.proj_in = torch.nn.Linear(64, width)
+ self.transformer_blocks = torch.nn.ModuleList(_Block(width) for _ in range(blocks))
+ self.proj_out = torch.nn.Linear(width, 64)
+
+ def forward(self, x):
+ x = self.proj_in(x)
+ for block in self.transformer_blocks:
+ x = block(x)
+ return self.proj_out(x)
+
+
+def test_lazy_stream_pinned_in_place_prefetches_and_is_bit_identical(monkeypatch):
+ if not torch.cuda.is_available():
+ pytest.skip("needs CUDA: diffusers stream group offload")
+ import inspect
+
+ from diffusers.hooks import apply_group_offloading
+
+ from core.inference.diffusion_offload_prefetch import module_prefetcher
+ from core.inference.diffusion_prequant import _move_groups_outside_inference_mode
+
+ monkeypatch.setattr(te, "h3_te_pin_allowed", lambda *_a, **_k: True)
+ params = inspect.signature(apply_group_offloading).parameters
+ if "record_stream" not in params or "low_cpu_mem_usage" not in params:
+ pytest.skip("diffusers without record_stream / low_cpu_mem_usage")
+ torch.manual_seed(0)
+ net = _DiT().eval()
+ ref = copy.deepcopy(net).cuda()
+ apply_group_offloading(
+ net,
+ onload_device = torch.device("cuda"),
+ offload_device = torch.device("cpu"),
+ offload_type = "block_level",
+ num_blocks_per_group = 1,
+ use_stream = True,
+ record_stream = True,
+ non_blocking = True,
+ low_cpu_mem_usage = True,
+ )
+ _move_groups_outside_inference_mode(net)
+ assert res.pin_streamed_top_level_group(net)
+ assert res.pin_streamed_block_groups(net) == (6, 6)
+ # one host copy: the weights read the pinned slab views themselves
+ for block in net.transformer_blocks:
+ assert block.lin.weight.device.type == "cpu" and block.lin.weight.is_pinned()
+ assert res.install_h3_stream_prefetch(net, "cuda") == 1 + len(net.transformer_blocks)
+ assert module_prefetcher(net) is not None
+ x = torch.randn(8, 64, device = "cuda")
+ with torch.inference_mode():
+ want = ref(x)
+ for _ in range(3):
+ assert torch.equal(net(x), want)