Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

Studio: keep MiniMax-H3's streamed top-level group on the card between denoise steps (#15796)

Daniel Hanc4c785c5fe8c
diff --git a/studio/backend/core/inference/diffusion_offload_prefetch.py b/studio/backend/core/inference/diffusion_offload_prefetch.py
index c6ca53ee304..96e4e6e3fc0 100644
--- a/studio/backend/core/inference/diffusion_offload_prefetch.py
+++ b/studio/backend/core/inference/diffusion_offload_prefetch.py
@@ -508,6 +508,12 @@ class GroupPrefetcher:
carried = getattr(self, "carried", None)
if carried:
carried.discard(gid)
+ if gid in self.ready and self._keeps_for_next(gid):
+ # the next forward runs it first: re-uploading the same weights would only wait on the copy
+ self.__dict__.setdefault("carried", set()).add(gid)
+ self.stats["kept"] = self.stats.get("kept", 0) + 1
+ self._fill()
+ return
if gid in self.ready:
self._release(group, self.ready.pop(gid))
else:
@@ -586,6 +592,19 @@ class GroupPrefetcher:
if ahead < self.depth and self.pos >= len(self.order) and self.may_carry():
self._carry(ahead)
+ def _keeps_for_next(self, gid: int) -> bool:
+ """``keep_first``: keep the top-level group on device; the next forward needs it first."""
+ order = self.order
+ return bool(
+ getattr(self, "keep_first", False)
+ and self.active
+ and self.on_order
+ and order
+ and self.pos >= len(order)
+ and order[0] == gid
+ and self.may_carry()
+ )
+
def carry_usable(self) -> bool:
return bool(
not getattr(self, "interleaved", False)
@@ -1182,6 +1201,18 @@ def cross_forward_render(pipe: Any, enabled: bool = True) -> Iterator[list]:
pass
+def end_cross_forward(pipe: Any) -> int:
+ """Disarm ``pipe``'s prefetchers, drop their carry; returns carried copies read so far."""
+ hits = 0
+ for pf in _pipe_prefetchers(pipe):
+ try:
+ _disarm(pf)
+ hits += int(pf.stats.get("carried_hits", 0))
+ except Exception: # noqa: BLE001 - leave the render's own error to surface
+ pass
+ return hits
+
+
_UNKNOWN = object()
diff --git a/studio/backend/core/inference/video.py b/studio/backend/core/inference/video.py
index 26db62c35d8..66cd00d610a 100644
--- a/studio/backend/core/inference/video.py
+++ b/studio/backend/core/inference/video.py
@@ -232,7 +232,11 @@ from .diffusion import (
from .diffusion_native_quant import native_quant_reason
from .diffusion_batched import is_oom_error
from .diffusion_block_graph import compile_pipe_below_offload_hooks
-from .diffusion_offload_prefetch import cross_forward_render, cross_forward_step
+from .diffusion_offload_prefetch import (
+ cross_forward_render,
+ cross_forward_step,
+ end_cross_forward,
+)
from .diffusion_precision import (
effective_te_quant,
normalize_te_quant,
@@ -3635,6 +3639,28 @@ def _video_seed_stays_resident(
return plan_keeps_transformer_resident(planned)
+def _h3_carry_by_scheduler(fam: Any, state: Any, has_step_callback: bool) -> bool:
+ """H3 modular has no step callback: its scheduler.step wrapper drives a streamed denoiser's cross-forward carry."""
+ from .video_minimax_h3_residency import h3_stream_carry_enabled
+ return (
+ not has_step_callback
+ and getattr(fam, "name", None) == "minimax-h3"
+ and bool(getattr(state, "denoiser_streamed", False))
+ and h3_stream_carry_enabled()
+ )
+
+
+def _scheduler_step_carry(pipe: Any, steps: int, on_done: Any, cancelled: Any) -> Any:
+ """``on_done`` after each scheduler.step, then the cross-forward step (forward k's step is index k - 1)."""
+
+ def done(enqueued: int) -> None:
+ on_done(enqueued)
+ if not cancelled():
+ cross_forward_step(pipe, enqueued - 1, total = steps)
+
+ return done
+
+
def _carried_call(pipe: Any, kwargs: dict, has_step_callback: bool) -> Any:
"""Armed on the render thread (the interleave check is per thread); needs the step callback to stop before decode."""
with cross_forward_render(pipe, enabled = has_step_callback):
@@ -12394,6 +12420,11 @@ class VideoBackend:
time.sleep(_BOUNDARY_MARK_RETRY_SECONDS)
if not marked:
_enter_decode_phase()
+ if carry_by_scheduler:
+ logger.info(
+ "video.h3_stream_carry: %d carried copies read across forwards so far",
+ end_cross_forward(pipe),
+ )
def _pump() -> None:
"""One poll, inside the capture hold-off. Advances the step from the GPU, and
@@ -12435,6 +12466,7 @@ class VideoBackend:
has_step_callback = "callback_on_step_end" in call_params
if has_step_callback:
kwargs["callback_on_step_end"] = _on_step
+ carry_by_scheduler = _h3_carry_by_scheduler(fam, state, has_step_callback)
@contextlib.contextmanager
def progress_ctx():
@@ -12444,7 +12476,13 @@ class VideoBackend:
# HunyuanVideo-1.5 and H3 modular have no step callback:
# wrap scheduler.step for progress + cancel.
stack.enter_context(
- _scheduler_step_progress(pipe, _on_scheduler_step_cancel, _tick)
+ _scheduler_step_progress(
+ pipe,
+ _on_scheduler_step_cancel,
+ _scheduler_step_carry(pipe, steps, _tick, cancel.is_set)
+ if carry_by_scheduler
+ else _tick,
+ )
)
stack.enter_context(scheduler_step_preview(pipe, previewer))
stack.enter_context(_denoise_phase(pipe, _enter_denoise))
@@ -12551,7 +12589,11 @@ class VideoBackend:
"video",
ltx25_render_call
if ltx25_render_call is not None
- else (lambda: _carried_call(pipe, kwargs, has_step_callback)),
+ else (
+ lambda: _carried_call(
+ pipe, kwargs, has_step_callback or carry_by_scheduler
+ )
+ ),
)
except _VideoGenerationCancelled:
# Unwinding by exception skips maybe_free_model_hooks(); offloaded modules would stay on GPU.
diff --git a/studio/backend/core/inference/video_minimax_h3_residency.py b/studio/backend/core/inference/video_minimax_h3_residency.py
index 3a4d7ebff41..2448ac22ea9 100644
--- a/studio/backend/core/inference/video_minimax_h3_residency.py
+++ b/studio/backend/core/inference/video_minimax_h3_residency.py
@@ -768,6 +768,17 @@ def defer_streamed_block_pins(
H3_STREAM_PREFETCH_ENV = "UNSLOTH_H3_STREAM_PREFETCH"
+H3_STREAM_CARRY_ENV = "UNSLOTH_H3_STREAM_CARRY"
+
+
+def h3_stream_carry_enabled() -> bool:
+ """Cross-forward carry for H3's scheduler.step-driven loop; ``UNSLOTH_H3_STREAM_CARRY=0`` off."""
+ return str(os.environ.get(H3_STREAM_CARRY_ENV, "")).strip().lower() not in (
+ "0",
+ "off",
+ "false",
+ "no",
+ )
def h3_stream_prefetch_depth(block_bytes: list, default_depth: int) -> int:
@@ -847,6 +858,8 @@ def install_h3_stream_prefetch(
group_payload_bytes(top) if top is not None else 0,
[group_payload_bytes(g) for g in blocks],
)
+ # a streamed top-level group runs for the whole forward, so only keeping it hides its upload
+ prefetcher.keep_first = h3_stream_carry_enabled()
# end() puts groups copied ahead but never run back on the host copy; kick() queues copies
for name in ("end", "kick"):
setattr(prefetcher, name, _outside_inference_mode(getattr(prefetcher, name)))
diff --git a/studio/backend/tests/test_h3_stream_carry.py b/studio/backend/tests/test_h3_stream_carry.py
new file mode 100644
index 00000000000..9ab88fb3b16
--- /dev/null
+++ b/studio/backend/tests/test_h3_stream_carry.py
@@ -0,0 +1,223 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
+
+"""MiniMax-H3 (no step callback) drives the cross-forward carry from its scheduler.step wrapper."""
+
+from __future__ import annotations
+
+import os
+import sys
+import types
+
+import pytest
+
+import core.inference.diffusion_offload_prefetch as op
+import core.inference.video as video
+from core.inference.video_minimax_h3_residency import H3_STREAM_CARRY_ENV, h3_stream_carry_enabled
+
+_TESTS_DIR = os.path.dirname(os.path.abspath(__file__))
+if _TESTS_DIR not in sys.path:
+ sys.path.insert(0, _TESTS_DIR)
+
+from test_diffusion_offload_cross_forward import ( # noqa: E402
+ _clean_env, # noqa: F401 - autouse fixture
+ _copies,
+ _cpu_prefetcher,
+ _forward,
+)
+
+
[email protected](autouse = True)
+def _no_carry_env(monkeypatch):
+ monkeypatch.delenv(H3_STREAM_CARRY_ENV, raising = False)
+
+
+def _h3_pipe(pf):
+ pipe = types.SimpleNamespace(
+ components = {"transformer": types.SimpleNamespace(**{op.PREFETCHER_ATTR: pf})},
+ _interrupt = False,
+ )
+ pipe.scheduler = types.SimpleNamespace(step = lambda *a, **k: "stepped")
+ return pipe
+
+
+def _h3_render(
+ pf,
+ groups,
+ steps,
+ calls = None,
+ cancelled = lambda: False,
+):
+ """H3's loop: ``steps`` forwards, scheduler.step after all but the last (H3 makes 29 calls for 30 steps)."""
+ pipe = _h3_pipe(pf)
+ calls = steps - 1 if calls is None else calls
+ ticks = []
+ done = video._scheduler_step_carry(pipe, steps, ticks.append, cancelled)
+ with op.cross_forward_render(pipe):
+ with video._scheduler_step_progress(pipe, lambda n: None, done):
+ for i in range(steps):
+ _forward(pf, groups)
+ if i < calls:
+ pipe.scheduler.step()
+ decode_view = (dict(pf.ready), set(pf.carried), pf.inflight_bytes)
+ hits = op.end_cross_forward(pipe)
+ return ticks, decode_view, hits
+
+
+def test_forwards_after_the_first_read_the_carry_and_nothing_reaches_the_decode():
+ pf, groups, log = _cpu_prefetcher("tabcde", depth = 2)
+ _forward(pf, groups)
+ log.clear()
+ ticks, (ready, carried, inflight), hits = _h3_render(pf, groups, steps = 6)
+ assert ticks == [1, 2, 3, 4, 5]
+ assert pf.stats["carried_hits"] == 10 and hits == 10
+ assert not ready and not carried and inflight == 0
+ assert pf.stats.get("carried_dropped", 0) == 0
+ assert _copies(log) == list("tabcde") * 6
+
+
+def test_one_call_per_step_also_ends_clean():
+ pf, groups, _ = _cpu_prefetcher("tabcde", depth = 2)
+ _forward(pf, groups)
+ _, (ready, carried, inflight), _ = _h3_render(pf, groups, steps = 5, calls = 5)
+ assert not ready and not carried and inflight == 0
+
+
+def test_too_few_step_calls_are_dropped_at_the_decode():
+ pf, groups, _ = _cpu_prefetcher("tabcde", depth = 2)
+ _forward(pf, groups)
+ pipe = _h3_pipe(pf)
+ done = video._scheduler_step_carry(pipe, 10, lambda n: None, lambda: False)
+ with op.cross_forward_render(pipe):
+ with video._scheduler_step_progress(pipe, lambda n: None, done):
+ for i in range(3):
+ _forward(pf, groups)
+ pipe.scheduler.step()
+ assert pf.carried # a render shorter than its step count would leave these
+ op.end_cross_forward(pipe)
+ assert not pf.ready and not pf.carried and pf.inflight_bytes == 0 and not pf.armed
+
+
+def test_cancel_skips_the_carry_step():
+ pf, groups, _ = _cpu_prefetcher("tabcde", depth = 2)
+ _forward(pf, groups)
+ pipe = _h3_pipe(pf)
+ seen = []
+ done = video._scheduler_step_carry(pipe, 10, seen.append, lambda: True)
+ with op.cross_forward_render(pipe):
+ _forward(pf, groups)
+ done(1)
+ assert (
+ seen == [1] and pf.armed
+ ) # the cancel raises before the next step; render exit disarms
+ assert not pf.armed and not pf.carried
+
+
+ "fam_name, streamed, has_cb, env, expected",
+ [
+ ("minimax-h3", True, False, None, True),
+ ("minimax-h3", True, False, "0", False),
+ ("minimax-h3", True, True, None, False),
+ ("minimax-h3", False, False, None, False),
+ ("hunyuanvideo-1.5", True, False, None, False),
+ ],
+)
+def test_gate(monkeypatch, fam_name, streamed, has_cb, env, expected):
+ if env is not None:
+ monkeypatch.setenv(H3_STREAM_CARRY_ENV, env)
+ fam = types.SimpleNamespace(name = fam_name)
+ state = types.SimpleNamespace(denoiser_streamed = "stream" if streamed else False)
+ assert video._h3_carry_by_scheduler(fam, state, has_cb) is expected
+
+
+ "value, on", [(None, True), ("1", True), ("0", False), ("off", False), ("no", False)]
+)
+def test_kill_switch(monkeypatch, value, on):
+ if value is not None:
+ monkeypatch.setenv(H3_STREAM_CARRY_ENV, value)
+ assert h3_stream_carry_enabled() is on
+
+
+def test_kill_switch_render_carries_nothing(monkeypatch):
+ monkeypatch.setenv(H3_STREAM_CARRY_ENV, "0")
+ pf, groups, log = _cpu_prefetcher("tabcde", depth = 2)
+ _forward(pf, groups)
+ log.clear()
+ pipe = _h3_pipe(pf)
+ gate = video._h3_carry_by_scheduler(
+ types.SimpleNamespace(name = "minimax-h3"),
+ types.SimpleNamespace(denoiser_streamed = "stream"),
+ False,
+ )
+ with op.cross_forward_render(pipe, enabled = gate):
+ for _ in range(4):
+ _forward(pf, groups)
+ assert not gate and not pf.stats.get("carried") and _copies(log) == list("tabcde") * 4
+
+
+def _top_forward(pf, top, blocks):
+ pf.begin()
+ pf.onload(top)
+ for g in blocks:
+ pf.onload(g)
+ pf.offload(g)
+ pf.offload(top)
+ pf.end()
+
+
+def _top_render(
+ keep,
+ steps = 6,
+ nbytes = 10,
+):
+ pf, groups, log = _cpu_prefetcher("tabcde", nbytes = nbytes, depth = 2)
+ pf.nbytes[id(groups[0])] = 2 * nbytes
+ pf.window = 2 * nbytes + 2 * nbytes
+ pf.keep_first = keep
+ top, blocks = groups[0], groups[1:]
+ _top_forward(pf, top, blocks)
+ log.clear()
+ peak_before = pf.peak_inflight_bytes
+ pipe = _h3_pipe(pf)
+ done = video._scheduler_step_carry(pipe, steps, lambda n: None, lambda: False)
+ with op.cross_forward_render(pipe):
+ with video._scheduler_step_progress(pipe, lambda n: None, done):
+ for i in range(steps):
+ _top_forward(pf, top, blocks)
+ if i < steps - 1:
+ pipe.scheduler.step()
+ decode_view = (dict(pf.ready), set(pf.carried), pf.inflight_bytes)
+ op.end_cross_forward(pipe)
+ return pf, log, decode_view, peak_before
+
+
+def test_streamed_top_group_stays_for_the_next_forward():
+ pf, log, (ready, carried, inflight), peak_before = _top_render(keep = True)
+ copies = _copies(log)
+ assert copies.count("t") == 1 and all(copies.count(b) == 6 for b in "abcde")
+ assert pf.stats["kept"] == 5
+ assert not ready and not carried and inflight == 0
+ assert pf.peak_inflight_bytes <= max(peak_before, pf.window)
+
+
+def test_without_keep_the_top_group_reuploads_every_forward():
+ pf, log, (ready, carried, inflight), _ = _top_render(keep = False)
+ assert _copies(log).count("t") == 6 and not pf.stats.get("kept")
+ assert not ready and not carried and inflight == 0
+
+
+def test_kept_top_group_is_released_when_the_render_is_cut_short():
+ pf, groups, log = _cpu_prefetcher("tabcde", depth = 2)
+ pf.keep_first = True
+ top, blocks = groups[0], groups[1:]
+ _top_forward(pf, top, blocks)
+ pipe = _h3_pipe(pf)
+ with pytest.raises(RuntimeError):
+ with op.cross_forward_render(pipe):
+ pf.armed = True
+ _top_forward(pf, top, blocks)
+ assert id(top) in pf.ready and id(top) in pf.carried
+ raise RuntimeError("cancelled")
+ assert not pf.ready and not pf.carried and pf.inflight_bytes == 0