Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
Studio: keep MiniMax-H3's streamed top-level group on the card between denoise steps (#15796)
Daniel Hanc4c785c5fe8c
diff --git a/studio/backend/core/inference/diffusion_offload_prefetch.py b/studio/backend/core/inference/diffusion_offload_prefetch.pyindex c6ca53ee304..96e4e6e3fc0 100644--- a/studio/backend/core/inference/diffusion_offload_prefetch.py+++ b/studio/backend/core/inference/diffusion_offload_prefetch.py@@ -508,6 +508,12 @@ class GroupPrefetcher:carried = getattr(self, "carried", None)if carried:carried.discard(gid)+ if gid in self.ready and self._keeps_for_next(gid):+ # the next forward runs it first: re-uploading the same weights would only wait on the copy+ self.__dict__.setdefault("carried", set()).add(gid)+ self.stats["kept"] = self.stats.get("kept", 0) + 1+ self._fill()+ returnif gid in self.ready:self._release(group, self.ready.pop(gid))else:@@ -586,6 +592,19 @@ class GroupPrefetcher:if ahead < self.depth and self.pos >= len(self.order) and self.may_carry():self._carry(ahead)+ def _keeps_for_next(self, gid: int) -> bool:+ """``keep_first``: keep the top-level group on device; the next forward needs it first."""+ order = self.order+ return bool(+ getattr(self, "keep_first", False)+ and self.active+ and self.on_order+ and order+ and self.pos >= len(order)+ and order[0] == gid+ and self.may_carry()+ )+def carry_usable(self) -> bool:return bool(not getattr(self, "interleaved", False)@@ -1182,6 +1201,18 @@ def cross_forward_render(pipe: Any, enabled: bool = True) -> Iterator[list]:pass+def end_cross_forward(pipe: Any) -> int:+ """Disarm ``pipe``'s prefetchers, drop their carry; returns carried copies read so far."""+ hits = 0+ for pf in _pipe_prefetchers(pipe):+ try:+ _disarm(pf)+ hits += int(pf.stats.get("carried_hits", 0))+ except Exception: # noqa: BLE001 - leave the render's own error to surface+ pass+ return hits++_UNKNOWN = object()diff --git a/studio/backend/core/inference/video.py b/studio/backend/core/inference/video.pyindex 26db62c35d8..66cd00d610a 100644--- a/studio/backend/core/inference/video.py+++ b/studio/backend/core/inference/video.py@@ -232,7 +232,11 @@ from .diffusion import (from .diffusion_native_quant import native_quant_reasonfrom .diffusion_batched import is_oom_errorfrom .diffusion_block_graph import compile_pipe_below_offload_hooks-from .diffusion_offload_prefetch import cross_forward_render, cross_forward_step+from .diffusion_offload_prefetch import (+ cross_forward_render,+ cross_forward_step,+ end_cross_forward,+)from .diffusion_precision import (effective_te_quant,normalize_te_quant,@@ -3635,6 +3639,28 @@ def _video_seed_stays_resident(return plan_keeps_transformer_resident(planned)+def _h3_carry_by_scheduler(fam: Any, state: Any, has_step_callback: bool) -> bool:+ """H3 modular has no step callback: its scheduler.step wrapper drives a streamed denoiser's cross-forward carry."""+ from .video_minimax_h3_residency import h3_stream_carry_enabled+ return (+ not has_step_callback+ and getattr(fam, "name", None) == "minimax-h3"+ and bool(getattr(state, "denoiser_streamed", False))+ and h3_stream_carry_enabled()+ )+++def _scheduler_step_carry(pipe: Any, steps: int, on_done: Any, cancelled: Any) -> Any:+ """``on_done`` after each scheduler.step, then the cross-forward step (forward k's step is index k - 1)."""++ def done(enqueued: int) -> None:+ on_done(enqueued)+ if not cancelled():+ cross_forward_step(pipe, enqueued - 1, total = steps)++ return done++def _carried_call(pipe: Any, kwargs: dict, has_step_callback: bool) -> Any:"""Armed on the render thread (the interleave check is per thread); needs the step callback to stop before decode."""with cross_forward_render(pipe, enabled = has_step_callback):@@ -12394,6 +12420,11 @@ class VideoBackend:time.sleep(_BOUNDARY_MARK_RETRY_SECONDS)if not marked:_enter_decode_phase()+ if carry_by_scheduler:+ logger.info(+ "video.h3_stream_carry: %d carried copies read across forwards so far",+ end_cross_forward(pipe),+ )def _pump() -> None:"""One poll, inside the capture hold-off. Advances the step from the GPU, and@@ -12435,6 +12466,7 @@ class VideoBackend:has_step_callback = "callback_on_step_end" in call_paramsif has_step_callback:kwargs["callback_on_step_end"] = _on_step+ carry_by_scheduler = _h3_carry_by_scheduler(fam, state, has_step_callback)@contextlib.contextmanagerdef progress_ctx():@@ -12444,7 +12476,13 @@ class VideoBackend:# HunyuanVideo-1.5 and H3 modular have no step callback:# wrap scheduler.step for progress + cancel.stack.enter_context(- _scheduler_step_progress(pipe, _on_scheduler_step_cancel, _tick)+ _scheduler_step_progress(+ pipe,+ _on_scheduler_step_cancel,+ _scheduler_step_carry(pipe, steps, _tick, cancel.is_set)+ if carry_by_scheduler+ else _tick,+ ))stack.enter_context(scheduler_step_preview(pipe, previewer))stack.enter_context(_denoise_phase(pipe, _enter_denoise))@@ -12551,7 +12589,11 @@ class VideoBackend:"video",ltx25_render_callif ltx25_render_call is not None- else (lambda: _carried_call(pipe, kwargs, has_step_callback)),+ else (+ lambda: _carried_call(+ pipe, kwargs, has_step_callback or carry_by_scheduler+ )+ ),)except _VideoGenerationCancelled:# Unwinding by exception skips maybe_free_model_hooks(); offloaded modules would stay on GPU.diff --git a/studio/backend/core/inference/video_minimax_h3_residency.py b/studio/backend/core/inference/video_minimax_h3_residency.pyindex 3a4d7ebff41..2448ac22ea9 100644--- a/studio/backend/core/inference/video_minimax_h3_residency.py+++ b/studio/backend/core/inference/video_minimax_h3_residency.py@@ -768,6 +768,17 @@ def defer_streamed_block_pins(H3_STREAM_PREFETCH_ENV = "UNSLOTH_H3_STREAM_PREFETCH"+H3_STREAM_CARRY_ENV = "UNSLOTH_H3_STREAM_CARRY"+++def h3_stream_carry_enabled() -> bool:+ """Cross-forward carry for H3's scheduler.step-driven loop; ``UNSLOTH_H3_STREAM_CARRY=0`` off."""+ return str(os.environ.get(H3_STREAM_CARRY_ENV, "")).strip().lower() not in (+ "0",+ "off",+ "false",+ "no",+ )def h3_stream_prefetch_depth(block_bytes: list, default_depth: int) -> int:@@ -847,6 +858,8 @@ def install_h3_stream_prefetch(group_payload_bytes(top) if top is not None else 0,[group_payload_bytes(g) for g in blocks],)+ # a streamed top-level group runs for the whole forward, so only keeping it hides its upload+ prefetcher.keep_first = h3_stream_carry_enabled()# end() puts groups copied ahead but never run back on the host copy; kick() queues copiesfor name in ("end", "kick"):setattr(prefetcher, name, _outside_inference_mode(getattr(prefetcher, name)))diff --git a/studio/backend/tests/test_h3_stream_carry.py b/studio/backend/tests/test_h3_stream_carry.pynew file mode 100644index 00000000000..9ab88fb3b16--- /dev/null+++ b/studio/backend/tests/test_h3_stream_carry.py@@ -0,0 +1,223 @@+# SPDX-License-Identifier: AGPL-3.0-only+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0++"""MiniMax-H3 (no step callback) drives the cross-forward carry from its scheduler.step wrapper."""++from __future__ import annotations++import os+import sys+import types++import pytest++import core.inference.diffusion_offload_prefetch as op+import core.inference.video as video+from core.inference.video_minimax_h3_residency import H3_STREAM_CARRY_ENV, h3_stream_carry_enabled++_TESTS_DIR = os.path.dirname(os.path.abspath(__file__))+if _TESTS_DIR not in sys.path:+ sys.path.insert(0, _TESTS_DIR)++from test_diffusion_offload_cross_forward import ( # noqa: E402+ _clean_env, # noqa: F401 - autouse fixture+ _copies,+ _cpu_prefetcher,+ _forward,+)++[email protected](autouse = True)+def _no_carry_env(monkeypatch):+ monkeypatch.delenv(H3_STREAM_CARRY_ENV, raising = False)+++def _h3_pipe(pf):+ pipe = types.SimpleNamespace(+ components = {"transformer": types.SimpleNamespace(**{op.PREFETCHER_ATTR: pf})},+ _interrupt = False,+ )+ pipe.scheduler = types.SimpleNamespace(step = lambda *a, **k: "stepped")+ return pipe+++def _h3_render(+ pf,+ groups,+ steps,+ calls = None,+ cancelled = lambda: False,+):+ """H3's loop: ``steps`` forwards, scheduler.step after all but the last (H3 makes 29 calls for 30 steps)."""+ pipe = _h3_pipe(pf)+ calls = steps - 1 if calls is None else calls+ ticks = []+ done = video._scheduler_step_carry(pipe, steps, ticks.append, cancelled)+ with op.cross_forward_render(pipe):+ with video._scheduler_step_progress(pipe, lambda n: None, done):+ for i in range(steps):+ _forward(pf, groups)+ if i < calls:+ pipe.scheduler.step()+ decode_view = (dict(pf.ready), set(pf.carried), pf.inflight_bytes)+ hits = op.end_cross_forward(pipe)+ return ticks, decode_view, hits+++def test_forwards_after_the_first_read_the_carry_and_nothing_reaches_the_decode():+ pf, groups, log = _cpu_prefetcher("tabcde", depth = 2)+ _forward(pf, groups)+ log.clear()+ ticks, (ready, carried, inflight), hits = _h3_render(pf, groups, steps = 6)+ assert ticks == [1, 2, 3, 4, 5]+ assert pf.stats["carried_hits"] == 10 and hits == 10+ assert not ready and not carried and inflight == 0+ assert pf.stats.get("carried_dropped", 0) == 0+ assert _copies(log) == list("tabcde") * 6+++def test_one_call_per_step_also_ends_clean():+ pf, groups, _ = _cpu_prefetcher("tabcde", depth = 2)+ _forward(pf, groups)+ _, (ready, carried, inflight), _ = _h3_render(pf, groups, steps = 5, calls = 5)+ assert not ready and not carried and inflight == 0+++def test_too_few_step_calls_are_dropped_at_the_decode():+ pf, groups, _ = _cpu_prefetcher("tabcde", depth = 2)+ _forward(pf, groups)+ pipe = _h3_pipe(pf)+ done = video._scheduler_step_carry(pipe, 10, lambda n: None, lambda: False)+ with op.cross_forward_render(pipe):+ with video._scheduler_step_progress(pipe, lambda n: None, done):+ for i in range(3):+ _forward(pf, groups)+ pipe.scheduler.step()+ assert pf.carried # a render shorter than its step count would leave these+ op.end_cross_forward(pipe)+ assert not pf.ready and not pf.carried and pf.inflight_bytes == 0 and not pf.armed+++def test_cancel_skips_the_carry_step():+ pf, groups, _ = _cpu_prefetcher("tabcde", depth = 2)+ _forward(pf, groups)+ pipe = _h3_pipe(pf)+ seen = []+ done = video._scheduler_step_carry(pipe, 10, seen.append, lambda: True)+ with op.cross_forward_render(pipe):+ _forward(pf, groups)+ done(1)+ assert (+ seen == [1] and pf.armed+ ) # the cancel raises before the next step; render exit disarms+ assert not pf.armed and not pf.carried+++ "fam_name, streamed, has_cb, env, expected",+ [+ ("minimax-h3", True, False, None, True),+ ("minimax-h3", True, False, "0", False),+ ("minimax-h3", True, True, None, False),+ ("minimax-h3", False, False, None, False),+ ("hunyuanvideo-1.5", True, False, None, False),+ ],+)+def test_gate(monkeypatch, fam_name, streamed, has_cb, env, expected):+ if env is not None:+ monkeypatch.setenv(H3_STREAM_CARRY_ENV, env)+ fam = types.SimpleNamespace(name = fam_name)+ state = types.SimpleNamespace(denoiser_streamed = "stream" if streamed else False)+ assert video._h3_carry_by_scheduler(fam, state, has_cb) is expected+++ "value, on", [(None, True), ("1", True), ("0", False), ("off", False), ("no", False)]+)+def test_kill_switch(monkeypatch, value, on):+ if value is not None:+ monkeypatch.setenv(H3_STREAM_CARRY_ENV, value)+ assert h3_stream_carry_enabled() is on+++def test_kill_switch_render_carries_nothing(monkeypatch):+ monkeypatch.setenv(H3_STREAM_CARRY_ENV, "0")+ pf, groups, log = _cpu_prefetcher("tabcde", depth = 2)+ _forward(pf, groups)+ log.clear()+ pipe = _h3_pipe(pf)+ gate = video._h3_carry_by_scheduler(+ types.SimpleNamespace(name = "minimax-h3"),+ types.SimpleNamespace(denoiser_streamed = "stream"),+ False,+ )+ with op.cross_forward_render(pipe, enabled = gate):+ for _ in range(4):+ _forward(pf, groups)+ assert not gate and not pf.stats.get("carried") and _copies(log) == list("tabcde") * 4+++def _top_forward(pf, top, blocks):+ pf.begin()+ pf.onload(top)+ for g in blocks:+ pf.onload(g)+ pf.offload(g)+ pf.offload(top)+ pf.end()+++def _top_render(+ keep,+ steps = 6,+ nbytes = 10,+):+ pf, groups, log = _cpu_prefetcher("tabcde", nbytes = nbytes, depth = 2)+ pf.nbytes[id(groups[0])] = 2 * nbytes+ pf.window = 2 * nbytes + 2 * nbytes+ pf.keep_first = keep+ top, blocks = groups[0], groups[1:]+ _top_forward(pf, top, blocks)+ log.clear()+ peak_before = pf.peak_inflight_bytes+ pipe = _h3_pipe(pf)+ done = video._scheduler_step_carry(pipe, steps, lambda n: None, lambda: False)+ with op.cross_forward_render(pipe):+ with video._scheduler_step_progress(pipe, lambda n: None, done):+ for i in range(steps):+ _top_forward(pf, top, blocks)+ if i < steps - 1:+ pipe.scheduler.step()+ decode_view = (dict(pf.ready), set(pf.carried), pf.inflight_bytes)+ op.end_cross_forward(pipe)+ return pf, log, decode_view, peak_before+++def test_streamed_top_group_stays_for_the_next_forward():+ pf, log, (ready, carried, inflight), peak_before = _top_render(keep = True)+ copies = _copies(log)+ assert copies.count("t") == 1 and all(copies.count(b) == 6 for b in "abcde")+ assert pf.stats["kept"] == 5+ assert not ready and not carried and inflight == 0+ assert pf.peak_inflight_bytes <= max(peak_before, pf.window)+++def test_without_keep_the_top_group_reuploads_every_forward():+ pf, log, (ready, carried, inflight), _ = _top_render(keep = False)+ assert _copies(log).count("t") == 6 and not pf.stats.get("kept")+ assert not ready and not carried and inflight == 0+++def test_kept_top_group_is_released_when_the_render_is_cut_short():+ pf, groups, log = _cpu_prefetcher("tabcde", depth = 2)+ pf.keep_first = True+ top, blocks = groups[0], groups[1:]+ _top_forward(pf, top, blocks)+ pipe = _h3_pipe(pf)+ with pytest.raises(RuntimeError):+ with op.cross_forward_render(pipe):+ pf.armed = True+ _top_forward(pf, top, blocks)+ assert id(top) in pf.ready and id(top) in pf.carried+ raise RuntimeError("cancelled")+ assert not pf.ready and not pf.carried and pf.inflight_bytes == 0