Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

Studio MiniMax-H3: keep the released bf16 denoiser resident beside a streamed conditioner on ~100 GB cards (#16877)

Daniel Haned3eec632c06
diff --git a/studio/backend/core/inference/video.py b/studio/backend/core/inference/video.py
index 854b11de0db..c27d54336ab 100644
--- a/studio/backend/core/inference/video.py
+++ b/studio/backend/core/inference/video.py
@@ -2793,14 +2793,25 @@ def h3_streamed_int8_supported(target: Any = None) -> bool:
def _h3_others_bytes(
- fam: Any, components: Any, *, te_scheme: Optional[str], dtype: Any, rotating: bool
+ fam: Any,
+ components: Any,
+ *,
+ te_scheme: Optional[str],
+ dtype: Any,
+ rotating: bool,
+ te_streamed: bool = False,
) -> int:
"""Bytes beside the denoiser. ``rotating``: conditioner and VAEs rotate (the 40 GB reserve evicts
- one before placing the other), so only the larger counts; otherwise they add up."""
+ one before placing the other), so only the larger counts; otherwise they add up. ``te_streamed``: the
+ conditioner counts at its streamed encode footprint."""
import torch
scale = 2.0 if dtype is torch.float32 else 1.0
- text_encoder_gb = h3_te_resident_gb(te_scheme, bf16_gb = components[1])
+ if te_streamed:
+ from .video_minimax_h3_te import H3_TE_STREAMED_GB
+ text_encoder_gb = H3_TE_STREAMED_GB
+ else:
+ text_encoder_gb = h3_te_resident_gb(te_scheme, bf16_gb = components[1])
headroom_bytes = (
estimate_video_runtime_mib(
width = fam.resolution_presets[0][0],
@@ -2821,6 +2832,7 @@ def _h3_dense_denoiser_resident_bytes(
te_scheme: Optional[str],
dtype: Any,
rotating: bool = False,
+ te_streamed: bool = False,
) -> Optional[tuple[int, int]]:
"""Denoiser size is measured from the built module, not the family table's released dense figure."""
components = getattr(fam, "bf16_components_gb", None)
@@ -2839,7 +2851,12 @@ def _h3_dense_denoiser_resident_bytes(
if denoiser_bytes <= 0:
return None
others = _h3_others_bytes(
- fam, components, te_scheme = te_scheme, dtype = dtype, rotating = rotating
+ fam,
+ components,
+ te_scheme = te_scheme,
+ dtype = dtype,
+ rotating = rotating,
+ te_streamed = te_streamed,
)
return int(denoiser_bytes), others
except Exception: # noqa: BLE001 -- an unanswerable estimate keeps the rotation as it is
@@ -3122,6 +3139,95 @@ def _h3_dense_denoiser_fits(sizes: Optional[tuple[int, int]], free_bytes: Option
return int(free_bytes) >= int(denoiser_bytes) + int(others_bytes)
+def _h3_dense_pin_plan(
+ rotating_sizes: Optional[tuple[int, int]],
+ streamed_sizes: Optional[tuple[int, int]],
+ free_bytes: Optional[int],
+) -> Optional[str]:
+ """``"pin"`` (fits beside a rotating conditioner), ``"pin_stream_te"`` (only beside a streamed one) or None."""
+ if _h3_dense_denoiser_fits(rotating_sizes, free_bytes):
+ return "pin"
+ if _h3_dense_denoiser_fits(streamed_sizes, free_bytes):
+ return "pin_stream_te"
+ return None
+
+
+def _h3_dense_demote(
+ pipe: Any,
+ denoiser: Any,
+ *,
+ logger: Any = None,
+) -> bool:
+ """Move a streamed-conditioner-pinned denoiser back into the CPU-offload rotation; False = unchanged."""
+ try:
+ manager = getattr(pipe, "_components_manager", None)
+ hooks = list(getattr(manager, "model_hooks", None) or ())
+ if any(getattr(hook, "model", None) is denoiser for hook in hooks):
+ return True
+ device = getattr(manager, "_auto_offload_device", None)
+ name = next(
+ (k for k, v in (getattr(manager, "components", None) or {}).items() if v is denoiser),
+ None,
+ )
+ if denoiser is None or not hooks or device is None or name is None:
+ return False
+ from .diffusion_memory import _available_system_memory_mib
+ from .diffusion_prequant import tensor_payload_bytes
+
+ need_mib = sum(tensor_payload_bytes(t) for t in denoiser.parameters()) >> 20
+ available_mib = _available_system_memory_mib()
+ if available_mib is None or available_mib < need_mib + max(4096, need_mib // 10):
+ return False
+ from diffusers.modular_pipelines.components_manager import custom_offload_with_hook
+
+ hook = custom_offload_with_hook(
+ name, denoiser, device, offload_strategy = getattr(manager, "_offload_strategy", None)
+ )
+ for other in hooks:
+ if other.hook.execution_device == hook.hook.execution_device:
+ hook.add_other_hook(other)
+ other.add_other_hook(hook)
+ manager.model_hooks = hooks + [hook]
+ hook.offload()
+ clear_gpu_cache()
+ if logger is not None:
+ logger.info(
+ "video.h3_denoiser_placement: this request needs the rotation's VRAM floor, so the bf16 denoiser "
+ "rejoins the CPU-offload rotation until the next load"
+ )
+ return True
+ except Exception as exc: # noqa: BLE001 -- the request guard refuses as before
+ if logger is not None:
+ logger.warning("video.h3_denoiser_placement: could not rejoin the rotation: %s", exc)
+ return False
+
+
+def _h3_group_offload_available() -> bool:
+ try:
+ from diffusers.hooks import apply_group_offloading # noqa: F401
+ except Exception: # noqa: BLE001 -- no group offloading: the conditioner cannot stream
+ return False
+ return True
+
+
+def _h3_dense_unpin(
+ manager: Any,
+ denoiser: Any,
+ device: Any,
+ *,
+ logger: Any = None,
+) -> None:
+ """Return a pinned released denoiser to the CPU-offload rotation (re-hooks every component)."""
+ try:
+ denoiser.to("cpu")
+ manager.disable_auto_cpu_offload()
+ manager.enable_auto_cpu_offload(device = device, memory_reserve_margin = "40GB")
+ except Exception as exc: # noqa: BLE001 -- the load continues with what placement it has
+ logger.warning(
+ "video.h3_denoiser_placement: could not return the denoiser to the rotation: %s", exc
+ )
+
+
def _h3_residency_budget_bytes(
residency: Any,
device: str,
@@ -11889,6 +11995,7 @@ class VideoBackend:
)
offload_policy = "none"
denoiser_pinned = False
+ dense_compile_held = False
denoiser_streamed: Optional[str] = None
denoiser_single_host_copy = False
te_streamed: Optional[str] = None
@@ -12128,9 +12235,27 @@ class VideoBackend:
except Exception: # noqa: BLE001 -- unreadable free memory keeps the rotation
free_bytes = None
if sizes is not None and free_bytes is not None:
- denoiser_bytes, others_bytes = sizes
- if _h3_dense_denoiser_fits(sizes, free_bytes):
- from .diffusion_prequant import pin_prequantized_module
+ from .video_minimax_h3_te import h3_te_stream_enabled
+
+ streamed_sizes = (
+ _h3_dense_denoiser_resident_bytes(
+ fam,
+ denoiser = denoiser,
+ te_scheme = text_encoder_quant_engaged,
+ dtype = dtype,
+ rotating = True,
+ te_streamed = True,
+ )
+ if h3_te_stream_enabled() and _h3_group_offload_available()
+ else None
+ )
+ pin_plan = _h3_dense_pin_plan(sizes, streamed_sizes, free_bytes)
+ from .diffusion_prequant import pin_prequantized_module
+
+ if pin_plan == "pin_stream_te":
+ from .video_minimax_h3_te import stream_h3_text_encoder
+
+ # Denoiser first: pinning the conditioner first evicts its cached pages (66 GB re-read).
denoiser_pinned = pin_prequantized_module(
manager,
denoiser,
@@ -12138,6 +12263,39 @@ class VideoBackend:
logger = logger,
label = "released bfloat16 denoiser",
)
+ te_streamed = stream_h3_text_encoder(
+ manager,
+ getattr(pipe, "text_encoder", None),
+ device,
+ logger = logger,
+ )
+ if te_streamed:
+ sizes = streamed_sizes
+ else:
+ # Sized for the stream: beside a rotating conditioner it OOMs at the encode.
+ _h3_dense_unpin(manager, denoiser, device, logger = logger)
+ denoiser_pinned = False
+ pin_plan = None
+ elif pin_plan == "pin":
+ denoiser_pinned = pin_prequantized_module(
+ manager,
+ denoiser,
+ device,
+ logger = logger,
+ label = "released bfloat16 denoiser",
+ )
+ denoiser_bytes, others_bytes = sizes
+ if pin_plan is not None:
+ if te_streamed:
+ # Compile is not bit-identical here (G4: LPIPS 0.115 over 20 steps): stay eager.
+ dense_compile_held = bool(denoiser_pinned)
+ logger.info(
+ "video.h3_denoiser_placement: %.1f GB free holds the denoiser beside the "
+ "streamed conditioner (%.1f GB), not beside a rotating one, so it stays "
+ "resident",
+ free_bytes / 1e9,
+ (denoiser_bytes + others_bytes) / 1e9,
+ )
else:
logger.info(
"video.h3_denoiser_placement: %.1f GB free is under the %.1f GB the "
@@ -12148,7 +12306,14 @@ class VideoBackend:
)
# Speed layer applied here after offload (compile needs denoiser_pinned); not in load_pipeline.
- if effective_speed in (SPEED_DEFAULT, SPEED_MAX) and not (
+ if effective_speed in (SPEED_DEFAULT, SPEED_MAX) and dense_compile_held:
+ logger.info(
+ "video.speed_mode: the released bf16 denoiser is resident beside the streamed "
+ "conditioner; the regional compile is held off so the output matches the rotation "
+ "it replaces; keeping the lossless optimisations"
+ )
+ effective_speed = SPEED_EAGER
+ elif effective_speed in (SPEED_DEFAULT, SPEED_MAX) and not (
denoiser_pinned or denoiser_streamed
):
# Compile only a resident or streamed denoiser; in the rotation the graph fights the onload hooks.
@@ -12451,7 +12616,7 @@ class VideoBackend:
else ""
)
+ (
- "; the quantized conditioner streams leaf by leaf (group offload)"
+ "; the conditioner streams leaf by leaf (group offload)"
if te_streamed
else ""
),
@@ -12465,7 +12630,12 @@ class VideoBackend:
else "regional compile over the resident pre-quantized denoiser"
)
if "compiled" in speed_optims
- else "lossless optimisations only; the denoiser is in the CPU-offload rotation",
+ else (
+ "lossless optimisations only; the bf16 denoiser is resident beside the "
+ "streamed conditioner (compile held off: output unchanged)"
+ if dense_compile_held
+ else "lossless optimisations only; the denoiser is in the CPU-offload rotation"
+ ),
),
"attention_backend": (
attention_backend,
@@ -13286,6 +13456,29 @@ class VideoBackend:
),
)
)
+ if (
+ available_vram_gb + 0.25 < required_vram_gb
+ and getattr(state, "denoiser_pinned", False)
+ and getattr(state, "te_streamed", False)
+ and state.transformer_quant is None
+ and state.offload_policy != "none"
+ ):
+ # Pin was sized for the default clip; a longer one the rotation fits must not be refused.
+ rotation_gb = estimate_h3_diffusers_vram_gb(
+ width,
+ height,
+ frames,
+ text_encoder_gb = h3_te_resident_gb(
+ state.text_encoder_quant, bf16_gb = H3_TEXT_ENCODER_BF16_GB
+ ),
+ text_encoder_streamed = True,
+ )
+ if available_vram_gb + 0.25 >= rotation_gb and _h3_dense_demote(
+ state.pipe,
+ getattr(state.pipe, h3_denoiser_component(state.h3_task), None),
+ logger = logger,
+ ):
+ required_vram_gb = rotation_gb
if available_vram_gb + 0.25 < required_vram_gb:
raise RuntimeError(
f"MiniMax-H3 needs about {required_vram_gb:.1f} GB available "
diff --git a/studio/backend/tests/test_h3_bf16_streamed_te_pin.py b/studio/backend/tests/test_h3_bf16_streamed_te_pin.py
new file mode 100644
index 00000000000..d6062dc3b23
--- /dev/null
+++ b/studio/backend/tests/test_h3_bf16_streamed_te_pin.py
@@ -0,0 +1,228 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
+
+"""Released bf16 MiniMax-H3 denoiser placement beside a streamed conditioner."""
+
+import inspect
+
+import pytest
+
+
+def _family():
+ from core.inference.video_families import detect_video_family
+ return detect_video_family("MiniMaxAI/MiniMax-H3")
+
+
+class _Denoiser:
+ def parameters(self):
+ import torch
+ return iter([torch.empty(8, dtype = torch.bfloat16)])
+
+ def buffers(self):
+ return iter(())
+
+
+def _sizes(
+ monkeypatch,
+ *,
+ te_scheme,
+ te_streamed,
+ denoiser_gb = 66.3,
+):
+ import torch
+
+ from core.inference import diffusion_prequant
+ from core.inference import video as vid
+
+ monkeypatch.setattr(
+ diffusion_prequant, "tensor_payload_bytes", lambda t: int(denoiser_gb * 1e9)
+ )
+ return vid._h3_dense_denoiser_resident_bytes(
+ _family(),
+ denoiser = _Denoiser(),
+ te_scheme = te_scheme,
+ dtype = torch.bfloat16,
+ rotating = True,
+ te_streamed = te_streamed,
+ )
+
+
+def _plan(
+ monkeypatch,
+ free_gb,
+ te_scheme = None,
+):
+ from core.inference import video as vid
+
+ rotating = _sizes(monkeypatch, te_scheme = te_scheme, te_streamed = False)
+ streamed = _sizes(monkeypatch, te_scheme = te_scheme, te_streamed = True)
+ return vid._h3_dense_pin_plan(rotating, streamed, int(free_gb * 1e9))
+
+
+def test_streamed_conditioner_is_sized_at_its_encode_footprint(monkeypatch):
+ from core.inference.video_minimax_h3_te import H3_TE_STREAMED_GB
+
+ fam = _family()
+ rotating = _sizes(monkeypatch, te_scheme = None, te_streamed = False)
+ streamed = _sizes(monkeypatch, te_scheme = None, te_streamed = True)
+ assert rotating[0] == streamed[0]
+ vae_gb = fam.bf16_components_gb[2]
+ expected = int(fam.bf16_components_gb[1] * 1e9) - int(max(H3_TE_STREAMED_GB, vae_gb) * 1e9)
+ assert abs((rotating[1] - streamed[1]) - expected) < 1000
+ import torch
+
+ from core.inference import video as vid
+
+ default = vid._h3_dense_denoiser_resident_bytes(
+ fam, denoiser = _Denoiser(), te_scheme = None, dtype = torch.bfloat16, rotating = True
+ )
+ assert default == rotating
+
+
+ "free_gb, te_scheme, expected",
+ [
+ (101.4, None, "pin_stream_te"),
+ (101.4, "int8", "pin_stream_te"),
+ (178.0, None, "pin"),
+ (178.0, "int8", "pin"),
+ (79.0, None, None),
+ (47.0, None, None),
+ (12.0, "int8", None),
+ ],
+)
+def test_bf16_denoiser_placement_by_card(monkeypatch, free_gb, te_scheme, expected):
+ assert _plan(monkeypatch, free_gb, te_scheme) == expected
+
+
+def test_unreadable_or_unsized_keeps_the_rotation():
+ from core.inference import video as vid
+
+ assert vid._h3_dense_pin_plan(None, None, 100 * 10**9) is None
+ assert vid._h3_dense_pin_plan((1, 1), (1, 1), None) is None
+ assert vid._h3_dense_pin_plan((60 * 10**9, 60 * 10**9), None, 100 * 10**9) is None
+
+
+def test_load_moves_the_denoiser_before_pinning_the_conditioner_and_falls_back_when_refused():
+ from core.inference import video as vid
+
+ src = inspect.getsource(vid.VideoBackend)
+ start = src.index('if pin_plan == "pin_stream_te":')
+ end = src.index('elif pin_plan == "pin":', start)
+ block = src[start:end]
+ pin_at = block.index("pin_prequantized_module(")
+ stream_at = block.index("stream_h3_text_encoder(\n")
+ assert pin_at < stream_at
+ refused = block[stream_at:]
+ assert "_h3_dense_unpin(manager, denoiser, device, logger = logger)" in refused
+ assert "pin_plan = None" in refused and "denoiser_pinned = False" in refused
+ assert (
+ "_h3_group_offload_available()" in src[src.rindex("streamed_sizes = (", 0, start) : start]
+ )
+
+
+def test_unpin_returns_the_denoiser_to_the_rotation():
+ from core.inference import video as vid
+
+ calls = []
+
+ class _Module:
+ def to(self, device):
+ calls.append(("to", device))
+
+ class _Manager:
+ def disable_auto_cpu_offload(self):
+ calls.append(("disable",))
+
+ def enable_auto_cpu_offload(
+ self,
+ device = None,
+ memory_reserve_margin = None,
+ ):
+ calls.append(("enable", device, memory_reserve_margin))
+
+ vid._h3_dense_unpin(_Manager(), _Module(), "cuda")
+ assert calls == [("to", "cpu"), ("disable",), ("enable", "cuda", "40GB")]
+
+
+def test_streamed_conditioner_tier_keeps_the_eager_tier_of_the_rotation_it_replaces():
+ from core.inference import video as vid
+
+ src = inspect.getsource(vid.VideoBackend)
+ start = src.index("pin_plan = _h3_dense_pin_plan(")
+ held = src.index("dense_compile_held = bool(denoiser_pinned)", start)
+ assert held < src.index("video.h3_denoiser_placement: %.1f GB free is under", start)
+ gate = src.index("if effective_speed in (SPEED_DEFAULT, SPEED_MAX) and dense_compile_held:")
+ assert src.index("effective_speed = SPEED_EAGER", gate) < src.index(
+ "snapshot_backend_flags()", gate
+ )
+
+
+class _Hook:
+ def __init__(
+ self,
+ model,
+ device = "cuda:0",
+ ):
+ self.model = model
+ self.hook = type("H", (), {"execution_device": device})()
+ self.others = []
+ self.offloaded = False
+
+ def add_other_hook(self, other):
+ self.others.append(other)
+
+ def offload(self):
+ self.offloaded = True
+
+
+def _demote_fixture(monkeypatch, available_mib):
+ import torch
+
+ cm = pytest.importorskip("diffusers.modular_pipelines.components_manager")
+
+ from core.inference import diffusion_memory
+ from core.inference import video as vid
+
+ denoiser = torch.nn.Linear(4, 4)
+ conditioner_vae = object()
+ manager = type("M", (), {})()
+ manager.components = {"transformer_1": denoiser, "vae_2": conditioner_vae}
+ manager.model_hooks = [_Hook(conditioner_vae)]
+ manager._auto_offload_device = "cuda:0"
+ manager._offload_strategy = None
+ pipe = type("P", (), {})()
+ pipe._components_manager = manager
+ monkeypatch.setattr(diffusion_memory, "_available_system_memory_mib", lambda: available_mib)
+ monkeypatch.setattr(
+ cm,
+ "custom_offload_with_hook",
+ lambda name, model, device, offload_strategy = None: _Hook(model, device),
+ )
+ monkeypatch.setattr(vid, "clear_gpu_cache", lambda: None)
+ return vid, pipe, manager, denoiser
+
+
+def test_a_request_past_the_pin_budget_returns_the_denoiser_to_the_rotation(monkeypatch):
+ vid, pipe, manager, denoiser = _demote_fixture(monkeypatch, 200_000)
+ assert vid._h3_dense_demote(pipe, denoiser)
+ added = manager.model_hooks[-1]
+ assert added.model is denoiser and added.offloaded
+ assert added in manager.model_hooks[0].others and manager.model_hooks[0] in added.others
+ assert vid._h3_dense_demote(pipe, denoiser) and len(manager.model_hooks) == 2
+
+
+def test_no_host_room_keeps_the_pin_and_the_refusal(monkeypatch):
+ vid, pipe, manager, denoiser = _demote_fixture(monkeypatch, 1)
+ assert not vid._h3_dense_demote(pipe, denoiser)
+ assert len(manager.model_hooks) == 1
+
+
+def test_request_guard_tries_the_rotation_before_refusing():
+ from core.inference import video as vid
+
+ src = inspect.getsource(vid.VideoBackend)
+ demote = src.index("_h3_dense_demote(\n")
+ refuse = src.index('f"MiniMax-H3 needs about {required_vram_gb:.1f} GB available "')
+ assert demote < refuse
+ assert "required_vram_gb = rotation_gb" in src[demote:refuse]