Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
Studio MiniMax-H3: keep the released bf16 denoiser resident beside a streamed conditioner on ~100 GB cards (#16877)
Daniel Haned3eec632c06
diff --git a/studio/backend/core/inference/video.py b/studio/backend/core/inference/video.pyindex 854b11de0db..c27d54336ab 100644--- a/studio/backend/core/inference/video.py+++ b/studio/backend/core/inference/video.py@@ -2793,14 +2793,25 @@ def h3_streamed_int8_supported(target: Any = None) -> bool:def _h3_others_bytes(- fam: Any, components: Any, *, te_scheme: Optional[str], dtype: Any, rotating: bool+ fam: Any,+ components: Any,+ *,+ te_scheme: Optional[str],+ dtype: Any,+ rotating: bool,+ te_streamed: bool = False,) -> int:"""Bytes beside the denoiser. ``rotating``: conditioner and VAEs rotate (the 40 GB reserve evicts- one before placing the other), so only the larger counts; otherwise they add up."""+ one before placing the other), so only the larger counts; otherwise they add up. ``te_streamed``: the+ conditioner counts at its streamed encode footprint."""import torchscale = 2.0 if dtype is torch.float32 else 1.0- text_encoder_gb = h3_te_resident_gb(te_scheme, bf16_gb = components[1])+ if te_streamed:+ from .video_minimax_h3_te import H3_TE_STREAMED_GB+ text_encoder_gb = H3_TE_STREAMED_GB+ else:+ text_encoder_gb = h3_te_resident_gb(te_scheme, bf16_gb = components[1])headroom_bytes = (estimate_video_runtime_mib(width = fam.resolution_presets[0][0],@@ -2821,6 +2832,7 @@ def _h3_dense_denoiser_resident_bytes(te_scheme: Optional[str],dtype: Any,rotating: bool = False,+ te_streamed: bool = False,) -> Optional[tuple[int, int]]:"""Denoiser size is measured from the built module, not the family table's released dense figure."""components = getattr(fam, "bf16_components_gb", None)@@ -2839,7 +2851,12 @@ def _h3_dense_denoiser_resident_bytes(if denoiser_bytes <= 0:return Noneothers = _h3_others_bytes(- fam, components, te_scheme = te_scheme, dtype = dtype, rotating = rotating+ fam,+ components,+ te_scheme = te_scheme,+ dtype = dtype,+ rotating = rotating,+ te_streamed = te_streamed,)return int(denoiser_bytes), othersexcept Exception: # noqa: BLE001 -- an unanswerable estimate keeps the rotation as it is@@ -3122,6 +3139,95 @@ def _h3_dense_denoiser_fits(sizes: Optional[tuple[int, int]], free_bytes: Optionreturn int(free_bytes) >= int(denoiser_bytes) + int(others_bytes)+def _h3_dense_pin_plan(+ rotating_sizes: Optional[tuple[int, int]],+ streamed_sizes: Optional[tuple[int, int]],+ free_bytes: Optional[int],+) -> Optional[str]:+ """``"pin"`` (fits beside a rotating conditioner), ``"pin_stream_te"`` (only beside a streamed one) or None."""+ if _h3_dense_denoiser_fits(rotating_sizes, free_bytes):+ return "pin"+ if _h3_dense_denoiser_fits(streamed_sizes, free_bytes):+ return "pin_stream_te"+ return None+++def _h3_dense_demote(+ pipe: Any,+ denoiser: Any,+ *,+ logger: Any = None,+) -> bool:+ """Move a streamed-conditioner-pinned denoiser back into the CPU-offload rotation; False = unchanged."""+ try:+ manager = getattr(pipe, "_components_manager", None)+ hooks = list(getattr(manager, "model_hooks", None) or ())+ if any(getattr(hook, "model", None) is denoiser for hook in hooks):+ return True+ device = getattr(manager, "_auto_offload_device", None)+ name = next(+ (k for k, v in (getattr(manager, "components", None) or {}).items() if v is denoiser),+ None,+ )+ if denoiser is None or not hooks or device is None or name is None:+ return False+ from .diffusion_memory import _available_system_memory_mib+ from .diffusion_prequant import tensor_payload_bytes++ need_mib = sum(tensor_payload_bytes(t) for t in denoiser.parameters()) >> 20+ available_mib = _available_system_memory_mib()+ if available_mib is None or available_mib < need_mib + max(4096, need_mib // 10):+ return False+ from diffusers.modular_pipelines.components_manager import custom_offload_with_hook++ hook = custom_offload_with_hook(+ name, denoiser, device, offload_strategy = getattr(manager, "_offload_strategy", None)+ )+ for other in hooks:+ if other.hook.execution_device == hook.hook.execution_device:+ hook.add_other_hook(other)+ other.add_other_hook(hook)+ manager.model_hooks = hooks + [hook]+ hook.offload()+ clear_gpu_cache()+ if logger is not None:+ logger.info(+ "video.h3_denoiser_placement: this request needs the rotation's VRAM floor, so the bf16 denoiser "+ "rejoins the CPU-offload rotation until the next load"+ )+ return True+ except Exception as exc: # noqa: BLE001 -- the request guard refuses as before+ if logger is not None:+ logger.warning("video.h3_denoiser_placement: could not rejoin the rotation: %s", exc)+ return False+++def _h3_group_offload_available() -> bool:+ try:+ from diffusers.hooks import apply_group_offloading # noqa: F401+ except Exception: # noqa: BLE001 -- no group offloading: the conditioner cannot stream+ return False+ return True+++def _h3_dense_unpin(+ manager: Any,+ denoiser: Any,+ device: Any,+ *,+ logger: Any = None,+) -> None:+ """Return a pinned released denoiser to the CPU-offload rotation (re-hooks every component)."""+ try:+ denoiser.to("cpu")+ manager.disable_auto_cpu_offload()+ manager.enable_auto_cpu_offload(device = device, memory_reserve_margin = "40GB")+ except Exception as exc: # noqa: BLE001 -- the load continues with what placement it has+ logger.warning(+ "video.h3_denoiser_placement: could not return the denoiser to the rotation: %s", exc+ )++def _h3_residency_budget_bytes(residency: Any,device: str,@@ -11889,6 +11995,7 @@ class VideoBackend:)offload_policy = "none"denoiser_pinned = False+ dense_compile_held = Falsedenoiser_streamed: Optional[str] = Nonedenoiser_single_host_copy = Falsete_streamed: Optional[str] = None@@ -12128,9 +12235,27 @@ class VideoBackend:except Exception: # noqa: BLE001 -- unreadable free memory keeps the rotationfree_bytes = Noneif sizes is not None and free_bytes is not None:- denoiser_bytes, others_bytes = sizes- if _h3_dense_denoiser_fits(sizes, free_bytes):- from .diffusion_prequant import pin_prequantized_module+ from .video_minimax_h3_te import h3_te_stream_enabled++ streamed_sizes = (+ _h3_dense_denoiser_resident_bytes(+ fam,+ denoiser = denoiser,+ te_scheme = text_encoder_quant_engaged,+ dtype = dtype,+ rotating = True,+ te_streamed = True,+ )+ if h3_te_stream_enabled() and _h3_group_offload_available()+ else None+ )+ pin_plan = _h3_dense_pin_plan(sizes, streamed_sizes, free_bytes)+ from .diffusion_prequant import pin_prequantized_module++ if pin_plan == "pin_stream_te":+ from .video_minimax_h3_te import stream_h3_text_encoder++ # Denoiser first: pinning the conditioner first evicts its cached pages (66 GB re-read).denoiser_pinned = pin_prequantized_module(manager,denoiser,@@ -12138,6 +12263,39 @@ class VideoBackend:logger = logger,label = "released bfloat16 denoiser",)+ te_streamed = stream_h3_text_encoder(+ manager,+ getattr(pipe, "text_encoder", None),+ device,+ logger = logger,+ )+ if te_streamed:+ sizes = streamed_sizes+ else:+ # Sized for the stream: beside a rotating conditioner it OOMs at the encode.+ _h3_dense_unpin(manager, denoiser, device, logger = logger)+ denoiser_pinned = False+ pin_plan = None+ elif pin_plan == "pin":+ denoiser_pinned = pin_prequantized_module(+ manager,+ denoiser,+ device,+ logger = logger,+ label = "released bfloat16 denoiser",+ )+ denoiser_bytes, others_bytes = sizes+ if pin_plan is not None:+ if te_streamed:+ # Compile is not bit-identical here (G4: LPIPS 0.115 over 20 steps): stay eager.+ dense_compile_held = bool(denoiser_pinned)+ logger.info(+ "video.h3_denoiser_placement: %.1f GB free holds the denoiser beside the "+ "streamed conditioner (%.1f GB), not beside a rotating one, so it stays "+ "resident",+ free_bytes / 1e9,+ (denoiser_bytes + others_bytes) / 1e9,+ )else:logger.info("video.h3_denoiser_placement: %.1f GB free is under the %.1f GB the "@@ -12148,7 +12306,14 @@ class VideoBackend:)# Speed layer applied here after offload (compile needs denoiser_pinned); not in load_pipeline.- if effective_speed in (SPEED_DEFAULT, SPEED_MAX) and not (+ if effective_speed in (SPEED_DEFAULT, SPEED_MAX) and dense_compile_held:+ logger.info(+ "video.speed_mode: the released bf16 denoiser is resident beside the streamed "+ "conditioner; the regional compile is held off so the output matches the rotation "+ "it replaces; keeping the lossless optimisations"+ )+ effective_speed = SPEED_EAGER+ elif effective_speed in (SPEED_DEFAULT, SPEED_MAX) and not (denoiser_pinned or denoiser_streamed):# Compile only a resident or streamed denoiser; in the rotation the graph fights the onload hooks.@@ -12451,7 +12616,7 @@ class VideoBackend:else "")+ (- "; the quantized conditioner streams leaf by leaf (group offload)"+ "; the conditioner streams leaf by leaf (group offload)"if te_streamedelse ""),@@ -12465,7 +12630,12 @@ class VideoBackend:else "regional compile over the resident pre-quantized denoiser")if "compiled" in speed_optims- else "lossless optimisations only; the denoiser is in the CPU-offload rotation",+ else (+ "lossless optimisations only; the bf16 denoiser is resident beside the "+ "streamed conditioner (compile held off: output unchanged)"+ if dense_compile_held+ else "lossless optimisations only; the denoiser is in the CPU-offload rotation"+ ),),"attention_backend": (attention_backend,@@ -13286,6 +13456,29 @@ class VideoBackend:),))+ if (+ available_vram_gb + 0.25 < required_vram_gb+ and getattr(state, "denoiser_pinned", False)+ and getattr(state, "te_streamed", False)+ and state.transformer_quant is None+ and state.offload_policy != "none"+ ):+ # Pin was sized for the default clip; a longer one the rotation fits must not be refused.+ rotation_gb = estimate_h3_diffusers_vram_gb(+ width,+ height,+ frames,+ text_encoder_gb = h3_te_resident_gb(+ state.text_encoder_quant, bf16_gb = H3_TEXT_ENCODER_BF16_GB+ ),+ text_encoder_streamed = True,+ )+ if available_vram_gb + 0.25 >= rotation_gb and _h3_dense_demote(+ state.pipe,+ getattr(state.pipe, h3_denoiser_component(state.h3_task), None),+ logger = logger,+ ):+ required_vram_gb = rotation_gbif available_vram_gb + 0.25 < required_vram_gb:raise RuntimeError(f"MiniMax-H3 needs about {required_vram_gb:.1f} GB available "diff --git a/studio/backend/tests/test_h3_bf16_streamed_te_pin.py b/studio/backend/tests/test_h3_bf16_streamed_te_pin.pynew file mode 100644index 00000000000..d6062dc3b23--- /dev/null+++ b/studio/backend/tests/test_h3_bf16_streamed_te_pin.py@@ -0,0 +1,228 @@+# SPDX-License-Identifier: AGPL-3.0-only+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0++"""Released bf16 MiniMax-H3 denoiser placement beside a streamed conditioner."""++import inspect++import pytest+++def _family():+ from core.inference.video_families import detect_video_family+ return detect_video_family("MiniMaxAI/MiniMax-H3")+++class _Denoiser:+ def parameters(self):+ import torch+ return iter([torch.empty(8, dtype = torch.bfloat16)])++ def buffers(self):+ return iter(())+++def _sizes(+ monkeypatch,+ *,+ te_scheme,+ te_streamed,+ denoiser_gb = 66.3,+):+ import torch++ from core.inference import diffusion_prequant+ from core.inference import video as vid++ monkeypatch.setattr(+ diffusion_prequant, "tensor_payload_bytes", lambda t: int(denoiser_gb * 1e9)+ )+ return vid._h3_dense_denoiser_resident_bytes(+ _family(),+ denoiser = _Denoiser(),+ te_scheme = te_scheme,+ dtype = torch.bfloat16,+ rotating = True,+ te_streamed = te_streamed,+ )+++def _plan(+ monkeypatch,+ free_gb,+ te_scheme = None,+):+ from core.inference import video as vid++ rotating = _sizes(monkeypatch, te_scheme = te_scheme, te_streamed = False)+ streamed = _sizes(monkeypatch, te_scheme = te_scheme, te_streamed = True)+ return vid._h3_dense_pin_plan(rotating, streamed, int(free_gb * 1e9))+++def test_streamed_conditioner_is_sized_at_its_encode_footprint(monkeypatch):+ from core.inference.video_minimax_h3_te import H3_TE_STREAMED_GB++ fam = _family()+ rotating = _sizes(monkeypatch, te_scheme = None, te_streamed = False)+ streamed = _sizes(monkeypatch, te_scheme = None, te_streamed = True)+ assert rotating[0] == streamed[0]+ vae_gb = fam.bf16_components_gb[2]+ expected = int(fam.bf16_components_gb[1] * 1e9) - int(max(H3_TE_STREAMED_GB, vae_gb) * 1e9)+ assert abs((rotating[1] - streamed[1]) - expected) < 1000+ import torch++ from core.inference import video as vid++ default = vid._h3_dense_denoiser_resident_bytes(+ fam, denoiser = _Denoiser(), te_scheme = None, dtype = torch.bfloat16, rotating = True+ )+ assert default == rotating+++ "free_gb, te_scheme, expected",+ [+ (101.4, None, "pin_stream_te"),+ (101.4, "int8", "pin_stream_te"),+ (178.0, None, "pin"),+ (178.0, "int8", "pin"),+ (79.0, None, None),+ (47.0, None, None),+ (12.0, "int8", None),+ ],+)+def test_bf16_denoiser_placement_by_card(monkeypatch, free_gb, te_scheme, expected):+ assert _plan(monkeypatch, free_gb, te_scheme) == expected+++def test_unreadable_or_unsized_keeps_the_rotation():+ from core.inference import video as vid++ assert vid._h3_dense_pin_plan(None, None, 100 * 10**9) is None+ assert vid._h3_dense_pin_plan((1, 1), (1, 1), None) is None+ assert vid._h3_dense_pin_plan((60 * 10**9, 60 * 10**9), None, 100 * 10**9) is None+++def test_load_moves_the_denoiser_before_pinning_the_conditioner_and_falls_back_when_refused():+ from core.inference import video as vid++ src = inspect.getsource(vid.VideoBackend)+ start = src.index('if pin_plan == "pin_stream_te":')+ end = src.index('elif pin_plan == "pin":', start)+ block = src[start:end]+ pin_at = block.index("pin_prequantized_module(")+ stream_at = block.index("stream_h3_text_encoder(\n")+ assert pin_at < stream_at+ refused = block[stream_at:]+ assert "_h3_dense_unpin(manager, denoiser, device, logger = logger)" in refused+ assert "pin_plan = None" in refused and "denoiser_pinned = False" in refused+ assert (+ "_h3_group_offload_available()" in src[src.rindex("streamed_sizes = (", 0, start) : start]+ )+++def test_unpin_returns_the_denoiser_to_the_rotation():+ from core.inference import video as vid++ calls = []++ class _Module:+ def to(self, device):+ calls.append(("to", device))++ class _Manager:+ def disable_auto_cpu_offload(self):+ calls.append(("disable",))++ def enable_auto_cpu_offload(+ self,+ device = None,+ memory_reserve_margin = None,+ ):+ calls.append(("enable", device, memory_reserve_margin))++ vid._h3_dense_unpin(_Manager(), _Module(), "cuda")+ assert calls == [("to", "cpu"), ("disable",), ("enable", "cuda", "40GB")]+++def test_streamed_conditioner_tier_keeps_the_eager_tier_of_the_rotation_it_replaces():+ from core.inference import video as vid++ src = inspect.getsource(vid.VideoBackend)+ start = src.index("pin_plan = _h3_dense_pin_plan(")+ held = src.index("dense_compile_held = bool(denoiser_pinned)", start)+ assert held < src.index("video.h3_denoiser_placement: %.1f GB free is under", start)+ gate = src.index("if effective_speed in (SPEED_DEFAULT, SPEED_MAX) and dense_compile_held:")+ assert src.index("effective_speed = SPEED_EAGER", gate) < src.index(+ "snapshot_backend_flags()", gate+ )+++class _Hook:+ def __init__(+ self,+ model,+ device = "cuda:0",+ ):+ self.model = model+ self.hook = type("H", (), {"execution_device": device})()+ self.others = []+ self.offloaded = False++ def add_other_hook(self, other):+ self.others.append(other)++ def offload(self):+ self.offloaded = True+++def _demote_fixture(monkeypatch, available_mib):+ import torch++ cm = pytest.importorskip("diffusers.modular_pipelines.components_manager")++ from core.inference import diffusion_memory+ from core.inference import video as vid++ denoiser = torch.nn.Linear(4, 4)+ conditioner_vae = object()+ manager = type("M", (), {})()+ manager.components = {"transformer_1": denoiser, "vae_2": conditioner_vae}+ manager.model_hooks = [_Hook(conditioner_vae)]+ manager._auto_offload_device = "cuda:0"+ manager._offload_strategy = None+ pipe = type("P", (), {})()+ pipe._components_manager = manager+ monkeypatch.setattr(diffusion_memory, "_available_system_memory_mib", lambda: available_mib)+ monkeypatch.setattr(+ cm,+ "custom_offload_with_hook",+ lambda name, model, device, offload_strategy = None: _Hook(model, device),+ )+ monkeypatch.setattr(vid, "clear_gpu_cache", lambda: None)+ return vid, pipe, manager, denoiser+++def test_a_request_past_the_pin_budget_returns_the_denoiser_to_the_rotation(monkeypatch):+ vid, pipe, manager, denoiser = _demote_fixture(monkeypatch, 200_000)+ assert vid._h3_dense_demote(pipe, denoiser)+ added = manager.model_hooks[-1]+ assert added.model is denoiser and added.offloaded+ assert added in manager.model_hooks[0].others and manager.model_hooks[0] in added.others+ assert vid._h3_dense_demote(pipe, denoiser) and len(manager.model_hooks) == 2+++def test_no_host_room_keeps_the_pin_and_the_refusal(monkeypatch):+ vid, pipe, manager, denoiser = _demote_fixture(monkeypatch, 1)+ assert not vid._h3_dense_demote(pipe, denoiser)+ assert len(manager.model_hooks) == 1+++def test_request_guard_tries_the_rotation_before_refusing():+ from core.inference import video as vid++ src = inspect.getsource(vid.VideoBackend)+ demote = src.index("_h3_dense_demote(\n")+ refuse = src.index('f"MiniMax-H3 needs about {required_vram_gb:.1f} GB available "')+ assert demote < refuse+ assert "required_vram_gb = rotation_gb" in src[demote:refuse]