Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
Studio MiniMax-H3: park the VAEs when a render returns, so later requests keep their residency budget (#17356)
Daniel Han3bcaf2d24234
diff --git a/studio/backend/core/inference/video.py b/studio/backend/core/inference/video.pyindex b83ffa96f36..e508e957006 100644--- a/studio/backend/core/inference/video.py+++ b/studio/backend/core/inference/video.py@@ -12865,6 +12865,7 @@ class VideoBackend:if offload_policy != "none" and (te_streamed or denoiser_streamed):# Only the VAEs rotate now, evicting each other every render. Installed last: the levers above replace weights.from .video_minimax_h3_residency import install_pinned_swap+for vae_name in ("vae", "audio_vae"):try:install_pinned_swap(@@ -12872,6 +12873,9 @@ class VideoBackend:)except Exception as exc: # noqa: BLE001 -- stock moveslogger.warning("video.h3_pinned_swap: %s: %s", vae_name, exc)+ from .video_minimax_h3_residency import install_h3_rotation_park++ install_h3_rotation_park(pipe, park = bool(denoiser_streamed), logger = logger)if te_streamed == "stream_lazy":# Last host copy, so available RAM is final.from .video_minimax_h3_te import h3_te_resident_enabled, keep_h3_te_resident@@ -14373,7 +14377,12 @@ class VideoBackend:reset_static_step_skip(_skip_pipe(state, pipe), None)if fam.modular_workflow:from .video_minimax_h3_vae import settle_h3_vae_fallback+settle_h3_vae_fallback(state, pipe)+ from .video_minimax_h3_residency import park_h3_rotation++ # The next request's VRAM floor reads free memory before its first forward evicts these.+ park_h3_rotation(pipe)if cancel.is_set():raise RuntimeError(VIDEO_CANCELLED_MSG)diff --git a/studio/backend/core/inference/video_minimax_h3_residency.py b/studio/backend/core/inference/video_minimax_h3_residency.pyindex 3dac656a97d..045ce556370 100644--- a/studio/backend/core/inference/video_minimax_h3_residency.py+++ b/studio/backend/core/inference/video_minimax_h3_residency.py@@ -7,6 +7,7 @@ streamed."""from __future__ import annotations+import loggingimport osfrom typing import Any, Optional@@ -1481,3 +1482,111 @@ def install_pinned_swap(sum(t.numel() * t.element_size() for t in host.values()) / 1e9,)return True+++H3_ROTATION_PARK_ENV = "UNSLOTH_H3_ROTATION_PARK"+_PARK_ATTR = "_unsloth_h3_rotation_park"+_COMPONENTS_MANAGER_LOGGER = "diffusers.modular_pipelines.components_manager"+_FALLBACK_WARNING = "no combination of models to offload to cpu is found"+++def h3_rotation_park_enabled() -> bool:+ return str(os.environ.get(H3_ROTATION_PARK_ENV, "")).strip().lower() not in (+ "0",+ "off",+ "false",+ "no",+ )+++class _DropFallbackWarning(logging.Filter):+ def filter(self, record: logging.LogRecord) -> bool:+ try:+ return _FALLBACK_WARNING not in record.getMessage()+ except Exception: # noqa: BLE001 -- an unformattable record is not ours to drop+ return True+++class H3QuietOffloadStrategy:+ """The wrapped strategy, minus its "offloading all models" warning (evict-all is this tier's policy)."""++ def __init__(self, inner: Any) -> None:+ self.inner = inner++ def __getattr__(self, name: str) -> Any:+ inner = self.__dict__.get("inner")+ if inner is None:+ raise AttributeError(name)+ return getattr(inner, name)++ def __call__(self, hooks: Any, model_id: Any, model: Any, execution_device: Any) -> Any:+ log = logging.getLogger(_COMPONENTS_MANAGER_LOGGER)+ quiet = _DropFallbackWarning()+ log.addFilter(quiet)+ try:+ return self.inner(+ hooks = hooks, model_id = model_id, model = model, execution_device = execution_device+ )+ finally:+ log.removeFilter(quiet)+++def install_h3_rotation_park(+ pipe: Any,+ *,+ park: bool,+ logger: Any = None,+) -> bool:+ """Streamed tiers only. ``park`` (streamed denoiser): evict the rotation when a render returns, not at the next+ forward, so the next request's VRAM floor stops counting the last decode's audio VAE."""+ if pipe is None or not h3_rotation_park_enabled():+ return False+ manager = getattr(pipe, "_components_manager", None)+ hooks = list(getattr(manager, "model_hooks", None) or ())+ if not hooks or getattr(manager, "_auto_offload_device", None) is None:+ return False+ wrapped: dict[int, Any] = {}++ def _quiet(strategy: Any) -> Any:+ if strategy is None or isinstance(strategy, H3QuietOffloadStrategy):+ return strategy+ if id(strategy) not in wrapped:+ wrapped[id(strategy)] = H3QuietOffloadStrategy(strategy)+ return wrapped[id(strategy)]++ for hook in hooks:+ inner = getattr(hook, "hook", None)+ if inner is not None and hasattr(inner, "offload_strategy"):+ inner.offload_strategy = _quiet(inner.offload_strategy)+ if hasattr(manager, "_offload_strategy"):+ manager._offload_strategy = _quiet(manager._offload_strategy)+ setattr(pipe, _PARK_ATTR, bool(park))+ if logger is not None:+ logger.info(+ "video.h3_rotation: evicting every resident component is this tier's policy, so the diffusers "+ "fallback warning is muted; %s %s (%s=0 keeps both as before)",+ ", ".join(str(getattr(hook, "model_id", "?")) for hook in hooks),+ "park on the host when a render returns, so the next request's VRAM floor and resident set see that "+ "room free"+ if park+ else "stay until they are evicted (the conditioner's pre-hook may be skipped by a prompt-cache hit)",+ H3_ROTATION_PARK_ENV,+ )+ return True+++def park_h3_rotation(pipe: Any) -> bool:+ """Evict resident rotating components (the pre-hooks' own eviction); never raises."""+ if pipe is None or not getattr(pipe, _PARK_ATTR, False):+ return False+ try:+ manager = getattr(pipe, "_components_manager", None)+ device = getattr(manager, "_auto_offload_device", None)+ if manager is None or device is None:+ return False+ from .diffusion_prequant import _evict_rotation_hook++ _evict_rotation_hook(manager, device)(None, None)+ return True+ except Exception: # noqa: BLE001 -- the next render's pre-hook evicts them as before+ return Falsediff --git a/studio/backend/tests/test_h3_rotation_park.py b/studio/backend/tests/test_h3_rotation_park.pynew file mode 100644index 00000000000..5fb597be3e6--- /dev/null+++ b/studio/backend/tests/test_h3_rotation_park.py@@ -0,0 +1,184 @@+# SPDX-License-Identifier: AGPL-3.0-only+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0++"""MiniMax-H3 streamed tiers: rotating VAEs park on render return; diffusers' evict-all warning is muted."""++from __future__ import annotations++import logging+import pathlib+import types++import pytest++torch = pytest.importorskip("torch")++from core.inference import video_minimax_h3_residency as residency # noqa: E402++_GB = 1024**3+++class _Sized(torch.nn.Module):+ def __init__(self, gb: float):+ super().__init__()+ self.w = torch.nn.Parameter(torch.zeros(4))+ self._gb = gb++ def get_memory_footprint(self):+ return int(self._gb * _GB)+++class _Records(logging.Handler):+ def __init__(self):+ super().__init__(logging.DEBUG)+ self.messages: list[str] = []++ def emit(self, record):+ self.messages.append(record.getMessage())+++def cm_log():+ log = logging.getLogger(residency._COMPONENTS_MANAGER_LOGGER)+ handler = _Records()+ level = log.level+ log.addHandler(handler)+ log.setLevel(logging.DEBUG)+ try:+ yield handler+ finally:+ log.removeHandler(handler)+ log.setLevel(level)+++def _real_manager(monkeypatch):+ cm = pytest.importorskip("diffusers.modular_pipelines.components_manager")+ pytest.importorskip("accelerate")+ manager = cm.ComponentsManager()+ manager.add("vae", _Sized(4.0))+ manager.add("audio_vae", _Sized(0.61))+ monkeypatch.setattr(+ torch.cuda, "mem_get_info", lambda *a, **k: (30 * _GB, 80 * _GB), raising = False+ )+ manager.enable_auto_cpu_offload(device = "cuda", memory_reserve_margin = "40GB")+ return manager+++def _hook(manager, name):+ return next(+ h+ for h in manager.model_hooks+ if h.model_id.split("_")[0] == name.split("_")[0]+ and (("audio" in h.model_id) == ("audio" in name))+ )+++def test_strategy_keeps_the_diffusers_decision_and_mutes_only_its_warning(monkeypatch, cm_log):+ manager = _real_manager(monkeypatch)+ pipe = types.SimpleNamespace(_components_manager = manager)+ vae, audio = _hook(manager, "vae"), _hook(manager, "audio_vae")+ stock = audio.hook.offload_strategy+ # 30 GB free under a 40 GB margin: no subset frees enough, so diffusers evicts every resident model.+ expected = stock(+ hooks = [vae],+ model_id = "audio_vae",+ model = audio.model,+ execution_device = torch.device("cuda:0"),+ )+ assert expected == [vae]+ assert any("no combination of models" in m for m in cm_log.messages)+ cm_log.messages.clear()++ assert residency.install_h3_rotation_park(pipe, park = True)+ quiet = audio.hook.offload_strategy+ assert isinstance(quiet, residency.H3QuietOffloadStrategy) and quiet.inner is stock+ assert manager._offload_strategy is quiet and vae.hook.offload_strategy is quiet+ assert quiet.memory_reserve_margin == stock.memory_reserve_margin+ got = quiet(+ hooks = [vae],+ model_id = "audio_vae",+ model = audio.model,+ execution_device = torch.device("cuda:0"),+ )+ assert got == expected+ assert not any("no combination of models" in m for m in cm_log.messages)+ assert any("search for models to offload" in m for m in cm_log.messages)+ assert not any(+ isinstance(f, residency._DropFallbackWarning)+ for f in logging.getLogger(residency._COMPONENTS_MANAGER_LOGGER).filters+ )+ monkeypatch.setattr(+ torch.cuda, "mem_get_info", lambda *a, **k: (60 * _GB, 80 * _GB), raising = False+ )+ assert (+ quiet(+ hooks = [vae],+ model_id = "audio_vae",+ model = audio.model,+ execution_device = torch.device("cuda:0"),+ )+ == []+ )+ assert residency.install_h3_rotation_park(pipe, park = True)+ assert audio.hook.offload_strategy is quiet+++def test_kill_switch_keeps_stock(monkeypatch, cm_log):+ monkeypatch.setenv(residency.H3_ROTATION_PARK_ENV, "0")+ manager = _real_manager(monkeypatch)+ pipe = types.SimpleNamespace(_components_manager = manager)+ stock = _hook(manager, "vae").hook.offload_strategy+ assert not residency.install_h3_rotation_park(pipe, park = True)+ assert _hook(manager, "vae").hook.offload_strategy is stock+ assert not getattr(pipe, residency._PARK_ATTR, False)+ assert not residency.park_h3_rotation(pipe)+++class _FakeHook:+ def __init__(self, name, device):+ self.model_id = name+ self.model = torch.nn.Linear(2, 2, device = device)+ self.offloads = 0+ self.hook = types.SimpleNamespace(offload_strategy = None)++ def offload(self):+ self.offloads += 1+ self.model.to_empty(device = "cpu")+++def _fake_pipe(*, parked_device = "meta"):+ vae = _FakeHook("vae", "cpu")+ audio = _FakeHook("audio_vae", parked_device) # still on the device after the last decode+ manager = types.SimpleNamespace(+ model_hooks = [vae, audio], _auto_offload_device = torch.device("meta"), _offload_strategy = None+ )+ return types.SimpleNamespace(_components_manager = manager), vae, audio+++def test_render_return_parks_what_the_next_forward_would_evict():+ pipe, vae, audio = _fake_pipe()+ logger = types.SimpleNamespace(lines = [], info = lambda msg, *a: logger.lines.append(msg % a))+ assert residency.install_h3_rotation_park(pipe, park = True, logger = logger)+ assert len(logger.lines) == 1 and "video.h3_rotation" in logger.lines[0]+ assert residency.park_h3_rotation(pipe)+ assert audio.offloads == 1 and audio.model.weight.device.type == "cpu"+ assert vae.offloads == 0+++def test_no_park_without_a_streamed_denoiser():+ pipe, _vae, audio = _fake_pipe()+ assert residency.install_h3_rotation_park(pipe, park = False)+ assert not residency.park_h3_rotation(pipe)+ assert audio.offloads == 0 and audio.model.weight.device.type == "meta"+++def test_video_wires_install_and_render_end_park():+ src = (+ pathlib.Path(__file__).resolve().parents[1] / "core" / "inference" / "video.py"+ ).read_text()+ swap = src.index("install_pinned_swap(\n")+ install = src.index("install_h3_rotation_park(pipe, park = bool(denoiser_streamed)")+ assert swap < install < swap + 800+ settle = src.index("settle_h3_vae_fallback(state, pipe)")+ park = src.index("park_h3_rotation(pipe)")+ assert settle < park < settle + 400