Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

Studio MiniMax-H3: keep only the conditioner hidden state H3 reads (about 2 GiB less at 4096 tokens) (#17095)

Daniel Han90f5bcf51ca9
diff --git a/studio/backend/core/inference/video.py b/studio/backend/core/inference/video.py
index 133a105d7a0..a454b85e7ee 100644
--- a/studio/backend/core/inference/video.py
+++ b/studio/backend/core/inference/video.py
@@ -12326,7 +12326,16 @@ class VideoBackend:
autoopt.finalize(h3_audit, speed_optims, resolved)
apply_comfy_flow_shift(pipe, getattr(fam, "comfy_flow_shift", None), logger)
from . import diffusion_prompt_cache
+ from .video_minimax_h3_te import install_h3_te_read_capture
+ try:
+ install_h3_te_read_capture(
+ pipe.text_encoder,
+ layer = getattr(pipe, "text_encoder_layer", 50),
+ logger = logger,
+ )
+ except Exception: # noqa: BLE001 - a lazy component that is not loaded
+ pass
diffusion_prompt_cache.install(
pipe,
identity = {
diff --git a/studio/backend/core/inference/video_minimax_h3_te.py b/studio/backend/core/inference/video_minimax_h3_te.py
index 2a81aecf85b..f315d28f09a 100644
--- a/studio/backend/core/inference/video_minimax_h3_te.py
+++ b/studio/backend/core/inference/video_minimax_h3_te.py
@@ -906,3 +906,78 @@ def _copy_host_tensors_(entries: list, undo: Optional[list] = None) -> int:
undo.append((group, tensor, host, new))
copied += new.numel() * new.element_size()
return copied
+
+
+# "0": the H3 conditioner records every hidden state again.
+H3_TE_READ_CAPTURE_ENV = "UNSLOTH_H3_TE_READ_CAPTURE"
+
+
+def h3_te_read_capture_enabled() -> bool:
+ import os
+ return str(os.environ.get(H3_TE_READ_CAPTURE_ENV, "")).strip().lower() not in (
+ "0",
+ "off",
+ "false",
+ "no",
+ )
+
+
+def install_h3_te_read_capture(
+ text_encoder: Any,
+ *,
+ layer: int = H3_TE_READ_LAYER,
+ logger: Any = None,
+) -> bool:
+ """H3 reads only ``hidden_states[layer]``: run without recording and keep layer ``layer``'s input via a pre-hook.
+ Past the DeepStack layers that input equals ``hidden_states[layer]`` in transformers 4.x and 5.x (5.x records
+ those layers before the visual merge), so DeepStack layers are refused. Idempotent; never raises."""
+ try:
+ model = getattr(text_encoder, "model", None)
+ layers = getattr(getattr(model, "language_model", None), "layers", None)
+ if model is None or layers is None or not isinstance(layer, int):
+ return False
+ if getattr(model, "_unsloth_h3_read_capture", None) is not None:
+ return True
+ config = getattr(text_encoder, "config", None)
+ deepstack = getattr(
+ getattr(config, "vision_config", None), "deepstack_visual_indexes", None
+ )
+ deepstack_layers = len(deepstack) if deepstack is not None else None
+ if deepstack_layers is None or not (deepstack_layers < layer < len(layers)):
+ return False
+ stock = model.forward
+ target = layers[layer]
+ except Exception: # noqa: BLE001 - optimisation only
+ return False
+
+ @functools.wraps(stock)
+ def forward(*args: Any, **kwargs: Any) -> Any:
+ if not kwargs.get("output_hidden_states") or not h3_te_read_capture_enabled():
+ return stock(*args, **kwargs)
+ captured: list = []
+
+ def _keep(_module: Any, hook_args: tuple, hook_kwargs: dict) -> None:
+ captured.append(hook_args[0] if hook_args else hook_kwargs.get("hidden_states"))
+
+ handle = target.register_forward_pre_hook(_keep, with_kwargs = True)
+ try:
+ out = stock(*args, **{**kwargs, "output_hidden_states": False})
+ finally:
+ handle.remove()
+ if len(captured) != 1 or captured[0] is None:
+ del out, captured
+ return stock(*args, **kwargs)
+ states = [None] * (len(layers) + 1)
+ states[layer] = captured[0]
+ out.hidden_states = tuple(states)
+ return out
+
+ forward.__wrapped__ = stock
+ try:
+ model.forward = forward
+ model._unsloth_h3_read_capture = layer
+ except Exception: # noqa: BLE001
+ return False
+ if logger is not None:
+ logger.info("video.h3_te_read_capture: on (keeps hidden_states[%d] only)", layer)
+ return True
diff --git a/studio/backend/tests/test_h3_te_read_capture.py b/studio/backend/tests/test_h3_te_read_capture.py
new file mode 100644
index 00000000000..7a97a3fb05e
--- /dev/null
+++ b/studio/backend/tests/test_h3_te_read_capture.py
@@ -0,0 +1,131 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
+
+"""CPU tests for ``install_h3_te_read_capture`` on a tiny random Qwen3-VL conditioner."""
+
+from __future__ import annotations
+
+import pytest
+
+torch = pytest.importorskip("torch")
+transformers = pytest.importorskip("transformers")
+
+from core.inference.video_minimax_h3_te import ( # noqa: E402
+ H3_TE_READ_CAPTURE_ENV,
+ install_h3_te_read_capture,
+)
+
+_IMAGE = 1000
+
+
+def _tiny(layers = 6, deepstack = (0, 1, 2)):
+ cls = getattr(transformers, "Qwen3VLForConditionalGeneration", None)
+ if cls is None:
+ pytest.skip("transformers without Qwen3-VL")
+ config = transformers.Qwen3VLConfig(
+ text_config = dict(
+ hidden_size = 16,
+ intermediate_size = 32,
+ num_hidden_layers = layers,
+ num_attention_heads = 4,
+ num_key_value_heads = 2,
+ head_dim = 8,
+ vocab_size = 1024,
+ rope_scaling = {
+ "mrope_interleaved": True,
+ "mrope_section": [2, 1, 1],
+ "rope_type": "default",
+ },
+ rope_theta = 10000,
+ ),
+ vision_config = dict(
+ depth = 3,
+ hidden_size = 16,
+ intermediate_size = 32,
+ num_heads = 4,
+ out_hidden_size = 16,
+ patch_size = 16,
+ spatial_merge_size = 2,
+ temporal_patch_size = 2,
+ num_position_embeddings = 64,
+ deepstack_visual_indexes = list(deepstack),
+ ),
+ image_token_id = _IMAGE,
+ video_token_id = 1001,
+ vision_start_token_id = 1002,
+ vision_end_token_id = 1003,
+ )
+ torch.manual_seed(0)
+ return cls(config).eval()
+
+
+def _inputs(with_image):
+ ids = [5, 6, 7, 8, 9]
+ kwargs = {}
+ if with_image:
+ grid = torch.tensor([[1, 4, 6]])
+ ids = [1002] + [_IMAGE] * 6 + [1003] + ids
+ kwargs = {
+ "pixel_values": torch.randn(
+ 24, 3 * 2 * 16 * 16, generator = torch.Generator().manual_seed(1)
+ ),
+ "image_grid_thw": grid,
+ }
+ input_ids = torch.tensor([ids])
+ mm = torch.tensor([[1 if i == _IMAGE else 0 for i in ids]])
+ return dict(
+ input_ids = input_ids,
+ attention_mask = torch.ones_like(input_ids),
+ mm_token_type_ids = mm,
+ use_cache = False,
+ **kwargs,
+ )
+
+
[email protected](autouse = True)
+def _env(monkeypatch):
+ monkeypatch.delenv(H3_TE_READ_CAPTURE_ENV, raising = False)
+
+
[email protected]("with_image", [False, True])
[email protected]("layer", [4, 5])
+def test_capture_returns_the_read_state_bit_for_bit_and_drops_the_rest(with_image, layer):
+ model = _tiny()
+ kw = _inputs(with_image)
+ with torch.no_grad():
+ stock = model.model(output_hidden_states = True, **kw).hidden_states
+ assert install_h3_te_read_capture(model, layer = layer)
+ assert install_h3_te_read_capture(model, layer = layer) # idempotent
+ with torch.no_grad():
+ out = model.model(output_hidden_states = True, **kw)
+ assert len(out.hidden_states) == len(stock)
+ assert torch.equal(out.hidden_states[layer], stock[layer])
+ assert all(s is None for i, s in enumerate(out.hidden_states) if i != layer)
+ with torch.no_grad():
+ plain = model.model(output_hidden_states = False, **kw)
+ assert torch.equal(plain.last_hidden_state, out.last_hidden_state)
+ # no hook is left behind on the read layer
+ assert not model.model.language_model.layers[layer]._forward_pre_hooks
+
+
+def test_kill_switch_restores_every_state(monkeypatch):
+ model = _tiny()
+ kw = _inputs(True)
+ assert install_h3_te_read_capture(model, layer = 4)
+ monkeypatch.setenv(H3_TE_READ_CAPTURE_ENV, "0")
+ with torch.no_grad():
+ out = model.model(output_hidden_states = True, **kw)
+ assert all(s is not None for s in out.hidden_states)
+
+
[email protected]("layer", [0, 3, 6, 7])
+def test_declines_deepstack_and_out_of_range_layers(layer):
+ # transformers 5.x records layers 1..3 before the DeepStack merge, which the pre-hook would see after it
+ model = _tiny()
+ assert not install_h3_te_read_capture(model, layer = layer)
+ assert getattr(model.model, "_unsloth_h3_read_capture", None) is None
+
+
+def test_declines_modules_without_a_decoder_stack():
+ assert not install_h3_te_read_capture(torch.nn.Linear(2, 2))
+ assert not install_h3_te_read_capture(None)