Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

Studio MiniMax-H3: run the text stream once per render, pack the sequence in place, look up the streamed conditioner's vocabulary rows on the host (#17203)

Daniel Hancf47cf5dd007
diff --git a/studio/backend/core/inference/video.py b/studio/backend/core/inference/video.py
index 9ef8587db6a..a77724fee9f 100644
--- a/studio/backend/core/inference/video.py
+++ b/studio/backend/core/inference/video.py
@@ -11714,6 +11714,10 @@ class VideoBackend:
**({"pretrained_model_name_or_path": offline_sources} if offline_sources else {}),
**({"revision": component_revisions} if component_revisions else {}),
)
+ # Before any offload hook wraps the denoiser's forward (the hooks then wrap this one).
+ from .video_minimax_h3_forward import install_h3_forward_on_pipe
+
+ install_h3_forward_on_pipe(pipe, logger = logger)
# Pre-cast the decoder: fp32 VAE under fp16 autocast keeps both copies; encoder stays for keyframes.
from .video_minimax_h3 import trim_h3_video_vae
@@ -13611,6 +13615,11 @@ class VideoBackend:
)
stack.enter_context(scheduler_step_preview(pipe, previewer))
stack.enter_context(_denoise_phase(pipe, _enter_denoise))
+ if fam.name == "minimax-h3":
+ from .video_minimax_h3_forward import h3_render_scope
+
+ # H3's text stream runs once inside this render, dropped after it.
+ stack.enter_context(h3_render_scope(pipe, logger))
if float_seeds is not None:
stack.enter_context(_seed_before_first_forward(float_seeds))
# No video family has a denoise-to-decode callback, so the decoder itself reports the decode phase.
diff --git a/studio/backend/core/inference/video_minimax_h3_forward.py b/studio/backend/core/inference/video_minimax_h3_forward.py
new file mode 100644
index 00000000000..ee778071f99
--- /dev/null
+++ b/studio/backend/core/inference/video_minimax_h3_forward.py
@@ -0,0 +1,322 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
+
+"""MiniMax-H3 denoiser outer forward, op for op the diffusers 0.42.1 one (installed only over the mirrored hash), with:
+
+* the text stream run once per render inside ``h3_render_scope``, keyed on the conditioning tensor and LoRA scale; an
+ inference tensor has no version counter, so an in-place edit inside a render is not seen
+ (``UNSLOTH_H3_REFINER_CACHE=0`` disables).
+* the packed sequence filled in place with ``index_copy_`` (``UNSLOTH_H3_INPLACE_PACK=0`` disables).
+"""
+
+from __future__ import annotations
+
+import contextlib
+import hashlib
+import inspect
+import os
+import threading
+import types
+from typing import Any, Iterator, Optional
+
+REFINER_CACHE_ENV = "UNSLOTH_H3_REFINER_CACHE"
+INPLACE_PACK_ENV = "UNSLOTH_H3_INPLACE_PACK"
+
+_CLASS_NAME = "MiniMaxH3Transformer3DModel"
+# sha256[:16] of inspect.getsource(MiniMaxH3Transformer3DModel.forward) this module mirrors (diffusers 0.42.1).
+_KNOWN_FORWARD_SHA = frozenset({"ae200f2868ad388f"})
+
+_PATCHED_ATTR = "_unsloth_h3_forward"
+_SCOPE_ATTR = "_unsloth_h3_text_cache"
+STATS_ATTR = "_unsloth_h3_forward_stats"
+DENOISER_COMPONENTS = ("transformer", "transformer_ref")
+
+_LOCK = threading.Lock()
+_SHA_CACHE: dict = {}
+
+
+def _switch(name: str) -> bool:
+ return (os.environ.get(name) or "").strip().lower() not in ("0", "off", "false", "no")
+
+
+def refiner_cache_enabled() -> bool:
+ return _switch(REFINER_CACHE_ENV)
+
+
+def inplace_pack_enabled() -> bool:
+ return _switch(INPLACE_PACK_ENV)
+
+
+def _forward_sha(cls: type) -> Optional[str]:
+ if cls in _SHA_CACHE:
+ return _SHA_CACHE[cls]
+ try:
+ sha = hashlib.sha256(inspect.getsource(cls.forward).encode()).hexdigest()[:16]
+ except Exception: # noqa: BLE001
+ sha = None
+ _SHA_CACHE[cls] = sha
+ return sha
+
+
+def _stats(module: Any) -> dict:
+ stats = module.__dict__.get(STATS_ATTR)
+ if stats is None:
+ stats = {"text_hits": 0, "text_misses": 0, "inplace_packs": 0, "stock": 0}
+ module.__dict__[STATS_ATTR] = stats
+ return stats
+
+
+class _TextCache:
+ """One render's text-stream rows: the conditioning tensor they came from (held, so its storage cannot be
+ reused by another tensor while the entry lives) and everything else that decides them."""
+
+ __slots__ = ("source", "key", "value")
+
+ def __init__(self) -> None:
+ self.source = None
+ self.key = None
+ self.value = None
+
+
+def _source_key(t: Any, dtype: Any) -> tuple:
+ version = None if t.is_inference() else t._version # inference tensors carry no version counter
+ return (t.data_ptr(), tuple(t.shape), t.dtype, str(t.device), version, dtype)
+
+
+def _text_rows(
+ self: Any,
+ encoder_hidden_states: Any,
+ lora_scale: Any = None,
+) -> Any:
+ from diffusers.models.modeling_utils import get_parameter_dtype
+
+ dtype = get_parameter_dtype(self.context_embedder)
+ cache = self.__dict__.get(_SCOPE_ATTR)
+ if cache is None or not refiner_cache_enabled():
+ text_embeds = self.context_embedder(encoder_hidden_states.to(dtype))
+ return self.token_refiner(text_embeds)
+ # the LoRA scale apply_lora_scale applied to this call scales the refiner's adapters too
+ key = (_source_key(encoder_hidden_states, dtype), lora_scale)
+ if cache.value is not None and cache.source is encoder_hidden_states and cache.key == key:
+ _stats(self)["text_hits"] += 1
+ return cache.value
+ text_embeds = self.context_embedder(encoder_hidden_states.to(dtype))
+ text_embeds = self.token_refiner(text_embeds)
+ cache.source, cache.key, cache.value = encoder_hidden_states, key, text_embeds
+ _stats(self)["text_misses"] += 1
+ return text_embeds
+
+
+def _body(
+ self: Any,
+ hidden_states: Any,
+ audio_hidden_states: Any,
+ encoder_hidden_states: Any,
+ timestep: Any,
+ timestep_indices: Any,
+ token_tags: Any,
+ position_ids: Any,
+ video_indices: Any,
+ audio_indices: Any,
+ text_indices: Any,
+ attention_kwargs: Optional[dict] = None,
+ return_dict: bool = True,
+ _unsloth_lora_scale: Any = None,
+) -> Any:
+ """``MiniMaxH3Transformer3DModel.forward`` (diffusers 0.42.1) op for op, except the text stream and the pack."""
+ from diffusers.models.modeling_utils import get_parameter_dtype
+ from diffusers.models.transformers.transformer_minimax_h3 import (
+ MINIMAX_H3_MODALITY_NUM,
+ MiniMaxH3TransformerOutput,
+ )
+
+ if position_ids.ndim != 2 or position_ids.shape[-1] != 3:
+ raise ValueError(
+ f"`position_ids` must be a `(seq_len, 3)` tensor, got {list(position_ids.shape)}."
+ )
+ sequence_length = position_ids.shape[0]
+ if token_tags.shape != (sequence_length,) or timestep_indices.shape != (sequence_length,):
+ raise ValueError(
+ "`token_tags` and `timestep_indices` must both be `(seq_len,)` tensors matching `position_ids`, got "
+ f"{list(token_tags.shape)} and {list(timestep_indices.shape)} for seq_len={sequence_length}."
+ )
+
+ rotary_emb = self.rope(position_ids)
+
+ video_embeds = self.proj_in(hidden_states.to(get_parameter_dtype(self.proj_in)))
+ audio_embeds = self.audio_proj_in(
+ audio_hidden_states.to(get_parameter_dtype(self.audio_proj_in))
+ )
+ text_embeds = _text_rows(self, encoder_hidden_states, _unsloth_lora_scale)
+
+ hidden_states = text_embeds.new_zeros(
+ (text_embeds.shape[0], sequence_length, text_embeds.shape[-1])
+ )
+ if inplace_pack_enabled():
+ hidden_states.index_copy_(1, text_indices, text_embeds)
+ hidden_states.index_copy_(1, video_indices, video_embeds.to(text_embeds.dtype))
+ hidden_states.index_copy_(1, audio_indices, audio_embeds.to(text_embeds.dtype))
+ _stats(self)["inplace_packs"] += 1
+ else:
+ hidden_states = hidden_states.index_copy(1, text_indices, text_embeds)
+ hidden_states = hidden_states.index_copy(
+ 1, video_indices, video_embeds.to(text_embeds.dtype)
+ )
+ hidden_states = hidden_states.index_copy(
+ 1, audio_indices, audio_embeds.to(text_embeds.dtype)
+ )
+ del video_embeds, audio_embeds
+
+ temb = self.time_proj(timestep)
+ temb = self.time_embedder(temb.to(get_parameter_dtype(self.time_embedder)))
+
+ adaln_indices = timestep_indices * MINIMAX_H3_MODALITY_NUM + token_tags
+
+ for block in self.transformer_blocks:
+ hidden_states = block(hidden_states, temb, adaln_indices, rotary_emb)
+
+ hidden_states = self.norm_out(hidden_states, temb, timestep_indices).to(
+ get_parameter_dtype(self.proj_out)
+ )
+ video_output = self.proj_out(hidden_states).index_select(1, video_indices)
+ audio_output = self.audio_proj_out(hidden_states).index_select(1, audio_indices)
+
+ if not return_dict:
+ return (video_output, audio_output)
+ return MiniMaxH3TransformerOutput(sample = video_output, audio_sample = audio_output)
+
+
+_SCALED_BODY = None
+
+
+def _scaled_body() -> Any:
+ """``_body`` under the same ``apply_lora_scale`` the stock forward carries (built once, lazily)."""
+ global _SCALED_BODY
+ if _SCALED_BODY is None:
+ from diffusers.utils import apply_lora_scale
+ _SCALED_BODY = apply_lora_scale("attention_kwargs")(_body)
+ return _SCALED_BODY
+
+
+def _forward(
+ self: Any,
+ hidden_states: Any,
+ audio_hidden_states: Any,
+ encoder_hidden_states: Any,
+ timestep: Any,
+ timestep_indices: Any,
+ token_tags: Any,
+ position_ids: Any,
+ video_indices: Any,
+ audio_indices: Any,
+ text_indices: Any,
+ attention_kwargs: Optional[dict] = None,
+ return_dict: bool = True,
+) -> Any:
+ import torch
+
+ use_cache = self.__dict__.get(_SCOPE_ATTR) is not None and refiner_cache_enabled()
+ extra = {}
+ if torch.is_grad_enabled() or not (use_cache or inplace_pack_enabled()):
+ _stats(self)["stock"] += 1
+ fn = type(self).forward
+ else:
+ fn = _scaled_body()
+ # apply_lora_scale pops the scale before the body runs; the text cache keys on it
+ extra["_unsloth_lora_scale"] = (
+ attention_kwargs.get("scale") if isinstance(attention_kwargs, dict) else None
+ )
+ return fn(
+ self,
+ hidden_states = hidden_states,
+ audio_hidden_states = audio_hidden_states,
+ encoder_hidden_states = encoder_hidden_states,
+ timestep = timestep,
+ timestep_indices = timestep_indices,
+ token_tags = token_tags,
+ position_ids = position_ids,
+ video_indices = video_indices,
+ audio_indices = audio_indices,
+ text_indices = text_indices,
+ attention_kwargs = attention_kwargs,
+ return_dict = return_dict,
+ **extra,
+ )
+
+
+def _hooked(module: Any) -> bool:
+ return getattr(module, "_hf_hook", None) is not None or bool(
+ getattr(getattr(module, "_diffusers_hook", None), "hooks", None)
+ )
+
+
+def install_h3_forward(transformer: Any, logger: Any = None) -> bool:
+ """Bind the mirrored forward before any offload hook wraps ``forward``; idempotent, never raises, False = stock."""
+ try:
+ if transformer is None or type(transformer).__name__ != _CLASS_NAME:
+ return False
+ if transformer.__dict__.get(_PATCHED_ATTR):
+ return True
+ if not (refiner_cache_enabled() or inplace_pack_enabled()):
+ return False
+ with _LOCK:
+ if _forward_sha(type(transformer)) not in _KNOWN_FORWARD_SHA:
+ if logger is not None:
+ logger.info("video.h3_forward: unknown denoiser forward source, keeping stock")
+ return False
+ if "forward" in transformer.__dict__ or _hooked(transformer):
+ if logger is not None:
+ logger.info("video.h3_forward: denoiser forward already wrapped, keeping stock")
+ return False
+ transformer.forward = types.MethodType(_forward, transformer)
+ transformer.__dict__[_PATCHED_ATTR] = True
+ _stats(transformer)
+ except Exception as exc: # noqa: BLE001 -- an optimisation, never fail a load
+ if logger is not None:
+ logger.warning("video.h3_forward: install skipped: %s", exc)
+ return False
+ if logger is not None:
+ logger.info(
+ "video.h3_forward: text stream once per render (%s), packed sequence in place (%s)",
+ "on" if refiner_cache_enabled() else "off",
+ "on" if inplace_pack_enabled() else "off",
+ )
+ return True
+
+
+def install_h3_forward_on_pipe(pipe: Any, logger: Any = None) -> int:
+ return sum(
+ 1
+ for name in DENOISER_COMPONENTS
+ if install_h3_forward(getattr(pipe, name, None), logger = logger)
+ )
+
+
+def h3_render_scope(pipe: Any, logger: Any = None) -> Iterator[None]:
+ """One render: the text-stream cache exists only inside, and is dropped on exit or error."""
+ modules = [
+ m
+ for m in (getattr(pipe, name, None) for name in DENOISER_COMPONENTS)
+ if m is not None and m.__dict__.get(_PATCHED_ATTR)
+ ]
+ before = []
+ for m in modules:
+ m.__dict__[_SCOPE_ATTR] = _TextCache()
+ before.append(dict(_stats(m)))
+ try:
+ yield
+ finally:
+ for m, start in zip(modules, before):
+ m.__dict__.pop(_SCOPE_ATTR, None)
+ if logger is not None:
+ now = _stats(m)
+ delta = {k: now[k] - start.get(k, 0) for k in now}
+ logger.info(
+ "video.h3_forward: render ran the text stream %d time(s), reused it %d time(s), "
+ "%d in-place packs, %d stock calls",
+ delta["text_misses"],
+ delta["text_hits"],
+ delta["inplace_packs"],
+ delta["stock"],
+ )
diff --git a/studio/backend/core/inference/video_minimax_h3_te.py b/studio/backend/core/inference/video_minimax_h3_te.py
index f315d28f09a..ae3680139cf 100644
--- a/studio/backend/core/inference/video_minimax_h3_te.py
+++ b/studio/backend/core/inference/video_minimax_h3_te.py
@@ -606,6 +606,115 @@ def _fill_arena(arena: Any, *, entries: list) -> None:
list(pool.map(run, jobs))
+# The streamed conditioner's vocabulary table (151936 x 5120 bf16, 1.56 GB) stays on the host: the prompt's rows are
+# gathered there and only they are uploaded, not the whole table per encode. ``UNSLOTH_H3_HOST_EMBED_ROWS=0`` streams
+# the table like every other leaf.
+H3_HOST_EMBED_ROWS_ENV = "UNSLOTH_H3_HOST_EMBED_ROWS"
+_HOST_ROWS_ATTR = "_unsloth_host_rows_device"
+
+
+def h3_host_embed_rows_enabled() -> bool:
+ import os
+ return str(os.environ.get(H3_HOST_EMBED_ROWS_ENV, "")).strip().lower() not in (
+ "0",
+ "off",
+ "false",
+ "no",
+ )
+
+
+def _host_rows_forward(self: Any, input: Any) -> Any:
+ """``nn.Embedding.forward`` on the host table (a gather: bit-identical), the rows sent to the onload device."""
+ import torch.nn.functional as F
+
+ weight = self.weight
+ ids = input if input.device == weight.device else input.to(weight.device)
+ rows = F.embedding(
+ ids,
+ weight,
+ self.padding_idx,
+ self.max_norm,
+ self.norm_type,
+ self.scale_grad_by_freq,
+ self.sparse,
+ )
+ return rows.to(self.__dict__[_HOST_ROWS_ATTR])
+
+
+def keep_h3_embedding_on_host(target: Any, logger: Any = None) -> bool:
+ """Take the group-offloaded language embedding out of the stream, its lookup done on the host. Run right after
+ ``apply_group_offloading``; True when installed. Plain frozen bf16/fp16/fp32 tables only. Never raises."""
+ try:
+ if not h3_host_embed_rows_enabled():
+ return False
+ import types
+
+ import torch
+
+ getter = getattr(target, "get_input_embeddings", None)
+ emb = getter() if callable(getter) else None
+ if type(emb) is not torch.nn.Embedding or _HOST_ROWS_ATTR in emb.__dict__:
+ return False
+ weight = emb.weight
+ if (
+ type(weight.data) is not torch.Tensor
+ or weight.device.type != "cpu"
+ or weight.requires_grad
+ or emb.max_norm is not None
+ or emb.sparse
+ or weight.dtype not in (torch.bfloat16, torch.float16, torch.float32)
+ ):
+ return False
+ registry = getattr(emb, "_diffusers_hook", None)
+ hooks = getattr(registry, "hooks", None) or {}
+ # a streamed (lazy prefetch) setup also tracks the leaf's first execution; anything else keeps the stock leaf
+ if "group_offloading" not in hooks or not set(hooks) <= {
+ "group_offloading",
+ "layer_execution_tracker",
+ }:
+ return False
+ group = getattr(hooks["group_offloading"], "group", None)
+ if group is None or list(getattr(group, "modules", None) or ()) != [emb]:
+ return False
+ onload = getattr(group, "onload_device", None)
+ if onload is None:
+ return False
+ name = next((n for n, m in target.named_modules() if m is emb), None)
+ lazy = (getattr(getattr(target, "_diffusers_hook", None), "hooks", None) or {}).get(
+ "lazy_prefetch_group_offloading"
+ )
+ emb.__dict__[_HOST_ROWS_ATTR] = torch.device(onload)
+ try:
+ if "layer_execution_tracker" in hooks:
+ registry.remove_hook("layer_execution_tracker", recurse = False)
+ registry.remove_hook("group_offloading", recurse = False)
+ finally:
+ # set even if a removal failed half way: the table is on the host, so the lookup must be too
+ emb.forward = types.MethodType(_host_rows_forward, emb)
+ names = getattr(lazy, "_layer_execution_tracker_module_names", None)
+ if isinstance(names, set) and name is not None:
+ names.discard(name) # no longer streamed: the first forward must not expect it
+ # diffusers may hold the table's pinned twin; the lookup reads what the parameter points at now
+ host = getattr(group, "cpu_param_dict", None) or {}
+ pinned = host.get(weight)
+ if (
+ pinned is not None
+ and pinned.device.type == "cpu"
+ and weight.data_ptr() != pinned.data_ptr()
+ ):
+ weight.data = pinned
+ except Exception as exc: # noqa: BLE001 -- the streamed leaf still works
+ if logger is not None:
+ logger.warning("video.h3_te_stream: embedding stays streamed: %s", exc)
+ return False
+ if logger is not None:
+ logger.info(
+ "video.h3_te_stream: vocabulary table (%.2f GB) stays on the host, prompt rows uploaded",
+ weight.numel() * weight.element_size() / 1e9,
+ )
+ return True
+
+
def stream_h3_text_encoder(
manager: Any,
text_encoder: Any,
@@ -677,6 +786,7 @@ def stream_h3_text_encoder(
install_group_offload_buffer_restore()
install_group_offload_hooks_eager()
apply_group_offloading(target, **kwargs)
+ keep_h3_embedding_on_host(target, logger = logger)
# Image prompts: Qwen3-VL's position interpolation reads the offloaded embedding's CPU device (transformers 5.5).
_pin_vision_embedding_device(target)
except Exception as exc: # noqa: BLE001 -- keep the rotation
diff --git a/studio/backend/tests/test_h3_forward.py b/studio/backend/tests/test_h3_forward.py
new file mode 100644
index 00000000000..11d1a9ba50f
--- /dev/null
+++ b/studio/backend/tests/test_h3_forward.py
@@ -0,0 +1,326 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
+
+"""CPU tests for the mirrored MiniMax-H3 denoiser forward (text stream once per render, packed sequence in place)
+on a tiny random diffusers H3 transformer."""
+
+from __future__ import annotations
+
+import inspect
+
+import pytest
+
+torch = pytest.importorskip("torch")
+h3_mod = pytest.importorskip("diffusers.models.transformers.transformer_minimax_h3")
+
+from core.inference import video_minimax_h3_forward as h3f # noqa: E402
+
+needs_mirror = pytest.mark.skipif(
+ h3f._forward_sha(h3_mod.MiniMaxH3Transformer3DModel) not in h3f._KNOWN_FORWARD_SHA,
+ reason = "installed diffusers' H3 forward is not the mirrored source",
+)
+
+T_ROWS, V_ROWS, A_ROWS = 5, 12, 4
+
+
+def _model():
+ torch.manual_seed(0)
+ m = h3_mod.MiniMaxH3Transformer3DModel(
+ num_attention_heads = 2,
+ attention_head_dim = 16,
+ hidden_size = 32,
+ num_layers = 2,
+ num_refiner_layers = 2,
+ ffn_dim = 64,
+ in_channels = 4,
+ audio_in_channels = 6,
+ patch_size = (1, 1, 1),
+ text_dim = 24,
+ freq_dim = 16,
+ time_embed_hidden_dim = 32,
+ time_embed_dim = 16,
+ rope_freq_dim = 2,
+ ).eval()
+ with torch.no_grad():
+ for p in m.parameters():
+ p.copy_(torch.randn_like(p) * 0.1)
+ return m
+
+
+def _layout():
+ s = T_ROWS + V_ROWS + A_ROWS
+ perm = torch.randperm(s, generator = torch.Generator().manual_seed(1))
+ text_indices = perm[:T_ROWS].sort().values
+ video_indices = perm[T_ROWS : T_ROWS + V_ROWS].sort().values
+ audio_indices = perm[T_ROWS + V_ROWS :].sort().values
+ token_tags = torch.zeros(s, dtype = torch.long)
+ token_tags[video_indices] = 1
+ token_tags[audio_indices] = 2
+ position_ids = torch.randint(0, 6, (s, 3), generator = torch.Generator().manual_seed(2))
+ timestep_indices = torch.zeros(s, dtype = torch.long)
+ timestep_indices[video_indices[:3]] = 1
+ return dict(
+ token_tags = token_tags,
+ position_ids = position_ids,
+ video_indices = video_indices,
+ audio_indices = audio_indices,
+ text_indices = text_indices,
+ timestep_indices = timestep_indices,
+ )
+
+
+def _step_inputs(i, enc):
+ g = torch.Generator().manual_seed(100 + i)
+ return dict(
+ hidden_states = torch.randn(1, V_ROWS, 4, generator = g),
+ audio_hidden_states = torch.randn(1, A_ROWS, 6, generator = g),
+ encoder_hidden_states = enc,
+ timestep = torch.tensor([1000.0 - 100 * i, 990.0]),
+ return_dict = False,
+ )
+
+
+def _run(
+ model,
+ enc,
+ steps = 4,
+ layout = None,
+):
+ layout = layout or _layout()
+ outs = []
+ with torch.no_grad():
+ for i in range(steps):
+ outs.append(model(**_step_inputs(i, enc), **layout))
+ return outs
+
+
+def _enc(seed = 3):
+ return torch.randn(1, T_ROWS, 24, generator = torch.Generator().manual_seed(seed))
+
+
+class _Pipe:
+ def __init__(self, transformer):
+ self.transformer = transformer
+
+
+def _assert_same(a, b):
+ assert len(a) == len(b)
+ for (va, aa), (vb, ab) in zip(a, b):
+ assert torch.equal(va, vb) and torch.equal(aa, ab)
+
+
+@needs_mirror
+def test_bit_identical_and_text_stream_runs_once_per_render(monkeypatch):
+ monkeypatch.delenv(h3f.REFINER_CACHE_ENV, raising = False)
+ monkeypatch.delenv(h3f.INPLACE_PACK_ENV, raising = False)
+ stock = _run(_model(), _enc())
+ m = _model()
+ calls = {"refiner": 0, "embedder": 0}
+ m.token_refiner.register_forward_hook(
+ lambda *a: calls.__setitem__("refiner", calls["refiner"] + 1)
+ )
+ m.context_embedder.register_forward_hook(
+ lambda *a: calls.__setitem__("embedder", calls["embedder"] + 1)
+ )
+ assert h3f.install_h3_forward(m) is True
+ assert h3f.install_h3_forward(m) is True # idempotent
+ with h3f.h3_render_scope(_Pipe(m)):
+ got = _run(m, _enc())
+ _assert_same(stock, got)
+ assert calls == {"refiner": 1, "embedder": 1}
+ stats = m.__dict__[h3f.STATS_ATTR]
+ assert stats["text_misses"] == 1 and stats["text_hits"] == 3 and stats["inplace_packs"] == 4
+ # the scope is gone after the render: nothing holds the conditioning
+ assert h3f._SCOPE_ATTR not in m.__dict__
+
+
+@needs_mirror
+def test_cache_only_inside_a_render_and_keyed_on_the_conditioning(monkeypatch):
+ monkeypatch.delenv(h3f.REFINER_CACHE_ENV, raising = False)
+ m = _model()
+ h3f.install_h3_forward(m)
+ stats = m.__dict__[h3f.STATS_ATTR]
+ _run(m, _enc(), steps = 2) # no render scope: recomputed each call
+ assert stats["text_misses"] == 0 and stats["text_hits"] == 0
+ stock = _model()
+ layout = _layout()
+ with h3f.h3_render_scope(_Pipe(m)):
+ a = _run(m, _enc(3), steps = 2, layout = layout)
+ b = _run(
+ m, _enc(4), steps = 2, layout = layout
+ ) # another caption tensor: a miss, not the old rows
+ assert stats["text_misses"] == 2 and stats["text_hits"] == 2
+ _assert_same(a, _run(stock, _enc(3), steps = 2, layout = layout))
+ _assert_same(b, _run(stock, _enc(4), steps = 2, layout = layout))
+ # an in-place edit of the same tensor (normal tensor: version counter) misses too
+ enc = _enc(5)
+ with h3f.h3_render_scope(_Pipe(m)):
+ _run(m, enc, steps = 1, layout = layout)
+ enc.mul_(2.0)
+ got = _run(m, enc, steps = 1, layout = layout)
+ _assert_same(got, _run(stock, enc, steps = 1, layout = layout))
+ assert stats["text_misses"] == 4
+
+
+@needs_mirror
+def test_inference_mode_render(monkeypatch):
+ monkeypatch.delenv(h3f.REFINER_CACHE_ENV, raising = False)
+ m, stock = _model(), _model()
+ h3f.install_h3_forward(m)
+ layout = _layout()
+ with torch.inference_mode():
+ enc = _enc()
+ ref = [stock(**_step_inputs(i, enc), **layout) for i in range(3)]
+ with h3f.h3_render_scope(_Pipe(m)):
+ got = [m(**_step_inputs(i, enc), **layout) for i in range(3)]
+ _assert_same(ref, got)
+ assert m.__dict__[h3f.STATS_ATTR]["text_hits"] == 2
+
+
+@needs_mirror
[email protected]("env", [h3f.REFINER_CACHE_ENV, h3f.INPLACE_PACK_ENV])
+def test_kill_switches(monkeypatch, env):
+ monkeypatch.setenv(env, "0")
+ m = _model()
+ assert h3f.install_h3_forward(m) is True
+ with h3f.h3_render_scope(_Pipe(m)):
+ got = _run(m, _enc())
+ _assert_same(_run(_model(), _enc()), got)
+ stats = m.__dict__[h3f.STATS_ATTR]
+ if env == h3f.REFINER_CACHE_ENV:
+ assert stats["text_misses"] == 0 and stats["text_hits"] == 0 and stats["inplace_packs"] == 4
+ else:
+ assert stats["text_hits"] == 3 and stats["inplace_packs"] == 0
+ monkeypatch.setenv(h3f.REFINER_CACHE_ENV, "0")
+ monkeypatch.setenv(h3f.INPLACE_PACK_ENV, "0")
+ fresh = _model()
+ assert h3f.install_h3_forward(fresh) is False
+ assert "forward" not in fresh.__dict__
+
+
+@needs_mirror
+def test_no_allocating_full_sequence_copies(monkeypatch):
+ from torch.utils._python_dispatch import TorchDispatchMode
+
+ class Count(TorchDispatchMode):
+ def __init__(self):
+ super().__init__()
+ self.ops = {}
+
+ def __torch_dispatch__(
+ self,
+ func,
+ types,
+ args = (),
+ kwargs = None,
+ ):
+ name = str(func.overloadpacket)
+ self.ops[name] = self.ops.get(name, 0) + 1
+ return func(*args, **(kwargs or {}))
+
+ def count(model, scope):
+ with Count() as c:
+ if scope:
+ with h3f.h3_render_scope(_Pipe(model)):
+ _run(model, _enc(), steps = 2)
+ else:
+ _run(model, _enc(), steps = 2)
+ return c.ops
+
+ monkeypatch.delenv(h3f.INPLACE_PACK_ENV, raising = False)
+ stock = count(_model(), False)
+ m = _model()
+ h3f.install_h3_forward(m)
+ patched = count(m, True)
+ assert stock.get("aten.index_copy", 0) == 6 and stock.get("aten.index_copy_", 0) == 0
+ assert patched.get("aten.index_copy", 0) == 0 and patched.get("aten.index_copy_", 0) == 6
+ # the refiner's attention + MLP ran for one of the two steps only
+ assert patched.get("aten.mm", 0) + patched.get("aten.addmm", 0) < stock.get(
+ "aten.mm", 0
+ ) + stock.get("aten.addmm", 0)
+
+
+@needs_mirror
+def test_training_and_grad_keep_stock(monkeypatch):
+ m = _model()
+ h3f.install_h3_forward(m)
+ layout = _layout()
+ with h3f.h3_render_scope(_Pipe(m)):
+ v, a = m(**_step_inputs(0, _enc()), **layout)
+ assert v.requires_grad
+ stats = m.__dict__[h3f.STATS_ATTR]
+ assert stats["stock"] == 1 and stats["text_misses"] == 0
+
+
+@needs_mirror
+def test_signature_and_hooks_wrap_the_mirrored_forward():
+ from diffusers.hooks import HookRegistry, ModelHook
+
+ m = _model()
+ h3f.install_h3_forward(m)
+ stock_params = list(inspect.signature(h3_mod.MiniMaxH3Transformer3DModel.forward).parameters)
+ # the modular denoise step filters its layout kwargs by this signature
+ assert list(inspect.signature(m.forward).parameters) == stock_params[1:]
+
+ seen = []
+
+ class Pre(ModelHook):
+ def pre_forward(self, module, *args, **kwargs):
+ seen.append(1)
+ return args, kwargs
+
+ HookRegistry.check_if_exists_or_initialize(m).register_hook(Pre(), "probe")
+ with h3f.h3_render_scope(_Pipe(m)):
+ _run(m, _enc(), steps = 2)
+ assert seen == [1, 1] and m.__dict__[h3f.STATS_ATTR]["text_hits"] == 1
+
+ hooked = _model()
+ HookRegistry.check_if_exists_or_initialize(hooked).register_hook(Pre(), "probe")
+ assert h3f.install_h3_forward(hooked) is False # never under an existing hook
+
+
+def test_unknown_source_and_other_classes_keep_stock(monkeypatch):
+ class Other(torch.nn.Module):
+ def forward(self, x):
+ return x
+
+ assert h3f.install_h3_forward(Other()) is False
+ assert h3f.install_h3_forward(None) is False
+ m = _model()
+ monkeypatch.setattr(h3f, "_KNOWN_FORWARD_SHA", frozenset({"0" * 16}))
+ assert h3f.install_h3_forward(m) is False
+ assert "forward" not in m.__dict__
+
+
+@needs_mirror
+def test_regional_compile_adds_no_graphs(monkeypatch):
+ from torch._dynamo.testing import CompileCounter
+
+ def graphs(install):
+ torch._dynamo.reset()
+ m = _model()
+ if install:
+ h3f.install_h3_forward(m)
+ cnt = CompileCounter()
+ m.compile_repeated_blocks(backend = cnt, fullgraph = True, dynamic = False)
+ with h3f.h3_render_scope(_Pipe(m)):
+ _run(m, _enc(), steps = 4)
+ return cnt.frame_count
+
+ stock, patched = graphs(False), graphs(True)
+ assert patched <= stock
+
+
+@needs_mirror
+def test_lora_scale_is_part_of_the_key(monkeypatch):
+ monkeypatch.delenv(h3f.REFINER_CACHE_ENV, raising = False)
+ m = _model()
+ h3f.install_h3_forward(m)
+ layout = _layout()
+ enc = _enc()
+ with torch.no_grad(), h3f.h3_render_scope(_Pipe(m)):
+ m(**_step_inputs(0, enc), **layout)
+ m(**_step_inputs(1, enc), **layout, attention_kwargs = {"scale": 0.5})
+ m(**_step_inputs(2, enc), **layout, attention_kwargs = {"scale": 0.5})
+ stats = m.__dict__[h3f.STATS_ATTR]
+ assert stats["text_misses"] == 2 and stats["text_hits"] == 1
diff --git a/studio/backend/tests/test_h3_te_host_rows.py b/studio/backend/tests/test_h3_te_host_rows.py
new file mode 100644
index 00000000000..86392746b1b
--- /dev/null
+++ b/studio/backend/tests/test_h3_te_host_rows.py
@@ -0,0 +1,121 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
+
+"""CPU tests: the streamed MiniMax-H3 conditioner's vocabulary table stays on the host (bit-identical lookups)."""
+
+from __future__ import annotations
+
+import pytest
+
+torch = pytest.importorskip("torch")
+
+from core.inference import video_minimax_h3_te as te # noqa: E402
+
+
+class _TinyLM(torch.nn.Module):
+ def __init__(self):
+ super().__init__()
+ torch.manual_seed(0)
+ self.embed_tokens = torch.nn.Embedding(50, 8, padding_idx = 0).to(torch.bfloat16)
+ self.proj = torch.nn.Linear(8, 8).to(torch.bfloat16)
+
+ def get_input_embeddings(self):
+ return self.embed_tokens
+
+ def forward(self, input_ids):
+ return self.proj(self.embed_tokens(input_ids))
+
+
+def _streamed(install):
+ from diffusers.hooks import apply_group_offloading
+
+ m = _TinyLM().requires_grad_(False)
+ apply_group_offloading(
+ m,
+ onload_device = torch.device("cpu"),
+ offload_device = torch.device("cpu"),
+ offload_type = "leaf_level",
+ use_stream = False,
+ )
+ installed = te.keep_h3_embedding_on_host(m) if install else False
+ return m, installed
+
+
+def test_host_rows_bit_identical_and_out_of_the_stream(monkeypatch):
+ pytest.importorskip("diffusers.hooks")
+ monkeypatch.delenv(te.H3_HOST_EMBED_ROWS_ENV, raising = False)
+ ref_mod, _ = _streamed(False)
+ m, installed = _streamed(True)
+ assert installed is True
+ assert "group_offloading" not in (m.embed_tokens._diffusers_hook.hooks or {})
+ assert "group_offloading" in m.proj._diffusers_hook.hooks # every other leaf still streams
+ ids = torch.tensor([[0, 3, 7, 49, 3]])
+ with torch.no_grad():
+ assert torch.equal(ref_mod(ids), m(ids))
+ assert torch.equal(ref_mod.embed_tokens(ids), m.embed_tokens(ids))
+ assert m.embed_tokens.weight.device.type == "cpu"
+ assert (
+ te.keep_h3_embedding_on_host(m) is False
+ ) # already taken out: a second call changes nothing
+
+
+def test_host_rows_kill_switch_and_refusals(monkeypatch):
+ pytest.importorskip("diffusers.hooks")
+ monkeypatch.setenv(te.H3_HOST_EMBED_ROWS_ENV, "0")
+ m, installed = _streamed(True)
+ assert installed is False and "group_offloading" in m.embed_tokens._diffusers_hook.hooks
+ monkeypatch.delenv(te.H3_HOST_EMBED_ROWS_ENV)
+ # not group offloaded: nothing to take out
+ plain = _TinyLM()
+ assert te.keep_h3_embedding_on_host(plain) is False
+ assert "forward" not in plain.embed_tokens.__dict__
+
+
+def test_host_rows_under_lazy_prefetch_tracking(monkeypatch):
+ pytest.importorskip("diffusers.hooks")
+ from diffusers.hooks import group_offloading as go
+
+ monkeypatch.delenv(te.H3_HOST_EMBED_ROWS_ENV, raising = False)
+ from diffusers.hooks import apply_group_offloading
+
+ def build():
+ m = _TinyLM().requires_grad_(False)
+ apply_group_offloading(
+ m,
+ onload_device = torch.device("cpu"),
+ offload_device = torch.device("cpu"),
+ offload_type = "leaf_level",
+ use_stream = False,
+ )
+ # what use_stream=True adds (diffusers' own helper): a root group + a lazy prefetch hook tracking every leaf
+ cpu = torch.device("cpu")
+ config = go.GroupOffloadingConfig(
+ onload_device = cpu,
+ offload_device = cpu,
+ offload_type = go.GroupOffloadingType.LEAF_LEVEL,
+ non_blocking = False,
+ record_stream = False,
+ low_cpu_mem_usage = False,
+ )
+ root = go.ModuleGroup(
+ modules = [],
+ offload_device = cpu,
+ onload_device = cpu,
+ offload_leader = m,
+ onload_leader = m,
+ onload_self = True,
+ group_id = "lazy_leafs",
+ )
+ go._apply_lazy_group_offloading_hook(m, root, config = config)
+ return m
+
+ ref_mod, m = build(), build()
+ assert "layer_execution_tracker" in m.embed_tokens._diffusers_hook.hooks
+ assert te.keep_h3_embedding_on_host(m) is True
+ assert not m.embed_tokens._diffusers_hook.hooks
+ lazy = m._diffusers_hook.hooks["lazy_prefetch_group_offloading"]
+ assert "embed_tokens" not in lazy._layer_execution_tracker_module_names
+ ids = torch.tensor([[1, 2, 3]])
+ with torch.no_grad():
+ for _ in range(2): # the first forward builds the prefetch chain, the second uses it
+ assert torch.equal(ref_mod(ids), m(ids))