Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
Studio MiniMax-H3: run the text stream once per render, pack the sequence in place, look up the streamed conditioner's vocabulary rows on the host (#17203)
Daniel Hancf47cf5dd007
diff --git a/studio/backend/core/inference/video.py b/studio/backend/core/inference/video.pyindex 9ef8587db6a..a77724fee9f 100644--- a/studio/backend/core/inference/video.py+++ b/studio/backend/core/inference/video.py@@ -11714,6 +11714,10 @@ class VideoBackend:**({"pretrained_model_name_or_path": offline_sources} if offline_sources else {}),**({"revision": component_revisions} if component_revisions else {}),)+ # Before any offload hook wraps the denoiser's forward (the hooks then wrap this one).+ from .video_minimax_h3_forward import install_h3_forward_on_pipe++ install_h3_forward_on_pipe(pipe, logger = logger)# Pre-cast the decoder: fp32 VAE under fp16 autocast keeps both copies; encoder stays for keyframes.from .video_minimax_h3 import trim_h3_video_vae@@ -13611,6 +13615,11 @@ class VideoBackend:)stack.enter_context(scheduler_step_preview(pipe, previewer))stack.enter_context(_denoise_phase(pipe, _enter_denoise))+ if fam.name == "minimax-h3":+ from .video_minimax_h3_forward import h3_render_scope++ # H3's text stream runs once inside this render, dropped after it.+ stack.enter_context(h3_render_scope(pipe, logger))if float_seeds is not None:stack.enter_context(_seed_before_first_forward(float_seeds))# No video family has a denoise-to-decode callback, so the decoder itself reports the decode phase.diff --git a/studio/backend/core/inference/video_minimax_h3_forward.py b/studio/backend/core/inference/video_minimax_h3_forward.pynew file mode 100644index 00000000000..ee778071f99--- /dev/null+++ b/studio/backend/core/inference/video_minimax_h3_forward.py@@ -0,0 +1,322 @@+# SPDX-License-Identifier: AGPL-3.0-only+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0++"""MiniMax-H3 denoiser outer forward, op for op the diffusers 0.42.1 one (installed only over the mirrored hash), with:++* the text stream run once per render inside ``h3_render_scope``, keyed on the conditioning tensor and LoRA scale; an+ inference tensor has no version counter, so an in-place edit inside a render is not seen+ (``UNSLOTH_H3_REFINER_CACHE=0`` disables).+* the packed sequence filled in place with ``index_copy_`` (``UNSLOTH_H3_INPLACE_PACK=0`` disables).+"""++from __future__ import annotations++import contextlib+import hashlib+import inspect+import os+import threading+import types+from typing import Any, Iterator, Optional++REFINER_CACHE_ENV = "UNSLOTH_H3_REFINER_CACHE"+INPLACE_PACK_ENV = "UNSLOTH_H3_INPLACE_PACK"++_CLASS_NAME = "MiniMaxH3Transformer3DModel"+# sha256[:16] of inspect.getsource(MiniMaxH3Transformer3DModel.forward) this module mirrors (diffusers 0.42.1).+_KNOWN_FORWARD_SHA = frozenset({"ae200f2868ad388f"})++_PATCHED_ATTR = "_unsloth_h3_forward"+_SCOPE_ATTR = "_unsloth_h3_text_cache"+STATS_ATTR = "_unsloth_h3_forward_stats"+DENOISER_COMPONENTS = ("transformer", "transformer_ref")++_LOCK = threading.Lock()+_SHA_CACHE: dict = {}+++def _switch(name: str) -> bool:+ return (os.environ.get(name) or "").strip().lower() not in ("0", "off", "false", "no")+++def refiner_cache_enabled() -> bool:+ return _switch(REFINER_CACHE_ENV)+++def inplace_pack_enabled() -> bool:+ return _switch(INPLACE_PACK_ENV)+++def _forward_sha(cls: type) -> Optional[str]:+ if cls in _SHA_CACHE:+ return _SHA_CACHE[cls]+ try:+ sha = hashlib.sha256(inspect.getsource(cls.forward).encode()).hexdigest()[:16]+ except Exception: # noqa: BLE001+ sha = None+ _SHA_CACHE[cls] = sha+ return sha+++def _stats(module: Any) -> dict:+ stats = module.__dict__.get(STATS_ATTR)+ if stats is None:+ stats = {"text_hits": 0, "text_misses": 0, "inplace_packs": 0, "stock": 0}+ module.__dict__[STATS_ATTR] = stats+ return stats+++class _TextCache:+ """One render's text-stream rows: the conditioning tensor they came from (held, so its storage cannot be+ reused by another tensor while the entry lives) and everything else that decides them."""++ __slots__ = ("source", "key", "value")++ def __init__(self) -> None:+ self.source = None+ self.key = None+ self.value = None+++def _source_key(t: Any, dtype: Any) -> tuple:+ version = None if t.is_inference() else t._version # inference tensors carry no version counter+ return (t.data_ptr(), tuple(t.shape), t.dtype, str(t.device), version, dtype)+++def _text_rows(+ self: Any,+ encoder_hidden_states: Any,+ lora_scale: Any = None,+) -> Any:+ from diffusers.models.modeling_utils import get_parameter_dtype++ dtype = get_parameter_dtype(self.context_embedder)+ cache = self.__dict__.get(_SCOPE_ATTR)+ if cache is None or not refiner_cache_enabled():+ text_embeds = self.context_embedder(encoder_hidden_states.to(dtype))+ return self.token_refiner(text_embeds)+ # the LoRA scale apply_lora_scale applied to this call scales the refiner's adapters too+ key = (_source_key(encoder_hidden_states, dtype), lora_scale)+ if cache.value is not None and cache.source is encoder_hidden_states and cache.key == key:+ _stats(self)["text_hits"] += 1+ return cache.value+ text_embeds = self.context_embedder(encoder_hidden_states.to(dtype))+ text_embeds = self.token_refiner(text_embeds)+ cache.source, cache.key, cache.value = encoder_hidden_states, key, text_embeds+ _stats(self)["text_misses"] += 1+ return text_embeds+++def _body(+ self: Any,+ hidden_states: Any,+ audio_hidden_states: Any,+ encoder_hidden_states: Any,+ timestep: Any,+ timestep_indices: Any,+ token_tags: Any,+ position_ids: Any,+ video_indices: Any,+ audio_indices: Any,+ text_indices: Any,+ attention_kwargs: Optional[dict] = None,+ return_dict: bool = True,+ _unsloth_lora_scale: Any = None,+) -> Any:+ """``MiniMaxH3Transformer3DModel.forward`` (diffusers 0.42.1) op for op, except the text stream and the pack."""+ from diffusers.models.modeling_utils import get_parameter_dtype+ from diffusers.models.transformers.transformer_minimax_h3 import (+ MINIMAX_H3_MODALITY_NUM,+ MiniMaxH3TransformerOutput,+ )++ if position_ids.ndim != 2 or position_ids.shape[-1] != 3:+ raise ValueError(+ f"`position_ids` must be a `(seq_len, 3)` tensor, got {list(position_ids.shape)}."+ )+ sequence_length = position_ids.shape[0]+ if token_tags.shape != (sequence_length,) or timestep_indices.shape != (sequence_length,):+ raise ValueError(+ "`token_tags` and `timestep_indices` must both be `(seq_len,)` tensors matching `position_ids`, got "+ f"{list(token_tags.shape)} and {list(timestep_indices.shape)} for seq_len={sequence_length}."+ )++ rotary_emb = self.rope(position_ids)++ video_embeds = self.proj_in(hidden_states.to(get_parameter_dtype(self.proj_in)))+ audio_embeds = self.audio_proj_in(+ audio_hidden_states.to(get_parameter_dtype(self.audio_proj_in))+ )+ text_embeds = _text_rows(self, encoder_hidden_states, _unsloth_lora_scale)++ hidden_states = text_embeds.new_zeros(+ (text_embeds.shape[0], sequence_length, text_embeds.shape[-1])+ )+ if inplace_pack_enabled():+ hidden_states.index_copy_(1, text_indices, text_embeds)+ hidden_states.index_copy_(1, video_indices, video_embeds.to(text_embeds.dtype))+ hidden_states.index_copy_(1, audio_indices, audio_embeds.to(text_embeds.dtype))+ _stats(self)["inplace_packs"] += 1+ else:+ hidden_states = hidden_states.index_copy(1, text_indices, text_embeds)+ hidden_states = hidden_states.index_copy(+ 1, video_indices, video_embeds.to(text_embeds.dtype)+ )+ hidden_states = hidden_states.index_copy(+ 1, audio_indices, audio_embeds.to(text_embeds.dtype)+ )+ del video_embeds, audio_embeds++ temb = self.time_proj(timestep)+ temb = self.time_embedder(temb.to(get_parameter_dtype(self.time_embedder)))++ adaln_indices = timestep_indices * MINIMAX_H3_MODALITY_NUM + token_tags++ for block in self.transformer_blocks:+ hidden_states = block(hidden_states, temb, adaln_indices, rotary_emb)++ hidden_states = self.norm_out(hidden_states, temb, timestep_indices).to(+ get_parameter_dtype(self.proj_out)+ )+ video_output = self.proj_out(hidden_states).index_select(1, video_indices)+ audio_output = self.audio_proj_out(hidden_states).index_select(1, audio_indices)++ if not return_dict:+ return (video_output, audio_output)+ return MiniMaxH3TransformerOutput(sample = video_output, audio_sample = audio_output)+++_SCALED_BODY = None+++def _scaled_body() -> Any:+ """``_body`` under the same ``apply_lora_scale`` the stock forward carries (built once, lazily)."""+ global _SCALED_BODY+ if _SCALED_BODY is None:+ from diffusers.utils import apply_lora_scale+ _SCALED_BODY = apply_lora_scale("attention_kwargs")(_body)+ return _SCALED_BODY+++def _forward(+ self: Any,+ hidden_states: Any,+ audio_hidden_states: Any,+ encoder_hidden_states: Any,+ timestep: Any,+ timestep_indices: Any,+ token_tags: Any,+ position_ids: Any,+ video_indices: Any,+ audio_indices: Any,+ text_indices: Any,+ attention_kwargs: Optional[dict] = None,+ return_dict: bool = True,+) -> Any:+ import torch++ use_cache = self.__dict__.get(_SCOPE_ATTR) is not None and refiner_cache_enabled()+ extra = {}+ if torch.is_grad_enabled() or not (use_cache or inplace_pack_enabled()):+ _stats(self)["stock"] += 1+ fn = type(self).forward+ else:+ fn = _scaled_body()+ # apply_lora_scale pops the scale before the body runs; the text cache keys on it+ extra["_unsloth_lora_scale"] = (+ attention_kwargs.get("scale") if isinstance(attention_kwargs, dict) else None+ )+ return fn(+ self,+ hidden_states = hidden_states,+ audio_hidden_states = audio_hidden_states,+ encoder_hidden_states = encoder_hidden_states,+ timestep = timestep,+ timestep_indices = timestep_indices,+ token_tags = token_tags,+ position_ids = position_ids,+ video_indices = video_indices,+ audio_indices = audio_indices,+ text_indices = text_indices,+ attention_kwargs = attention_kwargs,+ return_dict = return_dict,+ **extra,+ )+++def _hooked(module: Any) -> bool:+ return getattr(module, "_hf_hook", None) is not None or bool(+ getattr(getattr(module, "_diffusers_hook", None), "hooks", None)+ )+++def install_h3_forward(transformer: Any, logger: Any = None) -> bool:+ """Bind the mirrored forward before any offload hook wraps ``forward``; idempotent, never raises, False = stock."""+ try:+ if transformer is None or type(transformer).__name__ != _CLASS_NAME:+ return False+ if transformer.__dict__.get(_PATCHED_ATTR):+ return True+ if not (refiner_cache_enabled() or inplace_pack_enabled()):+ return False+ with _LOCK:+ if _forward_sha(type(transformer)) not in _KNOWN_FORWARD_SHA:+ if logger is not None:+ logger.info("video.h3_forward: unknown denoiser forward source, keeping stock")+ return False+ if "forward" in transformer.__dict__ or _hooked(transformer):+ if logger is not None:+ logger.info("video.h3_forward: denoiser forward already wrapped, keeping stock")+ return False+ transformer.forward = types.MethodType(_forward, transformer)+ transformer.__dict__[_PATCHED_ATTR] = True+ _stats(transformer)+ except Exception as exc: # noqa: BLE001 -- an optimisation, never fail a load+ if logger is not None:+ logger.warning("video.h3_forward: install skipped: %s", exc)+ return False+ if logger is not None:+ logger.info(+ "video.h3_forward: text stream once per render (%s), packed sequence in place (%s)",+ "on" if refiner_cache_enabled() else "off",+ "on" if inplace_pack_enabled() else "off",+ )+ return True+++def install_h3_forward_on_pipe(pipe: Any, logger: Any = None) -> int:+ return sum(+ 1+ for name in DENOISER_COMPONENTS+ if install_h3_forward(getattr(pipe, name, None), logger = logger)+ )+++def h3_render_scope(pipe: Any, logger: Any = None) -> Iterator[None]:+ """One render: the text-stream cache exists only inside, and is dropped on exit or error."""+ modules = [+ m+ for m in (getattr(pipe, name, None) for name in DENOISER_COMPONENTS)+ if m is not None and m.__dict__.get(_PATCHED_ATTR)+ ]+ before = []+ for m in modules:+ m.__dict__[_SCOPE_ATTR] = _TextCache()+ before.append(dict(_stats(m)))+ try:+ yield+ finally:+ for m, start in zip(modules, before):+ m.__dict__.pop(_SCOPE_ATTR, None)+ if logger is not None:+ now = _stats(m)+ delta = {k: now[k] - start.get(k, 0) for k in now}+ logger.info(+ "video.h3_forward: render ran the text stream %d time(s), reused it %d time(s), "+ "%d in-place packs, %d stock calls",+ delta["text_misses"],+ delta["text_hits"],+ delta["inplace_packs"],+ delta["stock"],+ )diff --git a/studio/backend/core/inference/video_minimax_h3_te.py b/studio/backend/core/inference/video_minimax_h3_te.pyindex f315d28f09a..ae3680139cf 100644--- a/studio/backend/core/inference/video_minimax_h3_te.py+++ b/studio/backend/core/inference/video_minimax_h3_te.py@@ -606,6 +606,115 @@ def _fill_arena(arena: Any, *, entries: list) -> None:list(pool.map(run, jobs))+# The streamed conditioner's vocabulary table (151936 x 5120 bf16, 1.56 GB) stays on the host: the prompt's rows are+# gathered there and only they are uploaded, not the whole table per encode. ``UNSLOTH_H3_HOST_EMBED_ROWS=0`` streams+# the table like every other leaf.+H3_HOST_EMBED_ROWS_ENV = "UNSLOTH_H3_HOST_EMBED_ROWS"+_HOST_ROWS_ATTR = "_unsloth_host_rows_device"+++def h3_host_embed_rows_enabled() -> bool:+ import os+ return str(os.environ.get(H3_HOST_EMBED_ROWS_ENV, "")).strip().lower() not in (+ "0",+ "off",+ "false",+ "no",+ )+++def _host_rows_forward(self: Any, input: Any) -> Any:+ """``nn.Embedding.forward`` on the host table (a gather: bit-identical), the rows sent to the onload device."""+ import torch.nn.functional as F++ weight = self.weight+ ids = input if input.device == weight.device else input.to(weight.device)+ rows = F.embedding(+ ids,+ weight,+ self.padding_idx,+ self.max_norm,+ self.norm_type,+ self.scale_grad_by_freq,+ self.sparse,+ )+ return rows.to(self.__dict__[_HOST_ROWS_ATTR])+++def keep_h3_embedding_on_host(target: Any, logger: Any = None) -> bool:+ """Take the group-offloaded language embedding out of the stream, its lookup done on the host. Run right after+ ``apply_group_offloading``; True when installed. Plain frozen bf16/fp16/fp32 tables only. Never raises."""+ try:+ if not h3_host_embed_rows_enabled():+ return False+ import types++ import torch++ getter = getattr(target, "get_input_embeddings", None)+ emb = getter() if callable(getter) else None+ if type(emb) is not torch.nn.Embedding or _HOST_ROWS_ATTR in emb.__dict__:+ return False+ weight = emb.weight+ if (+ type(weight.data) is not torch.Tensor+ or weight.device.type != "cpu"+ or weight.requires_grad+ or emb.max_norm is not None+ or emb.sparse+ or weight.dtype not in (torch.bfloat16, torch.float16, torch.float32)+ ):+ return False+ registry = getattr(emb, "_diffusers_hook", None)+ hooks = getattr(registry, "hooks", None) or {}+ # a streamed (lazy prefetch) setup also tracks the leaf's first execution; anything else keeps the stock leaf+ if "group_offloading" not in hooks or not set(hooks) <= {+ "group_offloading",+ "layer_execution_tracker",+ }:+ return False+ group = getattr(hooks["group_offloading"], "group", None)+ if group is None or list(getattr(group, "modules", None) or ()) != [emb]:+ return False+ onload = getattr(group, "onload_device", None)+ if onload is None:+ return False+ name = next((n for n, m in target.named_modules() if m is emb), None)+ lazy = (getattr(getattr(target, "_diffusers_hook", None), "hooks", None) or {}).get(+ "lazy_prefetch_group_offloading"+ )+ emb.__dict__[_HOST_ROWS_ATTR] = torch.device(onload)+ try:+ if "layer_execution_tracker" in hooks:+ registry.remove_hook("layer_execution_tracker", recurse = False)+ registry.remove_hook("group_offloading", recurse = False)+ finally:+ # set even if a removal failed half way: the table is on the host, so the lookup must be too+ emb.forward = types.MethodType(_host_rows_forward, emb)+ names = getattr(lazy, "_layer_execution_tracker_module_names", None)+ if isinstance(names, set) and name is not None:+ names.discard(name) # no longer streamed: the first forward must not expect it+ # diffusers may hold the table's pinned twin; the lookup reads what the parameter points at now+ host = getattr(group, "cpu_param_dict", None) or {}+ pinned = host.get(weight)+ if (+ pinned is not None+ and pinned.device.type == "cpu"+ and weight.data_ptr() != pinned.data_ptr()+ ):+ weight.data = pinned+ except Exception as exc: # noqa: BLE001 -- the streamed leaf still works+ if logger is not None:+ logger.warning("video.h3_te_stream: embedding stays streamed: %s", exc)+ return False+ if logger is not None:+ logger.info(+ "video.h3_te_stream: vocabulary table (%.2f GB) stays on the host, prompt rows uploaded",+ weight.numel() * weight.element_size() / 1e9,+ )+ return True++def stream_h3_text_encoder(manager: Any,text_encoder: Any,@@ -677,6 +786,7 @@ def stream_h3_text_encoder(install_group_offload_buffer_restore()install_group_offload_hooks_eager()apply_group_offloading(target, **kwargs)+ keep_h3_embedding_on_host(target, logger = logger)# Image prompts: Qwen3-VL's position interpolation reads the offloaded embedding's CPU device (transformers 5.5)._pin_vision_embedding_device(target)except Exception as exc: # noqa: BLE001 -- keep the rotationdiff --git a/studio/backend/tests/test_h3_forward.py b/studio/backend/tests/test_h3_forward.pynew file mode 100644index 00000000000..11d1a9ba50f--- /dev/null+++ b/studio/backend/tests/test_h3_forward.py@@ -0,0 +1,326 @@+# SPDX-License-Identifier: AGPL-3.0-only+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0++"""CPU tests for the mirrored MiniMax-H3 denoiser forward (text stream once per render, packed sequence in place)+on a tiny random diffusers H3 transformer."""++from __future__ import annotations++import inspect++import pytest++torch = pytest.importorskip("torch")+h3_mod = pytest.importorskip("diffusers.models.transformers.transformer_minimax_h3")++from core.inference import video_minimax_h3_forward as h3f # noqa: E402++needs_mirror = pytest.mark.skipif(+ h3f._forward_sha(h3_mod.MiniMaxH3Transformer3DModel) not in h3f._KNOWN_FORWARD_SHA,+ reason = "installed diffusers' H3 forward is not the mirrored source",+)++T_ROWS, V_ROWS, A_ROWS = 5, 12, 4+++def _model():+ torch.manual_seed(0)+ m = h3_mod.MiniMaxH3Transformer3DModel(+ num_attention_heads = 2,+ attention_head_dim = 16,+ hidden_size = 32,+ num_layers = 2,+ num_refiner_layers = 2,+ ffn_dim = 64,+ in_channels = 4,+ audio_in_channels = 6,+ patch_size = (1, 1, 1),+ text_dim = 24,+ freq_dim = 16,+ time_embed_hidden_dim = 32,+ time_embed_dim = 16,+ rope_freq_dim = 2,+ ).eval()+ with torch.no_grad():+ for p in m.parameters():+ p.copy_(torch.randn_like(p) * 0.1)+ return m+++def _layout():+ s = T_ROWS + V_ROWS + A_ROWS+ perm = torch.randperm(s, generator = torch.Generator().manual_seed(1))+ text_indices = perm[:T_ROWS].sort().values+ video_indices = perm[T_ROWS : T_ROWS + V_ROWS].sort().values+ audio_indices = perm[T_ROWS + V_ROWS :].sort().values+ token_tags = torch.zeros(s, dtype = torch.long)+ token_tags[video_indices] = 1+ token_tags[audio_indices] = 2+ position_ids = torch.randint(0, 6, (s, 3), generator = torch.Generator().manual_seed(2))+ timestep_indices = torch.zeros(s, dtype = torch.long)+ timestep_indices[video_indices[:3]] = 1+ return dict(+ token_tags = token_tags,+ position_ids = position_ids,+ video_indices = video_indices,+ audio_indices = audio_indices,+ text_indices = text_indices,+ timestep_indices = timestep_indices,+ )+++def _step_inputs(i, enc):+ g = torch.Generator().manual_seed(100 + i)+ return dict(+ hidden_states = torch.randn(1, V_ROWS, 4, generator = g),+ audio_hidden_states = torch.randn(1, A_ROWS, 6, generator = g),+ encoder_hidden_states = enc,+ timestep = torch.tensor([1000.0 - 100 * i, 990.0]),+ return_dict = False,+ )+++def _run(+ model,+ enc,+ steps = 4,+ layout = None,+):+ layout = layout or _layout()+ outs = []+ with torch.no_grad():+ for i in range(steps):+ outs.append(model(**_step_inputs(i, enc), **layout))+ return outs+++def _enc(seed = 3):+ return torch.randn(1, T_ROWS, 24, generator = torch.Generator().manual_seed(seed))+++class _Pipe:+ def __init__(self, transformer):+ self.transformer = transformer+++def _assert_same(a, b):+ assert len(a) == len(b)+ for (va, aa), (vb, ab) in zip(a, b):+ assert torch.equal(va, vb) and torch.equal(aa, ab)+++@needs_mirror+def test_bit_identical_and_text_stream_runs_once_per_render(monkeypatch):+ monkeypatch.delenv(h3f.REFINER_CACHE_ENV, raising = False)+ monkeypatch.delenv(h3f.INPLACE_PACK_ENV, raising = False)+ stock = _run(_model(), _enc())+ m = _model()+ calls = {"refiner": 0, "embedder": 0}+ m.token_refiner.register_forward_hook(+ lambda *a: calls.__setitem__("refiner", calls["refiner"] + 1)+ )+ m.context_embedder.register_forward_hook(+ lambda *a: calls.__setitem__("embedder", calls["embedder"] + 1)+ )+ assert h3f.install_h3_forward(m) is True+ assert h3f.install_h3_forward(m) is True # idempotent+ with h3f.h3_render_scope(_Pipe(m)):+ got = _run(m, _enc())+ _assert_same(stock, got)+ assert calls == {"refiner": 1, "embedder": 1}+ stats = m.__dict__[h3f.STATS_ATTR]+ assert stats["text_misses"] == 1 and stats["text_hits"] == 3 and stats["inplace_packs"] == 4+ # the scope is gone after the render: nothing holds the conditioning+ assert h3f._SCOPE_ATTR not in m.__dict__+++@needs_mirror+def test_cache_only_inside_a_render_and_keyed_on_the_conditioning(monkeypatch):+ monkeypatch.delenv(h3f.REFINER_CACHE_ENV, raising = False)+ m = _model()+ h3f.install_h3_forward(m)+ stats = m.__dict__[h3f.STATS_ATTR]+ _run(m, _enc(), steps = 2) # no render scope: recomputed each call+ assert stats["text_misses"] == 0 and stats["text_hits"] == 0+ stock = _model()+ layout = _layout()+ with h3f.h3_render_scope(_Pipe(m)):+ a = _run(m, _enc(3), steps = 2, layout = layout)+ b = _run(+ m, _enc(4), steps = 2, layout = layout+ ) # another caption tensor: a miss, not the old rows+ assert stats["text_misses"] == 2 and stats["text_hits"] == 2+ _assert_same(a, _run(stock, _enc(3), steps = 2, layout = layout))+ _assert_same(b, _run(stock, _enc(4), steps = 2, layout = layout))+ # an in-place edit of the same tensor (normal tensor: version counter) misses too+ enc = _enc(5)+ with h3f.h3_render_scope(_Pipe(m)):+ _run(m, enc, steps = 1, layout = layout)+ enc.mul_(2.0)+ got = _run(m, enc, steps = 1, layout = layout)+ _assert_same(got, _run(stock, enc, steps = 1, layout = layout))+ assert stats["text_misses"] == 4+++@needs_mirror+def test_inference_mode_render(monkeypatch):+ monkeypatch.delenv(h3f.REFINER_CACHE_ENV, raising = False)+ m, stock = _model(), _model()+ h3f.install_h3_forward(m)+ layout = _layout()+ with torch.inference_mode():+ enc = _enc()+ ref = [stock(**_step_inputs(i, enc), **layout) for i in range(3)]+ with h3f.h3_render_scope(_Pipe(m)):+ got = [m(**_step_inputs(i, enc), **layout) for i in range(3)]+ _assert_same(ref, got)+ assert m.__dict__[h3f.STATS_ATTR]["text_hits"] == 2+++@needs_mirror[email protected]("env", [h3f.REFINER_CACHE_ENV, h3f.INPLACE_PACK_ENV])+def test_kill_switches(monkeypatch, env):+ monkeypatch.setenv(env, "0")+ m = _model()+ assert h3f.install_h3_forward(m) is True+ with h3f.h3_render_scope(_Pipe(m)):+ got = _run(m, _enc())+ _assert_same(_run(_model(), _enc()), got)+ stats = m.__dict__[h3f.STATS_ATTR]+ if env == h3f.REFINER_CACHE_ENV:+ assert stats["text_misses"] == 0 and stats["text_hits"] == 0 and stats["inplace_packs"] == 4+ else:+ assert stats["text_hits"] == 3 and stats["inplace_packs"] == 0+ monkeypatch.setenv(h3f.REFINER_CACHE_ENV, "0")+ monkeypatch.setenv(h3f.INPLACE_PACK_ENV, "0")+ fresh = _model()+ assert h3f.install_h3_forward(fresh) is False+ assert "forward" not in fresh.__dict__+++@needs_mirror+def test_no_allocating_full_sequence_copies(monkeypatch):+ from torch.utils._python_dispatch import TorchDispatchMode++ class Count(TorchDispatchMode):+ def __init__(self):+ super().__init__()+ self.ops = {}++ def __torch_dispatch__(+ self,+ func,+ types,+ args = (),+ kwargs = None,+ ):+ name = str(func.overloadpacket)+ self.ops[name] = self.ops.get(name, 0) + 1+ return func(*args, **(kwargs or {}))++ def count(model, scope):+ with Count() as c:+ if scope:+ with h3f.h3_render_scope(_Pipe(model)):+ _run(model, _enc(), steps = 2)+ else:+ _run(model, _enc(), steps = 2)+ return c.ops++ monkeypatch.delenv(h3f.INPLACE_PACK_ENV, raising = False)+ stock = count(_model(), False)+ m = _model()+ h3f.install_h3_forward(m)+ patched = count(m, True)+ assert stock.get("aten.index_copy", 0) == 6 and stock.get("aten.index_copy_", 0) == 0+ assert patched.get("aten.index_copy", 0) == 0 and patched.get("aten.index_copy_", 0) == 6+ # the refiner's attention + MLP ran for one of the two steps only+ assert patched.get("aten.mm", 0) + patched.get("aten.addmm", 0) < stock.get(+ "aten.mm", 0+ ) + stock.get("aten.addmm", 0)+++@needs_mirror+def test_training_and_grad_keep_stock(monkeypatch):+ m = _model()+ h3f.install_h3_forward(m)+ layout = _layout()+ with h3f.h3_render_scope(_Pipe(m)):+ v, a = m(**_step_inputs(0, _enc()), **layout)+ assert v.requires_grad+ stats = m.__dict__[h3f.STATS_ATTR]+ assert stats["stock"] == 1 and stats["text_misses"] == 0+++@needs_mirror+def test_signature_and_hooks_wrap_the_mirrored_forward():+ from diffusers.hooks import HookRegistry, ModelHook++ m = _model()+ h3f.install_h3_forward(m)+ stock_params = list(inspect.signature(h3_mod.MiniMaxH3Transformer3DModel.forward).parameters)+ # the modular denoise step filters its layout kwargs by this signature+ assert list(inspect.signature(m.forward).parameters) == stock_params[1:]++ seen = []++ class Pre(ModelHook):+ def pre_forward(self, module, *args, **kwargs):+ seen.append(1)+ return args, kwargs++ HookRegistry.check_if_exists_or_initialize(m).register_hook(Pre(), "probe")+ with h3f.h3_render_scope(_Pipe(m)):+ _run(m, _enc(), steps = 2)+ assert seen == [1, 1] and m.__dict__[h3f.STATS_ATTR]["text_hits"] == 1++ hooked = _model()+ HookRegistry.check_if_exists_or_initialize(hooked).register_hook(Pre(), "probe")+ assert h3f.install_h3_forward(hooked) is False # never under an existing hook+++def test_unknown_source_and_other_classes_keep_stock(monkeypatch):+ class Other(torch.nn.Module):+ def forward(self, x):+ return x++ assert h3f.install_h3_forward(Other()) is False+ assert h3f.install_h3_forward(None) is False+ m = _model()+ monkeypatch.setattr(h3f, "_KNOWN_FORWARD_SHA", frozenset({"0" * 16}))+ assert h3f.install_h3_forward(m) is False+ assert "forward" not in m.__dict__+++@needs_mirror+def test_regional_compile_adds_no_graphs(monkeypatch):+ from torch._dynamo.testing import CompileCounter++ def graphs(install):+ torch._dynamo.reset()+ m = _model()+ if install:+ h3f.install_h3_forward(m)+ cnt = CompileCounter()+ m.compile_repeated_blocks(backend = cnt, fullgraph = True, dynamic = False)+ with h3f.h3_render_scope(_Pipe(m)):+ _run(m, _enc(), steps = 4)+ return cnt.frame_count++ stock, patched = graphs(False), graphs(True)+ assert patched <= stock+++@needs_mirror+def test_lora_scale_is_part_of_the_key(monkeypatch):+ monkeypatch.delenv(h3f.REFINER_CACHE_ENV, raising = False)+ m = _model()+ h3f.install_h3_forward(m)+ layout = _layout()+ enc = _enc()+ with torch.no_grad(), h3f.h3_render_scope(_Pipe(m)):+ m(**_step_inputs(0, enc), **layout)+ m(**_step_inputs(1, enc), **layout, attention_kwargs = {"scale": 0.5})+ m(**_step_inputs(2, enc), **layout, attention_kwargs = {"scale": 0.5})+ stats = m.__dict__[h3f.STATS_ATTR]+ assert stats["text_misses"] == 2 and stats["text_hits"] == 1diff --git a/studio/backend/tests/test_h3_te_host_rows.py b/studio/backend/tests/test_h3_te_host_rows.pynew file mode 100644index 00000000000..86392746b1b--- /dev/null+++ b/studio/backend/tests/test_h3_te_host_rows.py@@ -0,0 +1,121 @@+# SPDX-License-Identifier: AGPL-3.0-only+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0++"""CPU tests: the streamed MiniMax-H3 conditioner's vocabulary table stays on the host (bit-identical lookups)."""++from __future__ import annotations++import pytest++torch = pytest.importorskip("torch")++from core.inference import video_minimax_h3_te as te # noqa: E402+++class _TinyLM(torch.nn.Module):+ def __init__(self):+ super().__init__()+ torch.manual_seed(0)+ self.embed_tokens = torch.nn.Embedding(50, 8, padding_idx = 0).to(torch.bfloat16)+ self.proj = torch.nn.Linear(8, 8).to(torch.bfloat16)++ def get_input_embeddings(self):+ return self.embed_tokens++ def forward(self, input_ids):+ return self.proj(self.embed_tokens(input_ids))+++def _streamed(install):+ from diffusers.hooks import apply_group_offloading++ m = _TinyLM().requires_grad_(False)+ apply_group_offloading(+ m,+ onload_device = torch.device("cpu"),+ offload_device = torch.device("cpu"),+ offload_type = "leaf_level",+ use_stream = False,+ )+ installed = te.keep_h3_embedding_on_host(m) if install else False+ return m, installed+++def test_host_rows_bit_identical_and_out_of_the_stream(monkeypatch):+ pytest.importorskip("diffusers.hooks")+ monkeypatch.delenv(te.H3_HOST_EMBED_ROWS_ENV, raising = False)+ ref_mod, _ = _streamed(False)+ m, installed = _streamed(True)+ assert installed is True+ assert "group_offloading" not in (m.embed_tokens._diffusers_hook.hooks or {})+ assert "group_offloading" in m.proj._diffusers_hook.hooks # every other leaf still streams+ ids = torch.tensor([[0, 3, 7, 49, 3]])+ with torch.no_grad():+ assert torch.equal(ref_mod(ids), m(ids))+ assert torch.equal(ref_mod.embed_tokens(ids), m.embed_tokens(ids))+ assert m.embed_tokens.weight.device.type == "cpu"+ assert (+ te.keep_h3_embedding_on_host(m) is False+ ) # already taken out: a second call changes nothing+++def test_host_rows_kill_switch_and_refusals(monkeypatch):+ pytest.importorskip("diffusers.hooks")+ monkeypatch.setenv(te.H3_HOST_EMBED_ROWS_ENV, "0")+ m, installed = _streamed(True)+ assert installed is False and "group_offloading" in m.embed_tokens._diffusers_hook.hooks+ monkeypatch.delenv(te.H3_HOST_EMBED_ROWS_ENV)+ # not group offloaded: nothing to take out+ plain = _TinyLM()+ assert te.keep_h3_embedding_on_host(plain) is False+ assert "forward" not in plain.embed_tokens.__dict__+++def test_host_rows_under_lazy_prefetch_tracking(monkeypatch):+ pytest.importorskip("diffusers.hooks")+ from diffusers.hooks import group_offloading as go++ monkeypatch.delenv(te.H3_HOST_EMBED_ROWS_ENV, raising = False)+ from diffusers.hooks import apply_group_offloading++ def build():+ m = _TinyLM().requires_grad_(False)+ apply_group_offloading(+ m,+ onload_device = torch.device("cpu"),+ offload_device = torch.device("cpu"),+ offload_type = "leaf_level",+ use_stream = False,+ )+ # what use_stream=True adds (diffusers' own helper): a root group + a lazy prefetch hook tracking every leaf+ cpu = torch.device("cpu")+ config = go.GroupOffloadingConfig(+ onload_device = cpu,+ offload_device = cpu,+ offload_type = go.GroupOffloadingType.LEAF_LEVEL,+ non_blocking = False,+ record_stream = False,+ low_cpu_mem_usage = False,+ )+ root = go.ModuleGroup(+ modules = [],+ offload_device = cpu,+ onload_device = cpu,+ offload_leader = m,+ onload_leader = m,+ onload_self = True,+ group_id = "lazy_leafs",+ )+ go._apply_lazy_group_offloading_hook(m, root, config = config)+ return m++ ref_mod, m = build(), build()+ assert "layer_execution_tracker" in m.embed_tokens._diffusers_hook.hooks+ assert te.keep_h3_embedding_on_host(m) is True+ assert not m.embed_tokens._diffusers_hook.hooks+ lazy = m._diffusers_hook.hooks["lazy_prefetch_group_offloading"]+ assert "embed_tokens" not in lazy._layer_execution_tracker_module_names+ ids = torch.tensor([[1, 2, 3]])+ with torch.no_grad():+ for _ in range(2): # the first forward builds the prefetch chain, the second uses it+ assert torch.equal(ref_mod(ids), m(ids))