Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

Studio MiniMax-H3: keep conditioner layers on the card beside a resident denoiser, size its host pins to their weights (#17355)

Daniel Han4d709a58c8f2
diff --git a/studio/backend/core/inference/video.py b/studio/backend/core/inference/video.py
index a108805529e..b83ffa96f36 100644
--- a/studio/backend/core/inference/video.py
+++ b/studio/backend/core/inference/video.py
@@ -3255,7 +3255,11 @@ def _h3_residency_budget_bytes(
if free is None:
return 0
cached = cuda_reusable_cache_bytes(device)
- available = int(free) + cached + int(residency.resident_bytes())
+ # The conditioner's device share is demotable like a resident block (and it yields first).
+ held = getattr(residency, "device_bytes", None)
+ available = (
+ int(free) + cached + int(held() if callable(held) else residency.resident_bytes())
+ )
te_gb = (
H3_TE_STREAMED_GB
if te_streamed
@@ -12449,6 +12453,11 @@ class VideoBackend:
_te, logger = logger
)
)
+ from .video_minimax_h3_te import h3_te_device_share
+
+ residency.te_share = h3_te_device_share(
+ _te_for_yield, device, te_streamed, logger = logger
+ )
big_w, big_h = max(
fam.resolution_presets, key = lambda wh: wh[0] * wh[1]
)
diff --git a/studio/backend/core/inference/video_minimax_h3_residency.py b/studio/backend/core/inference/video_minimax_h3_residency.py
index 1bb154bf0bf..3dac656a97d 100644
--- a/studio/backend/core/inference/video_minimax_h3_residency.py
+++ b/studio/backend/core/inference/video_minimax_h3_residency.py
@@ -627,6 +627,8 @@ class H3Residency:
# bytes of slabs the last ``apply`` compacted; a compaction that could not fit every slab retries next re-fit
self.compacted_bytes = 0
self.compact_pending = False
+ # video_minimax_h3_te.H3TeDeviceShare
+ self.te_share: Any = None
@property
def usable(self) -> bool:
@@ -644,6 +646,17 @@ class H3Residency:
total = self.top_bytes if (self.top is not None and is_resident(self.top)) else 0
return total + sum(b for g, b in zip(self.blocks, self.block_bytes) if is_resident(g))
+ def device_bytes(self) -> int:
+ te = getattr(self, "te_share", None)
+ return self.resident_bytes() + (te.resident_bytes() if te is not None else 0)
+
+ def te_budget(self, budget_bytes: int, top: bool, n: int) -> int:
+ """Zero unless the whole denoiser is resident: it is read every step, the conditioner once."""
+ if n < len(self.blocks) or (self.top is not None and not top):
+ return 0
+ planned = (self.top_bytes if top else 0) + sum(self.block_bytes[:n])
+ return max(0, int(budget_bytes) - planned)
+
def plan(
self,
budget_bytes: int,
@@ -704,7 +717,21 @@ class H3Residency:
) -> tuple[bool, int]:
"""Re-fit the resident set to ``budget_bytes`` (the device bytes the resident set may occupy)."""
top, n = self.plan(budget_bytes)
+ te = getattr(self, "te_share", None)
+ te_left = self.te_budget(budget_bytes, top, n) if te is not None else 0
+ te_n = te.plan(te_left) if te is not None else 0
+ if te is not None:
+ # the conditioner gives its share back before any denoiser block is promoted
+ te.shrink(te_n)
self.apply(top, n)
+ if te is not None:
+ try:
+ te.grow(te_n)
+ except Exception as exc: # noqa: BLE001 -- the conditioner streams as before
+ te.shrink(0)
+ if self.logger is not None:
+ self.logger.warning("video.h3_te_resident_device: staying streamed: %s", exc)
+ te.report(te_left, force = initial)
if initial:
self.max_blocks = n
released = getattr(self, "released_bytes", 0)
@@ -738,6 +765,13 @@ def release_all(residency: "H3Residency", *, logger: Any = None) -> None:
"""Return every group of ``residency`` to streaming, including one whose onload failed partway. Never raises."""
import torch
+ te = getattr(residency, "te_share", None)
+ if te is not None:
+ try:
+ te.shrink(0)
+ except Exception as exc: # noqa: BLE001
+ if logger is not None:
+ logger.warning("video.h3_te_resident_device: could not release the share: %s", exc)
groups = list(reversed(residency.blocks))
if residency.top is not None:
groups.append(residency.top)
diff --git a/studio/backend/core/inference/video_minimax_h3_te.py b/studio/backend/core/inference/video_minimax_h3_te.py
index 6eb7348dda3..7f2de2c3b99 100644
--- a/studio/backend/core/inference/video_minimax_h3_te.py
+++ b/studio/backend/core/inference/video_minimax_h3_te.py
@@ -498,21 +498,38 @@ def _pinned_arena(size: int, fill: Any) -> Any:
return arena
+# Arenas sized to their payload (0.61 GB audio VAE pinned 2.15 GB); the host-looked-up vocab table stays mapped. =0 reverts.
+H3_HOST_PIN_TRIM_ENV = "UNSLOTH_H3_HOST_PIN_TRIM"
+_PIN_PAGE = 4096
+
+
+def h3_host_pin_trim_enabled() -> bool:
+ import os
+ return str(os.environ.get(H3_HOST_PIN_TRIM_ENV, "")).strip().lower() not in (
+ "0",
+ "off",
+ "false",
+ "no",
+ )
+
+
def pin_module_in_place(
module: Any,
*,
arena_bytes: int = _PIN_ARENA_BYTES,
_arena_factory: Any = None,
repoint: bool = False,
+ skip: tuple = (),
) -> int:
"""Replaces tensors in _parameters/_buffers rather than .data, which would keep the mmap alive."""
import torch
+ skipped = {id(t) for t in skip if t is not None}
slots: dict[int, list] = {} # id(tensor) -> [tensor, [(owner dict, name, is_param)]]
for sub in module.modules():
for table, is_param in ((sub._parameters, True), (sub._buffers, False)):
for name, tensor in list(table.items()):
- if tensor is None:
+ if tensor is None or id(tensor) in skipped:
continue
entry = slots.setdefault(id(tensor), [tensor, []])
entry[1].append((table, name, is_param))
@@ -537,6 +554,10 @@ def pin_module_in_place(
slot[2].append((tensor, owners, slot[1]))
slot[1] += need
pinned += nbytes
+ if h3_host_pin_trim_enabled():
+ # placement is final: an arena's unused tail would only be zeroed and locked
+ for arena in arenas:
+ arena[0] = min(arena[0], _round_up(arena[1], _PIN_PAGE))
placed = [] # (tensor, owners, pinned view)
for size, _used, entries in arenas:
fill = functools.partial(_fill_arena, entries = entries)
@@ -638,6 +659,43 @@ def _host_rows_forward(self: Any, input: Any) -> Any:
return rows.to(self.__dict__[_HOST_ROWS_ATTR])
+def _host_rows_table(target: Any) -> Any:
+ """The table ``keep_h3_embedding_on_host`` will take out of the stream (same static checks), or None."""
+ try:
+ import torch
+
+ if target is None or not h3_host_embed_rows_enabled():
+ return None
+ getter = getattr(target, "get_input_embeddings", None)
+ emb = getter() if callable(getter) else None
+ if type(emb) is not torch.nn.Embedding or _HOST_ROWS_ATTR in emb.__dict__:
+ return None
+ weight = emb.weight
+ if (
+ type(weight.data) is not torch.Tensor
+ or weight.device.type != "cpu"
+ or emb.max_norm is not None
+ or emb.sparse
+ or weight.dtype not in (torch.bfloat16, torch.float16, torch.float32)
+ ):
+ return None
+ return weight
+ except Exception: # noqa: BLE001 -- pin it with the rest
+ return None
+
+
+def _pin_streamed_table(target: Any, weight: Any) -> None:
+ """The table stayed in the stream after all: pin it as the whole-module pin would have."""
+ from .diffusion_memory import _offload_groups
+
+ owner = next(m for m in target.modules() if m._parameters.get("weight") is weight)
+ pin_module_in_place(owner, repoint = True)
+ for group in _offload_groups(target):
+ cpu = getattr(group, "cpu_param_dict", None)
+ if isinstance(cpu, dict) and weight in cpu:
+ cpu[weight] = weight.data
+
+
def keep_h3_embedding_on_host(target: Any, logger: Any = None) -> bool:
"""Take the group-offloaded language embedding out of the stream, its lookup done on the host. Run right after
``apply_group_offloading``; True when installed. Plain frozen bf16/fp16/fp32 tables only. Never raises."""
@@ -754,13 +812,20 @@ def stream_h3_text_encoder(
pin = False
elif pin is None:
pin = h3_te_pin_allowed(_module_payload_bytes(text_encoder), onload)
+ # Looked up on the host, so a pinned copy buys nothing; needs diffusers to take the pinned rest as it is.
+ table = (
+ _host_rows_table(target)
+ if pin and h3_host_pin_trim_enabled() and "low_cpu_mem_usage" in params
+ else None
+ )
if pin:
try:
- pin_module_in_place(text_encoder)
+ pinned_bytes = pin_module_in_place(text_encoder, skip = (table,))
except Exception as exc: # noqa: BLE001 -- pinned host RAM refused: stream unpinned
if logger is not None:
logger.warning("video.h3_te_stream: pinning failed, streaming unpinned: %s", exc)
pin = False
+ table = None
kwargs: dict[str, Any] = {
"onload_device": onload,
"offload_device": torch.device("cpu"),
@@ -774,7 +839,8 @@ def stream_h3_text_encoder(
kwargs["record_stream"] = True
if "low_cpu_mem_usage" in params:
# Already pinned in place, so diffusers' pin_memory() returns the same tensor: no second host copy.
- kwargs["low_cpu_mem_usage"] = not pin
+ # With the table left mapped, the onload pins only what is not pinned yet: nothing once the table is out.
+ kwargs["low_cpu_mem_usage"] = not pin or table is not None
elif not pin:
return None
try:
@@ -783,7 +849,22 @@ def stream_h3_text_encoder(
install_group_offload_buffer_restore()
install_group_offload_hooks_eager()
apply_group_offloading(target, **kwargs)
- keep_h3_embedding_on_host(target, logger = logger)
+ on_host = keep_h3_embedding_on_host(target, logger = logger)
+ if table is not None:
+ if on_host:
+ if logger is not None:
+ logger.info(
+ "video.h3_host_pin_trim: conditioner pins %.2f GB in arenas sized to their weights, its "
+ "vocabulary table (%.2f GB) stays on its mapping",
+ pinned_bytes / 1e9,
+ table.numel() * table.element_size() / 1e9,
+ )
+ else:
+ try:
+ _pin_streamed_table(target, table)
+ except Exception as exc: # noqa: BLE001 -- streams pageable: correct, slower
+ if logger is not None:
+ logger.warning("video.h3_host_pin_trim: table not pinned: %s", exc)
# Image prompts: Qwen3-VL's position interpolation reads the offloaded embedding's CPU device (transformers 5.5).
_pin_vision_embedding_device(target)
except Exception as exc: # noqa: BLE001 -- keep the rotation
@@ -1015,6 +1096,232 @@ def _copy_host_tensors_(entries: list, undo: Optional[list] = None) -> int:
return copied
+# Once every denoiser block is resident, leftover budget keeps conditioner leaf groups on device in read order (48 GB:
+# ~0.8 s per render saved); requests needing more take it back first. Vision tower keeps streaming. =0 disables.
+H3_TE_DEVICE_RESIDENT_ENV = "UNSLOTH_H3_TE_DEVICE_RESIDENT"
+_TE_DEVICE_ATTR = "_unsloth_te_device"
+_TE_SAVED_IO_ATTR = "_unsloth_te_saved_io"
+# Planned device bytes per tensor: the allocator's 512-byte rounding plus an unsplit remainder (at most 1 MiB).
+_ALLOC_ROUND = 512
+_ALLOC_SPLIT_SLACK = 1 << 20
+
+
+def h3_te_device_resident_enabled() -> bool:
+ import os
+ return str(os.environ.get(H3_TE_DEVICE_RESIDENT_ENV, "")).strip().lower() not in (
+ "0",
+ "off",
+ "false",
+ "no",
+ )
+
+
+def _planned_device_bytes(group: Any) -> int:
+ from .video_minimax_h3_residency import _group_tensors, _tensor_bytes
+
+ total = 0
+ for t in _group_tensors(group):
+ n = _tensor_bytes(t)
+ total += _round_up(n, _ALLOC_ROUND) + (_ALLOC_SPLIT_SLACK if n > _ALLOC_SPLIT_SLACK else 0)
+ return total
+
+
+def _allocated_bytes(device: Any) -> Optional[int]:
+ try:
+ import torch
+
+ dev = torch.device(device)
+ if dev.type != "cuda" or not torch.cuda.is_available():
+ return None
+ return int(torch.cuda.memory_allocated(dev))
+ except Exception: # noqa: BLE001
+ return None
+
+
+def _te_resident_onload(group: Any) -> Any:
+ """Keep the side-stream sync: the next group may still be arriving via the previous group's prefetch."""
+ stream = getattr(group, "stream", None)
+
+ def onload_(*_a: Any, **_k: Any) -> None:
+ if stream is not None:
+ stream.synchronize()
+
+ try:
+ import torch
+ return torch.compiler.disable(onload_)
+ except Exception: # noqa: BLE001
+ return onload_
+
+
+def _te_noop(*_a: Any, **_k: Any) -> None:
+ return None
+
+
+class H3TeDeviceShare:
+ """The streamed conditioner's leaf groups resident on the device: a prefix in read order (language layers first)."""
+
+ def __init__(
+ self,
+ text_encoder: Any,
+ device: Any,
+ *,
+ logger: Any = None,
+ ) -> None:
+ from .diffusion_memory import _offload_groups
+ from .video_minimax_h3_residency import _group_tensors
+
+ target = getattr(text_encoder, "model", None)
+ self.device = device
+ self.logger = logger
+ self.groups: list = []
+ self.layer_of: list = []
+ names = {id(m): n for n, m in target.named_modules()} if target is not None else {}
+ for group in _offload_groups(target) if target is not None else []:
+ mods = list(getattr(group, "modules", None) or ())
+ leader = mods[0] if mods else getattr(group, "onload_leader", None)
+ parts = names.get(id(leader), "").split(".")
+ if (
+ "visual" in parts
+ or getattr(group, "stream", None) is None
+ or not isinstance(getattr(group, "cpu_param_dict", None), dict)
+ or not _group_tensors(group)
+ ):
+ continue
+ layer = None
+ if "layers" in parts and parts.index("layers") + 1 < len(parts):
+ index = parts[parts.index("layers") + 1]
+ layer = int(index) if index.isdigit() else None
+ self.groups.append(group)
+ self.layer_of.append(layer)
+ self.planned = [_planned_device_bytes(g) for g in self.groups]
+ self.layers = len({x for x in self.layer_of if x is not None})
+ self._logged: Optional[int] = None
+
+ @property
+ def usable(self) -> bool:
+ return bool(self.groups)
+
+ def resident_count(self) -> int:
+ n = 0
+ for group in self.groups:
+ if not group.__dict__.get(_TE_DEVICE_ATTR):
+ break
+ n += 1
+ return n
+
+ def resident_bytes(self) -> int:
+ """Measured at onload; planned bytes off CUDA."""
+ return sum(
+ int(group.__dict__.get(_TE_DEVICE_ATTR) or 0)
+ for group in self.groups
+ if group.__dict__.get(_TE_DEVICE_ATTR)
+ )
+
+ def plan(self, budget_bytes: int) -> int:
+ n = 0
+ left = int(budget_bytes)
+ while n < len(self.groups) and self.planned[n] <= left:
+ left -= self.planned[n]
+ n += 1
+ return n
+
+ def shrink(self, n: int) -> None:
+ import torch
+ demoted = False
+ for group in reversed(self.groups[n:]):
+ if not group.__dict__.get(_TE_DEVICE_ATTR):
+ continue
+ if not demoted and torch.cuda.is_available():
+ torch.cuda.current_stream().synchronize()
+ saved_on, saved_off = group.__dict__.get(_TE_SAVED_IO_ATTR, (None, None))
+ for name, saved in (("onload_", saved_on), ("offload_", saved_off)):
+ if saved is not None:
+ setattr(group, name, saved)
+ else:
+ group.__dict__.pop(name, None)
+ group.__dict__.pop(_TE_DEVICE_ATTR, None)
+ group.offload_() # back to its host copy, which was never dropped
+ demoted = True
+
+ def grow(self, n: int) -> None:
+ import torch
+ for group in self.groups[:n]:
+ if group.__dict__.get(_TE_DEVICE_ATTR):
+ continue
+ before = _allocated_bytes(self.device)
+ stream = getattr(group, "stream", None)
+ try:
+ group.onload_()
+ if stream is not None:
+ stream.synchronize()
+ if torch.cuda.is_available():
+ torch.cuda.current_stream().synchronize()
+ except Exception:
+ group.offload_() # a half-onloaded group back on its host copy
+ raise
+ after = _allocated_bytes(self.device)
+ held = after - before if before is not None and after is not None else None
+ if held is None or held <= 0:
+ held = _planned_device_bytes(group)
+ group.__dict__[_TE_SAVED_IO_ATTR] = (
+ group.__dict__.get("onload_"),
+ group.__dict__.get("offload_"),
+ )
+ group.onload_ = _te_resident_onload(group)
+ group.offload_ = _te_noop
+ group.__dict__[_TE_DEVICE_ATTR] = max(1, int(held))
+
+ def report(
+ self,
+ budget_bytes: int,
+ *,
+ force: bool = False,
+ ) -> None:
+ n = self.resident_count()
+ if self.logger is None or (n == self._logged and not force):
+ return
+ self._logged = n
+ full = sum(
+ 1
+ for layer in {x for x in self.layer_of if x is not None}
+ if all(
+ self.groups[i].__dict__.get(_TE_DEVICE_ATTR)
+ for i, x in enumerate(self.layer_of)
+ if x == layer
+ )
+ )
+ self.logger.info(
+ "video.h3_te_resident_device: %d of %d TE layers resident on %s (%.2f GB, %d of %d leaf groups, "
+ "%.2f GB left beside the resident denoiser)",
+ full,
+ self.layers,
+ self.device,
+ self.resident_bytes() / 1e9,
+ n,
+ len(self.groups),
+ max(0, budget_bytes) / 1e9,
+ )
+
+
+def h3_te_device_share(
+ text_encoder: Any,
+ device: Any,
+ mode: Any,
+ *,
+ logger: Any = None,
+) -> Optional[H3TeDeviceShare]:
+ """None unless the conditioner streams with a side stream. Never raises."""
+ try:
+ if mode not in ("stream", "stream_lazy") or not h3_te_device_resident_enabled():
+ return None
+ share = H3TeDeviceShare(text_encoder, device, logger = logger)
+ return share if share.usable else None
+ except Exception as exc: # noqa: BLE001 -- the conditioner streams as before
+ if logger is not None:
+ logger.warning("video.h3_te_resident_device: staying streamed: %s", exc)
+ return None
+
+
# "0": the H3 conditioner records every hidden state again.
H3_TE_READ_CAPTURE_ENV = "UNSLOTH_H3_TE_READ_CAPTURE"
diff --git a/studio/backend/tests/test_h3_host_pin_trim.py b/studio/backend/tests/test_h3_host_pin_trim.py
new file mode 100644
index 00000000000..7446fa42bb2
--- /dev/null
+++ b/studio/backend/tests/test_h3_host_pin_trim.py
@@ -0,0 +1,225 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
+
+"""CPU tests: MiniMax-H3 host pin arenas trimmed to payload; the host-looked-up vocab table stays mapped."""
+
+from __future__ import annotations
+
+import inspect
+import logging
+import types
+
+import pytest
+
+torch = pytest.importorskip("torch")
+
+from core.inference import video_minimax_h3_te as te # noqa: E402
+
+
[email protected](autouse = True)
+def _env(monkeypatch):
+ monkeypatch.delenv("UNSLOTH_H3_HOST_PIN_TRIM", raising = False)
+ monkeypatch.delenv(te.H3_HOST_EMBED_ROWS_ENV, raising = False)
+ monkeypatch.delenv("UNSLOTH_DIFFUSION_GROUP_OFFLOAD_PIN", raising = False)
+
+
+def _module(kb):
+ torch.manual_seed(0)
+ m = torch.nn.Sequential(*[torch.nn.Linear(64, 64, bias = False) for _ in range(kb // 16)])
+ return m # 16 KiB per fp32 64x64 weight
+
+
+def _pin(m, arena_bytes):
+ sizes = []
+
+ def factory(size):
+ sizes.append(size)
+ return torch.zeros(size, dtype = torch.uint8)
+
+ payload = te.pin_module_in_place(m, arena_bytes = arena_bytes, _arena_factory = factory)
+ return payload, sizes
+
+
+def test_an_arena_holds_what_it_is_given_not_a_whole_arena():
+ """Scaled: 304 KiB of weights against 1 MiB arenas (the 0.61 GB audio VAE against 2 GiB ones)."""
+ m = _module(304)
+ expect = {k: v.clone() for k, v in m.state_dict().items()}
+ payload, sizes = _pin(m, 1 << 20)
+ assert payload == 304 * 1024
+ assert sizes == [304 * 1024] # page-exact, not 1 MiB
+ for k, v in m.state_dict().items():
+ assert torch.equal(v, expect[k])
+
+
+def test_full_arenas_stay_full_and_only_tails_are_trimmed():
+ m = _module(2 * 1024 + 48) # 2 MiB + 48 KiB against 1 MiB arenas
+ payload, sizes = _pin(m, 1 << 20)
+ assert sizes == [1 << 20, 1 << 20, 48 * 1024]
+ assert sum(sizes) == payload
+
+
+def test_trim_kill_switch_pins_whole_arenas(monkeypatch):
+ assert _pin(_module(304), 1 << 20)[1] == [304 * 1024]
+ monkeypatch.setenv("UNSLOTH_H3_HOST_PIN_TRIM", "0")
+ _payload, sizes = _pin(_module(304), 1 << 20)
+ assert sizes == [1 << 20]
+
+
+def _host(monkeypatch, *, total_gb, available_gb, limit_gib, used_gib):
+ import psutil
+ from utils import host_memory
+
+ gib = 1 << 30
+ monkeypatch.setattr(
+ host_memory,
+ "cgroup_memory_budgets",
+ lambda *a, **k: [((limit_gib - used_gib) * gib, limit_gib * gib)],
+ )
+ monkeypatch.setattr(
+ psutil,
+ "virtual_memory",
+ lambda: types.SimpleNamespace(total = int(total_gb * 1e9), available = int(available_gb * 1e9)),
+ )
+
+
+def test_pins_stay_inside_the_cgroup_budget_they_were_admitted_against(monkeypatch):
+ """Pinned bytes = admitted payload, not arena-rounded (which overran the granted cgroup headroom)."""
+ import core.inference.diffusion_memory as dm
+
+ monkeypatch.setattr(dm, "_pinned_memory_capped", lambda: False)
+ _host(monkeypatch, total_gb = 94, available_gb = 80, limit_gib = 70, used_gib = 50)
+ # min(80 GB available, 20 GiB cgroup headroom) less max(4 GiB, 15% of the 70 GiB limit)
+ assert dm._pin_budget_mib() == 20 * 1024 - int(70 * 1024 * 0.15)
+ assert te.h3_te_pin_allowed(int(9.0 * 2**30))
+ assert not te.h3_te_pin_allowed(int(9.6 * 2**30))
+ # a payload one arena-tail over a whole arena: pinned bytes == payload rounded to a page
+ m = _module(1024 + 304)
+ payload, sizes = _pin(m, 1 << 20)
+ assert sum(sizes) == -(-payload // 4096) * 4096
+
+
+class _TinyLM(torch.nn.Module):
+ def __init__(self):
+ super().__init__()
+ torch.manual_seed(0)
+ self.embed_tokens = torch.nn.Embedding(512, 32, padding_idx = 0).to(torch.bfloat16)
+ self.proj = torch.nn.Linear(32, 32).to(torch.bfloat16)
+
+ def get_input_embeddings(self):
+ return self.embed_tokens
+
+ def forward(self, input_ids):
+ return self.proj(self.embed_tokens(input_ids))
+
+
+class _TE(torch.nn.Module):
+ def __init__(self):
+ super().__init__()
+ self.model = _TinyLM()
+
+
+def _stream_pinned(monkeypatch):
+ pytest.importorskip("diffusers.hooks")
+ import diffusers.hooks as hooks
+
+ import core.inference.diffusion_memory as dm
+ import core.inference.diffusion_prequant as prequant
+ from core.inference import host_pinning
+
+ real_apply = hooks.apply_group_offloading
+ seen = {}
+
+ def fake_apply(
+ module,
+ onload_device,
+ offload_device = torch.device("cpu"),
+ offload_type = "block_level",
+ use_stream = False,
+ non_blocking = False,
+ record_stream = False,
+ low_cpu_mem_usage = False,
+ ):
+ seen.update(low_cpu_mem_usage = low_cpu_mem_usage, use_stream = use_stream)
+ real_apply(
+ module,
+ onload_device = torch.device("cpu"),
+ offload_device = torch.device("cpu"),
+ offload_type = "leaf_level",
+ use_stream = False,
+ )
+
+ assert "low_cpu_mem_usage" in inspect.signature(fake_apply).parameters
+ monkeypatch.setattr(hooks, "apply_group_offloading", fake_apply)
+ monkeypatch.setattr(host_pinning, "host_pinning_allowed", lambda device = None: True)
+ monkeypatch.setattr(prequant, "_unhook_from_manager", lambda *a, **k: True)
+ monkeypatch.setattr(prequant, "_evict_rotation_hook", lambda *a, **k: (lambda *x, **y: None))
+ monkeypatch.setattr(dm, "install_group_offload_hooks_eager", lambda: False)
+ monkeypatch.setattr(dm, "_pin_vision_embedding_device", lambda target: None)
+ arenas = []
+
+ def arena(size, fill):
+ buf = torch.zeros(size, dtype = torch.uint8)
+ fill(buf)
+ arenas.append(buf)
+ return buf
+
+ monkeypatch.setattr(te, "_pinned_arena", arena)
+ reference = _TE()
+ streamed = _TE()
+ original_table = streamed.model.embed_tokens.weight.data_ptr()
+ logger = logging.getLogger("h3_pin_trim_test")
+ mode = te.stream_h3_text_encoder(object(), streamed, "cuda", pin = True, logger = logger)
+ return reference, streamed, mode, seen, arenas, original_table
+
+
+def _in_arena(t, arenas):
+ return t.untyped_storage().data_ptr() in {a.untyped_storage().data_ptr() for a in arenas}
+
+
+def test_the_vocabulary_table_stays_on_its_mapping_and_lookups_are_identical(monkeypatch, caplog):
+ with caplog.at_level(logging.INFO, logger = "h3_pin_trim_test"):
+ reference, streamed, mode, seen, arenas, original = _stream_pinned(monkeypatch)
+ assert mode == "stream"
+ table = streamed.model.embed_tokens.weight
+ assert table.data_ptr() == original and not _in_arena(table, arenas)
+ assert _in_arena(streamed.model.proj.weight, arenas)
+ assert (
+ sum(a.numel() for a in arenas) < table.numel() * table.element_size()
+ ) # the table is not pinned
+ assert (
+ seen["low_cpu_mem_usage"] is True
+ ) # diffusers keeps the pinned rest as is, makes no pinned twin
+ ids = torch.tensor([[0, 3, 7, 511, 3]])
+ with torch.no_grad():
+ assert torch.equal(reference.model(ids), streamed.model(ids))
+ assert any(m.startswith("video.h3_host_pin_trim: conditioner pins") for m in caplog.messages)
+
+
+def test_trim_kill_switch_pins_the_table_again(monkeypatch):
+ _r, streamed, _m, _s, arenas, _o = _stream_pinned(monkeypatch)
+ assert not _in_arena(streamed.model.embed_tokens.weight, arenas)
+ monkeypatch.setenv("UNSLOTH_H3_HOST_PIN_TRIM", "0")
+ reference, streamed, mode, seen, arenas, original = _stream_pinned(monkeypatch)
+ assert mode == "stream"
+ assert _in_arena(streamed.model.embed_tokens.weight, arenas)
+ assert seen["low_cpu_mem_usage"] is False
+ ids = torch.tensor([[1, 2, 3]])
+ with torch.no_grad():
+ assert torch.equal(reference.model(ids), streamed.model(ids))
+
+
+def test_a_table_left_in_the_stream_is_pinned_after_all(monkeypatch):
+ """Host rows off (or refused): the table streams, so it must not stream from pageable memory."""
+ monkeypatch.setattr(te, "keep_h3_embedding_on_host", lambda target, logger = None: False)
+ reference, streamed, mode, seen, arenas, original = _stream_pinned(monkeypatch)
+ table = streamed.model.embed_tokens.weight
+ assert seen["low_cpu_mem_usage"] is True # left out of the up-front pin...
+ assert _in_arena(table, arenas) # ...and pinned once it stayed in the stream
+ from core.inference.diffusion_memory import _offload_groups
+
+ hosts = [
+ g.cpu_param_dict.get(table)
+ for g in _offload_groups(streamed.model)
+ if table in (getattr(g, "cpu_param_dict", None) or {})
+ ]
+ assert all(h is None or h.data_ptr() == table.data_ptr() for h in hosts)
diff --git a/studio/backend/tests/test_h3_te_device_resident.py b/studio/backend/tests/test_h3_te_device_resident.py
new file mode 100644
index 00000000000..8f1c8dd0918
--- /dev/null
+++ b/studio/backend/tests/test_h3_te_device_resident.py
@@ -0,0 +1,339 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
+
+"""CPU tests: MiniMax-H3 conditioner device share takes only leftover budget and yields before any block demotes."""
+
+from __future__ import annotations
+
+import logging
+import random
+
+import pytest
+
+torch = pytest.importorskip("torch")
+
+
+class _Stream:
+ def __init__(self):
+ self.syncs = 0
+
+ def synchronize(self):
+ self.syncs += 1
+
+
+class _TeGroup:
+ """A diffusers-style streamed leaf group: onload gives the weights new storage, offload re-points at the host copy."""
+
+ def __init__(
+ self,
+ leaf,
+ events = None,
+ tag = None,
+ ):
+ self.modules = [leaf]
+ self.parameters = []
+ self.buffers = []
+ self.stream = _Stream()
+ self.onload_leader = leaf
+ self.cpu_param_dict = {p: p.data for p in leaf.parameters()}
+ self.where = "cpu"
+ self.on = 0
+ self.off = 0
+ self.events = events
+ self.tag = tag
+
+ def onload_(self):
+ self.on += 1
+ for p, host in self.cpu_param_dict.items():
+ p.data = host.clone()
+ self.where = "cuda"
+ if self.events is not None:
+ self.events.append(("on", self.tag))
+
+ def offload_(self):
+ self.off += 1
+ for p in self.modules[0].parameters():
+ p.data = self.cpu_param_dict[p]
+ self.where = "cpu"
+ if self.events is not None:
+ self.events.append(("off", self.tag))
+
+
+class _Block(_TeGroup):
+ pass
+
+
+class _Model(torch.nn.Module):
+ def __init__(
+ self,
+ n_layers = 3,
+ width = 16,
+ ):
+ super().__init__()
+ torch.manual_seed(0)
+ self.visual = torch.nn.Sequential(torch.nn.Linear(width, width))
+ self.language_model = torch.nn.Module()
+ self.language_model.layers = torch.nn.ModuleList()
+ for _ in range(n_layers):
+ layer = torch.nn.Module()
+ layer.q = torch.nn.Linear(width, width, bias = False)
+ layer.mlp = torch.nn.Linear(width, 4 * width, bias = False)
+ self.language_model.layers.append(layer)
+
+ def forward(self, x):
+ for layer in self.language_model.layers:
+ x = x + layer.mlp(layer.q(x)).view(*x.shape[:-1], 4, -1).sum(-2)
+ return x
+
+
+class _TE(torch.nn.Module):
+ def __init__(self, **kw):
+ super().__init__()
+ self.model = _Model(**kw)
+
+
+def _share(
+ monkeypatch,
+ te_module,
+ events = None,
+ logger = None,
+):
+ import core.inference.diffusion_memory as dm
+ from core.inference.video_minimax_h3_te import H3TeDeviceShare
+
+ groups = [
+ _TeGroup(m, events, name)
+ for name, m in te_module.model.named_modules()
+ if isinstance(m, torch.nn.Linear)
+ ]
+ monkeypatch.setattr(dm, "_offload_groups", lambda target: list(groups))
+ return H3TeDeviceShare(te_module, "cuda", logger = logger), groups
+
+
+def _residency(
+ block_bytes,
+ events = None,
+ top_bytes = 0,
+):
+ from core.inference.video_minimax_h3_residency import H3Residency, group_payload_bytes
+
+ r = H3Residency.__new__(H3Residency)
+
+ def block(nbytes, tag):
+ return _Block(torch.nn.Linear(1, nbytes // 4, bias = False), events, tag)
+
+ r.top = block(top_bytes, "top") if top_bytes else None
+ r.blocks = [block(b, f"b{i}") for i, b in enumerate(block_bytes)]
+ r.device = "cuda"
+ r.logger = None
+ r.top_bytes = group_payload_bytes(r.top) if r.top is not None else 0
+ r.block_bytes = [group_payload_bytes(g) for g in r.blocks]
+ r.max_blocks = 0
+ r.te_share = None
+ return r
+
+
[email protected](autouse = True)
+def _cpu(monkeypatch):
+ monkeypatch.setattr(torch.cuda, "is_available", lambda: False)
+ monkeypatch.delenv("UNSLOTH_H3_TE_DEVICE_RESIDENT", raising = False)
+ monkeypatch.delenv("UNSLOTH_H3_RESIDENT_UNPIN", raising = False)
+ monkeypatch.delenv("UNSLOTH_H3_HOST_PIN_TRIM", raising = False)
+
+
+def test_share_reads_language_layers_in_order_and_leaves_the_vision_tower_streamed(monkeypatch):
+ te_module = _TE()
+ share, groups = _share(monkeypatch, te_module)
+ assert [g.tag for g in share.groups] == [
+ f"language_model.layers.{i}.{leaf}" for i in range(3) for leaf in ("q", "mlp")
+ ]
+ assert share.layer_of == [0, 0, 1, 1, 2, 2] and share.layers == 3
+ assert all(g.tag != "visual.0" for g in share.groups)
+
+
+def test_budget_fits_every_block_then_a_conditioner_prefix(monkeypatch):
+ te_module = _TE()
+ share, _ = _share(monkeypatch, te_module)
+ r = _residency([4096] * 4)
+ r.te_share = share
+ room = sum(share.planned[:3])
+ r.fit(sum(r.block_bytes) + room, initial = True)
+ assert r.resident_blocks() == 4
+ assert share.resident_count() == 3
+ assert share.resident_bytes() <= room
+ assert r.device_bytes() == r.resident_bytes() + share.resident_bytes()
+ assert [g.where for g in share.groups] == ["cuda"] * 3 + ["cpu"] * 3
+
+
+def test_nothing_for_the_conditioner_unless_every_block_is_resident(monkeypatch):
+ """12 / 16 GB: the denoiser keeps a block prefix, and the leftover (smaller than a block) is not handed out."""
+ te_module = _TE()
+ share, _ = _share(monkeypatch, te_module)
+ r = _residency([4096] * 4)
+ r.te_share = share
+ r.fit(3 * r.block_bytes[0] + r.block_bytes[0] - 1, initial = True)
+ assert r.resident_blocks() == 3
+ assert share.plan(r.block_bytes[0] - 1) > 0 # it would fit the leftover
+ assert share.resident_count() == 0 and share.resident_bytes() == 0
+ # a streamed top-level group, or one block short: nothing either
+ r2 = _residency([4096] * 2, top_bytes = 8192)
+ assert r2.te_budget(10**9, False, len(r2.blocks)) == 0
+ assert r2.te_budget(10**9, True, len(r2.blocks) - 1) == 0
+ assert r2.te_budget(10**9, True, len(r2.blocks)) == 10**9 - r2.top_bytes - sum(r2.block_bytes)
+
+
+def test_a_bigger_request_takes_the_share_back_before_demoting_a_block(monkeypatch):
+ events: list = []
+ te_module = _TE()
+ share, groups = _share(monkeypatch, te_module, events)
+ r = _residency([4096] * 4, events)
+ r.te_share = share
+ full = sum(r.block_bytes)
+ r.fit(full + sum(share.planned), initial = True)
+ assert share.resident_count() == len(share.groups) and r.resident_blocks() == 4
+ events.clear()
+ # exactly the denoiser's bytes: the share goes, every block stays
+ r.fit(full)
+ assert share.resident_count() == 0 and r.resident_blocks() == 4
+ assert all(kind == "off" and not tag.startswith("b") for kind, tag in events), events
+ events.clear()
+ r.fit(full + sum(share.planned[:2]))
+ assert share.resident_count() == 2
+ events.clear()
+ # one byte short: the share is given back first, then the last block
+ r.fit(full - 1)
+ offs = [tag for kind, tag in events if kind == "off"]
+ assert offs[-1] == "b3" and all(not t.startswith("b") for t in offs[:-1]), offs
+ assert share.resident_count() == 0 and r.resident_blocks() == 3
+
+
+def test_resident_groups_keep_the_side_stream_sync_and_skip_the_moves(monkeypatch):
+ te_module = _TE()
+ share, groups = _share(monkeypatch, te_module)
+ share.grow(2)
+ g = share.groups[0]
+ assert g.where == "cuda" and g.on == 1
+ syncs = g.stream.syncs
+ g.onload_() # the group before it prefetching it: no move, but the stream is synced
+ assert g.on == 1 and g.stream.syncs == syncs + 1
+ g.offload_()
+ assert g.off == 0 and g.where == "cuda"
+ share.shrink(0)
+ assert "onload_" not in g.__dict__ and "offload_" not in g.__dict__
+ assert g.where == "cpu" and g.off == 1
+ for p, host in g.cpu_param_dict.items():
+ assert p.data.data_ptr() == host.data_ptr() # back on the host copy it never dropped
+
+
+def test_output_is_bit_identical_resident_or_streamed(monkeypatch):
+ te_module = _TE()
+ x = torch.randn(2, 16)
+ with torch.no_grad():
+ expect = te_module.model(x)
+ share, _ = _share(monkeypatch, te_module)
+ for n in (6, 3, 0, 5):
+ share.shrink(n)
+ share.grow(n)
+ with torch.no_grad():
+ assert torch.equal(te_module.model(x), expect)
+
+
+def test_the_budget_counts_the_share_as_demotable(monkeypatch):
+ """The share lowers free VRAM, so the budget must add it back or every request would shrink it."""
+ import core.inference.diffusion_memory as dm
+ from core.inference import video as vid
+
+ te_module = _TE()
+ share, _ = _share(monkeypatch, te_module)
+ r = _residency([4096] * 4)
+ r.fit(10**9, initial = True)
+ base_free = 30 * 10**9
+ monkeypatch.setattr(dm, "cuda_reusable_cache_bytes", lambda device = None: 0)
+ monkeypatch.setattr(vid, "_h3_signed_free_device_bytes", lambda device: base_free)
+ kw = dict(width = 960, height = 544, frames = 124, te_streamed = True, te_scheme = "int8")
+ without = vid._h3_residency_budget_bytes(r, "cuda", **kw)
+ r.te_share = share
+ share.grow(4)
+ held = share.resident_bytes()
+ assert held > 0
+ monkeypatch.setattr(vid, "_h3_signed_free_device_bytes", lambda device: base_free - held)
+ assert vid._h3_residency_budget_bytes(r, "cuda", **kw) == without
+
+
+def test_randomized_requests_keep_the_denoiser_and_the_device_peak_of_the_base(monkeypatch):
+ """Simulated card: denoiser residency matches base at every request and never overcommits the card."""
+ rng = random.Random(7)
+ for trial in range(40):
+ blocks = [rng.choice([4096, 8192]) for _ in range(rng.randint(1, 5))]
+ te_module = _TE(n_layers = rng.randint(1, 4))
+ share, _ = _share(monkeypatch, te_module)
+ head, base = _residency(blocks), _residency(blocks)
+ head.te_share = share
+ card = sum(head.block_bytes) + rng.randint(0, sum(share.planned) * 2)
+ for k in range(8):
+ need = rng.randint(0, card // 3)
+ outs = []
+ for r in (base, head):
+ used = r.device_bytes()
+ free = card - used
+ budget = free + r.device_bytes() - need
+ r.fit(max(0, budget), initial = k == 0)
+ peak = r.device_bytes() + need
+ assert peak <= card, (trial, k, peak, card)
+ outs.append(r.resident_blocks())
+ assert outs[0] == outs[1], (trial, k)
+
+
+def test_kill_switch_and_modes(monkeypatch):
+ from core.inference.video_minimax_h3_te import h3_te_device_share
+
+ te_module = _TE()
+ _share(monkeypatch, te_module)
+ assert h3_te_device_share(te_module, "cuda", "stream") is not None
+ assert h3_te_device_share(te_module, "cuda", "stream_lazy") is not None
+ assert h3_te_device_share(te_module, "cuda", "sync") is None # integrated GPU: one memory
+ assert h3_te_device_share(te_module, "cuda", None) is None
+ monkeypatch.setenv("UNSLOTH_H3_TE_DEVICE_RESIDENT", "0")
+ assert h3_te_device_share(te_module, "cuda", "stream") is None
+
+
+def test_engagement_log_line(monkeypatch, caplog):
+ logger = logging.getLogger("h3_te_device_test")
+ te_module = _TE()
+ share, _ = _share(monkeypatch, te_module, logger = logger)
+ r = _residency([4096] * 2)
+ r.te_share = share
+ with caplog.at_level(logging.INFO, logger = "h3_te_device_test"):
+ r.fit(sum(r.block_bytes) + sum(share.planned[:4]), initial = True)
+ lines = [m for m in caplog.messages if m.startswith("video.h3_te_resident_device:")]
+ assert len(lines) == 1 and lines[0].startswith(
+ "video.h3_te_resident_device: 2 of 3 TE layers resident on cuda"
+ ), lines
+
+
+def test_a_failed_onload_leaves_the_conditioner_streamed(monkeypatch):
+ te_module = _TE()
+ share, groups = _share(monkeypatch, te_module)
+ r = _residency([4096])
+ r.te_share = share
+
+ def boom():
+ raise RuntimeError("CUDA out of memory")
+
+ share.groups[2].onload_ = boom
+ r.fit(10**9, initial = True)
+ assert r.resident_blocks() == 1
+ assert share.resident_count() == 0 and all(g.where == "cpu" for g in share.groups)
+
+
+def test_release_all_returns_the_share(monkeypatch):
+ from core.inference.video_minimax_h3_residency import release_all
+
+ te_module = _TE()
+ share, _ = _share(monkeypatch, te_module)
+ r = _residency([4096])
+ r.te_share = share
+ r.fit(10**9, initial = True)
+ assert share.resident_count() == len(share.groups)
+ release_all(r)
+ assert share.resident_count() == 0 and all(g.where == "cpu" for g in share.groups)