Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
Studio MiniMax-H3: keep conditioner layers on the card beside a resident denoiser, size its host pins to their weights (#17355)
Daniel Han4d709a58c8f2
diff --git a/studio/backend/core/inference/video.py b/studio/backend/core/inference/video.pyindex a108805529e..b83ffa96f36 100644--- a/studio/backend/core/inference/video.py+++ b/studio/backend/core/inference/video.py@@ -3255,7 +3255,11 @@ def _h3_residency_budget_bytes(if free is None:return 0cached = cuda_reusable_cache_bytes(device)- available = int(free) + cached + int(residency.resident_bytes())+ # The conditioner's device share is demotable like a resident block (and it yields first).+ held = getattr(residency, "device_bytes", None)+ available = (+ int(free) + cached + int(held() if callable(held) else residency.resident_bytes())+ )te_gb = (H3_TE_STREAMED_GBif te_streamed@@ -12449,6 +12453,11 @@ class VideoBackend:_te, logger = logger))+ from .video_minimax_h3_te import h3_te_device_share++ residency.te_share = h3_te_device_share(+ _te_for_yield, device, te_streamed, logger = logger+ )big_w, big_h = max(fam.resolution_presets, key = lambda wh: wh[0] * wh[1])diff --git a/studio/backend/core/inference/video_minimax_h3_residency.py b/studio/backend/core/inference/video_minimax_h3_residency.pyindex 1bb154bf0bf..3dac656a97d 100644--- a/studio/backend/core/inference/video_minimax_h3_residency.py+++ b/studio/backend/core/inference/video_minimax_h3_residency.py@@ -627,6 +627,8 @@ class H3Residency:# bytes of slabs the last ``apply`` compacted; a compaction that could not fit every slab retries next re-fitself.compacted_bytes = 0self.compact_pending = False+ # video_minimax_h3_te.H3TeDeviceShare+ self.te_share: Any = None@propertydef usable(self) -> bool:@@ -644,6 +646,17 @@ class H3Residency:total = self.top_bytes if (self.top is not None and is_resident(self.top)) else 0return total + sum(b for g, b in zip(self.blocks, self.block_bytes) if is_resident(g))+ def device_bytes(self) -> int:+ te = getattr(self, "te_share", None)+ return self.resident_bytes() + (te.resident_bytes() if te is not None else 0)++ def te_budget(self, budget_bytes: int, top: bool, n: int) -> int:+ """Zero unless the whole denoiser is resident: it is read every step, the conditioner once."""+ if n < len(self.blocks) or (self.top is not None and not top):+ return 0+ planned = (self.top_bytes if top else 0) + sum(self.block_bytes[:n])+ return max(0, int(budget_bytes) - planned)+def plan(self,budget_bytes: int,@@ -704,7 +717,21 @@ class H3Residency:) -> tuple[bool, int]:"""Re-fit the resident set to ``budget_bytes`` (the device bytes the resident set may occupy)."""top, n = self.plan(budget_bytes)+ te = getattr(self, "te_share", None)+ te_left = self.te_budget(budget_bytes, top, n) if te is not None else 0+ te_n = te.plan(te_left) if te is not None else 0+ if te is not None:+ # the conditioner gives its share back before any denoiser block is promoted+ te.shrink(te_n)self.apply(top, n)+ if te is not None:+ try:+ te.grow(te_n)+ except Exception as exc: # noqa: BLE001 -- the conditioner streams as before+ te.shrink(0)+ if self.logger is not None:+ self.logger.warning("video.h3_te_resident_device: staying streamed: %s", exc)+ te.report(te_left, force = initial)if initial:self.max_blocks = nreleased = getattr(self, "released_bytes", 0)@@ -738,6 +765,13 @@ def release_all(residency: "H3Residency", *, logger: Any = None) -> None:"""Return every group of ``residency`` to streaming, including one whose onload failed partway. Never raises."""import torch+ te = getattr(residency, "te_share", None)+ if te is not None:+ try:+ te.shrink(0)+ except Exception as exc: # noqa: BLE001+ if logger is not None:+ logger.warning("video.h3_te_resident_device: could not release the share: %s", exc)groups = list(reversed(residency.blocks))if residency.top is not None:groups.append(residency.top)diff --git a/studio/backend/core/inference/video_minimax_h3_te.py b/studio/backend/core/inference/video_minimax_h3_te.pyindex 6eb7348dda3..7f2de2c3b99 100644--- a/studio/backend/core/inference/video_minimax_h3_te.py+++ b/studio/backend/core/inference/video_minimax_h3_te.py@@ -498,21 +498,38 @@ def _pinned_arena(size: int, fill: Any) -> Any:return arena+# Arenas sized to their payload (0.61 GB audio VAE pinned 2.15 GB); the host-looked-up vocab table stays mapped. =0 reverts.+H3_HOST_PIN_TRIM_ENV = "UNSLOTH_H3_HOST_PIN_TRIM"+_PIN_PAGE = 4096+++def h3_host_pin_trim_enabled() -> bool:+ import os+ return str(os.environ.get(H3_HOST_PIN_TRIM_ENV, "")).strip().lower() not in (+ "0",+ "off",+ "false",+ "no",+ )++def pin_module_in_place(module: Any,*,arena_bytes: int = _PIN_ARENA_BYTES,_arena_factory: Any = None,repoint: bool = False,+ skip: tuple = (),) -> int:"""Replaces tensors in _parameters/_buffers rather than .data, which would keep the mmap alive."""import torch+ skipped = {id(t) for t in skip if t is not None}slots: dict[int, list] = {} # id(tensor) -> [tensor, [(owner dict, name, is_param)]]for sub in module.modules():for table, is_param in ((sub._parameters, True), (sub._buffers, False)):for name, tensor in list(table.items()):- if tensor is None:+ if tensor is None or id(tensor) in skipped:continueentry = slots.setdefault(id(tensor), [tensor, []])entry[1].append((table, name, is_param))@@ -537,6 +554,10 @@ def pin_module_in_place(slot[2].append((tensor, owners, slot[1]))slot[1] += needpinned += nbytes+ if h3_host_pin_trim_enabled():+ # placement is final: an arena's unused tail would only be zeroed and locked+ for arena in arenas:+ arena[0] = min(arena[0], _round_up(arena[1], _PIN_PAGE))placed = [] # (tensor, owners, pinned view)for size, _used, entries in arenas:fill = functools.partial(_fill_arena, entries = entries)@@ -638,6 +659,43 @@ def _host_rows_forward(self: Any, input: Any) -> Any:return rows.to(self.__dict__[_HOST_ROWS_ATTR])+def _host_rows_table(target: Any) -> Any:+ """The table ``keep_h3_embedding_on_host`` will take out of the stream (same static checks), or None."""+ try:+ import torch++ if target is None or not h3_host_embed_rows_enabled():+ return None+ getter = getattr(target, "get_input_embeddings", None)+ emb = getter() if callable(getter) else None+ if type(emb) is not torch.nn.Embedding or _HOST_ROWS_ATTR in emb.__dict__:+ return None+ weight = emb.weight+ if (+ type(weight.data) is not torch.Tensor+ or weight.device.type != "cpu"+ or emb.max_norm is not None+ or emb.sparse+ or weight.dtype not in (torch.bfloat16, torch.float16, torch.float32)+ ):+ return None+ return weight+ except Exception: # noqa: BLE001 -- pin it with the rest+ return None+++def _pin_streamed_table(target: Any, weight: Any) -> None:+ """The table stayed in the stream after all: pin it as the whole-module pin would have."""+ from .diffusion_memory import _offload_groups++ owner = next(m for m in target.modules() if m._parameters.get("weight") is weight)+ pin_module_in_place(owner, repoint = True)+ for group in _offload_groups(target):+ cpu = getattr(group, "cpu_param_dict", None)+ if isinstance(cpu, dict) and weight in cpu:+ cpu[weight] = weight.data++def keep_h3_embedding_on_host(target: Any, logger: Any = None) -> bool:"""Take the group-offloaded language embedding out of the stream, its lookup done on the host. Run right after``apply_group_offloading``; True when installed. Plain frozen bf16/fp16/fp32 tables only. Never raises."""@@ -754,13 +812,20 @@ def stream_h3_text_encoder(pin = Falseelif pin is None:pin = h3_te_pin_allowed(_module_payload_bytes(text_encoder), onload)+ # Looked up on the host, so a pinned copy buys nothing; needs diffusers to take the pinned rest as it is.+ table = (+ _host_rows_table(target)+ if pin and h3_host_pin_trim_enabled() and "low_cpu_mem_usage" in params+ else None+ )if pin:try:- pin_module_in_place(text_encoder)+ pinned_bytes = pin_module_in_place(text_encoder, skip = (table,))except Exception as exc: # noqa: BLE001 -- pinned host RAM refused: stream unpinnedif logger is not None:logger.warning("video.h3_te_stream: pinning failed, streaming unpinned: %s", exc)pin = False+ table = Nonekwargs: dict[str, Any] = {"onload_device": onload,"offload_device": torch.device("cpu"),@@ -774,7 +839,8 @@ def stream_h3_text_encoder(kwargs["record_stream"] = Trueif "low_cpu_mem_usage" in params:# Already pinned in place, so diffusers' pin_memory() returns the same tensor: no second host copy.- kwargs["low_cpu_mem_usage"] = not pin+ # With the table left mapped, the onload pins only what is not pinned yet: nothing once the table is out.+ kwargs["low_cpu_mem_usage"] = not pin or table is not Noneelif not pin:return Nonetry:@@ -783,7 +849,22 @@ def stream_h3_text_encoder(install_group_offload_buffer_restore()install_group_offload_hooks_eager()apply_group_offloading(target, **kwargs)- keep_h3_embedding_on_host(target, logger = logger)+ on_host = keep_h3_embedding_on_host(target, logger = logger)+ if table is not None:+ if on_host:+ if logger is not None:+ logger.info(+ "video.h3_host_pin_trim: conditioner pins %.2f GB in arenas sized to their weights, its "+ "vocabulary table (%.2f GB) stays on its mapping",+ pinned_bytes / 1e9,+ table.numel() * table.element_size() / 1e9,+ )+ else:+ try:+ _pin_streamed_table(target, table)+ except Exception as exc: # noqa: BLE001 -- streams pageable: correct, slower+ if logger is not None:+ logger.warning("video.h3_host_pin_trim: table not pinned: %s", exc)# Image prompts: Qwen3-VL's position interpolation reads the offloaded embedding's CPU device (transformers 5.5)._pin_vision_embedding_device(target)except Exception as exc: # noqa: BLE001 -- keep the rotation@@ -1015,6 +1096,232 @@ def _copy_host_tensors_(entries: list, undo: Optional[list] = None) -> int:return copied+# Once every denoiser block is resident, leftover budget keeps conditioner leaf groups on device in read order (48 GB:+# ~0.8 s per render saved); requests needing more take it back first. Vision tower keeps streaming. =0 disables.+H3_TE_DEVICE_RESIDENT_ENV = "UNSLOTH_H3_TE_DEVICE_RESIDENT"+_TE_DEVICE_ATTR = "_unsloth_te_device"+_TE_SAVED_IO_ATTR = "_unsloth_te_saved_io"+# Planned device bytes per tensor: the allocator's 512-byte rounding plus an unsplit remainder (at most 1 MiB).+_ALLOC_ROUND = 512+_ALLOC_SPLIT_SLACK = 1 << 20+++def h3_te_device_resident_enabled() -> bool:+ import os+ return str(os.environ.get(H3_TE_DEVICE_RESIDENT_ENV, "")).strip().lower() not in (+ "0",+ "off",+ "false",+ "no",+ )+++def _planned_device_bytes(group: Any) -> int:+ from .video_minimax_h3_residency import _group_tensors, _tensor_bytes++ total = 0+ for t in _group_tensors(group):+ n = _tensor_bytes(t)+ total += _round_up(n, _ALLOC_ROUND) + (_ALLOC_SPLIT_SLACK if n > _ALLOC_SPLIT_SLACK else 0)+ return total+++def _allocated_bytes(device: Any) -> Optional[int]:+ try:+ import torch++ dev = torch.device(device)+ if dev.type != "cuda" or not torch.cuda.is_available():+ return None+ return int(torch.cuda.memory_allocated(dev))+ except Exception: # noqa: BLE001+ return None+++def _te_resident_onload(group: Any) -> Any:+ """Keep the side-stream sync: the next group may still be arriving via the previous group's prefetch."""+ stream = getattr(group, "stream", None)++ def onload_(*_a: Any, **_k: Any) -> None:+ if stream is not None:+ stream.synchronize()++ try:+ import torch+ return torch.compiler.disable(onload_)+ except Exception: # noqa: BLE001+ return onload_+++def _te_noop(*_a: Any, **_k: Any) -> None:+ return None+++class H3TeDeviceShare:+ """The streamed conditioner's leaf groups resident on the device: a prefix in read order (language layers first)."""++ def __init__(+ self,+ text_encoder: Any,+ device: Any,+ *,+ logger: Any = None,+ ) -> None:+ from .diffusion_memory import _offload_groups+ from .video_minimax_h3_residency import _group_tensors++ target = getattr(text_encoder, "model", None)+ self.device = device+ self.logger = logger+ self.groups: list = []+ self.layer_of: list = []+ names = {id(m): n for n, m in target.named_modules()} if target is not None else {}+ for group in _offload_groups(target) if target is not None else []:+ mods = list(getattr(group, "modules", None) or ())+ leader = mods[0] if mods else getattr(group, "onload_leader", None)+ parts = names.get(id(leader), "").split(".")+ if (+ "visual" in parts+ or getattr(group, "stream", None) is None+ or not isinstance(getattr(group, "cpu_param_dict", None), dict)+ or not _group_tensors(group)+ ):+ continue+ layer = None+ if "layers" in parts and parts.index("layers") + 1 < len(parts):+ index = parts[parts.index("layers") + 1]+ layer = int(index) if index.isdigit() else None+ self.groups.append(group)+ self.layer_of.append(layer)+ self.planned = [_planned_device_bytes(g) for g in self.groups]+ self.layers = len({x for x in self.layer_of if x is not None})+ self._logged: Optional[int] = None++ @property+ def usable(self) -> bool:+ return bool(self.groups)++ def resident_count(self) -> int:+ n = 0+ for group in self.groups:+ if not group.__dict__.get(_TE_DEVICE_ATTR):+ break+ n += 1+ return n++ def resident_bytes(self) -> int:+ """Measured at onload; planned bytes off CUDA."""+ return sum(+ int(group.__dict__.get(_TE_DEVICE_ATTR) or 0)+ for group in self.groups+ if group.__dict__.get(_TE_DEVICE_ATTR)+ )++ def plan(self, budget_bytes: int) -> int:+ n = 0+ left = int(budget_bytes)+ while n < len(self.groups) and self.planned[n] <= left:+ left -= self.planned[n]+ n += 1+ return n++ def shrink(self, n: int) -> None:+ import torch+ demoted = False+ for group in reversed(self.groups[n:]):+ if not group.__dict__.get(_TE_DEVICE_ATTR):+ continue+ if not demoted and torch.cuda.is_available():+ torch.cuda.current_stream().synchronize()+ saved_on, saved_off = group.__dict__.get(_TE_SAVED_IO_ATTR, (None, None))+ for name, saved in (("onload_", saved_on), ("offload_", saved_off)):+ if saved is not None:+ setattr(group, name, saved)+ else:+ group.__dict__.pop(name, None)+ group.__dict__.pop(_TE_DEVICE_ATTR, None)+ group.offload_() # back to its host copy, which was never dropped+ demoted = True++ def grow(self, n: int) -> None:+ import torch+ for group in self.groups[:n]:+ if group.__dict__.get(_TE_DEVICE_ATTR):+ continue+ before = _allocated_bytes(self.device)+ stream = getattr(group, "stream", None)+ try:+ group.onload_()+ if stream is not None:+ stream.synchronize()+ if torch.cuda.is_available():+ torch.cuda.current_stream().synchronize()+ except Exception:+ group.offload_() # a half-onloaded group back on its host copy+ raise+ after = _allocated_bytes(self.device)+ held = after - before if before is not None and after is not None else None+ if held is None or held <= 0:+ held = _planned_device_bytes(group)+ group.__dict__[_TE_SAVED_IO_ATTR] = (+ group.__dict__.get("onload_"),+ group.__dict__.get("offload_"),+ )+ group.onload_ = _te_resident_onload(group)+ group.offload_ = _te_noop+ group.__dict__[_TE_DEVICE_ATTR] = max(1, int(held))++ def report(+ self,+ budget_bytes: int,+ *,+ force: bool = False,+ ) -> None:+ n = self.resident_count()+ if self.logger is None or (n == self._logged and not force):+ return+ self._logged = n+ full = sum(+ 1+ for layer in {x for x in self.layer_of if x is not None}+ if all(+ self.groups[i].__dict__.get(_TE_DEVICE_ATTR)+ for i, x in enumerate(self.layer_of)+ if x == layer+ )+ )+ self.logger.info(+ "video.h3_te_resident_device: %d of %d TE layers resident on %s (%.2f GB, %d of %d leaf groups, "+ "%.2f GB left beside the resident denoiser)",+ full,+ self.layers,+ self.device,+ self.resident_bytes() / 1e9,+ n,+ len(self.groups),+ max(0, budget_bytes) / 1e9,+ )+++def h3_te_device_share(+ text_encoder: Any,+ device: Any,+ mode: Any,+ *,+ logger: Any = None,+) -> Optional[H3TeDeviceShare]:+ """None unless the conditioner streams with a side stream. Never raises."""+ try:+ if mode not in ("stream", "stream_lazy") or not h3_te_device_resident_enabled():+ return None+ share = H3TeDeviceShare(text_encoder, device, logger = logger)+ return share if share.usable else None+ except Exception as exc: # noqa: BLE001 -- the conditioner streams as before+ if logger is not None:+ logger.warning("video.h3_te_resident_device: staying streamed: %s", exc)+ return None++# "0": the H3 conditioner records every hidden state again.H3_TE_READ_CAPTURE_ENV = "UNSLOTH_H3_TE_READ_CAPTURE"diff --git a/studio/backend/tests/test_h3_host_pin_trim.py b/studio/backend/tests/test_h3_host_pin_trim.pynew file mode 100644index 00000000000..7446fa42bb2--- /dev/null+++ b/studio/backend/tests/test_h3_host_pin_trim.py@@ -0,0 +1,225 @@+# SPDX-License-Identifier: AGPL-3.0-only+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0++"""CPU tests: MiniMax-H3 host pin arenas trimmed to payload; the host-looked-up vocab table stays mapped."""++from __future__ import annotations++import inspect+import logging+import types++import pytest++torch = pytest.importorskip("torch")++from core.inference import video_minimax_h3_te as te # noqa: E402++[email protected](autouse = True)+def _env(monkeypatch):+ monkeypatch.delenv("UNSLOTH_H3_HOST_PIN_TRIM", raising = False)+ monkeypatch.delenv(te.H3_HOST_EMBED_ROWS_ENV, raising = False)+ monkeypatch.delenv("UNSLOTH_DIFFUSION_GROUP_OFFLOAD_PIN", raising = False)+++def _module(kb):+ torch.manual_seed(0)+ m = torch.nn.Sequential(*[torch.nn.Linear(64, 64, bias = False) for _ in range(kb // 16)])+ return m # 16 KiB per fp32 64x64 weight+++def _pin(m, arena_bytes):+ sizes = []++ def factory(size):+ sizes.append(size)+ return torch.zeros(size, dtype = torch.uint8)++ payload = te.pin_module_in_place(m, arena_bytes = arena_bytes, _arena_factory = factory)+ return payload, sizes+++def test_an_arena_holds_what_it_is_given_not_a_whole_arena():+ """Scaled: 304 KiB of weights against 1 MiB arenas (the 0.61 GB audio VAE against 2 GiB ones)."""+ m = _module(304)+ expect = {k: v.clone() for k, v in m.state_dict().items()}+ payload, sizes = _pin(m, 1 << 20)+ assert payload == 304 * 1024+ assert sizes == [304 * 1024] # page-exact, not 1 MiB+ for k, v in m.state_dict().items():+ assert torch.equal(v, expect[k])+++def test_full_arenas_stay_full_and_only_tails_are_trimmed():+ m = _module(2 * 1024 + 48) # 2 MiB + 48 KiB against 1 MiB arenas+ payload, sizes = _pin(m, 1 << 20)+ assert sizes == [1 << 20, 1 << 20, 48 * 1024]+ assert sum(sizes) == payload+++def test_trim_kill_switch_pins_whole_arenas(monkeypatch):+ assert _pin(_module(304), 1 << 20)[1] == [304 * 1024]+ monkeypatch.setenv("UNSLOTH_H3_HOST_PIN_TRIM", "0")+ _payload, sizes = _pin(_module(304), 1 << 20)+ assert sizes == [1 << 20]+++def _host(monkeypatch, *, total_gb, available_gb, limit_gib, used_gib):+ import psutil+ from utils import host_memory++ gib = 1 << 30+ monkeypatch.setattr(+ host_memory,+ "cgroup_memory_budgets",+ lambda *a, **k: [((limit_gib - used_gib) * gib, limit_gib * gib)],+ )+ monkeypatch.setattr(+ psutil,+ "virtual_memory",+ lambda: types.SimpleNamespace(total = int(total_gb * 1e9), available = int(available_gb * 1e9)),+ )+++def test_pins_stay_inside_the_cgroup_budget_they_were_admitted_against(monkeypatch):+ """Pinned bytes = admitted payload, not arena-rounded (which overran the granted cgroup headroom)."""+ import core.inference.diffusion_memory as dm++ monkeypatch.setattr(dm, "_pinned_memory_capped", lambda: False)+ _host(monkeypatch, total_gb = 94, available_gb = 80, limit_gib = 70, used_gib = 50)+ # min(80 GB available, 20 GiB cgroup headroom) less max(4 GiB, 15% of the 70 GiB limit)+ assert dm._pin_budget_mib() == 20 * 1024 - int(70 * 1024 * 0.15)+ assert te.h3_te_pin_allowed(int(9.0 * 2**30))+ assert not te.h3_te_pin_allowed(int(9.6 * 2**30))+ # a payload one arena-tail over a whole arena: pinned bytes == payload rounded to a page+ m = _module(1024 + 304)+ payload, sizes = _pin(m, 1 << 20)+ assert sum(sizes) == -(-payload // 4096) * 4096+++class _TinyLM(torch.nn.Module):+ def __init__(self):+ super().__init__()+ torch.manual_seed(0)+ self.embed_tokens = torch.nn.Embedding(512, 32, padding_idx = 0).to(torch.bfloat16)+ self.proj = torch.nn.Linear(32, 32).to(torch.bfloat16)++ def get_input_embeddings(self):+ return self.embed_tokens++ def forward(self, input_ids):+ return self.proj(self.embed_tokens(input_ids))+++class _TE(torch.nn.Module):+ def __init__(self):+ super().__init__()+ self.model = _TinyLM()+++def _stream_pinned(monkeypatch):+ pytest.importorskip("diffusers.hooks")+ import diffusers.hooks as hooks++ import core.inference.diffusion_memory as dm+ import core.inference.diffusion_prequant as prequant+ from core.inference import host_pinning++ real_apply = hooks.apply_group_offloading+ seen = {}++ def fake_apply(+ module,+ onload_device,+ offload_device = torch.device("cpu"),+ offload_type = "block_level",+ use_stream = False,+ non_blocking = False,+ record_stream = False,+ low_cpu_mem_usage = False,+ ):+ seen.update(low_cpu_mem_usage = low_cpu_mem_usage, use_stream = use_stream)+ real_apply(+ module,+ onload_device = torch.device("cpu"),+ offload_device = torch.device("cpu"),+ offload_type = "leaf_level",+ use_stream = False,+ )++ assert "low_cpu_mem_usage" in inspect.signature(fake_apply).parameters+ monkeypatch.setattr(hooks, "apply_group_offloading", fake_apply)+ monkeypatch.setattr(host_pinning, "host_pinning_allowed", lambda device = None: True)+ monkeypatch.setattr(prequant, "_unhook_from_manager", lambda *a, **k: True)+ monkeypatch.setattr(prequant, "_evict_rotation_hook", lambda *a, **k: (lambda *x, **y: None))+ monkeypatch.setattr(dm, "install_group_offload_hooks_eager", lambda: False)+ monkeypatch.setattr(dm, "_pin_vision_embedding_device", lambda target: None)+ arenas = []++ def arena(size, fill):+ buf = torch.zeros(size, dtype = torch.uint8)+ fill(buf)+ arenas.append(buf)+ return buf++ monkeypatch.setattr(te, "_pinned_arena", arena)+ reference = _TE()+ streamed = _TE()+ original_table = streamed.model.embed_tokens.weight.data_ptr()+ logger = logging.getLogger("h3_pin_trim_test")+ mode = te.stream_h3_text_encoder(object(), streamed, "cuda", pin = True, logger = logger)+ return reference, streamed, mode, seen, arenas, original_table+++def _in_arena(t, arenas):+ return t.untyped_storage().data_ptr() in {a.untyped_storage().data_ptr() for a in arenas}+++def test_the_vocabulary_table_stays_on_its_mapping_and_lookups_are_identical(monkeypatch, caplog):+ with caplog.at_level(logging.INFO, logger = "h3_pin_trim_test"):+ reference, streamed, mode, seen, arenas, original = _stream_pinned(monkeypatch)+ assert mode == "stream"+ table = streamed.model.embed_tokens.weight+ assert table.data_ptr() == original and not _in_arena(table, arenas)+ assert _in_arena(streamed.model.proj.weight, arenas)+ assert (+ sum(a.numel() for a in arenas) < table.numel() * table.element_size()+ ) # the table is not pinned+ assert (+ seen["low_cpu_mem_usage"] is True+ ) # diffusers keeps the pinned rest as is, makes no pinned twin+ ids = torch.tensor([[0, 3, 7, 511, 3]])+ with torch.no_grad():+ assert torch.equal(reference.model(ids), streamed.model(ids))+ assert any(m.startswith("video.h3_host_pin_trim: conditioner pins") for m in caplog.messages)+++def test_trim_kill_switch_pins_the_table_again(monkeypatch):+ _r, streamed, _m, _s, arenas, _o = _stream_pinned(monkeypatch)+ assert not _in_arena(streamed.model.embed_tokens.weight, arenas)+ monkeypatch.setenv("UNSLOTH_H3_HOST_PIN_TRIM", "0")+ reference, streamed, mode, seen, arenas, original = _stream_pinned(monkeypatch)+ assert mode == "stream"+ assert _in_arena(streamed.model.embed_tokens.weight, arenas)+ assert seen["low_cpu_mem_usage"] is False+ ids = torch.tensor([[1, 2, 3]])+ with torch.no_grad():+ assert torch.equal(reference.model(ids), streamed.model(ids))+++def test_a_table_left_in_the_stream_is_pinned_after_all(monkeypatch):+ """Host rows off (or refused): the table streams, so it must not stream from pageable memory."""+ monkeypatch.setattr(te, "keep_h3_embedding_on_host", lambda target, logger = None: False)+ reference, streamed, mode, seen, arenas, original = _stream_pinned(monkeypatch)+ table = streamed.model.embed_tokens.weight+ assert seen["low_cpu_mem_usage"] is True # left out of the up-front pin...+ assert _in_arena(table, arenas) # ...and pinned once it stayed in the stream+ from core.inference.diffusion_memory import _offload_groups++ hosts = [+ g.cpu_param_dict.get(table)+ for g in _offload_groups(streamed.model)+ if table in (getattr(g, "cpu_param_dict", None) or {})+ ]+ assert all(h is None or h.data_ptr() == table.data_ptr() for h in hosts)diff --git a/studio/backend/tests/test_h3_te_device_resident.py b/studio/backend/tests/test_h3_te_device_resident.pynew file mode 100644index 00000000000..8f1c8dd0918--- /dev/null+++ b/studio/backend/tests/test_h3_te_device_resident.py@@ -0,0 +1,339 @@+# SPDX-License-Identifier: AGPL-3.0-only+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0++"""CPU tests: MiniMax-H3 conditioner device share takes only leftover budget and yields before any block demotes."""++from __future__ import annotations++import logging+import random++import pytest++torch = pytest.importorskip("torch")+++class _Stream:+ def __init__(self):+ self.syncs = 0++ def synchronize(self):+ self.syncs += 1+++class _TeGroup:+ """A diffusers-style streamed leaf group: onload gives the weights new storage, offload re-points at the host copy."""++ def __init__(+ self,+ leaf,+ events = None,+ tag = None,+ ):+ self.modules = [leaf]+ self.parameters = []+ self.buffers = []+ self.stream = _Stream()+ self.onload_leader = leaf+ self.cpu_param_dict = {p: p.data for p in leaf.parameters()}+ self.where = "cpu"+ self.on = 0+ self.off = 0+ self.events = events+ self.tag = tag++ def onload_(self):+ self.on += 1+ for p, host in self.cpu_param_dict.items():+ p.data = host.clone()+ self.where = "cuda"+ if self.events is not None:+ self.events.append(("on", self.tag))++ def offload_(self):+ self.off += 1+ for p in self.modules[0].parameters():+ p.data = self.cpu_param_dict[p]+ self.where = "cpu"+ if self.events is not None:+ self.events.append(("off", self.tag))+++class _Block(_TeGroup):+ pass+++class _Model(torch.nn.Module):+ def __init__(+ self,+ n_layers = 3,+ width = 16,+ ):+ super().__init__()+ torch.manual_seed(0)+ self.visual = torch.nn.Sequential(torch.nn.Linear(width, width))+ self.language_model = torch.nn.Module()+ self.language_model.layers = torch.nn.ModuleList()+ for _ in range(n_layers):+ layer = torch.nn.Module()+ layer.q = torch.nn.Linear(width, width, bias = False)+ layer.mlp = torch.nn.Linear(width, 4 * width, bias = False)+ self.language_model.layers.append(layer)++ def forward(self, x):+ for layer in self.language_model.layers:+ x = x + layer.mlp(layer.q(x)).view(*x.shape[:-1], 4, -1).sum(-2)+ return x+++class _TE(torch.nn.Module):+ def __init__(self, **kw):+ super().__init__()+ self.model = _Model(**kw)+++def _share(+ monkeypatch,+ te_module,+ events = None,+ logger = None,+):+ import core.inference.diffusion_memory as dm+ from core.inference.video_minimax_h3_te import H3TeDeviceShare++ groups = [+ _TeGroup(m, events, name)+ for name, m in te_module.model.named_modules()+ if isinstance(m, torch.nn.Linear)+ ]+ monkeypatch.setattr(dm, "_offload_groups", lambda target: list(groups))+ return H3TeDeviceShare(te_module, "cuda", logger = logger), groups+++def _residency(+ block_bytes,+ events = None,+ top_bytes = 0,+):+ from core.inference.video_minimax_h3_residency import H3Residency, group_payload_bytes++ r = H3Residency.__new__(H3Residency)++ def block(nbytes, tag):+ return _Block(torch.nn.Linear(1, nbytes // 4, bias = False), events, tag)++ r.top = block(top_bytes, "top") if top_bytes else None+ r.blocks = [block(b, f"b{i}") for i, b in enumerate(block_bytes)]+ r.device = "cuda"+ r.logger = None+ r.top_bytes = group_payload_bytes(r.top) if r.top is not None else 0+ r.block_bytes = [group_payload_bytes(g) for g in r.blocks]+ r.max_blocks = 0+ r.te_share = None+ return r++[email protected](autouse = True)+def _cpu(monkeypatch):+ monkeypatch.setattr(torch.cuda, "is_available", lambda: False)+ monkeypatch.delenv("UNSLOTH_H3_TE_DEVICE_RESIDENT", raising = False)+ monkeypatch.delenv("UNSLOTH_H3_RESIDENT_UNPIN", raising = False)+ monkeypatch.delenv("UNSLOTH_H3_HOST_PIN_TRIM", raising = False)+++def test_share_reads_language_layers_in_order_and_leaves_the_vision_tower_streamed(monkeypatch):+ te_module = _TE()+ share, groups = _share(monkeypatch, te_module)+ assert [g.tag for g in share.groups] == [+ f"language_model.layers.{i}.{leaf}" for i in range(3) for leaf in ("q", "mlp")+ ]+ assert share.layer_of == [0, 0, 1, 1, 2, 2] and share.layers == 3+ assert all(g.tag != "visual.0" for g in share.groups)+++def test_budget_fits_every_block_then_a_conditioner_prefix(monkeypatch):+ te_module = _TE()+ share, _ = _share(monkeypatch, te_module)+ r = _residency([4096] * 4)+ r.te_share = share+ room = sum(share.planned[:3])+ r.fit(sum(r.block_bytes) + room, initial = True)+ assert r.resident_blocks() == 4+ assert share.resident_count() == 3+ assert share.resident_bytes() <= room+ assert r.device_bytes() == r.resident_bytes() + share.resident_bytes()+ assert [g.where for g in share.groups] == ["cuda"] * 3 + ["cpu"] * 3+++def test_nothing_for_the_conditioner_unless_every_block_is_resident(monkeypatch):+ """12 / 16 GB: the denoiser keeps a block prefix, and the leftover (smaller than a block) is not handed out."""+ te_module = _TE()+ share, _ = _share(monkeypatch, te_module)+ r = _residency([4096] * 4)+ r.te_share = share+ r.fit(3 * r.block_bytes[0] + r.block_bytes[0] - 1, initial = True)+ assert r.resident_blocks() == 3+ assert share.plan(r.block_bytes[0] - 1) > 0 # it would fit the leftover+ assert share.resident_count() == 0 and share.resident_bytes() == 0+ # a streamed top-level group, or one block short: nothing either+ r2 = _residency([4096] * 2, top_bytes = 8192)+ assert r2.te_budget(10**9, False, len(r2.blocks)) == 0+ assert r2.te_budget(10**9, True, len(r2.blocks) - 1) == 0+ assert r2.te_budget(10**9, True, len(r2.blocks)) == 10**9 - r2.top_bytes - sum(r2.block_bytes)+++def test_a_bigger_request_takes_the_share_back_before_demoting_a_block(monkeypatch):+ events: list = []+ te_module = _TE()+ share, groups = _share(monkeypatch, te_module, events)+ r = _residency([4096] * 4, events)+ r.te_share = share+ full = sum(r.block_bytes)+ r.fit(full + sum(share.planned), initial = True)+ assert share.resident_count() == len(share.groups) and r.resident_blocks() == 4+ events.clear()+ # exactly the denoiser's bytes: the share goes, every block stays+ r.fit(full)+ assert share.resident_count() == 0 and r.resident_blocks() == 4+ assert all(kind == "off" and not tag.startswith("b") for kind, tag in events), events+ events.clear()+ r.fit(full + sum(share.planned[:2]))+ assert share.resident_count() == 2+ events.clear()+ # one byte short: the share is given back first, then the last block+ r.fit(full - 1)+ offs = [tag for kind, tag in events if kind == "off"]+ assert offs[-1] == "b3" and all(not t.startswith("b") for t in offs[:-1]), offs+ assert share.resident_count() == 0 and r.resident_blocks() == 3+++def test_resident_groups_keep_the_side_stream_sync_and_skip_the_moves(monkeypatch):+ te_module = _TE()+ share, groups = _share(monkeypatch, te_module)+ share.grow(2)+ g = share.groups[0]+ assert g.where == "cuda" and g.on == 1+ syncs = g.stream.syncs+ g.onload_() # the group before it prefetching it: no move, but the stream is synced+ assert g.on == 1 and g.stream.syncs == syncs + 1+ g.offload_()+ assert g.off == 0 and g.where == "cuda"+ share.shrink(0)+ assert "onload_" not in g.__dict__ and "offload_" not in g.__dict__+ assert g.where == "cpu" and g.off == 1+ for p, host in g.cpu_param_dict.items():+ assert p.data.data_ptr() == host.data_ptr() # back on the host copy it never dropped+++def test_output_is_bit_identical_resident_or_streamed(monkeypatch):+ te_module = _TE()+ x = torch.randn(2, 16)+ with torch.no_grad():+ expect = te_module.model(x)+ share, _ = _share(monkeypatch, te_module)+ for n in (6, 3, 0, 5):+ share.shrink(n)+ share.grow(n)+ with torch.no_grad():+ assert torch.equal(te_module.model(x), expect)+++def test_the_budget_counts_the_share_as_demotable(monkeypatch):+ """The share lowers free VRAM, so the budget must add it back or every request would shrink it."""+ import core.inference.diffusion_memory as dm+ from core.inference import video as vid++ te_module = _TE()+ share, _ = _share(monkeypatch, te_module)+ r = _residency([4096] * 4)+ r.fit(10**9, initial = True)+ base_free = 30 * 10**9+ monkeypatch.setattr(dm, "cuda_reusable_cache_bytes", lambda device = None: 0)+ monkeypatch.setattr(vid, "_h3_signed_free_device_bytes", lambda device: base_free)+ kw = dict(width = 960, height = 544, frames = 124, te_streamed = True, te_scheme = "int8")+ without = vid._h3_residency_budget_bytes(r, "cuda", **kw)+ r.te_share = share+ share.grow(4)+ held = share.resident_bytes()+ assert held > 0+ monkeypatch.setattr(vid, "_h3_signed_free_device_bytes", lambda device: base_free - held)+ assert vid._h3_residency_budget_bytes(r, "cuda", **kw) == without+++def test_randomized_requests_keep_the_denoiser_and_the_device_peak_of_the_base(monkeypatch):+ """Simulated card: denoiser residency matches base at every request and never overcommits the card."""+ rng = random.Random(7)+ for trial in range(40):+ blocks = [rng.choice([4096, 8192]) for _ in range(rng.randint(1, 5))]+ te_module = _TE(n_layers = rng.randint(1, 4))+ share, _ = _share(monkeypatch, te_module)+ head, base = _residency(blocks), _residency(blocks)+ head.te_share = share+ card = sum(head.block_bytes) + rng.randint(0, sum(share.planned) * 2)+ for k in range(8):+ need = rng.randint(0, card // 3)+ outs = []+ for r in (base, head):+ used = r.device_bytes()+ free = card - used+ budget = free + r.device_bytes() - need+ r.fit(max(0, budget), initial = k == 0)+ peak = r.device_bytes() + need+ assert peak <= card, (trial, k, peak, card)+ outs.append(r.resident_blocks())+ assert outs[0] == outs[1], (trial, k)+++def test_kill_switch_and_modes(monkeypatch):+ from core.inference.video_minimax_h3_te import h3_te_device_share++ te_module = _TE()+ _share(monkeypatch, te_module)+ assert h3_te_device_share(te_module, "cuda", "stream") is not None+ assert h3_te_device_share(te_module, "cuda", "stream_lazy") is not None+ assert h3_te_device_share(te_module, "cuda", "sync") is None # integrated GPU: one memory+ assert h3_te_device_share(te_module, "cuda", None) is None+ monkeypatch.setenv("UNSLOTH_H3_TE_DEVICE_RESIDENT", "0")+ assert h3_te_device_share(te_module, "cuda", "stream") is None+++def test_engagement_log_line(monkeypatch, caplog):+ logger = logging.getLogger("h3_te_device_test")+ te_module = _TE()+ share, _ = _share(monkeypatch, te_module, logger = logger)+ r = _residency([4096] * 2)+ r.te_share = share+ with caplog.at_level(logging.INFO, logger = "h3_te_device_test"):+ r.fit(sum(r.block_bytes) + sum(share.planned[:4]), initial = True)+ lines = [m for m in caplog.messages if m.startswith("video.h3_te_resident_device:")]+ assert len(lines) == 1 and lines[0].startswith(+ "video.h3_te_resident_device: 2 of 3 TE layers resident on cuda"+ ), lines+++def test_a_failed_onload_leaves_the_conditioner_streamed(monkeypatch):+ te_module = _TE()+ share, groups = _share(monkeypatch, te_module)+ r = _residency([4096])+ r.te_share = share++ def boom():+ raise RuntimeError("CUDA out of memory")++ share.groups[2].onload_ = boom+ r.fit(10**9, initial = True)+ assert r.resident_blocks() == 1+ assert share.resident_count() == 0 and all(g.where == "cpu" for g in share.groups)+++def test_release_all_returns_the_share(monkeypatch):+ from core.inference.video_minimax_h3_residency import release_all++ te_module = _TE()+ share, _ = _share(monkeypatch, te_module)+ r = _residency([4096])+ r.te_share = share+ r.fit(10**9, initial = True)+ assert share.resident_count() == len(share.groups)+ release_all(r)+ assert share.resident_count() == 0 and all(g.where == "cpu" for g in share.groups)