Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
Studio H3: release resident denoiser blocks' host copies and give the RAM to the conditioner (#16639)
Daniel Han4d74b01a20d0
diff --git a/studio/backend/core/inference/video.py b/studio/backend/core/inference/video.pyindex e9a92ef312c..e14007b7a12 100644--- a/studio/backend/core/inference/video.py+++ b/studio/backend/core/inference/video.py@@ -11329,6 +11329,21 @@ class VideoBackend:try:residency = H3Residency(denoiser, device, logger = logger)if residency.usable:+ from .video_minimax_h3_te import yield_h3_te_resident++ _te_for_yield = getattr(pipe, "text_encoder", None)+ residency.host_yield = (+ lambda nbytes, _te = _te_for_yield: yield_h3_te_resident(+ _te, nbytes, logger = logger+ )+ )+ from .video_minimax_h3_te import top_up_h3_te_resident++ residency.host_top_up = (+ lambda _te = _te_for_yield: top_up_h3_te_resident(+ _te, logger = logger+ )+ )big_w, big_h = max(fam.resolution_presets, key = lambda wh: wh[0] * wh[1])diff --git a/studio/backend/core/inference/video_minimax_h3_residency.py b/studio/backend/core/inference/video_minimax_h3_residency.pyindex 2448ac22ea9..ee1e66f2364 100644--- a/studio/backend/core/inference/video_minimax_h3_residency.py+++ b/studio/backend/core/inference/video_minimax_h3_residency.py@@ -126,6 +126,186 @@ def is_resident(group: Any) -> bool:return bool(getattr(group, "_unsloth_resident", False))+# A resident group's host copy is only read again on demote, so it is released while the group is resident and+# rebuilt from the device copy (bit-exact) on demote: the H3 L4 load gives those ~11 GB to the conditioner instead.+H3_RESIDENT_UNPIN_ENV = "UNSLOTH_H3_RESIDENT_UNPIN"+_RELEASED_ATTR = "_unsloth_host_released"+# Floor (MiB) of the available host RAM a rebuild leaves; the load's own reserve (``encoder_resident_keep_mib``, what+# the conditioner share was sized to leave) applies when larger. Below it the host-yield callback gives RAM back first.+H3_REBUILD_KEEP_MIB = 2048+++def h3_resident_unpin_enabled() -> bool:+ return str(os.environ.get(H3_RESIDENT_UNPIN_ENV, "")).strip().lower() not in (+ "0",+ "off",+ "false",+ "no",+ )+++def _host_leaf_bytes(tensor: Any) -> int:+ return sum(int(leaf.numel()) * int(leaf.element_size()) for leaf in _host_leaves(tensor))+++def released_host_bytes(group: Any) -> int:+ return int(getattr(group, _RELEASED_ATTR, 0) or 0)+++def release_host_copy(group: Any, logger: Any = None) -> int:+ """Drop the host copy of a resident ``group`` (its weights live on the device); returns bytes released, 0 if kept.+ Kept when off, when a background pin still owns the group, or when any weight still reads its host copy."""+ if not h3_resident_unpin_enabled() or not is_resident(group) or released_host_bytes(group):+ return 0+ cpu = getattr(group, "cpu_param_dict", None)+ if not isinstance(cpu, dict) or not cpu:+ return 0+ pinner = getattr(group, "_unsloth_background_pin", None)+ wait = getattr(pinner, "wait", None)+ if callable(wait):+ try:+ wait(group)+ except Exception: # noqa: BLE001 -- unknown pin state: keep the copy+ return 0+ tensors = _group_tensors(group)+ if any(t not in cpu for t in tensors):+ return 0+ for t in tensors:+ host = cpu[t]+ if getattr(host, "device", None) is None or host.device.type != "cpu" or _reads(t, host):+ return 0 # a weight still points at its host copy: not on the device+ nbytes = sum(_host_leaf_bytes(cpu[t]) for t in tensors)+ group.cpu_param_dict = {}+ setattr(group, _RELEASED_ATTR, max(1, nbytes))+ return nbytes+++def _host_copy_of(tensor: Any) -> Any:+ """Host copy of a device ``tensor`` in the layout ``_pin_host_tensor`` gives (pinned where CUDA is up)."""+ import torch+ from torch.utils._python_dispatch import is_traceable_wrapper_subclass++ if is_traceable_wrapper_subclass(tensor):+ names, ctx = tensor.__tensor_flatten__()+ inner = {name: _host_copy_of(getattr(tensor, name)) for name in names}+ return type(tensor).__tensor_unflatten__(inner, ctx, tensor.size(), tensor.stride())+ data = tensor.detach()+ if type(data) is torch.nn.Parameter:+ data = data.data+ host = data.to("cpu", copy = True)+ if not torch.cuda.is_available():+ return host+ return _pin_host_tensor(host)+++def rebuild_host_copy(+ group: Any,+ *,+ host_yield: Any = None,+ logger: Any = None,+) -> int:+ """Re-create a released group's host copy from its device weights (an exact copy). ``host_yield(nbytes)`` is asked+ for RAM first when the copy would leave under ``H3_REBUILD_KEEP_MIB`` available. Returns bytes rebuilt."""+ need = released_host_bytes(group)+ if not need:+ return 0+ import torch++ if host_yield is not None:+ try:+ from .diffusion_small_host import encoder_resident_keep_mib, host_ram_mib+ total, available = host_ram_mib()+ if available is not None:+ keep = max(H3_REBUILD_KEEP_MIB, int(encoder_resident_keep_mib(total)))+ deficit = (need >> 20) + keep - int(available)+ if deficit > 0:+ host_yield(deficit << 20)+ except Exception as exc: # noqa: BLE001 -- rebuild anyway+ if logger is not None:+ logger.warning(+ "video.h3_residency: host RAM check before a rebuild failed: %s", exc+ )+ if torch.cuda.is_available():+ torch.cuda.current_stream().synchronize()+ host = {}+ with torch.no_grad():+ for t in _group_tensors(group):+ source = t if _is_wrapper(t) else t.data+ host[t] = _host_copy_of(source)+ group.cpu_param_dict = host+ setattr(group, _RELEASED_ATTR, 0)+ return need+++def _is_wrapper(tensor: Any) -> bool:+ from torch.utils._python_dispatch import is_traceable_wrapper_subclass+ return is_traceable_wrapper_subclass(tensor)+++# A slab still alive only for a few small weights of other groups (first-fit packs them into earlier slabs' tails)+# is compacted: those weights move to their own pinned copies so the slab is freed. Fraction of the slab still used.+H3_COMPACT_MAX_LIVE_FRACTION = 0.25+_H3_COMPACT_MIN_SLAB = 64 << 20+++def compact_host_slabs(groups: list, logger: Any = None) -> int:+ """Move the host weights that keep a mostly-released pinned slab alive into fresh pinned copies (exact copies,+ re-pointed like the arena's own pin), so the slab returns to the OS. Returns bytes of slabs freed (estimated)."""+ from .diffusion_pinned_arena import _repoint++ live: dict = {}+ sizes: dict = {}+ for group in groups:+ cpu = getattr(group, "cpu_param_dict", None) or {}+ for tensor, host in list(cpu.items()):+ for leaf in _host_leaves(host):+ if leaf.device.type != "cpu":+ continue+ storage = leaf.untyped_storage()+ key = storage.data_ptr()+ sizes[key] = int(storage.nbytes())+ live.setdefault(key, []).append(+ (group, tensor, host, int(leaf.numel()) * int(leaf.element_size()))+ )+ freed = moved = 0+ for key, entries in live.items():+ size = sizes[key]+ used = sum(e[3] for e in entries)+ if size < _H3_COMPACT_MIN_SLAB or used > H3_COMPACT_MAX_LIVE_FRACTION * size:+ continue+ for group, tensor, host, _n in entries:+ cpu = group.cpu_param_dict+ if cpu.get(tensor) is not host:+ continue # already moved through another leaf+ new = _pin_host_tensor(host)+ cpu[tensor] = new+ if _reads(tensor, host):+ _repoint(tensor, new)+ moved += _host_leaf_bytes(host)+ freed += size+ if freed:+ try:+ from .diffusion_offload_prefetch import bump_placement_epoch+ bump_placement_epoch()+ except Exception: # noqa: BLE001+ pass+ if logger is not None:+ logger.info(+ "video.h3_residency: compacted %.2f GB of pinned slabs (moved %.1f MB of other blocks' weights)",+ freed / 1e9,+ moved / 1e6,+ )+ return freed+++def _host_empty_cache() -> None:+ try:+ from .diffusion_memory import _host_empty_cache as empty+ empty()+ except Exception: # noqa: BLE001+ pass++def make_resident(group: Any) -> None:"""Onload ``group`` once and make its hooks' onload / offload no-ops, so it stays on the device."""import torch@@ -150,12 +330,30 @@ def make_resident(group: Any) -> None:group._unsloth_resident = True-def demote(group: Any) -> None:- """Undo ``make_resident``: restore the streaming onload / offload and move the weights back to the host copy."""+def demote(+ group: Any,+ *,+ host_yield: Any = None,+ logger: Any = None,+ rebuild: bool = True,+) -> None:+ """Undo ``make_resident``: restore the streaming onload / offload and move the weights back to the host copy.+ A released host copy is rebuilt from the device first; ``rebuild=False`` (unload) drops the weights instead."""import torchif not is_resident(group):return+ if released_host_bytes(group):+ if rebuild:+ nbytes = rebuild_host_copy(group, host_yield = host_yield, logger = logger)+ if logger is not None:+ logger.info(+ "video.h3_residency: rebuilt a demoted block's host copy from the device (%.2f GB)",+ nbytes / 1e9,+ )+ else:+ group._unsloth_resident = False+ returnsaved_on, saved_off = getattr(group, "_unsloth_saved_io", (None, None))for name, saved in (("onload_", saved_on), ("offload_", saved_off)):if saved is not None:@@ -184,6 +382,12 @@ class H3Residency:self.top_bytes = group_payload_bytes(self.top) if self.top is not None else 0self.block_bytes = [group_payload_bytes(g) for g in self.blocks]self.max_blocks = 0+ # host_yield(nbytes): frees host RAM (the conditioner's resident share) before a demoted block is rebuilt+ self.host_yield: Any = None+ # host_top_up(): hands RAM a re-fit released to the conditioner's resident share (after the load only)+ self.host_top_up: Any = None+ self.released_bytes = 0+ self.topped_bytes = 0@propertydef usable(self) -> bool:@@ -224,15 +428,26 @@ class H3Residency:import torchfor i in range(len(self.blocks) - 1, n - 1, -1):- demote(self.blocks[i])+ demote(self.blocks[i], host_yield = getattr(self, "host_yield", None), logger = self.logger)if self.top is not None and not top:- demote(self.top)+ demote(self.top, host_yield = getattr(self, "host_yield", None), logger = self.logger)if torch.cuda.is_available():torch.cuda.empty_cache()if self.top is not None and top:make_resident(self.top)for i in range(n):make_resident(self.blocks[i])+ # Block groups only: the top-level group keeps its own pinned copy (it is re-pinned by its own path).+ freed = sum(release_host_copy(self.blocks[i], logger = self.logger) for i in range(n))+ if freed:+ groups = list(self.blocks) + ([self.top] if self.top is not None else [])+ try:+ compact_host_slabs(groups, logger = self.logger)+ except Exception as exc: # noqa: BLE001 -- the released copies are still dropped+ if self.logger is not None:+ self.logger.warning("video.h3_residency: slab compaction skipped: %s", exc)+ _host_empty_cache()+ self.released_bytes = sum(released_host_bytes(g) for g in self.blocks)def fit(self,@@ -245,15 +460,27 @@ class H3Residency:self.apply(top, n)if initial:self.max_blocks = n+ released = getattr(self, "released_bytes", 0)+ top_up = getattr(self, "host_top_up", None)+ if released > getattr(self, "topped_bytes", 0) and callable(top_up):+ try:+ top_up()+ except Exception as exc: # noqa: BLE001 -- the conditioner stays mapped+ if self.logger is not None:+ self.logger.warning("video.h3_residency: conditioner top-up failed: %s", exc)+ self.topped_bytes = released+ elif released < getattr(self, "topped_bytes", 0):+ self.topped_bytes = releasedif self.logger is not None:self.logger.info("video.h3_residency: %d of %d denoiser blocks resident (+ top-level %s), %.2f GB on the device, "- "budget %.2f GB",+ "budget %.2f GB, %.2f GB of resident host copies released",n,len(self.blocks),"resident" if top else "streamed",self.resident_bytes() / 1e9,budget_bytes / 1e9,+ self.released_bytes / 1e9,)return top, n@@ -268,7 +495,7 @@ def release_all(residency: "H3Residency", *, logger: Any = None) -> None:for group in groups:try:if is_resident(group):- demote(group)+ demote(group, host_yield = getattr(residency, "host_yield", None), logger = logger)else:group.offload_()except Exception as exc: # noqa: BLE001diff --git a/studio/backend/core/inference/video_minimax_h3_te.py b/studio/backend/core/inference/video_minimax_h3_te.pyindex bcaa59f3ca7..e2633975630 100644--- a/studio/backend/core/inference/video_minimax_h3_te.py+++ b/studio/backend/core/inference/video_minimax_h3_te.py@@ -715,7 +715,9 @@ def keep_h3_te_resident(target = getattr(text_encoder, "model", None)if target is None or not h3_te_resident_enabled():return 0- entries = _streamed_host_tensors(target, groups)+ undo = _undo_list(target) # marks the share as managed: later calls top it up+ copied_ids = {id(e[3]) for e in undo}+ entries = [e for e in _streamed_host_tensors(target, groups) if id(e[2]) not in copied_ids]need = -(-sum(h.numel() * h.element_size() for _g, _t, h in entries) // (1 << 20))if need <= 0:return 0@@ -748,7 +750,7 @@ def keep_h3_te_resident(if nbytes <= budget:chosen.append(entry)budget -= nbytes- copied = _copy_host_tensors_(chosen)+ copied = _copy_host_tensors_(chosen, undo = undo)mib = copied >> 20if logger is not None:logger.info(@@ -764,7 +766,69 @@ def keep_h3_te_resident(return 0-def _copy_host_tensors_(entries: list) -> int:+_TE_RESIDENT_UNDO_ATTR = "_unsloth_te_resident_undo"+++def _undo_list(target: Any) -> list:+ undo = getattr(target, _TE_RESIDENT_UNDO_ATTR, None)+ if undo is None:+ undo = []+ try:+ setattr(target, _TE_RESIDENT_UNDO_ATTR, undo)+ except Exception: # noqa: BLE001 -- no undo: the share cannot yield+ return []+ return undo+++def top_up_h3_te_resident(text_encoder: Any, *, logger: Any = None) -> int:+ """``keep_h3_te_resident`` again after the load (a re-fit released resident denoiser host copies): copies the+ next share of the still-mapped conditioner. Only for a share the load already manages. Returns MiB copied."""+ target = getattr(text_encoder, "model", None)+ if target is None or getattr(target, _TE_RESIDENT_UNDO_ATTR, None) is None:+ return 0+ return keep_h3_te_resident(text_encoder, logger = logger)+++def yield_h3_te_resident(+ text_encoder: Any,+ nbytes: int,+ *,+ logger: Any = None,+) -> int:+ """Return up to ``nbytes`` of the conditioner's resident share to its memory mapping (last copied first: vision,+ then the deepest language layers), so the anonymous copies are freed. Returns bytes given back. Never raises."""+ try:+ target = getattr(text_encoder, "model", None)+ undo = getattr(target, _TE_RESIDENT_UNDO_ATTR, None) if target is not None else None+ if not undo or nbytes <= 0:+ return 0+ import torch++ freed = 0+ with torch.no_grad():+ while undo and freed < nbytes:+ group, tensor, mapped, copy = undo.pop()+ cpu = getattr(group, "cpu_param_dict", None)+ if cpu is not None and cpu.get(tensor) is copy:+ cpu[tensor] = mapped+ if tensor.device.type == "cpu" and tensor.data_ptr() == copy.data_ptr():+ tensor.data = mapped+ freed += copy.numel() * copy.element_size()+ del copy+ if logger is not None:+ logger.info(+ "video.h3_te_resident: %d MiB of the conditioner's resident share returned to its mapping "+ "(host RAM for a demoted denoiser block)",+ freed >> 20,+ )+ return freed+ except Exception as exc: # noqa: BLE001+ if logger is not None:+ logger.warning("video.h3_te_resident: could not yield the resident share: %s", exc)+ return 0+++def _copy_host_tensors_(entries: list, undo: Optional[list] = None) -> int:import torchfrom concurrent.futures import ThreadPoolExecutor@@ -783,5 +847,7 @@ def _copy_host_tensors_(entries: list) -> int:cpu[tensor] = newif tensor.device.type == "cpu" and tensor.data_ptr() == host.data_ptr():tensor.data = new+ if undo is not None:+ undo.append((group, tensor, host, new))copied += new.numel() * new.element_size()return copieddiff --git a/studio/backend/tests/test_h3_resident_unpin.py b/studio/backend/tests/test_h3_resident_unpin.pynew file mode 100644index 00000000000..7e084316a72--- /dev/null+++ b/studio/backend/tests/test_h3_resident_unpin.py@@ -0,0 +1,276 @@+# SPDX-License-Identifier: AGPL-3.0-only+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0++"""MiniMax-H3 streamed denoiser: a block resident on the card drops its host copy (the conditioner's resident share+takes that RAM), and a demoted block gets an exact host copy back from the device before it streams again."""++from __future__ import annotations++import logging++import pytest++torch = pytest.importorskip("torch")+++class _HostGroup:+ """A diffusers-style offload group: ``cpu_param_dict`` holds the host copy, onload gives the weight new storage."""++ def __init__(+ self,+ seed: int,+ n: int = 64,+ ):+ g = torch.Generator().manual_seed(seed)+ lin = torch.nn.Linear(n, n, bias = True)+ with torch.no_grad():+ lin.weight.copy_(torch.randn(n, n, generator = g))+ lin.bias.copy_(torch.randn(n, generator = g))+ self.modules = [lin]+ self.parameters = []+ self.buffers = []+ self.stream = None+ self.cpu_param_dict = {p: p.data for p in lin.parameters()}+ self.where = "cpu"++ def onload_(self):+ for p, host in self.cpu_param_dict.items():+ p.data = host.clone() # the "device" copy: distinct storage+ self.where = "cuda"++ def offload_(self):+ for p in list(self.modules[0].parameters()):+ p.data = self.cpu_param_dict[p]+ self.where = "cpu"+++def _residency(n_blocks: int, logger = None):+ from core.inference.video_minimax_h3_residency import H3Residency, group_payload_bytes++ r = H3Residency.__new__(H3Residency)+ r.top = None+ r.blocks = [_HostGroup(i) for i in range(n_blocks)]+ r.device = "cuda"+ r.logger = logger+ r.top_bytes = 0+ r.block_bytes = [group_payload_bytes(g) for g in r.blocks]+ r.max_blocks = 0+ r.host_yield = None+ r.released_bytes = 0+ return r++[email protected](autouse = True)+def _no_cuda(monkeypatch):+ monkeypatch.setattr(torch.cuda, "is_available", lambda: False)+ monkeypatch.delenv("UNSLOTH_H3_RESIDENT_UNPIN", raising = False)+++def test_a_resident_block_drops_its_host_copy():+ from core.inference import video_minimax_h3_residency as res++ r = _residency(4)+ r.fit(r.block_bytes[0] * 2, initial = True)+ assert r.resident_blocks() == 2+ for g in r.blocks[:2]:+ assert g.cpu_param_dict == {}+ assert res.released_host_bytes(g) == res.group_payload_bytes(g)+ for g in r.blocks[2:]:+ assert len(g.cpu_param_dict) == 2 and res.released_host_bytes(g) == 0+ assert r.released_bytes == sum(r.block_bytes[:2])+++def test_a_demoted_block_streams_from_an_exact_rebuilt_copy(caplog):+ r = _residency(3, logger = logging.getLogger("h3test"))+ before = [[t.clone() for t in g.modules[0].parameters()] for g in r.blocks]+ r.fit(sum(r.block_bytes), initial = True)+ assert all(g.cpu_param_dict == {} for g in r.blocks)+ with caplog.at_level(logging.INFO, logger = "h3test"):+ r.fit(r.block_bytes[0]) # a bigger request: blocks 1 and 2 go back to streaming+ assert r.resident_blocks() == 1+ assert "rebuilt a demoted block's host copy from the device" in caplog.text+ for g, ref in zip(r.blocks[1:], before[1:]):+ assert g.where == "cpu" and len(g.cpu_param_dict) == 2+ for p, want in zip(g.modules[0].parameters(), ref):+ assert torch.equal(p.data, want)+ assert (+ p.data.data_ptr() == g.cpu_param_dict[p].data_ptr()+ ) # offload re-pointed at the rebuilt copy+ g.onload_() # streams again from the rebuilt copy+ for p, want in zip(g.modules[0].parameters(), ref):+ assert torch.equal(p.data, want)+ r.fit(sum(r.block_bytes)) # promoted again: released again+ assert all(g.cpu_param_dict == {} for g in r.blocks)+++def test_the_unpin_kill_switch_keeps_the_host_copy(monkeypatch):+ monkeypatch.setenv("UNSLOTH_H3_RESIDENT_UNPIN", "0")+ r = _residency(2)+ r.fit(sum(r.block_bytes), initial = True)+ assert r.resident_blocks() == 2+ assert all(len(g.cpu_param_dict) == 2 for g in r.blocks) and r.released_bytes == 0+++def test_a_block_still_reading_its_host_copy_is_not_released():+ from core.inference import video_minimax_h3_residency as res++ g = _HostGroup(0)+ res.make_resident(g)+ g.offload_ = res._noop+ for p, host in g.cpu_param_dict.items():+ p.data = host # onload left it on the host+ assert res.release_host_copy(g) == 0 and len(g.cpu_param_dict) == 2+++def test_a_rebuild_asks_the_conditioner_share_for_ram_when_the_host_is_tight(monkeypatch):+ from core.inference import diffusion_small_host++ asked = []+ monkeypatch.setattr(diffusion_small_host, "host_ram_mib", lambda: (16384, 100))+ r = _residency(2)+ r.host_yield = lambda nbytes: asked.append(nbytes) or nbytes+ r.fit(sum(r.block_bytes), initial = True)+ r.fit(0)+ assert len(asked) == 2 and all(n >= (2048 - 100) << 20 for n in asked)+ assert all(len(g.cpu_param_dict) == 2 for g in r.blocks)+++def test_release_all_rebuilds_every_released_block():+ from core.inference import video_minimax_h3_residency as res++ r = _residency(3)+ ref = [[t.clone() for t in g.modules[0].parameters()] for g in r.blocks]+ r.fit(sum(r.block_bytes), initial = True)+ res.release_all(r)+ assert r.resident_bytes() == 0+ for g, want in zip(r.blocks, ref):+ assert all(torch.equal(p.data, w) for p, w in zip(g.modules[0].parameters(), want))+++def test_the_conditioner_share_yields_back_to_its_mapping():+ from core.inference import video_minimax_h3_te as te++ class _TE:+ pass++ lin = torch.nn.Linear(32, 32)+ mapped = {p: p.data for p in lin.parameters()}+ group = type("G", (), {})()+ group.cpu_param_dict = dict(mapped)+ holder = _TE()+ holder.model = torch.nn.Module()+ entries = [(group, p, h) for p, h in mapped.items()]+ copied = te._copy_host_tensors_(entries, undo = te._undo_list(holder.model))+ assert copied == sum(h.numel() * h.element_size() for h in mapped.values())+ assert all(group.cpu_param_dict[p] is not mapped[p] for p in mapped)+ freed = te.yield_h3_te_resident(holder, 1)+ assert freed > 0+ weight_or_bias = [p for p in mapped if group.cpu_param_dict[p] is mapped[p]]+ assert len(weight_or_bias) == 1 # last copied first, only as much as asked+ te.yield_h3_te_resident(holder, 1 << 30)+ assert all(group.cpu_param_dict[p] is mapped[p] for p in mapped)+ assert all(p.data.data_ptr() == mapped[p].data_ptr() for p in mapped)+++def test_a_refit_that_releases_more_hands_it_to_the_conditioner_once():+ r = _residency(4)+ r.topped_bytes = 0+ calls = []+ r.host_top_up = lambda: calls.append(r.released_bytes)+ r.fit(r.block_bytes[0], initial = True) # the load's fit (a larger preset): one block+ r.fit(sum(r.block_bytes)) # the request's fit: every block, more released+ r.fit(sum(r.block_bytes)) # same request again: nothing new+ assert calls == [r.block_bytes[0], sum(r.block_bytes)]+ r.fit(r.block_bytes[0]) # demote: less released, no top-up+ assert len(calls) == 2 and r.topped_bytes == r.block_bytes[0]+++def test_the_conditioner_top_up_copies_only_what_is_still_mapped(monkeypatch):+ from core.inference import diffusion_small_host+ from core.inference import video_minimax_h3_te as te++ lin = torch.nn.Linear(32, 32)+ group = type("G", (), {})()+ group.modules = [lin]+ group.parameters = []+ group.buffers = []+ group.cpu_param_dict = {p: p.data for p in lin.parameters()}++ class _TE:+ pass++ holder = _TE()+ holder.model = torch.nn.Module()+ monkeypatch.setattr(diffusion_small_host, "host_ram_mib", lambda: (65536, 65536))+ monkeypatch.setattr(diffusion_small_host, "host_commit_mib", lambda: None)+ monkeypatch.setattr(te, "_H3_TE_RESIDENT_MIN_MIB", 0)+ assert te.top_up_h3_te_resident(holder) == 0 # not managed by a load yet+ te.keep_h3_te_resident(holder, groups = [group])+ undo = getattr(holder.model, te._TE_RESIDENT_UNDO_ATTR)+ assert len(undo) == 2+ te.keep_h3_te_resident(holder, groups = [group]) # a top-up: nothing still mapped+ assert len(undo) == 2+++def test_a_mostly_released_slab_is_compacted_so_it_can_be_freed(monkeypatch):+ """First-fit packs a later block's small weights into an earlier slab's tail: once the resident blocks drop their+ copies, those few weights alone would keep the whole slab pinned. They move to their own copies (exact)."""+ from core.inference import video_minimax_h3_residency as res++ monkeypatch.setattr(res, "_H3_COMPACT_MIN_SLAB", 0)+ monkeypatch.setattr(res, "_pin_host_tensor", lambda src: src.clone())+ slab = torch.arange(4096, dtype = torch.float32)+ big = _HostGroup(0, n = 60) # 60*60 + 60 = 3660 floats: most of the slab+ small = _HostGroup(1, n = 4) # 20 floats in the slab's tail+ off = 0+ for g in (big, small):+ for p in g.modules[0].parameters():+ view = slab[off : off + p.numel()].view(p.shape)+ view.copy_(p.data)+ g.cpu_param_dict[p] = view+ p.data = view+ off += p.numel()+ want = [p.data.clone() for p in small.modules[0].parameters()]+ r = _residency(0)+ r.blocks = [big, small]+ r.block_bytes = [res.group_payload_bytes(g) for g in r.blocks]+ r.fit(r.block_bytes[0], initial = True)+ assert r.resident_blocks() == 1 and big.cpu_param_dict == {}+ for p, w in zip(small.modules[0].parameters(), want):+ host = small.cpu_param_dict[p]+ assert host.untyped_storage().data_ptr() != slab.untyped_storage().data_ptr()+ assert torch.equal(host, w)+ assert p.data.data_ptr() == host.data_ptr() # the streamed weight reads its new copy+++def test_a_well_used_slab_is_left_alone(monkeypatch):+ from core.inference import video_minimax_h3_residency as res++ monkeypatch.setattr(res, "_H3_COMPACT_MIN_SLAB", 0)+ slab = torch.zeros(30) # 20 of 30 floats still used+ g = _HostGroup(0, n = 4)+ off = 0+ for p in g.modules[0].parameters():+ g.cpu_param_dict[p] = slab[off : off + p.numel()].view(p.shape)+ off += p.numel()+ assert res.compact_host_slabs([g]) == 0+ assert all(+ h.untyped_storage().data_ptr() == slab.untyped_storage().data_ptr()+ for h in g.cpu_param_dict.values()+ )+++def test_a_rebuild_restores_the_load_reserve_not_just_the_floor(monkeypatch):+ """A demote after the conditioner share was topped up must give back enough for the load's own reserve (an L4+ renderer was OOM-killed when only a 2 GiB floor was restored)."""+ from core.inference import diffusion_small_host++ monkeypatch.setattr(diffusion_small_host, "host_ram_mib", lambda: (53000, 5000))+ monkeypatch.setattr(diffusion_small_host, "encoder_resident_keep_mib", lambda total: 7982)+ asked = []+ r = _residency(1)+ r.host_yield = lambda nbytes: asked.append(nbytes) or nbytes+ r.fit(sum(r.block_bytes), initial = True)+ r.fit(0)+ need_mib = r.block_bytes[0] >> 20+ assert asked == [(need_mib + 7982 - 5000) << 20] # the 2 GiB floor alone would ask for nothing