Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

Studio H3: release resident denoiser blocks' host copies and give the RAM to the conditioner (#16639)

Daniel Han4d74b01a20d0
diff --git a/studio/backend/core/inference/video.py b/studio/backend/core/inference/video.py
index e9a92ef312c..e14007b7a12 100644
--- a/studio/backend/core/inference/video.py
+++ b/studio/backend/core/inference/video.py
@@ -11329,6 +11329,21 @@ class VideoBackend:
try:
residency = H3Residency(denoiser, device, logger = logger)
if residency.usable:
+ from .video_minimax_h3_te import yield_h3_te_resident
+
+ _te_for_yield = getattr(pipe, "text_encoder", None)
+ residency.host_yield = (
+ lambda nbytes, _te = _te_for_yield: yield_h3_te_resident(
+ _te, nbytes, logger = logger
+ )
+ )
+ from .video_minimax_h3_te import top_up_h3_te_resident
+
+ residency.host_top_up = (
+ lambda _te = _te_for_yield: top_up_h3_te_resident(
+ _te, logger = logger
+ )
+ )
big_w, big_h = max(
fam.resolution_presets, key = lambda wh: wh[0] * wh[1]
)
diff --git a/studio/backend/core/inference/video_minimax_h3_residency.py b/studio/backend/core/inference/video_minimax_h3_residency.py
index 2448ac22ea9..ee1e66f2364 100644
--- a/studio/backend/core/inference/video_minimax_h3_residency.py
+++ b/studio/backend/core/inference/video_minimax_h3_residency.py
@@ -126,6 +126,186 @@ def is_resident(group: Any) -> bool:
return bool(getattr(group, "_unsloth_resident", False))
+# A resident group's host copy is only read again on demote, so it is released while the group is resident and
+# rebuilt from the device copy (bit-exact) on demote: the H3 L4 load gives those ~11 GB to the conditioner instead.
+H3_RESIDENT_UNPIN_ENV = "UNSLOTH_H3_RESIDENT_UNPIN"
+_RELEASED_ATTR = "_unsloth_host_released"
+# Floor (MiB) of the available host RAM a rebuild leaves; the load's own reserve (``encoder_resident_keep_mib``, what
+# the conditioner share was sized to leave) applies when larger. Below it the host-yield callback gives RAM back first.
+H3_REBUILD_KEEP_MIB = 2048
+
+
+def h3_resident_unpin_enabled() -> bool:
+ return str(os.environ.get(H3_RESIDENT_UNPIN_ENV, "")).strip().lower() not in (
+ "0",
+ "off",
+ "false",
+ "no",
+ )
+
+
+def _host_leaf_bytes(tensor: Any) -> int:
+ return sum(int(leaf.numel()) * int(leaf.element_size()) for leaf in _host_leaves(tensor))
+
+
+def released_host_bytes(group: Any) -> int:
+ return int(getattr(group, _RELEASED_ATTR, 0) or 0)
+
+
+def release_host_copy(group: Any, logger: Any = None) -> int:
+ """Drop the host copy of a resident ``group`` (its weights live on the device); returns bytes released, 0 if kept.
+ Kept when off, when a background pin still owns the group, or when any weight still reads its host copy."""
+ if not h3_resident_unpin_enabled() or not is_resident(group) or released_host_bytes(group):
+ return 0
+ cpu = getattr(group, "cpu_param_dict", None)
+ if not isinstance(cpu, dict) or not cpu:
+ return 0
+ pinner = getattr(group, "_unsloth_background_pin", None)
+ wait = getattr(pinner, "wait", None)
+ if callable(wait):
+ try:
+ wait(group)
+ except Exception: # noqa: BLE001 -- unknown pin state: keep the copy
+ return 0
+ tensors = _group_tensors(group)
+ if any(t not in cpu for t in tensors):
+ return 0
+ for t in tensors:
+ host = cpu[t]
+ if getattr(host, "device", None) is None or host.device.type != "cpu" or _reads(t, host):
+ return 0 # a weight still points at its host copy: not on the device
+ nbytes = sum(_host_leaf_bytes(cpu[t]) for t in tensors)
+ group.cpu_param_dict = {}
+ setattr(group, _RELEASED_ATTR, max(1, nbytes))
+ return nbytes
+
+
+def _host_copy_of(tensor: Any) -> Any:
+ """Host copy of a device ``tensor`` in the layout ``_pin_host_tensor`` gives (pinned where CUDA is up)."""
+ import torch
+ from torch.utils._python_dispatch import is_traceable_wrapper_subclass
+
+ if is_traceable_wrapper_subclass(tensor):
+ names, ctx = tensor.__tensor_flatten__()
+ inner = {name: _host_copy_of(getattr(tensor, name)) for name in names}
+ return type(tensor).__tensor_unflatten__(inner, ctx, tensor.size(), tensor.stride())
+ data = tensor.detach()
+ if type(data) is torch.nn.Parameter:
+ data = data.data
+ host = data.to("cpu", copy = True)
+ if not torch.cuda.is_available():
+ return host
+ return _pin_host_tensor(host)
+
+
+def rebuild_host_copy(
+ group: Any,
+ *,
+ host_yield: Any = None,
+ logger: Any = None,
+) -> int:
+ """Re-create a released group's host copy from its device weights (an exact copy). ``host_yield(nbytes)`` is asked
+ for RAM first when the copy would leave under ``H3_REBUILD_KEEP_MIB`` available. Returns bytes rebuilt."""
+ need = released_host_bytes(group)
+ if not need:
+ return 0
+ import torch
+
+ if host_yield is not None:
+ try:
+ from .diffusion_small_host import encoder_resident_keep_mib, host_ram_mib
+ total, available = host_ram_mib()
+ if available is not None:
+ keep = max(H3_REBUILD_KEEP_MIB, int(encoder_resident_keep_mib(total)))
+ deficit = (need >> 20) + keep - int(available)
+ if deficit > 0:
+ host_yield(deficit << 20)
+ except Exception as exc: # noqa: BLE001 -- rebuild anyway
+ if logger is not None:
+ logger.warning(
+ "video.h3_residency: host RAM check before a rebuild failed: %s", exc
+ )
+ if torch.cuda.is_available():
+ torch.cuda.current_stream().synchronize()
+ host = {}
+ with torch.no_grad():
+ for t in _group_tensors(group):
+ source = t if _is_wrapper(t) else t.data
+ host[t] = _host_copy_of(source)
+ group.cpu_param_dict = host
+ setattr(group, _RELEASED_ATTR, 0)
+ return need
+
+
+def _is_wrapper(tensor: Any) -> bool:
+ from torch.utils._python_dispatch import is_traceable_wrapper_subclass
+ return is_traceable_wrapper_subclass(tensor)
+
+
+# A slab still alive only for a few small weights of other groups (first-fit packs them into earlier slabs' tails)
+# is compacted: those weights move to their own pinned copies so the slab is freed. Fraction of the slab still used.
+H3_COMPACT_MAX_LIVE_FRACTION = 0.25
+_H3_COMPACT_MIN_SLAB = 64 << 20
+
+
+def compact_host_slabs(groups: list, logger: Any = None) -> int:
+ """Move the host weights that keep a mostly-released pinned slab alive into fresh pinned copies (exact copies,
+ re-pointed like the arena's own pin), so the slab returns to the OS. Returns bytes of slabs freed (estimated)."""
+ from .diffusion_pinned_arena import _repoint
+
+ live: dict = {}
+ sizes: dict = {}
+ for group in groups:
+ cpu = getattr(group, "cpu_param_dict", None) or {}
+ for tensor, host in list(cpu.items()):
+ for leaf in _host_leaves(host):
+ if leaf.device.type != "cpu":
+ continue
+ storage = leaf.untyped_storage()
+ key = storage.data_ptr()
+ sizes[key] = int(storage.nbytes())
+ live.setdefault(key, []).append(
+ (group, tensor, host, int(leaf.numel()) * int(leaf.element_size()))
+ )
+ freed = moved = 0
+ for key, entries in live.items():
+ size = sizes[key]
+ used = sum(e[3] for e in entries)
+ if size < _H3_COMPACT_MIN_SLAB or used > H3_COMPACT_MAX_LIVE_FRACTION * size:
+ continue
+ for group, tensor, host, _n in entries:
+ cpu = group.cpu_param_dict
+ if cpu.get(tensor) is not host:
+ continue # already moved through another leaf
+ new = _pin_host_tensor(host)
+ cpu[tensor] = new
+ if _reads(tensor, host):
+ _repoint(tensor, new)
+ moved += _host_leaf_bytes(host)
+ freed += size
+ if freed:
+ try:
+ from .diffusion_offload_prefetch import bump_placement_epoch
+ bump_placement_epoch()
+ except Exception: # noqa: BLE001
+ pass
+ if logger is not None:
+ logger.info(
+ "video.h3_residency: compacted %.2f GB of pinned slabs (moved %.1f MB of other blocks' weights)",
+ freed / 1e9,
+ moved / 1e6,
+ )
+ return freed
+
+
+def _host_empty_cache() -> None:
+ try:
+ from .diffusion_memory import _host_empty_cache as empty
+ empty()
+ except Exception: # noqa: BLE001
+ pass
+
+
def make_resident(group: Any) -> None:
"""Onload ``group`` once and make its hooks' onload / offload no-ops, so it stays on the device."""
import torch
@@ -150,12 +330,30 @@ def make_resident(group: Any) -> None:
group._unsloth_resident = True
-def demote(group: Any) -> None:
- """Undo ``make_resident``: restore the streaming onload / offload and move the weights back to the host copy."""
+def demote(
+ group: Any,
+ *,
+ host_yield: Any = None,
+ logger: Any = None,
+ rebuild: bool = True,
+) -> None:
+ """Undo ``make_resident``: restore the streaming onload / offload and move the weights back to the host copy.
+ A released host copy is rebuilt from the device first; ``rebuild=False`` (unload) drops the weights instead."""
import torch
if not is_resident(group):
return
+ if released_host_bytes(group):
+ if rebuild:
+ nbytes = rebuild_host_copy(group, host_yield = host_yield, logger = logger)
+ if logger is not None:
+ logger.info(
+ "video.h3_residency: rebuilt a demoted block's host copy from the device (%.2f GB)",
+ nbytes / 1e9,
+ )
+ else:
+ group._unsloth_resident = False
+ return
saved_on, saved_off = getattr(group, "_unsloth_saved_io", (None, None))
for name, saved in (("onload_", saved_on), ("offload_", saved_off)):
if saved is not None:
@@ -184,6 +382,12 @@ class H3Residency:
self.top_bytes = group_payload_bytes(self.top) if self.top is not None else 0
self.block_bytes = [group_payload_bytes(g) for g in self.blocks]
self.max_blocks = 0
+ # host_yield(nbytes): frees host RAM (the conditioner's resident share) before a demoted block is rebuilt
+ self.host_yield: Any = None
+ # host_top_up(): hands RAM a re-fit released to the conditioner's resident share (after the load only)
+ self.host_top_up: Any = None
+ self.released_bytes = 0
+ self.topped_bytes = 0
@property
def usable(self) -> bool:
@@ -224,15 +428,26 @@ class H3Residency:
import torch
for i in range(len(self.blocks) - 1, n - 1, -1):
- demote(self.blocks[i])
+ demote(self.blocks[i], host_yield = getattr(self, "host_yield", None), logger = self.logger)
if self.top is not None and not top:
- demote(self.top)
+ demote(self.top, host_yield = getattr(self, "host_yield", None), logger = self.logger)
if torch.cuda.is_available():
torch.cuda.empty_cache()
if self.top is not None and top:
make_resident(self.top)
for i in range(n):
make_resident(self.blocks[i])
+ # Block groups only: the top-level group keeps its own pinned copy (it is re-pinned by its own path).
+ freed = sum(release_host_copy(self.blocks[i], logger = self.logger) for i in range(n))
+ if freed:
+ groups = list(self.blocks) + ([self.top] if self.top is not None else [])
+ try:
+ compact_host_slabs(groups, logger = self.logger)
+ except Exception as exc: # noqa: BLE001 -- the released copies are still dropped
+ if self.logger is not None:
+ self.logger.warning("video.h3_residency: slab compaction skipped: %s", exc)
+ _host_empty_cache()
+ self.released_bytes = sum(released_host_bytes(g) for g in self.blocks)
def fit(
self,
@@ -245,15 +460,27 @@ class H3Residency:
self.apply(top, n)
if initial:
self.max_blocks = n
+ released = getattr(self, "released_bytes", 0)
+ top_up = getattr(self, "host_top_up", None)
+ if released > getattr(self, "topped_bytes", 0) and callable(top_up):
+ try:
+ top_up()
+ except Exception as exc: # noqa: BLE001 -- the conditioner stays mapped
+ if self.logger is not None:
+ self.logger.warning("video.h3_residency: conditioner top-up failed: %s", exc)
+ self.topped_bytes = released
+ elif released < getattr(self, "topped_bytes", 0):
+ self.topped_bytes = released
if self.logger is not None:
self.logger.info(
"video.h3_residency: %d of %d denoiser blocks resident (+ top-level %s), %.2f GB on the device, "
- "budget %.2f GB",
+ "budget %.2f GB, %.2f GB of resident host copies released",
n,
len(self.blocks),
"resident" if top else "streamed",
self.resident_bytes() / 1e9,
budget_bytes / 1e9,
+ self.released_bytes / 1e9,
)
return top, n
@@ -268,7 +495,7 @@ def release_all(residency: "H3Residency", *, logger: Any = None) -> None:
for group in groups:
try:
if is_resident(group):
- demote(group)
+ demote(group, host_yield = getattr(residency, "host_yield", None), logger = logger)
else:
group.offload_()
except Exception as exc: # noqa: BLE001
diff --git a/studio/backend/core/inference/video_minimax_h3_te.py b/studio/backend/core/inference/video_minimax_h3_te.py
index bcaa59f3ca7..e2633975630 100644
--- a/studio/backend/core/inference/video_minimax_h3_te.py
+++ b/studio/backend/core/inference/video_minimax_h3_te.py
@@ -715,7 +715,9 @@ def keep_h3_te_resident(
target = getattr(text_encoder, "model", None)
if target is None or not h3_te_resident_enabled():
return 0
- entries = _streamed_host_tensors(target, groups)
+ undo = _undo_list(target) # marks the share as managed: later calls top it up
+ copied_ids = {id(e[3]) for e in undo}
+ entries = [e for e in _streamed_host_tensors(target, groups) if id(e[2]) not in copied_ids]
need = -(-sum(h.numel() * h.element_size() for _g, _t, h in entries) // (1 << 20))
if need <= 0:
return 0
@@ -748,7 +750,7 @@ def keep_h3_te_resident(
if nbytes <= budget:
chosen.append(entry)
budget -= nbytes
- copied = _copy_host_tensors_(chosen)
+ copied = _copy_host_tensors_(chosen, undo = undo)
mib = copied >> 20
if logger is not None:
logger.info(
@@ -764,7 +766,69 @@ def keep_h3_te_resident(
return 0
-def _copy_host_tensors_(entries: list) -> int:
+_TE_RESIDENT_UNDO_ATTR = "_unsloth_te_resident_undo"
+
+
+def _undo_list(target: Any) -> list:
+ undo = getattr(target, _TE_RESIDENT_UNDO_ATTR, None)
+ if undo is None:
+ undo = []
+ try:
+ setattr(target, _TE_RESIDENT_UNDO_ATTR, undo)
+ except Exception: # noqa: BLE001 -- no undo: the share cannot yield
+ return []
+ return undo
+
+
+def top_up_h3_te_resident(text_encoder: Any, *, logger: Any = None) -> int:
+ """``keep_h3_te_resident`` again after the load (a re-fit released resident denoiser host copies): copies the
+ next share of the still-mapped conditioner. Only for a share the load already manages. Returns MiB copied."""
+ target = getattr(text_encoder, "model", None)
+ if target is None or getattr(target, _TE_RESIDENT_UNDO_ATTR, None) is None:
+ return 0
+ return keep_h3_te_resident(text_encoder, logger = logger)
+
+
+def yield_h3_te_resident(
+ text_encoder: Any,
+ nbytes: int,
+ *,
+ logger: Any = None,
+) -> int:
+ """Return up to ``nbytes`` of the conditioner's resident share to its memory mapping (last copied first: vision,
+ then the deepest language layers), so the anonymous copies are freed. Returns bytes given back. Never raises."""
+ try:
+ target = getattr(text_encoder, "model", None)
+ undo = getattr(target, _TE_RESIDENT_UNDO_ATTR, None) if target is not None else None
+ if not undo or nbytes <= 0:
+ return 0
+ import torch
+
+ freed = 0
+ with torch.no_grad():
+ while undo and freed < nbytes:
+ group, tensor, mapped, copy = undo.pop()
+ cpu = getattr(group, "cpu_param_dict", None)
+ if cpu is not None and cpu.get(tensor) is copy:
+ cpu[tensor] = mapped
+ if tensor.device.type == "cpu" and tensor.data_ptr() == copy.data_ptr():
+ tensor.data = mapped
+ freed += copy.numel() * copy.element_size()
+ del copy
+ if logger is not None:
+ logger.info(
+ "video.h3_te_resident: %d MiB of the conditioner's resident share returned to its mapping "
+ "(host RAM for a demoted denoiser block)",
+ freed >> 20,
+ )
+ return freed
+ except Exception as exc: # noqa: BLE001
+ if logger is not None:
+ logger.warning("video.h3_te_resident: could not yield the resident share: %s", exc)
+ return 0
+
+
+def _copy_host_tensors_(entries: list, undo: Optional[list] = None) -> int:
import torch
from concurrent.futures import ThreadPoolExecutor
@@ -783,5 +847,7 @@ def _copy_host_tensors_(entries: list) -> int:
cpu[tensor] = new
if tensor.device.type == "cpu" and tensor.data_ptr() == host.data_ptr():
tensor.data = new
+ if undo is not None:
+ undo.append((group, tensor, host, new))
copied += new.numel() * new.element_size()
return copied
diff --git a/studio/backend/tests/test_h3_resident_unpin.py b/studio/backend/tests/test_h3_resident_unpin.py
new file mode 100644
index 00000000000..7e084316a72
--- /dev/null
+++ b/studio/backend/tests/test_h3_resident_unpin.py
@@ -0,0 +1,276 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
+
+"""MiniMax-H3 streamed denoiser: a block resident on the card drops its host copy (the conditioner's resident share
+takes that RAM), and a demoted block gets an exact host copy back from the device before it streams again."""
+
+from __future__ import annotations
+
+import logging
+
+import pytest
+
+torch = pytest.importorskip("torch")
+
+
+class _HostGroup:
+ """A diffusers-style offload group: ``cpu_param_dict`` holds the host copy, onload gives the weight new storage."""
+
+ def __init__(
+ self,
+ seed: int,
+ n: int = 64,
+ ):
+ g = torch.Generator().manual_seed(seed)
+ lin = torch.nn.Linear(n, n, bias = True)
+ with torch.no_grad():
+ lin.weight.copy_(torch.randn(n, n, generator = g))
+ lin.bias.copy_(torch.randn(n, generator = g))
+ self.modules = [lin]
+ self.parameters = []
+ self.buffers = []
+ self.stream = None
+ self.cpu_param_dict = {p: p.data for p in lin.parameters()}
+ self.where = "cpu"
+
+ def onload_(self):
+ for p, host in self.cpu_param_dict.items():
+ p.data = host.clone() # the "device" copy: distinct storage
+ self.where = "cuda"
+
+ def offload_(self):
+ for p in list(self.modules[0].parameters()):
+ p.data = self.cpu_param_dict[p]
+ self.where = "cpu"
+
+
+def _residency(n_blocks: int, logger = None):
+ from core.inference.video_minimax_h3_residency import H3Residency, group_payload_bytes
+
+ r = H3Residency.__new__(H3Residency)
+ r.top = None
+ r.blocks = [_HostGroup(i) for i in range(n_blocks)]
+ r.device = "cuda"
+ r.logger = logger
+ r.top_bytes = 0
+ r.block_bytes = [group_payload_bytes(g) for g in r.blocks]
+ r.max_blocks = 0
+ r.host_yield = None
+ r.released_bytes = 0
+ return r
+
+
[email protected](autouse = True)
+def _no_cuda(monkeypatch):
+ monkeypatch.setattr(torch.cuda, "is_available", lambda: False)
+ monkeypatch.delenv("UNSLOTH_H3_RESIDENT_UNPIN", raising = False)
+
+
+def test_a_resident_block_drops_its_host_copy():
+ from core.inference import video_minimax_h3_residency as res
+
+ r = _residency(4)
+ r.fit(r.block_bytes[0] * 2, initial = True)
+ assert r.resident_blocks() == 2
+ for g in r.blocks[:2]:
+ assert g.cpu_param_dict == {}
+ assert res.released_host_bytes(g) == res.group_payload_bytes(g)
+ for g in r.blocks[2:]:
+ assert len(g.cpu_param_dict) == 2 and res.released_host_bytes(g) == 0
+ assert r.released_bytes == sum(r.block_bytes[:2])
+
+
+def test_a_demoted_block_streams_from_an_exact_rebuilt_copy(caplog):
+ r = _residency(3, logger = logging.getLogger("h3test"))
+ before = [[t.clone() for t in g.modules[0].parameters()] for g in r.blocks]
+ r.fit(sum(r.block_bytes), initial = True)
+ assert all(g.cpu_param_dict == {} for g in r.blocks)
+ with caplog.at_level(logging.INFO, logger = "h3test"):
+ r.fit(r.block_bytes[0]) # a bigger request: blocks 1 and 2 go back to streaming
+ assert r.resident_blocks() == 1
+ assert "rebuilt a demoted block's host copy from the device" in caplog.text
+ for g, ref in zip(r.blocks[1:], before[1:]):
+ assert g.where == "cpu" and len(g.cpu_param_dict) == 2
+ for p, want in zip(g.modules[0].parameters(), ref):
+ assert torch.equal(p.data, want)
+ assert (
+ p.data.data_ptr() == g.cpu_param_dict[p].data_ptr()
+ ) # offload re-pointed at the rebuilt copy
+ g.onload_() # streams again from the rebuilt copy
+ for p, want in zip(g.modules[0].parameters(), ref):
+ assert torch.equal(p.data, want)
+ r.fit(sum(r.block_bytes)) # promoted again: released again
+ assert all(g.cpu_param_dict == {} for g in r.blocks)
+
+
+def test_the_unpin_kill_switch_keeps_the_host_copy(monkeypatch):
+ monkeypatch.setenv("UNSLOTH_H3_RESIDENT_UNPIN", "0")
+ r = _residency(2)
+ r.fit(sum(r.block_bytes), initial = True)
+ assert r.resident_blocks() == 2
+ assert all(len(g.cpu_param_dict) == 2 for g in r.blocks) and r.released_bytes == 0
+
+
+def test_a_block_still_reading_its_host_copy_is_not_released():
+ from core.inference import video_minimax_h3_residency as res
+
+ g = _HostGroup(0)
+ res.make_resident(g)
+ g.offload_ = res._noop
+ for p, host in g.cpu_param_dict.items():
+ p.data = host # onload left it on the host
+ assert res.release_host_copy(g) == 0 and len(g.cpu_param_dict) == 2
+
+
+def test_a_rebuild_asks_the_conditioner_share_for_ram_when_the_host_is_tight(monkeypatch):
+ from core.inference import diffusion_small_host
+
+ asked = []
+ monkeypatch.setattr(diffusion_small_host, "host_ram_mib", lambda: (16384, 100))
+ r = _residency(2)
+ r.host_yield = lambda nbytes: asked.append(nbytes) or nbytes
+ r.fit(sum(r.block_bytes), initial = True)
+ r.fit(0)
+ assert len(asked) == 2 and all(n >= (2048 - 100) << 20 for n in asked)
+ assert all(len(g.cpu_param_dict) == 2 for g in r.blocks)
+
+
+def test_release_all_rebuilds_every_released_block():
+ from core.inference import video_minimax_h3_residency as res
+
+ r = _residency(3)
+ ref = [[t.clone() for t in g.modules[0].parameters()] for g in r.blocks]
+ r.fit(sum(r.block_bytes), initial = True)
+ res.release_all(r)
+ assert r.resident_bytes() == 0
+ for g, want in zip(r.blocks, ref):
+ assert all(torch.equal(p.data, w) for p, w in zip(g.modules[0].parameters(), want))
+
+
+def test_the_conditioner_share_yields_back_to_its_mapping():
+ from core.inference import video_minimax_h3_te as te
+
+ class _TE:
+ pass
+
+ lin = torch.nn.Linear(32, 32)
+ mapped = {p: p.data for p in lin.parameters()}
+ group = type("G", (), {})()
+ group.cpu_param_dict = dict(mapped)
+ holder = _TE()
+ holder.model = torch.nn.Module()
+ entries = [(group, p, h) for p, h in mapped.items()]
+ copied = te._copy_host_tensors_(entries, undo = te._undo_list(holder.model))
+ assert copied == sum(h.numel() * h.element_size() for h in mapped.values())
+ assert all(group.cpu_param_dict[p] is not mapped[p] for p in mapped)
+ freed = te.yield_h3_te_resident(holder, 1)
+ assert freed > 0
+ weight_or_bias = [p for p in mapped if group.cpu_param_dict[p] is mapped[p]]
+ assert len(weight_or_bias) == 1 # last copied first, only as much as asked
+ te.yield_h3_te_resident(holder, 1 << 30)
+ assert all(group.cpu_param_dict[p] is mapped[p] for p in mapped)
+ assert all(p.data.data_ptr() == mapped[p].data_ptr() for p in mapped)
+
+
+def test_a_refit_that_releases_more_hands_it_to_the_conditioner_once():
+ r = _residency(4)
+ r.topped_bytes = 0
+ calls = []
+ r.host_top_up = lambda: calls.append(r.released_bytes)
+ r.fit(r.block_bytes[0], initial = True) # the load's fit (a larger preset): one block
+ r.fit(sum(r.block_bytes)) # the request's fit: every block, more released
+ r.fit(sum(r.block_bytes)) # same request again: nothing new
+ assert calls == [r.block_bytes[0], sum(r.block_bytes)]
+ r.fit(r.block_bytes[0]) # demote: less released, no top-up
+ assert len(calls) == 2 and r.topped_bytes == r.block_bytes[0]
+
+
+def test_the_conditioner_top_up_copies_only_what_is_still_mapped(monkeypatch):
+ from core.inference import diffusion_small_host
+ from core.inference import video_minimax_h3_te as te
+
+ lin = torch.nn.Linear(32, 32)
+ group = type("G", (), {})()
+ group.modules = [lin]
+ group.parameters = []
+ group.buffers = []
+ group.cpu_param_dict = {p: p.data for p in lin.parameters()}
+
+ class _TE:
+ pass
+
+ holder = _TE()
+ holder.model = torch.nn.Module()
+ monkeypatch.setattr(diffusion_small_host, "host_ram_mib", lambda: (65536, 65536))
+ monkeypatch.setattr(diffusion_small_host, "host_commit_mib", lambda: None)
+ monkeypatch.setattr(te, "_H3_TE_RESIDENT_MIN_MIB", 0)
+ assert te.top_up_h3_te_resident(holder) == 0 # not managed by a load yet
+ te.keep_h3_te_resident(holder, groups = [group])
+ undo = getattr(holder.model, te._TE_RESIDENT_UNDO_ATTR)
+ assert len(undo) == 2
+ te.keep_h3_te_resident(holder, groups = [group]) # a top-up: nothing still mapped
+ assert len(undo) == 2
+
+
+def test_a_mostly_released_slab_is_compacted_so_it_can_be_freed(monkeypatch):
+ """First-fit packs a later block's small weights into an earlier slab's tail: once the resident blocks drop their
+ copies, those few weights alone would keep the whole slab pinned. They move to their own copies (exact)."""
+ from core.inference import video_minimax_h3_residency as res
+
+ monkeypatch.setattr(res, "_H3_COMPACT_MIN_SLAB", 0)
+ monkeypatch.setattr(res, "_pin_host_tensor", lambda src: src.clone())
+ slab = torch.arange(4096, dtype = torch.float32)
+ big = _HostGroup(0, n = 60) # 60*60 + 60 = 3660 floats: most of the slab
+ small = _HostGroup(1, n = 4) # 20 floats in the slab's tail
+ off = 0
+ for g in (big, small):
+ for p in g.modules[0].parameters():
+ view = slab[off : off + p.numel()].view(p.shape)
+ view.copy_(p.data)
+ g.cpu_param_dict[p] = view
+ p.data = view
+ off += p.numel()
+ want = [p.data.clone() for p in small.modules[0].parameters()]
+ r = _residency(0)
+ r.blocks = [big, small]
+ r.block_bytes = [res.group_payload_bytes(g) for g in r.blocks]
+ r.fit(r.block_bytes[0], initial = True)
+ assert r.resident_blocks() == 1 and big.cpu_param_dict == {}
+ for p, w in zip(small.modules[0].parameters(), want):
+ host = small.cpu_param_dict[p]
+ assert host.untyped_storage().data_ptr() != slab.untyped_storage().data_ptr()
+ assert torch.equal(host, w)
+ assert p.data.data_ptr() == host.data_ptr() # the streamed weight reads its new copy
+
+
+def test_a_well_used_slab_is_left_alone(monkeypatch):
+ from core.inference import video_minimax_h3_residency as res
+
+ monkeypatch.setattr(res, "_H3_COMPACT_MIN_SLAB", 0)
+ slab = torch.zeros(30) # 20 of 30 floats still used
+ g = _HostGroup(0, n = 4)
+ off = 0
+ for p in g.modules[0].parameters():
+ g.cpu_param_dict[p] = slab[off : off + p.numel()].view(p.shape)
+ off += p.numel()
+ assert res.compact_host_slabs([g]) == 0
+ assert all(
+ h.untyped_storage().data_ptr() == slab.untyped_storage().data_ptr()
+ for h in g.cpu_param_dict.values()
+ )
+
+
+def test_a_rebuild_restores_the_load_reserve_not_just_the_floor(monkeypatch):
+ """A demote after the conditioner share was topped up must give back enough for the load's own reserve (an L4
+ renderer was OOM-killed when only a 2 GiB floor was restored)."""
+ from core.inference import diffusion_small_host
+
+ monkeypatch.setattr(diffusion_small_host, "host_ram_mib", lambda: (53000, 5000))
+ monkeypatch.setattr(diffusion_small_host, "encoder_resident_keep_mib", lambda total: 7982)
+ asked = []
+ r = _residency(1)
+ r.host_yield = lambda nbytes: asked.append(nbytes) or nbytes
+ r.fit(sum(r.block_bytes), initial = True)
+ r.fit(0)
+ need_mib = r.block_bytes[0] >> 20
+ assert asked == [(need_mib + 7982 - 5000) << 20] # the 2 GiB floor alone would ask for nothing