Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

Studio MiniMax-H3: skip the up-front denoiser slab pin when every block will be resident (-16 GiB host peak on A100 40 GB) (#17354)

Daniel Han61e18dd5f112
diff --git a/studio/backend/core/inference/diffusion_prequant.py b/studio/backend/core/inference/diffusion_prequant.py
index 34672067a4e..1edde1bc589 100644
--- a/studio/backend/core/inference/diffusion_prequant.py
+++ b/studio/backend/core/inference/diffusion_prequant.py
@@ -3388,8 +3388,10 @@ def stream_prequantized_module(
*,
logger: Any = None,
label: str = "pre-quantized denoiser",
+ defer_pin: bool = False,
) -> Optional[str]:
- """Streams via group offloading; raises once unhooked, as a resident fallback would OOM each render."""
+ """Streams via group offloading; raises once unhooked, as a resident fallback would OOM each render.
+ ``defer_pin``: lazy host copies, no up-front pin (the caller pins what does not end up resident)."""
if not torchao_group_offload_supported():
return None
import inspect
@@ -3441,7 +3443,9 @@ def stream_prequantized_module(
kwargs["non_blocking"] = True
if "record_stream" in params:
kwargs["record_stream"] = True
- if "low_cpu_mem_usage" in params:
+ if "low_cpu_mem_usage" in params and defer_pin:
+ kwargs["low_cpu_mem_usage"] = True
+ elif "low_cpu_mem_usage" in params:
from itertools import chain
payload_mib = sum(
tensor_payload_bytes(t) for t in chain(module.parameters(), module.buffers())
diff --git a/studio/backend/core/inference/video.py b/studio/backend/core/inference/video.py
index 6b0902848fa..a108805529e 100644
--- a/studio/backend/core/inference/video.py
+++ b/studio/backend/core/inference/video.py
@@ -3274,6 +3274,47 @@ def _h3_residency_budget_bytes(
return 0
+def _h3_resident_pin_deferral(
+ fam: Any, denoiser: Any, device: str, *, te_streamed: bool, te_scheme: Optional[str]
+) -> Optional[int]:
+ """Denoiser bytes if the initial fit is predicted to hold it whole (so pins can be deferred), else None."""
+ try:
+ from .video_minimax_h3_residency import (
+ h3_defer_resident_pin_enabled,
+ h3_dit_resident_enabled,
+ h3_resident_unpin_enabled,
+ )
+
+ if denoiser is None or not str(device).startswith("cuda"):
+ return None
+ if not (
+ h3_defer_resident_pin_enabled()
+ and h3_dit_resident_enabled()
+ and h3_resident_unpin_enabled()
+ ):
+ return None
+ from itertools import chain
+
+ from .diffusion_prequant import tensor_payload_bytes
+
+ total = sum(
+ tensor_payload_bytes(t) for t in chain(denoiser.parameters(), denoiser.buffers())
+ )
+ big_w, big_h = max(fam.resolution_presets, key = lambda wh: wh[0] * wh[1])
+ budget = _h3_residency_budget_bytes(
+ types.SimpleNamespace(resident_bytes = lambda: 0),
+ device,
+ width = big_w,
+ height = big_h,
+ frames = fam.default_num_frames,
+ te_streamed = te_streamed,
+ te_scheme = te_scheme,
+ )
+ return int(total) if 0 < total <= budget else None
+ except Exception: # noqa: BLE001 -- unpredictable: pin up front
+ return None
+
+
def _h3_placement_tier(
*,
quantised: bool,
@@ -12253,8 +12294,25 @@ class VideoBackend:
),
logger = logger,
)
+ pin_deferred_bytes = (
+ _h3_resident_pin_deferral(
+ fam,
+ denoiser,
+ device,
+ te_streamed = bool(te_streamed),
+ te_scheme = text_encoder_quant_engaged,
+ )
+ if stream
+ else None
+ )
placed = (
- stream_prequantized_module(manager, denoiser, device, logger = logger)
+ stream_prequantized_module(
+ manager,
+ denoiser,
+ device,
+ logger = logger,
+ **({"defer_pin": True} if pin_deferred_bytes is not None else {}),
+ )
if stream
else None
)
@@ -12286,6 +12344,9 @@ class VideoBackend:
h3_dit_resident_enabled,
)
+ pin_deferred = False
+ pin_deferred_budget = 0
+
if denoiser_streamed in ("stream", "stream_lazy"):
from .video_minimax_h3_residency import pin_streamed_top_level_group
@@ -12294,6 +12355,37 @@ class VideoBackend:
except Exception as exc: # noqa: BLE001 -- a speed-up only
logger.warning("video.h3_top_group: %s", exc)
block_pinner = None
+ if (
+ pin_deferred_bytes is not None
+ and denoiser_streamed == "stream_lazy"
+ ):
+ from .video_minimax_h3_residency import defer_resident_block_pins
+ try:
+ residency = H3Residency(denoiser, device, logger = logger)
+ big_w, big_h = max(
+ fam.resolution_presets, key = lambda wh: wh[0] * wh[1]
+ )
+ pin_deferred_budget = _h3_residency_budget_bytes(
+ residency,
+ device,
+ width = big_w,
+ height = big_h,
+ frames = fam.default_num_frames,
+ te_streamed = bool(te_streamed),
+ te_scheme = text_encoder_quant_engaged,
+ )
+ pin_deferred = defer_resident_block_pins(
+ residency, pin_deferred_budget, logger = logger
+ )
+ except Exception as exc: # noqa: BLE001 -- pin as the lazy path does
+ logger.warning("video.h3_block_pin: %s", exc)
+ pin_deferred = False
+ if pin_deferred:
+ # every block is promoted and released by the fit below
+ denoiser_streamed = "stream"
+ denoiser_single_host_copy = True
+ else:
+ residency = None
if denoiser_streamed == "stream_lazy":
from .video_minimax_h3_residency import (
defer_streamed_block_pins,
@@ -12339,7 +12431,8 @@ class VideoBackend:
"stream_lazy",
):
try:
- residency = H3Residency(denoiser, device, logger = logger)
+ if not pin_deferred:
+ residency = H3Residency(denoiser, device, logger = logger)
if residency.usable:
from .video_minimax_h3_te import yield_h3_te_resident
@@ -12360,7 +12453,9 @@ class VideoBackend:
fam.resolution_presets, key = lambda wh: wh[0] * wh[1]
)
residency.fit(
- _h3_residency_budget_bytes(
+ pin_deferred_budget
+ if pin_deferred
+ else _h3_residency_budget_bytes(
residency,
device,
width = big_w,
@@ -12382,6 +12477,12 @@ class VideoBackend:
from .video_minimax_h3_residency import release_all
release_all(residency, logger = logger)
residency = None
+ if pin_deferred:
+ from .video_minimax_h3_residency import pin_deferred_block_groups
+ try:
+ pin_deferred_block_groups(denoiser, logger = logger)
+ except Exception as exc: # noqa: BLE001 -- they pin on each onload
+ logger.warning("video.h3_block_pin: %s", exc)
else:
denoiser_pinned = pin_prequantized_module(
manager, denoiser, device, logger = logger
diff --git a/studio/backend/core/inference/video_minimax_h3_residency.py b/studio/backend/core/inference/video_minimax_h3_residency.py
index ea1b82855ea..1bb154bf0bf 100644
--- a/studio/backend/core/inference/video_minimax_h3_residency.py
+++ b/studio/backend/core/inference/video_minimax_h3_residency.py
@@ -428,19 +428,131 @@ def _host_empty_cache() -> None:
pass
+# A fully resident initial fit skips the up-front block pin (18 s, 20 GB pinned transient on 48 GB); leftovers pin after.
+H3_DEFER_RESIDENT_PIN_ENV = "UNSLOTH_H3_DEFER_RESIDENT_PIN"
+_PIN_DEFERRED_ATTR = "_unsloth_pin_deferred"
+
+
+def h3_defer_resident_pin_enabled() -> bool:
+ return str(os.environ.get(H3_DEFER_RESIDENT_PIN_ENV, "")).strip().lower() not in (
+ "0",
+ "off",
+ "false",
+ "no",
+ )
+
+
+def _onload_pageable(group: Any) -> bool:
+ """diffusers' stream onload of a pin-deferred group, read from the pageable host copy without pinning."""
+ from contextlib import nullcontext
+
+ cpu = getattr(group, "cpu_param_dict", None)
+ process = getattr(group, "_process_tensors_from_modules", None)
+ if not isinstance(cpu, dict) or not cpu or not callable(process):
+ return False
+ import torch
+
+ stream = getattr(group, "stream", None)
+ if stream is not None:
+ stream.synchronize()
+ current = torch.cuda.current_stream() if stream is not None else None
+ with nullcontext() if stream is None else torch.cuda.stream(stream):
+ process(cpu, default_stream = current)
+ return True
+
+
+def _host_alias(tensor: Any) -> Any:
+ """A host-copy object distinct from ``tensor`` over the same storage. diffusers' lazy copy of a torchao weight is
+ the weight itself, so the onload's swap_tensors would carry that "host copy" to the device as well."""
+ if _is_wrapper(tensor):
+ names, ctx = tensor.__tensor_flatten__()
+ inner = {name: getattr(tensor, name) for name in names}
+ return type(tensor).__tensor_unflatten__(inner, ctx, tensor.size(), tensor.stride())
+ return tensor.detach()
+
+
+def defer_resident_block_pins(
+ residency: "H3Residency",
+ budget_bytes: int,
+ logger: Any = None,
+) -> bool:
+ """Engage the deferral if ``budget_bytes`` holds top + every block; call before the prefetch and initial fit."""
+ if not h3_defer_resident_pin_enabled() or not h3_resident_unpin_enabled():
+ return False
+ if not residency.usable:
+ return False
+ top, n = residency.plan(budget_bytes)
+ if (residency.top is not None and not top) or n != len(residency.blocks):
+ return False
+ for group in residency.blocks:
+ cpu = getattr(group, "cpu_param_dict", None)
+ if (
+ not getattr(group, "low_cpu_mem_usage", False)
+ or getattr(group, "stream", None) is None
+ or getattr(group, "offload_to_disk_path", None)
+ or not isinstance(cpu, dict)
+ or not cpu
+ or is_resident(group)
+ ):
+ return False
+ try:
+ aliases = [
+ {t: _host_alias(src) for t, src in group.cpu_param_dict.items() if src is t}
+ for group in residency.blocks
+ ]
+ except Exception: # noqa: BLE001 -- unknown wrapper: pin as the lazy path does
+ return False
+ for group, alias in zip(residency.blocks, aliases):
+ group.cpu_param_dict.update(alias)
+ group.low_cpu_mem_usage = False
+ setattr(group, _PIN_DEFERRED_ATTR, True)
+ if logger is not None:
+ logger.info(
+ "video.h3_block_pin: %d block groups will be resident, pin deferred (%.2f GB)",
+ len(residency.blocks),
+ sum(residency.block_bytes) / 1e9,
+ )
+ return True
+
+
+def pin_deferred_block_groups(transformer: Any, logger: Any = None) -> tuple[int, int]:
+ """Pin the deferred blocks the initial fit left streamed; returns (groups pinned, block groups)."""
+ _top, blocks = h3_offload_groups(transformer)
+ left = [g for g in blocks if getattr(g, _PIN_DEFERRED_ATTR, False) and not is_resident(g)]
+ if not left:
+ return 0, len(blocks)
+ for group in left:
+ group.low_cpu_mem_usage = True
+ setattr(group, _PIN_DEFERRED_ATTR, False)
+ if logger is not None:
+ logger.info(
+ "video.h3_block_pin: %d of %d pin-deferred block groups stay streamed, pinning them now",
+ len(left),
+ len(blocks),
+ )
+ return pin_streamed_block_groups(transformer, logger = logger)
+
+
def make_resident(group: Any) -> None:
"""Onload ``group`` once and make its hooks' onload / offload no-ops, so it stays on the device."""
import torch
if is_resident(group):
return
- # Instance attribute: an instance override is an onload path too, except the prefetcher's own.
- prefetcher = getattr(group, "_unsloth_prefetcher", None)
- owns = getattr(prefetcher, "owns", None)
- if callable(owns) and owns(group):
- _outside_inference_mode(type(group).onload_)(group)
- else:
- group.onload_()
+ promoted = False
+ if getattr(group, _PIN_DEFERRED_ATTR, False):
+ try:
+ promoted = _outside_inference_mode(_onload_pageable)(group)
+ except Exception: # noqa: BLE001 -- diffusers' onload re-copies every tensor from the host copy
+ promoted = False
+ if not promoted:
+ # Instance attribute: an instance override is an onload path too, except the prefetcher's own.
+ prefetcher = getattr(group, "_unsloth_prefetcher", None)
+ owns = getattr(prefetcher, "owns", None)
+ if callable(owns) and owns(group):
+ _outside_inference_mode(type(group).onload_)(group)
+ else:
+ group.onload_()
stream = getattr(group, "stream", None)
if stream is not None:
stream.synchronize()
@@ -450,6 +562,8 @@ def make_resident(group: Any) -> None:
group.onload_ = _resident_onload(group)
group.offload_ = _noop
group._unsloth_resident = True
+ if getattr(group, _PIN_DEFERRED_ATTR, False):
+ setattr(group, _PIN_DEFERRED_ATTR, False)
def demote(
diff --git a/studio/backend/tests/test_h3_defer_resident_pin.py b/studio/backend/tests/test_h3_defer_resident_pin.py
new file mode 100644
index 00000000000..68dde15e7b3
--- /dev/null
+++ b/studio/backend/tests/test_h3_defer_resident_pin.py
@@ -0,0 +1,400 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
+
+"""MiniMax-H3: a fully resident initial fit defers the up-front block pin (UNSLOTH_H3_DEFER_RESIDENT_PIN)."""
+
+from __future__ import annotations
+
+import logging
+import types
+
+import pytest
+
+torch = pytest.importorskip("torch")
+
+DEFER_ENV = "UNSLOTH_H3_DEFER_RESIDENT_PIN"
+
+
+class _Stream:
+ def synchronize(self):
+ return None
+
+
+class _LazyGroup:
+ """A lazily streamed diffusers block group: ``cpu_param_dict`` maps each weight to its pageable host copy (the
+ weight itself, as diffusers' lazy ``_to_cpu`` gives a torchao weight)."""
+
+ def __init__(
+ self,
+ seed: int,
+ n: int = 32,
+ ):
+ g = torch.Generator().manual_seed(seed)
+ lin = torch.nn.Linear(n, n, bias = True)
+ with torch.no_grad():
+ lin.weight.copy_(torch.randn(n, n, generator = g))
+ lin.bias.copy_(torch.randn(n, generator = g))
+ self.modules = [lin]
+ self.parameters = []
+ self.buffers = []
+ self.stream = _Stream()
+ self.low_cpu_mem_usage = True
+ self.offload_to_disk_path = None
+ self.cpu_param_dict = {p: p for p in lin.parameters()}
+ self.original = {p: p.detach().clone() for p in lin.parameters()}
+ self.onloads = 0
+ self.processed_with: list = []
+
+ def _process_tensors_from_modules(
+ self,
+ pinned_memory = None,
+ default_stream = None,
+ ):
+ self.processed_with.append((pinned_memory, default_stream))
+ for p in list(self.modules[0].parameters()):
+ p.data = pinned_memory[p].detach().clone() # the "device" copy: distinct storage
+
+ def onload_(self):
+ self.onloads += 1
+ self._process_tensors_from_modules(
+ {p: t.detach().clone() for p, t in self.cpu_param_dict.items()}
+ )
+
+ def offload_(self):
+ for p in list(self.modules[0].parameters()):
+ p.data = self.cpu_param_dict[p]
+
+ def _init_cpu_param_dict(self):
+ return {p: p.detach().clone() for p in self.modules[0].parameters()}
+
+
+def _residency(n_blocks: int, logger = None):
+ from core.inference.video_minimax_h3_residency import H3Residency, group_payload_bytes
+
+ r = H3Residency.__new__(H3Residency)
+ r.top = None
+ r.blocks = [_LazyGroup(i) for i in range(n_blocks)]
+ r.device = "cuda"
+ r.logger = logger
+ r.top_bytes = 0
+ r.block_bytes = [group_payload_bytes(g) for g in r.blocks]
+ r.max_blocks = 0
+ r.host_yield = None
+ r.host_top_up = None
+ r.released_bytes = 0
+ r.topped_bytes = 0
+ r.compacted_bytes = 0
+ r.compact_pending = False
+ return r
+
+
+_CURRENT = object()
+
+
[email protected](autouse = True)
+def _env(monkeypatch):
+ from contextlib import nullcontext
+
+ monkeypatch.setattr(torch.cuda, "is_available", lambda: False)
+ monkeypatch.setattr(torch.cuda, "stream", lambda s: nullcontext())
+ monkeypatch.setattr(torch.cuda, "current_stream", lambda *a: _CURRENT)
+ for name in (
+ DEFER_ENV,
+ "UNSLOTH_H3_RESIDENT_UNPIN",
+ "UNSLOTH_H3_DIT_RESIDENT",
+ "UNSLOTH_H3_STREAM_BLOCK_PIN",
+ ):
+ monkeypatch.delenv(name, raising = False)
+
+
+# -- prediction before placement -------------------------------------------------------------------------------------
+
+
+def _fam():
+ return types.SimpleNamespace(
+ resolution_presets = ((640, 384), (1344, 768)), default_num_frames = 121
+ )
+
+
+def _predict(monkeypatch, budget):
+ from core.inference import video
+
+ seen = {}
+
+ def fake_budget(residency, device, **kw):
+ seen["resident"] = residency.resident_bytes()
+ seen.update(kw)
+ return budget
+
+ monkeypatch.setattr(video, "_h3_residency_budget_bytes", fake_budget)
+ net = torch.nn.Sequential(torch.nn.Linear(64, 64), torch.nn.Linear(64, 64))
+ total = sum(p.numel() * p.element_size() for p in net.parameters())
+ got = video._h3_resident_pin_deferral(_fam(), net, "cuda", te_streamed = True, te_scheme = "int8")
+ return got, total, seen
+
+
+def test_prediction_defers_when_the_initial_fit_holds_the_whole_denoiser(monkeypatch):
+ got, total, seen = _predict(monkeypatch, budget = 10**9)
+ assert got == total
+ # the fit's own sizing: largest preset, default length, nothing resident yet, streamed conditioner
+ assert (seen["width"], seen["height"], seen["frames"]) == (1344, 768, 121)
+ assert seen["resident"] == 0 and seen["te_streamed"] is True and seen["te_scheme"] == "int8"
+
+
+def test_prediction_keeps_the_up_front_pin_on_a_partial_fit(monkeypatch):
+ # 12 / 16 GB cards: the budget holds only part of the denoiser
+ _got, total, _seen = _predict(monkeypatch, budget = 0)
+ assert _got is None
+ got, _total, _seen = _predict(monkeypatch, budget = total - 1)
+ assert got is None
+
+
+ "env",
+ [
+ {DEFER_ENV: "0"},
+ {"UNSLOTH_H3_DIT_RESIDENT": "0"},
+ {"UNSLOTH_H3_RESIDENT_UNPIN": "0"},
+ ],
+)
+def test_prediction_kill_switches(monkeypatch, env):
+ for k, v in env.items():
+ monkeypatch.setenv(k, v)
+ got, _total, _seen = _predict(monkeypatch, budget = 10**12)
+ assert got is None
+
+
+def test_prediction_needs_a_cuda_device(monkeypatch):
+ from core.inference import video
+
+ monkeypatch.setattr(video, "_h3_residency_budget_bytes", lambda *a, **k: 10**12)
+ net = torch.nn.Linear(8, 8)
+ assert (
+ video._h3_resident_pin_deferral(_fam(), net, "cpu", te_streamed = True, te_scheme = None)
+ is None
+ )
+ assert (
+ video._h3_resident_pin_deferral(_fam(), None, "cuda", te_streamed = True, te_scheme = None)
+ is None
+ )
+
+
+# -- streaming without the up-front pin ------------------------------------------------------------------------------
+
+
+def _patch_stream(monkeypatch, plan):
+ pytest.importorskip("diffusers.hooks.group_offloading")
+ import diffusers.hooks as hooks
+
+ from core.inference import diffusion_memory as dm
+ from core.inference import diffusion_prequant as dp
+ from core.inference import host_pinning
+
+ monkeypatch.setattr(dp, "torchao_group_offload_supported", lambda: True)
+ monkeypatch.setattr(dp, "_unhook_from_manager", lambda *a, **k: True)
+ monkeypatch.setattr(dp, "_weights_pinnable", lambda m: True)
+ monkeypatch.setattr(dp, "_move_groups_outside_inference_mode", lambda m: None)
+ monkeypatch.setattr(dp, "_evict_rotation_hook", lambda *a: (lambda m, a: None))
+ monkeypatch.setattr(dm, "install_group_offload_buffer_restore", lambda: None)
+ monkeypatch.setattr(dm, "install_group_offload_hooks_eager", lambda: None)
+ monkeypatch.setattr(host_pinning, "host_pinning_allowed", lambda *a, **k: True)
+ plans = []
+
+ def fake_plan(*a, **k):
+ plans.append(a)
+ return plan
+
+ monkeypatch.setattr(dm, "_streamed_pin_plan", fake_plan)
+ applied = {}
+
+ def fake_apply(
+ module,
+ *,
+ onload_device,
+ offload_device,
+ offload_type,
+ num_blocks_per_group,
+ use_stream,
+ non_blocking = False,
+ record_stream = False,
+ low_cpu_mem_usage = False,
+ ):
+ applied["low_cpu_mem_usage"] = low_cpu_mem_usage
+
+ monkeypatch.setattr(hooks, "apply_group_offloading", fake_apply)
+ return dp, plans, applied
+
+
+def test_deferred_stream_skips_the_up_front_arena_pin(monkeypatch):
+ dp, plans, applied = _patch_stream(monkeypatch, plan = (True, True))
+ net = torch.nn.Sequential(torch.nn.Linear(64, 64))
+ mode = dp.stream_prequantized_module(object(), net, "cuda", defer_pin = True)
+ assert mode == "stream_lazy"
+ assert applied["low_cpu_mem_usage"] is True
+ assert plans == [] # no pin plan, no arena
+ assert getattr(net, "_unsloth_pin_arena_bytes", None) is None
+
+
+def test_undeferred_stream_still_pins_up_front(monkeypatch):
+ dp, plans, applied = _patch_stream(monkeypatch, plan = (True, True))
+ net = torch.nn.Sequential(torch.nn.Linear(64, 64))
+ mode = dp.stream_prequantized_module(object(), net, "cuda")
+ assert mode == "stream" and applied["low_cpu_mem_usage"] is False and len(plans) == 1
+
+
+# -- the residency fit -----------------------------------------------------------------------------------------------
+
+
+def test_deferral_engages_when_the_fit_holds_every_block(caplog):
+ from core.inference import video_minimax_h3_residency as res
+
+ logger = logging.getLogger("test_h3_defer")
+ r = _residency(4, logger)
+ with caplog.at_level(logging.INFO, logger = "test_h3_defer"):
+ assert res.defer_resident_block_pins(r, sum(r.block_bytes), logger = logger)
+ assert any(
+ "4 block groups will be resident, pin deferred" in rec.getMessage()
+ for rec in caplog.records
+ )
+ for g in r.blocks:
+ assert g.low_cpu_mem_usage is False
+ # host copy is its own object over the same storage: the onload's swap cannot carry it to the device
+ for p, host in g.cpu_param_dict.items():
+ assert host is not p and host.data_ptr() == p.data_ptr()
+
+
+def test_deferral_does_not_engage_on_a_partial_fit():
+ from core.inference import video_minimax_h3_residency as res
+
+ r = _residency(4)
+ assert not res.defer_resident_block_pins(r, sum(r.block_bytes) - 1)
+ for g in r.blocks:
+ assert g.low_cpu_mem_usage is True and not getattr(g, "_unsloth_pin_deferred", False)
+ assert all(host is p for p, host in g.cpu_param_dict.items())
+
+
+def test_kill_switch_keeps_the_blocks_lazy(monkeypatch):
+ from core.inference import video_minimax_h3_residency as res
+
+ monkeypatch.setenv(DEFER_ENV, "0")
+ r = _residency(3)
+ assert not res.defer_resident_block_pins(r, 10**12)
+ assert all(g.low_cpu_mem_usage is True for g in r.blocks)
+
+
+def test_deferred_blocks_promote_from_the_pageable_copy_and_release_it():
+ from core.inference import video_minimax_h3_residency as res
+
+ r = _residency(4)
+ assert res.defer_resident_block_pins(r, sum(r.block_bytes))
+ hosts = [dict(g.cpu_param_dict) for g in r.blocks]
+ top, n = r.fit(sum(r.block_bytes), initial = True)
+ assert n == 4 and r.resident_blocks() == 4
+ for g, host in zip(r.blocks, hosts):
+ # one direct copy from the pageable host copy, not diffusers' per-tensor pinning onload
+ assert g.onloads == 0 and len(g.processed_with) == 1
+ assert g.processed_with[0][0] == host
+ # recorded on the compute stream, as diffusers' stream onload does
+ assert g.processed_with[0][1] is _CURRENT
+ assert not getattr(g, "_unsloth_pin_deferred", False)
+ assert g.cpu_param_dict == {} and res.released_host_bytes(g) == res.group_payload_bytes(g)
+ for p in g.modules[0].parameters():
+ assert torch.equal(p, g.original[p])
+ assert r.released_bytes == sum(r.block_bytes)
+
+
+def test_a_demoted_deferred_block_streams_from_a_rebuilt_host_copy():
+ from core.inference import video_minimax_h3_residency as res
+
+ r = _residency(3)
+ assert res.defer_resident_block_pins(r, sum(r.block_bytes))
+ r.fit(sum(r.block_bytes), initial = True)
+ r.fit(r.block_bytes[0]) # a bigger request
+ assert r.resident_blocks() == 1
+ for g in r.blocks[1:]:
+ assert res.released_host_bytes(g) == 0 and len(g.cpu_param_dict) == 2
+ for p in g.modules[0].parameters():
+ assert p.data_ptr() == g.cpu_param_dict[p].data_ptr()
+ assert torch.equal(p, g.original[p])
+ g.onload_() # streams again through the normal onload
+ assert g.onloads == 1
+
+
+def test_a_smaller_fit_than_predicted_pins_the_leftover_blocks(monkeypatch):
+ from core.inference import video_minimax_h3_residency as res
+
+ r = _residency(4)
+ assert res.defer_resident_block_pins(r, sum(r.block_bytes))
+ r.fit(sum(r.block_bytes[:2]), initial = True) # the prediction was wrong
+ assert r.resident_blocks() == 2
+ seen = {}
+
+ def fake_pin(transformer, logger = None):
+ seen["lazy"] = [g for g in r.blocks if g.low_cpu_mem_usage]
+ return len(seen["lazy"]), len(r.blocks)
+
+ monkeypatch.setattr(res, "h3_offload_groups", lambda t: (None, list(r.blocks)))
+ monkeypatch.setattr(res, "pin_streamed_block_groups", fake_pin)
+ assert res.pin_deferred_block_groups(object()) == (2, 4)
+ assert seen["lazy"] == r.blocks[2:]
+ assert not any(getattr(g, "_unsloth_pin_deferred", False) for g in r.blocks)
+
+
+def test_leftover_pin_is_a_no_op_when_every_block_went_resident(monkeypatch):
+ from core.inference import video_minimax_h3_residency as res
+
+ r = _residency(3)
+ assert res.defer_resident_block_pins(r, sum(r.block_bytes))
+ r.fit(sum(r.block_bytes), initial = True)
+ called = []
+ monkeypatch.setattr(res, "h3_offload_groups", lambda t: (None, list(r.blocks)))
+ monkeypatch.setattr(res, "pin_streamed_block_groups", lambda *a, **k: called.append(1))
+ assert res.pin_deferred_block_groups(object()) == (0, 3)
+ assert called == []
+
+
+def test_a_torchao_host_alias_survives_the_onload_swap():
+ torchao = pytest.importorskip("torchao")
+ from torchao.quantization import Int8DynamicActivationInt8WeightConfig, quantize_
+
+ from core.inference.video_minimax_h3_residency import _host_alias
+
+ lin = torch.nn.Sequential(torch.nn.Linear(64, 64))
+ try:
+ quantize_(lin, Int8DynamicActivationInt8WeightConfig(version = 2))
+ except TypeError:
+ quantize_(lin, Int8DynamicActivationInt8WeightConfig())
+ w = lin[0].weight
+ if not hasattr(w, "__tensor_flatten__"):
+ pytest.skip("not a wrapper subclass in this torchao")
+ names, _ctx = w.__tensor_flatten__()
+ before = {n: getattr(w, n).data_ptr() for n in names}
+ alias = _host_alias(w)
+ assert alias is not w and type(alias) is type(w)
+ moved = _host_alias(w)
+ for n in names:
+ setattr(moved, n, getattr(w, n).clone()) # the "device" copy
+ torch.utils.swap_tensors(w, moved)
+ assert {n: getattr(alias, n).data_ptr() for n in names} == before
+ assert all(getattr(w, n).data_ptr() != before[n] for n in names)
+ del torchao
+
+
+def test_the_direct_promote_drives_diffusers_own_tensor_transfer():
+ """Guards diffusers' private ``_process_tensors_from_modules(pinned_memory, default_stream)``: the promote reads the
+ group's host-copy dict as its sources."""
+ go = pytest.importorskip("diffusers.hooks.group_offloading")
+ from core.inference import video_minimax_h3_residency as res
+
+ lin = torch.nn.Linear(16, 16)
+ group = go.ModuleGroup(
+ modules = [lin],
+ offload_device = torch.device("cpu"),
+ onload_device = torch.device("cpu"),
+ offload_leader = lin,
+ )
+ host = {p: p.detach().clone() for p in lin.parameters()}
+ group.cpu_param_dict = dict(host)
+ assert res._onload_pageable(group)
+ for p in lin.parameters():
+ assert p.data_ptr() == host[p].data_ptr()