Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
# SPDX-License-Identifier: AGPL-3.0-only
# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
"""Hermetic CPU-only tests for the diffusion memory planner and applier, with no torch or GPU."""
from __future__ import annotations
import sys
import types
import core.inference.diffusion_memory as diffusion_memory
import pytest
from core.inference.diffusion_memory import (
DEFAULT_GROUP_BLOCKS,
DEFAULT_IMAGE_HEIGHT,
DEFAULT_IMAGE_WIDTH,
MEMORY_MODE_BALANCED,
MEMORY_MODE_FAST,
MEMORY_MODE_LOW_VRAM,
OFFLOAD_GROUP,
OFFLOAD_MODEL,
OFFLOAD_NONE,
OFFLOAD_SEQUENTIAL,
OFFLOAD_STREAMING,
DeviceMemory,
MemoryPlan,
apply_memory_plan,
estimate_gguf_resident_mib,
estimate_image_runtime_mib,
normalize_memory_mode,
plan_diffusion_memory,
refine_memory_plan_for_components,
snapshot_device_memory,
)
def _target(
*,
device = "cuda",
backend = "cuda",
supports_offload = True,
):
"""Duck-typed DiffusionDeviceTarget stand-in with only the fields the planner reads."""
return types.SimpleNamespace(
device = device,
backend = backend,
supports_model_cpu_offload = supports_offload,
)
def _discrete(free_mib, total_mib = None):
return DeviceMemory("cuda", "cuda", "discrete_vram", free_mib, total_mib or free_mib)
def test_normalize_memory_mode_accepts_and_rejects():
assert normalize_memory_mode(None) is None
assert normalize_memory_mode(" ") is None
assert normalize_memory_mode("LOW-VRAM") == "low_vram"
assert normalize_memory_mode("Balanced") == "balanced"
with pytest.raises(ValueError):
normalize_memory_mode("ultra")
class _FakeNativeFunction:
def __init__(self, result = 0):
self.argtypes = None
self.restype = None
self.calls = []
self.result = result
def __call__(self, *args):
self.calls.append(args)
return self.result
@pytest.mark.parametrize(
("platform_name", "library_name", "symbol", "native_args"),
[
("linux", None, "malloc_trim", (0,)),
("darwin", None, "malloc_zone_pressure_relief", (None, 0)),
("win32", "ucrtbase.dll", "_heapmin", ()),
],
)
def test_host_memory_reclaimer_uses_and_caches_the_native_api(
monkeypatch, platform_name, library_name, symbol, native_args
):
native = _FakeNativeFunction()
library_loads = []
def fake_cdll(name):
library_loads.append(name)
return types.SimpleNamespace(**{symbol: native})
monkeypatch.setattr(diffusion_memory.sys, "platform", platform_name)
monkeypatch.setattr(diffusion_memory.ctypes, "CDLL", fake_cdll)
diffusion_memory._resolve_host_memory_reclaimer.cache_clear()
try:
assert diffusion_memory.reclaim_offload_host_memory(OFFLOAD_MODEL) is True
assert diffusion_memory.reclaim_offload_host_memory(OFFLOAD_MODEL) is True
finally:
diffusion_memory._resolve_host_memory_reclaimer.cache_clear()
assert library_loads == [library_name]
assert native.calls == [native_args, native_args]
assert native.argtypes is not None and native.restype is not None
def test_host_memory_reclaimer_windows_falls_back_and_logs_heap_failure_once(monkeypatch):
native = _FakeNativeFunction(result = -1)
library_loads = []
warnings = []
def fake_cdll(name):
library_loads.append(name)
if name == "ucrtbase.dll":
raise OSError("UCRT unavailable")
return types.SimpleNamespace(_heapmin = native)
monkeypatch.setattr(diffusion_memory.sys, "platform", "win32")
monkeypatch.setattr(diffusion_memory.ctypes, "CDLL", fake_cdll)
monkeypatch.setattr(diffusion_memory, "_host_memory_reclaim_warning_logged", False)
diffusion_memory._resolve_host_memory_reclaimer.cache_clear()
logger = types.SimpleNamespace(warning = lambda *args: warnings.append(args))
try:
assert diffusion_memory.reclaim_offload_host_memory(OFFLOAD_MODEL, logger) is False
assert diffusion_memory.reclaim_offload_host_memory(OFFLOAD_MODEL, logger) is False
finally:
diffusion_memory._resolve_host_memory_reclaimer.cache_clear()
assert library_loads == ["ucrtbase.dll", "msvcrt.dll"]
assert native.calls == [(), ()]
assert len(warnings) == 1
def test_host_memory_reclaimer_is_policy_scoped_and_best_effort(monkeypatch):
calls = []
monkeypatch.setattr(
diffusion_memory, "_resolve_host_memory_reclaimer", lambda: lambda: calls.append(0)
)
for policy in (OFFLOAD_NONE, OFFLOAD_GROUP, OFFLOAD_STREAMING, OFFLOAD_SEQUENTIAL):
assert diffusion_memory.reclaim_offload_host_memory(policy) is False
assert calls == []
assert diffusion_memory.reclaim_offload_host_memory(OFFLOAD_MODEL) is True
assert calls == [0]
def resolver_failure():
raise OSError("allocator unavailable")
monkeypatch.setattr(diffusion_memory, "_resolve_host_memory_reclaimer", resolver_failure)
assert diffusion_memory.reclaim_offload_host_memory(OFFLOAD_MODEL) is False
def call_failure():
raise RuntimeError("pressure API failed")
monkeypatch.setattr(diffusion_memory, "_resolve_host_memory_reclaimer", lambda: call_failure)
assert diffusion_memory.reclaim_offload_host_memory(OFFLOAD_MODEL) is False
def test_reclaim_host_memory_ignores_the_offload_policy(monkeypatch):
calls = []
monkeypatch.setattr(
diffusion_memory, "_resolve_host_memory_reclaimer", lambda: lambda: calls.append(0)
)
assert diffusion_memory.reclaim_host_memory() is True
assert calls == [0]
for policy in (OFFLOAD_NONE, OFFLOAD_GROUP, OFFLOAD_SEQUENTIAL):
assert diffusion_memory.reclaim_offload_host_memory(policy) is False
assert calls == [0]
def call_failure():
raise OSError("allocator unavailable")
monkeypatch.setattr(diffusion_memory, "_resolve_host_memory_reclaimer", lambda: call_failure)
assert diffusion_memory.reclaim_host_memory() is False
monkeypatch.setattr(diffusion_memory, "_resolve_host_memory_reclaimer", lambda: None)
assert diffusion_memory.reclaim_host_memory() is False
def test_host_memory_reclaimer_caches_unsupported_or_missing_apis(monkeypatch):
loads = []
monkeypatch.setattr(diffusion_memory.sys, "platform", "linux")
monkeypatch.setattr(
diffusion_memory.ctypes,
"CDLL",
lambda name: loads.append(name) or types.SimpleNamespace(),
)
diffusion_memory._resolve_host_memory_reclaimer.cache_clear()
try:
assert diffusion_memory.reclaim_offload_host_memory(OFFLOAD_MODEL) is False
assert diffusion_memory.reclaim_offload_host_memory(OFFLOAD_MODEL) is False
finally:
diffusion_memory._resolve_host_memory_reclaimer.cache_clear()
assert loads == [None]
monkeypatch.setattr(diffusion_memory.sys, "platform", "haiku")
monkeypatch.setattr(
diffusion_memory.ctypes,
"CDLL",
lambda name: pytest.fail("unsupported platforms must not load an allocator library"),
)
diffusion_memory._resolve_host_memory_reclaimer.cache_clear()
try:
assert diffusion_memory.reclaim_offload_host_memory(OFFLOAD_MODEL) is False
finally:
diffusion_memory._resolve_host_memory_reclaimer.cache_clear()
@pytest.mark.skipif(
not sys.platform.startswith("linux"),
reason = "exercises the real glibc allocator symbol",
)
def test_host_memory_reclaimer_calls_the_real_glibc_symbol():
"""Spy on the real CDLL, not a stub: a fresh CDLL(None).malloc_trim would carry default argtypes."""
import ctypes
if not hasattr(ctypes.CDLL(None), "malloc_trim"):
pytest.skip("no malloc_trim on this libc (musl); the resolver correctly declines")
real_cdll = ctypes.CDLL
captured = {}
def spy(name, *args, **kwargs):
library = real_cdll(name, *args, **kwargs)
captured["library"] = library
return library
monkeypatch_target = diffusion_memory.ctypes
original = monkeypatch_target.CDLL
monkeypatch_target.CDLL = spy
diffusion_memory._resolve_host_memory_reclaimer.cache_clear()
try:
assert diffusion_memory.reclaim_offload_host_memory(OFFLOAD_MODEL) is True
pressure = captured["library"].malloc_trim
assert pressure.argtypes == [ctypes.c_size_t]
assert pressure.restype is ctypes.c_int
# glibc: 1 = released to the OS, 0 = could not.
assert pressure(0) in (0, 1)
finally:
monkeypatch_target.CDLL = original
diffusion_memory._resolve_host_memory_reclaimer.cache_clear()
def test_host_memory_reclaimer_survives_a_python_without_ctypes(monkeypatch):
"""Without _ctypes (no libffi), the reclaimer must degrade to None, not stop backends from starting."""
monkeypatch.setattr(diffusion_memory, "ctypes", None)
diffusion_memory._resolve_host_memory_reclaimer.cache_clear()
try:
assert diffusion_memory._resolve_host_memory_reclaimer() is None
assert diffusion_memory.reclaim_offload_host_memory(OFFLOAD_MODEL) is False
assert diffusion_memory.reclaim_offload_host_memory(OFFLOAD_NONE) is False
finally:
diffusion_memory._resolve_host_memory_reclaimer.cache_clear()
def test_host_memory_reclaimer_reports_an_unsupported_allocator_once(monkeypatch):
"""The call site discards the return value, so an allocator we can never trim has to say so
itself or a permanent no-op is invisible while host RAM climbs."""
messages = []
logger = types.SimpleNamespace(
info = lambda fmt, *args: messages.append(fmt % args),
warning = lambda fmt, *args: messages.append(fmt % args),
)
monkeypatch.setattr(diffusion_memory, "_host_memory_reclaim_unsupported_logged", False)
monkeypatch.setattr(diffusion_memory.sys, "platform", "haiku")
diffusion_memory._resolve_host_memory_reclaimer.cache_clear()
try:
assert diffusion_memory.reclaim_offload_host_memory(OFFLOAD_MODEL, logger = logger) is False
assert diffusion_memory.reclaim_offload_host_memory(OFFLOAD_MODEL, logger = logger) is False
finally:
diffusion_memory._resolve_host_memory_reclaimer.cache_clear()
assert len(messages) == 1, messages
assert "haiku" in messages[0]
def test_estimate_gguf_resident_mib_matches_packed_size():
# GGUF stays packed on device and dequantises per matmul, so resident ~= on-disk size plus margin.
assert estimate_gguf_resident_mib(1000) == 1050
assert estimate_gguf_resident_mib(7220) == 7581
assert estimate_gguf_resident_mib(None) is None
def test_estimate_image_runtime_scales_with_pixels_and_family():
base = estimate_image_runtime_mib(width = DEFAULT_IMAGE_WIDTH, height = DEFAULT_IMAGE_HEIGHT)
bigger = estimate_image_runtime_mib(width = 2048, height = 2048)
assert bigger > base
turbo = estimate_image_runtime_mib(
width = DEFAULT_IMAGE_WIDTH, height = DEFAULT_IMAGE_HEIGHT, family = "z-image-turbo"
)
assert turbo < base
def test_cpu_target_never_offloads_but_tiles():
plan = plan_diffusion_memory(
target = _target(device = "cpu", backend = "cpu", supports_offload = False),
device_memory = DeviceMemory("cpu", "cpu", "system_memory", 8000, 16000),
model_dense_mib = 4000,
runtime_headroom_mib = 2000,
)
assert plan.offload_policy == OFFLOAD_NONE
# CPU/MPS have no separate device pool, so VAE tiling caps the spike.
assert plan.vae_tiling and plan.vae_slicing
def test_mps_unified_never_auto_offloads():
plan = plan_diffusion_memory(
target = _target(device = "mps", backend = "mps", supports_offload = False),
device_memory = DeviceMemory("mps", "mps", "unified_memory", 4000, 32000),
model_dense_mib = 20000,
runtime_headroom_mib = 4000,
)
assert plan.offload_policy == OFFLOAD_NONE
assert any("unified" in r for r in plan.reasons)
def test_unified_cuda_skips_offload_even_if_offload_capable():
# An integrated CUDA SoC has unified memory; CPU offload would free nothing.
plan = plan_diffusion_memory(
target = _target(device = "cuda", backend = "cuda", supports_offload = True),
device_memory = DeviceMemory("cuda", "cuda", "unified_memory", 2000, 16000),
model_dense_mib = 12000,
runtime_headroom_mib = 4000,
)
assert plan.offload_policy == OFFLOAD_NONE
def test_auto_resident_when_roomy():
plan = plan_diffusion_memory(
target = _target(),
device_memory = _discrete(80000),
model_dense_mib = 12000,
runtime_headroom_mib = 4000,
)
assert plan.offload_policy == OFFLOAD_NONE
assert plan.vae_tiling is False and plan.vae_slicing is False
def test_auto_model_offload_on_tight_fit():
# 24 GB free -> reserve 2400 -> budget 21600, 0.85*budget = 18360; 21000 lands between.
plan = plan_diffusion_memory(
target = _target(),
device_memory = _discrete(24000, 24000),
model_dense_mib = 16000,
runtime_headroom_mib = 4000,
base_overhead_mib = 1000,
)
assert plan.offload_policy == OFFLOAD_MODEL
assert plan.vae_tiling is True
def test_auto_group_offload_when_transformer_overflows_but_companions_fit():
plan = plan_diffusion_memory(
target = _target(),
device_memory = _discrete(8000, 8000),
model_dense_mib = 40000,
companion_dense_mib = 1500,
runtime_headroom_mib = 1000,
base_overhead_mib = 1000,
)
assert plan.offload_policy == OFFLOAD_GROUP
# Group keeps the VAE resident, so slicing (exact) but not tiling (lossy).
assert plan.vae_slicing is True and plan.vae_tiling is False
def test_auto_model_offload_when_companions_exceed_budget():
plan = plan_diffusion_memory(
target = _target(),
device_memory = _discrete(8000, 8000),
model_dense_mib = 40000,
companion_dense_mib = 30000,
runtime_headroom_mib = 4000,
)
assert plan.offload_policy == OFFLOAD_MODEL
def test_auto_model_offload_when_companion_size_unknown():
# Without a companion estimate group cannot be proven to fit, so take the safest cut.
plan = plan_diffusion_memory(
target = _target(),
device_memory = _discrete(8000, 8000),
model_dense_mib = 40000,
runtime_headroom_mib = 4000,
)
assert plan.offload_policy == OFFLOAD_MODEL
def test_auto_stays_resident_when_budget_unknown():
plan = plan_diffusion_memory(
target = _target(),
device_memory = _discrete(None, None),
model_dense_mib = 40000,
runtime_headroom_mib = 4000,
)
assert plan.offload_policy == OFFLOAD_NONE
assert any("unknown" in r for r in plan.reasons)
def test_explicit_modes_force_policy_regardless_of_budget():
roomy = _discrete(80000)
assert (
plan_diffusion_memory(
target = _target(),
device_memory = roomy,
model_dense_mib = 1000,
runtime_headroom_mib = 1000,
requested_mode = MEMORY_MODE_FAST,
).offload_policy
== OFFLOAD_NONE
)
assert (
plan_diffusion_memory(
target = _target(),
device_memory = roomy,
model_dense_mib = 1000,
runtime_headroom_mib = 1000,
requested_mode = MEMORY_MODE_BALANCED,
).offload_policy
== OFFLOAD_GROUP
)
assert (
plan_diffusion_memory(
target = _target(),
device_memory = roomy,
model_dense_mib = 1000,
runtime_headroom_mib = 1000,
requested_mode = MEMORY_MODE_LOW_VRAM,
).offload_policy
== OFFLOAD_MODEL
)
def test_fast_falls_back_to_model_offload_when_it_does_not_fit():
plan = plan_diffusion_memory(
target = _target(),
device_memory = _discrete(8000, 8000),
model_dense_mib = 40000,
runtime_headroom_mib = 4000,
requested_mode = MEMORY_MODE_FAST,
)
assert plan.offload_policy == OFFLOAD_MODEL
def test_explicit_cpu_offload_overrides_resident_auto_choice():
plan = plan_diffusion_memory(
target = _target(),
device_memory = _discrete(80000),
model_dense_mib = 4000,
runtime_headroom_mib = 2000,
explicit_offload = True,
)
assert plan.offload_policy == OFFLOAD_MODEL
assert any("explicit cpu_offload" in r for r in plan.reasons)
def test_explicit_memory_mode_wins_over_legacy_cpu_offload():
# memory_mode is documented to override cpu_offload.
plan = plan_diffusion_memory(
target = _target(),
device_memory = _discrete(80000),
model_dense_mib = 4000,
runtime_headroom_mib = 2000,
requested_mode = MEMORY_MODE_FAST,
explicit_offload = True,
)
assert plan.offload_policy == OFFLOAD_NONE
assert not any("explicit cpu_offload" in r for r in plan.reasons)
def test_explicit_cpu_offload_ignored_on_cpu_target():
plan = plan_diffusion_memory(
target = _target(device = "cpu", backend = "cpu", supports_offload = False),
device_memory = DeviceMemory("cpu", "cpu", "system_memory", 8000, 16000),
model_dense_mib = 4000,
runtime_headroom_mib = 2000,
explicit_offload = True,
)
assert plan.offload_policy == OFFLOAD_NONE
def test_snapshot_cpu_target_uses_system_memory(monkeypatch):
import core.inference.diffusion_memory as mem
monkeypatch.setattr(mem, "_system_memory_mib", lambda: (16000, 9000))
snap = snapshot_device_memory(_target(device = "cpu", backend = "cpu"))
assert snap.memory_kind == "system_memory"
assert snap.free_mib == 9000 and snap.total_mib == 16000
def test_snapshot_cuda_reads_mem_get_info(monkeypatch):
import sys
fake_torch = types.ModuleType("torch")
fake_torch.cuda = types.SimpleNamespace(
mem_get_info = lambda: (10 * 1024 * 1024 * 1024, 24 * 1024 * 1024 * 1024),
get_device_properties = lambda i: types.SimpleNamespace(integrated = False),
)
monkeypatch.setitem(sys.modules, "torch", fake_torch)
snap = snapshot_device_memory(_target())
assert snap.memory_kind == "discrete_vram"
assert snap.free_mib == 10 * 1024 and snap.total_mib == 24 * 1024
def test_snapshot_never_raises_on_probe_failure(monkeypatch):
import sys
fake_torch = types.ModuleType("torch")
def _boom():
raise RuntimeError("no cuda")
fake_torch.cuda = types.SimpleNamespace(mem_get_info = _boom)
monkeypatch.setitem(sys.modules, "torch", fake_torch)
snap = snapshot_device_memory(_target())
assert snap.free_mib is None and snap.total_mib is None
class _RecordingPipe:
def __init__(self) -> None:
self.calls: list[str] = []
self.offload_device = None
def to(self, device):
self.calls.append(f"to:{device}")
return self
def enable_model_cpu_offload(self, device = None):
self.calls.append("model_offload")
self.offload_device = device
def enable_sequential_cpu_offload(self, device = None):
self.calls.append("sequential_offload")
self.offload_device = device
def enable_vae_tiling(self):
self.calls.append("vae_tiling")
def enable_vae_slicing(self):
self.calls.append("vae_slicing")
def _plan(policy, *, tiling):
return plan_diffusion_memory(
target = _target(),
device_memory = _discrete(80000) if policy == OFFLOAD_NONE else _discrete(4000, 8000),
model_dense_mib = 1000 if policy == OFFLOAD_NONE else 40000,
runtime_headroom_mib = 1000,
requested_mode = {
OFFLOAD_NONE: MEMORY_MODE_FAST,
OFFLOAD_GROUP: MEMORY_MODE_BALANCED,
OFFLOAD_MODEL: MEMORY_MODE_LOW_VRAM,
}[policy],
)
def _manual_plan(policy, *, tiling):
"""Build a plan for a policy the auto/explicit modes no longer emit (sequential)."""
return MemoryPlan(
requested_mode = "manual",
offload_policy = policy,
vae_tiling = tiling,
vae_slicing = tiling,
device_memory = _discrete(4000, 8000),
estimates = {},
)
def test_apply_none_places_resident():
pipe = _RecordingPipe()
effective, tiled = apply_memory_plan(pipe, _plan(OFFLOAD_NONE, tiling = False), device = "cuda")
assert pipe.calls == ["to:cuda"]
assert effective == OFFLOAD_NONE and tiled is False
def test_apply_model_offload_engages_offload_and_tiling():
pipe = _RecordingPipe()
effective, tiled = apply_memory_plan(pipe, _plan(OFFLOAD_MODEL, tiling = True), device = "cuda")
assert "model_offload" in pipe.calls
assert "to:cuda" not in pipe.calls
assert "vae_tiling" in pipe.calls and "vae_slicing" in pipe.calls
assert effective == OFFLOAD_MODEL and tiled is True
assert pipe.offload_device == "cuda"
def test_apply_model_offload_passes_target_device():
# enable_model_cpu_offload defaults to CUDA, so XPU and others need the device forwarded.
pipe = _RecordingPipe()
apply_memory_plan(pipe, _plan(OFFLOAD_MODEL, tiling = False), device = "xpu")
assert pipe.offload_device == "xpu"
def test_apply_vae_tiling_falls_back_to_vae_submodule():
# Z-Image-style pipeline: no pipeline-level enable_vae_tiling, only pipe.vae.
class _VaeOnly:
def __init__(self):
self.vae = types.SimpleNamespace(
tiled = False,
sliced = False,
enable_tiling = self._tile,
enable_slicing = self._slice,
)
def _tile(self):
self.vae.tiled = True
def _slice(self):
self.vae.sliced = True
def enable_model_cpu_offload(self, device = None):
self.offloaded = True
pipe = _VaeOnly()
effective, tiled = apply_memory_plan(pipe, _plan(OFFLOAD_MODEL, tiling = True), device = "cuda")
assert tiled is True and pipe.vae.tiled and pipe.vae.sliced
def test_apply_group_falls_back_to_model_without_transformer():
# No .transformer, so group offload cannot engage and the applier falls back to whole-module.
pipe = _RecordingPipe()
effective, _ = apply_memory_plan(pipe, _plan(OFFLOAD_GROUP, tiling = True), device = "cuda")
assert effective == OFFLOAD_MODEL and "model_offload" in pipe.calls
def _install_fake_torch_and_hooks(monkeypatch, apply_group_offloading):
"""Fake torch.nn.Module + diffusers.hooks.apply_group_offloading for _apply_group_offload."""
import sys
class _Mod:
pass
fake_torch = types.ModuleType("torch")
fake_torch.nn = types.SimpleNamespace(Module = _Mod)
fake_torch.device = lambda d: types.SimpleNamespace(type = str(d).split(":")[0])
monkeypatch.setitem(sys.modules, "torch", fake_torch)
if "diffusers" not in sys.modules:
monkeypatch.setitem(sys.modules, "diffusers", types.ModuleType("diffusers"))
fake_hooks = types.ModuleType("diffusers.hooks")
fake_hooks.apply_group_offloading = apply_group_offloading
monkeypatch.setitem(sys.modules, "diffusers.hooks", fake_hooks)
return _Mod
def test_apply_group_partial_hooks_propagates_not_crash_fallback(monkeypatch):
# A partially hooked pipe is rejected by enable_model_cpu_offload, so the failure must propagate.
import core.inference.diffusion_memory as mem
calls = {"n": 0}
def _apply(module, **kw):
calls["n"] += 1
if calls["n"] >= 2:
raise RuntimeError("OOM on second DiT")
Mod = _install_fake_torch_and_hooks(monkeypatch, _apply)
class _DualPipe:
transformer = Mod()
transformer_2 = Mod()
components: dict = {}
with pytest.raises(RuntimeError, match = "OOM on second DiT"):
mem._apply_group_offload(_DualPipe(), "cuda", logger = None)
assert calls["n"] == 2
def test_apply_group_single_transformer_failure_falls_back(monkeypatch):
import core.inference.diffusion_memory as mem
def _apply(module, **kw):
raise RuntimeError("OOM before any hook")
Mod = _install_fake_torch_and_hooks(monkeypatch, _apply)
class _SinglePipe:
transformer = Mod()
components: dict = {}
assert mem._apply_group_offload(_SinglePipe(), "cuda", logger = None) is False
def test_apply_group_fallback_enables_vae_tiling():
# A group plan leaves tiling off; falling back to whole-module offload must turn it on.
plan = _plan(OFFLOAD_GROUP, tiling = True)
assert plan.vae_tiling is False
pipe = _RecordingPipe() # no .transformer -> group offload falls back to model
effective, tiled = apply_memory_plan(pipe, plan, device = "cuda")
assert effective == OFFLOAD_MODEL
assert tiled is True and "vae_tiling" in pipe.calls
def _install_sized_torch(monkeypatch):
import sys
class Tensor:
def __init__(self, size_mib):
self.size_mib = size_mib
def numel(self):
return self.size_mib * 1024 * 1024
def element_size(self):
return 1
class Module:
def __init__(self, size_mib = 0):
self.tensor = Tensor(size_mib)
self.moved_to = None
def parameters(self, recurse = True):
return [self.tensor]
def buffers(self, recurse = True):
return []
def to(self, device):
self.moved_to = device
return self
fake_torch = types.ModuleType("torch")
fake_torch.nn = types.SimpleNamespace(Module = Module)
fake_torch.device = lambda value: types.SimpleNamespace(type = str(value).split(":")[0])
monkeypatch.setitem(sys.modules, "torch", fake_torch)
return Module
def test_refine_model_offload_streams_only_when_a_component_exceeds_budget(monkeypatch):
Module = _install_sized_torch(monkeypatch)
plan = MemoryPlan(
requested_mode = "low_vram",
offload_policy = OFFLOAD_MODEL,
vae_tiling = True,
vae_slicing = True,
device_memory = _discrete(8000),
estimates = {"safe_device_budget_mib": 6000},
)
fitting_transformer = Module(4000)
fits = types.SimpleNamespace(
transformer = fitting_transformer,
components = {"transformer": fitting_transformer, "text_encoder": Module(5500)},
)
assert refine_memory_plan_for_components(fits, plan) is plan
transformer = Module(2500)
oversized = types.SimpleNamespace(
transformer = transformer,
components = {"transformer": transformer, "text_encoder": Module(7500)},
)
refined = refine_memory_plan_for_components(oversized, plan)
assert refined.offload_policy == OFFLOAD_STREAMING
assert refined.estimates["largest_component_mib"] == 7500
assert any("text_encoder" in reason for reason in refined.reasons)
def test_refine_with_headroom_streams_a_component_that_fits_only_without_activations(monkeypatch):
"""Video: a GGUF DiT under the budget by itself but not with the forward's headroom beside it (LTX-2.5 Q4_K_M at a
16 GiB budget OOMed in its first forward under whole-module offload) streams; the bare check keeps the plan."""
Module = _install_sized_torch(monkeypatch)
plan = MemoryPlan(
requested_mode = "auto",
offload_policy = OFFLOAD_MODEL,
vae_tiling = True,
vae_slicing = True,
device_memory = _discrete(16384),
estimates = {
"safe_device_budget_mib": 15000,
"runtime_headroom_mib": 3000,
"base_overhead_mib": 500,
},
)
transformer = Module(13300)
pipe = types.SimpleNamespace(
transformer = transformer, components = {"transformer": transformer, "connectors": Module(6000)}
)
assert refine_memory_plan_for_components(pipe, plan) is plan
refined = refine_memory_plan_for_components(pipe, plan, headroom_mib = 3500)
assert refined.offload_policy == OFFLOAD_STREAMING
assert any("runtime headroom" in reason for reason in refined.reasons)
small = Module(9000)
roomy = types.SimpleNamespace(transformer = small, components = {"transformer": small})
assert refine_memory_plan_for_components(roomy, plan, headroom_mib = 3500) is plan
def test_largest_streamable_companion_mib_measures_only_the_text_encoders(monkeypatch):
from core.inference.diffusion_memory import largest_streamable_companion_mib
Module = _install_sized_torch(monkeypatch)
transformer = Module(9000)
pipe = types.SimpleNamespace(
transformer = transformer,
components = {
"transformer": transformer,
"text_encoder": Module(1200),
"text_encoder_2": Module(4800),
"vae": Module(9500),
},
)
assert largest_streamable_companion_mib(pipe) == 4800
only_dit = types.SimpleNamespace(
transformer = transformer, components = {"transformer": transformer}
)
assert largest_streamable_companion_mib(only_dit) is None
def test_refine_keeps_model_offload_for_a_torchao_transformer(monkeypatch):
"""A torchao transformer's bf16-shaped size must not force streaming."""
Module = _install_sized_torch(monkeypatch)
plan = MemoryPlan(
requested_mode = "low_vram",
offload_policy = OFFLOAD_MODEL,
vae_tiling = True,
vae_slicing = True,
device_memory = _discrete(8000),
estimates = {"safe_device_budget_mib": 6000},
)
transformer = Module(7500)
pipe = types.SimpleNamespace(
transformer = transformer,
components = {"transformer": transformer, "text_encoder": Module(1200)},
)
assert refine_memory_plan_for_components(pipe, plan).offload_policy == OFFLOAD_STREAMING
monkeypatch.setattr(diffusion_memory, "_pipe_denoisers_hold_torchao", lambda pipe: True)
assert refine_memory_plan_for_components(pipe, plan) is plan
def test_refine_keeps_model_offload_when_streaming_cannot_help(monkeypatch):
"""Only a component streaming can actually hook justifies leaving whole-module offload."""
Module = _install_sized_torch(monkeypatch)
plan = MemoryPlan(
requested_mode = "low_vram",
offload_policy = OFFLOAD_MODEL,
vae_tiling = True,
vae_slicing = True,
device_memory = _discrete(8000),
estimates = {"safe_device_budget_mib": 6000},
)
# The oversized component has no granular hook, so streaming would onload it whole anyway.
transformer = Module(2500)
fat_vae = types.SimpleNamespace(
transformer = transformer,
components = {
"transformer": transformer,
"text_encoder": Module(1200),
"vae": Module(7500),
},
)
assert refine_memory_plan_for_components(fat_vae, plan) is plan
# Streaming holds every unstreamed component at once: 3000 + 3500 > 6000.
transformer = Module(2000)
fat_resident = types.SimpleNamespace(
transformer = transformer,
components = {
"transformer": transformer,
"text_encoder": Module(6500),
"vae": Module(3000),
"image_encoder": Module(3500),
},
)
assert refine_memory_plan_for_components(fat_resident, plan) is plan
transformer = Module(2000)
slim_resident = types.SimpleNamespace(
transformer = transformer,
components = {
"transformer": transformer,
"text_encoder": Module(6500),
"vae": Module(300),
"image_encoder": Module(500),
},
)
refined = refine_memory_plan_for_components(slim_resident, plan)
assert refined.offload_policy == OFFLOAD_STREAMING
assert refined.estimates["streaming_resident_mib"] == 800
def test_apply_streaming_uses_block_and_leaf_hooks_with_bounded_cpu_memory(monkeypatch):
Module = _install_sized_torch(monkeypatch)
calls = []
def _apply(
module,
*,
onload_device,
offload_device,
offload_type,
num_blocks_per_group = None,
use_stream = False,
non_blocking = False,
record_stream = True,
low_cpu_mem_usage = False,
):
calls.append(
(
module,
offload_type,
num_blocks_per_group,
use_stream,
non_blocking,
record_stream,
low_cpu_mem_usage,
)
)
import sys
if "diffusers" not in sys.modules:
monkeypatch.setitem(sys.modules, "diffusers", types.ModuleType("diffusers"))
hooks = types.ModuleType("diffusers.hooks")
hooks.apply_group_offloading = _apply
monkeypatch.setitem(sys.modules, "diffusers.hooks", hooks)
transformer = Module(2500)
text_encoder = Module(7500)
vae = Module(200)
pipe = types.SimpleNamespace(
transformer = transformer,
components = {
"transformer": transformer,
"text_encoder": text_encoder,
"vae": vae,
},
)
import core.inference.diffusion_memory as mem
# No pinnable host RAM: streamed modules stay unpinned; record_stream keeps the compute stream undrained.
monkeypatch.delenv(mem.STREAMING_PREFETCH_ENV, raising = False)
monkeypatch.delenv(mem.GROUP_OFFLOAD_PIN_ENV, raising = False)
monkeypatch.setattr(mem, "_pin_budget_mib", lambda: 0)
monkeypatch.setattr(mem, "_pinned_memory_capped", lambda: False)
effective, _ = apply_memory_plan(
pipe, _manual_plan(OFFLOAD_STREAMING, tiling = True), device = "cuda"
)
assert effective == OFFLOAD_STREAMING
assert vae.moved_to.type == "cuda"
assert [(call[1], call[2]) for call in calls] == [
("block_level", DEFAULT_GROUP_BLOCKS),
("leaf_level", None),
]
assert all(call[3:] == (True, True, True, True) for call in calls)
def _streaming_apply_kwargs(
monkeypatch,
budget,
*,
env = None,
request_background = False,
):
"""{component: apply kwargs} and the modules handed to a deferred pinner, for one streaming apply."""
import sys
import core.inference.diffusion_memory as mem
Module = _install_sized_torch(monkeypatch)
seen: dict = {}
deferred: list = []
def _apply(
module,
*,
onload_device,
offload_device,
offload_type,
num_blocks_per_group = None,
use_stream = False,
non_blocking = False,
record_stream = False,
low_cpu_mem_usage = False,
):
seen[module.name] = {
"use_stream": use_stream,
"record_stream": record_stream,
"low_cpu_mem_usage": low_cpu_mem_usage,
"offload_type": offload_type,
}
if "diffusers" not in sys.modules:
monkeypatch.setitem(sys.modules, "diffusers", types.ModuleType("diffusers"))
hooks = types.ModuleType("diffusers.hooks")
hooks.apply_group_offloading = _apply
monkeypatch.setitem(sys.modules, "diffusers.hooks", hooks)
if env is None:
monkeypatch.delenv(mem.STREAMING_PREFETCH_ENV, raising = False)
else:
monkeypatch.setenv(mem.STREAMING_PREFETCH_ENV, env)
monkeypatch.delenv(mem.GROUP_OFFLOAD_PIN_ENV, raising = False)
monkeypatch.delenv(mem.BACKGROUND_PIN_ENV, raising = False)
monkeypatch.setattr(mem, "_pin_budget_mib", lambda: budget)
monkeypatch.setattr(mem, "_pinned_memory_capped", lambda: False)
monkeypatch.setattr(mem, "install_group_pin_wait", lambda: True)
monkeypatch.setattr(
mem,
"_defer_pinning",
lambda pipe, module, device, logger: deferred.append(module.name) or True,
)
parts = {"transformer": Module(4000), "text_encoder": Module(7500), "vae": Module(200)}
for name, module in parts.items():
module.name = name
pipe = types.SimpleNamespace(transformer = parts["transformer"], components = parts)
if request_background:
mem.request_background_pins(pipe)
mem._apply_streaming_offload(pipe, "cuda", None)
return seen, deferred
def test_streaming_pins_the_transformer_within_the_host_budget(monkeypatch):
# Unpinned copies are re-pinned per onload, and record_stream=False syncs the compute stream per group.
seen, deferred = _streaming_apply_kwargs(monkeypatch, 6000)
assert seen["transformer"]["low_cpu_mem_usage"] is False
assert seen["transformer"]["record_stream"] is True
assert seen["text_encoder"]["low_cpu_mem_usage"] is True
assert seen["text_encoder"]["record_stream"] is True
assert deferred == []
def test_streaming_pins_everything_on_a_ram_rich_host(monkeypatch):
seen, _ = _streaming_apply_kwargs(monkeypatch, 40_000)
assert all(kw["low_cpu_mem_usage"] is False for kw in seen.values()), seen
def test_streaming_pins_off_the_load_path_when_asked(monkeypatch):
seen, deferred = _streaming_apply_kwargs(monkeypatch, 6000, request_background = True)
assert seen["transformer"]["low_cpu_mem_usage"] is True
assert seen["transformer"]["record_stream"] is True
assert deferred == ["transformer"]
def test_streaming_prefetch_kill_switch_restores_the_old_kwargs(monkeypatch):
seen, deferred = _streaming_apply_kwargs(monkeypatch, 40_000, env = "0", request_background = True)
assert all(
kw["low_cpu_mem_usage"] is True and kw["record_stream"] is False for kw in seen.values()
), seen
assert deferred == []
def test_apply_streaming_does_not_fall_back_to_known_oom_path(monkeypatch):
import core.inference.diffusion_memory as mem
def _fail(*args, **kwargs):
raise RuntimeError("granular streaming offload could not be enabled")
monkeypatch.setattr(mem, "_apply_streaming_offload", _fail)
pipe = _RecordingPipe()
with pytest.raises(RuntimeError, match = "granular streaming offload could not be enabled"):
apply_memory_plan(pipe, _manual_plan(OFFLOAD_STREAMING, tiling = True), device = "cuda")
assert "model_offload" not in pipe.calls
def test_apply_sequential_offload():
pipe = _RecordingPipe()
effective, _ = apply_memory_plan(
pipe, _manual_plan(OFFLOAD_SEQUENTIAL, tiling = True), device = "cuda"
)
assert "sequential_offload" in pipe.calls and "to:cuda" not in pipe.calls
assert effective == OFFLOAD_SEQUENTIAL
assert pipe.offload_device == "cuda"
def test_apply_sequential_falls_back_to_model_offload_when_unsupported():
# Sequential offload is unreliable for GGUF on some diffusers versions.
class _NoSeqPipe(_RecordingPipe):
def enable_sequential_cpu_offload(self, device = None):
raise RuntimeError("sequential offload not supported for this transformer")
pipe = _NoSeqPipe()
effective, _ = apply_memory_plan(
pipe, _manual_plan(OFFLOAD_SEQUENTIAL, tiling = True), device = "cuda"
)
assert effective == OFFLOAD_MODEL
assert "model_offload" in pipe.calls
def test_apply_tolerates_pipe_without_vae_savers():
class _Bare:
def __init__(self):
self.moved = None
def to(self, device):
self.moved = device
bare = _Bare()
_, tiled = apply_memory_plan(bare, _plan(OFFLOAD_NONE, tiling = False), device = "cpu")
assert bare.moved == "cpu" and tiled is False
def test_settled_snapshot_takes_max_free_over_reads(monkeypatch):
# A transient foreign allocation can only shrink free, so keep the max across reads.
from core.inference import diffusion_memory as dm
reads = [
DeviceMemory("cuda", "cuda", "discrete_vram", free_mib = 60_000, total_mib = 183_359),
DeviceMemory("cuda", "cuda", "discrete_vram", free_mib = 170_000, total_mib = 183_359),
DeviceMemory("cuda", "cuda", "discrete_vram", free_mib = 170_000, total_mib = 183_359),
]
monkeypatch.setattr(dm, "snapshot_device_memory", lambda target: reads.pop(0))
snap = dm.settled_snapshot_device_memory(_target(device = "cuda"), attempts = 3, delay_s = 0)
assert snap.free_mib == 170_000
def test_settled_snapshot_stops_early_when_device_already_idle(monkeypatch):
from core.inference import diffusion_memory as dm
calls = []
def fake_snapshot(target):
calls.append(1)
return DeviceMemory("cuda", "cuda", "discrete_vram", free_mib = 170_000, total_mib = 183_359)
monkeypatch.setattr(dm, "snapshot_device_memory", fake_snapshot)
snap = dm.settled_snapshot_device_memory(_target(device = "cuda"), attempts = 3, delay_s = 0)
assert snap.free_mib == 170_000
assert calls == [1]
def _fake_torch_allocator(monkeypatch, *, reserved: int, allocated: int):
"""Stub torch.cuda's allocator counters for the reclaimable snapshot."""
import sys
torch = types.ModuleType("torch")
torch.cuda = types.SimpleNamespace(
memory_reserved = lambda: reserved,
memory_allocated = lambda: allocated,
)
monkeypatch.setitem(sys.modules, "torch", torch)
def test_reclaimable_snapshot_credits_cached_blocks_without_flushing(monkeypatch):
# mem_get_info counts the caching allocator's reusable blocks as used, so a warm card looks full.
# The guard must not mistake that for a shortfall or pay empty_cache() per image.
from core.inference import diffusion_memory as dm
monkeypatch.setattr(
dm,
"snapshot_device_memory",
lambda target: DeviceMemory(
"cuda", "cuda", "discrete_vram", free_mib = 2_000, total_mib = 16_302
),
)
_fake_torch_allocator(monkeypatch, reserved = 6 * 1024**3, allocated = 2 * 1024**3)
snap = dm.reclaimable_snapshot_device_memory(_target(device = "cuda"))
assert snap.free_mib == 2_000 + 4 * 1024
def test_reclaimable_snapshot_does_not_credit_a_live_cuda_graph_pool(monkeypatch):
# A captured graph's pool is reserved but unusable by others while it lives, so it counts as used.
from core.inference import diffusion_cuda_graph as cg
from core.inference import diffusion_memory as dm
monkeypatch.setattr(
dm,
"snapshot_device_memory",
lambda target: DeviceMemory(
"cuda", "cuda", "discrete_vram", free_mib = 2_000, total_mib = 16_302
),
)
_fake_torch_allocator(monkeypatch, reserved = 6 * 1024**3, allocated = 2 * 1024**3)
monkeypatch.setattr(cg, "live_pool_free_bytes", lambda: 3 * 1024**3)
assert dm.reclaimable_snapshot_device_memory(_target(device = "cuda")).free_mib == 2_000 + 1024
monkeypatch.setattr(cg, "live_pool_free_bytes", lambda: 5 * 1024**3)
assert dm.reclaimable_snapshot_device_memory(_target(device = "cuda")).free_mib == 2_000
def test_reclaimable_snapshot_never_claims_more_than_the_card(monkeypatch):
# The credit is arithmetic: a bogus allocator reading must not invent memory.
from core.inference import diffusion_memory as dm
monkeypatch.setattr(
dm,
"snapshot_device_memory",
lambda target: DeviceMemory(
"cuda", "cuda", "discrete_vram", free_mib = 15_000, total_mib = 16_302
),
)
_fake_torch_allocator(monkeypatch, reserved = 40 * 1024**3, allocated = 0)
assert dm.reclaimable_snapshot_device_memory(_target(device = "cuda")).free_mib == 16_302
def test_reclaimable_snapshot_falls_back_when_the_allocator_is_unreadable(monkeypatch):
from core.inference import diffusion_memory as dm
import sys
monkeypatch.setattr(
dm,
"snapshot_device_memory",
lambda target: DeviceMemory(
"cuda", "cuda", "discrete_vram", free_mib = 2_000, total_mib = 16_302
),
)
torch = types.ModuleType("torch")
def _boom():
raise RuntimeError("no cuda context")
torch.cuda = types.SimpleNamespace(memory_reserved = _boom, memory_allocated = _boom)
monkeypatch.setitem(sys.modules, "torch", torch)
assert dm.reclaimable_snapshot_device_memory(_target(device = "cuda")).free_mib == 2_000
def test_reclaimable_snapshot_passthrough_off_cuda(monkeypatch):
from core.inference import diffusion_memory as dm
monkeypatch.setattr(
dm,
"snapshot_device_memory",
lambda target: DeviceMemory(
"mps", "mps", "unified_memory", free_mib = 8_000, total_mib = 16_000
),
)
_fake_torch_allocator(monkeypatch, reserved = 6 * 1024**3, allocated = 0)
assert dm.reclaimable_snapshot_device_memory(_target(device = "mps")).free_mib == 8_000
def test_settled_snapshot_passthrough_off_cuda(monkeypatch):
from core.inference import diffusion_memory as dm
calls = []
def fake_snapshot(target):
calls.append(1)
return DeviceMemory("mps", "mps", "unified_memory", free_mib = 8_000, total_mib = 16_000)
monkeypatch.setattr(dm, "snapshot_device_memory", fake_snapshot)
snap = dm.settled_snapshot_device_memory(_target(device = "mps"), attempts = 3, delay_s = 0)
assert snap.memory_kind == "unified_memory"
assert calls == [1]
def test_plan_fits_total_capacity():
# True when required fits (total - reserve) * 0.85: only then can a settled retry help.
from core.inference.diffusion_memory import plan_fits_total_capacity
def plan(
required,
total,
kind = "discrete_vram",
):
return types.SimpleNamespace(
estimates = {"resident_required_mib": required},
device_memory = DeviceMemory("cuda", "cuda", kind, free_mib = 1, total_mib = total),
)
assert plan_fits_total_capacity(plan(90_228, 183_359)) is True
# 0.85 * (183,359 - 4,096 capped reserve) = 152,373
assert plan_fits_total_capacity(plan(160_000, 183_359)) is False
assert plan_fits_total_capacity(plan(None, 183_359)) is False
assert plan_fits_total_capacity(plan(90_228, None)) is False
assert plan_fits_total_capacity(types.SimpleNamespace()) is False
# Apple Silicon shares one pool and PYTORCH_MPS_HIGH_WATERMARK_RATIO=0.0 lifts the allocator limit,
# so an oversized load is SIGKILLed by the OS; the load-time refusal replaces that with a message.
_MPS_TOTAL_MIB = 16 * 1024
_MPS_FREE_MIB = int(_MPS_TOTAL_MIB * 0.80)
def _unified_plan(
*,
model_dense_mib,
runtime_headroom_mib = 3072,
free_mib = _MPS_FREE_MIB,
total_mib = _MPS_TOTAL_MIB,
kind = "unified_memory",
device = "mps",
):
"""Built by the shipped planner, so the tests exercise real budget arithmetic, not a copy."""
return plan_diffusion_memory(
target = _target(device = device, backend = device, supports_offload = False),
device_memory = DeviceMemory(device, device, kind, free_mib, total_mib),
model_dense_mib = model_dense_mib,
runtime_headroom_mib = runtime_headroom_mib,
)
def test_mps_plan_is_tight_past_the_resident_margin_only():
from core.inference.diffusion_memory import mps_plan_tight
budget = _unified_plan(model_dense_mib = 1000).estimates["safe_device_budget_mib"]
# resident_required = weights + the 3072 runtime headroom + the 2048 base overhead
at_margin = int(budget * 0.85) - 3072 - 2048
assert not mps_plan_tight(_unified_plan(model_dense_mib = at_margin))
assert mps_plan_tight(_unified_plan(model_dense_mib = at_margin + 1))
# far past their budgets, but only MPS changes: discrete VRAM, an integrated GPU and the CPU keep their rules
for kind, device in (
("discrete_vram", "cuda"),
("unified_memory", "cuda"),
("system_memory", "cpu"),
):
other = _unified_plan(model_dense_mib = 4 * budget, kind = kind, device = device)
assert other.estimates["resident_required_mib"] > other.estimates["safe_device_budget_mib"]
assert not mps_plan_tight(other), (kind, device)
assert not mps_plan_tight(_unified_plan(model_dense_mib = at_margin + 1, free_mib = None))
assert not mps_plan_tight(None)
def test_unified_oversize_refuses_and_names_family_and_both_numbers():
from core.inference.diffusion_memory import unified_memory_shortfall_message
# 16 GiB Mac, 12.8 GiB free, 20% unified reserve: ~9.5 GiB budget.
plan = _unified_plan(model_dense_mib = 24 * 1024)
assert plan.offload_policy == OFFLOAD_NONE
message = unified_memory_shortfall_message(plan, family = "wan2.2-ti2v-5b")
assert message is not None
assert "wan2.2-ti2v-5b" in message
assert "about 26 GB of memory for its weights" in message # 24 GiB weights + 2 GiB overhead
assert "about 10 GB is usable" in message # 12.8 GiB free, less 20% of 16 GiB
assert "13 GB currently free" in message
assert "smaller or more quantized model" in message
assert "UNSLOTH_DIFFUSION_ALLOW_OVERSIZED_LOAD=1" in message
def test_unified_oversize_ignores_the_soft_runtime_headroom():
"""Refusal budgets weights only; VAE tiling cuts the decode peak, so marginal loads are not refused."""
from core.inference.diffusion_memory import unified_memory_shortfall_message
budget = _MPS_FREE_MIB - max(2048, int(_MPS_TOTAL_MIB * 0.20))
weights = budget - 2048 # weights + the 2048 base overhead land exactly on the budget
plan = _unified_plan(model_dense_mib = weights, runtime_headroom_mib = 8192)
assert plan.estimates["resident_required_mib"] > budget # refused if headroom counted
assert unified_memory_shortfall_message(plan) is None
assert unified_memory_shortfall_message(_unified_plan(model_dense_mib = weights + 1)) is not None
def test_unified_oversize_never_refuses_discrete_vram():
"""Discrete VRAM keeps its fallback ladder: an oversized model streams from host RAM under
group / whole-module offload, so refusing it would break a load that works today."""
from core.inference.diffusion_memory import unified_memory_shortfall_message
plan = plan_diffusion_memory(
target = _target(device = "cuda", backend = "cuda", supports_offload = True),
device_memory = DeviceMemory("cuda", "cuda", "discrete_vram", 12_000, 16_384),
model_dense_mib = 80 * 1024,
runtime_headroom_mib = 6963,
)
assert plan.offload_policy == OFFLOAD_MODEL
assert unified_memory_shortfall_message(plan, family = "ltx-2") is None
def test_unified_oversize_never_refuses_plain_cpu_system_memory():
# A CPU target has swap and is an opt-in fringe path: unchanged.
from core.inference.diffusion_memory import unified_memory_shortfall_message
plan = _unified_plan(model_dense_mib = 80 * 1024, kind = "system_memory", device = "cpu")
assert unified_memory_shortfall_message(plan) is None
@pytest.mark.parametrize(
"kwargs",
[
{"free_mib": None},
{"model_dense_mib": None},
],
)
def test_unified_oversize_fails_open_on_unknown_inputs(kwargs):
"""Matches the planner's own "budget or model size unknown; staying resident": never turn a
failed probe into a refusal."""
from core.inference.diffusion_memory import unified_memory_shortfall_message
kwargs = {"model_dense_mib": 80 * 1024, **kwargs}
assert unified_memory_shortfall_message(_unified_plan(**kwargs)) is None
def test_unified_oversize_env_override_attempts_the_load_anyway(monkeypatch):
from core.inference.diffusion_memory import (
UNIFIED_OVERSIZE_ENV,
raise_on_unified_memory_shortfall,
unified_memory_shortfall_message,
)
plan = _unified_plan(model_dense_mib = 80 * 1024)
assert unified_memory_shortfall_message(plan) is not None
for value in ("1", "true", "YES", "on"):
monkeypatch.setenv(UNIFIED_OVERSIZE_ENV, value)
assert unified_memory_shortfall_message(plan) is None
raise_on_unified_memory_shortfall(plan)
monkeypatch.setenv(UNIFIED_OVERSIZE_ENV, "0")
assert unified_memory_shortfall_message(plan) is not None
def test_unified_oversize_message_survives_a_malformed_plan():
from core.inference.diffusion_memory import unified_memory_shortfall_message
assert unified_memory_shortfall_message(types.SimpleNamespace()) is None
def test_raise_on_unified_memory_shortfall_raises_runtime_error_with_the_message():
"""RuntimeError, not a 409: loader threads' _run_load copies it to load_progress for the UI toast."""
from core.inference.diffusion_memory import (
raise_on_unified_memory_shortfall,
unified_memory_shortfall_message,
)
plan = _unified_plan(model_dense_mib = 80 * 1024)
with pytest.raises(RuntimeError) as excinfo:
raise_on_unified_memory_shortfall(plan, family = "ltx-2")
assert str(excinfo.value) == unified_memory_shortfall_message(plan, family = "ltx-2")
raise_on_unified_memory_shortfall(_unified_plan(model_dense_mib = 1024))
def test_unified_oversize_decision_matrix_for_the_real_video_families():
"""The video family tables must refuse oversize models exactly where weights cannot fit on Mac RAM."""
from core.inference.video_families import _FAMILIES
from core.inference.diffusion_memory import (
DEFAULT_BASE_OVERHEAD_MIB,
estimate_video_runtime_mib,
unified_memory_shortfall_message,
)
mib_per_gb = 1000.0**3 / (1024.0 * 1024.0) # the tables are DECIMAL GB
# family: RAM sizes (GiB) at which the load must be refused.
expected_refusals = {
# 144.2 GB bf16: refused on every Mac here by design; users reach H3 via GGUF or prequant artifacts.
"minimax-h3": {16, 24, 32, 64, 96, 128},
"ltx-2": {16, 24, 32, 64, 96},
# 75.1 GB of bf16 weights (the Gemma 4 encoder counted, though it lives in its worker process).
"ltx-2.5": {16, 24, 32, 64, 96},
"wan2.2-ti2v-5b": {16, 24, 32},
"wan2.2-t2v-a14b": {16, 24, 32, 64, 96},
# Wan 1.3B (14.7 GB) misses a 24 GiB Mac's ~14.4 GiB budget; fits from 32 GiB up.
"wan2.1-t2v-1.3b": {16, 24},
# 40.5 GB (one 14B DiT, same encoder + VAE): plus overhead it just misses the ~38.4 GiB budget
# at 64 GiB, and fits at 96 GiB and above (unlike the two-expert A14B at 69.1 GB).
"wan2.1-t2v-14b": {16, 24, 32, 64},
"hunyuanvideo-1.5": {16, 24, 32},
"hunyuanvideo-1.5-720p": {16, 24, 32},
"cosmos3-nano": {16, 24, 32},
"cosmos3-edge": set(),
}
assert {f.name for f in _FAMILIES} == set(
expected_refusals
), "a video family was added or renamed: extend the expected refusal matrix"
for fam in _FAMILIES:
width, height = fam.resolution_presets[0]
dense = int(sum(fam.bf16_components_gb) * mib_per_gb)
headroom = estimate_video_runtime_mib(
width = width, height = height, num_frames = fam.default_num_frames
)
for ram_gib in (16, 24, 32, 64, 96, 128):
total = ram_gib * 1024
plan = _unified_plan(
model_dense_mib = dense,
runtime_headroom_mib = headroom,
free_mib = int(total * 0.80),
total_mib = total,
)
assert plan.offload_policy == OFFLOAD_NONE
refused = unified_memory_shortfall_message(plan, family = fam.name) is not None
assert refused is (ram_gib in expected_refusals[fam.name]), (
f"{fam.name} at {ram_gib} GiB: refused={refused}, "
f"weights+overhead={dense + DEFAULT_BASE_OVERHEAD_MIB} MiB, "
f"budget={plan.estimates['safe_device_budget_mib']} MiB"
)
def test_unified_memory_policy_cannot_express_a_misfit():
"""Why the dense-quant fast path needed its own explicit check: on unified memory the planner
returns OFFLOAD_NONE for ANY size (offload just shuffles bytes within one pool), so the
"policy is not none" test the dense re-plan relied on can never fire there."""
from core.inference.diffusion_memory import (
DeviceMemory,
OFFLOAD_NONE,
plan_diffusion_memory,
unified_memory_shortfall_message,
)
target = type("T", (), {"supports_model_cpu_offload": True, "device": "cuda"})()
memory = DeviceMemory("cuda", "cuda", "unified_memory", 8_000, 16_000)
plan = plan_diffusion_memory(
target = target,
device_memory = memory,
model_dense_mib = 90_000,
runtime_headroom_mib = 1_000,
)
assert plan.offload_policy == OFFLOAD_NONE
assert unified_memory_shortfall_message(plan, family = "flux.1") is not None
# Text encoders run once before step 0, yet group offload keeps them resident all denoise;
# streaming them keeps the group tier instead of dropping to whole-module offload.
# Free 15,870 of 16,305 MiB, reserve max(2048, 10%) = 2048: budget is exactly 13,822 MiB.
_16G_FREE_MIB = 15_870
_16G_TOTAL_MIB = 16_305
_16G_BUDGET_MIB = 13_822
# Z-Image int8: transformer 6451 + companions 7820 (text encoders 7629, VAE the rest).
_ZIMAGE_MODEL_DENSE_MIB = 14_271
_ZIMAGE_COMPANION_MIB = 7_820
_ZIMAGE_TEXT_ENCODER_MIB = 7_629
def test_safe_budget_matches_the_reported_16g_card():
# Anchors every number below: fails first if the reserve rule changes.
from core.inference.diffusion_memory import _safe_device_budget_mib
assert _safe_device_budget_mib(_discrete(_16G_FREE_MIB, _16G_TOTAL_MIB)) == _16G_BUDGET_MIB
def _zimage_plan(text_encoder_dense_mib):
return plan_diffusion_memory(
target = _target(),
device_memory = _discrete(_16G_FREE_MIB, _16G_TOTAL_MIB),
model_dense_mib = _ZIMAGE_MODEL_DENSE_MIB,
companion_dense_mib = _ZIMAGE_COMPANION_MIB,
text_encoder_dense_mib = text_encoder_dense_mib,
runtime_headroom_mib = 8192,
base_overhead_mib = 2048,
)
def test_streamed_text_encoders_rescue_the_48_minute_plan():
# Group floor 7820 + 8192 + 2048 = 18,060 > 13,822; streamed encoders give 10,431, which fits.
before = _zimage_plan(None)
assert before.offload_policy == OFFLOAD_MODEL
assert before.stream_text_encoders is False
assert before.estimates["group_floor_streamed_te_mib"] is None
after = _zimage_plan(_ZIMAGE_TEXT_ENCODER_MIB)
assert after.offload_policy == OFFLOAD_GROUP
assert after.stream_text_encoders is True
assert after.estimates["resident_required_mib"] == 24_511
assert after.estimates["group_floor_mib"] == 18_060
assert after.estimates["group_floor_streamed_te_mib"] == 10_431
assert any("text encoders" in reason for reason in after.reasons)
assert after.vae_slicing is True and after.vae_tiling is False
# The applier and status/logging read the public dict.
assert after.as_public_dict()["stream_text_encoders"] is True
@pytest.mark.parametrize("mode", [None, MEMORY_MODE_FAST])
def test_plain_group_is_preferred_over_streaming_the_text_encoders(mode):
# Tier must only rescue whole-module offload; auto and fast reach it by separate branches.
plan = plan_diffusion_memory(
target = _target(),
device_memory = _discrete(_16G_FREE_MIB, _16G_TOTAL_MIB),
model_dense_mib = 40_000,
companion_dense_mib = 1_500,
text_encoder_dense_mib = 1_400,
runtime_headroom_mib = 8192,
base_overhead_mib = 2048,
requested_mode = mode,
)
assert plan.offload_policy == OFFLOAD_GROUP
assert plan.stream_text_encoders is False
def test_streamed_text_encoders_still_fall_through_when_even_that_floor_is_over():
plan = plan_diffusion_memory(
target = _target(),
device_memory = _discrete(_16G_FREE_MIB, _16G_TOTAL_MIB),
model_dense_mib = 40_000,
companion_dense_mib = 20_000,
text_encoder_dense_mib = 1_000,
runtime_headroom_mib = 8192,
base_overhead_mib = 2048,
)
assert plan.offload_policy == OFFLOAD_MODEL
assert plan.stream_text_encoders is False
def test_fast_mode_also_reaches_the_streamed_text_encoder_tier():
# fast has its own does-not-fit branch; it must offer the same ladder as auto.
plan = plan_diffusion_memory(
target = _target(),
device_memory = _discrete(_16G_FREE_MIB, _16G_TOTAL_MIB),
model_dense_mib = _ZIMAGE_MODEL_DENSE_MIB,
companion_dense_mib = _ZIMAGE_COMPANION_MIB,
text_encoder_dense_mib = _ZIMAGE_TEXT_ENCODER_MIB,
runtime_headroom_mib = 8192,
base_overhead_mib = 2048,
requested_mode = MEMORY_MODE_FAST,
)
assert plan.offload_policy == OFFLOAD_GROUP
assert plan.stream_text_encoders is True
def test_text_encoder_split_larger_than_the_companions_clamps_at_zero():
# The terms come from different sources, so a split exceeding the total must floor at 0.
plan = plan_diffusion_memory(
target = _target(),
device_memory = _discrete(_16G_FREE_MIB, _16G_TOTAL_MIB),
model_dense_mib = 40_000,
companion_dense_mib = 7_820,
text_encoder_dense_mib = 9_999,
runtime_headroom_mib = 8192,
base_overhead_mib = 2048,
)
assert plan.estimates["group_floor_streamed_te_mib"] == 10_240 # 0 + 8192 + 2048
def _legacy_offload_policy(
*, budget, model_dense_mib, companion_dense_mib, runtime_headroom_mib, base_overhead_mib
):
"""Pre-streamed-encoder auto policy, kept separate so back-compat tests catch new-tier leaks."""
required = model_dense_mib + runtime_headroom_mib + base_overhead_mib
if required <= int(budget * 0.85):
return OFFLOAD_NONE
if companion_dense_mib is None:
return OFFLOAD_MODEL
group_floor = companion_dense_mib + runtime_headroom_mib + base_overhead_mib
return OFFLOAD_GROUP if group_floor <= budget else OFFLOAD_MODEL
def test_no_text_encoder_split_reproduces_the_previous_decision():
# Back-compat: with no split the planner must land exactly where it did before.
from core.inference.diffusion_memory import _safe_device_budget_mib
for free, total in ((6_000, 8_192), (11_000, 12_288), (15_870, 16_305), (80_000, 81_920)):
for model_dense in (2_000, 14_271, 40_000):
for companion in (None, 200, 7_820, 30_000):
for headroom in (1_000, 8_192):
memory = _discrete(free, total)
plan = plan_diffusion_memory(
target = _target(),
device_memory = memory,
model_dense_mib = model_dense,
companion_dense_mib = companion,
runtime_headroom_mib = headroom,
base_overhead_mib = 2048,
)
expected = _legacy_offload_policy(
budget = _safe_device_budget_mib(memory),
model_dense_mib = model_dense,
companion_dense_mib = companion,
runtime_headroom_mib = headroom,
base_overhead_mib = 2048,
)
where = (free, total, model_dense, companion, headroom)
assert plan.offload_policy == expected, where
assert plan.stream_text_encoders is False, where
assert plan.estimates["group_floor_streamed_te_mib"] is None, where
def _stream_te_pipe(monkeypatch):
"""Two-encoder pipe that records which modules got group-offload hooks or stayed resident."""
applied: list[Any] = []
Mod = _install_fake_torch_and_hooks(monkeypatch, lambda module, **kw: applied.append(module))
class _Comp(Mod):
def __init__(self, name):
self.name = name
self.placed = None
def to(self, device):
self.placed = device
return self
transformer = _Comp("transformer")
text_encoder = _Comp("text_encoder")
text_encoder_2 = _Comp("text_encoder_2")
vae = _Comp("vae")
class _Pipe:
pass
pipe = _Pipe()
pipe.transformer = transformer
pipe.components = {
"transformer": transformer,
"text_encoder": text_encoder,
"text_encoder_2": text_encoder_2,
"vae": vae,
}
return pipe, applied, transformer, text_encoder, text_encoder_2, vae
def test_apply_group_offload_leaves_text_encoders_resident_by_default(monkeypatch):
import core.inference.diffusion_memory as mem
pipe, applied, transformer, te, te2, vae = _stream_te_pipe(monkeypatch)
assert mem._apply_group_offload(pipe, "cuda", logger = None) is True
assert applied == [transformer]
assert te.placed is not None and te2.placed is not None and vae.placed is not None
def test_apply_group_offload_streams_text_encoders_when_asked(monkeypatch):
# Placing text encoders resident would defeat the tier: their bytes are what did not fit.
import core.inference.diffusion_memory as mem
pipe, applied, transformer, te, te2, vae = _stream_te_pipe(monkeypatch)
assert mem._apply_group_offload(pipe, "cuda", logger = None, stream_text_encoders = True) is True
assert applied == [transformer, te, te2]
assert te.placed is None and te2.placed is None
assert vae.placed is not None
def _stream_te_kwargs(monkeypatch, **call_kw):
"""Use a signature-complete fake so the group-offload kwargs gate is actually exercised."""
seen: dict = {}
def _apply(
module,
onload_device = None,
offload_device = None,
offload_type = None,
num_blocks_per_group = None,
non_blocking = False,
use_stream = False,
record_stream = False,
low_cpu_mem_usage = False,
**_,
):
seen[getattr(module, "name", "?")] = dict(
use_stream = use_stream,
non_blocking = non_blocking,
record_stream = record_stream,
low_cpu_mem_usage = low_cpu_mem_usage,
offload_type = offload_type,
num_blocks_per_group = num_blocks_per_group,
)
import core.inference.diffusion_memory as mem
pipe, _applied, *_ = _stream_te_pipe(monkeypatch)
# Swap, don't re-install: a second install mints a new Module class, breaking isinstance.
_swap_group_offloading(monkeypatch, _apply)
assert mem._apply_group_offload(pipe, "cuda", logger = None, **call_kw) is True
return seen
def test_the_split_leaves_the_weights_only_estimate_terms_untouched():
# The unified refusal uses model_dense_mib, base_overhead_mib and safe_device_budget_mib;
# the split must not move any of them.
without = _zimage_plan(None).estimates
with_split = _zimage_plan(_ZIMAGE_TEXT_ENCODER_MIB).estimates
for key in ("safe_device_budget_mib", "model_dense_mib", "base_overhead_mib"):
assert without[key] == with_split[key], key
assert without["group_floor_mib"] == with_split["group_floor_mib"]
assert without["text_encoder_dense_mib"] is None
assert with_split["text_encoder_dense_mib"] == _ZIMAGE_TEXT_ENCODER_MIB
assert with_split["group_floor_streamed_te_mib"] < with_split["group_floor_mib"]
def test_streaming_the_text_encoders_does_not_pin_host_memory(monkeypatch):
# Hosts may lack RAM, so streamed text encoders must not be pinned.
import core.inference.diffusion_memory as mem
monkeypatch.delenv(mem.GROUP_OFFLOAD_PIN_ENV, raising = False)
monkeypatch.setattr(mem, "_pin_budget_mib", lambda: 0)
seen = _stream_te_kwargs(monkeypatch, stream_text_encoders = True)
assert seen, "the applier never reached diffusers"
assert all(kw["low_cpu_mem_usage"] is True for kw in seen.values()), seen
def test_the_unchanged_group_tier_still_pins_for_speed(monkeypatch):
# The existing group plan was chosen because companions fit, so pinning stays the default.
seen = _stream_te_kwargs(monkeypatch)
assert seen, "the applier never reached diffusers"
assert all(kw["low_cpu_mem_usage"] is False for kw in seen.values()), seen
assert all(kw["use_stream"] and kw["non_blocking"] for kw in seen.values()), seen
def _swap_group_offloading(monkeypatch, apply_group_offloading):
"""Swap apply_group_offloading in place; reinstalling the fake mints a new Module class."""
import sys
monkeypatch.setattr(
sys.modules["diffusers.hooks"], "apply_group_offloading", apply_group_offloading
)
def test_a_text_encoder_that_refuses_group_offload_stays_resident(monkeypatch):
# By the encoder stage the transformer is hooked and whole-module fallback is gone,
# so an encoder offload refusal must degrade to resident, not fail the load.
import core.inference.diffusion_memory as mem
applied: list[Any] = []
def _apply(module, **kw):
if getattr(module, "name", "").startswith("text_encoder"):
raise ValueError("no block list on this text encoder")
applied.append(module)
pipe, _unused, transformer, te, te2, vae = _stream_te_pipe(monkeypatch)
_swap_group_offloading(monkeypatch, _apply)
assert mem._apply_group_offload(pipe, "cuda", logger = None, stream_text_encoders = True) is True
assert applied == [transformer]
assert te.placed is not None and te2.placed is not None
assert vae.placed is not None
def test_one_refusing_text_encoder_does_not_cost_the_other_its_streaming(monkeypatch):
# Each encoder is decided alone, so one refusing encoder must not revert the whole rescue.
import core.inference.diffusion_memory as mem
applied: list[Any] = []
def _apply(module, **kw):
if getattr(module, "name", "") == "text_encoder":
raise ValueError("no block list on this text encoder")
applied.append(module)
pipe, _unused, transformer, te, te2, _vae = _stream_te_pipe(monkeypatch)
_swap_group_offloading(monkeypatch, _apply)
assert mem._apply_group_offload(pipe, "cuda", logger = None, stream_text_encoders = True) is True
assert applied == [transformer, te2]
assert te.placed is not None
assert te2.placed is None
def test_a_failing_dit_keeps_its_all_or_nothing_semantics(monkeypatch):
# Encoder tolerance only: a DiT failing after another hooked still propagates (partial denoiser).
import core.inference.diffusion_memory as mem
calls = {"n": 0}
def _apply(module, **kw):
calls["n"] += 1
if calls["n"] >= 2:
raise RuntimeError("OOM on the second DiT")
Mod = _install_fake_torch_and_hooks(monkeypatch, _apply)
class _DualPipe:
pass
pipe = _DualPipe()
pipe.transformer = Mod()
pipe.transformer_2 = Mod()
pipe.components = {"transformer": pipe.transformer, "transformer_2": pipe.transformer_2}
with pytest.raises(RuntimeError, match = "OOM on the second DiT"):
mem._apply_group_offload(pipe, "cuda", logger = None, stream_text_encoders = True)
def test_apply_memory_plan_threads_the_stream_flag_to_the_group_applier(monkeypatch):
# The flag must reach _apply_group_offload, else the encoders are placed resident and OOM.
import core.inference.diffusion_memory as mem
seen = {}
def _fake(
pipe,
device,
logger,
*,
stream_text_encoders = False,
):
seen["stream_text_encoders"] = stream_text_encoders
return True
monkeypatch.setattr(mem, "_apply_group_offload", _fake)
apply_memory_plan(_RecordingPipe(), _zimage_plan(_ZIMAGE_TEXT_ENCODER_MIB), device = "cuda")
assert seen["stream_text_encoders"] is True
apply_memory_plan(_RecordingPipe(), _plan(OFFLOAD_GROUP, tiling = True), device = "cuda")
assert seen["stream_text_encoders"] is False
# Load-time plans budget 1024x1024; larger frames OOM on Linux, and on Windows WDDM spill
# into system RAM silently and freeze the desktop. This guard refuses before sampling.
# The base repo carries the distilled marker, so the estimate is the discounted one (0.85).
_TURBO_HINT = (
"z-image Z-Image-Turbo-Q4_K_S.gguf unsloth/Z-Image-Turbo-GGUF Tongyi-MAI/Z-Image-Turbo"
)
def _shortfall(
width,
height,
memory = None,
**kw,
):
from core.inference.diffusion_memory import image_activation_shortfall_message
return image_activation_shortfall_message(
device_memory = memory if memory is not None else _discrete(_16G_FREE_MIB, _16G_TOTAL_MIB),
width = width,
height = height,
family = kw.pop("family", _TURBO_HINT),
**kw,
)
def test_estimate_image_runtime_scales_with_the_real_dimensions():
# 1088x1920 is 1.99x the area of 1024x1024, so the headroom must be ~2x.
base = estimate_image_runtime_mib(width = 1024, height = 1024)
tall = estimate_image_runtime_mib(width = 1088, height = 1920)
assert base == 8192
assert tall == 16_320
assert 1.95 < tall / base < 2.05
assert estimate_image_runtime_mib(width = 1088, height = 1920, family = _TURBO_HINT) == 13_872
def test_guard_refuses_the_oversized_frame_and_passes_the_default_one():
# 13,872 MiB of working memory against a 13,822 MiB budget: refuse.
message = _shortfall(1088, 1920)
assert message is not None
assert "1088x1920" in message
# Quote the total compared (activations + 2,048 MiB overhead), not activations alone.
assert "15.55 GB" in message # needed, overhead included
assert "2.00 GB of fixed overhead" in message
assert "13.50 GB" in message
assert "15.50 GB" in message
assert "smaller resolution" in message
assert "UNSLOTH_DIFFUSION_ALLOW_OVERSIZED_GENERATE" in message
assert _shortfall(1024, 1024) is None
def test_guard_counts_the_base_overhead_alongside_the_activations():
# Base overhead shares the pass's tensors; omitting it lets 13,872 MiB pass unflagged.
from core.inference.diffusion_memory import _safe_device_budget_mib
reported_card = _discrete(16_302, 16_302)
assert _safe_device_budget_mib(reported_card) == 14_254
assert _shortfall(1088, 1920, memory = reported_card) is not None
# Zeroing the overhead makes it silent again, so the term is load-bearing.
assert _shortfall(1088, 1920, memory = reported_card, base_overhead_mib = 0) is None
def test_guard_never_refuses_at_or_below_the_resolution_the_load_planned_for():
# Flat headroom is a planning figure, so the guard acts only above the planned resolution.
small = _discrete(
int(8 * 1024 * 0.97), 8 * 1024
) # safe budget 5898 MiB, under the 6963 default
assert _shortfall(1024, 1024, memory = small) is None
assert _shortfall(512, 512, memory = small) is None
assert _shortfall(1088, 1920, memory = small) is not None
def test_guard_scales_with_batch_size():
assert _shortfall(1024, 1024, batch_size = 1) is None
assert _shortfall(1024, 1024, batch_size = 4) is not None
assert "at a batch of 4" in _shortfall(1024, 1024, batch_size = 4)
def test_guard_is_skipped_on_unified_memory():
# On unified memory the load-time unified refusal owns this device class.
unified = DeviceMemory("cuda", "cuda", "unified_memory", _16G_FREE_MIB, _16G_TOTAL_MIB)
assert _shortfall(1088, 1920, memory = unified) is None
mps = DeviceMemory("mps", "mps", "unified_memory", _16G_FREE_MIB, _16G_TOTAL_MIB)
assert _shortfall(1088, 1920, memory = mps) is None
def test_guard_is_skipped_when_free_memory_is_unknown():
blind = DeviceMemory("cuda", "cuda", "discrete_vram", None, _16G_TOTAL_MIB)
assert _shortfall(1088, 1920, memory = blind) is None
def test_guard_is_skipped_off_cuda_and_rocm():
# ROCm reports device "cuda" so stays covered; XPU / CPU allocators differ and are untouched.
xpu = DeviceMemory("xpu", "xpu", "discrete_vram", _16G_FREE_MIB, _16G_TOTAL_MIB)
assert _shortfall(1088, 1920, memory = xpu) is None
cpu = DeviceMemory("cpu", "cpu", "system_memory", _16G_FREE_MIB, _16G_TOTAL_MIB)
assert _shortfall(1088, 1920, memory = cpu) is None
def test_guard_env_override_lets_an_oversized_generation_through(monkeypatch):
from core.inference.diffusion_memory import OVERSIZED_GENERATE_ENV
for value in ("1", "true", "YES", " on "):
monkeypatch.setenv(OVERSIZED_GENERATE_ENV, value)
assert _shortfall(1088, 1920) is None, value
for value in ("0", "false", "", "maybe"):
monkeypatch.setenv(OVERSIZED_GENERATE_ENV, value)
assert _shortfall(1088, 1920) is not None, value
def test_guard_fails_open_when_the_probe_raises():
# A broken probe must never cost a generation that would have worked.
class _Exploding:
device = "cuda"
memory_kind = "discrete_vram"
is_unified = False
total_mib = _16G_TOTAL_MIB
@property
def free_mib(self):
raise RuntimeError("mem_get_info exploded")
assert _shortfall(1088, 1920, memory = _Exploding()) is None
def test_raiser_raises_valueerror_so_the_route_answers_400():
# ValueError, not RuntimeError: the route maps ValueError to a 400 with the reason;
# RuntimeError is reserved for sentinels and otherwise becomes an opaque 500.
from core.inference.diffusion_memory import raise_on_image_activation_shortfall
with pytest.raises(ValueError, match = "1088x1920"):
raise_on_image_activation_shortfall(
device_memory = _discrete(_16G_FREE_MIB, _16G_TOTAL_MIB),
width = 1088,
height = 1920,
family = _TURBO_HINT,
)
raise_on_image_activation_shortfall(
device_memory = _discrete(_16G_FREE_MIB, _16G_TOTAL_MIB),
width = 1024,
height = 1024,
family = _TURBO_HINT,
)
def _zimage_plan_on(memory, text_encoder_dense_mib):
return plan_diffusion_memory(
target = _target(),
device_memory = memory,
model_dense_mib = _ZIMAGE_MODEL_DENSE_MIB,
companion_dense_mib = _ZIMAGE_COMPANION_MIB,
text_encoder_dense_mib = text_encoder_dense_mib,
runtime_headroom_mib = 8192,
base_overhead_mib = 2048,
)
def test_default_resolution_plans_identically_across_card_sizes():
# At the default resolution the guard is silent and the no-split plan is unchanged.
from core.inference.diffusion_memory import _safe_device_budget_mib
for gigabytes in (8, 12, 16, 24, 32, 48, 80):
total = gigabytes * 1024
memory = _discrete(int(total * 0.97), total)
assert _shortfall(1024, 1024, memory = memory) is None, gigabytes
expected = _legacy_offload_policy(
budget = _safe_device_budget_mib(memory),
model_dense_mib = _ZIMAGE_MODEL_DENSE_MIB,
companion_dense_mib = _ZIMAGE_COMPANION_MIB,
runtime_headroom_mib = 8192,
base_overhead_mib = 2048,
)
assert _zimage_plan_on(memory, None).offload_policy == expected, gigabytes
def test_the_refusal_reports_the_number_it_compared():
"""A refusal that quotes less than it compared reads as a bug in the guard. On the 15.92 GiB
card in #8188 (14,254 MiB usable) it printed "needs about 13.55 GB ... only about 13.92 GB
usable" and then refused, which is a contradiction on its face."""
from core.inference.diffusion_memory import DEFAULT_BASE_OVERHEAD_MIB
message = _shortfall(1088, 1920, base_overhead_mib = 1_024)
assert message is not None
assert "14.55 GB of working memory" in message
assert "1.00 GB of fixed overhead" in message
assert "13.55 GB of working memory" in _shortfall(1088, 1920, base_overhead_mib = 0)
assert DEFAULT_BASE_OVERHEAD_MIB > 0
def test_the_activation_refusal_is_its_own_error_type():
"""OpenAI route sanitises errors to a bare 500, so refusal needs its own type; still a ValueError."""
import pytest as _pytest
from core.inference.diffusion_memory import (
ImageActivationShortfallError,
raise_on_image_activation_shortfall,
)
assert issubclass(ImageActivationShortfallError, ValueError)
with _pytest.raises(ImageActivationShortfallError):
raise_on_image_activation_shortfall(
device_memory = _discrete(_16G_FREE_MIB, _16G_TOTAL_MIB),
width = 1088,
height = 1920,
family = _TURBO_HINT,
)
raise_on_image_activation_shortfall(
device_memory = _discrete(_16G_FREE_MIB, _16G_TOTAL_MIB),
width = 1024,
height = 1024,
family = _TURBO_HINT,
)
def test_the_streamed_encoders_are_hooked_leaf_by_leaf(monkeypatch):
# Text encoders are not uniform block stacks, so they must use leaf_level, not the DiTs' block_level.
seen = _stream_te_kwargs(monkeypatch, stream_text_encoders = True)
assert seen, "the applier never reached diffusers"
encoders = {name: kw for name, kw in seen.items() if str(name).startswith("text_encoder")}
denoisers = {name: kw for name, kw in seen.items() if not str(name).startswith("text_encoder")}
assert encoders and denoisers, seen
for name, kw in encoders.items():
assert kw["offload_type"] == "leaf_level", name
# diffusers only requires num_blocks_per_group for block_level.
assert kw["num_blocks_per_group"] is None, name
for name, kw in denoisers.items():
assert kw["offload_type"] == "block_level", name
assert kw["num_blocks_per_group"] is not None, name
def test_the_batch_remedy_appears_only_when_a_batch_was_budgeted():
"""Smaller-batch advice only when a batch was budgeted, since it cannot change a single-image
refusal."""
single = _shortfall(1088, 1920)
assert single is not None
assert "smaller batch size" not in single
assert "smaller resolution," in single
batched = _shortfall(1024, 1024, batch_size = 4)
assert batched is not None
assert "at a batch of 4" in batched
assert "or a smaller batch size" in batched
def test_cpu_offload_hooks_take_the_indexed_card_not_a_bare_cuda():
# Without an index diffusers falls back to _offload_gpu_id 0 and onloads to cuda:0 (0.39),
# so a load pinned to another GPU would page onto the wrong one.
for build, policy, call in (
(_plan, OFFLOAD_MODEL, "model_offload"),
(_manual_plan, OFFLOAD_SEQUENTIAL, "sequential_offload"),
):
pipe = _RecordingPipe()
apply_memory_plan(
pipe, build(policy, tiling = False), device = "cuda", placement_device = "cuda:1"
)
assert call in pipe.calls
assert pipe.offload_device == "cuda:1"
def test_the_resident_placement_follows_the_selected_card_too():
pipe = _RecordingPipe()
apply_memory_plan(
pipe, _plan(OFFLOAD_NONE, tiling = False), device = "cuda", placement_device = "cuda:1"
)
assert pipe.calls == ["to:cuda:1"]
def test_an_automatic_load_still_hands_over_a_bare_device():
for policy, expected in (
(OFFLOAD_NONE, ["to:cuda"]),
(OFFLOAD_MODEL, ["model_offload"]),
):
pipe = _RecordingPipe()
apply_memory_plan(pipe, _plan(policy, tiling = False), device = "cuda")
assert expected[0] in pipe.calls
if policy is OFFLOAD_MODEL:
assert pipe.offload_device == "cuda"
def _a100_40gb():
"""An idle A100-SXM4-40GB, the card the pipeline-load report came from."""
return diffusion_memory.DeviceMemory("cuda", "cuda", "discrete_vram", 40200, 40960)
def _offloadable():
return types.SimpleNamespace(supports_model_cpu_offload = True)
def test_a_pipeline_load_without_a_companion_split_says_unknown_not_over_budget():
# An unknown split fails both group tiers by construction; it must not be reported as over budget.
plan = diffusion_memory.plan_diffusion_memory(
target = _offloadable(),
device_memory = _a100_40gb(),
model_dense_mib = 31566,
runtime_headroom_mib = 8192,
companion_dense_mib = None,
text_encoder_dense_mib = None,
)
assert plan.offload_policy == OFFLOAD_MODEL
reason = "; ".join(plan.reasons)
assert "companion size unknown" in reason
assert "exceed budget" not in reason
def test_the_same_load_reaches_group_offload_once_the_split_is_known():
# Control: Qwen-Image-2.1 on 40 GB, 31,566 MiB weights do not fit but the 28,264 MiB group floor does.
plan = diffusion_memory.plan_diffusion_memory(
target = _offloadable(),
device_memory = _a100_40gb(),
model_dense_mib = 31566,
runtime_headroom_mib = 8192,
companion_dense_mib = 18024,
text_encoder_dense_mib = 16689,
)
assert plan.offload_policy == OFFLOAD_GROUP
assert plan.vae_tiling is False
_Q21_TRANSFORMER_MIB = 7_650
_Q21_VAE_MIB = 1_288
_Q21_TE_MIB = 8_959
_24G_FREE_MIB = 23_000
_24G_TOTAL_MIB = 24_576 # budget 23000 - 2457 = 20543
def _q21_plan(
free_mib,
total_mib,
*,
text_encoder_dense_mib = _Q21_TE_MIB,
mode = None,
):
return plan_diffusion_memory(
target = _target(),
device_memory = _discrete(free_mib, total_mib),
model_dense_mib = _Q21_TRANSFORMER_MIB + _Q21_VAE_MIB + _Q21_TE_MIB,
companion_dense_mib = _Q21_VAE_MIB + _Q21_TE_MIB,
text_encoder_dense_mib = text_encoder_dense_mib,
runtime_headroom_mib = 8192,
base_overhead_mib = 2048,
requested_mode = mode,
)
def test_the_transformer_stays_resident_when_only_the_encoder_does_not_fit():
plan = _q21_plan(_24G_FREE_MIB, _24G_TOTAL_MIB)
assert plan.estimates["resident_transformer_floor_mib"] == 19_178
assert plan.offload_policy == OFFLOAD_GROUP
assert plan.stream_transformer is False and plan.stream_text_encoders is True
assert plan.estimates["group_floor_mib"] <= plan.estimates["safe_device_budget_mib"]
assert plan.as_public_dict()["stream_transformer"] is False
def test_the_resident_transformer_tier_needs_the_encoder_split():
plan = _q21_plan(_24G_FREE_MIB, _24G_TOTAL_MIB, text_encoder_dense_mib = None)
assert plan.estimates["resident_transformer_floor_mib"] is None
assert plan.offload_policy == OFFLOAD_GROUP
assert plan.stream_transformer is True and plan.stream_text_encoders is False
def test_a_card_too_small_for_the_resident_transformer_still_streams_it():
plan = _q21_plan(_16G_FREE_MIB, _16G_TOTAL_MIB)
assert plan.offload_policy == OFFLOAD_GROUP
assert plan.stream_transformer is True and plan.stream_text_encoders is True
def test_fast_mode_prefers_the_resident_transformer_over_streaming_it():
plan = _q21_plan(_24G_FREE_MIB, _24G_TOTAL_MIB, mode = MEMORY_MODE_FAST)
assert plan.offload_policy == OFFLOAD_GROUP
assert plan.stream_transformer is False and plan.stream_text_encoders is True
def test_every_other_tier_reports_a_streamed_or_resident_transformer_consistently():
roomy = _q21_plan(80_000, 81_920)
assert roomy.offload_policy == OFFLOAD_NONE and roomy.stream_transformer is True
tiny = _q21_plan(6_000, 8_192)
assert tiny.offload_policy == OFFLOAD_MODEL and tiny.stream_transformer is True
balanced = _q21_plan(_24G_FREE_MIB, _24G_TOTAL_MIB, mode = MEMORY_MODE_BALANCED)
assert balanced.offload_policy == OFFLOAD_GROUP and balanced.stream_transformer is True
def test_apply_group_offload_can_keep_the_transformer_resident(monkeypatch):
import core.inference.diffusion_memory as mem
pipe, applied, transformer, te, te2, vae = _stream_te_pipe(monkeypatch)
assert (
mem._apply_group_offload(
pipe, "cuda", logger = None, stream_text_encoders = True, stream_transformer = False
)
is True
)
assert applied == [te, te2]
assert transformer.placed is not None and vae.placed is not None
assert te.placed is None and te2.placed is None
def test_the_resident_transformer_tier_streams_encoders_at_leaf_level(monkeypatch):
seen = _stream_te_kwargs(monkeypatch, stream_text_encoders = True, stream_transformer = False)
assert "transformer" not in seen
assert seen["text_encoder"]["offload_type"] == "leaf_level"
assert seen["text_encoder_2"]["offload_type"] == "leaf_level"
def test_a_refusing_encoder_hands_the_resident_transformer_tier_to_whole_module_offload(
monkeypatch,
):
import core.inference.diffusion_memory as mem
def _apply(module, **kw):
raise ValueError("no leaves to stream")
pipe, _unused, transformer, te, te2, vae = _stream_te_pipe(monkeypatch)
_swap_group_offloading(monkeypatch, _apply)
assert (
mem._apply_group_offload(
pipe, "cuda", logger = None, stream_text_encoders = True, stream_transformer = False
)
is False
)
def test_apply_group_offload_refuses_a_plan_that_streams_nothing(monkeypatch):
import core.inference.diffusion_memory as mem
pipe, applied, *_ = _stream_te_pipe(monkeypatch)
assert mem._apply_group_offload(pipe, "cuda", logger = None, stream_transformer = False) is False
assert applied == []
def test_apply_memory_plan_threads_the_resident_transformer_flag(monkeypatch):
import core.inference.diffusion_memory as mem
seen = {}
def _fake(pipe, device, logger, **kw):
seen.clear()
seen.update(kw)
return True
monkeypatch.setattr(mem, "_apply_group_offload", _fake)
policy, _tiling = apply_memory_plan(
_RecordingPipe(), _q21_plan(_24G_FREE_MIB, _24G_TOTAL_MIB), device = "cuda"
)
assert policy == OFFLOAD_GROUP
assert seen == {"stream_text_encoders": True, "stream_transformer": False}
apply_memory_plan(_RecordingPipe(), _q21_plan(_16G_FREE_MIB, _16G_TOTAL_MIB), device = "cuda")
assert seen == {"stream_text_encoders": True}
def _sized_stream_te_kwargs(monkeypatch, sizes, budget, **call_kw):
import core.inference.diffusion_memory as mem
monkeypatch.delenv(mem.GROUP_OFFLOAD_PIN_ENV, raising = False)
monkeypatch.setattr(mem, "_pin_budget_mib", lambda: budget)
monkeypatch.setattr(mem, "_pinned_memory_capped", lambda: False, raising = False)
monkeypatch.setattr(mem, "_module_host_mib", lambda m: sizes.get(getattr(m, "name", ""), 0))
return _stream_te_kwargs(monkeypatch, **call_kw)
def test_a_ram_rich_host_pins_everything_the_tier_streams(monkeypatch):
seen = _sized_stream_te_kwargs(
monkeypatch,
{"transformer": 7000, "text_encoder": 8000, "text_encoder_2": 500},
40_000,
stream_text_encoders = True,
)
assert all(kw["low_cpu_mem_usage"] is False for kw in seen.values()), seen
def test_the_transformer_is_pinned_before_the_encoders(monkeypatch):
seen = _sized_stream_te_kwargs(
monkeypatch,
{"transformer": 7000, "text_encoder": 8000, "text_encoder_2": 500},
10_000,
stream_text_encoders = True,
)
assert seen["transformer"]["low_cpu_mem_usage"] is False
assert seen["text_encoder"]["low_cpu_mem_usage"] is True
assert seen["text_encoder_2"]["low_cpu_mem_usage"] is True
def test_the_resident_transformer_tier_pins_its_encoders_when_ram_allows(monkeypatch):
seen = _sized_stream_te_kwargs(
monkeypatch,
{"transformer": 7000, "text_encoder": 8000, "text_encoder_2": 500},
10_000,
stream_text_encoders = True,
stream_transformer = False,
)
assert "transformer" not in seen
assert seen["text_encoder"]["low_cpu_mem_usage"] is False
assert seen["text_encoder_2"]["low_cpu_mem_usage"] is False
def _background_pin_calls(monkeypatch, sizes, budget, **call_kw):
"""(low_cpu_mem_usage per module, modules handed to a deferred pinner) for one apply."""
import core.inference.diffusion_memory as mem
usage: dict = {}
deferred: list = []
pipe, *_ = _stream_te_pipe(monkeypatch)
def _apply(
module,
onload_device = None,
offload_device = None,
offload_type = None,
num_blocks_per_group = None,
non_blocking = False,
use_stream = False,
record_stream = False,
low_cpu_mem_usage = False,
**_,
):
usage[module.name] = low_cpu_mem_usage
_swap_group_offloading(monkeypatch, _apply)
monkeypatch.delenv(mem.GROUP_OFFLOAD_PIN_ENV, raising = False)
monkeypatch.setattr(mem, "_pin_budget_mib", lambda: budget)
monkeypatch.setattr(mem, "_pinned_memory_capped", lambda: False, raising = False)
monkeypatch.setattr(mem, "_module_host_mib", lambda m: sizes.get(getattr(m, "name", ""), 0))
monkeypatch.setattr(
mem,
"_defer_pinning",
lambda pipe, module, device, logger: deferred.append(module.name) or True,
)
assert mem._apply_group_offload(pipe, "cuda", logger = None, **call_kw) is True
return usage, deferred
_BG_SIZES = {"transformer": 7000, "text_encoder": 8000, "text_encoder_2": 500}
def test_a_video_load_defers_every_planned_pin_off_the_load_path(monkeypatch):
# Pinning at load reads every streamed byte first (125-139 s for LTX-2.3); pinning is deferred, not reduced.
usage, deferred = _background_pin_calls(
monkeypatch, _BG_SIZES, 40_000, stream_text_encoders = True, background_pin = True
)
assert all(v is True for v in usage.values()), usage
assert deferred == ["transformer", "text_encoder", "text_encoder_2"]
def test_a_module_the_ram_gate_left_unpinned_gets_no_pinner(monkeypatch):
usage, deferred = _background_pin_calls(
monkeypatch, _BG_SIZES, 10_000, stream_text_encoders = True, background_pin = True
)
assert usage == {"transformer": True, "text_encoder": True, "text_encoder_2": True}, usage
assert deferred == ["transformer"]
def test_a_resident_dit_tier_defers_only_its_encoders(monkeypatch):
usage, deferred = _background_pin_calls(
monkeypatch,
_BG_SIZES,
10_000,
stream_text_encoders = True,
stream_transformer = False,
background_pin = True,
)
assert "transformer" not in usage
assert deferred == ["text_encoder", "text_encoder_2"]
def test_without_background_pin_the_apply_still_pins_eagerly(monkeypatch):
usage, deferred = _background_pin_calls(
monkeypatch, _BG_SIZES, 40_000, stream_text_encoders = True
)
assert all(v is False for v in usage.values()), usage
assert deferred == []
def test_the_background_pin_env_restores_eager_pinning(monkeypatch):
import core.inference.diffusion_memory as mem
monkeypatch.setenv(mem.BACKGROUND_PIN_ENV, "0")
usage, deferred = _background_pin_calls(
monkeypatch, _BG_SIZES, 40_000, stream_text_encoders = True, background_pin = True
)
assert all(v is False for v in usage.values()), usage
assert deferred == []
def test_a_pipe_request_turns_background_pinning_on(monkeypatch):
# Video asks through the pipe, so apply_memory_plan and its stubs keep their signature.
import core.inference.diffusion_memory as mem
usage, deferred = _background_pin_calls(
monkeypatch, _BG_SIZES, 40_000, stream_text_encoders = True
)
assert deferred == []
import sys
pipe, *_ = _stream_te_pipe(monkeypatch)
seen: dict = {}
def _apply(
module,
low_cpu_mem_usage = False,
use_stream = False,
**_,
):
seen[module.name] = low_cpu_mem_usage
_swap_group_offloading(monkeypatch, _apply)
monkeypatch.setattr(
mem,
"_defer_pinning",
lambda pipe, module, device, logger: deferred.append(module.name) or True,
)
mem.request_background_pins(pipe)
assert mem._apply_group_offload(pipe, "cuda", logger = None, stream_text_encoders = True) is True
assert "transformer" in deferred, deferred
@pytest.mark.parametrize("stream_transformer", [True, False])
def test_windows_and_wsl_never_pin_the_streamed_tiers(monkeypatch, stream_transformer):
import core.inference.diffusion_memory as mem
seen = _sized_stream_te_kwargs(
monkeypatch,
{"transformer": 7000, "text_encoder": 8000, "text_encoder_2": 500},
1_000_000,
stream_text_encoders = True,
stream_transformer = stream_transformer,
)
assert all(kw["low_cpu_mem_usage"] is False for kw in seen.values()), seen
monkeypatch.delattr(mem, "_pinned_memory_capped", raising = False)
monkeypatch.undo()
monkeypatch.delenv(mem.GROUP_OFFLOAD_PIN_ENV, raising = False)
monkeypatch.setattr(mem, "_pin_budget_mib", lambda: 1_000_000)
monkeypatch.setattr(mem.sys, "platform", "win32")
seen = _stream_te_kwargs(
monkeypatch, stream_text_encoders = True, stream_transformer = stream_transformer
)
assert seen and all(kw["low_cpu_mem_usage"] is True for kw in seen.values()), seen
monkeypatch.setenv(mem.GROUP_OFFLOAD_PIN_ENV, "1")
assert mem._streamed_pin_plan(7000, 8000) == (True, True)
def test_a_first_encoder_refusal_is_unhooked_before_whole_module_offload(monkeypatch):
import core.inference.diffusion_memory as mem
def _apply(module, **kw):
if getattr(module, "name", "") == "text_encoder":
raise RuntimeError("CUDA error: out of memory (pin_memory)")
pipe, _unused, transformer, te, te2, vae = _stream_te_pipe(monkeypatch)
_swap_group_offloading(monkeypatch, _apply)
unhooked: list[str] = []
monkeypatch.setattr(
mem, "_remove_group_offload_hooks", lambda m: unhooked.append(m.name), raising = False
)
assert (
mem._apply_group_offload(
pipe, "cuda", logger = None, stream_text_encoders = True, stream_transformer = False
)
is False
)
assert unhooked == ["text_encoder"]
def test_an_encoder_kept_resident_after_a_partial_leaf_apply_is_unhooked(monkeypatch):
"""A refusal that raised after hooking part of the encoder must not leave those hooks on the resident encoder."""
import core.inference.diffusion_memory as mem
def _apply(module, **kw):
if getattr(module, "name", "") == "text_encoder":
raise RuntimeError("leaf offload failed part way")
pipe, _unused, transformer, te, te2, vae = _stream_te_pipe(monkeypatch)
_swap_group_offloading(monkeypatch, _apply)
unhooked: list[str] = []
monkeypatch.setattr(
mem, "_remove_group_offload_hooks", lambda m: unhooked.append(m.name), raising = False
)
assert (
mem._apply_group_offload(
pipe, "cuda", logger = None, stream_text_encoders = True, stream_transformer = True
)
is True
)
assert unhooked == ["text_encoder"]
def test_pinned_memory_cap_detection(monkeypatch, tmp_path):
import builtins
import core.inference.diffusion_memory as mem
monkeypatch.setattr(mem.sys, "platform", "win32")
assert mem._pinned_memory_capped() is True
monkeypatch.setattr(mem.sys, "platform", "linux")
real_open = builtins.open
for text, capped in (
("Linux version 6.6.87.2-microsoft-standard-WSL2 (gcc)", True),
("Linux version 6.17.0-1007-aws (gcc)", False),
):
monkeypatch.setattr(
builtins,
"open",
lambda path, *a, _t = text, **k: (
__import__("io").StringIO(_t)
if path == "/proc/version"
else real_open(path, *a, **k)
),
)
assert mem._pinned_memory_capped() is capped
def test_unknown_host_memory_never_pins(monkeypatch):
seen = _sized_stream_te_kwargs(
monkeypatch, {"transformer": 1, "text_encoder": 1}, None, stream_text_encoders = True
)
assert all(kw["low_cpu_mem_usage"] is True for kw in seen.values()), seen
@pytest.mark.parametrize("value, pinned", [("0", True), ("1", False)])
def test_the_pin_env_overrides_the_ram_gate(monkeypatch, value, pinned):
import core.inference.diffusion_memory as mem
monkeypatch.setattr(mem, "_pin_budget_mib", lambda: 40_000 if value == "0" else 0)
monkeypatch.setenv(mem.GROUP_OFFLOAD_PIN_ENV, value)
seen = _stream_te_kwargs(monkeypatch, stream_text_encoders = True)
assert all(kw["low_cpu_mem_usage"] is pinned for kw in seen.values()), seen
def test_the_pin_budget_leaves_the_reserve_free(monkeypatch):
import core.inference.diffusion_memory as mem
monkeypatch.setattr(mem, "_cgroup_memory_limit_mib", lambda: None)
monkeypatch.setattr(mem, "_system_memory_mib", lambda: (65_536, 30_000))
monkeypatch.setattr(mem, "_available_system_memory_mib", lambda: 30_000)
assert mem._pin_budget_mib() == 30_000 - int(65_536 * 0.15)
# 64 GiB container on a 512 GiB host: the reserve comes from the container cgroup.
monkeypatch.setattr(mem, "_cgroup_memory_limit_mib", lambda: 65_536)
monkeypatch.setattr(mem, "_system_memory_mib", lambda: (524_288, 60_000))
monkeypatch.setattr(mem, "_available_system_memory_mib", lambda: 60_000)
assert mem._pin_budget_mib() == 60_000 - int(65_536 * 0.15)
monkeypatch.setattr(mem, "_cgroup_memory_limit_mib", lambda: None)
monkeypatch.setattr(mem, "_system_memory_mib", lambda: (16_384, 3_000))
monkeypatch.setattr(mem, "_available_system_memory_mib", lambda: 3_000)
assert mem._pin_budget_mib() == 0
monkeypatch.setattr(mem, "_system_memory_mib", lambda: (None, None))
assert mem._pin_budget_mib() is None
def test_a_late_encoder_refusal_streams_the_transformer_before_placing_the_encoder(monkeypatch):
import core.inference.diffusion_memory as mem
order: list[str] = []
def _apply(module, **kw):
if getattr(module, "name", "") == "text_encoder_2":
raise ValueError("no leaves to stream")
order.append("hook:" + module.name)
pipe, _unused, transformer, te, te2, vae = _stream_te_pipe(monkeypatch)
_swap_group_offloading(monkeypatch, _apply)
original_to = type(te2).to
def _to(self, device):
if self is te2:
order.append("place:text_encoder_2")
return original_to(self, device)
monkeypatch.setattr(type(te2), "to", _to)
assert (
mem._apply_group_offload(
pipe, "cuda", logger = None, stream_text_encoders = True, stream_transformer = False
)
is True
)
assert order == ["hook:text_encoder", "hook:transformer", "place:text_encoder_2"]
def test_pinned_host_pricing_rounds_each_tensor_to_a_power_of_two(monkeypatch):
import core.inference.diffusion_memory as mem
class _T:
def __init__(self, nbytes):
self._n = nbytes
def numel(self):
return self._n
def element_size(self):
return 1
class _M:
def parameters(self, recurse = True):
return [_T(3 * 1024 * 1024), _T(5 * 1024 * 1024)]
def buffers(self, recurse = True):
return [_T(1024 * 1024)]
assert mem._module_host_mib(_M()) == 13
def test_only_resident_and_encoder_only_plans_keep_the_transformer_in_place():
from types import SimpleNamespace
import core.inference.diffusion_memory as mem
keeps = mem.plan_keeps_transformer_resident
assert keeps(SimpleNamespace(offload_policy = OFFLOAD_NONE)) is True
assert keeps(SimpleNamespace(offload_policy = OFFLOAD_GROUP, stream_transformer = False)) is True
assert keeps(SimpleNamespace(offload_policy = OFFLOAD_GROUP, stream_transformer = True)) is False
assert keeps(SimpleNamespace(offload_policy = OFFLOAD_GROUP)) is False
for policy in (OFFLOAD_MODEL, mem.OFFLOAD_STREAMING, mem.OFFLOAD_SEQUENTIAL):
assert keeps(SimpleNamespace(offload_policy = policy, stream_transformer = False)) is False
assert keeps(_q21_plan(_24G_FREE_MIB, _24G_TOTAL_MIB)) is True
assert keeps(_q21_plan(_16G_FREE_MIB, _16G_TOTAL_MIB)) is False
class _TorchaoLikeTensor:
"""Stands in for a torchao subclass: only its defining module is read."""
_TorchaoLikeTensor.__module__ = "torchao.quantization.linear_activation_quantized_tensor"
class _ParamsModule:
def __init__(self, *tensors):
self._tensors = tensors
def parameters(self):
return iter(self._tensors)
def test_torchao_weights_are_told_apart_from_gguf_and_plain_ones():
import core.inference.diffusion_memory as mem
class _GGUFParameter:
pass
_GGUFParameter.__module__ = "diffusers.quantizers.gguf.utils"
assert mem._holds_torchao_weights(_ParamsModule(object(), _TorchaoLikeTensor())) is True
assert mem._holds_torchao_weights(_ParamsModule(_GGUFParameter(), object())) is False
class _Broken:
def parameters(self):
raise RuntimeError("no parameters")
assert mem._holds_torchao_weights(_Broken()) is False
def test_a_refusing_encoder_never_hands_a_torchao_transformer_to_whole_module_offload(monkeypatch):
# Whole-module offload moves the transformer every call, which torchao weights do not survive.
import core.inference.diffusion_memory as mem
pipe = _RecordingPipe()
pipe.transformer = _ParamsModule(_TorchaoLikeTensor())
monkeypatch.setattr(mem, "_apply_group_offload", lambda *a, **k: False)
with pytest.raises(RuntimeError, match = "cannot move torchao weights"):
apply_memory_plan(pipe, _q21_plan(_24G_FREE_MIB, _24G_TOTAL_MIB), device = "cuda")
assert "model_offload" not in pipe.calls
def test_a_refusing_encoder_still_falls_back_for_a_dense_transformer(monkeypatch):
import core.inference.diffusion_memory as mem
pipe = _RecordingPipe()
pipe.transformer = _ParamsModule(object())
monkeypatch.setattr(mem, "_apply_group_offload", lambda *a, **k: False)
policy, _tiling = apply_memory_plan(
pipe, _q21_plan(_24G_FREE_MIB, _24G_TOTAL_MIB), device = "cuda"
)
assert policy == OFFLOAD_MODEL
def test_a_late_encoder_refusal_never_streams_a_torchao_transformer(monkeypatch):
import core.inference.diffusion_memory as mem
order: list[str] = []
def _apply(module, **kw):
if getattr(module, "name", "") == "text_encoder_2":
raise ValueError("no leaves to stream")
order.append("hook:" + module.name)
pipe, _unused, transformer, te, te2, vae = _stream_te_pipe(monkeypatch)
_swap_group_offloading(monkeypatch, _apply)
monkeypatch.setattr(mem, "_pipe_denoisers_hold_torchao", lambda _pipe: True)
with pytest.raises(ValueError, match = "no leaves to stream"):
mem._apply_group_offload(
pipe, "cuda", logger = None, stream_text_encoders = True, stream_transformer = False
)
assert order == ["hook:text_encoder"]
assert te2.placed is None
def test_streamed_transformer_runs_pinned_and_overlapped_on_a_real_gpu(monkeypatch):
torch = pytest.importorskip("torch")
if not torch.cuda.is_available():
pytest.skip("needs a CUDA device")
pytest.importorskip("diffusers.hooks")
import core.inference.diffusion_memory as mem
monkeypatch.delenv("UNSLOTH_DIFFUSION_STREAMING_PREFETCH", raising = False)
monkeypatch.delenv(mem.GROUP_OFFLOAD_PIN_ENV, raising = False)
monkeypatch.setattr(mem, "_pin_budget_mib", lambda: 1 << 20)
monkeypatch.setattr(mem, "_pinned_memory_capped", lambda: False)
class Net(torch.nn.Module):
def __init__(self):
super().__init__()
self.proj_in = torch.nn.Linear(64, 64)
self.blocks = torch.nn.ModuleList(torch.nn.Linear(64, 64) for _ in range(6))
def forward(self, x):
x = self.proj_in(x)
for block in self.blocks:
x = x + torch.tanh(block(x))
return x
torch.manual_seed(0)
net = Net().eval()
x = torch.randn(8, 64)
with torch.no_grad():
want = net(x)
pipe = types.SimpleNamespace(transformer = net, components = {"transformer": net})
mem._apply_streaming_offload(pipe, "cuda", None)
groups = [g for g in mem._offload_groups(net) if g.stream is not None]
assert groups and all(g.record_stream for g in groups)
assert all(t.is_pinned() for g in groups for t in g.cpu_param_dict.values())
with torch.no_grad():
for _ in range(3): # the first forward traces the order, the later ones prefetch
got = net(x.cuda())
torch.cuda.synchronize()
assert torch.allclose(got.cpu(), want, atol = 1e-5)
def _top_group_module(
monkeypatch,
*,
stream = None,
torchao = False,
):
"""A module whose diffusers registry holds a top-level group, on a fake diffusers.hooks.group_offloading."""
import sys
torch = pytest.importorskip("torch")
go = types.ModuleType("diffusers.hooks.group_offloading")
go._GROUP_OFFLOADING = "group_offloading"
go._is_torchao_tensor = lambda t: torchao
hooks = types.ModuleType("diffusers.hooks")
hooks.group_offloading = go
monkeypatch.setitem(sys.modules, "diffusers.hooks", hooks)
monkeypatch.setitem(sys.modules, "diffusers.hooks.group_offloading", go)
if "diffusers" not in sys.modules:
monkeypatch.setitem(sys.modules, "diffusers", types.ModuleType("diffusers"))
leaf = torch.nn.Linear(4, 4)
group = types.SimpleNamespace(
stream = stream,
offload_to_disk_path = None,
onload_device = types.SimpleNamespace(type = "cuda"),
modules = [leaf],
parameters = [],
buffers = [],
onload_ = "diffusers",
offload_ = "diffusers",
)
hook = types.SimpleNamespace(group = group)
module = types.SimpleNamespace(
_diffusers_hook = types.SimpleNamespace(
get_hook = lambda name: hook if name == "group_offloading" else None
)
)
return module, group
@pytest.mark.parametrize(
"env", ["UNSLOTH_DIFFUSION_PIN_TOP_GROUP", "UNSLOTH_DIFFUSION_GROUP_OFFLOAD_PIN"]
)
def test_top_level_group_kill_switch(monkeypatch, env):
import core.inference.diffusion_memory as mem
module, group = _top_group_module(monkeypatch)
monkeypatch.setattr(mem, "_pin_budget_mib", lambda: 1 << 20)
monkeypatch.setattr(mem, "_pinned_memory_capped", lambda: False)
monkeypatch.setenv(env, "0")
assert mem._pin_top_level_group(module) is False
assert group.onload_ == "diffusers" and group.offload_ == "diffusers"
def test_top_level_group_respects_the_pinned_allocator_rounding(monkeypatch):
import core.inference.diffusion_memory as mem
module, group = _top_group_module(monkeypatch)
monkeypatch.delenv(mem.PIN_TOP_GROUP_ENV, raising = False)
monkeypatch.delenv(mem.GROUP_OFFLOAD_PIN_ENV, raising = False)
monkeypatch.setattr(mem, "_pinned_memory_capped", lambda: False)
# A 3 MiB weight pins as a 4 MiB block, so a 3 MiB budget must refuse it.
group.modules = [
types.SimpleNamespace(
parameters = lambda: [__import__("torch").empty(3 << 18)], buffers = lambda: []
)
]
monkeypatch.setattr(mem, "_pin_budget_mib", lambda: 3)
assert mem._pin_top_level_group(module) is False
assert group.onload_ == "diffusers"
def test_top_level_group_counts_against_the_running_pin_total(monkeypatch):
import core.inference.diffusion_memory as mem
module, group = _top_group_module(monkeypatch)
monkeypatch.delenv(mem.PIN_TOP_GROUP_ENV, raising = False)
monkeypatch.delenv(mem.GROUP_OFFLOAD_PIN_ENV, raising = False)
monkeypatch.setattr(mem, "_pinned_memory_capped", lambda: False)
group.modules = [
types.SimpleNamespace(
parameters = lambda: [__import__("torch").empty(1 << 18)], buffers = lambda: []
)
]
monkeypatch.setattr(mem, "_pin_budget_mib", lambda: 8)
# pin_memory raises on a host with no CUDA device (CPU CI), so a plain copy stands in.
torch = pytest.importorskip("torch")
monkeypatch.setattr(torch.Tensor, "pin_memory", lambda self: self.clone())
assert mem._pin_top_level_group(module, None, [8]) is False
assert group.onload_ == "diffusers"
total = [4]
assert mem._pin_top_level_group(module, None, total) is True
assert total == [5]
@pytest.mark.parametrize("kind", ["streamed_group", "torchao"])
def test_top_level_group_left_alone_when_not_the_streamless_top_group(monkeypatch, kind):
import core.inference.diffusion_memory as mem
module, group = _top_group_module(
monkeypatch,
stream = object() if kind == "streamed_group" else None,
torchao = kind == "torchao",
)
monkeypatch.delenv(mem.PIN_TOP_GROUP_ENV, raising = False)
monkeypatch.setattr(mem, "_pin_budget_mib", lambda: 1 << 20)
monkeypatch.setattr(mem, "_pinned_memory_capped", lambda: False)
assert mem._pin_top_level_group(module) is False
assert group.onload_ == "diffusers"
@pytest.mark.parametrize("tier", ["group", "streaming"])
def test_top_level_weights_onload_from_one_pinned_copy_on_a_real_gpu(monkeypatch, tier):
torch = pytest.importorskip("torch")
if not torch.cuda.is_available():
pytest.skip("needs a CUDA device")
pytest.importorskip("diffusers.hooks")
import core.inference.diffusion_memory as mem
monkeypatch.delenv("UNSLOTH_DIFFUSION_PIN_TOP_GROUP", raising = False)
monkeypatch.delenv(mem.GROUP_OFFLOAD_PIN_ENV, raising = False)
monkeypatch.setattr(mem, "_pin_budget_mib", lambda: 1 << 20)
monkeypatch.setattr(mem, "_pinned_memory_capped", lambda: False)
class Net(torch.nn.Module):
def __init__(self):
super().__init__()
self.proj_in = torch.nn.Linear(64, 64)
self.blocks = torch.nn.ModuleList(torch.nn.Linear(64, 64) for _ in range(4))
self.proj_out = torch.nn.Linear(64, 64)
def forward(self, x):
x = self.proj_in(x)
for block in self.blocks:
x = x + torch.tanh(block(x))
return self.proj_out(x)
torch.manual_seed(0)
net = Net().eval()
x = torch.randn(8, 64)
with torch.no_grad():
want = net(x)
pipe = types.SimpleNamespace(transformer = net, components = {"transformer": net})
if tier == "group":
assert mem._apply_group_offload(pipe, "cuda", logger = None) is True
else:
mem._apply_streaming_offload(pipe, "cuda", None)
with torch.no_grad():
got = [net(x.cuda()).cpu() for _ in range(3)]
torch.cuda.synchronize()
# Offloaded onto the same pinned host copy: no device-to-host copy into a fresh buffer.
ptr = net.proj_out.weight.data_ptr()
assert net.proj_out.weight.device.type == "cpu" and net.proj_out.weight.is_pinned()
with torch.no_grad():
got.append(net(x.cuda()).cpu())
torch.cuda.synchronize()
assert net.proj_out.weight.data_ptr() == ptr
assert all(torch.allclose(g, want, atol = 1e-5) for g in got)
def test_top_level_weights_replaced_while_offloaded_are_picked_up(monkeypatch):
torch = pytest.importorskip("torch")
if not torch.cuda.is_available():
pytest.skip("needs a CUDA device")
pytest.importorskip("diffusers.hooks")
import copy
import core.inference.diffusion_memory as mem
monkeypatch.delenv("UNSLOTH_DIFFUSION_PIN_TOP_GROUP", raising = False)
monkeypatch.setattr(mem, "_pin_budget_mib", lambda: 1 << 20)
monkeypatch.setattr(mem, "_pinned_memory_capped", lambda: False)
class Net(torch.nn.Module):
def __init__(self):
super().__init__()
self.blocks = torch.nn.ModuleList(torch.nn.Linear(16, 16) for _ in range(3))
self.proj_out = torch.nn.Linear(16, 16)
def forward(self, x):
for block in self.blocks:
x = x + torch.tanh(block(x))
return self.proj_out(x)
torch.manual_seed(0)
net = Net().eval()
ref = copy.deepcopy(net)
x = torch.randn(4, 16)
pipe = types.SimpleNamespace(transformer = net, components = {"transformer": net})
mem._apply_streaming_offload(pipe, "cuda", None)
with torch.no_grad():
for _ in range(2):
net(x.cuda())
torch.cuda.synchronize()
net.proj_out.weight.data = net.proj_out.weight.data * 2
ref.proj_out.weight.data = ref.proj_out.weight.data * 2
got = net(x.cuda()).cpu()
want = ref(x)
assert torch.allclose(got, want, atol = 1e-5)
def test_streaming_counts_pinned_encoders_before_a_torchao_denoiser_pins(monkeypatch):
# The torchao denoiser pins within a running total that must include plan-pinned encoders.
import core.inference.diffusion_memory as mem
seen_totals: dict = {}
real = mem._torchao_group_offload_kwargs
def _spy(
module,
kwargs,
pinned_mib = None,
):
seen_totals[module.name] = pinned_mib[0] if pinned_mib else None
return real(module, kwargs, pinned_mib)
monkeypatch.setattr(mem, "_torchao_group_offload_kwargs", _spy)
_streaming_apply_kwargs(monkeypatch, 40_000)
# The 7500 MiB encoder, rounded up to a power of two like the pinned allocator.
assert seen_totals["transformer"] == 8192
@pytest.mark.parametrize("tier", ["group", "streaming"])
def test_top_level_weights_never_copied_back_when_the_pin_budget_is_spent(monkeypatch, tier):
"""No pinnable host RAM (a 31 GB Kaggle host holding a 10 GB DiT and an 11 GB encoder reports 0 MiB): the top-level
group still must not copy its weights device to host on every offload. It re-points to the host tensors it was
loaded from, so the host pointer survives every forward and the output is unchanged."""
torch = pytest.importorskip("torch")
if not torch.cuda.is_available():
pytest.skip("needs a CUDA device")
pytest.importorskip("diffusers.hooks")
import core.inference.diffusion_memory as mem
monkeypatch.delenv("UNSLOTH_DIFFUSION_PIN_TOP_GROUP", raising = False)
monkeypatch.delenv(mem.GROUP_OFFLOAD_PIN_ENV, raising = False)
monkeypatch.setattr(mem, "_pin_budget_mib", lambda: 0)
monkeypatch.setattr(mem, "_pinned_memory_capped", lambda: False)
class Net(torch.nn.Module):
def __init__(self):
super().__init__()
self.proj_in = torch.nn.Linear(1024, 1024)
self.blocks = torch.nn.ModuleList(torch.nn.Linear(1024, 1024) for _ in range(4))
self.proj_out = torch.nn.Linear(1024, 1024)
def forward(self, x):
x = self.proj_in(x)
for block in self.blocks:
x = x + torch.tanh(block(x))
return self.proj_out(x)
torch.manual_seed(0)
net = Net().eval() # 4 MiB per Linear: a whole MiB, so a 0 MiB budget really refuses the pin
x = torch.randn(8, 1024)
with torch.no_grad():
want = net(x)
ptr = net.proj_out.weight.data_ptr()
pipe = types.SimpleNamespace(transformer = net, components = {"transformer": net})
if tier == "group":
assert mem._apply_group_offload(pipe, "cuda", logger = None) is True
else:
mem._apply_streaming_offload(pipe, "cuda", None)
groups = [g for g in mem._offload_groups(net) if getattr(g, "offload_leader", None) is net]
assert groups and all(getattr(g, "_unsloth_top_no_copy_back", False) for g in groups)
assert not any(getattr(g, "_unsloth_pinned_top", False) for g in groups)
got = []
for _ in range(3):
with torch.no_grad():
got.append(net(x.cuda()).cpu())
torch.cuda.synchronize()
assert net.proj_out.weight.device.type == "cpu"
assert net.proj_out.weight.data_ptr() == ptr
assert all(torch.equal(g, got[0]) for g in got)
assert all(torch.allclose(g, want, atol = 1e-5) for g in got)
def test_top_level_copy_back_skip_kill_switch_and_scope(monkeypatch):
import core.inference.diffusion_memory as mem
module, group = _top_group_module(monkeypatch)
monkeypatch.setenv(mem.PIN_TOP_GROUP_ENV, "0")
assert mem._skip_top_level_copy_back(module) is False
assert group.onload_ == "diffusers" and group.offload_ == "diffusers"
monkeypatch.delenv(mem.PIN_TOP_GROUP_ENV, raising = False)
for kind in ("streamed_group", "torchao"):
module, group = _top_group_module(
monkeypatch,
stream = object() if kind == "streamed_group" else None,
torchao = kind == "torchao",
)
assert mem._skip_top_level_copy_back(module) is False
assert group.onload_ == "diffusers"
module, group = _top_group_module(monkeypatch)
weight = group.modules[0].weight
host_ptr = weight.data_ptr()
assert mem._skip_top_level_copy_back(module) is True
assert group.onload_ != "diffusers" and group._unsloth_top_no_copy_back is True
group.offload_()
assert weight.data_ptr() == host_ptr
assert mem._skip_top_level_copy_back(module) is False
monkeypatch.setattr(mem, "_pin_budget_mib", lambda: 1 << 20)
monkeypatch.setattr(mem, "_pinned_memory_capped", lambda: False)
assert mem._pin_top_level_group(module) is False
def test_both_offload_tiers_fall_back_to_the_no_copy_back_top_group():
import ast
import inspect
import core.inference.diffusion_memory as mem
for fn in (mem._apply_group_offload, mem._apply_streaming_offload):
src = ast.unparse(ast.parse(inspect.getsource(fn).lstrip()))
assert "_skip_top_level_copy_back(module, logger)" in src, fn.__name__
assert (
"not _pin_top_level_group(module, logger, pinned_mib, device=onload)" in src
), fn.__name__
def test_refine_headroom_applies_to_the_denoiser_not_a_larger_text_encoder(monkeypatch):
"""The denoise headroom prices only denoisers: a text encoder larger than the DiT runs in its own phase, so a
whole-module plan where each fits its phase stays whole-module."""
Module = _install_sized_torch(monkeypatch)
plan = MemoryPlan(
requested_mode = "auto",
offload_policy = OFFLOAD_MODEL,
vae_tiling = True,
vae_slicing = True,
device_memory = _discrete(16384),
estimates = {"safe_device_budget_mib": 14000},
)
transformer = Module(6000)
pipe = types.SimpleNamespace(
transformer = transformer,
components = {"transformer": transformer, "text_encoder": Module(10000), "vae": Module(1000)},
)
assert refine_memory_plan_for_components(pipe, plan, headroom_mib = 6144) is plan
big = Module(9000)
tight = types.SimpleNamespace(
transformer = big, components = {"transformer": big, "text_encoder": Module(10000)}
)
refined = refine_memory_plan_for_components(tight, plan, headroom_mib = 6144)
assert refined.offload_policy == OFFLOAD_STREAMING
assert "loaded transformer is 9000 MiB (+6144 MiB runtime headroom)" in refined.reasons[-1]