Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

Fix diffusion offload host RAM retention (#10184)

Wasim Yousef Said4a108e0ca445
diff --git a/scripts/diffusion_bench.py b/scripts/diffusion_bench.py
index 11691700cf2..20105d89296 100644
--- a/scripts/diffusion_bench.py
+++ b/scripts/diffusion_bench.py
@@ -119,6 +119,27 @@ def _gpu_name() -> Optional[str]:
return None
+def _process_rss_bytes() -> Optional[int]:
+ """Best-effort current-process RSS, without making the benchmark depend on psutil."""
+ try:
+ import psutil
+ return int(psutil.Process().memory_info().rss)
+ except Exception:
+ pass
+ try:
+ if sys.platform.startswith("linux"):
+ resident_pages = int(Path("/proc/self/statm").read_text().split()[1])
+ return resident_pages * int(os.sysconf("SC_PAGE_SIZE"))
+ except Exception:
+ pass
+ return None
+
+
+def _finite_json_number(value: float) -> Optional[float]:
+ """Keep benchmark JSON RFC-compliant when a metric is infinite or unavailable."""
+ return value if math.isfinite(value) else None
+
+
def _versions() -> dict[str, Optional[str]]:
out: dict[str, Optional[str]] = {"torch": None, "diffusers": None}
try:
@@ -231,13 +252,18 @@ def _run(args: argparse.Namespace) -> dict[str, Any]:
status = backend.status()
print(f" loaded: {status}", flush = True)
+ rss_after_load = _process_rss_bytes()
+
# ── warmup (discarded) ──
for _ in range(max(0, args.warmup)):
_generate_once(backend, args)
+ rss_after_warmup = _process_rss_bytes()
+
# ── measured generations (fixed seed -> deterministic) ──
_cuda_reset_peak()
latencies: list[float] = []
+ rss_after_generations: list[Optional[int]] = []
first_image = None
for i in range(max(1, args.iters)):
_cuda_sync()
@@ -245,11 +271,19 @@ def _run(args: argparse.Namespace) -> dict[str, Any]:
image = _generate_once(backend, args)
_cuda_sync()
latencies.append(time.time() - g0)
+ rss_after_generations.append(_process_rss_bytes())
if first_image is None:
first_image = image
print(f" gen[{i}] {latencies[-1]:.3f}s", flush = True)
total = sum(latencies)
+ measured_rss = [value for value in rss_after_generations if value is not None]
+ post_warmup_rss_growth = (
+ max(0, max(measured_rss) - rss_after_warmup)
+ if rss_after_warmup is not None and measured_rss
+ else None
+ )
+
gen_metrics = {
"iters": len(latencies),
"warmup": max(0, args.warmup),
@@ -260,6 +294,12 @@ def _run(args: argparse.Namespace) -> dict[str, Any]:
if total > 0
else None,
"peak_vram_bytes": _cuda_peak_alloc(),
+ "host_rss": {
+ "after_load_bytes": rss_after_load,
+ "after_warmup_bytes": rss_after_warmup,
+ "after_each_generation_bytes": rss_after_generations,
+ "post_warmup_growth_bytes": post_warmup_rss_growth,
+ },
}
# The fixed-seed image is the accuracy anchor.
@@ -337,6 +377,10 @@ def _write_baseline(args: argparse.Namespace) -> int:
f"peak_vram={metrics['generate'].get('peak_vram_bytes')}",
flush = True,
)
+
+ rss_growth = (metrics["generate"].get("host_rss") or {}).get("post_warmup_growth_bytes")
+ if rss_growth is not None:
+ print(f" host RSS growth after warmup: {rss_growth / 2**20:.1f} MiB", flush = True)
return 0
@@ -345,6 +389,16 @@ def _compare(args: argparse.Namespace) -> int:
baseline = json.loads(baseline_path.read_text())
out_dir = Path(args.out_dir).resolve()
args._image_out = out_dir / "compare.png"
+ # --write-baseline takes any path, so a baseline can be sitting on a name this run writes.
+ # Refuse before the generation is paid for, not after.
+ for written in (out_dir / "compare.json", args._image_out):
+ if baseline_path == written:
+ print(
+ f"error: baseline {baseline_path} is the file this run writes; "
+ f"pass a different --out-dir or rename the baseline",
+ file = sys.stderr,
+ )
+ return 2
# Refuse a noisy cross-hardware / cross-dtype comparison unless forced.
base_env = baseline.get("env", {})
@@ -383,6 +437,9 @@ def _compare(args: argparse.Namespace) -> int:
cur_peak = cur_gen.get("peak_vram_bytes")
vram_reg = ((cur_peak - base_peak) / base_peak) if (base_peak and cur_peak) else 0.0
+ base_rss_growth = (base_gen.get("host_rss") or {}).get("post_warmup_growth_bytes")
+ cur_rss_growth = (cur_gen.get("host_rss") or {}).get("post_warmup_growth_bytes")
+
print("\n=== REGRESSION REPORT ===", flush = True)
print(f" {'metric':<22}{'baseline':>16}{'current':>16}{'delta':>12}", flush = True)
print(
@@ -394,6 +451,19 @@ def _compare(args: argparse.Namespace) -> int:
f" {'peak_vram_MB':<22}{base_peak / 1e6:>16.1f}{cur_peak / 1e6:>16.1f}{vram_reg * 100:>11.1f}%",
flush = True,
)
+
+ if base_rss_growth is not None or cur_rss_growth is not None:
+ base_rss_label = f"{base_rss_growth / 2**20:.1f}" if base_rss_growth is not None else "-"
+ cur_rss_label = f"{cur_rss_growth / 2**20:.1f}" if cur_rss_growth is not None else "-"
+ rss_delta_label = (
+ f"{(cur_rss_growth - base_rss_growth) / 2**20:+.1f}"
+ if base_rss_growth is not None and cur_rss_growth is not None
+ else "-"
+ )
+ print(
+ f" {'host_rss_growth_MiB':<22}{base_rss_label:>16}{cur_rss_label:>16}{rss_delta_label:>12}",
+ flush = True,
+ )
print(f" {'psnr_dB(vs ref)':<22}{'-':>16}{psnr:>16.2f}{'':>12}", flush = True)
failures = []
@@ -403,11 +473,30 @@ def _compare(args: argparse.Namespace) -> int:
)
if base_peak and cur_peak and vram_reg > args.max_vram_regression:
failures.append(f"peak VRAM +{vram_reg * 100:.1f}% > {args.max_vram_regression * 100:.0f}%")
+
+ if args.max_host_rss_growth_mib is not None:
+ if cur_rss_growth is None:
+ failures.append("host RSS unavailable; cannot verify the configured growth limit")
+ elif cur_rss_growth > args.max_host_rss_growth_mib * 2**20:
+ failures.append(
+ f"host RSS growth {cur_rss_growth / 2**20:.1f} MiB > "
+ f"{args.max_host_rss_growth_mib:.1f} MiB"
+ )
if math.isnan(psnr):
failures.append("PSNR reference image missing; cannot verify output quality")
elif psnr < args.min_psnr:
failures.append(f"PSNR {psnr:.2f}dB < {args.min_psnr:.1f}dB (output changed)")
+ metrics["comparison"] = {
+ "baseline_json": str(baseline_path),
+ "psnr_db": _finite_json_number(psnr),
+ "latency_regression": latency_reg,
+ "vram_regression": vram_reg,
+ "failures": list(failures),
+ }
+ out_dir.mkdir(parents = True, exist_ok = True)
+ (out_dir / "compare.json").write_text(json.dumps(metrics, indent = 2, allow_nan = False))
+
if failures:
print("\n FAIL: " + "; ".join(failures), flush = True)
return 1
@@ -504,6 +593,12 @@ def _build_parser() -> argparse.ArgumentParser:
default = 0.10,
help = "fail if peak generation VRAM rises by more than this fraction",
)
+ p.add_argument(
+ "--max-host-rss-growth-mib",
+ type = float,
+ default = None,
+ help = "fail comparison if peak post-warmup process RSS growth exceeds this many MiB",
+ )
p.add_argument(
"--min-psnr",
type = float,
@@ -523,6 +618,8 @@ def _build_parser() -> argparse.ArgumentParser:
def main(argv: Optional[list[str]] = None) -> int:
args = _build_parser().parse_args(argv)
+ if isinstance(args.gguf, str):
+ args.gguf = args.gguf.strip() or None
if bool(args.write_baseline) == bool(args.compare):
print("error: pass exactly one of --write-baseline / --compare", file = sys.stderr)
return 2
diff --git a/studio/backend/core/inference/diffusion.py b/studio/backend/core/inference/diffusion.py
index 579c560776b..64f1f8b6ee8 100644
--- a/studio/backend/core/inference/diffusion.py
+++ b/studio/backend/core/inference/diffusion.py
@@ -100,6 +100,7 @@ from .diffusion_memory import (
raise_on_image_activation_shortfall,
raise_on_unified_memory_shortfall,
reclaimable_snapshot_device_memory,
+ reclaim_offload_host_memory,
refine_memory_plan_for_components,
settled_snapshot_device_memory,
snapshot_device_memory,
@@ -6215,6 +6216,9 @@ class DiffusionBackend:
raise RuntimeError(DIFFUSION_CANCELLED_MSG)
if self._active_generate_cancel is cancel:
self._active_generate_cancel = None
+
+ # Components are offloaded and transfer copies dropped; return the pages now.
+ reclaim_offload_host_memory(state.offload_policy, logger = logger)
# Count the finished generation (drives deferred speed); a batch is one generation.
object.__setattr__(state, "generation_count", state.generation_count + 1)
return {
diff --git a/studio/backend/core/inference/diffusion_memory.py b/studio/backend/core/inference/diffusion_memory.py
index e35fdd4d798..b2076b2aec4 100644
--- a/studio/backend/core/inference/diffusion_memory.py
+++ b/studio/backend/core/inference/diffusion_memory.py
@@ -16,9 +16,11 @@ imported lazily.
from __future__ import annotations
+import functools
import os
+import sys
from dataclasses import dataclass, replace
-from typing import Any, Optional
+from typing import Any, Callable, Optional
# ── memory modes (operator intent) ───────────────────────────────────────────
@@ -53,6 +55,100 @@ DEFAULT_IMAGE_HEIGHT = 1024
DEFAULT_BASE_OVERHEAD_MIB = 2048
+_host_memory_reclaim_warning_logged = False
+_host_memory_reclaim_unsupported_logged = False
+
+# Optional: a Python without _ctypes must not break the inference stack that imports this module.
+# Must stay a module attribute, not a lazy local: the reclaimer tests monkeypatch it.
+try:
+ import ctypes
+except Exception: # noqa: BLE001
+ ctypes = None # type: ignore[assignment]
+
+
[email protected]_cache(maxsize = 1)
+def _resolve_host_memory_reclaimer() -> Optional[Callable[[], None]]:
+ """Resolve this process allocator's native pressure API once, if the OS exposes one."""
+ if ctypes is None:
+ return None
+ try:
+ if sys.platform.startswith("linux"):
+ allocator = ctypes.CDLL(None)
+ pressure = allocator.malloc_trim
+ pressure.argtypes = [ctypes.c_size_t]
+ pressure.restype = ctypes.c_int
+
+ def reclaim() -> None:
+ pressure(0)
+
+ elif sys.platform == "darwin":
+ allocator = ctypes.CDLL(None)
+ pressure = allocator.malloc_zone_pressure_relief
+ pressure.argtypes = [ctypes.c_void_p, ctypes.c_size_t]
+ pressure.restype = ctypes.c_size_t
+
+ def reclaim() -> None:
+ pressure(None, 0)
+
+ elif sys.platform == "win32":
+ pressure = None
+ for library_name in ("ucrtbase.dll", "msvcrt.dll"):
+ try:
+ allocator = ctypes.CDLL(library_name)
+ pressure = allocator._heapmin
+ break
+ except Exception:
+ continue
+ if pressure is None:
+ return None
+ pressure.argtypes = []
+ pressure.restype = ctypes.c_int
+
+ def reclaim() -> None:
+ if pressure() == -1:
+ raise OSError("_heapmin failed")
+
+ else:
+ return None
+ except Exception:
+ return None
+ return reclaim
+
+
+def reclaim_offload_host_memory(offload_policy: str, logger: Any = None) -> bool:
+ """Return unused allocator pages after whole-model CPU offload, without touching live
+ tensors, Python GC, or device caches. Unsupported allocators and failures are non-fatal."""
+ global _host_memory_reclaim_warning_logged
+ global _host_memory_reclaim_unsupported_logged
+ if offload_policy != OFFLOAD_MODEL:
+ return False
+ try:
+ reclaim = _resolve_host_memory_reclaimer()
+ if reclaim is None:
+ # The call site discards the result, so a permanent no-op is otherwise invisible.
+ if logger is not None and not _host_memory_reclaim_unsupported_logged:
+ _host_memory_reclaim_unsupported_logged = True
+ try:
+ logger.info(
+ "diffusion.memory: no host allocator pressure API on this platform "
+ "(%s); offloaded host pages will not be returned early",
+ sys.platform,
+ )
+ except Exception: # noqa: BLE001
+ pass
+ return False
+ reclaim()
+ return True
+ except Exception as exc: # noqa: BLE001
+ if logger is not None and not _host_memory_reclaim_warning_logged:
+ _host_memory_reclaim_warning_logged = True
+ try:
+ logger.warning("diffusion.memory: host allocator reclamation failed: %s", exc)
+ except Exception: # noqa: BLE001
+ pass
+ return False
+
+
def normalize_memory_mode(value: Optional[str]) -> Optional[str]:
"""Lower/strip a requested mode (accepting dashes); None passes through. Raises ValueError
for an unsupported mode so the route rejects it as a 4xx before any GPU work."""
diff --git a/studio/backend/core/inference/video.py b/studio/backend/core/inference/video.py
index e5abcd78901..c6c87c9522d 100644
--- a/studio/backend/core/inference/video.py
+++ b/studio/backend/core/inference/video.py
@@ -79,6 +79,7 @@ from .diffusion_memory import (
normalize_memory_mode,
plan_diffusion_memory,
raise_on_unified_memory_shortfall,
+ reclaim_offload_host_memory,
settled_snapshot_device_memory,
)
from .diffusion_speed import (
@@ -5711,6 +5712,15 @@ class VideoBackend:
raise RuntimeError(VIDEO_CANCELLED_MSG)
duration_s = len(video_frames) / float(out_fps) if out_fps else 0.0
self._gen = {"active": False}
+ # Deregister under cancel_generate's own lock before the trim: it blocks for a few
+ # hundred ms after the last is_set() check, so a Stop landing there would be
+ # answered true while this clip is still returned and persisted.
+ with self._lock:
+ if cancel.is_set():
+ raise RuntimeError(VIDEO_CANCELLED_MSG)
+ if self._active_generate_cancel is cancel:
+ self._active_generate_cancel = None
+ reclaim_offload_host_memory(state.offload_policy, logger = logger)
return {
"mp4_bytes": mp4_bytes,
"seed": int(seed),
diff --git a/studio/backend/tests/test_diffusion_backend.py b/studio/backend/tests/test_diffusion_backend.py
index 2e99b8132f0..a4467d4e502 100644
--- a/studio/backend/tests/test_diffusion_backend.py
+++ b/studio/backend/tests/test_diffusion_backend.py
@@ -6496,6 +6496,57 @@ def test_generate_non_oom_error_is_not_retried(fake_runtime, tmp_path):
assert pipe.batch_attempts == [4] # no backoff retries on a non-OOM error
+def test_generate_reclaims_model_offload_memory_once_after_success(
+ fake_runtime, tmp_path, monkeypatch
+):
+ from core.inference import diffusion as dmod
+
+ backend = _load_zimage_backend(tmp_path)
+ trace = []
+
+ def fake_reclaim(policy, logger = None):
+ if policy == "model":
+ trace.append("reclaim")
+ return True
+ return False
+
+ monkeypatch.setattr(dmod, "reclaim_offload_host_memory", fake_reclaim, raising = False)
+ monkeypatch.setattr(dmod.compile_cache, "register_shape", lambda *a, **k: None)
+ monkeypatch.setattr(dmod.compile_cache, "save", lambda *a, **k: trace.append("save"))
+
+ object.__setattr__(backend._state, "offload_policy", "model")
+ object.__setattr__(backend._state, "pipe", _CountingPipe(max_images = 2))
+ out = backend.generate(prompt = "p", seeds = [1, 2, 3, 4])
+ assert len(out["images"]) == 4
+ assert trace == ["save", "reclaim"]
+
+ for policy in ("none", "group", "streaming", "sequential"):
+ object.__setattr__(backend._state, "offload_policy", policy)
+ object.__setattr__(backend._state, "pipe", _FakePipe())
+ backend.generate(prompt = policy)
+ assert trace.count("reclaim") == 1
+
+ object.__setattr__(backend._state, "offload_policy", "model")
+ object.__setattr__(backend._state, "pipe", _BoomPipe())
+ with pytest.raises(RuntimeError, match = "shape mismatch"):
+ backend.generate(prompt = "failed")
+
+ class _CancellingPipe(_FakePipe):
+ def __call__(
+ self,
+ *,
+ prompt = None,
+ **kwargs,
+ ):
+ assert backend.cancel_generate() is True
+ return super().__call__(prompt = prompt, **kwargs)
+
+ object.__setattr__(backend._state, "pipe", _CancellingPipe())
+ with pytest.raises(RuntimeError, match = DIFFUSION_CANCELLED_MSG):
+ backend.generate(prompt = "cancelled")
+ assert trace.count("reclaim") == 1
+
+
def test_generate_broadcasts_negative_prompt_across_a_mixed_prompt_batch(fake_runtime, tmp_path):
# A prompt list needs a matching negative list: encode_prompt asserts equal lengths, and pipes that encode the negative separately would build batch-1 embeds against batch-N latents.
backend = _load_zimage_backend(tmp_path)
diff --git a/studio/backend/tests/test_diffusion_bench.py b/studio/backend/tests/test_diffusion_bench.py
new file mode 100644
index 00000000000..36034add329
--- /dev/null
+++ b/studio/backend/tests/test_diffusion_bench.py
@@ -0,0 +1,163 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
+
+"""Hermetic tests for the standalone diffusion benchmark report."""
+
+from __future__ import annotations
+
+import importlib.util
+import json
+import math
+import sys
+from pathlib import Path
+from types import SimpleNamespace
+
+import pytest
+
+
+_REPO_ROOT = Path(__file__).resolve().parents[3]
+_SPEC = importlib.util.spec_from_file_location(
+ "diffusion_bench", _REPO_ROOT / "scripts" / "diffusion_bench.py"
+)
+assert _SPEC is not None and _SPEC.loader is not None
+_DIFFUSION_BENCH = importlib.util.module_from_spec(_SPEC)
+_SPEC.loader.exec_module(_DIFFUSION_BENCH)
+
+
+ ("psnr", "expected_exit"),
+ ((math.inf, 0), (-math.inf, 1), (math.nan, 1)),
+)
+def test_compare_report_serializes_non_finite_psnr_as_strict_json(
+ tmp_path, monkeypatch, psnr, expected_exit
+):
+ reference = tmp_path / "reference.png"
+ reference.write_bytes(b"reference")
+ baseline = {
+ "env": {
+ "gpu_name": "test-gpu",
+ "status": {"device": "cuda", "dtype": "bfloat16"},
+ },
+ "generate": {
+ "median_latency_s": 1.0,
+ "peak_vram_bytes": 100,
+ "host_rss": {"post_warmup_growth_bytes": 0},
+ },
+ "accuracy": {"reference_png": str(reference)},
+ }
+ baseline_path = tmp_path / "baseline.json"
+ baseline_path.write_text(json.dumps(baseline))
+ current = {
+ "env": {"status": {"device": "cuda", "dtype": "bfloat16"}},
+ "generate": {
+ "median_latency_s": 1.0,
+ "peak_vram_bytes": 100,
+ "host_rss": {"post_warmup_growth_bytes": 0},
+ },
+ }
+ monkeypatch.setattr(_DIFFUSION_BENCH, "_gpu_name", lambda: "test-gpu")
+ monkeypatch.setattr(_DIFFUSION_BENCH, "_run", lambda args: current)
+ monkeypatch.setattr(_DIFFUSION_BENCH, "_psnr", lambda ref, candidate: psnr)
+ out_dir = tmp_path / "out"
+ args = SimpleNamespace(
+ compare = str(baseline_path),
+ out_dir = str(out_dir),
+ force_compare = False,
+ max_latency_regression = 0.1,
+ max_vram_regression = 0.1,
+ max_host_rss_growth_mib = 1.0,
+ min_psnr = 30.0,
+ )
+
+ assert _DIFFUSION_BENCH._compare(args) == expected_exit
+ report_text = (out_dir / "compare.json").read_text()
+ report = json.loads(
+ report_text,
+ parse_constant = lambda token: pytest.fail(f"non-standard JSON constant: {token}"),
+ )
+ assert report["comparison"]["psnr_db"] is None
+
+
+def test_process_rss_is_plausible_or_absent():
+ value = _DIFFUSION_BENCH._process_rss_bytes()
+ assert value is None or (isinstance(value, int) and 2**20 < value < 2**44)
+
+
[email protected](not sys.platform.startswith("linux"), reason = "/proc fallback is Linux only")
+def test_process_rss_falls_back_to_proc_without_psutil(monkeypatch):
+ """The /proc branch is unreachable wherever psutil is installed, which is everywhere the
+ benchmark normally runs, so drive it directly rather than leaving it unexercised."""
+ real_import = __import__
+
+ def without_psutil(name, *args, **kwargs):
+ if name == "psutil":
+ raise ImportError("psutil unavailable")
+ return real_import(name, *args, **kwargs)
+
+ monkeypatch.setattr("builtins.__import__", without_psutil)
+ value = _DIFFUSION_BENCH._process_rss_bytes()
+ assert isinstance(value, int) and value > 0
+
+
[email protected]("name", ["compare.json", "compare.png"])
+def test_compare_refuses_to_overwrite_its_own_baseline(tmp_path, monkeypatch, name):
+ """--write-baseline accepts any path, so a baseline can legitimately be sitting on one of the
+ names the compare run writes. Refuse instead of destroying the reference metrics, and refuse
+ BEFORE the generation rather than after paying for it.
+
+ _run and _psnr are stubbed so that without the guard this reaches the real write and
+ clobbers the baseline; otherwise the case would pass for the wrong reason.
+ """
+ out_dir = tmp_path / "out"
+ out_dir.mkdir()
+ reference = tmp_path / "reference.png"
+ reference.write_bytes(b"reference")
+ baseline_path = out_dir / name
+ baseline_path.write_text(
+ json.dumps(
+ {
+ "env": {"gpu_name": "test-gpu", "status": {"device": "cuda", "dtype": "bfloat16"}},
+ "generate": {
+ "median_latency_s": 1.0,
+ "peak_vram_bytes": 100,
+ "host_rss": {"post_warmup_growth_bytes": 0},
+ },
+ "accuracy": {"reference_png": str(reference)},
+ }
+ )
+ )
+ before = baseline_path.read_text()
+
+ monkeypatch.setattr(_DIFFUSION_BENCH, "_gpu_name", lambda: "test-gpu")
+ monkeypatch.setattr(
+ _DIFFUSION_BENCH,
+ "_run",
+ lambda args: {
+ "env": {"status": {"device": "cuda", "dtype": "bfloat16"}},
+ "generate": {
+ "median_latency_s": 1.0,
+ "peak_vram_bytes": 100,
+ "host_rss": {"post_warmup_growth_bytes": 0},
+ },
+ },
+ )
+ monkeypatch.setattr(_DIFFUSION_BENCH, "_psnr", lambda ref, candidate: 99.0)
+
+ args = SimpleNamespace(
+ compare = str(baseline_path),
+ out_dir = str(out_dir),
+ force_compare = False,
+ max_latency_regression = 0.1,
+ max_vram_regression = 0.1,
+ max_host_rss_growth_mib = None,
+ min_psnr = 30.0,
+ )
+ assert _DIFFUSION_BENCH._compare(args) == 2
+ assert baseline_path.read_text() == before, "the baseline was overwritten"
+
+
+def test_host_rss_guard_is_off_unless_requested():
+ """Back-compat for existing scripted callers: the new threshold only engages when asked
+ for, so an invocation written before this flag existed behaves exactly as it did."""
+ args = _DIFFUSION_BENCH._build_parser().parse_args(["--write-baseline", "out.json"])
+ assert args.max_host_rss_growth_mib is None
diff --git a/studio/backend/tests/test_diffusion_memory.py b/studio/backend/tests/test_diffusion_memory.py
index 4b7de2ee014..3fca544b59a 100644
--- a/studio/backend/tests/test_diffusion_memory.py
+++ b/studio/backend/tests/test_diffusion_memory.py
@@ -10,8 +10,10 @@ matrix and the applier's pipeline calls are exercised in isolation.
from __future__ import annotations
+import sys
import types
+import core.inference.diffusion_memory as diffusion_memory
import pytest
from core.inference.diffusion_memory import (
@@ -69,6 +71,204 @@ def test_normalize_memory_mode_accepts_and_rejects():
normalize_memory_mode("ultra")
+class _FakeNativeFunction:
+ def __init__(self, result = 0):
+ self.argtypes = None
+ self.restype = None
+ self.calls = []
+ self.result = result
+
+ def __call__(self, *args):
+ self.calls.append(args)
+ return self.result
+
+
+ ("platform_name", "library_name", "symbol", "native_args"),
+ [
+ ("linux", None, "malloc_trim", (0,)),
+ ("darwin", None, "malloc_zone_pressure_relief", (None, 0)),
+ ("win32", "ucrtbase.dll", "_heapmin", ()),
+ ],
+)
+def test_host_memory_reclaimer_uses_and_caches_the_native_api(
+ monkeypatch, platform_name, library_name, symbol, native_args
+):
+ native = _FakeNativeFunction()
+ library_loads = []
+
+ def fake_cdll(name):
+ library_loads.append(name)
+ return types.SimpleNamespace(**{symbol: native})
+
+ monkeypatch.setattr(diffusion_memory.sys, "platform", platform_name)
+ monkeypatch.setattr(diffusion_memory.ctypes, "CDLL", fake_cdll)
+ diffusion_memory._resolve_host_memory_reclaimer.cache_clear()
+ try:
+ assert diffusion_memory.reclaim_offload_host_memory(OFFLOAD_MODEL) is True
+ assert diffusion_memory.reclaim_offload_host_memory(OFFLOAD_MODEL) is True
+ finally:
+ diffusion_memory._resolve_host_memory_reclaimer.cache_clear()
+
+ assert library_loads == [library_name]
+ assert native.calls == [native_args, native_args]
+ assert native.argtypes is not None and native.restype is not None
+
+
+def test_host_memory_reclaimer_windows_falls_back_and_logs_heap_failure_once(monkeypatch):
+ native = _FakeNativeFunction(result = -1)
+ library_loads = []
+ warnings = []
+
+ def fake_cdll(name):
+ library_loads.append(name)
+ if name == "ucrtbase.dll":
+ raise OSError("UCRT unavailable")
+ return types.SimpleNamespace(_heapmin = native)
+
+ monkeypatch.setattr(diffusion_memory.sys, "platform", "win32")
+ monkeypatch.setattr(diffusion_memory.ctypes, "CDLL", fake_cdll)
+ monkeypatch.setattr(diffusion_memory, "_host_memory_reclaim_warning_logged", False)
+ diffusion_memory._resolve_host_memory_reclaimer.cache_clear()
+ logger = types.SimpleNamespace(warning = lambda *args: warnings.append(args))
+ try:
+ assert diffusion_memory.reclaim_offload_host_memory(OFFLOAD_MODEL, logger) is False
+ assert diffusion_memory.reclaim_offload_host_memory(OFFLOAD_MODEL, logger) is False
+ finally:
+ diffusion_memory._resolve_host_memory_reclaimer.cache_clear()
+
+ assert library_loads == ["ucrtbase.dll", "msvcrt.dll"]
+ assert native.calls == [(), ()]
+ assert len(warnings) == 1
+
+
+def test_host_memory_reclaimer_is_policy_scoped_and_best_effort(monkeypatch):
+ calls = []
+ monkeypatch.setattr(
+ diffusion_memory, "_resolve_host_memory_reclaimer", lambda: lambda: calls.append(0)
+ )
+
+ for policy in (OFFLOAD_NONE, OFFLOAD_GROUP, OFFLOAD_STREAMING, OFFLOAD_SEQUENTIAL):
+ assert diffusion_memory.reclaim_offload_host_memory(policy) is False
+ assert calls == []
+ assert diffusion_memory.reclaim_offload_host_memory(OFFLOAD_MODEL) is True
+ assert calls == [0]
+
+ def resolver_failure():
+ raise OSError("allocator unavailable")
+
+ monkeypatch.setattr(diffusion_memory, "_resolve_host_memory_reclaimer", resolver_failure)
+ assert diffusion_memory.reclaim_offload_host_memory(OFFLOAD_MODEL) is False
+
+ def call_failure():
+ raise RuntimeError("pressure API failed")
+
+ monkeypatch.setattr(diffusion_memory, "_resolve_host_memory_reclaimer", lambda: call_failure)
+ assert diffusion_memory.reclaim_offload_host_memory(OFFLOAD_MODEL) is False
+
+
+def test_host_memory_reclaimer_caches_unsupported_or_missing_apis(monkeypatch):
+ loads = []
+ monkeypatch.setattr(diffusion_memory.sys, "platform", "linux")
+ monkeypatch.setattr(
+ diffusion_memory.ctypes,
+ "CDLL",
+ lambda name: loads.append(name) or types.SimpleNamespace(),
+ )
+ diffusion_memory._resolve_host_memory_reclaimer.cache_clear()
+ try:
+ assert diffusion_memory.reclaim_offload_host_memory(OFFLOAD_MODEL) is False
+ assert diffusion_memory.reclaim_offload_host_memory(OFFLOAD_MODEL) is False
+ finally:
+ diffusion_memory._resolve_host_memory_reclaimer.cache_clear()
+ assert loads == [None]
+
+ monkeypatch.setattr(diffusion_memory.sys, "platform", "haiku")
+ monkeypatch.setattr(
+ diffusion_memory.ctypes,
+ "CDLL",
+ lambda name: pytest.fail("unsupported platforms must not load an allocator library"),
+ )
+ diffusion_memory._resolve_host_memory_reclaimer.cache_clear()
+ try:
+ assert diffusion_memory.reclaim_offload_host_memory(OFFLOAD_MODEL) is False
+ finally:
+ diffusion_memory._resolve_host_memory_reclaimer.cache_clear()
+
+
+ not sys.platform.startswith("linux"),
+ reason = "exercises the real glibc allocator symbol",
+)
+def test_host_memory_reclaimer_calls_the_real_glibc_symbol():
+ """Every other reclaimer test replaces ctypes, so none of them would notice if the library,
+ the symbol name or the ABI stopped resolving against a real libc. Spy on the real CDLL
+ instead of standing in for it, so the assertions land on the very function pointer the
+ resolver configured (a fresh CDLL(None).malloc_trim would carry default argtypes and prove
+ nothing)."""
+ import ctypes
+
+ if not hasattr(ctypes.CDLL(None), "malloc_trim"):
+ pytest.skip("no malloc_trim on this libc (musl); the resolver correctly declines")
+
+ real_cdll = ctypes.CDLL
+ captured = {}
+
+ def spy(name, *args, **kwargs):
+ library = real_cdll(name, *args, **kwargs)
+ captured["library"] = library
+ return library
+
+ monkeypatch_target = diffusion_memory.ctypes
+ original = monkeypatch_target.CDLL
+ monkeypatch_target.CDLL = spy
+ diffusion_memory._resolve_host_memory_reclaimer.cache_clear()
+ try:
+ assert diffusion_memory.reclaim_offload_host_memory(OFFLOAD_MODEL) is True
+ pressure = captured["library"].malloc_trim
+ assert pressure.argtypes == [ctypes.c_size_t]
+ assert pressure.restype is ctypes.c_int
+ # glibc: 1 = released to the OS, 0 = could not.
+ assert pressure(0) in (0, 1)
+ finally:
+ monkeypatch_target.CDLL = original
+ diffusion_memory._resolve_host_memory_reclaimer.cache_clear()
+
+
+def test_host_memory_reclaimer_survives_a_python_without_ctypes(monkeypatch):
+ """A Python built without libffi has no _ctypes. This module is imported by diffusion,
+ video, llama_cpp, sd_cpp_backend and routes.inference, so a hard ctypes dependency would
+ turn a missing optional optimisation into a backend that cannot start at all."""
+ monkeypatch.setattr(diffusion_memory, "ctypes", None)
+ diffusion_memory._resolve_host_memory_reclaimer.cache_clear()
+ try:
+ assert diffusion_memory._resolve_host_memory_reclaimer() is None
+ assert diffusion_memory.reclaim_offload_host_memory(OFFLOAD_MODEL) is False
+ assert diffusion_memory.reclaim_offload_host_memory(OFFLOAD_NONE) is False
+ finally:
+ diffusion_memory._resolve_host_memory_reclaimer.cache_clear()
+
+
+def test_host_memory_reclaimer_reports_an_unsupported_allocator_once(monkeypatch):
+ """The call site discards the return value, so an allocator we can never trim has to say so
+ itself or a permanent no-op is invisible while host RAM climbs."""
+ messages = []
+ logger = types.SimpleNamespace(
+ info = lambda fmt, *args: messages.append(fmt % args),
+ warning = lambda fmt, *args: messages.append(fmt % args),
+ )
+ monkeypatch.setattr(diffusion_memory, "_host_memory_reclaim_unsupported_logged", False)
+ monkeypatch.setattr(diffusion_memory.sys, "platform", "haiku")
+ diffusion_memory._resolve_host_memory_reclaimer.cache_clear()
+ try:
+ assert diffusion_memory.reclaim_offload_host_memory(OFFLOAD_MODEL, logger = logger) is False
+ assert diffusion_memory.reclaim_offload_host_memory(OFFLOAD_MODEL, logger = logger) is False
+ finally:
+ diffusion_memory._resolve_host_memory_reclaimer.cache_clear()
+ assert len(messages) == 1, messages
+ assert "haiku" in messages[0]
+
+
# ── filename / size estimates ─────────────────────────────────────────────────
diff --git a/studio/backend/tests/test_video_backend.py b/studio/backend/tests/test_video_backend.py
index 83bd43578bd..f48d1dcdb67 100644
--- a/studio/backend/tests/test_video_backend.py
+++ b/studio/backend/tests/test_video_backend.py
@@ -877,6 +877,66 @@ def test_load_generate_unload_gguf(fake_runtime, tmp_path):
assert status["loaded"] is False
+def test_generate_reclaims_model_offload_host_memory(fake_runtime, tmp_path, monkeypatch):
+ """Video shares the image path's whole-model offload lifecycle, and its weights are larger,
+ so the same host-allocator retention applies. Reclaim once per successful generation, only
+ under the 'model' policy, and only after progress has been cleared."""
+ from core.inference import video as video_mod
+
+ trace = []
+ monkeypatch.setattr(
+ video_mod,
+ "reclaim_offload_host_memory",
+ lambda policy, logger = None: trace.append(policy) or True,
+ )
+
+ backend = VideoBackend()
+ _load_gguf(backend, tmp_path)
+
+ for policy in ("none", "group", "streaming", "sequential", "model"):
+ backend._state = dataclasses.replace(backend._state, offload_policy = policy)
+ backend.generate(prompt = "a sloth surfing", width = 256, height = 256, num_frames = 9, fps = 8)
+ assert trace == ["none", "group", "streaming", "sequential", "model"]
+
+ from core.inference import diffusion_memory
+
+ monkeypatch.setattr(diffusion_memory, "_resolve_host_memory_reclaimer", lambda: lambda: None)
+ assert [diffusion_memory.reclaim_offload_host_memory(p) for p in trace] == [
+ False,
+ False,
+ False,
+ False,
+ True,
+ ]
+
+ # Progress must already be idle: the trim can block for a few hundred ms.
+ assert backend.generate_progress()["active"] is False
+
+
+def test_a_stop_during_the_reclaim_cannot_be_answered_true(fake_runtime, tmp_path, monkeypatch):
+ """The trim blocks for a few hundred ms after the last is_set() check, so a Stop landing in
+ that window used to be told it succeeded while the clip was still returned and persisted.
+ The event must be deregistered before the trim, under the lock cancel_generate takes."""
+ from core.inference import video as video_mod
+
+ backend = VideoBackend()
+ _load_gguf(backend, tmp_path)
+ backend._state = dataclasses.replace(backend._state, offload_policy = "model")
+
+ answered = []
+
+ def stop_midway(policy, logger = None):
+ # Press Stop while the stubbed blocking trim runs.
+ answered.append(backend.cancel_generate())
+ return True
+
+ monkeypatch.setattr(video_mod, "reclaim_offload_host_memory", stop_midway)
+
+ result = backend.generate(prompt = "a sloth surfing", width = 256, height = 256, num_frames = 9, fps = 8)
+ assert result["mp4_bytes"] == b"MP4"
+ assert answered == [False], "cancel_generate answered true for a clip that was still persisted"
+
+
def test_load_holds_generate_lock_across_placement(fake_runtime, tmp_path, monkeypatch):
# The video load must hold _generate_lock across GPU placement so an unload -- which barriers on that lock -- cannot
# hand the GPU away mid-move. unload() must block until placement releases it, and the superseded load then aborts.
@@ -1592,9 +1652,9 @@ def test_video_speed_off_suppresses_auto_dtype_quant(fake_runtime, monkeypatch):
# select reseeds to none, so after the user changes Speed and reloads, quantisation stays
# pinned off for no reason they can see.
resolved = status["resolved"]["transformer_quant"]
- assert (
- resolved["requested"] is None
- ), f"the record reports {resolved['requested']!r} as the user's request; nothing was asked for"
+ assert resolved["requested"] is None, (
+ f"the record reports {resolved['requested']!r} as the user's request; nothing was asked for"
+ )
assert resolved["source"] == "auto"
# Control: with speed NOT off the auto precision promotion still engages, so the suppression above is specific to speed=off.
@@ -3103,9 +3163,9 @@ def test_the_h3_loader_optimises_the_partition_it_denoises_with():
]
assert len(calls) == 1, f"{helper} is called {len(calls)} times"
first = calls[0].args[0]
- assert (
- getattr(first, "id", None) == "speed_view"
- ), f"{helper} is handed {ast.dump(first)}, not the denoiser view"
+ assert getattr(first, "id", None) == "speed_view", (
+ f"{helper} is handed {ast.dump(first)}, not the denoiser view"
+ )
def test_download_plan_adds_no_prequant_entry_when_bfloat16_is_pinned(monkeypatch):
@@ -4770,9 +4830,9 @@ def test_unload_fences_a_generation_queued_behind_its_barrier(fake_runtime, tmp_
queued = _run_teardown_race(backend, backend.unload)
- assert (
- "out" not in queued
- ), "a generation queued behind the unload barrier ran against a pipeline being torn down"
+ assert "out" not in queued, (
+ "a generation queued behind the unload barrier ran against a pipeline being torn down"
+ )
assert queued.get("error") in (VIDEO_NOT_LOADED_MSG, VIDEO_CANCELLED_MSG), queued
assert backend._state is None
assert backend._teardown_waiters == 0 # the fence drained
@@ -4785,9 +4845,9 @@ def test_superseding_load_fences_a_generation_queued_behind_its_barrier(fake_run
queued = _run_teardown_race(backend, lambda: _load_gguf(backend, tmp_path))
- assert (
- "out" not in queued
- ), "a generation queued behind the load barrier ran against a pipeline being torn down"
+ assert "out" not in queued, (
+ "a generation queued behind the load barrier ran against a pipeline being torn down"
+ )
assert queued.get("error") in (VIDEO_NOT_LOADED_MSG, VIDEO_CANCELLED_MSG), queued
assert backend._teardown_waiters == 0 # the fence drained
@@ -5033,9 +5093,9 @@ def test_the_h3_conditioner_falls_back_to_a_cache_that_predates_the_move(monkeyp
monkeypatch.setattr(diffusion_families, "_upstream_is_cached", _probe)
assert te.h3_te_quant_source("int8") == te.H3_LEGACY_TE_QUANT_REPO
- assert asked == [
- (te.H3_LEGACY_TE_QUANT_REPO, (te.H3_TE_QUANT_FILES["int8"],), True)
- ], "the conditioner must be probed by its own filename, in both cache roots"
+ assert asked == [(te.H3_LEGACY_TE_QUANT_REPO, (te.H3_TE_QUANT_FILES["int8"],), True)], (
+ "the conditioner must be probed by its own filename, in both cache roots"
+ )
monkeypatch.setattr(diffusion_families, "_upstream_is_cached", lambda *a, **k: False)
assert te.h3_te_quant_source("int8") == te.H3_TE_QUANT_REPO
@@ -5316,9 +5376,9 @@ def test_the_h3_native_path_pins_cfg_scale_to_one():
assert calls, "video.py no longer builds native video params"
for call in calls:
pinned = [kw for kw in call.keywords if kw.arg == "cfg_scale"]
- assert (
- pinned
- ), "the H3 native path must pass cfg_scale explicitly, not fall back to a default"
+ assert pinned, (
+ "the H3 native path must pass cfg_scale explicitly, not fall back to a default"
+ )
value = pinned[0].value
assert isinstance(value, ast.Constant), (
"cfg_scale must be a literal 1.0 on the H3 native path; forwarding the request's "
@@ -7961,9 +8021,9 @@ def test_plan_is_unchanged_when_no_text_encoder_quant_is_requested(fake_runtime,
continue
calls.clear()
VideoBackend().load_pipeline(fam.base_repo, model_kind = "pipeline")
- assert calls[0]["model_dense_mib"] == int(
- sum(fam.bf16_components_gb) * _MIB_PER_GB
- ), fam.name
+ assert calls[0]["model_dense_mib"] == int(sum(fam.bf16_components_gb) * _MIB_PER_GB), (
+ fam.name
+ )
def test_dense_quant_replan_uses_the_scaled_text_encoder(fake_runtime, monkeypatch):
@@ -8660,9 +8720,9 @@ def test_generation_in_flight_tracks_a_background_job(fake_runtime, tmp_path, mo
assert video_mod.generation_in_flight() is False
backend.begin_generate(prompt = "a clip")
assert rendering.wait(10), "the generate worker never started"
- assert (
- video_mod.generation_in_flight() is True
- ), "liveness cannot tell this backend from a dead one while it renders a clip"
+ assert video_mod.generation_in_flight() is True, (
+ "liveness cannot tell this backend from a dead one while it renders a clip"
+ )
assert inside["in_flight"] is True
release.set()
@@ -8967,9 +9027,9 @@ def test_a_direct_worker_call_keeps_the_outcome_it_recorded(fake_runtime, tmp_pa
backend._run_generate(cancel_event = threading.Event(), prompt = "a clip")
progress = backend.generate_progress()
- assert (
- progress["phase"] == "completed"
- ), f"a direct call recorded {progress['phase']!r}; the backstop overwrote its result"
+ assert progress["phase"] == "completed", (
+ f"a direct call recorded {progress['phase']!r}; the backstop overwrote its result"
+ )
assert progress["video"]["id"] == "direct"
@@ -8990,6 +9050,6 @@ def test_a_direct_worker_call_keeps_its_cancellation(fake_runtime, tmp_path, mon
progress = backend.generate_progress()
assert progress["phase"] == "failed"
- assert (
- progress["error"] == VIDEO_CANCELLED_MSG
- ), f"a direct call reported {progress['error']!r} instead of the cancellation sentinel"
+ assert progress["error"] == VIDEO_CANCELLED_MSG, (
+ f"a direct call reported {progress['error']!r} instead of the cancellation sentinel"
+ )