Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

Studio diffusion: let a 12.7 GB host load FLUX.2-klein-9B by spilling the int8 denoiser to a scratch file (#15911)

Daniel Han2572ed547f72
diff --git a/studio/backend/core/inference/diffusion.py b/studio/backend/core/inference/diffusion.py
index 221455a1337..be5a34bac95 100644
--- a/studio/backend/core/inference/diffusion.py
+++ b/studio/backend/core/inference/diffusion.py
@@ -174,6 +174,7 @@ from .diffusion_small_host import (
host_commit_mib,
host_ram_mib,
resolve_snapshot_dir,
+ spill_capacity_mib,
stored_components,
torch_dtype_map as small_host_torch_dtype_map,
)
@@ -11995,6 +11996,7 @@ class DiffusionBackend:
lora_active = lora_active,
host_commit_mib = host_commit_mib(),
pread_libraries = ("transformers",) if pread_available() else (),
+ spill_free_mib = spill_capacity_mib(),
)
if decision.engaged or decision.refuse:
logger.info(
@@ -12005,6 +12007,13 @@ class DiffusionBackend:
)
elif decision.available_mib is not None:
logger.info("diffusion.small_host: engaged=False (%s)", decision.reason)
+ if decision.int8_spill:
+ logger.info(
+ "diffusion.small_host: int8 denoiser spills to a memory-mapped scratch file (host estimate %d MiB "
+ "instead of %d MiB)",
+ decision.spill_route_host_mib,
+ decision.route_host_mib,
+ )
if decision.refuse:
raise RuntimeError(decision.refuse)
return decision
@@ -12030,6 +12039,7 @@ class DiffusionBackend:
mark,
prepare_streamed_encoder_,
quantize_int8_weight_,
+ spill_dir,
)
device = target.torch_device
@@ -12051,16 +12061,29 @@ class DiffusionBackend:
else:
# pageable: pinning rounds blocks to powers of two (11.3 GB of FLUX.1 int8 held 18 GB pinned)
act_int8 = int8_act_family(fam) and int8_act_device_ok(device)
- stats = quantize_int8_weight_(
- module,
- compute_dtype = dtype,
- work_device = device,
- keep_device = "cpu",
- act_int8 = act_int8,
- )
+ spill_to = spill_dir() if getattr(decision, "int8_spill", False) else None
+ try:
+ stats = quantize_int8_weight_(
+ module,
+ compute_dtype = dtype,
+ work_device = device,
+ keep_device = "cpu",
+ act_int8 = act_int8,
+ spill_dir = spill_to,
+ )
+ except OSError as exc:
+ if spill_to is None:
+ raise
+ # in-RAM route does not fit: no fallback
+ raise RuntimeError(
+ f"Not enough system RAM to load this model, and the int8 scratch file in {spill_to} "
+ f"could not be created ({exc}). Point UNSLOTH_DIFFUSION_SMALL_HOST_SPILL_DIR at a disk "
+ f"with {decision.route_host_mib / 1024 + 2:.0f} GB free."
+ ) from exc
info["components"][name] = (
f"int8 weights ({stats['int8_bytes'] >> 20} MiB, {stats['linears']} linears"
+ (", int8 activations" if act_int8 else "")
+ + (", memory-mapped scratch file" if stats.get("spilled_bytes") else "")
+ ")"
)
if load_token is not None:
diff --git a/studio/backend/core/inference/diffusion_memory.py b/studio/backend/core/inference/diffusion_memory.py
index 671c74f19c2..5ecf1b9a38d 100644
--- a/studio/backend/core/inference/diffusion_memory.py
+++ b/studio/backend/core/inference/diffusion_memory.py
@@ -3389,10 +3389,11 @@ _MEASURED_DENSE_BF16_EAGER_PEAK_MIB: dict[str, tuple[int, int]] = {
# Small-host int8 DiT, eager: (peak MiB above resident weights at 1024^2, largest int8 DiT MiB covered).
-# FLUX.2-klein-4B, Kaggle T4 fp16, 4 steps, sliced VAE. "0" = learned peak alone.
+# T4 fp16, 4 steps, sliced VAE: FLUX.2-klein-4B 2805 (Kaggle), 9B 2459 (Colab); the 4B's covers both.
+# "0" = learned peak alone.
MEASURED_ACTIVATION_SMALL_HOST_ENV = "UNSLOTH_DIFFUSION_MEASURED_ACTIVATION_SMALL_HOST"
_MEASURED_SMALL_HOST_INT8_EAGER_PEAK_MIB: dict[str, tuple[int, int]] = {
- "flux.2-klein": (2805, 3800),
+ "flux.2-klein": (2805, 8700),
}
diff --git a/studio/backend/core/inference/diffusion_small_host.py b/studio/backend/core/inference/diffusion_small_host.py
index 9acee08907a..b2c10a678ec 100644
--- a/studio/backend/core/inference/diffusion_small_host.py
+++ b/studio/backend/core/inference/diffusion_small_host.py
@@ -7,6 +7,8 @@ from __future__ import annotations
import json
import os
+import sys
+import uuid
from dataclasses import dataclass, field
from pathlib import Path
from typing import Any, Optional
@@ -48,16 +50,156 @@ def host_ram_check_disabled() -> bool:
return _env(HOST_RAM_CHECK_ENV) in ("0", "off", "false", "no")
+# Int8 spill: the route's int8 DiT weights live in a memory-mapped scratch file, not anonymous host RAM.
+# Auto only when the in-RAM route does not fit; 0 = off, 1 = always.
+SPILL_ENV = "UNSLOTH_DIFFUSION_SMALL_HOST_SPILL"
+SPILL_DIR_ENV = "UNSLOTH_DIFFUSION_SMALL_HOST_SPILL_DIR"
+_SPILL_DISK_MARGIN_MIB = 2048
+# host bytes quantisation holds at once (bias, scale, D2H staging)
+_SPILL_TRANSIENT_MIB = 256
+_SPILL_ALIGN = 64
+_SPILL_PREFIX = "unsloth-int8-spill-"
+# a spill here would be RAM again
+_RAM_FILESYSTEMS = frozenset({"tmpfs", "ramfs", "devtmpfs", "hugetlbfs", "rootfs"})
+
+
+def spill_disabled() -> bool:
+ return _env(SPILL_ENV) in ("0", "off", "false", "no")
+
+
+def spill_forced() -> bool:
+ return _env(SPILL_ENV) in ("1", "on", "true", "yes", "force")
+
+
+def spill_dir() -> Path:
+ raw = (os.environ.get(SPILL_DIR_ENV) or "").strip()
+ if raw:
+ return Path(os.path.expanduser(raw))
+ try:
+ from utils.paths.storage_roots import cache_root as studio_cache_root
+ return studio_cache_root() / "diffusion_int8_spill"
+ except Exception: # noqa: BLE001
+ return Path.home() / ".cache" / "unsloth" / "diffusion_int8_spill"
+
+
+def _fs_type(path: Path) -> Optional[str]:
+ try:
+ real = os.path.realpath(path)
+ best, kind = "", None
+ with open("/proc/mounts", encoding = "utf-8") as fh:
+ for line in fh:
+ parts = line.split()
+ if len(parts) < 3:
+ continue
+ mnt = parts[1].replace("\\040", " ")
+ inside = real == mnt or real.startswith(mnt.rstrip("/") + "/")
+ if inside and len(mnt) >= len(best):
+ best, kind = mnt, parts[2]
+ return kind
+ except Exception: # noqa: BLE001
+ return None
+
+
+def spill_capacity_mib(directory: Any = None) -> Optional[int]:
+ """Free MiB for the spill, None if off, not Linux, RAM-backed or unwritable."""
+ if spill_disabled() or not sys.platform.startswith("linux"):
+ return None
+ try:
+ import shutil
+
+ d = Path(directory) if directory is not None else spill_dir()
+ # nearest existing ancestor: no directory is created unless the spill runs
+ while not d.exists() and d.parent != d:
+ d = d.parent
+ kind = _fs_type(d)
+ if kind is None or kind in _RAM_FILESYSTEMS or not os.access(d, os.W_OK):
+ return None
+ return int(shutil.disk_usage(d).free // _MIB)
+ except Exception: # noqa: BLE001
+ return None
+
+
+def _sweep_stale_spills(directory: Path) -> None:
+ try:
+ for f in directory.glob(f"{_SPILL_PREFIX}*.bin"):
+ try:
+ pid = int(f.name[len(_SPILL_PREFIX) :].split("-", 1)[0])
+ except ValueError:
+ continue
+ if pid == os.getpid():
+ continue
+ try:
+ os.kill(pid, 0)
+ continue
+ except ProcessLookupError:
+ pass
+ except PermissionError:
+ continue
+ try:
+ f.unlink()
+ except OSError:
+ pass
+ except Exception: # noqa: BLE001
+ pass
+
+
+class Int8Spill:
+ """Preallocated, unlinked scratch file mapped MAP_SHARED; ``store`` returns a file-backed copy of an int8 tensor."""
+
+ def __init__(self, directory: Any, nbytes: int):
+ import torch
+
+ d = Path(directory)
+ d.mkdir(parents = True, exist_ok = True)
+ _sweep_stale_spills(d)
+ self.nbytes = max(_SPILL_ALIGN, int(nbytes))
+ path = d / f"{_SPILL_PREFIX}{os.getpid()}-{uuid.uuid4().hex[:8]}.bin"
+ fd = os.open(path, os.O_RDWR | os.O_CREAT | os.O_EXCL, 0o600)
+ try:
+ try:
+ # reserve now: ENOSPC under a shared mapping is SIGBUS
+ os.posix_fallocate(fd, 0, self.nbytes)
+ finally:
+ os.close(fd)
+ self.flat = torch.from_file(str(path), shared = True, size = self.nbytes, dtype = torch.int8)
+ finally:
+ try:
+ path.unlink()
+ except OSError:
+ pass
+ self.offset = 0
+
+ def store(self, q: Any) -> Any:
+ n = int(q.numel())
+ start = -(-self.offset // _SPILL_ALIGN) * _SPILL_ALIGN
+ if start + n > self.nbytes:
+ raise RuntimeError("int8 spill file sized too small")
+ view = self.flat[start : start + n].view(q.shape)
+ view.copy_(q)
+ self.offset = start + n
+ return view
+
+
+def spill_bytes_for(numels: list) -> int:
+ return sum(-(-int(n) // _SPILL_ALIGN) * _SPILL_ALIGN for n in numels)
+
+
_ST_DTYPES = {"BF16": "bfloat16", "F16": "float16", "F32": "float32"}
def _safetensors_bytes_by_dtype(path: Path, keep: tuple[str, ...] = ()) -> dict[str, int]:
+ return _header_sizes(path, keep)[0]
+
+
+def _header_sizes(path: Path, keep: tuple[str, ...] = ()) -> tuple[dict[str, int], int]:
+ """(bytes per dtype plus "kept", largest embedding bytes)."""
from .diffusion_component_sizes import read_safetensors_header
out: dict[str, int] = {}
+ embed = 0
header = read_safetensors_header(path, max_bytes = 256 * _MIB)
if header is None: # unreadable header: no claim about this file
- return out
+ return out, embed
for name, meta in header.items():
if name == "__metadata__":
continue
@@ -67,9 +209,16 @@ def _safetensors_bytes_by_dtype(path: Path, keep: tuple[str, ...] = ()) -> dict[
out[key] = out.get(key, 0) + int(end) - int(start)
if keep and any(f".{k}." in f".{name}." for k in keep):
out["kept"] = out.get("kept", 0) + int(end) - int(start)
+ if _is_embedding_name(name):
+ embed = max(embed, int(end) - int(start))
except Exception: # noqa: BLE001
continue
- return out
+ return out, embed
+
+
+def _is_embedding_name(name: str) -> bool:
+ leaf = name.rsplit(".", 2)
+ return "embed" in name or (len(leaf) >= 2 and leaf[-2] == "shared")
@dataclass(frozen = True)
@@ -80,6 +229,8 @@ class StoredComponent:
# stored MiB of ``_keep_in_fp32_modules`` tensors: they convert to fp32 on the host even on the route
kept_fp32_mib: int = 0
library: str = ""
+ # largest embedding (text encoders), converted on the host
+ embed_mib: int = 0
def _keep_fp32_from_config(sub: Path) -> tuple[str, ...]:
@@ -117,9 +268,12 @@ def stored_components(snapshot: Any) -> dict[str, StoredComponent]:
index = {}
for sub in sorted(p for p in root.iterdir() if p.is_dir()):
totals: dict[str, int] = {}
+ embed = 0
keep = _keep_fp32_from_config(sub) if sub.name.startswith("text_encoder") else ()
for f in _weight_files(sub):
- for k, v in _safetensors_bytes_by_dtype(f, keep).items():
+ sizes, largest = _header_sizes(f, keep)
+ embed = max(embed, largest)
+ for k, v in sizes.items():
totals[k] = totals.get(k, 0) + v
kept = totals.pop("kept", 0)
if not totals:
@@ -128,7 +282,12 @@ def stored_components(snapshot: Any) -> dict[str, StoredComponent]:
entry = index.get(sub.name)
library = str(entry[0]) if isinstance(entry, list) and entry and entry[0] else ""
out[sub.name] = StoredComponent(
- sub.name, -(-sum(totals.values()) // _MIB), dtype, -(-kept // _MIB), library
+ sub.name,
+ -(-sum(totals.values()) // _MIB),
+ dtype,
+ -(-kept // _MIB),
+ library,
+ -(-embed // _MIB) if sub.name.startswith("text_encoder") else 0,
)
except Exception: # noqa: BLE001 - no claim
return {}
@@ -213,6 +372,8 @@ class SmallHostDecision:
# component -> stored dtype it loads at (memory-mapped)
storage_dtypes: dict[str, str] = field(default_factory = dict)
refuse: Optional[str] = None
+ int8_spill: bool = False
+ spill_route_host_mib: int = 0
def decide_small_host(
@@ -225,11 +386,12 @@ def decide_small_host(
lora_active: bool = False,
host_commit_mib: Optional[int] = None,
pread_libraries: tuple[str, ...] = (),
+ spill_free_mib: Optional[int] = None,
) -> SmallHostDecision:
"""Whether this pipeline load takes the small-host route. Only CUDA loads converting large bf16 components
(fp16 / fp32 compute) change; same-dtype loads already stay memory-mapped. ``host_commit_mib`` (Windows)
also routes a dense load whose commit need does not fit: on Windows commit exhaustion is an access
- violation in native code, not a Python error (#13432, #14513)."""
+ violation in native code, not a Python error (#13432, #14513). ``spill_free_mib``: ``spill_capacity_mib``, None = no spill."""
if small_host_disabled():
return SmallHostDecision(False, f"{SMALL_HOST_ENV}=0")
if str(device) != "cuda":
@@ -303,13 +465,17 @@ def decide_small_host(
route,
**extra,
)
+ spill, spill_route = _spill_choice(
+ converted, compute, route, reserve, host_available_mib, spill_free_mib
+ )
+ need = spill_route if spill else route
refuse = None
if (
host_available_mib is not None
- and route + reserve > int(host_available_mib)
+ and need + reserve > int(host_available_mib)
and not host_ram_check_disabled()
):
- refuse = _refusal(route, reserve, host_available_mib, "even the low-memory route")
+ refuse = _refusal(need, reserve, host_available_mib, "even the low-memory route")
why = (
f"{SMALL_HOST_ENV}=1"
if forced and fits
@@ -325,10 +491,36 @@ def decide_small_host(
route,
storage_dtypes = storage,
refuse = refuse,
+ int8_spill = spill,
+ spill_route_host_mib = spill_route if spill else 0,
**extra,
)
+def _spill_choice(
+ converted: dict[str, StoredComponent],
+ compute: str,
+ route: int,
+ reserve: int,
+ available: Optional[int],
+ spill_free_mib: Optional[int],
+) -> tuple[bool, int]:
+ """(spill, host MiB of the spilled route). Auto only when the in-RAM route does not fit."""
+ dit = sum(c.mib // 2 for n, c in converted.items() if n in INT8_DENOISER_NAMES)
+ if not dit or spill_free_mib is None or spill_disabled():
+ return False, route
+ short = available is not None and route + reserve > int(available)
+ if not (short or spill_forced()) or dit + _SPILL_DISK_MARGIN_MIB > int(spill_free_mib):
+ return False, route
+ # route omits the converted encoder embedding
+ embed = sum(
+ c.embed_mib * _itemsize(compute) // _itemsize(c.dtype)
+ for n, c in converted.items()
+ if n.startswith("text_encoder")
+ )
+ return True, route - dit + embed + _SPILL_TRANSIENT_MIB
+
+
def _refusal(need: int, reserve: int, available: Optional[int], what: str) -> str:
return (
f"Not enough system RAM to load this model: {what} needs about {(need + reserve) / 1024:.1f} GB "
@@ -480,10 +672,12 @@ def quantize_int8_weight_(
work_device: Any,
keep_device: Any = "cpu",
act_int8: bool = False,
+ spill_dir: Any = None,
) -> dict[str, int]:
"""Replace every large ``nn.Linear`` with per-row int8 weights in place, one Linear at a time (quantised on
``work_device``, stored on ``keep_device``); everything else casts to ``compute_dtype``. ``act_int8`` lets the
- new Linears run W8A8 where ``Int8WeightLinear`` allows it."""
+ new Linears run W8A8 where ``Int8WeightLinear`` allows it. ``spill_dir``: int8 weights go to an ``Int8Spill``
+ there (same bytes, file-backed)."""
import torch
cls = int8_linear_class()
@@ -493,6 +687,12 @@ def quantize_int8_weight_(
targets = [
(name, child) for name, child in module.named_modules() if type(child) is torch.nn.Linear
]
+ spill = None
+ if spill_dir is not None:
+ sizes = [
+ lin.weight.numel() for _, lin in targets if lin.weight.numel() >= _INT8_MIN_ELEMENTS
+ ]
+ spill = Int8Spill(spill_dir, spill_bytes_for(sizes))
for name, lin in targets:
if lin.weight.numel() < _INT8_MIN_ELEMENTS:
stats["dense_linears"] += 1
@@ -503,7 +703,7 @@ def quantize_int8_weight_(
q = torch.round(w / scale).clamp_(-127, 127).to(torch.int8)
bias = None if lin.bias is None else lin.bias.detach().to(keep, compute_dtype)
new = cls(
- q.to(keep),
+ q.to(keep) if spill is None else spill.store(q),
scale.to(compute_dtype).to(keep),
bias,
lin.in_features,
@@ -516,6 +716,8 @@ def quantize_int8_weight_(
setattr(parent, attr, new)
stats["linears"] += 1
stats["int8_bytes"] += new.qweight.numel()
+ if spill is not None:
+ stats["spilled_bytes"] = stats.get("spilled_bytes", 0) + new.qweight.numel()
with torch.no_grad():
for sub in module.modules():
if isinstance(sub, cls):
diff --git a/studio/backend/tests/test_diffusion_measured_placement_dense.py b/studio/backend/tests/test_diffusion_measured_placement_dense.py
index 7833f9d4acf..8404d5a8c81 100644
--- a/studio/backend/tests/test_diffusion_measured_placement_dense.py
+++ b/studio/backend/tests/test_diffusion_measured_placement_dense.py
@@ -250,6 +250,27 @@ def test_small_host_int8_klein_places_from_its_measured_seed(klein_pipe, monkeyp
assert resident + KLEIN_SMALL_HOST_HEADROOM + dm.DEFAULT_BASE_OVERHEAD_MIB <= 12758
+KLEIN9_SMALL_HOST_LOADED = {
+ "transformer": (8664, "dit"),
+ "text_encoder": (11124, "text_encoder"),
+ "vae": (160, "other"),
+}
+
+
+def test_small_host_int8_klein9_places_from_the_shared_seed(klein_pipe, monkeypatch):
+ # unseeded on a 12.7 GB T4: 2212 of 8664 MiB resident, the rest re-read from the spill every step
+ pipe = _small_host_int8_pipe(klein_pipe, monkeypatch)
+ monkeypatch.setattr(dm, "_loaded_component_mib", lambda pipe: dict(KLEIN9_SMALL_HOST_LOADED))
+ plan = _flat_plan(12758)
+ new = _refine(pipe, plan)
+ assert new is not plan
+ assert new.estimates["measured_runtime_headroom_mib"] == KLEIN_SMALL_HOST_HEADROOM
+ assert new.estimates["measured_small_host_int8"] is True
+ assert 6000 <= (new.resident_transformer_mib or 0) < 8664
+ resident = new.resident_transformer_mib + (new.resident_text_encoder_mib or 0) + 160
+ assert resident + KLEIN_SMALL_HOST_HEADROOM + dm.DEFAULT_BASE_OVERHEAD_MIB <= 12758
+
+
def test_small_host_int8_seed_kill_switch_and_scope(klein_pipe, monkeypatch):
pipe = _small_host_int8_pipe(klein_pipe, monkeypatch)
plan = _flat_plan(12758)
@@ -258,6 +279,7 @@ def test_small_host_int8_seed_kill_switch_and_scope(klein_pipe, monkeypatch):
monkeypatch.delenv(SMALL_HOST_SEED_ENV)
assert _refine(pipe, plan, family = "flux.1") is plan
assert dm._small_host_int8_seed_mib("flux.2-klein", "default", 3700, 2) is None
+ assert dm._small_host_int8_seed_mib("flux.2-klein", "off", 8664, 2) == 2805
assert dm._small_host_int8_seed_mib("flux.2-klein", "off", 9000, 2) is None
assert dm._small_host_int8_seed_mib("flux.2-klein", "off", 3700, 4) == 5610
pipe._unsloth_measured_reserve = (
diff --git a/studio/backend/tests/test_diffusion_small_host_spill.py b/studio/backend/tests/test_diffusion_small_host_spill.py
new file mode 100644
index 00000000000..c8a9ac1c8a5
--- /dev/null
+++ b/studio/backend/tests/test_diffusion_small_host_spill.py
@@ -0,0 +1,286 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
+
+"""Small-host int8 spill: the denoiser's int8 weights live in a memory-mapped scratch file instead of host RAM.
+Sized to a Colab standard T4 (12.7 GB host RAM) loading FLUX.2-klein-9B; CPU only."""
+
+from __future__ import annotations
+
+import json
+import os
+import struct
+import subprocess
+import sys
+import textwrap
+from pathlib import Path
+
+import pytest
+import torch
+
+import core.inference.diffusion_small_host as sh
+
+KLEIN9 = {
+ "transformer": sh.StoredComponent("transformer", 17317, "bfloat16", 0, "diffusers"),
+ "text_encoder": sh.StoredComponent(
+ "text_encoder", 15623, "bfloat16", 0, "transformers", embed_mib = 1187
+ ),
+ "vae": sh.StoredComponent("vae", 161, "bfloat16", 0, "diffusers"),
+}
+COLAB_TOTAL = 12977
+COLAB_AVAILABLE = 11161
+ROOMY_DISK = 200 * 1024
+
+_LINUX = sys.platform.startswith("linux")
+
+
[email protected](autouse = True)
+def _clean_env(monkeypatch):
+ for name in (
+ sh.SMALL_HOST_ENV,
+ sh.HOST_RAM_CHECK_ENV,
+ sh.SPILL_ENV,
+ sh.SPILL_DIR_ENV,
+ ):
+ monkeypatch.delenv(name, raising = False)
+
+
+def _decide(
+ comps = KLEIN9,
+ total = COLAB_TOTAL,
+ available = COLAB_AVAILABLE,
+ **kw,
+):
+ return sh.decide_small_host(
+ comps,
+ torch.float16,
+ device = "cuda",
+ host_total_mib = total,
+ host_available_mib = available,
+ **kw,
+ )
+
+
+def test_klein9_on_colab_refuses_without_spill():
+ d = _decide()
+ assert d.engaged and not d.int8_spill
+ assert d.refuse and "11.5 GB" in d.refuse and "10.9 GB" in d.refuse
+
+
+def test_klein9_on_colab_spills_and_loads():
+ d = _decide(spill_free_mib = ROOMY_DISK)
+ assert d.engaged and d.int8_spill and d.refuse is None
+ assert d.storage_dtypes == {"transformer": "bfloat16", "text_encoder": "bfloat16"}
+ # int8 DiT leaves the host estimate; the encoder's converted embedding joins it
+ assert d.spill_route_host_mib == 1187 + sh._SPILL_TRANSIENT_MIB
+ assert d.route_host_mib == 17317 // 2
+
+
+def test_spill_kill_switch_keeps_the_refusal(monkeypatch):
+ monkeypatch.setenv(sh.SPILL_ENV, "0")
+ d = _decide(spill_free_mib = ROOMY_DISK)
+ assert not d.int8_spill and d.refuse == _decide().refuse
+
+
+def test_spill_needs_disk_room():
+ d = _decide(spill_free_mib = 17317 // 2 + sh._SPILL_DISK_MARGIN_MIB - 1)
+ assert not d.int8_spill and d.refuse == _decide().refuse
+
+
+def test_spill_route_still_refuses_a_host_too_small_for_it():
+ d = _decide(available = 3000, spill_free_mib = ROOMY_DISK)
+ assert d.int8_spill and d.refuse and "Not enough system RAM" in d.refuse
+
+
[email protected]("available", [11800, 14000, 20000, 30000, 40000])
+def test_hosts_that_load_today_keep_their_exact_decision(available):
+ base = _decide(available = available)
+ head = _decide(available = available, spill_free_mib = ROOMY_DISK)
+ assert base.refuse is None and not head.int8_spill
+ assert (base.engaged, base.reason, base.storage_dtypes, base.refuse, base.route_host_mib) == (
+ head.engaged,
+ head.reason,
+ head.storage_dtypes,
+ head.refuse,
+ head.route_host_mib,
+ )
+
+
+def test_spill_forced_on_a_host_that_fits(monkeypatch):
+ monkeypatch.setenv(sh.SPILL_ENV, "1")
+ d = _decide(available = 30000, spill_free_mib = ROOMY_DISK)
+ assert d.engaged and d.int8_spill and d.refuse is None
+
+
+def test_no_denoiser_no_spill():
+ comps = {"text_encoder": KLEIN9["text_encoder"]}
+ d = _decide(comps, available = 2000, spill_free_mib = ROOMY_DISK)
+ assert not d.int8_spill
+
+
+def test_check_disabled_still_spills(monkeypatch):
+ # the in-RAM route was OOM-killed with the check off: spill whenever it does not fit
+ monkeypatch.setenv(sh.HOST_RAM_CHECK_ENV, "0")
+ d = _decide(spill_free_mib = ROOMY_DISK)
+ assert d.int8_spill and d.refuse is None
+
+
+def _write_st(path: Path, tensors: dict) -> None:
+ header, off = {}, 0
+ for name, (dtype, shape) in tensors.items():
+ n = 2
+ for s in shape:
+ n *= s
+ header[name] = {"dtype": dtype, "shape": list(shape), "data_offsets": [off, off + n]}
+ off += n
+ blob = json.dumps(header).encode()
+ path.write_bytes(struct.pack("<Q", len(blob)) + blob + b"\0" * off)
+
+
+def test_stored_components_reads_the_encoder_embedding(tmp_path):
+ te = tmp_path / "text_encoder"
+ te.mkdir()
+ _write_st(
+ te / "model.safetensors",
+ {
+ "model.embed_tokens.weight": ("BF16", (1024, 512)),
+ "model.layers.0.mlp.up_proj.weight": ("BF16", (512, 512)),
+ },
+ )
+ dit = tmp_path / "transformer"
+ dit.mkdir()
+ _write_st(
+ dit / "diffusion_pytorch_model.safetensors", {"x_embedder.weight": ("BF16", (1024, 1024))}
+ )
+ comps = sh.stored_components(tmp_path)
+ assert comps["text_encoder"].embed_mib == 1
+ assert comps["transformer"].embed_mib == 0
+ assert sh._safetensors_bytes_by_dtype(te / "model.safetensors") == {
+ "bfloat16": 1024 * 512 * 2 + 512 * 512 * 2
+ }
+
+
+class _Tiny(torch.nn.Module):
+ def __init__(self):
+ super().__init__()
+ self.a = torch.nn.Linear(2048, 2048)
+ self.b = torch.nn.Linear(2048, 4096, bias = False)
+ self.small = torch.nn.Linear(64, 64)
+ self.norm = torch.nn.LayerNorm(2048)
+
+ def forward(self, x):
+ return self.b(self.norm(self.a(x)))
+
+
+def _tiny(seed = 0):
+ torch.manual_seed(seed)
+ return _Tiny().to(torch.bfloat16)
+
+
[email protected](not _LINUX, reason = "spill is Linux only")
+def test_spilled_int8_is_bit_identical_and_file_backed(tmp_path):
+ ram, spilled = _tiny(), _tiny()
+ s_ram = sh.quantize_int8_weight_(ram, compute_dtype = torch.float16, work_device = "cpu")
+ s_sp = sh.quantize_int8_weight_(
+ spilled, compute_dtype = torch.float16, work_device = "cpu", spill_dir = tmp_path
+ )
+ assert s_ram["linears"] == s_sp["linears"] == 2
+ assert s_sp["spilled_bytes"] == s_sp["int8_bytes"] and s_ram.get("spilled_bytes", 0) == 0
+ for name in ("a", "b"):
+ r, s = getattr(ram, name), getattr(spilled, name)
+ assert torch.equal(r.qweight, s.qweight) and torch.equal(r.scale, s.scale)
+ assert s.qweight.untyped_storage().filename is not None
+ assert r.qweight.untyped_storage().filename is None
+ assert not list(tmp_path.iterdir())
+ x = torch.randn(3, 2048, dtype = torch.float16)
+ with torch.no_grad():
+ assert torch.equal(ram(x), spilled(x))
+
+
[email protected](not _LINUX, reason = "spill is Linux only")
+def test_spill_capacity_reads_free_disk_and_refuses_ram_filesystems(tmp_path, monkeypatch):
+ free = sh.spill_capacity_mib(tmp_path)
+ assert free is not None and free > 0
+ if Path("/dev/shm").is_dir() and sh._fs_type(Path("/dev/shm")) == "tmpfs":
+ assert sh.spill_capacity_mib(Path("/dev/shm")) is None
+ monkeypatch.setenv(sh.SPILL_ENV, "0")
+ assert sh.spill_capacity_mib(tmp_path) is None
+
+
+def test_spill_capacity_is_none_off_linux(tmp_path, monkeypatch):
+ monkeypatch.setattr(sh.sys, "platform", "win32")
+ assert sh.spill_capacity_mib(tmp_path) is None
+
+
+def test_spill_dir_env_wins(tmp_path, monkeypatch):
+ monkeypatch.setenv(sh.SPILL_DIR_ENV, str(tmp_path / "x"))
+ assert sh.spill_dir() == tmp_path / "x"
+
+
[email protected](not _LINUX, reason = "spill is Linux only")
+def test_stale_spill_files_of_dead_processes_are_swept(tmp_path):
+ dead = tmp_path / f"{sh._SPILL_PREFIX}999999999-abc.bin"
+ dead.write_bytes(b"x")
+ mine = tmp_path / f"{sh._SPILL_PREFIX}{os.getpid()}-abc.bin"
+ mine.write_bytes(b"x")
+ sh._sweep_stale_spills(tmp_path)
+ assert not dead.exists() and mine.exists()
+
+
+_RSS_CHILD = textwrap.dedent(
+ """
+ import sys, torch
+ sys.path.insert(0, {backend!r})
+ import core.inference.diffusion_small_host as sh
+
+ def anon():
+ for line in open("/proc/self/status"):
+ if line.startswith("RssAnon:"):
+ return int(line.split()[1]) // 1024
+ import safetensors.torch as st
+ torch.manual_seed(0)
+ def build():
+ return torch.nn.Sequential(*[torch.nn.Linear(4096, 4096, bias=False) for _ in range(16)])
+ src = {tmp!r} + "/src.safetensors"
+ st.save_file({{k: v.to(torch.bfloat16) for k, v in build().state_dict().items()}}, src)
+ with torch.device("meta"):
+ m = build()
+ # memory-mapped bf16 source, as the route loads the stored shards
+ m.load_state_dict(st.load_file(src), assign=True)
+ before = anon()
+ kw = dict(spill_dir={spill!r}) if {spill!r} else dict()
+ sh.quantize_int8_weight_(m, compute_dtype=torch.float16, work_device="cpu", **kw)
+ print("DELTA", anon() - before)
+ """
+)
+
+
[email protected](not _LINUX, reason = "RssAnon is Linux only")
[email protected]("spill", [False, True])
+def test_int8_host_residency_bound(tmp_path, spill):
+ # 16 x 4096^2 int8 = 256 MiB: the in-RAM route keeps all of it anonymous, the spill keeps it file-backed
+ backend = str(Path(sh.__file__).resolve().parents[2])
+ spill_dir = tmp_path / "spill"
+ spill_dir.mkdir()
+ code = _RSS_CHILD.format(
+ backend = backend, tmp = str(tmp_path), spill = str(spill_dir) if spill else ""
+ )
+ out = subprocess.run(
+ [sys.executable, "-c", code],
+ capture_output = True,
+ text = True,
+ env = {**os.environ, "CUDA_VISIBLE_DEVICES": ""},
+ timeout = 300,
+ )
+ assert out.returncode == 0, out.stderr[-2000:]
+ delta = int(out.stdout.split("DELTA")[-1].split()[0])
+ if spill:
+ assert delta < 64, delta
+ else:
+ assert delta > 200, delta
+
+
[email protected](not _LINUX, reason = "spill is Linux only")
+def test_spill_capacity_creates_nothing(tmp_path):
+ target = tmp_path / "a" / "b"
+ assert sh.spill_capacity_mib(target) is not None
+ assert not (tmp_path / "a").exists()