Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
Studio diffusion: let a 12.7 GB host load FLUX.2-klein-9B by spilling the int8 denoiser to a scratch file (#15911)
Daniel Han2572ed547f72
diff --git a/studio/backend/core/inference/diffusion.py b/studio/backend/core/inference/diffusion.pyindex 221455a1337..be5a34bac95 100644--- a/studio/backend/core/inference/diffusion.py+++ b/studio/backend/core/inference/diffusion.py@@ -174,6 +174,7 @@ from .diffusion_small_host import (host_commit_mib,host_ram_mib,resolve_snapshot_dir,+ spill_capacity_mib,stored_components,torch_dtype_map as small_host_torch_dtype_map,)@@ -11995,6 +11996,7 @@ class DiffusionBackend:lora_active = lora_active,host_commit_mib = host_commit_mib(),pread_libraries = ("transformers",) if pread_available() else (),+ spill_free_mib = spill_capacity_mib(),)if decision.engaged or decision.refuse:logger.info(@@ -12005,6 +12007,13 @@ class DiffusionBackend:)elif decision.available_mib is not None:logger.info("diffusion.small_host: engaged=False (%s)", decision.reason)+ if decision.int8_spill:+ logger.info(+ "diffusion.small_host: int8 denoiser spills to a memory-mapped scratch file (host estimate %d MiB "+ "instead of %d MiB)",+ decision.spill_route_host_mib,+ decision.route_host_mib,+ )if decision.refuse:raise RuntimeError(decision.refuse)return decision@@ -12030,6 +12039,7 @@ class DiffusionBackend:mark,prepare_streamed_encoder_,quantize_int8_weight_,+ spill_dir,)device = target.torch_device@@ -12051,16 +12061,29 @@ class DiffusionBackend:else:# pageable: pinning rounds blocks to powers of two (11.3 GB of FLUX.1 int8 held 18 GB pinned)act_int8 = int8_act_family(fam) and int8_act_device_ok(device)- stats = quantize_int8_weight_(- module,- compute_dtype = dtype,- work_device = device,- keep_device = "cpu",- act_int8 = act_int8,- )+ spill_to = spill_dir() if getattr(decision, "int8_spill", False) else None+ try:+ stats = quantize_int8_weight_(+ module,+ compute_dtype = dtype,+ work_device = device,+ keep_device = "cpu",+ act_int8 = act_int8,+ spill_dir = spill_to,+ )+ except OSError as exc:+ if spill_to is None:+ raise+ # in-RAM route does not fit: no fallback+ raise RuntimeError(+ f"Not enough system RAM to load this model, and the int8 scratch file in {spill_to} "+ f"could not be created ({exc}). Point UNSLOTH_DIFFUSION_SMALL_HOST_SPILL_DIR at a disk "+ f"with {decision.route_host_mib / 1024 + 2:.0f} GB free."+ ) from excinfo["components"][name] = (f"int8 weights ({stats['int8_bytes'] >> 20} MiB, {stats['linears']} linears"+ (", int8 activations" if act_int8 else "")+ + (", memory-mapped scratch file" if stats.get("spilled_bytes") else "")+ ")")if load_token is not None:diff --git a/studio/backend/core/inference/diffusion_memory.py b/studio/backend/core/inference/diffusion_memory.pyindex 671c74f19c2..5ecf1b9a38d 100644--- a/studio/backend/core/inference/diffusion_memory.py+++ b/studio/backend/core/inference/diffusion_memory.py@@ -3389,10 +3389,11 @@ _MEASURED_DENSE_BF16_EAGER_PEAK_MIB: dict[str, tuple[int, int]] = {# Small-host int8 DiT, eager: (peak MiB above resident weights at 1024^2, largest int8 DiT MiB covered).-# FLUX.2-klein-4B, Kaggle T4 fp16, 4 steps, sliced VAE. "0" = learned peak alone.+# T4 fp16, 4 steps, sliced VAE: FLUX.2-klein-4B 2805 (Kaggle), 9B 2459 (Colab); the 4B's covers both.+# "0" = learned peak alone.MEASURED_ACTIVATION_SMALL_HOST_ENV = "UNSLOTH_DIFFUSION_MEASURED_ACTIVATION_SMALL_HOST"_MEASURED_SMALL_HOST_INT8_EAGER_PEAK_MIB: dict[str, tuple[int, int]] = {- "flux.2-klein": (2805, 3800),+ "flux.2-klein": (2805, 8700),}diff --git a/studio/backend/core/inference/diffusion_small_host.py b/studio/backend/core/inference/diffusion_small_host.pyindex 9acee08907a..b2c10a678ec 100644--- a/studio/backend/core/inference/diffusion_small_host.py+++ b/studio/backend/core/inference/diffusion_small_host.py@@ -7,6 +7,8 @@ from __future__ import annotationsimport jsonimport os+import sys+import uuidfrom dataclasses import dataclass, fieldfrom pathlib import Pathfrom typing import Any, Optional@@ -48,16 +50,156 @@ def host_ram_check_disabled() -> bool:return _env(HOST_RAM_CHECK_ENV) in ("0", "off", "false", "no")+# Int8 spill: the route's int8 DiT weights live in a memory-mapped scratch file, not anonymous host RAM.+# Auto only when the in-RAM route does not fit; 0 = off, 1 = always.+SPILL_ENV = "UNSLOTH_DIFFUSION_SMALL_HOST_SPILL"+SPILL_DIR_ENV = "UNSLOTH_DIFFUSION_SMALL_HOST_SPILL_DIR"+_SPILL_DISK_MARGIN_MIB = 2048+# host bytes quantisation holds at once (bias, scale, D2H staging)+_SPILL_TRANSIENT_MIB = 256+_SPILL_ALIGN = 64+_SPILL_PREFIX = "unsloth-int8-spill-"+# a spill here would be RAM again+_RAM_FILESYSTEMS = frozenset({"tmpfs", "ramfs", "devtmpfs", "hugetlbfs", "rootfs"})+++def spill_disabled() -> bool:+ return _env(SPILL_ENV) in ("0", "off", "false", "no")+++def spill_forced() -> bool:+ return _env(SPILL_ENV) in ("1", "on", "true", "yes", "force")+++def spill_dir() -> Path:+ raw = (os.environ.get(SPILL_DIR_ENV) or "").strip()+ if raw:+ return Path(os.path.expanduser(raw))+ try:+ from utils.paths.storage_roots import cache_root as studio_cache_root+ return studio_cache_root() / "diffusion_int8_spill"+ except Exception: # noqa: BLE001+ return Path.home() / ".cache" / "unsloth" / "diffusion_int8_spill"+++def _fs_type(path: Path) -> Optional[str]:+ try:+ real = os.path.realpath(path)+ best, kind = "", None+ with open("/proc/mounts", encoding = "utf-8") as fh:+ for line in fh:+ parts = line.split()+ if len(parts) < 3:+ continue+ mnt = parts[1].replace("\\040", " ")+ inside = real == mnt or real.startswith(mnt.rstrip("/") + "/")+ if inside and len(mnt) >= len(best):+ best, kind = mnt, parts[2]+ return kind+ except Exception: # noqa: BLE001+ return None+++def spill_capacity_mib(directory: Any = None) -> Optional[int]:+ """Free MiB for the spill, None if off, not Linux, RAM-backed or unwritable."""+ if spill_disabled() or not sys.platform.startswith("linux"):+ return None+ try:+ import shutil++ d = Path(directory) if directory is not None else spill_dir()+ # nearest existing ancestor: no directory is created unless the spill runs+ while not d.exists() and d.parent != d:+ d = d.parent+ kind = _fs_type(d)+ if kind is None or kind in _RAM_FILESYSTEMS or not os.access(d, os.W_OK):+ return None+ return int(shutil.disk_usage(d).free // _MIB)+ except Exception: # noqa: BLE001+ return None+++def _sweep_stale_spills(directory: Path) -> None:+ try:+ for f in directory.glob(f"{_SPILL_PREFIX}*.bin"):+ try:+ pid = int(f.name[len(_SPILL_PREFIX) :].split("-", 1)[0])+ except ValueError:+ continue+ if pid == os.getpid():+ continue+ try:+ os.kill(pid, 0)+ continue+ except ProcessLookupError:+ pass+ except PermissionError:+ continue+ try:+ f.unlink()+ except OSError:+ pass+ except Exception: # noqa: BLE001+ pass+++class Int8Spill:+ """Preallocated, unlinked scratch file mapped MAP_SHARED; ``store`` returns a file-backed copy of an int8 tensor."""++ def __init__(self, directory: Any, nbytes: int):+ import torch++ d = Path(directory)+ d.mkdir(parents = True, exist_ok = True)+ _sweep_stale_spills(d)+ self.nbytes = max(_SPILL_ALIGN, int(nbytes))+ path = d / f"{_SPILL_PREFIX}{os.getpid()}-{uuid.uuid4().hex[:8]}.bin"+ fd = os.open(path, os.O_RDWR | os.O_CREAT | os.O_EXCL, 0o600)+ try:+ try:+ # reserve now: ENOSPC under a shared mapping is SIGBUS+ os.posix_fallocate(fd, 0, self.nbytes)+ finally:+ os.close(fd)+ self.flat = torch.from_file(str(path), shared = True, size = self.nbytes, dtype = torch.int8)+ finally:+ try:+ path.unlink()+ except OSError:+ pass+ self.offset = 0++ def store(self, q: Any) -> Any:+ n = int(q.numel())+ start = -(-self.offset // _SPILL_ALIGN) * _SPILL_ALIGN+ if start + n > self.nbytes:+ raise RuntimeError("int8 spill file sized too small")+ view = self.flat[start : start + n].view(q.shape)+ view.copy_(q)+ self.offset = start + n+ return view+++def spill_bytes_for(numels: list) -> int:+ return sum(-(-int(n) // _SPILL_ALIGN) * _SPILL_ALIGN for n in numels)++_ST_DTYPES = {"BF16": "bfloat16", "F16": "float16", "F32": "float32"}def _safetensors_bytes_by_dtype(path: Path, keep: tuple[str, ...] = ()) -> dict[str, int]:+ return _header_sizes(path, keep)[0]+++def _header_sizes(path: Path, keep: tuple[str, ...] = ()) -> tuple[dict[str, int], int]:+ """(bytes per dtype plus "kept", largest embedding bytes)."""from .diffusion_component_sizes import read_safetensors_headerout: dict[str, int] = {}+ embed = 0header = read_safetensors_header(path, max_bytes = 256 * _MIB)if header is None: # unreadable header: no claim about this file- return out+ return out, embedfor name, meta in header.items():if name == "__metadata__":continue@@ -67,9 +209,16 @@ def _safetensors_bytes_by_dtype(path: Path, keep: tuple[str, ...] = ()) -> dict[out[key] = out.get(key, 0) + int(end) - int(start)if keep and any(f".{k}." in f".{name}." for k in keep):out["kept"] = out.get("kept", 0) + int(end) - int(start)+ if _is_embedding_name(name):+ embed = max(embed, int(end) - int(start))except Exception: # noqa: BLE001continue- return out+ return out, embed+++def _is_embedding_name(name: str) -> bool:+ leaf = name.rsplit(".", 2)+ return "embed" in name or (len(leaf) >= 2 and leaf[-2] == "shared")@dataclass(frozen = True)@@ -80,6 +229,8 @@ class StoredComponent:# stored MiB of ``_keep_in_fp32_modules`` tensors: they convert to fp32 on the host even on the routekept_fp32_mib: int = 0library: str = ""+ # largest embedding (text encoders), converted on the host+ embed_mib: int = 0def _keep_fp32_from_config(sub: Path) -> tuple[str, ...]:@@ -117,9 +268,12 @@ def stored_components(snapshot: Any) -> dict[str, StoredComponent]:index = {}for sub in sorted(p for p in root.iterdir() if p.is_dir()):totals: dict[str, int] = {}+ embed = 0keep = _keep_fp32_from_config(sub) if sub.name.startswith("text_encoder") else ()for f in _weight_files(sub):- for k, v in _safetensors_bytes_by_dtype(f, keep).items():+ sizes, largest = _header_sizes(f, keep)+ embed = max(embed, largest)+ for k, v in sizes.items():totals[k] = totals.get(k, 0) + vkept = totals.pop("kept", 0)if not totals:@@ -128,7 +282,12 @@ def stored_components(snapshot: Any) -> dict[str, StoredComponent]:entry = index.get(sub.name)library = str(entry[0]) if isinstance(entry, list) and entry and entry[0] else ""out[sub.name] = StoredComponent(- sub.name, -(-sum(totals.values()) // _MIB), dtype, -(-kept // _MIB), library+ sub.name,+ -(-sum(totals.values()) // _MIB),+ dtype,+ -(-kept // _MIB),+ library,+ -(-embed // _MIB) if sub.name.startswith("text_encoder") else 0,)except Exception: # noqa: BLE001 - no claimreturn {}@@ -213,6 +372,8 @@ class SmallHostDecision:# component -> stored dtype it loads at (memory-mapped)storage_dtypes: dict[str, str] = field(default_factory = dict)refuse: Optional[str] = None+ int8_spill: bool = False+ spill_route_host_mib: int = 0def decide_small_host(@@ -225,11 +386,12 @@ def decide_small_host(lora_active: bool = False,host_commit_mib: Optional[int] = None,pread_libraries: tuple[str, ...] = (),+ spill_free_mib: Optional[int] = None,) -> SmallHostDecision:"""Whether this pipeline load takes the small-host route. Only CUDA loads converting large bf16 components(fp16 / fp32 compute) change; same-dtype loads already stay memory-mapped. ``host_commit_mib`` (Windows)also routes a dense load whose commit need does not fit: on Windows commit exhaustion is an access- violation in native code, not a Python error (#13432, #14513)."""+ violation in native code, not a Python error (#13432, #14513). ``spill_free_mib``: ``spill_capacity_mib``, None = no spill."""if small_host_disabled():return SmallHostDecision(False, f"{SMALL_HOST_ENV}=0")if str(device) != "cuda":@@ -303,13 +465,17 @@ def decide_small_host(route,**extra,)+ spill, spill_route = _spill_choice(+ converted, compute, route, reserve, host_available_mib, spill_free_mib+ )+ need = spill_route if spill else routerefuse = Noneif (host_available_mib is not None- and route + reserve > int(host_available_mib)+ and need + reserve > int(host_available_mib)and not host_ram_check_disabled()):- refuse = _refusal(route, reserve, host_available_mib, "even the low-memory route")+ refuse = _refusal(need, reserve, host_available_mib, "even the low-memory route")why = (f"{SMALL_HOST_ENV}=1"if forced and fits@@ -325,10 +491,36 @@ def decide_small_host(route,storage_dtypes = storage,refuse = refuse,+ int8_spill = spill,+ spill_route_host_mib = spill_route if spill else 0,**extra,)+def _spill_choice(+ converted: dict[str, StoredComponent],+ compute: str,+ route: int,+ reserve: int,+ available: Optional[int],+ spill_free_mib: Optional[int],+) -> tuple[bool, int]:+ """(spill, host MiB of the spilled route). Auto only when the in-RAM route does not fit."""+ dit = sum(c.mib // 2 for n, c in converted.items() if n in INT8_DENOISER_NAMES)+ if not dit or spill_free_mib is None or spill_disabled():+ return False, route+ short = available is not None and route + reserve > int(available)+ if not (short or spill_forced()) or dit + _SPILL_DISK_MARGIN_MIB > int(spill_free_mib):+ return False, route+ # route omits the converted encoder embedding+ embed = sum(+ c.embed_mib * _itemsize(compute) // _itemsize(c.dtype)+ for n, c in converted.items()+ if n.startswith("text_encoder")+ )+ return True, route - dit + embed + _SPILL_TRANSIENT_MIB++def _refusal(need: int, reserve: int, available: Optional[int], what: str) -> str:return (f"Not enough system RAM to load this model: {what} needs about {(need + reserve) / 1024:.1f} GB "@@ -480,10 +672,12 @@ def quantize_int8_weight_(work_device: Any,keep_device: Any = "cpu",act_int8: bool = False,+ spill_dir: Any = None,) -> dict[str, int]:"""Replace every large ``nn.Linear`` with per-row int8 weights in place, one Linear at a time (quantised on``work_device``, stored on ``keep_device``); everything else casts to ``compute_dtype``. ``act_int8`` lets the- new Linears run W8A8 where ``Int8WeightLinear`` allows it."""+ new Linears run W8A8 where ``Int8WeightLinear`` allows it. ``spill_dir``: int8 weights go to an ``Int8Spill``+ there (same bytes, file-backed)."""import torchcls = int8_linear_class()@@ -493,6 +687,12 @@ def quantize_int8_weight_(targets = [(name, child) for name, child in module.named_modules() if type(child) is torch.nn.Linear]+ spill = None+ if spill_dir is not None:+ sizes = [+ lin.weight.numel() for _, lin in targets if lin.weight.numel() >= _INT8_MIN_ELEMENTS+ ]+ spill = Int8Spill(spill_dir, spill_bytes_for(sizes))for name, lin in targets:if lin.weight.numel() < _INT8_MIN_ELEMENTS:stats["dense_linears"] += 1@@ -503,7 +703,7 @@ def quantize_int8_weight_(q = torch.round(w / scale).clamp_(-127, 127).to(torch.int8)bias = None if lin.bias is None else lin.bias.detach().to(keep, compute_dtype)new = cls(- q.to(keep),+ q.to(keep) if spill is None else spill.store(q),scale.to(compute_dtype).to(keep),bias,lin.in_features,@@ -516,6 +716,8 @@ def quantize_int8_weight_(setattr(parent, attr, new)stats["linears"] += 1stats["int8_bytes"] += new.qweight.numel()+ if spill is not None:+ stats["spilled_bytes"] = stats.get("spilled_bytes", 0) + new.qweight.numel()with torch.no_grad():for sub in module.modules():if isinstance(sub, cls):diff --git a/studio/backend/tests/test_diffusion_measured_placement_dense.py b/studio/backend/tests/test_diffusion_measured_placement_dense.pyindex 7833f9d4acf..8404d5a8c81 100644--- a/studio/backend/tests/test_diffusion_measured_placement_dense.py+++ b/studio/backend/tests/test_diffusion_measured_placement_dense.py@@ -250,6 +250,27 @@ def test_small_host_int8_klein_places_from_its_measured_seed(klein_pipe, monkeypassert resident + KLEIN_SMALL_HOST_HEADROOM + dm.DEFAULT_BASE_OVERHEAD_MIB <= 12758+KLEIN9_SMALL_HOST_LOADED = {+ "transformer": (8664, "dit"),+ "text_encoder": (11124, "text_encoder"),+ "vae": (160, "other"),+}+++def test_small_host_int8_klein9_places_from_the_shared_seed(klein_pipe, monkeypatch):+ # unseeded on a 12.7 GB T4: 2212 of 8664 MiB resident, the rest re-read from the spill every step+ pipe = _small_host_int8_pipe(klein_pipe, monkeypatch)+ monkeypatch.setattr(dm, "_loaded_component_mib", lambda pipe: dict(KLEIN9_SMALL_HOST_LOADED))+ plan = _flat_plan(12758)+ new = _refine(pipe, plan)+ assert new is not plan+ assert new.estimates["measured_runtime_headroom_mib"] == KLEIN_SMALL_HOST_HEADROOM+ assert new.estimates["measured_small_host_int8"] is True+ assert 6000 <= (new.resident_transformer_mib or 0) < 8664+ resident = new.resident_transformer_mib + (new.resident_text_encoder_mib or 0) + 160+ assert resident + KLEIN_SMALL_HOST_HEADROOM + dm.DEFAULT_BASE_OVERHEAD_MIB <= 12758++def test_small_host_int8_seed_kill_switch_and_scope(klein_pipe, monkeypatch):pipe = _small_host_int8_pipe(klein_pipe, monkeypatch)plan = _flat_plan(12758)@@ -258,6 +279,7 @@ def test_small_host_int8_seed_kill_switch_and_scope(klein_pipe, monkeypatch):monkeypatch.delenv(SMALL_HOST_SEED_ENV)assert _refine(pipe, plan, family = "flux.1") is planassert dm._small_host_int8_seed_mib("flux.2-klein", "default", 3700, 2) is None+ assert dm._small_host_int8_seed_mib("flux.2-klein", "off", 8664, 2) == 2805assert dm._small_host_int8_seed_mib("flux.2-klein", "off", 9000, 2) is Noneassert dm._small_host_int8_seed_mib("flux.2-klein", "off", 3700, 4) == 5610pipe._unsloth_measured_reserve = (diff --git a/studio/backend/tests/test_diffusion_small_host_spill.py b/studio/backend/tests/test_diffusion_small_host_spill.pynew file mode 100644index 00000000000..c8a9ac1c8a5--- /dev/null+++ b/studio/backend/tests/test_diffusion_small_host_spill.py@@ -0,0 +1,286 @@+# SPDX-License-Identifier: AGPL-3.0-only+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0++"""Small-host int8 spill: the denoiser's int8 weights live in a memory-mapped scratch file instead of host RAM.+Sized to a Colab standard T4 (12.7 GB host RAM) loading FLUX.2-klein-9B; CPU only."""++from __future__ import annotations++import json+import os+import struct+import subprocess+import sys+import textwrap+from pathlib import Path++import pytest+import torch++import core.inference.diffusion_small_host as sh++KLEIN9 = {+ "transformer": sh.StoredComponent("transformer", 17317, "bfloat16", 0, "diffusers"),+ "text_encoder": sh.StoredComponent(+ "text_encoder", 15623, "bfloat16", 0, "transformers", embed_mib = 1187+ ),+ "vae": sh.StoredComponent("vae", 161, "bfloat16", 0, "diffusers"),+}+COLAB_TOTAL = 12977+COLAB_AVAILABLE = 11161+ROOMY_DISK = 200 * 1024++_LINUX = sys.platform.startswith("linux")++[email protected](autouse = True)+def _clean_env(monkeypatch):+ for name in (+ sh.SMALL_HOST_ENV,+ sh.HOST_RAM_CHECK_ENV,+ sh.SPILL_ENV,+ sh.SPILL_DIR_ENV,+ ):+ monkeypatch.delenv(name, raising = False)+++def _decide(+ comps = KLEIN9,+ total = COLAB_TOTAL,+ available = COLAB_AVAILABLE,+ **kw,+):+ return sh.decide_small_host(+ comps,+ torch.float16,+ device = "cuda",+ host_total_mib = total,+ host_available_mib = available,+ **kw,+ )+++def test_klein9_on_colab_refuses_without_spill():+ d = _decide()+ assert d.engaged and not d.int8_spill+ assert d.refuse and "11.5 GB" in d.refuse and "10.9 GB" in d.refuse+++def test_klein9_on_colab_spills_and_loads():+ d = _decide(spill_free_mib = ROOMY_DISK)+ assert d.engaged and d.int8_spill and d.refuse is None+ assert d.storage_dtypes == {"transformer": "bfloat16", "text_encoder": "bfloat16"}+ # int8 DiT leaves the host estimate; the encoder's converted embedding joins it+ assert d.spill_route_host_mib == 1187 + sh._SPILL_TRANSIENT_MIB+ assert d.route_host_mib == 17317 // 2+++def test_spill_kill_switch_keeps_the_refusal(monkeypatch):+ monkeypatch.setenv(sh.SPILL_ENV, "0")+ d = _decide(spill_free_mib = ROOMY_DISK)+ assert not d.int8_spill and d.refuse == _decide().refuse+++def test_spill_needs_disk_room():+ d = _decide(spill_free_mib = 17317 // 2 + sh._SPILL_DISK_MARGIN_MIB - 1)+ assert not d.int8_spill and d.refuse == _decide().refuse+++def test_spill_route_still_refuses_a_host_too_small_for_it():+ d = _decide(available = 3000, spill_free_mib = ROOMY_DISK)+ assert d.int8_spill and d.refuse and "Not enough system RAM" in d.refuse++[email protected]("available", [11800, 14000, 20000, 30000, 40000])+def test_hosts_that_load_today_keep_their_exact_decision(available):+ base = _decide(available = available)+ head = _decide(available = available, spill_free_mib = ROOMY_DISK)+ assert base.refuse is None and not head.int8_spill+ assert (base.engaged, base.reason, base.storage_dtypes, base.refuse, base.route_host_mib) == (+ head.engaged,+ head.reason,+ head.storage_dtypes,+ head.refuse,+ head.route_host_mib,+ )+++def test_spill_forced_on_a_host_that_fits(monkeypatch):+ monkeypatch.setenv(sh.SPILL_ENV, "1")+ d = _decide(available = 30000, spill_free_mib = ROOMY_DISK)+ assert d.engaged and d.int8_spill and d.refuse is None+++def test_no_denoiser_no_spill():+ comps = {"text_encoder": KLEIN9["text_encoder"]}+ d = _decide(comps, available = 2000, spill_free_mib = ROOMY_DISK)+ assert not d.int8_spill+++def test_check_disabled_still_spills(monkeypatch):+ # the in-RAM route was OOM-killed with the check off: spill whenever it does not fit+ monkeypatch.setenv(sh.HOST_RAM_CHECK_ENV, "0")+ d = _decide(spill_free_mib = ROOMY_DISK)+ assert d.int8_spill and d.refuse is None+++def _write_st(path: Path, tensors: dict) -> None:+ header, off = {}, 0+ for name, (dtype, shape) in tensors.items():+ n = 2+ for s in shape:+ n *= s+ header[name] = {"dtype": dtype, "shape": list(shape), "data_offsets": [off, off + n]}+ off += n+ blob = json.dumps(header).encode()+ path.write_bytes(struct.pack("<Q", len(blob)) + blob + b"\0" * off)+++def test_stored_components_reads_the_encoder_embedding(tmp_path):+ te = tmp_path / "text_encoder"+ te.mkdir()+ _write_st(+ te / "model.safetensors",+ {+ "model.embed_tokens.weight": ("BF16", (1024, 512)),+ "model.layers.0.mlp.up_proj.weight": ("BF16", (512, 512)),+ },+ )+ dit = tmp_path / "transformer"+ dit.mkdir()+ _write_st(+ dit / "diffusion_pytorch_model.safetensors", {"x_embedder.weight": ("BF16", (1024, 1024))}+ )+ comps = sh.stored_components(tmp_path)+ assert comps["text_encoder"].embed_mib == 1+ assert comps["transformer"].embed_mib == 0+ assert sh._safetensors_bytes_by_dtype(te / "model.safetensors") == {+ "bfloat16": 1024 * 512 * 2 + 512 * 512 * 2+ }+++class _Tiny(torch.nn.Module):+ def __init__(self):+ super().__init__()+ self.a = torch.nn.Linear(2048, 2048)+ self.b = torch.nn.Linear(2048, 4096, bias = False)+ self.small = torch.nn.Linear(64, 64)+ self.norm = torch.nn.LayerNorm(2048)++ def forward(self, x):+ return self.b(self.norm(self.a(x)))+++def _tiny(seed = 0):+ torch.manual_seed(seed)+ return _Tiny().to(torch.bfloat16)++[email protected](not _LINUX, reason = "spill is Linux only")+def test_spilled_int8_is_bit_identical_and_file_backed(tmp_path):+ ram, spilled = _tiny(), _tiny()+ s_ram = sh.quantize_int8_weight_(ram, compute_dtype = torch.float16, work_device = "cpu")+ s_sp = sh.quantize_int8_weight_(+ spilled, compute_dtype = torch.float16, work_device = "cpu", spill_dir = tmp_path+ )+ assert s_ram["linears"] == s_sp["linears"] == 2+ assert s_sp["spilled_bytes"] == s_sp["int8_bytes"] and s_ram.get("spilled_bytes", 0) == 0+ for name in ("a", "b"):+ r, s = getattr(ram, name), getattr(spilled, name)+ assert torch.equal(r.qweight, s.qweight) and torch.equal(r.scale, s.scale)+ assert s.qweight.untyped_storage().filename is not None+ assert r.qweight.untyped_storage().filename is None+ assert not list(tmp_path.iterdir())+ x = torch.randn(3, 2048, dtype = torch.float16)+ with torch.no_grad():+ assert torch.equal(ram(x), spilled(x))++[email protected](not _LINUX, reason = "spill is Linux only")+def test_spill_capacity_reads_free_disk_and_refuses_ram_filesystems(tmp_path, monkeypatch):+ free = sh.spill_capacity_mib(tmp_path)+ assert free is not None and free > 0+ if Path("/dev/shm").is_dir() and sh._fs_type(Path("/dev/shm")) == "tmpfs":+ assert sh.spill_capacity_mib(Path("/dev/shm")) is None+ monkeypatch.setenv(sh.SPILL_ENV, "0")+ assert sh.spill_capacity_mib(tmp_path) is None+++def test_spill_capacity_is_none_off_linux(tmp_path, monkeypatch):+ monkeypatch.setattr(sh.sys, "platform", "win32")+ assert sh.spill_capacity_mib(tmp_path) is None+++def test_spill_dir_env_wins(tmp_path, monkeypatch):+ monkeypatch.setenv(sh.SPILL_DIR_ENV, str(tmp_path / "x"))+ assert sh.spill_dir() == tmp_path / "x"++[email protected](not _LINUX, reason = "spill is Linux only")+def test_stale_spill_files_of_dead_processes_are_swept(tmp_path):+ dead = tmp_path / f"{sh._SPILL_PREFIX}999999999-abc.bin"+ dead.write_bytes(b"x")+ mine = tmp_path / f"{sh._SPILL_PREFIX}{os.getpid()}-abc.bin"+ mine.write_bytes(b"x")+ sh._sweep_stale_spills(tmp_path)+ assert not dead.exists() and mine.exists()+++_RSS_CHILD = textwrap.dedent(+ """+ import sys, torch+ sys.path.insert(0, {backend!r})+ import core.inference.diffusion_small_host as sh++ def anon():+ for line in open("/proc/self/status"):+ if line.startswith("RssAnon:"):+ return int(line.split()[1]) // 1024+ import safetensors.torch as st+ torch.manual_seed(0)+ def build():+ return torch.nn.Sequential(*[torch.nn.Linear(4096, 4096, bias=False) for _ in range(16)])+ src = {tmp!r} + "/src.safetensors"+ st.save_file({{k: v.to(torch.bfloat16) for k, v in build().state_dict().items()}}, src)+ with torch.device("meta"):+ m = build()+ # memory-mapped bf16 source, as the route loads the stored shards+ m.load_state_dict(st.load_file(src), assign=True)+ before = anon()+ kw = dict(spill_dir={spill!r}) if {spill!r} else dict()+ sh.quantize_int8_weight_(m, compute_dtype=torch.float16, work_device="cpu", **kw)+ print("DELTA", anon() - before)+ """+)++[email protected](not _LINUX, reason = "RssAnon is Linux only")[email protected]("spill", [False, True])+def test_int8_host_residency_bound(tmp_path, spill):+ # 16 x 4096^2 int8 = 256 MiB: the in-RAM route keeps all of it anonymous, the spill keeps it file-backed+ backend = str(Path(sh.__file__).resolve().parents[2])+ spill_dir = tmp_path / "spill"+ spill_dir.mkdir()+ code = _RSS_CHILD.format(+ backend = backend, tmp = str(tmp_path), spill = str(spill_dir) if spill else ""+ )+ out = subprocess.run(+ [sys.executable, "-c", code],+ capture_output = True,+ text = True,+ env = {**os.environ, "CUDA_VISIBLE_DEVICES": ""},+ timeout = 300,+ )+ assert out.returncode == 0, out.stderr[-2000:]+ delta = int(out.stdout.split("DELTA")[-1].split()[0])+ if spill:+ assert delta < 64, delta+ else:+ assert delta > 200, delta++[email protected](not _LINUX, reason = "spill is Linux only")+def test_spill_capacity_creates_nothing(tmp_path):+ target = tmp_path / "a" / "b"+ assert sh.spill_capacity_mib(target) is not None+ assert not (tmp_path / "a").exists()