Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
# SPDX-License-Identifier: AGPL-3.0-only
# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
"""Low-RAM load route, used only when a dense load would not fit RAM; UNSLOTH_DIFFUSION_SMALL_HOST=0|1."""
from __future__ import annotations
import json
import os
import re
import sys
import uuid
from dataclasses import dataclass, field
from pathlib import Path
from typing import Any, Optional
from .rocm_bf16 import is_rocm_torch
SMALL_HOST_ENV = "UNSLOTH_DIFFUSION_SMALL_HOST"
HOST_RAM_CHECK_ENV = "UNSLOTH_DIFFUSION_HOST_RAM_CHECK"
# test hook: caps every host reading the guards take (total, available, commit)
HOST_RAM_LIMIT_ENV = "UNSLOTH_DIFFUSION_HOST_RAM_LIMIT_MIB"
SMALL_HOST_ATTR = "_unsloth_small_host"
_MIB = 1024 * 1024
_STREAM_MIN_MIB = 512
_HOST_RESERVE_MIN_MIB = 3072
_HOST_RESERVE_FRACTION = 0.10
# Host RAM the fp8-storage rule (``diffusion_comfy_quant.fp8_storage_wanted``) keeps free when it lets a single file
# widen. The single-file refusal reserves at most this much, so it never refuses a widened load that rule chose.
FP8_STORAGE_HOST_RESERVE_MIB = 4096
# fp16 T5 keeps ``wo`` in fp32 (``_keep_in_fp32_modules``): 9.5 GB stored, 11.0 GB loaded.
_CONVERT_MARGIN = 1.15
_INT8_MIN_ELEMENTS = 1 << 22
# DiT denoisers the route stores as int8; a UNet (mostly convs) converts dense like any other non-encoder component
INT8_DENOISER_NAMES = ("transformer", "transformer_2", "unconditional_transformer")
def _env(name: str) -> str:
return (os.environ.get(name) or "").strip().lower()
def small_host_disabled() -> bool:
return _env(SMALL_HOST_ENV) in ("0", "off", "false", "no")
def small_host_forced() -> bool:
return _env(SMALL_HOST_ENV) in ("1", "on", "true", "yes", "force")
def host_ram_check_disabled() -> bool:
return _env(HOST_RAM_CHECK_ENV) in ("0", "off", "false", "no")
# Int8 spill: the route's int8 DiT weights live in a memory-mapped scratch file, not anonymous host RAM.
# Auto only when the in-RAM route does not fit; 0 = off, 1 = always.
SPILL_ENV = "UNSLOTH_DIFFUSION_SMALL_HOST_SPILL"
SPILL_DIR_ENV = "UNSLOTH_DIFFUSION_SMALL_HOST_SPILL_DIR"
_SPILL_DISK_MARGIN_MIB = 2048
# host bytes quantisation holds at once (bias, scale, D2H staging)
_SPILL_TRANSIENT_MIB = 256
_SPILL_ALIGN = 64
_SPILL_PREFIX = "unsloth-int8-spill-"
# a spill here would be RAM again
_RAM_FILESYSTEMS = frozenset({"tmpfs", "ramfs", "devtmpfs", "hugetlbfs", "rootfs"})
def spill_disabled() -> bool:
return _env(SPILL_ENV) in ("0", "off", "false", "no")
# A first load fetches the weights before the host-RAM decision, which reads their headers.
DOWNLOAD_FIRST_ENV = "UNSLOTH_DIFFUSION_SMALL_HOST_DOWNLOAD_FIRST"
def download_first_enabled() -> bool:
return _env(DOWNLOAD_FIRST_ENV) not in ("0", "off", "false", "no")
def spill_forced() -> bool:
return _env(SPILL_ENV) in ("1", "on", "true", "yes", "force")
def spill_dir() -> Path:
raw = (os.environ.get(SPILL_DIR_ENV) or "").strip()
if raw:
return Path(os.path.expanduser(raw))
try:
from utils.paths.storage_roots import cache_root as studio_cache_root
return studio_cache_root() / "diffusion_int8_spill"
except Exception: # noqa: BLE001
return Path.home() / ".cache" / "unsloth" / "diffusion_int8_spill"
def _fs_type(path: Path) -> Optional[str]:
try:
real = os.path.realpath(path)
best, kind = "", None
with open("/proc/mounts", encoding = "utf-8") as fh:
for line in fh:
parts = line.split()
if len(parts) < 3:
continue
mnt = parts[1].replace("\\040", " ")
inside = real == mnt or real.startswith(mnt.rstrip("/") + "/")
if inside and len(mnt) >= len(best):
best, kind = mnt, parts[2]
return kind
except Exception: # noqa: BLE001
return None
def spill_capacity_mib(directory: Any = None) -> Optional[int]:
"""Free MiB for the spill, None if off, not Linux, RAM-backed or unwritable."""
if spill_disabled() or not sys.platform.startswith("linux"):
return None
try:
import shutil
d = Path(directory) if directory is not None else spill_dir()
# nearest existing ancestor: no directory is created unless the spill runs
while not d.exists() and d.parent != d:
d = d.parent
kind = _fs_type(d)
if kind is None or kind in _RAM_FILESYSTEMS or not os.access(d, os.W_OK):
return None
return int(shutil.disk_usage(d).free // _MIB)
except Exception: # noqa: BLE001
return None
def _sweep_stale_spills(directory: Path) -> None:
try:
for f in directory.glob(f"{_SPILL_PREFIX}*.bin"):
try:
pid = int(f.name[len(_SPILL_PREFIX) :].split("-", 1)[0])
except ValueError:
continue
if pid == os.getpid():
continue
try:
os.kill(pid, 0)
continue
except ProcessLookupError:
pass
except PermissionError:
continue
try:
f.unlink()
except OSError:
pass
except Exception: # noqa: BLE001
pass
class Int8Spill:
"""Preallocated, unlinked scratch file mapped MAP_SHARED; ``store`` returns a file-backed copy of an int8 tensor."""
def __init__(self, directory: Any, nbytes: int):
import torch
d = Path(directory)
d.mkdir(parents = True, exist_ok = True)
_sweep_stale_spills(d)
self.nbytes = max(_SPILL_ALIGN, int(nbytes))
path = d / f"{_SPILL_PREFIX}{os.getpid()}-{uuid.uuid4().hex[:8]}.bin"
fd = os.open(path, os.O_RDWR | os.O_CREAT | os.O_EXCL, 0o600)
try:
try:
# reserve now: ENOSPC under a shared mapping is SIGBUS
os.posix_fallocate(fd, 0, self.nbytes)
finally:
os.close(fd)
self.flat = torch.from_file(str(path), shared = True, size = self.nbytes, dtype = torch.int8)
finally:
try:
path.unlink()
except OSError:
pass
self.offset = 0
def store(self, q: Any) -> Any:
n = int(q.numel())
start = -(-self.offset // _SPILL_ALIGN) * _SPILL_ALIGN
if start + n > self.nbytes:
raise RuntimeError("int8 spill file sized too small")
view = self.flat[start : start + n].view(q.shape)
view.copy_(q)
self.offset = start + n
return view
def spill_bytes_for(numels: list) -> int:
return sum(-(-int(n) // _SPILL_ALIGN) * _SPILL_ALIGN for n in numels)
_ST_DTYPES = {"BF16": "bfloat16", "F16": "float16", "F32": "float32"}
def _safetensors_bytes_by_dtype(path: Path, keep: tuple[str, ...] = ()) -> dict[str, int]:
return _header_sizes(path, keep)[0]
def _header_sizes(path: Path, keep: tuple[str, ...] = ()) -> tuple[dict[str, int], int]:
"""(bytes per dtype plus "kept", largest embedding bytes)."""
from .diffusion_component_sizes import read_safetensors_header
out: dict[str, int] = {}
embed = 0
header = read_safetensors_header(path, max_bytes = 256 * _MIB)
if header is None: # unreadable header: no claim about this file
return out, embed
for name, meta in header.items():
if name == "__metadata__":
continue
try:
start, end = meta["data_offsets"]
key = _ST_DTYPES.get(str(meta.get("dtype")), str(meta.get("dtype")).lower())
out[key] = out.get(key, 0) + int(end) - int(start)
if keep and any(f".{k}." in f".{name}." for k in keep):
out["kept"] = out.get("kept", 0) + int(end) - int(start)
if _is_embedding_name(name):
embed = max(embed, int(end) - int(start))
except Exception: # noqa: BLE001
continue
return out, embed
def _is_embedding_name(name: str) -> bool:
leaf = name.rsplit(".", 2)
return "embed" in name or (len(leaf) >= 2 and leaf[-2] == "shared")
@dataclass(frozen = True)
class StoredComponent:
name: str
mib: int
dtype: str # dominant stored float dtype ("bfloat16", "float16", "float32", ...)
# stored MiB of ``_keep_in_fp32_modules`` tensors: they convert to fp32 on the host even on the route
kept_fp32_mib: int = 0
library: str = ""
# largest embedding (text encoders), converted on the host
embed_mib: int = 0
def _keep_fp32_from_config(sub: Path) -> tuple[str, ...]:
try:
arch = (
json.loads((sub / "config.json").read_text(encoding = "utf-8")).get("architectures")
or [None]
)[0]
if not arch:
return ()
import transformers
return _keep_fp32_patterns(getattr(transformers, str(arch), None))
except Exception: # noqa: BLE001
return ()
def _weight_files(sub: Path) -> list[Path]:
files = sorted(sub.glob("*.safetensors"))
# ``model.fp16.safetensors`` next to ``model.safetensors`` is a variant from_pretrained does not read
plain = [f for f in files if "." not in f.name[: -len(".safetensors")]]
return plain or files
def snapshot_weights_complete(
snapshot: Any,
skip: Any = (),
variant: Optional[str] = None,
) -> bool:
"""Check pipeline weights for the requested variant, excluding caller-supplied components."""
try:
root = Path(snapshot)
index = json.loads((root / "model_index.json").read_text(encoding = "utf-8"))
except Exception: # noqa: BLE001
return True
for name, entry in index.items():
if name.startswith("_") or name in skip:
continue
if not (isinstance(entry, list) and len(entry) >= 2 and entry[0] and entry[1]):
continue
sub = root / name
if not sub.is_dir():
return False
files = list(sub.glob("*.safetensors")) + list(sub.glob("*.bin"))
shard_indexes = list(sub.glob("*.index.json"))
if variant:
marks = (f".{variant}.", f".{variant}-")
files = [f for f in files if any(m in f.name for m in marks)]
shard_indexes = [
f for f in sub.glob("*.index*.json") if any(m in f.name for m in marks)
]
if not files and (sub / "config.json").is_file():
return False # a model folder without weights; tokenizer / scheduler folders have no config.json
indexed_shards = set()
for shard_index in shard_indexes:
try:
shards = set(
json.loads(shard_index.read_text(encoding = "utf-8"))
.get("weight_map", {})
.values()
)
except Exception: # noqa: BLE001
return False
if any(not (sub / shard).is_file() for shard in shards):
return False
indexed_shards.update(shards)
# All numbered shards still need the index the loader opens.
if any(
re.search(r"-\d{5}-of-\d{5}(?:\.[^.]+)?\.(?:safetensors|bin)$", f.name)
and f.name not in indexed_shards
for f in files
):
return False
return True
def stored_components(snapshot: Any) -> dict[str, StoredComponent]:
"""Per component: stored MiB and dominant dtype, from safetensors headers only."""
out: dict[str, StoredComponent] = {}
try:
root = Path(snapshot)
if not root.is_dir():
return out
try:
index = json.loads((root / "model_index.json").read_text(encoding = "utf-8"))
except Exception: # noqa: BLE001
index = {}
for sub in sorted(p for p in root.iterdir() if p.is_dir()):
totals: dict[str, int] = {}
embed = 0
keep = _keep_fp32_from_config(sub) if sub.name.startswith("text_encoder") else ()
for f in _weight_files(sub):
sizes, largest = _header_sizes(f, keep)
embed = max(embed, largest)
for k, v in sizes.items():
totals[k] = totals.get(k, 0) + v
kept = totals.pop("kept", 0)
if not totals:
continue
dtype = max(totals.items(), key = lambda kv: kv[1])[0]
entry = index.get(sub.name)
library = str(entry[0]) if isinstance(entry, list) and entry and entry[0] else ""
out[sub.name] = StoredComponent(
sub.name,
-(-sum(totals.values()) // _MIB),
dtype,
-(-kept // _MIB),
library,
-(-embed // _MIB) if sub.name.startswith("text_encoder") else 0,
)
except Exception: # noqa: BLE001 - no claim
return {}
return out
def resolve_snapshot_dir(repo_or_dir: Any, cache_dir: Any = None) -> Optional[Path]:
"""Local diffusers dir for a repo id (cache only) or path."""
try:
p = Path(str(repo_or_dir)).expanduser()
if p.is_dir():
return p
from huggingface_hub import try_to_load_from_cache
hit = try_to_load_from_cache(str(repo_or_dir), "model_index.json", cache_dir = cache_dir)
if isinstance(hit, str):
return Path(hit).parent
except Exception: # noqa: BLE001
return None
return None
def _host_ram_limit_mib() -> int:
try:
return max(0, int((os.environ.get(HOST_RAM_LIMIT_ENV) or "").strip()))
except ValueError:
return 0
def _capped(value: Optional[int], limit: int) -> Optional[int]:
if limit <= 0:
return value
return limit if value is None else min(value, limit)
def host_ram_mib() -> tuple[Optional[int], Optional[int]]:
"""(total, available) host MiB, both capped by an enforcing cgroup and the ``HOST_RAM_LIMIT_ENV`` test hook."""
from .diffusion_memory import _available_system_memory_mib, _host_ram_capacity_mib
limit = _host_ram_limit_mib()
return _capped(_host_ram_capacity_mib(), limit), _capped(_available_system_memory_mib(), limit)
def host_commit_mib() -> Optional[int]:
"""Windows commit headroom MiB (None elsewhere or unreadable), capped by the ``HOST_RAM_LIMIT_ENV`` test hook."""
from .diffusion_memory import _windows_commit_headroom_mib
headroom = _windows_commit_headroom_mib()
return None if headroom is None else _capped(headroom, _host_ram_limit_mib())
def host_reserve_mib(total_mib: Optional[int]) -> int:
return max(_HOST_RESERVE_MIN_MIB, int((total_mib or 0) * _HOST_RESERVE_FRACTION))
def single_file_host_reserve_mib(total_mib: Optional[int]) -> int:
"""Reserve of the single-file host RAM refusal: the small-host reserve, capped at the fp8-storage rule's. Hosts
up to 40 GB are unchanged; above, 10% would refuse loads the fp8-storage rule widens because they fit."""
return min(host_reserve_mib(total_mib), FP8_STORAGE_HOST_RESERVE_MIB)
def _dtype_name(dtype: Any) -> str:
return str(dtype).replace("torch.", "")
def _itemsize(name: str) -> int:
return {"float32": 4, "bfloat16": 2, "float16": 2, "float8_e4m3fn": 1, "float8_e5m2": 1}.get(
name, 2
)
@dataclass(frozen = True)
class SmallHostDecision:
engaged: bool
reason: str
available_mib: Optional[int] = None
total_mib: Optional[int] = None
dense_host_mib: int = 0
route_host_mib: int = 0
commit_mib: Optional[int] = None
dense_commit_mib: int = 0
# component -> stored dtype it loads at (memory-mapped)
storage_dtypes: dict[str, str] = field(default_factory = dict)
refuse: Optional[str] = None
int8_spill: bool = False
spill_route_host_mib: int = 0
def decide_small_host(
components: dict[str, StoredComponent],
compute_dtype: Any,
*,
device: str,
host_total_mib: Optional[int],
host_available_mib: Optional[int],
lora_active: bool = False,
host_commit_mib: Optional[int] = None,
pread_libraries: tuple[str, ...] = (),
spill_free_mib: Optional[int] = None,
) -> SmallHostDecision:
"""Whether this pipeline load takes the small-host route. Only CUDA loads converting large bf16 components
(fp16 / fp32 compute) change; same-dtype loads already stay memory-mapped. ``host_commit_mib`` (Windows)
also routes a dense load whose commit need does not fit: on Windows commit exhaustion is an access
violation in native code, not a Python error (#13432, #14513). ``spill_free_mib``: ``spill_capacity_mib``, None = no spill."""
if small_host_disabled():
return SmallHostDecision(False, f"{SMALL_HOST_ENV}=0")
if str(device) != "cuda":
return SmallHostDecision(False, "not a CUDA load")
compute = _dtype_name(compute_dtype)
if compute not in ("float16", "float32"):
return SmallHostDecision(False, f"compute dtype {compute} needs no conversion")
converted = {
name: comp
for name, comp in components.items()
if comp.dtype == "bfloat16" and comp.mib >= _STREAM_MIN_MIB
}
if not converted:
return SmallHostDecision(False, "no large bf16 component to convert")
dense = int(
sum(c.mib * _itemsize(compute) / _itemsize(c.dtype) for c in converted.values())
* _CONVERT_MARGIN
)
# encoders stay memory-mapped except fp32-kept layers; a DiT lands as int8
route = sum(
c.mib // 2
if n in INT8_DENOISER_NAMES
else c.kept_fp32_mib * 4 // _itemsize(c.dtype)
if n.startswith("text_encoder")
else c.mib * _itemsize(compute) // _itemsize(c.dtype)
for n, c in converted.items()
)
reserve = host_reserve_mib(host_total_mib)
# Windows charges each copy-on-write safetensors view to commit until the load ends: stored bytes plus the
# converted copy; a component read through ``pread`` (``pread_libraries``) holds no view. Linux / macOS do not
# enforce commit.
dense_commit = dense + int(
sum(c.mib for c in converted.values() if c.library not in pread_libraries) * _CONVERT_MARGIN
)
storage = {name: comp.dtype for name, comp in converted.items()}
forced = small_host_forced()
if host_available_mib is None and not forced:
return SmallHostDecision(False, "host RAM unreadable", dense_host_mib = dense)
commit = None if host_commit_mib is None else int(host_commit_mib)
commit_fits = commit is None or dense_commit + reserve <= commit
fits = (
host_available_mib is not None
and dense + reserve <= int(host_available_mib)
and commit_fits
)
commit_note = (
""
if commit is None
else f", {commit} MiB commit headroom (dense needs ~{dense_commit} MiB)"
)
extra = {"commit_mib": commit, "dense_commit_mib": dense_commit}
if fits and not forced:
return SmallHostDecision(
False,
f"dense {compute} load ({dense} MiB) fits {host_available_mib} MiB available host RAM{commit_note}",
host_available_mib,
host_total_mib,
dense,
route,
**extra,
)
if lora_active and not forced:
# the int8 denoiser cannot carry adapters
return SmallHostDecision(
False,
f"LoRA adapters need the dense denoiser (dense load ~{dense} MiB, {host_available_mib} MiB available"
f"{commit_note})",
host_available_mib,
host_total_mib,
dense,
route,
**extra,
)
spill, spill_route = _spill_choice(
converted, compute, route, reserve, host_available_mib, spill_free_mib
)
need = spill_route if spill else route
refuse = None
if (
host_available_mib is not None
and need + reserve > int(host_available_mib)
and not host_ram_check_disabled()
):
refuse = _refusal(need, reserve, host_available_mib, "even the low-memory route")
why = (
f"{SMALL_HOST_ENV}=1"
if forced and fits
else f"dense {compute} load needs ~{dense} MiB + {reserve} MiB reserve of host RAM, "
f"{host_available_mib} MiB available{commit_note}"
)
return SmallHostDecision(
True,
why,
host_available_mib,
host_total_mib,
dense,
route,
storage_dtypes = storage,
refuse = refuse,
int8_spill = spill,
spill_route_host_mib = spill_route if spill else 0,
**extra,
)
def _spill_choice(
converted: dict[str, StoredComponent],
compute: str,
route: int,
reserve: int,
available: Optional[int],
spill_free_mib: Optional[int],
) -> tuple[bool, int]:
"""(spill, host MiB of the spilled route). Auto only when the in-RAM route does not fit."""
dit = sum(c.mib // 2 for n, c in converted.items() if n in INT8_DENOISER_NAMES)
if not dit or spill_free_mib is None or spill_disabled():
return False, route
short = available is not None and route + reserve > int(available)
if not (short or spill_forced()) or dit + _SPILL_DISK_MARGIN_MIB > int(spill_free_mib):
return False, route
# route omits the converted encoder embedding
embed = sum(
c.embed_mib * _itemsize(compute) // _itemsize(c.dtype)
for n, c in converted.items()
if n.startswith("text_encoder")
)
return True, route - dit + embed + _SPILL_TRANSIENT_MIB
def _refusal(need: int, reserve: int, available: Optional[int], what: str) -> str:
return (
f"Not enough system RAM to load this model: {what} needs about {(need + reserve) / 1024:.1f} GB "
f"of host memory and {((available or 0)) / 1024:.1f} GB is available. Close other applications, "
f"pick a smaller or GGUF model, or set {HOST_RAM_CHECK_ENV}=0 to try anyway."
)
def torch_dtype_map(decision: SmallHostDecision, compute_dtype: Any) -> Any:
"""``torch_dtype`` for ``DiffusionPipeline.from_pretrained``: stored dtype for the routed components."""
if not decision.engaged or not decision.storage_dtypes:
return compute_dtype
import torch
mapping: dict[str, Any] = {"default": compute_dtype}
for name, dtype in decision.storage_dtypes.items():
mapping[name] = getattr(torch, dtype)
return mapping
INT8_ACT_ENV = "UNSLOTH_DIFFUSION_SMALL_HOST_INT8_ACT"
# Measured on a T4 (LPIPS inside the fp32-vs-bf16 spread). qwen-image-edit shares the DiT: 2.03x per step, same
# LPIPS distance to the bf16 reference as weight-only int8.
INT8_ACT_FAMILIES = frozenset({"qwen-image", "qwen-image-edit"})
# torch._int_mm: rows M > 16, K and N multiples of 8.
_INT8_ACT_MIN_ROWS = 17
_INT8_ACT_ALIGN = 8
_INT8_ACT_DEVICE_OK: dict[int, bool] = {}
_INT8_ACT_COUNTS = {"int8_act": 0, "dequant": 0}
def int8_act_disabled() -> bool:
return _env(INT8_ACT_ENV) in ("0", "off", "false", "no")
def int8_act_family(fam: Any) -> bool:
"""Whether ``fam``'s int8-stored denoiser may run W8A8 (still gated per device and per call)."""
if int8_act_disabled():
return False
name = str(getattr(fam, "name", fam) or "").strip().lower()
return name in INT8_ACT_FAMILIES
def int8_act_counts() -> dict[str, int]:
return dict(_INT8_ACT_COUNTS)
def int8_act_device_ok(device: Any) -> bool:
"""sm_75 CUDA only (fp32-promoted, int8 tensor cores; on sm_80+ bf16 eager W8A8 does not win; ROCm capabilities
differ), and ``torch._int_mm`` must return the exact integer product there."""
import torch
try:
dev = torch.device(device)
if dev.type != "cuda" or is_rocm_torch(torch):
return False
idx = dev.index if dev.index is not None else torch.cuda.current_device()
except Exception: # noqa: BLE001
return False
hit = _INT8_ACT_DEVICE_OK.get(idx)
if hit is not None:
return hit
ok = False
try:
if tuple(torch.cuda.get_device_capability(idx)) == (7, 5) and hasattr(torch, "_int_mm"):
g = torch.Generator(device = "cpu").manual_seed(0)
a = torch.randint(-127, 128, (32, 64), generator = g, dtype = torch.int8)
b = torch.randint(-127, 128, (48, 64), generator = g, dtype = torch.int8)
want = a.to(torch.int64) @ b.to(torch.int64).t()
got = torch._int_mm(a.to(dev), b.to(dev).t())
ok = bool(torch.equal(got.cpu().to(torch.int64), want))
except Exception: # noqa: BLE001 - no usable int8 GEMM: keep the dequantised path
ok = False
_INT8_ACT_DEVICE_OK[idx] = ok
return ok
def enable_int8_act_(module: Any) -> int:
"""Let every int8-stored Linear of ``module`` run W8A8 (an explicit int8 request on an fp16-only card); returns
how many. The per-call gate is unchanged: float32 CUDA input on sm_75 with an exact ``_int_mm``, else dequantise.
``UNSLOTH_DIFFUSION_SMALL_HOST_INT8_ACT=0`` keeps them dequantised."""
if int8_act_disabled() or _INT8_CLS is None:
return 0
count = 0
for sub in module.modules():
if isinstance(sub, _INT8_CLS):
sub.act_int8 = True
count += 1
return count
def _int8_linear_class():
import torch
import torch.nn.functional as F
class Int8WeightLinear(torch.nn.Module):
"""Linear with int8 per-output-channel weights, dequantised to the input dtype for each forward.
``act_int8``: eligible float32 CUDA calls quantise activations per row and run ``torch._int_mm`` instead."""
def __init__(self, qweight, scale, bias, in_features: int, out_features: int):
super().__init__()
self.in_features = in_features
self.out_features = out_features
self.qweight = torch.nn.Parameter(qweight, requires_grad = False)
self.scale = torch.nn.Parameter(scale, requires_grad = False)
self.bias = None if bias is None else torch.nn.Parameter(bias, requires_grad = False)
self.act_int8 = False
def _int8_act_ok(self, x) -> bool:
if not self.act_int8 or x.dtype is not torch.float32 or not x.is_cuda:
return False
if self.in_features % _INT8_ACT_ALIGN or self.out_features % _INT8_ACT_ALIGN:
return False
if x.numel() // max(1, self.in_features) < _INT8_ACT_MIN_ROWS:
return False
return int8_act_device_ok(x.device)
def _forward_int8_act(self, x):
x2 = x.reshape(-1, self.in_features)
xs = torch.linalg.vector_norm(x2, ord = float("inf"), dim = 1, keepdim = True)
xs = xs.clamp_min_(1e-12).div_(127.0)
xq = torch.div(x2, xs).round_().clamp_(-127, 127).to(torch.int8)
acc = torch._int_mm(xq, self.qweight.t())
y = torch.mul(acc, xs).mul_(self.scale.reshape(1, -1).to(torch.float32))
if self.bias is not None:
y.add_(self.bias.to(torch.float32))
return y.reshape(*x.shape[:-1], self.out_features)
def forward(self, x):
# Eager-only: mutating a global int under Dynamo recompiles every call.
count = not torch.compiler.is_compiling()
if self._int8_act_ok(x):
if count:
_INT8_ACT_COUNTS["int8_act"] += 1
return self._forward_int8_act(x)
if self.act_int8 and count:
_INT8_ACT_COUNTS["dequant"] += 1
scale = self.scale
if scale.dtype == x.dtype:
# int8 * float promotes: cast + scale in one pass, bit-identical (int8 -> fp16/fp32 is exact).
w = torch.mul(self.qweight, scale)
else:
w = self.qweight.to(x.dtype) * scale.to(x.dtype)
return F.linear(x, w, None if self.bias is None else self.bias.to(x.dtype))
def extra_repr(self) -> str:
return f"in_features={self.in_features}, out_features={self.out_features}, int8 weight"
return Int8WeightLinear
_INT8_CLS = None
def int8_linear_class():
global _INT8_CLS
if _INT8_CLS is None:
_INT8_CLS = _int8_linear_class()
return _INT8_CLS
def quantize_int8_weight_(
module: Any,
*,
compute_dtype: Any,
work_device: Any,
keep_device: Any = "cpu",
act_int8: bool = False,
spill_dir: Any = None,
) -> dict[str, int]:
"""Replace every large ``nn.Linear`` with per-row int8 weights in place, one Linear at a time (quantised on
``work_device``, stored on ``keep_device``); everything else casts to ``compute_dtype``. ``act_int8`` lets the
new Linears run W8A8 where ``Int8WeightLinear`` allows it. ``spill_dir``: int8 weights go to an ``Int8Spill``
there (same bytes, file-backed)."""
import torch
cls = int8_linear_class()
work = torch.device(work_device)
keep = torch.device(keep_device)
stats = {"linears": 0, "dense_linears": 0, "int8_bytes": 0}
targets = [
(name, child) for name, child in module.named_modules() if type(child) is torch.nn.Linear
]
spill = None
if spill_dir is not None:
sizes = [
lin.weight.numel() for _, lin in targets if lin.weight.numel() >= _INT8_MIN_ELEMENTS
]
spill = Int8Spill(spill_dir, spill_bytes_for(sizes))
for name, lin in targets:
if lin.weight.numel() < _INT8_MIN_ELEMENTS:
stats["dense_linears"] += 1
continue
with torch.no_grad():
w = lin.weight.detach().to(work, non_blocking = False).float()
scale = w.abs().amax(dim = 1, keepdim = True).clamp_min(1e-12) / 127.0
q = torch.round(w / scale).clamp_(-127, 127).to(torch.int8)
bias = None if lin.bias is None else lin.bias.detach().to(keep, compute_dtype)
new = cls(
q.to(keep) if spill is None else spill.store(q),
scale.to(compute_dtype).to(keep),
bias,
lin.in_features,
lin.out_features,
)
new.act_int8 = bool(act_int8)
del w, q, scale
parent_name, _, attr = name.rpartition(".")
parent = module.get_submodule(parent_name) if parent_name else module
setattr(parent, attr, new)
stats["linears"] += 1
stats["int8_bytes"] += new.qweight.numel()
if spill is not None:
stats["spilled_bytes"] = stats.get("spilled_bytes", 0) + new.qweight.numel()
with torch.no_grad():
for sub in module.modules():
if isinstance(sub, cls):
continue
for pname, p in list(sub.named_parameters(recurse = False)):
if p.dtype == torch.bfloat16 and p.dtype != compute_dtype:
p.data = p.data.to(keep, compute_dtype)
elif p.device != keep:
p.data = p.data.to(keep)
for bname, b in list(sub.named_buffers(recurse = False)):
if b is not None and b.dtype == torch.bfloat16 and b.dtype != compute_dtype:
sub._buffers[bname] = b.to(keep, compute_dtype)
if work.type == "cuda":
torch.cuda.empty_cache()
return stats
def _keep_fp32_patterns(module: Any) -> tuple[str, ...]:
pats = getattr(module, "_keep_in_fp32_modules", None) or ()
strict = getattr(module, "_keep_in_fp32_modules_strict", None) or ()
return tuple(str(p) for p in (*pats, *strict))
def cast_resident_(module: Any, device: Any, compute_dtype: Any) -> None:
"""Move ``module`` to ``device`` one tensor at a time, cast as the dense load would (fp32 for kept modules)."""
import torch
dev = torch.device(device)
keep = _keep_fp32_patterns(module)
with torch.no_grad():
for mname, sub in module.named_modules():
want = torch.float32 if keep and any(k in mname for k in keep) else compute_dtype
for pname, p in list(sub.named_parameters(recurse = False)):
dt = want if p.dtype == torch.bfloat16 else p.dtype
p.data = p.data.to(dev).to(dt)
for bname, b in list(sub.named_buffers(recurse = False)):
if b is None:
continue
dt = want if b.dtype == torch.bfloat16 else b.dtype
sub._buffers[bname] = b.to(dev).to(dt)
def prepare_streamed_encoder_(module: Any, compute_dtype: Any) -> int:
"""Layerwise-cast a memory-mapped bf16 encoder: Linear / Embedding weights keep their stored dtype, everything
else converts now, including the owner of the first float parameter (``module.dtype`` sets the prompt-embedding
dtype). Returns the MiB left memory-mapped."""
import torch
from diffusers.hooks import apply_layerwise_casting
keep = _keep_fp32_patterns(module)
streamable = (torch.nn.Linear, torch.nn.Embedding)
def _kept(mname: str) -> bool:
return bool(keep) and any(k in mname for k in keep)
def _want(mname: str) -> Any:
return torch.float32 if _kept(mname) else compute_dtype
def _convert(sub: Any, want: Any) -> None:
# fp32 buffers (RoPE ``inv_freq``) stay fp32, as in the dense load
for pname, p in list(sub.named_parameters(recurse = False)):
if p.dtype == torch.bfloat16 and p.dtype != want:
p.data = p.data.to(want)
for bname, b in list(sub.named_buffers(recurse = False)):
if b is not None and b.dtype == torch.bfloat16 and b.dtype != want:
sub._buffers[bname] = b.to(want)
first_owner = None
for pname, p in module.named_parameters():
if p.is_floating_point():
first_owner = pname.rpartition(".")[0]
break
streamed = 0
with torch.no_grad():
for mname, sub in module.named_modules():
want = _want(mname)
# kept-fp32 Linears convert now: T5 casts its input to ``wo.weight.dtype`` before the hook runs
if (
type(sub) in streamable
and not _kept(mname)
and mname != first_owner
and sub.weight.dtype != want
):
storage = sub.weight.dtype
streamed += sub.weight.numel() * sub.weight.element_size()
apply_layerwise_casting(
sub,
storage_dtype = storage,
compute_dtype = want,
skip_modules_pattern = None,
skip_modules_classes = None,
)
sub._unsloth_mmap_streamed = True
continue
_convert(sub, want)
return streamed // _MIB
# Small hosts drop mmapped encoder pages between prompts (FLUX.1 T5 on a 31 GB T4: ~7 GB re-read, 18-24 s/prompt).
ENCODER_RESIDENT_ENV = "UNSLOTH_DIFFUSION_SMALL_HOST_TE_RESIDENT"
# Host RAM still taken after the route: pinned DiT staging (~2 GB on FLUX.1) + encoder prefetch ring.
_ENCODER_RESIDENT_RUNTIME_MIB = 2560
def _mmap_streamed_tensors(module: Any) -> list:
"""(owner, name, tensor) of every CPU parameter / buffer left in storage dtype by ``prepare_streamed_encoder_``."""
out: list = []
seen: set = set()
for sub in module.modules():
if not getattr(sub, "_unsloth_mmap_streamed", False):
continue
for kind, items in (
("p", sub.named_parameters(recurse = False)),
("b", sub.named_buffers(recurse = False)),
):
for name, t in items:
if t is None or t.device.type != "cpu" or t.numel() == 0:
continue
key = t.untyped_storage().data_ptr()
if key in seen:
continue
seen.add(key)
out.append((sub, kind, name, t))
return out
def streamed_encoder_mib(module: Any) -> int:
total = sum(int(t.numel()) * int(t.element_size()) for *_x, t in _mmap_streamed_tensors(module))
return -(-total // _MIB)
def encoder_resident_mode() -> Optional[bool]:
value = _env(ENCODER_RESIDENT_ENV)
if value in ("0", "off", "false", "no"):
return False
if value in ("1", "on", "true", "yes"):
return True
return None
def encoder_resident_keep_mib(host_total_mib: Optional[int]) -> int:
return host_reserve_mib(host_total_mib) + _ENCODER_RESIDENT_RUNTIME_MIB
def encoder_resident_share_mib(
streamed_mib: int,
*,
host_total_mib: Optional[int],
host_available_mib: Optional[int],
host_commit_mib: Optional[int] = None,
min_share_mib: int = 0,
) -> tuple[int, str]:
"""MiB of ``streamed_mib`` that fit under ``decide_encoder_resident``'s reserve; ``host_available_mib`` read after placement."""
if streamed_mib <= 0:
return 0, "no memory-mapped encoder weights"
if host_available_mib is None:
return 0, "host RAM unreadable"
keep = encoder_resident_keep_mib(host_total_mib)
room = int(host_available_mib) - keep
if host_commit_mib is not None:
room = min(room, int(host_commit_mib) - keep)
share = max(0, min(int(streamed_mib), room))
if share <= 0 or share < int(min_share_mib):
return (
0,
f"{host_available_mib} MiB available host RAM leaves {max(0, room)} MiB above the "
f"{keep} MiB the load keeps",
)
return (
share,
f"{share} of {streamed_mib} MiB leaves {int(host_available_mib) - share} MiB of "
f"{host_available_mib} MiB available host RAM (keeps {keep} MiB)",
)
def decide_encoder_resident(
streamed_mib: int,
*,
host_total_mib: Optional[int],
host_available_mib: Optional[int],
host_commit_mib: Optional[int] = None,
int8_spill: bool = False,
) -> tuple[bool, str]:
"""``host_available_mib`` is read after placement, so it already excludes the int8 DiT."""
mode = encoder_resident_mode()
if streamed_mib <= 0:
return False, "no memory-mapped encoder weights"
if mode is False:
return False, f"{ENCODER_RESIDENT_ENV}=0"
if mode is True:
return True, f"{ENCODER_RESIDENT_ENV}=1"
if int8_spill:
return False, "the int8 denoiser already spills to disk"
if host_available_mib is None:
return False, "host RAM unreadable"
keep = encoder_resident_keep_mib(host_total_mib)
left = int(host_available_mib) - int(streamed_mib)
if left < keep:
return (
False,
f"{streamed_mib} MiB would leave {left} MiB of {host_available_mib} MiB available host RAM, "
f"under the {keep} MiB the load keeps",
)
if host_commit_mib is not None and int(host_commit_mib) - int(streamed_mib) < keep:
return (
False,
f"{streamed_mib} MiB would leave {int(host_commit_mib) - int(streamed_mib)} MiB of commit headroom, "
f"under the {keep} MiB the load keeps",
)
return (
True,
f"{streamed_mib} MiB leaves {left} MiB of {host_available_mib} MiB available host RAM",
)
def materialize_streamed_encoder_(module: Any, cancel_check: Any = None) -> int:
"""Must run before group offload records the host tensors. Returns MiB copied."""
import torch
copied = 0
with torch.no_grad():
for sub, kind, name, t in _mmap_streamed_tensors(module):
fresh = torch.empty_like(t, memory_format = torch.contiguous_format)
fresh.copy_(t)
if kind == "p":
getattr(sub, name).data = fresh
else:
sub._buffers[name] = fresh
copied += int(t.numel()) * int(t.element_size())
if cancel_check is not None:
cancel_check()
return -(-copied // _MIB)
def module_dtype_ok(module: Any, compute_dtype: Any) -> bool:
try:
return getattr(module, "dtype", compute_dtype) == compute_dtype
except Exception: # noqa: BLE001
return True
def mark(pipe: Any, info: dict) -> None:
try:
setattr(pipe, SMALL_HOST_ATTR, dict(info))
except Exception: # noqa: BLE001
pass
def engaged_on(pipe: Any) -> Optional[dict]:
return getattr(pipe, SMALL_HOST_ATTR, None)
ENCODER_PREFETCH_ENV = "UNSLOTH_DIFFUSION_SMALL_HOST_PREFETCH"
ENCODER_PREFETCH_ATTR = "_unsloth_encoder_prefetch"
# Pinned staging ring: the only host bytes this adds.
_STAGE_SLOT_BYTES = 32 * _MIB
_STAGE_SLOTS = 4
# Device bytes copied ahead: a few groups (T5-XXL MLP weight 80 MiB, Qwen2.5-VL 130 MiB).
_PREFETCH_MAX_BYTES = 384 * _MIB
_PREFETCH_MIN_BYTES = 128 * _MIB
def encoder_prefetch_disabled() -> bool:
return _env(ENCODER_PREFETCH_ENV) in ("0", "off", "false", "no")
def _group_tensors(group: Any) -> list:
"""(tensor, host source) pairs of a diffusers offload group, in the order diffusers onloads them."""
out: list = []
seen: set[int] = set()
cpu = getattr(group, "cpu_param_dict", None) or {}
for t in (
[p for m in group.modules for p in m.parameters()]
+ [b for m in group.modules for b in m.buffers()]
+ list(getattr(group, "parameters", None) or [])
+ list(getattr(group, "buffers", None) or [])
):
if id(t) in seen:
continue
seen.add(id(t))
out.append((t, cpu.get(t, t)))
return out
def _prefetch_budget_bytes(device: Any) -> int:
"""Bytes of encoder groups the prefetcher may hold on ``device`` ahead of compute."""
try:
import torch
from .diffusion_memory import user_budget_reserve_bytes
free, total = torch.cuda.mem_get_info(device)
reserve = user_budget_reserve_bytes(device, total)
free = max(0, int(free) - reserve)
if reserve > 0 and free < _PREFETCH_MIN_BYTES:
return 0
return max(_PREFETCH_MIN_BYTES, min(_PREFETCH_MAX_BYTES, free // 8))
except Exception: # noqa: BLE001
return _PREFETCH_MIN_BYTES
def _prefetch_budget_lowered(device: Any) -> bool:
"""True when the user's VRAM Budget keeps a reserve on ``device``."""
try:
import torch
from .diffusion_memory import user_budget_reserve_bytes
_free, total = torch.cuda.mem_get_info(device)
return user_budget_reserve_bytes(device, total) > 0
except Exception: # noqa: BLE001
return False
class _EncoderPrefetcher:
"""Copies a memory-mapped encoder's offload groups ahead of its forward, in the previous forward's order, on a
worker thread through a pinned ring (diffusers pins each group on the calling thread). Off-order groups and worker
errors fall back to a synchronous copy."""
def __init__(self, module: Any, groups: list, device: Any):
import threading
import torch
self.module = module
self.groups = groups
dev = torch.device(device)
if dev.type == "cuda" and dev.index is None:
dev = torch.device("cuda", torch.cuda.current_device())
self.device = dev
self.stream = None # side stream, created on the first copy
self.compute = None # the stream the encoder runs on, read when a forward begins
self.order: list = [] # group ids in the last forward's onload order
self.seen: list = []
self.active = False
self.cond = threading.Condition()
self.ready: dict = {} # id(group) -> (device tensors, event)
self.inflight = 0
self.budget = _PREFETCH_MIN_BYTES
self.strict = False # lowered Budget: an ahead group must fit the window, else it copies synchronously
self.skipped: dict = {} # group id -> times the worker left to a synchronous copy
self.pos = 0
self.stop = False
self.worker = None
self.error: Optional[BaseException] = None
self.stage: list = []
self.stage_events: list = []
self.slot = 0
# The worker and a synchronous onload copy (a group skipped under a lowered Budget) share the staging ring.
self.copy_lock = threading.Lock()
self.stats = {"prefetched": 0, "sync": 0, "passes": 0}
self.by_id = {id(g): g for g in groups}
def _side_stream(self) -> Any:
import torch
if self.stream is None:
self.stream = torch.cuda.Stream(device = self.device)
return self.stream
def _ensure_stage(self) -> None:
import torch
if not self.stage:
# inference_mode is thread-local: the worker cannot write inference tensors made on the forward thread.
with torch.inference_mode(False):
self.stage = [
torch.empty(_STAGE_SLOT_BYTES, dtype = torch.uint8, pin_memory = True)
for _ in range(_STAGE_SLOTS)
]
self.stage_events = [None] * _STAGE_SLOTS
def _copy_group(self, group: Any) -> tuple:
"""Device copies of ``group``'s host tensors, issued on the side stream. Returns (tensors, event, bytes)."""
# Held across staging write, H2D enqueue and slot event so the other thread cannot reuse a slot in between.
with self.copy_lock:
return self._copy_group_locked(group)
def _copy_group_locked(self, group: Any) -> tuple:
import torch
self._ensure_stage()
stream = self._side_stream()
compute = self.compute or torch.cuda.current_stream(self.device)
moved: list = []
nbytes = 0
with torch.cuda.stream(stream):
for _t, src in _group_tensors(group):
n = int(src.numel()) * int(src.element_size())
if src.device.type != "cpu" or n == 0 or not src.is_contiguous() or src.is_pinned():
with torch.cuda.stream(compute):
moved.append(
src.to(
self.device,
non_blocking = src.device.type != "cpu" or src.is_pinned(),
)
)
continue
nbytes += n
# Copy waits on the compute stream so it covers the last user of a freed block.
with torch.cuda.stream(compute):
dst = torch.empty(src.shape, dtype = src.dtype, device = self.device)
queued = torch.cuda.Event()
queued.record(compute)
stream.wait_event(queued)
sb = src.reshape(-1).view(torch.uint8)
db = dst.reshape(-1).view(torch.uint8)
off = 0
while off < n:
k = min(_STAGE_SLOT_BYTES, n - off)
i = self.slot
self.slot = (i + 1) % _STAGE_SLOTS
ev = self.stage_events[i]
if ev is not None:
ev.synchronize()
buf = self.stage[i][:k]
buf.copy_(sb[off : off + k])
db[off : off + k].copy_(buf, non_blocking = True)
ev = torch.cuda.Event()
ev.record(stream)
self.stage_events[i] = ev
off += k
# A dropped entry (halt, exception) frees dst before the copy lands.
dst.record_stream(stream)
moved.append(dst)
done = torch.cuda.Event()
done.record(stream)
return moved, done, nbytes
def _group_bytes(self, group: Any) -> int:
"""Host bytes :meth:`_copy_group` would stage for ``group``."""
n = 0
for _t, src in _group_tensors(group):
if src.device.type == "cpu" and src.is_contiguous() and not src.is_pinned():
n += int(src.numel()) * int(src.element_size())
return n
def _attach(self, group: Any, moved: list, done: Any) -> None:
import torch
cur = torch.cuda.current_stream(self.device)
cur.wait_event(done)
if getattr(group, "stream", None) is not None:
# diffusers fences its own-stream prefetch only in the next group's onload_, which this replaces.
cur.wait_stream(group.stream)
for (t, _src), dev in zip(_group_tensors(group), moved):
t.data = dev
dev.record_stream(cur)
def _run(self, order: list) -> None:
import torch
try:
if self.device.type == "cuda":
torch.cuda.set_device(self.device)
for gid in order:
group = self.by_id.get(gid)
need = self._group_bytes(group) if self.strict and group is not None else 0
with self.cond:
if need > self.budget:
self.skipped[gid] = self.skipped.get(gid, 0) + 1
self.cond.notify_all()
continue
# A group the forward onloads twice waits for its first copy to be taken.
while not self.stop and (
(
self.inflight + need > self.budget
if self.strict
else self.inflight > 0 and self.inflight >= self.budget
)
or gid in self.ready
):
self.cond.wait(0.05)
if self.stop:
return
if group is None:
continue
moved, done, n = self._copy_group(group)
with self.cond:
if self.stop:
return
self.ready[gid] = (moved, done, n)
self.inflight += n
self.cond.notify_all()
except BaseException as exc: # noqa: BLE001 - every group falls back to a synchronous copy
with self.cond:
self.error = exc
self.cond.notify_all()
def _halt(self) -> None:
with self.cond:
self.stop = True
self.cond.notify_all()
if self.worker is not None:
self.worker.join()
self.worker = None
with self.cond:
self.ready.clear()
self.skipped.clear()
self.inflight = 0
def begin(self) -> None:
import threading
import torch
self._halt()
self.stop = False
self.error = None
self.seen = []
self.pos = 0
self.active = True
self.stats["passes"] += 1
self.compute = (
torch.cuda.current_stream(self.device) if self.device.type == "cuda" else None
)
if not self.order:
return # first forward: synchronous copies, and it records the order
self.budget = _prefetch_budget_bytes(self.device)
if self.budget <= 0:
return # no budgeted room: every group copies synchronously, as on the first forward
self.strict = _prefetch_budget_lowered(self.device)
self._order_after_compute()
self.worker = threading.Thread(
target = self._run, args = (list(self.order),), daemon = True, name = "unsloth-encoder-prefetch"
)
self.worker.start()
def _order_after_compute(self) -> None:
"""The worker's copies must not overtake work the compute stream already queued (the prompt's input ids)."""
import torch
self._side_stream().wait_stream(torch.cuda.current_stream(self.device))
def end(self) -> None:
if not self.active:
return
self.active = False
self._halt()
if self.seen:
self.order = list(self.seen)
def onload(self, group: Any) -> None:
gid = id(group)
if self.active:
self.seen.append(gid)
entry = None
if self.worker is not None:
expected = self.order[self.pos] if self.pos < len(self.order) else None
if expected == gid:
self.pos += 1
with self.cond:
while (
gid not in self.ready
and not self.skipped.get(gid)
and self.error is None
and self.worker.is_alive()
):
self.cond.wait(0.05)
entry = self.ready.pop(gid, None)
if entry is None and self.skipped.get(gid):
self.skipped[gid] -= 1
if entry is not None:
self.inflight -= entry[2]
self.cond.notify_all()
else:
# Off the recorded order: stop prefetching for this forward; the next one re-records the order.
self._halt()
if entry is None:
moved, done, _n = self._copy_group(group)
self.stats["sync"] += 1
else:
moved, done, _n = entry
self.stats["prefetched"] += 1
self._attach(group, moved, done)
def install_encoder_prefetch(
module: Any,
device: Any,
logger: Any = None,
) -> int:
"""Swap each streamed offload group's ``onload_`` of a small-host text encoder for the prefetching copy. Groups made
resident are left alone. Returns the number of groups covered (0: unchanged, e.g. kill switch or no CUDA)."""
if encoder_prefetch_disabled():
return 0
try:
import torch
if torch.device(device).type != "cuda" or not torch.cuda.is_available():
return 0
from .diffusion_memory import GROUP_OFFLOAD_PIN_ENV, _offload_groups
if _env(GROUP_OFFLOAD_PIN_ENV) in ("0", "off", "false", "no"):
return 0 # the ring is pinned memory: the user's "pin nothing" override wins
from .host_pinning import host_pinning_allowed
if not host_pinning_allowed(device):
return 0 # integrated GPU: pinned memory only takes RAM from the GPU
groups = [
g
for g in _offload_groups(module)
if not getattr(g, "_unsloth_resident", False)
and getattr(g, "stream", None) is not None
and not getattr(g, "offload_to_disk_path", None)
]
if not groups:
return 0
pf = _EncoderPrefetcher(module, groups, device)
pf._ensure_stage() # a pinned allocation failing here keeps diffusers' onload, not a failed render
disable = getattr(getattr(torch, "compiler", None), "disable", None)
for group in groups:
def onload_(
*_a: Any,
_g: Any = group,
**_k: Any,
) -> None:
pf.onload(_g)
group.onload_ = disable(onload_) if callable(disable) else onload_
module.register_forward_pre_hook(lambda *_a, **_k: pf.begin())
module.register_forward_hook(lambda *_a, **_k: pf.end(), always_call = True)
setattr(module, ENCODER_PREFETCH_ATTR, pf)
if logger is not None:
logger.info(
"diffusion.small_host: %s streams %d offload groups through a prefetching copy",
type(module).__name__,
len(groups),
)
return len(groups)
except Exception as exc: # noqa: BLE001 - keep diffusers' own onload
if logger is not None:
logger.warning("diffusion.small_host: encoder prefetch unavailable (%s)", exc)
return 0