Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

Studio: build offloaded video int8 denoisers per shard when the dense load does not fit host RAM (#16290)

Daniel Han8af3535ff130
diff --git a/studio/backend/core/inference/diffusion_native_cache.py b/studio/backend/core/inference/diffusion_native_cache.py
index 7c354efccfd..0d03a1e715e 100644
--- a/studio/backend/core/inference/diffusion_native_cache.py
+++ b/studio/backend/core/inference/diffusion_native_cache.py
@@ -545,6 +545,8 @@ _STREAM_DTYPES = {
"U8": "uint8",
"BOOL": "bool",
}
+# ``cast_floats``: floating dtypes a bf16 load casts per tensor exactly as diffusers' loader does.
+_CAST_DTYPES = {"F32": "float32", "F16": "float16", "F64": "float64"}
def stream_build_enabled() -> bool:
@@ -552,9 +554,14 @@ def stream_build_enabled() -> bool:
return value not in ("0", "off", "false", "no")
-def _dense_shards(source: Any, component: str, cache_dir: Optional[str]) -> Optional[list[str]]:
+def _dense_shards(
+ source: Any,
+ component: str,
+ cache_dir: Optional[str],
+ weights_name: str = "diffusion_pytorch_model",
+) -> Optional[list[str]]:
"""The shard files ``from_pretrained(source, subfolder = component)`` reads (index first, as diffusers), or None."""
- stem = f"{component}/diffusion_pytorch_model"
+ stem = f"{component}/{weights_name}"
index = _cache._local_file(str(source), f"{stem}.safetensors.index.json", cache_dir)
if index is not None:
with open(index, "r", encoding = "utf-8") as fh:
@@ -566,18 +573,39 @@ def _dense_shards(source: Any, component: str, cache_dir: Optional[str]) -> Opti
return [str(single)] if single else None
-def _shard_entries(path: str) -> Optional[tuple[int, dict]]:
- """(data start, ``{key: header entry}``) of a shard holding only dtypes in ``_STREAM_DTYPES``, else None."""
+def _shard_entries(path: str, dtypes: Optional[dict] = None) -> Optional[tuple[int, dict]]:
+ """(data start, ``{key: header entry}``) of a shard holding only dtypes in ``dtypes`` (default
+ ``_STREAM_DTYPES``), else None."""
+ dtypes = _STREAM_DTYPES if dtypes is None else dtypes
raw = _cache.read_header_bytes(path)
if raw is None:
return None
header = json.loads(raw[8:].decode("utf-8"))
header.pop("__metadata__", None)
- if any(entry.get("dtype") not in _STREAM_DTYPES for entry in header.values()):
+ if any(entry.get("dtype") not in dtypes for entry in header.values()):
return None
return len(raw), header
+def _stored_dtype(tag: str) -> Any:
+ import torch
+ return getattr(torch, _STREAM_DTYPES.get(tag) or _CAST_DTYPES[tag])
+
+
+def _loaded_dtype(key: str, tag: str, dtype: Any, keep_in_fp32: Any) -> Any:
+ """The dtype ``from_pretrained(torch_dtype = dtype)`` gives the stored ``key``: diffusers'
+ ``load_model_dict_into_meta`` rule (a floating param under a ``_keep_in_fp32_modules`` name goes fp32, any other
+ floating param goes ``dtype``; integers stay)."""
+ import torch
+
+ stored = _stored_dtype(tag)
+ if not stored.is_floating_point:
+ return stored
+ if keep_in_fp32 and any(m in key.split(".") for m in keep_in_fp32):
+ return torch.float32
+ return dtype
+
+
def _map_tensor(fd: int, start: int, entry: dict) -> Any:
"""One tensor over its own private mapping, unmapped when freed (a whole-shard mapping stays resident while any
tensor survives)."""
@@ -586,7 +614,7 @@ def _map_tensor(fd: int, start: int, entry: dict) -> Any:
import torch
begin, end = (int(v) for v in entry["data_offsets"])
- dtype = getattr(torch, _STREAM_DTYPES[entry["dtype"]])
+ dtype = _stored_dtype(entry["dtype"])
shape = [int(d) for d in entry["shape"]]
count = 1
for d in shape:
@@ -616,9 +644,15 @@ def build_from_shards(
config_kwargs: Optional[dict] = None,
logger: Any = None,
cancel_check: Optional[Any] = None,
+ cast_floats: bool = False,
+ reraise_oom: bool = False,
) -> Any:
"""Build the native ``component`` per tensor, bit-identical to the in-place build, or None for the dense load
- (also on cancel, which the caller re-checks). Never raises."""
+ (also on cancel, which the caller re-checks). Never raises, except an out-of-memory under ``reraise_oom``.
+
+ ``cast_floats``: fp32 / fp16 shards and ``_keep_in_fp32_modules`` classes are built too, each tensor cast as the
+ dense bf16 load casts it (one Linear's bf16 copy at a time). Inside ``quantise_host_linears_on(device)`` each
+ Linear is quantised on ``device`` and returned to the host, as the in-place build there does."""
cancelled = [False]
def _check() -> None:
@@ -634,7 +668,12 @@ def build_from_shards(
try:
import torch
- from .diffusion_native_quant import NATIVE_INT8, native_linear_class
+ from .diffusion_native_quant import (
+ _HOST_QUANT_DEVICE,
+ NATIVE_INT8,
+ _HostLinearMover,
+ native_linear_class,
+ )
from .diffusion_transformer_quant import exclude_tokens_for_scheme, make_filter_fn
def skip(why: str) -> None:
@@ -644,7 +683,8 @@ def build_from_shards(
return None
if dtype is not torch.bfloat16:
return skip(f"loads at {str(dtype).replace('torch.', '')}, not bfloat16")
- if getattr(transformer_cls, "_keep_in_fp32_modules", None):
+ keep_in_fp32 = list(getattr(transformer_cls, "_keep_in_fp32_modules", None) or ())
+ if keep_in_fp32 and not cast_floats:
return skip("the class keeps fp32 modules")
scheme = spec["scheme"]
cache_dir = (config_kwargs or {}).get("cache_dir")
@@ -653,11 +693,16 @@ def build_from_shards(
return skip("dense shards not found locally")
if _cache._hub_source_stale(shards, logger):
return None
+ allowed = {**_STREAM_DTYPES, **_CAST_DTYPES} if cast_floats else _STREAM_DTYPES
entries = []
for shard in shards:
- found = _shard_entries(shard)
+ found = _shard_entries(shard, allowed)
if found is None:
- return skip("a shard holds a dtype other than bf16")
+ return skip(
+ "a shard holds a dtype other than bf16"
+ if not cast_floats
+ else "a shard holds a dtype the per-tensor cast does not cover"
+ )
entries.append((shard, *found))
stored = {key: entry for _shard, _start, header in entries for key, entry in header.items()}
_check()
@@ -682,7 +727,9 @@ def build_from_shards(
mod.weight = torch.nn.Parameter(
torch.empty(
mod.weight.shape,
- dtype = getattr(torch, _STREAM_DTYPES[entry["dtype"]]),
+ dtype = _loaded_dtype(f"{name}.weight", entry["dtype"], dtype, keep_in_fp32)
+ if cast_floats
+ else _stored_dtype(entry["dtype"]),
device = "meta",
),
requires_grad = mod.weight.requires_grad,
@@ -696,6 +743,12 @@ def build_from_shards(
act_int8 = scheme == NATIVE_INT8 and bool(spec["act_int8"])
rot_group = int(spec["rot_group"]) if act_int8 else 0
cls = native_linear_class()
+
+ def make(linear: Any) -> Any:
+ return cls(linear, scheme, act_int8 = act_int8, rot_group = rot_group)
+
+ mover_device = _HOST_QUANT_DEVICE.get()
+ mover = _HostLinearMover(mover_device) if mover_device is not None else None
state: dict = {}
built = 0
for shard, start, header in entries:
@@ -706,6 +759,9 @@ def build_from_shards(
):
_check()
tensor = _map_tensor(fd, start, entry)
+ if cast_floats:
+ # What load_model_dict_into_meta does: ``param.to(dtype)`` (a no-op at the stored dtype).
+ tensor = tensor.to(_loaded_dtype(key, entry["dtype"], dtype, keep_in_fp32))
fqn, _, leaf = key.rpartition(".")
if leaf != "weight" or fqn not in quant:
state[key] = tensor
@@ -722,7 +778,7 @@ def build_from_shards(
)
stand_in.weight = torch.nn.Parameter(tensor, requires_grad = False)
del tensor
- layer = cls(stand_in, scheme, act_int8 = act_int8, rot_group = rot_group)
+ layer = mover.build(stand_in, make) if mover is not None else make(stand_in)
del stand_in
setattr(parent, attr, layer)
state[f"{fqn}.weight_q"] = layer.weight_q
@@ -731,6 +787,8 @@ def build_from_shards(
finally:
# The mappings outlive the descriptor; each closes with its tensor.
os.close(fd)
+ if mover is not None:
+ mover.close(logger)
if built != len(quant):
return skip(f"{len(quant) - built} quantizable weights missing from the shards")
model.load_state_dict(state, strict = True, assign = True)
@@ -759,6 +817,8 @@ def build_from_shards(
except Exception as exc: # noqa: BLE001 - the caller loads the dense module whole instead
if cancelled[0]:
return None
+ if reraise_oom and _is_oom(exc):
+ raise
_cache._log(
logger,
"warning",
@@ -767,6 +827,19 @@ def build_from_shards(
return None
+def _is_oom(exc: BaseException) -> bool:
+ if isinstance(exc, MemoryError):
+ return True
+ try:
+ import torch
+ oom = getattr(torch, "OutOfMemoryError", None)
+ if oom is not None and isinstance(exc, oom):
+ return True
+ except Exception: # noqa: BLE001
+ pass
+ return "out of memory" in str(exc).lower()
+
+
def build_spec(transformer: Any, family: Optional[str]) -> Optional[dict]:
"""``native_spec`` of an in-memory native build, read from its layers, or None (no native layers, or a mix)."""
try:
diff --git a/studio/backend/core/inference/video.py b/studio/backend/core/inference/video.py
index 7f96621e7da..a44d3802893 100644
--- a/studio/backend/core/inference/video.py
+++ b/studio/backend/core/inference/video.py
@@ -3627,6 +3627,52 @@ def _read_native_quant_from_cache(
return {}
+def _video_native_stream_build(
+ fam: Any,
+ spec: Optional[dict],
+ *,
+ target: Any,
+ dtype: Any,
+ base: Optional[str],
+ source: Optional[str],
+ config_kwargs: dict,
+ skip: tuple = (),
+ cancel_check: Any = None,
+) -> dict[str, Any]:
+ """``{component: module}`` for EVERY denoiser built per shard into the native cache when the dense load would
+ not fit host RAM (``video_native_stream``), or ``{}``. Raises only a cancel or an out-of-memory."""
+ if spec is None:
+ return {}
+ try:
+ from utils.torch_warmup import close_dynamo_import_window
+ close_dynamo_import_window(logger) # a no-op once the load closed it above
+ except Exception as exc: # noqa: BLE001 - optimisation only
+ logger.debug("dynamo pre-import skipped: %r", exc)
+ try:
+ import diffusers
+
+ from .video_native_stream import build_into_cache
+ transformer_cls = getattr(diffusers, str(getattr(fam, "transformer_class", "")), None)
+ except Exception as exc: # noqa: BLE001 - the dense load still runs
+ logger.warning("video.native_stream: skipped (%s)", exc)
+ return {}
+ return build_into_cache(
+ fam,
+ spec,
+ transformer_cls,
+ target = target,
+ dtype = dtype,
+ base = base,
+ source = source,
+ config_kwargs = config_kwargs,
+ skip = skip,
+ # The device the runtime quant would build each host Linear on, so the codes match it bit for bit.
+ host_quant_device = _host_quant_device(target),
+ logger = logger,
+ cancel_check = cancel_check,
+ )
+
+
# The planner DECIDED against the seed (not "never ran"); a load must not re-take it after teardown.
DENOISER_SEED_DECLINED = "__declined__"
@@ -9069,22 +9115,39 @@ class VideoBackend:
if kind == "pipeline" and not denoiser_injected
else None
)
- native_cache_hits = _read_native_quant_from_cache(
- fam,
- native_cache_spec,
- base = base,
- source = _base_local_dir or fetch_base,
- config_kwargs = {}
+ native_config_kwargs: dict[str, Any] = (
+ {}
if _base_local_dir
else {
"cache_dir": hub_cache_dir(),
"local_files_only": local_files_only,
**({"token": hf_token} if hf_token else {}),
**_pinned_base_kwargs(),
- },
+ }
+ )
+ native_cache_hits = _read_native_quant_from_cache(
+ fam,
+ native_cache_spec,
+ base = base,
+ source = _base_local_dir or fetch_base,
+ config_kwargs = native_config_kwargs,
cancel_check = lambda: self._raise_if_video_load_superseded(_load_token),
)
self._raise_if_video_load_superseded(_load_token)
+ if not native_cache_hits and native_cache_spec is not None:
+ # A miss whose dense load would not fit host RAM: build the same native experts per shard into the cache.
+ native_cache_hits = _video_native_stream_build(
+ fam,
+ native_cache_spec,
+ target = target,
+ dtype = dtype,
+ base = base,
+ source = _base_local_dir or fetch_base,
+ config_kwargs = native_config_kwargs,
+ skip = tuple(pipe_kwargs),
+ cancel_check = lambda: self._raise_if_video_load_superseded(_load_token),
+ )
+ self._raise_if_video_load_superseded(_load_token)
pipe_kwargs.update(native_cache_hits)
# Injection is best-effort (a missing checkpoint falls back to the dense encoder), but the scoped pre-download
# already dropped those shards and from_pretrained cannot re-fetch from a local snapshot dir, so top them up
diff --git a/studio/backend/core/inference/video_native_stream.py b/studio/backend/core/inference/video_native_stream.py
new file mode 100644
index 00000000000..06875177eac
--- /dev/null
+++ b/studio/backend/core/inference/video_native_stream.py
@@ -0,0 +1,301 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
+
+"""Native int8 video denoisers built per shard into the local prequant cache when the dense load would not fit host
+RAM (Wan2.2-T2V-A14B: two fp32 14B experts, 57 GB at bf16, on a 53 GB host); the dense experts are never held in RAM.
+``UNSLOTH_VIDEO_NATIVE_STREAM_BUILD=0`` keeps the dense load, ``=1`` streams even where it fits.
+"""
+
+from __future__ import annotations
+
+import gc
+import json
+import os
+import time
+import types
+from typing import Any, Callable, Optional
+
+VIDEO_NATIVE_STREAM_ENV = "UNSLOTH_VIDEO_NATIVE_STREAM_BUILD"
+_MIB = 1024 * 1024
+_WEIGHT_NAMES = ("model", "diffusion_pytorch_model")
+
+
+def _env() -> str:
+ return (os.environ.get(VIDEO_NATIVE_STREAM_ENV) or "").strip().lower()
+
+
+def stream_disabled() -> bool:
+ return _env() in ("0", "off", "false", "no")
+
+
+def stream_forced() -> bool:
+ return _env() in ("1", "on", "true", "yes", "force")
+
+
+def decide(
+ dense_mib: Optional[int],
+ *,
+ available_mib: Optional[int],
+ reserve_mib: int,
+ forced: bool = False,
+ disabled: bool = False,
+) -> tuple[bool, str]:
+ """(engage, reason). Engages only when the dense pipeline load provably exceeds available host RAM (or forced);
+ an unreadable size or host keeps the dense load, as before."""
+ if disabled:
+ return False, f"{VIDEO_NATIVE_STREAM_ENV}=0"
+ if forced:
+ return True, f"{VIDEO_NATIVE_STREAM_ENV}=1"
+ if dense_mib is None:
+ return False, "dense denoiser size unreadable"
+ if available_mib is None:
+ return False, "host RAM unreadable"
+ need = int(dense_mib) + int(reserve_mib)
+ if need <= int(available_mib):
+ return False, (
+ f"dense load (~{int(dense_mib)} MiB + {int(reserve_mib)} MiB reserve) fits "
+ f"{int(available_mib)} MiB available host RAM"
+ )
+ return True, (
+ f"dense load needs ~{int(dense_mib)} MiB + {int(reserve_mib)} MiB reserve of host RAM, "
+ f"{int(available_mib)} MiB available"
+ )
+
+
+def _component_mib(
+ source: str,
+ component: str,
+ cache_dir: Optional[str],
+ *,
+ weights_names: tuple,
+ keep_in_fp32: Any = None,
+) -> Optional[int]:
+ """MiB ``component`` occupies once loaded at half precision (fp32 tensors halve, kept-fp32 names stay), 0 with
+ no weight file, None when a header is unreadable."""
+ from .diffusion_native_cache import _dense_shards
+ from .video_small_host import loaded_mib
+
+ for name in weights_names:
+ shards = _dense_shards(source, component, cache_dir, weights_name = name)
+ if shards:
+ return loaded_mib(
+ types.SimpleNamespace(shards = shards, parts = {"keep_in_fp32": keep_in_fp32})
+ )
+ return 0
+
+
+def dense_load_mib(
+ source: str,
+ denoisers: tuple,
+ *,
+ cache_dir: Optional[str],
+ transformer_cls: Any,
+ skip: tuple = (),
+) -> tuple[Optional[int], dict]:
+ """(total MiB, ``{component: MiB}``) the dense pipeline load holds in host RAM: every denoiser plus the weighted
+ companions ``model_index.json`` lists that are not already injected (``skip``). Denoiser size None = unknown."""
+ from . import diffusion_prequant_cache as _cache
+
+ keep = list(getattr(transformer_cls, "_keep_in_fp32_modules", None) or ())
+ sizes: dict = {}
+ for component in denoisers:
+ mib = _component_mib(
+ source,
+ component,
+ cache_dir,
+ weights_names = ("diffusion_pytorch_model",),
+ keep_in_fp32 = keep,
+ )
+ if not mib:
+ return None, sizes
+ sizes[component] = int(mib)
+ try:
+ index = _cache._local_file(str(source), "model_index.json", cache_dir)
+ names = []
+ if index is not None:
+ with open(index, "r", encoding = "utf-8") as fh:
+ raw = json.load(fh) or {}
+ names = [
+ k
+ for k, v in raw.items()
+ if not k.startswith("_")
+ and isinstance(v, (list, tuple))
+ and len(v) == 2
+ and all(v)
+ and k not in denoisers
+ and k not in skip
+ ]
+ for component in names:
+ mib = _component_mib(source, component, cache_dir, weights_names = _WEIGHT_NAMES)
+ if mib:
+ sizes[component] = int(mib)
+ except Exception: # noqa: BLE001 -- companions only sharpen the estimate; the denoisers decide alone
+ pass
+ return sum(sizes.values()), sizes
+
+
+def build_into_cache(
+ fam: Any,
+ spec: Optional[dict],
+ transformer_cls: Any,
+ *,
+ target: Any,
+ dtype: Any,
+ base: Optional[str],
+ source: Optional[str],
+ config_kwargs: Optional[dict] = None,
+ skip: tuple = (),
+ host_quant_device: Any = None,
+ logger: Any = None,
+ cancel_check: Optional[Callable[[], None]] = None,
+) -> dict:
+ """``{component: native module}`` for EVERY denoiser, built per shard and read back from the prequant cache, or
+ ``{}`` for the dense load. A cancel returns ``{}`` (the caller re-checks it); an out-of-memory raises."""
+ if spec is None or transformer_cls is None or not source:
+ return {}
+ if getattr(target, "device", None) != "cuda":
+ return {}
+ from . import diffusion_native_cache as NC
+ from .diffusion_native_quant import quantise_host_linears_on
+ from .diffusion_small_host import host_ram_mib, host_reserve_mib
+ from .video_denoiser_prequant import denoiser_components
+
+ if not NC.stream_build_enabled():
+ return {}
+ components = tuple(denoiser_components(fam))
+ source = str(source)
+ cache_dir = (config_kwargs or {}).get("cache_dir")
+ try:
+ dense_mib, sizes = dense_load_mib(
+ source, components, cache_dir = cache_dir, transformer_cls = transformer_cls, skip = skip
+ )
+ total, available = host_ram_mib()
+ engaged, why = decide(
+ dense_mib,
+ available_mib = available,
+ reserve_mib = host_reserve_mib(total),
+ forced = stream_forced(),
+ disabled = stream_disabled(),
+ )
+ except Exception as exc: # noqa: BLE001 -- unanswerable: the dense load, as before
+ engaged, why, sizes = False, f"sizing failed ({type(exc).__name__}: {exc})", {}
+ if logger is not None:
+ logger.info(
+ "video.native_stream: engaged=%s (%s); %s",
+ engaged,
+ why,
+ ", ".join(f"{k}={v} MiB" for k, v in sizes.items()) or "no sizes",
+ )
+ if not engaged:
+ return {}
+ t0 = time.perf_counter()
+ family = getattr(fam, "name", None)
+ if logger is not None:
+ logger.info(
+ "video.native_stream: building the native %s %s of %s per shard into the local prequant cache "
+ "(the dense denoisers are not loaded; %s=0 loads them dense)",
+ spec.get("scheme"),
+ "/".join(components),
+ family,
+ VIDEO_NATIVE_STREAM_ENV,
+ )
+
+ def build(component: str) -> Any:
+ """One expert per shard, or None (logged; a cancel raises)."""
+ with quantise_host_linears_on(host_quant_device):
+ module = NC.build_from_shards(
+ transformer_cls,
+ source,
+ spec,
+ family = family,
+ dtype = dtype,
+ component = component,
+ config_kwargs = config_kwargs,
+ logger = logger,
+ cancel_check = cancel_check,
+ cast_floats = True,
+ reraise_oom = True,
+ )
+ if module is None:
+ if cancel_check is not None:
+ cancel_check()
+ why = "could not be built per shard"
+ elif NC.build_spec(module, family) != spec:
+ module, why = None, "built a recipe other than the cache key's"
+ if module is None and logger is not None:
+ logger.warning(
+ "video.native_stream: %s %s; loading the dense denoisers", component, why
+ )
+ return module
+
+ kept: dict = {}
+ paths: dict = {}
+ for component in components:
+ if cancel_check is not None:
+ cancel_check()
+ hit = NC.lookup(
+ fam, spec, base_repo = base, source_base = source, component = component, logger = logger
+ )
+ if hit is not None:
+ paths[component] = hit
+ continue
+ module = build(component)
+ if module is None:
+ return {}
+ path = NC.save(
+ module,
+ fam,
+ spec,
+ base_repo = base,
+ source_base = source,
+ component = component,
+ logger = logger,
+ cancel_check = cancel_check,
+ )
+ if path is None:
+ # Unwritable cache: keep this expert in memory, still never dense.
+ kept[component] = module
+ else:
+ paths[component] = path
+ module = None
+ gc.collect()
+ out: dict = dict(kept)
+ for component in list(paths):
+ if cancel_check is not None:
+ cancel_check()
+ module = NC.load(
+ paths[component],
+ transformer_cls,
+ source,
+ component = component,
+ spec = spec,
+ config_kwargs = config_kwargs,
+ logger = logger,
+ cancel_check = cancel_check,
+ )
+ if module is None:
+ if cancel_check is not None:
+ cancel_check()
+ # Evicted since written (small cache budget): rebuild, kept in memory.
+ if logger is not None:
+ logger.warning(
+ "video.native_stream: the cached %s did not read back; rebuilding it per shard",
+ component,
+ )
+ module = build(component)
+ if module is None:
+ return {}
+ paths.pop(component)
+ kept[component] = module
+ out[component] = module
+ if logger is not None:
+ logger.info(
+ "video.native_stream: %d native %s denoiser(s) ready in %.1f s (%d from the local prequant cache, "
+ "%d kept in memory)",
+ len(out),
+ spec.get("scheme"),
+ time.perf_counter() - t0,
+ len(paths),
+ len(kept),
+ )
+ return {c: out[c] for c in components}
diff --git a/studio/backend/tests/test_video_native_stream_build.py b/studio/backend/tests/test_video_native_stream_build.py
new file mode 100644
index 00000000000..bc7d8403289
--- /dev/null
+++ b/studio/backend/tests/test_video_native_stream_build.py
@@ -0,0 +1,512 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
+
+"""A fresh offloaded two-expert Wan load whose dense experts would not fit host RAM builds the native int8 per shard
+into the prequant cache: same bytes as the in-place runtime build, never a dense expert in RAM."""
+
+from __future__ import annotations
+
+import gc
+import sys
+import types
+import weakref
+from pathlib import Path
+
+import pytest
+
+torch = pytest.importorskip("torch")
+diffusers = pytest.importorskip("diffusers")
+pytest.importorskip("safetensors")
+pytest.importorskip("accelerate")
+
+BACKEND = Path(__file__).resolve().parents[1]
+if str(BACKEND) not in sys.path:
+ sys.path.insert(0, str(BACKEND))
+
+from core.inference import diffusion_native_cache as NC # noqa: E402
+from core.inference import diffusion_prequant_cache as C # noqa: E402
+from core.inference import diffusion_small_host as SH # noqa: E402
+from core.inference import diffusion_transformer_quant as tq # noqa: E402
+from core.inference import video_native_stream as VS # noqa: E402
+from core.inference.diffusion_native_quant import is_native_linear, quantise_host_linears_on # noqa: E402
+from core.inference.video_families import detect_video_family # noqa: E402
+
+FAMILY = "wan2.2-t2v-a14b"
+BASE = "Wan-AI/Wan2.2-T2V-A14B-Diffusers"
+COMPONENTS = ("transformer", "transformer_2")
+CLS = diffusers.WanTransformer3DModel
+CUDA = types.SimpleNamespace(device = "cuda", dtype = torch.bfloat16)
+
+
+def _family():
+ return detect_video_family(BASE, override = FAMILY)
+
+
+def _tiny_config() -> dict:
+ # 2 x 256 = 512 wide: every attention / FFN Linear passes the 512 filter and the g256 ConvRot divides it.
+ return dict(
+ patch_size = (1, 2, 2),
+ num_attention_heads = 2,
+ attention_head_dim = 256,
+ in_channels = 4,
+ out_channels = 4,
+ text_dim = 64,
+ freq_dim = 32,
+ ffn_dim = 1024,
+ num_layers = 2,
+ cross_attn_norm = True,
+ qk_norm = "rms_norm_across_heads",
+ eps = 1e-6,
+ rope_max_seq_len = 64,
+ )
+
+
+def _write_base(root: Path, dtype, sharded: bool) -> str:
+ """A two-expert Wan base like the real A14B repo: fp32 experts on disk (``dtype``), plus a model_index.json."""
+ kwargs = {"max_shard_size": "2MB"} if sharded else {}
+ for seed, component in enumerate(COMPONENTS):
+ torch.manual_seed(seed)
+ model = CLS(**_tiny_config()).to(dtype)
+ model.save_pretrained(str(root / component), safe_serialization = True, **kwargs)
+ (root / "model_index.json").write_text(
+ '{"_class_name": "WanPipeline", "transformer": ["diffusers", "WanTransformer3DModel"], '
+ '"transformer_2": ["diffusers", "WanTransformer3DModel"], "scheduler": ["diffusers", "X"], '
+ '"tokenizer": ["transformers", "T"], "text_encoder": [null, null]}'
+ )
+ return str(root)
+
+
[email protected](params = ["fp32-single", "fp32-sharded", "bf16-single"])
+def base_dir(tmp_path, request):
+ dtype = torch.bfloat16 if request.param.startswith("bf16") else torch.float32
+ return _write_base(
+ tmp_path / "Wan2.2-T2V-A14B-Diffusers", dtype, request.param.endswith("sharded")
+ )
+
+
+def fp32_dir(tmp_path):
+ return _write_base(tmp_path / "Wan2.2-T2V-A14B-Diffusers", torch.float32, True)
+
+
[email protected](autouse = True)
+def cache_root(tmp_path, monkeypatch):
+ root = tmp_path / "cache"
+ monkeypatch.setenv(C.ENV_DIR, str(root))
+ for env in (C.ENV_MODE, C.ENV_ASYNC, NC.STREAM_BUILD_ENV, VS.VIDEO_NATIVE_STREAM_ENV):
+ monkeypatch.delenv(env, raising = False)
+ monkeypatch.setattr(C, "_hub_source_stale", lambda shards, logger = None: False)
+ return root
+
+
+def _spec(act_int8 = True):
+ return NC.native_spec("int8", FAMILY, act_int8 = act_int8)
+
+
+def _runtime_build(
+ base_dir,
+ component,
+ act_int8 = True,
+ device = None,
+):
+ """Today's offloaded load: dense bf16 from_pretrained, then the in-place native pass (host linears on ``device``)."""
+ model = CLS.from_pretrained(base_dir, subfolder = component, torch_dtype = torch.bfloat16)
+ view = types.SimpleNamespace(transformer = model)
+ with quantise_host_linears_on(device):
+ assert (
+ tq._quantize_native(
+ view,
+ "int8",
+ family = FAMILY,
+ min_features = tq.DEFAULT_MIN_LINEAR_FEATURES,
+ act_int8 = act_int8,
+ logger = None,
+ )
+ == "int8"
+ )
+ return model
+
+
+def _streamed(
+ base_dir,
+ component,
+ act_int8 = True,
+ device = None,
+ **kwargs,
+):
+ with quantise_host_linears_on(device):
+ return NC.build_from_shards(
+ CLS,
+ base_dir,
+ _spec(act_int8),
+ family = FAMILY,
+ dtype = kwargs.pop("dtype", torch.bfloat16),
+ component = component,
+ cast_floats = kwargs.pop("cast_floats", True),
+ **kwargs,
+ )
+
+
+def _assert_same_module(a, b):
+ sa, sb = a.state_dict(), b.state_dict()
+ assert sa.keys() == sb.keys()
+ for k in sa:
+ assert sa[k].dtype == sb[k].dtype and torch.equal(sa[k], sb[k]), k
+ ba, bb = (
+ dict(a.named_buffers()),
+ dict(b.named_buffers()),
+ ) # non-persistent too (rope tables, rot_h)
+ assert ba.keys() == bb.keys()
+ for k in ba:
+ assert ba[k].dtype == bb[k].dtype and torch.equal(ba[k], bb[k]), k
+ for (na, ma), (nb, mb) in zip(a.named_modules(), b.named_modules()):
+ assert na == nb and type(ma).__name__ == type(mb).__name__, na
+ assert ma.extra_repr() == mb.extra_repr(), na
+ pa, pb = dict(a.named_parameters()), dict(b.named_parameters())
+ assert pa.keys() == pb.keys()
+ for k in pa:
+ assert pa[k].dtype == pb[k].dtype and pa[k].requires_grad == pb[k].requires_grad, k
+
+
+def _forward(model):
+ g = torch.Generator().manual_seed(7)
+ hidden = torch.randn(1, 4, 3, 8, 8, generator = g).to(torch.bfloat16)
+ text = torch.randn(1, 6, 64, generator = g).to(torch.bfloat16)
+ with torch.no_grad():
+ return model(hidden, torch.tensor([500]), text, return_dict = False)[0]
+
+
+# -- decision ----------------------------------------------------------------------------------------------------
+
+
+ "dense, available, reserve, forced, disabled, engaged",
+ [
+ (
+ 58_600,
+ 51_000,
+ 5_300,
+ False,
+ False,
+ True,
+ ), # A14B experts alone on the 53 GB Colab L4 host
+ (69_000, 29_000, 3_100, False, False, True), # A14B pipeline on the 31 GB Kaggle T4 host
+ (
+ 20_000,
+ 51_000,
+ 5_300,
+ False,
+ False,
+ False,
+ ), # Wan2.2-TI2V-5B: the dense load fits, unchanged
+ (46_000, 51_000, 5_300, False, False, True), # fits only without the reserve: engage
+ (46_000, 51_000, 5_300, False, True, False), # kill switch
+ (20_000, 51_000, 5_300, True, False, True), # forced
+ (None, 51_000, 5_300, False, False, False), # unreadable size: the dense load, as before
+ (58_600, None, 5_300, False, False, False), # unreadable host: the dense load, as before
+ (58_600, 51_000, 5_300, True, True, False), # off wins over force
+ ],
+)
+def test_decision(dense, available, reserve, forced, disabled, engaged):
+ got, why = VS.decide(
+ dense, available_mib = available, reserve_mib = reserve, forced = forced, disabled = disabled
+ )
+ assert got is engaged and why
+
+
+def test_env_switches(monkeypatch):
+ for value, off, on in (
+ ("0", True, False),
+ ("off", True, False),
+ ("1", False, True),
+ ("", False, False),
+ ):
+ monkeypatch.setenv(VS.VIDEO_NATIVE_STREAM_ENV, value)
+ assert (VS.stream_disabled(), VS.stream_forced()) == (off, on)
+
+
+def test_dense_size_prices_loaded_dtype_and_keep_in_fp32(fp32_dir, tmp_path):
+ total, sizes = VS.dense_load_mib(fp32_dir, COMPONENTS, cache_dir = None, transformer_cls = CLS)
+ assert set(sizes) == set(COMPONENTS)
+ model = CLS.from_pretrained(fp32_dir, subfolder = "transformer", torch_dtype = torch.bfloat16)
+ loaded = sum(t.numel() * t.element_size() for t in model.state_dict().values())
+ assert sizes["transformer"] == -(-loaded // (1024 * 1024))
+ assert total == sum(sizes.values())
+ # a companion with weights counts, one already injected does not; a missing expert is "unknown"
+ te = Path(fp32_dir) / "text_encoder"
+ te.mkdir()
+ from safetensors.torch import save_file
+
+ save_file({"w": torch.zeros(1024, 1024)}, str(te / "model.safetensors"))
+ idx = Path(fp32_dir) / "model_index.json"
+ idx.write_text(
+ idx.read_text().replace(
+ '"text_encoder": [null, null]', '"text_encoder": ["transformers", "T"]'
+ )
+ )
+ with_te, sizes = VS.dense_load_mib(fp32_dir, COMPONENTS, cache_dir = None, transformer_cls = CLS)
+ assert sizes["text_encoder"] == 2 and with_te == total + 2
+ assert (
+ VS.dense_load_mib(
+ fp32_dir, COMPONENTS, cache_dir = None, transformer_cls = CLS, skip = ("text_encoder",)
+ )[0]
+ == total
+ )
+ assert (
+ VS.dense_load_mib(str(tmp_path / "nope"), COMPONENTS, cache_dir = None, transformer_cls = CLS)[
+ 0
+ ]
+ is None
+ )
+
+
+# -- bit identity ------------------------------------------------------------------------------------------------
+
+
[email protected]("act_int8", [True, False])
[email protected]("device", [None, "cpu"])
+def test_per_shard_build_is_bit_identical_to_the_runtime_build(base_dir, act_int8, device):
+ for component in COMPONENTS:
+ ref = _runtime_build(base_dir, component, act_int8, device)
+ got = _streamed(base_dir, component, act_int8, device)
+ assert got is not None
+ assert got._unsloth_runtime_quant_info == {"scheme": "int8", "native": True}
+ assert NC.build_spec(got, FAMILY) == _spec(act_int8)
+ _assert_same_module(ref, got)
+ # the deliberate fp32 linears (_keep_in_fp32_modules) stay dense fp32, as in the dense load
+ kept = [n for n, m in got.named_modules() if isinstance(m, torch.nn.Linear)]
+ assert kept and all(
+ dict(got.named_modules())[n].weight.dtype == dict(ref.named_modules())[n].weight.dtype
+ for n in kept
+ )
+ assert sum(is_native_linear(m) for m in got.modules()) > 0
+ assert torch.equal(_forward(ref), _forward(got))
+
+
+def test_cache_entries_are_byte_identical_to_the_runtime_entries(fp32_dir, tmp_path, monkeypatch):
+ fam = _family()
+ for component in COMPONENTS:
+ entries = []
+ for arm, build in (("ref", _runtime_build), ("streamed", _streamed)):
+ monkeypatch.setenv(C.ENV_DIR, str(tmp_path / f"cache_{arm}"))
+ out = NC.save(
+ build(fp32_dir, component),
+ fam,
+ _spec(),
+ base_repo = BASE,
+ source_base = fp32_dir,
+ component = component,
+ )
+ assert isinstance(out, str)
+ entries.append(Path(out))
+ assert entries[0].parent.name == entries[1].parent.name
+ assert entries[0].read_bytes() == entries[1].read_bytes()
+
+
+def test_default_mode_still_declines_fp32_and_keep_in_fp32(fp32_dir):
+ # The image path's call (no cast_floats) keeps its old contract.
+ assert _streamed(fp32_dir, "transformer", cast_floats = False) is None
+
+
+def test_mover_builds_every_quantised_linear(fp32_dir, monkeypatch):
+ from core.inference import diffusion_native_quant as NQ
+
+ seen = []
+ real = NQ._HostLinearMover.build
+
+ def spy(self, linear, make):
+ seen.append(str(self.device))
+ return real(self, linear, make)
+
+ monkeypatch.setattr(NQ._HostLinearMover, "build", spy)
+ got = _streamed(fp32_dir, "transformer", device = "cpu")
+ assert got is not None
+ assert len(seen) == sum(is_native_linear(m) for m in got.modules()) and set(seen) == {"cpu"}
+
+
+def test_oom_is_raised_not_swallowed(fp32_dir, monkeypatch):
+ from core.inference import diffusion_native_quant as NQ
+
+ real = NQ.native_linear_class()
+
+ class Boom(real):
+ def __init__(self, *a, **k):
+ raise torch.OutOfMemoryError("CUDA out of memory. Tried to allocate 1 GiB")
+
+ monkeypatch.setattr(NQ, "native_linear_class", lambda: Boom)
+ with pytest.raises(torch.OutOfMemoryError):
+ _streamed(fp32_dir, "transformer", reraise_oom = True)
+ # the image path's call keeps returning None
+ assert _streamed(fp32_dir, "transformer", cast_floats = False, reraise_oom = False) is None
+
+
+# -- the load-time route -----------------------------------------------------------------------------------------
+
+
+def _small_host(
+ monkeypatch,
+ available_mib = 1,
+ total_mib = 53 * 1024,
+):
+ monkeypatch.setattr(SH, "host_ram_mib", lambda: (total_mib, available_mib))
+
+
+def _route(base_dir, **kwargs):
+ from core.inference import video as V
+ return V._video_native_stream_build(
+ _family(),
+ kwargs.pop("spec", _spec()),
+ target = kwargs.pop("target", CUDA),
+ dtype = torch.bfloat16,
+ base = BASE,
+ source = base_dir,
+ config_kwargs = {},
+ **kwargs,
+ )
+
+
+def _refuse_dense(monkeypatch):
+ """The dense expert load must never run on this route."""
+
+ def refuse(*a, **k):
+ raise AssertionError("dense from_pretrained of a denoiser")
+
+ monkeypatch.setattr(CLS, "from_pretrained", refuse)
+
+
+def no_dense(monkeypatch):
+ _refuse_dense(monkeypatch)
+
+
+class _Log:
+ def __init__(self):
+ self.lines = []
+
+ def _add(self, msg, *args):
+ self.lines.append(str(msg) % args if args else str(msg))
+
+ info = warning = debug = _add
+
+
+def test_small_host_builds_every_expert_into_the_cache_and_the_returning_load_reads_it(
+ fp32_dir, monkeypatch
+):
+ from core.inference import video as V
+
+ refs = {c: _runtime_build(fp32_dir, c, act_int8 = True) for c in COMPONENTS}
+ _small_host(monkeypatch)
+ _refuse_dense(monkeypatch)
+ log = _Log()
+ monkeypatch.setattr(V, "logger", log)
+ got = _route(fp32_dir)
+ assert tuple(got) == COMPONENTS
+ for component, module in got.items():
+ assert module._unsloth_runtime_quant_info["cache_path"]
+ _assert_same_module(refs[component], module)
+ text = "\n".join(log.lines)
+ assert "video.native_stream: engaged=True" in text
+ assert "building the native int8 transformer/transformer_2" in text
+ assert (
+ "2 native int8 denoiser(s) ready" in text
+ and "(2 from the local prequant cache, 0 kept in memory)" in text
+ )
+ # The returning load: the existing read serves both experts.
+ hits = V._read_native_quant_from_cache(
+ _family(), _spec(), base = BASE, source = fp32_dir, config_kwargs = {}
+ )
+ assert set(hits) == set(COMPONENTS)
+
+
+def test_one_expert_at_a_time(fp32_dir, monkeypatch, no_dense):
+ _small_host(monkeypatch)
+ real = NC.build_from_shards
+ alive: list = []
+
+ def build(*a, **k):
+ gc.collect()
+ assert all(ref() is None for ref in alive), "the previous expert's build is still alive"
+ module = real(*a, **k)
+ alive.append(weakref.ref(module))
+ return module
+
+ monkeypatch.setattr(NC, "build_from_shards", build)
+ assert tuple(_route(fp32_dir)) == COMPONENTS
+ assert len(alive) == 2
+
+
+def test_fits_host_keeps_the_dense_load(fp32_dir, monkeypatch, cache_root):
+ _small_host(monkeypatch, available_mib = 10**7)
+ assert _route(fp32_dir) == {}
+ assert not cache_root.exists() or not any(cache_root.iterdir())
+
+
+def test_kill_switches(fp32_dir, monkeypatch, cache_root):
+ _small_host(monkeypatch)
+ for env in (VS.VIDEO_NATIVE_STREAM_ENV, NC.STREAM_BUILD_ENV):
+ monkeypatch.setenv(env, "0")
+ assert _route(fp32_dir) == {}
+ monkeypatch.delenv(env)
+ assert not cache_root.exists() or not any(cache_root.iterdir())
+
+
+def test_forced_streams_on_a_big_host(fp32_dir, monkeypatch, no_dense):
+ _small_host(monkeypatch, available_mib = 10**7)
+ monkeypatch.setenv(VS.VIDEO_NATIVE_STREAM_ENV, "1")
+ assert tuple(_route(fp32_dir)) == COMPONENTS
+
+
+def test_not_cuda_or_no_spec_is_a_no_op(fp32_dir, monkeypatch):
+ _small_host(monkeypatch)
+ assert _route(fp32_dir, target = types.SimpleNamespace(device = "mps", dtype = torch.bfloat16)) == {}
+ assert _route(fp32_dir, spec = None) == {}
+
+
+def test_unwritable_cache_keeps_the_built_experts(fp32_dir, monkeypatch, no_dense):
+ _small_host(monkeypatch)
+ monkeypatch.setenv(C.ENV_MODE, "0")
+ got = _route(fp32_dir)
+ assert tuple(got) == COMPONENTS
+ assert all("cache_path" not in m._unsloth_runtime_quant_info for m in got.values())
+
+
+def test_a_cached_expert_from_an_interrupted_build_is_reused(fp32_dir, monkeypatch, no_dense):
+ _small_host(monkeypatch)
+ first = _streamed(fp32_dir, "transformer")
+ assert NC.save(
+ first, _family(), _spec(), base_repo = BASE, source_base = fp32_dir, component = "transformer"
+ )
+ calls = []
+ real = NC.build_from_shards
+ monkeypatch.setattr(
+ NC, "build_from_shards", lambda *a, **k: calls.append(k["component"]) or real(*a, **k)
+ )
+ assert tuple(_route(fp32_dir)) == COMPONENTS
+ assert calls == ["transformer_2"]
+
+
+def test_a_failed_expert_build_keeps_the_dense_load(fp32_dir, monkeypatch):
+ _small_host(monkeypatch)
+ monkeypatch.setattr(NC, "build_from_shards", lambda *a, **k: None)
+ assert _route(fp32_dir) == {}
+
+
+def test_a_cancel_raises(fp32_dir, monkeypatch, no_dense):
+ _small_host(monkeypatch)
+ calls = []
+
+ def cancel():
+ calls.append(1)
+ if len(calls) > 5:
+ raise RuntimeError("Video load was cancelled or superseded.")
+
+ with pytest.raises(RuntimeError, match = "cancelled"):
+ _route(fp32_dir, cancel_check = cancel)
+
+
+def test_an_entry_that_does_not_read_back_is_rebuilt_in_memory(fp32_dir, monkeypatch, no_dense):
+ _small_host(monkeypatch)
+ monkeypatch.setattr(NC, "load", lambda *a, **k: None)
+ got = _route(fp32_dir)
+ assert tuple(got) == COMPONENTS
+ assert all("cache_path" not in m._unsloth_runtime_quant_info for m in got.values())