Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
Studio: build offloaded video int8 denoisers per shard when the dense load does not fit host RAM (#16290)
Daniel Han8af3535ff130
diff --git a/studio/backend/core/inference/diffusion_native_cache.py b/studio/backend/core/inference/diffusion_native_cache.pyindex 7c354efccfd..0d03a1e715e 100644--- a/studio/backend/core/inference/diffusion_native_cache.py+++ b/studio/backend/core/inference/diffusion_native_cache.py@@ -545,6 +545,8 @@ _STREAM_DTYPES = {"U8": "uint8","BOOL": "bool",}+# ``cast_floats``: floating dtypes a bf16 load casts per tensor exactly as diffusers' loader does.+_CAST_DTYPES = {"F32": "float32", "F16": "float16", "F64": "float64"}def stream_build_enabled() -> bool:@@ -552,9 +554,14 @@ def stream_build_enabled() -> bool:return value not in ("0", "off", "false", "no")-def _dense_shards(source: Any, component: str, cache_dir: Optional[str]) -> Optional[list[str]]:+def _dense_shards(+ source: Any,+ component: str,+ cache_dir: Optional[str],+ weights_name: str = "diffusion_pytorch_model",+) -> Optional[list[str]]:"""The shard files ``from_pretrained(source, subfolder = component)`` reads (index first, as diffusers), or None."""- stem = f"{component}/diffusion_pytorch_model"+ stem = f"{component}/{weights_name}"index = _cache._local_file(str(source), f"{stem}.safetensors.index.json", cache_dir)if index is not None:with open(index, "r", encoding = "utf-8") as fh:@@ -566,18 +573,39 @@ def _dense_shards(source: Any, component: str, cache_dir: Optional[str]) -> Optireturn [str(single)] if single else None-def _shard_entries(path: str) -> Optional[tuple[int, dict]]:- """(data start, ``{key: header entry}``) of a shard holding only dtypes in ``_STREAM_DTYPES``, else None."""+def _shard_entries(path: str, dtypes: Optional[dict] = None) -> Optional[tuple[int, dict]]:+ """(data start, ``{key: header entry}``) of a shard holding only dtypes in ``dtypes`` (default+ ``_STREAM_DTYPES``), else None."""+ dtypes = _STREAM_DTYPES if dtypes is None else dtypesraw = _cache.read_header_bytes(path)if raw is None:return Noneheader = json.loads(raw[8:].decode("utf-8"))header.pop("__metadata__", None)- if any(entry.get("dtype") not in _STREAM_DTYPES for entry in header.values()):+ if any(entry.get("dtype") not in dtypes for entry in header.values()):return Nonereturn len(raw), header+def _stored_dtype(tag: str) -> Any:+ import torch+ return getattr(torch, _STREAM_DTYPES.get(tag) or _CAST_DTYPES[tag])+++def _loaded_dtype(key: str, tag: str, dtype: Any, keep_in_fp32: Any) -> Any:+ """The dtype ``from_pretrained(torch_dtype = dtype)`` gives the stored ``key``: diffusers'+ ``load_model_dict_into_meta`` rule (a floating param under a ``_keep_in_fp32_modules`` name goes fp32, any other+ floating param goes ``dtype``; integers stay)."""+ import torch++ stored = _stored_dtype(tag)+ if not stored.is_floating_point:+ return stored+ if keep_in_fp32 and any(m in key.split(".") for m in keep_in_fp32):+ return torch.float32+ return dtype++def _map_tensor(fd: int, start: int, entry: dict) -> Any:"""One tensor over its own private mapping, unmapped when freed (a whole-shard mapping stays resident while anytensor survives)."""@@ -586,7 +614,7 @@ def _map_tensor(fd: int, start: int, entry: dict) -> Any:import torchbegin, end = (int(v) for v in entry["data_offsets"])- dtype = getattr(torch, _STREAM_DTYPES[entry["dtype"]])+ dtype = _stored_dtype(entry["dtype"])shape = [int(d) for d in entry["shape"]]count = 1for d in shape:@@ -616,9 +644,15 @@ def build_from_shards(config_kwargs: Optional[dict] = None,logger: Any = None,cancel_check: Optional[Any] = None,+ cast_floats: bool = False,+ reraise_oom: bool = False,) -> Any:"""Build the native ``component`` per tensor, bit-identical to the in-place build, or None for the dense load- (also on cancel, which the caller re-checks). Never raises."""+ (also on cancel, which the caller re-checks). Never raises, except an out-of-memory under ``reraise_oom``.++ ``cast_floats``: fp32 / fp16 shards and ``_keep_in_fp32_modules`` classes are built too, each tensor cast as the+ dense bf16 load casts it (one Linear's bf16 copy at a time). Inside ``quantise_host_linears_on(device)`` each+ Linear is quantised on ``device`` and returned to the host, as the in-place build there does."""cancelled = [False]def _check() -> None:@@ -634,7 +668,12 @@ def build_from_shards(try:import torch- from .diffusion_native_quant import NATIVE_INT8, native_linear_class+ from .diffusion_native_quant import (+ _HOST_QUANT_DEVICE,+ NATIVE_INT8,+ _HostLinearMover,+ native_linear_class,+ )from .diffusion_transformer_quant import exclude_tokens_for_scheme, make_filter_fndef skip(why: str) -> None:@@ -644,7 +683,8 @@ def build_from_shards(return Noneif dtype is not torch.bfloat16:return skip(f"loads at {str(dtype).replace('torch.', '')}, not bfloat16")- if getattr(transformer_cls, "_keep_in_fp32_modules", None):+ keep_in_fp32 = list(getattr(transformer_cls, "_keep_in_fp32_modules", None) or ())+ if keep_in_fp32 and not cast_floats:return skip("the class keeps fp32 modules")scheme = spec["scheme"]cache_dir = (config_kwargs or {}).get("cache_dir")@@ -653,11 +693,16 @@ def build_from_shards(return skip("dense shards not found locally")if _cache._hub_source_stale(shards, logger):return None+ allowed = {**_STREAM_DTYPES, **_CAST_DTYPES} if cast_floats else _STREAM_DTYPESentries = []for shard in shards:- found = _shard_entries(shard)+ found = _shard_entries(shard, allowed)if found is None:- return skip("a shard holds a dtype other than bf16")+ return skip(+ "a shard holds a dtype other than bf16"+ if not cast_floats+ else "a shard holds a dtype the per-tensor cast does not cover"+ )entries.append((shard, *found))stored = {key: entry for _shard, _start, header in entries for key, entry in header.items()}_check()@@ -682,7 +727,9 @@ def build_from_shards(mod.weight = torch.nn.Parameter(torch.empty(mod.weight.shape,- dtype = getattr(torch, _STREAM_DTYPES[entry["dtype"]]),+ dtype = _loaded_dtype(f"{name}.weight", entry["dtype"], dtype, keep_in_fp32)+ if cast_floats+ else _stored_dtype(entry["dtype"]),device = "meta",),requires_grad = mod.weight.requires_grad,@@ -696,6 +743,12 @@ def build_from_shards(act_int8 = scheme == NATIVE_INT8 and bool(spec["act_int8"])rot_group = int(spec["rot_group"]) if act_int8 else 0cls = native_linear_class()++ def make(linear: Any) -> Any:+ return cls(linear, scheme, act_int8 = act_int8, rot_group = rot_group)++ mover_device = _HOST_QUANT_DEVICE.get()+ mover = _HostLinearMover(mover_device) if mover_device is not None else Nonestate: dict = {}built = 0for shard, start, header in entries:@@ -706,6 +759,9 @@ def build_from_shards():_check()tensor = _map_tensor(fd, start, entry)+ if cast_floats:+ # What load_model_dict_into_meta does: ``param.to(dtype)`` (a no-op at the stored dtype).+ tensor = tensor.to(_loaded_dtype(key, entry["dtype"], dtype, keep_in_fp32))fqn, _, leaf = key.rpartition(".")if leaf != "weight" or fqn not in quant:state[key] = tensor@@ -722,7 +778,7 @@ def build_from_shards()stand_in.weight = torch.nn.Parameter(tensor, requires_grad = False)del tensor- layer = cls(stand_in, scheme, act_int8 = act_int8, rot_group = rot_group)+ layer = mover.build(stand_in, make) if mover is not None else make(stand_in)del stand_insetattr(parent, attr, layer)state[f"{fqn}.weight_q"] = layer.weight_q@@ -731,6 +787,8 @@ def build_from_shards(finally:# The mappings outlive the descriptor; each closes with its tensor.os.close(fd)+ if mover is not None:+ mover.close(logger)if built != len(quant):return skip(f"{len(quant) - built} quantizable weights missing from the shards")model.load_state_dict(state, strict = True, assign = True)@@ -759,6 +817,8 @@ def build_from_shards(except Exception as exc: # noqa: BLE001 - the caller loads the dense module whole insteadif cancelled[0]:return None+ if reraise_oom and _is_oom(exc):+ raise_cache._log(logger,"warning",@@ -767,6 +827,19 @@ def build_from_shards(return None+def _is_oom(exc: BaseException) -> bool:+ if isinstance(exc, MemoryError):+ return True+ try:+ import torch+ oom = getattr(torch, "OutOfMemoryError", None)+ if oom is not None and isinstance(exc, oom):+ return True+ except Exception: # noqa: BLE001+ pass+ return "out of memory" in str(exc).lower()++def build_spec(transformer: Any, family: Optional[str]) -> Optional[dict]:"""``native_spec`` of an in-memory native build, read from its layers, or None (no native layers, or a mix)."""try:diff --git a/studio/backend/core/inference/video.py b/studio/backend/core/inference/video.pyindex 7f96621e7da..a44d3802893 100644--- a/studio/backend/core/inference/video.py+++ b/studio/backend/core/inference/video.py@@ -3627,6 +3627,52 @@ def _read_native_quant_from_cache(return {}+def _video_native_stream_build(+ fam: Any,+ spec: Optional[dict],+ *,+ target: Any,+ dtype: Any,+ base: Optional[str],+ source: Optional[str],+ config_kwargs: dict,+ skip: tuple = (),+ cancel_check: Any = None,+) -> dict[str, Any]:+ """``{component: module}`` for EVERY denoiser built per shard into the native cache when the dense load would+ not fit host RAM (``video_native_stream``), or ``{}``. Raises only a cancel or an out-of-memory."""+ if spec is None:+ return {}+ try:+ from utils.torch_warmup import close_dynamo_import_window+ close_dynamo_import_window(logger) # a no-op once the load closed it above+ except Exception as exc: # noqa: BLE001 - optimisation only+ logger.debug("dynamo pre-import skipped: %r", exc)+ try:+ import diffusers++ from .video_native_stream import build_into_cache+ transformer_cls = getattr(diffusers, str(getattr(fam, "transformer_class", "")), None)+ except Exception as exc: # noqa: BLE001 - the dense load still runs+ logger.warning("video.native_stream: skipped (%s)", exc)+ return {}+ return build_into_cache(+ fam,+ spec,+ transformer_cls,+ target = target,+ dtype = dtype,+ base = base,+ source = source,+ config_kwargs = config_kwargs,+ skip = skip,+ # The device the runtime quant would build each host Linear on, so the codes match it bit for bit.+ host_quant_device = _host_quant_device(target),+ logger = logger,+ cancel_check = cancel_check,+ )++# The planner DECIDED against the seed (not "never ran"); a load must not re-take it after teardown.DENOISER_SEED_DECLINED = "__declined__"@@ -9069,22 +9115,39 @@ class VideoBackend:if kind == "pipeline" and not denoiser_injectedelse None)- native_cache_hits = _read_native_quant_from_cache(- fam,- native_cache_spec,- base = base,- source = _base_local_dir or fetch_base,- config_kwargs = {}+ native_config_kwargs: dict[str, Any] = (+ {}if _base_local_direlse {"cache_dir": hub_cache_dir(),"local_files_only": local_files_only,**({"token": hf_token} if hf_token else {}),**_pinned_base_kwargs(),- },+ }+ )+ native_cache_hits = _read_native_quant_from_cache(+ fam,+ native_cache_spec,+ base = base,+ source = _base_local_dir or fetch_base,+ config_kwargs = native_config_kwargs,cancel_check = lambda: self._raise_if_video_load_superseded(_load_token),)self._raise_if_video_load_superseded(_load_token)+ if not native_cache_hits and native_cache_spec is not None:+ # A miss whose dense load would not fit host RAM: build the same native experts per shard into the cache.+ native_cache_hits = _video_native_stream_build(+ fam,+ native_cache_spec,+ target = target,+ dtype = dtype,+ base = base,+ source = _base_local_dir or fetch_base,+ config_kwargs = native_config_kwargs,+ skip = tuple(pipe_kwargs),+ cancel_check = lambda: self._raise_if_video_load_superseded(_load_token),+ )+ self._raise_if_video_load_superseded(_load_token)pipe_kwargs.update(native_cache_hits)# Injection is best-effort (a missing checkpoint falls back to the dense encoder), but the scoped pre-download# already dropped those shards and from_pretrained cannot re-fetch from a local snapshot dir, so top them updiff --git a/studio/backend/core/inference/video_native_stream.py b/studio/backend/core/inference/video_native_stream.pynew file mode 100644index 00000000000..06875177eac--- /dev/null+++ b/studio/backend/core/inference/video_native_stream.py@@ -0,0 +1,301 @@+# SPDX-License-Identifier: AGPL-3.0-only+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0++"""Native int8 video denoisers built per shard into the local prequant cache when the dense load would not fit host+RAM (Wan2.2-T2V-A14B: two fp32 14B experts, 57 GB at bf16, on a 53 GB host); the dense experts are never held in RAM.+``UNSLOTH_VIDEO_NATIVE_STREAM_BUILD=0`` keeps the dense load, ``=1`` streams even where it fits.+"""++from __future__ import annotations++import gc+import json+import os+import time+import types+from typing import Any, Callable, Optional++VIDEO_NATIVE_STREAM_ENV = "UNSLOTH_VIDEO_NATIVE_STREAM_BUILD"+_MIB = 1024 * 1024+_WEIGHT_NAMES = ("model", "diffusion_pytorch_model")+++def _env() -> str:+ return (os.environ.get(VIDEO_NATIVE_STREAM_ENV) or "").strip().lower()+++def stream_disabled() -> bool:+ return _env() in ("0", "off", "false", "no")+++def stream_forced() -> bool:+ return _env() in ("1", "on", "true", "yes", "force")+++def decide(+ dense_mib: Optional[int],+ *,+ available_mib: Optional[int],+ reserve_mib: int,+ forced: bool = False,+ disabled: bool = False,+) -> tuple[bool, str]:+ """(engage, reason). Engages only when the dense pipeline load provably exceeds available host RAM (or forced);+ an unreadable size or host keeps the dense load, as before."""+ if disabled:+ return False, f"{VIDEO_NATIVE_STREAM_ENV}=0"+ if forced:+ return True, f"{VIDEO_NATIVE_STREAM_ENV}=1"+ if dense_mib is None:+ return False, "dense denoiser size unreadable"+ if available_mib is None:+ return False, "host RAM unreadable"+ need = int(dense_mib) + int(reserve_mib)+ if need <= int(available_mib):+ return False, (+ f"dense load (~{int(dense_mib)} MiB + {int(reserve_mib)} MiB reserve) fits "+ f"{int(available_mib)} MiB available host RAM"+ )+ return True, (+ f"dense load needs ~{int(dense_mib)} MiB + {int(reserve_mib)} MiB reserve of host RAM, "+ f"{int(available_mib)} MiB available"+ )+++def _component_mib(+ source: str,+ component: str,+ cache_dir: Optional[str],+ *,+ weights_names: tuple,+ keep_in_fp32: Any = None,+) -> Optional[int]:+ """MiB ``component`` occupies once loaded at half precision (fp32 tensors halve, kept-fp32 names stay), 0 with+ no weight file, None when a header is unreadable."""+ from .diffusion_native_cache import _dense_shards+ from .video_small_host import loaded_mib++ for name in weights_names:+ shards = _dense_shards(source, component, cache_dir, weights_name = name)+ if shards:+ return loaded_mib(+ types.SimpleNamespace(shards = shards, parts = {"keep_in_fp32": keep_in_fp32})+ )+ return 0+++def dense_load_mib(+ source: str,+ denoisers: tuple,+ *,+ cache_dir: Optional[str],+ transformer_cls: Any,+ skip: tuple = (),+) -> tuple[Optional[int], dict]:+ """(total MiB, ``{component: MiB}``) the dense pipeline load holds in host RAM: every denoiser plus the weighted+ companions ``model_index.json`` lists that are not already injected (``skip``). Denoiser size None = unknown."""+ from . import diffusion_prequant_cache as _cache++ keep = list(getattr(transformer_cls, "_keep_in_fp32_modules", None) or ())+ sizes: dict = {}+ for component in denoisers:+ mib = _component_mib(+ source,+ component,+ cache_dir,+ weights_names = ("diffusion_pytorch_model",),+ keep_in_fp32 = keep,+ )+ if not mib:+ return None, sizes+ sizes[component] = int(mib)+ try:+ index = _cache._local_file(str(source), "model_index.json", cache_dir)+ names = []+ if index is not None:+ with open(index, "r", encoding = "utf-8") as fh:+ raw = json.load(fh) or {}+ names = [+ k+ for k, v in raw.items()+ if not k.startswith("_")+ and isinstance(v, (list, tuple))+ and len(v) == 2+ and all(v)+ and k not in denoisers+ and k not in skip+ ]+ for component in names:+ mib = _component_mib(source, component, cache_dir, weights_names = _WEIGHT_NAMES)+ if mib:+ sizes[component] = int(mib)+ except Exception: # noqa: BLE001 -- companions only sharpen the estimate; the denoisers decide alone+ pass+ return sum(sizes.values()), sizes+++def build_into_cache(+ fam: Any,+ spec: Optional[dict],+ transformer_cls: Any,+ *,+ target: Any,+ dtype: Any,+ base: Optional[str],+ source: Optional[str],+ config_kwargs: Optional[dict] = None,+ skip: tuple = (),+ host_quant_device: Any = None,+ logger: Any = None,+ cancel_check: Optional[Callable[[], None]] = None,+) -> dict:+ """``{component: native module}`` for EVERY denoiser, built per shard and read back from the prequant cache, or+ ``{}`` for the dense load. A cancel returns ``{}`` (the caller re-checks it); an out-of-memory raises."""+ if spec is None or transformer_cls is None or not source:+ return {}+ if getattr(target, "device", None) != "cuda":+ return {}+ from . import diffusion_native_cache as NC+ from .diffusion_native_quant import quantise_host_linears_on+ from .diffusion_small_host import host_ram_mib, host_reserve_mib+ from .video_denoiser_prequant import denoiser_components++ if not NC.stream_build_enabled():+ return {}+ components = tuple(denoiser_components(fam))+ source = str(source)+ cache_dir = (config_kwargs or {}).get("cache_dir")+ try:+ dense_mib, sizes = dense_load_mib(+ source, components, cache_dir = cache_dir, transformer_cls = transformer_cls, skip = skip+ )+ total, available = host_ram_mib()+ engaged, why = decide(+ dense_mib,+ available_mib = available,+ reserve_mib = host_reserve_mib(total),+ forced = stream_forced(),+ disabled = stream_disabled(),+ )+ except Exception as exc: # noqa: BLE001 -- unanswerable: the dense load, as before+ engaged, why, sizes = False, f"sizing failed ({type(exc).__name__}: {exc})", {}+ if logger is not None:+ logger.info(+ "video.native_stream: engaged=%s (%s); %s",+ engaged,+ why,+ ", ".join(f"{k}={v} MiB" for k, v in sizes.items()) or "no sizes",+ )+ if not engaged:+ return {}+ t0 = time.perf_counter()+ family = getattr(fam, "name", None)+ if logger is not None:+ logger.info(+ "video.native_stream: building the native %s %s of %s per shard into the local prequant cache "+ "(the dense denoisers are not loaded; %s=0 loads them dense)",+ spec.get("scheme"),+ "/".join(components),+ family,+ VIDEO_NATIVE_STREAM_ENV,+ )++ def build(component: str) -> Any:+ """One expert per shard, or None (logged; a cancel raises)."""+ with quantise_host_linears_on(host_quant_device):+ module = NC.build_from_shards(+ transformer_cls,+ source,+ spec,+ family = family,+ dtype = dtype,+ component = component,+ config_kwargs = config_kwargs,+ logger = logger,+ cancel_check = cancel_check,+ cast_floats = True,+ reraise_oom = True,+ )+ if module is None:+ if cancel_check is not None:+ cancel_check()+ why = "could not be built per shard"+ elif NC.build_spec(module, family) != spec:+ module, why = None, "built a recipe other than the cache key's"+ if module is None and logger is not None:+ logger.warning(+ "video.native_stream: %s %s; loading the dense denoisers", component, why+ )+ return module++ kept: dict = {}+ paths: dict = {}+ for component in components:+ if cancel_check is not None:+ cancel_check()+ hit = NC.lookup(+ fam, spec, base_repo = base, source_base = source, component = component, logger = logger+ )+ if hit is not None:+ paths[component] = hit+ continue+ module = build(component)+ if module is None:+ return {}+ path = NC.save(+ module,+ fam,+ spec,+ base_repo = base,+ source_base = source,+ component = component,+ logger = logger,+ cancel_check = cancel_check,+ )+ if path is None:+ # Unwritable cache: keep this expert in memory, still never dense.+ kept[component] = module+ else:+ paths[component] = path+ module = None+ gc.collect()+ out: dict = dict(kept)+ for component in list(paths):+ if cancel_check is not None:+ cancel_check()+ module = NC.load(+ paths[component],+ transformer_cls,+ source,+ component = component,+ spec = spec,+ config_kwargs = config_kwargs,+ logger = logger,+ cancel_check = cancel_check,+ )+ if module is None:+ if cancel_check is not None:+ cancel_check()+ # Evicted since written (small cache budget): rebuild, kept in memory.+ if logger is not None:+ logger.warning(+ "video.native_stream: the cached %s did not read back; rebuilding it per shard",+ component,+ )+ module = build(component)+ if module is None:+ return {}+ paths.pop(component)+ kept[component] = module+ out[component] = module+ if logger is not None:+ logger.info(+ "video.native_stream: %d native %s denoiser(s) ready in %.1f s (%d from the local prequant cache, "+ "%d kept in memory)",+ len(out),+ spec.get("scheme"),+ time.perf_counter() - t0,+ len(paths),+ len(kept),+ )+ return {c: out[c] for c in components}diff --git a/studio/backend/tests/test_video_native_stream_build.py b/studio/backend/tests/test_video_native_stream_build.pynew file mode 100644index 00000000000..bc7d8403289--- /dev/null+++ b/studio/backend/tests/test_video_native_stream_build.py@@ -0,0 +1,512 @@+# SPDX-License-Identifier: AGPL-3.0-only+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0++"""A fresh offloaded two-expert Wan load whose dense experts would not fit host RAM builds the native int8 per shard+into the prequant cache: same bytes as the in-place runtime build, never a dense expert in RAM."""++from __future__ import annotations++import gc+import sys+import types+import weakref+from pathlib import Path++import pytest++torch = pytest.importorskip("torch")+diffusers = pytest.importorskip("diffusers")+pytest.importorskip("safetensors")+pytest.importorskip("accelerate")++BACKEND = Path(__file__).resolve().parents[1]+if str(BACKEND) not in sys.path:+ sys.path.insert(0, str(BACKEND))++from core.inference import diffusion_native_cache as NC # noqa: E402+from core.inference import diffusion_prequant_cache as C # noqa: E402+from core.inference import diffusion_small_host as SH # noqa: E402+from core.inference import diffusion_transformer_quant as tq # noqa: E402+from core.inference import video_native_stream as VS # noqa: E402+from core.inference.diffusion_native_quant import is_native_linear, quantise_host_linears_on # noqa: E402+from core.inference.video_families import detect_video_family # noqa: E402++FAMILY = "wan2.2-t2v-a14b"+BASE = "Wan-AI/Wan2.2-T2V-A14B-Diffusers"+COMPONENTS = ("transformer", "transformer_2")+CLS = diffusers.WanTransformer3DModel+CUDA = types.SimpleNamespace(device = "cuda", dtype = torch.bfloat16)+++def _family():+ return detect_video_family(BASE, override = FAMILY)+++def _tiny_config() -> dict:+ # 2 x 256 = 512 wide: every attention / FFN Linear passes the 512 filter and the g256 ConvRot divides it.+ return dict(+ patch_size = (1, 2, 2),+ num_attention_heads = 2,+ attention_head_dim = 256,+ in_channels = 4,+ out_channels = 4,+ text_dim = 64,+ freq_dim = 32,+ ffn_dim = 1024,+ num_layers = 2,+ cross_attn_norm = True,+ qk_norm = "rms_norm_across_heads",+ eps = 1e-6,+ rope_max_seq_len = 64,+ )+++def _write_base(root: Path, dtype, sharded: bool) -> str:+ """A two-expert Wan base like the real A14B repo: fp32 experts on disk (``dtype``), plus a model_index.json."""+ kwargs = {"max_shard_size": "2MB"} if sharded else {}+ for seed, component in enumerate(COMPONENTS):+ torch.manual_seed(seed)+ model = CLS(**_tiny_config()).to(dtype)+ model.save_pretrained(str(root / component), safe_serialization = True, **kwargs)+ (root / "model_index.json").write_text(+ '{"_class_name": "WanPipeline", "transformer": ["diffusers", "WanTransformer3DModel"], '+ '"transformer_2": ["diffusers", "WanTransformer3DModel"], "scheduler": ["diffusers", "X"], '+ '"tokenizer": ["transformers", "T"], "text_encoder": [null, null]}'+ )+ return str(root)++[email protected](params = ["fp32-single", "fp32-sharded", "bf16-single"])+def base_dir(tmp_path, request):+ dtype = torch.bfloat16 if request.param.startswith("bf16") else torch.float32+ return _write_base(+ tmp_path / "Wan2.2-T2V-A14B-Diffusers", dtype, request.param.endswith("sharded")+ )+++def fp32_dir(tmp_path):+ return _write_base(tmp_path / "Wan2.2-T2V-A14B-Diffusers", torch.float32, True)++[email protected](autouse = True)+def cache_root(tmp_path, monkeypatch):+ root = tmp_path / "cache"+ monkeypatch.setenv(C.ENV_DIR, str(root))+ for env in (C.ENV_MODE, C.ENV_ASYNC, NC.STREAM_BUILD_ENV, VS.VIDEO_NATIVE_STREAM_ENV):+ monkeypatch.delenv(env, raising = False)+ monkeypatch.setattr(C, "_hub_source_stale", lambda shards, logger = None: False)+ return root+++def _spec(act_int8 = True):+ return NC.native_spec("int8", FAMILY, act_int8 = act_int8)+++def _runtime_build(+ base_dir,+ component,+ act_int8 = True,+ device = None,+):+ """Today's offloaded load: dense bf16 from_pretrained, then the in-place native pass (host linears on ``device``)."""+ model = CLS.from_pretrained(base_dir, subfolder = component, torch_dtype = torch.bfloat16)+ view = types.SimpleNamespace(transformer = model)+ with quantise_host_linears_on(device):+ assert (+ tq._quantize_native(+ view,+ "int8",+ family = FAMILY,+ min_features = tq.DEFAULT_MIN_LINEAR_FEATURES,+ act_int8 = act_int8,+ logger = None,+ )+ == "int8"+ )+ return model+++def _streamed(+ base_dir,+ component,+ act_int8 = True,+ device = None,+ **kwargs,+):+ with quantise_host_linears_on(device):+ return NC.build_from_shards(+ CLS,+ base_dir,+ _spec(act_int8),+ family = FAMILY,+ dtype = kwargs.pop("dtype", torch.bfloat16),+ component = component,+ cast_floats = kwargs.pop("cast_floats", True),+ **kwargs,+ )+++def _assert_same_module(a, b):+ sa, sb = a.state_dict(), b.state_dict()+ assert sa.keys() == sb.keys()+ for k in sa:+ assert sa[k].dtype == sb[k].dtype and torch.equal(sa[k], sb[k]), k+ ba, bb = (+ dict(a.named_buffers()),+ dict(b.named_buffers()),+ ) # non-persistent too (rope tables, rot_h)+ assert ba.keys() == bb.keys()+ for k in ba:+ assert ba[k].dtype == bb[k].dtype and torch.equal(ba[k], bb[k]), k+ for (na, ma), (nb, mb) in zip(a.named_modules(), b.named_modules()):+ assert na == nb and type(ma).__name__ == type(mb).__name__, na+ assert ma.extra_repr() == mb.extra_repr(), na+ pa, pb = dict(a.named_parameters()), dict(b.named_parameters())+ assert pa.keys() == pb.keys()+ for k in pa:+ assert pa[k].dtype == pb[k].dtype and pa[k].requires_grad == pb[k].requires_grad, k+++def _forward(model):+ g = torch.Generator().manual_seed(7)+ hidden = torch.randn(1, 4, 3, 8, 8, generator = g).to(torch.bfloat16)+ text = torch.randn(1, 6, 64, generator = g).to(torch.bfloat16)+ with torch.no_grad():+ return model(hidden, torch.tensor([500]), text, return_dict = False)[0]+++# -- decision ----------------------------------------------------------------------------------------------------+++ "dense, available, reserve, forced, disabled, engaged",+ [+ (+ 58_600,+ 51_000,+ 5_300,+ False,+ False,+ True,+ ), # A14B experts alone on the 53 GB Colab L4 host+ (69_000, 29_000, 3_100, False, False, True), # A14B pipeline on the 31 GB Kaggle T4 host+ (+ 20_000,+ 51_000,+ 5_300,+ False,+ False,+ False,+ ), # Wan2.2-TI2V-5B: the dense load fits, unchanged+ (46_000, 51_000, 5_300, False, False, True), # fits only without the reserve: engage+ (46_000, 51_000, 5_300, False, True, False), # kill switch+ (20_000, 51_000, 5_300, True, False, True), # forced+ (None, 51_000, 5_300, False, False, False), # unreadable size: the dense load, as before+ (58_600, None, 5_300, False, False, False), # unreadable host: the dense load, as before+ (58_600, 51_000, 5_300, True, True, False), # off wins over force+ ],+)+def test_decision(dense, available, reserve, forced, disabled, engaged):+ got, why = VS.decide(+ dense, available_mib = available, reserve_mib = reserve, forced = forced, disabled = disabled+ )+ assert got is engaged and why+++def test_env_switches(monkeypatch):+ for value, off, on in (+ ("0", True, False),+ ("off", True, False),+ ("1", False, True),+ ("", False, False),+ ):+ monkeypatch.setenv(VS.VIDEO_NATIVE_STREAM_ENV, value)+ assert (VS.stream_disabled(), VS.stream_forced()) == (off, on)+++def test_dense_size_prices_loaded_dtype_and_keep_in_fp32(fp32_dir, tmp_path):+ total, sizes = VS.dense_load_mib(fp32_dir, COMPONENTS, cache_dir = None, transformer_cls = CLS)+ assert set(sizes) == set(COMPONENTS)+ model = CLS.from_pretrained(fp32_dir, subfolder = "transformer", torch_dtype = torch.bfloat16)+ loaded = sum(t.numel() * t.element_size() for t in model.state_dict().values())+ assert sizes["transformer"] == -(-loaded // (1024 * 1024))+ assert total == sum(sizes.values())+ # a companion with weights counts, one already injected does not; a missing expert is "unknown"+ te = Path(fp32_dir) / "text_encoder"+ te.mkdir()+ from safetensors.torch import save_file++ save_file({"w": torch.zeros(1024, 1024)}, str(te / "model.safetensors"))+ idx = Path(fp32_dir) / "model_index.json"+ idx.write_text(+ idx.read_text().replace(+ '"text_encoder": [null, null]', '"text_encoder": ["transformers", "T"]'+ )+ )+ with_te, sizes = VS.dense_load_mib(fp32_dir, COMPONENTS, cache_dir = None, transformer_cls = CLS)+ assert sizes["text_encoder"] == 2 and with_te == total + 2+ assert (+ VS.dense_load_mib(+ fp32_dir, COMPONENTS, cache_dir = None, transformer_cls = CLS, skip = ("text_encoder",)+ )[0]+ == total+ )+ assert (+ VS.dense_load_mib(str(tmp_path / "nope"), COMPONENTS, cache_dir = None, transformer_cls = CLS)[+ 0+ ]+ is None+ )+++# -- bit identity ------------------------------------------------------------------------------------------------++[email protected]("act_int8", [True, False])[email protected]("device", [None, "cpu"])+def test_per_shard_build_is_bit_identical_to_the_runtime_build(base_dir, act_int8, device):+ for component in COMPONENTS:+ ref = _runtime_build(base_dir, component, act_int8, device)+ got = _streamed(base_dir, component, act_int8, device)+ assert got is not None+ assert got._unsloth_runtime_quant_info == {"scheme": "int8", "native": True}+ assert NC.build_spec(got, FAMILY) == _spec(act_int8)+ _assert_same_module(ref, got)+ # the deliberate fp32 linears (_keep_in_fp32_modules) stay dense fp32, as in the dense load+ kept = [n for n, m in got.named_modules() if isinstance(m, torch.nn.Linear)]+ assert kept and all(+ dict(got.named_modules())[n].weight.dtype == dict(ref.named_modules())[n].weight.dtype+ for n in kept+ )+ assert sum(is_native_linear(m) for m in got.modules()) > 0+ assert torch.equal(_forward(ref), _forward(got))+++def test_cache_entries_are_byte_identical_to_the_runtime_entries(fp32_dir, tmp_path, monkeypatch):+ fam = _family()+ for component in COMPONENTS:+ entries = []+ for arm, build in (("ref", _runtime_build), ("streamed", _streamed)):+ monkeypatch.setenv(C.ENV_DIR, str(tmp_path / f"cache_{arm}"))+ out = NC.save(+ build(fp32_dir, component),+ fam,+ _spec(),+ base_repo = BASE,+ source_base = fp32_dir,+ component = component,+ )+ assert isinstance(out, str)+ entries.append(Path(out))+ assert entries[0].parent.name == entries[1].parent.name+ assert entries[0].read_bytes() == entries[1].read_bytes()+++def test_default_mode_still_declines_fp32_and_keep_in_fp32(fp32_dir):+ # The image path's call (no cast_floats) keeps its old contract.+ assert _streamed(fp32_dir, "transformer", cast_floats = False) is None+++def test_mover_builds_every_quantised_linear(fp32_dir, monkeypatch):+ from core.inference import diffusion_native_quant as NQ++ seen = []+ real = NQ._HostLinearMover.build++ def spy(self, linear, make):+ seen.append(str(self.device))+ return real(self, linear, make)++ monkeypatch.setattr(NQ._HostLinearMover, "build", spy)+ got = _streamed(fp32_dir, "transformer", device = "cpu")+ assert got is not None+ assert len(seen) == sum(is_native_linear(m) for m in got.modules()) and set(seen) == {"cpu"}+++def test_oom_is_raised_not_swallowed(fp32_dir, monkeypatch):+ from core.inference import diffusion_native_quant as NQ++ real = NQ.native_linear_class()++ class Boom(real):+ def __init__(self, *a, **k):+ raise torch.OutOfMemoryError("CUDA out of memory. Tried to allocate 1 GiB")++ monkeypatch.setattr(NQ, "native_linear_class", lambda: Boom)+ with pytest.raises(torch.OutOfMemoryError):+ _streamed(fp32_dir, "transformer", reraise_oom = True)+ # the image path's call keeps returning None+ assert _streamed(fp32_dir, "transformer", cast_floats = False, reraise_oom = False) is None+++# -- the load-time route -----------------------------------------------------------------------------------------+++def _small_host(+ monkeypatch,+ available_mib = 1,+ total_mib = 53 * 1024,+):+ monkeypatch.setattr(SH, "host_ram_mib", lambda: (total_mib, available_mib))+++def _route(base_dir, **kwargs):+ from core.inference import video as V+ return V._video_native_stream_build(+ _family(),+ kwargs.pop("spec", _spec()),+ target = kwargs.pop("target", CUDA),+ dtype = torch.bfloat16,+ base = BASE,+ source = base_dir,+ config_kwargs = {},+ **kwargs,+ )+++def _refuse_dense(monkeypatch):+ """The dense expert load must never run on this route."""++ def refuse(*a, **k):+ raise AssertionError("dense from_pretrained of a denoiser")++ monkeypatch.setattr(CLS, "from_pretrained", refuse)+++def no_dense(monkeypatch):+ _refuse_dense(monkeypatch)+++class _Log:+ def __init__(self):+ self.lines = []++ def _add(self, msg, *args):+ self.lines.append(str(msg) % args if args else str(msg))++ info = warning = debug = _add+++def test_small_host_builds_every_expert_into_the_cache_and_the_returning_load_reads_it(+ fp32_dir, monkeypatch+):+ from core.inference import video as V++ refs = {c: _runtime_build(fp32_dir, c, act_int8 = True) for c in COMPONENTS}+ _small_host(monkeypatch)+ _refuse_dense(monkeypatch)+ log = _Log()+ monkeypatch.setattr(V, "logger", log)+ got = _route(fp32_dir)+ assert tuple(got) == COMPONENTS+ for component, module in got.items():+ assert module._unsloth_runtime_quant_info["cache_path"]+ _assert_same_module(refs[component], module)+ text = "\n".join(log.lines)+ assert "video.native_stream: engaged=True" in text+ assert "building the native int8 transformer/transformer_2" in text+ assert (+ "2 native int8 denoiser(s) ready" in text+ and "(2 from the local prequant cache, 0 kept in memory)" in text+ )+ # The returning load: the existing read serves both experts.+ hits = V._read_native_quant_from_cache(+ _family(), _spec(), base = BASE, source = fp32_dir, config_kwargs = {}+ )+ assert set(hits) == set(COMPONENTS)+++def test_one_expert_at_a_time(fp32_dir, monkeypatch, no_dense):+ _small_host(monkeypatch)+ real = NC.build_from_shards+ alive: list = []++ def build(*a, **k):+ gc.collect()+ assert all(ref() is None for ref in alive), "the previous expert's build is still alive"+ module = real(*a, **k)+ alive.append(weakref.ref(module))+ return module++ monkeypatch.setattr(NC, "build_from_shards", build)+ assert tuple(_route(fp32_dir)) == COMPONENTS+ assert len(alive) == 2+++def test_fits_host_keeps_the_dense_load(fp32_dir, monkeypatch, cache_root):+ _small_host(monkeypatch, available_mib = 10**7)+ assert _route(fp32_dir) == {}+ assert not cache_root.exists() or not any(cache_root.iterdir())+++def test_kill_switches(fp32_dir, monkeypatch, cache_root):+ _small_host(monkeypatch)+ for env in (VS.VIDEO_NATIVE_STREAM_ENV, NC.STREAM_BUILD_ENV):+ monkeypatch.setenv(env, "0")+ assert _route(fp32_dir) == {}+ monkeypatch.delenv(env)+ assert not cache_root.exists() or not any(cache_root.iterdir())+++def test_forced_streams_on_a_big_host(fp32_dir, monkeypatch, no_dense):+ _small_host(monkeypatch, available_mib = 10**7)+ monkeypatch.setenv(VS.VIDEO_NATIVE_STREAM_ENV, "1")+ assert tuple(_route(fp32_dir)) == COMPONENTS+++def test_not_cuda_or_no_spec_is_a_no_op(fp32_dir, monkeypatch):+ _small_host(monkeypatch)+ assert _route(fp32_dir, target = types.SimpleNamespace(device = "mps", dtype = torch.bfloat16)) == {}+ assert _route(fp32_dir, spec = None) == {}+++def test_unwritable_cache_keeps_the_built_experts(fp32_dir, monkeypatch, no_dense):+ _small_host(monkeypatch)+ monkeypatch.setenv(C.ENV_MODE, "0")+ got = _route(fp32_dir)+ assert tuple(got) == COMPONENTS+ assert all("cache_path" not in m._unsloth_runtime_quant_info for m in got.values())+++def test_a_cached_expert_from_an_interrupted_build_is_reused(fp32_dir, monkeypatch, no_dense):+ _small_host(monkeypatch)+ first = _streamed(fp32_dir, "transformer")+ assert NC.save(+ first, _family(), _spec(), base_repo = BASE, source_base = fp32_dir, component = "transformer"+ )+ calls = []+ real = NC.build_from_shards+ monkeypatch.setattr(+ NC, "build_from_shards", lambda *a, **k: calls.append(k["component"]) or real(*a, **k)+ )+ assert tuple(_route(fp32_dir)) == COMPONENTS+ assert calls == ["transformer_2"]+++def test_a_failed_expert_build_keeps_the_dense_load(fp32_dir, monkeypatch):+ _small_host(monkeypatch)+ monkeypatch.setattr(NC, "build_from_shards", lambda *a, **k: None)+ assert _route(fp32_dir) == {}+++def test_a_cancel_raises(fp32_dir, monkeypatch, no_dense):+ _small_host(monkeypatch)+ calls = []++ def cancel():+ calls.append(1)+ if len(calls) > 5:+ raise RuntimeError("Video load was cancelled or superseded.")++ with pytest.raises(RuntimeError, match = "cancelled"):+ _route(fp32_dir, cancel_check = cancel)+++def test_an_entry_that_does_not_read_back_is_rebuilt_in_memory(fp32_dir, monkeypatch, no_dense):+ _small_host(monkeypatch)+ monkeypatch.setattr(NC, "load", lambda *a, **k: None)+ got = _route(fp32_dir)+ assert tuple(got) == COMPONENTS+ assert all("cache_path" not in m._unsloth_runtime_quant_info for m in got.values())