Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

Studio video: cache the resident runtime-quantized int8 DiT so returning loads skip the requantize (#16291)

Daniel Hanb830247af5cb
diff --git a/studio/backend/core/inference/diffusion_prequant_cache.py b/studio/backend/core/inference/diffusion_prequant_cache.py
index f2c00116077..a74e50d8208 100644
--- a/studio/backend/core/inference/diffusion_prequant_cache.py
+++ b/studio/backend/core/inference/diffusion_prequant_cache.py
@@ -1125,11 +1125,13 @@ def save_runtime_quant(
logger: Any = None,
cancel_check: Optional[Any] = None,
defer: bool = False,
+ host_snapshot: bool = False,
) -> Any:
"""Persist a just-quantized transformer; returns the artifact path or None (logged). Never raises.
``base_repo`` is recorded as ``base_model_id``; ``source_base`` is where the dense weights were read (a mirror).
``defer`` runs every check now and returns a ``PendingWrite`` owning host tensors only; falls back to the
- synchronous write when the build is off-host, RAM is short, another write runs, or ASYNC=0."""
+ synchronous write when the build is off-host, RAM is short, another write runs, or ASYNC=0.
+ ``host_snapshot``: copy a device build to host now and write after the load; if it cannot defer, skip it."""
if not enabled():
return None
t0 = time.perf_counter()
@@ -1288,7 +1290,7 @@ def save_runtime_quant(
job = _WriteJob(key, root / key, flat, header, parts, nbytes, t0, logger)
flat = header = None
if defer:
- why = _defer_blocker(job)
+ why = _defer_blocker(job, host_snapshot = host_snapshot, cancel_check = cancel_check)
if why is None:
state_dict = None
pending = PendingWrite(job, stack.pop_all())
@@ -1301,6 +1303,13 @@ def save_runtime_quant(
f"source shards, pins {job.pinned_map_bytes / 2**30:.2f} GiB of mappings)",
)
return pending
+ if host_snapshot and async_enabled():
+ _log(
+ logger,
+ "info",
+ f"skipped ({why}; a resident build is never written inside the load)",
+ )
+ return None
_log(logger, "info", f"writing {key} before the load returns ({why})")
state_dict = None
return job.run(cancel_check)
@@ -1393,6 +1402,19 @@ class _WriteJob:
fd, stamp = entry
return _FileSlice(fd, file_offset + int(t.data_ptr()) - lo, t, stamp)
+ def snapshot_to_host(self, cancel_check: Optional[Any] = None) -> None:
+ """Copy to host so the writer never touches the device (a render may be capturing a CUDA graph)."""
+ out: dict = {}
+ for name, t in self.flat.items():
+ if cancel_check is not None:
+ cancel_check()
+ if isinstance(t, _FlatRef):
+ t = t.tensor()
+ if getattr(getattr(t, "device", None), "type", "cpu") != "cpu":
+ t = t.detach().to("cpu")
+ out[name] = t
+ self.flat = out
+
def close(self) -> None:
fds, self.fds = self.fds, {}
for fd, _stamp in fds.values():
@@ -1500,22 +1522,39 @@ def _host_memory_bytes() -> tuple[Optional[int], Optional[int]]:
return total, avail
-def _defer_blocker(job: _WriteJob) -> Optional[str]:
+def _defer_blocker(
+ job: _WriteJob,
+ host_snapshot: bool = False,
+ cancel_check: Optional[Any] = None,
+) -> Optional[str]:
"""Why ``job`` must be written now, or None: it must be wholly on the host and what it keeps alive must fit."""
if not async_enabled():
return f"{ENV_ASYNC}=0"
with _lock:
if any(not p.done() for p in _BACKGROUND):
return "another background cache write is running"
+ off_host = 0
for t in job.tensors():
if isinstance(t, _FlatRef):
t = t.tensor()
device = getattr(t, "device", None)
if getattr(device, "type", None) != "cpu":
- return f"the build is on {device}, not the host"
+ if not host_snapshot:
+ return f"the build is on {device}, not the host"
+ off_host += _tensor_bytes(t)
total, avail = _host_memory_bytes()
if total is None:
return "host memory unknown"
+ if off_host:
+ reserve = max(_ASYNC_RESERVE_BYTES, int(total * _ASYNC_RESERVE_FRACTION))
+ if avail < off_host + reserve:
+ return (
+ f"host RAM short for a host copy of the build: {avail / 2**30:.1f} GiB available, the copy is "
+ f"{off_host / 2**30:.1f} GiB and needs {reserve / 2**30:.1f} GiB spare"
+ )
+ job.snapshot_to_host(cancel_check)
+ job.resolve()
+ return None
if not job.release:
job.resolve()
keeps = job.held_bytes + job.pinned_map_bytes
diff --git a/studio/backend/core/inference/video.py b/studio/backend/core/inference/video.py
index 6139c6769d2..7f96621e7da 100644
--- a/studio/backend/core/inference/video.py
+++ b/studio/backend/core/inference/video.py
@@ -3067,6 +3067,36 @@ def _video_auto_denoiser_scheme(
base_repo: Optional[str],
speed_mode: Optional[str] = None,
hf_token: Optional[str] = None,
+) -> Optional[str]:
+ """Seed scheme: a hosted checkpoint, else (auto only) the local cached resident build; None = dense shards."""
+ scheme = _video_hosted_denoiser_scheme(
+ fam,
+ target = target,
+ requested = requested,
+ base_repo = base_repo,
+ speed_mode = speed_mode,
+ hf_token = hf_token,
+ )
+ if scheme is not None:
+ return scheme
+ return _video_cached_resident_scheme(
+ fam,
+ target = target,
+ requested = requested,
+ base_repo = base_repo,
+ speed_mode = speed_mode,
+ hf_token = hf_token,
+ )
+
+
+def _video_hosted_denoiser_scheme(
+ fam: Any,
+ *,
+ target: Any,
+ requested: Optional[str],
+ base_repo: Optional[str],
+ speed_mode: Optional[str] = None,
+ hf_token: Optional[str] = None,
) -> Optional[str]:
"""The scheme a CONVENTIONAL video load would seed from a hosted checkpoint, or None. ``hf_token``
reaches the hosted header probe, so a gated artifact is judged, not read as unknown."""
@@ -3128,18 +3158,20 @@ def _save_runtime_quant_to_cache(
base: Optional[str],
source: Optional[str],
cancel_check: Any = None,
-) -> None:
- """Cache a runtime-quantized single denoiser; MoE is skipped (no lookup slot for the second expert)."""
+ defer: bool = False,
+) -> Any:
+ """Cache a runtime-quantized single denoiser; MoE is skipped (no lookup slot for the second expert).
+ ``defer``: return a ``PendingWrite`` to start after publish, or None. Never raises."""
try:
from .video_denoiser_prequant import denoiser_components
views = list(views or ())
if len(denoiser_components(fam)) != 1 or len(views) != 1:
logger.info("diffusion.prequant_cache: skipped (multi-denoiser video family)")
- return
+ return None
from .diffusion_prequant_cache import save_runtime_quant
- save_runtime_quant(
+ return save_runtime_quant(
getattr(views[0], "transformer", None),
fam,
base_repo = base,
@@ -3147,9 +3179,126 @@ def _save_runtime_quant_to_cache(
component = "transformer",
logger = logger,
cancel_check = cancel_check,
+ **({"defer": True, "host_snapshot": True} if defer else {}),
)
except Exception as exc: # noqa: BLE001 - a cache write never fails a load
logger.warning("diffusion.prequant_cache: skipped (%s)", exc)
+ return None
+
+
+# "0": no local prequant cache for AUTO resident torchao DiTs (explicit schemes unaffected).
+VIDEO_RESIDENT_QUANT_CACHE_ENV = "UNSLOTH_VIDEO_RESIDENT_QUANT_CACHE"
+_RESIDENT_QUANT_WRITE_ATTR = "_unsloth_resident_quant_write"
+
+
+def _resident_quant_cache_enabled() -> bool:
+ raw = (os.environ.get(VIDEO_RESIDENT_QUANT_CACHE_ENV) or "").strip().lower()
+ if raw in ("0", "off", "false", "no", "none"):
+ return False
+ try:
+ from .diffusion_prequant_cache import enabled
+ return bool(enabled())
+ except Exception: # noqa: BLE001
+ return False
+
+
+def _is_prequant_cache_source(source: Any) -> bool:
+ if getattr(source, "kind", None) != "path":
+ return False
+ try:
+ from .diffusion_prequant_cache import path_in_cache
+ return bool(path_in_cache(getattr(source, "location", None)))
+ except Exception: # noqa: BLE001
+ return False
+
+
+def _video_cached_resident_scheme(
+ fam: Any,
+ *,
+ target: Any,
+ requested: Optional[str],
+ base_repo: Optional[str],
+ speed_mode: Optional[str] = None,
+ hf_token: Optional[str] = None,
+) -> Optional[str]:
+ """AUTO's own resident pick if the local cache holds exactly that build, else None. Only that scheme is
+ served, so an explicit int8 entry cannot turn later auto loads into int8. Never raises."""
+ try:
+ # Unset is auto: the Studio page omits the field.
+ auto = requested is None or str(requested).strip().lower() in ("", "auto")
+ if not auto:
+ return None
+ if speed_mode is not None and str(speed_mode).strip().lower() == SPEED_OFF:
+ return None
+ if getattr(fam, "modular_workflow", None) or not _resident_quant_cache_enabled():
+ return None
+ from .video_denoiser_prequant import denoiser_components, denoiser_prequant_sources
+
+ if len(denoiser_components(fam)) != 1:
+ return None
+ import torch
+
+ # Entries are bf16 builds only (the save refuses any other dtype).
+ if getattr(target, "dtype", None) is not torch.bfloat16:
+ return None
+ from .diffusion_prequant import restricted_prequant_load_supported
+ from .diffusion_prequant_cache import CACHEABLE_SCHEMES
+
+ cached = {}
+ for scheme in CACHEABLE_SCHEMES:
+ sources = denoiser_prequant_sources(fam, scheme, base_repo, hf_token = hf_token)
+ if sources and all(_is_prequant_cache_source(src) for src in sources.values()):
+ cached[scheme] = sources
+ if not cached:
+ return None
+ pick = select_transformer_quant_scheme(target, TQ_AUTO, family = getattr(fam, "name", None))
+ if pick not in cached or not restricted_prequant_load_supported(pick):
+ return None
+ return pick
+ except Exception: # noqa: BLE001 -- unanswerable: the dense shards load and quantise as before
+ return None
+
+
+def _discard_resident_quant_write(owner: Any, token: Any = None) -> None:
+ slot = getattr(owner, "_resident_quant_write", None)
+ if slot is None or (token is not None and slot[0] != token):
+ return
+ owner._resident_quant_write = None
+ try:
+ slot[1].discard()
+ except Exception: # noqa: BLE001 - best effort
+ pass
+
+
+def _start_resident_quant_write(owner: Any, pipe: Any, token: Any) -> None:
+ slot = getattr(owner, "_resident_quant_write", None)
+ if slot is None or slot[0] != token:
+ return
+ owner._resident_quant_write = None
+ try:
+ setattr(pipe, _RESIDENT_QUANT_WRITE_ATTR, slot[1])
+ slot[1].start()
+ except Exception as exc: # noqa: BLE001 - a cache write never fails a load
+ logger.warning("diffusion.prequant_cache: resident background write skipped (%s)", exc)
+ try:
+ slot[1].discard()
+ except Exception: # noqa: BLE001
+ pass
+
+
+def _finish_resident_quant_write(pipe: Any) -> None:
+ """Teardown: let a running write finish (bounded), then cancel."""
+ write = getattr(pipe, _RESIDENT_QUANT_WRITE_ATTR, None)
+ if write is None:
+ return
+ try:
+ from .diffusion_prequant_cache import FINISH_TIMEOUT_SECONDS
+ if not write.wait(FINISH_TIMEOUT_SECONDS):
+ write.cancel()
+ write.wait(30.0)
+ setattr(pipe, _RESIDENT_QUANT_WRITE_ATTR, None)
+ except Exception: # noqa: BLE001 - teardown is best effort
+ pass
def _read_cast_cache(
@@ -5588,6 +5737,7 @@ class VideoBackend:
except Exception as exc: # noqa: BLE001 -- surfaced via load_progress
# A failed load never commits state; roll back speed globals here, token-scoped to this load.
self._rollback_precommit_globals(token)
+ _discard_resident_quant_write(self, token)
# Token-scoped: a newer LTX-2.5 load may already hold the slot with its own worker.
self._close_pending_ltx25_encoder(token)
if self._load_token != token:
@@ -9415,6 +9565,30 @@ class VideoBackend:
f"{comfy_info.get('dequantized', 0)} dequantized"
)
logger.info("video.transformer_quant: %s", transformer_quant_source)
+ elif (
+ denoiser_injected
+ and denoiser_seed_sources
+ and all(_is_prequant_cache_source(src) for src in denoiser_seed_sources.values())
+ ):
+ transformer_quant_engaged = denoiser_seed_scheme
+ transformer_quant_source = (
+ f"runtime {denoiser_seed_scheme} build read back from this machine's prequant cache of "
+ "the same build"
+ )
+ logger.info(
+ "video.transformer_quant: read the resident %s denoiser for %s from the local prequant "
+ "cache (%s); the dense denoiser shards are not loaded or quantised",
+ denoiser_seed_scheme,
+ fam.name,
+ ", ".join(
+ sorted(
+ {
+ Path(str(src.location)).parent.name
+ for src in denoiser_seed_sources.values()
+ }
+ )
+ ),
+ )
elif denoiser_injected:
transformer_quant_engaged = denoiser_seed_scheme
transformer_quant_source = (
@@ -9561,7 +9735,6 @@ class VideoBackend:
cancel_check = lambda: self._raise_if_video_load_superseded(_load_token),
)
self._raise_if_video_load_superseded(_load_token)
- # Only explicit schemes read the cache back, so an auto save would be a write nothing opens.
elif transformer_quant_pinned is not None:
_save_runtime_quant_to_cache(
fam,
@@ -9571,6 +9744,25 @@ class VideoBackend:
cancel_check = lambda: self._raise_if_video_load_superseded(_load_token),
)
self._raise_if_video_load_superseded(_load_token)
+ # Auto resident build: write after publish so the first render never waits on disk.
+ elif (
+ not video_offload
+ and native_scheme is None
+ and normalize_transformer_quant(transformer_quant) == TQ_AUTO
+ and _resident_quant_cache_enabled()
+ ):
+ _discard_resident_quant_write(self)
+ pending = _save_runtime_quant_to_cache(
+ fam,
+ views,
+ base = base,
+ source = _base_local_dir or fetch_base,
+ cancel_check = lambda: self._raise_if_video_load_superseded(_load_token),
+ defer = True,
+ )
+ if hasattr(pending, "start"):
+ self._resident_quant_write = (_load_token, pending)
+ self._raise_if_video_load_superseded(_load_token)
elif transformer_quant_pinned is not None:
transformer_quant_decline = (
f"'{transformer_quant_pinned}' did not engage on family '{fam.name}' with this "
@@ -10246,6 +10438,7 @@ class VideoBackend:
if native_writes is not None:
setattr(pipe, _NATIVE_WRITES_ATTR, native_writes)
native_writes.start()
+ _start_resident_quant_write(self, pipe, _load_token)
return self.status()
@staticmethod
@@ -13556,6 +13749,7 @@ class VideoBackend:
# A cast cache write still reads the components the teardown frees.
_finish_cast_writes(getattr(state, "pipe", None))
_finish_native_writes(getattr(state, "pipe", None))
+ _finish_resident_quant_write(getattr(state, "pipe", None))
uninstall_cfg_parallel(getattr(state, "pipe", None))
# Before clear_gpu_cache(), or the graph pool stays reserved.
diffusion_cuda_graph.uninstall_all(
diff --git a/studio/backend/tests/test_video_resident_quant_cache.py b/studio/backend/tests/test_video_resident_quant_cache.py
new file mode 100644
index 00000000000..f5003fc1390
--- /dev/null
+++ b/studio/backend/tests/test_video_resident_quant_cache.py
@@ -0,0 +1,539 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
+
+"""AUTO resident runtime-quantized video DiT and the local prequant cache (CPU only, tiny random model)."""
+
+from __future__ import annotations
+
+import inspect
+import sys
+import types
+from pathlib import Path
+
+import pytest
+
+torch = pytest.importorskip("torch")
+pytest.importorskip("torchao")
+pytest.importorskip("safetensors")
+diffusers = pytest.importorskip("diffusers")
+
+BACKEND = Path(__file__).resolve().parents[1]
+if str(BACKEND) not in sys.path:
+ sys.path.insert(0, str(BACKEND))
+
+from core.inference import diffusion_prequant_cache as C # noqa: E402
+from core.inference import prequant_safetensors as PS # noqa: E402
+from core.inference import video as vid # noqa: E402
+
+FAMILY = "hunyuanvideo-1.5"
+CLS = "HunyuanVideo15Transformer3DModel"
+
+
+def env(tmp_path, monkeypatch):
+ root = tmp_path / "cache"
+ monkeypatch.setenv(C.ENV_DIR, str(root))
+ for name in (C.ENV_MODE, C.ENV_MAX_GB, C.ENV_ASYNC, vid.VIDEO_RESIDENT_QUANT_CACHE_ENV):
+ monkeypatch.delenv(name, raising = False)
+ monkeypatch.delenv("UNSLOTH_ALLOW_LOCAL_PREQUANT_PATH", raising = False)
+ C._HEADER_MEMO.clear()
+ getattr(C, "_UNREACHABLE", {}).clear()
+ monkeypatch.setattr(C, "_hub_source_stale", lambda shards, logger = None: False)
+ yield root
+ C.cancel_background_writes(timeout = 10.0)
+
+
+def _fam(base: Path, **over) -> types.SimpleNamespace:
+ kw = dict(
+ name = FAMILY,
+ transformer_class = CLS,
+ base_repo = str(base),
+ is_moe = False,
+ modular_workflow = None,
+ prequant_repos = (),
+ )
+ kw.update(over)
+ return types.SimpleNamespace(**kw)
+
+
+def _tiny_model(seed: int = 0):
+ torch.manual_seed(seed)
+ model = diffusers.HunyuanVideo15Transformer3DModel(
+ in_channels = 9,
+ out_channels = 4,
+ num_attention_heads = 4,
+ attention_head_dim = 128, # 512-wide linears: over the 512 min_features floor, so they quantise
+ num_layers = 1,
+ num_refiner_layers = 1,
+ mlp_ratio = 2.0,
+ text_embed_dim = 64,
+ text_embed_2_dim = 32,
+ image_embed_dim = 32,
+ rope_axes_dim = (16, 56, 56),
+ target_size = 64,
+ )
+ return model.to(torch.bfloat16).eval()
+
+
+def _write_base(
+ tmp_path: Path,
+ seed: int = 0,
+ name: str = "base",
+):
+ model = _tiny_model(seed)
+ base = tmp_path / name
+ model.save_pretrained(str(base / "transformer"), safe_serialization = True)
+ return base, model
+
+
+def _flattenable(scheme: str) -> bool:
+ try:
+ from torchao.quantization import quantize_
+
+ from core.inference.diffusion_transformer_quant import _make_quant_config
+
+ m = torch.nn.Sequential(torch.nn.Linear(512, 512)).to(torch.bfloat16)
+ quantize_(m, _make_quant_config(scheme))
+ helpers = PS._torchao_helpers()
+ if helpers is None:
+ return False
+ helpers[0](m.state_dict())
+ return True
+ except Exception: # noqa: BLE001
+ return False
+
+
[email protected](scope = "module")
+def scheme() -> str:
+ """int8 (the resident auto pick) where this torchao writes it (>= 0.18, Int8Tensor); else fp8; else skip."""
+ for candidate in ("int8", "fp8"):
+ if _flattenable(candidate):
+ return candidate
+ pytest.skip("this torchao cannot write a quantized build as safetensors on the CPU")
+
+
+def _quantize(model, scheme: str, monkeypatch):
+ from core.inference import diffusion_transformer_quant as tq
+
+ monkeypatch.setattr(tq, "select_transformer_quant_scheme", lambda *a, **k: scheme)
+ monkeypatch.setattr(tq, "native_quant_scheme", lambda *a, **k: None)
+ target = types.SimpleNamespace(device = "cpu", dtype = torch.bfloat16)
+ pipe = types.SimpleNamespace(transformer = model)
+ assert tq.quantize_transformer(pipe, target, mode = "auto", family = FAMILY) == scheme
+ assert tq.transformer_is_quantised(model)
+ return model
+
+
+def _flat_equal(a: dict, b: dict) -> list:
+ fa, _ = C._flatten(a)
+ fb, _ = C._flatten(b)
+ assert set(fa) == set(fb)
+ bad = []
+ for k in fa:
+ x, y = fa[k], fb[k]
+ x = x.tensor() if isinstance(x, C._FlatRef) else x
+ y = y.tensor() if isinstance(y, C._FlatRef) else y
+ if x.dtype != y.dtype or tuple(x.shape) != tuple(y.shape):
+ bad.append(k)
+ continue
+ if not torch.equal(
+ x.contiguous().reshape(-1).view(torch.uint8),
+ y.contiguous().reshape(-1).view(torch.uint8),
+ ):
+ bad.append(k)
+ return bad
+
+
+def _views(model):
+ return [types.SimpleNamespace(transformer = model)]
+
+
+def _bf16_target():
+ return types.SimpleNamespace(device = "cuda", dtype = torch.bfloat16)
+
+
+def test_the_deferred_resident_write_reads_back_bit_identically(env, tmp_path, scheme, monkeypatch):
+ from core.inference.diffusion_prequant import PrequantSource, load_prequantized_transformer
+
+ base, model = _write_base(tmp_path)
+ fam = _fam(base)
+ fresh = _quantize(model, scheme, monkeypatch)
+
+ pending = vid._save_runtime_quant_to_cache(
+ fam, _views(fresh), base = str(base), source = str(base), defer = True
+ )
+ assert hasattr(pending, "start"), "the auto resident write must be deferred to after publish"
+ # Nothing is published before start(): a load that fails before publish leaves no entry.
+ assert C.lookup(fam, scheme, base_repo = str(base)) is None
+ assert pending.start() and pending.wait(120)
+ path = C.lookup(fam, scheme, base_repo = str(base))
+ assert path is not None and Path(path).is_file()
+ # Keyed exactly like the explicit save (same source fingerprint, scheme, filters, toolchain).
+ key, _parts = C._key_for(fam, scheme, str(base), "transformer", base_id = str(base))
+ assert Path(path).parent.name == key
+
+ loaded = load_prequantized_transformer(
+ diffusers.HunyuanVideo15Transformer3DModel,
+ str(base),
+ PrequantSource(kind = "path", location = path, filename = None),
+ device = "cpu",
+ dtype = torch.bfloat16,
+ scheme = scheme,
+ min_features = 512,
+ component = "transformer",
+ local_files_only = True,
+ )
+ assert loaded is not None
+ assert _flat_equal(fresh.state_dict(), loaded.state_dict()) == []
+ assert {n for n, _ in fresh.named_buffers()} == {n for n, _ in loaded.named_buffers()}
+ g = torch.Generator().manual_seed(1)
+ inputs = dict(
+ hidden_states = torch.randn(1, 9, 1, 4, 4, generator = g).to(torch.bfloat16),
+ timestep = torch.tensor([500.0]).to(torch.bfloat16),
+ encoder_hidden_states = torch.randn(1, 8, 64, generator = g).to(torch.bfloat16),
+ encoder_attention_mask = torch.ones(1, 8, dtype = torch.bool),
+ encoder_hidden_states_2 = torch.randn(1, 4, 32, generator = g).to(torch.bfloat16),
+ encoder_attention_mask_2 = torch.ones(1, 4, dtype = torch.bool),
+ image_embeds = torch.zeros(1, 4, 32).to(torch.bfloat16),
+ return_dict = False,
+ )
+ with torch.no_grad():
+ want = fresh(**inputs)[0]
+ got = loaded(**inputs)[0]
+ assert torch.equal(want.contiguous().view(torch.uint8), got.contiguous().view(torch.uint8))
+
+
+def test_a_returning_auto_load_seeds_exactly_that_build(env, tmp_path, scheme, monkeypatch):
+ base, model = _write_base(tmp_path)
+ fam = _fam(base)
+ monkeypatch.setattr(vid, "select_transformer_quant_scheme", lambda *a, **k: scheme)
+ kw = dict(target = _bf16_target(), requested = "auto", base_repo = str(base), speed_mode = None)
+ unset = dict(kw, requested = None)
+ assert vid._video_auto_denoiser_scheme(fam, **kw) is None
+ assert vid._video_auto_denoiser_scheme(fam, **unset) is None
+
+ _quantize(model, scheme, monkeypatch)
+ pending = vid._save_runtime_quant_to_cache(
+ fam, _views(model), base = str(base), source = str(base), defer = True
+ )
+ assert pending.start() and pending.wait(120)
+ assert vid._video_auto_denoiser_scheme(fam, **kw) == scheme
+ # The default Studio request (transformer_quant omitted) replays the same entry.
+ assert vid._video_auto_denoiser_scheme(fam, **unset) == scheme
+ assert vid._video_auto_denoiser_scheme(fam, **dict(unset, requested = "none")) is None
+
+ from core.inference.video_denoiser_prequant import denoiser_prequant_sources
+
+ sources = denoiser_prequant_sources(fam, scheme, str(base))
+ assert sources and all(vid._is_prequant_cache_source(s) for s in sources.values())
+
+
+def _stub_decision(
+ monkeypatch,
+ *,
+ cached = ("int8",),
+ pick = "int8",
+):
+ import core.inference.diffusion_prequant as pq
+ import core.inference.video_denoiser_prequant as vdp
+
+ calls = {"select": 0}
+
+ def _sources(
+ fam,
+ scheme,
+ base_repo,
+ *,
+ hf_token = None,
+ ):
+ if scheme not in cached:
+ return None
+ return {
+ "transformer": types.SimpleNamespace(
+ kind = "path", location = f"/c/{scheme}/model.safetensors"
+ )
+ }
+
+ def _select(*a, **k):
+ calls["select"] += 1
+ return pick
+
+ monkeypatch.setattr(vdp, "denoiser_prequant_sources", _sources)
+ monkeypatch.setattr(
+ vid, "_is_prequant_cache_source", lambda src: str(src.location).startswith("/c/")
+ )
+ monkeypatch.setattr(vid, "select_transformer_quant_scheme", _select)
+ monkeypatch.setattr(pq, "restricted_prequant_load_supported", lambda scheme = None, *a: True)
+ return calls
+
+
+def _decide(fam = None, **over):
+ kw = dict(target = _bf16_target(), requested = "auto", base_repo = "org/base", speed_mode = None)
+ kw.update(over)
+ return vid._video_cached_resident_scheme(fam or _fam(Path("/nonexistent")), **kw)
+
+
+def test_auto_replays_only_its_own_pick(env, monkeypatch):
+ _stub_decision(monkeypatch)
+ assert _decide() == "int8"
+ # An explicit int8 entry must not turn auto's fp8 pick into int8.
+ _stub_decision(monkeypatch, cached = ("int8",), pick = "fp8")
+ assert _decide() is None
+ _stub_decision(monkeypatch, cached = ("int8",), pick = None)
+ assert _decide() is None
+
+
[email protected]("requested", [None, "", " Auto ", "auto"])
+def test_an_unset_request_replays_exactly_like_auto(env, monkeypatch, requested):
+ """The Studio video page omits transformer_quant for the hardware ladder: unset is the default user load."""
+ _stub_decision(monkeypatch)
+ assert _decide(requested = requested) == "int8"
+ _stub_decision(monkeypatch, cached = ("int8",), pick = "fp8")
+ assert _decide(requested = requested) is None
+ calls = _stub_decision(monkeypatch, cached = ())
+ assert _decide(requested = requested) is None and calls["select"] == 0
+
+
+def test_the_load_maps_unset_to_auto_before_the_seed_and_the_write():
+ """The write branch and the seed call see ``transformer_quant`` after load_pipeline turned unset into auto."""
+ src = inspect.getsource(vid.VideoBackend.load_pipeline)
+ mapped = src.index('transformer_quant = "off" if speed_off else TQ_AUTO')
+ unset = src[mapped - 600 : mapped]
+ assert "if transformer_quant is None or str(transformer_quant).strip().lower() in (" in unset
+ seed = src.index(
+ "denoiser_seed_scheme = _video_auto_denoiser_planned or _video_auto_denoiser_scheme("
+ )
+ write = src.index(
+ "_save_runtime_quant_to_cache(", src.index("_save_runtime_quant_to_cache(") + 1
+ )
+ assert mapped < seed < write
+ assert "requested = transformer_quant," in src[seed : seed + 300]
+
+
+def test_no_entry_asks_nothing_and_changes_nothing(env, monkeypatch):
+ calls = _stub_decision(monkeypatch, cached = ())
+ assert _decide() is None
+ assert calls["select"] == 0
+
+
+def test_explicit_unset_speed_off_modular_moe_and_non_bf16_never_replay(env, monkeypatch):
+ _stub_decision(monkeypatch)
+ assert _decide(requested = "int8") is None
+ assert _decide(requested = "off") is None
+ assert _decide(requested = "none") is None
+ assert _decide(speed_mode = "off") is None
+ assert _decide(speed_mode = " OFF ") is None
+ assert _decide(speed_mode = "off", requested = None) is None
+ assert _decide(_fam(Path("/x"), modular_workflow = "fl2va")) is None
+ assert _decide(_fam(Path("/x"), is_moe = True)) is None
+ assert _decide(target = types.SimpleNamespace(device = "cuda", dtype = torch.float16)) is None
+ assert _decide() == "int8"
+
+
+def test_a_hosted_or_operator_source_is_not_a_cache_replay(env, monkeypatch):
+ import core.inference.video_denoiser_prequant as vdp
+
+ _stub_decision(monkeypatch)
+ monkeypatch.setattr(
+ vdp,
+ "denoiser_prequant_sources",
+ lambda fam, scheme, base, hf_token = None: {
+ "transformer": types.SimpleNamespace(kind = "repo", location = "org/hosted")
+ },
+ )
+ assert _decide() is None
+
+
+def test_an_install_that_cannot_open_the_entry_keeps_the_dense_build(env, monkeypatch):
+ import core.inference.diffusion_prequant as pq
+
+ _stub_decision(monkeypatch)
+ monkeypatch.setattr(pq, "restricted_prequant_load_supported", lambda scheme = None, *a: False)
+ assert _decide() is None
+
+
+def test_kill_switches(env, monkeypatch):
+ _stub_decision(monkeypatch)
+ assert vid._resident_quant_cache_enabled()
+ monkeypatch.setenv(vid.VIDEO_RESIDENT_QUANT_CACHE_ENV, "0")
+ assert not vid._resident_quant_cache_enabled()
+ assert _decide() is None
+ monkeypatch.delenv(vid.VIDEO_RESIDENT_QUANT_CACHE_ENV)
+ monkeypatch.setenv(C.ENV_MODE, "off")
+ assert not vid._resident_quant_cache_enabled()
+ assert _decide() is None
+
+
+def test_the_hosted_seed_wins_and_the_cache_is_only_a_fallback(env, monkeypatch):
+ seen = []
+ monkeypatch.setattr(vid, "_video_hosted_denoiser_scheme", lambda fam, **kw: "nvfp4")
+ monkeypatch.setattr(
+ vid, "_video_cached_resident_scheme", lambda fam, **kw: seen.append(1) or "int8"
+ )
+ kw = dict(target = None, requested = "auto", base_repo = "b", speed_mode = None)
+ assert vid._video_auto_denoiser_scheme(None, **kw) == "nvfp4" and seen == []
+ monkeypatch.setattr(vid, "_video_hosted_denoiser_scheme", lambda fam, **kw: None)
+ assert vid._video_auto_denoiser_scheme(None, **kw) == "int8" and seen == [1]
+
+
+class _FakeDeviceTensor:
+ """Reports a CUDA device; ``.to('cpu')`` hands back the real host tensor (a CPU-only host has no device)."""
+
+ def __init__(self, t):
+ self._t = t
+ self.device = types.SimpleNamespace(type = "cuda")
+ self.dtype = t.dtype
+ self.shape = t.shape
+
+ def numel(self):
+ return self._t.numel()
+
+ def element_size(self):
+ return self._t.element_size()
+
+ def detach(self):
+ return self
+
+ def to(self, device):
+ assert device == "cpu"
+ return self._t.clone()
+
+
+def _job(tmp_path, flat):
+ return C._WriteJob(
+ "k" * 32,
+ tmp_path / "k",
+ flat,
+ {},
+ {},
+ sum(C._tensor_bytes(t) for t in flat.values()),
+ 0.0,
+ None,
+ )
+
+
+def test_the_snapshot_moves_every_device_tensor_to_the_host(env, tmp_path, monkeypatch):
+ a, b = torch.randn(8, 8).to(torch.bfloat16), torch.randint(-128, 127, (16,), dtype = torch.int8)
+ job = _job(tmp_path, {"a": _FakeDeviceTensor(a), "b": b})
+ monkeypatch.setattr(C, "_host_memory_bytes", lambda: (64 << 30, 32 << 30))
+ assert C._defer_blocker(job, host_snapshot = True) is None
+ assert all(getattr(t, "device", None).type == "cpu" for t in job.tensors())
+ assert torch.equal(job.flat["a"], a) and torch.equal(job.flat["b"], b)
+ # Without host_snapshot a device build is never deferred (the image path's rule, unchanged).
+ job2 = _job(tmp_path, {"a": _FakeDeviceTensor(a)})
+ assert "not the host" in C._defer_blocker(job2)
+
+
+def test_a_snapshot_that_does_not_fit_host_ram_is_refused(env, tmp_path, monkeypatch):
+ a = torch.zeros(1024, 1024, dtype = torch.bfloat16)
+ job = _job(tmp_path, {"a": _FakeDeviceTensor(a)})
+ monkeypatch.setattr(C, "_host_memory_bytes", lambda: (8 << 30, 1 << 30))
+ why = C._defer_blocker(job, host_snapshot = True)
+ assert why is not None and "host RAM short" in why
+ assert isinstance(job.flat["a"], _FakeDeviceTensor)
+
+
+def test_a_resident_write_that_cannot_defer_is_skipped_not_written_in_the_load(
+ env, tmp_path, scheme, monkeypatch
+):
+ base, model = _write_base(tmp_path)
+ fam = _fam(base)
+ _quantize(model, scheme, monkeypatch)
+ monkeypatch.setattr(
+ C, "_defer_blocker", lambda job, **kw: "another background cache write is running"
+ )
+ assert (
+ vid._save_runtime_quant_to_cache(
+ fam, _views(model), base = str(base), source = str(base), defer = True
+ )
+ is None
+ )
+ assert C.lookup(fam, scheme, base_repo = str(base)) is None
+ monkeypatch.setenv(C.ENV_ASYNC, "0")
+ monkeypatch.setattr(C, "_defer_blocker", lambda job, **kw: f"{C.ENV_ASYNC}=0")
+ out = vid._save_runtime_quant_to_cache(
+ fam, _views(model), base = str(base), source = str(base), defer = True
+ )
+ assert isinstance(out, str) and C.lookup(fam, scheme, base_repo = str(base)) == out
+
+
+def test_a_discarded_write_releases_its_slot_and_leaves_no_entry(
+ env, tmp_path, scheme, monkeypatch
+):
+ base, model = _write_base(tmp_path)
+ fam = _fam(base)
+ _quantize(model, scheme, monkeypatch)
+ pending = vid._save_runtime_quant_to_cache(
+ fam, _views(model), base = str(base), source = str(base), defer = True
+ )
+ owner = types.SimpleNamespace(_resident_quant_write = (7, pending))
+ vid._discard_resident_quant_write(owner, 8) # another load's token: untouched
+ assert owner._resident_quant_write is not None and not pending.done()
+ vid._discard_resident_quant_write(owner, 7)
+ assert owner._resident_quant_write is None and pending.done()
+ assert C.lookup(fam, scheme, base_repo = str(base)) is None
+ assert not any(not p.done() for p in C._BACKGROUND)
+
+
+def test_publish_starts_only_this_loads_write_and_teardown_finishes_it(env):
+ events = []
+
+ class _W:
+ def start(self):
+ events.append("start")
+ return True
+
+ def wait(self, timeout = None):
+ events.append("wait")
+ return True
+
+ def cancel(self):
+ events.append("cancel")
+
+ def discard(self):
+ events.append("discard")
+
+ w = _W()
+ owner = types.SimpleNamespace(_resident_quant_write = (1, w))
+ pipe = types.SimpleNamespace()
+ vid._start_resident_quant_write(owner, pipe, 2)
+ assert events == [] and owner._resident_quant_write is not None
+ vid._start_resident_quant_write(owner, pipe, 1)
+ assert events == ["start"] and owner._resident_quant_write is None
+ assert getattr(pipe, vid._RESIDENT_QUANT_WRITE_ATTR) is w
+ vid._finish_resident_quant_write(pipe)
+ assert events == ["start", "wait"] and getattr(pipe, vid._RESIDENT_QUANT_WRITE_ATTR) is None
+
+
+def test_the_load_writes_auto_resident_builds_deferred_and_keeps_explicit_writes_inline():
+ src = inspect.getsource(vid.VideoBackend.load_pipeline)
+ explicit_at = src.index("_save_runtime_quant_to_cache(")
+ assert "elif transformer_quant_pinned is not None:" in src[explicit_at - 200 : explicit_at]
+ assert "defer = True" not in src[explicit_at : explicit_at + 400]
+ auto_at = src.index("_save_runtime_quant_to_cache(", explicit_at + 1)
+ guard = src[auto_at - 700 : auto_at]
+ for needle in (
+ "not video_offload",
+ "native_scheme is None",
+ "normalize_transformer_quant(transformer_quant) == TQ_AUTO",
+ "_resident_quant_cache_enabled()",
+ ):
+ assert needle in guard, needle
+ assert "defer = True" in src[auto_at : auto_at + 400]
+ assert "source = _base_local_dir or fetch_base," in src[auto_at : auto_at + 400]
+ assert "_start_resident_quant_write(self, pipe, _load_token)" in src
+ assert src.index("_start_resident_quant_write(") > src.index(
+ '"video.loaded: %s (%s, %s, offload=%s'
+ )
+
+
+def test_a_failed_load_discards_and_an_unload_finishes_the_write():
+ backend_src = inspect.getsource(vid.VideoBackend)
+ assert "_discard_resident_quant_write(self, token)" in backend_src
+ assert '_finish_resident_quant_write(getattr(state, "pipe", None))' in backend_src
+
+
+def test_a_cache_seed_says_so_in_the_log_and_the_resolved_source():
+ src = inspect.getsource(vid.VideoBackend.load_pipeline)
+ assert "read the resident %s denoiser for %s from the local prequant" in src
+ assert "read back from this machine's prequant cache" in src