Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
Studio video: cache the resident runtime-quantized int8 DiT so returning loads skip the requantize (#16291)
Daniel Hanb830247af5cb
diff --git a/studio/backend/core/inference/diffusion_prequant_cache.py b/studio/backend/core/inference/diffusion_prequant_cache.pyindex f2c00116077..a74e50d8208 100644--- a/studio/backend/core/inference/diffusion_prequant_cache.py+++ b/studio/backend/core/inference/diffusion_prequant_cache.py@@ -1125,11 +1125,13 @@ def save_runtime_quant(logger: Any = None,cancel_check: Optional[Any] = None,defer: bool = False,+ host_snapshot: bool = False,) -> Any:"""Persist a just-quantized transformer; returns the artifact path or None (logged). Never raises.``base_repo`` is recorded as ``base_model_id``; ``source_base`` is where the dense weights were read (a mirror).``defer`` runs every check now and returns a ``PendingWrite`` owning host tensors only; falls back to the- synchronous write when the build is off-host, RAM is short, another write runs, or ASYNC=0."""+ synchronous write when the build is off-host, RAM is short, another write runs, or ASYNC=0.+ ``host_snapshot``: copy a device build to host now and write after the load; if it cannot defer, skip it."""if not enabled():return Nonet0 = time.perf_counter()@@ -1288,7 +1290,7 @@ def save_runtime_quant(job = _WriteJob(key, root / key, flat, header, parts, nbytes, t0, logger)flat = header = Noneif defer:- why = _defer_blocker(job)+ why = _defer_blocker(job, host_snapshot = host_snapshot, cancel_check = cancel_check)if why is None:state_dict = Nonepending = PendingWrite(job, stack.pop_all())@@ -1301,6 +1303,13 @@ def save_runtime_quant(f"source shards, pins {job.pinned_map_bytes / 2**30:.2f} GiB of mappings)",)return pending+ if host_snapshot and async_enabled():+ _log(+ logger,+ "info",+ f"skipped ({why}; a resident build is never written inside the load)",+ )+ return None_log(logger, "info", f"writing {key} before the load returns ({why})")state_dict = Nonereturn job.run(cancel_check)@@ -1393,6 +1402,19 @@ class _WriteJob:fd, stamp = entryreturn _FileSlice(fd, file_offset + int(t.data_ptr()) - lo, t, stamp)+ def snapshot_to_host(self, cancel_check: Optional[Any] = None) -> None:+ """Copy to host so the writer never touches the device (a render may be capturing a CUDA graph)."""+ out: dict = {}+ for name, t in self.flat.items():+ if cancel_check is not None:+ cancel_check()+ if isinstance(t, _FlatRef):+ t = t.tensor()+ if getattr(getattr(t, "device", None), "type", "cpu") != "cpu":+ t = t.detach().to("cpu")+ out[name] = t+ self.flat = out+def close(self) -> None:fds, self.fds = self.fds, {}for fd, _stamp in fds.values():@@ -1500,22 +1522,39 @@ def _host_memory_bytes() -> tuple[Optional[int], Optional[int]]:return total, avail-def _defer_blocker(job: _WriteJob) -> Optional[str]:+def _defer_blocker(+ job: _WriteJob,+ host_snapshot: bool = False,+ cancel_check: Optional[Any] = None,+) -> Optional[str]:"""Why ``job`` must be written now, or None: it must be wholly on the host and what it keeps alive must fit."""if not async_enabled():return f"{ENV_ASYNC}=0"with _lock:if any(not p.done() for p in _BACKGROUND):return "another background cache write is running"+ off_host = 0for t in job.tensors():if isinstance(t, _FlatRef):t = t.tensor()device = getattr(t, "device", None)if getattr(device, "type", None) != "cpu":- return f"the build is on {device}, not the host"+ if not host_snapshot:+ return f"the build is on {device}, not the host"+ off_host += _tensor_bytes(t)total, avail = _host_memory_bytes()if total is None:return "host memory unknown"+ if off_host:+ reserve = max(_ASYNC_RESERVE_BYTES, int(total * _ASYNC_RESERVE_FRACTION))+ if avail < off_host + reserve:+ return (+ f"host RAM short for a host copy of the build: {avail / 2**30:.1f} GiB available, the copy is "+ f"{off_host / 2**30:.1f} GiB and needs {reserve / 2**30:.1f} GiB spare"+ )+ job.snapshot_to_host(cancel_check)+ job.resolve()+ return Noneif not job.release:job.resolve()keeps = job.held_bytes + job.pinned_map_bytesdiff --git a/studio/backend/core/inference/video.py b/studio/backend/core/inference/video.pyindex 6139c6769d2..7f96621e7da 100644--- a/studio/backend/core/inference/video.py+++ b/studio/backend/core/inference/video.py@@ -3067,6 +3067,36 @@ def _video_auto_denoiser_scheme(base_repo: Optional[str],speed_mode: Optional[str] = None,hf_token: Optional[str] = None,+) -> Optional[str]:+ """Seed scheme: a hosted checkpoint, else (auto only) the local cached resident build; None = dense shards."""+ scheme = _video_hosted_denoiser_scheme(+ fam,+ target = target,+ requested = requested,+ base_repo = base_repo,+ speed_mode = speed_mode,+ hf_token = hf_token,+ )+ if scheme is not None:+ return scheme+ return _video_cached_resident_scheme(+ fam,+ target = target,+ requested = requested,+ base_repo = base_repo,+ speed_mode = speed_mode,+ hf_token = hf_token,+ )+++def _video_hosted_denoiser_scheme(+ fam: Any,+ *,+ target: Any,+ requested: Optional[str],+ base_repo: Optional[str],+ speed_mode: Optional[str] = None,+ hf_token: Optional[str] = None,) -> Optional[str]:"""The scheme a CONVENTIONAL video load would seed from a hosted checkpoint, or None. ``hf_token``reaches the hosted header probe, so a gated artifact is judged, not read as unknown."""@@ -3128,18 +3158,20 @@ def _save_runtime_quant_to_cache(base: Optional[str],source: Optional[str],cancel_check: Any = None,-) -> None:- """Cache a runtime-quantized single denoiser; MoE is skipped (no lookup slot for the second expert)."""+ defer: bool = False,+) -> Any:+ """Cache a runtime-quantized single denoiser; MoE is skipped (no lookup slot for the second expert).+ ``defer``: return a ``PendingWrite`` to start after publish, or None. Never raises."""try:from .video_denoiser_prequant import denoiser_componentsviews = list(views or ())if len(denoiser_components(fam)) != 1 or len(views) != 1:logger.info("diffusion.prequant_cache: skipped (multi-denoiser video family)")- return+ return Nonefrom .diffusion_prequant_cache import save_runtime_quant- save_runtime_quant(+ return save_runtime_quant(getattr(views[0], "transformer", None),fam,base_repo = base,@@ -3147,9 +3179,126 @@ def _save_runtime_quant_to_cache(component = "transformer",logger = logger,cancel_check = cancel_check,+ **({"defer": True, "host_snapshot": True} if defer else {}),)except Exception as exc: # noqa: BLE001 - a cache write never fails a loadlogger.warning("diffusion.prequant_cache: skipped (%s)", exc)+ return None+++# "0": no local prequant cache for AUTO resident torchao DiTs (explicit schemes unaffected).+VIDEO_RESIDENT_QUANT_CACHE_ENV = "UNSLOTH_VIDEO_RESIDENT_QUANT_CACHE"+_RESIDENT_QUANT_WRITE_ATTR = "_unsloth_resident_quant_write"+++def _resident_quant_cache_enabled() -> bool:+ raw = (os.environ.get(VIDEO_RESIDENT_QUANT_CACHE_ENV) or "").strip().lower()+ if raw in ("0", "off", "false", "no", "none"):+ return False+ try:+ from .diffusion_prequant_cache import enabled+ return bool(enabled())+ except Exception: # noqa: BLE001+ return False+++def _is_prequant_cache_source(source: Any) -> bool:+ if getattr(source, "kind", None) != "path":+ return False+ try:+ from .diffusion_prequant_cache import path_in_cache+ return bool(path_in_cache(getattr(source, "location", None)))+ except Exception: # noqa: BLE001+ return False+++def _video_cached_resident_scheme(+ fam: Any,+ *,+ target: Any,+ requested: Optional[str],+ base_repo: Optional[str],+ speed_mode: Optional[str] = None,+ hf_token: Optional[str] = None,+) -> Optional[str]:+ """AUTO's own resident pick if the local cache holds exactly that build, else None. Only that scheme is+ served, so an explicit int8 entry cannot turn later auto loads into int8. Never raises."""+ try:+ # Unset is auto: the Studio page omits the field.+ auto = requested is None or str(requested).strip().lower() in ("", "auto")+ if not auto:+ return None+ if speed_mode is not None and str(speed_mode).strip().lower() == SPEED_OFF:+ return None+ if getattr(fam, "modular_workflow", None) or not _resident_quant_cache_enabled():+ return None+ from .video_denoiser_prequant import denoiser_components, denoiser_prequant_sources++ if len(denoiser_components(fam)) != 1:+ return None+ import torch++ # Entries are bf16 builds only (the save refuses any other dtype).+ if getattr(target, "dtype", None) is not torch.bfloat16:+ return None+ from .diffusion_prequant import restricted_prequant_load_supported+ from .diffusion_prequant_cache import CACHEABLE_SCHEMES++ cached = {}+ for scheme in CACHEABLE_SCHEMES:+ sources = denoiser_prequant_sources(fam, scheme, base_repo, hf_token = hf_token)+ if sources and all(_is_prequant_cache_source(src) for src in sources.values()):+ cached[scheme] = sources+ if not cached:+ return None+ pick = select_transformer_quant_scheme(target, TQ_AUTO, family = getattr(fam, "name", None))+ if pick not in cached or not restricted_prequant_load_supported(pick):+ return None+ return pick+ except Exception: # noqa: BLE001 -- unanswerable: the dense shards load and quantise as before+ return None+++def _discard_resident_quant_write(owner: Any, token: Any = None) -> None:+ slot = getattr(owner, "_resident_quant_write", None)+ if slot is None or (token is not None and slot[0] != token):+ return+ owner._resident_quant_write = None+ try:+ slot[1].discard()+ except Exception: # noqa: BLE001 - best effort+ pass+++def _start_resident_quant_write(owner: Any, pipe: Any, token: Any) -> None:+ slot = getattr(owner, "_resident_quant_write", None)+ if slot is None or slot[0] != token:+ return+ owner._resident_quant_write = None+ try:+ setattr(pipe, _RESIDENT_QUANT_WRITE_ATTR, slot[1])+ slot[1].start()+ except Exception as exc: # noqa: BLE001 - a cache write never fails a load+ logger.warning("diffusion.prequant_cache: resident background write skipped (%s)", exc)+ try:+ slot[1].discard()+ except Exception: # noqa: BLE001+ pass+++def _finish_resident_quant_write(pipe: Any) -> None:+ """Teardown: let a running write finish (bounded), then cancel."""+ write = getattr(pipe, _RESIDENT_QUANT_WRITE_ATTR, None)+ if write is None:+ return+ try:+ from .diffusion_prequant_cache import FINISH_TIMEOUT_SECONDS+ if not write.wait(FINISH_TIMEOUT_SECONDS):+ write.cancel()+ write.wait(30.0)+ setattr(pipe, _RESIDENT_QUANT_WRITE_ATTR, None)+ except Exception: # noqa: BLE001 - teardown is best effort+ passdef _read_cast_cache(@@ -5588,6 +5737,7 @@ class VideoBackend:except Exception as exc: # noqa: BLE001 -- surfaced via load_progress# A failed load never commits state; roll back speed globals here, token-scoped to this load.self._rollback_precommit_globals(token)+ _discard_resident_quant_write(self, token)# Token-scoped: a newer LTX-2.5 load may already hold the slot with its own worker.self._close_pending_ltx25_encoder(token)if self._load_token != token:@@ -9415,6 +9565,30 @@ class VideoBackend:f"{comfy_info.get('dequantized', 0)} dequantized")logger.info("video.transformer_quant: %s", transformer_quant_source)+ elif (+ denoiser_injected+ and denoiser_seed_sources+ and all(_is_prequant_cache_source(src) for src in denoiser_seed_sources.values())+ ):+ transformer_quant_engaged = denoiser_seed_scheme+ transformer_quant_source = (+ f"runtime {denoiser_seed_scheme} build read back from this machine's prequant cache of "+ "the same build"+ )+ logger.info(+ "video.transformer_quant: read the resident %s denoiser for %s from the local prequant "+ "cache (%s); the dense denoiser shards are not loaded or quantised",+ denoiser_seed_scheme,+ fam.name,+ ", ".join(+ sorted(+ {+ Path(str(src.location)).parent.name+ for src in denoiser_seed_sources.values()+ }+ )+ ),+ )elif denoiser_injected:transformer_quant_engaged = denoiser_seed_schemetransformer_quant_source = (@@ -9561,7 +9735,6 @@ class VideoBackend:cancel_check = lambda: self._raise_if_video_load_superseded(_load_token),)self._raise_if_video_load_superseded(_load_token)- # Only explicit schemes read the cache back, so an auto save would be a write nothing opens.elif transformer_quant_pinned is not None:_save_runtime_quant_to_cache(fam,@@ -9571,6 +9744,25 @@ class VideoBackend:cancel_check = lambda: self._raise_if_video_load_superseded(_load_token),)self._raise_if_video_load_superseded(_load_token)+ # Auto resident build: write after publish so the first render never waits on disk.+ elif (+ not video_offload+ and native_scheme is None+ and normalize_transformer_quant(transformer_quant) == TQ_AUTO+ and _resident_quant_cache_enabled()+ ):+ _discard_resident_quant_write(self)+ pending = _save_runtime_quant_to_cache(+ fam,+ views,+ base = base,+ source = _base_local_dir or fetch_base,+ cancel_check = lambda: self._raise_if_video_load_superseded(_load_token),+ defer = True,+ )+ if hasattr(pending, "start"):+ self._resident_quant_write = (_load_token, pending)+ self._raise_if_video_load_superseded(_load_token)elif transformer_quant_pinned is not None:transformer_quant_decline = (f"'{transformer_quant_pinned}' did not engage on family '{fam.name}' with this "@@ -10246,6 +10438,7 @@ class VideoBackend:if native_writes is not None:setattr(pipe, _NATIVE_WRITES_ATTR, native_writes)native_writes.start()+ _start_resident_quant_write(self, pipe, _load_token)return self.status()@staticmethod@@ -13556,6 +13749,7 @@ class VideoBackend:# A cast cache write still reads the components the teardown frees._finish_cast_writes(getattr(state, "pipe", None))_finish_native_writes(getattr(state, "pipe", None))+ _finish_resident_quant_write(getattr(state, "pipe", None))uninstall_cfg_parallel(getattr(state, "pipe", None))# Before clear_gpu_cache(), or the graph pool stays reserved.diffusion_cuda_graph.uninstall_all(diff --git a/studio/backend/tests/test_video_resident_quant_cache.py b/studio/backend/tests/test_video_resident_quant_cache.pynew file mode 100644index 00000000000..f5003fc1390--- /dev/null+++ b/studio/backend/tests/test_video_resident_quant_cache.py@@ -0,0 +1,539 @@+# SPDX-License-Identifier: AGPL-3.0-only+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0++"""AUTO resident runtime-quantized video DiT and the local prequant cache (CPU only, tiny random model)."""++from __future__ import annotations++import inspect+import sys+import types+from pathlib import Path++import pytest++torch = pytest.importorskip("torch")+pytest.importorskip("torchao")+pytest.importorskip("safetensors")+diffusers = pytest.importorskip("diffusers")++BACKEND = Path(__file__).resolve().parents[1]+if str(BACKEND) not in sys.path:+ sys.path.insert(0, str(BACKEND))++from core.inference import diffusion_prequant_cache as C # noqa: E402+from core.inference import prequant_safetensors as PS # noqa: E402+from core.inference import video as vid # noqa: E402++FAMILY = "hunyuanvideo-1.5"+CLS = "HunyuanVideo15Transformer3DModel"+++def env(tmp_path, monkeypatch):+ root = tmp_path / "cache"+ monkeypatch.setenv(C.ENV_DIR, str(root))+ for name in (C.ENV_MODE, C.ENV_MAX_GB, C.ENV_ASYNC, vid.VIDEO_RESIDENT_QUANT_CACHE_ENV):+ monkeypatch.delenv(name, raising = False)+ monkeypatch.delenv("UNSLOTH_ALLOW_LOCAL_PREQUANT_PATH", raising = False)+ C._HEADER_MEMO.clear()+ getattr(C, "_UNREACHABLE", {}).clear()+ monkeypatch.setattr(C, "_hub_source_stale", lambda shards, logger = None: False)+ yield root+ C.cancel_background_writes(timeout = 10.0)+++def _fam(base: Path, **over) -> types.SimpleNamespace:+ kw = dict(+ name = FAMILY,+ transformer_class = CLS,+ base_repo = str(base),+ is_moe = False,+ modular_workflow = None,+ prequant_repos = (),+ )+ kw.update(over)+ return types.SimpleNamespace(**kw)+++def _tiny_model(seed: int = 0):+ torch.manual_seed(seed)+ model = diffusers.HunyuanVideo15Transformer3DModel(+ in_channels = 9,+ out_channels = 4,+ num_attention_heads = 4,+ attention_head_dim = 128, # 512-wide linears: over the 512 min_features floor, so they quantise+ num_layers = 1,+ num_refiner_layers = 1,+ mlp_ratio = 2.0,+ text_embed_dim = 64,+ text_embed_2_dim = 32,+ image_embed_dim = 32,+ rope_axes_dim = (16, 56, 56),+ target_size = 64,+ )+ return model.to(torch.bfloat16).eval()+++def _write_base(+ tmp_path: Path,+ seed: int = 0,+ name: str = "base",+):+ model = _tiny_model(seed)+ base = tmp_path / name+ model.save_pretrained(str(base / "transformer"), safe_serialization = True)+ return base, model+++def _flattenable(scheme: str) -> bool:+ try:+ from torchao.quantization import quantize_++ from core.inference.diffusion_transformer_quant import _make_quant_config++ m = torch.nn.Sequential(torch.nn.Linear(512, 512)).to(torch.bfloat16)+ quantize_(m, _make_quant_config(scheme))+ helpers = PS._torchao_helpers()+ if helpers is None:+ return False+ helpers[0](m.state_dict())+ return True+ except Exception: # noqa: BLE001+ return False++[email protected](scope = "module")+def scheme() -> str:+ """int8 (the resident auto pick) where this torchao writes it (>= 0.18, Int8Tensor); else fp8; else skip."""+ for candidate in ("int8", "fp8"):+ if _flattenable(candidate):+ return candidate+ pytest.skip("this torchao cannot write a quantized build as safetensors on the CPU")+++def _quantize(model, scheme: str, monkeypatch):+ from core.inference import diffusion_transformer_quant as tq++ monkeypatch.setattr(tq, "select_transformer_quant_scheme", lambda *a, **k: scheme)+ monkeypatch.setattr(tq, "native_quant_scheme", lambda *a, **k: None)+ target = types.SimpleNamespace(device = "cpu", dtype = torch.bfloat16)+ pipe = types.SimpleNamespace(transformer = model)+ assert tq.quantize_transformer(pipe, target, mode = "auto", family = FAMILY) == scheme+ assert tq.transformer_is_quantised(model)+ return model+++def _flat_equal(a: dict, b: dict) -> list:+ fa, _ = C._flatten(a)+ fb, _ = C._flatten(b)+ assert set(fa) == set(fb)+ bad = []+ for k in fa:+ x, y = fa[k], fb[k]+ x = x.tensor() if isinstance(x, C._FlatRef) else x+ y = y.tensor() if isinstance(y, C._FlatRef) else y+ if x.dtype != y.dtype or tuple(x.shape) != tuple(y.shape):+ bad.append(k)+ continue+ if not torch.equal(+ x.contiguous().reshape(-1).view(torch.uint8),+ y.contiguous().reshape(-1).view(torch.uint8),+ ):+ bad.append(k)+ return bad+++def _views(model):+ return [types.SimpleNamespace(transformer = model)]+++def _bf16_target():+ return types.SimpleNamespace(device = "cuda", dtype = torch.bfloat16)+++def test_the_deferred_resident_write_reads_back_bit_identically(env, tmp_path, scheme, monkeypatch):+ from core.inference.diffusion_prequant import PrequantSource, load_prequantized_transformer++ base, model = _write_base(tmp_path)+ fam = _fam(base)+ fresh = _quantize(model, scheme, monkeypatch)++ pending = vid._save_runtime_quant_to_cache(+ fam, _views(fresh), base = str(base), source = str(base), defer = True+ )+ assert hasattr(pending, "start"), "the auto resident write must be deferred to after publish"+ # Nothing is published before start(): a load that fails before publish leaves no entry.+ assert C.lookup(fam, scheme, base_repo = str(base)) is None+ assert pending.start() and pending.wait(120)+ path = C.lookup(fam, scheme, base_repo = str(base))+ assert path is not None and Path(path).is_file()+ # Keyed exactly like the explicit save (same source fingerprint, scheme, filters, toolchain).+ key, _parts = C._key_for(fam, scheme, str(base), "transformer", base_id = str(base))+ assert Path(path).parent.name == key++ loaded = load_prequantized_transformer(+ diffusers.HunyuanVideo15Transformer3DModel,+ str(base),+ PrequantSource(kind = "path", location = path, filename = None),+ device = "cpu",+ dtype = torch.bfloat16,+ scheme = scheme,+ min_features = 512,+ component = "transformer",+ local_files_only = True,+ )+ assert loaded is not None+ assert _flat_equal(fresh.state_dict(), loaded.state_dict()) == []+ assert {n for n, _ in fresh.named_buffers()} == {n for n, _ in loaded.named_buffers()}+ g = torch.Generator().manual_seed(1)+ inputs = dict(+ hidden_states = torch.randn(1, 9, 1, 4, 4, generator = g).to(torch.bfloat16),+ timestep = torch.tensor([500.0]).to(torch.bfloat16),+ encoder_hidden_states = torch.randn(1, 8, 64, generator = g).to(torch.bfloat16),+ encoder_attention_mask = torch.ones(1, 8, dtype = torch.bool),+ encoder_hidden_states_2 = torch.randn(1, 4, 32, generator = g).to(torch.bfloat16),+ encoder_attention_mask_2 = torch.ones(1, 4, dtype = torch.bool),+ image_embeds = torch.zeros(1, 4, 32).to(torch.bfloat16),+ return_dict = False,+ )+ with torch.no_grad():+ want = fresh(**inputs)[0]+ got = loaded(**inputs)[0]+ assert torch.equal(want.contiguous().view(torch.uint8), got.contiguous().view(torch.uint8))+++def test_a_returning_auto_load_seeds_exactly_that_build(env, tmp_path, scheme, monkeypatch):+ base, model = _write_base(tmp_path)+ fam = _fam(base)+ monkeypatch.setattr(vid, "select_transformer_quant_scheme", lambda *a, **k: scheme)+ kw = dict(target = _bf16_target(), requested = "auto", base_repo = str(base), speed_mode = None)+ unset = dict(kw, requested = None)+ assert vid._video_auto_denoiser_scheme(fam, **kw) is None+ assert vid._video_auto_denoiser_scheme(fam, **unset) is None++ _quantize(model, scheme, monkeypatch)+ pending = vid._save_runtime_quant_to_cache(+ fam, _views(model), base = str(base), source = str(base), defer = True+ )+ assert pending.start() and pending.wait(120)+ assert vid._video_auto_denoiser_scheme(fam, **kw) == scheme+ # The default Studio request (transformer_quant omitted) replays the same entry.+ assert vid._video_auto_denoiser_scheme(fam, **unset) == scheme+ assert vid._video_auto_denoiser_scheme(fam, **dict(unset, requested = "none")) is None++ from core.inference.video_denoiser_prequant import denoiser_prequant_sources++ sources = denoiser_prequant_sources(fam, scheme, str(base))+ assert sources and all(vid._is_prequant_cache_source(s) for s in sources.values())+++def _stub_decision(+ monkeypatch,+ *,+ cached = ("int8",),+ pick = "int8",+):+ import core.inference.diffusion_prequant as pq+ import core.inference.video_denoiser_prequant as vdp++ calls = {"select": 0}++ def _sources(+ fam,+ scheme,+ base_repo,+ *,+ hf_token = None,+ ):+ if scheme not in cached:+ return None+ return {+ "transformer": types.SimpleNamespace(+ kind = "path", location = f"/c/{scheme}/model.safetensors"+ )+ }++ def _select(*a, **k):+ calls["select"] += 1+ return pick++ monkeypatch.setattr(vdp, "denoiser_prequant_sources", _sources)+ monkeypatch.setattr(+ vid, "_is_prequant_cache_source", lambda src: str(src.location).startswith("/c/")+ )+ monkeypatch.setattr(vid, "select_transformer_quant_scheme", _select)+ monkeypatch.setattr(pq, "restricted_prequant_load_supported", lambda scheme = None, *a: True)+ return calls+++def _decide(fam = None, **over):+ kw = dict(target = _bf16_target(), requested = "auto", base_repo = "org/base", speed_mode = None)+ kw.update(over)+ return vid._video_cached_resident_scheme(fam or _fam(Path("/nonexistent")), **kw)+++def test_auto_replays_only_its_own_pick(env, monkeypatch):+ _stub_decision(monkeypatch)+ assert _decide() == "int8"+ # An explicit int8 entry must not turn auto's fp8 pick into int8.+ _stub_decision(monkeypatch, cached = ("int8",), pick = "fp8")+ assert _decide() is None+ _stub_decision(monkeypatch, cached = ("int8",), pick = None)+ assert _decide() is None++[email protected]("requested", [None, "", " Auto ", "auto"])+def test_an_unset_request_replays_exactly_like_auto(env, monkeypatch, requested):+ """The Studio video page omits transformer_quant for the hardware ladder: unset is the default user load."""+ _stub_decision(monkeypatch)+ assert _decide(requested = requested) == "int8"+ _stub_decision(monkeypatch, cached = ("int8",), pick = "fp8")+ assert _decide(requested = requested) is None+ calls = _stub_decision(monkeypatch, cached = ())+ assert _decide(requested = requested) is None and calls["select"] == 0+++def test_the_load_maps_unset_to_auto_before_the_seed_and_the_write():+ """The write branch and the seed call see ``transformer_quant`` after load_pipeline turned unset into auto."""+ src = inspect.getsource(vid.VideoBackend.load_pipeline)+ mapped = src.index('transformer_quant = "off" if speed_off else TQ_AUTO')+ unset = src[mapped - 600 : mapped]+ assert "if transformer_quant is None or str(transformer_quant).strip().lower() in (" in unset+ seed = src.index(+ "denoiser_seed_scheme = _video_auto_denoiser_planned or _video_auto_denoiser_scheme("+ )+ write = src.index(+ "_save_runtime_quant_to_cache(", src.index("_save_runtime_quant_to_cache(") + 1+ )+ assert mapped < seed < write+ assert "requested = transformer_quant," in src[seed : seed + 300]+++def test_no_entry_asks_nothing_and_changes_nothing(env, monkeypatch):+ calls = _stub_decision(monkeypatch, cached = ())+ assert _decide() is None+ assert calls["select"] == 0+++def test_explicit_unset_speed_off_modular_moe_and_non_bf16_never_replay(env, monkeypatch):+ _stub_decision(monkeypatch)+ assert _decide(requested = "int8") is None+ assert _decide(requested = "off") is None+ assert _decide(requested = "none") is None+ assert _decide(speed_mode = "off") is None+ assert _decide(speed_mode = " OFF ") is None+ assert _decide(speed_mode = "off", requested = None) is None+ assert _decide(_fam(Path("/x"), modular_workflow = "fl2va")) is None+ assert _decide(_fam(Path("/x"), is_moe = True)) is None+ assert _decide(target = types.SimpleNamespace(device = "cuda", dtype = torch.float16)) is None+ assert _decide() == "int8"+++def test_a_hosted_or_operator_source_is_not_a_cache_replay(env, monkeypatch):+ import core.inference.video_denoiser_prequant as vdp++ _stub_decision(monkeypatch)+ monkeypatch.setattr(+ vdp,+ "denoiser_prequant_sources",+ lambda fam, scheme, base, hf_token = None: {+ "transformer": types.SimpleNamespace(kind = "repo", location = "org/hosted")+ },+ )+ assert _decide() is None+++def test_an_install_that_cannot_open_the_entry_keeps_the_dense_build(env, monkeypatch):+ import core.inference.diffusion_prequant as pq++ _stub_decision(monkeypatch)+ monkeypatch.setattr(pq, "restricted_prequant_load_supported", lambda scheme = None, *a: False)+ assert _decide() is None+++def test_kill_switches(env, monkeypatch):+ _stub_decision(monkeypatch)+ assert vid._resident_quant_cache_enabled()+ monkeypatch.setenv(vid.VIDEO_RESIDENT_QUANT_CACHE_ENV, "0")+ assert not vid._resident_quant_cache_enabled()+ assert _decide() is None+ monkeypatch.delenv(vid.VIDEO_RESIDENT_QUANT_CACHE_ENV)+ monkeypatch.setenv(C.ENV_MODE, "off")+ assert not vid._resident_quant_cache_enabled()+ assert _decide() is None+++def test_the_hosted_seed_wins_and_the_cache_is_only_a_fallback(env, monkeypatch):+ seen = []+ monkeypatch.setattr(vid, "_video_hosted_denoiser_scheme", lambda fam, **kw: "nvfp4")+ monkeypatch.setattr(+ vid, "_video_cached_resident_scheme", lambda fam, **kw: seen.append(1) or "int8"+ )+ kw = dict(target = None, requested = "auto", base_repo = "b", speed_mode = None)+ assert vid._video_auto_denoiser_scheme(None, **kw) == "nvfp4" and seen == []+ monkeypatch.setattr(vid, "_video_hosted_denoiser_scheme", lambda fam, **kw: None)+ assert vid._video_auto_denoiser_scheme(None, **kw) == "int8" and seen == [1]+++class _FakeDeviceTensor:+ """Reports a CUDA device; ``.to('cpu')`` hands back the real host tensor (a CPU-only host has no device)."""++ def __init__(self, t):+ self._t = t+ self.device = types.SimpleNamespace(type = "cuda")+ self.dtype = t.dtype+ self.shape = t.shape++ def numel(self):+ return self._t.numel()++ def element_size(self):+ return self._t.element_size()++ def detach(self):+ return self++ def to(self, device):+ assert device == "cpu"+ return self._t.clone()+++def _job(tmp_path, flat):+ return C._WriteJob(+ "k" * 32,+ tmp_path / "k",+ flat,+ {},+ {},+ sum(C._tensor_bytes(t) for t in flat.values()),+ 0.0,+ None,+ )+++def test_the_snapshot_moves_every_device_tensor_to_the_host(env, tmp_path, monkeypatch):+ a, b = torch.randn(8, 8).to(torch.bfloat16), torch.randint(-128, 127, (16,), dtype = torch.int8)+ job = _job(tmp_path, {"a": _FakeDeviceTensor(a), "b": b})+ monkeypatch.setattr(C, "_host_memory_bytes", lambda: (64 << 30, 32 << 30))+ assert C._defer_blocker(job, host_snapshot = True) is None+ assert all(getattr(t, "device", None).type == "cpu" for t in job.tensors())+ assert torch.equal(job.flat["a"], a) and torch.equal(job.flat["b"], b)+ # Without host_snapshot a device build is never deferred (the image path's rule, unchanged).+ job2 = _job(tmp_path, {"a": _FakeDeviceTensor(a)})+ assert "not the host" in C._defer_blocker(job2)+++def test_a_snapshot_that_does_not_fit_host_ram_is_refused(env, tmp_path, monkeypatch):+ a = torch.zeros(1024, 1024, dtype = torch.bfloat16)+ job = _job(tmp_path, {"a": _FakeDeviceTensor(a)})+ monkeypatch.setattr(C, "_host_memory_bytes", lambda: (8 << 30, 1 << 30))+ why = C._defer_blocker(job, host_snapshot = True)+ assert why is not None and "host RAM short" in why+ assert isinstance(job.flat["a"], _FakeDeviceTensor)+++def test_a_resident_write_that_cannot_defer_is_skipped_not_written_in_the_load(+ env, tmp_path, scheme, monkeypatch+):+ base, model = _write_base(tmp_path)+ fam = _fam(base)+ _quantize(model, scheme, monkeypatch)+ monkeypatch.setattr(+ C, "_defer_blocker", lambda job, **kw: "another background cache write is running"+ )+ assert (+ vid._save_runtime_quant_to_cache(+ fam, _views(model), base = str(base), source = str(base), defer = True+ )+ is None+ )+ assert C.lookup(fam, scheme, base_repo = str(base)) is None+ monkeypatch.setenv(C.ENV_ASYNC, "0")+ monkeypatch.setattr(C, "_defer_blocker", lambda job, **kw: f"{C.ENV_ASYNC}=0")+ out = vid._save_runtime_quant_to_cache(+ fam, _views(model), base = str(base), source = str(base), defer = True+ )+ assert isinstance(out, str) and C.lookup(fam, scheme, base_repo = str(base)) == out+++def test_a_discarded_write_releases_its_slot_and_leaves_no_entry(+ env, tmp_path, scheme, monkeypatch+):+ base, model = _write_base(tmp_path)+ fam = _fam(base)+ _quantize(model, scheme, monkeypatch)+ pending = vid._save_runtime_quant_to_cache(+ fam, _views(model), base = str(base), source = str(base), defer = True+ )+ owner = types.SimpleNamespace(_resident_quant_write = (7, pending))+ vid._discard_resident_quant_write(owner, 8) # another load's token: untouched+ assert owner._resident_quant_write is not None and not pending.done()+ vid._discard_resident_quant_write(owner, 7)+ assert owner._resident_quant_write is None and pending.done()+ assert C.lookup(fam, scheme, base_repo = str(base)) is None+ assert not any(not p.done() for p in C._BACKGROUND)+++def test_publish_starts_only_this_loads_write_and_teardown_finishes_it(env):+ events = []++ class _W:+ def start(self):+ events.append("start")+ return True++ def wait(self, timeout = None):+ events.append("wait")+ return True++ def cancel(self):+ events.append("cancel")++ def discard(self):+ events.append("discard")++ w = _W()+ owner = types.SimpleNamespace(_resident_quant_write = (1, w))+ pipe = types.SimpleNamespace()+ vid._start_resident_quant_write(owner, pipe, 2)+ assert events == [] and owner._resident_quant_write is not None+ vid._start_resident_quant_write(owner, pipe, 1)+ assert events == ["start"] and owner._resident_quant_write is None+ assert getattr(pipe, vid._RESIDENT_QUANT_WRITE_ATTR) is w+ vid._finish_resident_quant_write(pipe)+ assert events == ["start", "wait"] and getattr(pipe, vid._RESIDENT_QUANT_WRITE_ATTR) is None+++def test_the_load_writes_auto_resident_builds_deferred_and_keeps_explicit_writes_inline():+ src = inspect.getsource(vid.VideoBackend.load_pipeline)+ explicit_at = src.index("_save_runtime_quant_to_cache(")+ assert "elif transformer_quant_pinned is not None:" in src[explicit_at - 200 : explicit_at]+ assert "defer = True" not in src[explicit_at : explicit_at + 400]+ auto_at = src.index("_save_runtime_quant_to_cache(", explicit_at + 1)+ guard = src[auto_at - 700 : auto_at]+ for needle in (+ "not video_offload",+ "native_scheme is None",+ "normalize_transformer_quant(transformer_quant) == TQ_AUTO",+ "_resident_quant_cache_enabled()",+ ):+ assert needle in guard, needle+ assert "defer = True" in src[auto_at : auto_at + 400]+ assert "source = _base_local_dir or fetch_base," in src[auto_at : auto_at + 400]+ assert "_start_resident_quant_write(self, pipe, _load_token)" in src+ assert src.index("_start_resident_quant_write(") > src.index(+ '"video.loaded: %s (%s, %s, offload=%s'+ )+++def test_a_failed_load_discards_and_an_unload_finishes_the_write():+ backend_src = inspect.getsource(vid.VideoBackend)+ assert "_discard_resident_quant_write(self, token)" in backend_src+ assert '_finish_resident_quant_write(getattr(state, "pipe", None))' in backend_src+++def test_a_cache_seed_says_so_in_the_log_and_the_resolved_source():+ src = inspect.getsource(vid.VideoBackend.load_pipeline)+ assert "read the resident %s denoiser for %s from the local prequant" in src+ assert "read back from this machine's prequant cache" in src