Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

Studio video small host: price the encoder converted beside the cast misses, cast an oversized DiT from its source shards (#16700)

Daniel Han3c88a25fdf2b
diff --git a/studio/backend/core/inference/diffusion_cast_cache.py b/studio/backend/core/inference/diffusion_cast_cache.py
index 46e8e15180f..cedb3378567 100644
--- a/studio/backend/core/inference/diffusion_cast_cache.py
+++ b/studio/backend/core/inference/diffusion_cast_cache.py
@@ -772,110 +772,226 @@ def save(
logger, "info", f"{cand.component}: not cached (rescaled in place for fp16 decode)"
)
return None
- names = list(cand.names)
reader = _Reader(module, check)
why = _matches_source(cand, reader)
if why is not None:
_log(logger, "info", f"{cand.component}: not cached ({why})")
return None
- import torch
+ return _persist(cand, reader, logger = logger, t0 = t0)
+ except Exception as exc: # noqa: BLE001 - a cache write never fails a load
+ _log(
+ logger,
+ "warning",
+ f"{cand.component}: save failed: {type(exc).__name__}: {str(exc)[:200]}",
+ )
+ return None
+
- nbytes = 0
- for _n, dtype, shape in names:
- count = 1
- for d in shape:
- count *= int(d)
- nbytes += count * torch.empty((), dtype = dtype).element_size()
- budget = _cache.max_cache_bytes()
- if budget is not None and nbytes > budget:
- _log(logger, "info", f"{cand.component}: not cached (larger than the cache budget)")
+def _persist(
+ cand: CastCandidate,
+ reader: Any,
+ *,
+ logger: Any = None,
+ t0: float = 0.0,
+) -> Optional[str]:
+ """Write ``cand.names`` through ``reader`` into the cast cache; returns the artifact path or None. Raises."""
+ import torch
+
+ root = _cache.cache_root()
+ names = list(cand.names)
+ nbytes = 0
+ for _n, dtype, shape in names:
+ count = 1
+ for d in shape:
+ count *= int(d)
+ nbytes += count * torch.empty((), dtype = dtype).element_size()
+ budget = _cache.max_cache_bytes()
+ if budget is not None and nbytes > budget:
+ _log(logger, "info", f"{cand.component}: not cached (larger than the cache budget)")
+ return None
+ with _cache._writer_lock(root, cand.key) as locked:
+ if not locked:
return None
- with _cache._writer_lock(root, cand.key) as locked:
- if not locked:
+ existing = _cache._entry(cand.key)
+ if existing is not None:
+ with _cache._lock:
+ rejected = os.path.realpath(str(existing)) in _cache._REJECTED
+ if not rejected:
return None
- existing = _cache._entry(cand.key)
- if existing is not None:
- with _cache._lock:
- rejected = os.path.realpath(str(existing)) in _cache._REJECTED
- if not rejected:
- return None
- _cache._reclaim(root / cand.key)
- root.mkdir(parents = True, exist_ok = True)
- need = nbytes + max(_cache._DISK_MARGIN_BYTES, nbytes // 20)
+ _cache._reclaim(root / cand.key)
+ root.mkdir(parents = True, exist_ok = True)
+ need = nbytes + max(_cache._DISK_MARGIN_BYTES, nbytes // 20)
+ free = shutil.disk_usage(str(root)).free
+ if free < need:
+ _cache.evict(root = root, extra_free_bytes = need - free, keep = cand.key, logger = logger)
free = shutil.disk_usage(str(root)).free
- if free < need:
- _cache.evict(root = root, extra_free_bytes = need - free, keep = cand.key, logger = logger)
- free = shutil.disk_usage(str(root)).free
- if free < need:
- _log(
- logger,
- "info",
- f"{cand.component}: not cached (disk short: {free / 2**30:.1f} GiB free, "
- f"need {need / 2**30:.1f} GiB)",
- )
- return None
- source_bytes = sum(os.path.getsize(os.path.realpath(s)) for s in cand.shards)
- meta = {
- "key": cand.key,
- "component": cand.component,
- "class": cand.class_name,
- "dtype": cand.parts["dtype"],
- "source_gib": round(source_bytes / 2**30, 2),
- "torch_version": str(torch.__version__),
- }
- kdir = root / cand.key
- kdir.mkdir(parents = True, exist_ok = True)
- shutil.copyfile(cand.config_path, kdir / "config.json")
- fd, tmp = tempfile.mkstemp(
- dir = str(kdir), prefix = f".{_cache.FILE_NAME}.", suffix = _cache._TEMP_SUFFIX
+ if free < need:
+ _log(
+ logger,
+ "info",
+ f"{cand.component}: not cached (disk short: {free / 2**30:.1f} GiB free, "
+ f"need {need / 2**30:.1f} GiB)",
)
- os.close(fd)
- hasher = hashlib.sha256()
- try:
- size = _write(
- tmp,
- cand,
- {CAST_META_KEY: json.dumps(meta, sort_keys = True)},
- hasher,
- reader,
- )
- os.replace(tmp, kdir / _cache.FILE_NAME)
- tmp = None
- finally:
- if tmp is not None:
- try:
- os.unlink(tmp)
- except OSError:
- pass
- sha = hasher.hexdigest()
- _cache._record_verified(kdir, sha)
- manifest = {
- "key": cand.key,
- "bytes": size,
- "sha256": sha,
- "created": time.time(),
- "parts": cand.parts,
- }
- _cache._atomic_bytes(
- kdir / _cache.MANIFEST_NAME,
- json.dumps(manifest, indent = 1, default = str).encode("utf-8"),
+ return None
+ source_bytes = sum(os.path.getsize(os.path.realpath(s)) for s in cand.shards)
+ meta = {
+ "key": cand.key,
+ "component": cand.component,
+ "class": cand.class_name,
+ "dtype": cand.parts["dtype"],
+ "source_gib": round(source_bytes / 2**30, 2),
+ "torch_version": str(torch.__version__),
+ }
+ kdir = root / cand.key
+ kdir.mkdir(parents = True, exist_ok = True)
+ shutil.copyfile(cand.config_path, kdir / "config.json")
+ fd, tmp = tempfile.mkstemp(
+ dir = str(kdir), prefix = f".{_cache.FILE_NAME}.", suffix = _cache._TEMP_SUFFIX
+ )
+ os.close(fd)
+ hasher = hashlib.sha256()
+ try:
+ size = _write(
+ tmp,
+ cand,
+ {CAST_META_KEY: json.dumps(meta, sort_keys = True)},
+ hasher,
+ reader,
)
- _cache._touch(kdir)
- _log(
- logger,
- "info",
- f"{cand.component}: saved the {cand.parts['dtype']} cast ({cand.key}, {size / 2**30:.2f} GiB from "
- f"{source_bytes / 2**30:.2f} GiB of source shards, in {time.perf_counter() - t0:.1f} s)",
+ os.replace(tmp, kdir / _cache.FILE_NAME)
+ tmp = None
+ finally:
+ if tmp is not None:
+ try:
+ os.unlink(tmp)
+ except OSError:
+ pass
+ sha = hasher.hexdigest()
+ _cache._record_verified(kdir, sha)
+ manifest = {
+ "key": cand.key,
+ "bytes": size,
+ "sha256": sha,
+ "created": time.time(),
+ "parts": cand.parts,
+ }
+ _cache._atomic_bytes(
+ kdir / _cache.MANIFEST_NAME,
+ json.dumps(manifest, indent = 1, default = str).encode("utf-8"),
)
- _cache.evict(logger = logger, keep = cand.key)
- return str(kdir / _cache.FILE_NAME)
- except Exception as exc: # noqa: BLE001 - a cache write never fails a load
+ _cache._touch(kdir)
+ _log(
+ logger,
+ "info",
+ f"{cand.component}: saved the {cand.parts['dtype']} cast ({cand.key}, {size / 2**30:.2f} GiB from "
+ f"{source_bytes / 2**30:.2f} GiB of source shards, in {time.perf_counter() - t0:.1f} s)",
+ )
+ _cache.evict(logger = logger, keep = cand.key)
+ return str(kdir / _cache.FILE_NAME)
+
+
+class _ShardReader:
+ """``tensor(name)`` straight from the source shards, cast to the cached dtype: one tensor in RAM at a time."""
+
+ def __init__(self, cand: CastCandidate, where: dict, check: Any) -> None:
+ self.cand = cand
+ self.where = where
+ self.check = check
+ self.dtypes = {n: d for n, d, _s in cand.names}
+ self._open: dict = {}
+
+ def tensor(self, name: str) -> Any:
+ from safetensors import safe_open
+
+ shard = self.where[name]
+ fh = self._open.get(shard)
+ if fh is None:
+ for old in self._open.values():
+ old.__exit__(None, None, None)
+ self._open = {shard: safe_open(shard, framework = "pt").__enter__()}
+ fh = self._open[shard]
+ return fh.get_tensor(name).to(self.dtypes[name])
+
+ def close(self) -> None:
+ for fh in self._open.values():
+ try:
+ fh.__exit__(None, None, None)
+ except Exception: # noqa: BLE001
+ pass
+ self._open = {}
+
+
+def _source_names(cand: CastCandidate) -> tuple[Optional[list], dict]:
+ """``([(name, dtype, shape)], {name: shard})`` a ``from_pretrained(torch_dtype = cand.dtype)`` load would hold, from
+ an empty-weights instance of the class (diffusers only) and the shard headers; (None, {}) when they disagree."""
+ import torch
+
+ if cand.library != "diffusers":
+ return None, {}
+ from accelerate import init_empty_weights
+
+ config = cand.cls.load_config(os.path.dirname(cand.config_path))
+ with init_empty_weights():
+ model = cand.cls.from_config(config)
+ keep = tuple(str(m) for m in (getattr(cand.cls, "_keep_in_fp32_modules", None) or ()))
+ tied = _tied_keys(model)
+ where: dict = {}
+ tags: dict = {}
+ for shard in cand.shards:
+ header = _shard_header(shard) or {}
+ for key, entry in header.items():
+ if key != "__metadata__" and isinstance(entry, dict):
+ where[key] = shard
+ tags[key] = entry
+ names = []
+ for name, tensor in model.state_dict(keep_vars = True).items():
+ if name in tied and name not in where:
+ continue
+ entry = tags.get(name)
+ if entry is None or [int(d) for d in entry.get("shape", ())] != [
+ int(d) for d in tensor.shape
+ ]:
+ return None, {}
+ dtype = tensor.dtype
+ if dtype.is_floating_point:
+ dtype = torch.float32 if any(k in name.split(".") for k in keep) else cand.dtype
+ names.append((name, dtype, tuple(int(d) for d in tensor.shape)))
+ return names, where
+
+
+def save_from_source(
+ cand: CastCandidate,
+ *,
+ logger: Any = None,
+ check: Optional[Any] = None,
+) -> Optional[str]:
+ """Write ``cand``'s cast straight from its source shards, one tensor in RAM at a time, for a component whose
+ dense load would not fit host RAM; the load then maps the cached file. Returns the path or None. Never raises."""
+ check = check or (lambda: None)
+ t0 = time.perf_counter()
+ reader = None
+ try:
+ names, where = _source_names(cand)
+ if not names:
+ _log(
+ logger,
+ "info",
+ f"{cand.component}: not cached from source (its tensors do not match the class)",
+ )
+ return None
+ cand.names = names
+ reader = _ShardReader(cand, where, check)
+ return _persist(cand, reader, logger = logger, t0 = t0)
+ except Exception as exc: # noqa: BLE001 - the dense load still runs
_log(
logger,
"warning",
- f"{cand.component}: save failed: {type(exc).__name__}: {str(exc)[:200]}",
+ f"{cand.component}: source cast failed: {type(exc).__name__}: {str(exc)[:200]}",
)
return None
+ finally:
+ if reader is not None:
+ reader.close()
class CastWrites:
diff --git a/studio/backend/core/inference/video.py b/studio/backend/core/inference/video.py
index ac2d1cdc055..2495f4b3999 100644
--- a/studio/backend/core/inference/video.py
+++ b/studio/backend/core/inference/video.py
@@ -3452,8 +3452,18 @@ def _read_cast_cache(
if not cands:
return []
if device is not None:
- from .video_small_host import stage_misses
- stage_misses(cands, device, logger = logger, cancel_check = cancel_check)
+ from .video_small_host import converted_others_mib, stage_misses
+ others = converted_others_mib(
+ source,
+ cands,
+ torch_dtype = pipe_kwargs.get("torch_dtype"),
+ cache_dir = None
+ if source and os.path.isdir(str(source))
+ else pipe_kwargs.get("cache_dir"),
+ revision = revision,
+ skip = tuple(pipe_kwargs),
+ )
+ stage_misses(cands, device, logger = logger, cancel_check = cancel_check, others_mib = others)
hits = read_all(cands, logger = logger, cancel_check = cancel_check)
pipe_kwargs.update(hits)
if hits:
diff --git a/studio/backend/core/inference/video_small_host.py b/studio/backend/core/inference/video_small_host.py
index 6045d2539cf..bfa2ae9c180 100644
--- a/studio/backend/core/inference/video_small_host.py
+++ b/studio/backend/core/inference/video_small_host.py
@@ -17,7 +17,7 @@ import re
import time
from dataclasses import dataclass, field
from pathlib import Path
-from typing import Any, Callable, Optional
+from typing import Any, Callable, Iterable, Optional
SMALL_HOST_ENV = "UNSLOTH_VIDEO_SMALL_HOST"
_MIB = 1024 * 1024
@@ -83,6 +83,10 @@ class StagePlan:
available_mib: Optional[int] = None
reserve_mib: int = 0
skipped: dict = field(default_factory = dict)
+ # component -> MiB of it placed on the card when it is staged split across the card and host RAM
+ split: dict = field(default_factory = dict)
+ # MiB the same load converts in host RAM outside the cast cache (a bf16 encoder cast to fp16)
+ others_mib: int = 0
def plan_stage(
@@ -93,48 +97,83 @@ def plan_stage(
device_free_mib: Optional[int],
forced: bool = False,
disabled: bool = False,
+ others_mib: int = 0,
+ source_castable: Optional[Iterable] = None,
) -> StagePlan:
- """``misses``: component -> loaded MiB. Stage largest first (each must fit the card) until the rest fits host RAM."""
+ """``misses``: component -> loaded MiB. Stage largest first until the rest fits host RAM; ``others_mib`` is what
+ the same load converts in host RAM besides them (it cannot be staged, so it is priced, not moved). A miss larger
+ than the card is staged split: the card holds what fits, host RAM the rest (when that rest fits). Only a
+ component in ``source_castable`` (None = every one) can be split, since its cast is written from the source
+ shards; any other one too big for the card is skipped and the planner moves on."""
if disabled:
return StagePlan(False, f"{SMALL_HOST_ENV}=0")
sizes = {k: int(v) for k, v in misses.items() if v is not None}
dense = sum(sizes.values())
+ others = max(0, int(others_mib or 0))
if not sizes:
- return StagePlan(False, "no cast-cache miss to convert", dense_mib = 0)
+ return StagePlan(False, "no cast-cache miss to convert", dense_mib = 0, others_mib = others)
if available_mib is None and not forced:
- return StagePlan(False, "host RAM unreadable", dense_mib = dense)
- fits = available_mib is not None and dense + reserve_mib <= int(available_mib)
+ return StagePlan(False, "host RAM unreadable", dense_mib = dense, others_mib = others)
+ beside = f" + {others} MiB of other converted components" if others else ""
+ fits = available_mib is not None and dense + others + reserve_mib <= int(available_mib)
if fits and not forced:
return StagePlan(
False,
- f"dense conversion ({dense} MiB) fits {available_mib} MiB available host RAM",
+ f"dense conversion ({dense} MiB{beside}) fits {available_mib} MiB available host RAM",
dense_mib = dense,
available_mib = available_mib,
reserve_mib = reserve_mib,
+ others_mib = others,
)
stage = []
skipped = {}
+ split = {}
left = dense
for name, mib in sorted(sizes.items(), key = lambda kv: (-kv[1], kv[0])):
- if not forced and available_mib is not None and left + reserve_mib <= int(available_mib):
+ if (
+ not forced
+ and available_mib is not None
+ and left + others + reserve_mib <= int(available_mib)
+ ):
break
if mib < _STAGE_MIN_MIB:
skipped[name] = "small"
continue
- if device_free_mib is None or mib + _DEVICE_MARGIN_MIB > int(device_free_mib):
- skipped[name] = (
- f"needs {mib + _DEVICE_MARGIN_MIB} MiB of the card, {device_free_mib} MiB free"
- )
+ card = None if device_free_mib is None else int(device_free_mib) - _DEVICE_MARGIN_MIB
+ if card is not None and mib <= card:
+ stage.append(name)
+ left -= mib
+ continue
+ # Split: the card's share plus a host share that fits beside the rest of the load's conversions.
+ host_room = (
+ None
+ if available_mib is None
+ else int(available_mib) - reserve_mib - others - (left - mib)
+ )
+ if (
+ card is not None
+ and (source_castable is None or name in source_castable)
+ and card >= _STAGE_MIN_MIB
+ and host_room is not None
+ and mib - card <= host_room
+ ):
+ stage.append(name)
+ split[name] = card
+ left -= mib
continue
- stage.append(name)
- left -= mib
+ skipped[name] = (
+ f"needs {mib + _DEVICE_MARGIN_MIB} MiB of the card, {device_free_mib} MiB free, and its host share "
+ f"does not fit {host_room} MiB"
+ )
why = (
f"{SMALL_HOST_ENV}=1"
if forced
- else f"dense conversion needs ~{dense} MiB + {reserve_mib} MiB reserve of host RAM, "
+ else f"dense conversion needs ~{dense} MiB{beside} + {reserve_mib} MiB reserve of host RAM, "
f"{available_mib} MiB available"
)
- return StagePlan(bool(stage), why, tuple(stage), dense, available_mib, reserve_mib, skipped)
+ return StagePlan(
+ bool(stage), why, tuple(stage), dense, available_mib, reserve_mib, skipped, split, others
+ )
def _component_dir(cand: Any) -> str:
@@ -206,12 +245,66 @@ def _device_free_mib(device: Any) -> Optional[int]:
return None
+def converted_others_mib(
+ source: Any,
+ cands: list,
+ *,
+ torch_dtype: Any,
+ cache_dir: Optional[str] = None,
+ revision: Optional[str] = None,
+ skip: Any = (),
+) -> int:
+ """MiB the pipeline load converts in host RAM outside the cast cache: half-precision shards of a component that
+ is not a cast candidate but loads at the other half dtype (bf16 -> fp16 on a float16 card), which become anonymous
+ memory beside the cast-cache misses. 0 when unreadable."""
+ try:
+ import torch
+
+ from . import diffusion_cast_cache as cc
+
+ index = cc._read_json(
+ cc._cache._local_file(str(source), "model_index.json", cache_dir, revision)
+ )
+ if not isinstance(index, dict):
+ return 0
+ names = {c.component for c in cands}
+ skip = set(skip or ())
+ total = 0
+ for component, spec in index.items():
+ if component.startswith("_") or component in skip or component in names:
+ continue
+ if (
+ not isinstance(spec, (list, tuple))
+ or len(spec) != 2
+ or spec[0] not in cc._LIBRARIES
+ ):
+ continue
+ dtype = cc._component_dtype(torch_dtype, component)
+ if not cc._is_half(dtype):
+ continue
+ want = "F16" if dtype == torch.float16 else "BF16"
+ for shard in (
+ cc._shards_of(str(source), component, cc._STEMS[spec[0]], cache_dir, revision) or ()
+ ):
+ header = cc._shard_header(shard) or {}
+ for name, entry in header.items():
+ if name == "__metadata__" or not isinstance(entry, dict):
+ continue
+ if entry.get("dtype") in ("F16", "BF16") and entry.get("dtype") != want:
+ begin, end = (int(v) for v in entry.get("data_offsets") or (0, 0))
+ total += end - begin
+ return -(-total // _MIB)
+ except Exception: # noqa: BLE001 - unpriced, as before
+ return 0
+
+
def stage_misses(
cands: list,
device: Any,
*,
logger: Any = None,
cancel_check: Optional[Callable[[], None]] = None,
+ others_mib: int = 0,
) -> list:
"""Stage the misses of ``cands`` a dense load would not fit; returns the staged components. Never raises except
a cancel."""
@@ -235,6 +328,10 @@ def stage_misses(
device_free_mib = _device_free_mib(device),
forced = stage_forced(),
disabled = stage_disabled(),
+ others_mib = others_mib,
+ source_castable = {
+ c.component for c in cands if getattr(c, "library", None) == "diffusers"
+ },
)
if logger is not None and (plan.engaged or plan.skipped or misses):
logger.info(
@@ -246,11 +343,47 @@ def stage_misses(
)
done = []
by_name = {c.component: c for c in cands}
+ from .diffusion_cast_cache import save_from_source
+
for name in plan.stage:
if cancel_check is not None:
cancel_check()
+ if plan.split.get(name):
+ # A card / host split through accelerate's cpu offload brings its host share back as meta tensors
+ # (nothing to write); the source shards cast one tensor at a time instead, card untouched. If that
+ # fails the component is left to the normal load.
+ t0 = time.perf_counter()
+ if save_from_source(by_name[name], logger = logger, check = cancel_check) is not None:
+ done.append(name)
+ if logger is not None:
+ logger.info(
+ "video.small_host: %s cast written from its source shards in %.1f s (host RAM: one "
+ "tensor at a time); the load maps it",
+ name,
+ time.perf_counter() - t0,
+ )
+ continue
if stage_on_device(by_name[name], device, logger = logger, cancel_check = cancel_check):
done.append(name)
+ if plan.skipped and not stage_disabled():
+ # Neither the card nor a card / host split takes it: write its cast from the source shards one tensor at a
+ # time, so the load maps the cached file instead of converting the whole component in host RAM. This runs
+ # even when nothing else was staged (a lone DiT too big for the card is the case it is for).
+ for name, why in plan.skipped.items():
+ if name not in by_name or why == "small":
+ continue
+ if cancel_check is not None:
+ cancel_check()
+ t0 = time.perf_counter()
+ if save_from_source(by_name[name], logger = logger, check = cancel_check) is not None:
+ done.append(name)
+ if logger is not None:
+ logger.info(
+ "video.small_host: %s cast written from its source shards in %.1f s (host RAM: one "
+ "tensor at a time); the load maps it",
+ name,
+ time.perf_counter() - t0,
+ )
return done
except Exception as exc: # noqa: BLE001 - the dense load still runs
if cancel_check is not None:
diff --git a/studio/backend/tests/test_diffusion_cast_cache.py b/studio/backend/tests/test_diffusion_cast_cache.py
index 43efb39fa5f..deca08e4601 100644
--- a/studio/backend/tests/test_diffusion_cast_cache.py
+++ b/studio/backend/tests/test_diffusion_cast_cache.py
@@ -291,3 +291,70 @@ def test_resident_device_module_is_read_back_bit_identical(pipe_dir):
hit = CC.read(cand)
for k, v in hit.state_dict().items():
assert v.dtype == reference[k].dtype and torch.equal(_bits(v), _bits(reference[k])), k
+
+
+def test_source_cast_matches_the_live_module_cast_bit_for_bit(pipe_dir, tmp_path, monkeypatch):
+ """A component too big for host RAM is cast from its source shards one tensor at a time: the cached file is the
+ same file the live-module write produces (same key, same bytes), and reads back as the dense load."""
+ cand = _by_name(CC.candidates(pipe_dir, torch_dtype = FP16_DEFAULT))["transformer"]
+ path = CC.save_from_source(cand)
+ assert path is not None and C.path_in_cache(path)
+ from_source = Path(path).read_bytes()
+ hit = CC.read(_by_name(CC.candidates(pipe_dir, torch_dtype = FP16_DEFAULT))["transformer"])
+ assert hit is not None
+ dense = _dense(cand)
+ _identical(dense, hit)
+ # the live-module write of the same component, into a second cache root: identical bytes (bar the timestamp-free
+ # metadata, which is the same too)
+ monkeypatch.setenv(C.ENV_DIR, str(tmp_path / "cache_live"))
+ live = _by_name(CC.candidates(pipe_dir, torch_dtype = FP16_DEFAULT))["transformer"]
+ live_path = _save(live, dense)
+ assert live_path is not None
+ assert Path(live_path).read_bytes() == from_source
+
+
+def test_source_cast_refuses_a_class_its_shards_do_not_match(pipe_dir):
+ cand = _by_name(CC.candidates(pipe_dir, torch_dtype = FP16_DEFAULT))["transformer"]
+ cand.cls = (
+ diffusers.UNet2DModel
+ ) # a different architecture: names / shapes disagree with the shards
+ cand.config_path = str(pipe_dir / "transformer" / "config.json")
+ assert CC.save_from_source(cand) is None
+ # transformers components are not cast from source (their dense load is the small one)
+ te = _by_name(CC.candidates(pipe_dir, torch_dtype = FP16_DEFAULT))["text_encoder"]
+ assert CC.save_from_source(te) is None
+
+
+def test_source_cast_of_a_hunyuanvideo15_denoiser_reads_back_as_the_dense_load(tmp_path):
+ """The HunyuanVideo-1.5 case: its fp32 DiT (31 GB) does not fit a 31 GB host beside the converted encoder."""
+ torch.manual_seed(0)
+ model = diffusers.HunyuanVideo15Transformer3DModel(
+ in_channels = 9,
+ out_channels = 4,
+ num_attention_heads = 2,
+ attention_head_dim = 32,
+ num_layers = 1,
+ num_refiner_layers = 1,
+ mlp_ratio = 2.0,
+ text_embed_dim = 64,
+ text_embed_2_dim = 32,
+ image_embed_dim = 32,
+ rope_axes_dim = (8, 12, 12),
+ target_size = 64,
+ ).to(torch.float32)
+ root = tmp_path / "HunyuanVideo-1.5-tiny"
+ model.save_pretrained(root / "transformer", max_shard_size = "200KB")
+ (root / "model_index.json").write_text(
+ json.dumps(
+ {
+ "_class_name": "HunyuanVideo15Pipeline",
+ "transformer": ["diffusers", "HunyuanVideo15Transformer3DModel"],
+ }
+ )
+ )
+ cand = _by_name(CC.candidates(root, torch_dtype = FP16_DEFAULT))["transformer"]
+ assert len(cand.shards) > 1
+ assert CC.save_from_source(cand) is not None
+ hit = CC.read(_by_name(CC.candidates(root, torch_dtype = FP16_DEFAULT))["transformer"])
+ assert hit is not None
+ _identical(_dense(cand), hit)
diff --git a/studio/backend/tests/test_video_small_host.py b/studio/backend/tests/test_video_small_host.py
index 390bbff94bb..ba61e848c9e 100644
--- a/studio/backend/tests/test_video_small_host.py
+++ b/studio/backend/tests/test_video_small_host.py
@@ -60,11 +60,19 @@ def test_stages_largest_first_until_the_rest_fits():
assert plan.stage == ("text_encoder", "transformer")
-def test_card_too_small_for_the_encoder_keeps_the_dense_load():
+def test_card_too_small_for_the_encoder_stages_it_split():
plan = VS.plan_stage(WAN13, available_mib = 10100, reserve_mib = 3072, device_free_mib = 11000)
- # the encoder does not fit the card; the DiT alone does but cannot rescue the load: staged anyway, logged
+ # the encoder does not fit the card: the card takes what fits, host RAM the 2.5 GB rest, and the DiT then
+ # converts densely beside it
+ assert plan.stage == ("text_encoder",)
+ assert plan.split == {"text_encoder": 11000 - VS._DEVICE_MARGIN_MIB}
+
+
+def test_a_split_whose_host_share_does_not_fit_keeps_the_dense_load():
+ plan = VS.plan_stage(WAN13, available_mib = 10100, reserve_mib = 3072, device_free_mib = 4000)
+ # 9.5 GB of the encoder would stay in host RAM beside the DiT: no split; the DiT alone is staged, logged
assert "text_encoder" in plan.skipped
- assert plan.stage == ("transformer",)
+ assert plan.stage == ("transformer",) and plan.split == {}
def test_kill_switch_and_force(monkeypatch):
@@ -265,3 +273,180 @@ def test_non_cuda_device_never_stages(pipe_dir, monkeypatch):
calls = _wire(monkeypatch, available = 10)
assert VS.stage_misses(list(_cands(pipe_dir).values()), "cpu") == []
assert calls == []
+
+
+# HunyuanVideo-1.5 on a Kaggle T4 (31 GB host, 15 GB card): the fp32 DiT converts to fp16 (18.4 GiB, a cast-cache
+# miss) while the bf16 Qwen2.5-VL-7B converts to fp16 beside it (15.6 GiB, not a cast candidate). Pricing the DiT
+# alone let every load through and the host ran out of RAM (rc -9).
+HV15_T4 = {"transformer": 18827}
+HV15_TE_MIB = 15600
+
+
+def test_the_encoder_converted_beside_the_dit_is_priced():
+ old = VS.plan_stage(HV15_T4, available_mib = 29238, reserve_mib = 3174, device_free_mib = 14600)
+ assert not old.engaged # the DiT alone "fits"
+ plan = VS.plan_stage(
+ HV15_T4,
+ available_mib = 29238,
+ reserve_mib = 3174,
+ device_free_mib = 14600,
+ others_mib = HV15_TE_MIB,
+ )
+ assert plan.engaged and plan.stage == ("transformer",)
+ # 15 GB card: split, 13.8 GB on the card and 4.9 GB in host RAM beside the encoder's 15.6
+ assert plan.split == {"transformer": 14600 - VS._DEVICE_MARGIN_MIB}
+ assert plan.others_mib == HV15_TE_MIB and "other converted components" in plan.reason
+
+
+def test_others_that_fit_change_nothing():
+ plan = VS.plan_stage(
+ HV15_T4,
+ available_mib = 60000,
+ reserve_mib = 6000,
+ device_free_mib = 14600,
+ others_mib = HV15_TE_MIB,
+ )
+ assert not plan.engaged and "fits" in plan.reason
+
+
+def test_converted_others_counts_only_a_half_dtype_change(pipe_dir):
+ import safetensors.torch as st
+
+ te = pipe_dir / "text_encoder"
+ for f in te.glob("*.safetensors"):
+ f.unlink()
+ st.save_file({"w": torch.zeros(1024, 512, dtype = torch.bfloat16)}, str(te / "model.safetensors"))
+ cands = [c for c in CC.candidates(pipe_dir, torch_dtype = FP16_DEFAULT)]
+ assert {c.component for c in cands} == {
+ "transformer"
+ } # the bf16 encoder is not a cast candidate
+ fp16 = VS.converted_others_mib(pipe_dir, cands, torch_dtype = FP16_DEFAULT)
+ assert fp16 == 1 # 1 MiB of bf16 converted to fp16 in host RAM
+ bf16 = VS.converted_others_mib(
+ pipe_dir, cands, torch_dtype = {"default": torch.bfloat16, "vae": torch.float32}
+ )
+ assert bf16 == 0 # same dtype: memory-mapped, nothing converted
+ assert (
+ VS.converted_others_mib(pipe_dir, cands, torch_dtype = FP16_DEFAULT, skip = ("text_encoder",))
+ == 0
+ )
+
+
+def test_device_load_kwargs_put_the_whole_component_on_the_card():
+ import types
+ from unittest import mock
+
+ cand = types.SimpleNamespace()
+ with mock.patch.object(
+ CC, "_load_kwargs", lambda c: {"torch_dtype": torch.float16, "output_loading_info": True}
+ ):
+ whole = VS._device_load_kwargs(cand, "cuda:0")
+ assert whole["device_map"] == {"": "cuda:0"} and "max_memory" not in whole
+ assert "output_loading_info" not in whole
+
+
+def test_a_dit_neither_the_card_nor_a_split_takes_is_cast_from_its_shards(pipe_dir, monkeypatch):
+ """The HunyuanVideo-1.5 T4 case: the fp16 DiT is bigger than the card and its host share does not fit, so its cast
+ is written from the source shards one tensor at a time and the load maps it (no dense conversion in host RAM)."""
+ from core.inference import video as V
+
+ sizes = {"text_encoder": 100, "transformer": 20000}
+ monkeypatch.setattr(VS, "loaded_mib", lambda cand: sizes[cand.component])
+ calls = _wire(monkeypatch, available = 3000, total = 13000)
+ pipe_kwargs = {"torch_dtype": dict(FP16_DEFAULT), "cache_dir": None, "local_files_only": True}
+ written = []
+ real = CC.save_from_source
+ monkeypatch.setattr(
+ CC,
+ "save_from_source",
+ lambda cand, **kw: written.append(cand.component) or real(cand, **kw),
+ )
+ missed = V._read_cast_cache(
+ pipe_kwargs, source = str(pipe_dir), revision = None, base = None, device = "cuda"
+ )
+ assert written == ["transformer"] and ("transformer", "cuda") not in calls
+ assert "transformer" in pipe_kwargs and "transformer" not in [c.component for c in missed]
+ _identical(pipe_kwargs["transformer"], _dense(_cands(pipe_dir)["transformer"]))
+
+
+def test_source_cast_respects_the_kill_switch(pipe_dir, monkeypatch):
+ from core.inference import video as V
+
+ monkeypatch.setenv(VS.SMALL_HOST_ENV, "0")
+ sizes = {"text_encoder": 100, "transformer": 20000}
+ monkeypatch.setattr(VS, "loaded_mib", lambda cand: sizes[cand.component])
+ _wire(monkeypatch, available = 3000, total = 13000)
+ pipe_kwargs = {"torch_dtype": dict(FP16_DEFAULT), "cache_dir": None, "local_files_only": True}
+ V._read_cast_cache(pipe_kwargs, source = str(pipe_dir), revision = None, base = None, device = "cuda")
+ assert "transformer" not in pipe_kwargs
+
+
+def test_a_split_plan_casts_from_the_source_shards_instead(pipe_dir, monkeypatch):
+ """A DiT a little over the card was split card / host through accelerate's cpu offload, whose host share loads
+ as meta tensors (nothing to write, on the Kaggle T4 the dense load then ran out of host RAM): it is cast from
+ its source shards instead, without touching the card."""
+ from core.inference import video as V
+
+ sizes = {"text_encoder": 100, "transformer": 16000}
+ monkeypatch.setattr(VS, "loaded_mib", lambda cand: sizes[cand.component])
+ calls = _wire(monkeypatch, available = 18000, total = 20000)
+ written = []
+ real = CC.save_from_source
+ monkeypatch.setattr(
+ CC,
+ "save_from_source",
+ lambda cand, **kw: written.append(cand.component) or real(cand, **kw),
+ )
+ pipe_kwargs = {"torch_dtype": dict(FP16_DEFAULT), "cache_dir": None, "local_files_only": True}
+ V._read_cast_cache(pipe_kwargs, source = str(pipe_dir), revision = None, base = None, device = "cuda")
+ assert written == ["transformer"] and ("transformer", "cuda") not in calls
+ _identical(pipe_kwargs["transformer"], _dense(_cands(pipe_dir)["transformer"]))
+
+
+def test_a_split_plan_whose_source_cast_fails_is_left_to_the_normal_load(pipe_dir, monkeypatch):
+ """No accelerate card / host split as a fallback (its host share comes back as meta tensors): a split-planned
+ component whose source cast fails is not staged at all, so the pipeline loads it as on main."""
+ from core.inference import video as V
+
+ sizes = {"text_encoder": 100, "transformer": 16000}
+ monkeypatch.setattr(VS, "loaded_mib", lambda cand: sizes[cand.component])
+ calls = _wire(monkeypatch, available = 18000, total = 20000)
+ monkeypatch.setattr(CC, "save_from_source", lambda cand, **kw: None)
+ pipe_kwargs = {"torch_dtype": dict(FP16_DEFAULT), "cache_dir": None, "local_files_only": True}
+ V._read_cast_cache(pipe_kwargs, source = str(pipe_dir), revision = None, base = None, device = "cuda")
+ assert ("transformer", "cuda") not in calls and "transformer" not in pipe_kwargs
+
+
+def test_a_lone_dit_too_big_for_the_card_is_still_cast_from_its_shards(pipe_dir, monkeypatch):
+ """Nothing else staged: the plan is not engaged, but a skipped component too big for the card and the host still
+ takes the source cast, so the load maps it instead of converting it densely."""
+ from core.inference import video as V
+
+ sizes = {"text_encoder": 100, "transformer": 20000}
+ monkeypatch.setattr(VS, "loaded_mib", lambda cand: sizes[cand.component])
+ _wire(monkeypatch, available = 3000, total = 13000)
+ monkeypatch.setattr(VS, "_STAGE_MIN_MIB", 512)
+ written = []
+ real = CC.save_from_source
+ monkeypatch.setattr(
+ CC,
+ "save_from_source",
+ lambda cand, **kw: written.append(cand.component) or real(cand, **kw),
+ )
+ pipe_kwargs = {"torch_dtype": dict(FP16_DEFAULT), "cache_dir": None, "local_files_only": True}
+ V._read_cast_cache(pipe_kwargs, source = str(pipe_dir), revision = None, base = None, device = "cuda")
+ assert written == ["transformer"] and "transformer" in pipe_kwargs
+
+
+def test_a_transformers_encoder_over_the_card_is_not_split_planned():
+ """Its cast cannot be written from the source shards (diffusers components only), so it is skipped and the
+ planner goes on to stage the DiT, as before the split path existed."""
+ plan = VS.plan_stage(
+ WAN13,
+ available_mib = 16000,
+ reserve_mib = 3000,
+ device_free_mib = 11000,
+ source_castable = {"transformer"},
+ )
+ assert "text_encoder" in plan.skipped and plan.split == {}
+ assert plan.stage == ("transformer",)