Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
Studio video small host: price the encoder converted beside the cast misses, cast an oversized DiT from its source shards (#16700)
Daniel Han3c88a25fdf2b
diff --git a/studio/backend/core/inference/diffusion_cast_cache.py b/studio/backend/core/inference/diffusion_cast_cache.pyindex 46e8e15180f..cedb3378567 100644--- a/studio/backend/core/inference/diffusion_cast_cache.py+++ b/studio/backend/core/inference/diffusion_cast_cache.py@@ -772,110 +772,226 @@ def save(logger, "info", f"{cand.component}: not cached (rescaled in place for fp16 decode)")return None- names = list(cand.names)reader = _Reader(module, check)why = _matches_source(cand, reader)if why is not None:_log(logger, "info", f"{cand.component}: not cached ({why})")return None- import torch+ return _persist(cand, reader, logger = logger, t0 = t0)+ except Exception as exc: # noqa: BLE001 - a cache write never fails a load+ _log(+ logger,+ "warning",+ f"{cand.component}: save failed: {type(exc).__name__}: {str(exc)[:200]}",+ )+ return None+- nbytes = 0- for _n, dtype, shape in names:- count = 1- for d in shape:- count *= int(d)- nbytes += count * torch.empty((), dtype = dtype).element_size()- budget = _cache.max_cache_bytes()- if budget is not None and nbytes > budget:- _log(logger, "info", f"{cand.component}: not cached (larger than the cache budget)")+def _persist(+ cand: CastCandidate,+ reader: Any,+ *,+ logger: Any = None,+ t0: float = 0.0,+) -> Optional[str]:+ """Write ``cand.names`` through ``reader`` into the cast cache; returns the artifact path or None. Raises."""+ import torch++ root = _cache.cache_root()+ names = list(cand.names)+ nbytes = 0+ for _n, dtype, shape in names:+ count = 1+ for d in shape:+ count *= int(d)+ nbytes += count * torch.empty((), dtype = dtype).element_size()+ budget = _cache.max_cache_bytes()+ if budget is not None and nbytes > budget:+ _log(logger, "info", f"{cand.component}: not cached (larger than the cache budget)")+ return None+ with _cache._writer_lock(root, cand.key) as locked:+ if not locked:return None- with _cache._writer_lock(root, cand.key) as locked:- if not locked:+ existing = _cache._entry(cand.key)+ if existing is not None:+ with _cache._lock:+ rejected = os.path.realpath(str(existing)) in _cache._REJECTED+ if not rejected:return None- existing = _cache._entry(cand.key)- if existing is not None:- with _cache._lock:- rejected = os.path.realpath(str(existing)) in _cache._REJECTED- if not rejected:- return None- _cache._reclaim(root / cand.key)- root.mkdir(parents = True, exist_ok = True)- need = nbytes + max(_cache._DISK_MARGIN_BYTES, nbytes // 20)+ _cache._reclaim(root / cand.key)+ root.mkdir(parents = True, exist_ok = True)+ need = nbytes + max(_cache._DISK_MARGIN_BYTES, nbytes // 20)+ free = shutil.disk_usage(str(root)).free+ if free < need:+ _cache.evict(root = root, extra_free_bytes = need - free, keep = cand.key, logger = logger)free = shutil.disk_usage(str(root)).free- if free < need:- _cache.evict(root = root, extra_free_bytes = need - free, keep = cand.key, logger = logger)- free = shutil.disk_usage(str(root)).free- if free < need:- _log(- logger,- "info",- f"{cand.component}: not cached (disk short: {free / 2**30:.1f} GiB free, "- f"need {need / 2**30:.1f} GiB)",- )- return None- source_bytes = sum(os.path.getsize(os.path.realpath(s)) for s in cand.shards)- meta = {- "key": cand.key,- "component": cand.component,- "class": cand.class_name,- "dtype": cand.parts["dtype"],- "source_gib": round(source_bytes / 2**30, 2),- "torch_version": str(torch.__version__),- }- kdir = root / cand.key- kdir.mkdir(parents = True, exist_ok = True)- shutil.copyfile(cand.config_path, kdir / "config.json")- fd, tmp = tempfile.mkstemp(- dir = str(kdir), prefix = f".{_cache.FILE_NAME}.", suffix = _cache._TEMP_SUFFIX+ if free < need:+ _log(+ logger,+ "info",+ f"{cand.component}: not cached (disk short: {free / 2**30:.1f} GiB free, "+ f"need {need / 2**30:.1f} GiB)",)- os.close(fd)- hasher = hashlib.sha256()- try:- size = _write(- tmp,- cand,- {CAST_META_KEY: json.dumps(meta, sort_keys = True)},- hasher,- reader,- )- os.replace(tmp, kdir / _cache.FILE_NAME)- tmp = None- finally:- if tmp is not None:- try:- os.unlink(tmp)- except OSError:- pass- sha = hasher.hexdigest()- _cache._record_verified(kdir, sha)- manifest = {- "key": cand.key,- "bytes": size,- "sha256": sha,- "created": time.time(),- "parts": cand.parts,- }- _cache._atomic_bytes(- kdir / _cache.MANIFEST_NAME,- json.dumps(manifest, indent = 1, default = str).encode("utf-8"),+ return None+ source_bytes = sum(os.path.getsize(os.path.realpath(s)) for s in cand.shards)+ meta = {+ "key": cand.key,+ "component": cand.component,+ "class": cand.class_name,+ "dtype": cand.parts["dtype"],+ "source_gib": round(source_bytes / 2**30, 2),+ "torch_version": str(torch.__version__),+ }+ kdir = root / cand.key+ kdir.mkdir(parents = True, exist_ok = True)+ shutil.copyfile(cand.config_path, kdir / "config.json")+ fd, tmp = tempfile.mkstemp(+ dir = str(kdir), prefix = f".{_cache.FILE_NAME}.", suffix = _cache._TEMP_SUFFIX+ )+ os.close(fd)+ hasher = hashlib.sha256()+ try:+ size = _write(+ tmp,+ cand,+ {CAST_META_KEY: json.dumps(meta, sort_keys = True)},+ hasher,+ reader,)- _cache._touch(kdir)- _log(- logger,- "info",- f"{cand.component}: saved the {cand.parts['dtype']} cast ({cand.key}, {size / 2**30:.2f} GiB from "- f"{source_bytes / 2**30:.2f} GiB of source shards, in {time.perf_counter() - t0:.1f} s)",+ os.replace(tmp, kdir / _cache.FILE_NAME)+ tmp = None+ finally:+ if tmp is not None:+ try:+ os.unlink(tmp)+ except OSError:+ pass+ sha = hasher.hexdigest()+ _cache._record_verified(kdir, sha)+ manifest = {+ "key": cand.key,+ "bytes": size,+ "sha256": sha,+ "created": time.time(),+ "parts": cand.parts,+ }+ _cache._atomic_bytes(+ kdir / _cache.MANIFEST_NAME,+ json.dumps(manifest, indent = 1, default = str).encode("utf-8"),)- _cache.evict(logger = logger, keep = cand.key)- return str(kdir / _cache.FILE_NAME)- except Exception as exc: # noqa: BLE001 - a cache write never fails a load+ _cache._touch(kdir)+ _log(+ logger,+ "info",+ f"{cand.component}: saved the {cand.parts['dtype']} cast ({cand.key}, {size / 2**30:.2f} GiB from "+ f"{source_bytes / 2**30:.2f} GiB of source shards, in {time.perf_counter() - t0:.1f} s)",+ )+ _cache.evict(logger = logger, keep = cand.key)+ return str(kdir / _cache.FILE_NAME)+++class _ShardReader:+ """``tensor(name)`` straight from the source shards, cast to the cached dtype: one tensor in RAM at a time."""++ def __init__(self, cand: CastCandidate, where: dict, check: Any) -> None:+ self.cand = cand+ self.where = where+ self.check = check+ self.dtypes = {n: d for n, d, _s in cand.names}+ self._open: dict = {}++ def tensor(self, name: str) -> Any:+ from safetensors import safe_open++ shard = self.where[name]+ fh = self._open.get(shard)+ if fh is None:+ for old in self._open.values():+ old.__exit__(None, None, None)+ self._open = {shard: safe_open(shard, framework = "pt").__enter__()}+ fh = self._open[shard]+ return fh.get_tensor(name).to(self.dtypes[name])++ def close(self) -> None:+ for fh in self._open.values():+ try:+ fh.__exit__(None, None, None)+ except Exception: # noqa: BLE001+ pass+ self._open = {}+++def _source_names(cand: CastCandidate) -> tuple[Optional[list], dict]:+ """``([(name, dtype, shape)], {name: shard})`` a ``from_pretrained(torch_dtype = cand.dtype)`` load would hold, from+ an empty-weights instance of the class (diffusers only) and the shard headers; (None, {}) when they disagree."""+ import torch++ if cand.library != "diffusers":+ return None, {}+ from accelerate import init_empty_weights++ config = cand.cls.load_config(os.path.dirname(cand.config_path))+ with init_empty_weights():+ model = cand.cls.from_config(config)+ keep = tuple(str(m) for m in (getattr(cand.cls, "_keep_in_fp32_modules", None) or ()))+ tied = _tied_keys(model)+ where: dict = {}+ tags: dict = {}+ for shard in cand.shards:+ header = _shard_header(shard) or {}+ for key, entry in header.items():+ if key != "__metadata__" and isinstance(entry, dict):+ where[key] = shard+ tags[key] = entry+ names = []+ for name, tensor in model.state_dict(keep_vars = True).items():+ if name in tied and name not in where:+ continue+ entry = tags.get(name)+ if entry is None or [int(d) for d in entry.get("shape", ())] != [+ int(d) for d in tensor.shape+ ]:+ return None, {}+ dtype = tensor.dtype+ if dtype.is_floating_point:+ dtype = torch.float32 if any(k in name.split(".") for k in keep) else cand.dtype+ names.append((name, dtype, tuple(int(d) for d in tensor.shape)))+ return names, where+++def save_from_source(+ cand: CastCandidate,+ *,+ logger: Any = None,+ check: Optional[Any] = None,+) -> Optional[str]:+ """Write ``cand``'s cast straight from its source shards, one tensor in RAM at a time, for a component whose+ dense load would not fit host RAM; the load then maps the cached file. Returns the path or None. Never raises."""+ check = check or (lambda: None)+ t0 = time.perf_counter()+ reader = None+ try:+ names, where = _source_names(cand)+ if not names:+ _log(+ logger,+ "info",+ f"{cand.component}: not cached from source (its tensors do not match the class)",+ )+ return None+ cand.names = names+ reader = _ShardReader(cand, where, check)+ return _persist(cand, reader, logger = logger, t0 = t0)+ except Exception as exc: # noqa: BLE001 - the dense load still runs_log(logger,"warning",- f"{cand.component}: save failed: {type(exc).__name__}: {str(exc)[:200]}",+ f"{cand.component}: source cast failed: {type(exc).__name__}: {str(exc)[:200]}",)return None+ finally:+ if reader is not None:+ reader.close()class CastWrites:diff --git a/studio/backend/core/inference/video.py b/studio/backend/core/inference/video.pyindex ac2d1cdc055..2495f4b3999 100644--- a/studio/backend/core/inference/video.py+++ b/studio/backend/core/inference/video.py@@ -3452,8 +3452,18 @@ def _read_cast_cache(if not cands:return []if device is not None:- from .video_small_host import stage_misses- stage_misses(cands, device, logger = logger, cancel_check = cancel_check)+ from .video_small_host import converted_others_mib, stage_misses+ others = converted_others_mib(+ source,+ cands,+ torch_dtype = pipe_kwargs.get("torch_dtype"),+ cache_dir = None+ if source and os.path.isdir(str(source))+ else pipe_kwargs.get("cache_dir"),+ revision = revision,+ skip = tuple(pipe_kwargs),+ )+ stage_misses(cands, device, logger = logger, cancel_check = cancel_check, others_mib = others)hits = read_all(cands, logger = logger, cancel_check = cancel_check)pipe_kwargs.update(hits)if hits:diff --git a/studio/backend/core/inference/video_small_host.py b/studio/backend/core/inference/video_small_host.pyindex 6045d2539cf..bfa2ae9c180 100644--- a/studio/backend/core/inference/video_small_host.py+++ b/studio/backend/core/inference/video_small_host.py@@ -17,7 +17,7 @@ import reimport timefrom dataclasses import dataclass, fieldfrom pathlib import Path-from typing import Any, Callable, Optional+from typing import Any, Callable, Iterable, OptionalSMALL_HOST_ENV = "UNSLOTH_VIDEO_SMALL_HOST"_MIB = 1024 * 1024@@ -83,6 +83,10 @@ class StagePlan:available_mib: Optional[int] = Nonereserve_mib: int = 0skipped: dict = field(default_factory = dict)+ # component -> MiB of it placed on the card when it is staged split across the card and host RAM+ split: dict = field(default_factory = dict)+ # MiB the same load converts in host RAM outside the cast cache (a bf16 encoder cast to fp16)+ others_mib: int = 0def plan_stage(@@ -93,48 +97,83 @@ def plan_stage(device_free_mib: Optional[int],forced: bool = False,disabled: bool = False,+ others_mib: int = 0,+ source_castable: Optional[Iterable] = None,) -> StagePlan:- """``misses``: component -> loaded MiB. Stage largest first (each must fit the card) until the rest fits host RAM."""+ """``misses``: component -> loaded MiB. Stage largest first until the rest fits host RAM; ``others_mib`` is what+ the same load converts in host RAM besides them (it cannot be staged, so it is priced, not moved). A miss larger+ than the card is staged split: the card holds what fits, host RAM the rest (when that rest fits). Only a+ component in ``source_castable`` (None = every one) can be split, since its cast is written from the source+ shards; any other one too big for the card is skipped and the planner moves on."""if disabled:return StagePlan(False, f"{SMALL_HOST_ENV}=0")sizes = {k: int(v) for k, v in misses.items() if v is not None}dense = sum(sizes.values())+ others = max(0, int(others_mib or 0))if not sizes:- return StagePlan(False, "no cast-cache miss to convert", dense_mib = 0)+ return StagePlan(False, "no cast-cache miss to convert", dense_mib = 0, others_mib = others)if available_mib is None and not forced:- return StagePlan(False, "host RAM unreadable", dense_mib = dense)- fits = available_mib is not None and dense + reserve_mib <= int(available_mib)+ return StagePlan(False, "host RAM unreadable", dense_mib = dense, others_mib = others)+ beside = f" + {others} MiB of other converted components" if others else ""+ fits = available_mib is not None and dense + others + reserve_mib <= int(available_mib)if fits and not forced:return StagePlan(False,- f"dense conversion ({dense} MiB) fits {available_mib} MiB available host RAM",+ f"dense conversion ({dense} MiB{beside}) fits {available_mib} MiB available host RAM",dense_mib = dense,available_mib = available_mib,reserve_mib = reserve_mib,+ others_mib = others,)stage = []skipped = {}+ split = {}left = densefor name, mib in sorted(sizes.items(), key = lambda kv: (-kv[1], kv[0])):- if not forced and available_mib is not None and left + reserve_mib <= int(available_mib):+ if (+ not forced+ and available_mib is not None+ and left + others + reserve_mib <= int(available_mib)+ ):breakif mib < _STAGE_MIN_MIB:skipped[name] = "small"continue- if device_free_mib is None or mib + _DEVICE_MARGIN_MIB > int(device_free_mib):- skipped[name] = (- f"needs {mib + _DEVICE_MARGIN_MIB} MiB of the card, {device_free_mib} MiB free"- )+ card = None if device_free_mib is None else int(device_free_mib) - _DEVICE_MARGIN_MIB+ if card is not None and mib <= card:+ stage.append(name)+ left -= mib+ continue+ # Split: the card's share plus a host share that fits beside the rest of the load's conversions.+ host_room = (+ None+ if available_mib is None+ else int(available_mib) - reserve_mib - others - (left - mib)+ )+ if (+ card is not None+ and (source_castable is None or name in source_castable)+ and card >= _STAGE_MIN_MIB+ and host_room is not None+ and mib - card <= host_room+ ):+ stage.append(name)+ split[name] = card+ left -= mibcontinue- stage.append(name)- left -= mib+ skipped[name] = (+ f"needs {mib + _DEVICE_MARGIN_MIB} MiB of the card, {device_free_mib} MiB free, and its host share "+ f"does not fit {host_room} MiB"+ )why = (f"{SMALL_HOST_ENV}=1"if forced- else f"dense conversion needs ~{dense} MiB + {reserve_mib} MiB reserve of host RAM, "+ else f"dense conversion needs ~{dense} MiB{beside} + {reserve_mib} MiB reserve of host RAM, "f"{available_mib} MiB available")- return StagePlan(bool(stage), why, tuple(stage), dense, available_mib, reserve_mib, skipped)+ return StagePlan(+ bool(stage), why, tuple(stage), dense, available_mib, reserve_mib, skipped, split, others+ )def _component_dir(cand: Any) -> str:@@ -206,12 +245,66 @@ def _device_free_mib(device: Any) -> Optional[int]:return None+def converted_others_mib(+ source: Any,+ cands: list,+ *,+ torch_dtype: Any,+ cache_dir: Optional[str] = None,+ revision: Optional[str] = None,+ skip: Any = (),+) -> int:+ """MiB the pipeline load converts in host RAM outside the cast cache: half-precision shards of a component that+ is not a cast candidate but loads at the other half dtype (bf16 -> fp16 on a float16 card), which become anonymous+ memory beside the cast-cache misses. 0 when unreadable."""+ try:+ import torch++ from . import diffusion_cast_cache as cc++ index = cc._read_json(+ cc._cache._local_file(str(source), "model_index.json", cache_dir, revision)+ )+ if not isinstance(index, dict):+ return 0+ names = {c.component for c in cands}+ skip = set(skip or ())+ total = 0+ for component, spec in index.items():+ if component.startswith("_") or component in skip or component in names:+ continue+ if (+ not isinstance(spec, (list, tuple))+ or len(spec) != 2+ or spec[0] not in cc._LIBRARIES+ ):+ continue+ dtype = cc._component_dtype(torch_dtype, component)+ if not cc._is_half(dtype):+ continue+ want = "F16" if dtype == torch.float16 else "BF16"+ for shard in (+ cc._shards_of(str(source), component, cc._STEMS[spec[0]], cache_dir, revision) or ()+ ):+ header = cc._shard_header(shard) or {}+ for name, entry in header.items():+ if name == "__metadata__" or not isinstance(entry, dict):+ continue+ if entry.get("dtype") in ("F16", "BF16") and entry.get("dtype") != want:+ begin, end = (int(v) for v in entry.get("data_offsets") or (0, 0))+ total += end - begin+ return -(-total // _MIB)+ except Exception: # noqa: BLE001 - unpriced, as before+ return 0++def stage_misses(cands: list,device: Any,*,logger: Any = None,cancel_check: Optional[Callable[[], None]] = None,+ others_mib: int = 0,) -> list:"""Stage the misses of ``cands`` a dense load would not fit; returns the staged components. Never raises excepta cancel."""@@ -235,6 +328,10 @@ def stage_misses(device_free_mib = _device_free_mib(device),forced = stage_forced(),disabled = stage_disabled(),+ others_mib = others_mib,+ source_castable = {+ c.component for c in cands if getattr(c, "library", None) == "diffusers"+ },)if logger is not None and (plan.engaged or plan.skipped or misses):logger.info(@@ -246,11 +343,47 @@ def stage_misses()done = []by_name = {c.component: c for c in cands}+ from .diffusion_cast_cache import save_from_source+for name in plan.stage:if cancel_check is not None:cancel_check()+ if plan.split.get(name):+ # A card / host split through accelerate's cpu offload brings its host share back as meta tensors+ # (nothing to write); the source shards cast one tensor at a time instead, card untouched. If that+ # fails the component is left to the normal load.+ t0 = time.perf_counter()+ if save_from_source(by_name[name], logger = logger, check = cancel_check) is not None:+ done.append(name)+ if logger is not None:+ logger.info(+ "video.small_host: %s cast written from its source shards in %.1f s (host RAM: one "+ "tensor at a time); the load maps it",+ name,+ time.perf_counter() - t0,+ )+ continueif stage_on_device(by_name[name], device, logger = logger, cancel_check = cancel_check):done.append(name)+ if plan.skipped and not stage_disabled():+ # Neither the card nor a card / host split takes it: write its cast from the source shards one tensor at a+ # time, so the load maps the cached file instead of converting the whole component in host RAM. This runs+ # even when nothing else was staged (a lone DiT too big for the card is the case it is for).+ for name, why in plan.skipped.items():+ if name not in by_name or why == "small":+ continue+ if cancel_check is not None:+ cancel_check()+ t0 = time.perf_counter()+ if save_from_source(by_name[name], logger = logger, check = cancel_check) is not None:+ done.append(name)+ if logger is not None:+ logger.info(+ "video.small_host: %s cast written from its source shards in %.1f s (host RAM: one "+ "tensor at a time); the load maps it",+ name,+ time.perf_counter() - t0,+ )return doneexcept Exception as exc: # noqa: BLE001 - the dense load still runsif cancel_check is not None:diff --git a/studio/backend/tests/test_diffusion_cast_cache.py b/studio/backend/tests/test_diffusion_cast_cache.pyindex 43efb39fa5f..deca08e4601 100644--- a/studio/backend/tests/test_diffusion_cast_cache.py+++ b/studio/backend/tests/test_diffusion_cast_cache.py@@ -291,3 +291,70 @@ def test_resident_device_module_is_read_back_bit_identical(pipe_dir):hit = CC.read(cand)for k, v in hit.state_dict().items():assert v.dtype == reference[k].dtype and torch.equal(_bits(v), _bits(reference[k])), k+++def test_source_cast_matches_the_live_module_cast_bit_for_bit(pipe_dir, tmp_path, monkeypatch):+ """A component too big for host RAM is cast from its source shards one tensor at a time: the cached file is the+ same file the live-module write produces (same key, same bytes), and reads back as the dense load."""+ cand = _by_name(CC.candidates(pipe_dir, torch_dtype = FP16_DEFAULT))["transformer"]+ path = CC.save_from_source(cand)+ assert path is not None and C.path_in_cache(path)+ from_source = Path(path).read_bytes()+ hit = CC.read(_by_name(CC.candidates(pipe_dir, torch_dtype = FP16_DEFAULT))["transformer"])+ assert hit is not None+ dense = _dense(cand)+ _identical(dense, hit)+ # the live-module write of the same component, into a second cache root: identical bytes (bar the timestamp-free+ # metadata, which is the same too)+ monkeypatch.setenv(C.ENV_DIR, str(tmp_path / "cache_live"))+ live = _by_name(CC.candidates(pipe_dir, torch_dtype = FP16_DEFAULT))["transformer"]+ live_path = _save(live, dense)+ assert live_path is not None+ assert Path(live_path).read_bytes() == from_source+++def test_source_cast_refuses_a_class_its_shards_do_not_match(pipe_dir):+ cand = _by_name(CC.candidates(pipe_dir, torch_dtype = FP16_DEFAULT))["transformer"]+ cand.cls = (+ diffusers.UNet2DModel+ ) # a different architecture: names / shapes disagree with the shards+ cand.config_path = str(pipe_dir / "transformer" / "config.json")+ assert CC.save_from_source(cand) is None+ # transformers components are not cast from source (their dense load is the small one)+ te = _by_name(CC.candidates(pipe_dir, torch_dtype = FP16_DEFAULT))["text_encoder"]+ assert CC.save_from_source(te) is None+++def test_source_cast_of_a_hunyuanvideo15_denoiser_reads_back_as_the_dense_load(tmp_path):+ """The HunyuanVideo-1.5 case: its fp32 DiT (31 GB) does not fit a 31 GB host beside the converted encoder."""+ torch.manual_seed(0)+ model = diffusers.HunyuanVideo15Transformer3DModel(+ in_channels = 9,+ out_channels = 4,+ num_attention_heads = 2,+ attention_head_dim = 32,+ num_layers = 1,+ num_refiner_layers = 1,+ mlp_ratio = 2.0,+ text_embed_dim = 64,+ text_embed_2_dim = 32,+ image_embed_dim = 32,+ rope_axes_dim = (8, 12, 12),+ target_size = 64,+ ).to(torch.float32)+ root = tmp_path / "HunyuanVideo-1.5-tiny"+ model.save_pretrained(root / "transformer", max_shard_size = "200KB")+ (root / "model_index.json").write_text(+ json.dumps(+ {+ "_class_name": "HunyuanVideo15Pipeline",+ "transformer": ["diffusers", "HunyuanVideo15Transformer3DModel"],+ }+ )+ )+ cand = _by_name(CC.candidates(root, torch_dtype = FP16_DEFAULT))["transformer"]+ assert len(cand.shards) > 1+ assert CC.save_from_source(cand) is not None+ hit = CC.read(_by_name(CC.candidates(root, torch_dtype = FP16_DEFAULT))["transformer"])+ assert hit is not None+ _identical(_dense(cand), hit)diff --git a/studio/backend/tests/test_video_small_host.py b/studio/backend/tests/test_video_small_host.pyindex 390bbff94bb..ba61e848c9e 100644--- a/studio/backend/tests/test_video_small_host.py+++ b/studio/backend/tests/test_video_small_host.py@@ -60,11 +60,19 @@ def test_stages_largest_first_until_the_rest_fits():assert plan.stage == ("text_encoder", "transformer")-def test_card_too_small_for_the_encoder_keeps_the_dense_load():+def test_card_too_small_for_the_encoder_stages_it_split():plan = VS.plan_stage(WAN13, available_mib = 10100, reserve_mib = 3072, device_free_mib = 11000)- # the encoder does not fit the card; the DiT alone does but cannot rescue the load: staged anyway, logged+ # the encoder does not fit the card: the card takes what fits, host RAM the 2.5 GB rest, and the DiT then+ # converts densely beside it+ assert plan.stage == ("text_encoder",)+ assert plan.split == {"text_encoder": 11000 - VS._DEVICE_MARGIN_MIB}+++def test_a_split_whose_host_share_does_not_fit_keeps_the_dense_load():+ plan = VS.plan_stage(WAN13, available_mib = 10100, reserve_mib = 3072, device_free_mib = 4000)+ # 9.5 GB of the encoder would stay in host RAM beside the DiT: no split; the DiT alone is staged, loggedassert "text_encoder" in plan.skipped- assert plan.stage == ("transformer",)+ assert plan.stage == ("transformer",) and plan.split == {}def test_kill_switch_and_force(monkeypatch):@@ -265,3 +273,180 @@ def test_non_cuda_device_never_stages(pipe_dir, monkeypatch):calls = _wire(monkeypatch, available = 10)assert VS.stage_misses(list(_cands(pipe_dir).values()), "cpu") == []assert calls == []+++# HunyuanVideo-1.5 on a Kaggle T4 (31 GB host, 15 GB card): the fp32 DiT converts to fp16 (18.4 GiB, a cast-cache+# miss) while the bf16 Qwen2.5-VL-7B converts to fp16 beside it (15.6 GiB, not a cast candidate). Pricing the DiT+# alone let every load through and the host ran out of RAM (rc -9).+HV15_T4 = {"transformer": 18827}+HV15_TE_MIB = 15600+++def test_the_encoder_converted_beside_the_dit_is_priced():+ old = VS.plan_stage(HV15_T4, available_mib = 29238, reserve_mib = 3174, device_free_mib = 14600)+ assert not old.engaged # the DiT alone "fits"+ plan = VS.plan_stage(+ HV15_T4,+ available_mib = 29238,+ reserve_mib = 3174,+ device_free_mib = 14600,+ others_mib = HV15_TE_MIB,+ )+ assert plan.engaged and plan.stage == ("transformer",)+ # 15 GB card: split, 13.8 GB on the card and 4.9 GB in host RAM beside the encoder's 15.6+ assert plan.split == {"transformer": 14600 - VS._DEVICE_MARGIN_MIB}+ assert plan.others_mib == HV15_TE_MIB and "other converted components" in plan.reason+++def test_others_that_fit_change_nothing():+ plan = VS.plan_stage(+ HV15_T4,+ available_mib = 60000,+ reserve_mib = 6000,+ device_free_mib = 14600,+ others_mib = HV15_TE_MIB,+ )+ assert not plan.engaged and "fits" in plan.reason+++def test_converted_others_counts_only_a_half_dtype_change(pipe_dir):+ import safetensors.torch as st++ te = pipe_dir / "text_encoder"+ for f in te.glob("*.safetensors"):+ f.unlink()+ st.save_file({"w": torch.zeros(1024, 512, dtype = torch.bfloat16)}, str(te / "model.safetensors"))+ cands = [c for c in CC.candidates(pipe_dir, torch_dtype = FP16_DEFAULT)]+ assert {c.component for c in cands} == {+ "transformer"+ } # the bf16 encoder is not a cast candidate+ fp16 = VS.converted_others_mib(pipe_dir, cands, torch_dtype = FP16_DEFAULT)+ assert fp16 == 1 # 1 MiB of bf16 converted to fp16 in host RAM+ bf16 = VS.converted_others_mib(+ pipe_dir, cands, torch_dtype = {"default": torch.bfloat16, "vae": torch.float32}+ )+ assert bf16 == 0 # same dtype: memory-mapped, nothing converted+ assert (+ VS.converted_others_mib(pipe_dir, cands, torch_dtype = FP16_DEFAULT, skip = ("text_encoder",))+ == 0+ )+++def test_device_load_kwargs_put_the_whole_component_on_the_card():+ import types+ from unittest import mock++ cand = types.SimpleNamespace()+ with mock.patch.object(+ CC, "_load_kwargs", lambda c: {"torch_dtype": torch.float16, "output_loading_info": True}+ ):+ whole = VS._device_load_kwargs(cand, "cuda:0")+ assert whole["device_map"] == {"": "cuda:0"} and "max_memory" not in whole+ assert "output_loading_info" not in whole+++def test_a_dit_neither_the_card_nor_a_split_takes_is_cast_from_its_shards(pipe_dir, monkeypatch):+ """The HunyuanVideo-1.5 T4 case: the fp16 DiT is bigger than the card and its host share does not fit, so its cast+ is written from the source shards one tensor at a time and the load maps it (no dense conversion in host RAM)."""+ from core.inference import video as V++ sizes = {"text_encoder": 100, "transformer": 20000}+ monkeypatch.setattr(VS, "loaded_mib", lambda cand: sizes[cand.component])+ calls = _wire(monkeypatch, available = 3000, total = 13000)+ pipe_kwargs = {"torch_dtype": dict(FP16_DEFAULT), "cache_dir": None, "local_files_only": True}+ written = []+ real = CC.save_from_source+ monkeypatch.setattr(+ CC,+ "save_from_source",+ lambda cand, **kw: written.append(cand.component) or real(cand, **kw),+ )+ missed = V._read_cast_cache(+ pipe_kwargs, source = str(pipe_dir), revision = None, base = None, device = "cuda"+ )+ assert written == ["transformer"] and ("transformer", "cuda") not in calls+ assert "transformer" in pipe_kwargs and "transformer" not in [c.component for c in missed]+ _identical(pipe_kwargs["transformer"], _dense(_cands(pipe_dir)["transformer"]))+++def test_source_cast_respects_the_kill_switch(pipe_dir, monkeypatch):+ from core.inference import video as V++ monkeypatch.setenv(VS.SMALL_HOST_ENV, "0")+ sizes = {"text_encoder": 100, "transformer": 20000}+ monkeypatch.setattr(VS, "loaded_mib", lambda cand: sizes[cand.component])+ _wire(monkeypatch, available = 3000, total = 13000)+ pipe_kwargs = {"torch_dtype": dict(FP16_DEFAULT), "cache_dir": None, "local_files_only": True}+ V._read_cast_cache(pipe_kwargs, source = str(pipe_dir), revision = None, base = None, device = "cuda")+ assert "transformer" not in pipe_kwargs+++def test_a_split_plan_casts_from_the_source_shards_instead(pipe_dir, monkeypatch):+ """A DiT a little over the card was split card / host through accelerate's cpu offload, whose host share loads+ as meta tensors (nothing to write, on the Kaggle T4 the dense load then ran out of host RAM): it is cast from+ its source shards instead, without touching the card."""+ from core.inference import video as V++ sizes = {"text_encoder": 100, "transformer": 16000}+ monkeypatch.setattr(VS, "loaded_mib", lambda cand: sizes[cand.component])+ calls = _wire(monkeypatch, available = 18000, total = 20000)+ written = []+ real = CC.save_from_source+ monkeypatch.setattr(+ CC,+ "save_from_source",+ lambda cand, **kw: written.append(cand.component) or real(cand, **kw),+ )+ pipe_kwargs = {"torch_dtype": dict(FP16_DEFAULT), "cache_dir": None, "local_files_only": True}+ V._read_cast_cache(pipe_kwargs, source = str(pipe_dir), revision = None, base = None, device = "cuda")+ assert written == ["transformer"] and ("transformer", "cuda") not in calls+ _identical(pipe_kwargs["transformer"], _dense(_cands(pipe_dir)["transformer"]))+++def test_a_split_plan_whose_source_cast_fails_is_left_to_the_normal_load(pipe_dir, monkeypatch):+ """No accelerate card / host split as a fallback (its host share comes back as meta tensors): a split-planned+ component whose source cast fails is not staged at all, so the pipeline loads it as on main."""+ from core.inference import video as V++ sizes = {"text_encoder": 100, "transformer": 16000}+ monkeypatch.setattr(VS, "loaded_mib", lambda cand: sizes[cand.component])+ calls = _wire(monkeypatch, available = 18000, total = 20000)+ monkeypatch.setattr(CC, "save_from_source", lambda cand, **kw: None)+ pipe_kwargs = {"torch_dtype": dict(FP16_DEFAULT), "cache_dir": None, "local_files_only": True}+ V._read_cast_cache(pipe_kwargs, source = str(pipe_dir), revision = None, base = None, device = "cuda")+ assert ("transformer", "cuda") not in calls and "transformer" not in pipe_kwargs+++def test_a_lone_dit_too_big_for_the_card_is_still_cast_from_its_shards(pipe_dir, monkeypatch):+ """Nothing else staged: the plan is not engaged, but a skipped component too big for the card and the host still+ takes the source cast, so the load maps it instead of converting it densely."""+ from core.inference import video as V++ sizes = {"text_encoder": 100, "transformer": 20000}+ monkeypatch.setattr(VS, "loaded_mib", lambda cand: sizes[cand.component])+ _wire(monkeypatch, available = 3000, total = 13000)+ monkeypatch.setattr(VS, "_STAGE_MIN_MIB", 512)+ written = []+ real = CC.save_from_source+ monkeypatch.setattr(+ CC,+ "save_from_source",+ lambda cand, **kw: written.append(cand.component) or real(cand, **kw),+ )+ pipe_kwargs = {"torch_dtype": dict(FP16_DEFAULT), "cache_dir": None, "local_files_only": True}+ V._read_cast_cache(pipe_kwargs, source = str(pipe_dir), revision = None, base = None, device = "cuda")+ assert written == ["transformer"] and "transformer" in pipe_kwargs+++def test_a_transformers_encoder_over_the_card_is_not_split_planned():+ """Its cast cannot be written from the source shards (diffusers components only), so it is skipped and the+ planner goes on to stage the DiT, as before the split path existed."""+ plan = VS.plan_stage(+ WAN13,+ available_mib = 16000,+ reserve_mib = 3000,+ device_free_mib = 11000,+ source_castable = {"transformer"},+ )+ assert "text_encoder" in plan.skipped and plan.split == {}+ assert plan.stage == ("transformer",)