Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

Studio video: stage an fp32 encoder on the card when its dense cast would not fit host RAM (#15975)

Daniel Hand3702be73973
diff --git a/studio/backend/core/inference/video.py b/studio/backend/core/inference/video.py
index d9c0c036e06..6139c6769d2 100644
--- a/studio/backend/core/inference/video.py
+++ b/studio/backend/core/inference/video.py
@@ -3159,8 +3159,10 @@ def _read_cast_cache(
revision: Optional[str],
base: Optional[str],
cancel_check: Any = None,
+ device: Any = None,
) -> list:
- """Inject cached float16 / bfloat16 casts into ``pipe_kwargs``; returns the unserved candidates. Never raises."""
+ """Inject cached float16 / bfloat16 casts into ``pipe_kwargs``; returns the unserved candidates. Never raises.
+ ``device``: a miss whose dense conversion would not fit host RAM is first staged there (``video_small_host``)."""
try:
from .diffusion_cast_cache import candidates, read_all
@@ -3177,6 +3179,9 @@ def _read_cast_cache(
)
if not cands:
return []
+ if device is not None:
+ from .video_small_host import stage_misses
+ stage_misses(cands, device, logger = logger, cancel_check = cancel_check)
hits = read_all(cands, logger = logger, cancel_check = cancel_check)
pipe_kwargs.update(hits)
if hits:
@@ -8974,6 +8979,7 @@ class VideoBackend:
revision = None if _base_local_dir else _base_fetch_revision,
base = base,
cancel_check = lambda: self._raise_if_video_load_superseded(_load_token),
+ device = target.torch_device if target.device == "cuda" else None,
)
# The pre-downloaded snapshot dir keeps from_pretrained off the hub; hub id when pre-download was skipped.
pipe = pipeline_cls.from_pretrained(
diff --git a/studio/backend/core/inference/video_small_host.py b/studio/backend/core/inference/video_small_host.py
new file mode 100644
index 00000000000..6045d2539cf
--- /dev/null
+++ b/studio/backend/core/inference/video_small_host.py
@@ -0,0 +1,262 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
+
+"""Stage cast-cache misses too big for host RAM (e.g. fp32 UMT5 on a 12.7 GB host gets OOM-killed) onto the card.
+
+Each such miss loads alone onto the device (transformers streams per tensor), is written to the cast cache from the
+device tensors and freed; the normal cast-cache read then serves it, same bits. ``UNSLOTH_VIDEO_SMALL_HOST=0`` off,
+``=1`` stages every miss that fits the card.
+"""
+
+from __future__ import annotations
+
+import gc
+import json
+import os
+import re
+import time
+from dataclasses import dataclass, field
+from pathlib import Path
+from typing import Any, Callable, Optional
+
+SMALL_HOST_ENV = "UNSLOTH_VIDEO_SMALL_HOST"
+_MIB = 1024 * 1024
+# device bytes the staged load needs above its own weights (CUDA context grows, allocator rounding)
+_DEVICE_MARGIN_MIB = 768
+# a candidate this small converts in RAM even on a small host
+_STAGE_MIN_MIB = 512
+
+
+def _env() -> str:
+ return (os.environ.get(SMALL_HOST_ENV) or "").strip().lower()
+
+
+def stage_disabled() -> bool:
+ return _env() in ("0", "off", "false", "no")
+
+
+def stage_forced() -> bool:
+ return _env() in ("1", "on", "true", "yes", "force")
+
+
+def _keep_regex(patterns: Any) -> Optional[Any]:
+ pats = [str(p) for p in (patterns or ()) if p]
+ if not pats:
+ return None
+ # transformers' own rule (modeling_utils: ``(^|\.)module($|\.)``)
+ return re.compile("|".join(rf"((^|\.){re.escape(p)}($|\.))" for p in pats))
+
+
+def loaded_mib(cand: Any) -> Optional[int]:
+ """MiB ``cand`` occupies once loaded at its half dtype: fp32 tensors halve except ``_keep_in_fp32_modules``."""
+ from .diffusion_cast_cache import _shard_header
+
+ keep = _keep_regex((getattr(cand, "parts", None) or {}).get("keep_in_fp32"))
+ total = 0
+ for shard in getattr(cand, "shards", None) or ():
+ header = _shard_header(shard)
+ if header is None:
+ return None
+ for name, entry in header.items():
+ if not isinstance(entry, dict):
+ continue
+ try:
+ begin, end = (int(v) for v in entry.get("data_offsets") or (0, 0))
+ except (TypeError, ValueError):
+ return None
+ size = end - begin
+ tag = entry.get("dtype")
+ if tag == "F64":
+ size = size // 4 if keep is None or not keep.search(name) else size // 2
+ elif tag == "F32" and (keep is None or not keep.search(name)):
+ size //= 2
+ total += size
+ return -(-total // _MIB)
+
+
+@dataclass(frozen = True)
+class StagePlan:
+ engaged: bool
+ reason: str
+ stage: tuple = ()
+ dense_mib: int = 0
+ available_mib: Optional[int] = None
+ reserve_mib: int = 0
+ skipped: dict = field(default_factory = dict)
+
+
+def plan_stage(
+ misses: dict,
+ *,
+ available_mib: Optional[int],
+ reserve_mib: int,
+ device_free_mib: Optional[int],
+ forced: bool = False,
+ disabled: bool = False,
+) -> StagePlan:
+ """``misses``: component -> loaded MiB. Stage largest first (each must fit the card) until the rest fits host RAM."""
+ if disabled:
+ return StagePlan(False, f"{SMALL_HOST_ENV}=0")
+ sizes = {k: int(v) for k, v in misses.items() if v is not None}
+ dense = sum(sizes.values())
+ if not sizes:
+ return StagePlan(False, "no cast-cache miss to convert", dense_mib = 0)
+ if available_mib is None and not forced:
+ return StagePlan(False, "host RAM unreadable", dense_mib = dense)
+ fits = available_mib is not None and dense + reserve_mib <= int(available_mib)
+ if fits and not forced:
+ return StagePlan(
+ False,
+ f"dense conversion ({dense} MiB) fits {available_mib} MiB available host RAM",
+ dense_mib = dense,
+ available_mib = available_mib,
+ reserve_mib = reserve_mib,
+ )
+ stage = []
+ skipped = {}
+ left = dense
+ for name, mib in sorted(sizes.items(), key = lambda kv: (-kv[1], kv[0])):
+ if not forced and available_mib is not None and left + reserve_mib <= int(available_mib):
+ break
+ if mib < _STAGE_MIN_MIB:
+ skipped[name] = "small"
+ continue
+ if device_free_mib is None or mib + _DEVICE_MARGIN_MIB > int(device_free_mib):
+ skipped[name] = (
+ f"needs {mib + _DEVICE_MARGIN_MIB} MiB of the card, {device_free_mib} MiB free"
+ )
+ continue
+ stage.append(name)
+ left -= mib
+ why = (
+ f"{SMALL_HOST_ENV}=1"
+ if forced
+ else f"dense conversion needs ~{dense} MiB + {reserve_mib} MiB reserve of host RAM, "
+ f"{available_mib} MiB available"
+ )
+ return StagePlan(bool(stage), why, tuple(stage), dense, available_mib, reserve_mib, skipped)
+
+
+def _component_dir(cand: Any) -> str:
+ return str(Path(cand.config_path).parent)
+
+
+def _device_load_kwargs(cand: Any, device: Any) -> dict:
+ from .diffusion_cast_cache import _load_kwargs
+
+ kwargs = _load_kwargs(cand)
+ kwargs.pop("output_loading_info", None)
+ kwargs["device_map"] = {"": str(device)}
+ return kwargs
+
+
+def stage_on_device(
+ cand: Any,
+ device: Any,
+ *,
+ logger: Any = None,
+ cancel_check: Optional[Callable[[], None]] = None,
+) -> bool:
+ """Load ``cand`` alone onto ``device``, write its cast-cache entry from the device tensors, free it."""
+ import torch
+
+ from .diffusion_cast_cache import _state_names, save
+
+ t0 = time.perf_counter()
+ module = None
+ try:
+ module = cand.cls.from_pretrained(_component_dir(cand), **_device_load_kwargs(cand, device))
+ module.eval()
+ if cancel_check is not None:
+ cancel_check()
+ names = _state_names(module)
+ if not names:
+ return False
+ cand.names = names
+ path = save(cand, module, logger = logger, check = cancel_check)
+ if logger is not None:
+ logger.info(
+ "video.small_host: %s staged on %s in %.1f s (%s)",
+ cand.component,
+ device,
+ time.perf_counter() - t0,
+ "cast cache written" if path else "not written",
+ )
+ return bool(path)
+ finally:
+ module = None
+ gc.collect()
+ try:
+ if torch.cuda.is_available():
+ torch.cuda.empty_cache()
+ except Exception: # noqa: BLE001
+ pass
+
+
+def _device_free_mib(device: Any) -> Optional[int]:
+ try:
+ import torch
+
+ dev = torch.device(device)
+ if dev.type != "cuda" or not torch.cuda.is_available():
+ return None
+ free, _total = torch.cuda.mem_get_info(dev)
+ return int(free // _MIB)
+ except Exception: # noqa: BLE001
+ return None
+
+
+def stage_misses(
+ cands: list,
+ device: Any,
+ *,
+ logger: Any = None,
+ cancel_check: Optional[Callable[[], None]] = None,
+) -> list:
+ """Stage the misses of ``cands`` a dense load would not fit; returns the staged components. Never raises except
+ a cancel."""
+ if not cands or device is None:
+ return []
+ try:
+ import torch
+
+ if torch.device(device).type != "cuda":
+ return []
+ from .diffusion_cast_cache import cached_components
+ from .diffusion_small_host import host_ram_mib, host_reserve_mib
+
+ cached = set(cached_components(cands))
+ misses = {c.component: loaded_mib(c) for c in cands if c.component not in cached}
+ total, available = host_ram_mib()
+ plan = plan_stage(
+ misses,
+ available_mib = available,
+ reserve_mib = host_reserve_mib(total),
+ device_free_mib = _device_free_mib(device),
+ forced = stage_forced(),
+ disabled = stage_disabled(),
+ )
+ if logger is not None and (plan.engaged or plan.skipped or misses):
+ logger.info(
+ "video.small_host: engaged=%s (%s); stage=%s%s",
+ plan.engaged,
+ plan.reason,
+ list(plan.stage),
+ f" skipped={json.dumps(plan.skipped)}" if plan.skipped else "",
+ )
+ done = []
+ by_name = {c.component: c for c in cands}
+ for name in plan.stage:
+ if cancel_check is not None:
+ cancel_check()
+ if stage_on_device(by_name[name], device, logger = logger, cancel_check = cancel_check):
+ done.append(name)
+ return done
+ except Exception as exc: # noqa: BLE001 - the dense load still runs
+ if cancel_check is not None:
+ cancel_check()
+ if logger is not None:
+ logger.warning(
+ "video.small_host: staging skipped (%s: %s)", type(exc).__name__, str(exc)[:200]
+ )
+ return []
diff --git a/studio/backend/tests/test_video_small_host.py b/studio/backend/tests/test_video_small_host.py
new file mode 100644
index 00000000000..390bbff94bb
--- /dev/null
+++ b/studio/backend/tests/test_video_small_host.py
@@ -0,0 +1,267 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
+
+"""Video small-host staging: routing, kill switch, bit-identical staged entry."""
+
+from __future__ import annotations
+
+import json
+import sys
+from pathlib import Path
+
+import pytest
+
+torch = pytest.importorskip("torch")
+diffusers = pytest.importorskip("diffusers")
+transformers = pytest.importorskip("transformers")
+pytest.importorskip("safetensors")
+
+BACKEND = Path(__file__).resolve().parents[1]
+if str(BACKEND) not in sys.path:
+ sys.path.insert(0, str(BACKEND))
+
+from core.inference import diffusion_cast_cache as CC # noqa: E402
+from core.inference import diffusion_prequant_cache as C # noqa: E402
+from core.inference import video_small_host as VS # noqa: E402
+
+FP16_DEFAULT = {"vae": torch.float32, "default": torch.float16}
+# Wan2.1-T2V-1.3B on a Colab T4: UMT5 13.4 GB loaded (wo kept fp32), DiT 2.8 GB; 12.7 GB host, ~9.9 GB available
+WAN13 = {"text_encoder": 12748, "transformer": 2709}
+
+
[email protected](autouse = True)
+def _clean_env(tmp_path, monkeypatch):
+ monkeypatch.setenv(C.ENV_DIR, str(tmp_path / "cache"))
+ monkeypatch.delenv(CC.CAST_CACHE_ENV, raising = False)
+ monkeypatch.delenv(C.ENV_MODE, raising = False)
+ monkeypatch.delenv(VS.SMALL_HOST_ENV, raising = False)
+ monkeypatch.setattr(C, "_hub_source_stale", lambda shards, logger = None: False)
+ with C._lock:
+ C._REJECTED.clear()
+
+
+def test_big_host_is_unchanged():
+ plan = VS.plan_stage(WAN13, available_mib = 48000, reserve_mib = 5300, device_free_mib = 22000)
+ assert not plan.engaged and plan.stage == ()
+ assert "fits" in plan.reason
+
+
+def test_small_t4_host_stages_the_encoder_only():
+ plan = VS.plan_stage(WAN13, available_mib = 10100, reserve_mib = 3072, device_free_mib = 14600)
+ assert plan.engaged
+ # the DiT's dense conversion (2.7 GB) fits once the encoder is off the host
+ assert plan.stage == ("text_encoder",)
+ assert plan.dense_mib == sum(WAN13.values())
+
+
+def test_stages_largest_first_until_the_rest_fits():
+ misses = {"text_encoder": 9000, "transformer": 6000, "transformer_2": 6000}
+ plan = VS.plan_stage(misses, available_mib = 12000, reserve_mib = 3000, device_free_mib = 40000)
+ assert plan.stage == ("text_encoder", "transformer")
+
+
+def test_card_too_small_for_the_encoder_keeps_the_dense_load():
+ plan = VS.plan_stage(WAN13, available_mib = 10100, reserve_mib = 3072, device_free_mib = 11000)
+ # the encoder does not fit the card; the DiT alone does but cannot rescue the load: staged anyway, logged
+ assert "text_encoder" in plan.skipped
+ assert plan.stage == ("transformer",)
+
+
+def test_kill_switch_and_force(monkeypatch):
+ monkeypatch.setenv(VS.SMALL_HOST_ENV, "0")
+ assert VS.stage_disabled()
+ plan = VS.plan_stage(
+ WAN13,
+ available_mib = 100,
+ reserve_mib = 3072,
+ device_free_mib = 14600,
+ disabled = VS.stage_disabled(),
+ )
+ assert not plan.engaged
+ monkeypatch.setenv(VS.SMALL_HOST_ENV, "1")
+ plan = VS.plan_stage(
+ WAN13,
+ available_mib = 48000,
+ reserve_mib = 5300,
+ device_free_mib = 22000,
+ forced = VS.stage_forced(),
+ )
+ assert plan.stage == ("text_encoder", "transformer")
+
+
+def test_unreadable_host_ram_changes_nothing():
+ plan = VS.plan_stage(WAN13, available_mib = None, reserve_mib = 3072, device_free_mib = 14600)
+ assert not plan.engaged
+
+
+def test_small_candidates_are_never_staged():
+ plan = VS.plan_stage({"vae": 200}, available_mib = 100, reserve_mib = 3072, device_free_mib = 14600)
+ assert not plan.engaged and plan.skipped == {"vae": "small"}
+
+
+def _tiny_umt5():
+ torch.manual_seed(1)
+ config = transformers.UMT5Config(
+ vocab_size = 64,
+ d_model = 32,
+ d_kv = 8,
+ d_ff = 64,
+ num_layers = 2,
+ num_heads = 4,
+ relative_attention_num_buckets = 8,
+ )
+ return transformers.UMT5EncoderModel(config).to(torch.float32)
+
+
+def _tiny_wan():
+ torch.manual_seed(0)
+ return diffusers.WanTransformer3DModel(
+ patch_size = (1, 2, 2),
+ num_attention_heads = 2,
+ attention_head_dim = 16,
+ in_channels = 4,
+ out_channels = 4,
+ text_dim = 32,
+ freq_dim = 16,
+ ffn_dim = 64,
+ num_layers = 2,
+ cross_attn_norm = True,
+ qk_norm = "rms_norm_across_heads",
+ eps = 1e-6,
+ rope_max_seq_len = 32,
+ ).to(torch.float32)
+
+
+def pipe_dir(tmp_path):
+ root = tmp_path / "Wan2.1-T2V-tiny-Diffusers"
+ _tiny_wan().save_pretrained(root / "transformer", max_shard_size = "60KB")
+ _tiny_umt5().save_pretrained(root / "text_encoder")
+ (root / "vae").mkdir()
+ (root / "model_index.json").write_text(
+ json.dumps(
+ {
+ "_class_name": "WanPipeline",
+ "transformer": ["diffusers", "WanTransformer3DModel"],
+ "text_encoder": ["transformers", "UMT5EncoderModel"],
+ "vae": ["diffusers", "AutoencoderKLWan"],
+ "tokenizer": ["transformers", "T5TokenizerFast"],
+ "scheduler": ["diffusers", "UniPCMultistepScheduler"],
+ }
+ )
+ )
+ return root
+
+
+def _cands(pipe_dir):
+ return {c.component: c for c in CC.candidates(pipe_dir, torch_dtype = FP16_DEFAULT)}
+
+
+def _dense(cand):
+ kwargs = CC._load_kwargs(cand)
+ kwargs.pop("output_loading_info")
+ return cand.cls.from_pretrained(str(Path(cand.source) / cand.component), **kwargs)
+
+
+def _identical(a, b):
+ sa, sb = a.state_dict(), b.state_dict()
+ assert sa.keys() == sb.keys()
+ for k in sa:
+ assert sa[k].dtype == sb[k].dtype, k
+ assert sa[k].device.type == "cpu" and sb[k].device.type == "cpu", k
+ assert torch.equal(
+ sa[k].reshape(-1).view(torch.uint8), sb[k].reshape(-1).view(torch.uint8)
+ ), k
+
+
+def test_loaded_mib_keeps_wo_wide(pipe_dir):
+ cand = _cands(pipe_dir)["text_encoder"]
+ assert cand.parts["keep_in_fp32"] == ["wo"]
+ dense = _dense(cand)
+ want = sum(t.numel() * t.element_size() for t in dense.state_dict().values())
+ # tied embed_tokens is stored once; the estimate counts stored tensors at their loaded width
+ got_bytes = 0
+ for shard in cand.shards:
+ for name, e in CC._shard_header(shard).items():
+ b, end = e["data_offsets"]
+ got_bytes += (end - b) if ".wo." in f".{name}" else (end - b) // 2
+ assert VS.loaded_mib(cand) == -(-got_bytes // VS._MIB)
+ assert got_bytes <= want
+
+
[email protected]("component", ["text_encoder", "transformer"])
+def test_staged_entry_reads_bit_identical_to_the_dense_load(pipe_dir, component):
+ cand = _cands(pipe_dir)[component]
+ assert VS.stage_on_device(cand, "cpu")
+ hit = CC.read(_cands(pipe_dir)[component])
+ assert hit is not None
+ _identical(hit, _dense(cand))
+ if component == "text_encoder":
+ assert hit.encoder.block[0].layer[1].DenseReluDense.wo.weight.dtype == torch.float32
+ assert hit.shared.weight.dtype == torch.float16
+
+
+def _wire(
+ monkeypatch,
+ available,
+ total = 13000,
+ free = 14600,
+):
+ from core.inference import diffusion_small_host as SH
+
+ calls = []
+ real = VS.stage_on_device
+
+ def fake_stage(cand, device, **kw):
+ calls.append((cand.component, str(device)))
+ return real(cand, "cpu", **kw)
+
+ monkeypatch.setattr(VS, "stage_on_device", fake_stage)
+ monkeypatch.setattr(VS, "_device_free_mib", lambda device: free)
+ monkeypatch.setattr(VS, "_STAGE_MIN_MIB", 0)
+ monkeypatch.setattr(SH, "host_ram_mib", lambda: (total, available))
+ return calls
+
+
+def test_video_reader_stages_then_serves_the_encoder_on_a_small_host(pipe_dir, monkeypatch):
+ from core.inference import video as V
+
+ sizes = {"text_encoder": 9000, "transformer": 2700}
+ monkeypatch.setattr(VS, "loaded_mib", lambda cand: sizes[cand.component])
+ calls = _wire(monkeypatch, available = 10000)
+ pipe_kwargs = {"torch_dtype": dict(FP16_DEFAULT), "cache_dir": None, "local_files_only": True}
+ missed = V._read_cast_cache(
+ pipe_kwargs, source = str(pipe_dir), revision = None, base = None, device = "cuda"
+ )
+ assert calls == [("text_encoder", "cuda")]
+ assert [c.component for c in missed] == ["transformer"]
+ _identical(pipe_kwargs["text_encoder"], _dense(_cands(pipe_dir)["text_encoder"]))
+
+
+def test_video_reader_on_a_big_host_never_stages(pipe_dir, monkeypatch):
+ from core.inference import video as V
+
+ calls = _wire(monkeypatch, available = 48000, total = 52000)
+ pipe_kwargs = {"torch_dtype": dict(FP16_DEFAULT), "cache_dir": None, "local_files_only": True}
+ missed = V._read_cast_cache(
+ pipe_kwargs, source = str(pipe_dir), revision = None, base = None, device = "cuda"
+ )
+ assert calls == []
+ assert sorted(c.component for c in missed) == ["text_encoder", "transformer"]
+ assert "text_encoder" not in pipe_kwargs
+
+
+def test_video_reader_kill_switch(pipe_dir, monkeypatch):
+ from core.inference import video as V
+
+ monkeypatch.setenv(VS.SMALL_HOST_ENV, "0")
+ calls = _wire(monkeypatch, available = 10)
+ pipe_kwargs = {"torch_dtype": dict(FP16_DEFAULT), "cache_dir": None, "local_files_only": True}
+ V._read_cast_cache(pipe_kwargs, source = str(pipe_dir), revision = None, base = None, device = "cuda")
+ assert calls == []
+
+
+def test_non_cuda_device_never_stages(pipe_dir, monkeypatch):
+ calls = _wire(monkeypatch, available = 10)
+ assert VS.stage_misses(list(_cands(pipe_dir).values()), "cpu") == []
+ assert calls == []