Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
Studio video: stage an fp32 encoder on the card when its dense cast would not fit host RAM (#15975)
Daniel Hand3702be73973
diff --git a/studio/backend/core/inference/video.py b/studio/backend/core/inference/video.pyindex d9c0c036e06..6139c6769d2 100644--- a/studio/backend/core/inference/video.py+++ b/studio/backend/core/inference/video.py@@ -3159,8 +3159,10 @@ def _read_cast_cache(revision: Optional[str],base: Optional[str],cancel_check: Any = None,+ device: Any = None,) -> list:- """Inject cached float16 / bfloat16 casts into ``pipe_kwargs``; returns the unserved candidates. Never raises."""+ """Inject cached float16 / bfloat16 casts into ``pipe_kwargs``; returns the unserved candidates. Never raises.+ ``device``: a miss whose dense conversion would not fit host RAM is first staged there (``video_small_host``)."""try:from .diffusion_cast_cache import candidates, read_all@@ -3177,6 +3179,9 @@ def _read_cast_cache()if not cands:return []+ if device is not None:+ from .video_small_host import stage_misses+ stage_misses(cands, device, logger = logger, cancel_check = cancel_check)hits = read_all(cands, logger = logger, cancel_check = cancel_check)pipe_kwargs.update(hits)if hits:@@ -8974,6 +8979,7 @@ class VideoBackend:revision = None if _base_local_dir else _base_fetch_revision,base = base,cancel_check = lambda: self._raise_if_video_load_superseded(_load_token),+ device = target.torch_device if target.device == "cuda" else None,)# The pre-downloaded snapshot dir keeps from_pretrained off the hub; hub id when pre-download was skipped.pipe = pipeline_cls.from_pretrained(diff --git a/studio/backend/core/inference/video_small_host.py b/studio/backend/core/inference/video_small_host.pynew file mode 100644index 00000000000..6045d2539cf--- /dev/null+++ b/studio/backend/core/inference/video_small_host.py@@ -0,0 +1,262 @@+# SPDX-License-Identifier: AGPL-3.0-only+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0++"""Stage cast-cache misses too big for host RAM (e.g. fp32 UMT5 on a 12.7 GB host gets OOM-killed) onto the card.++Each such miss loads alone onto the device (transformers streams per tensor), is written to the cast cache from the+device tensors and freed; the normal cast-cache read then serves it, same bits. ``UNSLOTH_VIDEO_SMALL_HOST=0`` off,+``=1`` stages every miss that fits the card.+"""++from __future__ import annotations++import gc+import json+import os+import re+import time+from dataclasses import dataclass, field+from pathlib import Path+from typing import Any, Callable, Optional++SMALL_HOST_ENV = "UNSLOTH_VIDEO_SMALL_HOST"+_MIB = 1024 * 1024+# device bytes the staged load needs above its own weights (CUDA context grows, allocator rounding)+_DEVICE_MARGIN_MIB = 768+# a candidate this small converts in RAM even on a small host+_STAGE_MIN_MIB = 512+++def _env() -> str:+ return (os.environ.get(SMALL_HOST_ENV) or "").strip().lower()+++def stage_disabled() -> bool:+ return _env() in ("0", "off", "false", "no")+++def stage_forced() -> bool:+ return _env() in ("1", "on", "true", "yes", "force")+++def _keep_regex(patterns: Any) -> Optional[Any]:+ pats = [str(p) for p in (patterns or ()) if p]+ if not pats:+ return None+ # transformers' own rule (modeling_utils: ``(^|\.)module($|\.)``)+ return re.compile("|".join(rf"((^|\.){re.escape(p)}($|\.))" for p in pats))+++def loaded_mib(cand: Any) -> Optional[int]:+ """MiB ``cand`` occupies once loaded at its half dtype: fp32 tensors halve except ``_keep_in_fp32_modules``."""+ from .diffusion_cast_cache import _shard_header++ keep = _keep_regex((getattr(cand, "parts", None) or {}).get("keep_in_fp32"))+ total = 0+ for shard in getattr(cand, "shards", None) or ():+ header = _shard_header(shard)+ if header is None:+ return None+ for name, entry in header.items():+ if not isinstance(entry, dict):+ continue+ try:+ begin, end = (int(v) for v in entry.get("data_offsets") or (0, 0))+ except (TypeError, ValueError):+ return None+ size = end - begin+ tag = entry.get("dtype")+ if tag == "F64":+ size = size // 4 if keep is None or not keep.search(name) else size // 2+ elif tag == "F32" and (keep is None or not keep.search(name)):+ size //= 2+ total += size+ return -(-total // _MIB)+++@dataclass(frozen = True)+class StagePlan:+ engaged: bool+ reason: str+ stage: tuple = ()+ dense_mib: int = 0+ available_mib: Optional[int] = None+ reserve_mib: int = 0+ skipped: dict = field(default_factory = dict)+++def plan_stage(+ misses: dict,+ *,+ available_mib: Optional[int],+ reserve_mib: int,+ device_free_mib: Optional[int],+ forced: bool = False,+ disabled: bool = False,+) -> StagePlan:+ """``misses``: component -> loaded MiB. Stage largest first (each must fit the card) until the rest fits host RAM."""+ if disabled:+ return StagePlan(False, f"{SMALL_HOST_ENV}=0")+ sizes = {k: int(v) for k, v in misses.items() if v is not None}+ dense = sum(sizes.values())+ if not sizes:+ return StagePlan(False, "no cast-cache miss to convert", dense_mib = 0)+ if available_mib is None and not forced:+ return StagePlan(False, "host RAM unreadable", dense_mib = dense)+ fits = available_mib is not None and dense + reserve_mib <= int(available_mib)+ if fits and not forced:+ return StagePlan(+ False,+ f"dense conversion ({dense} MiB) fits {available_mib} MiB available host RAM",+ dense_mib = dense,+ available_mib = available_mib,+ reserve_mib = reserve_mib,+ )+ stage = []+ skipped = {}+ left = dense+ for name, mib in sorted(sizes.items(), key = lambda kv: (-kv[1], kv[0])):+ if not forced and available_mib is not None and left + reserve_mib <= int(available_mib):+ break+ if mib < _STAGE_MIN_MIB:+ skipped[name] = "small"+ continue+ if device_free_mib is None or mib + _DEVICE_MARGIN_MIB > int(device_free_mib):+ skipped[name] = (+ f"needs {mib + _DEVICE_MARGIN_MIB} MiB of the card, {device_free_mib} MiB free"+ )+ continue+ stage.append(name)+ left -= mib+ why = (+ f"{SMALL_HOST_ENV}=1"+ if forced+ else f"dense conversion needs ~{dense} MiB + {reserve_mib} MiB reserve of host RAM, "+ f"{available_mib} MiB available"+ )+ return StagePlan(bool(stage), why, tuple(stage), dense, available_mib, reserve_mib, skipped)+++def _component_dir(cand: Any) -> str:+ return str(Path(cand.config_path).parent)+++def _device_load_kwargs(cand: Any, device: Any) -> dict:+ from .diffusion_cast_cache import _load_kwargs++ kwargs = _load_kwargs(cand)+ kwargs.pop("output_loading_info", None)+ kwargs["device_map"] = {"": str(device)}+ return kwargs+++def stage_on_device(+ cand: Any,+ device: Any,+ *,+ logger: Any = None,+ cancel_check: Optional[Callable[[], None]] = None,+) -> bool:+ """Load ``cand`` alone onto ``device``, write its cast-cache entry from the device tensors, free it."""+ import torch++ from .diffusion_cast_cache import _state_names, save++ t0 = time.perf_counter()+ module = None+ try:+ module = cand.cls.from_pretrained(_component_dir(cand), **_device_load_kwargs(cand, device))+ module.eval()+ if cancel_check is not None:+ cancel_check()+ names = _state_names(module)+ if not names:+ return False+ cand.names = names+ path = save(cand, module, logger = logger, check = cancel_check)+ if logger is not None:+ logger.info(+ "video.small_host: %s staged on %s in %.1f s (%s)",+ cand.component,+ device,+ time.perf_counter() - t0,+ "cast cache written" if path else "not written",+ )+ return bool(path)+ finally:+ module = None+ gc.collect()+ try:+ if torch.cuda.is_available():+ torch.cuda.empty_cache()+ except Exception: # noqa: BLE001+ pass+++def _device_free_mib(device: Any) -> Optional[int]:+ try:+ import torch++ dev = torch.device(device)+ if dev.type != "cuda" or not torch.cuda.is_available():+ return None+ free, _total = torch.cuda.mem_get_info(dev)+ return int(free // _MIB)+ except Exception: # noqa: BLE001+ return None+++def stage_misses(+ cands: list,+ device: Any,+ *,+ logger: Any = None,+ cancel_check: Optional[Callable[[], None]] = None,+) -> list:+ """Stage the misses of ``cands`` a dense load would not fit; returns the staged components. Never raises except+ a cancel."""+ if not cands or device is None:+ return []+ try:+ import torch++ if torch.device(device).type != "cuda":+ return []+ from .diffusion_cast_cache import cached_components+ from .diffusion_small_host import host_ram_mib, host_reserve_mib++ cached = set(cached_components(cands))+ misses = {c.component: loaded_mib(c) for c in cands if c.component not in cached}+ total, available = host_ram_mib()+ plan = plan_stage(+ misses,+ available_mib = available,+ reserve_mib = host_reserve_mib(total),+ device_free_mib = _device_free_mib(device),+ forced = stage_forced(),+ disabled = stage_disabled(),+ )+ if logger is not None and (plan.engaged or plan.skipped or misses):+ logger.info(+ "video.small_host: engaged=%s (%s); stage=%s%s",+ plan.engaged,+ plan.reason,+ list(plan.stage),+ f" skipped={json.dumps(plan.skipped)}" if plan.skipped else "",+ )+ done = []+ by_name = {c.component: c for c in cands}+ for name in plan.stage:+ if cancel_check is not None:+ cancel_check()+ if stage_on_device(by_name[name], device, logger = logger, cancel_check = cancel_check):+ done.append(name)+ return done+ except Exception as exc: # noqa: BLE001 - the dense load still runs+ if cancel_check is not None:+ cancel_check()+ if logger is not None:+ logger.warning(+ "video.small_host: staging skipped (%s: %s)", type(exc).__name__, str(exc)[:200]+ )+ return []diff --git a/studio/backend/tests/test_video_small_host.py b/studio/backend/tests/test_video_small_host.pynew file mode 100644index 00000000000..390bbff94bb--- /dev/null+++ b/studio/backend/tests/test_video_small_host.py@@ -0,0 +1,267 @@+# SPDX-License-Identifier: AGPL-3.0-only+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0++"""Video small-host staging: routing, kill switch, bit-identical staged entry."""++from __future__ import annotations++import json+import sys+from pathlib import Path++import pytest++torch = pytest.importorskip("torch")+diffusers = pytest.importorskip("diffusers")+transformers = pytest.importorskip("transformers")+pytest.importorskip("safetensors")++BACKEND = Path(__file__).resolve().parents[1]+if str(BACKEND) not in sys.path:+ sys.path.insert(0, str(BACKEND))++from core.inference import diffusion_cast_cache as CC # noqa: E402+from core.inference import diffusion_prequant_cache as C # noqa: E402+from core.inference import video_small_host as VS # noqa: E402++FP16_DEFAULT = {"vae": torch.float32, "default": torch.float16}+# Wan2.1-T2V-1.3B on a Colab T4: UMT5 13.4 GB loaded (wo kept fp32), DiT 2.8 GB; 12.7 GB host, ~9.9 GB available+WAN13 = {"text_encoder": 12748, "transformer": 2709}++[email protected](autouse = True)+def _clean_env(tmp_path, monkeypatch):+ monkeypatch.setenv(C.ENV_DIR, str(tmp_path / "cache"))+ monkeypatch.delenv(CC.CAST_CACHE_ENV, raising = False)+ monkeypatch.delenv(C.ENV_MODE, raising = False)+ monkeypatch.delenv(VS.SMALL_HOST_ENV, raising = False)+ monkeypatch.setattr(C, "_hub_source_stale", lambda shards, logger = None: False)+ with C._lock:+ C._REJECTED.clear()+++def test_big_host_is_unchanged():+ plan = VS.plan_stage(WAN13, available_mib = 48000, reserve_mib = 5300, device_free_mib = 22000)+ assert not plan.engaged and plan.stage == ()+ assert "fits" in plan.reason+++def test_small_t4_host_stages_the_encoder_only():+ plan = VS.plan_stage(WAN13, available_mib = 10100, reserve_mib = 3072, device_free_mib = 14600)+ assert plan.engaged+ # the DiT's dense conversion (2.7 GB) fits once the encoder is off the host+ assert plan.stage == ("text_encoder",)+ assert plan.dense_mib == sum(WAN13.values())+++def test_stages_largest_first_until_the_rest_fits():+ misses = {"text_encoder": 9000, "transformer": 6000, "transformer_2": 6000}+ plan = VS.plan_stage(misses, available_mib = 12000, reserve_mib = 3000, device_free_mib = 40000)+ assert plan.stage == ("text_encoder", "transformer")+++def test_card_too_small_for_the_encoder_keeps_the_dense_load():+ plan = VS.plan_stage(WAN13, available_mib = 10100, reserve_mib = 3072, device_free_mib = 11000)+ # the encoder does not fit the card; the DiT alone does but cannot rescue the load: staged anyway, logged+ assert "text_encoder" in plan.skipped+ assert plan.stage == ("transformer",)+++def test_kill_switch_and_force(monkeypatch):+ monkeypatch.setenv(VS.SMALL_HOST_ENV, "0")+ assert VS.stage_disabled()+ plan = VS.plan_stage(+ WAN13,+ available_mib = 100,+ reserve_mib = 3072,+ device_free_mib = 14600,+ disabled = VS.stage_disabled(),+ )+ assert not plan.engaged+ monkeypatch.setenv(VS.SMALL_HOST_ENV, "1")+ plan = VS.plan_stage(+ WAN13,+ available_mib = 48000,+ reserve_mib = 5300,+ device_free_mib = 22000,+ forced = VS.stage_forced(),+ )+ assert plan.stage == ("text_encoder", "transformer")+++def test_unreadable_host_ram_changes_nothing():+ plan = VS.plan_stage(WAN13, available_mib = None, reserve_mib = 3072, device_free_mib = 14600)+ assert not plan.engaged+++def test_small_candidates_are_never_staged():+ plan = VS.plan_stage({"vae": 200}, available_mib = 100, reserve_mib = 3072, device_free_mib = 14600)+ assert not plan.engaged and plan.skipped == {"vae": "small"}+++def _tiny_umt5():+ torch.manual_seed(1)+ config = transformers.UMT5Config(+ vocab_size = 64,+ d_model = 32,+ d_kv = 8,+ d_ff = 64,+ num_layers = 2,+ num_heads = 4,+ relative_attention_num_buckets = 8,+ )+ return transformers.UMT5EncoderModel(config).to(torch.float32)+++def _tiny_wan():+ torch.manual_seed(0)+ return diffusers.WanTransformer3DModel(+ patch_size = (1, 2, 2),+ num_attention_heads = 2,+ attention_head_dim = 16,+ in_channels = 4,+ out_channels = 4,+ text_dim = 32,+ freq_dim = 16,+ ffn_dim = 64,+ num_layers = 2,+ cross_attn_norm = True,+ qk_norm = "rms_norm_across_heads",+ eps = 1e-6,+ rope_max_seq_len = 32,+ ).to(torch.float32)+++def pipe_dir(tmp_path):+ root = tmp_path / "Wan2.1-T2V-tiny-Diffusers"+ _tiny_wan().save_pretrained(root / "transformer", max_shard_size = "60KB")+ _tiny_umt5().save_pretrained(root / "text_encoder")+ (root / "vae").mkdir()+ (root / "model_index.json").write_text(+ json.dumps(+ {+ "_class_name": "WanPipeline",+ "transformer": ["diffusers", "WanTransformer3DModel"],+ "text_encoder": ["transformers", "UMT5EncoderModel"],+ "vae": ["diffusers", "AutoencoderKLWan"],+ "tokenizer": ["transformers", "T5TokenizerFast"],+ "scheduler": ["diffusers", "UniPCMultistepScheduler"],+ }+ )+ )+ return root+++def _cands(pipe_dir):+ return {c.component: c for c in CC.candidates(pipe_dir, torch_dtype = FP16_DEFAULT)}+++def _dense(cand):+ kwargs = CC._load_kwargs(cand)+ kwargs.pop("output_loading_info")+ return cand.cls.from_pretrained(str(Path(cand.source) / cand.component), **kwargs)+++def _identical(a, b):+ sa, sb = a.state_dict(), b.state_dict()+ assert sa.keys() == sb.keys()+ for k in sa:+ assert sa[k].dtype == sb[k].dtype, k+ assert sa[k].device.type == "cpu" and sb[k].device.type == "cpu", k+ assert torch.equal(+ sa[k].reshape(-1).view(torch.uint8), sb[k].reshape(-1).view(torch.uint8)+ ), k+++def test_loaded_mib_keeps_wo_wide(pipe_dir):+ cand = _cands(pipe_dir)["text_encoder"]+ assert cand.parts["keep_in_fp32"] == ["wo"]+ dense = _dense(cand)+ want = sum(t.numel() * t.element_size() for t in dense.state_dict().values())+ # tied embed_tokens is stored once; the estimate counts stored tensors at their loaded width+ got_bytes = 0+ for shard in cand.shards:+ for name, e in CC._shard_header(shard).items():+ b, end = e["data_offsets"]+ got_bytes += (end - b) if ".wo." in f".{name}" else (end - b) // 2+ assert VS.loaded_mib(cand) == -(-got_bytes // VS._MIB)+ assert got_bytes <= want++[email protected]("component", ["text_encoder", "transformer"])+def test_staged_entry_reads_bit_identical_to_the_dense_load(pipe_dir, component):+ cand = _cands(pipe_dir)[component]+ assert VS.stage_on_device(cand, "cpu")+ hit = CC.read(_cands(pipe_dir)[component])+ assert hit is not None+ _identical(hit, _dense(cand))+ if component == "text_encoder":+ assert hit.encoder.block[0].layer[1].DenseReluDense.wo.weight.dtype == torch.float32+ assert hit.shared.weight.dtype == torch.float16+++def _wire(+ monkeypatch,+ available,+ total = 13000,+ free = 14600,+):+ from core.inference import diffusion_small_host as SH++ calls = []+ real = VS.stage_on_device++ def fake_stage(cand, device, **kw):+ calls.append((cand.component, str(device)))+ return real(cand, "cpu", **kw)++ monkeypatch.setattr(VS, "stage_on_device", fake_stage)+ monkeypatch.setattr(VS, "_device_free_mib", lambda device: free)+ monkeypatch.setattr(VS, "_STAGE_MIN_MIB", 0)+ monkeypatch.setattr(SH, "host_ram_mib", lambda: (total, available))+ return calls+++def test_video_reader_stages_then_serves_the_encoder_on_a_small_host(pipe_dir, monkeypatch):+ from core.inference import video as V++ sizes = {"text_encoder": 9000, "transformer": 2700}+ monkeypatch.setattr(VS, "loaded_mib", lambda cand: sizes[cand.component])+ calls = _wire(monkeypatch, available = 10000)+ pipe_kwargs = {"torch_dtype": dict(FP16_DEFAULT), "cache_dir": None, "local_files_only": True}+ missed = V._read_cast_cache(+ pipe_kwargs, source = str(pipe_dir), revision = None, base = None, device = "cuda"+ )+ assert calls == [("text_encoder", "cuda")]+ assert [c.component for c in missed] == ["transformer"]+ _identical(pipe_kwargs["text_encoder"], _dense(_cands(pipe_dir)["text_encoder"]))+++def test_video_reader_on_a_big_host_never_stages(pipe_dir, monkeypatch):+ from core.inference import video as V++ calls = _wire(monkeypatch, available = 48000, total = 52000)+ pipe_kwargs = {"torch_dtype": dict(FP16_DEFAULT), "cache_dir": None, "local_files_only": True}+ missed = V._read_cast_cache(+ pipe_kwargs, source = str(pipe_dir), revision = None, base = None, device = "cuda"+ )+ assert calls == []+ assert sorted(c.component for c in missed) == ["text_encoder", "transformer"]+ assert "text_encoder" not in pipe_kwargs+++def test_video_reader_kill_switch(pipe_dir, monkeypatch):+ from core.inference import video as V++ monkeypatch.setenv(VS.SMALL_HOST_ENV, "0")+ calls = _wire(monkeypatch, available = 10)+ pipe_kwargs = {"torch_dtype": dict(FP16_DEFAULT), "cache_dir": None, "local_files_only": True}+ V._read_cast_cache(pipe_kwargs, source = str(pipe_dir), revision = None, base = None, device = "cuda")+ assert calls == []+++def test_non_cuda_device_never_stages(pipe_dir, monkeypatch):+ calls = _wire(monkeypatch, available = 10)+ assert VS.stage_misses(list(_cands(pipe_dir).values()), "cpu") == []+ assert calls == []