Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
Studio diffusion: download a partially cached snapshot before sizing it for small hosts (#16080)
Daniel Hana00356eecf58
diff --git a/studio/backend/core/inference/diffusion.py b/studio/backend/core/inference/diffusion.pyindex 99afcfee4bc..0612bc58128 100644--- a/studio/backend/core/inference/diffusion.py+++ b/studio/backend/core/inference/diffusion.py@@ -175,6 +175,7 @@ from .diffusion_small_host import (host_commit_mib,host_ram_mib,resolve_snapshot_dir,+ snapshot_weights_complete,spill_capacity_mib,stored_components,torch_dtype_map as small_host_torch_dtype_map,@@ -9322,6 +9323,17 @@ class DiffusionBackend:self._stop_load_prefetch()else:# bf16 -> fp16 conversion materialises in host RAM (35 GB for FLUX.1)+ # Re-judged on current pipe_kwargs: a since-supplied component can complete the twins.+ _decision_variant = (+ local_fp16_variant(+ _variant_snapshot,+ load_dtype = dtype,+ torch_dtype = pipe_kwargs.get("torch_dtype"),+ supplied = tuple(pipe_kwargs),+ )+ if _fp16_variant_eligible+ else None+ )small_host = self._small_host_decision(_base_local_dir or fetch_base,pipe_kwargs,@@ -9331,10 +9343,13 @@ class DiffusionBackend:fetch = _small_host_fetch(pipeline_cls,fetch_base,- pipe_kwargs,+ {**pipe_kwargs, "variant": _decision_variant}+ if _decision_variant is not None+ else pipe_kwargs,_base_local_dir,local_files_only,),+ variant = _decision_variant,)if small_host is not None and small_host.engaged:pipe_kwargs["torch_dtype"] = small_host_torch_dtype_map(@@ -12064,14 +12079,21 @@ class DiffusionBackend:*,lora_active: bool,fetch: Optional[Callable[[], Any]] = None,+ variant: Optional[str] = None,) -> Any:"""Small-host decision for a pipeline load; raises its refusal, None without a local snapshot.- ``fetch`` downloads what ``from_pretrained`` would when the snapshot holds no weights yet."""+ ``fetch`` downloads what ``from_pretrained`` would when the snapshot holds no weights yet; ``variant``+ is the weight layout the load reads."""snapshot = resolve_snapshot_dir(source, hub_cache_dir())if (fetch is not Noneand small_host_download_first()- and (snapshot is None or not stored_components(snapshot))+ and (+ snapshot is None+ or not stored_components(snapshot)+ # interrupted download: sizing only the cached part hides missing components+ or not snapshot_weights_complete(snapshot, skip = tuple(pipe_kwargs), variant = variant)+ )):fetched = fetch()snapshot = resolve_snapshot_dir(fetched, None) if fetched else snapshotdiff --git a/studio/backend/core/inference/diffusion_small_host.py b/studio/backend/core/inference/diffusion_small_host.pyindex 13f2fe2fced..5025bef1410 100644--- a/studio/backend/core/inference/diffusion_small_host.py+++ b/studio/backend/core/inference/diffusion_small_host.py@@ -263,6 +263,50 @@ def _weight_files(sub: Path) -> list[Path]:return plain or files+def snapshot_weights_complete(+ snapshot: Any,+ skip: Any = (),+ variant: Optional[str] = None,+) -> bool:+ """False when a ``model_index.json`` component lacks weights or index-listed shards; True without an index.+ ``skip`` names caller-supplied components; ``variant`` checks only that layout's files."""+ try:+ root = Path(snapshot)+ index = json.loads((root / "model_index.json").read_text(encoding = "utf-8"))+ except Exception: # noqa: BLE001+ return True+ for name, entry in index.items():+ if name.startswith("_") or name in skip:+ continue+ if not (isinstance(entry, list) and len(entry) >= 2 and entry[0] and entry[1]):+ continue+ sub = root / name+ if not sub.is_dir():+ return False+ files = list(sub.glob("*.safetensors")) + list(sub.glob("*.bin"))+ shard_indexes = list(sub.glob("*.index.json"))+ if variant:+ marks = (f".{variant}.", f".{variant}-")+ files = [f for f in files if any(m in f.name for m in marks)]+ shard_indexes = [+ f for f in sub.glob("*.index*.json") if any(m in f.name for m in marks)+ ]+ if not files and (sub / "config.json").is_file():+ return False # a model folder without weights; tokenizer / scheduler folders have no config.json+ for shard_index in shard_indexes:+ try:+ shards = set(+ json.loads(shard_index.read_text(encoding = "utf-8"))+ .get("weight_map", {})+ .values()+ )+ except Exception: # noqa: BLE001+ return False+ if any(not (sub / shard).is_file() for shard in shards):+ return False+ return True++def stored_components(snapshot: Any) -> dict[str, StoredComponent]:"""Per component: stored MiB and dominant dtype, from safetensors headers only."""out: dict[str, StoredComponent] = {}diff --git a/studio/backend/tests/test_diffusion_small_host.py b/studio/backend/tests/test_diffusion_small_host.pyindex fc15a0d1cba..204558ee75c 100644--- a/studio/backend/tests/test_diffusion_small_host.py+++ b/studio/backend/tests/test_diffusion_small_host.py@@ -558,3 +558,200 @@ def test_small_host_fetch_only_for_an_online_unstaged_hub_load():assert dif._small_host_fetch(_Pipe, "org/model", kw, None, True) is Noneassert dif._small_host_fetch(_Pipe, "org/model", kw, "/staged", False) is Noneassert dif._small_host_fetch(object, "org/model", kw, None, False) is None+++def _header_only(+ root,+ name,+ mib,+ filename = "diffusion_pytorch_model.safetensors",+):+ sub = root / name+ sub.mkdir(parents = True, exist_ok = True)+ raw = json.dumps(+ {+ "weight": {+ "dtype": "BF16",+ "shape": [mib * 1024 * 1024 // 2],+ "data_offsets": [0, mib * 1024 * 1024],+ }+ }+ ).encode()+ (sub / filename).write_bytes(struct.pack("<Q", len(raw)) + raw)++[email protected]("cached", ["configs_only", "vae_only", "one_dit_shard"])+def test_a_partially_cached_snapshot_downloads_before_the_decision(tmp_path, monkeypatch, cached):+ """An interrupted download leaves some weights cached; sizing them alone hides the large denoiser."""+ from core.inference import diffusion as dif++ monkeypatch.delenv(sh.DOWNLOAD_FIRST_ENV, raising = False)+ partial, full = tmp_path / "partial", tmp_path / "full"+ index = {+ "_class_name": "Flux2KleinPipeline",+ "scheduler": ["diffusers", "FlowMatchEulerDiscreteScheduler"],+ "transformer": ["diffusers", "Flux2Transformer2DModel"],+ "text_encoder": ["transformers", "Qwen3Model"],+ "vae": ["diffusers", "AutoencoderKLFlux2"],+ }+ for root in (partial, full):+ root.mkdir()+ (root / "model_index.json").write_text(json.dumps(index))+ (root / "scheduler").mkdir()+ (root / "scheduler" / "scheduler_config.json").write_text("{}")+ _header_only(full, "transformer", 17317)+ _header_only(full, "text_encoder", 15623, "model.safetensors")+ _header_only(full, "vae", 161)+ if cached == "vae_only":+ _header_only(partial, "vae", 161)+ if cached == "one_dit_shard":+ _header_only(+ partial, "transformer", 4096, "diffusion_pytorch_model-00001-of-00005.safetensors"+ )+ (partial / "transformer" / "diffusion_pytorch_model.safetensors.index.json").write_text(+ json.dumps(+ {+ "weight_map": {+ f"w{i}": f"diffusion_pytorch_model-0000{i}-of-00005.safetensors"+ for i in range(1, 6)+ }+ }+ )+ )+ monkeypatch.setattr(dif, "host_ram_mib", lambda: (12977, 11161))+ monkeypatch.setattr(dif, "host_commit_mib", lambda: None)+ monkeypatch.setattr(dif, "pread_available", lambda: False)+ monkeypatch.setattr(dif, "spill_capacity_mib", lambda: 204800)+ calls = []++ def fetch():+ calls.append(1)+ return str(full)++ d = dif.DiffusionBackend()._small_host_decision(+ str(partial),+ {},+ types.SimpleNamespace(device = "cuda"),+ torch.float16,+ lora_active = False,+ fetch = fetch,+ )+ assert calls == [1]+ assert d.engaged and d.int8_spill+++def test_a_complete_snapshot_with_a_supplied_component_skips_the_download(tmp_path):+ index = {+ "transformer": ["diffusers", "X"],+ "vae": ["diffusers", "Y"],+ "tokenizer": ["transformers", "T"],+ }+ (tmp_path / "model_index.json").write_text(json.dumps(index))+ _header_only(tmp_path, "vae", 1)+ (tmp_path / "tokenizer").mkdir()+ (tmp_path / "tokenizer" / "tokenizer_config.json").write_text("{}")+ assert sh.snapshot_weights_complete(tmp_path, skip = ("transformer",))+ assert not sh.snapshot_weights_complete(tmp_path)+++def test_complete_fp16_twins_with_an_orphan_default_index_skip_the_download(tmp_path, monkeypatch):+ # A cached fp16 load reads the twins: the absent default shards an orphan index lists are not owed.+ from core.inference import diffusion as dif++ monkeypatch.delenv(sh.DOWNLOAD_FIRST_ENV, raising = False)+ index = {"transformer": ["diffusers", "X"], "vae": ["diffusers", "Y"]}+ (tmp_path / "model_index.json").write_text(json.dumps(index))+ _header_only(tmp_path, "transformer", 1, "diffusion_pytorch_model.fp16.safetensors")+ _header_only(tmp_path, "vae", 1, "diffusion_pytorch_model.fp16.safetensors")+ (tmp_path / "transformer" / "diffusion_pytorch_model.safetensors.index.json").write_text(+ json.dumps({"weight_map": {"w": "diffusion_pytorch_model-00001-of-00002.safetensors"}})+ )+ assert not sh.snapshot_weights_complete(tmp_path)+ assert sh.snapshot_weights_complete(tmp_path, variant = "fp16")+ monkeypatch.setattr(dif, "host_ram_mib", lambda: (64 * 1024, 60 * 1024))+ monkeypatch.setattr(dif, "host_commit_mib", lambda: None)+ monkeypatch.setattr(dif, "pread_available", lambda: False)+ calls = []+ dif.DiffusionBackend()._small_host_decision(+ str(tmp_path),+ {},+ types.SimpleNamespace(device = "cuda"),+ torch.float16,+ lora_active = False,+ fetch = lambda: calls.append(1),+ variant = "fp16",+ )+ assert calls == []+++def test_twins_completed_by_a_supplied_component_skip_the_early_download(tmp_path, monkeypatch):+ """A since-supplied component completes the twins: the early download must not fetch the default weights."""+ import ast+ import inspect++ from core.inference import diffusion as dif+ from core.inference.diffusion_fp16_variant import local_fp16_variant++ monkeypatch.delenv(sh.DOWNLOAD_FIRST_ENV, raising = False)+ index = {+ "transformer": ["diffusers", "SD3Transformer2DModel"],+ "text_encoder": ["transformers", "CLIPTextModel"],+ "vae": ["diffusers", "AutoencoderKL"],+ }+ (tmp_path / "model_index.json").write_text(json.dumps(index))+ _header_only(tmp_path, "transformer", 1, "diffusion_pytorch_model.fp16.safetensors")+ _header_only(tmp_path, "text_encoder", 1, "model.fp16.safetensors")+ _header_only(tmp_path, "vae", 1) # default name only: no twin+ (tmp_path / "transformer" / "diffusion_pytorch_model.safetensors.index.json").write_text(+ json.dumps({"weight_map": {"w": "diffusion_pytorch_model-00001-of-00002.safetensors"}})+ )+ pipe_kwargs = {"torch_dtype": torch.float16, "vae": object()}+ assert (+ local_fp16_variant(str(tmp_path), load_dtype = torch.float16, supplied = tuple(pipe_kwargs))+ == "fp16"+ )++ src = inspect.getsource(dif)+ tree = ast.parse(src)+ call = next(+ n+ for n in ast.walk(tree)+ if isinstance(n, ast.Call)+ and getattr(n.func, "attr", None) == "_small_host_decision"+ and any(k.arg == "variant" for k in n.keywords)+ )+ arg = next(k.value for k in call.keywords if k.arg == "variant")+ assert isinstance(arg, ast.Name)+ assign = max(+ (+ n+ for n in ast.walk(tree)+ if isinstance(n, ast.Assign)+ and n.lineno < call.lineno+ and any(isinstance(t, ast.Name) and t.id == arg.id for t in n.targets)+ ),+ key = lambda n: n.lineno,+ )+ env = {+ "local_fp16_variant": local_fp16_variant,+ "_variant_snapshot": str(tmp_path),+ "dtype": torch.float16,+ "pipe_kwargs": pipe_kwargs,+ "_te_replaced": (),+ "_fp16_variant_eligible": True,+ }+ variant = eval(compile(ast.Expression(assign.value), "<call site>", "eval"), env)+ monkeypatch.setattr(dif, "host_ram_mib", lambda: (64 * 1024, 60 * 1024))+ monkeypatch.setattr(dif, "host_commit_mib", lambda: None)+ monkeypatch.setattr(dif, "pread_available", lambda: False)+ calls = []+ dif.DiffusionBackend()._small_host_decision(+ str(tmp_path),+ pipe_kwargs,+ types.SimpleNamespace(device = "cuda"),+ torch.float16,+ lora_active = False,+ fetch = lambda: calls.append(1),+ variant = variant,+ )+ assert variant == "fp16" and calls == []