Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

Studio diffusion: a first load downloads the weights before the small-host decision reads them (#15974)

Daniel Han16cfa8a64a93
diff --git a/studio/backend/core/inference/diffusion.py b/studio/backend/core/inference/diffusion.py
index be5a34bac95..e3e41ad5bac 100644
--- a/studio/backend/core/inference/diffusion.py
+++ b/studio/backend/core/inference/diffusion.py
@@ -170,6 +170,7 @@ from .diffusion_flux3 import REFERENCE_AREA as FLUX3_REFERENCE_AREA
from .diffusion_small_host import (
INT8_DENOISER_NAMES,
decide_small_host,
+ download_first_enabled as small_host_download_first,
engaged_on as small_host_engaged_on,
host_commit_mib,
host_ram_mib,
@@ -3338,6 +3339,22 @@ def _peak_model_identity(
return identity
+def _small_host_fetch(
+ pipeline_cls: Any,
+ repo: Any,
+ pipe_kwargs: dict,
+ local_dir: Optional[str],
+ local_files_only: bool,
+) -> Optional[Callable[[], Any]]:
+ """``from_pretrained``'s own download, run before the small-host decision; None offline or with a staged snapshot."""
+ if local_files_only or local_dir or not isinstance(repo, str) or _is_local_path(repo):
+ return None
+ download = getattr(pipeline_cls, "download", None)
+ if not callable(download):
+ return None
+ return lambda: download(repo, **pipe_kwargs)
+
+
def _conditioning_cache_hits(pipe: Any) -> int:
"""Encodes served from the in-memory prompt cache or the persistent conditioning cache so far: a render with a
hit skipped the text-encoder phase, so its peak covers only part of what a new prompt needs."""
@@ -9264,6 +9281,13 @@ class DiffusionBackend:
target,
dtype,
lora_active = _has_active_lora(loras),
+ fetch = _small_host_fetch(
+ pipeline_cls,
+ fetch_base,
+ pipe_kwargs,
+ _base_local_dir,
+ local_files_only,
+ ),
)
if small_host is not None and small_host.engaged:
pipe_kwargs["torch_dtype"] = small_host_torch_dtype_map(
@@ -9583,6 +9607,13 @@ class DiffusionBackend:
target,
dtype,
lora_active = _has_active_lora(loras),
+ fetch = _small_host_fetch(
+ pipeline_cls,
+ fetch_base,
+ pipe_kwargs,
+ _base_local_dir,
+ local_files_only,
+ ),
)
if small_host is not None and small_host.engaged:
pipe_kwargs["torch_dtype"] = small_host_torch_dtype_map(
@@ -11974,10 +12005,30 @@ class DiffusionBackend:
return unified_memory_shortfall_message(plan, family = fam.name)
def _small_host_decision(
- self, source: Any, pipe_kwargs: dict, target: Any, dtype: Any, *, lora_active: bool
+ self,
+ source: Any,
+ pipe_kwargs: dict,
+ target: Any,
+ dtype: Any,
+ *,
+ lora_active: bool,
+ fetch: Optional[Callable[[], Any]] = None,
) -> Any:
- """Small-host decision for a pipeline load; raises its refusal, None without a local snapshot."""
+ """Small-host decision for a pipeline load; raises its refusal, None without a local snapshot.
+ ``fetch`` downloads what ``from_pretrained`` would when the snapshot holds no weights yet."""
snapshot = resolve_snapshot_dir(source, hub_cache_dir())
+ if (
+ fetch is not None
+ and small_host_download_first()
+ and (snapshot is None or not stored_components(snapshot))
+ ):
+ fetched = fetch()
+ snapshot = resolve_snapshot_dir(fetched, None) if fetched else snapshot
+ if snapshot is not None:
+ logger.info(
+ "diffusion.small_host: downloaded the weights before the host-RAM decision (%s)",
+ snapshot,
+ )
if snapshot is None:
return None
comps = {
diff --git a/studio/backend/core/inference/diffusion_small_host.py b/studio/backend/core/inference/diffusion_small_host.py
index b2c10a678ec..13f2fe2fced 100644
--- a/studio/backend/core/inference/diffusion_small_host.py
+++ b/studio/backend/core/inference/diffusion_small_host.py
@@ -67,6 +67,14 @@ def spill_disabled() -> bool:
return _env(SPILL_ENV) in ("0", "off", "false", "no")
+# A first load fetches the weights before the host-RAM decision, which reads their headers.
+DOWNLOAD_FIRST_ENV = "UNSLOTH_DIFFUSION_SMALL_HOST_DOWNLOAD_FIRST"
+
+
+def download_first_enabled() -> bool:
+ return _env(DOWNLOAD_FIRST_ENV) not in ("0", "off", "false", "no")
+
+
def spill_forced() -> bool:
return _env(SPILL_ENV) in ("1", "on", "true", "yes", "force")
diff --git a/studio/backend/tests/test_diffusion_small_host.py b/studio/backend/tests/test_diffusion_small_host.py
index b00d075c7bc..fc15a0d1cba 100644
--- a/studio/backend/tests/test_diffusion_small_host.py
+++ b/studio/backend/tests/test_diffusion_small_host.py
@@ -479,3 +479,82 @@ def test_lora_refused_and_status_reports_the_int8_route():
types.SimpleNamespace(_unsloth_small_host = {"components": {"transformer": "cast on device"}})
)
assert not _small_host_int8(object())
+
+
+def _decide_with_fetch(
+ tmp_path,
+ monkeypatch,
+ *,
+ weights_cached,
+ fetch_result = "dl",
+):
+ """The pipeline decision on ``tmp_path`` (a config-only or a full snapshot) with a counting ``fetch``."""
+ from core.inference import diffusion as dif
+
+ full = tmp_path / "dl"
+ full.mkdir()
+ monkeypatch.setattr(dif, "host_ram_mib", lambda: (13 * 1024, 11 * 1024))
+ monkeypatch.setattr(dif, "host_commit_mib", lambda: None)
+ monkeypatch.setattr(dif, "pread_available", lambda: False)
+ monkeypatch.setattr(
+ dif,
+ "stored_components",
+ lambda snapshot: dict(QWEN_GGUF_COMPANIONS)
+ if (weights_cached or str(snapshot) == str(full))
+ else {},
+ )
+ calls = []
+
+ def fetch():
+ calls.append(1)
+ return str(full) if fetch_result == "dl" else fetch_result
+
+ d = dif.DiffusionBackend()._small_host_decision(
+ str(tmp_path),
+ {"transformer": object()},
+ types.SimpleNamespace(device = "cuda"),
+ torch.float16,
+ lora_active = False,
+ fetch = fetch,
+ )
+ return d, calls
+
+
+def test_a_config_only_snapshot_downloads_before_the_decision(tmp_path, monkeypatch):
+ monkeypatch.delenv(sh.DOWNLOAD_FIRST_ENV, raising = False)
+ d, calls = _decide_with_fetch(tmp_path, monkeypatch, weights_cached = False)
+ assert calls == [1]
+ assert d is not None and d.engaged
+
+
+def test_cached_weights_decide_without_a_download(tmp_path, monkeypatch):
+ monkeypatch.delenv(sh.DOWNLOAD_FIRST_ENV, raising = False)
+ d, calls = _decide_with_fetch(tmp_path, monkeypatch, weights_cached = True)
+ assert calls == [] and d.engaged
+
+
[email protected]("raw", ["0", "off", "false", "no"])
+def test_download_first_kill_switch(tmp_path, monkeypatch, raw):
+ monkeypatch.setenv(sh.DOWNLOAD_FIRST_ENV, raw)
+ d, calls = _decide_with_fetch(tmp_path, monkeypatch, weights_cached = False)
+ assert calls == []
+ assert not d.engaged
+
+
+def test_small_host_fetch_only_for_an_online_unstaged_hub_load():
+ from core.inference import diffusion as dif
+
+ seen = []
+
+ class _Pipe:
+ @classmethod
+ def download(cls, repo, **kwargs):
+ seen.append((repo, sorted(kwargs)))
+ return "/snap"
+
+ kw = {"torch_dtype": torch.float16, "transformer": object()}
+ fetch = dif._small_host_fetch(_Pipe, "org/model", kw, None, False)
+ assert fetch() == "/snap" and seen == [("org/model", ["torch_dtype", "transformer"])]
+ assert dif._small_host_fetch(_Pipe, "org/model", kw, None, True) is None
+ assert dif._small_host_fetch(_Pipe, "org/model", kw, "/staged", False) is None
+ assert dif._small_host_fetch(object, "org/model", kw, None, False) is None