Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

Studio: keep the loaded model if a GGUF download fails or is cancelled (#15846)

Nilaya1fa6e609d02
diff --git a/studio/backend/core/inference/llama_cpp.py b/studio/backend/core/inference/llama_cpp.py
index 4c42641ef45..7d76e5c201d 100644
--- a/studio/backend/core/inference/llama_cpp.py
+++ b/studio/backend/core/inference/llama_cpp.py
@@ -23178,9 +23178,8 @@ class LlamaCppBackend:
_pv_diffusion_unpinnable = bool(
_paravirtual_cpu_forced and not self.diffusion_split_supported()
)
- _preflight_model_path = None
- self._pending_variant_fallback = None
- if hf_repo and (_vulkan_ordinal_pin or _cpu_only_pin or _pv_diffusion_unpinnable):
+ if hf_repo:
+ # Resolve repo casing once so the GGUF and companions share one cache entry.
_resolved_repo = _resolve_repo_id_casing(hf_repo)
if _resolved_repo != hf_repo:
logger.info(
@@ -23189,6 +23188,9 @@ class LlamaCppBackend:
hf_repo,
)
hf_repo = _resolved_repo
+ _preflight_model_path = None
+ self._pending_variant_fallback = None
+ if hf_repo and (_vulkan_ordinal_pin or _cpu_only_pin or _pv_diffusion_unpinnable):
with _hf_offline_if_unreachable():
_preflight_model_path = self._download_gguf(
hf_repo = hf_repo,
@@ -23267,6 +23269,150 @@ class LlamaCppBackend:
)
raise ValueError(_early_non_chat)
+ # Resolve the canonical MTP mode once; it decides which drafter is downloaded and launched.
+ _spec_canon = _canonicalize_spec_mode(speculative_type) or "auto"
+ # #11308: DFlash2 + --split-mode tensor aborts at startup (ggml-org/llama.cpp#27819) and the
+ # route falls back to layer split, so Auto prefers a loadable MTP sidecar on tensor instead.
+ # _estimate_gguf_required_gb mirrors this.
+ _auto_tensor_split = _spec_canon == "auto" and _effective_tensor_parallel(
+ extra_args, tensor_parallel
+ )
+
+ def _auto_dflash_blocked_by_tensor() -> bool:
+ if (
+ _auto_tensor_split
+ and mtp_draft_path
+ and not _extra_args_mtp_draft_path(extra_args, env = _child_spec_env(extra_args))
+ ):
+ try:
+ from utils.models.gguf_metadata import read_gguf_nextn_predict_layers
+ return bool(
+ _launch_caps(binary).get("mtp_token")
+ and not (read_gguf_nextn_predict_layers(model_path) or 0) > 0
+ and _mtp_drafter_loads_standalone(mtp_draft_path)
+ )
+ except Exception:
+ return False
+ return False
+
+ def _dflash_breaks_on_images() -> bool:
+ if not dflash_draft_path or disable_vision or extra_args_disable_mmproj(extra_args):
+ return False
+ try:
+ projector = self._resolve_launch_mmproj_path(
+ model_path = model_path, mmproj_path = mmproj_path, extra_args = extra_args
+ )
+ except Exception:
+ return False
+ return _dflash_drafter_breaks_on_images(model_path, dflash_draft_path, projector)
+
+ _unloadable_mtp_draft_path: Optional[str] = None
+ # Downloads run before the teardown so a failed or cancelled fetch keeps the resident model;
+ # sidecar fetches write flags on self, so the resident's come back until the teardown commits.
+ _resident_sidecar_flags = (
+ self._dspark_sidecar_absent,
+ self._dflash_sidecar_absent,
+ self._dflash_retry_needed,
+ )
+ self._dflash_retry_needed = False
+ model_path = _preflight_model_path
+ try:
+ if hf_repo:
+ # Scope HF_HUB_OFFLINE to the download block only when DNS is dead.
+ with _hf_offline_if_unreachable():
+ if model_path is None:
+ # Revalidate the carried file before skipping download resolution.
+ model_path = self._verified_cached_gguf(intent, hf_repo, hf_variant)
+ if model_path:
+ logger.info("Reusing verified cached GGUF: %s", model_path)
+ if model_path is None:
+ model_path = self._download_gguf(
+ hf_repo = hf_repo,
+ hf_variant = hf_variant,
+ hf_token = hf_token,
+ cancel_event = download_cancel_event,
+ )
+ # Vision off does not skip the fetch: an mmproj may be an audio encoder instead.
+ if (
+ is_vision
+ and not mmproj_path
+ and not extra_args_disable_mmproj(extra_args)
+ ):
+ mmproj_path = self._download_mmproj(
+ hf_repo = hf_repo,
+ hf_token = hf_token,
+ cancel_event = download_cancel_event,
+ near_path = model_path,
+ )
+ # Fetch the separate MTP drafter; no size gate (helps even sub-3B).
+ # The MTP/ fallback is qwen4exp only, whose GGUFs carry no head.
+ if (
+ not mtp_draft_path
+ and _spec_canon in ("auto", "mtp", "mtp+ngram")
+ and not _extra_args_set_spec_type(extra_args)
+ ):
+ mtp_draft_path = self._download_mtp(
+ hf_repo = hf_repo,
+ hf_token = hf_token,
+ cancel_event = download_cancel_event,
+ near_path = model_path,
+ allow_nested = self._gguf_path_wants_nested_mtp(model_path),
+ )
+ # "auto" is included: DSpark is the default whenever the repo
+ # ships a sidecar. Repos without one no-op, exactly like the
+ # MTP fetch above, so this costs one listing and no download.
+ if (
+ not dspark_draft_path
+ and _spec_canon in ("auto", "dspark")
+ and not _extra_args_set_spec_type(extra_args)
+ ):
+ dspark_draft_path = self._download_dspark(
+ hf_repo = hf_repo,
+ hf_token = hf_token,
+ cancel_event = download_cancel_event,
+ near_path = model_path,
+ binary = binary,
+ caps_probe = _launch_caps,
+ )
+ # Auto fetch stands down once DSpark resolves (it wins the promotion), using the same
+ # capability-aware 'resolved'. An explicit 'dflash' still fetches.
+ if (
+ not dflash_draft_path
+ and _spec_canon in ("auto", "dflash")
+ and not _auto_dflash_blocked_by_tensor()
+ and not self._dspark_wins_auto(
+ binary = binary,
+ dspark_draft_path = dspark_draft_path,
+ spec_canon = _spec_canon,
+ extra_args = extra_args,
+ )
+ and not _extra_args_set_spec_type(extra_args)
+ ):
+ dflash_draft_path = self._download_dflash(
+ hf_repo = hf_repo,
+ hf_token = hf_token,
+ cancel_event = download_cancel_event,
+ near_path = model_path,
+ binary = binary,
+ )
+ elif gguf_path:
+ if not Path(gguf_path).is_file():
+ raise FileNotFoundError(f"GGUF file not found: {gguf_path}")
+ model_path = gguf_path
+ else:
+ raise ValueError("Either gguf_path or hf_repo must be provided")
+ finally:
+ _load_sidecar_flags = (
+ self._dspark_sidecar_absent,
+ self._dflash_sidecar_absent,
+ self._dflash_retry_needed,
+ )
+ (
+ self._dspark_sidecar_absent,
+ self._dflash_sidecar_absent,
+ self._dflash_retry_needed,
+ ) = _resident_sidecar_flags
+
# The probe can take a moment on a slow Hub, so re-read the cancel flag rather
# than tearing down a healthy server for a load that was called off.
if _load_cancelled():
@@ -23306,142 +23452,15 @@ class LlamaCppBackend:
self._variant_fallback_warning = None
# Reset at the teardown, not where the binary resolves: early bails keep the old server.
self._launch_binary_revision = self._binary_revision(binary)
- # Cleared per load: a local-file load never reaches the DFlash fetch.
- self._dflash_retry_needed = False
-
- # ── Phase 2: download (NO lock held, so cancel can proceed) ──
- # Resolve the canonical MTP mode once; it decides which drafter is downloaded and launched.
- _spec_canon = _canonicalize_spec_mode(speculative_type) or "auto"
- # #11308: DFlash2 + --split-mode tensor aborts at startup (ggml-org/llama.cpp#27819) and the
- # route falls back to layer split, so Auto prefers a loadable MTP sidecar on tensor instead.
- # _estimate_gguf_required_gb mirrors this.
- _auto_tensor_split = _spec_canon == "auto" and _effective_tensor_parallel(
- extra_args, tensor_parallel
- )
-
- def _auto_dflash_blocked_by_tensor() -> bool:
- if (
- _auto_tensor_split
- and mtp_draft_path
- and not _extra_args_mtp_draft_path(extra_args, env = _child_spec_env(extra_args))
- ):
- try:
- from utils.models.gguf_metadata import read_gguf_nextn_predict_layers
- return bool(
- _launch_caps(binary).get("mtp_token")
- and not (read_gguf_nextn_predict_layers(model_path) or 0) > 0
- and _mtp_drafter_loads_standalone(mtp_draft_path)
- )
- except Exception:
- return False
- return False
-
- def _dflash_breaks_on_images() -> bool:
- if not dflash_draft_path or disable_vision or extra_args_disable_mmproj(extra_args):
- return False
- try:
- projector = self._resolve_launch_mmproj_path(
- model_path = model_path, mmproj_path = mmproj_path, extra_args = extra_args
- )
- except Exception:
- return False
- return _dflash_drafter_breaks_on_images(model_path, dflash_draft_path, projector)
-
- _unloadable_mtp_draft_path: Optional[str] = None
- # Scope HF_HUB_OFFLINE to the download block only when DNS is dead.
- if hf_repo:
- # Resolve repo casing once so the GGUF and companions share one cache entry.
- _resolved_repo = _resolve_repo_id_casing(hf_repo)
- if _resolved_repo != hf_repo:
- logger.info(
- "Using cached repo_id casing '%s' for requested '%s'",
- _resolved_repo,
- hf_repo,
- )
- hf_repo = _resolved_repo
- with _hf_offline_if_unreachable():
- model_path = _preflight_model_path
- if model_path is None:
- # Revalidate the carried file before skipping download resolution.
- model_path = self._verified_cached_gguf(intent, hf_repo, hf_variant)
- if model_path:
- logger.info("Reusing verified cached GGUF: %s", model_path)
- if model_path is None:
- model_path = self._download_gguf(
- hf_repo = hf_repo,
- hf_variant = hf_variant,
- hf_token = hf_token,
- cancel_event = download_cancel_event,
- )
- if self._pending_variant_fallback:
- hf_variant, self._variant_fallback_warning = self._pending_variant_fallback
- # Replayed by the respawn and a variant-less Apply, which would re-download.
- intent = replace(intent, hf_variant = hf_variant)
- # Vision off does not skip the fetch: an mmproj may be an audio encoder instead.
- if is_vision and not mmproj_path and not extra_args_disable_mmproj(extra_args):
- mmproj_path = self._download_mmproj(
- hf_repo = hf_repo,
- hf_token = hf_token,
- cancel_event = download_cancel_event,
- near_path = model_path,
- )
- # Fetch the separate MTP drafter; no size gate (helps even sub-3B).
- # The MTP/ fallback is qwen4exp only, whose GGUFs carry no head.
- if (
- not mtp_draft_path
- and _spec_canon in ("auto", "mtp", "mtp+ngram")
- and not _extra_args_set_spec_type(extra_args)
- ):
- mtp_draft_path = self._download_mtp(
- hf_repo = hf_repo,
- hf_token = hf_token,
- cancel_event = download_cancel_event,
- near_path = model_path,
- allow_nested = self._gguf_path_wants_nested_mtp(model_path),
- )
- # "auto" is included: DSpark is the default whenever the repo
- # ships a sidecar. Repos without one no-op, exactly like the
- # MTP fetch above, so this costs one listing and no download.
- if (
- not dspark_draft_path
- and _spec_canon in ("auto", "dspark")
- and not _extra_args_set_spec_type(extra_args)
- ):
- dspark_draft_path = self._download_dspark(
- hf_repo = hf_repo,
- hf_token = hf_token,
- cancel_event = download_cancel_event,
- near_path = model_path,
- binary = binary,
- caps_probe = _launch_caps,
- )
- # Auto fetch stands down once DSpark resolves (it wins the promotion), using the same
- # capability-aware 'resolved'. An explicit 'dflash' still fetches.
- if (
- not dflash_draft_path
- and _spec_canon in ("auto", "dflash")
- and not _auto_dflash_blocked_by_tensor()
- and not self._dspark_wins_auto(
- binary = binary,
- dspark_draft_path = dspark_draft_path,
- spec_canon = _spec_canon,
- extra_args = extra_args,
- )
- and not _extra_args_set_spec_type(extra_args)
- ):
- dflash_draft_path = self._download_dflash(
- hf_repo = hf_repo,
- hf_token = hf_token,
- cancel_event = download_cancel_event,
- near_path = model_path,
- binary = binary,
- )
- elif gguf_path:
- if not Path(gguf_path).is_file():
- raise FileNotFoundError(f"GGUF file not found: {gguf_path}")
- model_path = gguf_path
- else:
- raise ValueError("Either gguf_path or hf_repo must be provided")
+ (
+ self._dspark_sidecar_absent,
+ self._dflash_sidecar_absent,
+ self._dflash_retry_needed,
+ ) = _load_sidecar_flags
+ if hf_repo and self._pending_variant_fallback:
+ hf_variant, self._variant_fallback_warning = self._pending_variant_fallback
+ # Replayed by the respawn and a variant-less Apply, which would re-download.
+ intent = replace(intent, hf_variant = hf_variant)
# Set identifier early so _read_gguf_metadata can use it (DeepSeek).
self._model_identifier = model_identifier
diff --git a/studio/backend/tests/test_gpu_memory_mode.py b/studio/backend/tests/test_gpu_memory_mode.py
index 5a4a8ffda7d..cd9682ca986 100644
--- a/studio/backend/tests/test_gpu_memory_mode.py
+++ b/studio/backend/tests/test_gpu_memory_mode.py
@@ -596,9 +596,9 @@ def test_remote_vulkan_diffusion_preflight_runs_before_teardown(monkeypatch):
preflight = src.index("_preflight_model_path = self._download_gguf(")
teardown = src.index("# ── Phase 1: kill old process")
assert preflight < teardown
- phase_two = src.index("model_path = _preflight_model_path", teardown)
- download = src.index("model_path = self._download_gguf(", teardown)
- assert phase_two < download
+ carried = src.index("model_path = _preflight_model_path")
+ download = src.index("model_path = self._download_gguf(", carried)
+ assert preflight < carried < download < teardown
def test_local_vulkan_diffusion_preflight_runs_before_teardown():
diff --git a/studio/backend/tests/test_llama_cpp_remote_non_chat_gguf_preflight.py b/studio/backend/tests/test_llama_cpp_remote_non_chat_gguf_preflight.py
index 38d45e64c56..0cf3ac79131 100644
--- a/studio/backend/tests/test_llama_cpp_remote_non_chat_gguf_preflight.py
+++ b/studio/backend/tests/test_llama_cpp_remote_non_chat_gguf_preflight.py
@@ -290,32 +290,126 @@ def test_a_non_chat_repo_load_keeps_the_resident_model_and_never_downloads(
assert order == []
-def test_a_chat_repo_load_still_proceeds_past_the_teardown(monkeypatch):
- backend = LlamaCppBackend()
- order: list[str] = []
- _repo_load(monkeypatch, backend, order)
+_SIDECARS = ("_download_mmproj", "_download_mtp", "_download_dspark", "_download_dflash")
- def _download(**_kwargs):
- order.append("download")
- raise RuntimeError("stop here")
- monkeypatch.setattr(backend, "_download_gguf", _download)
+def _chat_repo_load(monkeypatch, backend, order, **downloads):
+ _repo_load(monkeypatch, backend, order)
+ monkeypatch.setattr(
+ backend, "_download_gguf", lambda **_kwargs: order.append("download") or "/cache/m.gguf"
+ )
+ for name in _SIDECARS:
+ monkeypatch.setattr(backend, name, lambda **_kwargs: None)
+ for name, fn in downloads.items():
+ monkeypatch.setattr(backend, name, fn)
monkeypatch.setattr(
diffusion_compat,
"_read_gguf_header",
lambda *_a, **_k: _gguf_bytes(arch = "qwen3", declared_kv = 32),
)
- with pytest.raises(RuntimeError, match = "stop here"):
- backend.load_model(
- GgufLoadIntent(
- hf_repo = "unsloth/Qwen3-0.6B-GGUF",
- hf_variant = "Q4_K_M",
- model_identifier = "unsloth/Qwen3-0.6B-GGUF",
- )
+ def _launch(*_a, **_k):
+ order.append("launch")
+ raise RuntimeError("stop here")
+
+ monkeypatch.setattr(backend, "_read_gguf_metadata", _launch)
+ return backend.load_model(
+ GgufLoadIntent(
+ hf_repo = "unsloth/Qwen3-VL-2B-Instruct-GGUF",
+ hf_variant = "Q4_K_M",
+ model_identifier = "unsloth/Qwen3-VL-2B-Instruct-GGUF",
+ is_vision = True,
)
+ )
- assert order == ["kill", "download"]
+
+def _connection_reset(**_kwargs):
+ raise RuntimeError("connection reset")
+
+
+def test_a_chat_repo_load_still_proceeds_past_the_teardown(monkeypatch):
+ order: list[str] = []
+ with pytest.raises(RuntimeError, match = "stop here"):
+ _chat_repo_load(monkeypatch, LlamaCppBackend(), order)
+ assert order == ["download", "kill", "launch"]
+
+
+def test_a_failed_download_leaves_the_resident_model_running(monkeypatch):
+ order: list[str] = []
+
+ def _download(**_kwargs):
+ order.append("download")
+ raise RuntimeError("connection reset")
+
+ with pytest.raises(RuntimeError, match = "connection reset"):
+ _chat_repo_load(monkeypatch, LlamaCppBackend(), order, _download_gguf = _download)
+ assert order == ["download"]
+
+
[email protected]("sidecar", _SIDECARS)
+def test_a_failed_sidecar_download_leaves_the_resident_model_running(monkeypatch, sidecar):
+ order: list[str] = []
+ with pytest.raises(RuntimeError, match = "connection reset"):
+ _chat_repo_load(monkeypatch, LlamaCppBackend(), order, **{sidecar: _connection_reset})
+ assert order == ["download"]
+
+
[email protected]("sidecar", _SIDECARS)
+def test_a_sidecar_download_cancelled_mid_fetch_leaves_the_resident_model_running(
+ monkeypatch, sidecar
+):
+ backend = LlamaCppBackend()
+ order: list[str] = []
+ cancel = {sidecar: lambda **_k: backend._cancel_event.set()}
+ assert _chat_repo_load(monkeypatch, backend, order, **cancel) is False
+ assert order == ["download"]
+
+
+def _sidecar_outcome(backend):
+ def _dspark(**_kwargs):
+ backend._dspark_sidecar_absent = False
+
+ def _dflash(**_kwargs):
+ backend._dflash_sidecar_absent = False
+ backend._dflash_retry_needed = True
+
+ return {"_download_dspark": _dspark, "_download_dflash": _dflash}
+
+
+def _sidecar_flags(backend):
+ return (
+ backend._dspark_sidecar_absent,
+ backend._dflash_sidecar_absent,
+ backend._dflash_retry_needed,
+ )
+
+
+def test_a_failed_load_keeps_the_resident_servers_sidecar_flags(monkeypatch):
+ backend = LlamaCppBackend()
+ backend._dspark_sidecar_absent = backend._dflash_sidecar_absent = True
+ downloads = _sidecar_outcome(backend)
+ fetched = downloads["_download_dflash"]
+
+ def _dflash_then_fail(**kwargs):
+ fetched(**kwargs)
+ raise RuntimeError("connection reset")
+
+ downloads["_download_dflash"] = _dflash_then_fail
+ order: list[str] = []
+ with pytest.raises(RuntimeError, match = "connection reset"):
+ _chat_repo_load(monkeypatch, backend, order, **downloads)
+ assert order == ["download"]
+ assert _sidecar_flags(backend) == (True, True, False)
+
+
+def test_a_committed_load_takes_the_fetched_sidecar_flags(monkeypatch):
+ backend = LlamaCppBackend()
+ backend._dspark_sidecar_absent = backend._dflash_sidecar_absent = True
+ order: list[str] = []
+ with pytest.raises(RuntimeError, match = "stop here"):
+ _chat_repo_load(monkeypatch, backend, order, **_sidecar_outcome(backend))
+ assert order == ["download", "kill", "launch"]
+ assert _sidecar_flags(backend) == (False, False, True)
def test_a_preflighted_file_is_judged_without_a_second_request(monkeypatch, tmp_path):
@@ -408,7 +502,7 @@ def test_the_per_load_resets_sit_below_the_teardown_in_source():
src = inspect.getsource(llama_cpp_module.LlamaCppBackend.load_model)
teardown = src.index("# ── Phase 1: kill old process")
assert src.index("self._launch_binary_revision = self._binary_revision(binary)") > teardown
- assert src.index("self._dflash_retry_needed = False") > teardown
+ assert src.index(") = _load_sidecar_flags") > teardown
def test_the_cached_probe_is_verified_the_way_the_loader_verifies_it(monkeypatch, tmp_path):
@@ -802,8 +896,8 @@ def test_the_launch_reuses_the_carried_file_instead_of_resolving_it_again():
src = inspect.getsource(llama_cpp_module.LlamaCppBackend.load_model)
teardown = src.index("# ── Phase 1: kill old process")
reuse = src.index("self._verified_cached_gguf(intent, hf_repo, hf_variant)")
- download = src.index("model_path = self._download_gguf(", teardown)
- assert teardown < reuse < download, "the carried file is not consulted before the download"
+ download = src.index("model_path = self._download_gguf(", reuse)
+ assert reuse < download < teardown, "the carried file is not consulted before the download"
def test_a_carried_path_outside_the_active_cache_is_not_reused(monkeypatch, tmp_path):
@@ -882,17 +976,19 @@ def test_the_launch_opens_the_carried_file_instead_of_resolving_it(monkeypatch,
diffusion_compat, "_read_gguf_header", lambda *_a, **_k: _gguf_bytes(arch = "llama")
)
- def _stop(**_kwargs):
+ for name in _SIDECARS:
+ monkeypatch.setattr(backend, name, lambda **_k: None)
+
+ def _stop(*_a, **_k):
raise RuntimeError("stop here")
- # _download_mtp is the first fetch past the download decision.
- monkeypatch.setattr(backend, "_download_mtp", _stop)
+ monkeypatch.setattr(backend, "_read_gguf_metadata", _stop)
with pytest.raises(RuntimeError, match = "stop here"):
backend.load_model(_intent(verified_gguf = verified))
return order
assert _load(_verified(cached)) == ["kill"]
- assert _load(None) == ["kill", "download"]
+ assert _load(None) == ["download", "kill"]
def _launchable_backend(monkeypatch) -> LlamaCppBackend: