Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
Studio: keep the loaded model if a GGUF download fails or is cancelled (#15846)
Nilaya1fa6e609d02
diff --git a/studio/backend/core/inference/llama_cpp.py b/studio/backend/core/inference/llama_cpp.pyindex 4c42641ef45..7d76e5c201d 100644--- a/studio/backend/core/inference/llama_cpp.py+++ b/studio/backend/core/inference/llama_cpp.py@@ -23178,9 +23178,8 @@ class LlamaCppBackend:_pv_diffusion_unpinnable = bool(_paravirtual_cpu_forced and not self.diffusion_split_supported())- _preflight_model_path = None- self._pending_variant_fallback = None- if hf_repo and (_vulkan_ordinal_pin or _cpu_only_pin or _pv_diffusion_unpinnable):+ if hf_repo:+ # Resolve repo casing once so the GGUF and companions share one cache entry._resolved_repo = _resolve_repo_id_casing(hf_repo)if _resolved_repo != hf_repo:logger.info(@@ -23189,6 +23188,9 @@ class LlamaCppBackend:hf_repo,)hf_repo = _resolved_repo+ _preflight_model_path = None+ self._pending_variant_fallback = None+ if hf_repo and (_vulkan_ordinal_pin or _cpu_only_pin or _pv_diffusion_unpinnable):with _hf_offline_if_unreachable():_preflight_model_path = self._download_gguf(hf_repo = hf_repo,@@ -23267,6 +23269,150 @@ class LlamaCppBackend:)raise ValueError(_early_non_chat)+ # Resolve the canonical MTP mode once; it decides which drafter is downloaded and launched.+ _spec_canon = _canonicalize_spec_mode(speculative_type) or "auto"+ # #11308: DFlash2 + --split-mode tensor aborts at startup (ggml-org/llama.cpp#27819) and the+ # route falls back to layer split, so Auto prefers a loadable MTP sidecar on tensor instead.+ # _estimate_gguf_required_gb mirrors this.+ _auto_tensor_split = _spec_canon == "auto" and _effective_tensor_parallel(+ extra_args, tensor_parallel+ )++ def _auto_dflash_blocked_by_tensor() -> bool:+ if (+ _auto_tensor_split+ and mtp_draft_path+ and not _extra_args_mtp_draft_path(extra_args, env = _child_spec_env(extra_args))+ ):+ try:+ from utils.models.gguf_metadata import read_gguf_nextn_predict_layers+ return bool(+ _launch_caps(binary).get("mtp_token")+ and not (read_gguf_nextn_predict_layers(model_path) or 0) > 0+ and _mtp_drafter_loads_standalone(mtp_draft_path)+ )+ except Exception:+ return False+ return False++ def _dflash_breaks_on_images() -> bool:+ if not dflash_draft_path or disable_vision or extra_args_disable_mmproj(extra_args):+ return False+ try:+ projector = self._resolve_launch_mmproj_path(+ model_path = model_path, mmproj_path = mmproj_path, extra_args = extra_args+ )+ except Exception:+ return False+ return _dflash_drafter_breaks_on_images(model_path, dflash_draft_path, projector)++ _unloadable_mtp_draft_path: Optional[str] = None+ # Downloads run before the teardown so a failed or cancelled fetch keeps the resident model;+ # sidecar fetches write flags on self, so the resident's come back until the teardown commits.+ _resident_sidecar_flags = (+ self._dspark_sidecar_absent,+ self._dflash_sidecar_absent,+ self._dflash_retry_needed,+ )+ self._dflash_retry_needed = False+ model_path = _preflight_model_path+ try:+ if hf_repo:+ # Scope HF_HUB_OFFLINE to the download block only when DNS is dead.+ with _hf_offline_if_unreachable():+ if model_path is None:+ # Revalidate the carried file before skipping download resolution.+ model_path = self._verified_cached_gguf(intent, hf_repo, hf_variant)+ if model_path:+ logger.info("Reusing verified cached GGUF: %s", model_path)+ if model_path is None:+ model_path = self._download_gguf(+ hf_repo = hf_repo,+ hf_variant = hf_variant,+ hf_token = hf_token,+ cancel_event = download_cancel_event,+ )+ # Vision off does not skip the fetch: an mmproj may be an audio encoder instead.+ if (+ is_vision+ and not mmproj_path+ and not extra_args_disable_mmproj(extra_args)+ ):+ mmproj_path = self._download_mmproj(+ hf_repo = hf_repo,+ hf_token = hf_token,+ cancel_event = download_cancel_event,+ near_path = model_path,+ )+ # Fetch the separate MTP drafter; no size gate (helps even sub-3B).+ # The MTP/ fallback is qwen4exp only, whose GGUFs carry no head.+ if (+ not mtp_draft_path+ and _spec_canon in ("auto", "mtp", "mtp+ngram")+ and not _extra_args_set_spec_type(extra_args)+ ):+ mtp_draft_path = self._download_mtp(+ hf_repo = hf_repo,+ hf_token = hf_token,+ cancel_event = download_cancel_event,+ near_path = model_path,+ allow_nested = self._gguf_path_wants_nested_mtp(model_path),+ )+ # "auto" is included: DSpark is the default whenever the repo+ # ships a sidecar. Repos without one no-op, exactly like the+ # MTP fetch above, so this costs one listing and no download.+ if (+ not dspark_draft_path+ and _spec_canon in ("auto", "dspark")+ and not _extra_args_set_spec_type(extra_args)+ ):+ dspark_draft_path = self._download_dspark(+ hf_repo = hf_repo,+ hf_token = hf_token,+ cancel_event = download_cancel_event,+ near_path = model_path,+ binary = binary,+ caps_probe = _launch_caps,+ )+ # Auto fetch stands down once DSpark resolves (it wins the promotion), using the same+ # capability-aware 'resolved'. An explicit 'dflash' still fetches.+ if (+ not dflash_draft_path+ and _spec_canon in ("auto", "dflash")+ and not _auto_dflash_blocked_by_tensor()+ and not self._dspark_wins_auto(+ binary = binary,+ dspark_draft_path = dspark_draft_path,+ spec_canon = _spec_canon,+ extra_args = extra_args,+ )+ and not _extra_args_set_spec_type(extra_args)+ ):+ dflash_draft_path = self._download_dflash(+ hf_repo = hf_repo,+ hf_token = hf_token,+ cancel_event = download_cancel_event,+ near_path = model_path,+ binary = binary,+ )+ elif gguf_path:+ if not Path(gguf_path).is_file():+ raise FileNotFoundError(f"GGUF file not found: {gguf_path}")+ model_path = gguf_path+ else:+ raise ValueError("Either gguf_path or hf_repo must be provided")+ finally:+ _load_sidecar_flags = (+ self._dspark_sidecar_absent,+ self._dflash_sidecar_absent,+ self._dflash_retry_needed,+ )+ (+ self._dspark_sidecar_absent,+ self._dflash_sidecar_absent,+ self._dflash_retry_needed,+ ) = _resident_sidecar_flags+# The probe can take a moment on a slow Hub, so re-read the cancel flag rather# than tearing down a healthy server for a load that was called off.if _load_cancelled():@@ -23306,142 +23452,15 @@ class LlamaCppBackend:self._variant_fallback_warning = None# Reset at the teardown, not where the binary resolves: early bails keep the old server.self._launch_binary_revision = self._binary_revision(binary)- # Cleared per load: a local-file load never reaches the DFlash fetch.- self._dflash_retry_needed = False-- # ── Phase 2: download (NO lock held, so cancel can proceed) ──- # Resolve the canonical MTP mode once; it decides which drafter is downloaded and launched.- _spec_canon = _canonicalize_spec_mode(speculative_type) or "auto"- # #11308: DFlash2 + --split-mode tensor aborts at startup (ggml-org/llama.cpp#27819) and the- # route falls back to layer split, so Auto prefers a loadable MTP sidecar on tensor instead.- # _estimate_gguf_required_gb mirrors this.- _auto_tensor_split = _spec_canon == "auto" and _effective_tensor_parallel(- extra_args, tensor_parallel- )-- def _auto_dflash_blocked_by_tensor() -> bool:- if (- _auto_tensor_split- and mtp_draft_path- and not _extra_args_mtp_draft_path(extra_args, env = _child_spec_env(extra_args))- ):- try:- from utils.models.gguf_metadata import read_gguf_nextn_predict_layers- return bool(- _launch_caps(binary).get("mtp_token")- and not (read_gguf_nextn_predict_layers(model_path) or 0) > 0- and _mtp_drafter_loads_standalone(mtp_draft_path)- )- except Exception:- return False- return False-- def _dflash_breaks_on_images() -> bool:- if not dflash_draft_path or disable_vision or extra_args_disable_mmproj(extra_args):- return False- try:- projector = self._resolve_launch_mmproj_path(- model_path = model_path, mmproj_path = mmproj_path, extra_args = extra_args- )- except Exception:- return False- return _dflash_drafter_breaks_on_images(model_path, dflash_draft_path, projector)-- _unloadable_mtp_draft_path: Optional[str] = None- # Scope HF_HUB_OFFLINE to the download block only when DNS is dead.- if hf_repo:- # Resolve repo casing once so the GGUF and companions share one cache entry.- _resolved_repo = _resolve_repo_id_casing(hf_repo)- if _resolved_repo != hf_repo:- logger.info(- "Using cached repo_id casing '%s' for requested '%s'",- _resolved_repo,- hf_repo,- )- hf_repo = _resolved_repo- with _hf_offline_if_unreachable():- model_path = _preflight_model_path- if model_path is None:- # Revalidate the carried file before skipping download resolution.- model_path = self._verified_cached_gguf(intent, hf_repo, hf_variant)- if model_path:- logger.info("Reusing verified cached GGUF: %s", model_path)- if model_path is None:- model_path = self._download_gguf(- hf_repo = hf_repo,- hf_variant = hf_variant,- hf_token = hf_token,- cancel_event = download_cancel_event,- )- if self._pending_variant_fallback:- hf_variant, self._variant_fallback_warning = self._pending_variant_fallback- # Replayed by the respawn and a variant-less Apply, which would re-download.- intent = replace(intent, hf_variant = hf_variant)- # Vision off does not skip the fetch: an mmproj may be an audio encoder instead.- if is_vision and not mmproj_path and not extra_args_disable_mmproj(extra_args):- mmproj_path = self._download_mmproj(- hf_repo = hf_repo,- hf_token = hf_token,- cancel_event = download_cancel_event,- near_path = model_path,- )- # Fetch the separate MTP drafter; no size gate (helps even sub-3B).- # The MTP/ fallback is qwen4exp only, whose GGUFs carry no head.- if (- not mtp_draft_path- and _spec_canon in ("auto", "mtp", "mtp+ngram")- and not _extra_args_set_spec_type(extra_args)- ):- mtp_draft_path = self._download_mtp(- hf_repo = hf_repo,- hf_token = hf_token,- cancel_event = download_cancel_event,- near_path = model_path,- allow_nested = self._gguf_path_wants_nested_mtp(model_path),- )- # "auto" is included: DSpark is the default whenever the repo- # ships a sidecar. Repos without one no-op, exactly like the- # MTP fetch above, so this costs one listing and no download.- if (- not dspark_draft_path- and _spec_canon in ("auto", "dspark")- and not _extra_args_set_spec_type(extra_args)- ):- dspark_draft_path = self._download_dspark(- hf_repo = hf_repo,- hf_token = hf_token,- cancel_event = download_cancel_event,- near_path = model_path,- binary = binary,- caps_probe = _launch_caps,- )- # Auto fetch stands down once DSpark resolves (it wins the promotion), using the same- # capability-aware 'resolved'. An explicit 'dflash' still fetches.- if (- not dflash_draft_path- and _spec_canon in ("auto", "dflash")- and not _auto_dflash_blocked_by_tensor()- and not self._dspark_wins_auto(- binary = binary,- dspark_draft_path = dspark_draft_path,- spec_canon = _spec_canon,- extra_args = extra_args,- )- and not _extra_args_set_spec_type(extra_args)- ):- dflash_draft_path = self._download_dflash(- hf_repo = hf_repo,- hf_token = hf_token,- cancel_event = download_cancel_event,- near_path = model_path,- binary = binary,- )- elif gguf_path:- if not Path(gguf_path).is_file():- raise FileNotFoundError(f"GGUF file not found: {gguf_path}")- model_path = gguf_path- else:- raise ValueError("Either gguf_path or hf_repo must be provided")+ (+ self._dspark_sidecar_absent,+ self._dflash_sidecar_absent,+ self._dflash_retry_needed,+ ) = _load_sidecar_flags+ if hf_repo and self._pending_variant_fallback:+ hf_variant, self._variant_fallback_warning = self._pending_variant_fallback+ # Replayed by the respawn and a variant-less Apply, which would re-download.+ intent = replace(intent, hf_variant = hf_variant)# Set identifier early so _read_gguf_metadata can use it (DeepSeek).self._model_identifier = model_identifierdiff --git a/studio/backend/tests/test_gpu_memory_mode.py b/studio/backend/tests/test_gpu_memory_mode.pyindex 5a4a8ffda7d..cd9682ca986 100644--- a/studio/backend/tests/test_gpu_memory_mode.py+++ b/studio/backend/tests/test_gpu_memory_mode.py@@ -596,9 +596,9 @@ def test_remote_vulkan_diffusion_preflight_runs_before_teardown(monkeypatch):preflight = src.index("_preflight_model_path = self._download_gguf(")teardown = src.index("# ── Phase 1: kill old process")assert preflight < teardown- phase_two = src.index("model_path = _preflight_model_path", teardown)- download = src.index("model_path = self._download_gguf(", teardown)- assert phase_two < download+ carried = src.index("model_path = _preflight_model_path")+ download = src.index("model_path = self._download_gguf(", carried)+ assert preflight < carried < download < teardowndef test_local_vulkan_diffusion_preflight_runs_before_teardown():diff --git a/studio/backend/tests/test_llama_cpp_remote_non_chat_gguf_preflight.py b/studio/backend/tests/test_llama_cpp_remote_non_chat_gguf_preflight.pyindex 38d45e64c56..0cf3ac79131 100644--- a/studio/backend/tests/test_llama_cpp_remote_non_chat_gguf_preflight.py+++ b/studio/backend/tests/test_llama_cpp_remote_non_chat_gguf_preflight.py@@ -290,32 +290,126 @@ def test_a_non_chat_repo_load_keeps_the_resident_model_and_never_downloads(assert order == []-def test_a_chat_repo_load_still_proceeds_past_the_teardown(monkeypatch):- backend = LlamaCppBackend()- order: list[str] = []- _repo_load(monkeypatch, backend, order)+_SIDECARS = ("_download_mmproj", "_download_mtp", "_download_dspark", "_download_dflash")- def _download(**_kwargs):- order.append("download")- raise RuntimeError("stop here")- monkeypatch.setattr(backend, "_download_gguf", _download)+def _chat_repo_load(monkeypatch, backend, order, **downloads):+ _repo_load(monkeypatch, backend, order)+ monkeypatch.setattr(+ backend, "_download_gguf", lambda **_kwargs: order.append("download") or "/cache/m.gguf"+ )+ for name in _SIDECARS:+ monkeypatch.setattr(backend, name, lambda **_kwargs: None)+ for name, fn in downloads.items():+ monkeypatch.setattr(backend, name, fn)monkeypatch.setattr(diffusion_compat,"_read_gguf_header",lambda *_a, **_k: _gguf_bytes(arch = "qwen3", declared_kv = 32),)- with pytest.raises(RuntimeError, match = "stop here"):- backend.load_model(- GgufLoadIntent(- hf_repo = "unsloth/Qwen3-0.6B-GGUF",- hf_variant = "Q4_K_M",- model_identifier = "unsloth/Qwen3-0.6B-GGUF",- )+ def _launch(*_a, **_k):+ order.append("launch")+ raise RuntimeError("stop here")++ monkeypatch.setattr(backend, "_read_gguf_metadata", _launch)+ return backend.load_model(+ GgufLoadIntent(+ hf_repo = "unsloth/Qwen3-VL-2B-Instruct-GGUF",+ hf_variant = "Q4_K_M",+ model_identifier = "unsloth/Qwen3-VL-2B-Instruct-GGUF",+ is_vision = True,)+ )- assert order == ["kill", "download"]++def _connection_reset(**_kwargs):+ raise RuntimeError("connection reset")+++def test_a_chat_repo_load_still_proceeds_past_the_teardown(monkeypatch):+ order: list[str] = []+ with pytest.raises(RuntimeError, match = "stop here"):+ _chat_repo_load(monkeypatch, LlamaCppBackend(), order)+ assert order == ["download", "kill", "launch"]+++def test_a_failed_download_leaves_the_resident_model_running(monkeypatch):+ order: list[str] = []++ def _download(**_kwargs):+ order.append("download")+ raise RuntimeError("connection reset")++ with pytest.raises(RuntimeError, match = "connection reset"):+ _chat_repo_load(monkeypatch, LlamaCppBackend(), order, _download_gguf = _download)+ assert order == ["download"]++[email protected]("sidecar", _SIDECARS)+def test_a_failed_sidecar_download_leaves_the_resident_model_running(monkeypatch, sidecar):+ order: list[str] = []+ with pytest.raises(RuntimeError, match = "connection reset"):+ _chat_repo_load(monkeypatch, LlamaCppBackend(), order, **{sidecar: _connection_reset})+ assert order == ["download"]++[email protected]("sidecar", _SIDECARS)+def test_a_sidecar_download_cancelled_mid_fetch_leaves_the_resident_model_running(+ monkeypatch, sidecar+):+ backend = LlamaCppBackend()+ order: list[str] = []+ cancel = {sidecar: lambda **_k: backend._cancel_event.set()}+ assert _chat_repo_load(monkeypatch, backend, order, **cancel) is False+ assert order == ["download"]+++def _sidecar_outcome(backend):+ def _dspark(**_kwargs):+ backend._dspark_sidecar_absent = False++ def _dflash(**_kwargs):+ backend._dflash_sidecar_absent = False+ backend._dflash_retry_needed = True++ return {"_download_dspark": _dspark, "_download_dflash": _dflash}+++def _sidecar_flags(backend):+ return (+ backend._dspark_sidecar_absent,+ backend._dflash_sidecar_absent,+ backend._dflash_retry_needed,+ )+++def test_a_failed_load_keeps_the_resident_servers_sidecar_flags(monkeypatch):+ backend = LlamaCppBackend()+ backend._dspark_sidecar_absent = backend._dflash_sidecar_absent = True+ downloads = _sidecar_outcome(backend)+ fetched = downloads["_download_dflash"]++ def _dflash_then_fail(**kwargs):+ fetched(**kwargs)+ raise RuntimeError("connection reset")++ downloads["_download_dflash"] = _dflash_then_fail+ order: list[str] = []+ with pytest.raises(RuntimeError, match = "connection reset"):+ _chat_repo_load(monkeypatch, backend, order, **downloads)+ assert order == ["download"]+ assert _sidecar_flags(backend) == (True, True, False)+++def test_a_committed_load_takes_the_fetched_sidecar_flags(monkeypatch):+ backend = LlamaCppBackend()+ backend._dspark_sidecar_absent = backend._dflash_sidecar_absent = True+ order: list[str] = []+ with pytest.raises(RuntimeError, match = "stop here"):+ _chat_repo_load(monkeypatch, backend, order, **_sidecar_outcome(backend))+ assert order == ["download", "kill", "launch"]+ assert _sidecar_flags(backend) == (False, False, True)def test_a_preflighted_file_is_judged_without_a_second_request(monkeypatch, tmp_path):@@ -408,7 +502,7 @@ def test_the_per_load_resets_sit_below_the_teardown_in_source():src = inspect.getsource(llama_cpp_module.LlamaCppBackend.load_model)teardown = src.index("# ── Phase 1: kill old process")assert src.index("self._launch_binary_revision = self._binary_revision(binary)") > teardown- assert src.index("self._dflash_retry_needed = False") > teardown+ assert src.index(") = _load_sidecar_flags") > teardowndef test_the_cached_probe_is_verified_the_way_the_loader_verifies_it(monkeypatch, tmp_path):@@ -802,8 +896,8 @@ def test_the_launch_reuses_the_carried_file_instead_of_resolving_it_again():src = inspect.getsource(llama_cpp_module.LlamaCppBackend.load_model)teardown = src.index("# ── Phase 1: kill old process")reuse = src.index("self._verified_cached_gguf(intent, hf_repo, hf_variant)")- download = src.index("model_path = self._download_gguf(", teardown)- assert teardown < reuse < download, "the carried file is not consulted before the download"+ download = src.index("model_path = self._download_gguf(", reuse)+ assert reuse < download < teardown, "the carried file is not consulted before the download"def test_a_carried_path_outside_the_active_cache_is_not_reused(monkeypatch, tmp_path):@@ -882,17 +976,19 @@ def test_the_launch_opens_the_carried_file_instead_of_resolving_it(monkeypatch,diffusion_compat, "_read_gguf_header", lambda *_a, **_k: _gguf_bytes(arch = "llama"))- def _stop(**_kwargs):+ for name in _SIDECARS:+ monkeypatch.setattr(backend, name, lambda **_k: None)++ def _stop(*_a, **_k):raise RuntimeError("stop here")- # _download_mtp is the first fetch past the download decision.- monkeypatch.setattr(backend, "_download_mtp", _stop)+ monkeypatch.setattr(backend, "_read_gguf_metadata", _stop)with pytest.raises(RuntimeError, match = "stop here"):backend.load_model(_intent(verified_gguf = verified))return orderassert _load(_verified(cached)) == ["kill"]- assert _load(None) == ["kill", "download"]+ assert _load(None) == ["download", "kill"]def _launchable_backend(monkeypatch) -> LlamaCppBackend: