Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

Studio: on low disk, load the largest GGUF quant that fits and show which one loaded (#11052)

Nilayce66f60c9c07
diff --git a/studio/backend/core/inference/llama_cpp.py b/studio/backend/core/inference/llama_cpp.py
index b2e40b8beeb..70b5edcf5bd 100644
--- a/studio/backend/core/inference/llama_cpp.py
+++ b/studio/backend/core/inference/llama_cpp.py
@@ -6698,6 +6698,9 @@ class LlamaCppBackend:
# Reset by _begin_load_warnings so one load's notice is never reported against
# the next.
self._last_load_warning: Optional[str] = None
+ # Same, for a quant fallback: the download fills the pair, the launch publishes it.
+ self._variant_fallback_warning: Optional[str] = None
+ self._pending_variant_fallback: Optional[tuple[str, str]] = None
# Set per launch by _record_carveout_advice; None on nearly every load.
self._last_carveout_advice: Optional[dict] = None
self._model_identifier: Optional[str] = None
@@ -13253,10 +13256,11 @@ class LlamaCppBackend:
@property
def last_load_warning(self) -> Optional[str]:
"""Advisory notice from the most recent load, or None. Read by the route."""
- return self._last_load_warning
+ notices = (self._variant_fallback_warning, self._last_load_warning)
+ return " ".join(notice for notice in notices if notice) or None
def _begin_load_warnings(self) -> None:
- """Drop the previous load's advisory notice, at the point a load commits to
+ """Drop the previous load's memory notice, at the point a load commits to
replacing the resident server. Without it a warned load followed by a
comfortable one would report the first load's notice against the second.
@@ -15599,23 +15603,25 @@ class LlamaCppBackend:
# ── Variant fallback ────────────────────────────────────────────
@staticmethod
- def _find_smallest_fitting_variant(
+ def _find_fitting_variant(
hf_repo: str,
free_bytes: int,
+ requested_bytes: int,
hf_token: Optional[str] = None,
+ requested_file: Optional[str] = None,
) -> Optional[tuple[str, int, list[str]]]:
- """Find the smallest GGUF variant (including all shards) that fits.
+ """Find a GGUF variant (including all shards) smaller than the requested one.
- Groups split shards by variant prefix and sums their sizes (e.g.
- UD-Q4_K_XL with 9 shards of 50 GB each = 450 GB total).
+ The largest that leaves ``_DISK_RESERVE_BYTES`` free, else the fewest new bytes.
Returns (first_shard_filename, total_size_bytes, extra_shards) or None.
"""
try:
from huggingface_hub import get_paths_info, list_repo_files
+ from core.inference.openai_auto_download import _DISK_RESERVE_BYTES
files = list_repo_files(hf_repo, token = hf_token)
- from hub.utils.gguf import drop_shadowed_appledouble_names
+ from hub.utils.gguf import drop_shadowed_appledouble_names, gguf_checkpoint_family
gguf_files = [
f
@@ -15624,6 +15630,10 @@ class LlamaCppBackend:
and not _is_companion_gguf_path(f)
and not _is_big_endian_gguf_path(f)
]
+ # A sibling checkpoint's smaller quant is different weights, not a fallback.
+ if requested_file is not None:
+ checkpoint = gguf_checkpoint_family(requested_file)
+ gguf_files = [f for f in gguf_files if gguf_checkpoint_family(f) == checkpoint]
if not gguf_files:
return None
@@ -15639,23 +15649,30 @@ class LlamaCppBackend:
variants.setdefault(key, []).append(f)
# Sum shard sizes per variant, track the first shard (for download)
- variant_sizes: list[tuple[str, int, list[str]]] = []
+ fitting: list[tuple[str, int, list[str], int]] = []
for key, shard_files in variants.items():
total = sum(size_map.get(f, 0) for f in shard_files)
first = sorted(shard_files)[0]
- variant_sizes.append((first, total, shard_files))
-
- # Smallest that fits
- variant_sizes.sort(key = lambda x: x[1])
- for first_file, total_size, shard_files in variant_sizes:
- if total_size > 0 and total_size <= free_bytes:
- return (
- first_file,
- total_size,
- [path for path in sorted(shard_files) if path != first_file],
- )
+ extra = [path for path in sorted(shard_files) if path != first]
+ cached = _cached_complete_candidate(hf_repo, first, extra)
+ needed = (
+ 0
+ if cached and _cached_candidate_matches_revision_size(hf_repo, cached, hf_token)
+ else total
+ )
+ if 0 < total < requested_bytes and needed <= free_bytes:
+ fitting.append((first, total, extra, needed))
+ if not fitting:
+ return None
- return None
+ roomy = [v for v in fitting if v[3] + _DISK_RESERVE_BYTES <= free_bytes]
+ # Only cached variants can tie on new bytes, and the larger costs no more.
+ first_file, total_size, extra_shards, _ = (
+ max(roomy, key = lambda v: v[1])
+ if roomy
+ else min(fitting, key = lambda v: (v[3], -v[1]))
+ )
+ return first_file, total_size, extra_shards
except Exception:
return None
@@ -16992,17 +17009,26 @@ class LlamaCppBackend:
f"Not enough disk space to download {gguf_filename}. "
f"Only {free_gb:.1f} GB free in {download_cache_dir}"
)
- smaller = self._find_smallest_fitting_variant(
+ smaller = self._find_fitting_variant(
hf_repo,
free_bytes,
+ total_bytes,
hf_token,
+ requested_file = gguf_filename,
)
if smaller:
fallback_file, fallback_size, fallback_shards = smaller
- logger.info(
- f"Selected variant too large ({total_gb:.1f} GB), "
- f"falling back to {fallback_file} ({fallback_size / (1024**3):.1f} GB)"
+ from utils.models.model_config import _gguf_variant_key
+
+ fallback_variant = _gguf_variant_key(fallback_file)
+ notice = (
+ f"Not enough disk space to download {hf_variant} "
+ f"({total_gb:.1f} GB needed, {free_gb:.1f} GB free), so "
+ f"{fallback_variant} ({fallback_size / (1024**3):.1f} GB) "
+ "was loaded instead."
)
+ logger.warning(notice)
+ self._pending_variant_fallback = (fallback_variant, notice)
gguf_filename = fallback_file
gguf_extra_shards = fallback_shards
@@ -21414,6 +21440,7 @@ class LlamaCppBackend:
_paravirtual_cpu_forced and not self.diffusion_split_supported()
)
_preflight_model_path = None
+ self._pending_variant_fallback = None
if hf_repo and (_vulkan_ordinal_pin or _cpu_only_pin or _pv_diffusion_unpinnable):
_resolved_repo = _resolve_repo_id_casing(hf_repo)
if _resolved_repo != hf_repo:
@@ -21542,6 +21569,7 @@ class LlamaCppBackend:
self._cleanup_cpu_fallback_runtime()
self._mmproj_fallback_reason = None
+ self._variant_fallback_warning = None
# Both describe the process just killed, so they are reset HERE rather than
# where the binary is resolved: everything above can bail with the old server
# still running (a stand-down, a non-chat refusal, a cancel), and resetting on
@@ -21596,6 +21624,10 @@ class LlamaCppBackend:
hf_token = hf_token,
cancel_event = download_cancel_event,
)
+ if self._pending_variant_fallback:
+ hf_variant, self._variant_fallback_warning = self._pending_variant_fallback
+ # Replayed by the respawn and a variant-less Apply, which would re-download.
+ intent = replace(intent, hf_variant = hf_variant)
# Auto-download mmproj for vision models unless opted out. Vision
# switched off does NOT opt out: remote discovery calls any mmproj
# filename vision, and only the file's metadata separates an image
diff --git a/studio/backend/models/inference.py b/studio/backend/models/inference.py
index 0dec97eb9e2..cc5d05ab2d5 100644
--- a/studio/backend/models/inference.py
+++ b/studio/backend/models/inference.py
@@ -1442,7 +1442,8 @@ class LoadResponse(_InferenceRuntimeFields):
None,
description = "Non-blocking advisory about this load, or null. Set when the "
"weights do not fit in free VRAM plus available system RAM, so llama.cpp pages "
- "them in from disk and generation will be slow. The model still loaded.",
+ "them in from disk and generation will be slow, or when the requested GGUF quant "
+ "did not fit on disk and a smaller one was loaded. The model still loaded.",
)
carveout_advice: Optional[dict] = Field(
None,
@@ -1570,6 +1571,11 @@ class InferenceStatusResponse(_InferenceRuntimeFields):
False, description = "Whether the active model came from a local filesystem path"
)
gguf_variant: Optional[str] = Field(None, description = "GGUF quantization variant (e.g. Q4_K_M)")
+ memory_warning: Optional[str] = Field(
+ None,
+ description = "Non-blocking advisory about the active load, or null: the "
+ "memory_warning its load response carried, kept while that model is running.",
+ )
loading: List[str] = Field(default_factory = list, description = "Models currently being loaded")
loaded: List[str] = Field(default_factory = list, description = "Models currently loaded")
inference: Optional[Dict[str, Any]] = Field(
diff --git a/studio/backend/routes/inference.py b/studio/backend/routes/inference.py
index e1d8ab7210c..1e46fe6dcf1 100644
--- a/studio/backend/routes/inference.py
+++ b/studio/backend/routes/inference.py
@@ -18247,6 +18247,7 @@ async def get_status(current_subject: str = Depends(get_current_subject)):
llama_backend, _native_grant_backed, _model_id
),
gguf_variant = llama_backend.hf_variant,
+ memory_warning = llama_backend.last_load_warning,
loading = _loading,
# Plus anything the Unsloth registry still holds: the GGUF load
# only unloaded the ACTIVE one, so a model cached behind it is
diff --git a/studio/backend/tests/llama_backend_double.py b/studio/backend/tests/llama_backend_double.py
index 37e7f167298..26e7a9daeb7 100644
--- a/studio/backend/tests/llama_backend_double.py
+++ b/studio/backend/tests/llama_backend_double.py
@@ -72,6 +72,7 @@ class FakeLlamaCppBackend:
requested_extra_args = None
spec_fallback_reason = None
spec_drafter_kind = None
+ last_load_warning = None
def __getattr__(self, name):
"""Runtime fields answer as the real backend does; anything else raises.
diff --git a/studio/backend/tests/test_gguf_load_cache_reuse.py b/studio/backend/tests/test_gguf_load_cache_reuse.py
index 3812d67ed7a..09c583f4bec 100644
--- a/studio/backend/tests/test_gguf_load_cache_reuse.py
+++ b/studio/backend/tests/test_gguf_load_cache_reuse.py
@@ -15,9 +15,10 @@ import sys
import threading
import types as _types
from contextlib import contextmanager, nullcontext
+from dataclasses import replace
from pathlib import Path
from types import SimpleNamespace
-from unittest.mock import patch
+from unittest.mock import Mock, patch
import pytest
@@ -84,6 +85,7 @@ from core.inference.llama_cpp import (
gguf_load_in_flight,
hf_gguf_load_in_flight,
)
+from core.inference.openai_auto_download import _DISK_RESERVE_BYTES
REPO = "unsloth/gemma-test-GGUF"
@@ -130,6 +132,53 @@ def _fail_get_paths_info(*_args, **_kwargs):
raise AssertionError("cached reuse must return before the sizing preflight")
+GIB = 1024**3
+_LOW_DISK_SIZES = {
+ "gemma-test-UD-IQ1_S.gguf": GIB,
+ "gemma-test-Q4_K_M.gguf": 2 * GIB,
+ "gemma-test-Q6_K.gguf": 6 * GIB,
+ "gemma-test-Q8_0.gguf": 8 * GIB,
+}
+
+
+@contextmanager
+def _low_disk_hub(
+ free: int,
+ *,
+ snap: Path | None = None,
+ served: str | None = None,
+ sizes: dict[str, int] | None = None,
+):
+ sizes = sizes or _LOW_DISK_SIZES
+ downloaded: list[str] = []
+
+ def fake_get_paths_info(
+ _repo,
+ paths,
+ *,
+ revision = None,
+ token = None,
+ ):
+ on_disk = snap is not None and revision == snap.name
+ return [
+ _types.SimpleNamespace(path = path, size = 4 if on_disk else sizes[path]) for path in paths
+ ]
+
+ def fake_download(_repo, filename, *_args, **_kwargs):
+ downloaded.append(filename)
+ return served or f"/fake/{REPO}/{filename}"
+
+ usage = _types.SimpleNamespace(total = 100 * GIB, used = 100 * GIB - free, free = free)
+ with (
+ patch("huggingface_hub.list_repo_files", lambda *_a, **_k: list(sizes)),
+ patch("huggingface_hub.get_paths_info", fake_get_paths_info),
+ patch("huggingface_hub.try_to_load_from_cache", lambda *_a, **_k: None),
+ patch("shutil.disk_usage", lambda *_a, **_k: usage),
+ patch("core.inference.llama_cpp.hf_hub_download_with_xet_fallback", fake_download),
+ ):
+ yield downloaded
+
+
def _load_route_module(name: str, relative_path: str):
"""Import a route module under a private name so patches can't leak."""
spec = importlib.util.spec_from_file_location(name, Path(_BACKEND_DIR) / relative_path)
@@ -528,7 +577,7 @@ class TestLoadReusesCachedCopy:
patch("shutil.disk_usage", lambda *_a, **_k: _types.SimpleNamespace(free = 10)),
patch.object(
backend,
- "_find_smallest_fitting_variant",
+ "_find_fitting_variant",
lambda *_a, **_k: (fallback, 4, []),
),
patch("core.inference.llama_cpp.hf_hub_download_with_xet_fallback", _fail_download),
@@ -537,6 +586,133 @@ class TestLoadReusesCachedCopy:
assert out == str(snap / fallback)
+ def test_low_disk_fallback_picks_largest_variant_leaving_reserve(self, hf_cache):
+ backend = LlamaCppBackend()
+ with _low_disk_hub(_DISK_RESERVE_BYTES + 5 * GIB // 2) as downloaded:
+ out = backend._download_gguf(hf_repo = REPO, hf_variant = "Q8_0")
+
+ assert downloaded == ["gemma-test-Q4_K_M.gguf"]
+ assert out == f"/fake/{REPO}/gemma-test-Q4_K_M.gguf"
+ variant, notice = backend._pending_variant_fallback
+ assert variant == "Q4_K_M"
+ assert "Q8_0" in notice and "Q4_K_M (2.0 GB)" in notice
+
+ def test_low_disk_fallback_keeps_smallest_when_none_leaves_reserve(self, hf_cache):
+ backend = LlamaCppBackend()
+ with _low_disk_hub(_DISK_RESERVE_BYTES + GIB // 2) as downloaded:
+ backend._download_gguf(hf_repo = REPO, hf_variant = "Q8_0")
+
+ assert downloaded == ["gemma-test-UD-IQ1_S.gguf"]
+ assert backend._pending_variant_fallback[0] == "UD-IQ1_S"
+
+ def test_low_disk_fallback_reuses_cached_variant_before_downloading(self, hf_cache):
+ backend = LlamaCppBackend()
+ snap = _build_cache(hf_cache, REPO, {"gemma-test-Q4_K_M.gguf": 4})
+ with _low_disk_hub(_DISK_RESERVE_BYTES + GIB // 2, snap = snap) as downloaded:
+ out = backend._download_gguf(hf_repo = REPO, hf_variant = "Q8_0")
+
+ assert downloaded == []
+ assert out == str(snap / "gemma-test-Q4_K_M.gguf")
+
+ def test_low_disk_fallback_prefers_the_largest_cached_variant_under_the_reserve(self, hf_cache):
+ backend = LlamaCppBackend()
+ snap = _build_cache(
+ hf_cache, REPO, {"gemma-test-Q4_K_M.gguf": 4, "gemma-test-Q6_K.gguf": 4}
+ )
+ with _low_disk_hub(GIB // 2, snap = snap) as downloaded:
+ out = backend._download_gguf(hf_repo = REPO, hf_variant = "Q8_0")
+
+ assert downloaded == []
+ assert out == str(snap / "gemma-test-Q6_K.gguf")
+
+ def test_low_disk_fallback_never_picks_a_larger_cached_variant(self, hf_cache):
+ backend = LlamaCppBackend()
+ snap = _build_cache(hf_cache, REPO, {"gemma-test-Q8_0.gguf": 4})
+ with _low_disk_hub(_DISK_RESERVE_BYTES + GIB // 2, snap = snap) as downloaded:
+ backend._download_gguf(hf_repo = REPO, hf_variant = "Q6_K")
+
+ assert downloaded == ["gemma-test-UD-IQ1_S.gguf"]
+
+ def test_low_disk_fallback_stays_on_the_requested_checkpoint(self, hf_cache):
+ backend = LlamaCppBackend()
+ sizes = {
+ "gemma-test-Q4_K_M.gguf": 2 * GIB,
+ "gemma-test-Q8_0.gguf": 16 * GIB,
+ "distilled/gemma-test-distilled-Q6_K.gguf": 6 * GIB,
+ }
+ with _low_disk_hub(_DISK_RESERVE_BYTES + 13 * GIB // 2, sizes = sizes) as downloaded:
+ backend._download_gguf(hf_repo = REPO, hf_variant = "Q8_0")
+
+ assert downloaded == ["gemma-test-Q4_K_M.gguf"]
+ assert backend._pending_variant_fallback[0] == "Q4_K_M"
+
+ def test_low_disk_fallback_ignores_a_truncated_cached_variant(self, hf_cache):
+ backend = LlamaCppBackend()
+ _build_cache(hf_cache, REPO, {"gemma-test-Q6_K.gguf": 4})
+ with _low_disk_hub(_DISK_RESERVE_BYTES + GIB // 2) as downloaded:
+ backend._download_gguf(hf_repo = REPO, hf_variant = "Q8_0")
+
+ assert downloaded == ["gemma-test-UD-IQ1_S.gguf"]
+
+ def test_low_disk_without_any_fitting_variant_refuses(self, hf_cache):
+ backend = LlamaCppBackend()
+ with _low_disk_hub(GIB // 2), pytest.raises(RuntimeError, match = "any variant"):
+ backend._download_gguf(hf_repo = REPO, hf_variant = "Q8_0")
+
+ def test_a_fallback_and_a_memory_notice_are_both_reported(self):
+ backend = LlamaCppBackend()
+ backend._variant_fallback_warning = "Q4_K_M was loaded instead."
+ backend._record_load_warning("The model does not fit in GPU memory.")
+
+ assert backend.last_load_warning == (
+ "Q4_K_M was loaded instead. The model does not fit in GPU memory."
+ )
+
+ # The arch-crash retry calls this again after the download; the teardown clears both.
+ backend._begin_load_warnings()
+ assert backend.last_load_warning == "Q4_K_M was loaded instead."
+
+ def test_low_disk_fallback_load_records_the_served_variant(self, hf_cache, tmp_path):
+ backend = LlamaCppBackend()
+ served = tmp_path / "gemma-test-Q4_K_M.gguf"
+ served.write_bytes(b"GGUF" + b"\0" * 4096)
+ backend._find_llama_server_binary = lambda *_a, **_k: "/usr/bin/true"
+ backend._remote_non_chat_gguf_refusal = lambda **_k: None
+ backend._non_chat_gguf_refusal_for_path = lambda *_a, **_k: None
+ backend._non_chat_gguf_refusal = lambda *_a, **_k: None
+ backend._kill_process = lambda *_a, **_k: None
+ backend._wait_for_health = lambda timeout = 600.0, interval = 0.5, cancelled = None: True
+
+ def _start(cmd, env, **_kwargs):
+ backend._process = Mock(pid = 424242, returncode = None)
+ backend._process.poll.return_value = None
+ backend._stdout_lines = ["build: 6543", "main: loading model"]
+ return backend._process
+
+ backend._start_llama_process = _start
+ requested = GgufLoadIntent(
+ model_identifier = REPO,
+ hf_repo = REPO,
+ hf_variant = "Q8_0",
+ speculative_type = "none",
+ n_ctx = 4096,
+ )
+ with _low_disk_hub(_DISK_RESERVE_BYTES + 5 * GIB // 2, served = str(served)):
+ assert backend.load_model(requested) is True
+
+ assert backend.hf_variant == "Q4_K_M"
+ assert "Q4_K_M" in (backend.last_load_warning or "")
+ assert not backend.matches_load_source(requested)
+ assert backend.matches_load_source(replace(requested, hf_variant = "Q4_K_M"))
+ # Replayed by the crash respawn, so it must name the quant that is running.
+ assert backend.last_load_intent.hf_variant == "Q4_K_M"
+
+ with _low_disk_hub(100 * GIB, served = str(served)):
+ assert backend.load_model(requested) is True
+
+ assert backend.hf_variant == "Q8_0"
+ assert backend.last_load_warning is None
+
def test_companion_prefers_main_snapshot_sibling(self, hf_cache):
"""A cached mmproj is reused from the main model's snapshot."""
backend = LlamaCppBackend()
diff --git a/studio/backend/tests/test_inference_status_loaded_gguf.py b/studio/backend/tests/test_inference_status_loaded_gguf.py
index 4c35afc6090..2aefc8976e4 100644
--- a/studio/backend/tests/test_inference_status_loaded_gguf.py
+++ b/studio/backend/tests/test_inference_status_loaded_gguf.py
@@ -206,6 +206,18 @@ def test_status_publishes_mmproj_cpu_recovery(status_route):
assert status.mmproj_fallback_reason == "cpu_offload"
+def test_status_publishes_the_running_load_warning(status_route):
+ assert status_route(_StatusBackend("org/A-GGUF")).memory_warning is None
+
+ backend = _StatusBackend("org/A-GGUF")
+ backend.last_load_warning = (
+ "Not enough disk space to download BF16 (7.5 GB needed, 7.5 GB free), "
+ "so Q4_1 (2.4 GB) was loaded instead."
+ )
+
+ assert status_route(backend).memory_warning == backend.last_load_warning
+
+
def test_status_publishes_an_explicit_text_only_mmproj_fallback(status_route):
backend = _StatusBackend("org/Vision-GGUF")
backend.is_vision = False
diff --git a/studio/frontend/src/features/chat/api/chat-api.ts b/studio/frontend/src/features/chat/api/chat-api.ts
index c8d28a90452..b82bf1a59e6 100644
--- a/studio/frontend/src/features/chat/api/chat-api.ts
+++ b/studio/frontend/src/features/chat/api/chat-api.ts
@@ -21,6 +21,7 @@ import {
type ModelRuntime,
withModelLoadNotice,
} from "@/lib/model-lifecycle-events";
+import { showLoadWarning } from "../utils/load-warning-toast";
import type {
MessageRecord,
ModelType,
@@ -299,6 +300,7 @@ export async function loadModel(
// passed -- a cached Hub candidate is requested by its loadId while the runtime
// keeps `loaded.model`, and the unload is issued with the second.
showCarveoutAdvice(loaded.carveout_advice, loaded.model, payload.model_path);
+ showLoadWarning(loaded.memory_warning);
return loaded;
},
);
diff --git a/studio/frontend/src/features/chat/lib/apply-inference-status-to-store.ts b/studio/frontend/src/features/chat/lib/apply-inference-status-to-store.ts
index 54316d798bf..f79c0386732 100644
--- a/studio/frontend/src/features/chat/lib/apply-inference-status-to-store.ts
+++ b/studio/frontend/src/features/chat/lib/apply-inference-status-to-store.ts
@@ -34,6 +34,7 @@ import {
isMultimodalResponse,
} from "../types/api";
import type { ChatModelRow } from "../types/runtime";
+import { showLoadWarning } from "../utils/load-warning-toast";
import { resolveQwenThinkingParams } from "../utils/qwen-sampling-table";
import { sameGpuSelection } from "@/hooks/gpu-selection";
import { resolveBatchSizeSeed } from "./resolve-batch-size-seed";
@@ -186,6 +187,9 @@ export function applyActiveModelStatusToStore(
const hydratingExistingModel =
previousCheckpoint !== checkpointId ||
previousGgufVariant !== (status.gguf_variant ?? null);
+ if (hydratingExistingModel) {
+ showLoadWarning(status.memory_warning);
+ }
const supportsReasoning = status.supports_reasoning ?? false;
const reasoningAlwaysOn = status.reasoning_always_on ?? false;
const reasoningStyle = status.reasoning_style ?? "enable_thinking";
diff --git a/studio/frontend/src/features/chat/types/api.ts b/studio/frontend/src/features/chat/types/api.ts
index adda79a9ced..81cd62427df 100644
--- a/studio/frontend/src/features/chat/types/api.ts
+++ b/studio/frontend/src/features/chat/types/api.ts
@@ -222,6 +222,7 @@ export interface LoadModelResponse {
* dedicated to it than the weights need. Unknown-shaped on purpose so an older or
* proxied backend cannot render "undefined GB"; narrowed by parseCarveoutAdvice. */
carveout_advice?: unknown;
+ memory_warning?: string | null;
is_diffusion?: boolean;
/** GPU-layer count the diffusion runner was ASKED for, when it differs from what it applied: a shim
* without --ngl runs Auto, so gpu_layers reports -1 while this carries the request. */
@@ -335,6 +336,7 @@ export interface InferenceStatusResponse {
* without --ngl runs Auto, so gpu_layers reports -1 while this carries the request. */
diffusion_requested_ngl?: number | null;
gguf_variant?: string | null;
+ memory_warning?: string | null;
is_audio?: boolean;
audio_type?: string | null;
has_audio_input?: boolean;
diff --git a/studio/frontend/src/features/chat/utils/load-warning-toast.ts b/studio/frontend/src/features/chat/utils/load-warning-toast.ts
new file mode 100644
index 00000000000..e10280d4c66
--- /dev/null
+++ b/studio/frontend/src/features/chat/utils/load-warning-toast.ts
@@ -0,0 +1,20 @@
+// SPDX-License-Identifier: AGPL-3.0-only
+// Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
+
+import { toast } from "@/lib/toast";
+
+/** One id, so the second call REPLACES rather than stacks. */
+export const LOAD_WARNING_TOAST_ID = "model-load-warning";
+
+export function showLoadWarning(warning: string | null | undefined): void {
+ if (!warning) {
+ toast.dismiss(LOAD_WARNING_TOAST_ID);
+ return;
+ }
+ toast.warning("Model loaded with a warning", {
+ id: LOAD_WARNING_TOAST_ID,
+ description: warning,
+ duration: 12000,
+ closeButton: true,
+ });
+}
diff --git a/studio/frontend/tests/chat-save-conflict-detail.test.ts b/studio/frontend/tests/chat-save-conflict-detail.test.ts
index 4b0ef3bad5e..1e33af2c6b5 100644
--- a/studio/frontend/tests/chat-save-conflict-detail.test.ts
+++ b/studio/frontend/tests/chat-save-conflict-detail.test.ts
@@ -63,6 +63,7 @@ function harness(response: ReturnType<typeof jsonResponse>) {
notifyChatHistoryUpdated: () => {},
isCoalescedHistoryEvent: () => false,
},
+ "../utils/load-warning-toast": { showLoadWarning: () => {} },
"./generation-length.ts": {},
"./gguf-variants-request": {},
"./padded-response": { assertCompletedPaddedBody: () => {} },
diff --git a/studio/frontend/tests/load-warning-toast.test.ts b/studio/frontend/tests/load-warning-toast.test.ts
new file mode 100644
index 00000000000..12e354f0efa
--- /dev/null
+++ b/studio/frontend/tests/load-warning-toast.test.ts
@@ -0,0 +1,109 @@
+// SPDX-License-Identifier: AGPL-3.0-only
+// Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
+
+import assert from "node:assert/strict";
+import { register } from "node:module";
+import test from "node:test";
+
+import { readSrc } from "./helpers/kit.ts";
+import { loadWithStubs } from "./helpers/module-stubs.ts";
+
+register("./helpers/toast-resolver.mjs", import.meta.url);
+
+const { calls } = await import("./helpers/toast-stub.mjs");
+const { LOAD_WARNING_TOAST_ID, showLoadWarning } = await import(
+ "../src/features/chat/utils/load-warning-toast.ts"
+);
+
+const NOTICE =
+ "Not enough disk space to download BF16 (7.5 GB needed, 7.5 GB free), so Q4_1 (2.4 GB) was loaded instead.";
+const FROM_STATUS_ON_MODEL_CHANGE =
+ /if \(hydratingExistingModel\) \{\s*showLoadWarning\(status\.memory_warning\);/;
+
+type ChatApi = {
+ loadModel: (payload: Record<string, unknown>) => Promise<Record<string, unknown>>;
+};
+
+function chatApi(body: Record<string, unknown>) {
+ const shown: (string | null | undefined)[] = [];
+ const module = loadWithStubs<ChatApi>(
+ new URL("../src/features/chat/api/chat-api.ts", import.meta.url),
+ {
+ "@/features/auth": {
+ authFetch: async () => ({
+ status: 200,
+ ok: true,
+ headers: { get: () => null },
+ json: async () => body,
+ }),
+ },
+ "@/lib/format-fastapi-error": { formatApiErrorBody: () => null },
+ "../types": {},
+ "../types/api": {},
+ "../utils/chat-history-revision": {
+ notifyChatHistoryUpdated: () => {},
+ isCoalescedHistoryEvent: () => false,
+ },
+ "../utils/load-warning-toast": {
+ showLoadWarning: (warning: string | null | undefined) => shown.push(warning),
+ },
+ "./generation-length.ts": {},
+ "./gguf-variants-request": {},
+ "./padded-response": { assertCompletedPaddedBody: () => {} },
+ "@/features/hf-auth": { prepareHfTokenForUse: async () => ({ proceed: true }) },
+ "@/features/igpu-carveout": {
+ dismissCarveoutAdviceForModel: () => {},
+ showCarveoutAdvice: () => {},
+ },
+ "@/features/hub/lib/abort-signals": {},
+ "@/features/hub/lib/hub-token-header": { hubTokenHeader: () => ({}) },
+ "@/features/hub/lib/network": { isHuggingFaceOffline: () => false },
+ "@/features/native-intents/api": { consumeNativePathToken: () => undefined },
+ "@/lib/model-lifecycle-events": {
+ withModelLoadNotice: async (
+ _runtime: string,
+ _path: string | null,
+ run: () => Promise<unknown>,
+ ) => run(),
+ },
+ },
+ );
+ return { module, shown };
+}
+
+test("a load warning raises one warning toast carrying the backend's text", () => {
+ calls.length = 0;
+ showLoadWarning(NOTICE);
+ assert.equal(calls.length, 1);
+ assert.equal(calls[0].kind, "warning");
+ assert.equal(calls[0].options?.id, LOAD_WARNING_TOAST_ID);
+ assert.equal(calls[0].options?.description, NOTICE);
+});
+
+test("a model with no warning takes the previous model's warning down", () => {
+ calls.length = 0;
+ showLoadWarning(null);
+ showLoadWarning(undefined);
+ assert.deepEqual(calls, [
+ { kind: "dismiss", id: LOAD_WARNING_TOAST_ID },
+ { kind: "dismiss", id: LOAD_WARNING_TOAST_ID },
+ ]);
+});
+
+test("a load response's warning reaches the notice, and its absence takes one down", async () => {
+ const warned = chatApi({ status: "loaded", model: "m", memory_warning: NOTICE });
+ await warned.module.loadModel({ model_path: "m" });
+ assert.deepEqual(warned.shown, [NOTICE]);
+
+ const quiet = chatApi({ status: "loaded", model: "m" });
+ await quiet.module.loadModel({ model_path: "m" });
+ assert.deepEqual(quiet.shown, [undefined]);
+});
+
+// No test drives applyActiveModelStatusToStore, so a model loaded elsewhere is matched here.
+test("a model loaded elsewhere reaches the toast on the model change", () => {
+ assert.match(
+ readSrc("features/chat/lib/apply-inference-status-to-store.ts"),
+ FROM_STATUS_ON_MODEL_CHANGE,
+ );
+});
diff --git a/unsloth_cli/commands/start.py b/unsloth_cli/commands/start.py
index 041c7acc2f9..7c5664bdba7 100644
--- a/unsloth_cli/commands/start.py
+++ b/unsloth_cli/commands/start.py
@@ -4075,6 +4075,16 @@ def _connect(
# That server was started FROM these knobs, so inferring a target here would reload what was just loaded.
infer_resident = server is None,
)
+ status = _inference_status(base, key) if model else {}
+ # A GGUF can be active while the resolved entry is another resident model.
+ if status.get("memory_warning") and any(
+ _model_id_matches(
+ (entry or {}).get("id"), status_id, allow_casefold = is_loopback_url(base)
+ )
+ for status_id in (status.get("active_model"), status.get("model_identifier"))
+ if status_id
+ ):
+ typer.echo(f"Warning: {status['memory_warning']}", err = True)
except BaseException:
_shutdown_auto_served()
raise
diff --git a/unsloth_cli/tests/test_start.py b/unsloth_cli/tests/test_start.py
index f36b2f1b5bb..29e56559864 100644
--- a/unsloth_cli/tests/test_start.py
+++ b/unsloth_cli/tests/test_start.py
@@ -1586,6 +1586,60 @@ def test_connect_claude_no_launch(fake_studio):
assert ".claude/settings.json" not in result.output
+def test_connect_prints_the_running_models_load_warning(fake_studio, monkeypatch):
+ notice = (
+ "Not enough disk space to download BF16 (7.5 GB needed, 7.5 GB free), "
+ "so Q4_1 (2.4 GB) was loaded instead."
+ )
+ http_json = start._http_json
+
+ def with_warning(
+ method,
+ url,
+ token,
+ payload = None,
+ timeout = 30,
+ error = None,
+ ):
+ if url.endswith("/api/inference/status"):
+ return {"is_gguf": True, "model_identifier": MODEL["id"], "memory_warning": notice}
+ return http_json(method, url, token, payload, timeout, error)
+
+ monkeypatch.setattr(start, "_http_json", with_warning)
+ result = CliRunner().invoke(start.start_app, ["claude", "--no-launch", "--model", MODEL["id"]])
+
+ assert result.exit_code == 0, result.output
+ assert f"Warning: {notice}" in result.stderr
+ assert f"Warning: {notice}" not in result.stdout
+
+
+def test_connect_skips_the_load_warning_of_another_active_model(fake_studio, monkeypatch):
+ http_json = start._http_json
+
+ def other_model_warns(
+ method,
+ url,
+ token,
+ payload = None,
+ timeout = 30,
+ error = None,
+ ):
+ if url.endswith("/api/inference/status"):
+ return {
+ "is_gguf": True,
+ "active_model": "unsloth/Other-GGUF",
+ "model_identifier": "unsloth/Other-GGUF",
+ "memory_warning": "Not enough disk space to download BF16, so Q4_1 was loaded instead.",
+ }
+ return http_json(method, url, token, payload, timeout, error)
+
+ monkeypatch.setattr(start, "_http_json", other_model_warns)
+ result = CliRunner().invoke(start.start_app, ["claude", "--no-launch", "--model", MODEL["id"]])
+
+ assert result.exit_code == 0, result.output
+ assert "Warning: Not enough disk space" not in result.output
+
+
def test_connect_claude_session_settings_follow_forwarded_settings(fake_studio):
forwarded = json.dumps({"env": {"CLAUDE_CODE_USE_FOUNDRY": "1"}})
result = CliRunner().invoke(