Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

Studio: say when a model only partly fits on the GPU (#8812)

Daniel Han01b8b84c5e6f
diff --git a/studio/backend/core/inference/llama_cpp.py b/studio/backend/core/inference/llama_cpp.py
index 03dc68ade04..62cd457b8c9 100644
--- a/studio/backend/core/inference/llama_cpp.py
+++ b/studio/backend/core/inference/llama_cpp.py
@@ -729,13 +729,40 @@ _GPU_OFFLOAD_MARKERS = (
_OFFLOADED_LAYERS_RE = re.compile(
r"offloaded\s+(\d+)\s*/\s*(\d+)\s+layers?\s+to\s+gpu", re.IGNORECASE
)
+# llama.cpp #23021 moved libllama INFO (offload counts, device table, buffer sizes) to trace (4).
+_LLAMA_TRACE_VERBOSITY = 4
+_TRACE_VERBOSITY_HELP_RE = re.compile(r"\b4:\s*trace\b", re.IGNORECASE)
+_LOG_VERBOSITY_FLAGS = frozenset(
+ {"-lv", "--verbosity", "--log-verbosity", "-v", "--verbose", "--log-verbose", "--log-disable"}
+)
+# Past this, the active llama-server log keeps only warnings and errors.
+_LLAMA_LOG_FULL_BYTES = 64 * 1024 * 1024
+_LOG_LEVEL_TOKEN_RE = re.compile(r"^\S+ ([A-Z]) ")
+# The startup head is never trimmed: its load lines are read for the server's whole life.
+_STDOUT_TRIM_AT = 20000
+_STDOUT_TAIL_KEEP = 5000
+_STDOUT_HEAD_MAX = 5000
+
+
+def _trace_verbosity_args(caps: dict, extra_args: Optional[Iterable[str]], environ) -> list[str]:
+ """``--verbosity 4`` when the build hides load lines at its default and the user chose none."""
+ if not caps.get("supports_trace_verbosity"):
+ return []
+ if _extra_args_set_any_flag(extra_args, _LOG_VERBOSITY_FLAGS):
+ return []
+ if "LLAMA_ARG_LOG_VERBOSITY" in environ:
+ return []
+ return ["--verbosity", str(_LLAMA_TRACE_VERBOSITY)]
+
+
_GPU_MODEL_BUFFER_RE = re.compile(
r"\b(?:CUDA|ROCm|ROCM|HIP|SYCL)(\d+)\s+model buffer size\s*=\s*"
r"([0-9]+(?:\.[0-9]+)?)\s+MiB\b",
re.IGNORECASE,
)
_DEVICE_ROW_RE = re.compile(
- r"-\s*(CUDA|ROCm|ROCM|HIP|Metal|Vulkan|SYCL|OpenCL|MUSA|CANN|CPU)\w*\s*:",
+ # MTL: ggml-metal names its devices MTL0, MTL1 (GGML_METAL_NAME), never "Metal".
+ r"-\s*(CUDA|ROCm|ROCM|HIP|Metal|MTL|Vulkan|SYCL|OpenCL|MUSA|CANN|CPU)\w*\s*:",
re.IGNORECASE,
)
_GPU_DEVICE_PREFIXES = (
@@ -743,6 +770,7 @@ _GPU_DEVICE_PREFIXES = (
"rocm",
"hip",
"metal",
+ "mtl",
"vulkan",
"sycl",
"opencl",
@@ -751,6 +779,46 @@ _GPU_DEVICE_PREFIXES = (
)
+def parse_gpu_offload_counts(lines: "list[str]") -> "Optional[tuple[int, int]]":
+ """``(offloaded, total)`` for the largest-M line, or None; ties keep the first
+ line since llama.cpp logs the main model before its drafter."""
+ max_total = -1
+ offloaded_at_max = 0
+ for line in lines:
+ match = _OFFLOADED_LAYERS_RE.search(line)
+ if not match:
+ continue
+ offloaded, total = int(match.group(1)), int(match.group(2))
+ if total > max_total:
+ max_total, offloaded_at_max = total, offloaded
+ if max_total <= 0:
+ return None
+ return offloaded_at_max, max_total
+
+
+def llama_saw_gpu_device(lines: "list[str]") -> Optional[bool]:
+ """Whether llama.cpp's device table (rows after the header) lists a GPU, or None
+ with no table. All-CPU means the GPU backend failed to load."""
+ after_header = False
+ saw_device_row = False
+ saw_gpu_device = False
+ for line in lines:
+ if "device_info:" in line:
+ after_header = True
+ continue
+ if not after_header:
+ continue
+ match = _DEVICE_ROW_RE.search(line)
+ if not match:
+ continue
+ saw_device_row = True
+ if match.group(1).lower().startswith(_GPU_DEVICE_PREFIXES):
+ saw_gpu_device = True
+ if not saw_device_row:
+ return None
+ return saw_gpu_device
+
+
def classify_gpu_offload_lines(lines: "list[str]") -> Optional[bool]:
"""True if the model landed on a GPU, False if it stayed on CPU despite GPU
intent, None when the log has no usable signal."""
@@ -786,23 +854,8 @@ def classify_gpu_offload_lines(lines: "list[str]") -> Optional[bool]:
# device_info: lists *available* devices (printed whenever a GPU backend is
# visible), not where the model loaded, so it can only disconfirm: an
# all-CPU table means no usable GPU. A visible GPU device is not proof the
- # model used it, so it does not return True. Rows after the header only.
- after_header = False
- saw_device_row = False
- saw_gpu_device = False
- for line in lines:
- if "device_info:" in line:
- after_header = True
- continue
- if not after_header:
- continue
- match = _DEVICE_ROW_RE.search(line)
- if not match:
- continue
- saw_device_row = True
- if match.group(1).lower().startswith(_GPU_DEVICE_PREFIXES):
- saw_gpu_device = True
- if saw_device_row and not saw_gpu_device:
+ # model used it, so it does not return True.
+ if llama_saw_gpu_device(lines) is False:
return False
return None
@@ -7320,6 +7373,12 @@ class LlamaCppBackend:
self._stats_logger = None # vLLM-style engine-stats poller, set on load
# Set by _classify_gpu_offload after _wait_for_health.
self._gpu_offload_active: Optional[bool] = None
+ # (offloaded, total) layers llama.cpp reported for this load, when it said.
+ self._gpu_offload_layers: Optional[tuple[int, int]] = None
+ # Whether the user's own extras pinned the layer split for this load.
+ self._offload_overridden: bool = False
+ # Studio saw a GPU for this load but llama.cpp's device table did not.
+ self._gpu_backend_unavailable: bool = False
# Diffusion only: the split the running child was asked for.
self._diffusion_requested_ngl: Optional[int] = None
self._context_length: Optional[int] = None
@@ -7794,6 +7853,25 @@ class LlamaCppBackend:
slots = 1
return max(1, slots)
+ @property
+ def offloaded_layers(self) -> Optional[int]:
+ """Layers llama.cpp actually put on a GPU, when its log said so."""
+ return self._gpu_offload_layers[0] if self._gpu_offload_layers else None
+
+ @property
+ def offload_total_layers(self) -> Optional[int]:
+ return self._gpu_offload_layers[1] if self._gpu_offload_layers else None
+
+ @property
+ def gpu_backend_unavailable(self) -> bool:
+ """Studio found a GPU for this load but llama.cpp reported none."""
+ return self._gpu_backend_unavailable
+
+ @property
+ def offload_overridden(self) -> bool:
+ """Whether the user's own extras (Auto mode keeps -ngl) chose this split."""
+ return self._offload_overridden
+
@property
def requested_parallel_slots(self) -> int:
"""--parallel the last load asked for, before any fit-time reduction.
@@ -9123,6 +9201,7 @@ class LlamaCppBackend:
# Fails CLOSED, unlike --jinja: a build without --reasoning exits on it.
"supports_reasoning_flag": False,
"supports_no_mmproj_offload": False,
+ "supports_trace_verbosity": False,
"supports_video_fps": False,
"supports_load_mode": False,
"spec_draft_ngl_flag": None,
@@ -9157,6 +9236,7 @@ class LlamaCppBackend:
spec_draft_n_max_flag: Optional[str] = None
spec_draft_n_max_default: Optional[int] = None
supports_kv_unified = False
+ supports_trace_verbosity = False
# See the fallback dict: these fail open.
supports_no_context_shift = True
supports_jinja = True
@@ -9372,6 +9452,12 @@ class LlamaCppBackend:
spec_draft_n_max_default = int(_n_max_default.group(1))
supports_kv_unified = _is_real("--kv-unified")
+ # Fails closed: an older build's level 4 may be per-token debug on the decode path.
+ supports_trace_verbosity = bool(
+ probe_ok
+ and _is_real("--verbosity")
+ and _TRACE_VERBOSITY_HELP_RE.search(blocks.get("--verbosity") or "")
+ )
# Only once the help actually parsed AND `--help` succeeded. Empty
# `blocks` means the probe told us nothing, and a nonzero exit means
# the listing may be a fragment; neither may read as "absent" for a
@@ -9509,6 +9595,7 @@ class LlamaCppBackend:
"reasoning_budget_probe_inconclusive": not (probe_ok and help_nonempty),
"supports_reasoning_flag": supports_reasoning_flag,
"supports_no_mmproj_offload": supports_no_mmproj_offload,
+ "supports_trace_verbosity": supports_trace_verbosity,
"supports_video_fps": supports_video_fps,
"supports_load_mode": supports_load_mode,
"spec_draft_ngl_flag": spec_draft_ngl_flag,
@@ -17490,15 +17577,25 @@ class LlamaCppBackend:
post-mortem has the full output even if the crash predates the
drain-thread join in ``_wait_for_health``.
"""
+ startup_len: Optional[int] = None
+ log_bytes = 0
try:
for line in self._process.stdout:
line = line.rstrip()
if line:
- self._stdout_lines.append(line)
+ lines = self._stdout_lines
+ lines.append(line)
+ if len(lines) > _STDOUT_TRIM_AT:
+ # Before readiness the whole log so far is startup: keep its head too.
+ head = _STDOUT_HEAD_MAX if startup_len is None else startup_len
+ head = min(head, _STDOUT_HEAD_MAX, len(lines))
+ del lines[head : len(lines) - _STDOUT_TAIL_KEEP]
# Two forms across llama.cpp builds. Only readiness lines wake the
# probe: waking on every tensor-load log spins startup.
line_lower = line.lower()
if "server is listening" in line_lower or "model loaded" in line_lower:
+ if startup_len is None:
+ startup_len = len(self._stdout_lines)
health_probe_event = getattr(self, "_health_probe_event", None)
if health_probe_event is not None:
health_probe_event.set()
@@ -17506,6 +17603,19 @@ class LlamaCppBackend:
fh = getattr(self, "_llama_log_fh", None)
if fh is not None:
try:
+ if log_bytes >= _LLAMA_LOG_FULL_BYTES:
+ level = _LOG_LEVEL_TOKEN_RE.match(line)
+ if not (
+ level.group(1) in "WE"
+ if level is not None
+ else ("error" in line_lower or "fail" in line_lower)
+ ):
+ continue
+ elif log_bytes + len(line) + 1 >= _LLAMA_LOG_FULL_BYTES:
+ fh.write(
+ "[studio] log past 64 MiB: keeping warnings and errors only\n"
+ )
+ log_bytes += len(line) + 1
fh.write(line + "\n")
fh.flush()
except (ValueError, OSError):
@@ -27252,6 +27362,7 @@ class LlamaCppBackend:
# Enable Jinja chat template rendering
if _caps.get("supports_jinja", True):
cmd.extend(["--jinja"])
+ cmd.extend(_trace_verbosity_args(_caps, extra_args, os.environ))
# KV cache data type
_valid_cache_types = _VALID_CACHE_TYPES
@@ -31089,6 +31200,46 @@ class LlamaCppBackend:
gpu_indices is not None or use_fit or gpu_memory_mode == "manual",
_detected_gpus,
)
+ # CPU-only hosts log 0/N too, so a zero needs an expected GPU; a positive count stands.
+ _offload_counts = (
+ None
+ if _arch_gate_forced_cpu or _deliberate_cpu_only
+ else parse_gpu_offload_counts(self._stdout_lines)
+ )
+ if (
+ _offload_counts is not None
+ and _offload_counts[0] <= 0
+ and not (
+ (_detected_gpus or _metal_capable_host())
+ and (gpu_indices is not None or use_fit or gpu_memory_mode == "manual")
+ )
+ ):
+ _offload_counts = None
+ self._gpu_offload_layers = _offload_counts
+ self._gpu_backend_unavailable = bool(_detected_gpus or _metal_capable_host()) and (
+ llama_saw_gpu_device(self._stdout_lines) is False
+ )
+ # Not _GPU_OFFLOAD_OVERRIDE_FLAGS: --fit on is exactly the split to report.
+ self._offload_overridden = (
+ _extra_args_set_any_flag(extra_args, _GPU_LAYER_FLAGS)
+ or _device_selection_is_cpu(
+ self._strip_device_extra_args(extra_args)
+ if _gpu_ids_own_device_flags
+ else extra_args,
+ env,
+ )
+ or _env_fixes_gpu_layers(env)
+ )
+ if (
+ self._gpu_offload_layers is not None
+ and 0 < self._gpu_offload_layers[0] < self._gpu_offload_layers[1]
+ ):
+ logger.warning(
+ "llama-server offloaded %d of %d layers to GPU; the rest run on "
+ "CPU, which is much slower for a dense model.",
+ self._gpu_offload_layers[0],
+ self._gpu_offload_layers[1],
+ )
if self._gpu_offload_active is False and not _deliberate_cpu_only:
logger.warning(
"llama-server appears to have loaded the model entirely "
@@ -32452,6 +32603,9 @@ class LlamaCppBackend:
# loading) server as VRAM-resident rather than reading the killed
# server's stale zero-offload flag until the health probe reclassifies.
self._gpu_offload_active = None
+ self._gpu_offload_layers = None
+ self._offload_overridden = False
+ self._gpu_backend_unavailable = False
# Drives _wait_for_vram_settle in the next load_model; set in finally
# so both in-process and frontend Apply paths record the kill.
self._last_kill_monotonic = time.monotonic()
diff --git a/studio/backend/models/inference.py b/studio/backend/models/inference.py
index 68d58a2c5db..d61bac1507b 100644
--- a/studio/backend/models/inference.py
+++ b/studio/backend/models/inference.py
@@ -1485,6 +1485,34 @@ class _InferenceRuntimeFields(BaseModel):
-1,
description = "Manual mode: requested --gpu-layers value (-1 = Auto/--fit, or when not manual).",
)
+ offloaded_layers: Optional[int] = Field(
+ None,
+ description = (
+ "Layers llama.cpp actually placed on a GPU, when it reported the count. "
+ "In Auto mode the placement is llama.cpp's own (--fit on), so this is the "
+ "only account of what happened; gpu_layers above is the REQUEST."
+ ),
+ )
+ offload_total_layers: Optional[int] = Field(
+ None,
+ description = "Total layers in the model, alongside offloaded_layers.",
+ )
+ gpu_backend_unavailable: bool = Field(
+ False,
+ description = (
+ "Studio detected a GPU for this load but llama.cpp's own device table "
+ "reported none, meaning its GPU backend did not initialise. Separates a "
+ "backend failure from a fit that placed no layers; both log 0/M."
+ ),
+ )
+ offload_overridden: bool = Field(
+ False,
+ description = (
+ "Whether the user's own llama-server extras pinned the layer split. "
+ "Auto mode respects an inherited -ngl instead of stripping it, so a "
+ "deliberate placement can arrive with gpu_memory_mode still 'auto'."
+ ),
+ )
cpu_fallback_reason: Optional[Literal["vulkan_startup_crash"]] = Field(
None,
description = (
diff --git a/studio/backend/tests/llama_backend_double.py b/studio/backend/tests/llama_backend_double.py
index 26e7a9daeb7..e7b24915f98 100644
--- a/studio/backend/tests/llama_backend_double.py
+++ b/studio/backend/tests/llama_backend_double.py
@@ -55,6 +55,8 @@ class FakeLlamaCppBackend:
gpu_layers = -1
n_cpu_moe = 0
n_moe_layers = 0
+ gpu_backend_unavailable = False
+ offload_overridden = False
# Private: the only name the real backend has for these, so the one production falls back to.
_is_audio = False
_has_audio_input = False
diff --git a/studio/backend/tests/test_launch_flags_are_capability_gated.py b/studio/backend/tests/test_launch_flags_are_capability_gated.py
index 45ea8358a47..e508fa695a1 100644
--- a/studio/backend/tests/test_launch_flags_are_capability_gated.py
+++ b/studio/backend/tests/test_launch_flags_are_capability_gated.py
@@ -504,3 +504,138 @@ class TestTheFlaglessFixupKeepsMlaKAndVEqual:
cmd, _ = _flagless_v_cache_fixup(known_off = True, cmd = self.CMD, env = {}, mla = False)
assert cmd[cmd.index("--cache-type-k") + 1] == "q8_0"
assert cmd[cmd.index("--cache-type-v") + 1] == "f16"
+
+
+# The -lv block of b11160 and later (llama.cpp "logs : reduce", #23021).
+TRACE_HELP = NEW_HELP + (
+ "-lv, --verbosity, --log-verbosity N Set the verbosity threshold. Messages with a higher verbosity will be\n"
+ " ignored. Values:\n"
+ " - 0: generic output\n"
+ " - 1: error\n"
+ " - 2: warning\n"
+ " - 3: info\n"
+ " - 4: trace (more info)\n"
+ " - 5: debug\n"
+ " (default: 3)\n"
+)
+PRE_TRACE_HELP = NEW_HELP + (
+ "-lv, --verbosity, --log-verbosity N Set the verbosity threshold. Messages with a higher verbosity will be\n"
+ " ignored.\n"
+)
+
+
+class TestTraceVerbosityRestoresTheLoadLog:
+ """Load lines the offload report and classifiers read are trace-level on current builds."""
+
+ def test_a_trace_build_is_detected(self, tmp_path, monkeypatch):
+ assert probe(tmp_path, monkeypatch, TRACE_HELP)["supports_trace_verbosity"] is True
+
+ @pytest.mark.parametrize(
+ "help_text,returncode",
+ [(PRE_TRACE_HELP, 0), (NEW_HELP, 0), (TRACE_HELP, 1), ("", 0)],
+ )
+ def test_anything_else_fails_closed(self, tmp_path, monkeypatch, help_text, returncode):
+ caps = probe(tmp_path, monkeypatch, help_text, returncode)
+ assert caps["supports_trace_verbosity"] is False
+
+ def test_an_unprobeable_binary_fails_closed(self, tmp_path):
+ caps = LlamaCppBackend.probe_server_capabilities(str(tmp_path / "absent"))
+ assert caps["supports_trace_verbosity"] is False
+
+ def test_a_trace_build_gets_level_4(self):
+ args = llama_cpp_module._trace_verbosity_args({"supports_trace_verbosity": True}, [], {})
+ assert args == ["--verbosity", "4"]
+
+ @pytest.mark.parametrize(
+ "extra_args",
+ [["-lv", "2"], ["--verbosity=5"], ["--log-verbosity", "1"], ["-v"], ["--log-disable"]],
+ )
+ def test_the_users_own_logging_choice_wins(self, extra_args):
+ caps = {"supports_trace_verbosity": True}
+ assert llama_cpp_module._trace_verbosity_args(caps, extra_args, {}) == []
+
+ def test_the_env_form_wins_too(self):
+ caps = {"supports_trace_verbosity": True}
+ env = {"LLAMA_ARG_LOG_VERBOSITY": "3"}
+ assert llama_cpp_module._trace_verbosity_args(caps, None, env) == []
+
+ def test_other_builds_get_nothing(self):
+ assert llama_cpp_module._trace_verbosity_args({}, None, {}) == []
+
+ def test_the_launch_emits_it(self):
+ src = inspect.getsource(LlamaCppBackend.load_model)
+ assert "cmd.extend(_trace_verbosity_args(_caps, extra_args, os.environ))" in src
+
+
+class TestTheStdoutBufferIsBounded:
+ """Level 4 adds about 30 lines a request; the buffer lives as long as the server."""
+
+ def _drain(self, lines):
+ backend = LlamaCppBackend.__new__(LlamaCppBackend)
+ backend._stdout_lines = []
+ backend._process = _types.SimpleNamespace(stdout = iter(f"{l}\n" for l in lines))
+ backend._drain_stdout()
+ return backend._stdout_lines
+
+ def test_the_startup_head_and_the_tail_survive(self):
+ startup = [f"load {i}" for i in range(200)] + ["main: server is listening on 127.0.0.1"]
+ requests = [f"request {i}" for i in range(60000)]
+ kept = self._drain(startup + requests)
+ assert len(kept) <= llama_cpp_module._STDOUT_TRIM_AT
+ assert kept[: len(startup)] == startup
+ assert kept[-1] == "request 59999"
+ assert len(kept) >= len(startup) + llama_cpp_module._STDOUT_TAIL_KEEP
+
+ def test_a_short_log_is_untouched(self):
+ lines = [f"line {i}" for i in range(500)]
+ assert self._drain(lines) == lines
+
+
+def test_a_flood_before_readiness_keeps_the_startup_head():
+ backend = LlamaCppBackend.__new__(LlamaCppBackend)
+ backend._stdout_lines = []
+ lines = ["load_tensors: offloaded 13/37 layers to GPU"] + [f"trace {i}" for i in range(30000)]
+ backend._process = _types.SimpleNamespace(stdout = iter(f"{l}\n" for l in lines))
+ backend._drain_stdout()
+ assert backend._stdout_lines[0] == lines[0]
+ assert len(backend._stdout_lines) <= llama_cpp_module._STDOUT_TRIM_AT
+
+
+def test_the_offload_report_covers_metal_and_env_pinned_layers():
+ src = inspect.getsource(LlamaCppBackend.load_model)
+ assert "(_detected_gpus or _metal_capable_host())" in src
+ assert "_offload_counts[0] <= 0" in src
+ start = src.index("self._offload_overridden = ")
+ assignment = src[start : src.index("\n if ", start)]
+ # A --device the gpu_ids pin stripped never reached the child, so it pins nothing.
+ assert "self._strip_device_extra_args(extra_args)" in assignment
+ assert "if _gpu_ids_own_device_flags" in assignment
+ assert "_env_fixes_gpu_layers(env)" in assignment
+
+
+def test_the_active_log_file_stops_growing_past_its_cap(tmp_path, monkeypatch):
+ monkeypatch.setattr(llama_cpp_module, "_LLAMA_LOG_FULL_BYTES", 4096)
+ backend = LlamaCppBackend.__new__(LlamaCppBackend)
+ backend._stdout_lines = []
+ log = tmp_path / "llama.log"
+ backend._llama_log_fh = open(log, "w", encoding = "utf-8")
+ lines = [f"0.00.{i:06d} I slot launch: id 0 | task {i} | padding padding" for i in range(2000)]
+ lines += ["0.09.000000 W srv a warning after the cap", "\trepeat_last_n = 64", "error: boom"]
+ backend._process = _types.SimpleNamespace(stdout = iter(f"{l}\n" for l in lines))
+ backend._drain_stdout()
+ backend._llama_log_fh.close()
+ text = log.read_text(encoding = "utf-8")
+ assert len(text) < 4096 + 512
+ assert "keeping warnings and errors only" in text
+ assert "a warning after the cap" in text and "error: boom" in text
+ assert "repeat_last_n" not in text
+
+
+def test_metal_device_rows_count_as_gpu():
+ rows = [
+ "device_info:",
+ " - MTL0 : Apple M3 Max (98304 MiB, 98303 MiB free)",
+ " - CPU : Apple M3 Max",
+ ]
+ assert llama_cpp_module.llama_saw_gpu_device(rows) is True
+ assert llama_cpp_module.llama_saw_gpu_device([rows[0], rows[2]]) is False
diff --git a/studio/backend/tests/test_llama_cpp_context_fit.py b/studio/backend/tests/test_llama_cpp_context_fit.py
index 7856491e45e..45ce1c0f8cb 100644
--- a/studio/backend/tests/test_llama_cpp_context_fit.py
+++ b/studio/backend/tests/test_llama_cpp_context_fit.py
@@ -82,6 +82,7 @@ from core.inference.llama_cpp import (
_CTX_FIT_VRAM_FRACTION,
LlamaCppBackend,
classify_gpu_offload_lines,
+ parse_gpu_offload_counts,
)
from core.inference.llama_server_args import parse_ctx_override, resolve_requested_ctx
@@ -1049,3 +1050,83 @@ class TestAppleNoKvMetadataFloor:
apple_budget_mib = 23_000,
)
assert plan["c_arg"] == 100_000 # explicit honored even without KV sizing
+
+
+class TestPartialOffloadIsReportable:
+ """The counts behind the boolean."""
+
+ def test_a_split_load_keeps_both_numbers(self):
+ assert parse_gpu_offload_counts(["load_tensors: offloaded 38/60 layers to GPU"]) == (38, 60)
+
+ def test_the_boolean_cannot_tell_these_apart(self):
+ split = ["load_tensors: offloaded 12/60 layers to GPU"]
+ whole = ["load_tensors: offloaded 60/60 layers to GPU"]
+ assert classify_gpu_offload_lines(split) is classify_gpu_offload_lines(whole) is True
+ assert parse_gpu_offload_counts(split) != parse_gpu_offload_counts(whole)
+
+ def test_the_main_model_wins_over_a_draft(self):
+ lines = [
+ "load_tensors: offloaded 3/3 layers to GPU",
+ "load_tensors: offloaded 12/60 layers to GPU",
+ ]
+ assert parse_gpu_offload_counts(lines) == (12, 60)
+
+ def test_a_drafter_of_equal_size_does_not_mask_the_main_model(self):
+ # Tie: first line wins, the main model is logged before its drafter.
+ lines = [
+ "load_tensors: offloaded 16/32 layers to GPU",
+ "load_tensors: offloaded 32/32 layers to GPU",
+ ]
+ assert parse_gpu_offload_counts(lines) == (16, 32)
+
+ def test_no_counted_line_is_not_a_guess(self):
+ assert parse_gpu_offload_counts(["INFO starting server"]) is None
+ assert parse_gpu_offload_counts([]) is None
+
+ def test_a_zero_total_is_not_reportable(self):
+ assert parse_gpu_offload_counts(["offloaded 0/0 layers to GPU"]) is None
+
+ def test_an_extras_ngl_reads_as_the_users_own_placement(self):
+ from core.inference.llama_cpp import (
+ _GPU_OFFLOAD_OVERRIDE_FLAGS,
+ _extra_args_set_any_flag,
+ )
+
+ assert _extra_args_set_any_flag(["-ngl", "20"], _GPU_OFFLOAD_OVERRIDE_FLAGS)
+ assert _extra_args_set_any_flag(["--gpu-layers=20"], _GPU_OFFLOAD_OVERRIDE_FLAGS)
+ assert not _extra_args_set_any_flag(["--threads", "8"], _GPU_OFFLOAD_OVERRIDE_FLAGS)
+ assert not _extra_args_set_any_flag(None, _GPU_OFFLOAD_OVERRIDE_FLAGS)
+
+ def test_an_all_cpu_device_table_means_the_backend_did_not_load(self):
+ from core.inference.llama_cpp import llama_saw_gpu_device
+
+ cpu_only = [
+ "device_info: available devices",
+ " - CPU: 12 cores",
+ ]
+ with_gpu = [
+ "device_info: available devices",
+ " - CUDA0: NVIDIA GeForce RTX 4090",
+ " - CPU: 12 cores",
+ ]
+ assert llama_saw_gpu_device(cpu_only) is False
+ assert llama_saw_gpu_device(with_gpu) is True
+ # No table at all is unknown, not a failure.
+ assert llama_saw_gpu_device(["INFO starting server"]) is None
+ assert llama_saw_gpu_device([" - CUDA0: something"]) is None
+
+ def test_a_fit_on_in_extras_is_not_a_pinned_split(self):
+ from core.inference.llama_cpp import _GPU_LAYER_FLAGS, _extra_args_set_any_flag
+
+ assert not _extra_args_set_any_flag(["--fit", "on"], _GPU_LAYER_FLAGS)
+ assert not _extra_args_set_any_flag(["--fit", "off"], _GPU_LAYER_FLAGS)
+ assert _extra_args_set_any_flag(["-ngl", "20"], _GPU_LAYER_FLAGS)
+
+ def test_a_cpu_device_is_deliberate_placement_too(self):
+ from core.inference.llama_cpp import _device_selection_is_cpu
+
+ assert _device_selection_is_cpu(["--device", "cpu"])
+ assert _device_selection_is_cpu(["-dev", "none"])
+ assert _device_selection_is_cpu(None, {"LLAMA_ARG_DEVICE": "cpu"})
+ assert not _device_selection_is_cpu(["--device", "CUDA0"])
+ assert not _device_selection_is_cpu(None)
diff --git a/studio/frontend/src/features/audio/audio-page.tsx b/studio/frontend/src/features/audio/audio-page.tsx
index 657a99c2974..476e596b602 100644
--- a/studio/frontend/src/features/audio/audio-page.tsx
+++ b/studio/frontend/src/features/audio/audio-page.tsx
@@ -50,6 +50,8 @@ import {
listGgufVariants,
listLoras,
loadModel,
+ offloadCountsFrom,
+ offloadWarning,
requestLocalPromptQueueStop,
unloadModel,
useChatRuntimeStore,
@@ -1069,9 +1071,16 @@ export function AudioPage({
);
if (!isCurrent()) return;
if (res.is_audio && isTtsAudioType(res.audio_type)) {
- toast.success(`Model loaded (${res.audio_type ?? "audio"})`, {
- id: toastId,
- });
+ const offloadNotice = offloadWarning(offloadCountsFrom(res));
+ const showToast = offloadNotice ? toast.warning : toast.success;
+ showToast(
+ `Model loaded (${res.audio_type ?? "audio"})${offloadNotice?.titleSuffix ?? ""}`,
+ {
+ id: toastId,
+ description: offloadNotice?.description,
+ duration: offloadNotice ? 8000 : undefined,
+ },
+ );
// Only the native runtime and GGUF can be held in RAM.
if (
wantsCpu &&
diff --git a/studio/frontend/src/features/chat/api/chat-adapter.ts b/studio/frontend/src/features/chat/api/chat-adapter.ts
index 23019c1db17..eb76ff72009 100644
--- a/studio/frontend/src/features/chat/api/chat-adapter.ts
+++ b/studio/frontend/src/features/chat/api/chat-adapter.ts
@@ -368,6 +368,11 @@ import {
supportsChatGenerationRuns,
} from "./chat-generation-api";
import { isDurableRunCandidate, turnRequiresLegacyStream } from "./durable-gate";
+import {
+ type OffloadCounts,
+ offloadCountsFrom,
+ offloadWarning,
+} from "../lib/partial-offload";
// Small models (<=9B) answer from memory, so "auto" forces retrieval for them.
const AUTOINJECT_AUTO_MAX_SIZE_B = 9;
@@ -3098,6 +3103,7 @@ async function autoLoadSmallestModel(options?: AutoLoadOptions): Promise<{
message: string,
cpuFallbackReason?: CpuFallbackReason | null,
mmprojFallbackReason?: MmprojFallbackReason | null,
+ offloadCounts?: OffloadCounts,
): void => {
// Both reasons composed: nesting them as `mmproj ? ... : cpu ? ...` dropped the CPU message.
// That combination is reachable and is the case this feature exists for; see loadFallbackNotice.
@@ -3105,6 +3111,7 @@ async function autoLoadSmallestModel(options?: AutoLoadOptions): Promise<{
message,
cpuFallbackReason,
mmprojFallbackReason,
+ offloadWarning(offloadCounts ?? {}),
);
const options = {
description: notice.description,
@@ -3702,6 +3709,7 @@ async function autoLoadSmallestModel(options?: AutoLoadOptions): Promise<{
candidate.successLabel,
loadResp.cpu_fallback_reason,
loadResp.mmproj_fallback_reason,
+ offloadCountsFrom(loadResp),
);
});
return true;
@@ -4030,6 +4038,7 @@ async function autoLoadSmallestModel(options?: AutoLoadOptions): Promise<{
`Loaded ${DEFAULT_CHAT_MODEL_LABEL} (${DEFAULT_CHAT_MODEL_VARIANT})`,
loadResp.cpu_fallback_reason,
loadResp.mmproj_fallback_reason,
+ offloadCountsFrom(loadResp),
);
});
return { loaded: true, blockedByTrustRemoteCode: false };
diff --git a/studio/frontend/src/features/chat/hooks/use-chat-model-runtime.ts b/studio/frontend/src/features/chat/hooks/use-chat-model-runtime.ts
index aa3db0e8fc4..9e5cf0ba510 100644
--- a/studio/frontend/src/features/chat/hooks/use-chat-model-runtime.ts
+++ b/studio/frontend/src/features/chat/hooks/use-chat-model-runtime.ts
@@ -32,6 +32,11 @@ import {
type ReloadHint,
serverWideReloadRequired,
} from "../lib/server-wide-reload";
+import {
+ type OffloadCounts,
+ offloadCountsFrom,
+ offloadWarning,
+} from "../lib/partial-offload";
import { isSettingsRouteAbsent } from "@/features/settings/api/settings-route-absent";
import {
failureLogPath,
@@ -1860,6 +1865,7 @@ export function useChatModelRuntime() {
let downloadComplete = isDownloaded || isCachedLora;
let cpuFallbackReason: CpuFallbackReason | null = null;
let mmprojFallbackReason: MmprojFallbackReason | null = null;
+ let offloadCounts: OffloadCounts = {};
try {
async function performLoad(): Promise<void> {
if (abortCtrl.signal.aborted) throw new Error("Cancelled");
@@ -2508,6 +2514,7 @@ export function useChatModelRuntime() {
});
cpuFallbackReason = loadResponse.cpu_fallback_reason ?? null;
mmprojFallbackReason = loadResponse.mmproj_fallback_reason ?? null;
+ offloadCounts = offloadCountsFrom(loadResponse);
// If cancelled while loading, do not show the model as active: it is being unloaded.
if (abortCtrl.signal.aborted) throw new Error("Cancelled");
@@ -3334,6 +3341,7 @@ export function useChatModelRuntime() {
`${toastDisplayName} loaded`,
cpuFallbackReason,
mmprojFallbackReason,
+ offloadWarning(offloadCounts),
);
const loadedTitle = notice.title;
const loadedDescription = notice.description;
diff --git a/studio/frontend/src/features/chat/index.ts b/studio/frontend/src/features/chat/index.ts
index ec26d44932e..2759cc1e974 100644
--- a/studio/frontend/src/features/chat/index.ts
+++ b/studio/frontend/src/features/chat/index.ts
@@ -77,6 +77,7 @@ export {
GPU_LAYERS_AUTO,
} from "./stores/chat-runtime-store";
export { resolveStagedDiffusionClassification } from "./lib/gpu-placement";
+export { offloadCountsFrom, offloadWarning } from "./lib/partial-offload";
export {
preferFullToolOutput,
preferSanitizedFullToolOutput,
diff --git a/studio/frontend/src/features/chat/lib/partial-offload.ts b/studio/frontend/src/features/chat/lib/partial-offload.ts
new file mode 100644
index 00000000000..ce5401809c1
--- /dev/null
+++ b/studio/frontend/src/features/chat/lib/partial-offload.ts
@@ -0,0 +1,99 @@
+// SPDX-License-Identifier: AGPL-3.0-only
+// Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
+
+import { CPU_FALLBACK_MESSAGE } from "../utils/mmproj-fallback";
+
+// React-free so node:test can import it; shared by every model load path.
+
+export interface OffloadCounts {
+ offloaded?: number | null;
+ total?: number | null;
+ gpuMemoryMode?: string | null;
+ gpuLayers?: number | null;
+ /** The user's own `-ngl` in llama-server extras, which Auto mode respects. */
+ offloadOverridden?: boolean | null;
+ cpuFallbackReason?: string | null;
+ gpuBackendUnavailable?: boolean | null;
+}
+
+export interface OffloadCountsSource {
+ offloaded_layers?: number | null;
+ offload_total_layers?: number | null;
+ gpu_memory_mode?: string | null;
+ gpu_layers?: number | null;
+ offload_overridden?: boolean | null;
+ cpu_fallback_reason?: string | null;
+ gpu_backend_unavailable?: boolean | null;
+}
+
+/** Pick the split out of a load response, so the load paths cannot drift. */
+export function offloadCountsFrom(
+ response: OffloadCountsSource,
+): OffloadCounts {
+ return {
+ offloaded: response.offloaded_layers,
+ total: response.offload_total_layers,
+ gpuMemoryMode: response.gpu_memory_mode,
+ gpuLayers: response.gpu_layers,
+ offloadOverridden: response.offload_overridden,
+ cpuFallbackReason: response.cpu_fallback_reason,
+ gpuBackendUnavailable: response.gpu_backend_unavailable,
+ };
+}
+
+export interface OffloadWarning {
+ /** Appended to the loaded title, e.g. "Qwen3 loaded, partly on CPU". */
+ titleSuffix: string;
+ description: string;
+}
+
+/** Why a loaded model is not fully on the GPU, or `null`. A known cpu_fallback_reason
+ * wins over the counts: a recovered Vulkan crash also logs 0/M, and is not a size problem.
+ * Manual pins only count with gpuLayers >= 0 (Manual + Auto layers is llama.cpp's fit). */
+export function offloadWarning(counts: OffloadCounts): OffloadWarning | null {
+ const {
+ offloaded,
+ total,
+ gpuMemoryMode,
+ gpuLayers,
+ offloadOverridden,
+ cpuFallbackReason,
+ gpuBackendUnavailable,
+ } = counts;
+ if (cpuFallbackReason === "vulkan_startup_crash") {
+ return {
+ titleSuffix: " on CPU",
+ description: CPU_FALLBACK_MESSAGE,
+ };
+ }
+ // An unrecognised reason is still a reason: say nothing rather than guess at it.
+ if (cpuFallbackReason) return null;
+ if (typeof offloaded !== "number" || typeof total !== "number") return null;
+ if (total <= 0 || offloaded >= total) return null;
+ // Before the pin check: a GPU backend that failed to load is never what was asked for.
+ if (offloaded <= 0 && gpuBackendUnavailable && gpuLayers !== 0) {
+ return {
+ titleSuffix: ", on CPU",
+ description:
+ "The GPU was found but llama.cpp could not use it, so the model runs " +
+ "entirely on CPU and generation will be slow. This is a backend " +
+ "problem rather than a size one, so a smaller quantization will not " +
+ "help. The llama-server log in Settings > Logs has the reason.",
+ };
+ }
+ const manualPin =
+ gpuMemoryMode === "manual" &&
+ typeof gpuLayers === "number" &&
+ gpuLayers >= 0;
+ if (manualPin || offloadOverridden) return null;
+ if (offloaded <= 0) {
+ return {
+ titleSuffix: ", on CPU",
+ description: `None of the ${total} layers fit on the GPU, so the model runs entirely on CPU and generation will be slow. A smaller quantization or a shorter context may leave room on the GPU.`,
+ };
+ }
+ return {
+ titleSuffix: ", partly on CPU",
+ description: `${offloaded} of ${total} layers are on the GPU. The rest run on CPU, so generation will be slower. A smaller quantization or a shorter context may let more of it fit on the GPU.`,
+ };
+}
diff --git a/studio/frontend/src/features/chat/shared-composer.tsx b/studio/frontend/src/features/chat/shared-composer.tsx
index 068a6e26737..b9b453e268c 100644
--- a/studio/frontend/src/features/chat/shared-composer.tsx
+++ b/studio/frontend/src/features/chat/shared-composer.tsx
@@ -2,6 +2,7 @@
// Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
import { mlxRuntimeStateFrom } from "./lib/mlx-runtime-state";
+import { offloadCountsFrom, offloadWarning } from "./lib/partial-offload";
import {
clearedServerTuningState,
committedServerTuningState,
@@ -1707,6 +1708,13 @@ export function SharedComposer({
// Persist the GPU Memory mode on a non-diffusion GGUF compare-load too, so an applied manual
// choice survives a restart.
persistGpuMemoryModeOnLoad(resp, effectiveGpuMemoryMode);
+ const compareOffload = offloadWarning(offloadCountsFrom(resp));
+ if (compareOffload) {
+ toast.warning(
+ `${compareModelDisplayName(sel.id)} loaded${compareOffload.titleSuffix}`,
+ { description: compareOffload.description, duration: 8000 },
+ );
+ }
upgradeUnloadedActive = false;
const store = useChatRuntimeStore.getState();
store.setCheckpoint(
diff --git a/studio/frontend/src/features/chat/types/api.ts b/studio/frontend/src/features/chat/types/api.ts
index aac56bc45f8..4cae8f67425 100644
--- a/studio/frontend/src/features/chat/types/api.ts
+++ b/studio/frontend/src/features/chat/types/api.ts
@@ -285,6 +285,10 @@ export interface LoadModelResponse {
gpu_memory_mode?: "auto" | "manual";
gpu_layers?: number;
/** Set when an automatic Vulkan startup crash was recovered by loading on CPU. */
+ offloaded_layers?: number | null;
+ offload_total_layers?: number | null;
+ offload_overridden?: boolean | null;
+ gpu_backend_unavailable?: boolean | null;
cpu_fallback_reason?: CpuFallbackReason | null;
/** How Unsloth recovered after a multimodal projector failed at startup. */
mmproj_fallback_reason?: MmprojFallbackReason | null;
@@ -397,6 +401,10 @@ export interface InferenceStatusResponse {
gpu_memory_mode?: "auto" | "manual";
gpu_layers?: number;
/** Set while the active model is a recovered CPU-only Vulkan load. */
+ offloaded_layers?: number | null;
+ offload_total_layers?: number | null;
+ offload_overridden?: boolean | null;
+ gpu_backend_unavailable?: boolean | null;
cpu_fallback_reason?: CpuFallbackReason | null;
/** How the active GGUF recovered after a multimodal projector startup failure. */
mmproj_fallback_reason?: MmprojFallbackReason | null;
diff --git a/studio/frontend/src/features/chat/utils/mmproj-fallback.ts b/studio/frontend/src/features/chat/utils/mmproj-fallback.ts
index e4281171331..7d62fee6ca3 100644
--- a/studio/frontend/src/features/chat/utils/mmproj-fallback.ts
+++ b/studio/frontend/src/features/chat/utils/mmproj-fallback.ts
@@ -60,8 +60,11 @@ export function loadFallbackNotice(
baseTitle: string,
cpuFallbackReason: CpuFallbackReason | null | undefined,
mmprojFallbackReason: MmprojFallbackReason | null | undefined,
+ offloadNotice?: { titleSuffix: string; description: string } | null,
): { title: string; description: string | undefined; degraded: boolean } {
const textOnly = isTextOnlyMmprojFallback(mmprojFallbackReason);
+ // A known CPU fallback already explains where the layers went.
+ const offload = cpuFallbackReason ? null : (offloadNotice ?? null);
// When the whole model is on the CPU, saying the projector is too adds nothing. Losing vision
// entirely is a different fact and is always said.
@@ -70,6 +73,10 @@ export function loadFallbackNotice(
suffix = " on CPU, without vision";
} else if (cpuFallbackReason) {
suffix = " on CPU";
+ } else if (offload) {
+ suffix = textOnly
+ ? `${offload.titleSuffix}, without vision`
+ : offload.titleSuffix;
} else if (mmprojFallbackReason === "cpu_offload") {
suffix = " with vision on CPU";
} else if (textOnly) {
@@ -81,6 +88,9 @@ export function loadFallbackNotice(
if (cpuFallbackReason) {
parts.push(CPU_FALLBACK_MESSAGE);
}
+ if (offload) {
+ parts.push(offload.description);
+ }
if (mmprojFallbackReason) {
parts.push(mmprojFallbackMessage(mmprojFallbackReason));
}
@@ -88,6 +98,6 @@ export function loadFallbackNotice(
return {
title: `${baseTitle}${suffix}`,
description: parts.length > 0 ? parts.join(" ") : undefined,
- degraded: Boolean(cpuFallbackReason || mmprojFallbackReason),
+ degraded: Boolean(cpuFallbackReason || offload || mmprojFallbackReason),
};
}
diff --git a/studio/frontend/src/features/recipe-studio/hooks/use-recipe-executions.ts b/studio/frontend/src/features/recipe-studio/hooks/use-recipe-executions.ts
index c42949bbeb9..8ef377bc27a 100644
--- a/studio/frontend/src/features/recipe-studio/hooks/use-recipe-executions.ts
+++ b/studio/frontend/src/features/recipe-studio/hooks/use-recipe-executions.ts
@@ -2,7 +2,13 @@
// Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
import { usePlatformStore } from "@/config/env";
-import { getInferenceStatus, loadModel, validateModel } from "@/features/chat";
+import {
+ getInferenceStatus,
+ loadModel,
+ offloadCountsFrom,
+ offloadWarning,
+ validateModel,
+} from "@/features/chat";
import { unpinnedLoadContext } from "@/features/chat/presets/preset-policy";
// eslint-disable-next-line no-restricted-imports -- Avoid the hub barrel's React and download-manager exports.
import { isOllamaModelId } from "@/features/hub/lib/model-identity";
@@ -338,7 +344,7 @@ async function loadLocalModelSelection(
// A recipe's own target loads the way an unpinned chat model does; restoring the
// model it displaced replays what that model's own load asked for.
const platform = usePlatformStore.getState();
- await loadModel({
+ const loadResp = await loadModel({
// biome-ignore lint/style/useNamingConvention: api schema
model_path: target,
// biome-ignore lint/style/useNamingConvention: api schema
@@ -372,15 +378,18 @@ async function loadLocalModelSelection(
// biome-ignore lint/style/useNamingConvention: api schema
tensor_parallel: false,
});
+ const offloadNotice = offloadWarning(offloadCountsFrom(loadResp));
const successOptions = {
- description: undefined,
- duration: 2000,
+ description: offloadNotice?.description,
+ duration: offloadNotice ? 8000 : 2000,
icon: undefined,
};
+ const title = `Loaded ${modelLabel}${offloadNotice?.titleSuffix ?? ""}`;
+ const showToast = offloadNotice ? toast.warning : toast.success;
if (loadToastDismissed) {
- toast.success(`Loaded ${modelLabel}`, successOptions);
+ showToast(title, successOptions);
} else {
- toast.success(`Loaded ${modelLabel}`, { ...successOptions, id: toastId });
+ showToast(title, { ...successOptions, id: toastId });
}
return null;
} catch (error) {
diff --git a/studio/frontend/tests/auto-load-cpu-fallback-toast.test.ts b/studio/frontend/tests/auto-load-cpu-fallback-toast.test.ts
index 53c6ec84c14..5b2da9effd3 100644
--- a/studio/frontend/tests/auto-load-cpu-fallback-toast.test.ts
+++ b/studio/frontend/tests/auto-load-cpu-fallback-toast.test.ts
@@ -11,6 +11,9 @@ registerBundlerResolver();
const { CPU_FALLBACK_MESSAGE, loadFallbackNotice } = await import(
"../src/features/chat/utils/mmproj-fallback.ts"
);
+const { offloadWarning } = await import(
+ "../src/features/chat/lib/partial-offload.ts"
+);
// Hoisted: biome's useTopLevelRegex flags a literal recompiled per call.
const ON_CPU_SUFFIX = / on CPU$/;
@@ -20,6 +23,10 @@ const PICKS_WARNING = /notice\.degraded \? toast\.warning : toast\.success/;
const PASSES_DESCRIPTION = /description: notice\.description/;
const FORWARDS_REASON = /cpu_fallback_reason/;
const CALL_SITES = /\n\s*showAutoLoadSuccess\([\s\S]*?\);/g;
+const PARTIAL_SPLIT = /38 of 60 layers are on the GPU/;
+const TEXT_ONLY = /text-only mode/;
+const OFFLOAD_WARNING_CALL = /offloadWarning\(offloadCounts/;
+const FORWARDS_COUNTS = /offloadCountsFrom\(loadResp\)/;
// Asserted against the source like the other chat-adapter tests: importing the
// module would drag in the stores and the toast layer for one closure.
@@ -60,6 +67,51 @@ test("the auto-load toast is driven by that verdict, not by its own copy of it",
assert.match(helper, PASSES_DESCRIPTION);
});
+test("an auto-load that only partly reached the GPU warns too", () => {
+ const offload = offloadWarning({ offloaded: 38, total: 60 });
+ const notice = loadFallbackNotice("Loaded Qwen3", null, null, offload);
+ assert.equal(notice.degraded, true);
+ assert.equal(notice.title, "Loaded Qwen3, partly on CPU");
+ assert.match(notice.description ?? "", PARTIAL_SPLIT);
+
+ const start = source.indexOf("const showAutoLoadSuccess = (");
+ const helper = source.slice(start, source.indexOf("\n };", start));
+ assert.match(helper, OFFLOAD_WARNING_CALL);
+});
+
+test("a known CPU fallback is not also reported as a split", () => {
+ const offload = offloadWarning({ offloaded: 0, total: 60 });
+ const notice = loadFallbackNotice(
+ "Loaded Qwen3",
+ "vulkan_startup_crash",
+ null,
+ offload,
+ );
+ assert.equal(notice.title, "Loaded Qwen3 on CPU");
+ assert.equal(notice.description, CPU_FALLBACK_MESSAGE);
+});
+
+test("a split load that also lost vision says both", () => {
+ const offload = offloadWarning({ offloaded: 38, total: 60 });
+ const notice = loadFallbackNotice(
+ "Loaded Qwen3",
+ null,
+ "projector_incompatible",
+ offload,
+ );
+ assert.equal(notice.title, "Loaded Qwen3, partly on CPU, without vision");
+ assert.match(notice.description ?? "", PARTIAL_SPLIT);
+ assert.match(notice.description ?? "", TEXT_ONLY);
+});
+
+test("every auto-load path forwards the offload counts", () => {
+ const calls = source.match(CALL_SITES) ?? [];
+ assert.ok(calls.length > 0, "no showAutoLoadSuccess call sites found");
+ for (const call of calls) {
+ assert.match(call, FORWARDS_COUNTS);
+ }
+});
+
test("every auto-load path forwards the load response's fallback reason", () => {
const calls = source.match(CALL_SITES) ?? [];
assert.ok(calls.length > 0, "no showAutoLoadSuccess call sites found");
diff --git a/studio/frontend/tests/offload-warning-load-paths.test.ts b/studio/frontend/tests/offload-warning-load-paths.test.ts
new file mode 100644
index 00000000000..477ff7ff915
--- /dev/null
+++ b/studio/frontend/tests/offload-warning-load-paths.test.ts
@@ -0,0 +1,40 @@
+// SPDX-License-Identifier: AGPL-3.0-only
+// Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
+
+import assert from "node:assert/strict";
+import { readFileSync } from "node:fs";
+import test from "node:test";
+import { fileURLToPath } from "node:url";
+
+// Every load path must surface the offload warning, not a plain success toast.
+const LOAD_PATHS = [
+ "../src/features/chat/hooks/use-chat-model-runtime.ts",
+ "../src/features/chat/api/chat-adapter.ts",
+ "../src/features/chat/shared-composer.tsx",
+ "../src/features/recipe-studio/hooks/use-recipe-executions.ts",
+ "../src/features/audio/audio-page.tsx",
+];
+
+test("every user-facing load path consults the offload warning", () => {
+ for (const path of LOAD_PATHS) {
+ const source = readFileSync(
+ fileURLToPath(new URL(path, import.meta.url)),
+ "utf8",
+ );
+ assert.match(source, /offloadWarning\(/, `${path} ignores the split`);
+ }
+});
+
+test("a recipe run no longer claims plain success unconditionally", () => {
+ const source = readFileSync(
+ fileURLToPath(
+ new URL(
+ "../src/features/recipe-studio/hooks/use-recipe-executions.ts",
+ import.meta.url,
+ ),
+ ),
+ "utf8",
+ );
+ assert.match(source, /offloadNotice \? toast\.warning : toast\.success/);
+ assert.doesNotMatch(source, /toast\.success\(`Loaded \$\{modelLabel\}`/);
+});
diff --git a/studio/frontend/tests/partial-offload.test.ts b/studio/frontend/tests/partial-offload.test.ts
new file mode 100644
index 00000000000..ae1661387aa
--- /dev/null
+++ b/studio/frontend/tests/partial-offload.test.ts
@@ -0,0 +1,197 @@
+// SPDX-License-Identifier: AGPL-3.0-only
+// Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
+
+import assert from "node:assert/strict";
+import test from "node:test";
+
+import { registerBundlerResolver } from "./helpers/kit.ts";
+
+registerBundlerResolver();
+
+const { offloadCountsFrom, offloadWarning } = await import(
+ "../src/features/chat/lib/partial-offload.ts"
+);
+
+test("a split load is reported", () => {
+ assert.match(
+ offloadWarning({ offloaded: 38, total: 60 })?.description ?? "",
+ /^38 of 60 layers are on the GPU\./,
+ );
+ assert.equal(
+ offloadWarning({ offloaded: 38, total: 60 })?.titleSuffix,
+ ", partly on CPU",
+ );
+ assert.notEqual(offloadWarning({ offloaded: 1, total: 60 }), null);
+});
+
+test("a full offload is the normal case and says nothing", () => {
+ assert.equal(offloadWarning({ offloaded: 60, total: 60 }), null);
+});
+
+test("no layers on the GPU is the worst case, not an excluded one", () => {
+ const warning = offloadWarning({ offloaded: 0, total: 60 });
+ assert.equal(warning?.titleSuffix, ", on CPU");
+ assert.match(warning?.description ?? "", /^None of the 60 layers fit/);
+});
+
+test("a load that reported no counts says nothing", () => {
+ assert.equal(offloadWarning({}), null);
+ assert.equal(offloadWarning({ offloaded: null, total: null }), null);
+ assert.equal(offloadWarning({ offloaded: 38, total: null }), null);
+ assert.equal(offloadWarning({ offloaded: undefined, total: 60 }), null);
+});
+
+test("a nonsense total cannot produce a warning", () => {
+ assert.equal(offloadWarning({ offloaded: 5, total: 0 }), null);
+ assert.equal(offloadWarning({ offloaded: 5, total: -1 }), null);
+ assert.equal(offloadWarning({ offloaded: 61, total: 60 }), null);
+});
+
+test("a split the user pinned themselves is not warned about", () => {
+ assert.equal(
+ offloadWarning({
+ offloaded: 20,
+ total: 60,
+ gpuMemoryMode: "manual",
+ gpuLayers: 20,
+ }),
+ null,
+ );
+ assert.notEqual(
+ offloadWarning({ offloaded: 20, total: 60, gpuMemoryMode: "auto" }),
+ null,
+ );
+ // Absent (an older backend) is treated as automatic, which is the default.
+ assert.notEqual(offloadWarning({ offloaded: 20, total: 60 }), null);
+});
+
+test("an -ngl passed through extras counts as the user's own choice", () => {
+ assert.equal(
+ offloadWarning({
+ offloaded: 20,
+ total: 60,
+ gpuMemoryMode: "auto",
+ offloadOverridden: true,
+ }),
+ null,
+ );
+ assert.equal(
+ offloadWarning({ offloaded: 0, total: 60, offloadOverridden: true }),
+ null,
+ );
+});
+
+test("every load path reads the response the same way", () => {
+ assert.deepEqual(
+ offloadCountsFrom({
+ // biome-ignore lint/style/useNamingConvention: api schema
+ offloaded_layers: 38,
+ // biome-ignore lint/style/useNamingConvention: api schema
+ offload_total_layers: 60,
+ // biome-ignore lint/style/useNamingConvention: api schema
+ gpu_memory_mode: "auto",
+ // biome-ignore lint/style/useNamingConvention: api schema
+ gpu_layers: -1,
+ // biome-ignore lint/style/useNamingConvention: api schema
+ offload_overridden: false,
+ // biome-ignore lint/style/useNamingConvention: api schema
+ cpu_fallback_reason: null,
+ // biome-ignore lint/style/useNamingConvention: api schema
+ gpu_backend_unavailable: false,
+ }),
+ {
+ offloaded: 38,
+ total: 60,
+ gpuMemoryMode: "auto",
+ gpuLayers: -1,
+ offloadOverridden: false,
+ cpuFallbackReason: null,
+ gpuBackendUnavailable: false,
+ },
+ );
+});
+
+test("Manual mode with GPU Layers on Auto is still an automatic spill", () => {
+ assert.notEqual(
+ offloadWarning({
+ offloaded: 20,
+ total: 60,
+ gpuMemoryMode: "manual",
+ gpuLayers: -1,
+ }),
+ null,
+ );
+ assert.notEqual(
+ offloadWarning({ offloaded: 0, total: 60, gpuMemoryMode: "manual" }),
+ null,
+ );
+});
+
+test("a GPU that llama.cpp could not use is not a size problem", () => {
+ const broken = offloadWarning({
+ offloaded: 0,
+ total: 60,
+ gpuBackendUnavailable: true,
+ });
+ assert.match(broken?.description ?? "", /could not use it/);
+ assert.doesNotMatch(broken?.description ?? "", /may leave room|may let more/);
+ assert.match(
+ offloadWarning({ offloaded: 0, total: 60 })?.description ?? "",
+ /None of the 60 layers fit/,
+ );
+ assert.match(
+ offloadWarning({ offloaded: 20, total: 60, gpuBackendUnavailable: true })
+ ?.description ?? "",
+ /^20 of 60 layers/,
+ );
+});
+
+test("a known reason for the CPU wins over the counts", () => {
+ const warning = offloadWarning({
+ offloaded: 0,
+ total: 60,
+ cpuFallbackReason: "vulkan_startup_crash",
+ });
+ assert.equal(warning?.titleSuffix, " on CPU");
+ assert.match(warning?.description ?? "", /Vulkan backend crashed/);
+ assert.doesNotMatch(warning?.description ?? "", /smaller quantization/);
+ // An unrecognised reason is still a reason: say nothing rather than guess.
+ assert.equal(
+ offloadWarning({ offloaded: 0, total: 60, cpuFallbackReason: "something" }),
+ null,
+ );
+});
+
+test("a requested GPU split that got no GPU at all still warns", () => {
+ for (const counts of [
+ { gpuMemoryMode: "manual", gpuLayers: 20 },
+ { gpuMemoryMode: "auto", gpuLayers: -1, offloadOverridden: true },
+ ]) {
+ const warning = offloadWarning({
+ offloaded: 0,
+ total: 60,
+ gpuBackendUnavailable: true,
+ ...counts,
+ });
+ assert.match(warning?.description ?? "", /could not use it/);
+ }
+ // Manual 0 layers asked for the CPU.
+ assert.equal(
+ offloadWarning({
+ offloaded: 0,
+ total: 60,
+ gpuBackendUnavailable: true,
+ gpuMemoryMode: "manual",
+ gpuLayers: 0,
+ }),
+ null,
+ );
+});
+
+test("the advice does not promise that a smaller quantization fits", () => {
+ for (const offloaded of [0, 38]) {
+ const text = offloadWarning({ offloaded, total: 60 })?.description ?? "";
+ assert.match(text, /smaller quantization or a shorter context may/);
+ assert.doesNotMatch(text, /would fit entirely|would leave room/);
+ }
+});
diff --git a/tests/studio/test_chat_autoload_failure_gate.py b/tests/studio/test_chat_autoload_failure_gate.py
index 8c7ed9c3c8a..4c380b24f80 100644
--- a/tests/studio/test_chat_autoload_failure_gate.py
+++ b/tests/studio/test_chat_autoload_failure_gate.py
@@ -387,9 +387,11 @@ function loadFallbackNotice(
baseTitle: string,
cpuFallbackReason: any,
mmprojFallbackReason: any,
+ offloadNotice?: any,
) {
const parts: string[] = [];
if (cpuFallbackReason) parts.push(`cpu:${cpuFallbackReason}`);
+ else if (offloadNotice) parts.push(`offload:${offloadNotice.titleSuffix}`);
if (mmprojFallbackReason) parts.push(`mmproj:${mmprojFallbackReason}`);
return {
title: parts.length > 0 ? `${baseTitle} (${parts.join(", ")})` : baseTitle,
@@ -397,6 +399,17 @@ function loadFallbackNotice(
degraded: parts.length > 0,
};
}
+// From ../lib/partial-offload: the auto-load success toast reports a layer split through them.
+// Real semantics are tested in studio/frontend/tests/partial-offload.test.ts.
+function offloadCountsFrom(response: any) {
+ return { offloaded: response?.offloaded_layers, total: response?.offload_total_layers };
+}
+function offloadWarning(counts: any) {
+ const { offloaded, total } = counts ?? {};
+ if (typeof offloaded !== "number" || typeof total !== "number") return null;
+ if (total <= 0 || offloaded >= total) return null;
+ return { titleSuffix: ", partly on CPU", description: `${offloaded} of ${total}` };
+}
async function prepareHfTokenForUse(token: any) {
EVENTS.push({ kind: "prepareHfToken" });
if (SCENARIO.tokenDecision === "decline") return { proceed: false, token };
diff --git a/tests/studio/test_model_picker_contracts.py b/tests/studio/test_model_picker_contracts.py
index 73bf24437b5..762b9ddc080 100644
--- a/tests/studio/test_model_picker_contracts.py
+++ b/tests/studio/test_model_picker_contracts.py
@@ -425,9 +425,10 @@ def test_recipe_model_load_toast_is_persistent_and_dismissible():
assert "closeButton: true" in model_load
assert "icon: createLoadingToastIcon()" in model_load
assert "onDismiss:" in model_load
- assert "description: undefined" in model_load
+ # A plain success clears the loading description and lasts 2 s; only a layer split says more.
+ assert "description: offloadNotice?.description" in model_load
assert "icon: undefined" in model_load
- assert "duration: 2000" in model_load
+ assert "duration: offloadNotice ? 8000 : 2000" in model_load
toast_lib = _read("lib/toast.ts")
assert "createElement(Spinner" in toast_lib