Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
Studio: say when a model only partly fits on the GPU (#8812)
Daniel Han01b8b84c5e6f
diff --git a/studio/backend/core/inference/llama_cpp.py b/studio/backend/core/inference/llama_cpp.pyindex 03dc68ade04..62cd457b8c9 100644--- a/studio/backend/core/inference/llama_cpp.py+++ b/studio/backend/core/inference/llama_cpp.py@@ -729,13 +729,40 @@ _GPU_OFFLOAD_MARKERS = (_OFFLOADED_LAYERS_RE = re.compile(r"offloaded\s+(\d+)\s*/\s*(\d+)\s+layers?\s+to\s+gpu", re.IGNORECASE)+# llama.cpp #23021 moved libllama INFO (offload counts, device table, buffer sizes) to trace (4).+_LLAMA_TRACE_VERBOSITY = 4+_TRACE_VERBOSITY_HELP_RE = re.compile(r"\b4:\s*trace\b", re.IGNORECASE)+_LOG_VERBOSITY_FLAGS = frozenset(+ {"-lv", "--verbosity", "--log-verbosity", "-v", "--verbose", "--log-verbose", "--log-disable"}+)+# Past this, the active llama-server log keeps only warnings and errors.+_LLAMA_LOG_FULL_BYTES = 64 * 1024 * 1024+_LOG_LEVEL_TOKEN_RE = re.compile(r"^\S+ ([A-Z]) ")+# The startup head is never trimmed: its load lines are read for the server's whole life.+_STDOUT_TRIM_AT = 20000+_STDOUT_TAIL_KEEP = 5000+_STDOUT_HEAD_MAX = 5000+++def _trace_verbosity_args(caps: dict, extra_args: Optional[Iterable[str]], environ) -> list[str]:+ """``--verbosity 4`` when the build hides load lines at its default and the user chose none."""+ if not caps.get("supports_trace_verbosity"):+ return []+ if _extra_args_set_any_flag(extra_args, _LOG_VERBOSITY_FLAGS):+ return []+ if "LLAMA_ARG_LOG_VERBOSITY" in environ:+ return []+ return ["--verbosity", str(_LLAMA_TRACE_VERBOSITY)]++_GPU_MODEL_BUFFER_RE = re.compile(r"\b(?:CUDA|ROCm|ROCM|HIP|SYCL)(\d+)\s+model buffer size\s*=\s*"r"([0-9]+(?:\.[0-9]+)?)\s+MiB\b",re.IGNORECASE,)_DEVICE_ROW_RE = re.compile(- r"-\s*(CUDA|ROCm|ROCM|HIP|Metal|Vulkan|SYCL|OpenCL|MUSA|CANN|CPU)\w*\s*:",+ # MTL: ggml-metal names its devices MTL0, MTL1 (GGML_METAL_NAME), never "Metal".+ r"-\s*(CUDA|ROCm|ROCM|HIP|Metal|MTL|Vulkan|SYCL|OpenCL|MUSA|CANN|CPU)\w*\s*:",re.IGNORECASE,)_GPU_DEVICE_PREFIXES = (@@ -743,6 +770,7 @@ _GPU_DEVICE_PREFIXES = ("rocm","hip","metal",+ "mtl","vulkan","sycl","opencl",@@ -751,6 +779,46 @@ _GPU_DEVICE_PREFIXES = ()+def parse_gpu_offload_counts(lines: "list[str]") -> "Optional[tuple[int, int]]":+ """``(offloaded, total)`` for the largest-M line, or None; ties keep the first+ line since llama.cpp logs the main model before its drafter."""+ max_total = -1+ offloaded_at_max = 0+ for line in lines:+ match = _OFFLOADED_LAYERS_RE.search(line)+ if not match:+ continue+ offloaded, total = int(match.group(1)), int(match.group(2))+ if total > max_total:+ max_total, offloaded_at_max = total, offloaded+ if max_total <= 0:+ return None+ return offloaded_at_max, max_total+++def llama_saw_gpu_device(lines: "list[str]") -> Optional[bool]:+ """Whether llama.cpp's device table (rows after the header) lists a GPU, or None+ with no table. All-CPU means the GPU backend failed to load."""+ after_header = False+ saw_device_row = False+ saw_gpu_device = False+ for line in lines:+ if "device_info:" in line:+ after_header = True+ continue+ if not after_header:+ continue+ match = _DEVICE_ROW_RE.search(line)+ if not match:+ continue+ saw_device_row = True+ if match.group(1).lower().startswith(_GPU_DEVICE_PREFIXES):+ saw_gpu_device = True+ if not saw_device_row:+ return None+ return saw_gpu_device++def classify_gpu_offload_lines(lines: "list[str]") -> Optional[bool]:"""True if the model landed on a GPU, False if it stayed on CPU despite GPUintent, None when the log has no usable signal."""@@ -786,23 +854,8 @@ def classify_gpu_offload_lines(lines: "list[str]") -> Optional[bool]:# device_info: lists *available* devices (printed whenever a GPU backend is# visible), not where the model loaded, so it can only disconfirm: an# all-CPU table means no usable GPU. A visible GPU device is not proof the- # model used it, so it does not return True. Rows after the header only.- after_header = False- saw_device_row = False- saw_gpu_device = False- for line in lines:- if "device_info:" in line:- after_header = True- continue- if not after_header:- continue- match = _DEVICE_ROW_RE.search(line)- if not match:- continue- saw_device_row = True- if match.group(1).lower().startswith(_GPU_DEVICE_PREFIXES):- saw_gpu_device = True- if saw_device_row and not saw_gpu_device:+ # model used it, so it does not return True.+ if llama_saw_gpu_device(lines) is False:return Falsereturn None@@ -7320,6 +7373,12 @@ class LlamaCppBackend:self._stats_logger = None # vLLM-style engine-stats poller, set on load# Set by _classify_gpu_offload after _wait_for_health.self._gpu_offload_active: Optional[bool] = None+ # (offloaded, total) layers llama.cpp reported for this load, when it said.+ self._gpu_offload_layers: Optional[tuple[int, int]] = None+ # Whether the user's own extras pinned the layer split for this load.+ self._offload_overridden: bool = False+ # Studio saw a GPU for this load but llama.cpp's device table did not.+ self._gpu_backend_unavailable: bool = False# Diffusion only: the split the running child was asked for.self._diffusion_requested_ngl: Optional[int] = Noneself._context_length: Optional[int] = None@@ -7794,6 +7853,25 @@ class LlamaCppBackend:slots = 1return max(1, slots)+ @property+ def offloaded_layers(self) -> Optional[int]:+ """Layers llama.cpp actually put on a GPU, when its log said so."""+ return self._gpu_offload_layers[0] if self._gpu_offload_layers else None++ @property+ def offload_total_layers(self) -> Optional[int]:+ return self._gpu_offload_layers[1] if self._gpu_offload_layers else None++ @property+ def gpu_backend_unavailable(self) -> bool:+ """Studio found a GPU for this load but llama.cpp reported none."""+ return self._gpu_backend_unavailable++ @property+ def offload_overridden(self) -> bool:+ """Whether the user's own extras (Auto mode keeps -ngl) chose this split."""+ return self._offload_overridden+@propertydef requested_parallel_slots(self) -> int:"""--parallel the last load asked for, before any fit-time reduction.@@ -9123,6 +9201,7 @@ class LlamaCppBackend:# Fails CLOSED, unlike --jinja: a build without --reasoning exits on it."supports_reasoning_flag": False,"supports_no_mmproj_offload": False,+ "supports_trace_verbosity": False,"supports_video_fps": False,"supports_load_mode": False,"spec_draft_ngl_flag": None,@@ -9157,6 +9236,7 @@ class LlamaCppBackend:spec_draft_n_max_flag: Optional[str] = Nonespec_draft_n_max_default: Optional[int] = Nonesupports_kv_unified = False+ supports_trace_verbosity = False# See the fallback dict: these fail open.supports_no_context_shift = Truesupports_jinja = True@@ -9372,6 +9452,12 @@ class LlamaCppBackend:spec_draft_n_max_default = int(_n_max_default.group(1))supports_kv_unified = _is_real("--kv-unified")+ # Fails closed: an older build's level 4 may be per-token debug on the decode path.+ supports_trace_verbosity = bool(+ probe_ok+ and _is_real("--verbosity")+ and _TRACE_VERBOSITY_HELP_RE.search(blocks.get("--verbosity") or "")+ )# Only once the help actually parsed AND `--help` succeeded. Empty# `blocks` means the probe told us nothing, and a nonzero exit means# the listing may be a fragment; neither may read as "absent" for a@@ -9509,6 +9595,7 @@ class LlamaCppBackend:"reasoning_budget_probe_inconclusive": not (probe_ok and help_nonempty),"supports_reasoning_flag": supports_reasoning_flag,"supports_no_mmproj_offload": supports_no_mmproj_offload,+ "supports_trace_verbosity": supports_trace_verbosity,"supports_video_fps": supports_video_fps,"supports_load_mode": supports_load_mode,"spec_draft_ngl_flag": spec_draft_ngl_flag,@@ -17490,15 +17577,25 @@ class LlamaCppBackend:post-mortem has the full output even if the crash predates thedrain-thread join in ``_wait_for_health``."""+ startup_len: Optional[int] = None+ log_bytes = 0try:for line in self._process.stdout:line = line.rstrip()if line:- self._stdout_lines.append(line)+ lines = self._stdout_lines+ lines.append(line)+ if len(lines) > _STDOUT_TRIM_AT:+ # Before readiness the whole log so far is startup: keep its head too.+ head = _STDOUT_HEAD_MAX if startup_len is None else startup_len+ head = min(head, _STDOUT_HEAD_MAX, len(lines))+ del lines[head : len(lines) - _STDOUT_TAIL_KEEP]# Two forms across llama.cpp builds. Only readiness lines wake the# probe: waking on every tensor-load log spins startup.line_lower = line.lower()if "server is listening" in line_lower or "model loaded" in line_lower:+ if startup_len is None:+ startup_len = len(self._stdout_lines)health_probe_event = getattr(self, "_health_probe_event", None)if health_probe_event is not None:health_probe_event.set()@@ -17506,6 +17603,19 @@ class LlamaCppBackend:fh = getattr(self, "_llama_log_fh", None)if fh is not None:try:+ if log_bytes >= _LLAMA_LOG_FULL_BYTES:+ level = _LOG_LEVEL_TOKEN_RE.match(line)+ if not (+ level.group(1) in "WE"+ if level is not None+ else ("error" in line_lower or "fail" in line_lower)+ ):+ continue+ elif log_bytes + len(line) + 1 >= _LLAMA_LOG_FULL_BYTES:+ fh.write(+ "[studio] log past 64 MiB: keeping warnings and errors only\n"+ )+ log_bytes += len(line) + 1fh.write(line + "\n")fh.flush()except (ValueError, OSError):@@ -27252,6 +27362,7 @@ class LlamaCppBackend:# Enable Jinja chat template renderingif _caps.get("supports_jinja", True):cmd.extend(["--jinja"])+ cmd.extend(_trace_verbosity_args(_caps, extra_args, os.environ))# KV cache data type_valid_cache_types = _VALID_CACHE_TYPES@@ -31089,6 +31200,46 @@ class LlamaCppBackend:gpu_indices is not None or use_fit or gpu_memory_mode == "manual",_detected_gpus,)+ # CPU-only hosts log 0/N too, so a zero needs an expected GPU; a positive count stands.+ _offload_counts = (+ None+ if _arch_gate_forced_cpu or _deliberate_cpu_only+ else parse_gpu_offload_counts(self._stdout_lines)+ )+ if (+ _offload_counts is not None+ and _offload_counts[0] <= 0+ and not (+ (_detected_gpus or _metal_capable_host())+ and (gpu_indices is not None or use_fit or gpu_memory_mode == "manual")+ )+ ):+ _offload_counts = None+ self._gpu_offload_layers = _offload_counts+ self._gpu_backend_unavailable = bool(_detected_gpus or _metal_capable_host()) and (+ llama_saw_gpu_device(self._stdout_lines) is False+ )+ # Not _GPU_OFFLOAD_OVERRIDE_FLAGS: --fit on is exactly the split to report.+ self._offload_overridden = (+ _extra_args_set_any_flag(extra_args, _GPU_LAYER_FLAGS)+ or _device_selection_is_cpu(+ self._strip_device_extra_args(extra_args)+ if _gpu_ids_own_device_flags+ else extra_args,+ env,+ )+ or _env_fixes_gpu_layers(env)+ )+ if (+ self._gpu_offload_layers is not None+ and 0 < self._gpu_offload_layers[0] < self._gpu_offload_layers[1]+ ):+ logger.warning(+ "llama-server offloaded %d of %d layers to GPU; the rest run on "+ "CPU, which is much slower for a dense model.",+ self._gpu_offload_layers[0],+ self._gpu_offload_layers[1],+ )if self._gpu_offload_active is False and not _deliberate_cpu_only:logger.warning("llama-server appears to have loaded the model entirely "@@ -32452,6 +32603,9 @@ class LlamaCppBackend:# loading) server as VRAM-resident rather than reading the killed# server's stale zero-offload flag until the health probe reclassifies.self._gpu_offload_active = None+ self._gpu_offload_layers = None+ self._offload_overridden = False+ self._gpu_backend_unavailable = False# Drives _wait_for_vram_settle in the next load_model; set in finally# so both in-process and frontend Apply paths record the kill.self._last_kill_monotonic = time.monotonic()diff --git a/studio/backend/models/inference.py b/studio/backend/models/inference.pyindex 68d58a2c5db..d61bac1507b 100644--- a/studio/backend/models/inference.py+++ b/studio/backend/models/inference.py@@ -1485,6 +1485,34 @@ class _InferenceRuntimeFields(BaseModel):-1,description = "Manual mode: requested --gpu-layers value (-1 = Auto/--fit, or when not manual).",)+ offloaded_layers: Optional[int] = Field(+ None,+ description = (+ "Layers llama.cpp actually placed on a GPU, when it reported the count. "+ "In Auto mode the placement is llama.cpp's own (--fit on), so this is the "+ "only account of what happened; gpu_layers above is the REQUEST."+ ),+ )+ offload_total_layers: Optional[int] = Field(+ None,+ description = "Total layers in the model, alongside offloaded_layers.",+ )+ gpu_backend_unavailable: bool = Field(+ False,+ description = (+ "Studio detected a GPU for this load but llama.cpp's own device table "+ "reported none, meaning its GPU backend did not initialise. Separates a "+ "backend failure from a fit that placed no layers; both log 0/M."+ ),+ )+ offload_overridden: bool = Field(+ False,+ description = (+ "Whether the user's own llama-server extras pinned the layer split. "+ "Auto mode respects an inherited -ngl instead of stripping it, so a "+ "deliberate placement can arrive with gpu_memory_mode still 'auto'."+ ),+ )cpu_fallback_reason: Optional[Literal["vulkan_startup_crash"]] = Field(None,description = (diff --git a/studio/backend/tests/llama_backend_double.py b/studio/backend/tests/llama_backend_double.pyindex 26e7a9daeb7..e7b24915f98 100644--- a/studio/backend/tests/llama_backend_double.py+++ b/studio/backend/tests/llama_backend_double.py@@ -55,6 +55,8 @@ class FakeLlamaCppBackend:gpu_layers = -1n_cpu_moe = 0n_moe_layers = 0+ gpu_backend_unavailable = False+ offload_overridden = False# Private: the only name the real backend has for these, so the one production falls back to._is_audio = False_has_audio_input = Falsediff --git a/studio/backend/tests/test_launch_flags_are_capability_gated.py b/studio/backend/tests/test_launch_flags_are_capability_gated.pyindex 45ea8358a47..e508fa695a1 100644--- a/studio/backend/tests/test_launch_flags_are_capability_gated.py+++ b/studio/backend/tests/test_launch_flags_are_capability_gated.py@@ -504,3 +504,138 @@ class TestTheFlaglessFixupKeepsMlaKAndVEqual:cmd, _ = _flagless_v_cache_fixup(known_off = True, cmd = self.CMD, env = {}, mla = False)assert cmd[cmd.index("--cache-type-k") + 1] == "q8_0"assert cmd[cmd.index("--cache-type-v") + 1] == "f16"+++# The -lv block of b11160 and later (llama.cpp "logs : reduce", #23021).+TRACE_HELP = NEW_HELP + (+ "-lv, --verbosity, --log-verbosity N Set the verbosity threshold. Messages with a higher verbosity will be\n"+ " ignored. Values:\n"+ " - 0: generic output\n"+ " - 1: error\n"+ " - 2: warning\n"+ " - 3: info\n"+ " - 4: trace (more info)\n"+ " - 5: debug\n"+ " (default: 3)\n"+)+PRE_TRACE_HELP = NEW_HELP + (+ "-lv, --verbosity, --log-verbosity N Set the verbosity threshold. Messages with a higher verbosity will be\n"+ " ignored.\n"+)+++class TestTraceVerbosityRestoresTheLoadLog:+ """Load lines the offload report and classifiers read are trace-level on current builds."""++ def test_a_trace_build_is_detected(self, tmp_path, monkeypatch):+ assert probe(tmp_path, monkeypatch, TRACE_HELP)["supports_trace_verbosity"] is True++ @pytest.mark.parametrize(+ "help_text,returncode",+ [(PRE_TRACE_HELP, 0), (NEW_HELP, 0), (TRACE_HELP, 1), ("", 0)],+ )+ def test_anything_else_fails_closed(self, tmp_path, monkeypatch, help_text, returncode):+ caps = probe(tmp_path, monkeypatch, help_text, returncode)+ assert caps["supports_trace_verbosity"] is False++ def test_an_unprobeable_binary_fails_closed(self, tmp_path):+ caps = LlamaCppBackend.probe_server_capabilities(str(tmp_path / "absent"))+ assert caps["supports_trace_verbosity"] is False++ def test_a_trace_build_gets_level_4(self):+ args = llama_cpp_module._trace_verbosity_args({"supports_trace_verbosity": True}, [], {})+ assert args == ["--verbosity", "4"]++ @pytest.mark.parametrize(+ "extra_args",+ [["-lv", "2"], ["--verbosity=5"], ["--log-verbosity", "1"], ["-v"], ["--log-disable"]],+ )+ def test_the_users_own_logging_choice_wins(self, extra_args):+ caps = {"supports_trace_verbosity": True}+ assert llama_cpp_module._trace_verbosity_args(caps, extra_args, {}) == []++ def test_the_env_form_wins_too(self):+ caps = {"supports_trace_verbosity": True}+ env = {"LLAMA_ARG_LOG_VERBOSITY": "3"}+ assert llama_cpp_module._trace_verbosity_args(caps, None, env) == []++ def test_other_builds_get_nothing(self):+ assert llama_cpp_module._trace_verbosity_args({}, None, {}) == []++ def test_the_launch_emits_it(self):+ src = inspect.getsource(LlamaCppBackend.load_model)+ assert "cmd.extend(_trace_verbosity_args(_caps, extra_args, os.environ))" in src+++class TestTheStdoutBufferIsBounded:+ """Level 4 adds about 30 lines a request; the buffer lives as long as the server."""++ def _drain(self, lines):+ backend = LlamaCppBackend.__new__(LlamaCppBackend)+ backend._stdout_lines = []+ backend._process = _types.SimpleNamespace(stdout = iter(f"{l}\n" for l in lines))+ backend._drain_stdout()+ return backend._stdout_lines++ def test_the_startup_head_and_the_tail_survive(self):+ startup = [f"load {i}" for i in range(200)] + ["main: server is listening on 127.0.0.1"]+ requests = [f"request {i}" for i in range(60000)]+ kept = self._drain(startup + requests)+ assert len(kept) <= llama_cpp_module._STDOUT_TRIM_AT+ assert kept[: len(startup)] == startup+ assert kept[-1] == "request 59999"+ assert len(kept) >= len(startup) + llama_cpp_module._STDOUT_TAIL_KEEP++ def test_a_short_log_is_untouched(self):+ lines = [f"line {i}" for i in range(500)]+ assert self._drain(lines) == lines+++def test_a_flood_before_readiness_keeps_the_startup_head():+ backend = LlamaCppBackend.__new__(LlamaCppBackend)+ backend._stdout_lines = []+ lines = ["load_tensors: offloaded 13/37 layers to GPU"] + [f"trace {i}" for i in range(30000)]+ backend._process = _types.SimpleNamespace(stdout = iter(f"{l}\n" for l in lines))+ backend._drain_stdout()+ assert backend._stdout_lines[0] == lines[0]+ assert len(backend._stdout_lines) <= llama_cpp_module._STDOUT_TRIM_AT+++def test_the_offload_report_covers_metal_and_env_pinned_layers():+ src = inspect.getsource(LlamaCppBackend.load_model)+ assert "(_detected_gpus or _metal_capable_host())" in src+ assert "_offload_counts[0] <= 0" in src+ start = src.index("self._offload_overridden = ")+ assignment = src[start : src.index("\n if ", start)]+ # A --device the gpu_ids pin stripped never reached the child, so it pins nothing.+ assert "self._strip_device_extra_args(extra_args)" in assignment+ assert "if _gpu_ids_own_device_flags" in assignment+ assert "_env_fixes_gpu_layers(env)" in assignment+++def test_the_active_log_file_stops_growing_past_its_cap(tmp_path, monkeypatch):+ monkeypatch.setattr(llama_cpp_module, "_LLAMA_LOG_FULL_BYTES", 4096)+ backend = LlamaCppBackend.__new__(LlamaCppBackend)+ backend._stdout_lines = []+ log = tmp_path / "llama.log"+ backend._llama_log_fh = open(log, "w", encoding = "utf-8")+ lines = [f"0.00.{i:06d} I slot launch: id 0 | task {i} | padding padding" for i in range(2000)]+ lines += ["0.09.000000 W srv a warning after the cap", "\trepeat_last_n = 64", "error: boom"]+ backend._process = _types.SimpleNamespace(stdout = iter(f"{l}\n" for l in lines))+ backend._drain_stdout()+ backend._llama_log_fh.close()+ text = log.read_text(encoding = "utf-8")+ assert len(text) < 4096 + 512+ assert "keeping warnings and errors only" in text+ assert "a warning after the cap" in text and "error: boom" in text+ assert "repeat_last_n" not in text+++def test_metal_device_rows_count_as_gpu():+ rows = [+ "device_info:",+ " - MTL0 : Apple M3 Max (98304 MiB, 98303 MiB free)",+ " - CPU : Apple M3 Max",+ ]+ assert llama_cpp_module.llama_saw_gpu_device(rows) is True+ assert llama_cpp_module.llama_saw_gpu_device([rows[0], rows[2]]) is Falsediff --git a/studio/backend/tests/test_llama_cpp_context_fit.py b/studio/backend/tests/test_llama_cpp_context_fit.pyindex 7856491e45e..45ce1c0f8cb 100644--- a/studio/backend/tests/test_llama_cpp_context_fit.py+++ b/studio/backend/tests/test_llama_cpp_context_fit.py@@ -82,6 +82,7 @@ from core.inference.llama_cpp import (_CTX_FIT_VRAM_FRACTION,LlamaCppBackend,classify_gpu_offload_lines,+ parse_gpu_offload_counts,)from core.inference.llama_server_args import parse_ctx_override, resolve_requested_ctx@@ -1049,3 +1050,83 @@ class TestAppleNoKvMetadataFloor:apple_budget_mib = 23_000,)assert plan["c_arg"] == 100_000 # explicit honored even without KV sizing+++class TestPartialOffloadIsReportable:+ """The counts behind the boolean."""++ def test_a_split_load_keeps_both_numbers(self):+ assert parse_gpu_offload_counts(["load_tensors: offloaded 38/60 layers to GPU"]) == (38, 60)++ def test_the_boolean_cannot_tell_these_apart(self):+ split = ["load_tensors: offloaded 12/60 layers to GPU"]+ whole = ["load_tensors: offloaded 60/60 layers to GPU"]+ assert classify_gpu_offload_lines(split) is classify_gpu_offload_lines(whole) is True+ assert parse_gpu_offload_counts(split) != parse_gpu_offload_counts(whole)++ def test_the_main_model_wins_over_a_draft(self):+ lines = [+ "load_tensors: offloaded 3/3 layers to GPU",+ "load_tensors: offloaded 12/60 layers to GPU",+ ]+ assert parse_gpu_offload_counts(lines) == (12, 60)++ def test_a_drafter_of_equal_size_does_not_mask_the_main_model(self):+ # Tie: first line wins, the main model is logged before its drafter.+ lines = [+ "load_tensors: offloaded 16/32 layers to GPU",+ "load_tensors: offloaded 32/32 layers to GPU",+ ]+ assert parse_gpu_offload_counts(lines) == (16, 32)++ def test_no_counted_line_is_not_a_guess(self):+ assert parse_gpu_offload_counts(["INFO starting server"]) is None+ assert parse_gpu_offload_counts([]) is None++ def test_a_zero_total_is_not_reportable(self):+ assert parse_gpu_offload_counts(["offloaded 0/0 layers to GPU"]) is None++ def test_an_extras_ngl_reads_as_the_users_own_placement(self):+ from core.inference.llama_cpp import (+ _GPU_OFFLOAD_OVERRIDE_FLAGS,+ _extra_args_set_any_flag,+ )++ assert _extra_args_set_any_flag(["-ngl", "20"], _GPU_OFFLOAD_OVERRIDE_FLAGS)+ assert _extra_args_set_any_flag(["--gpu-layers=20"], _GPU_OFFLOAD_OVERRIDE_FLAGS)+ assert not _extra_args_set_any_flag(["--threads", "8"], _GPU_OFFLOAD_OVERRIDE_FLAGS)+ assert not _extra_args_set_any_flag(None, _GPU_OFFLOAD_OVERRIDE_FLAGS)++ def test_an_all_cpu_device_table_means_the_backend_did_not_load(self):+ from core.inference.llama_cpp import llama_saw_gpu_device++ cpu_only = [+ "device_info: available devices",+ " - CPU: 12 cores",+ ]+ with_gpu = [+ "device_info: available devices",+ " - CUDA0: NVIDIA GeForce RTX 4090",+ " - CPU: 12 cores",+ ]+ assert llama_saw_gpu_device(cpu_only) is False+ assert llama_saw_gpu_device(with_gpu) is True+ # No table at all is unknown, not a failure.+ assert llama_saw_gpu_device(["INFO starting server"]) is None+ assert llama_saw_gpu_device([" - CUDA0: something"]) is None++ def test_a_fit_on_in_extras_is_not_a_pinned_split(self):+ from core.inference.llama_cpp import _GPU_LAYER_FLAGS, _extra_args_set_any_flag++ assert not _extra_args_set_any_flag(["--fit", "on"], _GPU_LAYER_FLAGS)+ assert not _extra_args_set_any_flag(["--fit", "off"], _GPU_LAYER_FLAGS)+ assert _extra_args_set_any_flag(["-ngl", "20"], _GPU_LAYER_FLAGS)++ def test_a_cpu_device_is_deliberate_placement_too(self):+ from core.inference.llama_cpp import _device_selection_is_cpu++ assert _device_selection_is_cpu(["--device", "cpu"])+ assert _device_selection_is_cpu(["-dev", "none"])+ assert _device_selection_is_cpu(None, {"LLAMA_ARG_DEVICE": "cpu"})+ assert not _device_selection_is_cpu(["--device", "CUDA0"])+ assert not _device_selection_is_cpu(None)diff --git a/studio/frontend/src/features/audio/audio-page.tsx b/studio/frontend/src/features/audio/audio-page.tsxindex 657a99c2974..476e596b602 100644--- a/studio/frontend/src/features/audio/audio-page.tsx+++ b/studio/frontend/src/features/audio/audio-page.tsx@@ -50,6 +50,8 @@ import {listGgufVariants,listLoras,loadModel,+ offloadCountsFrom,+ offloadWarning,requestLocalPromptQueueStop,unloadModel,useChatRuntimeStore,@@ -1069,9 +1071,16 @@ export function AudioPage({);if (!isCurrent()) return;if (res.is_audio && isTtsAudioType(res.audio_type)) {- toast.success(`Model loaded (${res.audio_type ?? "audio"})`, {- id: toastId,- });+ const offloadNotice = offloadWarning(offloadCountsFrom(res));+ const showToast = offloadNotice ? toast.warning : toast.success;+ showToast(+ `Model loaded (${res.audio_type ?? "audio"})${offloadNotice?.titleSuffix ?? ""}`,+ {+ id: toastId,+ description: offloadNotice?.description,+ duration: offloadNotice ? 8000 : undefined,+ },+ );// Only the native runtime and GGUF can be held in RAM.if (wantsCpu &&diff --git a/studio/frontend/src/features/chat/api/chat-adapter.ts b/studio/frontend/src/features/chat/api/chat-adapter.tsindex 23019c1db17..eb76ff72009 100644--- a/studio/frontend/src/features/chat/api/chat-adapter.ts+++ b/studio/frontend/src/features/chat/api/chat-adapter.ts@@ -368,6 +368,11 @@ import {supportsChatGenerationRuns,} from "./chat-generation-api";import { isDurableRunCandidate, turnRequiresLegacyStream } from "./durable-gate";+import {+ type OffloadCounts,+ offloadCountsFrom,+ offloadWarning,+} from "../lib/partial-offload";// Small models (<=9B) answer from memory, so "auto" forces retrieval for them.const AUTOINJECT_AUTO_MAX_SIZE_B = 9;@@ -3098,6 +3103,7 @@ async function autoLoadSmallestModel(options?: AutoLoadOptions): Promise<{message: string,cpuFallbackReason?: CpuFallbackReason | null,mmprojFallbackReason?: MmprojFallbackReason | null,+ offloadCounts?: OffloadCounts,): void => {// Both reasons composed: nesting them as `mmproj ? ... : cpu ? ...` dropped the CPU message.// That combination is reachable and is the case this feature exists for; see loadFallbackNotice.@@ -3105,6 +3111,7 @@ async function autoLoadSmallestModel(options?: AutoLoadOptions): Promise<{message,cpuFallbackReason,mmprojFallbackReason,+ offloadWarning(offloadCounts ?? {}),);const options = {description: notice.description,@@ -3702,6 +3709,7 @@ async function autoLoadSmallestModel(options?: AutoLoadOptions): Promise<{candidate.successLabel,loadResp.cpu_fallback_reason,loadResp.mmproj_fallback_reason,+ offloadCountsFrom(loadResp),);});return true;@@ -4030,6 +4038,7 @@ async function autoLoadSmallestModel(options?: AutoLoadOptions): Promise<{`Loaded ${DEFAULT_CHAT_MODEL_LABEL} (${DEFAULT_CHAT_MODEL_VARIANT})`,loadResp.cpu_fallback_reason,loadResp.mmproj_fallback_reason,+ offloadCountsFrom(loadResp),);});return { loaded: true, blockedByTrustRemoteCode: false };diff --git a/studio/frontend/src/features/chat/hooks/use-chat-model-runtime.ts b/studio/frontend/src/features/chat/hooks/use-chat-model-runtime.tsindex aa3db0e8fc4..9e5cf0ba510 100644--- a/studio/frontend/src/features/chat/hooks/use-chat-model-runtime.ts+++ b/studio/frontend/src/features/chat/hooks/use-chat-model-runtime.ts@@ -32,6 +32,11 @@ import {type ReloadHint,serverWideReloadRequired,} from "../lib/server-wide-reload";+import {+ type OffloadCounts,+ offloadCountsFrom,+ offloadWarning,+} from "../lib/partial-offload";import { isSettingsRouteAbsent } from "@/features/settings/api/settings-route-absent";import {failureLogPath,@@ -1860,6 +1865,7 @@ export function useChatModelRuntime() {let downloadComplete = isDownloaded || isCachedLora;let cpuFallbackReason: CpuFallbackReason | null = null;let mmprojFallbackReason: MmprojFallbackReason | null = null;+ let offloadCounts: OffloadCounts = {};try {async function performLoad(): Promise<void> {if (abortCtrl.signal.aborted) throw new Error("Cancelled");@@ -2508,6 +2514,7 @@ export function useChatModelRuntime() {});cpuFallbackReason = loadResponse.cpu_fallback_reason ?? null;mmprojFallbackReason = loadResponse.mmproj_fallback_reason ?? null;+ offloadCounts = offloadCountsFrom(loadResponse);// If cancelled while loading, do not show the model as active: it is being unloaded.if (abortCtrl.signal.aborted) throw new Error("Cancelled");@@ -3334,6 +3341,7 @@ export function useChatModelRuntime() {`${toastDisplayName} loaded`,cpuFallbackReason,mmprojFallbackReason,+ offloadWarning(offloadCounts),);const loadedTitle = notice.title;const loadedDescription = notice.description;diff --git a/studio/frontend/src/features/chat/index.ts b/studio/frontend/src/features/chat/index.tsindex ec26d44932e..2759cc1e974 100644--- a/studio/frontend/src/features/chat/index.ts+++ b/studio/frontend/src/features/chat/index.ts@@ -77,6 +77,7 @@ export {GPU_LAYERS_AUTO,} from "./stores/chat-runtime-store";export { resolveStagedDiffusionClassification } from "./lib/gpu-placement";+export { offloadCountsFrom, offloadWarning } from "./lib/partial-offload";export {preferFullToolOutput,preferSanitizedFullToolOutput,diff --git a/studio/frontend/src/features/chat/lib/partial-offload.ts b/studio/frontend/src/features/chat/lib/partial-offload.tsnew file mode 100644index 00000000000..ce5401809c1--- /dev/null+++ b/studio/frontend/src/features/chat/lib/partial-offload.ts@@ -0,0 +1,99 @@+// SPDX-License-Identifier: AGPL-3.0-only+// Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0++import { CPU_FALLBACK_MESSAGE } from "../utils/mmproj-fallback";++// React-free so node:test can import it; shared by every model load path.++export interface OffloadCounts {+ offloaded?: number | null;+ total?: number | null;+ gpuMemoryMode?: string | null;+ gpuLayers?: number | null;+ /** The user's own `-ngl` in llama-server extras, which Auto mode respects. */+ offloadOverridden?: boolean | null;+ cpuFallbackReason?: string | null;+ gpuBackendUnavailable?: boolean | null;+}++export interface OffloadCountsSource {+ offloaded_layers?: number | null;+ offload_total_layers?: number | null;+ gpu_memory_mode?: string | null;+ gpu_layers?: number | null;+ offload_overridden?: boolean | null;+ cpu_fallback_reason?: string | null;+ gpu_backend_unavailable?: boolean | null;+}++/** Pick the split out of a load response, so the load paths cannot drift. */+export function offloadCountsFrom(+ response: OffloadCountsSource,+): OffloadCounts {+ return {+ offloaded: response.offloaded_layers,+ total: response.offload_total_layers,+ gpuMemoryMode: response.gpu_memory_mode,+ gpuLayers: response.gpu_layers,+ offloadOverridden: response.offload_overridden,+ cpuFallbackReason: response.cpu_fallback_reason,+ gpuBackendUnavailable: response.gpu_backend_unavailable,+ };+}++export interface OffloadWarning {+ /** Appended to the loaded title, e.g. "Qwen3 loaded, partly on CPU". */+ titleSuffix: string;+ description: string;+}++/** Why a loaded model is not fully on the GPU, or `null`. A known cpu_fallback_reason+ * wins over the counts: a recovered Vulkan crash also logs 0/M, and is not a size problem.+ * Manual pins only count with gpuLayers >= 0 (Manual + Auto layers is llama.cpp's fit). */+export function offloadWarning(counts: OffloadCounts): OffloadWarning | null {+ const {+ offloaded,+ total,+ gpuMemoryMode,+ gpuLayers,+ offloadOverridden,+ cpuFallbackReason,+ gpuBackendUnavailable,+ } = counts;+ if (cpuFallbackReason === "vulkan_startup_crash") {+ return {+ titleSuffix: " on CPU",+ description: CPU_FALLBACK_MESSAGE,+ };+ }+ // An unrecognised reason is still a reason: say nothing rather than guess at it.+ if (cpuFallbackReason) return null;+ if (typeof offloaded !== "number" || typeof total !== "number") return null;+ if (total <= 0 || offloaded >= total) return null;+ // Before the pin check: a GPU backend that failed to load is never what was asked for.+ if (offloaded <= 0 && gpuBackendUnavailable && gpuLayers !== 0) {+ return {+ titleSuffix: ", on CPU",+ description:+ "The GPU was found but llama.cpp could not use it, so the model runs " ++ "entirely on CPU and generation will be slow. This is a backend " ++ "problem rather than a size one, so a smaller quantization will not " ++ "help. The llama-server log in Settings > Logs has the reason.",+ };+ }+ const manualPin =+ gpuMemoryMode === "manual" &&+ typeof gpuLayers === "number" &&+ gpuLayers >= 0;+ if (manualPin || offloadOverridden) return null;+ if (offloaded <= 0) {+ return {+ titleSuffix: ", on CPU",+ description: `None of the ${total} layers fit on the GPU, so the model runs entirely on CPU and generation will be slow. A smaller quantization or a shorter context may leave room on the GPU.`,+ };+ }+ return {+ titleSuffix: ", partly on CPU",+ description: `${offloaded} of ${total} layers are on the GPU. The rest run on CPU, so generation will be slower. A smaller quantization or a shorter context may let more of it fit on the GPU.`,+ };+}diff --git a/studio/frontend/src/features/chat/shared-composer.tsx b/studio/frontend/src/features/chat/shared-composer.tsxindex 068a6e26737..b9b453e268c 100644--- a/studio/frontend/src/features/chat/shared-composer.tsx+++ b/studio/frontend/src/features/chat/shared-composer.tsx@@ -2,6 +2,7 @@// Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0import { mlxRuntimeStateFrom } from "./lib/mlx-runtime-state";+import { offloadCountsFrom, offloadWarning } from "./lib/partial-offload";import {clearedServerTuningState,committedServerTuningState,@@ -1707,6 +1708,13 @@ export function SharedComposer({// Persist the GPU Memory mode on a non-diffusion GGUF compare-load too, so an applied manual// choice survives a restart.persistGpuMemoryModeOnLoad(resp, effectiveGpuMemoryMode);+ const compareOffload = offloadWarning(offloadCountsFrom(resp));+ if (compareOffload) {+ toast.warning(+ `${compareModelDisplayName(sel.id)} loaded${compareOffload.titleSuffix}`,+ { description: compareOffload.description, duration: 8000 },+ );+ }upgradeUnloadedActive = false;const store = useChatRuntimeStore.getState();store.setCheckpoint(diff --git a/studio/frontend/src/features/chat/types/api.ts b/studio/frontend/src/features/chat/types/api.tsindex aac56bc45f8..4cae8f67425 100644--- a/studio/frontend/src/features/chat/types/api.ts+++ b/studio/frontend/src/features/chat/types/api.ts@@ -285,6 +285,10 @@ export interface LoadModelResponse {gpu_memory_mode?: "auto" | "manual";gpu_layers?: number;/** Set when an automatic Vulkan startup crash was recovered by loading on CPU. */+ offloaded_layers?: number | null;+ offload_total_layers?: number | null;+ offload_overridden?: boolean | null;+ gpu_backend_unavailable?: boolean | null;cpu_fallback_reason?: CpuFallbackReason | null;/** How Unsloth recovered after a multimodal projector failed at startup. */mmproj_fallback_reason?: MmprojFallbackReason | null;@@ -397,6 +401,10 @@ export interface InferenceStatusResponse {gpu_memory_mode?: "auto" | "manual";gpu_layers?: number;/** Set while the active model is a recovered CPU-only Vulkan load. */+ offloaded_layers?: number | null;+ offload_total_layers?: number | null;+ offload_overridden?: boolean | null;+ gpu_backend_unavailable?: boolean | null;cpu_fallback_reason?: CpuFallbackReason | null;/** How the active GGUF recovered after a multimodal projector startup failure. */mmproj_fallback_reason?: MmprojFallbackReason | null;diff --git a/studio/frontend/src/features/chat/utils/mmproj-fallback.ts b/studio/frontend/src/features/chat/utils/mmproj-fallback.tsindex e4281171331..7d62fee6ca3 100644--- a/studio/frontend/src/features/chat/utils/mmproj-fallback.ts+++ b/studio/frontend/src/features/chat/utils/mmproj-fallback.ts@@ -60,8 +60,11 @@ export function loadFallbackNotice(baseTitle: string,cpuFallbackReason: CpuFallbackReason | null | undefined,mmprojFallbackReason: MmprojFallbackReason | null | undefined,+ offloadNotice?: { titleSuffix: string; description: string } | null,): { title: string; description: string | undefined; degraded: boolean } {const textOnly = isTextOnlyMmprojFallback(mmprojFallbackReason);+ // A known CPU fallback already explains where the layers went.+ const offload = cpuFallbackReason ? null : (offloadNotice ?? null);// When the whole model is on the CPU, saying the projector is too adds nothing. Losing vision// entirely is a different fact and is always said.@@ -70,6 +73,10 @@ export function loadFallbackNotice(suffix = " on CPU, without vision";} else if (cpuFallbackReason) {suffix = " on CPU";+ } else if (offload) {+ suffix = textOnly+ ? `${offload.titleSuffix}, without vision`+ : offload.titleSuffix;} else if (mmprojFallbackReason === "cpu_offload") {suffix = " with vision on CPU";} else if (textOnly) {@@ -81,6 +88,9 @@ export function loadFallbackNotice(if (cpuFallbackReason) {parts.push(CPU_FALLBACK_MESSAGE);}+ if (offload) {+ parts.push(offload.description);+ }if (mmprojFallbackReason) {parts.push(mmprojFallbackMessage(mmprojFallbackReason));}@@ -88,6 +98,6 @@ export function loadFallbackNotice(return {title: `${baseTitle}${suffix}`,description: parts.length > 0 ? parts.join(" ") : undefined,- degraded: Boolean(cpuFallbackReason || mmprojFallbackReason),+ degraded: Boolean(cpuFallbackReason || offload || mmprojFallbackReason),};}diff --git a/studio/frontend/src/features/recipe-studio/hooks/use-recipe-executions.ts b/studio/frontend/src/features/recipe-studio/hooks/use-recipe-executions.tsindex c42949bbeb9..8ef377bc27a 100644--- a/studio/frontend/src/features/recipe-studio/hooks/use-recipe-executions.ts+++ b/studio/frontend/src/features/recipe-studio/hooks/use-recipe-executions.ts@@ -2,7 +2,13 @@// Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0import { usePlatformStore } from "@/config/env";-import { getInferenceStatus, loadModel, validateModel } from "@/features/chat";+import {+ getInferenceStatus,+ loadModel,+ offloadCountsFrom,+ offloadWarning,+ validateModel,+} from "@/features/chat";import { unpinnedLoadContext } from "@/features/chat/presets/preset-policy";// eslint-disable-next-line no-restricted-imports -- Avoid the hub barrel's React and download-manager exports.import { isOllamaModelId } from "@/features/hub/lib/model-identity";@@ -338,7 +344,7 @@ async function loadLocalModelSelection(// A recipe's own target loads the way an unpinned chat model does; restoring the// model it displaced replays what that model's own load asked for.const platform = usePlatformStore.getState();- await loadModel({+ const loadResp = await loadModel({// biome-ignore lint/style/useNamingConvention: api schemamodel_path: target,// biome-ignore lint/style/useNamingConvention: api schema@@ -372,15 +378,18 @@ async function loadLocalModelSelection(// biome-ignore lint/style/useNamingConvention: api schematensor_parallel: false,});+ const offloadNotice = offloadWarning(offloadCountsFrom(loadResp));const successOptions = {- description: undefined,- duration: 2000,+ description: offloadNotice?.description,+ duration: offloadNotice ? 8000 : 2000,icon: undefined,};+ const title = `Loaded ${modelLabel}${offloadNotice?.titleSuffix ?? ""}`;+ const showToast = offloadNotice ? toast.warning : toast.success;if (loadToastDismissed) {- toast.success(`Loaded ${modelLabel}`, successOptions);+ showToast(title, successOptions);} else {- toast.success(`Loaded ${modelLabel}`, { ...successOptions, id: toastId });+ showToast(title, { ...successOptions, id: toastId });}return null;} catch (error) {diff --git a/studio/frontend/tests/auto-load-cpu-fallback-toast.test.ts b/studio/frontend/tests/auto-load-cpu-fallback-toast.test.tsindex 53c6ec84c14..5b2da9effd3 100644--- a/studio/frontend/tests/auto-load-cpu-fallback-toast.test.ts+++ b/studio/frontend/tests/auto-load-cpu-fallback-toast.test.ts@@ -11,6 +11,9 @@ registerBundlerResolver();const { CPU_FALLBACK_MESSAGE, loadFallbackNotice } = await import("../src/features/chat/utils/mmproj-fallback.ts");+const { offloadWarning } = await import(+ "../src/features/chat/lib/partial-offload.ts"+);// Hoisted: biome's useTopLevelRegex flags a literal recompiled per call.const ON_CPU_SUFFIX = / on CPU$/;@@ -20,6 +23,10 @@ const PICKS_WARNING = /notice\.degraded \? toast\.warning : toast\.success/;const PASSES_DESCRIPTION = /description: notice\.description/;const FORWARDS_REASON = /cpu_fallback_reason/;const CALL_SITES = /\n\s*showAutoLoadSuccess\([\s\S]*?\);/g;+const PARTIAL_SPLIT = /38 of 60 layers are on the GPU/;+const TEXT_ONLY = /text-only mode/;+const OFFLOAD_WARNING_CALL = /offloadWarning\(offloadCounts/;+const FORWARDS_COUNTS = /offloadCountsFrom\(loadResp\)/;// Asserted against the source like the other chat-adapter tests: importing the// module would drag in the stores and the toast layer for one closure.@@ -60,6 +67,51 @@ test("the auto-load toast is driven by that verdict, not by its own copy of it",assert.match(helper, PASSES_DESCRIPTION);});+test("an auto-load that only partly reached the GPU warns too", () => {+ const offload = offloadWarning({ offloaded: 38, total: 60 });+ const notice = loadFallbackNotice("Loaded Qwen3", null, null, offload);+ assert.equal(notice.degraded, true);+ assert.equal(notice.title, "Loaded Qwen3, partly on CPU");+ assert.match(notice.description ?? "", PARTIAL_SPLIT);++ const start = source.indexOf("const showAutoLoadSuccess = (");+ const helper = source.slice(start, source.indexOf("\n };", start));+ assert.match(helper, OFFLOAD_WARNING_CALL);+});++test("a known CPU fallback is not also reported as a split", () => {+ const offload = offloadWarning({ offloaded: 0, total: 60 });+ const notice = loadFallbackNotice(+ "Loaded Qwen3",+ "vulkan_startup_crash",+ null,+ offload,+ );+ assert.equal(notice.title, "Loaded Qwen3 on CPU");+ assert.equal(notice.description, CPU_FALLBACK_MESSAGE);+});++test("a split load that also lost vision says both", () => {+ const offload = offloadWarning({ offloaded: 38, total: 60 });+ const notice = loadFallbackNotice(+ "Loaded Qwen3",+ null,+ "projector_incompatible",+ offload,+ );+ assert.equal(notice.title, "Loaded Qwen3, partly on CPU, without vision");+ assert.match(notice.description ?? "", PARTIAL_SPLIT);+ assert.match(notice.description ?? "", TEXT_ONLY);+});++test("every auto-load path forwards the offload counts", () => {+ const calls = source.match(CALL_SITES) ?? [];+ assert.ok(calls.length > 0, "no showAutoLoadSuccess call sites found");+ for (const call of calls) {+ assert.match(call, FORWARDS_COUNTS);+ }+});+test("every auto-load path forwards the load response's fallback reason", () => {const calls = source.match(CALL_SITES) ?? [];assert.ok(calls.length > 0, "no showAutoLoadSuccess call sites found");diff --git a/studio/frontend/tests/offload-warning-load-paths.test.ts b/studio/frontend/tests/offload-warning-load-paths.test.tsnew file mode 100644index 00000000000..477ff7ff915--- /dev/null+++ b/studio/frontend/tests/offload-warning-load-paths.test.ts@@ -0,0 +1,40 @@+// SPDX-License-Identifier: AGPL-3.0-only+// Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0++import assert from "node:assert/strict";+import { readFileSync } from "node:fs";+import test from "node:test";+import { fileURLToPath } from "node:url";++// Every load path must surface the offload warning, not a plain success toast.+const LOAD_PATHS = [+ "../src/features/chat/hooks/use-chat-model-runtime.ts",+ "../src/features/chat/api/chat-adapter.ts",+ "../src/features/chat/shared-composer.tsx",+ "../src/features/recipe-studio/hooks/use-recipe-executions.ts",+ "../src/features/audio/audio-page.tsx",+];++test("every user-facing load path consults the offload warning", () => {+ for (const path of LOAD_PATHS) {+ const source = readFileSync(+ fileURLToPath(new URL(path, import.meta.url)),+ "utf8",+ );+ assert.match(source, /offloadWarning\(/, `${path} ignores the split`);+ }+});++test("a recipe run no longer claims plain success unconditionally", () => {+ const source = readFileSync(+ fileURLToPath(+ new URL(+ "../src/features/recipe-studio/hooks/use-recipe-executions.ts",+ import.meta.url,+ ),+ ),+ "utf8",+ );+ assert.match(source, /offloadNotice \? toast\.warning : toast\.success/);+ assert.doesNotMatch(source, /toast\.success\(`Loaded \$\{modelLabel\}`/);+});diff --git a/studio/frontend/tests/partial-offload.test.ts b/studio/frontend/tests/partial-offload.test.tsnew file mode 100644index 00000000000..ae1661387aa--- /dev/null+++ b/studio/frontend/tests/partial-offload.test.ts@@ -0,0 +1,197 @@+// SPDX-License-Identifier: AGPL-3.0-only+// Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0++import assert from "node:assert/strict";+import test from "node:test";++import { registerBundlerResolver } from "./helpers/kit.ts";++registerBundlerResolver();++const { offloadCountsFrom, offloadWarning } = await import(+ "../src/features/chat/lib/partial-offload.ts"+);++test("a split load is reported", () => {+ assert.match(+ offloadWarning({ offloaded: 38, total: 60 })?.description ?? "",+ /^38 of 60 layers are on the GPU\./,+ );+ assert.equal(+ offloadWarning({ offloaded: 38, total: 60 })?.titleSuffix,+ ", partly on CPU",+ );+ assert.notEqual(offloadWarning({ offloaded: 1, total: 60 }), null);+});++test("a full offload is the normal case and says nothing", () => {+ assert.equal(offloadWarning({ offloaded: 60, total: 60 }), null);+});++test("no layers on the GPU is the worst case, not an excluded one", () => {+ const warning = offloadWarning({ offloaded: 0, total: 60 });+ assert.equal(warning?.titleSuffix, ", on CPU");+ assert.match(warning?.description ?? "", /^None of the 60 layers fit/);+});++test("a load that reported no counts says nothing", () => {+ assert.equal(offloadWarning({}), null);+ assert.equal(offloadWarning({ offloaded: null, total: null }), null);+ assert.equal(offloadWarning({ offloaded: 38, total: null }), null);+ assert.equal(offloadWarning({ offloaded: undefined, total: 60 }), null);+});++test("a nonsense total cannot produce a warning", () => {+ assert.equal(offloadWarning({ offloaded: 5, total: 0 }), null);+ assert.equal(offloadWarning({ offloaded: 5, total: -1 }), null);+ assert.equal(offloadWarning({ offloaded: 61, total: 60 }), null);+});++test("a split the user pinned themselves is not warned about", () => {+ assert.equal(+ offloadWarning({+ offloaded: 20,+ total: 60,+ gpuMemoryMode: "manual",+ gpuLayers: 20,+ }),+ null,+ );+ assert.notEqual(+ offloadWarning({ offloaded: 20, total: 60, gpuMemoryMode: "auto" }),+ null,+ );+ // Absent (an older backend) is treated as automatic, which is the default.+ assert.notEqual(offloadWarning({ offloaded: 20, total: 60 }), null);+});++test("an -ngl passed through extras counts as the user's own choice", () => {+ assert.equal(+ offloadWarning({+ offloaded: 20,+ total: 60,+ gpuMemoryMode: "auto",+ offloadOverridden: true,+ }),+ null,+ );+ assert.equal(+ offloadWarning({ offloaded: 0, total: 60, offloadOverridden: true }),+ null,+ );+});++test("every load path reads the response the same way", () => {+ assert.deepEqual(+ offloadCountsFrom({+ // biome-ignore lint/style/useNamingConvention: api schema+ offloaded_layers: 38,+ // biome-ignore lint/style/useNamingConvention: api schema+ offload_total_layers: 60,+ // biome-ignore lint/style/useNamingConvention: api schema+ gpu_memory_mode: "auto",+ // biome-ignore lint/style/useNamingConvention: api schema+ gpu_layers: -1,+ // biome-ignore lint/style/useNamingConvention: api schema+ offload_overridden: false,+ // biome-ignore lint/style/useNamingConvention: api schema+ cpu_fallback_reason: null,+ // biome-ignore lint/style/useNamingConvention: api schema+ gpu_backend_unavailable: false,+ }),+ {+ offloaded: 38,+ total: 60,+ gpuMemoryMode: "auto",+ gpuLayers: -1,+ offloadOverridden: false,+ cpuFallbackReason: null,+ gpuBackendUnavailable: false,+ },+ );+});++test("Manual mode with GPU Layers on Auto is still an automatic spill", () => {+ assert.notEqual(+ offloadWarning({+ offloaded: 20,+ total: 60,+ gpuMemoryMode: "manual",+ gpuLayers: -1,+ }),+ null,+ );+ assert.notEqual(+ offloadWarning({ offloaded: 0, total: 60, gpuMemoryMode: "manual" }),+ null,+ );+});++test("a GPU that llama.cpp could not use is not a size problem", () => {+ const broken = offloadWarning({+ offloaded: 0,+ total: 60,+ gpuBackendUnavailable: true,+ });+ assert.match(broken?.description ?? "", /could not use it/);+ assert.doesNotMatch(broken?.description ?? "", /may leave room|may let more/);+ assert.match(+ offloadWarning({ offloaded: 0, total: 60 })?.description ?? "",+ /None of the 60 layers fit/,+ );+ assert.match(+ offloadWarning({ offloaded: 20, total: 60, gpuBackendUnavailable: true })+ ?.description ?? "",+ /^20 of 60 layers/,+ );+});++test("a known reason for the CPU wins over the counts", () => {+ const warning = offloadWarning({+ offloaded: 0,+ total: 60,+ cpuFallbackReason: "vulkan_startup_crash",+ });+ assert.equal(warning?.titleSuffix, " on CPU");+ assert.match(warning?.description ?? "", /Vulkan backend crashed/);+ assert.doesNotMatch(warning?.description ?? "", /smaller quantization/);+ // An unrecognised reason is still a reason: say nothing rather than guess.+ assert.equal(+ offloadWarning({ offloaded: 0, total: 60, cpuFallbackReason: "something" }),+ null,+ );+});++test("a requested GPU split that got no GPU at all still warns", () => {+ for (const counts of [+ { gpuMemoryMode: "manual", gpuLayers: 20 },+ { gpuMemoryMode: "auto", gpuLayers: -1, offloadOverridden: true },+ ]) {+ const warning = offloadWarning({+ offloaded: 0,+ total: 60,+ gpuBackendUnavailable: true,+ ...counts,+ });+ assert.match(warning?.description ?? "", /could not use it/);+ }+ // Manual 0 layers asked for the CPU.+ assert.equal(+ offloadWarning({+ offloaded: 0,+ total: 60,+ gpuBackendUnavailable: true,+ gpuMemoryMode: "manual",+ gpuLayers: 0,+ }),+ null,+ );+});++test("the advice does not promise that a smaller quantization fits", () => {+ for (const offloaded of [0, 38]) {+ const text = offloadWarning({ offloaded, total: 60 })?.description ?? "";+ assert.match(text, /smaller quantization or a shorter context may/);+ assert.doesNotMatch(text, /would fit entirely|would leave room/);+ }+});diff --git a/tests/studio/test_chat_autoload_failure_gate.py b/tests/studio/test_chat_autoload_failure_gate.pyindex 8c7ed9c3c8a..4c380b24f80 100644--- a/tests/studio/test_chat_autoload_failure_gate.py+++ b/tests/studio/test_chat_autoload_failure_gate.py@@ -387,9 +387,11 @@ function loadFallbackNotice(baseTitle: string,cpuFallbackReason: any,mmprojFallbackReason: any,+ offloadNotice?: any,) {const parts: string[] = [];if (cpuFallbackReason) parts.push(`cpu:${cpuFallbackReason}`);+ else if (offloadNotice) parts.push(`offload:${offloadNotice.titleSuffix}`);if (mmprojFallbackReason) parts.push(`mmproj:${mmprojFallbackReason}`);return {title: parts.length > 0 ? `${baseTitle} (${parts.join(", ")})` : baseTitle,@@ -397,6 +399,17 @@ function loadFallbackNotice(degraded: parts.length > 0,};}+// From ../lib/partial-offload: the auto-load success toast reports a layer split through them.+// Real semantics are tested in studio/frontend/tests/partial-offload.test.ts.+function offloadCountsFrom(response: any) {+ return { offloaded: response?.offloaded_layers, total: response?.offload_total_layers };+}+function offloadWarning(counts: any) {+ const { offloaded, total } = counts ?? {};+ if (typeof offloaded !== "number" || typeof total !== "number") return null;+ if (total <= 0 || offloaded >= total) return null;+ return { titleSuffix: ", partly on CPU", description: `${offloaded} of ${total}` };+}async function prepareHfTokenForUse(token: any) {EVENTS.push({ kind: "prepareHfToken" });if (SCENARIO.tokenDecision === "decline") return { proceed: false, token };diff --git a/tests/studio/test_model_picker_contracts.py b/tests/studio/test_model_picker_contracts.pyindex 73bf24437b5..762b9ddc080 100644--- a/tests/studio/test_model_picker_contracts.py+++ b/tests/studio/test_model_picker_contracts.py@@ -425,9 +425,10 @@ def test_recipe_model_load_toast_is_persistent_and_dismissible():assert "closeButton: true" in model_loadassert "icon: createLoadingToastIcon()" in model_loadassert "onDismiss:" in model_load- assert "description: undefined" in model_load+ # A plain success clears the loading description and lasts 2 s; only a layer split says more.+ assert "description: offloadNotice?.description" in model_loadassert "icon: undefined" in model_load- assert "duration: 2000" in model_load+ assert "duration: offloadNotice ? 8000 : 2000" in model_loadtoast_lib = _read("lib/toast.ts")assert "createElement(Spinner" in toast_lib