Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
Studio: retry as one sequence when llama.cpp refuses a unified KV cache (#10371)
Daniel Hane37730dcd772
diff --git a/studio/backend/core/inference/llama_cpp.py b/studio/backend/core/inference/llama_cpp.pyindex b1027ec684d..6be0583401d 100644--- a/studio/backend/core/inference/llama_cpp.py+++ b/studio/backend/core/inference/llama_cpp.py@@ -77,6 +77,7 @@ from core.inference.llama_server_args import (_GPU_LAYER_FLAGS,_LAYER_OFFLOAD_FLAGS,_MOE_OFFLOAD_FLAGS,+ _PARALLEL_FLAGS,_SPLIT_MODE_FLAGS,_TENSOR_SPLIT_FLAGS,_effective_tensor_parallel,@@ -17693,6 +17694,79 @@ class LlamaCppBackend:"""return env.pop("LLAMA_ARG_FLASH_ATTN", None) is not None+ # On the wording, not an architecture name, so the next model needs no list.+ # glm5next spells it as the GGML_ASSERT of ggml-org/llama.cpp#27754.+ _KV_UNIFIED_REFUSED_MARKERS = (+ "a unified kv cache is only supported with a single sequence",+ "needs one sequence per stream",+ )++ @staticmethod+ def _is_kv_unified_refused(output: str) -> bool:+ low = (output or "").lower()+ return any(marker in low for marker in LlamaCppBackend._KV_UNIFIED_REFUSED_MARKERS)++ @staticmethod+ def _with_single_sequence(cmd: list[str]) -> Optional[list[str]]:+ """Return cmd re-run as one sequence, or None when it already is one.++ Studio adds --kv-unified itself above one slot, so this reverses Studio's+ choice, not the user's. Every alias goes, from _PARALLEL_FLAGS so it cannot+ drift from the denylist; llama.cpp is last-wins.+ """+ out: list[str] = []+ saw_kv_unified = False+ multi_slot = False+ skip_value = False+ for i, tok in enumerate(cmd):+ if skip_value:+ skip_value = False+ continue+ name = _flag_name(tok)+ if name in ("--kv-unified", "-kvu"):+ saw_kv_unified = True+ # The flag is bare, but an inline "=1" or a separate "1" reach us.+ if (+ "=" not in tok+ and cmd[i + 1 : i + 2]+ # Case-sensitive, like llama.cpp's own bool parse.+ and cmd[i + 1] in _LLAMA_ARG_TRUE_FALSE_AUTO_VALUES+ ):+ skip_value = True+ continue+ if name in _PARALLEL_FLAGS:+ if "=" in tok:+ value = tok.partition("=")[2]+ elif tok != name:+ # The attached short, -np8, which _flag_name peels to -np.+ value = tok[len(name) :]+ else:+ # Valueless --parallel: eating the next token would delete a flag.+ value = cmd[i + 1] if i + 1 < len(cmd) else ""+ skip_value = _flag_name(value) is None+ try:+ multi_slot = multi_slot or int(value.strip()) > 1+ except (AttributeError, TypeError, ValueError):+ pass+ out.extend([name, "1"])+ continue+ out.append(tok)+ if not saw_kv_unified and not multi_slot:+ return None+ if not any(_flag_name(tok) in _PARALLEL_FLAGS for tok in out):+ out.extend(["--parallel", "1"])+ return out++ @staticmethod+ def _drop_env_single_sequence(env: MutableMapping[str, str]) -> bool:+ """Drop inherited unified-cache and slot-count env before that retry.++ llama.cpp reads its environment before argv. Dropped, not negated with+ --no-kv-unified, which needs every build reaching here to know it.+ """+ dropped = env.pop("LLAMA_ARG_KV_UNIFIED", None) is not None+ return env.pop("LLAMA_ARG_N_PARALLEL", None) is not None or dropped+@staticmethoddef _strip_mmproj_args(cmd: list[str]) -> list[str]:"""Return cmd without the '--mmproj <path>' pair (text-only retry).@@ -23692,6 +23766,11 @@ class LlamaCppBackend:_startup_output) or self._is_tensor_quant_kv_unsupported(_startup_output)_hip_rocr_mismatch = self._is_bundled_hip_rocr_mismatch(_startup_output)+ # No fit retry reaches it, and the rung below needs this+ # launch's argv. Whole buffer: it arrives with a backtrace.+ _capability_crash = _tensor_capability_crash or self._is_kv_unified_refused(+ "\n".join(self._stdout_lines)+ )if (not _did_rocm_retryand _startup_crashed@@ -23722,7 +23801,7 @@ class LlamaCppBackend:if (not _did_fit_retryand _startup_crashed- and not _tensor_capability_crash+ and not _capability_crashand not _hip_rocr_mismatch):# A spill-planned launch that crashed on startup. The@@ -23754,7 +23833,7 @@ class LlamaCppBackend:not _did_fit_retryand fully_gpu_offloadedand _startup_crashed- and not _tensor_capability_crash+ and not _capability_crashand not _hip_rocr_mismatch):# We forced --fit off because Unsloth's (conservative) VRAM@@ -23822,7 +23901,7 @@ class LlamaCppBackend:not _did_fit_retryand _fit_retry_allowedand _startup_crashed- and not _tensor_capability_crash+ and not _capability_crashand not _hip_rocr_mismatch):logger.warning(@@ -24517,6 +24596,34 @@ class LlamaCppBackend:self._memory_state = resolve_effective_memory_state(cmd, env)healthy = _spawn_and_wait(cmd, label = "-archfallback")+ # Studio adds --kv-unified itself above one slot, so nothing the user+ # changes reaches it: retry at one slot, context intact. It aborts, so+ # this MUST stay ahead of the flash-attn rung, which takes any signal+ # crash; on the message, not the exit, so Windows lands here too.+ if not healthy and not _load_cancelled():+ _kvu_cmd = (+ self._with_single_sequence(_last_spawn_cmd)+ if self._is_kv_unified_refused("\n".join(self._stdout_lines))+ else None+ )+ if _kvu_cmd is not None:+ logger.warning(+ "llama-server refused a unified KV cache with more than "+ "one sequence; retrying with one slot and --kv-unified "+ "dropped. Concurrent requests will queue."+ )+ self._kill_process()+ if self._drop_env_single_sequence(env):+ logger.info(+ "Dropped inherited LLAMA_ARG_KV_UNIFIED / "+ "LLAMA_ARG_N_PARALLEL for the single-sequence retry."+ )+ cmd = _kvu_cmd+ # Read by admission control; left as-is Studio over-admits.+ n_parallel = 1 # allow-slot-clamp: llama-server refused more+ kv_cache_unified = False+ healthy = _spawn_and_wait(_kvu_cmd, label = "-single-seq")+# Flash-attention kernels hard-crash at startup on some ROCm/GPU# builds (frequently inside the vision tower). Disabling FA keeps# both vision and MTP, so retry that way before dropping either.diff --git a/studio/backend/core/inference/llama_server_args.py b/studio/backend/core/inference/llama_server_args.pyindex f15855abea1..7a9205b31fd 100644--- a/studio/backend/core/inference/llama_server_args.py+++ b/studio/backend/core/inference/llama_server_args.py@@ -35,12 +35,16 @@ BATCH_MAX = 65536CTX_CHECKPOINTS_MAX = 256CACHE_RAM_MAX_MIB = 1024 * 1024+# Slot-count aliases in one place: the denial below, its #9510 hint and the single-sequence retry must cover the same+# set, or a spelling one of them misses reaches llama-server unnoticed.+_PARALLEL_FLAGS: frozenset[str] = frozenset({"-np", "--parallel", "--n-parallel"})+# Each group = every alias (short + long) of one hard-denied flag. Extend the matching group when llama.cpp adds a new# alias._DENYLIST_GROUPS: tuple[frozenset[str], ...] = (# Parallel slots: owned by typer --parallel and LoadRequest.n_parallel; a pass-through would desync the slot# bookkeeping from llama-server.- frozenset({"-np", "--parallel", "--n-parallel"}),+ _PARALLEL_FLAGS,# Model identity: a second -m would load a different model than Unsloth thinks it loaded# Model identity: Unsloth resolves it from LoadRequest; a second -m would load a different model than Unsloth thinks# it loaded.@@ -275,7 +279,7 @@ def validate_extra_args(args: Optional[Iterable[str]]) -> list[str]:# #9510: users reaching for `--parallel 1` hit this refusal with no pointer to the supported knob# Why (#9510): users reaching for `--parallel 1` to cap concurrent predictions on a local model hit this# refusal with no pointer to the supported knob; name it.- if flag in {"-np", "--parallel", "--n-parallel"}:+ if flag in _PARALLEL_FLAGS:message += "; set n_parallel on the load request (parallel decode slots) instead"raise ValueError(message)if flag is None:diff --git a/studio/backend/tests/test_llama_cpp_single_sequence_retry.py b/studio/backend/tests/test_llama_cpp_single_sequence_retry.pynew file mode 100644index 00000000000..e0ec554d1df--- /dev/null+++ b/studio/backend/tests/test_llama_cpp_single_sequence_retry.py@@ -0,0 +1,171 @@+# SPDX-License-Identifier: AGPL-3.0-only+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0++"""The single-sequence retry for architectures that refuse a unified KV cache.++Studio appends ``--kv-unified`` itself above one slot, so nothing the user changes+in the UI reaches the flag that stops the model loading.+"""++from __future__ import annotations++import os+import sys++sys.path.insert(0, os.path.join(os.path.dirname(__file__), ".."))++from core.inference.llama_cpp import LlamaCppBackend++# Verbatim from a Strix Halo report: llama-server refusing GLM-5.3-Flash.+_REFUSAL = (+ "0.56.012.623 E llama_init_from_model: failed to initialize the context: "+ "glm5next: the pooled indexer needs one sequence per stream, so a unified "+ "KV cache is only supported with a single sequence"+)+++# What the binary prints: the full buffer is scanned, so the backtrace cannot bury it.+_ASSERT = (+ "/build/llama.cpp/src/models/glm5next.cpp:1018: GGML_ASSERT(n_ps == 1 && "+ '"the per-cell pool view needs one sequence per stream") failed\n'+ "[New LWP 4242]\n#0 0x00007f0000000000 in abort ()"+)+++def test_the_reported_refusal_is_recognised():+ assert LlamaCppBackend._is_kv_unified_refused(_REFUSAL)+++def test_the_assert_the_binary_actually_prints_is_recognised():+ assert LlamaCppBackend._is_kv_unified_refused(_ASSERT)+++def test_an_unrelated_failure_is_not():+ assert not LlamaCppBackend._is_kv_unified_refused(+ "error loading model: unknown model architecture: 'qwen4exp'"+ )+ assert not LlamaCppBackend._is_kv_unified_refused("")+ # minimax-m3 still loads: retrying it would cost three slots for nothing.+ assert not LlamaCppBackend._is_kv_unified_refused(+ "minimax_m3: unified KV cache with n_seq_max > 1; MSA needs per-sequence "+ "streams -> running DENSE attention. Drop --kv-unified to enable MSA."+ )+++def test_the_reported_launch_becomes_one_slot_with_no_unified_cache():+ cmd = [+ "llama-server",+ "-m",+ "GLM-5.3-Flash-UD-IQ4_XS-00001-of-00004.gguf",+ "--parallel",+ "4",+ "--flash-attn",+ "on",+ "--no-context-shift",+ "-c",+ "128000",+ "--gpu-layers",+ "47",+ "--fit",+ "off",+ "--kv-unified",+ "--jinja",+ ]++ out = LlamaCppBackend._with_single_sequence(cmd)++ assert out is not None+ assert "--kv-unified" not in out+ assert out[out.index("--parallel") + 1] == "1"+ assert out[out.index("-c") + 1] == "128000"+ assert out[out.index("--gpu-layers") + 1] == "47"+ assert "--jinja" in out and "--no-context-shift" in out+++def test_a_parallel_surviving_in_the_extras_tail_is_rewritten_too():+ """llama.cpp is last-wins and extras are appended after Unsloth's flags."""+ cmd = ["llama-server", "--parallel", "4", "--kv-unified", "-np", "8"]++ out = LlamaCppBackend._with_single_sequence(cmd)++ assert out == ["llama-server", "--parallel", "1", "-np", "1"]+++def test_every_spelling_of_the_two_flags_is_handled():+ cmd = ["llama-server", "--parallel=4", "-kvu", "--alias", "m"]+ assert LlamaCppBackend._with_single_sequence(cmd) == [+ "llama-server",+ "--parallel",+ "1",+ "--alias",+ "m",+ ]++ cmd = ["llama-server", "-np8", "--kv-unified", "1"]+ assert LlamaCppBackend._with_single_sequence(cmd) == ["llama-server", "-np", "1"]+++def test_every_alias_of_the_slot_count_is_rewritten():+ """--n-parallel is in the denylist group too."""+ out = LlamaCppBackend._with_single_sequence(+ ["llama-server", "--parallel", "4", "--kv-unified", "--n-parallel", "8"]+ )++ assert out == ["llama-server", "--parallel", "1", "--n-parallel", "1"]+++def test_a_valueless_slot_flag_does_not_swallow_the_next_flag():+ """Malformed, but eating the token behind it would delete --kv-unified."""+ out = LlamaCppBackend._with_single_sequence(["llama-server", "--parallel", "--kv-unified"])++ assert out == ["llama-server", "--parallel", "1"]+++def test_a_command_already_running_one_sequence_has_nothing_to_retry():+ assert LlamaCppBackend._with_single_sequence(["llama-server", "-c", "8192"]) is None+ assert LlamaCppBackend._with_single_sequence(["llama-server", "--parallel", "1"]) is None+++def test_a_slot_count_is_added_when_the_command_carried_none():+ out = LlamaCppBackend._with_single_sequence(["llama-server", "--kv-unified"])++ assert out == ["llama-server", "--parallel", "1"]+++def test_the_inherited_environment_is_dropped_so_it_cannot_undo_the_retry():+ """llama.cpp applies its environment before parsing argv."""+ env = {"LLAMA_ARG_KV_UNIFIED": "1", "LLAMA_ARG_N_PARALLEL": "4", "PATH": "/usr/bin"}++ assert LlamaCppBackend._drop_env_single_sequence(env) is True+ assert "LLAMA_ARG_KV_UNIFIED" not in env+ assert "LLAMA_ARG_N_PARALLEL" not in env+ assert env["PATH"] == "/usr/bin"+++def test_a_clean_environment_reports_nothing_dropped():+ env = {"PATH": "/usr/bin"}++ assert LlamaCppBackend._drop_env_single_sequence(env) is False+ assert env == {"PATH": "/usr/bin"}+++def test_the_fit_recovery_rungs_stand_down_for_this_refusal():+ """A startup crash, so the --fit rungs would take it first and waste a load."""+ import inspect++ src = inspect.getsource(LlamaCppBackend.load_model)+ assert "_capability_crash = _tensor_capability_crash or self._is_kv_unified_refused" in src+ # Every --fit rung, and only those: the HIP rung keeps its narrower gate.+ assert src.count("and not _capability_crash") == 3+ assert src.count("and not _tensor_capability_crash") == 1+++def test_the_retry_commits_the_one_slot_geometry_it_launched():+ # Committed after the ladder, so the retry must overwrite what it reverses.+ import inspect++ src = inspect.getsource(LlamaCppBackend.load_model)+ start = src.index('label = "-single-seq"')+ block = src[src.rindex("cmd = _kvu_cmd", 0, start) : start]+ assert "n_parallel = 1" in block+ assert "kv_cache_unified = False" in blockdiff --git a/studio/backend/tests/test_tp_vision_regression.py b/studio/backend/tests/test_tp_vision_regression.pyindex 72bbad179e3..3ff2fd9ecac 100644--- a/studio/backend/tests/test_tp_vision_regression.py+++ b/studio/backend/tests/test_tp_vision_regression.py@@ -617,11 +617,10 @@ def test_fit_off_retry_skipped_on_a_tensor_capability_crash():"""The fit-independent --fit off retry is skipped on the split-axis marker, elsethe model crashes a second time before the latch records it (reviewer.py, #6659).- The same skip now also covers the pre-b9455 refusal of a quantized KV cache in- tensor mode (ggml-org/llama.cpp#23792): both are capabilities the binary lacks,- so a second spawn to let it offload cannot help and costs a full model load.- Hence the guard's name is _tensor_capability_crash rather than the split-axis- one it started as.+ It now also covers the pre-b9455 quantized-KV refusal in tensor mode+ (ggml-org/llama.cpp#23792) and the unified-cache refusal: no second spawn helps+ any of the three, and each costs a full model load. Hence _capability_crash,+ with _tensor_capability_crash left as the half the ROCm rung gates on."""src = inspect.getsource(LlamaCppBackend.load_model)retry = src.find('run_cmd = [*run_cmd, "--fit", "off"]')@@ -629,11 +628,12 @@ def test_fit_off_retry_skipped_on_a_tensor_capability_crash():guard = src[max(0, retry - 1000) : retry]assert "_fit_retry_allowed" in guard and "_startup_crashed" in guardassert (- "not _tensor_capability_crash" in guard- ), "the fit-off retry must be skipped when the crash is a tensor capability limit"- # Both markers feed it, so neither can be dropped without this failing.+ "not _capability_crash" in guard+ ), "the fit-off retry must be skipped when the crash is a capability limit"+ # All three markers feed it, so none can be dropped without this failing.assert "_is_tensor_split_assert" in srcassert "_is_tensor_quant_kv_unsupported" in src+ assert "_is_kv_unified_refused" in srcdef test_is_abort_exit_recognizes_windows_crt_abort():