Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
Studio: keep the llama-server prompt cache on a shared-memory GPU (#10382)
Daniel Hanc2bb6e340c94
diff --git a/studio/backend/core/inference/llama_cpp.py b/studio/backend/core/inference/llama_cpp.pyindex f65b3c5cbc6..b1027ec684d 100644--- a/studio/backend/core/inference/llama_cpp.py+++ b/studio/backend/core/inference/llama_cpp.py@@ -71,6 +71,8 @@ from core.inference.context_window import ()from core.inference.stream_errors import stream_error_from_chunkfrom core.inference.llama_server_args import (+ _CACHE_RAM_FLAGS,+ _CTX_CHECKPOINTS_FLAGS,_DEVICE_FLAGS,_GPU_LAYER_FLAGS,_LAYER_OFFLOAD_FLAGS,@@ -8806,6 +8808,42 @@ class LlamaCppBackend:return arch_by_idreturn arch_by_id+ @staticmethod+ def _offload_target_shares_system_memory(+ *, is_vulkan_backend: bool, shared_gpu_ids, detected_gpus, gpu_indices+ ) -> bool:+ """True when EVERY device this launch offloads to reports host RAM as its VRAM.++ An iGPU or a unified-memory APU has no separate pool and no bus between the+ two: a buffer llama.cpp calls "host RAM" and one it calls "VRAM" are the same+ physical memory. Tuning that exists to avoid moving bytes across PCI-E+ therefore buys nothing there, and any recovery path it disables is pure loss.++ Fails closed. A mixed selection, an unreadable inventory or a launch that+ pins nothing all answer False, so the caller keeps the behaviour it has.+ The two index spaces are kept apart the way the rest of this file keeps+ them: Vulkan ordinals against shared_gpu_ids, physical HIP/CUDA ids+ against the unified-memory sets.+ """+ devices = (+ list(gpu_indices) if gpu_indices else [idx for idx, _free in (detected_gpus or ())]+ )+ if not devices:+ return False+ if is_vulkan_backend:+ shared = set(shared_gpu_ids or ())+ return bool(shared) and all(idx in shared for idx in devices)+ # ROCm only, deliberately. The one caller is the Windows full-offload branch and+ # every integrated CUDA part is Linux-only, so _integrated_cuda_gpu_ids() could+ # only answer False here -- at the price of a get_device_properties() call, which+ # leaks a ~700 MiB primary context (see _get_gpu_memory). The ROCm helper answers+ # empty on a CUDA torch without touching the device.+ try:+ unified = LlamaCppBackend._rocm_unified_memory_gpu_ids()+ except Exception:+ return False+ return bool(unified) and all(idx in unified for idx in devices)+@staticmethoddef _amd_apu_wants_unified_memory(gpu_indices = None) -> bool:"""True only for AMD unified-memory APUs, where GGML_CUDA_ENABLE_UNIFIED_MEMORY@@ -17248,6 +17286,82 @@ class LlamaCppBackend:return []return discrete if 0 < len(discrete) < len(_selected) else []+ @staticmethod+ def _cache_tuning_target_unknown(+ extra_args: Optional[Iterable[str]],+ gpu_ids: Optional[Iterable[int]],+ env: Mapping[str, str],+ ) -> bool:+ """Whether the device this cache tuning would be chosen against is the one+ the child actually gets.++ A user device selection is not stripped on an automatic load and llama.cpp+ reads it last (argv) or first (env, before argv either way), so it, not the+ automatic placement, names the target. Both spellings count: only an explicit+ ``gpu_ids`` clears LLAMA_ARG_DEVICE, so on an automatic load the env twin+ survives into the child verbatim. Fail-closed by design -- the failure that+ matters is emitting --cache-ram 0 against a shared pool, while keeping the+ prompt cache on a discrete card only forgoes a tuning.+ """+ if gpu_ids is not None:+ return False+ return bool(+ _extra_args_set_any_flag(extra_args, _DEVICE_FLAGS)+ or str(env.get("LLAMA_ARG_DEVICE", "")).strip()+ )++ @staticmethod+ def _retry_cache_tuning_flags(+ cmd: list[str],+ *,+ cache_ram: Optional[int],+ ctx_checkpoints: Optional[int],+ server_caps: Mapping[str, Any],+ ) -> list[str]:+ """The cache tuning to append when an arch-crash retry lands on a discrete GPU.++ The extras were appended to ``cmd`` long before the retry, so anything added+ here wins the last-wins parse -- the reverse of the initial launch, where a+ typed --cache-ram overrides the tuning. So a setting the command already+ states is skipped, keeping that precedence rather than zeroing a value the+ panel still shows. An explicit field is handled by the caller's own+ None checks, exactly as at launch.+ """+ # Every spelling llama.cpp accepts, from the sets the arg layer already keeps: a+ # short -cram or -ctxcp states the setting exactly as the long form does, so+ # appending after one would zero a value the user typed and the panel shows.+ stated = {_flag_name(str(token)) for token in cmd}+ flags: list[str] = []+ if (+ cache_ram is None+ and server_caps.get("supports_cache_ram")+ and not (_CACHE_RAM_FLAGS & stated)+ ):+ flags.extend(["--cache-ram", "0"])+ checkpoints_flag = server_caps.get("ctx_checkpoints_flag")+ if ctx_checkpoints is None and checkpoints_flag and not (_CTX_CHECKPOINTS_FLAGS & stated):+ flags.extend([str(checkpoints_flag), "0"])+ return flags++ @staticmethod+ def _without_flag_pairs(cmd: list[str], pairs: list[str]) -> list[str]:+ """Remove flag/value pairs THIS process appended, by exact token match.++ Only ever called with tokens the cache tuning emitted itself, every one of them+ a flag followed by its value, so the two-token step cannot swallow a following+ user extra the way it would for a valueless flag (the reason the mlock path+ refuses to strip at all). A pair that is no longer present is skipped rather+ than searched for again, so a repeated call is a no-op.+ """+ out = list(cmd)+ for index in range(0, len(pairs) - 1, 2):+ flag, value = pairs[index], pairs[index + 1]+ for at in range(len(out) - 1):+ if out[at] == flag and out[at + 1] == value:+ del out[at : at + 2]+ break+ return out+@staticmethoddef _without_tensor_split(cmd: list[str]) -> Optional[list[str]]:"""Return cmd with ``--tensor-split``/``-ts`` removed (both spellings and the@@ -22474,7 +22588,58 @@ class LlamaCppBackend:# Windows + full offload: drop the host-RAM KV checkpoints that cause# WDDM/PCI-E overhead, but keep prompt caching (in-VRAM prefix reuse) so# a repeated prompt is not re-prefilled on every request. #5692.- if sys.platform == "win32" and full_offload_tuning_active:+ # ... unless the offload target IS system RAM. #5692 is a discrete card,+ # whose host-RAM checkpoints cross PCI-E under WDDM, so dropping them is+ # a saving. An iGPU has one pool and no bus, so the flags only remove the+ # prompt cache and every chance of reusing a prefix; --cache-ram 0 takes+ # --cache-idle-slots down with it. Measured on a Strix Halo: a 48.8 h+ # session spent 44.3 h re-ingesting, 3.38 M prompt against 72 k generated.+ # Behind the platform gate on purpose: the helper reads torch, and a+ # launch that never reaches this block would spend a device probe on a+ # question with no consumer.+ #+ # gpu_indices is the picker's answer, not always the child's: with no+ # gpu_ids a user --device survives in the extras and wins last-wins over+ # the generated pin. Decline rather than re-derive the target from argv --+ # the failure that matters is --cache-ram 0 against a shared pool, and+ # keeping the prompt cache on a discrete card only forgoes a tuning.+ # The exact tokens the tuning appended, so the arch-crash respawn can+ # take them back off when it lands on a different device class.+ _cache_flags_emitted: list[str] = []+ # extra_args, not the memory policy's list: that one is built later, and+ # a gpu_ids pin is the case _strip_device_extra_args removes the flag in.+ # The env twin counts too: only an explicit gpu_ids clears+ # LLAMA_ARG_DEVICE, and llama.cpp reads it before argv, so an automatic+ # load can place against a target the generated pin never names.+ _cache_target_unknown = self._cache_tuning_target_unknown(+ extra_args, gpu_ids, os.environ+ )+ _shared_memory_offload = (+ sys.platform == "win32"+ and full_offload_tuning_active+ and (+ _cache_target_unknown+ or self._offload_target_shares_system_memory(+ is_vulkan_backend = is_vulkan_backend,+ shared_gpu_ids = _shared_gpu_ids,+ detected_gpus = _detected_gpus,+ gpu_indices = gpu_indices,+ )+ )+ )+ if _cache_target_unknown and sys.platform == "win32" and full_offload_tuning_active:+ logger.info(+ "Keeping the llama-server prompt cache: a user --device in the extra "+ "arguments overrides the automatic placement, so the offload target "+ "this tuning would be chosen against is not the one the child gets."+ )+ elif _shared_memory_offload:+ logger.info(+ "Keeping the llama-server prompt cache: this load offloads to a GPU "+ "that shares system memory, so --cache-ram 0 and --ctx-checkpoints 0 "+ "would remove prefix reuse without saving a transfer."+ )+ elif sys.platform == "win32" and full_offload_tuning_active:unsupported_cache_flags: list[str] = []# An explicit control wins over the platform tuning: llama.cpp# is last-wins, so a 0 emitted after it would overrule the panel@@ -22482,15 +22647,16 @@ class LlamaCppBackend:if cache_ram is not None:passelif server_caps.get("supports_cache_ram"):- cmd.extend(["--cache-ram", "0"])+ _cache_flags_emitted.extend(["--cache-ram", "0"])else:unsupported_cache_flags.append("--cache-ram")if ctx_checkpoints is not None:passelif server_caps.get("ctx_checkpoints_flag"):- cmd.extend([str(server_caps["ctx_checkpoints_flag"]), "0"])+ _cache_flags_emitted.extend([str(server_caps["ctx_checkpoints_flag"]), "0"])else:unsupported_cache_flags.append("--ctx-checkpoints")+ cmd.extend(_cache_flags_emitted)if unsupported_cache_flags:logger.info("Skipping unsupported Windows cache flags for llama-server: %s",@@ -24121,6 +24287,43 @@ class LlamaCppBackend:self._kill_process()gpu_indices = _remaining_unified_gpu_indices = _remaining+ # The tuning was decided against the devices that just crashed,+ # and this respawn reuses `cmd`, so a discrete-to-APU retry would+ # carry --cache-ram 0 onto a shared pool and the reverse would+ # reach a discrete card without it. Re-decide against the retry.+ if sys.platform == "win32" and full_offload_tuning_active:+ _retry_shared = _cache_target_unknown or (+ self._offload_target_shares_system_memory(+ is_vulkan_backend = is_vulkan_backend,+ shared_gpu_ids = _shared_gpu_ids,+ detected_gpus = _retry_rows or _detected_gpus,+ gpu_indices = _remaining,+ )+ )+ if _retry_shared and _cache_flags_emitted:+ # Exactly the tokens this policy appended, each a flag+ # with its value, so the strip cannot eat a user extra.+ cmd = self._without_flag_pairs(cmd, _cache_flags_emitted)+ _cache_flags_emitted = []+ logger.info(+ "Retry lands on a GPU that shares system memory: dropped "+ "--cache-ram 0 and --ctx-checkpoints 0 so the prompt cache "+ "survives."+ )+ elif not _retry_shared and not _cache_flags_emitted:+ _retry_flags = self._retry_cache_tuning_flags(+ cmd,+ cache_ram = cache_ram,+ ctx_checkpoints = ctx_checkpoints,+ server_caps = server_caps,+ )+ if _retry_flags:+ cmd.extend(_retry_flags)+ _cache_flags_emitted = list(_retry_flags)+ logger.info(+ "Retry lands on a GPU with its own memory: applied the "+ "Windows full-offload cache tuning."+ )# GGML_CUDA_ENABLE_UNIFIED_MEMORY was decided for the CRASHED# set. The canonical #7624 shape crashes on the APU and retries# on the dGPU, where it is harmful, so withdraw it, but onlydiff --git a/studio/backend/tests/test_server_tuning_flags.py b/studio/backend/tests/test_server_tuning_flags.pyindex 3c29eb38ec7..dab11687c70 100644--- a/studio/backend/tests/test_server_tuning_flags.py+++ b/studio/backend/tests/test_server_tuning_flags.py@@ -300,7 +300,10 @@ def test_the_checkpoint_flag_falls_back_to_the_legacy_spelling():# and the emission uses the recorded name, not a hard-coded oneload = inspect.getsource(llama_cpp.LlamaCppBackend.load_model)assert "cmd.extend([str(_ctxcp_flag), str(int(ctx_checkpoints))])" in load- assert 'cmd.extend([str(server_caps["ctx_checkpoints_flag"]), "0"])' in load+ # The Windows tuning's 0 goes through the list the arch-crash respawn takes its own+ # flags back off, so the sink is that list rather than cmd. What it pins is unchanged.+ assert '_cache_flags_emitted.extend([str(server_caps["ctx_checkpoints_flag"]), "0"])' in load+ assert "cmd.extend(_cache_flags_emitted)" in loaddef test_an_unsupported_draft_cache_dtype_is_dropped_not_launched():diff --git a/studio/backend/tests/test_shared_memory_offload_keeps_the_prompt_cache.py b/studio/backend/tests/test_shared_memory_offload_keeps_the_prompt_cache.pynew file mode 100644index 00000000000..6a65f54f2f0--- /dev/null+++ b/studio/backend/tests/test_shared_memory_offload_keeps_the_prompt_cache.py@@ -0,0 +1,351 @@+# SPDX-License-Identifier: AGPL-3.0-only+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0++"""Windows full-offload tuning (#5692) drops the llama-server prompt cache to avoid+WDDM/PCI-E traffic. An iGPU or APU has one memory pool and no bus, so the same flags+save nothing and remove prefix reuse instead.++Measured on a Strix Halo running the Vulkan build with -ngl -1: one 48.8 h session+spent 44.3 h re-ingesting prompts, 3.38 M prompt tokens against 72 k generated, with+`--cache-ram 0 --ctx-checkpoints 0` in its own argv and+`--cache-idle-slots requires --cache-ram, disabling` in llama-server's first lines.++The predicate fails closed everywhere it cannot prove the whole target is shared, so+a discrete card keeps the tuning it was written for.+"""++from __future__ import annotations++import types++import pytest++from core.inference.llama_cpp import LlamaCppBackend++_VISIBLE_DEVICE_MASKS = ("HIP_VISIBLE_DEVICES", "ROCR_VISIBLE_DEVICES", "CUDA_VISIBLE_DEVICES")++[email protected](autouse = True)+def _no_inherited_gpu_mask(monkeypatch):+ """The ROCm arm asks about physical ids on a faked torch host. A mask inherited+ from the shell remaps them onto ids the fake host does not have."""+ for _mask in _VISIBLE_DEVICE_MASKS:+ monkeypatch.delenv(_mask, raising = False)+++def _shares(**kwargs) -> bool:+ return LlamaCppBackend._offload_target_shares_system_memory(**kwargs)+++def _vulkan(+ shared_gpu_ids,+ detected,+ gpu_indices = None,+) -> bool:+ return _shares(+ is_vulkan_backend = True,+ shared_gpu_ids = shared_gpu_ids,+ detected_gpus = detected,+ gpu_indices = gpu_indices,+ )+++def _fake_torch(archs, *, hip = "6.4.0"):+ torch = types.ModuleType("torch")+ torch.version = types.SimpleNamespace(hip = hip)+ torch.cuda = types.SimpleNamespace(+ is_available = lambda: True,+ device_count = lambda: len(archs),+ get_device_properties = lambda i: types.SimpleNamespace(gcnArchName = archs[i]),+ )+ return torch+++# ── Vulkan, where the shared set is ggml's compact ordinals ──+++def test_an_igpu_only_target_shares_system_memory():+ assert _vulkan({0}, [(0, 170079)], gpu_indices = [0]) is True+++def test_an_unpinned_load_reads_the_detected_list():+ """The auto-Vulkan arm pins nothing at this point, which is the shape the+ Strix Halo session ran in."""+ assert _vulkan({0}, [(0, 170079)]) is True+++def test_a_discrete_card_beside_the_igpu_keeps_the_tuning():+ assert _vulkan({0}, [(0, 170079), (1, 24000)], gpu_indices = [0, 1]) is False+ assert _vulkan({0}, [(0, 170079), (1, 24000)]) is False+++def test_pinning_only_the_igpu_of_a_mixed_host_still_shares():+ assert _vulkan({0}, [(0, 170079), (1, 24000)], gpu_indices = [0]) is True+++def test_a_discrete_only_host_keeps_the_tuning():+ assert _vulkan(set(), [(0, 24000)], gpu_indices = [0]) is False+++def test_a_load_with_no_device_at_all_fails_closed():+ assert _vulkan({0}, []) is False+ assert _vulkan(set(), []) is False+++# ── ROCm and CUDA, where the ids are physical ──+++def test_an_amd_apu_shares_system_memory(monkeypatch):+ monkeypatch.setitem(__import__("sys").modules, "torch", _fake_torch(["gfx1151"]))+ assert (+ _shares(+ is_vulkan_backend = False,+ shared_gpu_ids = set(),+ detected_gpus = [(0, 60000)],+ gpu_indices = [0],+ )+ is True+ )+++def test_a_discrete_amd_card_keeps_the_tuning(monkeypatch):+ monkeypatch.setitem(__import__("sys").modules, "torch", _fake_torch(["gfx1100"]))+ assert (+ _shares(+ is_vulkan_backend = False,+ shared_gpu_ids = set(),+ detected_gpus = [(0, 24000)],+ gpu_indices = [0],+ )+ is False+ )+++def test_an_apu_next_to_a_discrete_card_keeps_the_tuning(monkeypatch):+ monkeypatch.setitem(__import__("sys").modules, "torch", _fake_torch(["gfx1151", "gfx1100"]))+ assert (+ _shares(+ is_vulkan_backend = False,+ shared_gpu_ids = set(),+ detected_gpus = [(0, 60000), (1, 24000)],+ gpu_indices = [0, 1],+ )+ is False+ )+++def test_an_unreadable_inventory_fails_closed(monkeypatch):+ """No torch, so neither unified-memory set can be built. The tuning stays as it+ is rather than being dropped on a guess."""+ monkeypatch.setitem(__import__("sys").modules, "torch", None)+ assert (+ _shares(+ is_vulkan_backend = False,+ shared_gpu_ids = set(),+ detected_gpus = [(0, 60000)],+ gpu_indices = [0],+ )+ is False+ )+++def test_the_vulkan_shared_set_is_not_read_as_physical_ids(monkeypatch):+ """Vulkan ordinals and HIP ids are different spaces. A ROCm launch must not+ inherit a shared set that was built from Vulkan's numbering."""+ monkeypatch.setitem(__import__("sys").modules, "torch", _fake_torch(["gfx1100"]))+ assert (+ _shares(+ is_vulkan_backend = False,+ shared_gpu_ids = {0},+ detected_gpus = [(0, 24000)],+ gpu_indices = [0],+ )+ is False+ )+++def test_a_cuda_host_is_answered_without_touching_the_device(monkeypatch):+ """The predicate must not create a CUDA primary context to answer.++ ``torch.cuda.get_device_properties`` initialises CUDA in the backend process+ and never gives the memory back (~700 MiB), which is VRAM the child+ llama-server then cannot use. On a CUDA host the answer is False either way,+ so it has to come from ``torch.version.hip`` alone. The fake raises on any+ device probe, so a reintroduced ``_integrated_cuda_gpu_ids()`` call fails+ this test rather than merely costing memory in production."""++ class _DeviceProbed(BaseException):+ """BaseException on purpose: every helper in this family swallows+ ``Exception`` per device, so an ordinary error would be caught and the+ test would pass against the very code it exists to catch."""++ def _explode(_ordinal):+ raise _DeviceProbed("probed the CUDA device to answer a Windows-only question")++ torch = types.ModuleType("torch")+ torch.version = types.SimpleNamespace(hip = None)+ torch.cuda = types.SimpleNamespace(+ is_available = lambda: True,+ device_count = lambda: 1,+ get_device_properties = _explode,+ )+ monkeypatch.setitem(__import__("sys").modules, "torch", torch)+ assert (+ _shares(+ is_vulkan_backend = False,+ shared_gpu_ids = set(),+ detected_gpus = [(0, 24000)],+ gpu_indices = [0],+ )+ is False+ )+++# ── _without_flag_pairs: the strip the arch-crash respawn uses ──+++def test_only_the_pairs_this_policy_emitted_are_stripped():+ """The respawn reuses the argv it already built, so taking the tuning back off has+ to be an exact-token removal. Every pair is a flag with its value, which is what+ makes the two-token step safe: a valueless flag would swallow the user extra that+ follows it, which is why the mlock path refuses to strip at all."""+ cmd = [+ "llama-server",+ "--cache-ram",+ "0",+ "--ctx-checkpoints",+ "0",+ "--cache-ram",+ "4096",+ "--verbose",+ ]+ out = LlamaCppBackend._without_flag_pairs(cmd, ["--cache-ram", "0", "--ctx-checkpoints", "0"])+ # The user's own --cache-ram 4096 survives; only the emitted zeros go.+ assert out == ["llama-server", "--cache-ram", "4096", "--verbose"]+++def test_stripping_a_pair_that_is_gone_is_a_no_op():+ cmd = ["llama-server", "--verbose"]+ assert LlamaCppBackend._without_flag_pairs(cmd, ["--cache-ram", "0"]) == cmd+++def test_a_trailing_flag_without_its_value_is_left_alone():+ # Defensive: an odd-length pair list would otherwise index past the end.+ cmd = ["llama-server", "--cache-ram"]+ assert LlamaCppBackend._without_flag_pairs(cmd, ["--cache-ram"]) == cmd+++# ── the target the tuning is chosen against ──+++def test_a_user_device_flag_makes_the_target_unknown():+ assert LlamaCppBackend._cache_tuning_target_unknown(["--device", "ROCm1"], None, {})+++def test_an_inherited_device_env_makes_the_target_unknown():+ # Only an explicit gpu_ids clears the env twin, and llama.cpp reads it before argv, so+ # the generated pin is not what the child places against. Reading argv alone emitted+ # --cache-ram 0 at an APU the picker had paired with a discrete card.+ assert LlamaCppBackend._cache_tuning_target_unknown(None, None, {"LLAMA_ARG_DEVICE": "ROCm0"})+ assert not LlamaCppBackend._cache_tuning_target_unknown(None, None, {"LLAMA_ARG_DEVICE": " "})+++def test_an_explicit_pin_owns_the_placement_so_the_target_is_known():+ # The control: gpu_ids clears both spellings, so neither can name another device.+ assert not LlamaCppBackend._cache_tuning_target_unknown(+ ["--device", "ROCm1"], [0], {"LLAMA_ARG_DEVICE": "ROCm0"}+ )+ assert not LlamaCppBackend._cache_tuning_target_unknown(None, None, {})+++# ── the arch-crash retry keeps the launch's precedence ──++_CAPS = {"supports_cache_ram": True, "ctx_checkpoints_flag": "--ctx-checkpoints"}+++def test_the_retry_applies_the_tuning_when_nothing_states_it():+ assert LlamaCppBackend._retry_cache_tuning_flags(+ ["llama-server", "-m", "x.gguf"],+ cache_ram = None,+ ctx_checkpoints = None,+ server_caps = _CAPS,+ ) == ["--cache-ram", "0", "--ctx-checkpoints", "0"]+++def test_the_retry_does_not_overrule_a_cache_flag_the_command_already_states():+ # The extras sit in cmd already, so appending here wins last-wins and zeroes a value+ # the panel still shows -- the reverse of the launch, where the extras win.+ flags = LlamaCppBackend._retry_cache_tuning_flags(+ ["llama-server", "-m", "x.gguf", "--cache-ram", "8192"],+ cache_ram = None,+ ctx_checkpoints = None,+ server_caps = _CAPS,+ )+ assert flags == ["--ctx-checkpoints", "0"]++ assert (+ LlamaCppBackend._retry_cache_tuning_flags(+ ["llama-server", "--cache-ram=8192", "--ctx-checkpoints=4"],+ cache_ram = None,+ ctx_checkpoints = None,+ server_caps = _CAPS,+ )+ == []+ )+++def test_the_retry_reads_the_short_spellings_as_the_same_settings():+ # -cram and -ctxcp are aliases llama.cpp accepts and the panel offers, so appending+ # the long form after one would silently zero the user's value.+ assert (+ LlamaCppBackend._retry_cache_tuning_flags(+ ["llama-server", "-cram", "8192", "-ctxcp", "4"],+ cache_ram = None,+ ctx_checkpoints = None,+ server_caps = _CAPS,+ )+ == []+ )++ # One at a time, so a single alias cannot stand in for the pair.+ assert LlamaCppBackend._retry_cache_tuning_flags(+ ["llama-server", "-cram", "8192"],+ cache_ram = None,+ ctx_checkpoints = None,+ server_caps = _CAPS,+ ) == ["--ctx-checkpoints", "0"]+ assert LlamaCppBackend._retry_cache_tuning_flags(+ ["llama-server", "-ctxcp=4"],+ cache_ram = None,+ ctx_checkpoints = None,+ server_caps = _CAPS,+ ) == ["--cache-ram", "0"]+++def test_the_retry_reads_the_other_checkpoint_alias_as_the_same_setting():+ # A build advertising --ctx-checkpoints can still be handed --swa-checkpoints.+ assert LlamaCppBackend._retry_cache_tuning_flags(+ ["llama-server", "--swa-checkpoints", "4"],+ cache_ram = None,+ ctx_checkpoints = None,+ server_caps = _CAPS,+ ) == ["--cache-ram", "0"]+++def test_the_retry_skips_what_the_build_and_the_fields_already_own():+ # An explicit field, as at launch, and a build without the capability.+ assert (+ LlamaCppBackend._retry_cache_tuning_flags(+ ["llama-server"], cache_ram = 4096, ctx_checkpoints = 8, server_caps = _CAPS+ )+ == []+ )+ assert (+ LlamaCppBackend._retry_cache_tuning_flags(+ ["llama-server"],+ cache_ram = None,+ ctx_checkpoints = None,+ server_caps = {"supports_cache_ram": False, "ctx_checkpoints_flag": None},+ )+ == []+ )