Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

Studio: keep the llama-server prompt cache on a shared-memory GPU (#10382)

Daniel Hanc2bb6e340c94
diff --git a/studio/backend/core/inference/llama_cpp.py b/studio/backend/core/inference/llama_cpp.py
index f65b3c5cbc6..b1027ec684d 100644
--- a/studio/backend/core/inference/llama_cpp.py
+++ b/studio/backend/core/inference/llama_cpp.py
@@ -71,6 +71,8 @@ from core.inference.context_window import (
)
from core.inference.stream_errors import stream_error_from_chunk
from core.inference.llama_server_args import (
+ _CACHE_RAM_FLAGS,
+ _CTX_CHECKPOINTS_FLAGS,
_DEVICE_FLAGS,
_GPU_LAYER_FLAGS,
_LAYER_OFFLOAD_FLAGS,
@@ -8806,6 +8808,42 @@ class LlamaCppBackend:
return arch_by_id
return arch_by_id
+ @staticmethod
+ def _offload_target_shares_system_memory(
+ *, is_vulkan_backend: bool, shared_gpu_ids, detected_gpus, gpu_indices
+ ) -> bool:
+ """True when EVERY device this launch offloads to reports host RAM as its VRAM.
+
+ An iGPU or a unified-memory APU has no separate pool and no bus between the
+ two: a buffer llama.cpp calls "host RAM" and one it calls "VRAM" are the same
+ physical memory. Tuning that exists to avoid moving bytes across PCI-E
+ therefore buys nothing there, and any recovery path it disables is pure loss.
+
+ Fails closed. A mixed selection, an unreadable inventory or a launch that
+ pins nothing all answer False, so the caller keeps the behaviour it has.
+ The two index spaces are kept apart the way the rest of this file keeps
+ them: Vulkan ordinals against shared_gpu_ids, physical HIP/CUDA ids
+ against the unified-memory sets.
+ """
+ devices = (
+ list(gpu_indices) if gpu_indices else [idx for idx, _free in (detected_gpus or ())]
+ )
+ if not devices:
+ return False
+ if is_vulkan_backend:
+ shared = set(shared_gpu_ids or ())
+ return bool(shared) and all(idx in shared for idx in devices)
+ # ROCm only, deliberately. The one caller is the Windows full-offload branch and
+ # every integrated CUDA part is Linux-only, so _integrated_cuda_gpu_ids() could
+ # only answer False here -- at the price of a get_device_properties() call, which
+ # leaks a ~700 MiB primary context (see _get_gpu_memory). The ROCm helper answers
+ # empty on a CUDA torch without touching the device.
+ try:
+ unified = LlamaCppBackend._rocm_unified_memory_gpu_ids()
+ except Exception:
+ return False
+ return bool(unified) and all(idx in unified for idx in devices)
+
@staticmethod
def _amd_apu_wants_unified_memory(gpu_indices = None) -> bool:
"""True only for AMD unified-memory APUs, where GGML_CUDA_ENABLE_UNIFIED_MEMORY
@@ -17248,6 +17286,82 @@ class LlamaCppBackend:
return []
return discrete if 0 < len(discrete) < len(_selected) else []
+ @staticmethod
+ def _cache_tuning_target_unknown(
+ extra_args: Optional[Iterable[str]],
+ gpu_ids: Optional[Iterable[int]],
+ env: Mapping[str, str],
+ ) -> bool:
+ """Whether the device this cache tuning would be chosen against is the one
+ the child actually gets.
+
+ A user device selection is not stripped on an automatic load and llama.cpp
+ reads it last (argv) or first (env, before argv either way), so it, not the
+ automatic placement, names the target. Both spellings count: only an explicit
+ ``gpu_ids`` clears LLAMA_ARG_DEVICE, so on an automatic load the env twin
+ survives into the child verbatim. Fail-closed by design -- the failure that
+ matters is emitting --cache-ram 0 against a shared pool, while keeping the
+ prompt cache on a discrete card only forgoes a tuning.
+ """
+ if gpu_ids is not None:
+ return False
+ return bool(
+ _extra_args_set_any_flag(extra_args, _DEVICE_FLAGS)
+ or str(env.get("LLAMA_ARG_DEVICE", "")).strip()
+ )
+
+ @staticmethod
+ def _retry_cache_tuning_flags(
+ cmd: list[str],
+ *,
+ cache_ram: Optional[int],
+ ctx_checkpoints: Optional[int],
+ server_caps: Mapping[str, Any],
+ ) -> list[str]:
+ """The cache tuning to append when an arch-crash retry lands on a discrete GPU.
+
+ The extras were appended to ``cmd`` long before the retry, so anything added
+ here wins the last-wins parse -- the reverse of the initial launch, where a
+ typed --cache-ram overrides the tuning. So a setting the command already
+ states is skipped, keeping that precedence rather than zeroing a value the
+ panel still shows. An explicit field is handled by the caller's own
+ None checks, exactly as at launch.
+ """
+ # Every spelling llama.cpp accepts, from the sets the arg layer already keeps: a
+ # short -cram or -ctxcp states the setting exactly as the long form does, so
+ # appending after one would zero a value the user typed and the panel shows.
+ stated = {_flag_name(str(token)) for token in cmd}
+ flags: list[str] = []
+ if (
+ cache_ram is None
+ and server_caps.get("supports_cache_ram")
+ and not (_CACHE_RAM_FLAGS & stated)
+ ):
+ flags.extend(["--cache-ram", "0"])
+ checkpoints_flag = server_caps.get("ctx_checkpoints_flag")
+ if ctx_checkpoints is None and checkpoints_flag and not (_CTX_CHECKPOINTS_FLAGS & stated):
+ flags.extend([str(checkpoints_flag), "0"])
+ return flags
+
+ @staticmethod
+ def _without_flag_pairs(cmd: list[str], pairs: list[str]) -> list[str]:
+ """Remove flag/value pairs THIS process appended, by exact token match.
+
+ Only ever called with tokens the cache tuning emitted itself, every one of them
+ a flag followed by its value, so the two-token step cannot swallow a following
+ user extra the way it would for a valueless flag (the reason the mlock path
+ refuses to strip at all). A pair that is no longer present is skipped rather
+ than searched for again, so a repeated call is a no-op.
+ """
+ out = list(cmd)
+ for index in range(0, len(pairs) - 1, 2):
+ flag, value = pairs[index], pairs[index + 1]
+ for at in range(len(out) - 1):
+ if out[at] == flag and out[at + 1] == value:
+ del out[at : at + 2]
+ break
+ return out
+
@staticmethod
def _without_tensor_split(cmd: list[str]) -> Optional[list[str]]:
"""Return cmd with ``--tensor-split``/``-ts`` removed (both spellings and the
@@ -22474,7 +22588,58 @@ class LlamaCppBackend:
# Windows + full offload: drop the host-RAM KV checkpoints that cause
# WDDM/PCI-E overhead, but keep prompt caching (in-VRAM prefix reuse) so
# a repeated prompt is not re-prefilled on every request. #5692.
- if sys.platform == "win32" and full_offload_tuning_active:
+ # ... unless the offload target IS system RAM. #5692 is a discrete card,
+ # whose host-RAM checkpoints cross PCI-E under WDDM, so dropping them is
+ # a saving. An iGPU has one pool and no bus, so the flags only remove the
+ # prompt cache and every chance of reusing a prefix; --cache-ram 0 takes
+ # --cache-idle-slots down with it. Measured on a Strix Halo: a 48.8 h
+ # session spent 44.3 h re-ingesting, 3.38 M prompt against 72 k generated.
+ # Behind the platform gate on purpose: the helper reads torch, and a
+ # launch that never reaches this block would spend a device probe on a
+ # question with no consumer.
+ #
+ # gpu_indices is the picker's answer, not always the child's: with no
+ # gpu_ids a user --device survives in the extras and wins last-wins over
+ # the generated pin. Decline rather than re-derive the target from argv --
+ # the failure that matters is --cache-ram 0 against a shared pool, and
+ # keeping the prompt cache on a discrete card only forgoes a tuning.
+ # The exact tokens the tuning appended, so the arch-crash respawn can
+ # take them back off when it lands on a different device class.
+ _cache_flags_emitted: list[str] = []
+ # extra_args, not the memory policy's list: that one is built later, and
+ # a gpu_ids pin is the case _strip_device_extra_args removes the flag in.
+ # The env twin counts too: only an explicit gpu_ids clears
+ # LLAMA_ARG_DEVICE, and llama.cpp reads it before argv, so an automatic
+ # load can place against a target the generated pin never names.
+ _cache_target_unknown = self._cache_tuning_target_unknown(
+ extra_args, gpu_ids, os.environ
+ )
+ _shared_memory_offload = (
+ sys.platform == "win32"
+ and full_offload_tuning_active
+ and (
+ _cache_target_unknown
+ or self._offload_target_shares_system_memory(
+ is_vulkan_backend = is_vulkan_backend,
+ shared_gpu_ids = _shared_gpu_ids,
+ detected_gpus = _detected_gpus,
+ gpu_indices = gpu_indices,
+ )
+ )
+ )
+ if _cache_target_unknown and sys.platform == "win32" and full_offload_tuning_active:
+ logger.info(
+ "Keeping the llama-server prompt cache: a user --device in the extra "
+ "arguments overrides the automatic placement, so the offload target "
+ "this tuning would be chosen against is not the one the child gets."
+ )
+ elif _shared_memory_offload:
+ logger.info(
+ "Keeping the llama-server prompt cache: this load offloads to a GPU "
+ "that shares system memory, so --cache-ram 0 and --ctx-checkpoints 0 "
+ "would remove prefix reuse without saving a transfer."
+ )
+ elif sys.platform == "win32" and full_offload_tuning_active:
unsupported_cache_flags: list[str] = []
# An explicit control wins over the platform tuning: llama.cpp
# is last-wins, so a 0 emitted after it would overrule the panel
@@ -22482,15 +22647,16 @@ class LlamaCppBackend:
if cache_ram is not None:
pass
elif server_caps.get("supports_cache_ram"):
- cmd.extend(["--cache-ram", "0"])
+ _cache_flags_emitted.extend(["--cache-ram", "0"])
else:
unsupported_cache_flags.append("--cache-ram")
if ctx_checkpoints is not None:
pass
elif server_caps.get("ctx_checkpoints_flag"):
- cmd.extend([str(server_caps["ctx_checkpoints_flag"]), "0"])
+ _cache_flags_emitted.extend([str(server_caps["ctx_checkpoints_flag"]), "0"])
else:
unsupported_cache_flags.append("--ctx-checkpoints")
+ cmd.extend(_cache_flags_emitted)
if unsupported_cache_flags:
logger.info(
"Skipping unsupported Windows cache flags for llama-server: %s",
@@ -24121,6 +24287,43 @@ class LlamaCppBackend:
self._kill_process()
gpu_indices = _remaining
_unified_gpu_indices = _remaining
+ # The tuning was decided against the devices that just crashed,
+ # and this respawn reuses `cmd`, so a discrete-to-APU retry would
+ # carry --cache-ram 0 onto a shared pool and the reverse would
+ # reach a discrete card without it. Re-decide against the retry.
+ if sys.platform == "win32" and full_offload_tuning_active:
+ _retry_shared = _cache_target_unknown or (
+ self._offload_target_shares_system_memory(
+ is_vulkan_backend = is_vulkan_backend,
+ shared_gpu_ids = _shared_gpu_ids,
+ detected_gpus = _retry_rows or _detected_gpus,
+ gpu_indices = _remaining,
+ )
+ )
+ if _retry_shared and _cache_flags_emitted:
+ # Exactly the tokens this policy appended, each a flag
+ # with its value, so the strip cannot eat a user extra.
+ cmd = self._without_flag_pairs(cmd, _cache_flags_emitted)
+ _cache_flags_emitted = []
+ logger.info(
+ "Retry lands on a GPU that shares system memory: dropped "
+ "--cache-ram 0 and --ctx-checkpoints 0 so the prompt cache "
+ "survives."
+ )
+ elif not _retry_shared and not _cache_flags_emitted:
+ _retry_flags = self._retry_cache_tuning_flags(
+ cmd,
+ cache_ram = cache_ram,
+ ctx_checkpoints = ctx_checkpoints,
+ server_caps = server_caps,
+ )
+ if _retry_flags:
+ cmd.extend(_retry_flags)
+ _cache_flags_emitted = list(_retry_flags)
+ logger.info(
+ "Retry lands on a GPU with its own memory: applied the "
+ "Windows full-offload cache tuning."
+ )
# GGML_CUDA_ENABLE_UNIFIED_MEMORY was decided for the CRASHED
# set. The canonical #7624 shape crashes on the APU and retries
# on the dGPU, where it is harmful, so withdraw it, but only
diff --git a/studio/backend/tests/test_server_tuning_flags.py b/studio/backend/tests/test_server_tuning_flags.py
index 3c29eb38ec7..dab11687c70 100644
--- a/studio/backend/tests/test_server_tuning_flags.py
+++ b/studio/backend/tests/test_server_tuning_flags.py
@@ -300,7 +300,10 @@ def test_the_checkpoint_flag_falls_back_to_the_legacy_spelling():
# and the emission uses the recorded name, not a hard-coded one
load = inspect.getsource(llama_cpp.LlamaCppBackend.load_model)
assert "cmd.extend([str(_ctxcp_flag), str(int(ctx_checkpoints))])" in load
- assert 'cmd.extend([str(server_caps["ctx_checkpoints_flag"]), "0"])' in load
+ # The Windows tuning's 0 goes through the list the arch-crash respawn takes its own
+ # flags back off, so the sink is that list rather than cmd. What it pins is unchanged.
+ assert '_cache_flags_emitted.extend([str(server_caps["ctx_checkpoints_flag"]), "0"])' in load
+ assert "cmd.extend(_cache_flags_emitted)" in load
def test_an_unsupported_draft_cache_dtype_is_dropped_not_launched():
diff --git a/studio/backend/tests/test_shared_memory_offload_keeps_the_prompt_cache.py b/studio/backend/tests/test_shared_memory_offload_keeps_the_prompt_cache.py
new file mode 100644
index 00000000000..6a65f54f2f0
--- /dev/null
+++ b/studio/backend/tests/test_shared_memory_offload_keeps_the_prompt_cache.py
@@ -0,0 +1,351 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
+
+"""Windows full-offload tuning (#5692) drops the llama-server prompt cache to avoid
+WDDM/PCI-E traffic. An iGPU or APU has one memory pool and no bus, so the same flags
+save nothing and remove prefix reuse instead.
+
+Measured on a Strix Halo running the Vulkan build with -ngl -1: one 48.8 h session
+spent 44.3 h re-ingesting prompts, 3.38 M prompt tokens against 72 k generated, with
+`--cache-ram 0 --ctx-checkpoints 0` in its own argv and
+`--cache-idle-slots requires --cache-ram, disabling` in llama-server's first lines.
+
+The predicate fails closed everywhere it cannot prove the whole target is shared, so
+a discrete card keeps the tuning it was written for.
+"""
+
+from __future__ import annotations
+
+import types
+
+import pytest
+
+from core.inference.llama_cpp import LlamaCppBackend
+
+_VISIBLE_DEVICE_MASKS = ("HIP_VISIBLE_DEVICES", "ROCR_VISIBLE_DEVICES", "CUDA_VISIBLE_DEVICES")
+
+
[email protected](autouse = True)
+def _no_inherited_gpu_mask(monkeypatch):
+ """The ROCm arm asks about physical ids on a faked torch host. A mask inherited
+ from the shell remaps them onto ids the fake host does not have."""
+ for _mask in _VISIBLE_DEVICE_MASKS:
+ monkeypatch.delenv(_mask, raising = False)
+
+
+def _shares(**kwargs) -> bool:
+ return LlamaCppBackend._offload_target_shares_system_memory(**kwargs)
+
+
+def _vulkan(
+ shared_gpu_ids,
+ detected,
+ gpu_indices = None,
+) -> bool:
+ return _shares(
+ is_vulkan_backend = True,
+ shared_gpu_ids = shared_gpu_ids,
+ detected_gpus = detected,
+ gpu_indices = gpu_indices,
+ )
+
+
+def _fake_torch(archs, *, hip = "6.4.0"):
+ torch = types.ModuleType("torch")
+ torch.version = types.SimpleNamespace(hip = hip)
+ torch.cuda = types.SimpleNamespace(
+ is_available = lambda: True,
+ device_count = lambda: len(archs),
+ get_device_properties = lambda i: types.SimpleNamespace(gcnArchName = archs[i]),
+ )
+ return torch
+
+
+# ── Vulkan, where the shared set is ggml's compact ordinals ──
+
+
+def test_an_igpu_only_target_shares_system_memory():
+ assert _vulkan({0}, [(0, 170079)], gpu_indices = [0]) is True
+
+
+def test_an_unpinned_load_reads_the_detected_list():
+ """The auto-Vulkan arm pins nothing at this point, which is the shape the
+ Strix Halo session ran in."""
+ assert _vulkan({0}, [(0, 170079)]) is True
+
+
+def test_a_discrete_card_beside_the_igpu_keeps_the_tuning():
+ assert _vulkan({0}, [(0, 170079), (1, 24000)], gpu_indices = [0, 1]) is False
+ assert _vulkan({0}, [(0, 170079), (1, 24000)]) is False
+
+
+def test_pinning_only_the_igpu_of_a_mixed_host_still_shares():
+ assert _vulkan({0}, [(0, 170079), (1, 24000)], gpu_indices = [0]) is True
+
+
+def test_a_discrete_only_host_keeps_the_tuning():
+ assert _vulkan(set(), [(0, 24000)], gpu_indices = [0]) is False
+
+
+def test_a_load_with_no_device_at_all_fails_closed():
+ assert _vulkan({0}, []) is False
+ assert _vulkan(set(), []) is False
+
+
+# ── ROCm and CUDA, where the ids are physical ──
+
+
+def test_an_amd_apu_shares_system_memory(monkeypatch):
+ monkeypatch.setitem(__import__("sys").modules, "torch", _fake_torch(["gfx1151"]))
+ assert (
+ _shares(
+ is_vulkan_backend = False,
+ shared_gpu_ids = set(),
+ detected_gpus = [(0, 60000)],
+ gpu_indices = [0],
+ )
+ is True
+ )
+
+
+def test_a_discrete_amd_card_keeps_the_tuning(monkeypatch):
+ monkeypatch.setitem(__import__("sys").modules, "torch", _fake_torch(["gfx1100"]))
+ assert (
+ _shares(
+ is_vulkan_backend = False,
+ shared_gpu_ids = set(),
+ detected_gpus = [(0, 24000)],
+ gpu_indices = [0],
+ )
+ is False
+ )
+
+
+def test_an_apu_next_to_a_discrete_card_keeps_the_tuning(monkeypatch):
+ monkeypatch.setitem(__import__("sys").modules, "torch", _fake_torch(["gfx1151", "gfx1100"]))
+ assert (
+ _shares(
+ is_vulkan_backend = False,
+ shared_gpu_ids = set(),
+ detected_gpus = [(0, 60000), (1, 24000)],
+ gpu_indices = [0, 1],
+ )
+ is False
+ )
+
+
+def test_an_unreadable_inventory_fails_closed(monkeypatch):
+ """No torch, so neither unified-memory set can be built. The tuning stays as it
+ is rather than being dropped on a guess."""
+ monkeypatch.setitem(__import__("sys").modules, "torch", None)
+ assert (
+ _shares(
+ is_vulkan_backend = False,
+ shared_gpu_ids = set(),
+ detected_gpus = [(0, 60000)],
+ gpu_indices = [0],
+ )
+ is False
+ )
+
+
+def test_the_vulkan_shared_set_is_not_read_as_physical_ids(monkeypatch):
+ """Vulkan ordinals and HIP ids are different spaces. A ROCm launch must not
+ inherit a shared set that was built from Vulkan's numbering."""
+ monkeypatch.setitem(__import__("sys").modules, "torch", _fake_torch(["gfx1100"]))
+ assert (
+ _shares(
+ is_vulkan_backend = False,
+ shared_gpu_ids = {0},
+ detected_gpus = [(0, 24000)],
+ gpu_indices = [0],
+ )
+ is False
+ )
+
+
+def test_a_cuda_host_is_answered_without_touching_the_device(monkeypatch):
+ """The predicate must not create a CUDA primary context to answer.
+
+ ``torch.cuda.get_device_properties`` initialises CUDA in the backend process
+ and never gives the memory back (~700 MiB), which is VRAM the child
+ llama-server then cannot use. On a CUDA host the answer is False either way,
+ so it has to come from ``torch.version.hip`` alone. The fake raises on any
+ device probe, so a reintroduced ``_integrated_cuda_gpu_ids()`` call fails
+ this test rather than merely costing memory in production."""
+
+ class _DeviceProbed(BaseException):
+ """BaseException on purpose: every helper in this family swallows
+ ``Exception`` per device, so an ordinary error would be caught and the
+ test would pass against the very code it exists to catch."""
+
+ def _explode(_ordinal):
+ raise _DeviceProbed("probed the CUDA device to answer a Windows-only question")
+
+ torch = types.ModuleType("torch")
+ torch.version = types.SimpleNamespace(hip = None)
+ torch.cuda = types.SimpleNamespace(
+ is_available = lambda: True,
+ device_count = lambda: 1,
+ get_device_properties = _explode,
+ )
+ monkeypatch.setitem(__import__("sys").modules, "torch", torch)
+ assert (
+ _shares(
+ is_vulkan_backend = False,
+ shared_gpu_ids = set(),
+ detected_gpus = [(0, 24000)],
+ gpu_indices = [0],
+ )
+ is False
+ )
+
+
+# ── _without_flag_pairs: the strip the arch-crash respawn uses ──
+
+
+def test_only_the_pairs_this_policy_emitted_are_stripped():
+ """The respawn reuses the argv it already built, so taking the tuning back off has
+ to be an exact-token removal. Every pair is a flag with its value, which is what
+ makes the two-token step safe: a valueless flag would swallow the user extra that
+ follows it, which is why the mlock path refuses to strip at all."""
+ cmd = [
+ "llama-server",
+ "--cache-ram",
+ "0",
+ "--ctx-checkpoints",
+ "0",
+ "--cache-ram",
+ "4096",
+ "--verbose",
+ ]
+ out = LlamaCppBackend._without_flag_pairs(cmd, ["--cache-ram", "0", "--ctx-checkpoints", "0"])
+ # The user's own --cache-ram 4096 survives; only the emitted zeros go.
+ assert out == ["llama-server", "--cache-ram", "4096", "--verbose"]
+
+
+def test_stripping_a_pair_that_is_gone_is_a_no_op():
+ cmd = ["llama-server", "--verbose"]
+ assert LlamaCppBackend._without_flag_pairs(cmd, ["--cache-ram", "0"]) == cmd
+
+
+def test_a_trailing_flag_without_its_value_is_left_alone():
+ # Defensive: an odd-length pair list would otherwise index past the end.
+ cmd = ["llama-server", "--cache-ram"]
+ assert LlamaCppBackend._without_flag_pairs(cmd, ["--cache-ram"]) == cmd
+
+
+# ── the target the tuning is chosen against ──
+
+
+def test_a_user_device_flag_makes_the_target_unknown():
+ assert LlamaCppBackend._cache_tuning_target_unknown(["--device", "ROCm1"], None, {})
+
+
+def test_an_inherited_device_env_makes_the_target_unknown():
+ # Only an explicit gpu_ids clears the env twin, and llama.cpp reads it before argv, so
+ # the generated pin is not what the child places against. Reading argv alone emitted
+ # --cache-ram 0 at an APU the picker had paired with a discrete card.
+ assert LlamaCppBackend._cache_tuning_target_unknown(None, None, {"LLAMA_ARG_DEVICE": "ROCm0"})
+ assert not LlamaCppBackend._cache_tuning_target_unknown(None, None, {"LLAMA_ARG_DEVICE": " "})
+
+
+def test_an_explicit_pin_owns_the_placement_so_the_target_is_known():
+ # The control: gpu_ids clears both spellings, so neither can name another device.
+ assert not LlamaCppBackend._cache_tuning_target_unknown(
+ ["--device", "ROCm1"], [0], {"LLAMA_ARG_DEVICE": "ROCm0"}
+ )
+ assert not LlamaCppBackend._cache_tuning_target_unknown(None, None, {})
+
+
+# ── the arch-crash retry keeps the launch's precedence ──
+
+_CAPS = {"supports_cache_ram": True, "ctx_checkpoints_flag": "--ctx-checkpoints"}
+
+
+def test_the_retry_applies_the_tuning_when_nothing_states_it():
+ assert LlamaCppBackend._retry_cache_tuning_flags(
+ ["llama-server", "-m", "x.gguf"],
+ cache_ram = None,
+ ctx_checkpoints = None,
+ server_caps = _CAPS,
+ ) == ["--cache-ram", "0", "--ctx-checkpoints", "0"]
+
+
+def test_the_retry_does_not_overrule_a_cache_flag_the_command_already_states():
+ # The extras sit in cmd already, so appending here wins last-wins and zeroes a value
+ # the panel still shows -- the reverse of the launch, where the extras win.
+ flags = LlamaCppBackend._retry_cache_tuning_flags(
+ ["llama-server", "-m", "x.gguf", "--cache-ram", "8192"],
+ cache_ram = None,
+ ctx_checkpoints = None,
+ server_caps = _CAPS,
+ )
+ assert flags == ["--ctx-checkpoints", "0"]
+
+ assert (
+ LlamaCppBackend._retry_cache_tuning_flags(
+ ["llama-server", "--cache-ram=8192", "--ctx-checkpoints=4"],
+ cache_ram = None,
+ ctx_checkpoints = None,
+ server_caps = _CAPS,
+ )
+ == []
+ )
+
+
+def test_the_retry_reads_the_short_spellings_as_the_same_settings():
+ # -cram and -ctxcp are aliases llama.cpp accepts and the panel offers, so appending
+ # the long form after one would silently zero the user's value.
+ assert (
+ LlamaCppBackend._retry_cache_tuning_flags(
+ ["llama-server", "-cram", "8192", "-ctxcp", "4"],
+ cache_ram = None,
+ ctx_checkpoints = None,
+ server_caps = _CAPS,
+ )
+ == []
+ )
+
+ # One at a time, so a single alias cannot stand in for the pair.
+ assert LlamaCppBackend._retry_cache_tuning_flags(
+ ["llama-server", "-cram", "8192"],
+ cache_ram = None,
+ ctx_checkpoints = None,
+ server_caps = _CAPS,
+ ) == ["--ctx-checkpoints", "0"]
+ assert LlamaCppBackend._retry_cache_tuning_flags(
+ ["llama-server", "-ctxcp=4"],
+ cache_ram = None,
+ ctx_checkpoints = None,
+ server_caps = _CAPS,
+ ) == ["--cache-ram", "0"]
+
+
+def test_the_retry_reads_the_other_checkpoint_alias_as_the_same_setting():
+ # A build advertising --ctx-checkpoints can still be handed --swa-checkpoints.
+ assert LlamaCppBackend._retry_cache_tuning_flags(
+ ["llama-server", "--swa-checkpoints", "4"],
+ cache_ram = None,
+ ctx_checkpoints = None,
+ server_caps = _CAPS,
+ ) == ["--cache-ram", "0"]
+
+
+def test_the_retry_skips_what_the_build_and_the_fields_already_own():
+ # An explicit field, as at launch, and a build without the capability.
+ assert (
+ LlamaCppBackend._retry_cache_tuning_flags(
+ ["llama-server"], cache_ram = 4096, ctx_checkpoints = 8, server_caps = _CAPS
+ )
+ == []
+ )
+ assert (
+ LlamaCppBackend._retry_cache_tuning_flags(
+ ["llama-server"],
+ cache_ram = None,
+ ctx_checkpoints = None,
+ server_caps = {"supports_cache_ram": False, "ctx_checkpoints_flag": None},
+ )
+ == []
+ )