Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

Studio: skip the Keep resident page-lock when the llama-server --help probe is inconclusive (#15833)

Daniel Han1b3ad974ea6a
diff --git a/studio/backend/core/inference/llama_cpp.py b/studio/backend/core/inference/llama_cpp.py
index 233d7458fbc..480818103b8 100644
--- a/studio/backend/core/inference/llama_cpp.py
+++ b/studio/backend/core/inference/llama_cpp.py
@@ -134,6 +134,8 @@ from core.inference.llama_server_args import (
fit_target_margin_in,
MANAGED_DIO_FLAGS,
no_reserve_requires_dio,
+ keep_resident_lock_applicable,
+ keep_resident_lock_flags,
load_mode_support,
resolve_launch_load_mode,
resolve_effective_load_state,
@@ -26133,7 +26135,7 @@ class LlamaCppBackend:
extra_args,
binary = binary,
requested_load_mode = load_mode,
- supports_load_mode = bool(server_caps.get("supports_load_mode")),
+ supports_load_mode = load_mode_support(server_caps),
settings = _mem_settings,
layers_fixed = _caller_owns_budget or _paravirtual_cpu_forced,
mtp_may_engage = _mtp_will_engage,
@@ -26312,9 +26314,7 @@ class LlamaCppBackend:
extra_args,
binary = binary,
requested_load_mode = load_mode,
- supports_load_mode = bool(
- server_caps.get("supports_load_mode")
- ),
+ supports_load_mode = load_mode_support(server_caps),
settings = _mem_settings,
layers_fixed = _caller_owns_budget or _paravirtual_cpu_forced,
mtp_may_engage = False,
@@ -28965,7 +28965,7 @@ class LlamaCppBackend:
# Armed here, not at load start: rows from before the Hub download are stale.
_arm_load_probe_memo()
_mem_dio_possible = no_reserve_requires_dio(
- supports_load_mode = bool(server_caps.get("supports_load_mode")),
+ supports_load_mode = load_mode_support(server_caps),
gpu_offload_confirmed = True,
)
# NOT gated on the toggle: the answer is this launch's recorded placement
@@ -29015,7 +29015,7 @@ class LlamaCppBackend:
)
_mem_managed, _mem_extras = apply_model_memory_policy(
extra_args,
- supports_load_mode = bool(server_caps.get("supports_load_mode")),
+ supports_load_mode = load_mode_support(server_caps),
weights_in_host_memory = _mem_host_resident,
gpu_offload_confirmed = _mem_gpu_offload_confirmed,
env = _fit_load_mode_env_view,
@@ -29171,6 +29171,19 @@ class LlamaCppBackend:
and is_vulkan_backend
and not self._vulkan_probe_answered(binary)
)
+ # An unknown build gets no lock, so none is owed either.
+ self._memory_mlock_applicable = keep_resident_lock_applicable(
+ self._memory_mlock_applicable, load_mode_support(server_caps)
+ )
+ if (
+ _mem_should_mlock
+ and _mem_host_resident
+ and load_mode_support(server_caps) is None
+ ):
+ logger.info(
+ "Model Memory: skipping page-lock, the llama-server --help "
+ "probe was inconclusive and newer builds reject --mlock."
+ )
if _mem_should_mlock and not _mem_host_resident:
logger.info(
"Model Memory: skipping page-lock, the weights are fully "
@@ -30618,19 +30631,20 @@ class LlamaCppBackend:
)
# With --fit off, -ngl defaults to every layer, so the host-resident verdict may no longer hold.
if _mem_should_mlock and not _mem_host_resident:
- _run.extend(
- ["--load-mode", "mmap+mlock"]
- if server_caps.get("supports_load_mode")
- else ["--mlock"]
+ _retry_lock = keep_resident_lock_flags(
+ load_mode_support(server_caps)
)
+ _run.extend(_retry_lock)
_mem_host_resident = True
- self._memory_mlock_applicable = True
- self._memory_policy_active = True
- logger.info(
- "Model Memory: re-applying the page-lock for "
- "the --fit on retry; the full-offload "
- "prediction did not hold."
- )
+ # An unknown build gets no lock, so none is owed either.
+ self._memory_mlock_applicable = bool(_retry_lock)
+ if _retry_lock:
+ self._memory_policy_active = True
+ logger.info(
+ "Model Memory: re-applying the page-lock for "
+ "the --fit on retry; the full-offload "
+ "prediction did not hold."
+ )
run_cmd = _run
self._record_memory_state(_run, env)
_did_fit_retry = True
@@ -31458,7 +31472,7 @@ class LlamaCppBackend:
if _retry_host_resident and not _mem_host_resident:
_retry_managed, _ = apply_model_memory_policy(
extra_args,
- supports_load_mode = bool(server_caps.get("supports_load_mode")),
+ supports_load_mode = load_mode_support(server_caps),
# Not the requested load mode: the first launch
# emitted it and this call only ADDS, so a second
# copy would land after the extras and mark
@@ -31474,7 +31488,9 @@ class LlamaCppBackend:
# flag. Nothing is stripped, so kept extras survive.
cmd.extend(_retry_managed)
_mem_host_resident = _retry_host_resident
- self._memory_mlock_applicable = _retry_host_resident
+ self._memory_mlock_applicable = keep_resident_lock_applicable(
+ _retry_host_resident, load_mode_support(server_caps)
+ )
# Stays active if it already was: the flag also
# covers extras this policy touched and env it
# scrubbed, neither of which the respawn undoes.
diff --git a/studio/backend/core/inference/llama_server_args.py b/studio/backend/core/inference/llama_server_args.py
index 15cdb96eef1..b9716d65129 100644
--- a/studio/backend/core/inference/llama_server_args.py
+++ b/studio/backend/core/inference/llama_server_args.py
@@ -1308,9 +1308,11 @@ def strip_context_only(args: Optional[Iterable[str]]) -> Optional[list[str]]:
MANAGED_DIO_FLAGS: tuple[str, ...] = ("--load-mode", "dio")
-def no_reserve_requires_dio(*, supports_load_mode: bool, gpu_offload_confirmed: bool) -> bool:
+def no_reserve_requires_dio(
+ *, supports_load_mode: Optional[bool], gpu_offload_confirmed: bool
+) -> bool:
"""Windows keeps a full offload mapping resident, so DirectIO is owed only on confirmed offload and on a build supporting --load-mode."""
- return sys.platform == "win32" and supports_load_mode and gpu_offload_confirmed
+ return sys.platform == "win32" and bool(supports_load_mode) and gpu_offload_confirmed
def load_mode_support(server_caps: Mapping[str, object]) -> Optional[bool]:
@@ -1322,6 +1324,19 @@ def load_mode_support(server_caps: Mapping[str, object]) -> Optional[bool]:
return None
+def keep_resident_lock_flags(supports_load_mode: Optional[bool]) -> list[str]:
+ """Page-lock spelling for this build; none when unknown, since newer builds exit on --mlock."""
+ if supports_load_mode is None:
+ return []
+ # mmap+mlock, not bare mlock, matches what --mlock meant with default mmap.
+ return ["--load-mode", "mmap+mlock"] if supports_load_mode else ["--mlock"]
+
+
+def keep_resident_lock_applicable(host_resident: bool, supports_load_mode: Optional[bool]) -> bool:
+ """Whether a missing lock is owed; an unknown build gets none, so a reload must not ask for one."""
+ return bool(host_resident) and supports_load_mode is not None
+
+
def resolve_launch_load_mode(
extra_args: Optional[Iterable[str]],
*,
@@ -1357,7 +1372,7 @@ def resolve_launch_load_mode(
def apply_model_memory_policy(
extra_args: Optional[Iterable[str]],
*,
- supports_load_mode: bool = False,
+ supports_load_mode: Optional[bool] = False,
weights_in_host_memory: bool = True,
gpu_offload_confirmed: bool = False,
env: Optional[Mapping[str, str]] = None,
@@ -1400,9 +1415,9 @@ def apply_model_memory_policy(
):
# On confirmed full offload, Windows streams via dio since the GGUF cannot be partially unmapped; per-model modes still win later.
managed.extend(MANAGED_DIO_FLAGS)
- if keep_resident and not no_ram_reserve and weights_in_host_memory:
- # Before the extras, like the managed block: mmap+mlock, not bare mlock, matches what --mlock meant with default mmap.
- managed.extend(["--load-mode", "mmap+mlock"] if supports_load_mode else ["--mlock"])
+ lock = keep_resident_lock_flags(supports_load_mode)
+ if keep_resident and not no_ram_reserve and weights_in_host_memory and lock:
+ managed.extend(lock)
tokens = strip_shadowing_flags(
tokens,
strip_context = False,
diff --git a/studio/backend/tests/test_gpu_arch_gate_os_matrix_7624.py b/studio/backend/tests/test_gpu_arch_gate_os_matrix_7624.py
index 9dbc346c4a9..e73a36dea8b 100644
--- a/studio/backend/tests/test_gpu_arch_gate_os_matrix_7624.py
+++ b/studio/backend/tests/test_gpu_arch_gate_os_matrix_7624.py
@@ -2715,6 +2715,10 @@ class TestInheritedSplitEnvGoesWithTheArgvStrip:
assert env.get("LLAMA_ARG_TENSOR_SPLIT") == "3,1"
+# A build whose --help probe answered (old spellings only): an unanswered probe gets no lock (#15828).
+_PROBED_CAPS = {"help_probe_ok": True, "flags": ["--mlock", "--no-mmap"]}
+
+
class TestTheForcedCpuLaunchAppliesThePageLock:
"""Residency must be judged on the post-gate devices, or a forced-CPU load skips the page-lock."""
@@ -2745,6 +2749,7 @@ class TestTheForcedCpuLaunchAppliesThePageLock:
capture = capture,
backend = backend,
intent_kwargs = {"gpu_memory_mode": "manual", "gpu_layers": 33},
+ server_caps = _PROBED_CAPS,
)
return launches, capture
@@ -2844,6 +2849,7 @@ class TestTheApuRetryRecomputesThePageLock:
returncode = 1,
output = "ROCm error: device kernel image is invalid",
capture = capture,
+ server_caps = _PROBED_CAPS,
)
return launches, capture
@@ -2883,6 +2889,7 @@ class TestTheApuRetryRecomputesThePageLock:
capture = capture,
# -c 8192 is the entry a valueless-flag scan would eat with --no-mmap.
intent_kwargs = {"extra_args": ["--no-mmap", "-c", "8192"]},
+ server_caps = _PROBED_CAPS,
)
retry = [(cmd, env) for cmd, env in launches if env.get("ROCR_VISIBLE_DEVICES") == "1"]
assert retry, [_visibility(e) for _c, e in launches]
diff --git a/studio/backend/tests/test_load_mode_caps_unknown.py b/studio/backend/tests/test_load_mode_caps_unknown.py
index b1eddf043d9..a2eb32e3d64 100644
--- a/studio/backend/tests/test_load_mode_caps_unknown.py
+++ b/studio/backend/tests/test_load_mode_caps_unknown.py
@@ -102,3 +102,77 @@ def test_a_build_listing_load_mode_gets_it(tmp_path):
_fake_server(tmp_path, f"cat <<'X'\n{help_text}\nX\nexit 0")
)
assert _launch_flags(caps) == ["--load-mode", "none"]
+
+
+_KEEP_RESIDENT = (True, False)
+
+
+ "support, expected",
+ [(True, ["--load-mode", "mmap+mlock"]), (False, ["--mlock"]), (None, [])],
+)
+def test_keep_resident_lock_spelling_follows_the_probe(support, expected):
+ from core.inference.llama_server_args import apply_model_memory_policy, keep_resident_lock_flags
+
+ assert keep_resident_lock_flags(support) == expected
+ managed, extras = apply_model_memory_policy(
+ ["--temp", "0.7"],
+ supports_load_mode = support,
+ weights_in_host_memory = True,
+ settings = _KEEP_RESIDENT,
+ )
+ assert managed == expected
+ assert extras == ["--temp", "0.7"]
+
+
+def test_unknown_support_leaves_a_typed_lock_alone():
+ from core.inference.llama_server_args import apply_model_memory_policy
+ assert apply_model_memory_policy(
+ ["--mlock"],
+ supports_load_mode = None,
+ weights_in_host_memory = True,
+ settings = _KEEP_RESIDENT,
+ ) == ([], ["--mlock"])
+
+
[email protected]("help_body", ["exit 1", "exit 0"])
+def test_a_skipped_lock_does_not_demand_a_reload(tmp_path, monkeypatch, help_body):
+ import utils.model_memory_settings as mm
+ from core.inference.llama_server_args import (
+ apply_model_memory_policy,
+ keep_resident_lock_applicable,
+ load_mode_support,
+ memory_state_satisfies_settings,
+ resolve_effective_memory_state,
+ )
+
+ monkeypatch.setattr(mm, "get_keep_resident", lambda: True)
+ monkeypatch.setattr(mm, "get_no_ram_reserve", lambda: False)
+ caps = LlamaCppBackend.probe_server_capabilities(_fake_server(tmp_path, help_body))
+ support = load_mode_support(caps)
+ managed, extras = apply_model_memory_policy(
+ [], supports_load_mode = support, weights_in_host_memory = True, settings = _KEEP_RESIDENT
+ )
+ assert managed == []
+ state = resolve_effective_memory_state([*managed, *extras], {})
+ applicable = keep_resident_lock_applicable(True, support)
+ assert applicable is False
+ assert memory_state_satisfies_settings(state, bool(managed), applicable) is True
+
+
[email protected]("support", [True, False])
+def test_a_known_build_still_owes_the_lock(support):
+ from core.inference.llama_server_args import keep_resident_lock_applicable
+ assert keep_resident_lock_applicable(True, support) is True
+ assert keep_resident_lock_applicable(False, support) is False
+
+
+def test_no_launch_site_reads_the_raw_load_mode_cap():
+ import inspect
+
+ from core.inference import llama_cpp
+
+ source = inspect.getsource(llama_cpp)
+ assert 'server_caps.get("supports_load_mode")' not in source
+ assert "server_caps['supports_load_mode']" not in source
+ assert 'server_caps["supports_load_mode"]' not in source