Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
Studio: skip the Keep resident page-lock when the llama-server --help probe is inconclusive (#15833)
Daniel Han1b3ad974ea6a
diff --git a/studio/backend/core/inference/llama_cpp.py b/studio/backend/core/inference/llama_cpp.pyindex 233d7458fbc..480818103b8 100644--- a/studio/backend/core/inference/llama_cpp.py+++ b/studio/backend/core/inference/llama_cpp.py@@ -134,6 +134,8 @@ from core.inference.llama_server_args import (fit_target_margin_in,MANAGED_DIO_FLAGS,no_reserve_requires_dio,+ keep_resident_lock_applicable,+ keep_resident_lock_flags,load_mode_support,resolve_launch_load_mode,resolve_effective_load_state,@@ -26133,7 +26135,7 @@ class LlamaCppBackend:extra_args,binary = binary,requested_load_mode = load_mode,- supports_load_mode = bool(server_caps.get("supports_load_mode")),+ supports_load_mode = load_mode_support(server_caps),settings = _mem_settings,layers_fixed = _caller_owns_budget or _paravirtual_cpu_forced,mtp_may_engage = _mtp_will_engage,@@ -26312,9 +26314,7 @@ class LlamaCppBackend:extra_args,binary = binary,requested_load_mode = load_mode,- supports_load_mode = bool(- server_caps.get("supports_load_mode")- ),+ supports_load_mode = load_mode_support(server_caps),settings = _mem_settings,layers_fixed = _caller_owns_budget or _paravirtual_cpu_forced,mtp_may_engage = False,@@ -28965,7 +28965,7 @@ class LlamaCppBackend:# Armed here, not at load start: rows from before the Hub download are stale._arm_load_probe_memo()_mem_dio_possible = no_reserve_requires_dio(- supports_load_mode = bool(server_caps.get("supports_load_mode")),+ supports_load_mode = load_mode_support(server_caps),gpu_offload_confirmed = True,)# NOT gated on the toggle: the answer is this launch's recorded placement@@ -29015,7 +29015,7 @@ class LlamaCppBackend:)_mem_managed, _mem_extras = apply_model_memory_policy(extra_args,- supports_load_mode = bool(server_caps.get("supports_load_mode")),+ supports_load_mode = load_mode_support(server_caps),weights_in_host_memory = _mem_host_resident,gpu_offload_confirmed = _mem_gpu_offload_confirmed,env = _fit_load_mode_env_view,@@ -29171,6 +29171,19 @@ class LlamaCppBackend:and is_vulkan_backendand not self._vulkan_probe_answered(binary))+ # An unknown build gets no lock, so none is owed either.+ self._memory_mlock_applicable = keep_resident_lock_applicable(+ self._memory_mlock_applicable, load_mode_support(server_caps)+ )+ if (+ _mem_should_mlock+ and _mem_host_resident+ and load_mode_support(server_caps) is None+ ):+ logger.info(+ "Model Memory: skipping page-lock, the llama-server --help "+ "probe was inconclusive and newer builds reject --mlock."+ )if _mem_should_mlock and not _mem_host_resident:logger.info("Model Memory: skipping page-lock, the weights are fully "@@ -30618,19 +30631,20 @@ class LlamaCppBackend:)# With --fit off, -ngl defaults to every layer, so the host-resident verdict may no longer hold.if _mem_should_mlock and not _mem_host_resident:- _run.extend(- ["--load-mode", "mmap+mlock"]- if server_caps.get("supports_load_mode")- else ["--mlock"]+ _retry_lock = keep_resident_lock_flags(+ load_mode_support(server_caps))+ _run.extend(_retry_lock)_mem_host_resident = True- self._memory_mlock_applicable = True- self._memory_policy_active = True- logger.info(- "Model Memory: re-applying the page-lock for "- "the --fit on retry; the full-offload "- "prediction did not hold."- )+ # An unknown build gets no lock, so none is owed either.+ self._memory_mlock_applicable = bool(_retry_lock)+ if _retry_lock:+ self._memory_policy_active = True+ logger.info(+ "Model Memory: re-applying the page-lock for "+ "the --fit on retry; the full-offload "+ "prediction did not hold."+ )run_cmd = _runself._record_memory_state(_run, env)_did_fit_retry = True@@ -31458,7 +31472,7 @@ class LlamaCppBackend:if _retry_host_resident and not _mem_host_resident:_retry_managed, _ = apply_model_memory_policy(extra_args,- supports_load_mode = bool(server_caps.get("supports_load_mode")),+ supports_load_mode = load_mode_support(server_caps),# Not the requested load mode: the first launch# emitted it and this call only ADDS, so a second# copy would land after the extras and mark@@ -31474,7 +31488,9 @@ class LlamaCppBackend:# flag. Nothing is stripped, so kept extras survive.cmd.extend(_retry_managed)_mem_host_resident = _retry_host_resident- self._memory_mlock_applicable = _retry_host_resident+ self._memory_mlock_applicable = keep_resident_lock_applicable(+ _retry_host_resident, load_mode_support(server_caps)+ )# Stays active if it already was: the flag also# covers extras this policy touched and env it# scrubbed, neither of which the respawn undoes.diff --git a/studio/backend/core/inference/llama_server_args.py b/studio/backend/core/inference/llama_server_args.pyindex 15cdb96eef1..b9716d65129 100644--- a/studio/backend/core/inference/llama_server_args.py+++ b/studio/backend/core/inference/llama_server_args.py@@ -1308,9 +1308,11 @@ def strip_context_only(args: Optional[Iterable[str]]) -> Optional[list[str]]:MANAGED_DIO_FLAGS: tuple[str, ...] = ("--load-mode", "dio")-def no_reserve_requires_dio(*, supports_load_mode: bool, gpu_offload_confirmed: bool) -> bool:+def no_reserve_requires_dio(+ *, supports_load_mode: Optional[bool], gpu_offload_confirmed: bool+) -> bool:"""Windows keeps a full offload mapping resident, so DirectIO is owed only on confirmed offload and on a build supporting --load-mode."""- return sys.platform == "win32" and supports_load_mode and gpu_offload_confirmed+ return sys.platform == "win32" and bool(supports_load_mode) and gpu_offload_confirmeddef load_mode_support(server_caps: Mapping[str, object]) -> Optional[bool]:@@ -1322,6 +1324,19 @@ def load_mode_support(server_caps: Mapping[str, object]) -> Optional[bool]:return None+def keep_resident_lock_flags(supports_load_mode: Optional[bool]) -> list[str]:+ """Page-lock spelling for this build; none when unknown, since newer builds exit on --mlock."""+ if supports_load_mode is None:+ return []+ # mmap+mlock, not bare mlock, matches what --mlock meant with default mmap.+ return ["--load-mode", "mmap+mlock"] if supports_load_mode else ["--mlock"]+++def keep_resident_lock_applicable(host_resident: bool, supports_load_mode: Optional[bool]) -> bool:+ """Whether a missing lock is owed; an unknown build gets none, so a reload must not ask for one."""+ return bool(host_resident) and supports_load_mode is not None++def resolve_launch_load_mode(extra_args: Optional[Iterable[str]],*,@@ -1357,7 +1372,7 @@ def resolve_launch_load_mode(def apply_model_memory_policy(extra_args: Optional[Iterable[str]],*,- supports_load_mode: bool = False,+ supports_load_mode: Optional[bool] = False,weights_in_host_memory: bool = True,gpu_offload_confirmed: bool = False,env: Optional[Mapping[str, str]] = None,@@ -1400,9 +1415,9 @@ def apply_model_memory_policy():# On confirmed full offload, Windows streams via dio since the GGUF cannot be partially unmapped; per-model modes still win later.managed.extend(MANAGED_DIO_FLAGS)- if keep_resident and not no_ram_reserve and weights_in_host_memory:- # Before the extras, like the managed block: mmap+mlock, not bare mlock, matches what --mlock meant with default mmap.- managed.extend(["--load-mode", "mmap+mlock"] if supports_load_mode else ["--mlock"])+ lock = keep_resident_lock_flags(supports_load_mode)+ if keep_resident and not no_ram_reserve and weights_in_host_memory and lock:+ managed.extend(lock)tokens = strip_shadowing_flags(tokens,strip_context = False,diff --git a/studio/backend/tests/test_gpu_arch_gate_os_matrix_7624.py b/studio/backend/tests/test_gpu_arch_gate_os_matrix_7624.pyindex 9dbc346c4a9..e73a36dea8b 100644--- a/studio/backend/tests/test_gpu_arch_gate_os_matrix_7624.py+++ b/studio/backend/tests/test_gpu_arch_gate_os_matrix_7624.py@@ -2715,6 +2715,10 @@ class TestInheritedSplitEnvGoesWithTheArgvStrip:assert env.get("LLAMA_ARG_TENSOR_SPLIT") == "3,1"+# A build whose --help probe answered (old spellings only): an unanswered probe gets no lock (#15828).+_PROBED_CAPS = {"help_probe_ok": True, "flags": ["--mlock", "--no-mmap"]}++class TestTheForcedCpuLaunchAppliesThePageLock:"""Residency must be judged on the post-gate devices, or a forced-CPU load skips the page-lock."""@@ -2745,6 +2749,7 @@ class TestTheForcedCpuLaunchAppliesThePageLock:capture = capture,backend = backend,intent_kwargs = {"gpu_memory_mode": "manual", "gpu_layers": 33},+ server_caps = _PROBED_CAPS,)return launches, capture@@ -2844,6 +2849,7 @@ class TestTheApuRetryRecomputesThePageLock:returncode = 1,output = "ROCm error: device kernel image is invalid",capture = capture,+ server_caps = _PROBED_CAPS,)return launches, capture@@ -2883,6 +2889,7 @@ class TestTheApuRetryRecomputesThePageLock:capture = capture,# -c 8192 is the entry a valueless-flag scan would eat with --no-mmap.intent_kwargs = {"extra_args": ["--no-mmap", "-c", "8192"]},+ server_caps = _PROBED_CAPS,)retry = [(cmd, env) for cmd, env in launches if env.get("ROCR_VISIBLE_DEVICES") == "1"]assert retry, [_visibility(e) for _c, e in launches]diff --git a/studio/backend/tests/test_load_mode_caps_unknown.py b/studio/backend/tests/test_load_mode_caps_unknown.pyindex b1eddf043d9..a2eb32e3d64 100644--- a/studio/backend/tests/test_load_mode_caps_unknown.py+++ b/studio/backend/tests/test_load_mode_caps_unknown.py@@ -102,3 +102,77 @@ def test_a_build_listing_load_mode_gets_it(tmp_path):_fake_server(tmp_path, f"cat <<'X'\n{help_text}\nX\nexit 0"))assert _launch_flags(caps) == ["--load-mode", "none"]+++_KEEP_RESIDENT = (True, False)+++ "support, expected",+ [(True, ["--load-mode", "mmap+mlock"]), (False, ["--mlock"]), (None, [])],+)+def test_keep_resident_lock_spelling_follows_the_probe(support, expected):+ from core.inference.llama_server_args import apply_model_memory_policy, keep_resident_lock_flags++ assert keep_resident_lock_flags(support) == expected+ managed, extras = apply_model_memory_policy(+ ["--temp", "0.7"],+ supports_load_mode = support,+ weights_in_host_memory = True,+ settings = _KEEP_RESIDENT,+ )+ assert managed == expected+ assert extras == ["--temp", "0.7"]+++def test_unknown_support_leaves_a_typed_lock_alone():+ from core.inference.llama_server_args import apply_model_memory_policy+ assert apply_model_memory_policy(+ ["--mlock"],+ supports_load_mode = None,+ weights_in_host_memory = True,+ settings = _KEEP_RESIDENT,+ ) == ([], ["--mlock"])++[email protected]("help_body", ["exit 1", "exit 0"])+def test_a_skipped_lock_does_not_demand_a_reload(tmp_path, monkeypatch, help_body):+ import utils.model_memory_settings as mm+ from core.inference.llama_server_args import (+ apply_model_memory_policy,+ keep_resident_lock_applicable,+ load_mode_support,+ memory_state_satisfies_settings,+ resolve_effective_memory_state,+ )++ monkeypatch.setattr(mm, "get_keep_resident", lambda: True)+ monkeypatch.setattr(mm, "get_no_ram_reserve", lambda: False)+ caps = LlamaCppBackend.probe_server_capabilities(_fake_server(tmp_path, help_body))+ support = load_mode_support(caps)+ managed, extras = apply_model_memory_policy(+ [], supports_load_mode = support, weights_in_host_memory = True, settings = _KEEP_RESIDENT+ )+ assert managed == []+ state = resolve_effective_memory_state([*managed, *extras], {})+ applicable = keep_resident_lock_applicable(True, support)+ assert applicable is False+ assert memory_state_satisfies_settings(state, bool(managed), applicable) is True++[email protected]("support", [True, False])+def test_a_known_build_still_owes_the_lock(support):+ from core.inference.llama_server_args import keep_resident_lock_applicable+ assert keep_resident_lock_applicable(True, support) is True+ assert keep_resident_lock_applicable(False, support) is False+++def test_no_launch_site_reads_the_raw_load_mode_cap():+ import inspect++ from core.inference import llama_cpp++ source = inspect.getsource(llama_cpp)+ assert 'server_caps.get("supports_load_mode")' not in source+ assert "server_caps['supports_load_mode']" not in source+ assert 'server_caps["supports_load_mode"]' not in source