Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
fix(studio): avoid resident GGUF mappings for Windows no-reserve loads (#10618)
Lee Jackson159207880cc4
diff --git a/studio/backend/core/inference/_vulkan_probe.py b/studio/backend/core/inference/_vulkan_probe.pyindex dfde5a3b344..3a29480cf62 100644--- a/studio/backend/core/inference/_vulkan_probe.py+++ b/studio/backend/core/inference/_vulkan_probe.py@@ -6,7 +6,7 @@Run in a short-lived subprocess (``python _vulkan_probe.py <bindir>``) so theVulkan instance never lives in the long-running backend process. Loads thebundled ggml Vulkan backend from ``<bindir>`` and prints one-``<idx>\\t<free_bytes>\\t<is_igpu>\\t<total_bytes>\\t<name>`` line per device to+``<idx>\\t<free_bytes>\\t<is_igpu>\\t<total_bytes>\\t<name>\\t<type_known>`` line per device tostdout. Indices are ggml's own Vulkan device ordinals, which need not matchnvidia-smi order. ``is_igpu`` (from ggml's device type) is ``1`` for anintegrated GPU sharing system RAM. ``total_bytes`` is the device-local heap;@@ -26,17 +26,23 @@ import sys_GGML_BACKEND_DEVICE_TYPE_IGPU = 2-def _igpu_flags_and_names(base, lib, count: int) -> tuple[list[bool], list[str]]:- """Per-device integrated-GPU flags and descriptions via ggml's backend registry.+def _igpu_flags_and_names(base, lib, count: int) -> tuple[list[bool], list[str], list[bool]]:+ """Per-device integrated-GPU flags, descriptions, and whether the type was READ.The Vulkan reg enumerates devices in the same order as``ggml_backend_vk_get_device_memory`` (each context uses ``ctx->device =i``), so reg index == device ordinal. Returns all-False / empty-name on anyfailure so the reader never over-caps a discrete card and the memoryreadings still get through.++ The third list says whether each flag is an ANSWER. Without it "not integrated"+ and "could not tell" are the same value, which is safe for a caller that only+ skips a page-lock and wrong for one choosing a loader: a DirectIO decision taken+ on an unread type buffers an iGPU's host-backed weights."""flags = [False] * countnames = [""] * count+ known = [False] * counttry:lib.ggml_backend_vk_reg.restype = ctypes.c_void_plib.ggml_backend_vk_reg.argtypes = []@@ -48,12 +54,12 @@ def _igpu_flags_and_names(base, lib, count: int) -> tuple[list[bool], list[str]]base.ggml_backend_dev_type.argtypes = [ctypes.c_void_p]reg = lib.ggml_backend_vk_reg()if not reg:- return flags, names+ return flags, names, knowndev_count = base.ggml_backend_reg_dev_count(reg)except Exception:# Best-effort: any failure degrades to "discrete"/"unnamed" so the memory readings still get through instead of# crashing the probe.- return flags, names+ return flags, names, known# Bound outside the type-detection try above: a ggml-base without the description symbol (older/custom build) must# degrade to unnamed devices, not abort before the iGPU flags are read (which would count an iGPU's shared RAM as@@ -77,6 +83,7 @@ def _igpu_flags_and_names(base, lib, count: int) -> tuple[list[bool], list[str]]continuetry:flags[i] = base.ggml_backend_dev_type(dev) == _GGML_BACKEND_DEVICE_TYPE_IGPU+ known[i] = Trueexcept Exception:passfor function in name_functions:@@ -89,7 +96,7 @@ def _igpu_flags_and_names(base, lib, count: int) -> tuple[list[bool], list[str]]name = raw_name.decode("utf-8", errors = "replace")names[i] = name.replace("\t", " ").replace("\r", " ").replace("\n", " ").strip()break- return flags, names+ return flags, names, knowndef main() -> int:@@ -151,12 +158,15 @@ def main() -> int:]count = lib.ggml_backend_vk_get_device_count()- igpu, names = _igpu_flags_and_names(base, lib, count)+ igpu, names, known = _igpu_flags_and_names(base, lib, count)rows = []for i in range(count):free, total = ctypes.c_size_t(0), ctypes.c_size_t(0)lib.ggml_backend_vk_get_device_memory(i, ctypes.byref(free), ctypes.byref(total))- rows.append("%d\t%d\t%d\t%d\t%s" % (i, free.value, int(igpu[i]), total.value, names[i]))+ rows.append(+ "%d\t%d\t%d\t%d\t%s\t%d"+ % (i, free.value, int(igpu[i]), total.value, names[i], int(known[i]))+ )sys.stdout.write("\n".join(rows))return 0diff --git a/studio/backend/core/inference/llama_cpp.py b/studio/backend/core/inference/llama_cpp.pyindex cd9d4193129..0aa089ccbca 100644--- a/studio/backend/core/inference/llama_cpp.py+++ b/studio/backend/core/inference/llama_cpp.py@@ -96,6 +96,10 @@ from core.inference.llama_server_args import (memory_state_satisfies_settings,fit_is_effectively_on,fit_target_margin_in,+ MANAGED_DIO_FLAGS,+ no_reserve_requires_dio,+ resolve_launch_load_mode,+ resolve_effective_load_state,resolve_effective_memory_state,scrub_denied_env,scrub_memory_env,@@ -3537,7 +3541,8 @@ def _with_gguf_load_marker(load: Callable):load_cancel_event: Optional[threading.Event] = None,):hf_repo = intent.hf_repo- with gguf_load_in_flight(hf_repo):+ # Must not leak: polling outside a load needs LIVE VRAM, never this snapshot.+ with _load_probe_memo_scope(), gguf_load_in_flight(hf_repo):if hf_repo and _hub_download_blocks_gguf_load(hf_repo,intent.hf_variant,@@ -5524,6 +5529,21 @@ def _without_subsequence(tokens: List[str], run: List[str]) -> List[str]:return list(tokens)+def _count_subsequence(tokens: List[str], run: List[str]) -> int:+ """How many non-overlapping contiguous occurrences of ``run`` are in ``tokens``."""+ if not run:+ return 0+ total = 0+ i = 0+ while i <= len(tokens) - len(run):+ if tokens[i : i + len(run)] == run:+ total += 1+ i += len(run)+ else:+ i += 1+ return total++def _subsequence_index(tokens: List[str], run: List[str], hint: int) -> int:"""Where ``run`` sits in ``tokens``, given it was appended at ``hint``.@@ -6124,8 +6144,117 @@ def _prepend_loader_dir(existing: str, lib_dir: str) -> str:_CPU_RUNTIME_OWNER_FILE = "UNSLOTH_OWNER_PID"# GPU backends the staged CPU-only runtime must not carry over. Module level: compiled once.+# Vulkan rows for the placement decision in progress. THREAD-LOCAL: system-info polling+# reaches the same probe from other threads and wants live VRAM, and a shared dict let+# such a poll seed the load's memo with minutes-old rows.+_LOAD_PROBE_STATE = threading.local()+++def _load_probe_memo_armed() -> bool:+ return getattr(_LOAD_PROBE_STATE, "armed", False)+++def _load_probe_memo_get(kind, binary):+ rows = getattr(_LOAD_PROBE_STATE, "rows", None)+ return rows.get((kind, binary)) if rows else None+++def _load_probe_memo_put(kind, binary, rows) -> None:+ if not _load_probe_memo_armed():+ return+ store = getattr(_LOAD_PROBE_STATE, "rows", None)+ if store is None:+ store = _LOAD_PROBE_STATE.rows = {}+ store[(kind, binary)] = rows+++def _arm_load_probe_memo() -> None:+ """Start memoising the launch probes for the placement decision, on this thread+ only. Keyed by kind: the placement asks two different subprocesses, the device+ enumeration and, for a Vulkan target, the discrete-versus-shared probe.++ Called at the placement work rather than at the load call, which also covers the+ Hub download: rows captured before a multi-minute download would price the fit+ against VRAM that has since been allocated. `_load_probe_memo_scope` on the load+ call owns the lifetime, so this never has to be unwound by hand.+ """+ _LOAD_PROBE_STATE.armed = True+ _LOAD_PROBE_STATE.rows = {}+++def _load_probe_memo_scope():+ """Guarantee the memo cannot outlive a load however it exits. Arming is the+ narrower `_arm_load_probe_memo`, taken around the placement decision."""+ try:+ yield+ finally:+ _LOAD_PROBE_STATE.armed = False+ _LOAD_PROBE_STATE.rows = None+++# "Probed, no usable answer", kept apart from None = "not probed yet".+_MISSING = object()++# Discrete by construction: `_weights_in_host_memory` classifies them upstream. Vulkan+# gets the shared-memory probe; SYCL/OpenCL/MUSA/CANN may be integrated, so they decline.+_SELF_EVIDENTLY_DISCRETE = frozenset({"cuda"})++# AMD needs its classifier to have RUN: the empty set means "no APU" or "no ROCm torch to+# ask", and there is no Windows ROCm wheel, so reading it as discrete gave a Strix Halo dio.+_NEEDS_ROCM_CLASSIFICATION = frozenset({"rocm", "hip"})++# The rungs narrow exactly these, so a memo keyed on the binary alone answered a narrowed+# set with the original set's verdict.+_DEVICE_VISIBILITY_ENV = (+ "CUDA_VISIBLE_DEVICES",+ "HIP_VISIBLE_DEVICES",+ "ROCR_VISIBLE_DEVICES",+ "GGML_VK_VISIBLE_DEVICES",+ "GGML_BACKEND_PATH",+ "LLAMA_ARG_DEVICE",+ "LLAMA_ARG_MAIN_GPU",+)++# A header, then one indented `<id>: <desc> (<total> MiB, <free> MiB free)` per device;+# ` (none)` or nothing on a CPU-only build. The id has no internal spaces, which excludes+# the ` Device 0: ...` lines ggml_cuda_init writes ABOVE the header.+_LISTED_DEVICE_RE = re.compile(r"^\s+(\S+):\s")+_LIST_DEVICES_HEADER = "Available devices:"+++def _parse_listed_devices(text: Optional[str]) -> Optional[list[str]]:+ """Device ids from ``--list-devices`` output, or None when it had no answer.++ Only lines BELOW the header count: backends log their own initialisation above it,+ which is noise that happens to contain colons.+ """+ if not text:+ return None+ lines = text.splitlines()+ for index, line in enumerate(lines):+ if line.strip() == _LIST_DEVICES_HEADER:+ break+ else:+ # No header: an old build or unreadable output, neither of which is evidence.+ return None+ devices = []+ for line in lines[index + 1 :]:+ if not line.strip():+ continue+ if not line[:1].isspace():+ break # dedent ends the block+ match = _LISTED_DEVICE_RE.match(line)+ if match:+ devices.append(match.group(1))+ return devices+++# Exact soname or a real versioned form (`libggml-cuda.so.1`), not a stem prefix: that also+# matched `ggml-cuda.dll.disabled`, the name left behind by DISABLING the backend._GGML_GPU_BACKEND_RE = re.compile(- r"^(?:lib)?ggml-(?:cuda|hip|vulkan|metal|sycl|opencl|musa|cann|virtgpu)"+ r"^(?:lib)?ggml-(?:cann|cuda|hip|metal|musa|opencl|sycl|virtgpu|vulkan)"+ r"(?:\.dll|\.so(?:\.\d+)*|(?:\.\d+)*\.dylib)$")@@ -6393,7 +6522,7 @@ class LlamaCppBackend:# settings route reports a stale budget instead of nagging on every save.self._vram_fraction_launched: Optional[float] = None# Budget a load has committed to but not published, covering planning ->- # Popen as _memory_launch_pending does for Model Memory placement.+ # Popen as _memory_pending_launch does for Model Memory placement.self._vram_fraction_pending: Optional[float] = None# --batch-size / --ubatch-size the last load asked for; none = defaults or extras / envself._requested_n_batch: Optional[int] = None@@ -6559,17 +6688,29 @@ class LlamaCppBackend:# load-mode). _memory_policy_active records that the launch differed# from an unmanaged one, so disabling the toggles can undo it.self._memory_state: Optional[tuple[bool, bool]] = None+ # mmap and dio share the pair above, so without this they are indistinguishable.+ self._memory_direct_io: Optional[bool] = None+ # The same chain one toggle apart; against _memory_direct_io that is "would+ # reloading change the loader".+ self._memory_dio_applicable: bool = False+ # The managed tokens, so a rung that gives up the offload can take them back out.+ self._memory_dio_flags: list[str] = []+ # Compared against by the strip, so it can never remove a hand-typed pair.+ self._memory_dio_user_tokens: list[str] = []self._memory_policy_active: bool = False+ # What still marks the launch with the managed flags removed (scrubbed env, vetoed+ # extra), so a withdrawing rung can spot a child now equal to an unmanaged one.+ self._memory_policy_extras_touched: bool = False# False when a launch is fully offloaded to a discrete GPU, where# page-locking host RAM is skipped on purpose.self._memory_mlock_applicable: bool = True# The "--load-mode none" tokens the FIT emitted, so paths that replace the# placement can take them back out. Empty for a mode the user asked for.self._fit_load_mode_flags: list[str] = []- # True between recording a launch's placement and the child being- # spawned, so a save landing in that window still has a launch to- # compare against.- self._memory_launch_pending: bool = False+ # The pair a launch in flight is committed to, None when none is. ONE attribute:+ # a separate marker and snapshot read out of step answered a save with+ # reload_required=false about a child already committed to the pre-save flags.+ self._memory_pending_launch: Optional[tuple[bool, bool]] = None# True when the resident model came from an explicit UI load rather than# the OpenAI API, so the idle unload can be scoped to API-loaded models.# Not on GgufLoadIntent: that is compared for equality to detect@@ -8391,6 +8532,438 @@ class LlamaCppBackend:backends.add(backend)return frozenset(backends)+ def _record_memory_state(+ self,+ argv,+ env = None,+ ) -> None:+ """Record the ``(mlock, reserves_ram)`` a launch really runs with, and+ whether it streams, from ONE parse. A dozen paths record a state; a+ second parse at each of them would drift from this one."""+ mlock, reserves_ram, direct_io = resolve_effective_load_state(argv, env)+ self._memory_state = (mlock, reserves_ram)+ self._memory_direct_io = direct_io++ def _drop_managed_dio(+ self,+ argv,+ reason: str,+ clear_record: bool = True,+ ):+ """Take the no-reserve DirectIO pair back out of an argv whose placement+ is no longer the confirmed full offload it was chosen for.++ ``use_mmap`` covers mmap / mmap+mlock / auto only, so under dio every+ layer llama.cpp leaves on the CPU is read into an allocated buffer+ instead of a mapping the kernel can page: the reservation the setting+ exists to avoid. Only Unsloth's own tokens; a user's --load-mode is+ theirs, exactly as with the fit's mode.++ ``clear_record`` False when the caller strips a COPY and `cmd` still+ carries the pair, which is the arm the fit's own mode already takes for+ the same reason: the arch-crash and CPU rungs respawn from `cmd`, and+ forgetting the tokens here leaves their own strip a no-op on an argv that+ still has them. The applicable flag goes either way, because it describes+ the child about to spawn, and the `_mem_policy_for_cmd` snapshot restores+ `cmd`'s value for the rung that goes back to it.+ """+ if not self._memory_dio_flags:+ self._memory_dio_applicable = False+ return argv+ # The record says a pair exists SOMEWHERE, not in THIS argv: a rung that strips a+ # copy keeps it for `cmd`, so strip only while this argv holds more than the user wrote.+ if _count_subsequence(list(argv), self._memory_dio_flags) <= _count_subsequence(+ list(getattr(self, "_memory_dio_user_tokens", []) or []), self._memory_dio_flags+ ):+ if clear_record:+ self._memory_dio_flags = []+ self._memory_dio_applicable = False+ return argv+ stripped = _without_subsequence(argv, self._memory_dio_flags)+ if clear_record:+ self._memory_dio_flags = []+ self._memory_dio_applicable = False+ # The pair may have been the only mark, and a child equal to an unmanaged one must+ # not be torn down when the toggles go off.+ self._memory_policy_active = self._memory_policy_extras_touched+ logger.info("Model Memory: dropping the managed --load-mode dio; %s", reason)+ return stripped++ @staticmethod+ def _vulkan_offload_is_discrete(binary: Optional[str], gpu_indices = None) -> bool:+ """True only when the probe ANSWERED and every device in play is discrete.++ ``_vulkan_targets_are_igpus`` folds "probe failed" into "not an iGPU",+ which is the safe direction for its own caller (that one only skips a+ page-lock) and the wrong one here: an iGPU's VRAM is system RAM, so+ confirming an offload we could not read hands DirectIO to weights that+ are really host-backed. Absence of an answer declines.+ """+ try:+ rows = LlamaCppBackend._run_vulkan_probe(binary)+ except Exception:+ return False+ if not rows:+ return False+ wanted = set(gpu_indices) if gpu_indices else None+ selected = [r for r in rows if wanted is None or r["index"] in wanted]+ if not selected:+ return False+ # EVERY ordinal, not merely some: a partial intersection confirmed an unclassified one.+ if wanted is not None and wanted - {r["index"] for r in selected}:+ return False+ # The probe reports all-False when the type query fails, so "not integrated" and+ # "unread" share a value. Right for the page-lock caller, wrong here.+ if not all(r.get("type_known") for r in selected):+ return False+ return not any(r["is_igpu"] for r in selected)++ @classmethod+ def _gpu_offload_confirmed(+ cls,+ binary: Optional[str],+ env: Optional[Mapping[str, str]],+ gpu_indices,+ host_resident: bool,+ dio_possible: bool,+ extra_args = None,+ ) -> bool:+ """Whether this launch really puts the weights on a discrete GPU.++ The one predicate behind managed DirectIO. Confirming wrongly is worse than+ not acting: DirectIO over host-resident weights replaces a pageable mapping+ with a model-sized allocated buffer. Two questions, and anything unanswered+ declines:++ 1. Will these devices exist for the child? ``--list-devices`` is the loader's+ own verdict, so a missing runtime, a disabled plugin, a wrong-vendor plugin+ and a ``GGML_BACKEND_PATH`` plugin all resolve without reading a filename.+ 2. Is the device discrete? ggml does not report it. Vulkan has the separate+ probe, CUDA and ROCm are classified upstream by `_weights_in_host_memory`,+ and a SYCL or OpenCL id can be an integrated GPU nothing here recognises.++ ``dio_possible`` is the platform-and-build gate, NOT the toggle: off Windows+ no answer can reach a flag, so the probes are not worth spawning. Gating on+ the toggle would be wrong, because the verdict is recorded as this launch's+ placement and compared against a later save.+ """+ if not dio_possible or host_resident:+ return False+ devices = cls._enumerated_gpu_devices(binary, env)+ # A pass-through `--device` is appended last and decides placement, so the+ # auto-selected ordinals are not what runs. Its values are ggml ids.+ override = cls._effective_device_ids(extra_args, env)+ if override is not None:+ if any(d.lower() in _CPU_DEVICE_VALUES for d in override):+ return False+ listed = {d.lower() for d in (devices or [])}+ if not override or not {d.lower() for d in override} <= listed:+ return False+ gpu_indices = [+ cls._device_ordinal(d) for d in override if cls._device_ordinal(d) is not None+ ]+ devices = [d for d in (devices or []) if d.lower() in {o.lower() for o in override}]+ if not cls._offload_devices_are_live(devices, gpu_indices):+ return False+ # EVERY selected device: returning the first kind's verdict confirmed a mixed set+ # that still had weights on an integrated GPU.+ selected = cls._selected_devices(devices, gpu_indices)+ backends = {cls._device_backend(d) for d in selected}+ if backends - {"vulkan"} - _SELF_EVIDENTLY_DISCRETE - _NEEDS_ROCM_CLASSIFICATION:+ return False+ # An AMD id only counts once something has actually looked at the device.+ if (backends & _NEEDS_ROCM_CLASSIFICATION) and not cls._rocm_classification_answered():+ return False+ if "vulkan" in backends:+ # Only the VULKAN ordinals: a CUDA target's 0 otherwise satisfies Vulkan row 0.+ vulkan_ordinals = [+ cls._device_ordinal(d) for d in selected if cls._device_backend(d) == "vulkan"+ ]+ if any(o is None for o in vulkan_ordinals):+ return False+ return cls._vulkan_offload_is_discrete(binary, vulkan_ordinals)+ return True++ @staticmethod+ def _device_ordinal(device: str) -> Optional[int]:+ """The trailing ordinal of a ggml device id, or None."""+ match = re.search(r"(\d+)$", device)+ return int(match.group(1)) if match else None++ @staticmethod+ def _compact_ordinals(+ gpu_indices,+ env: Optional[Mapping[str, str]],+ is_vulkan = False,+ ):+ """Physical ordinals translated into the child's COMPACT space.++ A mask reindexes survivors from 0, so the adapter this launch calls physical 1+ is ``CUDA0`` to the child, and comparing the two rejected a narrowed launch that+ was fully offloaded. The mask lists survivors in order, so a physical id's+ position in it IS its compact ordinal. Unchanged with no mask, or one we cannot+ map.+ """+ if not gpu_indices or not env or is_vulkan:+ return gpu_indices+ # GGML_VK_VISIBLE_DEVICES is deliberately absent: ggml passes it through, so a+ # Vulkan ordinal is ALREADY compact and translating it again picked another row.+ for name in ("HIP_VISIBLE_DEVICES", "ROCR_VISIBLE_DEVICES", "CUDA_VISIBLE_DEVICES"):+ raw = env.get(name)+ if not raw or not str(raw).strip():+ continue+ try:+ order = [int(part.strip()) for part in str(raw).split(",") if part.strip()]+ except ValueError:+ return gpu_indices+ if not order:+ return gpu_indices+ try:+ mapped = [order.index(int(i)) for i in gpu_indices if int(i) in order]+ except (TypeError, ValueError):+ return gpu_indices+ return mapped or gpu_indices+ return gpu_indices++ @staticmethod+ def _physical_ordinals(+ compact,+ env: Optional[Mapping[str, str]],+ is_vulkan = False,+ ):+ """The inverse of `_compact_ordinals`: compact ordinals back to physical ids.++ `--device` names compact ids, but `_amd_apu_wants_unified_memory` is documented+ to take PHYSICAL ones, so a selection that arrives compact has to be mapped+ back before residency can classify it.+ """+ if not compact or not env or is_vulkan:+ return compact+ for name in ("HIP_VISIBLE_DEVICES", "ROCR_VISIBLE_DEVICES", "CUDA_VISIBLE_DEVICES"):+ raw = env.get(name)+ if not raw or not str(raw).strip():+ continue+ try:+ order = [int(part.strip()) for part in str(raw).split(",") if part.strip()]+ except ValueError:+ return compact+ if not order:+ return compact+ try:+ mapped = [order[int(i)] for i in compact if 0 <= int(i) < len(order)]+ except (TypeError, ValueError):+ return compact+ return mapped or compact+ return compact++ @classmethod+ def _effective_main_gpu(cls, extra_args, env) -> Optional[int]:+ """The sole device a ``--split-mode none`` launch puts every weight on.++ llama.cpp keeps only ``devices[main_gpu]``, and ``--main-gpu`` can name a device+ OUTSIDE the automatic plan, so it replaces the selection rather than narrowing+ it. Missing that confirmed a discrete card while the child put the weights on a+ shared-memory iGPU.+ """+ if not _split_mode_confines_to_one_device(extra_args, env):+ return None+ value = _extra_args_device(extra_args, {"--main-gpu", "-mg"})+ if value is None and env:+ value = env.get("LLAMA_ARG_MAIN_GPU")+ if value is None:+ return None+ try:+ return int(str(value).strip())+ except (TypeError, ValueError):+ return None++ @classmethod+ def _effective_gpu_selection(+ cls,+ binary,+ env,+ gpu_indices,+ extra_args,+ dio_possible = True,+ is_vulkan = False,+ ):+ """The ordinals the child will really use.++ A pass-through ``--device`` is appended last, so it beats the auto selection.+ Resolved once and handed to every consumer, because the host-residency verdict+ and the confirmation must be about the SAME devices: computed apart, an override+ onto a unified-memory APU was priced against the discrete card it replaced.++ Returns ``(physical, compact)``, because the consumers need the SAME selection+ in DIFFERENT spaces and collapsing them broke one or the other:+ `_amd_apu_wants_unified_memory` is documented to take physical ids, while+ `--list-devices` reports the child's compact ordinals.++ Unchanged when there is no override, when it names no GPU, or when it names a+ device the build never enumerated; the confirmation declines those on their own+ terms, and answering them here would hide the reason.+ """+ # Gated like the confirmation: with no dio decision to follow, the enumeration+ # cannot change an outcome and is not worth a subprocess.+ if not dio_possible:+ return gpu_indices, gpu_indices+ override = cls._effective_device_ids(extra_args, env)+ if override and not any(d.lower() in _CPU_DEVICE_VALUES for d in override):+ listed = {d.lower() for d in (cls._enumerated_gpu_devices(binary, env) or [])}+ if {d.lower() for d in override} <= listed:+ # `--device` names compact ids, so physical is the derived form.+ ordinals = [cls._device_ordinal(d) for d in override]+ compact = [o for o in ordinals if o is not None]+ if not compact:+ return gpu_indices, cls._compact_ordinals(gpu_indices, env, is_vulkan)+ return cls._physical_ordinals(compact, env, is_vulkan), compact+ return gpu_indices, cls._compact_ordinals(gpu_indices, env, is_vulkan)+ # Otherwise PHYSICAL, and `--main-gpu` under `-sm none` replaces the selection.+ main_gpu = cls._effective_main_gpu(extra_args, env)+ if main_gpu is not None:+ # `--main-gpu` indexes the FILTERED list, so it too arrives compact.+ return cls._physical_ordinals([main_gpu], env, is_vulkan), [main_gpu]+ return gpu_indices, cls._compact_ordinals(gpu_indices, env, is_vulkan)++ @classmethod+ def _effective_device_ids(cls, extra_args, env) -> Optional[list[str]]:+ """The explicit ``--device`` selection, or None when there is none.++ Same precedence llama.cpp uses and `_device_selection_is_cpu` already mirrors:+ the last argv value, then ``LLAMA_ARG_DEVICE``. The values are ggml device ids,+ the same namespace ``--list-devices`` prints, so they compare directly.+ """+ value = _extra_args_main_device(extra_args)+ if value is None and env:+ value = env.get("LLAMA_ARG_DEVICE")+ if value is None:+ return None+ return [d.strip() for d in str(value).split(",") if d.strip()]++ @staticmethod+ def _selected_devices(devices: Optional[list[str]], gpu_indices) -> list[str]:+ """The enumerated GPU devices this launch will use. ggml ids are+ ``<Backend><ordinal>``; the CPU device is not a placement target, and with no+ pin every GPU device is in play."""+ if not devices:+ return []+ gpu = [d for d in devices if not d.upper().startswith("CPU")]+ if not gpu_indices:+ return gpu+ wanted = {int(i) for i in gpu_indices}+ picked = []+ for device in gpu:+ match = re.search(r"(\d+)$", device)+ if match and int(match.group(1)) in wanted:+ picked.append(device)+ return picked++ @classmethod+ def _offload_devices_are_live(cls, devices: Optional[list[str]], gpu_indices) -> bool:+ """Whether every device this launch pins was enumerated by the build.++ Replaces "do the host's GPUs exist": a device the host reports but the child+ cannot open is the case that confirmed an offload which never happened.++ By ORDINAL COVERAGE, not device count: a multi-backend build can enumerate the+ same ordinal twice (``CUDA0`` and ``Vulkan0``), and counting ids rejected a+ pinned launch whose devices were all live.+ """+ selected = cls._selected_devices(devices, gpu_indices)+ if not selected:+ return False+ if not gpu_indices:+ return True+ covered = {cls._device_ordinal(d) for d in selected}+ return {int(i) for i in gpu_indices} <= covered++ @staticmethod+ def _device_backend(device: str) -> str:+ """The ggml backend behind a device id, lowercased. What `_is_vulkan_backend`+ and the plugin-root scan approximated from filenames: the build reports what it+ actually loaded, so an external ``GGML_BACKEND_PATH`` plugin names itself too."""+ return re.sub(r"\d+$", "", device).strip().lower()++ @classmethod+ def _devices_are_vulkan(cls, devices: Optional[list[str]], gpu_indices) -> bool:+ """Whether any device in play is a Vulkan one, read off the build's own ids."""+ return any(+ cls._device_backend(d) == "vulkan" for d in cls._selected_devices(devices, gpu_indices)+ )++ @classmethod+ def _enumerated_gpu_devices(+ cls,+ binary: Optional[str] = None,+ env: Optional[Mapping[str, str]] = None,+ ) -> Optional[list[str]]:+ """The ggml device ids ``--list-devices`` reports, or None for no usable answer.++ Tri-state like ``sd_cpp_accelerator_device_verdict``: ``[]`` is the build+ saying it has no devices, None is an older build rejecting the flag, a timeout+ or a crash. Both decline, but empty is a real answer and is not folded in.+ """+ binary = binary or cls._find_llama_server_binary()+ if not binary:+ return None+ key = (binary, cls._device_visibility_key(env))+ memo = _load_probe_memo_get("devices", key)+ if memo is not None:+ return None if memo is _MISSING else memo+ devices = cls._run_list_devices(binary, env)+ _load_probe_memo_put("devices", key, _MISSING if devices is None else devices)+ return devices++ @staticmethod+ def _device_visibility_key(env: Optional[Mapping[str, str]]):+ """What this environment lets the child see, as a memo key. A rung that masks+ an adapter has a different answer coming, and without this a narrowing onto a+ usable discrete GPU kept the original set's failure."""+ source = os.environ if env is None else env+ return tuple((name, source.get(name)) for name in _DEVICE_VISIBILITY_ENV)++ @staticmethod+ def _run_list_devices(+ binary: str, env: Optional[Mapping[str, str]] = None+ ) -> Optional[list[str]]:+ """``--list-devices`` once, parsed. The flag prints and exits, so this leaves no+ server behind; it is on the argv denylist for that reason, which does not apply+ to Unsloth calling it deliberately."""+ try:+ probe_env = LlamaCppBackend._llama_server_env_for_binary(binary)+ except Exception:+ probe_env = child_env_without_native_path_secret()+ # The probe must see what the CHILD sees: on Windows only+ # `_llama_server_env_for_binary` puts the managed venv's CUDA runtime on PATH, so+ # the caller's REMOVALS are replayed rather than its whole mapping.+ if env is not None:+ # BOTH directions: removals alone dropped a narrowing rung's new mask.+ for name in _DEVICE_VISIBILITY_ENV:+ if name in env:+ probe_env[name] = env[name]+ else:+ probe_env.pop(name, None)+ for name in list(probe_env):+ if name not in env and name in os.environ:+ probe_env.pop(name, None)+ try:+ result = subprocess.run(+ [binary, "--list-devices"],+ capture_output = True,+ encoding = "utf-8",+ errors = "replace",+ timeout = 30,+ env = utf8_child_env(probe_env),+ **_windows_hidden_subprocess_kwargs(),+ )+ except Exception as e:+ logger.debug(f"llama-server --list-devices failed: {e}")+ return None+ if result.returncode != 0:+ logger.debug(f"llama-server --list-devices exited {result.returncode}")+ return None+ return _parse_listed_devices(result.stdout)+@staticmethoddef _is_vulkan_backend(binary: Optional[str] = None) -> bool:"""True if the installed llama.cpp build is Vulkan-only.@@ -8574,6 +9147,19 @@ class LlamaCppBackend:)return tuple(int(i) for i in order)+ @staticmethod+ def _vulkan_probe_answered(binary: Optional[str]) -> bool:+ """Whether the Vulkan device probe produced rows at all.++ `_vulkan_targets_are_igpus` folds "no answer" into "not integrated", which is+ right for the page-lock question it was written for and not something the+ bookkeeping can read as a measurement.+ """+ try:+ return bool(LlamaCppBackend._run_vulkan_probe(binary))+ except Exception:+ return False+@staticmethoddef _vulkan_targets_are_igpus(binary: Optional[str], gpu_indices = None) -> bool:"""True when any Vulkan device in play is integrated.@@ -8767,6 +9353,41 @@ class LlamaCppBackend:except Exception:return False+ @staticmethod+ def _rocm_classification_answered() -> bool:+ """Whether `_rocm_unified_memory_gpu_ids` could actually look at the devices.++ Kept separate from it because the two questions have opposite safe answers:+ skipping a page-lock may treat "no APU" and "could not tell" alike, choosing+ a loader may not. DirectIO over a unified-memory APU replaces a pageable+ mapping with a model-sized allocated buffer.++ Answering it means reading every device, not finding the machinery to read+ them with, so this repeats the classify call rather than trusting a count.+ """+ try:+ import torch++ if not LlamaCppBackend._torch_is_rocm(torch):+ return False+ if not (hasattr(torch, "cuda") and torch.cuda.is_available()):+ return False+ from core.training.worker import _rocm_classify_unified_memory++ count = torch.cuda.device_count()+ if count <= 0:+ return False+ # Actually classify them. An enumerable device is not a classified one: the+ # loop below skips whatever it cannot read with `except: continue`, so that+ # device is simply absent from the unified set, which reads as discrete.+ # Every VISIBLE device, not just the selected ones, because mapping a ggml+ # ordinal onto a torch one is a translation this does not own.+ for ordinal in range(count):+ _rocm_classify_unified_memory(torch.cuda.get_device_properties(ordinal))+ return True+ except Exception:+ return False+@staticmethoddef _rocm_unified_memory_gpu_ids() -> set[int]:"""PHYSICAL ids of visible ROCm GPUs whose "VRAM" is shared system RAM.@@ -10870,8 +11491,14 @@ class LlamaCppBackend:binary = binary or LlamaCppBackend._find_llama_server_binary()if not binary:return []+ # Memoised per load: a 15s subprocess now asked for from several places. Cleared+ # by _load_probe_memo_scope, so a device change between loads is never cached.+ _memo = _load_probe_memo_get("vulkan", binary)+ if _memo is not None:+ return _memobinary_dir = _llama_lib_dir(binary)if not _lib_dir_has_ggml_backend(binary_dir, "vulkan"):+ _load_probe_memo_put("vulkan", binary, [])return []env = child_env_without_native_path_secret()@@ -10902,16 +11529,20 @@ class LlamaCppBackend:logger.debug(f"vulkan GPU probe exited {result.returncode}: {result.stderr.strip()}")+ _load_probe_memo_put("vulkan", binary, [])return []except Exception as e:logger.debug(f"vulkan GPU probe failed: {e}")+ # Memoised too: an unanswered probe comes straight back here, so an uncached+ # timeout is paid twice over and again per rung.+ _load_probe_memo_put("vulkan", binary, [])return []rows: list[dict] = []for line in result.stdout.strip().splitlines():parts = line.split("\t")# 4 columns from an older probe (no name); 5 with the name column.- if len(parts) not in (4, 5):+ if len(parts) not in (4, 5, 6):continuetry:rows.append(@@ -10920,12 +11551,15 @@ class LlamaCppBackend:"free_mib": int(parts[1]) // (1024 * 1024),"is_igpu": parts[2] == "1","total_mib": int(parts[3]) // (1024 * 1024),- "name": parts[4].strip() if len(parts) == 5 else "",+ "name": parts[4].strip() if len(parts) >= 5 else "",+ # Absent on an older staged probe, so it reads unknown and declines.+ "type_known": len(parts) >= 6 and parts[5].strip() == "1",})except ValueError:continuerows.sort(key = lambda r: r["index"])+ _load_probe_memo_put("vulkan", binary, rows)return rows@staticmethod@@ -12809,10 +13443,8 @@ class LlamaCppBackend:# Warn against the FULL search path, inherited entries included: a hand-installed CUDA# toolkit puts cudart64_*.dll on PATH without the venv or CUDA_PATH knowing, and warning# on the prepended directories alone told working custom setups to repair a fine install.- LlamaCppBackend._warn_missing_windows_cuda_runtime(- binary_dir,- path_dirs + [d for d in existing_path.split(";") if d],- )+ _full_search_path = path_dirs + [d for d in existing_path.split(";") if d]+ LlamaCppBackend._warn_missing_windows_cuda_runtime(binary_dir, _full_search_path)# ROCm: the prebuilt bundles rocblas.dll but NOT the Tensile# kernel files (rocblas/library/*.dat + *.hsaco); the DLL searches@@ -15339,10 +15971,15 @@ class LlamaCppBackend:self._swa_full = Falseself._kv_cache_unified = Falseself._memory_state = None+ self._memory_direct_io = None+ self._memory_dio_applicable = False+ self._memory_dio_flags = []+ self._memory_dio_user_tokens = []self._memory_policy_active = False+ self._memory_policy_extras_touched = Falseself._memory_mlock_applicable = Trueself._fit_load_mode_flags = []- self._memory_launch_pending = False+ self._memory_pending_launch = Noneself._n_ubatch = self._DEFAULT_N_UBATCHself._requested_n_batch = Noneself._requested_n_ubatch = None@@ -19235,6 +19872,10 @@ class LlamaCppBackend:"Load mode: dropping the fit's --load-mode none for the CPU ""fallback; it runs entirely from host RAM.")+ # And the managed pair: this replay appends "--device none".+ replay = self._drop_managed_dio(+ replay, "the CPU fallback runs entirely from host RAM", clear_record = False+ )# A user's own "--load-mode none" / "--no-mmap" survives that strip, by design,# and on this rung it is no longer the mode they were priced for: the replay# appends "--gpu-layers 0 --fit off --device none", so nothing credits VRAM and@@ -19550,6 +20191,9 @@ class LlamaCppBackend:finally:self._vram_fraction_pending = Noneself._binary_revision_pending = None+ # Same for the placement marker: released at the end of the call instead, a+ # finished load could blank the marker of a queued one that had published.+ self._memory_pending_launch = None@_with_gguf_load_markerdef load_model(@@ -24004,7 +24648,7 @@ class LlamaCppBackend:# GPU with full offload it would hold a second copy of the model# in system RAM and do nothing for VRAM, so it is not emitted# and the idle-unload veto carries residency by itself.- from utils.model_memory_settings import should_mlock+ from utils.model_memory_settings import capture_model_memory_settings# fully_gpu_offloaded is only set by the auto branch. Manual mode# and a user -ngl reach the same placement by their own routes,@@ -24032,16 +24676,50 @@ class LlamaCppBackend:if gpu_ids is not None:_mem_extra_args = self._strip_device_extra_args(extra_args)self._clear_device_placement_env(_mem_env)+ # ONE read for every decision below, published as one act: the window a+ # save must not fall through opens at the CAPTURE, not the spawn. It+ # carries the pair because `_memory_state` is None until the flags+ # resolve, which the comparator reads as "not governed".+ _mem_settings = capture_model_memory_settings(+ lambda pair: setattr(self, "_memory_pending_launch", pair)+ )+ _mem_keep_resident, _mem_no_reserve = _mem_settings+ _mem_should_mlock = _mem_keep_resident and not _mem_no_reserve+ # Armed HERE, not at the load call: that also spans the Hub download, and+ # rows from before it price the fit against VRAM since allocated.+ _arm_load_probe_memo()+ _mem_dio_possible = no_reserve_requires_dio(+ supports_load_mode = bool(server_caps.get("supports_load_mode")),+ gpu_offload_confirmed = True,+ )+ # NOT gated on the toggle: the answer is this launch's recorded placement+ # and a LATER save is compared against it, so gating made the record mean+ # "we did not look". Costs one subprocess, on Windows Vulkan builds only.+ _mem_probe_for_dio = _mem_dio_possible+ # The ordinals the CHILD will use, resolved once for every consumer: a+ # pass-through `--device` wins, so an override onto a unified-memory APU+ # was otherwise priced against the discrete card it replaced. Physical for+ # the classifier, compact for the build's device list. One selection.+ _mem_physical_indices, _mem_compact_indices = self._effective_gpu_selection(+ binary,+ _mem_env,+ gpu_indices,+ _mem_extra_args,+ _mem_dio_possible,+ is_vulkan_backend,+ )_mem_host_resident = self._weights_in_host_memory(fully_gpu_offloaded = fully_gpu_offloaded,gpu_memory_mode = gpu_memory_mode,gpu_layers = gpu_layers,extra_args = _mem_extra_args,- gpu_indices = gpu_indices,+ gpu_indices = _mem_physical_indices,is_vulkan_backend = is_vulkan_backend,binary = binary,env = _mem_env,- probe_vulkan = should_mlock(),+ # _mem_should_mlock is always False under no-reserve, so gating on it+ # alone made the DirectIO branch unreachable on the Vulkan build.+ probe_vulkan = _mem_should_mlock or _mem_probe_for_dio,# Over the built cmd AND the extras, so Unsloth's own --fit# counts and a later user --fit still wins by last-arg.fit_active = fit_is_effectively_on([*cmd, *(_mem_extra_args or [])], _mem_env),@@ -24053,10 +24731,38 @@ class LlamaCppBackend:# relaunch undoes.if _arch_gate_forced_cpu:_mem_host_resident = True+ # Moved up: the managed DirectIO defers to an inherited mode and runs first.+ _fit_load_mode_env_view = dict(_mem_env)+ scrub_memory_env(_fit_load_mode_env_view, _mem_settings)+ # The view as it would be under no-reserve: the scrub is what the SETTINGS+ # do, so asking against the live view answered for the wrong toggle.+ _mem_env_view_no_reserve = dict(_mem_env)+ scrub_memory_env(_mem_env_view_no_reserve, (_mem_keep_resident, True))+ # POSITIVE, not "not host-resident": that predicate only gates skipping a+ # page-lock, so it errs True for an unprobed device.+ # NOT gated on where the projector lands: mtmd_context_params has no+ # use_mmap or load_mode, and clip.cpp reads the mmproj through its own+ # ifstream, so gating on it only withheld dio from the weights that respond.+ _mem_gpu_offload_confirmed = self._gpu_offload_confirmed(+ binary,+ _mem_env,+ _mem_compact_indices,+ _mem_host_resident,+ _mem_dio_possible,+ _mem_extra_args,+ )_mem_managed, _mem_extras = apply_model_memory_policy(extra_args,supports_load_mode = bool(server_caps.get("supports_load_mode")),weights_in_host_memory = _mem_host_resident,+ gpu_offload_confirmed = _mem_gpu_offload_confirmed,+ env = _fit_load_mode_env_view,+ settings = _mem_settings,+ )+ # What the user wrote, so the strip can tell their pair from ours.+ self._memory_dio_user_tokens = list(_mem_extras or [])+ self._memory_dio_flags = (+ list(_mem_managed) if tuple(_mem_managed) == MANAGED_DIO_FLAGS else [])# After Model Memory and on the extras it returned, because it# defers to those settings: while either one owns host placement@@ -24073,8 +24779,6 @@ class LlamaCppBackend:# beat one silently. Only the FIT's mode stands aside, since a per-model# pick is a choice made for THIS model. Asked AFTER the same scrub the# child gets, so a var a Model Memory toggle drops vetoes nothing.- _fit_load_mode_env_view = dict(_mem_env)- scrub_memory_env(_fit_load_mode_env_view)if (_fit_load_modeand not load_mode@@ -24106,7 +24810,119 @@ class LlamaCppBackend:supports_load_mode = bool(server_caps.get("supports_load_mode")),weights_in_host_memory = _mem_host_resident,requested_load_mode = _resolved_load_mode,+ settings = _mem_settings,+ )++ # Whether a relaunch under no-reserve would really stream. Placement alone+ # is not enough: a per-model mmap, an extra or an inherited choice lands+ # after the managed pair and wins by last-arg, so it demanded a reload no+ # relaunch could satisfy. Toggle-independent, so a LATER save is compared+ # against this launch instead of reading as satisfied.+ def _ask(pair, env_view):+ return resolve_launch_load_mode(+ extra_args,+ supports_load_mode = bool(server_caps.get("supports_load_mode")),+ weights_in_host_memory = _mem_host_resident,+ gpu_offload_confirmed = _mem_gpu_offload_confirmed,+ requested_load_mode = _resolved_load_mode,+ env = env_view,+ settings = pair,+ )++ # Would a relaunch ADD the pair. The POLICY emitting it is the question,+ # not the effective mode: a user's own "dio" already streams, and a reload+ # to reach the mode the child is in is a notice that never clears.+ _hypo_emitted, _hypo_effective = _ask(+ (_mem_keep_resident, True), _mem_env_view_no_reserve)+ self._memory_dio_applicable = _hypo_emitted and _hypo_effective++ # The child with the policy off entirely, which is what activity asks.+ _off_view = dict(_mem_env)+ scrub_memory_env(_off_view, (False, False))++ def _mem_env_for(child_env):+ """`_mem_env`'s placement scrubbing with the CHILD's current+ visibility. `_mem_env` predates the gate, so probing with it reused+ the unnarrowed set's verdict."""+ if child_env is None:+ return _mem_env+ view = dict(_mem_env)+ for _name in _DEVICE_VISIBILITY_ENV:+ if _name in child_env:+ view[_name] = child_env[_name]+ else:+ view.pop(_name, None)+ return view++ def _dio_decision_for(+ devices,+ *,+ fully_offloaded,+ child_env = None,+ ):+ """``(pair, applicable, active)`` for a CHANGED device set.++ Everything the launch above asks, asked again for the devices a+ rung has just narrowed to: the host-residency verdict, the+ backend and probe confirmation, the live and forced-on answers,+ and the toggles-off comparison that decides activity. The rungs+ used to ask a subset each, and every missing piece became its own+ review round: a partial offload getting the pair, a narrowed set+ never gaining it, a redundant pair recorded as activity.+ """+ _rung_env = _mem_env_for(child_env)+ _rung_physical, _rung_compact = self._effective_gpu_selection(+ binary,+ _rung_env,+ devices,+ _mem_extra_args,+ _mem_dio_possible,+ is_vulkan_backend,+ )+ host_resident = self._weights_in_host_memory(+ fully_gpu_offloaded = fully_offloaded,+ gpu_memory_mode = gpu_memory_mode,+ gpu_layers = gpu_layers,+ extra_args = _mem_extra_args,+ gpu_indices = _rung_physical,+ is_vulkan_backend = is_vulkan_backend,+ binary = binary,+ env = _rung_env,+ probe_vulkan = _mem_probe_for_dio or _mem_should_mlock,+ fit_active = fit_is_effectively_on(+ [*cmd, *(_mem_extra_args or [])], _rung_env+ ),+ )+ confirmed = self._gpu_offload_confirmed(+ binary,+ _rung_env,+ _rung_compact,+ host_resident,+ _mem_dio_possible,+ _mem_extra_args,+ )++ def _for(pair, env_view):+ return resolve_launch_load_mode(+ extra_args,+ supports_load_mode = bool(server_caps.get("supports_load_mode")),+ weights_in_host_memory = host_resident,+ gpu_offload_confirmed = confirmed,+ requested_load_mode = _resolved_load_mode,+ env = env_view,+ settings = pair,+ )++ live_emitted, live_effective = _for(_mem_settings, _fit_load_mode_env_view)+ hypo_emitted, hypo_effective = _for(+ (_mem_keep_resident, True), _mem_env_view_no_reserve+ )+ pair = list(MANAGED_DIO_FLAGS) if (live_emitted and live_effective) else []+ # Redundant with the user's own loader: nothing a relaunch could undo.+ active = bool(pair) and live_effective != _for((False, False), _off_view)[1]+ return pair, (hypo_emitted and hypo_effective), active+# Only when the FIT chose it: a user's own pick survives every fallback# below, but a conclusion about a placement has to go when that# placement does.@@ -24115,8 +24931,16 @@ class LlamaCppBackend:)# Remembered so the reload hint and the duplicate-load comparator do# not demand an mlock this launch deliberately skipped.- self._memory_mlock_applicable = _mem_host_resident- if should_mlock() and not _mem_host_resident:+ # A probe forced for the DirectIO decision must not make the page-lock+ # bookkeeping LESS conservative: an unreadable Vulkan probe answers "not an+ # iGPU", which recorded the lock inapplicable on maybe-host-backed weights.+ self._memory_mlock_applicable = _mem_host_resident or (+ _mem_probe_for_dio+ and not _mem_should_mlock+ and is_vulkan_backend+ and not self._vulkan_probe_answered(binary)+ )+ if _mem_should_mlock and not _mem_host_resident:logger.info("Model Memory: skipping page-lock, the weights are fully ""offloaded to a discrete GPU; residency is kept by not "@@ -24125,11 +24949,17 @@ class LlamaCppBackend:if _mem_managed:cmd.extend(_mem_managed)logger.info(- "Model Memory: keeping weights pinned in place (%s)",+ "Model Memory: applying loading policy (%s)"," ".join(_mem_managed),)if _load_mode_managed:cmd.extend(_load_mode_managed)+ # Protected like the extras: the per-model selection is the user's too,+ # so a rung that stripped a COPY left it behind for the next strip.+ self._memory_dio_user_tokens = [+ *self._memory_dio_user_tokens,+ *_load_mode_managed,+ ]logger.info("Load mode: %s", " ".join(_load_mode_managed))# User pass-through args go last. Placement flags are removed@@ -24220,7 +25050,9 @@ class LlamaCppBackend:# llama.cpp reads LLAMA_ARG_MLOCK / _MMAP / _LOAD_MODE before argv,# so stripping the tokens alone would leave an inherited value in# force. Only when a toggle is on, so existing setups are untouched.- _mem_scrubbed = scrub_memory_env(env)+ # _mem_settings, not the live toggles: a save landing in between scrubbed+ # the env under the new pair while the argv came from the old one.+ _mem_scrubbed = scrub_memory_env(env, _mem_settings)if _mem_scrubbed:logger.info("Model Memory owns placement; dropped inherited %s",@@ -24244,7 +25076,7 @@ class LlamaCppBackend:# mmap/mmap+mlock/auto), so leaving it out records the launch as# non-reserving and a later "Don't reserve system RAM" is judged already# satisfied, keeping the reservation instead of relaunching with mmap.- self._memory_state = resolve_effective_memory_state(+ self._record_memory_state(list(_mem_managed) + list(_load_mode_managed) + list(_mem_extras), env)# Did the policy change this launch at all: emitted a flag,@@ -24256,17 +25088,43 @@ class LlamaCppBackend:_mem_policy_touched_extras = bool(_mem_scrubbed) or _mem_extras != list(extra_args or [])- self._memory_policy_active = bool(_mem_managed) or _mem_policy_touched_extras+ # Activity is "would turning the toggles off change this child".+ # bool(_mem_managed) is not: a later mmap shadows the pair and a user's own+ # "dio" makes it redundant, both leaving the child equal to an unmanaged+ # one, so counting either made turning no-reserve OFF demand an inert+ # reload. The keep-resident block cannot be shadowed.+ _mem_managed_is_effective = bool(_mem_managed) and (+ tuple(_mem_managed) != MANAGED_DIO_FLAGS+ or self._memory_direct_io != _ask((False, False), _off_view)[1]+ )+ self._memory_policy_active = _mem_managed_is_effective or _mem_policy_touched_extras+ self._memory_policy_extras_touched = _mem_policy_touched_extras+# What `cmd` itself means, snapshotted before any respawn edits it.+ # The dio TOKENS, not just applicability: the fit-off retry records a pair+ # `cmd` never carried, and a fallback respawning `cmd` read that as+ # "already has it". Copied, so a later strip cannot reach back in.# _spawn_and_wait's --fit retries append a page-lock to THEIR argv# and write the policy back; the arch-crash retry (#7624) respawns# `cmd`, which never carried that lock, so it restores these.- _mem_policy_for_cmd = (- _mem_host_resident,- self._memory_state,- self._memory_policy_active,- self._memory_mlock_applicable,- )+ def _snapshot_policy_for_cmd():+ """What `cmd` means RIGHT NOW, for the arch-crash retry to restore.++ A function because every site that mutates `cmd`'s managed pair has+ to retake it, and the proactive gate did not: the retry restored a+ pre-gate snapshot over a post-gate command, appending a second pair+ after an addition or believing an absent one was still there."""+ return (+ _mem_host_resident,+ self._memory_state,+ self._memory_direct_io,+ self._memory_dio_applicable,+ list(self._memory_dio_flags),+ self._memory_policy_active,+ self._memory_mlock_applicable,+ )++ _mem_policy_for_cmd = _snapshot_policy_for_cmd()# Omitting --threads relies on llama.cpp's physical-core default, so# drop an inherited LLAMA_ARG_THREADS that would otherwise feed the# arg handler and silently force hardware_concurrency(). #5692@@ -24812,6 +25670,36 @@ class LlamaCppBackend:_child_gpu_physical_ids = tuple(int(i) for i in _survivors)# Narrower than any pin above, so it replaces it._launch_pinned_ids = list(_survivors)+ # Taken against the UNNARROWED set, where an unsupported APU+ # answered host-resident for the whole launch; this gate removes+ # it. fully_offloaded False: the fitter still owns placement.+ _gate_dio, _gate_applicable, _gate_active = _dio_decision_for(+ _survivors, fully_offloaded = False, child_env = env+ )+ self._memory_dio_applicable = _gate_applicable+ # BOTH directions: narrowing can also take the offload away, and+ # the pair then reads CPU-resident layers into allocated buffers.+ if self._memory_dio_flags and not _gate_dio:+ cmd = self._drop_managed_dio(+ cmd,+ "the arch gate's surviving GPU(s) no longer confirm a "+ "full offload",+ )+ self._memory_dio_applicable = _gate_applicable+ self._record_memory_state(cmd, env)+ _mem_policy_for_cmd = _snapshot_policy_for_cmd()+ elif _gate_dio and not self._memory_dio_flags:+ cmd = [*cmd, *_gate_dio]+ self._memory_dio_flags = list(_gate_dio)+ self._memory_policy_active = _gate_active or self._memory_policy_active+ self._record_memory_state(cmd, env)+ _mem_policy_for_cmd = _snapshot_policy_for_cmd()+ logger.info(+ "Model Memory: applying %s; the arch gate pins this "+ "launch to discrete GPU(s) %s.",+ " ".join(_gate_dio),+ _survivors,+ )# And the carve-out advice with it: upstream priced the# UNNARROWED set, which _rocm_selected_pool_mib declines on a# mixed host, so a model outgrowing the surviving APU's@@ -24928,13 +25816,8 @@ class LlamaCppBackend:# reload comparator judges this child against a mode it no longer# runs. The snapshot too: the arch-crash retry restores it over# `cmd`, which now carries the override.- self._memory_state = resolve_effective_memory_state(cmd, env)- _mem_policy_for_cmd = (- _mem_host_resident,- self._memory_state,- self._memory_policy_active,- self._memory_mlock_applicable,- )+ self._record_memory_state(cmd, env)+ _mem_policy_for_cmd = _snapshot_policy_for_cmd()self._record_load_warning(_offload_msg)# Captured before any text-only fallback strips it from cmd.@@ -24991,7 +25874,7 @@ class LlamaCppBackend:# system RAM". From the argv, not _mem_managed + _mem_extras:# this descends from _last_spawn_cmd, which carries any page-lock# the --fit on retry appended, and rebuilding would drop it.- self._memory_state = resolve_effective_memory_state(stripped, env)+ self._record_memory_state(stripped, env)logger.info("Load mode: dropping the fit's --load-mode none for the ""--flash-attn off retry; the no-flash launch has a "@@ -25104,7 +25987,9 @@ class LlamaCppBackend:self._health_wait_cancelled = Truereturn False# is_active covers it from here, so drop the pre-spawn flag.- self._memory_launch_pending = False+ # Re-armed below if this attempt does not come up: the retry rungs+ # respawn from the SAME captured settings.+ self._memory_pending_launch = None# Background thread to drain stdout (prevents pipe deadlock)self._stdout_thread = threading.Thread(@@ -25119,6 +26004,10 @@ class LlamaCppBackend:return Trueif getattr(self, "_health_wait_cancelled", False):return False+ # The window is open again: the rungs below respawn from the SAME+ # captured settings while a crashed child leaves is_active false.+ # Here rather than per rung, so a later rung cannot forget it.+ self._memory_pending_launch = _mem_settings# Read once, like the wait itself: a cleared reference is a# teardown, not a startup crash, and re-reading the# attribute per term would race the shutdown thread again.@@ -25199,7 +26088,7 @@ class LlamaCppBackend:self._llama_log_path,)run_cmd = _reverted- self._memory_state = resolve_effective_memory_state(run_cmd, env)+ self._record_memory_state(run_cmd, env)_did_fit_retry = Truecontinueif (@@ -25247,12 +26136,19 @@ class LlamaCppBackend:"for the --fit on retry; the fit it was derived ""from did not hold.")+ # And the managed pair: the offload it was chosen for did+ # not hold.+ _run = self._drop_managed_dio(+ _run,+ "the --fit on retry gives up the confirmed full offload",+ clear_record = False,+ )# The full-offload prediction that suppressed the# page-lock was the very thing that just proved# wrong, so llama.cpp may now leave weights in host# RAM. Re-arm residency for the retry (last-wins, so# appending is enough) and re-record the state.- if should_mlock() and not _mem_host_resident:+ if _mem_should_mlock and not _mem_host_resident:_run.extend(["--load-mode", "mmap+mlock"]if server_caps.get("supports_load_mode")@@ -25267,7 +26163,7 @@ class LlamaCppBackend:"prediction did not hold.")run_cmd = _run- self._memory_state = resolve_effective_memory_state(_run, env)+ self._record_memory_state(_run, env)_did_fit_retry = Truecontinueif (@@ -25292,23 +26188,22 @@ class LlamaCppBackend:# host-resident verdict can stop holding. Drop a# page-lock the retry no longer needs rather than# reserving a full host copy of a fully offloaded model.- if _mem_managed and _mem_host_resident:- if not self._weights_in_host_memory(- fully_gpu_offloaded = True,- gpu_memory_mode = gpu_memory_mode,- gpu_layers = gpu_layers,- extra_args = _mem_extra_args,- gpu_indices = gpu_indices,- is_vulkan_backend = is_vulkan_backend,- binary = binary,- env = _mem_env,- ):+ if _mem_host_resident and not self._weights_in_host_memory(+ fully_gpu_offloaded = True,+ gpu_memory_mode = gpu_memory_mode,+ gpu_layers = gpu_layers,+ extra_args = _mem_extra_args,+ gpu_indices = gpu_indices,+ is_vulkan_backend = is_vulkan_backend,+ binary = binary,+ env = _mem_env,+ ):+ _mem_host_resident = False+ # So a later save is not compared against a lock this+ # launch dropped.+ self._memory_mlock_applicable = False+ if _mem_managed:run_cmd = _without_subsequence(run_cmd, _mem_managed)- _mem_host_resident = False- # Recorded so a later "keep resident" save is not- # compared against a lock this launch dropped,- # which would demand a pointless reload.- self._memory_mlock_applicable = False# The managed flag was the policy's only mark on# this child unless it also scrubbed or stripped,# and a child equal to an unmanaged one must not@@ -25318,7 +26213,28 @@ class LlamaCppBackend:"Model Memory: dropping the page-lock for ""the --fit off retry; it offloads every layer.")- self._memory_state = resolve_effective_memory_state(run_cmd, env)++ # The other direction, which the page-lock arm cannot+ # reach: nothing was emitted BECAUSE the fitted attempt+ # read host-resident, the verdict this retry overturns.+ # Appended, so a hand-typed flag still wins last. The+ # fitter is OFF here, so -ngl is every layer.+ _retry_dio, _retry_applicable, _retry_active = _dio_decision_for(+ gpu_indices, fully_offloaded = True, child_env = env+ )+ self._memory_dio_applicable = _retry_applicable+ if _retry_dio and not self._memory_dio_flags:+ run_cmd = [*run_cmd, *_retry_dio]+ self._memory_dio_flags = list(_retry_dio)+ self._memory_policy_active = (+ _retry_active or self._memory_policy_active+ )+ logger.info(+ "Model Memory: applying %s for the --fit off "+ "retry; it offloads every layer.",+ " ".join(_retry_dio),+ )+ self._record_memory_state(run_cmd, env)_did_fit_retry = Truecontinuereturn False@@ -25440,8 +26356,10 @@ class LlamaCppBackend:# with: the record has to describe the argv that started, whichever# of the two changed it, or the reload comparator judges this child# against a mode it does not have.- if self._fit_load_mode_flags or _cpu_pageable_note:- self._memory_state = resolve_effective_memory_state(_last_spawn_cmd, env)+ #+ # Unconditional now: the replay also withdraws the managed DirectIO, a+ # third way to go stale, and all of them land in _last_spawn_cmd.+ self._record_memory_state(_last_spawn_cmd, env)intent = self._apply_cpu_fallback_state(intent,is_vision = fallback_has_mmproj,@@ -25564,9 +26482,11 @@ class LlamaCppBackend:# `cmd` before anything spawns. The arch-crash rung re-derives it# from `cmd` too, so the _mem_policy_for_cmd snapshot it restores# cannot put the stale pair back. The pageable override rewrites- # the same field, so either one having fired makes the record stale.- if self._fit_load_mode_flags or _replay_pageable_note:- self._memory_state = resolve_effective_memory_state(cmd, env)+ # the same field, as does the managed DirectIO withdrawal, so the+ # record follows the argv about to spawn. The snapshot with it:+ # this replay REPLACED `cmd`.+ self._record_memory_state(cmd, env)+ _mem_policy_for_cmd = _snapshot_policy_for_cmd()# The preflight above priced and, where needed, rewrote the argv# this replay was built FROM; an override settled here is news it# could not have carried. Appended to whatever notice is recorded,@@ -25614,13 +26534,18 @@ class LlamaCppBackend:# window sees no active backend and reports no reload while the# child is already committed. Popen clears it, and so does every# exit below, so a failed spawn cannot leave it stuck on.- self._memory_launch_pending = True+ self._memory_pending_launch = _mem_settingsself._vram_fraction_pending = _budget_priced_placement()healthy = Falsetry:healthy = _spawn_and_wait(cmd)finally:- self._memory_launch_pending = False+ # Only once a child is up, where is_active takes over. An+ # unconditional clear undid the re-arm and left the outer rungs (arch+ # gate, KV-unified, flash-attention) respawning with nothing marking+ # the window. `_serial_load_scope` releases it either way.+ if healthy:+ self._memory_pending_launch = Noneif not healthy and _finish_cancelled_health_wait("Load cancelled during the llama-server health wait"):@@ -25866,17 +26791,49 @@ class LlamaCppBackend:(_mem_host_resident,self._memory_state,+ self._memory_direct_io,+ self._memory_dio_applicable,+ self._memory_dio_flags,self._memory_policy_active,self._memory_mlock_applicable,) = _mem_policy_for_cmd+ # AFTER the restore, which would otherwise put back the+ # applicability the strip cleared. Re-asked on the narrowed set in+ # BOTH directions: narrowing onto the surviving discrete card can+ # GAIN an offload the original set never had, so guarding on an+ # existing pair left the successful retry on mmap.+ _arch_dio, _arch_applicable, _arch_active = _dio_decision_for(+ _remaining, fully_offloaded = fully_gpu_offloaded, child_env = env+ )+ self._memory_dio_applicable = _arch_applicable+ _dio_left_cmd = False+ if self._memory_dio_flags and not _arch_dio:+ _dio_left_cmd = True+ cmd = self._drop_managed_dio(+ cmd,+ "the arch-crash retry's remaining devices no longer "+ "confirm a full offload",+ )+ self._memory_dio_applicable = _arch_applicable+ elif _arch_dio and not self._memory_dio_flags:+ _dio_left_cmd = True+ cmd = [*cmd, *_arch_dio]+ self._memory_dio_flags = list(_arch_dio)+ self._memory_policy_active = _arch_active or self._memory_policy_active+ logger.info(+ "Model Memory: applying %s; the arch-crash retry's "+ "remaining GPU(s) %s confirm a full offload.",+ " ".join(_arch_dio),+ _remaining,+ )# ...except the load-mode pair just removed: the snapshot was# taken while `cmd` still carried it, so restoring it would# record a reservation this respawn no longer makes. From the# stripped argv, NOT rebuilt from _mem_managed + _mem_extras: by# here `cmd` may be a fallback argv those parts no longer add up# to, dropping whatever memory-relevant flags they added.- if _fit_mode_left_cmd:- self._memory_state = resolve_effective_memory_state(cmd, env)+ if _fit_mode_left_cmd or _dio_left_cmd:+ self._record_memory_state(cmd, env)# Residency is a property of the DEVICES, which the retry just# changed: crash on the discrete card, land on the# unified-memory APU, and the weights are host-backed after@@ -25894,7 +26851,10 @@ class LlamaCppBackend:is_vulkan_backend = is_vulkan_backend,binary = binary,env = env,- probe_vulkan = should_mlock(),+ # Widened like the other two sites: _mem_should_mlock is always+ # False under no-reserve, so alone it asked with the probe off,+ # and an unprobed device answers the conservative True.+ probe_vulkan = _mem_should_mlock or _mem_probe_for_dio,fit_active = fit_is_effectively_on([*cmd, *(_mem_extra_args or [])], env),)# Lock-ADDING direction only: `cmd` carries a policy-emitted@@ -25914,6 +26874,9 @@ class LlamaCppBackend:# copy would land after the extras and mark# _memory_policy_active for a launch it never touched.weights_in_host_memory = _retry_host_resident,+ # The launch's own pair: a save landing before this retry+ # would otherwise pick flags this launch never saw.+ settings = _mem_settings,)if _retry_managed:# After the user extras, so llama.cpp's last-wins parse@@ -25928,11 +26891,10 @@ class LlamaCppBackend:self._memory_policy_active = (bool(_retry_managed) or self._memory_policy_active)- # In argv order -- extras first, then the appended- # lock -- because the resolver is last-wins too.- self._memory_state = resolve_effective_memory_state(- list(_mem_extras) + list(_retry_managed), env- )+ # From `cmd`: it may carry the managed pair, which the parts no+ # longer add up to. Rebuilding from them recorded a mapped load+ # for a streaming child, and the comparator then looped.+ self._record_memory_state(cmd, env)logger.info("Arch-crash retry changed where the weights live; ""recomputed Model Memory (%s).",@@ -25968,7 +26930,7 @@ class LlamaCppBackend:self._amend_load_warning(_retry_pageable_note)# From the argv, like the fit-strip above: `cmd` is what# the respawn runs, and the record has to match it.- self._memory_state = resolve_effective_memory_state(cmd, env)+ self._record_memory_state(cmd, env)# And the carve-out advice with them. _begin_load_warnings()# dropped the one priced for the crashed placement, but the# respawn can land on a unified-memory APU whose allocation the@@ -26312,15 +27274,17 @@ class LlamaCppBackend:# Same as the no-flash rung: the record follows the# argv, or a stale "reserving" spends a full reload# the running server already satisfies.- self._memory_state = resolve_effective_memory_state(- _stripped_cpu_projector_cmd, env- )+ self._record_memory_state(_stripped_cpu_projector_cmd, env)logger.info("Load mode: dropping the fit's --load-mode none ""for the CPU-projector retry; it moves the ""projector into host RAM the fit credited to VRAM.")_cpu_projector_cmd = _stripped_cpu_projector_cmd+ # The managed pair stays: this retry moves the PROJECTOR, which+ # clip.cpp loads through its own ifstream, and the main model is+ # still fully offloaded. The block above is the fit's VRAM+ # accounting, which the move really does void.logger.warning("llama-server failed while loading this model's GPU ""vision projector (--mmproj); retrying with the "@@ -27365,6 +28329,9 @@ class LlamaCppBackend:self._memory_state,self._memory_policy_active,self._memory_mlock_applicable,+ self._memory_direct_io,+ self._memory_dio_applicable,+ bool(self._memory_dio_flags),):logger.info("Model Memory policy changed since launch; forcing a reload")return False@@ -27563,9 +28530,14 @@ class LlamaCppBackend:self._swa_full = Falseself._kv_cache_unified = Falseself._memory_state = None+ self._memory_direct_io = None+ self._memory_dio_applicable = False+ self._memory_dio_flags = []+ self._memory_dio_user_tokens = []self._memory_policy_active = False+ self._memory_policy_extras_touched = Falseself._memory_mlock_applicable = True- self._memory_launch_pending = False+ self._memory_pending_launch = Noneself._vram_fraction_pending = Noneself._loaded_by_user_action = Falseself._n_ubatch = self._DEFAULT_N_UBATCHdiff --git a/studio/backend/core/inference/llama_server_args.py b/studio/backend/core/inference/llama_server_args.pyindex 087f20cf95b..b5353e33247 100644--- a/studio/backend/core/inference/llama_server_args.py+++ b/studio/backend/core/inference/llama_server_args.py@@ -1121,11 +1121,81 @@ def strip_context_only(args: Optional[Iterable[str]]) -> Optional[list[str]]:)+# The managed block the policy emits, named so the launch can spot its own tokens and+# take them back out when the placement they were chosen for stops holding.+MANAGED_DIO_FLAGS: tuple[str, ...] = ("--load-mode", "dio")+++def no_reserve_requires_dio(*, supports_load_mode: bool, gpu_offload_confirmed: bool) -> bool:+ """Whether "Don't reserve system RAM" owes this launch ``--load-mode dio``.++ Windows keeps the whole GGUF mapping resident after a full offload, because+ ``unmap_fragment`` is a no-op there (#9033), so the setting only means+ anything on that platform once the offload is confirmed and the build+ understands the flag.++ Placement only. Whether a loader choice further down the chain then overrides+ the pair is not asked here: ``resolve_launch_load_mode`` answers that by+ resolving the argv the policy really produces.+ """+ return sys.platform == "win32" and supports_load_mode and gpu_offload_confirmed+++def resolve_launch_load_mode(+ extra_args: Optional[Iterable[str]],+ *,+ supports_load_mode: bool,+ weights_in_host_memory: bool,+ gpu_offload_confirmed: bool,+ requested_load_mode: Optional[str],+ env: Optional[Mapping[str, str]],+ settings: tuple[bool, bool],+) -> tuple[bool, bool]:+ """``(policy_emitted_dio, effective_dio)`` under this ``(keep_resident, no_ram_reserve)``.++ Runs the REAL policy chain and resolves the argv it produces, rather than+ assembling a hypothetical by hand. The launch calls it for each settings pair+ it needs an answer about, so those answers cannot disagree about one launch.++ Asking by hand is what went wrong before: the env view and the surviving+ extras are scrubbed and stripped BY the settings, so a hypothetical built+ from the live ones silently answered for the wrong toggle.++ The two halves are DIFFERENT questions and conflating them is the other way+ to get this wrong. ``effective_dio`` is what the child runs, which a user's+ own ``dio`` satisfies on its own. ``policy_emitted_dio`` is whether THIS+ policy put the pair there, which is what "did the policy change this launch"+ and "would a relaunch add something" actually turn on.+ """+ managed, extras = apply_model_memory_policy(+ extra_args,+ supports_load_mode = supports_load_mode,+ weights_in_host_memory = weights_in_host_memory,+ gpu_offload_confirmed = gpu_offload_confirmed,+ env = env,+ settings = settings,+ )+ selected, extras = apply_load_mode_policy(+ extras,+ supports_load_mode = supports_load_mode,+ weights_in_host_memory = weights_in_host_memory,+ requested_load_mode = requested_load_mode,+ settings = settings,+ )+ return (+ tuple(managed) == MANAGED_DIO_FLAGS,+ resolve_effective_direct_io([*managed, *selected, *extras], env),+ )++def apply_model_memory_policy(extra_args: Optional[Iterable[str]],*,supports_load_mode: bool = False,weights_in_host_memory: bool = True,+ gpu_offload_confirmed: bool = False,+ env: Optional[Mapping[str, str]] = None,+ settings: Optional[tuple[bool, bool]] = None,) -> tuple[list[str], list[str]]:"""Resolve the Model Memory settings into llama-server flags, returning ``(managed_flags,extras)``: what Unsloth emits itself, and the user's extras with any vetoed flag removed.@@ -1137,20 +1207,42 @@ def apply_model_memory_policy(idle-unload veto alone. Every other load-mode-bearing flag is stripped from the emitted extras,because a trailing one resets the whole mode and would drop the mlock.- "Don't reserve system RAM" drops ``--mlock`` / ``--no-mmap``, leaving the default mmap path.- With both off nothing is stripped, so a hand-typed flag still applies. The per-model Mmap/Mlock- control is resolved separately, by ``apply_load_mode_policy``, which runs after this and defers- to it.+ "Don't reserve system RAM" drops ``--mlock`` / ``--no-mmap``, leaving the+ default mmap path except on Windows with confirmed full GPU offload, where+ supported builds use DirectIO to avoid retaining the resident file mapping.+ With both off nothing is stripped, so a hand-typed flag still applies.++ ``gpu_offload_confirmed`` is that confirmation, and the caller has to+ establish it POSITIVELY. ``not weights_in_host_memory`` is not the same+ thing: that predicate exists to skip a page-lock, so it errs towards True+ for a device it did not probe and towards False for an offload the build+ cannot perform, and neither direction is safe for choosing a loader.++ ``env`` is the child's environment AFTER ``scrub_memory_env``. An inherited+ loader choice that survives the scrub is a non-reserving one the settings+ disclaim, and argv beats the environment in llama.cpp, so the managed+ DirectIO stands aside for it the way the fit's own mode does.++ ``settings`` is a ``(keep_resident, no_ram_reserve)`` the caller already read+ as one coherent snapshot. A launch that has to consult the toggles itself,+ to decide whether a placement answer is worth probing for, must hand that+ same pair down: re-reading here would let a save landing in between gate the+ probe on one value and the flags on another.++ The per-model Mmap/Mlock control is resolved separately, by+ ``apply_load_mode_policy``, which runs after this and defers to it."""- try:- from utils.model_memory_settings import get_model_memory_settings- except Exception:- # Settings unavailable (bare unit-test import): behave as before.- return [], list(extra_args or [])-- # One snapshot for both decisions: read separately, a save landing between them strips for one setting and locks- # for the other, so a saved --mlock could survive a committed no-reserve.- keep_resident, no_ram_reserve = get_model_memory_settings()+ if settings is None:+ try:+ from utils.model_memory_settings import get_model_memory_settings+ except Exception:+ # Settings unavailable (bare unit-test import): behave as before.+ return [], list(extra_args or [])++ # One snapshot for both decisions: read separately, a save landing between them+ # let a saved --mlock survive a committed no-reserve.+ settings = get_model_memory_settings()+ keep_resident, no_ram_reserve = settingstokens = list(extra_args or [])if no_ram_reserve:tokens = strip_shadowing_flags(@@ -1166,6 +1258,19 @@ def apply_model_memory_policy(tokens = _strip_reserving_load_modes(tokens)managed: list[str] = []+ if (+ no_ram_reserve+ and no_reserve_requires_dio(+ supports_load_mode = supports_load_mode,+ gpu_offload_confirmed = gpu_offload_confirmed,+ )+ # An inherited choice surviving the scrub is a non-reserving one the settings+ # disclaim, and argv beats the environment, so the pair stands aside for it.+ and not memory_env_selects_load_mode(env)+ ):+ # Windows cannot partially unmap the GGUF after offload (unmap_fragment is a+ # no-op), so stream for this confirmed placement. Per-model modes still win later.+ managed.extend(MANAGED_DIO_FLAGS)if keep_resident and not no_ram_reserve and weights_in_host_memory:# Before the extras, like the rest of the managed block. mmap+mlock, not bare mlock: it matches what --mlock# meant alongside the default mmap.@@ -1190,6 +1295,7 @@ def apply_load_mode_policy(supports_load_mode: bool = False,weights_in_host_memory: bool = True,requested_load_mode: Optional[str] = None,+ settings: Optional[tuple[bool, bool]] = None,) -> tuple[list[str], list[str]]:"""Resolve the per-model Mmap/Mlock control into llama-server flags, returning ``(managed_flags,extras)`` like ``apply_model_memory_policy``, and meant to run straight after it on the extras@@ -1209,12 +1315,14 @@ def apply_load_mode_policy(mode = _normalize_load_mode_value(requested_load_mode)if not mode:return [], tokens- try:- from utils.model_memory_settings import get_model_memory_settings- keep_resident, no_ram_reserve = get_model_memory_settings()- except Exception:- # Settings unavailable (bare unit-test import): nothing to defer to.- keep_resident, no_ram_reserve = False, False+ if settings is None:+ try:+ from utils.model_memory_settings import get_model_memory_settings+ settings = get_model_memory_settings()+ except Exception:+ # Settings unavailable (bare unit-test import): nothing to defer to.+ settings = (False, False)+ keep_resident, no_ram_reserve = settingsif keep_resident and not no_ram_reserve and weights_in_host_memory:logger.info("Model Memory: 'Keep model in GPU memory' owns the load mode; "@@ -1280,8 +1388,15 @@ def _strip_reserving_load_modes(tokens: list[str]) -> list[str]:return out-def model_memory_owns_placement() -> bool:- """True when either toggle is on, so the child env must be scrubbed."""+def model_memory_owns_placement(settings: Optional[tuple[bool, bool]] = None) -> bool:+ """True when either toggle is on, so the child env must be scrubbed.++ ``settings`` answers for a GIVEN ``(keep_resident, no_ram_reserve)`` instead of+ the live one, which is what lets the launch ask what it WOULD scrub under a+ setting that is not currently on.+ """+ if settings is not None:+ return settings[0] or settings[1]try:from utils.model_memory_settings import get_keep_resident, get_no_ram_reserveexcept Exception:@@ -1457,13 +1572,15 @@ def memory_env_selects_load_mode(env: Optional[Mapping[str, str]]) -> bool:return False-def scrub_memory_env(env: dict) -> list[str]:- """Drop inherited memory placement the settings override. Returns the names removed, for- logging. A no-op with both toggles off, so an existing LLAMA_ARG_MLOCK deployment keeps- working untouched. Only the values that actually lock or reserve go: an inherited- ``LLAMA_ARG_DIO=1`` is a loader choice, not a reservation, and no-reserve has no quarrel with- it."""- if not model_memory_owns_placement():+def scrub_memory_env(env: dict, settings: Optional[tuple[bool, bool]] = None) -> list[str]:+ """Drop inherited memory placement the settings override.++ Returns the names removed, for logging. A no-op with both toggles off, so an+ existing LLAMA_ARG_MLOCK deployment keeps working untouched. Only the values+ that actually lock or reserve go: an inherited ``LLAMA_ARG_DIO=1`` is a+ loader choice, not a reservation, and no-reserve has no quarrel with it.+ """+ if not model_memory_owns_placement(settings):return []removed = [name@@ -1647,40 +1764,68 @@ _LOAD_MODE_RESERVING_VALUES = frozenset({"none", "mlock"})def resolve_effective_memory_state(argv: Optional[Iterable[str]], env: Optional[Mapping[str, str]] = None) -> tuple[bool, bool]:- """``(mlock, reserves_ram)`` the child will actually run with. Mirrors llama.cpp: env supplies- defaults, argv overrides last-wins. Used to compare a running process against the current- settings, so the reload hint reflects the launched state rather than only what Unsloth- emitted."""+ """``(mlock, reserves_ram)`` the child will actually run with.++ Mirrors llama.cpp: env supplies defaults, argv overrides last-wins. Used to+ compare a running process against the current settings, so the reload hint+ reflects the launched state rather than only what Unsloth emitted.+ """+ mlock, reserves_ram, _direct_io = resolve_effective_load_state(argv, env)+ return mlock, reserves_ram+++def resolve_effective_direct_io(+ argv: Optional[Iterable[str]], env: Optional[Mapping[str, str]] = None+) -> bool:+ """Whether the child will actually run DirectIO.++ ``mmap`` and ``dio`` both hold no full host copy, so they are the same+ ``(mlock, reserves_ram)`` and the reload comparator cannot tell them apart.+ Where the Windows no-reserve policy owes a launch dio, that difference is+ the whole decision, so it is resolved from the same parse rather than a+ second one that could drift from it.+ """+ return resolve_effective_load_state(argv, env)[2]+++def resolve_effective_load_state(+ argv: Optional[Iterable[str]], env: Optional[Mapping[str, str]] = None+) -> tuple[bool, bool, bool]:+ """``(mlock, reserves_ram, direct_io)``. Every branch that resolves the mode+ assigns all three, so the DirectIO bit cannot fall out of step with the pair."""env = env or {}mlock = Falsereserves_ram = False+ direct_io = False# Each var runs the SAME handler as its flag, so it assigns the whole mode and a later one overwrites an earlier# one, in llama.cpp's registration order. Measured: LLAMA_ARG_MLOCK=1 with LLAMA_ARG_MMAP=on or LLAMA_ARG_DIO=0# leaves the child unlocked. Only the mlock bit, like the argv --mlock below: "mlock" vs "mmap+mlock" is not# observable and changes no decision.if str(env.get("LLAMA_ARG_MLOCK", "")).strip().lower() in _ENV_TRUE_VALUES:- mlock = True- # Every option with a negative form also answers to LLAMA_ARG_NO_<NAME>: upstream rewrites the name and, if that- # var EXISTS, forces the value falsey whatever it says, before reading the affirmative one. Measured:+ # Either mode it can mean maps or buffers the weights; neither streams.+ mlock, direct_io = True, False+ # Every option with a negative form also answers to LLAMA_ARG_NO_<NAME>: upstream rewrites the name and, if that var+ # EXISTS, forces the value falsey whatever it says, before reading the affirmative one. Measured:# LLAMA_ARG_NO_MMAP=0 still disables mmap, and it beats LLAMA_ARG_MMAP=on. --mlock has no negative form, so# LLAMA_ARG_NO_MLOCK does nothing. LLAMA_ARG_MMAP is whether to mmap, so "off" means mmap disabled ("none")._mmap_env = "0" if "LLAMA_ARG_NO_MMAP" in env else str(env.get("LLAMA_ARG_MMAP", ""))_mmap_env = _mmap_env.strip().lower()if _mmap_env in _ENV_TRUE_VALUES:- mlock, reserves_ram = False, False+ mlock, reserves_ram, direct_io = False, False, Falseelif _mmap_env in _ENV_FALSE_VALUES:- mlock, reserves_ram = False, True+ mlock, reserves_ram, direct_io = False, True, False# LLAMA_ARG_DIO likewise: on selects DirectIO, off selects "none"._dio_env = "0" if "LLAMA_ARG_NO_DIO" in env else str(env.get("LLAMA_ARG_DIO", ""))_dio_env = _dio_env.strip().lower()if _dio_env in _ENV_TRUE_VALUES:- mlock, reserves_ram = False, False+ mlock, reserves_ram, direct_io = False, False, Trueelif _dio_env in _ENV_FALSE_VALUES:- mlock, reserves_ram = False, True+ mlock, reserves_ram, direct_io = False, True, False_mode_env = str(env.get("LLAMA_ARG_LOAD_MODE", "")).strip().lower()if _mode_env:mlock = _mode_env in _LOAD_MODE_MLOCK_VALUESreserves_ram = _mode_env in _LOAD_MODE_RESERVING_VALUES+ direct_io = _mode_env == "dio"tokens = [str(a) for a in (argv or [])]i, n = 0, len(tokens)@@ -1692,30 +1837,34 @@ def resolve_effective_memory_state(continueif flag in _MLOCK_FLAGS:# Only the mlock bit: which of "mlock" / "mmap+mlock" this maps to is not observable and changes no- # decision- mlock = True+ # decision. Both of them do, however, replace a DirectIO mode set earlier.+ mlock, direct_io = True, Falsei += 1elif flag in _NO_MMAP_FLAGS:# Deprecated selector for the whole "none" mode, so it clears the mlock too: measured, "--mlock --no-mmap"# leaves the child unlocked while "--no-mmap --mlock" locks it.mlock = Falsereserves_ram = True+ direct_io = Falsei += 1elif flag in _DIO_ON_FLAGS:# Deprecated load-mode selector: resets the mode, so the mlock goes. DirectIO streams the weights, so it# holds no full host copy.mlock = Falsereserves_ram = False+ direct_io = Truei += 1elif flag in _DIO_OFF_FLAGS:# Not "plain mmap": upstream maps these to mode `none`, like --no-mmap, which reads the weights into a# full host buffer.mlock = Falsereserves_ram = True+ direct_io = Falsei += 1elif flag == "--mmap":mlock = Falsereserves_ram = False+ direct_io = Falsei += 1elif flag in _LOAD_MODE_FLAGS:if "=" in tok:@@ -1728,16 +1877,20 @@ def resolve_effective_memory_state(if value:mlock = value in _LOAD_MODE_MLOCK_VALUESreserves_ram = value in _LOAD_MODE_RESERVING_VALUES+ direct_io = value == "dio"i += stepelse:i += 1- return mlock, reserves_ram+ return mlock, reserves_ram, direct_iodef memory_state_satisfies_settings(state: Optional[tuple[bool, bool]],policy_active: bool = False,mlock_applicable: bool = True,+ direct_io: Optional[bool] = None,+ dio_applicable: bool = False,+ dio_managed: bool = False,) -> bool:"""True when a launched ``(mlock, reserves_ram)`` matches the settings.@@ -1752,10 +1905,19 @@ def memory_state_satisfies_settings(longer applies, so any of those has to be undone on the next launch, while a launch it nevertouched is left alone.- ``mlock_applicable`` is False when the launch is fully offloaded to a discrete GPU, where- page-locking host RAM buys nothing and is deliberately not emitted. Residency there is the- idle-unload veto, which needs no relaunch, so demanding mlock would ask for a reload that can- never satisfy the check.+ ``mlock_applicable`` is False when the launch is fully offloaded to a+ discrete GPU, where page-locking host RAM buys nothing and is deliberately+ not emitted. Residency there is the idle-unload veto, which needs no+ relaunch, so demanding mlock would ask for a reload that can never satisfy+ the check.++ ``direct_io`` is whether the launch actually streams, and ``dio_applicable``+ whether ``no_reserve_requires_dio`` held for its platform, build and+ placement. Default mmap and DirectIO are the same ``(mlock, reserves_ram)``,+ so without these a Windows full-offload process launched on the default+ mapping reports as satisfying a no-reserve that would now emit dio, and both+ the reload hint and the duplicate-load fast path leave it running. None means+ a caller that does not track it, which never forces a reload."""if state is None:return True@@ -1766,7 +1928,16 @@ def memory_state_satisfies_settings(mlock, reserves_ram = stateif get_no_ram_reserve():# mlock_applicable only excuses a MISSING lock; a live reservation still has to go, wherever the weights are.- return not (mlock or reserves_ram)+ if mlock or reserves_ram:+ return False+ return not (dio_applicable and direct_io is False)if get_keep_resident():+ # The managed DirectIO has to go when no-reserve does: with residency on the+ # policy emits a page-lock or nothing, so a streaming child contradicts the+ # settings however the lock reads, and `not mlock_applicable` accepted it.+ # `dio_managed`, not `policy_active`: that aggregate is also set by a scrubbed+ # env var, so a user's own `dio` left reload_required permanently true.+ if direct_io and dio_applicable and dio_managed:+ return Falsereturn mlock or not mlock_applicablereturn not policy_activediff --git a/studio/backend/routes/settings.py b/studio/backend/routes/settings.pyindex efae98c47a8..47bb944ae13 100644--- a/studio/backend/routes/settings.py+++ b/studio/backend/routes/settings.py@@ -930,40 +930,84 @@ _NO_LAUNCH = object()def _active_launch_placement():- """``(state, policy_active, mlock_applicable)`` for the running child. ``state`` is ``_NO_LAUNCH``- when nothing is running or coming up, so the caller can tell "no process" from "a process with no- load-mode"."""+ """``(state, policy_active, mlock_applicable, direct_io, dio_applicable, dio_managed,+ pending_settings)`` for the running child.++ ``state`` is ``_NO_LAUNCH`` when nothing is running or coming up, so the+ caller can tell "no process" apart from "a process with no load-mode".+ """try:from routes.inference import get_llama_cpp_backendbackend = get_llama_cpp_backend()- pending = bool(getattr(backend, "_memory_launch_pending", False))- if not backend.is_active and not pending:- return _NO_LAUNCH, False, True+ # Read ONCE: two reads can straddle the marker clear, leaving a killed child's+ # placement answering for the launch that replaced it. One attribute carries both+ # "is a launch pending" and "what is it committed to".+ pending = getattr(backend, "_memory_pending_launch", None)+ if not backend.is_active and pending is None:+ return _NO_LAUNCH, False, True, None, False, False, Nonereturn (getattr(backend, "_memory_state", None),bool(getattr(backend, "_memory_policy_active", False)),bool(getattr(backend, "_memory_mlock_applicable", True)),+ getattr(backend, "_memory_direct_io", None),+ bool(getattr(backend, "_memory_dio_applicable", False)),+ # The pair the POLICY emitted, not the aggregate: a user's own `dio` must+ # not be withdrawn on their behalf.+ bool(getattr(backend, "_memory_dio_flags", None)),+ pending,)except Exception:- return _NO_LAUNCH, False, True+ return _NO_LAUNCH, False, True, None, False, False, None+++def _launch_effect_of(settings):+ """The part of ``(keep_resident, no_ram_reserve)`` a launch can express.++ Mirrors ``should_mlock``: the page-lock is emitted only when residency is on and+ no-reserve is off, so with no-reserve on the residency toggle reaches no flag.+ """+ keep_resident, no_ram_reserve = settings+ return (keep_resident and not no_ram_reserve, no_ram_reserve)def _model_memory_reload_required() -> bool:- """True when the loaded process's memory placement contradicts the settings. Compares the state the child- ACTUALLY launched with (env defaults plus last-wins argv, so a user-supplied --mlock / --no-mmap counts)- against what the current settings would produce. The idle-unload veto applies immediately, so only- placement can be stale. Keyed on is_active, not is_loaded: a save that lands while a load is still- passing its health check would otherwise report no reload while the child is already committed to the- pre-save flags."""- state, policy_active, mlock_applicable = _active_launch_placement()+ """True when the loaded process's memory placement contradicts the settings.++ Compares the state the child ACTUALLY launched with -- env defaults plus+ last-wins argv, so a user-supplied --mlock / --no-mmap counts -- against+ what the current settings would produce. The idle-unload veto applies+ immediately (the loop re-reads each poll), so only placement can be stale.++ Keyed on is_active, not is_loaded: a save that lands while a load is still+ passing its health check would otherwise report no reload while the child is+ already committed to the pre-save flags. _memory_launch_pending covers the+ same window before Popen, where the placement is decided but _process is+ still None.+ """+ state, policy_active, mlock_applicable, direct_io, dio_applicable, dio_managed, pending = (+ _active_launch_placement()+ )if state is _NO_LAUNCH:return False+ # A launch in flight has no resolved flags and the comparator reads None as "not+ # governed", so answer from its snapshot. Whenever one is pending, NOT only when+ # `state` is None: replacing a model leaves the old child's `_memory_state` behind.+ if pending is not None:+ from utils.model_memory_settings import get_model_memory_settings++ # By EFFECT, not the literal pair: no-reserve wins over keep-resident for every+ # loader flag, so flipping keep-resident under it only moves the idle-unload veto,+ # which the loop re-reads each poll. The raw tuple asked for an inexpressible reload.+ return _launch_effect_of(get_model_memory_settings()) != _launch_effect_of(pending)+# Same predicate the duplicate-load comparator uses.from core.inference.llama_server_args import memory_state_satisfies_settings- return not memory_state_satisfies_settings(state, policy_active, mlock_applicable)+ return not memory_state_satisfies_settings(+ state, policy_active, mlock_applicable, direct_io, dio_applicable, dio_managed+ )def _model_memory_mlock_active(want_mlock: bool) -> bool:@@ -974,7 +1018,9 @@ def _model_memory_mlock_active(want_mlock: bool) -> bool:user's own --mlock counts, since the resolver reads the launched argv."""if not want_mlock:return False- state, _policy_active, _applicable = _active_launch_placement()+ state, _policy_active, _applicable, _direct_io, _dio_applicable, _dio_managed, _pending = (+ _active_launch_placement()+ )if state is _NO_LAUNCH:return Truereturn bool(state and state[0])diff --git a/studio/backend/tests/test_amd_apu_unified_memory.py b/studio/backend/tests/test_amd_apu_unified_memory.pyindex 265144df908..65759939634 100644--- a/studio/backend/tests/test_amd_apu_unified_memory.py+++ b/studio/backend/tests/test_amd_apu_unified_memory.py@@ -601,3 +601,49 @@ class TestTheEnableSwitch:self._strix_halo(monkeypatch)assert LlamaCppBackend._unified_memory_for_launch([0], 100 * _GIB) is Trueassert LlamaCppBackend._unified_memory_for_launch([0], None) is False+++class TestTheDirectIoGateNeedsEveryDeviceRead:+ """`_rocm_classification_answered` guards a loader choice, so it must mean "the+ devices were classified", not "a classifier was importable and something+ enumerated". `_rocm_unified_memory_gpu_ids` drops any device it cannot read with+ `except: continue`, and an absent device reads as discrete, which is how an+ unclassified APU would be handed DirectIO over its own system RAM."""++ @staticmethod+ def _torch(archs, *, raises_on = None):+ t = _fake_torch("6.2.0", list(archs))++ def props(i):+ if raises_on is not None and i == raises_on:+ raise RuntimeError("HIP error: invalid device ordinal")+ return types.SimpleNamespace(gcnArchName = archs[i])++ t.cuda.get_device_properties = props+ return t++ def test_every_device_readable_answers(self, monkeypatch):+ monkeypatch.setitem(sys.modules, "torch", self._torch(["gfx1100", "gfx1201"]))+ assert LlamaCppBackend._rocm_classification_answered() is True++ def test_one_unreadable_device_declines(self, monkeypatch):+ """The discrete gfx1100 beside it is exactly the trap: the set comes back+ empty, the launch looks like two discrete cards, and the unreadable one may+ be the APU."""+ monkeypatch.setitem(sys.modules, "torch", self._torch(["gfx1100", "gfx1151"], raises_on = 1))+ assert LlamaCppBackend._rocm_unified_memory_gpu_ids() == set()+ assert LlamaCppBackend._rocm_classification_answered() is False++ def test_the_only_device_being_unreadable_declines(self, monkeypatch):+ monkeypatch.setitem(sys.modules, "torch", self._torch(["gfx1151"], raises_on = 0))+ assert LlamaCppBackend._rocm_classification_answered() is False++ def test_no_devices_declines(self, monkeypatch):+ monkeypatch.setitem(sys.modules, "torch", self._torch([]))+ assert LlamaCppBackend._rocm_classification_answered() is False++ def test_non_rocm_torch_declines(self, monkeypatch):+ """No Windows ROCm wheel, so this is the usual answer on the one platform+ the DirectIO decision runs on."""+ monkeypatch.setitem(sys.modules, "torch", _fake_torch(None, ["sm_90"]))+ assert LlamaCppBackend._rocm_classification_answered() is Falsediff --git a/studio/backend/tests/test_gpu_arch_gate_consumers_7624.py b/studio/backend/tests/test_gpu_arch_gate_consumers_7624.pyindex a29da8cfcc5..5ce70d59280 100644--- a/studio/backend/tests/test_gpu_arch_gate_consumers_7624.py+++ b/studio/backend/tests/test_gpu_arch_gate_consumers_7624.py@@ -612,15 +612,28 @@ class TestArchRetryRestoresTheMemoryPolicy:# Snapshotted at the launch, not re-derived at the retry: re-probing# residency for the SURVIVORS would mark an APU survivor mlock-applicable# against a lock-free argv, turning every later duplicate load into a reload.- assert "_mem_policy_for_cmd = (" in text- _snap = [- _line.strip().rstrip(",")- for _line in text.split("_mem_policy_for_cmd = (")[1].split(")")[0].splitlines()- if _line.strip()- ]+ # One helper, so every site that mutates `cmd` retakes the same shape.+ assert "def _snapshot_policy_for_cmd():" in text+ assert "_mem_policy_for_cmd = _snapshot_policy_for_cmd()" in text+ _body = text.split("def _snapshot_policy_for_cmd():")[1].split("return (")[1]+ _members = []+ for _line in _body.splitlines():+ if _line.strip() == ")":+ break+ if _line.strip():+ _members.append(_line.strip().rstrip(","))+ _snap = _membersassert _snap == ["_mem_host_resident","self._memory_state",+ # With the pair: a stale DirectIO bit beside a restored one is the drift+ # the snapshot exists to prevent.+ "self._memory_direct_io",+ # Same reason: a rung that strips a COPY clears it while `cmd` still owes+ # DirectIO.+ "self._memory_dio_applicable",+ # Copied in, so a later strip cannot reach the snapshot the fallback uses.+ "list(self._memory_dio_flags)","self._memory_policy_active","self._memory_mlock_applicable",]@@ -641,6 +654,13 @@ class TestArchRetryRestoresTheMemoryPolicy:assert _restored == ["_mem_host_resident","self._memory_state",+ # With the pair: a stale DirectIO bit beside a restored one is the drift+ # the snapshot exists to prevent.+ "self._memory_direct_io",+ # Same reason: a rung that strips a COPY clears it while `cmd` still owes+ # DirectIO.+ "self._memory_dio_applicable",+ "self._memory_dio_flags","self._memory_policy_active","self._memory_mlock_applicable",]@@ -1047,3 +1067,43 @@ class TestGatedTensorModeStillDeduplicates:backend._tensor_parallel = Truerequest = dict(_GATED_REQUEST, tensor_parallel = False)assert backend.adopt_load_intent_if_matched(GgufLoadIntent(**request)) is False+++class TestArchRetryAsksResidencyTheSameWayTheLaunchDid:+ """The rung's residency question has to be the launch's question.++ An unprobed device answers the conservative "host resident", and+ `_mem_should_mlock` is always False under no-reserve, so gating the rung's probe+ on it alone contradicted the launch's own verdict for the same devices. Not+ cosmetic: the rung re-records the placement.+ """++ def _load_model_source(self):+ import inspect++ from core.inference.llama_cpp import LlamaCppBackend+ return inspect.getsource(LlamaCppBackend.load_model)++ def test_every_probe_vulkan_gate_admits_the_directio_probe(self):+ """Across all three sites: the defect was that they disagreed."""+ gates = [+ line.strip()+ for line in self._load_model_source().splitlines()+ if line.strip().startswith("probe_vulkan = ")+ ]+ assert len(gates) == 3, gates+ assert all("_mem_probe_for_dio" in gate for gate in gates), gates++ def test_the_residency_arm_records_from_the_argv(self):+ """`cmd` may carry the managed DirectIO pair, which `_mem_extras ++ _retry_managed` does not add up to. Rebuilding from the parts records a mapped+ load for a streaming child, and the comparator then asks for a reload that+ relaunching reproduces."""+ src = self._load_model_source()+ marker = "Arch-crash retry changed where the weights live"+ assert marker in src+ arm = src[: src.index(marker)]+ arm = arm[arm.rindex("_retry_host_resident and not _mem_host_resident") :]+ compact = "".join(arm.split())+ assert "self._record_memory_state(cmd,env)" in compact+ assert "self._record_memory_state(list(_mem_extras)" not in compactdiff --git a/studio/backend/tests/test_llama_cpp_vulkan_probe.py b/studio/backend/tests/test_llama_cpp_vulkan_probe.pyindex b02a94d1cfd..ecb72f14271 100644--- a/studio/backend/tests/test_llama_cpp_vulkan_probe.py+++ b/studio/backend/tests/test_llama_cpp_vulkan_probe.py@@ -80,10 +80,13 @@ def test_missing_description_symbol_keeps_igpu_detection():)lib = _types.SimpleNamespace(ggml_backend_vk_reg = _FakeCFunction(1))- flags, names = _igpu_flags_and_names(base, lib, 1)+ flags, names, known = _igpu_flags_and_names(base, lib, 1)assert flags == [True]assert names == ["Legacy Vulkan iGPU"]+ # The type WAS read, which separates a real "not integrated" from a failed query;+ # only the former may be trusted for a DirectIO decision.+ assert known == [True]def _make_vulkan_install(tmp_path: Path) -> str:diff --git a/studio/backend/tests/test_load_mode_fit_matrix.py b/studio/backend/tests/test_load_mode_fit_matrix.pyindex d6ff8af2b39..9b4672b8ee1 100644--- a/studio/backend/tests/test_load_mode_fit_matrix.py+++ b/studio/backend/tests/test_load_mode_fit_matrix.py@@ -707,6 +707,7 @@ def test_the_cpu_fallback_drops_the_fits_load_mode(monkeypatch):from unittest import mockbackend = LlamaCppBackend.__new__(LlamaCppBackend)+ backend._memory_dio_flags = []backend._fit_load_mode_flags = ["--load-mode", "none"]replay = ["llama-server",@@ -742,6 +743,7 @@ def test_the_cpu_fallback_keeps_a_load_mode_the_user_asked_for(monkeypatch):from unittest import mockbackend = LlamaCppBackend.__new__(LlamaCppBackend)+ backend._memory_dio_flags = []backend._fit_load_mode_flags = [] # user pick: nothing recordedreplay = ["llama-server", "-m", "model.gguf", "--load-mode", "none"]with (@@ -1750,7 +1752,8 @@ def test_an_inherited_loader_mode_wins_over_the_fits_pick():]# Asked of the child's environment AFTER the same scrub it gets, so a var a# Model Memory toggle drops vetoes nothing.- assert "scrub_memory_env(_fit_load_mode_env_view)" in arm+ # Now handed the launch's snapshot explicitly, the same value it read before.+ assert "scrub_memory_env(_fit_load_mode_env_view,_mem_settings)" in arm# Assert the CONDITIONS, not one rendering: the formatter is free to wrap this,# which breaks a single-string pin. Each clause is asserted on its own.assert "_fit_load_mode" in arm and "notload_mode" in arm@@ -2152,9 +2155,9 @@ def test_the_no_flash_rung_recomputes_the_memory_record():# Bounded at the next definition, so this proves the recompute is in the# helper and not merely somewhere later in load_model.helper = helper[: helper.index("_spawn_and_wait")]- assert "self._memory_state" in helper+ assert "self._record_memory_state" in helpercompact = "".join(helper.split())- assert "resolve_effective_memory_state(stripped" in compact+ assert "self._record_memory_state(stripped" in compactdef test_the_cpu_projector_rung_recomputes_the_memory_record():@@ -2165,9 +2168,9 @@ def test_the_cpu_projector_rung_recomputes_the_memory_record():src = inspect.getsource(B.load_model)arm = src[: src.index('"-mmproj-cpu"')]arm = arm[arm.rindex("_with_mmproj_offload_disabled") :]- assert "self._memory_state" in arm+ assert "self._record_memory_state" in armcompact = "".join(arm.split())- assert "resolve_effective_memory_state(_stripped_cpu_projector_cmd" in compact+ assert "self._record_memory_state(_stripped_cpu_projector_cmd" in compactdef test_the_arch_crash_rung_records_from_the_argv_not_the_parts():@@ -2194,7 +2197,7 @@ def test_the_arch_crash_rung_records_from_the_argv_not_the_parts():# Reachability: single call expression, whitespace stripped, so a reformat# that wraps the call cannot break it.compact = "".join(inspect.getsource(B.load_model).split())- assert "resolve_effective_memory_state(cmd,env)" in compact+ assert "self._record_memory_state(cmd,env)" in compact# ------------------ round 12: the CPU fallback is a rung that strips the loader@@ -2218,6 +2221,7 @@ def test_the_cpu_replay_and_the_launch_record_disagree(monkeypatch):)backend = LlamaCppBackend.__new__(LlamaCppBackend)+ backend._memory_dio_flags = []backend._fit_load_mode_flags = ["--load-mode", FIT_MODE]launched = ["llama-server", "-m", "model.gguf", "--load-mode", FIT_MODE]with (@@ -2263,7 +2267,7 @@ def test_the_crash_path_cpu_fallback_recomputes_the_memory_record():# Bounded at the normalisation that closes the recovery, so this proves the# recompute is in the arm and not merely somewhere later in load_model.arm = arm[: arm.index("_apply_cpu_fallback_state")]- assert "resolve_effective_memory_state(_last_spawn_cmd,env)" in arm+ assert "self._record_memory_state(_last_spawn_cmd,env)" in armdef test_the_replayed_cpu_fallback_recomputes_the_memory_record():@@ -2277,4 +2281,4 @@ def test_the_replayed_cpu_fallback_recomputes_the_memory_record():# crash path's, which reads the same helper without it.arm = src[src.index("allow_manual_cpu=True") :]arm = arm[: arm.index("_apply_cpu_fallback_state")]- assert "resolve_effective_memory_state(cmd,env)" in arm+ assert "self._record_memory_state(cmd,env)" in armdiff --git a/studio/backend/tests/test_model_memory_settings.py b/studio/backend/tests/test_model_memory_settings.pyindex f193efd6c5c..484856ef18c 100644--- a/studio/backend/tests/test_model_memory_settings.py+++ b/studio/backend/tests/test_model_memory_settings.py@@ -47,6 +47,8 @@ def policy(monkeypatch):extras,supports_load_mode = False,weights_in_host_memory = True,+ gpu_offload_confirmed = False,+ env = None,):monkeypatch.setattr(mm, "get_keep_resident", lambda: keep_resident)monkeypatch.setattr(mm, "get_no_ram_reserve", lambda: no_ram_reserve)@@ -55,6 +57,8 @@ def policy(monkeypatch):extras,supports_load_mode = supports_load_mode,weights_in_host_memory = weights_in_host_memory,+ gpu_offload_confirmed = gpu_offload_confirmed,+ env = env,)return run@@ -1212,23 +1216,32 @@ class TestVulkanIgpuDetection:return LlamaCppBackend._vulkan_targets_are_igpusdef test_all_igpus(self, monkeypatch):- rows = [{"index": 0, "is_igpu": True}]+ rows = [{"index": 0, "is_igpu": True, "type_known": True}]assert self._probe(monkeypatch, rows)("bin", None) is Truedef test_a_mixed_set_still_has_host_weights(self, monkeypatch):"""A split puts part of the model on the iGPU, whose VRAM is system RAM,so those pages are as evictable as if it were the only device."""- rows = [{"index": 0, "is_igpu": True}, {"index": 1, "is_igpu": False}]+ rows = [+ {"index": 0, "is_igpu": True, "type_known": True},+ {"index": 1, "is_igpu": False, "type_known": True},+ ]assert self._probe(monkeypatch, rows)("bin", None) is Truedef test_only_the_selected_devices_count(self, monkeypatch):- rows = [{"index": 0, "is_igpu": True}, {"index": 1, "is_igpu": False}]+ rows = [+ {"index": 0, "is_igpu": True, "type_known": True},+ {"index": 1, "is_igpu": False, "type_known": True},+ ]assert self._probe(monkeypatch, rows)("bin", [0]) is Trueassert self._probe(monkeypatch, rows)("bin", [1]) is Falseassert self._probe(monkeypatch, rows)("bin", [0, 1]) is Truedef test_discrete_only_stays_no(self, monkeypatch):- rows = [{"index": 0, "is_igpu": False}, {"index": 1, "is_igpu": False}]+ rows = [+ {"index": 0, "is_igpu": False, "type_known": True},+ {"index": 1, "is_igpu": False, "type_known": True},+ ]assert self._probe(monkeypatch, rows)("bin", None) is Falsedef test_an_unreadable_probe_answers_no(self, monkeypatch):@@ -1377,7 +1390,7 @@ class TestPreSpawnWindow:backend = _fake_backend(is_active = False,- _memory_launch_pending = True,+ _memory_pending_launch = (True, False),_memory_state = (True, False),_memory_policy_active = True,)@@ -1395,7 +1408,7 @@ class TestPreSpawnWindow:import routes.settings as rsbackend = _fake_backend(- _memory_launch_pending = True,+ _memory_pending_launch = (True, False),_memory_state = (True, False),_memory_policy_active = True,)@@ -2285,7 +2298,7 @@ class TestFitOffRetryDropsTheLock:"_without_subsequence(run_cmd, _mem_managed)","_mem_host_resident = False","self._memory_mlock_applicable = False",- "resolve_effective_memory_state(run_cmd, env)",+ "self._record_memory_state(run_cmd, env)",):assert needle in tail, needle@@ -2318,9 +2331,15 @@ class TestFitOffRetryClearsPolicyActivity:import inspectsrc = inspect.getsource(LlamaCppBackend.load_model)+ # Whitespace-normalised, or pre-commit.ci's reflow reads as a behaviour change.+ # No longer bare bool(_mem_managed): a pair a later mmap shadows changes nothing+ # observable, so it is not activity. The other half is what the retry reuses.+ flat = "".join(src.split())assert (- "self._memory_policy_active = bool(_mem_managed) or _mem_policy_touched_extras" in src+ "self._memory_policy_active=_mem_managed_is_effectiveor_mem_policy_touched_extras"+ in flat)+ assert "self._memory_policy_extras_touched=_mem_policy_touched_extras" in flatbranch = src.find('run_cmd = [*run_cmd, "--fit", "off"]')assert branch != -1end = src.find("return False", branch)@@ -2443,3 +2462,2269 @@ class TestTheEffectiveFitterState:src = inspect.getsource(LlamaCppBackend.load_model)assert "fit_active = fit_is_effectively_on(" in srcassert "[*cmd, *(_mem_extra_args or [])], _mem_env" in src+++class TestWindowsNoReserveStreaming:+ @pytest.mark.parametrize("keep_resident", [False, True])+ def test_full_offload_streams_and_none_cannot_restore_mmap(+ self, policy, monkeypatch, keep_resident+ ):+ monkeypatch.setattr(_lsa.sys, "platform", "win32")+ managed, extras = policy(+ keep_resident,+ True,+ ["--no-mmap", "--temp", "0.7"],+ supports_load_mode = True,+ weights_in_host_memory = False,+ gpu_offload_confirmed = True,+ )+ selected, extras = _lsa.apply_load_mode_policy(+ extras,+ supports_load_mode = True,+ weights_in_host_memory = False,+ requested_load_mode = "none",+ )+ assert managed + selected + extras == ["--load-mode", "dio", "--temp", "0.7"]++ @pytest.mark.parametrize(+ "platform,host,supported",+ [+ ("linux", False, True),+ ("darwin", False, True),+ ("win32", True, True),+ ("win32", False, False),+ ],+ )+ def test_other_placements_and_legacy_builds_keep_their_policy(+ self, policy, monkeypatch, platform, host, supported+ ):+ monkeypatch.setattr(_lsa.sys, "platform", platform)+ managed, extras = policy(+ False,+ True,+ ["--mlock"],+ supports_load_mode = supported,+ weights_in_host_memory = host,+ gpu_offload_confirmed = not host,+ )+ assert managed == []+ assert extras == []++ def test_explicit_mmap_remains_an_override(self, policy, monkeypatch):+ monkeypatch.setattr(_lsa.sys, "platform", "win32")+ managed, extras = policy(+ False,+ True,+ [],+ supports_load_mode = True,+ weights_in_host_memory = False,+ gpu_offload_confirmed = True,+ )+ selected, extras = _lsa.apply_load_mode_policy(+ extras,+ supports_load_mode = True,+ weights_in_host_memory = False,+ requested_load_mode = "mmap",+ )+ assert managed + selected == ["--load-mode", "dio", "--load-mode", "mmap"]+++def _dio(+ extras = None,+ *,+ supports = True,+ host = False,+ confirmed = True,+ requested = None,+ env = None,+ no_reserve = True,+ keep = False,+):+ """``(policy_emitted, effective)`` through the real policy chain."""+ return _lsa.resolve_launch_load_mode(+ extras or [],+ supports_load_mode = supports,+ weights_in_host_memory = host,+ gpu_offload_confirmed = confirmed,+ requested_load_mode = requested,+ env = env,+ settings = (keep, no_reserve),+ )+++class TestTheDioPolicy:+ """The whole feature: on Windows, a confirmed full offload under no-reserve+ streams instead of mapping, because Windows keeps the GGUF mapping resident+ after offload (unmap_fragment is a no-op there, #9033). Everything else keeps+ llama.cpp's default."""++ @pytest.fixture(autouse = True)+ def _win(self, monkeypatch):+ monkeypatch.setattr(_lsa.sys, "platform", "win32")++ def test_a_confirmed_full_offload_streams(self):+ assert _dio() == (True, True)++ @pytest.mark.parametrize(+ "kwargs",+ [+ {"no_reserve": False}, # the setting is off+ {"supports": False}, # a build with no --load-mode+ {"confirmed": False}, # placement not established, incl. host-resident+ ],+ ids = ["toggle-off", "legacy-build", "unconfirmed"],+ )+ def test_every_leg_is_required(self, kwargs):+ assert _dio(**kwargs) == (False, False)++ def test_host_residency_reaches_the_branch_as_an_unconfirmed_offload(self):+ """The launch hands host residency to the confirmation, so the two cannot+ disagree; the policy takes the confirmation and does not re-derive it."""+ from core.inference.llama_cpp import LlamaCppBackend+ import inspect++ flat = "".join(inspect.getsource(LlamaCppBackend.load_model).split())+ assert "_mem_gpu_offload_confirmed=self._gpu_offload_confirmed(" in flat+ assert "binary,_mem_env,_mem_compact_indices,_mem_host_resident," in flat+ # and host residency was priced against those SAME ordinals+ assert "gpu_indices=_mem_physical_indices," in flat+ # and the owner declines on host residency before probing anything+ owner = "".join(inspect.getsource(LlamaCppBackend._gpu_offload_confirmed).split())+ assert owner.index("ifnotdio_possibleorhost_resident:returnFalse") < owner.index(+ "_enumerated_gpu_devices("+ )++ def test_other_platforms_keep_mmap(self, monkeypatch):+ for platform in ("linux", "darwin"):+ monkeypatch.setattr(_lsa.sys, "platform", platform)+ assert _dio() == (False, False)++ @pytest.mark.parametrize(+ "env",+ [+ {"LLAMA_ARG_MMAP": "1"},+ {"LLAMA_ARG_LOAD_MODE": "mmap"},+ {"LLAMA_ARG_DIO": "1"},+ ],+ )+ def test_an_inherited_loader_choice_wins(self, env):+ """no-reserve owns the RESERVATION, not the loader, so these survive the+ scrub, and argv beats the environment in llama.cpp."""+ assert _lsa.scrub_memory_env(dict(env), (False, True)) == []+ emitted, effective = _dio(env = env)+ # The pair stands aside for any of them; only an inherited dio still streams.+ assert emitted is False+ assert effective is (env.get("LLAMA_ARG_DIO") == "1")++ def test_a_reserving_env_var_is_scrubbed_and_does_not_veto(self):+ scrubbed = {"LLAMA_ARG_NO_MMAP": "1"}+ assert _lsa.scrub_memory_env(scrubbed, (False, True)) == ["LLAMA_ARG_NO_MMAP"]+ assert _dio(env = scrubbed) == (True, True)++ def test_a_per_model_mmap_wins_by_last_arg(self):+ # The policy still emits; the user's selector wins the resolve.+ assert _dio(requested = "mmap") == (True, False)++ def test_a_per_model_reserving_mode_is_dropped_and_dio_stands(self):+ """no-reserve vetoes none/mlock/mmap+mlock, so they cannot shadow it."""+ for mode in ("none", "mlock", "mmap+mlock"):+ assert _dio(requested = mode) == (True, True)++ def test_a_hand_typed_extra_wins_by_last_arg(self):+ assert _dio(["--load-mode", "mmap"]) == (True, False)++ def test_a_reserving_extra_is_stripped_and_dio_stands(self):+ assert _dio(["--no-mmap"]) == (True, True)++ @pytest.mark.parametrize("how", ["requested", "extras", "env"])+ def test_a_user_dio_streams_without_the_policy_emitting(self, how):+ """The two halves are different questions. Conflating them made a retry+ append a redundant pair and mark the launch active, and with both toggles+ off `not policy_active` then fails forever: an endless reload of a healthy+ child. The user's own dio streams; the policy contributed nothing."""+ kw = {"no_reserve": False}+ if how == "requested":+ kw["requested"] = "dio"+ elif how == "extras":+ kw["extras"] = ["--load-mode", "dio"]+ else:+ kw["env"] = {"LLAMA_ARG_DIO": "1"}+ emitted, effective = _dio(**kw)+ assert (emitted, effective) == (False, True)+++class TestTheReloadComparator:+ """mmap and dio are both "no full host copy", so the reservation bits alone+ cannot tell them apart. The launch records both what it runs and what a+ relaunch under no-reserve would run; the difference is the whole question."""++ @staticmethod+ def _no_reserve(monkeypatch, on = True):+ import utils.model_memory_settings as mm+ monkeypatch.setattr(mm, "get_keep_resident", lambda: False)+ monkeypatch.setattr(mm, "get_no_ram_reserve", lambda: on)++ def test_the_pair_alone_cannot_tell_the_two_apart(self):+ assert resolve_effective_memory_state([]) == (False, False)+ assert resolve_effective_memory_state(["--load-mode", "dio"]) == (False, False)+ assert _lsa.resolve_effective_direct_io([]) is False+ assert _lsa.resolve_effective_direct_io(["--load-mode", "dio"]) is True++ def test_a_mapped_child_that_would_stream_needs_a_reload(self, monkeypatch):+ self._no_reserve(monkeypatch)+ assert not memory_state_satisfies_settings((False, False), False, False, False, True)++ def test_a_streaming_child_is_satisfied(self, monkeypatch):+ self._no_reserve(monkeypatch)+ assert memory_state_satisfies_settings((False, False), True, False, True, True)++ def test_a_child_a_relaunch_would_not_change_is_satisfied(self, monkeypatch):+ """Linux, a legacy build, an explicit mmap: reloading resolves to the same+ loader, so demanding it would be a notice that never clears."""+ self._no_reserve(monkeypatch)+ assert memory_state_satisfies_settings((False, False), False, True, False, False)++ def test_a_caller_that_does_not_track_it_never_forces_a_reload(self, monkeypatch):+ self._no_reserve(monkeypatch)+ assert memory_state_satisfies_settings((False, False), False, False, None, True)++ def test_a_live_reservation_still_loses(self, monkeypatch):+ self._no_reserve(monkeypatch)+ assert not memory_state_satisfies_settings((False, True), True, False, True, True)++ def test_a_shadowed_pair_is_not_policy_activity(self, monkeypatch):+ """With the toggle off, a child whose only mark was an inert pair is equal+ to an unmanaged one and must not be torn down."""+ self._no_reserve(monkeypatch, on = False)+ assert memory_state_satisfies_settings((False, False), False, False)+++class TestTheLaunchAsksTheSameQuestionTwice:+ def test_the_applicability_comes_from_the_policy_chain(self):+ """Not hand-assembled: the env view and the surviving extras are scrubbed+ and stripped BY the settings, so a hypothetical built from the live ones+ answers for the wrong toggle. That was the root of six review rounds."""+ from core.inference.llama_cpp import LlamaCppBackend+ import inspect++ src = inspect.getsource(LlamaCppBackend.load_model)+ flat = "".join(src.split())+ assert "self._memory_dio_applicable=_hypo_emittedand_hypo_effective" in flat+ assert "_ask((_mem_keep_resident,True),_mem_env_view_no_reserve)" in flat+ # activity asks the same way, for the toggles-off child+ assert "_ask((False,False),_off_view)" in flat+ # the hypothetical env view is scrubbed for the hypothetical toggle+ assert "scrub_memory_env(_mem_env_view_no_reserve,(_mem_keep_resident,True))" in flat+ # and the machinery it replaced is gone+ for retired in (+ "managed_dio_applies",+ "_mem_dio_survives_chain",+ "_mem_dio_placement_unlooked",+ ):+ assert retired not in flat, retired+++class TestTheLaunchWithdrawsTheDio:+ """The pair is chosen for a confirmed full offload. Under dio llama.cpp does+ not map the file (use_mmap covers mmap / mmap+mlock / auto only), so every+ rung that hands the placement back must take it with it, or the layers left+ on the CPU are allocated buffers instead of a pageable mapping."""++ @staticmethod+ def _src():+ from core.inference.llama_cpp import LlamaCppBackend+ import inspect+ return inspect.getsource(LlamaCppBackend.load_model)++ def test_the_three_weight_moving_rungs_strip_it(self):+ from core.inference.llama_cpp import LlamaCppBackend+ import inspect++ launch, replay = (+ self._src(),+ inspect.getsource(LlamaCppBackend._prepare_cpu_fallback_launch),+ )+ # --fit on retry, arch-crash retry and the proactive arch gate here, CPU replay in+ # the builder. The gate strips as well as adds, or a narrowed set keeps the pair.+ assert launch.count("self._drop_managed_dio(") == 3+ assert replay.count("self._drop_managed_dio(") == 1+ # Only the rung that strips `cmd` itself may forget the tokens.+ assert launch.count("clear_record = False") == 1+ assert replay.count("clear_record = False") == 1++ def test_the_projector_retry_does_not(self):+ """It moves the PROJECTOR, and clip.cpp loads that through its own ifstream+ with no use_mmap or load_mode to consult (mtmd_context_params has neither),+ so the pair says nothing about it and the weights are still offloaded."""+ arm = self._src()+ arm = arm[: arm.index('"-mmproj-cpu"')]+ arm = arm[arm.rindex("_with_mmproj_offload_disabled") :]+ assert "_drop_managed_dio" not in arm++ def test_the_arch_retry_re_asks_instead_of_dropping(self):+ """It only narrows visibility and keeps the all-layer --fit off command, so+ a surviving discrete GPU can still be a confirmed full offload. Dropping+ unconditionally left a healthy child on mmap with applicability cleared."""+ src = self._src()+ arm = src[src.index("_arch_dio, _arch_applicable, _arch_active =") :]+ arm = arm[: arm.index("self._record_memory_state(cmd, env)")]+ assert "_dio_decision_for(" in arm+ assert "_remaining" in arm+ # and the restore still precedes any bookkeeping the strip would clear+ assert src.index(") = _mem_policy_for_cmd") < src.index("_dio_left_cmd = False")++ def test_a_copy_strip_leaves_the_tokens_nameable(self):+ from core.inference.llama_cpp import LlamaCppBackend++ b = LlamaCppBackend.__new__(LlamaCppBackend)+ b._memory_dio_flags = list(_lsa.MANAGED_DIO_FLAGS)+ b._memory_dio_applicable = True+ b._memory_policy_active = True+ b._memory_policy_extras_touched = False+ cmd = ["--model", "m.gguf", *_lsa.MANAGED_DIO_FLAGS]+ assert b._drop_managed_dio(list(cmd), "copy", clear_record = False) == ["--model", "m.gguf"]+ assert b._memory_dio_flags == list(_lsa.MANAGED_DIO_FLAGS)+ # ...and the pair was this policy's only mark, so the child is unmanaged now.+ assert b._memory_policy_active is False+ assert b._drop_managed_dio(cmd, "cmd") == ["--model", "m.gguf"]+ assert b._memory_dio_flags == []++ def test_a_scrubbed_var_or_vetoed_extra_keeps_activity(self):+ from core.inference.llama_cpp import LlamaCppBackend++ b = LlamaCppBackend.__new__(LlamaCppBackend)+ b._memory_dio_flags = list(_lsa.MANAGED_DIO_FLAGS)+ b._memory_dio_applicable = True+ b._memory_policy_active = True+ b._memory_policy_extras_touched = True+ b._drop_managed_dio(["-m", "x", *_lsa.MANAGED_DIO_FLAGS], "cpu")+ assert b._memory_policy_active is True+++class TestThePlacementProbes:+ """Two ways to mistake a placement, both of which handed DirectIO to weights+ that really sit in host RAM or withheld it from weights that do not."""++ def test_an_unanswered_vulkan_probe_declines(self, monkeypatch):+ """_vulkan_targets_are_igpus folds "probe failed" into "not an iGPU",+ which is safe where it only skips a page-lock and wrong here: an iGPU's+ VRAM is system RAM."""+ from core.inference.llama_cpp import LlamaCppBackend++ def boom(binary = None):+ raise OSError("probe timed out")++ monkeypatch.setattr(LlamaCppBackend, "_run_vulkan_probe", staticmethod(boom))+ assert not LlamaCppBackend._vulkan_offload_is_discrete("llama-server")+ monkeypatch.setattr(+ LlamaCppBackend, "_run_vulkan_probe", staticmethod(lambda binary = None: [])+ )+ assert not LlamaCppBackend._vulkan_offload_is_discrete("llama-server")++ def test_an_igpu_in_play_declines(self, monkeypatch):+ from core.inference.llama_cpp import LlamaCppBackend++ rows = [+ {"index": 0, "is_igpu": True, "type_known": True},+ {"index": 1, "is_igpu": False, "type_known": True},+ ]+ monkeypatch.setattr(+ LlamaCppBackend, "_run_vulkan_probe", staticmethod(lambda binary = None: rows)+ )+ assert not LlamaCppBackend._vulkan_offload_is_discrete("llama-server")+ assert not LlamaCppBackend._vulkan_offload_is_discrete("llama-server", [0])+ assert LlamaCppBackend._vulkan_offload_is_discrete("llama-server", [1])++ def test_the_probe_is_never_gated_on_the_toggle(self):+ """Gating it made the recorded placement mean "we did not look", which is+ not the same as "not applicable"."""+ from core.inference.llama_cpp import LlamaCppBackend+ import inspect++ flat = "".join(inspect.getsource(LlamaCppBackend.load_model).split())+ assert "_mem_probe_for_dio=_mem_dio_possible" in flat+ assert "_mem_probe_for_dio=_mem_dio_possibleand_mem_no_reserve" not in flat+++class TestEveryDeviceSetChangeReAsks:+ """A rung that changes the effective device set changes the placement, and the+ pair is a question about placement. Each rung used to ask a SUBSET of what the+ launch asks, and every missing piece became its own review round: a partial+ offload getting the pair, a narrowed set never gaining it, a redundant pair+ recorded as activity. They now share one decision."""++ @staticmethod+ def _src():+ from core.inference.llama_cpp import LlamaCppBackend+ import inspect+ return inspect.getsource(LlamaCppBackend.load_model)++ def test_the_partial_helpers_are_gone(self):+ src = self._src()+ for retired in ("_managed_dio_for_confirmed_offload", "_refresh_dio_for_devices"):+ assert retired not in src, retired++ def test_all_three_rungs_use_the_one_decision(self):+ src = self._src()+ assert src.count("_dio_decision_for(") == 4 # 1 def + 3 rungs+ flat = "".join(src.split())+ assert "_dio_decision_for(_survivors,fully_offloaded=False,child_env=env)" in flat+ assert "_dio_decision_for(gpu_indices,fully_offloaded=True,child_env=env)" in flat+ assert (+ "_dio_decision_for(_remaining,fully_offloaded=fully_gpu_offloaded,child_env=env)"+ in flat+ )++ def test_the_decision_re_runs_the_placement_check(self):+ """Backend and probe discreteness are not a full offload: the fitter may+ still leave layers on the CPU, where dio buffers them."""+ src = self._src()+ arm = src[src.index("def _dio_decision_for(") :]+ arm = arm[: arm.index("return pair,")]+ flat = "".join(arm.split())+ assert "host_resident=self._weights_in_host_memory(" in flat+ # residency is carried INTO the confirmation, and the rung probes its OWN visibility+ assert "_rung_env=_mem_env_for(child_env)" in flat+ assert (+ "self._gpu_offload_confirmed(binary,_rung_env,_rung_compact,host_resident,"+ "_mem_dio_possible,_mem_extra_args,)" in flat+ )+ assert "fully_gpu_offloaded=fully_offloaded," in flat++ def test_the_decision_answers_all_three_questions(self):+ src = self._src()+ arm = src[src.index("def _dio_decision_for(") :]+ arm = arm[: arm.index("return pair,")]+ flat = "".join(arm.split())+ assert "_for(_mem_settings,_fit_load_mode_env_view)" in flat # live+ assert "_for((_mem_keep_resident,True),_mem_env_view_no_reserve)" in flat # forced on+ assert "_for((False,False),_off_view)" in flat # toggles off++ def test_the_reactive_gate_can_gain_the_pair_not_only_lose_it(self):+ """Narrowing onto the surviving discrete card can gain a full offload the+ original set (with an unsupported APU in it) never had."""+ src = self._src()+ arm = src[src.index("_arch_dio, _arch_applicable, _arch_active = _dio_decision_for") :]+ arm = arm[: arm.index("self._record_memory_state(cmd, env)")]+ assert "elif _arch_dio and not self._memory_dio_flags:" in arm+ assert "cmd = [*cmd, *_arch_dio]" in arm++ def test_no_rung_sets_activity_unconditionally(self):+ """A pair redundant with a loader the user picked changes nothing a+ relaunch could undo, so appending it is not activity."""+ src = self._src()+ # Each dio rung ORs its answer in rather than asserting True. (A fourth OR is the+ # pre-existing mlock re-arm.)+ for marker in ("_gate_active", "_retry_active", "_arch_active"):+ assert f"{marker} or self._memory_policy_active" in src, marker+++class TestTheVulkanProbeMemoIsScopedToThePlacement:+ """Several placement decisions in one launch want the same rows and each probe+ is a subprocess behind a 15s timeout. The memo is armed only around that work+ and is THREAD-LOCAL: system-info polling reaches the same probe from other+ threads at any time and needs live free/used VRAM, and the load call also spans+ the Hub download, so rows taken before it would price a fit against VRAM that+ has since been allocated."""++ @staticmethod+ def _count_probes(+ monkeypatch,+ m,+ *,+ raising = False,+ ):+ calls = []+ monkeypatch.setattr(m, "_llama_lib_dir", lambda b: Path("/nope"))+ monkeypatch.setattr(m, "_lib_dir_has_ggml_backend", lambda d, n: True)++ def run(*a, **k):+ calls.append(1)+ if raising:+ raise OSError("probe timed out")+ return type("R", (), {"returncode": 0, "stdout": "0 1 2 0 dGPU", "stderr": ""})()++ monkeypatch.setattr(m.subprocess, "run", run)+ return calls++ def test_outside_a_load_every_ask_is_live(self, monkeypatch):+ """Stale rows here would report free/used VRAM captured before llama-server+ allocated, and a transient empty result would stick indefinitely."""+ import core.inference.llama_cpp as m++ calls = self._count_probes(monkeypatch, m)+ for _ in range(3):+ m.LlamaCppBackend._run_vulkan_probe("llama-server")+ assert len(calls) == 3+++class TestTheBackendPathIsEvidenceOnlyWhenItHoldsAPlugin:+ """_binary_ships_no_gpu_backend answering False means "I cannot say it ships+ none", not "it ships one": it returns False for a static layout, an unreadable+ directory, and any nonempty GGML_BACKEND_PATH. Reading that as a positive is+ the fail-open this check exists to prevent."""++ @staticmethod+ def _no_verdict(monkeypatch):+ from core.inference.llama_cpp import LlamaCppBackend+ monkeypatch.setattr(+ LlamaCppBackend,+ "_binary_ships_no_gpu_backend",+ staticmethod(lambda binary = None, env = None: False),+ )+++class TestTheSnapshotCarriesTheDioTokens:+ def test_the_flags_round_trip_with_the_base_command(self):+ """The fit-off retry can append the pair to its OWN run_cmd and record it+ while `cmd` never carried one; the fallback respawning `cmd` then read that+ stale record as "already has it" and could not append what its own devices+ confirm."""+ from core.inference.llama_cpp import LlamaCppBackend+ import inspect++ src = inspect.getsource(LlamaCppBackend.load_model)+ # One snapshot helper, copying in so a later strip cannot reach back into it.+ assert "list(self._memory_dio_flags)," in src+ assert "self._memory_dio_flags,\n" in src # the restore+ # every site that takes the snapshot goes through the helper+ assert src.count("_mem_policy_for_cmd = _snapshot_policy_for_cmd()") >= 4+++class TestASaveDuringPlacementIsAnswered:+ """The marker makes the route see a pending launch, but `_memory_state` is None+ until the flags resolve and the comparator reads None as "not governed". What+ the child is committed to from the snapshot onwards is the toggle pair, so the+ route answers from that."""++ def test_the_launch_publishes_what_it_is_committed_to(self):+ from core.inference.llama_cpp import LlamaCppBackend+ import inspect++ src = "".join(inspect.getsource(LlamaCppBackend.load_model).split())+ # Capture and publication are ONE act: no window, no ordering to get wrong.+ assert (+ 'capture_model_memory_settings(lambdapair:setattr(self,"_memory_pending_launch",pair))'+ in src+ )+ # and nothing re-publishes it before the placement work; the recovery rungs+ # further down legitimately re-arm after a failed attempt+ head = src[+ src.index("capture_model_memory_settings(") : src.index("_arm_load_probe_memo()")+ ]+ assert "self._memory_pending_launch=" not in head++ def test_a_save_that_changes_a_toggle_asks_for_a_reload(self, monkeypatch):+ import routes.settings as rs+ import utils.model_memory_settings as mm++ monkeypatch.setattr(+ rs,+ "_active_launch_placement",+ lambda: (None, False, True, None, False, False, (False, False)),+ )+ monkeypatch.setattr(mm, "get_model_memory_settings", lambda: (False, True))+ assert rs._model_memory_reload_required() is True++ def test_a_save_matching_the_committed_pair_does_not(self, monkeypatch):+ import routes.settings as rs+ import utils.model_memory_settings as mm++ monkeypatch.setattr(+ rs,+ "_active_launch_placement",+ lambda: (None, False, True, None, False, False, (False, True)),+ )+ monkeypatch.setattr(mm, "get_model_memory_settings", lambda: (False, True))+ assert rs._model_memory_reload_required() is False+++class TestTheMarkerIsReleasedWithTheLoadLock:+ """A token assigned at decorator entry was installed before the call had the+ lock, so a merely queued load stole ownership. `_serial_load_scope` already+ solves this shape for the fraction marker: release on the way out of the LOCK,+ because a queued load arms its own the instant it takes it."""++ def test_the_scope_releases_it(self):+ from core.inference.llama_cpp import LlamaCppBackend+ import inspect++ src = inspect.getsource(LlamaCppBackend._serial_load_scope)+ assert "self._memory_pending_launch = None" in src+ # released beside the markers that already had this treatment+ assert "self._vram_fraction_pending = None" in src++ def test_the_load_call_no_longer_owns_it(self):+ import core.inference.llama_cpp as m+ import inspect++ assert not hasattr(m, "_pending_placement_cleared")+ src = inspect.getsource(m._with_gguf_load_marker)+ assert "_pending_placement_cleared" not in src+ assert "_memory_pending_token" not in inspect.getsource(m.LlamaCppBackend.load_model)++ def test_the_publish_is_inside_the_lock(self):+ """Releasing on lock-exit is only correct if the publish is inside it."""+ from core.inference.llama_cpp import LlamaCppBackend+ import inspect++ src = inspect.getsource(LlamaCppBackend.load_model)+ assert src.index("with self._serial_load_scope():") < src.index(+ "self._memory_pending_launch = _mem_settings"+ )+++class TestAReplacementLoadIsNotAnsweredByTheOldChild:+ """Replacing a model kills the old process without clearing its `_memory_state`,+ so a save during the new launch's probe was compared against a placement that+ belongs to a child already gone."""++ def test_the_pending_snapshot_wins_over_a_stale_state(self, monkeypatch):+ import routes.settings as rs+ import utils.model_memory_settings as mm++ # the killed child's state is still present and non-None+ monkeypatch.setattr(+ rs,+ "_active_launch_placement",+ lambda: ((False, False), False, True, False, False, False, (False, False)),+ )+ monkeypatch.setattr(mm, "get_model_memory_settings", lambda: (False, True))+ assert rs._model_memory_reload_required() is True++ def test_the_route_does_not_gate_on_a_none_state(self):+ import routes.settings as rs+ import inspect++ src = inspect.getsource(rs._model_memory_reload_required)+ assert "if pending is not None:" in src+ assert "if state is None and pending is not None:" not in src+ # one read, so the two cannot straddle the marker clear+ assert "_pending_launch_settings" not in src+ assert src.count("_active_launch_placement()") == 1+++class TestAllThreeCudaFamiliesAreRequired:+ """ggml-cuda imports cublas64, which imports cublasLt64; LoadLibrary returns+ NULL unless all three resolve. Same set REAL_UPSTREAM_CUDART_BUNDLE pins in+ test_windows_gpu_detection_mock.py."""++ @staticmethod+ def _cuda_build(tmp_path):+ (tmp_path / "ggml-cuda.dll").write_text("")+ libs = tmp_path / "libs"+ libs.mkdir()+ return libs+++class TestARetryKeepsThePlacementWindowOpen:+ """The marker is dropped after Popen because is_active covers it, but a crashed+ child is not active either, and the retry rungs redo the placement work and+ spawn from the SAME captured settings."""++ def test_a_failed_attempt_re_arms_it(self):+ from core.inference.llama_cpp import LlamaCppBackend+ import inspect++ src = inspect.getsource(LlamaCppBackend.load_model)+ drop = src.index("# is_active covers it from here, so drop the pre-spawn flag.")+ rearm = src.index("self._memory_pending_launch = _mem_settings", drop)+ crashed = src.index("_crashed_proc = self._process", drop)+ # re-armed before the crash is even classified, so every rung inherits it+ assert drop < rearm < crashed++ def test_the_lock_still_owns_the_release(self):+ from core.inference.llama_cpp import LlamaCppBackend+ import inspect+ assert "self._memory_pending_launch = None" in inspect.getsource(+ LlamaCppBackend._serial_load_scope+ )+++class TestThePendingCompareIsByEffect:+ """no-reserve wins over keep-resident for every loader flag, so flipping+ residency while no-reserve is on changes nothing the child launches with. Only+ the idle-unload veto moves, and the loop re-reads that each poll."""++ @pytest.mark.parametrize(+ "live,pending,reload_required",+ [+ ((True, True), (False, True), False), # residency flip under no-reserve+ ((False, True), (True, True), False), # ...either direction+ ((False, True), (False, False), True), # no-reserve itself changed+ ((True, False), (False, False), True), # the page-lock changed+ ((False, False), (False, False), False),+ ],+ )+ def test_only_a_change_the_launch_can_express_asks_for_a_reload(+ self, monkeypatch, live, pending, reload_required+ ):+ import routes.settings as rs+ import utils.model_memory_settings as mm++ monkeypatch.setattr(+ rs,+ "_active_launch_placement",+ lambda: (None, False, True, None, False, False, pending),+ )+ monkeypatch.setattr(mm, "get_model_memory_settings", lambda: live)+ assert rs._model_memory_reload_required() is reload_required++ def test_the_effect_mirrors_should_mlock(self):+ import routes.settings as rs+ for keep in (True, False):+ for no_res in (True, False):+ mlock_bit, _ = rs._launch_effect_of((keep, no_res))+ assert mlock_bit == (keep and not no_res)+++class TestNoNestedHelperIsUsedBeforeItsDef:+ """A nested `def` in `load_model` binds the name as a local, so a call above it+ raises UnboundLocalError at runtime rather than failing import or lint. That+ shipped once and aborted every full-offload launch."""++ def test_every_nested_def_precedes_its_calls(self):+ import ast+ import inspect+ from core.inference.llama_cpp import LlamaCppBackend++ src = inspect.getsource(LlamaCppBackend.load_model)+ tree = ast.parse(inspect.cleandoc("\n".join(src.splitlines()[1:])))+ fn = tree.body[0]++ defined_at = {}+ for node in ast.walk(fn):+ if isinstance(node, (ast.FunctionDef, ast.AsyncFunctionDef)) and node is not fn:+ defined_at.setdefault(node.name, node.lineno)++ offenders = []+ for node in ast.walk(fn):+ if (+ isinstance(node, ast.Call)+ and isinstance(node.func, ast.Name)+ and node.func.id in defined_at+ and node.lineno < defined_at[node.func.id]+ ):+ offenders.append(+ f"{node.func.id} called at {node.lineno}, defined at "+ f"{defined_at[node.func.id]}"+ )+ assert not offenders, "UnboundLocalError at runtime: " + "; ".join(offenders)+++class TestThePendingWindowHasNoGaps:+ """Three ways the window closed early: the publish order, the spawn caller's+ unconditional clear, and the outer recovery rungs. The first is closed by+ construction now. Two attributes held one fact between them, so every writer had+ to order its stores and the reader had to mirror that; one attribute (None, or+ the pair) is assigned atomically, leaving no ordering to assert."""++ def test_the_two_attribute_marker_is_gone(self):+ import inspect+ import routes.settings as rs+ from core.inference.llama_cpp import LlamaCppBackend++ for src in (+ inspect.getsource(LlamaCppBackend.load_model),+ inspect.getsource(LlamaCppBackend._serial_load_scope),+ inspect.getsource(rs._active_launch_placement),+ ):+ assert "_memory_launch_pending" not in src+ assert "_memory_pending_settings" not in src++ def test_the_reader_takes_one_value(self):+ import ast+ import inspect+ import routes.settings as rs++ # module-level function, so its source already sits at column 0+ tree = ast.parse(inspect.getsource(rs._active_launch_placement))+ reads = [+ node+ for node in ast.walk(tree)+ if isinstance(node, ast.Constant) and node.value == "_memory_pending_launch"+ ]+ assert len(reads) == 1, "the pending launch must be sampled exactly once"++ def test_every_arming_site_carries_the_snapshot(self):+ import inspect+ from core.inference.llama_cpp import LlamaCppBackend++ src = inspect.getsource(LlamaCppBackend.load_model)+ armed = [+ line.strip() for line in src.splitlines() if "self._memory_pending_launch =" in line+ ]+ assert armed, "the launch must publish its pending window"+ for line in armed:+ # either it arms WITH the captured pair, or it clears+ assert line.endswith("= _mem_settings") or line.endswith("= None"), line++ def test_the_spawn_caller_clears_only_on_success(self):+ """An unconditional clear undid the in-spawn re-arm and left the outer+ recovery rungs respawning from the captured settings unmarked."""+ import inspect+ from core.inference.llama_cpp import LlamaCppBackend++ src = inspect.getsource(LlamaCppBackend.load_model)+ arm = src[src.index("healthy = _spawn_and_wait(cmd)") :][:900]+ flat = "".join(arm.split())+ assert "ifhealthy:self._memory_pending_launch=None" in flat++ def test_the_lock_is_still_the_backstop(self):+ import inspect+ from core.inference.llama_cpp import LlamaCppBackend+ assert "self._memory_pending_launch = None" in inspect.getsource(+ LlamaCppBackend._serial_load_scope+ )+++class TestOneLaunchReadsOneSettingsSnapshot:+ """`load_model` captures the pair once and decides the argv from it, so every+ consumer inside the launch must read that snapshot. The child-environment scrub+ read the live settings, so a save landing mid-launch gave the process flags from+ one pair and an environment scrubbed under the other."""++ def test_no_scrub_in_load_model_reads_the_live_settings(self):+ import ast+ import inspect+ from core.inference.llama_cpp import LlamaCppBackend++ src = inspect.getsource(LlamaCppBackend.load_model)+ tree = ast.parse(inspect.cleandoc("\n".join(src.splitlines()[1:])))++ bare = [+ node.lineno+ for node in ast.walk(tree.body[0])+ if isinstance(node, ast.Call)+ and isinstance(node.func, ast.Name)+ and node.func.id == "scrub_memory_env"+ and len(node.args) < 2+ and not any(kw.arg == "settings" for kw in node.keywords)+ ]+ assert not bare, (+ "scrub_memory_env called without the launch's settings snapshot at "+ f"line(s) {bare}; it would re-read the live toggles mid-launch"+ )++ def test_the_child_environment_scrub_uses_the_snapshot(self):+ import inspect+ from core.inference.llama_cpp import LlamaCppBackend++ src = "".join(inspect.getsource(LlamaCppBackend.load_model).split())+ assert "_mem_scrubbed=scrub_memory_env(env,_mem_settings)" in src++ def test_an_explicit_snapshot_overrides_the_live_settings(self, monkeypatch):+ from core.inference.llama_server_args import scrub_memory_env+ from utils import model_memory_settings++ def _live(keep_resident, no_ram_reserve):+ monkeypatch.setattr(model_memory_settings, "get_keep_resident", lambda: keep_resident)+ monkeypatch.setattr(model_memory_settings, "get_no_ram_reserve", lambda: no_ram_reserve)++ # Live settings own placement, but the snapshot says neither toggle was on, so+ # this launch leaves the inherited value alone.+ _live(True, False)+ env = {"LLAMA_ARG_MLOCK": "1"}+ assert scrub_memory_env(env, (False, False)) == []+ assert env == {"LLAMA_ARG_MLOCK": "1"}++ # and the converse: the snapshot owns placement even though the live pair does not+ _live(False, False)+ env = {"LLAMA_ARG_MLOCK": "1"}+ assert scrub_memory_env(env, (True, False)) == ["LLAMA_ARG_MLOCK"]+ assert env == {}++ # with no snapshot it falls back to the live pair, as the other scrub sites do+ _live(True, False)+ env = {"LLAMA_ARG_MLOCK": "1"}+ assert scrub_memory_env(env) == ["LLAMA_ARG_MLOCK"]+++class TestOnlyALoadablePluginCountsAsAGpuBackend:+ """A disabled backend leaves `ggml-cuda.dll.bak` behind, which is precisely when+ the build ships none. A prefix match read those as a backend, so a CPU-only install+ could be confirmed for full offload and take DirectIO over host-resident weights."""++ LOADABLE = (+ "ggml-cuda.dll",+ "ggml-vulkan.dll",+ "libggml-cuda.so",+ "libggml-cuda.so.1",+ "libggml-cuda.so.0.0.1",+ "libggml-hip.so.2",+ "libggml-metal.dylib",+ "libggml-metal.1.dylib",+ )+ NOT_LOADABLE = (+ "ggml-cuda.dll.bak",+ "ggml-cuda.dll.disabled",+ "ggml-cuda.dll.old",+ "ggml-cuda-notes.txt",+ "ggml-cuda.txt",+ "ggml-cudafoo.dll",+ "ggml-cuda",+ "libggml-cuda.so.x",+ )++ def test_the_gpu_backend_pattern_matches_only_loadable_names(self):+ from core.inference.llama_cpp import _GGML_GPU_BACKEND_RE+ for name in self.LOADABLE:+ assert _GGML_GPU_BACKEND_RE.match(name), name+ for name in self.NOT_LOADABLE:+ assert not _GGML_GPU_BACKEND_RE.match(name), name+++class TestRecoveryRungsReadTheLaunchSnapshot:+ """Every rung of one launch decides from the pair that launch captured. The fit-on+ and arch-crash recoveries called live `should_mlock()` and passed no `settings`, so+ a save mid-launch gave the retry toggles the rest of the launch never saw."""++ def test_load_model_never_calls_the_live_mlock_helper(self):+ import inspect+ from core.inference.llama_cpp import LlamaCppBackend++ src = "".join(inspect.getsource(LlamaCppBackend.load_model).split())+ assert "should_mlock()" not in src++ def test_the_live_helper_is_not_even_imported_into_the_launch(self):+ import inspect+ from core.inference.llama_cpp import LlamaCppBackend++ src = "".join(inspect.getsource(LlamaCppBackend.load_model).split())+ assert "importshould_mlock" not in src+ assert ",should_mlock" not in src++ def test_every_policy_call_in_load_model_carries_the_snapshot(self):+ import ast+ import inspect+ from core.inference.llama_cpp import LlamaCppBackend++ src = inspect.getsource(LlamaCppBackend.load_model)+ tree = ast.parse(inspect.cleandoc("\n".join(src.splitlines()[1:])))+ bare = [+ node.lineno+ for node in ast.walk(tree.body[0])+ if isinstance(node, ast.Call)+ and isinstance(node.func, ast.Name)+ and node.func.id == "apply_model_memory_policy"+ and not any(kw.arg == "settings" for kw in node.keywords)+ ]+ assert not bare, f"apply_model_memory_policy without the launch snapshot at line(s) {bare}"+++class TestAnyInstalledGpuPluginMustBeLoadable:+ """A CUDA-only runtime check was the single fail-OPEN path into+ `_mem_gpu_offload_confirmed`. `ggml-hip.dll` present with `amdhip64*.dll` absent+ loads no more than a CUDA build without cudart: llama.cpp reports no devices and+ keeps the weights on the CPU, while the host inventory still lists the card. The+ launch was then confirmed for full offload and given managed DirectIO, trading a+ pageable mapping for a model-sized allocated buffer."""++ def _install(self, tmp_path, plugin, runtime_names):+ (tmp_path / plugin).write_text("")+ libs = tmp_path / "libs"+ libs.mkdir(exist_ok = True)+ for name in runtime_names:+ (libs / name).write_text("")+ return [str(libs)]+++# Real `llama-server --list-devices` output, captured not invented: a CUDA build with+# two cards, and the two ways a build with nothing prints it. The noise above the+# header is what ggml_cuda_init writes.+_LIST_DEVICES_GPU = """ggml_cuda_init: found 2 CUDA devices (Total VRAM: 72627 MiB):+ Device 0: NVIDIA RTX 6000 Ada Generation, compute capability 8.9, VMM: yes, VRAM: 48504 MiB+ Device 1: NVIDIA GeForce RTX 3090, compute capability 8.6, VMM: yes, VRAM: 24123 MiB+Available devices:+ CUDA0: NVIDIA RTX 6000 Ada Generation (48504 MiB, 47172 MiB free)+ CUDA1: NVIDIA GeForce RTX 3090 (24123 MiB, 21855 MiB free)+"""+_LIST_DEVICES_NONE = "Available devices:\n (none)\n"+_LIST_DEVICES_BARE = "Available devices:\n"+_LIST_DEVICES_VULKAN = (+ "Available devices:\n Vulkan0: AMD Radeon 8060S (16384 MiB, 15000 MiB free)\n"+)+++class TestTheBuildsOwnDeviceListIsTheEvidence:+ """The confirmation used to be six filename checks standing in for one question:+ will the child place the weights on a discrete GPU. Each was a proxy that a build+ enumerating nothing could still satisfy. `--list-devices` is the loader's own+ verdict, so a missing runtime, a disabled plugin, a wrong-vendor plugin and a+ GGML_BACKEND_PATH plugin all resolve without reading a filename."""++ def test_it_parses_a_real_gpu_listing(self):+ from core.inference.llama_cpp import _parse_listed_devices+ assert _parse_listed_devices(_LIST_DEVICES_GPU) == ["CUDA0", "CUDA1"]++ def test_the_init_noise_above_the_header_is_not_a_device(self):+ """` Device 0: ...` is indented and has a colon, so only the header keeps it out."""+ from core.inference.llama_cpp import _parse_listed_devices+ assert "Device" not in "".join(_parse_listed_devices(_LIST_DEVICES_GPU))++ @pytest.mark.parametrize("text", [_LIST_DEVICES_NONE, _LIST_DEVICES_BARE])+ def test_both_renderings_of_no_devices_are_an_answer(self, text):+ """Different builds print `(none)` or nothing at all. Both mean zero devices,+ which is a real answer and must not collapse into "could not tell"."""+ from core.inference.llama_cpp import _parse_listed_devices+ assert _parse_listed_devices(text) == []++ @pytest.mark.parametrize(+ "text", ["", None, "error: unknown argument --list-devices\n", "garbage\n"]+ )+ def test_no_header_is_no_answer(self, text):+ """An older build that rejects the flag is not evidence of having no devices."""+ from core.inference.llama_cpp import _parse_listed_devices+ assert _parse_listed_devices(text) is None+++class TestOnlyEnumeratedDevicesConfirmAnOffload:+ def _confirm(+ self,+ monkeypatch,+ devices,+ gpu_indices = None,+ host_resident = False,+ discrete = True,+ dio_possible = True,+ ):+ from core.inference.llama_cpp import LlamaCppBackend++ monkeypatch.setattr(+ LlamaCppBackend,+ "_enumerated_gpu_devices",+ classmethod(lambda cls, binary = None, env = None: devices),+ )+ monkeypatch.setattr(+ LlamaCppBackend,+ "_vulkan_offload_is_discrete",+ staticmethod(lambda binary, idx = None: discrete),+ )+ return LlamaCppBackend._gpu_offload_confirmed(+ "llama-server", {}, gpu_indices, host_resident, dio_possible+ )++ def test_an_enumerated_gpu_confirms(self, monkeypatch):+ assert self._confirm(monkeypatch, ["CUDA0"]) is True++ def test_a_build_reporting_no_devices_declines(self, monkeypatch):+ """The CUDA-without-cudart, HIP-without-amdhip64, disabled-plugin and+ wrong-vendor cases all arrive here as an empty list."""+ assert self._confirm(monkeypatch, []) is False++ def test_an_unanswered_probe_declines(self, monkeypatch):+ assert self._confirm(monkeypatch, None) is False++ def test_host_resident_weights_decline_without_probing(self, monkeypatch):+ assert self._confirm(monkeypatch, ["CUDA0"], host_resident = True) is False++ def test_a_platform_that_cannot_use_dio_never_probes(self, monkeypatch):+ """Off Windows no answer can reach a flag, now or after a later save, so the+ subprocess is not worth spawning."""+ from core.inference.llama_cpp import LlamaCppBackend++ def _boom(+ cls,+ binary = None,+ env = None,+ ):+ raise AssertionError("probed when the answer cannot matter")++ monkeypatch.setattr(LlamaCppBackend, "_enumerated_gpu_devices", classmethod(_boom))+ assert (+ LlamaCppBackend._gpu_offload_confirmed("llama-server", {}, None, False, False) is False+ )++ def test_a_cpu_only_listing_is_not_an_offload_target(self, monkeypatch):+ assert self._confirm(monkeypatch, ["CPU"]) is False++ def test_a_pinned_device_the_build_never_listed_declines(self, monkeypatch):+ """The stale-pin case: the host reports a card the child cannot open."""+ assert self._confirm(monkeypatch, ["CUDA0"], gpu_indices = [1]) is False++ def test_every_pinned_device_must_be_listed(self, monkeypatch):+ assert self._confirm(monkeypatch, ["CUDA0"], gpu_indices = [0, 1]) is False+ assert self._confirm(monkeypatch, ["CUDA0", "CUDA1"], gpu_indices = [0, 1]) is True++ def test_a_vulkan_target_still_needs_the_discreteness_probe(self, monkeypatch):+ """--list-devices does not say whether the VRAM is carved out of system RAM,+ and an iGPU full offload is still host-backed."""+ assert self._confirm(monkeypatch, ["Vulkan0"], discrete = False) is False+ assert self._confirm(monkeypatch, ["Vulkan0"], discrete = True) is True++ def test_a_non_vulkan_target_does_not_pay_for_that_probe(self, monkeypatch):+ assert self._confirm(monkeypatch, ["CUDA0"], discrete = False) is True++ def test_an_external_vulkan_plugin_names_itself_like_any_other(self):+ """What the plugin-root scan was approximating: the build lists the device it+ loaded, wherever the plugin came from."""+ from core.inference.llama_cpp import LlamaCppBackend, _parse_listed_devices++ devices = _parse_listed_devices(_LIST_DEVICES_VULKAN)+ assert LlamaCppBackend._devices_are_vulkan(devices, None) is True+++class TestTheDeviceListIsProbedOncePerLoad:+ def test_while_armed_repeated_asks_spawn_one_probe(self, monkeypatch):+ import core.inference.llama_cpp as m++ calls = []+ monkeypatch.setattr(+ m.LlamaCppBackend,+ "_run_list_devices",+ staticmethod(lambda binary, env = None: calls.append(binary) or ["CUDA0"]),+ )+ m._arm_load_probe_memo()+ try:+ for _ in range(3):+ assert m.LlamaCppBackend._enumerated_gpu_devices("llama-server", {}) == ["CUDA0"]+ assert len(calls) == 1+ finally:+ m._LOAD_PROBE_STATE.armed = False+ m._LOAD_PROBE_STATE.rows = None++ def test_an_unanswered_probe_is_memoised_too(self, monkeypatch):+ """Otherwise a build that hangs on the flag pays the timeout once per rung."""+ import core.inference.llama_cpp as m++ calls = []+ monkeypatch.setattr(+ m.LlamaCppBackend,+ "_run_list_devices",+ staticmethod(lambda binary, env = None: calls.append(binary) or None),+ )+ m._arm_load_probe_memo()+ try:+ for _ in range(3):+ assert m.LlamaCppBackend._enumerated_gpu_devices("llama-server", {}) is None+ assert len(calls) == 1+ finally:+ m._LOAD_PROBE_STATE.armed = False+ m._LOAD_PROBE_STATE.rows = None++ def test_the_two_probes_do_not_share_a_slot(self, monkeypatch):+ """One memo, two kinds: the device list and the Vulkan discreteness rows."""+ import core.inference.llama_cpp as m++ m._arm_load_probe_memo()+ try:+ m._load_probe_memo_put("devices", "b", ["CUDA0"])+ m._load_probe_memo_put("vulkan", "b", [{"index": 0}])+ assert m._load_probe_memo_get("devices", "b") == ["CUDA0"]+ assert m._load_probe_memo_get("vulkan", "b") == [{"index": 0}]+ finally:+ m._LOAD_PROBE_STATE.armed = False+ m._LOAD_PROBE_STATE.rows = None++ def test_before_arming_every_ask_is_live(self, monkeypatch):+ import core.inference.llama_cpp as m++ calls = []+ monkeypatch.setattr(+ m.LlamaCppBackend,+ "_run_list_devices",+ staticmethod(lambda binary, env = None: calls.append(binary) or ["CUDA0"]),+ )+ m._LOAD_PROBE_STATE.armed = False+ m._LOAD_PROBE_STATE.rows = None+ for _ in range(2):+ m.LlamaCppBackend._enumerated_gpu_devices("llama-server", {})+ assert len(calls) == 2++ def test_the_load_owns_the_scope_and_the_placement_arms_it(self):+ import inspect+ import core.inference.llama_cpp as m++ assert "_load_probe_memo_scope()" in inspect.getsource(m._with_gguf_load_marker)+ assert "_arm_load_probe_memo()" in inspect.getsource(m.LlamaCppBackend.load_model)++ def test_the_scope_clears_at_the_end(self):+ import core.inference.llama_cpp as m++ m._arm_load_probe_memo()+ with m._load_probe_memo_scope():+ pass+ assert m._load_probe_memo_armed() is False+ assert m._load_probe_memo_get("devices", "b") is None+++class TestTheLaunchPublishesItsPlacementWindow:+ """Kept from the class that also held the old CUDA-runtime checks: the window+ itself is still real, the filename checks that shared the class are not."""++ def test_the_snapshot_is_published_before_any_placement_work(self):+ import inspect+ from core.inference.llama_cpp import LlamaCppBackend++ src = inspect.getsource(LlamaCppBackend.load_model)+ set_at = src.index("capture_model_memory_settings(")+ assert set_at < src.index("_arm_load_probe_memo()")+ assert set_at < src.index("_mem_gpu_offload_confirmed = self._gpu_offload_confirmed(")+++class TestTheCaptureAndThePublicationAreOneAct:+ """A launch is committed to the pair from the moment it READS it, so publishing+ afterwards, however soon, left a window where a save was answered from a state in+ which the launch did not exist yet."""++ def _mod(self):+ import utils.model_memory_settings as mm+ return mm++ def test_it_publishes_the_pair_it_read(self, monkeypatch):+ mm = self._mod()+ monkeypatch.setattr(mm, "get_model_memory_settings", lambda: (True, False))+ monkeypatch.setattr(mm, "_pair_generations", lambda: (0, 0))+ seen = []+ assert mm.capture_model_memory_settings(seen.append) == (True, False)+ assert seen[-1] == (True, False)++ def test_a_save_inside_the_window_republishes_the_newer_pair(self, monkeypatch):+ """The write bumps a generation, which is how this module already detects a+ save racing a read."""+ mm = self._mod()+ pairs = [(True, False), (False, True), (False, True)]+ gens = [(0, 0), (0, 1), (0, 1), (0, 1), (0, 1)]++ monkeypatch.setattr(mm, "get_model_memory_settings", lambda: pairs.pop(0))+ monkeypatch.setattr(mm, "_pair_generations", lambda: gens.pop(0))+ seen = []+ final = mm.capture_model_memory_settings(seen.append)+ # published stale, then corrected, and both values agree at the end+ assert seen == [(True, False), (False, True)]+ assert final == (False, True) == seen[-1]++ def test_a_quiet_capture_publishes_once(self, monkeypatch):+ mm = self._mod()+ monkeypatch.setattr(mm, "get_model_memory_settings", lambda: (False, True))+ monkeypatch.setattr(mm, "_pair_generations", lambda: (3, 7))+ seen = []+ mm.capture_model_memory_settings(seen.append)+ assert len(seen) == 1++ def test_a_write_storm_still_terminates(self, monkeypatch):+ """Bounded like the read it mirrors, so a pathological writer cannot spin."""+ mm = self._mod()+ counter = iter(range(10_000))+ monkeypatch.setattr(mm, "get_model_memory_settings", lambda: (True, True))+ monkeypatch.setattr(mm, "_pair_generations", lambda: (next(counter), 0))+ seen = []+ assert mm.capture_model_memory_settings(seen.append) == (True, True)+ assert len(seen) == mm._MAX_REREADS+++class TestTheProbeSeesWhatTheChildWillSee:+ """On Windows the CUDA runtime comes from the managed venv, and only+ `_llama_server_env_for_binary` puts it on PATH. Probing the raw environment+ reported no devices for installs whose child loads CUDA perfectly."""++ def test_the_probe_builds_the_child_env(self, monkeypatch):+ import core.inference.llama_cpp as m++ seen = {}+ monkeypatch.setattr(+ m.LlamaCppBackend,+ "_llama_server_env_for_binary",+ staticmethod(lambda b, **kw: {"PATH": "/venv/torch/lib", "KEEP": "1"}),+ )++ class _R:+ returncode = 0+ stdout = "Available devices:\n CUDA0: x (1 MiB, 1 MiB free)\n"++ monkeypatch.setattr(m.subprocess, "run", lambda *a, **kw: seen.update(kw) or _R())+ assert m.LlamaCppBackend._run_list_devices("llama-server", None) == ["CUDA0"]+ assert seen["env"]["PATH"] == "/venv/torch/lib"++ def test_the_callers_placement_removals_are_replayed(self, monkeypatch):+ """A manual-mode or gpu_ids load hides devices from the child on purpose, so+ the probe must not see them either -- while keeping the native paths."""+ import core.inference.llama_cpp as m++ monkeypatch.setenv("CUDA_VISIBLE_DEVICES", "1")+ monkeypatch.setattr(+ m.LlamaCppBackend,+ "_llama_server_env_for_binary",+ staticmethod(lambda b, **kw: {"PATH": "/venv/torch/lib", "CUDA_VISIBLE_DEVICES": "1"}),+ )+ seen = {}++ class _R:+ returncode = 0+ stdout = "Available devices:\n"++ monkeypatch.setattr(m.subprocess, "run", lambda *a, **kw: seen.update(kw) or _R())+ # the caller's view dropped the pin, so the probe drops it too+ m.LlamaCppBackend._run_list_devices("llama-server", {"PATH": "ignored"})+ assert "CUDA_VISIBLE_DEVICES" not in seen["env"]+ assert seen["env"]["PATH"] == "/venv/torch/lib"++ def test_an_unbuildable_env_still_probes(self, monkeypatch):+ import core.inference.llama_cpp as m++ def _boom(b, **kw):+ raise OSError("no")++ monkeypatch.setattr(m.LlamaCppBackend, "_llama_server_env_for_binary", staticmethod(_boom))++ class _R:+ returncode = 0+ stdout = "Available devices:\n CUDA0: x (1 MiB, 1 MiB free)\n"++ monkeypatch.setattr(m.subprocess, "run", lambda *a, **kw: _R())+ assert m.LlamaCppBackend._run_list_devices("llama-server", None) == ["CUDA0"]+++class TestOnlyAClassifiableDeviceConfirms:+ """`--list-devices` proves a device is LIVE, not discrete. A SYCL or OpenCL build+ on an integrated Intel GPU enumerates `SYCL0` happily, and nothing here can tell+ its VRAM is system RAM."""++ def _confirm(+ self,+ monkeypatch,+ devices,+ discrete = True,+ rocm_classified = True,+ ):+ from core.inference.llama_cpp import LlamaCppBackend++ monkeypatch.setattr(+ LlamaCppBackend,+ "_enumerated_gpu_devices",+ classmethod(lambda cls, binary = None, env = None: devices),+ )+ monkeypatch.setattr(+ LlamaCppBackend,+ "_vulkan_offload_is_discrete",+ staticmethod(lambda binary, idx = None: discrete),+ )+ monkeypatch.setattr(+ LlamaCppBackend,+ "_rocm_classification_answered",+ staticmethod(lambda: rocm_classified),+ )+ return LlamaCppBackend._gpu_offload_confirmed("llama-server", {}, None, False, True)++ @pytest.mark.parametrize("device", ["CUDA0", "ROCm0", "HIP0"])+ def test_a_backend_classified_upstream_confirms(self, monkeypatch, device):+ assert self._confirm(monkeypatch, [device]) is True++ @pytest.mark.parametrize("device", ["ROCm0", "HIP0"])+ def test_an_amd_device_nothing_classified_declines(self, monkeypatch, device):+ """The empty set means "no APU here" OR "no ROCm torch to ask", and only the+ first is a fact about the hardware. There is no Windows ROCm wheel, so on the+ one platform this runs on the second is the usual case."""+ assert self._confirm(monkeypatch, [device], rocm_classified = False) is False++ def test_cuda_does_not_need_the_rocm_classifier(self, monkeypatch):+ """It has its own upstream answer, so the AMD gate must not narrow it."""+ assert self._confirm(monkeypatch, ["CUDA0"], rocm_classified = False) is True++ @pytest.mark.parametrize("device", ["SYCL0", "OpenCL0", "MUSA0", "CANN0", "Metal0"])+ def test_a_backend_that_can_be_integrated_declines(self, monkeypatch, device):+ assert self._confirm(monkeypatch, [device]) is False++ def test_a_mixed_set_needs_every_device_classifiable(self, monkeypatch):+ assert self._confirm(monkeypatch, ["CUDA0", "SYCL1"]) is False++ def test_vulkan_still_goes_through_its_probe(self, monkeypatch):+ assert self._confirm(monkeypatch, ["Vulkan0"], discrete = False) is False+ assert self._confirm(monkeypatch, ["Vulkan0"], discrete = True) is True++ def test_the_backend_is_read_off_the_id(self):+ from core.inference.llama_cpp import LlamaCppBackend as B++ assert B._device_backend("CUDA0") == "cuda"+ assert B._device_backend("Vulkan10") == "vulkan"+ assert B._device_backend("ROCm0") == "rocm"+ assert B._device_backend("SYCL0") == "sycl"+++class TestEverySelectedBackendMustBeClassifiable:+ """A multi-backend build can enumerate a discrete Vulkan device beside an+ unclassifiable SYCL one. Returning whichever kind was found first confirmed a set+ that still had weights on an integrated GPU."""++ def _confirm(+ self,+ monkeypatch,+ devices,+ discrete = True,+ ):+ from core.inference.llama_cpp import LlamaCppBackend++ monkeypatch.setattr(+ LlamaCppBackend,+ "_enumerated_gpu_devices",+ classmethod(lambda cls, binary = None, env = None: devices),+ )+ monkeypatch.setattr(+ LlamaCppBackend,+ "_vulkan_offload_is_discrete",+ staticmethod(lambda binary, idx = None: discrete),+ )+ return LlamaCppBackend._gpu_offload_confirmed("llama-server", {}, None, False, True)++ def test_a_discrete_vulkan_beside_an_unclassifiable_device_declines(self, monkeypatch):+ assert self._confirm(monkeypatch, ["Vulkan0", "SYCL1"], discrete = True) is False++ def test_a_discrete_vulkan_beside_a_known_discrete_device_confirms(self, monkeypatch):+ assert self._confirm(monkeypatch, ["Vulkan0", "CUDA1"], discrete = True) is True++ def test_that_pair_still_obeys_the_vulkan_probe(self, monkeypatch):+ assert self._confirm(monkeypatch, ["Vulkan0", "CUDA1"], discrete = False) is False++ def test_an_unclassifiable_device_alone_declines(self, monkeypatch):+ assert self._confirm(monkeypatch, ["OpenCL0"]) is False+++class TestTheDeviceMemoFollowsVisibility:+ """The child enumerates what the environment lets it see, so the memo is keyed on+ that too. Keyed on the binary alone, a rung that masks an unsupported adapter kept+ the original set's failure and could never gain DirectIO."""++ def test_a_changed_mask_re_probes(self, monkeypatch):+ import core.inference.llama_cpp as m++ calls = []++ def _run(binary, env = None):+ calls.append(dict(env or {}))+ return ["CUDA0"] if (env or {}).get("ROCR_VISIBLE_DEVICES") else None++ monkeypatch.setattr(m.LlamaCppBackend, "_run_list_devices", staticmethod(_run))+ m._arm_load_probe_memo()+ try:+ assert m.LlamaCppBackend._enumerated_gpu_devices("b", {}) is None+ # the gate masks the bad adapter; the narrowed child is a new question+ assert m.LlamaCppBackend._enumerated_gpu_devices(+ "b", {"ROCR_VISIBLE_DEVICES": "1"}+ ) == ["CUDA0"]+ assert len(calls) == 2+ finally:+ m._LOAD_PROBE_STATE.armed = False+ m._LOAD_PROBE_STATE.rows = None++ def test_the_same_visibility_is_still_probed_once(self, monkeypatch):+ import core.inference.llama_cpp as m++ calls = []+ monkeypatch.setattr(+ m.LlamaCppBackend,+ "_run_list_devices",+ staticmethod(lambda binary, env = None: calls.append(1) or ["CUDA0"]),+ )+ m._arm_load_probe_memo()+ try:+ for _ in range(3):+ m.LlamaCppBackend._enumerated_gpu_devices("b", {"CUDA_VISIBLE_DEVICES": "0"})+ assert len(calls) == 1+ finally:+ m._LOAD_PROBE_STATE.armed = False+ m._LOAD_PROBE_STATE.rows = None++ def test_the_key_covers_the_masks_that_matter(self):+ from core.inference.llama_cpp import _DEVICE_VISIBILITY_ENV+ assert {+ "CUDA_VISIBLE_DEVICES",+ "HIP_VISIBLE_DEVICES",+ "ROCR_VISIBLE_DEVICES",+ "GGML_VK_VISIBLE_DEVICES",+ "GGML_BACKEND_PATH",+ } <= set(_DEVICE_VISIBILITY_ENV)+++class TestANarrowingRungProbesItsOwnVisibility:+ """`_mem_env` is the snapshot from before the gate, so probing with it reused the+ unnarrowed set's verdict. Keying the memo on visibility only helps if the rung+ actually passes the narrowed environment in."""++ def test_every_rung_passes_the_child_environment(self):+ import inspect+ from core.inference.llama_cpp import LlamaCppBackend++ flat = "".join(inspect.getsource(LlamaCppBackend.load_model).split())+ assert flat.count("child_env=env") == 3++ def test_the_view_takes_visibility_from_the_child(self, monkeypatch):+ import core.inference.llama_cpp as m++ # the helper is a closure, so exercise its rule directly+ base = {"KEEP": "1", "CUDA_VISIBLE_DEVICES": "0,1"}+ child = {"CUDA_VISIBLE_DEVICES": "1"}+ view = dict(base)+ for name in m._DEVICE_VISIBILITY_ENV:+ if name in child:+ view[name] = child[name]+ else:+ view.pop(name, None)+ assert view == {"KEEP": "1", "CUDA_VISIBLE_DEVICES": "1"}+ assert m.LlamaCppBackend._device_visibility_key(+ view+ ) != m.LlamaCppBackend._device_visibility_key(base)+++class TestTheStripNeverEatsAUserAuthoredPair:+ """The record says a managed pair exists somewhere, not that it is in THIS argv.+ A rung that strips a copy keeps the record for `cmd`, so an argv built from that+ copy matched again and removed the user's own `--load-mode dio`."""++ def _backend(self):+ from core.inference.llama_cpp import LlamaCppBackend++ b = LlamaCppBackend.__new__(LlamaCppBackend)+ b._memory_dio_flags = ["--load-mode", "dio"]+ b._memory_dio_user_tokens = []+ b._memory_dio_applicable = True+ b._memory_policy_active = True+ b._memory_policy_extras_touched = False+ return b++ def test_the_managed_pair_is_still_removed(self):+ b = self._backend()+ argv = ["llama-server", "--load-mode", "dio"]+ assert b._drop_managed_dio(argv, "test") == ["llama-server"]++ def test_a_user_pair_alone_survives(self):+ b = self._backend()+ b._memory_dio_user_tokens = ["--load-mode", "dio"]+ argv = ["llama-server", "--load-mode", "dio"]+ # only the user's occurrence is left, so there is nothing of ours to take+ assert b._drop_managed_dio(argv, "test") == argv++ def test_ours_goes_and_theirs_stays_when_both_are_present(self):+ b = self._backend()+ b._memory_dio_user_tokens = ["--load-mode", "dio"]+ argv = ["llama-server", "--load-mode", "dio", "--load-mode", "dio"]+ out = b._drop_managed_dio(argv, "test")+ assert out == ["llama-server", "--load-mode", "dio"]++ def test_the_applicability_still_clears_either_way(self):+ b = self._backend()+ b._memory_dio_user_tokens = ["--load-mode", "dio"]+ b._drop_managed_dio(["llama-server", "--load-mode", "dio"], "test")+ assert b._memory_dio_applicable is False++ def test_the_counter_is_non_overlapping(self):+ from core.inference.llama_cpp import _count_subsequence++ assert _count_subsequence(["a", "a", "a"], ["a", "a"]) == 1+ assert _count_subsequence(["a", "a", "a", "a"], ["a", "a"]) == 2+ assert _count_subsequence(["x"], ["a"]) == 0+ assert _count_subsequence(["a"], []) == 0+++class TestTheProbeSeesTheNarrowedVisibility:+ """Removals alone left a narrowing rung's new mask out of the probe, so it+ re-enumerated the original adapter set and repeated the failure the narrowing+ existed to clear."""++ def test_a_new_mask_reaches_the_probe(self, monkeypatch):+ import core.inference.llama_cpp as m++ monkeypatch.delenv("CUDA_VISIBLE_DEVICES", raising = False)+ monkeypatch.setattr(+ m.LlamaCppBackend,+ "_llama_server_env_for_binary",+ staticmethod(lambda b, **kw: {"PATH": "/venv/lib"}),+ )+ seen = {}++ class _R:+ returncode = 0+ stdout = "Available devices:\n CUDA0: x (1 MiB, 1 MiB free)\n"++ monkeypatch.setattr(m.subprocess, "run", lambda *a, **kw: seen.update(kw) or _R())+ m.LlamaCppBackend._run_list_devices("b", {"CUDA_VISIBLE_DEVICES": "1", "PATH": "x"})+ assert seen["env"]["CUDA_VISIBLE_DEVICES"] == "1"+ assert seen["env"]["PATH"] == "/venv/lib"++ def test_a_changed_mask_overrides_the_inherited_one(self, monkeypatch):+ import core.inference.llama_cpp as m++ monkeypatch.setenv("ROCR_VISIBLE_DEVICES", "0,1")+ monkeypatch.setattr(+ m.LlamaCppBackend,+ "_llama_server_env_for_binary",+ staticmethod(lambda b, **kw: {"ROCR_VISIBLE_DEVICES": "0,1"}),+ )+ seen = {}++ class _R:+ returncode = 0+ stdout = "Available devices:\n"++ monkeypatch.setattr(m.subprocess, "run", lambda *a, **kw: seen.update(kw) or _R())+ m.LlamaCppBackend._run_list_devices("b", {"ROCR_VISIBLE_DEVICES": "1"})+ assert seen["env"]["ROCR_VISIBLE_DEVICES"] == "1"+++class TestLivenessCountsOrdinalsNotIds:+ """A multi-backend build can enumerate one ordinal twice, as CUDA0 and Vulkan0.+ Counting ids rejected a pinned launch whose devices were all live."""++ def test_one_ordinal_under_two_backends_is_still_covered(self):+ from core.inference.llama_cpp import LlamaCppBackend as B+ assert B._offload_devices_are_live(["CUDA0", "Vulkan0"], [0]) is True++ def test_a_missing_ordinal_still_declines(self):+ from core.inference.llama_cpp import LlamaCppBackend as B+ assert B._offload_devices_are_live(["CUDA0", "Vulkan0"], [0, 1]) is False++ def test_every_requested_ordinal_must_appear(self):+ from core.inference.llama_cpp import LlamaCppBackend as B+ assert B._offload_devices_are_live(["CUDA0", "CUDA1"], [0, 1]) is True+++class TestAPassThroughDeviceOverrideDecidesPlacement:+ """`--device` is appended last and decides where the child really puts the+ weights, so confirming against the auto-selected ordinals could emit DirectIO for+ a host-backed device the user pinned."""++ def _confirm(+ self,+ monkeypatch,+ devices,+ gpu_indices,+ extra_args = None,+ env = None,+ discrete = True,+ ):+ from core.inference.llama_cpp import LlamaCppBackend++ monkeypatch.setattr(+ LlamaCppBackend,+ "_enumerated_gpu_devices",+ classmethod(lambda cls, binary = None, e = None: devices),+ )+ monkeypatch.setattr(+ LlamaCppBackend,+ "_vulkan_offload_is_discrete",+ staticmethod(lambda binary, idx = None: discrete),+ )+ return LlamaCppBackend._gpu_offload_confirmed(+ "llama-server", env or {}, gpu_indices, False, True, extra_args+ )++ def test_an_override_onto_an_unclassifiable_device_declines(self, monkeypatch):+ assert self._confirm(monkeypatch, ["CUDA0", "SYCL1"], [0], ["--device", "SYCL1"]) is False++ def test_an_override_onto_a_discrete_device_confirms(self, monkeypatch):+ assert self._confirm(monkeypatch, ["CUDA0", "SYCL1"], [1], ["--device", "CUDA0"]) is True++ def test_an_override_naming_a_device_the_build_lacks_declines(self, monkeypatch):+ assert self._confirm(monkeypatch, ["CUDA0"], [0], ["--device", "CUDA3"]) is False++ def test_a_cpu_override_declines(self, monkeypatch):+ assert self._confirm(monkeypatch, ["CUDA0"], [0], ["--device", "none"]) is False++ def test_the_env_twin_is_honoured(self, monkeypatch):+ assert (+ self._confirm(monkeypatch, ["CUDA0", "SYCL1"], [0], None, {"LLAMA_ARG_DEVICE": "SYCL1"})+ is False+ )++ def test_argv_beats_the_env_twin(self, monkeypatch):+ assert (+ self._confirm(+ monkeypatch,+ ["CUDA0", "SYCL1"],+ [1],+ ["--device", "CUDA0"],+ {"LLAMA_ARG_DEVICE": "SYCL1"},+ )+ is True+ )++ def test_no_override_still_uses_the_auto_selection(self, monkeypatch):+ assert self._confirm(monkeypatch, ["CUDA0", "SYCL1"], [0]) is True+++class TestOneEffectiveDeviceSetFeedsEveryConsumer:+ """The host-residency verdict and the confirmation have to be about the SAME+ devices. Resolved apart, an override onto a unified-memory APU was priced against+ the discrete card it replaced, and DirectIO went to weights in host RAM."""++ def _resolve(+ self,+ monkeypatch,+ devices,+ gpu_indices,+ extra_args = None,+ env = None,+ ):+ from core.inference.llama_cpp import LlamaCppBackend+ monkeypatch.setattr(+ LlamaCppBackend,+ "_enumerated_gpu_devices",+ classmethod(lambda cls, binary = None, e = None: devices),+ )+ return LlamaCppBackend._effective_gpu_selection(+ "llama-server", env or {}, gpu_indices, extra_args, True+ )[1]++ def test_an_override_replaces_the_auto_selection(self, monkeypatch):+ assert self._resolve(monkeypatch, ["CUDA0", "ROCm1"], [0], ["--device", "ROCm1"]) == [1]++ def test_the_env_twin_counts(self, monkeypatch):+ assert self._resolve(+ monkeypatch, ["CUDA0", "ROCm1"], [0], None, {"LLAMA_ARG_DEVICE": "ROCm1"}+ ) == [1]++ def test_no_override_leaves_the_auto_selection(self, monkeypatch):+ assert self._resolve(monkeypatch, ["CUDA0", "ROCm1"], [0]) == [0]++ def test_a_cpu_override_leaves_it_for_the_confirmation_to_decline(self, monkeypatch):+ assert self._resolve(monkeypatch, ["CUDA0"], [0], ["--device", "none"]) == [0]++ def test_an_unlisted_override_leaves_it_for_the_confirmation_to_decline(self, monkeypatch):+ assert self._resolve(monkeypatch, ["CUDA0"], [0], ["--device", "CUDA7"]) == [0]++ def test_both_consumers_get_the_resolved_set(self):+ import inspect+ from core.inference.llama_cpp import LlamaCppBackend++ flat = "".join(inspect.getsource(LlamaCppBackend.load_model).split())+ # main path: resolved once, then used for residency AND confirmation+ assert "_mem_physical_indices,_mem_compact_indices=self._effective_gpu_selection(" in flat+ assert "gpu_indices=_mem_physical_indices," in flat+ assert "binary,_mem_env,_mem_compact_indices,_mem_host_resident," in flat+ # the rung resolves its own narrowed set the same way+ assert (+ "_rung_physical,_rung_compact=self._effective_gpu_selection(binary,_rung_env,"+ "devices,_mem_extra_args,_mem_dio_possible,is_vulkan_backend,)" in flat+ )+++class TestTheSnapshotIsRetakenAfterEveryCmdMutation:+ """The arch-crash retry restores `_mem_policy_for_cmd` over `cmd`. Any site that+ adds or removes the managed pair on `cmd` has to retake it first, or the retry+ appends a second pair after an addition, or believes an absent one is still there+ after a removal."""++ def _src(self):+ import inspect+ from core.inference.llama_cpp import LlamaCppBackend+ return inspect.getsource(LlamaCppBackend.load_model)++ def test_the_snapshot_comes_from_one_helper(self):+ src = self._src()+ assert "def _snapshot_policy_for_cmd():" in src+ assert "_mem_policy_for_cmd = (" not in src # no inline copies left++ def test_the_proactive_gate_retakes_it_on_both_arms(self):+ src = self._src()+ gate = src[src.index("the arch gate's surviving GPU(s) no longer confirm a") :]+ gate = gate[: gate.index("_gated_carveout_need")]+ # once for the removal arm, once for the addition arm+ assert gate.count("_mem_policy_for_cmd = _snapshot_policy_for_cmd()") == 2++ def test_every_mutation_before_the_restore_retakes_it(self):+ """A `_record_memory_state(cmd, ...)` means `cmd` changed meaning. Only the+ mutations BEFORE the arch-crash restore matter; the ones inside the retry run+ after the snapshot has been consumed and have nothing left to feed."""+ src = self._src()+ lines = [ln.strip() for ln in src.splitlines()]+ restore = next(i for i, ln in enumerate(lines) if ln.endswith("= _mem_policy_for_cmd"))+ checked = 0+ for i, line in enumerate(lines[:restore]):+ if line == "self._record_memory_state(cmd, env)":+ checked += 1+ window = lines[i + 1 : i + 4]+ assert any(+ "_mem_policy_for_cmd = _snapshot_policy_for_cmd()" in w for w in window+ ), f"no retake after the cmd mutation at offset {i}"+ assert checked >= 4, "expected several pre-restore cmd mutations to guard"+++class TestMaskedDevicesCompareInCompactSpace:+ """A visibility mask reindexes survivors from 0, so the adapter this launch calls+ physical 1 is `CUDA0` to the child. Comparing physical against compact rejected a+ narrowed launch that was fully offloaded, the opposite of the narrowing's purpose."""++ def test_a_survivor_is_translated_to_its_compact_ordinal(self):+ from core.inference.llama_cpp import LlamaCppBackend as B+ assert B._compact_ordinals([1], {"CUDA_VISIBLE_DEVICES": "1"}) == [0]++ def test_the_position_in_the_mask_is_the_ordinal(self):+ from core.inference.llama_cpp import LlamaCppBackend as B+ assert B._compact_ordinals([3], {"ROCR_VISIBLE_DEVICES": "2,3"}) == [1]+ assert B._compact_ordinals([2, 3], {"ROCR_VISIBLE_DEVICES": "2,3"}) == [0, 1]++ def test_no_mask_leaves_the_ordinals_alone(self):+ from core.inference.llama_cpp import LlamaCppBackend as B+ assert B._compact_ordinals([1], {}) == [1]+ assert B._compact_ordinals([1], {"CUDA_VISIBLE_DEVICES": ""}) == [1]++ def test_an_unmappable_mask_leaves_them_alone(self):+ from core.inference.llama_cpp import LlamaCppBackend as B+ assert B._compact_ordinals([1], {"CUDA_VISIBLE_DEVICES": "GPU-abc"}) == [1]++ def test_a_masked_survivor_now_reads_as_live(self, monkeypatch):+ """The end-to-end shape of the bug: mask keeps physical 1, the child reports+ CUDA0, and the launch must still see its pinned device as enumerated."""+ from core.inference.llama_cpp import LlamaCppBackend as B++ monkeypatch.setattr(+ B,+ "_enumerated_gpu_devices",+ classmethod(lambda cls, binary = None, env = None: ["CUDA0"]),+ )+ effective = B._effective_gpu_selection(+ "llama-server", {"CUDA_VISIBLE_DEVICES": "1"}, [1], None, True+ )[1]+ assert effective == [0]+ assert B._offload_devices_are_live(["CUDA0"], effective) is True+++class TestSplitModeNoneFollowsTheMainGpu:+ """llama.cpp keeps only `devices[main_gpu]`, and `--main-gpu` can name a device+ OUTSIDE the automatic plan, so it replaces the selection rather than narrowing it.+ Confirming the plan's discrete card while the child used a shared-memory iGPU is+ exactly the case DirectIO must not take."""++ def _eff(+ self,+ monkeypatch,+ gpu_indices,+ extra_args = None,+ env = None,+ devices = ("CUDA0", "Vulkan1"),+ ):+ from core.inference.llama_cpp import LlamaCppBackend as B+ monkeypatch.setattr(+ B,+ "_enumerated_gpu_devices",+ classmethod(lambda cls, binary = None, e = None: list(devices)),+ )+ return B._effective_gpu_selection("llama-server", env or {}, gpu_indices, extra_args, True)[+ 0+ ]++ def test_main_gpu_replaces_the_plan(self, monkeypatch):+ assert self._eff(monkeypatch, [0], ["--split-mode", "none", "--main-gpu", "1"]) == [1]++ def test_the_short_flag_counts(self, monkeypatch):+ assert self._eff(monkeypatch, [0], ["-sm", "none", "-mg", "1"]) == [1]++ def test_the_env_twin_counts(self, monkeypatch):+ assert self._eff(+ monkeypatch, [0], ["--split-mode", "none"], {"LLAMA_ARG_MAIN_GPU": "1"}+ ) == [1]++ def test_without_split_mode_none_it_is_not_a_replacement(self, monkeypatch):+ """With a layer split the weights spread over the whole selection, so the plan+ is still what to check."""+ assert self._eff(monkeypatch, [0], ["--main-gpu", "1"]) == [0]++ def test_an_unparsable_main_gpu_leaves_the_plan(self, monkeypatch):+ assert self._eff(monkeypatch, [0], ["--split-mode", "none", "--main-gpu", "nope"]) == [0]++ def test_the_redirected_device_is_what_gets_classified(self, monkeypatch):+ """The whole point: a Vulkan iGPU target must reach the Vulkan probe rather+ than being confirmed off the plan's CUDA card."""+ from core.inference.llama_cpp import LlamaCppBackend as B++ monkeypatch.setattr(+ B,+ "_enumerated_gpu_devices",+ classmethod(lambda cls, binary = None, e = None: ["CUDA0", "Vulkan1"]),+ )+ monkeypatch.setattr(+ B, "_vulkan_offload_is_discrete", staticmethod(lambda binary, idx = None: False)+ )+ effective = B._effective_gpu_selection(+ "llama-server", {}, [0], ["--split-mode", "none", "--main-gpu", "1"], True+ )[1]+ assert effective == [1]+ assert (+ B._gpu_offload_confirmed(+ "llama-server",+ {},+ effective,+ False,+ True,+ ["--split-mode", "none", "--main-gpu", "1"],+ )+ is False+ )++ def test_a_main_gpu_override_is_also_gated(self, monkeypatch):+ """Off the DirectIO path nothing is resolved, so no behaviour changes."""+ from core.inference.llama_cpp import LlamaCppBackend as B+ assert B._effective_gpu_selection(+ "llama-server", {}, [0], ["--split-mode", "none", "--main-gpu", "1"], False+ )[1] == [0]+++class TestVulkanOrdinalsAreAlreadyCompact:+ """ggml passes GGML_VK_VISIBLE_DEVICES through and enumerates what survives, so a+ Vulkan ordinal is already compact, the same thing `_run_vulkan_probe` documents+ about its own rows. Translating one again pointed the residency and discreteness+ checks at a different probe row."""++ def test_a_vulkan_launch_is_not_translated(self):+ from core.inference.llama_cpp import LlamaCppBackend as B+ assert B._compact_ordinals([1], {"GGML_VK_VISIBLE_DEVICES": "1,2"}, True) == [1]++ def test_the_vulkan_mask_is_not_even_consulted(self):+ from core.inference.llama_cpp import LlamaCppBackend as B+ assert B._compact_ordinals([1], {"GGML_VK_VISIBLE_DEVICES": "1,2"}) == [1]++ def test_a_cuda_launch_is_still_translated(self):+ from core.inference.llama_cpp import LlamaCppBackend as B+ assert B._compact_ordinals([1], {"CUDA_VISIBLE_DEVICES": "1,2"}) == [0]++ def test_the_resolver_passes_the_backend_through(self, monkeypatch):+ from core.inference.llama_cpp import LlamaCppBackend as B++ monkeypatch.setattr(+ B,+ "_enumerated_gpu_devices",+ classmethod(lambda cls, binary = None, env = None: ["Vulkan0", "Vulkan1"]),+ )+ env = {"GGML_VK_VISIBLE_DEVICES": "1,2"}+ assert B._effective_gpu_selection("b", env, [1], None, True, True)[1] == [1]+++class TestAnUnreadDeviceTypeIsNotDiscreteEvidence:+ """The probe reports all-False when the type query fails, so "not integrated" and+ "could not read the type" shared a value. That default is right for the page-lock+ caller and wrong for a loader choice."""++ def _rows(self, monkeypatch, rows):+ from core.inference.llama_cpp import LlamaCppBackend as B+ monkeypatch.setattr(B, "_run_vulkan_probe", staticmethod(lambda binary = None: rows))+ return B._vulkan_offload_is_discrete("llama-server", None)++ def test_a_known_discrete_device_confirms(self, monkeypatch):+ assert self._rows(monkeypatch, [{"index": 0, "is_igpu": False, "type_known": True}]) is True++ def test_an_unread_type_declines(self, monkeypatch):+ assert (+ self._rows(monkeypatch, [{"index": 0, "is_igpu": False, "type_known": False}]) is False+ )++ def test_one_unread_device_declines_the_set(self, monkeypatch):+ assert (+ self._rows(+ monkeypatch,+ [+ {"index": 0, "is_igpu": False, "type_known": True},+ {"index": 1, "is_igpu": False, "type_known": False},+ ],+ )+ is False+ )++ def test_a_known_igpu_still_declines(self, monkeypatch):+ assert self._rows(monkeypatch, [{"index": 0, "is_igpu": True, "type_known": True}]) is False++ def test_an_older_probe_without_the_column_reads_as_unknown(self):+ """A staged probe predating the column cannot tell the two apart, so it+ declines rather than being trusted."""+ from core.inference.llama_cpp import LlamaCppBackend as B++ assert B._vulkan_offload_is_discrete.__doc__ # predicate still documented+ row = {"index": 0, "is_igpu": False}+ assert row.get("type_known") is None++ def test_the_probe_emits_the_column(self):+ import inspect+ from core.inference import _vulkan_probe++ src = inspect.getsource(_vulkan_probe)+ assert "known[i] = True" in src+ assert "int(known[i])" in src+++class TestThePerModelPairIsUserAuthoredToo:+ """A per-model `dio` selection is the user's choice as much as a hand-typed flag.+ Counting only the extras, a rung that stripped a COPY left the per-model pair+ behind and the next strip took that one instead."""++ def test_the_launch_records_the_per_model_pair(self):+ import inspect+ from core.inference.llama_cpp import LlamaCppBackend++ flat = "".join(inspect.getsource(LlamaCppBackend.load_model).split())+ assert (+ "self._memory_dio_user_tokens=[*self._memory_dio_user_tokens,*_load_mode_managed,]"+ in flat+ )++ def test_a_per_model_pair_alone_survives_the_strip(self):+ from core.inference.llama_cpp import LlamaCppBackend++ b = LlamaCppBackend.__new__(LlamaCppBackend)+ b._memory_dio_flags = ["--load-mode", "dio"]+ # extras empty, but the per-model selection put one pair on the command+ b._memory_dio_user_tokens = ["--load-mode", "dio"]+ b._memory_dio_applicable = True+ b._memory_policy_active = True+ b._memory_policy_extras_touched = False+ argv = ["llama-server", "--load-mode", "dio"]+ assert b._drop_managed_dio(argv, "test") == argv+++class TestResidencyWithdrawsTheNoReserveDio:+ """A no-reserve launch's managed DirectIO has to go when no-reserve does: with+ residency on and no-reserve off the policy emits a page-lock or nothing, never+ dio, so a streaming child contradicts the new settings."""++ def _satisfied(+ self,+ monkeypatch,+ *,+ direct_io,+ dio_applicable,+ policy_active,+ mlock_applicable = False,+ state = (False, False),+ dio_managed = None,+ ):+ import utils.model_memory_settings as mm+ from core.inference.llama_server_args import memory_state_satisfies_settings++ monkeypatch.setattr(mm, "get_no_ram_reserve", lambda: False)+ monkeypatch.setattr(mm, "get_keep_resident", lambda: True)+ return memory_state_satisfies_settings(+ state,+ policy_active,+ mlock_applicable,+ direct_io,+ dio_applicable,+ policy_active if dio_managed is None else dio_managed,+ )++ def test_the_aggregate_activity_bit_is_not_what_decides(self, monkeypatch):+ """A scrubbed env var or a vetoed extra also sets `policy_active`, so a user's+ own dio beside an inherited LLAMA_ARG_MLOCK matched it and every relaunch+ produced the same child and set it again, leaving reload_required stuck on."""+ assert (+ self._satisfied(+ monkeypatch,+ direct_io = True,+ dio_applicable = True,+ policy_active = True,+ dio_managed = False,+ )+ is True+ )++ def test_an_active_managed_dio_demands_a_reload(self, monkeypatch):+ assert (+ self._satisfied(monkeypatch, direct_io = True, dio_applicable = True, policy_active = True)+ is False+ )++ def test_a_user_authored_dio_does_not(self, monkeypatch):+ """Theirs to keep; the policy never touched this child."""+ assert (+ self._satisfied(monkeypatch, direct_io = True, dio_applicable = True, policy_active = False)+ is True+ )++ def test_a_non_streaming_child_is_unaffected(self, monkeypatch):+ assert (+ self._satisfied(monkeypatch, direct_io = False, dio_applicable = True, policy_active = True)+ is True+ )++ def test_the_mlock_case_still_answers_as_before(self, monkeypatch):+ assert (+ self._satisfied(+ monkeypatch,+ direct_io = None,+ dio_applicable = False,+ policy_active = False,+ mlock_applicable = True,+ state = (True, False),+ )+ is True+ )+++class TestTheSelectionCarriesBothOrdinalSpaces:+ """The consumers need the SAME selection in DIFFERENT spaces, and collapsing them+ broke one or the other: `_amd_apu_wants_unified_memory` is documented to take+ physical ids, while `--list-devices` reports the child's compact ordinals."""++ def _sel(+ self,+ monkeypatch,+ gpu_indices,+ env = None,+ extra_args = None,+ devices = ("CUDA0",),+ is_vulkan = False,+ ):+ from core.inference.llama_cpp import LlamaCppBackend as B+ monkeypatch.setattr(+ B,+ "_enumerated_gpu_devices",+ classmethod(lambda cls, binary = None, e = None: list(devices)),+ )+ return B._effective_gpu_selection(+ "llama-server", env or {}, gpu_indices, extra_args, True, is_vulkan+ )++ def test_a_mask_splits_the_two_spaces(self, monkeypatch):+ """Physical APU 1 exposed as compact 0: residency must still classify 1."""+ physical, compact = self._sel(monkeypatch, [1], {"CUDA_VISIBLE_DEVICES": "1"})+ assert physical == [1]+ assert compact == [0]++ def test_no_mask_leaves_them_equal(self, monkeypatch):+ physical, compact = self._sel(monkeypatch, [1])+ assert physical == compact == [1]++ def test_a_device_override_is_mapped_back_to_physical(self, monkeypatch):+ """`--device` names compact ids, so the physical form is the derived one."""+ physical, compact = self._sel(+ monkeypatch,+ [0],+ {"CUDA_VISIBLE_DEVICES": "2,3"},+ ["--device", "CUDA1"],+ devices = ("CUDA0", "CUDA1"),+ )+ assert compact == [1]+ assert physical == [3]++ def test_main_gpu_is_compact_and_maps_back_to_physical(self, monkeypatch):+ """`--main-gpu` indexes the child's FILTERED list, so with one masked survivor+ `--main-gpu 0` is physical 1."""+ physical, compact = self._sel(+ monkeypatch,+ [0],+ {"CUDA_VISIBLE_DEVICES": "1"},+ ["--split-mode", "none", "--main-gpu", "0"],+ )+ assert compact == [0]+ assert physical == [1]++ def test_vulkan_keeps_the_two_identical(self, monkeypatch):+ physical, compact = self._sel(+ monkeypatch,+ [1],+ {"GGML_VK_VISIBLE_DEVICES": "1,2"},+ devices = ("Vulkan0", "Vulkan1"),+ is_vulkan = True,+ )+ assert physical == compact == [1]++ def test_the_inverse_round_trips(self):+ from core.inference.llama_cpp import LlamaCppBackend as B++ env = {"CUDA_VISIBLE_DEVICES": "2,3"}+ assert B._compact_ordinals([3], env) == [1]+ assert B._physical_ordinals([1], env) == [3]+ assert B._physical_ordinals(B._compact_ordinals([2, 3], env), env) == [2, 3]++ def test_each_consumer_gets_its_own_space(self):+ import inspect+ from core.inference.llama_cpp import LlamaCppBackend++ flat = "".join(inspect.getsource(LlamaCppBackend.load_model).split())+ # residency classifies physical ids+ assert "gpu_indices=_mem_physical_indices," in flat+ # the confirmation matches the build's compact device list+ assert "binary,_mem_env,_mem_compact_indices,_mem_host_resident," in flat+ # and the rung does the same with its narrowed set+ assert "gpu_indices=_rung_physical," in flat+ assert "self._gpu_offload_confirmed(binary,_rung_env,_rung_compact,host_resident," in flat+++class TestTheVulkanProbeMustCoverItsOwnTargets:+ """`gpu_indices` carries ordinals from every selected backend, but the probe+ reports Vulkan rows. Passing all of them let a CUDA target's 0 satisfy Vulkan row+ 0 and leave the real Vulkan device unprobed but treated as discrete."""++ def _confirm(self, monkeypatch, devices, rows):+ from core.inference.llama_cpp import LlamaCppBackend as B++ monkeypatch.setattr(+ B,+ "_enumerated_gpu_devices",+ classmethod(lambda cls, binary = None, env = None: list(devices)),+ )+ monkeypatch.setattr(B, "_run_vulkan_probe", staticmethod(lambda binary = None: rows))+ return B._gpu_offload_confirmed("llama-server", {}, None, False, True)++ def test_a_cuda_ordinal_cannot_satisfy_a_vulkan_row(self, monkeypatch):+ """Selected CUDA0 and Vulkan1, probe holds only Vulkan0: the Vulkan target is+ unprobed, so this must decline."""+ assert (+ self._confirm(+ monkeypatch,+ ["CUDA0", "Vulkan1"],+ [{"index": 0, "is_igpu": False, "type_known": True}],+ )+ is False+ )++ def test_covering_the_vulkan_target_confirms(self, monkeypatch):+ assert (+ self._confirm(+ monkeypatch,+ ["CUDA0", "Vulkan1"],+ [{"index": 1, "is_igpu": False, "type_known": True}],+ )+ is True+ )++ def test_an_igpu_on_the_covered_target_still_declines(self, monkeypatch):+ assert (+ self._confirm(+ monkeypatch,+ ["CUDA0", "Vulkan1"],+ [{"index": 1, "is_igpu": True, "type_known": True}],+ )+ is False+ )++ def test_the_predicate_requires_full_coverage(self, monkeypatch):+ from core.inference.llama_cpp import LlamaCppBackend as B++ monkeypatch.setattr(+ B,+ "_run_vulkan_probe",+ staticmethod(lambda binary = None: [{"index": 0, "is_igpu": False, "type_known": True}]),+ )+ assert B._vulkan_offload_is_discrete("llama-server", [0]) is True+ assert B._vulkan_offload_is_discrete("llama-server", [0, 1]) is False+++class TestAForcedProbeKeepsMlockConservative:+ """The probe is forced for the DirectIO decision even when the page-lock question+ would not have run it. An unreadable Vulkan probe answers "not an iGPU", which+ turned host residency off and recorded the lock as inapplicable, and the+ comparator then excused a missing lock on weights that may be host-backed."""++ def test_an_unanswered_probe_keeps_the_lock_applicable(self):+ import inspect+ from core.inference.llama_cpp import LlamaCppBackend++ flat = "".join(inspect.getsource(LlamaCppBackend.load_model).split())+ assert (+ "self._memory_mlock_applicable=_mem_host_residentor("+ "_mem_probe_for_dioandnot_mem_should_mlockandis_vulkan_backendand"+ "notself._vulkan_probe_answered(binary))" in flat+ )++ def test_the_answered_predicate_is_not_the_igpu_one(self, monkeypatch):+ from core.inference.llama_cpp import LlamaCppBackend as B++ monkeypatch.setattr(B, "_run_vulkan_probe", staticmethod(lambda binary = None: []))+ assert B._vulkan_probe_answered("llama-server") is False+ # and the iGPU question still answers False for the same empty probe, which is+ # why it cannot be read as a measurement+ assert B._vulkan_targets_are_igpus("llama-server", None) is False++ monkeypatch.setattr(+ B,+ "_run_vulkan_probe",+ staticmethod(lambda binary = None: [{"index": 0, "is_igpu": False, "type_known": True}]),+ )+ assert B._vulkan_probe_answered("llama-server") is Truediff --git a/studio/backend/utils/model_memory_settings.py b/studio/backend/utils/model_memory_settings.pyindex 20ede1c4ea2..0befef56567 100644--- a/studio/backend/utils/model_memory_settings.py+++ b/studio/backend/utils/model_memory_settings.py@@ -6,8 +6,9 @@``keep_resident`` -- weights never go back to system RAM while loaded: no idleauto-unload, and ``--mlock`` so the OS cannot page them out and re-fault them in.-``no_ram_reserve`` -- no full host-RAM copy: keeps llama.cpp's default mmap path-and drops ``--no-mmap`` / ``--mlock``.+``no_ram_reserve`` -- avoids locked or reserved weight buffers. Uses DirectIO on+supported Windows builds when full GPU offload is confirmed, otherwise keeps+llama.cpp's default mmap path. Required CPU buffers can still use host RAM.Both on means "live in VRAM, keep no RAM copy, never idle-unload". ``--mlock`` isitself a full-model RAM reservation, so ``no_ram_reserve`` wins on that flag.@@ -121,6 +122,27 @@ def _pair_generations() -> tuple[int, int]:)+def capture_model_memory_settings(publish) -> tuple[bool, bool]:+ """Read the pair and publish it, with no window in between for a save to fall through.++ ``get_model_memory_settings`` closes the window INSIDE the read; this closes the one+ after it. A launch is committed to the pair from the moment it reads it, so a save+ landing before the publication is answered from a state where the launch does not+ exist yet: ``reload_required=false`` about a child that will run the pre-save flags.++ Detected rather than locked, as this module already handles the read: the write+ bumps a generation, so a capture whose generation moved republishes the newer pair.+ Holding ``_cache_lock`` instead would mean holding it across the read's DB I/O.+ """+ for _attempt in range(_MAX_REREADS):+ before = _pair_generations()+ pair = get_model_memory_settings()+ publish(pair)+ if _pair_generations() == before:+ return pair+ return pair++def get_model_memory_settings() -> tuple[bool, bool]:"""``(keep_resident, no_ram_reserve)`` from ONE coherent snapshot.diff --git a/studio/frontend/src/features/model-picker/components/model-config-page.tsx b/studio/frontend/src/features/model-picker/components/model-config-page.tsxindex c1f4f855ebc..90575e0d7be 100644--- a/studio/frontend/src/features/model-picker/components/model-config-page.tsx+++ b/studio/frontend/src/features/model-picker/components/model-config-page.tsx@@ -191,8 +191,8 @@ const LOAD_MODE_LABELS: Record<(typeof LOAD_MODES)[number], string> = {dio: "DirectIO",};-// What "Don't reserve system RAM" vetoes: mmap maps and dio streams, so neither holds a full-// host copy. Mirrors _LOAD_MODE_MLOCK_VALUES and _LOAD_MODE_RESERVING_VALUES.+// What "Don't reserve system RAM" vetoes: allocated or locked host weight buffers. Windows mmap can still hold the+// file pages resident, which is what the DirectIO policy is for.// Mirrors _LOAD_MODE_MLOCK_VALUES | _LOAD_MODE_RESERVING_VALUES in llama_server_args.py.const RAM_RESERVING_LOAD_MODES = new Set(["none", "mlock", "mmap+mlock"]);@@ -212,7 +212,7 @@ function loadModeOverrideNotice(: "This will be replaced by mmap+mlock: Keep model in GPU memory, in Settings, owns how the weights are held.";}if (settings.noRamReserve && RAM_RESERVING_LOAD_MODES.has(mode)) {- return "This will be removed and the load runs the default mmap path: Don't reserve system RAM, in Settings, owns how the weights are held.";+ return "This will be removed: Don't reserve system RAM, in Settings, chooses the loading policy. Supported Windows builds skip the mapping for a full GPU offload; other placements use the default mmap path.";}return null;}diff --git a/studio/frontend/src/i18n/locales/ar.ts b/studio/frontend/src/i18n/locales/ar.tsindex 75872a70c19..bc4f04d789b 100644--- a/studio/frontend/src/i18n/locales/ar.ts+++ b/studio/frontend/src/i18n/locales/ar.ts@@ -1239,8 +1239,8 @@ export const ar = {keepResidentDescription: "يبقى في ذاكرة كرت الرسوميات بين الرسائل.",keepResidentHint: "لا تُعاد الأوزان إلى ذاكرة النظام ما دام النموذج محمَّلاً. يعطّل التفريغ التلقائي عند الخمول، وإذا كانت الأوزان موجودة فعلاً في ذاكرة المضيف (ذاكرة موحّدة أو تفريغ جزئي إلى المعالج الرسومي) فإنه يمرّر --mlock أيضاً حتى لا يُخرجها النظام من الذاكرة ويعيد رفعها عند رسالتك التالية.",noRamReserve: "عدم حجز ذاكرة النظام للنموذج",- noRamReserveDescription: "لا يحتفظ بنسخة كاملة في ذاكرة النظام.",- noRamReserveHint: "ينقل الأوزان إلى ذاكرة كرت الرسوميات بدل الاحتفاظ بنسخة كاملة في ذاكرة النظام. يُبقي تحميل llama.cpp المعتمد على تعيين الذاكرة ويحذف --no-mmap و--mlock.",+ noRamReserveDescription: "يقلل ذاكرة النظام المحجوزة لأوزان النموذج.",+ noRamReserveHint: "يتخطى التحميل المعتمد على تعيين الذاكرة في إصدارات ويندوز المدعومة عندما يكون النموذج محمّلاً بالكامل على كرت الرسوميات، فلا تبقى صفحاته مقيمة. وإلا يُبقي التحميل المعتمد على تعيين الذاكرة. قد تستهلك مخازن المعالج المطلوبة ذاكرة النظام. ويحذف --no-mmap و--mlock.",mlockVetoed: "يبقى --mlock معطَّلاً: تثبيت النموذج سيحجز ذاكرة تكفي حجمه كاملاً. ولا يزال التفريغ التلقائي عند الخمول معطَّلاً.",memlockCapped: "يحدّ هذا النظام الذاكرة المقفلة عند {limit}. لن يُثبَّت نموذج أكبر بالكامل؛ ارفع الحد باستخدام ulimit -l.",reloadRequired: "أعد تحميل النموذج لتطبيق خيارات الذاكرة الجديدة.",diff --git a/studio/frontend/src/i18n/locales/de.ts b/studio/frontend/src/i18n/locales/de.tsindex 1e71cdc2f15..396058cb508 100644--- a/studio/frontend/src/i18n/locales/de.ts+++ b/studio/frontend/src/i18n/locales/de.ts@@ -1266,8 +1266,8 @@ export const de = {keepResidentDescription: "Zwischen Prompts im VRAM bleiben.",keepResidentHint: "Die Gewichte werden nicht an den System-RAM zurückgegeben, solange das Modell geladen bleibt. Deaktiviert das automatische Entladen im Leerlauf und übergibt zusätzlich --mlock, wenn die Gewichte tatsächlich im Host-RAM liegen (Unified Memory oder teilweises GPU-Offload), damit das Betriebssystem sie nicht auslagert und beim nächsten Prompt neu hochlädt.",noRamReserve: "Keinen System-RAM für das Modell reservieren",- noRamReserveDescription: "Keine vollständige Kopie im RAM behalten.",- noRamReserveHint: "Die Gewichte werden in den VRAM gestreamt, statt eine vollständige Kopie im RAM zu halten. Behält das speicherabgebildete Laden von llama.cpp bei und entfernt --no-mmap und --mlock.",+ noRamReserveDescription: "Weniger RAM für die Modellgewichte belegen.",+ noRamReserveHint: "Überspringt auf unterstützten Windows-Builds das speicherabgebildete Laden, wenn das Modell vollständig auf die GPU ausgelagert ist, sodass dessen Seiten nicht im Speicher gehalten werden. Andernfalls bleibt das speicherabgebildete Laden erhalten. Erforderliche CPU-Puffer können weiterhin RAM belegen. Entfernt --no-mmap und --mlock.",mlockVetoed: "--mlock bleibt aus: das Fixieren des Modells würde RAM für das gesamte Modell reservieren. Das automatische Entladen im Leerlauf bleibt deaktiviert.",memlockCapped: "Dieses System begrenzt gesperrten Speicher auf {limit}. Ein größeres Modell wird nicht vollständig fixiert; erhöhen Sie das Limit mit ulimit -l.",reloadRequired: "Modell neu laden, um die neuen Speicheroptionen anzuwenden.",diff --git a/studio/frontend/src/i18n/locales/en.ts b/studio/frontend/src/i18n/locales/en.tsindex 414b8f6c033..198aca96171 100644--- a/studio/frontend/src/i18n/locales/en.ts+++ b/studio/frontend/src/i18n/locales/en.ts@@ -1233,8 +1233,8 @@ export const en = {keepResidentDescription: "Stay in VRAM between prompts.",keepResidentHint: "Don't hand the weights back to system RAM while the model stays loaded. Turns off idle auto-unload, and when the weights do sit in host RAM (unified memory, or a partial GPU offload) it also passes --mlock so the OS can't page them out and re-upload them on your next prompt.",noRamReserve: "Don't reserve system RAM for the model",- noRamReserveDescription: "Keep no full copy in RAM.",- noRamReserveHint: "Stream the weights into VRAM instead of keeping a full copy in RAM. Keeps llama.cpp's memory-mapped loading and drops --no-mmap and --mlock.",+ noRamReserveDescription: "Reduce host RAM held for model weights.",+ noRamReserveHint: "Skips memory-mapped loading on supported Windows builds when the model is fully offloaded to the GPU, so its pages are not held resident. Otherwise keeps memory-mapped loading. Required CPU buffers can still use RAM. Drops --no-mmap and --mlock.",mlockVetoed: "--mlock stays off: pinning the model in place would reserve RAM for all of it. Idle auto-unload is still disabled.",memlockCapped: "This system caps locked memory at {limit}. A larger model will not be fully pinned; raise the limit with ulimit -l.",reloadRequired: "Reload the model to apply the new memory flags.",diff --git a/studio/frontend/src/i18n/locales/es.ts b/studio/frontend/src/i18n/locales/es.tsindex e51b4c53e60..89bfdf1f7f9 100644--- a/studio/frontend/src/i18n/locales/es.ts+++ b/studio/frontend/src/i18n/locales/es.ts@@ -1261,8 +1261,8 @@ export const es = {keepResidentDescription: "Permanece en la VRAM entre mensajes.",keepResidentHint: "No devuelve los pesos a la RAM del sistema mientras el modelo siga cargado. Desactiva la descarga automática por inactividad y, cuando los pesos sí residen en la RAM del host (memoria unificada o descarga parcial a la GPU), también pasa --mlock para que el sistema operativo no los pagine ni los vuelva a subir en tu siguiente mensaje.",noRamReserve: "No reservar RAM del sistema para el modelo",- noRamReserveDescription: "No mantiene una copia completa en la RAM.",- noRamReserveHint: "Transfiere los pesos a la VRAM en lugar de mantener una copia completa en la RAM. Conserva la carga mapeada en memoria de llama.cpp y elimina --no-mmap y --mlock.",+ noRamReserveDescription: "Reduce la RAM ocupada por los pesos del modelo.",+ noRamReserveHint: "Omite la carga mapeada en memoria en las compilaciones de Windows compatibles cuando el modelo está totalmente descargado en la GPU, de modo que sus páginas no quedan residentes. En caso contrario conserva la carga mapeada en memoria. Los búferes de CPU necesarios pueden seguir usando RAM. Elimina --no-mmap y --mlock.",mlockVetoed: "--mlock permanece desactivado: fijar el modelo reservaría RAM para todo él. La descarga automática por inactividad sigue desactivada.",memlockCapped: "Este sistema limita la memoria bloqueada a {limit}. Un modelo mayor no quedará fijado por completo; aumenta el límite con ulimit -l.",reloadRequired: "Vuelve a cargar el modelo para aplicar las nuevas opciones de memoria.",diff --git a/studio/frontend/src/i18n/locales/fr.ts b/studio/frontend/src/i18n/locales/fr.tsindex 5a874c62af6..f7eb4ee4e59 100644--- a/studio/frontend/src/i18n/locales/fr.ts+++ b/studio/frontend/src/i18n/locales/fr.ts@@ -1265,8 +1265,8 @@ export const fr = {keepResidentDescription: "Reste en VRAM entre les messages.",keepResidentHint: "Ne rend pas les poids à la RAM système tant que le modèle reste chargé. Désactive le déchargement automatique en veille et, lorsque les poids résident réellement en RAM hôte (mémoire unifiée ou déchargement GPU partiel), passe aussi --mlock afin que le système ne les décharge pas pour les retransférer au prochain message.",noRamReserve: "Ne pas réserver de RAM système pour le modèle",- noRamReserveDescription: "Ne garde aucune copie complète en RAM.",- noRamReserveHint: "Transfère les poids vers la VRAM au lieu d'en garder une copie complète en RAM. Conserve le chargement mappé en mémoire de llama.cpp et supprime --no-mmap et --mlock.",+ noRamReserveDescription: "Réduit la RAM occupée par les poids du modèle.",+ noRamReserveHint: "Ignore le chargement mappé en mémoire sur les versions Windows compatibles lorsque le modèle est entièrement déchargé sur le GPU, afin que ses pages ne restent pas résidentes. Sinon, conserve le chargement mappé en mémoire. Les tampons CPU nécessaires peuvent toujours occuper de la RAM. Supprime --no-mmap et --mlock.",mlockVetoed: "--mlock reste désactivé : épingler le modèle réserverait de la RAM pour l'intégralité de celui-ci. Le déchargement automatique en veille reste désactivé.",memlockCapped: "Ce système limite la mémoire verrouillée à {limit}. Un modèle plus grand ne sera pas entièrement épinglé ; augmentez la limite avec ulimit -l.",reloadRequired: "Rechargez le modèle pour appliquer les nouvelles options de mémoire.",diff --git a/studio/frontend/src/i18n/locales/hi.ts b/studio/frontend/src/i18n/locales/hi.tsindex 9ab1e655659..8e152c7c544 100644--- a/studio/frontend/src/i18n/locales/hi.ts+++ b/studio/frontend/src/i18n/locales/hi.ts@@ -1242,8 +1242,8 @@ export const hi = {keepResidentDescription: "प्रॉम्प्ट के बीच VRAM में बना रहता है।",keepResidentHint: "जब तक मॉडल लोड रहता है, वेट्स सिस्टम RAM को वापस नहीं दिए जाते। निष्क्रिय होने पर स्वतः अनलोड बंद कर देता है, और जब वेट्स वास्तव में होस्ट RAM में रहते हैं (यूनिफाइड मेमोरी, या आंशिक GPU ऑफ़लोड) तब --mlock भी भेजता है, ताकि OS उन्हें पेज आउट करके आपके अगले प्रॉम्प्ट पर दोबारा अपलोड न करे।",noRamReserve: "मॉडल के लिए सिस्टम RAM आरक्षित न करें",- noRamReserveDescription: "RAM में पूरी प्रति नहीं रखता।",- noRamReserveHint: "RAM में पूरी प्रति रखने के बजाय वेट्स को VRAM में भेजता है। llama.cpp की मेमोरी-मैप्ड लोडिंग बनाए रखता है और --no-mmap तथा --mlock हटा देता है।",+ noRamReserveDescription: "मॉडल वेट्स के लिए रखी गई RAM घटाता है।",+ noRamReserveHint: "जब मॉडल पूरी तरह GPU पर ऑफ़लोड हो, तो समर्थित Windows बिल्ड पर मेमोरी-मैप्ड लोडिंग छोड़ देता है, ताकि उसके पेज रेज़िडेंट न रहें। अन्यथा मेमोरी-मैप्ड लोडिंग बनाए रखता है। आवश्यक CPU बफ़र अब भी RAM ले सकते हैं। --no-mmap तथा --mlock हटा देता है।",mlockVetoed: "--mlock बंद ही रहता है: मॉडल को पिन करने से उसकी पूरी मात्रा के लिए RAM आरक्षित हो जाएगी। निष्क्रिय होने पर स्वतः अनलोड अब भी अक्षम है।",memlockCapped: "यह सिस्टम लॉक की गई मेमोरी को {limit} तक सीमित करता है। इससे बड़ा मॉडल पूरी तरह पिन नहीं होगा; ulimit -l से सीमा बढ़ाएँ।",reloadRequired: "नई मेमोरी सेटिंग्स लागू करने के लिए मॉडल दोबारा लोड करें।",diff --git a/studio/frontend/src/i18n/locales/it.ts b/studio/frontend/src/i18n/locales/it.tsindex fd45c5c2bcd..f82d5433dcb 100644--- a/studio/frontend/src/i18n/locales/it.ts+++ b/studio/frontend/src/i18n/locales/it.ts@@ -1232,8 +1232,8 @@ export const it = {keepResidentDescription: "Resta nella VRAM tra un prompt e l'altro.",keepResidentHint: "Non restituisce i pesi alla RAM di sistema finché il modello resta caricato. Disattiva lo scaricamento automatico in inattività e, quando i pesi risiedono davvero nella RAM host (memoria unificata o offload parziale sulla GPU), passa anche --mlock, così il sistema operativo non li pagina per ricaricarli al prompt successivo.",noRamReserve: "Non riservare RAM di sistema per il modello",- noRamReserveDescription: "Non tiene una copia completa in RAM.",- noRamReserveHint: "Trasferisce i pesi nella VRAM invece di tenerne una copia completa in RAM. Mantiene il caricamento mappato in memoria di llama.cpp e rimuove --no-mmap e --mlock.",+ noRamReserveDescription: "Riduce la RAM occupata dai pesi del modello.",+ noRamReserveHint: "Salta il caricamento mappato in memoria sulle build Windows supportate quando il modello è interamente scaricato sulla GPU, così le sue pagine non restano residenti. Altrimenti mantiene il caricamento mappato in memoria. I buffer CPU necessari possono comunque occupare RAM. Rimuove --no-mmap e --mlock.",mlockVetoed: "--mlock resta disattivato: bloccare il modello riserverebbe RAM per l'intero modello. Lo scaricamento automatico in inattività resta disattivato.",memlockCapped: "Questo sistema limita la memoria bloccata a {limit}. Un modello più grande non verrà bloccato del tutto; aumenta il limite con ulimit -l.",reloadRequired: "Ricarica il modello per applicare le nuove opzioni di memoria.",diff --git a/studio/frontend/src/i18n/locales/ja.ts b/studio/frontend/src/i18n/locales/ja.tsindex efa0fe37fc6..82dab7a1b6a 100644--- a/studio/frontend/src/i18n/locales/ja.ts+++ b/studio/frontend/src/i18n/locales/ja.ts@@ -1221,8 +1221,8 @@ export const ja = {keepResidentDescription: "プロンプト間も VRAM に常駐します。",keepResidentHint: "モデルがロードされている間、重みをシステム RAM に戻しません。アイドル時の自動アンロードを無効にし、重みが実際にホスト RAM 上にある場合(ユニファイドメモリ、または GPU への部分オフロード)は --mlock も渡すため、OS が重みをページアウトして次のプロンプトで再アップロードすることがなくなります。",noRamReserve: "モデル用にシステム RAM を確保しない",- noRamReserveDescription: "RAM に完全なコピーを保持しません。",- noRamReserveHint: "RAM に完全なコピーを保持せず、重みを VRAM へ転送します。llama.cpp のメモリマップ読み込みを維持し、--no-mmap と --mlock を除去します。",+ noRamReserveDescription: "モデルの重みが占有する RAM を減らします。",+ noRamReserveHint: "モデルが GPU に完全にオフロードされている場合、対応する Windows ビルドではメモリマップ読み込みを行わず、ファイルのページが常駐しないようにします。それ以外ではメモリマップ読み込みを維持します。必要な CPU バッファは RAM を使用することがあります。--no-mmap と --mlock を除去します。",mlockVetoed: "--mlock は無効のままです。モデルを固定するとモデル全体分の RAM を確保することになります。アイドル時の自動アンロードは引き続き無効です。",memlockCapped: "このシステムはロック可能なメモリを {limit} に制限しています。これより大きいモデルは完全には固定されません。ulimit -l で上限を引き上げてください。",reloadRequired: "新しいメモリ設定を適用するにはモデルを再読み込みしてください。",diff --git a/studio/frontend/src/i18n/locales/ko.ts b/studio/frontend/src/i18n/locales/ko.tsindex f1e33294ee2..bfb3923165a 100644--- a/studio/frontend/src/i18n/locales/ko.ts+++ b/studio/frontend/src/i18n/locales/ko.ts@@ -1235,8 +1235,8 @@ export const ko = {keepResidentDescription: "프롬프트 사이에도 VRAM에 상주합니다.",keepResidentHint: "모델이 로드되어 있는 동안 가중치를 시스템 RAM으로 되돌리지 않습니다. 유휴 시 자동 언로드를 끄고, 가중치가 실제로 호스트 RAM에 있는 경우(통합 메모리 또는 부분 GPU 오프로드)에는 --mlock도 전달하여 운영체제가 가중치를 페이지 아웃했다가 다음 프롬프트에서 다시 업로드하지 않도록 합니다.",noRamReserve: "모델을 위해 시스템 RAM을 예약하지 않음",- noRamReserveDescription: "RAM에 전체 사본을 두지 않습니다.",- noRamReserveHint: "RAM에 전체 사본을 두지 않고 가중치를 VRAM으로 전송합니다. llama.cpp의 메모리 매핑 로딩을 유지하고 --no-mmap과 --mlock을 제거합니다.",+ noRamReserveDescription: "모델 가중치가 차지하는 RAM을 줄입니다.",+ noRamReserveHint: "모델이 GPU에 완전히 오프로드된 경우 지원되는 Windows 빌드에서 메모리 매핑 로딩을 건너뛰어 파일 페이지가 상주하지 않도록 합니다. 그렇지 않으면 메모리 매핑 로딩을 유지합니다. 필요한 CPU 버퍼는 여전히 RAM을 사용할 수 있습니다. --no-mmap과 --mlock을 제거합니다.",mlockVetoed: "--mlock은 꺼진 상태로 유지됩니다. 모델을 고정하면 모델 전체 크기만큼 RAM을 예약하게 됩니다. 유휴 시 자동 언로드는 계속 비활성화됩니다.",memlockCapped: "이 시스템은 잠금 메모리를 {limit}로 제한합니다. 더 큰 모델은 완전히 고정되지 않습니다. ulimit -l로 한도를 늘리세요.",reloadRequired: "새 메모리 옵션을 적용하려면 모델을 다시 로드하세요.",diff --git a/studio/frontend/src/i18n/locales/pt-br.ts b/studio/frontend/src/i18n/locales/pt-br.tsindex 7751a513776..4d110da488a 100644--- a/studio/frontend/src/i18n/locales/pt-br.ts+++ b/studio/frontend/src/i18n/locales/pt-br.ts@@ -1249,8 +1249,8 @@ export const ptBR = {keepResidentDescription: "Permanece na VRAM entre os prompts.",keepResidentHint: "Não devolve os pesos à RAM do sistema enquanto o modelo continuar carregado. Desativa a descarga automática por inatividade e, quando os pesos de fato ficam na RAM do host (memória unificada ou offload parcial para a GPU), também passa --mlock, para que o sistema operacional não os pagine e os reenvie no próximo prompt.",noRamReserve: "Não reservar RAM do sistema para o modelo",- noRamReserveDescription: "Não mantém uma cópia completa na RAM.",- noRamReserveHint: "Transfere os pesos para a VRAM em vez de manter uma cópia completa na RAM. Mantém o carregamento mapeado em memória do llama.cpp e remove --no-mmap e --mlock.",+ noRamReserveDescription: "Reduz a RAM ocupada pelos pesos do modelo.",+ noRamReserveHint: "Ignora o carregamento mapeado em memória nas versões compatíveis do Windows quando o modelo está totalmente descarregado na GPU, para que suas páginas não fiquem residentes. Caso contrário, mantém o carregamento mapeado em memória. Buffers de CPU necessários ainda podem usar RAM. Remove --no-mmap e --mlock.",mlockVetoed: "--mlock continua desativado: fixar o modelo reservaria RAM para todo ele. A descarga automática por inatividade continua desativada.",memlockCapped: "Este sistema limita a memória bloqueada a {limit}. Um modelo maior não será totalmente fixado; aumente o limite com ulimit -l.",reloadRequired: "Recarregue o modelo para aplicar as novas opções de memória.",diff --git a/studio/frontend/src/i18n/locales/ru.ts b/studio/frontend/src/i18n/locales/ru.tsindex 524390dcbfd..a98c7abc5ae 100644--- a/studio/frontend/src/i18n/locales/ru.ts+++ b/studio/frontend/src/i18n/locales/ru.ts@@ -1245,8 +1245,8 @@ export const ru = {keepResidentDescription: "Остаётся в видеопамяти между запросами.",keepResidentHint: "Веса не возвращаются в системную ОЗУ, пока модель загружена. Отключает автовыгрузку по простою, а если веса действительно находятся в ОЗУ хоста (единая память или частичная выгрузка на GPU), дополнительно передаёт --mlock, чтобы ОС не вытесняла их и не загружала заново при следующем запросе.",noRamReserve: "Не резервировать системную ОЗУ под модель",- noRamReserveDescription: "Не хранит полную копию в ОЗУ.",- noRamReserveHint: "Веса передаются в видеопамять вместо хранения полной копии в ОЗУ. Сохраняет загрузку через отображение файла в память в llama.cpp и убирает --no-mmap и --mlock.",+ noRamReserveDescription: "Уменьшает объём ОЗУ, занятый весами модели.",+ noRamReserveHint: "Пропускает загрузку через отображение файла в память в поддерживаемых сборках Windows, когда модель полностью выгружена на видеокарту, чтобы её страницы не оставались в памяти. В остальных случаях сохраняет загрузку через отображение файла в память. Необходимые буферы ЦП всё равно могут занимать ОЗУ. Убирает --no-mmap и --mlock.",mlockVetoed: "--mlock остаётся выключенным: закрепление модели зарезервировало бы ОЗУ под неё целиком. Автовыгрузка по простою по-прежнему отключена.",memlockCapped: "Система ограничивает блокируемую память значением {limit}. Модель большего размера не будет закреплена полностью; увеличьте лимит через ulimit -l.",reloadRequired: "Перезагрузите модель, чтобы применить новые параметры памяти.",diff --git a/studio/frontend/src/i18n/locales/zh-CN.ts b/studio/frontend/src/i18n/locales/zh-CN.tsindex 16f21c6c317..2166294c2aa 100644--- a/studio/frontend/src/i18n/locales/zh-CN.ts+++ b/studio/frontend/src/i18n/locales/zh-CN.ts@@ -1211,8 +1211,8 @@ export const zhCN = {keepResidentDescription: "在两次提问之间保持驻留显存。",keepResidentHint: "模型保持加载期间,不把权重交还给系统内存。关闭空闲自动卸载;当权重确实位于主机内存中时(统一内存,或部分卸载到 GPU),还会传入 --mlock,使操作系统无法将其换出、并在你下次提问时重新上传。",noRamReserve: "不为模型预留系统内存",- noRamReserveDescription: "不在内存中保留完整副本。",- noRamReserveHint: "将权重传输到显存,而不在内存中保留完整副本。保持 llama.cpp 的内存映射加载方式,并移除 --no-mmap 和 --mlock。",+ noRamReserveDescription: "减少模型权重占用的内存。",+ noRamReserveHint: "当模型完全卸载到 GPU 时,在受支持的 Windows 版本上跳过内存映射加载,使其页面不再常驻;否则保持内存映射加载。必要的 CPU 缓冲区仍可能占用内存。移除 --no-mmap 和 --mlock。",mlockVetoed: "--mlock 保持关闭:锁定模型会为其全部内容预留内存。空闲自动卸载仍处于禁用状态。",memlockCapped: "本系统将锁定内存限制为 {limit}。更大的模型无法被完全锁定;请使用 ulimit -l 提高上限。",reloadRequired: "重新加载模型以应用新的内存选项。",