Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
Studio: memory-map an oversized load on an integrated GPU and read its lookup tables from disk (#16857)
oobabooga3db62df24308
diff --git a/studio/backend/core/inference/llama_cpp.py b/studio/backend/core/inference/llama_cpp.pyindex 5fc861e8fe3..81d725f0630 100644--- a/studio/backend/core/inference/llama_cpp.py+++ b/studio/backend/core/inference/llama_cpp.py@@ -130,6 +130,7 @@ from core.inference.llama_server_args import (extra_args_select_load_mode,fit_is_enabled_in,force_pageable_load,+ load_mode_left_auto,memory_env_selects_load_mode,memory_state_satisfies_settings,fit_is_effectively_on,@@ -4125,6 +4126,15 @@ _VRAM_FLOOR_RESERVE_MIB = 512.0# llama-server's own --fit-target default ("target margin per device for --fit,# ... default: 1024"): what the fallback fitter leaves when we pass nothing._LLAMA_FIT_TARGET_DEFAULT_MIB = 1024.0+# Extra --fit-target for a load paged on a Windows iGPU, whose GPU share and mapped pages share one+# memory: on a Strix Halo 8 GiB still aborted at warm-up, 16 GiB ran. Linux runs slower with it.+_IGPU_PAGED_FIT_HEADROOM_MIB = 15360+++def _paged_load_needs_fit_headroom() -> bool:+ """Whether a load paged on an iGPU needs ``_IGPU_PAGED_FIT_HEADROOM_MIB``."""+ return sys.platform == "win32"+# Windows + NVIDIA since driver 536.40: WDDM spills VRAM overflow to host RAM instead of failing, so the reserve matches llama.cpp fit margin._WINDOWS_SYSMEM_FALLBACK_RESERVE_MIB = _LLAMA_FIT_TARGET_DEFAULT_MIB@@ -7779,6 +7789,11 @@ _PAGEABLE_OVERRIDE_NOTE = ("RAM up front and cannot complete at this size, so Unsloth loaded it with memory ""mapping instead.")+# The override where nobody chose the mode (auto is none on an iGPU); lazy and fit flags are logged.+_PAGEABLE_AUTO_OVERRIDE_NOTE = (+ " Unsloth turned on memory mapping, which llama.cpp leaves off by default on this "+ "integrated GPU."+)# The override is reported, not undone: the running server is already memory-mapped._PAGEABLE_OVERRIDE_TEXT_ONLY_NOTICE = ("This model was set to load without memory mapping, and with its vision projector "@@ -7787,6 +7802,13 @@ _PAGEABLE_OVERRIDE_TEXT_ONLY_NOTICE = ("which the weights alone would have fit. The running server keeps the memory ""mapping it started with; load the model again to run it as you asked.")+_PAGEABLE_AUTO_OVERRIDE_TEXT_ONLY_NOTICE = (+ "With its vision projector in system RAM this model could run out of memory without "+ "memory mapping, llama.cpp's default on this integrated GPU, so Unsloth loaded it with "+ "memory mapping instead. The projector then failed to start and this session is "+ "text-only, which the weights alone would have fit. The running server keeps the "+ "memory mapping it started with."+)_CGROUP_ROOT = "/sys/fs/cgroup"_PROC_SELF_CGROUP = "/proc/self/cgroup"@@ -10366,6 +10388,7 @@ class LlamaCppBackend:"supports_video_timestamp_interval": False,"supports_video_ffmpeg_dir": None,"supports_load_mode": False,+ "supports_load_mode_auto": False,"supports_moe_cache": False,"supports_moe_cache_auto": False,"supports_lazy_mode": False,@@ -10437,6 +10460,7 @@ class LlamaCppBackend:supports_video_timestamp_interval = Falsesupports_video_ffmpeg_dir = Nonesupports_load_mode = False+ supports_load_mode_auto = Falsesupports_moe_cache = Falsesupports_moe_cache_auto = Falsesupports_lazy_mode = False@@ -10651,6 +10675,14 @@ class LlamaCppBackend:# --load-mode supersedes --mlock / --no-mmap, which are deprecated.# Pre-initialised above: a failed probe must fall back, not raise.supports_load_mode = _is_real("--load-mode")+ # "auto: mmap, unless a device does not support it": without it, mmap is the default everywhere.+ supports_load_mode_auto = bool(+ supports_load_mode+ and re.search(+ r"(?<![a-z0-9_+-])auto(?![a-z0-9_+-])",+ (blocks.get("--load-mode") or "").lower(),+ )+ )# "auto" is a fork addition; upstream builds exit on it, so fail closed.supports_moe_cache = bool(probe_ok and _is_real("--moe-cache-mib"))supports_moe_cache_auto = bool(@@ -10764,6 +10796,7 @@ class LlamaCppBackend:"supports_video_timestamp_interval": supports_video_timestamp_interval,"supports_video_ffmpeg_dir": supports_video_ffmpeg_dir,"supports_load_mode": supports_load_mode,+ "supports_load_mode_auto": supports_load_mode_auto,"supports_moe_cache": supports_moe_cache,"supports_moe_cache_auto": supports_moe_cache_auto,"supports_lazy_mode": supports_lazy_mode,@@ -15671,12 +15704,131 @@ class LlamaCppBackend:"yes",)+ def _auto_load_mode_unmapped(+ self,+ server_caps: Optional[Mapping[str, object]],+ *,+ child_has_no_gpu: bool = False,+ **devices,+ ) -> bool:+ """Whether llama.cpp resolves ``--load-mode auto`` to ``none`` here: a device lacking+ ``mmap_support`` turns both auto modes off (``load_tensors``); ``devices`` go to+ ``_lazy_auto_resolves_off``."""+ if child_has_no_gpu or not (server_caps or {}).get("supports_load_mode_auto"):+ return False+ try:+ return bool(self._lazy_auto_resolves_off(**devices))+ except Exception as e: # noqa: BLE001 -- unknown devices: keep the launch as is+ logger.debug("Could not tell whether llama.cpp maps this load: %s", e)+ return False++ def _paged_lookup_table_flags(+ self,+ model_path: Optional[str],+ caps: Optional[Mapping[str, object]],+ argv: Iterable[str],+ env: Optional[Mapping[str, str]] = None,+ ) -> list[str]:+ """``[<lazy flag>, "on"]`` to stream the tables auto streams elsewhere: mmap prefetch skips+ only lazy ranges (llama-mmap.cpp). Empty when a lazy mode is set or nothing would stream."""+ caps = caps or {}+ if not model_path or not caps.get("supports_lazy_mode"):+ return []+ flag = str(caps.get("lazy_mode_flag") or "--lazy-mode")+ source_env = os.environ if env is None else env+ flags, env_name = _LAZY_MODE_SPELLINGS.get(flag, (frozenset({flag}), ""))+ argv = list(argv or ())+ if (env_name and str(source_env.get(env_name) or "").strip()) or _extra_args_set_any_flag(+ argv, flags+ ):+ return []+ if self._lazy_read_host_bytes(model_path, caps = caps, extra_args = argv, env = source_env) <= 0:+ return []+ return [flag, "on"]++ @staticmethod+ def _paged_fit_headroom_flags(+ argv: Iterable[str],+ env: Optional[Mapping[str, str]],+ user_args: Optional[Iterable[str]],+ gpu_count: int = 1,+ ) -> list[str]:+ """``--fit-target`` raised by ``_IGPU_PAGED_FIT_HEADROOM_MIB`` over the launch's margin.+ Empty off Windows, with the fitter off, a user margin, a per-device list, or a second+ device, which the broadcast margin would starve."""+ args = [str(a) for a in argv or ()]+ if not _paged_load_needs_fit_headroom() or gpu_count > 1:+ return []+ if (+ _extra_args_device(args, {"--rpc"})+ or str((env or {}).get("LLAMA_ARG_RPC") or "").strip()+ ):+ return []+ devices = _extra_args_device(args, {"--device", "-dev"})+ if devices is None:+ devices = str((env or {}).get("LLAMA_ARG_DEVICE") or "")+ if (+ len(+ [+ d+ for d in devices.split(",")+ if d.strip().lower() not in ("", *_CPU_DEVICE_VALUES)+ ]+ )+ > 1+ ):+ return []+ if not fit_is_effectively_on(args, env) or _fit_target_values(user_args or (), env):+ return []+ raw = _last_flag_value(args, {"-fitt", "--fit-target"})+ if (+ raw is not None+ and len([p for p in str(raw).replace("/", ",").split(",") if p.strip()]) > 1+ ):+ return []+ have = _single_device_fit_margin(args, None)+ base = _LLAMA_FIT_TARGET_DEFAULT_MIB if have is None else have+ return ["--fit-target", str(math.ceil(base + _IGPU_PAGED_FIT_HEADROOM_MIB))]++ @staticmethod+ def _margins_with_paged_headroom(+ margins: Optional[Mapping[int, float]],+ headroom: Iterable[str],+ sign: int = 1,+ ) -> Optional[Mapping[int, float]]:+ """The spawn respells ``--fit-target`` per device from ``margins``, so they carry the headroom too."""+ if not margins or not list(headroom):+ return margins+ return {i: m + sign * _IGPU_PAGED_FIT_HEADROOM_MIB for i, m in margins.items()}++ @staticmethod+ def _withdraw_paged_headroom(+ cmd: list[str], margins: Optional[Mapping[int, float]], headroom: list[str]+ ) -> tuple[list[str], Optional[Mapping[int, float]]]:+ """``cmd`` and ``margins`` without the iGPU paging headroom, for a respawn off the iGPU."""+ return (+ LlamaCppBackend._without_flag_pairs(cmd, headroom),+ LlamaCppBackend._margins_with_paged_headroom(margins, headroom, sign = -1),+ )+def _page_an_oversized_unmapped_load(- self, cmd: Iterable[str], env: Optional[dict], oversized+ self,+ cmd: Iterable[str],+ env: Optional[dict],+ oversized,+ *,+ auto_unmapped: bool = False,+ lazy_table_flags: Optional[Callable[[list], list]] = None,+ fit_headroom_flags: Optional[Callable[[list], list]] = None,) -> tuple[list, Optional[str]]:- """Returns (cmd, note); rewrites an oversized non-pageable load to mmap so it finishes; judges effective env state, not argv; never blocks."""+ """Returns (cmd, note); rewrites an oversized non-pageable load to mmap so it finishes; judges effective env state, not argv; never blocks.++ ``auto_unmapped``: auto is none for this child, so an unset mode is rewritten too, and+ ``lazy_table_flags(argv)`` / ``fit_headroom_flags(argv)`` are appended."""original = cmd if isinstance(cmd, list) else list(cmd or ())- _mlock_now, _reserves_ram_now = resolve_effective_memory_state(original, env)+ _mlock_now, _reserves_ram_now = resolve_effective_memory_state(+ original, env, auto_unmapped = auto_unmapped+ )if not _reserves_ram_now:return original, Nonetry:@@ -15686,15 +15838,33 @@ class LlamaCppBackend:# Advisory pricing: an unreadable host or GGUF leaves the launch alone.logger.debug("Could not price the load for a pageable override.", exc_info = True)return original, None- paged, overridden = force_pageable_load(original, env)+ from_auto = auto_unmapped and load_mode_left_auto(original, env)+ paged, overridden = force_pageable_load(original, env, auto_unmapped = auto_unmapped)if not overridden:return original, None+ lazy: list = []+ if auto_unmapped and lazy_table_flags is not None:+ try:+ lazy = list(lazy_table_flags(paged) or [])+ except Exception: # noqa: BLE001 -- the mapping alone still lets the load finish+ logger.debug("Lookup tables: could not price lazy reads.", exc_info = True)+ paged = [*paged, *lazy]+ headroom: list = []+ if auto_unmapped and fit_headroom_flags is not None:+ try:+ headroom = list(fit_headroom_flags(paged) or [])+ except Exception: # noqa: BLE001 -- the mapping still applies without the margin+ logger.debug("Fit headroom: could not read the launch's margin.", exc_info = True)+ paged = [*paged, *headroom]logger.warning("Overriding the unmapped load mode (%s) for an oversized model: it would "- "allocate the whole model in host RAM instead of paging it in.",+ "allocate the whole model in host RAM instead of paging it in.%s%s",", ".join(overridden),+ f" Lookup tables: {' '.join(lazy)}." if lazy else "",+ f" GPU headroom: {' '.join(headroom)}." if headroom else "",)- return paged, _PAGEABLE_OVERRIDE_NOTE+ note = _PAGEABLE_AUTO_OVERRIDE_NOTE if from_auto else _PAGEABLE_OVERRIDE_NOTE+ return paged, note@propertydef last_load_warning(self) -> Optional[str]:@@ -15750,7 +15920,11 @@ class LlamaCppBackend:# The discrete guard never charged the projector, so its verdict stands.self._last_load_warning = host_msg + suffixelif suffix.strip():- self._last_load_warning = _PAGEABLE_OVERRIDE_TEXT_ONLY_NOTICE+ self._last_load_warning = (+ _PAGEABLE_AUTO_OVERRIDE_TEXT_ONLY_NOTICE+ if _PAGEABLE_AUTO_OVERRIDE_NOTE in suffix+ else _PAGEABLE_OVERRIDE_TEXT_ONLY_NOTICE+ )else:self._last_load_warning = Nonelogger.info(@@ -23527,6 +23701,8 @@ class LlamaCppBackend:_fit_target_delta_by_idx: Optional[dict[int, float]] = None# Studio's emitted margins per card, spelled into --fit-target at spawn._fit_target_margins_by_idx: Optional[dict[int, float]] = None+ # --fit-target pair a paged iGPU launch appended, withdrawn if a retry leaves the iGPU.+ _paged_headroom: list = []# Set only if that margin actually reached the command line, which is# what makes the child budget-priced in a mode the planner skipped._fit_target_priced = False@@ -30597,8 +30773,38 @@ class LlamaCppBackend:))+ # On an iGPU auto is none; same devices as the guard above.+ _launch_auto_unmapped = self._auto_load_mode_unmapped(+ server_caps,+ child_has_no_gpu = _child_has_no_gpu,+ gpu_indices = [row[0] for row in _reachable_gpus] or None,+ detected_gpus = _reachable_gpus,+ shared_gpu_ids = _shared_gpu_ids,+ is_vulkan_backend = is_vulkan_backend,+ probe_integrated_cuda = False,+ extra_args = cmd,+ env = env,+ device_list = False,+ integrated_gpu_ids = _vulkan_igpu_ids,+ )+ _paged_headroom = []cmd, _pageable_note = self._page_an_oversized_unmapped_load(- cmd, env, _launch_is_oversized+ cmd,+ env,+ _launch_is_oversized,+ auto_unmapped = _launch_auto_unmapped,+ lazy_table_flags = lambda argv: self._paged_lookup_table_flags(+ model_path, server_caps, argv, env+ ),+ fit_headroom_flags = lambda argv: _paged_headroom.extend(+ self._paged_fit_headroom_flags(+ argv, env, extra_args, gpu_count = len(_reachable_gpus)+ )+ )+ or _paged_headroom,+ )+ _fit_target_margins_by_idx = self._margins_with_paged_headroom(+ _fit_target_margins_by_idx, _paged_headroom)if _pageable_note:# Record the argv that actually launches, and snapshot it too, since arch-crash retry restores it.@@ -32181,8 +32387,41 @@ class LlamaCppBackend:))+ _retry_auto_unmapped = self._auto_load_mode_unmapped(+ server_caps,+ gpu_indices = [row[0] for row in _retry_rows] or None,+ detected_gpus = _retry_rows,+ shared_gpu_ids = _shared_gpu_ids,+ is_vulkan_backend = is_vulkan_backend,+ probe_integrated_cuda = False,+ extra_args = cmd,+ env = env,+ device_list = False,+ integrated_gpu_ids = _vulkan_igpu_ids,+ )+ if _paged_headroom and not _retry_auto_unmapped:+ cmd, _fit_target_margins_by_idx = self._withdraw_paged_headroom(+ cmd, _fit_target_margins_by_idx, _paged_headroom+ )+ _paged_headroom = []+ _retry_headroom: list = []cmd, _retry_pageable_note = self._page_an_oversized_unmapped_load(- cmd, env, _retry_is_oversized+ cmd,+ env,+ _retry_is_oversized,+ auto_unmapped = _retry_auto_unmapped,+ lazy_table_flags = lambda argv: self._paged_lookup_table_flags(+ model_path, server_caps, argv, env+ ),+ fit_headroom_flags = lambda argv: _retry_headroom.extend(+ self._paged_fit_headroom_flags(+ argv, env, extra_args, gpu_count = len(_retry_rows)+ )+ )+ or _retry_headroom,+ )+ _fit_target_margins_by_idx = self._margins_with_paged_headroom(+ _fit_target_margins_by_idx, _retry_headroom)if _retry_pageable_note:# Onto whichever notice first-notice-wins kept above; adiff --git a/studio/backend/core/inference/llama_server_args.py b/studio/backend/core/inference/llama_server_args.pyindex 6708b4d641c..974bed3c2b0 100644--- a/studio/backend/core/inference/llama_server_args.py+++ b/studio/backend/core/inference/llama_server_args.py@@ -1716,10 +1716,17 @@ def _pageable_env_value(def force_pageable_load(- argv: Optional[Iterable[str]], env: Optional[dict] = None+ argv: Optional[Iterable[str]],+ env: Optional[dict] = None,+ *,+ auto_unmapped: bool = False,) -> tuple[list[str], list[str]]:- """Rewrites none/mlock to pageable: effective locks become mmap+mlock, shadowed locks stay dropped; decided by pre-rewrite state, not tokens."""+ """Rewrites none/mlock to pageable: effective locks become mmap+mlock, shadowed locks stay dropped; decided by pre-rewrite state, not tokens.++ ``auto_unmapped``: auto is none on this child, so an unset mode gets ``--load-mode mmap``."""tokens = [str(a) for a in (argv or [])]+ if auto_unmapped and load_mode_left_auto(tokens, env):+ return [*tokens, "--load-mode", "mmap"], ["auto, which is none on this GPU"]# Only RAM-reserving launches are rewritten; pageable dio or mmap keeps every token, mlock included._mlock_now, _reserves_now = resolve_effective_memory_state(tokens, env)drop_shadowed_mlock = _reserves_now and not _mlock_now@@ -1772,6 +1779,9 @@ def force_pageable_load(else:env[name] = new_valueoverridden.append(name)+ # Dropping none/--no-mmap leaves auto, which is none again here.+ if overridden and auto_unmapped and load_mode_left_auto(out, env):+ out.extend(["--load-mode", "mmap"])return out, overridden@@ -1793,10 +1803,15 @@ _LOAD_MODE_RESERVING_VALUES = frozenset({"none", "mlock"})def resolve_effective_memory_state(- argv: Optional[Iterable[str]], env: Optional[Mapping[str, str]] = None+ argv: Optional[Iterable[str]],+ env: Optional[Mapping[str, str]] = None,+ *,+ auto_unmapped: bool = False,) -> tuple[bool, bool]:"""What the child runs with: env supplies defaults, argv overrides last-wins, as llama.cpp does."""- mlock, reserves_ram, _direct_io = resolve_effective_load_state(argv, env)+ mlock, reserves_ram, _direct_io = resolve_effective_load_state(+ argv, env, auto_unmapped = auto_unmapped+ )return mlock, reserves_ram@@ -1808,37 +1823,63 @@ def resolve_effective_direct_io(def resolve_effective_load_state(- argv: Optional[Iterable[str]], env: Optional[Mapping[str, str]] = None+ argv: Optional[Iterable[str]],+ env: Optional[Mapping[str, str]] = None,+ *,+ auto_unmapped: bool = False,) -> tuple[bool, bool, bool]:"""``(mlock, reserves_ram, direct_io)``. Every branch that resolves the mode- assigns all three, so the DirectIO bit cannot fall out of step with the pair."""+ assigns all three, so the DirectIO bit cannot fall out of step with the pair.++ ``auto_unmapped``: a device lacks ``mmap_support`` (an iGPU), so ``auto`` is ``none``+ (``load_tensors``), not mmap."""+ mlock, reserves_ram, direct_io, auto = _resolve_load_state(argv, env)+ if auto and auto_unmapped:+ return False, True, False+ return mlock, reserves_ram, direct_io+++def load_mode_left_auto(+ argv: Optional[Iterable[str]], env: Optional[Mapping[str, str]] = None+) -> bool:+ """Whether nothing selects a mode other than ``auto``. Legacy flags count: only builds+ without ``auto`` accept them."""+ return _resolve_load_state(argv, env)[3]+++def _resolve_load_state(+ argv: Optional[Iterable[str]], env: Optional[Mapping[str, str]] = None+) -> tuple[bool, bool, bool, bool]:+ """``(mlock, reserves_ram, direct_io, auto)``, last selector wins, env before argv."""env = env or {}mlock = Falsereserves_ram = Falsedirect_io = False+ auto = True# Env vars run the flag's handler in registration order, so a later one overwrites the mode.if str(env.get("LLAMA_ARG_MLOCK", "")).strip().lower() in _ENV_TRUE_VALUES:# Either mode it can mean maps or buffers the weights; neither streams.- mlock, direct_io = True, False+ mlock, direct_io, auto = True, False, False# LLAMA_ARG_NO_<NAME> present forces false whatever the value, and beats the affirmative var._mmap_env = "0" if "LLAMA_ARG_NO_MMAP" in env else str(env.get("LLAMA_ARG_MMAP", ""))_mmap_env = _mmap_env.strip().lower()if _mmap_env in _ENV_TRUE_VALUES:- mlock, reserves_ram, direct_io = False, False, False+ mlock, reserves_ram, direct_io, auto = False, False, False, Falseelif _mmap_env in _ENV_FALSE_VALUES:- mlock, reserves_ram, direct_io = False, True, False+ mlock, reserves_ram, direct_io, auto = False, True, False, False# LLAMA_ARG_DIO likewise: on selects DirectIO, off selects "none"._dio_env = "0" if "LLAMA_ARG_NO_DIO" in env else str(env.get("LLAMA_ARG_DIO", ""))_dio_env = _dio_env.strip().lower()if _dio_env in _ENV_TRUE_VALUES:- mlock, reserves_ram, direct_io = False, False, True+ mlock, reserves_ram, direct_io, auto = False, False, True, Falseelif _dio_env in _ENV_FALSE_VALUES:- mlock, reserves_ram, direct_io = False, True, False+ mlock, reserves_ram, direct_io, auto = False, True, False, False_mode_env = str(env.get("LLAMA_ARG_LOAD_MODE", "")).strip().lower()if _mode_env:mlock = _mode_env in _LOAD_MODE_MLOCK_VALUESreserves_ram = _mode_env in _LOAD_MODE_RESERVING_VALUESdirect_io = _mode_env == "dio"+ auto = _mode_env == "auto"tokens = [str(a) for a in (argv or [])]i, n = 0, len(tokens)@@ -1850,30 +1891,34 @@ def resolve_effective_load_state(continueif flag in _MLOCK_FLAGS:# Only the mlock bit matters: mlock and mmap+mlock both replace an earlier DirectIO mode.- mlock, direct_io = True, False+ mlock, direct_io, auto = True, False, Falsei += 1elif flag in _NO_MMAP_FLAGS:# --no-mmap resets the whole mode, clearing an earlier --mlock.mlock = Falsereserves_ram = Truedirect_io = False+ auto = Falsei += 1elif flag in _DIO_ON_FLAGS:# Deprecated load-mode selector resets the mode, dropping mlock; DirectIO holds no full host copy.mlock = Falsereserves_ram = Falsedirect_io = True+ auto = Falsei += 1elif flag in _DIO_OFF_FLAGS:# Not plain mmap: upstream maps these to mode `none` (full host buffer).mlock = Falsereserves_ram = Truedirect_io = False+ auto = Falsei += 1elif flag == "--mmap":mlock = Falsereserves_ram = Falsedirect_io = False+ auto = Falsei += 1elif flag in _LOAD_MODE_FLAGS:if "=" in tok:@@ -1887,10 +1932,11 @@ def resolve_effective_load_state(mlock = value in _LOAD_MODE_MLOCK_VALUESreserves_ram = value in _LOAD_MODE_RESERVING_VALUESdirect_io = value == "dio"+ auto = value == "auto"i += stepelse:i += 1- return mlock, reserves_ram, direct_io+ return mlock, reserves_ram, direct_io, autodef memory_state_satisfies_settings(diff --git a/studio/backend/tests/test_llama_cpp_placement.py b/studio/backend/tests/test_llama_cpp_placement.pyindex 9700da85412..88b5eddb68b 100644--- a/studio/backend/tests/test_llama_cpp_placement.py+++ b/studio/backend/tests/test_llama_cpp_placement.py@@ -2679,6 +2679,247 @@ def test_an_unmapped_apu_load_that_fits_is_left_exactly_as_asked(tmp_path, monkeassert backend.last_load_warning is None+_GiB = 1024**3+# The DeepSeek-V4.1-Flash UD-IQ1_S Engram table: 256 x 768022850 IQ4_NL rows.+_DEEPSEEK41_TABLE = 110_595_290_400+++def _igpu_backend(+ tmp_path,+ monkeypatch,+ *,+ avail_mib,+ gguf_gb = 216.97,+ table = _DEEPSEEK41_TABLE,+ igpu = True,+ load_mode_auto = True,+):+ """An integrated CUDA SoC (RTX Spark N1X, DGX Spark): both auto modes resolve off."""+ backend, gguf = _apu_backend(+ tmp_path, gguf_gb = gguf_gb, avail_mib = avail_mib, monkeypatch = monkeypatch+ )+ backend._lazy_auto_resolves_off = lambda **_kw: igpu+ backend._gguf_tensor_scan = lambda _path: (+ "deepseek41",+ {+ "per_layer_token_embd.weight": table,+ "blk.0.ffn_up_exps.weight": int(gguf_gb * _GiB) - table,+ },+ 0,+ )+ caps = dict(LlamaCppBackend.probe_server_capabilities.__func__(LlamaCppBackend, None))+ caps.update(+ supports_load_mode = True,+ supports_load_mode_auto = load_mode_auto,+ supports_lazy_mode = True,+ lazy_mode_flag = "--lazy-mode",+ )+ monkeypatch.setattr(LlamaCppBackend, "probe_server_capabilities", lambda *a, **k: caps)+ return backend, gguf+++from core.inference.llama_server_args import resolve_effective_load_state # noqa: E402+++ "extra_args, env",+ [(["--no-mmap"], {}), ([], {"LLAMA_ARG_NO_MMAP": "1"}), (["--load-mode=none"], {})],+ ids = ["no-mmap", "env-no-mmap", "load-mode-none-equals"],+)+def test_every_unmapped_selector_on_an_igpu_ends_up_mapped(tmp_path, monkeypatch, extra_args, env):+ """Removing the selector is not enough there: the mode the child resolves must be pageable."""+ backend, gguf = _igpu_backend(tmp_path, monkeypatch, avail_mib = 17 * 1024)+ for name, value in env.items():+ monkeypatch.setenv(name, value)++ launched = _launch(backend, gguf, extra_args = extra_args)++ assert (+ resolve_effective_load_state(launched["cmd"], launched["env"], auto_unmapped = True)[1]+ is False+ )+ assert "was set to load without memory mapping" in (backend.last_load_warning or "")+++def _modes(cmd, flag):+ return [cmd[i + 1] for i, tok in enumerate(cmd) if tok == flag and i + 1 < len(cmd)]++[email protected]("platform", ["win32", "linux"])+def test_an_oversized_igpu_load_left_on_auto_is_mapped_and_streams_its_table(+ tmp_path, monkeypatch, platform+):+ """On an iGPU auto is none, so a 217 GB model on auto read its 103 GiB table into one buffer."""+ monkeypatch.setattr(+ llama_cpp_module, "_paged_load_needs_fit_headroom", lambda: platform == "win32"+ )+ backend, gguf = _igpu_backend(tmp_path, monkeypatch, avail_mib = 17 * 1024)+ logged = _override_log(monkeypatch)++ cmd = _launch(backend, gguf)["cmd"]++ assert _modes(cmd, "--load-mode") == ["mmap"], cmd+ assert _modes(cmd, "--lazy-mode") == ["on"], cmd+ assert resolve_effective_load_state(cmd, {}, auto_unmapped = True)[1] is False+ windows = platform == "win32"+ assert _modes(cmd, "--fit-target")[-1:] == (["16384"] if windows else []), cmd+ warning = backend.last_load_warning or ""+ assert warning.endswith(+ " Unsloth turned on memory mapping, which llama.cpp leaves off by default on this "+ "integrated GPU."+ ), warning+ assert "was set to load without memory mapping" not in warning+ override = [line for line in logged if "Overriding the unmapped load mode" in line]+ assert override and "Lookup tables: --lazy-mode on." in override[0], logged+ assert ("GPU headroom: --fit-target 16384." in override[0]) is windows, override+ assert backend._memory_state == (False, False)+++def test_an_igpu_load_that_fits_keeps_llama_cpps_default(tmp_path, monkeypatch):+ """The control: room to run, so nothing is overridden; none is the faster load on an iGPU."""+ backend, gguf = _igpu_backend(tmp_path, monkeypatch, avail_mib = 400 * 1024)++ cmd = _launch(backend, gguf)["cmd"]++ assert "--load-mode" not in cmd and "--lazy-mode" not in cmd, cmd+ assert "--fit-target" not in cmd, cmd+ assert backend.last_load_warning is None+++ "igpu, load_mode_auto",+ [(False, True), (True, False)],+ ids = ["discrete-gpu-auto-is-mmap", "build-without-auto"],+)+def test_auto_is_left_alone_where_it_already_maps(tmp_path, monkeypatch, igpu, load_mode_auto):+ """A device with mmap support, or a build whose default is mmap everywhere, already pages."""+ backend, gguf = _igpu_backend(+ tmp_path, monkeypatch, avail_mib = 17 * 1024, igpu = igpu, load_mode_auto = load_mode_auto+ )++ cmd = _launch(backend, gguf)["cmd"]++ assert "--load-mode" not in cmd and "--lazy-mode" not in cmd, cmd+ assert "--fit-target" not in cmd, cmd+ assert "memory mapping instead" not in (backend.last_load_warning or "")+++def test_a_user_lazy_mode_survives_the_igpu_override(tmp_path, monkeypatch):+ """The mapping is owed, the lazy mode is the user's: only an unset one is restored."""+ backend, gguf = _igpu_backend(tmp_path, monkeypatch, avail_mib = 17 * 1024)++ cmd = _launch(backend, gguf, extra_args = ["--lazy-mode", "off"])["cmd"]++ assert _modes(cmd, "--load-mode") == ["mmap"], cmd+ assert _modes(cmd, "--lazy-mode") == ["off"], cmd+++def test_a_user_fit_target_survives_the_igpu_override(tmp_path, monkeypatch):+ monkeypatch.setattr(llama_cpp_module, "_paged_load_needs_fit_headroom", lambda: True)+ backend, gguf = _igpu_backend(tmp_path, monkeypatch, avail_mib = 17 * 1024)++ cmd = _launch(backend, gguf, extra_args = ["--fit-target", "4096"])["cmd"]++ assert _modes(cmd, "--load-mode") == ["mmap"], cmd+ assert _modes(cmd, "--fit-target") == ["4096"], cmd+++ "argv, env, user, expected",+ [+ (["--fit", "on"], {}, [], ["--fit-target", "16384"]),+ (["--fit", "on", "--fit-target", "3072"], {}, [], ["--fit-target", "18432"]),+ (["--fit", "off"], {}, [], []),+ (["--fit", "on", "--fit-target", "1024,2048"], {}, [], []),+ (["--fit", "on", "--fit-target", "4096"], {}, ["--fit-target", "4096"], []),+ (["--fit", "on"], {"LLAMA_ARG_FIT_TARGET": "2048"}, [], []),+ (["--fit", "on", "--device", "Vulkan0,Vulkan1"], {}, [], []),+ (["--fit", "on"], {"LLAMA_ARG_DEVICE": "Vulkan0,Vulkan1"}, [], []),+ (["--fit", "on", "--device", "Vulkan0,none"], {}, [], ["--fit-target", "16384"]),+ (["--fit", "on", "--rpc", "host:50052"], {}, [], []),+ ],+ ids = [+ "default",+ "adds-to-studio-margin",+ "fitter-off",+ "per-device-list",+ "user-argv",+ "user-env",+ "mixed-device-list",+ "mixed-device-env",+ "one-device-and-cpu",+ "rpc",+ ],+)+def test_the_igpu_fit_headroom(monkeypatch, argv, env, user, expected):+ monkeypatch.setattr(llama_cpp_module, "_paged_load_needs_fit_headroom", lambda: True)+ assert LlamaCppBackend._paged_fit_headroom_flags(argv, env, user) == expected+ assert LlamaCppBackend._paged_fit_headroom_flags(argv, env, user, gpu_count = 2) == []+ monkeypatch.setattr(llama_cpp_module, "_paged_load_needs_fit_headroom", lambda: False)+ assert LlamaCppBackend._paged_fit_headroom_flags(argv, env, user) == []+++def test_the_igpu_fit_headroom_survives_the_per_device_margin(tmp_path, monkeypatch):+ """A non-default budget spells --fit-target per device at spawn; that rewrite keeps the headroom."""+ monkeypatch.setattr(llama_cpp_module, "_paged_load_needs_fit_headroom", lambda: True)+ monkeypatch.setattr(llama_cpp_module, "_active_vram_fraction", lambda: 0.9)+ monkeypatch.setattr(LlamaCppBackend, "_GPU_IDS_ARE_PCI_INDICES", True)+ monkeypatch.setenv("CUDA_DEVICE_ORDER", "PCI_BUS_ID")+ monkeypatch.delenv("CUDA_VISIBLE_DEVICES", raising = False)+ backend, gguf = _igpu_backend(tmp_path, monkeypatch, avail_mib = 17 * 1024)+ LlamaCppBackend.probe_server_capabilities()["supports_fit_target"] = True++ cmd = _launch(backend, gguf)["cmd"]++ assert _modes(cmd, "--load-mode") == ["mmap"], cmd+ assert (+ float(_modes(cmd, "--fit-target")[-1]) > llama_cpp_module._IGPU_PAGED_FIT_HEADROOM_MIB+ ), cmd+++def test_a_retry_off_the_igpu_withdraws_the_paging_headroom():+ """The arch-crash respawn on a discrete sibling must not keep the iGPU's 15 GiB margin."""+ import inspect++ headroom = ["--fit-target", "16384"]+ cmd = ["-m", "x.gguf", "--fit-target", "1024", "--load-mode", "mmap", *headroom]+ added = LlamaCppBackend._margins_with_paged_headroom({0: 1024.0, 1: 2048.0}, headroom)+ assert added == {0: 16384.0, 1: 17408.0}+ out, margins = LlamaCppBackend._withdraw_paged_headroom(cmd, added, headroom)+ assert out == ["-m", "x.gguf", "--fit-target", "1024", "--load-mode", "mmap"]+ assert margins == {0: 1024.0, 1: 2048.0}+ assert LlamaCppBackend._withdraw_paged_headroom(cmd, None, []) == (cmd, None)++ src = inspect.getsource(LlamaCppBackend.load_model)+ retry = src[src.index("_retry_auto_unmapped = self._auto_load_mode_unmapped(") :]+ withdraw = retry.index("if _paged_headroom and not _retry_auto_unmapped:")+ assert withdraw < retry.index("self._page_an_oversized_unmapped_load(")+ assert "auto_unmapped = _retry_auto_unmapped" in retry+++def test_an_igpu_model_without_a_table_is_only_mapped(tmp_path, monkeypatch):+ backend, gguf = _igpu_backend(tmp_path, monkeypatch, avail_mib = 17 * 1024, table = 0)++ cmd = _launch(backend, gguf)["cmd"]++ assert _modes(cmd, "--load-mode") == ["mmap"], cmd+ assert "--lazy-mode" not in cmd, cmd+++def test_an_explicit_none_on_an_igpu_also_streams_the_table(tmp_path, monkeypatch):+ """The rewrite of a user's none keeps its wording and gains the lazy read on an iGPU."""+ backend, gguf = _igpu_backend(tmp_path, monkeypatch, avail_mib = 17 * 1024)++ launched = _launch(backend, gguf, extra_args = ["--load-mode", "none"])+ cmd = launched["cmd"]++ assert not _unmapped_tokens(cmd), cmd+ assert _modes(cmd, "--load-mode")[-1:] == ["mmap"], cmd+ assert resolve_effective_load_state(cmd, launched["env"], auto_unmapped = True)[1] is False+ assert _modes(cmd, "--lazy-mode") == ["on"], cmd+ assert "was set to load without memory mapping" in (backend.last_load_warning or "")++def _unmapped_tokens(cmd):"""The tokens in ``cmd`` that select a mode llama.cpp does not mmap."""out = []@@ -5779,7 +6020,7 @@ def test_a_device_list_still_sees_an_auto_filtered_vulkan_igpu(monkeypatch):assert src.index("_vulkan_igpu_ids = (") < src.index("_gpu_mem = self._vulkan_auto_gpu_memory(_gpu_mem)")- assert src.count("integrated_gpu_ids = _vulkan_igpu_ids") == 5+ assert src.count("integrated_gpu_ids = _vulkan_igpu_ids") == 7def test_the_oversized_readings_share_their_preflights_ram_snapshot():@@ -7075,3 +7316,32 @@ def test_the_checkpoint_discount_removes_only_the_checkpoints():without = backend._slots_that_fit_on_gpu(*args, ctx_checkpoints = 0, **kw)assert with_ckpts == withoutassert without[1] is False and without[2] < 8+++ "note, expected",+ [+ (+ llama_cpp_module._PAGEABLE_AUTO_OVERRIDE_NOTE,+ llama_cpp_module._PAGEABLE_AUTO_OVERRIDE_TEXT_ONLY_NOTICE,+ ),+ (+ llama_cpp_module._PAGEABLE_OVERRIDE_NOTE,+ llama_cpp_module._PAGEABLE_OVERRIDE_TEXT_ONLY_NOTICE,+ ),+ ],+ ids = ["igpu-auto", "user-none"],+)+def test_the_text_only_notice_keeps_who_chose_the_unmapped_mode(note, expected):+ """The iGPU note must not claim the user asked for an unmapped load."""+ backend = LlamaCppBackend()+ apu_msg = "This model needs about 70 GB but only about 60 GB of memory is available."+ backend._last_load_warning = apu_msg + note+ backend._reprice_after_dropping_pinned_projector(+ apu_msg = apu_msg,+ host_msg = None,+ model_size = 50 * 1024**3,+ pinned_bytes = 2 * 1024**3,+ avail_mib = 60 * 1024,+ )+ assert backend._last_load_warning == expecteddiff --git a/studio/backend/tests/test_llama_server_args.py b/studio/backend/tests/test_llama_server_args.pyindex a3bf4865bc4..97d0a917d04 100644--- a/studio/backend/tests/test_llama_server_args.py+++ b/studio/backend/tests/test_llama_server_args.py@@ -1568,3 +1568,73 @@ def test_translate_ngram_mod_does_not_consume_a_following_flag(chain_with_mtp):out = translate_ngram_mod_args(args, "legacy", chain_with_mtp = chain_with_mtp)assert out[-2:] == ["--top-k", "/restricted/model.gguf"]validate_extra_args(out)+++ "argv, env, auto",+ [+ ([], {}, True),+ (["--load-mode", "auto"], {}, True),+ (["--load-mode=auto"], {}, True),+ (["--load-mode", "none", "--load-mode", "auto"], {}, True),+ ([], {"LLAMA_ARG_LOAD_MODE": "auto"}, True),+ (["--load-mode", "auto"], {"LLAMA_ARG_LOAD_MODE": "mmap"}, True),+ (["--load-mode", "mmap"], {}, False),+ (["--load-mode", "none"], {}, False),+ ([], {"LLAMA_ARG_LOAD_MODE": "mmap"}, False),+ (["--mlock"], {}, False),+ (["--mmap"], {}, False),+ (["--no-mmap"], {}, False),+ ([], {"LLAMA_ARG_NO_MMAP": "0"}, False),+ ([], {"LLAMA_ARG_DIO": "1"}, False),+ (["--load-mode"], {}, True),+ ],+)+def test_load_mode_left_auto(argv, env, auto):+ assert _lsa.load_mode_left_auto(argv, env) is auto+++def test_auto_reserves_ram_only_where_llama_cpp_resolves_it_to_none():+ assert _lsa.resolve_effective_load_state([], {}) == (False, False, False)+ assert _lsa.resolve_effective_load_state([], {}, auto_unmapped = True) == (False, True, False)+ assert _lsa.resolve_effective_memory_state(["--load-mode", "auto"], {}, auto_unmapped = True) == (+ False,+ True,+ )+ assert _lsa.resolve_effective_load_state(["--load-mode", "mmap"], {}, auto_unmapped = True) == (+ False,+ False,+ False,+ )+ assert _lsa.resolve_effective_load_state(["--load-mode", "dio"], {}, auto_unmapped = True) == (+ False,+ False,+ True,+ )+++def test_the_pageable_rewrite_maps_an_unselected_mode_on_an_igpu():+ out, overridden = _lsa.force_pageable_load(["--fit", "on"], {}, auto_unmapped = True)+ assert out == ["--fit", "on", "--load-mode", "mmap"] and overridden+ assert _lsa.resolve_effective_memory_state(out, {}, auto_unmapped = True) == (False, False)+ # An inherited LLAMA_ARG_LOAD_MODE=auto loses to the appended argv.+ env = {"LLAMA_ARG_LOAD_MODE": "auto"}+ out, _ = _lsa.force_pageable_load([], env, auto_unmapped = True)+ assert _lsa.resolve_effective_memory_state(out, env, auto_unmapped = True) == (False, False)+ for argv, env in (+ (["--load-mode", "none"], {}),+ (["--no-mmap"], {}),+ ([], {"LLAMA_ARG_NO_MMAP": "1"}),+ ):+ out, overridden = _lsa.force_pageable_load(argv, env, auto_unmapped = True)+ assert overridden and out[-2:] == ["--load-mode", "mmap"], (argv, env, out)+ assert _lsa.resolve_effective_memory_state(out, env, auto_unmapped = True) == (False, False)+ assert _lsa.force_pageable_load(["--load-mode", "mlock"], {}, auto_unmapped = True)[0] == [+ "--load-mode",+ "mmap+mlock",+ ]+ assert _lsa.force_pageable_load([], {}) == ([], [])+ assert _lsa.force_pageable_load(["--load-mode", "mmap"], {}, auto_unmapped = True) == (+ ["--load-mode", "mmap"],+ [],+ )diff --git a/studio/backend/tests/test_load_mode_caps_unknown.py b/studio/backend/tests/test_load_mode_caps_unknown.pyindex a2eb32e3d64..2cfa89fd1a7 100644--- a/studio/backend/tests/test_load_mode_caps_unknown.py+++ b/studio/backend/tests/test_load_mode_caps_unknown.py@@ -27,6 +27,9 @@ from core.inference.llama_cpp import LlamaCppBackendfrom core.inference.llama_server_args import apply_load_mode_policy, resolve_launch_load_mode_NO_SETTINGS = (False, False)+posix_only = pytest.mark.skipif(+ sys.platform == "win32", reason = "shell script stands in for the binary"+)def _fake_server(tmp_path, body: str) -> str:@@ -81,12 +84,14 @@ def test_a_failed_help_probe_emits_no_load_flag(tmp_path):assert _launch_flags(caps) == []+@posix_onlydef test_an_empty_help_probe_emits_no_load_flag(tmp_path):caps = LlamaCppBackend.probe_server_capabilities(_fake_server(tmp_path, "exit 0"))assert caps["help_probe_ok"] is True and not caps["flags"]assert _launch_flags(caps) == []+@posix_onlydef test_a_build_listing_only_the_old_flags_still_gets_them(tmp_path):help_text = "--no-mmap do not memory-map model\n--mlock lock model in RAM"caps = LlamaCppBackend.probe_server_capabilities(@@ -96,6 +101,7 @@ def test_a_build_listing_only_the_old_flags_still_gets_them(tmp_path):assert _launch_flags(caps) == ["--no-mmap"]+@posix_onlydef test_a_build_listing_load_mode_gets_it(tmp_path):help_text = "--load-mode MODE how to load the model"caps = LlamaCppBackend.probe_server_capabilities(@@ -176,3 +182,45 @@ def test_no_launch_site_reads_the_raw_load_mode_cap():assert 'server_caps.get("supports_load_mode")' not in sourceassert "server_caps['supports_load_mode']" not in sourceassert 'server_caps["supports_load_mode"]' not in source+++# llama-server --help for --load-mode, verbatim: b11643 (ggml-org/llama.cpp#26081 added+# "auto", which resolves to none on an iGPU) and the earlier #20834 block, default mmap.+_LOAD_MODE_HELP_WITH_AUTO = """\+-lm, --load-mode MODE model loading mode (default: auto)+ - auto: mmap, unless a device does not support it+ - none: no special loading mode+ - mmap: memory-map model (if mmap disabled, slower load but may reduce+ pageouts if not using mlock)+ - mlock: force system to keep model in RAM rather than swapping or+ compressing+ - mmap+mlock: mmap + force system to keep model in RAM rather than+ swapping or compressing+ - dio: use DirectIO if available+ (env: LLAMA_ARG_LOAD_MODE)"""+_LOAD_MODE_HELP_WITHOUT_AUTO = """\+-lm, --load-mode MODE model loading mode (default: mmap)+ - none: no special loading mode+ - mmap: memory-map model (if mmap disabled, slower load but may reduce+ pageouts if not using mlock)+ - mlock: mmap + force system to keep model in RAM rather than swapping+ or compressing+ - dio: use DirectIO if available+ (env: LLAMA_ARG_LOAD_MODE)"""+++@posix_only+ "help_text, auto",+ [+ (_LOAD_MODE_HELP_WITH_AUTO, True),+ (_LOAD_MODE_HELP_WITHOUT_AUTO, False),+ ("--mmap memory-map model (default: auto)", False),+ ],+ ids = ["auto-build", "load-mode-without-auto", "legacy-flags-only"],+)+def test_the_probe_reads_whether_the_build_lists_load_mode_auto(tmp_path, help_text, auto):+ caps = LlamaCppBackend.probe_server_capabilities(+ _fake_server(tmp_path, f"cat <<'X'\n{help_text}\nX\nexit 0")+ )+ assert caps["supports_load_mode_auto"] is auto