Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

Studio: memory-map an oversized load on an integrated GPU and read its lookup tables from disk (#16857)

oobabooga3db62df24308
diff --git a/studio/backend/core/inference/llama_cpp.py b/studio/backend/core/inference/llama_cpp.py
index 5fc861e8fe3..81d725f0630 100644
--- a/studio/backend/core/inference/llama_cpp.py
+++ b/studio/backend/core/inference/llama_cpp.py
@@ -130,6 +130,7 @@ from core.inference.llama_server_args import (
extra_args_select_load_mode,
fit_is_enabled_in,
force_pageable_load,
+ load_mode_left_auto,
memory_env_selects_load_mode,
memory_state_satisfies_settings,
fit_is_effectively_on,
@@ -4125,6 +4126,15 @@ _VRAM_FLOOR_RESERVE_MIB = 512.0
# llama-server's own --fit-target default ("target margin per device for --fit,
# ... default: 1024"): what the fallback fitter leaves when we pass nothing.
_LLAMA_FIT_TARGET_DEFAULT_MIB = 1024.0
+# Extra --fit-target for a load paged on a Windows iGPU, whose GPU share and mapped pages share one
+# memory: on a Strix Halo 8 GiB still aborted at warm-up, 16 GiB ran. Linux runs slower with it.
+_IGPU_PAGED_FIT_HEADROOM_MIB = 15360
+
+
+def _paged_load_needs_fit_headroom() -> bool:
+ """Whether a load paged on an iGPU needs ``_IGPU_PAGED_FIT_HEADROOM_MIB``."""
+ return sys.platform == "win32"
+
# Windows + NVIDIA since driver 536.40: WDDM spills VRAM overflow to host RAM instead of failing, so the reserve matches llama.cpp fit margin.
_WINDOWS_SYSMEM_FALLBACK_RESERVE_MIB = _LLAMA_FIT_TARGET_DEFAULT_MIB
@@ -7779,6 +7789,11 @@ _PAGEABLE_OVERRIDE_NOTE = (
"RAM up front and cannot complete at this size, so Unsloth loaded it with memory "
"mapping instead."
)
+# The override where nobody chose the mode (auto is none on an iGPU); lazy and fit flags are logged.
+_PAGEABLE_AUTO_OVERRIDE_NOTE = (
+ " Unsloth turned on memory mapping, which llama.cpp leaves off by default on this "
+ "integrated GPU."
+)
# The override is reported, not undone: the running server is already memory-mapped.
_PAGEABLE_OVERRIDE_TEXT_ONLY_NOTICE = (
"This model was set to load without memory mapping, and with its vision projector "
@@ -7787,6 +7802,13 @@ _PAGEABLE_OVERRIDE_TEXT_ONLY_NOTICE = (
"which the weights alone would have fit. The running server keeps the memory "
"mapping it started with; load the model again to run it as you asked."
)
+_PAGEABLE_AUTO_OVERRIDE_TEXT_ONLY_NOTICE = (
+ "With its vision projector in system RAM this model could run out of memory without "
+ "memory mapping, llama.cpp's default on this integrated GPU, so Unsloth loaded it with "
+ "memory mapping instead. The projector then failed to start and this session is "
+ "text-only, which the weights alone would have fit. The running server keeps the "
+ "memory mapping it started with."
+)
_CGROUP_ROOT = "/sys/fs/cgroup"
_PROC_SELF_CGROUP = "/proc/self/cgroup"
@@ -10366,6 +10388,7 @@ class LlamaCppBackend:
"supports_video_timestamp_interval": False,
"supports_video_ffmpeg_dir": None,
"supports_load_mode": False,
+ "supports_load_mode_auto": False,
"supports_moe_cache": False,
"supports_moe_cache_auto": False,
"supports_lazy_mode": False,
@@ -10437,6 +10460,7 @@ class LlamaCppBackend:
supports_video_timestamp_interval = False
supports_video_ffmpeg_dir = None
supports_load_mode = False
+ supports_load_mode_auto = False
supports_moe_cache = False
supports_moe_cache_auto = False
supports_lazy_mode = False
@@ -10651,6 +10675,14 @@ class LlamaCppBackend:
# --load-mode supersedes --mlock / --no-mmap, which are deprecated.
# Pre-initialised above: a failed probe must fall back, not raise.
supports_load_mode = _is_real("--load-mode")
+ # "auto: mmap, unless a device does not support it": without it, mmap is the default everywhere.
+ supports_load_mode_auto = bool(
+ supports_load_mode
+ and re.search(
+ r"(?<![a-z0-9_+-])auto(?![a-z0-9_+-])",
+ (blocks.get("--load-mode") or "").lower(),
+ )
+ )
# "auto" is a fork addition; upstream builds exit on it, so fail closed.
supports_moe_cache = bool(probe_ok and _is_real("--moe-cache-mib"))
supports_moe_cache_auto = bool(
@@ -10764,6 +10796,7 @@ class LlamaCppBackend:
"supports_video_timestamp_interval": supports_video_timestamp_interval,
"supports_video_ffmpeg_dir": supports_video_ffmpeg_dir,
"supports_load_mode": supports_load_mode,
+ "supports_load_mode_auto": supports_load_mode_auto,
"supports_moe_cache": supports_moe_cache,
"supports_moe_cache_auto": supports_moe_cache_auto,
"supports_lazy_mode": supports_lazy_mode,
@@ -15671,12 +15704,131 @@ class LlamaCppBackend:
"yes",
)
+ def _auto_load_mode_unmapped(
+ self,
+ server_caps: Optional[Mapping[str, object]],
+ *,
+ child_has_no_gpu: bool = False,
+ **devices,
+ ) -> bool:
+ """Whether llama.cpp resolves ``--load-mode auto`` to ``none`` here: a device lacking
+ ``mmap_support`` turns both auto modes off (``load_tensors``); ``devices`` go to
+ ``_lazy_auto_resolves_off``."""
+ if child_has_no_gpu or not (server_caps or {}).get("supports_load_mode_auto"):
+ return False
+ try:
+ return bool(self._lazy_auto_resolves_off(**devices))
+ except Exception as e: # noqa: BLE001 -- unknown devices: keep the launch as is
+ logger.debug("Could not tell whether llama.cpp maps this load: %s", e)
+ return False
+
+ def _paged_lookup_table_flags(
+ self,
+ model_path: Optional[str],
+ caps: Optional[Mapping[str, object]],
+ argv: Iterable[str],
+ env: Optional[Mapping[str, str]] = None,
+ ) -> list[str]:
+ """``[<lazy flag>, "on"]`` to stream the tables auto streams elsewhere: mmap prefetch skips
+ only lazy ranges (llama-mmap.cpp). Empty when a lazy mode is set or nothing would stream."""
+ caps = caps or {}
+ if not model_path or not caps.get("supports_lazy_mode"):
+ return []
+ flag = str(caps.get("lazy_mode_flag") or "--lazy-mode")
+ source_env = os.environ if env is None else env
+ flags, env_name = _LAZY_MODE_SPELLINGS.get(flag, (frozenset({flag}), ""))
+ argv = list(argv or ())
+ if (env_name and str(source_env.get(env_name) or "").strip()) or _extra_args_set_any_flag(
+ argv, flags
+ ):
+ return []
+ if self._lazy_read_host_bytes(model_path, caps = caps, extra_args = argv, env = source_env) <= 0:
+ return []
+ return [flag, "on"]
+
+ @staticmethod
+ def _paged_fit_headroom_flags(
+ argv: Iterable[str],
+ env: Optional[Mapping[str, str]],
+ user_args: Optional[Iterable[str]],
+ gpu_count: int = 1,
+ ) -> list[str]:
+ """``--fit-target`` raised by ``_IGPU_PAGED_FIT_HEADROOM_MIB`` over the launch's margin.
+ Empty off Windows, with the fitter off, a user margin, a per-device list, or a second
+ device, which the broadcast margin would starve."""
+ args = [str(a) for a in argv or ()]
+ if not _paged_load_needs_fit_headroom() or gpu_count > 1:
+ return []
+ if (
+ _extra_args_device(args, {"--rpc"})
+ or str((env or {}).get("LLAMA_ARG_RPC") or "").strip()
+ ):
+ return []
+ devices = _extra_args_device(args, {"--device", "-dev"})
+ if devices is None:
+ devices = str((env or {}).get("LLAMA_ARG_DEVICE") or "")
+ if (
+ len(
+ [
+ d
+ for d in devices.split(",")
+ if d.strip().lower() not in ("", *_CPU_DEVICE_VALUES)
+ ]
+ )
+ > 1
+ ):
+ return []
+ if not fit_is_effectively_on(args, env) or _fit_target_values(user_args or (), env):
+ return []
+ raw = _last_flag_value(args, {"-fitt", "--fit-target"})
+ if (
+ raw is not None
+ and len([p for p in str(raw).replace("/", ",").split(",") if p.strip()]) > 1
+ ):
+ return []
+ have = _single_device_fit_margin(args, None)
+ base = _LLAMA_FIT_TARGET_DEFAULT_MIB if have is None else have
+ return ["--fit-target", str(math.ceil(base + _IGPU_PAGED_FIT_HEADROOM_MIB))]
+
+ @staticmethod
+ def _margins_with_paged_headroom(
+ margins: Optional[Mapping[int, float]],
+ headroom: Iterable[str],
+ sign: int = 1,
+ ) -> Optional[Mapping[int, float]]:
+ """The spawn respells ``--fit-target`` per device from ``margins``, so they carry the headroom too."""
+ if not margins or not list(headroom):
+ return margins
+ return {i: m + sign * _IGPU_PAGED_FIT_HEADROOM_MIB for i, m in margins.items()}
+
+ @staticmethod
+ def _withdraw_paged_headroom(
+ cmd: list[str], margins: Optional[Mapping[int, float]], headroom: list[str]
+ ) -> tuple[list[str], Optional[Mapping[int, float]]]:
+ """``cmd`` and ``margins`` without the iGPU paging headroom, for a respawn off the iGPU."""
+ return (
+ LlamaCppBackend._without_flag_pairs(cmd, headroom),
+ LlamaCppBackend._margins_with_paged_headroom(margins, headroom, sign = -1),
+ )
+
def _page_an_oversized_unmapped_load(
- self, cmd: Iterable[str], env: Optional[dict], oversized
+ self,
+ cmd: Iterable[str],
+ env: Optional[dict],
+ oversized,
+ *,
+ auto_unmapped: bool = False,
+ lazy_table_flags: Optional[Callable[[list], list]] = None,
+ fit_headroom_flags: Optional[Callable[[list], list]] = None,
) -> tuple[list, Optional[str]]:
- """Returns (cmd, note); rewrites an oversized non-pageable load to mmap so it finishes; judges effective env state, not argv; never blocks."""
+ """Returns (cmd, note); rewrites an oversized non-pageable load to mmap so it finishes; judges effective env state, not argv; never blocks.
+
+ ``auto_unmapped``: auto is none for this child, so an unset mode is rewritten too, and
+ ``lazy_table_flags(argv)`` / ``fit_headroom_flags(argv)`` are appended."""
original = cmd if isinstance(cmd, list) else list(cmd or ())
- _mlock_now, _reserves_ram_now = resolve_effective_memory_state(original, env)
+ _mlock_now, _reserves_ram_now = resolve_effective_memory_state(
+ original, env, auto_unmapped = auto_unmapped
+ )
if not _reserves_ram_now:
return original, None
try:
@@ -15686,15 +15838,33 @@ class LlamaCppBackend:
# Advisory pricing: an unreadable host or GGUF leaves the launch alone.
logger.debug("Could not price the load for a pageable override.", exc_info = True)
return original, None
- paged, overridden = force_pageable_load(original, env)
+ from_auto = auto_unmapped and load_mode_left_auto(original, env)
+ paged, overridden = force_pageable_load(original, env, auto_unmapped = auto_unmapped)
if not overridden:
return original, None
+ lazy: list = []
+ if auto_unmapped and lazy_table_flags is not None:
+ try:
+ lazy = list(lazy_table_flags(paged) or [])
+ except Exception: # noqa: BLE001 -- the mapping alone still lets the load finish
+ logger.debug("Lookup tables: could not price lazy reads.", exc_info = True)
+ paged = [*paged, *lazy]
+ headroom: list = []
+ if auto_unmapped and fit_headroom_flags is not None:
+ try:
+ headroom = list(fit_headroom_flags(paged) or [])
+ except Exception: # noqa: BLE001 -- the mapping still applies without the margin
+ logger.debug("Fit headroom: could not read the launch's margin.", exc_info = True)
+ paged = [*paged, *headroom]
logger.warning(
"Overriding the unmapped load mode (%s) for an oversized model: it would "
- "allocate the whole model in host RAM instead of paging it in.",
+ "allocate the whole model in host RAM instead of paging it in.%s%s",
", ".join(overridden),
+ f" Lookup tables: {' '.join(lazy)}." if lazy else "",
+ f" GPU headroom: {' '.join(headroom)}." if headroom else "",
)
- return paged, _PAGEABLE_OVERRIDE_NOTE
+ note = _PAGEABLE_AUTO_OVERRIDE_NOTE if from_auto else _PAGEABLE_OVERRIDE_NOTE
+ return paged, note
@property
def last_load_warning(self) -> Optional[str]:
@@ -15750,7 +15920,11 @@ class LlamaCppBackend:
# The discrete guard never charged the projector, so its verdict stands.
self._last_load_warning = host_msg + suffix
elif suffix.strip():
- self._last_load_warning = _PAGEABLE_OVERRIDE_TEXT_ONLY_NOTICE
+ self._last_load_warning = (
+ _PAGEABLE_AUTO_OVERRIDE_TEXT_ONLY_NOTICE
+ if _PAGEABLE_AUTO_OVERRIDE_NOTE in suffix
+ else _PAGEABLE_OVERRIDE_TEXT_ONLY_NOTICE
+ )
else:
self._last_load_warning = None
logger.info(
@@ -23527,6 +23701,8 @@ class LlamaCppBackend:
_fit_target_delta_by_idx: Optional[dict[int, float]] = None
# Studio's emitted margins per card, spelled into --fit-target at spawn.
_fit_target_margins_by_idx: Optional[dict[int, float]] = None
+ # --fit-target pair a paged iGPU launch appended, withdrawn if a retry leaves the iGPU.
+ _paged_headroom: list = []
# Set only if that margin actually reached the command line, which is
# what makes the child budget-priced in a mode the planner skipped.
_fit_target_priced = False
@@ -30597,8 +30773,38 @@ class LlamaCppBackend:
)
)
+ # On an iGPU auto is none; same devices as the guard above.
+ _launch_auto_unmapped = self._auto_load_mode_unmapped(
+ server_caps,
+ child_has_no_gpu = _child_has_no_gpu,
+ gpu_indices = [row[0] for row in _reachable_gpus] or None,
+ detected_gpus = _reachable_gpus,
+ shared_gpu_ids = _shared_gpu_ids,
+ is_vulkan_backend = is_vulkan_backend,
+ probe_integrated_cuda = False,
+ extra_args = cmd,
+ env = env,
+ device_list = False,
+ integrated_gpu_ids = _vulkan_igpu_ids,
+ )
+ _paged_headroom = []
cmd, _pageable_note = self._page_an_oversized_unmapped_load(
- cmd, env, _launch_is_oversized
+ cmd,
+ env,
+ _launch_is_oversized,
+ auto_unmapped = _launch_auto_unmapped,
+ lazy_table_flags = lambda argv: self._paged_lookup_table_flags(
+ model_path, server_caps, argv, env
+ ),
+ fit_headroom_flags = lambda argv: _paged_headroom.extend(
+ self._paged_fit_headroom_flags(
+ argv, env, extra_args, gpu_count = len(_reachable_gpus)
+ )
+ )
+ or _paged_headroom,
+ )
+ _fit_target_margins_by_idx = self._margins_with_paged_headroom(
+ _fit_target_margins_by_idx, _paged_headroom
)
if _pageable_note:
# Record the argv that actually launches, and snapshot it too, since arch-crash retry restores it.
@@ -32181,8 +32387,41 @@ class LlamaCppBackend:
)
)
+ _retry_auto_unmapped = self._auto_load_mode_unmapped(
+ server_caps,
+ gpu_indices = [row[0] for row in _retry_rows] or None,
+ detected_gpus = _retry_rows,
+ shared_gpu_ids = _shared_gpu_ids,
+ is_vulkan_backend = is_vulkan_backend,
+ probe_integrated_cuda = False,
+ extra_args = cmd,
+ env = env,
+ device_list = False,
+ integrated_gpu_ids = _vulkan_igpu_ids,
+ )
+ if _paged_headroom and not _retry_auto_unmapped:
+ cmd, _fit_target_margins_by_idx = self._withdraw_paged_headroom(
+ cmd, _fit_target_margins_by_idx, _paged_headroom
+ )
+ _paged_headroom = []
+ _retry_headroom: list = []
cmd, _retry_pageable_note = self._page_an_oversized_unmapped_load(
- cmd, env, _retry_is_oversized
+ cmd,
+ env,
+ _retry_is_oversized,
+ auto_unmapped = _retry_auto_unmapped,
+ lazy_table_flags = lambda argv: self._paged_lookup_table_flags(
+ model_path, server_caps, argv, env
+ ),
+ fit_headroom_flags = lambda argv: _retry_headroom.extend(
+ self._paged_fit_headroom_flags(
+ argv, env, extra_args, gpu_count = len(_retry_rows)
+ )
+ )
+ or _retry_headroom,
+ )
+ _fit_target_margins_by_idx = self._margins_with_paged_headroom(
+ _fit_target_margins_by_idx, _retry_headroom
)
if _retry_pageable_note:
# Onto whichever notice first-notice-wins kept above; a
diff --git a/studio/backend/core/inference/llama_server_args.py b/studio/backend/core/inference/llama_server_args.py
index 6708b4d641c..974bed3c2b0 100644
--- a/studio/backend/core/inference/llama_server_args.py
+++ b/studio/backend/core/inference/llama_server_args.py
@@ -1716,10 +1716,17 @@ def _pageable_env_value(
def force_pageable_load(
- argv: Optional[Iterable[str]], env: Optional[dict] = None
+ argv: Optional[Iterable[str]],
+ env: Optional[dict] = None,
+ *,
+ auto_unmapped: bool = False,
) -> tuple[list[str], list[str]]:
- """Rewrites none/mlock to pageable: effective locks become mmap+mlock, shadowed locks stay dropped; decided by pre-rewrite state, not tokens."""
+ """Rewrites none/mlock to pageable: effective locks become mmap+mlock, shadowed locks stay dropped; decided by pre-rewrite state, not tokens.
+
+ ``auto_unmapped``: auto is none on this child, so an unset mode gets ``--load-mode mmap``."""
tokens = [str(a) for a in (argv or [])]
+ if auto_unmapped and load_mode_left_auto(tokens, env):
+ return [*tokens, "--load-mode", "mmap"], ["auto, which is none on this GPU"]
# Only RAM-reserving launches are rewritten; pageable dio or mmap keeps every token, mlock included.
_mlock_now, _reserves_now = resolve_effective_memory_state(tokens, env)
drop_shadowed_mlock = _reserves_now and not _mlock_now
@@ -1772,6 +1779,9 @@ def force_pageable_load(
else:
env[name] = new_value
overridden.append(name)
+ # Dropping none/--no-mmap leaves auto, which is none again here.
+ if overridden and auto_unmapped and load_mode_left_auto(out, env):
+ out.extend(["--load-mode", "mmap"])
return out, overridden
@@ -1793,10 +1803,15 @@ _LOAD_MODE_RESERVING_VALUES = frozenset({"none", "mlock"})
def resolve_effective_memory_state(
- argv: Optional[Iterable[str]], env: Optional[Mapping[str, str]] = None
+ argv: Optional[Iterable[str]],
+ env: Optional[Mapping[str, str]] = None,
+ *,
+ auto_unmapped: bool = False,
) -> tuple[bool, bool]:
"""What the child runs with: env supplies defaults, argv overrides last-wins, as llama.cpp does."""
- mlock, reserves_ram, _direct_io = resolve_effective_load_state(argv, env)
+ mlock, reserves_ram, _direct_io = resolve_effective_load_state(
+ argv, env, auto_unmapped = auto_unmapped
+ )
return mlock, reserves_ram
@@ -1808,37 +1823,63 @@ def resolve_effective_direct_io(
def resolve_effective_load_state(
- argv: Optional[Iterable[str]], env: Optional[Mapping[str, str]] = None
+ argv: Optional[Iterable[str]],
+ env: Optional[Mapping[str, str]] = None,
+ *,
+ auto_unmapped: bool = False,
) -> tuple[bool, bool, bool]:
"""``(mlock, reserves_ram, direct_io)``. Every branch that resolves the mode
- assigns all three, so the DirectIO bit cannot fall out of step with the pair."""
+ assigns all three, so the DirectIO bit cannot fall out of step with the pair.
+
+ ``auto_unmapped``: a device lacks ``mmap_support`` (an iGPU), so ``auto`` is ``none``
+ (``load_tensors``), not mmap."""
+ mlock, reserves_ram, direct_io, auto = _resolve_load_state(argv, env)
+ if auto and auto_unmapped:
+ return False, True, False
+ return mlock, reserves_ram, direct_io
+
+
+def load_mode_left_auto(
+ argv: Optional[Iterable[str]], env: Optional[Mapping[str, str]] = None
+) -> bool:
+ """Whether nothing selects a mode other than ``auto``. Legacy flags count: only builds
+ without ``auto`` accept them."""
+ return _resolve_load_state(argv, env)[3]
+
+
+def _resolve_load_state(
+ argv: Optional[Iterable[str]], env: Optional[Mapping[str, str]] = None
+) -> tuple[bool, bool, bool, bool]:
+ """``(mlock, reserves_ram, direct_io, auto)``, last selector wins, env before argv."""
env = env or {}
mlock = False
reserves_ram = False
direct_io = False
+ auto = True
# Env vars run the flag's handler in registration order, so a later one overwrites the mode.
if str(env.get("LLAMA_ARG_MLOCK", "")).strip().lower() in _ENV_TRUE_VALUES:
# Either mode it can mean maps or buffers the weights; neither streams.
- mlock, direct_io = True, False
+ mlock, direct_io, auto = True, False, False
# LLAMA_ARG_NO_<NAME> present forces false whatever the value, and beats the affirmative var.
_mmap_env = "0" if "LLAMA_ARG_NO_MMAP" in env else str(env.get("LLAMA_ARG_MMAP", ""))
_mmap_env = _mmap_env.strip().lower()
if _mmap_env in _ENV_TRUE_VALUES:
- mlock, reserves_ram, direct_io = False, False, False
+ mlock, reserves_ram, direct_io, auto = False, False, False, False
elif _mmap_env in _ENV_FALSE_VALUES:
- mlock, reserves_ram, direct_io = False, True, False
+ mlock, reserves_ram, direct_io, auto = False, True, False, False
# LLAMA_ARG_DIO likewise: on selects DirectIO, off selects "none".
_dio_env = "0" if "LLAMA_ARG_NO_DIO" in env else str(env.get("LLAMA_ARG_DIO", ""))
_dio_env = _dio_env.strip().lower()
if _dio_env in _ENV_TRUE_VALUES:
- mlock, reserves_ram, direct_io = False, False, True
+ mlock, reserves_ram, direct_io, auto = False, False, True, False
elif _dio_env in _ENV_FALSE_VALUES:
- mlock, reserves_ram, direct_io = False, True, False
+ mlock, reserves_ram, direct_io, auto = False, True, False, False
_mode_env = str(env.get("LLAMA_ARG_LOAD_MODE", "")).strip().lower()
if _mode_env:
mlock = _mode_env in _LOAD_MODE_MLOCK_VALUES
reserves_ram = _mode_env in _LOAD_MODE_RESERVING_VALUES
direct_io = _mode_env == "dio"
+ auto = _mode_env == "auto"
tokens = [str(a) for a in (argv or [])]
i, n = 0, len(tokens)
@@ -1850,30 +1891,34 @@ def resolve_effective_load_state(
continue
if flag in _MLOCK_FLAGS:
# Only the mlock bit matters: mlock and mmap+mlock both replace an earlier DirectIO mode.
- mlock, direct_io = True, False
+ mlock, direct_io, auto = True, False, False
i += 1
elif flag in _NO_MMAP_FLAGS:
# --no-mmap resets the whole mode, clearing an earlier --mlock.
mlock = False
reserves_ram = True
direct_io = False
+ auto = False
i += 1
elif flag in _DIO_ON_FLAGS:
# Deprecated load-mode selector resets the mode, dropping mlock; DirectIO holds no full host copy.
mlock = False
reserves_ram = False
direct_io = True
+ auto = False
i += 1
elif flag in _DIO_OFF_FLAGS:
# Not plain mmap: upstream maps these to mode `none` (full host buffer).
mlock = False
reserves_ram = True
direct_io = False
+ auto = False
i += 1
elif flag == "--mmap":
mlock = False
reserves_ram = False
direct_io = False
+ auto = False
i += 1
elif flag in _LOAD_MODE_FLAGS:
if "=" in tok:
@@ -1887,10 +1932,11 @@ def resolve_effective_load_state(
mlock = value in _LOAD_MODE_MLOCK_VALUES
reserves_ram = value in _LOAD_MODE_RESERVING_VALUES
direct_io = value == "dio"
+ auto = value == "auto"
i += step
else:
i += 1
- return mlock, reserves_ram, direct_io
+ return mlock, reserves_ram, direct_io, auto
def memory_state_satisfies_settings(
diff --git a/studio/backend/tests/test_llama_cpp_placement.py b/studio/backend/tests/test_llama_cpp_placement.py
index 9700da85412..88b5eddb68b 100644
--- a/studio/backend/tests/test_llama_cpp_placement.py
+++ b/studio/backend/tests/test_llama_cpp_placement.py
@@ -2679,6 +2679,247 @@ def test_an_unmapped_apu_load_that_fits_is_left_exactly_as_asked(tmp_path, monke
assert backend.last_load_warning is None
+_GiB = 1024**3
+# The DeepSeek-V4.1-Flash UD-IQ1_S Engram table: 256 x 768022850 IQ4_NL rows.
+_DEEPSEEK41_TABLE = 110_595_290_400
+
+
+def _igpu_backend(
+ tmp_path,
+ monkeypatch,
+ *,
+ avail_mib,
+ gguf_gb = 216.97,
+ table = _DEEPSEEK41_TABLE,
+ igpu = True,
+ load_mode_auto = True,
+):
+ """An integrated CUDA SoC (RTX Spark N1X, DGX Spark): both auto modes resolve off."""
+ backend, gguf = _apu_backend(
+ tmp_path, gguf_gb = gguf_gb, avail_mib = avail_mib, monkeypatch = monkeypatch
+ )
+ backend._lazy_auto_resolves_off = lambda **_kw: igpu
+ backend._gguf_tensor_scan = lambda _path: (
+ "deepseek41",
+ {
+ "per_layer_token_embd.weight": table,
+ "blk.0.ffn_up_exps.weight": int(gguf_gb * _GiB) - table,
+ },
+ 0,
+ )
+ caps = dict(LlamaCppBackend.probe_server_capabilities.__func__(LlamaCppBackend, None))
+ caps.update(
+ supports_load_mode = True,
+ supports_load_mode_auto = load_mode_auto,
+ supports_lazy_mode = True,
+ lazy_mode_flag = "--lazy-mode",
+ )
+ monkeypatch.setattr(LlamaCppBackend, "probe_server_capabilities", lambda *a, **k: caps)
+ return backend, gguf
+
+
+from core.inference.llama_server_args import resolve_effective_load_state # noqa: E402
+
+
+ "extra_args, env",
+ [(["--no-mmap"], {}), ([], {"LLAMA_ARG_NO_MMAP": "1"}), (["--load-mode=none"], {})],
+ ids = ["no-mmap", "env-no-mmap", "load-mode-none-equals"],
+)
+def test_every_unmapped_selector_on_an_igpu_ends_up_mapped(tmp_path, monkeypatch, extra_args, env):
+ """Removing the selector is not enough there: the mode the child resolves must be pageable."""
+ backend, gguf = _igpu_backend(tmp_path, monkeypatch, avail_mib = 17 * 1024)
+ for name, value in env.items():
+ monkeypatch.setenv(name, value)
+
+ launched = _launch(backend, gguf, extra_args = extra_args)
+
+ assert (
+ resolve_effective_load_state(launched["cmd"], launched["env"], auto_unmapped = True)[1]
+ is False
+ )
+ assert "was set to load without memory mapping" in (backend.last_load_warning or "")
+
+
+def _modes(cmd, flag):
+ return [cmd[i + 1] for i, tok in enumerate(cmd) if tok == flag and i + 1 < len(cmd)]
+
+
[email protected]("platform", ["win32", "linux"])
+def test_an_oversized_igpu_load_left_on_auto_is_mapped_and_streams_its_table(
+ tmp_path, monkeypatch, platform
+):
+ """On an iGPU auto is none, so a 217 GB model on auto read its 103 GiB table into one buffer."""
+ monkeypatch.setattr(
+ llama_cpp_module, "_paged_load_needs_fit_headroom", lambda: platform == "win32"
+ )
+ backend, gguf = _igpu_backend(tmp_path, monkeypatch, avail_mib = 17 * 1024)
+ logged = _override_log(monkeypatch)
+
+ cmd = _launch(backend, gguf)["cmd"]
+
+ assert _modes(cmd, "--load-mode") == ["mmap"], cmd
+ assert _modes(cmd, "--lazy-mode") == ["on"], cmd
+ assert resolve_effective_load_state(cmd, {}, auto_unmapped = True)[1] is False
+ windows = platform == "win32"
+ assert _modes(cmd, "--fit-target")[-1:] == (["16384"] if windows else []), cmd
+ warning = backend.last_load_warning or ""
+ assert warning.endswith(
+ " Unsloth turned on memory mapping, which llama.cpp leaves off by default on this "
+ "integrated GPU."
+ ), warning
+ assert "was set to load without memory mapping" not in warning
+ override = [line for line in logged if "Overriding the unmapped load mode" in line]
+ assert override and "Lookup tables: --lazy-mode on." in override[0], logged
+ assert ("GPU headroom: --fit-target 16384." in override[0]) is windows, override
+ assert backend._memory_state == (False, False)
+
+
+def test_an_igpu_load_that_fits_keeps_llama_cpps_default(tmp_path, monkeypatch):
+ """The control: room to run, so nothing is overridden; none is the faster load on an iGPU."""
+ backend, gguf = _igpu_backend(tmp_path, monkeypatch, avail_mib = 400 * 1024)
+
+ cmd = _launch(backend, gguf)["cmd"]
+
+ assert "--load-mode" not in cmd and "--lazy-mode" not in cmd, cmd
+ assert "--fit-target" not in cmd, cmd
+ assert backend.last_load_warning is None
+
+
+ "igpu, load_mode_auto",
+ [(False, True), (True, False)],
+ ids = ["discrete-gpu-auto-is-mmap", "build-without-auto"],
+)
+def test_auto_is_left_alone_where_it_already_maps(tmp_path, monkeypatch, igpu, load_mode_auto):
+ """A device with mmap support, or a build whose default is mmap everywhere, already pages."""
+ backend, gguf = _igpu_backend(
+ tmp_path, monkeypatch, avail_mib = 17 * 1024, igpu = igpu, load_mode_auto = load_mode_auto
+ )
+
+ cmd = _launch(backend, gguf)["cmd"]
+
+ assert "--load-mode" not in cmd and "--lazy-mode" not in cmd, cmd
+ assert "--fit-target" not in cmd, cmd
+ assert "memory mapping instead" not in (backend.last_load_warning or "")
+
+
+def test_a_user_lazy_mode_survives_the_igpu_override(tmp_path, monkeypatch):
+ """The mapping is owed, the lazy mode is the user's: only an unset one is restored."""
+ backend, gguf = _igpu_backend(tmp_path, monkeypatch, avail_mib = 17 * 1024)
+
+ cmd = _launch(backend, gguf, extra_args = ["--lazy-mode", "off"])["cmd"]
+
+ assert _modes(cmd, "--load-mode") == ["mmap"], cmd
+ assert _modes(cmd, "--lazy-mode") == ["off"], cmd
+
+
+def test_a_user_fit_target_survives_the_igpu_override(tmp_path, monkeypatch):
+ monkeypatch.setattr(llama_cpp_module, "_paged_load_needs_fit_headroom", lambda: True)
+ backend, gguf = _igpu_backend(tmp_path, monkeypatch, avail_mib = 17 * 1024)
+
+ cmd = _launch(backend, gguf, extra_args = ["--fit-target", "4096"])["cmd"]
+
+ assert _modes(cmd, "--load-mode") == ["mmap"], cmd
+ assert _modes(cmd, "--fit-target") == ["4096"], cmd
+
+
+ "argv, env, user, expected",
+ [
+ (["--fit", "on"], {}, [], ["--fit-target", "16384"]),
+ (["--fit", "on", "--fit-target", "3072"], {}, [], ["--fit-target", "18432"]),
+ (["--fit", "off"], {}, [], []),
+ (["--fit", "on", "--fit-target", "1024,2048"], {}, [], []),
+ (["--fit", "on", "--fit-target", "4096"], {}, ["--fit-target", "4096"], []),
+ (["--fit", "on"], {"LLAMA_ARG_FIT_TARGET": "2048"}, [], []),
+ (["--fit", "on", "--device", "Vulkan0,Vulkan1"], {}, [], []),
+ (["--fit", "on"], {"LLAMA_ARG_DEVICE": "Vulkan0,Vulkan1"}, [], []),
+ (["--fit", "on", "--device", "Vulkan0,none"], {}, [], ["--fit-target", "16384"]),
+ (["--fit", "on", "--rpc", "host:50052"], {}, [], []),
+ ],
+ ids = [
+ "default",
+ "adds-to-studio-margin",
+ "fitter-off",
+ "per-device-list",
+ "user-argv",
+ "user-env",
+ "mixed-device-list",
+ "mixed-device-env",
+ "one-device-and-cpu",
+ "rpc",
+ ],
+)
+def test_the_igpu_fit_headroom(monkeypatch, argv, env, user, expected):
+ monkeypatch.setattr(llama_cpp_module, "_paged_load_needs_fit_headroom", lambda: True)
+ assert LlamaCppBackend._paged_fit_headroom_flags(argv, env, user) == expected
+ assert LlamaCppBackend._paged_fit_headroom_flags(argv, env, user, gpu_count = 2) == []
+ monkeypatch.setattr(llama_cpp_module, "_paged_load_needs_fit_headroom", lambda: False)
+ assert LlamaCppBackend._paged_fit_headroom_flags(argv, env, user) == []
+
+
+def test_the_igpu_fit_headroom_survives_the_per_device_margin(tmp_path, monkeypatch):
+ """A non-default budget spells --fit-target per device at spawn; that rewrite keeps the headroom."""
+ monkeypatch.setattr(llama_cpp_module, "_paged_load_needs_fit_headroom", lambda: True)
+ monkeypatch.setattr(llama_cpp_module, "_active_vram_fraction", lambda: 0.9)
+ monkeypatch.setattr(LlamaCppBackend, "_GPU_IDS_ARE_PCI_INDICES", True)
+ monkeypatch.setenv("CUDA_DEVICE_ORDER", "PCI_BUS_ID")
+ monkeypatch.delenv("CUDA_VISIBLE_DEVICES", raising = False)
+ backend, gguf = _igpu_backend(tmp_path, monkeypatch, avail_mib = 17 * 1024)
+ LlamaCppBackend.probe_server_capabilities()["supports_fit_target"] = True
+
+ cmd = _launch(backend, gguf)["cmd"]
+
+ assert _modes(cmd, "--load-mode") == ["mmap"], cmd
+ assert (
+ float(_modes(cmd, "--fit-target")[-1]) > llama_cpp_module._IGPU_PAGED_FIT_HEADROOM_MIB
+ ), cmd
+
+
+def test_a_retry_off_the_igpu_withdraws_the_paging_headroom():
+ """The arch-crash respawn on a discrete sibling must not keep the iGPU's 15 GiB margin."""
+ import inspect
+
+ headroom = ["--fit-target", "16384"]
+ cmd = ["-m", "x.gguf", "--fit-target", "1024", "--load-mode", "mmap", *headroom]
+ added = LlamaCppBackend._margins_with_paged_headroom({0: 1024.0, 1: 2048.0}, headroom)
+ assert added == {0: 16384.0, 1: 17408.0}
+ out, margins = LlamaCppBackend._withdraw_paged_headroom(cmd, added, headroom)
+ assert out == ["-m", "x.gguf", "--fit-target", "1024", "--load-mode", "mmap"]
+ assert margins == {0: 1024.0, 1: 2048.0}
+ assert LlamaCppBackend._withdraw_paged_headroom(cmd, None, []) == (cmd, None)
+
+ src = inspect.getsource(LlamaCppBackend.load_model)
+ retry = src[src.index("_retry_auto_unmapped = self._auto_load_mode_unmapped(") :]
+ withdraw = retry.index("if _paged_headroom and not _retry_auto_unmapped:")
+ assert withdraw < retry.index("self._page_an_oversized_unmapped_load(")
+ assert "auto_unmapped = _retry_auto_unmapped" in retry
+
+
+def test_an_igpu_model_without_a_table_is_only_mapped(tmp_path, monkeypatch):
+ backend, gguf = _igpu_backend(tmp_path, monkeypatch, avail_mib = 17 * 1024, table = 0)
+
+ cmd = _launch(backend, gguf)["cmd"]
+
+ assert _modes(cmd, "--load-mode") == ["mmap"], cmd
+ assert "--lazy-mode" not in cmd, cmd
+
+
+def test_an_explicit_none_on_an_igpu_also_streams_the_table(tmp_path, monkeypatch):
+ """The rewrite of a user's none keeps its wording and gains the lazy read on an iGPU."""
+ backend, gguf = _igpu_backend(tmp_path, monkeypatch, avail_mib = 17 * 1024)
+
+ launched = _launch(backend, gguf, extra_args = ["--load-mode", "none"])
+ cmd = launched["cmd"]
+
+ assert not _unmapped_tokens(cmd), cmd
+ assert _modes(cmd, "--load-mode")[-1:] == ["mmap"], cmd
+ assert resolve_effective_load_state(cmd, launched["env"], auto_unmapped = True)[1] is False
+ assert _modes(cmd, "--lazy-mode") == ["on"], cmd
+ assert "was set to load without memory mapping" in (backend.last_load_warning or "")
+
+
def _unmapped_tokens(cmd):
"""The tokens in ``cmd`` that select a mode llama.cpp does not mmap."""
out = []
@@ -5779,7 +6020,7 @@ def test_a_device_list_still_sees_an_auto_filtered_vulkan_igpu(monkeypatch):
assert src.index("_vulkan_igpu_ids = (") < src.index(
"_gpu_mem = self._vulkan_auto_gpu_memory(_gpu_mem)"
)
- assert src.count("integrated_gpu_ids = _vulkan_igpu_ids") == 5
+ assert src.count("integrated_gpu_ids = _vulkan_igpu_ids") == 7
def test_the_oversized_readings_share_their_preflights_ram_snapshot():
@@ -7075,3 +7316,32 @@ def test_the_checkpoint_discount_removes_only_the_checkpoints():
without = backend._slots_that_fit_on_gpu(*args, ctx_checkpoints = 0, **kw)
assert with_ckpts == without
assert without[1] is False and without[2] < 8
+
+
+ "note, expected",
+ [
+ (
+ llama_cpp_module._PAGEABLE_AUTO_OVERRIDE_NOTE,
+ llama_cpp_module._PAGEABLE_AUTO_OVERRIDE_TEXT_ONLY_NOTICE,
+ ),
+ (
+ llama_cpp_module._PAGEABLE_OVERRIDE_NOTE,
+ llama_cpp_module._PAGEABLE_OVERRIDE_TEXT_ONLY_NOTICE,
+ ),
+ ],
+ ids = ["igpu-auto", "user-none"],
+)
+def test_the_text_only_notice_keeps_who_chose_the_unmapped_mode(note, expected):
+ """The iGPU note must not claim the user asked for an unmapped load."""
+ backend = LlamaCppBackend()
+ apu_msg = "This model needs about 70 GB but only about 60 GB of memory is available."
+ backend._last_load_warning = apu_msg + note
+ backend._reprice_after_dropping_pinned_projector(
+ apu_msg = apu_msg,
+ host_msg = None,
+ model_size = 50 * 1024**3,
+ pinned_bytes = 2 * 1024**3,
+ avail_mib = 60 * 1024,
+ )
+ assert backend._last_load_warning == expected
diff --git a/studio/backend/tests/test_llama_server_args.py b/studio/backend/tests/test_llama_server_args.py
index a3bf4865bc4..97d0a917d04 100644
--- a/studio/backend/tests/test_llama_server_args.py
+++ b/studio/backend/tests/test_llama_server_args.py
@@ -1568,3 +1568,73 @@ def test_translate_ngram_mod_does_not_consume_a_following_flag(chain_with_mtp):
out = translate_ngram_mod_args(args, "legacy", chain_with_mtp = chain_with_mtp)
assert out[-2:] == ["--top-k", "/restricted/model.gguf"]
validate_extra_args(out)
+
+
+ "argv, env, auto",
+ [
+ ([], {}, True),
+ (["--load-mode", "auto"], {}, True),
+ (["--load-mode=auto"], {}, True),
+ (["--load-mode", "none", "--load-mode", "auto"], {}, True),
+ ([], {"LLAMA_ARG_LOAD_MODE": "auto"}, True),
+ (["--load-mode", "auto"], {"LLAMA_ARG_LOAD_MODE": "mmap"}, True),
+ (["--load-mode", "mmap"], {}, False),
+ (["--load-mode", "none"], {}, False),
+ ([], {"LLAMA_ARG_LOAD_MODE": "mmap"}, False),
+ (["--mlock"], {}, False),
+ (["--mmap"], {}, False),
+ (["--no-mmap"], {}, False),
+ ([], {"LLAMA_ARG_NO_MMAP": "0"}, False),
+ ([], {"LLAMA_ARG_DIO": "1"}, False),
+ (["--load-mode"], {}, True),
+ ],
+)
+def test_load_mode_left_auto(argv, env, auto):
+ assert _lsa.load_mode_left_auto(argv, env) is auto
+
+
+def test_auto_reserves_ram_only_where_llama_cpp_resolves_it_to_none():
+ assert _lsa.resolve_effective_load_state([], {}) == (False, False, False)
+ assert _lsa.resolve_effective_load_state([], {}, auto_unmapped = True) == (False, True, False)
+ assert _lsa.resolve_effective_memory_state(["--load-mode", "auto"], {}, auto_unmapped = True) == (
+ False,
+ True,
+ )
+ assert _lsa.resolve_effective_load_state(["--load-mode", "mmap"], {}, auto_unmapped = True) == (
+ False,
+ False,
+ False,
+ )
+ assert _lsa.resolve_effective_load_state(["--load-mode", "dio"], {}, auto_unmapped = True) == (
+ False,
+ False,
+ True,
+ )
+
+
+def test_the_pageable_rewrite_maps_an_unselected_mode_on_an_igpu():
+ out, overridden = _lsa.force_pageable_load(["--fit", "on"], {}, auto_unmapped = True)
+ assert out == ["--fit", "on", "--load-mode", "mmap"] and overridden
+ assert _lsa.resolve_effective_memory_state(out, {}, auto_unmapped = True) == (False, False)
+ # An inherited LLAMA_ARG_LOAD_MODE=auto loses to the appended argv.
+ env = {"LLAMA_ARG_LOAD_MODE": "auto"}
+ out, _ = _lsa.force_pageable_load([], env, auto_unmapped = True)
+ assert _lsa.resolve_effective_memory_state(out, env, auto_unmapped = True) == (False, False)
+ for argv, env in (
+ (["--load-mode", "none"], {}),
+ (["--no-mmap"], {}),
+ ([], {"LLAMA_ARG_NO_MMAP": "1"}),
+ ):
+ out, overridden = _lsa.force_pageable_load(argv, env, auto_unmapped = True)
+ assert overridden and out[-2:] == ["--load-mode", "mmap"], (argv, env, out)
+ assert _lsa.resolve_effective_memory_state(out, env, auto_unmapped = True) == (False, False)
+ assert _lsa.force_pageable_load(["--load-mode", "mlock"], {}, auto_unmapped = True)[0] == [
+ "--load-mode",
+ "mmap+mlock",
+ ]
+ assert _lsa.force_pageable_load([], {}) == ([], [])
+ assert _lsa.force_pageable_load(["--load-mode", "mmap"], {}, auto_unmapped = True) == (
+ ["--load-mode", "mmap"],
+ [],
+ )
diff --git a/studio/backend/tests/test_load_mode_caps_unknown.py b/studio/backend/tests/test_load_mode_caps_unknown.py
index a2eb32e3d64..2cfa89fd1a7 100644
--- a/studio/backend/tests/test_load_mode_caps_unknown.py
+++ b/studio/backend/tests/test_load_mode_caps_unknown.py
@@ -27,6 +27,9 @@ from core.inference.llama_cpp import LlamaCppBackend
from core.inference.llama_server_args import apply_load_mode_policy, resolve_launch_load_mode
_NO_SETTINGS = (False, False)
+posix_only = pytest.mark.skipif(
+ sys.platform == "win32", reason = "shell script stands in for the binary"
+)
def _fake_server(tmp_path, body: str) -> str:
@@ -81,12 +84,14 @@ def test_a_failed_help_probe_emits_no_load_flag(tmp_path):
assert _launch_flags(caps) == []
+@posix_only
def test_an_empty_help_probe_emits_no_load_flag(tmp_path):
caps = LlamaCppBackend.probe_server_capabilities(_fake_server(tmp_path, "exit 0"))
assert caps["help_probe_ok"] is True and not caps["flags"]
assert _launch_flags(caps) == []
+@posix_only
def test_a_build_listing_only_the_old_flags_still_gets_them(tmp_path):
help_text = "--no-mmap do not memory-map model\n--mlock lock model in RAM"
caps = LlamaCppBackend.probe_server_capabilities(
@@ -96,6 +101,7 @@ def test_a_build_listing_only_the_old_flags_still_gets_them(tmp_path):
assert _launch_flags(caps) == ["--no-mmap"]
+@posix_only
def test_a_build_listing_load_mode_gets_it(tmp_path):
help_text = "--load-mode MODE how to load the model"
caps = LlamaCppBackend.probe_server_capabilities(
@@ -176,3 +182,45 @@ def test_no_launch_site_reads_the_raw_load_mode_cap():
assert 'server_caps.get("supports_load_mode")' not in source
assert "server_caps['supports_load_mode']" not in source
assert 'server_caps["supports_load_mode"]' not in source
+
+
+# llama-server --help for --load-mode, verbatim: b11643 (ggml-org/llama.cpp#26081 added
+# "auto", which resolves to none on an iGPU) and the earlier #20834 block, default mmap.
+_LOAD_MODE_HELP_WITH_AUTO = """\
+-lm, --load-mode MODE model loading mode (default: auto)
+ - auto: mmap, unless a device does not support it
+ - none: no special loading mode
+ - mmap: memory-map model (if mmap disabled, slower load but may reduce
+ pageouts if not using mlock)
+ - mlock: force system to keep model in RAM rather than swapping or
+ compressing
+ - mmap+mlock: mmap + force system to keep model in RAM rather than
+ swapping or compressing
+ - dio: use DirectIO if available
+ (env: LLAMA_ARG_LOAD_MODE)"""
+_LOAD_MODE_HELP_WITHOUT_AUTO = """\
+-lm, --load-mode MODE model loading mode (default: mmap)
+ - none: no special loading mode
+ - mmap: memory-map model (if mmap disabled, slower load but may reduce
+ pageouts if not using mlock)
+ - mlock: mmap + force system to keep model in RAM rather than swapping
+ or compressing
+ - dio: use DirectIO if available
+ (env: LLAMA_ARG_LOAD_MODE)"""
+
+
+@posix_only
+ "help_text, auto",
+ [
+ (_LOAD_MODE_HELP_WITH_AUTO, True),
+ (_LOAD_MODE_HELP_WITHOUT_AUTO, False),
+ ("--mmap memory-map model (default: auto)", False),
+ ],
+ ids = ["auto-build", "load-mode-without-auto", "legacy-flags-only"],
+)
+def test_the_probe_reads_whether_the_build_lists_load_mode_auto(tmp_path, help_text, auto):
+ caps = LlamaCppBackend.probe_server_capabilities(
+ _fake_server(tmp_path, f"cat <<'X'\n{help_text}\nX\nexit 0")
+ )
+ assert caps["supports_load_mode_auto"] is auto