Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

Studio: size a GPU cache for spilled MoE experts with --moe-cache-mib auto (#12951)

Daniel Hanb15cda912a6c
diff --git a/studio/backend/core/inference/llama_cpp.py b/studio/backend/core/inference/llama_cpp.py
index 8ee38bf42e4..34ff04eb483 100644
--- a/studio/backend/core/inference/llama_cpp.py
+++ b/studio/backend/core/inference/llama_cpp.py
@@ -99,6 +99,7 @@ from core.inference.llama_server_args import (
_TENSOR_SPLIT_FLAGS,
_effective_tensor_parallel,
_flag_name,
+ _last_flag_value,
_tensor_parallel_matches_loaded,
LLAMA_CTX_CHECKPOINTS_DEFAULT,
apply_load_mode_policy,
@@ -7092,6 +7093,132 @@ def _expert_spill_places_tensors_on_cpu(
return _args_place_tensors_on_cpu(kept) or _env_places_tensors_on_cpu(source_env)
+# GPU cache of host-resident experts (ggml-org/llama.cpp#29887); "auto" is sized by the
+# fork's fitter (unslothai/llama.cpp#251). One run, so a retry can strip it.
+_MOE_CACHE_AUTO_TOKENS = ("--moe-cache-mib", "auto")
+# User placement: the fitter aborts on most of it, and the cache refuses >1 device (--rpc adds one).
+_MOE_CACHE_USER_OWNED_FLAGS = (
+ frozenset({"--moe-cache-mib", "-ot", "--override-tensor", "--rpc"})
+ | _CPU_FFN_COUNT_FLAGS
+ | _DEVICE_FLAGS
+ | _GPU_LAYER_FLAGS
+ | _MOE_OFFLOAD_FLAGS
+ | _SPLIT_MODE_FLAGS
+ | _TENSOR_SPLIT_FLAGS
+)
+_MOE_CACHE_USER_OWNED_ENV = (
+ "LLAMA_ARG_MOE_CACHE_MIB",
+ "LLAMA_ARG_OVERRIDE_TENSOR",
+ "LLAMA_ARG_CPU_MOE",
+ "LLAMA_ARG_N_CPU_MOE",
+ "LLAMA_ARG_N_CPU_FFN",
+ "LLAMA_ARG_DEVICE",
+ "LLAMA_ARG_MAIN_GPU",
+ "LLAMA_ARG_SPLIT_MODE",
+ "LLAMA_ARG_TENSOR_SPLIT",
+ "LLAMA_ARG_RPC",
+)
+# Exact startup errors: healthy launches also log "moe cache auto:" and "MoE cache size =".
+_MOE_CACHE_ERROR_RE = re.compile(
+ r"MoE cache (?:requires a GPU backend|does not support tensor parallelism"
+ r"|requires a MoE model|is too small to hold the experts of one token)"
+ r"|failed to (?:create the MoE cache context|allocate the MoE cache buffers)"
+ r"|the MoE cache is too small for the experts selected in layer",
+ re.IGNORECASE,
+)
+
+# TENSOR_READ_LAZY tensors (src/models/gemma4.cpp, qwen4exp.cpp): under --lazy-mode auto one
+# over 4 GiB stays mmap'd under every load mode (lazy_read::add), needing no pinned RAM or VRAM.
+_LAZY_READ_TENSORS: dict[str, tuple[str, ...]] = {
+ "gemma4": ("per_layer_token_embd.weight",),
+ "qwen4exp": ("per_layer_token_embd.weight",),
+}
+_LAZY_READ_AUTO_MIN_BYTES = 4 * 1024**3
+# --tensor-read-lazy predates ggml-org/llama.cpp#27969; a build reads only its own spelling.
+_LAZY_MODE_SPELLINGS: dict[str, tuple[frozenset[str], str]] = {
+ "--lazy-mode": (frozenset({"-lzm", "--lazy-mode"}), "LLAMA_ARG_LAZY_MODE"),
+ "--tensor-read-lazy": (frozenset({"--tensor-read-lazy"}), "LLAMA_ARG_TENSOR_READ_LAZY"),
+}
+
+
+# llama-server's --cache-ram default (MiB): the host-RAM prompt cache it grows into.
+_LLAMA_CACHE_RAM_DEFAULT_MIB = 8192
+
+
+def _prompt_cache_host_bytes(
+ cache_ram: Optional[int],
+ extra_args: Optional[Iterable[str]],
+ env: Optional[Mapping[str, str]] = None,
+ *,
+ supports_cache_ram: bool,
+) -> Optional[int]:
+ """The host RAM llama-server's prompt cache may grow into, or None when it is
+ unbounded (``-1``) or unreadable. Env first, then the launch's own value, then a
+ last-wins pass-through flag; zero on a build without the cache."""
+ if not supports_cache_ram:
+ return 0
+ source_env = os.environ if env is None else env
+ value: object = _LLAMA_CACHE_RAM_DEFAULT_MIB
+ if str(source_env.get("LLAMA_ARG_CACHE_RAM") or "").strip():
+ value = source_env.get("LLAMA_ARG_CACHE_RAM")
+ if cache_ram is not None:
+ value = cache_ram
+ try:
+ override = _last_flag_value(extra_args, _CACHE_RAM_FLAGS)
+ if override is not None:
+ value = override
+ mib = int(str(value).strip())
+ except (TypeError, ValueError):
+ return None
+ return None if mib < 0 else mib * 1024 * 1024
+
+
+def _argv_loads_pinned(argv: Iterable[str], env: Optional[Mapping[str, str]] = None) -> bool:
+ """Whether ``argv`` (with the env twins) loads the weights into a pinned host
+ copy, i.e. ``--load-mode none``: no mmap, no page-lock, no DirectIO."""
+ mlock, reserves_ram, direct_io = resolve_effective_load_state(list(argv), env)
+ return bool(reserves_ram and not mlock and not direct_io)
+
+
+def _effective_lazy_mode(
+ extra_args: Optional[Iterable[str]],
+ env: Optional[Mapping[str, str]] = None,
+ *,
+ flag: str = "--lazy-mode",
+) -> str:
+ """llama.cpp's lazy mode for this launch: env first, then last-wins argv, both in
+ the build's own spelling ``flag``.
+
+ A value llama-server would refuse reads as "off", which discounts nothing."""
+ source_env = os.environ if env is None else env
+ flags, env_name = _LAZY_MODE_SPELLINGS.get(flag, _LAZY_MODE_SPELLINGS["--lazy-mode"])
+ mode = "auto"
+ value = str(source_env.get(env_name) or "").strip().lower()
+ if value:
+ mode = value
+ try:
+ value = _last_flag_value(extra_args, flags)
+ except ValueError:
+ return "off"
+ if value:
+ mode = value.strip().lower()
+ return mode if mode in ("auto", "on") else "off"
+
+
+def _lazy_read_bytes(
+ architecture: Optional[str], tensor_bytes: Mapping[str, int], mode: str
+) -> int:
+ """Bytes of ``tensor_bytes`` llama.cpp reads lazily under ``mode``."""
+ if mode not in ("auto", "on"):
+ return 0
+ names = _LAZY_READ_TENSORS.get(architecture or "", ())
+ return sum(
+ int(size)
+ for name, size in tensor_bytes.items()
+ if name in names and (mode == "on" or int(size) > _LAZY_READ_AUTO_MIN_BYTES)
+ )
+
+
def _build_ngram_mod_flags(
caps: Optional[dict],
n_match: int = 24,
@@ -7971,6 +8098,7 @@ class LlamaCppBackend:
# The expert-spill micro-batch raise: the pair before it, and the argv tokens.
self._moe_spill_batch_restore: Optional[tuple[Optional[int], Optional[int]]] = None
self._moe_spill_batch_tokens: Optional[tuple[list[str], list[str]]] = None
+ self._moe_cache_flags: List[str] = []
# The tuning group the last load asked for; none = defaults, or left to
# extras / env. What was REQUESTED, not what ran: Model Memory can replace
# the load mode, and Windows full-offload tuning owns the two cache knobs.
@@ -9819,6 +9947,10 @@ class LlamaCppBackend:
"supports_trace_verbosity": False,
"supports_video_fps": False,
"supports_load_mode": False,
+ "supports_moe_cache": False,
+ "supports_moe_cache_auto": False,
+ "supports_lazy_mode": False,
+ "lazy_mode_flag": None,
"spec_draft_ngl_flag": None,
"spec_draft_cache_k_flag": None,
"spec_draft_cache_v_flag": None,
@@ -9873,6 +10005,10 @@ class LlamaCppBackend:
supports_no_mmproj_offload = False
supports_video_fps = False
supports_load_mode = False
+ supports_moe_cache = False
+ supports_moe_cache_auto = False
+ supports_lazy_mode = False
+ lazy_mode_flag = None
spec_draft_ngl_flag = None
spec_draft_cache_k_flag = None
spec_draft_cache_v_flag = None
@@ -10118,6 +10254,20 @@ class LlamaCppBackend:
# --load-mode supersedes --mlock / --no-mmap, which are deprecated.
# Pre-initialised above: a failed probe must fall back, not raise.
supports_load_mode = _is_real("--load-mode")
+ # "auto" is a fork addition; upstream builds exit on it, so fail closed.
+ supports_moe_cache = bool(probe_ok and _is_real("--moe-cache-mib"))
+ supports_moe_cache_auto = bool(
+ supports_moe_cache
+ and re.search(
+ r"(?<![a-z0-9_-])auto(?![a-z0-9_-])",
+ (blocks.get("--moe-cache-mib") or "").lower(),
+ )
+ )
+ if probe_ok:
+ lazy_mode_flag = next(
+ (f for f in ("--lazy-mode", "--tensor-read-lazy") if _is_real(f)), None
+ )
+ supports_lazy_mode = lazy_mode_flag is not None
# Record WHICH alias this build has: --spec-draft-ngl only landed in
# b8955, and a build exposing only --gpu-layers-draft would refuse to
# start on the newer name. Long forms only, since the block parser above
@@ -10216,6 +10366,10 @@ class LlamaCppBackend:
"supports_trace_verbosity": supports_trace_verbosity,
"supports_video_fps": supports_video_fps,
"supports_load_mode": supports_load_mode,
+ "supports_moe_cache": supports_moe_cache,
+ "supports_moe_cache_auto": supports_moe_cache_auto,
+ "supports_lazy_mode": supports_lazy_mode,
+ "lazy_mode_flag": lazy_mode_flag,
"spec_draft_ngl_flag": spec_draft_ngl_flag,
"spec_draft_cache_k_flag": spec_draft_cache_k_flag,
"spec_draft_cache_v_flag": spec_draft_cache_v_flag,
@@ -10398,6 +10552,163 @@ class LlamaCppBackend:
return total
+ @staticmethod
+ def _gguf_scan_tensor_bytes(path: str) -> "tuple[Optional[str], dict[str, int], int]":
+ """(architecture, lazy-read candidate bytes by name, routed expert bytes) from one
+ GGUF header. Streamed: ``GGUFReader`` materialises the vocabulary under the load
+ lock. A quant type gguf predates is sized from the next tensor's offset."""
+ from gguf.constants import GGML_QUANT_SIZES
+ from core.inference.offload_layout import _BLOCK_RE, _MOE_EXPERT_RE
+
+ wanted = {name.encode() for names in _LAZY_READ_TENSORS.values() for name in names}
+ architecture: Optional[str] = None
+ alignment = 32 # GGUF's default when general.alignment is absent
+ named: dict[str, int] = {}
+ experts = 0
+ offsets: list[int] = []
+ unsized: list[tuple[str, int]] = []
+ with open(path, "rb") as f:
+ if struct.unpack("<I", f.read(4))[0] != 0x46554747: # b"GGUF"
+ return None, {}, 0
+ # Version 1 used 32-bit counts and lengths, which this layout misreads.
+ if struct.unpack("<I", f.read(4))[0] < 2:
+ return None, {}, 0
+ n_tensors, n_kv = struct.unpack("<QQ", f.read(16))
+ for _ in range(n_kv):
+ key = f.read(struct.unpack("<Q", f.read(8))[0])
+ value_type = struct.unpack("<I", f.read(4))[0]
+ if key == b"general.architecture" and value_type == 8:
+ raw = f.read(struct.unpack("<Q", f.read(8))[0])
+ architecture = raw.decode("utf-8", "replace")
+ continue
+ if key == b"general.alignment" and value_type == 4:
+ alignment = max(1, struct.unpack("<I", f.read(4))[0])
+ continue
+ LlamaCppBackend._gguf_skip_value(f, value_type)
+ for _ in range(n_tensors):
+ raw_name = f.read(struct.unpack("<Q", f.read(8))[0])
+ n_dims = struct.unpack("<I", f.read(4))[0]
+ dims = struct.unpack(f"<{n_dims}Q", f.read(8 * n_dims))
+ ggml_type = struct.unpack("<I", f.read(4))[0]
+ offset = struct.unpack("<Q", f.read(8))[0]
+ offsets.append(offset)
+ block_match = _BLOCK_RE.match(raw_name.decode("utf-8", "replace"))
+ is_expert = bool(block_match and _MOE_EXPERT_RE.match(block_match.group(2)))
+ if not is_expert and raw_name not in wanted:
+ continue
+ name = raw_name.decode("utf-8", "replace")
+ block = GGML_QUANT_SIZES.get(ggml_type)
+ if block is None:
+ unsized.append((name, offset))
+ continue
+ size = math.prod(dims) // block[0] * block[1]
+ if is_expert:
+ experts += size
+ else:
+ named[name] = size
+ if unsized:
+ data_start = -(-f.tell() // alignment) * alignment
+ data_end = max(0, os.path.getsize(path) - data_start)
+ ordered = sorted(offsets)
+ for name, offset in unsized:
+ later = ordered[bisect.bisect_right(ordered, offset) :]
+ size = max(0, (later[0] if later else data_end) - offset)
+ block_match = _BLOCK_RE.match(name)
+ if block_match and _MOE_EXPERT_RE.match(block_match.group(2)):
+ experts += size
+ else:
+ named[name] = size
+ return architecture, named, experts
+
+ def _gguf_tensor_scan(
+ self, model_path: Optional[str]
+ ) -> "Optional[tuple[Optional[str], dict[str, int], int]]":
+ """``_gguf_scan_tensor_bytes`` over every shard, cached per file identity; None
+ when a shard is missing or unreadable."""
+ from core.inference.offload_layout import split_shard_paths
+
+ if not model_path:
+ return None
+ paths = split_shard_paths(model_path) or [model_path]
+ try:
+ key = tuple((p, st.st_size, st.st_mtime_ns) for p in paths for st in (os.stat(p),))
+ except OSError:
+ return None
+ cached = getattr(self, "_gguf_tensor_scan_cache", None)
+ if cached is not None and cached[0] == key:
+ return cached[1]
+ try:
+ architecture: Optional[str] = None
+ named: dict[str, int] = {}
+ experts = 0
+ for p in paths:
+ arch, part, part_experts = self._gguf_scan_tensor_bytes(p)
+ # Shard 1 carries the metadata.
+ architecture = architecture or arch
+ for name, size in part.items():
+ named[name] = named.get(name, 0) + size
+ experts += part_experts
+ result = (architecture, named, experts)
+ except Exception as e: # unreadable or truncated: discount nothing
+ logger.debug("GGUF tensor scan failed for %s (%s)", model_path, e)
+ result = None
+ self._gguf_tensor_scan_cache = (key, result)
+ return result
+
+ def _lazy_read_host_bytes(
+ self,
+ model_path: Optional[str],
+ *,
+ caps: Optional[Mapping[str, object]],
+ extra_args: Optional[Iterable[str]] = None,
+ env: Optional[Mapping[str, str]] = None,
+ auto_resolves_off: bool = False,
+ ) -> int:
+ """Bytes llama.cpp leaves mmap'd from disk under every load mode (``_LAZY_READ_TENSORS``);
+ zero without lazy-read support, under ``off``, or on an unreadable header.
+ ``auto_resolves_off``: llama.cpp turns ``auto`` into ``off`` for these devices."""
+ if not model_path or not (caps or {}).get("supports_lazy_mode"):
+ return 0
+ mode = _effective_lazy_mode(
+ extra_args, env, flag = str((caps or {}).get("lazy_mode_flag") or "--lazy-mode")
+ )
+ if mode == "off" or (mode == "auto" and auto_resolves_off):
+ return 0
+ scan = self._gguf_tensor_scan(model_path)
+ if scan is None:
+ return 0
+ architecture, named, _experts = scan
+ return _lazy_read_bytes(architecture, named, mode)
+
+ def _lazy_auto_resolves_off(
+ self,
+ *,
+ gpu_indices: Optional[Iterable[int]],
+ detected_gpus: Optional[Iterable[tuple]],
+ shared_gpu_ids: Optional[Iterable[int]],
+ is_vulkan_backend: bool,
+ probe_integrated_cuda: bool = True,
+ ) -> bool:
+ """Whether llama.cpp resolves ``--lazy-mode auto`` to ``off`` for this child: a model
+ device without mmap support (ggml-org/llama.cpp#28160), i.e. every selected device
+ integrated, since the default list holds iGPUs only when no discrete GPU is seen.
+ ``probe_integrated_cuda`` False uses only a cached SoC answer (no new context)."""
+ rows = [(row[0], row[1]) for row in (detected_gpus or ())]
+ if self._offload_target_shares_system_memory(
+ is_vulkan_backend = is_vulkan_backend,
+ shared_gpu_ids = shared_gpu_ids,
+ detected_gpus = rows,
+ gpu_indices = gpu_indices,
+ ):
+ return True
+ if is_vulkan_backend:
+ return False
+ if not probe_integrated_cuda and not self._integrated_cuda_probe_is_free():
+ return False
+ return self._integrated_cuda_selection_is_all_shared(
+ list(gpu_indices) if gpu_indices else None
+ )
+
@staticmethod
def _installed_ggml_backends(binary: Optional[str] = None) -> frozenset[str]:
"""Backend libraries shipped beside llama-server."""
@@ -14851,6 +15162,27 @@ class LlamaCppBackend:
if not model_bytes or (not gpus and not child_has_no_gpu):
return None
shared = set(shared_gpu_ids or ())
+ # Same lazy-read discount as the fit, or this rewrite undoes its pinned verdict.
+ try:
+ model_bytes = max(
+ 1,
+ model_bytes
+ - self._lazy_read_host_bytes(
+ model_path,
+ caps = self.probe_server_capabilities(argv[0]),
+ extra_args = argv,
+ env = _env,
+ auto_resolves_off = self._lazy_auto_resolves_off(
+ gpu_indices = [row[0] for row in gpus] if gpus else None,
+ detected_gpus = gpus,
+ shared_gpu_ids = shared,
+ is_vulkan_backend = bool(shared) or self._is_vulkan_backend(argv[0]),
+ probe_integrated_cuda = False,
+ ),
+ ),
+ )
+ except Exception as e: # noqa: BLE001 -- unverified discount: price the whole file
+ logger.debug("Lazy-read discount skipped in the host preflight: %s", e)
free_vram_mib = sum(max(0, row[1]) for row in gpus if row[0] not in shared)
heap_free_mib, heap_bytes = self._shared_heap_budget(gpus, shared, model_bytes, argv, _env)
offload_bytes = model_bytes - free_vram_mib * 1024 * 1024
@@ -15594,6 +15926,8 @@ class LlamaCppBackend:
avail_mib: Optional[int] = None,
extra_args: Optional[Iterable[str]] = None,
env: Optional[Mapping[str, str]] = None,
+ lazy_read_bytes: int = 0,
+ announce: bool = True,
) -> Optional[str]:
"""``"none"`` when the fit proves the load needs no demand paging, else None.
@@ -15661,6 +15995,9 @@ class LlamaCppBackend:
enabled fitter is free to lower it (common/fit.cpp aborts only on a count the
user really set), and a pass-through ``--fit-target`` raises the margin above
anything ``_ctx_integrity_flags`` emitted.
+
+ ``lazy_read_bytes`` (``_lazy_read_host_bytes``) leaves the footprint: it stays
+ mmap'd and pages through the headroom. ``announce`` False silences a what-if call.
"""
from utils.hardware import is_apple_silicon
@@ -15706,6 +16043,9 @@ class LlamaCppBackend:
+ max(0, pipeline_overhead_bytes)
+ max(0, soft_overhead)
)
+ _lazy_discount = max(0, int(lazy_read_bytes or 0))
+ if _lazy_discount:
+ footprint = max(1, footprint - _lazy_discount)
# Unified memory is ONE pool: an APU's free VRAM IS the host RAM the spill
# comes out of, so adding both fits a model twice into memory that holds it
# once. shared_gpu_ids already names Vulkan iGPUs; the ROCm APUs it cannot see
@@ -15739,11 +16079,19 @@ class LlamaCppBackend:
vram_margin_mib = fit_margin_mib,
avail_mib = avail_mib,
):
- logger.info(
- "Load mode: the whole load (%.1f GB) fits without paging, using --load-mode %s.",
- footprint / (1024**3),
- self._FIT_LOAD_MODE,
- )
+ if announce:
+ logger.info(
+ "Load mode: the whole load (%.1f GB) fits without paging, using "
+ "--load-mode %s%s.",
+ footprint / (1024**3),
+ self._FIT_LOAD_MODE,
+ (
+ f" ({_lazy_discount / (1024**3):.1f} GB of lazily read tables stay "
+ "on disk)"
+ if _lazy_discount
+ else ""
+ ),
+ )
return self._FIT_LOAD_MODE
return None
@@ -18198,6 +18546,9 @@ class LlamaCppBackend:
else:
for _ in range(alen):
LlamaCppBackend._gguf_skip_value(f, atype)
+ else:
+ # Its size is unknown, so every later read would be misaligned.
+ raise ValueError(f"unknown GGUF value type {vtype}")
@staticmethod
def _gguf_read_array_value(f, atype: int, alen: int) -> Optional[list]:
@@ -18781,10 +19132,10 @@ class LlamaCppBackend:
self._gguf_skip_value(f, vtype)
else:
self._gguf_skip_value(f, vtype)
- except (struct.error, UnicodeDecodeError):
+ except (struct.error, UnicodeDecodeError, ValueError):
# Truncated input (e.g. HTTP byte-range header
- # fetch); break so the resolver fallback runs on
- # what we have.
+ # fetch), or a value type with no known size; break
+ # so the resolver fallback runs on what we have.
break
else:
kv_complete = True
@@ -23372,6 +23723,61 @@ class LlamaCppBackend:
return [*argv[:i], *before, *argv[i + len(raised) :]]
return argv
+ def _drop_moe_cache(
+ self,
+ argv: "list[str]",
+ why: str = "",
+ ) -> "list[str]":
+ """``argv`` without the ``--moe-cache-mib auto`` this launch added; a user's is kept."""
+ tokens = list(getattr(self, "_moe_cache_flags", None) or ())
+ stripped = _without_subsequence(list(argv), tokens)
+ if tokens and stripped != list(argv) and why:
+ logger.info("MoE cache: dropping --moe-cache-mib auto for the %s.", why)
+ return stripped
+
+ def _moe_cache_auto_eligible(
+ self,
+ *,
+ caps: Optional[Mapping[str, object]],
+ gpu_memory_mode: str,
+ use_fit: bool,
+ experts_on_host: bool,
+ discrete_gpu: bool,
+ gpu_indices: Optional[Iterable[int]],
+ detected_gpus: Optional[Iterable[tuple[int, int]]],
+ is_vulkan_backend: bool,
+ tensor_parallel: bool,
+ extra_args: Optional[Iterable[str]],
+ env: Optional[Mapping[str, str]] = None,
+ ) -> bool:
+ """Whether this launch may get ``--moe-cache-mib auto`` (the caller still checks RAM
+ and the final load mode): a build with ``auto``, ``--fit on``, spilling experts, one
+ discrete CUDA / ROCm GPU (the cache is single-device), and no user placement."""
+ if not (caps or {}).get("supports_moe_cache_auto"):
+ return False
+ if gpu_memory_mode == "manual" or not use_fit or not experts_on_host:
+ return False
+ if not discrete_gpu or is_vulkan_backend or tensor_parallel or self.n_moe_layers <= 0:
+ return False
+ devices = (
+ {int(i) for i in gpu_indices}
+ if gpu_indices is not None
+ else {int(idx) for idx, _free in (detected_gpus or ())}
+ )
+ if len(devices) != 1:
+ return False
+ source_env = os.environ if env is None else env
+ if _extra_args_set_any_flag(extra_args, _MOE_CACHE_USER_OWNED_FLAGS):
+ return False
+ if any(str(source_env.get(name) or "").strip() for name in _MOE_CACHE_USER_OWNED_ENV):
+ return False
+ if _env_fixes_gpu_layers(source_env):
+ return False
+ try:
+ return fit_is_effectively_on(list(extra_args or ()), source_env)
+ except ValueError: # a valueless --fit: llama-server refuses the launch anyway
+ return False
+
def _discrete_gpu_for_expert_spill(
self,
gpu_indices: Optional[Iterable[int]],
@@ -23442,8 +23848,9 @@ class LlamaCppBackend:
replay = self._drop_managed_dio(
replay, "the CPU fallback runs entirely from host RAM", clear_record = False
)
- # Nothing to stream without a GPU: put back the pre-raise batch pair.
+ # Nothing to stream without a GPU; the expert cache needs one too.
replay = self._undo_moe_spill_batch(replay)
+ replay = self._drop_moe_cache(replay, "CPU fallback")
# A user's own "--load-mode none" / "--no-mmap" survives that strip, by design,
# and on this rung it is no longer the mode they were priced for: the replay
# appends "--gpu-layers 0 --fit off --device none", so nothing credits VRAM and
@@ -25059,6 +25466,8 @@ class LlamaCppBackend:
self._pending_plan_mib = {}
# Set by the expert-spill raise after the fit.
self._moe_spill_batch_restore: Optional[tuple[Optional[int], Optional[int]]] = None
+ _moe_cache_candidate = False
+ _moe_cache_ram_ok = False
_shared_gpus = frozenset()
# Sized inputs for the tensor-spill planner, None when the fit never
# priced them. Bound before the try like _placement_verdict_partial:
@@ -27387,17 +27796,18 @@ class LlamaCppBackend:
and not _device_selection_is_cpu(extra_args, os.environ)
and (use_fit or _expert_spill_places_tensors_on_cpu(extra_args, os.environ))
)
+ _moe_spill_discrete_gpu = bool(
+ _moe_experts_on_host
+ and self._discrete_gpu_for_expert_spill(
+ gpu_indices, _detected_gpus, _shared_gpu_ids
+ )
+ )
_spill_n_batch, _spill_n_ubatch = _moe_spill_batch_ubatch(
n_batch,
n_ubatch,
n_moe_layers = self.n_moe_layers,
experts_on_host = _moe_experts_on_host,
- discrete_gpu = (
- _moe_experts_on_host
- and self._discrete_gpu_for_expert_spill(
- gpu_indices, _detected_gpus, _shared_gpu_ids
- )
- ),
+ discrete_gpu = _moe_spill_discrete_gpu,
user_named_batch = _user_named_batch,
)
if _spill_n_ubatch != n_ubatch:
@@ -27422,6 +27832,20 @@ class LlamaCppBackend:
per_device_tensor = True,
) or (self._TENSOR_PARALLEL_BUFFER_RESERVE_MIB * 1024 * 1024)
+ _moe_cache_candidate = self._moe_cache_auto_eligible(
+ caps = server_caps,
+ gpu_memory_mode = gpu_memory_mode,
+ use_fit = use_fit,
+ experts_on_host = _moe_experts_on_host,
+ discrete_gpu = _moe_spill_discrete_gpu,
+ gpu_indices = gpu_indices,
+ detected_gpus = _detected_gpus,
+ is_vulkan_backend = is_vulkan_backend,
+ tensor_parallel = tensor_parallel,
+ extra_args = extra_args,
+ env = os.environ,
+ )
+
# Pass the final slot and micro-batch values instead of the defaults
# captured before slot reduction.
_mtp_reserve_bytes = (
@@ -27726,7 +28150,20 @@ class LlamaCppBackend:
_kv_bytes(effective_ctx, _effective_ctx_checkpoints)
- _kv_bytes(effective_ctx, 0),
)
- _fit_load_mode = self._fit_derived_load_mode(
+ # A lazily read table stays mmap'd under every load mode: not pinned RAM.
+ _fit_lazy_bytes = self._lazy_read_host_bytes(
+ model_path,
+ caps = server_caps,
+ extra_args = _fit_extras,
+ env = _fit_env,
+ auto_resolves_off = self._lazy_auto_resolves_off(
+ gpu_indices = gpu_indices,
+ detected_gpus = _detected_gpus,
+ shared_gpu_ids = _shared_gpu_ids,
+ is_vulkan_backend = is_vulkan_backend,
+ ),
+ )
+ _fit_load_mode_kwargs = dict(
model_size = _fit_model_size,
mmproj_pinned_bytes = _mmproj_pinned_bytes
+ (_fit_env_mmproj_bytes if _fit_env_mmproj_on_host else 0),
@@ -27783,7 +28220,44 @@ class LlamaCppBackend:
# launch's own -ngl.
extra_args = _fit_extras,
env = _fit_env,
+ lazy_read_bytes = _fit_lazy_bytes,
)
+ _fit_load_mode = self._fit_derived_load_mode(**_fit_load_mode_kwargs)
+ if _moe_cache_candidate:
+ # The cache frees no host RAM: every routed expert plus the prompt
+ # cache must fit pinned (moved out of model_size, not counted twice).
+ _moe_expert_bytes = (self._gguf_tensor_scan(model_path) or (None, {}, 0))[2]
+ _moe_prompt_cache_bytes = _prompt_cache_host_bytes(
+ cache_ram,
+ _fit_extras,
+ _fit_env,
+ supports_cache_ram = bool(server_caps.get("supports_cache_ram")),
+ )
+ # An unknown size (unsized inherited projector) admits nothing.
+ _moe_cache_ram_ok = bool(
+ _fit_load_mode_kwargs["model_size"]
+ and _moe_expert_bytes > 0
+ and _moe_prompt_cache_bytes is not None
+ and self._fit_derived_load_mode(
+ **{
+ **_fit_load_mode_kwargs,
+ "model_size": max(
+ 1, _fit_load_mode_kwargs["model_size"] - _moe_expert_bytes
+ ),
+ "host_only_bytes": _fit_load_mode_kwargs["host_only_bytes"]
+ + _moe_expert_bytes
+ + _moe_prompt_cache_bytes,
+ "announce": False,
+ }
+ )
+ == self._FIT_LOAD_MODE
+ )
+ if not _moe_cache_ram_ok:
+ logger.info(
+ "MoE cache: system RAM cannot hold the %.1f GB of routed "
+ "experts pinned; launching without --moe-cache-mib.",
+ _moe_expert_bytes / (1024**3),
+ )
except Exception as e:
logger.warning(f"GPU selection failed ({e}), using --fit on")
gpu_indices, use_fit = None, True
@@ -27793,6 +28267,7 @@ class LlamaCppBackend:
_placement_verdict_partial = False
# Same reasoning: an unpriced load keeps llama.cpp's default.
_fit_load_mode = None
+ _moe_cache_candidate = _moe_cache_ram_ok = False
# And a half-built snapshot is not a price either: the throw can
# land after the dict exists but before the later terms are added
# to it, which would plan against a footprint missing them.
@@ -27901,6 +28376,8 @@ class LlamaCppBackend:
# abstains, while the RAM this launch actually loads into cannot hold
# them. "none" and "mlock" do not mmap, so that is an OOM kill.
_apu_ram_oversized = False
+ # model_size less a lazily read table; the text-only reprice charges the same.
+ _apu_resident_model_size = model_size
# The two shortfall notices this load recorded, kept verbatim (and
# un-amended) so the text-only fallback further down can re-price them
# once it drops the CPU-pinned projector whose bytes they charged. Both
@@ -27944,11 +28421,28 @@ class LlamaCppBackend:
_apu_ram_part = (
"APU" if self._amd_apu_wants_unified_memory(gpu_indices) else "SoC"
)
+ _apu_resident_model_size = max(
+ 1,
+ model_size
+ - self._lazy_read_host_bytes(
+ model_path,
+ caps = server_caps,
+ extra_args = extra_args,
+ env = os.environ,
+ auto_resolves_off = self._lazy_auto_resolves_off(
+ gpu_indices = gpu_indices,
+ detected_gpus = _detected_gpus,
+ shared_gpu_ids = _shared_gpu_ids,
+ is_vulkan_backend = False,
+ probe_integrated_cuda = False,
+ ),
+ ),
+ )
_ram_msg = self._apu_ram_shortfall_message(
# A pinned projector left model_size but not system RAM, and
# this guard exists to stop an oversize load being OOM-killed
# mid-read, so it has to weigh the projector either way.
- model_size + _mmproj_pinned_bytes,
+ _apu_resident_model_size + _mmproj_pinned_bytes,
_apu_avail_mib,
part = _apu_ram_part,
)
@@ -28216,6 +28710,8 @@ class LlamaCppBackend:
# Set when a positional --tensor-split is emitted, so the env block
# can pin CUDA to PCI order even without a GPU subset (see below).
manual_tensor_split_emitted = False
+ # The MoE cache's "auto" sizing runs only under llama.cpp's fitter.
+ _placed_by_fitter = False
if gpu_memory_mode == "manual" and gpu_layers >= 0:
# Pin the user's layer count and disable auto-fit. --fit off
# also means _ctx_integrity_flags must not add --fit-ctx.
@@ -28307,6 +28803,7 @@ class LlamaCppBackend:
)
else:
cmd.extend(["--fit", "on"])
+ _placed_by_fitter = True
elif gpu_indices is not None:
# Fits on selected GPU(s) -- force all layers on GPU. --fit off is
# required: without it llama.cpp's default --fit on second-guesses
@@ -29483,6 +29980,20 @@ class LlamaCppBackend:
cmd.extend(_pv_split_mode_pin)
if _pv_device_pin:
cmd.extend(_pv_device_pin)
+ # Pinned loads only: under mmap the cache measured slower than none.
+ self._moe_cache_flags = []
+ if (
+ _moe_cache_candidate
+ and _moe_cache_ram_ok
+ and _placed_by_fitter
+ and _argv_loads_pinned(cmd, _fit_load_mode_env_view)
+ ):
+ self._moe_cache_flags = list(_MOE_CACHE_AUTO_TOKENS)
+ cmd.extend(self._moe_cache_flags)
+ logger.info(
+ "MoE experts spill to pinned system RAM on one GPU; adding "
+ "--moe-cache-mib auto so llama.cpp caches hot experts in VRAM."
+ )
# A build with no --flash-attn cannot run a quantized V cache
# either: llama.cpp aborts init with "V cache quantization
@@ -30466,10 +30977,10 @@ class LlamaCppBackend:
def _spawn_and_wait(run_cmd, *, label = ""):
"""Start llama-server with run_cmd and wait for health.
- Up to three launches: the first, one ROCm env correction,
- and one --fit recovery. Separate flags, so a library mix
- does not spend the fit slot and a VRAM crash after the
- correction can still fit-retry.
+ Up to four launches: the first, one ROCm env correction, one
+ retry without the MoE expert cache, and one --fit recovery.
+ Separate flags, so a library mix does not spend the fit slot
+ and a VRAM crash after the correction can still fit-retry.
"""
# _mem_host_resident too: the --fit on retry re-arms the
# page-lock and writes it back, which without this makes the
@@ -30483,9 +30994,15 @@ class LlamaCppBackend:
# and projector placement, and each retry changes one of them.
if label:
run_cmd = self._drop_tensor_spill(run_cmd, label.lstrip("-") or "retry")
+ # The cache only rides the first pinned launch: every retry re-places.
+ if label or not _argv_loads_pinned(run_cmd, env):
+ run_cmd = self._drop_moe_cache(
+ run_cmd, f"{label.lstrip('-') or 'unpinned'} launch"
+ )
_fit_retry_allowed = self._fit_off_retry_eligible(run_cmd, use_fit)
_did_fit_retry = False
- for _spawn_attempt in (0, 1, 2):
+ _did_moe_cache_retry = False
+ for _spawn_attempt in (0, 1, 2, 3):
# Defensive kill: drop an orphan Popen a concurrent load may
# have stored before we overwrite the reference (#5161).
# Also reaps the crashed first attempt on the retry pass.
@@ -30654,6 +31171,29 @@ class LlamaCppBackend:
env["LD_LIBRARY_PATH"] = _retry_ld
_did_rocm_retry = True
continue
+ if (
+ not _did_moe_cache_retry
+ and _startup_crashed
+ and not _capability_crash
+ and not _hip_rocr_mismatch
+ ):
+ # OOM or a cache error with the cache on: same placement without it.
+ _uncached = self._drop_moe_cache(run_cmd)
+ _crash_text = "\n".join(self._stdout_lines[-200:])
+ if _uncached != run_cmd and (
+ _MOE_CACHE_ERROR_RE.search(_crash_text)
+ or self._is_gpu_memory_start_failure(_crash_text)
+ ):
+ logger.warning(
+ "llama-server crashed during startup (exit code %s) "
+ "with --moe-cache-mib auto; retrying once without "
+ "the MoE expert cache. Crash log: %s",
+ _crashed_proc.returncode,
+ self._llama_log_path,
+ )
+ run_cmd = _uncached
+ _did_moe_cache_retry = True
+ continue
if (
not _did_fit_retry
and _startup_crashed
@@ -31324,8 +31864,25 @@ class LlamaCppBackend:
_retry_apu_msg = None
if model_size is not None and _retry_wants_unified:
_apu_avail_mib = self._available_system_memory_mib()
+ _apu_resident_model_size = max(
+ 1,
+ model_size
+ - self._lazy_read_host_bytes(
+ model_path,
+ caps = server_caps,
+ extra_args = cmd,
+ env = env,
+ auto_resolves_off = self._lazy_auto_resolves_off(
+ gpu_indices = _remaining,
+ detected_gpus = _detected_gpus,
+ shared_gpu_ids = _shared_gpu_ids,
+ is_vulkan_backend = is_vulkan_backend,
+ probe_integrated_cuda = False,
+ ),
+ ),
+ )
_retry_apu_msg = self._apu_ram_shortfall_message(
- model_size + _mmproj_pinned_bytes,
+ _apu_resident_model_size + _mmproj_pinned_bytes,
_apu_avail_mib,
)
# Same contract as the preflight above: the opt-out takes
@@ -32151,7 +32708,7 @@ class LlamaCppBackend:
self._reprice_after_dropping_pinned_projector(
apu_msg = _apu_ram_msg,
host_msg = _host_ram_msg,
- model_size = model_size,
+ model_size = _apu_resident_model_size,
pinned_bytes = _mmproj_pinned_bytes,
avail_mib = _apu_avail_mib,
part = _apu_ram_part,
diff --git a/studio/backend/tests/test_llama_cpp_placement.py b/studio/backend/tests/test_llama_cpp_placement.py
index b6e7a7fc18b..547d17e3f5a 100644
--- a/studio/backend/tests/test_llama_cpp_placement.py
+++ b/studio/backend/tests/test_llama_cpp_placement.py
@@ -4281,3 +4281,870 @@ def test_the_spill_raise_helper_only_raises():
assert _moe_spill_batch_ubatch(None, None, **{**on, off: False}) == (None, None)
assert _moe_spill_batch_ubatch(None, None, **{**on, "user_named_batch": True}) == (None, None)
assert _moe_spill_batch_ubatch(None, None, **{**on, "n_moe_layers": 0}) == (None, None)
+
+
+# MoE experts in host RAM: --moe-cache-mib auto, and lazily read tables.
+
+_MOE_CACHE = ["--moe-cache-mib", "auto"]
+
+
+def _has_moe_cache(cmd):
+ return any(cmd[i : i + 2] == _MOE_CACHE for i in range(len(cmd) - 1))
+
+
+def _write_help_binary(tmp_path, name, help_text):
+ script = tmp_path / name
+ script.write_text("#!/bin/sh\ncat <<'HELP'\n" + help_text + "\nHELP\n")
+ script.chmod(0o755)
+ return str(script)
+
+
+_HELP_BASE = (
+ "-m, --model FNAME model path\n"
+ "--load-mode MODE how to load the model\n"
+)
+
+
[email protected](sys.platform == "win32", reason = "shell script stands in for llama-server")
+ "extra_help, cache, auto, lazy",
+ [
+ ("", False, False, False),
+ (
+ "--moe-cache-mib N GPU cache size in MiB for the MoE experts "
+ "kept in the CPU (default: 0, disabled)\n",
+ True,
+ False,
+ False,
+ ),
+ (
+ "--moe-cache-mib N|auto GPU cache size in MiB for the MoE experts "
+ "kept in the CPU, or auto to size it from free VRAM (default: 0, disabled)\n"
+ "-lzm, --lazy-mode MODE on-demand reading of certain tensors\n",
+ True,
+ True,
+ True,
+ ),
+ ("--tensor-read-lazy MODE on-demand reading\n", False, False, True),
+ ],
+ ids = ["old_build", "upstream_cache", "fork_auto", "pre_rename_lazy"],
+)
+def test_the_probe_reads_the_moe_cache_and_lazy_mode_flags(
+ tmp_path, monkeypatch, extra_help, cache, auto, lazy
+):
+ monkeypatch.setattr(LlamaCppBackend, "_capability_cache", {})
+ binary = _write_help_binary(
+ tmp_path, f"llama-server-{cache}{auto}{lazy}", _HELP_BASE + extra_help
+ )
+ caps = LlamaCppBackend.probe_server_capabilities(binary)
+
+ assert caps["help_probe_ok"] is True
+ assert (
+ caps["supports_moe_cache"],
+ caps["supports_moe_cache_auto"],
+ caps["supports_lazy_mode"],
+ ) == (
+ cache,
+ auto,
+ lazy,
+ )
+ assert caps["lazy_mode_flag"] == (
+ ("--tensor-read-lazy" if "--tensor-read-lazy" in extra_help else "--lazy-mode")
+ if lazy
+ else None
+ )
+
+
+def test_a_failed_probe_advertises_no_moe_cache():
+ caps = LlamaCppBackend.probe_server_capabilities("/nonexistent/llama-server")
+ assert not caps["supports_moe_cache_auto"] and not caps["supports_lazy_mode"]
+
+
+_CACHE_CAPS_ON = dict(
+ supports_load_mode = True,
+ # The 8 GiB --cache-ram default is charged to the cache's RAM admission.
+ supports_cache_ram = True,
+ supports_moe_cache = True,
+ supports_moe_cache_auto = True,
+ supports_lazy_mode = True,
+)
+
+
+def _moe_cache_host(monkeypatch, _discrete_linux_host):
+ import utils.hardware as hardware
+ import utils.model_memory_settings as mm
+
+ # A discrete CUDA host on every runner: macOS CI is Apple Silicon.
+ monkeypatch.setattr(hardware, "is_apple_silicon", lambda: False)
+
+ monkeypatch.setattr(mm, "get_model_memory_settings", lambda: (False, False))
+ monkeypatch.setattr(mm, "get_keep_resident", lambda: False)
+ monkeypatch.setattr(mm, "get_no_ram_reserve", lambda: False)
+ monkeypatch.setattr(mm, "should_mlock", lambda: False)
+ for name in (
+ "LLAMA_ARG_MOE_CACHE_MIB",
+ "LLAMA_ARG_OVERRIDE_TENSOR",
+ "LLAMA_ARG_N_GPU_LAYERS",
+ "LLAMA_ARG_DEVICE",
+ "LLAMA_ARG_FIT",
+ "LLAMA_ARG_LOAD_MODE",
+ "LLAMA_ARG_MMAP",
+ "LLAMA_ARG_NO_MMAP",
+ "LLAMA_ARG_LAZY_MODE",
+ "LLAMA_ARG_CACHE_RAM",
+ "LLAMA_ARG_N_CPU_FFN",
+ "LLAMA_ARG_RPC",
+ ):
+ monkeypatch.delenv(name, raising = False)
+
+
+def _cache_launch(
+ tmp_path,
+ *,
+ caps = None,
+ moe = True,
+ spilled = True,
+ gpus = 1,
+ ram_gib = 256,
+ expert_gib = 15,
+ lazy = None,
+ arch = "qwen3moe",
+ size_gib = None,
+ memory = None,
+ host_guard = False,
+ **load_kwargs,
+):
+ """One launch of an MoE (or dense) model, priced so the fit can answer
+ "none". 20 GiB on an 8 GiB card spills; 1 GiB on a 40 GiB card does not.
+ ``host_guard`` puts the real host-RAM preflight back."""
+ if memory is None:
+ memory = (
+ [(i, 8_000, 16_000) for i in range(gpus)]
+ if spilled
+ else [(i, 40_000, 48_000) for i in range(gpus)]
+ )
+ if size_gib is None:
+ size_gib = 20 if spilled else 1
+ backend, gguf = _moe_backend(tmp_path, size_gib = size_gib, memory = memory, moe = moe)
+ backend._can_estimate_kv = lambda: True
+ backend._estimate_kv_cache_bytes = lambda *a, **k: _GIB // 4
+ backend._available_system_memory_mib = lambda: int(ram_gib * 1024)
+ backend._gguf_tensor_scan = lambda _path: (
+ arch,
+ dict(lazy or {}),
+ int(expert_gib * _GIB) if moe else 0,
+ )
+ full_caps = dict(LlamaCppBackend.probe_server_capabilities.__func__(LlamaCppBackend, None))
+ full_caps.update(_CACHE_CAPS_ON if caps is None else caps)
+ backend.probe_server_capabilities = lambda _binary = None: full_caps
+ if host_guard:
+ _restore_host_guard(backend)
+ cmd = _launch(backend, gguf, **load_kwargs)["cmd"]
+ return backend, cmd
+
+
+def _load_mode(cmd):
+ values = [cmd[i + 1] for i, tok in enumerate(cmd) if tok == "--load-mode"]
+ return values[-1] if values else None
+
+
+def test_a_spilled_moe_on_one_gpu_loading_pinned_gets_the_cache(tmp_path, _moe_cache_host):
+ backend, cmd = _cache_launch(tmp_path)
+
+ assert cmd[cmd.index("--fit") + 1] == "on", cmd
+ assert _load_mode(cmd) == "none", cmd
+ assert _has_moe_cache(cmd), cmd
+ assert cmd.count("--moe-cache-mib") == 1
+ # Next to the expert-spill micro-batch, which it never replaces.
+ assert _ubatch_values(cmd) == ["2048"], cmd
+ assert backend._moe_cache_flags == _MOE_CACHE
+
+
+ "cell, kwargs",
+ [
+ ("dense", dict(moe = False)),
+ ("no_spill", dict(spilled = False)),
+ ("two_gpus", dict(gpus = 2)),
+ # 15 GiB experts + 8 GiB prompt cache > 16 GiB RAM; the 20 GiB load alone fits.
+ ("ram_cannot_pin_experts", dict(ram_gib = 24)),
+ ("ram_cannot_pin_anything", dict(ram_gib = 8)),
+ (
+ "old_build",
+ dict(
+ caps = dict(_CACHE_CAPS_ON, supports_moe_cache = False, supports_moe_cache_auto = False)
+ ),
+ ),
+ ("upstream_build_no_auto", dict(caps = dict(_CACHE_CAPS_ON, supports_moe_cache_auto = False))),
+ ("user_mmap", dict(load_mode = "mmap")),
+ ("user_mmap_extra", dict(extra_args = ["--load-mode", "mmap"])),
+ ("manual_layers", dict(gpu_memory_mode = "manual", gpu_layers = 40)),
+ ],
+ ids = lambda v: v if isinstance(v, str) else "",
+)
+def test_the_moe_cache_stays_off(tmp_path, _moe_cache_host, cell, kwargs):
+ backend, cmd = _cache_launch(tmp_path, **kwargs)
+
+ assert not _has_moe_cache(cmd) and "--moe-cache-mib" not in cmd, (cell, cmd)
+ assert backend._moe_cache_flags == []
+ if cell == "ram_cannot_pin_experts":
+ # Pinned without the cache: the cache never demotes a pinned load.
+ assert _load_mode(cmd) == "none", cmd
+ if cell == "ram_cannot_pin_anything":
+ assert _load_mode(cmd) is None, cmd
+
+
+ "extra_args",
+ [
+ ["-ot", r"blk\.\d+\.ffn_.*_exps\.=CPU"],
+ ["--override-tensor", "exps=CUDA0"],
+ ["-ngl", "30"],
+ ["--n-gpu-layers", "99"],
+ ["-cmoe"],
+ ["--cpu-moe"],
+ ["--n-cpu-moe", "10"],
+ ["--moe-cache-mib", "4096"],
+ ["--fit", "off"],
+ ["--device", "CUDA0"],
+ ["--tensor-split", "1,0"],
+ ["-ncffn", "1"],
+ ["--n-cpu-ffn=1"],
+ ["--rpc", "127.0.0.1:50052"],
+ ],
+ ids = [
+ "ot",
+ "override_tensor",
+ "ngl",
+ "n_gpu_layers",
+ "cmoe",
+ "cpu_moe",
+ "n_cpu_moe",
+ "user_cache",
+ "fit_off",
+ "device",
+ "tensor_split",
+ "ncffn",
+ "n_cpu_ffn_equals",
+ "rpc",
+ ],
+)
+def test_a_user_placement_flag_keeps_the_moe_cache_off(tmp_path, _moe_cache_host, extra_args):
+ _backend, cmd = _cache_launch(tmp_path, extra_args = extra_args)
+
+ assert not _has_moe_cache(cmd), cmd
+ if "--moe-cache-mib" in extra_args:
+ assert cmd.count("--moe-cache-mib") == 1 and cmd[cmd.index("--moe-cache-mib") + 1] == "4096"
+
+
+ "name, value",
+ [
+ ("LLAMA_ARG_MOE_CACHE_MIB", "2048"),
+ ("LLAMA_ARG_N_GPU_LAYERS", "20"),
+ ("LLAMA_ARG_OVERRIDE_TENSOR", "exps=CPU"),
+ ("LLAMA_ARG_FIT", "off"),
+ ("LLAMA_ARG_N_CPU_FFN", "1"),
+ ("LLAMA_ARG_RPC", "127.0.0.1:50052"),
+ ],
+)
+def test_an_inherited_placement_keeps_the_moe_cache_off(
+ tmp_path, _moe_cache_host, monkeypatch, name, value
+):
+ monkeypatch.setenv(name, value)
+ _backend, cmd = _cache_launch(tmp_path)
+
+ assert not _has_moe_cache(cmd), cmd
+
+
+def test_an_unsized_inherited_projector_skips_the_cache_not_the_placement(
+ tmp_path, _moe_cache_host, monkeypatch
+):
+ # An unsized LLAMA_ARG_MMPROJ_URL leaves the fit's model size unknown.
+ monkeypatch.setenv("LLAMA_ARG_MMPROJ_URL", "https://example.invalid/mmproj.gguf")
+ warnings = _override_log(monkeypatch)
+ backend, cmd = _cache_launch(tmp_path)
+
+ assert not any("GPU selection failed" in line for line in warnings), warnings
+ assert not _has_moe_cache(cmd) and backend._moe_cache_flags == []
+
+
+def test_the_moe_cache_eligibility_gates(tmp_path, _moe_cache_host):
+ backend, _gguf = _moe_backend(tmp_path, **_SPILLED)
+ on = dict(
+ caps = _CACHE_CAPS_ON,
+ gpu_memory_mode = "auto",
+ use_fit = True,
+ experts_on_host = True,
+ discrete_gpu = True,
+ gpu_indices = [0],
+ detected_gpus = [(0, 8_000), (1, 8_000)],
+ is_vulkan_backend = False,
+ tensor_parallel = False,
+ extra_args = [],
+ env = {},
+ )
+ assert backend._moe_cache_auto_eligible(**on) is True
+ for off in (
+ dict(use_fit = False),
+ dict(experts_on_host = False),
+ dict(discrete_gpu = False),
+ dict(is_vulkan_backend = True),
+ dict(tensor_parallel = True),
+ dict(gpu_memory_mode = "manual"),
+ dict(gpu_indices = [0, 1]),
+ dict(gpu_indices = None),
+ dict(caps = {}),
+ dict(extra_args = ["--fit", "off"]),
+ dict(env = {"LLAMA_ARG_FIT": "off"}),
+ ):
+ assert backend._moe_cache_auto_eligible(**{**on, **off}) is False, off
+ assert backend._moe_cache_auto_eligible(
+ **{**on, "gpu_indices": None, "detected_gpus": [(0, 8_000)]}
+ )
+ backend._n_experts = None
+ assert backend._moe_cache_auto_eligible(**on) is False
+
+
+def test_every_retry_drops_the_moe_cache_first():
+ backend = LlamaCppBackend()
+ argv = ["llama-server", "-m", "x.gguf", "--fit", "on", "--load-mode", "none", *_MOE_CACHE]
+ # Not this launch's: a user's identical flag is theirs.
+ assert backend._drop_moe_cache(argv, "test") == argv
+
+ backend._moe_cache_flags = list(_MOE_CACHE)
+ assert backend._drop_moe_cache(argv, "test") == argv[:-2]
+ # The CPU replay hands back the uncached argv as well.
+ assert backend._drop_moe_cache(argv[:-2], "test") == argv[:-2]
+
+
+ "crash_line, retried",
+ [
+ (
+ "llama_init_from_model: failed to initialize the context: MoE cache is too small to hold the experts of one token",
+ True,
+ ),
+ (
+ "ggml_backend_cuda_buffer_type_alloc_buffer: allocating 4096.00 MiB on device 0: cudaMalloc failed: out of memory",
+ True,
+ ),
+ ("error: unknown model architecture: 'foo'", False),
+ # Logged on every healthy cache setup: a later failure is not the cache's.
+ (
+ "common_fit_params: moe cache auto: 12288 MiB, 96 experts resident\n"
+ "llama_moe_cache_init: MoE cache size = 12288.00 MiB\n"
+ "error: unknown model architecture: 'foo'",
+ False,
+ ),
+ (
+ "llama_moe_cache_init: MoE cache size = 12288.00 MiB\n"
+ "llama-moe-cache.cpp:412: GGML_ABORT: the MoE cache is too small for the "
+ "experts selected in layer 7",
+ True,
+ ),
+ ("failed to allocate the MoE cache buffers", True),
+ ],
+ ids = ["cache_error", "cuda_oom", "unrelated", "info_lines_then_unrelated", "abort", "alloc"],
+)
+def test_a_cache_crash_retries_once_without_the_cache(
+ tmp_path, _moe_cache_host, crash_line, retried
+):
+ spawned = []
+
+ def fake_popen(cmd, **kwargs):
+ if not cmd or str(cmd[0]) != "/fake/llama-server":
+ return _REAL_POPEN(cmd, **kwargs)
+ spawned.append(list(cmd))
+ crashed = len(spawned) == 1
+ return type(
+ "Process",
+ (),
+ {
+ "pid": 100 + len(spawned),
+ "stdout": iter([crash_line + "\n"]) if crashed else iter(()),
+ "returncode": 1 if crashed else None,
+ "poll": lambda self: 1 if crashed else None,
+ "terminate": lambda self: None,
+ "wait": lambda self, timeout = None: 1 if crashed else 0,
+ "kill": lambda self: None,
+ },
+ )()
+
+ backend, gguf = _moe_backend(tmp_path, **_SPILLED)
+ backend._can_estimate_kv = lambda: True
+ backend._estimate_kv_cache_bytes = lambda *a, **k: _GIB // 4
+ backend._available_system_memory_mib = lambda: 256 * 1024
+ backend._gguf_tensor_scan = lambda _path: ("qwen3moe", {}, 15 * _GIB)
+ full_caps = dict(LlamaCppBackend.probe_server_capabilities.__func__(LlamaCppBackend, None))
+ full_caps.update(_CACHE_CAPS_ON)
+ backend.probe_server_capabilities = lambda _binary = None: full_caps
+
+ def health(timeout, **_kw):
+ if len(spawned) == 1:
+ backend._stdout_lines = [crash_line]
+ return False
+ return True
+
+ backend._wait_for_health = health
+ with patch.object(subprocess, "Popen", side_effect = fake_popen):
+ try:
+ backend.load_model(GgufLoadIntent(gguf_path = str(gguf), model_identifier = "test"))
+ except Exception:
+ pass
+
+ assert _has_moe_cache(spawned[0]), spawned[0]
+ if retried:
+ assert len(spawned) >= 2 and not _has_moe_cache(spawned[1]), spawned
+ assert _without_cache(spawned[0]) == spawned[1]
+ else:
+ assert len(spawned) < 2 or spawned[1] != _without_cache(spawned[0])
+
+
+def _without_cache(cmd):
+ for i in range(len(cmd) - 1):
+ if cmd[i : i + 2] == _MOE_CACHE:
+ return cmd[:i] + cmd[i + 2 :]
+ return list(cmd)
+
+
+# Qwen3.8-Flash-Next IQ1_S (arch qwen4exp), sizes from its GGUF header.
+_QWEN38_TOTAL = int(67.55 * _GIB)
+_QWEN38_PLE = int(26.82 * _GIB) # per_layer_token_embd.weight, TENSOR_READ_LAZY
+_QWEN38_EXPERTS = int(37.11 * _GIB)
+
+
+class _RamStub:
+ """Only what the load-mode predicate touches."""
+
+ def __init__(self, avail_mib):
+ self._avail_mib = avail_mib
+
+ def _available_system_memory_mib(self):
+ return self._avail_mib
+
+ def _amd_apu_wants_unified_memory(self, gpu_indices = None):
+ return False
+
+ _fits_without_paging = LlamaCppBackend._fits_without_paging
+ _FIT_LOAD_MODE = LlamaCppBackend._FIT_LOAD_MODE
+
+
+def _qwen38_mode(
+ lazy_bytes,
+ host_only_bytes = 0,
+ monkeypatch = None,
+):
+ # 16 GiB card (15.5 free, 1 GiB margin), 30 GiB RAM available, 0.5 GiB KV + compute.
+ return LlamaCppBackend._fit_derived_load_mode(
+ _RamStub(30 * 1024),
+ model_size = _QWEN38_TOTAL,
+ kv_cache_bytes = _GIB // 2,
+ compute_buffer_flat = _GIB // 2,
+ host_only_bytes = host_only_bytes,
+ gpus = [(0, int(15.5 * 1024))],
+ gpu_indices = [0],
+ fit_margin_mib = 1024,
+ lazy_read_bytes = lazy_bytes,
+ extra_args = [],
+ env = {},
+ )
+
+
+def test_a_lazily_read_table_is_not_pinned_ram(monkeypatch):
+ import utils.hardware as hardware
+
+ monkeypatch.setattr(hardware, "is_apple_silicon", lambda: False)
+ # 67.55 + 1 GiB against 14.5 GiB of VRAM leaves 54 GiB for 28 GiB of RAM: mmap.
+ assert _qwen38_mode(0) is None
+ # Without the 26.82 GiB table: 41.7 GiB, 27.2 GiB of RAM, which fits pinned.
+ assert _qwen38_mode(_QWEN38_PLE) == LlamaCppBackend._FIT_LOAD_MODE
+ # The cache needs all 37.11 GiB of experts pinned: pinned without it.
+ assert _qwen38_mode(_QWEN38_PLE, host_only_bytes = _QWEN38_EXPERTS) is None
+
+
+ "arch, size, extra_args, env, expected",
+ [
+ ("qwen4exp", _QWEN38_PLE, [], {}, _QWEN38_PLE),
+ ("gemma4", _QWEN38_PLE, [], {}, _QWEN38_PLE),
+ # Another architecture's table of the same name is read like any tensor.
+ ("qwen3moe", _QWEN38_PLE, [], {}, 0),
+ # At or under 4 GiB, auto loads it normally; "on" reads it lazily anyway.
+ ("qwen4exp", 4 * _GIB, [], {}, 0),
+ ("qwen4exp", 4 * _GIB, ["-lzm", "on"], {}, 4 * _GIB),
+ ("qwen4exp", _QWEN38_PLE, ["-lzm", "off"], {}, 0),
+ ("qwen4exp", _QWEN38_PLE, ["--lazy-mode=off"], {}, 0),
+ ("qwen4exp", _QWEN38_PLE, [], {"LLAMA_ARG_LAZY_MODE": "off"}, 0),
+ # Last wins over the env twin.
+ (
+ "qwen4exp",
+ _QWEN38_PLE,
+ ["--lazy-mode", "auto"],
+ {"LLAMA_ARG_LAZY_MODE": "off"},
+ _QWEN38_PLE,
+ ),
+ ],
+ ids = [
+ "qwen4exp",
+ "gemma4",
+ "other_arch",
+ "under_4gib",
+ "on",
+ "lzm_off",
+ "inline_off",
+ "env_off",
+ "argv_beats_env",
+ ],
+)
+def test_lazy_table_bytes(arch, size, extra_args, env, expected):
+ backend = LlamaCppBackend()
+ backend._gguf_tensor_scan = lambda _path: (arch, {"per_layer_token_embd.weight": size}, 0)
+ caps = {"supports_lazy_mode": True}
+ got = backend._lazy_read_host_bytes("/m.gguf", caps = caps, extra_args = extra_args, env = env)
+ assert got == expected
+ # A build without lazy reads loads every tensor.
+ assert backend._lazy_read_host_bytes("/m.gguf", caps = {}, extra_args = extra_args, env = env) == 0
+
+
+def _write_tensor_gguf(path, architecture, tensors):
+ """A GGUF with real tensor infos: (name, dims, ggml_type) each, F32 data laid out
+ back to back."""
+
+ def string(value):
+ data = value.encode()
+ return struct.pack("<Q", len(data)) + data
+
+ header = struct.pack("<IIQQ", 0x46554747, 3, len(tensors), 1)
+ header += string("general.architecture") + struct.pack("<I", 8) + string(architecture)
+ offset = 0
+ for name, dims, ggml_type in tensors:
+ header += string(name) + struct.pack("<I", len(dims)) + struct.pack(f"<{len(dims)}Q", *dims)
+ header += struct.pack("<I", ggml_type) + struct.pack("<Q", offset)
+ n = 1
+ for d in dims:
+ n *= d
+ offset += -(-n * 4 // 32) * 32
+ header += b"\0" * (-len(header) % 32)
+ path.write_bytes(header + b"\0" * offset)
+ return path
+
+
+def test_the_header_scan_sums_shards(tmp_path):
+ f32 = 0
+ _write_tensor_gguf(
+ tmp_path / "m-00001-of-00002.gguf",
+ "qwen4exp",
+ [
+ ("per_layer_token_embd.weight", (8, 4), f32),
+ ("blk.0.ffn_up_exps.weight", (4, 4, 2), f32),
+ ],
+ )
+ _write_tensor_gguf(
+ tmp_path / "m-00002-of-00002.gguf",
+ "",
+ [("blk.1.ffn_down_exps.weight", (4, 4, 2), f32), ("blk.1.attn_q.weight", (4, 4), f32)],
+ )
+ backend = LlamaCppBackend()
+ arch, named, experts = backend._gguf_tensor_scan(str(tmp_path / "m-00001-of-00002.gguf"))
+
+ assert arch == "qwen4exp"
+ assert named == {"per_layer_token_embd.weight": 8 * 4 * 4}
+ assert experts == 2 * 4 * 4 * 2 * 4
+ # A missing shard reads as unknown, which discounts nothing.
+ (tmp_path / "m-00002-of-00002.gguf").unlink()
+ assert LlamaCppBackend()._gguf_tensor_scan(str(tmp_path / "m-00001-of-00002.gguf")) is None
+
+
+def test_the_launch_discounts_the_lazy_table(tmp_path, _moe_cache_host):
+ """The same predicate through the launch: a 20 GiB MoE whose 12 GiB n-gram table
+ is lazy fits 8 GiB of VRAM plus 12 GiB of RAM only without the table."""
+ plain = _cache_launch(
+ tmp_path, ram_gib = 12, caps = dict(_CACHE_CAPS_ON, supports_moe_cache_auto = False)
+ )[1]
+ assert _load_mode(plain) is None, plain
+
+ lazy = {"per_layer_token_embd.weight": 12 * _GIB}
+ no_auto = dict(_CACHE_CAPS_ON, supports_moe_cache_auto = False)
+ _b, cmd = _cache_launch(tmp_path, ram_gib = 12, lazy = lazy, caps = no_auto)
+ assert _load_mode(cmd) is None, "qwen3moe does not mark the table lazy"
+
+ _b, cmd = _cache_launch(tmp_path, ram_gib = 12, lazy = lazy, arch = "qwen4exp", caps = no_auto)
+ assert _load_mode(cmd) == "none", cmd
+ # And "-lzm off" makes it an ordinary tensor again.
+ _b, cmd = _cache_launch(
+ tmp_path, ram_gib = 12, lazy = lazy, arch = "qwen4exp", caps = no_auto, extra_args = ["-lzm", "off"]
+ )
+ assert _load_mode(cmd) is None, cmd
+
+
[email protected]("ram_gib, cached", [(60, True), (40, False)])
+def test_the_cache_admission_counts_the_experts_once(tmp_path, _moe_cache_host, ram_gib, cached):
+ """Qwen3.8-Flash-Next IQ1_S on one 24 GiB card. With the lazy table on disk the
+ load is ~41 GiB; the cache needs the 37.11 GiB of experts plus the 8 GiB prompt
+ cache pinned in RAM, which 60 GiB holds and 40 GiB does not. Charging the experts
+ to RAM on top of model_size, as well as in it, refused the 60 GiB host."""
+ backend, cmd = _cache_launch(
+ tmp_path,
+ size_gib = _QWEN38_TOTAL / _GIB,
+ memory = [(0, 24_000, 24_576)],
+ ram_gib = ram_gib,
+ expert_gib = _QWEN38_EXPERTS / _GIB,
+ lazy = {"per_layer_token_embd.weight": _QWEN38_PLE},
+ arch = "qwen4exp",
+ )
+
+ # Pinned either way: the cache never demotes a pinned load.
+ assert _load_mode(cmd) == "none", cmd
+ assert _has_moe_cache(cmd) is cached, cmd
+ assert backend._moe_cache_flags == (_MOE_CACHE if cached else [])
+
+
[email protected]("ram_gib, cached", [(45, False), (60, True)])
+def test_qwen38_on_a_16gib_card_launches_pinned_past_the_host_guard(
+ tmp_path, _moe_cache_host, ram_gib, cached
+):
+ """Qwen3.8-Flash-Next IQ1_S on one 16 GiB card, end to end with the real host-RAM
+ preflight. The fit leaves the 26.82 GiB lazy table out and picks "none"; the
+ preflight charged the whole 67.55 GiB file, read a shortfall, and the pageable
+ rewrite took the pinned mode (and with it the cache) back out."""
+ backend, cmd = _cache_launch(
+ tmp_path,
+ size_gib = _QWEN38_TOTAL / _GIB,
+ memory = [(0, 15_500, 16_384)],
+ ram_gib = ram_gib,
+ expert_gib = _QWEN38_EXPERTS / _GIB,
+ lazy = {"per_layer_token_embd.weight": _QWEN38_PLE},
+ arch = "qwen4exp",
+ host_guard = True,
+ )
+
+ assert _load_mode(cmd) == "none", cmd
+ assert _has_moe_cache(cmd) is cached, cmd
+ assert backend.last_load_warning is None, backend.last_load_warning
+
+
+def test_the_host_guard_still_charges_a_table_read_in_full(tmp_path, _moe_cache_host):
+ """Under "-lzm off" the table is an ordinary tensor: the same host is short."""
+ _backend, cmd = _cache_launch(
+ tmp_path,
+ size_gib = _QWEN38_TOTAL / _GIB,
+ memory = [(0, 15_500, 16_384)],
+ ram_gib = 45,
+ expert_gib = _QWEN38_EXPERTS / _GIB,
+ lazy = {"per_layer_token_embd.weight": _QWEN38_PLE},
+ arch = "qwen4exp",
+ host_guard = True,
+ extra_args = ["-lzm", "off"],
+ )
+
+ assert _load_mode(cmd) is None and not _has_moe_cache(cmd), cmd
+
+
+def test_an_igpu_only_child_gets_no_auto_lazy_discount(tmp_path, _moe_cache_host, monkeypatch):
+ """llama.cpp turns lazy auto off on a device without mmap support, so the fit that
+ pins the lazy table's model on a discrete card leaves it mapped here."""
+ monkeypatch.setattr(LlamaCppBackend, "_lazy_auto_resolves_off", lambda self, **_kw: True)
+ lazy = {"per_layer_token_embd.weight": 12 * _GIB}
+ no_auto = dict(_CACHE_CAPS_ON, supports_moe_cache_auto = False)
+
+ _b, cmd = _cache_launch(tmp_path, ram_gib = 12, lazy = lazy, arch = "qwen4exp", caps = no_auto)
+ assert _load_mode(cmd) is None, cmd
+ # An explicit "on" is read lazily on any device.
+ _b, cmd = _cache_launch(
+ tmp_path, ram_gib = 12, lazy = lazy, arch = "qwen4exp", caps = no_auto, extra_args = ["-lzm", "on"]
+ )
+ assert _load_mode(cmd) == "none", cmd
+
+
+ "vulkan, gpu_indices, shared, rocm_unified, cuda_integrated, expected",
+ [
+ (False, [0], set(), set(), set(), False),
+ (True, [0], {0}, set(), set(), True),
+ # llama.cpp drops an iGPU from its default list once a discrete GPU is seen.
+ (True, None, {1}, set(), set(), False),
+ (False, [0], set(), {0}, set(), True),
+ (False, [0, 1], set(), {0}, set(), False),
+ (False, [0], set(), set(), {0}, True),
+ (False, None, set(), set(), {0, 1}, True),
+ ],
+ ids = [
+ "discrete",
+ "vulkan_igpu",
+ "vulkan_mixed",
+ "rocm_apu",
+ "rocm_mixed",
+ "cuda_soc",
+ "cuda_soc_unpinned",
+ ],
+)
+def test_lazy_auto_resolves_off_only_where_llama_cpp_does(
+ monkeypatch, vulkan, gpu_indices, shared, rocm_unified, cuda_integrated, expected
+):
+ monkeypatch.setattr(
+ LlamaCppBackend, "_rocm_unified_memory_gpu_ids", staticmethod(lambda: set(rocm_unified))
+ )
+ monkeypatch.setattr(
+ LlamaCppBackend, "_integrated_cuda_gpu_ids", staticmethod(lambda: set(cuda_integrated))
+ )
+ monkeypatch.setattr(
+ LlamaCppBackend, "_resolve_visible_physical_ids", staticmethod(lambda: [0, 1])
+ )
+ got = LlamaCppBackend()._lazy_auto_resolves_off(
+ gpu_indices = gpu_indices,
+ detected_gpus = [(0, 8_000), (1, 8_000)],
+ shared_gpu_ids = shared,
+ is_vulkan_backend = vulkan,
+ )
+ assert got is expected
+
+ backend = LlamaCppBackend()
+ backend._gguf_tensor_scan = lambda _path: (
+ "qwen4exp",
+ {"per_layer_token_embd.weight": _QWEN38_PLE},
+ 0,
+ )
+ caps = {"supports_lazy_mode": True}
+ auto = backend._lazy_read_host_bytes("/m.gguf", caps = caps, env = {}, auto_resolves_off = got)
+ on = backend._lazy_read_host_bytes(
+ "/m.gguf", caps = caps, extra_args = ["-lzm", "on"], env = {}, auto_resolves_off = got
+ )
+ assert (auto, on) == ((0 if expected else _QWEN38_PLE), _QWEN38_PLE)
+
+
[email protected]("lazy_args, warned", [(["-lzm", "on"], False), ([], True)])
+def test_the_apu_guard_prices_only_resident_weights(tmp_path, monkeypatch, lazy_args, warned):
+ """A 64.6 GiB model with a 26.82 GiB table on an APU with 46 GiB of RAM. "-lzm on"
+ leaves the table on disk, so the unmapped load fits; auto is off on an APU, so the
+ whole file loads and the guard still warns and remaps."""
+ backend, gguf = _apu_backend(
+ tmp_path, gguf_gb = 64.6, avail_mib = 46 * 1024, monkeypatch = monkeypatch
+ )
+ monkeypatch.setattr(LlamaCppBackend, "_rocm_unified_memory_gpu_ids", staticmethod(lambda: {0}))
+ backend._gguf_tensor_scan = lambda _path: (
+ "qwen4exp",
+ {"per_layer_token_embd.weight": _QWEN38_PLE},
+ 0,
+ )
+ caps = dict(LlamaCppBackend.probe_server_capabilities.__func__(LlamaCppBackend, None))
+ caps.update(_CACHE_CAPS_ON)
+ backend.probe_server_capabilities = lambda _binary = None: caps
+
+ cmd = _launch(backend, gguf, extra_args = ["--load-mode", "none", *lazy_args])["cmd"]
+
+ assert ("unified-memory APU" in (backend.last_load_warning or "")) is warned
+ assert bool(_unmapped_tokens(cmd)) is not warned, cmd
+
+
+ "flag, extra_args, env, expected",
+ [
+ ("--lazy-mode", ["-lzm", "off"], {}, 0),
+ ("--lazy-mode", [], {"LLAMA_ARG_LAZY_MODE": "off"}, 0),
+ # The old spelling's env twin is not read by a renamed build.
+ ("--lazy-mode", [], {"LLAMA_ARG_TENSOR_READ_LAZY": "off"}, _QWEN38_PLE),
+ ("--tensor-read-lazy", ["--tensor-read-lazy", "off"], {}, 0),
+ ("--tensor-read-lazy", [], {"LLAMA_ARG_TENSOR_READ_LAZY": "off"}, 0),
+ # Nor the new one's by a build from before the rename.
+ ("--tensor-read-lazy", [], {"LLAMA_ARG_LAZY_MODE": "off"}, _QWEN38_PLE),
+ ],
+ ids = [
+ "new_flag",
+ "new_env",
+ "new_ignores_old_env",
+ "old_flag",
+ "old_env",
+ "old_ignores_new_env",
+ ],
+)
+def test_only_the_probed_lazy_spelling_is_honoured(flag, extra_args, env, expected):
+ backend = LlamaCppBackend()
+ backend._gguf_tensor_scan = lambda _path: (
+ "qwen4exp",
+ {"per_layer_token_embd.weight": _QWEN38_PLE},
+ 0,
+ )
+ caps = {"supports_lazy_mode": True, "lazy_mode_flag": flag}
+ got = backend._lazy_read_host_bytes("/m.gguf", caps = caps, extra_args = extra_args, env = env)
+ assert got == expected
+
+
+def _write_writer_gguf(
+ path,
+ tensors,
+ *,
+ split_max_tensors = 0,
+):
+ """``tensors``: (name, numpy array, raw ggml type or None) each, written by
+ gguf-py's own writer, split llama.cpp-style when ``split_max_tensors`` is set."""
+ import gguf
+
+ writer = gguf.GGUFWriter(path, "qwen4exp", split_max_tensors = split_max_tensors)
+ writer.add_string("tokenizer.ggml.model", "gpt2")
+ writer.add_array("tokenizer.ggml.tokens", ["a", "b", "<c>"])
+ for name, array, raw_dtype in tensors:
+ writer.add_tensor(name, array, raw_dtype = raw_dtype)
+ writer.write_header_to_file()
+ writer.write_kv_data_to_file()
+ writer.write_tensors_to_file()
+ writer.close()
+
+
+def test_the_header_scan_reads_writer_output(tmp_path, monkeypatch):
+ np = pytest.importorskip("numpy")
+ gguf = pytest.importorskip("gguf")
+ from gguf.constants import GGML_QUANT_SIZES, GGMLQuantizationType
+
+ q8 = GGMLQuantizationType.Q8_0
+ q8_block, q8_bytes = GGML_QUANT_SIZES[q8]
+ # Q8_0 rows of 64 elements: two blocks each, stored as raw bytes.
+ q8_rows = np.zeros((4, 2 * q8_bytes), dtype = np.uint8)
+ tensors = [
+ ("per_layer_token_embd.weight", np.zeros((8, 16), dtype = np.float32), None),
+ ("blk.0.ffn_up_exps.weight", np.zeros((2, 4, 32), dtype = np.float16), None),
+ ("blk.0.attn_q.weight", np.zeros((4, 32), dtype = np.float32), None),
+ ("blk.1.ffn_down_exps.weight", q8_rows, q8),
+ ]
+ _write_writer_gguf(tmp_path / "m.gguf", tensors, split_max_tensors = 2)
+ shards = sorted(p.name for p in tmp_path.glob("*.gguf"))
+ assert shards == ["m-00001-of-00002.gguf", "m-00002-of-00002.gguf"], shards
+ first = str(tmp_path / shards[0])
+ expected_experts = 2 * 4 * 32 * 2 + 4 * 64 // q8_block * q8_bytes
+
+ arch, named, experts = LlamaCppBackend()._gguf_tensor_scan(first)
+ assert arch == "qwen4exp"
+ assert named == {"per_layer_token_embd.weight": 8 * 16 * 4}
+ assert experts == expected_experts
+
+ # Unknown quant type: sized from the next offset / end of file, within one alignment.
+ monkeypatch.delitem(GGML_QUANT_SIZES, q8)
+ monkeypatch.delitem(GGML_QUANT_SIZES, GGMLQuantizationType.F16)
+ _arch, named, experts = LlamaCppBackend()._gguf_tensor_scan(first)
+ assert named == {"per_layer_token_embd.weight": 8 * 16 * 4}
+ assert expected_experts <= experts < expected_experts + 2 * gguf.GGUF_DEFAULT_ALIGNMENT
+
+ # A missing shard reads as unknown, which discounts nothing.
+ (tmp_path / shards[1]).unlink()
+ assert LlamaCppBackend()._gguf_tensor_scan(first) is None
+
+
+def test_the_header_scan_refuses_what_it_cannot_parse(tmp_path):
+ path = _write_tensor_gguf(
+ tmp_path / "v1.gguf", "qwen4exp", [("per_layer_token_embd.weight", (8, 4), 0)]
+ )
+ data = bytearray(path.read_bytes())
+ # GGUF v1 used 32-bit counts: nothing is read from it.
+ data[4:8] = struct.pack("<I", 1)
+ path.write_bytes(bytes(data))
+ assert LlamaCppBackend._gguf_scan_tensor_bytes(str(path)) == (None, {}, 0)
+
+ # An unknown KV value type has no size to skip, so the scan gives up.
+ header = struct.pack("<IIQQ", 0x46554747, 3, 0, 1)
+ header += struct.pack("<Q", 3) + b"odd" + struct.pack("<I", 99) + b"\0" * 8
+ bad = tmp_path / "bad.gguf"
+ bad.write_bytes(header)
+ with pytest.raises(ValueError):
+ LlamaCppBackend._gguf_scan_tensor_bytes(str(bad))
+ assert LlamaCppBackend()._gguf_tensor_scan(str(bad)) is None
diff --git a/studio/backend/tests/test_llama_cpp_single_sequence_retry.py b/studio/backend/tests/test_llama_cpp_single_sequence_retry.py
index e0ec554d1df..d3f9337d21c 100644
--- a/studio/backend/tests/test_llama_cpp_single_sequence_retry.py
+++ b/studio/backend/tests/test_llama_cpp_single_sequence_retry.py
@@ -155,8 +155,9 @@ def test_the_fit_recovery_rungs_stand_down_for_this_refusal():
src = inspect.getsource(LlamaCppBackend.load_model)
assert "_capability_crash = _tensor_capability_crash or self._is_kv_unified_refused" in src
- # Every --fit rung, and only those: the HIP rung keeps its narrower gate.
- assert src.count("and not _capability_crash") == 3
+ # Every --fit rung and the MoE cache rung, and only those: the HIP rung keeps its
+ # narrower gate.
+ assert src.count("and not _capability_crash") == 4
assert src.count("and not _tensor_capability_crash") == 1