Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
Studio: size a GPU cache for spilled MoE experts with --moe-cache-mib auto (#12951)
Daniel Hanb15cda912a6c
diff --git a/studio/backend/core/inference/llama_cpp.py b/studio/backend/core/inference/llama_cpp.pyindex 8ee38bf42e4..34ff04eb483 100644--- a/studio/backend/core/inference/llama_cpp.py+++ b/studio/backend/core/inference/llama_cpp.py@@ -99,6 +99,7 @@ from core.inference.llama_server_args import (_TENSOR_SPLIT_FLAGS,_effective_tensor_parallel,_flag_name,+ _last_flag_value,_tensor_parallel_matches_loaded,LLAMA_CTX_CHECKPOINTS_DEFAULT,apply_load_mode_policy,@@ -7092,6 +7093,132 @@ def _expert_spill_places_tensors_on_cpu(return _args_place_tensors_on_cpu(kept) or _env_places_tensors_on_cpu(source_env)+# GPU cache of host-resident experts (ggml-org/llama.cpp#29887); "auto" is sized by the+# fork's fitter (unslothai/llama.cpp#251). One run, so a retry can strip it.+_MOE_CACHE_AUTO_TOKENS = ("--moe-cache-mib", "auto")+# User placement: the fitter aborts on most of it, and the cache refuses >1 device (--rpc adds one).+_MOE_CACHE_USER_OWNED_FLAGS = (+ frozenset({"--moe-cache-mib", "-ot", "--override-tensor", "--rpc"})+ | _CPU_FFN_COUNT_FLAGS+ | _DEVICE_FLAGS+ | _GPU_LAYER_FLAGS+ | _MOE_OFFLOAD_FLAGS+ | _SPLIT_MODE_FLAGS+ | _TENSOR_SPLIT_FLAGS+)+_MOE_CACHE_USER_OWNED_ENV = (+ "LLAMA_ARG_MOE_CACHE_MIB",+ "LLAMA_ARG_OVERRIDE_TENSOR",+ "LLAMA_ARG_CPU_MOE",+ "LLAMA_ARG_N_CPU_MOE",+ "LLAMA_ARG_N_CPU_FFN",+ "LLAMA_ARG_DEVICE",+ "LLAMA_ARG_MAIN_GPU",+ "LLAMA_ARG_SPLIT_MODE",+ "LLAMA_ARG_TENSOR_SPLIT",+ "LLAMA_ARG_RPC",+)+# Exact startup errors: healthy launches also log "moe cache auto:" and "MoE cache size =".+_MOE_CACHE_ERROR_RE = re.compile(+ r"MoE cache (?:requires a GPU backend|does not support tensor parallelism"+ r"|requires a MoE model|is too small to hold the experts of one token)"+ r"|failed to (?:create the MoE cache context|allocate the MoE cache buffers)"+ r"|the MoE cache is too small for the experts selected in layer",+ re.IGNORECASE,+)++# TENSOR_READ_LAZY tensors (src/models/gemma4.cpp, qwen4exp.cpp): under --lazy-mode auto one+# over 4 GiB stays mmap'd under every load mode (lazy_read::add), needing no pinned RAM or VRAM.+_LAZY_READ_TENSORS: dict[str, tuple[str, ...]] = {+ "gemma4": ("per_layer_token_embd.weight",),+ "qwen4exp": ("per_layer_token_embd.weight",),+}+_LAZY_READ_AUTO_MIN_BYTES = 4 * 1024**3+# --tensor-read-lazy predates ggml-org/llama.cpp#27969; a build reads only its own spelling.+_LAZY_MODE_SPELLINGS: dict[str, tuple[frozenset[str], str]] = {+ "--lazy-mode": (frozenset({"-lzm", "--lazy-mode"}), "LLAMA_ARG_LAZY_MODE"),+ "--tensor-read-lazy": (frozenset({"--tensor-read-lazy"}), "LLAMA_ARG_TENSOR_READ_LAZY"),+}+++# llama-server's --cache-ram default (MiB): the host-RAM prompt cache it grows into.+_LLAMA_CACHE_RAM_DEFAULT_MIB = 8192+++def _prompt_cache_host_bytes(+ cache_ram: Optional[int],+ extra_args: Optional[Iterable[str]],+ env: Optional[Mapping[str, str]] = None,+ *,+ supports_cache_ram: bool,+) -> Optional[int]:+ """The host RAM llama-server's prompt cache may grow into, or None when it is+ unbounded (``-1``) or unreadable. Env first, then the launch's own value, then a+ last-wins pass-through flag; zero on a build without the cache."""+ if not supports_cache_ram:+ return 0+ source_env = os.environ if env is None else env+ value: object = _LLAMA_CACHE_RAM_DEFAULT_MIB+ if str(source_env.get("LLAMA_ARG_CACHE_RAM") or "").strip():+ value = source_env.get("LLAMA_ARG_CACHE_RAM")+ if cache_ram is not None:+ value = cache_ram+ try:+ override = _last_flag_value(extra_args, _CACHE_RAM_FLAGS)+ if override is not None:+ value = override+ mib = int(str(value).strip())+ except (TypeError, ValueError):+ return None+ return None if mib < 0 else mib * 1024 * 1024+++def _argv_loads_pinned(argv: Iterable[str], env: Optional[Mapping[str, str]] = None) -> bool:+ """Whether ``argv`` (with the env twins) loads the weights into a pinned host+ copy, i.e. ``--load-mode none``: no mmap, no page-lock, no DirectIO."""+ mlock, reserves_ram, direct_io = resolve_effective_load_state(list(argv), env)+ return bool(reserves_ram and not mlock and not direct_io)+++def _effective_lazy_mode(+ extra_args: Optional[Iterable[str]],+ env: Optional[Mapping[str, str]] = None,+ *,+ flag: str = "--lazy-mode",+) -> str:+ """llama.cpp's lazy mode for this launch: env first, then last-wins argv, both in+ the build's own spelling ``flag``.++ A value llama-server would refuse reads as "off", which discounts nothing."""+ source_env = os.environ if env is None else env+ flags, env_name = _LAZY_MODE_SPELLINGS.get(flag, _LAZY_MODE_SPELLINGS["--lazy-mode"])+ mode = "auto"+ value = str(source_env.get(env_name) or "").strip().lower()+ if value:+ mode = value+ try:+ value = _last_flag_value(extra_args, flags)+ except ValueError:+ return "off"+ if value:+ mode = value.strip().lower()+ return mode if mode in ("auto", "on") else "off"+++def _lazy_read_bytes(+ architecture: Optional[str], tensor_bytes: Mapping[str, int], mode: str+) -> int:+ """Bytes of ``tensor_bytes`` llama.cpp reads lazily under ``mode``."""+ if mode not in ("auto", "on"):+ return 0+ names = _LAZY_READ_TENSORS.get(architecture or "", ())+ return sum(+ int(size)+ for name, size in tensor_bytes.items()+ if name in names and (mode == "on" or int(size) > _LAZY_READ_AUTO_MIN_BYTES)+ )++def _build_ngram_mod_flags(caps: Optional[dict],n_match: int = 24,@@ -7971,6 +8098,7 @@ class LlamaCppBackend:# The expert-spill micro-batch raise: the pair before it, and the argv tokens.self._moe_spill_batch_restore: Optional[tuple[Optional[int], Optional[int]]] = Noneself._moe_spill_batch_tokens: Optional[tuple[list[str], list[str]]] = None+ self._moe_cache_flags: List[str] = []# The tuning group the last load asked for; none = defaults, or left to# extras / env. What was REQUESTED, not what ran: Model Memory can replace# the load mode, and Windows full-offload tuning owns the two cache knobs.@@ -9819,6 +9947,10 @@ class LlamaCppBackend:"supports_trace_verbosity": False,"supports_video_fps": False,"supports_load_mode": False,+ "supports_moe_cache": False,+ "supports_moe_cache_auto": False,+ "supports_lazy_mode": False,+ "lazy_mode_flag": None,"spec_draft_ngl_flag": None,"spec_draft_cache_k_flag": None,"spec_draft_cache_v_flag": None,@@ -9873,6 +10005,10 @@ class LlamaCppBackend:supports_no_mmproj_offload = Falsesupports_video_fps = Falsesupports_load_mode = False+ supports_moe_cache = False+ supports_moe_cache_auto = False+ supports_lazy_mode = False+ lazy_mode_flag = Nonespec_draft_ngl_flag = Nonespec_draft_cache_k_flag = Nonespec_draft_cache_v_flag = None@@ -10118,6 +10254,20 @@ class LlamaCppBackend:# --load-mode supersedes --mlock / --no-mmap, which are deprecated.# Pre-initialised above: a failed probe must fall back, not raise.supports_load_mode = _is_real("--load-mode")+ # "auto" is a fork addition; upstream builds exit on it, so fail closed.+ supports_moe_cache = bool(probe_ok and _is_real("--moe-cache-mib"))+ supports_moe_cache_auto = bool(+ supports_moe_cache+ and re.search(+ r"(?<![a-z0-9_-])auto(?![a-z0-9_-])",+ (blocks.get("--moe-cache-mib") or "").lower(),+ )+ )+ if probe_ok:+ lazy_mode_flag = next(+ (f for f in ("--lazy-mode", "--tensor-read-lazy") if _is_real(f)), None+ )+ supports_lazy_mode = lazy_mode_flag is not None# Record WHICH alias this build has: --spec-draft-ngl only landed in# b8955, and a build exposing only --gpu-layers-draft would refuse to# start on the newer name. Long forms only, since the block parser above@@ -10216,6 +10366,10 @@ class LlamaCppBackend:"supports_trace_verbosity": supports_trace_verbosity,"supports_video_fps": supports_video_fps,"supports_load_mode": supports_load_mode,+ "supports_moe_cache": supports_moe_cache,+ "supports_moe_cache_auto": supports_moe_cache_auto,+ "supports_lazy_mode": supports_lazy_mode,+ "lazy_mode_flag": lazy_mode_flag,"spec_draft_ngl_flag": spec_draft_ngl_flag,"spec_draft_cache_k_flag": spec_draft_cache_k_flag,"spec_draft_cache_v_flag": spec_draft_cache_v_flag,@@ -10398,6 +10552,163 @@ class LlamaCppBackend:return total+ @staticmethod+ def _gguf_scan_tensor_bytes(path: str) -> "tuple[Optional[str], dict[str, int], int]":+ """(architecture, lazy-read candidate bytes by name, routed expert bytes) from one+ GGUF header. Streamed: ``GGUFReader`` materialises the vocabulary under the load+ lock. A quant type gguf predates is sized from the next tensor's offset."""+ from gguf.constants import GGML_QUANT_SIZES+ from core.inference.offload_layout import _BLOCK_RE, _MOE_EXPERT_RE++ wanted = {name.encode() for names in _LAZY_READ_TENSORS.values() for name in names}+ architecture: Optional[str] = None+ alignment = 32 # GGUF's default when general.alignment is absent+ named: dict[str, int] = {}+ experts = 0+ offsets: list[int] = []+ unsized: list[tuple[str, int]] = []+ with open(path, "rb") as f:+ if struct.unpack("<I", f.read(4))[0] != 0x46554747: # b"GGUF"+ return None, {}, 0+ # Version 1 used 32-bit counts and lengths, which this layout misreads.+ if struct.unpack("<I", f.read(4))[0] < 2:+ return None, {}, 0+ n_tensors, n_kv = struct.unpack("<QQ", f.read(16))+ for _ in range(n_kv):+ key = f.read(struct.unpack("<Q", f.read(8))[0])+ value_type = struct.unpack("<I", f.read(4))[0]+ if key == b"general.architecture" and value_type == 8:+ raw = f.read(struct.unpack("<Q", f.read(8))[0])+ architecture = raw.decode("utf-8", "replace")+ continue+ if key == b"general.alignment" and value_type == 4:+ alignment = max(1, struct.unpack("<I", f.read(4))[0])+ continue+ LlamaCppBackend._gguf_skip_value(f, value_type)+ for _ in range(n_tensors):+ raw_name = f.read(struct.unpack("<Q", f.read(8))[0])+ n_dims = struct.unpack("<I", f.read(4))[0]+ dims = struct.unpack(f"<{n_dims}Q", f.read(8 * n_dims))+ ggml_type = struct.unpack("<I", f.read(4))[0]+ offset = struct.unpack("<Q", f.read(8))[0]+ offsets.append(offset)+ block_match = _BLOCK_RE.match(raw_name.decode("utf-8", "replace"))+ is_expert = bool(block_match and _MOE_EXPERT_RE.match(block_match.group(2)))+ if not is_expert and raw_name not in wanted:+ continue+ name = raw_name.decode("utf-8", "replace")+ block = GGML_QUANT_SIZES.get(ggml_type)+ if block is None:+ unsized.append((name, offset))+ continue+ size = math.prod(dims) // block[0] * block[1]+ if is_expert:+ experts += size+ else:+ named[name] = size+ if unsized:+ data_start = -(-f.tell() // alignment) * alignment+ data_end = max(0, os.path.getsize(path) - data_start)+ ordered = sorted(offsets)+ for name, offset in unsized:+ later = ordered[bisect.bisect_right(ordered, offset) :]+ size = max(0, (later[0] if later else data_end) - offset)+ block_match = _BLOCK_RE.match(name)+ if block_match and _MOE_EXPERT_RE.match(block_match.group(2)):+ experts += size+ else:+ named[name] = size+ return architecture, named, experts++ def _gguf_tensor_scan(+ self, model_path: Optional[str]+ ) -> "Optional[tuple[Optional[str], dict[str, int], int]]":+ """``_gguf_scan_tensor_bytes`` over every shard, cached per file identity; None+ when a shard is missing or unreadable."""+ from core.inference.offload_layout import split_shard_paths++ if not model_path:+ return None+ paths = split_shard_paths(model_path) or [model_path]+ try:+ key = tuple((p, st.st_size, st.st_mtime_ns) for p in paths for st in (os.stat(p),))+ except OSError:+ return None+ cached = getattr(self, "_gguf_tensor_scan_cache", None)+ if cached is not None and cached[0] == key:+ return cached[1]+ try:+ architecture: Optional[str] = None+ named: dict[str, int] = {}+ experts = 0+ for p in paths:+ arch, part, part_experts = self._gguf_scan_tensor_bytes(p)+ # Shard 1 carries the metadata.+ architecture = architecture or arch+ for name, size in part.items():+ named[name] = named.get(name, 0) + size+ experts += part_experts+ result = (architecture, named, experts)+ except Exception as e: # unreadable or truncated: discount nothing+ logger.debug("GGUF tensor scan failed for %s (%s)", model_path, e)+ result = None+ self._gguf_tensor_scan_cache = (key, result)+ return result++ def _lazy_read_host_bytes(+ self,+ model_path: Optional[str],+ *,+ caps: Optional[Mapping[str, object]],+ extra_args: Optional[Iterable[str]] = None,+ env: Optional[Mapping[str, str]] = None,+ auto_resolves_off: bool = False,+ ) -> int:+ """Bytes llama.cpp leaves mmap'd from disk under every load mode (``_LAZY_READ_TENSORS``);+ zero without lazy-read support, under ``off``, or on an unreadable header.+ ``auto_resolves_off``: llama.cpp turns ``auto`` into ``off`` for these devices."""+ if not model_path or not (caps or {}).get("supports_lazy_mode"):+ return 0+ mode = _effective_lazy_mode(+ extra_args, env, flag = str((caps or {}).get("lazy_mode_flag") or "--lazy-mode")+ )+ if mode == "off" or (mode == "auto" and auto_resolves_off):+ return 0+ scan = self._gguf_tensor_scan(model_path)+ if scan is None:+ return 0+ architecture, named, _experts = scan+ return _lazy_read_bytes(architecture, named, mode)++ def _lazy_auto_resolves_off(+ self,+ *,+ gpu_indices: Optional[Iterable[int]],+ detected_gpus: Optional[Iterable[tuple]],+ shared_gpu_ids: Optional[Iterable[int]],+ is_vulkan_backend: bool,+ probe_integrated_cuda: bool = True,+ ) -> bool:+ """Whether llama.cpp resolves ``--lazy-mode auto`` to ``off`` for this child: a model+ device without mmap support (ggml-org/llama.cpp#28160), i.e. every selected device+ integrated, since the default list holds iGPUs only when no discrete GPU is seen.+ ``probe_integrated_cuda`` False uses only a cached SoC answer (no new context)."""+ rows = [(row[0], row[1]) for row in (detected_gpus or ())]+ if self._offload_target_shares_system_memory(+ is_vulkan_backend = is_vulkan_backend,+ shared_gpu_ids = shared_gpu_ids,+ detected_gpus = rows,+ gpu_indices = gpu_indices,+ ):+ return True+ if is_vulkan_backend:+ return False+ if not probe_integrated_cuda and not self._integrated_cuda_probe_is_free():+ return False+ return self._integrated_cuda_selection_is_all_shared(+ list(gpu_indices) if gpu_indices else None+ )+@staticmethoddef _installed_ggml_backends(binary: Optional[str] = None) -> frozenset[str]:"""Backend libraries shipped beside llama-server."""@@ -14851,6 +15162,27 @@ class LlamaCppBackend:if not model_bytes or (not gpus and not child_has_no_gpu):return Noneshared = set(shared_gpu_ids or ())+ # Same lazy-read discount as the fit, or this rewrite undoes its pinned verdict.+ try:+ model_bytes = max(+ 1,+ model_bytes+ - self._lazy_read_host_bytes(+ model_path,+ caps = self.probe_server_capabilities(argv[0]),+ extra_args = argv,+ env = _env,+ auto_resolves_off = self._lazy_auto_resolves_off(+ gpu_indices = [row[0] for row in gpus] if gpus else None,+ detected_gpus = gpus,+ shared_gpu_ids = shared,+ is_vulkan_backend = bool(shared) or self._is_vulkan_backend(argv[0]),+ probe_integrated_cuda = False,+ ),+ ),+ )+ except Exception as e: # noqa: BLE001 -- unverified discount: price the whole file+ logger.debug("Lazy-read discount skipped in the host preflight: %s", e)free_vram_mib = sum(max(0, row[1]) for row in gpus if row[0] not in shared)heap_free_mib, heap_bytes = self._shared_heap_budget(gpus, shared, model_bytes, argv, _env)offload_bytes = model_bytes - free_vram_mib * 1024 * 1024@@ -15594,6 +15926,8 @@ class LlamaCppBackend:avail_mib: Optional[int] = None,extra_args: Optional[Iterable[str]] = None,env: Optional[Mapping[str, str]] = None,+ lazy_read_bytes: int = 0,+ announce: bool = True,) -> Optional[str]:"""``"none"`` when the fit proves the load needs no demand paging, else None.@@ -15661,6 +15995,9 @@ class LlamaCppBackend:enabled fitter is free to lower it (common/fit.cpp aborts only on a count theuser really set), and a pass-through ``--fit-target`` raises the margin aboveanything ``_ctx_integrity_flags`` emitted.++ ``lazy_read_bytes`` (``_lazy_read_host_bytes``) leaves the footprint: it stays+ mmap'd and pages through the headroom. ``announce`` False silences a what-if call."""from utils.hardware import is_apple_silicon@@ -15706,6 +16043,9 @@ class LlamaCppBackend:+ max(0, pipeline_overhead_bytes)+ max(0, soft_overhead))+ _lazy_discount = max(0, int(lazy_read_bytes or 0))+ if _lazy_discount:+ footprint = max(1, footprint - _lazy_discount)# Unified memory is ONE pool: an APU's free VRAM IS the host RAM the spill# comes out of, so adding both fits a model twice into memory that holds it# once. shared_gpu_ids already names Vulkan iGPUs; the ROCm APUs it cannot see@@ -15739,11 +16079,19 @@ class LlamaCppBackend:vram_margin_mib = fit_margin_mib,avail_mib = avail_mib,):- logger.info(- "Load mode: the whole load (%.1f GB) fits without paging, using --load-mode %s.",- footprint / (1024**3),- self._FIT_LOAD_MODE,- )+ if announce:+ logger.info(+ "Load mode: the whole load (%.1f GB) fits without paging, using "+ "--load-mode %s%s.",+ footprint / (1024**3),+ self._FIT_LOAD_MODE,+ (+ f" ({_lazy_discount / (1024**3):.1f} GB of lazily read tables stay "+ "on disk)"+ if _lazy_discount+ else ""+ ),+ )return self._FIT_LOAD_MODEreturn None@@ -18198,6 +18546,9 @@ class LlamaCppBackend:else:for _ in range(alen):LlamaCppBackend._gguf_skip_value(f, atype)+ else:+ # Its size is unknown, so every later read would be misaligned.+ raise ValueError(f"unknown GGUF value type {vtype}")@staticmethoddef _gguf_read_array_value(f, atype: int, alen: int) -> Optional[list]:@@ -18781,10 +19132,10 @@ class LlamaCppBackend:self._gguf_skip_value(f, vtype)else:self._gguf_skip_value(f, vtype)- except (struct.error, UnicodeDecodeError):+ except (struct.error, UnicodeDecodeError, ValueError):# Truncated input (e.g. HTTP byte-range header- # fetch); break so the resolver fallback runs on- # what we have.+ # fetch), or a value type with no known size; break+ # so the resolver fallback runs on what we have.breakelse:kv_complete = True@@ -23372,6 +23723,61 @@ class LlamaCppBackend:return [*argv[:i], *before, *argv[i + len(raised) :]]return argv+ def _drop_moe_cache(+ self,+ argv: "list[str]",+ why: str = "",+ ) -> "list[str]":+ """``argv`` without the ``--moe-cache-mib auto`` this launch added; a user's is kept."""+ tokens = list(getattr(self, "_moe_cache_flags", None) or ())+ stripped = _without_subsequence(list(argv), tokens)+ if tokens and stripped != list(argv) and why:+ logger.info("MoE cache: dropping --moe-cache-mib auto for the %s.", why)+ return stripped++ def _moe_cache_auto_eligible(+ self,+ *,+ caps: Optional[Mapping[str, object]],+ gpu_memory_mode: str,+ use_fit: bool,+ experts_on_host: bool,+ discrete_gpu: bool,+ gpu_indices: Optional[Iterable[int]],+ detected_gpus: Optional[Iterable[tuple[int, int]]],+ is_vulkan_backend: bool,+ tensor_parallel: bool,+ extra_args: Optional[Iterable[str]],+ env: Optional[Mapping[str, str]] = None,+ ) -> bool:+ """Whether this launch may get ``--moe-cache-mib auto`` (the caller still checks RAM+ and the final load mode): a build with ``auto``, ``--fit on``, spilling experts, one+ discrete CUDA / ROCm GPU (the cache is single-device), and no user placement."""+ if not (caps or {}).get("supports_moe_cache_auto"):+ return False+ if gpu_memory_mode == "manual" or not use_fit or not experts_on_host:+ return False+ if not discrete_gpu or is_vulkan_backend or tensor_parallel or self.n_moe_layers <= 0:+ return False+ devices = (+ {int(i) for i in gpu_indices}+ if gpu_indices is not None+ else {int(idx) for idx, _free in (detected_gpus or ())}+ )+ if len(devices) != 1:+ return False+ source_env = os.environ if env is None else env+ if _extra_args_set_any_flag(extra_args, _MOE_CACHE_USER_OWNED_FLAGS):+ return False+ if any(str(source_env.get(name) or "").strip() for name in _MOE_CACHE_USER_OWNED_ENV):+ return False+ if _env_fixes_gpu_layers(source_env):+ return False+ try:+ return fit_is_effectively_on(list(extra_args or ()), source_env)+ except ValueError: # a valueless --fit: llama-server refuses the launch anyway+ return False+def _discrete_gpu_for_expert_spill(self,gpu_indices: Optional[Iterable[int]],@@ -23442,8 +23848,9 @@ class LlamaCppBackend:replay = self._drop_managed_dio(replay, "the CPU fallback runs entirely from host RAM", clear_record = False)- # Nothing to stream without a GPU: put back the pre-raise batch pair.+ # Nothing to stream without a GPU; the expert cache needs one too.replay = self._undo_moe_spill_batch(replay)+ replay = self._drop_moe_cache(replay, "CPU fallback")# A user's own "--load-mode none" / "--no-mmap" survives that strip, by design,# and on this rung it is no longer the mode they were priced for: the replay# appends "--gpu-layers 0 --fit off --device none", so nothing credits VRAM and@@ -25059,6 +25466,8 @@ class LlamaCppBackend:self._pending_plan_mib = {}# Set by the expert-spill raise after the fit.self._moe_spill_batch_restore: Optional[tuple[Optional[int], Optional[int]]] = None+ _moe_cache_candidate = False+ _moe_cache_ram_ok = False_shared_gpus = frozenset()# Sized inputs for the tensor-spill planner, None when the fit never# priced them. Bound before the try like _placement_verdict_partial:@@ -27387,17 +27796,18 @@ class LlamaCppBackend:and not _device_selection_is_cpu(extra_args, os.environ)and (use_fit or _expert_spill_places_tensors_on_cpu(extra_args, os.environ)))+ _moe_spill_discrete_gpu = bool(+ _moe_experts_on_host+ and self._discrete_gpu_for_expert_spill(+ gpu_indices, _detected_gpus, _shared_gpu_ids+ )+ )_spill_n_batch, _spill_n_ubatch = _moe_spill_batch_ubatch(n_batch,n_ubatch,n_moe_layers = self.n_moe_layers,experts_on_host = _moe_experts_on_host,- discrete_gpu = (- _moe_experts_on_host- and self._discrete_gpu_for_expert_spill(- gpu_indices, _detected_gpus, _shared_gpu_ids- )- ),+ discrete_gpu = _moe_spill_discrete_gpu,user_named_batch = _user_named_batch,)if _spill_n_ubatch != n_ubatch:@@ -27422,6 +27832,20 @@ class LlamaCppBackend:per_device_tensor = True,) or (self._TENSOR_PARALLEL_BUFFER_RESERVE_MIB * 1024 * 1024)+ _moe_cache_candidate = self._moe_cache_auto_eligible(+ caps = server_caps,+ gpu_memory_mode = gpu_memory_mode,+ use_fit = use_fit,+ experts_on_host = _moe_experts_on_host,+ discrete_gpu = _moe_spill_discrete_gpu,+ gpu_indices = gpu_indices,+ detected_gpus = _detected_gpus,+ is_vulkan_backend = is_vulkan_backend,+ tensor_parallel = tensor_parallel,+ extra_args = extra_args,+ env = os.environ,+ )+# Pass the final slot and micro-batch values instead of the defaults# captured before slot reduction._mtp_reserve_bytes = (@@ -27726,7 +28150,20 @@ class LlamaCppBackend:_kv_bytes(effective_ctx, _effective_ctx_checkpoints)- _kv_bytes(effective_ctx, 0),)- _fit_load_mode = self._fit_derived_load_mode(+ # A lazily read table stays mmap'd under every load mode: not pinned RAM.+ _fit_lazy_bytes = self._lazy_read_host_bytes(+ model_path,+ caps = server_caps,+ extra_args = _fit_extras,+ env = _fit_env,+ auto_resolves_off = self._lazy_auto_resolves_off(+ gpu_indices = gpu_indices,+ detected_gpus = _detected_gpus,+ shared_gpu_ids = _shared_gpu_ids,+ is_vulkan_backend = is_vulkan_backend,+ ),+ )+ _fit_load_mode_kwargs = dict(model_size = _fit_model_size,mmproj_pinned_bytes = _mmproj_pinned_bytes+ (_fit_env_mmproj_bytes if _fit_env_mmproj_on_host else 0),@@ -27783,7 +28220,44 @@ class LlamaCppBackend:# launch's own -ngl.extra_args = _fit_extras,env = _fit_env,+ lazy_read_bytes = _fit_lazy_bytes,)+ _fit_load_mode = self._fit_derived_load_mode(**_fit_load_mode_kwargs)+ if _moe_cache_candidate:+ # The cache frees no host RAM: every routed expert plus the prompt+ # cache must fit pinned (moved out of model_size, not counted twice).+ _moe_expert_bytes = (self._gguf_tensor_scan(model_path) or (None, {}, 0))[2]+ _moe_prompt_cache_bytes = _prompt_cache_host_bytes(+ cache_ram,+ _fit_extras,+ _fit_env,+ supports_cache_ram = bool(server_caps.get("supports_cache_ram")),+ )+ # An unknown size (unsized inherited projector) admits nothing.+ _moe_cache_ram_ok = bool(+ _fit_load_mode_kwargs["model_size"]+ and _moe_expert_bytes > 0+ and _moe_prompt_cache_bytes is not None+ and self._fit_derived_load_mode(+ **{+ **_fit_load_mode_kwargs,+ "model_size": max(+ 1, _fit_load_mode_kwargs["model_size"] - _moe_expert_bytes+ ),+ "host_only_bytes": _fit_load_mode_kwargs["host_only_bytes"]+ + _moe_expert_bytes+ + _moe_prompt_cache_bytes,+ "announce": False,+ }+ )+ == self._FIT_LOAD_MODE+ )+ if not _moe_cache_ram_ok:+ logger.info(+ "MoE cache: system RAM cannot hold the %.1f GB of routed "+ "experts pinned; launching without --moe-cache-mib.",+ _moe_expert_bytes / (1024**3),+ )except Exception as e:logger.warning(f"GPU selection failed ({e}), using --fit on")gpu_indices, use_fit = None, True@@ -27793,6 +28267,7 @@ class LlamaCppBackend:_placement_verdict_partial = False# Same reasoning: an unpriced load keeps llama.cpp's default._fit_load_mode = None+ _moe_cache_candidate = _moe_cache_ram_ok = False# And a half-built snapshot is not a price either: the throw can# land after the dict exists but before the later terms are added# to it, which would plan against a footprint missing them.@@ -27901,6 +28376,8 @@ class LlamaCppBackend:# abstains, while the RAM this launch actually loads into cannot hold# them. "none" and "mlock" do not mmap, so that is an OOM kill._apu_ram_oversized = False+ # model_size less a lazily read table; the text-only reprice charges the same.+ _apu_resident_model_size = model_size# The two shortfall notices this load recorded, kept verbatim (and# un-amended) so the text-only fallback further down can re-price them# once it drops the CPU-pinned projector whose bytes they charged. Both@@ -27944,11 +28421,28 @@ class LlamaCppBackend:_apu_ram_part = ("APU" if self._amd_apu_wants_unified_memory(gpu_indices) else "SoC")+ _apu_resident_model_size = max(+ 1,+ model_size+ - self._lazy_read_host_bytes(+ model_path,+ caps = server_caps,+ extra_args = extra_args,+ env = os.environ,+ auto_resolves_off = self._lazy_auto_resolves_off(+ gpu_indices = gpu_indices,+ detected_gpus = _detected_gpus,+ shared_gpu_ids = _shared_gpu_ids,+ is_vulkan_backend = False,+ probe_integrated_cuda = False,+ ),+ ),+ )_ram_msg = self._apu_ram_shortfall_message(# A pinned projector left model_size but not system RAM, and# this guard exists to stop an oversize load being OOM-killed# mid-read, so it has to weigh the projector either way.- model_size + _mmproj_pinned_bytes,+ _apu_resident_model_size + _mmproj_pinned_bytes,_apu_avail_mib,part = _apu_ram_part,)@@ -28216,6 +28710,8 @@ class LlamaCppBackend:# Set when a positional --tensor-split is emitted, so the env block# can pin CUDA to PCI order even without a GPU subset (see below).manual_tensor_split_emitted = False+ # The MoE cache's "auto" sizing runs only under llama.cpp's fitter.+ _placed_by_fitter = Falseif gpu_memory_mode == "manual" and gpu_layers >= 0:# Pin the user's layer count and disable auto-fit. --fit off# also means _ctx_integrity_flags must not add --fit-ctx.@@ -28307,6 +28803,7 @@ class LlamaCppBackend:)else:cmd.extend(["--fit", "on"])+ _placed_by_fitter = Trueelif gpu_indices is not None:# Fits on selected GPU(s) -- force all layers on GPU. --fit off is# required: without it llama.cpp's default --fit on second-guesses@@ -29483,6 +29980,20 @@ class LlamaCppBackend:cmd.extend(_pv_split_mode_pin)if _pv_device_pin:cmd.extend(_pv_device_pin)+ # Pinned loads only: under mmap the cache measured slower than none.+ self._moe_cache_flags = []+ if (+ _moe_cache_candidate+ and _moe_cache_ram_ok+ and _placed_by_fitter+ and _argv_loads_pinned(cmd, _fit_load_mode_env_view)+ ):+ self._moe_cache_flags = list(_MOE_CACHE_AUTO_TOKENS)+ cmd.extend(self._moe_cache_flags)+ logger.info(+ "MoE experts spill to pinned system RAM on one GPU; adding "+ "--moe-cache-mib auto so llama.cpp caches hot experts in VRAM."+ )# A build with no --flash-attn cannot run a quantized V cache# either: llama.cpp aborts init with "V cache quantization@@ -30466,10 +30977,10 @@ class LlamaCppBackend:def _spawn_and_wait(run_cmd, *, label = ""):"""Start llama-server with run_cmd and wait for health.- Up to three launches: the first, one ROCm env correction,- and one --fit recovery. Separate flags, so a library mix- does not spend the fit slot and a VRAM crash after the- correction can still fit-retry.+ Up to four launches: the first, one ROCm env correction, one+ retry without the MoE expert cache, and one --fit recovery.+ Separate flags, so a library mix does not spend the fit slot+ and a VRAM crash after the correction can still fit-retry."""# _mem_host_resident too: the --fit on retry re-arms the# page-lock and writes it back, which without this makes the@@ -30483,9 +30994,15 @@ class LlamaCppBackend:# and projector placement, and each retry changes one of them.if label:run_cmd = self._drop_tensor_spill(run_cmd, label.lstrip("-") or "retry")+ # The cache only rides the first pinned launch: every retry re-places.+ if label or not _argv_loads_pinned(run_cmd, env):+ run_cmd = self._drop_moe_cache(+ run_cmd, f"{label.lstrip('-') or 'unpinned'} launch"+ )_fit_retry_allowed = self._fit_off_retry_eligible(run_cmd, use_fit)_did_fit_retry = False- for _spawn_attempt in (0, 1, 2):+ _did_moe_cache_retry = False+ for _spawn_attempt in (0, 1, 2, 3):# Defensive kill: drop an orphan Popen a concurrent load may# have stored before we overwrite the reference (#5161).# Also reaps the crashed first attempt on the retry pass.@@ -30654,6 +31171,29 @@ class LlamaCppBackend:env["LD_LIBRARY_PATH"] = _retry_ld_did_rocm_retry = Truecontinue+ if (+ not _did_moe_cache_retry+ and _startup_crashed+ and not _capability_crash+ and not _hip_rocr_mismatch+ ):+ # OOM or a cache error with the cache on: same placement without it.+ _uncached = self._drop_moe_cache(run_cmd)+ _crash_text = "\n".join(self._stdout_lines[-200:])+ if _uncached != run_cmd and (+ _MOE_CACHE_ERROR_RE.search(_crash_text)+ or self._is_gpu_memory_start_failure(_crash_text)+ ):+ logger.warning(+ "llama-server crashed during startup (exit code %s) "+ "with --moe-cache-mib auto; retrying once without "+ "the MoE expert cache. Crash log: %s",+ _crashed_proc.returncode,+ self._llama_log_path,+ )+ run_cmd = _uncached+ _did_moe_cache_retry = True+ continueif (not _did_fit_retryand _startup_crashed@@ -31324,8 +31864,25 @@ class LlamaCppBackend:_retry_apu_msg = Noneif model_size is not None and _retry_wants_unified:_apu_avail_mib = self._available_system_memory_mib()+ _apu_resident_model_size = max(+ 1,+ model_size+ - self._lazy_read_host_bytes(+ model_path,+ caps = server_caps,+ extra_args = cmd,+ env = env,+ auto_resolves_off = self._lazy_auto_resolves_off(+ gpu_indices = _remaining,+ detected_gpus = _detected_gpus,+ shared_gpu_ids = _shared_gpu_ids,+ is_vulkan_backend = is_vulkan_backend,+ probe_integrated_cuda = False,+ ),+ ),+ )_retry_apu_msg = self._apu_ram_shortfall_message(- model_size + _mmproj_pinned_bytes,+ _apu_resident_model_size + _mmproj_pinned_bytes,_apu_avail_mib,)# Same contract as the preflight above: the opt-out takes@@ -32151,7 +32708,7 @@ class LlamaCppBackend:self._reprice_after_dropping_pinned_projector(apu_msg = _apu_ram_msg,host_msg = _host_ram_msg,- model_size = model_size,+ model_size = _apu_resident_model_size,pinned_bytes = _mmproj_pinned_bytes,avail_mib = _apu_avail_mib,part = _apu_ram_part,diff --git a/studio/backend/tests/test_llama_cpp_placement.py b/studio/backend/tests/test_llama_cpp_placement.pyindex b6e7a7fc18b..547d17e3f5a 100644--- a/studio/backend/tests/test_llama_cpp_placement.py+++ b/studio/backend/tests/test_llama_cpp_placement.py@@ -4281,3 +4281,870 @@ def test_the_spill_raise_helper_only_raises():assert _moe_spill_batch_ubatch(None, None, **{**on, off: False}) == (None, None)assert _moe_spill_batch_ubatch(None, None, **{**on, "user_named_batch": True}) == (None, None)assert _moe_spill_batch_ubatch(None, None, **{**on, "n_moe_layers": 0}) == (None, None)+++# MoE experts in host RAM: --moe-cache-mib auto, and lazily read tables.++_MOE_CACHE = ["--moe-cache-mib", "auto"]+++def _has_moe_cache(cmd):+ return any(cmd[i : i + 2] == _MOE_CACHE for i in range(len(cmd) - 1))+++def _write_help_binary(tmp_path, name, help_text):+ script = tmp_path / name+ script.write_text("#!/bin/sh\ncat <<'HELP'\n" + help_text + "\nHELP\n")+ script.chmod(0o755)+ return str(script)+++_HELP_BASE = (+ "-m, --model FNAME model path\n"+ "--load-mode MODE how to load the model\n"+)++[email protected](sys.platform == "win32", reason = "shell script stands in for llama-server")+ "extra_help, cache, auto, lazy",+ [+ ("", False, False, False),+ (+ "--moe-cache-mib N GPU cache size in MiB for the MoE experts "+ "kept in the CPU (default: 0, disabled)\n",+ True,+ False,+ False,+ ),+ (+ "--moe-cache-mib N|auto GPU cache size in MiB for the MoE experts "+ "kept in the CPU, or auto to size it from free VRAM (default: 0, disabled)\n"+ "-lzm, --lazy-mode MODE on-demand reading of certain tensors\n",+ True,+ True,+ True,+ ),+ ("--tensor-read-lazy MODE on-demand reading\n", False, False, True),+ ],+ ids = ["old_build", "upstream_cache", "fork_auto", "pre_rename_lazy"],+)+def test_the_probe_reads_the_moe_cache_and_lazy_mode_flags(+ tmp_path, monkeypatch, extra_help, cache, auto, lazy+):+ monkeypatch.setattr(LlamaCppBackend, "_capability_cache", {})+ binary = _write_help_binary(+ tmp_path, f"llama-server-{cache}{auto}{lazy}", _HELP_BASE + extra_help+ )+ caps = LlamaCppBackend.probe_server_capabilities(binary)++ assert caps["help_probe_ok"] is True+ assert (+ caps["supports_moe_cache"],+ caps["supports_moe_cache_auto"],+ caps["supports_lazy_mode"],+ ) == (+ cache,+ auto,+ lazy,+ )+ assert caps["lazy_mode_flag"] == (+ ("--tensor-read-lazy" if "--tensor-read-lazy" in extra_help else "--lazy-mode")+ if lazy+ else None+ )+++def test_a_failed_probe_advertises_no_moe_cache():+ caps = LlamaCppBackend.probe_server_capabilities("/nonexistent/llama-server")+ assert not caps["supports_moe_cache_auto"] and not caps["supports_lazy_mode"]+++_CACHE_CAPS_ON = dict(+ supports_load_mode = True,+ # The 8 GiB --cache-ram default is charged to the cache's RAM admission.+ supports_cache_ram = True,+ supports_moe_cache = True,+ supports_moe_cache_auto = True,+ supports_lazy_mode = True,+)+++def _moe_cache_host(monkeypatch, _discrete_linux_host):+ import utils.hardware as hardware+ import utils.model_memory_settings as mm++ # A discrete CUDA host on every runner: macOS CI is Apple Silicon.+ monkeypatch.setattr(hardware, "is_apple_silicon", lambda: False)++ monkeypatch.setattr(mm, "get_model_memory_settings", lambda: (False, False))+ monkeypatch.setattr(mm, "get_keep_resident", lambda: False)+ monkeypatch.setattr(mm, "get_no_ram_reserve", lambda: False)+ monkeypatch.setattr(mm, "should_mlock", lambda: False)+ for name in (+ "LLAMA_ARG_MOE_CACHE_MIB",+ "LLAMA_ARG_OVERRIDE_TENSOR",+ "LLAMA_ARG_N_GPU_LAYERS",+ "LLAMA_ARG_DEVICE",+ "LLAMA_ARG_FIT",+ "LLAMA_ARG_LOAD_MODE",+ "LLAMA_ARG_MMAP",+ "LLAMA_ARG_NO_MMAP",+ "LLAMA_ARG_LAZY_MODE",+ "LLAMA_ARG_CACHE_RAM",+ "LLAMA_ARG_N_CPU_FFN",+ "LLAMA_ARG_RPC",+ ):+ monkeypatch.delenv(name, raising = False)+++def _cache_launch(+ tmp_path,+ *,+ caps = None,+ moe = True,+ spilled = True,+ gpus = 1,+ ram_gib = 256,+ expert_gib = 15,+ lazy = None,+ arch = "qwen3moe",+ size_gib = None,+ memory = None,+ host_guard = False,+ **load_kwargs,+):+ """One launch of an MoE (or dense) model, priced so the fit can answer+ "none". 20 GiB on an 8 GiB card spills; 1 GiB on a 40 GiB card does not.+ ``host_guard`` puts the real host-RAM preflight back."""+ if memory is None:+ memory = (+ [(i, 8_000, 16_000) for i in range(gpus)]+ if spilled+ else [(i, 40_000, 48_000) for i in range(gpus)]+ )+ if size_gib is None:+ size_gib = 20 if spilled else 1+ backend, gguf = _moe_backend(tmp_path, size_gib = size_gib, memory = memory, moe = moe)+ backend._can_estimate_kv = lambda: True+ backend._estimate_kv_cache_bytes = lambda *a, **k: _GIB // 4+ backend._available_system_memory_mib = lambda: int(ram_gib * 1024)+ backend._gguf_tensor_scan = lambda _path: (+ arch,+ dict(lazy or {}),+ int(expert_gib * _GIB) if moe else 0,+ )+ full_caps = dict(LlamaCppBackend.probe_server_capabilities.__func__(LlamaCppBackend, None))+ full_caps.update(_CACHE_CAPS_ON if caps is None else caps)+ backend.probe_server_capabilities = lambda _binary = None: full_caps+ if host_guard:+ _restore_host_guard(backend)+ cmd = _launch(backend, gguf, **load_kwargs)["cmd"]+ return backend, cmd+++def _load_mode(cmd):+ values = [cmd[i + 1] for i, tok in enumerate(cmd) if tok == "--load-mode"]+ return values[-1] if values else None+++def test_a_spilled_moe_on_one_gpu_loading_pinned_gets_the_cache(tmp_path, _moe_cache_host):+ backend, cmd = _cache_launch(tmp_path)++ assert cmd[cmd.index("--fit") + 1] == "on", cmd+ assert _load_mode(cmd) == "none", cmd+ assert _has_moe_cache(cmd), cmd+ assert cmd.count("--moe-cache-mib") == 1+ # Next to the expert-spill micro-batch, which it never replaces.+ assert _ubatch_values(cmd) == ["2048"], cmd+ assert backend._moe_cache_flags == _MOE_CACHE+++ "cell, kwargs",+ [+ ("dense", dict(moe = False)),+ ("no_spill", dict(spilled = False)),+ ("two_gpus", dict(gpus = 2)),+ # 15 GiB experts + 8 GiB prompt cache > 16 GiB RAM; the 20 GiB load alone fits.+ ("ram_cannot_pin_experts", dict(ram_gib = 24)),+ ("ram_cannot_pin_anything", dict(ram_gib = 8)),+ (+ "old_build",+ dict(+ caps = dict(_CACHE_CAPS_ON, supports_moe_cache = False, supports_moe_cache_auto = False)+ ),+ ),+ ("upstream_build_no_auto", dict(caps = dict(_CACHE_CAPS_ON, supports_moe_cache_auto = False))),+ ("user_mmap", dict(load_mode = "mmap")),+ ("user_mmap_extra", dict(extra_args = ["--load-mode", "mmap"])),+ ("manual_layers", dict(gpu_memory_mode = "manual", gpu_layers = 40)),+ ],+ ids = lambda v: v if isinstance(v, str) else "",+)+def test_the_moe_cache_stays_off(tmp_path, _moe_cache_host, cell, kwargs):+ backend, cmd = _cache_launch(tmp_path, **kwargs)++ assert not _has_moe_cache(cmd) and "--moe-cache-mib" not in cmd, (cell, cmd)+ assert backend._moe_cache_flags == []+ if cell == "ram_cannot_pin_experts":+ # Pinned without the cache: the cache never demotes a pinned load.+ assert _load_mode(cmd) == "none", cmd+ if cell == "ram_cannot_pin_anything":+ assert _load_mode(cmd) is None, cmd+++ "extra_args",+ [+ ["-ot", r"blk\.\d+\.ffn_.*_exps\.=CPU"],+ ["--override-tensor", "exps=CUDA0"],+ ["-ngl", "30"],+ ["--n-gpu-layers", "99"],+ ["-cmoe"],+ ["--cpu-moe"],+ ["--n-cpu-moe", "10"],+ ["--moe-cache-mib", "4096"],+ ["--fit", "off"],+ ["--device", "CUDA0"],+ ["--tensor-split", "1,0"],+ ["-ncffn", "1"],+ ["--n-cpu-ffn=1"],+ ["--rpc", "127.0.0.1:50052"],+ ],+ ids = [+ "ot",+ "override_tensor",+ "ngl",+ "n_gpu_layers",+ "cmoe",+ "cpu_moe",+ "n_cpu_moe",+ "user_cache",+ "fit_off",+ "device",+ "tensor_split",+ "ncffn",+ "n_cpu_ffn_equals",+ "rpc",+ ],+)+def test_a_user_placement_flag_keeps_the_moe_cache_off(tmp_path, _moe_cache_host, extra_args):+ _backend, cmd = _cache_launch(tmp_path, extra_args = extra_args)++ assert not _has_moe_cache(cmd), cmd+ if "--moe-cache-mib" in extra_args:+ assert cmd.count("--moe-cache-mib") == 1 and cmd[cmd.index("--moe-cache-mib") + 1] == "4096"+++ "name, value",+ [+ ("LLAMA_ARG_MOE_CACHE_MIB", "2048"),+ ("LLAMA_ARG_N_GPU_LAYERS", "20"),+ ("LLAMA_ARG_OVERRIDE_TENSOR", "exps=CPU"),+ ("LLAMA_ARG_FIT", "off"),+ ("LLAMA_ARG_N_CPU_FFN", "1"),+ ("LLAMA_ARG_RPC", "127.0.0.1:50052"),+ ],+)+def test_an_inherited_placement_keeps_the_moe_cache_off(+ tmp_path, _moe_cache_host, monkeypatch, name, value+):+ monkeypatch.setenv(name, value)+ _backend, cmd = _cache_launch(tmp_path)++ assert not _has_moe_cache(cmd), cmd+++def test_an_unsized_inherited_projector_skips_the_cache_not_the_placement(+ tmp_path, _moe_cache_host, monkeypatch+):+ # An unsized LLAMA_ARG_MMPROJ_URL leaves the fit's model size unknown.+ monkeypatch.setenv("LLAMA_ARG_MMPROJ_URL", "https://example.invalid/mmproj.gguf")+ warnings = _override_log(monkeypatch)+ backend, cmd = _cache_launch(tmp_path)++ assert not any("GPU selection failed" in line for line in warnings), warnings+ assert not _has_moe_cache(cmd) and backend._moe_cache_flags == []+++def test_the_moe_cache_eligibility_gates(tmp_path, _moe_cache_host):+ backend, _gguf = _moe_backend(tmp_path, **_SPILLED)+ on = dict(+ caps = _CACHE_CAPS_ON,+ gpu_memory_mode = "auto",+ use_fit = True,+ experts_on_host = True,+ discrete_gpu = True,+ gpu_indices = [0],+ detected_gpus = [(0, 8_000), (1, 8_000)],+ is_vulkan_backend = False,+ tensor_parallel = False,+ extra_args = [],+ env = {},+ )+ assert backend._moe_cache_auto_eligible(**on) is True+ for off in (+ dict(use_fit = False),+ dict(experts_on_host = False),+ dict(discrete_gpu = False),+ dict(is_vulkan_backend = True),+ dict(tensor_parallel = True),+ dict(gpu_memory_mode = "manual"),+ dict(gpu_indices = [0, 1]),+ dict(gpu_indices = None),+ dict(caps = {}),+ dict(extra_args = ["--fit", "off"]),+ dict(env = {"LLAMA_ARG_FIT": "off"}),+ ):+ assert backend._moe_cache_auto_eligible(**{**on, **off}) is False, off+ assert backend._moe_cache_auto_eligible(+ **{**on, "gpu_indices": None, "detected_gpus": [(0, 8_000)]}+ )+ backend._n_experts = None+ assert backend._moe_cache_auto_eligible(**on) is False+++def test_every_retry_drops_the_moe_cache_first():+ backend = LlamaCppBackend()+ argv = ["llama-server", "-m", "x.gguf", "--fit", "on", "--load-mode", "none", *_MOE_CACHE]+ # Not this launch's: a user's identical flag is theirs.+ assert backend._drop_moe_cache(argv, "test") == argv++ backend._moe_cache_flags = list(_MOE_CACHE)+ assert backend._drop_moe_cache(argv, "test") == argv[:-2]+ # The CPU replay hands back the uncached argv as well.+ assert backend._drop_moe_cache(argv[:-2], "test") == argv[:-2]+++ "crash_line, retried",+ [+ (+ "llama_init_from_model: failed to initialize the context: MoE cache is too small to hold the experts of one token",+ True,+ ),+ (+ "ggml_backend_cuda_buffer_type_alloc_buffer: allocating 4096.00 MiB on device 0: cudaMalloc failed: out of memory",+ True,+ ),+ ("error: unknown model architecture: 'foo'", False),+ # Logged on every healthy cache setup: a later failure is not the cache's.+ (+ "common_fit_params: moe cache auto: 12288 MiB, 96 experts resident\n"+ "llama_moe_cache_init: MoE cache size = 12288.00 MiB\n"+ "error: unknown model architecture: 'foo'",+ False,+ ),+ (+ "llama_moe_cache_init: MoE cache size = 12288.00 MiB\n"+ "llama-moe-cache.cpp:412: GGML_ABORT: the MoE cache is too small for the "+ "experts selected in layer 7",+ True,+ ),+ ("failed to allocate the MoE cache buffers", True),+ ],+ ids = ["cache_error", "cuda_oom", "unrelated", "info_lines_then_unrelated", "abort", "alloc"],+)+def test_a_cache_crash_retries_once_without_the_cache(+ tmp_path, _moe_cache_host, crash_line, retried+):+ spawned = []++ def fake_popen(cmd, **kwargs):+ if not cmd or str(cmd[0]) != "/fake/llama-server":+ return _REAL_POPEN(cmd, **kwargs)+ spawned.append(list(cmd))+ crashed = len(spawned) == 1+ return type(+ "Process",+ (),+ {+ "pid": 100 + len(spawned),+ "stdout": iter([crash_line + "\n"]) if crashed else iter(()),+ "returncode": 1 if crashed else None,+ "poll": lambda self: 1 if crashed else None,+ "terminate": lambda self: None,+ "wait": lambda self, timeout = None: 1 if crashed else 0,+ "kill": lambda self: None,+ },+ )()++ backend, gguf = _moe_backend(tmp_path, **_SPILLED)+ backend._can_estimate_kv = lambda: True+ backend._estimate_kv_cache_bytes = lambda *a, **k: _GIB // 4+ backend._available_system_memory_mib = lambda: 256 * 1024+ backend._gguf_tensor_scan = lambda _path: ("qwen3moe", {}, 15 * _GIB)+ full_caps = dict(LlamaCppBackend.probe_server_capabilities.__func__(LlamaCppBackend, None))+ full_caps.update(_CACHE_CAPS_ON)+ backend.probe_server_capabilities = lambda _binary = None: full_caps++ def health(timeout, **_kw):+ if len(spawned) == 1:+ backend._stdout_lines = [crash_line]+ return False+ return True++ backend._wait_for_health = health+ with patch.object(subprocess, "Popen", side_effect = fake_popen):+ try:+ backend.load_model(GgufLoadIntent(gguf_path = str(gguf), model_identifier = "test"))+ except Exception:+ pass++ assert _has_moe_cache(spawned[0]), spawned[0]+ if retried:+ assert len(spawned) >= 2 and not _has_moe_cache(spawned[1]), spawned+ assert _without_cache(spawned[0]) == spawned[1]+ else:+ assert len(spawned) < 2 or spawned[1] != _without_cache(spawned[0])+++def _without_cache(cmd):+ for i in range(len(cmd) - 1):+ if cmd[i : i + 2] == _MOE_CACHE:+ return cmd[:i] + cmd[i + 2 :]+ return list(cmd)+++# Qwen3.8-Flash-Next IQ1_S (arch qwen4exp), sizes from its GGUF header.+_QWEN38_TOTAL = int(67.55 * _GIB)+_QWEN38_PLE = int(26.82 * _GIB) # per_layer_token_embd.weight, TENSOR_READ_LAZY+_QWEN38_EXPERTS = int(37.11 * _GIB)+++class _RamStub:+ """Only what the load-mode predicate touches."""++ def __init__(self, avail_mib):+ self._avail_mib = avail_mib++ def _available_system_memory_mib(self):+ return self._avail_mib++ def _amd_apu_wants_unified_memory(self, gpu_indices = None):+ return False++ _fits_without_paging = LlamaCppBackend._fits_without_paging+ _FIT_LOAD_MODE = LlamaCppBackend._FIT_LOAD_MODE+++def _qwen38_mode(+ lazy_bytes,+ host_only_bytes = 0,+ monkeypatch = None,+):+ # 16 GiB card (15.5 free, 1 GiB margin), 30 GiB RAM available, 0.5 GiB KV + compute.+ return LlamaCppBackend._fit_derived_load_mode(+ _RamStub(30 * 1024),+ model_size = _QWEN38_TOTAL,+ kv_cache_bytes = _GIB // 2,+ compute_buffer_flat = _GIB // 2,+ host_only_bytes = host_only_bytes,+ gpus = [(0, int(15.5 * 1024))],+ gpu_indices = [0],+ fit_margin_mib = 1024,+ lazy_read_bytes = lazy_bytes,+ extra_args = [],+ env = {},+ )+++def test_a_lazily_read_table_is_not_pinned_ram(monkeypatch):+ import utils.hardware as hardware++ monkeypatch.setattr(hardware, "is_apple_silicon", lambda: False)+ # 67.55 + 1 GiB against 14.5 GiB of VRAM leaves 54 GiB for 28 GiB of RAM: mmap.+ assert _qwen38_mode(0) is None+ # Without the 26.82 GiB table: 41.7 GiB, 27.2 GiB of RAM, which fits pinned.+ assert _qwen38_mode(_QWEN38_PLE) == LlamaCppBackend._FIT_LOAD_MODE+ # The cache needs all 37.11 GiB of experts pinned: pinned without it.+ assert _qwen38_mode(_QWEN38_PLE, host_only_bytes = _QWEN38_EXPERTS) is None+++ "arch, size, extra_args, env, expected",+ [+ ("qwen4exp", _QWEN38_PLE, [], {}, _QWEN38_PLE),+ ("gemma4", _QWEN38_PLE, [], {}, _QWEN38_PLE),+ # Another architecture's table of the same name is read like any tensor.+ ("qwen3moe", _QWEN38_PLE, [], {}, 0),+ # At or under 4 GiB, auto loads it normally; "on" reads it lazily anyway.+ ("qwen4exp", 4 * _GIB, [], {}, 0),+ ("qwen4exp", 4 * _GIB, ["-lzm", "on"], {}, 4 * _GIB),+ ("qwen4exp", _QWEN38_PLE, ["-lzm", "off"], {}, 0),+ ("qwen4exp", _QWEN38_PLE, ["--lazy-mode=off"], {}, 0),+ ("qwen4exp", _QWEN38_PLE, [], {"LLAMA_ARG_LAZY_MODE": "off"}, 0),+ # Last wins over the env twin.+ (+ "qwen4exp",+ _QWEN38_PLE,+ ["--lazy-mode", "auto"],+ {"LLAMA_ARG_LAZY_MODE": "off"},+ _QWEN38_PLE,+ ),+ ],+ ids = [+ "qwen4exp",+ "gemma4",+ "other_arch",+ "under_4gib",+ "on",+ "lzm_off",+ "inline_off",+ "env_off",+ "argv_beats_env",+ ],+)+def test_lazy_table_bytes(arch, size, extra_args, env, expected):+ backend = LlamaCppBackend()+ backend._gguf_tensor_scan = lambda _path: (arch, {"per_layer_token_embd.weight": size}, 0)+ caps = {"supports_lazy_mode": True}+ got = backend._lazy_read_host_bytes("/m.gguf", caps = caps, extra_args = extra_args, env = env)+ assert got == expected+ # A build without lazy reads loads every tensor.+ assert backend._lazy_read_host_bytes("/m.gguf", caps = {}, extra_args = extra_args, env = env) == 0+++def _write_tensor_gguf(path, architecture, tensors):+ """A GGUF with real tensor infos: (name, dims, ggml_type) each, F32 data laid out+ back to back."""++ def string(value):+ data = value.encode()+ return struct.pack("<Q", len(data)) + data++ header = struct.pack("<IIQQ", 0x46554747, 3, len(tensors), 1)+ header += string("general.architecture") + struct.pack("<I", 8) + string(architecture)+ offset = 0+ for name, dims, ggml_type in tensors:+ header += string(name) + struct.pack("<I", len(dims)) + struct.pack(f"<{len(dims)}Q", *dims)+ header += struct.pack("<I", ggml_type) + struct.pack("<Q", offset)+ n = 1+ for d in dims:+ n *= d+ offset += -(-n * 4 // 32) * 32+ header += b"\0" * (-len(header) % 32)+ path.write_bytes(header + b"\0" * offset)+ return path+++def test_the_header_scan_sums_shards(tmp_path):+ f32 = 0+ _write_tensor_gguf(+ tmp_path / "m-00001-of-00002.gguf",+ "qwen4exp",+ [+ ("per_layer_token_embd.weight", (8, 4), f32),+ ("blk.0.ffn_up_exps.weight", (4, 4, 2), f32),+ ],+ )+ _write_tensor_gguf(+ tmp_path / "m-00002-of-00002.gguf",+ "",+ [("blk.1.ffn_down_exps.weight", (4, 4, 2), f32), ("blk.1.attn_q.weight", (4, 4), f32)],+ )+ backend = LlamaCppBackend()+ arch, named, experts = backend._gguf_tensor_scan(str(tmp_path / "m-00001-of-00002.gguf"))++ assert arch == "qwen4exp"+ assert named == {"per_layer_token_embd.weight": 8 * 4 * 4}+ assert experts == 2 * 4 * 4 * 2 * 4+ # A missing shard reads as unknown, which discounts nothing.+ (tmp_path / "m-00002-of-00002.gguf").unlink()+ assert LlamaCppBackend()._gguf_tensor_scan(str(tmp_path / "m-00001-of-00002.gguf")) is None+++def test_the_launch_discounts_the_lazy_table(tmp_path, _moe_cache_host):+ """The same predicate through the launch: a 20 GiB MoE whose 12 GiB n-gram table+ is lazy fits 8 GiB of VRAM plus 12 GiB of RAM only without the table."""+ plain = _cache_launch(+ tmp_path, ram_gib = 12, caps = dict(_CACHE_CAPS_ON, supports_moe_cache_auto = False)+ )[1]+ assert _load_mode(plain) is None, plain++ lazy = {"per_layer_token_embd.weight": 12 * _GIB}+ no_auto = dict(_CACHE_CAPS_ON, supports_moe_cache_auto = False)+ _b, cmd = _cache_launch(tmp_path, ram_gib = 12, lazy = lazy, caps = no_auto)+ assert _load_mode(cmd) is None, "qwen3moe does not mark the table lazy"++ _b, cmd = _cache_launch(tmp_path, ram_gib = 12, lazy = lazy, arch = "qwen4exp", caps = no_auto)+ assert _load_mode(cmd) == "none", cmd+ # And "-lzm off" makes it an ordinary tensor again.+ _b, cmd = _cache_launch(+ tmp_path, ram_gib = 12, lazy = lazy, arch = "qwen4exp", caps = no_auto, extra_args = ["-lzm", "off"]+ )+ assert _load_mode(cmd) is None, cmd++[email protected]("ram_gib, cached", [(60, True), (40, False)])+def test_the_cache_admission_counts_the_experts_once(tmp_path, _moe_cache_host, ram_gib, cached):+ """Qwen3.8-Flash-Next IQ1_S on one 24 GiB card. With the lazy table on disk the+ load is ~41 GiB; the cache needs the 37.11 GiB of experts plus the 8 GiB prompt+ cache pinned in RAM, which 60 GiB holds and 40 GiB does not. Charging the experts+ to RAM on top of model_size, as well as in it, refused the 60 GiB host."""+ backend, cmd = _cache_launch(+ tmp_path,+ size_gib = _QWEN38_TOTAL / _GIB,+ memory = [(0, 24_000, 24_576)],+ ram_gib = ram_gib,+ expert_gib = _QWEN38_EXPERTS / _GIB,+ lazy = {"per_layer_token_embd.weight": _QWEN38_PLE},+ arch = "qwen4exp",+ )++ # Pinned either way: the cache never demotes a pinned load.+ assert _load_mode(cmd) == "none", cmd+ assert _has_moe_cache(cmd) is cached, cmd+ assert backend._moe_cache_flags == (_MOE_CACHE if cached else [])++[email protected]("ram_gib, cached", [(45, False), (60, True)])+def test_qwen38_on_a_16gib_card_launches_pinned_past_the_host_guard(+ tmp_path, _moe_cache_host, ram_gib, cached+):+ """Qwen3.8-Flash-Next IQ1_S on one 16 GiB card, end to end with the real host-RAM+ preflight. The fit leaves the 26.82 GiB lazy table out and picks "none"; the+ preflight charged the whole 67.55 GiB file, read a shortfall, and the pageable+ rewrite took the pinned mode (and with it the cache) back out."""+ backend, cmd = _cache_launch(+ tmp_path,+ size_gib = _QWEN38_TOTAL / _GIB,+ memory = [(0, 15_500, 16_384)],+ ram_gib = ram_gib,+ expert_gib = _QWEN38_EXPERTS / _GIB,+ lazy = {"per_layer_token_embd.weight": _QWEN38_PLE},+ arch = "qwen4exp",+ host_guard = True,+ )++ assert _load_mode(cmd) == "none", cmd+ assert _has_moe_cache(cmd) is cached, cmd+ assert backend.last_load_warning is None, backend.last_load_warning+++def test_the_host_guard_still_charges_a_table_read_in_full(tmp_path, _moe_cache_host):+ """Under "-lzm off" the table is an ordinary tensor: the same host is short."""+ _backend, cmd = _cache_launch(+ tmp_path,+ size_gib = _QWEN38_TOTAL / _GIB,+ memory = [(0, 15_500, 16_384)],+ ram_gib = 45,+ expert_gib = _QWEN38_EXPERTS / _GIB,+ lazy = {"per_layer_token_embd.weight": _QWEN38_PLE},+ arch = "qwen4exp",+ host_guard = True,+ extra_args = ["-lzm", "off"],+ )++ assert _load_mode(cmd) is None and not _has_moe_cache(cmd), cmd+++def test_an_igpu_only_child_gets_no_auto_lazy_discount(tmp_path, _moe_cache_host, monkeypatch):+ """llama.cpp turns lazy auto off on a device without mmap support, so the fit that+ pins the lazy table's model on a discrete card leaves it mapped here."""+ monkeypatch.setattr(LlamaCppBackend, "_lazy_auto_resolves_off", lambda self, **_kw: True)+ lazy = {"per_layer_token_embd.weight": 12 * _GIB}+ no_auto = dict(_CACHE_CAPS_ON, supports_moe_cache_auto = False)++ _b, cmd = _cache_launch(tmp_path, ram_gib = 12, lazy = lazy, arch = "qwen4exp", caps = no_auto)+ assert _load_mode(cmd) is None, cmd+ # An explicit "on" is read lazily on any device.+ _b, cmd = _cache_launch(+ tmp_path, ram_gib = 12, lazy = lazy, arch = "qwen4exp", caps = no_auto, extra_args = ["-lzm", "on"]+ )+ assert _load_mode(cmd) == "none", cmd+++ "vulkan, gpu_indices, shared, rocm_unified, cuda_integrated, expected",+ [+ (False, [0], set(), set(), set(), False),+ (True, [0], {0}, set(), set(), True),+ # llama.cpp drops an iGPU from its default list once a discrete GPU is seen.+ (True, None, {1}, set(), set(), False),+ (False, [0], set(), {0}, set(), True),+ (False, [0, 1], set(), {0}, set(), False),+ (False, [0], set(), set(), {0}, True),+ (False, None, set(), set(), {0, 1}, True),+ ],+ ids = [+ "discrete",+ "vulkan_igpu",+ "vulkan_mixed",+ "rocm_apu",+ "rocm_mixed",+ "cuda_soc",+ "cuda_soc_unpinned",+ ],+)+def test_lazy_auto_resolves_off_only_where_llama_cpp_does(+ monkeypatch, vulkan, gpu_indices, shared, rocm_unified, cuda_integrated, expected+):+ monkeypatch.setattr(+ LlamaCppBackend, "_rocm_unified_memory_gpu_ids", staticmethod(lambda: set(rocm_unified))+ )+ monkeypatch.setattr(+ LlamaCppBackend, "_integrated_cuda_gpu_ids", staticmethod(lambda: set(cuda_integrated))+ )+ monkeypatch.setattr(+ LlamaCppBackend, "_resolve_visible_physical_ids", staticmethod(lambda: [0, 1])+ )+ got = LlamaCppBackend()._lazy_auto_resolves_off(+ gpu_indices = gpu_indices,+ detected_gpus = [(0, 8_000), (1, 8_000)],+ shared_gpu_ids = shared,+ is_vulkan_backend = vulkan,+ )+ assert got is expected++ backend = LlamaCppBackend()+ backend._gguf_tensor_scan = lambda _path: (+ "qwen4exp",+ {"per_layer_token_embd.weight": _QWEN38_PLE},+ 0,+ )+ caps = {"supports_lazy_mode": True}+ auto = backend._lazy_read_host_bytes("/m.gguf", caps = caps, env = {}, auto_resolves_off = got)+ on = backend._lazy_read_host_bytes(+ "/m.gguf", caps = caps, extra_args = ["-lzm", "on"], env = {}, auto_resolves_off = got+ )+ assert (auto, on) == ((0 if expected else _QWEN38_PLE), _QWEN38_PLE)++[email protected]("lazy_args, warned", [(["-lzm", "on"], False), ([], True)])+def test_the_apu_guard_prices_only_resident_weights(tmp_path, monkeypatch, lazy_args, warned):+ """A 64.6 GiB model with a 26.82 GiB table on an APU with 46 GiB of RAM. "-lzm on"+ leaves the table on disk, so the unmapped load fits; auto is off on an APU, so the+ whole file loads and the guard still warns and remaps."""+ backend, gguf = _apu_backend(+ tmp_path, gguf_gb = 64.6, avail_mib = 46 * 1024, monkeypatch = monkeypatch+ )+ monkeypatch.setattr(LlamaCppBackend, "_rocm_unified_memory_gpu_ids", staticmethod(lambda: {0}))+ backend._gguf_tensor_scan = lambda _path: (+ "qwen4exp",+ {"per_layer_token_embd.weight": _QWEN38_PLE},+ 0,+ )+ caps = dict(LlamaCppBackend.probe_server_capabilities.__func__(LlamaCppBackend, None))+ caps.update(_CACHE_CAPS_ON)+ backend.probe_server_capabilities = lambda _binary = None: caps++ cmd = _launch(backend, gguf, extra_args = ["--load-mode", "none", *lazy_args])["cmd"]++ assert ("unified-memory APU" in (backend.last_load_warning or "")) is warned+ assert bool(_unmapped_tokens(cmd)) is not warned, cmd+++ "flag, extra_args, env, expected",+ [+ ("--lazy-mode", ["-lzm", "off"], {}, 0),+ ("--lazy-mode", [], {"LLAMA_ARG_LAZY_MODE": "off"}, 0),+ # The old spelling's env twin is not read by a renamed build.+ ("--lazy-mode", [], {"LLAMA_ARG_TENSOR_READ_LAZY": "off"}, _QWEN38_PLE),+ ("--tensor-read-lazy", ["--tensor-read-lazy", "off"], {}, 0),+ ("--tensor-read-lazy", [], {"LLAMA_ARG_TENSOR_READ_LAZY": "off"}, 0),+ # Nor the new one's by a build from before the rename.+ ("--tensor-read-lazy", [], {"LLAMA_ARG_LAZY_MODE": "off"}, _QWEN38_PLE),+ ],+ ids = [+ "new_flag",+ "new_env",+ "new_ignores_old_env",+ "old_flag",+ "old_env",+ "old_ignores_new_env",+ ],+)+def test_only_the_probed_lazy_spelling_is_honoured(flag, extra_args, env, expected):+ backend = LlamaCppBackend()+ backend._gguf_tensor_scan = lambda _path: (+ "qwen4exp",+ {"per_layer_token_embd.weight": _QWEN38_PLE},+ 0,+ )+ caps = {"supports_lazy_mode": True, "lazy_mode_flag": flag}+ got = backend._lazy_read_host_bytes("/m.gguf", caps = caps, extra_args = extra_args, env = env)+ assert got == expected+++def _write_writer_gguf(+ path,+ tensors,+ *,+ split_max_tensors = 0,+):+ """``tensors``: (name, numpy array, raw ggml type or None) each, written by+ gguf-py's own writer, split llama.cpp-style when ``split_max_tensors`` is set."""+ import gguf++ writer = gguf.GGUFWriter(path, "qwen4exp", split_max_tensors = split_max_tensors)+ writer.add_string("tokenizer.ggml.model", "gpt2")+ writer.add_array("tokenizer.ggml.tokens", ["a", "b", "<c>"])+ for name, array, raw_dtype in tensors:+ writer.add_tensor(name, array, raw_dtype = raw_dtype)+ writer.write_header_to_file()+ writer.write_kv_data_to_file()+ writer.write_tensors_to_file()+ writer.close()+++def test_the_header_scan_reads_writer_output(tmp_path, monkeypatch):+ np = pytest.importorskip("numpy")+ gguf = pytest.importorskip("gguf")+ from gguf.constants import GGML_QUANT_SIZES, GGMLQuantizationType++ q8 = GGMLQuantizationType.Q8_0+ q8_block, q8_bytes = GGML_QUANT_SIZES[q8]+ # Q8_0 rows of 64 elements: two blocks each, stored as raw bytes.+ q8_rows = np.zeros((4, 2 * q8_bytes), dtype = np.uint8)+ tensors = [+ ("per_layer_token_embd.weight", np.zeros((8, 16), dtype = np.float32), None),+ ("blk.0.ffn_up_exps.weight", np.zeros((2, 4, 32), dtype = np.float16), None),+ ("blk.0.attn_q.weight", np.zeros((4, 32), dtype = np.float32), None),+ ("blk.1.ffn_down_exps.weight", q8_rows, q8),+ ]+ _write_writer_gguf(tmp_path / "m.gguf", tensors, split_max_tensors = 2)+ shards = sorted(p.name for p in tmp_path.glob("*.gguf"))+ assert shards == ["m-00001-of-00002.gguf", "m-00002-of-00002.gguf"], shards+ first = str(tmp_path / shards[0])+ expected_experts = 2 * 4 * 32 * 2 + 4 * 64 // q8_block * q8_bytes++ arch, named, experts = LlamaCppBackend()._gguf_tensor_scan(first)+ assert arch == "qwen4exp"+ assert named == {"per_layer_token_embd.weight": 8 * 16 * 4}+ assert experts == expected_experts++ # Unknown quant type: sized from the next offset / end of file, within one alignment.+ monkeypatch.delitem(GGML_QUANT_SIZES, q8)+ monkeypatch.delitem(GGML_QUANT_SIZES, GGMLQuantizationType.F16)+ _arch, named, experts = LlamaCppBackend()._gguf_tensor_scan(first)+ assert named == {"per_layer_token_embd.weight": 8 * 16 * 4}+ assert expected_experts <= experts < expected_experts + 2 * gguf.GGUF_DEFAULT_ALIGNMENT++ # A missing shard reads as unknown, which discounts nothing.+ (tmp_path / shards[1]).unlink()+ assert LlamaCppBackend()._gguf_tensor_scan(first) is None+++def test_the_header_scan_refuses_what_it_cannot_parse(tmp_path):+ path = _write_tensor_gguf(+ tmp_path / "v1.gguf", "qwen4exp", [("per_layer_token_embd.weight", (8, 4), 0)]+ )+ data = bytearray(path.read_bytes())+ # GGUF v1 used 32-bit counts: nothing is read from it.+ data[4:8] = struct.pack("<I", 1)+ path.write_bytes(bytes(data))+ assert LlamaCppBackend._gguf_scan_tensor_bytes(str(path)) == (None, {}, 0)++ # An unknown KV value type has no size to skip, so the scan gives up.+ header = struct.pack("<IIQQ", 0x46554747, 3, 0, 1)+ header += struct.pack("<Q", 3) + b"odd" + struct.pack("<I", 99) + b"\0" * 8+ bad = tmp_path / "bad.gguf"+ bad.write_bytes(header)+ with pytest.raises(ValueError):+ LlamaCppBackend._gguf_scan_tensor_bytes(str(bad))+ assert LlamaCppBackend()._gguf_tensor_scan(str(bad)) is Nonediff --git a/studio/backend/tests/test_llama_cpp_single_sequence_retry.py b/studio/backend/tests/test_llama_cpp_single_sequence_retry.pyindex e0ec554d1df..d3f9337d21c 100644--- a/studio/backend/tests/test_llama_cpp_single_sequence_retry.py+++ b/studio/backend/tests/test_llama_cpp_single_sequence_retry.py@@ -155,8 +155,9 @@ def test_the_fit_recovery_rungs_stand_down_for_this_refusal():src = inspect.getsource(LlamaCppBackend.load_model)assert "_capability_crash = _tensor_capability_crash or self._is_kv_unified_refused" in src- # Every --fit rung, and only those: the HIP rung keeps its narrower gate.- assert src.count("and not _capability_crash") == 3+ # Every --fit rung and the MoE cache rung, and only those: the HIP rung keeps its+ # narrower gate.+ assert src.count("and not _capability_crash") == 4assert src.count("and not _tensor_capability_crash") == 1