Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
Studio: price per-layer embedding and Engram lookup tables as host or disk memory, not VRAM (#14465)
Daniel Han2b1476fcb1dc
diff --git a/studio/backend/core/inference/llama_cpp.py b/studio/backend/core/inference/llama_cpp.pyindex c654bc6a16b..d36ba0f61ce 100644--- a/studio/backend/core/inference/llama_cpp.py+++ b/studio/backend/core/inference/llama_cpp.py@@ -12,6 +12,7 @@ import atexitimport bisectimport weakrefimport contextlib+import contextvarsimport ctypesimport errnoimport functools@@ -82,6 +83,14 @@ from core.inference.context_window import (turn_is_servable,)from core.inference.llama_tool_schema import llama_grammar_tools+from core.inference.lookup_tables import (+ HOST_FORCED_TABLE_ARCHS as _HOST_FORCED_TABLE_ARCHS,+ HOST_FORCED_TABLE_MARKER as _HOST_FORCED_TABLE_MARKER,+ LAZY_READ_AUTO_MIN_BYTES as _LOOKUP_LAZY_READ_AUTO_MIN_BYTES,+ LAZY_READ_TENSORS as _LOOKUP_LAZY_READ_TENSORS,+ is_per_layer_embedding as _is_per_layer_embedding,+ lazy_read_bytes as _lookup_lazy_read_bytes,+)from core.inference.stream_errors import stream_error_from_chunkfrom hub.utils.hf_errors import modelscope_missingfrom hub.utils.hf_tokens import (@@ -6659,6 +6668,17 @@ def _extra_args_tensor_split(return parts+def _pass_through_reprices_cache(extra_args: Optional[Iterable[str]], n_parallel) -> bool:+ """Whether an argv -c 0 or a larger --parallel changes the cache Studio priced."""+ try:+ if parse_ctx_override(extra_args) == 0:+ return True+ except ValueError:+ return True+ override = _extra_args_n_parallel(extra_args, {})+ return override is not None and override > int(n_parallel or 1)++def _extra_args_n_parallel(extra_args: Optional[Iterable[str]], env: Optional[Mapping[str, str]] = None) -> Optional[int]:@@ -7066,13 +7086,9 @@ _MOE_CACHE_ERROR_RE = re.compile(re.IGNORECASE,)-# TENSOR_READ_LAZY tensors (src/models/gemma4.cpp, qwen4exp.cpp): under --lazy-mode auto one-# over 4 GiB stays mmap'd under every load mode (lazy_read::add), needing no pinned RAM or VRAM.-_LAZY_READ_TENSORS: dict[str, tuple[str, ...]] = {- "gemma4": ("per_layer_token_embd.weight",),- "qwen4exp": ("per_layer_token_embd.weight",),-}-_LAZY_READ_AUTO_MIN_BYTES = 4 * 1024**3+_LAZY_READ_TENSORS = _LOOKUP_LAZY_READ_TENSORS+_LAZY_READ_AUTO_MIN_BYTES = _LOOKUP_LAZY_READ_AUTO_MIN_BYTES+_LAZY_RESIDENT_MARGIN_FRAC = 0.10# Embedded NextN/MTP heads llama.cpp skips at load unless --spec-type draft-mtp, measured:# Nemotron 3.5 Super loads 2987 MiB less with MTP off (b11642 + ggml-org#30520)._NEXTN_SKIPPED_WITHOUT_MTP_ARCHS = frozenset({"nemotron_h_moe"})@@ -7122,6 +7138,29 @@ def _argv_loads_pinned(argv: Iterable[str], env: Optional[Mapping[str, str]] = Nreturn bool(reserves_ram and not mlock and not direct_io)+_GPU_MEMORY_SKIP_TORCH: "contextvars.ContextVar[bool]" = contextvars.ContextVar(+ "_GPU_MEMORY_SKIP_TORCH", default = False+)+++def _gpu_device_node_present() -> bool:+ """Whether this host exposes a GPU device node (ROCm, NVIDIA, WSL)."""+ if sys.platform == "darwin":+ return False+ if os.name == "nt":+ return True+ return any(os.path.exists(p) for p in ("/dev/kfd", "/dev/nvidiactl", "/dev/dxg"))+++def _without_torch_gpu_probe():+ token = _GPU_MEMORY_SKIP_TORCH.set(True)+ try:+ yield+ finally:+ _GPU_MEMORY_SKIP_TORCH.reset(token)++def _effective_lazy_mode(extra_args: Optional[Iterable[str]],env: Optional[Mapping[str, str]] = None,@@ -7151,16 +7190,111 @@ def _lazy_read_bytes(architecture: Optional[str], tensor_bytes: Mapping[str, int], mode: str) -> int:"""Bytes of ``tensor_bytes`` llama.cpp reads lazily under ``mode``."""- if mode not in ("auto", "on"):- return 0- names = _LAZY_READ_TENSORS.get(architecture or "", ())- return sum(- int(size)- for name, size in tensor_bytes.items()- if name in names and (mode == "on" or int(size) > _LAZY_READ_AUTO_MIN_BYTES)+ return _lookup_lazy_read_bytes(architecture, tensor_bytes, mode)+++_HOST_INPUT_HEAD_NAMES = frozenset({"token_embd.weight", "output.weight"})+++def _is_host_input_or_head_tensor(name: str) -> bool:+ """token_embd, output.weight and any per-layer lookup table: the host-pinned input tensors."""+ return name in _HOST_INPUT_HEAD_NAMES or _is_per_layer_embedding(name)+++def _override_places_tensor_on_device(+ name: str,+ extra_args: Optional[Iterable[str]] = None,+ env: Optional[Mapping[str, str]] = None,+) -> bool:+ """Whether an ``--override-tensor`` (argv or env) may put ``name`` on a device."""+ source_env = os.environ if env is None else env+ values: list[str] = []+ env_value = str(source_env.get("LLAMA_ARG_OVERRIDE_TENSOR") or "").strip()+ if env_value:+ values.append(env_value)+ args = [str(a) for a in extra_args or ()]+ for i, raw in enumerate(args):+ if _flag_name(raw) in {"-ot", "--override-tensor"}:+ _, eq, inline = raw.partition("=")+ value = inline if eq else (args[i + 1] if i + 1 < len(args) else "")+ if value:+ values.append(value)+ for value in values:+ for part in value.split(","):+ pattern, eq, target = part.rpartition("=")+ if not eq or not pattern:+ continue+ on_host = _override_targets_host(f"{pattern}={target}")+ if "[:" in pattern:+ # A POSIX class is std::regex syntax Python reads literally: unknown, so a host+ if not on_host:+ return True+ continue+ try:+ if not re.search(pattern, name):+ continue+ except re.error:+ return True+ return not on_host+ return False++[email protected]_cache(maxsize = 8)+def _host_forced_in_files(key: tuple) -> frozenset[str]:+ """``_host_forced_table_archs`` for the files ``key`` names (path, size, mtime)."""+ import mmap++ for path, _size, _mtime in key:+ try:+ with open(path, "rb") as f, mmap.mmap(f.fileno(), 0, access = mmap.ACCESS_READ) as m:+ if m.find(_HOST_FORCED_TABLE_MARKER) >= 0:+ return _HOST_FORCED_TABLE_ARCHS+ except (OSError, ValueError):+ continue+ return frozenset()+++def _lazy_read_names(+ architecture: Optional[str], tensor_bytes: Mapping[str, int], mode: str+) -> frozenset[str]:+ """Names of ``tensor_bytes`` llama.cpp reads lazily under ``mode``. It picks the lazy+ (CPU) buffer before applying ``-ot``, so these stay on the host whatever an override says."""+ return frozenset(+ name+ for name, size in (tensor_bytes or {}).items()+ if _lazy_read_bytes(architecture, {name: size}, mode))+def _host_pinned_input_bytes(+ scan: "Optional[tuple[Optional[str], Mapping[str, int], int]]",+ extra_args: Optional[Iterable[str]] = None,+ env: Optional[Mapping[str, str]] = None,+ *,+ lazy_mode: str = "off",+ host_forced: Iterable[str] = (),+) -> int:+ """Bytes llama.cpp keeps on the host however many layers it offloads."""+ if not scan:+ return 0+ architecture, named, _experts = scan+ streamed = _lazy_read_names(architecture, named, lazy_mode)+ total = 0+ for name, size in (named or {}).items():+ if _is_per_layer_embedding(name):+ if (+ name in streamed+ or architecture in host_forced+ or not _override_places_tensor_on_device(name, extra_args, env)+ ):+ total += int(size)+ token_embd = int((named or {}).get("token_embd.weight") or 0)+ if token_embd and int((named or {}).get("output.weight") or 0):+ if not _override_places_tensor_on_device("token_embd.weight", extra_args, env):+ total += token_embd+ return total++def _build_ngram_mod_flags(caps: Optional[dict],n_match: int = 24,@@ -7630,6 +7764,10 @@ _GGML_GPU_BACKEND_RE = re.compile(r"^(?:lib)?ggml-(?:cann|cuda|hip|metal|musa|opencl|sycl|virtgpu|vulkan)"r"(?:\.dll|\.so(?:\.\d+)*|(?:\.\d+)*\.dylib)$")+# libllama the same way: a `.bak` / `.disabled` copy is not the library the child loads.+_LLAMA_SHARED_LIB_RE = re.compile(+ r"^(?:lib)?llama(?:\.dll|\.so(?:\.\d+)*|(?:\.\d+)*\.dylib)$", re.IGNORECASE+)def _cpu_runtime_owner_alive(staged_dir: Path) -> bool:@@ -8119,6 +8257,8 @@ class LlamaCppBackend:self._memory_mlock_applicable: bool = True# --load-mode none tokens FIT emitted, removed by paths replacing placement; empty if user chose one.self._fit_load_mode_flags: list[str] = []+ self._lazy_mode_flags: list[str] = []+ self._apple_lazy_flags: list[str] = []# Mixed-pin --tensor-split tokens; the --fit on retry strips them (fit.cpp aborts on a set split).self._mixed_split_flags: list[str] = []# The pair a launch in flight is committed to, None when none is. ONE attribute:@@ -10521,7 +10661,9 @@ class LlamaCppBackend:named: dict[str, int] = {}experts = 0for p in paths:- arch, part, part_experts = self._gguf_scan_tensor_bytes(p)+ arch, part, part_experts = self._gguf_scan_tensor_bytes(+ p, also = _is_host_input_or_head_tensor+ )# Shard 1 carries the metadata.architecture = architecture or archfor name, size in part.items():@@ -10599,11 +10741,78 @@ class LlamaCppBackend:shared_gpu_ids: Optional[Iterable[int]],is_vulkan_backend: bool,probe_integrated_cuda: bool = True,+ extra_args: Optional[Iterable[str]] = None,+ env: Optional[Mapping[str, str]] = None,+ device_list: Optional[bool] = None,+ tensor_parallel: Optional[bool] = None,+ integrated_gpu_ids: Optional[Iterable[int]] = None,+ child_gpu_order: Optional[Sequence[int]] = None,) -> bool:- """Whether llama.cpp resolves ``--lazy-mode auto`` to ``off`` for this child: a model- device without mmap support (ggml-org/llama.cpp#28160), i.e. every selected device- integrated, since the default list holds iGPUs only when no discrete GPU is seen.- ``probe_integrated_cuda`` False uses only a cached SoC answer (no new context)."""+ """Whether llama.cpp resolves ``--lazy-mode auto`` to ``off`` for this child.++ ``child_gpu_order``: the ids the child's visibility mask will hold, in order+ (``_planned_child_gpu_order``), for a preview whose ``env`` does not carry that+ mask yet; a compact ``CUDA<k>`` / ``ROCm<k>`` names entry k of it."""+ tensor = (+ _effective_tensor_parallel(extra_args, False, env)+ if tensor_parallel is None+ else bool(tensor_parallel)+ )+ selected = None+ override = self._effective_device_ids(extra_args, env)+ one_device = not tensor and _split_mode_confines_to_one_device(extra_args, env)+ if one_device:+ main = self._effective_main_gpu(extra_args, env)+ main = 0 if main is None else main+ if main < 0:+ return False+ if override is not None and one_device:+ override = [d for d in override if d.lower() not in _CPU_DEVICE_VALUES]+ override = override[main : main + 1]+ if override is not None:+ # An RPC device reports mmap support (ggml-rpc.cpp device_get_props), so it never turns+ # auto off.+ gpu_names = [+ d+ for d in override+ if d.lower() not in _CPU_DEVICE_VALUES and not d.strip().lower().startswith("rpc")+ ]+ ordinals = [self._device_ordinal(d) for d in gpu_names]+ if all(o is not None for o in ordinals):+ if child_gpu_order is not None and not is_vulkan_backend:+ _order = [int(i) for i in child_gpu_order]+ selected = [_order[o] for o in ordinals if 0 <= o < len(_order)] or _order+ else:+ selected = list(self._physical_ordinals(ordinals, env, is_vulkan_backend) or ())+ elif tensor or (device_list if device_list is not None else is_vulkan_backend):+ try:+ selected = list(gpu_indices or ()) or None+ except TypeError:+ selected = None+ if selected is None and tensor:+ selected = [row[0] for row in (detected_gpus or ())] or None+ if selected is not None:+ if selected and override is None and one_device:+ main = self._effective_main_gpu(extra_args, env)+ main = 0 if main is None else main+ selected = selected[main : main + 1] if main >= 0 else []+ selected = set(selected)+ if not selected:+ return False+ if selected:+ if is_vulkan_backend:+ return bool(selected & (set(shared_gpu_ids or ()) | set(integrated_gpu_ids or ())))+ try:+ if selected & LlamaCppBackend._rocm_unified_memory_gpu_ids():+ return True+ except Exception:+ pass+ if not probe_integrated_cuda and not self._integrated_cuda_probe_is_free():+ return False+ try:+ return bool(selected & LlamaCppBackend._integrated_cuda_gpu_ids())+ except Exception:+ return Falserows = [(row[0], row[1]) for row in (detected_gpus or ())]if self._offload_target_shares_system_memory(is_vulkan_backend = is_vulkan_backend,@@ -10620,6 +10829,303 @@ class LlamaCppBackend:list(gpu_indices) if gpu_indices else None)+ def _child_runs_on_cpu(+ self,+ binary: Optional[str],+ extra_args: Optional[Iterable[str]],+ env: Optional[Mapping[str, str]],+ *,+ cpu_forced: bool = False,+ tensor_parallel: Optional[bool] = None,+ ) -> bool:+ """Whether the child gets no model device (CPU build, CPU --device or forced CPU launch)."""+ if (+ cpu_forced+ or self._binary_ships_no_gpu_backend(binary, env)+ or _device_selection_is_cpu(extra_args, env)+ ):+ return True+ tensor = (+ _effective_tensor_parallel(extra_args, False, env)+ if tensor_parallel is None+ else bool(tensor_parallel)+ )+ # Split mode none with a negative main GPU clears the model's devices (src/llama.cpp).+ if not tensor and _split_mode_confines_to_one_device(extra_args, env):+ main = self._effective_main_gpu(extra_args, env)+ return main is not None and main < 0+ return False++ @staticmethod+ def _metal_placement_unset(+ extra_args: Optional[Iterable[str]], env: Optional[Mapping[str, str]], *, layers_fixed: bool+ ) -> bool:+ """Whether nothing but Studio places this Metal launch, so a measured discount may pin full+ offload."""+ source_env = os.environ if env is None else env+ return not (+ layers_fixed+ or _extra_args_set_any_flag(extra_args, _GPU_LAYER_FLAGS)+ or _env_fixes_gpu_layers(source_env)+ or _device_selection_is_cpu(extra_args, source_env)+ or _args_place_tensors_on_cpu(extra_args)+ or _env_places_tensors_on_cpu(source_env)+ or _extra_args_set_any_flag(extra_args, _FIT_CONTROL_FLAGS)+ or any(source_env.get(name) for name in _FIT_CONTROL_ENV_VARS)+ or fit_is_enabled_in(extra_args)+ # A pass-through adapter sits on the base tensor's buffer (llama-adapter.cpp:335) and+ # nothing here charges it: keep the discount unspent.+ or _sidecar_adapter_paths(extra_args)+ )++ @staticmethod+ def _host_forced_table_archs(binary: Optional[str] = None) -> frozenset[str]:+ """Archs whose table the build keeps on the host whatever -ot says; none if unreadable."""+ binary = binary or LlamaCppBackend._find_llama_server_binary()+ if not binary:+ return frozenset()+ try:+ resolved = _resolve_llama_binary(binary)+ libs = sorted(+ p+ for p in resolved.parent.iterdir()+ if _LLAMA_SHARED_LIB_RE.match(p.name) and p.is_file()+ )+ key = tuple(+ (str(p), st.st_size, st.st_mtime_ns)+ for p in (libs or [resolved])+ for st in (p.stat(),)+ )+ except OSError:+ return frozenset()+ return _host_forced_in_files(key)++ def lookup_tables_stream_by_default(+ self,+ extra_args: Optional[Iterable[str]] = None,+ env: Optional[Mapping[str, str]] = None,+ *,+ gpu_ids: Optional[Iterable[int]] = None,+ tensor_parallel: Optional[bool] = None,+ ) -> bool:+ """Whether a launch on this host leaves lazily read lookup tables on disk."""+ _env = dict(os.environ if env is None else env)+ try:+ binary = self._find_llama_server_binary()+ caps = self.probe_server_capabilities(binary)+ if not caps.get("supports_lazy_mode"):+ return False+ mode = _effective_lazy_mode(+ extra_args, _env, flag = str(caps.get("lazy_mode_flag") or "--lazy-mode")+ )+ if mode != "auto":+ return mode == "on"+ tensor = (+ _effective_tensor_parallel(extra_args, False, _env)+ if tensor_parallel is None+ else bool(tensor_parallel)+ )+ self._reconcile_split_mode_env(_env, tensor_parallel = tensor)+ cpu_child = self._child_runs_on_cpu(binary, extra_args, _env, tensor_parallel = tensor)+ vulkan = not cpu_child and self._is_vulkan_backend(binary)+ rows = []+ if not cpu_child:+ with _without_torch_gpu_probe():+ rows = list(self._get_gpu_memory(binary, for_llama_server = True) or ())+ # An empty Vulkan inventory is a failed probe: it may be an iGPU, where auto is off.+ if not rows and (vulkan or _gpu_device_node_present()):+ return False+ shared = {idx for idx, _free, total in rows if total <= 0} if vulkan else set()+ picked = [int(i) for i in gpu_ids] if gpu_ids else None+ return not self._lazy_auto_resolves_off(+ gpu_indices = picked,+ detected_gpus = [(idx, free) for idx, free, _total in rows],+ shared_gpu_ids = shared,+ is_vulkan_backend = vulkan,+ probe_integrated_cuda = False,+ extra_args = extra_args,+ env = _env,+ device_list = None if picked else False,+ tensor_parallel = tensor_parallel,+ integrated_gpu_ids = shared,+ )+ except Exception:+ return False++ def _managed_lazy_mode_flags(+ self,+ model_path: Optional[str],+ *,+ caps: Optional[Mapping[str, object]],+ extra_args: Optional[Iterable[str]] = None,+ env: Optional[Mapping[str, str]] = None,+ auto_resolves_off: bool = False,+ resident_fits: Callable[[int], Optional[bool]],+ binary: Optional[str] = None,+ ) -> list[str]:+ """``[<lazy flag>, "off"]`` to keep a lazily read lookup table resident, else ``[]``."""+ caps = caps or {}+ if not model_path or not caps.get("supports_lazy_mode"):+ return []+ flag = str(caps.get("lazy_mode_flag") or "--lazy-mode")+ source_env = os.environ if env is None else env+ flags, env_name = _LAZY_MODE_SPELLINGS.get(flag, (frozenset({flag}), ""))+ if (env_name and str(source_env.get(env_name) or "").strip()) or _extra_args_set_any_flag(+ extra_args, flags+ ):+ return []+ if auto_resolves_off:+ return []+ table = self._lazy_read_host_bytes(+ model_path, caps = caps, extra_args = extra_args, env = source_env+ )+ if table <= 0:+ return []+ scan = self._gguf_tensor_scan(model_path)+ if scan is None or (+ scan[0] not in self._host_forced_table_archs(binary)+ and any(+ _override_places_tensor_on_device(name, extra_args, source_env)+ for name in scan[1]+ if _is_per_layer_embedding(name)+ )+ ):+ return []+ try:+ fits = resident_fits(table)+ except Exception as e: # noqa: BLE001 -- unpriced: keep llama.cpp's default+ logger.debug("Lookup tables: resident fit unpriced (%s)", e)+ fits = None+ key = (model_path, table, bool(fits))+ if getattr(self, "_lazy_mode_decision_logged", None) != key:+ self._lazy_mode_decision_logged = key+ if fits:+ logger.info(+ "Lookup tables: keeping the %.1f GB lookup table in RAM (%s off); "+ "free RAM covers it beside the weights and the KV cache.",+ table / (1024**3),+ flag,+ )+ else:+ logger.info(+ "Lookup tables: streaming the %.1f GB lookup table from disk "+ "(llama.cpp's lazy read); RAM cannot hold it beside the weights and "+ "the KV cache. Prefill and decode can be slower and vary.",+ table / (1024**3),+ )+ return [flag, "off"] if fits else []++ def _apple_lookup_table_discount(+ self,+ probe_discount: int,+ model_path: Optional[str],+ *,+ caps: Optional[Mapping[str, object]],+ extra_args: Optional[Iterable[str]],+ model_size_fit: int,+ need_at_ctx: Optional[int],+ budget_mib: int,+ cache_ram: Optional[int],+ env: Optional[Mapping[str, str]] = None,+ binary: Optional[str] = None,+ cpu_forced: bool = False,+ ) -> int:+ """The Metal embedding discount with the lazily read lookup table settled."""+ source_env = os.environ if env is None else env+ self._apple_lazy_flags = []+ table = self._lazy_read_host_bytes(+ model_path, caps = caps, extra_args = extra_args, env = source_env+ )+ if table <= 0:+ auto_table = self._lazy_read_host_bytes(model_path, caps = caps, extra_args = (), env = {})+ if (+ auto_table > 0+ and _effective_lazy_mode(+ extra_args,+ source_env,+ flag = str((caps or {}).get("lazy_mode_flag") or "--lazy-mode"),+ )+ == "off"+ ):+ return max(0, int(probe_discount) - auto_table)+ return probe_discount+ if _sidecar_adapter_paths(extra_args):+ return probe_discount+ scan = self._gguf_tensor_scan(model_path)+ streamed = (+ _lazy_read_names(+ scan[0],+ scan[1],+ _effective_lazy_mode(+ extra_args,+ source_env,+ flag = str((caps or {}).get("lazy_mode_flag") or "--lazy-mode"),+ ),+ )+ if scan is not None+ else frozenset()+ )+ if (+ scan is not None+ and scan[0] not in self._host_forced_table_archs(binary)+ and any(+ _override_places_tensor_on_device(name, extra_args, source_env)+ for name in scan[1]+ if _is_per_layer_embedding(name) and name not in streamed+ )+ ):+ return probe_discount++ def _resident_fits(table_bytes: int) -> Optional[bool]:+ if self._child_runs_on_cpu(binary, extra_args, source_env, cpu_forced = cpu_forced):+ return None+ if any(+ str(source_env.get(name) or "").strip()+ for name in ("LLAMA_ARG_MMPROJ", "LLAMA_ARG_MMPROJ_URL")+ ):+ return None+ avail = self._available_system_memory_mib()+ prompt = _prompt_cache_host_bytes(+ cache_ram,+ extra_args,+ source_env,+ supports_cache_ram = bool((caps or {}).get("supports_cache_ram")),+ )+ if (+ avail is None+ or prompt is None+ or need_at_ctx is None+ or budget_mib <= 0+ or not self._can_estimate_kv()+ ):+ return None+ need = (+ int(model_size_fit)+ + int(need_at_ctx)+ + int(prompt)+ + int(table_bytes * _LAZY_RESIDENT_MARGIN_FRAC)+ )+ ceiling_mib = min(int(budget_mib), int(avail) - _HOST_RAM_HEADROOM_MIB)+ return need <= max(0, ceiling_mib) * 1024 * 1024++ self._apple_lazy_flags = self._managed_lazy_mode_flags(+ model_path,+ caps = caps,+ extra_args = extra_args,+ env = source_env,+ resident_fits = _resident_fits,+ binary = binary,+ )+ if self._apple_lazy_flags:+ return max(0, int(probe_discount) - table)+ if table > probe_discount:+ logger.info(+ "Metal fit: %.2f GB of lookup tables are read lazily from disk, outside "+ "the Metal working set; not charged to unified memory.",+ table / (1024**3),+ )+ return max(int(probe_discount), table)+@staticmethoddef _installed_ggml_backends(binary: Optional[str] = None) -> frozenset[str]:"""Backend libraries shipped beside llama-server."""@@ -11121,6 +11627,59 @@ class LlamaCppBackend:return Nonereturn ordered+ @staticmethod+ def _picker_ordered_gpu_indices(gpu_ids, gpu_indices, tp_tensor_split):+ """``(gpu_indices, tp_tensor_split)`` in the picker's order; a reorder the shares cannot+ follow is vetoed."""+ if not gpu_ids:+ return gpu_indices, tp_tensor_split+ picked = [int(i) for i in gpu_ids]+ if gpu_indices is None:+ return picked, tp_tensor_split+ before = [int(i) for i in gpu_indices]+ kept = set(before)+ reordered = [i for i in picked if i in kept] + [i for i in before if i not in set(picked)]+ if tp_tensor_split and len(tp_tensor_split) == len(before):+ share_by_id = dict(zip(before, tp_tensor_split))+ if all(i in share_by_id for i in reordered):+ return reordered, [share_by_id[i] for i in reordered]+ elif not tp_tensor_split:+ return reordered, tp_tensor_split+ return gpu_indices, tp_tensor_split++ def _planned_child_gpu_order(+ self,+ *,+ gpu_ids,+ gpu_indices,+ tp_tensor_split,+ extra_args,+ env,+ is_vulkan_backend: bool,+ cpu_forced: bool = False,+ ) -> "tuple[Optional[list[int]], Optional[list[int]]]":+ """``(device order, visibility mask order)`` the spawn will give the child; mask None when+ inherited."""+ order, split = self._picker_ordered_gpu_indices(gpu_ids, gpu_indices, tp_tensor_split)+ if order is None or is_vulkan_backend or cpu_forced:+ return (None if order is None else [int(i) for i in order]), None+ order = [int(i) for i in order]+ mask = list(order)+ if not gpu_ids:+ inherited = self._inherited_child_gpu_order(mask)+ if (+ inherited is not None+ and not _extra_args_have_tensor_split(extra_args, env)+ and (not split or len(split) == len(mask))+ ):+ mask = inherited+ try:+ if LlamaCppBackend._pinned_child_visibility(mask) is None:+ return order, None+ except Exception:+ return order, None+ return order, mask+@staticmethoddef _repoint_emitted_tensor_split(cmd: list, old_order: list[int], new_order: list[int]@@ -12915,20 +13474,7 @@ class LlamaCppBackend:"""Embedding bytes proven absent from the Metal working set; fails closed, CPU weights staycharged."""source_env = os.environ if env is None else env- if (- layers_fixed- or _extra_args_set_any_flag(extra_args, _GPU_LAYER_FLAGS)- or _env_fixes_gpu_layers(source_env)- or _device_selection_is_cpu(extra_args, source_env)- or _args_place_tensors_on_cpu(extra_args)- or _env_places_tensors_on_cpu(source_env)- or _extra_args_set_any_flag(extra_args, _FIT_CONTROL_FLAGS)- or any(source_env.get(name) for name in _FIT_CONTROL_ENV_VARS)- # Extras are appended after the pin, while inherited values are overridden by it.- or fit_is_enabled_in(extra_args)- # A pass-through adapter's bytes are not in the probe or model_size, so do not spend the discount.- or _sidecar_adapter_paths(extra_args)- ):+ if not self._metal_placement_unset(extra_args, source_env, layers_fixed = layers_fixed):return 0env_view = dict(source_env)scrub_memory_env(env_view, settings)@@ -12955,7 +13501,7 @@ class LlamaCppBackend:layout = self._tensor_spill_layout(model_path, all_shards = True)if layout is None or not layout.complete:return 0- embeddings = int(layout.token_embd_bytes)+ embeddings = int(layout.token_embd_bytes) + int(layout.per_layer_embd_bytes)if embeddings <= 0 or (mtp_may_engage and layout.has_excluded_blocks):return 0measured = self._metal_measured_model_mib(binary, model_path)@@ -13935,7 +14481,11 @@ class LlamaCppBackend:return rocm_gpus# ── Torch fallback (covers AMD ROCm and missing nvidia-smi) ──- gpus = LlamaCppBackend._get_gpu_memory_torch(binary, for_llama_server)+ gpus = (+ []+ if _GPU_MEMORY_SKIP_TORCH.get()+ else LlamaCppBackend._get_gpu_memory_torch(binary, for_llama_server)+ )if not gpus and mig_card_rows:LlamaCppBackend._GPU_IDS_ARE_PCI_INDICES = Truereturn LlamaCppBackend._widen_integrated_cuda_rows(mig_card_rows)@@ -14437,6 +14987,7 @@ class LlamaCppBackend:avail_mib: Optional[int] = None,shared_gpu_ids: Iterable[int] = (),honor_opt_out: bool = True,+ integrated_gpu_ids: Iterable[int] = (),) -> Optional[str]:"""Advisory shortfall warning, never blocks; UNSLOTH_ALLOW_HOST_OFFLOAD=1 silences only the message, never the pageable-load override."""argv = [str(a) for a in cmd or ()]@@ -14479,6 +15030,10 @@ class LlamaCppBackend:shared_gpu_ids = shared,is_vulkan_backend = bool(shared) or self._is_vulkan_backend(argv[0]),probe_integrated_cuda = False,+ extra_args = argv,+ env = _env,+ device_list = False,+ integrated_gpu_ids = integrated_gpu_ids,),),)@@ -14875,6 +15430,24 @@ class LlamaCppBackend:"GPU, so the whole model is charged to system RAM.")+ def _unified_pool_gpu_ids(+ self, rows, *, gpu_indices, shared_gpu_ids, is_vulkan_backend: bool+ ) -> set:+ """Ids in ``rows`` whose free VRAM is the host RAM pool (Vulkan iGPUs, ROCm APUs, CUDA+ SoCs)."""+ rows = list(rows or ())+ shared = set(shared_gpu_ids or ())+ pinned = list(gpu_indices) if gpu_indices is not None else [row[0] for row in rows]+ if rows and not is_vulkan_backend and self._amd_apu_wants_unified_memory(pinned):+ # Per device: a discrete card beside an APU still has its own pool.+ shared |= {row[0] for row in rows if self._amd_apu_wants_unified_memory([row[0]])}+ # Integrated CUDA SoCs (Jetson, DGX Spark) too. Cache only: _get_gpu_memory filled it.+ _integrated = LlamaCppBackend._INTEGRATED_CUDA_IDS.get(+ LlamaCppBackend._integrated_cuda_mask_key(), ()+ )+ shared |= {row[0] for row in rows if row[0] in _integrated}+ return shared+def _fits_without_paging(self,footprint_bytes: Optional[int],@@ -15053,10 +15626,11 @@ class LlamaCppBackend:is_vulkan_backend = is_vulkan_backend,gpu_indices = gpu_indices,):- logger.info(- "Load mode: pass-through arguments override the placement this fit "- "priced; charging the whole load to host RAM."- )+ if announce:+ logger.info(+ "Load mode: pass-through arguments override the placement this fit "+ "priced; charging the whole load to host RAM."+ )rows, gpu_indices = [], Nonefootprint = (model_size@@ -15082,16 +15656,12 @@ class LlamaCppBackend:# comes out of, so adding both fits a model twice into memory that holds it# once. shared_gpu_ids already names Vulkan iGPUs; the ROCm APUs it cannot see# are added here. Both end up priced against host RAM alone.- shared = set(shared_gpu_ids or ())- pinned = list(gpu_indices) if gpu_indices is not None else [idx for idx, _free in rows]- if rows and not is_vulkan_backend and self._amd_apu_wants_unified_memory(pinned):- # Per device: a discrete card next to an APU still has its own pool.- shared |= {idx for idx, _free in rows if self._amd_apu_wants_unified_memory([idx])}- # Integrated CUDA SoCs (Jetson, DGX Spark) too. Cache only: _get_gpu_memory filled it.- _integrated = LlamaCppBackend._INTEGRATED_CUDA_IDS.get(- LlamaCppBackend._integrated_cuda_mask_key(), ()+ shared = self._unified_pool_gpu_ids(+ rows,+ gpu_indices = gpu_indices,+ shared_gpu_ids = shared_gpu_ids,+ is_vulkan_backend = is_vulkan_backend,)- shared |= {idx for idx, _free in rows if idx in _integrated}# --no-kv-offload puts the WHOLE cache in host RAM whatever the layer placement# says (llama-kv-cache.cpp upgrades a layer's buffer type only inside# `if (offload)`), so free VRAM may not pay for it. Resolved off the same@@ -15667,6 +16237,24 @@ class LlamaCppBackend:for name in ("LLAMA_ARG_SPLIT_MODE", "LLAMA_ARG_TENSOR_SPLIT"):env.pop(name, None)+ @classmethod+ def _reconcile_split_mode_env(cls, env: dict[str, str], *, tensor_parallel: bool) -> None:+ """Clear inherited LLAMA_ARG_* split env that contradicts Studio's split, so previews price+ what the child runs."""+ if not tensor_parallel:+ # Layer split: clear a non-layer inherited split mode (and any paired+ # tensor-split) so the child can't override the layer plan.+ _inherited_sm = (env.get("LLAMA_ARG_SPLIT_MODE") or "").strip().lower()+ if _inherited_sm and _inherited_sm != "layer":+ env.pop("LLAMA_ARG_SPLIT_MODE", None)+ env.pop("LLAMA_ARG_TENSOR_SPLIT", None)+ else:+ # Unsloth owns the tensor split: it emits --tensor-split when it picks an+ # uneven one (CLI wins) and nothing when an even split is safe. Clear any+ # inherited LLAMA_ARG_TENSOR_SPLIT so the even case can't be overridden by a+ # stale env (the layer branch above clears it too).+ env.pop("LLAMA_ARG_TENSOR_SPLIT", None)+@staticmethoddef _discrete_tier(gpus: list,@@ -18527,6 +19115,8 @@ class LlamaCppBackend:self._memory_policy_extras_touched = Falseself._memory_mlock_applicable = Trueself._fit_load_mode_flags = []+ self._lazy_mode_flags = []+ self._apple_lazy_flags = []self._mixed_split_flags = []self._memory_pending_launch = Noneself._n_ubatch = self._DEFAULT_N_UBATCH@@ -21929,6 +22519,7 @@ class LlamaCppBackend:"Load mode: dropping the fit's --load-mode none for the CPU ""fallback; it runs entirely from host RAM.")+ replay = self._drop_managed_lazy_mode(replay, "CPU fallback")# And the managed pair: this replay appends "--device none".replay = self._drop_managed_dio(replay, "the CPU fallback runs entirely from host RAM", clear_record = False@@ -23254,6 +23845,7 @@ class LlamaCppBackend:# Seed before the try (except falls through to the launch; the probe may throw first). Capture before manual empties gpus so defaults stay GPU-aware._detected_gpus: list[tuple[int, int]] = []_shared_gpu_ids: set[int] = set()+ _vulkan_igpu_ids: frozenset[int] = frozenset()_placement_ctx_checkpoints = 0# Set when the arch gate emptied a non-empty GPU pool, so the env# block below masks the child onto the CPU. Bound before the try for@@ -23309,6 +23901,8 @@ class LlamaCppBackend:# for llama.cpp's own default. Bound before the try like the verdict# flags above: the except path falls through to the launch, which reads it._fit_load_mode: Optional[str] = None+ _managed_lazy_flags: Optional[list[str]] = None+ _host_input_embd = 0# Layer-fallback min GPUs; raised below on a tensor downgrade. Bound# before the try so the --fit-on except path still has it (no UnboundLocal)._layer_min_gpus = 1@@ -23427,6 +24021,11 @@ class LlamaCppBackend:"get GPU inference back.",_present,)+ _vulkan_igpu_ids = (+ frozenset(idx for idx, _free, total in _gpu_mem if total <= 0)+ if is_vulkan_backend+ else frozenset()+ )if is_vulkan_backend and not gpu_ids and gpu_memory_mode != "manual":_gpu_mem = self._vulkan_auto_gpu_memory(_gpu_mem)gpus = [(idx, free) for idx, free, _t in _gpu_mem]@@ -24270,6 +24869,68 @@ class LlamaCppBackend:floor_ctx,)+ def _host_input_bytes_now(weights: int) -> int:+ if not (+ gpus+ and not _integrated_gpus+ and not _shared_gpu_ids+ and not _shared_pool_gpus+ and not _metal_capable_host()+ and not self._amd_apu_wants_unified_memory([i for i, _f in gpus])+ and not set(+ LlamaCppBackend._INTEGRATED_CUDA_IDS.get(+ LlamaCppBackend._integrated_cuda_mask_key(), ()+ )+ ).intersection(i for i, _f in gpus)+ ):+ return 0+ # A streamed table stays on the host whatever -ot says.+ _inp_extras = (+ self._strip_device_extra_args(extra_args)+ if _gpu_ids_own_device_flags+ else extra_args+ )+ _inp_env = dict(os.environ)+ if gpu_memory_mode == "manual":+ self._clear_manual_placement_env(_inp_env)+ if _gpu_ids_own_device_flags:+ self._clear_device_placement_env(_inp_env)+ self._reconcile_split_mode_env(_inp_env, tensor_parallel = tensor_parallel)+ try:+ _inp_lazy = "off"+ if server_caps.get("supports_lazy_mode"):+ _inp_lazy = _effective_lazy_mode(+ _inp_extras,+ _inp_env,+ flag = str(server_caps.get("lazy_mode_flag") or "--lazy-mode"),+ )+ if _inp_lazy == "auto" and self._lazy_auto_resolves_off(+ gpu_indices = (+ gpu_indices+ if gpu_indices is not None or not gpu_ids+ else [int(i) for i in gpu_ids]+ ),+ detected_gpus = _detected_gpus,+ shared_gpu_ids = _shared_gpu_ids,+ is_vulkan_backend = is_vulkan_backend,+ extra_args = _inp_extras,+ env = _inp_env,+ tensor_parallel = tensor_parallel,+ integrated_gpu_ids = _vulkan_igpu_ids,+ ):+ _inp_lazy = "off"+ found = _host_pinned_input_bytes(+ self._gguf_tensor_scan(model_path),+ _inp_extras,+ _inp_env,+ lazy_mode = _inp_lazy,+ host_forced = self._host_forced_table_archs(binary),+ )+ except Exception as e: # unread header: charge the whole file+ logger.debug("Host-pinned input embeddings unpriced: %s", e)+ return 0+ return min(max(0, found), max(0, int(weights) - 1))+_mm_budgeted_gpus = [(_idx, _free) for _idx, _free in gpus if total_by_idx.get(_idx, 0) > 0]@@ -24300,6 +24961,7 @@ class LlamaCppBackend:)_mm_need = (gguf_size+ - _host_input_bytes_now(gguf_size)+ mmproj_size+ _compute_buffer_pipeline+ self._CUDA_CONTEXT_RESERVE_BYTES@@ -24415,6 +25077,7 @@ class LlamaCppBackend:_soft += self._MTP_DRAFT_COMPUTE_BYTESreturn (model_size+ - _host_input_bytes_now(model_size)+ (_layer_compute_buffer(slots)if slots@@ -24802,8 +25465,19 @@ class LlamaCppBackend:if (not gpus or len(_mm_budgeted_gpus) < len(gpus))else 0)+ _host_input_embd = _host_input_bytes_now(model_size)+ if _host_input_embd:+ logger.info(+ "GPU fit: %.2f GB of input embeddings and lookup tables stay "+ "in host memory; not charged to VRAM.",+ _host_input_embd / (1024**3),+ )model_size_fit = (- model_size + _compute_buffer_pipeline + _soft_overhead + _shared_pool_mmproj+ model_size+ - _host_input_embd+ + _compute_buffer_pipeline+ + _soft_overhead+ + _shared_pool_mmproj)def _subset_model_size(n_gpus: int) -> int:@@ -24848,9 +25522,9 @@ class LlamaCppBackend:_tp_flat_mtp,_mtp_bytes(min(2048, effective_ctx) if effective_ctx > 0 else 2048),)- _tp_required_mib = (model_size + _tp_mtp_floor + _soft_overhead) / (- 1024 * 1024- )+ _tp_required_mib = (+ model_size - _host_input_embd + _tp_mtp_floor + _soft_overhead+ ) / (1024 * 1024)if _tp_weight_budget_mib <= _tp_required_mib:logger.info("Tensor parallelism requested but the pooled VRAM "@@ -24881,6 +25555,7 @@ class LlamaCppBackend:)model_size_fit = (model_size+ - _host_input_embd+ _compute_buffer_pipeline+ _soft_overhead+ _shared_pool_mmproj@@ -24904,7 +25579,7 @@ class LlamaCppBackend:tp_tensor_split,) = self._plan_tensor_parallel(tp_gpus,- model_size,+ max(1, model_size - _host_input_embd),target_ctx,cache_type_kv = cache_type_kv,scratch_cache_type_kv = _scratch_cache_type_kv,@@ -25380,6 +26055,71 @@ class LlamaCppBackend:layers_fixed = _caller_owns_budget or _paravirtual_cpu_forced,mtp_may_engage = _mtp_will_engage,)+ _apple_probe_embd = _apple_host_embd+ # Metal refuses past the wired limit; None is unknown, 0 used up.+ _apple_wired = self._apple_metal_wired_headroom_bytes()+ _apple_resident_budget_mib = (+ _apple_fit_budget_mib+ if _apple_wired is None+ else min(+ _apple_fit_budget_mib,+ int(+ _apple_wired+ // (1024 * 1024)+ * max(0.0, 1.0 - _flat_mtp_reserve)+ ),+ )+ )+ _apple_extras = (+ self._strip_device_extra_args(extra_args)+ if _gpu_ids_own_device_flags+ else extra_args+ )+ _apple_env = dict(os.environ)+ if gpu_memory_mode == "manual":+ self._clear_manual_placement_env(_apple_env)+ if _gpu_ids_own_device_flags:+ self._clear_device_placement_env(_apple_env)+ self._reconcile_split_mode_env(_apple_env, tensor_parallel = tensor_parallel)+ _apple_host_embd = self._apple_lookup_table_discount(+ _apple_host_embd,+ model_path,+ caps = server_caps,+ extra_args = _apple_extras,+ env = _apple_env,+ model_size_fit = model_size_fit,+ need_at_ctx = (+ (+ _kv_bytes(effective_ctx, _fit_ctx_checkpoints)+ + _cc_bytes(effective_ctx)+ + _mtp_bytes(effective_ctx)+ + _apple_cpu_draft+ )+ if effective_ctx > 0+ and not _pass_through_reprices_cache(extra_args, n_parallel)+ and not _flat_mtp_engages+ and (+ _apple_cpu_draft := self._cpu_resident_draft_bytes(+ effective_ctx,+ drafter_path = _cpu_draft_path,+ draft_cache_type_k = _mtp_draft_ck,+ draft_cache_type_v = _mtp_draft_cv,+ n_parallel = n_parallel,+ swa_full = swa_full,+ kv_unified = planned_kv_unified,+ n_ubatch = _effective_ubatch,+ flash_attn = planned_flash_attn,+ )+ )+ is not None+ else None+ ),+ budget_mib = _apple_resident_budget_mib,+ cache_ram = cache_ram,+ binary = binary,+ cpu_forced = _paravirtual_cpu_forced or _arch_gate_forced_cpu,+ )+ _managed_lazy_flags = list(self._apple_lazy_flags)_apple_model_size_fit = model_size_fit - _apple_host_embdif _apple_host_embd:logger.info(@@ -25693,7 +26433,17 @@ class LlamaCppBackend:effective_ctx = max_available_ctx = _vram_fit_ctx# Unmeasured floors still rely on llama.cpp's fitter._metal_full_offload_pinned = bool(- _apple_host_embd and _apple_measured_ceiling is not None+ _apple_measured_ceiling is not None+ and (+ _apple_probe_embd+ or (+ _apple_host_embd+ and not _paravirtual_cpu_forced+ and self._metal_placement_unset(+ extra_args, os.environ, layers_fixed = _caller_owns_budget+ )+ )+ ))# Weights over the budget launch here too and the fitter places# them, so it must see the same wired limit the refusals do.@@ -25915,6 +26665,7 @@ class LlamaCppBackend:_child_env = dict(os.environ)if _gpu_ids_own_device_flags:self._clear_device_placement_env(_child_env)+ self._reconcile_split_mode_env(_child_env, tensor_parallel = tensor_parallel)if _pass_through_overrides_placement(_child_extras,_child_env,@@ -25988,21 +26739,35 @@ class LlamaCppBackend:supports_fit_target = _supports_fit_target,)# A lazily read table stays mmap'd on the host, never on the GPU.+ _child_order, _child_mask = self._planned_child_gpu_order(+ gpu_ids = gpu_ids,+ gpu_indices = gpu_indices,+ tp_tensor_split = tp_tensor_split,+ extra_args = _child_extras,+ env = _child_env,+ is_vulkan_backend = is_vulkan_backend,+ cpu_forced = _paravirtual_cpu_forced or _arch_gate_forced_cpu,+ )_lazy_host = self._lazy_read_host_bytes(model_path,caps = server_caps,extra_args = _child_extras,env = _child_env,auto_resolves_off = self._lazy_auto_resolves_off(- gpu_indices = gpu_indices,+ gpu_indices = _child_order,detected_gpus = _detected_gpus,shared_gpu_ids = _shared_gpu_ids,is_vulkan_backend = is_vulkan_backend,+ extra_args = _child_extras,+ env = _child_env,+ tensor_parallel = tensor_parallel,+ integrated_gpu_ids = _vulkan_igpu_ids,+ child_gpu_order = _child_mask,),)_need = (model_size_fit- - _lazy_host+ - max(0, _lazy_host - _host_input_embd)- _model_size_fit_cb+ _compute_buffer_pipeline+ (@@ -26533,7 +27298,9 @@ class LlamaCppBackend:_on = list(gpu_indices or [idx for idx, _ in gpus])if _on:self._pending_plan_mib = _gpu_plan_mib(- gguf_size + mmproj_size + kv_cache_bytes, _on, gpus+ max(1, gguf_size - _host_input_embd) + mmproj_size + kv_cache_bytes,+ _on,+ gpus,)if (not gpus@@ -26578,6 +27345,7 @@ class LlamaCppBackend:self._clear_manual_placement_env(_fit_env)if _gpu_ids_own_device_flags:self._clear_device_placement_env(_fit_env)+ self._reconcile_split_mode_env(_fit_env, tensor_parallel = tensor_parallel)# What the placement branch below will emit, so the effective# (last-wins) fitter state is readable before the final argv exists.# Every reachable branch is fixed by use_fit.@@ -26691,17 +27459,44 @@ class LlamaCppBackend:- _kv_bytes(effective_ctx, 0),)# A lazily read table stays mmap'd under every load mode: not pinned RAM.+ # An empty inventory beside a GPU may hide an iGPU, where auto is off: priced+ # resident.+ _fit_child_order, _fit_child_mask = self._planned_child_gpu_order(+ gpu_ids = gpu_ids,+ gpu_indices = gpu_indices,+ tp_tensor_split = tp_tensor_split,+ extra_args = _fit_extras,+ env = _fit_env,+ is_vulkan_backend = is_vulkan_backend,+ cpu_forced = _paravirtual_cpu_forced or _arch_gate_forced_cpu,+ )+ _fit_lazy_auto_off = (+ not _detected_gpus+ and (is_vulkan_backend or _gpu_device_node_present())+ and not self._child_runs_on_cpu(+ binary,+ _fit_extras,+ _fit_env,+ cpu_forced = _paravirtual_cpu_forced or _arch_gate_forced_cpu,+ tensor_parallel = tensor_parallel,+ )+ ) or self._lazy_auto_resolves_off(+ gpu_indices = _fit_child_order,+ detected_gpus = _detected_gpus,+ shared_gpu_ids = _shared_gpu_ids,+ is_vulkan_backend = is_vulkan_backend,+ extra_args = _fit_extras,+ env = _fit_env,+ tensor_parallel = tensor_parallel,+ integrated_gpu_ids = _vulkan_igpu_ids,+ child_gpu_order = _fit_child_mask,+ )_fit_lazy_bytes = self._lazy_read_host_bytes(model_path,caps = server_caps,extra_args = _fit_extras,env = _fit_env,- auto_resolves_off = self._lazy_auto_resolves_off(- gpu_indices = gpu_indices,- detected_gpus = _detected_gpus,- shared_gpu_ids = _shared_gpu_ids,- is_vulkan_backend = is_vulkan_backend,- ),+ auto_resolves_off = _fit_lazy_auto_off,)_fit_load_mode_kwargs = dict(model_size = _fit_model_size,@@ -26756,6 +27551,69 @@ class LlamaCppBackend:env = _fit_env,lazy_read_bytes = _fit_lazy_bytes,)+ _resident_inputs = max(0, _host_input_embd - _fit_lazy_bytes)+ if _resident_inputs and _fit_load_mode_kwargs["model_size"]:+ _fit_load_mode_kwargs["model_size"] = max(+ 1, _fit_load_mode_kwargs["model_size"] - _resident_inputs+ )+ _fit_load_mode_kwargs["host_only_bytes"] += _resident_inputs+ if _managed_lazy_flags is None:+ _lazy_prompt_cache = _prompt_cache_host_bytes(+ cache_ram,+ _fit_extras,+ _fit_env,+ supports_cache_ram = bool(server_caps.get("supports_cache_ram")),+ )++ def _table_resident_fits(table: int) -> Optional[bool]:+ if (+ _lazy_prompt_cache is None+ or not _fit_load_mode_kwargs["model_size"]+ or effective_ctx <= 0+ or _pass_through_reprices_cache(_fit_extras, n_parallel)+ # A CPU child cannot use the cards this fit credits.+ or self._child_runs_on_cpu(+ binary,+ _fit_extras,+ _fit_env,+ cpu_forced = _paravirtual_cpu_forced or _arch_gate_forced_cpu,+ tensor_parallel = tensor_parallel,+ )+ ):+ return None+ return (+ self._fit_derived_load_mode(+ **{+ **_fit_load_mode_kwargs,+ "lazy_read_bytes": table,+ "host_only_bytes": _fit_load_mode_kwargs["host_only_bytes"]+ + table+ + _lazy_prompt_cache+ + int(table * _LAZY_RESIDENT_MARGIN_FRAC),+ "announce": False,+ }+ )+ == self._FIT_LOAD_MODE+ )++ _managed_lazy_flags = self._managed_lazy_mode_flags(+ model_path,+ caps = server_caps,+ extra_args = _fit_extras,+ env = _fit_env,+ auto_resolves_off = _fit_lazy_auto_off,+ resident_fits = _table_resident_fits,+ binary = binary,+ )+ if _managed_lazy_flags and _fit_lazy_bytes:+ if _fit_load_mode_kwargs["model_size"]:+ _fit_load_mode_kwargs["model_size"] = max(+ 1, _fit_load_mode_kwargs["model_size"] - _fit_lazy_bytes+ )+ _fit_load_mode_kwargs["host_only_bytes"] += _fit_lazy_bytes+ _fit_load_mode_kwargs["lazy_read_bytes"] = 0+ _fit_lazy_bytes = 0+ _spill_inputs["lazy_read_bytes"] = _fit_lazy_bytes_fit_load_mode = self._fit_derived_load_mode(**_fit_load_mode_kwargs)if _moe_cache_candidate:_moe_cache_switch = _moe_switch(_MOE_CACHE_ENV)@@ -26825,6 +27683,7 @@ class LlamaCppBackend:_placement_verdict_partial = False# Same reasoning: an unpriced load keeps llama.cpp's default._fit_load_mode = None+ _managed_lazy_flags = None_moe_cache_candidate = _moe_cache_ram_ok = False# And a half-built snapshot is not a price either: the throw can# land after the dict exists but before the later terms are added@@ -26869,7 +27728,9 @@ class LlamaCppBackend:):capped_only = _ctx_capped_for_vram and not _placement_verdict_partialasked_ctx = _ctx_before_fit if capped_only else effective_ctx- need_mib = (gguf_size + mmproj_size + _kv_bytes(asked_ctx)) // (1024 * 1024)+ need_mib = (+ max(1, gguf_size - _host_input_embd) + mmproj_size + _kv_bytes(asked_ctx)+ ) // (1024 * 1024)free_mib = sum(free for _, free in gpus)need_gb, free_gb = need_mib / 1024, free_mib / 1024# The probe's own classifiers: an integrated CUDA SoC can report a real total.@@ -26900,24 +27761,9 @@ class LlamaCppBackend:# Pin the whole picked set so the model can't spill onto an unpicked GPU.if gpu_ids:- _picked_order = [int(i) for i in gpu_ids]- if gpu_indices is None:- gpu_indices = _picked_order- else:- # Survivors keep the user's drag order.- _kept = {int(i) for i in gpu_indices}- _before_reorder = [int(i) for i in gpu_indices]- _reordered = [i for i in _picked_order if i in _kept] + [- int(i) for i in gpu_indices if int(i) not in set(_picked_order)- ]- # Shares are positional over ascending indices; veto rather than half-apply a reorder.- if tp_tensor_split and len(tp_tensor_split) == len(_before_reorder):- _share_by_id = dict(zip(_before_reorder, tp_tensor_split))- if all(i in _share_by_id for i in _reordered):- tp_tensor_split = [_share_by_id[i] for i in _reordered]- gpu_indices = _reordered- elif not tp_tensor_split:- gpu_indices = _reordered+ gpu_indices, tp_tensor_split = self._picker_ordered_gpu_indices(+ gpu_ids, gpu_indices, tp_tensor_split+ )# Auto Vulkan fit prefers discrete GPUs and keeps that pool pinned.elif (is_vulkan_backend@@ -26974,20 +27820,55 @@ class LlamaCppBackend:_apu_ram_part = ("APU" if self._amd_apu_wants_unified_memory(gpu_indices) else "SoC")+ _apu_lazy_extras = (+ self._strip_device_extra_args(extra_args)+ if _gpu_ids_own_device_flags+ else extra_args+ )+ _apu_lazy_env = dict(os.environ)+ if gpu_memory_mode == "manual":+ self._clear_manual_placement_env(_apu_lazy_env)+ if _gpu_ids_own_device_flags:+ self._clear_device_placement_env(_apu_lazy_env)+ self._reconcile_split_mode_env(_apu_lazy_env, tensor_parallel = tensor_parallel)+ _apu_child_order, _apu_child_mask = self._planned_child_gpu_order(+ gpu_ids = gpu_ids,+ gpu_indices = gpu_indices,+ tp_tensor_split = tp_tensor_split,+ extra_args = _apu_lazy_extras,+ env = _apu_lazy_env,+ is_vulkan_backend = False,+ cpu_forced = _paravirtual_cpu_forced or _arch_gate_forced_cpu,+ )_apu_resident_model_size = max(1,model_size- self._lazy_read_host_bytes(model_path,caps = server_caps,- extra_args = extra_args,- env = os.environ,- auto_resolves_off = self._lazy_auto_resolves_off(- gpu_indices = gpu_indices,+ extra_args = _apu_lazy_extras,+ env = _apu_lazy_env,+ # A failed GPU probe still launches on the APU, where auto is off.+ auto_resolves_off = (+ not _detected_gpus+ and not self._child_runs_on_cpu(+ binary,+ _apu_lazy_extras,+ _apu_lazy_env,+ cpu_forced = _paravirtual_cpu_forced or _arch_gate_forced_cpu,+ tensor_parallel = tensor_parallel,+ )+ )+ or self._lazy_auto_resolves_off(+ gpu_indices = _apu_child_order,detected_gpus = _detected_gpus,shared_gpu_ids = _shared_gpu_ids,is_vulkan_backend = False,probe_integrated_cuda = False,+ extra_args = _apu_lazy_extras,+ env = _apu_lazy_env,+ tensor_parallel = tensor_parallel,+ child_gpu_order = _apu_child_mask,),),)@@ -27453,7 +28334,7 @@ class LlamaCppBackend:_sanitized_split,tp_gpus,gpu_indices,- model_size,+ max(1, model_size - _host_input_embd),effective_ctx,cache_type_kv = cache_type_kv,n_parallel = n_parallel,@@ -28228,6 +29109,12 @@ class LlamaCppBackend:]logger.info("Load mode: %s", " ".join(_load_mode_managed))+ # Before the extras, so a user's own lazy mode wins by last-arg.+ self._lazy_mode_flags = list(_managed_lazy_flags or [])+ if self._lazy_mode_flags:+ cmd.extend(self._lazy_mode_flags)+ logger.info("Lookup tables: %s", " ".join(self._lazy_mode_flags))+# A flag validate_extra_args refuses has an env twin llama.cpp reads before argv, so both must go.if _mem_extras:_emit_extra_args = self._strip_mmproj_args(list(_mem_extras))@@ -28426,20 +29313,9 @@ class LlamaCppBackend:# Clears an inherited non-layer LLAMA_ARG_SPLIT_MODE so the child cannot override the layer plan.env[_ct_var] = _ct_norm- # Strip inherited LLAMA_ARG_* split env; the CLI strip cannot reach it.- if not tensor_parallel:- # Clear any inherited LLAMA_ARG_TENSOR_SPLIT so a stale env cannot override an even split.- _inherited_sm = (env.get("LLAMA_ARG_SPLIT_MODE") or "").strip().lower()- if _inherited_sm and _inherited_sm != "layer":- env.pop("LLAMA_ARG_SPLIT_MODE", None)- env.pop("LLAMA_ARG_TENSOR_SPLIT", None)- else:- # Unsloth owns the tensor split: it emits --tensor-split when it- # picks an uneven one (CLI wins) and nothing when an even split- # is safe. Clear any inherited LLAMA_ARG_TENSOR_SPLIT so the even- # case can't be overridden by a stale env (the layer branch above- # clears it too).- env.pop("LLAMA_ARG_TENSOR_SPLIT", None)+ # Reconcile the inherited LLAMA_ARG_* split env with Unsloth's final+ # decision (the previews above apply the same helper).+ self._reconcile_split_mode_env(env, tensor_parallel = tensor_parallel)# A gpu_ids pin also owns inherited device placement.if _gpu_ids_own_device_flags:@@ -28521,8 +29397,19 @@ class LlamaCppBackend:layout = self._tensor_spill_layout(model_path, all_shards = True)if layout is None or not getattr(layout, "complete", True):return None- if int(getattr(layout, "lm_head_bytes", 0) or 0):+ if int(getattr(layout, "lm_head_bytes", 0) or 0) and not (+ _override_places_tensor_on_device("token_embd.weight", run_argv, env)+ ):need -= int(getattr(layout, "token_embd_bytes", 0) or 0)+ per_layer = int(getattr(layout, "per_layer_embd_bytes", 0) or 0)+ scan = self._gguf_tensor_scan(model_path) if per_layer else None+ if scan is not None and any(+ _override_places_tensor_on_device(name, run_argv, env)+ for name in scan[1]+ if _is_per_layer_embedding(name)+ ):+ per_layer = 0+ need -= per_layerif not engages and self._nextn_predict_layers:need -= int(getattr(layout, "excluded_block_bytes", 0) or 0)return need@@ -28948,6 +29835,7 @@ class LlamaCppBackend:child_has_no_gpu = _child_has_no_gpu,avail_mib = _preflight_avail_mib,shared_gpu_ids = _shared_gpu_ids,+ integrated_gpu_ids = _vulkan_igpu_ids,)if (_offload_msg@@ -28972,8 +29860,10 @@ class LlamaCppBackend:_reachable_gpus,env,child_has_no_gpu = _child_has_no_gpu,+ avail_mib = _preflight_avail_mib,shared_gpu_ids = _shared_gpu_ids,honor_opt_out = False,+ integrated_gpu_ids = _vulkan_igpu_ids,))@@ -29084,6 +29974,9 @@ class LlamaCppBackend:# Single revocation point for the tensor-spill plan; label is empty only on the first spawn.if label:run_cmd = self._drop_tensor_spill(run_cmd, label.lstrip("-") or "retry")+ run_cmd = self._drop_managed_lazy_mode(+ run_cmd, label.lstrip("-") or "retry"+ )# The cache only rides the first pinned launch: every retry re-places.if label or not _argv_loads_pinned(run_cmd, env):run_cmd = self._drop_moe_cache(@@ -29093,6 +29986,8 @@ class LlamaCppBackend:_did_fit_retry = False_did_moe_cache_retry = Falsefor _spawn_attempt in (0, 1, 2, 3):+ if _spawn_attempt:+ run_cmd = self._drop_managed_lazy_mode(run_cmd, "startup")# Defensive kill: drop an orphan Popen a concurrent load may# have stored before we overwrite the reference (#5161).# Also reaps the crashed first attempt on the retry pass.@@ -30213,40 +31108,15 @@ class LlamaCppBackend:_apu_ram_msg = None_host_ram_msg = None_apu_avail_mib = None- # The opt-out hides only the message; _retry_apu_msg stays set for _retry_is_oversized.- _retry_apu_msg = None- if model_size is not None and _retry_wants_unified:- _apu_avail_mib = self._available_system_memory_mib()- _apu_resident_model_size = max(- 1,- model_size- - self._lazy_read_host_bytes(- model_path,- caps = server_caps,- extra_args = cmd,- env = env,- auto_resolves_off = self._lazy_auto_resolves_off(- gpu_indices = _remaining,- detected_gpus = _detected_gpus,- shared_gpu_ids = _shared_gpu_ids,- is_vulkan_backend = is_vulkan_backend,- probe_integrated_cuda = False,- ),- ),- )- _retry_apu_msg = self._apu_ram_shortfall_message(- _apu_resident_model_size + _mmproj_pinned_bytes,- _apu_avail_mib,- )- # Cache tuning was decided on the crashed devices; re-decide it, since cmd is reused.- if not self._host_offload_warning_opted_out():- _apu_ram_msg = _retry_apu_msg- self._record_load_warning(_retry_apu_msg)+ # Read before the kill frees the crashed child's memory, which would inflate the reading.+ _retry_apu_avail_mib = (+ self._available_system_memory_mib()+ if model_size is not None and _retry_wants_unified+ else None+ )+ _retry_host_avail_mib = self._available_system_memory_mib()# host guard credited the whole pool; the respawn reaches only _remaining_retry_rows = [row for row in _detected_gpus if row[0] in set(_remaining)]- _retry_host_msg = self._launch_host_shortfall_message(cmd, _retry_rows, env)- _host_ram_msg = _retry_host_msg- self._record_load_warning(_retry_host_msg)logger.warning(f"llama-server crashed with a HIP kernel-image error on "f"GPU(s) {_crashed} -- the llama.cpp build has no kernels "@@ -30381,6 +31251,41 @@ class LlamaCppBackend:"Arch-crash retry leaves one GPU; dropped tensor ""parallelism, which needs at least two.")+ cmd = self._drop_managed_lazy_mode(cmd, "device fallback")+ _retry_apu_msg = None+ if model_size is not None and _retry_wants_unified:+ _apu_avail_mib = _retry_apu_avail_mib+ _apu_resident_model_size = max(+ 1,+ model_size+ - self._lazy_read_host_bytes(+ model_path,+ caps = server_caps,+ extra_args = cmd,+ env = env,+ auto_resolves_off = self._lazy_auto_resolves_off(+ gpu_indices = _remaining,+ detected_gpus = _detected_gpus,+ shared_gpu_ids = _shared_gpu_ids,+ is_vulkan_backend = is_vulkan_backend,+ probe_integrated_cuda = False,+ extra_args = cmd,+ env = env,+ ),+ ),+ )+ _retry_apu_msg = self._apu_ram_shortfall_message(+ _apu_resident_model_size + _mmproj_pinned_bytes,+ _apu_avail_mib,+ )+ if not self._host_offload_warning_opted_out():+ _apu_ram_msg = _retry_apu_msg+ self._record_load_warning(_retry_apu_msg)+ _retry_host_msg = self._launch_host_shortfall_message(+ cmd, _retry_rows, env, avail_mib = _retry_host_avail_mib+ )+ _host_ram_msg = _retry_host_msg+ self._record_load_warning(_retry_host_msg)# Restore what cmd means: a --fit retry may have recorded a page-lock that cmd never carried._fit_mode_left_cmd = bool(self._fit_load_mode_flags)if self._fit_load_mode_flags:@@ -30507,7 +31412,11 @@ class LlamaCppBackend:return Falsereturn bool(self._launch_host_shortfall_message(- cmd, _retry_rows, env, honor_opt_out = False+ cmd,+ _retry_rows,+ env,+ honor_opt_out = False,+ avail_mib = _retry_host_avail_mib,))@@ -30882,7 +31791,7 @@ class LlamaCppBackend:flash_attn = planned_flash_attn,)self._pending_plan_mib = _gpu_plan_mib(- gguf_size + mmproj_size + kv_cache_bytes,+ max(1, gguf_size - _host_input_embd) + mmproj_size + kv_cache_bytes,list(self._pending_plan_mib),gpus,)@@ -31085,6 +31994,7 @@ class LlamaCppBackend:)self._mmproj_fallback_reason = "projector_startup_failure"cmd = self._strip_mmproj_args(_vision_gpu_cmd)+ cmd = self._drop_managed_lazy_mode(cmd, "text-only retry")# Keep a trailing sidecar pair: rewriting it would drop the drafter's# reserve.if [@@ -33386,6 +34296,7 @@ class LlamaCppBackend:host_ram_headroom_bytes = _HOST_RAM_HEADROOM_MIB * 1024 * 1024,# -nkvo moves cache out of VRAM, so it shrinks the deficit.kv_on_host = not _kv_offload_from_args(extra_args, env = source_env),+ lazy_read_bytes = int(inputs.get("lazy_read_bytes") or 0),# GPU-resident allocations the layout cannot see, since it is built# from the target GGUF alone: the vision projector and the MTP draft# reserve. Both are in the footprint that produced the use_fit@@ -33415,6 +34326,20 @@ class LlamaCppBackend:return []return ["-ngl", "-1", "--fit", "off", *tokens]+ def _drop_managed_lazy_mode(self, run_cmd: "list[str]", why: str) -> "list[str]":+ """Take Unsloth's ``--lazy-mode off`` back out of an argv for a retry."""+ if not getattr(self, "_lazy_mode_flags", None):+ return run_cmd+ stripped = _without_subsequence(run_cmd, self._lazy_mode_flags)+ if stripped == run_cmd:+ return run_cmd+ logger.info(+ "Lookup tables: dropping %s for the %s retry; llama.cpp streams the table.",+ " ".join(self._lazy_mode_flags),+ why,+ )+ return stripped+def _drop_tensor_spill(self, run_cmd: "list[str]", why: str) -> "list[str]":"""Retries invalidate the plan, so placement returns to --fit on; the record stays for CPUfallback."""diff --git a/studio/backend/core/inference/lookup_tables.py b/studio/backend/core/inference/lookup_tables.pynew file mode 100644index 00000000000..71e880dcce9--- /dev/null+++ b/studio/backend/core/inference/lookup_tables.py@@ -0,0 +1,43 @@+# SPDX-License-Identifier: AGPL-3.0-only+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0++"""Lookup-only embedding tables (per-layer embeddings, Engram) and how llama.cpp places them."""++from __future__ import annotations++from typing import Mapping, Optional++LAZY_READ_TENSORS: dict[str, tuple[str, ...]] = {+ "gemma4": ("per_layer_token_embd.weight",),+ "qwen4exp": ("per_layer_token_embd.weight",),+ "deepseek41": ("per_layer_token_embd.weight",),+}+LAZY_READ_AUTO_MIN_BYTES = 4 * 1024**3+# Archs whose table a build marked by HOST_FORCED_TABLE_MARKER keeps on the host whatever -ot+# says. Studio's prebuilt b11734-mix-c208e5f logs "per_layer_token_embd.weight (105471 MiB+# iq4_nl) must stay in host memory; ignoring placement on CUDA0, using CPU" for the DeepSeek-V4.1+# Engram table; upstream b11870 gathers it in the graph and honours the override.+HOST_FORCED_TABLE_ARCHS = frozenset({"deepseek41"})+HOST_FORCED_TABLE_MARKER = b"must stay in host memory; ignoring placement on"+PER_LAYER_EMBD_PREFIX = "per_layer_token_embd"+++def is_per_layer_embedding(name: str) -> bool:+ """Whether ``name`` is a per-layer (PLE / Engram) input embedding table."""+ return str(name).startswith(PER_LAYER_EMBD_PREFIX)+++def lazy_read_bytes(+ architecture: Optional[str],+ tensor_bytes: Mapping[str, int],+ mode: str = "auto",+) -> int:+ """Bytes of ``tensor_bytes`` llama.cpp reads lazily under ``mode``."""+ if mode not in ("auto", "on"):+ return 0+ names = LAZY_READ_TENSORS.get(architecture or "", ())+ return sum(+ int(size)+ for name, size in tensor_bytes.items()+ if name in names and (mode == "on" or int(size) > LAZY_READ_AUTO_MIN_BYTES)+ )diff --git a/studio/backend/core/inference/memory_contract.py b/studio/backend/core/inference/memory_contract.pyindex cf07ecfd259..5b345b92690 100644--- a/studio/backend/core/inference/memory_contract.py+++ b/studio/backend/core/inference/memory_contract.py@@ -145,6 +145,8 @@ def build_memory_estimate(layer_count = getattr(breakdown, "layer_count", None),gpu_layers = getattr(breakdown, "gpu_layers", None),moe_offload_unmodelled = bool(moe_offload_unmodelled),+ lookup_table_bytes = int(getattr(breakdown, "lookup_table_bytes", 0) or 0),+ lookup_table_on_disk = bool(getattr(breakdown, "lookup_table_on_disk", False)),mlx_offload = (Noneif getattr(breakdown, "offloadable", None) is None@@ -189,6 +191,8 @@ def project_estimate_memory_response(estimate: MemoryEstimate) -> dict:"layer_count": estimate.layer_count,"gpu_layers": estimate.gpu_layers,"moe_offload_unmodelled": estimate.moe_offload_unmodelled,+ "lookup_table_bytes": estimate.lookup_table_bytes,+ "lookup_table_on_disk": estimate.lookup_table_on_disk,"mlx_offload": estimate.mlx_offload,}@@ -206,6 +210,8 @@ def project_kv_cache_estimate(return {# The quant-file meaning. See the module docstring."weights_bytes": estimate.quant_file_bytes or None,+ # Off the GPU and inside weights_bytes, so the bar can take it off the VRAM segment.+ "lookup_table_bytes": estimate.lookup_table_bytes or None,"kv_bytes": kv_bytes or None,"native_context": estimate.native_context,"spec_bytes": spec_bytes,diff --git a/studio/backend/core/inference/offload_layout.py b/studio/backend/core/inference/offload_layout.pyindex 7911b4869b2..a65d07920bd 100644--- a/studio/backend/core/inference/offload_layout.py+++ b/studio/backend/core/inference/offload_layout.py@@ -16,6 +16,8 @@ from dataclasses import dataclass, fieldfrom typing import Optionalimport os+from core.inference.lookup_tables import is_per_layer_embedding+logger = logging.getLogger(__name__)# Hybrid Mamba-2 archs storing head_count_kv (and on Bolt expert_used_count) per layer.@@ -52,6 +54,7 @@ class ModelLayout:lm_head_bytes: int = 0# llama.cpp pins dev_input to CPU: never VRAM, but charged to host RAMtoken_embd_bytes: int = 0+ per_layer_embd_bytes: int = 0# Every tensor's bytes, excluded blocks included; the file size less its metadata.tensor_bytes: int = 0# output_norm and friends: GPU-resident, too small to be worth spilling.@@ -259,6 +262,7 @@ def _layout_from_readers(readers) -> ModelLayout:resident: dict[int, int] = {}lm_head = 0token_embd = 0+ per_layer_embd = 0other_resident = 0for tensor in (t for r in readers for t in r.tensors):@@ -276,8 +280,11 @@ def _layout_from_readers(readers) -> ModelLayout:else:resident[index] = resident.get(index, 0) + nbytescontinue+ # Before the token_embd substring below, which would otherwise swallow the per-layer table.+ if is_per_layer_embedding(name):+ per_layer_embd += nbytes# token_embd_norm is a repeating-layer tensor, not a host-pinned input embedding.- if "token_embd" in name and not name.startswith("token_embd_norm"):+ elif "token_embd" in name and not name.startswith("token_embd_norm"):token_embd += nbyteselif name == "output.weight":lm_head += nbytes@@ -315,6 +322,7 @@ def _layout_from_readers(readers) -> ModelLayout:blocks = blocks,lm_head_bytes = lm_head,token_embd_bytes = token_embd,+ per_layer_embd_bytes = per_layer_embd,tensor_bytes = sum(int(t.n_bytes) for r in readers for t in r.tensors),other_resident_bytes = other_resident,kv_bytes_per_token_f16 = kv_per_token,diff --git a/studio/backend/core/inference/offload_planner.py b/studio/backend/core/inference/offload_planner.pyindex 6e1ea8ece53..622d5db5a95 100644--- a/studio/backend/core/inference/offload_planner.py+++ b/studio/backend/core/inference/offload_planner.py@@ -97,6 +97,7 @@ class PlanOptions:kv_quant_type: str = "q8_0"# -nkvo (or LLAMA_ARG_KV_OFFLOAD false) puts the whole KV cache on host (llama-kv-cache.cpp), so it is not charged to VRAMkv_on_host: bool = False+ lazy_read_bytes: int = 0@dataclass(frozen = True)@@ -792,8 +793,8 @@ def _finish(spilled_bytes = sum(b.spillable_bytes for b in chosen) + (layout.lm_head_bytes if spill_lm_head else 0)- # token_embd is host-resident on every launch, so it is host RAM this plan must pay for even when nothing is spilled- host_bytes = layout.token_embd_bytes + spilled_bytes+ lazy = min(max(0, int(opts.lazy_read_bytes or 0)), layout.per_layer_embd_bytes)+ host_bytes = layout.token_embd_bytes + layout.per_layer_embd_bytes - lazy + spilled_bytesif opts.kv_on_host:# -nkvo cache lives in host RAM; the mmap decision must count it.host_bytes += (diff --git a/studio/backend/hub/schemas/inventory.py b/studio/backend/hub/schemas/inventory.pyindex a718ad47c01..698857f24af 100644--- a/studio/backend/hub/schemas/inventory.py+++ b/studio/backend/hub/schemas/inventory.py@@ -46,6 +46,11 @@ class GgufVariantDetail(BaseModel):)size_bytes: int = Field(0, description = "File size in bytes")download_size_bytes: int = Field(0, description = "Total bytes needed to download this variant")+ lookup_table_bytes: int = Field(+ 0,+ description = "Bytes of lazily read lookup tables (per-layer embeddings / Engram) "+ "llama.cpp streams from disk by default. Part of size_bytes; 0 when none or unknown.",+ )pending_drafter_filename: Optional[str] = Field(None,description = (diff --git a/studio/backend/hub/services/models/gguf_variants.py b/studio/backend/hub/services/models/gguf_variants.pyindex cbbd9894517..5acfd685588 100644--- a/studio/backend/hub/services/models/gguf_variants.py+++ b/studio/backend/hub/services/models/gguf_variants.py@@ -12,6 +12,7 @@ import contextvarsimport functoolsimport hashlibimport json+import osimport reimport threadingimport time@@ -1248,6 +1249,346 @@ async def _mlx_precision_variants_answer(return VariantsAnswer(response, None, cache_authorized = False)+_LOOKUP_REMOTE_ENV = "UNSLOTH_LOOKUP_TABLE_REMOTE"+_LOOKUP_DEADLINE_S = 2.0+_LOOKUP_CONCURRENCY = 4+_LOOKUP_ARCH_TTL_S = 6 * 3600.0+_LOOKUP_ARCH_FAIL_TTL_S = 600.0+_lookup_arch_cache: dict[tuple[str, str], tuple[Optional[str], float]] = {}+_lookup_arch_lock = threading.Lock()+# Key of a read in flight -> its done event, so a second listing waits for it instead of skipping.+_lookup_inflight: dict = {}+_LOOKUP_POOL = ThreadPoolExecutor(+ max_workers = _LOOKUP_CONCURRENCY, thread_name_prefix = "gguf-lookup-tables"+)+++async def _lookup_thread(fn, /, *args):+ loop = asyncio.get_running_loop()+ call = functools.partial(contextvars.copy_context().run, fn, *args)+ return await loop.run_in_executor(_LOOKUP_POOL, call)+++def _claim_lookup(key) -> tuple[Optional[threading.Event], Optional[threading.Event]]:+ """``(None, own)`` once ``key`` is claimed here, else ``(held, None)``: the done event+ of the read already holding it."""+ with _lookup_arch_lock:+ held = _lookup_inflight.get(key)+ if held is not None:+ return held, None+ own = _lookup_inflight[key] = threading.Event()+ return None, own+++def _release_lookup(key, own: threading.Event) -> None:+ """Release ``key`` if ``own`` still holds it: a late worker never drops a newer claim."""+ with _lookup_arch_lock:+ if _lookup_inflight.get(key) is own:+ del _lookup_inflight[key]+ own.set()+++async def _wait_for_lookup(done: threading.Event, deadline: float) -> bool:+ """Whether another listing's read of the same key ended before ``deadline`` (loop time)."""+ loop = asyncio.get_running_loop()+ while not done.is_set():+ if loop.time() >= deadline:+ return False+ await asyncio.sleep(0.02)+ return True+++def _release_unrun_lookup(started: list, key, own, task: "asyncio.Future") -> None:+ """Release the inflight key of a read that never ran."""+ if not started and (task.cancelled() or task.exception() is not None):+ _release_lookup(key, own)+++async def _lookup_await_once(key, deadline: float, fn, /, *args):+ """``fn(*args)`` on the lookup pool, awaited until ``deadline`` (loop time). A call+ already in flight under ``key`` is waited for, then ``fn`` reruns on its cached answer+ (or retries, when that call left none)."""+ while True:+ held, own = _claim_lookup(key)+ if own is not None:+ break+ if not await _wait_for_lookup(held, deadline):+ return None++ started: list = []+ # Set at the deadline: a worker freeing up mid-cancel skips the call instead of running late.+ expired = threading.Event()++ def _run():+ started.append(True)+ try:+ return None if expired.is_set() else fn(*args)+ finally:+ _release_lookup(key, own)++ task = asyncio.ensure_future(_lookup_thread(_run))+ task.add_done_callback(functools.partial(_release_unrun_lookup, started, key, own))+ loop = asyncio.get_running_loop()+ done, _pending = await asyncio.wait({task}, timeout = max(0.0, deadline - loop.time()))+ if task not in done:+ expired.set()+ task.cancel()+ return None+ if task.cancelled() or task.exception() is not None:+ return None+ return task.result()+++def _cached_hub_repo_architecture(repo_id: str, hf_token: Optional[str]):+ """``(known, architecture)`` from the cache alone; never blocks on the Hub."""+ key = (repo_id.lower(), hf_cache_scan.token_fingerprint(hf_token))+ with _lookup_arch_lock:+ hit = _lookup_arch_cache.get(key)+ if hit is not None and hit[1] > time.monotonic():+ return True, hit[0]+ return False, None+++_LOOKUP_HOST_TTL_S = 60.0+_lookup_host_streams: list = [None, 0.0]+++def _host_streams_lookup_tables() -> bool:+ """Whether a default launch here streams the tables (an iGPU / APU reads them resident)."""+ now = time.monotonic()+ with _lookup_arch_lock:+ streams, expires = _lookup_host_streams+ if streams is not None and expires > now:+ return streams+ try:+ from routes.inference import get_llama_cpp_backend+ streams = bool(get_llama_cpp_backend().lookup_tables_stream_by_default())+ except Exception: # noqa: BLE001 -- unknown: price the tables as resident+ streams = False+ with _lookup_arch_lock:+ _lookup_host_streams[:] = [streams, now + _LOOKUP_HOST_TTL_S]+ return streams+++def _listing_lazy_mode() -> str:+ """The lazy mode a default launch here runs under."""+ try:+ from core.inference.llama_cpp import _effective_lazy_mode+ from routes.inference import get_llama_cpp_backend++ flag = get_llama_cpp_backend().probe_server_capabilities().get("lazy_mode_flag")+ mode = _effective_lazy_mode(None, os.environ, flag = str(flag or "--lazy-mode"))+ except Exception: # noqa: BLE001 -- unknown build: the default threshold+ return "auto"+ return "on" if mode == "on" else "auto"+++def _lookup_remote_enabled() -> bool:+ return (os.environ.get(_LOOKUP_REMOTE_ENV) or "").strip().lower() not in (+ "0",+ "off",+ "false",+ "no",+ )+++def _hub_repo_architecture(repo_id: str, hf_token: Optional[str]) -> Optional[str]:+ """``general.architecture`` the Hub reports for a GGUF repo, cached per repo and token."""+ key = (repo_id.lower(), hf_cache_scan.token_fingerprint(hf_token))+ now = time.monotonic()+ with _lookup_arch_lock:+ hit = _lookup_arch_cache.get(key)+ if hit is not None and hit[1] > now:+ return hit[0]+ arch: Optional[str] = None+ ttl = _LOOKUP_ARCH_FAIL_TTL_S+ try:+ from huggingface_hub import HfApi++ info = HfApi().model_info(repo_id, expand = ["gguf"], token = hf_token or None, timeout = 5)+ gguf_meta = getattr(info, "gguf", None) or {}+ arch = str(gguf_meta.get("architecture") or "") or None+ ttl = _LOOKUP_ARCH_TTL_S+ except Exception as exc: # noqa: BLE001 -- unknown arch: no remote reads+ logger.debug("lookup tables: no architecture for %s (%s)", repo_id, exc)+ with _lookup_arch_lock:+ _lookup_arch_cache[key] = (arch, now + ttl)+ return arch+++def _local_variant_gguf(+ answer: "VariantsAnswer",+ repo_id: str,+ variant,+ local_path: Optional[str] = None,+) -> Optional[str]:+ """The on-disk main GGUF behind a listed row, or None."""+ filename = str(variant.filename or "")+ sources = answer.variant_context_sources or {}+ candidates = [sources.get(str(variant.quant or "").lower())]+ roots = [repo_id] if is_local_path(repo_id) else []+ if answer.context_source:+ roots.append(answer.context_source)+ pinned = _snapshot_scope_for_request(repo_id, local_path)+ if pinned is not None:+ roots.append(str(pinned))+ for root in roots:+ path = Path(root).expanduser()+ candidates.append(str((path.parent if path.is_file() else path) / filename))+ for candidate in candidates:+ if candidate and os.path.isfile(candidate):+ return candidate+ if (+ variant.downloaded+ and answer.cache_authorized+ and not is_local_path(repo_id)+ and _is_valid_repo_id(repo_id)+ ):+ try:+ from huggingface_hub import try_to_load_from_cache++ repo_dir = _repo_cache_dir_for_request(repo_id, local_path)+ found = try_to_load_from_cache(repo_id, filename, cache_dir = str(repo_dir.parent))+ if isinstance(found, str) and os.path.isfile(found):+ return found+ except Exception: # noqa: BLE001 -- not cached here+ return None+ return None+++def _remote_lookup_bytes(+ repo_id: str,+ filename: str,+ hf_token,+ identity,+ mode = "auto",+ known_arch: Optional[str] = None,+) -> int:+ from hub.utils.gguf_lookup_tables import remote_lookup_info+ return remote_lookup_info(+ repo_id, filename, hf_token, identity = identity, mode = mode, known_arch = known_arch+ )[1]+++async def _with_lookup_tables(+ answer: "VariantsAnswer",+ repo_id: str,+ *,+ hf_token: Optional[str],+ offline: bool,+ prefer_local_cache: bool,+ local_path: Optional[str] = None,+) -> "VariantsAnswer":+ """``answer`` with ``lookup_table_bytes`` filled per variant."""+ try:+ from core.inference.lookup_tables import LAZY_READ_AUTO_MIN_BYTES+ from hub.utils.gguf_lookup_tables import (+ lazy_architecture,+ local_lookup_info,+ remote_cached,+ )++ variants = list(answer.response.variants or ())+ # A host that reads the tables resident lists no discount: no header is worth reading.+ if not variants or not await listing_thread(True, _host_streams_lookup_tables):+ return answer+ mode = await listing_thread(True, _listing_lazy_mode)+ found: dict[int, int] = {}+ arch: Optional[str] = None+ unresolved = []+ for index, v in enumerate(variants):+ if not str(v.filename or "").lower().endswith(".gguf"):+ continue+ # An update's cached file is the old revision: online, read the published header.+ published = v.update_available and not offline and not prefer_local_cache+ path = (+ None+ if published and not is_local_path(repo_id)+ else await listing_thread(True, _local_variant_gguf, answer, repo_id, v, local_path)+ )+ if path:+ local_arch, table = await listing_thread(True, local_lookup_info, path, mode)+ # Only a lazy one speaks for the repo: a repo can hold several families.+ if lazy_architecture(local_arch):+ arch = arch or local_arch+ if local_arch is not None:+ found[index] = table+ continue+ # Not on disk, or unreadable there (a shard still downloading): the Hub's header.+ unresolved.append((index, v))+ remote_ok = (+ unresolved+ and not offline+ and not prefer_local_cache+ and _lookup_remote_enabled()+ and not is_local_path(repo_id)+ and _is_valid_repo_id(repo_id)+ )+ if remote_ok:+ # Its own try: a Hub failure must not drop the local answers above.+ try:+ deadline = asyncio.get_running_loop().time() + _LOOKUP_DEADLINE_S+ if arch is None:+ known, arch = _cached_hub_repo_architecture(repo_id, hf_token)+ if not known and (+ mode == "on"+ or any(+ int(v.size_bytes or 0) > LAZY_READ_AUTO_MIN_BYTES+ for _i, v in unresolved+ )+ ):+ arch = await _lookup_await_once(+ ("arch", repo_id.lower(), hf_cache_scan.token_fingerprint(hf_token)),+ deadline,+ _hub_repo_architecture,+ repo_id,+ hf_token,+ )+ if lazy_architecture(arch):+ _fingerprint = hf_cache_scan.token_fingerprint(hf_token)+ tasks = {}+ for index, v in unresolved:+ identity = int(v.size_bytes or 0)+ # Under auto only a tensor over the threshold streams.+ if mode != "on" and 0 < identity <= LAZY_READ_AUTO_MIN_BYTES:+ continue+ cached = remote_cached(repo_id, v.filename, None, identity, mode = mode)+ if cached is not None:+ found[index] = cached[1]+ continue+ key = ("table", repo_id.lower(), v.filename, identity, mode, _fingerprint)+ task = asyncio.ensure_future(+ _lookup_await_once(+ key,+ deadline,+ _remote_lookup_bytes,+ repo_id,+ v.filename,+ hf_token,+ identity,+ mode,+ arch,+ )+ )+ tasks[task] = index+ for task, value in zip(tasks, await asyncio.gather(*tasks)):+ found[tasks[task]] = int(value or 0)+ except Exception as exc: # noqa: BLE001 -- keep the local answers+ logger.debug("lookup tables: Hub reads skipped for %s (%s)", repo_id, exc)+ if not any(found.values()):+ return answer+ updated = [+ v.model_copy(update = {"lookup_table_bytes": int(found.get(i, 0) or 0)})+ if found.get(i)+ else v+ for i, v in enumerate(variants)+ ]+ return answer._replace(response = answer.response.model_copy(update = {"variants": updated}))+ except Exception as exc: # noqa: BLE001 -- the listing never fails on this+ logger.debug("lookup tables: listing annotation skipped for %s (%s)", repo_id, exc)+ return answer++async def get_gguf_variants_answer(repo_id: str,prefer_local_cache: bool = False,@@ -2389,7 +2730,14 @@ async def get_gguf_variants_answer()if mlx_answer is not None:return mlx_answer- return answer+ return await _with_lookup_tables(+ answer,+ repo_id,+ hf_token = hf_token,+ offline = bool(offline),+ prefer_local_cache = bool(prefer_local_cache),+ local_path = local_path,+ )except HTTPException as exc:# Offline, a repo with no cached GGUF raises here, before the MLX fallback above.if (diff --git a/studio/backend/hub/utils/gguf_header_delta.py b/studio/backend/hub/utils/gguf_header_delta.pyindex d2ecba77076..0298c61331a 100644--- a/studio/backend/hub/utils/gguf_header_delta.py+++ b/studio/backend/hub/utils/gguf_header_delta.py@@ -492,6 +492,10 @@ def hub_range_fetcher(with _ranged_stream(get_session(), url, headers) as resp:if resp.status_code != 206:raise ValueError(f"range request answered {resp.status_code}")+ headers = getattr(resp, "headers", None) or {}+ total = str(headers.get("Content-Range") or "").rpartition("/")[2]+ if total.isdigit():+ fetch.file_size = int(total)return b"".join(_iter_body(resp, 1 << 20))return fetchdiff --git a/studio/backend/hub/utils/gguf_lookup_tables.py b/studio/backend/hub/utils/gguf_lookup_tables.pynew file mode 100644index 00000000000..5b3719c1df7--- /dev/null+++ b/studio/backend/hub/utils/gguf_lookup_tables.py@@ -0,0 +1,330 @@+# SPDX-License-Identifier: AGPL-3.0-only+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0++"""Bytes of the lookup-only tables (per-layer embeddings / Engram) llama.cpp streams from disk."""++from __future__ import annotations++import bisect+import os+import re+import struct+import threading+import time+from collections import OrderedDict+from typing import Callable, Optional++from core.inference.lookup_tables import LAZY_READ_TENSORS, lazy_read_bytes++_SPLIT_RE = re.compile(r"^(.*)-(\d{5})-of-(\d{5})(\.gguf)$", re.IGNORECASE)+_SCALAR_SIZES = {0: 1, 1: 1, 2: 2, 3: 2, 4: 4, 5: 4, 6: 4, 7: 1, 10: 8, 11: 8, 12: 8}+_STRING, _ARRAY, _UINT32 = 8, 9, 4+_FIRST_READ = 8 << 20+# A later split shard carries only the split keys: its header is a few KiB.+_SPLIT_SHARD_FIRST_READ = 64 << 10+MAX_HEADER_BYTES = 64 << 20++_WANTED = {name.encode() for names in LAZY_READ_TENSORS.values() for name in names}+++class _NeedMore(Exception):+ pass+++def lazy_architecture(arch: Optional[str]) -> bool:+ """Whether ``arch`` flags a lookup table llama.cpp may read lazily."""+ return bool(arch) and arch in LAZY_READ_TENSORS+++def parse_header(+ buf: bytes,+ file_size: Optional[int] = None,+ arch_only: bool = False,+) -> tuple[Optional[str], dict[str, int]]:+ """(architecture, {lazy-candidate tensor name: bytes}) from a GGUF v2/v3 header. A quant+ type gguf predates is sized from the next tensor's offset (or ``file_size``), as the+ launch scanner does. ``arch_only`` stops at the architecture key."""+ from gguf.constants import GGML_QUANT_SIZES++ pos = 0++ def take(n: int) -> bytes:+ nonlocal pos+ if pos + n > len(buf):+ raise _NeedMore()+ out = buf[pos : pos + n]+ pos += n+ return out++ def u32() -> int:+ return struct.unpack("<I", take(4))[0]++ def u64() -> int:+ return struct.unpack("<Q", take(8))[0]++ def string() -> bytes:+ n = u64()+ if n > (1 << 30):+ raise ValueError("implausible string length")+ return take(n)++ def skip_value(vtype: int, depth: int = 0) -> None:+ if depth > 8:+ raise ValueError("implausible array nesting")+ if vtype in _SCALAR_SIZES:+ take(_SCALAR_SIZES[vtype])+ elif vtype == _STRING:+ string()+ elif vtype == _ARRAY:+ item = u32()+ count = u64()+ if item in _SCALAR_SIZES:+ take(_SCALAR_SIZES[item] * count)+ elif item == _STRING:+ for _ in range(count):+ string()+ else:+ for _ in range(count):+ skip_value(item, depth + 1)+ else:+ raise ValueError(f"unknown value type {vtype}")++ if take(4) != b"GGUF":+ raise ValueError("not a GGUF")+ if u32() not in (2, 3):+ raise ValueError("unsupported GGUF version")+ n_tensors = u64()+ n_kv = u64()+ if n_tensors > 10_000_000 or n_kv > 10_000_000:+ raise ValueError("implausible header counts")+ arch: Optional[str] = None+ alignment = 32 # GGUF's default when general.alignment is absent+ for _ in range(n_kv):+ key = string()+ vtype = u32()+ if key == b"general.architecture" and vtype == _STRING:+ arch = string().decode("utf-8", "replace")+ if arch_only:+ return arch, {}+ elif key == b"general.alignment" and vtype == _UINT32:+ alignment = max(1, u32())+ else:+ skip_value(vtype)+ if arch_only:+ return arch, {}+ named: dict[str, int] = {}+ offsets: list[int] = []+ unsized: list[tuple[str, int]] = []+ for _ in range(n_tensors):+ name = string()+ n_dims = u32()+ if n_dims > 16:+ raise ValueError("implausible tensor rank")+ dims = [u64() for _ in range(n_dims)]+ ggml_type = u32()+ offset = u64()+ offsets.append(offset)+ if name not in _WANTED:+ continue+ block = GGML_QUANT_SIZES.get(ggml_type)+ if block is None:+ unsized.append((name.decode("utf-8", "replace"), offset))+ continue+ n = 1+ for d in dims:+ n *= int(d)+ named[name.decode("utf-8", "replace")] = n // block[0] * block[1]+ if unsized:+ data_start = -(-pos // alignment) * alignment+ ordered = sorted(offsets)+ for name, offset in unsized:+ later = ordered[bisect.bisect_right(ordered, offset) :]+ if later:+ named[name] = later[0] - offset+ elif file_size is not None:+ named[name] = max(0, file_size - data_start - offset)+ return arch, named+++def _read_growing(+ read: Callable[[int], bytes],+ cap: int = MAX_HEADER_BYTES,+ *,+ first_read: int = _FIRST_READ,+ file_size: Optional[Callable[[], Optional[int]]] = None,+ arch_only: bool = False,+):+ want = min(first_read, cap)+ while True:+ buf = read(want)+ try:+ return parse_header(+ buf, file_size() if file_size is not None else None, arch_only = arch_only+ )+ except _NeedMore:+ if len(buf) < want or want >= cap:+ raise ValueError("GGUF header unreadable within the cap")+ want = min(want * 2, cap)+++def header_of_local(path: str) -> tuple[Optional[str], dict[str, int]]:+ with open(path, "rb") as f:+ size = os.fstat(f.fileno()).st_size++ def read(n: int) -> bytes:+ f.seek(0)+ return f.read(n)++ return _read_growing(read, file_size = lambda: size)+++def shard_names(filename: str) -> list[str]:+ """Every shard of ``filename`` in order (itself alone when it is not a split)."""+ head, sep, base = str(filename).replace("\\", "/").rpartition("/")+ match = _SPLIT_RE.match(base)+ if not match:+ return [filename]+ prefix, _index, total, ext = match.groups()+ return [+ f"{head}{sep}{prefix}-{i:05d}-of-{int(total):05d}{ext}" for i in range(1, int(total) + 1)+ ]+++_LOCAL_CACHE: "OrderedDict[tuple, tuple[Optional[str], int]]" = OrderedDict()+_REMOTE_CACHE: "OrderedDict[tuple, tuple[tuple[Optional[str], int], float]]" = OrderedDict()+_REMOTE_TTL_S = 6 * 3600.0+_CACHE_LIMIT = 512+_lock = threading.Lock()+++def _remember(cache: OrderedDict, key, value) -> None:+ with _lock:+ cache[key] = value+ cache.move_to_end(key)+ while len(cache) > _CACHE_LIMIT:+ cache.popitem(last = False)+++MAX_SHARDS_READ = 8+++def _scan_shards(+ read_header: Callable[[str], tuple],+ names: list[str],+ mode: str = "auto",+ max_shards: Optional[int] = MAX_SHARDS_READ,+ known_arch: Optional[str] = None,+):+ """(architecture, lazy table bytes under ``mode``), stopping at the first shard carrying the+ table."""+ arch: Optional[str] = None+ order = names[:max_shards]+ hinted = lazy_architecture(known_arch) and len(names) > 1+ if hinted:+ order = names[1:max_shards] + names[:1]+ for name in order:+ part_arch, named = read_header(name)+ if hinted and not arch and not part_arch:+ if not named:+ continue+ arch = read_header(names[0], True)[0]+ arch = part_arch or arch+ if not lazy_architecture(arch):+ return arch, 0+ found = lazy_read_bytes(arch, named, mode)+ if found or named:+ return arch, found+ return arch, 0+++def local_lookup_info(path: Optional[str], mode: str = "auto") -> tuple[Optional[str], int]:+ """(architecture, table bytes llama.cpp reads lazily under ``mode``) for a local GGUF."""+ if not path:+ return None, 0+ try:+ names = shard_names(path)+ key = (mode,) + tuple(+ (p, st.st_size, st.st_mtime_ns) for p in names for st in (os.stat(p),)+ )+ except OSError:+ return None, 0+ with _lock:+ hit = _LOCAL_CACHE.get(key)+ if hit is not None:+ return hit+ try:+ # Local headers are cheap: every shard, as the launch scanner reads them.+ answer = _scan_shards(header_of_local, names, mode, max_shards = None)+ except Exception: # noqa: BLE001 -- unreadable header: price the whole file, as before+ answer = (None, 0)+ _remember(_LOCAL_CACHE, key, answer)+ return answer+++def local_lookup_table_bytes(path: Optional[str]) -> int:+ return local_lookup_info(path)[1]+++def remote_cached(+ repo_id: str,+ filename: str,+ revision: Optional[str],+ identity = None,+ *,+ mode: str = "auto",+):+ """The cached remote answer under lazy ``mode``, keyed on the file's identity (its listed size by default), so a re-upload under the same name is read again."""+ with _lock:+ hit = _REMOTE_CACHE.get((repo_id.lower(), filename, revision or "", identity, mode))+ if hit is None or hit[1] <= time.monotonic():+ return None+ return hit[0]+++def remote_lookup_info(+ repo_id: str,+ filename: str,+ token = None,+ *,+ revision: Optional[str] = None,+ identity = None,+ fetcher_fn: Optional[Callable] = None,+ mode: str = "auto",+ known_arch: Optional[str] = None,+) -> tuple[Optional[str], int]:+ """(architecture, lazily read table bytes) from Range reads of a Hub GGUF's header(s)."""+ # Hub repo ids are case-insensitive.+ key = (repo_id.lower(), filename, revision or "", identity, mode)+ hit = remote_cached(repo_id, filename, revision, identity, mode = mode)+ if hit is not None:+ return hit+ if fetcher_fn is None:+ from hub.utils.gguf_header_delta import hub_range_fetcher as fetcher_fn++ names = shard_names(filename)++ def read_header(name: str, arch_only: bool = False):+ fetch = fetcher_fn(repo_id, name, token, revision = revision)+ buf = b""++ def read(n: int) -> bytes:+ nonlocal buf+ if n > len(buf):+ buf += fetch(len(buf), n - 1)+ return buf[:n]++ return _read_growing(+ read,+ first_read = (+ _FIRST_READ if name == names[0] and not arch_only else _SPLIT_SHARD_FIRST_READ+ ),+ file_size = lambda: getattr(fetch, "file_size", None),+ arch_only = arch_only,+ )++ try:+ answer = _scan_shards(read_header, names, mode, known_arch = known_arch)+ except Exception: # noqa: BLE001 -- unreachable or unreadable: today's behaviour+ return None, 0+ _remember(_REMOTE_CACHE, key, (answer, time.monotonic() + _REMOTE_TTL_S))+ return answerdiff --git a/studio/backend/models/inference.py b/studio/backend/models/inference.pyindex 0e4b5874922..7b19c099da9 100644--- a/studio/backend/models/inference.py+++ b/studio/backend/models/inference.py@@ -1322,6 +1322,17 @@ class EstimateMemoryResponse(BaseModel):description = "True when --n-cpu-moe is set: experts move per-tensor, not ""per-block, so the GPU/host split above ignores it and reads high.",)+ lookup_table_bytes: int = Field(+ 0,+ description = "Lookup-only tables (per-layer embeddings / Engram) in the model "+ "file. Never in gpu_bytes; excluded from weights_bytes and total_bytes when "+ "lookup_table_on_disk is true.",+ )+ lookup_table_on_disk: bool = Field(+ False,+ description = "True when llama.cpp streams those tables from disk (its default "+ "lazy mode), so only the rows in use are paged in.",+ )mlx_offload: Optional[MlxOffloadEstimate] = Field(None,description = "MLX: what the load reads from disk instead of memory, priced in "@@ -1434,6 +1445,12 @@ class MemoryEstimate(BaseModel):moe_offload_unmodelled: bool = Field(False, description = "--n-cpu-moe is set, so the GPU figure reads high")+ lookup_table_bytes: int = Field(+ 0, description = "Lookup-only tables in the model file, never charged to the GPU"+ )+ lookup_table_on_disk: bool = Field(+ False, description = "Those tables are streamed from disk, not resident"+ )mlx_offload: Optional[MlxOffloadEstimate] = Field(None, description = "MLX: what the load reads from disk; null when not MLX")diff --git a/studio/backend/models/models.py b/studio/backend/models/models.pyindex d89c011d305..daaf86754d5 100644--- a/studio/backend/models/models.py+++ b/studio/backend/models/models.py@@ -192,6 +192,11 @@ class GgufVariantDetail(BaseModel):)size_bytes: int = Field(0, description = "File size in bytes")download_size_bytes: int = Field(0, description = "Total bytes needed to download this variant")+ lookup_table_bytes: int = Field(+ 0,+ description = "Bytes of lazily read lookup tables (per-layer embeddings / Engram) "+ "llama.cpp streams from disk by default. Part of size_bytes; 0 when none or unknown.",+ )# Mirrors hub.schemas.inventory.GgufVariantDetail; see display_label above.pending_drafter_filename: Optional[str] = Field(None, description = "Missing separate drafter when the main GGUF is already cached"diff --git a/studio/backend/routes/inference.py b/studio/backend/routes/inference.pyindex cfcd82d0ba4..9c29fc543d6 100644--- a/studio/backend/routes/inference.py+++ b/studio/backend/routes/inference.py@@ -13249,6 +13249,8 @@ class _GgufMemoryBreakdown(NamedTuple):gpu_layers: Optional[int]# Host-resident checkpoint share of kv_bytes. Keep last for positional callers.kv_checkpoint_bytes: int = 0+ lookup_table_bytes: int = 0+ lookup_table_on_disk: bool = Falsedef _gguf_offloaded_layer_fraction(@@ -13849,6 +13851,7 @@ def _gguf_memory_breakdown(tensor_split_possible: bool = True,# A GPU pin owns placement: load_model strips device flags and env before launch.device_pin_governs: bool = False,+ selected_gpu_ids: Optional[list[int]] = None,) -> Optional[_GgufMemoryBreakdown]:"""None if nothing sizes. Files are cached; the KV and compute buffers are re-derived per call."""from core.inference.llama_server_args import (@@ -14064,6 +14067,223 @@ def _gguf_memory_breakdown(# Only the main weight is split by --gpu-layers; companions have their own placement flags.main_bytes = min(gpu_files_bytes, LlamaCppBackend._get_gguf_size_bytes(gguf_path))companion_bytes = max(0, gpu_files_bytes - main_bytes)+ from core.inference.llama_cpp import (+ _effective_lazy_mode,+ _is_per_layer_embedding,+ _override_places_tensor_on_device,+ _without_torch_gpu_probe,+ )+ from hub.utils.gguf_lookup_tables import local_lookup_info++ _lt_backend = get_llama_cpp_backend()+ try:+ _lt_flag = str(+ _lt_backend.probe_server_capabilities().get("lazy_mode_flag") or "--lazy-mode"+ )+ except Exception: # noqa: BLE001 -- unknown build: the default spelling+ _lt_flag = "--lazy-mode"+ _lt_arch, _lt_bytes = local_lookup_info(+ gguf_path,+ "on" if _effective_lazy_mode(extras, os.environ, flag = _lt_flag) == "on" else "auto",+ )+ lookup_table_bytes = min(main_bytes, _lt_bytes)+ lookup_table_on_disk = False+ _lt_extras, _lt_env = extras, dict(os.environ)+ if selected_gpu_ids or device_pin_governs:+ _lt_extras = LlamaCppBackend._strip_device_extra_args(extras)+ LlamaCppBackend._clear_device_placement_env(_lt_env)+ if gpu_memory_mode == "manual":+ LlamaCppBackend._clear_manual_placement_env(_lt_env)+ LlamaCppBackend._reconcile_split_mode_env(_lt_env, tensor_parallel = bool(tensor_parallel))+ if lookup_table_bytes:+ lookup_table_on_disk = _lt_backend.lookup_tables_stream_by_default(+ _lt_extras,+ _lt_env,+ gpu_ids = selected_gpu_ids,+ tensor_parallel = tensor_parallel,+ )+ if not lookup_table_on_disk:+ try:+ _lt_scan = _lt_backend._gguf_tensor_scan(gguf_path)+ except Exception: # noqa: BLE001 -- unread header: keep the lazy answer+ _lt_scan = None+ if _lt_scan:+ _lt_arch = _lt_arch or _lt_scan[0]+ _resident = sum(+ int(v) for k, v in (_lt_scan[1] or {}).items() if _is_per_layer_embedding(k)+ )+ lookup_table_bytes = min(main_bytes, max(lookup_table_bytes, _resident))++ def _lt_override_moves_table() -> bool:+ if _lt_arch in _lt_backend._host_forced_table_archs():+ return False # this build keeps the table on the host whatever -ot says+ try:+ scan = _lt_backend._gguf_tensor_scan(gguf_path)+ except Exception: # noqa: BLE001 -- unread header: the canonical name alone+ scan = None+ names = [k for k in ((scan[1] if scan else None) or {}) if _is_per_layer_embedding(k)]+ return any(+ _override_places_tensor_on_device(name, _lt_extras, _lt_env)+ for name in names or ["per_layer_token_embd.weight"]+ )++ _lt_device_bytes = 0+ if lookup_table_bytes:+ if not lookup_table_on_disk and _lt_override_moves_table():+ _lt_device_bytes, lookup_table_bytes = lookup_table_bytes, 0+ if lookup_table_on_disk:+ from core.inference.llama_cpp import (+ _HOST_RAM_HEADROOM_MIB,+ _LAZY_MODE_SPELLINGS,+ _LAZY_RESIDENT_MARGIN_FRAC,+ _extra_args_set_any_flag,+ _kv_offload_from_args,+ _device_selection_is_cpu,+ _pass_through_overrides_placement,+ _pass_through_reprices_cache,+ _prompt_cache_host_bytes,+ _sidecar_adapter_bytes,+ _sidecar_adapter_paths,+ )+ from core.inference.llama_server_args import fit_target_margin_in, resolve_requested_ctx++ _lt_flags, _lt_env_name = _LAZY_MODE_SPELLINGS.get(_lt_flag, (frozenset({_lt_flag}), ""))+ _lt_user = bool(+ (_lt_env_name and str(_lt_env.get(_lt_env_name) or "").strip())+ or _extra_args_set_any_flag(_lt_extras, _lt_flags)+ )+ _lt_avail = LlamaCppBackend._available_system_memory_mib()+ _lt_device_share = max(0, main_bytes - lookup_table_bytes)+ _lt_forced_host = int(_lt_device_share * max(0.0, 1.0 - gpu_fraction))+ _lt_table_host = lookup_table_bytes + int(lookup_table_bytes * _LAZY_RESIDENT_MARGIN_FRAC)++ def _lt_resident_fits() -> bool:+ if not _lt_avail or not runtime.kv_estimable:+ return False # unsized: the launch keeps llama.cpp's default too+ if _sidecar_adapter_bytes(_lt_extras) is None:+ return False+ # A pass-through -c 0 or larger --parallel sizes a cache runtime did not price.+ if _pass_through_reprices_cache(_lt_extras, n_parallel):+ return False+ # A CPU child keeps llama.cpp's default; Manual Auto layers without a context leave it+ # to --fit.+ if _lt_backend._child_runs_on_cpu(+ _lt_backend._find_llama_server_binary(),+ _lt_extras,+ _lt_env,+ cpu_forced = bool(_gpu_placement_paravirtual()),+ tensor_parallel = bool(tensor_parallel),+ ):+ return False+ _lt_manual = gpu_memory_mode == "manual"+ if (+ _lt_manual+ and (gpu_layers if gpu_layers is not None else -1) < 0+ and resolve_requested_ctx(_lt_extras, n_ctx) <= 0+ ):+ return False+ prompt = _prompt_cache_host_bytes(+ None,+ _lt_extras,+ _lt_env,+ supports_cache_ram = bool(+ (_lt_backend.probe_server_capabilities() or {}).get("supports_cache_ram")+ ),+ )+ if prompt is None:+ return False+ host_only = _lt_forced_host + _lt_table_host + prompt+ if not _kv_offload_from_args(_lt_extras, env = _lt_env):+ host_only += runtime.kv_bytes+ if not drafter_on_gpu:+ host_only += host_drafter_bytes + drafter_runtime_bytes+ _lt_mm = _charged_projector_bytes(config, extras, disable_vision)+ _lt_mm = _lt_mm if _lt_mm and _lt_mm <= companion_bytes else 0+ _lt_mm_runtime = int(_lt_mm * (LlamaCppBackend._MMPROJ_VRAM_SAFETY - 1.0))+ if _lt_mm and _resolved_mmproj_offload(llama_extra_args) is False:+ host_only += _lt_mm + _lt_mm_runtime+ footprint = (+ _lt_device_share+ + companion_bytes+ + host_drafter_bytes+ + _lt_mm_runtime+ + adapter_bytes+ + runtime.kv_bytes+ + runtime.compute_bytes+ + drafter_runtime_bytes+ + _lt_table_host+ + prompt+ )+ from utils.hardware import is_apple_silicon++ if is_apple_silicon():+ if _sidecar_adapter_paths(_lt_extras):+ return False+ budget = _lt_backend._apple_metal_memory_budget_bytes()+ if budget <= 0:+ return False+ # Metal refuses past the wired limit: None is unknown, 0 used up.+ wired = _lt_backend._apple_metal_wired_headroom_bytes()+ if wired is not None:+ budget = min(budget, wired)+ if budget <= 0:+ return False+ return footprint <= min(+ budget, max(0, _lt_avail - _HOST_RAM_HEADROOM_MIB) * 1024 * 1024+ )+ try:+ with _without_torch_gpu_probe():+ rows = list(+ _lt_backend._get_gpu_memory(+ _lt_backend._find_llama_server_binary(), for_llama_server = True+ )+ or ()+ )+ except Exception: # noqa: BLE001 -- unread cards: the whole load in RAM+ rows = []+ pick = sorted(int(i) for i in selected_gpu_ids) if selected_gpu_ids else None+ _lt_vulkan = LlamaCppBackend._is_vulkan_backend()+ # Manual mode plans no cards at launch (gpus = []): the whole load is RAM.+ if (+ not rows+ or _lt_manual+ or _pass_through_overrides_placement(+ _lt_extras,+ _lt_env,+ len(pick) if pick is not None else len(rows),+ is_vulkan_backend = _lt_vulkan,+ gpu_indices = pick,+ )+ ):+ rows, pick = [], None+ _lt_rows = [(idx, free) for idx, free, _total in rows]+ return bool(+ _lt_backend._fits_without_paging(+ footprint,+ _lt_rows,+ gpu_indices = pick,+ # Vulkan reports total 0 for an iGPU; ROCm APUs and integrated CUDA SoCs pool+ # host RAM.+ shared_gpu_ids = _lt_backend._unified_pool_gpu_ids(+ _lt_rows,+ gpu_indices = pick,+ shared_gpu_ids = (+ {idx for idx, _free, total in rows if total <= 0}+ if _lt_vulkan+ else set()+ ),+ is_vulkan_backend = _lt_vulkan,+ ),+ host_only_bytes = host_only,+ vram_margin_mib = max(1024.0, fit_target_margin_in(_lt_extras, _lt_env) or 0.0),+ avail_mib = _lt_avail,+ )+ )++ if not _lt_user and not _lt_override_moves_table() and _lt_resident_fits():+ lookup_table_on_disk = False+ if lookup_table_on_disk:+ weights_bytes = max(0, weights_bytes - lookup_table_bytes)# Projector is the only charged companion that can sit off-GPU (--no-mmproj-offload).host_companion_bytes = 0# The projector costs more than its file: the encoder's buffers run about 1.3x it,@@ -14078,7 +14298,8 @@ def _gguf_memory_breakdown(if _resolved_mmproj_offload(llama_extra_args) is False:host_companion_bytes = min(companion_bytes, mmproj_bytes)gpu_weights = (- int(main_bytes * gpu_fraction)+ int((main_bytes - lookup_table_bytes - _lt_device_bytes) * gpu_fraction)+ + _lt_device_bytes+ int(adapter_bytes * gpu_fraction)+ companion_bytes- host_companion_bytes@@ -14144,6 +14365,8 @@ def _gguf_memory_breakdown(layer_count = layer_count,gpu_layers = gpu_layers if gpu_memory_mode == "manual" else None,kv_checkpoint_bytes = runtime.kv_checkpoint_bytes,+ lookup_table_bytes = lookup_table_bytes,+ lookup_table_on_disk = lookup_table_on_disk,)@@ -20357,6 +20580,7 @@ async def estimate_memory(tensor_split_possible = _tensor_split_possible(request.selected_gpu_ids or None),# A pin names the cards; the launch strips conflicting device flags.device_pin_governs = bool(request.selected_gpu_ids),+ selected_gpu_ids = pinned_gpu_ids,)breakdown = price(n_ctx = request.n_ctx or 0)if breakdown is None:diff --git a/studio/backend/routes/models.py b/studio/backend/routes/models.pyindex 95b84eff37b..eb738f8daaf 100644--- a/studio/backend/routes/models.py+++ b/studio/backend/routes/models.py@@ -4065,6 +4065,7 @@ async def get_kv_cache_estimate(null = {"kv_bytes": None,"weights_bytes": None,+ "lookup_table_bytes": None,"native_context": None,"spec_bytes": None,"n_ctx": None,@@ -4633,6 +4634,7 @@ async def get_gguf_variants(download_size_bytes = int(getattr(v, "download_size_bytes", v.size_bytes) or v.size_bytes),+ lookup_table_bytes = int(getattr(v, "lookup_table_bytes", 0) or 0),pending_drafter_filename = getattr(v, "pending_drafter_filename", None),pending_drafter_size_bytes = int(getattr(v, "pending_drafter_size_bytes", 0) or 0diff --git a/studio/backend/tests/test_compute_buffer.py b/studio/backend/tests/test_compute_buffer.pyindex bdb788a5a4e..a2a7703fcf1 100644--- a/studio/backend/tests/test_compute_buffer.py+++ b/studio/backend/tests/test_compute_buffer.py@@ -816,7 +816,8 @@ class TestPipelineParallelPredicate:def test_layer_path_scrubs_a_non_layer_split_mode_env(self):import inspect- compact = "".join(inspect.getsource(LlamaCppBackend.load_model).split())+ assert "self._reconcile_split_mode_env(env" in inspect.getsource(LlamaCppBackend.load_model)+ compact = "".join(inspect.getsource(LlamaCppBackend._reconcile_split_mode_env).split())assert 'if_inherited_smand_inherited_sm!="layer":' in compactassert 'env.pop("LLAMA_ARG_SPLIT_MODE",None)' in compactdiff --git a/studio/backend/tests/test_empty_gpu_probe_reason.py b/studio/backend/tests/test_empty_gpu_probe_reason.pyindex a51fe535456..ac0cc22b461 100644--- a/studio/backend/tests/test_empty_gpu_probe_reason.py+++ b/studio/backend/tests/test_empty_gpu_probe_reason.py@@ -265,7 +265,7 @@ class TestItDoesNotAssertTheLaunchOutcome:src = inspect.getsource(mod.LlamaCppBackend.load_model)warn = src.find("could not enumerate any GPU")- restore = src.find("gpu_indices = _picked_order")+ restore = src.find("self._picker_ordered_gpu_indices(")assert warn != -1 and restore != -1assert warn < restore, "if the pin moved above the warning, re-derive this guard"diff --git a/studio/backend/tests/test_llama_cpp_placement.py b/studio/backend/tests/test_llama_cpp_placement.pyindex b239f23913e..267c073c438 100644--- a/studio/backend/tests/test_llama_cpp_placement.py+++ b/studio/backend/tests/test_llama_cpp_placement.py@@ -162,7 +162,7 @@ def _launch("Process",(),{- "pid": 123,+ "pid": 2**31 - 1,"stdout": (),"poll": lambda self: None,"terminate": lambda self: None,@@ -1189,7 +1189,7 @@ def _launch_with_failing_drafter("Process",(),{- "pid": 123,+ "pid": 2**31 - 1,"stdout": (),"poll": lambda self: None,"terminate": lambda self: None,@@ -1297,7 +1297,7 @@ def test_a_drafter_that_fails_to_start_gives_its_slots_back(tmp_path):"Process",(),{- "pid": 123,+ "pid": 2**31 - 1,"stdout": (),"poll": lambda self: None,"terminate": lambda self: None,@@ -3267,7 +3267,7 @@ def _launch_with_text_only_fallback(backend, gguf, **load_kwargs):"Process",(),{- "pid": 123,+ "pid": 2**31 - 1,"stdout": (),"poll": lambda self: None,"terminate": lambda self: None,@@ -3454,7 +3454,7 @@ def _launch_with_vulkan_cpu_replay("Process",(),{- "pid": 123,+ "pid": 2**31 - 1,"stdout": (),"returncode": rc,"poll": lambda self: rc,@@ -5039,6 +5039,7 @@ class _RamStub:return False_fits_without_paging = LlamaCppBackend._fits_without_paging+ _unified_pool_gpu_ids = LlamaCppBackend._unified_pool_gpu_ids_FIT_LOAD_MODE = LlamaCppBackend._FIT_LOAD_MODE@@ -5303,15 +5304,33 @@ def test_an_igpu_only_child_gets_no_auto_lazy_discount(tmp_path, _moe_cache_host(False, [0, 1], set(), {0}, set(), False),(False, [0], set(), set(), {0}, True),(False, None, set(), set(), {0, 1}, True),+ (True, [0, 1], {1}, set(), set(), True),+ (True, [0, 1], set(), set(), set(), False),+ (True, [0, 1], {0, 1}, set(), set(), True),+ (True, None, set(), set(), set(), False),+ (True, None, {0, 1}, set(), set(), True),+ (False, [0, 1], set(), set(), {1}, False),+ (False, [0, 1], set(), set(), set(), False),+ (False, None, set(), set(), set(), False),+ (False, None, set(), set(), {1}, False),],ids = ["discrete","vulkan_igpu",- "vulkan_mixed",+ "vulkan_mixed_default","rocm_apu","rocm_mixed","cuda_soc","cuda_soc_unpinned",+ "vulkan_mixed_explicit",+ "vulkan_all_discrete_explicit",+ "vulkan_all_integrated_explicit",+ "vulkan_discrete_default",+ "vulkan_all_integrated_default",+ "cuda_mixed_mask",+ "cuda_all_discrete_explicit",+ "cuda_discrete_default",+ "cuda_mixed_default",],)def test_lazy_auto_resolves_off_only_where_llama_cpp_does(@@ -5348,6 +5367,270 @@ def test_lazy_auto_resolves_off_only_where_llama_cpp_does(assert (auto, on) == ((0 if expected else _QWEN38_PLE), _QWEN38_PLE)+def _mixed_cuda_soc_host(monkeypatch):+ """A discrete card (0) beside an integrated SoC GPU (1) on a CUDA build."""+ for name, value in (+ ("_rocm_unified_memory_gpu_ids", set()),+ ("_integrated_cuda_gpu_ids", {1}),+ ("_integrated_cuda_probe_is_free", True),+ ("_resolve_visible_physical_ids", [0, 1]),+ ):+ monkeypatch.setattr(LlamaCppBackend, name, staticmethod(lambda value = value: value))+++def _mixed_cuda_launch(+ tmp_path,+ ram_gib = 12,+ memory = ((0, 8_000, 16_000), (1, 1_000, 1_000)),+ **kw,+):+ return _cache_launch(+ tmp_path,+ ram_gib = ram_gib,+ lazy = {"per_layer_token_embd.weight": 12 * _GIB},+ arch = "qwen4exp",+ caps = dict(_CACHE_CAPS_ON, supports_moe_cache_auto = False),+ memory = list(memory),+ **kw,+ )+++def _apu_qwen38(+ tmp_path,+ monkeypatch,+ *,+ gguf_gb,+ avail_gib,+ memory,+ apu_ids = (0,),+):+ backend, gguf = _apu_backend(+ tmp_path, gguf_gb = gguf_gb, avail_mib = avail_gib * 1024, monkeypatch = monkeypatch+ )+ backend._get_gpu_memory = lambda *_a, **_kw: list(memory)+ backend._get_gpu_free_memory = lambda *_a, **_kw: [(i, f) for i, f, _t in memory]+ monkeypatch.setattr(+ LlamaCppBackend, "_rocm_unified_memory_gpu_ids", staticmethod(lambda: set(apu_ids))+ )+ backend._gguf_tensor_scan = lambda _path: (+ "qwen4exp",+ {"per_layer_token_embd.weight": _QWEN38_PLE},+ 0,+ )+ caps = dict(LlamaCppBackend.probe_server_capabilities.__func__(LlamaCppBackend, None))+ caps.update(_CACHE_CAPS_ON)+ backend.probe_server_capabilities = lambda _binary = None: caps+ return backend, gguf+++ "load_kwargs",+ [dict(), dict(gpu_ids = [0, 1])],+ ids = ["default_selection", "picker_mask"],+)+def test_a_mixed_cuda_host_keeps_the_lazy_discount_through_the_preflight(+ tmp_path, _moe_cache_host, monkeypatch, load_kwargs+):+ """No --device list."""+ _mixed_cuda_soc_host(monkeypatch)+ backend, cmd = _mixed_cuda_launch(tmp_path, host_guard = True, **load_kwargs)+ assert _load_mode(cmd) == "none", cmd+ assert backend.last_load_warning is None, backend.last_load_warning+++def test_an_explicit_device_list_naming_an_igpu_turns_lazy_auto_off(monkeypatch):+ """A real --device list is used as given, iGPU included, so ANY integrated device turns auto off."""+ _mixed_cuda_soc_host(monkeypatch)+ off = LlamaCppBackend()._lazy_auto_resolves_off+ kw = dict(+ gpu_indices = [0, 1],+ detected_gpus = [(0, 8_000), (1, 1_000)],+ shared_gpu_ids = set(),+ is_vulkan_backend = False,+ )+ assert off(**kw) is False+ for extra_args, env, want in (+ (["--device", "CUDA0,CUDA1"], {}, True),+ ([], {"LLAMA_ARG_DEVICE": "CUDA0,CUDA1"}, True),+ (["-dev", "CUDA1"], {}, True),+ (["--device", "CUDA0"], {}, False),+ (["--device", "CUDA0"], {"CUDA_VISIBLE_DEVICES": "1"}, True),+ (["--device", "none"], {}, False),+ ):+ assert off(**kw, extra_args = extra_args, env = env) is want, (extra_args, env)+ vk = dict(kw, is_vulkan_backend = True, shared_gpu_ids = {1})+ assert off(**vk) is True+ assert off(**dict(vk, gpu_indices = None)) is False+ assert off(**vk, device_list = False) is False+ dev = ["--device", "Vulkan0,Vulkan1"]+ assert off(**vk, device_list = False, extra_args = dev, env = {}) is True+++def test_an_explicit_pick_reaches_every_lazy_decision(tmp_path, _moe_cache_host, monkeypatch):+ """gpu_ids becomes the child's selection, so every lazy decision of the fit must see it."""+ _mixed_cuda_soc_host(monkeypatch)+ seen = []+ real = LlamaCppBackend._lazy_auto_resolves_off++ def spy(self, **kw):+ seen.append(kw.get("gpu_indices"))+ return real(self, **kw)++ monkeypatch.setattr(LlamaCppBackend, "_lazy_auto_resolves_off", spy)+ _mixed_cuda_launch(tmp_path, gpu_ids = [0, 1])+ assert seen, "no lazy decision ran"+ assert all(sel is not None and set(sel) <= {0, 1} for sel in seen), seen+++def test_split_mode_none_checks_only_the_main_device(monkeypatch):+ """-sm none keeps only devices[main_gpu] of the model list (src/llama.cpp llama_prepare_model_devices)."""+ _mixed_cuda_soc_host(monkeypatch)+ off = LlamaCppBackend()._lazy_auto_resolves_off+ vk = dict(+ gpu_indices = None,+ detected_gpus = [(0, 8_000), (1, 1_000)],+ shared_gpu_ids = {1},+ is_vulkan_backend = True,+ env = {},+ )+ dev = ["--device", "Vulkan0,Vulkan1"]+ assert off(**vk, extra_args = dev) is True+ assert off(**vk, extra_args = [*dev, "-sm", "none"]) is False+ assert off(**vk, extra_args = [*dev, "-sm", "none", "-mg", "1"]) is True+ assert off(**vk, extra_args = [*dev, "--split-mode", "none", "-mg", "-1"]) is False+ pick = dict(vk, extra_args = ["-sm", "none"])+ assert off(**dict(pick, gpu_indices = [0, 1])) is False+ assert off(**dict(pick, gpu_indices = [1, 0])) is True+ cuda = dict(+ vk, is_vulkan_backend = False, shared_gpu_ids = set(), env = {"LLAMA_ARG_SPLIT_MODE": "none"}+ )+ assert off(**cuda, extra_args = ["--device", "CUDA1,CUDA0"]) is True+ assert off(**cuda, extra_args = ["--device", "CUDA0,CUDA1"]) is False+++def test_the_apu_guard_reads_the_childs_device_list(tmp_path, monkeypatch):+ """APU 0 beside a discrete card 1, the user naming the APU with --device ROCm0."""+ monkeypatch.setattr(+ LlamaCppBackend, "_resolve_visible_physical_ids", staticmethod(lambda: [0, 1])+ )+ backend, gguf = _apu_qwen38(+ tmp_path,+ monkeypatch,+ gguf_gb = 64.6,+ avail_gib = 46,+ memory = [(0, 60_000, 60_000), (1, 8_000, 8_000)],+ )++ cmd = _launch(backend, gguf, extra_args = ["--load-mode", "none", "--device", "ROCm0"])["cmd"]++ assert "unified-memory APU" in (backend.last_load_warning or ""), backend.last_load_warning+ assert not _unmapped_tokens(cmd), cmd+++def test_the_arch_crash_retry_prices_ram_against_its_own_mask():+ """The retry's RAM checks run after its mask / device / split rewrites and see the retry argv and env."""+ import inspect++ src = inspect.getsource(LlamaCppBackend.load_model)+ retry = src[src.index("_retry_wants_unified = self._amd_apu_wants_unified_memory(") :]+ retry = retry[: retry.index('label = "-archfallback"')]+ host = retry.index("_retry_host_msg = self._launch_host_shortfall_message(")+ apu = retry.index("_retry_apu_msg = self._apu_ram_shortfall_message(")+ for rewrite in (+ "self._emit_child_gpu_visibility(",+ "_widen_pin_ids_for_companion_devices(",+ "self._clear_split_placement_env(env)",+ "self._without_tensor_split(cmd)",+ ):+ assert retry.index(rewrite) < min(host, apu), rewrite+ lazy = retry[retry.rindex("self._lazy_auto_resolves_off(", 0, apu) : apu]+ assert "extra_args = cmd" in lazy and "env = env" in lazy, lazy+ kill = retry.index("self._kill_process()")+ assert retry.index("_retry_host_avail_mib = self._available_system_memory_mib()") < kill+ assert "avail_mib = _retry_host_avail_mib" in retry[host:]+++def test_tensor_mode_checks_every_visible_device(monkeypatch):+ """--split-mode tensor builds one Meta device from every visible GPU, mmap only if all members support it (ggml-backend-meta.cpp)."""+ _mixed_cuda_soc_host(monkeypatch)+ off = LlamaCppBackend()._lazy_auto_resolves_off+ cuda = dict(+ gpu_indices = None,+ detected_gpus = [(0, 8_000), (1, 1_000)],+ shared_gpu_ids = set(),+ is_vulkan_backend = False,+ )+ assert off(**cuda) is False+ assert off(**cuda, tensor_parallel = True) is True+ assert off(**cuda, extra_args = ["-sm", "tensor"], env = {}) is True+ assert off(**cuda, extra_args = [], env = {"LLAMA_ARG_SPLIT_MODE": "tensor"}) is True+ assert off(**cuda, extra_args = ["-sm", "tensor"], env = {}, tensor_parallel = False) is False+ assert off(**dict(cuda, gpu_indices = [0]), tensor_parallel = True) is False+ vk = dict(cuda, is_vulkan_backend = True, shared_gpu_ids = {1})+ assert off(**vk, device_list = False) is False+ assert off(**vk, device_list = False, extra_args = ["--split-mode", "tensor"], env = {}) is True+++def test_a_mixed_tensor_launch_prices_the_lazy_table(tmp_path, _moe_cache_host, monkeypatch):+ """Manual tensor parallelism over a discrete card and an integrated SoC, asking for --load-mode none."""+ _mixed_cuda_soc_host(monkeypatch)+ monkeypatch.setattr(+ LlamaCppBackend, "_effective_gpu_count", staticmethod(lambda gpu_indices = None: 2)+ )+ kwargs = dict(+ ram_gib = 6,+ memory = [(0, 8_000, 16_000), (1, 8_000, 8_000)],+ host_guard = True,+ gpu_memory_mode = "manual",+ gpu_layers = 20,+ load_mode = "none",+ )+ backend, layer = _mixed_cuda_launch(tmp_path, **kwargs)+ assert _load_mode(layer) == "none", layer+ assert backend.last_load_warning is None, backend.last_load_warning+ backend, cmd = _mixed_cuda_launch(tmp_path, tensor_parallel = True, **kwargs)+ assert cmd[cmd.index("--split-mode") + 1] == "tensor", cmd+ assert _load_mode(cmd) is None, cmd+ assert backend.last_load_warning, cmd+++def test_a_device_list_still_sees_an_auto_filtered_vulkan_igpu(monkeypatch):+ """Automatic Vulkan placement drops iGPUs from its candidates, so they leave _shared_gpu_ids."""+ import inspect++ off = LlamaCppBackend()._lazy_auto_resolves_off+ kw = dict(+ gpu_indices = None,+ detected_gpus = [(0, 8_000)],+ shared_gpu_ids = set(),+ is_vulkan_backend = True,+ extra_args = ["--device", "Vulkan0,Vulkan1"],+ env = {},+ )+ assert off(**kw) is False+ assert off(**kw, integrated_gpu_ids = {1}) is True+ assert off(**dict(kw, extra_args = ["--device", "Vulkan0"]), integrated_gpu_ids = {1}) is False+ src = inspect.getsource(LlamaCppBackend.load_model)+ assert src.index("_vulkan_igpu_ids = (") < src.index(+ "_gpu_mem = self._vulkan_auto_gpu_memory(_gpu_mem)"+ )+ assert src.count("integrated_gpu_ids = _vulkan_igpu_ids") == 5+++def test_the_oversized_readings_share_their_preflights_ram_snapshot():+ """Under the opt-out the oversized reading re-prices the shortfall with its neighbouring preflight's RAM figure."""+ import inspect++ src = inspect.getsource(LlamaCppBackend.load_model)+ for name, snapshot in (+ ("def _launch_is_oversized()", "avail_mib = _preflight_avail_mib"),+ ("def _retry_is_oversized()", "avail_mib = _retry_host_avail_mib"),+ ):+ body = src[src.index(name) :]+ body = body[: body.index("self._page_an_oversized_unmapped_load(")]+ assert snapshot in body, (name, body)++@pytest.mark.parametrize("lazy_args, warned", [(["-lzm", "on"], False), ([], True)])def test_the_apu_guard_prices_only_resident_weights(tmp_path, monkeypatch, lazy_args, warned):"""A 64.6 GiB model with a 26.82 GiB table on an APU with 46 GiB of RAM. "-lzm on"@@ -5372,6 +5655,17 @@ def test_the_apu_guard_prices_only_resident_weights(tmp_path, monkeypatch, lazy_assert bool(_unmapped_tokens(cmd)) is not warned, cmd[email protected]("memory", [[(0, 60_000, 60_000)], []], ids = ["probed", "failed_probe"])+def test_a_failed_gpu_probe_keeps_the_apu_guard_pricing_the_table(tmp_path, monkeypatch, memory):+ """An empty inventory on a ROCm APU still launches a HIP child, where auto is off, so the guard charges the table."""+ backend, gguf = _apu_qwen38(tmp_path, monkeypatch, gguf_gb = 73.45, avail_gib = 58, memory = memory)++ cmd = _launch(backend, gguf, extra_args = ["--load-mode", "none"])["cmd"]++ assert "unified-memory APU" in (backend.last_load_warning or "")+ assert not _unmapped_tokens(cmd), cmd++@pytest.mark.parametrize("flag, extra_args, env, expected",[diff --git a/studio/backend/tests/test_load_mode_fit_matrix.py b/studio/backend/tests/test_load_mode_fit_matrix.pyindex 084550cb73e..039c665a32c 100644--- a/studio/backend/tests/test_load_mode_fit_matrix.py+++ b/studio/backend/tests/test_load_mode_fit_matrix.py@@ -54,6 +54,7 @@ class _Stub:return self._is_apu_fits_without_paging = LlamaCppBackend._fits_without_paging+ _unified_pool_gpu_ids = LlamaCppBackend._unified_pool_gpu_ids_FIT_LOAD_MODE = LlamaCppBackend._FIT_LOAD_MODEdiff --git a/studio/backend/tests/test_lookup_table_fit.py b/studio/backend/tests/test_lookup_table_fit.pynew file mode 100644index 00000000000..275fc67f2a5--- /dev/null+++ b/studio/backend/tests/test_lookup_table_fit.py@@ -0,0 +1,1200 @@+# SPDX-License-Identifier: AGPL-3.0-only+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0++"""Lookup-only embedding tables (per-layer embeddings, the DeepSeek-V4.1 Engram table) in the launch fit, the offload layout and the planner."""++from __future__ import annotations++import os+import subprocess+import sys+from pathlib import Path++import pytest++_BACKEND_DIR = str(Path(__file__).resolve().parent.parent)+_TESTS_DIR = str(Path(__file__).resolve().parent)+for _d in (_BACKEND_DIR, _TESTS_DIR):+ if _d not in sys.path:+ sys.path.insert(0, _d)++import test_llama_cpp_placement as placement # noqa: E402, I001+from test_gpu_arch_gate_os_matrix_7624 import probe_env # noqa: F401, I001 -- fixture by name+from test_llama_cpp_placement import ( # noqa: F401, I001 -- fixtures by name+ _cache_launch,+ _discrete_linux_host,+ _moe_cache_host,+)+from test_offload_planner import _StubReader, _StubTensor, _shard_fields, _shard_tensors++from core.inference import llama_cpp as lc # noqa: E402+from core.inference.llama_cpp import LlamaCppBackend # noqa: E402+from core.inference.offload_layout import ( # noqa: E402+ BlockLayout,+ ModelLayout,+ _layout_from_reader,+)+from core.inference.offload_planner import PlanOptions, _finish # noqa: E402++GIB = 1024**3+PLE = "per_layer_token_embd.weight"+OT_TABLE = ["-ot", "per_layer_token_embd=CUDA0"]++DSV41_TOTAL = int(216.98 * GIB)+DSV41_ENGRAM = int(103.00 * GIB)+DSV41_TOKEN_EMBD = int(0.51 * GIB)+DSV41_OUTPUT = int(0.51 * GIB)++FLASH_NEXT_TOTAL = int(73.45 * GIB)+FLASH_NEXT_PLE = int(26.82 * GIB)+FLASH_NEXT_TOKEN_EMBD = int(0.41 * GIB)+FLASH_NEXT_OUTPUT = int(0.33 * GIB)++GEMMA3N_TOTAL = int(4.23 * GIB)+GEMMA3N_PLE = int(1.64 * GIB)+GEMMA3N_TOKEN_EMBD = int(0.28 * GIB)++_CAPS = {"supports_lazy_mode": True}+_FLASH_NAMED = {+ PLE: FLASH_NEXT_PLE,+ "token_embd.weight": FLASH_NEXT_TOKEN_EMBD,+ "output.weight": FLASH_NEXT_OUTPUT,+}+_DSV41_NAMED = {+ PLE: DSV41_ENGRAM,+ "token_embd.weight": DSV41_TOKEN_EMBD,+ "output.weight": DSV41_OUTPUT,+}+_GEMMA3N_NAMED = {PLE: GEMMA3N_PLE, "token_embd.weight": GEMMA3N_TOKEN_EMBD}+_INHERITED_CPU_SPLIT = {"LLAMA_ARG_SPLIT_MODE": "none", "LLAMA_ARG_MAIN_GPU": "-1"}+_RPC_LIST = ["--rpc", "10.0.0.2:50052", "--device", "Vulkan0,RPC0"]+_ONE = _RPC_LIST + ["--split-mode", "none"]+++def _scan_backend(arch, named):+ backend = LlamaCppBackend()+ backend._gguf_tensor_scan = lambda _path: (arch, dict(named), 0)+ return backend+++def _arg(cmd, flag):+ return cmd[cmd.index(flag) + 1]+++def _lazy_values(cmd):+ return [cmd[i + 1] for i, tok in enumerate(cmd) if tok in ("--lazy-mode", "-lzm")]+++def _setenv(monkeypatch, env):+ for name, value in env.items():+ monkeypatch.setenv(name, value)+++def _full_caps():+ caps = dict(LlamaCppBackend.probe_server_capabilities.__func__(LlamaCppBackend, None))+ caps.update(placement._CACHE_CAPS_ON)+ return lambda _binary = None: caps+++def _flash_next_launch(+ tmp_path,+ *,+ ram_gib = 128,+ memory = ((0, 23_800, 24_576),),+ **kwargs,+):+ return _cache_launch(+ tmp_path,+ size_gib = FLASH_NEXT_TOTAL / GIB,+ memory = list(memory),+ ram_gib = ram_gib,+ expert_gib = 40,+ lazy = _FLASH_NAMED,+ arch = "qwen4exp",+ **kwargs,+ )+++def _flash_next_on_two_cards(+ tmp_path,+ extra_args = (),+ **kwargs,+):+ """Flash-Next (73.45 GiB, 26.82 GiB table) on 2x32 GiB."""+ return _cache_launch(+ tmp_path,+ moe = False,+ size_gib = FLASH_NEXT_TOTAL / GIB,+ memory = [(0, 32_000, 32_768), (1, 32_000, 32_768)],+ ram_gib = 128,+ lazy = _FLASH_NAMED,+ arch = "qwen4exp",+ extra_args = list(extra_args),+ **kwargs,+ )+++def _gemma3n_launch(tmp_path, *, memory, ram_gib, **kwargs):+ return _cache_launch(+ tmp_path,+ moe = False,+ size_gib = GEMMA3N_TOTAL / GIB,+ memory = memory,+ lazy = _GEMMA3N_NAMED,+ arch = "gemma3n",+ ram_gib = ram_gib,+ **kwargs,+ )+++def _spy(monkeypatch, name, record):+ """Wrap a LlamaCppBackend method; ``record(answer, args, kwargs)`` appends what it saw."""+ seen = []+ real = getattr(LlamaCppBackend, name)++ def spy(self, *a, **k):+ answer = real(self, *a, **k)+ entry = record(answer, a, k)+ if entry is not None:+ seen.append(entry)+ return answer++ monkeypatch.setattr(LlamaCppBackend, name, spy)+ return seen+++def _spy_fit_load_mode(monkeypatch):+ return _spy(+ monkeypatch,+ "_fit_derived_load_mode",+ lambda _ans, _a, k: k if k.get("announce", True) else None,+ )+++def _apple_backend(arch, named, avail_gib):+ backend = _scan_backend(arch, named)+ backend._available_system_memory_mib = lambda: int(avail_gib * 1024)+ backend._can_estimate_kv = lambda: True+ return backend+++def _apple_discount(+ backend,+ probe = 0,+ *,+ model_size_fit = FLASH_NEXT_TOTAL,+ budget_gib = 96,+ caps = None,+ extra_args = (),+ env = None,+ **kwargs,+):+ return backend._apple_lookup_table_discount(+ probe,+ "/m.gguf",+ caps = dict(caps if caps is not None else dict(_CAPS, supports_cache_ram = True)),+ extra_args = list(extra_args),+ model_size_fit = model_size_fit,+ need_at_ctx = kwargs.pop("need_at_ctx", GIB),+ budget_mib = int(budget_gib * 1024),+ cache_ram = None,+ env = env or {},+ **kwargs,+ )+++def _managed_flags(+ backend,+ extra_args = (),+ resident_fits = lambda _t: True,+ caps = _CAPS,+ **kwargs,+):+ return backend._managed_lazy_mode_flags(+ "/m.gguf",+ caps = caps,+ extra_args = list(extra_args),+ env = {},+ resident_fits = resident_fits,+ **kwargs,+ )+++ "extra_args, auto_off, expected",+ [+ ([], False, DSV41_ENGRAM),+ ([], True, 0),+ (["-lzm", "off"], False, 0),+ (["-lzm", "on"], True, DSV41_ENGRAM),+ ],+ ids = ["auto", "igpu_auto", "off", "on_igpu"],+)+def test_the_deepseek41_engram_table_is_read_lazily(extra_args, auto_off, expected):+ backend = _scan_backend("deepseek41", {PLE: DSV41_ENGRAM})+ got = backend._lazy_read_host_bytes(+ "/m.gguf", caps = _CAPS, extra_args = extra_args, env = {}, auto_resolves_off = auto_off+ )+ assert got == expected+++def test_the_dsv41_host_preflight_no_longer_charges_the_engram_table(monkeypatch):+ """DSV4.1 UD-IQ1_S on a 24 GiB card with 64 GiB of RAM."""+ backend = _scan_backend("deepseek41", {PLE: DSV41_ENGRAM})+ backend._get_gguf_size_bytes = lambda _path: DSV41_TOTAL+ backend.probe_server_capabilities = lambda _binary = None: dict(_CAPS)+ backend._is_vulkan_backend = lambda _binary = None: False+ monkeypatch.setattr(LlamaCppBackend, "_lazy_auto_resolves_off", lambda self, **_kw: False)+ monkeypatch.delenv("UNSLOTH_ALLOW_HOST_OFFLOAD", raising = False)+ seen = []+ backend._host_offload_shortfall_message = (+ lambda offload, _avail: seen.append(offload) or "short"+ )+ msg = backend._launch_host_shortfall_message(+ ["/fake/llama-server", "-m", "/m.gguf"], [(0, 23_800)], env = {}, avail_mib = 64 * 1024+ )+ assert msg == "short"+ assert seen == [DSV41_TOTAL - DSV41_ENGRAM - 23_800 * 1024 * 1024]+ assert 90 * GIB < seen[0] < 91 * GIB+++def _gemma3n_layout(with_output = False):+ tensors = _shard_tensors(range(4)) + [+ _StubTensor("token_embd.weight", GEMMA3N_TOKEN_EMBD),+ _StubTensor(PLE, GEMMA3N_PLE),+ _StubTensor("output_norm.weight", 4096),+ ]+ if with_output:+ tensors.append(_StubTensor("output.weight", GEMMA3N_TOKEN_EMBD))+ return _layout_from_reader(_StubReader(_shard_fields(**{"llama.block_count": 4}), tensors))+++def test_the_per_layer_table_is_not_token_embd():+ layout = _gemma3n_layout()+ assert layout.complete+ assert layout.token_embd_bytes == GEMMA3N_TOKEN_EMBD+ assert layout.per_layer_embd_bytes == GEMMA3N_PLE+++def test_a_tied_file_duplicates_token_embd_but_not_the_per_layer_table():+ """gemma-3n is tied."""+ assert _gemma3n_layout().other_resident_bytes == GEMMA3N_TOKEN_EMBD + 4096+ untied = _gemma3n_layout(with_output = True)+ assert untied.other_resident_bytes == 4096+ assert untied.lm_head_bytes == GEMMA3N_TOKEN_EMBD+++def test_a_lazy_table_does_not_deny_load_mode_none_to_spilled_experts():+ """Flash-Next on 24 GiB + a 64 GB box with 48 GiB available."""+ layout = ModelLayout(+ arch = "qwen4exp",+ n_layers = 40,+ n_attention_layers = 10,+ blocks = tuple(+ BlockLayout(index = i, spillable_bytes = GIB, resident_bytes = int(0.14 * GIB))+ for i in range(40)+ ),+ lm_head_bytes = FLASH_NEXT_OUTPUT,+ token_embd_bytes = FLASH_NEXT_TOKEN_EMBD,+ per_layer_embd_bytes = FLASH_NEXT_PLE,+ tensor_bytes = FLASH_NEXT_TOTAL,+ other_resident_bytes = int(0.29 * GIB),+ kv_bytes_per_token_f16 = 40 * 1024,+ n_ctx_train = 262144,+ is_moe = True,+ n_expert = 256,+ n_expert_used = 8,+ complete = True,+ )+ chosen = list(layout.blocks[:25])+ lazy = _finish(+ layout, PlanOptions(lazy_read_bytes = FLASH_NEXT_PLE), 8192, chosen, False, 48 * GIB+ )+ assert lazy.load_mode_none+ assert lazy.host_bytes == FLASH_NEXT_TOKEN_EMBD + 25 * GIB+ resident = _finish(layout, PlanOptions(), 8192, chosen, False, 48 * GIB)+ assert not resident.load_mode_none+ assert resident.host_bytes == FLASH_NEXT_TOKEN_EMBD + FLASH_NEXT_PLE + 25 * GIB+++def test_host_pinned_inputs_for_the_three_families():+ pinned = lc._host_pinned_input_bytes+ assert pinned(("gemma3n", _GEMMA3N_NAMED, 0)) == GEMMA3N_PLE+ assert pinned(("qwen4exp", _FLASH_NAMED, 0)) == FLASH_NEXT_PLE + FLASH_NEXT_TOKEN_EMBD+ assert pinned(("deepseek41", _DSV41_NAMED, 0)) == DSV41_ENGRAM + DSV41_TOKEN_EMBD+ assert pinned(None) == 0+++def test_an_override_onto_a_device_keeps_a_resident_table_charged():+ on_device, pinned = lc._override_places_tensor_on_device, lc._host_pinned_input_bytes+ assert on_device(PLE, OT_TABLE)+ assert not on_device(PLE, ["-ot", "per_layer=CPU"])+ assert not on_device(PLE, ["-ot", "per_layer=CPU,embd=CUDA0"])+ assert not on_device(PLE, ["-ot", "exps=CPU"])+ assert on_device(PLE, [], {"LLAMA_ARG_OVERRIDE_TENSOR": "embd=CUDA0"})+ assert pinned(("qwen4exp", {PLE: FLASH_NEXT_PLE}, 0), OT_TABLE, {}) == 0+ # Upstream b11870 gathers the Engram rows in the graph, so -ot places it like any other table;+ # a build that keeps it on the host whatever -ot says still discounts it.+ engram = ("deepseek41", {PLE: DSV41_ENGRAM}, 0)+ assert pinned(engram, OT_TABLE, {}) == 0+ assert pinned(engram, OT_TABLE, {}, host_forced = {"deepseek41"}) == DSV41_ENGRAM+++def test_gemma3n_fits_a_small_card_once_its_tables_leave_the_vram_charge(tmp_path, _moe_cache_host):+ """gemma-3n-E4B Q4_K_M (4.23 GiB)."""+ _backend, cmd = _gemma3n_launch(tmp_path, memory = [(0, 9_000, 10_240)], ram_gib = 16)+ assert (_arg(cmd, "--fit"), _arg(cmd, "-ngl")) == ("off", "-1"), cmd+ assert _lazy_values(cmd) == [], cmd+++def test_an_override_moving_the_table_to_the_gpu_keeps_it_charged(tmp_path, _moe_cache_host):+ _backend, cmd = _gemma3n_launch(+ tmp_path, memory = [(0, 9_000, 10_240)], ram_gib = 16, extra_args = OT_TABLE+ )+ assert _arg(cmd, "--fit") == "on", cmd++[email protected]("ram_gib, none_ok", [(2, False), (16, True)])+def test_resident_input_tables_are_host_ram_in_the_load_mode_fit(+ tmp_path, _moe_cache_host, ram_gib, none_ok+):+ """gemma3n's table is resident (never lazily read) and host-pinned."""+ _backend, cmd = _gemma3n_launch(tmp_path, memory = [(0, 23_000, 24_576)], ram_gib = ram_gib)+ assert ("--load-mode" in cmd and _arg(cmd, "--load-mode") == "none") is none_ok, cmd+++ "ram_gib, lazy",+ [(128, ["off"]), (58, [])],+ ids = [+ "flash_next_keeps_its_table_resident_where_ram_holds_it",+ "flash_next_streams_its_table_on_a_64gb_box",+ ],+)+def test_flash_next_keeps_its_table_resident_where_ram_holds_it(+ tmp_path, _moe_cache_host, ram_gib, lazy+):+ """24 GiB card; 58 GiB of RAM is a 64 GB box."""+ backend, cmd = _flash_next_launch(tmp_path, ram_gib = ram_gib)+ assert _lazy_values(cmd) == lazy, cmd+ assert backend._lazy_mode_flags == (["--lazy-mode", "off"] if lazy else [])+++def test_a_stale_spelling_from_another_build_is_not_the_users_choice(+ tmp_path, _moe_cache_host, monkeypatch+):+ """llama.cpp reads only its own spelling, so an older build's env twin is not a user choice."""+ monkeypatch.setenv("LLAMA_ARG_TENSOR_READ_LAZY", "off")+ assert _lazy_values(_flash_next_launch(tmp_path)[1]) == ["off"]+++def test_the_resident_table_is_charged_once_in_the_load_mode_fit(tmp_path, _moe_cache_host):+ """After the resident verdict the table moves from the pooled weights to host-only RAM."""+ _backend, cmd = _flash_next_launch(tmp_path, ram_gib = 72)+ assert _lazy_values(cmd) == ["off"], cmd+ assert _arg(cmd, "--load-mode") == "none", cmd+++def test_a_retry_hands_the_table_back_to_llama_cpp():+ backend = LlamaCppBackend()+ backend._lazy_mode_flags = ["--lazy-mode", "off"]+ argv = ["/fake/llama-server", "-m", "/m.gguf", "--lazy-mode", "off", "-c", "4096"]+ assert backend._drop_managed_lazy_mode(argv, "CPU fallback") == [+ "/fake/llama-server",+ "-m",+ "/m.gguf",+ "-c",+ "4096",+ ]+ backend._lazy_mode_flags = []+ assert backend._drop_managed_lazy_mode(argv, "CPU fallback") == argv+++def test_a_device_override_keeps_the_table_streamed(tmp_path, _moe_cache_host):+ """Read lazily, the table sits on a CPU buffer whatever -ot says."""+ backend, cmd = _flash_next_launch(tmp_path, extra_args = OT_TABLE)+ assert _lazy_values(cmd) == [], cmd+ assert backend._lazy_mode_flags == []+++def test_the_in_place_fit_on_retry_hands_the_table_back(tmp_path, _moe_cache_host, monkeypatch):+ """A full offload launches with --fit off; its failed first spawn retries without the flag."""+ cmds = []++ def launch(+ backend,+ gguf,+ model_identifier = "test",+ **kwargs,+ ):+ def fake_popen(cmd, **kw):+ if not cmd or str(cmd[0]) != "/fake/llama-server":+ return placement._REAL_POPEN(cmd, **kw)+ spawn = "-m" in cmd+ if spawn:+ cmds.append(list(cmd))+ rc = (1 if len(cmds) == 1 else None) if spawn else 0+ return type(+ "Process",+ (),+ {+ "pid": 2**31 - 1,+ "stdout": (),+ "returncode": rc,+ "poll": lambda self: rc,+ "terminate": lambda self: None,+ "wait": lambda self, timeout = None: rc or 0,+ "kill": lambda self: None,+ },+ )()++ backend._wait_for_health = lambda timeout = None, **_kw: len(cmds) > 1+ with placement.patch.object(subprocess, "Popen", side_effect = fake_popen):+ backend.load_model(+ placement.GgufLoadIntent(+ gguf_path = str(gguf), model_identifier = model_identifier, **kwargs+ )+ )+ return {"cmd": cmds[-1]}++ monkeypatch.setattr(placement, "_launch", launch)+ _flash_next_launch(tmp_path, memory = [(0, 96_000, 98_304)])+ assert [_lazy_values(c) for c in cmds] == [["off"], []], cmds+++ "extra_args, expected",+ [(["-lzm", "auto"], ["auto"]), (["--lazy-mode", "on"], ["on"]), (["-lzm", "off"], ["off"])],+ ids = ["user_auto", "user_on", "user_off"],+)+def test_a_users_lazy_mode_wins(tmp_path, _moe_cache_host, extra_args, expected):+ backend, cmd = _flash_next_launch(tmp_path, extra_args = extra_args)+ assert _lazy_values(cmd) == expected, cmd+ assert backend._lazy_mode_flags == []+++def test_an_inherited_lazy_mode_wins(tmp_path, _moe_cache_host, monkeypatch):+ monkeypatch.setenv("LLAMA_ARG_LAZY_MODE", "on")+ assert _lazy_values(_flash_next_launch(tmp_path)[1]) == []+++def test_an_igpu_child_is_left_to_llama_cpp(tmp_path, _moe_cache_host, monkeypatch):+ monkeypatch.setattr(LlamaCppBackend, "_lazy_auto_resolves_off", lambda self, **_kw: True)+ assert _lazy_values(_flash_next_launch(tmp_path)[1]) == []+++def test_a_build_without_lazy_mode_gets_no_flag(tmp_path, _moe_cache_host):+ caps = dict(placement._CACHE_CAPS_ON, supports_lazy_mode = False)+ assert _lazy_values(_flash_next_launch(tmp_path, caps = caps)[1]) == []+++def test_a_paravirtual_launch_keeps_llama_cpps_lazy_default(tmp_path, _moe_cache_host, monkeypatch):+ monkeypatch.setattr(lc, "_metal_device_is_paravirtual", lambda: True)+ assert _lazy_values(_flash_next_launch(tmp_path)[1]) == []+++def test_dsv41_streams_its_engram_table_on_24gb_plus_64gb(tmp_path, _moe_cache_host):+ _backend, cmd = _cache_launch(+ tmp_path,+ size_gib = DSV41_TOTAL / GIB,+ memory = [(0, 23_800, 24_576)],+ ram_gib = 58,+ expert_gib = 100,+ lazy = _DSV41_NAMED,+ arch = "deepseek41",+ )+ assert _lazy_values(cmd) == [], cmd+++def test_the_decision_is_logged_once(monkeypatch):+ seen = []+ monkeypatch.setattr(lc.logger, "info", lambda msg, *a, **k: seen.append(str(msg)))+ backend = _scan_backend("qwen4exp", {PLE: FLASH_NEXT_PLE})+ for _ in range(3):+ assert _managed_flags(backend) == ["--lazy-mode", "off"]+ assert sum("Lookup tables: keeping" in m for m in seen) == 1+++def test_an_unpriced_fit_keeps_llama_cpp_default():+ backend = _scan_backend("qwen4exp", {PLE: FLASH_NEXT_PLE})++ def boom(_table):+ raise RuntimeError("unpriced")++ for verdict in (lambda _t: None, lambda _t: False, boom):+ assert _managed_flags(backend, resident_fits = verdict) == []+ old = {"supports_lazy_mode": True, "lazy_mode_flag": "--tensor-read-lazy"}+ assert _managed_flags(backend, caps = old) == ["--tensor-read-lazy", "off"]+++ "probe, want",+ [(0, DSV41_ENGRAM), (DSV41_ENGRAM + DSV41_TOKEN_EMBD, DSV41_ENGRAM + DSV41_TOKEN_EMBD)],+ ids = [+ "a_failed_metal_probe_still_leaves_the_engram_table_out",+ "a_successful_probe_keeps_its_larger_measured_discount",+ ],+)+def test_a_failed_metal_probe_still_leaves_the_engram_table_out(probe, want):+ """DSV4.1 UD-IQ1_S on a 128 GB Mac."""+ backend = _apple_backend("deepseek41", {PLE: DSV41_ENGRAM}, avail_gib = 119)+ got = _apple_discount(backend, probe, model_size_fit = DSV41_TOTAL)+ assert got == want+ assert backend._apple_lazy_flags == []+ if not probe:+ assert (DSV41_TOTAL - got) / GIB == pytest.approx(113.98, abs = 0.01)+++ "avail_gib, budget_gib, flags, want",+ [(119, 96, ["--lazy-mode", "off"], 0), (58, 48, [], FLASH_NEXT_PLE)],+ ids = [+ "flash_next_goes_resident_on_a_128gb_mac_and_stays_charged",+ "flash_next_streams_on_a_64gb_mac",+ ],+)+def test_flash_next_goes_resident_on_a_128gb_mac_and_stays_charged(+ avail_gib, budget_gib, flags, want+):+ backend = _apple_backend("qwen4exp", {PLE: FLASH_NEXT_PLE}, avail_gib = avail_gib)+ assert _apple_discount(backend, budget_gib = budget_gib) == want+ assert backend._apple_lazy_flags == flags+++def test_an_unsized_kv_cache_proves_nothing_on_a_mac():+ backend = _apple_backend("qwen4exp", {PLE: FLASH_NEXT_PLE}, avail_gib = 119)+ backend._can_estimate_kv = lambda: False+ assert _apple_discount(backend) == FLASH_NEXT_PLE+ assert backend._apple_lazy_flags == []+++def test_a_users_lazy_off_on_a_mac_takes_the_table_back_out_of_the_probe_discount():+ backend = _apple_backend("qwen4exp", {PLE: FLASH_NEXT_PLE}, avail_gib = 119)+ got = _apple_discount(+ backend, FLASH_NEXT_PLE + FLASH_NEXT_TOKEN_EMBD, caps = _CAPS, extra_args = ["-lzm", "off"]+ )+ assert got == FLASH_NEXT_TOKEN_EMBD+ assert backend._apple_lazy_flags == []+++def test_gemma3n_gets_no_metal_table_discount():+ """Not TENSOR_READ_LAZY upstream and under 4 GiB: only the probe can discount it."""+ backend = _apple_backend("gemma3n", _GEMMA3N_NAMED, avail_gib = 12)+ assert _apple_discount(backend, model_size_fit = GEMMA3N_TOTAL, budget_gib = 10) == 0+ assert backend._apple_lazy_flags == []+++ "lazy_args, expected", [([], FLASH_NEXT_PLE), (["-lzm", "off"], 0)], ids = ["auto", "off"]+)+def test_a_metal_override_moves_the_table_only_when_it_is_not_streamed(lazy_args, expected):+ """llama.cpp picks the lazy buffer before -ot, so a streamed table stays off Metal."""+ backend = _apple_backend("qwen4exp", {PLE: FLASH_NEXT_PLE}, avail_gib = 58)+ got = _apple_discount(+ backend,+ caps = _CAPS,+ extra_args = ["-ot", "per_layer_token_embd=MTL0", *lazy_args],+ budget_gib = 48,+ )+ assert got == expected+++def test_module_table_names_deepseek41():+ assert "deepseek41" in lc._LAZY_READ_TENSORS++[email protected]("name", ["LLAMA_ARG_MMPROJ", "LLAMA_ARG_MMPROJ_URL"])+def test_an_inherited_projector_on_a_mac_keeps_the_table_streamed(name):+ """The env's projector is unpriced in the Metal footprint, so a Mac that would read the table resident leaves it to auto."""+ backend = _apple_backend("qwen4exp", {PLE: FLASH_NEXT_PLE}, avail_gib = 119)+ assert _apple_discount(backend, caps = _CAPS, env = {name: "/p/mmproj.gguf"}) == FLASH_NEXT_PLE+ assert backend._apple_lazy_flags == []+++ "extra_args, env",+ [+ (_RPC_LIST, {}),+ (["--rpc", "10.0.0.2:50052"], {}),+ ([], {"LLAMA_ARG_RPC": "10.0.0.2:50052"}),+ (["--rpc", "10.0.0.2:50052", "--device", "RPC0"], {}),+ (["--rpc", "10.0.0.2:50052", "--device", "Vulkan0"], {}),+ ([], {}),+ (_ONE + ["--main-gpu", "0"], {}),+ (_ONE + ["--main-gpu", "1"], {}),+ (_ONE + ["--main-gpu", "-1"], {}),+ (_RPC_LIST, {"LLAMA_ARG_SPLIT_MODE": "none", "LLAMA_ARG_MAIN_GPU": "0"}),+ (["--rpc", "10.0.0.2:50052", "--split-mode", "none", "--main-gpu", "-1"], {}),+ (["--rpc", "10.0.0.2:50052", "--split-mode", "none", "--main-gpu", "0"], {}),+ ],+)+def test_an_rpc_model_device_keeps_lazy_auto_on(extra_args, env):+ """RPC devices report mmap support (ggml-rpc.cpp device_get_props, b11734-mix-c208e5f and b11870)."""+ got = LlamaCppBackend()._lazy_auto_resolves_off(+ gpu_indices = [0],+ detected_gpus = [(0, 24_000)],+ shared_gpu_ids = set(),+ is_vulkan_backend = True,+ probe_integrated_cuda = False,+ extra_args = extra_args,+ env = env,+ )+ assert got is False+++def test_an_integrated_gpu_beside_an_rpc_device_still_turns_lazy_auto_off():+ got = LlamaCppBackend()._lazy_auto_resolves_off(+ gpu_indices = [0, 1],+ detected_gpus = [(0, 24_000), (1, 8_000)],+ shared_gpu_ids = {1},+ is_vulkan_backend = True,+ probe_integrated_cuda = False,+ extra_args = ["--rpc", "10.0.0.2:50052", "--device", "Vulkan1,RPC0"],+ env = {},+ )+ assert got is True+++def test_a_streamed_table_stays_on_the_host_under_a_device_override():+ """llama.cpp picks the lazy buffer before -ot (llama-model-loader.cpp create_tensor)."""+ pinned = lc._host_pinned_input_bytes+ scan = ("qwen4exp", {PLE: FLASH_NEXT_PLE}, 0)+ assert pinned(scan, OT_TABLE, {}, lazy_mode = "auto") == FLASH_NEXT_PLE+ assert pinned(scan, OT_TABLE, {}, lazy_mode = "off") == 0+ small = ("gemma4", {PLE: GEMMA3N_PLE}, 0)+ assert pinned(small, OT_TABLE, {}, lazy_mode = "auto") == 0+ assert pinned(small, OT_TABLE, {}, lazy_mode = "on") == GEMMA3N_PLE+++ "lazy_args, fit",+ [([], "off"), (["-lzm", "on"], "off"), (["-lzm", "off"], "on")],+ ids = ["auto", "on", "off"],+)+def test_a_device_override_leaves_a_streamed_table_off_the_vram_charge(+ tmp_path, _moe_cache_host, lazy_args, fit+):+ _backend, cmd = _flash_next_on_two_cards(tmp_path, ["-ot", "embd=CUDA0", *lazy_args])+ assert _arg(cmd, "--fit") == fit, cmd+ if fit == "off":+ assert _arg(cmd, "-ngl") == "-1", cmd+++def test_a_streamed_table_under_an_override_keeps_the_resident_embedding_host_only(+ tmp_path, _moe_cache_host, monkeypatch+):+ seen = _spy_fit_load_mode(monkeypatch)+ _flash_next_on_two_cards(tmp_path, OT_TABLE)+ assert seen and seen[-1]["lazy_read_bytes"] == FLASH_NEXT_PLE+ assert seen[-1]["host_only_bytes"] >= FLASH_NEXT_TOKEN_EMBD, seen[-1]+++ "forced, flags", [(frozenset(), []), (frozenset({"deepseek41"}), ["--lazy-mode", "off"])]+)+def test_a_device_override_on_deepseek41_follows_the_builds_placement(monkeypatch, forced, flags):+ """Where -ot moves the Engram table, -lzm off would make it GPU-resident while priced on the host."""+ monkeypatch.setattr(+ LlamaCppBackend, "_host_forced_table_archs", staticmethod(lambda binary = None: forced)+ )+ backend = _scan_backend("deepseek41", {PLE: DSV41_ENGRAM})+ assert _managed_flags(backend, OT_TABLE) == flags+++def _server_beside(tmp_path, name, payload):+ server = tmp_path / "llama-server"+ server.write_bytes(b"\x7fELF server")+ (tmp_path / name).write_bytes(payload)+ return LlamaCppBackend._host_forced_table_archs(str(server))++[email protected]("marked", [True, False])+def test_the_host_forced_probe_reads_the_builds_loader(tmp_path, marked):+ """Studio's prebuilt b11734-mix-c208e5f carries the loader warning; upstream b11870 does not."""+ marker = lc._HOST_FORCED_TABLE_MARKER if marked else b"place it anywhere"+ got = _server_beside(tmp_path, "libllama.so.0.7.0", b"\x00" * 64 + marker + b" %s, using %s\n")+ assert got == (frozenset({"deepseek41"}) if marked else frozenset())+ gone = str(tmp_path / "gone" / "llama-server")+ assert LlamaCppBackend._host_forced_table_archs(gone) == frozenset()+++ "name, forced",+ [+ ("libllama.so.0.7.0", True),+ ("libllama.0.dylib", True),+ ("LLAMA.DLL", True),+ ("libllama.so.0.bak", False),+ ("llama.dll.disabled", False),+ ],+)+def test_the_host_forced_probe_reads_only_a_loadable_library(tmp_path, name, forced):+ got = _server_beside(tmp_path, name, lc._HOST_FORCED_TABLE_MARKER)+ assert got == (frozenset({"deepseek41"}) if forced else frozenset())+++def test_the_lazy_mode_choices_probe_the_launch_binary(monkeypatch):+ seen = []+ monkeypatch.setattr(+ LlamaCppBackend,+ "_host_forced_table_archs",+ staticmethod(lambda binary = None: seen.append(binary) or frozenset()),+ )+ backend = _apple_backend("deepseek41", {PLE: DSV41_ENGRAM}, avail_gib = 119)+ ot = ["-ot", "per_layer_token_embd=MTL0"]+ _managed_flags(backend, ot, binary = "/launch/llama-server")+ _apple_discount(+ backend,+ caps = _CAPS,+ extra_args = ot,+ model_size_fit = DSV41_TOTAL,+ binary = "/launch/llama-server",+ )+ assert seen and set(seen) == {"/launch/llama-server"}+++ "extra_args, resident",+ [([], True), (["--parallel", "1"], True), (["-c", "0"], False), (["--parallel", "8"], False)],+ ids = ["none", "same_parallel", "ctx_zero", "larger_parallel"],+)+def test_a_pass_through_that_reprices_the_cache_keeps_the_table_streamed(+ tmp_path, _moe_cache_host, extra_args, resident+):+ """-c 0 or a larger --parallel wins over the cache the resident verdict priced."""+ _backend, cmd = _flash_next_launch(tmp_path, extra_args = extra_args)+ assert (_lazy_values(cmd) == ["off"]) is resident, cmd+++ "extra_args, n_parallel, reprices",+ [+ ([], 1, False),+ (["-c", "0"], 1, True),+ (["--ctx-size", "8192"], 1, False),+ (["--parallel", "4"], 4, False),+ (["--parallel", "4"], 2, True),+ (["--parallel", "1"], 2, False),+ ],+)+def test_the_metal_resident_verdict_abstains_on_the_same_overrides(+ extra_args, n_parallel, reprices+):+ assert lc._pass_through_reprices_cache(extra_args, n_parallel) is reprices++[email protected]("cpu_build", [False, True])+def test_a_cpu_child_keeps_llama_cpps_lazy_default(+ tmp_path, _moe_cache_host, monkeypatch, cpu_build+):+ """The resident verdict credits the cards; a CPU-only build or --device none has none."""+ monkeypatch.setattr(+ LlamaCppBackend, "_binary_ships_no_gpu_backend", staticmethod(lambda *a, **k: cpu_build)+ )+ _backend, cmd = _flash_next_launch(tmp_path)+ assert (_lazy_values(cmd) == ["off"]) is not cpu_build, cmd+ _backend, cmd = _flash_next_launch(tmp_path, extra_args = ["--device", "none"])+ assert _lazy_values(cmd) == [], cmd+++def _metal_launched(tmp_path, monkeypatch, **kw):+ """A Metal load_model (16 GiB budget, 12 GiB weights); returns argv."""+ import test_metal_explicit_context_guard as metal+ return metal._launch(+ tmp_path,+ monkeypatch,+ n_ctx = 8192,+ kv_per_token = 32 * 1024,+ backend = metal._backend_with_embeddings(monkeypatch, embd = 0),+ real_fit = True,+ budget_bytes = 16 * GIB,+ weights_bytes = 12 * GIB,+ **kw,+ )["cmd"]+++ "extra_args, pinned",+ [+ (None, True),+ (["-ngl", "20"], False),+ (["--fit-target", "2048"], False),+ (["-ot", "blk=CPU"], False),+ ],+ ids = ["studio_owns", "user_layers", "user_fit_target", "user_cpu_override"],+)+def test_a_streamed_table_alone_never_pins_a_placement_the_user_owns(+ tmp_path, monkeypatch, extra_args, pinned+):+ """The table discount sizes the Metal fit; only Studio's own placement may be pinned on it."""+ from test_metal_explicit_context_guard import TestInputEmbeddingsLeftInTheFileMapping as T++ monkeypatch.setattr(+ LlamaCppBackend,+ "_apple_lookup_table_discount",+ lambda self, probe, *a, **k: max(int(probe), 6 * GIB),+ )+ cmd = _metal_launched(tmp_path, monkeypatch, extra_args = extra_args)+ assert (T._fit_tokens(cmd)[:4] == ["-ngl", "-1", "--fit", "off"]) is pinned, cmd+++def _spy_resident_inputs(monkeypatch):+ seen = []++ def spy(self, probe, *a, **k):+ seen.append(k)+ return int(probe)++ monkeypatch.setattr(LlamaCppBackend, "_apple_lookup_table_discount", spy)+ return seen+++def test_the_metal_resident_verdict_prices_the_context_checkpoints(tmp_path, monkeypatch):+ needs = []+ for ckpt in (0, GIB):+ seen = _spy_resident_inputs(monkeypatch)+ _metal_launched(+ tmp_path, monkeypatch, ckpt_bytes = ckpt, extra_args = ["--ctx-checkpoints", "8"]+ )+ needs.append(seen[-1]["need_at_ctx"])+ assert needs[1] > needs[0], needs+++ "kw, budget_gib",+ [+ ({"wired_bytes": 0}, 16),+ ({"wired_bytes": 8 * GIB}, 8),+ ({"wired_bytes": 32 * GIB}, 16),+ ({"wired_exhausted": True}, 0),+ ],+ ids = ["0-16", "8-8", "32-16", "used_up_wired_headroom_leaves_no_resident_budget"],+)+def test_the_metal_resident_verdict_obeys_the_wired_headroom(tmp_path, monkeypatch, kw, budget_gib):+ seen = _spy_resident_inputs(monkeypatch)+ _metal_launched(tmp_path, monkeypatch, **kw)+ assert seen[-1]["budget_mib"] == budget_gib * 1024+++ "extra_args, cpu_forced",+ [(["--device", "none"], False), ([], True)],+ ids = ["device_none", "paravirtual"],+)+def test_a_cpu_child_on_a_mac_keeps_the_table_streamed(extra_args, cpu_forced):+ backend = _apple_backend("qwen4exp", {PLE: FLASH_NEXT_PLE}, avail_gib = 119)+ got = _apple_discount(backend, extra_args = extra_args, cpu_forced = cpu_forced)+ assert backend._apple_lazy_flags == []+ assert got == FLASH_NEXT_PLE+++def test_the_metal_resident_verdict_abstains_on_an_unsized_mtp_draft(tmp_path, monkeypatch):+ """A flat MTP reserve prices no draft KV, so no resident verdict (as the discrete one)."""+ seen = _spy_resident_inputs(monkeypatch)+ _metal_launched(tmp_path, monkeypatch)+ assert seen[-1]["need_at_ctx"] is not None+ _metal_launched(tmp_path, monkeypatch, extra_args = ["--spec-type", "draft-mtp"])+ assert seen[-1]["need_at_ctx"] is None+++def test_a_rocm_arch_gated_cpu_launch_is_a_cpu_child(tmp_path, monkeypatch, probe_env):+ """No installed kernel covers either card, so the child runs on the CPU."""+ import test_gpu_arch_gate_os_matrix_7624 as gate++ forced = _spy(monkeypatch, "_child_runs_on_cpu", lambda _ans, _a, k: bool(k.get("cpu_forced")))+ backend = _scan_backend("qwen4exp", {PLE: FLASH_NEXT_PLE})+ gate._apply_os(monkeypatch, "linux", is_rocm = True)+ torch = gate._fake_torch(+ [gate._device("gfx1030", free_mib = 12049), gate._device("gfx1036", free_mib = 12176)],+ vendor = "amd",+ )+ gate._run_auto_load(+ monkeypatch,+ tmp_path,+ torch,+ ["gfx908"],+ returncode = None,+ backend = backend,+ server_caps = dict(_CAPS),+ intent_kwargs = {"gpu_memory_mode": "auto"},+ )+ assert backend._arch_gate_forced_cpu is True+ assert forced and all(forced), forced++[email protected]("node", [False, True])+def test_an_empty_inventory_beside_a_gpu_prices_the_table_resident_at_launch(+ tmp_path, _moe_cache_host, monkeypatch, node+):+ """A failed probe may hide an iGPU, where lazy auto is off (as the listing prices it)."""+ seen = _spy_fit_load_mode(monkeypatch)+ monkeypatch.setattr(lc, "_gpu_device_node_present", lambda: node)+ _flash_next_launch(tmp_path, ram_gib = 58, memory = [])+ assert seen, "no load-mode fit ran"+ assert seen[-1]["lazy_read_bytes"] == (0 if node else FLASH_NEXT_PLE), seen[-1]+++ "extra_args, env, cpu",+ [+ (["--split-mode", "none", "--main-gpu", "-1"], {}, True),+ (["-sm", "none", "-mg", "-1"], {}, True),+ ([], _INHERITED_CPU_SPLIT, True),+ (["-sm", "none", "-mg", "0"], {}, False),+ (["--main-gpu", "-1"], {}, False),+ (["-sm", "layer", "-mg", "-1"], {}, False),+ ],+)+def test_split_mode_none_with_a_negative_main_gpu_is_a_cpu_child(extra_args, env, cpu):+ """llama.cpp clears the model's devices for it (src/llama.cpp, b11734-mix-c208e5f)."""+ backend = LlamaCppBackend()+ backend._binary_ships_no_gpu_backend = lambda *a, **k: False+ assert backend._child_runs_on_cpu(None, extra_args, dict(env)) is cpu+++ "extra_args, env, resident",+ [+ (["-sm", "none", "-mg", "0"], {}, True),+ (["--split-mode", "none", "--main-gpu", "-1"], {}, False),+ (["--split-mode", "none"], {"LLAMA_ARG_MAIN_GPU": "-1"}, False),+ # load_model drops an inherited non-layer split mode on a layer launch: layer split on CUDA0.+ ([], _INHERITED_CPU_SPLIT, True),+ (["--main-gpu", "-1"], {"LLAMA_ARG_SPLIT_MODE": "none"}, True),+ ],+)+def test_a_main_gpu_of_minus_one_keeps_llama_cpps_lazy_default(+ tmp_path, _moe_cache_host, monkeypatch, extra_args, env, resident+):+ _setenv(monkeypatch, env)+ _backend, cmd = _flash_next_launch(tmp_path, extra_args = extra_args)+ assert (_lazy_values(cmd) == ["off"]) is resident, cmd+++def _spy_placement_previews(monkeypatch):+ """(helper, caller, env split mode, tensor_parallel) for every placement preview."""+ seen = []+ for name in ("_lazy_auto_resolves_off", "_child_runs_on_cpu", "_apple_lookup_table_discount"):+ real = getattr(LlamaCppBackend, name)++ def make(real, name):+ def spy(self, *a, **k):+ env = k.get("env", a[2] if name == "_child_runs_on_cpu" and len(a) > 2 else None)+ env = os.environ if env is None else env+ seen.append(+ (+ name,+ sys._getframe(1).f_code.co_name,+ env.get("LLAMA_ARG_SPLIT_MODE"),+ k.get("tensor_parallel"),+ )+ )+ return real(self, *a, **k)++ return spy++ monkeypatch.setattr(LlamaCppBackend, name, make(real, name))+ return seen++[email protected]("cards", [1, 2])+def test_every_launch_preview_sees_the_split_mode_the_child_gets(+ tmp_path, _moe_cache_host, monkeypatch, cards+):+ """load_model drops an inherited non-layer split mode before spawn; every preview must price that child."""+ _setenv(monkeypatch, _INHERITED_CPU_SPLIT)+ seen = _spy_placement_previews(monkeypatch)+ launch = _flash_next_launch if cards == 1 else _flash_next_on_two_cards+ _backend, cmd = launch(tmp_path)+ callers = {caller for _n, caller, _sm, _tp in seen}+ assert {"_host_input_bytes_now", "load_model", "_table_resident_fits"} <= callers, seen+ assert cards == 2 or "_fitter_vram_shortfall" in callers, seen+ assert [s for s in seen if s[2] is not None] == [], seen+ assert _lazy_values(cmd) == ["off"], cmd+++def test_a_tensor_launch_is_not_a_cpu_child_under_an_inherited_split_mode(+ tmp_path, _moe_cache_host, monkeypatch+):+ """Studio emits --split-mode tensor (argv beats env), so the inherited CPU split does not apply."""+ _setenv(monkeypatch, _INHERITED_CPU_SPLIT)+ seen = _spy_placement_previews(monkeypatch)+ _backend, cmd = _flash_next_on_two_cards(tmp_path, tensor_parallel = True)+ assert "tensor" in cmd, cmd+ cpu_calls = [s for s in seen if s[0] == "_child_runs_on_cpu" and s[1] == "_table_resident_fits"]+ assert cpu_calls and cpu_calls[-1][3] is True, seen+ assert _lazy_values(cmd) == ["off"], cmd++[email protected]("tensor, cpu", [(None, True), (False, True), (True, False)])+def test_the_cpu_child_check_follows_the_launch_tensor_state(tensor, cpu):+ backend = LlamaCppBackend()+ backend._binary_ships_no_gpu_backend = lambda *a, **k: False+ got = backend._child_runs_on_cpu(None, [], dict(_INHERITED_CPU_SPLIT), tensor_parallel = tensor)+ assert got is cpu+++def _apu_flash_next(tmp_path, monkeypatch, apu_ids):+ backend, gguf = placement._apu_backend(+ tmp_path, gguf_gb = 73.45, avail_mib = 58 * 1024, monkeypatch = monkeypatch+ )+ monkeypatch.setattr(+ LlamaCppBackend, "_rocm_unified_memory_gpu_ids", staticmethod(lambda: set(apu_ids))+ )+ backend._gguf_tensor_scan = lambda _path: ("qwen4exp", {PLE: placement._QWEN38_PLE}, 0)+ backend.probe_server_capabilities = _full_caps()+ return backend, gguf+++def test_a_failed_probe_apu_guard_ignores_a_split_mode_the_launch_drops(tmp_path, monkeypatch):+ """The HIP child runs layer split on the APU once load_model drops split mode none, so the guard charges its table."""+ _setenv(monkeypatch, _INHERITED_CPU_SPLIT)+ backend, gguf = _apu_flash_next(tmp_path, monkeypatch, {0})+ backend._get_gpu_memory = lambda *_a, **_kw: []+ backend._get_gpu_free_memory = lambda *_a, **_kw: []++ res = placement._launch(backend, gguf, extra_args = ["--load-mode", "none"])++ assert res["env"].get("LLAMA_ARG_SPLIT_MODE") is None+ assert "unified-memory APU" in (backend.last_load_warning or "")+ assert not placement._unmapped_tokens(res["cmd"]), res["cmd"]+++def test_the_metal_discount_sees_the_split_mode_the_child_gets(tmp_path, monkeypatch):+ _setenv(monkeypatch, _INHERITED_CPU_SPLIT)+ seen = _spy_placement_previews(monkeypatch)+ _metal_launched(tmp_path, monkeypatch)+ metal = [s for s in seen if s[0] == "_apple_lookup_table_discount"]+ assert metal and all(s[2] is None for s in metal), seen+++ "extra_args",+ [["--device", "ROCm0"], ["--device", "ROCm0", "--load-mode", "none"], []],+ ids = ["device", "device_unmapped", "no_device"],+)+def test_a_compact_device_names_the_card_the_child_mask_puts_first(+ tmp_path, monkeypatch, extra_args+):+ """Auto pins a ROCm APU at physical 1 beside a discrete card at 0, so the child's ROCm0 is the APU, not the parent's."""+ for name in ("CUDA_VISIBLE_DEVICES", "HIP_VISIBLE_DEVICES", "ROCR_VISIBLE_DEVICES"):+ monkeypatch.delenv(name, raising = False)+ backend, gguf = _apu_flash_next(tmp_path, monkeypatch, {1})+ rows = [(0, 16_000, 16_384), (1, 90_000, 92_160)]+ backend._get_gpu_memory = lambda *_a, **_kw: list(rows)+ backend._get_gpu_free_memory = lambda *_a, **_kw: [(i, f) for i, f, _t in rows]+ backend._amd_apu_wants_unified_memory = lambda ids = None, *a, **k: (+ ids is not None and bool(list(ids)) and {int(i) for i in ids} <= {1}+ )+ backend._select_gpus = lambda *a, **k: ([1], True)+ backend._can_estimate_kv = lambda: True+ backend._estimate_kv_cache_bytes = lambda *a, **k: GIB // 4+ seen = _spy(+ monkeypatch,+ "_lazy_auto_resolves_off",+ lambda answer, _a, _k: (sys._getframe(2).f_code.co_name, answer),+ )++ res = placement._launch(backend, gguf, extra_args = extra_args)++ assert res["env"].get("CUDA_VISIBLE_DEVICES") == "1", res["env"]+ previews = [s for s in seen if s[0] in ("load_model", "_fitter_vram_shortfall")]+ assert previews and all(answer for _c, answer in previews), seen+ assert "unified-memory APU" in (backend.last_load_warning or "")+ assert not placement._unmapped_tokens(res["cmd"]), res["cmd"]++[email protected]("ram_gib", [58, 256])[email protected]("layout", ["dgpu0_igpu1", "igpu0_dgpu1"])+def test_split_mode_none_reads_the_main_device_in_picker_order(+ tmp_path, _moe_cache_host, layout, ram_gib+):+ """gpu_ids [1, 0] emits --device Vulkan1,Vulkan0, so --split-mode none puts every weight on Vulkan1, not Vulkan0."""+ dgpu, igpu = (0, 1) if layout == "dgpu0_igpu1" else (1, 0)+ rows = sorted([(dgpu, 35 * 1024, 36 * 1024), (igpu, 55 * 1024, 0)])+ backend, gguf = placement._moe_backend(tmp_path, size_gib = 73.45, memory = rows, moe = False)+ backend._is_vulkan_backend = lambda _binary = None: True+ backend._can_estimate_kv = lambda: True+ backend._estimate_kv_cache_bytes = lambda *a, **k: GIB // 4+ backend._available_system_memory_mib = lambda: int(ram_gib * 1024)+ backend._gguf_tensor_scan = lambda _path: ("qwen4exp", {PLE: placement._QWEN38_PLE}, 0)+ backend.probe_server_capabilities = _full_caps()++ cmd = placement._launch(backend, gguf, gpu_ids = [1, 0], extra_args = ["--split-mode", "none"])[+ "cmd"+ ]++ assert _arg(cmd, "--device") == "Vulkan1,Vulkan0", cmd+ main_is_igpu = igpu == 1+ # An iGPU main device keeps the table resident (auto is off there): 73.45 GiB does+ # not fit 58 GiB of RAM unmapped, so the load stays mmap'd.+ if main_is_igpu and ram_gib == 58:+ assert placement._load_mode(cmd) is None, cmd+ # A discrete main device streams it unless RAM holds it, where Studio pins it resident.+ expected_lazy = ["off"] if not main_is_igpu and ram_gib == 256 else []+ assert _lazy_values(cmd) == expected_lazy, cmd+++ "gpu_ids, gpu_indices, split, inherited, order, mask",+ [+ (None, [0, 1], None, None, [0, 1], [0, 1]),+ ([1, 0], [0, 1], None, None, [1, 0], [1, 0]),+ ([1, 0], None, None, None, [1, 0], [1, 0]),+ ([1, 0], [0, 1], [3, 1], None, [1, 0], [1, 0]),+ ([1, 0], [0, 1], [3], None, [0, 1], [0, 1]),+ (None, [0, 1], None, [1, 0], [0, 1], [1, 0]),+ (None, [0, 1], [3, 1, 2], [1, 0], [0, 1], [0, 1]),+ (None, None, None, None, None, None),+ ],+ ids = [+ "ascending",+ "picker_reorder",+ "picker_unnarrowed",+ "picker_moves_shares",+ "picker_veto_shares",+ "inherited_order",+ "inherited_veto_shares",+ "unpinned",+ ],+)+def test_the_planned_child_order_matches_the_spawn(+ monkeypatch, gpu_ids, gpu_indices, split, inherited, order, mask+):+ monkeypatch.setattr(+ LlamaCppBackend, "_resolve_visible_physical_ids", staticmethod(lambda: inherited)+ )+ monkeypatch.setattr(+ LlamaCppBackend,+ "_pinned_child_visibility",+ staticmethod(lambda ids: ",".join(str(int(i)) for i in ids)),+ )+ backend = LlamaCppBackend()+ kwargs = dict(+ gpu_ids = gpu_ids, gpu_indices = gpu_indices, tp_tensor_split = split, extra_args = [], env = {}+ )+ plan = backend._planned_child_gpu_order+ assert plan(**kwargs, is_vulkan_backend = False) == (order, mask)+ # Vulkan has no mask; a CPU child gets none either.+ assert plan(**kwargs, is_vulkan_backend = True) == (order, None)+ assert plan(**kwargs, is_vulkan_backend = False, cpu_forced = True) == (order, None)+ # Ids the mask cannot name keep the inherited one: the parent's numbering stands.+ monkeypatch.setattr(LlamaCppBackend, "_pinned_child_visibility", staticmethod(lambda ids: None))+ assert plan(**kwargs, is_vulkan_backend = False) == (order, None)+++def test_a_compact_device_maps_through_the_planned_mask(monkeypatch):+ backend = LlamaCppBackend()+ backend._integrated_cuda_gpu_ids = lambda: set()+ monkeypatch.setattr(LlamaCppBackend, "_rocm_unified_memory_gpu_ids", staticmethod(lambda: {1}))+ common = dict(+ gpu_indices = [1],+ detected_gpus = [(0, 16_000), (1, 90_000)],+ shared_gpu_ids = set(),+ is_vulkan_backend = False,+ extra_args = ["--device", "ROCm0"],+ env = {},+ )+ # The parent's numbering: ROCm0 is the discrete card; the child's mask [1]: ROCm0 is the APU.+ assert backend._lazy_auto_resolves_off(**common) is False+ assert backend._lazy_auto_resolves_off(**common, child_gpu_order = [1]) is Truediff --git a/studio/backend/tests/test_lookup_table_pricing.py b/studio/backend/tests/test_lookup_table_pricing.pynew file mode 100644index 00000000000..991b529b9eb--- /dev/null+++ b/studio/backend/tests/test_lookup_table_pricing.py@@ -0,0 +1,1339 @@+# SPDX-License-Identifier: AGPL-3.0-only+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0++"""Lookup-only tables (per-layer embeddings / Engram) priced apart in the listing and the panel."""++from __future__ import annotations++import asyncio+import io+import os+import struct+import sys+import threading+import time+from pathlib import Path+from types import SimpleNamespace++import pytest++_TESTS_DIR = str(Path(__file__).resolve().parent)+if _TESTS_DIR not in sys.path:+ sys.path.insert(0, _TESTS_DIR)+_BACKEND_DIR = str(Path(__file__).resolve().parent.parent)+if _BACKEND_DIR not in sys.path:+ sys.path.insert(0, _BACKEND_DIR)++import test_kv_cache_estimation # noqa: E402,F401+import test_llama_cpp_placement # noqa: E402,F401 -- stubs jwt for routes.inference++import routes.inference as ri # noqa: E402+import utils.hardware as hw # noqa: E402+from core.inference import llama_cpp as lc # noqa: E402+from hub.schemas.inventory import GgufVariantDetail # noqa: E402+from hub.services.models import gguf_variants as gv # noqa: E402+from hub.utils import gguf_header_delta # noqa: E402+from hub.utils import gguf_lookup_tables as glt # noqa: E402++GIB = 1024**3+IQ4_NL, Q5_1, F16 = 20, 7, 1+PLE = "per_layer_token_embd.weight"++DSV41_TABLE = 256 * 768022850 // 32 * 18+FLASH_NEXT_TABLE = 160 * 320001536 // 32 * 18+GEMMA_TABLE = 8960 * 262144 // 32 * 24++_ATTN = {+ "block_count": 4,+ "context_length": 4096,+ "embedding_length": 256,+ "attention.head_count": 4,+ "attention.head_count_kv": 2,+ "attention.key_length": 64,+ "attention.value_length": 64,+}+++def _header(+ arch: str,+ tensors: list[tuple[str, list[int], int]],+ ints: dict | None = None,+ offsets: list[int] | None = None,+):+ """A GGUF v3 header: the architecture, uint32 metadata, and tensor infos with no data."""+ kv: list[tuple[str, object]] = [("general.architecture", arch)]+ kv += [(f"{arch}.{k}", int(v)) for k, v in (ints or {}).items()]+ buf = io.BytesIO()+ buf.write(b"GGUF")+ buf.write(struct.pack("<IQQ", 3, len(tensors), len(kv)))+ for key, value in kv:+ raw = key.encode()+ buf.write(struct.pack("<Q", len(raw)) + raw)+ if isinstance(value, str):+ data = value.encode()+ buf.write(struct.pack("<IQ", 8, len(data)) + data)+ else:+ buf.write(struct.pack("<II", 4, value))+ offset = 0+ for i, (name, dims, ggml_type) in enumerate(tensors):+ raw = name.encode()+ buf.write(struct.pack("<Q", len(raw)) + raw)+ buf.write(struct.pack("<I", len(dims)))+ buf.write(struct.pack(f"<{len(dims)}Q", *dims))+ buf.write(struct.pack("<IQ", ggml_type, offsets[i] if offsets else offset))+ offset += 1024+ return buf.getvalue()+++_LAZY_BLOB = _header("qwen4exp", [(PLE, [160, 320001536], IQ4_NL)], _ATTN)+_DSV41_EMBD = ("token_embd.weight", [4096, 129280], IQ4_NL)+_DSV41_PLE = (PLE, [256, 768022850], IQ4_NL)+++def _write(path: Path, data: bytes) -> str:+ path.parent.mkdir(parents = True, exist_ok = True)+ path.write_bytes(data)+ return str(path)+++def _dsv41(tmp_path: Path) -> str:+ return _write(+ tmp_path / "DeepSeek-V4.1-Flash-UD-IQ1_S.gguf",+ _header("deepseek41", [_DSV41_EMBD, _DSV41_PLE]),+ )+++def _flash_next(tmp_path: Path) -> str:+ tensors = [+ ("token_embd.weight", [2048, 151936], IQ4_NL),+ (PLE, [160, 320001536], IQ4_NL),+ ("blk.0.attn_q.weight", [256, 256], F16),+ ]+ return _write(+ tmp_path / "Qwen3.8-Flash-Next-UD-Q2_K_XL.gguf", _header("qwen4exp", tensors, _ATTN)+ )+++def _gemma(tmp_path: Path, arch = "gemma3n") -> str:+ tensors = [(PLE, [8960, 262144], Q5_1)]+ if arch == "gemma3n":+ tensors.insert(0, ("token_embd.weight", [2048, 262144], IQ4_NL))+ return _write(tmp_path / f"{arch}.gguf", _header(arch, tensors))+++def _blob_fetcher(+ blobs: dict,+ calls: list,+ delay: float = 0.0,+):+ """A Range fetcher over in-memory files that reports each file's size like Content-Range."""++ def fetcher(+ repo_id,+ name,+ token,+ revision = None,+ ):+ blob, size = blobs[name]++ def fetch(start, end):+ calls.append((name, start, end))+ if delay:+ time.sleep(delay)+ fetch.file_size = size+ return blob[start : end + 1]++ return fetch++ return fetcher+++def _answer(variants, sources = None):+ from hub.schemas.inventory import GgufVariantsResponse+ from hub.services.models.gguf_variants import VariantsAnswer+ return VariantsAnswer(+ GgufVariantsResponse(repo_id = "unsloth/X-GGUF", variants = variants),+ None,+ variant_context_sources = sources or {},+ )+++def _row(+ filename = "a.gguf",+ quant = "Q4",+ size = 40 * GIB,+ **kw,+):+ return GgufVariantDetail(filename = filename, quant = quant, size_bytes = size, **kw)+++def _listing(+ rows,+ repo,+ *,+ token = None,+ offline = False,+):+ """The coroutine; ``rows`` may be a ready answer."""+ answer = rows if hasattr(rows, "response") else _answer(rows)+ return gv._with_lookup_tables(+ answer, repo, hf_token = token, offline = offline, prefer_local_cache = False+ )+++def _list(rows, repo, **kw):+ return asyncio.run(_listing(rows, repo, **kw))+++def _tables(out):+ return [v.lookup_table_bytes for v in out.response.variants]+++def _arch(+ monkeypatch,+ known,+ arch = "qwen4exp",+):+ monkeypatch.setattr(gv, "_cached_hub_repo_architecture", lambda *a, **k: (known, arch))+++def _remote(monkeypatch, fn = None):+ monkeypatch.setattr(+ glt, "remote_lookup_info", fn or (lambda *a, **k: ("qwen4exp", FLASH_NEXT_TABLE))+ )+++def _overlap(+ first,+ second,+ wait = None,+):+ """Run ``second`` once ``first`` is underway (after 0.1 s, or once ``wait`` is set)."""++ async def both():+ a = asyncio.ensure_future(first)+ if wait is None:+ await asyncio.sleep(0.1)+ while wait is not None and not wait.is_set():+ await asyncio.sleep(0.01)+ return await asyncio.gather(a, second)++ return asyncio.run(both())+++def _stream_backend(monkeypatch, *, vulkan, gpus):+ backend = lc.LlamaCppBackend.__new__(lc.LlamaCppBackend)+ monkeypatch.setattr(backend, "_find_llama_server_binary", lambda: "/fake/llama-server")+ monkeypatch.setattr(+ backend,+ "probe_server_capabilities",+ lambda binary = None: {"supports_lazy_mode": True, "lazy_mode_flag": "--lazy-mode"},+ )+ monkeypatch.setattr(backend, "_is_vulkan_backend", lambda binary = None: vulkan)+ monkeypatch.setattr(backend, "_get_gpu_memory", lambda *a, **k: list(gpus))+ return backend+++def _no_gpu_backend(monkeypatch, cpu_build):+ monkeypatch.setattr(+ lc.LlamaCppBackend, "_binary_ships_no_gpu_backend", staticmethod(lambda *a, **k: cpu_build)+ )++[email protected](autouse = True)+def _streams_here(monkeypatch):+ monkeypatch.setattr(gv, "_host_streams_lookup_tables", lambda: True)+++def test_split_names_keep_the_extension_as_written():+ assert glt.shard_names("d/M-00001-of-00002.GGUF") == [+ "d/M-00001-of-00002.GGUF",+ "d/M-00002-of-00002.GGUF",+ ]+++def test_a_posix_class_pattern_is_not_proof_the_table_stays_on_the_host():+ """std::regex reads [[:alpha:]] as a class."""+ on_device = lc._override_places_tensor_on_device+ assert on_device(PLE, ["-ot", "per_[[:alpha:]_]+=CUDA0"], {})+ assert on_device(PLE, ["-ot", "per_[[:alpha:]_]+=CPU,per_layer=CUDA0"], {})+ assert not on_device(PLE, ["-ot", "per_[[:alpha:]_]+=CPU"], {})+++class TestHeaderScan:+ def test_real_table_sizes(self, tmp_path):+ assert glt.local_lookup_info(_dsv41(tmp_path)) == ("deepseek41", DSV41_TABLE)+ assert DSV41_TABLE / GIB == pytest.approx(103.00, abs = 0.01)+ assert glt.local_lookup_info(_flash_next(tmp_path)) == ("qwen4exp", FLASH_NEXT_TABLE)+ assert FLASH_NEXT_TABLE / GIB == pytest.approx(26.82, abs = 0.01)++ def test_gemma3n_is_not_lazily_read(self, tmp_path):+ assert glt.local_lookup_info(_gemma(tmp_path)) == ("gemma3n", 0)++ def test_a_small_table_stays_resident_under_auto(self, tmp_path):+ assert glt.local_lookup_table_bytes(_gemma(tmp_path, "gemma4")) == 0++ def test_an_explicit_lazy_mode_on_reads_a_small_table_lazily(self, tmp_path):+ small = _gemma(tmp_path, "gemma4")+ assert glt.local_lookup_info(small) == ("gemma4", 0)+ assert glt.local_lookup_info(small, "on") == ("gemma4", GEMMA_TABLE)++ @pytest.mark.parametrize(+ "second, want",+ [(True, DSV41_TABLE), (False, 0)],+ ids = ["split_file_finds_the_table_in_a_later_shard", "missing_shard"],+ )+ def test_split_file_finds_the_table_in_a_later_shard(self, tmp_path, second, want):+ first = _write(tmp_path / "m-00001-of-00002.gguf", _header("deepseek41", [_DSV41_EMBD]))+ if second:+ _write(tmp_path / "m-00002-of-00002.gguf", _header("", [_DSV41_PLE]))+ assert glt.local_lookup_table_bytes(first) == want++ def test_a_missing_shard_or_garbage_is_zero(self, tmp_path):+ assert glt.local_lookup_table_bytes(_write(tmp_path / "junk.gguf", b"not a gguf")) == 0+ assert glt.local_lookup_table_bytes(None) == 0++ def test_remote_reads_only_the_header_range(self, tmp_path):+ calls = []+ fetcher = _blob_fetcher({"Q2/flash-Q2.gguf": (_LAZY_BLOB, 40 * GIB)}, calls)++ def read():+ return glt.remote_lookup_info(+ "unsloth/Flash-Next-GGUF", "Q2/flash-Q2.gguf", identity = 1, fetcher_fn = fetcher+ )++ assert read() == ("qwen4exp", FLASH_NEXT_TABLE)+ assert calls and all(start == 0 for _name, start, _end in calls)+ calls.clear()+ assert read() == ("qwen4exp", FLASH_NEXT_TABLE)+ assert not calls+ glt._REMOTE_CACHE.clear()+ glt._REMOTE_TTL_S, ttl = -1.0, glt._REMOTE_TTL_S+ try:+ read()+ assert glt.remote_cached("unsloth/Flash-Next-GGUF", "Q2/flash-Q2.gguf", None, 1) is None+ finally:+ glt._REMOTE_TTL_S = ttl+ assert calls++ def test_remote_failure_is_zero_and_not_cached(self):+ def broken(*a, **k):+ def fetch(start, end):+ raise OSError("network down")++ return fetch++ assert glt.remote_lookup_info("o/r", "f.gguf", identity = 7, fetcher_fn = broken) == (None, 0)+ assert glt.remote_cached("o/r", "f.gguf", None, 7) is None++ def test_a_remote_answer_is_cached_per_lazy_mode(self):+ fetcher = _blob_fetcher({"f.gguf": (b"", 0)}, [])+ glt.remote_lookup_info("o/m", "f.gguf", identity = 11, fetcher_fn = fetcher)+ assert glt.remote_cached("o/m", "f.gguf", None, 11, mode = "on") is None+++class TestListing:+ @pytest.mark.parametrize(+ "streams, want",+ [(False, None), (True, DSV41_TABLE)],+ ids = [+ "a_host_that_reads_the_tables_resident_lists_no_discount",+ "a_downloaded_variant_is_read_from_its_own_header",+ ],+ )+ def test_a_downloaded_variant_is_read_from_its_own_header(+ self, tmp_path, monkeypatch, streams, want+ ):+ """An iGPU / APU resolves lazy auto to off, so the listing prices the whole file."""+ monkeypatch.setattr(gv, "_host_streams_lookup_tables", lambda: streams)+ monkeypatch.setattr(+ gv, "_hub_repo_architecture", lambda *a, **k: pytest.fail("no Hub call")+ )+ path = _dsv41(tmp_path)+ row = _row(os.path.basename(path), "UD-IQ1_S", 216 * GIB, downloaded = True)+ answer = _answer([row], {"ud-iq1_s": path})+ out = _list(answer, "unsloth/X-GGUF", offline = True)+ if want is None:+ assert out is answer+ else:+ assert _tables(out) == [want]+ assert _tables(answer) == [0]++ def test_both_variant_models_carry_the_field(self):+ from models.models import GgufVariantDetail as RouteDetail+ for model in (GgufVariantDetail, RouteDetail):+ assert model.model_fields["lookup_table_bytes"].default == 0++ def test_hub_only_quants_of_a_lazy_arch_are_range_read(self, monkeypatch):+ _arch(monkeypatch, True)+ _remote(monkeypatch)+ rows = [_row("Q2/a-00001-of-00003.gguf", "Q2"), _row("README.md", "x", 1)]+ assert _tables(_list(rows, "unsloth/Flash-Next-GGUF")) == [FLASH_NEXT_TABLE, 0]++ def test_a_small_repo_never_asks_the_hub_for_its_architecture(self, monkeypatch):+ """No row can hold a streamed table (each is under 4 GiB)."""+ _arch(monkeypatch, False, None)+ monkeypatch.setattr(+ gv, "_hub_repo_architecture", lambda *a, **k: pytest.fail("no Hub call")+ )+ answer = _answer([_row(size = GIB)])+ assert _list(answer, "o/small") is answer++ def test_a_cold_architecture_is_priced_on_the_first_listing(self, monkeypatch):+ fetched = []+ monkeypatch.setattr(+ gv, "_hub_repo_architecture", lambda repo, token: fetched.append(repo) or "qwen4exp"+ )+ _arch(monkeypatch, False, None)+ _remote(monkeypatch)+ assert _tables(_list([_row()], "o/cold")) == [FLASH_NEXT_TABLE]+ assert fetched == ["o/cold"]++ def test_a_slow_architecture_lookup_is_bounded_by_the_deadline(self, monkeypatch):+ """A late answer never holds the listing past its deadline, and is asked once."""+ release = threading.Event()+ fetched = []++ def slow_arch(repo, token):+ fetched.append(repo)+ release.wait(5)+ return "qwen4exp"++ monkeypatch.setattr(gv, "_LOOKUP_DEADLINE_S", 0.2)+ monkeypatch.setattr(gv, "_hub_repo_architecture", slow_arch)+ _arch(monkeypatch, False, None)+ _remote(monkeypatch, lambda *a, **k: pytest.fail("no header read yet"))+ answer = _answer([_row()])+ try:+ for _ in range(3):+ started = time.monotonic()+ assert _list(answer, "o/slow") is answer+ assert time.monotonic() - started < 2.0+ finally:+ release.set()+ for _ in range(50):+ if not gv._lookup_inflight:+ break+ time.sleep(0.05)+ assert fetched == ["o/slow"]++ def test_a_direct_file_source_finds_its_row_beside_it(self, tmp_path):+ model = tmp_path / "Qwen3.8-Flash-Next-UD-Q2_K_XL.gguf"+ model.write_bytes(b"GGUF")+ row = SimpleNamespace(filename = model.name, quant = "UD-Q2_K_XL", downloaded = True)+ for answer, repo in (+ (SimpleNamespace(variant_context_sources = None, context_source = str(model)), "o/r"),+ (SimpleNamespace(variant_context_sources = None, context_source = None), str(model)),+ ):+ assert gv._local_variant_gguf(answer, repo, row) == str(model)++ def test_a_requested_cache_copy_is_found_where_the_listing_found_it(self, tmp_path):+ """local_path naming a second cache root, or a snapshot other than refs/main, is searched."""+ repo_dir = tmp_path / "second_cache" / "models--o--lookup-cache-test"+ main_snap, pinned_snap = repo_dir / "snapshots" / "aaaa", repo_dir / "snapshots" / "bbbb"+ for snap, name in ((main_snap, "m-Q4.gguf"), (pinned_snap, "p-Q8.gguf")):+ snap.mkdir(parents = True)+ (snap / name).write_bytes(b"GGUF")+ (repo_dir / "refs").mkdir()+ (repo_dir / "refs" / "main").write_text("aaaa")+ answer = SimpleNamespace(+ variant_context_sources = None, context_source = None, cache_authorized = True+ )+ for name, root, snap in (+ ("m-Q4.gguf", repo_dir, main_snap),+ ("p-Q8.gguf", pinned_snap, pinned_snap),+ ):+ row = SimpleNamespace(filename = name, quant = "Q", downloaded = True)+ got = gv._local_variant_gguf(answer, "o/lookup-cache-test", row, str(root))+ assert got == str(snap / name)++ def test_a_read_the_pool_refuses_after_the_deadline_releases_its_key(self, monkeypatch):+ """Late refusals (the pool shut down at exit) never reach the read's own finally."""++ async def refused(fn, /, *args):+ await asyncio.sleep(0.01)+ raise RuntimeError("cannot schedule new futures after shutdown")++ monkeypatch.setattr(gv, "_lookup_thread", refused)+ key = ("arch", "o/refused")++ async def go():+ loop = asyncio.get_running_loop()+ assert await gv._lookup_await_once(key, loop.time(), lambda: 1) is None+ assert key in gv._lookup_inflight+ await asyncio.sleep(0.05)++ asyncio.run(go())+ assert key not in gv._lookup_inflight++ def test_a_cached_non_lazy_variant_does_not_veto_the_rest_of_the_repo(+ self, tmp_path, monkeypatch+ ):+ local = _write(+ tmp_path / "llama.gguf",+ _header("llama", [("token_embd.weight", [256, 1000], F16)], _ATTN),+ )+ _arch(monkeypatch, True)+ _remote(monkeypatch)+ rows = [_row("llama.gguf", "Q8_0", 1, downloaded = True), _row("flash/f.gguf", "Q2")]+ out = _list(_answer(rows, {"q8_0": local}), "o/mixed")+ assert _tables(out) == [0, FLASH_NEXT_TABLE]++ def test_one_listing_queues_every_uncached_header(self, monkeypatch):+ calls = []+ lock = threading.Lock()++ def read(repo, name, *a, **k):+ with lock:+ calls.append(name)+ return ("qwen4exp", FLASH_NEXT_TABLE)++ _arch(monkeypatch, True)+ _remote(monkeypatch, read)+ rows = [_row(f"q{i}/m.gguf", f"Q{i}", 40 * GIB + i) for i in range(10)]+ out = _list(rows, "o/many")+ assert sorted(calls) == sorted(r.filename for r in rows)+ assert all(_tables(out))++ @pytest.mark.parametrize("offline, want", [(False, 10 * GIB), (True, 40 * GIB)])+ def test_an_update_is_priced_from_the_published_header(+ self, monkeypatch, tmp_path, offline, want+ ):+ """The cached file is the old revision: online, an update's table comes from the Hub's header."""+ on_disk = tmp_path / "old.gguf"+ on_disk.write_bytes(b"x")+ _arch(monkeypatch, True)+ monkeypatch.setattr(gv, "_local_variant_gguf", lambda *a: str(on_disk))+ monkeypatch.setattr(+ glt, "local_lookup_info", lambda path, mode = "auto": ("qwen4exp", 40 * GIB)+ )+ _remote(monkeypatch, lambda *a, **k: ("qwen4exp", 10 * GIB))+ row = _row("u.gguf", "U", 70 * GIB + offline, downloaded = True, update_available = True)+ out = _list([row], f"o/updated-{offline}", offline = offline)+ assert _tables(out) == [want]++ def test_a_forced_lazy_mode_sizes_the_tables_it_streams(self, monkeypatch, tmp_path):+ modes = []++ def local(path, mode = "auto"):+ modes.append(("local", mode))+ return ("qwen4exp", FLASH_NEXT_TABLE)++ def read(+ *a,+ mode = "auto",+ **k,+ ):+ modes.append(("remote", mode))+ return ("qwen4exp", FLASH_NEXT_TABLE)++ monkeypatch.setattr(gv, "_listing_lazy_mode", lambda: "on")+ _arch(monkeypatch, True)+ _remote(monkeypatch, read)+ monkeypatch.setattr(glt, "local_lookup_info", local)+ on_disk = tmp_path / "local.gguf"+ on_disk.write_bytes(b"x")+ monkeypatch.setattr(+ gv, "_local_variant_gguf", lambda a, r, v, *_: str(on_disk) if v.quant == "L" else None+ )+ _list([_row("l.gguf", "L", 1, downloaded = True), _row("r.gguf", "R", 2)], "o/forced")+ assert sorted(modes) == [("local", "on"), ("remote", "on")]++ def test_a_forced_lazy_mode_looks_up_a_cold_repo_of_small_files(self, monkeypatch):+ monkeypatch.setattr(gv, "_listing_lazy_mode", lambda: "on")+ _arch(monkeypatch, False, None)+ monkeypatch.setattr(gv, "_hub_repo_architecture", lambda *a, **k: "gemma4")+ _remote(monkeypatch, lambda *a, **k: ("gemma4", 2 * GIB))+ assert _tables(_list([_row("s.gguf", size = 3 * GIB)], "o/small")) == [2 * GIB]++ @pytest.mark.parametrize(+ "stage",+ ["arch", "table"],+ ids = [+ "an_architecture_lookup_in_flight_for_another_token_does_not_block",+ "a_table_scan_in_flight_for_another_token_does_not_skip_the_row",+ ],+ )+ def test_an_architecture_lookup_in_flight_for_another_token_does_not_block(+ self, monkeypatch, stage+ ):+ asked = []+ second = threading.Event()++ def gate(token, answer):+ asked.append(token)+ if token == "hf_first":+ second.wait(1.5)+ else:+ second.set()+ return answer++ if stage == "arch":+ _arch(monkeypatch, False, None)+ monkeypatch.setattr(+ gv, "_hub_repo_architecture", lambda repo, token: gate(token, "qwen4exp")+ )+ _remote(monkeypatch)+ else:+ _arch(monkeypatch, True)+ _remote(+ monkeypatch,+ lambda repo, name, token, *a, **k: gate(token, ("qwen4exp", FLASH_NEXT_TABLE)),+ )+ rows = [_row(size = 40 * GIB + 7)]+ repo = f"o/token-{stage}"+ _first, other = _overlap(+ _listing(rows, repo, token = "hf_first"), _listing(rows, repo, token = "hf_second")+ )+ assert sorted(asked) == ["hf_first", "hf_second"]+ assert _tables(other) == [FLASH_NEXT_TABLE]++ @pytest.mark.parametrize(+ "vulkan, node, streams",+ [(False, True, False), (False, False, True), (True, False, False), (True, True, False)],+ )+ def test_an_empty_inventory_beside_a_gpu_node_prices_the_table_resident(+ self, monkeypatch, vulkan, node, streams+ ):+ """Without the torch fallback a ROCm APU can read as no rows; an empty Vulkan inventory may hide an iGPU."""+ backend = _stream_backend(monkeypatch, vulkan = vulkan, gpus = [])+ monkeypatch.setattr(backend, "_lazy_auto_resolves_off", lambda **k: False)+ monkeypatch.setattr(lc, "_gpu_device_node_present", lambda: node)+ assert backend.lookup_tables_stream_by_default(None, {}) is streams++ def test_a_stalled_hub_leaves_no_queue_after_the_deadline(self, monkeypatch):+ release = threading.Event()+ started = []+ lock = threading.Lock()++ def stalled(repo, name, *a, **k):+ with lock:+ started.append(name)+ release.wait(10)+ return ("qwen4exp", FLASH_NEXT_TABLE)++ monkeypatch.setattr(gv, "_LOOKUP_DEADLINE_S", 0.3)+ _arch(monkeypatch, True)+ _remote(monkeypatch, stalled)+ rows = [_row(f"s{i}/m.gguf", f"S{i}", 40 * GIB + i) for i in range(12)]++ async def listing_then_release():+ await _listing(rows, "o/stalled")+ await asyncio.sleep(0.3)+ release.set()+ await asyncio.sleep(0.5)++ try:+ asyncio.run(listing_then_release())+ finally:+ release.set()+ assert len(started) <= gv._LOOKUP_CONCURRENCY, started+ with gv._lookup_arch_lock:+ assert not [k for k in gv._lookup_inflight if k[:2] == ("table", "o/stalled")]++ def test_an_expired_architecture_lookup_leaves_no_queue(self, monkeypatch):+ release = threading.Event()+ ran = []++ def slow(name):+ ran.append(name)+ release.wait(10)+ return name++ async def main():+ loop = asyncio.get_running_loop()+ busy = [+ asyncio.ensure_future(gv._lookup_thread(slow, f"busy{i}"))+ for i in range(gv._LOOKUP_CONCURRENCY)+ ]+ await asyncio.sleep(0.2)+ got = await gv._lookup_await_once(+ ("arch", "o/late", "fp"), loop.time() + 0.2, slow, "late"+ )+ release.set()+ await asyncio.gather(*busy)+ await asyncio.sleep(0.3)+ return got++ try:+ got = asyncio.run(main())+ finally:+ release.set()+ assert got is None+ assert "late" not in ran+ with gv._lookup_arch_lock:+ assert ("arch", "o/late", "fp") not in gv._lookup_inflight++ def test_work_the_deadline_drops_never_runs_if_a_worker_reaches_it_late(self, monkeypatch):+ """A worker can pick up queued work before the cancel reaches the executor."""+ captured = []+ reads = []++ async def queued(fn, *args):+ captured.append((fn, args))+ await asyncio.sleep(10)++ def read(repo, name, *a, **k):+ reads.append(name)+ return ("qwen4exp", FLASH_NEXT_TABLE)++ monkeypatch.setattr(gv, "_lookup_thread", queued)+ monkeypatch.setattr(gv, "_LOOKUP_DEADLINE_S", 0.1)+ _arch(monkeypatch, True)+ _remote(monkeypatch, read)+ rows = [_row(f"q{i}/m.gguf", f"Q{i}", 40 * GIB + i) for i in range(3)]++ async def main():+ await _listing(rows, "o/late")+ loop = asyncio.get_running_loop()+ return await gv._lookup_await_once(+ ("arch", "o/late", "fp"), loop.time() + 0.1, read, "o/late", "arch", None+ )++ assert asyncio.run(main()) is None+ assert len(captured) == 4+ for fn, args in captured:+ fn(*args)+ assert reads == []++ def test_a_capability_read_never_falls_back_to_a_torch_context(self, monkeypatch):+ from utils.hardware import gpu_query++ B = lc.LlamaCppBackend+ monkeypatch.setattr(B, "_is_vulkan_backend", staticmethod(lambda binary = None: False))+ monkeypatch.setattr(B, "_get_gpu_memory_nvml", staticmethod(lambda: []))+ monkeypatch.setattr(B, "_get_gpu_memory_amd_smi", staticmethod(lambda binary = None, **k: []))+ monkeypatch.setattr(+ B,+ "_get_gpu_memory_torch",+ staticmethod(lambda *a, **k: pytest.fail("torch probe in a capability read")),+ )++ def no_smi(*a, **k):+ raise FileNotFoundError("nvidia-smi")++ monkeypatch.setattr(gpu_query, "run_nvidia_smi", no_smi)+ with lc._without_torch_gpu_probe():+ assert B._get_gpu_memory("/fake/llama-server", for_llama_server = True) == []+ assert lc._GPU_MEMORY_SKIP_TORCH.get() is False++ def test_offline_or_a_non_lazy_arch_never_reads_the_hub(self, monkeypatch):+ _remote(monkeypatch, lambda *a, **k: pytest.fail("no header read"))+ answer = _answer([_row(quant = "Q4_K_M", size = 1)])+ assert _list(answer, "o/r", offline = True) is answer+ monkeypatch.setattr(gv, "_hub_repo_architecture", lambda *a, **k: "llama")+ assert _list(answer, "o/r") is answer++ def test_a_failure_returns_the_listing_unchanged(self, monkeypatch):+ def boom(*a, **k):+ raise RuntimeError("boom")++ monkeypatch.setattr(gv, "_local_variant_gguf", boom)+ answer = _answer([_row(quant = "Q", size = 1)])+ assert _list(answer, "o/r") is answer+++def _static(monkeypatch, name, fn):+ monkeypatch.setattr(ri.LlamaCppBackend, name, staticmethod(fn))+++def _ram(monkeypatch, gib):+ _static(monkeypatch, "_available_system_memory_mib", lambda: int(gib * 1024))+++def _gpus(monkeypatch, *rows):+ _static(monkeypatch, "_get_gpu_memory", lambda binary = None, **k: list(rows))+++def _caps(monkeypatch, **caps):+ _static(monkeypatch, "probe_server_capabilities", lambda binary = None: {"found": True, **caps})+++def _apple(monkeypatch, budget_gib):+ monkeypatch.setattr(hw, "is_apple_silicon", lambda: True)+ _static(monkeypatch, "_apple_metal_memory_budget_bytes", lambda: int(budget_gib * GIB))+++class TestEstimate:+ WEIGHTS = 73.45 * GIB++ @pytest.fixture(autouse = True)+ def _clean_env(self, monkeypatch):+ monkeypatch.delenv("LLAMA_ARG_LAZY_MODE", raising = False)+ monkeypatch.delenv("LLAMA_ARG_OVERRIDE_TENSOR", raising = False)++ @pytest.fixture+ def breakdown(self, tmp_path, monkeypatch):+ path = _flash_next(tmp_path)+ monkeypatch.setattr(ri, "_gguf_resident_file_gb", lambda cfg, **kw: self.WEIGHTS / GIB)+ _static(monkeypatch, "_get_gguf_size_bytes", lambda p: int(self.WEIGHTS))+ _caps(monkeypatch, supports_lazy_mode = True)+ _static(monkeypatch, "_is_vulkan_backend", lambda b = None: False)+ _gpus(monkeypatch, (0, 22000, 24576))+ _static(monkeypatch, "_integrated_cuda_gpu_ids", lambda: set())+ monkeypatch.setattr(hw, "is_apple_silicon", lambda: False)+ _static(monkeypatch, "_apple_metal_memory_budget_bytes", lambda: 0)+ _ram(monkeypatch, 16)+ ri._estimate_files_cache.clear()+ ri._estimate_config_cache.clear()+ config = SimpleNamespace(+ identifier = "local/flash-next", gguf_file = path, is_gguf = True, gguf_mmproj_file = None+ )++ def run(gguf = None, **kw):+ cfg = config if gguf is None else SimpleNamespace(**{**vars(config), "gguf_file": gguf})+ return ri._gguf_memory_breakdown(+ cfg, gguf or path, **{"n_ctx": 4096, "gpu_memory_mode": "auto", **kw}+ )++ yield run+ ri._estimate_files_cache.clear()+ ri._estimate_config_cache.clear()++ def test_streamed_table_leaves_the_gpu_and_the_total(self, breakdown):+ b = breakdown()+ assert (b.lookup_table_bytes, b.lookup_table_on_disk) == (FLASH_NEXT_TABLE, True)+ assert b.weights_bytes == int(round(self.WEIGHTS)) - FLASH_NEXT_TABLE+ assert b.gpu_bytes == b.total_bytes == b.weights_bytes + b.kv_bytes + b.compute_bytes++ def test_lazy_mode_off_keeps_it_resident_but_still_off_the_gpu(self, breakdown):+ b = breakdown(llama_extra_args = ["-lzm", "off"])+ assert (b.lookup_table_bytes, b.lookup_table_on_disk) == (FLASH_NEXT_TABLE, False)+ assert b.weights_bytes == int(round(self.WEIGHTS))+ assert b.gpu_bytes == b.total_bytes - FLASH_NEXT_TABLE++ @pytest.mark.parametrize("extras", [[], ["-lzm", "auto"]], ids = ["managed", "user_auto"])+ def test_ram_that_holds_the_table_reads_it_resident_as_the_launch_does(+ self, breakdown, monkeypatch, extras+ ):+ """A user's own lazy mode wins over the managed choice, as at launch."""+ _ram(monkeypatch, 256)+ b = breakdown(llama_extra_args = extras)+ assert (b.lookup_table_bytes, b.lookup_table_on_disk) == (FLASH_NEXT_TABLE, bool(extras))++ @pytest.mark.parametrize(+ "extras, on_disk",+ [+ ([], False),+ (["--parallel", "1"], False),+ (["-c", "0"], True),+ (["--parallel", "8"], True),+ ],+ ids = ["none", "same_parallel", "ctx_zero", "larger_parallel"],+ )+ def test_a_pass_through_that_reprices_the_cache_keeps_the_table_streamed(+ self, breakdown, monkeypatch, extras, on_disk+ ):+ _ram(monkeypatch, 256)+ assert breakdown(llama_extra_args = extras).lookup_table_on_disk is on_disk++ @pytest.mark.parametrize("ram_gib", [66, 256])+ @pytest.mark.parametrize("cpu_build", [False, True])+ def test_a_cpu_child_keeps_the_table_streamed_as_the_launch_does(+ self, breakdown, monkeypatch, cpu_build, ram_gib+ ):+ _ram(monkeypatch, ram_gib)+ _static(monkeypatch, "_binary_ships_no_gpu_backend", lambda *a, **k: cpu_build)+ assert breakdown().lookup_table_on_disk is cpu_build+ assert breakdown(llama_extra_args = ["--device", "none"]).lookup_table_on_disk is True++ def test_a_mig_slice_reporting_no_total_keeps_its_vram_credit(self, breakdown, monkeypatch):+ """Total 0 means an iGPU only on Vulkan; a CUDA MIG or vGPU slice can report it with VRAM of its own."""+ _ram(monkeypatch, 66)+ _gpus(monkeypatch, (0, 22000, 0))+ assert breakdown().lookup_table_on_disk is False++ @pytest.mark.parametrize("pool", ["soc", "apu", "discrete"])+ def test_a_unified_pool_beside_a_card_is_priced_once_as_the_launch_does(+ self, breakdown, monkeypatch, pool+ ):+ """An integrated CUDA SoC or ROCm APU's free memory is the host RAM the proof already counts."""+ _ram(monkeypatch, 58)+ _gpus(monkeypatch, (0, 16000, 16384), (1, 50000, 65536))+ monkeypatch.setattr(+ ri.LlamaCppBackend,+ "_INTEGRATED_CUDA_IDS",+ {ri.LlamaCppBackend._integrated_cuda_mask_key(): (1,) if pool == "soc" else ()},+ )+ monkeypatch.setattr(+ ri.get_llama_cpp_backend(),+ "_amd_apu_wants_unified_memory",+ lambda ids = None, *a, **k: (pool == "apu" and ids is not None and bool(set(ids) & {1})),+ raising = False,+ )+ assert breakdown().lookup_table_on_disk is (pool != "discrete")++ @pytest.mark.parametrize("tensor", [False, True])+ def test_an_inherited_split_mode_the_launch_drops_keeps_the_table_resident(+ self, breakdown, monkeypatch, tensor+ ):+ """Layer launches drop an inherited split mode none and tensor launches override it: neither child is on the CPU."""+ monkeypatch.setenv("LLAMA_ARG_SPLIT_MODE", "none")+ monkeypatch.setenv("LLAMA_ARG_MAIN_GPU", "-1")+ _ram(monkeypatch, 256)+ if tensor:+ _gpus(monkeypatch, (0, 22000, 24576), (1, 22000, 24576))+ assert breakdown(tensor_parallel = tensor).lookup_table_on_disk is False++ @pytest.mark.parametrize(+ "ram_gib, kw, on_disk",+ [+ (66, {}, False),+ (66, {"gpu_memory_mode": "manual", "gpu_layers": -1}, True),+ (66, {"gpu_memory_mode": "manual", "gpu_layers": 20}, True),+ (256, {"gpu_memory_mode": "manual", "gpu_layers": -1}, False),+ (256, {"gpu_memory_mode": "manual", "gpu_layers": -1, "n_ctx": 0}, True),+ ],+ ids = [+ "auto_gpu_credit",+ "manual_ram_only",+ "manual_layers_ram_only",+ "manual_fits",+ "manual_no_ctx",+ ],+ )+ def test_manual_mode_proves_residency_as_the_launch_does(+ self, breakdown, monkeypatch, ram_gib, kw, on_disk+ ):+ """Manual launches plan no cards (gpus = []), and Auto layers with no context leave it to --fit."""+ _ram(monkeypatch, ram_gib)+ assert breakdown(**kw).lookup_table_on_disk is on_disk++ def test_a_paravirtual_metal_estimate_keeps_the_table_streamed(self, breakdown, monkeypatch):+ _ram(monkeypatch, 256)+ assert breakdown().lookup_table_on_disk is False+ monkeypatch.setattr(ri, "_gpu_placement_paravirtual", lambda: True)+ assert breakdown().lookup_table_on_disk is True++ @pytest.mark.parametrize(+ "wired_gib, on_disk", [(None, False), (0, True), (200, False), (60, True)]+ )+ def test_the_apple_estimate_obeys_the_wired_headroom(+ self, breakdown, monkeypatch, wired_gib, on_disk+ ):+ _apple(monkeypatch, 120)+ _ram(monkeypatch, 256)+ _static(+ monkeypatch,+ "_apple_metal_wired_headroom_bytes",+ lambda: None if wired_gib is None else wired_gib * GIB,+ )+ assert breakdown().lookup_table_on_disk is on_disk++ def test_a_cpu_pinned_projector_is_charged_to_ram_in_the_residency_proof(+ self, breakdown, monkeypatch+ ):+ """A 96 GB card holds everything but the table, so 34 GiB of RAM keeps the table resident."""+ MM = 6 * GIB+ monkeypatch.setattr(+ ri, "_gguf_resident_file_gb", lambda cfg, **kw: (self.WEIGHTS + MM) / GIB+ )+ monkeypatch.setattr(ri, "_charged_projector_bytes", lambda *a, **k: MM)+ _gpus(monkeypatch, (0, 96000, 98304))+ _ram(monkeypatch, 34)+ assert breakdown().lookup_table_on_disk is False+ assert breakdown(llama_extra_args = ["--no-mmproj-offload"]).lookup_table_on_disk is True++ def test_ram_the_spilled_weights_crowd_out_keeps_the_table_streamed(+ self, breakdown, monkeypatch+ ):+ monkeypatch.delenv("LLAMA_ARG_CACHE_RAM", raising = False)+ _ram(monkeypatch, 58)+ _caps(monkeypatch, supports_lazy_mode = True, supports_cache_ram = True)+ assert breakdown().lookup_table_on_disk is True+ _gpus(monkeypatch, (0, 96_000, 98_304))+ assert breakdown().lookup_table_on_disk is False++ @pytest.mark.parametrize("budget_gib, ram_gib, resident", [(48, 64, False), (180, 256, True)])+ def test_apple_prices_the_whole_load_against_unified_memory(+ self, breakdown, monkeypatch, budget_gib, ram_gib, resident+ ):+ _apple(monkeypatch, budget_gib)+ _ram(monkeypatch, ram_gib)+ assert breakdown().lookup_table_on_disk is (not resident)++ @pytest.mark.parametrize(+ "indexed",+ [True, False],+ ids = [+ "an_indexed_table_name_keeps_its_override",+ "a_device_override_keeps_the_table_streamed_as_the_launch_does",+ ],+ )+ def test_an_indexed_table_name_keeps_its_override(self, breakdown, monkeypatch, indexed):+ _ram(monkeypatch, 256)+ name = "per_layer_token_embd_0" if indexed else "per_layer_token_embd"+ if indexed:+ real = ri.LlamaCppBackend._gguf_tensor_scan++ def scan(self, path):+ arch, named, experts = real(self, path)+ return (+ arch,+ {k.replace(PLE, f"{name}.weight"): v for k, v in named.items()},+ experts,+ )++ monkeypatch.setattr(ri.LlamaCppBackend, "_gguf_tensor_scan", scan)+ assert breakdown(llama_extra_args = ["-ot", f"{name}=CUDA0"]).lookup_table_on_disk is True++ def test_a_device_override_charges_a_resident_table_wholly_to_the_gpu(+ self, breakdown, tmp_path+ ):+ """A layer fraction does not apply to a tensor an -ot pins."""+ b = breakdown(+ gguf = _gemma(tmp_path),+ gpu_memory_mode = "manual",+ gpu_layers = 0,+ llama_extra_args = ["-ot", "per_layer_token_embd=CUDA0"],+ )+ assert b.lookup_table_bytes == 0+ assert b.gpu_bytes >= GEMMA_TABLE++ @pytest.mark.parametrize(+ "igpu",+ [False, True],+ ids = [+ "a_build_without_lazy_mode_keeps_the_table_resident",+ "an_integrated_gpu_reads_the_table_resident",+ ],+ )+ def test_a_build_without_lazy_mode_keeps_the_table_resident(self, breakdown, monkeypatch, igpu):+ """A Vulkan iGPU has no mmap support, so llama.cpp resolves auto to off."""+ if igpu:+ _static(monkeypatch, "_is_vulkan_backend", lambda b = None: True)+ _gpus(monkeypatch, (0, 30000, 0))+ else:+ _caps(monkeypatch)+ b = breakdown()+ assert b.lookup_table_on_disk is False+ assert b.weights_bytes == int(round(self.WEIGHTS))++ def test_a_pin_on_the_integrated_gpu_reads_the_table_resident(self, breakdown, monkeypatch):+ """A mixed host streams by default (llama.cpp drops the iGPU), not when pinned to it."""+ _static(monkeypatch, "_is_vulkan_backend", lambda b = None: True)+ _gpus(monkeypatch, (0, 30000, 0), (1, 22000, 24576))+ assert breakdown().lookup_table_on_disk is True+ assert breakdown(selected_gpu_ids = [0]).lookup_table_on_disk is False+ assert breakdown(selected_gpu_ids = [1]).lookup_table_on_disk is True+ stale = breakdown(selected_gpu_ids = [0], llama_extra_args = ["--device", "Vulkan1"])+ assert stale.lookup_table_on_disk is False++ def test_an_explicit_lazy_mode_on_prices_every_flagged_table(self, breakdown, monkeypatch):+ modes = []+ real = glt.local_lookup_info+ monkeypatch.setattr(+ glt,+ "local_lookup_info",+ lambda path, mode = "auto": modes.append(mode) or real(path, mode),+ )+ breakdown(llama_extra_args = ["-lzm", "on"])+ breakdown()+ assert modes == ["on", "auto"]++ def test_a_resident_gemma3n_table_leaves_the_gpu_share_not_the_total(self, breakdown, tmp_path):+ """Not lazily read, but still a host-pinned input like the launcher prices it."""+ b = breakdown(gguf = _gemma(tmp_path))+ assert (b.lookup_table_bytes, b.lookup_table_on_disk) == (GEMMA_TABLE, False)+ assert b.weights_bytes == int(round(self.WEIGHTS))+ assert b.gpu_bytes == b.total_bytes - b.lookup_table_bytes++ def test_a_resident_table_moved_to_the_gpu_stays_in_the_gpu_share(self, breakdown):+ b = breakdown(llama_extra_args = ["-lzm", "off", "-ot", "per_layer_token_embd=CUDA0"])+ assert (b.lookup_table_bytes, b.lookup_table_on_disk) == (0, False)+ assert b.gpu_bytes == b.total_bytes++ def test_the_load_panel_response_carries_the_table(self, breakdown):+ from core.inference.memory_contract import (+ build_memory_estimate,+ project_estimate_memory_response,+ )+ from models.inference import EstimateMemoryResponse++ est = build_memory_estimate(breakdown(), quant_file_bytes = int(self.WEIGHTS))+ out = project_estimate_memory_response(est)+ assert (out["lookup_table_bytes"], out["lookup_table_on_disk"]) == (FLASH_NEXT_TABLE, True)+ assert EstimateMemoryResponse(**out).lookup_table_bytes == FLASH_NEXT_TABLE++ def test_the_memory_bar_response_carries_the_table(self, breakdown):+ """Its weights_bytes is the whole file, so the bar needs the table to take it off the VRAM segment."""+ from core.inference.memory_contract import build_memory_estimate, project_kv_cache_estimate++ est = build_memory_estimate(breakdown(), quant_file_bytes = int(self.WEIGHTS))+ out = project_kv_cache_estimate(est)+ assert (out["weights_bytes"], out["lookup_table_bytes"]) == (+ int(self.WEIGHTS),+ FLASH_NEXT_TABLE,+ )+++def _split_blobs(first_arch = "qwen4exp", first_embd = ("token_embd.weight", [2048, 151936], IQ4_NL)):+ names = [f"Q4/m-{i:05d}-of-00003.gguf" for i in (1, 2, 3)]+ headers = [+ _header(first_arch, [first_embd], _ATTN),+ _header("", [(PLE, [160, 320001536], IQ4_NL)]),+ _header("", [("blk.3.attn_q.weight", [256, 256], F16)]),+ ]+ return names, {n: (h, 40 * GIB) for n, h in zip(names, headers)}+++class TestHubDiscovery:+ # Engram stored in a quant type gguf predates (ggml type 43); the next tensor bounds it.+ UNKNOWN_TYPE = 43++ def test_an_unknown_quant_type_is_sized_from_the_next_offset(self, tmp_path):+ path = _write(+ tmp_path / "dsv41-unknown.gguf",+ _header(+ "deepseek41",+ [+ (PLE, [256, 768022850], self.UNKNOWN_TYPE),+ ("blk.0.attn_q.weight", [256, 256], F16),+ ],+ offsets = [0, DSV41_TABLE],+ ),+ )+ assert glt.local_lookup_info(path) == ("deepseek41", DSV41_TABLE)++ def test_an_unknown_quant_type_last_in_the_file_is_sized_from_the_file(self, tmp_path):+ header = _header(+ "deepseek41",+ [_DSV41_EMBD, (PLE, [256, 768022850], self.UNKNOWN_TYPE)],+ offsets = [0, 4096],+ )+ size = -(-len(header) // 32) * 32 + 4096 + DSV41_TABLE+ path = tmp_path / "dsv41-last.gguf"+ with open(path, "wb") as f:+ f.write(header)+ f.truncate(size)+ assert glt.local_lookup_info(str(path)) == ("deepseek41", DSV41_TABLE)+ got = glt.remote_lookup_info(+ "o/unknown-type",+ "m.gguf",+ identity = size,+ fetcher_fn = _blob_fetcher({"m.gguf": (header, size)}, []),+ )+ assert got == ("deepseek41", DSV41_TABLE)++ def test_local_discovery_reads_every_shard(self, tmp_path):+ first = _write(tmp_path / "big-00001-of-00010.gguf", _header("deepseek41", [_DSV41_EMBD]))+ for i in range(2, 10):+ _write(+ tmp_path / f"big-{i:05d}-of-00010.gguf",+ _header("", [(f"blk.{i}.attn_q.weight", [256, 256], F16)]),+ )+ _write(tmp_path / "big-00010-of-00010.gguf", _header("", [_DSV41_PLE]))+ assert glt.local_lookup_table_bytes(first) == DSV41_TABLE++ def test_a_later_split_shard_starts_with_a_small_range(self):+ names, blobs = _split_blobs()+ calls = []+ got = glt.remote_lookup_info(+ "o/small-range", names[0], identity = 1, fetcher_fn = _blob_fetcher(blobs, calls)+ )+ assert got == ("qwen4exp", FLASH_NEXT_TABLE)+ later = [end + 1 for name, _start, end in calls if name != names[0]]+ assert later and max(later) <= 64 << 10, calls++ def test_a_known_architecture_reads_the_later_shards_before_the_tokenizer(self):+ names, blobs = _split_blobs()+ calls = []+ got = glt.remote_lookup_info(+ "o/known-arch",+ names[0],+ identity = 1,+ fetcher_fn = _blob_fetcher(blobs, calls),+ known_arch = "qwen4exp",+ )+ assert got == ("qwen4exp", FLASH_NEXT_TABLE)+ # The table's shard, then only the first shard's metadata for its architecture.+ assert [name for name, _s, _e in calls] == [names[1], names[0]]+ assert calls[1][2] + 1 <= 64 << 10, calls++ def test_the_listing_forwards_the_known_architecture(self, monkeypatch):+ seen = []+ _arch(monkeypatch, True)+ _remote(monkeypatch, lambda *a, **k: seen.append(k.get("known_arch")) or ("qwen4exp", 1))+ _list([_row("Q4/m.gguf")], "o/forward")+ assert seen == ["qwen4exp"]++ def test_auto_mode_skips_files_too_small_to_hold_a_streamed_table(self, monkeypatch):+ read_names = []++ def read(repo, name, *a, **k):+ read_names.append(name)+ return ("gemma4", 5 * GIB if name == "BF16.gguf" else 0)++ _arch(monkeypatch, True, "gemma4")+ _remote(monkeypatch, read)+ rows = [_row("Q4_K_M.gguf", "Q4_K_M", 3 * GIB), _row("BF16.gguf", "BF16", 15 * GIB)]+ out = _list(rows, "o/small-files")+ assert read_names == ["BF16.gguf"]+ assert _tables(out) == [0, 5 * GIB]++ @pytest.mark.parametrize(+ "repos",+ [("o/shared-read", "o/shared-read"), ("O/Cased-GGUF", "o/cased-gguf")],+ ids = [+ "a_listing_shares_a_table_read_already_in_flight",+ "differently_cased_listings_share_one_header_read",+ ],+ )+ def test_a_listing_shares_a_table_read_already_in_flight(self, monkeypatch, repos):+ calls = []+ monkeypatch.setattr(+ gguf_header_delta,+ "hub_range_fetcher",+ _blob_fetcher({"Q4/shared.gguf": (_LAZY_BLOB, 40 * GIB)}, calls, delay = 0.3),+ )+ _arch(monkeypatch, True)+ glt._REMOTE_CACHE.clear()+ # Room for a cold gguf import under a loaded runner; the overlap is what is tested.+ monkeypatch.setattr(gv, "_LOOKUP_DEADLINE_S", 10.0)+ rows = [_row("Q4/shared.gguf")]+ outs = _overlap(_listing(rows, repos[0]), _listing(rows, repos[1]))+ assert [_tables(o) for o in outs] == [[FLASH_NEXT_TABLE]] * 2+ assert len(calls) == 1++ def test_a_listing_shares_an_architecture_lookup_already_in_flight(self, monkeypatch):+ cache: dict = {}+ lock = threading.Lock()+ asked = []++ def arch(repo, token):+ with lock:+ if repo in cache:+ return cache[repo]+ asked.append(repo)+ time.sleep(0.3)+ with lock:+ cache[repo] = "qwen4exp"+ return "qwen4exp"++ _arch(monkeypatch, False, None)+ monkeypatch.setattr(gv, "_hub_repo_architecture", arch)+ _remote(monkeypatch)+ monkeypatch.setattr(gv, "_LOOKUP_DEADLINE_S", 10.0)+ rows = [_row()]+ outs = _overlap(+ _listing(rows, "o/shared-arch", token = "hf_same"),+ _listing(rows, "o/shared-arch", token = "hf_same"),+ )+ assert [_tables(o) for o in outs] == [[FLASH_NEXT_TABLE]] * 2+ assert asked == ["o/shared-arch"]+++class TestListingFollowups:+ def test_a_repo_hint_never_names_another_familys_split_variant(self):+ names, blobs = _split_blobs("gemma3n", ("token_embd.weight", [2048, 262144], IQ4_NL))+ got = glt.remote_lookup_info(+ "o/mixed-families",+ names[0],+ identity = 1,+ fetcher_fn = _blob_fetcher(blobs, []),+ known_arch = "qwen4exp",+ )+ assert got == ("gemma3n", 0)++ def test_the_remote_cache_ignores_repo_id_case(self):+ fetcher = _blob_fetcher({"Q4/case.gguf": (_LAZY_BLOB, 40 * GIB)}, [])+ glt.remote_lookup_info("Unsloth/Case-GGUF", "Q4/case.gguf", identity = 3, fetcher_fn = fetcher)+ assert glt.remote_cached("unsloth/case-gguf", "Q4/case.gguf", None, 3) == (+ "qwen4exp",+ FLASH_NEXT_TABLE,+ )++ def test_nested_metadata_arrays_are_skipped(self):+ # Splice one array-of-uint32-arrays KV in after the header's counts.+ n_tensors, n_kv = struct.unpack_from("<QQ", _LAZY_BLOB, 8)+ key = b"x.nested"+ inner = struct.pack("<IQ", 4, 3) + struct.pack("<3I", 1, 2, 3)+ kv = struct.pack("<Q", len(key)) + key + struct.pack("<IIQ", 9, 9, 2) + inner * 2+ spliced = _LAZY_BLOB[:8] + struct.pack("<QQ", n_tensors, n_kv + 1) + kv + _LAZY_BLOB[24:]+ assert glt.parse_header(spliced) == ("qwen4exp", {PLE: FLASH_NEXT_TABLE})++ def test_a_late_release_keeps_a_newer_claim(self):+ key = ("table", "o/owner")+ _held, first = gv._claim_lookup(key)+ gv._release_lookup(key, first)+ _held, second = gv._claim_lookup(key)+ gv._release_lookup(key, first) # the cancelled worker, running late+ assert gv._lookup_inflight.get(key) is second+ gv._release_lookup(key, second)+ assert key not in gv._lookup_inflight++ def test_a_listing_retries_a_read_that_failed_while_it_waited(self, monkeypatch):+ attempts = []+ lock = threading.Lock()+ reading = threading.Event()++ def read(repo, name, *a, **k):+ with lock:+ attempts.append(name)+ first = len(attempts) == 1+ reading.set()+ time.sleep(0.3)+ if first:+ raise OSError("transient")+ return ("qwen4exp", FLASH_NEXT_TABLE)++ _arch(monkeypatch, True)+ _remote(monkeypatch, read)+ monkeypatch.setattr(gv, "_LOOKUP_DEADLINE_S", 10.0)+ rows = [_row("Q4/retry.gguf")]+ outs = _overlap(_listing(rows, "o/retry"), _listing(rows, "o/retry"), wait = reading)+ assert [_tables(o) for o in outs] == [[0], [FLASH_NEXT_TABLE]]+ assert len(attempts) == 2++ def test_an_unreadable_local_copy_falls_back_to_the_hub_header(self, tmp_path, monkeypatch):+ """The first shard is on disk, the second still downloading."""+ first = _write(+ tmp_path / "m-00001-of-00002.gguf",+ _header("qwen4exp", [("token_embd.weight", [2048, 151936], IQ4_NL)], _ATTN),+ )+ monkeypatch.setattr(gv, "_local_variant_gguf", lambda *a, **k: first)+ _arch(monkeypatch, True)+ _remote(monkeypatch)+ out = _list([_row("m-00001-of-00002.gguf")], "o/partial")+ assert _tables(out) == [FLASH_NEXT_TABLE]++ def test_a_host_that_reads_tables_resident_reads_no_header(self, monkeypatch):+ looked = []+ monkeypatch.setattr(gv, "_host_streams_lookup_tables", lambda: False)+ monkeypatch.setattr(gv, "_local_variant_gguf", lambda *a, **k: looked.append(a) or None)+ answer = _answer([_row("Q4/m.gguf")])+ assert _list(answer, "o/resident") is answer and looked == []++ def test_a_cpu_build_streams_the_tables_beside_an_empty_gpu_inventory(self, monkeypatch):+ """No model device on a CPU-only build, so lazy auto stays on whatever the GPU probe says."""+ backend = _stream_backend(monkeypatch, vulkan = False, gpus = [])+ monkeypatch.setattr(lc, "_gpu_device_node_present", lambda: True)+ _no_gpu_backend(monkeypatch, True)+ assert backend.lookup_tables_stream_by_default(None, {}) is True+ _no_gpu_backend(monkeypatch, False)+ assert backend.lookup_tables_stream_by_default(["--device", "none"], {}) is True+ assert backend.lookup_tables_stream_by_default(None, {}) is False+ cpu_split = ["--split-mode", "none", "--main-gpu", "-1"]+ assert backend.lookup_tables_stream_by_default(cpu_split, {}) is True+ # Not under tensor parallel: Studio's --split-mode tensor beats the env.+ assert (+ backend.lookup_tables_stream_by_default(+ ["--main-gpu", "-1"], {"LLAMA_ARG_SPLIT_MODE": "none"}, tensor_parallel = True+ )+ is False+ )++ @pytest.mark.parametrize(+ "env",+ [+ {},+ {"LLAMA_ARG_SPLIT_MODE": "none", "LLAMA_ARG_MAIN_GPU": "-1"},+ {"LLAMA_ARG_SPLIT_MODE": "row", "LLAMA_ARG_MAIN_GPU": "-1"},+ ],+ ids = ["clean", "inherited_none", "inherited_row"],+ )+ def test_a_listing_drops_the_split_mode_the_launch_drops(self, monkeypatch, env):+ """A layer launch clears an inherited non-layer split mode, so the child runs on the shared Vulkan iGPU."""+ backend = _stream_backend(monkeypatch, vulkan = True, gpus = [(0, 8000, 0)])+ _no_gpu_backend(monkeypatch, False)+ assert backend.lookup_tables_stream_by_default(None, env) is Falsediff --git a/studio/backend/tests/test_memory_estimate_contract_freeze.py b/studio/backend/tests/test_memory_estimate_contract_freeze.pyindex 25850c846a6..4b47b99df58 100644--- a/studio/backend/tests/test_memory_estimate_contract_freeze.py+++ b/studio/backend/tests/test_memory_estimate_contract_freeze.py@@ -95,6 +95,7 @@ _KV_CACHE_ESTIMATE_KEYS = frozenset({"kv_bytes","weights_bytes",+ "lookup_table_bytes","native_context","spec_bytes","n_ctx",@@ -136,6 +137,32 @@ class TestTheKvCacheEstimateEnvelope:"This is a narrowing and it breaks callers.")+ def test_the_null_answer_carries_the_same_keys(self, monkeypatch):+ """An unresolved quant answers every shipped key, nulled."""+ out = _call_route(+ monkeypatch,+ path = "",+ weights_bytes = 0,+ repo_id = "unsloth/contract-freeze-GGUF",+ speculative_type = None,+ )+ assert out is not None and out["kv_bytes"] is None+ assert set(out) == _KV_CACHE_ESTIMATE_KEYS++ def test_no_lookup_table_reads_as_none_not_zero(self, monkeypatch, tmp_path):+ """The route's rule for every term it itemizes: None for "no such term"."""+ gguf = _write_gguf(tmp_path / "model-Q4_K_M.gguf", _PLAIN_GQA)+ _reach_the_planner(monkeypatch, gguf)+ out = _call_route(+ monkeypatch,+ path = gguf,+ weights_bytes = 4_000_000_000,+ repo_id = "unsloth/contract-freeze-GGUF",+ speculative_type = None,+ )+ assert out["kv_bytes"] is not None+ assert out["lookup_table_bytes"] is None+def test_weights_bytes_is_the_quant_file_alone(self, monkeypatch, tmp_path):# Planner reached so the aggregate differs; otherwise this holds vacuously.quant_size = 4_123_456_789diff --git a/studio/backend/tests/test_mtp_vram_budget.py b/studio/backend/tests/test_mtp_vram_budget.pyindex 9c647e7fa01..8554d1ea447 100644--- a/studio/backend/tests/test_mtp_vram_budget.py+++ b/studio/backend/tests/test_mtp_vram_budget.py@@ -972,15 +972,19 @@ class TestExtraArgsMtpDetection:def test_load_model_clears_inherited_split_mode_on_layer(self):compact = "".join(inspect.getsource(LlamaCppBackend.load_model).split())- assert 'env.get("LLAMA_ARG_SPLIT_MODE")' in compact- assert '_inherited_sm!="layer"' in compact- assert 'env.pop("LLAMA_ARG_SPLIT_MODE",None)' in compact- assert 'env.pop("LLAMA_ARG_TENSOR_SPLIT",None)' in compact+ assert "self._reconcile_split_mode_env(env,tensor_parallel=tensor_parallel)" in compact+ for sm in ("none", "row", "tensor", " Row "):+ env = {"LLAMA_ARG_SPLIT_MODE": sm, "LLAMA_ARG_TENSOR_SPLIT": "1,1", "X": "1"}+ LlamaCppBackend._reconcile_split_mode_env(env, tensor_parallel = False)+ assert env == {"X": "1"}, sm+ env = {"LLAMA_ARG_SPLIT_MODE": "layer", "LLAMA_ARG_TENSOR_SPLIT": "1,1"}+ LlamaCppBackend._reconcile_split_mode_env(env, tensor_parallel = False)+ assert env == {"LLAMA_ARG_SPLIT_MODE": "layer", "LLAMA_ARG_TENSOR_SPLIT": "1,1"}def test_load_model_clears_tensor_split_env_in_tensor_mode(self):- src = inspect.getsource(LlamaCppBackend.load_model)- compact = "".join(src.split())- assert compact.count('env.pop("LLAMA_ARG_TENSOR_SPLIT",None)') >= 2+ env = {"LLAMA_ARG_SPLIT_MODE": "tensor", "LLAMA_ARG_TENSOR_SPLIT": "1,1"}+ LlamaCppBackend._reconcile_split_mode_env(env, tensor_parallel = True)+ assert env == {"LLAMA_ARG_SPLIT_MODE": "tensor"}def test_load_model_layer_compute_buffer_fallback(self):compact = "".join(inspect.getsource(LlamaCppBackend.load_model).split())diff --git a/studio/backend/tests/test_offload_planner.py b/studio/backend/tests/test_offload_planner.pyindex 1a26d4f286e..15134ea8fde 100644--- a/studio/backend/tests/test_offload_planner.py+++ b/studio/backend/tests/test_offload_planner.py@@ -927,7 +927,9 @@ def test_the_layout_reader_totals_every_tensor_from_a_real_gguf(tmp_path):layout = layout_from_gguf(str(path))assert layout.completeassert layout.tensor_bytes == 4 * (16 + 1024 + 4096 + 256 + 64)- assert layout.token_embd_bytes == 4 * (1024 + 4096)+ assert layout.token_embd_bytes == 4 * 1024+ assert layout.per_layer_embd_bytes == 4 * 4096+ assert layout.other_resident_bytes == 4 * (16 + 1024)@pytest.mark.parametrize("endianess", ["LITTLE", "BIG"])diff --git a/studio/backend/tests/test_offload_planner_seam.py b/studio/backend/tests/test_offload_planner_seam.pyindex c561f67eeb6..680a782720c 100644--- a/studio/backend/tests/test_offload_planner_seam.py+++ b/studio/backend/tests/test_offload_planner_seam.py@@ -1071,8 +1071,8 @@ def test_a_reconciliation_still_strips_a_non_layer_split_mode_env():"""Reachability anchor for the decline above: if load_model ever stopsscrubbing the inherited mode, the env form becomes plannable again and this isthe test that should fail and say so."""- src = inspect.getsource(LlamaCppBackend.load_model)- compact = "".join(src.split())+ assert "self._reconcile_split_mode_env(env" in inspect.getsource(LlamaCppBackend.load_model)+ compact = "".join(inspect.getsource(LlamaCppBackend._reconcile_split_mode_env).split())assert 'if_inherited_smand_inherited_sm!="layer":' in compactassert 'env.pop("LLAMA_ARG_SPLIT_MODE",None)' in compactassert 'env.pop("LLAMA_ARG_SPLIT_MODE",None)env.pop("LLAMA_ARG_TENSOR_SPLIT",None)' in compactdiff --git a/studio/backend/tests/test_tensor_parallel.py b/studio/backend/tests/test_tensor_parallel.pyindex 6b1c9c7b699..a79a73386f2 100644--- a/studio/backend/tests/test_tensor_parallel.py+++ b/studio/backend/tests/test_tensor_parallel.py@@ -1600,7 +1600,7 @@ def test_load_model_tensor_admission_and_capacity_gate_use_usable_budget():# The gate's required footprint must include the non-shrinkable MTP reserve,# not weights alone, or a separate-drafter MTP load can still overcommit.assert "_tp_mtp_floor" in src- assert "model_size + _tp_mtp_floor" in src+ assert "model_size - _host_input_embd + _tp_mtp_floor" in srcdef test_load_model_tensor_floor_keeps_flat_reserve_for_weights_only():diff --git a/studio/frontend/scripts/check-bundle-budget.ts b/studio/frontend/scripts/check-bundle-budget.tsindex 26a86325cde..4d297245fd2 100644--- a/studio/frontend/scripts/check-bundle-budget.ts+++ b/studio/frontend/scripts/check-bundle-budget.ts@@ -424,8 +424,10 @@ export const BUDGET = {// model picker (6,960,469 -> 6,960,551 B raw, 2,100,277 -> 2,100,453 B transfer against c9ddc5ab5e, one machine and// build, Node 26, 105 chunks). Raised by that growth.// Cosmos3 video families (#14554): +466 B raw / +218 B transfer. Raised by that.- transferBytes: 2_100_715,- rawBytes: 6_972_038,+ // Lookup-table pricing (#14465): +1,517 B raw / +470 B transfer on top of the lines above (measured merged with+ // 4db3eaeb55, Node 24, 99 chunks).+ transferBytes: 2_101_185,+ rawBytes: 6_973_555,};// Chunk count is not budgeted: splitting a page out raises it while lowering bytes.diff --git a/studio/frontend/src/features/chat/api/chat-api.ts b/studio/frontend/src/features/chat/api/chat-api.tsindex d70d52f6b32..87f631c0096 100644--- a/studio/frontend/src/features/chat/api/chat-api.ts+++ b/studio/frontend/src/features/chat/api/chat-api.ts@@ -1622,6 +1622,8 @@ export async function listGgufVariants(export interface KvCacheEstimate {kv_bytes: number | null;weights_bytes: number | null;+ /** Host / disk lookup tables, part of weights_bytes but never on the GPU; null when none. */+ lookup_table_bytes?: number | null;native_context: number | null;/** Extra MTP draft reserve; null for ngram or a model with no MTP head. */spec_bytes: number | null;diff --git a/studio/frontend/src/features/chat/types/api.ts b/studio/frontend/src/features/chat/types/api.tsindex 7e8d31b0adf..90c17efc0a1 100644--- a/studio/frontend/src/features/chat/types/api.ts+++ b/studio/frontend/src/features/chat/types/api.ts@@ -163,6 +163,8 @@ export interface GgufVariantDetail {display_label?: string | null;size_bytes: number;download_size_bytes?: number;+ /** Streamed lookup-table bytes inside `size_bytes`; 0 when none or on an older backend. */+ lookup_table_bytes?: number;pending_drafter_filename?: string | null;pending_drafter_size_bytes?: number;downloaded?: boolean;diff --git a/studio/frontend/src/features/hub/catalog/gguf-download-card.tsx b/studio/frontend/src/features/hub/catalog/gguf-download-card.tsxindex ccbcc8444eb..07f55e9b8c6 100644--- a/studio/frontend/src/features/hub/catalog/gguf-download-card.tsx+++ b/studio/frontend/src/features/hub/catalog/gguf-download-card.tsx@@ -33,7 +33,11 @@ import {import { useVramBudgetFraction } from "@/hooks/use-vram-budget-fraction";import { ChevronDownStandardIcon } from "@/lib/chevron-icons";import { copyToClipboard } from "@/lib/copy-to-clipboard";-import { type GgufFitClass, classifyGgufVariantFit } from "@/lib/gguf-fit";+import {+ type GgufFitClass,+ classifyGgufVariantFit,+ ggufVariantLookupTableBytes,+} from "@/lib/gguf-fit";import { toast } from "@/lib/toast";import { cn } from "@/lib/utils";import {@@ -145,6 +149,12 @@ const FIT_BADGE: Record<GgufFitClass, FitBadgeMeta> = {"No GPU VRAM detected. This GGUF may run with system RAM and CPU offload. Inference will be slower.",iconClassName: "text-sky-600 dark:text-sky-400",},+ streamed: {+ label: "Fits (lookup tables streamed from disk)",+ tooltip:+ "Fits with its lookup tables streamed from disk, llama.cpp's default for them. Prefill and decode can be slower and vary with disk speed.",+ iconClassName: "text-sky-600 dark:text-sky-400",+ },oom: {label: "Does not fit",// Not "won't fit": llama-server never refuses on size, it uses --fit. Matches the chat picker.@@ -162,12 +172,14 @@ const CHIP_DEFAULT =function QuantBadge({quant,fit,+ lookupTableBytes = 0,showFit = true,variant = "trigger",tooltipMode = "eager",}: {quant: string;fit: GgufFitClass;+ lookupTableBytes?: number;showFit?: boolean;variant?: "trigger" | "menu";tooltipMode?: "eager" | "lazy" | "none";@@ -237,6 +249,13 @@ function QuantBadge({</TooltipTrigger><TooltipContent side="top" sideOffset={4}>{meta.tooltip}+ {lookupTableBytes > 0 && (+ <span className="mt-1 block text-muted-foreground">+ Lookup tables: {formatFootprintBytes(lookupTableBytes)}+ {/* Streamed only when RAM cannot hold them; otherwise the load keeps them resident. */}+ {fit === "streamed" ? ", read from disk" : ", kept in RAM when it fits"}+ </span>+ )}</TooltipContent></Tooltip>);@@ -248,6 +267,7 @@ interface GgufVariantMenuItem {quant: string;label: string;fit: GgufFitClass;+ lookupTableBytes: number;downloaded: boolean;partial: boolean;downloadSizeLabel: string;@@ -347,6 +367,7 @@ function createGgufVariantMenuItems(quant: variant.quant,label: ggufVariantDisplayLabel(variant),fit: classifyGgufVariantFit(variant, resources),+ lookupTableBytes: ggufVariantLookupTableBytes(variant),downloaded: Boolean(variant.downloaded),partial: Boolean(variant.partial),downloadSizeLabel: ggufVariantTransferLabel(variant),@@ -580,6 +601,7 @@ const GgufVariantMenuRow = memo(function GgufVariantMenuRow({<QuantBadgequant={item.label}fit={item.fit}+ lookupTableBytes={item.lookupTableBytes}showFit={showFitInfo}variant="menu"tooltipMode="lazy"@@ -1144,6 +1166,7 @@ export function GgufDownloadCard({<QuantBadgequant={selectedLabel ?? selected.quant}fit={selectedFit ?? "oom"}+ lookupTableBytes={ggufVariantLookupTableBytes(selected)}showFit={showFitInfo}/>) : (@@ -1359,6 +1382,7 @@ export function GgufDownloadCard({repoId={repoId}quant={selected.quant}sizeBytes={selected.size_bytes}+ lookupTableBytes={ggufVariantLookupTableBytes(selected)}gpuGb={gpuGb}showReadout={true}className="px-1"diff --git a/studio/frontend/src/features/hub/catalog/local-on-device-card.tsx b/studio/frontend/src/features/hub/catalog/local-on-device-card.tsxindex 77b39ec3397..56c2a287625 100644--- a/studio/frontend/src/features/hub/catalog/local-on-device-card.tsx+++ b/studio/frontend/src/features/hub/catalog/local-on-device-card.tsx@@ -52,6 +52,7 @@ import {} from "../lib/gguf-filename";import {ggufVariantDisplayLabel,+ localGgufMemoryBarVariant,resolveLocalGgufVariant,sortLocalGgufVariants,} from "../lib/gguf-variant-sort";@@ -71,6 +72,7 @@ import { PathInfoButton } from "./path-info-button";import { TransportConflictDialog } from "./transport-conflict-dialog";import { useCardDelete } from "./use-card-delete";import { useGgufVariantFetchState } from "./use-gguf-variant-fetch-state";+import { ggufVariantLookupTableBytes } from "@/lib/gguf-fit";interface LocalOnDeviceCardProps {modelId: string;@@ -349,6 +351,10 @@ export function LocalOnDeviceCard({...variant,download_size_bytes:remoteVariant.download_size_bytes || variant.download_size_bytes,+ // An update's reading wins, even 0: the local header is the old revision.+ lookup_table_bytes: remoteVariant.update_available+ ? (remoteVariant.lookup_table_bytes ?? 0)+ : variant.lookup_table_bytes || remoteVariant.lookup_table_bytes,// Both sides measure the same cache; prefer local, fall back to remote.download_remaining_bytes:variant.download_remaining_bytes ??@@ -400,6 +406,10 @@ export function LocalOnDeviceCard({defaultVariant: currentVariantState.defaultVariant,});const selectedQuant = selectedVariant?.quant ?? null;+ const memoryBarVariant = localGgufMemoryBarVariant(+ selectedVariant,+ currentVariantState.variants,+ );const updateJobActive = useDownloadManagerStore((s) =>repoId? Boolean(@@ -649,7 +659,12 @@ export function LocalOnDeviceCard({repoId={repoId || localGgufPath}loadId={localGgufPath}quant={selectedQuant ?? ""}- sizeBytes={selectedVariant?.size_bytes}+ sizeBytes={memoryBarVariant?.size_bytes}+ lookupTableBytes={+ memoryBarVariant+ ? ggufVariantLookupTableBytes(memoryBarVariant)+ : 0+ }gpuGb={gpuGb}className="px-3 pb-2"/>diff --git a/studio/frontend/src/features/hub/inventory/api.ts b/studio/frontend/src/features/hub/inventory/api.tsindex db56c9d063c..9f78a2c37a1 100644--- a/studio/frontend/src/features/hub/inventory/api.ts+++ b/studio/frontend/src/features/hub/inventory/api.ts@@ -200,6 +200,8 @@ export interface GgufVariantDetail {display_label?: string | null;size_bytes: number;download_size_bytes?: number;+ /** Streamed lookup-table bytes inside `size_bytes`; 0 when none or on an older backend. */+ lookup_table_bytes?: number;pending_drafter_filename?: string | null;pending_drafter_size_bytes?: number;download_remaining_bytes?: number | null;diff --git a/studio/frontend/src/features/hub/lib/gguf-variant-sort.ts b/studio/frontend/src/features/hub/lib/gguf-variant-sort.tsindex e40f243ecc7..006714c84f9 100644--- a/studio/frontend/src/features/hub/lib/gguf-variant-sort.ts+++ b/studio/frontend/src/features/hub/lib/gguf-variant-sort.ts@@ -48,6 +48,7 @@ export function ggufVariantFitRank(return 1;case "partial":case "ram":+ case "streamed":return 2;default:return 3;@@ -130,3 +131,19 @@ export function resolveLocalGgufVariant<T extends { quant: string }>(}return variants[0] ?? null;}++// The memory bar loads the cached file, so an update's merged row (published+// lookup table, cached size) would mix two revisions; read the cached row.+export function localGgufMemoryBarVariant<T extends { quant: string }>(+ selected: (T & { update_available?: boolean }) | null,+ localVariants: readonly T[] | null | undefined,+): T | null {+ if (!selected?.update_available) {+ return selected;+ }+ return (+ localVariants?.find((variant) =>+ ggufVariantsMatch(variant.quant, selected.quant),+ ) ?? selected+ );+}diff --git a/studio/frontend/src/features/model-picker/api/memory-estimate.ts b/studio/frontend/src/features/model-picker/api/memory-estimate.tsindex b42482c3822..479106d8454 100644--- a/studio/frontend/src/features/model-picker/api/memory-estimate.ts+++ b/studio/frontend/src/features/model-picker/api/memory-estimate.ts@@ -46,6 +46,9 @@ export interface MemoryEstimate {gpuLayers: number | null;/** `--n-cpu-moe` is set, so the GPU figure ignores it and reads high. */moeOffloadUnmodelled: boolean;+ lookupTableBytes?: number;+ /** Tables streamed from disk, so excluded from `weightsBytes` and `totalBytes`. */+ lookupTableOnDisk?: boolean;/** MLX: share of the PLE tables / routed experts read from disk, priced in `weightsBytes`. */mlxPleOffload: number | null;mlxExpertOffload: number | null;@@ -140,6 +143,8 @@ interface ApiEstimateResponse {layer_count: number | null;gpu_layers: number | null;moe_offload_unmodelled: boolean;+ lookup_table_bytes?: number;+ lookup_table_on_disk?: boolean;mlx_offload?: { ple?: unknown; experts?: unknown; offloadable?: unknown } | null;}@@ -260,6 +265,8 @@ function toMemoryEstimate(body: ApiEstimateResponse): MemoryEstimate {layerCount: nullableCount(body.layer_count),gpuLayers: nullableCount(body.gpu_layers),moeOffloadUnmodelled: flag(body.moe_offload_unmodelled, false),+ lookupTableBytes: finiteBytes(body.lookup_table_bytes, 0),+ lookupTableOnDisk: flag(body.lookup_table_on_disk, false),mlxPleOffload: offloadFraction(body.mlx_offload?.ple),mlxExpertOffload: offloadFraction(body.mlx_offload?.experts),mlxOffloadable: offloadableParts(body.mlx_offload?.offloadable),diff --git a/studio/frontend/src/features/model-picker/components/memory-estimate-row.tsx b/studio/frontend/src/features/model-picker/components/memory-estimate-row.tsxindex ec6b43a1738..317474b7f36 100644--- a/studio/frontend/src/features/model-picker/components/memory-estimate-row.tsx+++ b/studio/frontend/src/features/model-picker/components/memory-estimate-row.tsx@@ -323,6 +323,15 @@ export function MemoryEstimateRow({value={formatMemoryGb(estimate.projectorRuntimeBytes)}/>)}+ {/* Streamed: outside Weights and Total, only the rows in use page in. Resident:+ already inside Weights, so the note says so rather than adding it twice. */}+ {(estimate.lookupTableBytes ?? 0) > 0 && (+ <MemoryBreakdownLine+ label="Lookup tables"+ value={formatMemoryGb(estimate.lookupTableBytes ?? 0)}+ note={estimate.lookupTableOnDisk ? "on disk" : "in RAM, part of Weights"}+ />+ )}{estimate.drafterRuntimeBytes > 0 && (<MemoryBreakdownLinelabel="Draft cache"diff --git a/studio/frontend/src/features/model-picker/components/model-selector/gguf-discovery.ts b/studio/frontend/src/features/model-picker/components/model-selector/gguf-discovery.tsindex 578b2fa416e..3512e40379c 100644--- a/studio/frontend/src/features/model-picker/components/model-selector/gguf-discovery.ts+++ b/studio/frontend/src/features/model-picker/components/model-selector/gguf-discovery.ts@@ -21,6 +21,12 @@ function withHubState<V extends GgufVariantDetail>(download_size_bytes: published.download_size_bytes,}: {}),+ // A newer revision's reading wins, even 0; else local, with the Hub filling an unread 0.+ lookup_table_bytes: published?.update_available+ ? (published.lookup_table_bytes ?? 0)+ : published+ ? local.lookup_table_bytes || published.lookup_table_bytes+ : local.lookup_table_bytes,};}diff --git a/studio/frontend/src/features/model-picker/components/model-selector/model-catalog.ts b/studio/frontend/src/features/model-picker/components/model-selector/model-catalog.tsindex 816a0c14da5..0cf6cf0ae0b 100644--- a/studio/frontend/src/features/model-picker/components/model-selector/model-catalog.ts+++ b/studio/frontend/src/features/model-picker/components/model-selector/model-catalog.ts@@ -15,6 +15,7 @@ import {type GgufFitClass,type GgufVariantSizes,classifyGgufFit as classifyGgufFitForDevice,+ classifyGgufFitWithLookupTables,ggufVariantFitSizeBytes,} from "../../../../lib/gguf-fit.ts";import {@@ -1349,7 +1350,9 @@ export function recommendedQuantForDevice<T extends GgufVariantSizes>(preferred: T | null = null,): T | null {// Judged on the download footprint, which includes companion GGUFs (mmproj, drafter).- const fitOfVariant = (variant: T) => fitOf(ggufVariantFitSizeBytes(variant));+ // A `streamed` quant (tables left on disk) runs but never outranks one that fits fully resident.+ const fitOfVariant = (variant: T) =>+ classifyGgufFitWithLookupTables(fitOf, variant);// Zero size means unknown (old backend or stat error) and must not compete as comfortable.const sized = variants.filter((variant) => ggufVariantFitSizeBytes(variant) > 0);if (sized.length === 0) return null;@@ -1384,6 +1387,7 @@ export interface QuantVariant {filename: string;size_bytes: number;downloaded?: boolean;+ lookup_table_bytes?: number;}/** The quant a bare group/repo click loads: largest downloaded non-OOM quant, else the repo default@@ -1396,7 +1400,12 @@ export function pickDefaultQuant(if (!variants || variants.length === 0) return null;const anyBudget = (budget.gpuGb || 0) > 0 || (budget.systemRamGb || 0) > 0;const runs = (v: QuantVariant) =>- ggufFitRuns(classifyGgufFit(v.size_bytes, budget));+ ggufFitRuns(+ classifyGgufFitWithLookupTables(+ (bytes) => classifyGgufFit(bytes, budget),+ { size_bytes: v.size_bytes, lookup_table_bytes: v.lookup_table_bytes },+ ),+ );const downloadedFitting = variants.filter((v) => v.downloaded && runs(v)).sort((a, b) => b.size_bytes - a.size_bytes);diff --git a/studio/frontend/src/features/model-picker/components/model-selector/pickers.tsx b/studio/frontend/src/features/model-picker/components/model-selector/pickers.tsxindex 8e7546e301e..fc746785825 100644--- a/studio/frontend/src/features/model-picker/components/model-selector/pickers.tsx+++ b/studio/frontend/src/features/model-picker/components/model-selector/pickers.tsx@@ -120,7 +120,8 @@ import { copyToClipboard } from "@/lib/copy-to-clipboard";import { diffusionRouteSearch } from "@/lib/diffusion-route-search";import {type GgufFitClass,- ggufVariantFitSizeBytes,+ classifyGgufFitWithLookupTables,+ ggufVariantLookupTableBytes,requiredGgufMemoryGb,} from "@/lib/gguf-fit";import { formatGiB } from "@/lib/memory/format";@@ -825,6 +826,11 @@ const VRAM_VERDICT: Record<PickerFitStatus, FitVerdict | null> = {tone: ORANGE,hint: "No GPU detected. Runs on system RAM and CPU. Expect much slower inference.",},+ streamed: {+ label: "Fits (lookup tables streamed from disk)",+ tone: AMBER,+ hint: "Fits with its lookup tables streamed from disk, llama.cpp's default for them. Prefill and decode can be slower and vary with disk speed.",+ },oom: OFFLOADS,exceeds: WONT_FIT,};@@ -2087,7 +2093,7 @@ function GgufVariantExpander({// One verdict per variant from the download footprint (incl. companions), so badge, order and star agree.const getVariantFit = useCallback((variant: GgufVariantDetail): GgufFitClass =>- getGgufFit(ggufVariantFitSizeBytes(variant)),+ classifyGgufFitWithLookupTables(getGgufFit, variant),[getGgufFit],);@@ -4948,11 +4954,13 @@ export function HubModelPicker({downloaded: ReadonlyMap<string, string | null>;visionByRepo: ReadonlyMap<string, boolean>;sizes: ReadonlyMap<string, number>;+ lookupTables: ReadonlyMap<string, number>;}>({validated: false,downloaded: new Map(),visionByRepo: new Map(),sizes: new Map(),+ lookupTables: new Map(),});const prunePinnedQuantValidation = useCallback((repoId: string, quant: string) => {@@ -5001,10 +5009,23 @@ export function HubModelPicker({(variant) =>[pinKey(repoId, variant.quant), variant.size_bytes] as const,),+ lookupTables: downloaded.map(+ (variant) =>+ [+ pinKey(repoId, variant.quant),+ ggufVariantLookupTableBytes(variant),+ ] as const,+ ),};} catch {// Hide unverifiable quants rather than claim a missing file is downloaded.- return { repoId, hasVision: undefined, downloaded: [], sizes: [] };+ return {+ repoId,+ hasVision: undefined,+ downloaded: [],+ sizes: [],+ lookupTables: [],+ };}}),).then((groups) => {@@ -5020,6 +5041,9 @@ export function HubModelPicker({),),sizes: new Map(groups.flatMap((group) => group.sizes)),+ lookupTables: new Map(+ groups.flatMap((group) => group.lookupTables),+ ),});}});@@ -6427,7 +6451,14 @@ export function HubModelPicker({diffusionTaskById.get(entry.repoId.toLowerCase()),)? undefined- : { repoId: entry.repoId, quant: entry.quant }+ : {+ repoId: entry.repoId,+ quant: entry.quant,+ sizeBytes,+ lookupTableBytes: pinnedQuantValidation.lookupTables.get(+ pinKey(entry.repoId, entry.quant),+ ),+ }}gpuGb={expanderGpuGb}alignMeta="device"@@ -6576,6 +6607,7 @@ export function HubModelPicker({repoId: c.repo_id,quant: variant.quant,sizeBytes: variant.size_bytes,+ lookupTableBytes: ggufVariantLookupTableBytes(variant),loadId: c.load_id,}}diff --git a/studio/frontend/src/hooks/use-model-memory.ts b/studio/frontend/src/hooks/use-model-memory.tsindex fcbf3daaf39..d5fff8cda03 100644--- a/studio/frontend/src/hooks/use-model-memory.ts+++ b/studio/frontend/src/hooks/use-model-memory.ts@@ -37,6 +37,7 @@ import {extraArgsAddResidentFiles,extraArgsOwnPlacement,extraArgsShapeKvCache,+ memoryBarWeightsBytes,} from "@/lib/model-memory";import { useInferenceGpuInfo } from "./use-gpu-info";import { useEffect, useMemo, useState, useSyncExternalStore } from "react";@@ -46,6 +47,7 @@ export interface ModelMemorySource {quant: string;/** Size from the listing, so the bar draws before the estimate arrives. */sizeBytes?: number | null;+ lookupTableBytes?: number | null;/** What selecting the row loads, so a duplicate HF cache cannot price a different copy. */loadId?: string | null;}@@ -263,7 +265,11 @@ async function fetchEstimate(.then((r) => {const estimate: Estimate = {kvBytes: r.kv_bytes,- weightsBytes: r.weights_bytes,+ // weights_bytes is the whole file; its lookup tables never reach the GPU.+ weightsBytes:+ r.weights_bytes == null+ ? null+ : Math.max(0, r.weights_bytes - (r.lookup_table_bytes ?? 0)),specBytes: r.spec_bytes,nCtx: r.n_ctx ?? 0,projectorBytes: r.projector_bytes ?? null,@@ -414,7 +420,7 @@ export function useModelMemory(if (estimate?.specUnpriced) return computeModelMemory({});// An env device pin is the one placement override budgetIsMeaningful cannot see.if (estimate?.inheritedDevicePin) return computeModelMemory({});- const weights = estimate?.weightsBytes ?? source?.sizeBytes;+ const weights = memoryBarWeightsBytes(estimate, source);return computeModelMemory({// The projector is resident alongside the weights.weightsBytes:@@ -441,5 +447,5 @@ export function useModelMemory(? plan.nCtx == null: !estimate.contextIsPinned,});- }, [plan, entry, source?.sizeBytes, budgetGb, budgetFraction]);+ }, [plan, entry, source?.sizeBytes, source?.lookupTableBytes, budgetGb, budgetFraction]);}diff --git a/studio/frontend/src/lib/gguf-fit.ts b/studio/frontend/src/lib/gguf-fit.tsindex 5d6d7399035..dcdc6df5bfd 100644--- a/studio/frontend/src/lib/gguf-fit.ts+++ b/studio/frontend/src/lib/gguf-fit.ts@@ -3,7 +3,13 @@/** Mirrors llama_cpp.py `_select_gpus`, so the Hub card and chat picker cannot disagree. */-export type GgufFitClass = "fits" | "marginal" | "partial" | "ram" | "oom";+export type GgufFitClass =+ | "fits"+ | "marginal"+ | "partial"+ | "ram"+ | "streamed"+ | "oom";export interface GgufFitInput {gpuGb?: number;@@ -97,6 +103,8 @@ export function classifyGgufFit(export interface GgufVariantSizes {size_bytes: number;download_size_bytes?: number;+ /** Lookup tables inside `size_bytes` llama.cpp reads lazily from disk; 0 on older backends. */+ lookup_table_bytes?: number;}/** The larger of `size_bytes` and `download_size_bytes` (companions like mmproj or a drafter).@@ -105,9 +113,30 @@ export function ggufVariantFitSizeBytes(variant: GgufVariantSizes): number {return Math.max(variant.size_bytes, variant.download_size_bytes ?? 0);}+export function ggufVariantLookupTableBytes(variant: GgufVariantSizes): number {+ const table = Number(variant.lookup_table_bytes ?? 0);+ if (!Number.isFinite(table) || table <= 0) return 0;+ return Math.min(table, ggufVariantFitSizeBytes(variant));+}++export function classifyGgufFitWithLookupTables(+ fitOf: (sizeBytes: number) => GgufFitClass,+ variant: GgufVariantSizes,+): GgufFitClass {+ const size = ggufVariantFitSizeBytes(variant);+ const full = fitOf(size);+ const table = ggufVariantLookupTableBytes(variant);+ if (table <= 0) return full;+ if (full !== "oom") return full;+ return fitOf(size - table) === "oom" ? full : "streamed";+}+export function classifyGgufVariantFit(variant: GgufVariantSizes,input: GgufFitInput,): GgufFitClass {- return classifyGgufFit(ggufVariantFitSizeBytes(variant), input);+ return classifyGgufFitWithLookupTables(+ (sizeBytes) => classifyGgufFit(sizeBytes, input),+ variant,+ );}diff --git a/studio/frontend/src/lib/model-memory.ts b/studio/frontend/src/lib/model-memory.tsindex 51fdd3ad90e..dbfa6509035 100644--- a/studio/frontend/src/lib/model-memory.ts+++ b/studio/frontend/src/lib/model-memory.ts@@ -80,6 +80,25 @@ const EMPTY: ModelMemorySegments = {pressure: "normal",};+/** VRAM weights segment: only a host-resident lookup table comes off (a streamed one is already out). */+export function memoryBarWeightsBytes(+ estimate:+ | {+ weightsBytes?: number | null;+ lookupTableBytes?: number | null;+ lookupTableOnDisk?: boolean | null;+ }+ | undefined,+ source: { sizeBytes?: number | null; lookupTableBytes?: number | null } | undefined,+): number | null | undefined {+ if (estimate?.weightsBytes != null) {+ const resident = estimate.lookupTableOnDisk ? 0 : (estimate.lookupTableBytes ?? 0);+ return Math.max(0, estimate.weightsBytes - resident);+ }+ const size = source?.sizeBytes;+ return size == null ? size : Math.max(0, size - (source?.lookupTableBytes ?? 0));+}+function toGb(bytes?: number | null): number {return bytes && bytes > 0 ? bytes / BYTES_PER_GB : 0;}diff --git a/studio/frontend/tests/gguf-fit-lookup-tables.test.ts b/studio/frontend/tests/gguf-fit-lookup-tables.test.tsnew file mode 100644index 00000000000..32adcd319f0--- /dev/null+++ b/studio/frontend/tests/gguf-fit-lookup-tables.test.ts@@ -0,0 +1,91 @@+// SPDX-License-Identifier: AGPL-3.0-only+// Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0++// Streamed lookup tables must not be charged as resident. Sizes are real GGUF header figures.++import assert from "node:assert/strict";+import test from "node:test";++import {+ ggufFitRuns,+ pickDefaultQuant,+ recommendedQuantForDevice,+} from "../src/features/model-picker/components/model-selector/model-catalog.ts";+import {+ classifyGgufFit,+ classifyGgufFitWithLookupTables,+ classifyGgufVariantFit,+ ggufVariantLookupTableBytes,+} from "../src/lib/gguf-fit.ts";++const GIB = 1024 ** 3;+// 24 GB card + 64 GB RAM: Flash-Next without the table (54.6 GiB) just clears 23.28 + 32.+const HOST_24_64 = { gpuGb: 24, systemRamGb: 64 };++const flashNext = {+ quant: "UD-Q2_K_XL",+ size_bytes: 73.45 * GIB,+ lookup_table_bytes: 26.82 * GIB,+};+const dsv41 = {+ quant: "UD-IQ1_S",+ size_bytes: 216.98 * GIB,+ lookup_table_bytes: 103.0 * GIB,+};+const gemma3n = { quant: "Q4_K_M", size_bytes: 4.23 * GIB };++test("Flash-Next on 24 GB + 64 GB runs with its table on disk instead of reading OOM", () => {+ assert.equal(classifyGgufFit(flashNext.size_bytes, HOST_24_64), "oom");+ assert.equal(classifyGgufVariantFit(flashNext, HOST_24_64), "streamed");+ assert.equal(ggufFitRuns("streamed"), true);+});++test("DeepSeek-V4.1 UD-IQ1_S on 24 GB + 64 GB stays OOM without the Engram table", () => {+ assert.equal(classifyGgufVariantFit(dsv41, HOST_24_64), "oom");+});++test("a variant with no table keeps today's verdict", () => {+ const host = { gpuGb: 8, systemRamGb: 16 };+ assert.equal(+ classifyGgufVariantFit(gemma3n, host),+ classifyGgufFit(gemma3n.size_bytes, host),+ );+ assert.equal(classifyGgufVariantFit(gemma3n, host), "fits");+ assert.equal(ggufVariantLookupTableBytes(gemma3n), 0);+});++test("a table that would not change the verdict leaves it alone", () => {+ const big = { gpuGb: 192, systemRamGb: 256 };+ assert.equal(classifyGgufVariantFit(flashNext, big), "fits");+});++test("a streamed quant runs but ranks below one that fits with everything resident", () => {+ const fitOf = (bytes: number) => classifyGgufFit(bytes, HOST_24_64);+ const comfortable = { quant: "Q2", size_bytes: 15 * GIB, lookup_table_bytes: 0 };+ assert.equal(recommendedQuantForDevice([flashNext], fitOf), flashNext);+ assert.equal(recommendedQuantForDevice([flashNext, comfortable], fitOf), comfortable);+ const smaller = { quant: "UD-IQ1_S", size_bytes: 30 * GIB, lookup_table_bytes: 0 };+ assert.notEqual(classifyGgufVariantFit(smaller, HOST_24_64), "streamed");+});++test("a resident fit the table only improves keeps its own verdict", () => {+ const fitOf = (bytes: number) =>+ bytes > 60 * GIB ? "partial" : bytes > 40 * GIB ? "marginal" : "fits";+ const variant = { quant: "Q4", size_bytes: 70 * GIB, lookup_table_bytes: 20 * GIB };+ assert.equal(classifyGgufFitWithLookupTables(fitOf, variant), "partial");+});++test("a malformed table size is ignored", () => {+ const bytes = (lookup_table_bytes: number) =>+ ggufVariantLookupTableBytes({ size_bytes: GIB, lookup_table_bytes });+ assert.equal(bytes(Number.NaN), 0);+ assert.equal(bytes(5 * GIB), GIB);+});++test("a bare repo click picks the streamed quant rather than the smallest fallback", () => {+ const variants = [+ { ...flashNext, filename: "Q2/a-00001-of-00003.gguf", downloaded: true },+ { quant: "Q8_0", filename: "Q8/a.gguf", size_bytes: 160 * GIB, lookup_table_bytes: 26.82 * GIB },+ ];+ assert.equal(pickDefaultQuant(variants, "Q8_0", HOST_24_64)?.quant, "UD-Q2_K_XL");+});diff --git a/studio/frontend/tests/hub-local-gguf-selection.test.ts b/studio/frontend/tests/hub-local-gguf-selection.test.tsindex 56cbc536521..a7f8127becc 100644--- a/studio/frontend/tests/hub-local-gguf-selection.test.ts+++ b/studio/frontend/tests/hub-local-gguf-selection.test.ts@@ -143,3 +143,37 @@ test("a blank-quant variant remains eligible for the ranked fallback", () => {const sorted = sortLocalGgufVariants([oversized, blank], { gpuGb: 24 });assert.equal(resolveLocalGgufVariant(sorted, {}), blank);});++test("the memory bar reads the cached revision while an update is published", async () => {+ const { localGgufMemoryBarVariant } = await import(+ "../src/features/hub/lib/gguf-variant-sort.ts"+ );+ const { ggufVariantLookupTableBytes } = await import(+ "../src/lib/gguf-fit.ts"+ );+ const GiB = 1024 ** 3;+ const cached = {+ quant: "Q4_K_M",+ size_bytes: 70 * GiB,+ lookup_table_bytes: 20 * GiB,+ };+ // The merged row keeps the cached size but carries the published table.+ const merged = {+ ...cached,+ lookup_table_bytes: 30 * GiB,+ download_size_bytes: 80 * GiB,+ update_available: true,+ };+ const barVariant = localGgufMemoryBarVariant(merged, [+ { quant: "Q8_0", size_bytes: 1, lookup_table_bytes: 0 },+ cached,+ ]);+ assert.equal(barVariant?.size_bytes, 70 * GiB);+ assert.equal(ggufVariantLookupTableBytes(barVariant!), 20 * GiB);++ // No update, or no cached row, keeps the selected variant.+ const current = { ...cached, update_available: false };+ assert.equal(localGgufMemoryBarVariant(current, [cached]), current);+ assert.equal(localGgufMemoryBarVariant(merged, null), merged);+ assert.equal(localGgufMemoryBarVariant(null, [cached]), null);+});diff --git a/studio/frontend/tests/hub-row-selection.test.ts b/studio/frontend/tests/hub-row-selection.test.tsindex 1163c284513..78bb8795764 100644--- a/studio/frontend/tests/hub-row-selection.test.ts+++ b/studio/frontend/tests/hub-row-selection.test.ts@@ -48,8 +48,8 @@ test("the picker reads one fit per variant, companions included", () => {const at = PICKERS.indexOf("const getVariantFit = useCallback(");assert.notEqual(at, -1, "getVariantFit is missing");assert.match(- PICKERS.slice(at, at + 260),- /ggufVariantFitSizeBytes\(variant\)/,+ PICKERS.slice(at, at + 400),+ /ggufVariantFitSizeBytes\(variant\)|classifyGgufFitWithLookupTables\(getGgufFit, variant\)/,);assert.equal(PICKERS.match(/getVariantFit\(v\)/g)?.length, 2);});diff --git a/studio/frontend/tests/lookup-table-memory-bar.test.ts b/studio/frontend/tests/lookup-table-memory-bar.test.tsnew file mode 100644index 00000000000..92e62e08ccc--- /dev/null+++ b/studio/frontend/tests/lookup-table-memory-bar.test.ts@@ -0,0 +1,79 @@+// SPDX-License-Identifier: AGPL-3.0-only+// Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0++// Lookup tables never sit in VRAM. Sizes are real GGUF header figures.++import assert from "node:assert/strict";+import test from "node:test";++import type { KvCacheEstimate } from "../src/features/chat/api/chat-api.ts";+import { readSrcAsync, registerBundlerResolver } from "./helpers/kit.ts";++registerBundlerResolver();++const { memoryBarWeightsBytes } = await import("../src/lib/model-memory.ts");+const pickers = await readSrcAsync(+ "features/model-picker/components/model-selector/pickers.tsx",+);++const GIB = 1024 ** 3;+const FLASH = { sizeBytes: 73.45 * GIB, lookupTableBytes: 26.82 * GIB };++test("before the estimate, the listing's streamed table comes off the file size", () => {+ assert.equal(+ memoryBarWeightsBytes(undefined, FLASH),+ FLASH.sizeBytes - FLASH.lookupTableBytes,+ );+ assert.equal(memoryBarWeightsBytes(undefined, {}), undefined);+});++test("a streamed table the estimate already left out is not taken off twice", () => {+ const weightsBytes = FLASH.sizeBytes - FLASH.lookupTableBytes;+ const estimate = {+ weightsBytes,+ lookupTableBytes: FLASH.lookupTableBytes,+ lookupTableOnDisk: true,+ };+ assert.equal(memoryBarWeightsBytes(estimate, FLASH), weightsBytes);+});++test("a host-resident table inside the estimate's weights is not charted as VRAM", () => {+ assert.equal(+ memoryBarWeightsBytes(+ { weightsBytes: 4.23 * GIB, lookupTableBytes: 1.64 * GIB, lookupTableOnDisk: false },+ { sizeBytes: 4.23 * GIB, lookupTableBytes: 0 },+ ),+ 4.23 * GIB - 1.64 * GIB,+ );+});++test("a pinned quant row hands the bar the same listing figures as the sole-quant row", () => {+ const start = pickers.indexOf("const renderPinnedQuantRow");+ assert.ok(start > 0);+ const body = pickers.slice(start, pickers.indexOf("gpuGb={expanderGpuGb}", start));+ assert.match(body, /repoId: entry\.repoId,\s*quant: entry\.quant,[\s\S]*sizeBytes,/);+ assert.match(body, /lookupTableBytes: pinnedQuantValidation\.lookupTables\.get\(/);+ assert.match(pickers, /lookupTables: downloaded\.map\([\s\S]*?ggufVariantLookupTableBytes\(variant\)/);+});++test("the on-device card takes an update's table reading, a zero included", async () => {+ const card = await readSrcAsync("features/hub/catalog/local-on-device-card.tsx");+ assert.match(+ card,+ /lookup_table_bytes: remoteVariant\.update_available\s*\?\s*\(remoteVariant\.lookup_table_bytes \?\? 0\)\s*:\s*variant\.lookup_table_bytes \|\| remoteVariant\.lookup_table_bytes/,+ );+});++test("the memory bar's estimate takes the lookup table off the whole-file weights", async () => {+ // GET /kv-cache-estimate's weights_bytes is the quant file alone, streamed or resident table included.+ const hook = await readSrcAsync("hooks/use-model-memory.ts");+ assert.match(+ hook,+ /weightsBytes:\s*r\.weights_bytes == null\s*\?\s*null\s*:\s*Math\.max\(0, r\.weights_bytes - \(r\.lookup_table_bytes \?\? 0\)\)/,+ );+});++test("the kv-cache-estimate type carries the route's null for a model without a table", () => {+ const noTable: Pick<KvCacheEstimate, "lookup_table_bytes"> = { lookup_table_bytes: null };+ assert.equal(noTable.lookup_table_bytes ?? 0, 0);+});diff --git a/studio/frontend/tests/picker-gguf-discovery.test.ts b/studio/frontend/tests/picker-gguf-discovery.test.tsindex 0e7eb21979b..f522beed83c 100644--- a/studio/frontend/tests/picker-gguf-discovery.test.ts+++ b/studio/frontend/tests/picker-gguf-discovery.test.ts@@ -314,3 +314,50 @@ test("an up-to-date cached quant keeps the Hub's companion-inclusive size", asynassert.equal(result.variants[0].download_size_bytes, 900);assert.equal(result.variants[0].cache_path, "/cache/old");});++const GIB = 1024 ** 3;+type Variant = GgufVariantsResponse["variants"][number];++async function mergedRow(cached: Partial<Variant>, hub: Partial<Variant>) {+ const merged = await loadPickerGgufVariants(+ async (localOnly) => ({+ ...local,+ variants: [{ ...local.variants[0], ...(localOnly ? cached : hub) }],+ }),+ options,+ () => {},+ );+ return merged.variants[0];+}++test("an update's table size travels with the update's file size", async () => {+ const row = await mergedRow(+ { size_bytes: 70 * GIB, lookup_table_bytes: 20 * GIB },+ { size_bytes: 80 * GIB, lookup_table_bytes: 30 * GIB, update_available: true },+ );+ assert.equal(row.size_bytes, 80 * GIB);+ assert.equal(row.lookup_table_bytes, 30 * GIB);+});++for (const [name, hub, table] of [+ [+ "a Hub row with no table reading keeps the local header's",+ { lookup_table_bytes: 0 },+ 20 * GIB,+ ],+ [+ "an up-to-date local header's table wins over the Hub's bounded reading",+ { lookup_table_bytes: 12 * GIB },+ 20 * GIB,+ ],+ [+ "an update that drops the table is taken at its word",+ { lookup_table_bytes: 0, update_available: true },+ 0,+ ],+] as const) {+ test(name, async () => {+ const row = await mergedRow({ lookup_table_bytes: 20 * GIB }, hub);+ assert.equal(row.lookup_table_bytes, table);+ });+}