Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

Studio: price per-layer embedding and Engram lookup tables as host or disk memory, not VRAM (#14465)

Daniel Han2b1476fcb1dc
diff --git a/studio/backend/core/inference/llama_cpp.py b/studio/backend/core/inference/llama_cpp.py
index c654bc6a16b..d36ba0f61ce 100644
--- a/studio/backend/core/inference/llama_cpp.py
+++ b/studio/backend/core/inference/llama_cpp.py
@@ -12,6 +12,7 @@ import atexit
import bisect
import weakref
import contextlib
+import contextvars
import ctypes
import errno
import functools
@@ -82,6 +83,14 @@ from core.inference.context_window import (
turn_is_servable,
)
from core.inference.llama_tool_schema import llama_grammar_tools
+from core.inference.lookup_tables import (
+ HOST_FORCED_TABLE_ARCHS as _HOST_FORCED_TABLE_ARCHS,
+ HOST_FORCED_TABLE_MARKER as _HOST_FORCED_TABLE_MARKER,
+ LAZY_READ_AUTO_MIN_BYTES as _LOOKUP_LAZY_READ_AUTO_MIN_BYTES,
+ LAZY_READ_TENSORS as _LOOKUP_LAZY_READ_TENSORS,
+ is_per_layer_embedding as _is_per_layer_embedding,
+ lazy_read_bytes as _lookup_lazy_read_bytes,
+)
from core.inference.stream_errors import stream_error_from_chunk
from hub.utils.hf_errors import modelscope_missing
from hub.utils.hf_tokens import (
@@ -6659,6 +6668,17 @@ def _extra_args_tensor_split(
return parts
+def _pass_through_reprices_cache(extra_args: Optional[Iterable[str]], n_parallel) -> bool:
+ """Whether an argv -c 0 or a larger --parallel changes the cache Studio priced."""
+ try:
+ if parse_ctx_override(extra_args) == 0:
+ return True
+ except ValueError:
+ return True
+ override = _extra_args_n_parallel(extra_args, {})
+ return override is not None and override > int(n_parallel or 1)
+
+
def _extra_args_n_parallel(
extra_args: Optional[Iterable[str]], env: Optional[Mapping[str, str]] = None
) -> Optional[int]:
@@ -7066,13 +7086,9 @@ _MOE_CACHE_ERROR_RE = re.compile(
re.IGNORECASE,
)
-# TENSOR_READ_LAZY tensors (src/models/gemma4.cpp, qwen4exp.cpp): under --lazy-mode auto one
-# over 4 GiB stays mmap'd under every load mode (lazy_read::add), needing no pinned RAM or VRAM.
-_LAZY_READ_TENSORS: dict[str, tuple[str, ...]] = {
- "gemma4": ("per_layer_token_embd.weight",),
- "qwen4exp": ("per_layer_token_embd.weight",),
-}
-_LAZY_READ_AUTO_MIN_BYTES = 4 * 1024**3
+_LAZY_READ_TENSORS = _LOOKUP_LAZY_READ_TENSORS
+_LAZY_READ_AUTO_MIN_BYTES = _LOOKUP_LAZY_READ_AUTO_MIN_BYTES
+_LAZY_RESIDENT_MARGIN_FRAC = 0.10
# Embedded NextN/MTP heads llama.cpp skips at load unless --spec-type draft-mtp, measured:
# Nemotron 3.5 Super loads 2987 MiB less with MTP off (b11642 + ggml-org#30520).
_NEXTN_SKIPPED_WITHOUT_MTP_ARCHS = frozenset({"nemotron_h_moe"})
@@ -7122,6 +7138,29 @@ def _argv_loads_pinned(argv: Iterable[str], env: Optional[Mapping[str, str]] = N
return bool(reserves_ram and not mlock and not direct_io)
+_GPU_MEMORY_SKIP_TORCH: "contextvars.ContextVar[bool]" = contextvars.ContextVar(
+ "_GPU_MEMORY_SKIP_TORCH", default = False
+)
+
+
+def _gpu_device_node_present() -> bool:
+ """Whether this host exposes a GPU device node (ROCm, NVIDIA, WSL)."""
+ if sys.platform == "darwin":
+ return False
+ if os.name == "nt":
+ return True
+ return any(os.path.exists(p) for p in ("/dev/kfd", "/dev/nvidiactl", "/dev/dxg"))
+
+
+def _without_torch_gpu_probe():
+ token = _GPU_MEMORY_SKIP_TORCH.set(True)
+ try:
+ yield
+ finally:
+ _GPU_MEMORY_SKIP_TORCH.reset(token)
+
+
def _effective_lazy_mode(
extra_args: Optional[Iterable[str]],
env: Optional[Mapping[str, str]] = None,
@@ -7151,16 +7190,111 @@ def _lazy_read_bytes(
architecture: Optional[str], tensor_bytes: Mapping[str, int], mode: str
) -> int:
"""Bytes of ``tensor_bytes`` llama.cpp reads lazily under ``mode``."""
- if mode not in ("auto", "on"):
- return 0
- names = _LAZY_READ_TENSORS.get(architecture or "", ())
- return sum(
- int(size)
- for name, size in tensor_bytes.items()
- if name in names and (mode == "on" or int(size) > _LAZY_READ_AUTO_MIN_BYTES)
+ return _lookup_lazy_read_bytes(architecture, tensor_bytes, mode)
+
+
+_HOST_INPUT_HEAD_NAMES = frozenset({"token_embd.weight", "output.weight"})
+
+
+def _is_host_input_or_head_tensor(name: str) -> bool:
+ """token_embd, output.weight and any per-layer lookup table: the host-pinned input tensors."""
+ return name in _HOST_INPUT_HEAD_NAMES or _is_per_layer_embedding(name)
+
+
+def _override_places_tensor_on_device(
+ name: str,
+ extra_args: Optional[Iterable[str]] = None,
+ env: Optional[Mapping[str, str]] = None,
+) -> bool:
+ """Whether an ``--override-tensor`` (argv or env) may put ``name`` on a device."""
+ source_env = os.environ if env is None else env
+ values: list[str] = []
+ env_value = str(source_env.get("LLAMA_ARG_OVERRIDE_TENSOR") or "").strip()
+ if env_value:
+ values.append(env_value)
+ args = [str(a) for a in extra_args or ()]
+ for i, raw in enumerate(args):
+ if _flag_name(raw) in {"-ot", "--override-tensor"}:
+ _, eq, inline = raw.partition("=")
+ value = inline if eq else (args[i + 1] if i + 1 < len(args) else "")
+ if value:
+ values.append(value)
+ for value in values:
+ for part in value.split(","):
+ pattern, eq, target = part.rpartition("=")
+ if not eq or not pattern:
+ continue
+ on_host = _override_targets_host(f"{pattern}={target}")
+ if "[:" in pattern:
+ # A POSIX class is std::regex syntax Python reads literally: unknown, so a host
+ if not on_host:
+ return True
+ continue
+ try:
+ if not re.search(pattern, name):
+ continue
+ except re.error:
+ return True
+ return not on_host
+ return False
+
+
[email protected]_cache(maxsize = 8)
+def _host_forced_in_files(key: tuple) -> frozenset[str]:
+ """``_host_forced_table_archs`` for the files ``key`` names (path, size, mtime)."""
+ import mmap
+
+ for path, _size, _mtime in key:
+ try:
+ with open(path, "rb") as f, mmap.mmap(f.fileno(), 0, access = mmap.ACCESS_READ) as m:
+ if m.find(_HOST_FORCED_TABLE_MARKER) >= 0:
+ return _HOST_FORCED_TABLE_ARCHS
+ except (OSError, ValueError):
+ continue
+ return frozenset()
+
+
+def _lazy_read_names(
+ architecture: Optional[str], tensor_bytes: Mapping[str, int], mode: str
+) -> frozenset[str]:
+ """Names of ``tensor_bytes`` llama.cpp reads lazily under ``mode``. It picks the lazy
+ (CPU) buffer before applying ``-ot``, so these stay on the host whatever an override says."""
+ return frozenset(
+ name
+ for name, size in (tensor_bytes or {}).items()
+ if _lazy_read_bytes(architecture, {name: size}, mode)
)
+def _host_pinned_input_bytes(
+ scan: "Optional[tuple[Optional[str], Mapping[str, int], int]]",
+ extra_args: Optional[Iterable[str]] = None,
+ env: Optional[Mapping[str, str]] = None,
+ *,
+ lazy_mode: str = "off",
+ host_forced: Iterable[str] = (),
+) -> int:
+ """Bytes llama.cpp keeps on the host however many layers it offloads."""
+ if not scan:
+ return 0
+ architecture, named, _experts = scan
+ streamed = _lazy_read_names(architecture, named, lazy_mode)
+ total = 0
+ for name, size in (named or {}).items():
+ if _is_per_layer_embedding(name):
+ if (
+ name in streamed
+ or architecture in host_forced
+ or not _override_places_tensor_on_device(name, extra_args, env)
+ ):
+ total += int(size)
+ token_embd = int((named or {}).get("token_embd.weight") or 0)
+ if token_embd and int((named or {}).get("output.weight") or 0):
+ if not _override_places_tensor_on_device("token_embd.weight", extra_args, env):
+ total += token_embd
+ return total
+
+
def _build_ngram_mod_flags(
caps: Optional[dict],
n_match: int = 24,
@@ -7630,6 +7764,10 @@ _GGML_GPU_BACKEND_RE = re.compile(
r"^(?:lib)?ggml-(?:cann|cuda|hip|metal|musa|opencl|sycl|virtgpu|vulkan)"
r"(?:\.dll|\.so(?:\.\d+)*|(?:\.\d+)*\.dylib)$"
)
+# libllama the same way: a `.bak` / `.disabled` copy is not the library the child loads.
+_LLAMA_SHARED_LIB_RE = re.compile(
+ r"^(?:lib)?llama(?:\.dll|\.so(?:\.\d+)*|(?:\.\d+)*\.dylib)$", re.IGNORECASE
+)
def _cpu_runtime_owner_alive(staged_dir: Path) -> bool:
@@ -8119,6 +8257,8 @@ class LlamaCppBackend:
self._memory_mlock_applicable: bool = True
# --load-mode none tokens FIT emitted, removed by paths replacing placement; empty if user chose one.
self._fit_load_mode_flags: list[str] = []
+ self._lazy_mode_flags: list[str] = []
+ self._apple_lazy_flags: list[str] = []
# Mixed-pin --tensor-split tokens; the --fit on retry strips them (fit.cpp aborts on a set split).
self._mixed_split_flags: list[str] = []
# The pair a launch in flight is committed to, None when none is. ONE attribute:
@@ -10521,7 +10661,9 @@ class LlamaCppBackend:
named: dict[str, int] = {}
experts = 0
for p in paths:
- arch, part, part_experts = self._gguf_scan_tensor_bytes(p)
+ arch, part, part_experts = self._gguf_scan_tensor_bytes(
+ p, also = _is_host_input_or_head_tensor
+ )
# Shard 1 carries the metadata.
architecture = architecture or arch
for name, size in part.items():
@@ -10599,11 +10741,78 @@ class LlamaCppBackend:
shared_gpu_ids: Optional[Iterable[int]],
is_vulkan_backend: bool,
probe_integrated_cuda: bool = True,
+ extra_args: Optional[Iterable[str]] = None,
+ env: Optional[Mapping[str, str]] = None,
+ device_list: Optional[bool] = None,
+ tensor_parallel: Optional[bool] = None,
+ integrated_gpu_ids: Optional[Iterable[int]] = None,
+ child_gpu_order: Optional[Sequence[int]] = None,
) -> bool:
- """Whether llama.cpp resolves ``--lazy-mode auto`` to ``off`` for this child: a model
- device without mmap support (ggml-org/llama.cpp#28160), i.e. every selected device
- integrated, since the default list holds iGPUs only when no discrete GPU is seen.
- ``probe_integrated_cuda`` False uses only a cached SoC answer (no new context)."""
+ """Whether llama.cpp resolves ``--lazy-mode auto`` to ``off`` for this child.
+
+ ``child_gpu_order``: the ids the child's visibility mask will hold, in order
+ (``_planned_child_gpu_order``), for a preview whose ``env`` does not carry that
+ mask yet; a compact ``CUDA<k>`` / ``ROCm<k>`` names entry k of it."""
+ tensor = (
+ _effective_tensor_parallel(extra_args, False, env)
+ if tensor_parallel is None
+ else bool(tensor_parallel)
+ )
+ selected = None
+ override = self._effective_device_ids(extra_args, env)
+ one_device = not tensor and _split_mode_confines_to_one_device(extra_args, env)
+ if one_device:
+ main = self._effective_main_gpu(extra_args, env)
+ main = 0 if main is None else main
+ if main < 0:
+ return False
+ if override is not None and one_device:
+ override = [d for d in override if d.lower() not in _CPU_DEVICE_VALUES]
+ override = override[main : main + 1]
+ if override is not None:
+ # An RPC device reports mmap support (ggml-rpc.cpp device_get_props), so it never turns
+ # auto off.
+ gpu_names = [
+ d
+ for d in override
+ if d.lower() not in _CPU_DEVICE_VALUES and not d.strip().lower().startswith("rpc")
+ ]
+ ordinals = [self._device_ordinal(d) for d in gpu_names]
+ if all(o is not None for o in ordinals):
+ if child_gpu_order is not None and not is_vulkan_backend:
+ _order = [int(i) for i in child_gpu_order]
+ selected = [_order[o] for o in ordinals if 0 <= o < len(_order)] or _order
+ else:
+ selected = list(self._physical_ordinals(ordinals, env, is_vulkan_backend) or ())
+ elif tensor or (device_list if device_list is not None else is_vulkan_backend):
+ try:
+ selected = list(gpu_indices or ()) or None
+ except TypeError:
+ selected = None
+ if selected is None and tensor:
+ selected = [row[0] for row in (detected_gpus or ())] or None
+ if selected is not None:
+ if selected and override is None and one_device:
+ main = self._effective_main_gpu(extra_args, env)
+ main = 0 if main is None else main
+ selected = selected[main : main + 1] if main >= 0 else []
+ selected = set(selected)
+ if not selected:
+ return False
+ if selected:
+ if is_vulkan_backend:
+ return bool(selected & (set(shared_gpu_ids or ()) | set(integrated_gpu_ids or ())))
+ try:
+ if selected & LlamaCppBackend._rocm_unified_memory_gpu_ids():
+ return True
+ except Exception:
+ pass
+ if not probe_integrated_cuda and not self._integrated_cuda_probe_is_free():
+ return False
+ try:
+ return bool(selected & LlamaCppBackend._integrated_cuda_gpu_ids())
+ except Exception:
+ return False
rows = [(row[0], row[1]) for row in (detected_gpus or ())]
if self._offload_target_shares_system_memory(
is_vulkan_backend = is_vulkan_backend,
@@ -10620,6 +10829,303 @@ class LlamaCppBackend:
list(gpu_indices) if gpu_indices else None
)
+ def _child_runs_on_cpu(
+ self,
+ binary: Optional[str],
+ extra_args: Optional[Iterable[str]],
+ env: Optional[Mapping[str, str]],
+ *,
+ cpu_forced: bool = False,
+ tensor_parallel: Optional[bool] = None,
+ ) -> bool:
+ """Whether the child gets no model device (CPU build, CPU --device or forced CPU launch)."""
+ if (
+ cpu_forced
+ or self._binary_ships_no_gpu_backend(binary, env)
+ or _device_selection_is_cpu(extra_args, env)
+ ):
+ return True
+ tensor = (
+ _effective_tensor_parallel(extra_args, False, env)
+ if tensor_parallel is None
+ else bool(tensor_parallel)
+ )
+ # Split mode none with a negative main GPU clears the model's devices (src/llama.cpp).
+ if not tensor and _split_mode_confines_to_one_device(extra_args, env):
+ main = self._effective_main_gpu(extra_args, env)
+ return main is not None and main < 0
+ return False
+
+ @staticmethod
+ def _metal_placement_unset(
+ extra_args: Optional[Iterable[str]], env: Optional[Mapping[str, str]], *, layers_fixed: bool
+ ) -> bool:
+ """Whether nothing but Studio places this Metal launch, so a measured discount may pin full
+ offload."""
+ source_env = os.environ if env is None else env
+ return not (
+ layers_fixed
+ or _extra_args_set_any_flag(extra_args, _GPU_LAYER_FLAGS)
+ or _env_fixes_gpu_layers(source_env)
+ or _device_selection_is_cpu(extra_args, source_env)
+ or _args_place_tensors_on_cpu(extra_args)
+ or _env_places_tensors_on_cpu(source_env)
+ or _extra_args_set_any_flag(extra_args, _FIT_CONTROL_FLAGS)
+ or any(source_env.get(name) for name in _FIT_CONTROL_ENV_VARS)
+ or fit_is_enabled_in(extra_args)
+ # A pass-through adapter sits on the base tensor's buffer (llama-adapter.cpp:335) and
+ # nothing here charges it: keep the discount unspent.
+ or _sidecar_adapter_paths(extra_args)
+ )
+
+ @staticmethod
+ def _host_forced_table_archs(binary: Optional[str] = None) -> frozenset[str]:
+ """Archs whose table the build keeps on the host whatever -ot says; none if unreadable."""
+ binary = binary or LlamaCppBackend._find_llama_server_binary()
+ if not binary:
+ return frozenset()
+ try:
+ resolved = _resolve_llama_binary(binary)
+ libs = sorted(
+ p
+ for p in resolved.parent.iterdir()
+ if _LLAMA_SHARED_LIB_RE.match(p.name) and p.is_file()
+ )
+ key = tuple(
+ (str(p), st.st_size, st.st_mtime_ns)
+ for p in (libs or [resolved])
+ for st in (p.stat(),)
+ )
+ except OSError:
+ return frozenset()
+ return _host_forced_in_files(key)
+
+ def lookup_tables_stream_by_default(
+ self,
+ extra_args: Optional[Iterable[str]] = None,
+ env: Optional[Mapping[str, str]] = None,
+ *,
+ gpu_ids: Optional[Iterable[int]] = None,
+ tensor_parallel: Optional[bool] = None,
+ ) -> bool:
+ """Whether a launch on this host leaves lazily read lookup tables on disk."""
+ _env = dict(os.environ if env is None else env)
+ try:
+ binary = self._find_llama_server_binary()
+ caps = self.probe_server_capabilities(binary)
+ if not caps.get("supports_lazy_mode"):
+ return False
+ mode = _effective_lazy_mode(
+ extra_args, _env, flag = str(caps.get("lazy_mode_flag") or "--lazy-mode")
+ )
+ if mode != "auto":
+ return mode == "on"
+ tensor = (
+ _effective_tensor_parallel(extra_args, False, _env)
+ if tensor_parallel is None
+ else bool(tensor_parallel)
+ )
+ self._reconcile_split_mode_env(_env, tensor_parallel = tensor)
+ cpu_child = self._child_runs_on_cpu(binary, extra_args, _env, tensor_parallel = tensor)
+ vulkan = not cpu_child and self._is_vulkan_backend(binary)
+ rows = []
+ if not cpu_child:
+ with _without_torch_gpu_probe():
+ rows = list(self._get_gpu_memory(binary, for_llama_server = True) or ())
+ # An empty Vulkan inventory is a failed probe: it may be an iGPU, where auto is off.
+ if not rows and (vulkan or _gpu_device_node_present()):
+ return False
+ shared = {idx for idx, _free, total in rows if total <= 0} if vulkan else set()
+ picked = [int(i) for i in gpu_ids] if gpu_ids else None
+ return not self._lazy_auto_resolves_off(
+ gpu_indices = picked,
+ detected_gpus = [(idx, free) for idx, free, _total in rows],
+ shared_gpu_ids = shared,
+ is_vulkan_backend = vulkan,
+ probe_integrated_cuda = False,
+ extra_args = extra_args,
+ env = _env,
+ device_list = None if picked else False,
+ tensor_parallel = tensor_parallel,
+ integrated_gpu_ids = shared,
+ )
+ except Exception:
+ return False
+
+ def _managed_lazy_mode_flags(
+ self,
+ model_path: Optional[str],
+ *,
+ caps: Optional[Mapping[str, object]],
+ extra_args: Optional[Iterable[str]] = None,
+ env: Optional[Mapping[str, str]] = None,
+ auto_resolves_off: bool = False,
+ resident_fits: Callable[[int], Optional[bool]],
+ binary: Optional[str] = None,
+ ) -> list[str]:
+ """``[<lazy flag>, "off"]`` to keep a lazily read lookup table resident, else ``[]``."""
+ caps = caps or {}
+ if not model_path or not caps.get("supports_lazy_mode"):
+ return []
+ flag = str(caps.get("lazy_mode_flag") or "--lazy-mode")
+ source_env = os.environ if env is None else env
+ flags, env_name = _LAZY_MODE_SPELLINGS.get(flag, (frozenset({flag}), ""))
+ if (env_name and str(source_env.get(env_name) or "").strip()) or _extra_args_set_any_flag(
+ extra_args, flags
+ ):
+ return []
+ if auto_resolves_off:
+ return []
+ table = self._lazy_read_host_bytes(
+ model_path, caps = caps, extra_args = extra_args, env = source_env
+ )
+ if table <= 0:
+ return []
+ scan = self._gguf_tensor_scan(model_path)
+ if scan is None or (
+ scan[0] not in self._host_forced_table_archs(binary)
+ and any(
+ _override_places_tensor_on_device(name, extra_args, source_env)
+ for name in scan[1]
+ if _is_per_layer_embedding(name)
+ )
+ ):
+ return []
+ try:
+ fits = resident_fits(table)
+ except Exception as e: # noqa: BLE001 -- unpriced: keep llama.cpp's default
+ logger.debug("Lookup tables: resident fit unpriced (%s)", e)
+ fits = None
+ key = (model_path, table, bool(fits))
+ if getattr(self, "_lazy_mode_decision_logged", None) != key:
+ self._lazy_mode_decision_logged = key
+ if fits:
+ logger.info(
+ "Lookup tables: keeping the %.1f GB lookup table in RAM (%s off); "
+ "free RAM covers it beside the weights and the KV cache.",
+ table / (1024**3),
+ flag,
+ )
+ else:
+ logger.info(
+ "Lookup tables: streaming the %.1f GB lookup table from disk "
+ "(llama.cpp's lazy read); RAM cannot hold it beside the weights and "
+ "the KV cache. Prefill and decode can be slower and vary.",
+ table / (1024**3),
+ )
+ return [flag, "off"] if fits else []
+
+ def _apple_lookup_table_discount(
+ self,
+ probe_discount: int,
+ model_path: Optional[str],
+ *,
+ caps: Optional[Mapping[str, object]],
+ extra_args: Optional[Iterable[str]],
+ model_size_fit: int,
+ need_at_ctx: Optional[int],
+ budget_mib: int,
+ cache_ram: Optional[int],
+ env: Optional[Mapping[str, str]] = None,
+ binary: Optional[str] = None,
+ cpu_forced: bool = False,
+ ) -> int:
+ """The Metal embedding discount with the lazily read lookup table settled."""
+ source_env = os.environ if env is None else env
+ self._apple_lazy_flags = []
+ table = self._lazy_read_host_bytes(
+ model_path, caps = caps, extra_args = extra_args, env = source_env
+ )
+ if table <= 0:
+ auto_table = self._lazy_read_host_bytes(model_path, caps = caps, extra_args = (), env = {})
+ if (
+ auto_table > 0
+ and _effective_lazy_mode(
+ extra_args,
+ source_env,
+ flag = str((caps or {}).get("lazy_mode_flag") or "--lazy-mode"),
+ )
+ == "off"
+ ):
+ return max(0, int(probe_discount) - auto_table)
+ return probe_discount
+ if _sidecar_adapter_paths(extra_args):
+ return probe_discount
+ scan = self._gguf_tensor_scan(model_path)
+ streamed = (
+ _lazy_read_names(
+ scan[0],
+ scan[1],
+ _effective_lazy_mode(
+ extra_args,
+ source_env,
+ flag = str((caps or {}).get("lazy_mode_flag") or "--lazy-mode"),
+ ),
+ )
+ if scan is not None
+ else frozenset()
+ )
+ if (
+ scan is not None
+ and scan[0] not in self._host_forced_table_archs(binary)
+ and any(
+ _override_places_tensor_on_device(name, extra_args, source_env)
+ for name in scan[1]
+ if _is_per_layer_embedding(name) and name not in streamed
+ )
+ ):
+ return probe_discount
+
+ def _resident_fits(table_bytes: int) -> Optional[bool]:
+ if self._child_runs_on_cpu(binary, extra_args, source_env, cpu_forced = cpu_forced):
+ return None
+ if any(
+ str(source_env.get(name) or "").strip()
+ for name in ("LLAMA_ARG_MMPROJ", "LLAMA_ARG_MMPROJ_URL")
+ ):
+ return None
+ avail = self._available_system_memory_mib()
+ prompt = _prompt_cache_host_bytes(
+ cache_ram,
+ extra_args,
+ source_env,
+ supports_cache_ram = bool((caps or {}).get("supports_cache_ram")),
+ )
+ if (
+ avail is None
+ or prompt is None
+ or need_at_ctx is None
+ or budget_mib <= 0
+ or not self._can_estimate_kv()
+ ):
+ return None
+ need = (
+ int(model_size_fit)
+ + int(need_at_ctx)
+ + int(prompt)
+ + int(table_bytes * _LAZY_RESIDENT_MARGIN_FRAC)
+ )
+ ceiling_mib = min(int(budget_mib), int(avail) - _HOST_RAM_HEADROOM_MIB)
+ return need <= max(0, ceiling_mib) * 1024 * 1024
+
+ self._apple_lazy_flags = self._managed_lazy_mode_flags(
+ model_path,
+ caps = caps,
+ extra_args = extra_args,
+ env = source_env,
+ resident_fits = _resident_fits,
+ binary = binary,
+ )
+ if self._apple_lazy_flags:
+ return max(0, int(probe_discount) - table)
+ if table > probe_discount:
+ logger.info(
+ "Metal fit: %.2f GB of lookup tables are read lazily from disk, outside "
+ "the Metal working set; not charged to unified memory.",
+ table / (1024**3),
+ )
+ return max(int(probe_discount), table)
+
@staticmethod
def _installed_ggml_backends(binary: Optional[str] = None) -> frozenset[str]:
"""Backend libraries shipped beside llama-server."""
@@ -11121,6 +11627,59 @@ class LlamaCppBackend:
return None
return ordered
+ @staticmethod
+ def _picker_ordered_gpu_indices(gpu_ids, gpu_indices, tp_tensor_split):
+ """``(gpu_indices, tp_tensor_split)`` in the picker's order; a reorder the shares cannot
+ follow is vetoed."""
+ if not gpu_ids:
+ return gpu_indices, tp_tensor_split
+ picked = [int(i) for i in gpu_ids]
+ if gpu_indices is None:
+ return picked, tp_tensor_split
+ before = [int(i) for i in gpu_indices]
+ kept = set(before)
+ reordered = [i for i in picked if i in kept] + [i for i in before if i not in set(picked)]
+ if tp_tensor_split and len(tp_tensor_split) == len(before):
+ share_by_id = dict(zip(before, tp_tensor_split))
+ if all(i in share_by_id for i in reordered):
+ return reordered, [share_by_id[i] for i in reordered]
+ elif not tp_tensor_split:
+ return reordered, tp_tensor_split
+ return gpu_indices, tp_tensor_split
+
+ def _planned_child_gpu_order(
+ self,
+ *,
+ gpu_ids,
+ gpu_indices,
+ tp_tensor_split,
+ extra_args,
+ env,
+ is_vulkan_backend: bool,
+ cpu_forced: bool = False,
+ ) -> "tuple[Optional[list[int]], Optional[list[int]]]":
+ """``(device order, visibility mask order)`` the spawn will give the child; mask None when
+ inherited."""
+ order, split = self._picker_ordered_gpu_indices(gpu_ids, gpu_indices, tp_tensor_split)
+ if order is None or is_vulkan_backend or cpu_forced:
+ return (None if order is None else [int(i) for i in order]), None
+ order = [int(i) for i in order]
+ mask = list(order)
+ if not gpu_ids:
+ inherited = self._inherited_child_gpu_order(mask)
+ if (
+ inherited is not None
+ and not _extra_args_have_tensor_split(extra_args, env)
+ and (not split or len(split) == len(mask))
+ ):
+ mask = inherited
+ try:
+ if LlamaCppBackend._pinned_child_visibility(mask) is None:
+ return order, None
+ except Exception:
+ return order, None
+ return order, mask
+
@staticmethod
def _repoint_emitted_tensor_split(
cmd: list, old_order: list[int], new_order: list[int]
@@ -12915,20 +13474,7 @@ class LlamaCppBackend:
"""Embedding bytes proven absent from the Metal working set; fails closed, CPU weights stay
charged."""
source_env = os.environ if env is None else env
- if (
- layers_fixed
- or _extra_args_set_any_flag(extra_args, _GPU_LAYER_FLAGS)
- or _env_fixes_gpu_layers(source_env)
- or _device_selection_is_cpu(extra_args, source_env)
- or _args_place_tensors_on_cpu(extra_args)
- or _env_places_tensors_on_cpu(source_env)
- or _extra_args_set_any_flag(extra_args, _FIT_CONTROL_FLAGS)
- or any(source_env.get(name) for name in _FIT_CONTROL_ENV_VARS)
- # Extras are appended after the pin, while inherited values are overridden by it.
- or fit_is_enabled_in(extra_args)
- # A pass-through adapter's bytes are not in the probe or model_size, so do not spend the discount.
- or _sidecar_adapter_paths(extra_args)
- ):
+ if not self._metal_placement_unset(extra_args, source_env, layers_fixed = layers_fixed):
return 0
env_view = dict(source_env)
scrub_memory_env(env_view, settings)
@@ -12955,7 +13501,7 @@ class LlamaCppBackend:
layout = self._tensor_spill_layout(model_path, all_shards = True)
if layout is None or not layout.complete:
return 0
- embeddings = int(layout.token_embd_bytes)
+ embeddings = int(layout.token_embd_bytes) + int(layout.per_layer_embd_bytes)
if embeddings <= 0 or (mtp_may_engage and layout.has_excluded_blocks):
return 0
measured = self._metal_measured_model_mib(binary, model_path)
@@ -13935,7 +14481,11 @@ class LlamaCppBackend:
return rocm_gpus
# ── Torch fallback (covers AMD ROCm and missing nvidia-smi) ──
- gpus = LlamaCppBackend._get_gpu_memory_torch(binary, for_llama_server)
+ gpus = (
+ []
+ if _GPU_MEMORY_SKIP_TORCH.get()
+ else LlamaCppBackend._get_gpu_memory_torch(binary, for_llama_server)
+ )
if not gpus and mig_card_rows:
LlamaCppBackend._GPU_IDS_ARE_PCI_INDICES = True
return LlamaCppBackend._widen_integrated_cuda_rows(mig_card_rows)
@@ -14437,6 +14987,7 @@ class LlamaCppBackend:
avail_mib: Optional[int] = None,
shared_gpu_ids: Iterable[int] = (),
honor_opt_out: bool = True,
+ integrated_gpu_ids: Iterable[int] = (),
) -> Optional[str]:
"""Advisory shortfall warning, never blocks; UNSLOTH_ALLOW_HOST_OFFLOAD=1 silences only the message, never the pageable-load override."""
argv = [str(a) for a in cmd or ()]
@@ -14479,6 +15030,10 @@ class LlamaCppBackend:
shared_gpu_ids = shared,
is_vulkan_backend = bool(shared) or self._is_vulkan_backend(argv[0]),
probe_integrated_cuda = False,
+ extra_args = argv,
+ env = _env,
+ device_list = False,
+ integrated_gpu_ids = integrated_gpu_ids,
),
),
)
@@ -14875,6 +15430,24 @@ class LlamaCppBackend:
"GPU, so the whole model is charged to system RAM."
)
+ def _unified_pool_gpu_ids(
+ self, rows, *, gpu_indices, shared_gpu_ids, is_vulkan_backend: bool
+ ) -> set:
+ """Ids in ``rows`` whose free VRAM is the host RAM pool (Vulkan iGPUs, ROCm APUs, CUDA
+ SoCs)."""
+ rows = list(rows or ())
+ shared = set(shared_gpu_ids or ())
+ pinned = list(gpu_indices) if gpu_indices is not None else [row[0] for row in rows]
+ if rows and not is_vulkan_backend and self._amd_apu_wants_unified_memory(pinned):
+ # Per device: a discrete card beside an APU still has its own pool.
+ shared |= {row[0] for row in rows if self._amd_apu_wants_unified_memory([row[0]])}
+ # Integrated CUDA SoCs (Jetson, DGX Spark) too. Cache only: _get_gpu_memory filled it.
+ _integrated = LlamaCppBackend._INTEGRATED_CUDA_IDS.get(
+ LlamaCppBackend._integrated_cuda_mask_key(), ()
+ )
+ shared |= {row[0] for row in rows if row[0] in _integrated}
+ return shared
+
def _fits_without_paging(
self,
footprint_bytes: Optional[int],
@@ -15053,10 +15626,11 @@ class LlamaCppBackend:
is_vulkan_backend = is_vulkan_backend,
gpu_indices = gpu_indices,
):
- logger.info(
- "Load mode: pass-through arguments override the placement this fit "
- "priced; charging the whole load to host RAM."
- )
+ if announce:
+ logger.info(
+ "Load mode: pass-through arguments override the placement this fit "
+ "priced; charging the whole load to host RAM."
+ )
rows, gpu_indices = [], None
footprint = (
model_size
@@ -15082,16 +15656,12 @@ class LlamaCppBackend:
# comes out of, so adding both fits a model twice into memory that holds it
# once. shared_gpu_ids already names Vulkan iGPUs; the ROCm APUs it cannot see
# are added here. Both end up priced against host RAM alone.
- shared = set(shared_gpu_ids or ())
- pinned = list(gpu_indices) if gpu_indices is not None else [idx for idx, _free in rows]
- if rows and not is_vulkan_backend and self._amd_apu_wants_unified_memory(pinned):
- # Per device: a discrete card next to an APU still has its own pool.
- shared |= {idx for idx, _free in rows if self._amd_apu_wants_unified_memory([idx])}
- # Integrated CUDA SoCs (Jetson, DGX Spark) too. Cache only: _get_gpu_memory filled it.
- _integrated = LlamaCppBackend._INTEGRATED_CUDA_IDS.get(
- LlamaCppBackend._integrated_cuda_mask_key(), ()
+ shared = self._unified_pool_gpu_ids(
+ rows,
+ gpu_indices = gpu_indices,
+ shared_gpu_ids = shared_gpu_ids,
+ is_vulkan_backend = is_vulkan_backend,
)
- shared |= {idx for idx, _free in rows if idx in _integrated}
# --no-kv-offload puts the WHOLE cache in host RAM whatever the layer placement
# says (llama-kv-cache.cpp upgrades a layer's buffer type only inside
# `if (offload)`), so free VRAM may not pay for it. Resolved off the same
@@ -15667,6 +16237,24 @@ class LlamaCppBackend:
for name in ("LLAMA_ARG_SPLIT_MODE", "LLAMA_ARG_TENSOR_SPLIT"):
env.pop(name, None)
+ @classmethod
+ def _reconcile_split_mode_env(cls, env: dict[str, str], *, tensor_parallel: bool) -> None:
+ """Clear inherited LLAMA_ARG_* split env that contradicts Studio's split, so previews price
+ what the child runs."""
+ if not tensor_parallel:
+ # Layer split: clear a non-layer inherited split mode (and any paired
+ # tensor-split) so the child can't override the layer plan.
+ _inherited_sm = (env.get("LLAMA_ARG_SPLIT_MODE") or "").strip().lower()
+ if _inherited_sm and _inherited_sm != "layer":
+ env.pop("LLAMA_ARG_SPLIT_MODE", None)
+ env.pop("LLAMA_ARG_TENSOR_SPLIT", None)
+ else:
+ # Unsloth owns the tensor split: it emits --tensor-split when it picks an
+ # uneven one (CLI wins) and nothing when an even split is safe. Clear any
+ # inherited LLAMA_ARG_TENSOR_SPLIT so the even case can't be overridden by a
+ # stale env (the layer branch above clears it too).
+ env.pop("LLAMA_ARG_TENSOR_SPLIT", None)
+
@staticmethod
def _discrete_tier(
gpus: list,
@@ -18527,6 +19115,8 @@ class LlamaCppBackend:
self._memory_policy_extras_touched = False
self._memory_mlock_applicable = True
self._fit_load_mode_flags = []
+ self._lazy_mode_flags = []
+ self._apple_lazy_flags = []
self._mixed_split_flags = []
self._memory_pending_launch = None
self._n_ubatch = self._DEFAULT_N_UBATCH
@@ -21929,6 +22519,7 @@ class LlamaCppBackend:
"Load mode: dropping the fit's --load-mode none for the CPU "
"fallback; it runs entirely from host RAM."
)
+ replay = self._drop_managed_lazy_mode(replay, "CPU fallback")
# And the managed pair: this replay appends "--device none".
replay = self._drop_managed_dio(
replay, "the CPU fallback runs entirely from host RAM", clear_record = False
@@ -23254,6 +23845,7 @@ class LlamaCppBackend:
# Seed before the try (except falls through to the launch; the probe may throw first). Capture before manual empties gpus so defaults stay GPU-aware.
_detected_gpus: list[tuple[int, int]] = []
_shared_gpu_ids: set[int] = set()
+ _vulkan_igpu_ids: frozenset[int] = frozenset()
_placement_ctx_checkpoints = 0
# Set when the arch gate emptied a non-empty GPU pool, so the env
# block below masks the child onto the CPU. Bound before the try for
@@ -23309,6 +23901,8 @@ class LlamaCppBackend:
# for llama.cpp's own default. Bound before the try like the verdict
# flags above: the except path falls through to the launch, which reads it.
_fit_load_mode: Optional[str] = None
+ _managed_lazy_flags: Optional[list[str]] = None
+ _host_input_embd = 0
# Layer-fallback min GPUs; raised below on a tensor downgrade. Bound
# before the try so the --fit-on except path still has it (no UnboundLocal).
_layer_min_gpus = 1
@@ -23427,6 +24021,11 @@ class LlamaCppBackend:
"get GPU inference back.",
_present,
)
+ _vulkan_igpu_ids = (
+ frozenset(idx for idx, _free, total in _gpu_mem if total <= 0)
+ if is_vulkan_backend
+ else frozenset()
+ )
if is_vulkan_backend and not gpu_ids and gpu_memory_mode != "manual":
_gpu_mem = self._vulkan_auto_gpu_memory(_gpu_mem)
gpus = [(idx, free) for idx, free, _t in _gpu_mem]
@@ -24270,6 +24869,68 @@ class LlamaCppBackend:
floor_ctx,
)
+ def _host_input_bytes_now(weights: int) -> int:
+ if not (
+ gpus
+ and not _integrated_gpus
+ and not _shared_gpu_ids
+ and not _shared_pool_gpus
+ and not _metal_capable_host()
+ and not self._amd_apu_wants_unified_memory([i for i, _f in gpus])
+ and not set(
+ LlamaCppBackend._INTEGRATED_CUDA_IDS.get(
+ LlamaCppBackend._integrated_cuda_mask_key(), ()
+ )
+ ).intersection(i for i, _f in gpus)
+ ):
+ return 0
+ # A streamed table stays on the host whatever -ot says.
+ _inp_extras = (
+ self._strip_device_extra_args(extra_args)
+ if _gpu_ids_own_device_flags
+ else extra_args
+ )
+ _inp_env = dict(os.environ)
+ if gpu_memory_mode == "manual":
+ self._clear_manual_placement_env(_inp_env)
+ if _gpu_ids_own_device_flags:
+ self._clear_device_placement_env(_inp_env)
+ self._reconcile_split_mode_env(_inp_env, tensor_parallel = tensor_parallel)
+ try:
+ _inp_lazy = "off"
+ if server_caps.get("supports_lazy_mode"):
+ _inp_lazy = _effective_lazy_mode(
+ _inp_extras,
+ _inp_env,
+ flag = str(server_caps.get("lazy_mode_flag") or "--lazy-mode"),
+ )
+ if _inp_lazy == "auto" and self._lazy_auto_resolves_off(
+ gpu_indices = (
+ gpu_indices
+ if gpu_indices is not None or not gpu_ids
+ else [int(i) for i in gpu_ids]
+ ),
+ detected_gpus = _detected_gpus,
+ shared_gpu_ids = _shared_gpu_ids,
+ is_vulkan_backend = is_vulkan_backend,
+ extra_args = _inp_extras,
+ env = _inp_env,
+ tensor_parallel = tensor_parallel,
+ integrated_gpu_ids = _vulkan_igpu_ids,
+ ):
+ _inp_lazy = "off"
+ found = _host_pinned_input_bytes(
+ self._gguf_tensor_scan(model_path),
+ _inp_extras,
+ _inp_env,
+ lazy_mode = _inp_lazy,
+ host_forced = self._host_forced_table_archs(binary),
+ )
+ except Exception as e: # unread header: charge the whole file
+ logger.debug("Host-pinned input embeddings unpriced: %s", e)
+ return 0
+ return min(max(0, found), max(0, int(weights) - 1))
+
_mm_budgeted_gpus = [
(_idx, _free) for _idx, _free in gpus if total_by_idx.get(_idx, 0) > 0
]
@@ -24300,6 +24961,7 @@ class LlamaCppBackend:
)
_mm_need = (
gguf_size
+ - _host_input_bytes_now(gguf_size)
+ mmproj_size
+ _compute_buffer_pipeline
+ self._CUDA_CONTEXT_RESERVE_BYTES
@@ -24415,6 +25077,7 @@ class LlamaCppBackend:
_soft += self._MTP_DRAFT_COMPUTE_BYTES
return (
model_size
+ - _host_input_bytes_now(model_size)
+ (
_layer_compute_buffer(slots)
if slots
@@ -24802,8 +25465,19 @@ class LlamaCppBackend:
if (not gpus or len(_mm_budgeted_gpus) < len(gpus))
else 0
)
+ _host_input_embd = _host_input_bytes_now(model_size)
+ if _host_input_embd:
+ logger.info(
+ "GPU fit: %.2f GB of input embeddings and lookup tables stay "
+ "in host memory; not charged to VRAM.",
+ _host_input_embd / (1024**3),
+ )
model_size_fit = (
- model_size + _compute_buffer_pipeline + _soft_overhead + _shared_pool_mmproj
+ model_size
+ - _host_input_embd
+ + _compute_buffer_pipeline
+ + _soft_overhead
+ + _shared_pool_mmproj
)
def _subset_model_size(n_gpus: int) -> int:
@@ -24848,9 +25522,9 @@ class LlamaCppBackend:
_tp_flat_mtp,
_mtp_bytes(min(2048, effective_ctx) if effective_ctx > 0 else 2048),
)
- _tp_required_mib = (model_size + _tp_mtp_floor + _soft_overhead) / (
- 1024 * 1024
- )
+ _tp_required_mib = (
+ model_size - _host_input_embd + _tp_mtp_floor + _soft_overhead
+ ) / (1024 * 1024)
if _tp_weight_budget_mib <= _tp_required_mib:
logger.info(
"Tensor parallelism requested but the pooled VRAM "
@@ -24881,6 +25555,7 @@ class LlamaCppBackend:
)
model_size_fit = (
model_size
+ - _host_input_embd
+ _compute_buffer_pipeline
+ _soft_overhead
+ _shared_pool_mmproj
@@ -24904,7 +25579,7 @@ class LlamaCppBackend:
tp_tensor_split,
) = self._plan_tensor_parallel(
tp_gpus,
- model_size,
+ max(1, model_size - _host_input_embd),
target_ctx,
cache_type_kv = cache_type_kv,
scratch_cache_type_kv = _scratch_cache_type_kv,
@@ -25380,6 +26055,71 @@ class LlamaCppBackend:
layers_fixed = _caller_owns_budget or _paravirtual_cpu_forced,
mtp_may_engage = _mtp_will_engage,
)
+ _apple_probe_embd = _apple_host_embd
+ # Metal refuses past the wired limit; None is unknown, 0 used up.
+ _apple_wired = self._apple_metal_wired_headroom_bytes()
+ _apple_resident_budget_mib = (
+ _apple_fit_budget_mib
+ if _apple_wired is None
+ else min(
+ _apple_fit_budget_mib,
+ int(
+ _apple_wired
+ // (1024 * 1024)
+ * max(0.0, 1.0 - _flat_mtp_reserve)
+ ),
+ )
+ )
+ _apple_extras = (
+ self._strip_device_extra_args(extra_args)
+ if _gpu_ids_own_device_flags
+ else extra_args
+ )
+ _apple_env = dict(os.environ)
+ if gpu_memory_mode == "manual":
+ self._clear_manual_placement_env(_apple_env)
+ if _gpu_ids_own_device_flags:
+ self._clear_device_placement_env(_apple_env)
+ self._reconcile_split_mode_env(_apple_env, tensor_parallel = tensor_parallel)
+ _apple_host_embd = self._apple_lookup_table_discount(
+ _apple_host_embd,
+ model_path,
+ caps = server_caps,
+ extra_args = _apple_extras,
+ env = _apple_env,
+ model_size_fit = model_size_fit,
+ need_at_ctx = (
+ (
+ _kv_bytes(effective_ctx, _fit_ctx_checkpoints)
+ + _cc_bytes(effective_ctx)
+ + _mtp_bytes(effective_ctx)
+ + _apple_cpu_draft
+ )
+ if effective_ctx > 0
+ and not _pass_through_reprices_cache(extra_args, n_parallel)
+ and not _flat_mtp_engages
+ and (
+ _apple_cpu_draft := self._cpu_resident_draft_bytes(
+ effective_ctx,
+ drafter_path = _cpu_draft_path,
+ draft_cache_type_k = _mtp_draft_ck,
+ draft_cache_type_v = _mtp_draft_cv,
+ n_parallel = n_parallel,
+ swa_full = swa_full,
+ kv_unified = planned_kv_unified,
+ n_ubatch = _effective_ubatch,
+ flash_attn = planned_flash_attn,
+ )
+ )
+ is not None
+ else None
+ ),
+ budget_mib = _apple_resident_budget_mib,
+ cache_ram = cache_ram,
+ binary = binary,
+ cpu_forced = _paravirtual_cpu_forced or _arch_gate_forced_cpu,
+ )
+ _managed_lazy_flags = list(self._apple_lazy_flags)
_apple_model_size_fit = model_size_fit - _apple_host_embd
if _apple_host_embd:
logger.info(
@@ -25693,7 +26433,17 @@ class LlamaCppBackend:
effective_ctx = max_available_ctx = _vram_fit_ctx
# Unmeasured floors still rely on llama.cpp's fitter.
_metal_full_offload_pinned = bool(
- _apple_host_embd and _apple_measured_ceiling is not None
+ _apple_measured_ceiling is not None
+ and (
+ _apple_probe_embd
+ or (
+ _apple_host_embd
+ and not _paravirtual_cpu_forced
+ and self._metal_placement_unset(
+ extra_args, os.environ, layers_fixed = _caller_owns_budget
+ )
+ )
+ )
)
# Weights over the budget launch here too and the fitter places
# them, so it must see the same wired limit the refusals do.
@@ -25915,6 +26665,7 @@ class LlamaCppBackend:
_child_env = dict(os.environ)
if _gpu_ids_own_device_flags:
self._clear_device_placement_env(_child_env)
+ self._reconcile_split_mode_env(_child_env, tensor_parallel = tensor_parallel)
if _pass_through_overrides_placement(
_child_extras,
_child_env,
@@ -25988,21 +26739,35 @@ class LlamaCppBackend:
supports_fit_target = _supports_fit_target,
)
# A lazily read table stays mmap'd on the host, never on the GPU.
+ _child_order, _child_mask = self._planned_child_gpu_order(
+ gpu_ids = gpu_ids,
+ gpu_indices = gpu_indices,
+ tp_tensor_split = tp_tensor_split,
+ extra_args = _child_extras,
+ env = _child_env,
+ is_vulkan_backend = is_vulkan_backend,
+ cpu_forced = _paravirtual_cpu_forced or _arch_gate_forced_cpu,
+ )
_lazy_host = self._lazy_read_host_bytes(
model_path,
caps = server_caps,
extra_args = _child_extras,
env = _child_env,
auto_resolves_off = self._lazy_auto_resolves_off(
- gpu_indices = gpu_indices,
+ gpu_indices = _child_order,
detected_gpus = _detected_gpus,
shared_gpu_ids = _shared_gpu_ids,
is_vulkan_backend = is_vulkan_backend,
+ extra_args = _child_extras,
+ env = _child_env,
+ tensor_parallel = tensor_parallel,
+ integrated_gpu_ids = _vulkan_igpu_ids,
+ child_gpu_order = _child_mask,
),
)
_need = (
model_size_fit
- - _lazy_host
+ - max(0, _lazy_host - _host_input_embd)
- _model_size_fit_cb
+ _compute_buffer_pipeline
+ (
@@ -26533,7 +27298,9 @@ class LlamaCppBackend:
_on = list(gpu_indices or [idx for idx, _ in gpus])
if _on:
self._pending_plan_mib = _gpu_plan_mib(
- gguf_size + mmproj_size + kv_cache_bytes, _on, gpus
+ max(1, gguf_size - _host_input_embd) + mmproj_size + kv_cache_bytes,
+ _on,
+ gpus,
)
if (
not gpus
@@ -26578,6 +27345,7 @@ class LlamaCppBackend:
self._clear_manual_placement_env(_fit_env)
if _gpu_ids_own_device_flags:
self._clear_device_placement_env(_fit_env)
+ self._reconcile_split_mode_env(_fit_env, tensor_parallel = tensor_parallel)
# What the placement branch below will emit, so the effective
# (last-wins) fitter state is readable before the final argv exists.
# Every reachable branch is fixed by use_fit.
@@ -26691,17 +27459,44 @@ class LlamaCppBackend:
- _kv_bytes(effective_ctx, 0),
)
# A lazily read table stays mmap'd under every load mode: not pinned RAM.
+ # An empty inventory beside a GPU may hide an iGPU, where auto is off: priced
+ # resident.
+ _fit_child_order, _fit_child_mask = self._planned_child_gpu_order(
+ gpu_ids = gpu_ids,
+ gpu_indices = gpu_indices,
+ tp_tensor_split = tp_tensor_split,
+ extra_args = _fit_extras,
+ env = _fit_env,
+ is_vulkan_backend = is_vulkan_backend,
+ cpu_forced = _paravirtual_cpu_forced or _arch_gate_forced_cpu,
+ )
+ _fit_lazy_auto_off = (
+ not _detected_gpus
+ and (is_vulkan_backend or _gpu_device_node_present())
+ and not self._child_runs_on_cpu(
+ binary,
+ _fit_extras,
+ _fit_env,
+ cpu_forced = _paravirtual_cpu_forced or _arch_gate_forced_cpu,
+ tensor_parallel = tensor_parallel,
+ )
+ ) or self._lazy_auto_resolves_off(
+ gpu_indices = _fit_child_order,
+ detected_gpus = _detected_gpus,
+ shared_gpu_ids = _shared_gpu_ids,
+ is_vulkan_backend = is_vulkan_backend,
+ extra_args = _fit_extras,
+ env = _fit_env,
+ tensor_parallel = tensor_parallel,
+ integrated_gpu_ids = _vulkan_igpu_ids,
+ child_gpu_order = _fit_child_mask,
+ )
_fit_lazy_bytes = self._lazy_read_host_bytes(
model_path,
caps = server_caps,
extra_args = _fit_extras,
env = _fit_env,
- auto_resolves_off = self._lazy_auto_resolves_off(
- gpu_indices = gpu_indices,
- detected_gpus = _detected_gpus,
- shared_gpu_ids = _shared_gpu_ids,
- is_vulkan_backend = is_vulkan_backend,
- ),
+ auto_resolves_off = _fit_lazy_auto_off,
)
_fit_load_mode_kwargs = dict(
model_size = _fit_model_size,
@@ -26756,6 +27551,69 @@ class LlamaCppBackend:
env = _fit_env,
lazy_read_bytes = _fit_lazy_bytes,
)
+ _resident_inputs = max(0, _host_input_embd - _fit_lazy_bytes)
+ if _resident_inputs and _fit_load_mode_kwargs["model_size"]:
+ _fit_load_mode_kwargs["model_size"] = max(
+ 1, _fit_load_mode_kwargs["model_size"] - _resident_inputs
+ )
+ _fit_load_mode_kwargs["host_only_bytes"] += _resident_inputs
+ if _managed_lazy_flags is None:
+ _lazy_prompt_cache = _prompt_cache_host_bytes(
+ cache_ram,
+ _fit_extras,
+ _fit_env,
+ supports_cache_ram = bool(server_caps.get("supports_cache_ram")),
+ )
+
+ def _table_resident_fits(table: int) -> Optional[bool]:
+ if (
+ _lazy_prompt_cache is None
+ or not _fit_load_mode_kwargs["model_size"]
+ or effective_ctx <= 0
+ or _pass_through_reprices_cache(_fit_extras, n_parallel)
+ # A CPU child cannot use the cards this fit credits.
+ or self._child_runs_on_cpu(
+ binary,
+ _fit_extras,
+ _fit_env,
+ cpu_forced = _paravirtual_cpu_forced or _arch_gate_forced_cpu,
+ tensor_parallel = tensor_parallel,
+ )
+ ):
+ return None
+ return (
+ self._fit_derived_load_mode(
+ **{
+ **_fit_load_mode_kwargs,
+ "lazy_read_bytes": table,
+ "host_only_bytes": _fit_load_mode_kwargs["host_only_bytes"]
+ + table
+ + _lazy_prompt_cache
+ + int(table * _LAZY_RESIDENT_MARGIN_FRAC),
+ "announce": False,
+ }
+ )
+ == self._FIT_LOAD_MODE
+ )
+
+ _managed_lazy_flags = self._managed_lazy_mode_flags(
+ model_path,
+ caps = server_caps,
+ extra_args = _fit_extras,
+ env = _fit_env,
+ auto_resolves_off = _fit_lazy_auto_off,
+ resident_fits = _table_resident_fits,
+ binary = binary,
+ )
+ if _managed_lazy_flags and _fit_lazy_bytes:
+ if _fit_load_mode_kwargs["model_size"]:
+ _fit_load_mode_kwargs["model_size"] = max(
+ 1, _fit_load_mode_kwargs["model_size"] - _fit_lazy_bytes
+ )
+ _fit_load_mode_kwargs["host_only_bytes"] += _fit_lazy_bytes
+ _fit_load_mode_kwargs["lazy_read_bytes"] = 0
+ _fit_lazy_bytes = 0
+ _spill_inputs["lazy_read_bytes"] = _fit_lazy_bytes
_fit_load_mode = self._fit_derived_load_mode(**_fit_load_mode_kwargs)
if _moe_cache_candidate:
_moe_cache_switch = _moe_switch(_MOE_CACHE_ENV)
@@ -26825,6 +27683,7 @@ class LlamaCppBackend:
_placement_verdict_partial = False
# Same reasoning: an unpriced load keeps llama.cpp's default.
_fit_load_mode = None
+ _managed_lazy_flags = None
_moe_cache_candidate = _moe_cache_ram_ok = False
# And a half-built snapshot is not a price either: the throw can
# land after the dict exists but before the later terms are added
@@ -26869,7 +27728,9 @@ class LlamaCppBackend:
):
capped_only = _ctx_capped_for_vram and not _placement_verdict_partial
asked_ctx = _ctx_before_fit if capped_only else effective_ctx
- need_mib = (gguf_size + mmproj_size + _kv_bytes(asked_ctx)) // (1024 * 1024)
+ need_mib = (
+ max(1, gguf_size - _host_input_embd) + mmproj_size + _kv_bytes(asked_ctx)
+ ) // (1024 * 1024)
free_mib = sum(free for _, free in gpus)
need_gb, free_gb = need_mib / 1024, free_mib / 1024
# The probe's own classifiers: an integrated CUDA SoC can report a real total.
@@ -26900,24 +27761,9 @@ class LlamaCppBackend:
# Pin the whole picked set so the model can't spill onto an unpicked GPU.
if gpu_ids:
- _picked_order = [int(i) for i in gpu_ids]
- if gpu_indices is None:
- gpu_indices = _picked_order
- else:
- # Survivors keep the user's drag order.
- _kept = {int(i) for i in gpu_indices}
- _before_reorder = [int(i) for i in gpu_indices]
- _reordered = [i for i in _picked_order if i in _kept] + [
- int(i) for i in gpu_indices if int(i) not in set(_picked_order)
- ]
- # Shares are positional over ascending indices; veto rather than half-apply a reorder.
- if tp_tensor_split and len(tp_tensor_split) == len(_before_reorder):
- _share_by_id = dict(zip(_before_reorder, tp_tensor_split))
- if all(i in _share_by_id for i in _reordered):
- tp_tensor_split = [_share_by_id[i] for i in _reordered]
- gpu_indices = _reordered
- elif not tp_tensor_split:
- gpu_indices = _reordered
+ gpu_indices, tp_tensor_split = self._picker_ordered_gpu_indices(
+ gpu_ids, gpu_indices, tp_tensor_split
+ )
# Auto Vulkan fit prefers discrete GPUs and keeps that pool pinned.
elif (
is_vulkan_backend
@@ -26974,20 +27820,55 @@ class LlamaCppBackend:
_apu_ram_part = (
"APU" if self._amd_apu_wants_unified_memory(gpu_indices) else "SoC"
)
+ _apu_lazy_extras = (
+ self._strip_device_extra_args(extra_args)
+ if _gpu_ids_own_device_flags
+ else extra_args
+ )
+ _apu_lazy_env = dict(os.environ)
+ if gpu_memory_mode == "manual":
+ self._clear_manual_placement_env(_apu_lazy_env)
+ if _gpu_ids_own_device_flags:
+ self._clear_device_placement_env(_apu_lazy_env)
+ self._reconcile_split_mode_env(_apu_lazy_env, tensor_parallel = tensor_parallel)
+ _apu_child_order, _apu_child_mask = self._planned_child_gpu_order(
+ gpu_ids = gpu_ids,
+ gpu_indices = gpu_indices,
+ tp_tensor_split = tp_tensor_split,
+ extra_args = _apu_lazy_extras,
+ env = _apu_lazy_env,
+ is_vulkan_backend = False,
+ cpu_forced = _paravirtual_cpu_forced or _arch_gate_forced_cpu,
+ )
_apu_resident_model_size = max(
1,
model_size
- self._lazy_read_host_bytes(
model_path,
caps = server_caps,
- extra_args = extra_args,
- env = os.environ,
- auto_resolves_off = self._lazy_auto_resolves_off(
- gpu_indices = gpu_indices,
+ extra_args = _apu_lazy_extras,
+ env = _apu_lazy_env,
+ # A failed GPU probe still launches on the APU, where auto is off.
+ auto_resolves_off = (
+ not _detected_gpus
+ and not self._child_runs_on_cpu(
+ binary,
+ _apu_lazy_extras,
+ _apu_lazy_env,
+ cpu_forced = _paravirtual_cpu_forced or _arch_gate_forced_cpu,
+ tensor_parallel = tensor_parallel,
+ )
+ )
+ or self._lazy_auto_resolves_off(
+ gpu_indices = _apu_child_order,
detected_gpus = _detected_gpus,
shared_gpu_ids = _shared_gpu_ids,
is_vulkan_backend = False,
probe_integrated_cuda = False,
+ extra_args = _apu_lazy_extras,
+ env = _apu_lazy_env,
+ tensor_parallel = tensor_parallel,
+ child_gpu_order = _apu_child_mask,
),
),
)
@@ -27453,7 +28334,7 @@ class LlamaCppBackend:
_sanitized_split,
tp_gpus,
gpu_indices,
- model_size,
+ max(1, model_size - _host_input_embd),
effective_ctx,
cache_type_kv = cache_type_kv,
n_parallel = n_parallel,
@@ -28228,6 +29109,12 @@ class LlamaCppBackend:
]
logger.info("Load mode: %s", " ".join(_load_mode_managed))
+ # Before the extras, so a user's own lazy mode wins by last-arg.
+ self._lazy_mode_flags = list(_managed_lazy_flags or [])
+ if self._lazy_mode_flags:
+ cmd.extend(self._lazy_mode_flags)
+ logger.info("Lookup tables: %s", " ".join(self._lazy_mode_flags))
+
# A flag validate_extra_args refuses has an env twin llama.cpp reads before argv, so both must go.
if _mem_extras:
_emit_extra_args = self._strip_mmproj_args(list(_mem_extras))
@@ -28426,20 +29313,9 @@ class LlamaCppBackend:
# Clears an inherited non-layer LLAMA_ARG_SPLIT_MODE so the child cannot override the layer plan.
env[_ct_var] = _ct_norm
- # Strip inherited LLAMA_ARG_* split env; the CLI strip cannot reach it.
- if not tensor_parallel:
- # Clear any inherited LLAMA_ARG_TENSOR_SPLIT so a stale env cannot override an even split.
- _inherited_sm = (env.get("LLAMA_ARG_SPLIT_MODE") or "").strip().lower()
- if _inherited_sm and _inherited_sm != "layer":
- env.pop("LLAMA_ARG_SPLIT_MODE", None)
- env.pop("LLAMA_ARG_TENSOR_SPLIT", None)
- else:
- # Unsloth owns the tensor split: it emits --tensor-split when it
- # picks an uneven one (CLI wins) and nothing when an even split
- # is safe. Clear any inherited LLAMA_ARG_TENSOR_SPLIT so the even
- # case can't be overridden by a stale env (the layer branch above
- # clears it too).
- env.pop("LLAMA_ARG_TENSOR_SPLIT", None)
+ # Reconcile the inherited LLAMA_ARG_* split env with Unsloth's final
+ # decision (the previews above apply the same helper).
+ self._reconcile_split_mode_env(env, tensor_parallel = tensor_parallel)
# A gpu_ids pin also owns inherited device placement.
if _gpu_ids_own_device_flags:
@@ -28521,8 +29397,19 @@ class LlamaCppBackend:
layout = self._tensor_spill_layout(model_path, all_shards = True)
if layout is None or not getattr(layout, "complete", True):
return None
- if int(getattr(layout, "lm_head_bytes", 0) or 0):
+ if int(getattr(layout, "lm_head_bytes", 0) or 0) and not (
+ _override_places_tensor_on_device("token_embd.weight", run_argv, env)
+ ):
need -= int(getattr(layout, "token_embd_bytes", 0) or 0)
+ per_layer = int(getattr(layout, "per_layer_embd_bytes", 0) or 0)
+ scan = self._gguf_tensor_scan(model_path) if per_layer else None
+ if scan is not None and any(
+ _override_places_tensor_on_device(name, run_argv, env)
+ for name in scan[1]
+ if _is_per_layer_embedding(name)
+ ):
+ per_layer = 0
+ need -= per_layer
if not engages and self._nextn_predict_layers:
need -= int(getattr(layout, "excluded_block_bytes", 0) or 0)
return need
@@ -28948,6 +29835,7 @@ class LlamaCppBackend:
child_has_no_gpu = _child_has_no_gpu,
avail_mib = _preflight_avail_mib,
shared_gpu_ids = _shared_gpu_ids,
+ integrated_gpu_ids = _vulkan_igpu_ids,
)
if (
_offload_msg
@@ -28972,8 +29860,10 @@ class LlamaCppBackend:
_reachable_gpus,
env,
child_has_no_gpu = _child_has_no_gpu,
+ avail_mib = _preflight_avail_mib,
shared_gpu_ids = _shared_gpu_ids,
honor_opt_out = False,
+ integrated_gpu_ids = _vulkan_igpu_ids,
)
)
@@ -29084,6 +29974,9 @@ class LlamaCppBackend:
# Single revocation point for the tensor-spill plan; label is empty only on the first spawn.
if label:
run_cmd = self._drop_tensor_spill(run_cmd, label.lstrip("-") or "retry")
+ run_cmd = self._drop_managed_lazy_mode(
+ run_cmd, label.lstrip("-") or "retry"
+ )
# The cache only rides the first pinned launch: every retry re-places.
if label or not _argv_loads_pinned(run_cmd, env):
run_cmd = self._drop_moe_cache(
@@ -29093,6 +29986,8 @@ class LlamaCppBackend:
_did_fit_retry = False
_did_moe_cache_retry = False
for _spawn_attempt in (0, 1, 2, 3):
+ if _spawn_attempt:
+ run_cmd = self._drop_managed_lazy_mode(run_cmd, "startup")
# Defensive kill: drop an orphan Popen a concurrent load may
# have stored before we overwrite the reference (#5161).
# Also reaps the crashed first attempt on the retry pass.
@@ -30213,40 +31108,15 @@ class LlamaCppBackend:
_apu_ram_msg = None
_host_ram_msg = None
_apu_avail_mib = None
- # The opt-out hides only the message; _retry_apu_msg stays set for _retry_is_oversized.
- _retry_apu_msg = None
- if model_size is not None and _retry_wants_unified:
- _apu_avail_mib = self._available_system_memory_mib()
- _apu_resident_model_size = max(
- 1,
- model_size
- - self._lazy_read_host_bytes(
- model_path,
- caps = server_caps,
- extra_args = cmd,
- env = env,
- auto_resolves_off = self._lazy_auto_resolves_off(
- gpu_indices = _remaining,
- detected_gpus = _detected_gpus,
- shared_gpu_ids = _shared_gpu_ids,
- is_vulkan_backend = is_vulkan_backend,
- probe_integrated_cuda = False,
- ),
- ),
- )
- _retry_apu_msg = self._apu_ram_shortfall_message(
- _apu_resident_model_size + _mmproj_pinned_bytes,
- _apu_avail_mib,
- )
- # Cache tuning was decided on the crashed devices; re-decide it, since cmd is reused.
- if not self._host_offload_warning_opted_out():
- _apu_ram_msg = _retry_apu_msg
- self._record_load_warning(_retry_apu_msg)
+ # Read before the kill frees the crashed child's memory, which would inflate the reading.
+ _retry_apu_avail_mib = (
+ self._available_system_memory_mib()
+ if model_size is not None and _retry_wants_unified
+ else None
+ )
+ _retry_host_avail_mib = self._available_system_memory_mib()
# host guard credited the whole pool; the respawn reaches only _remaining
_retry_rows = [row for row in _detected_gpus if row[0] in set(_remaining)]
- _retry_host_msg = self._launch_host_shortfall_message(cmd, _retry_rows, env)
- _host_ram_msg = _retry_host_msg
- self._record_load_warning(_retry_host_msg)
logger.warning(
f"llama-server crashed with a HIP kernel-image error on "
f"GPU(s) {_crashed} -- the llama.cpp build has no kernels "
@@ -30381,6 +31251,41 @@ class LlamaCppBackend:
"Arch-crash retry leaves one GPU; dropped tensor "
"parallelism, which needs at least two."
)
+ cmd = self._drop_managed_lazy_mode(cmd, "device fallback")
+ _retry_apu_msg = None
+ if model_size is not None and _retry_wants_unified:
+ _apu_avail_mib = _retry_apu_avail_mib
+ _apu_resident_model_size = max(
+ 1,
+ model_size
+ - self._lazy_read_host_bytes(
+ model_path,
+ caps = server_caps,
+ extra_args = cmd,
+ env = env,
+ auto_resolves_off = self._lazy_auto_resolves_off(
+ gpu_indices = _remaining,
+ detected_gpus = _detected_gpus,
+ shared_gpu_ids = _shared_gpu_ids,
+ is_vulkan_backend = is_vulkan_backend,
+ probe_integrated_cuda = False,
+ extra_args = cmd,
+ env = env,
+ ),
+ ),
+ )
+ _retry_apu_msg = self._apu_ram_shortfall_message(
+ _apu_resident_model_size + _mmproj_pinned_bytes,
+ _apu_avail_mib,
+ )
+ if not self._host_offload_warning_opted_out():
+ _apu_ram_msg = _retry_apu_msg
+ self._record_load_warning(_retry_apu_msg)
+ _retry_host_msg = self._launch_host_shortfall_message(
+ cmd, _retry_rows, env, avail_mib = _retry_host_avail_mib
+ )
+ _host_ram_msg = _retry_host_msg
+ self._record_load_warning(_retry_host_msg)
# Restore what cmd means: a --fit retry may have recorded a page-lock that cmd never carried.
_fit_mode_left_cmd = bool(self._fit_load_mode_flags)
if self._fit_load_mode_flags:
@@ -30507,7 +31412,11 @@ class LlamaCppBackend:
return False
return bool(
self._launch_host_shortfall_message(
- cmd, _retry_rows, env, honor_opt_out = False
+ cmd,
+ _retry_rows,
+ env,
+ honor_opt_out = False,
+ avail_mib = _retry_host_avail_mib,
)
)
@@ -30882,7 +31791,7 @@ class LlamaCppBackend:
flash_attn = planned_flash_attn,
)
self._pending_plan_mib = _gpu_plan_mib(
- gguf_size + mmproj_size + kv_cache_bytes,
+ max(1, gguf_size - _host_input_embd) + mmproj_size + kv_cache_bytes,
list(self._pending_plan_mib),
gpus,
)
@@ -31085,6 +31994,7 @@ class LlamaCppBackend:
)
self._mmproj_fallback_reason = "projector_startup_failure"
cmd = self._strip_mmproj_args(_vision_gpu_cmd)
+ cmd = self._drop_managed_lazy_mode(cmd, "text-only retry")
# Keep a trailing sidecar pair: rewriting it would drop the drafter's
# reserve.
if [
@@ -33386,6 +34296,7 @@ class LlamaCppBackend:
host_ram_headroom_bytes = _HOST_RAM_HEADROOM_MIB * 1024 * 1024,
# -nkvo moves cache out of VRAM, so it shrinks the deficit.
kv_on_host = not _kv_offload_from_args(extra_args, env = source_env),
+ lazy_read_bytes = int(inputs.get("lazy_read_bytes") or 0),
# GPU-resident allocations the layout cannot see, since it is built
# from the target GGUF alone: the vision projector and the MTP draft
# reserve. Both are in the footprint that produced the use_fit
@@ -33415,6 +34326,20 @@ class LlamaCppBackend:
return []
return ["-ngl", "-1", "--fit", "off", *tokens]
+ def _drop_managed_lazy_mode(self, run_cmd: "list[str]", why: str) -> "list[str]":
+ """Take Unsloth's ``--lazy-mode off`` back out of an argv for a retry."""
+ if not getattr(self, "_lazy_mode_flags", None):
+ return run_cmd
+ stripped = _without_subsequence(run_cmd, self._lazy_mode_flags)
+ if stripped == run_cmd:
+ return run_cmd
+ logger.info(
+ "Lookup tables: dropping %s for the %s retry; llama.cpp streams the table.",
+ " ".join(self._lazy_mode_flags),
+ why,
+ )
+ return stripped
+
def _drop_tensor_spill(self, run_cmd: "list[str]", why: str) -> "list[str]":
"""Retries invalidate the plan, so placement returns to --fit on; the record stays for CPU
fallback."""
diff --git a/studio/backend/core/inference/lookup_tables.py b/studio/backend/core/inference/lookup_tables.py
new file mode 100644
index 00000000000..71e880dcce9
--- /dev/null
+++ b/studio/backend/core/inference/lookup_tables.py
@@ -0,0 +1,43 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
+
+"""Lookup-only embedding tables (per-layer embeddings, Engram) and how llama.cpp places them."""
+
+from __future__ import annotations
+
+from typing import Mapping, Optional
+
+LAZY_READ_TENSORS: dict[str, tuple[str, ...]] = {
+ "gemma4": ("per_layer_token_embd.weight",),
+ "qwen4exp": ("per_layer_token_embd.weight",),
+ "deepseek41": ("per_layer_token_embd.weight",),
+}
+LAZY_READ_AUTO_MIN_BYTES = 4 * 1024**3
+# Archs whose table a build marked by HOST_FORCED_TABLE_MARKER keeps on the host whatever -ot
+# says. Studio's prebuilt b11734-mix-c208e5f logs "per_layer_token_embd.weight (105471 MiB
+# iq4_nl) must stay in host memory; ignoring placement on CUDA0, using CPU" for the DeepSeek-V4.1
+# Engram table; upstream b11870 gathers it in the graph and honours the override.
+HOST_FORCED_TABLE_ARCHS = frozenset({"deepseek41"})
+HOST_FORCED_TABLE_MARKER = b"must stay in host memory; ignoring placement on"
+PER_LAYER_EMBD_PREFIX = "per_layer_token_embd"
+
+
+def is_per_layer_embedding(name: str) -> bool:
+ """Whether ``name`` is a per-layer (PLE / Engram) input embedding table."""
+ return str(name).startswith(PER_LAYER_EMBD_PREFIX)
+
+
+def lazy_read_bytes(
+ architecture: Optional[str],
+ tensor_bytes: Mapping[str, int],
+ mode: str = "auto",
+) -> int:
+ """Bytes of ``tensor_bytes`` llama.cpp reads lazily under ``mode``."""
+ if mode not in ("auto", "on"):
+ return 0
+ names = LAZY_READ_TENSORS.get(architecture or "", ())
+ return sum(
+ int(size)
+ for name, size in tensor_bytes.items()
+ if name in names and (mode == "on" or int(size) > LAZY_READ_AUTO_MIN_BYTES)
+ )
diff --git a/studio/backend/core/inference/memory_contract.py b/studio/backend/core/inference/memory_contract.py
index cf07ecfd259..5b345b92690 100644
--- a/studio/backend/core/inference/memory_contract.py
+++ b/studio/backend/core/inference/memory_contract.py
@@ -145,6 +145,8 @@ def build_memory_estimate(
layer_count = getattr(breakdown, "layer_count", None),
gpu_layers = getattr(breakdown, "gpu_layers", None),
moe_offload_unmodelled = bool(moe_offload_unmodelled),
+ lookup_table_bytes = int(getattr(breakdown, "lookup_table_bytes", 0) or 0),
+ lookup_table_on_disk = bool(getattr(breakdown, "lookup_table_on_disk", False)),
mlx_offload = (
None
if getattr(breakdown, "offloadable", None) is None
@@ -189,6 +191,8 @@ def project_estimate_memory_response(estimate: MemoryEstimate) -> dict:
"layer_count": estimate.layer_count,
"gpu_layers": estimate.gpu_layers,
"moe_offload_unmodelled": estimate.moe_offload_unmodelled,
+ "lookup_table_bytes": estimate.lookup_table_bytes,
+ "lookup_table_on_disk": estimate.lookup_table_on_disk,
"mlx_offload": estimate.mlx_offload,
}
@@ -206,6 +210,8 @@ def project_kv_cache_estimate(
return {
# The quant-file meaning. See the module docstring.
"weights_bytes": estimate.quant_file_bytes or None,
+ # Off the GPU and inside weights_bytes, so the bar can take it off the VRAM segment.
+ "lookup_table_bytes": estimate.lookup_table_bytes or None,
"kv_bytes": kv_bytes or None,
"native_context": estimate.native_context,
"spec_bytes": spec_bytes,
diff --git a/studio/backend/core/inference/offload_layout.py b/studio/backend/core/inference/offload_layout.py
index 7911b4869b2..a65d07920bd 100644
--- a/studio/backend/core/inference/offload_layout.py
+++ b/studio/backend/core/inference/offload_layout.py
@@ -16,6 +16,8 @@ from dataclasses import dataclass, field
from typing import Optional
import os
+from core.inference.lookup_tables import is_per_layer_embedding
+
logger = logging.getLogger(__name__)
# Hybrid Mamba-2 archs storing head_count_kv (and on Bolt expert_used_count) per layer.
@@ -52,6 +54,7 @@ class ModelLayout:
lm_head_bytes: int = 0
# llama.cpp pins dev_input to CPU: never VRAM, but charged to host RAM
token_embd_bytes: int = 0
+ per_layer_embd_bytes: int = 0
# Every tensor's bytes, excluded blocks included; the file size less its metadata.
tensor_bytes: int = 0
# output_norm and friends: GPU-resident, too small to be worth spilling.
@@ -259,6 +262,7 @@ def _layout_from_readers(readers) -> ModelLayout:
resident: dict[int, int] = {}
lm_head = 0
token_embd = 0
+ per_layer_embd = 0
other_resident = 0
for tensor in (t for r in readers for t in r.tensors):
@@ -276,8 +280,11 @@ def _layout_from_readers(readers) -> ModelLayout:
else:
resident[index] = resident.get(index, 0) + nbytes
continue
+ # Before the token_embd substring below, which would otherwise swallow the per-layer table.
+ if is_per_layer_embedding(name):
+ per_layer_embd += nbytes
# token_embd_norm is a repeating-layer tensor, not a host-pinned input embedding.
- if "token_embd" in name and not name.startswith("token_embd_norm"):
+ elif "token_embd" in name and not name.startswith("token_embd_norm"):
token_embd += nbytes
elif name == "output.weight":
lm_head += nbytes
@@ -315,6 +322,7 @@ def _layout_from_readers(readers) -> ModelLayout:
blocks = blocks,
lm_head_bytes = lm_head,
token_embd_bytes = token_embd,
+ per_layer_embd_bytes = per_layer_embd,
tensor_bytes = sum(int(t.n_bytes) for r in readers for t in r.tensors),
other_resident_bytes = other_resident,
kv_bytes_per_token_f16 = kv_per_token,
diff --git a/studio/backend/core/inference/offload_planner.py b/studio/backend/core/inference/offload_planner.py
index 6e1ea8ece53..622d5db5a95 100644
--- a/studio/backend/core/inference/offload_planner.py
+++ b/studio/backend/core/inference/offload_planner.py
@@ -97,6 +97,7 @@ class PlanOptions:
kv_quant_type: str = "q8_0"
# -nkvo (or LLAMA_ARG_KV_OFFLOAD false) puts the whole KV cache on host (llama-kv-cache.cpp), so it is not charged to VRAM
kv_on_host: bool = False
+ lazy_read_bytes: int = 0
@dataclass(frozen = True)
@@ -792,8 +793,8 @@ def _finish(
spilled_bytes = sum(b.spillable_bytes for b in chosen) + (
layout.lm_head_bytes if spill_lm_head else 0
)
- # token_embd is host-resident on every launch, so it is host RAM this plan must pay for even when nothing is spilled
- host_bytes = layout.token_embd_bytes + spilled_bytes
+ lazy = min(max(0, int(opts.lazy_read_bytes or 0)), layout.per_layer_embd_bytes)
+ host_bytes = layout.token_embd_bytes + layout.per_layer_embd_bytes - lazy + spilled_bytes
if opts.kv_on_host:
# -nkvo cache lives in host RAM; the mmap decision must count it.
host_bytes += (
diff --git a/studio/backend/hub/schemas/inventory.py b/studio/backend/hub/schemas/inventory.py
index a718ad47c01..698857f24af 100644
--- a/studio/backend/hub/schemas/inventory.py
+++ b/studio/backend/hub/schemas/inventory.py
@@ -46,6 +46,11 @@ class GgufVariantDetail(BaseModel):
)
size_bytes: int = Field(0, description = "File size in bytes")
download_size_bytes: int = Field(0, description = "Total bytes needed to download this variant")
+ lookup_table_bytes: int = Field(
+ 0,
+ description = "Bytes of lazily read lookup tables (per-layer embeddings / Engram) "
+ "llama.cpp streams from disk by default. Part of size_bytes; 0 when none or unknown.",
+ )
pending_drafter_filename: Optional[str] = Field(
None,
description = (
diff --git a/studio/backend/hub/services/models/gguf_variants.py b/studio/backend/hub/services/models/gguf_variants.py
index cbbd9894517..5acfd685588 100644
--- a/studio/backend/hub/services/models/gguf_variants.py
+++ b/studio/backend/hub/services/models/gguf_variants.py
@@ -12,6 +12,7 @@ import contextvars
import functools
import hashlib
import json
+import os
import re
import threading
import time
@@ -1248,6 +1249,346 @@ async def _mlx_precision_variants_answer(
return VariantsAnswer(response, None, cache_authorized = False)
+_LOOKUP_REMOTE_ENV = "UNSLOTH_LOOKUP_TABLE_REMOTE"
+_LOOKUP_DEADLINE_S = 2.0
+_LOOKUP_CONCURRENCY = 4
+_LOOKUP_ARCH_TTL_S = 6 * 3600.0
+_LOOKUP_ARCH_FAIL_TTL_S = 600.0
+_lookup_arch_cache: dict[tuple[str, str], tuple[Optional[str], float]] = {}
+_lookup_arch_lock = threading.Lock()
+# Key of a read in flight -> its done event, so a second listing waits for it instead of skipping.
+_lookup_inflight: dict = {}
+_LOOKUP_POOL = ThreadPoolExecutor(
+ max_workers = _LOOKUP_CONCURRENCY, thread_name_prefix = "gguf-lookup-tables"
+)
+
+
+async def _lookup_thread(fn, /, *args):
+ loop = asyncio.get_running_loop()
+ call = functools.partial(contextvars.copy_context().run, fn, *args)
+ return await loop.run_in_executor(_LOOKUP_POOL, call)
+
+
+def _claim_lookup(key) -> tuple[Optional[threading.Event], Optional[threading.Event]]:
+ """``(None, own)`` once ``key`` is claimed here, else ``(held, None)``: the done event
+ of the read already holding it."""
+ with _lookup_arch_lock:
+ held = _lookup_inflight.get(key)
+ if held is not None:
+ return held, None
+ own = _lookup_inflight[key] = threading.Event()
+ return None, own
+
+
+def _release_lookup(key, own: threading.Event) -> None:
+ """Release ``key`` if ``own`` still holds it: a late worker never drops a newer claim."""
+ with _lookup_arch_lock:
+ if _lookup_inflight.get(key) is own:
+ del _lookup_inflight[key]
+ own.set()
+
+
+async def _wait_for_lookup(done: threading.Event, deadline: float) -> bool:
+ """Whether another listing's read of the same key ended before ``deadline`` (loop time)."""
+ loop = asyncio.get_running_loop()
+ while not done.is_set():
+ if loop.time() >= deadline:
+ return False
+ await asyncio.sleep(0.02)
+ return True
+
+
+def _release_unrun_lookup(started: list, key, own, task: "asyncio.Future") -> None:
+ """Release the inflight key of a read that never ran."""
+ if not started and (task.cancelled() or task.exception() is not None):
+ _release_lookup(key, own)
+
+
+async def _lookup_await_once(key, deadline: float, fn, /, *args):
+ """``fn(*args)`` on the lookup pool, awaited until ``deadline`` (loop time). A call
+ already in flight under ``key`` is waited for, then ``fn`` reruns on its cached answer
+ (or retries, when that call left none)."""
+ while True:
+ held, own = _claim_lookup(key)
+ if own is not None:
+ break
+ if not await _wait_for_lookup(held, deadline):
+ return None
+
+ started: list = []
+ # Set at the deadline: a worker freeing up mid-cancel skips the call instead of running late.
+ expired = threading.Event()
+
+ def _run():
+ started.append(True)
+ try:
+ return None if expired.is_set() else fn(*args)
+ finally:
+ _release_lookup(key, own)
+
+ task = asyncio.ensure_future(_lookup_thread(_run))
+ task.add_done_callback(functools.partial(_release_unrun_lookup, started, key, own))
+ loop = asyncio.get_running_loop()
+ done, _pending = await asyncio.wait({task}, timeout = max(0.0, deadline - loop.time()))
+ if task not in done:
+ expired.set()
+ task.cancel()
+ return None
+ if task.cancelled() or task.exception() is not None:
+ return None
+ return task.result()
+
+
+def _cached_hub_repo_architecture(repo_id: str, hf_token: Optional[str]):
+ """``(known, architecture)`` from the cache alone; never blocks on the Hub."""
+ key = (repo_id.lower(), hf_cache_scan.token_fingerprint(hf_token))
+ with _lookup_arch_lock:
+ hit = _lookup_arch_cache.get(key)
+ if hit is not None and hit[1] > time.monotonic():
+ return True, hit[0]
+ return False, None
+
+
+_LOOKUP_HOST_TTL_S = 60.0
+_lookup_host_streams: list = [None, 0.0]
+
+
+def _host_streams_lookup_tables() -> bool:
+ """Whether a default launch here streams the tables (an iGPU / APU reads them resident)."""
+ now = time.monotonic()
+ with _lookup_arch_lock:
+ streams, expires = _lookup_host_streams
+ if streams is not None and expires > now:
+ return streams
+ try:
+ from routes.inference import get_llama_cpp_backend
+ streams = bool(get_llama_cpp_backend().lookup_tables_stream_by_default())
+ except Exception: # noqa: BLE001 -- unknown: price the tables as resident
+ streams = False
+ with _lookup_arch_lock:
+ _lookup_host_streams[:] = [streams, now + _LOOKUP_HOST_TTL_S]
+ return streams
+
+
+def _listing_lazy_mode() -> str:
+ """The lazy mode a default launch here runs under."""
+ try:
+ from core.inference.llama_cpp import _effective_lazy_mode
+ from routes.inference import get_llama_cpp_backend
+
+ flag = get_llama_cpp_backend().probe_server_capabilities().get("lazy_mode_flag")
+ mode = _effective_lazy_mode(None, os.environ, flag = str(flag or "--lazy-mode"))
+ except Exception: # noqa: BLE001 -- unknown build: the default threshold
+ return "auto"
+ return "on" if mode == "on" else "auto"
+
+
+def _lookup_remote_enabled() -> bool:
+ return (os.environ.get(_LOOKUP_REMOTE_ENV) or "").strip().lower() not in (
+ "0",
+ "off",
+ "false",
+ "no",
+ )
+
+
+def _hub_repo_architecture(repo_id: str, hf_token: Optional[str]) -> Optional[str]:
+ """``general.architecture`` the Hub reports for a GGUF repo, cached per repo and token."""
+ key = (repo_id.lower(), hf_cache_scan.token_fingerprint(hf_token))
+ now = time.monotonic()
+ with _lookup_arch_lock:
+ hit = _lookup_arch_cache.get(key)
+ if hit is not None and hit[1] > now:
+ return hit[0]
+ arch: Optional[str] = None
+ ttl = _LOOKUP_ARCH_FAIL_TTL_S
+ try:
+ from huggingface_hub import HfApi
+
+ info = HfApi().model_info(repo_id, expand = ["gguf"], token = hf_token or None, timeout = 5)
+ gguf_meta = getattr(info, "gguf", None) or {}
+ arch = str(gguf_meta.get("architecture") or "") or None
+ ttl = _LOOKUP_ARCH_TTL_S
+ except Exception as exc: # noqa: BLE001 -- unknown arch: no remote reads
+ logger.debug("lookup tables: no architecture for %s (%s)", repo_id, exc)
+ with _lookup_arch_lock:
+ _lookup_arch_cache[key] = (arch, now + ttl)
+ return arch
+
+
+def _local_variant_gguf(
+ answer: "VariantsAnswer",
+ repo_id: str,
+ variant,
+ local_path: Optional[str] = None,
+) -> Optional[str]:
+ """The on-disk main GGUF behind a listed row, or None."""
+ filename = str(variant.filename or "")
+ sources = answer.variant_context_sources or {}
+ candidates = [sources.get(str(variant.quant or "").lower())]
+ roots = [repo_id] if is_local_path(repo_id) else []
+ if answer.context_source:
+ roots.append(answer.context_source)
+ pinned = _snapshot_scope_for_request(repo_id, local_path)
+ if pinned is not None:
+ roots.append(str(pinned))
+ for root in roots:
+ path = Path(root).expanduser()
+ candidates.append(str((path.parent if path.is_file() else path) / filename))
+ for candidate in candidates:
+ if candidate and os.path.isfile(candidate):
+ return candidate
+ if (
+ variant.downloaded
+ and answer.cache_authorized
+ and not is_local_path(repo_id)
+ and _is_valid_repo_id(repo_id)
+ ):
+ try:
+ from huggingface_hub import try_to_load_from_cache
+
+ repo_dir = _repo_cache_dir_for_request(repo_id, local_path)
+ found = try_to_load_from_cache(repo_id, filename, cache_dir = str(repo_dir.parent))
+ if isinstance(found, str) and os.path.isfile(found):
+ return found
+ except Exception: # noqa: BLE001 -- not cached here
+ return None
+ return None
+
+
+def _remote_lookup_bytes(
+ repo_id: str,
+ filename: str,
+ hf_token,
+ identity,
+ mode = "auto",
+ known_arch: Optional[str] = None,
+) -> int:
+ from hub.utils.gguf_lookup_tables import remote_lookup_info
+ return remote_lookup_info(
+ repo_id, filename, hf_token, identity = identity, mode = mode, known_arch = known_arch
+ )[1]
+
+
+async def _with_lookup_tables(
+ answer: "VariantsAnswer",
+ repo_id: str,
+ *,
+ hf_token: Optional[str],
+ offline: bool,
+ prefer_local_cache: bool,
+ local_path: Optional[str] = None,
+) -> "VariantsAnswer":
+ """``answer`` with ``lookup_table_bytes`` filled per variant."""
+ try:
+ from core.inference.lookup_tables import LAZY_READ_AUTO_MIN_BYTES
+ from hub.utils.gguf_lookup_tables import (
+ lazy_architecture,
+ local_lookup_info,
+ remote_cached,
+ )
+
+ variants = list(answer.response.variants or ())
+ # A host that reads the tables resident lists no discount: no header is worth reading.
+ if not variants or not await listing_thread(True, _host_streams_lookup_tables):
+ return answer
+ mode = await listing_thread(True, _listing_lazy_mode)
+ found: dict[int, int] = {}
+ arch: Optional[str] = None
+ unresolved = []
+ for index, v in enumerate(variants):
+ if not str(v.filename or "").lower().endswith(".gguf"):
+ continue
+ # An update's cached file is the old revision: online, read the published header.
+ published = v.update_available and not offline and not prefer_local_cache
+ path = (
+ None
+ if published and not is_local_path(repo_id)
+ else await listing_thread(True, _local_variant_gguf, answer, repo_id, v, local_path)
+ )
+ if path:
+ local_arch, table = await listing_thread(True, local_lookup_info, path, mode)
+ # Only a lazy one speaks for the repo: a repo can hold several families.
+ if lazy_architecture(local_arch):
+ arch = arch or local_arch
+ if local_arch is not None:
+ found[index] = table
+ continue
+ # Not on disk, or unreadable there (a shard still downloading): the Hub's header.
+ unresolved.append((index, v))
+ remote_ok = (
+ unresolved
+ and not offline
+ and not prefer_local_cache
+ and _lookup_remote_enabled()
+ and not is_local_path(repo_id)
+ and _is_valid_repo_id(repo_id)
+ )
+ if remote_ok:
+ # Its own try: a Hub failure must not drop the local answers above.
+ try:
+ deadline = asyncio.get_running_loop().time() + _LOOKUP_DEADLINE_S
+ if arch is None:
+ known, arch = _cached_hub_repo_architecture(repo_id, hf_token)
+ if not known and (
+ mode == "on"
+ or any(
+ int(v.size_bytes or 0) > LAZY_READ_AUTO_MIN_BYTES
+ for _i, v in unresolved
+ )
+ ):
+ arch = await _lookup_await_once(
+ ("arch", repo_id.lower(), hf_cache_scan.token_fingerprint(hf_token)),
+ deadline,
+ _hub_repo_architecture,
+ repo_id,
+ hf_token,
+ )
+ if lazy_architecture(arch):
+ _fingerprint = hf_cache_scan.token_fingerprint(hf_token)
+ tasks = {}
+ for index, v in unresolved:
+ identity = int(v.size_bytes or 0)
+ # Under auto only a tensor over the threshold streams.
+ if mode != "on" and 0 < identity <= LAZY_READ_AUTO_MIN_BYTES:
+ continue
+ cached = remote_cached(repo_id, v.filename, None, identity, mode = mode)
+ if cached is not None:
+ found[index] = cached[1]
+ continue
+ key = ("table", repo_id.lower(), v.filename, identity, mode, _fingerprint)
+ task = asyncio.ensure_future(
+ _lookup_await_once(
+ key,
+ deadline,
+ _remote_lookup_bytes,
+ repo_id,
+ v.filename,
+ hf_token,
+ identity,
+ mode,
+ arch,
+ )
+ )
+ tasks[task] = index
+ for task, value in zip(tasks, await asyncio.gather(*tasks)):
+ found[tasks[task]] = int(value or 0)
+ except Exception as exc: # noqa: BLE001 -- keep the local answers
+ logger.debug("lookup tables: Hub reads skipped for %s (%s)", repo_id, exc)
+ if not any(found.values()):
+ return answer
+ updated = [
+ v.model_copy(update = {"lookup_table_bytes": int(found.get(i, 0) or 0)})
+ if found.get(i)
+ else v
+ for i, v in enumerate(variants)
+ ]
+ return answer._replace(response = answer.response.model_copy(update = {"variants": updated}))
+ except Exception as exc: # noqa: BLE001 -- the listing never fails on this
+ logger.debug("lookup tables: listing annotation skipped for %s (%s)", repo_id, exc)
+ return answer
+
+
async def get_gguf_variants_answer(
repo_id: str,
prefer_local_cache: bool = False,
@@ -2389,7 +2730,14 @@ async def get_gguf_variants_answer(
)
if mlx_answer is not None:
return mlx_answer
- return answer
+ return await _with_lookup_tables(
+ answer,
+ repo_id,
+ hf_token = hf_token,
+ offline = bool(offline),
+ prefer_local_cache = bool(prefer_local_cache),
+ local_path = local_path,
+ )
except HTTPException as exc:
# Offline, a repo with no cached GGUF raises here, before the MLX fallback above.
if (
diff --git a/studio/backend/hub/utils/gguf_header_delta.py b/studio/backend/hub/utils/gguf_header_delta.py
index d2ecba77076..0298c61331a 100644
--- a/studio/backend/hub/utils/gguf_header_delta.py
+++ b/studio/backend/hub/utils/gguf_header_delta.py
@@ -492,6 +492,10 @@ def hub_range_fetcher(
with _ranged_stream(get_session(), url, headers) as resp:
if resp.status_code != 206:
raise ValueError(f"range request answered {resp.status_code}")
+ headers = getattr(resp, "headers", None) or {}
+ total = str(headers.get("Content-Range") or "").rpartition("/")[2]
+ if total.isdigit():
+ fetch.file_size = int(total)
return b"".join(_iter_body(resp, 1 << 20))
return fetch
diff --git a/studio/backend/hub/utils/gguf_lookup_tables.py b/studio/backend/hub/utils/gguf_lookup_tables.py
new file mode 100644
index 00000000000..5b3719c1df7
--- /dev/null
+++ b/studio/backend/hub/utils/gguf_lookup_tables.py
@@ -0,0 +1,330 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
+
+"""Bytes of the lookup-only tables (per-layer embeddings / Engram) llama.cpp streams from disk."""
+
+from __future__ import annotations
+
+import bisect
+import os
+import re
+import struct
+import threading
+import time
+from collections import OrderedDict
+from typing import Callable, Optional
+
+from core.inference.lookup_tables import LAZY_READ_TENSORS, lazy_read_bytes
+
+_SPLIT_RE = re.compile(r"^(.*)-(\d{5})-of-(\d{5})(\.gguf)$", re.IGNORECASE)
+_SCALAR_SIZES = {0: 1, 1: 1, 2: 2, 3: 2, 4: 4, 5: 4, 6: 4, 7: 1, 10: 8, 11: 8, 12: 8}
+_STRING, _ARRAY, _UINT32 = 8, 9, 4
+_FIRST_READ = 8 << 20
+# A later split shard carries only the split keys: its header is a few KiB.
+_SPLIT_SHARD_FIRST_READ = 64 << 10
+MAX_HEADER_BYTES = 64 << 20
+
+_WANTED = {name.encode() for names in LAZY_READ_TENSORS.values() for name in names}
+
+
+class _NeedMore(Exception):
+ pass
+
+
+def lazy_architecture(arch: Optional[str]) -> bool:
+ """Whether ``arch`` flags a lookup table llama.cpp may read lazily."""
+ return bool(arch) and arch in LAZY_READ_TENSORS
+
+
+def parse_header(
+ buf: bytes,
+ file_size: Optional[int] = None,
+ arch_only: bool = False,
+) -> tuple[Optional[str], dict[str, int]]:
+ """(architecture, {lazy-candidate tensor name: bytes}) from a GGUF v2/v3 header. A quant
+ type gguf predates is sized from the next tensor's offset (or ``file_size``), as the
+ launch scanner does. ``arch_only`` stops at the architecture key."""
+ from gguf.constants import GGML_QUANT_SIZES
+
+ pos = 0
+
+ def take(n: int) -> bytes:
+ nonlocal pos
+ if pos + n > len(buf):
+ raise _NeedMore()
+ out = buf[pos : pos + n]
+ pos += n
+ return out
+
+ def u32() -> int:
+ return struct.unpack("<I", take(4))[0]
+
+ def u64() -> int:
+ return struct.unpack("<Q", take(8))[0]
+
+ def string() -> bytes:
+ n = u64()
+ if n > (1 << 30):
+ raise ValueError("implausible string length")
+ return take(n)
+
+ def skip_value(vtype: int, depth: int = 0) -> None:
+ if depth > 8:
+ raise ValueError("implausible array nesting")
+ if vtype in _SCALAR_SIZES:
+ take(_SCALAR_SIZES[vtype])
+ elif vtype == _STRING:
+ string()
+ elif vtype == _ARRAY:
+ item = u32()
+ count = u64()
+ if item in _SCALAR_SIZES:
+ take(_SCALAR_SIZES[item] * count)
+ elif item == _STRING:
+ for _ in range(count):
+ string()
+ else:
+ for _ in range(count):
+ skip_value(item, depth + 1)
+ else:
+ raise ValueError(f"unknown value type {vtype}")
+
+ if take(4) != b"GGUF":
+ raise ValueError("not a GGUF")
+ if u32() not in (2, 3):
+ raise ValueError("unsupported GGUF version")
+ n_tensors = u64()
+ n_kv = u64()
+ if n_tensors > 10_000_000 or n_kv > 10_000_000:
+ raise ValueError("implausible header counts")
+ arch: Optional[str] = None
+ alignment = 32 # GGUF's default when general.alignment is absent
+ for _ in range(n_kv):
+ key = string()
+ vtype = u32()
+ if key == b"general.architecture" and vtype == _STRING:
+ arch = string().decode("utf-8", "replace")
+ if arch_only:
+ return arch, {}
+ elif key == b"general.alignment" and vtype == _UINT32:
+ alignment = max(1, u32())
+ else:
+ skip_value(vtype)
+ if arch_only:
+ return arch, {}
+ named: dict[str, int] = {}
+ offsets: list[int] = []
+ unsized: list[tuple[str, int]] = []
+ for _ in range(n_tensors):
+ name = string()
+ n_dims = u32()
+ if n_dims > 16:
+ raise ValueError("implausible tensor rank")
+ dims = [u64() for _ in range(n_dims)]
+ ggml_type = u32()
+ offset = u64()
+ offsets.append(offset)
+ if name not in _WANTED:
+ continue
+ block = GGML_QUANT_SIZES.get(ggml_type)
+ if block is None:
+ unsized.append((name.decode("utf-8", "replace"), offset))
+ continue
+ n = 1
+ for d in dims:
+ n *= int(d)
+ named[name.decode("utf-8", "replace")] = n // block[0] * block[1]
+ if unsized:
+ data_start = -(-pos // alignment) * alignment
+ ordered = sorted(offsets)
+ for name, offset in unsized:
+ later = ordered[bisect.bisect_right(ordered, offset) :]
+ if later:
+ named[name] = later[0] - offset
+ elif file_size is not None:
+ named[name] = max(0, file_size - data_start - offset)
+ return arch, named
+
+
+def _read_growing(
+ read: Callable[[int], bytes],
+ cap: int = MAX_HEADER_BYTES,
+ *,
+ first_read: int = _FIRST_READ,
+ file_size: Optional[Callable[[], Optional[int]]] = None,
+ arch_only: bool = False,
+):
+ want = min(first_read, cap)
+ while True:
+ buf = read(want)
+ try:
+ return parse_header(
+ buf, file_size() if file_size is not None else None, arch_only = arch_only
+ )
+ except _NeedMore:
+ if len(buf) < want or want >= cap:
+ raise ValueError("GGUF header unreadable within the cap")
+ want = min(want * 2, cap)
+
+
+def header_of_local(path: str) -> tuple[Optional[str], dict[str, int]]:
+ with open(path, "rb") as f:
+ size = os.fstat(f.fileno()).st_size
+
+ def read(n: int) -> bytes:
+ f.seek(0)
+ return f.read(n)
+
+ return _read_growing(read, file_size = lambda: size)
+
+
+def shard_names(filename: str) -> list[str]:
+ """Every shard of ``filename`` in order (itself alone when it is not a split)."""
+ head, sep, base = str(filename).replace("\\", "/").rpartition("/")
+ match = _SPLIT_RE.match(base)
+ if not match:
+ return [filename]
+ prefix, _index, total, ext = match.groups()
+ return [
+ f"{head}{sep}{prefix}-{i:05d}-of-{int(total):05d}{ext}" for i in range(1, int(total) + 1)
+ ]
+
+
+_LOCAL_CACHE: "OrderedDict[tuple, tuple[Optional[str], int]]" = OrderedDict()
+_REMOTE_CACHE: "OrderedDict[tuple, tuple[tuple[Optional[str], int], float]]" = OrderedDict()
+_REMOTE_TTL_S = 6 * 3600.0
+_CACHE_LIMIT = 512
+_lock = threading.Lock()
+
+
+def _remember(cache: OrderedDict, key, value) -> None:
+ with _lock:
+ cache[key] = value
+ cache.move_to_end(key)
+ while len(cache) > _CACHE_LIMIT:
+ cache.popitem(last = False)
+
+
+MAX_SHARDS_READ = 8
+
+
+def _scan_shards(
+ read_header: Callable[[str], tuple],
+ names: list[str],
+ mode: str = "auto",
+ max_shards: Optional[int] = MAX_SHARDS_READ,
+ known_arch: Optional[str] = None,
+):
+ """(architecture, lazy table bytes under ``mode``), stopping at the first shard carrying the
+ table."""
+ arch: Optional[str] = None
+ order = names[:max_shards]
+ hinted = lazy_architecture(known_arch) and len(names) > 1
+ if hinted:
+ order = names[1:max_shards] + names[:1]
+ for name in order:
+ part_arch, named = read_header(name)
+ if hinted and not arch and not part_arch:
+ if not named:
+ continue
+ arch = read_header(names[0], True)[0]
+ arch = part_arch or arch
+ if not lazy_architecture(arch):
+ return arch, 0
+ found = lazy_read_bytes(arch, named, mode)
+ if found or named:
+ return arch, found
+ return arch, 0
+
+
+def local_lookup_info(path: Optional[str], mode: str = "auto") -> tuple[Optional[str], int]:
+ """(architecture, table bytes llama.cpp reads lazily under ``mode``) for a local GGUF."""
+ if not path:
+ return None, 0
+ try:
+ names = shard_names(path)
+ key = (mode,) + tuple(
+ (p, st.st_size, st.st_mtime_ns) for p in names for st in (os.stat(p),)
+ )
+ except OSError:
+ return None, 0
+ with _lock:
+ hit = _LOCAL_CACHE.get(key)
+ if hit is not None:
+ return hit
+ try:
+ # Local headers are cheap: every shard, as the launch scanner reads them.
+ answer = _scan_shards(header_of_local, names, mode, max_shards = None)
+ except Exception: # noqa: BLE001 -- unreadable header: price the whole file, as before
+ answer = (None, 0)
+ _remember(_LOCAL_CACHE, key, answer)
+ return answer
+
+
+def local_lookup_table_bytes(path: Optional[str]) -> int:
+ return local_lookup_info(path)[1]
+
+
+def remote_cached(
+ repo_id: str,
+ filename: str,
+ revision: Optional[str],
+ identity = None,
+ *,
+ mode: str = "auto",
+):
+ """The cached remote answer under lazy ``mode``, keyed on the file's identity (its listed size by default), so a re-upload under the same name is read again."""
+ with _lock:
+ hit = _REMOTE_CACHE.get((repo_id.lower(), filename, revision or "", identity, mode))
+ if hit is None or hit[1] <= time.monotonic():
+ return None
+ return hit[0]
+
+
+def remote_lookup_info(
+ repo_id: str,
+ filename: str,
+ token = None,
+ *,
+ revision: Optional[str] = None,
+ identity = None,
+ fetcher_fn: Optional[Callable] = None,
+ mode: str = "auto",
+ known_arch: Optional[str] = None,
+) -> tuple[Optional[str], int]:
+ """(architecture, lazily read table bytes) from Range reads of a Hub GGUF's header(s)."""
+ # Hub repo ids are case-insensitive.
+ key = (repo_id.lower(), filename, revision or "", identity, mode)
+ hit = remote_cached(repo_id, filename, revision, identity, mode = mode)
+ if hit is not None:
+ return hit
+ if fetcher_fn is None:
+ from hub.utils.gguf_header_delta import hub_range_fetcher as fetcher_fn
+
+ names = shard_names(filename)
+
+ def read_header(name: str, arch_only: bool = False):
+ fetch = fetcher_fn(repo_id, name, token, revision = revision)
+ buf = b""
+
+ def read(n: int) -> bytes:
+ nonlocal buf
+ if n > len(buf):
+ buf += fetch(len(buf), n - 1)
+ return buf[:n]
+
+ return _read_growing(
+ read,
+ first_read = (
+ _FIRST_READ if name == names[0] and not arch_only else _SPLIT_SHARD_FIRST_READ
+ ),
+ file_size = lambda: getattr(fetch, "file_size", None),
+ arch_only = arch_only,
+ )
+
+ try:
+ answer = _scan_shards(read_header, names, mode, known_arch = known_arch)
+ except Exception: # noqa: BLE001 -- unreachable or unreadable: today's behaviour
+ return None, 0
+ _remember(_REMOTE_CACHE, key, (answer, time.monotonic() + _REMOTE_TTL_S))
+ return answer
diff --git a/studio/backend/models/inference.py b/studio/backend/models/inference.py
index 0e4b5874922..7b19c099da9 100644
--- a/studio/backend/models/inference.py
+++ b/studio/backend/models/inference.py
@@ -1322,6 +1322,17 @@ class EstimateMemoryResponse(BaseModel):
description = "True when --n-cpu-moe is set: experts move per-tensor, not "
"per-block, so the GPU/host split above ignores it and reads high.",
)
+ lookup_table_bytes: int = Field(
+ 0,
+ description = "Lookup-only tables (per-layer embeddings / Engram) in the model "
+ "file. Never in gpu_bytes; excluded from weights_bytes and total_bytes when "
+ "lookup_table_on_disk is true.",
+ )
+ lookup_table_on_disk: bool = Field(
+ False,
+ description = "True when llama.cpp streams those tables from disk (its default "
+ "lazy mode), so only the rows in use are paged in.",
+ )
mlx_offload: Optional[MlxOffloadEstimate] = Field(
None,
description = "MLX: what the load reads from disk instead of memory, priced in "
@@ -1434,6 +1445,12 @@ class MemoryEstimate(BaseModel):
moe_offload_unmodelled: bool = Field(
False, description = "--n-cpu-moe is set, so the GPU figure reads high"
)
+ lookup_table_bytes: int = Field(
+ 0, description = "Lookup-only tables in the model file, never charged to the GPU"
+ )
+ lookup_table_on_disk: bool = Field(
+ False, description = "Those tables are streamed from disk, not resident"
+ )
mlx_offload: Optional[MlxOffloadEstimate] = Field(
None, description = "MLX: what the load reads from disk; null when not MLX"
)
diff --git a/studio/backend/models/models.py b/studio/backend/models/models.py
index d89c011d305..daaf86754d5 100644
--- a/studio/backend/models/models.py
+++ b/studio/backend/models/models.py
@@ -192,6 +192,11 @@ class GgufVariantDetail(BaseModel):
)
size_bytes: int = Field(0, description = "File size in bytes")
download_size_bytes: int = Field(0, description = "Total bytes needed to download this variant")
+ lookup_table_bytes: int = Field(
+ 0,
+ description = "Bytes of lazily read lookup tables (per-layer embeddings / Engram) "
+ "llama.cpp streams from disk by default. Part of size_bytes; 0 when none or unknown.",
+ )
# Mirrors hub.schemas.inventory.GgufVariantDetail; see display_label above.
pending_drafter_filename: Optional[str] = Field(
None, description = "Missing separate drafter when the main GGUF is already cached"
diff --git a/studio/backend/routes/inference.py b/studio/backend/routes/inference.py
index cfcd82d0ba4..9c29fc543d6 100644
--- a/studio/backend/routes/inference.py
+++ b/studio/backend/routes/inference.py
@@ -13249,6 +13249,8 @@ class _GgufMemoryBreakdown(NamedTuple):
gpu_layers: Optional[int]
# Host-resident checkpoint share of kv_bytes. Keep last for positional callers.
kv_checkpoint_bytes: int = 0
+ lookup_table_bytes: int = 0
+ lookup_table_on_disk: bool = False
def _gguf_offloaded_layer_fraction(
@@ -13849,6 +13851,7 @@ def _gguf_memory_breakdown(
tensor_split_possible: bool = True,
# A GPU pin owns placement: load_model strips device flags and env before launch.
device_pin_governs: bool = False,
+ selected_gpu_ids: Optional[list[int]] = None,
) -> Optional[_GgufMemoryBreakdown]:
"""None if nothing sizes. Files are cached; the KV and compute buffers are re-derived per call."""
from core.inference.llama_server_args import (
@@ -14064,6 +14067,223 @@ def _gguf_memory_breakdown(
# Only the main weight is split by --gpu-layers; companions have their own placement flags.
main_bytes = min(gpu_files_bytes, LlamaCppBackend._get_gguf_size_bytes(gguf_path))
companion_bytes = max(0, gpu_files_bytes - main_bytes)
+ from core.inference.llama_cpp import (
+ _effective_lazy_mode,
+ _is_per_layer_embedding,
+ _override_places_tensor_on_device,
+ _without_torch_gpu_probe,
+ )
+ from hub.utils.gguf_lookup_tables import local_lookup_info
+
+ _lt_backend = get_llama_cpp_backend()
+ try:
+ _lt_flag = str(
+ _lt_backend.probe_server_capabilities().get("lazy_mode_flag") or "--lazy-mode"
+ )
+ except Exception: # noqa: BLE001 -- unknown build: the default spelling
+ _lt_flag = "--lazy-mode"
+ _lt_arch, _lt_bytes = local_lookup_info(
+ gguf_path,
+ "on" if _effective_lazy_mode(extras, os.environ, flag = _lt_flag) == "on" else "auto",
+ )
+ lookup_table_bytes = min(main_bytes, _lt_bytes)
+ lookup_table_on_disk = False
+ _lt_extras, _lt_env = extras, dict(os.environ)
+ if selected_gpu_ids or device_pin_governs:
+ _lt_extras = LlamaCppBackend._strip_device_extra_args(extras)
+ LlamaCppBackend._clear_device_placement_env(_lt_env)
+ if gpu_memory_mode == "manual":
+ LlamaCppBackend._clear_manual_placement_env(_lt_env)
+ LlamaCppBackend._reconcile_split_mode_env(_lt_env, tensor_parallel = bool(tensor_parallel))
+ if lookup_table_bytes:
+ lookup_table_on_disk = _lt_backend.lookup_tables_stream_by_default(
+ _lt_extras,
+ _lt_env,
+ gpu_ids = selected_gpu_ids,
+ tensor_parallel = tensor_parallel,
+ )
+ if not lookup_table_on_disk:
+ try:
+ _lt_scan = _lt_backend._gguf_tensor_scan(gguf_path)
+ except Exception: # noqa: BLE001 -- unread header: keep the lazy answer
+ _lt_scan = None
+ if _lt_scan:
+ _lt_arch = _lt_arch or _lt_scan[0]
+ _resident = sum(
+ int(v) for k, v in (_lt_scan[1] or {}).items() if _is_per_layer_embedding(k)
+ )
+ lookup_table_bytes = min(main_bytes, max(lookup_table_bytes, _resident))
+
+ def _lt_override_moves_table() -> bool:
+ if _lt_arch in _lt_backend._host_forced_table_archs():
+ return False # this build keeps the table on the host whatever -ot says
+ try:
+ scan = _lt_backend._gguf_tensor_scan(gguf_path)
+ except Exception: # noqa: BLE001 -- unread header: the canonical name alone
+ scan = None
+ names = [k for k in ((scan[1] if scan else None) or {}) if _is_per_layer_embedding(k)]
+ return any(
+ _override_places_tensor_on_device(name, _lt_extras, _lt_env)
+ for name in names or ["per_layer_token_embd.weight"]
+ )
+
+ _lt_device_bytes = 0
+ if lookup_table_bytes:
+ if not lookup_table_on_disk and _lt_override_moves_table():
+ _lt_device_bytes, lookup_table_bytes = lookup_table_bytes, 0
+ if lookup_table_on_disk:
+ from core.inference.llama_cpp import (
+ _HOST_RAM_HEADROOM_MIB,
+ _LAZY_MODE_SPELLINGS,
+ _LAZY_RESIDENT_MARGIN_FRAC,
+ _extra_args_set_any_flag,
+ _kv_offload_from_args,
+ _device_selection_is_cpu,
+ _pass_through_overrides_placement,
+ _pass_through_reprices_cache,
+ _prompt_cache_host_bytes,
+ _sidecar_adapter_bytes,
+ _sidecar_adapter_paths,
+ )
+ from core.inference.llama_server_args import fit_target_margin_in, resolve_requested_ctx
+
+ _lt_flags, _lt_env_name = _LAZY_MODE_SPELLINGS.get(_lt_flag, (frozenset({_lt_flag}), ""))
+ _lt_user = bool(
+ (_lt_env_name and str(_lt_env.get(_lt_env_name) or "").strip())
+ or _extra_args_set_any_flag(_lt_extras, _lt_flags)
+ )
+ _lt_avail = LlamaCppBackend._available_system_memory_mib()
+ _lt_device_share = max(0, main_bytes - lookup_table_bytes)
+ _lt_forced_host = int(_lt_device_share * max(0.0, 1.0 - gpu_fraction))
+ _lt_table_host = lookup_table_bytes + int(lookup_table_bytes * _LAZY_RESIDENT_MARGIN_FRAC)
+
+ def _lt_resident_fits() -> bool:
+ if not _lt_avail or not runtime.kv_estimable:
+ return False # unsized: the launch keeps llama.cpp's default too
+ if _sidecar_adapter_bytes(_lt_extras) is None:
+ return False
+ # A pass-through -c 0 or larger --parallel sizes a cache runtime did not price.
+ if _pass_through_reprices_cache(_lt_extras, n_parallel):
+ return False
+ # A CPU child keeps llama.cpp's default; Manual Auto layers without a context leave it
+ # to --fit.
+ if _lt_backend._child_runs_on_cpu(
+ _lt_backend._find_llama_server_binary(),
+ _lt_extras,
+ _lt_env,
+ cpu_forced = bool(_gpu_placement_paravirtual()),
+ tensor_parallel = bool(tensor_parallel),
+ ):
+ return False
+ _lt_manual = gpu_memory_mode == "manual"
+ if (
+ _lt_manual
+ and (gpu_layers if gpu_layers is not None else -1) < 0
+ and resolve_requested_ctx(_lt_extras, n_ctx) <= 0
+ ):
+ return False
+ prompt = _prompt_cache_host_bytes(
+ None,
+ _lt_extras,
+ _lt_env,
+ supports_cache_ram = bool(
+ (_lt_backend.probe_server_capabilities() or {}).get("supports_cache_ram")
+ ),
+ )
+ if prompt is None:
+ return False
+ host_only = _lt_forced_host + _lt_table_host + prompt
+ if not _kv_offload_from_args(_lt_extras, env = _lt_env):
+ host_only += runtime.kv_bytes
+ if not drafter_on_gpu:
+ host_only += host_drafter_bytes + drafter_runtime_bytes
+ _lt_mm = _charged_projector_bytes(config, extras, disable_vision)
+ _lt_mm = _lt_mm if _lt_mm and _lt_mm <= companion_bytes else 0
+ _lt_mm_runtime = int(_lt_mm * (LlamaCppBackend._MMPROJ_VRAM_SAFETY - 1.0))
+ if _lt_mm and _resolved_mmproj_offload(llama_extra_args) is False:
+ host_only += _lt_mm + _lt_mm_runtime
+ footprint = (
+ _lt_device_share
+ + companion_bytes
+ + host_drafter_bytes
+ + _lt_mm_runtime
+ + adapter_bytes
+ + runtime.kv_bytes
+ + runtime.compute_bytes
+ + drafter_runtime_bytes
+ + _lt_table_host
+ + prompt
+ )
+ from utils.hardware import is_apple_silicon
+
+ if is_apple_silicon():
+ if _sidecar_adapter_paths(_lt_extras):
+ return False
+ budget = _lt_backend._apple_metal_memory_budget_bytes()
+ if budget <= 0:
+ return False
+ # Metal refuses past the wired limit: None is unknown, 0 used up.
+ wired = _lt_backend._apple_metal_wired_headroom_bytes()
+ if wired is not None:
+ budget = min(budget, wired)
+ if budget <= 0:
+ return False
+ return footprint <= min(
+ budget, max(0, _lt_avail - _HOST_RAM_HEADROOM_MIB) * 1024 * 1024
+ )
+ try:
+ with _without_torch_gpu_probe():
+ rows = list(
+ _lt_backend._get_gpu_memory(
+ _lt_backend._find_llama_server_binary(), for_llama_server = True
+ )
+ or ()
+ )
+ except Exception: # noqa: BLE001 -- unread cards: the whole load in RAM
+ rows = []
+ pick = sorted(int(i) for i in selected_gpu_ids) if selected_gpu_ids else None
+ _lt_vulkan = LlamaCppBackend._is_vulkan_backend()
+ # Manual mode plans no cards at launch (gpus = []): the whole load is RAM.
+ if (
+ not rows
+ or _lt_manual
+ or _pass_through_overrides_placement(
+ _lt_extras,
+ _lt_env,
+ len(pick) if pick is not None else len(rows),
+ is_vulkan_backend = _lt_vulkan,
+ gpu_indices = pick,
+ )
+ ):
+ rows, pick = [], None
+ _lt_rows = [(idx, free) for idx, free, _total in rows]
+ return bool(
+ _lt_backend._fits_without_paging(
+ footprint,
+ _lt_rows,
+ gpu_indices = pick,
+ # Vulkan reports total 0 for an iGPU; ROCm APUs and integrated CUDA SoCs pool
+ # host RAM.
+ shared_gpu_ids = _lt_backend._unified_pool_gpu_ids(
+ _lt_rows,
+ gpu_indices = pick,
+ shared_gpu_ids = (
+ {idx for idx, _free, total in rows if total <= 0}
+ if _lt_vulkan
+ else set()
+ ),
+ is_vulkan_backend = _lt_vulkan,
+ ),
+ host_only_bytes = host_only,
+ vram_margin_mib = max(1024.0, fit_target_margin_in(_lt_extras, _lt_env) or 0.0),
+ avail_mib = _lt_avail,
+ )
+ )
+
+ if not _lt_user and not _lt_override_moves_table() and _lt_resident_fits():
+ lookup_table_on_disk = False
+ if lookup_table_on_disk:
+ weights_bytes = max(0, weights_bytes - lookup_table_bytes)
# Projector is the only charged companion that can sit off-GPU (--no-mmproj-offload).
host_companion_bytes = 0
# The projector costs more than its file: the encoder's buffers run about 1.3x it,
@@ -14078,7 +14298,8 @@ def _gguf_memory_breakdown(
if _resolved_mmproj_offload(llama_extra_args) is False:
host_companion_bytes = min(companion_bytes, mmproj_bytes)
gpu_weights = (
- int(main_bytes * gpu_fraction)
+ int((main_bytes - lookup_table_bytes - _lt_device_bytes) * gpu_fraction)
+ + _lt_device_bytes
+ int(adapter_bytes * gpu_fraction)
+ companion_bytes
- host_companion_bytes
@@ -14144,6 +14365,8 @@ def _gguf_memory_breakdown(
layer_count = layer_count,
gpu_layers = gpu_layers if gpu_memory_mode == "manual" else None,
kv_checkpoint_bytes = runtime.kv_checkpoint_bytes,
+ lookup_table_bytes = lookup_table_bytes,
+ lookup_table_on_disk = lookup_table_on_disk,
)
@@ -20357,6 +20580,7 @@ async def estimate_memory(
tensor_split_possible = _tensor_split_possible(request.selected_gpu_ids or None),
# A pin names the cards; the launch strips conflicting device flags.
device_pin_governs = bool(request.selected_gpu_ids),
+ selected_gpu_ids = pinned_gpu_ids,
)
breakdown = price(n_ctx = request.n_ctx or 0)
if breakdown is None:
diff --git a/studio/backend/routes/models.py b/studio/backend/routes/models.py
index 95b84eff37b..eb738f8daaf 100644
--- a/studio/backend/routes/models.py
+++ b/studio/backend/routes/models.py
@@ -4065,6 +4065,7 @@ async def get_kv_cache_estimate(
null = {
"kv_bytes": None,
"weights_bytes": None,
+ "lookup_table_bytes": None,
"native_context": None,
"spec_bytes": None,
"n_ctx": None,
@@ -4633,6 +4634,7 @@ async def get_gguf_variants(
download_size_bytes = int(
getattr(v, "download_size_bytes", v.size_bytes) or v.size_bytes
),
+ lookup_table_bytes = int(getattr(v, "lookup_table_bytes", 0) or 0),
pending_drafter_filename = getattr(v, "pending_drafter_filename", None),
pending_drafter_size_bytes = int(
getattr(v, "pending_drafter_size_bytes", 0) or 0
diff --git a/studio/backend/tests/test_compute_buffer.py b/studio/backend/tests/test_compute_buffer.py
index bdb788a5a4e..a2a7703fcf1 100644
--- a/studio/backend/tests/test_compute_buffer.py
+++ b/studio/backend/tests/test_compute_buffer.py
@@ -816,7 +816,8 @@ class TestPipelineParallelPredicate:
def test_layer_path_scrubs_a_non_layer_split_mode_env(self):
import inspect
- compact = "".join(inspect.getsource(LlamaCppBackend.load_model).split())
+ assert "self._reconcile_split_mode_env(env" in inspect.getsource(LlamaCppBackend.load_model)
+ compact = "".join(inspect.getsource(LlamaCppBackend._reconcile_split_mode_env).split())
assert 'if_inherited_smand_inherited_sm!="layer":' in compact
assert 'env.pop("LLAMA_ARG_SPLIT_MODE",None)' in compact
diff --git a/studio/backend/tests/test_empty_gpu_probe_reason.py b/studio/backend/tests/test_empty_gpu_probe_reason.py
index a51fe535456..ac0cc22b461 100644
--- a/studio/backend/tests/test_empty_gpu_probe_reason.py
+++ b/studio/backend/tests/test_empty_gpu_probe_reason.py
@@ -265,7 +265,7 @@ class TestItDoesNotAssertTheLaunchOutcome:
src = inspect.getsource(mod.LlamaCppBackend.load_model)
warn = src.find("could not enumerate any GPU")
- restore = src.find("gpu_indices = _picked_order")
+ restore = src.find("self._picker_ordered_gpu_indices(")
assert warn != -1 and restore != -1
assert warn < restore, "if the pin moved above the warning, re-derive this guard"
diff --git a/studio/backend/tests/test_llama_cpp_placement.py b/studio/backend/tests/test_llama_cpp_placement.py
index b239f23913e..267c073c438 100644
--- a/studio/backend/tests/test_llama_cpp_placement.py
+++ b/studio/backend/tests/test_llama_cpp_placement.py
@@ -162,7 +162,7 @@ def _launch(
"Process",
(),
{
- "pid": 123,
+ "pid": 2**31 - 1,
"stdout": (),
"poll": lambda self: None,
"terminate": lambda self: None,
@@ -1189,7 +1189,7 @@ def _launch_with_failing_drafter(
"Process",
(),
{
- "pid": 123,
+ "pid": 2**31 - 1,
"stdout": (),
"poll": lambda self: None,
"terminate": lambda self: None,
@@ -1297,7 +1297,7 @@ def test_a_drafter_that_fails_to_start_gives_its_slots_back(tmp_path):
"Process",
(),
{
- "pid": 123,
+ "pid": 2**31 - 1,
"stdout": (),
"poll": lambda self: None,
"terminate": lambda self: None,
@@ -3267,7 +3267,7 @@ def _launch_with_text_only_fallback(backend, gguf, **load_kwargs):
"Process",
(),
{
- "pid": 123,
+ "pid": 2**31 - 1,
"stdout": (),
"poll": lambda self: None,
"terminate": lambda self: None,
@@ -3454,7 +3454,7 @@ def _launch_with_vulkan_cpu_replay(
"Process",
(),
{
- "pid": 123,
+ "pid": 2**31 - 1,
"stdout": (),
"returncode": rc,
"poll": lambda self: rc,
@@ -5039,6 +5039,7 @@ class _RamStub:
return False
_fits_without_paging = LlamaCppBackend._fits_without_paging
+ _unified_pool_gpu_ids = LlamaCppBackend._unified_pool_gpu_ids
_FIT_LOAD_MODE = LlamaCppBackend._FIT_LOAD_MODE
@@ -5303,15 +5304,33 @@ def test_an_igpu_only_child_gets_no_auto_lazy_discount(tmp_path, _moe_cache_host
(False, [0, 1], set(), {0}, set(), False),
(False, [0], set(), set(), {0}, True),
(False, None, set(), set(), {0, 1}, True),
+ (True, [0, 1], {1}, set(), set(), True),
+ (True, [0, 1], set(), set(), set(), False),
+ (True, [0, 1], {0, 1}, set(), set(), True),
+ (True, None, set(), set(), set(), False),
+ (True, None, {0, 1}, set(), set(), True),
+ (False, [0, 1], set(), set(), {1}, False),
+ (False, [0, 1], set(), set(), set(), False),
+ (False, None, set(), set(), set(), False),
+ (False, None, set(), set(), {1}, False),
],
ids = [
"discrete",
"vulkan_igpu",
- "vulkan_mixed",
+ "vulkan_mixed_default",
"rocm_apu",
"rocm_mixed",
"cuda_soc",
"cuda_soc_unpinned",
+ "vulkan_mixed_explicit",
+ "vulkan_all_discrete_explicit",
+ "vulkan_all_integrated_explicit",
+ "vulkan_discrete_default",
+ "vulkan_all_integrated_default",
+ "cuda_mixed_mask",
+ "cuda_all_discrete_explicit",
+ "cuda_discrete_default",
+ "cuda_mixed_default",
],
)
def test_lazy_auto_resolves_off_only_where_llama_cpp_does(
@@ -5348,6 +5367,270 @@ def test_lazy_auto_resolves_off_only_where_llama_cpp_does(
assert (auto, on) == ((0 if expected else _QWEN38_PLE), _QWEN38_PLE)
+def _mixed_cuda_soc_host(monkeypatch):
+ """A discrete card (0) beside an integrated SoC GPU (1) on a CUDA build."""
+ for name, value in (
+ ("_rocm_unified_memory_gpu_ids", set()),
+ ("_integrated_cuda_gpu_ids", {1}),
+ ("_integrated_cuda_probe_is_free", True),
+ ("_resolve_visible_physical_ids", [0, 1]),
+ ):
+ monkeypatch.setattr(LlamaCppBackend, name, staticmethod(lambda value = value: value))
+
+
+def _mixed_cuda_launch(
+ tmp_path,
+ ram_gib = 12,
+ memory = ((0, 8_000, 16_000), (1, 1_000, 1_000)),
+ **kw,
+):
+ return _cache_launch(
+ tmp_path,
+ ram_gib = ram_gib,
+ lazy = {"per_layer_token_embd.weight": 12 * _GIB},
+ arch = "qwen4exp",
+ caps = dict(_CACHE_CAPS_ON, supports_moe_cache_auto = False),
+ memory = list(memory),
+ **kw,
+ )
+
+
+def _apu_qwen38(
+ tmp_path,
+ monkeypatch,
+ *,
+ gguf_gb,
+ avail_gib,
+ memory,
+ apu_ids = (0,),
+):
+ backend, gguf = _apu_backend(
+ tmp_path, gguf_gb = gguf_gb, avail_mib = avail_gib * 1024, monkeypatch = monkeypatch
+ )
+ backend._get_gpu_memory = lambda *_a, **_kw: list(memory)
+ backend._get_gpu_free_memory = lambda *_a, **_kw: [(i, f) for i, f, _t in memory]
+ monkeypatch.setattr(
+ LlamaCppBackend, "_rocm_unified_memory_gpu_ids", staticmethod(lambda: set(apu_ids))
+ )
+ backend._gguf_tensor_scan = lambda _path: (
+ "qwen4exp",
+ {"per_layer_token_embd.weight": _QWEN38_PLE},
+ 0,
+ )
+ caps = dict(LlamaCppBackend.probe_server_capabilities.__func__(LlamaCppBackend, None))
+ caps.update(_CACHE_CAPS_ON)
+ backend.probe_server_capabilities = lambda _binary = None: caps
+ return backend, gguf
+
+
+ "load_kwargs",
+ [dict(), dict(gpu_ids = [0, 1])],
+ ids = ["default_selection", "picker_mask"],
+)
+def test_a_mixed_cuda_host_keeps_the_lazy_discount_through_the_preflight(
+ tmp_path, _moe_cache_host, monkeypatch, load_kwargs
+):
+ """No --device list."""
+ _mixed_cuda_soc_host(monkeypatch)
+ backend, cmd = _mixed_cuda_launch(tmp_path, host_guard = True, **load_kwargs)
+ assert _load_mode(cmd) == "none", cmd
+ assert backend.last_load_warning is None, backend.last_load_warning
+
+
+def test_an_explicit_device_list_naming_an_igpu_turns_lazy_auto_off(monkeypatch):
+ """A real --device list is used as given, iGPU included, so ANY integrated device turns auto off."""
+ _mixed_cuda_soc_host(monkeypatch)
+ off = LlamaCppBackend()._lazy_auto_resolves_off
+ kw = dict(
+ gpu_indices = [0, 1],
+ detected_gpus = [(0, 8_000), (1, 1_000)],
+ shared_gpu_ids = set(),
+ is_vulkan_backend = False,
+ )
+ assert off(**kw) is False
+ for extra_args, env, want in (
+ (["--device", "CUDA0,CUDA1"], {}, True),
+ ([], {"LLAMA_ARG_DEVICE": "CUDA0,CUDA1"}, True),
+ (["-dev", "CUDA1"], {}, True),
+ (["--device", "CUDA0"], {}, False),
+ (["--device", "CUDA0"], {"CUDA_VISIBLE_DEVICES": "1"}, True),
+ (["--device", "none"], {}, False),
+ ):
+ assert off(**kw, extra_args = extra_args, env = env) is want, (extra_args, env)
+ vk = dict(kw, is_vulkan_backend = True, shared_gpu_ids = {1})
+ assert off(**vk) is True
+ assert off(**dict(vk, gpu_indices = None)) is False
+ assert off(**vk, device_list = False) is False
+ dev = ["--device", "Vulkan0,Vulkan1"]
+ assert off(**vk, device_list = False, extra_args = dev, env = {}) is True
+
+
+def test_an_explicit_pick_reaches_every_lazy_decision(tmp_path, _moe_cache_host, monkeypatch):
+ """gpu_ids becomes the child's selection, so every lazy decision of the fit must see it."""
+ _mixed_cuda_soc_host(monkeypatch)
+ seen = []
+ real = LlamaCppBackend._lazy_auto_resolves_off
+
+ def spy(self, **kw):
+ seen.append(kw.get("gpu_indices"))
+ return real(self, **kw)
+
+ monkeypatch.setattr(LlamaCppBackend, "_lazy_auto_resolves_off", spy)
+ _mixed_cuda_launch(tmp_path, gpu_ids = [0, 1])
+ assert seen, "no lazy decision ran"
+ assert all(sel is not None and set(sel) <= {0, 1} for sel in seen), seen
+
+
+def test_split_mode_none_checks_only_the_main_device(monkeypatch):
+ """-sm none keeps only devices[main_gpu] of the model list (src/llama.cpp llama_prepare_model_devices)."""
+ _mixed_cuda_soc_host(monkeypatch)
+ off = LlamaCppBackend()._lazy_auto_resolves_off
+ vk = dict(
+ gpu_indices = None,
+ detected_gpus = [(0, 8_000), (1, 1_000)],
+ shared_gpu_ids = {1},
+ is_vulkan_backend = True,
+ env = {},
+ )
+ dev = ["--device", "Vulkan0,Vulkan1"]
+ assert off(**vk, extra_args = dev) is True
+ assert off(**vk, extra_args = [*dev, "-sm", "none"]) is False
+ assert off(**vk, extra_args = [*dev, "-sm", "none", "-mg", "1"]) is True
+ assert off(**vk, extra_args = [*dev, "--split-mode", "none", "-mg", "-1"]) is False
+ pick = dict(vk, extra_args = ["-sm", "none"])
+ assert off(**dict(pick, gpu_indices = [0, 1])) is False
+ assert off(**dict(pick, gpu_indices = [1, 0])) is True
+ cuda = dict(
+ vk, is_vulkan_backend = False, shared_gpu_ids = set(), env = {"LLAMA_ARG_SPLIT_MODE": "none"}
+ )
+ assert off(**cuda, extra_args = ["--device", "CUDA1,CUDA0"]) is True
+ assert off(**cuda, extra_args = ["--device", "CUDA0,CUDA1"]) is False
+
+
+def test_the_apu_guard_reads_the_childs_device_list(tmp_path, monkeypatch):
+ """APU 0 beside a discrete card 1, the user naming the APU with --device ROCm0."""
+ monkeypatch.setattr(
+ LlamaCppBackend, "_resolve_visible_physical_ids", staticmethod(lambda: [0, 1])
+ )
+ backend, gguf = _apu_qwen38(
+ tmp_path,
+ monkeypatch,
+ gguf_gb = 64.6,
+ avail_gib = 46,
+ memory = [(0, 60_000, 60_000), (1, 8_000, 8_000)],
+ )
+
+ cmd = _launch(backend, gguf, extra_args = ["--load-mode", "none", "--device", "ROCm0"])["cmd"]
+
+ assert "unified-memory APU" in (backend.last_load_warning or ""), backend.last_load_warning
+ assert not _unmapped_tokens(cmd), cmd
+
+
+def test_the_arch_crash_retry_prices_ram_against_its_own_mask():
+ """The retry's RAM checks run after its mask / device / split rewrites and see the retry argv and env."""
+ import inspect
+
+ src = inspect.getsource(LlamaCppBackend.load_model)
+ retry = src[src.index("_retry_wants_unified = self._amd_apu_wants_unified_memory(") :]
+ retry = retry[: retry.index('label = "-archfallback"')]
+ host = retry.index("_retry_host_msg = self._launch_host_shortfall_message(")
+ apu = retry.index("_retry_apu_msg = self._apu_ram_shortfall_message(")
+ for rewrite in (
+ "self._emit_child_gpu_visibility(",
+ "_widen_pin_ids_for_companion_devices(",
+ "self._clear_split_placement_env(env)",
+ "self._without_tensor_split(cmd)",
+ ):
+ assert retry.index(rewrite) < min(host, apu), rewrite
+ lazy = retry[retry.rindex("self._lazy_auto_resolves_off(", 0, apu) : apu]
+ assert "extra_args = cmd" in lazy and "env = env" in lazy, lazy
+ kill = retry.index("self._kill_process()")
+ assert retry.index("_retry_host_avail_mib = self._available_system_memory_mib()") < kill
+ assert "avail_mib = _retry_host_avail_mib" in retry[host:]
+
+
+def test_tensor_mode_checks_every_visible_device(monkeypatch):
+ """--split-mode tensor builds one Meta device from every visible GPU, mmap only if all members support it (ggml-backend-meta.cpp)."""
+ _mixed_cuda_soc_host(monkeypatch)
+ off = LlamaCppBackend()._lazy_auto_resolves_off
+ cuda = dict(
+ gpu_indices = None,
+ detected_gpus = [(0, 8_000), (1, 1_000)],
+ shared_gpu_ids = set(),
+ is_vulkan_backend = False,
+ )
+ assert off(**cuda) is False
+ assert off(**cuda, tensor_parallel = True) is True
+ assert off(**cuda, extra_args = ["-sm", "tensor"], env = {}) is True
+ assert off(**cuda, extra_args = [], env = {"LLAMA_ARG_SPLIT_MODE": "tensor"}) is True
+ assert off(**cuda, extra_args = ["-sm", "tensor"], env = {}, tensor_parallel = False) is False
+ assert off(**dict(cuda, gpu_indices = [0]), tensor_parallel = True) is False
+ vk = dict(cuda, is_vulkan_backend = True, shared_gpu_ids = {1})
+ assert off(**vk, device_list = False) is False
+ assert off(**vk, device_list = False, extra_args = ["--split-mode", "tensor"], env = {}) is True
+
+
+def test_a_mixed_tensor_launch_prices_the_lazy_table(tmp_path, _moe_cache_host, monkeypatch):
+ """Manual tensor parallelism over a discrete card and an integrated SoC, asking for --load-mode none."""
+ _mixed_cuda_soc_host(monkeypatch)
+ monkeypatch.setattr(
+ LlamaCppBackend, "_effective_gpu_count", staticmethod(lambda gpu_indices = None: 2)
+ )
+ kwargs = dict(
+ ram_gib = 6,
+ memory = [(0, 8_000, 16_000), (1, 8_000, 8_000)],
+ host_guard = True,
+ gpu_memory_mode = "manual",
+ gpu_layers = 20,
+ load_mode = "none",
+ )
+ backend, layer = _mixed_cuda_launch(tmp_path, **kwargs)
+ assert _load_mode(layer) == "none", layer
+ assert backend.last_load_warning is None, backend.last_load_warning
+ backend, cmd = _mixed_cuda_launch(tmp_path, tensor_parallel = True, **kwargs)
+ assert cmd[cmd.index("--split-mode") + 1] == "tensor", cmd
+ assert _load_mode(cmd) is None, cmd
+ assert backend.last_load_warning, cmd
+
+
+def test_a_device_list_still_sees_an_auto_filtered_vulkan_igpu(monkeypatch):
+ """Automatic Vulkan placement drops iGPUs from its candidates, so they leave _shared_gpu_ids."""
+ import inspect
+
+ off = LlamaCppBackend()._lazy_auto_resolves_off
+ kw = dict(
+ gpu_indices = None,
+ detected_gpus = [(0, 8_000)],
+ shared_gpu_ids = set(),
+ is_vulkan_backend = True,
+ extra_args = ["--device", "Vulkan0,Vulkan1"],
+ env = {},
+ )
+ assert off(**kw) is False
+ assert off(**kw, integrated_gpu_ids = {1}) is True
+ assert off(**dict(kw, extra_args = ["--device", "Vulkan0"]), integrated_gpu_ids = {1}) is False
+ src = inspect.getsource(LlamaCppBackend.load_model)
+ assert src.index("_vulkan_igpu_ids = (") < src.index(
+ "_gpu_mem = self._vulkan_auto_gpu_memory(_gpu_mem)"
+ )
+ assert src.count("integrated_gpu_ids = _vulkan_igpu_ids") == 5
+
+
+def test_the_oversized_readings_share_their_preflights_ram_snapshot():
+ """Under the opt-out the oversized reading re-prices the shortfall with its neighbouring preflight's RAM figure."""
+ import inspect
+
+ src = inspect.getsource(LlamaCppBackend.load_model)
+ for name, snapshot in (
+ ("def _launch_is_oversized()", "avail_mib = _preflight_avail_mib"),
+ ("def _retry_is_oversized()", "avail_mib = _retry_host_avail_mib"),
+ ):
+ body = src[src.index(name) :]
+ body = body[: body.index("self._page_an_oversized_unmapped_load(")]
+ assert snapshot in body, (name, body)
+
+
@pytest.mark.parametrize("lazy_args, warned", [(["-lzm", "on"], False), ([], True)])
def test_the_apu_guard_prices_only_resident_weights(tmp_path, monkeypatch, lazy_args, warned):
"""A 64.6 GiB model with a 26.82 GiB table on an APU with 46 GiB of RAM. "-lzm on"
@@ -5372,6 +5655,17 @@ def test_the_apu_guard_prices_only_resident_weights(tmp_path, monkeypatch, lazy_
assert bool(_unmapped_tokens(cmd)) is not warned, cmd
[email protected]("memory", [[(0, 60_000, 60_000)], []], ids = ["probed", "failed_probe"])
+def test_a_failed_gpu_probe_keeps_the_apu_guard_pricing_the_table(tmp_path, monkeypatch, memory):
+ """An empty inventory on a ROCm APU still launches a HIP child, where auto is off, so the guard charges the table."""
+ backend, gguf = _apu_qwen38(tmp_path, monkeypatch, gguf_gb = 73.45, avail_gib = 58, memory = memory)
+
+ cmd = _launch(backend, gguf, extra_args = ["--load-mode", "none"])["cmd"]
+
+ assert "unified-memory APU" in (backend.last_load_warning or "")
+ assert not _unmapped_tokens(cmd), cmd
+
+
@pytest.mark.parametrize(
"flag, extra_args, env, expected",
[
diff --git a/studio/backend/tests/test_load_mode_fit_matrix.py b/studio/backend/tests/test_load_mode_fit_matrix.py
index 084550cb73e..039c665a32c 100644
--- a/studio/backend/tests/test_load_mode_fit_matrix.py
+++ b/studio/backend/tests/test_load_mode_fit_matrix.py
@@ -54,6 +54,7 @@ class _Stub:
return self._is_apu
_fits_without_paging = LlamaCppBackend._fits_without_paging
+ _unified_pool_gpu_ids = LlamaCppBackend._unified_pool_gpu_ids
_FIT_LOAD_MODE = LlamaCppBackend._FIT_LOAD_MODE
diff --git a/studio/backend/tests/test_lookup_table_fit.py b/studio/backend/tests/test_lookup_table_fit.py
new file mode 100644
index 00000000000..275fc67f2a5
--- /dev/null
+++ b/studio/backend/tests/test_lookup_table_fit.py
@@ -0,0 +1,1200 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
+
+"""Lookup-only embedding tables (per-layer embeddings, the DeepSeek-V4.1 Engram table) in the launch fit, the offload layout and the planner."""
+
+from __future__ import annotations
+
+import os
+import subprocess
+import sys
+from pathlib import Path
+
+import pytest
+
+_BACKEND_DIR = str(Path(__file__).resolve().parent.parent)
+_TESTS_DIR = str(Path(__file__).resolve().parent)
+for _d in (_BACKEND_DIR, _TESTS_DIR):
+ if _d not in sys.path:
+ sys.path.insert(0, _d)
+
+import test_llama_cpp_placement as placement # noqa: E402, I001
+from test_gpu_arch_gate_os_matrix_7624 import probe_env # noqa: F401, I001 -- fixture by name
+from test_llama_cpp_placement import ( # noqa: F401, I001 -- fixtures by name
+ _cache_launch,
+ _discrete_linux_host,
+ _moe_cache_host,
+)
+from test_offload_planner import _StubReader, _StubTensor, _shard_fields, _shard_tensors
+
+from core.inference import llama_cpp as lc # noqa: E402
+from core.inference.llama_cpp import LlamaCppBackend # noqa: E402
+from core.inference.offload_layout import ( # noqa: E402
+ BlockLayout,
+ ModelLayout,
+ _layout_from_reader,
+)
+from core.inference.offload_planner import PlanOptions, _finish # noqa: E402
+
+GIB = 1024**3
+PLE = "per_layer_token_embd.weight"
+OT_TABLE = ["-ot", "per_layer_token_embd=CUDA0"]
+
+DSV41_TOTAL = int(216.98 * GIB)
+DSV41_ENGRAM = int(103.00 * GIB)
+DSV41_TOKEN_EMBD = int(0.51 * GIB)
+DSV41_OUTPUT = int(0.51 * GIB)
+
+FLASH_NEXT_TOTAL = int(73.45 * GIB)
+FLASH_NEXT_PLE = int(26.82 * GIB)
+FLASH_NEXT_TOKEN_EMBD = int(0.41 * GIB)
+FLASH_NEXT_OUTPUT = int(0.33 * GIB)
+
+GEMMA3N_TOTAL = int(4.23 * GIB)
+GEMMA3N_PLE = int(1.64 * GIB)
+GEMMA3N_TOKEN_EMBD = int(0.28 * GIB)
+
+_CAPS = {"supports_lazy_mode": True}
+_FLASH_NAMED = {
+ PLE: FLASH_NEXT_PLE,
+ "token_embd.weight": FLASH_NEXT_TOKEN_EMBD,
+ "output.weight": FLASH_NEXT_OUTPUT,
+}
+_DSV41_NAMED = {
+ PLE: DSV41_ENGRAM,
+ "token_embd.weight": DSV41_TOKEN_EMBD,
+ "output.weight": DSV41_OUTPUT,
+}
+_GEMMA3N_NAMED = {PLE: GEMMA3N_PLE, "token_embd.weight": GEMMA3N_TOKEN_EMBD}
+_INHERITED_CPU_SPLIT = {"LLAMA_ARG_SPLIT_MODE": "none", "LLAMA_ARG_MAIN_GPU": "-1"}
+_RPC_LIST = ["--rpc", "10.0.0.2:50052", "--device", "Vulkan0,RPC0"]
+_ONE = _RPC_LIST + ["--split-mode", "none"]
+
+
+def _scan_backend(arch, named):
+ backend = LlamaCppBackend()
+ backend._gguf_tensor_scan = lambda _path: (arch, dict(named), 0)
+ return backend
+
+
+def _arg(cmd, flag):
+ return cmd[cmd.index(flag) + 1]
+
+
+def _lazy_values(cmd):
+ return [cmd[i + 1] for i, tok in enumerate(cmd) if tok in ("--lazy-mode", "-lzm")]
+
+
+def _setenv(monkeypatch, env):
+ for name, value in env.items():
+ monkeypatch.setenv(name, value)
+
+
+def _full_caps():
+ caps = dict(LlamaCppBackend.probe_server_capabilities.__func__(LlamaCppBackend, None))
+ caps.update(placement._CACHE_CAPS_ON)
+ return lambda _binary = None: caps
+
+
+def _flash_next_launch(
+ tmp_path,
+ *,
+ ram_gib = 128,
+ memory = ((0, 23_800, 24_576),),
+ **kwargs,
+):
+ return _cache_launch(
+ tmp_path,
+ size_gib = FLASH_NEXT_TOTAL / GIB,
+ memory = list(memory),
+ ram_gib = ram_gib,
+ expert_gib = 40,
+ lazy = _FLASH_NAMED,
+ arch = "qwen4exp",
+ **kwargs,
+ )
+
+
+def _flash_next_on_two_cards(
+ tmp_path,
+ extra_args = (),
+ **kwargs,
+):
+ """Flash-Next (73.45 GiB, 26.82 GiB table) on 2x32 GiB."""
+ return _cache_launch(
+ tmp_path,
+ moe = False,
+ size_gib = FLASH_NEXT_TOTAL / GIB,
+ memory = [(0, 32_000, 32_768), (1, 32_000, 32_768)],
+ ram_gib = 128,
+ lazy = _FLASH_NAMED,
+ arch = "qwen4exp",
+ extra_args = list(extra_args),
+ **kwargs,
+ )
+
+
+def _gemma3n_launch(tmp_path, *, memory, ram_gib, **kwargs):
+ return _cache_launch(
+ tmp_path,
+ moe = False,
+ size_gib = GEMMA3N_TOTAL / GIB,
+ memory = memory,
+ lazy = _GEMMA3N_NAMED,
+ arch = "gemma3n",
+ ram_gib = ram_gib,
+ **kwargs,
+ )
+
+
+def _spy(monkeypatch, name, record):
+ """Wrap a LlamaCppBackend method; ``record(answer, args, kwargs)`` appends what it saw."""
+ seen = []
+ real = getattr(LlamaCppBackend, name)
+
+ def spy(self, *a, **k):
+ answer = real(self, *a, **k)
+ entry = record(answer, a, k)
+ if entry is not None:
+ seen.append(entry)
+ return answer
+
+ monkeypatch.setattr(LlamaCppBackend, name, spy)
+ return seen
+
+
+def _spy_fit_load_mode(monkeypatch):
+ return _spy(
+ monkeypatch,
+ "_fit_derived_load_mode",
+ lambda _ans, _a, k: k if k.get("announce", True) else None,
+ )
+
+
+def _apple_backend(arch, named, avail_gib):
+ backend = _scan_backend(arch, named)
+ backend._available_system_memory_mib = lambda: int(avail_gib * 1024)
+ backend._can_estimate_kv = lambda: True
+ return backend
+
+
+def _apple_discount(
+ backend,
+ probe = 0,
+ *,
+ model_size_fit = FLASH_NEXT_TOTAL,
+ budget_gib = 96,
+ caps = None,
+ extra_args = (),
+ env = None,
+ **kwargs,
+):
+ return backend._apple_lookup_table_discount(
+ probe,
+ "/m.gguf",
+ caps = dict(caps if caps is not None else dict(_CAPS, supports_cache_ram = True)),
+ extra_args = list(extra_args),
+ model_size_fit = model_size_fit,
+ need_at_ctx = kwargs.pop("need_at_ctx", GIB),
+ budget_mib = int(budget_gib * 1024),
+ cache_ram = None,
+ env = env or {},
+ **kwargs,
+ )
+
+
+def _managed_flags(
+ backend,
+ extra_args = (),
+ resident_fits = lambda _t: True,
+ caps = _CAPS,
+ **kwargs,
+):
+ return backend._managed_lazy_mode_flags(
+ "/m.gguf",
+ caps = caps,
+ extra_args = list(extra_args),
+ env = {},
+ resident_fits = resident_fits,
+ **kwargs,
+ )
+
+
+ "extra_args, auto_off, expected",
+ [
+ ([], False, DSV41_ENGRAM),
+ ([], True, 0),
+ (["-lzm", "off"], False, 0),
+ (["-lzm", "on"], True, DSV41_ENGRAM),
+ ],
+ ids = ["auto", "igpu_auto", "off", "on_igpu"],
+)
+def test_the_deepseek41_engram_table_is_read_lazily(extra_args, auto_off, expected):
+ backend = _scan_backend("deepseek41", {PLE: DSV41_ENGRAM})
+ got = backend._lazy_read_host_bytes(
+ "/m.gguf", caps = _CAPS, extra_args = extra_args, env = {}, auto_resolves_off = auto_off
+ )
+ assert got == expected
+
+
+def test_the_dsv41_host_preflight_no_longer_charges_the_engram_table(monkeypatch):
+ """DSV4.1 UD-IQ1_S on a 24 GiB card with 64 GiB of RAM."""
+ backend = _scan_backend("deepseek41", {PLE: DSV41_ENGRAM})
+ backend._get_gguf_size_bytes = lambda _path: DSV41_TOTAL
+ backend.probe_server_capabilities = lambda _binary = None: dict(_CAPS)
+ backend._is_vulkan_backend = lambda _binary = None: False
+ monkeypatch.setattr(LlamaCppBackend, "_lazy_auto_resolves_off", lambda self, **_kw: False)
+ monkeypatch.delenv("UNSLOTH_ALLOW_HOST_OFFLOAD", raising = False)
+ seen = []
+ backend._host_offload_shortfall_message = (
+ lambda offload, _avail: seen.append(offload) or "short"
+ )
+ msg = backend._launch_host_shortfall_message(
+ ["/fake/llama-server", "-m", "/m.gguf"], [(0, 23_800)], env = {}, avail_mib = 64 * 1024
+ )
+ assert msg == "short"
+ assert seen == [DSV41_TOTAL - DSV41_ENGRAM - 23_800 * 1024 * 1024]
+ assert 90 * GIB < seen[0] < 91 * GIB
+
+
+def _gemma3n_layout(with_output = False):
+ tensors = _shard_tensors(range(4)) + [
+ _StubTensor("token_embd.weight", GEMMA3N_TOKEN_EMBD),
+ _StubTensor(PLE, GEMMA3N_PLE),
+ _StubTensor("output_norm.weight", 4096),
+ ]
+ if with_output:
+ tensors.append(_StubTensor("output.weight", GEMMA3N_TOKEN_EMBD))
+ return _layout_from_reader(_StubReader(_shard_fields(**{"llama.block_count": 4}), tensors))
+
+
+def test_the_per_layer_table_is_not_token_embd():
+ layout = _gemma3n_layout()
+ assert layout.complete
+ assert layout.token_embd_bytes == GEMMA3N_TOKEN_EMBD
+ assert layout.per_layer_embd_bytes == GEMMA3N_PLE
+
+
+def test_a_tied_file_duplicates_token_embd_but_not_the_per_layer_table():
+ """gemma-3n is tied."""
+ assert _gemma3n_layout().other_resident_bytes == GEMMA3N_TOKEN_EMBD + 4096
+ untied = _gemma3n_layout(with_output = True)
+ assert untied.other_resident_bytes == 4096
+ assert untied.lm_head_bytes == GEMMA3N_TOKEN_EMBD
+
+
+def test_a_lazy_table_does_not_deny_load_mode_none_to_spilled_experts():
+ """Flash-Next on 24 GiB + a 64 GB box with 48 GiB available."""
+ layout = ModelLayout(
+ arch = "qwen4exp",
+ n_layers = 40,
+ n_attention_layers = 10,
+ blocks = tuple(
+ BlockLayout(index = i, spillable_bytes = GIB, resident_bytes = int(0.14 * GIB))
+ for i in range(40)
+ ),
+ lm_head_bytes = FLASH_NEXT_OUTPUT,
+ token_embd_bytes = FLASH_NEXT_TOKEN_EMBD,
+ per_layer_embd_bytes = FLASH_NEXT_PLE,
+ tensor_bytes = FLASH_NEXT_TOTAL,
+ other_resident_bytes = int(0.29 * GIB),
+ kv_bytes_per_token_f16 = 40 * 1024,
+ n_ctx_train = 262144,
+ is_moe = True,
+ n_expert = 256,
+ n_expert_used = 8,
+ complete = True,
+ )
+ chosen = list(layout.blocks[:25])
+ lazy = _finish(
+ layout, PlanOptions(lazy_read_bytes = FLASH_NEXT_PLE), 8192, chosen, False, 48 * GIB
+ )
+ assert lazy.load_mode_none
+ assert lazy.host_bytes == FLASH_NEXT_TOKEN_EMBD + 25 * GIB
+ resident = _finish(layout, PlanOptions(), 8192, chosen, False, 48 * GIB)
+ assert not resident.load_mode_none
+ assert resident.host_bytes == FLASH_NEXT_TOKEN_EMBD + FLASH_NEXT_PLE + 25 * GIB
+
+
+def test_host_pinned_inputs_for_the_three_families():
+ pinned = lc._host_pinned_input_bytes
+ assert pinned(("gemma3n", _GEMMA3N_NAMED, 0)) == GEMMA3N_PLE
+ assert pinned(("qwen4exp", _FLASH_NAMED, 0)) == FLASH_NEXT_PLE + FLASH_NEXT_TOKEN_EMBD
+ assert pinned(("deepseek41", _DSV41_NAMED, 0)) == DSV41_ENGRAM + DSV41_TOKEN_EMBD
+ assert pinned(None) == 0
+
+
+def test_an_override_onto_a_device_keeps_a_resident_table_charged():
+ on_device, pinned = lc._override_places_tensor_on_device, lc._host_pinned_input_bytes
+ assert on_device(PLE, OT_TABLE)
+ assert not on_device(PLE, ["-ot", "per_layer=CPU"])
+ assert not on_device(PLE, ["-ot", "per_layer=CPU,embd=CUDA0"])
+ assert not on_device(PLE, ["-ot", "exps=CPU"])
+ assert on_device(PLE, [], {"LLAMA_ARG_OVERRIDE_TENSOR": "embd=CUDA0"})
+ assert pinned(("qwen4exp", {PLE: FLASH_NEXT_PLE}, 0), OT_TABLE, {}) == 0
+ # Upstream b11870 gathers the Engram rows in the graph, so -ot places it like any other table;
+ # a build that keeps it on the host whatever -ot says still discounts it.
+ engram = ("deepseek41", {PLE: DSV41_ENGRAM}, 0)
+ assert pinned(engram, OT_TABLE, {}) == 0
+ assert pinned(engram, OT_TABLE, {}, host_forced = {"deepseek41"}) == DSV41_ENGRAM
+
+
+def test_gemma3n_fits_a_small_card_once_its_tables_leave_the_vram_charge(tmp_path, _moe_cache_host):
+ """gemma-3n-E4B Q4_K_M (4.23 GiB)."""
+ _backend, cmd = _gemma3n_launch(tmp_path, memory = [(0, 9_000, 10_240)], ram_gib = 16)
+ assert (_arg(cmd, "--fit"), _arg(cmd, "-ngl")) == ("off", "-1"), cmd
+ assert _lazy_values(cmd) == [], cmd
+
+
+def test_an_override_moving_the_table_to_the_gpu_keeps_it_charged(tmp_path, _moe_cache_host):
+ _backend, cmd = _gemma3n_launch(
+ tmp_path, memory = [(0, 9_000, 10_240)], ram_gib = 16, extra_args = OT_TABLE
+ )
+ assert _arg(cmd, "--fit") == "on", cmd
+
+
[email protected]("ram_gib, none_ok", [(2, False), (16, True)])
+def test_resident_input_tables_are_host_ram_in_the_load_mode_fit(
+ tmp_path, _moe_cache_host, ram_gib, none_ok
+):
+ """gemma3n's table is resident (never lazily read) and host-pinned."""
+ _backend, cmd = _gemma3n_launch(tmp_path, memory = [(0, 23_000, 24_576)], ram_gib = ram_gib)
+ assert ("--load-mode" in cmd and _arg(cmd, "--load-mode") == "none") is none_ok, cmd
+
+
+ "ram_gib, lazy",
+ [(128, ["off"]), (58, [])],
+ ids = [
+ "flash_next_keeps_its_table_resident_where_ram_holds_it",
+ "flash_next_streams_its_table_on_a_64gb_box",
+ ],
+)
+def test_flash_next_keeps_its_table_resident_where_ram_holds_it(
+ tmp_path, _moe_cache_host, ram_gib, lazy
+):
+ """24 GiB card; 58 GiB of RAM is a 64 GB box."""
+ backend, cmd = _flash_next_launch(tmp_path, ram_gib = ram_gib)
+ assert _lazy_values(cmd) == lazy, cmd
+ assert backend._lazy_mode_flags == (["--lazy-mode", "off"] if lazy else [])
+
+
+def test_a_stale_spelling_from_another_build_is_not_the_users_choice(
+ tmp_path, _moe_cache_host, monkeypatch
+):
+ """llama.cpp reads only its own spelling, so an older build's env twin is not a user choice."""
+ monkeypatch.setenv("LLAMA_ARG_TENSOR_READ_LAZY", "off")
+ assert _lazy_values(_flash_next_launch(tmp_path)[1]) == ["off"]
+
+
+def test_the_resident_table_is_charged_once_in_the_load_mode_fit(tmp_path, _moe_cache_host):
+ """After the resident verdict the table moves from the pooled weights to host-only RAM."""
+ _backend, cmd = _flash_next_launch(tmp_path, ram_gib = 72)
+ assert _lazy_values(cmd) == ["off"], cmd
+ assert _arg(cmd, "--load-mode") == "none", cmd
+
+
+def test_a_retry_hands_the_table_back_to_llama_cpp():
+ backend = LlamaCppBackend()
+ backend._lazy_mode_flags = ["--lazy-mode", "off"]
+ argv = ["/fake/llama-server", "-m", "/m.gguf", "--lazy-mode", "off", "-c", "4096"]
+ assert backend._drop_managed_lazy_mode(argv, "CPU fallback") == [
+ "/fake/llama-server",
+ "-m",
+ "/m.gguf",
+ "-c",
+ "4096",
+ ]
+ backend._lazy_mode_flags = []
+ assert backend._drop_managed_lazy_mode(argv, "CPU fallback") == argv
+
+
+def test_a_device_override_keeps_the_table_streamed(tmp_path, _moe_cache_host):
+ """Read lazily, the table sits on a CPU buffer whatever -ot says."""
+ backend, cmd = _flash_next_launch(tmp_path, extra_args = OT_TABLE)
+ assert _lazy_values(cmd) == [], cmd
+ assert backend._lazy_mode_flags == []
+
+
+def test_the_in_place_fit_on_retry_hands_the_table_back(tmp_path, _moe_cache_host, monkeypatch):
+ """A full offload launches with --fit off; its failed first spawn retries without the flag."""
+ cmds = []
+
+ def launch(
+ backend,
+ gguf,
+ model_identifier = "test",
+ **kwargs,
+ ):
+ def fake_popen(cmd, **kw):
+ if not cmd or str(cmd[0]) != "/fake/llama-server":
+ return placement._REAL_POPEN(cmd, **kw)
+ spawn = "-m" in cmd
+ if spawn:
+ cmds.append(list(cmd))
+ rc = (1 if len(cmds) == 1 else None) if spawn else 0
+ return type(
+ "Process",
+ (),
+ {
+ "pid": 2**31 - 1,
+ "stdout": (),
+ "returncode": rc,
+ "poll": lambda self: rc,
+ "terminate": lambda self: None,
+ "wait": lambda self, timeout = None: rc or 0,
+ "kill": lambda self: None,
+ },
+ )()
+
+ backend._wait_for_health = lambda timeout = None, **_kw: len(cmds) > 1
+ with placement.patch.object(subprocess, "Popen", side_effect = fake_popen):
+ backend.load_model(
+ placement.GgufLoadIntent(
+ gguf_path = str(gguf), model_identifier = model_identifier, **kwargs
+ )
+ )
+ return {"cmd": cmds[-1]}
+
+ monkeypatch.setattr(placement, "_launch", launch)
+ _flash_next_launch(tmp_path, memory = [(0, 96_000, 98_304)])
+ assert [_lazy_values(c) for c in cmds] == [["off"], []], cmds
+
+
+ "extra_args, expected",
+ [(["-lzm", "auto"], ["auto"]), (["--lazy-mode", "on"], ["on"]), (["-lzm", "off"], ["off"])],
+ ids = ["user_auto", "user_on", "user_off"],
+)
+def test_a_users_lazy_mode_wins(tmp_path, _moe_cache_host, extra_args, expected):
+ backend, cmd = _flash_next_launch(tmp_path, extra_args = extra_args)
+ assert _lazy_values(cmd) == expected, cmd
+ assert backend._lazy_mode_flags == []
+
+
+def test_an_inherited_lazy_mode_wins(tmp_path, _moe_cache_host, monkeypatch):
+ monkeypatch.setenv("LLAMA_ARG_LAZY_MODE", "on")
+ assert _lazy_values(_flash_next_launch(tmp_path)[1]) == []
+
+
+def test_an_igpu_child_is_left_to_llama_cpp(tmp_path, _moe_cache_host, monkeypatch):
+ monkeypatch.setattr(LlamaCppBackend, "_lazy_auto_resolves_off", lambda self, **_kw: True)
+ assert _lazy_values(_flash_next_launch(tmp_path)[1]) == []
+
+
+def test_a_build_without_lazy_mode_gets_no_flag(tmp_path, _moe_cache_host):
+ caps = dict(placement._CACHE_CAPS_ON, supports_lazy_mode = False)
+ assert _lazy_values(_flash_next_launch(tmp_path, caps = caps)[1]) == []
+
+
+def test_a_paravirtual_launch_keeps_llama_cpps_lazy_default(tmp_path, _moe_cache_host, monkeypatch):
+ monkeypatch.setattr(lc, "_metal_device_is_paravirtual", lambda: True)
+ assert _lazy_values(_flash_next_launch(tmp_path)[1]) == []
+
+
+def test_dsv41_streams_its_engram_table_on_24gb_plus_64gb(tmp_path, _moe_cache_host):
+ _backend, cmd = _cache_launch(
+ tmp_path,
+ size_gib = DSV41_TOTAL / GIB,
+ memory = [(0, 23_800, 24_576)],
+ ram_gib = 58,
+ expert_gib = 100,
+ lazy = _DSV41_NAMED,
+ arch = "deepseek41",
+ )
+ assert _lazy_values(cmd) == [], cmd
+
+
+def test_the_decision_is_logged_once(monkeypatch):
+ seen = []
+ monkeypatch.setattr(lc.logger, "info", lambda msg, *a, **k: seen.append(str(msg)))
+ backend = _scan_backend("qwen4exp", {PLE: FLASH_NEXT_PLE})
+ for _ in range(3):
+ assert _managed_flags(backend) == ["--lazy-mode", "off"]
+ assert sum("Lookup tables: keeping" in m for m in seen) == 1
+
+
+def test_an_unpriced_fit_keeps_llama_cpp_default():
+ backend = _scan_backend("qwen4exp", {PLE: FLASH_NEXT_PLE})
+
+ def boom(_table):
+ raise RuntimeError("unpriced")
+
+ for verdict in (lambda _t: None, lambda _t: False, boom):
+ assert _managed_flags(backend, resident_fits = verdict) == []
+ old = {"supports_lazy_mode": True, "lazy_mode_flag": "--tensor-read-lazy"}
+ assert _managed_flags(backend, caps = old) == ["--tensor-read-lazy", "off"]
+
+
+ "probe, want",
+ [(0, DSV41_ENGRAM), (DSV41_ENGRAM + DSV41_TOKEN_EMBD, DSV41_ENGRAM + DSV41_TOKEN_EMBD)],
+ ids = [
+ "a_failed_metal_probe_still_leaves_the_engram_table_out",
+ "a_successful_probe_keeps_its_larger_measured_discount",
+ ],
+)
+def test_a_failed_metal_probe_still_leaves_the_engram_table_out(probe, want):
+ """DSV4.1 UD-IQ1_S on a 128 GB Mac."""
+ backend = _apple_backend("deepseek41", {PLE: DSV41_ENGRAM}, avail_gib = 119)
+ got = _apple_discount(backend, probe, model_size_fit = DSV41_TOTAL)
+ assert got == want
+ assert backend._apple_lazy_flags == []
+ if not probe:
+ assert (DSV41_TOTAL - got) / GIB == pytest.approx(113.98, abs = 0.01)
+
+
+ "avail_gib, budget_gib, flags, want",
+ [(119, 96, ["--lazy-mode", "off"], 0), (58, 48, [], FLASH_NEXT_PLE)],
+ ids = [
+ "flash_next_goes_resident_on_a_128gb_mac_and_stays_charged",
+ "flash_next_streams_on_a_64gb_mac",
+ ],
+)
+def test_flash_next_goes_resident_on_a_128gb_mac_and_stays_charged(
+ avail_gib, budget_gib, flags, want
+):
+ backend = _apple_backend("qwen4exp", {PLE: FLASH_NEXT_PLE}, avail_gib = avail_gib)
+ assert _apple_discount(backend, budget_gib = budget_gib) == want
+ assert backend._apple_lazy_flags == flags
+
+
+def test_an_unsized_kv_cache_proves_nothing_on_a_mac():
+ backend = _apple_backend("qwen4exp", {PLE: FLASH_NEXT_PLE}, avail_gib = 119)
+ backend._can_estimate_kv = lambda: False
+ assert _apple_discount(backend) == FLASH_NEXT_PLE
+ assert backend._apple_lazy_flags == []
+
+
+def test_a_users_lazy_off_on_a_mac_takes_the_table_back_out_of_the_probe_discount():
+ backend = _apple_backend("qwen4exp", {PLE: FLASH_NEXT_PLE}, avail_gib = 119)
+ got = _apple_discount(
+ backend, FLASH_NEXT_PLE + FLASH_NEXT_TOKEN_EMBD, caps = _CAPS, extra_args = ["-lzm", "off"]
+ )
+ assert got == FLASH_NEXT_TOKEN_EMBD
+ assert backend._apple_lazy_flags == []
+
+
+def test_gemma3n_gets_no_metal_table_discount():
+ """Not TENSOR_READ_LAZY upstream and under 4 GiB: only the probe can discount it."""
+ backend = _apple_backend("gemma3n", _GEMMA3N_NAMED, avail_gib = 12)
+ assert _apple_discount(backend, model_size_fit = GEMMA3N_TOTAL, budget_gib = 10) == 0
+ assert backend._apple_lazy_flags == []
+
+
+ "lazy_args, expected", [([], FLASH_NEXT_PLE), (["-lzm", "off"], 0)], ids = ["auto", "off"]
+)
+def test_a_metal_override_moves_the_table_only_when_it_is_not_streamed(lazy_args, expected):
+ """llama.cpp picks the lazy buffer before -ot, so a streamed table stays off Metal."""
+ backend = _apple_backend("qwen4exp", {PLE: FLASH_NEXT_PLE}, avail_gib = 58)
+ got = _apple_discount(
+ backend,
+ caps = _CAPS,
+ extra_args = ["-ot", "per_layer_token_embd=MTL0", *lazy_args],
+ budget_gib = 48,
+ )
+ assert got == expected
+
+
+def test_module_table_names_deepseek41():
+ assert "deepseek41" in lc._LAZY_READ_TENSORS
+
+
[email protected]("name", ["LLAMA_ARG_MMPROJ", "LLAMA_ARG_MMPROJ_URL"])
+def test_an_inherited_projector_on_a_mac_keeps_the_table_streamed(name):
+ """The env's projector is unpriced in the Metal footprint, so a Mac that would read the table resident leaves it to auto."""
+ backend = _apple_backend("qwen4exp", {PLE: FLASH_NEXT_PLE}, avail_gib = 119)
+ assert _apple_discount(backend, caps = _CAPS, env = {name: "/p/mmproj.gguf"}) == FLASH_NEXT_PLE
+ assert backend._apple_lazy_flags == []
+
+
+ "extra_args, env",
+ [
+ (_RPC_LIST, {}),
+ (["--rpc", "10.0.0.2:50052"], {}),
+ ([], {"LLAMA_ARG_RPC": "10.0.0.2:50052"}),
+ (["--rpc", "10.0.0.2:50052", "--device", "RPC0"], {}),
+ (["--rpc", "10.0.0.2:50052", "--device", "Vulkan0"], {}),
+ ([], {}),
+ (_ONE + ["--main-gpu", "0"], {}),
+ (_ONE + ["--main-gpu", "1"], {}),
+ (_ONE + ["--main-gpu", "-1"], {}),
+ (_RPC_LIST, {"LLAMA_ARG_SPLIT_MODE": "none", "LLAMA_ARG_MAIN_GPU": "0"}),
+ (["--rpc", "10.0.0.2:50052", "--split-mode", "none", "--main-gpu", "-1"], {}),
+ (["--rpc", "10.0.0.2:50052", "--split-mode", "none", "--main-gpu", "0"], {}),
+ ],
+)
+def test_an_rpc_model_device_keeps_lazy_auto_on(extra_args, env):
+ """RPC devices report mmap support (ggml-rpc.cpp device_get_props, b11734-mix-c208e5f and b11870)."""
+ got = LlamaCppBackend()._lazy_auto_resolves_off(
+ gpu_indices = [0],
+ detected_gpus = [(0, 24_000)],
+ shared_gpu_ids = set(),
+ is_vulkan_backend = True,
+ probe_integrated_cuda = False,
+ extra_args = extra_args,
+ env = env,
+ )
+ assert got is False
+
+
+def test_an_integrated_gpu_beside_an_rpc_device_still_turns_lazy_auto_off():
+ got = LlamaCppBackend()._lazy_auto_resolves_off(
+ gpu_indices = [0, 1],
+ detected_gpus = [(0, 24_000), (1, 8_000)],
+ shared_gpu_ids = {1},
+ is_vulkan_backend = True,
+ probe_integrated_cuda = False,
+ extra_args = ["--rpc", "10.0.0.2:50052", "--device", "Vulkan1,RPC0"],
+ env = {},
+ )
+ assert got is True
+
+
+def test_a_streamed_table_stays_on_the_host_under_a_device_override():
+ """llama.cpp picks the lazy buffer before -ot (llama-model-loader.cpp create_tensor)."""
+ pinned = lc._host_pinned_input_bytes
+ scan = ("qwen4exp", {PLE: FLASH_NEXT_PLE}, 0)
+ assert pinned(scan, OT_TABLE, {}, lazy_mode = "auto") == FLASH_NEXT_PLE
+ assert pinned(scan, OT_TABLE, {}, lazy_mode = "off") == 0
+ small = ("gemma4", {PLE: GEMMA3N_PLE}, 0)
+ assert pinned(small, OT_TABLE, {}, lazy_mode = "auto") == 0
+ assert pinned(small, OT_TABLE, {}, lazy_mode = "on") == GEMMA3N_PLE
+
+
+ "lazy_args, fit",
+ [([], "off"), (["-lzm", "on"], "off"), (["-lzm", "off"], "on")],
+ ids = ["auto", "on", "off"],
+)
+def test_a_device_override_leaves_a_streamed_table_off_the_vram_charge(
+ tmp_path, _moe_cache_host, lazy_args, fit
+):
+ _backend, cmd = _flash_next_on_two_cards(tmp_path, ["-ot", "embd=CUDA0", *lazy_args])
+ assert _arg(cmd, "--fit") == fit, cmd
+ if fit == "off":
+ assert _arg(cmd, "-ngl") == "-1", cmd
+
+
+def test_a_streamed_table_under_an_override_keeps_the_resident_embedding_host_only(
+ tmp_path, _moe_cache_host, monkeypatch
+):
+ seen = _spy_fit_load_mode(monkeypatch)
+ _flash_next_on_two_cards(tmp_path, OT_TABLE)
+ assert seen and seen[-1]["lazy_read_bytes"] == FLASH_NEXT_PLE
+ assert seen[-1]["host_only_bytes"] >= FLASH_NEXT_TOKEN_EMBD, seen[-1]
+
+
+ "forced, flags", [(frozenset(), []), (frozenset({"deepseek41"}), ["--lazy-mode", "off"])]
+)
+def test_a_device_override_on_deepseek41_follows_the_builds_placement(monkeypatch, forced, flags):
+ """Where -ot moves the Engram table, -lzm off would make it GPU-resident while priced on the host."""
+ monkeypatch.setattr(
+ LlamaCppBackend, "_host_forced_table_archs", staticmethod(lambda binary = None: forced)
+ )
+ backend = _scan_backend("deepseek41", {PLE: DSV41_ENGRAM})
+ assert _managed_flags(backend, OT_TABLE) == flags
+
+
+def _server_beside(tmp_path, name, payload):
+ server = tmp_path / "llama-server"
+ server.write_bytes(b"\x7fELF server")
+ (tmp_path / name).write_bytes(payload)
+ return LlamaCppBackend._host_forced_table_archs(str(server))
+
+
[email protected]("marked", [True, False])
+def test_the_host_forced_probe_reads_the_builds_loader(tmp_path, marked):
+ """Studio's prebuilt b11734-mix-c208e5f carries the loader warning; upstream b11870 does not."""
+ marker = lc._HOST_FORCED_TABLE_MARKER if marked else b"place it anywhere"
+ got = _server_beside(tmp_path, "libllama.so.0.7.0", b"\x00" * 64 + marker + b" %s, using %s\n")
+ assert got == (frozenset({"deepseek41"}) if marked else frozenset())
+ gone = str(tmp_path / "gone" / "llama-server")
+ assert LlamaCppBackend._host_forced_table_archs(gone) == frozenset()
+
+
+ "name, forced",
+ [
+ ("libllama.so.0.7.0", True),
+ ("libllama.0.dylib", True),
+ ("LLAMA.DLL", True),
+ ("libllama.so.0.bak", False),
+ ("llama.dll.disabled", False),
+ ],
+)
+def test_the_host_forced_probe_reads_only_a_loadable_library(tmp_path, name, forced):
+ got = _server_beside(tmp_path, name, lc._HOST_FORCED_TABLE_MARKER)
+ assert got == (frozenset({"deepseek41"}) if forced else frozenset())
+
+
+def test_the_lazy_mode_choices_probe_the_launch_binary(monkeypatch):
+ seen = []
+ monkeypatch.setattr(
+ LlamaCppBackend,
+ "_host_forced_table_archs",
+ staticmethod(lambda binary = None: seen.append(binary) or frozenset()),
+ )
+ backend = _apple_backend("deepseek41", {PLE: DSV41_ENGRAM}, avail_gib = 119)
+ ot = ["-ot", "per_layer_token_embd=MTL0"]
+ _managed_flags(backend, ot, binary = "/launch/llama-server")
+ _apple_discount(
+ backend,
+ caps = _CAPS,
+ extra_args = ot,
+ model_size_fit = DSV41_TOTAL,
+ binary = "/launch/llama-server",
+ )
+ assert seen and set(seen) == {"/launch/llama-server"}
+
+
+ "extra_args, resident",
+ [([], True), (["--parallel", "1"], True), (["-c", "0"], False), (["--parallel", "8"], False)],
+ ids = ["none", "same_parallel", "ctx_zero", "larger_parallel"],
+)
+def test_a_pass_through_that_reprices_the_cache_keeps_the_table_streamed(
+ tmp_path, _moe_cache_host, extra_args, resident
+):
+ """-c 0 or a larger --parallel wins over the cache the resident verdict priced."""
+ _backend, cmd = _flash_next_launch(tmp_path, extra_args = extra_args)
+ assert (_lazy_values(cmd) == ["off"]) is resident, cmd
+
+
+ "extra_args, n_parallel, reprices",
+ [
+ ([], 1, False),
+ (["-c", "0"], 1, True),
+ (["--ctx-size", "8192"], 1, False),
+ (["--parallel", "4"], 4, False),
+ (["--parallel", "4"], 2, True),
+ (["--parallel", "1"], 2, False),
+ ],
+)
+def test_the_metal_resident_verdict_abstains_on_the_same_overrides(
+ extra_args, n_parallel, reprices
+):
+ assert lc._pass_through_reprices_cache(extra_args, n_parallel) is reprices
+
+
[email protected]("cpu_build", [False, True])
+def test_a_cpu_child_keeps_llama_cpps_lazy_default(
+ tmp_path, _moe_cache_host, monkeypatch, cpu_build
+):
+ """The resident verdict credits the cards; a CPU-only build or --device none has none."""
+ monkeypatch.setattr(
+ LlamaCppBackend, "_binary_ships_no_gpu_backend", staticmethod(lambda *a, **k: cpu_build)
+ )
+ _backend, cmd = _flash_next_launch(tmp_path)
+ assert (_lazy_values(cmd) == ["off"]) is not cpu_build, cmd
+ _backend, cmd = _flash_next_launch(tmp_path, extra_args = ["--device", "none"])
+ assert _lazy_values(cmd) == [], cmd
+
+
+def _metal_launched(tmp_path, monkeypatch, **kw):
+ """A Metal load_model (16 GiB budget, 12 GiB weights); returns argv."""
+ import test_metal_explicit_context_guard as metal
+ return metal._launch(
+ tmp_path,
+ monkeypatch,
+ n_ctx = 8192,
+ kv_per_token = 32 * 1024,
+ backend = metal._backend_with_embeddings(monkeypatch, embd = 0),
+ real_fit = True,
+ budget_bytes = 16 * GIB,
+ weights_bytes = 12 * GIB,
+ **kw,
+ )["cmd"]
+
+
+ "extra_args, pinned",
+ [
+ (None, True),
+ (["-ngl", "20"], False),
+ (["--fit-target", "2048"], False),
+ (["-ot", "blk=CPU"], False),
+ ],
+ ids = ["studio_owns", "user_layers", "user_fit_target", "user_cpu_override"],
+)
+def test_a_streamed_table_alone_never_pins_a_placement_the_user_owns(
+ tmp_path, monkeypatch, extra_args, pinned
+):
+ """The table discount sizes the Metal fit; only Studio's own placement may be pinned on it."""
+ from test_metal_explicit_context_guard import TestInputEmbeddingsLeftInTheFileMapping as T
+
+ monkeypatch.setattr(
+ LlamaCppBackend,
+ "_apple_lookup_table_discount",
+ lambda self, probe, *a, **k: max(int(probe), 6 * GIB),
+ )
+ cmd = _metal_launched(tmp_path, monkeypatch, extra_args = extra_args)
+ assert (T._fit_tokens(cmd)[:4] == ["-ngl", "-1", "--fit", "off"]) is pinned, cmd
+
+
+def _spy_resident_inputs(monkeypatch):
+ seen = []
+
+ def spy(self, probe, *a, **k):
+ seen.append(k)
+ return int(probe)
+
+ monkeypatch.setattr(LlamaCppBackend, "_apple_lookup_table_discount", spy)
+ return seen
+
+
+def test_the_metal_resident_verdict_prices_the_context_checkpoints(tmp_path, monkeypatch):
+ needs = []
+ for ckpt in (0, GIB):
+ seen = _spy_resident_inputs(monkeypatch)
+ _metal_launched(
+ tmp_path, monkeypatch, ckpt_bytes = ckpt, extra_args = ["--ctx-checkpoints", "8"]
+ )
+ needs.append(seen[-1]["need_at_ctx"])
+ assert needs[1] > needs[0], needs
+
+
+ "kw, budget_gib",
+ [
+ ({"wired_bytes": 0}, 16),
+ ({"wired_bytes": 8 * GIB}, 8),
+ ({"wired_bytes": 32 * GIB}, 16),
+ ({"wired_exhausted": True}, 0),
+ ],
+ ids = ["0-16", "8-8", "32-16", "used_up_wired_headroom_leaves_no_resident_budget"],
+)
+def test_the_metal_resident_verdict_obeys_the_wired_headroom(tmp_path, monkeypatch, kw, budget_gib):
+ seen = _spy_resident_inputs(monkeypatch)
+ _metal_launched(tmp_path, monkeypatch, **kw)
+ assert seen[-1]["budget_mib"] == budget_gib * 1024
+
+
+ "extra_args, cpu_forced",
+ [(["--device", "none"], False), ([], True)],
+ ids = ["device_none", "paravirtual"],
+)
+def test_a_cpu_child_on_a_mac_keeps_the_table_streamed(extra_args, cpu_forced):
+ backend = _apple_backend("qwen4exp", {PLE: FLASH_NEXT_PLE}, avail_gib = 119)
+ got = _apple_discount(backend, extra_args = extra_args, cpu_forced = cpu_forced)
+ assert backend._apple_lazy_flags == []
+ assert got == FLASH_NEXT_PLE
+
+
+def test_the_metal_resident_verdict_abstains_on_an_unsized_mtp_draft(tmp_path, monkeypatch):
+ """A flat MTP reserve prices no draft KV, so no resident verdict (as the discrete one)."""
+ seen = _spy_resident_inputs(monkeypatch)
+ _metal_launched(tmp_path, monkeypatch)
+ assert seen[-1]["need_at_ctx"] is not None
+ _metal_launched(tmp_path, monkeypatch, extra_args = ["--spec-type", "draft-mtp"])
+ assert seen[-1]["need_at_ctx"] is None
+
+
+def test_a_rocm_arch_gated_cpu_launch_is_a_cpu_child(tmp_path, monkeypatch, probe_env):
+ """No installed kernel covers either card, so the child runs on the CPU."""
+ import test_gpu_arch_gate_os_matrix_7624 as gate
+
+ forced = _spy(monkeypatch, "_child_runs_on_cpu", lambda _ans, _a, k: bool(k.get("cpu_forced")))
+ backend = _scan_backend("qwen4exp", {PLE: FLASH_NEXT_PLE})
+ gate._apply_os(monkeypatch, "linux", is_rocm = True)
+ torch = gate._fake_torch(
+ [gate._device("gfx1030", free_mib = 12049), gate._device("gfx1036", free_mib = 12176)],
+ vendor = "amd",
+ )
+ gate._run_auto_load(
+ monkeypatch,
+ tmp_path,
+ torch,
+ ["gfx908"],
+ returncode = None,
+ backend = backend,
+ server_caps = dict(_CAPS),
+ intent_kwargs = {"gpu_memory_mode": "auto"},
+ )
+ assert backend._arch_gate_forced_cpu is True
+ assert forced and all(forced), forced
+
+
[email protected]("node", [False, True])
+def test_an_empty_inventory_beside_a_gpu_prices_the_table_resident_at_launch(
+ tmp_path, _moe_cache_host, monkeypatch, node
+):
+ """A failed probe may hide an iGPU, where lazy auto is off (as the listing prices it)."""
+ seen = _spy_fit_load_mode(monkeypatch)
+ monkeypatch.setattr(lc, "_gpu_device_node_present", lambda: node)
+ _flash_next_launch(tmp_path, ram_gib = 58, memory = [])
+ assert seen, "no load-mode fit ran"
+ assert seen[-1]["lazy_read_bytes"] == (0 if node else FLASH_NEXT_PLE), seen[-1]
+
+
+ "extra_args, env, cpu",
+ [
+ (["--split-mode", "none", "--main-gpu", "-1"], {}, True),
+ (["-sm", "none", "-mg", "-1"], {}, True),
+ ([], _INHERITED_CPU_SPLIT, True),
+ (["-sm", "none", "-mg", "0"], {}, False),
+ (["--main-gpu", "-1"], {}, False),
+ (["-sm", "layer", "-mg", "-1"], {}, False),
+ ],
+)
+def test_split_mode_none_with_a_negative_main_gpu_is_a_cpu_child(extra_args, env, cpu):
+ """llama.cpp clears the model's devices for it (src/llama.cpp, b11734-mix-c208e5f)."""
+ backend = LlamaCppBackend()
+ backend._binary_ships_no_gpu_backend = lambda *a, **k: False
+ assert backend._child_runs_on_cpu(None, extra_args, dict(env)) is cpu
+
+
+ "extra_args, env, resident",
+ [
+ (["-sm", "none", "-mg", "0"], {}, True),
+ (["--split-mode", "none", "--main-gpu", "-1"], {}, False),
+ (["--split-mode", "none"], {"LLAMA_ARG_MAIN_GPU": "-1"}, False),
+ # load_model drops an inherited non-layer split mode on a layer launch: layer split on CUDA0.
+ ([], _INHERITED_CPU_SPLIT, True),
+ (["--main-gpu", "-1"], {"LLAMA_ARG_SPLIT_MODE": "none"}, True),
+ ],
+)
+def test_a_main_gpu_of_minus_one_keeps_llama_cpps_lazy_default(
+ tmp_path, _moe_cache_host, monkeypatch, extra_args, env, resident
+):
+ _setenv(monkeypatch, env)
+ _backend, cmd = _flash_next_launch(tmp_path, extra_args = extra_args)
+ assert (_lazy_values(cmd) == ["off"]) is resident, cmd
+
+
+def _spy_placement_previews(monkeypatch):
+ """(helper, caller, env split mode, tensor_parallel) for every placement preview."""
+ seen = []
+ for name in ("_lazy_auto_resolves_off", "_child_runs_on_cpu", "_apple_lookup_table_discount"):
+ real = getattr(LlamaCppBackend, name)
+
+ def make(real, name):
+ def spy(self, *a, **k):
+ env = k.get("env", a[2] if name == "_child_runs_on_cpu" and len(a) > 2 else None)
+ env = os.environ if env is None else env
+ seen.append(
+ (
+ name,
+ sys._getframe(1).f_code.co_name,
+ env.get("LLAMA_ARG_SPLIT_MODE"),
+ k.get("tensor_parallel"),
+ )
+ )
+ return real(self, *a, **k)
+
+ return spy
+
+ monkeypatch.setattr(LlamaCppBackend, name, make(real, name))
+ return seen
+
+
[email protected]("cards", [1, 2])
+def test_every_launch_preview_sees_the_split_mode_the_child_gets(
+ tmp_path, _moe_cache_host, monkeypatch, cards
+):
+ """load_model drops an inherited non-layer split mode before spawn; every preview must price that child."""
+ _setenv(monkeypatch, _INHERITED_CPU_SPLIT)
+ seen = _spy_placement_previews(monkeypatch)
+ launch = _flash_next_launch if cards == 1 else _flash_next_on_two_cards
+ _backend, cmd = launch(tmp_path)
+ callers = {caller for _n, caller, _sm, _tp in seen}
+ assert {"_host_input_bytes_now", "load_model", "_table_resident_fits"} <= callers, seen
+ assert cards == 2 or "_fitter_vram_shortfall" in callers, seen
+ assert [s for s in seen if s[2] is not None] == [], seen
+ assert _lazy_values(cmd) == ["off"], cmd
+
+
+def test_a_tensor_launch_is_not_a_cpu_child_under_an_inherited_split_mode(
+ tmp_path, _moe_cache_host, monkeypatch
+):
+ """Studio emits --split-mode tensor (argv beats env), so the inherited CPU split does not apply."""
+ _setenv(monkeypatch, _INHERITED_CPU_SPLIT)
+ seen = _spy_placement_previews(monkeypatch)
+ _backend, cmd = _flash_next_on_two_cards(tmp_path, tensor_parallel = True)
+ assert "tensor" in cmd, cmd
+ cpu_calls = [s for s in seen if s[0] == "_child_runs_on_cpu" and s[1] == "_table_resident_fits"]
+ assert cpu_calls and cpu_calls[-1][3] is True, seen
+ assert _lazy_values(cmd) == ["off"], cmd
+
+
[email protected]("tensor, cpu", [(None, True), (False, True), (True, False)])
+def test_the_cpu_child_check_follows_the_launch_tensor_state(tensor, cpu):
+ backend = LlamaCppBackend()
+ backend._binary_ships_no_gpu_backend = lambda *a, **k: False
+ got = backend._child_runs_on_cpu(None, [], dict(_INHERITED_CPU_SPLIT), tensor_parallel = tensor)
+ assert got is cpu
+
+
+def _apu_flash_next(tmp_path, monkeypatch, apu_ids):
+ backend, gguf = placement._apu_backend(
+ tmp_path, gguf_gb = 73.45, avail_mib = 58 * 1024, monkeypatch = monkeypatch
+ )
+ monkeypatch.setattr(
+ LlamaCppBackend, "_rocm_unified_memory_gpu_ids", staticmethod(lambda: set(apu_ids))
+ )
+ backend._gguf_tensor_scan = lambda _path: ("qwen4exp", {PLE: placement._QWEN38_PLE}, 0)
+ backend.probe_server_capabilities = _full_caps()
+ return backend, gguf
+
+
+def test_a_failed_probe_apu_guard_ignores_a_split_mode_the_launch_drops(tmp_path, monkeypatch):
+ """The HIP child runs layer split on the APU once load_model drops split mode none, so the guard charges its table."""
+ _setenv(monkeypatch, _INHERITED_CPU_SPLIT)
+ backend, gguf = _apu_flash_next(tmp_path, monkeypatch, {0})
+ backend._get_gpu_memory = lambda *_a, **_kw: []
+ backend._get_gpu_free_memory = lambda *_a, **_kw: []
+
+ res = placement._launch(backend, gguf, extra_args = ["--load-mode", "none"])
+
+ assert res["env"].get("LLAMA_ARG_SPLIT_MODE") is None
+ assert "unified-memory APU" in (backend.last_load_warning or "")
+ assert not placement._unmapped_tokens(res["cmd"]), res["cmd"]
+
+
+def test_the_metal_discount_sees_the_split_mode_the_child_gets(tmp_path, monkeypatch):
+ _setenv(monkeypatch, _INHERITED_CPU_SPLIT)
+ seen = _spy_placement_previews(monkeypatch)
+ _metal_launched(tmp_path, monkeypatch)
+ metal = [s for s in seen if s[0] == "_apple_lookup_table_discount"]
+ assert metal and all(s[2] is None for s in metal), seen
+
+
+ "extra_args",
+ [["--device", "ROCm0"], ["--device", "ROCm0", "--load-mode", "none"], []],
+ ids = ["device", "device_unmapped", "no_device"],
+)
+def test_a_compact_device_names_the_card_the_child_mask_puts_first(
+ tmp_path, monkeypatch, extra_args
+):
+ """Auto pins a ROCm APU at physical 1 beside a discrete card at 0, so the child's ROCm0 is the APU, not the parent's."""
+ for name in ("CUDA_VISIBLE_DEVICES", "HIP_VISIBLE_DEVICES", "ROCR_VISIBLE_DEVICES"):
+ monkeypatch.delenv(name, raising = False)
+ backend, gguf = _apu_flash_next(tmp_path, monkeypatch, {1})
+ rows = [(0, 16_000, 16_384), (1, 90_000, 92_160)]
+ backend._get_gpu_memory = lambda *_a, **_kw: list(rows)
+ backend._get_gpu_free_memory = lambda *_a, **_kw: [(i, f) for i, f, _t in rows]
+ backend._amd_apu_wants_unified_memory = lambda ids = None, *a, **k: (
+ ids is not None and bool(list(ids)) and {int(i) for i in ids} <= {1}
+ )
+ backend._select_gpus = lambda *a, **k: ([1], True)
+ backend._can_estimate_kv = lambda: True
+ backend._estimate_kv_cache_bytes = lambda *a, **k: GIB // 4
+ seen = _spy(
+ monkeypatch,
+ "_lazy_auto_resolves_off",
+ lambda answer, _a, _k: (sys._getframe(2).f_code.co_name, answer),
+ )
+
+ res = placement._launch(backend, gguf, extra_args = extra_args)
+
+ assert res["env"].get("CUDA_VISIBLE_DEVICES") == "1", res["env"]
+ previews = [s for s in seen if s[0] in ("load_model", "_fitter_vram_shortfall")]
+ assert previews and all(answer for _c, answer in previews), seen
+ assert "unified-memory APU" in (backend.last_load_warning or "")
+ assert not placement._unmapped_tokens(res["cmd"]), res["cmd"]
+
+
[email protected]("ram_gib", [58, 256])
[email protected]("layout", ["dgpu0_igpu1", "igpu0_dgpu1"])
+def test_split_mode_none_reads_the_main_device_in_picker_order(
+ tmp_path, _moe_cache_host, layout, ram_gib
+):
+ """gpu_ids [1, 0] emits --device Vulkan1,Vulkan0, so --split-mode none puts every weight on Vulkan1, not Vulkan0."""
+ dgpu, igpu = (0, 1) if layout == "dgpu0_igpu1" else (1, 0)
+ rows = sorted([(dgpu, 35 * 1024, 36 * 1024), (igpu, 55 * 1024, 0)])
+ backend, gguf = placement._moe_backend(tmp_path, size_gib = 73.45, memory = rows, moe = False)
+ backend._is_vulkan_backend = lambda _binary = None: True
+ backend._can_estimate_kv = lambda: True
+ backend._estimate_kv_cache_bytes = lambda *a, **k: GIB // 4
+ backend._available_system_memory_mib = lambda: int(ram_gib * 1024)
+ backend._gguf_tensor_scan = lambda _path: ("qwen4exp", {PLE: placement._QWEN38_PLE}, 0)
+ backend.probe_server_capabilities = _full_caps()
+
+ cmd = placement._launch(backend, gguf, gpu_ids = [1, 0], extra_args = ["--split-mode", "none"])[
+ "cmd"
+ ]
+
+ assert _arg(cmd, "--device") == "Vulkan1,Vulkan0", cmd
+ main_is_igpu = igpu == 1
+ # An iGPU main device keeps the table resident (auto is off there): 73.45 GiB does
+ # not fit 58 GiB of RAM unmapped, so the load stays mmap'd.
+ if main_is_igpu and ram_gib == 58:
+ assert placement._load_mode(cmd) is None, cmd
+ # A discrete main device streams it unless RAM holds it, where Studio pins it resident.
+ expected_lazy = ["off"] if not main_is_igpu and ram_gib == 256 else []
+ assert _lazy_values(cmd) == expected_lazy, cmd
+
+
+ "gpu_ids, gpu_indices, split, inherited, order, mask",
+ [
+ (None, [0, 1], None, None, [0, 1], [0, 1]),
+ ([1, 0], [0, 1], None, None, [1, 0], [1, 0]),
+ ([1, 0], None, None, None, [1, 0], [1, 0]),
+ ([1, 0], [0, 1], [3, 1], None, [1, 0], [1, 0]),
+ ([1, 0], [0, 1], [3], None, [0, 1], [0, 1]),
+ (None, [0, 1], None, [1, 0], [0, 1], [1, 0]),
+ (None, [0, 1], [3, 1, 2], [1, 0], [0, 1], [0, 1]),
+ (None, None, None, None, None, None),
+ ],
+ ids = [
+ "ascending",
+ "picker_reorder",
+ "picker_unnarrowed",
+ "picker_moves_shares",
+ "picker_veto_shares",
+ "inherited_order",
+ "inherited_veto_shares",
+ "unpinned",
+ ],
+)
+def test_the_planned_child_order_matches_the_spawn(
+ monkeypatch, gpu_ids, gpu_indices, split, inherited, order, mask
+):
+ monkeypatch.setattr(
+ LlamaCppBackend, "_resolve_visible_physical_ids", staticmethod(lambda: inherited)
+ )
+ monkeypatch.setattr(
+ LlamaCppBackend,
+ "_pinned_child_visibility",
+ staticmethod(lambda ids: ",".join(str(int(i)) for i in ids)),
+ )
+ backend = LlamaCppBackend()
+ kwargs = dict(
+ gpu_ids = gpu_ids, gpu_indices = gpu_indices, tp_tensor_split = split, extra_args = [], env = {}
+ )
+ plan = backend._planned_child_gpu_order
+ assert plan(**kwargs, is_vulkan_backend = False) == (order, mask)
+ # Vulkan has no mask; a CPU child gets none either.
+ assert plan(**kwargs, is_vulkan_backend = True) == (order, None)
+ assert plan(**kwargs, is_vulkan_backend = False, cpu_forced = True) == (order, None)
+ # Ids the mask cannot name keep the inherited one: the parent's numbering stands.
+ monkeypatch.setattr(LlamaCppBackend, "_pinned_child_visibility", staticmethod(lambda ids: None))
+ assert plan(**kwargs, is_vulkan_backend = False) == (order, None)
+
+
+def test_a_compact_device_maps_through_the_planned_mask(monkeypatch):
+ backend = LlamaCppBackend()
+ backend._integrated_cuda_gpu_ids = lambda: set()
+ monkeypatch.setattr(LlamaCppBackend, "_rocm_unified_memory_gpu_ids", staticmethod(lambda: {1}))
+ common = dict(
+ gpu_indices = [1],
+ detected_gpus = [(0, 16_000), (1, 90_000)],
+ shared_gpu_ids = set(),
+ is_vulkan_backend = False,
+ extra_args = ["--device", "ROCm0"],
+ env = {},
+ )
+ # The parent's numbering: ROCm0 is the discrete card; the child's mask [1]: ROCm0 is the APU.
+ assert backend._lazy_auto_resolves_off(**common) is False
+ assert backend._lazy_auto_resolves_off(**common, child_gpu_order = [1]) is True
diff --git a/studio/backend/tests/test_lookup_table_pricing.py b/studio/backend/tests/test_lookup_table_pricing.py
new file mode 100644
index 00000000000..991b529b9eb
--- /dev/null
+++ b/studio/backend/tests/test_lookup_table_pricing.py
@@ -0,0 +1,1339 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
+
+"""Lookup-only tables (per-layer embeddings / Engram) priced apart in the listing and the panel."""
+
+from __future__ import annotations
+
+import asyncio
+import io
+import os
+import struct
+import sys
+import threading
+import time
+from pathlib import Path
+from types import SimpleNamespace
+
+import pytest
+
+_TESTS_DIR = str(Path(__file__).resolve().parent)
+if _TESTS_DIR not in sys.path:
+ sys.path.insert(0, _TESTS_DIR)
+_BACKEND_DIR = str(Path(__file__).resolve().parent.parent)
+if _BACKEND_DIR not in sys.path:
+ sys.path.insert(0, _BACKEND_DIR)
+
+import test_kv_cache_estimation # noqa: E402,F401
+import test_llama_cpp_placement # noqa: E402,F401 -- stubs jwt for routes.inference
+
+import routes.inference as ri # noqa: E402
+import utils.hardware as hw # noqa: E402
+from core.inference import llama_cpp as lc # noqa: E402
+from hub.schemas.inventory import GgufVariantDetail # noqa: E402
+from hub.services.models import gguf_variants as gv # noqa: E402
+from hub.utils import gguf_header_delta # noqa: E402
+from hub.utils import gguf_lookup_tables as glt # noqa: E402
+
+GIB = 1024**3
+IQ4_NL, Q5_1, F16 = 20, 7, 1
+PLE = "per_layer_token_embd.weight"
+
+DSV41_TABLE = 256 * 768022850 // 32 * 18
+FLASH_NEXT_TABLE = 160 * 320001536 // 32 * 18
+GEMMA_TABLE = 8960 * 262144 // 32 * 24
+
+_ATTN = {
+ "block_count": 4,
+ "context_length": 4096,
+ "embedding_length": 256,
+ "attention.head_count": 4,
+ "attention.head_count_kv": 2,
+ "attention.key_length": 64,
+ "attention.value_length": 64,
+}
+
+
+def _header(
+ arch: str,
+ tensors: list[tuple[str, list[int], int]],
+ ints: dict | None = None,
+ offsets: list[int] | None = None,
+):
+ """A GGUF v3 header: the architecture, uint32 metadata, and tensor infos with no data."""
+ kv: list[tuple[str, object]] = [("general.architecture", arch)]
+ kv += [(f"{arch}.{k}", int(v)) for k, v in (ints or {}).items()]
+ buf = io.BytesIO()
+ buf.write(b"GGUF")
+ buf.write(struct.pack("<IQQ", 3, len(tensors), len(kv)))
+ for key, value in kv:
+ raw = key.encode()
+ buf.write(struct.pack("<Q", len(raw)) + raw)
+ if isinstance(value, str):
+ data = value.encode()
+ buf.write(struct.pack("<IQ", 8, len(data)) + data)
+ else:
+ buf.write(struct.pack("<II", 4, value))
+ offset = 0
+ for i, (name, dims, ggml_type) in enumerate(tensors):
+ raw = name.encode()
+ buf.write(struct.pack("<Q", len(raw)) + raw)
+ buf.write(struct.pack("<I", len(dims)))
+ buf.write(struct.pack(f"<{len(dims)}Q", *dims))
+ buf.write(struct.pack("<IQ", ggml_type, offsets[i] if offsets else offset))
+ offset += 1024
+ return buf.getvalue()
+
+
+_LAZY_BLOB = _header("qwen4exp", [(PLE, [160, 320001536], IQ4_NL)], _ATTN)
+_DSV41_EMBD = ("token_embd.weight", [4096, 129280], IQ4_NL)
+_DSV41_PLE = (PLE, [256, 768022850], IQ4_NL)
+
+
+def _write(path: Path, data: bytes) -> str:
+ path.parent.mkdir(parents = True, exist_ok = True)
+ path.write_bytes(data)
+ return str(path)
+
+
+def _dsv41(tmp_path: Path) -> str:
+ return _write(
+ tmp_path / "DeepSeek-V4.1-Flash-UD-IQ1_S.gguf",
+ _header("deepseek41", [_DSV41_EMBD, _DSV41_PLE]),
+ )
+
+
+def _flash_next(tmp_path: Path) -> str:
+ tensors = [
+ ("token_embd.weight", [2048, 151936], IQ4_NL),
+ (PLE, [160, 320001536], IQ4_NL),
+ ("blk.0.attn_q.weight", [256, 256], F16),
+ ]
+ return _write(
+ tmp_path / "Qwen3.8-Flash-Next-UD-Q2_K_XL.gguf", _header("qwen4exp", tensors, _ATTN)
+ )
+
+
+def _gemma(tmp_path: Path, arch = "gemma3n") -> str:
+ tensors = [(PLE, [8960, 262144], Q5_1)]
+ if arch == "gemma3n":
+ tensors.insert(0, ("token_embd.weight", [2048, 262144], IQ4_NL))
+ return _write(tmp_path / f"{arch}.gguf", _header(arch, tensors))
+
+
+def _blob_fetcher(
+ blobs: dict,
+ calls: list,
+ delay: float = 0.0,
+):
+ """A Range fetcher over in-memory files that reports each file's size like Content-Range."""
+
+ def fetcher(
+ repo_id,
+ name,
+ token,
+ revision = None,
+ ):
+ blob, size = blobs[name]
+
+ def fetch(start, end):
+ calls.append((name, start, end))
+ if delay:
+ time.sleep(delay)
+ fetch.file_size = size
+ return blob[start : end + 1]
+
+ return fetch
+
+ return fetcher
+
+
+def _answer(variants, sources = None):
+ from hub.schemas.inventory import GgufVariantsResponse
+ from hub.services.models.gguf_variants import VariantsAnswer
+ return VariantsAnswer(
+ GgufVariantsResponse(repo_id = "unsloth/X-GGUF", variants = variants),
+ None,
+ variant_context_sources = sources or {},
+ )
+
+
+def _row(
+ filename = "a.gguf",
+ quant = "Q4",
+ size = 40 * GIB,
+ **kw,
+):
+ return GgufVariantDetail(filename = filename, quant = quant, size_bytes = size, **kw)
+
+
+def _listing(
+ rows,
+ repo,
+ *,
+ token = None,
+ offline = False,
+):
+ """The coroutine; ``rows`` may be a ready answer."""
+ answer = rows if hasattr(rows, "response") else _answer(rows)
+ return gv._with_lookup_tables(
+ answer, repo, hf_token = token, offline = offline, prefer_local_cache = False
+ )
+
+
+def _list(rows, repo, **kw):
+ return asyncio.run(_listing(rows, repo, **kw))
+
+
+def _tables(out):
+ return [v.lookup_table_bytes for v in out.response.variants]
+
+
+def _arch(
+ monkeypatch,
+ known,
+ arch = "qwen4exp",
+):
+ monkeypatch.setattr(gv, "_cached_hub_repo_architecture", lambda *a, **k: (known, arch))
+
+
+def _remote(monkeypatch, fn = None):
+ monkeypatch.setattr(
+ glt, "remote_lookup_info", fn or (lambda *a, **k: ("qwen4exp", FLASH_NEXT_TABLE))
+ )
+
+
+def _overlap(
+ first,
+ second,
+ wait = None,
+):
+ """Run ``second`` once ``first`` is underway (after 0.1 s, or once ``wait`` is set)."""
+
+ async def both():
+ a = asyncio.ensure_future(first)
+ if wait is None:
+ await asyncio.sleep(0.1)
+ while wait is not None and not wait.is_set():
+ await asyncio.sleep(0.01)
+ return await asyncio.gather(a, second)
+
+ return asyncio.run(both())
+
+
+def _stream_backend(monkeypatch, *, vulkan, gpus):
+ backend = lc.LlamaCppBackend.__new__(lc.LlamaCppBackend)
+ monkeypatch.setattr(backend, "_find_llama_server_binary", lambda: "/fake/llama-server")
+ monkeypatch.setattr(
+ backend,
+ "probe_server_capabilities",
+ lambda binary = None: {"supports_lazy_mode": True, "lazy_mode_flag": "--lazy-mode"},
+ )
+ monkeypatch.setattr(backend, "_is_vulkan_backend", lambda binary = None: vulkan)
+ monkeypatch.setattr(backend, "_get_gpu_memory", lambda *a, **k: list(gpus))
+ return backend
+
+
+def _no_gpu_backend(monkeypatch, cpu_build):
+ monkeypatch.setattr(
+ lc.LlamaCppBackend, "_binary_ships_no_gpu_backend", staticmethod(lambda *a, **k: cpu_build)
+ )
+
+
[email protected](autouse = True)
+def _streams_here(monkeypatch):
+ monkeypatch.setattr(gv, "_host_streams_lookup_tables", lambda: True)
+
+
+def test_split_names_keep_the_extension_as_written():
+ assert glt.shard_names("d/M-00001-of-00002.GGUF") == [
+ "d/M-00001-of-00002.GGUF",
+ "d/M-00002-of-00002.GGUF",
+ ]
+
+
+def test_a_posix_class_pattern_is_not_proof_the_table_stays_on_the_host():
+ """std::regex reads [[:alpha:]] as a class."""
+ on_device = lc._override_places_tensor_on_device
+ assert on_device(PLE, ["-ot", "per_[[:alpha:]_]+=CUDA0"], {})
+ assert on_device(PLE, ["-ot", "per_[[:alpha:]_]+=CPU,per_layer=CUDA0"], {})
+ assert not on_device(PLE, ["-ot", "per_[[:alpha:]_]+=CPU"], {})
+
+
+class TestHeaderScan:
+ def test_real_table_sizes(self, tmp_path):
+ assert glt.local_lookup_info(_dsv41(tmp_path)) == ("deepseek41", DSV41_TABLE)
+ assert DSV41_TABLE / GIB == pytest.approx(103.00, abs = 0.01)
+ assert glt.local_lookup_info(_flash_next(tmp_path)) == ("qwen4exp", FLASH_NEXT_TABLE)
+ assert FLASH_NEXT_TABLE / GIB == pytest.approx(26.82, abs = 0.01)
+
+ def test_gemma3n_is_not_lazily_read(self, tmp_path):
+ assert glt.local_lookup_info(_gemma(tmp_path)) == ("gemma3n", 0)
+
+ def test_a_small_table_stays_resident_under_auto(self, tmp_path):
+ assert glt.local_lookup_table_bytes(_gemma(tmp_path, "gemma4")) == 0
+
+ def test_an_explicit_lazy_mode_on_reads_a_small_table_lazily(self, tmp_path):
+ small = _gemma(tmp_path, "gemma4")
+ assert glt.local_lookup_info(small) == ("gemma4", 0)
+ assert glt.local_lookup_info(small, "on") == ("gemma4", GEMMA_TABLE)
+
+ @pytest.mark.parametrize(
+ "second, want",
+ [(True, DSV41_TABLE), (False, 0)],
+ ids = ["split_file_finds_the_table_in_a_later_shard", "missing_shard"],
+ )
+ def test_split_file_finds_the_table_in_a_later_shard(self, tmp_path, second, want):
+ first = _write(tmp_path / "m-00001-of-00002.gguf", _header("deepseek41", [_DSV41_EMBD]))
+ if second:
+ _write(tmp_path / "m-00002-of-00002.gguf", _header("", [_DSV41_PLE]))
+ assert glt.local_lookup_table_bytes(first) == want
+
+ def test_a_missing_shard_or_garbage_is_zero(self, tmp_path):
+ assert glt.local_lookup_table_bytes(_write(tmp_path / "junk.gguf", b"not a gguf")) == 0
+ assert glt.local_lookup_table_bytes(None) == 0
+
+ def test_remote_reads_only_the_header_range(self, tmp_path):
+ calls = []
+ fetcher = _blob_fetcher({"Q2/flash-Q2.gguf": (_LAZY_BLOB, 40 * GIB)}, calls)
+
+ def read():
+ return glt.remote_lookup_info(
+ "unsloth/Flash-Next-GGUF", "Q2/flash-Q2.gguf", identity = 1, fetcher_fn = fetcher
+ )
+
+ assert read() == ("qwen4exp", FLASH_NEXT_TABLE)
+ assert calls and all(start == 0 for _name, start, _end in calls)
+ calls.clear()
+ assert read() == ("qwen4exp", FLASH_NEXT_TABLE)
+ assert not calls
+ glt._REMOTE_CACHE.clear()
+ glt._REMOTE_TTL_S, ttl = -1.0, glt._REMOTE_TTL_S
+ try:
+ read()
+ assert glt.remote_cached("unsloth/Flash-Next-GGUF", "Q2/flash-Q2.gguf", None, 1) is None
+ finally:
+ glt._REMOTE_TTL_S = ttl
+ assert calls
+
+ def test_remote_failure_is_zero_and_not_cached(self):
+ def broken(*a, **k):
+ def fetch(start, end):
+ raise OSError("network down")
+
+ return fetch
+
+ assert glt.remote_lookup_info("o/r", "f.gguf", identity = 7, fetcher_fn = broken) == (None, 0)
+ assert glt.remote_cached("o/r", "f.gguf", None, 7) is None
+
+ def test_a_remote_answer_is_cached_per_lazy_mode(self):
+ fetcher = _blob_fetcher({"f.gguf": (b"", 0)}, [])
+ glt.remote_lookup_info("o/m", "f.gguf", identity = 11, fetcher_fn = fetcher)
+ assert glt.remote_cached("o/m", "f.gguf", None, 11, mode = "on") is None
+
+
+class TestListing:
+ @pytest.mark.parametrize(
+ "streams, want",
+ [(False, None), (True, DSV41_TABLE)],
+ ids = [
+ "a_host_that_reads_the_tables_resident_lists_no_discount",
+ "a_downloaded_variant_is_read_from_its_own_header",
+ ],
+ )
+ def test_a_downloaded_variant_is_read_from_its_own_header(
+ self, tmp_path, monkeypatch, streams, want
+ ):
+ """An iGPU / APU resolves lazy auto to off, so the listing prices the whole file."""
+ monkeypatch.setattr(gv, "_host_streams_lookup_tables", lambda: streams)
+ monkeypatch.setattr(
+ gv, "_hub_repo_architecture", lambda *a, **k: pytest.fail("no Hub call")
+ )
+ path = _dsv41(tmp_path)
+ row = _row(os.path.basename(path), "UD-IQ1_S", 216 * GIB, downloaded = True)
+ answer = _answer([row], {"ud-iq1_s": path})
+ out = _list(answer, "unsloth/X-GGUF", offline = True)
+ if want is None:
+ assert out is answer
+ else:
+ assert _tables(out) == [want]
+ assert _tables(answer) == [0]
+
+ def test_both_variant_models_carry_the_field(self):
+ from models.models import GgufVariantDetail as RouteDetail
+ for model in (GgufVariantDetail, RouteDetail):
+ assert model.model_fields["lookup_table_bytes"].default == 0
+
+ def test_hub_only_quants_of_a_lazy_arch_are_range_read(self, monkeypatch):
+ _arch(monkeypatch, True)
+ _remote(monkeypatch)
+ rows = [_row("Q2/a-00001-of-00003.gguf", "Q2"), _row("README.md", "x", 1)]
+ assert _tables(_list(rows, "unsloth/Flash-Next-GGUF")) == [FLASH_NEXT_TABLE, 0]
+
+ def test_a_small_repo_never_asks_the_hub_for_its_architecture(self, monkeypatch):
+ """No row can hold a streamed table (each is under 4 GiB)."""
+ _arch(monkeypatch, False, None)
+ monkeypatch.setattr(
+ gv, "_hub_repo_architecture", lambda *a, **k: pytest.fail("no Hub call")
+ )
+ answer = _answer([_row(size = GIB)])
+ assert _list(answer, "o/small") is answer
+
+ def test_a_cold_architecture_is_priced_on_the_first_listing(self, monkeypatch):
+ fetched = []
+ monkeypatch.setattr(
+ gv, "_hub_repo_architecture", lambda repo, token: fetched.append(repo) or "qwen4exp"
+ )
+ _arch(monkeypatch, False, None)
+ _remote(monkeypatch)
+ assert _tables(_list([_row()], "o/cold")) == [FLASH_NEXT_TABLE]
+ assert fetched == ["o/cold"]
+
+ def test_a_slow_architecture_lookup_is_bounded_by_the_deadline(self, monkeypatch):
+ """A late answer never holds the listing past its deadline, and is asked once."""
+ release = threading.Event()
+ fetched = []
+
+ def slow_arch(repo, token):
+ fetched.append(repo)
+ release.wait(5)
+ return "qwen4exp"
+
+ monkeypatch.setattr(gv, "_LOOKUP_DEADLINE_S", 0.2)
+ monkeypatch.setattr(gv, "_hub_repo_architecture", slow_arch)
+ _arch(monkeypatch, False, None)
+ _remote(monkeypatch, lambda *a, **k: pytest.fail("no header read yet"))
+ answer = _answer([_row()])
+ try:
+ for _ in range(3):
+ started = time.monotonic()
+ assert _list(answer, "o/slow") is answer
+ assert time.monotonic() - started < 2.0
+ finally:
+ release.set()
+ for _ in range(50):
+ if not gv._lookup_inflight:
+ break
+ time.sleep(0.05)
+ assert fetched == ["o/slow"]
+
+ def test_a_direct_file_source_finds_its_row_beside_it(self, tmp_path):
+ model = tmp_path / "Qwen3.8-Flash-Next-UD-Q2_K_XL.gguf"
+ model.write_bytes(b"GGUF")
+ row = SimpleNamespace(filename = model.name, quant = "UD-Q2_K_XL", downloaded = True)
+ for answer, repo in (
+ (SimpleNamespace(variant_context_sources = None, context_source = str(model)), "o/r"),
+ (SimpleNamespace(variant_context_sources = None, context_source = None), str(model)),
+ ):
+ assert gv._local_variant_gguf(answer, repo, row) == str(model)
+
+ def test_a_requested_cache_copy_is_found_where_the_listing_found_it(self, tmp_path):
+ """local_path naming a second cache root, or a snapshot other than refs/main, is searched."""
+ repo_dir = tmp_path / "second_cache" / "models--o--lookup-cache-test"
+ main_snap, pinned_snap = repo_dir / "snapshots" / "aaaa", repo_dir / "snapshots" / "bbbb"
+ for snap, name in ((main_snap, "m-Q4.gguf"), (pinned_snap, "p-Q8.gguf")):
+ snap.mkdir(parents = True)
+ (snap / name).write_bytes(b"GGUF")
+ (repo_dir / "refs").mkdir()
+ (repo_dir / "refs" / "main").write_text("aaaa")
+ answer = SimpleNamespace(
+ variant_context_sources = None, context_source = None, cache_authorized = True
+ )
+ for name, root, snap in (
+ ("m-Q4.gguf", repo_dir, main_snap),
+ ("p-Q8.gguf", pinned_snap, pinned_snap),
+ ):
+ row = SimpleNamespace(filename = name, quant = "Q", downloaded = True)
+ got = gv._local_variant_gguf(answer, "o/lookup-cache-test", row, str(root))
+ assert got == str(snap / name)
+
+ def test_a_read_the_pool_refuses_after_the_deadline_releases_its_key(self, monkeypatch):
+ """Late refusals (the pool shut down at exit) never reach the read's own finally."""
+
+ async def refused(fn, /, *args):
+ await asyncio.sleep(0.01)
+ raise RuntimeError("cannot schedule new futures after shutdown")
+
+ monkeypatch.setattr(gv, "_lookup_thread", refused)
+ key = ("arch", "o/refused")
+
+ async def go():
+ loop = asyncio.get_running_loop()
+ assert await gv._lookup_await_once(key, loop.time(), lambda: 1) is None
+ assert key in gv._lookup_inflight
+ await asyncio.sleep(0.05)
+
+ asyncio.run(go())
+ assert key not in gv._lookup_inflight
+
+ def test_a_cached_non_lazy_variant_does_not_veto_the_rest_of_the_repo(
+ self, tmp_path, monkeypatch
+ ):
+ local = _write(
+ tmp_path / "llama.gguf",
+ _header("llama", [("token_embd.weight", [256, 1000], F16)], _ATTN),
+ )
+ _arch(monkeypatch, True)
+ _remote(monkeypatch)
+ rows = [_row("llama.gguf", "Q8_0", 1, downloaded = True), _row("flash/f.gguf", "Q2")]
+ out = _list(_answer(rows, {"q8_0": local}), "o/mixed")
+ assert _tables(out) == [0, FLASH_NEXT_TABLE]
+
+ def test_one_listing_queues_every_uncached_header(self, monkeypatch):
+ calls = []
+ lock = threading.Lock()
+
+ def read(repo, name, *a, **k):
+ with lock:
+ calls.append(name)
+ return ("qwen4exp", FLASH_NEXT_TABLE)
+
+ _arch(monkeypatch, True)
+ _remote(monkeypatch, read)
+ rows = [_row(f"q{i}/m.gguf", f"Q{i}", 40 * GIB + i) for i in range(10)]
+ out = _list(rows, "o/many")
+ assert sorted(calls) == sorted(r.filename for r in rows)
+ assert all(_tables(out))
+
+ @pytest.mark.parametrize("offline, want", [(False, 10 * GIB), (True, 40 * GIB)])
+ def test_an_update_is_priced_from_the_published_header(
+ self, monkeypatch, tmp_path, offline, want
+ ):
+ """The cached file is the old revision: online, an update's table comes from the Hub's header."""
+ on_disk = tmp_path / "old.gguf"
+ on_disk.write_bytes(b"x")
+ _arch(monkeypatch, True)
+ monkeypatch.setattr(gv, "_local_variant_gguf", lambda *a: str(on_disk))
+ monkeypatch.setattr(
+ glt, "local_lookup_info", lambda path, mode = "auto": ("qwen4exp", 40 * GIB)
+ )
+ _remote(monkeypatch, lambda *a, **k: ("qwen4exp", 10 * GIB))
+ row = _row("u.gguf", "U", 70 * GIB + offline, downloaded = True, update_available = True)
+ out = _list([row], f"o/updated-{offline}", offline = offline)
+ assert _tables(out) == [want]
+
+ def test_a_forced_lazy_mode_sizes_the_tables_it_streams(self, monkeypatch, tmp_path):
+ modes = []
+
+ def local(path, mode = "auto"):
+ modes.append(("local", mode))
+ return ("qwen4exp", FLASH_NEXT_TABLE)
+
+ def read(
+ *a,
+ mode = "auto",
+ **k,
+ ):
+ modes.append(("remote", mode))
+ return ("qwen4exp", FLASH_NEXT_TABLE)
+
+ monkeypatch.setattr(gv, "_listing_lazy_mode", lambda: "on")
+ _arch(monkeypatch, True)
+ _remote(monkeypatch, read)
+ monkeypatch.setattr(glt, "local_lookup_info", local)
+ on_disk = tmp_path / "local.gguf"
+ on_disk.write_bytes(b"x")
+ monkeypatch.setattr(
+ gv, "_local_variant_gguf", lambda a, r, v, *_: str(on_disk) if v.quant == "L" else None
+ )
+ _list([_row("l.gguf", "L", 1, downloaded = True), _row("r.gguf", "R", 2)], "o/forced")
+ assert sorted(modes) == [("local", "on"), ("remote", "on")]
+
+ def test_a_forced_lazy_mode_looks_up_a_cold_repo_of_small_files(self, monkeypatch):
+ monkeypatch.setattr(gv, "_listing_lazy_mode", lambda: "on")
+ _arch(monkeypatch, False, None)
+ monkeypatch.setattr(gv, "_hub_repo_architecture", lambda *a, **k: "gemma4")
+ _remote(monkeypatch, lambda *a, **k: ("gemma4", 2 * GIB))
+ assert _tables(_list([_row("s.gguf", size = 3 * GIB)], "o/small")) == [2 * GIB]
+
+ @pytest.mark.parametrize(
+ "stage",
+ ["arch", "table"],
+ ids = [
+ "an_architecture_lookup_in_flight_for_another_token_does_not_block",
+ "a_table_scan_in_flight_for_another_token_does_not_skip_the_row",
+ ],
+ )
+ def test_an_architecture_lookup_in_flight_for_another_token_does_not_block(
+ self, monkeypatch, stage
+ ):
+ asked = []
+ second = threading.Event()
+
+ def gate(token, answer):
+ asked.append(token)
+ if token == "hf_first":
+ second.wait(1.5)
+ else:
+ second.set()
+ return answer
+
+ if stage == "arch":
+ _arch(monkeypatch, False, None)
+ monkeypatch.setattr(
+ gv, "_hub_repo_architecture", lambda repo, token: gate(token, "qwen4exp")
+ )
+ _remote(monkeypatch)
+ else:
+ _arch(monkeypatch, True)
+ _remote(
+ monkeypatch,
+ lambda repo, name, token, *a, **k: gate(token, ("qwen4exp", FLASH_NEXT_TABLE)),
+ )
+ rows = [_row(size = 40 * GIB + 7)]
+ repo = f"o/token-{stage}"
+ _first, other = _overlap(
+ _listing(rows, repo, token = "hf_first"), _listing(rows, repo, token = "hf_second")
+ )
+ assert sorted(asked) == ["hf_first", "hf_second"]
+ assert _tables(other) == [FLASH_NEXT_TABLE]
+
+ @pytest.mark.parametrize(
+ "vulkan, node, streams",
+ [(False, True, False), (False, False, True), (True, False, False), (True, True, False)],
+ )
+ def test_an_empty_inventory_beside_a_gpu_node_prices_the_table_resident(
+ self, monkeypatch, vulkan, node, streams
+ ):
+ """Without the torch fallback a ROCm APU can read as no rows; an empty Vulkan inventory may hide an iGPU."""
+ backend = _stream_backend(monkeypatch, vulkan = vulkan, gpus = [])
+ monkeypatch.setattr(backend, "_lazy_auto_resolves_off", lambda **k: False)
+ monkeypatch.setattr(lc, "_gpu_device_node_present", lambda: node)
+ assert backend.lookup_tables_stream_by_default(None, {}) is streams
+
+ def test_a_stalled_hub_leaves_no_queue_after_the_deadline(self, monkeypatch):
+ release = threading.Event()
+ started = []
+ lock = threading.Lock()
+
+ def stalled(repo, name, *a, **k):
+ with lock:
+ started.append(name)
+ release.wait(10)
+ return ("qwen4exp", FLASH_NEXT_TABLE)
+
+ monkeypatch.setattr(gv, "_LOOKUP_DEADLINE_S", 0.3)
+ _arch(monkeypatch, True)
+ _remote(monkeypatch, stalled)
+ rows = [_row(f"s{i}/m.gguf", f"S{i}", 40 * GIB + i) for i in range(12)]
+
+ async def listing_then_release():
+ await _listing(rows, "o/stalled")
+ await asyncio.sleep(0.3)
+ release.set()
+ await asyncio.sleep(0.5)
+
+ try:
+ asyncio.run(listing_then_release())
+ finally:
+ release.set()
+ assert len(started) <= gv._LOOKUP_CONCURRENCY, started
+ with gv._lookup_arch_lock:
+ assert not [k for k in gv._lookup_inflight if k[:2] == ("table", "o/stalled")]
+
+ def test_an_expired_architecture_lookup_leaves_no_queue(self, monkeypatch):
+ release = threading.Event()
+ ran = []
+
+ def slow(name):
+ ran.append(name)
+ release.wait(10)
+ return name
+
+ async def main():
+ loop = asyncio.get_running_loop()
+ busy = [
+ asyncio.ensure_future(gv._lookup_thread(slow, f"busy{i}"))
+ for i in range(gv._LOOKUP_CONCURRENCY)
+ ]
+ await asyncio.sleep(0.2)
+ got = await gv._lookup_await_once(
+ ("arch", "o/late", "fp"), loop.time() + 0.2, slow, "late"
+ )
+ release.set()
+ await asyncio.gather(*busy)
+ await asyncio.sleep(0.3)
+ return got
+
+ try:
+ got = asyncio.run(main())
+ finally:
+ release.set()
+ assert got is None
+ assert "late" not in ran
+ with gv._lookup_arch_lock:
+ assert ("arch", "o/late", "fp") not in gv._lookup_inflight
+
+ def test_work_the_deadline_drops_never_runs_if_a_worker_reaches_it_late(self, monkeypatch):
+ """A worker can pick up queued work before the cancel reaches the executor."""
+ captured = []
+ reads = []
+
+ async def queued(fn, *args):
+ captured.append((fn, args))
+ await asyncio.sleep(10)
+
+ def read(repo, name, *a, **k):
+ reads.append(name)
+ return ("qwen4exp", FLASH_NEXT_TABLE)
+
+ monkeypatch.setattr(gv, "_lookup_thread", queued)
+ monkeypatch.setattr(gv, "_LOOKUP_DEADLINE_S", 0.1)
+ _arch(monkeypatch, True)
+ _remote(monkeypatch, read)
+ rows = [_row(f"q{i}/m.gguf", f"Q{i}", 40 * GIB + i) for i in range(3)]
+
+ async def main():
+ await _listing(rows, "o/late")
+ loop = asyncio.get_running_loop()
+ return await gv._lookup_await_once(
+ ("arch", "o/late", "fp"), loop.time() + 0.1, read, "o/late", "arch", None
+ )
+
+ assert asyncio.run(main()) is None
+ assert len(captured) == 4
+ for fn, args in captured:
+ fn(*args)
+ assert reads == []
+
+ def test_a_capability_read_never_falls_back_to_a_torch_context(self, monkeypatch):
+ from utils.hardware import gpu_query
+
+ B = lc.LlamaCppBackend
+ monkeypatch.setattr(B, "_is_vulkan_backend", staticmethod(lambda binary = None: False))
+ monkeypatch.setattr(B, "_get_gpu_memory_nvml", staticmethod(lambda: []))
+ monkeypatch.setattr(B, "_get_gpu_memory_amd_smi", staticmethod(lambda binary = None, **k: []))
+ monkeypatch.setattr(
+ B,
+ "_get_gpu_memory_torch",
+ staticmethod(lambda *a, **k: pytest.fail("torch probe in a capability read")),
+ )
+
+ def no_smi(*a, **k):
+ raise FileNotFoundError("nvidia-smi")
+
+ monkeypatch.setattr(gpu_query, "run_nvidia_smi", no_smi)
+ with lc._without_torch_gpu_probe():
+ assert B._get_gpu_memory("/fake/llama-server", for_llama_server = True) == []
+ assert lc._GPU_MEMORY_SKIP_TORCH.get() is False
+
+ def test_offline_or_a_non_lazy_arch_never_reads_the_hub(self, monkeypatch):
+ _remote(monkeypatch, lambda *a, **k: pytest.fail("no header read"))
+ answer = _answer([_row(quant = "Q4_K_M", size = 1)])
+ assert _list(answer, "o/r", offline = True) is answer
+ monkeypatch.setattr(gv, "_hub_repo_architecture", lambda *a, **k: "llama")
+ assert _list(answer, "o/r") is answer
+
+ def test_a_failure_returns_the_listing_unchanged(self, monkeypatch):
+ def boom(*a, **k):
+ raise RuntimeError("boom")
+
+ monkeypatch.setattr(gv, "_local_variant_gguf", boom)
+ answer = _answer([_row(quant = "Q", size = 1)])
+ assert _list(answer, "o/r") is answer
+
+
+def _static(monkeypatch, name, fn):
+ monkeypatch.setattr(ri.LlamaCppBackend, name, staticmethod(fn))
+
+
+def _ram(monkeypatch, gib):
+ _static(monkeypatch, "_available_system_memory_mib", lambda: int(gib * 1024))
+
+
+def _gpus(monkeypatch, *rows):
+ _static(monkeypatch, "_get_gpu_memory", lambda binary = None, **k: list(rows))
+
+
+def _caps(monkeypatch, **caps):
+ _static(monkeypatch, "probe_server_capabilities", lambda binary = None: {"found": True, **caps})
+
+
+def _apple(monkeypatch, budget_gib):
+ monkeypatch.setattr(hw, "is_apple_silicon", lambda: True)
+ _static(monkeypatch, "_apple_metal_memory_budget_bytes", lambda: int(budget_gib * GIB))
+
+
+class TestEstimate:
+ WEIGHTS = 73.45 * GIB
+
+ @pytest.fixture(autouse = True)
+ def _clean_env(self, monkeypatch):
+ monkeypatch.delenv("LLAMA_ARG_LAZY_MODE", raising = False)
+ monkeypatch.delenv("LLAMA_ARG_OVERRIDE_TENSOR", raising = False)
+
+ @pytest.fixture
+ def breakdown(self, tmp_path, monkeypatch):
+ path = _flash_next(tmp_path)
+ monkeypatch.setattr(ri, "_gguf_resident_file_gb", lambda cfg, **kw: self.WEIGHTS / GIB)
+ _static(monkeypatch, "_get_gguf_size_bytes", lambda p: int(self.WEIGHTS))
+ _caps(monkeypatch, supports_lazy_mode = True)
+ _static(monkeypatch, "_is_vulkan_backend", lambda b = None: False)
+ _gpus(monkeypatch, (0, 22000, 24576))
+ _static(monkeypatch, "_integrated_cuda_gpu_ids", lambda: set())
+ monkeypatch.setattr(hw, "is_apple_silicon", lambda: False)
+ _static(monkeypatch, "_apple_metal_memory_budget_bytes", lambda: 0)
+ _ram(monkeypatch, 16)
+ ri._estimate_files_cache.clear()
+ ri._estimate_config_cache.clear()
+ config = SimpleNamespace(
+ identifier = "local/flash-next", gguf_file = path, is_gguf = True, gguf_mmproj_file = None
+ )
+
+ def run(gguf = None, **kw):
+ cfg = config if gguf is None else SimpleNamespace(**{**vars(config), "gguf_file": gguf})
+ return ri._gguf_memory_breakdown(
+ cfg, gguf or path, **{"n_ctx": 4096, "gpu_memory_mode": "auto", **kw}
+ )
+
+ yield run
+ ri._estimate_files_cache.clear()
+ ri._estimate_config_cache.clear()
+
+ def test_streamed_table_leaves_the_gpu_and_the_total(self, breakdown):
+ b = breakdown()
+ assert (b.lookup_table_bytes, b.lookup_table_on_disk) == (FLASH_NEXT_TABLE, True)
+ assert b.weights_bytes == int(round(self.WEIGHTS)) - FLASH_NEXT_TABLE
+ assert b.gpu_bytes == b.total_bytes == b.weights_bytes + b.kv_bytes + b.compute_bytes
+
+ def test_lazy_mode_off_keeps_it_resident_but_still_off_the_gpu(self, breakdown):
+ b = breakdown(llama_extra_args = ["-lzm", "off"])
+ assert (b.lookup_table_bytes, b.lookup_table_on_disk) == (FLASH_NEXT_TABLE, False)
+ assert b.weights_bytes == int(round(self.WEIGHTS))
+ assert b.gpu_bytes == b.total_bytes - FLASH_NEXT_TABLE
+
+ @pytest.mark.parametrize("extras", [[], ["-lzm", "auto"]], ids = ["managed", "user_auto"])
+ def test_ram_that_holds_the_table_reads_it_resident_as_the_launch_does(
+ self, breakdown, monkeypatch, extras
+ ):
+ """A user's own lazy mode wins over the managed choice, as at launch."""
+ _ram(monkeypatch, 256)
+ b = breakdown(llama_extra_args = extras)
+ assert (b.lookup_table_bytes, b.lookup_table_on_disk) == (FLASH_NEXT_TABLE, bool(extras))
+
+ @pytest.mark.parametrize(
+ "extras, on_disk",
+ [
+ ([], False),
+ (["--parallel", "1"], False),
+ (["-c", "0"], True),
+ (["--parallel", "8"], True),
+ ],
+ ids = ["none", "same_parallel", "ctx_zero", "larger_parallel"],
+ )
+ def test_a_pass_through_that_reprices_the_cache_keeps_the_table_streamed(
+ self, breakdown, monkeypatch, extras, on_disk
+ ):
+ _ram(monkeypatch, 256)
+ assert breakdown(llama_extra_args = extras).lookup_table_on_disk is on_disk
+
+ @pytest.mark.parametrize("ram_gib", [66, 256])
+ @pytest.mark.parametrize("cpu_build", [False, True])
+ def test_a_cpu_child_keeps_the_table_streamed_as_the_launch_does(
+ self, breakdown, monkeypatch, cpu_build, ram_gib
+ ):
+ _ram(monkeypatch, ram_gib)
+ _static(monkeypatch, "_binary_ships_no_gpu_backend", lambda *a, **k: cpu_build)
+ assert breakdown().lookup_table_on_disk is cpu_build
+ assert breakdown(llama_extra_args = ["--device", "none"]).lookup_table_on_disk is True
+
+ def test_a_mig_slice_reporting_no_total_keeps_its_vram_credit(self, breakdown, monkeypatch):
+ """Total 0 means an iGPU only on Vulkan; a CUDA MIG or vGPU slice can report it with VRAM of its own."""
+ _ram(monkeypatch, 66)
+ _gpus(monkeypatch, (0, 22000, 0))
+ assert breakdown().lookup_table_on_disk is False
+
+ @pytest.mark.parametrize("pool", ["soc", "apu", "discrete"])
+ def test_a_unified_pool_beside_a_card_is_priced_once_as_the_launch_does(
+ self, breakdown, monkeypatch, pool
+ ):
+ """An integrated CUDA SoC or ROCm APU's free memory is the host RAM the proof already counts."""
+ _ram(monkeypatch, 58)
+ _gpus(monkeypatch, (0, 16000, 16384), (1, 50000, 65536))
+ monkeypatch.setattr(
+ ri.LlamaCppBackend,
+ "_INTEGRATED_CUDA_IDS",
+ {ri.LlamaCppBackend._integrated_cuda_mask_key(): (1,) if pool == "soc" else ()},
+ )
+ monkeypatch.setattr(
+ ri.get_llama_cpp_backend(),
+ "_amd_apu_wants_unified_memory",
+ lambda ids = None, *a, **k: (pool == "apu" and ids is not None and bool(set(ids) & {1})),
+ raising = False,
+ )
+ assert breakdown().lookup_table_on_disk is (pool != "discrete")
+
+ @pytest.mark.parametrize("tensor", [False, True])
+ def test_an_inherited_split_mode_the_launch_drops_keeps_the_table_resident(
+ self, breakdown, monkeypatch, tensor
+ ):
+ """Layer launches drop an inherited split mode none and tensor launches override it: neither child is on the CPU."""
+ monkeypatch.setenv("LLAMA_ARG_SPLIT_MODE", "none")
+ monkeypatch.setenv("LLAMA_ARG_MAIN_GPU", "-1")
+ _ram(monkeypatch, 256)
+ if tensor:
+ _gpus(monkeypatch, (0, 22000, 24576), (1, 22000, 24576))
+ assert breakdown(tensor_parallel = tensor).lookup_table_on_disk is False
+
+ @pytest.mark.parametrize(
+ "ram_gib, kw, on_disk",
+ [
+ (66, {}, False),
+ (66, {"gpu_memory_mode": "manual", "gpu_layers": -1}, True),
+ (66, {"gpu_memory_mode": "manual", "gpu_layers": 20}, True),
+ (256, {"gpu_memory_mode": "manual", "gpu_layers": -1}, False),
+ (256, {"gpu_memory_mode": "manual", "gpu_layers": -1, "n_ctx": 0}, True),
+ ],
+ ids = [
+ "auto_gpu_credit",
+ "manual_ram_only",
+ "manual_layers_ram_only",
+ "manual_fits",
+ "manual_no_ctx",
+ ],
+ )
+ def test_manual_mode_proves_residency_as_the_launch_does(
+ self, breakdown, monkeypatch, ram_gib, kw, on_disk
+ ):
+ """Manual launches plan no cards (gpus = []), and Auto layers with no context leave it to --fit."""
+ _ram(monkeypatch, ram_gib)
+ assert breakdown(**kw).lookup_table_on_disk is on_disk
+
+ def test_a_paravirtual_metal_estimate_keeps_the_table_streamed(self, breakdown, monkeypatch):
+ _ram(monkeypatch, 256)
+ assert breakdown().lookup_table_on_disk is False
+ monkeypatch.setattr(ri, "_gpu_placement_paravirtual", lambda: True)
+ assert breakdown().lookup_table_on_disk is True
+
+ @pytest.mark.parametrize(
+ "wired_gib, on_disk", [(None, False), (0, True), (200, False), (60, True)]
+ )
+ def test_the_apple_estimate_obeys_the_wired_headroom(
+ self, breakdown, monkeypatch, wired_gib, on_disk
+ ):
+ _apple(monkeypatch, 120)
+ _ram(monkeypatch, 256)
+ _static(
+ monkeypatch,
+ "_apple_metal_wired_headroom_bytes",
+ lambda: None if wired_gib is None else wired_gib * GIB,
+ )
+ assert breakdown().lookup_table_on_disk is on_disk
+
+ def test_a_cpu_pinned_projector_is_charged_to_ram_in_the_residency_proof(
+ self, breakdown, monkeypatch
+ ):
+ """A 96 GB card holds everything but the table, so 34 GiB of RAM keeps the table resident."""
+ MM = 6 * GIB
+ monkeypatch.setattr(
+ ri, "_gguf_resident_file_gb", lambda cfg, **kw: (self.WEIGHTS + MM) / GIB
+ )
+ monkeypatch.setattr(ri, "_charged_projector_bytes", lambda *a, **k: MM)
+ _gpus(monkeypatch, (0, 96000, 98304))
+ _ram(monkeypatch, 34)
+ assert breakdown().lookup_table_on_disk is False
+ assert breakdown(llama_extra_args = ["--no-mmproj-offload"]).lookup_table_on_disk is True
+
+ def test_ram_the_spilled_weights_crowd_out_keeps_the_table_streamed(
+ self, breakdown, monkeypatch
+ ):
+ monkeypatch.delenv("LLAMA_ARG_CACHE_RAM", raising = False)
+ _ram(monkeypatch, 58)
+ _caps(monkeypatch, supports_lazy_mode = True, supports_cache_ram = True)
+ assert breakdown().lookup_table_on_disk is True
+ _gpus(monkeypatch, (0, 96_000, 98_304))
+ assert breakdown().lookup_table_on_disk is False
+
+ @pytest.mark.parametrize("budget_gib, ram_gib, resident", [(48, 64, False), (180, 256, True)])
+ def test_apple_prices_the_whole_load_against_unified_memory(
+ self, breakdown, monkeypatch, budget_gib, ram_gib, resident
+ ):
+ _apple(monkeypatch, budget_gib)
+ _ram(monkeypatch, ram_gib)
+ assert breakdown().lookup_table_on_disk is (not resident)
+
+ @pytest.mark.parametrize(
+ "indexed",
+ [True, False],
+ ids = [
+ "an_indexed_table_name_keeps_its_override",
+ "a_device_override_keeps_the_table_streamed_as_the_launch_does",
+ ],
+ )
+ def test_an_indexed_table_name_keeps_its_override(self, breakdown, monkeypatch, indexed):
+ _ram(monkeypatch, 256)
+ name = "per_layer_token_embd_0" if indexed else "per_layer_token_embd"
+ if indexed:
+ real = ri.LlamaCppBackend._gguf_tensor_scan
+
+ def scan(self, path):
+ arch, named, experts = real(self, path)
+ return (
+ arch,
+ {k.replace(PLE, f"{name}.weight"): v for k, v in named.items()},
+ experts,
+ )
+
+ monkeypatch.setattr(ri.LlamaCppBackend, "_gguf_tensor_scan", scan)
+ assert breakdown(llama_extra_args = ["-ot", f"{name}=CUDA0"]).lookup_table_on_disk is True
+
+ def test_a_device_override_charges_a_resident_table_wholly_to_the_gpu(
+ self, breakdown, tmp_path
+ ):
+ """A layer fraction does not apply to a tensor an -ot pins."""
+ b = breakdown(
+ gguf = _gemma(tmp_path),
+ gpu_memory_mode = "manual",
+ gpu_layers = 0,
+ llama_extra_args = ["-ot", "per_layer_token_embd=CUDA0"],
+ )
+ assert b.lookup_table_bytes == 0
+ assert b.gpu_bytes >= GEMMA_TABLE
+
+ @pytest.mark.parametrize(
+ "igpu",
+ [False, True],
+ ids = [
+ "a_build_without_lazy_mode_keeps_the_table_resident",
+ "an_integrated_gpu_reads_the_table_resident",
+ ],
+ )
+ def test_a_build_without_lazy_mode_keeps_the_table_resident(self, breakdown, monkeypatch, igpu):
+ """A Vulkan iGPU has no mmap support, so llama.cpp resolves auto to off."""
+ if igpu:
+ _static(monkeypatch, "_is_vulkan_backend", lambda b = None: True)
+ _gpus(monkeypatch, (0, 30000, 0))
+ else:
+ _caps(monkeypatch)
+ b = breakdown()
+ assert b.lookup_table_on_disk is False
+ assert b.weights_bytes == int(round(self.WEIGHTS))
+
+ def test_a_pin_on_the_integrated_gpu_reads_the_table_resident(self, breakdown, monkeypatch):
+ """A mixed host streams by default (llama.cpp drops the iGPU), not when pinned to it."""
+ _static(monkeypatch, "_is_vulkan_backend", lambda b = None: True)
+ _gpus(monkeypatch, (0, 30000, 0), (1, 22000, 24576))
+ assert breakdown().lookup_table_on_disk is True
+ assert breakdown(selected_gpu_ids = [0]).lookup_table_on_disk is False
+ assert breakdown(selected_gpu_ids = [1]).lookup_table_on_disk is True
+ stale = breakdown(selected_gpu_ids = [0], llama_extra_args = ["--device", "Vulkan1"])
+ assert stale.lookup_table_on_disk is False
+
+ def test_an_explicit_lazy_mode_on_prices_every_flagged_table(self, breakdown, monkeypatch):
+ modes = []
+ real = glt.local_lookup_info
+ monkeypatch.setattr(
+ glt,
+ "local_lookup_info",
+ lambda path, mode = "auto": modes.append(mode) or real(path, mode),
+ )
+ breakdown(llama_extra_args = ["-lzm", "on"])
+ breakdown()
+ assert modes == ["on", "auto"]
+
+ def test_a_resident_gemma3n_table_leaves_the_gpu_share_not_the_total(self, breakdown, tmp_path):
+ """Not lazily read, but still a host-pinned input like the launcher prices it."""
+ b = breakdown(gguf = _gemma(tmp_path))
+ assert (b.lookup_table_bytes, b.lookup_table_on_disk) == (GEMMA_TABLE, False)
+ assert b.weights_bytes == int(round(self.WEIGHTS))
+ assert b.gpu_bytes == b.total_bytes - b.lookup_table_bytes
+
+ def test_a_resident_table_moved_to_the_gpu_stays_in_the_gpu_share(self, breakdown):
+ b = breakdown(llama_extra_args = ["-lzm", "off", "-ot", "per_layer_token_embd=CUDA0"])
+ assert (b.lookup_table_bytes, b.lookup_table_on_disk) == (0, False)
+ assert b.gpu_bytes == b.total_bytes
+
+ def test_the_load_panel_response_carries_the_table(self, breakdown):
+ from core.inference.memory_contract import (
+ build_memory_estimate,
+ project_estimate_memory_response,
+ )
+ from models.inference import EstimateMemoryResponse
+
+ est = build_memory_estimate(breakdown(), quant_file_bytes = int(self.WEIGHTS))
+ out = project_estimate_memory_response(est)
+ assert (out["lookup_table_bytes"], out["lookup_table_on_disk"]) == (FLASH_NEXT_TABLE, True)
+ assert EstimateMemoryResponse(**out).lookup_table_bytes == FLASH_NEXT_TABLE
+
+ def test_the_memory_bar_response_carries_the_table(self, breakdown):
+ """Its weights_bytes is the whole file, so the bar needs the table to take it off the VRAM segment."""
+ from core.inference.memory_contract import build_memory_estimate, project_kv_cache_estimate
+
+ est = build_memory_estimate(breakdown(), quant_file_bytes = int(self.WEIGHTS))
+ out = project_kv_cache_estimate(est)
+ assert (out["weights_bytes"], out["lookup_table_bytes"]) == (
+ int(self.WEIGHTS),
+ FLASH_NEXT_TABLE,
+ )
+
+
+def _split_blobs(first_arch = "qwen4exp", first_embd = ("token_embd.weight", [2048, 151936], IQ4_NL)):
+ names = [f"Q4/m-{i:05d}-of-00003.gguf" for i in (1, 2, 3)]
+ headers = [
+ _header(first_arch, [first_embd], _ATTN),
+ _header("", [(PLE, [160, 320001536], IQ4_NL)]),
+ _header("", [("blk.3.attn_q.weight", [256, 256], F16)]),
+ ]
+ return names, {n: (h, 40 * GIB) for n, h in zip(names, headers)}
+
+
+class TestHubDiscovery:
+ # Engram stored in a quant type gguf predates (ggml type 43); the next tensor bounds it.
+ UNKNOWN_TYPE = 43
+
+ def test_an_unknown_quant_type_is_sized_from_the_next_offset(self, tmp_path):
+ path = _write(
+ tmp_path / "dsv41-unknown.gguf",
+ _header(
+ "deepseek41",
+ [
+ (PLE, [256, 768022850], self.UNKNOWN_TYPE),
+ ("blk.0.attn_q.weight", [256, 256], F16),
+ ],
+ offsets = [0, DSV41_TABLE],
+ ),
+ )
+ assert glt.local_lookup_info(path) == ("deepseek41", DSV41_TABLE)
+
+ def test_an_unknown_quant_type_last_in_the_file_is_sized_from_the_file(self, tmp_path):
+ header = _header(
+ "deepseek41",
+ [_DSV41_EMBD, (PLE, [256, 768022850], self.UNKNOWN_TYPE)],
+ offsets = [0, 4096],
+ )
+ size = -(-len(header) // 32) * 32 + 4096 + DSV41_TABLE
+ path = tmp_path / "dsv41-last.gguf"
+ with open(path, "wb") as f:
+ f.write(header)
+ f.truncate(size)
+ assert glt.local_lookup_info(str(path)) == ("deepseek41", DSV41_TABLE)
+ got = glt.remote_lookup_info(
+ "o/unknown-type",
+ "m.gguf",
+ identity = size,
+ fetcher_fn = _blob_fetcher({"m.gguf": (header, size)}, []),
+ )
+ assert got == ("deepseek41", DSV41_TABLE)
+
+ def test_local_discovery_reads_every_shard(self, tmp_path):
+ first = _write(tmp_path / "big-00001-of-00010.gguf", _header("deepseek41", [_DSV41_EMBD]))
+ for i in range(2, 10):
+ _write(
+ tmp_path / f"big-{i:05d}-of-00010.gguf",
+ _header("", [(f"blk.{i}.attn_q.weight", [256, 256], F16)]),
+ )
+ _write(tmp_path / "big-00010-of-00010.gguf", _header("", [_DSV41_PLE]))
+ assert glt.local_lookup_table_bytes(first) == DSV41_TABLE
+
+ def test_a_later_split_shard_starts_with_a_small_range(self):
+ names, blobs = _split_blobs()
+ calls = []
+ got = glt.remote_lookup_info(
+ "o/small-range", names[0], identity = 1, fetcher_fn = _blob_fetcher(blobs, calls)
+ )
+ assert got == ("qwen4exp", FLASH_NEXT_TABLE)
+ later = [end + 1 for name, _start, end in calls if name != names[0]]
+ assert later and max(later) <= 64 << 10, calls
+
+ def test_a_known_architecture_reads_the_later_shards_before_the_tokenizer(self):
+ names, blobs = _split_blobs()
+ calls = []
+ got = glt.remote_lookup_info(
+ "o/known-arch",
+ names[0],
+ identity = 1,
+ fetcher_fn = _blob_fetcher(blobs, calls),
+ known_arch = "qwen4exp",
+ )
+ assert got == ("qwen4exp", FLASH_NEXT_TABLE)
+ # The table's shard, then only the first shard's metadata for its architecture.
+ assert [name for name, _s, _e in calls] == [names[1], names[0]]
+ assert calls[1][2] + 1 <= 64 << 10, calls
+
+ def test_the_listing_forwards_the_known_architecture(self, monkeypatch):
+ seen = []
+ _arch(monkeypatch, True)
+ _remote(monkeypatch, lambda *a, **k: seen.append(k.get("known_arch")) or ("qwen4exp", 1))
+ _list([_row("Q4/m.gguf")], "o/forward")
+ assert seen == ["qwen4exp"]
+
+ def test_auto_mode_skips_files_too_small_to_hold_a_streamed_table(self, monkeypatch):
+ read_names = []
+
+ def read(repo, name, *a, **k):
+ read_names.append(name)
+ return ("gemma4", 5 * GIB if name == "BF16.gguf" else 0)
+
+ _arch(monkeypatch, True, "gemma4")
+ _remote(monkeypatch, read)
+ rows = [_row("Q4_K_M.gguf", "Q4_K_M", 3 * GIB), _row("BF16.gguf", "BF16", 15 * GIB)]
+ out = _list(rows, "o/small-files")
+ assert read_names == ["BF16.gguf"]
+ assert _tables(out) == [0, 5 * GIB]
+
+ @pytest.mark.parametrize(
+ "repos",
+ [("o/shared-read", "o/shared-read"), ("O/Cased-GGUF", "o/cased-gguf")],
+ ids = [
+ "a_listing_shares_a_table_read_already_in_flight",
+ "differently_cased_listings_share_one_header_read",
+ ],
+ )
+ def test_a_listing_shares_a_table_read_already_in_flight(self, monkeypatch, repos):
+ calls = []
+ monkeypatch.setattr(
+ gguf_header_delta,
+ "hub_range_fetcher",
+ _blob_fetcher({"Q4/shared.gguf": (_LAZY_BLOB, 40 * GIB)}, calls, delay = 0.3),
+ )
+ _arch(monkeypatch, True)
+ glt._REMOTE_CACHE.clear()
+ # Room for a cold gguf import under a loaded runner; the overlap is what is tested.
+ monkeypatch.setattr(gv, "_LOOKUP_DEADLINE_S", 10.0)
+ rows = [_row("Q4/shared.gguf")]
+ outs = _overlap(_listing(rows, repos[0]), _listing(rows, repos[1]))
+ assert [_tables(o) for o in outs] == [[FLASH_NEXT_TABLE]] * 2
+ assert len(calls) == 1
+
+ def test_a_listing_shares_an_architecture_lookup_already_in_flight(self, monkeypatch):
+ cache: dict = {}
+ lock = threading.Lock()
+ asked = []
+
+ def arch(repo, token):
+ with lock:
+ if repo in cache:
+ return cache[repo]
+ asked.append(repo)
+ time.sleep(0.3)
+ with lock:
+ cache[repo] = "qwen4exp"
+ return "qwen4exp"
+
+ _arch(monkeypatch, False, None)
+ monkeypatch.setattr(gv, "_hub_repo_architecture", arch)
+ _remote(monkeypatch)
+ monkeypatch.setattr(gv, "_LOOKUP_DEADLINE_S", 10.0)
+ rows = [_row()]
+ outs = _overlap(
+ _listing(rows, "o/shared-arch", token = "hf_same"),
+ _listing(rows, "o/shared-arch", token = "hf_same"),
+ )
+ assert [_tables(o) for o in outs] == [[FLASH_NEXT_TABLE]] * 2
+ assert asked == ["o/shared-arch"]
+
+
+class TestListingFollowups:
+ def test_a_repo_hint_never_names_another_familys_split_variant(self):
+ names, blobs = _split_blobs("gemma3n", ("token_embd.weight", [2048, 262144], IQ4_NL))
+ got = glt.remote_lookup_info(
+ "o/mixed-families",
+ names[0],
+ identity = 1,
+ fetcher_fn = _blob_fetcher(blobs, []),
+ known_arch = "qwen4exp",
+ )
+ assert got == ("gemma3n", 0)
+
+ def test_the_remote_cache_ignores_repo_id_case(self):
+ fetcher = _blob_fetcher({"Q4/case.gguf": (_LAZY_BLOB, 40 * GIB)}, [])
+ glt.remote_lookup_info("Unsloth/Case-GGUF", "Q4/case.gguf", identity = 3, fetcher_fn = fetcher)
+ assert glt.remote_cached("unsloth/case-gguf", "Q4/case.gguf", None, 3) == (
+ "qwen4exp",
+ FLASH_NEXT_TABLE,
+ )
+
+ def test_nested_metadata_arrays_are_skipped(self):
+ # Splice one array-of-uint32-arrays KV in after the header's counts.
+ n_tensors, n_kv = struct.unpack_from("<QQ", _LAZY_BLOB, 8)
+ key = b"x.nested"
+ inner = struct.pack("<IQ", 4, 3) + struct.pack("<3I", 1, 2, 3)
+ kv = struct.pack("<Q", len(key)) + key + struct.pack("<IIQ", 9, 9, 2) + inner * 2
+ spliced = _LAZY_BLOB[:8] + struct.pack("<QQ", n_tensors, n_kv + 1) + kv + _LAZY_BLOB[24:]
+ assert glt.parse_header(spliced) == ("qwen4exp", {PLE: FLASH_NEXT_TABLE})
+
+ def test_a_late_release_keeps_a_newer_claim(self):
+ key = ("table", "o/owner")
+ _held, first = gv._claim_lookup(key)
+ gv._release_lookup(key, first)
+ _held, second = gv._claim_lookup(key)
+ gv._release_lookup(key, first) # the cancelled worker, running late
+ assert gv._lookup_inflight.get(key) is second
+ gv._release_lookup(key, second)
+ assert key not in gv._lookup_inflight
+
+ def test_a_listing_retries_a_read_that_failed_while_it_waited(self, monkeypatch):
+ attempts = []
+ lock = threading.Lock()
+ reading = threading.Event()
+
+ def read(repo, name, *a, **k):
+ with lock:
+ attempts.append(name)
+ first = len(attempts) == 1
+ reading.set()
+ time.sleep(0.3)
+ if first:
+ raise OSError("transient")
+ return ("qwen4exp", FLASH_NEXT_TABLE)
+
+ _arch(monkeypatch, True)
+ _remote(monkeypatch, read)
+ monkeypatch.setattr(gv, "_LOOKUP_DEADLINE_S", 10.0)
+ rows = [_row("Q4/retry.gguf")]
+ outs = _overlap(_listing(rows, "o/retry"), _listing(rows, "o/retry"), wait = reading)
+ assert [_tables(o) for o in outs] == [[0], [FLASH_NEXT_TABLE]]
+ assert len(attempts) == 2
+
+ def test_an_unreadable_local_copy_falls_back_to_the_hub_header(self, tmp_path, monkeypatch):
+ """The first shard is on disk, the second still downloading."""
+ first = _write(
+ tmp_path / "m-00001-of-00002.gguf",
+ _header("qwen4exp", [("token_embd.weight", [2048, 151936], IQ4_NL)], _ATTN),
+ )
+ monkeypatch.setattr(gv, "_local_variant_gguf", lambda *a, **k: first)
+ _arch(monkeypatch, True)
+ _remote(monkeypatch)
+ out = _list([_row("m-00001-of-00002.gguf")], "o/partial")
+ assert _tables(out) == [FLASH_NEXT_TABLE]
+
+ def test_a_host_that_reads_tables_resident_reads_no_header(self, monkeypatch):
+ looked = []
+ monkeypatch.setattr(gv, "_host_streams_lookup_tables", lambda: False)
+ monkeypatch.setattr(gv, "_local_variant_gguf", lambda *a, **k: looked.append(a) or None)
+ answer = _answer([_row("Q4/m.gguf")])
+ assert _list(answer, "o/resident") is answer and looked == []
+
+ def test_a_cpu_build_streams_the_tables_beside_an_empty_gpu_inventory(self, monkeypatch):
+ """No model device on a CPU-only build, so lazy auto stays on whatever the GPU probe says."""
+ backend = _stream_backend(monkeypatch, vulkan = False, gpus = [])
+ monkeypatch.setattr(lc, "_gpu_device_node_present", lambda: True)
+ _no_gpu_backend(monkeypatch, True)
+ assert backend.lookup_tables_stream_by_default(None, {}) is True
+ _no_gpu_backend(monkeypatch, False)
+ assert backend.lookup_tables_stream_by_default(["--device", "none"], {}) is True
+ assert backend.lookup_tables_stream_by_default(None, {}) is False
+ cpu_split = ["--split-mode", "none", "--main-gpu", "-1"]
+ assert backend.lookup_tables_stream_by_default(cpu_split, {}) is True
+ # Not under tensor parallel: Studio's --split-mode tensor beats the env.
+ assert (
+ backend.lookup_tables_stream_by_default(
+ ["--main-gpu", "-1"], {"LLAMA_ARG_SPLIT_MODE": "none"}, tensor_parallel = True
+ )
+ is False
+ )
+
+ @pytest.mark.parametrize(
+ "env",
+ [
+ {},
+ {"LLAMA_ARG_SPLIT_MODE": "none", "LLAMA_ARG_MAIN_GPU": "-1"},
+ {"LLAMA_ARG_SPLIT_MODE": "row", "LLAMA_ARG_MAIN_GPU": "-1"},
+ ],
+ ids = ["clean", "inherited_none", "inherited_row"],
+ )
+ def test_a_listing_drops_the_split_mode_the_launch_drops(self, monkeypatch, env):
+ """A layer launch clears an inherited non-layer split mode, so the child runs on the shared Vulkan iGPU."""
+ backend = _stream_backend(monkeypatch, vulkan = True, gpus = [(0, 8000, 0)])
+ _no_gpu_backend(monkeypatch, False)
+ assert backend.lookup_tables_stream_by_default(None, env) is False
diff --git a/studio/backend/tests/test_memory_estimate_contract_freeze.py b/studio/backend/tests/test_memory_estimate_contract_freeze.py
index 25850c846a6..4b47b99df58 100644
--- a/studio/backend/tests/test_memory_estimate_contract_freeze.py
+++ b/studio/backend/tests/test_memory_estimate_contract_freeze.py
@@ -95,6 +95,7 @@ _KV_CACHE_ESTIMATE_KEYS = frozenset(
{
"kv_bytes",
"weights_bytes",
+ "lookup_table_bytes",
"native_context",
"spec_bytes",
"n_ctx",
@@ -136,6 +137,32 @@ class TestTheKvCacheEstimateEnvelope:
"This is a narrowing and it breaks callers."
)
+ def test_the_null_answer_carries_the_same_keys(self, monkeypatch):
+ """An unresolved quant answers every shipped key, nulled."""
+ out = _call_route(
+ monkeypatch,
+ path = "",
+ weights_bytes = 0,
+ repo_id = "unsloth/contract-freeze-GGUF",
+ speculative_type = None,
+ )
+ assert out is not None and out["kv_bytes"] is None
+ assert set(out) == _KV_CACHE_ESTIMATE_KEYS
+
+ def test_no_lookup_table_reads_as_none_not_zero(self, monkeypatch, tmp_path):
+ """The route's rule for every term it itemizes: None for "no such term"."""
+ gguf = _write_gguf(tmp_path / "model-Q4_K_M.gguf", _PLAIN_GQA)
+ _reach_the_planner(monkeypatch, gguf)
+ out = _call_route(
+ monkeypatch,
+ path = gguf,
+ weights_bytes = 4_000_000_000,
+ repo_id = "unsloth/contract-freeze-GGUF",
+ speculative_type = None,
+ )
+ assert out["kv_bytes"] is not None
+ assert out["lookup_table_bytes"] is None
+
def test_weights_bytes_is_the_quant_file_alone(self, monkeypatch, tmp_path):
# Planner reached so the aggregate differs; otherwise this holds vacuously.
quant_size = 4_123_456_789
diff --git a/studio/backend/tests/test_mtp_vram_budget.py b/studio/backend/tests/test_mtp_vram_budget.py
index 9c647e7fa01..8554d1ea447 100644
--- a/studio/backend/tests/test_mtp_vram_budget.py
+++ b/studio/backend/tests/test_mtp_vram_budget.py
@@ -972,15 +972,19 @@ class TestExtraArgsMtpDetection:
def test_load_model_clears_inherited_split_mode_on_layer(self):
compact = "".join(inspect.getsource(LlamaCppBackend.load_model).split())
- assert 'env.get("LLAMA_ARG_SPLIT_MODE")' in compact
- assert '_inherited_sm!="layer"' in compact
- assert 'env.pop("LLAMA_ARG_SPLIT_MODE",None)' in compact
- assert 'env.pop("LLAMA_ARG_TENSOR_SPLIT",None)' in compact
+ assert "self._reconcile_split_mode_env(env,tensor_parallel=tensor_parallel)" in compact
+ for sm in ("none", "row", "tensor", " Row "):
+ env = {"LLAMA_ARG_SPLIT_MODE": sm, "LLAMA_ARG_TENSOR_SPLIT": "1,1", "X": "1"}
+ LlamaCppBackend._reconcile_split_mode_env(env, tensor_parallel = False)
+ assert env == {"X": "1"}, sm
+ env = {"LLAMA_ARG_SPLIT_MODE": "layer", "LLAMA_ARG_TENSOR_SPLIT": "1,1"}
+ LlamaCppBackend._reconcile_split_mode_env(env, tensor_parallel = False)
+ assert env == {"LLAMA_ARG_SPLIT_MODE": "layer", "LLAMA_ARG_TENSOR_SPLIT": "1,1"}
def test_load_model_clears_tensor_split_env_in_tensor_mode(self):
- src = inspect.getsource(LlamaCppBackend.load_model)
- compact = "".join(src.split())
- assert compact.count('env.pop("LLAMA_ARG_TENSOR_SPLIT",None)') >= 2
+ env = {"LLAMA_ARG_SPLIT_MODE": "tensor", "LLAMA_ARG_TENSOR_SPLIT": "1,1"}
+ LlamaCppBackend._reconcile_split_mode_env(env, tensor_parallel = True)
+ assert env == {"LLAMA_ARG_SPLIT_MODE": "tensor"}
def test_load_model_layer_compute_buffer_fallback(self):
compact = "".join(inspect.getsource(LlamaCppBackend.load_model).split())
diff --git a/studio/backend/tests/test_offload_planner.py b/studio/backend/tests/test_offload_planner.py
index 1a26d4f286e..15134ea8fde 100644
--- a/studio/backend/tests/test_offload_planner.py
+++ b/studio/backend/tests/test_offload_planner.py
@@ -927,7 +927,9 @@ def test_the_layout_reader_totals_every_tensor_from_a_real_gguf(tmp_path):
layout = layout_from_gguf(str(path))
assert layout.complete
assert layout.tensor_bytes == 4 * (16 + 1024 + 4096 + 256 + 64)
- assert layout.token_embd_bytes == 4 * (1024 + 4096)
+ assert layout.token_embd_bytes == 4 * 1024
+ assert layout.per_layer_embd_bytes == 4 * 4096
+ assert layout.other_resident_bytes == 4 * (16 + 1024)
@pytest.mark.parametrize("endianess", ["LITTLE", "BIG"])
diff --git a/studio/backend/tests/test_offload_planner_seam.py b/studio/backend/tests/test_offload_planner_seam.py
index c561f67eeb6..680a782720c 100644
--- a/studio/backend/tests/test_offload_planner_seam.py
+++ b/studio/backend/tests/test_offload_planner_seam.py
@@ -1071,8 +1071,8 @@ def test_a_reconciliation_still_strips_a_non_layer_split_mode_env():
"""Reachability anchor for the decline above: if load_model ever stops
scrubbing the inherited mode, the env form becomes plannable again and this is
the test that should fail and say so."""
- src = inspect.getsource(LlamaCppBackend.load_model)
- compact = "".join(src.split())
+ assert "self._reconcile_split_mode_env(env" in inspect.getsource(LlamaCppBackend.load_model)
+ compact = "".join(inspect.getsource(LlamaCppBackend._reconcile_split_mode_env).split())
assert 'if_inherited_smand_inherited_sm!="layer":' in compact
assert 'env.pop("LLAMA_ARG_SPLIT_MODE",None)' in compact
assert 'env.pop("LLAMA_ARG_SPLIT_MODE",None)env.pop("LLAMA_ARG_TENSOR_SPLIT",None)' in compact
diff --git a/studio/backend/tests/test_tensor_parallel.py b/studio/backend/tests/test_tensor_parallel.py
index 6b1c9c7b699..a79a73386f2 100644
--- a/studio/backend/tests/test_tensor_parallel.py
+++ b/studio/backend/tests/test_tensor_parallel.py
@@ -1600,7 +1600,7 @@ def test_load_model_tensor_admission_and_capacity_gate_use_usable_budget():
# The gate's required footprint must include the non-shrinkable MTP reserve,
# not weights alone, or a separate-drafter MTP load can still overcommit.
assert "_tp_mtp_floor" in src
- assert "model_size + _tp_mtp_floor" in src
+ assert "model_size - _host_input_embd + _tp_mtp_floor" in src
def test_load_model_tensor_floor_keeps_flat_reserve_for_weights_only():
diff --git a/studio/frontend/scripts/check-bundle-budget.ts b/studio/frontend/scripts/check-bundle-budget.ts
index 26a86325cde..4d297245fd2 100644
--- a/studio/frontend/scripts/check-bundle-budget.ts
+++ b/studio/frontend/scripts/check-bundle-budget.ts
@@ -424,8 +424,10 @@ export const BUDGET = {
// model picker (6,960,469 -> 6,960,551 B raw, 2,100,277 -> 2,100,453 B transfer against c9ddc5ab5e, one machine and
// build, Node 26, 105 chunks). Raised by that growth.
// Cosmos3 video families (#14554): +466 B raw / +218 B transfer. Raised by that.
- transferBytes: 2_100_715,
- rawBytes: 6_972_038,
+ // Lookup-table pricing (#14465): +1,517 B raw / +470 B transfer on top of the lines above (measured merged with
+ // 4db3eaeb55, Node 24, 99 chunks).
+ transferBytes: 2_101_185,
+ rawBytes: 6_973_555,
};
// Chunk count is not budgeted: splitting a page out raises it while lowering bytes.
diff --git a/studio/frontend/src/features/chat/api/chat-api.ts b/studio/frontend/src/features/chat/api/chat-api.ts
index d70d52f6b32..87f631c0096 100644
--- a/studio/frontend/src/features/chat/api/chat-api.ts
+++ b/studio/frontend/src/features/chat/api/chat-api.ts
@@ -1622,6 +1622,8 @@ export async function listGgufVariants(
export interface KvCacheEstimate {
kv_bytes: number | null;
weights_bytes: number | null;
+ /** Host / disk lookup tables, part of weights_bytes but never on the GPU; null when none. */
+ lookup_table_bytes?: number | null;
native_context: number | null;
/** Extra MTP draft reserve; null for ngram or a model with no MTP head. */
spec_bytes: number | null;
diff --git a/studio/frontend/src/features/chat/types/api.ts b/studio/frontend/src/features/chat/types/api.ts
index 7e8d31b0adf..90c17efc0a1 100644
--- a/studio/frontend/src/features/chat/types/api.ts
+++ b/studio/frontend/src/features/chat/types/api.ts
@@ -163,6 +163,8 @@ export interface GgufVariantDetail {
display_label?: string | null;
size_bytes: number;
download_size_bytes?: number;
+ /** Streamed lookup-table bytes inside `size_bytes`; 0 when none or on an older backend. */
+ lookup_table_bytes?: number;
pending_drafter_filename?: string | null;
pending_drafter_size_bytes?: number;
downloaded?: boolean;
diff --git a/studio/frontend/src/features/hub/catalog/gguf-download-card.tsx b/studio/frontend/src/features/hub/catalog/gguf-download-card.tsx
index ccbcc8444eb..07f55e9b8c6 100644
--- a/studio/frontend/src/features/hub/catalog/gguf-download-card.tsx
+++ b/studio/frontend/src/features/hub/catalog/gguf-download-card.tsx
@@ -33,7 +33,11 @@ import {
import { useVramBudgetFraction } from "@/hooks/use-vram-budget-fraction";
import { ChevronDownStandardIcon } from "@/lib/chevron-icons";
import { copyToClipboard } from "@/lib/copy-to-clipboard";
-import { type GgufFitClass, classifyGgufVariantFit } from "@/lib/gguf-fit";
+import {
+ type GgufFitClass,
+ classifyGgufVariantFit,
+ ggufVariantLookupTableBytes,
+} from "@/lib/gguf-fit";
import { toast } from "@/lib/toast";
import { cn } from "@/lib/utils";
import {
@@ -145,6 +149,12 @@ const FIT_BADGE: Record<GgufFitClass, FitBadgeMeta> = {
"No GPU VRAM detected. This GGUF may run with system RAM and CPU offload. Inference will be slower.",
iconClassName: "text-sky-600 dark:text-sky-400",
},
+ streamed: {
+ label: "Fits (lookup tables streamed from disk)",
+ tooltip:
+ "Fits with its lookup tables streamed from disk, llama.cpp's default for them. Prefill and decode can be slower and vary with disk speed.",
+ iconClassName: "text-sky-600 dark:text-sky-400",
+ },
oom: {
label: "Does not fit",
// Not "won't fit": llama-server never refuses on size, it uses --fit. Matches the chat picker.
@@ -162,12 +172,14 @@ const CHIP_DEFAULT =
function QuantBadge({
quant,
fit,
+ lookupTableBytes = 0,
showFit = true,
variant = "trigger",
tooltipMode = "eager",
}: {
quant: string;
fit: GgufFitClass;
+ lookupTableBytes?: number;
showFit?: boolean;
variant?: "trigger" | "menu";
tooltipMode?: "eager" | "lazy" | "none";
@@ -237,6 +249,13 @@ function QuantBadge({
</TooltipTrigger>
<TooltipContent side="top" sideOffset={4}>
{meta.tooltip}
+ {lookupTableBytes > 0 && (
+ <span className="mt-1 block text-muted-foreground">
+ Lookup tables: {formatFootprintBytes(lookupTableBytes)}
+ {/* Streamed only when RAM cannot hold them; otherwise the load keeps them resident. */}
+ {fit === "streamed" ? ", read from disk" : ", kept in RAM when it fits"}
+ </span>
+ )}
</TooltipContent>
</Tooltip>
);
@@ -248,6 +267,7 @@ interface GgufVariantMenuItem {
quant: string;
label: string;
fit: GgufFitClass;
+ lookupTableBytes: number;
downloaded: boolean;
partial: boolean;
downloadSizeLabel: string;
@@ -347,6 +367,7 @@ function createGgufVariantMenuItems(
quant: variant.quant,
label: ggufVariantDisplayLabel(variant),
fit: classifyGgufVariantFit(variant, resources),
+ lookupTableBytes: ggufVariantLookupTableBytes(variant),
downloaded: Boolean(variant.downloaded),
partial: Boolean(variant.partial),
downloadSizeLabel: ggufVariantTransferLabel(variant),
@@ -580,6 +601,7 @@ const GgufVariantMenuRow = memo(function GgufVariantMenuRow({
<QuantBadge
quant={item.label}
fit={item.fit}
+ lookupTableBytes={item.lookupTableBytes}
showFit={showFitInfo}
variant="menu"
tooltipMode="lazy"
@@ -1144,6 +1166,7 @@ export function GgufDownloadCard({
<QuantBadge
quant={selectedLabel ?? selected.quant}
fit={selectedFit ?? "oom"}
+ lookupTableBytes={ggufVariantLookupTableBytes(selected)}
showFit={showFitInfo}
/>
) : (
@@ -1359,6 +1382,7 @@ export function GgufDownloadCard({
repoId={repoId}
quant={selected.quant}
sizeBytes={selected.size_bytes}
+ lookupTableBytes={ggufVariantLookupTableBytes(selected)}
gpuGb={gpuGb}
showReadout={true}
className="px-1"
diff --git a/studio/frontend/src/features/hub/catalog/local-on-device-card.tsx b/studio/frontend/src/features/hub/catalog/local-on-device-card.tsx
index 77b39ec3397..56c2a287625 100644
--- a/studio/frontend/src/features/hub/catalog/local-on-device-card.tsx
+++ b/studio/frontend/src/features/hub/catalog/local-on-device-card.tsx
@@ -52,6 +52,7 @@ import {
} from "../lib/gguf-filename";
import {
ggufVariantDisplayLabel,
+ localGgufMemoryBarVariant,
resolveLocalGgufVariant,
sortLocalGgufVariants,
} from "../lib/gguf-variant-sort";
@@ -71,6 +72,7 @@ import { PathInfoButton } from "./path-info-button";
import { TransportConflictDialog } from "./transport-conflict-dialog";
import { useCardDelete } from "./use-card-delete";
import { useGgufVariantFetchState } from "./use-gguf-variant-fetch-state";
+import { ggufVariantLookupTableBytes } from "@/lib/gguf-fit";
interface LocalOnDeviceCardProps {
modelId: string;
@@ -349,6 +351,10 @@ export function LocalOnDeviceCard({
...variant,
download_size_bytes:
remoteVariant.download_size_bytes || variant.download_size_bytes,
+ // An update's reading wins, even 0: the local header is the old revision.
+ lookup_table_bytes: remoteVariant.update_available
+ ? (remoteVariant.lookup_table_bytes ?? 0)
+ : variant.lookup_table_bytes || remoteVariant.lookup_table_bytes,
// Both sides measure the same cache; prefer local, fall back to remote.
download_remaining_bytes:
variant.download_remaining_bytes ??
@@ -400,6 +406,10 @@ export function LocalOnDeviceCard({
defaultVariant: currentVariantState.defaultVariant,
});
const selectedQuant = selectedVariant?.quant ?? null;
+ const memoryBarVariant = localGgufMemoryBarVariant(
+ selectedVariant,
+ currentVariantState.variants,
+ );
const updateJobActive = useDownloadManagerStore((s) =>
repoId
? Boolean(
@@ -649,7 +659,12 @@ export function LocalOnDeviceCard({
repoId={repoId || localGgufPath}
loadId={localGgufPath}
quant={selectedQuant ?? ""}
- sizeBytes={selectedVariant?.size_bytes}
+ sizeBytes={memoryBarVariant?.size_bytes}
+ lookupTableBytes={
+ memoryBarVariant
+ ? ggufVariantLookupTableBytes(memoryBarVariant)
+ : 0
+ }
gpuGb={gpuGb}
className="px-3 pb-2"
/>
diff --git a/studio/frontend/src/features/hub/inventory/api.ts b/studio/frontend/src/features/hub/inventory/api.ts
index db56c9d063c..9f78a2c37a1 100644
--- a/studio/frontend/src/features/hub/inventory/api.ts
+++ b/studio/frontend/src/features/hub/inventory/api.ts
@@ -200,6 +200,8 @@ export interface GgufVariantDetail {
display_label?: string | null;
size_bytes: number;
download_size_bytes?: number;
+ /** Streamed lookup-table bytes inside `size_bytes`; 0 when none or on an older backend. */
+ lookup_table_bytes?: number;
pending_drafter_filename?: string | null;
pending_drafter_size_bytes?: number;
download_remaining_bytes?: number | null;
diff --git a/studio/frontend/src/features/hub/lib/gguf-variant-sort.ts b/studio/frontend/src/features/hub/lib/gguf-variant-sort.ts
index e40f243ecc7..006714c84f9 100644
--- a/studio/frontend/src/features/hub/lib/gguf-variant-sort.ts
+++ b/studio/frontend/src/features/hub/lib/gguf-variant-sort.ts
@@ -48,6 +48,7 @@ export function ggufVariantFitRank(
return 1;
case "partial":
case "ram":
+ case "streamed":
return 2;
default:
return 3;
@@ -130,3 +131,19 @@ export function resolveLocalGgufVariant<T extends { quant: string }>(
}
return variants[0] ?? null;
}
+
+// The memory bar loads the cached file, so an update's merged row (published
+// lookup table, cached size) would mix two revisions; read the cached row.
+export function localGgufMemoryBarVariant<T extends { quant: string }>(
+ selected: (T & { update_available?: boolean }) | null,
+ localVariants: readonly T[] | null | undefined,
+): T | null {
+ if (!selected?.update_available) {
+ return selected;
+ }
+ return (
+ localVariants?.find((variant) =>
+ ggufVariantsMatch(variant.quant, selected.quant),
+ ) ?? selected
+ );
+}
diff --git a/studio/frontend/src/features/model-picker/api/memory-estimate.ts b/studio/frontend/src/features/model-picker/api/memory-estimate.ts
index b42482c3822..479106d8454 100644
--- a/studio/frontend/src/features/model-picker/api/memory-estimate.ts
+++ b/studio/frontend/src/features/model-picker/api/memory-estimate.ts
@@ -46,6 +46,9 @@ export interface MemoryEstimate {
gpuLayers: number | null;
/** `--n-cpu-moe` is set, so the GPU figure ignores it and reads high. */
moeOffloadUnmodelled: boolean;
+ lookupTableBytes?: number;
+ /** Tables streamed from disk, so excluded from `weightsBytes` and `totalBytes`. */
+ lookupTableOnDisk?: boolean;
/** MLX: share of the PLE tables / routed experts read from disk, priced in `weightsBytes`. */
mlxPleOffload: number | null;
mlxExpertOffload: number | null;
@@ -140,6 +143,8 @@ interface ApiEstimateResponse {
layer_count: number | null;
gpu_layers: number | null;
moe_offload_unmodelled: boolean;
+ lookup_table_bytes?: number;
+ lookup_table_on_disk?: boolean;
mlx_offload?: { ple?: unknown; experts?: unknown; offloadable?: unknown } | null;
}
@@ -260,6 +265,8 @@ function toMemoryEstimate(body: ApiEstimateResponse): MemoryEstimate {
layerCount: nullableCount(body.layer_count),
gpuLayers: nullableCount(body.gpu_layers),
moeOffloadUnmodelled: flag(body.moe_offload_unmodelled, false),
+ lookupTableBytes: finiteBytes(body.lookup_table_bytes, 0),
+ lookupTableOnDisk: flag(body.lookup_table_on_disk, false),
mlxPleOffload: offloadFraction(body.mlx_offload?.ple),
mlxExpertOffload: offloadFraction(body.mlx_offload?.experts),
mlxOffloadable: offloadableParts(body.mlx_offload?.offloadable),
diff --git a/studio/frontend/src/features/model-picker/components/memory-estimate-row.tsx b/studio/frontend/src/features/model-picker/components/memory-estimate-row.tsx
index ec6b43a1738..317474b7f36 100644
--- a/studio/frontend/src/features/model-picker/components/memory-estimate-row.tsx
+++ b/studio/frontend/src/features/model-picker/components/memory-estimate-row.tsx
@@ -323,6 +323,15 @@ export function MemoryEstimateRow({
value={formatMemoryGb(estimate.projectorRuntimeBytes)}
/>
)}
+ {/* Streamed: outside Weights and Total, only the rows in use page in. Resident:
+ already inside Weights, so the note says so rather than adding it twice. */}
+ {(estimate.lookupTableBytes ?? 0) > 0 && (
+ <MemoryBreakdownLine
+ label="Lookup tables"
+ value={formatMemoryGb(estimate.lookupTableBytes ?? 0)}
+ note={estimate.lookupTableOnDisk ? "on disk" : "in RAM, part of Weights"}
+ />
+ )}
{estimate.drafterRuntimeBytes > 0 && (
<MemoryBreakdownLine
label="Draft cache"
diff --git a/studio/frontend/src/features/model-picker/components/model-selector/gguf-discovery.ts b/studio/frontend/src/features/model-picker/components/model-selector/gguf-discovery.ts
index 578b2fa416e..3512e40379c 100644
--- a/studio/frontend/src/features/model-picker/components/model-selector/gguf-discovery.ts
+++ b/studio/frontend/src/features/model-picker/components/model-selector/gguf-discovery.ts
@@ -21,6 +21,12 @@ function withHubState<V extends GgufVariantDetail>(
download_size_bytes: published.download_size_bytes,
}
: {}),
+ // A newer revision's reading wins, even 0; else local, with the Hub filling an unread 0.
+ lookup_table_bytes: published?.update_available
+ ? (published.lookup_table_bytes ?? 0)
+ : published
+ ? local.lookup_table_bytes || published.lookup_table_bytes
+ : local.lookup_table_bytes,
};
}
diff --git a/studio/frontend/src/features/model-picker/components/model-selector/model-catalog.ts b/studio/frontend/src/features/model-picker/components/model-selector/model-catalog.ts
index 816a0c14da5..0cf6cf0ae0b 100644
--- a/studio/frontend/src/features/model-picker/components/model-selector/model-catalog.ts
+++ b/studio/frontend/src/features/model-picker/components/model-selector/model-catalog.ts
@@ -15,6 +15,7 @@ import {
type GgufFitClass,
type GgufVariantSizes,
classifyGgufFit as classifyGgufFitForDevice,
+ classifyGgufFitWithLookupTables,
ggufVariantFitSizeBytes,
} from "../../../../lib/gguf-fit.ts";
import {
@@ -1349,7 +1350,9 @@ export function recommendedQuantForDevice<T extends GgufVariantSizes>(
preferred: T | null = null,
): T | null {
// Judged on the download footprint, which includes companion GGUFs (mmproj, drafter).
- const fitOfVariant = (variant: T) => fitOf(ggufVariantFitSizeBytes(variant));
+ // A `streamed` quant (tables left on disk) runs but never outranks one that fits fully resident.
+ const fitOfVariant = (variant: T) =>
+ classifyGgufFitWithLookupTables(fitOf, variant);
// Zero size means unknown (old backend or stat error) and must not compete as comfortable.
const sized = variants.filter((variant) => ggufVariantFitSizeBytes(variant) > 0);
if (sized.length === 0) return null;
@@ -1384,6 +1387,7 @@ export interface QuantVariant {
filename: string;
size_bytes: number;
downloaded?: boolean;
+ lookup_table_bytes?: number;
}
/** The quant a bare group/repo click loads: largest downloaded non-OOM quant, else the repo default
@@ -1396,7 +1400,12 @@ export function pickDefaultQuant(
if (!variants || variants.length === 0) return null;
const anyBudget = (budget.gpuGb || 0) > 0 || (budget.systemRamGb || 0) > 0;
const runs = (v: QuantVariant) =>
- ggufFitRuns(classifyGgufFit(v.size_bytes, budget));
+ ggufFitRuns(
+ classifyGgufFitWithLookupTables(
+ (bytes) => classifyGgufFit(bytes, budget),
+ { size_bytes: v.size_bytes, lookup_table_bytes: v.lookup_table_bytes },
+ ),
+ );
const downloadedFitting = variants
.filter((v) => v.downloaded && runs(v))
.sort((a, b) => b.size_bytes - a.size_bytes);
diff --git a/studio/frontend/src/features/model-picker/components/model-selector/pickers.tsx b/studio/frontend/src/features/model-picker/components/model-selector/pickers.tsx
index 8e7546e301e..fc746785825 100644
--- a/studio/frontend/src/features/model-picker/components/model-selector/pickers.tsx
+++ b/studio/frontend/src/features/model-picker/components/model-selector/pickers.tsx
@@ -120,7 +120,8 @@ import { copyToClipboard } from "@/lib/copy-to-clipboard";
import { diffusionRouteSearch } from "@/lib/diffusion-route-search";
import {
type GgufFitClass,
- ggufVariantFitSizeBytes,
+ classifyGgufFitWithLookupTables,
+ ggufVariantLookupTableBytes,
requiredGgufMemoryGb,
} from "@/lib/gguf-fit";
import { formatGiB } from "@/lib/memory/format";
@@ -825,6 +826,11 @@ const VRAM_VERDICT: Record<PickerFitStatus, FitVerdict | null> = {
tone: ORANGE,
hint: "No GPU detected. Runs on system RAM and CPU. Expect much slower inference.",
},
+ streamed: {
+ label: "Fits (lookup tables streamed from disk)",
+ tone: AMBER,
+ hint: "Fits with its lookup tables streamed from disk, llama.cpp's default for them. Prefill and decode can be slower and vary with disk speed.",
+ },
oom: OFFLOADS,
exceeds: WONT_FIT,
};
@@ -2087,7 +2093,7 @@ function GgufVariantExpander({
// One verdict per variant from the download footprint (incl. companions), so badge, order and star agree.
const getVariantFit = useCallback(
(variant: GgufVariantDetail): GgufFitClass =>
- getGgufFit(ggufVariantFitSizeBytes(variant)),
+ classifyGgufFitWithLookupTables(getGgufFit, variant),
[getGgufFit],
);
@@ -4948,11 +4954,13 @@ export function HubModelPicker({
downloaded: ReadonlyMap<string, string | null>;
visionByRepo: ReadonlyMap<string, boolean>;
sizes: ReadonlyMap<string, number>;
+ lookupTables: ReadonlyMap<string, number>;
}>({
validated: false,
downloaded: new Map(),
visionByRepo: new Map(),
sizes: new Map(),
+ lookupTables: new Map(),
});
const prunePinnedQuantValidation = useCallback(
(repoId: string, quant: string) => {
@@ -5001,10 +5009,23 @@ export function HubModelPicker({
(variant) =>
[pinKey(repoId, variant.quant), variant.size_bytes] as const,
),
+ lookupTables: downloaded.map(
+ (variant) =>
+ [
+ pinKey(repoId, variant.quant),
+ ggufVariantLookupTableBytes(variant),
+ ] as const,
+ ),
};
} catch {
// Hide unverifiable quants rather than claim a missing file is downloaded.
- return { repoId, hasVision: undefined, downloaded: [], sizes: [] };
+ return {
+ repoId,
+ hasVision: undefined,
+ downloaded: [],
+ sizes: [],
+ lookupTables: [],
+ };
}
}),
).then((groups) => {
@@ -5020,6 +5041,9 @@ export function HubModelPicker({
),
),
sizes: new Map(groups.flatMap((group) => group.sizes)),
+ lookupTables: new Map(
+ groups.flatMap((group) => group.lookupTables),
+ ),
});
}
});
@@ -6427,7 +6451,14 @@ export function HubModelPicker({
diffusionTaskById.get(entry.repoId.toLowerCase()),
)
? undefined
- : { repoId: entry.repoId, quant: entry.quant }
+ : {
+ repoId: entry.repoId,
+ quant: entry.quant,
+ sizeBytes,
+ lookupTableBytes: pinnedQuantValidation.lookupTables.get(
+ pinKey(entry.repoId, entry.quant),
+ ),
+ }
}
gpuGb={expanderGpuGb}
alignMeta="device"
@@ -6576,6 +6607,7 @@ export function HubModelPicker({
repoId: c.repo_id,
quant: variant.quant,
sizeBytes: variant.size_bytes,
+ lookupTableBytes: ggufVariantLookupTableBytes(variant),
loadId: c.load_id,
}
}
diff --git a/studio/frontend/src/hooks/use-model-memory.ts b/studio/frontend/src/hooks/use-model-memory.ts
index fcbf3daaf39..d5fff8cda03 100644
--- a/studio/frontend/src/hooks/use-model-memory.ts
+++ b/studio/frontend/src/hooks/use-model-memory.ts
@@ -37,6 +37,7 @@ import {
extraArgsAddResidentFiles,
extraArgsOwnPlacement,
extraArgsShapeKvCache,
+ memoryBarWeightsBytes,
} from "@/lib/model-memory";
import { useInferenceGpuInfo } from "./use-gpu-info";
import { useEffect, useMemo, useState, useSyncExternalStore } from "react";
@@ -46,6 +47,7 @@ export interface ModelMemorySource {
quant: string;
/** Size from the listing, so the bar draws before the estimate arrives. */
sizeBytes?: number | null;
+ lookupTableBytes?: number | null;
/** What selecting the row loads, so a duplicate HF cache cannot price a different copy. */
loadId?: string | null;
}
@@ -263,7 +265,11 @@ async function fetchEstimate(
.then((r) => {
const estimate: Estimate = {
kvBytes: r.kv_bytes,
- weightsBytes: r.weights_bytes,
+ // weights_bytes is the whole file; its lookup tables never reach the GPU.
+ weightsBytes:
+ r.weights_bytes == null
+ ? null
+ : Math.max(0, r.weights_bytes - (r.lookup_table_bytes ?? 0)),
specBytes: r.spec_bytes,
nCtx: r.n_ctx ?? 0,
projectorBytes: r.projector_bytes ?? null,
@@ -414,7 +420,7 @@ export function useModelMemory(
if (estimate?.specUnpriced) return computeModelMemory({});
// An env device pin is the one placement override budgetIsMeaningful cannot see.
if (estimate?.inheritedDevicePin) return computeModelMemory({});
- const weights = estimate?.weightsBytes ?? source?.sizeBytes;
+ const weights = memoryBarWeightsBytes(estimate, source);
return computeModelMemory({
// The projector is resident alongside the weights.
weightsBytes:
@@ -441,5 +447,5 @@ export function useModelMemory(
? plan.nCtx == null
: !estimate.contextIsPinned,
});
- }, [plan, entry, source?.sizeBytes, budgetGb, budgetFraction]);
+ }, [plan, entry, source?.sizeBytes, source?.lookupTableBytes, budgetGb, budgetFraction]);
}
diff --git a/studio/frontend/src/lib/gguf-fit.ts b/studio/frontend/src/lib/gguf-fit.ts
index 5d6d7399035..dcdc6df5bfd 100644
--- a/studio/frontend/src/lib/gguf-fit.ts
+++ b/studio/frontend/src/lib/gguf-fit.ts
@@ -3,7 +3,13 @@
/** Mirrors llama_cpp.py `_select_gpus`, so the Hub card and chat picker cannot disagree. */
-export type GgufFitClass = "fits" | "marginal" | "partial" | "ram" | "oom";
+export type GgufFitClass =
+ | "fits"
+ | "marginal"
+ | "partial"
+ | "ram"
+ | "streamed"
+ | "oom";
export interface GgufFitInput {
gpuGb?: number;
@@ -97,6 +103,8 @@ export function classifyGgufFit(
export interface GgufVariantSizes {
size_bytes: number;
download_size_bytes?: number;
+ /** Lookup tables inside `size_bytes` llama.cpp reads lazily from disk; 0 on older backends. */
+ lookup_table_bytes?: number;
}
/** The larger of `size_bytes` and `download_size_bytes` (companions like mmproj or a drafter).
@@ -105,9 +113,30 @@ export function ggufVariantFitSizeBytes(variant: GgufVariantSizes): number {
return Math.max(variant.size_bytes, variant.download_size_bytes ?? 0);
}
+export function ggufVariantLookupTableBytes(variant: GgufVariantSizes): number {
+ const table = Number(variant.lookup_table_bytes ?? 0);
+ if (!Number.isFinite(table) || table <= 0) return 0;
+ return Math.min(table, ggufVariantFitSizeBytes(variant));
+}
+
+export function classifyGgufFitWithLookupTables(
+ fitOf: (sizeBytes: number) => GgufFitClass,
+ variant: GgufVariantSizes,
+): GgufFitClass {
+ const size = ggufVariantFitSizeBytes(variant);
+ const full = fitOf(size);
+ const table = ggufVariantLookupTableBytes(variant);
+ if (table <= 0) return full;
+ if (full !== "oom") return full;
+ return fitOf(size - table) === "oom" ? full : "streamed";
+}
+
export function classifyGgufVariantFit(
variant: GgufVariantSizes,
input: GgufFitInput,
): GgufFitClass {
- return classifyGgufFit(ggufVariantFitSizeBytes(variant), input);
+ return classifyGgufFitWithLookupTables(
+ (sizeBytes) => classifyGgufFit(sizeBytes, input),
+ variant,
+ );
}
diff --git a/studio/frontend/src/lib/model-memory.ts b/studio/frontend/src/lib/model-memory.ts
index 51fdd3ad90e..dbfa6509035 100644
--- a/studio/frontend/src/lib/model-memory.ts
+++ b/studio/frontend/src/lib/model-memory.ts
@@ -80,6 +80,25 @@ const EMPTY: ModelMemorySegments = {
pressure: "normal",
};
+/** VRAM weights segment: only a host-resident lookup table comes off (a streamed one is already out). */
+export function memoryBarWeightsBytes(
+ estimate:
+ | {
+ weightsBytes?: number | null;
+ lookupTableBytes?: number | null;
+ lookupTableOnDisk?: boolean | null;
+ }
+ | undefined,
+ source: { sizeBytes?: number | null; lookupTableBytes?: number | null } | undefined,
+): number | null | undefined {
+ if (estimate?.weightsBytes != null) {
+ const resident = estimate.lookupTableOnDisk ? 0 : (estimate.lookupTableBytes ?? 0);
+ return Math.max(0, estimate.weightsBytes - resident);
+ }
+ const size = source?.sizeBytes;
+ return size == null ? size : Math.max(0, size - (source?.lookupTableBytes ?? 0));
+}
+
function toGb(bytes?: number | null): number {
return bytes && bytes > 0 ? bytes / BYTES_PER_GB : 0;
}
diff --git a/studio/frontend/tests/gguf-fit-lookup-tables.test.ts b/studio/frontend/tests/gguf-fit-lookup-tables.test.ts
new file mode 100644
index 00000000000..32adcd319f0
--- /dev/null
+++ b/studio/frontend/tests/gguf-fit-lookup-tables.test.ts
@@ -0,0 +1,91 @@
+// SPDX-License-Identifier: AGPL-3.0-only
+// Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
+
+// Streamed lookup tables must not be charged as resident. Sizes are real GGUF header figures.
+
+import assert from "node:assert/strict";
+import test from "node:test";
+
+import {
+ ggufFitRuns,
+ pickDefaultQuant,
+ recommendedQuantForDevice,
+} from "../src/features/model-picker/components/model-selector/model-catalog.ts";
+import {
+ classifyGgufFit,
+ classifyGgufFitWithLookupTables,
+ classifyGgufVariantFit,
+ ggufVariantLookupTableBytes,
+} from "../src/lib/gguf-fit.ts";
+
+const GIB = 1024 ** 3;
+// 24 GB card + 64 GB RAM: Flash-Next without the table (54.6 GiB) just clears 23.28 + 32.
+const HOST_24_64 = { gpuGb: 24, systemRamGb: 64 };
+
+const flashNext = {
+ quant: "UD-Q2_K_XL",
+ size_bytes: 73.45 * GIB,
+ lookup_table_bytes: 26.82 * GIB,
+};
+const dsv41 = {
+ quant: "UD-IQ1_S",
+ size_bytes: 216.98 * GIB,
+ lookup_table_bytes: 103.0 * GIB,
+};
+const gemma3n = { quant: "Q4_K_M", size_bytes: 4.23 * GIB };
+
+test("Flash-Next on 24 GB + 64 GB runs with its table on disk instead of reading OOM", () => {
+ assert.equal(classifyGgufFit(flashNext.size_bytes, HOST_24_64), "oom");
+ assert.equal(classifyGgufVariantFit(flashNext, HOST_24_64), "streamed");
+ assert.equal(ggufFitRuns("streamed"), true);
+});
+
+test("DeepSeek-V4.1 UD-IQ1_S on 24 GB + 64 GB stays OOM without the Engram table", () => {
+ assert.equal(classifyGgufVariantFit(dsv41, HOST_24_64), "oom");
+});
+
+test("a variant with no table keeps today's verdict", () => {
+ const host = { gpuGb: 8, systemRamGb: 16 };
+ assert.equal(
+ classifyGgufVariantFit(gemma3n, host),
+ classifyGgufFit(gemma3n.size_bytes, host),
+ );
+ assert.equal(classifyGgufVariantFit(gemma3n, host), "fits");
+ assert.equal(ggufVariantLookupTableBytes(gemma3n), 0);
+});
+
+test("a table that would not change the verdict leaves it alone", () => {
+ const big = { gpuGb: 192, systemRamGb: 256 };
+ assert.equal(classifyGgufVariantFit(flashNext, big), "fits");
+});
+
+test("a streamed quant runs but ranks below one that fits with everything resident", () => {
+ const fitOf = (bytes: number) => classifyGgufFit(bytes, HOST_24_64);
+ const comfortable = { quant: "Q2", size_bytes: 15 * GIB, lookup_table_bytes: 0 };
+ assert.equal(recommendedQuantForDevice([flashNext], fitOf), flashNext);
+ assert.equal(recommendedQuantForDevice([flashNext, comfortable], fitOf), comfortable);
+ const smaller = { quant: "UD-IQ1_S", size_bytes: 30 * GIB, lookup_table_bytes: 0 };
+ assert.notEqual(classifyGgufVariantFit(smaller, HOST_24_64), "streamed");
+});
+
+test("a resident fit the table only improves keeps its own verdict", () => {
+ const fitOf = (bytes: number) =>
+ bytes > 60 * GIB ? "partial" : bytes > 40 * GIB ? "marginal" : "fits";
+ const variant = { quant: "Q4", size_bytes: 70 * GIB, lookup_table_bytes: 20 * GIB };
+ assert.equal(classifyGgufFitWithLookupTables(fitOf, variant), "partial");
+});
+
+test("a malformed table size is ignored", () => {
+ const bytes = (lookup_table_bytes: number) =>
+ ggufVariantLookupTableBytes({ size_bytes: GIB, lookup_table_bytes });
+ assert.equal(bytes(Number.NaN), 0);
+ assert.equal(bytes(5 * GIB), GIB);
+});
+
+test("a bare repo click picks the streamed quant rather than the smallest fallback", () => {
+ const variants = [
+ { ...flashNext, filename: "Q2/a-00001-of-00003.gguf", downloaded: true },
+ { quant: "Q8_0", filename: "Q8/a.gguf", size_bytes: 160 * GIB, lookup_table_bytes: 26.82 * GIB },
+ ];
+ assert.equal(pickDefaultQuant(variants, "Q8_0", HOST_24_64)?.quant, "UD-Q2_K_XL");
+});
diff --git a/studio/frontend/tests/hub-local-gguf-selection.test.ts b/studio/frontend/tests/hub-local-gguf-selection.test.ts
index 56cbc536521..a7f8127becc 100644
--- a/studio/frontend/tests/hub-local-gguf-selection.test.ts
+++ b/studio/frontend/tests/hub-local-gguf-selection.test.ts
@@ -143,3 +143,37 @@ test("a blank-quant variant remains eligible for the ranked fallback", () => {
const sorted = sortLocalGgufVariants([oversized, blank], { gpuGb: 24 });
assert.equal(resolveLocalGgufVariant(sorted, {}), blank);
});
+
+test("the memory bar reads the cached revision while an update is published", async () => {
+ const { localGgufMemoryBarVariant } = await import(
+ "../src/features/hub/lib/gguf-variant-sort.ts"
+ );
+ const { ggufVariantLookupTableBytes } = await import(
+ "../src/lib/gguf-fit.ts"
+ );
+ const GiB = 1024 ** 3;
+ const cached = {
+ quant: "Q4_K_M",
+ size_bytes: 70 * GiB,
+ lookup_table_bytes: 20 * GiB,
+ };
+ // The merged row keeps the cached size but carries the published table.
+ const merged = {
+ ...cached,
+ lookup_table_bytes: 30 * GiB,
+ download_size_bytes: 80 * GiB,
+ update_available: true,
+ };
+ const barVariant = localGgufMemoryBarVariant(merged, [
+ { quant: "Q8_0", size_bytes: 1, lookup_table_bytes: 0 },
+ cached,
+ ]);
+ assert.equal(barVariant?.size_bytes, 70 * GiB);
+ assert.equal(ggufVariantLookupTableBytes(barVariant!), 20 * GiB);
+
+ // No update, or no cached row, keeps the selected variant.
+ const current = { ...cached, update_available: false };
+ assert.equal(localGgufMemoryBarVariant(current, [cached]), current);
+ assert.equal(localGgufMemoryBarVariant(merged, null), merged);
+ assert.equal(localGgufMemoryBarVariant(null, [cached]), null);
+});
diff --git a/studio/frontend/tests/hub-row-selection.test.ts b/studio/frontend/tests/hub-row-selection.test.ts
index 1163c284513..78bb8795764 100644
--- a/studio/frontend/tests/hub-row-selection.test.ts
+++ b/studio/frontend/tests/hub-row-selection.test.ts
@@ -48,8 +48,8 @@ test("the picker reads one fit per variant, companions included", () => {
const at = PICKERS.indexOf("const getVariantFit = useCallback(");
assert.notEqual(at, -1, "getVariantFit is missing");
assert.match(
- PICKERS.slice(at, at + 260),
- /ggufVariantFitSizeBytes\(variant\)/,
+ PICKERS.slice(at, at + 400),
+ /ggufVariantFitSizeBytes\(variant\)|classifyGgufFitWithLookupTables\(getGgufFit, variant\)/,
);
assert.equal(PICKERS.match(/getVariantFit\(v\)/g)?.length, 2);
});
diff --git a/studio/frontend/tests/lookup-table-memory-bar.test.ts b/studio/frontend/tests/lookup-table-memory-bar.test.ts
new file mode 100644
index 00000000000..92e62e08ccc
--- /dev/null
+++ b/studio/frontend/tests/lookup-table-memory-bar.test.ts
@@ -0,0 +1,79 @@
+// SPDX-License-Identifier: AGPL-3.0-only
+// Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
+
+// Lookup tables never sit in VRAM. Sizes are real GGUF header figures.
+
+import assert from "node:assert/strict";
+import test from "node:test";
+
+import type { KvCacheEstimate } from "../src/features/chat/api/chat-api.ts";
+import { readSrcAsync, registerBundlerResolver } from "./helpers/kit.ts";
+
+registerBundlerResolver();
+
+const { memoryBarWeightsBytes } = await import("../src/lib/model-memory.ts");
+const pickers = await readSrcAsync(
+ "features/model-picker/components/model-selector/pickers.tsx",
+);
+
+const GIB = 1024 ** 3;
+const FLASH = { sizeBytes: 73.45 * GIB, lookupTableBytes: 26.82 * GIB };
+
+test("before the estimate, the listing's streamed table comes off the file size", () => {
+ assert.equal(
+ memoryBarWeightsBytes(undefined, FLASH),
+ FLASH.sizeBytes - FLASH.lookupTableBytes,
+ );
+ assert.equal(memoryBarWeightsBytes(undefined, {}), undefined);
+});
+
+test("a streamed table the estimate already left out is not taken off twice", () => {
+ const weightsBytes = FLASH.sizeBytes - FLASH.lookupTableBytes;
+ const estimate = {
+ weightsBytes,
+ lookupTableBytes: FLASH.lookupTableBytes,
+ lookupTableOnDisk: true,
+ };
+ assert.equal(memoryBarWeightsBytes(estimate, FLASH), weightsBytes);
+});
+
+test("a host-resident table inside the estimate's weights is not charted as VRAM", () => {
+ assert.equal(
+ memoryBarWeightsBytes(
+ { weightsBytes: 4.23 * GIB, lookupTableBytes: 1.64 * GIB, lookupTableOnDisk: false },
+ { sizeBytes: 4.23 * GIB, lookupTableBytes: 0 },
+ ),
+ 4.23 * GIB - 1.64 * GIB,
+ );
+});
+
+test("a pinned quant row hands the bar the same listing figures as the sole-quant row", () => {
+ const start = pickers.indexOf("const renderPinnedQuantRow");
+ assert.ok(start > 0);
+ const body = pickers.slice(start, pickers.indexOf("gpuGb={expanderGpuGb}", start));
+ assert.match(body, /repoId: entry\.repoId,\s*quant: entry\.quant,[\s\S]*sizeBytes,/);
+ assert.match(body, /lookupTableBytes: pinnedQuantValidation\.lookupTables\.get\(/);
+ assert.match(pickers, /lookupTables: downloaded\.map\([\s\S]*?ggufVariantLookupTableBytes\(variant\)/);
+});
+
+test("the on-device card takes an update's table reading, a zero included", async () => {
+ const card = await readSrcAsync("features/hub/catalog/local-on-device-card.tsx");
+ assert.match(
+ card,
+ /lookup_table_bytes: remoteVariant\.update_available\s*\?\s*\(remoteVariant\.lookup_table_bytes \?\? 0\)\s*:\s*variant\.lookup_table_bytes \|\| remoteVariant\.lookup_table_bytes/,
+ );
+});
+
+test("the memory bar's estimate takes the lookup table off the whole-file weights", async () => {
+ // GET /kv-cache-estimate's weights_bytes is the quant file alone, streamed or resident table included.
+ const hook = await readSrcAsync("hooks/use-model-memory.ts");
+ assert.match(
+ hook,
+ /weightsBytes:\s*r\.weights_bytes == null\s*\?\s*null\s*:\s*Math\.max\(0, r\.weights_bytes - \(r\.lookup_table_bytes \?\? 0\)\)/,
+ );
+});
+
+test("the kv-cache-estimate type carries the route's null for a model without a table", () => {
+ const noTable: Pick<KvCacheEstimate, "lookup_table_bytes"> = { lookup_table_bytes: null };
+ assert.equal(noTable.lookup_table_bytes ?? 0, 0);
+});
diff --git a/studio/frontend/tests/picker-gguf-discovery.test.ts b/studio/frontend/tests/picker-gguf-discovery.test.ts
index 0e7eb21979b..f522beed83c 100644
--- a/studio/frontend/tests/picker-gguf-discovery.test.ts
+++ b/studio/frontend/tests/picker-gguf-discovery.test.ts
@@ -314,3 +314,50 @@ test("an up-to-date cached quant keeps the Hub's companion-inclusive size", asyn
assert.equal(result.variants[0].download_size_bytes, 900);
assert.equal(result.variants[0].cache_path, "/cache/old");
});
+
+const GIB = 1024 ** 3;
+type Variant = GgufVariantsResponse["variants"][number];
+
+async function mergedRow(cached: Partial<Variant>, hub: Partial<Variant>) {
+ const merged = await loadPickerGgufVariants(
+ async (localOnly) => ({
+ ...local,
+ variants: [{ ...local.variants[0], ...(localOnly ? cached : hub) }],
+ }),
+ options,
+ () => {},
+ );
+ return merged.variants[0];
+}
+
+test("an update's table size travels with the update's file size", async () => {
+ const row = await mergedRow(
+ { size_bytes: 70 * GIB, lookup_table_bytes: 20 * GIB },
+ { size_bytes: 80 * GIB, lookup_table_bytes: 30 * GIB, update_available: true },
+ );
+ assert.equal(row.size_bytes, 80 * GIB);
+ assert.equal(row.lookup_table_bytes, 30 * GIB);
+});
+
+for (const [name, hub, table] of [
+ [
+ "a Hub row with no table reading keeps the local header's",
+ { lookup_table_bytes: 0 },
+ 20 * GIB,
+ ],
+ [
+ "an up-to-date local header's table wins over the Hub's bounded reading",
+ { lookup_table_bytes: 12 * GIB },
+ 20 * GIB,
+ ],
+ [
+ "an update that drops the table is taken at its word",
+ { lookup_table_bytes: 0, update_available: true },
+ 0,
+ ],
+] as const) {
+ test(name, async () => {
+ const row = await mergedRow({ lookup_table_bytes: 20 * GIB }, hub);
+ assert.equal(row.lookup_table_bytes, table);
+ });
+}