Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

Studio: stop charging CPU-mapped input embeddings to the Metal context budget (#11060)

oobabooga71c9a44cc0fa
diff --git a/studio/backend/core/inference/llama_cpp.py b/studio/backend/core/inference/llama_cpp.py
index 70b5edcf5bd..6b9f915ef57 100644
--- a/studio/backend/core/inference/llama_cpp.py
+++ b/studio/backend/core/inference/llama_cpp.py
@@ -4068,6 +4068,13 @@ _GPU_OFFLOAD_OVERRIDE_FLAGS = _LAYER_OFFLOAD_FLAGS
# presence-only (-cmoe takes no value, and an -ot pattern is not worth guessing
# at, which matches how the env side reads LLAMA_ARG_OVERRIDE_TENSOR).
_CPU_MOE_COUNT_FLAGS = frozenset({"-ncmoe", "--n-cpu-moe"})
+# The dense-model twin: common/arg.cpp turns a positive -ncffn into CPU overrides for the
+# first N layers' ffn_up/down/gate, exactly as -ncmoe does for expert weights.
+_CPU_FFN_COUNT_FLAGS = frozenset({"-ncffn", "--n-cpu-ffn"})
+# User controls over llama.cpp's own fitter. Studio emits neither on Metal, so a launch carrying
+# one is placed by a fitter the Metal memory probe did not run.
+_FIT_CONTROL_FLAGS = frozenset({"-fitt", "--fit-target", "-fitc", "--fit-ctx"})
+_FIT_CONTROL_ENV_VARS = ("LLAMA_ARG_FIT_TARGET", "LLAMA_ARG_FIT_CTX")
_CPU_PLACEMENT_PRESENCE_FLAGS = (_MOE_OFFLOAD_FLAGS - _CPU_MOE_COUNT_FLAGS) | frozenset(
{"-ot", "--override-tensor"}
)
@@ -4507,7 +4514,7 @@ def _args_place_tensors_on_cpu(extra_args: Optional[Iterable[str]] = None) -> bo
"""True when extras keep weights on the CPU, whatever the layer count says.
The arg-side twin of ``_env_places_tensors_on_cpu``, including its rule that
- a CPU-MoE count only counts when positive: ``--n-cpu-moe 0`` places nothing.
+ a CPU-MoE or CPU-FFN count only counts when positive: ``--n-cpu-moe 0`` places nothing.
"""
values = [str(raw) for raw in extra_args or ()]
for i, raw in enumerate(values):
@@ -4516,7 +4523,7 @@ def _args_place_tensors_on_cpu(extra_args: Optional[Iterable[str]] = None) -> bo
# non-empty, and guessing at the pattern is not worth it.
if flag in _CPU_PLACEMENT_PRESENCE_FLAGS:
return True
- if flag in _CPU_MOE_COUNT_FLAGS:
+ if flag in _CPU_MOE_COUNT_FLAGS | _CPU_FFN_COUNT_FLAGS:
_, eq, inline = raw.partition("=")
if _is_positive_int(inline if eq else (values[i + 1] if i + 1 < len(values) else "")):
return True
@@ -4535,6 +4542,7 @@ def _env_places_tensors_on_cpu(env: Optional[Mapping[str, str]] = None) -> bool:
source_env.get("LLAMA_ARG_OVERRIDE_TENSOR")
or source_env.get("LLAMA_ARG_CPU_MOE") in _LLAMA_ARG_TRUE_VALUES
or _is_positive_int(source_env.get("LLAMA_ARG_N_CPU_MOE"))
+ or _is_positive_int(source_env.get("LLAMA_ARG_N_CPU_FFN"))
)
@@ -5385,6 +5393,51 @@ def _normalized_load_mode(value: Optional[str]) -> Optional[str]:
return None if mode in {"", "auto"} else mode
+_BREAKDOWN_DEVICE_ROW_RE = re.compile(
+ r"\|\s*-\s*(?P<name>[^|(]+?)\s*\([^|]*\|\s*\d+\s*=\s*\d+\s*\+\s*\(\s*\d+\s*=\s*(?P<model>\d+)\s*\+"
+)
+_BREAKDOWN_HOST_ROW_RE = re.compile(
+ r"\|\s*-\s*(?P<name>[^|]+?)\s*\|\s*\d+\s*=\s*(?P<model>\d+)\s*\+"
+)
+
+
+def _parse_metal_memory_breakdown(output: str) -> Optional[tuple[int, int, int]]:
+ """Parse the first llama.cpp table as ``(metal_model_mib, host_model_mib, rows)``.
+
+ Require one Metal device row and sum the model column from every host row.
+ """
+ rows: list[str] = []
+ for line in output.splitlines():
+ if "common_memory_breakdown_print" not in line:
+ if rows:
+ break
+ continue
+ if "memory breakdown" in line:
+ if rows:
+ break
+ continue
+ rows.append(line)
+ metal: list[int] = []
+ host = 0
+ host_rows = 0
+ for row in rows:
+ device = _BREAKDOWN_DEVICE_ROW_RE.search(row)
+ if device:
+ if not device.group("name").strip().startswith("MTL"):
+ return None
+ metal.append(int(device.group("model")))
+ continue
+ hosted = _BREAKDOWN_HOST_ROW_RE.search(row)
+ if hosted:
+ host += int(hosted.group("model"))
+ host_rows += 1
+ continue
+ return None
+ if len(metal) != 1 or host_rows == 0:
+ return None
+ return metal[0], host, len(rows)
+
+
# The KV cache dtypes llama.cpp can map, shared by the emission guard and the
# budget. Module level because both the draft-cache block and the MTP reserve
# need it, and the main cache path's copy is a local inside load_model.
@@ -11461,6 +11514,150 @@ class LlamaCppBackend:
rec_bytes = min(rec_bytes, available)
return int(rec_bytes * _APPLE_UNIFIED_MEMORY_FRACTION)
+ def _metal_measured_model_mib(
+ self, binary: Optional[str], model_path: Optional[str]
+ ) -> Optional[tuple[int, int, int]]:
+ """Measure full-offload model buffers with the sibling ``llama-fit-params``.
+
+ Discounted launches pin the same placement. Results are cached by probe and model
+ identity; a missing, failed, or unreadable probe returns None.
+ """
+ if not binary or not model_path:
+ return None
+ try:
+ probe = _llama_lib_dir(binary) / "llama-fit-params"
+ probe_stat = probe.stat()
+ model_stat = os.stat(model_path)
+ except OSError:
+ return None
+ key = (
+ str(probe),
+ probe_stat.st_size,
+ probe_stat.st_mtime_ns,
+ model_path,
+ model_stat.st_size,
+ model_stat.st_mtime_ns,
+ )
+ cached = getattr(self, "_metal_model_mib_cache", None)
+ if cached is not None and cached[0] == key:
+ return cached[1]
+ measured: Optional[tuple[int, int, int]] = None
+ try:
+ env = self._llama_server_env_for_binary(str(binary))
+ for name in tuple(env):
+ if name.startswith("LLAMA_ARG_"):
+ env.pop(name, None)
+ result = subprocess.run(
+ [str(probe), "-m", model_path, "-ngl", "999", "-c", "512", "-lv", "4"],
+ capture_output = True,
+ text = True,
+ encoding = "utf-8",
+ errors = "replace",
+ # A cold Metal shader compile lands here instead of at the launch.
+ timeout = 120,
+ check = False,
+ env = env,
+ **_windows_hidden_subprocess_kwargs(),
+ )
+ if result.returncode == 0:
+ measured = _parse_metal_memory_breakdown(
+ (result.stdout or "") + "\n" + (result.stderr or "")
+ )
+ except Exception as e:
+ logger.debug(f"llama-fit-params memory probe failed: {e}")
+ self._metal_model_mib_cache = (key, measured)
+ return measured
+
+ def _metal_demand_paged_embedding_bytes(
+ self,
+ model_path: Optional[str],
+ extra_args: Optional[Iterable[str]],
+ *,
+ binary: Optional[str],
+ requested_load_mode: Optional[str],
+ supports_load_mode: bool,
+ settings: tuple[bool, bool],
+ layers_fixed: bool,
+ mtp_may_engage: bool,
+ env: Optional[Mapping[str, str]] = None,
+ ) -> int:
+ """Return mmap embedding bytes proven absent from the Metal working set.
+
+ Input embeddings live on the CPU but can still fall inside a Metal file mapping.
+ Fail closed unless the probe matches the launch placement and the loader leaves the
+ embeddings pageable. CPU fallback weights remain charged.
+ """
+ source_env = os.environ if env is None else env
+ if (
+ layers_fixed
+ or _extra_args_set_any_flag(extra_args, _GPU_LAYER_FLAGS)
+ or _env_fixes_gpu_layers(source_env)
+ or _device_selection_is_cpu(extra_args, source_env)
+ or _args_place_tensors_on_cpu(extra_args)
+ or _env_places_tensors_on_cpu(source_env)
+ or _extra_args_set_any_flag(extra_args, _FIT_CONTROL_FLAGS)
+ or any(source_env.get(name) for name in _FIT_CONTROL_ENV_VARS)
+ # Extras are appended after the pin, while inherited values are overridden by it.
+ or fit_is_enabled_in(extra_args)
+ # A pass-through adapter is resident on the base tensor's buffer
+ # (llama-adapter.cpp:335, :67) and neither the probe nor model_size carries it,
+ # and nothing on the Apple path charges it the way _fits_without_paging does
+ # off Metal. Spending the discount would hand that budget to the KV cache.
+ or _sidecar_adapter_paths(extra_args)
+ ):
+ return 0
+ env_view = dict(source_env)
+ scrub_memory_env(env_view, settings)
+ managed, extras = apply_model_memory_policy(
+ extra_args,
+ supports_load_mode = supports_load_mode,
+ weights_in_host_memory = True,
+ gpu_offload_confirmed = False,
+ env = env_view,
+ settings = settings,
+ )
+ selected, extras = apply_load_mode_policy(
+ extras,
+ supports_load_mode = supports_load_mode,
+ weights_in_host_memory = True,
+ requested_load_mode = requested_load_mode,
+ settings = settings,
+ )
+ mlock, reserves_ram, direct_io = resolve_effective_load_state(
+ [*managed, *selected, *extras], env_view
+ )
+ if mlock or reserves_ram or direct_io:
+ return 0
+ layout = self._tensor_spill_layout(model_path, all_shards = True)
+ if layout is None or not layout.complete:
+ return 0
+ embeddings = int(layout.token_embd_bytes)
+ if embeddings <= 0 or (mtp_may_engage and layout.has_excluded_blocks):
+ return 0
+ measured = self._metal_measured_model_mib(binary, model_path)
+ if measured is None:
+ logger.info(
+ "Metal fit: could not measure how llama.cpp maps this model (llama-fit-params "
+ "missing or unreadable), so its %.2f GB of input embeddings stay charged.",
+ embeddings / (1024**3),
+ )
+ return 0
+ metal_mib, host_mib, rows = measured
+ mib = 1024 * 1024
+ charged = (metal_mib + rows) * mib + max(0, host_mib * mib - embeddings)
+ # TENSOR_SKIP returns before the tensor exists (llama-model-loader.cpp:1125-1131), so a
+ # skipped block reaches no buffer and no table row. Those bytes are in the file and not
+ # in the probe, which is the signature the embeddings are being recognised by.
+ mapped = int(layout.tensor_bytes) - int(layout.excluded_block_bytes)
+ unmapped = min(embeddings, max(0, mapped - charged))
+ if not unmapped:
+ logger.info(
+ "Metal fit: llama.cpp maps this model's %.2f GB of input embeddings into Metal, "
+ "so they stay charged. A llama.cpp update may remove that.",
+ embeddings / (1024**3),
+ )
+ return unmapped
+
@staticmethod
def _rocm_arch_gate_keep(
binary: Optional[str], torch_mod, for_llama_server: bool
@@ -22283,6 +22480,14 @@ class LlamaCppBackend:
# the handler's own log line on purpose: test_tp_vision_regression
# string-searches this function's source for it to check ordering.)
_metal_ctx_refusal: Optional[str] = None
+ # A discounted Metal ceiling must launch the measured full-offload placement.
+ _metal_full_offload_pinned = False
+ # The fit and launch must use the same Model Memory snapshot.
+ from utils.model_memory_settings import capture_model_memory_settings
+
+ _mem_settings = capture_model_memory_settings(
+ lambda pair: setattr(self, "_memory_pending_launch", pair)
+ )
try:
gguf_size = self._get_gguf_size_bytes(model_path)
# Include GPU-loaded mmproj in the fit budget (#5825). GPU-loaded is
@@ -23900,7 +24105,25 @@ class LlamaCppBackend:
# this arm is reached only with no GPU enumerated, so no device has
# a budget of its own and _shared_pool_mmproj charged those bytes.
# Adding them again would price the encoder twice.
- _apple_model_size_fit = model_size_fit
+ # Exclude pageable input embeddings that the probe found outside Metal.
+ _apple_host_embd = self._metal_demand_paged_embedding_bytes(
+ model_path,
+ extra_args,
+ binary = binary,
+ requested_load_mode = load_mode,
+ supports_load_mode = bool(server_caps.get("supports_load_mode")),
+ settings = _mem_settings,
+ layers_fixed = _caller_owns_budget or _paravirtual_cpu_forced,
+ mtp_may_engage = _mtp_will_engage,
+ )
+ _apple_model_size_fit = model_size_fit - _apple_host_embd
+ if _apple_host_embd:
+ logger.info(
+ "Metal fit: %.2f GB of input embeddings stay in the CPU "
+ "file mapping; charging %.2f GB of weights to unified memory.",
+ _apple_host_embd / (1024**3),
+ _apple_model_size_fit / (1024**3),
+ )
def _apple_ctx_fit(target: int, min_ctx: int) -> int:
return self._fit_context_to_vram(
@@ -23963,7 +24186,7 @@ class LlamaCppBackend:
# and whenever the GGUF carries no context length and the
# request itself is that small.
_weights_over_budget = (
- model_size_fit / (1024 * 1024)
+ _apple_model_size_fit / (1024 * 1024)
) > _apple_fit_budget_mib
if _weights_over_budget:
# The fit priced nothing: no context rescues a load
@@ -24062,6 +24285,10 @@ class LlamaCppBackend:
cache_type_kv,
nothing_fits = _apple_nothing_fits,
)
+ # Unmeasured floors still rely on llama.cpp's fitter.
+ _metal_full_offload_pinned = bool(
+ _apple_host_embd and _apple_measured_ceiling is not None
+ )
# Prefer fewer serving slots on GPU over --fit on offload: when the extra
# --parallel slots push the footprint past the pin budget, llama-server
@@ -24576,6 +24803,7 @@ class LlamaCppBackend:
except Exception as e:
logger.warning(f"GPU selection failed ({e}), using --fit on")
gpu_indices, use_fit = None, True
+ _metal_full_offload_pinned = False
# Not a verdict: this arm never priced anything. Explicit, so
# the flag cannot survive from a previous load.
_placement_verdict_partial = False
@@ -24986,6 +25214,17 @@ class LlamaCppBackend:
# Single effective GPU: the split is never emitted, so
# don't report it as active via /status and /load.
self._tensor_split = None
+ elif use_fit and _metal_full_offload_pinned:
+ # Keep the launch on the full-offload mapping the discount measured.
+ cmd.extend(["-ngl", "-1", "--fit", "off"])
+ # Same claim as the pinned-GPU arm below, so it owes the same
+ # recovery: the --fit on retry after a startup crash is gated on
+ # this flag, and without it an optimistic discount would skip
+ # straight to the terminal fallbacks. -1 is llama.cpp's own
+ # default (llama_model_default_params), so the retry's fitter
+ # does not hit its "n_gpu_layers already set by user" abort
+ # (common/fit.cpp:462) and can still move layers off Metal.
+ fully_gpu_offloaded = True
elif use_fit:
# Unsloth could not prove a fit, so llama.cpp's fitter takes the
# placement. Its dense path fills "back to front with dense
@@ -25758,8 +25997,6 @@ class LlamaCppBackend:
# GPU with full offload it would hold a second copy of the model
# in system RAM and do nothing for VRAM, so it is not emitted
# and the idle-unload veto carries residency by itself.
- from utils.model_memory_settings import capture_model_memory_settings
-
# fully_gpu_offloaded is only set by the auto branch. Manual mode
# and a user -ngl reach the same placement by their own routes,
# so derive those too or they would still be pinned. The CPU
@@ -25786,13 +26023,6 @@ class LlamaCppBackend:
if gpu_ids is not None:
_mem_extra_args = self._strip_device_extra_args(extra_args)
self._clear_device_placement_env(_mem_env)
- # ONE read for every decision below, published as one act: the window a
- # save must not fall through opens at the CAPTURE, not the spawn. It
- # carries the pair because `_memory_state` is None until the flags
- # resolve, which the comparator reads as "not governed".
- _mem_settings = capture_model_memory_settings(
- lambda pair: setattr(self, "_memory_pending_launch", pair)
- )
_mem_keep_resident, _mem_no_reserve = _mem_settings
_mem_should_mlock = _mem_keep_resident and not _mem_no_reserve
# Armed HERE, not at the load call: that also spans the Hub download, and
diff --git a/studio/backend/core/inference/offload_layout.py b/studio/backend/core/inference/offload_layout.py
index 1b98122c948..4a099fb0c09 100644
--- a/studio/backend/core/inference/offload_layout.py
+++ b/studio/backend/core/inference/offload_layout.py
@@ -56,6 +56,8 @@ class ModelLayout:
# llama-model.cpp pins dev_input to the CPU unconditionally, so this is never charged to VRAM. Tracked because it IS
# charged to host RAM.
token_embd_bytes: int = 0
+ # Every tensor's bytes, excluded blocks included; the file size less its metadata.
+ tensor_bytes: int = 0
# output_norm and friends: GPU-resident, too small to be worth spilling.
other_resident_bytes: int = 0
# Attention cache for ONE token at f16, across the attention layers only.
@@ -256,7 +258,8 @@ def _layout_from_readers(readers) -> ModelLayout:
else:
resident[index] = resident.get(index, 0) + nbytes
continue
- if "token_embd" in name:
+ # token_embd_norm is a repeating-layer tensor, not a host-pinned input embedding.
+ if "token_embd" in name and not name.startswith("token_embd_norm"):
token_embd += nbytes
elif name == "output.weight":
lm_head += nbytes
@@ -306,6 +309,7 @@ def _layout_from_readers(readers) -> ModelLayout:
blocks = blocks,
lm_head_bytes = lm_head,
token_embd_bytes = token_embd,
+ tensor_bytes = sum(int(t.n_bytes) for r in readers for t in r.tensors),
other_resident_bytes = other_resident,
kv_bytes_per_token_f16 = kv_per_token,
recurrent_bytes = recurrent,
diff --git a/studio/backend/tests/test_metal_explicit_context_guard.py b/studio/backend/tests/test_metal_explicit_context_guard.py
index d10dc94cbfd..9e489c96ee5 100644
--- a/studio/backend/tests/test_metal_explicit_context_guard.py
+++ b/studio/backend/tests/test_metal_explicit_context_guard.py
@@ -951,3 +951,607 @@ class TestACpuPinnedProjectorOnUnifiedMemory:
**self._COMMON,
)
assert _ctx_values(captured["cmd"])[-1] == "40960"
+
+
+_GIB = 1024**3
+_MIB = 1024**2
+
+
+def _backend_with_embeddings(
+ monkeypatch,
+ *,
+ embd,
+ tensors = 12 * _GIB,
+ measured = "mapped as designed",
+ settings = (False, False),
+ layout = None,
+):
+ """Build a backend with controlled tensor layout and probe results.
+
+ The default result maps everything loadable except the input embeddings into Metal.
+ """
+ from core.inference.llama_server_args import MEMORY_ENV_VARS
+ from core.inference.offload_layout import ModelLayout
+ import utils.model_memory_settings as _mem_settings
+
+ monkeypatch.setattr(_mem_settings, "get_model_memory_settings", lambda: settings)
+ placement_vars = (
+ "LLAMA_ARG_OVERRIDE_TENSOR",
+ "LLAMA_ARG_CPU_MOE",
+ "LLAMA_ARG_N_CPU_MOE",
+ "LLAMA_ARG_N_CPU_FFN",
+ "LLAMA_ARG_N_GPU_LAYERS",
+ "LLAMA_ARG_DEVICE",
+ )
+ for name in (*MEMORY_ENV_VARS, *placement_vars):
+ monkeypatch.delenv(name, raising = False)
+ if layout is None:
+ layout = ModelLayout(complete = True, token_embd_bytes = embd, tensor_bytes = tensors)
+ if measured == "mapped as designed":
+ # Blocks the loader skips are in no buffer, so they are in no row of the table either.
+ metal = layout.tensor_bytes - embd - layout.excluded_block_bytes
+ measured = (metal // _MIB, embd // _MIB, 2)
+ backend = LlamaCppBackend()
+ backend._tensor_spill_layout = lambda _path, **_kw: layout
+ backend._metal_measured_model_mib = lambda _binary, _path: measured
+ return backend
+
+
+class TestInputEmbeddingsLeftInTheFileMapping:
+ """Exercise Metal fitting when input embeddings remain pageable on the CPU."""
+
+ _COMMON = dict(real_fit = True, budget_bytes = 16 * 1024**3, weights_bytes = 12 * 1024**3)
+
+ def _published(
+ self,
+ tmp_path,
+ monkeypatch,
+ *,
+ n_ctx,
+ kv_per_token = 32 * 1024,
+ extra_args = None,
+ gpu_memory_mode = "auto",
+ gpu_layers = -1,
+ **kw,
+ ):
+ return self._launched(
+ tmp_path,
+ monkeypatch,
+ n_ctx = n_ctx,
+ kv_per_token = kv_per_token,
+ extra_args = extra_args,
+ gpu_memory_mode = gpu_memory_mode,
+ gpu_layers = gpu_layers,
+ **kw,
+ )[:2]
+
+ def _launched(
+ self,
+ tmp_path,
+ monkeypatch,
+ *,
+ n_ctx,
+ kv_per_token = 32 * 1024,
+ extra_args = None,
+ gpu_memory_mode = "auto",
+ gpu_layers = -1,
+ budget_bytes = 16 * _GIB,
+ **kw,
+ ):
+ """``(published ceiling, launched context, argv)``."""
+ kw.setdefault("embd", 6 * 1024**3)
+ captured = _launch(
+ tmp_path,
+ monkeypatch,
+ n_ctx = n_ctx,
+ kv_per_token = kv_per_token,
+ extra_args = extra_args,
+ gpu_memory_mode = gpu_memory_mode,
+ gpu_layers = gpu_layers,
+ backend = _backend_with_embeddings(monkeypatch, **kw),
+ **{**self._COMMON, "budget_bytes": budget_bytes},
+ )
+ cmd = captured["cmd"]
+ return captured["backend"]._max_context_length, _ctx_values(cmd)[-1], cmd
+
+ @staticmethod
+ def _fit_tokens(cmd):
+ """The placement tokens this launch emitted, in order."""
+ out = []
+ for i, token in enumerate(cmd):
+ if token in ("-ngl", "--fit") and i + 1 < len(cmd):
+ out.extend([token, cmd[i + 1]])
+ return out
+
+ @pytest.mark.parametrize("n_ctx", [0, 32768])
+ def test_a_discounted_ceiling_launches_pinned_to_the_measured_placement(
+ self, tmp_path, monkeypatch, n_ctx
+ ):
+ """A discounted launch must preserve the probe's full-offload placement."""
+ from core.inference.llama_server_args import fit_is_effectively_on
+
+ published, _, cmd = self._launched(tmp_path, monkeypatch, n_ctx = n_ctx)
+ assert published > _FIT_MIN_CTX
+ assert self._fit_tokens(cmd) == ["-ngl", "-1", "--fit", "off"]
+ assert not fit_is_effectively_on(cmd, {})
+
+ def test_a_load_with_nothing_taken_out_keeps_the_fitter(self, tmp_path, monkeypatch):
+ _, _, cmd = self._launched(tmp_path, monkeypatch, n_ctx = 32768, embd = 0)
+ assert self._fit_tokens(cmd) == ["--fit", "on"]
+
+ def test_a_floor_the_discount_did_not_lift_keeps_the_fitter(self, tmp_path, monkeypatch):
+ published, _, cmd = self._launched(tmp_path, monkeypatch, n_ctx = 0, embd = _GIB)
+ assert published == _FIT_MIN_CTX
+ assert self._fit_tokens(cmd) == ["--fit", "on"]
+
+ @pytest.mark.parametrize("extra_args", [["--fit", "on"], ["--fit=on"], ["-fit", "1"]])
+ def test_an_extra_that_turns_the_fitter_back_on_keeps_the_charge(
+ self, tmp_path, monkeypatch, extra_args
+ ):
+ """A later pass-through flag can override Unsloth's placement pin."""
+ from core.inference.llama_server_args import fit_is_effectively_on
+
+ published, _, cmd = self._launched(
+ tmp_path, monkeypatch, n_ctx = 32768, extra_args = extra_args
+ )
+ assert published == _FIT_MIN_CTX
+ assert fit_is_effectively_on(cmd, {})
+ assert "-ngl" not in cmd
+
+ def test_a_pass_through_fit_off_still_matches_the_pin(self, tmp_path, monkeypatch):
+ from core.inference.llama_server_args import fit_is_effectively_on
+
+ published, _, cmd = self._launched(
+ tmp_path, monkeypatch, n_ctx = 32768, extra_args = ["--fit", "off"]
+ )
+ assert published > 32768
+ assert self._fit_tokens(cmd)[:4] == ["-ngl", "-1", "--fit", "off"]
+ assert not fit_is_effectively_on(cmd, {})
+
+ def test_a_placement_it_did_not_measure_keeps_the_fitter(self, tmp_path, monkeypatch):
+ _, _, cmd = self._launched(
+ tmp_path, monkeypatch, n_ctx = 32768, extra_args = ["--load-mode", "none"]
+ )
+ assert self._fit_tokens(cmd) == ["--fit", "on"]
+
+ def test_charged_whole_the_same_load_publishes_the_floor(self, tmp_path, monkeypatch):
+ published, _ = self._published(tmp_path, monkeypatch, n_ctx = 32768, embd = 0)
+ assert published == _FIT_MIN_CTX
+
+ def test_an_explicit_context_publishes_a_measured_ceiling(self, tmp_path, monkeypatch):
+ published, launched = self._published(tmp_path, monkeypatch, n_ctx = 32768)
+ assert launched == "32768"
+ assert published > 32768
+
+ def test_the_ceiling_follows_the_kv_cache_size(self, tmp_path, monkeypatch):
+ wide, _ = self._published(tmp_path, monkeypatch, n_ctx = 0, kv_per_token = 32 * 1024)
+ narrow, _ = self._published(tmp_path, monkeypatch, n_ctx = 0, kv_per_token = 64 * 1024)
+ assert _FIT_MIN_CTX < narrow < wide
+
+ def test_a_short_measured_ceiling_is_refused_past_rather_than_floored(
+ self, tmp_path, monkeypatch
+ ):
+ """A measured ceiling below the usual floor still refuses larger contexts."""
+ with pytest.raises(RuntimeError, match = "unified") as refused:
+ self._published(tmp_path, monkeypatch, n_ctx = 32768, kv_per_token = _FAT_KV)
+ assert 0 < _named_ceiling(str(refused.value)) < _FIT_MIN_CTX
+
+ def test_a_build_that_maps_the_embeddings_keeps_the_charge(self, tmp_path, monkeypatch):
+ published, _ = self._published(
+ tmp_path, monkeypatch, n_ctx = 32768, measured = (12 * 1024, 6 * 1024, 2)
+ )
+ assert published == _FIT_MIN_CTX
+
+ def test_a_draft_that_loads_the_targets_own_mtp_blocks_keeps_the_charge(
+ self, tmp_path, monkeypatch
+ ):
+ from core.inference.offload_layout import ModelLayout
+
+ layout = ModelLayout(
+ complete = True,
+ token_embd_bytes = 6 * _GIB,
+ tensor_bytes = 12 * _GIB,
+ has_excluded_blocks = True,
+ excluded_block_bytes = _GIB,
+ )
+ published, _ = self._published(
+ tmp_path,
+ monkeypatch,
+ n_ctx = 32768,
+ extra_args = ["--spec-type", "draft-mtp"],
+ layout = layout,
+ measured = (6 * 1024, 6 * 1024, 2),
+ )
+ assert published == _FIT_MIN_CTX
+
+ def test_no_measurement_keeps_the_charge(self, tmp_path, monkeypatch):
+ published, _ = self._published(tmp_path, monkeypatch, n_ctx = 32768, measured = None)
+ assert published == _FIT_MIN_CTX
+
+ def test_a_fixed_manual_layer_count_keeps_the_charge(self, tmp_path, monkeypatch):
+ published, _ = self._published(
+ tmp_path, monkeypatch, n_ctx = 32768, gpu_memory_mode = "manual", gpu_layers = 20
+ )
+ assert published == _FIT_MIN_CTX
+
+ def test_the_fit_and_the_launch_price_the_same_settings_read(self, tmp_path, monkeypatch):
+ """A settings change during loading must not split fit and launch decisions."""
+ import utils.model_memory_settings as _mem_settings
+
+ backend = _backend_with_embeddings(monkeypatch, embd = 6 * 1024**3)
+ reads = []
+
+ def settings():
+ reads.append(None)
+ return (False, False) if len(reads) == 1 else (True, False)
+
+ monkeypatch.setattr(_mem_settings, "get_model_memory_settings", settings)
+ captured = _launch(
+ tmp_path,
+ monkeypatch,
+ n_ctx = 32768,
+ kv_per_token = 32 * 1024,
+ backend = backend,
+ **self._COMMON,
+ )
+ locked = any("mlock" in str(token) for token in captured["cmd"])
+ discounted = captured["backend"]._max_context_length > _FIT_MIN_CTX
+ assert locked != discounted
+
+ @pytest.mark.parametrize(
+ "extra_args",
+ [
+ ["--no-mmap"],
+ ["--mlock"],
+ ["--load-mode", "none"],
+ ["--load-mode", "dio"],
+ ["--override-tensor", "per_layer_token_embd=CPU"],
+ ["--cpu-moe"],
+ ["-ncffn", "4"],
+ ["--fit-target", "2048"],
+ ],
+ )
+ def test_a_loader_or_placement_it_did_not_measure_keeps_the_charge(
+ self, tmp_path, monkeypatch, extra_args
+ ):
+ published, _ = self._published(tmp_path, monkeypatch, n_ctx = 32768, extra_args = extra_args)
+ assert published == _FIT_MIN_CTX
+
+
+class TestWhichLoadsLeaveTheEmbeddingsInTheMapping:
+ """Cover loader and placement inputs that control the discount."""
+
+ BYTES = 3 * _GIB
+ # The embeddings, less a MiB per measured table row for rounding.
+ DISCOUNT = BYTES - 2 * _MIB
+
+ def _bytes(
+ self,
+ monkeypatch,
+ *,
+ extra_args = None,
+ env = None,
+ load_mode = None,
+ layers_fixed = False,
+ mtp_may_engage = False,
+ **kw,
+ ):
+ kw.setdefault("embd", self.BYTES)
+ backend = _backend_with_embeddings(monkeypatch, **kw)
+ return backend._metal_demand_paged_embedding_bytes(
+ "/models/model.gguf",
+ extra_args,
+ binary = "/fake/llama-server",
+ requested_load_mode = load_mode,
+ supports_load_mode = True,
+ settings = kw.get("settings", (False, False)),
+ layers_fixed = layers_fixed,
+ mtp_may_engage = mtp_may_engage,
+ env = env or {},
+ )
+
+ @pytest.mark.parametrize("load_mode", [None, "auto", "mmap"])
+ def test_a_mapped_load_discounts_the_unmapped_embeddings(self, monkeypatch, load_mode):
+ assert self._bytes(monkeypatch, load_mode = load_mode) == self.DISCOUNT
+
+ def test_weights_llama_cpp_moved_to_the_cpu_stay_charged(self, monkeypatch):
+ """CPU fallback weights remain charged."""
+ tensors = 12 * _GIB
+ # 7 GiB Metal, 3 GiB embeddings, and 2 GiB CPU fallback.
+ measured = (7 * 1024, 5 * 1024, 3)
+ unmapped = self._bytes(monkeypatch, tensors = tensors, measured = measured)
+ charged = tensors - unmapped
+ assert charged >= 9 * _GIB
+ assert unmapped == 3 * _GIB - 3 * _MIB
+
+ def test_embeddings_mapped_into_metal_are_not_taken_out(self, monkeypatch):
+ """A collapsed Metal span can include the embeddings."""
+ tensors = 12 * _GIB
+ measured = (10 * 1024, 5 * 1024, 3) # the span covers embeddings, fallback is on the host
+ assert self._bytes(monkeypatch, tensors = tensors, measured = measured) == 0
+
+ def test_never_more_than_the_embeddings_comes_out(self, monkeypatch):
+ """Unaccounted bytes cannot increase the discount past the embeddings."""
+ measured = (4 * 1024, 3 * 1024, 2) # 7 GiB reported of a 12 GiB file
+ assert self._bytes(monkeypatch, tensors = 12 * _GIB, measured = measured) == self.BYTES
+
+ @staticmethod
+ def _layout_with_nextn(has_nextn):
+ from core.inference.offload_layout import ModelLayout
+ return ModelLayout(
+ complete = True,
+ token_embd_bytes = 3 * _GIB,
+ tensor_bytes = 12 * _GIB,
+ has_excluded_blocks = has_nextn,
+ excluded_block_bytes = 2 * _GIB if has_nextn else 0,
+ )
+
+ def test_its_own_mtp_blocks_loading_drops_it(self, monkeypatch):
+ """The probe cannot model target-embedded MTP blocks."""
+ layout = self._layout_with_nextn(True)
+ assert self._bytes(monkeypatch, layout = layout, mtp_may_engage = True) == 0
+
+ @pytest.mark.parametrize(("has_nextn", "mtp_may_engage"), [(True, False), (False, True)])
+ def test_mtp_blocks_that_stay_skipped_keep_it(self, monkeypatch, has_nextn, mtp_may_engage):
+ layout = self._layout_with_nextn(has_nextn)
+ unmapped = self._bytes(monkeypatch, layout = layout, mtp_may_engage = mtp_may_engage)
+ assert unmapped == self.DISCOUNT
+
+ def test_skipped_mtp_bytes_are_not_read_as_unmapped_embeddings(self, monkeypatch):
+ """TENSOR_SKIP keeps the trailing blocks out of every buffer AND every table row."""
+ layout = self._layout_with_nextn(True)
+ # 12 GiB file = 7 GiB trunk + 3 GiB embeddings + 2 GiB skipped MTP. The Metal span
+ # collapses over the embeddings, so nothing is demand-paged: 10 GiB Metal, the
+ # embeddings again in the host row, and no sign of the 2 GiB llama.cpp never created.
+ measured = (10 * 1024, 3 * 1024, 2)
+ assert self._bytes(monkeypatch, layout = layout, measured = measured) == 0
+
+ @pytest.mark.parametrize("load_mode", ["none", "mlock", "mmap+mlock", "dio"])
+ def test_a_holding_per_model_mode_drops_it(self, monkeypatch, load_mode):
+ assert self._bytes(monkeypatch, load_mode = load_mode) == 0
+
+ @pytest.mark.parametrize(
+ "extra_args",
+ [
+ ["--lora", "/a.gguf"],
+ ["--lora-scaled", "/a.gguf:0.5"],
+ ["--control-vector", "/v.gguf"],
+ ["--control-vector-scaled", "/v.gguf:0.5"],
+ ["--lora=/a.gguf"],
+ ],
+ )
+ def test_a_pass_through_adapter_drops_it(self, monkeypatch, extra_args):
+ """The probe loads the base GGUF alone, and no Apple term charges the adapter."""
+ assert self._bytes(monkeypatch, extra_args = extra_args) == 0
+
+ def test_the_adapter_gate_is_the_flag_set_the_other_consumers_price(self, monkeypatch):
+ from core.inference.llama_cpp import _SIDECAR_ADAPTER_FLAGS
+ for flag in _SIDECAR_ADAPTER_FLAGS:
+ assert self._bytes(monkeypatch, extra_args = [flag, "/a.gguf:0.5"]) == 0
+
+ def test_keep_model_in_gpu_memory_drops_it(self, monkeypatch):
+ assert self._bytes(monkeypatch, settings = (True, False)) == 0
+
+ def test_no_ram_reserve_vetoes_a_holding_mode_back_to_the_mapping(self, monkeypatch):
+ assert self._bytes(monkeypatch, load_mode = "none", settings = (False, True)) == self.DISCOUNT
+
+ @pytest.mark.parametrize(
+ "extra_args",
+ [["--no-mmap"], ["--mlock"], ["--load-mode=none"], ["-lm", "mmap+mlock"], ["--direct-io"]],
+ )
+ def test_a_holding_pass_through_flag_drops_it(self, monkeypatch, extra_args):
+ assert self._bytes(monkeypatch, extra_args = extra_args) == 0
+
+ def test_a_pass_through_mmap_keeps_it(self, monkeypatch):
+ assert self._bytes(monkeypatch, extra_args = ["--load-mode", "mmap"]) == self.DISCOUNT
+
+ @pytest.mark.parametrize("extra_args", [["--fit", "on"], ["--fit=on"], ["-fit", "true"]])
+ def test_a_pass_through_fit_on_drops_it(self, monkeypatch, extra_args):
+ assert self._bytes(monkeypatch, extra_args = extra_args) == 0
+
+ @pytest.mark.parametrize("extra_args", [["--fit", "off"], ["--fit=off"]])
+ def test_a_pass_through_fit_off_keeps_it(self, monkeypatch, extra_args):
+ assert self._bytes(monkeypatch, extra_args = extra_args) == self.DISCOUNT
+
+ def test_an_inherited_fit_on_keeps_it(self, monkeypatch):
+ assert self._bytes(monkeypatch, env = {"LLAMA_ARG_FIT": "on"}) == self.DISCOUNT
+
+ @pytest.mark.parametrize(
+ "env",
+ [{"LLAMA_ARG_MLOCK": "1"}, {"LLAMA_ARG_NO_MMAP": "0"}, {"LLAMA_ARG_LOAD_MODE": "none"}],
+ )
+ def test_a_holding_inherited_variable_drops_it(self, monkeypatch, env):
+ assert self._bytes(monkeypatch, env = env) == 0
+
+ @pytest.mark.parametrize(
+ "extra_args",
+ [
+ ["-ot", "token_embd=MTL0"],
+ ["--override-tensor", "per_layer_token_embd=CPU"],
+ ["--override-tensor=exps=CPU"],
+ ["--cpu-moe"],
+ ["--n-cpu-moe", "4"],
+ ["-ngl", "20"],
+ ["--device", "none"],
+ ["-ncffn", "4"],
+ ["--n-cpu-ffn=4"],
+ ["--fit-target", "2048"],
+ ["-fitt", "2048"],
+ ["--fit-ctx", "16384"],
+ ["-fitc=16384"],
+ ],
+ )
+ def test_a_pass_through_placement_drops_it(self, monkeypatch, extra_args):
+ assert self._bytes(monkeypatch, extra_args = extra_args) == 0
+
+ @pytest.mark.parametrize(
+ "env",
+ [
+ {"LLAMA_ARG_OVERRIDE_TENSOR": "exps=CPU"},
+ {"LLAMA_ARG_CPU_MOE": "1"},
+ {"LLAMA_ARG_N_GPU_LAYERS": "20"},
+ {"LLAMA_ARG_DEVICE": "none"},
+ {"LLAMA_ARG_N_CPU_FFN": "4"},
+ {"LLAMA_ARG_FIT_TARGET": "2048"},
+ {"LLAMA_ARG_FIT_CTX": "16384"},
+ ],
+ )
+ def test_an_inherited_placement_drops_it(self, monkeypatch, env):
+ assert self._bytes(monkeypatch, env = env) == 0
+
+ def test_a_request_with_a_fixed_layer_count_drops_it(self, monkeypatch):
+ assert self._bytes(monkeypatch, layers_fixed = True) == 0
+
+ def test_an_unreadable_layout_abstains(self, monkeypatch):
+ from core.inference.offload_layout import ModelLayout
+ assert self._bytes(monkeypatch, layout = ModelLayout()) == 0
+
+ def test_no_measurement_abstains_and_says_so(self, monkeypatch):
+ """Recorded off the module logger, not caplog.
+
+ The structlog stub at the top of this file is installed with
+ `sys.modules.setdefault`, so in a full run any module that imported the real
+ structlog first wins and the log never reaches a stdlib handler. Under xdist
+ that depends on which worker gets this file, which made the caplog spelling
+ pass alone and fail in CI.
+ """
+ import core.inference.llama_cpp as llama_cpp
+
+ said = []
+ monkeypatch.setattr(llama_cpp.logger, "info", lambda msg, *a, **kw: said.append(str(msg)))
+ assert self._bytes(monkeypatch, measured = None) == 0
+ assert any("could not measure" in line for line in said)
+
+
+# b10909-mix output from an 8 GB M1; the second table adds -ncffn 30.
+_E4B_TABLE = """0.00.408.825 I common_memory_breakdown_print: | memory breakdown [MiB] | total free self model context compute unaccounted |
+0.00.429.748 I common_memory_breakdown_print: | - MTL0 (Apple M1) | 5461 = 5460 + (3642 = 3025 + 28 + 589) + -3642 |
+0.00.429.748 I common_memory_breakdown_print: | - Host | 2352 = 2288 + 0 + 64 |
+0.00.444.070 I llama_fit_params: printing fitted CLI arguments to stdout...
+-c 512 -ngl 999
+"""
+_E4B_NCFFN_TABLE = """I common_memory_breakdown_print: | memory breakdown [MiB] | total free self model context compute unaccounted |
+I common_memory_breakdown_print: | - MTL0 (Apple M1) | 5461 = 5460 + (2196 = 1625 + 28 + 543) + -2196 |
+I common_memory_breakdown_print: | - Host | 2430 = 2288 + 0 + 142 |
+I common_memory_breakdown_print: | - CPU_REPACK | 1400 = 1400 + 0 + 0 |
+"""
+
+
+class TestTheMetalMemoryProbe:
+ """Parse and invoke llama-fit-params conservatively."""
+
+ def test_it_reads_the_metal_and_host_model_columns(self):
+ from core.inference.llama_cpp import _parse_metal_memory_breakdown
+ assert _parse_metal_memory_breakdown(_E4B_TABLE) == (3025, 2288, 2)
+
+ def test_every_host_buffer_type_is_summed(self):
+ from core.inference.llama_cpp import _parse_metal_memory_breakdown
+ assert _parse_metal_memory_breakdown(_E4B_NCFFN_TABLE) == (1625, 2288 + 1400, 3)
+
+ def test_only_the_first_table_is_read(self):
+ from core.inference.llama_cpp import _parse_metal_memory_breakdown
+ assert _parse_metal_memory_breakdown(_E4B_TABLE + _E4B_NCFFN_TABLE) == (3025, 2288, 2)
+
+ @pytest.mark.parametrize(
+ "output",
+ [
+ "",
+ "error: invalid argument: -lv",
+ _E4B_TABLE.replace("MTL0 (Apple M1)", "CUDA0 (RTX 4090)"),
+ _E4B_TABLE.replace("2352 = 2288 +", "2352"),
+ _E4B_TABLE.replace("3025 +", "x +"),
+ ],
+ )
+ def test_anything_else_is_unknown(self, output):
+ from core.inference.llama_cpp import _parse_metal_memory_breakdown
+ assert _parse_metal_memory_breakdown(output) is None
+
+ @staticmethod
+ def _install(tmp_path, script):
+ bin_dir = tmp_path / "bin"
+ bin_dir.mkdir()
+ (bin_dir / "llama-server").write_text("")
+ probe = bin_dir / "llama-fit-params"
+ probe.write_text(script)
+ probe.chmod(0o755)
+ model = tmp_path / "model.gguf"
+ model.write_bytes(b"GGUF")
+ return str(bin_dir / "llama-server"), str(model)
+
+ @pytest.mark.skipif(sys.platform == "win32", reason = "shell script stands in for the binary")
+ def test_it_runs_the_probe_beside_the_binary_and_caches_the_answer(self, tmp_path, monkeypatch):
+ calls = tmp_path / "calls"
+ table = _E4B_TABLE.replace("\n", "\\n")
+ script = f'#!/bin/sh\necho "$@" >> "{calls}"\nprintf "{table}"\n'
+ binary, model = self._install(tmp_path, script)
+ monkeypatch.setenv("LLAMA_ARG_N_GPU_LAYERS", "10")
+ backend = LlamaCppBackend()
+ assert backend._metal_measured_model_mib(binary, model) == (3025, 2288, 2)
+ assert backend._metal_measured_model_mib(binary, model) == (3025, 2288, 2)
+ argv = calls.read_text().splitlines()
+ assert argv == [f"-m {model} -ngl 999 -c 512 -lv 4"]
+
+ @pytest.mark.skipif(sys.platform == "win32", reason = "shell script stands in for the binary")
+ def test_a_failing_probe_is_unknown(self, tmp_path):
+ binary, model = self._install(tmp_path, "#!/bin/sh\nexit 1\n")
+ assert LlamaCppBackend()._metal_measured_model_mib(binary, model) is None
+
+ def test_no_probe_is_unknown(self, tmp_path):
+ (tmp_path / "llama-server").write_text("")
+ model = tmp_path / "model.gguf"
+ model.write_bytes(b"GGUF")
+ backend = LlamaCppBackend()
+ assert backend._metal_measured_model_mib(str(tmp_path / "llama-server"), str(model)) is None
+ assert backend._metal_measured_model_mib(None, str(model)) is None
+
+
+def test_every_forced_full_offload_arm_owes_the_fit_on_retry():
+ """A forced "-ngl -1 --fit off" must also claim the full offload.
+
+ The `--fit on` retry after a startup crash is gated on `fully_gpu_offloaded`,
+ and the tensor-spill recovery ahead of it is a no-op without a spill plan, so
+ an arm that pins the placement without setting the flag drops straight to the
+ terminal fallbacks when its estimate turns out optimistic. Checked at the
+ source, like the other invariants over this launch path, because the retry only
+ runs behind a real child crash.
+ """
+ import ast
+ import inspect
+ import textwrap
+
+ from core.inference.llama_cpp import LlamaCppBackend
+
+ tree = ast.parse(textwrap.dedent(inspect.getsource(LlamaCppBackend.load_model)))
+
+ def pins_full_offload(stmt):
+ """The emission as a DIRECT statement of the arm, so an enclosing `if` does
+ not also count as one."""
+ if not isinstance(stmt, ast.Expr) or not isinstance(stmt.value, ast.Call):
+ return False
+ call = stmt.value
+ if not isinstance(call.func, ast.Attribute) or call.func.attr != "extend":
+ return False
+ if not call.args or not isinstance(call.args[0], ast.List):
+ return False
+ values = [e.value for e in call.args[0].elts if isinstance(e, ast.Constant)]
+ return values == ["-ngl", "-1", "--fit", "off"]
+
+ def claims_full_offload(body):
+ for stmt in body:
+ if not isinstance(stmt, ast.Assign) or not isinstance(stmt.value, ast.Constant):
+ continue
+ if stmt.value.value is not True:
+ continue
+ if any(isinstance(t, ast.Name) and t.id == "fully_gpu_offloaded" for t in stmt.targets):
+ return True
+ return False
+
+ arms = [
+ branch
+ for node in ast.walk(tree)
+ if isinstance(node, ast.If)
+ for branch in (node.body, node.orelse)
+ if branch and any(pins_full_offload(stmt) for stmt in branch)
+ ]
+ assert len(arms) == 2, f"expected the two forced full-offload arms, found {len(arms)}"
+ assert all(claims_full_offload(arm) for arm in arms)
diff --git a/studio/backend/tests/test_model_memory_settings.py b/studio/backend/tests/test_model_memory_settings.py
index 484856ef18c..5fe558a5c59 100644
--- a/studio/backend/tests/test_model_memory_settings.py
+++ b/studio/backend/tests/test_model_memory_settings.py
@@ -1656,12 +1656,31 @@ class TestCpuMoeCountIsParsed:
(["-cmoe"], True),
(["-ot", "blk.*=CPU"], True),
(["--override-tensor", "x"], True),
+ # The dense-model count: common/arg.cpp turns it into CPU FFN overrides.
+ (["-ncffn", "0"], False),
+ (["-ncffn", "4"], True),
+ (["--n-cpu-ffn", "4"], True),
+ (["--n-cpu-ffn=4"], True),
+ (["--n_cpu_ffn", "4"], True),
],
)
def test_the_predicate(self, extras, expected):
from core.inference.llama_cpp import _args_place_tensors_on_cpu
assert _args_place_tensors_on_cpu(extras) is expected
+ @pytest.mark.parametrize(
+ ("env", "expected"),
+ [
+ ({}, False),
+ ({"LLAMA_ARG_N_CPU_MOE": "4"}, True),
+ ({"LLAMA_ARG_N_CPU_FFN": "0"}, False),
+ ({"LLAMA_ARG_N_CPU_FFN": "4"}, True),
+ ],
+ )
+ def test_the_env_predicate(self, env, expected):
+ from core.inference.llama_cpp import _env_places_tensors_on_cpu
+ assert _env_places_tensors_on_cpu(env) is expected
+
def test_it_is_the_same_predicate_the_pipeline_check_uses(self):
import ast
import inspect
diff --git a/studio/backend/tests/test_offload_planner.py b/studio/backend/tests/test_offload_planner.py
index 965eb6d40d9..792b020eedb 100644
--- a/studio/backend/tests/test_offload_planner.py
+++ b/studio/backend/tests/test_offload_planner.py
@@ -987,6 +987,44 @@ def test_a_single_file_gguf_is_still_read():
assert len(layout.blocks) == 64
+def test_the_layout_reader_totals_every_tensor_from_a_real_gguf(tmp_path):
+ """tensor_bytes includes every tensor while token_embd_bytes is embedding-only."""
+ import numpy as np
+ from gguf import GGUFWriter
+
+ path = tmp_path / "embd.gguf"
+ writer = GGUFWriter(str(path), "llama")
+ for key, value in _shard_fields().items():
+ if key.startswith("llama.") and isinstance(value, int):
+ writer.add_uint32(key, 1 if key == "llama.block_count" else value)
+ writer.add_tensor("output_norm.weight", np.zeros(16, dtype = np.float32))
+ writer.add_tensor("token_embd.weight", np.zeros(1024, dtype = np.float32))
+ writer.add_tensor("per_layer_token_embd.weight", np.zeros(4096, dtype = np.float32))
+ writer.add_tensor("blk.0.attn_q.weight", np.zeros(256, dtype = np.float32))
+ writer.add_tensor("blk.1.attn_q.weight", np.zeros(64, dtype = np.float32))
+ writer.write_header_to_file()
+ writer.write_kv_data_to_file()
+ writer.write_tensors_to_file()
+ writer.close()
+
+ layout = layout_from_gguf(str(path))
+ assert layout.complete
+ assert layout.tensor_bytes == 4 * (16 + 1024 + 4096 + 256 + 64)
+ assert layout.token_embd_bytes == 4 * (1024 + 4096)
+
+
+def test_token_embd_norm_is_not_an_input_embedding():
+ """token_embd_norm follows the repeating layers rather than the input layer."""
+ tensors = _shard_tensors(range(64)) + [
+ _StubTensor("token_embd.weight", 400 * MIB),
+ _StubTensor("token_embd_norm.weight", 4 * MIB),
+ _StubTensor("output.weight", 400 * MIB),
+ ]
+ layout = _layout_from_reader(_StubReader(_shard_fields(), tensors))
+ assert layout.token_embd_bytes == 400 * MIB
+ assert layout.other_resident_bytes == 4 * MIB
+
+
def test_a_split_gguf_abstains_instead_of_planning_on_one_shard():
"""GGUFReader memmaps the ONE path it is given, but llama.cpp reads
split.count off the first shard and loads every sibling
diff --git a/studio/install_llama_prebuilt.py b/studio/install_llama_prebuilt.py
index 369b73c408d..68d6cb2ed90 100644
--- a/studio/install_llama_prebuilt.py
+++ b/studio/install_llama_prebuilt.py
@@ -4318,6 +4318,8 @@ def runtime_patterns_for_install_kind(
"llama-server",
"llama-quantize",
"llama-diffusion-gemma-visual-server",
+ # Optional Metal memory probe; missing bundles keep the conservative estimate.
+ "llama-fit-params",
"lib*.dylib",
]
if install_kind in {
diff --git a/tests/studio/install/test_rocm_support.py b/tests/studio/install/test_rocm_support.py
index 76f7224a2d8..d4f3f5a72d1 100644
--- a/tests/studio/install/test_rocm_support.py
+++ b/tests/studio/install/test_rocm_support.py
@@ -446,6 +446,13 @@ class TestRuntimePatterns:
)
assert name in runtime_patterns_for_choice(choice)
+ def test_fit_params_kept_on_macos(self):
+ for kind in ("macos-arm64", "macos-x64"):
+ choice = AssetChoice(
+ repo = "", tag = "", name = "", url = "", source_label = "", install_kind = kind
+ )
+ assert "llama-fit-params" in runtime_patterns_for_choice(choice)
+
# TEST: install_llama_prebuilt.py -- HostInfo.has_rocm field