Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
Studio: stop charging CPU-mapped input embeddings to the Metal context budget (#11060)
oobabooga71c9a44cc0fa
diff --git a/studio/backend/core/inference/llama_cpp.py b/studio/backend/core/inference/llama_cpp.pyindex 70b5edcf5bd..6b9f915ef57 100644--- a/studio/backend/core/inference/llama_cpp.py+++ b/studio/backend/core/inference/llama_cpp.py@@ -4068,6 +4068,13 @@ _GPU_OFFLOAD_OVERRIDE_FLAGS = _LAYER_OFFLOAD_FLAGS# presence-only (-cmoe takes no value, and an -ot pattern is not worth guessing# at, which matches how the env side reads LLAMA_ARG_OVERRIDE_TENSOR)._CPU_MOE_COUNT_FLAGS = frozenset({"-ncmoe", "--n-cpu-moe"})+# The dense-model twin: common/arg.cpp turns a positive -ncffn into CPU overrides for the+# first N layers' ffn_up/down/gate, exactly as -ncmoe does for expert weights.+_CPU_FFN_COUNT_FLAGS = frozenset({"-ncffn", "--n-cpu-ffn"})+# User controls over llama.cpp's own fitter. Studio emits neither on Metal, so a launch carrying+# one is placed by a fitter the Metal memory probe did not run.+_FIT_CONTROL_FLAGS = frozenset({"-fitt", "--fit-target", "-fitc", "--fit-ctx"})+_FIT_CONTROL_ENV_VARS = ("LLAMA_ARG_FIT_TARGET", "LLAMA_ARG_FIT_CTX")_CPU_PLACEMENT_PRESENCE_FLAGS = (_MOE_OFFLOAD_FLAGS - _CPU_MOE_COUNT_FLAGS) | frozenset({"-ot", "--override-tensor"})@@ -4507,7 +4514,7 @@ def _args_place_tensors_on_cpu(extra_args: Optional[Iterable[str]] = None) -> bo"""True when extras keep weights on the CPU, whatever the layer count says.The arg-side twin of ``_env_places_tensors_on_cpu``, including its rule that- a CPU-MoE count only counts when positive: ``--n-cpu-moe 0`` places nothing.+ a CPU-MoE or CPU-FFN count only counts when positive: ``--n-cpu-moe 0`` places nothing."""values = [str(raw) for raw in extra_args or ()]for i, raw in enumerate(values):@@ -4516,7 +4523,7 @@ def _args_place_tensors_on_cpu(extra_args: Optional[Iterable[str]] = None) -> bo# non-empty, and guessing at the pattern is not worth it.if flag in _CPU_PLACEMENT_PRESENCE_FLAGS:return True- if flag in _CPU_MOE_COUNT_FLAGS:+ if flag in _CPU_MOE_COUNT_FLAGS | _CPU_FFN_COUNT_FLAGS:_, eq, inline = raw.partition("=")if _is_positive_int(inline if eq else (values[i + 1] if i + 1 < len(values) else "")):return True@@ -4535,6 +4542,7 @@ def _env_places_tensors_on_cpu(env: Optional[Mapping[str, str]] = None) -> bool:source_env.get("LLAMA_ARG_OVERRIDE_TENSOR")or source_env.get("LLAMA_ARG_CPU_MOE") in _LLAMA_ARG_TRUE_VALUESor _is_positive_int(source_env.get("LLAMA_ARG_N_CPU_MOE"))+ or _is_positive_int(source_env.get("LLAMA_ARG_N_CPU_FFN")))@@ -5385,6 +5393,51 @@ def _normalized_load_mode(value: Optional[str]) -> Optional[str]:return None if mode in {"", "auto"} else mode+_BREAKDOWN_DEVICE_ROW_RE = re.compile(+ r"\|\s*-\s*(?P<name>[^|(]+?)\s*\([^|]*\|\s*\d+\s*=\s*\d+\s*\+\s*\(\s*\d+\s*=\s*(?P<model>\d+)\s*\+"+)+_BREAKDOWN_HOST_ROW_RE = re.compile(+ r"\|\s*-\s*(?P<name>[^|]+?)\s*\|\s*\d+\s*=\s*(?P<model>\d+)\s*\+"+)+++def _parse_metal_memory_breakdown(output: str) -> Optional[tuple[int, int, int]]:+ """Parse the first llama.cpp table as ``(metal_model_mib, host_model_mib, rows)``.++ Require one Metal device row and sum the model column from every host row.+ """+ rows: list[str] = []+ for line in output.splitlines():+ if "common_memory_breakdown_print" not in line:+ if rows:+ break+ continue+ if "memory breakdown" in line:+ if rows:+ break+ continue+ rows.append(line)+ metal: list[int] = []+ host = 0+ host_rows = 0+ for row in rows:+ device = _BREAKDOWN_DEVICE_ROW_RE.search(row)+ if device:+ if not device.group("name").strip().startswith("MTL"):+ return None+ metal.append(int(device.group("model")))+ continue+ hosted = _BREAKDOWN_HOST_ROW_RE.search(row)+ if hosted:+ host += int(hosted.group("model"))+ host_rows += 1+ continue+ return None+ if len(metal) != 1 or host_rows == 0:+ return None+ return metal[0], host, len(rows)++# The KV cache dtypes llama.cpp can map, shared by the emission guard and the# budget. Module level because both the draft-cache block and the MTP reserve# need it, and the main cache path's copy is a local inside load_model.@@ -11461,6 +11514,150 @@ class LlamaCppBackend:rec_bytes = min(rec_bytes, available)return int(rec_bytes * _APPLE_UNIFIED_MEMORY_FRACTION)+ def _metal_measured_model_mib(+ self, binary: Optional[str], model_path: Optional[str]+ ) -> Optional[tuple[int, int, int]]:+ """Measure full-offload model buffers with the sibling ``llama-fit-params``.++ Discounted launches pin the same placement. Results are cached by probe and model+ identity; a missing, failed, or unreadable probe returns None.+ """+ if not binary or not model_path:+ return None+ try:+ probe = _llama_lib_dir(binary) / "llama-fit-params"+ probe_stat = probe.stat()+ model_stat = os.stat(model_path)+ except OSError:+ return None+ key = (+ str(probe),+ probe_stat.st_size,+ probe_stat.st_mtime_ns,+ model_path,+ model_stat.st_size,+ model_stat.st_mtime_ns,+ )+ cached = getattr(self, "_metal_model_mib_cache", None)+ if cached is not None and cached[0] == key:+ return cached[1]+ measured: Optional[tuple[int, int, int]] = None+ try:+ env = self._llama_server_env_for_binary(str(binary))+ for name in tuple(env):+ if name.startswith("LLAMA_ARG_"):+ env.pop(name, None)+ result = subprocess.run(+ [str(probe), "-m", model_path, "-ngl", "999", "-c", "512", "-lv", "4"],+ capture_output = True,+ text = True,+ encoding = "utf-8",+ errors = "replace",+ # A cold Metal shader compile lands here instead of at the launch.+ timeout = 120,+ check = False,+ env = env,+ **_windows_hidden_subprocess_kwargs(),+ )+ if result.returncode == 0:+ measured = _parse_metal_memory_breakdown(+ (result.stdout or "") + "\n" + (result.stderr or "")+ )+ except Exception as e:+ logger.debug(f"llama-fit-params memory probe failed: {e}")+ self._metal_model_mib_cache = (key, measured)+ return measured++ def _metal_demand_paged_embedding_bytes(+ self,+ model_path: Optional[str],+ extra_args: Optional[Iterable[str]],+ *,+ binary: Optional[str],+ requested_load_mode: Optional[str],+ supports_load_mode: bool,+ settings: tuple[bool, bool],+ layers_fixed: bool,+ mtp_may_engage: bool,+ env: Optional[Mapping[str, str]] = None,+ ) -> int:+ """Return mmap embedding bytes proven absent from the Metal working set.++ Input embeddings live on the CPU but can still fall inside a Metal file mapping.+ Fail closed unless the probe matches the launch placement and the loader leaves the+ embeddings pageable. CPU fallback weights remain charged.+ """+ source_env = os.environ if env is None else env+ if (+ layers_fixed+ or _extra_args_set_any_flag(extra_args, _GPU_LAYER_FLAGS)+ or _env_fixes_gpu_layers(source_env)+ or _device_selection_is_cpu(extra_args, source_env)+ or _args_place_tensors_on_cpu(extra_args)+ or _env_places_tensors_on_cpu(source_env)+ or _extra_args_set_any_flag(extra_args, _FIT_CONTROL_FLAGS)+ or any(source_env.get(name) for name in _FIT_CONTROL_ENV_VARS)+ # Extras are appended after the pin, while inherited values are overridden by it.+ or fit_is_enabled_in(extra_args)+ # A pass-through adapter is resident on the base tensor's buffer+ # (llama-adapter.cpp:335, :67) and neither the probe nor model_size carries it,+ # and nothing on the Apple path charges it the way _fits_without_paging does+ # off Metal. Spending the discount would hand that budget to the KV cache.+ or _sidecar_adapter_paths(extra_args)+ ):+ return 0+ env_view = dict(source_env)+ scrub_memory_env(env_view, settings)+ managed, extras = apply_model_memory_policy(+ extra_args,+ supports_load_mode = supports_load_mode,+ weights_in_host_memory = True,+ gpu_offload_confirmed = False,+ env = env_view,+ settings = settings,+ )+ selected, extras = apply_load_mode_policy(+ extras,+ supports_load_mode = supports_load_mode,+ weights_in_host_memory = True,+ requested_load_mode = requested_load_mode,+ settings = settings,+ )+ mlock, reserves_ram, direct_io = resolve_effective_load_state(+ [*managed, *selected, *extras], env_view+ )+ if mlock or reserves_ram or direct_io:+ return 0+ layout = self._tensor_spill_layout(model_path, all_shards = True)+ if layout is None or not layout.complete:+ return 0+ embeddings = int(layout.token_embd_bytes)+ if embeddings <= 0 or (mtp_may_engage and layout.has_excluded_blocks):+ return 0+ measured = self._metal_measured_model_mib(binary, model_path)+ if measured is None:+ logger.info(+ "Metal fit: could not measure how llama.cpp maps this model (llama-fit-params "+ "missing or unreadable), so its %.2f GB of input embeddings stay charged.",+ embeddings / (1024**3),+ )+ return 0+ metal_mib, host_mib, rows = measured+ mib = 1024 * 1024+ charged = (metal_mib + rows) * mib + max(0, host_mib * mib - embeddings)+ # TENSOR_SKIP returns before the tensor exists (llama-model-loader.cpp:1125-1131), so a+ # skipped block reaches no buffer and no table row. Those bytes are in the file and not+ # in the probe, which is the signature the embeddings are being recognised by.+ mapped = int(layout.tensor_bytes) - int(layout.excluded_block_bytes)+ unmapped = min(embeddings, max(0, mapped - charged))+ if not unmapped:+ logger.info(+ "Metal fit: llama.cpp maps this model's %.2f GB of input embeddings into Metal, "+ "so they stay charged. A llama.cpp update may remove that.",+ embeddings / (1024**3),+ )+ return unmapped+@staticmethoddef _rocm_arch_gate_keep(binary: Optional[str], torch_mod, for_llama_server: bool@@ -22283,6 +22480,14 @@ class LlamaCppBackend:# the handler's own log line on purpose: test_tp_vision_regression# string-searches this function's source for it to check ordering.)_metal_ctx_refusal: Optional[str] = None+ # A discounted Metal ceiling must launch the measured full-offload placement.+ _metal_full_offload_pinned = False+ # The fit and launch must use the same Model Memory snapshot.+ from utils.model_memory_settings import capture_model_memory_settings++ _mem_settings = capture_model_memory_settings(+ lambda pair: setattr(self, "_memory_pending_launch", pair)+ )try:gguf_size = self._get_gguf_size_bytes(model_path)# Include GPU-loaded mmproj in the fit budget (#5825). GPU-loaded is@@ -23900,7 +24105,25 @@ class LlamaCppBackend:# this arm is reached only with no GPU enumerated, so no device has# a budget of its own and _shared_pool_mmproj charged those bytes.# Adding them again would price the encoder twice.- _apple_model_size_fit = model_size_fit+ # Exclude pageable input embeddings that the probe found outside Metal.+ _apple_host_embd = self._metal_demand_paged_embedding_bytes(+ model_path,+ extra_args,+ binary = binary,+ requested_load_mode = load_mode,+ supports_load_mode = bool(server_caps.get("supports_load_mode")),+ settings = _mem_settings,+ layers_fixed = _caller_owns_budget or _paravirtual_cpu_forced,+ mtp_may_engage = _mtp_will_engage,+ )+ _apple_model_size_fit = model_size_fit - _apple_host_embd+ if _apple_host_embd:+ logger.info(+ "Metal fit: %.2f GB of input embeddings stay in the CPU "+ "file mapping; charging %.2f GB of weights to unified memory.",+ _apple_host_embd / (1024**3),+ _apple_model_size_fit / (1024**3),+ )def _apple_ctx_fit(target: int, min_ctx: int) -> int:return self._fit_context_to_vram(@@ -23963,7 +24186,7 @@ class LlamaCppBackend:# and whenever the GGUF carries no context length and the# request itself is that small._weights_over_budget = (- model_size_fit / (1024 * 1024)+ _apple_model_size_fit / (1024 * 1024)) > _apple_fit_budget_mibif _weights_over_budget:# The fit priced nothing: no context rescues a load@@ -24062,6 +24285,10 @@ class LlamaCppBackend:cache_type_kv,nothing_fits = _apple_nothing_fits,)+ # Unmeasured floors still rely on llama.cpp's fitter.+ _metal_full_offload_pinned = bool(+ _apple_host_embd and _apple_measured_ceiling is not None+ )# Prefer fewer serving slots on GPU over --fit on offload: when the extra# --parallel slots push the footprint past the pin budget, llama-server@@ -24576,6 +24803,7 @@ class LlamaCppBackend:except Exception as e:logger.warning(f"GPU selection failed ({e}), using --fit on")gpu_indices, use_fit = None, True+ _metal_full_offload_pinned = False# Not a verdict: this arm never priced anything. Explicit, so# the flag cannot survive from a previous load._placement_verdict_partial = False@@ -24986,6 +25214,17 @@ class LlamaCppBackend:# Single effective GPU: the split is never emitted, so# don't report it as active via /status and /load.self._tensor_split = None+ elif use_fit and _metal_full_offload_pinned:+ # Keep the launch on the full-offload mapping the discount measured.+ cmd.extend(["-ngl", "-1", "--fit", "off"])+ # Same claim as the pinned-GPU arm below, so it owes the same+ # recovery: the --fit on retry after a startup crash is gated on+ # this flag, and without it an optimistic discount would skip+ # straight to the terminal fallbacks. -1 is llama.cpp's own+ # default (llama_model_default_params), so the retry's fitter+ # does not hit its "n_gpu_layers already set by user" abort+ # (common/fit.cpp:462) and can still move layers off Metal.+ fully_gpu_offloaded = Trueelif use_fit:# Unsloth could not prove a fit, so llama.cpp's fitter takes the# placement. Its dense path fills "back to front with dense@@ -25758,8 +25997,6 @@ class LlamaCppBackend:# GPU with full offload it would hold a second copy of the model# in system RAM and do nothing for VRAM, so it is not emitted# and the idle-unload veto carries residency by itself.- from utils.model_memory_settings import capture_model_memory_settings-# fully_gpu_offloaded is only set by the auto branch. Manual mode# and a user -ngl reach the same placement by their own routes,# so derive those too or they would still be pinned. The CPU@@ -25786,13 +26023,6 @@ class LlamaCppBackend:if gpu_ids is not None:_mem_extra_args = self._strip_device_extra_args(extra_args)self._clear_device_placement_env(_mem_env)- # ONE read for every decision below, published as one act: the window a- # save must not fall through opens at the CAPTURE, not the spawn. It- # carries the pair because `_memory_state` is None until the flags- # resolve, which the comparator reads as "not governed".- _mem_settings = capture_model_memory_settings(- lambda pair: setattr(self, "_memory_pending_launch", pair)- )_mem_keep_resident, _mem_no_reserve = _mem_settings_mem_should_mlock = _mem_keep_resident and not _mem_no_reserve# Armed HERE, not at the load call: that also spans the Hub download, anddiff --git a/studio/backend/core/inference/offload_layout.py b/studio/backend/core/inference/offload_layout.pyindex 1b98122c948..4a099fb0c09 100644--- a/studio/backend/core/inference/offload_layout.py+++ b/studio/backend/core/inference/offload_layout.py@@ -56,6 +56,8 @@ class ModelLayout:# llama-model.cpp pins dev_input to the CPU unconditionally, so this is never charged to VRAM. Tracked because it IS# charged to host RAM.token_embd_bytes: int = 0+ # Every tensor's bytes, excluded blocks included; the file size less its metadata.+ tensor_bytes: int = 0# output_norm and friends: GPU-resident, too small to be worth spilling.other_resident_bytes: int = 0# Attention cache for ONE token at f16, across the attention layers only.@@ -256,7 +258,8 @@ def _layout_from_readers(readers) -> ModelLayout:else:resident[index] = resident.get(index, 0) + nbytescontinue- if "token_embd" in name:+ # token_embd_norm is a repeating-layer tensor, not a host-pinned input embedding.+ if "token_embd" in name and not name.startswith("token_embd_norm"):token_embd += nbyteselif name == "output.weight":lm_head += nbytes@@ -306,6 +309,7 @@ def _layout_from_readers(readers) -> ModelLayout:blocks = blocks,lm_head_bytes = lm_head,token_embd_bytes = token_embd,+ tensor_bytes = sum(int(t.n_bytes) for r in readers for t in r.tensors),other_resident_bytes = other_resident,kv_bytes_per_token_f16 = kv_per_token,recurrent_bytes = recurrent,diff --git a/studio/backend/tests/test_metal_explicit_context_guard.py b/studio/backend/tests/test_metal_explicit_context_guard.pyindex d10dc94cbfd..9e489c96ee5 100644--- a/studio/backend/tests/test_metal_explicit_context_guard.py+++ b/studio/backend/tests/test_metal_explicit_context_guard.py@@ -951,3 +951,607 @@ class TestACpuPinnedProjectorOnUnifiedMemory:**self._COMMON,)assert _ctx_values(captured["cmd"])[-1] == "40960"+++_GIB = 1024**3+_MIB = 1024**2+++def _backend_with_embeddings(+ monkeypatch,+ *,+ embd,+ tensors = 12 * _GIB,+ measured = "mapped as designed",+ settings = (False, False),+ layout = None,+):+ """Build a backend with controlled tensor layout and probe results.++ The default result maps everything loadable except the input embeddings into Metal.+ """+ from core.inference.llama_server_args import MEMORY_ENV_VARS+ from core.inference.offload_layout import ModelLayout+ import utils.model_memory_settings as _mem_settings++ monkeypatch.setattr(_mem_settings, "get_model_memory_settings", lambda: settings)+ placement_vars = (+ "LLAMA_ARG_OVERRIDE_TENSOR",+ "LLAMA_ARG_CPU_MOE",+ "LLAMA_ARG_N_CPU_MOE",+ "LLAMA_ARG_N_CPU_FFN",+ "LLAMA_ARG_N_GPU_LAYERS",+ "LLAMA_ARG_DEVICE",+ )+ for name in (*MEMORY_ENV_VARS, *placement_vars):+ monkeypatch.delenv(name, raising = False)+ if layout is None:+ layout = ModelLayout(complete = True, token_embd_bytes = embd, tensor_bytes = tensors)+ if measured == "mapped as designed":+ # Blocks the loader skips are in no buffer, so they are in no row of the table either.+ metal = layout.tensor_bytes - embd - layout.excluded_block_bytes+ measured = (metal // _MIB, embd // _MIB, 2)+ backend = LlamaCppBackend()+ backend._tensor_spill_layout = lambda _path, **_kw: layout+ backend._metal_measured_model_mib = lambda _binary, _path: measured+ return backend+++class TestInputEmbeddingsLeftInTheFileMapping:+ """Exercise Metal fitting when input embeddings remain pageable on the CPU."""++ _COMMON = dict(real_fit = True, budget_bytes = 16 * 1024**3, weights_bytes = 12 * 1024**3)++ def _published(+ self,+ tmp_path,+ monkeypatch,+ *,+ n_ctx,+ kv_per_token = 32 * 1024,+ extra_args = None,+ gpu_memory_mode = "auto",+ gpu_layers = -1,+ **kw,+ ):+ return self._launched(+ tmp_path,+ monkeypatch,+ n_ctx = n_ctx,+ kv_per_token = kv_per_token,+ extra_args = extra_args,+ gpu_memory_mode = gpu_memory_mode,+ gpu_layers = gpu_layers,+ **kw,+ )[:2]++ def _launched(+ self,+ tmp_path,+ monkeypatch,+ *,+ n_ctx,+ kv_per_token = 32 * 1024,+ extra_args = None,+ gpu_memory_mode = "auto",+ gpu_layers = -1,+ budget_bytes = 16 * _GIB,+ **kw,+ ):+ """``(published ceiling, launched context, argv)``."""+ kw.setdefault("embd", 6 * 1024**3)+ captured = _launch(+ tmp_path,+ monkeypatch,+ n_ctx = n_ctx,+ kv_per_token = kv_per_token,+ extra_args = extra_args,+ gpu_memory_mode = gpu_memory_mode,+ gpu_layers = gpu_layers,+ backend = _backend_with_embeddings(monkeypatch, **kw),+ **{**self._COMMON, "budget_bytes": budget_bytes},+ )+ cmd = captured["cmd"]+ return captured["backend"]._max_context_length, _ctx_values(cmd)[-1], cmd++ @staticmethod+ def _fit_tokens(cmd):+ """The placement tokens this launch emitted, in order."""+ out = []+ for i, token in enumerate(cmd):+ if token in ("-ngl", "--fit") and i + 1 < len(cmd):+ out.extend([token, cmd[i + 1]])+ return out++ @pytest.mark.parametrize("n_ctx", [0, 32768])+ def test_a_discounted_ceiling_launches_pinned_to_the_measured_placement(+ self, tmp_path, monkeypatch, n_ctx+ ):+ """A discounted launch must preserve the probe's full-offload placement."""+ from core.inference.llama_server_args import fit_is_effectively_on++ published, _, cmd = self._launched(tmp_path, monkeypatch, n_ctx = n_ctx)+ assert published > _FIT_MIN_CTX+ assert self._fit_tokens(cmd) == ["-ngl", "-1", "--fit", "off"]+ assert not fit_is_effectively_on(cmd, {})++ def test_a_load_with_nothing_taken_out_keeps_the_fitter(self, tmp_path, monkeypatch):+ _, _, cmd = self._launched(tmp_path, monkeypatch, n_ctx = 32768, embd = 0)+ assert self._fit_tokens(cmd) == ["--fit", "on"]++ def test_a_floor_the_discount_did_not_lift_keeps_the_fitter(self, tmp_path, monkeypatch):+ published, _, cmd = self._launched(tmp_path, monkeypatch, n_ctx = 0, embd = _GIB)+ assert published == _FIT_MIN_CTX+ assert self._fit_tokens(cmd) == ["--fit", "on"]++ @pytest.mark.parametrize("extra_args", [["--fit", "on"], ["--fit=on"], ["-fit", "1"]])+ def test_an_extra_that_turns_the_fitter_back_on_keeps_the_charge(+ self, tmp_path, monkeypatch, extra_args+ ):+ """A later pass-through flag can override Unsloth's placement pin."""+ from core.inference.llama_server_args import fit_is_effectively_on++ published, _, cmd = self._launched(+ tmp_path, monkeypatch, n_ctx = 32768, extra_args = extra_args+ )+ assert published == _FIT_MIN_CTX+ assert fit_is_effectively_on(cmd, {})+ assert "-ngl" not in cmd++ def test_a_pass_through_fit_off_still_matches_the_pin(self, tmp_path, monkeypatch):+ from core.inference.llama_server_args import fit_is_effectively_on++ published, _, cmd = self._launched(+ tmp_path, monkeypatch, n_ctx = 32768, extra_args = ["--fit", "off"]+ )+ assert published > 32768+ assert self._fit_tokens(cmd)[:4] == ["-ngl", "-1", "--fit", "off"]+ assert not fit_is_effectively_on(cmd, {})++ def test_a_placement_it_did_not_measure_keeps_the_fitter(self, tmp_path, monkeypatch):+ _, _, cmd = self._launched(+ tmp_path, monkeypatch, n_ctx = 32768, extra_args = ["--load-mode", "none"]+ )+ assert self._fit_tokens(cmd) == ["--fit", "on"]++ def test_charged_whole_the_same_load_publishes_the_floor(self, tmp_path, monkeypatch):+ published, _ = self._published(tmp_path, monkeypatch, n_ctx = 32768, embd = 0)+ assert published == _FIT_MIN_CTX++ def test_an_explicit_context_publishes_a_measured_ceiling(self, tmp_path, monkeypatch):+ published, launched = self._published(tmp_path, monkeypatch, n_ctx = 32768)+ assert launched == "32768"+ assert published > 32768++ def test_the_ceiling_follows_the_kv_cache_size(self, tmp_path, monkeypatch):+ wide, _ = self._published(tmp_path, monkeypatch, n_ctx = 0, kv_per_token = 32 * 1024)+ narrow, _ = self._published(tmp_path, monkeypatch, n_ctx = 0, kv_per_token = 64 * 1024)+ assert _FIT_MIN_CTX < narrow < wide++ def test_a_short_measured_ceiling_is_refused_past_rather_than_floored(+ self, tmp_path, monkeypatch+ ):+ """A measured ceiling below the usual floor still refuses larger contexts."""+ with pytest.raises(RuntimeError, match = "unified") as refused:+ self._published(tmp_path, monkeypatch, n_ctx = 32768, kv_per_token = _FAT_KV)+ assert 0 < _named_ceiling(str(refused.value)) < _FIT_MIN_CTX++ def test_a_build_that_maps_the_embeddings_keeps_the_charge(self, tmp_path, monkeypatch):+ published, _ = self._published(+ tmp_path, monkeypatch, n_ctx = 32768, measured = (12 * 1024, 6 * 1024, 2)+ )+ assert published == _FIT_MIN_CTX++ def test_a_draft_that_loads_the_targets_own_mtp_blocks_keeps_the_charge(+ self, tmp_path, monkeypatch+ ):+ from core.inference.offload_layout import ModelLayout++ layout = ModelLayout(+ complete = True,+ token_embd_bytes = 6 * _GIB,+ tensor_bytes = 12 * _GIB,+ has_excluded_blocks = True,+ excluded_block_bytes = _GIB,+ )+ published, _ = self._published(+ tmp_path,+ monkeypatch,+ n_ctx = 32768,+ extra_args = ["--spec-type", "draft-mtp"],+ layout = layout,+ measured = (6 * 1024, 6 * 1024, 2),+ )+ assert published == _FIT_MIN_CTX++ def test_no_measurement_keeps_the_charge(self, tmp_path, monkeypatch):+ published, _ = self._published(tmp_path, monkeypatch, n_ctx = 32768, measured = None)+ assert published == _FIT_MIN_CTX++ def test_a_fixed_manual_layer_count_keeps_the_charge(self, tmp_path, monkeypatch):+ published, _ = self._published(+ tmp_path, monkeypatch, n_ctx = 32768, gpu_memory_mode = "manual", gpu_layers = 20+ )+ assert published == _FIT_MIN_CTX++ def test_the_fit_and_the_launch_price_the_same_settings_read(self, tmp_path, monkeypatch):+ """A settings change during loading must not split fit and launch decisions."""+ import utils.model_memory_settings as _mem_settings++ backend = _backend_with_embeddings(monkeypatch, embd = 6 * 1024**3)+ reads = []++ def settings():+ reads.append(None)+ return (False, False) if len(reads) == 1 else (True, False)++ monkeypatch.setattr(_mem_settings, "get_model_memory_settings", settings)+ captured = _launch(+ tmp_path,+ monkeypatch,+ n_ctx = 32768,+ kv_per_token = 32 * 1024,+ backend = backend,+ **self._COMMON,+ )+ locked = any("mlock" in str(token) for token in captured["cmd"])+ discounted = captured["backend"]._max_context_length > _FIT_MIN_CTX+ assert locked != discounted++ @pytest.mark.parametrize(+ "extra_args",+ [+ ["--no-mmap"],+ ["--mlock"],+ ["--load-mode", "none"],+ ["--load-mode", "dio"],+ ["--override-tensor", "per_layer_token_embd=CPU"],+ ["--cpu-moe"],+ ["-ncffn", "4"],+ ["--fit-target", "2048"],+ ],+ )+ def test_a_loader_or_placement_it_did_not_measure_keeps_the_charge(+ self, tmp_path, monkeypatch, extra_args+ ):+ published, _ = self._published(tmp_path, monkeypatch, n_ctx = 32768, extra_args = extra_args)+ assert published == _FIT_MIN_CTX+++class TestWhichLoadsLeaveTheEmbeddingsInTheMapping:+ """Cover loader and placement inputs that control the discount."""++ BYTES = 3 * _GIB+ # The embeddings, less a MiB per measured table row for rounding.+ DISCOUNT = BYTES - 2 * _MIB++ def _bytes(+ self,+ monkeypatch,+ *,+ extra_args = None,+ env = None,+ load_mode = None,+ layers_fixed = False,+ mtp_may_engage = False,+ **kw,+ ):+ kw.setdefault("embd", self.BYTES)+ backend = _backend_with_embeddings(monkeypatch, **kw)+ return backend._metal_demand_paged_embedding_bytes(+ "/models/model.gguf",+ extra_args,+ binary = "/fake/llama-server",+ requested_load_mode = load_mode,+ supports_load_mode = True,+ settings = kw.get("settings", (False, False)),+ layers_fixed = layers_fixed,+ mtp_may_engage = mtp_may_engage,+ env = env or {},+ )++ @pytest.mark.parametrize("load_mode", [None, "auto", "mmap"])+ def test_a_mapped_load_discounts_the_unmapped_embeddings(self, monkeypatch, load_mode):+ assert self._bytes(monkeypatch, load_mode = load_mode) == self.DISCOUNT++ def test_weights_llama_cpp_moved_to_the_cpu_stay_charged(self, monkeypatch):+ """CPU fallback weights remain charged."""+ tensors = 12 * _GIB+ # 7 GiB Metal, 3 GiB embeddings, and 2 GiB CPU fallback.+ measured = (7 * 1024, 5 * 1024, 3)+ unmapped = self._bytes(monkeypatch, tensors = tensors, measured = measured)+ charged = tensors - unmapped+ assert charged >= 9 * _GIB+ assert unmapped == 3 * _GIB - 3 * _MIB++ def test_embeddings_mapped_into_metal_are_not_taken_out(self, monkeypatch):+ """A collapsed Metal span can include the embeddings."""+ tensors = 12 * _GIB+ measured = (10 * 1024, 5 * 1024, 3) # the span covers embeddings, fallback is on the host+ assert self._bytes(monkeypatch, tensors = tensors, measured = measured) == 0++ def test_never_more_than_the_embeddings_comes_out(self, monkeypatch):+ """Unaccounted bytes cannot increase the discount past the embeddings."""+ measured = (4 * 1024, 3 * 1024, 2) # 7 GiB reported of a 12 GiB file+ assert self._bytes(monkeypatch, tensors = 12 * _GIB, measured = measured) == self.BYTES++ @staticmethod+ def _layout_with_nextn(has_nextn):+ from core.inference.offload_layout import ModelLayout+ return ModelLayout(+ complete = True,+ token_embd_bytes = 3 * _GIB,+ tensor_bytes = 12 * _GIB,+ has_excluded_blocks = has_nextn,+ excluded_block_bytes = 2 * _GIB if has_nextn else 0,+ )++ def test_its_own_mtp_blocks_loading_drops_it(self, monkeypatch):+ """The probe cannot model target-embedded MTP blocks."""+ layout = self._layout_with_nextn(True)+ assert self._bytes(monkeypatch, layout = layout, mtp_may_engage = True) == 0++ @pytest.mark.parametrize(("has_nextn", "mtp_may_engage"), [(True, False), (False, True)])+ def test_mtp_blocks_that_stay_skipped_keep_it(self, monkeypatch, has_nextn, mtp_may_engage):+ layout = self._layout_with_nextn(has_nextn)+ unmapped = self._bytes(monkeypatch, layout = layout, mtp_may_engage = mtp_may_engage)+ assert unmapped == self.DISCOUNT++ def test_skipped_mtp_bytes_are_not_read_as_unmapped_embeddings(self, monkeypatch):+ """TENSOR_SKIP keeps the trailing blocks out of every buffer AND every table row."""+ layout = self._layout_with_nextn(True)+ # 12 GiB file = 7 GiB trunk + 3 GiB embeddings + 2 GiB skipped MTP. The Metal span+ # collapses over the embeddings, so nothing is demand-paged: 10 GiB Metal, the+ # embeddings again in the host row, and no sign of the 2 GiB llama.cpp never created.+ measured = (10 * 1024, 3 * 1024, 2)+ assert self._bytes(monkeypatch, layout = layout, measured = measured) == 0++ @pytest.mark.parametrize("load_mode", ["none", "mlock", "mmap+mlock", "dio"])+ def test_a_holding_per_model_mode_drops_it(self, monkeypatch, load_mode):+ assert self._bytes(monkeypatch, load_mode = load_mode) == 0++ @pytest.mark.parametrize(+ "extra_args",+ [+ ["--lora", "/a.gguf"],+ ["--lora-scaled", "/a.gguf:0.5"],+ ["--control-vector", "/v.gguf"],+ ["--control-vector-scaled", "/v.gguf:0.5"],+ ["--lora=/a.gguf"],+ ],+ )+ def test_a_pass_through_adapter_drops_it(self, monkeypatch, extra_args):+ """The probe loads the base GGUF alone, and no Apple term charges the adapter."""+ assert self._bytes(monkeypatch, extra_args = extra_args) == 0++ def test_the_adapter_gate_is_the_flag_set_the_other_consumers_price(self, monkeypatch):+ from core.inference.llama_cpp import _SIDECAR_ADAPTER_FLAGS+ for flag in _SIDECAR_ADAPTER_FLAGS:+ assert self._bytes(monkeypatch, extra_args = [flag, "/a.gguf:0.5"]) == 0++ def test_keep_model_in_gpu_memory_drops_it(self, monkeypatch):+ assert self._bytes(monkeypatch, settings = (True, False)) == 0++ def test_no_ram_reserve_vetoes_a_holding_mode_back_to_the_mapping(self, monkeypatch):+ assert self._bytes(monkeypatch, load_mode = "none", settings = (False, True)) == self.DISCOUNT++ @pytest.mark.parametrize(+ "extra_args",+ [["--no-mmap"], ["--mlock"], ["--load-mode=none"], ["-lm", "mmap+mlock"], ["--direct-io"]],+ )+ def test_a_holding_pass_through_flag_drops_it(self, monkeypatch, extra_args):+ assert self._bytes(monkeypatch, extra_args = extra_args) == 0++ def test_a_pass_through_mmap_keeps_it(self, monkeypatch):+ assert self._bytes(monkeypatch, extra_args = ["--load-mode", "mmap"]) == self.DISCOUNT++ @pytest.mark.parametrize("extra_args", [["--fit", "on"], ["--fit=on"], ["-fit", "true"]])+ def test_a_pass_through_fit_on_drops_it(self, monkeypatch, extra_args):+ assert self._bytes(monkeypatch, extra_args = extra_args) == 0++ @pytest.mark.parametrize("extra_args", [["--fit", "off"], ["--fit=off"]])+ def test_a_pass_through_fit_off_keeps_it(self, monkeypatch, extra_args):+ assert self._bytes(monkeypatch, extra_args = extra_args) == self.DISCOUNT++ def test_an_inherited_fit_on_keeps_it(self, monkeypatch):+ assert self._bytes(monkeypatch, env = {"LLAMA_ARG_FIT": "on"}) == self.DISCOUNT++ @pytest.mark.parametrize(+ "env",+ [{"LLAMA_ARG_MLOCK": "1"}, {"LLAMA_ARG_NO_MMAP": "0"}, {"LLAMA_ARG_LOAD_MODE": "none"}],+ )+ def test_a_holding_inherited_variable_drops_it(self, monkeypatch, env):+ assert self._bytes(monkeypatch, env = env) == 0++ @pytest.mark.parametrize(+ "extra_args",+ [+ ["-ot", "token_embd=MTL0"],+ ["--override-tensor", "per_layer_token_embd=CPU"],+ ["--override-tensor=exps=CPU"],+ ["--cpu-moe"],+ ["--n-cpu-moe", "4"],+ ["-ngl", "20"],+ ["--device", "none"],+ ["-ncffn", "4"],+ ["--n-cpu-ffn=4"],+ ["--fit-target", "2048"],+ ["-fitt", "2048"],+ ["--fit-ctx", "16384"],+ ["-fitc=16384"],+ ],+ )+ def test_a_pass_through_placement_drops_it(self, monkeypatch, extra_args):+ assert self._bytes(monkeypatch, extra_args = extra_args) == 0++ @pytest.mark.parametrize(+ "env",+ [+ {"LLAMA_ARG_OVERRIDE_TENSOR": "exps=CPU"},+ {"LLAMA_ARG_CPU_MOE": "1"},+ {"LLAMA_ARG_N_GPU_LAYERS": "20"},+ {"LLAMA_ARG_DEVICE": "none"},+ {"LLAMA_ARG_N_CPU_FFN": "4"},+ {"LLAMA_ARG_FIT_TARGET": "2048"},+ {"LLAMA_ARG_FIT_CTX": "16384"},+ ],+ )+ def test_an_inherited_placement_drops_it(self, monkeypatch, env):+ assert self._bytes(monkeypatch, env = env) == 0++ def test_a_request_with_a_fixed_layer_count_drops_it(self, monkeypatch):+ assert self._bytes(monkeypatch, layers_fixed = True) == 0++ def test_an_unreadable_layout_abstains(self, monkeypatch):+ from core.inference.offload_layout import ModelLayout+ assert self._bytes(monkeypatch, layout = ModelLayout()) == 0++ def test_no_measurement_abstains_and_says_so(self, monkeypatch):+ """Recorded off the module logger, not caplog.++ The structlog stub at the top of this file is installed with+ `sys.modules.setdefault`, so in a full run any module that imported the real+ structlog first wins and the log never reaches a stdlib handler. Under xdist+ that depends on which worker gets this file, which made the caplog spelling+ pass alone and fail in CI.+ """+ import core.inference.llama_cpp as llama_cpp++ said = []+ monkeypatch.setattr(llama_cpp.logger, "info", lambda msg, *a, **kw: said.append(str(msg)))+ assert self._bytes(monkeypatch, measured = None) == 0+ assert any("could not measure" in line for line in said)+++# b10909-mix output from an 8 GB M1; the second table adds -ncffn 30.+_E4B_TABLE = """0.00.408.825 I common_memory_breakdown_print: | memory breakdown [MiB] | total free self model context compute unaccounted |+0.00.429.748 I common_memory_breakdown_print: | - MTL0 (Apple M1) | 5461 = 5460 + (3642 = 3025 + 28 + 589) + -3642 |+0.00.429.748 I common_memory_breakdown_print: | - Host | 2352 = 2288 + 0 + 64 |+0.00.444.070 I llama_fit_params: printing fitted CLI arguments to stdout...+-c 512 -ngl 999+"""+_E4B_NCFFN_TABLE = """I common_memory_breakdown_print: | memory breakdown [MiB] | total free self model context compute unaccounted |+I common_memory_breakdown_print: | - MTL0 (Apple M1) | 5461 = 5460 + (2196 = 1625 + 28 + 543) + -2196 |+I common_memory_breakdown_print: | - Host | 2430 = 2288 + 0 + 142 |+I common_memory_breakdown_print: | - CPU_REPACK | 1400 = 1400 + 0 + 0 |+"""+++class TestTheMetalMemoryProbe:+ """Parse and invoke llama-fit-params conservatively."""++ def test_it_reads_the_metal_and_host_model_columns(self):+ from core.inference.llama_cpp import _parse_metal_memory_breakdown+ assert _parse_metal_memory_breakdown(_E4B_TABLE) == (3025, 2288, 2)++ def test_every_host_buffer_type_is_summed(self):+ from core.inference.llama_cpp import _parse_metal_memory_breakdown+ assert _parse_metal_memory_breakdown(_E4B_NCFFN_TABLE) == (1625, 2288 + 1400, 3)++ def test_only_the_first_table_is_read(self):+ from core.inference.llama_cpp import _parse_metal_memory_breakdown+ assert _parse_metal_memory_breakdown(_E4B_TABLE + _E4B_NCFFN_TABLE) == (3025, 2288, 2)++ @pytest.mark.parametrize(+ "output",+ [+ "",+ "error: invalid argument: -lv",+ _E4B_TABLE.replace("MTL0 (Apple M1)", "CUDA0 (RTX 4090)"),+ _E4B_TABLE.replace("2352 = 2288 +", "2352"),+ _E4B_TABLE.replace("3025 +", "x +"),+ ],+ )+ def test_anything_else_is_unknown(self, output):+ from core.inference.llama_cpp import _parse_metal_memory_breakdown+ assert _parse_metal_memory_breakdown(output) is None++ @staticmethod+ def _install(tmp_path, script):+ bin_dir = tmp_path / "bin"+ bin_dir.mkdir()+ (bin_dir / "llama-server").write_text("")+ probe = bin_dir / "llama-fit-params"+ probe.write_text(script)+ probe.chmod(0o755)+ model = tmp_path / "model.gguf"+ model.write_bytes(b"GGUF")+ return str(bin_dir / "llama-server"), str(model)++ @pytest.mark.skipif(sys.platform == "win32", reason = "shell script stands in for the binary")+ def test_it_runs_the_probe_beside_the_binary_and_caches_the_answer(self, tmp_path, monkeypatch):+ calls = tmp_path / "calls"+ table = _E4B_TABLE.replace("\n", "\\n")+ script = f'#!/bin/sh\necho "$@" >> "{calls}"\nprintf "{table}"\n'+ binary, model = self._install(tmp_path, script)+ monkeypatch.setenv("LLAMA_ARG_N_GPU_LAYERS", "10")+ backend = LlamaCppBackend()+ assert backend._metal_measured_model_mib(binary, model) == (3025, 2288, 2)+ assert backend._metal_measured_model_mib(binary, model) == (3025, 2288, 2)+ argv = calls.read_text().splitlines()+ assert argv == [f"-m {model} -ngl 999 -c 512 -lv 4"]++ @pytest.mark.skipif(sys.platform == "win32", reason = "shell script stands in for the binary")+ def test_a_failing_probe_is_unknown(self, tmp_path):+ binary, model = self._install(tmp_path, "#!/bin/sh\nexit 1\n")+ assert LlamaCppBackend()._metal_measured_model_mib(binary, model) is None++ def test_no_probe_is_unknown(self, tmp_path):+ (tmp_path / "llama-server").write_text("")+ model = tmp_path / "model.gguf"+ model.write_bytes(b"GGUF")+ backend = LlamaCppBackend()+ assert backend._metal_measured_model_mib(str(tmp_path / "llama-server"), str(model)) is None+ assert backend._metal_measured_model_mib(None, str(model)) is None+++def test_every_forced_full_offload_arm_owes_the_fit_on_retry():+ """A forced "-ngl -1 --fit off" must also claim the full offload.++ The `--fit on` retry after a startup crash is gated on `fully_gpu_offloaded`,+ and the tensor-spill recovery ahead of it is a no-op without a spill plan, so+ an arm that pins the placement without setting the flag drops straight to the+ terminal fallbacks when its estimate turns out optimistic. Checked at the+ source, like the other invariants over this launch path, because the retry only+ runs behind a real child crash.+ """+ import ast+ import inspect+ import textwrap++ from core.inference.llama_cpp import LlamaCppBackend++ tree = ast.parse(textwrap.dedent(inspect.getsource(LlamaCppBackend.load_model)))++ def pins_full_offload(stmt):+ """The emission as a DIRECT statement of the arm, so an enclosing `if` does+ not also count as one."""+ if not isinstance(stmt, ast.Expr) or not isinstance(stmt.value, ast.Call):+ return False+ call = stmt.value+ if not isinstance(call.func, ast.Attribute) or call.func.attr != "extend":+ return False+ if not call.args or not isinstance(call.args[0], ast.List):+ return False+ values = [e.value for e in call.args[0].elts if isinstance(e, ast.Constant)]+ return values == ["-ngl", "-1", "--fit", "off"]++ def claims_full_offload(body):+ for stmt in body:+ if not isinstance(stmt, ast.Assign) or not isinstance(stmt.value, ast.Constant):+ continue+ if stmt.value.value is not True:+ continue+ if any(isinstance(t, ast.Name) and t.id == "fully_gpu_offloaded" for t in stmt.targets):+ return True+ return False++ arms = [+ branch+ for node in ast.walk(tree)+ if isinstance(node, ast.If)+ for branch in (node.body, node.orelse)+ if branch and any(pins_full_offload(stmt) for stmt in branch)+ ]+ assert len(arms) == 2, f"expected the two forced full-offload arms, found {len(arms)}"+ assert all(claims_full_offload(arm) for arm in arms)diff --git a/studio/backend/tests/test_model_memory_settings.py b/studio/backend/tests/test_model_memory_settings.pyindex 484856ef18c..5fe558a5c59 100644--- a/studio/backend/tests/test_model_memory_settings.py+++ b/studio/backend/tests/test_model_memory_settings.py@@ -1656,12 +1656,31 @@ class TestCpuMoeCountIsParsed:(["-cmoe"], True),(["-ot", "blk.*=CPU"], True),(["--override-tensor", "x"], True),+ # The dense-model count: common/arg.cpp turns it into CPU FFN overrides.+ (["-ncffn", "0"], False),+ (["-ncffn", "4"], True),+ (["--n-cpu-ffn", "4"], True),+ (["--n-cpu-ffn=4"], True),+ (["--n_cpu_ffn", "4"], True),],)def test_the_predicate(self, extras, expected):from core.inference.llama_cpp import _args_place_tensors_on_cpuassert _args_place_tensors_on_cpu(extras) is expected+ @pytest.mark.parametrize(+ ("env", "expected"),+ [+ ({}, False),+ ({"LLAMA_ARG_N_CPU_MOE": "4"}, True),+ ({"LLAMA_ARG_N_CPU_FFN": "0"}, False),+ ({"LLAMA_ARG_N_CPU_FFN": "4"}, True),+ ],+ )+ def test_the_env_predicate(self, env, expected):+ from core.inference.llama_cpp import _env_places_tensors_on_cpu+ assert _env_places_tensors_on_cpu(env) is expected+def test_it_is_the_same_predicate_the_pipeline_check_uses(self):import astimport inspectdiff --git a/studio/backend/tests/test_offload_planner.py b/studio/backend/tests/test_offload_planner.pyindex 965eb6d40d9..792b020eedb 100644--- a/studio/backend/tests/test_offload_planner.py+++ b/studio/backend/tests/test_offload_planner.py@@ -987,6 +987,44 @@ def test_a_single_file_gguf_is_still_read():assert len(layout.blocks) == 64+def test_the_layout_reader_totals_every_tensor_from_a_real_gguf(tmp_path):+ """tensor_bytes includes every tensor while token_embd_bytes is embedding-only."""+ import numpy as np+ from gguf import GGUFWriter++ path = tmp_path / "embd.gguf"+ writer = GGUFWriter(str(path), "llama")+ for key, value in _shard_fields().items():+ if key.startswith("llama.") and isinstance(value, int):+ writer.add_uint32(key, 1 if key == "llama.block_count" else value)+ writer.add_tensor("output_norm.weight", np.zeros(16, dtype = np.float32))+ writer.add_tensor("token_embd.weight", np.zeros(1024, dtype = np.float32))+ writer.add_tensor("per_layer_token_embd.weight", np.zeros(4096, dtype = np.float32))+ writer.add_tensor("blk.0.attn_q.weight", np.zeros(256, dtype = np.float32))+ writer.add_tensor("blk.1.attn_q.weight", np.zeros(64, dtype = np.float32))+ writer.write_header_to_file()+ writer.write_kv_data_to_file()+ writer.write_tensors_to_file()+ writer.close()++ layout = layout_from_gguf(str(path))+ assert layout.complete+ assert layout.tensor_bytes == 4 * (16 + 1024 + 4096 + 256 + 64)+ assert layout.token_embd_bytes == 4 * (1024 + 4096)+++def test_token_embd_norm_is_not_an_input_embedding():+ """token_embd_norm follows the repeating layers rather than the input layer."""+ tensors = _shard_tensors(range(64)) + [+ _StubTensor("token_embd.weight", 400 * MIB),+ _StubTensor("token_embd_norm.weight", 4 * MIB),+ _StubTensor("output.weight", 400 * MIB),+ ]+ layout = _layout_from_reader(_StubReader(_shard_fields(), tensors))+ assert layout.token_embd_bytes == 400 * MIB+ assert layout.other_resident_bytes == 4 * MIB++def test_a_split_gguf_abstains_instead_of_planning_on_one_shard():"""GGUFReader memmaps the ONE path it is given, but llama.cpp readssplit.count off the first shard and loads every siblingdiff --git a/studio/install_llama_prebuilt.py b/studio/install_llama_prebuilt.pyindex 369b73c408d..68d6cb2ed90 100644--- a/studio/install_llama_prebuilt.py+++ b/studio/install_llama_prebuilt.py@@ -4318,6 +4318,8 @@ def runtime_patterns_for_install_kind("llama-server","llama-quantize","llama-diffusion-gemma-visual-server",+ # Optional Metal memory probe; missing bundles keep the conservative estimate.+ "llama-fit-params","lib*.dylib",]if install_kind in {diff --git a/tests/studio/install/test_rocm_support.py b/tests/studio/install/test_rocm_support.pyindex 76f7224a2d8..d4f3f5a72d1 100644--- a/tests/studio/install/test_rocm_support.py+++ b/tests/studio/install/test_rocm_support.py@@ -446,6 +446,13 @@ class TestRuntimePatterns:)assert name in runtime_patterns_for_choice(choice)+ def test_fit_params_kept_on_macos(self):+ for kind in ("macos-arm64", "macos-x64"):+ choice = AssetChoice(+ repo = "", tag = "", name = "", url = "", source_label = "", install_kind = kind+ )+ assert "llama-fit-params" in runtime_patterns_for_choice(choice)+# TEST: install_llama_prebuilt.py -- HostInfo.has_rocm field