Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

Studio: reserve the DSpark/DFlash sidecar in --fit-target and recover from a Metal out-of-memory (#14431)

Daniel Han215039af8d9a
diff --git a/studio/backend/core/inference/llama_cpp.py b/studio/backend/core/inference/llama_cpp.py
index ff84eda8ee8..c654bc6a16b 100644
--- a/studio/backend/core/inference/llama_cpp.py
+++ b/studio/backend/core/inference/llama_cpp.py
@@ -16,6 +16,7 @@ import ctypes
import errno
import functools
import hashlib
+import itertools
import json
import logging
import math
@@ -657,7 +658,33 @@ class _LlamaStreamCancelled(Exception):
# vulkan-hpp formats DeviceLostError as "vk::Queue::submit: ErrorDeviceLost".
_GPU_DEVICE_LOST_MARKERS = ("ErrorDeviceLost", "VK_ERROR_DEVICE_LOST")
+# ggml-metal latches a failed command buffer ("recreate the backend to recover"); the child then
+# answers "Compute error.".
+_METAL_BACKEND_POISONED_MARKERS = (
+ "backend is in error state from a previous command buffer failure",
+ "kIOGPUCommandBufferCallbackErrorOutOfMemory",
+)
+_DRAFT_CONTEXT_FAILURE_MARKERS = (
+ "llama_process(ctx_dft) failed",
+ "failed to process speculative batch",
+)
_DEVICE_LOST_LOG_GRACE_S = 2.0
+# Run after a terminal Metal OOM unload, outside every backend lock (routes/inference.py registers
+# them).
+_TERMINAL_RETIREMENT_HOOKS: list = []
+
+# Set during automatic recovery loads: only those inherit a Metal OOM strike.
+_AUTO_RECOVERY_LOAD = threading.local()
+
+
+def _automatic_recovery_load():
+ prev = getattr(_AUTO_RECOVERY_LOAD, "active", False)
+ _AUTO_RECOVERY_LOAD.active = True
+ try:
+ yield
+ finally:
+ _AUTO_RECOVERY_LOAD.active = prev
def _is_gpu_device_lost(exc: BaseException) -> bool:
@@ -4675,6 +4702,179 @@ def _binary_has_marker(path: str, _mtime_ns: int, _size: int, marker: bytes) ->
return False
+# No --help flag or build number announces it: read the trace string the new fit path compiles in.
+_FIT_MEASURES_BORROWED_DRAFT_MARKER = (
+ b"measuring the extra model next to an allocation-free target context"
+)
+_LLAMA_COMMON_LIB_NAMES = (
+ "libllama-common.so.0",
+ "libllama-common.so",
+ "libllama-common.0.dylib",
+ "libllama-common.dylib",
+ "llama-common.dll",
+)
+# The server links libllama-server-impl, which links libllama-common.
+_LLAMA_SERVER_IMPL_LIB_NAMES = (
+ "libllama-server-impl.so",
+ "libllama-server-impl.dylib",
+ "llama-server-impl.dll",
+)
+
+
+# (path, mtime_ns, size) -> embedded library names; the server binary is ~100 MB.
+_LINKED_LIB_NAMES_CACHE: dict = {}
+
+
+def _linked_llama_libs(binary: Path, lib_dir: Path) -> list:
+ """The server-impl / common libraries ``binary`` links by name (DT_NEEDED, LC_LOAD_DYLIB, PE
+ imports), not stray siblings."""
+ try:
+ st = binary.stat()
+ key = (str(binary), st.st_mtime_ns, st.st_size)
+ except OSError:
+ return []
+ embedded = _LINKED_LIB_NAMES_CACHE.get(key)
+ if embedded is None:
+ try:
+ data = binary.read_bytes()
+ except OSError:
+ return []
+ embedded = tuple(
+ tuple(n for n in names if n.encode() in data)
+ for names in (_LLAMA_SERVER_IMPL_LIB_NAMES, _LLAMA_COMMON_LIB_NAMES)
+ )
+ _LINKED_LIB_NAMES_CACHE[key] = embedded
+ linked = []
+ for names in embedded:
+ hit = next((n for n in names if (lib_dir / n).exists()), None)
+ if hit is not None:
+ linked.append(lib_dir / hit)
+ return linked
+
+
+_CUDA_LIKE_DEVICE_RE = re.compile(r"(?i)(cuda|rocm|hip)\d+$")
+
+
+def _spawn_draft_on_target(
+ run_cmd: List[str],
+ spawn_devices: Optional[str],
+ env: Optional[Mapping[str, str]] = None,
+) -> bool:
+ """Whether the spawned argv's CUDA / ROCm draft pin names the target's device."""
+ pin = _extra_args_draft_device_pin(run_cmd)
+ if pin is not None:
+ distinct = {d.strip() for d in pin.split(",") if d.strip()}
+ pin = distinct.pop() if len(distinct) == 1 else pin
+ if pin is None or not _CUDA_LIKE_DEVICE_RE.match(pin.strip()):
+ return True
+ if spawn_devices:
+ named = [d.strip().lower() for d in spawn_devices.split(",") if d.strip()]
+ if _split_mode_confines_to_one_device(run_cmd, env):
+ # Only devices[main_gpu] is kept (main_gpu 0 by default).
+ main = LlamaCppBackend._effective_main_gpu(run_cmd, env)
+ main = 0 if main is None else main
+ return 0 <= main < len(named) and pin.strip().lower() == named[main]
+ return set(named) == {d.strip().lower() for d in pin.split(",") if d.strip()}
+ return int(re.sub(r"\D", "", pin)) == (LlamaCppBackend._effective_main_gpu(run_cmd, env) or 0)
+
+
+def _sidecar_borrows_output(path: Optional[str]) -> bool:
+ """Whether the draft's logits use the target's output head (src/models/dflash.cpp); unknown
+ counts as borrowed."""
+ if not path:
+ return True
+ try:
+ from utils.models.gguf_metadata import _gguf_shard_has_named_tensor
+ from utils.models.model_config import colocated_split_shards
+
+ shards, complete = colocated_split_shards(Path(path))
+ return not (
+ complete
+ and any(
+ _gguf_shard_has_named_tensor(str(shard), "output.weight") is True
+ for shard in shards
+ )
+ )
+ except Exception:
+ return True
+
+
+def _sidecar_borrows_target_tensors(path: Optional[str]) -> bool:
+ """Whether the draft borrows token_embd or output, which the fitter cannot measure
+ (src/llama-context.cpp)."""
+ if not path:
+ return True
+ try:
+ from utils.models.gguf_metadata import _gguf_shard_has_named_tensor
+ from utils.models.model_config import colocated_split_shards
+
+ shards, complete = colocated_split_shards(Path(path))
+ if not complete or not shards:
+ return True
+ return any(
+ not any(_gguf_shard_has_named_tensor(str(shard), name) is True for shard in shards)
+ for name in ("token_embd.weight", "output.weight")
+ )
+ except Exception:
+ return True
+
+
+def _draft_pin_device_index(
+ pin: Optional[str], spawn_devices: Optional[str], device_count: int
+) -> Optional[int]:
+ """The child device index a one-card draft pin names, or None."""
+ names = {d.strip().lower() for d in (pin or "").split(",") if d.strip()}
+ if len(names) != 1:
+ return None
+ (name,) = names
+ if spawn_devices:
+ listed = [d.strip().lower() for d in spawn_devices.split(",") if d.strip()]
+ if name not in listed or len(set(listed)) != len(listed):
+ return None
+ return listed.index(name)
+ if not _CUDA_LIKE_DEVICE_RE.match(name):
+ return None
+ k = int(re.sub(r"\D", "", name))
+ return k if 0 <= k < device_count else None
+
+
+def _distinct_device_count(devices) -> int:
+ """Distinct names in a ``--device`` list: a repeated card is one memory pool."""
+ return len({d.strip().lower() for d in str(devices).split(",") if d.strip()})
+
+
+def _fit_measures_borrowed_drafts(bin_path: Optional[str]) -> bool:
+ """Whether ``bin_path``'s fitter sizes a borrowed-tensor draft itself; fails closed."""
+ if not bin_path:
+ return False
+ candidates = [Path(bin_path)]
+ try:
+ # A wrapper script execs the real server.
+ server = Path(_resolve_llama_binary(bin_path))
+ candidates.append(server)
+ lib_dir = _llama_lib_dir(bin_path)
+ for lib in _linked_llama_libs(server, lib_dir):
+ candidates.append(lib)
+ candidates.extend(l for l in _linked_llama_libs(lib, lib_dir) if l not in candidates)
+ except Exception: # noqa: BLE001 -- an unreadable lib dir is "not known"
+ pass
+ seen: set = set()
+ for candidate in candidates:
+ try:
+ candidate = candidate.resolve()
+ if candidate in seen or not candidate.is_file():
+ continue
+ seen.add(candidate)
+ st = candidate.stat()
+ except OSError:
+ continue
+ if _binary_has_marker(
+ str(candidate), st.st_mtime_ns, st.st_size, _FIT_MEASURES_BORROWED_DRAFT_MARKER
+ ):
+ return True
+ return False
+
+
def _swa_full_from_args_or_env(
extra_args: Optional[Iterable[str]], env: Optional[Mapping[str, str]] = None
) -> bool:
@@ -5606,6 +5806,155 @@ def _extra_args_mtp_draft_path(
return _extra_args_mtp_draft_source(extra_args, env)[0]
+def _single_device_fit_margin(
+ args: Iterable[str], env: Optional[Mapping[str, str]]
+) -> Optional[float]:
+ """The margin the one launched device keeps: the FIRST entry of a ``--fit-target`` list.
+
+ ``fit_target_margin_in`` takes the largest entry because it prices every device it
+ credits; a single-device launch only ever uses device 0's entry (common/arg.cpp), so a
+ larger later entry must not be added under the sidecar reserve."""
+ raw = _last_flag_value(args, {"-fitt", "--fit-target"})
+ if raw is None and env:
+ raw = env.get("LLAMA_ARG_FIT_TARGET")
+ if raw is None:
+ return None
+ first = str(raw).replace("/", ",").split(",")[0].strip()
+ try:
+ return float(first)
+ except ValueError:
+ return None
+
+
+def _fit_target_values(user_args: Iterable[str], env: Optional[Mapping[str, str]]) -> list[str]:
+ """Every --fit-target value llama.cpp applies, env first then argv order."""
+ found = []
+ if env and env.get("LLAMA_ARG_FIT_TARGET"):
+ found.append(str(env["LLAMA_ARG_FIT_TARGET"]))
+ tokens = [str(a) for a in user_args]
+ for i, raw in enumerate(tokens):
+ _, eq, inline = raw.partition("=")
+ if _flag_name(raw) in ("-fitt", "--fit-target"):
+ value = inline if eq else (tokens[i + 1] if i + 1 < len(tokens) else None)
+ if value is not None:
+ found.append(value)
+ return found
+
+
+def _fit_margins_by_device(
+ user_args: Optional[Iterable[str]],
+ env: Optional[Mapping[str, str]],
+ base_margin_mib: float,
+ count: int,
+) -> list[float]:
+ """Each card's user --fit-target margin as common/arg.cpp applies it; unset cards keep
+ ``base_margin_mib``."""
+ margins = [float(base_margin_mib)] * count
+ for raw in _fit_target_values(user_args or (), env):
+ entries = [e.strip() for e in str(raw).replace("/", ",").split(",") if e.strip()]
+ try:
+ values = [float(e) for e in entries]
+ except ValueError:
+ continue
+ if len(values) == 1:
+ margins = values * count
+ else:
+ margins[: len(values)] = values[:count]
+ return margins
+
+
+def _sidecar_reserve_margin_flags(
+ child_args: Iterable[str],
+ user_args: Optional[Iterable[str]],
+ env: Optional[Mapping[str, str]],
+ base_margin_mib: float,
+ reserve_mib: int,
+ *,
+ per_device: Optional[list[int]] = None,
+ base_by_device: Optional[list[float]] = None,
+) -> list[str]:
+ """``--fit-target`` keeping an unsized sidecar's reserve free on top of the user's margin, read
+ off the child's argv + env."""
+ args = [str(a) for a in child_args]
+ if not reserve_mib or not _extra_args_mtp_draft_path(args, env = env):
+ return []
+ if not fit_is_effectively_on(args, env):
+ return []
+ # A scalar --fit-target is broadcast to every device, so a multi-device launch needs a
+ # per-device list.
+ devices = _extra_args_main_device(args) or (os.environ if env is None else env).get(
+ "LLAMA_ARG_DEVICE"
+ )
+ if (
+ not per_device
+ and devices
+ and _distinct_device_count(devices) > 1
+ and not _split_mode_confines_to_one_device(args, env)
+ ):
+ return []
+ if per_device:
+ bases = (
+ base_by_device
+ if base_by_device and len(base_by_device) == len(per_device)
+ else [base_margin_mib] * len(per_device)
+ )
+ margins = _fit_margins_by_device(user_args, env, 0.0, len(per_device))
+ want_list = ",".join(
+ str(math.ceil(max(b, m) + r)) for b, m, r in zip(bases, margins, per_device)
+ )
+ if _last_flag_value(args, {"-fitt", "--fit-target"}) == want_list:
+ return []
+ return ["--fit-target", want_list]
+ # Studio's per-card margin, not the host-wide scalar a lowered budget sizes by the largest card.
+ if base_by_device and len(set(base_by_device)) == 1:
+ base_margin_mib = base_by_device[0]
+ want = math.ceil(
+ max(base_margin_mib, _single_device_fit_margin(user_args or (), env) or 0.0) + reserve_mib
+ )
+ have = _single_device_fit_margin(args, None)
+ if have is not None and have >= want:
+ return []
+ return ["--fit-target", str(want)]
+
+
+def _drop_sidecar_margins(
+ cmd: List[str],
+ integrity_flags: List[str],
+ rebuilt_integrity: List[str],
+ override_pairs: Iterable[List[str]],
+ *,
+ user_args: Iterable[str] = (),
+) -> List[str]:
+ """``cmd`` without Studio's sidecar margins; a user's own pairs survive, so it is idempotent."""
+ out = list(cmd)
+ user = [str(a) for a in user_args]
+
+ def _count(tokens: List[str], pair: List[str]) -> int:
+ return sum(tokens[i : i + len(pair)] == pair for i in range(len(tokens) - len(pair) + 1))
+
+ def _studio_owns(pair: List[str]) -> bool:
+ return bool(pair) and _count(out, pair) > _count(user, pair)
+
+ if "--fit-target" in integrity_flags:
+ old_at = integrity_flags.index("--fit-target")
+ old_margin = integrity_flags[old_at : old_at + 2]
+ new_margin: List[str] = []
+ if "--fit-target" in rebuilt_integrity:
+ new_at = rebuilt_integrity.index("--fit-target")
+ new_margin = rebuilt_integrity[new_at : new_at + 2]
+ at = _subsequence_index(out, old_margin, 0)
+ if out[at : at + 2] == old_margin and _studio_owns(old_margin):
+ out[at : at + 2] = new_margin
+ for pair in reversed(list(override_pairs)):
+ if not _studio_owns(pair):
+ continue
+ for i in range(len(out) - len(pair), -1, -1):
+ if out[i : i + len(pair)] == pair:
+ del out[i : i + len(pair)]
+ break
+ return out
+
+
_HF_DRAFT_FLAGS = frozenset({"--spec-draft-hf", "-hfd", "-hfrd", "--hf-repo-draft"})
_LOCAL_DRAFT_FLAGS = frozenset({"--model-draft", "--spec-draft-model", "-md"})
@@ -5679,6 +6028,9 @@ _BREAKDOWN_HOST_ROW_RE = re.compile(
)
+_METAL_FIT_PROBE_TIMEOUT_S = 120
+
+
def _parse_metal_memory_breakdown(output: str) -> Optional[tuple[int, int, int]]:
"""Requires one Metal device row; host model MiB sums the model column of every host row."""
rows: list[str] = []
@@ -5792,6 +6144,115 @@ def _draft_is_split_across_host(
return not (n_draft_layers and count > n_draft_layers)
+_DRAFT_OVERRIDE_TENSOR_FLAGS = frozenset(
+ {"-otd", "--override-tensor-draft", "--spec-draft-override-tensor"}
+)
+_DRAFT_CPU_MOE_FLAGS = frozenset({"--spec-draft-cpu-moe", "-cmoed", "--cpu-moe-draft"})
+_DRAFT_N_CPU_MOE_FLAGS = frozenset(
+ {"--spec-draft-n-cpu-moe", "--spec-draft-ncmoe", "-ncmoed", "--n-cpu-moe-draft"}
+)
+
+
+def _draft_card_share_unknown(
+ extra_args: Optional[Iterable[str]],
+ env: Optional[Mapping[str, str]] = None,
+ *,
+ n_draft_layers: Optional[int] = None,
+) -> bool:
+ """True when the drafter is not wholly on the target's card, so its whole file is not
+ the share to reserve there: a positive -ngld below its depth (split with the host), or
+ a draft tensor override, which llama.cpp applies per weight while picking buffers (the
+ same flags the fit-voiding list treats as placement changes), or the draft CPU-MoE
+ switches, which llama.cpp turns into draft tensor overrides."""
+ if _extra_args_set_any_flag(extra_args, _DRAFT_OVERRIDE_TENSOR_FLAGS | _DRAFT_CPU_MOE_FLAGS):
+ return True
+ source_env = os.environ if env is None else env
+ if source_env.get("LLAMA_ARG_SPEC_DRAFT_CPU_MOE") in _LLAMA_ARG_TRUE_VALUES:
+ return True
+ # Each positive count appends draft CPU overrides, so any positive one is a placement.
+ if _is_positive_int(source_env.get("LLAMA_ARG_SPEC_DRAFT_N_CPU_MOE")):
+ return True
+ args = [str(a) for a in extra_args] if extra_args else []
+ for i, raw in enumerate(args):
+ _, eq, inline = raw.partition("=")
+ if _flag_name(raw) in _DRAFT_N_CPU_MOE_FLAGS and _is_positive_int(
+ inline if eq else (args[i + 1] if i + 1 < len(args) else None)
+ ):
+ return True
+ return _draft_is_split_across_host(extra_args, env, n_draft_layers = n_draft_layers)
+
+
+def _sidecar_child_ctx(
+ args: Optional[Iterable[str]],
+ env: Optional[Mapping[str, str]],
+ native_ctx: Optional[int],
+ n_parallel: Optional[int],
+ kv_unified: bool,
+) -> int:
+ """The context the child allocates: last positive -c, else LLAMA_ARG_CTX_SIZE, else the native
+ length."""
+ ctx = parse_ctx_override(args)
+ if ctx is None:
+ try:
+ ctx = int(
+ str((os.environ if env is None else env).get("LLAMA_ARG_CTX_SIZE", "")).strip()
+ )
+ except (TypeError, ValueError):
+ ctx = None
+ if ctx and ctx > 0:
+ return ctx
+ # Without -c, --kv-unified-per-slot N sizes the pool to n_parallel * N
+ # (tools/server/server.cpp).
+ source_env = os.environ if env is None else env
+ tokens = [str(a) for a in args] if args else []
+ full_window = str(source_env.get("LLAMA_ARG_CTX_SIZE", "")).strip() == "0"
+ if str(source_env.get("LLAMA_ARG_FIT_CTX", "")).strip():
+ full_window = False
+ per_slot = source_env.get("LLAMA_ARG_KV_UNIFIED_PER_SLOT")
+ for i, raw in enumerate(tokens):
+ flag = _flag_name(raw)
+ if flag not in ("-c", "--ctx-size", "-fitc", "--fit-ctx", "--kv-unified-per-slot"):
+ continue
+ _, eq, inline = raw.partition("=")
+ value = inline if eq else (tokens[i + 1] if i + 1 < len(tokens) else "")
+ if flag == "--kv-unified-per-slot":
+ per_slot = value
+ elif flag in ("-fitc", "--fit-ctx"):
+ full_window = False
+ elif str(value).strip() == "0":
+ full_window = True
+ if _is_positive_int(per_slot) and not full_window:
+ return int(str(per_slot).strip()) * max(1, int(n_parallel or 1))
+ bound = int(native_ctx or 0) or _LLAMA_FIT_MIN_CTX
+ if not kv_unified:
+ bound *= max(1, int(n_parallel or 1))
+ return bound
+
+
+def _draft_n_ubatch(
+ args: Optional[Iterable[str]], env: Optional[Mapping[str, str]] = None
+) -> Optional[int]:
+ """A positive --spec-draft-n-ubatch (env twin overridden by argv), else None."""
+ found = (os.environ if env is None else env).get("LLAMA_ARG_SPEC_DRAFT_N_UBATCH")
+ tokens = [str(a) for a in args] if args else []
+ for i, raw in enumerate(tokens):
+ _, eq, inline = raw.partition("=")
+ if _flag_name(raw) == "--spec-draft-n-ubatch":
+ found = inline if eq else (tokens[i + 1] if i + 1 < len(tokens) else None)
+ return int(str(found).strip()) if _is_positive_int(found) else None
+
+
+def _capped_draft_n_ubatch(args, env, ctx, *batch_defaults) -> Optional[int]:
+ """--spec-draft-n-ubatch capped as llama_context caps it: at --batch-size, then a positive
+ context. ``batch_defaults`` are _named_batch_sizes' fallbacks."""
+ ub = _draft_n_ubatch(args, env)
+ if ub:
+ ub = min(ub, _named_batch_sizes(args, env, *batch_defaults)[0] & 0xFFFFFFFF)
+ if ctx and ctx > 0:
+ ub = min(ub, int(ctx))
+ return ub
+
+
_SIDECAR_ADAPTER_FLAGS = (
"--lora",
"--lora-scaled",
@@ -6148,8 +6609,8 @@ def _extra_args_split_mode(
found = str(raw).strip().lower()
args = [str(a) for a in extra_args] if extra_args else []
for i, arg in enumerate(args):
- name, _, inline = arg.partition("=")
- if name not in _SPLIT_MODE_FLAGS:
+ _, _, inline = arg.partition("=")
+ if _flag_name(arg) not in _SPLIT_MODE_FLAGS:
continue
value = inline if inline else (args[i + 1] if i + 1 < len(args) else "")
if value.strip():
@@ -6168,9 +6629,9 @@ def _extra_args_tensor_split(
raw = str(env_value).strip()
args = [str(a) for a in extra_args] if extra_args else []
for i, arg in enumerate(args):
- name, _, inline = arg.partition("=")
- if name not in _TENSOR_SPLIT_FLAGS:
+ if _flag_name(arg) not in _TENSOR_SPLIT_FLAGS:
continue
+ _, _, inline = arg.partition("=")
value = inline if inline else (args[i + 1] if i + 1 < len(args) else "")
if value.strip():
raw = value.strip()
@@ -7082,12 +7543,32 @@ def _arm_load_probe_memo() -> None:
@contextlib.contextmanager
def _load_probe_memo_scope():
"""Guarantee the memo cannot outlive a load however it exits. Arming is the
- narrower `_arm_load_probe_memo`, taken around the placement decision."""
+ narrower `_arm_load_probe_memo`, taken around the placement decision. The whole
+ load also keeps a llama-fit-params failure that is not cached across loads (a
+ timeout, a crash) and the Metal fit lines already logged, so a second budget pass
+ neither waits on the probe again nor repeats its message."""
+ prev = (
+ getattr(_LOAD_PROBE_STATE, "metal_transient", None),
+ getattr(_LOAD_PROBE_STATE, "logged", None),
+ )
+ _LOAD_PROBE_STATE.metal_transient = {}
+ _LOAD_PROBE_STATE.logged = set()
try:
yield
finally:
_LOAD_PROBE_STATE.armed = False
_LOAD_PROBE_STATE.rows = None
+ _LOAD_PROBE_STATE.metal_transient, _LOAD_PROBE_STATE.logged = prev
+
+
+def _info_once_per_load(msg: str, *args) -> None:
+ """``logger.info`` that a load says once (_load_probe_memo_scope); always outside one."""
+ seen = getattr(_LOAD_PROBE_STATE, "logged", None)
+ if seen is not None:
+ if (msg, args) in seen:
+ return
+ seen.add((msg, args))
+ logger.info(msg, *args)
# "Probed, no usable answer", kept apart from None = "not probed yet".
@@ -7601,6 +8082,9 @@ class LlamaCppBackend:
# True when the launch actually runs MTP (Unsloth- or user/env-driven);
# gates the watchdog and recovery so pass-through MTP is covered.
self._mtp_runtime_fallback_active = False
+ # (load key, count) of Metal OOM retirements, and the key that stopped respawning.
+ self._metal_oom_strikes = None
+ self._metal_oom_terminal_key = None
self._stdout_lines: list[str] = []
self._stdout_thread: Optional[threading.Thread] = None
# Wakes the health probe on subprocess readiness/exit, not left to the 500 ms fallback poll.
@@ -7685,7 +8169,14 @@ class LlamaCppBackend:
@property
def is_active(self) -> bool:
"""True if a llama-server process exists (loading or loaded)."""
- return self._process is not None
+ proc = self._process
+ if proc is None:
+ return False
+ return not (
+ not self._healthy
+ and proc is getattr(self, "_retired_process", None)
+ and getattr(self, "_metal_oom_terminal_key", None) is not None
+ )
def resident_memory_mib(self) -> Optional[int]:
"""Best-effort MiB the loaded model holds: its GPU model buffers when known, else the
@@ -9364,6 +9855,7 @@ class LlamaCppBackend:
"flash_attn_takes_value": True,
"supports_fit_ctx": False,
"supports_fit_target": False,
+ "fit_measures_borrowed_drafts": False,
"supports_cache_ram": False,
"supports_ctx_checkpoints": False,
"ctx_checkpoints_flag": None,
@@ -9399,16 +9891,26 @@ class LlamaCppBackend:
mtime_ns = 0
size = 0
cache_key = (bin_path, mtime_ns, size)
+ hit = None
with cls._capability_cache_lock:
cached = cls._capability_cache.get(cache_key)
if cached is not None:
- if not cached.get("mtp_probe_inconclusive"):
- return cached
retry_after = cls._capability_retry_after.get(cache_key)
- if retry_after is not None and time.monotonic() < retry_after:
- return cached
- cls._capability_cache.pop(cache_key, None)
- cls._capability_retry_after.pop(cache_key, None)
+ if not cached.get("mtp_probe_inconclusive") or (
+ retry_after is not None and time.monotonic() < retry_after
+ ):
+ hit = cached
+ else:
+ cls._capability_cache.pop(cache_key, None)
+ cls._capability_retry_after.pop(cache_key, None)
+ if hit is not None:
+ # A copy: the cached dict is shared across threads.
+ if hit.get("supports_fit_target"):
+ hit = {
+ **hit,
+ "fit_measures_borrowed_drafts": _fit_measures_borrowed_drafts(bin_path),
+ }
+ return hit
mtp_token: Optional[str] = None
supports_dspark = False
@@ -9746,6 +10248,8 @@ class LlamaCppBackend:
"flash_attn_takes_value": flash_attn_takes_value,
"supports_fit_ctx": supports_fit_ctx,
"supports_fit_target": supports_fit_target,
+ "fit_measures_borrowed_drafts": supports_fit_target
+ and _fit_measures_borrowed_drafts(bin_path),
"supports_cache_ram": supports_cache_ram,
"supports_ctx_checkpoints": supports_ctx_checkpoints,
"ctx_checkpoints_flag": ctx_checkpoints_flag,
@@ -12306,27 +12810,49 @@ class LlamaCppBackend:
self, binary: Optional[str], model_path: Optional[str]
) -> Optional[tuple[int, int, int]]:
"""Measures full-offload model buffers with the sibling llama-fit-params; None if that probe
- fails."""
+ fails, with the reason in ``_metal_probe_failure``."""
+ self._metal_probe_failure = None
if not binary or not model_path:
+ self._metal_probe_failure = "no binary or model path"
return None
try:
probe = _llama_lib_dir(binary) / "llama-fit-params"
+ except OSError:
+ self._metal_probe_failure = "llama-fit-params not found"
+ return None
+ try:
probe_stat = probe.stat()
+ except OSError:
+ self._metal_probe_failure = f"llama-fit-params not found at {probe}"
+ return None
+ try:
model_stat = os.stat(model_path)
except OSError:
+ self._metal_probe_failure = "model file unreadable"
+ return None
+ if os.name != "nt" and not os.access(probe, os.X_OK):
+ self._metal_probe_failure = f"llama-fit-params at {probe} is not executable"
return None
key = (
str(probe),
probe_stat.st_size,
probe_stat.st_mtime_ns,
+ probe_stat.st_mode,
model_path,
model_stat.st_size,
model_stat.st_mtime_ns,
)
cached = getattr(self, "_metal_model_mib_cache", None)
if cached is not None and cached[0] == key:
+ self._metal_probe_failure = cached[2]
return cached[1]
+ transient = getattr(_LOAD_PROBE_STATE, "metal_transient", None)
+ if transient is not None and key in transient:
+ self._metal_probe_failure = transient[key]
+ return None
measured: Optional[tuple[int, int, int]] = None
+ failure: Optional[str] = None
+ cacheable = True
try:
env = self._llama_server_env_for_binary(str(binary))
for name in tuple(env):
@@ -12339,18 +12865,38 @@ class LlamaCppBackend:
encoding = "utf-8",
errors = "replace",
# A cold Metal shader compile lands here instead of at the launch.
- timeout = 120,
+ timeout = _METAL_FIT_PROBE_TIMEOUT_S,
check = False,
env = env,
**_windows_hidden_subprocess_kwargs(),
)
- if result.returncode == 0:
- measured = _parse_metal_memory_breakdown(
- (result.stdout or "") + "\n" + (result.stderr or "")
- )
+ output = (result.stdout or "") + "\n" + (result.stderr or "")
+ if result.returncode != 0:
+ failure = f"llama-fit-params exited with code {result.returncode}"
+ cacheable = False
+ else:
+ measured = _parse_metal_memory_breakdown(output)
+ if measured is None:
+ failure = "llama-fit-params printed no Metal memory breakdown"
+ if failure:
+ tail = " | ".join(
+ line.strip() for line in output.strip().splitlines()[-3:] if line.strip()
+ )
+ if tail:
+ failure = f"{failure}: {tail[-300:]}"
+ except subprocess.TimeoutExpired:
+ failure = f"llama-fit-params timed out after {_METAL_FIT_PROBE_TIMEOUT_S} s"
+ cacheable = False
except Exception as e:
- logger.debug(f"llama-fit-params memory probe failed: {e}")
- self._metal_model_mib_cache = (key, measured)
+ failure = f"llama-fit-params could not run: {e}"
+ cacheable = False
+ if failure:
+ logger.debug(f"llama-fit-params memory probe failed: {failure}")
+ self._metal_probe_failure = failure
+ if cacheable:
+ self._metal_model_mib_cache = (key, measured, failure)
+ elif transient is not None:
+ transient[key] = failure
return measured
def _metal_demand_paged_embedding_bytes(
@@ -12414,9 +12960,10 @@ class LlamaCppBackend:
return 0
measured = self._metal_measured_model_mib(binary, model_path)
if measured is None:
- logger.info(
- "Metal fit: could not measure how llama.cpp maps this model (llama-fit-params "
- "missing or unreadable), so its %.2f GB of input embeddings stay charged.",
+ _info_once_per_load(
+ "Metal fit: could not measure how llama.cpp maps this model (%s), so its "
+ "%.2f GB of input embeddings stay charged.",
+ getattr(self, "_metal_probe_failure", None) or "llama-fit-params unavailable",
embeddings / (1024**3),
)
return 0
@@ -12427,7 +12974,7 @@ class LlamaCppBackend:
mapped = int(layout.tensor_bytes) - int(layout.excluded_block_bytes)
unmapped = min(embeddings, max(0, mapped - charged))
if not unmapped:
- logger.info(
+ _info_once_per_load(
"Metal fit: llama.cpp maps this model's %.2f GB of input embeddings into Metal, "
"so they stay charged. A llama.cpp update may remove that.",
embeddings / (1024**3),
@@ -16109,6 +16656,18 @@ class LlamaCppBackend:
draft_width = getattr(db, "_compute_activation_width", lambda: 0)()
if draft_width > 0:
width = draft_width + 2 * (getattr(self, "_embedding_length", None) or 0)
+ if getattr(db, "_architecture", None) == "dflash" and draft_width > 0:
+ # The DSpark/DFlash prompt graph grows with -ub (DeepSeek-V4.1 DSpark: 725.5 MiB at
+ # 512, 2900.5 MiB at 2048).
+ vocab = (
+ getattr(db, "_vocab_size", None)
+ or getattr(self, "_vocab_size", None)
+ or _ASSUMED_MAX_VOCAB
+ )
+ heads = (getattr(db, "_n_heads", None) or 0) * (
+ getattr(db, "_kv_key_length", None) or 0
+ )
+ width = max(width, 2 * vocab + 2 * heads + draft_width)
activations = int(width * ub * 4 * self._COMPUTE_BUFFER_SAFETY)
spec = verify_rows + max(0, ub * 2 * n_ctx + activations - self._MTP_DRAFT_COMPUTE_BYTES)
if getattr(db, "_architecture", None) == "dflash":
@@ -16825,6 +17384,7 @@ class LlamaCppBackend:
levelled = to_info = False
proc = self._process
device_lost_seen = False
+ metal_poison = {"draft": False}
try:
for line in proc.stdout:
line = line.rstrip()
@@ -16851,13 +17411,27 @@ class LlamaCppBackend:
levelled = True
to_info = level.group(1) in "WE" or (ready and level.group(1) == "I")
# E lines only: unprefixed lines can be a request dump with user text.
- if (
- not device_lost_seen
- and level.group(1) == "E"
- and any(marker in line for marker in _GPU_DEVICE_LOST_MARKERS)
- ):
- device_lost_seen = True
- self._retire_device_lost_server_soon(proc)
+ if level.group(1) == "E":
+ if any(m in line for m in _DRAFT_CONTEXT_FAILURE_MARKERS):
+ metal_poison["draft"] = True
+ if not device_lost_seen and any(
+ marker in line for marker in _GPU_DEVICE_LOST_MARKERS
+ ):
+ device_lost_seen = True
+ self._retire_device_lost_server_soon(proc)
+ elif (
+ not device_lost_seen
+ and not metal_poison.get("pending")
+ and any(
+ marker in line for marker in _METAL_BACKEND_POISONED_MARKERS
+ )
+ ):
+ # Not latched: a request can echo this text, so retirement confirms
+ # a compute failure first.
+ metal_poison["pending"] = True
+ self._retire_device_lost_server_soon(
+ proc, metal_poison = metal_poison
+ )
elif not levelled:
to_info = ready or _llama_line_is_warning_or_error(line, line_lower)
try:
@@ -17970,6 +18544,11 @@ class LlamaCppBackend:
self._kv_cache_context_total = None
# False means confirmed to hold no VRAM and makes the coordinator skip the unload, so claim it only for a zero-layer split.
self._gpu_offload_active = not holds_no_gpu
+ # A runner crash must neither replay the replaced chat load nor meet its Metal OOM latch.
+ self._last_load_intent = None
+ self._mtp_runtime_fallback_active = False
+ self._metal_oom_strikes = None
+ self._metal_oom_terminal_key = None
# Record the applied split and reset the other knobs so status and reload dedup agree.
self._gpu_memory_mode = gpu_memory_mode if manual_ngl is not None else "auto"
self._gpu_layers = manual_ngl if manual_ngl is not None else -1
@@ -22761,6 +23340,19 @@ class LlamaCppBackend:
_metal_full_offload_pinned = False
# --fit-target floor from the Metal wired headroom; 0 off Apple Silicon.
_metal_fit_target_floor_mib = 0.0
+ # --fit-target headroom for a DSpark/DFlash sidecar the child's fitter cannot size.
+ _sidecar_fit_reserve_mib = 0
+ _sidecar_fit_reserve_eligible_mib = 0
+ _sidecar_compute_mib = 0
+ _fit_device_count = 1
+ _sidecar_reserve_graph_mib = [0]
+ _sidecar_draft_layers: Optional[int] = None
+ _sidecar_owns_output = False
+ _sidecar_override_flags: List[str] = []
+ _sidecar_override_pairs: List[List[str]] = []
+ _sidecar_reserve_logged: list = []
+ _sidecar_reserve_reprice: Optional[Callable[..., int]] = None
+ _sidecar_kind_for_fit = None
# The fit and launch must use the same Model Memory snapshot.
from utils.model_memory_settings import capture_model_memory_settings
@@ -23235,6 +23827,98 @@ class LlamaCppBackend:
_cpu_draft_path = _mtp_draft_for_budget if _draft_on_cpu else None
if _draft_on_cpu:
_mtp_draft_for_budget = None
+
+ def _launch_split_env() -> dict:
+ e = dict(os.environ)
+ if not tensor_parallel:
+ sm = (e.get("LLAMA_ARG_SPLIT_MODE") or "").strip().lower()
+ if sm and sm != "layer":
+ e.pop("LLAMA_ARG_SPLIT_MODE", None)
+ e.pop("LLAMA_ARG_TENSOR_SPLIT", None)
+ return e
+
+ def _sidecar_planned_device_count() -> int:
+ count = len(gpu_indices if gpu_indices is not None else _detected_gpus)
+ if not _gpu_ids_own_device_flags:
+ named = _extra_args_main_device(extra_args) or os.environ.get(
+ "LLAMA_ARG_DEVICE"
+ )
+ if named:
+ count = _distinct_device_count(named)
+ if count > 1 and _split_mode_confines_to_one_device(
+ extra_args, _launch_split_env()
+ ):
+ return 1
+ return count
+
+ def _draft_on_target_device_now(apple_budget_mib: Optional[int] = None) -> bool:
+ # A draft pinned to another device allocates there (on Apple Silicon only
+ # MTL0 / Vulkan0 is the target's).
+ pin = _extra_args_draft_device_pin(extra_args)
+ if pin is None:
+ return True
+ distinct = {d.strip() for d in pin.split(",") if d.strip()}
+ pin = distinct.pop() if len(distinct) == 1 else pin
+ apple = _apple_budget_mib if apple_budget_mib is None else apple_budget_mib
+ # The picker strips competing device extras and env twins at emission, so
+ # this is the launch's placement.
+ _pl_args = (
+ self._strip_device_extra_args(extra_args)
+ if _gpu_ids_own_device_flags
+ else extra_args
+ )
+ _pl_env = _launch_split_env()
+ if gpu_memory_mode == "manual":
+ self._clear_manual_placement_env(_pl_env)
+ if _gpu_ids_own_device_flags:
+ self._clear_device_placement_env(_pl_env)
+ main = _extra_args_main_device(_pl_args)
+ if main is None and not _gpu_ids_own_device_flags:
+ main = _pl_env.get("LLAMA_ARG_DEVICE") or None
+ if main is None and is_vulkan_backend:
+ ids = (
+ gpu_indices
+ if gpu_indices is not None
+ else [g[0] for g in _detected_gpus]
+ )
+ main = ",".join(f"Vulkan{i}" for i in ids) or (
+ "Vulkan0" if apple > 0 else None
+ )
+ elif main is None and apple > 0:
+ main = "MTL0"
+ elif (main is None and pin.strip()) and all(
+ _CUDA_LIKE_DEVICE_RE.match(d.strip())
+ for d in pin.split(",")
+ if d.strip()
+ ):
+ # CUDA / ROCm pins are remapped into the child's mask: only the spawned
+ # argv can tell (_spawn_draft_on_target).
+ return True
+
+ if main and _split_mode_confines_to_one_device(_pl_args, _pl_env):
+ listed = [d.strip() for d in str(main).split(",") if d.strip()]
+ if len(listed) > 1:
+ at = self._effective_main_gpu(_pl_args, _pl_env)
+ at = 0 if at is None else at
+ main = listed[at] if 0 <= at < len(listed) else None
+
+ def _names(value: str) -> set:
+ return {d.strip().lower() for d in str(value).split(",") if d.strip()}
+
+ return bool(main) and bool(_names(pin)) and _names(pin) <= _names(main)
+
+ _draft_share_unknown = bool(
+ _mtp_draft_for_budget
+ and _draft_card_share_unknown(
+ extra_args,
+ os.environ,
+ n_draft_layers = getattr(
+ self._draft_backend_for(_mtp_draft_for_budget),
+ "_n_layers",
+ None,
+ ),
+ )
+ )
_mtp_draft_weights = 0
if _mtp_draft_for_budget:
try:
@@ -23306,6 +23990,18 @@ class LlamaCppBackend:
_kv_unified: bool = planned_kv_unified,
_n_ubatch: Optional[int] = _effective_ubatch,
) -> int:
+ # Applied here, not in a wrapper, so every gate taking this callback
+ # prices the draft micro-batch.
+ _n_ubatch = (
+ _capped_draft_n_ubatch(
+ extra_args,
+ os.environ,
+ ctx,
+ _emitted_n_batch(n_batch, _np),
+ n_ubatch,
+ )
+ or _n_ubatch
+ )
v = self._estimate_mtp_overhead_bytes(
ctx,
spec_draft_n_max = _n,
@@ -23688,6 +24384,8 @@ class LlamaCppBackend:
# llama.cpp loads a bare --model-draft anyway (has_dft()), so keep the reserve.
and not _extra_args_mtp_draft_path(extra_args, env = _spec_env)
and not _draft_cpu_no_embedded
+ and not _draft_share_unknown
+ and _draft_on_target_device_now(apple_budget_mib = 0)
and not tensor_parallel
and gpus
and effective_ctx > 0
@@ -24724,6 +25422,121 @@ class LlamaCppBackend:
+ _cc_bytes(ctx)
) / (1024 * 1024)
+ # Auto only: drop the drafter when the target fits Metal alone but not with
+ # the sidecar reserve.
+ if (
+ use_fit
+ and _mtp_will_engage
+ and _mtp_reserves_gpu
+ and _mtp_effective in ("dspark", "dflash")
+ and (_canonicalize_spec_mode(speculative_type) or "auto") == "auto"
+ and not _extra_args_set_spec_type(extra_args)
+ # A bare --model-draft in extras still loads (has_dft()).
+ and not _extra_args_mtp_draft_path(extra_args, env = _spec_env)
+ and not _draft_share_unknown
+ and _draft_on_target_device_now()
+ # Zero KV would read as a target that fits alone.
+ and self._can_estimate_kv()
+ ):
+ # _apple_fit_budget_mib already holds back a flat fraction for this
+ # drafter.
+ _sc_ctx = (
+ effective_ctx
+ if explicit_ctx
+ else min(_metal_floor_ctx, native_ctx_for_cap or _metal_floor_ctx)
+ )
+ _sc_target_mib = (
+ _apple_model_size_fit
+ + _kv_bytes(_sc_ctx, _fit_ctx_checkpoints)
+ + _cc_bytes(_sc_ctx)
+ ) / (1024 * 1024)
+ _sc_draft_ub = _capped_draft_n_ubatch(
+ extra_args,
+ os.environ,
+ _sc_ctx,
+ _emitted_n_batch(n_batch, n_parallel),
+ n_ubatch,
+ )
+ _sc_with_draft_mib = _sc_target_mib + self._sidecar_fit_reserve_mib(
+ _mtp_bytes(
+ _sc_ctx,
+ slots = _extra_args_n_parallel(extra_args, os.environ),
+ ubatch = _sc_draft_ub,
+ ),
+ mtp_draft_path,
+ flat_mib = math.ceil(_flat_mtp_reserve * max(0, _apple_budget_mib)),
+ )
+ # An explicit context is refused past the wired limit when that is
+ # readable.
+ _sc_wired_mib = (
+ self._apple_metal_wired_ceiling_bytes() // (1024 * 1024)
+ if explicit_ctx
+ and not _caller_owns_budget
+ and not _paravirtual_cpu_forced
+ else 0
+ )
+ _sc_over_limit = _apple_budget_mib < _sc_with_draft_mib or (
+ 0 < _sc_wired_mib < _sc_with_draft_mib
+ )
+ # Only when both do not fit: the draft-free embedding discount can cost
+ # a cold llama-fit-params run.
+ _sc_host_embd_without_draft = _apple_host_embd
+ _sc_target_only_mib = _sc_target_mib
+ if _sc_over_limit:
+ _sc_host_embd_without_draft = (
+ self._metal_demand_paged_embedding_bytes(
+ model_path,
+ extra_args,
+ binary = binary,
+ requested_load_mode = load_mode,
+ supports_load_mode = bool(
+ server_caps.get("supports_load_mode")
+ ),
+ settings = _mem_settings,
+ layers_fixed = _caller_owns_budget or _paravirtual_cpu_forced,
+ mtp_may_engage = False,
+ )
+ )
+ _sc_target_only_mib = _sc_target_mib - (
+ self._MTP_DRAFT_COMPUTE_BYTES
+ + _sc_host_embd_without_draft
+ - _apple_host_embd
+ ) / (1024 * 1024)
+ if _sc_target_only_mib <= _apple_budget_mib < _sc_with_draft_mib or (
+ _sc_wired_mib > 0
+ and _sc_target_only_mib <= _sc_wired_mib < _sc_with_draft_mib
+ ):
+ logger.info(
+ "Auto: the %s sidecar does not fit in unified memory "
+ "alongside the model, so it is dropped.",
+ "DSpark" if _mtp_effective == "dspark" else "DFlash",
+ )
+ _spec_dropped_no_vram = True
+ _mtp_will_engage = False
+ _mtp_reserves_gpu = False
+ mtp_overhead_fn = None
+ _mtp_kv_unsized = False
+ # Derived after the discrete drop: undo them, or Auto keeps a
+ # smaller context than Off.
+ _mtp_gpu_bytes_remain = False
+ _flat_mtp_engages = False
+ _flat_mtp_reserve = 0.0
+ _pin_fraction = _vram_frac
+ _apple_fit_budget_mib = _apple_budget_mib
+ _soft_overhead -= self._MTP_DRAFT_COMPUTE_BYTES
+ model_size_fit -= self._MTP_DRAFT_COMPUTE_BYTES
+ _apple_embd_logged = bool(_apple_host_embd)
+ _apple_host_embd = _sc_host_embd_without_draft
+ _apple_model_size_fit = model_size_fit - _apple_host_embd
+ if _apple_host_embd and not _apple_embd_logged:
+ logger.info(
+ "Metal fit: %.2f GB of input embeddings stay in the CPU "
+ "file mapping; charging %.2f GB of weights to unified "
+ "memory.",
+ _apple_host_embd / (1024**3),
+ _apple_model_size_fit / (1024**3),
+ )
+
if self._can_estimate_kv():
cap = _apple_ctx_fit(native_ctx_for_cap, _FIT_MIN_CTX)
_cap_footprint_mib = _apple_footprint_mib(cap)
@@ -25336,6 +26149,297 @@ class LlamaCppBackend:
if _mtp_will_engage
else 0
)
+ # DSpark/DFlash borrow the target's token_embd/output, so llama.cpp's fitter
+ # cannot measure the draft
+ # (common/fit.cpp "fitting without it"); carry the planner's reserve in the
+ # child's --fit-target.
+ _sidecar_kind_for_fit = (
+ _mtp_effective if _mtp_effective in ("dspark", "dflash") else None
+ )
+ if _extra_args_set_spec_type(extra_args):
+ _child_spec_types = _accumulated_spec_types(extra_args, env = _spec_env)
+ _sidecar_kind_for_fit = (
+ "dspark"
+ if "draft-dspark" in _child_spec_types
+ else ("dflash" if "draft-dflash" in _child_spec_types else None)
+ )
+ _fit_device_count = _sidecar_planned_device_count()
+ _draft_on_target_device = _draft_on_target_device_now()
+ if (
+ _mtp_will_engage
+ and _mtp_reserves_gpu
+ and _sidecar_kind_for_fit is not None
+ and _draft_on_target_device
+ # Builds whose fitter measures borrowed-tensor drafts need no reserve.
+ and not server_caps.get("fit_measures_borrowed_drafts")
+ and _sidecar_borrows_target_tensors(_mtp_draft_for_budget or mtp_draft_path)
+ ):
+ # Priced even under --fit off: a startup-OOM retry turns the fitter on with
+ # the drafter loaded.
+ _sidecar_auto_ctx = effective_ctx <= 0
+ _sidecar_reserve_ctx = (
+ _sidecar_child_ctx(
+ extra_args,
+ os.environ,
+ self._context_length,
+ n_parallel,
+ planned_kv_unified,
+ )
+ if _sidecar_auto_ctx
+ else effective_ctx
+ )
+ _sidecar_compute_mib = math.ceil(
+ self._MTP_DRAFT_COMPUTE_BYTES / (1024 * 1024)
+ )
+ _sidecar_reserve_path = _mtp_draft_for_budget or mtp_draft_path
+ try:
+ _sidecar_draft_layers = getattr(
+ self._draft_backend_for(_sidecar_reserve_path), "_n_layers", None
+ )
+ except Exception: # noqa: BLE001 - an unread drafter shares by memory alone
+ _sidecar_draft_layers = None
+ _sidecar_owns_output = not _sidecar_borrows_output(_sidecar_reserve_path)
+ # The fitter measures the target's rollback copies itself.
+ _sidecar_only_bytes = _mtp_bytes(
+ _sidecar_reserve_ctx, n_parallel, _effective_ubatch
+ )
+ if _target_rollback and _mtp_eff_n_max:
+ _sidecar_only_bytes -= (
+ self._rollback_state_bytes(n_parallel) * _mtp_eff_n_max
+ )
+ # An unsized sidecar carries the planner's flat fraction, or the fitter
+ # spends it on target layers.
+ _sidecar_flat_mib = 0
+ if _flat_mtp_reserve > 0:
+ _flat_pool_mib = (
+ _apple_budget_mib
+ if _apple_budget_mib > 0
+ else max(
+ (
+ free
+ for idx, free in (gpus or _detected_gpus)
+ if gpu_indices is None or idx in gpu_indices
+ ),
+ default = 0,
+ )
+ )
+ _sidecar_flat_mib = math.ceil(
+ _flat_mtp_reserve * max(0, _flat_pool_mib)
+ )
+ _sidecar_fit_reserve_eligible_mib = (
+ self._sidecar_fit_reserve_mib(
+ max(0, _sidecar_only_bytes),
+ _sidecar_reserve_path,
+ flat_mib = _sidecar_flat_mib,
+ )
+ + _sidecar_compute_mib
+ )
+
+ def _sidecar_child_on_discrete_gpus(
+ run_args: List[str], child_env: Mapping[str, str]
+ ) -> bool:
+ """Whether every card the drafter runs on is discrete; positive evidence
+ only."""
+ if _apple_budget_mib:
+ return False
+ active = gpu_indices
+ if (
+ not is_vulkan_backend
+ and _child_gpu_physical_ids is not None
+ and _child_gpu_physical_ids is not _parent_view_gpu_ids
+ ):
+ active = list(_child_gpu_physical_ids)
+ if not active:
+ return False
+ override = self._effective_device_ids(run_args, child_env)
+ if override is not None:
+ if not override or any(
+ d.lower() in _CPU_DEVICE_VALUES for d in override
+ ):
+ return False
+ # This backend's own names only: Vulkan0 is not CUDA0.
+ if not all(
+ (
+ re.fullmatch(r"(?i)vulkan\d+", d)
+ if is_vulkan_backend
+ else _CUDA_LIKE_DEVICE_RE.match(d)
+ )
+ for d in override
+ ):
+ return False
+ studio_pin = (
+ [f"vulkan{i}" for i in _vulkan_pin_ids]
+ if is_vulkan_backend and _vulkan_pin_ids
+ else None
+ )
+ if [d.lower() for d in override] != studio_pin:
+ listed = {
+ d.lower()
+ for d in (
+ self._enumerated_gpu_devices(binary, child_env) or []
+ )
+ }
+ if not {d.lower() for d in override} <= listed:
+ return False
+ seen = {idx for idx, _free in (_detected_gpus or ())}
+ shared = set(_shared_gpu_ids or ())
+ if (
+ is_vulkan_backend
+ and override
+ and not {int(d[6:]) for d in override} <= seen
+ ):
+ # Vulkan Auto drops its iGPUs from the probe.
+ return False
+ if (
+ seen
+ and not (seen & shared)
+ and self._discrete_gpu_for_expert_spill(
+ None, _detected_gpus, _shared_gpu_ids
+ )
+ ):
+ return True
+ # A mixed host: narrow only through an order the child provably shares.
+ if not is_vulkan_backend and (
+ self._positional_child_order_unproven(child_env)
+ or any(
+ not all(
+ t.strip().isdigit()
+ for t in str(child_env.get(name)).split(",")
+ if t.strip()
+ )
+ for name in (
+ "CUDA_VISIBLE_DEVICES",
+ "HIP_VISIBLE_DEVICES",
+ "ROCR_VISIBLE_DEVICES",
+ )
+ if child_env.get(name) is not None
+ )
+ ):
+ return False
+ active, compact = self._effective_gpu_selection(
+ binary, child_env, active, run_args, is_vulkan = is_vulkan_backend
+ )
+ pos = _draft_pin_device_index(
+ _extra_args_draft_device_pin(run_args),
+ ",".join(override) if override else None,
+ len(compact or active or _detected_gpus or ()),
+ )
+ if (
+ pos is None
+ and _extra_args_draft_device_pin(run_args) is None
+ and _split_mode_confines_to_one_device(run_args, child_env)
+ ):
+ main = self._effective_main_gpu(run_args, child_env)
+ pos = 0 if main is None else (main if override else None)
+ if pos is not None:
+ if override:
+ if not (compact and 0 <= pos < len(compact)):
+ return False
+ active = self._physical_ordinals(
+ [compact[pos]], child_env, is_vulkan_backend
+ )
+ elif is_vulkan_backend or not self._positional_child_order_unproven(
+ child_env
+ ):
+ active = self._physical_ordinals(
+ [pos], child_env, is_vulkan_backend
+ )
+ on = set(active) if active else seen
+ if not on or not on <= seen - shared:
+ return False
+ return self._discrete_gpu_for_expert_spill(
+ sorted(on), _detected_gpus, _shared_gpu_ids
+ )
+
+ def _sidecar_reserve_reprice(
+ run_args: List[str],
+ child_env: Mapping[str, str],
+ _ctx: int = _sidecar_reserve_ctx,
+ _n: int = _mtp_eff_n_max,
+ _dp: Optional[str] = _sidecar_reserve_path,
+ _w: int = _mtp_draft_weights,
+ _np: int = n_parallel,
+ _mtp: bool = _engaged_is_mtp,
+ _swa_full: bool = swa_full,
+ _kv_unified: bool = planned_kv_unified,
+ _ub: Optional[int] = _effective_ubatch,
+ _floor: int = _sidecar_compute_mib,
+ _flat: int = _sidecar_flat_mib,
+ _native: Optional[int] = self._context_length,
+ ) -> int:
+ # Read off the argv: a --flash-attn off respawn pads the draft cache to
+ # f16, extras can change slots.
+ _ck, _cv = _extra_args_draft_cache_types(run_args, child_env)
+ _np = _extra_args_n_parallel(run_args, child_env) or _np
+ _kv_unified = _kv_unified_from_args(
+ run_args, default = _kv_unified, env = child_env
+ )
+ try:
+ _ctx = _sidecar_child_ctx(
+ run_args, child_env, _native, _np, _kv_unified
+ )
+ except ValueError:
+ pass # a malformed -c is the launch's error to report, not this
+ # llama_context caps the micro-batch at the batch, and a causal batch at
+ # the context.
+ _ub = _capped_draft_n_ubatch(run_args, child_env, _ctx) or _ub
+ if _ub and _ctx and _ctx > 0:
+ _ub = min(_ub, int(_ctx))
+ _kv_kw = dict(
+ drafter_path = _dp,
+ draft_cache_type_k = _ck,
+ draft_cache_type_v = _cv,
+ n_parallel = _np,
+ swa_full = _swa_full,
+ kv_unified = _kv_unified,
+ n_ubatch = _ub,
+ flash_attn = _flash_attn_enabled_from_args(
+ run_args, default = planned_flash_attn, env = child_env
+ ),
+ )
+ _v = self._estimate_mtp_overhead_bytes(
+ _ctx,
+ spec_draft_n_max = _n,
+ draft_weights_bytes = _w,
+ mtp_keeps_target_ctx = _mtp,
+ target_rollback = False,
+ **_kv_kw,
+ )
+ # The draft context copies -nkvo, so its KV stays in host buffers.
+ if (
+ _v
+ and not _kv_offload_from_args(run_args, child_env)
+ and _sidecar_child_on_discrete_gpus(run_args, child_env)
+ ):
+ _host_kv = self._mtp_draft_kv_bytes(_ctx, **_kv_kw)
+ if _host_kv:
+ _v = max(0, _v - _host_kv)
+ try:
+ _sidecar_reserve_graph_mib[0] = math.ceil(
+ self._mtp_draft_compute_bytes(
+ _ctx,
+ drafter_path = _dp,
+ n_ubatch = _ub,
+ n_parallel = _np,
+ spec_draft_n_max = _n,
+ )
+ / (1024 * 1024)
+ )
+ except Exception: # noqa: BLE001 - unsized: the shares carry it
+ _sidecar_reserve_graph_mib[0] = 0
+ return (
+ self._sidecar_fit_reserve_mib(_v or 0, _dp, flat_mib = _flat) + _floor
+ )
+
+ if _draft_share_unknown:
+ # Drafter partly off the card: its weight share is unpriced, keep only
+ # the compute floor.
+ _sidecar_fit_reserve_eligible_mib = _sidecar_compute_mib
+ _sidecar_reserve_reprice = None
+ # A scalar --fit-target suits only a single-device launch; priced anyway,
+ # since a retry can narrow to one card.
+ if use_fit and (_apple_budget_mib > 0 or _fit_device_count == 1):
+ _sidecar_fit_reserve_mib = _sidecar_fit_reserve_eligible_mib
if _mtp_will_engage:
_mtp_note = self._mtp_reserve_note(
_mtp_reserve_bytes,
@@ -26250,14 +27354,20 @@ class LlamaCppBackend:
if _fit_target_priced and use_fit and not _metal_fit_target_floor_mib
else None
)
- if _metal_fit_target_floor_mib:
+ if _metal_fit_target_floor_mib or _sidecar_fit_reserve_mib:
# Rebuilt after the price is read: the floor is the machine's
- # margin, not the budget slider's.
+ # margin and the sidecar reserve the drafter's, not the budget
+ # slider's.
_integrity_flags = self._ctx_integrity_flags(
*_integrity_args,
fit_target_delta_mib = _fit_target_delta_mib,
fit_target_floor_mib = _metal_fit_target_floor_mib,
+ # The reserve rides on the spawn's trailing pair, which retry cleanup must
+ # tell apart from a user's value.
+ fit_target_reserve_mib = 0,
)
+ if _sidecar_fit_reserve_mib and not server_caps.get("supports_fit_target"):
+ _sidecar_fit_reserve_mib = 0
cmd.extend(_integrity_flags)
offload_overridden = _extra_args_set_any_flag(
extra_args, _GPU_OFFLOAD_OVERRIDE_FLAGS
@@ -26448,14 +27558,25 @@ class LlamaCppBackend:
if _gpu_ids_own_device_flags:
_spec_placement_extras = self._strip_device_extra_args(extra_args)
self._clear_device_placement_env(_spec_placement_env)
+ _spec_placement_argv = [*cmd, *(_spec_placement_extras or [])]
# A device the launch really carries, and not the CPU spelling of it.
# Only consulted where the probe found nothing: it is the one remaining
# way the child can still be pointed at a GPU.
_spec_device_pick = _extra_args_main_device(
- [*cmd, *(_spec_placement_extras or [])]
+ _spec_placement_argv
) or _spec_placement_env.get("LLAMA_ARG_DEVICE")
_spec_extras_pick_a_gpu = bool(_spec_device_pick) and not _device_selection_is_cpu(
- [*cmd, *(_spec_placement_extras or [])], _spec_placement_env
+ _spec_placement_argv, _spec_placement_env
+ )
+ # Log-only; a priced reserve is reported by the spawn (_log_sidecar_fit_reserve).
+ _sidecar_fit_sized = (
+ not fit_is_effectively_on(_spec_placement_argv, _spec_placement_env)
+ or bool(_sidecar_fit_reserve_eligible_mib)
+ or bool(server_caps.get("fit_measures_borrowed_drafts"))
+ or (
+ bool(launch_mtp_draft_path)
+ and not _sidecar_borrows_target_tensors(launch_mtp_draft_path)
+ )
)
spec_flags = self._build_speculative_flags(
speculative_type = speculative_type,
@@ -26469,9 +27590,9 @@ class LlamaCppBackend:
None if _spec_canon in ("dspark", "dflash") else launch_mtp_draft_path
),
dspark_draft_path = (launch_mtp_draft_path if _spec_canon == "dspark" else None),
- dspark_fit_sized = not use_fit,
+ dspark_fit_sized = _sidecar_fit_sized,
dflash_draft_path = (launch_mtp_draft_path if _spec_canon == "dflash" else None),
- dflash_fit_sized = not use_fit,
+ dflash_fit_sized = _sidecar_fit_sized,
# So the fallback says unopenable, not missing. DSpark/DFlash never
# reach the drop.
mtp_drafter_unloadable = bool(_unloadable_mtp_draft_path),
@@ -27123,6 +28244,36 @@ class LlamaCppBackend:
logger.info(
f"Appending user extra args to llama-server: {list(_emit_extra_args)}"
)
+ # Reconcile the reserve with the child as launched: extras can turn the fitter on or
+ # add a margin.
+ _sidecar_user_tokens = list(_emit_extra_args) if _mem_extras else []
+ if (
+ _sidecar_fit_reserve_eligible_mib
+ and server_caps.get("supports_fit_target")
+ # Re-read: a failed GPU selection falls back to every card.
+ and (_apple_budget_mib > 0 or _sidecar_planned_device_count() == 1)
+ ):
+ _sidecar_child_env = dict(os.environ)
+ if gpu_memory_mode == "manual":
+ self._clear_manual_placement_env(_sidecar_child_env)
+ _sidecar_override_flags = _sidecar_reserve_margin_flags(
+ cmd,
+ _emit_extra_args if _mem_extras else (),
+ _sidecar_child_env,
+ max(
+ _VRAM_FLOOR_RESERVE_MIB,
+ self._fit_target_margin_mib(
+ auto_fit = auto_fit,
+ fit_target_delta_mib = _fit_target_delta_mib,
+ floor_mib = _metal_fit_target_floor_mib,
+ ),
+ ),
+ _sidecar_fit_reserve_eligible_mib,
+ )
+ if _sidecar_override_flags:
+ _sidecar_fit_reserve_mib = _sidecar_fit_reserve_eligible_mib
+ _sidecar_override_pairs.append(list(_sidecar_override_flags))
+ cmd.extend(_sidecar_override_flags)
# Last so it wins: the Metal drafter CPU pin is a correctness fix.
if _pv_draft_cpu_pin:
cmd.extend(_pv_draft_cpu_pin)
@@ -27163,8 +28314,6 @@ class LlamaCppBackend:
kv_cache_unified = _kv_unified_from_args(cmd)
- logger.info(f"Starting llama-server: {' '.join(self._redacted_cmd_for_log(cmd))}")
-
# Library paths so llama-server finds its shared libs and CUDA DLLs.
env = self._llama_server_env_for_binary(binary)
# The GGUF header controls pooling; ignore inherited defaults.
@@ -27520,6 +28669,7 @@ class LlamaCppBackend:
_companion_added_device: Optional[list[str]] = None
if not is_vulkan_backend and _gpu_mem:
_child_gpu_physical_ids = self._unmasked_child_gpu_physical_ids()
+ _parent_view_gpu_ids = _child_gpu_physical_ids
# The CUDA SM gate goes here, where the child's GPU visibility is
# finally known: both arms below write CUDA_VISIBLE_DEVICES=-1, so no
@@ -27930,7 +29080,7 @@ class LlamaCppBackend:
# page-lock and writes it back, which without this makes the
# read below an UnboundLocalError instead.
nonlocal _last_spawn_cmd, _mem_host_resident, _did_rocm_retry
- nonlocal _sched_abort_seen
+ nonlocal _sched_abort_seen, _sidecar_fit_reserve_mib
# Single revocation point for the tensor-spill plan; label is empty only on the first spawn.
if label:
run_cmd = self._drop_tensor_spill(run_cmd, label.lstrip("-") or "retry")
@@ -27980,9 +29130,278 @@ class LlamaCppBackend:
# Best-effort; never block the load on logging.
logger.warning(f"Could not open llama-server log file: {e}")
self._llama_log_path = None
+ # Every launch: a retry can turn the fitter on or re-mask onto several
+ # cards.
+ _spawn_devices = _extra_args_main_device(run_cmd) or env.get(
+ "LLAMA_ARG_DEVICE"
+ )
+ _spawn_device_count = (
+ _distinct_device_count(_spawn_devices)
+ if _spawn_devices
+ else max(len(_child_gpu_physical_ids or ()), len(_detected_gpus))
+ if _child_gpu_physical_ids is None
+ or _child_gpu_physical_ids is _parent_view_gpu_ids
+ else len(_child_gpu_physical_ids)
+ )
+ if _sidecar_fit_reserve_eligible_mib:
+ # Re-derived each spawn: drop Studio's earlier reserve-bearing margins,
+ # keep the user's.
+ run_cmd = _drop_sidecar_margins(
+ list(run_cmd),
+ _integrity_flags,
+ self._ctx_integrity_flags(
+ *_integrity_args,
+ fit_target_delta_mib = _fit_target_delta_mib,
+ fit_target_floor_mib = _metal_fit_target_floor_mib,
+ ),
+ _sidecar_override_pairs,
+ user_args = _sidecar_user_tokens,
+ )
+ _sidecar_fit_reserve_mib = 0
run_cmd = self._per_device_fit_target_argv(
run_cmd, env, _fit_target_margins_by_idx
)
+ _spawn_base_by_dev: Optional[list[float]] = None
+ if _fit_target_margins_by_idx:
+ try:
+ _spelled = [
+ float(v)
+ for v in str(
+ _last_flag_value(run_cmd, {"-fitt", "--fit-target"})
+ ).split(",")
+ ]
+ except ValueError:
+ _spelled = []
+ if len(_spelled) == 1:
+ _spelled = _spelled * _spawn_device_count
+ if len(_spelled) == _spawn_device_count:
+ _spawn_base_by_dev = _spelled
+ # --rpc devices come first in the child's list, so local positional shares
+ # would land on the wrong cards.
+ _spawn_rpc = not _spawn_devices and bool(
+ (_extra_args_device(run_cmd, {"--rpc"}) or "").strip()
+ or str(env.get("LLAMA_ARG_RPC", "")).strip()
+ )
+ # Several cards: each keeps its share of the drafter (whole draft layers) in
+ # a per-device list.
+ _spawn_shares: Optional[list[float]] = [1.0]
+ _spawn_output_dev: Optional[int] = None
+ _spawn_pinned_at: Optional[int] = None
+ _spawn_cpu_only = _device_selection_is_cpu(run_cmd, env) or (
+ _child_gpu_physical_ids is not None
+ and _child_gpu_physical_ids is not _parent_view_gpu_ids
+ and not _child_gpu_physical_ids
+ )
+ if _spawn_cpu_only:
+ _spawn_shares = None
+ elif (
+ _spawn_device_count > 1
+ and _extra_args_split_mode(run_cmd, env) == "tensor"
+ ):
+ # --split-mode tensor folds the cards into one device: no per-device
+ # reserve.
+ _spawn_shares = None
+ elif _spawn_rpc:
+ _spawn_shares = None
+ elif _spawn_device_count > 1 and _split_mode_confines_to_one_device(
+ run_cmd, env
+ ):
+ if _spawn_base_by_dev and len(set(_spawn_base_by_dev)) > 1:
+ # llama.cpp keeps only devices[main_gpu] (src/llama.cpp): the
+ # reserve goes in entry 0 of a list that keeps the other margins.
+ _main = self._effective_main_gpu(run_cmd, env) or 0
+ if 0 <= _main < _spawn_device_count:
+ _spawn_shares = [
+ 1.0 if i == 0 else 0.0 for i in range(_spawn_device_count)
+ ]
+ _spawn_output_dev = 0
+ else:
+ _spawn_shares = None
+ elif _spawn_device_count > 1:
+ _spawn_shares = None
+ _spawn_pinned_at = _draft_pin_device_index(
+ _extra_args_draft_device_pin(run_cmd),
+ _spawn_devices,
+ _spawn_device_count,
+ )
+ if _spawn_pinned_at is not None:
+ _spawn_shares = [
+ 1.0 if i == _spawn_pinned_at else 0.0
+ for i in range(_spawn_device_count)
+ ]
+ _spawn_output_dev = _spawn_pinned_at
+ # Positional shares need the child to enumerate in the ids' order.
+ _spawn_order_unproven = self._positional_child_order_unproven(env)
+ if (
+ not is_vulkan_backend
+ and not _spawn_devices
+ and _extra_args_draft_device_pin(run_cmd) is None
+ and not _spawn_order_unproven
+ ):
+ _free_by_idx = dict(gpus or _detected_gpus or ())
+ if (
+ _child_gpu_physical_ids is None
+ or _child_gpu_physical_ids is _parent_view_gpu_ids
+ ):
+ _det_ids = [idx for idx, _free in _detected_gpus]
+ _spawn_ids = (
+ self._inherited_child_gpu_order(_det_ids) or _det_ids
+ )
+ else:
+ _spawn_ids = list(_child_gpu_physical_ids)
+ _spawn_free = [_free_by_idx.get(i) for i in _spawn_ids]
+ # The drafter inherits the target's --tensor-split
+ # (common_base_params_to_speculative).
+ _spawn_base = self._fit_target_margin_mib(
+ auto_fit = auto_fit,
+ fit_target_delta_mib = _fit_target_delta_mib,
+ floor_mib = _metal_fit_target_floor_mib,
+ )
+ _weights = None
+ if _spawn_free and all(f and f > 0 for f in _spawn_free):
+ _spawn_margins = _fit_margins_by_device(
+ extra_args, env, 0.0, len(_spawn_free)
+ )
+ _bases = (
+ _spawn_base_by_dev
+ if _spawn_base_by_dev
+ and len(_spawn_base_by_dev) == len(_spawn_free)
+ else [_spawn_base] * len(_spawn_free)
+ )
+ _weights = [
+ max(1.0, float(f) - max(b, m))
+ for f, m, b in zip(_spawn_free, _spawn_margins, _bases)
+ ]
+ _ts = _extra_args_tensor_split(run_cmd, env)
+ if _ts and _spawn_ids:
+ _ts_w = [
+ float(_ts[i]) if i < len(_ts) else 0.0
+ for i in range(len(_spawn_ids))
+ ]
+ if sum(_ts_w) > 0 and all(w >= 0 for w in _ts_w):
+ _weights = _ts_w
+ if _weights:
+ _layers = _sidecar_draft_layers or 0
+ _cum = list(
+ itertools.accumulate(w / sum(_weights) for w in _weights)
+ )
+ # "row" assigns blocks like "layer" (CUDA / ROCm register no
+ # split buffer type).
+ if _layers > 0:
+ # llama.cpp's layer split: block il goes to the first device
+ # whose cumulative share exceeds il / (n_layer + 1).
+ _counts = [0] * len(_weights)
+ for _il in range(_layers):
+ _dev = bisect.bisect_right(_cum, _il / (_layers + 1))
+ _counts[min(_dev, len(_weights) - 1)] += 1
+ _spawn_shares = [c / _layers for c in _counts]
+ else:
+ _spawn_shares = [w / sum(_weights) for w in _weights]
+ # Logits over a borrowed output run on the card holding the
+ # target's layer n_layer.
+ _t_layers = (
+ _layers
+ if _sidecar_owns_output
+ else int(getattr(self, "_n_layers", 0) or 0) or _layers
+ )
+ if _t_layers > 0:
+ _spawn_output_dev = min(
+ bisect.bisect_right(_cum, _t_layers / (_t_layers + 1)),
+ len(_weights) - 1,
+ )
+ if (
+ _sidecar_fit_reserve_eligible_mib
+ and _spawn_shares is not None
+ and (
+ _spawn_pinned_at is not None
+ or _spawn_draft_on_target(run_cmd, _spawn_devices, env)
+ )
+ and server_caps.get("supports_fit_target")
+ ):
+ _spawn_total = (
+ _sidecar_reserve_reprice(list(run_cmd), env)
+ if _sidecar_reserve_reprice is not None
+ else _sidecar_fit_reserve_eligible_mib
+ )
+ # The vocabulary-wide graph stays whole on the output card.
+ _spawn_graph = min(
+ _sidecar_reserve_graph_mib[0] if len(_spawn_shares) > 1 else 0,
+ max(0, _spawn_total - _sidecar_compute_mib),
+ )
+ _spawn_body = max(0, _spawn_total - _sidecar_compute_mib - _spawn_graph)
+ _last = (
+ _spawn_output_dev
+ if _spawn_output_dev is not None
+ else max(i for i, f in enumerate(_spawn_shares) if f > 0)
+ if any(f > 0 for f in _spawn_shares)
+ else len(_spawn_shares) - 1
+ )
+ _spawn_head = 0
+ if (
+ _sidecar_owns_output
+ and len(_spawn_shares) > 1
+ and not _draft_share_unknown
+ ):
+ _head_layout = self._tensor_spill_layout(
+ _sidecar_reserve_path, all_shards = True
+ )
+ if _head_layout is not None and _head_layout.complete:
+ _spawn_head = min(
+ _spawn_body,
+ math.ceil(_head_layout.lm_head_bytes / (1024 * 1024)),
+ )
+ _spawn_body -= _spawn_head
+ else:
+ _spawn_head = _spawn_body
+ _spawn_per_device = [
+ math.ceil(_spawn_body * f)
+ + (_sidecar_compute_mib if f > 0 or i == _last else 0)
+ + (_spawn_graph + _spawn_head if i == _last else 0)
+ for i, f in enumerate(_spawn_shares)
+ ]
+ _spawn_reserve = max(_spawn_per_device)
+ _spawn_flags = _sidecar_reserve_margin_flags(
+ run_cmd,
+ extra_args,
+ env,
+ self._fit_target_margin_mib(
+ auto_fit = auto_fit,
+ fit_target_delta_mib = _fit_target_delta_mib,
+ floor_mib = _metal_fit_target_floor_mib,
+ ),
+ _spawn_reserve,
+ per_device = _spawn_per_device
+ if len(_spawn_per_device) > 1
+ else None,
+ base_by_device = _spawn_base_by_dev,
+ )
+ if _spawn_flags:
+ run_cmd = [*run_cmd, *_spawn_flags]
+ if _spawn_total != _sidecar_fit_reserve_eligible_mib:
+ logger.info(
+ "Sidecar reserve repriced for this launch's argv: "
+ "%.2f GB.",
+ _spawn_total / 1024,
+ )
+ _sidecar_fit_reserve_mib = _spawn_reserve
+ if list(_spawn_flags) not in _sidecar_override_pairs:
+ _sidecar_override_pairs.append(list(_spawn_flags))
+ if (
+ _sidecar_fit_reserve_eligible_mib
+ and _sidecar_kind_for_fit
+ and not _spawn_cpu_only
+ ):
+ self._log_sidecar_fit_reserve(
+ run_cmd,
+ env,
+ _sidecar_kind_for_fit,
+ _sidecar_reserve_path,
+ _sidecar_fit_reserve_mib,
+ _sidecar_reserve_logged,
+ )
+ logger.info(
+ f"Starting llama-server: {' '.join(self._redacted_cmd_for_log(run_cmd))}"
+ )
_last_spawn_cmd = list(run_cmd)
# Read off the argv actually spawned rather than the intent, so
# every emitter is covered by construction.
@@ -28185,6 +29604,8 @@ class LlamaCppBackend:
# common/fit.cpp aborts a multi-device fit on a set split.
_run = _without_subsequence(_run, self._mixed_split_flags)
self._auto_tensor_split_emitted = None
+ # The --fit off launch emitted no margin; the next spawn adds the
+ # reserve.
# Same reasoning as the page-lock below, one step
# earlier: the fit picked "none" because it had proved
# the load fits, and that proof is what just failed, so
@@ -29164,6 +30585,28 @@ class LlamaCppBackend:
# Read by admission control; left as-is Studio over-admits.
n_parallel = 1 # allow-slot-clamp: llama-server refused more
kv_cache_unified = False
+ # The reserve was priced for the dropped slots: reprice for one sequence.
+ if (
+ _sidecar_fit_reserve_eligible_mib
+ and _sidecar_reserve_reprice is not None
+ ):
+ cmd = _drop_sidecar_margins(
+ cmd,
+ _integrity_flags,
+ self._ctx_integrity_flags(
+ *_integrity_args,
+ fit_target_delta_mib = _fit_target_delta_mib,
+ fit_target_floor_mib = _metal_fit_target_floor_mib,
+ ),
+ _sidecar_override_pairs,
+ user_args = _sidecar_user_tokens,
+ )
+ _sidecar_override_pairs.clear()
+ _sidecar_reserve_reprice = functools.partial(
+ _sidecar_reserve_reprice, _np = 1, _kv_unified = False
+ )
+ _sidecar_fit_reserve_eligible_mib = _sidecar_reserve_reprice(cmd, env)
+ _sidecar_fit_reserve_mib = 0
# LLAMA_ARG_SPEC_TYPE reaches the child only if extras set --spec-type; else the launch scrubs it.
if _auto_ckpt_emitted:
cmd = self._without_flag_pairs(cmd, _auto_ckpt_emitted)
@@ -29373,6 +30816,22 @@ class LlamaCppBackend:
strip_split_mode = False,
)
fallback_cmd = cmd[:_spec_at] + ["--spec-default"] + _fb_tail
+ # A user-owned drafter in the extras or env still loads and keeps its reserve.
+ if _sidecar_fit_reserve_mib and not _extra_args_mtp_draft_path(
+ fallback_cmd, env = env
+ ):
+ _fb_integrity = self._ctx_integrity_flags(
+ *_integrity_args,
+ fit_target_delta_mib = _fit_target_delta_mib,
+ fit_target_floor_mib = _metal_fit_target_floor_mib,
+ )
+ fallback_cmd = _drop_sidecar_margins(
+ fallback_cmd,
+ _integrity_flags,
+ _fb_integrity,
+ _sidecar_override_pairs,
+ user_args = _sidecar_user_tokens,
+ )
# Restore the cut slots; if they fail (unified KV refused), retry as cut.
_fb_cut = (
(
@@ -29626,9 +31085,15 @@ class LlamaCppBackend:
)
self._mmproj_fallback_reason = "projector_startup_failure"
cmd = self._strip_mmproj_args(_vision_gpu_cmd)
- cmd = self._per_device_fit_target_argv(
- cmd, env, _fit_target_margins_by_idx
- )
+ # Keep a trailing sidecar pair: rewriting it would drop the drafter's
+ # reserve.
+ if [
+ "--fit-target",
+ str(_last_flag_value(cmd, {"-fitt", "--fit-target"})),
+ ] not in _sidecar_override_pairs:
+ cmd = self._per_device_fit_target_argv(
+ cmd, env, _fit_target_margins_by_idx
+ )
_unified_withdraw_if_unneeded(cmd, why = "The text-only retry")
# This retry bypasses _spawn_and_wait, so refresh the launched-argv snapshot here.
_last_spawn_cmd = list(cmd)
@@ -29878,6 +31343,11 @@ class LlamaCppBackend:
# dropped: a respawn replays this snapshot arbitrarily later, so recovery
# re-resolves the file instead of trusting a path this request verified.
self._last_load_intent = replace(intent, verified_gguf = None)
+ if not getattr(_AUTO_RECOVERY_LOAD, "active", False):
+ # A caller's load starts a fresh OOM episode; only automatic recovery carries
+ # the strike count.
+ self._metal_oom_strikes = None
+ self._metal_oom_terminal_key = None
self._mtp_runtime_fallback_active = _mtp_active_for_launched_server
self._start_mtp_crash_watchdog()
@@ -30619,6 +32089,10 @@ class LlamaCppBackend:
def matches_load_source(self, intent: GgufLoadIntent) -> bool:
"""Whether the resident model has the intent's identity and weights."""
+ # A retired child stays recorded until respawned: an identical load must replace it.
+ _retired = getattr(self, "_retired_process", None)
+ if _retired is not None and getattr(self, "_process", None) is _retired:
+ return False
if not self.is_loaded or (self._model_identifier or "").lower() != (
intent.model_identifier.lower()
):
@@ -30791,6 +32265,8 @@ class LlamaCppBackend:
self._cpu_fallback_reason = None
self._last_load_intent = None
self._mtp_runtime_fallback_active = False
+ self._metal_oom_strikes = None
+ self._metal_oom_terminal_key = None
self._hf_variant = None
self._is_vision = False
self._disable_vision = False
@@ -31175,11 +32651,7 @@ class LlamaCppBackend:
finally:
self._kill_process_group(_pgid)
_surviving_descendants = self._terminate_descendants(_descendants)
- # getattr: teardown must tolerate a partially-built backend (failed
- # __init__ or a __new__-built instance), as with _llama_log_fh below.
- if getattr(self, "_stats_logger", None) is not None:
- self._stats_logger.stop()
- self._stats_logger = None
+ self._stop_stats_logger()
# Forget the pid only once confirmed gone, so a startup sweep can still reap survivors.
_killed_pid = getattr(self._process, "pid", None)
_exit_code = getattr(self._process, "poll", lambda: None)()
@@ -32312,18 +33784,25 @@ class LlamaCppBackend:
if resp.status_code != 200:
logger.debug(f"slot {entry.get('id')} restore returned HTTP {resp.status_code}")
- def _maybe_recover_from_mtp_crash(self, exc: Optional[BaseException] = None) -> bool:
+ def _maybe_recover_from_mtp_crash(
+ self,
+ exc: Optional[BaseException] = None,
+ *,
+ sidecar_oom: bool = False,
+ ) -> bool:
"""Schedule one background reload without MTP after a mid-generation death.
An MTP launch can pass /health and the load-time fallback, then abort on
a later decode (MTP+tensor flash-attn, Gemma 4 on Apple Silicon #13253).
Not a persistent ban: a fresh load re-tries MTP. Returns True if scheduled.
+ ``sidecar_oom`` is the Metal-retirement caller: an Auto DSpark / DFlash launch
+ whose server was just killed after running out of Metal memory.
"""
# Cheap async-safe gate: only our live MTP launch, not cancelled, with a
# snapshot to replay.
if self._cancel_event.is_set():
return False
- if not self._mtp_runtime_fallback_active:
+ if not (self._mtp_runtime_fallback_active or sidecar_oom):
return False
# Read before claiming: a raise after the claim strands the flag, and nothing
# else clears it, blocking every later respawn.
@@ -32331,6 +33810,11 @@ class LlamaCppBackend:
proc = self._process
if snapshot is None or proc is None:
return False
+ if self._metal_oom_is_terminal(snapshot):
+ return False
+ # Stopped on purpose (Metal OOM, lost device): the next request's respawn replays it.
+ if not sidecar_oom and proc is getattr(self, "_retired_process", None):
+ return False
# Still serving: a transient error, so leave the caller its normal retry.
if proc.poll() is None and self._server_socket_is_open():
return False
@@ -32349,11 +33833,12 @@ class LlamaCppBackend:
if proc.poll() is None:
logger.debug("Generation error but llama-server is alive; keeping MTP.")
return
- logger.warning(
- "llama-server exited mid-generation with MTP speculative "
- "decoding (%s); reloading without speculative decoding.",
- type(exc).__name__ if exc is not None else "server exited",
- )
+ if not sidecar_oom:
+ logger.warning(
+ "llama-server exited mid-generation with MTP speculative "
+ "decoding (%s); reloading without speculative decoding.",
+ type(exc).__name__ if exc is not None else "server exited",
+ )
# Re-check under the load lock so a newer load isn't clobbered.
requested_mode = snapshot.speculative_type
with self._serial_load_lock:
@@ -32367,7 +33852,14 @@ class LlamaCppBackend:
if self._last_load_intent != snapshot:
logger.info("MTP-crash reload skipped: load settings changed.")
return
+ if self._metal_oom_is_terminal(snapshot):
+ logger.info("MTP-crash reload skipped: out of Metal memory twice.")
+ return
+ if not sidecar_oom and proc is getattr(self, "_retired_process", None):
+ logger.info("MTP-crash reload skipped: the server was retired.")
+ return
unload_epoch = self._unload_epoch
+ dropped_draft = self._mtp_draft_path or self._mtp_draft_suppressed_path
# llama.cpp appends spec types, so strip them from extras.
_ea = list(snapshot.extra_args or ())
fallback_extra_args = snapshot.extra_args
@@ -32387,7 +33879,8 @@ class LlamaCppBackend:
speculative_type = "off",
extra_args = fallback_extra_args,
)
- started = bool(self.load_model(fallback))
+ with _automatic_recovery_load():
+ started = bool(self.load_model(fallback))
if not started:
logger.info("MTP-crash reload stopped before the replacement was ready.")
return
@@ -32413,13 +33906,18 @@ class LlamaCppBackend:
else _canonicalize_spec_mode(requested_mode)
)
self._spec_fallback_reason = "runtime_error"
+ # Keep the Auto identity so the same request matches instead of
+ # relaunching the sidecar.
+ if dropped_draft:
+ self._mtp_draft_path = None
+ self._mtp_draft_suppressed_path = dropped_draft
if undo_reload:
logger.info(
"MTP-crash reload undone: the model was unloaded during reload."
)
self.unload_model()
return
- logger.info("Reloaded without MTP after the mid-generation crash.")
+ logger.info("Reloaded without speculative decoding after the mid-generation crash.")
except Exception as e:
logger.error(f"Reload without MTP failed: {e}")
finally:
@@ -32745,6 +34243,29 @@ class LlamaCppBackend:
base = _VRAM_FLOOR_RESERVE_MIB if auto_fit else _LLAMA_FIT_TARGET_DEFAULT_MIB
return max(_VRAM_FLOOR_RESERVE_MIB, base + fit_target_delta_mib, floor_mib)
+ def _sidecar_fit_reserve_mib(
+ self,
+ reserve_bytes: int,
+ sidecar_path: Optional[str],
+ *,
+ flat_mib: int = 0,
+ ) -> int:
+ """MiB a DSpark/DFlash sidecar needs kept free by the child's ``--fit-target``.
+
+ The planner's byte-accurate reserve (draft weights + draft KV + draft compute +
+ target verification rows), never less than the sidecar file itself, which is
+ all that is known when the reserve fell back to the flat fraction (0 bytes).
+ ``flat_mib``: that fraction of the target's pool, passed only when the planner held
+ it back, so it is a floor: the child keeps at least what the plan kept.
+ Rounded up: a short MiB is a short allocation.
+ """
+ try:
+ file_bytes = int(self._get_gguf_size_bytes(sidecar_path) or 0) if sidecar_path else 0
+ except Exception:
+ file_bytes = 0
+ need = max(int(reserve_bytes or 0), file_bytes)
+ return max(math.ceil(need / (1024 * 1024)), max(0, int(flat_mib or 0)))
+
@staticmethod
def _fit_target_margins_mib(
*,
@@ -32812,6 +34333,67 @@ class LlamaCppBackend:
return False
return True
+ def _log_sidecar_fit_reserve(
+ self,
+ run_cmd: List[str],
+ env: Mapping[str, str],
+ kind: str,
+ sidecar_path: Optional[str],
+ reserved_mib: int,
+ logged: list,
+ ) -> None:
+ """Report whether the spawned argv reserves the DSpark / DFlash sidecar that
+ llama.cpp's --fit cannot size. Once per outcome per load (``logged`` holds the
+ last one), so a retry that changes it says so and a repeat stays quiet."""
+ if not fit_is_effectively_on(run_cmd, env) or not _extra_args_mtp_draft_path(
+ run_cmd, env = env
+ ):
+ return
+ outcome = (kind, bool(reserved_mib))
+ if logged and logged[-1] == outcome:
+ return
+ logged.append(outcome)
+ label = "DSpark" if kind == "dspark" else "DFlash"
+ if reserved_mib:
+ logger.info(
+ "%s under --fit on: llama.cpp cannot size the sidecar, so "
+ "--fit-target keeps %.2f GB free for it.",
+ label,
+ reserved_mib / 1024,
+ )
+ return
+ # Not a blocker: the fit step cannot measure a borrowed-tensor draft
+ # (llama-context.cpp:153-160).
+ logger.info(
+ "%s under --fit on: llama.cpp cannot size the sidecar during "
+ "fitting, so its ~%.1f GB is not reserved. Pin GPU Layers in "
+ "Manual mode if the load runs out of VRAM.",
+ label,
+ (self._get_gguf_size_bytes(sidecar_path) or 0) / 1e9,
+ )
+
+ def _positional_child_order_unproven(self, child_env: Mapping[str, str]) -> bool:
+ """Whether a per-card list built over the parent's GPU ids may not line up with the
+ child's CUDA enumeration (planning and spawn share it).
+
+ nvidia-smi ids follow PCI order, so the child must run under PCI_BUS_ID
+ (FASTEST_FIRST is CUDA's default) with a mask a resolver maps, unless the uuid
+ mask order is known. Torch-fallback ids are CUDA ordinals in the parent's own
+ order, so a child switched to another order (a GPU pick pins PCI_BUS_ID) counts
+ them differently. HIP ignores CUDA_DEVICE_ORDER.
+ """
+ child_order = str(child_env.get("CUDA_DEVICE_ORDER", "")).strip().upper()
+ if self._GPU_IDS_ARE_PCI_INDICES is True:
+ return (
+ child_order != "PCI_BUS_ID" or self._visibility_mask_is_unmappable()
+ ) and self._smi_uuid_mask_order() is None
+ if self._GPU_IDS_ARE_PCI_INDICES is False:
+ parent_order = (os.environ.get("CUDA_DEVICE_ORDER") or "").strip().upper()
+ return (parent_order or "FASTEST_FIRST") != (
+ child_order or "FASTEST_FIRST"
+ ) and not self._host_torch_is_rocm()
+ return False
+
@staticmethod
def _per_device_fit_target_argv(
cmd: list[str], env: Mapping[str, str], margins_by_idx: Optional[Mapping[int, float]]
@@ -32876,9 +34458,11 @@ class LlamaCppBackend:
*,
fit_target_delta_mib: float = 0.0,
fit_target_floor_mib: float = 0.0,
+ fit_target_reserve_mib: int = 0,
) -> list[str]:
"""Explicit --parallel drops the --kv-unified default and splits -c per slot; restore the
- shared pool."""
+ shared pool. ``fit_target_reserve_mib`` is added on top for a sidecar the fitter cannot
+ measure."""
flags: list[str] = []
if n_parallel > 1 and caps.get("supports_kv_unified"):
flags.append("--kv-unified")
@@ -32896,6 +34480,8 @@ class LlamaCppBackend:
fit_target_delta_mib = fit_target_delta_mib,
floor_mib = fit_target_floor_mib,
)
+ # On top of every margin, not max(): the sidecar's bytes must stay free.
+ _target += max(0, fit_target_reserve_mib)
# The legacy path stays silent when it would only restate llama.cpp's
# own default, so an untouched slider leaves the command unchanged.
if auto_fit or _target != _LLAMA_FIT_TARGET_DEFAULT_MIB:
@@ -33388,6 +34974,12 @@ class LlamaCppBackend:
intent = self._last_load_intent
if intent is None:
return False
+ if self._metal_oom_is_terminal(intent):
+ logger.error(
+ f"Not restarting llama-server for '{self._model_identifier}': "
+ "it ran out of Metal memory twice with these settings."
+ )
+ return False
epoch = self._unload_epoch
self._healthy = False
logger.warning(
@@ -33395,7 +34987,8 @@ class LlamaCppBackend:
f"(code {proc.returncode}); respawning to recover the session"
)
try:
- started = bool(self.load_model(intent))
+ with _automatic_recovery_load():
+ started = bool(self.load_model(intent))
except Exception as exc:
logger.error(f"Failed to respawn llama-server: {exc}")
return False
@@ -33406,45 +34999,239 @@ class LlamaCppBackend:
return False
return started
- def _retire_device_lost_server(self, served_by) -> None:
- """Kills the llama-server that hit VK_ERROR_DEVICE_LOST if still current: it survives but fails later requests; a dead child is handled by _respawn_if_dead."""
- with self._respawn_lock:
+ def _stop_stats_logger(self) -> None:
+ """Stop the engine-stats poller of the child going away. getattr: teardown must
+ tolerate a partially-built backend (failed __init__ or a __new__-built instance)."""
+ if getattr(self, "_stats_logger", None) is not None:
+ self._stats_logger.stop()
+ self._stats_logger = None
+
+ def _retire_device_lost_server(
+ self,
+ served_by,
+ metal_poison: Optional[dict] = None,
+ ) -> None:
+ """Kill the llama-server that hit a lost GPU device, if it is still current (#11453).
+
+ It survives VK_ERROR_DEVICE_LOST but fails every later request; a dead child is
+ what _respawn_if_dead already recovers. ``metal_poison`` marks the Metal variant:
+ a failed command buffer (usually out of memory) latches the backend the same way.
+ """
+ if metal_poison is not None:
+ # Outside the locks: the probe waits on the child up to its timeout.
if (
served_by is None
or served_by is not self._process
or served_by.poll() is not None
or self._cancel_event.is_set()
+ or not self._metal_compute_failure_confirmed()
):
+ metal_poison["pending"] = False
return
- # Else the MTP watchdog reads this kill as an MTP crash.
+ # _serial_load_lock too: no load may publish another model between the kill and the strike.
+ with self._respawn_lock, self._serial_load_lock:
+ if (
+ served_by is None
+ or served_by is not self._process
+ or served_by.poll() is not None
+ or self._cancel_event.is_set()
+ ):
+ return
+ # First, or the MTP watchdog reads this kill as an MTP crash.
self._stop_mtp_crash_watchdog()
+ self._retired_process = served_by
try:
served_by.kill()
served_by.wait(timeout = 10)
except (OSError, subprocess.TimeoutExpired):
pass
+ # Else it polls the dead child's port, which another server may reuse.
+ self._stop_stats_logger()
+ if metal_poison is not None:
+ # Under _respawn_lock, or a request could respawn the speculative config.
+ terminal = False
+ with self._lock:
+ if self._process is served_by and not self._cancel_event.is_set():
+ terminal = self._after_metal_poisoned_server(
+ bool(metal_poison.get("draft"))
+ )
+ if not terminal:
+ return
+ # Nothing will respawn it: unload it like any other model, still under
+ # _serial_load_lock.
+ self.unload_model()
+ # A reload queued on _serial_load_lock may land before the hooks run.
+ retired_epoch = self._unload_epoch
+ if metal_poison is not None:
+ # Outside every backend lock (gpu_arbiter._lock comes first).
+ for hook in list(_TERMINAL_RETIREMENT_HOOKS):
+ try:
+ hook(self, retired_epoch)
+ except Exception as exc: # noqa: BLE001 -- best effort, as a reap
+ logger.debug("Terminal retirement cleanup failed: %s", exc)
+ return
try:
logger.warning(
f"llama-server for '{self._model_identifier}' lost its GPU device "
- "(the graphics driver reset it); restarted it. On Intel Arc with "
- "Vulkan, GGML_VK_DISABLE_COOPMAT=1 may avoid the reset."
+ "(the graphics driver reset it); stopped it so the next request starts "
+ "a fresh one. On Intel Arc with Vulkan, GGML_VK_DISABLE_COOPMAT=1 may "
+ "avoid the reset."
)
except Exception:
# The logger raises once stdout is closed; recovery must not.
pass
- def _retire_device_lost_server_soon(self, served_by) -> None:
- """Retires a server whose log shows a device loss, after a grace for chat requests to retry
- first."""
+ def _metal_compute_failure_confirmed(self) -> bool:
+ """Whether the child really fails compute: a poisoned Metal backend rejects every
+ graph, so a one-token request comes back 500 "Compute error.". The log marker
+ alone is not proof, since request text can reach the server's error output. An
+ --embedding server is restricted to embeddings, so it is probed on the native
+ /embedding route, which computes under every pooling type but RANK (/v1/embeddings
+ 400s on NONE pooling before compute). RANK pools n_cls_out floats while
+ send_embedding reads n_embd_out of them, so a reranker (--rerank or --pooling rank)
+ is probed on /rerank, and one ranking only by the GGUF's default (/rerank refuses
+ it) is not probed at all."""
+ try:
+ embedding = bool(self.is_embedding_gguf)
+ except Exception:
+ embedding = False
+ args = [str(a) for a in (self._extra_args or ())]
+ # Argv only (load_model strips inherited LLAMA_ARG_POOLING); last-wins, as common/arg.cpp
+ # applies them.
+ pooling = None
+ for i, raw in enumerate(args):
+ flag = _flag_name(raw)
+ if flag in ("--embedding", "--embeddings"):
+ embedding = True
+ elif flag in ("--rerank", "--reranking"):
+ embedding = True
+ pooling = "rank"
+ elif flag == "--pooling":
+ _, eq, inline = raw.partition("=")
+ value = inline if eq else (args[i + 1] if i + 1 < len(args) else "")
+ pooling = value.strip().lower() or pooling
+ if embedding and pooling == "rank":
+ route, body = "/rerank", {"query": " ", "documents": [" "]}
+ elif embedding and pooling is None and getattr(self, "_pooling_type", None) == 4:
+ return False
+ elif embedding:
+ route, body = "/embedding", {"content": " "}
+ else:
+ route, body = "/completion", {"prompt": " ", "n_predict": 1, "stream": False}
+ try:
+ resp = httpx.post(
+ f"{self.base_url}{route}",
+ json = body,
+ headers = self._auth_headers,
+ timeout = 10.0,
+ trust_env = False,
+ )
+ if resp.status_code < 500:
+ return False
+ error = resp.json().get("error", {})
+ message = error.get("message", "") if isinstance(error, dict) else error
+ return (
+ str(message)
+ .rstrip()
+ .endswith(("Compute error.", "failed to process speculative batch"))
+ )
+ except Exception:
+ return False
+
+ @staticmethod
+ def _metal_oom_key(intent):
+ """The load an OOM strike is charged to: the speculative mode is what the fallback
+ changes, so it is left out and the no-drafter reload keeps the strike count. The
+ verified-cache hint is per request (the committed snapshot drops it), so it is left
+ out too, or a hinted explicit reload and the snapshot it commits disagree."""
+ try:
+ return replace(intent, speculative_type = None, verified_gguf = None)
+ except TypeError:
+ return intent
+
+ def _metal_oom_is_terminal(self, intent) -> bool:
+ """Whether ``intent`` already ran out of Metal memory twice (no more replays)."""
+ terminal = getattr(self, "_metal_oom_terminal_key", None)
+ return (
+ terminal is not None and intent is not None and self._metal_oom_key(intent) == terminal
+ )
+
+ @contextlib.contextmanager
+ def _while_still_retired(self, epoch):
+ """Hold _serial_load_lock, which every load takes, and yield whether this backend is
+ still the one a terminal Metal strike unloaded at ``epoch``: nothing spawned since and
+ no other unload. A cleanup acting on a replacement would kill it."""
+ with self._serial_load_lock:
+ yield self._process is None and getattr(self, "_unload_epoch", None) == epoch
+
+ def _after_metal_poisoned_server(self, draft_failed: bool) -> bool:
+ """Choose the recovery after a Metal-poisoned server was retired; True on the terminal
+ second strike."""
+ intent = self._last_load_intent
+ key = self._metal_oom_key(intent) if intent is not None else None
+ prior = getattr(self, "_metal_oom_strikes", None)
+ strikes = prior[1] + 1 if prior is not None and key is not None and prior[0] == key else 1
+ self._metal_oom_strikes = (key, strikes)
+ where = "the speculative draft context" if draft_failed else "a decode"
+ try:
+ name = self._model_identifier
+ if strikes >= 2:
+ # Not replaced: an identical explicit load must spawn a new child.
+ self._healthy = False
+ self._metal_oom_terminal_key = key
+ logger.error(
+ f"llama-server for '{name}' ran out of Metal memory again in {where} "
+ "after a restart; not restarting it. Lower the context length or the "
+ "GPU layers, turn off speculative decoding, or choose a smaller quant, "
+ "then load the model again."
+ )
+ return True
+ requested_mode = getattr(intent, "speculative_type", None)
+ recovery_extras = list(getattr(intent, "extra_args", None) or ())
+ # A bare --model-draft in the extras survives an Off reload (has_dft()).
+ auto_sidecar = (
+ self._speculative_type in ("draft-dspark", "draft-dflash")
+ and _canonicalize_spec_mode(requested_mode) in (None, "auto")
+ and not _extra_args_set_spec_type(recovery_extras)
+ and not _extra_args_mtp_draft_path(
+ recovery_extras, env = _child_spec_env(recovery_extras)
+ )
+ )
+ if auto_sidecar and self._maybe_recover_from_mtp_crash(sidecar_oom = True):
+ logger.warning(
+ f"llama-server for '{name}' ran out of Metal memory in {where} with the "
+ f"{self._speculative_type.split('-', 1)[1]} drafter and its Metal backend "
+ "cannot recover; reloading once without speculative decoding."
+ )
+ return False
+ logger.warning(
+ f"llama-server for '{name}' ran out of Metal memory in {where} and its Metal "
+ "backend cannot recover; stopped it so the next request starts a fresh one."
+ )
+ except Exception:
+ pass
+ return strikes >= 2
+
+ def _retire_device_lost_server_soon(
+ self,
+ served_by,
+ metal_poison: Optional[dict] = None,
+ ) -> None:
+ """Retire after a device loss in the server log, for the passthrough routes.
+
+ The grace lets a chat request that hit the same error retire and retry first.
+ """
def _retire():
time.sleep(_DEVICE_LOST_LOG_GRACE_S)
- self._retire_device_lost_server(served_by)
+ self._retire_device_lost_server(served_by, metal_poison = metal_poison)
try:
threading.Thread(target = _retire, daemon = True, name = "llama-device-lost").start()
except RuntimeError:
- pass
+ if metal_poison is not None:
+ # No retirement will run: let a later poison line try again.
+ metal_poison["pending"] = False
@contextlib.contextmanager
def _open_chat_stream_with_respawn_retry(
diff --git a/studio/backend/routes/inference.py b/studio/backend/routes/inference.py
index 943e37c5124..ee77adca9c6 100644
--- a/studio/backend/routes/inference.py
+++ b/studio/backend/routes/inference.py
@@ -165,6 +165,7 @@ from core.inference.llama_admission import (
from core.inference.tool_stream_exec import TOOL_APPROVAL_FLUSH_DELAY_S
from core.inference import model_slots
from core.inference.llama_cpp import (
+ _TERMINAL_RETIREMENT_HOOKS,
_llama_chunk_has_generated_output,
_local_ssl_context,
register_serving_backend,
@@ -11562,6 +11563,57 @@ def reap_dead_managed_engine(backend) -> None:
account_access.clear_resident("chat")
+def _orchestrator_occupied(orchestrator) -> bool:
+ """A standard, MLX or managed-engine model held or loading, as release_chat_gpu_claim counts it."""
+ if orchestrator is None:
+ return False
+ return bool(
+ getattr(orchestrator, "_managed_engine", None) is not None
+ or getattr(orchestrator, "active_model_name", None)
+ or tuple(getattr(orchestrator, "loading_models", ()) or ())
+ )
+
+
+def _primary_seat_occupied() -> bool:
+ """A non-GGUF model holds or is loading into the primary seat (call in the primary's slot)."""
+ return _orchestrator_occupied(_peek_inference_backend()) or _resident_npu_model() is not None
+
+
+def _release_after_terminal_retirement(backend, epoch) -> None:
+ """A llama-server stopped for good by a second Metal out-of-memory strike unloads itself
+ (llama_cpp._retire_device_lost_server); finish it as a manual unload would. A kept slot is
+ dropped at once (dead, it holds nothing a request could still use, so its refs do not keep
+ it) and leaves the primary's residency alone; the primary clears its own sharers. CHAT goes
+ once nothing else holds it. A reload that landed since the strike owns the slot and the
+ residency now, so nothing is touched then (checked under the lock every load takes).
+ A non-GGUF model holding or loading into the seat (primary or kept slot) owns it too."""
+ primary = backend is _llama_cpp_backend
+ with backend._while_still_retired(epoch) as retired:
+ filling = model_slots.loading[0] if model_slots.loading else None
+ if not retired or (filling is not None and filling.llama is backend):
+ return
+ for slot in [
+ s
+ for s in list(model_slots.slots)
+ if s.llama is backend and not _orchestrator_occupied(s.orchestrator)
+ ]:
+ try:
+ model_slots.drop(slot)
+ except Exception as exc: # noqa: BLE001 -- drop() keeps a stuck slot for retry
+ logger.warning("Could not drop a kept model after its Metal out-of-memory: %s", exc)
+ if primary and not model_slots.in_slot(None, _primary_seat_occupied):
+ account_access.clear_resident("chat")
+ # The arbiter lock comes before the backend's: released outside it, and release_if
+ # re-checks that nothing is resident.
+ if primary:
+ release_chat_gpu_claim()
+ else:
+ release_chat_after_kept_models()
+
+
+_TERMINAL_RETIREMENT_HOOKS.append(_release_after_terminal_retirement)
+
+
def reap_dead_worker(backend) -> None:
"""Cleared, never respawned: every routed request runs this, so an OOM respawn would loop."""
from core.inference.gpu_arbiter import CHAT, holds
@@ -14727,12 +14779,36 @@ def _wait_for_native_audio_gpu_free_gb(
async def _unload_llama_before_standard_load(llama_backend) -> None:
- """Tear down llama-server and wait for asynchronous driver VRAM reclaim."""
- if not llama_backend.is_loaded:
+ """Tear down llama-server and wait for asynchronous driver VRAM reclaim.
+
+ Unloads even when nothing is loaded, under the lock that every GGUF load and a terminal
+ Metal retirement's cleanup (_release_after_terminal_retirement) hold: the unload advances
+ the epoch, so a cleanup still pending sees this load replaced the retired server, and one
+ already past its check finishes first."""
+
+ def handoff() -> Optional[float]:
+ # getattr: a stand-in backend has no load to order against.
+ lock = getattr(llama_backend, "_serial_load_lock", None)
+ if lock is not None:
+ # As the unload would: a recovery reload holding the lock stops instead of finishing.
+ llama_backend._cancel_event.set()
+ with lock or nullcontext():
+ if not llama_backend.is_loaded:
+ if lock is None:
+ return None
+ kill_started = time.monotonic()
+ llama_backend.unload_model()
+ # A starting or recovering server was killed: wait for it like a loaded one.
+ last_kill = getattr(llama_backend, "_last_kill_monotonic", 0.0) or 0.0
+ return kill_started if last_kill >= kill_started else None
+ logger.info("Unloading GGUF model before loading Unsloth model")
+ kill_started = time.monotonic()
+ llama_backend.unload_model()
+ return kill_started
+
+ kill_started = await asyncio.to_thread(handoff)
+ if kill_started is None:
return
- logger.info("Unloading GGUF model before loading Unsloth model")
- kill_started = time.monotonic()
- await asyncio.to_thread(llama_backend.unload_model)
await asyncio.to_thread(
llama_backend._wait_for_vram_settle,
since_kill = kill_started,
diff --git a/studio/backend/tests/_sidecar_fit_helpers.py b/studio/backend/tests/_sidecar_fit_helpers.py
new file mode 100644
index 00000000000..eb3490fc4e0
--- /dev/null
+++ b/studio/backend/tests/_sidecar_fit_helpers.py
@@ -0,0 +1,323 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
+
+"""Shared launch harness for the DSpark/DFlash sidecar reserve tests."""
+
+from __future__ import annotations
+
+import subprocess
+import sys
+from pathlib import Path
+from unittest.mock import patch
+
+_TESTS_DIR = str(Path(__file__).resolve().parent)
+if _TESTS_DIR not in sys.path:
+ sys.path.insert(0, _TESTS_DIR)
+
+import test_llama_cpp_placement as placement # noqa: E402
+import test_metal_explicit_context_guard as metal_guard # noqa: E402
+from core.inference import llama_cpp as lc # noqa: E402
+from core.inference.llama_cpp import GgufLoadIntent, LlamaCppBackend # noqa: E402
+
+MIB = 1024 * 1024
+GIB = 1024 * MIB
+CAPS = {
+ "supports_dspark": True,
+ "supports_dflash": True,
+ "supports_fit_target": True,
+ "supports_fit_ctx": True,
+ "supports_kv_unified": True,
+ "spec_draft_n_max_flag": "--spec-draft-n-max",
+}
+# llama.cpp's default 1024 MiB margin plus the 8 GiB sidecar.
+FLOOR = 1024 + 8 * 1024
+ONE = ((0, 24_000, 24_000),)
+TWO = ((0, 24_000, 24_000), (1, 24_000, 24_000))
+MANUAL_AUTO = {"gpu_memory_mode": "manual", "gpu_layers": -1}
+DRAFT_KV = 768 * MIB
+NATIVE_MARKER = b"measuring the extra model next to an allocation-free target context"
+UNSET = object()
+
+
+def last(cmd, name):
+ """Last-wins value of ``name``, in either the spaced or the ``=`` spelling."""
+ value = None
+ for i, tok in enumerate(cmd):
+ if tok == name and i + 1 < len(cmd):
+ value = cmd[i + 1]
+ elif tok.startswith(name + "="):
+ value = tok.split("=", 1)[1]
+ return value
+
+
+def margins(cmd):
+ raw = last(cmd, "--fit-target") or "1024"
+ return [int(float(e)) for e in str(raw).split(",")]
+
+
+def margin(cmd):
+ return max(margins(cmd))
+
+
+def fit_targets(cmd):
+ return [cmd[i + 1] for i, a in enumerate(cmd[:-1]) if a == "--fit-target"]
+
+
+def sidecar(tmp_path, kind = "dspark"):
+ path = tmp_path / f"{kind}-model-Q8_0.gguf"
+ path.write_bytes(b"draft")
+ return str(path)
+
+
+def cuda_backend(
+ tmp_path,
+ memory = ONE,
+ vulkan = False,
+ draft_key = "dspark",
+):
+ backend, gguf = placement._backend(tmp_path, vulkan = vulkan, memory = list(memory))
+ backend._select_gpus = lambda *a, **k: (None, True)
+ backend.probe_server_capabilities = lambda _binary = None: dict(CAPS)
+ backend._get_gguf_size_bytes = lambda path: 8 * GIB if draft_key in str(path) else 1024
+ # A discrete-GPU host on every OS, not an Apple Silicon runner's unified budget.
+ backend._apple_metal_memory_budget_bytes = lambda: 0
+ return backend, gguf
+
+
+def cards(
+ tmp_path,
+ memory = TWO,
+ vulkan = False,
+):
+ backend, gguf = cuda_backend(tmp_path, memory = memory, vulkan = vulkan)
+ backend._arch_gate_survivors = lambda binary = None: []
+ return backend, gguf
+
+
+def dspark(backend, gguf, tmp_path, **kw):
+ return placement._launch(
+ backend, gguf, speculative_type = "dspark", dspark_draft_path = sidecar(tmp_path), **kw
+ )["cmd"]
+
+
+def per_token_estimate(backend, per_token):
+ backend._estimate_mtp_overhead_bytes = lambda n_ctx, **kw: (
+ 8 * GIB + max(0, int(n_ctx)) * per_token
+ )
+
+
+def slot_priced(backend):
+ backend._estimate_mtp_overhead_bytes = lambda n_ctx, n_parallel = 1, **kw: (
+ 8 * GIB + max(1, int(n_parallel or 1)) * GIB
+ )
+ return backend
+
+
+def three_block_drafter(backend):
+ read = backend._draft_backend_for
+
+ def three_blocks(path):
+ db = read(path)
+ db._n_layers = 3
+ return db
+
+ backend._draft_backend_for = three_blocks
+
+
+def elf(path, payload = b""):
+ path.write_bytes(b"\x7fELF" + payload + b"\x00")
+ return path
+
+
+def info_messages(monkeypatch):
+ seen = []
+ real_info = lc.logger.info
+
+ def info(msg, *args, **kwargs):
+ try:
+ seen.append(str(msg) % args if args else str(msg))
+ except TypeError:
+ seen.append(str(msg))
+ return real_info(msg, *args, **kwargs)
+
+ monkeypatch.setattr(lc.logger, "info", info)
+ return seen
+
+
+class _Spawns(list):
+ loaded = None
+
+
+def spawns(
+ backend,
+ gguf,
+ *,
+ healthy,
+ crash = lambda i, cmd: None,
+ stdout = lambda i, cmd: (),
+ late_exit = False,
+ **intent,
+):
+ """Every llama-server argv the load spawns. ``crash(i, cmd)`` is spawn i's exit code
+ (None = running), ``stdout`` its output, ``healthy`` its health answer. ``late_exit``
+ makes the first poll see the child running: it exits between the startup retry's check
+ and the outer --flash-attn off check."""
+ cmds = _Spawns()
+
+ def fake_popen(cmd, **kwargs):
+ if not cmd or str(cmd[0]) != "/fake/llama-server":
+ return placement._REAL_POPEN(cmd, **kwargs)
+ cmds.append(list(cmd))
+ rc = crash(len(cmds) - 1, list(cmd))
+ polls = [0]
+
+ def poll(self):
+ polls[0] += 1
+ return None if late_exit and polls[0] == 1 else rc
+
+ return type(
+ "Process",
+ (),
+ {
+ "pid": 2**31 - 1,
+ "stdout": (),
+ "returncode": rc,
+ "poll": poll,
+ "terminate": lambda self: None,
+ "wait": lambda self, timeout = None: rc or 0,
+ "kill": lambda self: None,
+ },
+ )()
+
+ def fake_health(timeout = None, **_kw):
+ i = len(cmds) - 1
+ backend._stdout_lines = list(stdout(i, cmds[-1]))
+ return healthy(i, cmds[-1])
+
+ backend._wait_for_health = fake_health
+ with patch.object(subprocess, "Popen", side_effect = fake_popen):
+ try:
+ cmds.loaded = backend.load_model(
+ GgufLoadIntent(gguf_path = str(gguf), model_identifier = "test", **intent)
+ )
+ except Exception:
+ pass
+ return cmds
+
+
+class _ReserveReport:
+ """Whether the user is told the sidecar is reserved: planning's verdict, then the
+ spawn's report off the argv it launched (the last outcome a retry logged)."""
+
+ def __init__(self):
+ self.planned = None
+ self.spawned: list = []
+
+ def __getitem__(self, key):
+ assert key == "fit_sized"
+ if self.planned is not True:
+ return self.planned
+ return self.spawned[-1] if self.spawned else True
+
+
+def spy_fit_sized(backend):
+ seen = _ReserveReport()
+ build = backend._build_speculative_flags
+ report = backend._log_sidecar_fit_reserve
+
+ def spy(**kw):
+ seen.planned = kw.get("dspark_fit_sized")
+ return build(**kw)
+
+ def log(run_cmd, env, kind, path, reserved_mib, logged):
+ before = len(logged)
+ report(run_cmd, env, kind, path, reserved_mib, logged)
+ if len(logged) > before:
+ seen.spawned.append(logged[-1][1])
+
+ backend._build_speculative_flags = spy
+ backend._log_sidecar_fit_reserve = log
+ return seen
+
+
+def host_kv_delta(
+ tmp_path,
+ monkeypatch,
+ *,
+ memory = ONE,
+ vulkan = False,
+ listed = UNSET,
+ apu = UNSET,
+ extras = (),
+ **kw,
+):
+ """MiB the -nkvo launch's largest margin is below the KV-on-card launch's.
+ ``listed`` is what --list-devices reports (None: no answer); ``apu`` the card whose
+ memory is the host's (None: none)."""
+ if listed is not UNSET:
+ monkeypatch.setattr(
+ LlamaCppBackend,
+ "_enumerated_gpu_devices",
+ classmethod(
+ lambda cls, binary = None, env = None: None if listed is None else list(listed)
+ ),
+ )
+ out = []
+ for name, more in (("on", []), ("off", ["-nkvo"])):
+ d = tmp_path / name
+ d.mkdir()
+ backend, gguf = cards(d, memory = memory, vulkan = vulkan)
+ backend._mtp_draft_kv_bytes = lambda *a, **k: DRAFT_KV
+ if apu is not UNSET:
+ backend._amd_apu_wants_unified_memory = lambda gpu_indices = None: (
+ apu is not None and (gpu_indices is None or apu in gpu_indices)
+ )
+ backend._integrated_cuda_unified_memory = lambda gpu_indices = None: False
+ out.append(margin(dspark(backend, gguf, d, extra_args = [*extras, *more], **kw)))
+ return out[0] - out[1]
+
+
+def metal_cmd(
+ tmp_path,
+ monkeypatch,
+ mode,
+ *,
+ budget_gib,
+ wired_gib = None,
+ sub = None,
+ n_ctx = 0,
+ **kw,
+):
+ """A real-fit Metal load with a DSpark sidecar; every GGUF here reads 4 GiB, the
+ sidecar too. Undoes ``monkeypatch`` so a test can load twice."""
+ sub = tmp_path / (sub or mode)
+ sub.mkdir()
+ path = sidecar(sub)
+ real_init = GgufLoadIntent.__init__
+
+ def init(self, *args, **kwargs):
+ kwargs.setdefault("dspark_draft_path", path)
+ kwargs.setdefault("speculative_type", mode)
+ real_init(self, *args, **kwargs)
+
+ monkeypatch.setattr(GgufLoadIntent, "__init__", init)
+ monkeypatch.setattr(
+ LlamaCppBackend,
+ "probe_server_capabilities",
+ classmethod(lambda cls, binary = None: dict(CAPS)),
+ )
+ backend = LlamaCppBackend()
+ wired_gib = budget_gib if wired_gib is None else wired_gib
+ cmd = metal_guard._launch(
+ sub,
+ monkeypatch,
+ n_ctx = n_ctx,
+ backend = backend,
+ real_fit = True,
+ budget_bytes = int(budget_gib * GIB),
+ wired_bytes = int(wired_gib * GIB),
+ weights_bytes = 4 * GIB,
+ **kw,
+ )["cmd"]
+ monkeypatch.undo()
+ return cmd, backend, path
diff --git a/studio/backend/tests/test_llama_cpp_metal_poisoned_respawn.py b/studio/backend/tests/test_llama_cpp_metal_poisoned_respawn.py
new file mode 100644
index 00000000000..5d79f530cba
--- /dev/null
+++ b/studio/backend/tests/test_llama_cpp_metal_poisoned_respawn.py
@@ -0,0 +1,990 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
+
+"""A llama-server whose Metal backend is poisoned is retired, respawned once, then unloaded."""
+
+from __future__ import annotations
+
+import asyncio
+import inspect
+import subprocess
+import sys
+import threading
+import time
+from dataclasses import dataclass, replace
+from typing import Optional
+from unittest.mock import patch
+
+import httpx
+import pytest
+
+import core.inference.llama_cpp as llama_cpp
+from core.inference import model_slots
+from core.inference.llama_cpp import GgufLoadIntent, LlamaCppBackend
+
+from . import _sidecar_fit_helpers as h
+
+# After a failed command buffer ggml-metal rejects every later graph until the backend is
+# recreated, so the child stays up answering "Compute error.". The reporter's lines:
+METAL_OOM = "0.30.864.334 E error: Insufficient Memory (00000008:kIOGPUCommandBufferCallbackErrorOutOfMemory)"
+METAL_LATCHED = (
+ "0.30.882.907 E ggml_metal_graph_compute: backend is in error state from a previous "
+ "command buffer failure - recreate the backend to recover"
+)
+DRAFT_FAILED = "0.22.891.174 E process: llama_process(ctx_dft) failed rc=-3 (n_tokens=14, offset=0)"
+
+
+@dataclass(frozen = True)
+class _Intent:
+ speculative_type: Optional[str] = None
+ extra_args: tuple = ()
+ gpu_ids: Optional[tuple] = None
+ verified_gguf: Optional[str] = None
+
+
+_OFF = _Intent(speculative_type = "off")
+_MTP = _Intent(speculative_type = "mtp")
+
+
+def _child(lines = ()):
+ code = "import sys, time\n"
+ for line in lines:
+ code += f"print({line!r}, flush = True)\n"
+ code += "time.sleep(120)\n"
+ return subprocess.Popen([sys.executable, "-c", code], stdout = subprocess.PIPE, text = True)
+
+
[email protected](autouse = True)
+def _studio_home(tmp_path, monkeypatch):
+ monkeypatch.setenv("UNSLOTH_STUDIO_HOME", str(tmp_path / "studio_home"))
+
+
+def harness(monkeypatch):
+ monkeypatch.setattr(llama_cpp, "_DEVICE_LOST_LOG_GRACE_S", 0.0)
+ children = []
+
+ def make(
+ intent: _Intent,
+ speculative_type: str = "draft-dspark",
+ lines = (),
+ ):
+ backend = LlamaCppBackend()
+ first = _child(lines)
+ children.append(first)
+ backend._process = first
+ backend._healthy = True
+ backend._model_identifier = "unsloth/DeepSeek-V4.1-Flash-1017-GGUF"
+ backend._last_load_intent = intent
+ backend._speculative_type = speculative_type
+ # The child here is a stub with no HTTP server: stand in for a compute that fails.
+ backend._metal_compute_failure_confirmed = lambda: True
+ loads = []
+
+ def fake_load_model(new_intent):
+ loads.append(new_intent)
+ replacement = _child()
+ children.append(replacement)
+ backend._process = replacement
+ backend._healthy = True
+ backend._last_load_intent = new_intent
+ backend._speculative_type = (
+ "none" if new_intent.speculative_type == "off" else speculative_type
+ )
+ return True
+
+ backend.load_model = fake_load_model
+ backend._server_socket_is_open = lambda: False
+ return backend, first, loads
+
+ yield make
+ for child in children:
+ if child.poll() is None:
+ child.kill()
+ child.wait()
+
+
+def _wait(predicate, timeout = 10.0):
+ deadline = time.monotonic() + timeout
+ while time.monotonic() < deadline:
+ if predicate():
+ return True
+ time.sleep(0.02)
+ return False
+
+
+def _poison(
+ backend,
+ process,
+ draft = False,
+):
+ backend._retire_device_lost_server(process, metal_poison = {"draft": draft})
+
+
+ "lines, retired",
+ [
+ ([METAL_OOM], True),
+ ([METAL_LATCHED], True),
+ (
+ [
+ "0.01.000.000 I slot launch_slot_: id 0 | prompt: what is kIOGPUCommandBufferCallbackErrorOutOfMemory?"
+ ],
+ False,
+ ),
+ (
+ ["what does backend is in error state from a previous command buffer failure mean?"],
+ False,
+ ),
+ ],
+ ids = ["oom_line", "latched_line", "info_line", "unprefixed_request_dump"],
+)
+def test_a_poisoned_metal_backend_in_the_log_retires_the_server(harness, lines, retired):
+ backend, first, _loads = harness(_OFF, "none", lines)
+ threading.Thread(target = backend._drain_stdout, daemon = True).start()
+ try:
+ first.wait(timeout = 5)
+ except subprocess.TimeoutExpired:
+ pass
+ assert (first.poll() is not None) is retired
+
+
+def test_echoed_marker_without_a_compute_failure_does_not_retire(harness):
+ """A grammar parse error prints the request's grammar, so a forged E-line can carry the
+ marker: retirement needs the child to really fail compute."""
+ backend, first, _loads = harness(_OFF, "none", [METAL_OOM])
+ backend._metal_compute_failure_confirmed = lambda: False
+ threading.Thread(target = backend._drain_stdout, daemon = True).start()
+ try:
+ first.wait(timeout = 1.5)
+ except subprocess.TimeoutExpired:
+ pass
+ assert first.poll() is None
+ assert getattr(backend, "_metal_oom_strikes", None) is None
+
+
+def test_auto_dspark_draft_oom_reloads_once_without_the_drafter_then_stops(harness):
+ backend, first, loads = harness(
+ _Intent(), "draft-dspark", [METAL_OOM, METAL_LATCHED, DRAFT_FAILED]
+ )
+ threading.Thread(target = backend._drain_stdout, daemon = True).start()
+
+ assert _wait(lambda: len(loads) == 1)
+ assert first.poll() is not None
+ assert loads[0].speculative_type == "off"
+ assert _wait(lambda: not backend._mtp_runtime_fallback_in_progress)
+
+ # The drafter-free server runs out of Metal memory too: the second strike is terminal.
+ second = backend._process
+ _poison(backend, second)
+ assert second.poll() is not None
+ assert backend._respawn_if_dead() is False
+ assert len(loads) == 1
+
+
+def test_explicit_dspark_is_not_silently_dropped(harness):
+ intent = _Intent(speculative_type = "dspark")
+ backend, first, loads = harness(intent, "draft-dspark")
+ _poison(backend, first, draft = True)
+ time.sleep(0.3)
+ # No drafter-free reload; the next request replays the user's own settings once.
+ assert loads == [] and first.poll() is not None
+ assert backend._respawn_if_dead() is True
+ assert loads == [intent]
+
+
+def test_retirement_keeps_a_user_owned_drafter(harness):
+ """Auto plus a bare --model-draft: an Off reload would still load that file."""
+ intent = _Intent(extra_args = ("--model-draft", "/models/custom-dspark.gguf"))
+ backend, first, loads = harness(intent, "draft-dspark")
+ _poison(backend, first, draft = True)
+ assert first.poll() is not None
+ _wait(lambda: bool(loads), timeout = 2.0)
+ assert all(load.speculative_type != "off" for load in loads)
+
+
+def test_a_launch_without_a_sidecar_respawns_once_then_stops(harness):
+ backend, first, loads = harness(_OFF, "none")
+ _poison(backend, first)
+ assert backend._respawn_if_dead() is True and loads == [_OFF]
+ _poison(backend, backend._process)
+ assert backend._respawn_if_dead() is False and loads == [_OFF]
+
+
+def test_a_terminal_second_oom_is_not_left_healthy(harness):
+ backend, first, _loads = harness(_OFF, "none")
+ _poison(backend, first)
+ assert backend._respawn_if_dead() is True
+ _poison(backend, backend._process)
+ assert backend._healthy is False
+
+
+def test_a_terminally_retired_server_is_not_active(harness):
+ """After the second Metal OOM nothing respawns the child, so delete, Always mode and the
+ training guard must not see a model still loading."""
+ backend, first, _loads = harness(_OFF, "none")
+ _poison(backend, first)
+ assert backend._respawn_if_dead() is True
+ assert backend.is_active
+ second = backend._process
+ _poison(backend, second)
+ assert second.poll() is not None
+ assert backend._process is None and backend._model_identifier is None
+ assert backend._respawn_if_dead() is False
+ assert backend.is_loaded is False
+ # routes/models.py refuses a delete with 409 while is_active or is_loaded.
+ assert backend.is_active is False
+
+
+def test_unload_clears_the_strikes(harness):
+ backend, first, _loads = harness(_OFF, "none")
+ backend._metal_oom_strikes = (_OFF, 2)
+ backend._metal_oom_terminal_key = _OFF
+ backend.unload_model()
+ assert backend._metal_oom_strikes is None and backend._metal_oom_terminal_key is None
+
+
+def test_a_model_swap_during_retirement_is_not_charged_the_strike(harness):
+ """An explicit load must not publish another model while the dead child is reaped."""
+ backend, first, _loads = harness(_OFF, "none")
+ real_kill = first.kill
+ blocked = {}
+
+ def kill_and_try_a_load():
+ real_kill()
+ got = []
+ t = threading.Thread(
+ target = lambda: got.append(backend._serial_load_lock.acquire(blocking = False))
+ )
+ t.start()
+ t.join()
+ blocked["load"] = got == [False]
+ if got == [True]:
+ backend._serial_load_lock.release()
+
+ first.kill = kill_and_try_a_load
+ _poison(backend, first)
+ assert blocked.get("load") is True
+ assert backend._metal_oom_strikes == (backend._metal_oom_key(_OFF), 1)
+
+
+def test_the_strike_is_recorded_before_a_respawn_can_run(harness):
+ """Retirement decides the recovery before releasing _respawn_lock, so a request racing
+ the kill cannot replay the speculative config first."""
+ backend, first, loads = harness(_Intent(), "draft-dspark")
+ seen = {}
+ real_after = backend._after_metal_poisoned_server
+
+ def after(draft_failed):
+ seen["respawn_lock_held"] = backend._respawn_lock.locked()
+ return real_after(draft_failed)
+
+ backend._after_metal_poisoned_server = after
+ _poison(backend, first, draft = True)
+ assert seen.get("respawn_lock_held") is True
+ assert _wait(lambda: len(loads) == 1)
+ assert loads[0].speculative_type == "off"
+
+
+def test_mtp_recovery_respects_the_terminal_strike(harness):
+ backend, first, loads = harness(_Intent(), "draft-dspark")
+ backend._metal_oom_terminal_key = backend._metal_oom_key(_Intent())
+ backend._mtp_runtime_fallback_active = True
+ first.kill()
+ first.wait()
+ assert backend._maybe_recover_from_mtp_crash(RuntimeError("boom")) is False
+ assert backend._maybe_recover_from_mtp_crash(sidecar_oom = True) is False
+ time.sleep(0.3)
+ assert loads == []
+
+
+def test_a_retirement_thread_that_cannot_start_releases_the_poison_flag(monkeypatch):
+ def refuse(self):
+ raise RuntimeError("can't start new thread")
+
+ monkeypatch.setattr(threading.Thread, "start", refuse)
+ poison = {"pending": True}
+ LlamaCppBackend()._retire_device_lost_server_soon(None, metal_poison = poison)
+ assert poison["pending"] is False
+
+
+# Confirming the poison: probe the route the server really computes on.
+
+
+def _compute_probe(
+ monkeypatch,
+ extras = None,
+ embedding = None,
+ message = "Compute error.",
+):
+ backend = LlamaCppBackend()
+ backend._port = 1
+ if extras is not None:
+ backend._extra_args = list(extras)
+ if embedding is not None:
+ monkeypatch.setattr(LlamaCppBackend, "is_embedding_gguf", property(lambda self: embedding))
+ resp = type(
+ "Resp", (), {"status_code": 500, "json": lambda self: {"error": {"message": message}}}
+ )()
+ with patch.object(llama_cpp.httpx, "post", return_value = resp, create = True) as post:
+ return backend._metal_compute_failure_confirmed(), post
+
+
+ "message, confirmed",
+ [
+ ("Compute error.", True),
+ ("decode() failed: failed to process speculative batch", True),
+ ("the request exceeds the available context size", False),
+ ],
+ ids = ["target", "draft", "unrelated"],
+)
+def test_the_draft_context_failure_confirms_the_poison(monkeypatch, message, confirmed):
+ assert _compute_probe(monkeypatch, message = message)[0] is confirmed
+
+
+ "env, embedding, extras, route, field",
+ [
+ # --rerank sets embedding mode with RANK pooling: /completion 501s before compute,
+ # and send_embedding would read n_embd_out floats out of RANK's n_cls_out.
+ ({}, False, ("--rerank",), "/rerank", "documents"),
+ ({}, False, ("--reranking",), "/rerank", "documents"),
+ ({}, False, ("--embeddings", "--pooling", "rank"), "/rerank", "documents"),
+ # A later --pooling replaces --rerank's RANK, as in common/arg.cpp.
+ ({}, False, ("--rerank", "--pooling", "mean"), "/embedding", "content"),
+ # /v1/embeddings 400s on NONE pooling, explicit or the model's default.
+ ({}, False, ("--embeddings", "--pooling", "none"), "/embedding", "content"),
+ ({}, True, (), "/embedding", "content"),
+ ({}, False, (), "/completion", None),
+ # --embedding restricts llama-server to embeddings: /completion never computes.
+ ({}, False, ("--embeddings",), "/embedding", None),
+ # load_model pops LLAMA_ARG_POOLING / _RERANKING / _EMBEDDINGS from the child's env.
+ ({"LLAMA_ARG_RERANKING": "1"}, False, (), "/completion", None),
+ ({"LLAMA_ARG_POOLING": "rank"}, True, (), "/embedding", None),
+ ],
+ ids = [
+ "rerank",
+ "reranking",
+ "pooling-rank",
+ "rerank-then-mean",
+ "pooling-none",
+ "embedding-gguf",
+ "completion",
+ "embeddings-flag",
+ "inherited-reranking-env",
+ "inherited-pooling-env",
+ ],
+)
+def test_embedding_and_rerank_servers_are_probed_where_they_compute(
+ monkeypatch, env, embedding, extras, route, field
+):
+ monkeypatch.delenv("LLAMA_ARG_POOLING", raising = False)
+ monkeypatch.delenv("LLAMA_ARG_RERANKING", raising = False)
+ for key, value in env.items():
+ monkeypatch.setenv(key, value)
+ confirmed, post = _compute_probe(monkeypatch, extras, embedding)
+ assert confirmed is True
+ assert post.call_args.args[0].endswith(route)
+ if field is not None:
+ assert field in post.call_args.kwargs["json"]
+
+
+def test_a_reranker_by_gguf_default_alone_is_not_probed(monkeypatch):
+ """--embeddings on a RANK-pooling GGUF: /rerank refuses (no --rerank) and /embedding would
+ over-read RANK's buffer, so nothing is sent and the server is not retired."""
+ monkeypatch.delenv("LLAMA_ARG_POOLING", raising = False)
+ monkeypatch.delenv("LLAMA_ARG_RERANKING", raising = False)
+ backend = LlamaCppBackend()
+ backend._port = 1
+ backend._pooling_type = 4
+ backend._extra_args = ["--embeddings"]
+ with patch.object(llama_cpp.httpx, "post", create = True) as post:
+ assert backend._metal_compute_failure_confirmed() is False
+ post.assert_not_called()
+
+
+# Metal OOM episodes: a strike belongs to one load; a caller's load starts a fresh one.
+
+
+def _stub_child():
+ return type(
+ "Process",
+ (),
+ {
+ "pid": 123,
+ "poll": lambda self: 0,
+ "kill": lambda self: None,
+ "wait": lambda self, timeout = None: 0,
+ },
+ )()
+
+
+def _strike(backend):
+ dead = _stub_child()
+ backend._process = dead
+ backend._retired_process = dead
+ backend._after_metal_poisoned_server(False)
+
+
+def _loaded(tmp_path, **kw):
+ backend, gguf = h.cuda_backend(tmp_path)
+ backend._speculative_type = "none"
+ h.placement._launch(backend, gguf, n_ctx = 4096, **kw)
+ return backend, gguf
+
+
[email protected]("retired", [False, True])
+def test_a_retired_child_is_not_reused_by_an_identical_load(tmp_path, retired):
+ """After a first retirement the killed child is still recorded and _healthy is still
+ true: an identical explicit load must spawn a replacement."""
+ gguf = tmp_path / "m.gguf"
+ gguf.write_bytes(b"gguf")
+ backend = LlamaCppBackend()
+ proc = _stub_child()
+ proc.poll = lambda: None
+ backend._process = proc
+ backend._healthy = True
+ backend._model_identifier = "m"
+ backend._gguf_path = str(gguf)
+ if retired:
+ backend._retire_device_lost_server(proc)
+ intent = GgufLoadIntent(gguf_path = str(gguf), model_identifier = "m")
+ assert backend.matches_load_source(intent) is (not retired)
+
+
+_HINT = ("someone/repo", "Q4_K_M", "/nonexistent/model.gguf", ())
+
+
+def test_the_strike_key_ignores_the_verified_cache_hint(tmp_path):
+ backend, gguf = h.cuda_backend(tmp_path)
+ plain = GgufLoadIntent(gguf_path = str(gguf), model_identifier = "test", n_ctx = 4096)
+ hinted = GgufLoadIntent(
+ gguf_path = str(gguf), model_identifier = "test", n_ctx = 4096, verified_gguf = _HINT
+ )
+ assert backend._metal_oom_key(hinted) == backend._metal_oom_key(plain)
+ backend._metal_oom_terminal_key = backend._metal_oom_key(plain)
+ assert backend._metal_oom_is_terminal(hinted)
+
+
+ "hint",
+ [None, _HINT],
+ ids = [
+ "test_explicit_reload_clears_a_terminal_metal_oom",
+ "test_a_hinted_explicit_reload_clears_a_terminal_metal_oom",
+ ],
+)
+def test_an_explicit_reload_clears_a_terminal_metal_oom(tmp_path, hint):
+ backend, gguf = h.cuda_backend(tmp_path)
+ # The terminal key comes from a committed snapshot, which never carries the hint.
+ snapshot = GgufLoadIntent(gguf_path = str(gguf), model_identifier = "test", n_ctx = 4096)
+ key = backend._metal_oom_key(snapshot)
+ backend._metal_oom_strikes = (key, 2)
+ backend._metal_oom_terminal_key = key
+ h.spawns(backend, gguf, healthy = lambda i, cmd: True, n_ctx = 4096, verified_gguf = hint)
+ assert backend._last_load_intent is not None
+ assert backend._last_load_intent.verified_gguf is None
+ assert backend._metal_oom_terminal_key is None
+ assert backend._metal_oom_strikes is None
+ assert not backend._metal_oom_is_terminal(backend._last_load_intent)
+
+
+def test_an_explicit_reload_with_changed_settings_clears_the_terminal_latch(tmp_path):
+ """The latch must not survive a reload that changes the GPU pick and re-arm once a
+ matching request writes gpu_ids back."""
+ backend, gguf = _loaded(tmp_path)
+ committed = backend._last_load_intent
+ assert committed is not None and committed.gpu_ids is None
+ _strike(backend)
+ _strike(backend)
+ assert backend._metal_oom_is_terminal(committed)
+ h.placement._launch(backend, gguf, n_ctx = 4096, gpu_ids = [0])
+ assert backend.is_active and backend.is_loaded
+ reloaded = backend._last_load_intent
+ assert list(reloaded.gpu_ids) == [0]
+ # Not vacuous: writing gpu_ids back reproduces the latched load exactly.
+ assert backend._metal_oom_key(replace(reloaded, gpu_ids = None)) == backend._metal_oom_key(
+ committed
+ )
+ assert backend._metal_oom_terminal_key is None
+ assert backend._metal_oom_strikes is None
+ backend._record_matching_gpu_request(None)
+ assert backend._metal_oom_is_terminal(backend._last_load_intent) is False
+
+
+def test_a_callers_load_starts_a_fresh_metal_oom_episode(tmp_path):
+ """A (one strike), then B, then A again: A's new session gets its own two strikes."""
+ backend, gguf = _loaded(tmp_path)
+ _strike(backend)
+ assert backend._metal_oom_strikes is not None
+ h.placement._launch(backend, gguf, n_ctx = 8192)
+ assert backend._metal_oom_strikes is None
+ h.placement._launch(backend, gguf, n_ctx = 4096)
+ _strike(backend)
+ assert backend._metal_oom_strikes[1] == 1
+ assert backend._metal_oom_terminal_key is None
+
+
+def test_automatic_recovery_keeps_the_strike(tmp_path):
+ """The respawn and the drafter-free reload replay under the recovery marker, which carries
+ the count, so a second OOM still stops respawning."""
+ backend, gguf = _loaded(tmp_path)
+ _strike(backend)
+ with llama_cpp._automatic_recovery_load():
+ h.placement._launch(backend, gguf, n_ctx = 4096)
+ assert backend._metal_oom_strikes[1] == 1
+ _strike(backend)
+ assert backend._metal_oom_terminal_key is not None
+
+
+def test_the_respawn_and_drafter_free_reload_run_as_automatic_recovery():
+ for fn in (LlamaCppBackend._respawn_if_dead, LlamaCppBackend._maybe_recover_from_mtp_crash):
+ assert "with _automatic_recovery_load():" in inspect.getsource(fn)
+
+
+def test_a_diffusion_load_drops_the_chat_servers_recovery_state(tmp_path, monkeypatch):
+ """A crashed diffusion runner must neither replay the chat load it replaced nor meet that
+ load's Metal OOM latch."""
+ import test_gpu_memory_mode as _gm
+
+ real_init = LlamaCppBackend.__init__
+
+ def init(self, *a, **k):
+ real_init(self, *a, **k)
+ self._last_load_intent = object()
+ self._mtp_runtime_fallback_active = True
+ self._metal_oom_strikes = ("chat", 2)
+ self._metal_oom_terminal_key = "chat"
+
+ monkeypatch.setattr(LlamaCppBackend, "__init__", init)
+ _cmd, _env, backend = _gm._launch_diffusion(
+ monkeypatch, tmp_path, gguf_gib = 4.0, gpus = _gm._TWO_3090, gpu_ids = None
+ )
+ assert backend._last_load_intent is None
+ assert backend._mtp_runtime_fallback_active is False
+ assert backend._metal_oom_strikes is None
+ assert backend._metal_oom_terminal_key is None
+
+
+# A server Studio retired on purpose is not an MTP crash: _respawn_if_dead replays it.
+
+
+def _mtp_server(harness):
+ backend, first, loads = harness(_MTP, "draft-mtp")
+ backend._mtp_runtime_fallback_active = True
+ return backend, first, loads
+
+
+def _retire(backend, first, metal):
+ if metal:
+ _poison(backend, first)
+ else:
+ backend._retire_device_lost_server(first)
+
+
[email protected]("metal", [True, False], ids = ["metal-strike-1", "vulkan-device-lost"])
+def test_a_retired_mtp_server_is_respawned_not_reloaded_without_mtp(harness, metal):
+ backend, first, loads = _mtp_server(harness)
+ _retire(backend, first, metal)
+ assert first.poll() is not None
+ assert backend._maybe_recover_from_mtp_crash(httpx.ConnectError("closed")) is False
+ assert backend._respawn_if_dead() is True
+ assert [load.speculative_type for load in loads] == ["mtp"]
+
+
+def test_a_real_mtp_crash_still_reloads_without_mtp(harness):
+ backend, first, loads = _mtp_server(harness)
+ first.kill()
+ first.wait()
+ assert backend._maybe_recover_from_mtp_crash(httpx.ConnectError("closed")) is True
+ assert _wait(lambda: bool(loads))
+ assert loads[-1].speculative_type == "off"
+
+
+def test_a_reload_queued_before_the_retirement_stands_down(harness):
+ """The request saw the port close while the child was still exiting; the retirement
+ lands during the queued reload's exit wait."""
+ backend, first, loads = _mtp_server(harness)
+ assert backend._maybe_recover_from_mtp_crash(httpx.ConnectError("closed")) is True
+ backend._retire_device_lost_server(first)
+ assert _wait(lambda: not backend._mtp_runtime_fallback_in_progress)
+ assert loads == []
+
+
+def test_the_retirement_is_published_before_the_kill(harness):
+ backend, first, _loads = _mtp_server(harness)
+ seen = []
+ kill = first.kill
+
+ def recording_kill():
+ seen.append(getattr(backend, "_retired_process", None) is first)
+ kill()
+
+ first.kill = recording_kill
+ backend._retire_device_lost_server(first)
+ assert seen == [True]
+
+
+# A retired child takes its engine-stats poller with it.
+
+
+class _Poller:
+ def __init__(self):
+ self.stopped = False
+
+ def stop(self):
+ self.stopped = True
+
+
[email protected]("metal", [True, False], ids = ["metal", "device-lost"])
+def test_a_retired_child_stops_its_stats_poller(harness, metal):
+ backend, first, _loads = harness(_OFF, "none")
+ poller = backend._stats_logger = _Poller()
+ _retire(backend, first, metal)
+ assert first.poll() is not None
+ assert poller.stopped and backend._stats_logger is None
+
+
+def test_a_terminal_metal_retirement_leaves_no_poller(harness):
+ backend, first, _loads = harness(_OFF, "none")
+ _poison(backend, first)
+ assert backend._respawn_if_dead() is True
+ poller = backend._stats_logger = _Poller()
+ _poison(backend, backend._process)
+ assert backend._process is None and backend._model_identifier is None
+ assert poller.stopped and backend._stats_logger is None
+
+
+def test_the_replacement_load_starts_its_own_poller(tmp_path, monkeypatch):
+ import core.inference.llama_stats as _stats
+
+ started = []
+ monkeypatch.setattr(
+ _stats,
+ "maybe_start_stats_logger",
+ lambda base_url, logger, headers = None: started.append(base_url) or _Poller(),
+ )
+ backend, gguf = h.cuda_backend(tmp_path)
+ caps = dict(h.CAPS, supports_metrics = True)
+ backend.probe_server_capabilities = lambda _binary = None: dict(caps)
+ h.placement._launch(backend, gguf, n_ctx = 4096)
+ # What a retirement leaves: the poller stopped and cleared.
+ backend._stats_logger.stop()
+ backend._stats_logger = None
+ h.placement._launch(backend, gguf, n_ctx = 4096, force_reload = True)
+ assert len(started) == 2 and isinstance(backend._stats_logger, _Poller)
+ assert not backend._stats_logger.stopped
+
+
+# A terminal Metal retirement is a full unload, and the routes release what it held.
+
+
+class _Fh:
+ def __init__(self):
+ self.closed = False
+
+ def close(self):
+ self.closed = True
+
+ def write(self, *_a):
+ return 0
+
+ def flush(self):
+ pass
+
+
+def _two_strikes(backend, first, monkeypatch, calls):
+ _poison(backend, first)
+ assert backend._respawn_if_dead() is True
+ fh = backend._llama_log_fh = _Fh()
+ monkeypatch.setattr(backend, "_clear_server_pid", lambda: calls.append("pid"))
+ monkeypatch.setattr(backend, "_unload_audio_codec", lambda: calls.append("codec"))
+ backend._planned_vram_mib = {0: 9000}
+ _poison(backend, backend._process)
+ return fh
+
+
+def test_the_terminal_strike_unloads_the_backend(harness, monkeypatch):
+ backend, first, _loads = harness(_OFF, "none")
+ calls = []
+ fh = _two_strikes(backend, first, monkeypatch, calls)
+ assert backend._process is None and backend._model_identifier is None
+ assert backend.is_active is False and backend.is_loaded is False
+ assert fh.closed and "pid" in calls and "codec" in calls
+ assert not backend._planned_vram_mib
+ assert backend._respawn_if_dead() is False
+
+
+def test_a_first_strike_still_respawns_with_the_same_settings(harness, monkeypatch):
+ backend, first, loads = harness(_OFF, "none")
+ seen = []
+ monkeypatch.setattr(llama_cpp, "_TERMINAL_RETIREMENT_HOOKS", [seen.append], raising = False)
+ _poison(backend, first)
+ assert backend._model_identifier is not None
+ assert backend._respawn_if_dead() is True
+ assert loads == [_OFF] and seen == []
+
+
+def test_the_terminal_strike_runs_the_cleanup_hooks_once(harness, monkeypatch):
+ backend, first, _loads = harness(_OFF, "none")
+ seen = []
+ monkeypatch.setattr(
+ llama_cpp, "_TERMINAL_RETIREMENT_HOOKS", [lambda *args: seen.append(args)], raising = False
+ )
+ _two_strikes(backend, first, monkeypatch, [])
+ assert seen == [(backend, backend._unload_epoch)]
+
+
+class _Orchestrator:
+ def __init__(self):
+ self.active_model_name = None
+ self.loading_models = ()
+ self.cleaned = False
+
+ def _cleanup(self):
+ self.cleaned = True
+
+
+def _kept_slot(refs = 0, orchestrator = None):
+ return model_slots.ExtraSlot(
+ llama = LlamaCppBackend(),
+ orchestrator = orchestrator or _Orchestrator(),
+ account = "a",
+ refs = refs,
+ )
+
+
+def _routes(monkeypatch):
+ import routes.inference as inf
+
+ assert inf._release_after_terminal_retirement in llama_cpp._TERMINAL_RETIREMENT_HOOKS
+ calls = []
+ monkeypatch.setattr(inf, "release_chat_after_kept_models", lambda: calls.append("kept") or True)
+ monkeypatch.setattr(inf, "release_chat_gpu_claim", lambda: calls.append("claim") or True)
+ monkeypatch.setattr(
+ inf.account_access, "clear_resident", lambda modality: calls.append(f"clear:{modality}")
+ )
+ return inf, calls
+
+
+def _release_primary(inf):
+ inf._release_after_terminal_retirement(
+ inf._llama_cpp_backend, inf._llama_cpp_backend._unload_epoch
+ )
+
+
+def test_a_retired_kept_slot_is_dropped_and_keeps_the_primarys_residency(monkeypatch):
+ """Dead, it holds nothing a request could still use, so a held one goes too; the chat
+ residency is the primary's, which may be a live zero-VRAM model."""
+ inf, calls = _routes(monkeypatch)
+ retired, other = _kept_slot(refs = 1), _kept_slot()
+ monkeypatch.setattr(model_slots, "slots", [retired, other])
+ monkeypatch.setattr(model_slots, "stuck", [])
+ inf._release_after_terminal_retirement(retired.llama, retired.llama._unload_epoch)
+ assert model_slots.slots == [other]
+ assert calls == ["kept"]
+ # Nothing dead is left for a later teardown to count as an unload.
+ monkeypatch.setattr(model_slots, "slots", [])
+ assert model_slots.unload_llama_slots() == 0
+
+
+def test_a_retired_primary_clears_its_own_residency(monkeypatch):
+ inf, calls = _routes(monkeypatch)
+ kept = _kept_slot()
+ monkeypatch.setattr(model_slots, "slots", [kept])
+ _release_primary(inf)
+ assert model_slots.slots == [kept]
+ assert calls == ["clear:chat", "claim"]
+
+
+def _cleanup_after(monkeypatch, inf, land):
+ """The routes cleanup, run after ``land`` (a reload queued on _serial_load_lock)."""
+
+ def hook(*args):
+ land()
+ inf._release_after_terminal_retirement(*args)
+
+ monkeypatch.setattr(llama_cpp, "_TERMINAL_RETIREMENT_HOOKS", [hook])
+
+
+def test_a_reload_before_the_cleanup_keeps_its_kept_slot(harness, monkeypatch):
+ inf, calls = _routes(monkeypatch)
+ backend, first, _loads = harness(_OFF, "none")
+ slot = model_slots.ExtraSlot(llama = backend, orchestrator = _Orchestrator(), account = "a")
+ monkeypatch.setattr(model_slots, "slots", [slot])
+ monkeypatch.setattr(model_slots, "stuck", [])
+ _cleanup_after(monkeypatch, inf, lambda: backend.load_model(_OFF))
+ _two_strikes(backend, first, monkeypatch, [])
+ assert model_slots.slots == [slot]
+ assert backend._process is not None and backend._process.poll() is None
+ assert calls == []
+
+
+def test_a_reload_before_the_cleanup_keeps_the_primarys_residency(harness, monkeypatch):
+ inf, calls = _routes(monkeypatch)
+ backend, first, _loads = harness(_OFF, "none")
+ monkeypatch.setattr(inf, "_llama_cpp_backend", backend)
+ monkeypatch.setattr(model_slots, "slots", [])
+ _cleanup_after(monkeypatch, inf, lambda: backend.load_model(_OFF))
+ _two_strikes(backend, first, monkeypatch, [])
+ assert backend._process is not None and backend._process.poll() is None
+ assert calls == []
+
+
+# A non-GGUF load that hands the GGUF primary off before the terminal cleanup.
+
+_MLX = "mlx-community/Qwen3-4B-4bit"
+
+
+def _standard_load_lands_first(monkeypatch, inf, backend, orchestrator):
+ def land():
+ asyncio.run(inf._unload_llama_before_standard_load(backend))
+ orchestrator.active_model_name = _MLX
+
+ _cleanup_after(monkeypatch, inf, land)
+
+
+def _no_settle(monkeypatch, backend, waits):
+ monkeypatch.setattr(backend, "_wait_for_vram_settle", lambda **kw: waits.append(kw))
+
+
+def test_a_standard_load_before_the_cleanup_keeps_the_primarys_residency(harness, monkeypatch):
+ inf, calls = _routes(monkeypatch)
+ backend, first, _loads = harness(_OFF, "none")
+ monkeypatch.setattr(inf, "_llama_cpp_backend", backend)
+ monkeypatch.setattr(model_slots, "slots", [])
+ std = _Orchestrator()
+ monkeypatch.setattr(inf, "_peek_inference_backend", lambda: std)
+ waits = []
+ _no_settle(monkeypatch, backend, waits)
+ _standard_load_lands_first(monkeypatch, inf, backend, std)
+ _two_strikes(backend, first, monkeypatch, [])
+ assert calls == []
+ # Nothing was running, so nothing to wait for.
+ assert waits == []
+
+
+def test_a_standard_load_before_the_cleanup_keeps_its_kept_slot(harness, monkeypatch):
+ inf, calls = _routes(monkeypatch)
+ backend, first, _loads = harness(_OFF, "none")
+ orchestrator = _Orchestrator()
+ slot = model_slots.ExtraSlot(llama = backend, orchestrator = orchestrator, account = "a")
+ monkeypatch.setattr(model_slots, "slots", [slot])
+ monkeypatch.setattr(model_slots, "stuck", [])
+ _no_settle(monkeypatch, backend, [])
+ _standard_load_lands_first(monkeypatch, inf, backend, orchestrator)
+ _two_strikes(backend, first, monkeypatch, [])
+ assert model_slots.slots == [slot]
+ assert orchestrator.cleaned is False
+ assert calls == []
+
+
+def test_the_standard_handoff_advances_the_epoch_of_an_unloaded_backend(monkeypatch):
+ import routes.inference as inf
+
+ backend = LlamaCppBackend()
+ waits = []
+ _no_settle(monkeypatch, backend, waits)
+ epoch = backend._unload_epoch
+ asyncio.run(inf._unload_llama_before_standard_load(backend))
+ assert backend._unload_epoch == epoch + 1
+ assert waits == []
+
+
+def test_the_standard_handoff_skips_the_wait_with_nothing_running(monkeypatch):
+ import routes.inference as inf
+
+ backend = LlamaCppBackend()
+ waits = []
+ _no_settle(monkeypatch, backend, waits)
+ asyncio.run(inf._unload_llama_before_standard_load(backend))
+ assert waits == []
+
+
+def test_the_standard_handoff_waits_for_a_cleanup_past_its_check(monkeypatch):
+ """A cleanup holding the lock already decided the backend is still retired: the standard
+ load may not publish its residency until that cleanup is done."""
+ import routes.inference as inf
+
+ backend = LlamaCppBackend()
+ _no_settle(monkeypatch, backend, [])
+ epoch = backend._unload_epoch
+ done = threading.Event()
+ with backend._while_still_retired(epoch) as retired:
+ assert retired
+ worker = threading.Thread(
+ target = lambda: (
+ asyncio.run(inf._unload_llama_before_standard_load(backend)),
+ done.set(),
+ )
+ )
+ worker.start()
+ assert not done.wait(0.3)
+ assert backend._unload_epoch == epoch
+ worker.join(10)
+ assert done.is_set() and backend._unload_epoch == epoch + 1
+
+
+ "healthy",
+ [True, False],
+ ids = [
+ "test_the_standard_handoff_still_waits_for_a_loaded_server_to_settle",
+ # Starting or recovering: a process, not loaded.
+ "test_the_standard_handoff_waits_after_killing_an_unhealthy_server",
+ ],
+)
+def test_the_standard_handoff_waits_after_killing_a_server(harness, monkeypatch, healthy):
+ import routes.inference as inf
+
+ backend, _first, _loads = harness(_OFF, "none")
+ backend._healthy = healthy
+ assert backend._process is not None and backend.is_loaded is healthy
+ waits = []
+ _no_settle(monkeypatch, backend, waits)
+ asyncio.run(inf._unload_llama_before_standard_load(backend))
+ assert backend._process is None
+ assert len(waits) == 1
+ if not healthy:
+ assert waits[0]["since_kill"] > 0
+
+
+def test_the_cleanup_leaves_an_occupied_primary_seat_alone(monkeypatch):
+ """No handoff ran (the epoch is still the retirement's): the non-GGUF primary still keeps
+ its sharers, and the CHAT release re-checks that the seat is idle."""
+ inf, calls = _routes(monkeypatch)
+ monkeypatch.setattr(model_slots, "slots", [])
+ for field, value in (
+ ("active_model_name", _MLX),
+ ("loading_models", (_MLX,)),
+ ("_managed_engine", object()),
+ ):
+ std = _Orchestrator()
+ setattr(std, field, value)
+ monkeypatch.setattr(inf, "_peek_inference_backend", lambda std = std: std)
+ calls.clear()
+ _release_primary(inf)
+ assert calls == ["claim"], field
+
+
+def test_the_cleanup_leaves_a_kept_slot_with_a_non_gguf_model_alone(monkeypatch):
+ inf, calls = _routes(monkeypatch)
+ monkeypatch.setattr(model_slots, "stuck", [])
+ for field, value in (("active_model_name", _MLX), ("loading_models", (_MLX,))):
+ orchestrator = _Orchestrator()
+ setattr(orchestrator, field, value)
+ slot = _kept_slot(orchestrator = orchestrator)
+ monkeypatch.setattr(model_slots, "slots", [slot])
+ calls.clear()
+ inf._release_after_terminal_retirement(slot.llama, slot.llama._unload_epoch)
+ assert model_slots.slots == [slot] and orchestrator.cleaned is False, field
+ assert calls == ["kept"]
+
+
+def test_an_idle_primary_seat_still_clears_its_residency(monkeypatch):
+ inf, calls = _routes(monkeypatch)
+ monkeypatch.setattr(model_slots, "slots", [])
+ monkeypatch.setattr(inf, "_peek_inference_backend", lambda: _Orchestrator())
+ _release_primary(inf)
+ assert calls == ["clear:chat", "claim"]
diff --git a/studio/backend/tests/test_metal_fit_params_probe_reasons.py b/studio/backend/tests/test_metal_fit_params_probe_reasons.py
new file mode 100644
index 00000000000..964b6872b75
--- /dev/null
+++ b/studio/backend/tests/test_metal_fit_params_probe_reasons.py
@@ -0,0 +1,141 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
+
+"""llama-fit-params probe: say why a measurement failed, and only cache answers that repeat."""
+
+import sys
+
+import pytest
+
+from core.inference import llama_cpp as lc
+from core.inference.llama_cpp import LlamaCppBackend
+
+_TABLE = """0.00.408.825 I common_memory_breakdown_print: | memory breakdown [MiB] | total free self model context compute unaccounted |
+0.00.429.748 I common_memory_breakdown_print: | - MTL0 (Apple M1) | 5461 = 5460 + (3642 = 3025 + 28 + 589) + -3642 |
+0.00.429.748 I common_memory_breakdown_print: | - Host | 2352 = 2288 + 0 + 64 |
+"""
+
+posix_only = pytest.mark.skipif(
+ sys.platform == "win32", reason = "shell script stands in for the binary"
+)
+
+
+def _install(
+ tmp_path,
+ script,
+ mode = 0o755,
+):
+ bin_dir = tmp_path / "bin"
+ bin_dir.mkdir()
+ (bin_dir / "llama-server").write_text("")
+ model = tmp_path / "model.gguf"
+ model.write_bytes(b"GGUF")
+ probe = bin_dir / "llama-fit-params"
+ if script is not None:
+ probe.write_text(script)
+ probe.chmod(mode)
+ return str(bin_dir / "llama-server"), str(model), probe
+
+
+def _counting(tmp_path, body):
+ calls = tmp_path / "calls"
+ return calls, f'#!/bin/sh\necho x >> "{calls}"\n{body}\n'
+
+
+def _ncalls(calls):
+ return len(calls.read_text(encoding = "utf-8").splitlines()) if calls.exists() else 0
+
+
+def test_missing_probe_is_named(tmp_path):
+ binary, model, _ = _install(tmp_path, None)
+ backend = LlamaCppBackend()
+ assert backend._metal_measured_model_mib(binary, model) is None
+ assert "not found" in backend._metal_probe_failure
+
+
+@posix_only
+def test_non_executable_probe_is_named_and_reprobed_after_repair(tmp_path):
+ table = _TABLE.replace("\n", "\\n")
+ calls, script = _counting(tmp_path, f'printf "{table}"')
+ binary, model, probe = _install(tmp_path, script, mode = 0o644)
+ backend = LlamaCppBackend()
+ assert backend._metal_measured_model_mib(binary, model) is None
+ assert "not executable" in backend._metal_probe_failure
+ assert _ncalls(calls) == 0
+ probe.chmod(0o755) # the installer's exec-bit repair
+ assert backend._metal_measured_model_mib(binary, model) == (3025, 2288, 2)
+ assert backend._metal_probe_failure is None
+
+
+@posix_only
+def test_nonzero_exit_is_named_with_its_output_and_not_cached(tmp_path):
+ calls, script = _counting(tmp_path, 'echo "ggml_metal_init: error: boom" >&2\nexit 3')
+ binary, model, _ = _install(tmp_path, script)
+ backend = LlamaCppBackend()
+ assert backend._metal_measured_model_mib(binary, model) is None
+ assert "exited with code 3" in backend._metal_probe_failure
+ assert "boom" in backend._metal_probe_failure
+ assert backend._metal_measured_model_mib(binary, model) is None
+ assert _ncalls(calls) == 2
+
+
+@posix_only
+def test_timeout_is_named_and_not_cached(tmp_path, monkeypatch):
+ monkeypatch.setattr(lc, "_METAL_FIT_PROBE_TIMEOUT_S", 0.5)
+ calls, script = _counting(tmp_path, "sleep 5")
+ binary, model, _ = _install(tmp_path, script)
+ backend = LlamaCppBackend()
+ assert backend._metal_measured_model_mib(binary, model) is None
+ assert "timed out" in backend._metal_probe_failure
+ assert backend._metal_measured_model_mib(binary, model) is None
+ assert _ncalls(calls) == 2
+
+
+@posix_only
+def test_unparsable_output_is_named_and_cached(tmp_path):
+ calls, script = _counting(tmp_path, 'echo "error: invalid argument: -lv"')
+ binary, model, _ = _install(tmp_path, script)
+ backend = LlamaCppBackend()
+ assert backend._metal_measured_model_mib(binary, model) is None
+ assert "no Metal memory breakdown" in backend._metal_probe_failure
+ assert "invalid argument" in backend._metal_probe_failure
+ assert backend._metal_measured_model_mib(binary, model) is None
+ assert "no Metal memory breakdown" in backend._metal_probe_failure
+ assert _ncalls(calls) == 1
+
+
+@posix_only
+def test_a_valid_table_still_parses_and_caches(tmp_path):
+ table = _TABLE.replace("\n", "\\n")
+ calls, script = _counting(tmp_path, f'printf "{table}"')
+ binary, model, _ = _install(tmp_path, script)
+ backend = LlamaCppBackend()
+ assert backend._metal_measured_model_mib(binary, model) == (3025, 2288, 2)
+ assert backend._metal_measured_model_mib(binary, model) == (3025, 2288, 2)
+ assert backend._metal_probe_failure is None
+ assert _ncalls(calls) == 1
+
+
+def test_the_launch_log_carries_the_reason():
+ """The "stay charged" line names the cause instead of a fixed guess."""
+ import inspect
+
+ src = inspect.getsource(LlamaCppBackend._metal_demand_paged_embedding_bytes)
+ assert "_metal_probe_failure" in src
+ assert "missing or unreadable" not in src
+
+
+@posix_only
+def test_a_failed_probe_runs_once_per_load(tmp_path):
+ calls, script = _counting(tmp_path, "exit 3")
+ binary, model, _ = _install(tmp_path, script)
+ backend = LlamaCppBackend()
+ with lc._load_probe_memo_scope():
+ assert backend._metal_measured_model_mib(binary, model) is None
+ assert backend._metal_measured_model_mib(binary, model) is None
+ assert "exited with code 3" in backend._metal_probe_failure
+ assert _ncalls(calls) == 1
+ # The next load probes again: a cold shader compile may have finished by then.
+ with lc._load_probe_memo_scope():
+ assert backend._metal_measured_model_mib(binary, model) is None
+ assert _ncalls(calls) == 2
diff --git a/studio/backend/tests/test_spec_sidecar_fit_reserve.py b/studio/backend/tests/test_spec_sidecar_fit_reserve.py
new file mode 100644
index 00000000000..b9c114f0e74
--- /dev/null
+++ b/studio/backend/tests/test_spec_sidecar_fit_reserve.py
@@ -0,0 +1,1064 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
+
+"""Single-card --fit-target reserve for DSpark/DFlash sidecars llama.cpp's fitter cannot size."""
+
+from __future__ import annotations
+
+import inspect
+import math
+import os
+import pathlib
+from pathlib import Path
+
+import pytest
+
+from . import _sidecar_fit_helpers as h
+from ._sidecar_fit_helpers import FLOOR, GIB, MIB, lc
+
+B = lc.LlamaCppBackend
+
+
[email protected](autouse = True)
+def _studio_home(tmp_path, monkeypatch):
+ monkeypatch.setenv("UNSLOTH_STUDIO_HOME", str(tmp_path / "studio_home"))
+ monkeypatch.delenv("LLAMA_ARG_DEVICE", raising = False)
+ monkeypatch.delenv("LLAMA_ARG_SPLIT_MODE", raising = False)
+ monkeypatch.setattr(lc, "_metal_capable_host", lambda: False)
+ monkeypatch.setattr(B, "_apple_metal_memory_budget_bytes", staticmethod(lambda: 0))
+
+
+def _launch_kind(
+ tmp_path,
+ kind = "dspark",
+ use_fit = True,
+ extra_args = None,
+):
+ backend, gguf = h.cuda_backend(tmp_path, draft_key = kind)
+ backend._select_gpus = lambda *a, **k: (None, True) if use_fit else ([0], False)
+ path = h.sidecar(tmp_path, kind)
+ cmd = h.placement._launch(
+ backend,
+ gguf,
+ speculative_type = kind,
+ extra_args = extra_args,
+ **{f"{kind}_draft_path": path},
+ )["cmd"]
+ return cmd, path
+
+
+class TestIntegrityFlags:
+ def test_reserve_rides_on_top_of_the_auto_fit_margin(self):
+ base = B._ctx_integrity_flags(1, True, True, 0, 0, h.CAPS)
+ flags = B._ctx_integrity_flags(1, True, True, 0, 0, h.CAPS, fit_target_reserve_mib = 7630)
+ assert int(h.last(flags, "--fit-target")) == int(h.last(base, "--fit-target")) + 7630
+
+ def test_reserve_rides_on_top_of_the_metal_floor(self):
+ flags = B._ctx_integrity_flags(
+ 1, True, False, 0, 0, h.CAPS, fit_target_floor_mib = 6464, fit_target_reserve_mib = 7630
+ )
+ assert int(h.last(flags, "--fit-target")) == 6464 + 7630
+
+ def test_reserve_makes_the_legacy_path_speak(self):
+ # The legacy path is silent at llama.cpp's own default; a reserve is not that.
+ assert "--fit-target" not in B._ctx_integrity_flags(1, True, False, 0, 0, h.CAPS)
+ flags = B._ctx_integrity_flags(1, True, False, 0, 0, h.CAPS, fit_target_reserve_mib = 100)
+ assert int(h.last(flags, "--fit-target")) == 1024 + 100
+
+ @pytest.mark.parametrize("use_fit, caps", [(False, h.CAPS), (True, {})])
+ def test_no_fit_or_no_flag_emits_nothing(self, use_fit, caps):
+ assert "--fit-target" not in B._ctx_integrity_flags(
+ 1, use_fit, False, 0, 0, caps, fit_target_reserve_mib = 100
+ )
+
+
+class TestReserveArithmetic:
+ def _backend(self, file_bytes):
+ backend = B()
+ backend._get_gguf_size_bytes = lambda _path: file_bytes
+ return backend
+
+ def test_planner_reserve_wins_when_larger(self):
+ assert self._backend(8 * GIB)._sidecar_fit_reserve_mib(9 * GIB, "/d.gguf") == 9 * 1024
+
+ def test_file_size_floors_a_flat_fallback(self):
+ assert self._backend(8 * GIB)._sidecar_fit_reserve_mib(0, "/d.gguf") == 8 * 1024
+
+ def test_rounds_up(self):
+ assert self._backend(0)._sidecar_fit_reserve_mib(MIB + 1, "/d.gguf") == 2
+
+ def test_nothing_known_is_zero(self):
+ assert self._backend(0)._sidecar_fit_reserve_mib(0, None) == 0
+
+
+def test_an_unsized_sidecar_carries_the_flat_reserve():
+ b = B.__new__(B)
+ b._get_gguf_size_bytes = lambda path: None
+ assert b._sidecar_fit_reserve_mib(0, "o/remote-dspark", flat_mib = 1200) == 1200
+ assert b._sidecar_fit_reserve_mib(2 * GIB, "o/remote-dspark", flat_mib = 1200) == 2048
+
+
+class TestDiscreteLaunch:
+ @pytest.mark.parametrize("kind", ["dspark", "dflash"])
+ def test_fit_on_keeps_the_sidecar_free(self, tmp_path, kind):
+ cmd, path = _launch_kind(tmp_path, kind = kind)
+ assert h.last(cmd, "--model-draft") == path
+ assert h.margin(cmd) >= FLOOR
+
+ def test_fit_off_has_no_fitter_to_tell(self, tmp_path):
+ cmd, path = _launch_kind(tmp_path, use_fit = False)
+ assert h.last(cmd, "--model-draft") == path
+ assert "--fit-target" not in cmd
+
+ def test_a_cpu_pinned_drafter_reserves_nothing(self, tmp_path):
+ cmd, path = _launch_kind(tmp_path, extra_args = ["--spec-draft-ngl", "0"])
+ assert h.last(cmd, "--model-draft") == path
+ assert h.margin(cmd) < FLOOR
+
+
+def test_the_integrity_margin_never_carries_the_reserve():
+ """The reserve rides only on the trailing pair the spawn appends, so it can neither make
+ an unpriced child look budget-priced nor be mistaken for a user's equal margin."""
+ src = Path(lc.__file__).read_text(encoding = "utf-8")
+ assert "fit_target_reserve_mib = _sidecar_fit_reserve_mib" not in src
+ assert "fit_target_reserve_mib = 0," in src
+
+
[email protected]("spec_type", ["draft-dspark", "draft-dflash"])
+def test_a_pass_through_sidecar_is_reserved_too(tmp_path, spec_type):
+ """Extras owning --spec-type replace Studio's emission, so the reserve follows the
+ sidecar the child really loads, even with Studio's own speculation off."""
+ backend, gguf = h.cuda_backend(tmp_path, draft_key = "sidecar")
+ path = tmp_path / "custom-sidecar-Q8_0.gguf"
+ path.write_bytes(b"draft")
+ cmd = h.placement._launch(
+ backend,
+ gguf,
+ speculative_type = "off",
+ extra_args = ["--spec-type", spec_type, "--model-draft", str(path)],
+ )["cmd"]
+ assert str(path) in cmd
+ assert h.margin(cmd) >= FLOOR
+
+
+# _sidecar_reserve_margin_flags: the spawn's one question of the argv + env it launches.
+
+
+ "args, env",
+ [
+ (["--model-draft", "/d.gguf"], {}),
+ (["-md", "/d.gguf"], {}),
+ (["--spec-draft-model", "/d.gguf"], {}),
+ (["--model-draft=/d.gguf"], {}),
+ ([], {"LLAMA_ARG_SPEC_DRAFT_MODEL": "/d.gguf"}),
+ ],
+ ids = ["long", "short", "spec-draft-model", "equals", "env"],
+)
+def test_every_draft_spelling_gets_the_reserve(args, env):
+ flags = lc._sidecar_reserve_margin_flags(["--fit", "on", *args], (), env, 1024, 8192)
+ assert flags == ["--fit-target", str(1024 + 8192)]
+
+
+ "args, env",
+ [(["--fit", "off"], {}), ([], {"LLAMA_ARG_FIT": "off"}), (["-fit", "off"], {})],
+ ids = ["argv", "env", "short"],
+)
+def test_no_reserve_when_the_fitter_is_off(args, env):
+ assert lc._sidecar_reserve_margin_flags(["-md", "/d.gguf", *args], (), env, 1024, 8192) == []
+
+
+def test_no_reserve_without_a_drafter():
+ assert lc._sidecar_reserve_margin_flags(["--fit", "on"], (), {}, 1024, 8192) == []
+
+
+ "user_args, env, want",
+ [
+ ((), {"LLAMA_ARG_FIT_TARGET": "12000"}, 12000 + 8192),
+ (("--fit-target", "16384"), {}, 16384 + 8192),
+ # A single-device launch keeps device 0's entry, not the list maximum.
+ (("-fitt", "512/20000"), {}, 1024 + 8192),
+ ],
+ ids = ["env", "argv", "per-device-list"],
+)
+def test_the_reserve_sits_on_top_of_the_users_margin(user_args, env, want):
+ flags = lc._sidecar_reserve_margin_flags(
+ ["-md", "/d.gguf", *user_args], user_args, env, 1024, 8192
+ )
+ assert flags == ["--fit-target", str(want)]
+
+
[email protected]("value", ["1024,65536", "1024/65536"], ids = ["comma", "slash"])
+def test_single_device_reserve_uses_the_first_list_entry(value):
+ flags = lc._sidecar_reserve_margin_flags(
+ ["--fit", "on", "-md", "/d.gguf", "--fit-target", value],
+ ("--fit-target", value),
+ {},
+ 1024,
+ 8192,
+ )
+ assert flags == ["--fit-target", str(1024 + 8192)]
+
+
+ "extras",
+ [["--fit-target", "1024"], ["-fitt", "1024"], ["--fit-target=1024"]],
+ ids = ["spaced", "short", "equals"],
+)
+def test_a_user_fit_target_keeps_the_reserve(tmp_path, extras):
+ backend, gguf = h.cuda_backend(tmp_path)
+ cmd = h.dspark(backend, gguf, tmp_path, extra_args = extras)
+ values = [
+ cmd[i + 1] if tok in ("--fit-target", "-fitt") else tok.split("=", 1)[1]
+ for i, tok in enumerate(cmd)
+ if tok in ("--fit-target", "-fitt") or tok.startswith("--fit-target=")
+ ]
+ assert int(values[-1]) >= FLOOR
+
+
+ "extras",
+ [["--fit", "on"], ["--fit=on"], ["-fit", "on"]],
+ ids = ["spaced", "equals", "short"],
+)
+def test_extras_that_turn_the_fitter_on_get_the_reserve(tmp_path, extras):
+ backend, gguf = h.cuda_backend(tmp_path)
+ backend._select_gpus = lambda *a, **k: ([0], False)
+ assert h.margin(h.dspark(backend, gguf, tmp_path, extra_args = extras)) >= FLOOR
+
+
+def test_an_inherited_env_margin_is_kept_under_the_reserve(tmp_path, monkeypatch):
+ monkeypatch.setenv("LLAMA_ARG_FIT_TARGET", "12000")
+ backend, gguf = h.cuda_backend(tmp_path)
+ assert h.margin(h.dspark(backend, gguf, tmp_path)) >= 12000 + 8 * 1024
+
+
+def test_the_reserve_carries_the_draft_compute_floor(tmp_path):
+ backend, gguf = h.cuda_backend(tmp_path)
+ assert h.margin(h.dspark(backend, gguf, tmp_path)) >= FLOOR + 224
+
+
+def test_full_draft_offload_still_gets_the_reserve(tmp_path):
+ backend, gguf = h.cuda_backend(tmp_path)
+ assert h.margin(h.dspark(backend, gguf, tmp_path)) >= FLOOR
+
+
+def test_manual_auto_context_prices_the_native_context(tmp_path):
+ backend, gguf = h.cuda_backend(tmp_path)
+ backend._context_length = 131072
+ h.per_token_estimate(backend, 64 * 1024) # 8 GiB of draft KV at 131072
+ cmd = h.dspark(backend, gguf, tmp_path, n_ctx = 0, **h.MANUAL_AUTO)
+ if "--fit-target" not in cmd:
+ pytest.skip("this host plans no fitter for Manual + Auto")
+ assert h.margin(cmd) >= 512 + 16 * 1024
+
+
+def test_explicit_manual_context_is_priced_at_that_context(tmp_path):
+ backend, gguf = h.cuda_backend(tmp_path)
+ backend._context_length = 131072
+ h.per_token_estimate(backend, 64 * 1024)
+ cmd = h.dspark(backend, gguf, tmp_path, n_ctx = 4096, **h.MANUAL_AUTO)
+ if "--fit-target" not in cmd:
+ pytest.skip("this host plans no fitter for Manual + Auto")
+ assert h.last(cmd, "-c") == "4096"
+ assert h.margin(cmd) < 512 + 12 * 1024
+
+
+def test_manual_auto_context_prices_an_inherited_context(tmp_path, monkeypatch):
+ monkeypatch.setenv("LLAMA_ARG_CTX_SIZE", "8192")
+ backend, gguf = h.cuda_backend(tmp_path)
+ backend._context_length = 131072
+ h.per_token_estimate(backend, 64 * 1024) # 0.5 GiB of draft KV at 8192
+ cmd = h.dspark(backend, gguf, tmp_path, n_ctx = 0, **h.MANUAL_AUTO)
+ assert "--fit-target" in cmd
+ assert h.margin(cmd) < 512 + 12 * 1024
+
+
+def test_an_automatic_context_is_priced_at_the_final_slots(tmp_path):
+ backend, gguf = h.cuda_backend(tmp_path)
+ backend._context_length = 32768
+ h.per_token_estimate(backend, 32 * 1024) # 1 GiB at 32768, 8 GiB at 8 x 32768
+ cmd = h.dspark(
+ backend,
+ gguf,
+ tmp_path,
+ n_ctx = 0,
+ n_parallel = 1,
+ extra_args = ("--parallel", "8", "--no-kv-unified"),
+ **h.MANUAL_AUTO,
+ )
+ assert "--fit-target" in cmd
+ assert h.margin(cmd) >= 512 + 16 * 1024
+
+
+def test_a_trailing_zero_context_is_priced_at_the_native_length(tmp_path):
+ backend, gguf = h.cuda_backend(tmp_path)
+ backend._context_length = 131072
+ h.per_token_estimate(backend, 64 * 1024)
+ cmd = h.dspark(
+ backend, gguf, tmp_path, gpu_memory_mode = "auto", n_ctx = 4096, extra_args = ["-c", "0"]
+ )
+ assert h.last(cmd, "-c") == "0", cmd
+ assert h.margin(cmd) >= 16 * 1024, cmd
+
+
+def test_an_unread_native_length_prices_the_fitters_minimum_context():
+ assert lc._sidecar_child_ctx([], {}, None, 1, True) == lc._LLAMA_FIT_MIN_CTX
+ assert lc._sidecar_child_ctx([], {}, None, 4, False) == 4 * lc._LLAMA_FIT_MIN_CTX
+ assert lc._sidecar_child_ctx(["-c", "8192"], {}, None, 4, False) == 8192
+
+
+ "args, env, want",
+ [
+ # No -c: the pool is n_parallel * per-slot limit (tools/server/server.cpp).
+ (["--kv-unified-per-slot", "2048"], {}, 4 * 2048),
+ (["--kv-unified-per-slot=65536"], {}, 4 * 65536),
+ ([], {"LLAMA_ARG_KV_UNIFIED_PER_SLOT": "2048"}, 4 * 2048),
+ (["-c", "16384", "--kv-unified-per-slot", "2048"], {}, 16384),
+ # -c 0 asks for the full window; a later --fit-ctx clears that.
+ (["-c", "0", "--kv-unified-per-slot", "2048"], {}, 131072),
+ (["-c", "0", "--fit-ctx", "8192", "--kv-unified-per-slot", "2048"], {}, 4 * 2048),
+ ([], {}, 131072),
+ ],
+)
+def test_the_per_slot_limit_sizes_the_priced_context(args, env, want):
+ assert lc._sidecar_child_ctx(args, env, 131072, 4, True) == want
+
+
+ "extras",
+ [("--parallel", "8"), ("-np", "8"), ("--parallel=8",)],
+ ids = ["long", "short", "equals"],
+)
+def test_pass_through_slots_are_priced_on_the_first_spawn(tmp_path, extras):
+ backend, gguf = h.cuda_backend(tmp_path)
+ h.slot_priced(backend)
+ cmd = h.dspark(backend, gguf, tmp_path, n_ctx = 4096, n_parallel = 1, extra_args = extras)
+ assert h.margin(cmd) >= FLOOR + 8 * 1024
+
+
+def test_fewer_final_slots_lower_the_reserve(tmp_path):
+ backend, gguf = h.cuda_backend(tmp_path)
+ h.slot_priced(backend)
+ cmd = h.dspark(
+ backend, gguf, tmp_path, n_ctx = 4096, n_parallel = 8, extra_args = ("--parallel", "1")
+ )
+ assert FLOOR + 1024 <= h.margin(cmd) < FLOOR + 2 * 1024
+ assert cmd.count("--fit-target") == 1
+
+
+def _ubatch_priced(backend):
+ backend._estimate_mtp_overhead_bytes = lambda n_ctx, n_ubatch = None, **kw: (
+ 8 * GIB + int(n_ubatch or 512) * MIB
+ )
+
+
+ "extras, env",
+ [
+ (("--batch-size", "4096", "--spec-draft-n-ubatch", "4096"), {}),
+ ((), {"LLAMA_ARG_BATCH": "4096", "LLAMA_ARG_SPEC_DRAFT_N_UBATCH": "4096"}),
+ ],
+ ids = ["flag", "env"],
+)
+def test_the_draft_micro_batch_is_priced(tmp_path, monkeypatch, extras, env):
+ for key, value in env.items():
+ monkeypatch.setenv(key, value)
+ backend, gguf = h.cuda_backend(tmp_path)
+ _ubatch_priced(backend)
+ cmd = h.dspark(backend, gguf, tmp_path, n_ctx = 4096, extra_args = extras)
+ assert h.margin(cmd) >= FLOOR + 4096
+
+
+def test_the_draft_micro_batch_is_capped_at_the_batch(tmp_path):
+ """llama_context caps n_ubatch at n_batch; the draft context inherits the target's 2048."""
+ backend, gguf = h.cuda_backend(tmp_path)
+ _ubatch_priced(backend)
+ cmd = h.dspark(
+ backend, gguf, tmp_path, n_ctx = 4096, extra_args = ["--spec-draft-n-ubatch", "8192"]
+ )
+ assert FLOOR + 2048 <= h.margin(cmd) < FLOOR + 4096
+
+
+def test_a_zero_draft_micro_batch_follows_the_target():
+ assert lc._draft_n_ubatch(["--spec-draft-n-ubatch", "0"], {}) is None
+ assert lc._draft_n_ubatch(["--spec-draft-n-ubatch=256"], {}) == 256
+
+
+def test_underscore_spellings_are_read_like_llama_cpp():
+ assert lc._draft_card_share_unknown(["--spec_draft_n_cpu_moe", "8"], {})
+ assert lc._draft_n_ubatch(["--spec_draft_n_ubatch", "2048"], {}) == 2048
+
+
+def _dsv41_dspark_pair():
+ """DeepSeek-V4.1 target and its DSpark drafter, dims read off the published GGUFs."""
+ target = B.__new__(B)
+ target._embedding_length, target._vocab_size = 5120, 129280
+ draft = B.__new__(B)
+ for k, v in dict(
+ _architecture = "dflash",
+ _embedding_length = 5120,
+ _n_layers = 3,
+ _n_heads = 64,
+ _n_kv_heads = 1,
+ _kv_key_length = 512,
+ _kv_value_length = 512,
+ _feed_forward_length = 2304,
+ _expert_feed_forward_length = 2304,
+ _vocab_size = 129280,
+ ).items():
+ setattr(draft, k, v)
+ target._draft_backend_for = lambda path: draft
+ return target
+
+
[email protected]("ub, measured_mib", [(512, 725.5), (2048, 2900.5)])
+def test_a_dspark_prompt_graph_is_priced_with_the_micro_batch(ub, measured_mib):
+ """llama.cpp b11734 reserves 725.5 MiB at -ub 512 and 2900.5 MiB at 2048 for this
+ drafter's compute; the old flat 349 MiB let the reserve fall 2.2 GiB short."""
+ b = _dsv41_dspark_pair()
+ got = (
+ b._mtp_draft_compute_bytes(8192, drafter_path = "/d.gguf", n_ubatch = ub)
+ + B._MTP_DRAFT_COMPUTE_BYTES
+ ) / MIB
+ assert measured_mib <= got <= measured_mib * 1.25
+
+
+_ROLLBACK = 4 * GIB
+
+
+def _rollback_backend(tmp_path):
+ backend, gguf = h.cuda_backend(tmp_path)
+ backend._rollback_state_bytes = lambda n_parallel = 1: _ROLLBACK
+ backend._estimate_mtp_overhead_bytes = (
+ lambda n_ctx, target_rollback = True, spec_draft_n_max = 0, **kw: (
+ 8 * GIB + (_ROLLBACK * spec_draft_n_max if target_rollback else 0)
+ )
+ )
+ return backend, gguf
+
+
+def test_initial_reserve_leaves_target_rollback_to_the_fitter(tmp_path):
+ backend, gguf = _rollback_backend(tmp_path)
+ assert FLOOR <= h.margin(h.dspark(backend, gguf, tmp_path)) < FLOOR + _ROLLBACK // MIB
+
+
+ "extras", [("-ngld", "1"), ("--spec-draft-ngl", "1")], ids = ["short", "long"]
+)
+def test_partial_draft_offload_gets_no_whole_file_reserve(tmp_path, extras):
+ backend, gguf = h.cuda_backend(tmp_path)
+ assert h.margin(h.dspark(backend, gguf, tmp_path, extra_args = extras)) < FLOOR
+
+
+ "extras",
+ [
+ ("-otd", ".*=CPU"),
+ ("--override-tensor-draft", ".*=CPU"),
+ ("--spec-draft-override-tensor", ".*=CPU"),
+ ("--spec-draft-override-tensor=.*=CPU",),
+ ],
+ ids = ["otd", "override-tensor-draft", "spec-draft-override-tensor", "equals"],
+)
+def test_a_draft_tensor_override_gets_no_whole_file_reserve(tmp_path, extras):
+ backend, gguf = h.cuda_backend(tmp_path)
+ cmd = h.dspark(backend, gguf, tmp_path, extra_args = extras)
+ assert h.margin(cmd) < FLOOR
+ assert extras[-1] in cmd
+
+
+def test_a_target_tensor_override_keeps_the_reserve(tmp_path):
+ backend, gguf = h.cuda_backend(tmp_path)
+ cmd = h.dspark(backend, gguf, tmp_path, extra_args = ("-ot", "blk\\.0\\.ffn_up=CPU"))
+ assert h.margin(cmd) >= FLOOR
+
+
+ "extras",
+ [
+ ("--spec-draft-cpu-moe",),
+ ("-cmoed",),
+ ("--spec-draft-n-cpu-moe", "99"),
+ ("--n-cpu-moe-draft=4",),
+ ],
+ ids = ["cpu-moe", "short", "n-cpu-moe", "n-equals"],
+)
+def test_draft_cpu_moe_gets_no_whole_file_reserve(tmp_path, extras):
+ backend, gguf = h.cuda_backend(tmp_path)
+ assert h.margin(h.dspark(backend, gguf, tmp_path, extra_args = extras)) < FLOOR
+
+
+ "env",
+ [{"LLAMA_ARG_SPEC_DRAFT_CPU_MOE": "1"}, {"LLAMA_ARG_SPEC_DRAFT_N_CPU_MOE": "8"}],
+ ids = ["cpu-moe", "n-cpu-moe"],
+)
+def test_draft_cpu_moe_from_the_env_is_a_placement_too(env):
+ assert lc._draft_card_share_unknown([], env)
+ assert not lc._draft_card_share_unknown([], {"LLAMA_ARG_SPEC_DRAFT_N_CPU_MOE": "0"})
+
+
+def test_a_positive_draft_cpu_moe_count_is_not_undone_by_a_later_zero():
+ """llama.cpp appends the draft CPU overrides per positive count; a later 0 adds none
+ but removes none either."""
+ assert lc._draft_card_share_unknown(["--spec-draft-n-cpu-moe", "8", "-ncmoed", "0"], {})
+ assert not lc._draft_card_share_unknown(["-ncmoed", "0"], {})
+
+
[email protected]("extras", [("-ngld", "1"), ("-otd", "blk\\.0\\.=CPU")])
+def test_a_partly_placed_drafter_keeps_its_compute_floor(tmp_path, extras):
+ backend, gguf = h.cuda_backend(tmp_path)
+ floor = math.ceil(B._MTP_DRAFT_COMPUTE_BYTES / MIB)
+ assert (
+ 1024 + floor <= h.margin(h.dspark(backend, gguf, tmp_path, extra_args = list(extras))) < FLOOR
+ )
+
+
+ "extras, env",
+ [(["-nkvo"], {}), (["--no-kv-offload"], {}), ([], {"LLAMA_ARG_NO_KV_OFFLOAD": "1"})],
+)
+def test_host_resident_draft_kv_is_not_reserved_on_the_card(tmp_path, monkeypatch, extras, env):
+ """The draft context copies the target's params (common_base_params_to_speculative), so
+ -nkvo puts the draft KV in host buffers too."""
+ for d in ("on", "off"):
+ (tmp_path / d).mkdir()
+
+ def margin_of(d, extras):
+ backend, gguf = h.cuda_backend(d)
+ backend._mtp_draft_kv_bytes = lambda *a, **k: h.DRAFT_KV
+ return h.margin(h.dspark(backend, gguf, d, extra_args = list(extras)))
+
+ on = margin_of(tmp_path / "on", [])
+ for key, value in env.items():
+ monkeypatch.setenv(key, value)
+ assert on - margin_of(tmp_path / "off", extras) >= 760
+
+
[email protected]("device", ["none", "cpu"])
+def test_a_cpu_child_gets_no_reserve_and_no_report(tmp_path, monkeypatch, device):
+ monkeypatch.setenv("CUDA_DEVICE_ORDER", "PCI_BUS_ID")
+ messages = h.info_messages(monkeypatch)
+ backend, gguf = h.cuda_backend(tmp_path)
+ cmd = h.dspark(backend, gguf, tmp_path, extra_args = ["--device", device])
+ assert h.margin(cmd) < 1024 + 4 * 1024
+ assert not any("cannot size the sidecar" in m for m in messages)
+
+
+def test_a_draft_with_its_own_embeddings_and_head_gets_no_reserve(tmp_path, monkeypatch):
+ """llama_context needs the target's context only when the draft lacks token_embd or output."""
+ monkeypatch.setattr(lc, "_sidecar_borrows_target_tensors", lambda path: False)
+ backend, gguf = h.cuda_backend(tmp_path)
+ assert h.margin(h.dspark(backend, gguf, tmp_path)) < FLOOR
+
+
+def _draft_with(monkeypatch, tmp_path, tensors):
+ import utils.models.gguf_metadata as gm
+ import utils.models.model_config as mc
+
+ path = tmp_path / "draft.gguf"
+ path.write_bytes(b"GGUF")
+ monkeypatch.setattr(mc, "colocated_split_shards", lambda p: ([p], True))
+ monkeypatch.setattr(gm, "_gguf_shard_has_named_tensor", lambda shard, name: name in tensors)
+ return str(path)
+
+
+ "present, borrows",
+ [
+ ({"token_embd.weight", "output.weight"}, False),
+ ({"token_embd.weight"}, True),
+ ({"output.weight"}, True),
+ (set(), True),
+ ],
+)
+def test_borrowed_tensor_detection(tmp_path, monkeypatch, present, borrows):
+ assert (
+ lc._sidecar_borrows_target_tensors(_draft_with(monkeypatch, tmp_path, present)) is borrows
+ )
+ assert lc._sidecar_borrows_target_tensors(None) is True
+
+
+ "tensors, borrows, borrows_output",
+ [
+ # Upstream DFlash borrows the target's head when a draft ships none, and no marker
+ # tells a build that ties it to the draft's embeddings apart.
+ ({"token_embd.weight"}, True, True),
+ ({"output.weight"}, True, False),
+ (set(), True, True),
+ ({"token_embd.weight", "output.weight"}, False, False),
+ ],
+)
+def test_a_draft_missing_a_tensor_borrows_it(
+ monkeypatch, tmp_path, tensors, borrows, borrows_output
+):
+ path = _draft_with(monkeypatch, tmp_path, tensors)
+ assert lc._sidecar_borrows_target_tensors(path) is borrows
+ assert lc._sidecar_borrows_output(path) is borrows_output
+
+
+def test_an_unread_draft_borrows_the_output():
+ assert lc._sidecar_borrows_output(None) is True
+
+
+def test_native_measurement_is_read_from_the_shared_common_library(tmp_path):
+ binary = h.elf(tmp_path / "llama-server", b"libllama-common.so.0")
+ real = h.elf(tmp_path / "libllama-common.so.0.0.1", b"..." + h.NATIVE_MARKER + b"...")
+ (tmp_path / "libllama-common.so.0").symlink_to(real.name)
+ (tmp_path / "libllama-common.so").symlink_to("libllama-common.so.0")
+ assert lc._fit_measures_borrowed_drafts(str(binary))
+
+
+def test_native_measurement_is_read_from_a_static_server(tmp_path):
+ assert lc._fit_measures_borrowed_drafts(str(h.elf(tmp_path / "llama-server", h.NATIVE_MARKER)))
+
+
+def test_native_measurement_is_read_through_the_installer_wrapper(tmp_path):
+ h.elf(tmp_path / "llama-server-real", h.NATIVE_MARKER)
+ wrapper = tmp_path / "llama-server"
+ wrapper.write_text('#!/bin/sh\nexec "$(dirname "$0")/llama-server-real" "$@"\n')
+ wrapper.chmod(0o755)
+ assert lc._fit_measures_borrowed_drafts(str(wrapper))
+
+
+def test_an_older_build_does_not_claim_native_measurement(tmp_path):
+ binary = h.elf(tmp_path / "llama-server")
+ h.elf(
+ tmp_path / "libllama-common.so",
+ b"failed to measure the memory of the extra model, fitting without it",
+ )
+ assert not lc._fit_measures_borrowed_drafts(str(binary))
+ assert not lc._fit_measures_borrowed_drafts(None)
+
+
+def test_only_the_linked_common_library_speaks_for_the_build(tmp_path):
+ binary = h.elf(tmp_path / "llama-server", b"libllama-common.so.0")
+ h.elf(tmp_path / "libllama-common.so.0")
+ h.elf(tmp_path / "libllama-common.so.0.9.9", h.NATIVE_MARKER)
+ assert not lc._fit_measures_borrowed_drafts(str(binary))
+ h.elf(tmp_path / "libllama-common.so.0", h.NATIVE_MARKER)
+ assert lc._fit_measures_borrowed_drafts(str(binary))
+
+
+def test_a_static_server_beside_a_stale_common_library_keeps_the_reserve(tmp_path):
+ binary = h.elf(tmp_path / "llama-server")
+ h.elf(tmp_path / "libllama-common.so.0", h.NATIVE_MARKER)
+ assert not lc._fit_measures_borrowed_drafts(str(binary))
+
+
+def test_the_common_library_is_followed_through_the_server_impl_library(tmp_path):
+ """b11734 layout: llama-server links libllama-server-impl.so, which links
+ libllama-common.so.0, where fit.cpp lives."""
+ binary = h.elf(tmp_path / "llama-server", b"libllama-server-impl.so")
+ h.elf(tmp_path / "libllama-server-impl.so", b"libllama-common.so.0")
+ h.elf(tmp_path / "libllama-common.so.0", h.NATIVE_MARKER)
+ assert lc._fit_measures_borrowed_drafts(str(binary))
+ h.elf(tmp_path / "libllama-server-impl.so")
+ assert not lc._fit_measures_borrowed_drafts(str(binary))
+
+
+ "linked", ["libllama-common.so.0", "libllama-common.0.dylib", "llama-common.dll"]
+)
+def test_a_server_that_names_its_common_library_is_read_through_it(tmp_path, linked):
+ binary = h.elf(tmp_path / "llama-server", linked.encode())
+ h.elf(tmp_path / linked, h.NATIVE_MARKER)
+ assert lc._fit_measures_borrowed_drafts(str(binary))
+
+
+def test_the_linked_library_names_are_read_once_per_binary_version(tmp_path, monkeypatch):
+ server = tmp_path / "llama-server"
+ server.write_bytes(b"\0libllama-common.so.0\0" * 4)
+ lib = tmp_path / "libllama-common.so.0"
+ lib.write_bytes(b"lib")
+ reads = []
+ real = pathlib.Path.read_bytes
+
+ def read_bytes(self):
+ reads.append(self.name)
+ return real(self)
+
+ monkeypatch.setattr(pathlib.Path, "read_bytes", read_bytes)
+ for _ in range(3):
+ assert lc._linked_llama_libs(server, tmp_path) == [lib]
+ assert reads.count("llama-server") == 1
+ # Existence is not cached.
+ lib.unlink()
+ assert lc._linked_llama_libs(server, tmp_path) == []
+ assert reads.count("llama-server") == 1
+ lib.write_bytes(b"lib")
+ server.write_bytes(b"\0libllama-server-impl.so\0")
+ st = server.stat()
+ os.utime(server, ns = (st.st_atime_ns, st.st_mtime_ns + 1_000_000_000))
+ (tmp_path / "libllama-server-impl.so").write_bytes(b"impl")
+ assert lc._linked_llama_libs(server, tmp_path) == [tmp_path / "libllama-server-impl.so"]
+ assert reads.count("llama-server") == 2
+
+
+def test_a_missing_binary_fails_closed(tmp_path):
+ caps = B.probe_server_capabilities(str(tmp_path / "missing"))
+ assert caps["fit_measures_borrowed_drafts"] is False
+
+
+def test_a_cached_probe_rereads_the_shared_library(tmp_path, monkeypatch):
+ """The capability cache is keyed on the server alone; a swapped libllama-common
+ must not keep an older answer."""
+ binary = h.elf(tmp_path / "llama-server", b"libllama-common.so")
+ st = binary.stat()
+ key = (str(binary), st.st_mtime_ns, st.st_size)
+ monkeypatch.setattr(B, "_exec_path_for_launch", staticmethod(lambda b: b))
+ monkeypatch.setitem(
+ B._capability_cache,
+ key,
+ {"found": True, "supports_fit_target": True, "fit_measures_borrowed_drafts": True},
+ )
+ assert B.probe_server_capabilities(str(binary))["fit_measures_borrowed_drafts"] is False
+ h.elf(tmp_path / "libllama-common.so", h.NATIVE_MARKER)
+ assert B.probe_server_capabilities(str(binary))["fit_measures_borrowed_drafts"] is True
+
+
[email protected]("native", [False, True], ids = ["older", "native"])
+def test_the_reserve_stands_down_when_the_fitter_measures_the_draft(tmp_path, native):
+ backend, gguf = h.cuda_backend(tmp_path)
+ backend.probe_server_capabilities = lambda _binary = None: {
+ **h.CAPS,
+ "fit_measures_borrowed_drafts": native,
+ }
+ cmd = h.dspark(backend, gguf, tmp_path)
+ assert "--model-draft" in cmd
+ assert (h.margin(cmd) >= FLOOR) is not native
+
+
+def test_a_default_slider_single_card_reserve_is_logged(tmp_path, monkeypatch):
+ backend, gguf = h.cuda_backend(tmp_path)
+ messages = h.info_messages(monkeypatch)
+ assert h.margin(h.dspark(backend, gguf, tmp_path)) >= FLOOR
+ assert any("llama.cpp cannot size the sidecar, so" in m for m in messages)
+
+
+def _warned(
+ tmp_path,
+ monkeypatch,
+ kind,
+ fit_target = True,
+):
+ messages = h.info_messages(monkeypatch)
+ backend, gguf = h.cuda_backend(tmp_path, draft_key = kind)
+ caps = dict(h.CAPS, supports_fit_target = fit_target)
+ backend.probe_server_capabilities = lambda _binary = None: dict(caps)
+ path = h.sidecar(tmp_path, kind)
+ cmd = h.placement._launch(backend, gguf, speculative_type = kind, **{f"{kind}_draft_path": path})[
+ "cmd"
+ ]
+ assert f"draft-{kind}" in cmd
+ return any("is not reserved" in m for m in messages)
+
+
[email protected]("kind", ["dspark", "dflash"])
+def test_a_self_contained_drafter_is_not_reported_unreserved(tmp_path, monkeypatch, kind):
+ monkeypatch.setattr(lc, "_sidecar_borrows_target_tensors", lambda path: False)
+ assert _warned(tmp_path, monkeypatch, kind) is False
+
+
[email protected]("kind", ["dspark", "dflash"])
+def test_a_borrowing_drafter_without_a_reserve_is_reported(tmp_path, monkeypatch, kind):
+ assert _warned(tmp_path, monkeypatch, kind, fit_target = False) is True
+
+
+def test_a_one_card_tensor_split_is_reserved_and_reported(tmp_path, monkeypatch):
+ """Tensor mode folds several cards into one meta device; one card has nothing to fold."""
+ backend, gguf = h.cards(tmp_path, h.ONE)
+ seen = h.spy_fit_sized(backend)
+ messages = h.info_messages(monkeypatch)
+ cmd = h.dspark(backend, gguf, tmp_path, extra_args = ["--split-mode", "tensor"])
+ assert h.margin(cmd) >= FLOOR
+ assert seen["fit_sized"] is True
+ assert not any("is not reserved" in m for m in messages)
+
+
+# Retries: the reserve is reconciled at every spawn off the argv being launched.
+
+
+def _flash_crash(i, cmd):
+ return -11 if h.last(cmd, "--flash-attn") == "on" else None
+
+
+def _flash_off(i, cmd):
+ return h.last(cmd, "--flash-attn") == "off"
+
+
+def test_the_drafter_free_startup_retry_drops_the_reserve(tmp_path):
+ backend, gguf = h.cuda_backend(tmp_path)
+ cmds = h.spawns(
+ backend,
+ gguf,
+ healthy = lambda i, cmd: "draft-dspark" not in cmd,
+ n_ctx = 4096,
+ speculative_type = "dspark",
+ dspark_draft_path = h.sidecar(tmp_path),
+ )
+ assert cmds.loaded
+ first, retry = cmds[0], cmds[-1]
+ assert int(h.last(first, "--fit-target")) >= FLOOR
+ assert "--model-draft" not in retry
+ assert h.margin(retry) < FLOOR
+
+
+def test_the_drafter_free_fallback_keeps_the_reserve_for_a_user_drafter(tmp_path):
+ """The fallback removes Studio's own block; a bare user --model-draft survives and
+ still loads, so its reserve must too."""
+ backend, gguf = h.cuda_backend(tmp_path)
+ custom = tmp_path / "custom-dspark-Q8_0.gguf"
+ custom.write_bytes(b"draft")
+ cmds = h.spawns(
+ backend,
+ gguf,
+ healthy = lambda i, cmd: "draft-dspark" not in cmd,
+ n_ctx = 4096,
+ speculative_type = "dspark",
+ dspark_draft_path = h.sidecar(tmp_path),
+ extra_args = ("--model-draft", str(custom)),
+ )
+ retry = cmds[-1]
+ if "draft-dspark" in retry:
+ pytest.skip("the user drafter kept the speculative block; no fallback ran")
+ assert str(custom) in retry
+ assert h.margin(retry) >= FLOOR
+
+
+def _spawn_source():
+ src = inspect.getsource(B.load_model)
+ spawn = src[src.index("def _spawn_and_wait(") :]
+ return src, spawn[: spawn.index("_last_spawn_cmd = list(run_cmd)")]
+
+
+def test_the_fit_on_startup_retry_carries_the_reserve():
+ """Every attempt of _spawn_and_wait (the --fit on retry re-enters its loop) reconciles
+ the reserve before spawning. Checked at the source: the retry needs a real crash."""
+ _src, spawn = _spawn_source()
+ assert "_sidecar_fit_reserve_eligible_mib" in spawn
+ assert "_sidecar_reserve_margin_flags(" in spawn
+
+
+def test_the_startup_retry_reads_the_child_not_a_literal_flag():
+ src, spawn = _spawn_source()
+ retry = src[src.index("retrying once with --fit on so it can offload") :]
+ retry = retry[: retry.index("_did_fit_retry = True")]
+ assert "_sidecar_reserve_margin_flags(" in spawn
+ assert '"--model-draft" in _run' not in retry
+ assert "fit_target_margin_in(extra_args, None)" not in retry
+
+
+def test_the_noflash_fit_flip_carries_the_reserve(tmp_path):
+ backend, gguf = h.cuda_backend(tmp_path)
+ backend._select_gpus = lambda *a, **k: ([0], False) # planned full offload: --fit off
+ cmds = h.spawns(
+ backend,
+ gguf,
+ crash = _flash_crash,
+ healthy = _flash_off,
+ late_exit = True,
+ n_ctx = 4096,
+ speculative_type = "dspark",
+ dspark_draft_path = h.sidecar(tmp_path),
+ )
+ noflash = [c for c in cmds if h.last(c, "--flash-attn") == "off"]
+ if not noflash or h.last(noflash[0], "--fit") != "on":
+ pytest.skip("no --flash-attn off retry with the fitter on in this harness")
+ assert "--model-draft" in noflash[0]
+ assert h.margin(noflash[0]) >= FLOOR
+
+
+def test_the_noflash_retry_reprices_the_reserve(tmp_path):
+ """--flash-attn off pads the draft V to f16, so the reserve grows."""
+ backend, gguf = h.cuda_backend(tmp_path)
+ backend._estimate_mtp_overhead_bytes = lambda n_ctx, draft_cache_type_v = None, **kw: (
+ 8 * GIB + (1 if str(draft_cache_type_v or "f16").lower().startswith("q") else 3) * GIB
+ )
+ cmds = h.spawns(
+ backend,
+ gguf,
+ crash = _flash_crash,
+ healthy = _flash_off,
+ n_ctx = 4096,
+ speculative_type = "dspark",
+ dspark_draft_path = h.sidecar(tmp_path),
+ extra_args = ("--cache-type-k-draft", "q4_0", "--cache-type-v-draft", "q4_0"),
+ )
+ noflash = [c for c in cmds if h.last(c, "--flash-attn") == "off"]
+ if not noflash or "--model-draft" not in noflash[0]:
+ pytest.skip("no --flash-attn off retry with the drafter in this harness")
+ assert h.margin(noflash[0]) >= h.margin(cmds[0]) + 2 * 1024
+
+
+def test_noflash_reprice_leaves_target_rollback_to_the_fitter(tmp_path):
+ backend, gguf = _rollback_backend(tmp_path)
+ cmds = h.spawns(
+ backend,
+ gguf,
+ crash = _flash_crash,
+ healthy = _flash_off,
+ n_ctx = 4096,
+ speculative_type = "dspark",
+ dspark_draft_path = h.sidecar(tmp_path),
+ )
+ noflash = [c for c in cmds if h.last(c, "--flash-attn") == "off" and "--model-draft" in c]
+ if not noflash:
+ pytest.skip("no --flash-attn off retry with the drafter in this harness")
+ assert h.margin(noflash[0]) < FLOOR + _ROLLBACK // MIB
+
+
+def test_a_retry_added_reserve_leaves_with_the_drafter(tmp_path):
+ backend, gguf = h.cuda_backend(tmp_path)
+ backend._select_gpus = lambda *a, **k: ([0], False) # planned full offload: --fit off
+ cmds = h.spawns(
+ backend,
+ gguf,
+ # flash on: signal crash (fit-on retry, then the --flash-attn off retry carries the
+ # reserve forward); flash off with the drafter: plain exit, so the drafter goes.
+ crash = lambda i, cmd: (
+ -11 if h.last(cmd, "--flash-attn") == "on" else (1 if "--model-draft" in cmd else None)
+ ),
+ healthy = lambda i, cmd: "--model-draft" not in cmd,
+ n_ctx = 4096,
+ speculative_type = "dspark",
+ dspark_draft_path = h.sidecar(tmp_path),
+ )
+ reserved = [c for c in cmds if "--model-draft" in c and h.margin(c) >= FLOOR]
+ if not reserved or "--model-draft" in cmds[-1]:
+ pytest.skip("no reserved retry followed by a drafter-free launch in this harness")
+ assert h.margin(cmds[-1]) < FLOOR
+
+
+_REFUSED = ("a unified KV cache is only supported with a single sequence",)
+
+
+def _one_slot_retry(backend, gguf, tmp_path, **kw):
+ return h.spawns(
+ backend,
+ gguf,
+ stdout = lambda i, cmd: _REFUSED if i == 0 else (),
+ healthy = lambda i, cmd: i > 0,
+ crash = lambda i, cmd: None if i > 0 else 1,
+ n_ctx = 4096,
+ n_parallel = 4,
+ speculative_type = "dspark",
+ dspark_draft_path = h.sidecar(tmp_path),
+ **kw,
+ )
+
+
+def test_the_one_slot_retry_reprices_the_reserve(tmp_path):
+ backend, gguf = h.cuda_backend(tmp_path)
+ cmds = _one_slot_retry(h.slot_priced(backend), gguf, tmp_path)
+ assert len(cmds) >= 2, [c[-6:] for c in cmds]
+ first, retry = cmds[0], cmds[1]
+ assert h.last(first, "--parallel") == "4"
+ assert h.last(retry, "--parallel") in (None, "1")
+ assert "--model-draft" in retry
+ assert h.margin(first) >= FLOOR + 4 * 1024
+ # Still reserved for the drafter and its one slot, not for the three slots dropped.
+ assert FLOOR + 1024 <= h.margin(retry) < FLOOR + 2 * 1024
+ assert retry.count("--fit-target") <= first.count("--fit-target")
+
+
+def test_the_one_slot_retry_keeps_a_user_margin_equal_to_the_reserve(tmp_path):
+ """A user --fit-target equal to Studio's reserved margin stays theirs: the retry drops
+ only Studio's trailing pair and reprices it for the one slot left."""
+ (tmp_path / "plain").mkdir()
+ backend, gguf = h.cuda_backend(tmp_path / "plain")
+ plain = h.spawns(
+ h.slot_priced(backend),
+ gguf,
+ healthy = lambda i, cmd: True,
+ n_ctx = 4096,
+ n_parallel = 4,
+ speculative_type = "dspark",
+ dspark_draft_path = h.sidecar(tmp_path),
+ )[0]
+ reserved = str(h.margin(plain))
+ backend, gguf = h.cuda_backend(tmp_path)
+ cmds = _one_slot_retry(
+ h.slot_priced(backend),
+ gguf,
+ tmp_path,
+ extra_args = ["--fit-target", reserved, "--fit-target", "1024"],
+ )
+ assert len(cmds) >= 2, [c[-6:] for c in cmds]
+ retry = h.fit_targets(cmds[1])
+ assert retry[-3:-1] == [reserved, "1024"], retry
+ assert FLOOR + 1024 <= int(retry[-1]) < FLOOR + 2 * 1024, retry
+
+
+# _drop_sidecar_margins: the fallback's cleanup takes back exactly the pairs Studio added.
+
+
+ "cmd, integrity, rebuilt, pairs, want",
+ [
+ # Single-sequence recovery already dropped --kv-unified and set --parallel 1.
+ (
+ ["--parallel", "1", "--fit", "on", "--fit-ctx", "4096", "--fit-target", "9252"],
+ ["--kv-unified", "--fit-ctx", "4096", "--fit-target", "9252"],
+ ["--kv-unified", "--fit-ctx", "4096", "--fit-target", "1024"],
+ [],
+ ["--parallel", "1", "--fit", "on", "--fit-ctx", "4096", "--fit-target", "1024"],
+ ),
+ (
+ ["--fit", "on", "--fit-target", "9225", "--fit-target", "1024", "--fit-target", "9225"],
+ [],
+ [],
+ [["--fit-target", "9225"]],
+ ["--fit", "on", "--fit-target", "9225", "--fit-target", "1024"],
+ ),
+ (
+ ["--fit", "on", "--fit-target", "9225", "--fit-target", "11273"],
+ [],
+ [],
+ [["--fit-target", "9225"], ["--fit-target", "11273"]],
+ ["--fit", "on"],
+ ),
+ ],
+ ids = [
+ "test_cleanup_swaps_only_the_margin_pair_after_a_rewrite",
+ "test_cleanup_removes_studios_trailing_pair_not_the_users",
+ "test_cleanup_removes_every_recorded_pair",
+ ],
+)
+def test_drop_sidecar_margins(cmd, integrity, rebuilt, pairs, want):
+ assert lc._drop_sidecar_margins(cmd, integrity, rebuilt, pairs) == want
+
+
+def test_a_repeated_cleanup_keeps_the_users_identical_margin():
+ integrity, rebuilt, user = (
+ ["--fit-target", "9449"],
+ ["--fit-target", "1024"],
+ ["--fit-target", "9449"],
+ )
+ # Studio's copy is already gone; only the user's remains.
+ cmd = ["llama-server", "-m", "m.gguf", "--fit-target", "1024", *user]
+ assert lc._drop_sidecar_margins(cmd, integrity, rebuilt, [], user_args = user) == cmd
+ cmd = ["llama-server", "--fit-target", "9449", *user]
+ assert lc._drop_sidecar_margins(cmd, integrity, rebuilt, [], user_args = user) == [
+ "llama-server",
+ "--fit-target",
+ "1024",
+ *user,
+ ]
+ # A trailing pair a launch appended goes; the user's identical earlier one stays.
+ pair = ["--fit-target", "17874"]
+ assert lc._drop_sidecar_margins(
+ ["llama-server", *pair, *pair], [], [], [pair], user_args = pair
+ ) == [
+ "llama-server",
+ *pair,
+ ]
+ assert lc._drop_sidecar_margins(["llama-server", *pair], [], [], [pair], user_args = pair) == [
+ "llama-server",
+ *pair,
+ ]
diff --git a/studio/backend/tests/test_spec_sidecar_fit_reserve_metal.py b/studio/backend/tests/test_spec_sidecar_fit_reserve_metal.py
new file mode 100644
index 00000000000..8412a10ef88
--- /dev/null
+++ b/studio/backend/tests/test_spec_sidecar_fit_reserve_metal.py
@@ -0,0 +1,222 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
+
+"""Metal: Auto drops a sidecar whose reserve stops the target fitting; explicit DSpark keeps it."""
+
+from __future__ import annotations
+
+import pytest
+
+from . import _sidecar_fit_helpers as h
+from ._sidecar_fit_helpers import GIB, lc, metal_cmd
+
+B = lc.LlamaCppBackend
+
+
[email protected](autouse = True)
+def _studio_home(tmp_path, monkeypatch):
+ monkeypatch.setenv("UNSLOTH_STUDIO_HOME", str(tmp_path / "studio_home"))
+
+
+def _embd(
+ monkeypatch,
+ value,
+ default = False,
+):
+ """Stub the pageable-embedding discount; excluded NextN blocks forfeit it while the
+ drafter may engage."""
+ calls = []
+
+ def embd(
+ self,
+ *a,
+ mtp_may_engage = default,
+ **k,
+ ):
+ calls.append(mtp_may_engage)
+ return value(mtp_may_engage)
+
+ monkeypatch.setattr(B, "_metal_demand_paged_embedding_bytes", embd)
+ return calls
+
+
+def _drafter_free_discount(mtp):
+ return 0 if mtp else 3 * GIB
+
+
+# Every GGUF here reads 4 GiB; the target with compute and soft overhead prices ~9.2 GiB,
+# so 12 GiB holds the target alone but not target + sidecar, and 16 GiB holds both.
+class TestMetalLaunch:
+ @pytest.mark.parametrize(
+ "extras, kept",
+ [
+ ([], False),
+ # Only one block on the card: not dropped for the whole file.
+ (["--spec-draft-ngl", "1"], True),
+ (["--cpu-moe-draft"], True),
+ # Its weights allocate on the RPC host, so Metal's budget is no reason to drop it.
+ (["--rpc", "192.0.2.1:50052", "--device", "MTL0", "--device-draft", "RPC0"], True),
+ ],
+ ids = [
+ "test_auto_drops_a_sidecar_that_does_not_fit",
+ "test_metal_auto_keeps_a_partially_offloaded_drafter",
+ "test_metal_auto_keeps_a_drafter_with_its_experts_on_cpu",
+ "test_metal_auto_keeps_a_drafter_pinned_to_another_device",
+ ],
+ )
+ def test_auto_at_12_gib(self, tmp_path, monkeypatch, extras, kept):
+ cmd, backend, _ = metal_cmd(tmp_path, monkeypatch, "auto", budget_gib = 12, extra_args = extras)
+ assert ("--model-draft" in cmd) is kept
+ assert (backend.spec_fallback_reason == "drafter_no_vram") is not kept
+
+ def test_explicit_dspark_keeps_it_with_the_reserve(self, tmp_path, monkeypatch):
+ cmd, _backend, path = metal_cmd(tmp_path, monkeypatch, "dspark", budget_gib = 12)
+ assert h.last(cmd, "--model-draft") == path
+ assert h.margin(cmd) >= 1024 + 4 * 1024
+
+ def test_auto_keeps_a_sidecar_that_fits(self, tmp_path, monkeypatch):
+ cmd, backend, path = metal_cmd(tmp_path, monkeypatch, "auto", budget_gib = 16)
+ assert h.last(cmd, "--model-draft") == path
+ assert h.margin(cmd) >= 1024 + 4 * 1024
+ assert backend.spec_fallback_reason is None
+
+
+ "budget_gib, kw, discount",
+ [
+ # The flat draft reserve and the draft compute charge go with the sidecar.
+ (12, {"kv_per_token": 128 * 1024}, None),
+ # Within 224 MiB of the target-only footprint the drafter's compute must not keep it.
+ (9.1, {}, None),
+ # The discount decided with the drafter engaged is decided again once it is dropped.
+ (9.1, {}, _drafter_free_discount),
+ ],
+ ids = [
+ "test_metal_auto_drop_releases_the_drafter_charges",
+ "test_metal_target_only_bound_leaves_out_the_drafter_compute",
+ "test_the_metal_auto_drop_recomputes_the_embedding_discount",
+ ],
+)
+def test_a_metal_auto_drop_matches_a_load_with_speculation_off(
+ tmp_path, monkeypatch, budget_gib, kw, discount
+):
+ def run(mode):
+ if discount is not None:
+ _embd(monkeypatch, discount)
+ return metal_cmd(tmp_path, monkeypatch, mode, budget_gib = budget_gib, **kw)[0]
+
+ auto, off = run("auto"), run("off")
+ assert "--model-draft" not in auto and "--model-draft" not in off
+ assert h.last(auto, "-c") == h.last(off, "-c")
+
+
+def test_metal_auto_drop_prices_the_target_with_the_drafter_free_discount(tmp_path, monkeypatch):
+ _embd(monkeypatch, _drafter_free_discount, default = True)
+ cmd, _backend, _ = metal_cmd(tmp_path, monkeypatch, "auto", budget_gib = 7.5)
+ assert "--model-draft" not in cmd
+
+
+def test_a_discount_the_drop_unlocks_is_logged(tmp_path, monkeypatch):
+ _embd(monkeypatch, _drafter_free_discount)
+ messages = h.info_messages(monkeypatch)
+ cmd, _backend, _ = metal_cmd(tmp_path, monkeypatch, "auto", budget_gib = 9.1)
+ assert "--model-draft" not in cmd
+ assert any("of input embeddings stay in the CPU file mapping" in m for m in messages)
+
+
+def test_the_draft_free_probe_waits_for_a_drafter_that_does_not_fit(tmp_path, monkeypatch):
+ """16 GiB holds both, so the draft-free discount (a cold llama-fit-params run) is never
+ needed."""
+ calls = _embd(monkeypatch, lambda mtp: 0)
+ cmd, _backend, _ = metal_cmd(tmp_path, monkeypatch, "auto", budget_gib = 16)
+ assert "--model-draft" in cmd
+ assert False not in calls
+
+
+def test_metal_auto_keeps_a_user_owned_bare_model_draft(tmp_path, monkeypatch):
+ """A bare --model-draft in the extras still loads (has_dft()), so Auto must not release
+ its reserve while the child loads it anyway."""
+ custom = tmp_path / "custom-dspark-Q8_0.gguf"
+ custom.write_bytes(b"draft")
+ cmd, backend, _ = metal_cmd(
+ tmp_path, monkeypatch, "auto", budget_gib = 12, extra_args = ["--model-draft", str(custom)]
+ )
+ assert str(custom) in cmd
+ assert backend.spec_fallback_reason != "drafter_no_vram"
+
+
[email protected]("pin, reserved", [("MTL0", True), ("mtl0", True), ("Vulkan0", False)])
+def test_a_metal_draft_pin_is_reserved_only_on_the_metal_device(
+ tmp_path, monkeypatch, pin, reserved
+):
+ cmd, _backend, _ = metal_cmd(
+ tmp_path, monkeypatch, "dspark", budget_gib = 16, extra_args = ["--device-draft", pin]
+ )
+ assert "--model-draft" in cmd
+ assert (h.margin(cmd) >= 1024 + 4 * 1024) is reserved
+
+
+def test_every_context_gate_prices_the_draft_micro_batch(tmp_path, monkeypatch):
+ """The Metal context search takes the draft callback directly, not through the wrapper
+ that reads --spec-draft-n-ubatch."""
+ seen = []
+ real = B._estimate_mtp_overhead_bytes
+
+ def spy(self, n_ctx, *args, **kwargs):
+ seen.append(kwargs.get("n_ubatch"))
+ return real(self, n_ctx, *args, **kwargs)
+
+ monkeypatch.setattr(B, "_estimate_mtp_overhead_bytes", spy)
+ cmd, _backend, _ = metal_cmd(
+ tmp_path,
+ monkeypatch,
+ "dspark",
+ budget_gib = 16,
+ kv_per_token = 32 * 1024,
+ extra_args = ["--spec-draft-n-ubatch", "32"],
+ )
+ assert "--model-draft" in cmd
+ # The first call only asks whether the reserve can be sized at all.
+ assert len(seen) > 2 and set(seen[1:]) == {32}, seen
+
+
[email protected]("budget_gib, wired_gib", [(20, 20), (15, 20), (20, 18), (20, 0), (30, 18)])
+def test_metal_auto_drops_a_sidecar_an_explicit_context_cannot_hold(
+ tmp_path, monkeypatch, budget_gib, wired_gib
+):
+ """An explicit context launches verbatim: 65,536 tokens at 128 KiB hold the target (Off
+ loads), not target + sidecar, so Auto drops the sidecar instead of refusing."""
+ kw = dict(n_ctx = 65_536, kv_per_token = 128 * 1024, budget_gib = budget_gib, wired_gib = wired_gib)
+ off, _, _ = metal_cmd(tmp_path, monkeypatch, "off", **kw)
+ assert h.last(off, "-c") == "65536"
+ auto, backend, _ = metal_cmd(tmp_path, monkeypatch, "auto", **kw)
+ assert "--model-draft" not in auto
+ assert backend.spec_fallback_reason == "drafter_no_vram"
+ assert h.last(auto, "-c") == "65536"
+
+
+def test_metal_auto_keeps_a_sidecar_an_explicit_context_holds(tmp_path, monkeypatch):
+ cmd, backend, _ = metal_cmd(
+ tmp_path, monkeypatch, "auto", n_ctx = 4096, kv_per_token = 1024, budget_gib = 20
+ )
+ assert "--model-draft" in cmd
+ assert backend.spec_fallback_reason != "drafter_no_vram"
+
+
+def test_metal_auto_keeps_the_sidecar_when_kv_cannot_be_estimated(tmp_path, monkeypatch):
+ """With no KV estimate the target alone is not known to fit either (the discrete drop's
+ own guard)."""
+ cmd, backend, _ = metal_cmd(tmp_path, monkeypatch, "auto", budget_gib = 10, can_estimate_kv = False)
+ assert "--model-draft" in cmd
+ assert backend.spec_fallback_reason != "drafter_no_vram"
+
+
+def test_an_unmeasured_model_is_reported_once_per_load(monkeypatch):
+ import test_metal_explicit_context_guard as metal_guard
+
+ messages = h.info_messages(monkeypatch)
+ case = metal_guard.TestWhichLoadsLeaveTheEmbeddingsInTheMapping()
+ with lc._load_probe_memo_scope():
+ assert case._bytes(monkeypatch, measured = None) == 0
+ assert case._bytes(monkeypatch, measured = None) == 0
+ assert sum("could not measure how llama.cpp maps" in m for m in messages) == 1
diff --git a/studio/backend/tests/test_spec_sidecar_fit_reserve_multi_gpu.py b/studio/backend/tests/test_spec_sidecar_fit_reserve_multi_gpu.py
new file mode 100644
index 00000000000..e77b152242e
--- /dev/null
+++ b/studio/backend/tests/test_spec_sidecar_fit_reserve_multi_gpu.py
@@ -0,0 +1,978 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
+
+"""Sidecar reserve across several cards: per-card shares, draft pins, split modes, host KV."""
+
+from __future__ import annotations
+
+import inspect
+import math
+from types import SimpleNamespace
+
+import pytest
+
+from . import _sidecar_fit_helpers as h
+from ._sidecar_fit_helpers import FLOOR, GIB, MANUAL_AUTO, TWO, lc
+
+B = lc.LlamaCppBackend
+_MIXED = ((0, 49_140, 49_140), (1, 24_576, 24_576))
+_VK_IGPU = ((0, 24_000, 0), (1, 24_000, 24_000))
+_COMPUTE_FLOOR = math.ceil(B._MTP_DRAFT_COMPUTE_BYTES / h.MIB)
+
+
[email protected](autouse = True)
+def _studio_home(tmp_path, monkeypatch):
+ monkeypatch.setenv("UNSLOTH_STUDIO_HOME", str(tmp_path / "studio_home"))
+ monkeypatch.setenv("CUDA_DEVICE_ORDER", "PCI_BUS_ID")
+ for k in ("CUDA_VISIBLE_DEVICES", "LLAMA_ARG_DEVICE", "LLAMA_ARG_SPLIT_MODE", "LLAMA_ARG_RPC"):
+ monkeypatch.delenv(k, raising = False)
+ # A discrete host on every runner, class-level budget reads included.
+ monkeypatch.setattr(lc, "_metal_capable_host", lambda: False)
+ monkeypatch.setattr(B, "_apple_metal_memory_budget_bytes", staticmethod(lambda: 0))
+
+
+def _pci_ids(monkeypatch):
+ """Studio's GPU ids are nvidia-smi's PCI indices and no mask needs a UUID resolver."""
+ monkeypatch.setattr(B, "_GPU_IDS_ARE_PCI_INDICES", True)
+ monkeypatch.setattr(B, "_smi_uuid_mask_order", staticmethod(lambda: None))
+ monkeypatch.setattr(B, "_visibility_mask_is_unmappable", staticmethod(lambda: False))
+
+
+def _budget(monkeypatch, _pci_ids):
+ """A 30% VRAM budget, so each card gets its own --fit-target margin."""
+ monkeypatch.setattr(lc, "_active_vram_fraction", lambda: 0.3)
+ monkeypatch.setattr(B, "_SMI_MIG_INDICES", frozenset())
+ monkeypatch.setattr(B, "_host_torch_is_rocm", staticmethod(lambda: False))
+
+
+def _band(m, lo, hi):
+ return 1024 + lo * 1024 <= m < 1024 + hi * 1024
+
+
+def test_multi_gpu_cuda_does_not_broadcast_the_reserve(tmp_path):
+ """A scalar --fit-target applies to every device: adding the whole sidecar to it would
+ withhold the sidecar on each card and push target layers to the host."""
+ backend, gguf = h.cuda_backend(tmp_path, memory = TWO)
+ cmd = h.dspark(backend, gguf, tmp_path)
+ assert h.last(cmd, "--model-draft") == str(tmp_path / "dspark-model-Q8_0.gguf")
+ assert h.margin(cmd) < FLOOR
+
+
+ "args, env",
+ [(["--device", "Vulkan0,Vulkan1"], {}), ([], {"LLAMA_ARG_DEVICE": "CUDA0,CUDA1"})],
+ ids = ["argv", "env"],
+)
+def test_no_scalar_reserve_on_a_multi_device_launch(args, env):
+ argv = ["--fit", "on", "-md", "/d.gguf", *args]
+ assert lc._sidecar_reserve_margin_flags(argv, (), env, 1024, 8192) == []
+
+
+def test_a_single_named_device_keeps_the_reserve():
+ argv = ["--fit", "on", "-md", "/d.gguf", "--device", "CUDA0"]
+ assert lc._sidecar_reserve_margin_flags(argv, (), {}, 1024, 8192) == ["--fit-target", "9216"]
+
+
+def test_extras_widening_the_device_list_get_no_scalar_reserve(tmp_path):
+ backend, gguf = h.cuda_backend(tmp_path, memory = TWO, vulkan = True)
+ backend._select_gpus = lambda *a, **k: ([0], True)
+ cmd = h.dspark(backend, gguf, tmp_path, extra_args = ("--device", "Vulkan0,Vulkan1"))
+ assert h.margin(cmd) < FLOOR
+
+
+def test_one_named_device_on_a_two_gpu_host_keeps_the_reserve(tmp_path):
+ backend, gguf = h.cuda_backend(tmp_path, memory = TWO, vulkan = True)
+ cmd = h.dspark(backend, gguf, tmp_path, extra_args = ("--device", "Vulkan0"))
+ assert h.last(cmd, "--device") == "Vulkan0"
+ assert h.margin(cmd) >= FLOOR
+
+
+def test_two_detected_gpus_without_a_device_pin_stay_unreserved(tmp_path):
+ backend, gguf = h.cuda_backend(tmp_path, memory = TWO, vulkan = True)
+ assert h.margin(h.dspark(backend, gguf, tmp_path)) < FLOOR
+
+
[email protected]("devices", ["CUDA0,CUDA0", "CUDA0, cuda0"])
+def test_a_repeated_device_is_one_card_for_the_reserve(tmp_path, devices):
+ backend, gguf = h.cuda_backend(tmp_path)
+ assert h.margin(h.dspark(backend, gguf, tmp_path, extra_args = ["--device", devices])) >= FLOOR
+
+
+def test_a_launch_the_arch_gate_narrows_to_one_card_gets_the_reserve(tmp_path):
+ backend, gguf = h.cuda_backend(tmp_path, memory = TWO)
+ backend._arch_gate_survivors = lambda binary = None: [1]
+ assert h.margin(h.dspark(backend, gguf, tmp_path)) >= FLOOR
+
+
+def test_an_arch_retry_onto_several_gpus_drops_the_scalar_reserve(tmp_path):
+ rows = ((0, 24_000, 24_000), (1, 12_000, 12_000), (2, 12_000, 12_000))
+ backend, gguf = h.cuda_backend(tmp_path, memory = rows)
+ backend._select_gpus = lambda *a, **k: ([0], True)
+ backend._arch_crash_retry_gpu_ids = lambda *a, **k: [1, 2]
+ crash = ("ggml_cuda_compute_forward: ROCm error: device kernel image is invalid",)
+ cmds = h.spawns(
+ backend,
+ gguf,
+ stdout = lambda i, cmd: crash if i == 0 else (),
+ healthy = lambda i, cmd: i > 0,
+ crash = lambda i, cmd: None if i > 0 else 1,
+ n_ctx = 4096,
+ speculative_type = "dspark",
+ dspark_draft_path = h.sidecar(tmp_path),
+ )
+ assert len(cmds) >= 2, [c[-6:] for c in cmds]
+ assert h.margin(cmds[0]) >= FLOOR
+ assert "--model-draft" in cmds[1]
+ assert h.margin(cmds[1]) < FLOOR
+
+
+def test_a_launch_left_on_two_cards_reserves_each_cards_share(tmp_path):
+ backend, gguf = h.cards(tmp_path)
+ margins = h.margins(h.dspark(backend, gguf, tmp_path))
+ assert len(margins) == 2
+ assert all(_band(m, 4, 5) for m in margins)
+
+
+ "memory, extras",
+ [
+ # 30,000 of 40,000 MiB free: three quarters of the ~8.3 GiB reserve on card 0.
+ (((0, 30_000, 30_000), (1, 10_000, 10_000)), []),
+ # The drafter inherits the target's --tensor-split even on equally free cards.
+ (TWO, ["--tensor-split", "3,1"]),
+ ],
+ ids = [
+ "test_unequal_cards_carry_their_own_shares",
+ "test_an_explicit_tensor_split_decides_the_draft_shares",
+ ],
+)
+def test_draft_shares_three_to_one(tmp_path, memory, extras):
+ backend, gguf = h.cards(tmp_path, memory)
+ big, small = h.margins(h.dspark(backend, gguf, tmp_path, extra_args = extras))
+ assert _band(big, 6, 7) and _band(small, 2, 3)
+
+
+def test_a_small_card_beside_a_large_one_is_not_over_reserved(tmp_path):
+ """A broadcast of the large card's share (~7.6 GiB) would exceed the 8,000 MiB card."""
+ backend, gguf = h.cards(tmp_path, ((0, 80_000, 80_000), (1, 8_000, 8_000)))
+ big, small = h.margins(h.dspark(backend, gguf, tmp_path))
+ assert big >= 1024 + 7 * 1024
+ assert small < 1024 + 1024
+
+
+def test_per_device_margins_follow_an_inherited_mask_order(tmp_path, monkeypatch):
+ """CUDA_VISIBLE_DEVICES=1,0 makes physical 1 the child's device 0."""
+ monkeypatch.setenv("CUDA_VISIBLE_DEVICES", "1,0")
+ backend, gguf = h.cards(tmp_path, ((0, 30_000, 30_000), (1, 10_000, 10_000)))
+ first, second = h.margins(h.dspark(backend, gguf, tmp_path))
+ assert first < second
+
+
+def _three_blocks(
+ tmp_path,
+ graph_gib = None,
+ target_layers = None,
+):
+ backend, gguf = h.cards(tmp_path)
+ h.three_block_drafter(backend)
+ if graph_gib is not None:
+ backend._mtp_draft_compute_bytes = lambda *a, **k: graph_gib * GIB
+ if target_layers is not None:
+ backend._n_layers = target_layers
+ return backend, gguf
+
+
+ "extras, bands",
+ [
+ # Equal cards: blocks 0-1 below 2/4, block 2 at it, so two on card 0 and one on card 1.
+ ([], [(5, 6), (2, 3)]),
+ # 3:2 keeps every block under 0.6 of n_layer + 1, so all three on card 0.
+ (["--tensor-split", "3,2"], [(8, 9), (0, 1)]),
+ ],
+)
+def test_draft_shares_follow_llama_cpps_whole_layer_split(tmp_path, extras, bands):
+ backend, gguf = _three_blocks(tmp_path)
+ margins = h.margins(h.dspark(backend, gguf, tmp_path, extra_args = extras))
+ for m, (lo, hi) in zip(margins, bands):
+ assert _band(m, lo, hi), margins
+
+
+def test_the_output_card_keeps_the_draft_compute_buffers_without_a_block(tmp_path):
+ """3:2 puts all three blocks on card 0, but the draft still runs (and writes its logits)
+ on the last card."""
+ backend, gguf = _three_blocks(tmp_path)
+ _big, small = h.margins(h.dspark(backend, gguf, tmp_path, extra_args = ["--tensor-split", "3,2"]))
+ assert small >= 1024 + _COMPUTE_FLOOR
+
+
+def test_the_output_card_keeps_the_whole_vocabulary_graph(tmp_path):
+ """The draft graph's logits over the borrowed vocabulary are sized on the output card,
+ so that part is not split by block shares."""
+ backend, gguf = _three_blocks(tmp_path, graph_gib = 3)
+ _big, small = h.margins(h.dspark(backend, gguf, tmp_path, extra_args = ["--tensor-split", "3,2"]))
+ assert small >= 1024 + _COMPUTE_FLOOR + 3 * 1024
+
+
+def test_a_split_that_leaves_the_last_card_out_keeps_the_graph_on_the_used_one(tmp_path):
+ backend, gguf = _three_blocks(tmp_path, graph_gib = 3)
+ used, idle = h.margins(h.dspark(backend, gguf, tmp_path, extra_args = ["--tensor-split", "1,0"]))
+ assert used >= 1024 + 3 * 1024
+ assert idle < 2048
+
+
+def test_the_graph_follows_the_targets_output_layer_not_the_drafters(tmp_path):
+ """DSpark borrows the target's output tensor: with one target block and a 3:2 split the
+ target's output layer (index 1 of 2 units = 0.5 < 0.6) is on card 0, so the graph is too."""
+ backend, gguf = _three_blocks(tmp_path, graph_gib = 3, target_layers = 1)
+ used, idle = h.margins(h.dspark(backend, gguf, tmp_path, extra_args = ["--tensor-split", "3,2"]))
+ assert used >= 1024 + 3 * 1024 + 4 * 1024
+ assert idle < 2048
+
+
+def test_a_draft_with_its_own_head_keeps_its_graph_on_its_own_output_card(tmp_path, monkeypatch):
+ """Same split, but the draft owns its head: its output layer (3 / 4 = 0.75) is on card 1."""
+ monkeypatch.setattr(lc, "_sidecar_borrows_output", lambda path: False)
+ backend, gguf = _three_blocks(tmp_path, graph_gib = 3, target_layers = 1)
+ _used, out = h.margins(h.dspark(backend, gguf, tmp_path, extra_args = ["--tensor-split", "3,2"]))
+ assert out >= 1024 + 3 * 1024
+
+
+def _owned_head_margins(tmp_path, monkeypatch, head_gib):
+ monkeypatch.setattr(lc, "_sidecar_borrows_output", lambda path: False)
+ backend, gguf = _three_blocks(tmp_path, target_layers = 1)
+ read = backend._tensor_spill_layout
+
+ def layout(path, **kw):
+ if "dspark" in str(path):
+ return SimpleNamespace(complete = True, lm_head_bytes = int(head_gib * GIB))
+ return read(path, **kw)
+
+ backend._tensor_spill_layout = layout
+ return h.margins(h.dspark(backend, gguf, tmp_path, extra_args = ["--tensor-split", "3,2"]))
+
+
+def test_an_owned_head_is_reserved_on_its_output_card(tmp_path, monkeypatch):
+ (tmp_path / "a").mkdir()
+ (tmp_path / "b").mkdir()
+ no_head = _owned_head_margins(tmp_path / "a", monkeypatch, 0)
+ head = _owned_head_margins(tmp_path / "b", monkeypatch, 2)
+ assert head[1] - no_head[1] >= 2000
+ assert no_head[0] - head[0] >= 2000
+
+
+def test_a_row_split_places_whole_draft_layers_like_a_layer_split(tmp_path):
+ """--split-mode row assigns blocks to devices like a layer split (llama-model.cpp)."""
+ (tmp_path / "row").mkdir()
+ backend, gguf = _three_blocks(tmp_path / "row")
+ row = h.margins(h.dspark(backend, gguf, tmp_path, extra_args = ["--split-mode", "row"]))
+ backend, gguf = _three_blocks(tmp_path)
+ layer = h.margins(h.dspark(backend, gguf, tmp_path))
+ assert row == layer
+ assert row[0] > row[1]
+
+
+def test_the_underscore_tensor_split_spelling_places_the_draft(tmp_path):
+ backend, gguf = _three_blocks(tmp_path)
+ margins = h.margins(h.dspark(backend, gguf, tmp_path, extra_args = ["--tensor_split", "1,0"]))
+ assert len(margins) == 2, margins
+ assert margins[0] >= FLOOR
+ assert margins[1] < 2048
+
+
+def test_a_tensor_split_launch_gets_no_per_device_reserve(tmp_path):
+ """--split-mode tensor folds the cards into one meta device: a per-card --fit-target
+ cannot hold the sidecar."""
+ backend, gguf = _three_blocks(tmp_path)
+ margins = h.margins(h.dspark(backend, gguf, tmp_path, extra_args = ["--split-mode", "tensor"]))
+ assert all(m < 2048 for m in margins), margins
+
+
+def test_shares_follow_each_cards_free_memory_past_its_own_margin(tmp_path):
+ """A user margin of 20,000 MiB leaves card 0 about 4 GB for weights against card 1's 23 GB."""
+ backend, gguf = h.cards(tmp_path)
+ margins = h.margins(
+ h.dspark(backend, gguf, tmp_path, extra_args = ["--fit-target", "20000,1024"])
+ )
+ assert len(margins) == 2
+ assert margins[0] < 24_000
+ assert margins[1] > 1024 + 4 * 1024
+
+
[email protected]("order, per_card", [("FASTEST_FIRST", False), ("PCI_BUS_ID", True)])
+def test_per_card_shares_need_a_provable_device_order(tmp_path, monkeypatch, order, per_card):
+ """nvidia-smi keys free memory by PCI order; under FASTEST_FIRST the child enumerates
+ its own way."""
+ monkeypatch.setenv("CUDA_DEVICE_ORDER", order)
+ monkeypatch.setattr(B, "_GPU_IDS_ARE_PCI_INDICES", True)
+ monkeypatch.setattr(B, "_smi_uuid_mask_order", staticmethod(lambda: None))
+ backend, gguf = h.cards(tmp_path, ((0, 80_000, 80_000), (1, 8_000, 8_000)))
+ assert (len(h.margins(h.dspark(backend, gguf, tmp_path))) == 2) is per_card
+
+
[email protected]("parent_order, reserved", [("FASTEST_FIRST", False), ("PCI_BUS_ID", True)])
+def test_torch_ordinals_in_another_order_than_the_child_get_no_positional_reserve(
+ tmp_path, monkeypatch, parent_order, reserved
+):
+ """No nvidia-smi: the ids are torch ordinals in the parent's CUDA_DEVICE_ORDER, while a
+ GPU pick pins the child to PCI_BUS_ID."""
+ monkeypatch.setenv("CUDA_DEVICE_ORDER", parent_order)
+ monkeypatch.setattr(B, "_GPU_IDS_ARE_PCI_INDICES", False)
+ monkeypatch.setattr(B, "_host_torch_is_rocm", staticmethod(lambda: False))
+ backend, gguf = h.cards(tmp_path, ((0, 80_000, 80_000), (1, 8_000, 8_000)))
+ seen = h.spy_fit_sized(backend)
+ cmd = h.dspark(backend, gguf, tmp_path, gpu_ids = [0, 1])
+ assert (len(h.margins(cmd)) == 2) is reserved
+ assert seen["fit_sized"] is reserved
+
+
+def test_the_order_check_is_one_helper_for_planning_and_spawn(monkeypatch):
+ monkeypatch.setattr(B, "_GPU_IDS_ARE_PCI_INDICES", False)
+ monkeypatch.setattr(B, "_host_torch_is_rocm", staticmethod(lambda: False))
+ monkeypatch.delenv("CUDA_DEVICE_ORDER", raising = False)
+ backend = B()
+ # Unset is CUDA's default, FASTEST_FIRST.
+ assert backend._positional_child_order_unproven({}) is False
+ assert backend._positional_child_order_unproven({"CUDA_DEVICE_ORDER": "FASTEST_FIRST"}) is False
+ assert backend._positional_child_order_unproven({"CUDA_DEVICE_ORDER": "PCI_BUS_ID"}) is True
+ # HIP ignores CUDA_DEVICE_ORDER.
+ monkeypatch.setattr(B, "_host_torch_is_rocm", staticmethod(lambda: True))
+ assert backend._positional_child_order_unproven({"CUDA_DEVICE_ORDER": "PCI_BUS_ID"}) is False
+
+
+def _mig_two_cards(tmp_path, monkeypatch):
+ monkeypatch.setattr(B, "_GPU_IDS_ARE_PCI_INDICES", True)
+ monkeypatch.setattr(B, "_smi_uuid_mask_order", staticmethod(lambda: None))
+ # A MIG mask (child order: card 1's slice, then card 0's) no resolver can map.
+ monkeypatch.setattr(B, "_visibility_mask_is_unmappable", staticmethod(lambda: True))
+ return h.cards(tmp_path, ((0, 16_000, 16_000), (1, 4_000, 4_864)))
+
+
+def test_an_unmappable_mig_mask_gets_no_positional_reserve(tmp_path, monkeypatch):
+ backend, gguf = _mig_two_cards(tmp_path, monkeypatch)
+ assert len(h.margins(h.dspark(backend, gguf, tmp_path))) == 1
+
+
+def test_an_unmappable_mig_mask_is_not_reported_as_reserved(tmp_path, monkeypatch):
+ backend, gguf = _mig_two_cards(tmp_path, monkeypatch)
+ seen = h.spy_fit_sized(backend)
+ h.dspark(backend, gguf, tmp_path)
+ assert seen["fit_sized"] is False
+
+
+def test_a_users_per_device_margins_stay_per_device_across_cards():
+ flags = lc._sidecar_reserve_margin_flags(
+ ["--model-draft", "/d.gguf", "--fit", "on"],
+ ["--fit-target", "1024,20000"],
+ {},
+ 1024,
+ 4213,
+ per_device = [4213, 4213],
+ )
+ assert flags == ["--fit-target", "5237,24213"]
+
+
+def test_a_scalar_user_margin_is_broadcast_under_per_device_shares():
+ flags = lc._sidecar_reserve_margin_flags(
+ ["--model-draft", "/d.gguf", "--fit", "on"],
+ ["--fit-target", "2048"],
+ {},
+ 1024,
+ 5000,
+ per_device = [5000, 500],
+ )
+ assert flags == ["--fit-target", "7048,2548"]
+
+
+ "user, env, want",
+ [
+ # A short list sets only the entries it names; the rest keep the earlier value.
+ (["--fit-target", "4096", "--fit-target", "2048,3072"], {}, "3048,4072,5096"),
+ # The env applies first, the argv after it.
+ ([], {"LLAMA_ARG_FIT_TARGET": "4096"}, "5096,5096,5096"),
+ (["--fit-target", "2048"], {"LLAMA_ARG_FIT_TARGET": "4096,4096,8192"}, "3048,3048,3048"),
+ (["--fit-target=2048,3072"], {"LLAMA_ARG_FIT_TARGET": "8192"}, "3048,4072,9192"),
+ (["--fit_target", "4096"], {}, "5096,5096,5096"),
+ (["--fit_target=2048,3072", "-fitt", "4096"], {}, "5096,5096,5096"),
+ ],
+)
+def test_per_device_margins_follow_llama_cpps_update_order(user, env, want):
+ flags = lc._sidecar_reserve_margin_flags(
+ ["--model-draft", "/d.gguf", "--fit", "on"],
+ user,
+ env,
+ 1024,
+ 1000,
+ per_device = [1000, 1000, 1000],
+ )
+ assert flags == ["--fit-target", want]
+
+
+ "user, base, reserve, per_device, by_device, want",
+ [
+ # The smaller card's lower margin is not raised to the scalar.
+ ([], 4000, 5000, [5000, 500], [4000, 1000], "9000,1500"),
+ (["--fit-target", "2048"], 1024, 100, [100, 100], [1024, 4096], "2148,4196"),
+ # A child masked to the smaller card carries that card's own margin.
+ ([], 4000, 5000, None, [1000], "6000"),
+ ],
+ ids = [
+ "test_studios_per_card_margins_are_kept_under_the_reserve",
+ "test_a_users_margin_still_wins_over_a_lower_per_card_base",
+ "test_one_card_keeps_its_own_emitted_base_under_the_reserve",
+ ],
+)
+def test_the_reserve_sits_on_each_cards_own_base(user, base, reserve, per_device, by_device, want):
+ kw = {} if per_device is None else {"per_device": per_device}
+ flags = lc._sidecar_reserve_margin_flags(
+ ["--model-draft", "/d.gguf", "--fit", "on"],
+ user,
+ {},
+ base,
+ reserve,
+ base_by_device = by_device,
+ **kw,
+ )
+ assert flags == ["--fit-target", want]
+
+
+def _base_margins(
+ tmp_path,
+ memory = _MIXED,
+ **kw,
+):
+ (tmp_path / "base").mkdir()
+ backend, gguf = h.cards(tmp_path / "base", memory)
+ return h.margins(h.placement._launch(backend, gguf, **kw)["cmd"])
+
+
+def test_each_card_keeps_its_budget_margin_under_the_reserve(tmp_path, _budget):
+ base = _base_margins(tmp_path)
+ assert len(base) == 2 and base[0] != base[1]
+ backend, gguf = h.cards(tmp_path, _MIXED)
+ reserved = h.margins(h.dspark(backend, gguf, tmp_path))
+ assert len(reserved) == 2
+ assert all(r > b for r, b in zip(reserved, base))
+ assert reserved[1] < 24_576
+
+
[email protected]("main_gpu", [[], ["--main-gpu", "1"]])
+def test_split_mode_none_keeps_the_other_cards_margins(tmp_path, _budget, main_gpu):
+ """llama.cpp keeps only devices[main_gpu] and the fitter reads margin 0 for it, so the
+ reserve goes in entry 0 whichever card is the main one."""
+ extras = ["--split-mode", "none", *main_gpu]
+ base = _base_margins(tmp_path, extra_args = extras)
+ backend, gguf = h.cards(tmp_path, _MIXED)
+ reserved = h.margins(h.dspark(backend, gguf, tmp_path, extra_args = extras))
+ assert len(reserved) == 2
+ assert reserved[0] >= base[0] + 8 * 1024
+ assert reserved[1] == base[1]
+
+
+def test_one_local_card_with_rpc_gets_no_broadcast_reserve(tmp_path, _budget):
+ """A scalar margin is broadcast to the remote devices too."""
+ extras = ["--rpc", "10.0.0.2:50052"]
+ (tmp_path / "base").mkdir()
+ backend, gguf = h.cuda_backend(tmp_path / "base", memory = _MIXED[:1])
+ base = h.margins(h.placement._launch(backend, gguf, extra_args = extras)["cmd"])
+ backend, gguf = h.cuda_backend(tmp_path, memory = _MIXED[:1])
+ assert h.margins(h.dspark(backend, gguf, tmp_path, extra_args = extras)) == base
+
+
+def test_an_rpc_launch_gets_no_positional_reserve_list(tmp_path, _budget):
+ backend, gguf = h.cards(tmp_path, _MIXED)
+ cmd = h.dspark(backend, gguf, tmp_path, extra_args = ["--rpc", "10.0.0.2:50052"])
+ assert len(h.margins(cmd)) == 1
+
+
+def test_the_text_only_retry_leaves_the_reserve_pair_alone():
+ source = "".join(inspect.getsource(B.load_model).split())
+ at = source.index("cmd=self._strip_mmproj_args(_vision_gpu_cmd)")
+ window = source[at : at + 400]
+ assert "notin_sidecar_override_pairs" in window
+ assert window.index("notin_sidecar_override_pairs") < window.index(
+ "_per_device_fit_target_argv("
+ )
+
+
[email protected]("draft_device, reserved", [("Vulkan1", False), ("Vulkan0", True)])
+def test_a_draft_pinned_to_another_vulkan_device_is_not_reserved_on_the_target(
+ tmp_path, draft_device, reserved
+):
+ backend, gguf = h.cuda_backend(tmp_path, memory = TWO, vulkan = True)
+ cmd = h.dspark(
+ backend, gguf, tmp_path, gpu_ids = [0], extra_args = ["--device-draft", draft_device]
+ )
+ assert (h.margin(cmd) >= FLOOR) is reserved
+
+
[email protected]("draft_device, reserved", [("Vulkan0", True), ("Vulkan1", False)])
+def test_a_stale_device_extra_does_not_decide_the_draft_device(tmp_path, draft_device, reserved):
+ """gpu_ids=[0] owns placement and strips the stale --device Vulkan1 at emission."""
+ backend, gguf = h.cuda_backend(tmp_path, memory = TWO, vulkan = True)
+ extras = ["--device", "Vulkan1", "--device-draft", draft_device]
+ cmd = h.dspark(backend, gguf, tmp_path, gpu_ids = [0], extra_args = extras)
+ assert (h.margin(cmd) >= FLOOR) is reserved
+
+
+ "pin, reserved", [("CUDA0", True), ("CUDA1", False), ("Vulkan0", False), ("RPC0", False)]
+)
+def test_a_cuda_draft_pin_on_the_only_target_card_keeps_the_reserve(tmp_path, pin, reserved):
+ """The child's mask numbers the single target card 0, so CUDA0 is the target."""
+ backend, gguf = h.cuda_backend(tmp_path)
+ cmd = h.dspark(backend, gguf, tmp_path, extra_args = ["--device-draft", pin])
+ assert "--model-draft" in cmd
+ assert (h.margin(cmd) >= FLOOR) is reserved
+
+
+ "extras, pin, reserved",
+ [
+ ([], "Vulkan1", True),
+ ([], "Vulkan0", True),
+ (["--split-mode", "none"], "Vulkan1", False),
+ (["--split-mode", "none"], "Vulkan0", True),
+ ],
+)
+def test_a_pin_on_any_studio_chosen_vulkan_card_is_on_the_target(tmp_path, extras, pin, reserved):
+ backend, gguf = h.cuda_backend(tmp_path, memory = TWO, vulkan = True)
+ cmd = h.dspark(
+ backend, gguf, tmp_path, gpu_ids = [0, 1], extra_args = [*extras, "--device-draft", pin]
+ )
+ assert (h.margin(cmd) >= FLOOR) is reserved
+
+
[email protected]("pin, reserved", [("CUDA1", True), ("CUDA0", False)])
+def test_an_inherited_target_device_decides_the_draft_pin(tmp_path, monkeypatch, pin, reserved):
+ monkeypatch.setenv("LLAMA_ARG_DEVICE", "CUDA1")
+ backend, gguf = h.cuda_backend(tmp_path, memory = TWO)
+ cmd = h.dspark(backend, gguf, tmp_path, extra_args = ["--spec-draft-device", pin])
+ assert (h.margin(cmd) >= FLOOR) is reserved
+
+
+ "argv, devices, on_target",
+ [
+ # The masked child's device 0 is the target.
+ (["--spec-draft-device", "CUDA0"], None, True),
+ (["--spec-draft-device", "CUDA1"], None, False),
+ # A companion widening: the target is --device CUDA0, the drafter its CUDA1.
+ (["--device", "CUDA0", "--spec-draft-device", "CUDA1"], "CUDA0", False),
+ (["--device", "CUDA1", "--spec-draft-device", "CUDA1"], "CUDA1", True),
+ # Settled at planning, where the backend is known.
+ (["--spec-draft-device", "RPC0"], None, True),
+ (["--spec-draft-device", "none"], None, True),
+ ],
+)
+def test_the_spawned_argv_decides_a_cuda_draft_pin(argv, devices, on_target):
+ assert lc._spawn_draft_on_target(argv, devices) is on_target
+
+
+def test_a_repeated_device_is_one_card_for_the_draft_pin():
+ argv = ["--device", "CUDA0,CUDA0", "--device-draft", "CUDA0"]
+ assert lc._spawn_draft_on_target(argv, "CUDA0,CUDA0")
+ assert not lc._spawn_draft_on_target(["--device-draft", "CUDA1"], "CUDA0,CUDA0")
+
+
+def test_a_repeated_draft_pin_is_one_card():
+ assert lc._spawn_draft_on_target(
+ ["--device", "CUDA0", "--device-draft", "CUDA0,CUDA0"], "CUDA0"
+ )
+
+
+_NONE_MAIN_1 = ["--split-mode", "none", "--main-gpu", "1"]
+_BOTH = ["--device", "CUDA0,CUDA1"]
+
+
+ "argv, devices, on_target",
+ [
+ # --split-mode none puts the target on --main-gpu, not device 0.
+ ([*_NONE_MAIN_1, "--spec-draft-device", "CUDA1"], None, True),
+ ([*_NONE_MAIN_1, "--spec-draft-device", "CUDA0"], None, False),
+ ([*_BOTH, *_NONE_MAIN_1, "--spec-draft-device", "CUDA1"], "CUDA0,CUDA1", True),
+ ([*_BOTH, *_NONE_MAIN_1, "--spec-draft-device", "CUDA0"], "CUDA0,CUDA1", False),
+ ],
+)
+def test_split_mode_none_compares_the_pin_with_the_main_gpu(argv, devices, on_target):
+ assert lc._spawn_draft_on_target(argv, devices, {}) is on_target
+
+
[email protected]("pin, on_target", [("CUDA0", True), ("CUDA1", False)])
+def test_split_mode_none_defaults_to_the_lists_first_device(pin, on_target):
+ argv = [*_BOTH, "--split-mode", "none", "--spec-draft-device", pin]
+ assert lc._spawn_draft_on_target(argv, "CUDA0,CUDA1", {}) is on_target
+
+
+def test_split_mode_none_keeps_the_whole_reserve_on_the_main_card(tmp_path):
+ backend, gguf = h.cards(tmp_path)
+ assert h.margin(h.dspark(backend, gguf, tmp_path, extra_args = ["--split-mode", "none"])) >= FLOOR
+
+
[email protected]("pin, reserved", [("CUDA0", True), ("CUDA1", False)])
+def test_split_mode_none_narrows_an_inherited_device_list_at_planning(
+ tmp_path, monkeypatch, pin, reserved
+):
+ monkeypatch.setenv("LLAMA_ARG_DEVICE", "CUDA0,CUDA1")
+ backend, gguf = h.cuda_backend(tmp_path, memory = TWO)
+ extras = ["--split-mode", "none", "--spec-draft-device", pin]
+ assert (h.margin(h.dspark(backend, gguf, tmp_path, extra_args = extras)) >= FLOOR) is reserved
+
+
+def test_planning_drops_an_inherited_split_mode_the_launch_clears(tmp_path, monkeypatch):
+ """A layer launch clears an inherited non-layer LLAMA_ARG_SPLIT_MODE before spawning, so
+ planning must not narrow the target to the main card on it."""
+ monkeypatch.setenv("LLAMA_ARG_DEVICE", "CUDA0,CUDA1")
+ monkeypatch.setenv("LLAMA_ARG_SPLIT_MODE", "none")
+ backend, gguf = h.cards(tmp_path)
+ assert (
+ h.margin(h.dspark(backend, gguf, tmp_path, extra_args = ["--device-draft", "CUDA1"])) >= FLOOR
+ )
+
+
+ "extras, at",
+ [
+ ([*_BOTH, "--spec-draft-device", "CUDA0"], 0),
+ ([*_BOTH, "--spec-draft-device", "CUDA1"], 1),
+ (["--spec-draft-device", "CUDA1"], 1),
+ ],
+)
+def test_a_draft_pinned_to_one_target_card_reserves_there(tmp_path, extras, at):
+ backend, gguf = h.cards(tmp_path)
+ margins = h.margins(h.dspark(backend, gguf, tmp_path, extra_args = extras))
+ assert len(margins) == 2, margins
+ assert margins[at] >= FLOOR, margins
+ assert margins[1 - at] < 1024 + 1024, margins
+
+
+ "pin, devices, count, at",
+ [
+ ("Vulkan1", "Vulkan0,Vulkan1", 2, 1),
+ ("vulkan0", "Vulkan0,Vulkan1", 2, 0),
+ ("Vulkan2", "Vulkan0,Vulkan1", 2, None),
+ ("Vulkan1", None, 2, None),
+ ("CUDA1", None, 2, 1),
+ ("CUDA0,CUDA1", "CUDA0,CUDA1", 2, None),
+ ],
+)
+def test_a_draft_pin_resolves_to_its_target_card(pin, devices, count, at):
+ assert lc._draft_pin_device_index(pin, devices, count) == at
+
+
+ "args, want",
+ [
+ (["--split_mode=tensor"], "tensor"),
+ (["--split_mode", "row"], "row"),
+ (["-sm", "layer"], "layer"),
+ ],
+)
+def test_the_underscore_split_mode_spelling_is_read(args, want):
+ assert lc._extra_args_split_mode(args, {}) == want
+
+
+# The fit-sized report: planning and spawn agree on where a reserve is provable.
+
+
+ "extras, caps_fit_target, fit_sized",
+ [
+ ([], True, True),
+ (["--rpc", "10.0.0.2:50052"], True, False),
+ (_BOTH, True, False),
+ ([], False, False),
+ ],
+)
+def test_fit_sized_is_reported_only_where_the_spawn_can_price(
+ tmp_path, extras, caps_fit_target, fit_sized
+):
+ backend, gguf = h.cards(tmp_path)
+ caps = dict(h.CAPS, supports_fit_target = caps_fit_target)
+ backend.probe_server_capabilities = lambda _binary = None: dict(caps)
+ seen = h.spy_fit_sized(backend)
+ h.dspark(backend, gguf, tmp_path, extra_args = extras)
+ assert seen["fit_sized"] is fit_sized
+
+
+def _past_one_card(tmp_path):
+ backend, gguf = h.cards(tmp_path)
+ backend._get_gguf_size_bytes = lambda path: (8 if "dspark" in str(path) else 30) * GIB
+ return backend, gguf
+
+
+def test_device_extras_a_gpu_ids_pin_strips_do_not_block_the_report(tmp_path):
+ backend, gguf = _past_one_card(tmp_path)
+ seen = h.spy_fit_sized(backend)
+ cmd = h.dspark(backend, gguf, tmp_path, gpu_ids = [0, 1], extra_args = _BOTH)
+ assert "--device" not in cmd
+ assert len(h.margins(cmd)) == 2
+ assert seen["fit_sized"] is True
+
+
[email protected]("pin, reserved", [("CUDA0,CUDA1", False), ("CUDA0", True)])
+def test_a_multi_device_draft_pin_is_not_reported_as_reserved(tmp_path, pin, reserved):
+ backend, gguf = _past_one_card(tmp_path)
+ seen = h.spy_fit_sized(backend)
+ cmd = h.dspark(backend, gguf, tmp_path, extra_args = [*_BOTH, "--device-draft", pin])
+ assert (len(h.margins(cmd)) == 2) is reserved
+ assert seen["fit_sized"] is reserved
+
+
+def test_an_unpinned_multi_card_vulkan_launch_gets_no_positional_reserve(tmp_path):
+ """Vulkan orders its devices its own way, so neither planning nor spawn reserves."""
+ backend, gguf = h.cards(tmp_path, vulkan = True)
+ seen = h.spy_fit_sized(backend)
+ cmd = h.dspark(backend, gguf, tmp_path, **MANUAL_AUTO)
+ assert "--device" not in cmd
+ assert seen["fit_sized"] is False
+ assert len(h.margins(cmd)) == 1
+
+
+ "extras, vulkan, reserved",
+ [
+ (["--device-draft", "CUDA2"], False, False),
+ # A repeated target entry leaves the pin's position unknown.
+ (["--device", "CUDA0,CUDA1,CUDA0", "--device-draft", "CUDA0"], False, False),
+ # A Vulkan name with no target list to find it in.
+ (["--device-draft", "Vulkan0"], True, False),
+ (["--device-draft", "CUDA1"], False, True),
+ ([*_BOTH, "--device-draft", "CUDA1"], False, True),
+ ],
+)
+def test_planning_reports_only_a_draft_pin_the_spawn_resolves(tmp_path, extras, vulkan, reserved):
+ backend, gguf = h.cards(tmp_path, vulkan = vulkan)
+ seen = h.spy_fit_sized(backend)
+ cmd = h.dspark(
+ backend, gguf, tmp_path, extra_args = list(extras), **(MANUAL_AUTO if vulkan else {})
+ )
+ assert (len(h.margins(cmd)) == 2) is reserved
+ assert seen["fit_sized"] is reserved
+
+
+def test_planning_resolves_a_draft_pin_against_studios_vulkan_pin(tmp_path):
+ """gpu_ids on Vulkan reaches the child as --device Vulkan0,Vulkan1 (added after planning)."""
+ backend, gguf = h.cards(tmp_path, vulkan = True)
+ seen = h.spy_fit_sized(backend)
+ cmd = h.dspark(
+ backend, gguf, tmp_path, gpu_ids = [0, 1], extra_args = ["--device-draft", "Vulkan1"]
+ )
+ assert len(h.margins(cmd)) == 2
+ assert seen["fit_sized"] is True
+
+
+ "memory, extras, caps_fit_target",
+ [
+ # --rpc devices come first in the child's list: no reserve on one local card.
+ (h.ONE, ["--rpc", "10.0.0.2:50052"], True),
+ (h.ONE, [], False),
+ # --split-mode none keeps the target on card 0, the drafter on card 1.
+ (TWO, ["--split-mode", "none", "--device-draft", "CUDA1"], True),
+ ],
+)
+def test_a_reserve_the_spawn_drops_is_reported_as_missing(
+ tmp_path, monkeypatch, memory, extras, caps_fit_target
+):
+ backend, gguf = h.cards(tmp_path, memory)
+ caps = dict(h.CAPS, supports_fit_target = caps_fit_target)
+ backend.probe_server_capabilities = lambda _binary = None: dict(caps)
+ seen = h.spy_fit_sized(backend)
+ messages = h.info_messages(monkeypatch)
+ cmd = h.dspark(backend, gguf, tmp_path, extra_args = list(extras))
+ assert h.margin(cmd) < 1024 + 4 * 1024
+ assert seen["fit_sized"] is False
+ assert not any("keeps" in m and "free for it" in m for m in messages)
+ assert any("is not reserved" in m for m in messages)
+
+
+def test_a_dflash_reserve_the_spawn_drops_is_reported_as_missing(tmp_path, monkeypatch):
+ backend, gguf = h.cards(tmp_path, h.ONE)
+ messages = h.info_messages(monkeypatch)
+ cmd = h.placement._launch(
+ backend,
+ gguf,
+ speculative_type = "dflash",
+ dflash_draft_path = h.sidecar(tmp_path, "dflash"),
+ extra_args = ["--rpc", "10.0.0.2:50052"],
+ )["cmd"]
+ assert "draft-dflash" in cmd
+ assert any(m.startswith("DFlash") and "is not reserved" in m for m in messages)
+ assert not any("keeps" in m and "free for it" in m for m in messages)
+
+
+def test_a_renumbered_companion_pin_is_reported_as_reserved(tmp_path):
+ """gpu_ids [1, 2] masks the child to those cards, so the request's CUDA2 reaches the
+ child as CUDA1."""
+ backend, gguf = h.cards(tmp_path, (*TWO, (2, 24_000, 24_000)))
+ seen = h.spy_fit_sized(backend)
+ cmd = h.dspark(backend, gguf, tmp_path, gpu_ids = [1, 2], extra_args = ["--device-draft", "CUDA2"])
+ margins = h.margins(cmd)
+ assert len(margins) == 2 and max(margins) >= FLOOR
+ assert seen["fit_sized"] is True
+
+
+def test_the_start_log_shows_the_argv_spawned(tmp_path, monkeypatch):
+ """The per-card reserve is spelled at spawn, so the logged command must be that one."""
+ backend, gguf = h.cards(tmp_path)
+ messages = h.info_messages(monkeypatch)
+ spelled = h.last(h.dspark(backend, gguf, tmp_path), "--fit-target")
+ assert "," in str(spelled)
+ starts = [m for m in messages if m.startswith("Starting llama-server:")]
+ assert starts and f"--fit-target {spelled}" in starts[-1]
+
+
+def test_auto_drops_a_drafter_pinned_across_both_target_cards(tmp_path):
+ backend, gguf, path = h.placement._tight_vram_backend(tmp_path, drafter_gb = 12.0)
+ backend._get_gpu_memory = lambda _binary = None, **_kw: [(0, 12288, 12288), (1, 12288, 12288)]
+ backend._get_gpu_free_memory = lambda _binary = None, **_kw: [(0, 12288), (1, 12288)]
+ backend._select_gpus = lambda *a, **k: ([0, 1], False)
+ backend._select_gpus_split_aware = lambda *a, **k: ([0, 1], False)
+ backend._arch_gate_survivors = lambda binary = None: []
+ cmd = h.placement._launch_auto_8k(
+ backend, gguf, dspark_draft_path = str(path), extra_args = ["--device-draft", "CUDA0,CUDA1"]
+ )["cmd"]
+ assert "--model-draft" not in cmd
+ assert backend.spec_fallback_reason == "drafter_no_vram"
+
+
+# Host-resident draft KV (-nkvo) leaves a card's reserve only on positive evidence that
+# the drafter's card is discrete; host memory is an APU's / iGPU's VRAM.
+
+_DGPU = ["CUDA0", "CUDA1"]
+_VULKAN = ["Vulkan0", "Vulkan1"]
+
+
+def test_a_discrete_card_picked_beside_an_apu_leaves_host_kv_out(tmp_path, monkeypatch):
+ delta = h.host_kv_delta(tmp_path, monkeypatch, memory = TWO, listed = _DGPU, apu = 0, gpu_ids = [1])
+ assert delta >= 760
+
+
[email protected]("device, shared", [("Vulkan0", True), ("Vulkan1", False)])
+def test_a_vulkan_igpu_picked_by_device_keeps_host_kv(tmp_path, monkeypatch, device, shared):
+ """Vulkan iGPU 0 (reports total 0) plus dGPU 1."""
+ delta = h.host_kv_delta(
+ tmp_path,
+ monkeypatch,
+ memory = _VK_IGPU,
+ vulkan = True,
+ listed = _VULKAN,
+ extras = ["--device", device],
+ **MANUAL_AUTO,
+ )
+ assert delta == 0 if shared else delta >= 760
+
+
[email protected]("device, discrete", [("CUDA1", True), ("ROCm1", True), ("CUDA0", False)])
+def test_a_named_card_beside_an_apu_decides_host_kv(tmp_path, monkeypatch, device, discrete):
+ delta = h.host_kv_delta(
+ tmp_path,
+ monkeypatch,
+ memory = TWO,
+ listed = [f"{device[:-1]}0", f"{device[:-1]}1"],
+ apu = 0,
+ extras = ["--device", device],
+ **MANUAL_AUTO,
+ )
+ assert delta >= 760 if discrete else delta == 0
+
+
+ "listed",
+ [_VULKAN, None],
+ ids = [
+ # Vulkan Auto drops iGPU 0 before the shared-card set is built: unknown, not discrete.
+ "test_a_vulkan_igpu_auto_filtered_out_keeps_host_kv",
+ # No --list-devices answer: the planned discrete card says nothing about the child's.
+ "test_an_override_the_build_does_not_list_keeps_host_kv",
+ ],
+)
+def test_an_unproven_vulkan_device_keeps_host_kv(tmp_path, monkeypatch, listed):
+ delta = h.host_kv_delta(
+ tmp_path,
+ monkeypatch,
+ memory = _VK_IGPU,
+ vulkan = True,
+ listed = listed,
+ apu = None,
+ extras = ["--device", "Vulkan0"],
+ )
+ assert delta == 0
+
+
+def test_a_one_card_draft_pin_on_the_discrete_card_leaves_host_kv_out(tmp_path, monkeypatch):
+ delta = h.host_kv_delta(
+ tmp_path, monkeypatch, memory = TWO, listed = _DGPU, apu = 0, extras = ["--device-draft", "CUDA1"]
+ )
+ assert delta >= 760
+
+
[email protected]("devices, left_out", [("CUDA1,CUDA0", True), ("CUDA0,CUDA1", False)])
+def test_split_mode_none_narrows_to_the_main_card(tmp_path, monkeypatch, devices, left_out):
+ """--split-mode none keeps devices[main_gpu], 0 by default: the first listed card."""
+ delta = h.host_kv_delta(
+ tmp_path,
+ monkeypatch,
+ memory = TWO,
+ listed = _DGPU,
+ apu = 0,
+ extras = ["--device", devices, "--split-mode", "none"],
+ )
+ assert delta >= 760 if left_out else delta == 0
+
+
+ "order, left_out",
+ [
+ # CUDA's default order: the child's CUDA1 need not be nvidia-smi's card 1.
+ ("FASTEST_FIRST", False),
+ ("PCI_BUS_ID", True),
+ ],
+)
+def test_a_mixed_host_narrows_only_through_a_proven_child_order(
+ tmp_path, monkeypatch, _pci_ids, order, left_out
+):
+ monkeypatch.setenv("CUDA_DEVICE_ORDER", order)
+ delta = h.host_kv_delta(
+ tmp_path, monkeypatch, memory = TWO, listed = _DGPU, apu = 0, extras = ["--device", "CUDA1"]
+ )
+ assert delta >= 760 if left_out else delta == 0
+
+
+def test_a_uuid_mask_is_not_read_as_physical_ids(tmp_path, monkeypatch, _pci_ids):
+ """CUDA_VISIBLE_DEVICES=GPU-<dgpu>,GPU-<apu>: the child's CUDA1 is the APU, which
+ _physical_ordinals (numeric masks only) would leave as card 1, the discrete one."""
+ monkeypatch.setenv("CUDA_VISIBLE_DEVICES", "GPU-11111111,GPU-00000000")
+ delta = h.host_kv_delta(
+ tmp_path, monkeypatch, memory = TWO, listed = _DGPU, apu = 0, extras = ["--device", "CUDA1"]
+ )
+ assert delta == 0
+
+
+def test_another_backends_device_name_is_not_read_as_cuda(tmp_path, monkeypatch, _pci_ids):
+ """A CUDA + Vulkan build lists Vulkan1 too, but it is not CUDA card 1."""
+ delta = h.host_kv_delta(
+ tmp_path,
+ monkeypatch,
+ memory = TWO,
+ listed = [*_DGPU, *_VULKAN],
+ apu = 0,
+ extras = ["--device", "Vulkan1"],
+ )
+ assert delta == 0
+
+
+def test_an_all_discrete_host_needs_no_ordinal_mapping(tmp_path, monkeypatch, _pci_ids):
+ """Control: no shared card anywhere, even under an order the child does not share."""
+ monkeypatch.setenv("CUDA_DEVICE_ORDER", "FASTEST_FIRST")
+ delta = h.host_kv_delta(
+ tmp_path, monkeypatch, memory = TWO, listed = _DGPU, apu = None, extras = ["--device", "CUDA1"]
+ )
+ assert delta >= 760
diff --git a/studio/setup.sh b/studio/setup.sh
index 0368157f831..2a1d601436e 100755
--- a/studio/setup.sh
+++ b/studio/setup.sh
@@ -5194,6 +5194,11 @@ else
if [ "$BUILD_OK" = true ]; then
run_quiet_no_exit "build llama-quantize" cmake --build "$_BUILD_TMP/build" --config Release --target llama-quantize -j"$NCPU" || true
+ # Best-effort on macOS: the Metal memory probe the prebuilt bundles ship too.
+ # Without it the Metal fit keeps a big model's input embeddings charged.
+ if [ "$(uname -s 2>/dev/null)" = "Darwin" ]; then
+ run_quiet_no_exit "build llama-fit-params" cmake --build "$_BUILD_TMP/build" --config Release --target llama-fit-params -j"$NCPU" || true
+ fi
# Best-effort: the target exists only on llama.cpp trees carrying the diffusion example.
run_quiet_no_exit "build diffusion visual server" cmake --build "$_BUILD_TMP/build" --config Release --target llama-diffusion-gemma-visual-server -j"$NCPU" || true
fi