Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
Studio: reserve the DSpark/DFlash sidecar in --fit-target and recover from a Metal out-of-memory (#14431)
Daniel Han215039af8d9a
diff --git a/studio/backend/core/inference/llama_cpp.py b/studio/backend/core/inference/llama_cpp.pyindex ff84eda8ee8..c654bc6a16b 100644--- a/studio/backend/core/inference/llama_cpp.py+++ b/studio/backend/core/inference/llama_cpp.py@@ -16,6 +16,7 @@ import ctypesimport errnoimport functoolsimport hashlib+import itertoolsimport jsonimport loggingimport math@@ -657,7 +658,33 @@ class _LlamaStreamCancelled(Exception):# vulkan-hpp formats DeviceLostError as "vk::Queue::submit: ErrorDeviceLost"._GPU_DEVICE_LOST_MARKERS = ("ErrorDeviceLost", "VK_ERROR_DEVICE_LOST")+# ggml-metal latches a failed command buffer ("recreate the backend to recover"); the child then+# answers "Compute error.".+_METAL_BACKEND_POISONED_MARKERS = (+ "backend is in error state from a previous command buffer failure",+ "kIOGPUCommandBufferCallbackErrorOutOfMemory",+)+_DRAFT_CONTEXT_FAILURE_MARKERS = (+ "llama_process(ctx_dft) failed",+ "failed to process speculative batch",+)_DEVICE_LOST_LOG_GRACE_S = 2.0+# Run after a terminal Metal OOM unload, outside every backend lock (routes/inference.py registers+# them).+_TERMINAL_RETIREMENT_HOOKS: list = []++# Set during automatic recovery loads: only those inherit a Metal OOM strike.+_AUTO_RECOVERY_LOAD = threading.local()+++def _automatic_recovery_load():+ prev = getattr(_AUTO_RECOVERY_LOAD, "active", False)+ _AUTO_RECOVERY_LOAD.active = True+ try:+ yield+ finally:+ _AUTO_RECOVERY_LOAD.active = prevdef _is_gpu_device_lost(exc: BaseException) -> bool:@@ -4675,6 +4702,179 @@ def _binary_has_marker(path: str, _mtime_ns: int, _size: int, marker: bytes) ->return False+# No --help flag or build number announces it: read the trace string the new fit path compiles in.+_FIT_MEASURES_BORROWED_DRAFT_MARKER = (+ b"measuring the extra model next to an allocation-free target context"+)+_LLAMA_COMMON_LIB_NAMES = (+ "libllama-common.so.0",+ "libllama-common.so",+ "libllama-common.0.dylib",+ "libllama-common.dylib",+ "llama-common.dll",+)+# The server links libllama-server-impl, which links libllama-common.+_LLAMA_SERVER_IMPL_LIB_NAMES = (+ "libllama-server-impl.so",+ "libllama-server-impl.dylib",+ "llama-server-impl.dll",+)+++# (path, mtime_ns, size) -> embedded library names; the server binary is ~100 MB.+_LINKED_LIB_NAMES_CACHE: dict = {}+++def _linked_llama_libs(binary: Path, lib_dir: Path) -> list:+ """The server-impl / common libraries ``binary`` links by name (DT_NEEDED, LC_LOAD_DYLIB, PE+ imports), not stray siblings."""+ try:+ st = binary.stat()+ key = (str(binary), st.st_mtime_ns, st.st_size)+ except OSError:+ return []+ embedded = _LINKED_LIB_NAMES_CACHE.get(key)+ if embedded is None:+ try:+ data = binary.read_bytes()+ except OSError:+ return []+ embedded = tuple(+ tuple(n for n in names if n.encode() in data)+ for names in (_LLAMA_SERVER_IMPL_LIB_NAMES, _LLAMA_COMMON_LIB_NAMES)+ )+ _LINKED_LIB_NAMES_CACHE[key] = embedded+ linked = []+ for names in embedded:+ hit = next((n for n in names if (lib_dir / n).exists()), None)+ if hit is not None:+ linked.append(lib_dir / hit)+ return linked+++_CUDA_LIKE_DEVICE_RE = re.compile(r"(?i)(cuda|rocm|hip)\d+$")+++def _spawn_draft_on_target(+ run_cmd: List[str],+ spawn_devices: Optional[str],+ env: Optional[Mapping[str, str]] = None,+) -> bool:+ """Whether the spawned argv's CUDA / ROCm draft pin names the target's device."""+ pin = _extra_args_draft_device_pin(run_cmd)+ if pin is not None:+ distinct = {d.strip() for d in pin.split(",") if d.strip()}+ pin = distinct.pop() if len(distinct) == 1 else pin+ if pin is None or not _CUDA_LIKE_DEVICE_RE.match(pin.strip()):+ return True+ if spawn_devices:+ named = [d.strip().lower() for d in spawn_devices.split(",") if d.strip()]+ if _split_mode_confines_to_one_device(run_cmd, env):+ # Only devices[main_gpu] is kept (main_gpu 0 by default).+ main = LlamaCppBackend._effective_main_gpu(run_cmd, env)+ main = 0 if main is None else main+ return 0 <= main < len(named) and pin.strip().lower() == named[main]+ return set(named) == {d.strip().lower() for d in pin.split(",") if d.strip()}+ return int(re.sub(r"\D", "", pin)) == (LlamaCppBackend._effective_main_gpu(run_cmd, env) or 0)+++def _sidecar_borrows_output(path: Optional[str]) -> bool:+ """Whether the draft's logits use the target's output head (src/models/dflash.cpp); unknown+ counts as borrowed."""+ if not path:+ return True+ try:+ from utils.models.gguf_metadata import _gguf_shard_has_named_tensor+ from utils.models.model_config import colocated_split_shards++ shards, complete = colocated_split_shards(Path(path))+ return not (+ complete+ and any(+ _gguf_shard_has_named_tensor(str(shard), "output.weight") is True+ for shard in shards+ )+ )+ except Exception:+ return True+++def _sidecar_borrows_target_tensors(path: Optional[str]) -> bool:+ """Whether the draft borrows token_embd or output, which the fitter cannot measure+ (src/llama-context.cpp)."""+ if not path:+ return True+ try:+ from utils.models.gguf_metadata import _gguf_shard_has_named_tensor+ from utils.models.model_config import colocated_split_shards++ shards, complete = colocated_split_shards(Path(path))+ if not complete or not shards:+ return True+ return any(+ not any(_gguf_shard_has_named_tensor(str(shard), name) is True for shard in shards)+ for name in ("token_embd.weight", "output.weight")+ )+ except Exception:+ return True+++def _draft_pin_device_index(+ pin: Optional[str], spawn_devices: Optional[str], device_count: int+) -> Optional[int]:+ """The child device index a one-card draft pin names, or None."""+ names = {d.strip().lower() for d in (pin or "").split(",") if d.strip()}+ if len(names) != 1:+ return None+ (name,) = names+ if spawn_devices:+ listed = [d.strip().lower() for d in spawn_devices.split(",") if d.strip()]+ if name not in listed or len(set(listed)) != len(listed):+ return None+ return listed.index(name)+ if not _CUDA_LIKE_DEVICE_RE.match(name):+ return None+ k = int(re.sub(r"\D", "", name))+ return k if 0 <= k < device_count else None+++def _distinct_device_count(devices) -> int:+ """Distinct names in a ``--device`` list: a repeated card is one memory pool."""+ return len({d.strip().lower() for d in str(devices).split(",") if d.strip()})+++def _fit_measures_borrowed_drafts(bin_path: Optional[str]) -> bool:+ """Whether ``bin_path``'s fitter sizes a borrowed-tensor draft itself; fails closed."""+ if not bin_path:+ return False+ candidates = [Path(bin_path)]+ try:+ # A wrapper script execs the real server.+ server = Path(_resolve_llama_binary(bin_path))+ candidates.append(server)+ lib_dir = _llama_lib_dir(bin_path)+ for lib in _linked_llama_libs(server, lib_dir):+ candidates.append(lib)+ candidates.extend(l for l in _linked_llama_libs(lib, lib_dir) if l not in candidates)+ except Exception: # noqa: BLE001 -- an unreadable lib dir is "not known"+ pass+ seen: set = set()+ for candidate in candidates:+ try:+ candidate = candidate.resolve()+ if candidate in seen or not candidate.is_file():+ continue+ seen.add(candidate)+ st = candidate.stat()+ except OSError:+ continue+ if _binary_has_marker(+ str(candidate), st.st_mtime_ns, st.st_size, _FIT_MEASURES_BORROWED_DRAFT_MARKER+ ):+ return True+ return False++def _swa_full_from_args_or_env(extra_args: Optional[Iterable[str]], env: Optional[Mapping[str, str]] = None) -> bool:@@ -5606,6 +5806,155 @@ def _extra_args_mtp_draft_path(return _extra_args_mtp_draft_source(extra_args, env)[0]+def _single_device_fit_margin(+ args: Iterable[str], env: Optional[Mapping[str, str]]+) -> Optional[float]:+ """The margin the one launched device keeps: the FIRST entry of a ``--fit-target`` list.++ ``fit_target_margin_in`` takes the largest entry because it prices every device it+ credits; a single-device launch only ever uses device 0's entry (common/arg.cpp), so a+ larger later entry must not be added under the sidecar reserve."""+ raw = _last_flag_value(args, {"-fitt", "--fit-target"})+ if raw is None and env:+ raw = env.get("LLAMA_ARG_FIT_TARGET")+ if raw is None:+ return None+ first = str(raw).replace("/", ",").split(",")[0].strip()+ try:+ return float(first)+ except ValueError:+ return None+++def _fit_target_values(user_args: Iterable[str], env: Optional[Mapping[str, str]]) -> list[str]:+ """Every --fit-target value llama.cpp applies, env first then argv order."""+ found = []+ if env and env.get("LLAMA_ARG_FIT_TARGET"):+ found.append(str(env["LLAMA_ARG_FIT_TARGET"]))+ tokens = [str(a) for a in user_args]+ for i, raw in enumerate(tokens):+ _, eq, inline = raw.partition("=")+ if _flag_name(raw) in ("-fitt", "--fit-target"):+ value = inline if eq else (tokens[i + 1] if i + 1 < len(tokens) else None)+ if value is not None:+ found.append(value)+ return found+++def _fit_margins_by_device(+ user_args: Optional[Iterable[str]],+ env: Optional[Mapping[str, str]],+ base_margin_mib: float,+ count: int,+) -> list[float]:+ """Each card's user --fit-target margin as common/arg.cpp applies it; unset cards keep+ ``base_margin_mib``."""+ margins = [float(base_margin_mib)] * count+ for raw in _fit_target_values(user_args or (), env):+ entries = [e.strip() for e in str(raw).replace("/", ",").split(",") if e.strip()]+ try:+ values = [float(e) for e in entries]+ except ValueError:+ continue+ if len(values) == 1:+ margins = values * count+ else:+ margins[: len(values)] = values[:count]+ return margins+++def _sidecar_reserve_margin_flags(+ child_args: Iterable[str],+ user_args: Optional[Iterable[str]],+ env: Optional[Mapping[str, str]],+ base_margin_mib: float,+ reserve_mib: int,+ *,+ per_device: Optional[list[int]] = None,+ base_by_device: Optional[list[float]] = None,+) -> list[str]:+ """``--fit-target`` keeping an unsized sidecar's reserve free on top of the user's margin, read+ off the child's argv + env."""+ args = [str(a) for a in child_args]+ if not reserve_mib or not _extra_args_mtp_draft_path(args, env = env):+ return []+ if not fit_is_effectively_on(args, env):+ return []+ # A scalar --fit-target is broadcast to every device, so a multi-device launch needs a+ # per-device list.+ devices = _extra_args_main_device(args) or (os.environ if env is None else env).get(+ "LLAMA_ARG_DEVICE"+ )+ if (+ not per_device+ and devices+ and _distinct_device_count(devices) > 1+ and not _split_mode_confines_to_one_device(args, env)+ ):+ return []+ if per_device:+ bases = (+ base_by_device+ if base_by_device and len(base_by_device) == len(per_device)+ else [base_margin_mib] * len(per_device)+ )+ margins = _fit_margins_by_device(user_args, env, 0.0, len(per_device))+ want_list = ",".join(+ str(math.ceil(max(b, m) + r)) for b, m, r in zip(bases, margins, per_device)+ )+ if _last_flag_value(args, {"-fitt", "--fit-target"}) == want_list:+ return []+ return ["--fit-target", want_list]+ # Studio's per-card margin, not the host-wide scalar a lowered budget sizes by the largest card.+ if base_by_device and len(set(base_by_device)) == 1:+ base_margin_mib = base_by_device[0]+ want = math.ceil(+ max(base_margin_mib, _single_device_fit_margin(user_args or (), env) or 0.0) + reserve_mib+ )+ have = _single_device_fit_margin(args, None)+ if have is not None and have >= want:+ return []+ return ["--fit-target", str(want)]+++def _drop_sidecar_margins(+ cmd: List[str],+ integrity_flags: List[str],+ rebuilt_integrity: List[str],+ override_pairs: Iterable[List[str]],+ *,+ user_args: Iterable[str] = (),+) -> List[str]:+ """``cmd`` without Studio's sidecar margins; a user's own pairs survive, so it is idempotent."""+ out = list(cmd)+ user = [str(a) for a in user_args]++ def _count(tokens: List[str], pair: List[str]) -> int:+ return sum(tokens[i : i + len(pair)] == pair for i in range(len(tokens) - len(pair) + 1))++ def _studio_owns(pair: List[str]) -> bool:+ return bool(pair) and _count(out, pair) > _count(user, pair)++ if "--fit-target" in integrity_flags:+ old_at = integrity_flags.index("--fit-target")+ old_margin = integrity_flags[old_at : old_at + 2]+ new_margin: List[str] = []+ if "--fit-target" in rebuilt_integrity:+ new_at = rebuilt_integrity.index("--fit-target")+ new_margin = rebuilt_integrity[new_at : new_at + 2]+ at = _subsequence_index(out, old_margin, 0)+ if out[at : at + 2] == old_margin and _studio_owns(old_margin):+ out[at : at + 2] = new_margin+ for pair in reversed(list(override_pairs)):+ if not _studio_owns(pair):+ continue+ for i in range(len(out) - len(pair), -1, -1):+ if out[i : i + len(pair)] == pair:+ del out[i : i + len(pair)]+ break+ return out++_HF_DRAFT_FLAGS = frozenset({"--spec-draft-hf", "-hfd", "-hfrd", "--hf-repo-draft"})_LOCAL_DRAFT_FLAGS = frozenset({"--model-draft", "--spec-draft-model", "-md"})@@ -5679,6 +6028,9 @@ _BREAKDOWN_HOST_ROW_RE = re.compile()+_METAL_FIT_PROBE_TIMEOUT_S = 120++def _parse_metal_memory_breakdown(output: str) -> Optional[tuple[int, int, int]]:"""Requires one Metal device row; host model MiB sums the model column of every host row."""rows: list[str] = []@@ -5792,6 +6144,115 @@ def _draft_is_split_across_host(return not (n_draft_layers and count > n_draft_layers)+_DRAFT_OVERRIDE_TENSOR_FLAGS = frozenset(+ {"-otd", "--override-tensor-draft", "--spec-draft-override-tensor"}+)+_DRAFT_CPU_MOE_FLAGS = frozenset({"--spec-draft-cpu-moe", "-cmoed", "--cpu-moe-draft"})+_DRAFT_N_CPU_MOE_FLAGS = frozenset(+ {"--spec-draft-n-cpu-moe", "--spec-draft-ncmoe", "-ncmoed", "--n-cpu-moe-draft"}+)+++def _draft_card_share_unknown(+ extra_args: Optional[Iterable[str]],+ env: Optional[Mapping[str, str]] = None,+ *,+ n_draft_layers: Optional[int] = None,+) -> bool:+ """True when the drafter is not wholly on the target's card, so its whole file is not+ the share to reserve there: a positive -ngld below its depth (split with the host), or+ a draft tensor override, which llama.cpp applies per weight while picking buffers (the+ same flags the fit-voiding list treats as placement changes), or the draft CPU-MoE+ switches, which llama.cpp turns into draft tensor overrides."""+ if _extra_args_set_any_flag(extra_args, _DRAFT_OVERRIDE_TENSOR_FLAGS | _DRAFT_CPU_MOE_FLAGS):+ return True+ source_env = os.environ if env is None else env+ if source_env.get("LLAMA_ARG_SPEC_DRAFT_CPU_MOE") in _LLAMA_ARG_TRUE_VALUES:+ return True+ # Each positive count appends draft CPU overrides, so any positive one is a placement.+ if _is_positive_int(source_env.get("LLAMA_ARG_SPEC_DRAFT_N_CPU_MOE")):+ return True+ args = [str(a) for a in extra_args] if extra_args else []+ for i, raw in enumerate(args):+ _, eq, inline = raw.partition("=")+ if _flag_name(raw) in _DRAFT_N_CPU_MOE_FLAGS and _is_positive_int(+ inline if eq else (args[i + 1] if i + 1 < len(args) else None)+ ):+ return True+ return _draft_is_split_across_host(extra_args, env, n_draft_layers = n_draft_layers)+++def _sidecar_child_ctx(+ args: Optional[Iterable[str]],+ env: Optional[Mapping[str, str]],+ native_ctx: Optional[int],+ n_parallel: Optional[int],+ kv_unified: bool,+) -> int:+ """The context the child allocates: last positive -c, else LLAMA_ARG_CTX_SIZE, else the native+ length."""+ ctx = parse_ctx_override(args)+ if ctx is None:+ try:+ ctx = int(+ str((os.environ if env is None else env).get("LLAMA_ARG_CTX_SIZE", "")).strip()+ )+ except (TypeError, ValueError):+ ctx = None+ if ctx and ctx > 0:+ return ctx+ # Without -c, --kv-unified-per-slot N sizes the pool to n_parallel * N+ # (tools/server/server.cpp).+ source_env = os.environ if env is None else env+ tokens = [str(a) for a in args] if args else []+ full_window = str(source_env.get("LLAMA_ARG_CTX_SIZE", "")).strip() == "0"+ if str(source_env.get("LLAMA_ARG_FIT_CTX", "")).strip():+ full_window = False+ per_slot = source_env.get("LLAMA_ARG_KV_UNIFIED_PER_SLOT")+ for i, raw in enumerate(tokens):+ flag = _flag_name(raw)+ if flag not in ("-c", "--ctx-size", "-fitc", "--fit-ctx", "--kv-unified-per-slot"):+ continue+ _, eq, inline = raw.partition("=")+ value = inline if eq else (tokens[i + 1] if i + 1 < len(tokens) else "")+ if flag == "--kv-unified-per-slot":+ per_slot = value+ elif flag in ("-fitc", "--fit-ctx"):+ full_window = False+ elif str(value).strip() == "0":+ full_window = True+ if _is_positive_int(per_slot) and not full_window:+ return int(str(per_slot).strip()) * max(1, int(n_parallel or 1))+ bound = int(native_ctx or 0) or _LLAMA_FIT_MIN_CTX+ if not kv_unified:+ bound *= max(1, int(n_parallel or 1))+ return bound+++def _draft_n_ubatch(+ args: Optional[Iterable[str]], env: Optional[Mapping[str, str]] = None+) -> Optional[int]:+ """A positive --spec-draft-n-ubatch (env twin overridden by argv), else None."""+ found = (os.environ if env is None else env).get("LLAMA_ARG_SPEC_DRAFT_N_UBATCH")+ tokens = [str(a) for a in args] if args else []+ for i, raw in enumerate(tokens):+ _, eq, inline = raw.partition("=")+ if _flag_name(raw) == "--spec-draft-n-ubatch":+ found = inline if eq else (tokens[i + 1] if i + 1 < len(tokens) else None)+ return int(str(found).strip()) if _is_positive_int(found) else None+++def _capped_draft_n_ubatch(args, env, ctx, *batch_defaults) -> Optional[int]:+ """--spec-draft-n-ubatch capped as llama_context caps it: at --batch-size, then a positive+ context. ``batch_defaults`` are _named_batch_sizes' fallbacks."""+ ub = _draft_n_ubatch(args, env)+ if ub:+ ub = min(ub, _named_batch_sizes(args, env, *batch_defaults)[0] & 0xFFFFFFFF)+ if ctx and ctx > 0:+ ub = min(ub, int(ctx))+ return ub++_SIDECAR_ADAPTER_FLAGS = ("--lora","--lora-scaled",@@ -6148,8 +6609,8 @@ def _extra_args_split_mode(found = str(raw).strip().lower()args = [str(a) for a in extra_args] if extra_args else []for i, arg in enumerate(args):- name, _, inline = arg.partition("=")- if name not in _SPLIT_MODE_FLAGS:+ _, _, inline = arg.partition("=")+ if _flag_name(arg) not in _SPLIT_MODE_FLAGS:continuevalue = inline if inline else (args[i + 1] if i + 1 < len(args) else "")if value.strip():@@ -6168,9 +6629,9 @@ def _extra_args_tensor_split(raw = str(env_value).strip()args = [str(a) for a in extra_args] if extra_args else []for i, arg in enumerate(args):- name, _, inline = arg.partition("=")- if name not in _TENSOR_SPLIT_FLAGS:+ if _flag_name(arg) not in _TENSOR_SPLIT_FLAGS:continue+ _, _, inline = arg.partition("=")value = inline if inline else (args[i + 1] if i + 1 < len(args) else "")if value.strip():raw = value.strip()@@ -7082,12 +7543,32 @@ def _arm_load_probe_memo() -> None:@contextlib.contextmanagerdef _load_probe_memo_scope():"""Guarantee the memo cannot outlive a load however it exits. Arming is the- narrower `_arm_load_probe_memo`, taken around the placement decision."""+ narrower `_arm_load_probe_memo`, taken around the placement decision. The whole+ load also keeps a llama-fit-params failure that is not cached across loads (a+ timeout, a crash) and the Metal fit lines already logged, so a second budget pass+ neither waits on the probe again nor repeats its message."""+ prev = (+ getattr(_LOAD_PROBE_STATE, "metal_transient", None),+ getattr(_LOAD_PROBE_STATE, "logged", None),+ )+ _LOAD_PROBE_STATE.metal_transient = {}+ _LOAD_PROBE_STATE.logged = set()try:yieldfinally:_LOAD_PROBE_STATE.armed = False_LOAD_PROBE_STATE.rows = None+ _LOAD_PROBE_STATE.metal_transient, _LOAD_PROBE_STATE.logged = prev+++def _info_once_per_load(msg: str, *args) -> None:+ """``logger.info`` that a load says once (_load_probe_memo_scope); always outside one."""+ seen = getattr(_LOAD_PROBE_STATE, "logged", None)+ if seen is not None:+ if (msg, args) in seen:+ return+ seen.add((msg, args))+ logger.info(msg, *args)# "Probed, no usable answer", kept apart from None = "not probed yet".@@ -7601,6 +8082,9 @@ class LlamaCppBackend:# True when the launch actually runs MTP (Unsloth- or user/env-driven);# gates the watchdog and recovery so pass-through MTP is covered.self._mtp_runtime_fallback_active = False+ # (load key, count) of Metal OOM retirements, and the key that stopped respawning.+ self._metal_oom_strikes = None+ self._metal_oom_terminal_key = Noneself._stdout_lines: list[str] = []self._stdout_thread: Optional[threading.Thread] = None# Wakes the health probe on subprocess readiness/exit, not left to the 500 ms fallback poll.@@ -7685,7 +8169,14 @@ class LlamaCppBackend:@propertydef is_active(self) -> bool:"""True if a llama-server process exists (loading or loaded)."""- return self._process is not None+ proc = self._process+ if proc is None:+ return False+ return not (+ not self._healthy+ and proc is getattr(self, "_retired_process", None)+ and getattr(self, "_metal_oom_terminal_key", None) is not None+ )def resident_memory_mib(self) -> Optional[int]:"""Best-effort MiB the loaded model holds: its GPU model buffers when known, else the@@ -9364,6 +9855,7 @@ class LlamaCppBackend:"flash_attn_takes_value": True,"supports_fit_ctx": False,"supports_fit_target": False,+ "fit_measures_borrowed_drafts": False,"supports_cache_ram": False,"supports_ctx_checkpoints": False,"ctx_checkpoints_flag": None,@@ -9399,16 +9891,26 @@ class LlamaCppBackend:mtime_ns = 0size = 0cache_key = (bin_path, mtime_ns, size)+ hit = Nonewith cls._capability_cache_lock:cached = cls._capability_cache.get(cache_key)if cached is not None:- if not cached.get("mtp_probe_inconclusive"):- return cachedretry_after = cls._capability_retry_after.get(cache_key)- if retry_after is not None and time.monotonic() < retry_after:- return cached- cls._capability_cache.pop(cache_key, None)- cls._capability_retry_after.pop(cache_key, None)+ if not cached.get("mtp_probe_inconclusive") or (+ retry_after is not None and time.monotonic() < retry_after+ ):+ hit = cached+ else:+ cls._capability_cache.pop(cache_key, None)+ cls._capability_retry_after.pop(cache_key, None)+ if hit is not None:+ # A copy: the cached dict is shared across threads.+ if hit.get("supports_fit_target"):+ hit = {+ **hit,+ "fit_measures_borrowed_drafts": _fit_measures_borrowed_drafts(bin_path),+ }+ return hitmtp_token: Optional[str] = Nonesupports_dspark = False@@ -9746,6 +10248,8 @@ class LlamaCppBackend:"flash_attn_takes_value": flash_attn_takes_value,"supports_fit_ctx": supports_fit_ctx,"supports_fit_target": supports_fit_target,+ "fit_measures_borrowed_drafts": supports_fit_target+ and _fit_measures_borrowed_drafts(bin_path),"supports_cache_ram": supports_cache_ram,"supports_ctx_checkpoints": supports_ctx_checkpoints,"ctx_checkpoints_flag": ctx_checkpoints_flag,@@ -12306,27 +12810,49 @@ class LlamaCppBackend:self, binary: Optional[str], model_path: Optional[str]) -> Optional[tuple[int, int, int]]:"""Measures full-offload model buffers with the sibling llama-fit-params; None if that probe- fails."""+ fails, with the reason in ``_metal_probe_failure``."""+ self._metal_probe_failure = Noneif not binary or not model_path:+ self._metal_probe_failure = "no binary or model path"return Nonetry:probe = _llama_lib_dir(binary) / "llama-fit-params"+ except OSError:+ self._metal_probe_failure = "llama-fit-params not found"+ return None+ try:probe_stat = probe.stat()+ except OSError:+ self._metal_probe_failure = f"llama-fit-params not found at {probe}"+ return None+ try:model_stat = os.stat(model_path)except OSError:+ self._metal_probe_failure = "model file unreadable"+ return None+ if os.name != "nt" and not os.access(probe, os.X_OK):+ self._metal_probe_failure = f"llama-fit-params at {probe} is not executable"return Nonekey = (str(probe),probe_stat.st_size,probe_stat.st_mtime_ns,+ probe_stat.st_mode,model_path,model_stat.st_size,model_stat.st_mtime_ns,)cached = getattr(self, "_metal_model_mib_cache", None)if cached is not None and cached[0] == key:+ self._metal_probe_failure = cached[2]return cached[1]+ transient = getattr(_LOAD_PROBE_STATE, "metal_transient", None)+ if transient is not None and key in transient:+ self._metal_probe_failure = transient[key]+ return Nonemeasured: Optional[tuple[int, int, int]] = None+ failure: Optional[str] = None+ cacheable = Truetry:env = self._llama_server_env_for_binary(str(binary))for name in tuple(env):@@ -12339,18 +12865,38 @@ class LlamaCppBackend:encoding = "utf-8",errors = "replace",# A cold Metal shader compile lands here instead of at the launch.- timeout = 120,+ timeout = _METAL_FIT_PROBE_TIMEOUT_S,check = False,env = env,**_windows_hidden_subprocess_kwargs(),)- if result.returncode == 0:- measured = _parse_metal_memory_breakdown(- (result.stdout or "") + "\n" + (result.stderr or "")- )+ output = (result.stdout or "") + "\n" + (result.stderr or "")+ if result.returncode != 0:+ failure = f"llama-fit-params exited with code {result.returncode}"+ cacheable = False+ else:+ measured = _parse_metal_memory_breakdown(output)+ if measured is None:+ failure = "llama-fit-params printed no Metal memory breakdown"+ if failure:+ tail = " | ".join(+ line.strip() for line in output.strip().splitlines()[-3:] if line.strip()+ )+ if tail:+ failure = f"{failure}: {tail[-300:]}"+ except subprocess.TimeoutExpired:+ failure = f"llama-fit-params timed out after {_METAL_FIT_PROBE_TIMEOUT_S} s"+ cacheable = Falseexcept Exception as e:- logger.debug(f"llama-fit-params memory probe failed: {e}")- self._metal_model_mib_cache = (key, measured)+ failure = f"llama-fit-params could not run: {e}"+ cacheable = False+ if failure:+ logger.debug(f"llama-fit-params memory probe failed: {failure}")+ self._metal_probe_failure = failure+ if cacheable:+ self._metal_model_mib_cache = (key, measured, failure)+ elif transient is not None:+ transient[key] = failurereturn measureddef _metal_demand_paged_embedding_bytes(@@ -12414,9 +12960,10 @@ class LlamaCppBackend:return 0measured = self._metal_measured_model_mib(binary, model_path)if measured is None:- logger.info(- "Metal fit: could not measure how llama.cpp maps this model (llama-fit-params "- "missing or unreadable), so its %.2f GB of input embeddings stay charged.",+ _info_once_per_load(+ "Metal fit: could not measure how llama.cpp maps this model (%s), so its "+ "%.2f GB of input embeddings stay charged.",+ getattr(self, "_metal_probe_failure", None) or "llama-fit-params unavailable",embeddings / (1024**3),)return 0@@ -12427,7 +12974,7 @@ class LlamaCppBackend:mapped = int(layout.tensor_bytes) - int(layout.excluded_block_bytes)unmapped = min(embeddings, max(0, mapped - charged))if not unmapped:- logger.info(+ _info_once_per_load("Metal fit: llama.cpp maps this model's %.2f GB of input embeddings into Metal, ""so they stay charged. A llama.cpp update may remove that.",embeddings / (1024**3),@@ -16109,6 +16656,18 @@ class LlamaCppBackend:draft_width = getattr(db, "_compute_activation_width", lambda: 0)()if draft_width > 0:width = draft_width + 2 * (getattr(self, "_embedding_length", None) or 0)+ if getattr(db, "_architecture", None) == "dflash" and draft_width > 0:+ # The DSpark/DFlash prompt graph grows with -ub (DeepSeek-V4.1 DSpark: 725.5 MiB at+ # 512, 2900.5 MiB at 2048).+ vocab = (+ getattr(db, "_vocab_size", None)+ or getattr(self, "_vocab_size", None)+ or _ASSUMED_MAX_VOCAB+ )+ heads = (getattr(db, "_n_heads", None) or 0) * (+ getattr(db, "_kv_key_length", None) or 0+ )+ width = max(width, 2 * vocab + 2 * heads + draft_width)activations = int(width * ub * 4 * self._COMPUTE_BUFFER_SAFETY)spec = verify_rows + max(0, ub * 2 * n_ctx + activations - self._MTP_DRAFT_COMPUTE_BYTES)if getattr(db, "_architecture", None) == "dflash":@@ -16825,6 +17384,7 @@ class LlamaCppBackend:levelled = to_info = Falseproc = self._processdevice_lost_seen = False+ metal_poison = {"draft": False}try:for line in proc.stdout:line = line.rstrip()@@ -16851,13 +17411,27 @@ class LlamaCppBackend:levelled = Trueto_info = level.group(1) in "WE" or (ready and level.group(1) == "I")# E lines only: unprefixed lines can be a request dump with user text.- if (- not device_lost_seen- and level.group(1) == "E"- and any(marker in line for marker in _GPU_DEVICE_LOST_MARKERS)- ):- device_lost_seen = True- self._retire_device_lost_server_soon(proc)+ if level.group(1) == "E":+ if any(m in line for m in _DRAFT_CONTEXT_FAILURE_MARKERS):+ metal_poison["draft"] = True+ if not device_lost_seen and any(+ marker in line for marker in _GPU_DEVICE_LOST_MARKERS+ ):+ device_lost_seen = True+ self._retire_device_lost_server_soon(proc)+ elif (+ not device_lost_seen+ and not metal_poison.get("pending")+ and any(+ marker in line for marker in _METAL_BACKEND_POISONED_MARKERS+ )+ ):+ # Not latched: a request can echo this text, so retirement confirms+ # a compute failure first.+ metal_poison["pending"] = True+ self._retire_device_lost_server_soon(+ proc, metal_poison = metal_poison+ )elif not levelled:to_info = ready or _llama_line_is_warning_or_error(line, line_lower)try:@@ -17970,6 +18544,11 @@ class LlamaCppBackend:self._kv_cache_context_total = None# False means confirmed to hold no VRAM and makes the coordinator skip the unload, so claim it only for a zero-layer split.self._gpu_offload_active = not holds_no_gpu+ # A runner crash must neither replay the replaced chat load nor meet its Metal OOM latch.+ self._last_load_intent = None+ self._mtp_runtime_fallback_active = False+ self._metal_oom_strikes = None+ self._metal_oom_terminal_key = None# Record the applied split and reset the other knobs so status and reload dedup agree.self._gpu_memory_mode = gpu_memory_mode if manual_ngl is not None else "auto"self._gpu_layers = manual_ngl if manual_ngl is not None else -1@@ -22761,6 +23340,19 @@ class LlamaCppBackend:_metal_full_offload_pinned = False# --fit-target floor from the Metal wired headroom; 0 off Apple Silicon._metal_fit_target_floor_mib = 0.0+ # --fit-target headroom for a DSpark/DFlash sidecar the child's fitter cannot size.+ _sidecar_fit_reserve_mib = 0+ _sidecar_fit_reserve_eligible_mib = 0+ _sidecar_compute_mib = 0+ _fit_device_count = 1+ _sidecar_reserve_graph_mib = [0]+ _sidecar_draft_layers: Optional[int] = None+ _sidecar_owns_output = False+ _sidecar_override_flags: List[str] = []+ _sidecar_override_pairs: List[List[str]] = []+ _sidecar_reserve_logged: list = []+ _sidecar_reserve_reprice: Optional[Callable[..., int]] = None+ _sidecar_kind_for_fit = None# The fit and launch must use the same Model Memory snapshot.from utils.model_memory_settings import capture_model_memory_settings@@ -23235,6 +23827,98 @@ class LlamaCppBackend:_cpu_draft_path = _mtp_draft_for_budget if _draft_on_cpu else Noneif _draft_on_cpu:_mtp_draft_for_budget = None++ def _launch_split_env() -> dict:+ e = dict(os.environ)+ if not tensor_parallel:+ sm = (e.get("LLAMA_ARG_SPLIT_MODE") or "").strip().lower()+ if sm and sm != "layer":+ e.pop("LLAMA_ARG_SPLIT_MODE", None)+ e.pop("LLAMA_ARG_TENSOR_SPLIT", None)+ return e++ def _sidecar_planned_device_count() -> int:+ count = len(gpu_indices if gpu_indices is not None else _detected_gpus)+ if not _gpu_ids_own_device_flags:+ named = _extra_args_main_device(extra_args) or os.environ.get(+ "LLAMA_ARG_DEVICE"+ )+ if named:+ count = _distinct_device_count(named)+ if count > 1 and _split_mode_confines_to_one_device(+ extra_args, _launch_split_env()+ ):+ return 1+ return count++ def _draft_on_target_device_now(apple_budget_mib: Optional[int] = None) -> bool:+ # A draft pinned to another device allocates there (on Apple Silicon only+ # MTL0 / Vulkan0 is the target's).+ pin = _extra_args_draft_device_pin(extra_args)+ if pin is None:+ return True+ distinct = {d.strip() for d in pin.split(",") if d.strip()}+ pin = distinct.pop() if len(distinct) == 1 else pin+ apple = _apple_budget_mib if apple_budget_mib is None else apple_budget_mib+ # The picker strips competing device extras and env twins at emission, so+ # this is the launch's placement.+ _pl_args = (+ self._strip_device_extra_args(extra_args)+ if _gpu_ids_own_device_flags+ else extra_args+ )+ _pl_env = _launch_split_env()+ if gpu_memory_mode == "manual":+ self._clear_manual_placement_env(_pl_env)+ if _gpu_ids_own_device_flags:+ self._clear_device_placement_env(_pl_env)+ main = _extra_args_main_device(_pl_args)+ if main is None and not _gpu_ids_own_device_flags:+ main = _pl_env.get("LLAMA_ARG_DEVICE") or None+ if main is None and is_vulkan_backend:+ ids = (+ gpu_indices+ if gpu_indices is not None+ else [g[0] for g in _detected_gpus]+ )+ main = ",".join(f"Vulkan{i}" for i in ids) or (+ "Vulkan0" if apple > 0 else None+ )+ elif main is None and apple > 0:+ main = "MTL0"+ elif (main is None and pin.strip()) and all(+ _CUDA_LIKE_DEVICE_RE.match(d.strip())+ for d in pin.split(",")+ if d.strip()+ ):+ # CUDA / ROCm pins are remapped into the child's mask: only the spawned+ # argv can tell (_spawn_draft_on_target).+ return True++ if main and _split_mode_confines_to_one_device(_pl_args, _pl_env):+ listed = [d.strip() for d in str(main).split(",") if d.strip()]+ if len(listed) > 1:+ at = self._effective_main_gpu(_pl_args, _pl_env)+ at = 0 if at is None else at+ main = listed[at] if 0 <= at < len(listed) else None++ def _names(value: str) -> set:+ return {d.strip().lower() for d in str(value).split(",") if d.strip()}++ return bool(main) and bool(_names(pin)) and _names(pin) <= _names(main)++ _draft_share_unknown = bool(+ _mtp_draft_for_budget+ and _draft_card_share_unknown(+ extra_args,+ os.environ,+ n_draft_layers = getattr(+ self._draft_backend_for(_mtp_draft_for_budget),+ "_n_layers",+ None,+ ),+ )+ )_mtp_draft_weights = 0if _mtp_draft_for_budget:try:@@ -23306,6 +23990,18 @@ class LlamaCppBackend:_kv_unified: bool = planned_kv_unified,_n_ubatch: Optional[int] = _effective_ubatch,) -> int:+ # Applied here, not in a wrapper, so every gate taking this callback+ # prices the draft micro-batch.+ _n_ubatch = (+ _capped_draft_n_ubatch(+ extra_args,+ os.environ,+ ctx,+ _emitted_n_batch(n_batch, _np),+ n_ubatch,+ )+ or _n_ubatch+ )v = self._estimate_mtp_overhead_bytes(ctx,spec_draft_n_max = _n,@@ -23688,6 +24384,8 @@ class LlamaCppBackend:# llama.cpp loads a bare --model-draft anyway (has_dft()), so keep the reserve.and not _extra_args_mtp_draft_path(extra_args, env = _spec_env)and not _draft_cpu_no_embedded+ and not _draft_share_unknown+ and _draft_on_target_device_now(apple_budget_mib = 0)and not tensor_paralleland gpusand effective_ctx > 0@@ -24724,6 +25422,121 @@ class LlamaCppBackend:+ _cc_bytes(ctx)) / (1024 * 1024)+ # Auto only: drop the drafter when the target fits Metal alone but not with+ # the sidecar reserve.+ if (+ use_fit+ and _mtp_will_engage+ and _mtp_reserves_gpu+ and _mtp_effective in ("dspark", "dflash")+ and (_canonicalize_spec_mode(speculative_type) or "auto") == "auto"+ and not _extra_args_set_spec_type(extra_args)+ # A bare --model-draft in extras still loads (has_dft()).+ and not _extra_args_mtp_draft_path(extra_args, env = _spec_env)+ and not _draft_share_unknown+ and _draft_on_target_device_now()+ # Zero KV would read as a target that fits alone.+ and self._can_estimate_kv()+ ):+ # _apple_fit_budget_mib already holds back a flat fraction for this+ # drafter.+ _sc_ctx = (+ effective_ctx+ if explicit_ctx+ else min(_metal_floor_ctx, native_ctx_for_cap or _metal_floor_ctx)+ )+ _sc_target_mib = (+ _apple_model_size_fit+ + _kv_bytes(_sc_ctx, _fit_ctx_checkpoints)+ + _cc_bytes(_sc_ctx)+ ) / (1024 * 1024)+ _sc_draft_ub = _capped_draft_n_ubatch(+ extra_args,+ os.environ,+ _sc_ctx,+ _emitted_n_batch(n_batch, n_parallel),+ n_ubatch,+ )+ _sc_with_draft_mib = _sc_target_mib + self._sidecar_fit_reserve_mib(+ _mtp_bytes(+ _sc_ctx,+ slots = _extra_args_n_parallel(extra_args, os.environ),+ ubatch = _sc_draft_ub,+ ),+ mtp_draft_path,+ flat_mib = math.ceil(_flat_mtp_reserve * max(0, _apple_budget_mib)),+ )+ # An explicit context is refused past the wired limit when that is+ # readable.+ _sc_wired_mib = (+ self._apple_metal_wired_ceiling_bytes() // (1024 * 1024)+ if explicit_ctx+ and not _caller_owns_budget+ and not _paravirtual_cpu_forced+ else 0+ )+ _sc_over_limit = _apple_budget_mib < _sc_with_draft_mib or (+ 0 < _sc_wired_mib < _sc_with_draft_mib+ )+ # Only when both do not fit: the draft-free embedding discount can cost+ # a cold llama-fit-params run.+ _sc_host_embd_without_draft = _apple_host_embd+ _sc_target_only_mib = _sc_target_mib+ if _sc_over_limit:+ _sc_host_embd_without_draft = (+ self._metal_demand_paged_embedding_bytes(+ model_path,+ extra_args,+ binary = binary,+ requested_load_mode = load_mode,+ supports_load_mode = bool(+ server_caps.get("supports_load_mode")+ ),+ settings = _mem_settings,+ layers_fixed = _caller_owns_budget or _paravirtual_cpu_forced,+ mtp_may_engage = False,+ )+ )+ _sc_target_only_mib = _sc_target_mib - (+ self._MTP_DRAFT_COMPUTE_BYTES+ + _sc_host_embd_without_draft+ - _apple_host_embd+ ) / (1024 * 1024)+ if _sc_target_only_mib <= _apple_budget_mib < _sc_with_draft_mib or (+ _sc_wired_mib > 0+ and _sc_target_only_mib <= _sc_wired_mib < _sc_with_draft_mib+ ):+ logger.info(+ "Auto: the %s sidecar does not fit in unified memory "+ "alongside the model, so it is dropped.",+ "DSpark" if _mtp_effective == "dspark" else "DFlash",+ )+ _spec_dropped_no_vram = True+ _mtp_will_engage = False+ _mtp_reserves_gpu = False+ mtp_overhead_fn = None+ _mtp_kv_unsized = False+ # Derived after the discrete drop: undo them, or Auto keeps a+ # smaller context than Off.+ _mtp_gpu_bytes_remain = False+ _flat_mtp_engages = False+ _flat_mtp_reserve = 0.0+ _pin_fraction = _vram_frac+ _apple_fit_budget_mib = _apple_budget_mib+ _soft_overhead -= self._MTP_DRAFT_COMPUTE_BYTES+ model_size_fit -= self._MTP_DRAFT_COMPUTE_BYTES+ _apple_embd_logged = bool(_apple_host_embd)+ _apple_host_embd = _sc_host_embd_without_draft+ _apple_model_size_fit = model_size_fit - _apple_host_embd+ if _apple_host_embd and not _apple_embd_logged:+ logger.info(+ "Metal fit: %.2f GB of input embeddings stay in the CPU "+ "file mapping; charging %.2f GB of weights to unified "+ "memory.",+ _apple_host_embd / (1024**3),+ _apple_model_size_fit / (1024**3),+ )+if self._can_estimate_kv():cap = _apple_ctx_fit(native_ctx_for_cap, _FIT_MIN_CTX)_cap_footprint_mib = _apple_footprint_mib(cap)@@ -25336,6 +26149,297 @@ class LlamaCppBackend:if _mtp_will_engageelse 0)+ # DSpark/DFlash borrow the target's token_embd/output, so llama.cpp's fitter+ # cannot measure the draft+ # (common/fit.cpp "fitting without it"); carry the planner's reserve in the+ # child's --fit-target.+ _sidecar_kind_for_fit = (+ _mtp_effective if _mtp_effective in ("dspark", "dflash") else None+ )+ if _extra_args_set_spec_type(extra_args):+ _child_spec_types = _accumulated_spec_types(extra_args, env = _spec_env)+ _sidecar_kind_for_fit = (+ "dspark"+ if "draft-dspark" in _child_spec_types+ else ("dflash" if "draft-dflash" in _child_spec_types else None)+ )+ _fit_device_count = _sidecar_planned_device_count()+ _draft_on_target_device = _draft_on_target_device_now()+ if (+ _mtp_will_engage+ and _mtp_reserves_gpu+ and _sidecar_kind_for_fit is not None+ and _draft_on_target_device+ # Builds whose fitter measures borrowed-tensor drafts need no reserve.+ and not server_caps.get("fit_measures_borrowed_drafts")+ and _sidecar_borrows_target_tensors(_mtp_draft_for_budget or mtp_draft_path)+ ):+ # Priced even under --fit off: a startup-OOM retry turns the fitter on with+ # the drafter loaded.+ _sidecar_auto_ctx = effective_ctx <= 0+ _sidecar_reserve_ctx = (+ _sidecar_child_ctx(+ extra_args,+ os.environ,+ self._context_length,+ n_parallel,+ planned_kv_unified,+ )+ if _sidecar_auto_ctx+ else effective_ctx+ )+ _sidecar_compute_mib = math.ceil(+ self._MTP_DRAFT_COMPUTE_BYTES / (1024 * 1024)+ )+ _sidecar_reserve_path = _mtp_draft_for_budget or mtp_draft_path+ try:+ _sidecar_draft_layers = getattr(+ self._draft_backend_for(_sidecar_reserve_path), "_n_layers", None+ )+ except Exception: # noqa: BLE001 - an unread drafter shares by memory alone+ _sidecar_draft_layers = None+ _sidecar_owns_output = not _sidecar_borrows_output(_sidecar_reserve_path)+ # The fitter measures the target's rollback copies itself.+ _sidecar_only_bytes = _mtp_bytes(+ _sidecar_reserve_ctx, n_parallel, _effective_ubatch+ )+ if _target_rollback and _mtp_eff_n_max:+ _sidecar_only_bytes -= (+ self._rollback_state_bytes(n_parallel) * _mtp_eff_n_max+ )+ # An unsized sidecar carries the planner's flat fraction, or the fitter+ # spends it on target layers.+ _sidecar_flat_mib = 0+ if _flat_mtp_reserve > 0:+ _flat_pool_mib = (+ _apple_budget_mib+ if _apple_budget_mib > 0+ else max(+ (+ free+ for idx, free in (gpus or _detected_gpus)+ if gpu_indices is None or idx in gpu_indices+ ),+ default = 0,+ )+ )+ _sidecar_flat_mib = math.ceil(+ _flat_mtp_reserve * max(0, _flat_pool_mib)+ )+ _sidecar_fit_reserve_eligible_mib = (+ self._sidecar_fit_reserve_mib(+ max(0, _sidecar_only_bytes),+ _sidecar_reserve_path,+ flat_mib = _sidecar_flat_mib,+ )+ + _sidecar_compute_mib+ )++ def _sidecar_child_on_discrete_gpus(+ run_args: List[str], child_env: Mapping[str, str]+ ) -> bool:+ """Whether every card the drafter runs on is discrete; positive evidence+ only."""+ if _apple_budget_mib:+ return False+ active = gpu_indices+ if (+ not is_vulkan_backend+ and _child_gpu_physical_ids is not None+ and _child_gpu_physical_ids is not _parent_view_gpu_ids+ ):+ active = list(_child_gpu_physical_ids)+ if not active:+ return False+ override = self._effective_device_ids(run_args, child_env)+ if override is not None:+ if not override or any(+ d.lower() in _CPU_DEVICE_VALUES for d in override+ ):+ return False+ # This backend's own names only: Vulkan0 is not CUDA0.+ if not all(+ (+ re.fullmatch(r"(?i)vulkan\d+", d)+ if is_vulkan_backend+ else _CUDA_LIKE_DEVICE_RE.match(d)+ )+ for d in override+ ):+ return False+ studio_pin = (+ [f"vulkan{i}" for i in _vulkan_pin_ids]+ if is_vulkan_backend and _vulkan_pin_ids+ else None+ )+ if [d.lower() for d in override] != studio_pin:+ listed = {+ d.lower()+ for d in (+ self._enumerated_gpu_devices(binary, child_env) or []+ )+ }+ if not {d.lower() for d in override} <= listed:+ return False+ seen = {idx for idx, _free in (_detected_gpus or ())}+ shared = set(_shared_gpu_ids or ())+ if (+ is_vulkan_backend+ and override+ and not {int(d[6:]) for d in override} <= seen+ ):+ # Vulkan Auto drops its iGPUs from the probe.+ return False+ if (+ seen+ and not (seen & shared)+ and self._discrete_gpu_for_expert_spill(+ None, _detected_gpus, _shared_gpu_ids+ )+ ):+ return True+ # A mixed host: narrow only through an order the child provably shares.+ if not is_vulkan_backend and (+ self._positional_child_order_unproven(child_env)+ or any(+ not all(+ t.strip().isdigit()+ for t in str(child_env.get(name)).split(",")+ if t.strip()+ )+ for name in (+ "CUDA_VISIBLE_DEVICES",+ "HIP_VISIBLE_DEVICES",+ "ROCR_VISIBLE_DEVICES",+ )+ if child_env.get(name) is not None+ )+ ):+ return False+ active, compact = self._effective_gpu_selection(+ binary, child_env, active, run_args, is_vulkan = is_vulkan_backend+ )+ pos = _draft_pin_device_index(+ _extra_args_draft_device_pin(run_args),+ ",".join(override) if override else None,+ len(compact or active or _detected_gpus or ()),+ )+ if (+ pos is None+ and _extra_args_draft_device_pin(run_args) is None+ and _split_mode_confines_to_one_device(run_args, child_env)+ ):+ main = self._effective_main_gpu(run_args, child_env)+ pos = 0 if main is None else (main if override else None)+ if pos is not None:+ if override:+ if not (compact and 0 <= pos < len(compact)):+ return False+ active = self._physical_ordinals(+ [compact[pos]], child_env, is_vulkan_backend+ )+ elif is_vulkan_backend or not self._positional_child_order_unproven(+ child_env+ ):+ active = self._physical_ordinals(+ [pos], child_env, is_vulkan_backend+ )+ on = set(active) if active else seen+ if not on or not on <= seen - shared:+ return False+ return self._discrete_gpu_for_expert_spill(+ sorted(on), _detected_gpus, _shared_gpu_ids+ )++ def _sidecar_reserve_reprice(+ run_args: List[str],+ child_env: Mapping[str, str],+ _ctx: int = _sidecar_reserve_ctx,+ _n: int = _mtp_eff_n_max,+ _dp: Optional[str] = _sidecar_reserve_path,+ _w: int = _mtp_draft_weights,+ _np: int = n_parallel,+ _mtp: bool = _engaged_is_mtp,+ _swa_full: bool = swa_full,+ _kv_unified: bool = planned_kv_unified,+ _ub: Optional[int] = _effective_ubatch,+ _floor: int = _sidecar_compute_mib,+ _flat: int = _sidecar_flat_mib,+ _native: Optional[int] = self._context_length,+ ) -> int:+ # Read off the argv: a --flash-attn off respawn pads the draft cache to+ # f16, extras can change slots.+ _ck, _cv = _extra_args_draft_cache_types(run_args, child_env)+ _np = _extra_args_n_parallel(run_args, child_env) or _np+ _kv_unified = _kv_unified_from_args(+ run_args, default = _kv_unified, env = child_env+ )+ try:+ _ctx = _sidecar_child_ctx(+ run_args, child_env, _native, _np, _kv_unified+ )+ except ValueError:+ pass # a malformed -c is the launch's error to report, not this+ # llama_context caps the micro-batch at the batch, and a causal batch at+ # the context.+ _ub = _capped_draft_n_ubatch(run_args, child_env, _ctx) or _ub+ if _ub and _ctx and _ctx > 0:+ _ub = min(_ub, int(_ctx))+ _kv_kw = dict(+ drafter_path = _dp,+ draft_cache_type_k = _ck,+ draft_cache_type_v = _cv,+ n_parallel = _np,+ swa_full = _swa_full,+ kv_unified = _kv_unified,+ n_ubatch = _ub,+ flash_attn = _flash_attn_enabled_from_args(+ run_args, default = planned_flash_attn, env = child_env+ ),+ )+ _v = self._estimate_mtp_overhead_bytes(+ _ctx,+ spec_draft_n_max = _n,+ draft_weights_bytes = _w,+ mtp_keeps_target_ctx = _mtp,+ target_rollback = False,+ **_kv_kw,+ )+ # The draft context copies -nkvo, so its KV stays in host buffers.+ if (+ _v+ and not _kv_offload_from_args(run_args, child_env)+ and _sidecar_child_on_discrete_gpus(run_args, child_env)+ ):+ _host_kv = self._mtp_draft_kv_bytes(_ctx, **_kv_kw)+ if _host_kv:+ _v = max(0, _v - _host_kv)+ try:+ _sidecar_reserve_graph_mib[0] = math.ceil(+ self._mtp_draft_compute_bytes(+ _ctx,+ drafter_path = _dp,+ n_ubatch = _ub,+ n_parallel = _np,+ spec_draft_n_max = _n,+ )+ / (1024 * 1024)+ )+ except Exception: # noqa: BLE001 - unsized: the shares carry it+ _sidecar_reserve_graph_mib[0] = 0+ return (+ self._sidecar_fit_reserve_mib(_v or 0, _dp, flat_mib = _flat) + _floor+ )++ if _draft_share_unknown:+ # Drafter partly off the card: its weight share is unpriced, keep only+ # the compute floor.+ _sidecar_fit_reserve_eligible_mib = _sidecar_compute_mib+ _sidecar_reserve_reprice = None+ # A scalar --fit-target suits only a single-device launch; priced anyway,+ # since a retry can narrow to one card.+ if use_fit and (_apple_budget_mib > 0 or _fit_device_count == 1):+ _sidecar_fit_reserve_mib = _sidecar_fit_reserve_eligible_mibif _mtp_will_engage:_mtp_note = self._mtp_reserve_note(_mtp_reserve_bytes,@@ -26250,14 +27354,20 @@ class LlamaCppBackend:if _fit_target_priced and use_fit and not _metal_fit_target_floor_mibelse None)- if _metal_fit_target_floor_mib:+ if _metal_fit_target_floor_mib or _sidecar_fit_reserve_mib:# Rebuilt after the price is read: the floor is the machine's- # margin, not the budget slider's.+ # margin and the sidecar reserve the drafter's, not the budget+ # slider's._integrity_flags = self._ctx_integrity_flags(*_integrity_args,fit_target_delta_mib = _fit_target_delta_mib,fit_target_floor_mib = _metal_fit_target_floor_mib,+ # The reserve rides on the spawn's trailing pair, which retry cleanup must+ # tell apart from a user's value.+ fit_target_reserve_mib = 0,)+ if _sidecar_fit_reserve_mib and not server_caps.get("supports_fit_target"):+ _sidecar_fit_reserve_mib = 0cmd.extend(_integrity_flags)offload_overridden = _extra_args_set_any_flag(extra_args, _GPU_OFFLOAD_OVERRIDE_FLAGS@@ -26448,14 +27558,25 @@ class LlamaCppBackend:if _gpu_ids_own_device_flags:_spec_placement_extras = self._strip_device_extra_args(extra_args)self._clear_device_placement_env(_spec_placement_env)+ _spec_placement_argv = [*cmd, *(_spec_placement_extras or [])]# A device the launch really carries, and not the CPU spelling of it.# Only consulted where the probe found nothing: it is the one remaining# way the child can still be pointed at a GPU._spec_device_pick = _extra_args_main_device(- [*cmd, *(_spec_placement_extras or [])]+ _spec_placement_argv) or _spec_placement_env.get("LLAMA_ARG_DEVICE")_spec_extras_pick_a_gpu = bool(_spec_device_pick) and not _device_selection_is_cpu(- [*cmd, *(_spec_placement_extras or [])], _spec_placement_env+ _spec_placement_argv, _spec_placement_env+ )+ # Log-only; a priced reserve is reported by the spawn (_log_sidecar_fit_reserve).+ _sidecar_fit_sized = (+ not fit_is_effectively_on(_spec_placement_argv, _spec_placement_env)+ or bool(_sidecar_fit_reserve_eligible_mib)+ or bool(server_caps.get("fit_measures_borrowed_drafts"))+ or (+ bool(launch_mtp_draft_path)+ and not _sidecar_borrows_target_tensors(launch_mtp_draft_path)+ ))spec_flags = self._build_speculative_flags(speculative_type = speculative_type,@@ -26469,9 +27590,9 @@ class LlamaCppBackend:None if _spec_canon in ("dspark", "dflash") else launch_mtp_draft_path),dspark_draft_path = (launch_mtp_draft_path if _spec_canon == "dspark" else None),- dspark_fit_sized = not use_fit,+ dspark_fit_sized = _sidecar_fit_sized,dflash_draft_path = (launch_mtp_draft_path if _spec_canon == "dflash" else None),- dflash_fit_sized = not use_fit,+ dflash_fit_sized = _sidecar_fit_sized,# So the fallback says unopenable, not missing. DSpark/DFlash never# reach the drop.mtp_drafter_unloadable = bool(_unloadable_mtp_draft_path),@@ -27123,6 +28244,36 @@ class LlamaCppBackend:logger.info(f"Appending user extra args to llama-server: {list(_emit_extra_args)}")+ # Reconcile the reserve with the child as launched: extras can turn the fitter on or+ # add a margin.+ _sidecar_user_tokens = list(_emit_extra_args) if _mem_extras else []+ if (+ _sidecar_fit_reserve_eligible_mib+ and server_caps.get("supports_fit_target")+ # Re-read: a failed GPU selection falls back to every card.+ and (_apple_budget_mib > 0 or _sidecar_planned_device_count() == 1)+ ):+ _sidecar_child_env = dict(os.environ)+ if gpu_memory_mode == "manual":+ self._clear_manual_placement_env(_sidecar_child_env)+ _sidecar_override_flags = _sidecar_reserve_margin_flags(+ cmd,+ _emit_extra_args if _mem_extras else (),+ _sidecar_child_env,+ max(+ _VRAM_FLOOR_RESERVE_MIB,+ self._fit_target_margin_mib(+ auto_fit = auto_fit,+ fit_target_delta_mib = _fit_target_delta_mib,+ floor_mib = _metal_fit_target_floor_mib,+ ),+ ),+ _sidecar_fit_reserve_eligible_mib,+ )+ if _sidecar_override_flags:+ _sidecar_fit_reserve_mib = _sidecar_fit_reserve_eligible_mib+ _sidecar_override_pairs.append(list(_sidecar_override_flags))+ cmd.extend(_sidecar_override_flags)# Last so it wins: the Metal drafter CPU pin is a correctness fix.if _pv_draft_cpu_pin:cmd.extend(_pv_draft_cpu_pin)@@ -27163,8 +28314,6 @@ class LlamaCppBackend:kv_cache_unified = _kv_unified_from_args(cmd)- logger.info(f"Starting llama-server: {' '.join(self._redacted_cmd_for_log(cmd))}")-# Library paths so llama-server finds its shared libs and CUDA DLLs.env = self._llama_server_env_for_binary(binary)# The GGUF header controls pooling; ignore inherited defaults.@@ -27520,6 +28669,7 @@ class LlamaCppBackend:_companion_added_device: Optional[list[str]] = Noneif not is_vulkan_backend and _gpu_mem:_child_gpu_physical_ids = self._unmasked_child_gpu_physical_ids()+ _parent_view_gpu_ids = _child_gpu_physical_ids# The CUDA SM gate goes here, where the child's GPU visibility is# finally known: both arms below write CUDA_VISIBLE_DEVICES=-1, so no@@ -27930,7 +29080,7 @@ class LlamaCppBackend:# page-lock and writes it back, which without this makes the# read below an UnboundLocalError instead.nonlocal _last_spawn_cmd, _mem_host_resident, _did_rocm_retry- nonlocal _sched_abort_seen+ nonlocal _sched_abort_seen, _sidecar_fit_reserve_mib# Single revocation point for the tensor-spill plan; label is empty only on the first spawn.if label:run_cmd = self._drop_tensor_spill(run_cmd, label.lstrip("-") or "retry")@@ -27980,9 +29130,278 @@ class LlamaCppBackend:# Best-effort; never block the load on logging.logger.warning(f"Could not open llama-server log file: {e}")self._llama_log_path = None+ # Every launch: a retry can turn the fitter on or re-mask onto several+ # cards.+ _spawn_devices = _extra_args_main_device(run_cmd) or env.get(+ "LLAMA_ARG_DEVICE"+ )+ _spawn_device_count = (+ _distinct_device_count(_spawn_devices)+ if _spawn_devices+ else max(len(_child_gpu_physical_ids or ()), len(_detected_gpus))+ if _child_gpu_physical_ids is None+ or _child_gpu_physical_ids is _parent_view_gpu_ids+ else len(_child_gpu_physical_ids)+ )+ if _sidecar_fit_reserve_eligible_mib:+ # Re-derived each spawn: drop Studio's earlier reserve-bearing margins,+ # keep the user's.+ run_cmd = _drop_sidecar_margins(+ list(run_cmd),+ _integrity_flags,+ self._ctx_integrity_flags(+ *_integrity_args,+ fit_target_delta_mib = _fit_target_delta_mib,+ fit_target_floor_mib = _metal_fit_target_floor_mib,+ ),+ _sidecar_override_pairs,+ user_args = _sidecar_user_tokens,+ )+ _sidecar_fit_reserve_mib = 0run_cmd = self._per_device_fit_target_argv(run_cmd, env, _fit_target_margins_by_idx)+ _spawn_base_by_dev: Optional[list[float]] = None+ if _fit_target_margins_by_idx:+ try:+ _spelled = [+ float(v)+ for v in str(+ _last_flag_value(run_cmd, {"-fitt", "--fit-target"})+ ).split(",")+ ]+ except ValueError:+ _spelled = []+ if len(_spelled) == 1:+ _spelled = _spelled * _spawn_device_count+ if len(_spelled) == _spawn_device_count:+ _spawn_base_by_dev = _spelled+ # --rpc devices come first in the child's list, so local positional shares+ # would land on the wrong cards.+ _spawn_rpc = not _spawn_devices and bool(+ (_extra_args_device(run_cmd, {"--rpc"}) or "").strip()+ or str(env.get("LLAMA_ARG_RPC", "")).strip()+ )+ # Several cards: each keeps its share of the drafter (whole draft layers) in+ # a per-device list.+ _spawn_shares: Optional[list[float]] = [1.0]+ _spawn_output_dev: Optional[int] = None+ _spawn_pinned_at: Optional[int] = None+ _spawn_cpu_only = _device_selection_is_cpu(run_cmd, env) or (+ _child_gpu_physical_ids is not None+ and _child_gpu_physical_ids is not _parent_view_gpu_ids+ and not _child_gpu_physical_ids+ )+ if _spawn_cpu_only:+ _spawn_shares = None+ elif (+ _spawn_device_count > 1+ and _extra_args_split_mode(run_cmd, env) == "tensor"+ ):+ # --split-mode tensor folds the cards into one device: no per-device+ # reserve.+ _spawn_shares = None+ elif _spawn_rpc:+ _spawn_shares = None+ elif _spawn_device_count > 1 and _split_mode_confines_to_one_device(+ run_cmd, env+ ):+ if _spawn_base_by_dev and len(set(_spawn_base_by_dev)) > 1:+ # llama.cpp keeps only devices[main_gpu] (src/llama.cpp): the+ # reserve goes in entry 0 of a list that keeps the other margins.+ _main = self._effective_main_gpu(run_cmd, env) or 0+ if 0 <= _main < _spawn_device_count:+ _spawn_shares = [+ 1.0 if i == 0 else 0.0 for i in range(_spawn_device_count)+ ]+ _spawn_output_dev = 0+ else:+ _spawn_shares = None+ elif _spawn_device_count > 1:+ _spawn_shares = None+ _spawn_pinned_at = _draft_pin_device_index(+ _extra_args_draft_device_pin(run_cmd),+ _spawn_devices,+ _spawn_device_count,+ )+ if _spawn_pinned_at is not None:+ _spawn_shares = [+ 1.0 if i == _spawn_pinned_at else 0.0+ for i in range(_spawn_device_count)+ ]+ _spawn_output_dev = _spawn_pinned_at+ # Positional shares need the child to enumerate in the ids' order.+ _spawn_order_unproven = self._positional_child_order_unproven(env)+ if (+ not is_vulkan_backend+ and not _spawn_devices+ and _extra_args_draft_device_pin(run_cmd) is None+ and not _spawn_order_unproven+ ):+ _free_by_idx = dict(gpus or _detected_gpus or ())+ if (+ _child_gpu_physical_ids is None+ or _child_gpu_physical_ids is _parent_view_gpu_ids+ ):+ _det_ids = [idx for idx, _free in _detected_gpus]+ _spawn_ids = (+ self._inherited_child_gpu_order(_det_ids) or _det_ids+ )+ else:+ _spawn_ids = list(_child_gpu_physical_ids)+ _spawn_free = [_free_by_idx.get(i) for i in _spawn_ids]+ # The drafter inherits the target's --tensor-split+ # (common_base_params_to_speculative).+ _spawn_base = self._fit_target_margin_mib(+ auto_fit = auto_fit,+ fit_target_delta_mib = _fit_target_delta_mib,+ floor_mib = _metal_fit_target_floor_mib,+ )+ _weights = None+ if _spawn_free and all(f and f > 0 for f in _spawn_free):+ _spawn_margins = _fit_margins_by_device(+ extra_args, env, 0.0, len(_spawn_free)+ )+ _bases = (+ _spawn_base_by_dev+ if _spawn_base_by_dev+ and len(_spawn_base_by_dev) == len(_spawn_free)+ else [_spawn_base] * len(_spawn_free)+ )+ _weights = [+ max(1.0, float(f) - max(b, m))+ for f, m, b in zip(_spawn_free, _spawn_margins, _bases)+ ]+ _ts = _extra_args_tensor_split(run_cmd, env)+ if _ts and _spawn_ids:+ _ts_w = [+ float(_ts[i]) if i < len(_ts) else 0.0+ for i in range(len(_spawn_ids))+ ]+ if sum(_ts_w) > 0 and all(w >= 0 for w in _ts_w):+ _weights = _ts_w+ if _weights:+ _layers = _sidecar_draft_layers or 0+ _cum = list(+ itertools.accumulate(w / sum(_weights) for w in _weights)+ )+ # "row" assigns blocks like "layer" (CUDA / ROCm register no+ # split buffer type).+ if _layers > 0:+ # llama.cpp's layer split: block il goes to the first device+ # whose cumulative share exceeds il / (n_layer + 1).+ _counts = [0] * len(_weights)+ for _il in range(_layers):+ _dev = bisect.bisect_right(_cum, _il / (_layers + 1))+ _counts[min(_dev, len(_weights) - 1)] += 1+ _spawn_shares = [c / _layers for c in _counts]+ else:+ _spawn_shares = [w / sum(_weights) for w in _weights]+ # Logits over a borrowed output run on the card holding the+ # target's layer n_layer.+ _t_layers = (+ _layers+ if _sidecar_owns_output+ else int(getattr(self, "_n_layers", 0) or 0) or _layers+ )+ if _t_layers > 0:+ _spawn_output_dev = min(+ bisect.bisect_right(_cum, _t_layers / (_t_layers + 1)),+ len(_weights) - 1,+ )+ if (+ _sidecar_fit_reserve_eligible_mib+ and _spawn_shares is not None+ and (+ _spawn_pinned_at is not None+ or _spawn_draft_on_target(run_cmd, _spawn_devices, env)+ )+ and server_caps.get("supports_fit_target")+ ):+ _spawn_total = (+ _sidecar_reserve_reprice(list(run_cmd), env)+ if _sidecar_reserve_reprice is not None+ else _sidecar_fit_reserve_eligible_mib+ )+ # The vocabulary-wide graph stays whole on the output card.+ _spawn_graph = min(+ _sidecar_reserve_graph_mib[0] if len(_spawn_shares) > 1 else 0,+ max(0, _spawn_total - _sidecar_compute_mib),+ )+ _spawn_body = max(0, _spawn_total - _sidecar_compute_mib - _spawn_graph)+ _last = (+ _spawn_output_dev+ if _spawn_output_dev is not None+ else max(i for i, f in enumerate(_spawn_shares) if f > 0)+ if any(f > 0 for f in _spawn_shares)+ else len(_spawn_shares) - 1+ )+ _spawn_head = 0+ if (+ _sidecar_owns_output+ and len(_spawn_shares) > 1+ and not _draft_share_unknown+ ):+ _head_layout = self._tensor_spill_layout(+ _sidecar_reserve_path, all_shards = True+ )+ if _head_layout is not None and _head_layout.complete:+ _spawn_head = min(+ _spawn_body,+ math.ceil(_head_layout.lm_head_bytes / (1024 * 1024)),+ )+ _spawn_body -= _spawn_head+ else:+ _spawn_head = _spawn_body+ _spawn_per_device = [+ math.ceil(_spawn_body * f)+ + (_sidecar_compute_mib if f > 0 or i == _last else 0)+ + (_spawn_graph + _spawn_head if i == _last else 0)+ for i, f in enumerate(_spawn_shares)+ ]+ _spawn_reserve = max(_spawn_per_device)+ _spawn_flags = _sidecar_reserve_margin_flags(+ run_cmd,+ extra_args,+ env,+ self._fit_target_margin_mib(+ auto_fit = auto_fit,+ fit_target_delta_mib = _fit_target_delta_mib,+ floor_mib = _metal_fit_target_floor_mib,+ ),+ _spawn_reserve,+ per_device = _spawn_per_device+ if len(_spawn_per_device) > 1+ else None,+ base_by_device = _spawn_base_by_dev,+ )+ if _spawn_flags:+ run_cmd = [*run_cmd, *_spawn_flags]+ if _spawn_total != _sidecar_fit_reserve_eligible_mib:+ logger.info(+ "Sidecar reserve repriced for this launch's argv: "+ "%.2f GB.",+ _spawn_total / 1024,+ )+ _sidecar_fit_reserve_mib = _spawn_reserve+ if list(_spawn_flags) not in _sidecar_override_pairs:+ _sidecar_override_pairs.append(list(_spawn_flags))+ if (+ _sidecar_fit_reserve_eligible_mib+ and _sidecar_kind_for_fit+ and not _spawn_cpu_only+ ):+ self._log_sidecar_fit_reserve(+ run_cmd,+ env,+ _sidecar_kind_for_fit,+ _sidecar_reserve_path,+ _sidecar_fit_reserve_mib,+ _sidecar_reserve_logged,+ )+ logger.info(+ f"Starting llama-server: {' '.join(self._redacted_cmd_for_log(run_cmd))}"+ )_last_spawn_cmd = list(run_cmd)# Read off the argv actually spawned rather than the intent, so# every emitter is covered by construction.@@ -28185,6 +29604,8 @@ class LlamaCppBackend:# common/fit.cpp aborts a multi-device fit on a set split._run = _without_subsequence(_run, self._mixed_split_flags)self._auto_tensor_split_emitted = None+ # The --fit off launch emitted no margin; the next spawn adds the+ # reserve.# Same reasoning as the page-lock below, one step# earlier: the fit picked "none" because it had proved# the load fits, and that proof is what just failed, so@@ -29164,6 +30585,28 @@ class LlamaCppBackend:# Read by admission control; left as-is Studio over-admits.n_parallel = 1 # allow-slot-clamp: llama-server refused morekv_cache_unified = False+ # The reserve was priced for the dropped slots: reprice for one sequence.+ if (+ _sidecar_fit_reserve_eligible_mib+ and _sidecar_reserve_reprice is not None+ ):+ cmd = _drop_sidecar_margins(+ cmd,+ _integrity_flags,+ self._ctx_integrity_flags(+ *_integrity_args,+ fit_target_delta_mib = _fit_target_delta_mib,+ fit_target_floor_mib = _metal_fit_target_floor_mib,+ ),+ _sidecar_override_pairs,+ user_args = _sidecar_user_tokens,+ )+ _sidecar_override_pairs.clear()+ _sidecar_reserve_reprice = functools.partial(+ _sidecar_reserve_reprice, _np = 1, _kv_unified = False+ )+ _sidecar_fit_reserve_eligible_mib = _sidecar_reserve_reprice(cmd, env)+ _sidecar_fit_reserve_mib = 0# LLAMA_ARG_SPEC_TYPE reaches the child only if extras set --spec-type; else the launch scrubs it.if _auto_ckpt_emitted:cmd = self._without_flag_pairs(cmd, _auto_ckpt_emitted)@@ -29373,6 +30816,22 @@ class LlamaCppBackend:strip_split_mode = False,)fallback_cmd = cmd[:_spec_at] + ["--spec-default"] + _fb_tail+ # A user-owned drafter in the extras or env still loads and keeps its reserve.+ if _sidecar_fit_reserve_mib and not _extra_args_mtp_draft_path(+ fallback_cmd, env = env+ ):+ _fb_integrity = self._ctx_integrity_flags(+ *_integrity_args,+ fit_target_delta_mib = _fit_target_delta_mib,+ fit_target_floor_mib = _metal_fit_target_floor_mib,+ )+ fallback_cmd = _drop_sidecar_margins(+ fallback_cmd,+ _integrity_flags,+ _fb_integrity,+ _sidecar_override_pairs,+ user_args = _sidecar_user_tokens,+ )# Restore the cut slots; if they fail (unified KV refused), retry as cut._fb_cut = ((@@ -29626,9 +31085,15 @@ class LlamaCppBackend:)self._mmproj_fallback_reason = "projector_startup_failure"cmd = self._strip_mmproj_args(_vision_gpu_cmd)- cmd = self._per_device_fit_target_argv(- cmd, env, _fit_target_margins_by_idx- )+ # Keep a trailing sidecar pair: rewriting it would drop the drafter's+ # reserve.+ if [+ "--fit-target",+ str(_last_flag_value(cmd, {"-fitt", "--fit-target"})),+ ] not in _sidecar_override_pairs:+ cmd = self._per_device_fit_target_argv(+ cmd, env, _fit_target_margins_by_idx+ )_unified_withdraw_if_unneeded(cmd, why = "The text-only retry")# This retry bypasses _spawn_and_wait, so refresh the launched-argv snapshot here._last_spawn_cmd = list(cmd)@@ -29878,6 +31343,11 @@ class LlamaCppBackend:# dropped: a respawn replays this snapshot arbitrarily later, so recovery# re-resolves the file instead of trusting a path this request verified.self._last_load_intent = replace(intent, verified_gguf = None)+ if not getattr(_AUTO_RECOVERY_LOAD, "active", False):+ # A caller's load starts a fresh OOM episode; only automatic recovery carries+ # the strike count.+ self._metal_oom_strikes = None+ self._metal_oom_terminal_key = Noneself._mtp_runtime_fallback_active = _mtp_active_for_launched_serverself._start_mtp_crash_watchdog()@@ -30619,6 +32089,10 @@ class LlamaCppBackend:def matches_load_source(self, intent: GgufLoadIntent) -> bool:"""Whether the resident model has the intent's identity and weights."""+ # A retired child stays recorded until respawned: an identical load must replace it.+ _retired = getattr(self, "_retired_process", None)+ if _retired is not None and getattr(self, "_process", None) is _retired:+ return Falseif not self.is_loaded or (self._model_identifier or "").lower() != (intent.model_identifier.lower()):@@ -30791,6 +32265,8 @@ class LlamaCppBackend:self._cpu_fallback_reason = Noneself._last_load_intent = Noneself._mtp_runtime_fallback_active = False+ self._metal_oom_strikes = None+ self._metal_oom_terminal_key = Noneself._hf_variant = Noneself._is_vision = Falseself._disable_vision = False@@ -31175,11 +32651,7 @@ class LlamaCppBackend:finally:self._kill_process_group(_pgid)_surviving_descendants = self._terminate_descendants(_descendants)- # getattr: teardown must tolerate a partially-built backend (failed- # __init__ or a __new__-built instance), as with _llama_log_fh below.- if getattr(self, "_stats_logger", None) is not None:- self._stats_logger.stop()- self._stats_logger = None+ self._stop_stats_logger()# Forget the pid only once confirmed gone, so a startup sweep can still reap survivors._killed_pid = getattr(self._process, "pid", None)_exit_code = getattr(self._process, "poll", lambda: None)()@@ -32312,18 +33784,25 @@ class LlamaCppBackend:if resp.status_code != 200:logger.debug(f"slot {entry.get('id')} restore returned HTTP {resp.status_code}")- def _maybe_recover_from_mtp_crash(self, exc: Optional[BaseException] = None) -> bool:+ def _maybe_recover_from_mtp_crash(+ self,+ exc: Optional[BaseException] = None,+ *,+ sidecar_oom: bool = False,+ ) -> bool:"""Schedule one background reload without MTP after a mid-generation death.An MTP launch can pass /health and the load-time fallback, then abort ona later decode (MTP+tensor flash-attn, Gemma 4 on Apple Silicon #13253).Not a persistent ban: a fresh load re-tries MTP. Returns True if scheduled.+ ``sidecar_oom`` is the Metal-retirement caller: an Auto DSpark / DFlash launch+ whose server was just killed after running out of Metal memory."""# Cheap async-safe gate: only our live MTP launch, not cancelled, with a# snapshot to replay.if self._cancel_event.is_set():return False- if not self._mtp_runtime_fallback_active:+ if not (self._mtp_runtime_fallback_active or sidecar_oom):return False# Read before claiming: a raise after the claim strands the flag, and nothing# else clears it, blocking every later respawn.@@ -32331,6 +33810,11 @@ class LlamaCppBackend:proc = self._processif snapshot is None or proc is None:return False+ if self._metal_oom_is_terminal(snapshot):+ return False+ # Stopped on purpose (Metal OOM, lost device): the next request's respawn replays it.+ if not sidecar_oom and proc is getattr(self, "_retired_process", None):+ return False# Still serving: a transient error, so leave the caller its normal retry.if proc.poll() is None and self._server_socket_is_open():return False@@ -32349,11 +33833,12 @@ class LlamaCppBackend:if proc.poll() is None:logger.debug("Generation error but llama-server is alive; keeping MTP.")return- logger.warning(- "llama-server exited mid-generation with MTP speculative "- "decoding (%s); reloading without speculative decoding.",- type(exc).__name__ if exc is not None else "server exited",- )+ if not sidecar_oom:+ logger.warning(+ "llama-server exited mid-generation with MTP speculative "+ "decoding (%s); reloading without speculative decoding.",+ type(exc).__name__ if exc is not None else "server exited",+ )# Re-check under the load lock so a newer load isn't clobbered.requested_mode = snapshot.speculative_typewith self._serial_load_lock:@@ -32367,7 +33852,14 @@ class LlamaCppBackend:if self._last_load_intent != snapshot:logger.info("MTP-crash reload skipped: load settings changed.")return+ if self._metal_oom_is_terminal(snapshot):+ logger.info("MTP-crash reload skipped: out of Metal memory twice.")+ return+ if not sidecar_oom and proc is getattr(self, "_retired_process", None):+ logger.info("MTP-crash reload skipped: the server was retired.")+ returnunload_epoch = self._unload_epoch+ dropped_draft = self._mtp_draft_path or self._mtp_draft_suppressed_path# llama.cpp appends spec types, so strip them from extras._ea = list(snapshot.extra_args or ())fallback_extra_args = snapshot.extra_args@@ -32387,7 +33879,8 @@ class LlamaCppBackend:speculative_type = "off",extra_args = fallback_extra_args,)- started = bool(self.load_model(fallback))+ with _automatic_recovery_load():+ started = bool(self.load_model(fallback))if not started:logger.info("MTP-crash reload stopped before the replacement was ready.")return@@ -32413,13 +33906,18 @@ class LlamaCppBackend:else _canonicalize_spec_mode(requested_mode))self._spec_fallback_reason = "runtime_error"+ # Keep the Auto identity so the same request matches instead of+ # relaunching the sidecar.+ if dropped_draft:+ self._mtp_draft_path = None+ self._mtp_draft_suppressed_path = dropped_draftif undo_reload:logger.info("MTP-crash reload undone: the model was unloaded during reload.")self.unload_model()return- logger.info("Reloaded without MTP after the mid-generation crash.")+ logger.info("Reloaded without speculative decoding after the mid-generation crash.")except Exception as e:logger.error(f"Reload without MTP failed: {e}")finally:@@ -32745,6 +34243,29 @@ class LlamaCppBackend:base = _VRAM_FLOOR_RESERVE_MIB if auto_fit else _LLAMA_FIT_TARGET_DEFAULT_MIBreturn max(_VRAM_FLOOR_RESERVE_MIB, base + fit_target_delta_mib, floor_mib)+ def _sidecar_fit_reserve_mib(+ self,+ reserve_bytes: int,+ sidecar_path: Optional[str],+ *,+ flat_mib: int = 0,+ ) -> int:+ """MiB a DSpark/DFlash sidecar needs kept free by the child's ``--fit-target``.++ The planner's byte-accurate reserve (draft weights + draft KV + draft compute ++ target verification rows), never less than the sidecar file itself, which is+ all that is known when the reserve fell back to the flat fraction (0 bytes).+ ``flat_mib``: that fraction of the target's pool, passed only when the planner held+ it back, so it is a floor: the child keeps at least what the plan kept.+ Rounded up: a short MiB is a short allocation.+ """+ try:+ file_bytes = int(self._get_gguf_size_bytes(sidecar_path) or 0) if sidecar_path else 0+ except Exception:+ file_bytes = 0+ need = max(int(reserve_bytes or 0), file_bytes)+ return max(math.ceil(need / (1024 * 1024)), max(0, int(flat_mib or 0)))+@staticmethoddef _fit_target_margins_mib(*,@@ -32812,6 +34333,67 @@ class LlamaCppBackend:return Falsereturn True+ def _log_sidecar_fit_reserve(+ self,+ run_cmd: List[str],+ env: Mapping[str, str],+ kind: str,+ sidecar_path: Optional[str],+ reserved_mib: int,+ logged: list,+ ) -> None:+ """Report whether the spawned argv reserves the DSpark / DFlash sidecar that+ llama.cpp's --fit cannot size. Once per outcome per load (``logged`` holds the+ last one), so a retry that changes it says so and a repeat stays quiet."""+ if not fit_is_effectively_on(run_cmd, env) or not _extra_args_mtp_draft_path(+ run_cmd, env = env+ ):+ return+ outcome = (kind, bool(reserved_mib))+ if logged and logged[-1] == outcome:+ return+ logged.append(outcome)+ label = "DSpark" if kind == "dspark" else "DFlash"+ if reserved_mib:+ logger.info(+ "%s under --fit on: llama.cpp cannot size the sidecar, so "+ "--fit-target keeps %.2f GB free for it.",+ label,+ reserved_mib / 1024,+ )+ return+ # Not a blocker: the fit step cannot measure a borrowed-tensor draft+ # (llama-context.cpp:153-160).+ logger.info(+ "%s under --fit on: llama.cpp cannot size the sidecar during "+ "fitting, so its ~%.1f GB is not reserved. Pin GPU Layers in "+ "Manual mode if the load runs out of VRAM.",+ label,+ (self._get_gguf_size_bytes(sidecar_path) or 0) / 1e9,+ )++ def _positional_child_order_unproven(self, child_env: Mapping[str, str]) -> bool:+ """Whether a per-card list built over the parent's GPU ids may not line up with the+ child's CUDA enumeration (planning and spawn share it).++ nvidia-smi ids follow PCI order, so the child must run under PCI_BUS_ID+ (FASTEST_FIRST is CUDA's default) with a mask a resolver maps, unless the uuid+ mask order is known. Torch-fallback ids are CUDA ordinals in the parent's own+ order, so a child switched to another order (a GPU pick pins PCI_BUS_ID) counts+ them differently. HIP ignores CUDA_DEVICE_ORDER.+ """+ child_order = str(child_env.get("CUDA_DEVICE_ORDER", "")).strip().upper()+ if self._GPU_IDS_ARE_PCI_INDICES is True:+ return (+ child_order != "PCI_BUS_ID" or self._visibility_mask_is_unmappable()+ ) and self._smi_uuid_mask_order() is None+ if self._GPU_IDS_ARE_PCI_INDICES is False:+ parent_order = (os.environ.get("CUDA_DEVICE_ORDER") or "").strip().upper()+ return (parent_order or "FASTEST_FIRST") != (+ child_order or "FASTEST_FIRST"+ ) and not self._host_torch_is_rocm()+ return False+@staticmethoddef _per_device_fit_target_argv(cmd: list[str], env: Mapping[str, str], margins_by_idx: Optional[Mapping[int, float]]@@ -32876,9 +34458,11 @@ class LlamaCppBackend:*,fit_target_delta_mib: float = 0.0,fit_target_floor_mib: float = 0.0,+ fit_target_reserve_mib: int = 0,) -> list[str]:"""Explicit --parallel drops the --kv-unified default and splits -c per slot; restore the- shared pool."""+ shared pool. ``fit_target_reserve_mib`` is added on top for a sidecar the fitter cannot+ measure."""flags: list[str] = []if n_parallel > 1 and caps.get("supports_kv_unified"):flags.append("--kv-unified")@@ -32896,6 +34480,8 @@ class LlamaCppBackend:fit_target_delta_mib = fit_target_delta_mib,floor_mib = fit_target_floor_mib,)+ # On top of every margin, not max(): the sidecar's bytes must stay free.+ _target += max(0, fit_target_reserve_mib)# The legacy path stays silent when it would only restate llama.cpp's# own default, so an untouched slider leaves the command unchanged.if auto_fit or _target != _LLAMA_FIT_TARGET_DEFAULT_MIB:@@ -33388,6 +34974,12 @@ class LlamaCppBackend:intent = self._last_load_intentif intent is None:return False+ if self._metal_oom_is_terminal(intent):+ logger.error(+ f"Not restarting llama-server for '{self._model_identifier}': "+ "it ran out of Metal memory twice with these settings."+ )+ return Falseepoch = self._unload_epochself._healthy = Falselogger.warning(@@ -33395,7 +34987,8 @@ class LlamaCppBackend:f"(code {proc.returncode}); respawning to recover the session")try:- started = bool(self.load_model(intent))+ with _automatic_recovery_load():+ started = bool(self.load_model(intent))except Exception as exc:logger.error(f"Failed to respawn llama-server: {exc}")return False@@ -33406,45 +34999,239 @@ class LlamaCppBackend:return Falsereturn started- def _retire_device_lost_server(self, served_by) -> None:- """Kills the llama-server that hit VK_ERROR_DEVICE_LOST if still current: it survives but fails later requests; a dead child is handled by _respawn_if_dead."""- with self._respawn_lock:+ def _stop_stats_logger(self) -> None:+ """Stop the engine-stats poller of the child going away. getattr: teardown must+ tolerate a partially-built backend (failed __init__ or a __new__-built instance)."""+ if getattr(self, "_stats_logger", None) is not None:+ self._stats_logger.stop()+ self._stats_logger = None++ def _retire_device_lost_server(+ self,+ served_by,+ metal_poison: Optional[dict] = None,+ ) -> None:+ """Kill the llama-server that hit a lost GPU device, if it is still current (#11453).++ It survives VK_ERROR_DEVICE_LOST but fails every later request; a dead child is+ what _respawn_if_dead already recovers. ``metal_poison`` marks the Metal variant:+ a failed command buffer (usually out of memory) latches the backend the same way.+ """+ if metal_poison is not None:+ # Outside the locks: the probe waits on the child up to its timeout.if (served_by is Noneor served_by is not self._processor served_by.poll() is not Noneor self._cancel_event.is_set()+ or not self._metal_compute_failure_confirmed()):+ metal_poison["pending"] = Falsereturn- # Else the MTP watchdog reads this kill as an MTP crash.+ # _serial_load_lock too: no load may publish another model between the kill and the strike.+ with self._respawn_lock, self._serial_load_lock:+ if (+ served_by is None+ or served_by is not self._process+ or served_by.poll() is not None+ or self._cancel_event.is_set()+ ):+ return+ # First, or the MTP watchdog reads this kill as an MTP crash.self._stop_mtp_crash_watchdog()+ self._retired_process = served_bytry:served_by.kill()served_by.wait(timeout = 10)except (OSError, subprocess.TimeoutExpired):pass+ # Else it polls the dead child's port, which another server may reuse.+ self._stop_stats_logger()+ if metal_poison is not None:+ # Under _respawn_lock, or a request could respawn the speculative config.+ terminal = False+ with self._lock:+ if self._process is served_by and not self._cancel_event.is_set():+ terminal = self._after_metal_poisoned_server(+ bool(metal_poison.get("draft"))+ )+ if not terminal:+ return+ # Nothing will respawn it: unload it like any other model, still under+ # _serial_load_lock.+ self.unload_model()+ # A reload queued on _serial_load_lock may land before the hooks run.+ retired_epoch = self._unload_epoch+ if metal_poison is not None:+ # Outside every backend lock (gpu_arbiter._lock comes first).+ for hook in list(_TERMINAL_RETIREMENT_HOOKS):+ try:+ hook(self, retired_epoch)+ except Exception as exc: # noqa: BLE001 -- best effort, as a reap+ logger.debug("Terminal retirement cleanup failed: %s", exc)+ returntry:logger.warning(f"llama-server for '{self._model_identifier}' lost its GPU device "- "(the graphics driver reset it); restarted it. On Intel Arc with "- "Vulkan, GGML_VK_DISABLE_COOPMAT=1 may avoid the reset."+ "(the graphics driver reset it); stopped it so the next request starts "+ "a fresh one. On Intel Arc with Vulkan, GGML_VK_DISABLE_COOPMAT=1 may "+ "avoid the reset.")except Exception:# The logger raises once stdout is closed; recovery must not.pass- def _retire_device_lost_server_soon(self, served_by) -> None:- """Retires a server whose log shows a device loss, after a grace for chat requests to retry- first."""+ def _metal_compute_failure_confirmed(self) -> bool:+ """Whether the child really fails compute: a poisoned Metal backend rejects every+ graph, so a one-token request comes back 500 "Compute error.". The log marker+ alone is not proof, since request text can reach the server's error output. An+ --embedding server is restricted to embeddings, so it is probed on the native+ /embedding route, which computes under every pooling type but RANK (/v1/embeddings+ 400s on NONE pooling before compute). RANK pools n_cls_out floats while+ send_embedding reads n_embd_out of them, so a reranker (--rerank or --pooling rank)+ is probed on /rerank, and one ranking only by the GGUF's default (/rerank refuses+ it) is not probed at all."""+ try:+ embedding = bool(self.is_embedding_gguf)+ except Exception:+ embedding = False+ args = [str(a) for a in (self._extra_args or ())]+ # Argv only (load_model strips inherited LLAMA_ARG_POOLING); last-wins, as common/arg.cpp+ # applies them.+ pooling = None+ for i, raw in enumerate(args):+ flag = _flag_name(raw)+ if flag in ("--embedding", "--embeddings"):+ embedding = True+ elif flag in ("--rerank", "--reranking"):+ embedding = True+ pooling = "rank"+ elif flag == "--pooling":+ _, eq, inline = raw.partition("=")+ value = inline if eq else (args[i + 1] if i + 1 < len(args) else "")+ pooling = value.strip().lower() or pooling+ if embedding and pooling == "rank":+ route, body = "/rerank", {"query": " ", "documents": [" "]}+ elif embedding and pooling is None and getattr(self, "_pooling_type", None) == 4:+ return False+ elif embedding:+ route, body = "/embedding", {"content": " "}+ else:+ route, body = "/completion", {"prompt": " ", "n_predict": 1, "stream": False}+ try:+ resp = httpx.post(+ f"{self.base_url}{route}",+ json = body,+ headers = self._auth_headers,+ timeout = 10.0,+ trust_env = False,+ )+ if resp.status_code < 500:+ return False+ error = resp.json().get("error", {})+ message = error.get("message", "") if isinstance(error, dict) else error+ return (+ str(message)+ .rstrip()+ .endswith(("Compute error.", "failed to process speculative batch"))+ )+ except Exception:+ return False++ @staticmethod+ def _metal_oom_key(intent):+ """The load an OOM strike is charged to: the speculative mode is what the fallback+ changes, so it is left out and the no-drafter reload keeps the strike count. The+ verified-cache hint is per request (the committed snapshot drops it), so it is left+ out too, or a hinted explicit reload and the snapshot it commits disagree."""+ try:+ return replace(intent, speculative_type = None, verified_gguf = None)+ except TypeError:+ return intent++ def _metal_oom_is_terminal(self, intent) -> bool:+ """Whether ``intent`` already ran out of Metal memory twice (no more replays)."""+ terminal = getattr(self, "_metal_oom_terminal_key", None)+ return (+ terminal is not None and intent is not None and self._metal_oom_key(intent) == terminal+ )++ @contextlib.contextmanager+ def _while_still_retired(self, epoch):+ """Hold _serial_load_lock, which every load takes, and yield whether this backend is+ still the one a terminal Metal strike unloaded at ``epoch``: nothing spawned since and+ no other unload. A cleanup acting on a replacement would kill it."""+ with self._serial_load_lock:+ yield self._process is None and getattr(self, "_unload_epoch", None) == epoch++ def _after_metal_poisoned_server(self, draft_failed: bool) -> bool:+ """Choose the recovery after a Metal-poisoned server was retired; True on the terminal+ second strike."""+ intent = self._last_load_intent+ key = self._metal_oom_key(intent) if intent is not None else None+ prior = getattr(self, "_metal_oom_strikes", None)+ strikes = prior[1] + 1 if prior is not None and key is not None and prior[0] == key else 1+ self._metal_oom_strikes = (key, strikes)+ where = "the speculative draft context" if draft_failed else "a decode"+ try:+ name = self._model_identifier+ if strikes >= 2:+ # Not replaced: an identical explicit load must spawn a new child.+ self._healthy = False+ self._metal_oom_terminal_key = key+ logger.error(+ f"llama-server for '{name}' ran out of Metal memory again in {where} "+ "after a restart; not restarting it. Lower the context length or the "+ "GPU layers, turn off speculative decoding, or choose a smaller quant, "+ "then load the model again."+ )+ return True+ requested_mode = getattr(intent, "speculative_type", None)+ recovery_extras = list(getattr(intent, "extra_args", None) or ())+ # A bare --model-draft in the extras survives an Off reload (has_dft()).+ auto_sidecar = (+ self._speculative_type in ("draft-dspark", "draft-dflash")+ and _canonicalize_spec_mode(requested_mode) in (None, "auto")+ and not _extra_args_set_spec_type(recovery_extras)+ and not _extra_args_mtp_draft_path(+ recovery_extras, env = _child_spec_env(recovery_extras)+ )+ )+ if auto_sidecar and self._maybe_recover_from_mtp_crash(sidecar_oom = True):+ logger.warning(+ f"llama-server for '{name}' ran out of Metal memory in {where} with the "+ f"{self._speculative_type.split('-', 1)[1]} drafter and its Metal backend "+ "cannot recover; reloading once without speculative decoding."+ )+ return False+ logger.warning(+ f"llama-server for '{name}' ran out of Metal memory in {where} and its Metal "+ "backend cannot recover; stopped it so the next request starts a fresh one."+ )+ except Exception:+ pass+ return strikes >= 2++ def _retire_device_lost_server_soon(+ self,+ served_by,+ metal_poison: Optional[dict] = None,+ ) -> None:+ """Retire after a device loss in the server log, for the passthrough routes.++ The grace lets a chat request that hit the same error retire and retry first.+ """def _retire():time.sleep(_DEVICE_LOST_LOG_GRACE_S)- self._retire_device_lost_server(served_by)+ self._retire_device_lost_server(served_by, metal_poison = metal_poison)try:threading.Thread(target = _retire, daemon = True, name = "llama-device-lost").start()except RuntimeError:- pass+ if metal_poison is not None:+ # No retirement will run: let a later poison line try again.+ metal_poison["pending"] = False@contextlib.contextmanagerdef _open_chat_stream_with_respawn_retry(diff --git a/studio/backend/routes/inference.py b/studio/backend/routes/inference.pyindex 943e37c5124..ee77adca9c6 100644--- a/studio/backend/routes/inference.py+++ b/studio/backend/routes/inference.py@@ -165,6 +165,7 @@ from core.inference.llama_admission import (from core.inference.tool_stream_exec import TOOL_APPROVAL_FLUSH_DELAY_Sfrom core.inference import model_slotsfrom core.inference.llama_cpp import (+ _TERMINAL_RETIREMENT_HOOKS,_llama_chunk_has_generated_output,_local_ssl_context,register_serving_backend,@@ -11562,6 +11563,57 @@ def reap_dead_managed_engine(backend) -> None:account_access.clear_resident("chat")+def _orchestrator_occupied(orchestrator) -> bool:+ """A standard, MLX or managed-engine model held or loading, as release_chat_gpu_claim counts it."""+ if orchestrator is None:+ return False+ return bool(+ getattr(orchestrator, "_managed_engine", None) is not None+ or getattr(orchestrator, "active_model_name", None)+ or tuple(getattr(orchestrator, "loading_models", ()) or ())+ )+++def _primary_seat_occupied() -> bool:+ """A non-GGUF model holds or is loading into the primary seat (call in the primary's slot)."""+ return _orchestrator_occupied(_peek_inference_backend()) or _resident_npu_model() is not None+++def _release_after_terminal_retirement(backend, epoch) -> None:+ """A llama-server stopped for good by a second Metal out-of-memory strike unloads itself+ (llama_cpp._retire_device_lost_server); finish it as a manual unload would. A kept slot is+ dropped at once (dead, it holds nothing a request could still use, so its refs do not keep+ it) and leaves the primary's residency alone; the primary clears its own sharers. CHAT goes+ once nothing else holds it. A reload that landed since the strike owns the slot and the+ residency now, so nothing is touched then (checked under the lock every load takes).+ A non-GGUF model holding or loading into the seat (primary or kept slot) owns it too."""+ primary = backend is _llama_cpp_backend+ with backend._while_still_retired(epoch) as retired:+ filling = model_slots.loading[0] if model_slots.loading else None+ if not retired or (filling is not None and filling.llama is backend):+ return+ for slot in [+ s+ for s in list(model_slots.slots)+ if s.llama is backend and not _orchestrator_occupied(s.orchestrator)+ ]:+ try:+ model_slots.drop(slot)+ except Exception as exc: # noqa: BLE001 -- drop() keeps a stuck slot for retry+ logger.warning("Could not drop a kept model after its Metal out-of-memory: %s", exc)+ if primary and not model_slots.in_slot(None, _primary_seat_occupied):+ account_access.clear_resident("chat")+ # The arbiter lock comes before the backend's: released outside it, and release_if+ # re-checks that nothing is resident.+ if primary:+ release_chat_gpu_claim()+ else:+ release_chat_after_kept_models()+++_TERMINAL_RETIREMENT_HOOKS.append(_release_after_terminal_retirement)++def reap_dead_worker(backend) -> None:"""Cleared, never respawned: every routed request runs this, so an OOM respawn would loop."""from core.inference.gpu_arbiter import CHAT, holds@@ -14727,12 +14779,36 @@ def _wait_for_native_audio_gpu_free_gb(async def _unload_llama_before_standard_load(llama_backend) -> None:- """Tear down llama-server and wait for asynchronous driver VRAM reclaim."""- if not llama_backend.is_loaded:+ """Tear down llama-server and wait for asynchronous driver VRAM reclaim.++ Unloads even when nothing is loaded, under the lock that every GGUF load and a terminal+ Metal retirement's cleanup (_release_after_terminal_retirement) hold: the unload advances+ the epoch, so a cleanup still pending sees this load replaced the retired server, and one+ already past its check finishes first."""++ def handoff() -> Optional[float]:+ # getattr: a stand-in backend has no load to order against.+ lock = getattr(llama_backend, "_serial_load_lock", None)+ if lock is not None:+ # As the unload would: a recovery reload holding the lock stops instead of finishing.+ llama_backend._cancel_event.set()+ with lock or nullcontext():+ if not llama_backend.is_loaded:+ if lock is None:+ return None+ kill_started = time.monotonic()+ llama_backend.unload_model()+ # A starting or recovering server was killed: wait for it like a loaded one.+ last_kill = getattr(llama_backend, "_last_kill_monotonic", 0.0) or 0.0+ return kill_started if last_kill >= kill_started else None+ logger.info("Unloading GGUF model before loading Unsloth model")+ kill_started = time.monotonic()+ llama_backend.unload_model()+ return kill_started++ kill_started = await asyncio.to_thread(handoff)+ if kill_started is None:return- logger.info("Unloading GGUF model before loading Unsloth model")- kill_started = time.monotonic()- await asyncio.to_thread(llama_backend.unload_model)await asyncio.to_thread(llama_backend._wait_for_vram_settle,since_kill = kill_started,diff --git a/studio/backend/tests/_sidecar_fit_helpers.py b/studio/backend/tests/_sidecar_fit_helpers.pynew file mode 100644index 00000000000..eb3490fc4e0--- /dev/null+++ b/studio/backend/tests/_sidecar_fit_helpers.py@@ -0,0 +1,323 @@+# SPDX-License-Identifier: AGPL-3.0-only+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0++"""Shared launch harness for the DSpark/DFlash sidecar reserve tests."""++from __future__ import annotations++import subprocess+import sys+from pathlib import Path+from unittest.mock import patch++_TESTS_DIR = str(Path(__file__).resolve().parent)+if _TESTS_DIR not in sys.path:+ sys.path.insert(0, _TESTS_DIR)++import test_llama_cpp_placement as placement # noqa: E402+import test_metal_explicit_context_guard as metal_guard # noqa: E402+from core.inference import llama_cpp as lc # noqa: E402+from core.inference.llama_cpp import GgufLoadIntent, LlamaCppBackend # noqa: E402++MIB = 1024 * 1024+GIB = 1024 * MIB+CAPS = {+ "supports_dspark": True,+ "supports_dflash": True,+ "supports_fit_target": True,+ "supports_fit_ctx": True,+ "supports_kv_unified": True,+ "spec_draft_n_max_flag": "--spec-draft-n-max",+}+# llama.cpp's default 1024 MiB margin plus the 8 GiB sidecar.+FLOOR = 1024 + 8 * 1024+ONE = ((0, 24_000, 24_000),)+TWO = ((0, 24_000, 24_000), (1, 24_000, 24_000))+MANUAL_AUTO = {"gpu_memory_mode": "manual", "gpu_layers": -1}+DRAFT_KV = 768 * MIB+NATIVE_MARKER = b"measuring the extra model next to an allocation-free target context"+UNSET = object()+++def last(cmd, name):+ """Last-wins value of ``name``, in either the spaced or the ``=`` spelling."""+ value = None+ for i, tok in enumerate(cmd):+ if tok == name and i + 1 < len(cmd):+ value = cmd[i + 1]+ elif tok.startswith(name + "="):+ value = tok.split("=", 1)[1]+ return value+++def margins(cmd):+ raw = last(cmd, "--fit-target") or "1024"+ return [int(float(e)) for e in str(raw).split(",")]+++def margin(cmd):+ return max(margins(cmd))+++def fit_targets(cmd):+ return [cmd[i + 1] for i, a in enumerate(cmd[:-1]) if a == "--fit-target"]+++def sidecar(tmp_path, kind = "dspark"):+ path = tmp_path / f"{kind}-model-Q8_0.gguf"+ path.write_bytes(b"draft")+ return str(path)+++def cuda_backend(+ tmp_path,+ memory = ONE,+ vulkan = False,+ draft_key = "dspark",+):+ backend, gguf = placement._backend(tmp_path, vulkan = vulkan, memory = list(memory))+ backend._select_gpus = lambda *a, **k: (None, True)+ backend.probe_server_capabilities = lambda _binary = None: dict(CAPS)+ backend._get_gguf_size_bytes = lambda path: 8 * GIB if draft_key in str(path) else 1024+ # A discrete-GPU host on every OS, not an Apple Silicon runner's unified budget.+ backend._apple_metal_memory_budget_bytes = lambda: 0+ return backend, gguf+++def cards(+ tmp_path,+ memory = TWO,+ vulkan = False,+):+ backend, gguf = cuda_backend(tmp_path, memory = memory, vulkan = vulkan)+ backend._arch_gate_survivors = lambda binary = None: []+ return backend, gguf+++def dspark(backend, gguf, tmp_path, **kw):+ return placement._launch(+ backend, gguf, speculative_type = "dspark", dspark_draft_path = sidecar(tmp_path), **kw+ )["cmd"]+++def per_token_estimate(backend, per_token):+ backend._estimate_mtp_overhead_bytes = lambda n_ctx, **kw: (+ 8 * GIB + max(0, int(n_ctx)) * per_token+ )+++def slot_priced(backend):+ backend._estimate_mtp_overhead_bytes = lambda n_ctx, n_parallel = 1, **kw: (+ 8 * GIB + max(1, int(n_parallel or 1)) * GIB+ )+ return backend+++def three_block_drafter(backend):+ read = backend._draft_backend_for++ def three_blocks(path):+ db = read(path)+ db._n_layers = 3+ return db++ backend._draft_backend_for = three_blocks+++def elf(path, payload = b""):+ path.write_bytes(b"\x7fELF" + payload + b"\x00")+ return path+++def info_messages(monkeypatch):+ seen = []+ real_info = lc.logger.info++ def info(msg, *args, **kwargs):+ try:+ seen.append(str(msg) % args if args else str(msg))+ except TypeError:+ seen.append(str(msg))+ return real_info(msg, *args, **kwargs)++ monkeypatch.setattr(lc.logger, "info", info)+ return seen+++class _Spawns(list):+ loaded = None+++def spawns(+ backend,+ gguf,+ *,+ healthy,+ crash = lambda i, cmd: None,+ stdout = lambda i, cmd: (),+ late_exit = False,+ **intent,+):+ """Every llama-server argv the load spawns. ``crash(i, cmd)`` is spawn i's exit code+ (None = running), ``stdout`` its output, ``healthy`` its health answer. ``late_exit``+ makes the first poll see the child running: it exits between the startup retry's check+ and the outer --flash-attn off check."""+ cmds = _Spawns()++ def fake_popen(cmd, **kwargs):+ if not cmd or str(cmd[0]) != "/fake/llama-server":+ return placement._REAL_POPEN(cmd, **kwargs)+ cmds.append(list(cmd))+ rc = crash(len(cmds) - 1, list(cmd))+ polls = [0]++ def poll(self):+ polls[0] += 1+ return None if late_exit and polls[0] == 1 else rc++ return type(+ "Process",+ (),+ {+ "pid": 2**31 - 1,+ "stdout": (),+ "returncode": rc,+ "poll": poll,+ "terminate": lambda self: None,+ "wait": lambda self, timeout = None: rc or 0,+ "kill": lambda self: None,+ },+ )()++ def fake_health(timeout = None, **_kw):+ i = len(cmds) - 1+ backend._stdout_lines = list(stdout(i, cmds[-1]))+ return healthy(i, cmds[-1])++ backend._wait_for_health = fake_health+ with patch.object(subprocess, "Popen", side_effect = fake_popen):+ try:+ cmds.loaded = backend.load_model(+ GgufLoadIntent(gguf_path = str(gguf), model_identifier = "test", **intent)+ )+ except Exception:+ pass+ return cmds+++class _ReserveReport:+ """Whether the user is told the sidecar is reserved: planning's verdict, then the+ spawn's report off the argv it launched (the last outcome a retry logged)."""++ def __init__(self):+ self.planned = None+ self.spawned: list = []++ def __getitem__(self, key):+ assert key == "fit_sized"+ if self.planned is not True:+ return self.planned+ return self.spawned[-1] if self.spawned else True+++def spy_fit_sized(backend):+ seen = _ReserveReport()+ build = backend._build_speculative_flags+ report = backend._log_sidecar_fit_reserve++ def spy(**kw):+ seen.planned = kw.get("dspark_fit_sized")+ return build(**kw)++ def log(run_cmd, env, kind, path, reserved_mib, logged):+ before = len(logged)+ report(run_cmd, env, kind, path, reserved_mib, logged)+ if len(logged) > before:+ seen.spawned.append(logged[-1][1])++ backend._build_speculative_flags = spy+ backend._log_sidecar_fit_reserve = log+ return seen+++def host_kv_delta(+ tmp_path,+ monkeypatch,+ *,+ memory = ONE,+ vulkan = False,+ listed = UNSET,+ apu = UNSET,+ extras = (),+ **kw,+):+ """MiB the -nkvo launch's largest margin is below the KV-on-card launch's.+ ``listed`` is what --list-devices reports (None: no answer); ``apu`` the card whose+ memory is the host's (None: none)."""+ if listed is not UNSET:+ monkeypatch.setattr(+ LlamaCppBackend,+ "_enumerated_gpu_devices",+ classmethod(+ lambda cls, binary = None, env = None: None if listed is None else list(listed)+ ),+ )+ out = []+ for name, more in (("on", []), ("off", ["-nkvo"])):+ d = tmp_path / name+ d.mkdir()+ backend, gguf = cards(d, memory = memory, vulkan = vulkan)+ backend._mtp_draft_kv_bytes = lambda *a, **k: DRAFT_KV+ if apu is not UNSET:+ backend._amd_apu_wants_unified_memory = lambda gpu_indices = None: (+ apu is not None and (gpu_indices is None or apu in gpu_indices)+ )+ backend._integrated_cuda_unified_memory = lambda gpu_indices = None: False+ out.append(margin(dspark(backend, gguf, d, extra_args = [*extras, *more], **kw)))+ return out[0] - out[1]+++def metal_cmd(+ tmp_path,+ monkeypatch,+ mode,+ *,+ budget_gib,+ wired_gib = None,+ sub = None,+ n_ctx = 0,+ **kw,+):+ """A real-fit Metal load with a DSpark sidecar; every GGUF here reads 4 GiB, the+ sidecar too. Undoes ``monkeypatch`` so a test can load twice."""+ sub = tmp_path / (sub or mode)+ sub.mkdir()+ path = sidecar(sub)+ real_init = GgufLoadIntent.__init__++ def init(self, *args, **kwargs):+ kwargs.setdefault("dspark_draft_path", path)+ kwargs.setdefault("speculative_type", mode)+ real_init(self, *args, **kwargs)++ monkeypatch.setattr(GgufLoadIntent, "__init__", init)+ monkeypatch.setattr(+ LlamaCppBackend,+ "probe_server_capabilities",+ classmethod(lambda cls, binary = None: dict(CAPS)),+ )+ backend = LlamaCppBackend()+ wired_gib = budget_gib if wired_gib is None else wired_gib+ cmd = metal_guard._launch(+ sub,+ monkeypatch,+ n_ctx = n_ctx,+ backend = backend,+ real_fit = True,+ budget_bytes = int(budget_gib * GIB),+ wired_bytes = int(wired_gib * GIB),+ weights_bytes = 4 * GIB,+ **kw,+ )["cmd"]+ monkeypatch.undo()+ return cmd, backend, pathdiff --git a/studio/backend/tests/test_llama_cpp_metal_poisoned_respawn.py b/studio/backend/tests/test_llama_cpp_metal_poisoned_respawn.pynew file mode 100644index 00000000000..5d79f530cba--- /dev/null+++ b/studio/backend/tests/test_llama_cpp_metal_poisoned_respawn.py@@ -0,0 +1,990 @@+# SPDX-License-Identifier: AGPL-3.0-only+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0++"""A llama-server whose Metal backend is poisoned is retired, respawned once, then unloaded."""++from __future__ import annotations++import asyncio+import inspect+import subprocess+import sys+import threading+import time+from dataclasses import dataclass, replace+from typing import Optional+from unittest.mock import patch++import httpx+import pytest++import core.inference.llama_cpp as llama_cpp+from core.inference import model_slots+from core.inference.llama_cpp import GgufLoadIntent, LlamaCppBackend++from . import _sidecar_fit_helpers as h++# After a failed command buffer ggml-metal rejects every later graph until the backend is+# recreated, so the child stays up answering "Compute error.". The reporter's lines:+METAL_OOM = "0.30.864.334 E error: Insufficient Memory (00000008:kIOGPUCommandBufferCallbackErrorOutOfMemory)"+METAL_LATCHED = (+ "0.30.882.907 E ggml_metal_graph_compute: backend is in error state from a previous "+ "command buffer failure - recreate the backend to recover"+)+DRAFT_FAILED = "0.22.891.174 E process: llama_process(ctx_dft) failed rc=-3 (n_tokens=14, offset=0)"+++@dataclass(frozen = True)+class _Intent:+ speculative_type: Optional[str] = None+ extra_args: tuple = ()+ gpu_ids: Optional[tuple] = None+ verified_gguf: Optional[str] = None+++_OFF = _Intent(speculative_type = "off")+_MTP = _Intent(speculative_type = "mtp")+++def _child(lines = ()):+ code = "import sys, time\n"+ for line in lines:+ code += f"print({line!r}, flush = True)\n"+ code += "time.sleep(120)\n"+ return subprocess.Popen([sys.executable, "-c", code], stdout = subprocess.PIPE, text = True)++[email protected](autouse = True)+def _studio_home(tmp_path, monkeypatch):+ monkeypatch.setenv("UNSLOTH_STUDIO_HOME", str(tmp_path / "studio_home"))+++def harness(monkeypatch):+ monkeypatch.setattr(llama_cpp, "_DEVICE_LOST_LOG_GRACE_S", 0.0)+ children = []++ def make(+ intent: _Intent,+ speculative_type: str = "draft-dspark",+ lines = (),+ ):+ backend = LlamaCppBackend()+ first = _child(lines)+ children.append(first)+ backend._process = first+ backend._healthy = True+ backend._model_identifier = "unsloth/DeepSeek-V4.1-Flash-1017-GGUF"+ backend._last_load_intent = intent+ backend._speculative_type = speculative_type+ # The child here is a stub with no HTTP server: stand in for a compute that fails.+ backend._metal_compute_failure_confirmed = lambda: True+ loads = []++ def fake_load_model(new_intent):+ loads.append(new_intent)+ replacement = _child()+ children.append(replacement)+ backend._process = replacement+ backend._healthy = True+ backend._last_load_intent = new_intent+ backend._speculative_type = (+ "none" if new_intent.speculative_type == "off" else speculative_type+ )+ return True++ backend.load_model = fake_load_model+ backend._server_socket_is_open = lambda: False+ return backend, first, loads++ yield make+ for child in children:+ if child.poll() is None:+ child.kill()+ child.wait()+++def _wait(predicate, timeout = 10.0):+ deadline = time.monotonic() + timeout+ while time.monotonic() < deadline:+ if predicate():+ return True+ time.sleep(0.02)+ return False+++def _poison(+ backend,+ process,+ draft = False,+):+ backend._retire_device_lost_server(process, metal_poison = {"draft": draft})+++ "lines, retired",+ [+ ([METAL_OOM], True),+ ([METAL_LATCHED], True),+ (+ [+ "0.01.000.000 I slot launch_slot_: id 0 | prompt: what is kIOGPUCommandBufferCallbackErrorOutOfMemory?"+ ],+ False,+ ),+ (+ ["what does backend is in error state from a previous command buffer failure mean?"],+ False,+ ),+ ],+ ids = ["oom_line", "latched_line", "info_line", "unprefixed_request_dump"],+)+def test_a_poisoned_metal_backend_in_the_log_retires_the_server(harness, lines, retired):+ backend, first, _loads = harness(_OFF, "none", lines)+ threading.Thread(target = backend._drain_stdout, daemon = True).start()+ try:+ first.wait(timeout = 5)+ except subprocess.TimeoutExpired:+ pass+ assert (first.poll() is not None) is retired+++def test_echoed_marker_without_a_compute_failure_does_not_retire(harness):+ """A grammar parse error prints the request's grammar, so a forged E-line can carry the+ marker: retirement needs the child to really fail compute."""+ backend, first, _loads = harness(_OFF, "none", [METAL_OOM])+ backend._metal_compute_failure_confirmed = lambda: False+ threading.Thread(target = backend._drain_stdout, daemon = True).start()+ try:+ first.wait(timeout = 1.5)+ except subprocess.TimeoutExpired:+ pass+ assert first.poll() is None+ assert getattr(backend, "_metal_oom_strikes", None) is None+++def test_auto_dspark_draft_oom_reloads_once_without_the_drafter_then_stops(harness):+ backend, first, loads = harness(+ _Intent(), "draft-dspark", [METAL_OOM, METAL_LATCHED, DRAFT_FAILED]+ )+ threading.Thread(target = backend._drain_stdout, daemon = True).start()++ assert _wait(lambda: len(loads) == 1)+ assert first.poll() is not None+ assert loads[0].speculative_type == "off"+ assert _wait(lambda: not backend._mtp_runtime_fallback_in_progress)++ # The drafter-free server runs out of Metal memory too: the second strike is terminal.+ second = backend._process+ _poison(backend, second)+ assert second.poll() is not None+ assert backend._respawn_if_dead() is False+ assert len(loads) == 1+++def test_explicit_dspark_is_not_silently_dropped(harness):+ intent = _Intent(speculative_type = "dspark")+ backend, first, loads = harness(intent, "draft-dspark")+ _poison(backend, first, draft = True)+ time.sleep(0.3)+ # No drafter-free reload; the next request replays the user's own settings once.+ assert loads == [] and first.poll() is not None+ assert backend._respawn_if_dead() is True+ assert loads == [intent]+++def test_retirement_keeps_a_user_owned_drafter(harness):+ """Auto plus a bare --model-draft: an Off reload would still load that file."""+ intent = _Intent(extra_args = ("--model-draft", "/models/custom-dspark.gguf"))+ backend, first, loads = harness(intent, "draft-dspark")+ _poison(backend, first, draft = True)+ assert first.poll() is not None+ _wait(lambda: bool(loads), timeout = 2.0)+ assert all(load.speculative_type != "off" for load in loads)+++def test_a_launch_without_a_sidecar_respawns_once_then_stops(harness):+ backend, first, loads = harness(_OFF, "none")+ _poison(backend, first)+ assert backend._respawn_if_dead() is True and loads == [_OFF]+ _poison(backend, backend._process)+ assert backend._respawn_if_dead() is False and loads == [_OFF]+++def test_a_terminal_second_oom_is_not_left_healthy(harness):+ backend, first, _loads = harness(_OFF, "none")+ _poison(backend, first)+ assert backend._respawn_if_dead() is True+ _poison(backend, backend._process)+ assert backend._healthy is False+++def test_a_terminally_retired_server_is_not_active(harness):+ """After the second Metal OOM nothing respawns the child, so delete, Always mode and the+ training guard must not see a model still loading."""+ backend, first, _loads = harness(_OFF, "none")+ _poison(backend, first)+ assert backend._respawn_if_dead() is True+ assert backend.is_active+ second = backend._process+ _poison(backend, second)+ assert second.poll() is not None+ assert backend._process is None and backend._model_identifier is None+ assert backend._respawn_if_dead() is False+ assert backend.is_loaded is False+ # routes/models.py refuses a delete with 409 while is_active or is_loaded.+ assert backend.is_active is False+++def test_unload_clears_the_strikes(harness):+ backend, first, _loads = harness(_OFF, "none")+ backend._metal_oom_strikes = (_OFF, 2)+ backend._metal_oom_terminal_key = _OFF+ backend.unload_model()+ assert backend._metal_oom_strikes is None and backend._metal_oom_terminal_key is None+++def test_a_model_swap_during_retirement_is_not_charged_the_strike(harness):+ """An explicit load must not publish another model while the dead child is reaped."""+ backend, first, _loads = harness(_OFF, "none")+ real_kill = first.kill+ blocked = {}++ def kill_and_try_a_load():+ real_kill()+ got = []+ t = threading.Thread(+ target = lambda: got.append(backend._serial_load_lock.acquire(blocking = False))+ )+ t.start()+ t.join()+ blocked["load"] = got == [False]+ if got == [True]:+ backend._serial_load_lock.release()++ first.kill = kill_and_try_a_load+ _poison(backend, first)+ assert blocked.get("load") is True+ assert backend._metal_oom_strikes == (backend._metal_oom_key(_OFF), 1)+++def test_the_strike_is_recorded_before_a_respawn_can_run(harness):+ """Retirement decides the recovery before releasing _respawn_lock, so a request racing+ the kill cannot replay the speculative config first."""+ backend, first, loads = harness(_Intent(), "draft-dspark")+ seen = {}+ real_after = backend._after_metal_poisoned_server++ def after(draft_failed):+ seen["respawn_lock_held"] = backend._respawn_lock.locked()+ return real_after(draft_failed)++ backend._after_metal_poisoned_server = after+ _poison(backend, first, draft = True)+ assert seen.get("respawn_lock_held") is True+ assert _wait(lambda: len(loads) == 1)+ assert loads[0].speculative_type == "off"+++def test_mtp_recovery_respects_the_terminal_strike(harness):+ backend, first, loads = harness(_Intent(), "draft-dspark")+ backend._metal_oom_terminal_key = backend._metal_oom_key(_Intent())+ backend._mtp_runtime_fallback_active = True+ first.kill()+ first.wait()+ assert backend._maybe_recover_from_mtp_crash(RuntimeError("boom")) is False+ assert backend._maybe_recover_from_mtp_crash(sidecar_oom = True) is False+ time.sleep(0.3)+ assert loads == []+++def test_a_retirement_thread_that_cannot_start_releases_the_poison_flag(monkeypatch):+ def refuse(self):+ raise RuntimeError("can't start new thread")++ monkeypatch.setattr(threading.Thread, "start", refuse)+ poison = {"pending": True}+ LlamaCppBackend()._retire_device_lost_server_soon(None, metal_poison = poison)+ assert poison["pending"] is False+++# Confirming the poison: probe the route the server really computes on.+++def _compute_probe(+ monkeypatch,+ extras = None,+ embedding = None,+ message = "Compute error.",+):+ backend = LlamaCppBackend()+ backend._port = 1+ if extras is not None:+ backend._extra_args = list(extras)+ if embedding is not None:+ monkeypatch.setattr(LlamaCppBackend, "is_embedding_gguf", property(lambda self: embedding))+ resp = type(+ "Resp", (), {"status_code": 500, "json": lambda self: {"error": {"message": message}}}+ )()+ with patch.object(llama_cpp.httpx, "post", return_value = resp, create = True) as post:+ return backend._metal_compute_failure_confirmed(), post+++ "message, confirmed",+ [+ ("Compute error.", True),+ ("decode() failed: failed to process speculative batch", True),+ ("the request exceeds the available context size", False),+ ],+ ids = ["target", "draft", "unrelated"],+)+def test_the_draft_context_failure_confirms_the_poison(monkeypatch, message, confirmed):+ assert _compute_probe(monkeypatch, message = message)[0] is confirmed+++ "env, embedding, extras, route, field",+ [+ # --rerank sets embedding mode with RANK pooling: /completion 501s before compute,+ # and send_embedding would read n_embd_out floats out of RANK's n_cls_out.+ ({}, False, ("--rerank",), "/rerank", "documents"),+ ({}, False, ("--reranking",), "/rerank", "documents"),+ ({}, False, ("--embeddings", "--pooling", "rank"), "/rerank", "documents"),+ # A later --pooling replaces --rerank's RANK, as in common/arg.cpp.+ ({}, False, ("--rerank", "--pooling", "mean"), "/embedding", "content"),+ # /v1/embeddings 400s on NONE pooling, explicit or the model's default.+ ({}, False, ("--embeddings", "--pooling", "none"), "/embedding", "content"),+ ({}, True, (), "/embedding", "content"),+ ({}, False, (), "/completion", None),+ # --embedding restricts llama-server to embeddings: /completion never computes.+ ({}, False, ("--embeddings",), "/embedding", None),+ # load_model pops LLAMA_ARG_POOLING / _RERANKING / _EMBEDDINGS from the child's env.+ ({"LLAMA_ARG_RERANKING": "1"}, False, (), "/completion", None),+ ({"LLAMA_ARG_POOLING": "rank"}, True, (), "/embedding", None),+ ],+ ids = [+ "rerank",+ "reranking",+ "pooling-rank",+ "rerank-then-mean",+ "pooling-none",+ "embedding-gguf",+ "completion",+ "embeddings-flag",+ "inherited-reranking-env",+ "inherited-pooling-env",+ ],+)+def test_embedding_and_rerank_servers_are_probed_where_they_compute(+ monkeypatch, env, embedding, extras, route, field+):+ monkeypatch.delenv("LLAMA_ARG_POOLING", raising = False)+ monkeypatch.delenv("LLAMA_ARG_RERANKING", raising = False)+ for key, value in env.items():+ monkeypatch.setenv(key, value)+ confirmed, post = _compute_probe(monkeypatch, extras, embedding)+ assert confirmed is True+ assert post.call_args.args[0].endswith(route)+ if field is not None:+ assert field in post.call_args.kwargs["json"]+++def test_a_reranker_by_gguf_default_alone_is_not_probed(monkeypatch):+ """--embeddings on a RANK-pooling GGUF: /rerank refuses (no --rerank) and /embedding would+ over-read RANK's buffer, so nothing is sent and the server is not retired."""+ monkeypatch.delenv("LLAMA_ARG_POOLING", raising = False)+ monkeypatch.delenv("LLAMA_ARG_RERANKING", raising = False)+ backend = LlamaCppBackend()+ backend._port = 1+ backend._pooling_type = 4+ backend._extra_args = ["--embeddings"]+ with patch.object(llama_cpp.httpx, "post", create = True) as post:+ assert backend._metal_compute_failure_confirmed() is False+ post.assert_not_called()+++# Metal OOM episodes: a strike belongs to one load; a caller's load starts a fresh one.+++def _stub_child():+ return type(+ "Process",+ (),+ {+ "pid": 123,+ "poll": lambda self: 0,+ "kill": lambda self: None,+ "wait": lambda self, timeout = None: 0,+ },+ )()+++def _strike(backend):+ dead = _stub_child()+ backend._process = dead+ backend._retired_process = dead+ backend._after_metal_poisoned_server(False)+++def _loaded(tmp_path, **kw):+ backend, gguf = h.cuda_backend(tmp_path)+ backend._speculative_type = "none"+ h.placement._launch(backend, gguf, n_ctx = 4096, **kw)+ return backend, gguf++[email protected]("retired", [False, True])+def test_a_retired_child_is_not_reused_by_an_identical_load(tmp_path, retired):+ """After a first retirement the killed child is still recorded and _healthy is still+ true: an identical explicit load must spawn a replacement."""+ gguf = tmp_path / "m.gguf"+ gguf.write_bytes(b"gguf")+ backend = LlamaCppBackend()+ proc = _stub_child()+ proc.poll = lambda: None+ backend._process = proc+ backend._healthy = True+ backend._model_identifier = "m"+ backend._gguf_path = str(gguf)+ if retired:+ backend._retire_device_lost_server(proc)+ intent = GgufLoadIntent(gguf_path = str(gguf), model_identifier = "m")+ assert backend.matches_load_source(intent) is (not retired)+++_HINT = ("someone/repo", "Q4_K_M", "/nonexistent/model.gguf", ())+++def test_the_strike_key_ignores_the_verified_cache_hint(tmp_path):+ backend, gguf = h.cuda_backend(tmp_path)+ plain = GgufLoadIntent(gguf_path = str(gguf), model_identifier = "test", n_ctx = 4096)+ hinted = GgufLoadIntent(+ gguf_path = str(gguf), model_identifier = "test", n_ctx = 4096, verified_gguf = _HINT+ )+ assert backend._metal_oom_key(hinted) == backend._metal_oom_key(plain)+ backend._metal_oom_terminal_key = backend._metal_oom_key(plain)+ assert backend._metal_oom_is_terminal(hinted)+++ "hint",+ [None, _HINT],+ ids = [+ "test_explicit_reload_clears_a_terminal_metal_oom",+ "test_a_hinted_explicit_reload_clears_a_terminal_metal_oom",+ ],+)+def test_an_explicit_reload_clears_a_terminal_metal_oom(tmp_path, hint):+ backend, gguf = h.cuda_backend(tmp_path)+ # The terminal key comes from a committed snapshot, which never carries the hint.+ snapshot = GgufLoadIntent(gguf_path = str(gguf), model_identifier = "test", n_ctx = 4096)+ key = backend._metal_oom_key(snapshot)+ backend._metal_oom_strikes = (key, 2)+ backend._metal_oom_terminal_key = key+ h.spawns(backend, gguf, healthy = lambda i, cmd: True, n_ctx = 4096, verified_gguf = hint)+ assert backend._last_load_intent is not None+ assert backend._last_load_intent.verified_gguf is None+ assert backend._metal_oom_terminal_key is None+ assert backend._metal_oom_strikes is None+ assert not backend._metal_oom_is_terminal(backend._last_load_intent)+++def test_an_explicit_reload_with_changed_settings_clears_the_terminal_latch(tmp_path):+ """The latch must not survive a reload that changes the GPU pick and re-arm once a+ matching request writes gpu_ids back."""+ backend, gguf = _loaded(tmp_path)+ committed = backend._last_load_intent+ assert committed is not None and committed.gpu_ids is None+ _strike(backend)+ _strike(backend)+ assert backend._metal_oom_is_terminal(committed)+ h.placement._launch(backend, gguf, n_ctx = 4096, gpu_ids = [0])+ assert backend.is_active and backend.is_loaded+ reloaded = backend._last_load_intent+ assert list(reloaded.gpu_ids) == [0]+ # Not vacuous: writing gpu_ids back reproduces the latched load exactly.+ assert backend._metal_oom_key(replace(reloaded, gpu_ids = None)) == backend._metal_oom_key(+ committed+ )+ assert backend._metal_oom_terminal_key is None+ assert backend._metal_oom_strikes is None+ backend._record_matching_gpu_request(None)+ assert backend._metal_oom_is_terminal(backend._last_load_intent) is False+++def test_a_callers_load_starts_a_fresh_metal_oom_episode(tmp_path):+ """A (one strike), then B, then A again: A's new session gets its own two strikes."""+ backend, gguf = _loaded(tmp_path)+ _strike(backend)+ assert backend._metal_oom_strikes is not None+ h.placement._launch(backend, gguf, n_ctx = 8192)+ assert backend._metal_oom_strikes is None+ h.placement._launch(backend, gguf, n_ctx = 4096)+ _strike(backend)+ assert backend._metal_oom_strikes[1] == 1+ assert backend._metal_oom_terminal_key is None+++def test_automatic_recovery_keeps_the_strike(tmp_path):+ """The respawn and the drafter-free reload replay under the recovery marker, which carries+ the count, so a second OOM still stops respawning."""+ backend, gguf = _loaded(tmp_path)+ _strike(backend)+ with llama_cpp._automatic_recovery_load():+ h.placement._launch(backend, gguf, n_ctx = 4096)+ assert backend._metal_oom_strikes[1] == 1+ _strike(backend)+ assert backend._metal_oom_terminal_key is not None+++def test_the_respawn_and_drafter_free_reload_run_as_automatic_recovery():+ for fn in (LlamaCppBackend._respawn_if_dead, LlamaCppBackend._maybe_recover_from_mtp_crash):+ assert "with _automatic_recovery_load():" in inspect.getsource(fn)+++def test_a_diffusion_load_drops_the_chat_servers_recovery_state(tmp_path, monkeypatch):+ """A crashed diffusion runner must neither replay the chat load it replaced nor meet that+ load's Metal OOM latch."""+ import test_gpu_memory_mode as _gm++ real_init = LlamaCppBackend.__init__++ def init(self, *a, **k):+ real_init(self, *a, **k)+ self._last_load_intent = object()+ self._mtp_runtime_fallback_active = True+ self._metal_oom_strikes = ("chat", 2)+ self._metal_oom_terminal_key = "chat"++ monkeypatch.setattr(LlamaCppBackend, "__init__", init)+ _cmd, _env, backend = _gm._launch_diffusion(+ monkeypatch, tmp_path, gguf_gib = 4.0, gpus = _gm._TWO_3090, gpu_ids = None+ )+ assert backend._last_load_intent is None+ assert backend._mtp_runtime_fallback_active is False+ assert backend._metal_oom_strikes is None+ assert backend._metal_oom_terminal_key is None+++# A server Studio retired on purpose is not an MTP crash: _respawn_if_dead replays it.+++def _mtp_server(harness):+ backend, first, loads = harness(_MTP, "draft-mtp")+ backend._mtp_runtime_fallback_active = True+ return backend, first, loads+++def _retire(backend, first, metal):+ if metal:+ _poison(backend, first)+ else:+ backend._retire_device_lost_server(first)++[email protected]("metal", [True, False], ids = ["metal-strike-1", "vulkan-device-lost"])+def test_a_retired_mtp_server_is_respawned_not_reloaded_without_mtp(harness, metal):+ backend, first, loads = _mtp_server(harness)+ _retire(backend, first, metal)+ assert first.poll() is not None+ assert backend._maybe_recover_from_mtp_crash(httpx.ConnectError("closed")) is False+ assert backend._respawn_if_dead() is True+ assert [load.speculative_type for load in loads] == ["mtp"]+++def test_a_real_mtp_crash_still_reloads_without_mtp(harness):+ backend, first, loads = _mtp_server(harness)+ first.kill()+ first.wait()+ assert backend._maybe_recover_from_mtp_crash(httpx.ConnectError("closed")) is True+ assert _wait(lambda: bool(loads))+ assert loads[-1].speculative_type == "off"+++def test_a_reload_queued_before_the_retirement_stands_down(harness):+ """The request saw the port close while the child was still exiting; the retirement+ lands during the queued reload's exit wait."""+ backend, first, loads = _mtp_server(harness)+ assert backend._maybe_recover_from_mtp_crash(httpx.ConnectError("closed")) is True+ backend._retire_device_lost_server(first)+ assert _wait(lambda: not backend._mtp_runtime_fallback_in_progress)+ assert loads == []+++def test_the_retirement_is_published_before_the_kill(harness):+ backend, first, _loads = _mtp_server(harness)+ seen = []+ kill = first.kill++ def recording_kill():+ seen.append(getattr(backend, "_retired_process", None) is first)+ kill()++ first.kill = recording_kill+ backend._retire_device_lost_server(first)+ assert seen == [True]+++# A retired child takes its engine-stats poller with it.+++class _Poller:+ def __init__(self):+ self.stopped = False++ def stop(self):+ self.stopped = True++[email protected]("metal", [True, False], ids = ["metal", "device-lost"])+def test_a_retired_child_stops_its_stats_poller(harness, metal):+ backend, first, _loads = harness(_OFF, "none")+ poller = backend._stats_logger = _Poller()+ _retire(backend, first, metal)+ assert first.poll() is not None+ assert poller.stopped and backend._stats_logger is None+++def test_a_terminal_metal_retirement_leaves_no_poller(harness):+ backend, first, _loads = harness(_OFF, "none")+ _poison(backend, first)+ assert backend._respawn_if_dead() is True+ poller = backend._stats_logger = _Poller()+ _poison(backend, backend._process)+ assert backend._process is None and backend._model_identifier is None+ assert poller.stopped and backend._stats_logger is None+++def test_the_replacement_load_starts_its_own_poller(tmp_path, monkeypatch):+ import core.inference.llama_stats as _stats++ started = []+ monkeypatch.setattr(+ _stats,+ "maybe_start_stats_logger",+ lambda base_url, logger, headers = None: started.append(base_url) or _Poller(),+ )+ backend, gguf = h.cuda_backend(tmp_path)+ caps = dict(h.CAPS, supports_metrics = True)+ backend.probe_server_capabilities = lambda _binary = None: dict(caps)+ h.placement._launch(backend, gguf, n_ctx = 4096)+ # What a retirement leaves: the poller stopped and cleared.+ backend._stats_logger.stop()+ backend._stats_logger = None+ h.placement._launch(backend, gguf, n_ctx = 4096, force_reload = True)+ assert len(started) == 2 and isinstance(backend._stats_logger, _Poller)+ assert not backend._stats_logger.stopped+++# A terminal Metal retirement is a full unload, and the routes release what it held.+++class _Fh:+ def __init__(self):+ self.closed = False++ def close(self):+ self.closed = True++ def write(self, *_a):+ return 0++ def flush(self):+ pass+++def _two_strikes(backend, first, monkeypatch, calls):+ _poison(backend, first)+ assert backend._respawn_if_dead() is True+ fh = backend._llama_log_fh = _Fh()+ monkeypatch.setattr(backend, "_clear_server_pid", lambda: calls.append("pid"))+ monkeypatch.setattr(backend, "_unload_audio_codec", lambda: calls.append("codec"))+ backend._planned_vram_mib = {0: 9000}+ _poison(backend, backend._process)+ return fh+++def test_the_terminal_strike_unloads_the_backend(harness, monkeypatch):+ backend, first, _loads = harness(_OFF, "none")+ calls = []+ fh = _two_strikes(backend, first, monkeypatch, calls)+ assert backend._process is None and backend._model_identifier is None+ assert backend.is_active is False and backend.is_loaded is False+ assert fh.closed and "pid" in calls and "codec" in calls+ assert not backend._planned_vram_mib+ assert backend._respawn_if_dead() is False+++def test_a_first_strike_still_respawns_with_the_same_settings(harness, monkeypatch):+ backend, first, loads = harness(_OFF, "none")+ seen = []+ monkeypatch.setattr(llama_cpp, "_TERMINAL_RETIREMENT_HOOKS", [seen.append], raising = False)+ _poison(backend, first)+ assert backend._model_identifier is not None+ assert backend._respawn_if_dead() is True+ assert loads == [_OFF] and seen == []+++def test_the_terminal_strike_runs_the_cleanup_hooks_once(harness, monkeypatch):+ backend, first, _loads = harness(_OFF, "none")+ seen = []+ monkeypatch.setattr(+ llama_cpp, "_TERMINAL_RETIREMENT_HOOKS", [lambda *args: seen.append(args)], raising = False+ )+ _two_strikes(backend, first, monkeypatch, [])+ assert seen == [(backend, backend._unload_epoch)]+++class _Orchestrator:+ def __init__(self):+ self.active_model_name = None+ self.loading_models = ()+ self.cleaned = False++ def _cleanup(self):+ self.cleaned = True+++def _kept_slot(refs = 0, orchestrator = None):+ return model_slots.ExtraSlot(+ llama = LlamaCppBackend(),+ orchestrator = orchestrator or _Orchestrator(),+ account = "a",+ refs = refs,+ )+++def _routes(monkeypatch):+ import routes.inference as inf++ assert inf._release_after_terminal_retirement in llama_cpp._TERMINAL_RETIREMENT_HOOKS+ calls = []+ monkeypatch.setattr(inf, "release_chat_after_kept_models", lambda: calls.append("kept") or True)+ monkeypatch.setattr(inf, "release_chat_gpu_claim", lambda: calls.append("claim") or True)+ monkeypatch.setattr(+ inf.account_access, "clear_resident", lambda modality: calls.append(f"clear:{modality}")+ )+ return inf, calls+++def _release_primary(inf):+ inf._release_after_terminal_retirement(+ inf._llama_cpp_backend, inf._llama_cpp_backend._unload_epoch+ )+++def test_a_retired_kept_slot_is_dropped_and_keeps_the_primarys_residency(monkeypatch):+ """Dead, it holds nothing a request could still use, so a held one goes too; the chat+ residency is the primary's, which may be a live zero-VRAM model."""+ inf, calls = _routes(monkeypatch)+ retired, other = _kept_slot(refs = 1), _kept_slot()+ monkeypatch.setattr(model_slots, "slots", [retired, other])+ monkeypatch.setattr(model_slots, "stuck", [])+ inf._release_after_terminal_retirement(retired.llama, retired.llama._unload_epoch)+ assert model_slots.slots == [other]+ assert calls == ["kept"]+ # Nothing dead is left for a later teardown to count as an unload.+ monkeypatch.setattr(model_slots, "slots", [])+ assert model_slots.unload_llama_slots() == 0+++def test_a_retired_primary_clears_its_own_residency(monkeypatch):+ inf, calls = _routes(monkeypatch)+ kept = _kept_slot()+ monkeypatch.setattr(model_slots, "slots", [kept])+ _release_primary(inf)+ assert model_slots.slots == [kept]+ assert calls == ["clear:chat", "claim"]+++def _cleanup_after(monkeypatch, inf, land):+ """The routes cleanup, run after ``land`` (a reload queued on _serial_load_lock)."""++ def hook(*args):+ land()+ inf._release_after_terminal_retirement(*args)++ monkeypatch.setattr(llama_cpp, "_TERMINAL_RETIREMENT_HOOKS", [hook])+++def test_a_reload_before_the_cleanup_keeps_its_kept_slot(harness, monkeypatch):+ inf, calls = _routes(monkeypatch)+ backend, first, _loads = harness(_OFF, "none")+ slot = model_slots.ExtraSlot(llama = backend, orchestrator = _Orchestrator(), account = "a")+ monkeypatch.setattr(model_slots, "slots", [slot])+ monkeypatch.setattr(model_slots, "stuck", [])+ _cleanup_after(monkeypatch, inf, lambda: backend.load_model(_OFF))+ _two_strikes(backend, first, monkeypatch, [])+ assert model_slots.slots == [slot]+ assert backend._process is not None and backend._process.poll() is None+ assert calls == []+++def test_a_reload_before_the_cleanup_keeps_the_primarys_residency(harness, monkeypatch):+ inf, calls = _routes(monkeypatch)+ backend, first, _loads = harness(_OFF, "none")+ monkeypatch.setattr(inf, "_llama_cpp_backend", backend)+ monkeypatch.setattr(model_slots, "slots", [])+ _cleanup_after(monkeypatch, inf, lambda: backend.load_model(_OFF))+ _two_strikes(backend, first, monkeypatch, [])+ assert backend._process is not None and backend._process.poll() is None+ assert calls == []+++# A non-GGUF load that hands the GGUF primary off before the terminal cleanup.++_MLX = "mlx-community/Qwen3-4B-4bit"+++def _standard_load_lands_first(monkeypatch, inf, backend, orchestrator):+ def land():+ asyncio.run(inf._unload_llama_before_standard_load(backend))+ orchestrator.active_model_name = _MLX++ _cleanup_after(monkeypatch, inf, land)+++def _no_settle(monkeypatch, backend, waits):+ monkeypatch.setattr(backend, "_wait_for_vram_settle", lambda **kw: waits.append(kw))+++def test_a_standard_load_before_the_cleanup_keeps_the_primarys_residency(harness, monkeypatch):+ inf, calls = _routes(monkeypatch)+ backend, first, _loads = harness(_OFF, "none")+ monkeypatch.setattr(inf, "_llama_cpp_backend", backend)+ monkeypatch.setattr(model_slots, "slots", [])+ std = _Orchestrator()+ monkeypatch.setattr(inf, "_peek_inference_backend", lambda: std)+ waits = []+ _no_settle(monkeypatch, backend, waits)+ _standard_load_lands_first(monkeypatch, inf, backend, std)+ _two_strikes(backend, first, monkeypatch, [])+ assert calls == []+ # Nothing was running, so nothing to wait for.+ assert waits == []+++def test_a_standard_load_before_the_cleanup_keeps_its_kept_slot(harness, monkeypatch):+ inf, calls = _routes(monkeypatch)+ backend, first, _loads = harness(_OFF, "none")+ orchestrator = _Orchestrator()+ slot = model_slots.ExtraSlot(llama = backend, orchestrator = orchestrator, account = "a")+ monkeypatch.setattr(model_slots, "slots", [slot])+ monkeypatch.setattr(model_slots, "stuck", [])+ _no_settle(monkeypatch, backend, [])+ _standard_load_lands_first(monkeypatch, inf, backend, orchestrator)+ _two_strikes(backend, first, monkeypatch, [])+ assert model_slots.slots == [slot]+ assert orchestrator.cleaned is False+ assert calls == []+++def test_the_standard_handoff_advances_the_epoch_of_an_unloaded_backend(monkeypatch):+ import routes.inference as inf++ backend = LlamaCppBackend()+ waits = []+ _no_settle(monkeypatch, backend, waits)+ epoch = backend._unload_epoch+ asyncio.run(inf._unload_llama_before_standard_load(backend))+ assert backend._unload_epoch == epoch + 1+ assert waits == []+++def test_the_standard_handoff_skips_the_wait_with_nothing_running(monkeypatch):+ import routes.inference as inf++ backend = LlamaCppBackend()+ waits = []+ _no_settle(monkeypatch, backend, waits)+ asyncio.run(inf._unload_llama_before_standard_load(backend))+ assert waits == []+++def test_the_standard_handoff_waits_for_a_cleanup_past_its_check(monkeypatch):+ """A cleanup holding the lock already decided the backend is still retired: the standard+ load may not publish its residency until that cleanup is done."""+ import routes.inference as inf++ backend = LlamaCppBackend()+ _no_settle(monkeypatch, backend, [])+ epoch = backend._unload_epoch+ done = threading.Event()+ with backend._while_still_retired(epoch) as retired:+ assert retired+ worker = threading.Thread(+ target = lambda: (+ asyncio.run(inf._unload_llama_before_standard_load(backend)),+ done.set(),+ )+ )+ worker.start()+ assert not done.wait(0.3)+ assert backend._unload_epoch == epoch+ worker.join(10)+ assert done.is_set() and backend._unload_epoch == epoch + 1+++ "healthy",+ [True, False],+ ids = [+ "test_the_standard_handoff_still_waits_for_a_loaded_server_to_settle",+ # Starting or recovering: a process, not loaded.+ "test_the_standard_handoff_waits_after_killing_an_unhealthy_server",+ ],+)+def test_the_standard_handoff_waits_after_killing_a_server(harness, monkeypatch, healthy):+ import routes.inference as inf++ backend, _first, _loads = harness(_OFF, "none")+ backend._healthy = healthy+ assert backend._process is not None and backend.is_loaded is healthy+ waits = []+ _no_settle(monkeypatch, backend, waits)+ asyncio.run(inf._unload_llama_before_standard_load(backend))+ assert backend._process is None+ assert len(waits) == 1+ if not healthy:+ assert waits[0]["since_kill"] > 0+++def test_the_cleanup_leaves_an_occupied_primary_seat_alone(monkeypatch):+ """No handoff ran (the epoch is still the retirement's): the non-GGUF primary still keeps+ its sharers, and the CHAT release re-checks that the seat is idle."""+ inf, calls = _routes(monkeypatch)+ monkeypatch.setattr(model_slots, "slots", [])+ for field, value in (+ ("active_model_name", _MLX),+ ("loading_models", (_MLX,)),+ ("_managed_engine", object()),+ ):+ std = _Orchestrator()+ setattr(std, field, value)+ monkeypatch.setattr(inf, "_peek_inference_backend", lambda std = std: std)+ calls.clear()+ _release_primary(inf)+ assert calls == ["claim"], field+++def test_the_cleanup_leaves_a_kept_slot_with_a_non_gguf_model_alone(monkeypatch):+ inf, calls = _routes(monkeypatch)+ monkeypatch.setattr(model_slots, "stuck", [])+ for field, value in (("active_model_name", _MLX), ("loading_models", (_MLX,))):+ orchestrator = _Orchestrator()+ setattr(orchestrator, field, value)+ slot = _kept_slot(orchestrator = orchestrator)+ monkeypatch.setattr(model_slots, "slots", [slot])+ calls.clear()+ inf._release_after_terminal_retirement(slot.llama, slot.llama._unload_epoch)+ assert model_slots.slots == [slot] and orchestrator.cleaned is False, field+ assert calls == ["kept"]+++def test_an_idle_primary_seat_still_clears_its_residency(monkeypatch):+ inf, calls = _routes(monkeypatch)+ monkeypatch.setattr(model_slots, "slots", [])+ monkeypatch.setattr(inf, "_peek_inference_backend", lambda: _Orchestrator())+ _release_primary(inf)+ assert calls == ["clear:chat", "claim"]diff --git a/studio/backend/tests/test_metal_fit_params_probe_reasons.py b/studio/backend/tests/test_metal_fit_params_probe_reasons.pynew file mode 100644index 00000000000..964b6872b75--- /dev/null+++ b/studio/backend/tests/test_metal_fit_params_probe_reasons.py@@ -0,0 +1,141 @@+# SPDX-License-Identifier: AGPL-3.0-only+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0++"""llama-fit-params probe: say why a measurement failed, and only cache answers that repeat."""++import sys++import pytest++from core.inference import llama_cpp as lc+from core.inference.llama_cpp import LlamaCppBackend++_TABLE = """0.00.408.825 I common_memory_breakdown_print: | memory breakdown [MiB] | total free self model context compute unaccounted |+0.00.429.748 I common_memory_breakdown_print: | - MTL0 (Apple M1) | 5461 = 5460 + (3642 = 3025 + 28 + 589) + -3642 |+0.00.429.748 I common_memory_breakdown_print: | - Host | 2352 = 2288 + 0 + 64 |+"""++posix_only = pytest.mark.skipif(+ sys.platform == "win32", reason = "shell script stands in for the binary"+)+++def _install(+ tmp_path,+ script,+ mode = 0o755,+):+ bin_dir = tmp_path / "bin"+ bin_dir.mkdir()+ (bin_dir / "llama-server").write_text("")+ model = tmp_path / "model.gguf"+ model.write_bytes(b"GGUF")+ probe = bin_dir / "llama-fit-params"+ if script is not None:+ probe.write_text(script)+ probe.chmod(mode)+ return str(bin_dir / "llama-server"), str(model), probe+++def _counting(tmp_path, body):+ calls = tmp_path / "calls"+ return calls, f'#!/bin/sh\necho x >> "{calls}"\n{body}\n'+++def _ncalls(calls):+ return len(calls.read_text(encoding = "utf-8").splitlines()) if calls.exists() else 0+++def test_missing_probe_is_named(tmp_path):+ binary, model, _ = _install(tmp_path, None)+ backend = LlamaCppBackend()+ assert backend._metal_measured_model_mib(binary, model) is None+ assert "not found" in backend._metal_probe_failure+++@posix_only+def test_non_executable_probe_is_named_and_reprobed_after_repair(tmp_path):+ table = _TABLE.replace("\n", "\\n")+ calls, script = _counting(tmp_path, f'printf "{table}"')+ binary, model, probe = _install(tmp_path, script, mode = 0o644)+ backend = LlamaCppBackend()+ assert backend._metal_measured_model_mib(binary, model) is None+ assert "not executable" in backend._metal_probe_failure+ assert _ncalls(calls) == 0+ probe.chmod(0o755) # the installer's exec-bit repair+ assert backend._metal_measured_model_mib(binary, model) == (3025, 2288, 2)+ assert backend._metal_probe_failure is None+++@posix_only+def test_nonzero_exit_is_named_with_its_output_and_not_cached(tmp_path):+ calls, script = _counting(tmp_path, 'echo "ggml_metal_init: error: boom" >&2\nexit 3')+ binary, model, _ = _install(tmp_path, script)+ backend = LlamaCppBackend()+ assert backend._metal_measured_model_mib(binary, model) is None+ assert "exited with code 3" in backend._metal_probe_failure+ assert "boom" in backend._metal_probe_failure+ assert backend._metal_measured_model_mib(binary, model) is None+ assert _ncalls(calls) == 2+++@posix_only+def test_timeout_is_named_and_not_cached(tmp_path, monkeypatch):+ monkeypatch.setattr(lc, "_METAL_FIT_PROBE_TIMEOUT_S", 0.5)+ calls, script = _counting(tmp_path, "sleep 5")+ binary, model, _ = _install(tmp_path, script)+ backend = LlamaCppBackend()+ assert backend._metal_measured_model_mib(binary, model) is None+ assert "timed out" in backend._metal_probe_failure+ assert backend._metal_measured_model_mib(binary, model) is None+ assert _ncalls(calls) == 2+++@posix_only+def test_unparsable_output_is_named_and_cached(tmp_path):+ calls, script = _counting(tmp_path, 'echo "error: invalid argument: -lv"')+ binary, model, _ = _install(tmp_path, script)+ backend = LlamaCppBackend()+ assert backend._metal_measured_model_mib(binary, model) is None+ assert "no Metal memory breakdown" in backend._metal_probe_failure+ assert "invalid argument" in backend._metal_probe_failure+ assert backend._metal_measured_model_mib(binary, model) is None+ assert "no Metal memory breakdown" in backend._metal_probe_failure+ assert _ncalls(calls) == 1+++@posix_only+def test_a_valid_table_still_parses_and_caches(tmp_path):+ table = _TABLE.replace("\n", "\\n")+ calls, script = _counting(tmp_path, f'printf "{table}"')+ binary, model, _ = _install(tmp_path, script)+ backend = LlamaCppBackend()+ assert backend._metal_measured_model_mib(binary, model) == (3025, 2288, 2)+ assert backend._metal_measured_model_mib(binary, model) == (3025, 2288, 2)+ assert backend._metal_probe_failure is None+ assert _ncalls(calls) == 1+++def test_the_launch_log_carries_the_reason():+ """The "stay charged" line names the cause instead of a fixed guess."""+ import inspect++ src = inspect.getsource(LlamaCppBackend._metal_demand_paged_embedding_bytes)+ assert "_metal_probe_failure" in src+ assert "missing or unreadable" not in src+++@posix_only+def test_a_failed_probe_runs_once_per_load(tmp_path):+ calls, script = _counting(tmp_path, "exit 3")+ binary, model, _ = _install(tmp_path, script)+ backend = LlamaCppBackend()+ with lc._load_probe_memo_scope():+ assert backend._metal_measured_model_mib(binary, model) is None+ assert backend._metal_measured_model_mib(binary, model) is None+ assert "exited with code 3" in backend._metal_probe_failure+ assert _ncalls(calls) == 1+ # The next load probes again: a cold shader compile may have finished by then.+ with lc._load_probe_memo_scope():+ assert backend._metal_measured_model_mib(binary, model) is None+ assert _ncalls(calls) == 2diff --git a/studio/backend/tests/test_spec_sidecar_fit_reserve.py b/studio/backend/tests/test_spec_sidecar_fit_reserve.pynew file mode 100644index 00000000000..b9c114f0e74--- /dev/null+++ b/studio/backend/tests/test_spec_sidecar_fit_reserve.py@@ -0,0 +1,1064 @@+# SPDX-License-Identifier: AGPL-3.0-only+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0++"""Single-card --fit-target reserve for DSpark/DFlash sidecars llama.cpp's fitter cannot size."""++from __future__ import annotations++import inspect+import math+import os+import pathlib+from pathlib import Path++import pytest++from . import _sidecar_fit_helpers as h+from ._sidecar_fit_helpers import FLOOR, GIB, MIB, lc++B = lc.LlamaCppBackend++[email protected](autouse = True)+def _studio_home(tmp_path, monkeypatch):+ monkeypatch.setenv("UNSLOTH_STUDIO_HOME", str(tmp_path / "studio_home"))+ monkeypatch.delenv("LLAMA_ARG_DEVICE", raising = False)+ monkeypatch.delenv("LLAMA_ARG_SPLIT_MODE", raising = False)+ monkeypatch.setattr(lc, "_metal_capable_host", lambda: False)+ monkeypatch.setattr(B, "_apple_metal_memory_budget_bytes", staticmethod(lambda: 0))+++def _launch_kind(+ tmp_path,+ kind = "dspark",+ use_fit = True,+ extra_args = None,+):+ backend, gguf = h.cuda_backend(tmp_path, draft_key = kind)+ backend._select_gpus = lambda *a, **k: (None, True) if use_fit else ([0], False)+ path = h.sidecar(tmp_path, kind)+ cmd = h.placement._launch(+ backend,+ gguf,+ speculative_type = kind,+ extra_args = extra_args,+ **{f"{kind}_draft_path": path},+ )["cmd"]+ return cmd, path+++class TestIntegrityFlags:+ def test_reserve_rides_on_top_of_the_auto_fit_margin(self):+ base = B._ctx_integrity_flags(1, True, True, 0, 0, h.CAPS)+ flags = B._ctx_integrity_flags(1, True, True, 0, 0, h.CAPS, fit_target_reserve_mib = 7630)+ assert int(h.last(flags, "--fit-target")) == int(h.last(base, "--fit-target")) + 7630++ def test_reserve_rides_on_top_of_the_metal_floor(self):+ flags = B._ctx_integrity_flags(+ 1, True, False, 0, 0, h.CAPS, fit_target_floor_mib = 6464, fit_target_reserve_mib = 7630+ )+ assert int(h.last(flags, "--fit-target")) == 6464 + 7630++ def test_reserve_makes_the_legacy_path_speak(self):+ # The legacy path is silent at llama.cpp's own default; a reserve is not that.+ assert "--fit-target" not in B._ctx_integrity_flags(1, True, False, 0, 0, h.CAPS)+ flags = B._ctx_integrity_flags(1, True, False, 0, 0, h.CAPS, fit_target_reserve_mib = 100)+ assert int(h.last(flags, "--fit-target")) == 1024 + 100++ @pytest.mark.parametrize("use_fit, caps", [(False, h.CAPS), (True, {})])+ def test_no_fit_or_no_flag_emits_nothing(self, use_fit, caps):+ assert "--fit-target" not in B._ctx_integrity_flags(+ 1, use_fit, False, 0, 0, caps, fit_target_reserve_mib = 100+ )+++class TestReserveArithmetic:+ def _backend(self, file_bytes):+ backend = B()+ backend._get_gguf_size_bytes = lambda _path: file_bytes+ return backend++ def test_planner_reserve_wins_when_larger(self):+ assert self._backend(8 * GIB)._sidecar_fit_reserve_mib(9 * GIB, "/d.gguf") == 9 * 1024++ def test_file_size_floors_a_flat_fallback(self):+ assert self._backend(8 * GIB)._sidecar_fit_reserve_mib(0, "/d.gguf") == 8 * 1024++ def test_rounds_up(self):+ assert self._backend(0)._sidecar_fit_reserve_mib(MIB + 1, "/d.gguf") == 2++ def test_nothing_known_is_zero(self):+ assert self._backend(0)._sidecar_fit_reserve_mib(0, None) == 0+++def test_an_unsized_sidecar_carries_the_flat_reserve():+ b = B.__new__(B)+ b._get_gguf_size_bytes = lambda path: None+ assert b._sidecar_fit_reserve_mib(0, "o/remote-dspark", flat_mib = 1200) == 1200+ assert b._sidecar_fit_reserve_mib(2 * GIB, "o/remote-dspark", flat_mib = 1200) == 2048+++class TestDiscreteLaunch:+ @pytest.mark.parametrize("kind", ["dspark", "dflash"])+ def test_fit_on_keeps_the_sidecar_free(self, tmp_path, kind):+ cmd, path = _launch_kind(tmp_path, kind = kind)+ assert h.last(cmd, "--model-draft") == path+ assert h.margin(cmd) >= FLOOR++ def test_fit_off_has_no_fitter_to_tell(self, tmp_path):+ cmd, path = _launch_kind(tmp_path, use_fit = False)+ assert h.last(cmd, "--model-draft") == path+ assert "--fit-target" not in cmd++ def test_a_cpu_pinned_drafter_reserves_nothing(self, tmp_path):+ cmd, path = _launch_kind(tmp_path, extra_args = ["--spec-draft-ngl", "0"])+ assert h.last(cmd, "--model-draft") == path+ assert h.margin(cmd) < FLOOR+++def test_the_integrity_margin_never_carries_the_reserve():+ """The reserve rides only on the trailing pair the spawn appends, so it can neither make+ an unpriced child look budget-priced nor be mistaken for a user's equal margin."""+ src = Path(lc.__file__).read_text(encoding = "utf-8")+ assert "fit_target_reserve_mib = _sidecar_fit_reserve_mib" not in src+ assert "fit_target_reserve_mib = 0," in src++[email protected]("spec_type", ["draft-dspark", "draft-dflash"])+def test_a_pass_through_sidecar_is_reserved_too(tmp_path, spec_type):+ """Extras owning --spec-type replace Studio's emission, so the reserve follows the+ sidecar the child really loads, even with Studio's own speculation off."""+ backend, gguf = h.cuda_backend(tmp_path, draft_key = "sidecar")+ path = tmp_path / "custom-sidecar-Q8_0.gguf"+ path.write_bytes(b"draft")+ cmd = h.placement._launch(+ backend,+ gguf,+ speculative_type = "off",+ extra_args = ["--spec-type", spec_type, "--model-draft", str(path)],+ )["cmd"]+ assert str(path) in cmd+ assert h.margin(cmd) >= FLOOR+++# _sidecar_reserve_margin_flags: the spawn's one question of the argv + env it launches.+++ "args, env",+ [+ (["--model-draft", "/d.gguf"], {}),+ (["-md", "/d.gguf"], {}),+ (["--spec-draft-model", "/d.gguf"], {}),+ (["--model-draft=/d.gguf"], {}),+ ([], {"LLAMA_ARG_SPEC_DRAFT_MODEL": "/d.gguf"}),+ ],+ ids = ["long", "short", "spec-draft-model", "equals", "env"],+)+def test_every_draft_spelling_gets_the_reserve(args, env):+ flags = lc._sidecar_reserve_margin_flags(["--fit", "on", *args], (), env, 1024, 8192)+ assert flags == ["--fit-target", str(1024 + 8192)]+++ "args, env",+ [(["--fit", "off"], {}), ([], {"LLAMA_ARG_FIT": "off"}), (["-fit", "off"], {})],+ ids = ["argv", "env", "short"],+)+def test_no_reserve_when_the_fitter_is_off(args, env):+ assert lc._sidecar_reserve_margin_flags(["-md", "/d.gguf", *args], (), env, 1024, 8192) == []+++def test_no_reserve_without_a_drafter():+ assert lc._sidecar_reserve_margin_flags(["--fit", "on"], (), {}, 1024, 8192) == []+++ "user_args, env, want",+ [+ ((), {"LLAMA_ARG_FIT_TARGET": "12000"}, 12000 + 8192),+ (("--fit-target", "16384"), {}, 16384 + 8192),+ # A single-device launch keeps device 0's entry, not the list maximum.+ (("-fitt", "512/20000"), {}, 1024 + 8192),+ ],+ ids = ["env", "argv", "per-device-list"],+)+def test_the_reserve_sits_on_top_of_the_users_margin(user_args, env, want):+ flags = lc._sidecar_reserve_margin_flags(+ ["-md", "/d.gguf", *user_args], user_args, env, 1024, 8192+ )+ assert flags == ["--fit-target", str(want)]++[email protected]("value", ["1024,65536", "1024/65536"], ids = ["comma", "slash"])+def test_single_device_reserve_uses_the_first_list_entry(value):+ flags = lc._sidecar_reserve_margin_flags(+ ["--fit", "on", "-md", "/d.gguf", "--fit-target", value],+ ("--fit-target", value),+ {},+ 1024,+ 8192,+ )+ assert flags == ["--fit-target", str(1024 + 8192)]+++ "extras",+ [["--fit-target", "1024"], ["-fitt", "1024"], ["--fit-target=1024"]],+ ids = ["spaced", "short", "equals"],+)+def test_a_user_fit_target_keeps_the_reserve(tmp_path, extras):+ backend, gguf = h.cuda_backend(tmp_path)+ cmd = h.dspark(backend, gguf, tmp_path, extra_args = extras)+ values = [+ cmd[i + 1] if tok in ("--fit-target", "-fitt") else tok.split("=", 1)[1]+ for i, tok in enumerate(cmd)+ if tok in ("--fit-target", "-fitt") or tok.startswith("--fit-target=")+ ]+ assert int(values[-1]) >= FLOOR+++ "extras",+ [["--fit", "on"], ["--fit=on"], ["-fit", "on"]],+ ids = ["spaced", "equals", "short"],+)+def test_extras_that_turn_the_fitter_on_get_the_reserve(tmp_path, extras):+ backend, gguf = h.cuda_backend(tmp_path)+ backend._select_gpus = lambda *a, **k: ([0], False)+ assert h.margin(h.dspark(backend, gguf, tmp_path, extra_args = extras)) >= FLOOR+++def test_an_inherited_env_margin_is_kept_under_the_reserve(tmp_path, monkeypatch):+ monkeypatch.setenv("LLAMA_ARG_FIT_TARGET", "12000")+ backend, gguf = h.cuda_backend(tmp_path)+ assert h.margin(h.dspark(backend, gguf, tmp_path)) >= 12000 + 8 * 1024+++def test_the_reserve_carries_the_draft_compute_floor(tmp_path):+ backend, gguf = h.cuda_backend(tmp_path)+ assert h.margin(h.dspark(backend, gguf, tmp_path)) >= FLOOR + 224+++def test_full_draft_offload_still_gets_the_reserve(tmp_path):+ backend, gguf = h.cuda_backend(tmp_path)+ assert h.margin(h.dspark(backend, gguf, tmp_path)) >= FLOOR+++def test_manual_auto_context_prices_the_native_context(tmp_path):+ backend, gguf = h.cuda_backend(tmp_path)+ backend._context_length = 131072+ h.per_token_estimate(backend, 64 * 1024) # 8 GiB of draft KV at 131072+ cmd = h.dspark(backend, gguf, tmp_path, n_ctx = 0, **h.MANUAL_AUTO)+ if "--fit-target" not in cmd:+ pytest.skip("this host plans no fitter for Manual + Auto")+ assert h.margin(cmd) >= 512 + 16 * 1024+++def test_explicit_manual_context_is_priced_at_that_context(tmp_path):+ backend, gguf = h.cuda_backend(tmp_path)+ backend._context_length = 131072+ h.per_token_estimate(backend, 64 * 1024)+ cmd = h.dspark(backend, gguf, tmp_path, n_ctx = 4096, **h.MANUAL_AUTO)+ if "--fit-target" not in cmd:+ pytest.skip("this host plans no fitter for Manual + Auto")+ assert h.last(cmd, "-c") == "4096"+ assert h.margin(cmd) < 512 + 12 * 1024+++def test_manual_auto_context_prices_an_inherited_context(tmp_path, monkeypatch):+ monkeypatch.setenv("LLAMA_ARG_CTX_SIZE", "8192")+ backend, gguf = h.cuda_backend(tmp_path)+ backend._context_length = 131072+ h.per_token_estimate(backend, 64 * 1024) # 0.5 GiB of draft KV at 8192+ cmd = h.dspark(backend, gguf, tmp_path, n_ctx = 0, **h.MANUAL_AUTO)+ assert "--fit-target" in cmd+ assert h.margin(cmd) < 512 + 12 * 1024+++def test_an_automatic_context_is_priced_at_the_final_slots(tmp_path):+ backend, gguf = h.cuda_backend(tmp_path)+ backend._context_length = 32768+ h.per_token_estimate(backend, 32 * 1024) # 1 GiB at 32768, 8 GiB at 8 x 32768+ cmd = h.dspark(+ backend,+ gguf,+ tmp_path,+ n_ctx = 0,+ n_parallel = 1,+ extra_args = ("--parallel", "8", "--no-kv-unified"),+ **h.MANUAL_AUTO,+ )+ assert "--fit-target" in cmd+ assert h.margin(cmd) >= 512 + 16 * 1024+++def test_a_trailing_zero_context_is_priced_at_the_native_length(tmp_path):+ backend, gguf = h.cuda_backend(tmp_path)+ backend._context_length = 131072+ h.per_token_estimate(backend, 64 * 1024)+ cmd = h.dspark(+ backend, gguf, tmp_path, gpu_memory_mode = "auto", n_ctx = 4096, extra_args = ["-c", "0"]+ )+ assert h.last(cmd, "-c") == "0", cmd+ assert h.margin(cmd) >= 16 * 1024, cmd+++def test_an_unread_native_length_prices_the_fitters_minimum_context():+ assert lc._sidecar_child_ctx([], {}, None, 1, True) == lc._LLAMA_FIT_MIN_CTX+ assert lc._sidecar_child_ctx([], {}, None, 4, False) == 4 * lc._LLAMA_FIT_MIN_CTX+ assert lc._sidecar_child_ctx(["-c", "8192"], {}, None, 4, False) == 8192+++ "args, env, want",+ [+ # No -c: the pool is n_parallel * per-slot limit (tools/server/server.cpp).+ (["--kv-unified-per-slot", "2048"], {}, 4 * 2048),+ (["--kv-unified-per-slot=65536"], {}, 4 * 65536),+ ([], {"LLAMA_ARG_KV_UNIFIED_PER_SLOT": "2048"}, 4 * 2048),+ (["-c", "16384", "--kv-unified-per-slot", "2048"], {}, 16384),+ # -c 0 asks for the full window; a later --fit-ctx clears that.+ (["-c", "0", "--kv-unified-per-slot", "2048"], {}, 131072),+ (["-c", "0", "--fit-ctx", "8192", "--kv-unified-per-slot", "2048"], {}, 4 * 2048),+ ([], {}, 131072),+ ],+)+def test_the_per_slot_limit_sizes_the_priced_context(args, env, want):+ assert lc._sidecar_child_ctx(args, env, 131072, 4, True) == want+++ "extras",+ [("--parallel", "8"), ("-np", "8"), ("--parallel=8",)],+ ids = ["long", "short", "equals"],+)+def test_pass_through_slots_are_priced_on_the_first_spawn(tmp_path, extras):+ backend, gguf = h.cuda_backend(tmp_path)+ h.slot_priced(backend)+ cmd = h.dspark(backend, gguf, tmp_path, n_ctx = 4096, n_parallel = 1, extra_args = extras)+ assert h.margin(cmd) >= FLOOR + 8 * 1024+++def test_fewer_final_slots_lower_the_reserve(tmp_path):+ backend, gguf = h.cuda_backend(tmp_path)+ h.slot_priced(backend)+ cmd = h.dspark(+ backend, gguf, tmp_path, n_ctx = 4096, n_parallel = 8, extra_args = ("--parallel", "1")+ )+ assert FLOOR + 1024 <= h.margin(cmd) < FLOOR + 2 * 1024+ assert cmd.count("--fit-target") == 1+++def _ubatch_priced(backend):+ backend._estimate_mtp_overhead_bytes = lambda n_ctx, n_ubatch = None, **kw: (+ 8 * GIB + int(n_ubatch or 512) * MIB+ )+++ "extras, env",+ [+ (("--batch-size", "4096", "--spec-draft-n-ubatch", "4096"), {}),+ ((), {"LLAMA_ARG_BATCH": "4096", "LLAMA_ARG_SPEC_DRAFT_N_UBATCH": "4096"}),+ ],+ ids = ["flag", "env"],+)+def test_the_draft_micro_batch_is_priced(tmp_path, monkeypatch, extras, env):+ for key, value in env.items():+ monkeypatch.setenv(key, value)+ backend, gguf = h.cuda_backend(tmp_path)+ _ubatch_priced(backend)+ cmd = h.dspark(backend, gguf, tmp_path, n_ctx = 4096, extra_args = extras)+ assert h.margin(cmd) >= FLOOR + 4096+++def test_the_draft_micro_batch_is_capped_at_the_batch(tmp_path):+ """llama_context caps n_ubatch at n_batch; the draft context inherits the target's 2048."""+ backend, gguf = h.cuda_backend(tmp_path)+ _ubatch_priced(backend)+ cmd = h.dspark(+ backend, gguf, tmp_path, n_ctx = 4096, extra_args = ["--spec-draft-n-ubatch", "8192"]+ )+ assert FLOOR + 2048 <= h.margin(cmd) < FLOOR + 4096+++def test_a_zero_draft_micro_batch_follows_the_target():+ assert lc._draft_n_ubatch(["--spec-draft-n-ubatch", "0"], {}) is None+ assert lc._draft_n_ubatch(["--spec-draft-n-ubatch=256"], {}) == 256+++def test_underscore_spellings_are_read_like_llama_cpp():+ assert lc._draft_card_share_unknown(["--spec_draft_n_cpu_moe", "8"], {})+ assert lc._draft_n_ubatch(["--spec_draft_n_ubatch", "2048"], {}) == 2048+++def _dsv41_dspark_pair():+ """DeepSeek-V4.1 target and its DSpark drafter, dims read off the published GGUFs."""+ target = B.__new__(B)+ target._embedding_length, target._vocab_size = 5120, 129280+ draft = B.__new__(B)+ for k, v in dict(+ _architecture = "dflash",+ _embedding_length = 5120,+ _n_layers = 3,+ _n_heads = 64,+ _n_kv_heads = 1,+ _kv_key_length = 512,+ _kv_value_length = 512,+ _feed_forward_length = 2304,+ _expert_feed_forward_length = 2304,+ _vocab_size = 129280,+ ).items():+ setattr(draft, k, v)+ target._draft_backend_for = lambda path: draft+ return target++[email protected]("ub, measured_mib", [(512, 725.5), (2048, 2900.5)])+def test_a_dspark_prompt_graph_is_priced_with_the_micro_batch(ub, measured_mib):+ """llama.cpp b11734 reserves 725.5 MiB at -ub 512 and 2900.5 MiB at 2048 for this+ drafter's compute; the old flat 349 MiB let the reserve fall 2.2 GiB short."""+ b = _dsv41_dspark_pair()+ got = (+ b._mtp_draft_compute_bytes(8192, drafter_path = "/d.gguf", n_ubatch = ub)+ + B._MTP_DRAFT_COMPUTE_BYTES+ ) / MIB+ assert measured_mib <= got <= measured_mib * 1.25+++_ROLLBACK = 4 * GIB+++def _rollback_backend(tmp_path):+ backend, gguf = h.cuda_backend(tmp_path)+ backend._rollback_state_bytes = lambda n_parallel = 1: _ROLLBACK+ backend._estimate_mtp_overhead_bytes = (+ lambda n_ctx, target_rollback = True, spec_draft_n_max = 0, **kw: (+ 8 * GIB + (_ROLLBACK * spec_draft_n_max if target_rollback else 0)+ )+ )+ return backend, gguf+++def test_initial_reserve_leaves_target_rollback_to_the_fitter(tmp_path):+ backend, gguf = _rollback_backend(tmp_path)+ assert FLOOR <= h.margin(h.dspark(backend, gguf, tmp_path)) < FLOOR + _ROLLBACK // MIB+++ "extras", [("-ngld", "1"), ("--spec-draft-ngl", "1")], ids = ["short", "long"]+)+def test_partial_draft_offload_gets_no_whole_file_reserve(tmp_path, extras):+ backend, gguf = h.cuda_backend(tmp_path)+ assert h.margin(h.dspark(backend, gguf, tmp_path, extra_args = extras)) < FLOOR+++ "extras",+ [+ ("-otd", ".*=CPU"),+ ("--override-tensor-draft", ".*=CPU"),+ ("--spec-draft-override-tensor", ".*=CPU"),+ ("--spec-draft-override-tensor=.*=CPU",),+ ],+ ids = ["otd", "override-tensor-draft", "spec-draft-override-tensor", "equals"],+)+def test_a_draft_tensor_override_gets_no_whole_file_reserve(tmp_path, extras):+ backend, gguf = h.cuda_backend(tmp_path)+ cmd = h.dspark(backend, gguf, tmp_path, extra_args = extras)+ assert h.margin(cmd) < FLOOR+ assert extras[-1] in cmd+++def test_a_target_tensor_override_keeps_the_reserve(tmp_path):+ backend, gguf = h.cuda_backend(tmp_path)+ cmd = h.dspark(backend, gguf, tmp_path, extra_args = ("-ot", "blk\\.0\\.ffn_up=CPU"))+ assert h.margin(cmd) >= FLOOR+++ "extras",+ [+ ("--spec-draft-cpu-moe",),+ ("-cmoed",),+ ("--spec-draft-n-cpu-moe", "99"),+ ("--n-cpu-moe-draft=4",),+ ],+ ids = ["cpu-moe", "short", "n-cpu-moe", "n-equals"],+)+def test_draft_cpu_moe_gets_no_whole_file_reserve(tmp_path, extras):+ backend, gguf = h.cuda_backend(tmp_path)+ assert h.margin(h.dspark(backend, gguf, tmp_path, extra_args = extras)) < FLOOR+++ "env",+ [{"LLAMA_ARG_SPEC_DRAFT_CPU_MOE": "1"}, {"LLAMA_ARG_SPEC_DRAFT_N_CPU_MOE": "8"}],+ ids = ["cpu-moe", "n-cpu-moe"],+)+def test_draft_cpu_moe_from_the_env_is_a_placement_too(env):+ assert lc._draft_card_share_unknown([], env)+ assert not lc._draft_card_share_unknown([], {"LLAMA_ARG_SPEC_DRAFT_N_CPU_MOE": "0"})+++def test_a_positive_draft_cpu_moe_count_is_not_undone_by_a_later_zero():+ """llama.cpp appends the draft CPU overrides per positive count; a later 0 adds none+ but removes none either."""+ assert lc._draft_card_share_unknown(["--spec-draft-n-cpu-moe", "8", "-ncmoed", "0"], {})+ assert not lc._draft_card_share_unknown(["-ncmoed", "0"], {})++[email protected]("extras", [("-ngld", "1"), ("-otd", "blk\\.0\\.=CPU")])+def test_a_partly_placed_drafter_keeps_its_compute_floor(tmp_path, extras):+ backend, gguf = h.cuda_backend(tmp_path)+ floor = math.ceil(B._MTP_DRAFT_COMPUTE_BYTES / MIB)+ assert (+ 1024 + floor <= h.margin(h.dspark(backend, gguf, tmp_path, extra_args = list(extras))) < FLOOR+ )+++ "extras, env",+ [(["-nkvo"], {}), (["--no-kv-offload"], {}), ([], {"LLAMA_ARG_NO_KV_OFFLOAD": "1"})],+)+def test_host_resident_draft_kv_is_not_reserved_on_the_card(tmp_path, monkeypatch, extras, env):+ """The draft context copies the target's params (common_base_params_to_speculative), so+ -nkvo puts the draft KV in host buffers too."""+ for d in ("on", "off"):+ (tmp_path / d).mkdir()++ def margin_of(d, extras):+ backend, gguf = h.cuda_backend(d)+ backend._mtp_draft_kv_bytes = lambda *a, **k: h.DRAFT_KV+ return h.margin(h.dspark(backend, gguf, d, extra_args = list(extras)))++ on = margin_of(tmp_path / "on", [])+ for key, value in env.items():+ monkeypatch.setenv(key, value)+ assert on - margin_of(tmp_path / "off", extras) >= 760++[email protected]("device", ["none", "cpu"])+def test_a_cpu_child_gets_no_reserve_and_no_report(tmp_path, monkeypatch, device):+ monkeypatch.setenv("CUDA_DEVICE_ORDER", "PCI_BUS_ID")+ messages = h.info_messages(monkeypatch)+ backend, gguf = h.cuda_backend(tmp_path)+ cmd = h.dspark(backend, gguf, tmp_path, extra_args = ["--device", device])+ assert h.margin(cmd) < 1024 + 4 * 1024+ assert not any("cannot size the sidecar" in m for m in messages)+++def test_a_draft_with_its_own_embeddings_and_head_gets_no_reserve(tmp_path, monkeypatch):+ """llama_context needs the target's context only when the draft lacks token_embd or output."""+ monkeypatch.setattr(lc, "_sidecar_borrows_target_tensors", lambda path: False)+ backend, gguf = h.cuda_backend(tmp_path)+ assert h.margin(h.dspark(backend, gguf, tmp_path)) < FLOOR+++def _draft_with(monkeypatch, tmp_path, tensors):+ import utils.models.gguf_metadata as gm+ import utils.models.model_config as mc++ path = tmp_path / "draft.gguf"+ path.write_bytes(b"GGUF")+ monkeypatch.setattr(mc, "colocated_split_shards", lambda p: ([p], True))+ monkeypatch.setattr(gm, "_gguf_shard_has_named_tensor", lambda shard, name: name in tensors)+ return str(path)+++ "present, borrows",+ [+ ({"token_embd.weight", "output.weight"}, False),+ ({"token_embd.weight"}, True),+ ({"output.weight"}, True),+ (set(), True),+ ],+)+def test_borrowed_tensor_detection(tmp_path, monkeypatch, present, borrows):+ assert (+ lc._sidecar_borrows_target_tensors(_draft_with(monkeypatch, tmp_path, present)) is borrows+ )+ assert lc._sidecar_borrows_target_tensors(None) is True+++ "tensors, borrows, borrows_output",+ [+ # Upstream DFlash borrows the target's head when a draft ships none, and no marker+ # tells a build that ties it to the draft's embeddings apart.+ ({"token_embd.weight"}, True, True),+ ({"output.weight"}, True, False),+ (set(), True, True),+ ({"token_embd.weight", "output.weight"}, False, False),+ ],+)+def test_a_draft_missing_a_tensor_borrows_it(+ monkeypatch, tmp_path, tensors, borrows, borrows_output+):+ path = _draft_with(monkeypatch, tmp_path, tensors)+ assert lc._sidecar_borrows_target_tensors(path) is borrows+ assert lc._sidecar_borrows_output(path) is borrows_output+++def test_an_unread_draft_borrows_the_output():+ assert lc._sidecar_borrows_output(None) is True+++def test_native_measurement_is_read_from_the_shared_common_library(tmp_path):+ binary = h.elf(tmp_path / "llama-server", b"libllama-common.so.0")+ real = h.elf(tmp_path / "libllama-common.so.0.0.1", b"..." + h.NATIVE_MARKER + b"...")+ (tmp_path / "libllama-common.so.0").symlink_to(real.name)+ (tmp_path / "libllama-common.so").symlink_to("libllama-common.so.0")+ assert lc._fit_measures_borrowed_drafts(str(binary))+++def test_native_measurement_is_read_from_a_static_server(tmp_path):+ assert lc._fit_measures_borrowed_drafts(str(h.elf(tmp_path / "llama-server", h.NATIVE_MARKER)))+++def test_native_measurement_is_read_through_the_installer_wrapper(tmp_path):+ h.elf(tmp_path / "llama-server-real", h.NATIVE_MARKER)+ wrapper = tmp_path / "llama-server"+ wrapper.write_text('#!/bin/sh\nexec "$(dirname "$0")/llama-server-real" "$@"\n')+ wrapper.chmod(0o755)+ assert lc._fit_measures_borrowed_drafts(str(wrapper))+++def test_an_older_build_does_not_claim_native_measurement(tmp_path):+ binary = h.elf(tmp_path / "llama-server")+ h.elf(+ tmp_path / "libllama-common.so",+ b"failed to measure the memory of the extra model, fitting without it",+ )+ assert not lc._fit_measures_borrowed_drafts(str(binary))+ assert not lc._fit_measures_borrowed_drafts(None)+++def test_only_the_linked_common_library_speaks_for_the_build(tmp_path):+ binary = h.elf(tmp_path / "llama-server", b"libllama-common.so.0")+ h.elf(tmp_path / "libllama-common.so.0")+ h.elf(tmp_path / "libllama-common.so.0.9.9", h.NATIVE_MARKER)+ assert not lc._fit_measures_borrowed_drafts(str(binary))+ h.elf(tmp_path / "libllama-common.so.0", h.NATIVE_MARKER)+ assert lc._fit_measures_borrowed_drafts(str(binary))+++def test_a_static_server_beside_a_stale_common_library_keeps_the_reserve(tmp_path):+ binary = h.elf(tmp_path / "llama-server")+ h.elf(tmp_path / "libllama-common.so.0", h.NATIVE_MARKER)+ assert not lc._fit_measures_borrowed_drafts(str(binary))+++def test_the_common_library_is_followed_through_the_server_impl_library(tmp_path):+ """b11734 layout: llama-server links libllama-server-impl.so, which links+ libllama-common.so.0, where fit.cpp lives."""+ binary = h.elf(tmp_path / "llama-server", b"libllama-server-impl.so")+ h.elf(tmp_path / "libllama-server-impl.so", b"libllama-common.so.0")+ h.elf(tmp_path / "libllama-common.so.0", h.NATIVE_MARKER)+ assert lc._fit_measures_borrowed_drafts(str(binary))+ h.elf(tmp_path / "libllama-server-impl.so")+ assert not lc._fit_measures_borrowed_drafts(str(binary))+++ "linked", ["libllama-common.so.0", "libllama-common.0.dylib", "llama-common.dll"]+)+def test_a_server_that_names_its_common_library_is_read_through_it(tmp_path, linked):+ binary = h.elf(tmp_path / "llama-server", linked.encode())+ h.elf(tmp_path / linked, h.NATIVE_MARKER)+ assert lc._fit_measures_borrowed_drafts(str(binary))+++def test_the_linked_library_names_are_read_once_per_binary_version(tmp_path, monkeypatch):+ server = tmp_path / "llama-server"+ server.write_bytes(b"\0libllama-common.so.0\0" * 4)+ lib = tmp_path / "libllama-common.so.0"+ lib.write_bytes(b"lib")+ reads = []+ real = pathlib.Path.read_bytes++ def read_bytes(self):+ reads.append(self.name)+ return real(self)++ monkeypatch.setattr(pathlib.Path, "read_bytes", read_bytes)+ for _ in range(3):+ assert lc._linked_llama_libs(server, tmp_path) == [lib]+ assert reads.count("llama-server") == 1+ # Existence is not cached.+ lib.unlink()+ assert lc._linked_llama_libs(server, tmp_path) == []+ assert reads.count("llama-server") == 1+ lib.write_bytes(b"lib")+ server.write_bytes(b"\0libllama-server-impl.so\0")+ st = server.stat()+ os.utime(server, ns = (st.st_atime_ns, st.st_mtime_ns + 1_000_000_000))+ (tmp_path / "libllama-server-impl.so").write_bytes(b"impl")+ assert lc._linked_llama_libs(server, tmp_path) == [tmp_path / "libllama-server-impl.so"]+ assert reads.count("llama-server") == 2+++def test_a_missing_binary_fails_closed(tmp_path):+ caps = B.probe_server_capabilities(str(tmp_path / "missing"))+ assert caps["fit_measures_borrowed_drafts"] is False+++def test_a_cached_probe_rereads_the_shared_library(tmp_path, monkeypatch):+ """The capability cache is keyed on the server alone; a swapped libllama-common+ must not keep an older answer."""+ binary = h.elf(tmp_path / "llama-server", b"libllama-common.so")+ st = binary.stat()+ key = (str(binary), st.st_mtime_ns, st.st_size)+ monkeypatch.setattr(B, "_exec_path_for_launch", staticmethod(lambda b: b))+ monkeypatch.setitem(+ B._capability_cache,+ key,+ {"found": True, "supports_fit_target": True, "fit_measures_borrowed_drafts": True},+ )+ assert B.probe_server_capabilities(str(binary))["fit_measures_borrowed_drafts"] is False+ h.elf(tmp_path / "libllama-common.so", h.NATIVE_MARKER)+ assert B.probe_server_capabilities(str(binary))["fit_measures_borrowed_drafts"] is True++[email protected]("native", [False, True], ids = ["older", "native"])+def test_the_reserve_stands_down_when_the_fitter_measures_the_draft(tmp_path, native):+ backend, gguf = h.cuda_backend(tmp_path)+ backend.probe_server_capabilities = lambda _binary = None: {+ **h.CAPS,+ "fit_measures_borrowed_drafts": native,+ }+ cmd = h.dspark(backend, gguf, tmp_path)+ assert "--model-draft" in cmd+ assert (h.margin(cmd) >= FLOOR) is not native+++def test_a_default_slider_single_card_reserve_is_logged(tmp_path, monkeypatch):+ backend, gguf = h.cuda_backend(tmp_path)+ messages = h.info_messages(monkeypatch)+ assert h.margin(h.dspark(backend, gguf, tmp_path)) >= FLOOR+ assert any("llama.cpp cannot size the sidecar, so" in m for m in messages)+++def _warned(+ tmp_path,+ monkeypatch,+ kind,+ fit_target = True,+):+ messages = h.info_messages(monkeypatch)+ backend, gguf = h.cuda_backend(tmp_path, draft_key = kind)+ caps = dict(h.CAPS, supports_fit_target = fit_target)+ backend.probe_server_capabilities = lambda _binary = None: dict(caps)+ path = h.sidecar(tmp_path, kind)+ cmd = h.placement._launch(backend, gguf, speculative_type = kind, **{f"{kind}_draft_path": path})[+ "cmd"+ ]+ assert f"draft-{kind}" in cmd+ return any("is not reserved" in m for m in messages)++[email protected]("kind", ["dspark", "dflash"])+def test_a_self_contained_drafter_is_not_reported_unreserved(tmp_path, monkeypatch, kind):+ monkeypatch.setattr(lc, "_sidecar_borrows_target_tensors", lambda path: False)+ assert _warned(tmp_path, monkeypatch, kind) is False++[email protected]("kind", ["dspark", "dflash"])+def test_a_borrowing_drafter_without_a_reserve_is_reported(tmp_path, monkeypatch, kind):+ assert _warned(tmp_path, monkeypatch, kind, fit_target = False) is True+++def test_a_one_card_tensor_split_is_reserved_and_reported(tmp_path, monkeypatch):+ """Tensor mode folds several cards into one meta device; one card has nothing to fold."""+ backend, gguf = h.cards(tmp_path, h.ONE)+ seen = h.spy_fit_sized(backend)+ messages = h.info_messages(monkeypatch)+ cmd = h.dspark(backend, gguf, tmp_path, extra_args = ["--split-mode", "tensor"])+ assert h.margin(cmd) >= FLOOR+ assert seen["fit_sized"] is True+ assert not any("is not reserved" in m for m in messages)+++# Retries: the reserve is reconciled at every spawn off the argv being launched.+++def _flash_crash(i, cmd):+ return -11 if h.last(cmd, "--flash-attn") == "on" else None+++def _flash_off(i, cmd):+ return h.last(cmd, "--flash-attn") == "off"+++def test_the_drafter_free_startup_retry_drops_the_reserve(tmp_path):+ backend, gguf = h.cuda_backend(tmp_path)+ cmds = h.spawns(+ backend,+ gguf,+ healthy = lambda i, cmd: "draft-dspark" not in cmd,+ n_ctx = 4096,+ speculative_type = "dspark",+ dspark_draft_path = h.sidecar(tmp_path),+ )+ assert cmds.loaded+ first, retry = cmds[0], cmds[-1]+ assert int(h.last(first, "--fit-target")) >= FLOOR+ assert "--model-draft" not in retry+ assert h.margin(retry) < FLOOR+++def test_the_drafter_free_fallback_keeps_the_reserve_for_a_user_drafter(tmp_path):+ """The fallback removes Studio's own block; a bare user --model-draft survives and+ still loads, so its reserve must too."""+ backend, gguf = h.cuda_backend(tmp_path)+ custom = tmp_path / "custom-dspark-Q8_0.gguf"+ custom.write_bytes(b"draft")+ cmds = h.spawns(+ backend,+ gguf,+ healthy = lambda i, cmd: "draft-dspark" not in cmd,+ n_ctx = 4096,+ speculative_type = "dspark",+ dspark_draft_path = h.sidecar(tmp_path),+ extra_args = ("--model-draft", str(custom)),+ )+ retry = cmds[-1]+ if "draft-dspark" in retry:+ pytest.skip("the user drafter kept the speculative block; no fallback ran")+ assert str(custom) in retry+ assert h.margin(retry) >= FLOOR+++def _spawn_source():+ src = inspect.getsource(B.load_model)+ spawn = src[src.index("def _spawn_and_wait(") :]+ return src, spawn[: spawn.index("_last_spawn_cmd = list(run_cmd)")]+++def test_the_fit_on_startup_retry_carries_the_reserve():+ """Every attempt of _spawn_and_wait (the --fit on retry re-enters its loop) reconciles+ the reserve before spawning. Checked at the source: the retry needs a real crash."""+ _src, spawn = _spawn_source()+ assert "_sidecar_fit_reserve_eligible_mib" in spawn+ assert "_sidecar_reserve_margin_flags(" in spawn+++def test_the_startup_retry_reads_the_child_not_a_literal_flag():+ src, spawn = _spawn_source()+ retry = src[src.index("retrying once with --fit on so it can offload") :]+ retry = retry[: retry.index("_did_fit_retry = True")]+ assert "_sidecar_reserve_margin_flags(" in spawn+ assert '"--model-draft" in _run' not in retry+ assert "fit_target_margin_in(extra_args, None)" not in retry+++def test_the_noflash_fit_flip_carries_the_reserve(tmp_path):+ backend, gguf = h.cuda_backend(tmp_path)+ backend._select_gpus = lambda *a, **k: ([0], False) # planned full offload: --fit off+ cmds = h.spawns(+ backend,+ gguf,+ crash = _flash_crash,+ healthy = _flash_off,+ late_exit = True,+ n_ctx = 4096,+ speculative_type = "dspark",+ dspark_draft_path = h.sidecar(tmp_path),+ )+ noflash = [c for c in cmds if h.last(c, "--flash-attn") == "off"]+ if not noflash or h.last(noflash[0], "--fit") != "on":+ pytest.skip("no --flash-attn off retry with the fitter on in this harness")+ assert "--model-draft" in noflash[0]+ assert h.margin(noflash[0]) >= FLOOR+++def test_the_noflash_retry_reprices_the_reserve(tmp_path):+ """--flash-attn off pads the draft V to f16, so the reserve grows."""+ backend, gguf = h.cuda_backend(tmp_path)+ backend._estimate_mtp_overhead_bytes = lambda n_ctx, draft_cache_type_v = None, **kw: (+ 8 * GIB + (1 if str(draft_cache_type_v or "f16").lower().startswith("q") else 3) * GIB+ )+ cmds = h.spawns(+ backend,+ gguf,+ crash = _flash_crash,+ healthy = _flash_off,+ n_ctx = 4096,+ speculative_type = "dspark",+ dspark_draft_path = h.sidecar(tmp_path),+ extra_args = ("--cache-type-k-draft", "q4_0", "--cache-type-v-draft", "q4_0"),+ )+ noflash = [c for c in cmds if h.last(c, "--flash-attn") == "off"]+ if not noflash or "--model-draft" not in noflash[0]:+ pytest.skip("no --flash-attn off retry with the drafter in this harness")+ assert h.margin(noflash[0]) >= h.margin(cmds[0]) + 2 * 1024+++def test_noflash_reprice_leaves_target_rollback_to_the_fitter(tmp_path):+ backend, gguf = _rollback_backend(tmp_path)+ cmds = h.spawns(+ backend,+ gguf,+ crash = _flash_crash,+ healthy = _flash_off,+ n_ctx = 4096,+ speculative_type = "dspark",+ dspark_draft_path = h.sidecar(tmp_path),+ )+ noflash = [c for c in cmds if h.last(c, "--flash-attn") == "off" and "--model-draft" in c]+ if not noflash:+ pytest.skip("no --flash-attn off retry with the drafter in this harness")+ assert h.margin(noflash[0]) < FLOOR + _ROLLBACK // MIB+++def test_a_retry_added_reserve_leaves_with_the_drafter(tmp_path):+ backend, gguf = h.cuda_backend(tmp_path)+ backend._select_gpus = lambda *a, **k: ([0], False) # planned full offload: --fit off+ cmds = h.spawns(+ backend,+ gguf,+ # flash on: signal crash (fit-on retry, then the --flash-attn off retry carries the+ # reserve forward); flash off with the drafter: plain exit, so the drafter goes.+ crash = lambda i, cmd: (+ -11 if h.last(cmd, "--flash-attn") == "on" else (1 if "--model-draft" in cmd else None)+ ),+ healthy = lambda i, cmd: "--model-draft" not in cmd,+ n_ctx = 4096,+ speculative_type = "dspark",+ dspark_draft_path = h.sidecar(tmp_path),+ )+ reserved = [c for c in cmds if "--model-draft" in c and h.margin(c) >= FLOOR]+ if not reserved or "--model-draft" in cmds[-1]:+ pytest.skip("no reserved retry followed by a drafter-free launch in this harness")+ assert h.margin(cmds[-1]) < FLOOR+++_REFUSED = ("a unified KV cache is only supported with a single sequence",)+++def _one_slot_retry(backend, gguf, tmp_path, **kw):+ return h.spawns(+ backend,+ gguf,+ stdout = lambda i, cmd: _REFUSED if i == 0 else (),+ healthy = lambda i, cmd: i > 0,+ crash = lambda i, cmd: None if i > 0 else 1,+ n_ctx = 4096,+ n_parallel = 4,+ speculative_type = "dspark",+ dspark_draft_path = h.sidecar(tmp_path),+ **kw,+ )+++def test_the_one_slot_retry_reprices_the_reserve(tmp_path):+ backend, gguf = h.cuda_backend(tmp_path)+ cmds = _one_slot_retry(h.slot_priced(backend), gguf, tmp_path)+ assert len(cmds) >= 2, [c[-6:] for c in cmds]+ first, retry = cmds[0], cmds[1]+ assert h.last(first, "--parallel") == "4"+ assert h.last(retry, "--parallel") in (None, "1")+ assert "--model-draft" in retry+ assert h.margin(first) >= FLOOR + 4 * 1024+ # Still reserved for the drafter and its one slot, not for the three slots dropped.+ assert FLOOR + 1024 <= h.margin(retry) < FLOOR + 2 * 1024+ assert retry.count("--fit-target") <= first.count("--fit-target")+++def test_the_one_slot_retry_keeps_a_user_margin_equal_to_the_reserve(tmp_path):+ """A user --fit-target equal to Studio's reserved margin stays theirs: the retry drops+ only Studio's trailing pair and reprices it for the one slot left."""+ (tmp_path / "plain").mkdir()+ backend, gguf = h.cuda_backend(tmp_path / "plain")+ plain = h.spawns(+ h.slot_priced(backend),+ gguf,+ healthy = lambda i, cmd: True,+ n_ctx = 4096,+ n_parallel = 4,+ speculative_type = "dspark",+ dspark_draft_path = h.sidecar(tmp_path),+ )[0]+ reserved = str(h.margin(plain))+ backend, gguf = h.cuda_backend(tmp_path)+ cmds = _one_slot_retry(+ h.slot_priced(backend),+ gguf,+ tmp_path,+ extra_args = ["--fit-target", reserved, "--fit-target", "1024"],+ )+ assert len(cmds) >= 2, [c[-6:] for c in cmds]+ retry = h.fit_targets(cmds[1])+ assert retry[-3:-1] == [reserved, "1024"], retry+ assert FLOOR + 1024 <= int(retry[-1]) < FLOOR + 2 * 1024, retry+++# _drop_sidecar_margins: the fallback's cleanup takes back exactly the pairs Studio added.+++ "cmd, integrity, rebuilt, pairs, want",+ [+ # Single-sequence recovery already dropped --kv-unified and set --parallel 1.+ (+ ["--parallel", "1", "--fit", "on", "--fit-ctx", "4096", "--fit-target", "9252"],+ ["--kv-unified", "--fit-ctx", "4096", "--fit-target", "9252"],+ ["--kv-unified", "--fit-ctx", "4096", "--fit-target", "1024"],+ [],+ ["--parallel", "1", "--fit", "on", "--fit-ctx", "4096", "--fit-target", "1024"],+ ),+ (+ ["--fit", "on", "--fit-target", "9225", "--fit-target", "1024", "--fit-target", "9225"],+ [],+ [],+ [["--fit-target", "9225"]],+ ["--fit", "on", "--fit-target", "9225", "--fit-target", "1024"],+ ),+ (+ ["--fit", "on", "--fit-target", "9225", "--fit-target", "11273"],+ [],+ [],+ [["--fit-target", "9225"], ["--fit-target", "11273"]],+ ["--fit", "on"],+ ),+ ],+ ids = [+ "test_cleanup_swaps_only_the_margin_pair_after_a_rewrite",+ "test_cleanup_removes_studios_trailing_pair_not_the_users",+ "test_cleanup_removes_every_recorded_pair",+ ],+)+def test_drop_sidecar_margins(cmd, integrity, rebuilt, pairs, want):+ assert lc._drop_sidecar_margins(cmd, integrity, rebuilt, pairs) == want+++def test_a_repeated_cleanup_keeps_the_users_identical_margin():+ integrity, rebuilt, user = (+ ["--fit-target", "9449"],+ ["--fit-target", "1024"],+ ["--fit-target", "9449"],+ )+ # Studio's copy is already gone; only the user's remains.+ cmd = ["llama-server", "-m", "m.gguf", "--fit-target", "1024", *user]+ assert lc._drop_sidecar_margins(cmd, integrity, rebuilt, [], user_args = user) == cmd+ cmd = ["llama-server", "--fit-target", "9449", *user]+ assert lc._drop_sidecar_margins(cmd, integrity, rebuilt, [], user_args = user) == [+ "llama-server",+ "--fit-target",+ "1024",+ *user,+ ]+ # A trailing pair a launch appended goes; the user's identical earlier one stays.+ pair = ["--fit-target", "17874"]+ assert lc._drop_sidecar_margins(+ ["llama-server", *pair, *pair], [], [], [pair], user_args = pair+ ) == [+ "llama-server",+ *pair,+ ]+ assert lc._drop_sidecar_margins(["llama-server", *pair], [], [], [pair], user_args = pair) == [+ "llama-server",+ *pair,+ ]diff --git a/studio/backend/tests/test_spec_sidecar_fit_reserve_metal.py b/studio/backend/tests/test_spec_sidecar_fit_reserve_metal.pynew file mode 100644index 00000000000..8412a10ef88--- /dev/null+++ b/studio/backend/tests/test_spec_sidecar_fit_reserve_metal.py@@ -0,0 +1,222 @@+# SPDX-License-Identifier: AGPL-3.0-only+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0++"""Metal: Auto drops a sidecar whose reserve stops the target fitting; explicit DSpark keeps it."""++from __future__ import annotations++import pytest++from . import _sidecar_fit_helpers as h+from ._sidecar_fit_helpers import GIB, lc, metal_cmd++B = lc.LlamaCppBackend++[email protected](autouse = True)+def _studio_home(tmp_path, monkeypatch):+ monkeypatch.setenv("UNSLOTH_STUDIO_HOME", str(tmp_path / "studio_home"))+++def _embd(+ monkeypatch,+ value,+ default = False,+):+ """Stub the pageable-embedding discount; excluded NextN blocks forfeit it while the+ drafter may engage."""+ calls = []++ def embd(+ self,+ *a,+ mtp_may_engage = default,+ **k,+ ):+ calls.append(mtp_may_engage)+ return value(mtp_may_engage)++ monkeypatch.setattr(B, "_metal_demand_paged_embedding_bytes", embd)+ return calls+++def _drafter_free_discount(mtp):+ return 0 if mtp else 3 * GIB+++# Every GGUF here reads 4 GiB; the target with compute and soft overhead prices ~9.2 GiB,+# so 12 GiB holds the target alone but not target + sidecar, and 16 GiB holds both.+class TestMetalLaunch:+ @pytest.mark.parametrize(+ "extras, kept",+ [+ ([], False),+ # Only one block on the card: not dropped for the whole file.+ (["--spec-draft-ngl", "1"], True),+ (["--cpu-moe-draft"], True),+ # Its weights allocate on the RPC host, so Metal's budget is no reason to drop it.+ (["--rpc", "192.0.2.1:50052", "--device", "MTL0", "--device-draft", "RPC0"], True),+ ],+ ids = [+ "test_auto_drops_a_sidecar_that_does_not_fit",+ "test_metal_auto_keeps_a_partially_offloaded_drafter",+ "test_metal_auto_keeps_a_drafter_with_its_experts_on_cpu",+ "test_metal_auto_keeps_a_drafter_pinned_to_another_device",+ ],+ )+ def test_auto_at_12_gib(self, tmp_path, monkeypatch, extras, kept):+ cmd, backend, _ = metal_cmd(tmp_path, monkeypatch, "auto", budget_gib = 12, extra_args = extras)+ assert ("--model-draft" in cmd) is kept+ assert (backend.spec_fallback_reason == "drafter_no_vram") is not kept++ def test_explicit_dspark_keeps_it_with_the_reserve(self, tmp_path, monkeypatch):+ cmd, _backend, path = metal_cmd(tmp_path, monkeypatch, "dspark", budget_gib = 12)+ assert h.last(cmd, "--model-draft") == path+ assert h.margin(cmd) >= 1024 + 4 * 1024++ def test_auto_keeps_a_sidecar_that_fits(self, tmp_path, monkeypatch):+ cmd, backend, path = metal_cmd(tmp_path, monkeypatch, "auto", budget_gib = 16)+ assert h.last(cmd, "--model-draft") == path+ assert h.margin(cmd) >= 1024 + 4 * 1024+ assert backend.spec_fallback_reason is None+++ "budget_gib, kw, discount",+ [+ # The flat draft reserve and the draft compute charge go with the sidecar.+ (12, {"kv_per_token": 128 * 1024}, None),+ # Within 224 MiB of the target-only footprint the drafter's compute must not keep it.+ (9.1, {}, None),+ # The discount decided with the drafter engaged is decided again once it is dropped.+ (9.1, {}, _drafter_free_discount),+ ],+ ids = [+ "test_metal_auto_drop_releases_the_drafter_charges",+ "test_metal_target_only_bound_leaves_out_the_drafter_compute",+ "test_the_metal_auto_drop_recomputes_the_embedding_discount",+ ],+)+def test_a_metal_auto_drop_matches_a_load_with_speculation_off(+ tmp_path, monkeypatch, budget_gib, kw, discount+):+ def run(mode):+ if discount is not None:+ _embd(monkeypatch, discount)+ return metal_cmd(tmp_path, monkeypatch, mode, budget_gib = budget_gib, **kw)[0]++ auto, off = run("auto"), run("off")+ assert "--model-draft" not in auto and "--model-draft" not in off+ assert h.last(auto, "-c") == h.last(off, "-c")+++def test_metal_auto_drop_prices_the_target_with_the_drafter_free_discount(tmp_path, monkeypatch):+ _embd(monkeypatch, _drafter_free_discount, default = True)+ cmd, _backend, _ = metal_cmd(tmp_path, monkeypatch, "auto", budget_gib = 7.5)+ assert "--model-draft" not in cmd+++def test_a_discount_the_drop_unlocks_is_logged(tmp_path, monkeypatch):+ _embd(monkeypatch, _drafter_free_discount)+ messages = h.info_messages(monkeypatch)+ cmd, _backend, _ = metal_cmd(tmp_path, monkeypatch, "auto", budget_gib = 9.1)+ assert "--model-draft" not in cmd+ assert any("of input embeddings stay in the CPU file mapping" in m for m in messages)+++def test_the_draft_free_probe_waits_for_a_drafter_that_does_not_fit(tmp_path, monkeypatch):+ """16 GiB holds both, so the draft-free discount (a cold llama-fit-params run) is never+ needed."""+ calls = _embd(monkeypatch, lambda mtp: 0)+ cmd, _backend, _ = metal_cmd(tmp_path, monkeypatch, "auto", budget_gib = 16)+ assert "--model-draft" in cmd+ assert False not in calls+++def test_metal_auto_keeps_a_user_owned_bare_model_draft(tmp_path, monkeypatch):+ """A bare --model-draft in the extras still loads (has_dft()), so Auto must not release+ its reserve while the child loads it anyway."""+ custom = tmp_path / "custom-dspark-Q8_0.gguf"+ custom.write_bytes(b"draft")+ cmd, backend, _ = metal_cmd(+ tmp_path, monkeypatch, "auto", budget_gib = 12, extra_args = ["--model-draft", str(custom)]+ )+ assert str(custom) in cmd+ assert backend.spec_fallback_reason != "drafter_no_vram"++[email protected]("pin, reserved", [("MTL0", True), ("mtl0", True), ("Vulkan0", False)])+def test_a_metal_draft_pin_is_reserved_only_on_the_metal_device(+ tmp_path, monkeypatch, pin, reserved+):+ cmd, _backend, _ = metal_cmd(+ tmp_path, monkeypatch, "dspark", budget_gib = 16, extra_args = ["--device-draft", pin]+ )+ assert "--model-draft" in cmd+ assert (h.margin(cmd) >= 1024 + 4 * 1024) is reserved+++def test_every_context_gate_prices_the_draft_micro_batch(tmp_path, monkeypatch):+ """The Metal context search takes the draft callback directly, not through the wrapper+ that reads --spec-draft-n-ubatch."""+ seen = []+ real = B._estimate_mtp_overhead_bytes++ def spy(self, n_ctx, *args, **kwargs):+ seen.append(kwargs.get("n_ubatch"))+ return real(self, n_ctx, *args, **kwargs)++ monkeypatch.setattr(B, "_estimate_mtp_overhead_bytes", spy)+ cmd, _backend, _ = metal_cmd(+ tmp_path,+ monkeypatch,+ "dspark",+ budget_gib = 16,+ kv_per_token = 32 * 1024,+ extra_args = ["--spec-draft-n-ubatch", "32"],+ )+ assert "--model-draft" in cmd+ # The first call only asks whether the reserve can be sized at all.+ assert len(seen) > 2 and set(seen[1:]) == {32}, seen++[email protected]("budget_gib, wired_gib", [(20, 20), (15, 20), (20, 18), (20, 0), (30, 18)])+def test_metal_auto_drops_a_sidecar_an_explicit_context_cannot_hold(+ tmp_path, monkeypatch, budget_gib, wired_gib+):+ """An explicit context launches verbatim: 65,536 tokens at 128 KiB hold the target (Off+ loads), not target + sidecar, so Auto drops the sidecar instead of refusing."""+ kw = dict(n_ctx = 65_536, kv_per_token = 128 * 1024, budget_gib = budget_gib, wired_gib = wired_gib)+ off, _, _ = metal_cmd(tmp_path, monkeypatch, "off", **kw)+ assert h.last(off, "-c") == "65536"+ auto, backend, _ = metal_cmd(tmp_path, monkeypatch, "auto", **kw)+ assert "--model-draft" not in auto+ assert backend.spec_fallback_reason == "drafter_no_vram"+ assert h.last(auto, "-c") == "65536"+++def test_metal_auto_keeps_a_sidecar_an_explicit_context_holds(tmp_path, monkeypatch):+ cmd, backend, _ = metal_cmd(+ tmp_path, monkeypatch, "auto", n_ctx = 4096, kv_per_token = 1024, budget_gib = 20+ )+ assert "--model-draft" in cmd+ assert backend.spec_fallback_reason != "drafter_no_vram"+++def test_metal_auto_keeps_the_sidecar_when_kv_cannot_be_estimated(tmp_path, monkeypatch):+ """With no KV estimate the target alone is not known to fit either (the discrete drop's+ own guard)."""+ cmd, backend, _ = metal_cmd(tmp_path, monkeypatch, "auto", budget_gib = 10, can_estimate_kv = False)+ assert "--model-draft" in cmd+ assert backend.spec_fallback_reason != "drafter_no_vram"+++def test_an_unmeasured_model_is_reported_once_per_load(monkeypatch):+ import test_metal_explicit_context_guard as metal_guard++ messages = h.info_messages(monkeypatch)+ case = metal_guard.TestWhichLoadsLeaveTheEmbeddingsInTheMapping()+ with lc._load_probe_memo_scope():+ assert case._bytes(monkeypatch, measured = None) == 0+ assert case._bytes(monkeypatch, measured = None) == 0+ assert sum("could not measure how llama.cpp maps" in m for m in messages) == 1diff --git a/studio/backend/tests/test_spec_sidecar_fit_reserve_multi_gpu.py b/studio/backend/tests/test_spec_sidecar_fit_reserve_multi_gpu.pynew file mode 100644index 00000000000..e77b152242e--- /dev/null+++ b/studio/backend/tests/test_spec_sidecar_fit_reserve_multi_gpu.py@@ -0,0 +1,978 @@+# SPDX-License-Identifier: AGPL-3.0-only+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0++"""Sidecar reserve across several cards: per-card shares, draft pins, split modes, host KV."""++from __future__ import annotations++import inspect+import math+from types import SimpleNamespace++import pytest++from . import _sidecar_fit_helpers as h+from ._sidecar_fit_helpers import FLOOR, GIB, MANUAL_AUTO, TWO, lc++B = lc.LlamaCppBackend+_MIXED = ((0, 49_140, 49_140), (1, 24_576, 24_576))+_VK_IGPU = ((0, 24_000, 0), (1, 24_000, 24_000))+_COMPUTE_FLOOR = math.ceil(B._MTP_DRAFT_COMPUTE_BYTES / h.MIB)++[email protected](autouse = True)+def _studio_home(tmp_path, monkeypatch):+ monkeypatch.setenv("UNSLOTH_STUDIO_HOME", str(tmp_path / "studio_home"))+ monkeypatch.setenv("CUDA_DEVICE_ORDER", "PCI_BUS_ID")+ for k in ("CUDA_VISIBLE_DEVICES", "LLAMA_ARG_DEVICE", "LLAMA_ARG_SPLIT_MODE", "LLAMA_ARG_RPC"):+ monkeypatch.delenv(k, raising = False)+ # A discrete host on every runner, class-level budget reads included.+ monkeypatch.setattr(lc, "_metal_capable_host", lambda: False)+ monkeypatch.setattr(B, "_apple_metal_memory_budget_bytes", staticmethod(lambda: 0))+++def _pci_ids(monkeypatch):+ """Studio's GPU ids are nvidia-smi's PCI indices and no mask needs a UUID resolver."""+ monkeypatch.setattr(B, "_GPU_IDS_ARE_PCI_INDICES", True)+ monkeypatch.setattr(B, "_smi_uuid_mask_order", staticmethod(lambda: None))+ monkeypatch.setattr(B, "_visibility_mask_is_unmappable", staticmethod(lambda: False))+++def _budget(monkeypatch, _pci_ids):+ """A 30% VRAM budget, so each card gets its own --fit-target margin."""+ monkeypatch.setattr(lc, "_active_vram_fraction", lambda: 0.3)+ monkeypatch.setattr(B, "_SMI_MIG_INDICES", frozenset())+ monkeypatch.setattr(B, "_host_torch_is_rocm", staticmethod(lambda: False))+++def _band(m, lo, hi):+ return 1024 + lo * 1024 <= m < 1024 + hi * 1024+++def test_multi_gpu_cuda_does_not_broadcast_the_reserve(tmp_path):+ """A scalar --fit-target applies to every device: adding the whole sidecar to it would+ withhold the sidecar on each card and push target layers to the host."""+ backend, gguf = h.cuda_backend(tmp_path, memory = TWO)+ cmd = h.dspark(backend, gguf, tmp_path)+ assert h.last(cmd, "--model-draft") == str(tmp_path / "dspark-model-Q8_0.gguf")+ assert h.margin(cmd) < FLOOR+++ "args, env",+ [(["--device", "Vulkan0,Vulkan1"], {}), ([], {"LLAMA_ARG_DEVICE": "CUDA0,CUDA1"})],+ ids = ["argv", "env"],+)+def test_no_scalar_reserve_on_a_multi_device_launch(args, env):+ argv = ["--fit", "on", "-md", "/d.gguf", *args]+ assert lc._sidecar_reserve_margin_flags(argv, (), env, 1024, 8192) == []+++def test_a_single_named_device_keeps_the_reserve():+ argv = ["--fit", "on", "-md", "/d.gguf", "--device", "CUDA0"]+ assert lc._sidecar_reserve_margin_flags(argv, (), {}, 1024, 8192) == ["--fit-target", "9216"]+++def test_extras_widening_the_device_list_get_no_scalar_reserve(tmp_path):+ backend, gguf = h.cuda_backend(tmp_path, memory = TWO, vulkan = True)+ backend._select_gpus = lambda *a, **k: ([0], True)+ cmd = h.dspark(backend, gguf, tmp_path, extra_args = ("--device", "Vulkan0,Vulkan1"))+ assert h.margin(cmd) < FLOOR+++def test_one_named_device_on_a_two_gpu_host_keeps_the_reserve(tmp_path):+ backend, gguf = h.cuda_backend(tmp_path, memory = TWO, vulkan = True)+ cmd = h.dspark(backend, gguf, tmp_path, extra_args = ("--device", "Vulkan0"))+ assert h.last(cmd, "--device") == "Vulkan0"+ assert h.margin(cmd) >= FLOOR+++def test_two_detected_gpus_without_a_device_pin_stay_unreserved(tmp_path):+ backend, gguf = h.cuda_backend(tmp_path, memory = TWO, vulkan = True)+ assert h.margin(h.dspark(backend, gguf, tmp_path)) < FLOOR++[email protected]("devices", ["CUDA0,CUDA0", "CUDA0, cuda0"])+def test_a_repeated_device_is_one_card_for_the_reserve(tmp_path, devices):+ backend, gguf = h.cuda_backend(tmp_path)+ assert h.margin(h.dspark(backend, gguf, tmp_path, extra_args = ["--device", devices])) >= FLOOR+++def test_a_launch_the_arch_gate_narrows_to_one_card_gets_the_reserve(tmp_path):+ backend, gguf = h.cuda_backend(tmp_path, memory = TWO)+ backend._arch_gate_survivors = lambda binary = None: [1]+ assert h.margin(h.dspark(backend, gguf, tmp_path)) >= FLOOR+++def test_an_arch_retry_onto_several_gpus_drops_the_scalar_reserve(tmp_path):+ rows = ((0, 24_000, 24_000), (1, 12_000, 12_000), (2, 12_000, 12_000))+ backend, gguf = h.cuda_backend(tmp_path, memory = rows)+ backend._select_gpus = lambda *a, **k: ([0], True)+ backend._arch_crash_retry_gpu_ids = lambda *a, **k: [1, 2]+ crash = ("ggml_cuda_compute_forward: ROCm error: device kernel image is invalid",)+ cmds = h.spawns(+ backend,+ gguf,+ stdout = lambda i, cmd: crash if i == 0 else (),+ healthy = lambda i, cmd: i > 0,+ crash = lambda i, cmd: None if i > 0 else 1,+ n_ctx = 4096,+ speculative_type = "dspark",+ dspark_draft_path = h.sidecar(tmp_path),+ )+ assert len(cmds) >= 2, [c[-6:] for c in cmds]+ assert h.margin(cmds[0]) >= FLOOR+ assert "--model-draft" in cmds[1]+ assert h.margin(cmds[1]) < FLOOR+++def test_a_launch_left_on_two_cards_reserves_each_cards_share(tmp_path):+ backend, gguf = h.cards(tmp_path)+ margins = h.margins(h.dspark(backend, gguf, tmp_path))+ assert len(margins) == 2+ assert all(_band(m, 4, 5) for m in margins)+++ "memory, extras",+ [+ # 30,000 of 40,000 MiB free: three quarters of the ~8.3 GiB reserve on card 0.+ (((0, 30_000, 30_000), (1, 10_000, 10_000)), []),+ # The drafter inherits the target's --tensor-split even on equally free cards.+ (TWO, ["--tensor-split", "3,1"]),+ ],+ ids = [+ "test_unequal_cards_carry_their_own_shares",+ "test_an_explicit_tensor_split_decides_the_draft_shares",+ ],+)+def test_draft_shares_three_to_one(tmp_path, memory, extras):+ backend, gguf = h.cards(tmp_path, memory)+ big, small = h.margins(h.dspark(backend, gguf, tmp_path, extra_args = extras))+ assert _band(big, 6, 7) and _band(small, 2, 3)+++def test_a_small_card_beside_a_large_one_is_not_over_reserved(tmp_path):+ """A broadcast of the large card's share (~7.6 GiB) would exceed the 8,000 MiB card."""+ backend, gguf = h.cards(tmp_path, ((0, 80_000, 80_000), (1, 8_000, 8_000)))+ big, small = h.margins(h.dspark(backend, gguf, tmp_path))+ assert big >= 1024 + 7 * 1024+ assert small < 1024 + 1024+++def test_per_device_margins_follow_an_inherited_mask_order(tmp_path, monkeypatch):+ """CUDA_VISIBLE_DEVICES=1,0 makes physical 1 the child's device 0."""+ monkeypatch.setenv("CUDA_VISIBLE_DEVICES", "1,0")+ backend, gguf = h.cards(tmp_path, ((0, 30_000, 30_000), (1, 10_000, 10_000)))+ first, second = h.margins(h.dspark(backend, gguf, tmp_path))+ assert first < second+++def _three_blocks(+ tmp_path,+ graph_gib = None,+ target_layers = None,+):+ backend, gguf = h.cards(tmp_path)+ h.three_block_drafter(backend)+ if graph_gib is not None:+ backend._mtp_draft_compute_bytes = lambda *a, **k: graph_gib * GIB+ if target_layers is not None:+ backend._n_layers = target_layers+ return backend, gguf+++ "extras, bands",+ [+ # Equal cards: blocks 0-1 below 2/4, block 2 at it, so two on card 0 and one on card 1.+ ([], [(5, 6), (2, 3)]),+ # 3:2 keeps every block under 0.6 of n_layer + 1, so all three on card 0.+ (["--tensor-split", "3,2"], [(8, 9), (0, 1)]),+ ],+)+def test_draft_shares_follow_llama_cpps_whole_layer_split(tmp_path, extras, bands):+ backend, gguf = _three_blocks(tmp_path)+ margins = h.margins(h.dspark(backend, gguf, tmp_path, extra_args = extras))+ for m, (lo, hi) in zip(margins, bands):+ assert _band(m, lo, hi), margins+++def test_the_output_card_keeps_the_draft_compute_buffers_without_a_block(tmp_path):+ """3:2 puts all three blocks on card 0, but the draft still runs (and writes its logits)+ on the last card."""+ backend, gguf = _three_blocks(tmp_path)+ _big, small = h.margins(h.dspark(backend, gguf, tmp_path, extra_args = ["--tensor-split", "3,2"]))+ assert small >= 1024 + _COMPUTE_FLOOR+++def test_the_output_card_keeps_the_whole_vocabulary_graph(tmp_path):+ """The draft graph's logits over the borrowed vocabulary are sized on the output card,+ so that part is not split by block shares."""+ backend, gguf = _three_blocks(tmp_path, graph_gib = 3)+ _big, small = h.margins(h.dspark(backend, gguf, tmp_path, extra_args = ["--tensor-split", "3,2"]))+ assert small >= 1024 + _COMPUTE_FLOOR + 3 * 1024+++def test_a_split_that_leaves_the_last_card_out_keeps_the_graph_on_the_used_one(tmp_path):+ backend, gguf = _three_blocks(tmp_path, graph_gib = 3)+ used, idle = h.margins(h.dspark(backend, gguf, tmp_path, extra_args = ["--tensor-split", "1,0"]))+ assert used >= 1024 + 3 * 1024+ assert idle < 2048+++def test_the_graph_follows_the_targets_output_layer_not_the_drafters(tmp_path):+ """DSpark borrows the target's output tensor: with one target block and a 3:2 split the+ target's output layer (index 1 of 2 units = 0.5 < 0.6) is on card 0, so the graph is too."""+ backend, gguf = _three_blocks(tmp_path, graph_gib = 3, target_layers = 1)+ used, idle = h.margins(h.dspark(backend, gguf, tmp_path, extra_args = ["--tensor-split", "3,2"]))+ assert used >= 1024 + 3 * 1024 + 4 * 1024+ assert idle < 2048+++def test_a_draft_with_its_own_head_keeps_its_graph_on_its_own_output_card(tmp_path, monkeypatch):+ """Same split, but the draft owns its head: its output layer (3 / 4 = 0.75) is on card 1."""+ monkeypatch.setattr(lc, "_sidecar_borrows_output", lambda path: False)+ backend, gguf = _three_blocks(tmp_path, graph_gib = 3, target_layers = 1)+ _used, out = h.margins(h.dspark(backend, gguf, tmp_path, extra_args = ["--tensor-split", "3,2"]))+ assert out >= 1024 + 3 * 1024+++def _owned_head_margins(tmp_path, monkeypatch, head_gib):+ monkeypatch.setattr(lc, "_sidecar_borrows_output", lambda path: False)+ backend, gguf = _three_blocks(tmp_path, target_layers = 1)+ read = backend._tensor_spill_layout++ def layout(path, **kw):+ if "dspark" in str(path):+ return SimpleNamespace(complete = True, lm_head_bytes = int(head_gib * GIB))+ return read(path, **kw)++ backend._tensor_spill_layout = layout+ return h.margins(h.dspark(backend, gguf, tmp_path, extra_args = ["--tensor-split", "3,2"]))+++def test_an_owned_head_is_reserved_on_its_output_card(tmp_path, monkeypatch):+ (tmp_path / "a").mkdir()+ (tmp_path / "b").mkdir()+ no_head = _owned_head_margins(tmp_path / "a", monkeypatch, 0)+ head = _owned_head_margins(tmp_path / "b", monkeypatch, 2)+ assert head[1] - no_head[1] >= 2000+ assert no_head[0] - head[0] >= 2000+++def test_a_row_split_places_whole_draft_layers_like_a_layer_split(tmp_path):+ """--split-mode row assigns blocks to devices like a layer split (llama-model.cpp)."""+ (tmp_path / "row").mkdir()+ backend, gguf = _three_blocks(tmp_path / "row")+ row = h.margins(h.dspark(backend, gguf, tmp_path, extra_args = ["--split-mode", "row"]))+ backend, gguf = _three_blocks(tmp_path)+ layer = h.margins(h.dspark(backend, gguf, tmp_path))+ assert row == layer+ assert row[0] > row[1]+++def test_the_underscore_tensor_split_spelling_places_the_draft(tmp_path):+ backend, gguf = _three_blocks(tmp_path)+ margins = h.margins(h.dspark(backend, gguf, tmp_path, extra_args = ["--tensor_split", "1,0"]))+ assert len(margins) == 2, margins+ assert margins[0] >= FLOOR+ assert margins[1] < 2048+++def test_a_tensor_split_launch_gets_no_per_device_reserve(tmp_path):+ """--split-mode tensor folds the cards into one meta device: a per-card --fit-target+ cannot hold the sidecar."""+ backend, gguf = _three_blocks(tmp_path)+ margins = h.margins(h.dspark(backend, gguf, tmp_path, extra_args = ["--split-mode", "tensor"]))+ assert all(m < 2048 for m in margins), margins+++def test_shares_follow_each_cards_free_memory_past_its_own_margin(tmp_path):+ """A user margin of 20,000 MiB leaves card 0 about 4 GB for weights against card 1's 23 GB."""+ backend, gguf = h.cards(tmp_path)+ margins = h.margins(+ h.dspark(backend, gguf, tmp_path, extra_args = ["--fit-target", "20000,1024"])+ )+ assert len(margins) == 2+ assert margins[0] < 24_000+ assert margins[1] > 1024 + 4 * 1024++[email protected]("order, per_card", [("FASTEST_FIRST", False), ("PCI_BUS_ID", True)])+def test_per_card_shares_need_a_provable_device_order(tmp_path, monkeypatch, order, per_card):+ """nvidia-smi keys free memory by PCI order; under FASTEST_FIRST the child enumerates+ its own way."""+ monkeypatch.setenv("CUDA_DEVICE_ORDER", order)+ monkeypatch.setattr(B, "_GPU_IDS_ARE_PCI_INDICES", True)+ monkeypatch.setattr(B, "_smi_uuid_mask_order", staticmethod(lambda: None))+ backend, gguf = h.cards(tmp_path, ((0, 80_000, 80_000), (1, 8_000, 8_000)))+ assert (len(h.margins(h.dspark(backend, gguf, tmp_path))) == 2) is per_card++[email protected]("parent_order, reserved", [("FASTEST_FIRST", False), ("PCI_BUS_ID", True)])+def test_torch_ordinals_in_another_order_than_the_child_get_no_positional_reserve(+ tmp_path, monkeypatch, parent_order, reserved+):+ """No nvidia-smi: the ids are torch ordinals in the parent's CUDA_DEVICE_ORDER, while a+ GPU pick pins the child to PCI_BUS_ID."""+ monkeypatch.setenv("CUDA_DEVICE_ORDER", parent_order)+ monkeypatch.setattr(B, "_GPU_IDS_ARE_PCI_INDICES", False)+ monkeypatch.setattr(B, "_host_torch_is_rocm", staticmethod(lambda: False))+ backend, gguf = h.cards(tmp_path, ((0, 80_000, 80_000), (1, 8_000, 8_000)))+ seen = h.spy_fit_sized(backend)+ cmd = h.dspark(backend, gguf, tmp_path, gpu_ids = [0, 1])+ assert (len(h.margins(cmd)) == 2) is reserved+ assert seen["fit_sized"] is reserved+++def test_the_order_check_is_one_helper_for_planning_and_spawn(monkeypatch):+ monkeypatch.setattr(B, "_GPU_IDS_ARE_PCI_INDICES", False)+ monkeypatch.setattr(B, "_host_torch_is_rocm", staticmethod(lambda: False))+ monkeypatch.delenv("CUDA_DEVICE_ORDER", raising = False)+ backend = B()+ # Unset is CUDA's default, FASTEST_FIRST.+ assert backend._positional_child_order_unproven({}) is False+ assert backend._positional_child_order_unproven({"CUDA_DEVICE_ORDER": "FASTEST_FIRST"}) is False+ assert backend._positional_child_order_unproven({"CUDA_DEVICE_ORDER": "PCI_BUS_ID"}) is True+ # HIP ignores CUDA_DEVICE_ORDER.+ monkeypatch.setattr(B, "_host_torch_is_rocm", staticmethod(lambda: True))+ assert backend._positional_child_order_unproven({"CUDA_DEVICE_ORDER": "PCI_BUS_ID"}) is False+++def _mig_two_cards(tmp_path, monkeypatch):+ monkeypatch.setattr(B, "_GPU_IDS_ARE_PCI_INDICES", True)+ monkeypatch.setattr(B, "_smi_uuid_mask_order", staticmethod(lambda: None))+ # A MIG mask (child order: card 1's slice, then card 0's) no resolver can map.+ monkeypatch.setattr(B, "_visibility_mask_is_unmappable", staticmethod(lambda: True))+ return h.cards(tmp_path, ((0, 16_000, 16_000), (1, 4_000, 4_864)))+++def test_an_unmappable_mig_mask_gets_no_positional_reserve(tmp_path, monkeypatch):+ backend, gguf = _mig_two_cards(tmp_path, monkeypatch)+ assert len(h.margins(h.dspark(backend, gguf, tmp_path))) == 1+++def test_an_unmappable_mig_mask_is_not_reported_as_reserved(tmp_path, monkeypatch):+ backend, gguf = _mig_two_cards(tmp_path, monkeypatch)+ seen = h.spy_fit_sized(backend)+ h.dspark(backend, gguf, tmp_path)+ assert seen["fit_sized"] is False+++def test_a_users_per_device_margins_stay_per_device_across_cards():+ flags = lc._sidecar_reserve_margin_flags(+ ["--model-draft", "/d.gguf", "--fit", "on"],+ ["--fit-target", "1024,20000"],+ {},+ 1024,+ 4213,+ per_device = [4213, 4213],+ )+ assert flags == ["--fit-target", "5237,24213"]+++def test_a_scalar_user_margin_is_broadcast_under_per_device_shares():+ flags = lc._sidecar_reserve_margin_flags(+ ["--model-draft", "/d.gguf", "--fit", "on"],+ ["--fit-target", "2048"],+ {},+ 1024,+ 5000,+ per_device = [5000, 500],+ )+ assert flags == ["--fit-target", "7048,2548"]+++ "user, env, want",+ [+ # A short list sets only the entries it names; the rest keep the earlier value.+ (["--fit-target", "4096", "--fit-target", "2048,3072"], {}, "3048,4072,5096"),+ # The env applies first, the argv after it.+ ([], {"LLAMA_ARG_FIT_TARGET": "4096"}, "5096,5096,5096"),+ (["--fit-target", "2048"], {"LLAMA_ARG_FIT_TARGET": "4096,4096,8192"}, "3048,3048,3048"),+ (["--fit-target=2048,3072"], {"LLAMA_ARG_FIT_TARGET": "8192"}, "3048,4072,9192"),+ (["--fit_target", "4096"], {}, "5096,5096,5096"),+ (["--fit_target=2048,3072", "-fitt", "4096"], {}, "5096,5096,5096"),+ ],+)+def test_per_device_margins_follow_llama_cpps_update_order(user, env, want):+ flags = lc._sidecar_reserve_margin_flags(+ ["--model-draft", "/d.gguf", "--fit", "on"],+ user,+ env,+ 1024,+ 1000,+ per_device = [1000, 1000, 1000],+ )+ assert flags == ["--fit-target", want]+++ "user, base, reserve, per_device, by_device, want",+ [+ # The smaller card's lower margin is not raised to the scalar.+ ([], 4000, 5000, [5000, 500], [4000, 1000], "9000,1500"),+ (["--fit-target", "2048"], 1024, 100, [100, 100], [1024, 4096], "2148,4196"),+ # A child masked to the smaller card carries that card's own margin.+ ([], 4000, 5000, None, [1000], "6000"),+ ],+ ids = [+ "test_studios_per_card_margins_are_kept_under_the_reserve",+ "test_a_users_margin_still_wins_over_a_lower_per_card_base",+ "test_one_card_keeps_its_own_emitted_base_under_the_reserve",+ ],+)+def test_the_reserve_sits_on_each_cards_own_base(user, base, reserve, per_device, by_device, want):+ kw = {} if per_device is None else {"per_device": per_device}+ flags = lc._sidecar_reserve_margin_flags(+ ["--model-draft", "/d.gguf", "--fit", "on"],+ user,+ {},+ base,+ reserve,+ base_by_device = by_device,+ **kw,+ )+ assert flags == ["--fit-target", want]+++def _base_margins(+ tmp_path,+ memory = _MIXED,+ **kw,+):+ (tmp_path / "base").mkdir()+ backend, gguf = h.cards(tmp_path / "base", memory)+ return h.margins(h.placement._launch(backend, gguf, **kw)["cmd"])+++def test_each_card_keeps_its_budget_margin_under_the_reserve(tmp_path, _budget):+ base = _base_margins(tmp_path)+ assert len(base) == 2 and base[0] != base[1]+ backend, gguf = h.cards(tmp_path, _MIXED)+ reserved = h.margins(h.dspark(backend, gguf, tmp_path))+ assert len(reserved) == 2+ assert all(r > b for r, b in zip(reserved, base))+ assert reserved[1] < 24_576++[email protected]("main_gpu", [[], ["--main-gpu", "1"]])+def test_split_mode_none_keeps_the_other_cards_margins(tmp_path, _budget, main_gpu):+ """llama.cpp keeps only devices[main_gpu] and the fitter reads margin 0 for it, so the+ reserve goes in entry 0 whichever card is the main one."""+ extras = ["--split-mode", "none", *main_gpu]+ base = _base_margins(tmp_path, extra_args = extras)+ backend, gguf = h.cards(tmp_path, _MIXED)+ reserved = h.margins(h.dspark(backend, gguf, tmp_path, extra_args = extras))+ assert len(reserved) == 2+ assert reserved[0] >= base[0] + 8 * 1024+ assert reserved[1] == base[1]+++def test_one_local_card_with_rpc_gets_no_broadcast_reserve(tmp_path, _budget):+ """A scalar margin is broadcast to the remote devices too."""+ extras = ["--rpc", "10.0.0.2:50052"]+ (tmp_path / "base").mkdir()+ backend, gguf = h.cuda_backend(tmp_path / "base", memory = _MIXED[:1])+ base = h.margins(h.placement._launch(backend, gguf, extra_args = extras)["cmd"])+ backend, gguf = h.cuda_backend(tmp_path, memory = _MIXED[:1])+ assert h.margins(h.dspark(backend, gguf, tmp_path, extra_args = extras)) == base+++def test_an_rpc_launch_gets_no_positional_reserve_list(tmp_path, _budget):+ backend, gguf = h.cards(tmp_path, _MIXED)+ cmd = h.dspark(backend, gguf, tmp_path, extra_args = ["--rpc", "10.0.0.2:50052"])+ assert len(h.margins(cmd)) == 1+++def test_the_text_only_retry_leaves_the_reserve_pair_alone():+ source = "".join(inspect.getsource(B.load_model).split())+ at = source.index("cmd=self._strip_mmproj_args(_vision_gpu_cmd)")+ window = source[at : at + 400]+ assert "notin_sidecar_override_pairs" in window+ assert window.index("notin_sidecar_override_pairs") < window.index(+ "_per_device_fit_target_argv("+ )++[email protected]("draft_device, reserved", [("Vulkan1", False), ("Vulkan0", True)])+def test_a_draft_pinned_to_another_vulkan_device_is_not_reserved_on_the_target(+ tmp_path, draft_device, reserved+):+ backend, gguf = h.cuda_backend(tmp_path, memory = TWO, vulkan = True)+ cmd = h.dspark(+ backend, gguf, tmp_path, gpu_ids = [0], extra_args = ["--device-draft", draft_device]+ )+ assert (h.margin(cmd) >= FLOOR) is reserved++[email protected]("draft_device, reserved", [("Vulkan0", True), ("Vulkan1", False)])+def test_a_stale_device_extra_does_not_decide_the_draft_device(tmp_path, draft_device, reserved):+ """gpu_ids=[0] owns placement and strips the stale --device Vulkan1 at emission."""+ backend, gguf = h.cuda_backend(tmp_path, memory = TWO, vulkan = True)+ extras = ["--device", "Vulkan1", "--device-draft", draft_device]+ cmd = h.dspark(backend, gguf, tmp_path, gpu_ids = [0], extra_args = extras)+ assert (h.margin(cmd) >= FLOOR) is reserved+++ "pin, reserved", [("CUDA0", True), ("CUDA1", False), ("Vulkan0", False), ("RPC0", False)]+)+def test_a_cuda_draft_pin_on_the_only_target_card_keeps_the_reserve(tmp_path, pin, reserved):+ """The child's mask numbers the single target card 0, so CUDA0 is the target."""+ backend, gguf = h.cuda_backend(tmp_path)+ cmd = h.dspark(backend, gguf, tmp_path, extra_args = ["--device-draft", pin])+ assert "--model-draft" in cmd+ assert (h.margin(cmd) >= FLOOR) is reserved+++ "extras, pin, reserved",+ [+ ([], "Vulkan1", True),+ ([], "Vulkan0", True),+ (["--split-mode", "none"], "Vulkan1", False),+ (["--split-mode", "none"], "Vulkan0", True),+ ],+)+def test_a_pin_on_any_studio_chosen_vulkan_card_is_on_the_target(tmp_path, extras, pin, reserved):+ backend, gguf = h.cuda_backend(tmp_path, memory = TWO, vulkan = True)+ cmd = h.dspark(+ backend, gguf, tmp_path, gpu_ids = [0, 1], extra_args = [*extras, "--device-draft", pin]+ )+ assert (h.margin(cmd) >= FLOOR) is reserved++[email protected]("pin, reserved", [("CUDA1", True), ("CUDA0", False)])+def test_an_inherited_target_device_decides_the_draft_pin(tmp_path, monkeypatch, pin, reserved):+ monkeypatch.setenv("LLAMA_ARG_DEVICE", "CUDA1")+ backend, gguf = h.cuda_backend(tmp_path, memory = TWO)+ cmd = h.dspark(backend, gguf, tmp_path, extra_args = ["--spec-draft-device", pin])+ assert (h.margin(cmd) >= FLOOR) is reserved+++ "argv, devices, on_target",+ [+ # The masked child's device 0 is the target.+ (["--spec-draft-device", "CUDA0"], None, True),+ (["--spec-draft-device", "CUDA1"], None, False),+ # A companion widening: the target is --device CUDA0, the drafter its CUDA1.+ (["--device", "CUDA0", "--spec-draft-device", "CUDA1"], "CUDA0", False),+ (["--device", "CUDA1", "--spec-draft-device", "CUDA1"], "CUDA1", True),+ # Settled at planning, where the backend is known.+ (["--spec-draft-device", "RPC0"], None, True),+ (["--spec-draft-device", "none"], None, True),+ ],+)+def test_the_spawned_argv_decides_a_cuda_draft_pin(argv, devices, on_target):+ assert lc._spawn_draft_on_target(argv, devices) is on_target+++def test_a_repeated_device_is_one_card_for_the_draft_pin():+ argv = ["--device", "CUDA0,CUDA0", "--device-draft", "CUDA0"]+ assert lc._spawn_draft_on_target(argv, "CUDA0,CUDA0")+ assert not lc._spawn_draft_on_target(["--device-draft", "CUDA1"], "CUDA0,CUDA0")+++def test_a_repeated_draft_pin_is_one_card():+ assert lc._spawn_draft_on_target(+ ["--device", "CUDA0", "--device-draft", "CUDA0,CUDA0"], "CUDA0"+ )+++_NONE_MAIN_1 = ["--split-mode", "none", "--main-gpu", "1"]+_BOTH = ["--device", "CUDA0,CUDA1"]+++ "argv, devices, on_target",+ [+ # --split-mode none puts the target on --main-gpu, not device 0.+ ([*_NONE_MAIN_1, "--spec-draft-device", "CUDA1"], None, True),+ ([*_NONE_MAIN_1, "--spec-draft-device", "CUDA0"], None, False),+ ([*_BOTH, *_NONE_MAIN_1, "--spec-draft-device", "CUDA1"], "CUDA0,CUDA1", True),+ ([*_BOTH, *_NONE_MAIN_1, "--spec-draft-device", "CUDA0"], "CUDA0,CUDA1", False),+ ],+)+def test_split_mode_none_compares_the_pin_with_the_main_gpu(argv, devices, on_target):+ assert lc._spawn_draft_on_target(argv, devices, {}) is on_target++[email protected]("pin, on_target", [("CUDA0", True), ("CUDA1", False)])+def test_split_mode_none_defaults_to_the_lists_first_device(pin, on_target):+ argv = [*_BOTH, "--split-mode", "none", "--spec-draft-device", pin]+ assert lc._spawn_draft_on_target(argv, "CUDA0,CUDA1", {}) is on_target+++def test_split_mode_none_keeps_the_whole_reserve_on_the_main_card(tmp_path):+ backend, gguf = h.cards(tmp_path)+ assert h.margin(h.dspark(backend, gguf, tmp_path, extra_args = ["--split-mode", "none"])) >= FLOOR++[email protected]("pin, reserved", [("CUDA0", True), ("CUDA1", False)])+def test_split_mode_none_narrows_an_inherited_device_list_at_planning(+ tmp_path, monkeypatch, pin, reserved+):+ monkeypatch.setenv("LLAMA_ARG_DEVICE", "CUDA0,CUDA1")+ backend, gguf = h.cuda_backend(tmp_path, memory = TWO)+ extras = ["--split-mode", "none", "--spec-draft-device", pin]+ assert (h.margin(h.dspark(backend, gguf, tmp_path, extra_args = extras)) >= FLOOR) is reserved+++def test_planning_drops_an_inherited_split_mode_the_launch_clears(tmp_path, monkeypatch):+ """A layer launch clears an inherited non-layer LLAMA_ARG_SPLIT_MODE before spawning, so+ planning must not narrow the target to the main card on it."""+ monkeypatch.setenv("LLAMA_ARG_DEVICE", "CUDA0,CUDA1")+ monkeypatch.setenv("LLAMA_ARG_SPLIT_MODE", "none")+ backend, gguf = h.cards(tmp_path)+ assert (+ h.margin(h.dspark(backend, gguf, tmp_path, extra_args = ["--device-draft", "CUDA1"])) >= FLOOR+ )+++ "extras, at",+ [+ ([*_BOTH, "--spec-draft-device", "CUDA0"], 0),+ ([*_BOTH, "--spec-draft-device", "CUDA1"], 1),+ (["--spec-draft-device", "CUDA1"], 1),+ ],+)+def test_a_draft_pinned_to_one_target_card_reserves_there(tmp_path, extras, at):+ backend, gguf = h.cards(tmp_path)+ margins = h.margins(h.dspark(backend, gguf, tmp_path, extra_args = extras))+ assert len(margins) == 2, margins+ assert margins[at] >= FLOOR, margins+ assert margins[1 - at] < 1024 + 1024, margins+++ "pin, devices, count, at",+ [+ ("Vulkan1", "Vulkan0,Vulkan1", 2, 1),+ ("vulkan0", "Vulkan0,Vulkan1", 2, 0),+ ("Vulkan2", "Vulkan0,Vulkan1", 2, None),+ ("Vulkan1", None, 2, None),+ ("CUDA1", None, 2, 1),+ ("CUDA0,CUDA1", "CUDA0,CUDA1", 2, None),+ ],+)+def test_a_draft_pin_resolves_to_its_target_card(pin, devices, count, at):+ assert lc._draft_pin_device_index(pin, devices, count) == at+++ "args, want",+ [+ (["--split_mode=tensor"], "tensor"),+ (["--split_mode", "row"], "row"),+ (["-sm", "layer"], "layer"),+ ],+)+def test_the_underscore_split_mode_spelling_is_read(args, want):+ assert lc._extra_args_split_mode(args, {}) == want+++# The fit-sized report: planning and spawn agree on where a reserve is provable.+++ "extras, caps_fit_target, fit_sized",+ [+ ([], True, True),+ (["--rpc", "10.0.0.2:50052"], True, False),+ (_BOTH, True, False),+ ([], False, False),+ ],+)+def test_fit_sized_is_reported_only_where_the_spawn_can_price(+ tmp_path, extras, caps_fit_target, fit_sized+):+ backend, gguf = h.cards(tmp_path)+ caps = dict(h.CAPS, supports_fit_target = caps_fit_target)+ backend.probe_server_capabilities = lambda _binary = None: dict(caps)+ seen = h.spy_fit_sized(backend)+ h.dspark(backend, gguf, tmp_path, extra_args = extras)+ assert seen["fit_sized"] is fit_sized+++def _past_one_card(tmp_path):+ backend, gguf = h.cards(tmp_path)+ backend._get_gguf_size_bytes = lambda path: (8 if "dspark" in str(path) else 30) * GIB+ return backend, gguf+++def test_device_extras_a_gpu_ids_pin_strips_do_not_block_the_report(tmp_path):+ backend, gguf = _past_one_card(tmp_path)+ seen = h.spy_fit_sized(backend)+ cmd = h.dspark(backend, gguf, tmp_path, gpu_ids = [0, 1], extra_args = _BOTH)+ assert "--device" not in cmd+ assert len(h.margins(cmd)) == 2+ assert seen["fit_sized"] is True++[email protected]("pin, reserved", [("CUDA0,CUDA1", False), ("CUDA0", True)])+def test_a_multi_device_draft_pin_is_not_reported_as_reserved(tmp_path, pin, reserved):+ backend, gguf = _past_one_card(tmp_path)+ seen = h.spy_fit_sized(backend)+ cmd = h.dspark(backend, gguf, tmp_path, extra_args = [*_BOTH, "--device-draft", pin])+ assert (len(h.margins(cmd)) == 2) is reserved+ assert seen["fit_sized"] is reserved+++def test_an_unpinned_multi_card_vulkan_launch_gets_no_positional_reserve(tmp_path):+ """Vulkan orders its devices its own way, so neither planning nor spawn reserves."""+ backend, gguf = h.cards(tmp_path, vulkan = True)+ seen = h.spy_fit_sized(backend)+ cmd = h.dspark(backend, gguf, tmp_path, **MANUAL_AUTO)+ assert "--device" not in cmd+ assert seen["fit_sized"] is False+ assert len(h.margins(cmd)) == 1+++ "extras, vulkan, reserved",+ [+ (["--device-draft", "CUDA2"], False, False),+ # A repeated target entry leaves the pin's position unknown.+ (["--device", "CUDA0,CUDA1,CUDA0", "--device-draft", "CUDA0"], False, False),+ # A Vulkan name with no target list to find it in.+ (["--device-draft", "Vulkan0"], True, False),+ (["--device-draft", "CUDA1"], False, True),+ ([*_BOTH, "--device-draft", "CUDA1"], False, True),+ ],+)+def test_planning_reports_only_a_draft_pin_the_spawn_resolves(tmp_path, extras, vulkan, reserved):+ backend, gguf = h.cards(tmp_path, vulkan = vulkan)+ seen = h.spy_fit_sized(backend)+ cmd = h.dspark(+ backend, gguf, tmp_path, extra_args = list(extras), **(MANUAL_AUTO if vulkan else {})+ )+ assert (len(h.margins(cmd)) == 2) is reserved+ assert seen["fit_sized"] is reserved+++def test_planning_resolves_a_draft_pin_against_studios_vulkan_pin(tmp_path):+ """gpu_ids on Vulkan reaches the child as --device Vulkan0,Vulkan1 (added after planning)."""+ backend, gguf = h.cards(tmp_path, vulkan = True)+ seen = h.spy_fit_sized(backend)+ cmd = h.dspark(+ backend, gguf, tmp_path, gpu_ids = [0, 1], extra_args = ["--device-draft", "Vulkan1"]+ )+ assert len(h.margins(cmd)) == 2+ assert seen["fit_sized"] is True+++ "memory, extras, caps_fit_target",+ [+ # --rpc devices come first in the child's list: no reserve on one local card.+ (h.ONE, ["--rpc", "10.0.0.2:50052"], True),+ (h.ONE, [], False),+ # --split-mode none keeps the target on card 0, the drafter on card 1.+ (TWO, ["--split-mode", "none", "--device-draft", "CUDA1"], True),+ ],+)+def test_a_reserve_the_spawn_drops_is_reported_as_missing(+ tmp_path, monkeypatch, memory, extras, caps_fit_target+):+ backend, gguf = h.cards(tmp_path, memory)+ caps = dict(h.CAPS, supports_fit_target = caps_fit_target)+ backend.probe_server_capabilities = lambda _binary = None: dict(caps)+ seen = h.spy_fit_sized(backend)+ messages = h.info_messages(monkeypatch)+ cmd = h.dspark(backend, gguf, tmp_path, extra_args = list(extras))+ assert h.margin(cmd) < 1024 + 4 * 1024+ assert seen["fit_sized"] is False+ assert not any("keeps" in m and "free for it" in m for m in messages)+ assert any("is not reserved" in m for m in messages)+++def test_a_dflash_reserve_the_spawn_drops_is_reported_as_missing(tmp_path, monkeypatch):+ backend, gguf = h.cards(tmp_path, h.ONE)+ messages = h.info_messages(monkeypatch)+ cmd = h.placement._launch(+ backend,+ gguf,+ speculative_type = "dflash",+ dflash_draft_path = h.sidecar(tmp_path, "dflash"),+ extra_args = ["--rpc", "10.0.0.2:50052"],+ )["cmd"]+ assert "draft-dflash" in cmd+ assert any(m.startswith("DFlash") and "is not reserved" in m for m in messages)+ assert not any("keeps" in m and "free for it" in m for m in messages)+++def test_a_renumbered_companion_pin_is_reported_as_reserved(tmp_path):+ """gpu_ids [1, 2] masks the child to those cards, so the request's CUDA2 reaches the+ child as CUDA1."""+ backend, gguf = h.cards(tmp_path, (*TWO, (2, 24_000, 24_000)))+ seen = h.spy_fit_sized(backend)+ cmd = h.dspark(backend, gguf, tmp_path, gpu_ids = [1, 2], extra_args = ["--device-draft", "CUDA2"])+ margins = h.margins(cmd)+ assert len(margins) == 2 and max(margins) >= FLOOR+ assert seen["fit_sized"] is True+++def test_the_start_log_shows_the_argv_spawned(tmp_path, monkeypatch):+ """The per-card reserve is spelled at spawn, so the logged command must be that one."""+ backend, gguf = h.cards(tmp_path)+ messages = h.info_messages(monkeypatch)+ spelled = h.last(h.dspark(backend, gguf, tmp_path), "--fit-target")+ assert "," in str(spelled)+ starts = [m for m in messages if m.startswith("Starting llama-server:")]+ assert starts and f"--fit-target {spelled}" in starts[-1]+++def test_auto_drops_a_drafter_pinned_across_both_target_cards(tmp_path):+ backend, gguf, path = h.placement._tight_vram_backend(tmp_path, drafter_gb = 12.0)+ backend._get_gpu_memory = lambda _binary = None, **_kw: [(0, 12288, 12288), (1, 12288, 12288)]+ backend._get_gpu_free_memory = lambda _binary = None, **_kw: [(0, 12288), (1, 12288)]+ backend._select_gpus = lambda *a, **k: ([0, 1], False)+ backend._select_gpus_split_aware = lambda *a, **k: ([0, 1], False)+ backend._arch_gate_survivors = lambda binary = None: []+ cmd = h.placement._launch_auto_8k(+ backend, gguf, dspark_draft_path = str(path), extra_args = ["--device-draft", "CUDA0,CUDA1"]+ )["cmd"]+ assert "--model-draft" not in cmd+ assert backend.spec_fallback_reason == "drafter_no_vram"+++# Host-resident draft KV (-nkvo) leaves a card's reserve only on positive evidence that+# the drafter's card is discrete; host memory is an APU's / iGPU's VRAM.++_DGPU = ["CUDA0", "CUDA1"]+_VULKAN = ["Vulkan0", "Vulkan1"]+++def test_a_discrete_card_picked_beside_an_apu_leaves_host_kv_out(tmp_path, monkeypatch):+ delta = h.host_kv_delta(tmp_path, monkeypatch, memory = TWO, listed = _DGPU, apu = 0, gpu_ids = [1])+ assert delta >= 760++[email protected]("device, shared", [("Vulkan0", True), ("Vulkan1", False)])+def test_a_vulkan_igpu_picked_by_device_keeps_host_kv(tmp_path, monkeypatch, device, shared):+ """Vulkan iGPU 0 (reports total 0) plus dGPU 1."""+ delta = h.host_kv_delta(+ tmp_path,+ monkeypatch,+ memory = _VK_IGPU,+ vulkan = True,+ listed = _VULKAN,+ extras = ["--device", device],+ **MANUAL_AUTO,+ )+ assert delta == 0 if shared else delta >= 760++[email protected]("device, discrete", [("CUDA1", True), ("ROCm1", True), ("CUDA0", False)])+def test_a_named_card_beside_an_apu_decides_host_kv(tmp_path, monkeypatch, device, discrete):+ delta = h.host_kv_delta(+ tmp_path,+ monkeypatch,+ memory = TWO,+ listed = [f"{device[:-1]}0", f"{device[:-1]}1"],+ apu = 0,+ extras = ["--device", device],+ **MANUAL_AUTO,+ )+ assert delta >= 760 if discrete else delta == 0+++ "listed",+ [_VULKAN, None],+ ids = [+ # Vulkan Auto drops iGPU 0 before the shared-card set is built: unknown, not discrete.+ "test_a_vulkan_igpu_auto_filtered_out_keeps_host_kv",+ # No --list-devices answer: the planned discrete card says nothing about the child's.+ "test_an_override_the_build_does_not_list_keeps_host_kv",+ ],+)+def test_an_unproven_vulkan_device_keeps_host_kv(tmp_path, monkeypatch, listed):+ delta = h.host_kv_delta(+ tmp_path,+ monkeypatch,+ memory = _VK_IGPU,+ vulkan = True,+ listed = listed,+ apu = None,+ extras = ["--device", "Vulkan0"],+ )+ assert delta == 0+++def test_a_one_card_draft_pin_on_the_discrete_card_leaves_host_kv_out(tmp_path, monkeypatch):+ delta = h.host_kv_delta(+ tmp_path, monkeypatch, memory = TWO, listed = _DGPU, apu = 0, extras = ["--device-draft", "CUDA1"]+ )+ assert delta >= 760++[email protected]("devices, left_out", [("CUDA1,CUDA0", True), ("CUDA0,CUDA1", False)])+def test_split_mode_none_narrows_to_the_main_card(tmp_path, monkeypatch, devices, left_out):+ """--split-mode none keeps devices[main_gpu], 0 by default: the first listed card."""+ delta = h.host_kv_delta(+ tmp_path,+ monkeypatch,+ memory = TWO,+ listed = _DGPU,+ apu = 0,+ extras = ["--device", devices, "--split-mode", "none"],+ )+ assert delta >= 760 if left_out else delta == 0+++ "order, left_out",+ [+ # CUDA's default order: the child's CUDA1 need not be nvidia-smi's card 1.+ ("FASTEST_FIRST", False),+ ("PCI_BUS_ID", True),+ ],+)+def test_a_mixed_host_narrows_only_through_a_proven_child_order(+ tmp_path, monkeypatch, _pci_ids, order, left_out+):+ monkeypatch.setenv("CUDA_DEVICE_ORDER", order)+ delta = h.host_kv_delta(+ tmp_path, monkeypatch, memory = TWO, listed = _DGPU, apu = 0, extras = ["--device", "CUDA1"]+ )+ assert delta >= 760 if left_out else delta == 0+++def test_a_uuid_mask_is_not_read_as_physical_ids(tmp_path, monkeypatch, _pci_ids):+ """CUDA_VISIBLE_DEVICES=GPU-<dgpu>,GPU-<apu>: the child's CUDA1 is the APU, which+ _physical_ordinals (numeric masks only) would leave as card 1, the discrete one."""+ monkeypatch.setenv("CUDA_VISIBLE_DEVICES", "GPU-11111111,GPU-00000000")+ delta = h.host_kv_delta(+ tmp_path, monkeypatch, memory = TWO, listed = _DGPU, apu = 0, extras = ["--device", "CUDA1"]+ )+ assert delta == 0+++def test_another_backends_device_name_is_not_read_as_cuda(tmp_path, monkeypatch, _pci_ids):+ """A CUDA + Vulkan build lists Vulkan1 too, but it is not CUDA card 1."""+ delta = h.host_kv_delta(+ tmp_path,+ monkeypatch,+ memory = TWO,+ listed = [*_DGPU, *_VULKAN],+ apu = 0,+ extras = ["--device", "Vulkan1"],+ )+ assert delta == 0+++def test_an_all_discrete_host_needs_no_ordinal_mapping(tmp_path, monkeypatch, _pci_ids):+ """Control: no shared card anywhere, even under an order the child does not share."""+ monkeypatch.setenv("CUDA_DEVICE_ORDER", "FASTEST_FIRST")+ delta = h.host_kv_delta(+ tmp_path, monkeypatch, memory = TWO, listed = _DGPU, apu = None, extras = ["--device", "CUDA1"]+ )+ assert delta >= 760diff --git a/studio/setup.sh b/studio/setup.shindex 0368157f831..2a1d601436e 100755--- a/studio/setup.sh+++ b/studio/setup.sh@@ -5194,6 +5194,11 @@ elseif [ "$BUILD_OK" = true ]; thenrun_quiet_no_exit "build llama-quantize" cmake --build "$_BUILD_TMP/build" --config Release --target llama-quantize -j"$NCPU" || true+ # Best-effort on macOS: the Metal memory probe the prebuilt bundles ship too.+ # Without it the Metal fit keeps a big model's input embeddings charged.+ if [ "$(uname -s 2>/dev/null)" = "Darwin" ]; then+ run_quiet_no_exit "build llama-fit-params" cmake --build "$_BUILD_TMP/build" --config Release --target llama-fit-params -j"$NCPU" || true+ fi# Best-effort: the target exists only on llama.cpp trees carrying the diffusion example.run_quiet_no_exit "build diffusion visual server" cmake --build "$_BUILD_TMP/build" --config Release --target llama-diffusion-gemma-visual-server -j"$NCPU" || truefi