Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

Studio: Auto drops serving slots before the MTP drafter (#13455)

Daniel Hanb93d4d1581bc
diff --git a/studio/backend/core/inference/llama_cpp.py b/studio/backend/core/inference/llama_cpp.py
index 02fbcee7212..adbc553523f 100644
--- a/studio/backend/core/inference/llama_cpp.py
+++ b/studio/backend/core/inference/llama_cpp.py
@@ -692,6 +692,8 @@ class GgufLoadIntent:
gpu_ids_are_vulkan_ordinals: Optional[bool] = None
n_threads: Optional[int] = None
n_parallel: int = 1
+ # Slots left at the default: Auto speculation may lower them to keep a drafter (#8872).
+ n_parallel_from_default: bool = False
# none follows the llama.cpp defaults (2048 / 512)
n_batch: Optional[int] = None
n_ubatch: Optional[int] = None
@@ -4579,6 +4581,8 @@ def _auto_mode_drops_mtp(
_MLA_MTP_FAST_ARCHS = frozenset({"glm5next", "glm5-next"})
# Smallest context Auto shrinks to so a fast MLA head still launches.
_FAST_MTP_MIN_CTX = 65536
+# Max share of the target-only Auto context a drafter may take at one slot.
+_MTP_SLOT_CTX_TRIM = 0.10
def _arch_has_fast_mla_mtp(architecture: Optional[str]) -> bool:
@@ -6843,6 +6847,23 @@ def _emitted_n_batch(n_batch: Optional[int], n_parallel: int) -> Optional[int]:
return max(int(n_batch), max(2, int(n_parallel or 1)))
+def _with_parallel_slots(
+ cmd: Sequence[str], slots: int, *, n_batch: Optional[int], supports_kv_unified: bool
+) -> list[str]:
+ """``cmd`` relaunched at ``slots``: Unsloth's own --parallel, its batch floor, and the
+ --kv-unified a multi-slot launch adds. Each is emitted before the extras, so the
+ first occurrence is the one to rewrite and a hand-typed flag still last-wins."""
+ out = [str(a) for a in cmd]
+ at = out.index("--parallel")
+ out[at + 1] = str(slots)
+ if n_batch is not None and "--batch-size" in out:
+ b = out.index("--batch-size")
+ out[b + 1] = str(_emitted_n_batch(n_batch, slots))
+ if slots > 1 and supports_kv_unified and "--kv-unified" not in out and "-kvu" not in out:
+ out[at + 2 : at + 2] = ["--kv-unified"]
+ return out
+
+
def _extra_args_split_mode(
extra_args: Optional[Iterable[str]], env: Optional[Mapping[str, str]] = None
) -> Optional[str]:
@@ -8277,6 +8298,7 @@ class LlamaCppBackend:
self._effective_parallel_slots: int = 1
# --parallel the last load asked for, before any fit-time reduction.
self._requested_n_parallel: int = 1
+ self._slots_cut_for_drafter: bool = False
# Budget the running child was sized against; None before any load, so the
# settings route reports a stale budget instead of nagging on every save.
self._vram_fraction_launched: Optional[float] = None
@@ -8864,6 +8886,7 @@ class LlamaCppBackend:
self._effective_parallel_slots = 1
# Cleared with the effective count so a stale value can't skew the dedupe.
self._requested_n_parallel = 1
+ self._slots_cut_for_drafter = False
self._requested_n_batch = None
self._requested_n_ubatch = None
self._requested_load_mode = None
@@ -9391,6 +9414,13 @@ class LlamaCppBackend:
return False
if not self._is_diffusion and self._requested_n_parallel != max(1, int(intent.n_parallel)):
return False
+ # The same count chosen rather than defaulted is one Auto may not cut.
+ if (
+ not self._is_diffusion
+ and self._slots_cut_for_drafter
+ and not intent.n_parallel_from_default
+ ):
+ return False
if not self._is_diffusion and (
self._requested_n_batch != intent.n_batch or self._requested_n_ubatch != intent.n_ubatch
):
@@ -26127,6 +26157,8 @@ class LlamaCppBackend:
# Auto dropped the drafter because only the target fit. Bound before the
# try: the launch below reads it either way.
_spec_dropped_no_vram = False
+ _mtp_slots_asked: Optional[int] = None
+ _mtp_slots_target_ctx: Optional[int] = None
# The projector runs on the CPU and so allocates no VRAM:
# --no-mmproj-offload clears mmproj_use_gpu and clip.cpp gates its whole
# GPU backend on that, leaving no partial offload. Only the paravirtual
@@ -26768,6 +26800,16 @@ class LlamaCppBackend:
- _cc_bytes(ctx, 1, slots, cache_type),
)
+ def _layer_compute_buffer(slots: int) -> int:
+ cb = self._estimate_compute_buffer_bytes(
+ n_ubatch = _ubatch_for_slots(slots),
+ n_parallel = slots,
+ per_device_tensor = False,
+ )
+ return (
+ cb if cb > 0 else self._TENSOR_PARALLEL_BUFFER_RESERVE_MIB * 1024 * 1024
+ )
+
# Layer-split compute buffer (one lump; tensor mode reserves it
# per device in _plan_tensor_parallel). Context-independent, so
# fold it into the model footprint for the branches below. Falls
@@ -27198,7 +27240,11 @@ class LlamaCppBackend:
else 0.0
)
- def _probe_base(drafter: bool, n: int) -> int:
+ def _probe_base(
+ drafter: bool,
+ n: int,
+ slots: int = 0,
+ ) -> int:
# model_size_fit's terms, before it is built below.
_soft = self._CUDA_CONTEXT_RESERVE_BYTES
if effective_is_vision and mmproj_size > 0:
@@ -27207,7 +27253,11 @@ class LlamaCppBackend:
_soft += self._MTP_DRAFT_COMPUTE_BYTES
return (
model_size
- + _compute_buffer_pipeline
+ + (
+ _layer_compute_buffer(slots)
+ if slots
+ else _compute_buffer_pipeline
+ )
+ _soft
+ max(0, n - 1) * _pipeline_overhead_bytes
)
@@ -27257,6 +27307,9 @@ class LlamaCppBackend:
_both_fit_somewhere = False
_probe_ctx = 0
_probe_need = _probe_have = 0.0
+ # (GPU count, target-only ctx, budget with drafter) per drafter-ranked subset.
+ _target_placements: list[tuple[int, int, float]] = []
+ _drafter_order = _probe_orders[-1]
for _probe_ranked in _probe_orders:
if _both_fit_somewhere:
break
@@ -27340,6 +27393,8 @@ class LlamaCppBackend:
_foot_w,
_budget_w,
)
+ if _probe_ranked is _drafter_order:
+ _target_placements.append((_n, _ctx_wo, _budget_w))
if _foot_w <= _budget_w:
_both_fit_somewhere = True
break
@@ -27399,6 +27454,104 @@ class LlamaCppBackend:
_budget_w / 1024,
)
break
+ # Fewer slots before no drafter. The walk takes the first subset holding the
+ # drafter, so it must keep its target-only ctx (less the trim at one slot).
+ # An extras --parallel wins at launch, so it is kept; the env twin is scrubbed.
+ if (
+ _target_fits_somewhere
+ and not _both_fit_somewhere
+ and n_parallel > 1
+ and intent.n_parallel_from_default
+ and _extra_args_n_parallel(extra_args, env = {}) is None
+ ):
+
+ def _mtp_foot_mib(n: int, ctx: int, s: int) -> float:
+ ub = _ubatch_for_slots(s)
+ return (
+ _probe_base(True, n, s)
+ + self._estimate_kv_cache_bytes(
+ ctx,
+ cache_type_kv,
+ n_parallel = s,
+ swa_full = swa_full,
+ kv_unified = planned_kv_unified,
+ n_ubatch = ub,
+ flash_attn = planned_flash_attn,
+ )
+ + _cc_bytes(ctx, n, s)
+ + _mtp_bytes(ctx, s, ub)
+ ) / (1024 * 1024)
+
+ def _ctx_with_drafter(n: int, ctx: int, budget: float, s: int) -> int:
+ if explicit_ctx:
+ return ctx if _mtp_foot_mib(n, ctx, s) <= budget else 0
+ ub = _ubatch_for_slots(s)
+ fitted = self._fit_context_to_vram(
+ ctx,
+ budget,
+ _probe_base(True, n, s),
+ cache_type_kv,
+ swa_full = swa_full,
+ n_parallel = s,
+ kv_unified = planned_kv_unified,
+ n_ubatch = ub,
+ flash_attn = planned_flash_attn,
+ mtp_engaged = True,
+ mtp_overhead_fn = (
+ functools.partial(mtp_overhead_fn, _np = s, _n_ubatch = ub)
+ if mtp_overhead_fn is not None
+ else None
+ ),
+ compute_ctx_bytes_fn = lambda c: _cc_bytes(c, n, s),
+ budget_frac = 1.0,
+ pooled = True,
+ total_mib = None,
+ ctx_checkpoints = _placement_ctx_checkpoints,
+ )
+ return (
+ fitted
+ if fitted > 0 and _mtp_foot_mib(n, fitted, s) <= budget
+ else 0
+ )
+
+ _mtp_slots = _mtp_slots_ctx = 0
+ for _s in range(n_parallel - 1, 0, -1):
+ _keep = 1 - (
+ _MTP_SLOT_CTX_TRIM if _s == 1 and not explicit_ctx else 0
+ )
+ for _n, _ctx_c, _budget_c in _target_placements:
+ _c = _ctx_with_drafter(_n, _ctx_c, _budget_c, _s)
+ if _c <= 0:
+ continue
+ if _c >= _ctx_c * _keep:
+ _mtp_slots, _mtp_slots_ctx = _s, _c
+ _mtp_slots_target_ctx = _ctx_c
+ break
+ if _mtp_slots:
+ break
+ if _mtp_slots:
+ logger.info(
+ "Auto: serving slots reduced %d -> %d so the drafter fits "
+ "in VRAM at context %d (target alone: %d). Set Parallel "
+ "Slots to keep %d without the drafter.",
+ n_parallel,
+ _mtp_slots,
+ _mtp_slots_ctx,
+ _probe_ctx,
+ n_parallel,
+ )
+ _mtp_slots_asked = n_parallel
+ n_parallel = _mtp_slots # allow-slot-clamp: the drafter fits in VRAM only at fewer slots
+ _effective_ubatch = _ubatch_for_slots(n_parallel)
+ _compute_buffer_pipeline = _layer_compute_buffer(n_parallel)
+ if mtp_overhead_fn is not None:
+ # Its defaults were bound at the requested count.
+ mtp_overhead_fn = functools.partial(
+ mtp_overhead_fn,
+ _np = n_parallel,
+ _n_ubatch = _effective_ubatch,
+ )
+ _both_fit_somewhere = True
if _target_fits_somewhere and not _both_fit_somewhere:
_spec_dropped_no_vram = True
_mtp_will_engage = False
@@ -28926,6 +29079,16 @@ class LlamaCppBackend:
_vram_fit_ctx = None
tp_tensor_split = None
effective_ctx = requested_ctx # fall back to original
+ # Nor is a slot cut for the drafter: requested count, no drafter, as before.
+ if _mtp_slots_asked is not None:
+ n_parallel = _mtp_slots_asked
+ _mtp_slots_asked = _mtp_slots_target_ctx = None
+ _effective_ubatch = _ubatch_for_slots(n_parallel)
+ _compute_buffer_pipeline = _layer_compute_buffer(n_parallel)
+ _spec_dropped_no_vram = True
+ _mtp_will_engage = False
+ mtp_overhead_fn = None
+ _mtp_kv_unsized = False
# A hand-set context unified memory cannot hold. Raised here so the handler
# above cannot turn it back into the launch it refuses, and ahead of the
@@ -33173,12 +33336,88 @@ class LlamaCppBackend:
strip_split_mode = False,
)
fallback_cmd = cmd[:_spec_at] + ["--spec-default"] + _fb_tail
+ # Restore the cut slots; if they fail (unified KV refused), retry as cut.
+ _fb_cut = (
+ (
+ fallback_cmd,
+ n_parallel,
+ effective_ctx,
+ (self._effective_context_length, self._max_context_length),
+ _auto_ctx_checkpoints,
+ _auto_ckpt_emitted,
+ )
+ if _mtp_slots_asked is not None
+ else None
+ )
+ if _fb_cut is not None:
+ n_parallel = _mtp_slots_asked
+ fallback_cmd = _with_parallel_slots(
+ fallback_cmd,
+ n_parallel,
+ n_batch = n_batch,
+ supports_kv_unified = bool(server_caps.get("supports_kv_unified")),
+ )
+ # The cut may have bought context; the restored count was priced at this.
+ if _mtp_slots_target_ctx and "-c" in fallback_cmd:
+ effective_ctx = _mtp_slots_target_ctx
+ fallback_cmd[fallback_cmd.index("-c") + 1] = str(effective_ctx)
+ # The ceiling was priced at the cut count; this is the one proven here.
+ self._effective_context_length = self._max_context_length = (
+ effective_ctx
+ )
+ # The checkpoint cap is per slot; same re-decide as the single-seq rung.
+ if _auto_ckpt_emitted:
+ fallback_cmd = self._without_flag_pairs(
+ fallback_cmd, _auto_ckpt_emitted
+ )
+ _auto_ctx_checkpoints = _decide_auto_ctx_checkpoints()
+ _auto_ckpt_emitted = _emit_auto_ctx_checkpoints(fallback_cmd)
+ kv_cache_unified = _kv_unified_from_args(fallback_cmd)
+ # The plan other loads place against was priced at the cut count.
+ _fb_cut_plan = dict(self._pending_plan_mib)
+ if self._pending_plan_mib and self._can_estimate_kv():
+ kv_cache_bytes = self._estimate_kv_cache_bytes(
+ effective_ctx,
+ cache_type_kv,
+ n_parallel = n_parallel,
+ swa_full = swa_full,
+ kv_unified = bool(kv_cache_unified),
+ n_ubatch = _ubatch_for_slots(n_parallel),
+ flash_attn = planned_flash_attn,
+ )
+ self._pending_plan_mib = _gpu_plan_mib(
+ gguf_size + mmproj_size + kv_cache_bytes,
+ list(self._pending_plan_mib),
+ gpus,
+ )
_unified_withdraw_if_unneeded(
fallback_cmd,
mtp_engages = False,
why = "The retry without speculative decoding",
)
healthy = _spawn_and_wait(fallback_cmd, label = "-retry")
+ if not healthy and _fb_cut is not None and not _load_cancelled():
+ logger.warning(
+ "The retry without speculative decoding failed at %d slots; "
+ "retrying at the %d the drafter launch ran.",
+ n_parallel,
+ _fb_cut[1],
+ )
+ self._kill_process()
+ (
+ fallback_cmd,
+ _fb_slots,
+ effective_ctx,
+ (self._effective_context_length, self._max_context_length),
+ _auto_ctx_checkpoints,
+ _auto_ckpt_emitted,
+ ) = _fb_cut
+ self._pending_plan_mib = _fb_cut_plan
+ n_parallel = (
+ _fb_slots # allow-slot-clamp: the restored count failed to start
+ )
+ kv_cache_unified = _kv_unified_from_args(fallback_cmd)
+ healthy = _spawn_and_wait(fallback_cmd, label = "-retry-cut")
if healthy:
self._speculative_type = "default"
_mtp_active_for_launched_server = False
@@ -33583,6 +33822,9 @@ class LlamaCppBackend:
self._requested_n_ctx = int(effective_ctx if _replayed_ctx_refit else n_ctx)
# Local n_parallel may have been reduced above; the snapshot has the ask.
self._requested_n_parallel = max(1, int(intent.n_parallel))
+ self._slots_cut_for_drafter = (
+ _mtp_slots_asked is not None and n_parallel < _mtp_slots_asked
+ )
# Commit with the rest of the known-good state: only a launch that got
# this far replaced the runtime, so a rejected preflight leaves the old
# marker intact, and the diffusion runner returns well before it without
diff --git a/studio/backend/routes/inference.py b/studio/backend/routes/inference.py
index 94502953dd4..cb95436faa9 100644
--- a/studio/backend/routes/inference.py
+++ b/studio/backend/routes/inference.py
@@ -8607,6 +8607,7 @@ def _gguf_request_intent(
extra_args = extra_args,
gpu_ids = gpu_ids,
n_parallel = n_parallel,
+ n_parallel_from_default = getattr(request, "n_parallel", None) is None,
)
settings.update(changes)
return replace(source, **settings)
diff --git a/studio/backend/tests/test_batch_sizes_per_load.py b/studio/backend/tests/test_batch_sizes_per_load.py
index 4d61418378d..a6e7bb60a14 100644
--- a/studio/backend/tests/test_batch_sizes_per_load.py
+++ b/studio/backend/tests/test_batch_sizes_per_load.py
@@ -655,8 +655,11 @@ def test_the_recorded_micro_batch_is_derived_from_the_slots_that_launched():
and node.func.id == "_ubatch_for_slots"
]
# sizing pass, embedding slot clamp, candidate split pricing, fit-time reduction, the
- # MoE expert-spill raise, then the post-launch record
- assert len(calls) == 6, f"expected six re-derivations, found {len(calls)}"
+ # MoE expert-spill raise, the post-launch record, the drafter probe's four
+ # (per-slot compute buffer, candidate footprint, one-slot trim, slot commit; #8872)
+ # the slot restore when GPU selection abandons that cut, and the plan repriced when a
+ # drafterless retry restores it
+ assert len(calls) == 12, f"expected twelve re-derivations, found {len(calls)}"
# the record must not reuse the sizing pass's value
compact = "".join(src.split())
assert "self._n_ubatch=max(0,int(self._DEFAULT_N_UBATCHif_launched_ubatchisNone" in compact
diff --git a/studio/backend/tests/test_ctx_checkpoint_host_budget.py b/studio/backend/tests/test_ctx_checkpoint_host_budget.py
index 77d3e3b918f..ac4c7aa29b4 100644
--- a/studio/backend/tests/test_ctx_checkpoint_host_budget.py
+++ b/studio/backend/tests/test_ctx_checkpoint_host_budget.py
@@ -477,7 +477,8 @@ class TestCheckpointsNeverReachAVramFigure:
source = inspect.getsource(LlamaCppBackend.load_model)
assert "0 if self._rollback_state_bytes(1) > 0 else _requested_ctx_checkpoints" in source
assert source.count("ctx_checkpoints = _fit_ctx_checkpoints,") == 1
- assert source.count("ctx_checkpoints = _placement_ctx_checkpoints,") == 6
+ # +1: the drafter probe's one-slot fit (#8872)
+ assert source.count("ctx_checkpoints = _placement_ctx_checkpoints,") == 7
assert source.count("_kv_bytes(effective_ctx, _effective_ctx_checkpoints)") == 1
def test_the_fit_is_unmoved_by_the_new_term(self):
diff --git a/studio/backend/tests/test_llama_cpp_placement.py b/studio/backend/tests/test_llama_cpp_placement.py
index 547d17e3f5a..e6b1a29cf16 100644
--- a/studio/backend/tests/test_llama_cpp_placement.py
+++ b/studio/backend/tests/test_llama_cpp_placement.py
@@ -5,6 +5,7 @@
from __future__ import annotations
+import dataclasses
import os
import struct
import subprocess
@@ -1109,6 +1110,369 @@ def test_auto_drops_the_drafter_when_only_the_target_fits(tmp_path):
assert backend.mtp_draft_path == str(sidecar)
+def _per_slot_mtp_backend(tmp_path: Path, *, gb_per_slot: float):
+ """The tight card with an embedded head whose reserve is all per-slot state."""
+ gb = 1024**3
+ backend, gguf, _sidecar = _tight_vram_backend(tmp_path, drafter_gb = 0.0)
+
+ def read_metadata(_path):
+ backend._nextn_predict_layers = 1
+ backend._context_length = 8192
+
+ backend._read_gguf_metadata = read_metadata
+ backend._estimate_mtp_overhead_bytes = (
+ lambda n_ctx, *args, _np = None, n_parallel = None, **kwargs: int(
+ gb_per_slot * gb * int(_np or n_parallel or 1)
+ )
+ )
+ backend.probe_server_capabilities = lambda _binary = None: {
+ "mtp_token": "draft-mtp",
+ "supports_ngram_mod": True,
+ "spec_draft_n_max_flag": "--spec-draft-n-max",
+ "supports_kv_unified": True,
+ }
+ return backend, gguf
+
+
+def _launch_default_slots(
+ backend,
+ gguf,
+ *,
+ n_parallel = 4,
+ **kwargs,
+):
+ return _launch_auto_8k(
+ backend, gguf, n_ctx = 0, n_parallel = n_parallel, n_parallel_from_default = True, **kwargs
+ )
+
+
+def _launched(result, flag: str) -> Optional[str]:
+ cmd = result["cmd"]
+ return cmd[cmd.index(flag) + 1] if flag in cmd else None
+
+
[email protected]("gb_per_slot, slots", [(1.5, "3"), (2.0, "2"), (5.0, "1")])
+def test_auto_drops_serving_slots_before_the_drafter(tmp_path, gb_per_slot, slots):
+ """#8872: the largest slot count that holds the drafter at the same context."""
+ backend, gguf = _per_slot_mtp_backend(tmp_path, gb_per_slot = gb_per_slot)
+
+ result = _launch_default_slots(backend, gguf)
+
+ assert _launched(result, "--spec-type") == "draft-mtp"
+ assert _launched(result, "--parallel") == slots
+ assert _launched(result, "-c") == "8192"
+ assert backend.spec_fallback_reason is None
+
+
[email protected]("n_parallel", [4, 1])
+def test_auto_drops_the_drafter_when_one_slot_cannot_hold_it(tmp_path, n_parallel):
+ backend, gguf = _per_slot_mtp_backend(tmp_path, gb_per_slot = 7.0)
+
+ result = _launch_default_slots(backend, gguf, n_parallel = n_parallel)
+
+ assert _launched(result, "--spec-type") != "draft-mtp"
+ assert _launched(result, "--parallel") == str(n_parallel)
+ assert backend.spec_fallback_reason == "drafter_no_vram"
+
+
+def test_a_chosen_slot_count_is_kept_and_the_drafter_dropped(tmp_path):
+ backend, gguf = _per_slot_mtp_backend(tmp_path, gb_per_slot = 2.0)
+
+ result = _launch_auto_8k(backend, gguf, n_ctx = 0, n_parallel = 4)
+
+ assert _launched(result, "--spec-type") != "draft-mtp"
+ assert _launched(result, "--parallel") == "4"
+ assert backend.spec_fallback_reason == "drafter_no_vram"
+
+
+def test_a_wider_subset_holding_the_drafter_does_not_justify_the_cut(tmp_path):
+ """Fewer slots fit the drafter at full context only across both cards, but the walk
+ takes the first card that holds it at any context, here a few hundred tokens."""
+ gb, mib = 1024**3, 1024 * 1024
+ backend, gguf = _backend(
+ tmp_path, vulkan = False, memory = [(0, 24_576, 24_576), (1, 24_576, 24_576)]
+ )
+
+ def read_metadata(_path):
+ backend._nextn_predict_layers = 1
+ backend._context_length = 8192
+
+ def fit(
+ requested,
+ budget_mib,
+ base,
+ *args,
+ mtp_engaged = False,
+ mtp_overhead_fn = None,
+ **kw,
+ ):
+ ctx = requested
+ while ctx > 0:
+ drafter = mtp_overhead_fn(ctx) if mtp_engaged and mtp_overhead_fn else 0
+ if (base + gb + drafter) / mib <= budget_mib:
+ return ctx
+ ctx -= 256
+ return 0
+
+ backend._read_gguf_metadata = read_metadata
+ backend._get_gguf_size_bytes = lambda _path: 16 * gb
+ backend._can_estimate_kv = lambda: True
+ backend._estimate_kv_cache_bytes = lambda *args, **kwargs: gb
+ backend._compute_buffer_ctx_bytes = lambda *args, **kwargs: 0
+ backend._estimate_compute_buffer_bytes = lambda **kwargs: 1
+ backend._mtp_draft_kv_bytes = lambda *args, **kwargs: 0
+ backend._estimate_mtp_overhead_bytes = (
+ lambda n_ctx, *args, _np = None, n_parallel = None, **kwargs: int(
+ 1.5 * gb * int(_np or n_parallel or 1) + 24 * gb * n_ctx / 8192
+ )
+ )
+ backend._fit_context_to_vram = fit
+ backend.probe_server_capabilities = lambda _binary = None: {
+ "mtp_token": "draft-mtp",
+ "supports_ngram_mod": True,
+ "spec_draft_n_max_flag": "--spec-draft-n-max",
+ "supports_kv_unified": True,
+ }
+
+ result = _launch_default_slots(backend, gguf)
+
+ assert _launched(result, "--spec-type") != "draft-mtp"
+ assert _launched(result, "--parallel") == "4"
+ assert _launched(result, "-c") == "8192"
+ assert backend.spec_fallback_reason == "drafter_no_vram"
+
+
[email protected]("extras", [["--parallel", "4"], ["-np=4"]])
+def test_a_slot_count_in_the_extras_is_kept(tmp_path, extras):
+ backend, gguf = _per_slot_mtp_backend(tmp_path, gb_per_slot = 2.0)
+
+ result = _launch_default_slots(backend, gguf, extra_args = extras)
+
+ assert _launched(result, "--spec-type") != "draft-mtp"
+ assert _launched(result, "--parallel") == "4"
+ assert backend.spec_fallback_reason == "drafter_no_vram"
+
+
+def _launch_with_failing_drafter(
+ backend,
+ gguf,
+ *,
+ refuses_unified_kv = False,
+ n_ctx = 0,
+):
+ cmds = []
+
+ def fake_popen(cmd, **kwargs):
+ if not cmd or str(cmd[0]) != "/fake/llama-server":
+ return _REAL_POPEN(cmd, **kwargs)
+ cmds.append(list(cmd))
+ return type(
+ "Process",
+ (),
+ {
+ "pid": 123,
+ "stdout": (),
+ "poll": lambda self: None,
+ "terminate": lambda self: None,
+ "wait": lambda self, timeout = None: 0,
+ "kill": lambda self: None,
+ },
+ )()
+
+ def fake_health(timeout = None, **_kw):
+ backend._stdout_lines = []
+ if "draft-mtp" in cmds[-1]:
+ return False
+ return not (refuses_unified_kv and "--kv-unified" in cmds[-1])
+
+ backend._wait_for_health = fake_health
+ with patch.object(subprocess, "Popen", side_effect = fake_popen):
+ assert backend.load_model(
+ GgufLoadIntent(
+ gguf_path = str(gguf),
+ model_identifier = "test",
+ n_ctx = n_ctx,
+ n_parallel = 4,
+ n_parallel_from_default = True,
+ speculative_type = "auto",
+ )
+ )
+ return cmds
+
+
+def test_a_drafterless_retry_that_fails_at_the_restored_slots_runs_as_cut(tmp_path):
+ backend, gguf = _per_slot_mtp_backend(tmp_path, gb_per_slot = 5.0)
+
+ cmds = _launch_with_failing_drafter(backend, gguf, refuses_unified_kv = True)
+
+ assert [c[c.index("--parallel") + 1] for c in cmds] == ["1", "4", "1"]
+ assert "--kv-unified" not in cmds[-1] and "draft-mtp" not in cmds[-1]
+ assert backend.effective_parallel_slots == 1
+
+
+ "refuses_unified_kv, ctxs", [(False, ["7424", "8192"]), (True, ["7424", "8192", "7424"])]
+)
+def test_the_restored_slots_relaunch_at_their_own_context(tmp_path, refuses_unified_kv, ctxs):
+ gb = 1024**3
+ backend, gguf = _per_slot_mtp_backend(tmp_path, gb_per_slot = 0.0)
+ backend._estimate_mtp_overhead_bytes = lambda n_ctx, *args, **kwargs: int(
+ 6.2 * gb * n_ctx / 8192
+ )
+ backend._fit_context_to_vram = lambda requested, *args, mtp_engaged = False, **kwargs: (
+ min(requested, 7424) if mtp_engaged else requested
+ )
+
+ cmds = _launch_with_failing_drafter(backend, gguf, refuses_unified_kv = refuses_unified_kv)
+
+ assert [c[c.index("-c") + 1] for c in cmds] == ctxs
+ assert backend._effective_context_length == int(ctxs[-1])
+ if not refuses_unified_kv:
+ assert backend._max_context_length == 8192
+
+
+def test_the_restored_slots_reprice_the_published_plan(tmp_path):
+ """Other loads place against this plan, so it is the restored count's, not the cut's."""
+ gb = 1024**3
+ backend, gguf = _per_slot_mtp_backend(tmp_path, gb_per_slot = 2.0)
+ backend._estimate_kv_cache_bytes = lambda *args, n_parallel = 1, **kwargs: (
+ gb // 4 * int(n_parallel or 1)
+ )
+
+ cmds = _launch_with_failing_drafter(backend, gguf)
+
+ assert [c[c.index("--parallel") + 1] for c in cmds] == ["2", "4"]
+ assert sum(backend._planned_vram_mib.values()) == (16 * gb + gb) // (1024 * 1024)
+
+
+def test_an_explicit_context_restores_its_ceiling(tmp_path):
+ backend, gguf = _per_slot_mtp_backend(tmp_path, gb_per_slot = 2.0)
+
+ cmds = _launch_with_failing_drafter(backend, gguf, n_ctx = 8192)
+
+ assert [c[c.index("--parallel") + 1] for c in cmds] == ["2", "4"]
+ assert [c[c.index("-c") + 1] for c in cmds] == ["8192", "8192"]
+ assert backend._max_context_length == backend._effective_context_length == 8192
+
+
+def test_an_env_slot_count_does_not_block_the_cut(tmp_path, monkeypatch):
+ """The launch scrubs LLAMA_ARG_N_PARALLEL, so it never reaches the server."""
+ monkeypatch.setenv("LLAMA_ARG_N_PARALLEL", "4")
+ backend, gguf = _per_slot_mtp_backend(tmp_path, gb_per_slot = 2.0)
+
+ result = _launch_default_slots(backend, gguf)
+
+ assert _launched(result, "--spec-type") == "draft-mtp"
+ assert _launched(result, "--parallel") == "2"
+
+
+def test_a_drafter_that_fails_to_start_gives_its_slots_back(tmp_path):
+ backend, gguf = _per_slot_mtp_backend(tmp_path, gb_per_slot = 2.0)
+ cmds = []
+
+ def fake_popen(cmd, **kwargs):
+ if not cmd or str(cmd[0]) != "/fake/llama-server":
+ return _REAL_POPEN(cmd, **kwargs)
+ cmds.append(list(cmd))
+ return type(
+ "Process",
+ (),
+ {
+ "pid": 123,
+ "stdout": (),
+ "poll": lambda self: None,
+ "terminate": lambda self: None,
+ "wait": lambda self, timeout = None: 0,
+ "kill": lambda self: None,
+ },
+ )()
+
+ def fake_health(timeout = None, **_kw):
+ backend._stdout_lines = []
+ return "draft-mtp" not in cmds[-1]
+
+ backend._wait_for_health = fake_health
+ with patch.object(subprocess, "Popen", side_effect = fake_popen):
+ assert backend.load_model(
+ GgufLoadIntent(
+ gguf_path = str(gguf),
+ model_identifier = "test",
+ n_ctx = 0,
+ n_parallel = 4,
+ n_parallel_from_default = True,
+ speculative_type = "auto",
+ )
+ )
+
+ first, retry = cmds[0], cmds[-1]
+ assert first[first.index("--parallel") + 1] == "2"
+ assert "draft-mtp" not in retry and "--spec-default" in retry
+ assert retry[retry.index("--parallel") + 1] == "4"
+ assert "--kv-unified" in retry
+ assert backend.effective_parallel_slots == 4
+
+
+def test_a_placement_abandoned_after_the_cut_gives_the_slots_back(tmp_path):
+ backend, gguf = _per_slot_mtp_backend(tmp_path, gb_per_slot = 2.0)
+
+ def boom(*_args, **_kwargs):
+ raise RuntimeError("placement bug")
+
+ backend._fit_derived_load_mode = boom
+ result = _launch_default_slots(backend, gguf)
+
+ assert _launched(result, "--fit") == "on"
+ assert _launched(result, "--parallel") == "4"
+ assert _launched(result, "--spec-type") != "draft-mtp"
+ assert backend.spec_fallback_reason == "drafter_no_vram"
+
+
+def test_a_chosen_count_does_not_dedupe_against_slots_cut_for_the_drafter(tmp_path):
+ backend, gguf = _per_slot_mtp_backend(tmp_path, gb_per_slot = 2.0)
+ result = _launch_default_slots(backend, gguf)
+ assert _launched(result, "--parallel") == "2"
+
+ chosen = GgufLoadIntent(
+ gguf_path = str(gguf),
+ model_identifier = "test",
+ n_ctx = 0,
+ n_parallel = 4,
+ speculative_type = "auto",
+ )
+ default = dataclasses.replace(chosen, n_parallel_from_default = True)
+ assert backend._runtime_matches_intent(default, None)
+ assert not backend._runtime_matches_intent(chosen, None)
+ # A diffusion runtime that replaced it ignores slot counts altogether.
+ backend._is_diffusion = True
+ assert backend._runtime_matches_intent(chosen, None) == backend._runtime_matches_intent(
+ default, None
+ )
+
+
[email protected]("ctx_with_drafter, keeps", [(7424, True), (6144, False)])
+def test_one_slot_may_trim_a_little_auto_context(tmp_path, ctx_with_drafter, keeps):
+ """Draft KV grows with context, so one slot can still need a few percent of it."""
+ gb = 1024**3
+ backend, gguf = _per_slot_mtp_backend(tmp_path, gb_per_slot = 0.0)
+ backend._estimate_mtp_overhead_bytes = lambda n_ctx, *args, **kwargs: int(
+ 6.2 * gb * n_ctx / 8192
+ )
+ backend._fit_context_to_vram = lambda requested, *args, mtp_engaged = False, **kwargs: (
+ min(requested, ctx_with_drafter) if mtp_engaged else requested
+ )
+
+ result = _launch_default_slots(backend, gguf)
+
+ if keeps:
+ assert _launched(result, "--spec-type") == "draft-mtp"
+ assert _launched(result, "--parallel") == "1"
+ assert _launched(result, "-c") == str(ctx_with_drafter)
+ else:
+ assert _launched(result, "--spec-type") != "draft-mtp"
+ assert _launched(result, "--parallel") == "4"
+ assert _launched(result, "-c") == "8192"
+
+
def test_auto_keeps_a_drafter_that_fits(tmp_path):
"""The drop is scoped to the shortfall: with room for both, nothing changes."""
backend, gguf, sidecar = _tight_vram_backend(tmp_path, drafter_gb = 1.5)
diff --git a/studio/backend/tests/test_vram_budget_settings.py b/studio/backend/tests/test_vram_budget_settings.py
index 10a5d06b0d6..4a97573cb19 100644
--- a/studio/backend/tests/test_vram_budget_settings.py
+++ b/studio/backend/tests/test_vram_budget_settings.py
@@ -585,10 +585,11 @@ class TestFloorReserve:
assert lc._vram_usable_mib(12_000, None, 1.0, pooled = True) == pytest.approx(12_000)
def test_every_pooled_caller_says_so(self):
- # Five call sites hand _fit_context_to_vram an absolute pool budget; each
- # has to be marked, since the flag is what keeps them from double-paying.
+ # Six call sites hand _fit_context_to_vram an absolute pool budget (the sixth is
+ # the drafter probe's one-slot fit, #8872); each has to be marked, since the flag
+ # is what keeps them from double-paying.
compact = _shared_setup_1()
- assert compact.count("budget_frac=1.0,pooled=True,total_mib=None,") == 5
+ assert compact.count("budget_frac=1.0,pooled=True,total_mib=None,") == 6
class TestRetriesAndDedup: