Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
Studio: Auto drops serving slots before the MTP drafter (#13455)
Daniel Hanb93d4d1581bc
diff --git a/studio/backend/core/inference/llama_cpp.py b/studio/backend/core/inference/llama_cpp.pyindex 02fbcee7212..adbc553523f 100644--- a/studio/backend/core/inference/llama_cpp.py+++ b/studio/backend/core/inference/llama_cpp.py@@ -692,6 +692,8 @@ class GgufLoadIntent:gpu_ids_are_vulkan_ordinals: Optional[bool] = Nonen_threads: Optional[int] = Nonen_parallel: int = 1+ # Slots left at the default: Auto speculation may lower them to keep a drafter (#8872).+ n_parallel_from_default: bool = False# none follows the llama.cpp defaults (2048 / 512)n_batch: Optional[int] = Nonen_ubatch: Optional[int] = None@@ -4579,6 +4581,8 @@ def _auto_mode_drops_mtp(_MLA_MTP_FAST_ARCHS = frozenset({"glm5next", "glm5-next"})# Smallest context Auto shrinks to so a fast MLA head still launches._FAST_MTP_MIN_CTX = 65536+# Max share of the target-only Auto context a drafter may take at one slot.+_MTP_SLOT_CTX_TRIM = 0.10def _arch_has_fast_mla_mtp(architecture: Optional[str]) -> bool:@@ -6843,6 +6847,23 @@ def _emitted_n_batch(n_batch: Optional[int], n_parallel: int) -> Optional[int]:return max(int(n_batch), max(2, int(n_parallel or 1)))+def _with_parallel_slots(+ cmd: Sequence[str], slots: int, *, n_batch: Optional[int], supports_kv_unified: bool+) -> list[str]:+ """``cmd`` relaunched at ``slots``: Unsloth's own --parallel, its batch floor, and the+ --kv-unified a multi-slot launch adds. Each is emitted before the extras, so the+ first occurrence is the one to rewrite and a hand-typed flag still last-wins."""+ out = [str(a) for a in cmd]+ at = out.index("--parallel")+ out[at + 1] = str(slots)+ if n_batch is not None and "--batch-size" in out:+ b = out.index("--batch-size")+ out[b + 1] = str(_emitted_n_batch(n_batch, slots))+ if slots > 1 and supports_kv_unified and "--kv-unified" not in out and "-kvu" not in out:+ out[at + 2 : at + 2] = ["--kv-unified"]+ return out++def _extra_args_split_mode(extra_args: Optional[Iterable[str]], env: Optional[Mapping[str, str]] = None) -> Optional[str]:@@ -8277,6 +8298,7 @@ class LlamaCppBackend:self._effective_parallel_slots: int = 1# --parallel the last load asked for, before any fit-time reduction.self._requested_n_parallel: int = 1+ self._slots_cut_for_drafter: bool = False# Budget the running child was sized against; None before any load, so the# settings route reports a stale budget instead of nagging on every save.self._vram_fraction_launched: Optional[float] = None@@ -8864,6 +8886,7 @@ class LlamaCppBackend:self._effective_parallel_slots = 1# Cleared with the effective count so a stale value can't skew the dedupe.self._requested_n_parallel = 1+ self._slots_cut_for_drafter = Falseself._requested_n_batch = Noneself._requested_n_ubatch = Noneself._requested_load_mode = None@@ -9391,6 +9414,13 @@ class LlamaCppBackend:return Falseif not self._is_diffusion and self._requested_n_parallel != max(1, int(intent.n_parallel)):return False+ # The same count chosen rather than defaulted is one Auto may not cut.+ if (+ not self._is_diffusion+ and self._slots_cut_for_drafter+ and not intent.n_parallel_from_default+ ):+ return Falseif not self._is_diffusion and (self._requested_n_batch != intent.n_batch or self._requested_n_ubatch != intent.n_ubatch):@@ -26127,6 +26157,8 @@ class LlamaCppBackend:# Auto dropped the drafter because only the target fit. Bound before the# try: the launch below reads it either way._spec_dropped_no_vram = False+ _mtp_slots_asked: Optional[int] = None+ _mtp_slots_target_ctx: Optional[int] = None# The projector runs on the CPU and so allocates no VRAM:# --no-mmproj-offload clears mmproj_use_gpu and clip.cpp gates its whole# GPU backend on that, leaving no partial offload. Only the paravirtual@@ -26768,6 +26800,16 @@ class LlamaCppBackend:- _cc_bytes(ctx, 1, slots, cache_type),)+ def _layer_compute_buffer(slots: int) -> int:+ cb = self._estimate_compute_buffer_bytes(+ n_ubatch = _ubatch_for_slots(slots),+ n_parallel = slots,+ per_device_tensor = False,+ )+ return (+ cb if cb > 0 else self._TENSOR_PARALLEL_BUFFER_RESERVE_MIB * 1024 * 1024+ )+# Layer-split compute buffer (one lump; tensor mode reserves it# per device in _plan_tensor_parallel). Context-independent, so# fold it into the model footprint for the branches below. Falls@@ -27198,7 +27240,11 @@ class LlamaCppBackend:else 0.0)- def _probe_base(drafter: bool, n: int) -> int:+ def _probe_base(+ drafter: bool,+ n: int,+ slots: int = 0,+ ) -> int:# model_size_fit's terms, before it is built below._soft = self._CUDA_CONTEXT_RESERVE_BYTESif effective_is_vision and mmproj_size > 0:@@ -27207,7 +27253,11 @@ class LlamaCppBackend:_soft += self._MTP_DRAFT_COMPUTE_BYTESreturn (model_size- + _compute_buffer_pipeline+ + (+ _layer_compute_buffer(slots)+ if slots+ else _compute_buffer_pipeline+ )+ _soft+ max(0, n - 1) * _pipeline_overhead_bytes)@@ -27257,6 +27307,9 @@ class LlamaCppBackend:_both_fit_somewhere = False_probe_ctx = 0_probe_need = _probe_have = 0.0+ # (GPU count, target-only ctx, budget with drafter) per drafter-ranked subset.+ _target_placements: list[tuple[int, int, float]] = []+ _drafter_order = _probe_orders[-1]for _probe_ranked in _probe_orders:if _both_fit_somewhere:break@@ -27340,6 +27393,8 @@ class LlamaCppBackend:_foot_w,_budget_w,)+ if _probe_ranked is _drafter_order:+ _target_placements.append((_n, _ctx_wo, _budget_w))if _foot_w <= _budget_w:_both_fit_somewhere = Truebreak@@ -27399,6 +27454,104 @@ class LlamaCppBackend:_budget_w / 1024,)break+ # Fewer slots before no drafter. The walk takes the first subset holding the+ # drafter, so it must keep its target-only ctx (less the trim at one slot).+ # An extras --parallel wins at launch, so it is kept; the env twin is scrubbed.+ if (+ _target_fits_somewhere+ and not _both_fit_somewhere+ and n_parallel > 1+ and intent.n_parallel_from_default+ and _extra_args_n_parallel(extra_args, env = {}) is None+ ):++ def _mtp_foot_mib(n: int, ctx: int, s: int) -> float:+ ub = _ubatch_for_slots(s)+ return (+ _probe_base(True, n, s)+ + self._estimate_kv_cache_bytes(+ ctx,+ cache_type_kv,+ n_parallel = s,+ swa_full = swa_full,+ kv_unified = planned_kv_unified,+ n_ubatch = ub,+ flash_attn = planned_flash_attn,+ )+ + _cc_bytes(ctx, n, s)+ + _mtp_bytes(ctx, s, ub)+ ) / (1024 * 1024)++ def _ctx_with_drafter(n: int, ctx: int, budget: float, s: int) -> int:+ if explicit_ctx:+ return ctx if _mtp_foot_mib(n, ctx, s) <= budget else 0+ ub = _ubatch_for_slots(s)+ fitted = self._fit_context_to_vram(+ ctx,+ budget,+ _probe_base(True, n, s),+ cache_type_kv,+ swa_full = swa_full,+ n_parallel = s,+ kv_unified = planned_kv_unified,+ n_ubatch = ub,+ flash_attn = planned_flash_attn,+ mtp_engaged = True,+ mtp_overhead_fn = (+ functools.partial(mtp_overhead_fn, _np = s, _n_ubatch = ub)+ if mtp_overhead_fn is not None+ else None+ ),+ compute_ctx_bytes_fn = lambda c: _cc_bytes(c, n, s),+ budget_frac = 1.0,+ pooled = True,+ total_mib = None,+ ctx_checkpoints = _placement_ctx_checkpoints,+ )+ return (+ fitted+ if fitted > 0 and _mtp_foot_mib(n, fitted, s) <= budget+ else 0+ )++ _mtp_slots = _mtp_slots_ctx = 0+ for _s in range(n_parallel - 1, 0, -1):+ _keep = 1 - (+ _MTP_SLOT_CTX_TRIM if _s == 1 and not explicit_ctx else 0+ )+ for _n, _ctx_c, _budget_c in _target_placements:+ _c = _ctx_with_drafter(_n, _ctx_c, _budget_c, _s)+ if _c <= 0:+ continue+ if _c >= _ctx_c * _keep:+ _mtp_slots, _mtp_slots_ctx = _s, _c+ _mtp_slots_target_ctx = _ctx_c+ break+ if _mtp_slots:+ break+ if _mtp_slots:+ logger.info(+ "Auto: serving slots reduced %d -> %d so the drafter fits "+ "in VRAM at context %d (target alone: %d). Set Parallel "+ "Slots to keep %d without the drafter.",+ n_parallel,+ _mtp_slots,+ _mtp_slots_ctx,+ _probe_ctx,+ n_parallel,+ )+ _mtp_slots_asked = n_parallel+ n_parallel = _mtp_slots # allow-slot-clamp: the drafter fits in VRAM only at fewer slots+ _effective_ubatch = _ubatch_for_slots(n_parallel)+ _compute_buffer_pipeline = _layer_compute_buffer(n_parallel)+ if mtp_overhead_fn is not None:+ # Its defaults were bound at the requested count.+ mtp_overhead_fn = functools.partial(+ mtp_overhead_fn,+ _np = n_parallel,+ _n_ubatch = _effective_ubatch,+ )+ _both_fit_somewhere = Trueif _target_fits_somewhere and not _both_fit_somewhere:_spec_dropped_no_vram = True_mtp_will_engage = False@@ -28926,6 +29079,16 @@ class LlamaCppBackend:_vram_fit_ctx = Nonetp_tensor_split = Noneeffective_ctx = requested_ctx # fall back to original+ # Nor is a slot cut for the drafter: requested count, no drafter, as before.+ if _mtp_slots_asked is not None:+ n_parallel = _mtp_slots_asked+ _mtp_slots_asked = _mtp_slots_target_ctx = None+ _effective_ubatch = _ubatch_for_slots(n_parallel)+ _compute_buffer_pipeline = _layer_compute_buffer(n_parallel)+ _spec_dropped_no_vram = True+ _mtp_will_engage = False+ mtp_overhead_fn = None+ _mtp_kv_unsized = False# A hand-set context unified memory cannot hold. Raised here so the handler# above cannot turn it back into the launch it refuses, and ahead of the@@ -33173,12 +33336,88 @@ class LlamaCppBackend:strip_split_mode = False,)fallback_cmd = cmd[:_spec_at] + ["--spec-default"] + _fb_tail+ # Restore the cut slots; if they fail (unified KV refused), retry as cut.+ _fb_cut = (+ (+ fallback_cmd,+ n_parallel,+ effective_ctx,+ (self._effective_context_length, self._max_context_length),+ _auto_ctx_checkpoints,+ _auto_ckpt_emitted,+ )+ if _mtp_slots_asked is not None+ else None+ )+ if _fb_cut is not None:+ n_parallel = _mtp_slots_asked+ fallback_cmd = _with_parallel_slots(+ fallback_cmd,+ n_parallel,+ n_batch = n_batch,+ supports_kv_unified = bool(server_caps.get("supports_kv_unified")),+ )+ # The cut may have bought context; the restored count was priced at this.+ if _mtp_slots_target_ctx and "-c" in fallback_cmd:+ effective_ctx = _mtp_slots_target_ctx+ fallback_cmd[fallback_cmd.index("-c") + 1] = str(effective_ctx)+ # The ceiling was priced at the cut count; this is the one proven here.+ self._effective_context_length = self._max_context_length = (+ effective_ctx+ )+ # The checkpoint cap is per slot; same re-decide as the single-seq rung.+ if _auto_ckpt_emitted:+ fallback_cmd = self._without_flag_pairs(+ fallback_cmd, _auto_ckpt_emitted+ )+ _auto_ctx_checkpoints = _decide_auto_ctx_checkpoints()+ _auto_ckpt_emitted = _emit_auto_ctx_checkpoints(fallback_cmd)+ kv_cache_unified = _kv_unified_from_args(fallback_cmd)+ # The plan other loads place against was priced at the cut count.+ _fb_cut_plan = dict(self._pending_plan_mib)+ if self._pending_plan_mib and self._can_estimate_kv():+ kv_cache_bytes = self._estimate_kv_cache_bytes(+ effective_ctx,+ cache_type_kv,+ n_parallel = n_parallel,+ swa_full = swa_full,+ kv_unified = bool(kv_cache_unified),+ n_ubatch = _ubatch_for_slots(n_parallel),+ flash_attn = planned_flash_attn,+ )+ self._pending_plan_mib = _gpu_plan_mib(+ gguf_size + mmproj_size + kv_cache_bytes,+ list(self._pending_plan_mib),+ gpus,+ )_unified_withdraw_if_unneeded(fallback_cmd,mtp_engages = False,why = "The retry without speculative decoding",)healthy = _spawn_and_wait(fallback_cmd, label = "-retry")+ if not healthy and _fb_cut is not None and not _load_cancelled():+ logger.warning(+ "The retry without speculative decoding failed at %d slots; "+ "retrying at the %d the drafter launch ran.",+ n_parallel,+ _fb_cut[1],+ )+ self._kill_process()+ (+ fallback_cmd,+ _fb_slots,+ effective_ctx,+ (self._effective_context_length, self._max_context_length),+ _auto_ctx_checkpoints,+ _auto_ckpt_emitted,+ ) = _fb_cut+ self._pending_plan_mib = _fb_cut_plan+ n_parallel = (+ _fb_slots # allow-slot-clamp: the restored count failed to start+ )+ kv_cache_unified = _kv_unified_from_args(fallback_cmd)+ healthy = _spawn_and_wait(fallback_cmd, label = "-retry-cut")if healthy:self._speculative_type = "default"_mtp_active_for_launched_server = False@@ -33583,6 +33822,9 @@ class LlamaCppBackend:self._requested_n_ctx = int(effective_ctx if _replayed_ctx_refit else n_ctx)# Local n_parallel may have been reduced above; the snapshot has the ask.self._requested_n_parallel = max(1, int(intent.n_parallel))+ self._slots_cut_for_drafter = (+ _mtp_slots_asked is not None and n_parallel < _mtp_slots_asked+ )# Commit with the rest of the known-good state: only a launch that got# this far replaced the runtime, so a rejected preflight leaves the old# marker intact, and the diffusion runner returns well before it withoutdiff --git a/studio/backend/routes/inference.py b/studio/backend/routes/inference.pyindex 94502953dd4..cb95436faa9 100644--- a/studio/backend/routes/inference.py+++ b/studio/backend/routes/inference.py@@ -8607,6 +8607,7 @@ def _gguf_request_intent(extra_args = extra_args,gpu_ids = gpu_ids,n_parallel = n_parallel,+ n_parallel_from_default = getattr(request, "n_parallel", None) is None,)settings.update(changes)return replace(source, **settings)diff --git a/studio/backend/tests/test_batch_sizes_per_load.py b/studio/backend/tests/test_batch_sizes_per_load.pyindex 4d61418378d..a6e7bb60a14 100644--- a/studio/backend/tests/test_batch_sizes_per_load.py+++ b/studio/backend/tests/test_batch_sizes_per_load.py@@ -655,8 +655,11 @@ def test_the_recorded_micro_batch_is_derived_from_the_slots_that_launched():and node.func.id == "_ubatch_for_slots"]# sizing pass, embedding slot clamp, candidate split pricing, fit-time reduction, the- # MoE expert-spill raise, then the post-launch record- assert len(calls) == 6, f"expected six re-derivations, found {len(calls)}"+ # MoE expert-spill raise, the post-launch record, the drafter probe's four+ # (per-slot compute buffer, candidate footprint, one-slot trim, slot commit; #8872)+ # the slot restore when GPU selection abandons that cut, and the plan repriced when a+ # drafterless retry restores it+ assert len(calls) == 12, f"expected twelve re-derivations, found {len(calls)}"# the record must not reuse the sizing pass's valuecompact = "".join(src.split())assert "self._n_ubatch=max(0,int(self._DEFAULT_N_UBATCHif_launched_ubatchisNone" in compactdiff --git a/studio/backend/tests/test_ctx_checkpoint_host_budget.py b/studio/backend/tests/test_ctx_checkpoint_host_budget.pyindex 77d3e3b918f..ac4c7aa29b4 100644--- a/studio/backend/tests/test_ctx_checkpoint_host_budget.py+++ b/studio/backend/tests/test_ctx_checkpoint_host_budget.py@@ -477,7 +477,8 @@ class TestCheckpointsNeverReachAVramFigure:source = inspect.getsource(LlamaCppBackend.load_model)assert "0 if self._rollback_state_bytes(1) > 0 else _requested_ctx_checkpoints" in sourceassert source.count("ctx_checkpoints = _fit_ctx_checkpoints,") == 1- assert source.count("ctx_checkpoints = _placement_ctx_checkpoints,") == 6+ # +1: the drafter probe's one-slot fit (#8872)+ assert source.count("ctx_checkpoints = _placement_ctx_checkpoints,") == 7assert source.count("_kv_bytes(effective_ctx, _effective_ctx_checkpoints)") == 1def test_the_fit_is_unmoved_by_the_new_term(self):diff --git a/studio/backend/tests/test_llama_cpp_placement.py b/studio/backend/tests/test_llama_cpp_placement.pyindex 547d17e3f5a..e6b1a29cf16 100644--- a/studio/backend/tests/test_llama_cpp_placement.py+++ b/studio/backend/tests/test_llama_cpp_placement.py@@ -5,6 +5,7 @@from __future__ import annotations+import dataclassesimport osimport structimport subprocess@@ -1109,6 +1110,369 @@ def test_auto_drops_the_drafter_when_only_the_target_fits(tmp_path):assert backend.mtp_draft_path == str(sidecar)+def _per_slot_mtp_backend(tmp_path: Path, *, gb_per_slot: float):+ """The tight card with an embedded head whose reserve is all per-slot state."""+ gb = 1024**3+ backend, gguf, _sidecar = _tight_vram_backend(tmp_path, drafter_gb = 0.0)++ def read_metadata(_path):+ backend._nextn_predict_layers = 1+ backend._context_length = 8192++ backend._read_gguf_metadata = read_metadata+ backend._estimate_mtp_overhead_bytes = (+ lambda n_ctx, *args, _np = None, n_parallel = None, **kwargs: int(+ gb_per_slot * gb * int(_np or n_parallel or 1)+ )+ )+ backend.probe_server_capabilities = lambda _binary = None: {+ "mtp_token": "draft-mtp",+ "supports_ngram_mod": True,+ "spec_draft_n_max_flag": "--spec-draft-n-max",+ "supports_kv_unified": True,+ }+ return backend, gguf+++def _launch_default_slots(+ backend,+ gguf,+ *,+ n_parallel = 4,+ **kwargs,+):+ return _launch_auto_8k(+ backend, gguf, n_ctx = 0, n_parallel = n_parallel, n_parallel_from_default = True, **kwargs+ )+++def _launched(result, flag: str) -> Optional[str]:+ cmd = result["cmd"]+ return cmd[cmd.index(flag) + 1] if flag in cmd else None++[email protected]("gb_per_slot, slots", [(1.5, "3"), (2.0, "2"), (5.0, "1")])+def test_auto_drops_serving_slots_before_the_drafter(tmp_path, gb_per_slot, slots):+ """#8872: the largest slot count that holds the drafter at the same context."""+ backend, gguf = _per_slot_mtp_backend(tmp_path, gb_per_slot = gb_per_slot)++ result = _launch_default_slots(backend, gguf)++ assert _launched(result, "--spec-type") == "draft-mtp"+ assert _launched(result, "--parallel") == slots+ assert _launched(result, "-c") == "8192"+ assert backend.spec_fallback_reason is None++[email protected]("n_parallel", [4, 1])+def test_auto_drops_the_drafter_when_one_slot_cannot_hold_it(tmp_path, n_parallel):+ backend, gguf = _per_slot_mtp_backend(tmp_path, gb_per_slot = 7.0)++ result = _launch_default_slots(backend, gguf, n_parallel = n_parallel)++ assert _launched(result, "--spec-type") != "draft-mtp"+ assert _launched(result, "--parallel") == str(n_parallel)+ assert backend.spec_fallback_reason == "drafter_no_vram"+++def test_a_chosen_slot_count_is_kept_and_the_drafter_dropped(tmp_path):+ backend, gguf = _per_slot_mtp_backend(tmp_path, gb_per_slot = 2.0)++ result = _launch_auto_8k(backend, gguf, n_ctx = 0, n_parallel = 4)++ assert _launched(result, "--spec-type") != "draft-mtp"+ assert _launched(result, "--parallel") == "4"+ assert backend.spec_fallback_reason == "drafter_no_vram"+++def test_a_wider_subset_holding_the_drafter_does_not_justify_the_cut(tmp_path):+ """Fewer slots fit the drafter at full context only across both cards, but the walk+ takes the first card that holds it at any context, here a few hundred tokens."""+ gb, mib = 1024**3, 1024 * 1024+ backend, gguf = _backend(+ tmp_path, vulkan = False, memory = [(0, 24_576, 24_576), (1, 24_576, 24_576)]+ )++ def read_metadata(_path):+ backend._nextn_predict_layers = 1+ backend._context_length = 8192++ def fit(+ requested,+ budget_mib,+ base,+ *args,+ mtp_engaged = False,+ mtp_overhead_fn = None,+ **kw,+ ):+ ctx = requested+ while ctx > 0:+ drafter = mtp_overhead_fn(ctx) if mtp_engaged and mtp_overhead_fn else 0+ if (base + gb + drafter) / mib <= budget_mib:+ return ctx+ ctx -= 256+ return 0++ backend._read_gguf_metadata = read_metadata+ backend._get_gguf_size_bytes = lambda _path: 16 * gb+ backend._can_estimate_kv = lambda: True+ backend._estimate_kv_cache_bytes = lambda *args, **kwargs: gb+ backend._compute_buffer_ctx_bytes = lambda *args, **kwargs: 0+ backend._estimate_compute_buffer_bytes = lambda **kwargs: 1+ backend._mtp_draft_kv_bytes = lambda *args, **kwargs: 0+ backend._estimate_mtp_overhead_bytes = (+ lambda n_ctx, *args, _np = None, n_parallel = None, **kwargs: int(+ 1.5 * gb * int(_np or n_parallel or 1) + 24 * gb * n_ctx / 8192+ )+ )+ backend._fit_context_to_vram = fit+ backend.probe_server_capabilities = lambda _binary = None: {+ "mtp_token": "draft-mtp",+ "supports_ngram_mod": True,+ "spec_draft_n_max_flag": "--spec-draft-n-max",+ "supports_kv_unified": True,+ }++ result = _launch_default_slots(backend, gguf)++ assert _launched(result, "--spec-type") != "draft-mtp"+ assert _launched(result, "--parallel") == "4"+ assert _launched(result, "-c") == "8192"+ assert backend.spec_fallback_reason == "drafter_no_vram"++[email protected]("extras", [["--parallel", "4"], ["-np=4"]])+def test_a_slot_count_in_the_extras_is_kept(tmp_path, extras):+ backend, gguf = _per_slot_mtp_backend(tmp_path, gb_per_slot = 2.0)++ result = _launch_default_slots(backend, gguf, extra_args = extras)++ assert _launched(result, "--spec-type") != "draft-mtp"+ assert _launched(result, "--parallel") == "4"+ assert backend.spec_fallback_reason == "drafter_no_vram"+++def _launch_with_failing_drafter(+ backend,+ gguf,+ *,+ refuses_unified_kv = False,+ n_ctx = 0,+):+ cmds = []++ def fake_popen(cmd, **kwargs):+ if not cmd or str(cmd[0]) != "/fake/llama-server":+ return _REAL_POPEN(cmd, **kwargs)+ cmds.append(list(cmd))+ return type(+ "Process",+ (),+ {+ "pid": 123,+ "stdout": (),+ "poll": lambda self: None,+ "terminate": lambda self: None,+ "wait": lambda self, timeout = None: 0,+ "kill": lambda self: None,+ },+ )()++ def fake_health(timeout = None, **_kw):+ backend._stdout_lines = []+ if "draft-mtp" in cmds[-1]:+ return False+ return not (refuses_unified_kv and "--kv-unified" in cmds[-1])++ backend._wait_for_health = fake_health+ with patch.object(subprocess, "Popen", side_effect = fake_popen):+ assert backend.load_model(+ GgufLoadIntent(+ gguf_path = str(gguf),+ model_identifier = "test",+ n_ctx = n_ctx,+ n_parallel = 4,+ n_parallel_from_default = True,+ speculative_type = "auto",+ )+ )+ return cmds+++def test_a_drafterless_retry_that_fails_at_the_restored_slots_runs_as_cut(tmp_path):+ backend, gguf = _per_slot_mtp_backend(tmp_path, gb_per_slot = 5.0)++ cmds = _launch_with_failing_drafter(backend, gguf, refuses_unified_kv = True)++ assert [c[c.index("--parallel") + 1] for c in cmds] == ["1", "4", "1"]+ assert "--kv-unified" not in cmds[-1] and "draft-mtp" not in cmds[-1]+ assert backend.effective_parallel_slots == 1+++ "refuses_unified_kv, ctxs", [(False, ["7424", "8192"]), (True, ["7424", "8192", "7424"])]+)+def test_the_restored_slots_relaunch_at_their_own_context(tmp_path, refuses_unified_kv, ctxs):+ gb = 1024**3+ backend, gguf = _per_slot_mtp_backend(tmp_path, gb_per_slot = 0.0)+ backend._estimate_mtp_overhead_bytes = lambda n_ctx, *args, **kwargs: int(+ 6.2 * gb * n_ctx / 8192+ )+ backend._fit_context_to_vram = lambda requested, *args, mtp_engaged = False, **kwargs: (+ min(requested, 7424) if mtp_engaged else requested+ )++ cmds = _launch_with_failing_drafter(backend, gguf, refuses_unified_kv = refuses_unified_kv)++ assert [c[c.index("-c") + 1] for c in cmds] == ctxs+ assert backend._effective_context_length == int(ctxs[-1])+ if not refuses_unified_kv:+ assert backend._max_context_length == 8192+++def test_the_restored_slots_reprice_the_published_plan(tmp_path):+ """Other loads place against this plan, so it is the restored count's, not the cut's."""+ gb = 1024**3+ backend, gguf = _per_slot_mtp_backend(tmp_path, gb_per_slot = 2.0)+ backend._estimate_kv_cache_bytes = lambda *args, n_parallel = 1, **kwargs: (+ gb // 4 * int(n_parallel or 1)+ )++ cmds = _launch_with_failing_drafter(backend, gguf)++ assert [c[c.index("--parallel") + 1] for c in cmds] == ["2", "4"]+ assert sum(backend._planned_vram_mib.values()) == (16 * gb + gb) // (1024 * 1024)+++def test_an_explicit_context_restores_its_ceiling(tmp_path):+ backend, gguf = _per_slot_mtp_backend(tmp_path, gb_per_slot = 2.0)++ cmds = _launch_with_failing_drafter(backend, gguf, n_ctx = 8192)++ assert [c[c.index("--parallel") + 1] for c in cmds] == ["2", "4"]+ assert [c[c.index("-c") + 1] for c in cmds] == ["8192", "8192"]+ assert backend._max_context_length == backend._effective_context_length == 8192+++def test_an_env_slot_count_does_not_block_the_cut(tmp_path, monkeypatch):+ """The launch scrubs LLAMA_ARG_N_PARALLEL, so it never reaches the server."""+ monkeypatch.setenv("LLAMA_ARG_N_PARALLEL", "4")+ backend, gguf = _per_slot_mtp_backend(tmp_path, gb_per_slot = 2.0)++ result = _launch_default_slots(backend, gguf)++ assert _launched(result, "--spec-type") == "draft-mtp"+ assert _launched(result, "--parallel") == "2"+++def test_a_drafter_that_fails_to_start_gives_its_slots_back(tmp_path):+ backend, gguf = _per_slot_mtp_backend(tmp_path, gb_per_slot = 2.0)+ cmds = []++ def fake_popen(cmd, **kwargs):+ if not cmd or str(cmd[0]) != "/fake/llama-server":+ return _REAL_POPEN(cmd, **kwargs)+ cmds.append(list(cmd))+ return type(+ "Process",+ (),+ {+ "pid": 123,+ "stdout": (),+ "poll": lambda self: None,+ "terminate": lambda self: None,+ "wait": lambda self, timeout = None: 0,+ "kill": lambda self: None,+ },+ )()++ def fake_health(timeout = None, **_kw):+ backend._stdout_lines = []+ return "draft-mtp" not in cmds[-1]++ backend._wait_for_health = fake_health+ with patch.object(subprocess, "Popen", side_effect = fake_popen):+ assert backend.load_model(+ GgufLoadIntent(+ gguf_path = str(gguf),+ model_identifier = "test",+ n_ctx = 0,+ n_parallel = 4,+ n_parallel_from_default = True,+ speculative_type = "auto",+ )+ )++ first, retry = cmds[0], cmds[-1]+ assert first[first.index("--parallel") + 1] == "2"+ assert "draft-mtp" not in retry and "--spec-default" in retry+ assert retry[retry.index("--parallel") + 1] == "4"+ assert "--kv-unified" in retry+ assert backend.effective_parallel_slots == 4+++def test_a_placement_abandoned_after_the_cut_gives_the_slots_back(tmp_path):+ backend, gguf = _per_slot_mtp_backend(tmp_path, gb_per_slot = 2.0)++ def boom(*_args, **_kwargs):+ raise RuntimeError("placement bug")++ backend._fit_derived_load_mode = boom+ result = _launch_default_slots(backend, gguf)++ assert _launched(result, "--fit") == "on"+ assert _launched(result, "--parallel") == "4"+ assert _launched(result, "--spec-type") != "draft-mtp"+ assert backend.spec_fallback_reason == "drafter_no_vram"+++def test_a_chosen_count_does_not_dedupe_against_slots_cut_for_the_drafter(tmp_path):+ backend, gguf = _per_slot_mtp_backend(tmp_path, gb_per_slot = 2.0)+ result = _launch_default_slots(backend, gguf)+ assert _launched(result, "--parallel") == "2"++ chosen = GgufLoadIntent(+ gguf_path = str(gguf),+ model_identifier = "test",+ n_ctx = 0,+ n_parallel = 4,+ speculative_type = "auto",+ )+ default = dataclasses.replace(chosen, n_parallel_from_default = True)+ assert backend._runtime_matches_intent(default, None)+ assert not backend._runtime_matches_intent(chosen, None)+ # A diffusion runtime that replaced it ignores slot counts altogether.+ backend._is_diffusion = True+ assert backend._runtime_matches_intent(chosen, None) == backend._runtime_matches_intent(+ default, None+ )++[email protected]("ctx_with_drafter, keeps", [(7424, True), (6144, False)])+def test_one_slot_may_trim_a_little_auto_context(tmp_path, ctx_with_drafter, keeps):+ """Draft KV grows with context, so one slot can still need a few percent of it."""+ gb = 1024**3+ backend, gguf = _per_slot_mtp_backend(tmp_path, gb_per_slot = 0.0)+ backend._estimate_mtp_overhead_bytes = lambda n_ctx, *args, **kwargs: int(+ 6.2 * gb * n_ctx / 8192+ )+ backend._fit_context_to_vram = lambda requested, *args, mtp_engaged = False, **kwargs: (+ min(requested, ctx_with_drafter) if mtp_engaged else requested+ )++ result = _launch_default_slots(backend, gguf)++ if keeps:+ assert _launched(result, "--spec-type") == "draft-mtp"+ assert _launched(result, "--parallel") == "1"+ assert _launched(result, "-c") == str(ctx_with_drafter)+ else:+ assert _launched(result, "--spec-type") != "draft-mtp"+ assert _launched(result, "--parallel") == "4"+ assert _launched(result, "-c") == "8192"++def test_auto_keeps_a_drafter_that_fits(tmp_path):"""The drop is scoped to the shortfall: with room for both, nothing changes."""backend, gguf, sidecar = _tight_vram_backend(tmp_path, drafter_gb = 1.5)diff --git a/studio/backend/tests/test_vram_budget_settings.py b/studio/backend/tests/test_vram_budget_settings.pyindex 10a5d06b0d6..4a97573cb19 100644--- a/studio/backend/tests/test_vram_budget_settings.py+++ b/studio/backend/tests/test_vram_budget_settings.py@@ -585,10 +585,11 @@ class TestFloorReserve:assert lc._vram_usable_mib(12_000, None, 1.0, pooled = True) == pytest.approx(12_000)def test_every_pooled_caller_says_so(self):- # Five call sites hand _fit_context_to_vram an absolute pool budget; each- # has to be marked, since the flag is what keeps them from double-paying.+ # Six call sites hand _fit_context_to_vram an absolute pool budget (the sixth is+ # the drafter probe's one-slot fit, #8872); each has to be marked, since the flag+ # is what keeps them from double-paying.compact = _shared_setup_1()- assert compact.count("budget_frac=1.0,pooled=True,total_mib=None,") == 5+ assert compact.count("budget_frac=1.0,pooled=True,total_mib=None,") == 6class TestRetriesAndDedup: