Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
Studio: make the GGUF memory estimate match the compute buffers llama.cpp allocates (#11043)
oobabooga820d377f13f7
diff --git a/studio/backend/core/inference/llama_cpp.py b/studio/backend/core/inference/llama_cpp.pyindex 6d32fb898e6..cfe11c45455 100644--- a/studio/backend/core/inference/llama_cpp.py+++ b/studio/backend/core/inference/llama_cpp.py@@ -3896,6 +3896,14 @@ def _kv_bytes_per_elem(cache_type: Optional[str]) -> float:}.get((cache_type or "f16").strip().lower(), 2.0)+# Upper bound on any current tokenizer, for output rows when a truncated header drops+# the token array. Above Llama 4 / Gemma 3 (256k), the widest shipping.+_ASSUMED_MAX_VOCAB = 262144+# Ceiling on the float32 activations a compute graph keeps per micro-batch token, for a+# header that cannot be read: 4 * n_ff at n_ff = 65536, above Llama 3.1 405B's 53056.+_ASSUMED_MAX_ACTIVATION_WIDTH = 262144++def _pad_kv_cells(cells: int) -> int:return ((cells + 255) // 256) * 256@@ -4689,6 +4697,24 @@ def _flash_attn_enabled_from_args(return enabled+def _planned_flash_attn(+ extra_args: Optional[Iterable[str]],+ supports_flash_attn: bool,+ v_cache_type: Optional[str] = None,+ architecture: Optional[str] = None,+) -> bool:+ """Flash attention as the launch command will resolve it.++ The managed --flash-attn on follows the environment and precedes user extras, so+ only extras can turn it off. A build without the flag gets neither the flag nor+ the inherited LLAMA_ARG_FLASH_ATTN. A quantized V cache forces it on, except on+ Grok, which llama.cpp always runs without it."""+ if not supports_flash_attn or architecture == "grok":+ return False+ enabled = _flash_attn_enabled_from_args(extra_args, default = True, env = {})+ return enabled or (v_cache_type or "f16").strip().lower() not in {"f16", "bf16", "f32"}++def _effective_spec_type(extra_args: Optional[Iterable[str]], env: Optional[Mapping[str, str]] = None) -> Optional[str]:@@ -7032,8 +7058,17 @@ class LlamaCppBackend:# llama_pooling_type from the GGUF; present and non-zero marks an embedding model.self._pooling_type: Optional[int] = None# For the compute-graph buffer estimate; vocab from the tokens array len.+ # feed_forward_length is the widest layer when the GGUF stores one per layer.self._feed_forward_length: Optional[int] = None+ self._expert_used_count: Optional[int] = None+ self._expert_feed_forward_length: Optional[int] = None+ self._expert_shared_feed_forward_length: Optional[int] = None+ self._expert_shared_count: Optional[int] = None+ # Gemma 3n / Gemma 4 per-layer input embedding width.+ self._embedding_length_per_layer_input: Optional[int] = Noneself._vocab_size: Optional[int] = None+ # Set from reserves_micro_batch_outputs() by the callers that know the binary.+ self._reserves_micro_batch_outputs: bool = False# Architecture-aware KV fields for 5-path estimationself._kv_key_length: Optional[int] = Noneself._kv_value_length: Optional[int] = None@@ -8515,6 +8550,8 @@ class LlamaCppBackend:_CAPABILITY_PROBE_RETRY_SECONDS = 30.0_CAPABILITY_PROBE_RETRY_MAX_SECONDS = 600.0_capability_cache: dict[tuple[str, int, int], dict[str, object]] = {}+ # (bin_path, mtime_ns, size) -> build number from `llama-server --version`, or None.+ _build_number_cache: dict[tuple[str, int, int], Optional[int]] = {}_capability_retry_after: dict[tuple[str, int, int], float] = {}_capability_retry_backoff: dict[tuple[str, int, int], float] = {}_capability_cache_lock = threading.Lock()@@ -8540,6 +8577,68 @@ class LlamaCppBackend:return Truereturn bool(cls._FLASH_ATTN_ENUM_RE.search(help_text))+ # ggml-org/llama.cpp#23861 capped the output rows a context reserves at the rows it+ # can emit. Builds before it reserve a row for every micro-batch token.+ _OUTPUT_ROWS_CAPPED_BUILD = 9460++ @classmethod+ def probe_build_number(cls, binary: Optional[str] = None) -> Optional[int]:+ """The llama.cpp build number from ``llama-server --version``, cached per binary.++ Current builds print ``version: X.Y.Z (build NNNN, commit ...)``, older ones+ ``version: NNNN (commit)``. None when the binary is missing, the probe fails,+ or the build reports 1 (a source build without tags)."""+ bin_path = cls._exec_path_for_launch(binary or cls._find_llama_server_binary())+ if not bin_path or not Path(bin_path).is_file():+ return None+ try:+ binary_stat = Path(bin_path).stat()+ cache_key = (bin_path, binary_stat.st_mtime_ns, binary_stat.st_size)+ except OSError:+ cache_key = (bin_path, 0, 0)+ with cls._capability_cache_lock:+ if cache_key in cls._build_number_cache:+ return cls._build_number_cache[cache_key]+ build: Optional[int] = None+ try:+ env = cls._llama_server_env_for_binary(bin_path)+ if sys.platform == "darwin":+ env["GGML_METAL_DEVICES"] = "0" # see probe_server_capabilities+ result = subprocess.run(+ [bin_path, "--version"],+ capture_output = True,+ text = True,+ encoding = "utf-8",+ errors = "replace",+ timeout = 10,+ check = False,+ env = env,+ **_windows_hidden_subprocess_kwargs(),+ )+ output = (result.stdout or "") + "\n" + (result.stderr or "")+ match = re.search(r"version:[^\r\n]*\bbuild\s+(\d+)\b", output) or re.search(+ r"version:\s*(\d+)\b(?!\.)", output+ )+ if match and int(match.group(1)) > 1:+ build = int(match.group(1))+ except Exception as exc:+ logger.debug("llama-server --version probe failed: %s", exc)+ with cls._capability_cache_lock:+ cls._build_number_cache[cache_key] = build+ return build++ @classmethod+ def reserves_micro_batch_outputs(cls, binary: Optional[str] = None) -> bool:+ """Whether this llama-server reserves an output row per micro-batch token.++ Known builds before b9460, including the older-macOS b9415 fallback, do.+ Unknown builds are priced as current, matching Studio's prebuilts."""+ try:+ build = cls.probe_build_number(binary)+ except Exception:+ return False+ return build is not None and build < cls._OUTPUT_ROWS_CAPPED_BUILD+_ADVERTISED_DEFAULT_RE = re.compile(r"\(default:\s*(-?\d+)")@classmethod@@ -14821,16 +14920,10 @@ class LlamaCppBackend:reserve_bytes_fn: Callable[[int], int],min_ctx: int = 8192,) -> int:- """Largest context in [``min_ctx``, ``ctx``] whose replicated per-device reserve- still fits the SMALLEST card of a layer-split subset; 0 when even ``min_ctx``- does not. The reserve shrinks with the context, so a subset- ``_every_gpu_holds_reserve`` turns down at the pooled context usually serves a- lower one: 40000 + 2500 MiB free holds 31488, not the 4096 a drop would leave.- Bisects the reserve, which is monotone in context on every branch but- proportional on only some: deepseek4's flat indexer term makes a closed form- answer 43341 on a 4000 MiB card, whose reserve there is 6048 MiB. The MiB- comparison mirrors ``_every_gpu_holds_reserve``, so that gate accepts what this- returns."""+ """Largest context in [min_ctx, ctx] whose reserve fits every selected GPU.++ Bisect because reserves are monotone but may include fixed costs. Use the+ same MiB comparison as _every_gpu_holds_reserve. Return 0 if min_ctx fails."""values = list(usable_mib)if not values or ctx < min_ctx:return 0@@ -15593,14 +15686,56 @@ class LlamaCppBackend:target_recurrent_copies = 0if target_rollback and spec_draft_n_max > 0:target_recurrent_copies = self._rollback_state_bytes(n_parallel) * spec_draft_n_max+ spec_compute = self._mtp_draft_compute_bytes(+ n_ctx,+ drafter_path = drafter_path,+ n_ubatch = n_ubatch,+ n_parallel = n_parallel,+ spec_draft_n_max = spec_draft_n_max,+ )if draft_kv is None:# KV unsized (exotic/remote drafter): still reserve known weights + any# MLA target copy so a large config can't launch over budget (the small# unsized draft KV rides in the cushion). Nothing known -> None, so the# caller keeps the flat fallback.total = weights + target_ctx_copy + target_recurrent_copies- return total if total > 0 else None- return draft_kv + weights + target_ctx_copy + target_recurrent_copies+ return total + spec_compute if total > 0 else None+ return draft_kv + weights + target_ctx_copy + target_recurrent_copies + spec_compute++ def _mtp_draft_compute_bytes(+ self,+ n_ctx: int,+ *,+ drafter_path: Optional[str] = None,+ n_ubatch: Optional[int] = None,+ n_parallel: int = 1,+ spec_draft_n_max: int = 0,+ ) -> int:+ """Draft mask and activations beyond the separately charged floor, plus+ target verification rows. Use the drafter's dimensions when available,+ otherwise the target's dimensions for an embedded head."""+ verify_rows = self._spec_verify_rows_bytes(n_parallel, spec_draft_n_max)+ if n_ctx <= 0:+ return verify_rows+ ub = max(1, int(self._DEFAULT_N_UBATCH if n_ubatch is None else n_ubatch))+ width = self._compute_activation_width()+ if drafter_path:+ db = self._draft_backend_for(drafter_path)+ draft_width = getattr(db, "_compute_activation_width", lambda: 0)()+ if draft_width > 0:+ width = draft_width + 2 * (getattr(self, "_embedding_length", None) or 0)+ activations = int(width * ub * 4 * self._COMPUTE_BUFFER_SAFETY)+ return verify_rows + max(0, ub * 2 * n_ctx + activations - self._MTP_DRAFT_COMPUTE_BYTES)++ def _spec_verify_rows_bytes(self, n_parallel: int, spec_draft_n_max: int) -> int:+ """Target output rows for verified draft tokens, wherever the drafter runs."""+ return (+ max(1, n_parallel)+ * max(0, spec_draft_n_max)+ * self._SPEC_VERIFY_ROW_COPIES+ * (getattr(self, "_vocab_size", None) or _ASSUMED_MAX_VOCAB)+ * 4+ )def _mtp_reserve_note(self,@@ -15667,7 +15802,7 @@ class LlamaCppBackend:)_DEFAULT_N_UBATCH = _DEFAULT_LLAMA_N_UBATCH- _COMPUTE_BUFFER_SAFETY = 1.15 # upper-bound margin on the compute-buffer estimate+ _COMPUTE_BUFFER_SAFETY = 1.10 # margin on the modelled activation width# Soft VRAM the modeled terms omit; charged to the fit budget on tight tiers (#6682)._CUDA_CONTEXT_RESERVE_BYTES = 320 * 1024 * 1024 # CUDA ctx + cuBLAS workspace (~330 MiB)_MMPROJ_VRAM_SAFETY = 1.4 # mmproj worst-case buffer vs file size (runtime ~1.3x)@@ -15683,30 +15818,10 @@ class LlamaCppBackend:# load while over-reserving only costs context. Zero cost on any other model,# where the recurrent state is nil._UNKNOWN_SPEC_DRAFT_N_MAX = 16- # The flash-attn KQ mask + attention scratch grow ~linearly with context; the flat- # _estimate_compute_buffer_bytes term only covers ctx -> 0. The per-token rate- # depends on the KV cache type: a QUANTIZED cache (q8_0/q5/q4/iq4) needs a- # context-sized dequant scratch that scales with n_embd, measured at 0.74-2.02 x- # n_embd across Qwen3.5/3.6 (2B/4B/9B/27B) and Gemma-4 (12B/31B) at q8_0; an- # f16/bf16/f32 cache skips the dequant and pays only the KQ mask, a flat n_ubatch*2- # bytes per context token regardless of n_embd (measured 1024 B/tok on Qwen-9B and- # Gemma-31B alike). So Qwen3.5-4B at 256k is 1.30 GiB at q8_0 vs 0.31 GiB at f16.- # 2.25 covers the worst quantized case (Qwen3.5-4B, ~2.0x) plus the under-modeled- # flat base; the mask safety covers the f16 base gap. Without this term, tight tiers- # at extreme context over-pin and spill to CPU (the 3% cushion is only ~0.25 GiB on- # an 8 GB card, far below the ~1-2.4 GiB quantized buffer at 256k): e.g. Qwen3.5-4B- # Q4 at 256k needs ~8.5 GiB on a real 8 GB card (weights 2.4 + KV 4.3 + compute 1.3- # + CUDA ctx) -> CPU spill; with this reserve the auto context caps to ~210k, fits.- _CTX_COMPUTE_BYTES_PER_EMBD = 2.25 # quantized KV, regular attention (dequant scratch)- _CTX_COMPUTE_BYTES_PER_EMBD_MLA = 1.25 # quantized KV, MLA (compressed attn: measured 0.94x)- _CTX_COMPUTE_F16_MASK_SAFETY = 1.5 # f16/bf16/f32 KV: KQ mask only (n_ubatch*2 B/tok)- # Per-device f16 ctx-linear rate steps 4x once the model spans >1 device under- # `-sm layer`: pipeline parallelism sets the ggml scheduler's n_copies to- # GGML_SCHED_MAX_COPIES == 4, and the [n_kv, n_ubatch] f16 KQ mask is a- # GGML_TENSOR_FLAG_INPUT, so 1 live copy becomes 4. Hence a step on "is split", not- # a ramp in device count: measured 2.00 B/tok/ubatch on 1 GPU and 8.00 on 2-8 GPUs,- # architecture- and --parallel-independent. Charging 1x on a split under-reserves- # 2.67x (= 8/(2*1.5)). llama.cpp declines it in _pipeline_parallel_disabled_by_args.+ # Vocabulary-wide float32 copies per verified draft token (logits and sampling).+ _SPEC_VERIFY_ROW_COPIES = 3+ # Pipeline parallelism uses GGML_SCHED_MAX_COPIES input copies per device.+ # This applies only when _pipeline_parallel_disabled_by_args is false._CTX_COMPUTE_SPLIT_MULT = 4# DeepSeek-V4 (deepseek4): its lightning indexer + sparse attention reserve a large# context-scaling compute buffer the rates above miss (present even with an f16@@ -15728,39 +15843,132 @@ class LlamaCppBackend:# path): measured 402.5 GiB reserve at 1M ctx pre-fix, ~402 KiB per token._INKLING_CTX_COMPUTE_DENSE_BYTES_PER_TOK = 402470 # per token at ub=512+ def _compute_activation_width(self) -> int:+ """Float32 activations the compute graph keeps live per micro-batch token.++ Take the widest dense, expert or recurrent block, then add per-layer+ embeddings. The 12 * n_embd floor covers attention and residual buffers.+ Return 0 without embedding dimensions.+ """++ # getattr: a backend built with __new__ carries only the dims it was given.+ def dim(name: str) -> int:+ return getattr(self, name, None) or 0++ n_embd = dim("_embedding_length")+ if n_embd <= 0:+ return 0++ widths = [12 * n_embd]+ if dim("_feed_forward_length") > 0:+ widths.append(4 * dim("_feed_forward_length"))+ n_used = dim("_expert_used_count")+ # Granite 4.0 H writes its expert width as feed_forward_length.+ n_ff_exp = dim("_expert_feed_forward_length") or (+ dim("_feed_forward_length") if dim("_n_experts") else 0+ )+ if n_used > 0 and n_ff_exp > 0:+ shared = dim("_expert_shared_feed_forward_length") or (+ n_ff_exp * dim("_expert_shared_count")+ )+ widths.append(n_used * (2 * n_embd + 3 * n_ff_exp) + 3 * shared)+ inner = dim("_ssm_inner_size")+ if inner > 0:+ conv_in = inner + 2 * (dim("_ssm_group_count") or 1) * dim("_ssm_state_size")+ widths.append(4 * conv_in)+ per_layer_input = dim("_embedding_length_per_layer_input") * dim("_n_layers")+ return max(widths) + per_layer_input+def _estimate_compute_buffer_bytes(self,*,n_ubatch: Optional[int] = None,n_parallel: int = 1,per_device_tensor: bool = False,+ vocab_ceiling: Optional[int] = None,) -> int:- """Per-device compute-graph buffer (bytes) from GGUF dims: a vocab-width- output buffer + activation scratch. Context-independent; scales with- ``--parallel`` (serving slots). Tensor mode materializes it on every device.- A slight upper bound over measured allocations; 0 when dims are missing."""- n_vocab = self._vocab_size or 0- n_embd = self._embedding_length or 0- if n_vocab <= 0 or n_embd <= 0:+ """Per-device compute-graph buffer (bytes) that does not grow with context.++ Includes activations, output rows, sliding-window masks and recurrent state.+ Since b9460 (#23861), output rows are capped at min(ubatch, slots); older+ builds reserve ubatch rows. Extra verification rows belong to the MTP reserve.++ Tensor splits use the same buffer per device, so per_device_tensor does not+ change the result. Return 0 for missing dimensions; vocab_ceiling supplies+ a fallback vocabulary size."""+ width = self._compute_activation_width()+ n_vocab = self._vocab_size or vocab_ceiling or 0+ if width <= 0 or n_vocab <= 0:return 0ub = max(1,int(self._DEFAULT_N_UBATCH if n_ubatch is None else n_ubatch),)par = max(1, int(n_parallel))- out_buffer = n_vocab * ub * 4 # f32 output/logits buffer- act_scratch = 4 * n_embd * ub * 4 # a few resident hidden-width buffers- if per_device_tensor:- # Output + comm/staging materialized on every device, every slot.- compute = 2 * act_scratch + out_buffer * par+ activations = width * ub * 4+ if self.is_embedding_gguf:+ outputs = (self._embedding_length or 0) * ub * 4+ elif getattr(self, "_reserves_micro_batch_outputs", False):+ # Before ggml-org/llama.cpp#23861: a row for every micro-batch token, once.+ # Measured on b9415 Vulkan: Gemma 4 E2B 515 MiB at ubatch 512, 137 on b10909.+ outputs = n_vocab * ub * 4else:- # Each extra concurrent slot adds one output buffer (chat decode sizes- # ~one logit row per slot). Embedding mode outputs the whole first- # micro-batch too, so it pays for every slot rather than slots past one.- # Matches measured chat {1:36,2:492,4:1388,8:3220} MiB.- output_slots = par if self.is_embedding_gguf else max(0, par - 1)- compute = act_scratch + out_buffer * output_slots- return int(compute * self._COMPUTE_BUFFER_SAFETY)+ outputs = n_vocab * min(ub, par) * 4+ swa_mask = 0+ sliding_window = self._effective_sliding_window()+ if sliding_window > 0:+ swa_mask = _pad_kv_cells(sliding_window * par + ub) * ub * 2+ # A recurrent block works on every slot's state at once: Qwen3.5 4B's buffer+ # grows 19.5 MiB from one slot to eight at the same micro-batch.+ inner = getattr(self, "_ssm_inner_size", None) or 0+ recurrent = 0+ if inner > 0:+ state = getattr(self, "_ssm_state_size", None) or 0+ conv_in = inner + 2 * (getattr(self, "_ssm_group_count", None) or 1) * state+ recurrent = par * (inner * state + 3 * conv_in) * 4+ return int((activations + outputs + recurrent) * self._COMPUTE_BUFFER_SAFETY + swa_mask)++ def _effective_sliding_window(self) -> int:+ """The sliding window when some layer really slides, else 0. Phi-4-mini+ declares a 262144 window over a 131072 context and no pattern, which bounds+ nothing."""+ window = getattr(self, "_sliding_window", None) or 0+ if window <= 0:+ return 0+ pattern = getattr(self, "_sliding_window_pattern", None)+ if pattern is not None:+ return window if any(pattern) else 0+ context = getattr(self, "_context_length", None) or 0+ return window if not context or window < context else 0++ def _attention_dequant_width(self) -> int:+ """K+V elements per cell copied to f16 from a quantized cache.++ Use the widest global layer: sliding-window layers do not scale with+ context. MLA uses compressed K only. Fall back to n_embd without head dims."""+ n_heads = getattr(self, "_n_heads", None)+ n_kv = getattr(self, "_n_kv_heads", None) or n_heads+ kv_lora_rank = getattr(self, "_kv_lora_rank", None)+ key_len = getattr(self, "_kv_key_length", None)+ val_len = getattr(self, "_kv_value_length", None)+ if kv_lora_rank is not None:+ rope_dim = getattr(self, "_key_length_mla", None) or 64+ return (getattr(self, "_n_kv_heads", None) or 1) * (key_len or kv_lora_rank + rope_dim)+ if key_len is None or val_len is None:+ if not (n_heads and n_kv):+ return getattr(self, "_embedding_length", None) or 0+ return 2 * n_kv * self._legacy_head_dim()+ by_layer = getattr(self, "_n_kv_heads_by_layer", None)+ pattern = (+ getattr(self, "_sliding_window_pattern", None)+ if getattr(self, "_sliding_window", None)+ else None+ )+ widest = 0+ for i, heads in enumerate(by_layer or ()):+ if not (pattern is not None and i < len(pattern) and pattern[i]):+ widest = max(widest, heads)+ return (widest or n_kv or 1) * (key_len + val_len)def _compute_buffer_ctx_bytes(self,@@ -15769,21 +15977,18 @@ class LlamaCppBackend:cache_type_kv: Optional[str] = None,*,layer_split: bool = False,+ flash_attn: bool = True,+ n_parallel: int = 1,) -> int:- """Context-linear growth of the per-device compute buffer (bytes), charged- on top of the flat ``_estimate_compute_buffer_bytes``. The flash-attn KQ- mask + attention scratch scale ~linearly with context and with the micro-- batch; the flat term only covers ctx -> 0. A quantized KV cache adds a- context-sized dequant scratch that scales with n_embd; f16/bf16/f32 pays only- the KQ mask, a flat n_ubatch*2 bytes per context token. ``cache_type_kv`` None- -> f16 (llama.cpp's default; env-set types are adopted by the caller via- ``_env_main_cache_type_for_budget``). Returns 0 when dims are missing or- ``n_ctx`` <= 0.-- ``layer_split`` adds the extra KQ-mask copies a multi-device split pays (see- ``_CTX_COMPUTE_SPLIT_MULT``) on every architecture except deepseek4, whose own- rate already subsumes them. Tensor mode passes False, having been measured- separately at the single-device rate (see ``_plan_tensor_parallel``)."""+ """Per-device context-dependent scratch, added to the flat compute buffer.++ Includes the KQ mask and quantized-cache dequantization. Without flash+ attention, also charge float32 scores and a float32 mask. Layer splits add+ scheduler copies, with sliding-window masks sized by n_parallel; tensor+ splits pass layer_split=False. DeepSeek and Inkling use separate rates.++ None cache type means f16; callers resolve environment overrides.+ Return 0 without n_embd or a positive context."""n_embd = self._embedding_length or 0if n_embd <= 0 or n_ctx <= 0:return 0@@ -15791,10 +15996,11 @@ class LlamaCppBackend:1,int(self._DEFAULT_N_UBATCH if n_ubatch is None else n_ubatch),)- # One KQ mask copy ([n_kv, n_ubatch] f16), n_embd-independent. Every rate below- # is a single-GPU total containing exactly one, so a split adds the extra copies.- mask_rate = ub * 2 * self._CTX_COMPUTE_F16_MASK_SAFETY- split_extra = (self._CTX_COMPUTE_SPLIT_MULT - 1) * mask_rate if layer_split else 0.0+ # One KQ mask copy ([n_kv, n_ubatch]), n_embd-independent: f16 under flash+ # attention, f32 without it (build_attn_inp_kq_mask). Every rate below is a+ # single-GPU total containing exactly one, so a split adds the extra copies.+ mask_rate = ub * (2 if flash_attn else 4)+ split_extra = (self._CTX_COMPUTE_SPLIT_MULT - 1) * mask_rate if layer_split else 0if getattr(self, "_architecture", None) == "deepseek4":# DSV4 indexer/CSA buffer (see constants): flat + linear, ub-scaled. Fires# for any KV type -- the indexer scratch is present even with an f16 cache.@@ -15820,23 +16026,27 @@ class LlamaCppBackend:# Both paths allocate the mask, and 8192 has only ~1.5x headroom over the# 5.6 KiB/tok banded measurement -- a split alone would overrun it.return int(rate * n_ctx * ub_scale + split_extra * n_ctx)+ per_tok = mask_rateif _kv_bytes_per_elem(cache_type_kv) < 2.0:- # Quantized cache: the dequant scratch dominates and scales with n_embd.- # MLA (compressed KV) needs far less of it: measured 0.94 x n_embd on- # GLM-5.2 and Kimi-K2.7 vs up to 2.02x on regular attention.- ub_scale = ub / self._DEFAULT_N_UBATCH- rate = (- self._CTX_COMPUTE_BYTES_PER_EMBD_MLA- if self._key_length_mla- else self._CTX_COMPUTE_BYTES_PER_EMBD- )- per_tok = rate * n_embd * ub_scale- else:- # f16/bf16/f32: only the KQ mask.- per_tok = mask_rate- # Only the mask goes 1x -> 4x, not the dequant scratch, so ADD the extra copies:- # max() would under-reserve the quantized path by the whole dequant scratch.- return int((per_tok + split_extra) * n_ctx)+ # Quantized cache: the f16 K/V copy, independent of the micro-batch.+ per_tok += self._attention_dequant_width() * 2+ if not flash_attn:+ n_head = getattr(self, "_n_heads", None) or max(1, n_embd // 64)+ per_tok += ub * n_head * 4+ if getattr(self, "_kv_lora_rank", None) is not None:+ # MLA decompresses a float32 copy of the latent cache to attend over it:+ # GLM-4.7-Flash grows 44 KiB per cell at ubatch 512, 2 KiB past the scores.+ per_tok += self._attention_dequant_width() * 4+ # Scheduler copies replicate masks and hidden states, but not dequant scratch.+ # Sliding-window copies use the same per-slot cell count as the flat buffer.+ split_fixed = 0+ if layer_split:+ split_fixed = self._CTX_COMPUTE_SPLIT_MULT * n_embd * ub * 4+ sliding_window = self._effective_sliding_window()+ if sliding_window > 0:+ swa_cells = _pad_kv_cells(sliding_window * max(1, int(n_parallel)) + ub)+ split_fixed += (self._CTX_COMPUTE_SPLIT_MULT - 1) * swa_cells * mask_rate+ return int((per_tok + split_extra) * n_ctx + split_fixed)def _slots_that_fit_on_gpu(self,@@ -15854,6 +16064,7 @@ class LlamaCppBackend:kv_unified: bool = True,flash_attn: bool = True,split_extra_bytes: int = 0,+ split_extra_for_slots: Optional[Callable[[int], int]] = None,ubatch_for_slots: Optional[Callable[[int], Optional[int]]] = None,mtp_bytes_for_slots: Optional[Callable[[int, Optional[int]], int]] = None,ctx_checkpoints: int = 0,@@ -15886,7 +16097,8 @@ class LlamaCppBackend:layers charges per slot. It takes the candidate micro-batch as well as the slotcount, since a reduced candidate lowers the batch floor and so the ubatch too;pricing the reserve at the requested pair over-charged every candidate, so one that- fits could be rejected and the load kept --fit."""+ fits could be rejected and the load kept --fit. ``split_extra_for_slots(slots)``+ re-prices the split step, whose sliding-window masks also scale with slots."""for slots in range(n_parallel if include_requested else n_parallel - 1, 0, -1):_ub = ubatch_for_slots(slots) if ubatch_for_slots else n_ubatchcb = self._estimate_compute_buffer_bytes(@@ -15916,7 +16128,9 @@ class LlamaCppBackend:total_by_idx = total_by_idx,per_device_overhead_bytes = per_device_overhead_bytes,min_gpus = min_gpus,- split_extra_bytes = split_extra_bytes,+ split_extra_bytes = (+ split_extra_for_slots(slots) if split_extra_for_slots else split_extra_bytes+ ),)if not use_fit:return gpu_indices, False, slots@@ -16580,6 +16794,11 @@ class LlamaCppBackend:self._pooling_type = Noneself._gguf_path = gguf_pathself._feed_forward_length = None+ self._expert_used_count = None+ self._expert_feed_forward_length = None+ self._expert_shared_feed_forward_length = None+ self._expert_shared_count = None+ self._embedding_length_per_layer_input = Noneself._vocab_size = Noneself._kv_key_length = Noneself._kv_value_length = None@@ -16700,6 +16919,11 @@ class LlamaCppBackend:f"{arch}.embedding_length": "embedding_length",f"{arch}.pooling_type": "pooling_type",f"{arch}.feed_forward_length": "feed_forward_length",+ f"{arch}.expert_used_count": "expert_used_count",+ f"{arch}.expert_feed_forward_length": "expert_feed_forward_length",+ f"{arch}.expert_shared_feed_forward_length": "expert_shared_feed_forward_length",+ f"{arch}.expert_shared_count": "expert_shared_count",+ f"{arch}.embedding_length_per_layer_input": "embedding_length_per_layer_input",f"{arch}.attention.key_length": "kv_key_length",f"{arch}.attention.value_length": "kv_value_length",f"{arch}.attention.sliding_window": "sliding_window",@@ -16762,6 +16986,8 @@ class LlamaCppBackend:elif attr == "sliding_window_pattern" and val_a is not None:self._sliding_window_pattern = [bool(x) for x in val_a]sliding_window_pattern_period = None+ elif attr == "feed_forward_length" and val_a:+ self._feed_forward_length = max(int(x) for x in val_a)else:self._gguf_skip_value(f, vtype)else:@@ -22342,6 +22568,7 @@ class LlamaCppBackend:raise LlamaServerNotFoundError(LLAMA_SERVER_NOT_FOUND_DETAIL)server_caps = _launch_caps(binary)+ self._reserves_micro_batch_outputs = self.reserves_micro_batch_outputs(binary)# Outside ``self._lock`` so /unload, /cancel, /status aren't# blocked. ``unload_model`` also records the kill, so the@@ -22474,6 +22701,13 @@ class LlamaCppBackend:None if _cache_type_from_env else cache_type_kv,extra_args,)+ # Price compute for the launch; KV separately covers the no-flash retry.+ _launch_flash_attn = _planned_flash_attn(+ extra_args,+ bool(server_caps.get("supports_flash_attn", True)),+ _planned_cache_pair[1],+ self._architecture,+ )# A user --split-mode in extras last-wins-overrides the toggle, and# an inherited tensor LLAMA_ARG_SPLIT_MODE flips it on (the child# would run tensor unbudgeted otherwise). The duplicate-load matchers@@ -23262,7 +23496,11 @@ class LlamaCppBackend:flash_attn = planned_flash_attn,)- def _cc_bytes(ctx: int, n_gpus: int = 1) -> int:+ def _cc_bytes(+ ctx: int,+ n_gpus: int = 1,+ slots: int = 0,+ ) -> int:# Context-linear compute-buffer growth (flash-attn KQ mask +# attention scratch); the flat _compute_buffer_pipeline folded# into model_size_fit only covers ctx -> 0. Charged per@@ -23276,18 +23514,21 @@ class LlamaCppBackend:# under-reserves ~(n-1)x it (e.g. Qwen3.5-397B on 3 GPUs). The# per-device rate also steps up once split, unless llama.cpp# declines the pipeline parallelism that causes the step.+ # A nonzero ``slots`` prices that candidate count and its micro-batch.return max(1, n_gpus) * self._compute_buffer_ctx_bytes(ctx,- _effective_ubatch,+ _ubatch_for_slots(slots) if slots else _effective_ubatch,_scratch_cache_type_kv,layer_split = n_gpus > 1 and not _pipeline_parallel_off,+ flash_attn = _launch_flash_attn,+ n_parallel = slots or n_parallel,)- def _cc_split_extra(ctx: int) -> int:+ def _cc_split_extra(ctx: int, slots: int = 0) -> int:# Per-device step from the single-device rate to the split one,# for the paths that must select GPUs before they know the# count. 0 when llama.cpp declines pipeline parallelism.- return max(0, _cc_bytes(ctx, 2) // 2 - _cc_bytes(ctx))+ return max(0, _cc_bytes(ctx, 2, slots) // 2 - _cc_bytes(ctx, 1, slots))# Layer-split compute buffer (one lump; tensor mode reserves it# per device in _plan_tensor_parallel). Context-independent, so@@ -23305,9 +23546,9 @@ class LlamaCppBackend:)# ...and the tensor-mode rate, reserved on EVERY selected device:- # the f32 output buffer and the comm staging are replicated rather- # than split, so the layer lump above is a fraction of what a tensor- # split allocates. Same flat fallback when the dims are missing.+ # each holds the whole buffer rather than a share of it, so the layer+ # lump above is one device of what a tensor split allocates. Same flat+ # fallback when the dims are missing._compute_buffer_tensor = self._estimate_compute_buffer_bytes(n_ubatch = _effective_ubatch,n_parallel = n_parallel,@@ -23451,11 +23692,12 @@ class LlamaCppBackend:# displaced does not run. The flat fraction below is gated on# this; the byte-accurate callback was not, so the fit went on# charging VRAM no drafter allocates.- # One term survives: a recurrent target's rollback snapshots sit- # in the TARGET context, so pinning the drafter to CPU does not- # move them. Charge those alone. Flat in ctx (the- # state is per-slot), hence the same _np/_n_ubatch keywords the- # replaced callback takes, so _mtp_bytes can still re-price slots.+ # Two terms survive: a recurrent target's rollback snapshots and+ # the verification output rows sit in the TARGET context, so+ # pinning the drafter to CPU does not move them. Charge those+ # alone. Flat in ctx (both are per-slot), hence the same+ # _np/_n_ubatch keywords the replaced callback takes, so+ # _mtp_bytes can still re-price slots.def _cpu_draft_target_state(_ctx: int,_np: int = n_parallel,@@ -23463,9 +23705,10 @@ class LlamaCppBackend:_n: int = _mtp_eff_n_max,_rollback: bool = _target_rollback,) -> int:- if not _rollback or _n <= 0:+ if _n <= 0:return 0- return self._rollback_state_bytes(_np) * _n+ rollback = self._rollback_state_bytes(_np) * _n if _rollback else 0+ return rollback + self._spec_verify_rows_bytes(_np, _n)mtp_overhead_fn = (_cpu_draft_target_state@@ -24597,7 +24840,7 @@ class LlamaCppBackend:swa_full = swa_full,kv_unified = planned_kv_unified,flash_attn = planned_flash_attn,- split_extra_bytes = _cc_split_extra(_reduce_ctx),+ split_extra_for_slots = lambda s: _cc_split_extra(_reduce_ctx, s),ubatch_for_slots = _ubatch_for_slots,mtp_bytes_for_slots = lambda s, ub: _mtp_bytes(_reduce_ctx, s, ub),ctx_checkpoints = _fit_ctx_checkpoints,diff --git a/studio/backend/routes/inference.py b/studio/backend/routes/inference.pyindex ee169f95947..9ceac27c872 100644--- a/studio/backend/routes/inference.py+++ b/studio/backend/routes/inference.py@@ -1613,12 +1613,12 @@ try:_emitted_n_batch,_extra_args_draft_device_pin,_extra_args_n_ubatch,- _flash_attn_enabled_from_args,_hf_offline_if_unreachable,_hf_offline_if_unreachable_for,_kv_bytes_per_elem,_kv_unified_from_args,_metal_device_is_paravirtual,+ _planned_flash_attn,_planned_main_cache_types,_planned_scratch_cache_type,_swa_full_from_args_or_env,@@ -1675,12 +1675,12 @@ except ImportError:_emitted_n_batch,_extra_args_draft_device_pin,_extra_args_n_ubatch,- _flash_attn_enabled_from_args,_hf_offline_if_unreachable,_hf_offline_if_unreachable_for,_kv_bytes_per_elem,_kv_unified_from_args,_metal_device_is_paravirtual,+ _planned_flash_attn,_planned_main_cache_types,_planned_scratch_cache_type,_swa_full_from_args_or_env,@@ -10494,10 +10494,6 @@ def _cached_repo_gguf_bytes(repo: str, hint: str = "") -> int:return 0-# Upper bound on any current tokenizer, used to rebuild the compute buffer when a-# truncated header drops the token array. Above Llama 4 / Gemma 3 (256k), the widest shipping.-_ASSUMED_MAX_VOCAB = 262144-# Canonical speculative modes whose launch opens no drafter file at all:# _build_speculative_flags returns out of each before it reaches --model-draft._SPEC_MODES_WITHOUT_A_DRAFTER = frozenset({"off", "ngram", "ngram-simple"})@@ -10641,7 +10637,7 @@ def _gguf_runtime_bytes(over-reserves on purpose; a panel quoting a number to a user wants the otherone, since a smaller ``-c`` in the extras is the context the user gets."""try:- from core.inference.llama_cpp import _batch_ubatch_for_mmproj+ from core.inference.llama_cpp import _ASSUMED_MAX_VOCAB, _batch_ubatch_for_mmprojfrom core.inference.llama_cpp import effective_ctx_checkpoints_for_capsfrom core.inference.llama_server_args import (parse_ctx_override,@@ -10651,6 +10647,14 @@ def _gguf_runtime_bytes(probe = _probe_backend()probe._model_identifier = model_identifierprobe._read_gguf_metadata(gguf_path)+ # An older llama.cpp reserves an output row per micro-batch token; priced as+ # the loader prices it. getattr: a stand-in backend has no probe.+ try:+ probe._reserves_micro_batch_outputs = bool(+ getattr(LlamaCppBackend, "reserves_micro_batch_outputs", lambda: False)()+ )+ except Exception as _rows_exc:+ logger.debug("llama-server build probe failed: %s", _rows_exc)# Price the same batch sizes used by load_model.n_batch, n_ubatch = _batch_ubatch_for_mmproj(0 if is_diffusion else launch_required_ubatch,@@ -10715,7 +10719,6 @@ def _gguf_runtime_bytes(# llama-server b10632, Qwen3-0.6B at 32k with -ctk/-ctv q4_0: 2296 MiB reserved# against 1008 MiB allocated, identical on the CPU and Vulkan builds.planned_v_type = planned_cache_types[1]- v_forces_flash_attn = planned_v_type not in {"f16", "bf16", "f32"}# the loader raises --batch-size to max(slots, 2) before launch, and llama.cpp# caps the micro-batch against it, so budget from the emitted value. Diffusion# takes neither flag, and SWA metadata prices the KV against the micro-batch,@@ -10787,24 +10790,23 @@ def _gguf_runtime_bytes(# so the default here is ON, not off. The false arm pads variable-width V# tensors to the model-wide maximum (_max_kv_value_width), which on an# architecture whose global and SWA layers disagree about n_embd_v_gqa- # inflates the whole cache and can call a load that fits an overflow. Resolved- # the way the launch resolves it: the managed default, narrowed by the- # capability probe, then llama.cpp's own env-then-last-wins-argv rule. A- # quantized V still forces it on top of all that, because llama-context.cpp- # turns it on itself rather than refusing the load.- _fa_default = True+ # inflates the whole cache and can call a load that fits an overflow.+ # Resolved the way the launch resolves it, narrowed by the capability probe.+ _fa_supported = Truetry:_fa_caps = LlamaCppBackend.probe_server_capabilities()# Same ``found`` test as the checkpoint probe above: the defaults dict# reports nothing supported, so keying on the flag alone would size every# unprobed host as flash-attention-less.if _fa_caps.get("found") and not _fa_caps.get("supports_flash_attn", True):- _fa_default = False+ _fa_supported = Falseexcept Exception as _fa_exc:logger.debug("flash-attention capability probe failed: %s", _fa_exc)- flash_attn = (- _flash_attn_enabled_from_args(llama_extra_args, default = _fa_default, env = os.environ)- or v_forces_flash_attn+ flash_attn = _planned_flash_attn(+ llama_extra_args,+ _fa_supported,+ planned_v_type,+ getattr(probe, "_architecture", None),)kv = probe._estimate_kv_cache_bytes(ctx,@@ -10854,38 +10856,13 @@ def _gguf_runtime_bytes(devices = max(1, int(n_devices))def _flat_buffer(per_device_tensor: bool) -> int:- """Flat compute buffer, rebuilt when the header is short of a vocab size.-- _estimate_compute_buffer_bytes returns 0 when vocab_size or embedding_length is- missing. Only the first is reachable: _vocab_size is set solely from the- tokenizer.ggml.tokens array length, which a truncated header drops while keeping- the dims. So rebuild from the real formula with a vocab ceiling rather than- substituting the loader's flat reserve. The loader can afford that reserve- because over-reserving there just shrinks the context; here it denies the load.- """- flat = probe._estimate_compute_buffer_bytes(+ """Use a vocabulary ceiling when a truncated header omits the token array."""+ return probe._estimate_compute_buffer_bytes(n_ubatch = effective_ubatch,n_parallel = slots,per_device_tensor = per_device_tensor,+ vocab_ceiling = _ASSUMED_MAX_VOCAB,)- if flat > 0:- return flat- # getattr: a bare backend double carries neither dims nor constants.- n_embd = getattr(probe, "_embedding_length", None) or 0- if n_embd <= 0:- return 0 # nothing to rebuild from; the total is floored below instead- ub = max(1, int(effective_ubatch or probe._DEFAULT_N_UBATCH))- act_scratch = 4 * n_embd * ub * 4- out_buffer = _ASSUMED_MAX_VOCAB * ub * 4- output_slots = (- slots if getattr(probe, "is_embedding_gguf", False) else max(0, slots - 1)- )- raw = (- 2 * act_scratch + out_buffer * slots- if per_device_tensor- else act_scratch + out_buffer * output_slots- )- return int(raw * probe._COMPUTE_BUFFER_SAFETY)if tensor_parallel:# mirrors _plan_tensor_parallel: per-device buffer and ctx growth on every device@@ -10910,6 +10887,8 @@ def _gguf_runtime_bytes(effective_ubatch,cache_type_for_scratch,layer_split = devices > 1 and not pipeline_parallel_off,+ flash_attn = flash_attn,+ n_parallel = slots,))if compute <= 0:@@ -10993,8 +10972,12 @@ def _remote_gguf_compute_reserve_gb(``required_ubatch`` matches the post-download launch."""- # remote dims are unreadable; only the kq mask, linear in ubatch x ctx, can be sized here- from core.inference.llama_cpp import _batch_ubatch_for_mmproj+ # remote dims are unreadable: the kq mask is sized exactly, the activations at a ceiling+ from core.inference.llama_cpp import (+ _ASSUMED_MAX_ACTIVATION_WIDTH,+ _ASSUMED_MAX_VOCAB,+ _batch_ubatch_for_mmproj,+ )from core.inference.llama_server_args import parse_ctx_overriden_batch, n_ubatch = _batch_ubatch_for_mmproj(@@ -11041,33 +11024,25 @@ def _remote_gguf_compute_reserve_gb(and not _pipeline_parallel_disabled_by_args(llama_extra_args, n_layers = None)else 1)- mask_bytes = (- budget_ctx- * effective_ubatch- * 2- * LlamaCppBackend._CTX_COMPUTE_F16_MASK_SAFETY- * devices- * split_mult- )- # The mask is only the context-linear half. The flat half needs dims that are- # unreadable remotely, but its dominant term needs only a vocab ceiling:- # llama.cpp reserves n_vocab * ubatch * 4 per slot past the first (every slot- # under tensor mode), so n_batch = n_ubatch = 32768 on two slots is ~32 GiB the- # mask does not cover, and omitting it let the guard admit an uncached load that- # then OOMs the training job. The activation scratch needs embedding_length and- # stays uncharged: the small half, and over-reserving denies the load outright.- # Scaled per device only in tensor mode, mirroring the local branch. The remote- # header is unknown, so reserve as if it enables embeddings.- _out_slots = max(1, n_parallel)- out_buffer_bytes = (- _ASSUMED_MAX_VOCAB- * effective_ubatch- * 4- * _out_slots- * (devices if tensor_parallel else 1)+ mask_bytes = budget_ctx * effective_ubatch * 2 * devices * split_mult+ # Unknown dimensions require an activation ceiling. Reserve output rows for the+ # maximum assumed draft depth. Tensor mode replicates both on every device.+ activation_bytes = _ASSUMED_MAX_ACTIVATION_WIDTH * effective_ubatch * 4+ output_rows = min(+ effective_ubatch,+ max(1, n_parallel) * (1 + LlamaCppBackend._UNKNOWN_SPEC_DRAFT_N_MAX),+ )+ try:+ if LlamaCppBackend.reserves_micro_batch_outputs():+ output_rows = effective_ubatch+ except Exception as _rows_exc:+ logger.debug("llama-server build probe failed: %s", _rows_exc)+ flat_bytes = (+ (activation_bytes + _ASSUMED_MAX_VOCAB * output_rows * 4)* LlamaCppBackend._COMPUTE_BUFFER_SAFETY+ * (devices if tensor_parallel else 1))- return (mask_bytes + out_buffer_bytes) / (1024**3)+ return (mask_bytes + flat_bytes) / (1024**3)return 0.0@@ -12585,6 +12560,8 @@ def _gguf_memory_breakdown(# The part of that reserve allocated in the TARGET's context rather than the# drafter's, so it stays with the target when the two are placed apart.target_spec_bytes = 0+ # The verification output rows in that reserve: target compute, not target KV.+ target_verify_bytes = 0drafter_on_gpu = host_drafter_bytes == 0charged_drafter = _charged_drafter_path(config, host_drafter_bytes or gpu_drafter_bytes, extras = extras@@ -12659,7 +12636,9 @@ def _gguf_memory_breakdown(n_ubatch = runtime.n_ubatch,flash_attn = bool(_draft_v_type) and _draft_v_type not in {"f16", "bf16", "f32"},)- drafter_runtime_bytes = (+ # The draft context's decode graph is charged on top, as the loader charges it:+ # the helper prices only what that graph grows beyond this floor.+ drafter_runtime_bytes = getattr(probe, "_MTP_DRAFT_COMPUTE_BYTES", 0) + (probe._estimate_mtp_overhead_bytes(runtime.n_ctx,spec_draft_n_max = draft_n_max,@@ -12691,6 +12670,10 @@ def _gguf_memory_breakdown(probe._mamba_recurrent_state_bytes(runtime.n_parallel) * draft_n_max)target_spec_bytes = min(target_spec_bytes, drafter_runtime_bytes)+ target_verify_bytes = min(+ probe._spec_verify_rows_bytes(runtime.n_parallel, draft_n_max),+ drafter_runtime_bytes - target_spec_bytes,+ )layer_count = runtime.layer_countgpu_fraction = _gguf_offloaded_layer_fraction(@@ -12766,7 +12749,8 @@ def _gguf_memory_breakdown(if gpu_fraction > 0.0:# Compute buffers land on the devices running layers.gpu_bytes += runtime.compute_bytes- # Target-side spec terms are target KV allocations and follow the target cache. The+ # Target-side spec KV follows the target cache, and its verification rows follow the+ # target's layers like the other compute buffers. The# drafter's own KV follows the drafter: a separate one does not inherit# --gpu-layers (llama.cpp overwrites it with the draft placement, default auto), so# at --gpu-layers 0 it is still on the GPU holding that cache. An embedded head is@@ -12774,8 +12758,14 @@ def _gguf_memory_breakdown(# drafter_on_gpu said yes even for a CPU-placed target -- but it is part of the# target's tensors and takes the target's context, so it goes where the target goes._drafter_on_gpu = (gpu_fraction > 0.0) if embedded_mtp else drafter_on_gpu- drafter_runtime_gpu_bytes = (target_spec_bytes if kv_on_gpu else 0) + (- (drafter_runtime_bytes - target_spec_bytes) if _drafter_on_gpu else 0+ drafter_runtime_gpu_bytes = (+ (target_spec_bytes if kv_on_gpu else 0)+ + (target_verify_bytes if gpu_fraction > 0.0 else 0)+ + (+ (drafter_runtime_bytes - target_spec_bytes - target_verify_bytes)+ if _drafter_on_gpu+ else 0+ ))gpu_bytes += drafter_runtime_gpu_bytes# The encoder's buffers sit with the projector, so --no-mmproj-offload takes themdiff --git a/studio/backend/routes/models.py b/studio/backend/routes/models.pyindex 2ed8bb52100..fb5190b2430 100644--- a/studio/backend/routes/models.py+++ b/studio/backend/routes/models.py@@ -4092,6 +4092,10 @@ async def get_kv_cache_estimate(# GPU, 3 without) and the rollback state is multiplied by it. An explicit 0 is still honoured.spec_draft_n_max = _effective_draft_n_max,)+ # Plus the draft decode graph's floor, which the helper leaves to+ # the loader's soft overhead.+ if spec is not None:+ spec += be._MTP_DRAFT_COMPUTE_BYTESexcept Exception as e:logger.debug(f"mtp overhead estimate failed for '{repo_id}' {quant}: {e}")diff --git a/studio/backend/tests/test_batch_sizes_per_load.py b/studio/backend/tests/test_batch_sizes_per_load.pyindex de13abca9c2..d6a7c83fb78 100644--- a/studio/backend/tests/test_batch_sizes_per_load.py+++ b/studio/backend/tests/test_batch_sizes_per_load.py@@ -403,12 +403,8 @@ def test_emitted_batch_clears_both_llama_server_floors(n_batch, n_parallel, expedef test_budgets_use_the_raised_batch_not_the_requested_one():- """llama.cpp caps the micro-batch against the batch it is GIVEN- (cparams.n_ubatch = min(cparams.n_batch, n_ubatch or n_batch)), and the loader- raises --batch-size to max(slots, 2) before launch. Budgeting from the requested- value instead planned n_batch=1 / n_ubatch=64 / 64 slots at a micro-batch of 1- and launched it at 64: ~2.2 GB of compute buffer the training guard never- charged, and an auto-fit sized for a graph the server does not build."""+ """Budget the micro-batch after the loader raises batch size to max(slots, 2).+ A batch of 1 with 64 slots and ubatch 64 must be priced at ubatch 64."""from unittest.mock import patchfrom routes import inference as route@@ -429,7 +425,17 @@ def test_budgets_use_the_raised_batch_not_the_requested_one():"/x.gguf", 32768, n_parallel = 64, n_batch = 1, n_ubatch = 1)assert raised == pytest.approx(explicit, abs = 0.01)- assert raised > unraised + 2.0+ # 63 more micro-batch tokens of activations and KQ mask, and 63 more output rows.+ probe = LlamaCppBackend()+ _header_reader(**_QWEN3_8B)(probe, "/x.gguf")+ grown = (+ probe._estimate_compute_buffer_bytes(n_ubatch = 64, n_parallel = 64)+ + probe._compute_buffer_ctx_bytes(32768, 64)+ - probe._estimate_compute_buffer_bytes(n_ubatch = 1, n_parallel = 64)+ - probe._compute_buffer_ctx_bytes(32768, 1)+ ) / 1024**3+ assert grown > 0.05+ assert raised - unraised == pytest.approx(grown, abs = 0.001)# the remote branch shares the floor: no dims, but the kq mask still growsfrom types import SimpleNamespace@@ -648,8 +654,9 @@ def test_the_recorded_micro_batch_is_derived_from_the_slots_that_launched():and isinstance(node.func, ast.Name)and node.func.id == "_ubatch_for_slots"]- # sizing pass, embedding slot clamp, fit-time reduction, then the post-launch record- assert len(calls) == 4, f"expected four re-derivations, found {len(calls)}"+ # sizing pass, embedding slot clamp, candidate split pricing, fit-time reduction, then+ # the post-launch record+ assert len(calls) == 5, f"expected five re-derivations, found {len(calls)}"# the record must not reuse the sizing pass's valuecompact = "".join(src.split())assert "self._n_ubatch=max(0,int(self._DEFAULT_N_UBATCHif_launched_ubatchisNone" in compact@@ -660,12 +667,27 @@ def test_the_recorded_micro_batch_is_derived_from_the_slots_that_launched():)+def test_the_guard_prices_the_output_rows_of_the_build_it_launches():+ """An older llama.cpp (before ggml-org/llama.cpp#23861) reserves an output row for+ every micro-batch token; the guard asks the binary rather than assuming the cap."""+ from unittest.mock import patch++ from routes import inference as route++ def _gb(older):+ with (+ patch.object(LlamaCppBackend, "_read_gguf_metadata", _header_reader(**_QWEN3_8B)),+ patch.object(LlamaCppBackend, "reserves_micro_batch_outputs", lambda *a, **k: older),+ ):+ return route._estimate_gguf_kv_gb("/x.gguf", 8192, n_parallel = 1, n_ubatch = 2048)++ rows = (2048 - 1) * 151936 * 4 * LlamaCppBackend._COMPUTE_BUFFER_SAFETY / 1024**3+ assert _gb(True) - _gb(False) == pytest.approx(rows, abs = 0.001)++def test_the_remote_guard_charges_the_flat_output_buffer():- """The KQ mask is only the context-linear half of the compute buffer. llama.cpp also- reserves n_vocab * ubatch * 4 per slot past the first, which is context-INdependent and- dwarfs the mask at large settings: n_batch = n_ubatch = 32768 on two slots is ~32 GiB- the mask never covers. Omitting it remotely let the coexistence guard admit an uncached- load that then OOMs the training job it exists to protect."""+ """Unknown headers still need an activation reserve beyond the KQ mask.+ Activations scale with ubatch; extra slots add output rows."""from types import SimpleNamespacefrom unittest.mock import patch@@ -698,11 +720,11 @@ def test_the_remote_guard_charges_the_flat_output_buffer():# Unset fields still launch at llama.cpp's known default 512-token micro-batch.assert blank_1 > 1.5- assert blank_4 > blank_1 + 1.5- # The remote header may enable embeddings, so the first output buffer is charged.+ # Slots add output rows, far below a second activation reserve.+ assert blank_1 < blank_4 < blank_1 + 0.5+ # The activation ceiling at a 32768-token micro-batch.assert big_1 > blank_1 + 30.0- # 262144 * 32768 * 4 = 32 GiB for the second slot too.- assert big_2 > big_1 + 30.0+ assert big_1 < big_2 < big_1 + 1.0# and it stays proportionate where the values are ordinaryassert typical_4 < 4.0@@ -730,7 +752,7 @@ def test_the_remote_guard_charges_the_flat_output_buffer():layer_1, layer_2 = _split(False, 1), _split(False, 2)tensor_1, tensor_2 = _split(True, 1), _split(True, 2)- # the second device adds only the ctx-linear mask, not another 32 GiB of logits+ # the second device adds only the ctx-linear mask, not another 35 GiB of activationsassert layer_2 - layer_1 < 30.0# tensor mode replicates the whole buffer on every card, so it does roughly doubleassert tensor_2 > tensor_1 * 1.8diff --git a/studio/backend/tests/test_chat_load_during_training.py b/studio/backend/tests/test_chat_load_during_training.pyindex 25015777baf..b52624dad74 100644--- a/studio/backend/tests/test_chat_load_during_training.py+++ b/studio/backend/tests/test_chat_load_during_training.py@@ -1306,12 +1306,7 @@ class TestValidateRefusesDuringTraining(unittest.TestCase):class TestRemoteGgufComputeReserve(unittest.TestCase):- """The compute reserve a remote GGUF estimate carries.-- Every other caller here patches it to zero to assert exact GB totals, so its arithmetic goes- unasserted even though it decides whether a load is refused. These pin the shape, not the- magnitude: retuning a safety factor keeps them passing, dropping a term does not.- """+ """Check the remote GGUF compute reserve without the other tests' zero-cost stub."""@classmethoddef setUpClass(cls):@@ -1329,27 +1324,56 @@ class TestRemoteGgufComputeReserve(unittest.TestCase):with patch.dict(os.environ, env, clear = True):return self.route._remote_gguf_compute_reserve_gb(max_seq_length = 4096, **kwargs)- def test_a_single_slot_still_reserves_an_output_buffer(self):- """llama-server allocates an output buffer for its one slot, but the old- max(0, n_parallel - 1) count reserved nothing there.-- The total is spelled out absolutely rather than compared against a neighbouring call: the- reserve is linear in slot count, so any two samples are one buffer apart under both the- old formula and the new one, and only an absolute anchor sees the floor.- """- from core.inference.llama_cpp import LlamaCppBackend+ def test_reserve_is_the_mask_plus_an_activation_ceiling(self):+ """Charge the activation ceiling once per micro-batch and output rows per slot."""+ from core.inference.llama_cpp import (+ _ASSUMED_MAX_ACTIVATION_WIDTH,+ _ASSUMED_MAX_VOCAB,+ LlamaCppBackend,+ )ubatch = LlamaCppBackend._DEFAULT_N_UBATCH- mask = 4096 * ubatch * 2 * LlamaCppBackend._CTX_COMPUTE_F16_MASK_SAFETY- per_slot = (- self.route._ASSUMED_MAX_VOCAB * ubatch * 4 * LlamaCppBackend._COMPUTE_BUFFER_SAFETY+ mask = 4096 * ubatch * 2++ def expected(slots):+ rows = min(ubatch, slots * (1 + LlamaCppBackend._UNKNOWN_SPEC_DRAFT_N_MAX))+ flat = _ASSUMED_MAX_ACTIVATION_WIDTH * ubatch * 4 + _ASSUMED_MAX_VOCAB * rows * 4+ return (mask + flat * LlamaCppBackend._COMPUTE_BUFFER_SAFETY) / (1024**3)++ self.assertAlmostEqual(self._reserve(n_parallel = 1), expected(1), places = 6)+ self.assertAlmostEqual(self._reserve(n_parallel = 2), expected(2), places = 6)+ # A second slot costs rows, far less than a second activation reserve.+ self.assertLess(+ self._reserve(n_parallel = 2) - self._reserve(n_parallel = 1),+ _ASSUMED_MAX_ACTIVATION_WIDTH * ubatch * 4 / (1024**3) / 4,)- self.assertAlmostEqual(self._reserve(n_parallel = 1), (mask + per_slot) / (1024**3), places = 6)- # One more buffer for a second slot, pinning the count as well as the floor.++ def test_an_older_build_reserves_a_row_per_micro_batch_token(self):+ """Before ggml-org/llama.cpp#23861 the output rows cover the whole micro-batch,+ which the remote guard charges when the binary reports such a build."""+ from core.inference.llama_cpp import _ASSUMED_MAX_VOCAB, LlamaCppBackend++ ubatch = LlamaCppBackend._DEFAULT_N_UBATCH+ with patch.object(LlamaCppBackend, "reserves_micro_batch_outputs", lambda *a, **k: False):+ current = self._reserve(n_parallel = 1)+ with patch.object(LlamaCppBackend, "reserves_micro_batch_outputs", lambda *a, **k: True):+ older = self._reserve(n_parallel = 1)+ rows = (ubatch - (1 + LlamaCppBackend._UNKNOWN_SPEC_DRAFT_N_MAX)) * _ASSUMED_MAX_VOCAB * 4self.assertAlmostEqual(- self._reserve(n_parallel = 2), (mask + 2 * per_slot) / (1024**3), places = 6+ older - current, rows * LlamaCppBackend._COMPUTE_BUFFER_SAFETY / (1024**3), places = 6)+ def test_tensor_mode_replicates_the_whole_buffer_on_every_device(self):+ """Tensor mode reserves the mask, activations and output rows on each device."""+ from core.inference.llama_cpp import LlamaCppBackend+ for older_build in (False, True):+ with patch.object(+ LlamaCppBackend, "reserves_micro_batch_outputs", lambda *a, _o = older_build, **k: _o+ ):+ single = self._reserve(n_parallel = 2)+ tensor = self._reserve(n_parallel = 2, n_devices = 2, tensor_parallel = True)+ self.assertAlmostEqual(tensor, 2 * single, places = 6)+def test_diffusion_reserves_nothing(self):"""The default micro-batch is a llama-server notion: a diffusion estimate has no ubatch toassume, and charging one would refuse loads that fit."""@@ -2829,6 +2853,7 @@ class TestEstimateGgufRequiredGb(unittest.TestCase):n_ubatch = None,n_parallel = 1,per_device_tensor = False,+ vocab_ceiling = None,):seen["compute_n_ubatch"] = n_ubatchreturn 0@@ -2840,6 +2865,8 @@ class TestEstimateGgufRequiredGb(unittest.TestCase):cache_type_kv = None,*,layer_split = False,+ flash_attn = True,+ n_parallel = 1,):return 0diff --git a/studio/backend/tests/test_compute_buffer.py b/studio/backend/tests/test_compute_buffer.pyindex 16381ece51b..727082ab887 100644--- a/studio/backend/tests/test_compute_buffer.py+++ b/studio/backend/tests/test_compute_buffer.py@@ -1,10 +1,7 @@# SPDX-License-Identifier: AGPL-3.0-only# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0-"""Tests for ``_estimate_compute_buffer_bytes``: it scales with ``--parallel``,-tensor exceeds pipeline, and it is a safe upper bound on the allocations measured-on real hardware (Qwen3.6-27B-MTP: parallel 1/2/4/8 -> 36/492/1388/3220 MiB single-GPU, ~600 MiB/device tensor). No GPU, subprocess, or GGUF I/O."""+"""Compute-buffer estimates checked against measured allocations and synthetic layouts."""from __future__ import annotations@@ -85,6 +82,15 @@ def _backend(return b+def _pad(cells):+ return ((cells + 255) // 256) * 256+++def _hidden_copies(embd, ub = 512):+ """The hidden-state copies a layer-split device holds, one per scheduler copy."""+ return LlamaCppBackend._CTX_COMPUTE_SPLIT_MULT * embd * ub * 4++def _backend_from_gguf_local(n_layers = 80,embd = 8192,@@ -110,76 +116,271 @@ def _backend_from_gguf_local(return b-# Measured ground truth (MiB) the estimate must upper-bound.-_PIPELINE_MEASURED = {1: 36, 2: 492, 4: 1388, 8: 3220}-_TENSOR_MEASURED_PER_DEVICE = 600---class TestSafeUpperBound:- """The estimate must be >= every measured allocation (never under-reserve)."""-- @pytest.mark.parametrize("parallel,measured", sorted(_PIPELINE_MEASURED.items()))- def test_pipeline_upper_bounds_measured(self, parallel, measured):- est = _backend()._estimate_compute_buffer_bytes(n_parallel = parallel) / MIB- assert est >= measured, f"under-reserved at parallel={parallel}: {est:.0f} < {measured}"-- @pytest.mark.parametrize("parallel,measured", sorted(_PIPELINE_MEASURED.items()))- def test_pipeline_not_wildly_over(self, parallel, measured):- # Stay within ~2x of measured so we don't waste context (the point of- # replacing the flat reserve). parallel=1 is tiny in absolute terms.- est = _backend()._estimate_compute_buffer_bytes(n_parallel = parallel) / MIB- assert est <= max(measured * 2.0, 128)-- def test_tensor_upper_bounds_measured(self):- est = _backend()._estimate_compute_buffer_bytes(n_parallel = 1, per_device_tensor = True) / MIB- assert est >= _TENSOR_MEASURED_PER_DEVICE-- def test_tensor_far_below_old_flat_reserve(self):- # The whole point: deterministic estimate << flat 5120 for this model.- est = _backend()._estimate_compute_buffer_bytes(n_parallel = 1, per_device_tensor = True) / MIB- assert est < LlamaCppBackend._TENSOR_PARALLEL_BUFFER_RESERVE_MIB---class TestScaling:- def test_grows_with_serving_slots(self):- b = _backend()- vals = [b._estimate_compute_buffer_bytes(n_parallel = p) for p in (1, 2, 4, 8)]- assert vals == sorted(vals) and vals[0] < vals[-1]-- def test_parallel_1_is_small(self):- # Single-token decode: a few tens of MiB, not gigabytes.- est = _backend()._estimate_compute_buffer_bytes(n_parallel = 1) / MIB- assert est < 128+_GEMMA4_12B_PATTERN = ([True] * 5 + [False]) * 8++# GGUF dims of models whose buffers were measured on the bundled llama.cpp b10909+# (RTX 6000 Ada, --flash-attn on, --kv-unified above one slot).+_DIMS = {+ "Llama-3.2-1B": dict(+ _vocab_size = 128256,+ _embedding_length = 2048,+ _feed_forward_length = 8192,+ _n_layers = 16,+ _n_heads = 32,+ _n_kv_heads = 8,+ _kv_key_length = 64,+ _kv_value_length = 64,+ ),+ "Qwen3-8B": dict(+ _vocab_size = 151936,+ _embedding_length = 4096,+ _feed_forward_length = 12288,+ _n_layers = 36,+ _n_heads = 32,+ _n_kv_heads = 8,+ _kv_key_length = 128,+ _kv_value_length = 128,+ ),+ "Qwen3.5-4B": dict(+ _vocab_size = 248320,+ _embedding_length = 2560,+ _feed_forward_length = 9216,+ _n_layers = 32,+ _n_heads = 16,+ _n_kv_heads = 4,+ _kv_key_length = 256,+ _kv_value_length = 256,+ _ssm_inner_size = 4096,+ _ssm_state_size = 128,+ _ssm_group_count = 16,+ _full_attention_interval = 4,+ ),+ "Qwen3.6-35B-A3B": dict(+ _vocab_size = 248320,+ _embedding_length = 2048,+ _expert_used_count = 8,+ _expert_feed_forward_length = 512,+ _expert_shared_feed_forward_length = 512,+ _n_layers = 40,+ _n_heads = 16,+ _n_kv_heads = 2,+ _kv_key_length = 256,+ _kv_value_length = 256,+ _ssm_inner_size = 4096,+ _ssm_state_size = 128,+ _ssm_group_count = 16,+ _full_attention_interval = 4,+ ),+ "Gemma-4-12B": dict(+ _vocab_size = 262144,+ _embedding_length = 3840,+ _feed_forward_length = 15360,+ _n_layers = 48,+ _n_heads = 16,+ _n_kv_heads = 8,+ _n_kv_heads_by_layer = [1 if not swa else 8 for swa in _GEMMA4_12B_PATTERN],+ _kv_key_length = 512,+ _kv_value_length = 512,+ _kv_key_length_swa = 256,+ _kv_value_length_swa = 256,+ _sliding_window = 1024,+ _sliding_window_pattern = _GEMMA4_12B_PATTERN,+ ),+ "Gemma-4-E4B": dict(+ _vocab_size = 262144,+ _embedding_length = 2560,+ _feed_forward_length = 10240,+ _embedding_length_per_layer_input = 256,+ _n_layers = 42,+ _n_heads = 8,+ _n_kv_heads = 2,+ _kv_key_length = 512,+ _kv_value_length = 512,+ _kv_key_length_swa = 256,+ _kv_value_length_swa = 256,+ _sliding_window = 512,+ _sliding_window_pattern = ([True] * 5 + [False]) * 7,+ ),+ "GLM-4.7-Flash": dict(+ _vocab_size = 154880,+ _embedding_length = 2048,+ _feed_forward_length = 10240,+ _expert_used_count = 4,+ _expert_feed_forward_length = 1536,+ _expert_shared_count = 1,+ _n_layers = 47,+ _n_heads = 20,+ _n_kv_heads = 1,+ _kv_key_length = 576,+ _kv_value_length = 512,+ _kv_lora_rank = 512,+ _key_length_mla = 256,+ ),+}+++def _measured(name):+ b = LlamaCppBackend()+ for attr, value in _DIMS[name].items():+ setattr(b, attr, value)+ return b- def test_tensor_exceeds_pipeline_at_same_parallel(self):- b = _backend()- pipe = b._estimate_compute_buffer_bytes(n_parallel = 1)- tens = b._estimate_compute_buffer_bytes(n_parallel = 1, per_device_tensor = True)- assert tens > pipe- def test_scales_with_vocab(self):- small = _backend(vocab = 32000)._estimate_compute_buffer_bytes(n_parallel = 4)- big = _backend(vocab = 256000)._estimate_compute_buffer_bytes(n_parallel = 4)- assert big > small+# llama-server's "CUDA0 compute buffer size" (MiB): (model, ubatch, slots, n_ctx, KV type).+_MEASURED_COMPUTE = [+ ("Llama-3.2-1B", 256, 4, 32768, "f16", 46.01),+ ("Llama-3.2-1B", 2048, 4, 32768, "f16", 368.04),+ ("Llama-3.2-1B", 512, 1, 32768, "f16", 92.01),+ ("Llama-3.2-1B", 512, 8, 32768, "f16", 92.01),+ ("Llama-3.2-1B", 512, 4, 131072, "f16", 188.01),+ ("Llama-3.2-1B", 512, 4, 131072, "q8_0", 400.04),+ ("Llama-3.2-1B", 2048, 4, 131072, "q8_0", 832.07),+ ("Qwen3-8B", 512, 4, 32768, "f16", 128.01),+ ("Qwen3-8B", 2048, 4, 131072, "f16", 896.04),+ ("Qwen3-8B", 512, 4, 131072, "q8_0", 672.09),+ ("Qwen3.5-4B", 512, 1, 32768, "f16", 96.02),+ ("Qwen3.5-4B", 512, 8, 32768, "f16", 115.52),+ ("Qwen3.5-4B", 2048, 4, 131072, "f16", 768.06),+ ("Qwen3.5-4B", 2048, 4, 131072, "q8_0", 1224.34),+ ("Qwen3.6-35B-A3B", 256, 4, 32768, "f16", 66.02),+ ("Qwen3.6-35B-A3B", 2048, 1, 32768, "f16", 528.2),+ ("Qwen3.6-35B-A3B", 512, 4, 131072, "q8_0", 436.31),+ ("Gemma-4-12B", 512, 1, 32768, "f16", 154.52),+ ("Gemma-4-12B", 1120, 4, 32768, "f16", 346.21),+ ("Gemma-4-12B", 2048, 8, 32768, "f16", 652.07),+ ("Gemma-4-12B", 2048, 4, 131072, "f16", 1020.07),+ ("Gemma-4-12B", 512, 4, 131072, "q8_0", 436.8),+ ("Gemma-4-E4B", 512, 4, 32768, "f16", 135.52),+ ("Gemma-4-E4B", 2048, 4, 131072, "q8_0", 1317.35),+ ("GLM-4.7-Flash", 512, 4, 32768, "f16", 110.01),+ ("GLM-4.7-Flash", 2048, 4, 131072, "f16", 824.04),+ ("GLM-4.7-Flash", 512, 4, 131072, "q8_0", 358.9),+]+++def _priced_mib(b, ub, slots, ctx, kv):+ flat = b._estimate_compute_buffer_bytes(n_ubatch = ub, n_parallel = slots)+ return (flat + b._compute_buffer_ctx_bytes(ctx, ub, kv)) / MIB+++class TestMeasuredComputeBuffer:+ """Flat plus context-linear must cover llama-server's own reserve without+ inflating it: the vocab-wide rows the old formula charged per slot came to 16 GiB+ for Gemma 4 12B at 8 slots and ubatch 2048, which reserves 652 MiB."""++ @pytest.mark.parametrize("name,ub,slots,ctx,kv,measured", _MEASURED_COMPUTE)+ def test_covers_the_measured_buffer(self, name, ub, slots, ctx, kv, measured):+ est = _priced_mib(_measured(name), ub, slots, ctx, kv)+ assert est >= measured, f"{name} ub{ub} p{slots} {ctx} {kv}: {est:.0f} < {measured}"++ @pytest.mark.parametrize("name,ub,slots,ctx,kv,measured", _MEASURED_COMPUTE)+ def test_stays_close_to_the_measured_buffer(self, name, ub, slots, ctx, kv, measured):+ # The quantized scratch and the activations peak at different points of the+ # graph but are charged together, the widest gap at a short context.+ est = _priced_mib(_measured(name), ub, slots, ctx, kv)+ assert est <= measured * (1.35 if kv == "f16" else 1.3), f"{name}: {est:.0f} vs {measured}"++ def test_the_reported_gemma4_vision_raise(self):+ # The micro-batch raise for the Gemma 4 projector, 512 -> 1120 at 4 slots and+ # 262144 cells: llama-server's buffer grows 455 MiB, the old estimate 2.5 GiB.+ b = _measured("Gemma-4-12B")+ raise_mib = _priced_mib(b, 1120, 4, 262144, "f16") - _priced_mib(b, 512, 4, 262144, "f16")+ assert 455 <= raise_mib <= 455 * 1.15+++class TestFlatBufferShape:+ def test_slots_add_output_rows_not_activations(self):+ b = _measured("Gemma-4-12B")+ one = b._estimate_compute_buffer_bytes(n_ubatch = 2048, n_parallel = 1)+ eight = b._estimate_compute_buffer_bytes(n_ubatch = 2048, n_parallel = 8)+ rows = 7 * 262144 * 4+ swa = (_pad(1024 * 8 + 2048) - _pad(1024 + 2048)) * 2048 * 2+ assert eight - one == pytest.approx(+ rows * LlamaCppBackend._COMPUTE_BUFFER_SAFETY + swa, rel = 1e-6+ )- def test_scales_with_ubatch(self):- b = _backend()- lo = b._estimate_compute_buffer_bytes(n_parallel = 4, n_ubatch = 256)- hi = b._estimate_compute_buffer_bytes(n_parallel = 4, n_ubatch = 1024)- assert hi > lo-- def test_embedding_mode_budgets_the_first_output_buffer(self):- chat = _backend()._estimate_compute_buffer_bytes(n_parallel = 1, n_ubatch = 512)- embedding = _backend(pooling_type = 2)._estimate_compute_buffer_bytes(- n_parallel = 1, n_ubatch = 512+ def test_scales_linearly_with_ubatch(self):+ b = _measured("Qwen3-8B")+ lo = b._estimate_compute_buffer_bytes(n_ubatch = 512, n_parallel = 1)+ hi = b._estimate_compute_buffer_bytes(n_ubatch = 1024, n_parallel = 1)+ rows = 151936 * 4 * LlamaCppBackend._COMPUTE_BUFFER_SAFETY+ assert hi - rows == pytest.approx(2 * (lo - rows), rel = 1e-6)++ def test_a_tensor_device_holds_the_single_device_buffer(self):+ # Qwen3 8B and Gemma 4 12B split over two GPUs reserve their single-GPU+ # buffer on each device, at ubatch 512 and 2048 alike.+ for name in ("Qwen3-8B", "Gemma-4-12B"):+ b = _measured(name)+ assert b._estimate_compute_buffer_bytes(+ n_ubatch = 2048, n_parallel = 4, per_device_tensor = True+ ) == b._estimate_compute_buffer_bytes(n_ubatch = 2048, n_parallel = 4)++ def test_embedding_rows_are_pooled_not_vocab_wide(self):+ b = _backend(vocab = 30522, embd = 384, pooling_type = 1)+ chat = _backend(vocab = 30522, embd = 384)+ assert b._estimate_compute_buffer_bytes(n_ubatch = 512, n_parallel = 4) > (+ chat._estimate_compute_buffer_bytes(n_ubatch = 512, n_parallel = 4)+ )+ # 512 pooled rows of 384 floats, far below 512 vocabulary rows.+ assert b._estimate_compute_buffer_bytes(n_ubatch = 512, n_parallel = 4) < 30522 * 512 * 4++ def test_widest_block_sets_the_width(self):+ # Dense FFN, routed experts, and the delta-net convolution each can be widest.+ assert _measured("Qwen3-8B")._compute_activation_width() == 4 * 12288+ assert _measured("Qwen3.6-35B-A3B")._compute_activation_width() == (+ 8 * (2 * 2048 + 3 * 512) + 3 * 512)- assert embedding > chat+ ssm = _backend(embd = 2560)+ ssm._ssm_inner_size, ssm._ssm_state_size, ssm._ssm_group_count = 4096, 128, 16+ assert ssm._compute_activation_width() == 4 * (4096 + 2 * 16 * 128)+ # Per-layer input embeddings ride on top of the widest block.+ assert _measured("Gemma-4-E4B")._compute_activation_width() == 4 * 10240 + 256 * 42++ def test_header_without_ffn_dims_uses_the_embedding_floor(self):+ assert _backend(embd = 4096)._compute_activation_width() == 12 * 4096++ def test_expert_width_falls_back_to_the_ffn_length(self):+ # Granite 4.0 H writes its 512-wide experts as feed_forward_length.+ b = _backend(vocab = 100352, embd = 1536)+ b._n_experts, b._expert_used_count = 64, 6+ b._feed_forward_length, b._expert_shared_feed_forward_length = 512, 1024+ assert b._compute_activation_width() == 6 * (2 * 1536 + 3 * 512) + 3 * 1024++ def test_granite_hybrid_is_covered(self):+ # Held out from the fit: llama-server reserved 80.0 MiB at one slot and 95.3+ # MiB at eight (ubatch 512, 32768 cells), the recurrent state per slot.+ b = _backend(vocab = 100352, embd = 1536)+ b._n_experts, b._expert_used_count = 64, 6+ b._feed_forward_length, b._expert_shared_feed_forward_length = 512, 1024+ b._ssm_inner_size, b._ssm_state_size, b._ssm_group_count = 3072, 128, 1+ for slots, measured in ((1, 80.0), (8, 95.3)):+ est = _priced_mib(b, 512, slots, 32768, "f16")+ assert measured <= est <= measured * 1.3++ def test_a_window_wider_than_the_context_does_not_slide(self):+ # Phi-4-mini declares a 262144 window over a 131072 context and no pattern;+ # pricing it as a sliding mask charged 12x its 92 MiB buffer.+ b = _backend(vocab = 200064, embd = 3072)+ b._feed_forward_length, b._context_length = 8192, 131072+ b._sliding_window, b._sliding_window_pattern = 262144, None+ assert b._effective_sliding_window() == 0+ assert 92.0 <= _priced_mib(b, 512, 4, 32768, "f16") <= 92.0 * 1.3+ b._sliding_window_pattern = [False] * 32+ assert b._effective_sliding_window() == 0+ b._sliding_window_pattern = [True, False] * 16+ assert b._effective_sliding_window() == 262144class TestFallback:def test_zero_when_vocab_missing(self):assert _backend(vocab = None)._estimate_compute_buffer_bytes(n_parallel = 4) == 0+ def test_vocab_ceiling_prices_a_header_without_a_token_array(self):+ b = _backend(vocab = None)+ assert b._estimate_compute_buffer_bytes(n_parallel = 4, vocab_ceiling = 262144) == (+ _backend(vocab = 262144)._estimate_compute_buffer_bytes(n_parallel = 4)+ )+def test_zero_when_embd_missing(self):assert _backend(embd = None)._estimate_compute_buffer_bytes(n_parallel = 4) == 0@@ -192,147 +393,78 @@ class TestFallback:assert gi == [0, 1] # both GPUs usable under the flat fallback-class TestParallel1Default:- """At Unsloth's default --parallel 1 the buffer is negligible in pipeline."""-- def test_default_n_parallel(self):- est = _backend()._estimate_compute_buffer_bytes() / MIB- assert est < 128--class TestContextLinearBuffer:- """``_compute_buffer_ctx_bytes``: the flash-attn KQ-mask + attention scratch- grow ~linearly with context; the flat estimate above only covers ctx -> 0.- Measured slope (q8_0 KV, ubatch 512) was 0.74-2.02 x n_embd; 2 x n_embd is the- worst-case upper bound the term must hold to."""+ """``_compute_buffer_ctx_bytes``: the KQ mask, n_ubatch * 2 bytes per cell, plus+ for a quantized cache an f16 copy of the widest global layer's K and V."""- # (model, n_embd, ctx, measured CUDA0 compute buffer MiB at that ctx, q8_0/ub512)+ # (model, n_embd, ctx, measured CUDA0 compute buffer MiB at that ctx, q8_0/ub512).+ # From an earlier build; kept as upper bounds on a header with only n_embd._MEASURED = [("Qwen3.5-2B", 2048, 262144, 796),- ("Qwen3.5-4B", 2560, 262144, 1330), # worst slope, 2.02 x n_embd+ ("Qwen3.5-4B", 2560, 262144, 1330),("Qwen3.5-9B", 4096, 262144, 1336),("Qwen3.6-27B", 5120, 262144, 1360),("Gemma-4-31B", 5376, 262144, 2392),]def test_zero_by_default(self):- # Omitted/zero ctx -> no term (keeps the flat callers unchanged).assert _backend()._compute_buffer_ctx_bytes(0) == 0def test_zero_when_embd_missing(self):assert _backend(embd = None)._compute_buffer_ctx_bytes(262144) == 0def test_grows_linearly_with_context(self):- b = _backend(embd = 4096)- a = b._compute_buffer_ctx_bytes(65536)- d = b._compute_buffer_ctx_bytes(131072)- assert d == pytest.approx(2 * a, rel = 1e-6)-- def test_scales_with_embd(self):- # The quantized (dequant-scratch) rate scales with n_embd; f16 (mask) does not.- small = _backend(embd = 2048)._compute_buffer_ctx_bytes(131072, cache_type_kv = "q8_0")- big = _backend(embd = 5120)._compute_buffer_ctx_bytes(131072, cache_type_kv = "q8_0")- assert big > small-- def test_scales_with_ubatch(self):- b = _backend(embd = 4096)- lo = b._compute_buffer_ctx_bytes(131072, n_ubatch = 256)- hi = b._compute_buffer_ctx_bytes(131072, n_ubatch = 1024)- assert hi > lo+ b = _measured("Qwen3-8B")+ for kv in ("f16", "q8_0"):+ a = b._compute_buffer_ctx_bytes(65536, cache_type_kv = kv)+ d = b._compute_buffer_ctx_bytes(131072, cache_type_kv = kv)+ assert d == pytest.approx(2 * a, rel = 1e-6)@pytest.mark.parametrize("name,embd,ctx,measured", _MEASURED)def test_upper_bounds_measured_compute_growth(self, name, embd, ctx, measured):- # flat term + context-linear term must cover the real (q8_0) buffer at full ctx.b = _backend(embd = embd)flat = b._estimate_compute_buffer_bytes(n_parallel = 1)total = (flat + b._compute_buffer_ctx_bytes(ctx, cache_type_kv = "q8_0")) / MIBassert total >= measured, f"{name}: under-reserved {total:.0f} < {measured}"- def test_worst_case_rate_covers_two_x_embd(self):- # >= 2 x n_embd bytes per context token at the default micro-batch (the worst- # measured quantized slope, Qwen3.5-4B), so flat + term upper-bounds the buffer.- embd = 4096- b = _backend(embd = embd)- per_tok = b._compute_buffer_ctx_bytes(100000, cache_type_kv = "q8_0") / 100000- assert per_tok >= 2 * embd-class TestContextBufferKVQuant:- """The context-linear rate depends on the KV cache type: a quantized cache adds a- context-sized dequant scratch (heavy); f16/bf16/f32 only pays the KQ mask (light).- Measured Qwen3.5-4B at 256k: 1.30 GiB (q8_0) vs 0.31 GiB (f16)."""-def test_quantized_heavier_than_f16(self):- b = _backend(embd = 4096)- q = b._compute_buffer_ctx_bytes(131072, cache_type_kv = "q8_0")- f = b._compute_buffer_ctx_bytes(131072, cache_type_kv = "f16")- assert q > f+ b = _measured("Qwen3-8B")+ assert b._compute_buffer_ctx_bytes(131072, cache_type_kv = "q8_0") > (+ b._compute_buffer_ctx_bytes(131072, cache_type_kv = "f16")+ )def test_none_cache_type_is_f16(self):- # None -> f16 (llama.cpp's default); the env-quantized case is covered by the- # KV budget's f16 over-reservation, so we take the lighter mask-only rate.- b = _backend(embd = 4096)- assert b._compute_buffer_ctx_bytes(- 131072, cache_type_kv = None- ) == b._compute_buffer_ctx_bytes(131072, cache_type_kv = "f16")+ b = _measured("Qwen3-8B")+ assert b._compute_buffer_ctx_bytes(131072, cache_type_kv = None) == (+ b._compute_buffer_ctx_bytes(131072, cache_type_kv = "f16")+ )@pytest.mark.parametrize("ct", ["f16", "bf16", "f32"])- def test_unquantized_uses_mask_only_rate(self, ct):- # f16/bf16/f32: KQ mask only, n_ubatch*2 B/tok, independent of n_embd.- b_small = _backend(embd = 2048)- b_big = _backend(embd = 8192)- per_small = b_small._compute_buffer_ctx_bytes(100000, cache_type_kv = ct) / 100000- per_big = b_big._compute_buffer_ctx_bytes(100000, cache_type_kv = ct) / 100000- assert per_small == per_big # no n_embd scaling on the f16 path- expected = 512 * 2 * LlamaCppBackend._CTX_COMPUTE_F16_MASK_SAFETY # ubatch 512- assert per_small == pytest.approx(expected, rel = 1e-6)+ def test_unquantized_pays_exactly_the_mask(self, ct):+ # n_ubatch * 2 bytes per cell whatever the model, as measured on every family.+ for name in _DIMS:+ b = _measured(name)+ for ub in (512, 2048):+ assert b._compute_buffer_ctx_bytes(100000, ub, ct) == ub * 2 * 100000@pytest.mark.parametrize("ct", ["q8_0", "q5_1", "q4_0", "iq4_nl"])- def test_quantized_types_use_heavy_rate(self, ct):- embd = 4096- b = _backend(embd = embd)- per_tok = b._compute_buffer_ctx_bytes(100000, cache_type_kv = ct) / 100000- assert per_tok == pytest.approx(- LlamaCppBackend._CTX_COMPUTE_BYTES_PER_EMBD * embd, rel = 1e-6- )-- def test_f16_covers_measured_mask(self):- # f16 buffer is ~mask only (~n_ubatch*2 B/tok); 0.5 x n_embd must cover the- # measured Qwen3.5-4B f16 slope (~0.4 x n_embd = 0.31 GiB at 256k).- b = _backend(embd = 2560) # Qwen3.5-4B- est = b._compute_buffer_ctx_bytes(262144, cache_type_kv = "f16") / MIB- assert est >= 320 # measured 0.31 GiB growth-+ def test_quantized_adds_the_widest_kv_copy_whatever_the_ubatch(self, ct):+ b = _measured("Qwen3-8B")+ for ub in (512, 2048):+ extra = b._compute_buffer_ctx_bytes(100000, ub, ct) - ub * 2 * 100000+ assert extra == 8 * (128 + 128) * 2 * 100000-class TestContextBufferMLA:- """MLA (compressed attention) needs a smaller quantized dequant scratch than- regular attention: measured 0.94 x n_embd on GLM-5.2 and Kimi-K2.7 vs up to- 2.02x on Qwen/Gemma. Charging the regular rate would badly over-reserve a tight- multi-GPU MLA pin (per-device scaling multiplies the error)."""+ def test_sliding_window_layers_do_not_set_the_rate(self):+ # Gemma 4 12B: 8 KV heads on the sliding layers, 1 on the global ones.+ assert _measured("Gemma-4-12B")._attention_dequant_width() == 1 * (512 + 512)- def test_mla_lighter_than_regular(self):- reg = _backend(embd = 6144, mla = None)._compute_buffer_ctx_bytes(262144, cache_type_kv = "q8_0")- mla = _backend(embd = 6144, mla = 256)._compute_buffer_ctx_bytes(262144, cache_type_kv = "q8_0")- assert mla < reg-- @pytest.mark.parametrize(- "name,embd,ctx,measured",- [- ("GLM-5.2", 6144, 754688, 4141), # per-device compute MiB at q8_0- ("Kimi-K2.7", 7168, 262144, 1690),- ],- )- def test_mla_rate_covers_measured(self, name, embd, ctx, measured):- b = _backend(embd = embd, mla = 256)- est = b._compute_buffer_ctx_bytes(ctx, cache_type_kv = "q8_0") / MIB- assert est >= measured, f"{name}: MLA under-reserved {est:.0f} < {measured}"+ def test_mla_copies_the_compressed_key_only(self):+ assert _measured("GLM-4.7-Flash")._attention_dequant_width() == 576- def test_mla_not_wildly_over(self):- # 1.25 x n_embd should stay within ~1.6x of the measured 0.94x (not 2.4x like- # the regular 2.25 rate would), so a multi-GPU MLA pin keeps its context.- b = _backend(embd = 6144, mla = 256)- est = b._compute_buffer_ctx_bytes(754688, cache_type_kv = "q8_0") / MIB- assert est <= 4141 * 1.7+ def test_header_without_head_dims_falls_back_to_embd(self):+ assert _backend(embd = 4096)._attention_dequant_width() == 4096class TestContextBufferDSV4:@@ -390,8 +522,7 @@ class TestContextBufferDSV4:# Regression guard: a non-deepseek4 model keeps the mask-only f16 rate.b = _backend(embd = 4096, arch = "llama")per_tok = b._compute_buffer_ctx_bytes(100000, cache_type_kv = "f16") / 100000- expected = 512 * 2 * LlamaCppBackend._CTX_COMPUTE_F16_MASK_SAFETY- assert per_tok == pytest.approx(expected, rel = 1e-6)+ assert per_tok == pytest.approx(512 * 2, rel = 1e-6)class TestContextBufferLayerSplit:@@ -426,11 +557,22 @@ class TestContextBufferLayerSplit:131072, cache_type_kv = "f16") == b._compute_buffer_ctx_bytes(131072, cache_type_kv = "f16", layer_split = False)- def test_split_is_exactly_the_multiplier(self):+ def test_split_is_the_multiplier_plus_the_hidden_state_copies(self):b = _backend(embd = 4096)one = b._compute_buffer_ctx_bytes(131072, cache_type_kv = "f16")many = b._compute_buffer_ctx_bytes(131072, cache_type_kv = "f16", layer_split = True)- assert many == pytest.approx(one * LlamaCppBackend._CTX_COMPUTE_SPLIT_MULT, rel = 1e-9)+ assert many == one * LlamaCppBackend._CTX_COMPUTE_SPLIT_MULT + _hidden_copies(4096)++ def test_matches_the_measured_two_gpu_split(self):+ # Qwen3 8B over an RTX 6000 Ada and an RTX 3090: 256.04 MiB on each device at+ # 32768 cells and ubatch 512, 2560.16 MiB at 131072 cells and ubatch 2048.+ b = _measured("Qwen3-8B")+ for ctx, ub, measured in ((32768, 512, 256.04), (131072, 2048, 2560.16)):+ per_device = (+ b._estimate_compute_buffer_bytes(n_ubatch = ub, n_parallel = 1)+ + b._compute_buffer_ctx_bytes(ctx, ub, "f16", layer_split = True)+ ) / MIB+ assert measured <= per_device <= measured * 1.1@pytest.mark.parametrize("name,n_gpus,ub,measured", _MEASURED)def test_upper_bounds_measured_per_device_rate(self, name, n_gpus, ub, measured):@@ -445,20 +587,17 @@ class TestContextBufferLayerSplit:assert per_tok >= measured * ub, f"{name} n={n_gpus}: {per_tok:.0f} < {measured * ub:.0f}"@pytest.mark.parametrize("name,n_gpus,ub,measured", _MEASURED)- def test_not_wildly_over_measured_per_device_rate(self, name, n_gpus, ub, measured):- # The step is a correction, not extra headroom: still the 1.5 safety factor.+ def test_charges_exactly_the_measured_per_device_rate(self, name, n_gpus, ub, measured):+ # The mask is priced exactly; only the split's hidden-state copies sit on top.b = _backend(embd = 4096)- per_tok = (- b._compute_buffer_ctx_bytes(- 100000, n_ubatch = ub, cache_type_kv = "f16", layer_split = n_gpus > 1- )- / 100000+ hidden = _hidden_copies(4096, ub) if n_gpus > 1 else 0+ charged = b._compute_buffer_ctx_bytes(+ 100000, n_ubatch = ub, cache_type_kv = "f16", layer_split = n_gpus > 1)- expected = measured * ub * LlamaCppBackend._CTX_COMPUTE_F16_MASK_SAFETY- assert per_tok == pytest.approx(expected, rel = 1e-6)+ assert charged - hidden == pytest.approx(measured * ub * 100000, rel = 1e-6)def test_pre_fix_split_reserve_was_short(self):- # The bug: without the step a split reserved 8/(2*1.5) = 2.67x too little.+ # The bug: without the step a split reserved a quarter of its masks.b = _backend(embd = 4096)one = b._compute_buffer_ctx_bytes(1048576, cache_type_kv = "f16")measured = self._RATE_SPLIT * 512 * 1048576@@ -485,7 +624,7 @@ class TestContextBufferLayerSplit:mask = b._compute_buffer_ctx_bytes(131072, cache_type_kv = "f16")single = b._compute_buffer_ctx_bytes(131072, cache_type_kv = ct)split = b._compute_buffer_ctx_bytes(131072, cache_type_kv = ct, layer_split = True)- delta = (LlamaCppBackend._CTX_COMPUTE_SPLIT_MULT - 1) * mask+ delta = (LlamaCppBackend._CTX_COMPUTE_SPLIT_MULT - 1) * mask + _hidden_copies(embd)assert split == pytest.approx(single + delta, rel = 1e-9)@pytest.mark.parametrize("embd,mla", [(2048, None), (2560, None), (8192, None), (7168, 576)])@@ -520,9 +659,7 @@ class TestContextBufferLayerSplit:131072, cache_type_kv = "f16", layer_split = True) == b._compute_buffer_ctx_bytes(131072, cache_type_kv = "f16")measured = TestContextBufferDSV4._MEASURED_1M_GIB * 1024**3 / 1048576- split_masks = (LlamaCppBackend._CTX_COMPUTE_SPLIT_MULT - 1) * (- 512 * 2 * LlamaCppBackend._CTX_COMPUTE_F16_MASK_SAFETY- )+ split_masks = (LlamaCppBackend._CTX_COMPUTE_SPLIT_MULT - 1) * 512 * 2assert LlamaCppBackend._DSV4_CTX_COMPUTE_BYTES_PER_TOK - measured >= split_masks@@ -554,9 +691,7 @@ class TestContextBufferInklingSplit:@pytest.mark.parametrize("ct", ["f16", None, "q8_0"])def test_split_adds_exactly_the_extra_mask_copies(self, ct):- delta = (LlamaCppBackend._CTX_COMPUTE_SPLIT_MULT - 1) * (- self._UB * 2 * LlamaCppBackend._CTX_COMPUTE_F16_MASK_SAFETY- )+ delta = (LlamaCppBackend._CTX_COMPUTE_SPLIT_MULT - 1) * self._UB * 2assert self._rate(ct, True) == pytest.approx(self._rate(ct, False) + delta, rel = 1e-9)def test_dense_fallback_delta_is_present_but_tiny(self):@@ -595,7 +730,9 @@ class TestLayerSplitWiring:indent = len(src[start]) - len(src[start].lstrip())body = [src[start]]for line in src[start + 1 :]:- if line.strip() and len(line) - len(line.lstrip()) <= indent:+ # A one-per-line signature closes with ") -> int:" at the def's indent.+ closes_signature = line.lstrip().startswith(")")+ if line.strip() and len(line) - len(line.lstrip()) <= indent and not closes_signature:breakbody.append(line)return "\n".join(body)@@ -956,12 +1093,12 @@ class TestPerDeviceSplitReserve:assert card1_usable < reserve_mib # would OOM card 1 at loaddef test_subset_is_capped_not_rejected_when_a_card_cannot_hold_its_reserve(self):- # 1762.72 MiB usable on card 1 holds 31488, not the pooled 262144. Rejecting+ # 1762.72 MiB usable on card 1 holds 30720, not the pooled 262144. Rejecting# the subset instead drops auto to the 4096 fallback for no reason.b = self._fit_backend()gpus, totals = self._HETEROGENEOUSassert self._drive(b, gpus, totals, 20_480, 262144, cap = False) == (None, 0)- assert self._drive(b, gpus, totals, 20_480, 262144) == ([0, 1], 31_488)+ assert self._drive(b, gpus, totals, 20_480, 262144) == ([0, 1], 30_720)def test_homogeneous_gpus_are_unaffected(self):b = self._fit_backend()@@ -1029,14 +1166,15 @@ class TestPerDeviceReserveCap:reserve = lambda c: ((self._OH + b._compute_buffer_ctx_bytes(c, self._UB, "f16", layer_split = True)) / MIB)- assert reserve(31_488) == 1762.0 <= usable- assert reserve(31_744) == 1768.0 > usable+ # 1024 MiB overhead + 256 MiB of hidden-state copies + 4 masks at 1/64 MiB a cell.+ assert reserve(30_720) == 1760.0 <= usable+ assert reserve(30_976) == 1764.0 > usabledef test_capped_context_still_fits_the_pooled_budget(self):b = self._fit_backend()gpus, totals = self._HETEROGENEOUSidx, ctx = self._drive(b, gpus, totals, 20_480, 262144)- assert idx == [0, 1] and ctx == 31_488+ assert idx == [0, 1] and ctx == 30_720pool = sum(g[1] - 0.03 * totals[g[0]] for g in gpus)cc = 2 * b._compute_buffer_ctx_bytes(ctx, self._UB, "f16", layer_split = True)footprint = (20_480 * MIB + self._OH + b._estimate_kv_cache_bytes(ctx) + cc) / MIB@@ -1087,7 +1225,7 @@ class TestPerDeviceReserveCap:totals = {0: 49_152, 1: 24_576, 2: 24_576}assert self._drive(b, gpus, totals, model_mib, 262144, cap = False) == (None, 0)idx, ctx = self._drive(b, gpus, totals, model_mib, 262144)- assert idx == [0, 1] and ctx == 31_488+ assert idx == [0, 1] and ctx == 30_720def test_helper_edges(self):cap = LlamaCppBackend._cap_ctx_to_per_device_reserve@@ -1248,7 +1386,7 @@ class TestSplitRateRecheckAfterSelection:for node in ast.walk(ast.parse(textwrap.dedent(source)))if isinstance(node, ast.Call)for keyword in node.keywords- if keyword.arg == "split_extra_bytes"+ if keyword.arg in ("split_extra_bytes", "split_extra_for_slots")]# Projector floor pin, explicit-context pin, reduced-slot retry, per-candidate# re-fit. A fifth has to come here and say which context it prices at.@@ -1261,14 +1399,20 @@ class TestSplitRateRecheckAfterSelection:assert len(wired) == 4, wired# Each passes the step at a context of its own, so none is exempt and none# hardcodes one: `_cc_split_extra(4096)` would not match.+ # The reduced-slot search re-prices the step for each candidate slot count.for expression in wired:- assert re.fullmatch(r"_cc_split_extra\(\w+\)", expression), expression+ assert re.fullmatch(+ r"_cc_split_extra\(\w+\)|lambda s: _cc_split_extra\(\w+, s\)", expression+ ), expressionassert "gpu_indices,use_fit=self._select_gpus_split_aware(" in load# The step rides _cc_bytes' pipelining gate, so it is 0 when llama.cpp declines.- assert "returnmax(0,_cc_bytes(ctx,2)//2-_cc_bytes(ctx))" in load+ assert "returnmax(0,_cc_bytes(ctx,2,slots)//2-_cc_bytes(ctx,1,slots))" in loadslots = "".join(inspect.getsource(LlamaCppBackend._slots_that_fit_on_gpu).split())assert "self._select_gpus_split_aware(" in slots- assert "split_extra_bytes=split_extra_bytes," in slots+ assert (+ "split_extra_bytes=(split_extra_for_slots(slots)ifsplit_extra_for_slotselsesplit_extra_bytes),"+ in slots+ )# ── The scratch rate keys off the LIGHTER axis ───────────────────────────────@@ -1370,8 +1514,9 @@ class TestTensorFitPricesTheQuantizedAxis:axis alone the planner advertises the full 262144; the quantized axiscannot hold it."""b = _backend_from_gguf_local()- optimistic = self._plan(b, "f16", None, 2048)[0]- honest = self._plan(b, "f16", "q4_0", 2048)[0]+ # 36 GB cards, so the KV cache rather than the native length bounds the context.+ optimistic = self._plan(b, "f16", None, 2048, per_gpu = 36_000)[0]+ honest = self._plan(b, "f16", "q4_0", 2048, per_gpu = 36_000)[0]assert honest < optimistic, (honest, optimistic)# Still a real context, not the 2048 floor: the fix must not collapse the@@ -1381,7 +1526,7 @@ class TestTensorFitPricesTheQuantizedAxis:# pinning: the asymmetric pair pays f16 KV bytes on both axes (the heavier# axis budgets storage) AND the full quantized dequant scratch. Both terms# conservative is the point; neither one alone describes this launch.- assert honest < self._plan(b, "q4_0", "q4_0", 2048)[0]+ assert honest < self._plan(b, "q4_0", "q4_0", 2048, per_gpu = 36_000)[0]def test_a_symmetric_request_is_unchanged(self):"""Default and explicit scratch type agree when both axes match, so no@@ -1390,3 +1535,171 @@ class TestTensorFitPricesTheQuantizedAxis:for kv in ("f16", "q8_0", "q4_0"):for ub in (512, 2048):assert self._plan(b, kv, None, ub) == self._plan(b, kv, kv, ub)+++class TestSpeculativeComputeBuffers:+ """Draft buffers must cover growth beyond the 224 MiB floor.+ Measurements below are from Qwen3.8 27B's embedded head."""++ def _qwen38(self):+ b = _backend(vocab = 248320, embd = 5120)+ b._feed_forward_length = 17408+ return b++ @pytest.mark.parametrize(+ "ub,ctx,measured", [(512, 32768, 130.02), (1120, 32768, 284.41), (1120, 131072, 428.78)]+ )+ def test_floor_plus_growth_covers_the_draft_context(self, ub, ctx, measured):+ b = self._qwen38()+ charged = (+ LlamaCppBackend._MTP_DRAFT_COMPUTE_BYTES + b._mtp_draft_compute_bytes(ctx, n_ubatch = ub)+ ) / MIB+ assert charged >= measured++ def test_short_context_and_micro_batch_stay_on_the_floor(self):+ assert self._qwen38()._mtp_draft_compute_bytes(4096, n_ubatch = 256) == 0++ def test_growth_follows_the_mask_once_past_the_floor(self):+ b = self._qwen38()+ lo = b._mtp_draft_compute_bytes(131072, n_ubatch = 1120)+ hi = b._mtp_draft_compute_bytes(262144, n_ubatch = 1120)+ assert hi - lo == 1120 * 2 * 131072++ def test_verify_rows_scale_with_slots_and_depth(self):+ b = self._qwen38()+ base = b._mtp_draft_compute_bytes(4096, n_ubatch = 256)+ rows = b._mtp_draft_compute_bytes(4096, n_ubatch = 256, n_parallel = 4, spec_draft_n_max = 2)+ assert rows - base == 4 * 2 * LlamaCppBackend._SPEC_VERIFY_ROW_COPIES * 248320 * 4++ def test_the_reserve_carries_it(self):+ b = self._qwen38()+ b._n_layers, b._n_kv_heads, b._kv_key_length, b._kv_value_length = 65, 4, 256, 256+ b._nextn_predict_layers = 1+ kwargs = dict(n_parallel = 1, target_rollback = False, mtp_keeps_target_ctx = False)+ short = b._estimate_mtp_overhead_bytes(4096, n_ubatch = 256, **kwargs)+ long = b._estimate_mtp_overhead_bytes(262144, n_ubatch = 1120, **kwargs)+ draft_kv = b._mtp_draft_kv_bytes(262144, n_ubatch = 1120) - b._mtp_draft_kv_bytes(+ 4096, n_ubatch = 256+ )+ assert long - short == draft_kv + b._mtp_draft_compute_bytes(262144, n_ubatch = 1120)+++class TestFlashAttentionOff:+ """Without flash attention llama.cpp materialises [n_kv, n_ubatch, n_head] float32+ attention scores and builds the KQ mask as float32. Measured on b10909 (RTX 6000+ Ada), f16 cache, llama-server's CUDA0 compute buffer in MiB."""++ _MEASURED = [+ ("Llama-3.2-1B", 512, 4, 32768, 2130.01),+ ("Llama-3.2-1B", 2048, 4, 32768, 8520.02),+ ("Llama-3.2-1B", 512, 4, 65536, 4242.01),+ ("Qwen3-8B", 1120, 4, 32768, 4698.76),+ ("Qwen3.5-4B", 512, 4, 65536, 2214.01),+ ("Qwen3.6-35B-A3B", 2048, 4, 32768, 4504.18),+ ("Gemma-4-12B", 1120, 4, 32768, 2540.8),+ ("Gemma-4-12B", 512, 4, 65536, 2248.01),+ ("Gemma-4-E4B", 512, 4, 65536, 1217.51),+ ("GLM-4.7-Flash", 1120, 4, 32768, 3086.3),+ ("GLM-4.7-Flash", 512, 4, 65536, 2818.9),+ ]++ @pytest.mark.parametrize("name,ub,slots,ctx,measured", _MEASURED)+ def test_covers_the_measured_buffer(self, name, ub, slots, ctx, measured):+ b = _measured(name)+ est = (+ b._estimate_compute_buffer_bytes(n_ubatch = ub, n_parallel = slots)+ + b._compute_buffer_ctx_bytes(ctx, ub, "f16", flash_attn = False)+ ) / MIB+ assert measured <= est <= measured * 1.1, f"{name}: {est:.0f} vs {measured}"++ def test_the_rate_is_the_score_matrix_plus_a_float32_mask(self):+ b = _measured("Gemma-4-12B")+ on = b._compute_buffer_ctx_bytes(100000, 1120, "f16")+ off = b._compute_buffer_ctx_bytes(100000, 1120, "f16", flash_attn = False)+ assert on == 1120 * 2 * 100000+ assert off == 1120 * (4 + 16 * 4) * 100000+++class TestLayerSplitSlidingWindowSlots:+ """The replicated sliding-window masks of a layer split hold the window for every+ slot, as the single-device mask does. Measured per GPU across an RTX 6000 Ada and an+ RTX 3090, Gemma 4 12B at ubatch 1120."""++ @pytest.mark.parametrize("slots,ctx,measured", [(8, 65536, 971), (16, 131072, 1601)])+ def test_covers_the_measured_split(self, slots, ctx, measured):+ b = _measured("Gemma-4-12B")+ per_device = (+ b._estimate_compute_buffer_bytes(n_ubatch = 1120, n_parallel = slots)+ + b._compute_buffer_ctx_bytes(ctx, 1120, "f16", layer_split = True, n_parallel = slots)+ ) / MIB+ one_slot_window = (+ b._estimate_compute_buffer_bytes(n_ubatch = 1120, n_parallel = slots)+ + b._compute_buffer_ctx_bytes(ctx, 1120, "f16", layer_split = True)+ ) / MIB+ assert one_slot_window < measured <= per_device <= measured * 1.1+++class TestOutputRowsByBuild:+ """Before ggml-org/llama.cpp#23861 (b9460) a context reserved an output row for every+ micro-batch token, once, whatever the slot count: on Vulkan, Gemma 4 E2B at ubatch+ 512 reserved 515 MiB on b9415 and 137 MiB on b10909, the same at one slot or four."""++ @pytest.mark.parametrize(+ "output,build",+ [+ ("version: 0.4.0-dev (build 10909, commit 329b6160f)\nbuilt with GNU", 10909),+ ("version: 9415 (06d26dfdf)\nbuilt with GNU 11.4.0", 9415),+ ("version: 1 (unknown)\n", None),+ ("no version here", None),+ ],+ )+ def test_the_build_is_read_from_either_version_format(+ self, tmp_path, monkeypatch, output, build+ ):+ import subprocess+ import types++ binary = tmp_path / "llama-server"+ binary.write_bytes(b"")+ monkeypatch.setattr(LlamaCppBackend, "_build_number_cache", {})+ monkeypatch.setattr(+ subprocess,+ "run",+ lambda *a, **k: types.SimpleNamespace(stdout = output, stderr = "", returncode = 0),+ )+ assert LlamaCppBackend.probe_build_number(str(binary)) == build++ @pytest.mark.parametrize(+ "build,legacy", [(9415, True), (9459, True), (9460, False), (None, False)]+ )+ def test_only_a_known_older_build_reserves_micro_batch_rows(self, monkeypatch, build, legacy):+ monkeypatch.setattr(+ LlamaCppBackend, "probe_build_number", classmethod(lambda cls, binary = None: build)+ )+ assert LlamaCppBackend.reserves_micro_batch_outputs() is legacy++ def test_an_older_build_is_charged_a_row_per_micro_batch_token(self):+ # Measured b9415 Vulkan compute buffers (MiB) at 32768 cells, one and four slots.+ cases = [+ (dict(vocab = 128256, embd = 2048, ff = 8192), 512, 254.50),+ (dict(vocab = 151936, embd = 4096, ff = 12288), 2048, 1667.05),+ (dict(vocab = 262144, embd = 1536, ff = 12288, ple = 256 * 35), 2048, 2704.08),+ ]+ for dims, ub, measured in cases:+ b = _backend(vocab = dims["vocab"], embd = dims["embd"])+ b._feed_forward_length = dims["ff"]+ b._embedding_length_per_layer_input = 0+ b._n_layers = 1+ if "ple" in dims:+ b._embedding_length_per_layer_input, b._n_layers = 256, 35+ b._reserves_micro_batch_outputs = True+ for slots in (1, 4):+ est = (+ b._estimate_compute_buffer_bytes(n_ubatch = ub, n_parallel = slots)+ + b._compute_buffer_ctx_bytes(32768, ub, "f16")+ ) / MIB+ assert measured <= est <= measured * 1.5, (dims, slots, est)+ b._reserves_micro_batch_outputs = False+ assert b._estimate_compute_buffer_bytes(n_ubatch = ub, n_parallel = 1) < (+ dims["vocab"] * ub * 4+ )diff --git a/studio/backend/tests/test_cuda_sm_gate.py b/studio/backend/tests/test_cuda_sm_gate.pyindex ae586feafe8..eab730014dc 100644--- a/studio/backend/tests/test_cuda_sm_gate.py+++ b/studio/backend/tests/test_cuda_sm_gate.py@@ -217,6 +217,10 @@ def _gated_backend(monkeypatch.setattr(LlamaCppBackend, "_is_unsloth_managed_binary", staticmethod(lambda _binary: True))+ # The build probe runs `--version`, which the Popen patch below would count as a launch.+ monkeypatch.setattr(+ LlamaCppBackend, "probe_build_number", classmethod(lambda cls, binary = None: None)+ )backend = LlamaCppBackend()backend._get_gpu_memory = lambda _binary = None, **_kw: []backend._get_gpu_free_memory = lambda _binary = None, **_kw: []diff --git a/studio/backend/tests/test_estimator_flash_attn_parity.py b/studio/backend/tests/test_estimator_flash_attn_parity.pyindex d9c32b525b2..0d6be183208 100644--- a/studio/backend/tests/test_estimator_flash_attn_parity.py+++ b/studio/backend/tests/test_estimator_flash_attn_parity.py@@ -336,3 +336,50 @@ def test_pass_through_adapters_are_charged_and_follow_the_base_placement(assert (missing is not None and missing.adapters_unsized), "an unsizable adapter has to mark the total a floor rather than vanish"+++def _runtime(path, extras = None):+ runtime = ri._gguf_runtime_bytes(path, 32768, extras, 1, None, False)+ return runtime.kv_bytes, runtime.compute_bytes+++def _flash_attn_caps(monkeypatch, supported):+ monkeypatch.setattr(+ ri.LlamaCppBackend,+ "probe_server_capabilities",+ classmethod(lambda cls, binary = None: {"found": True, "supports_flash_attn": supported}),+ )+++def test_flash_attention_follows_the_launch_argv_not_the_inherited_env(+ monkeypatch, qwen3_shaped_gguf+):+ """The managed --flash-attn on is parsed after LLAMA_ARG_FLASH_ATTN, so only the+ extras can turn it off; a build without the flag drops both flag and env."""+ monkeypatch.delenv("LLAMA_ARG_FLASH_ATTN", raising = False)+ _flash_attn_caps(monkeypatch, True)+ on = _runtime(qwen3_shaped_gguf)+ off = _runtime(qwen3_shaped_gguf, ["--flash-attn", "off"])+ assert off[1] > on[1]++ monkeypatch.setenv("LLAMA_ARG_FLASH_ATTN", "off")+ assert _runtime(qwen3_shaped_gguf) == on++ monkeypatch.setenv("LLAMA_ARG_FLASH_ATTN", "on")+ _flash_attn_caps(monkeypatch, False)+ assert _runtime(qwen3_shaped_gguf) == off+++def test_grok_is_priced_without_flash_attention(monkeypatch, qwen3_shaped_gguf):+ """llama.cpp forces flash attention off for Grok whatever the launch asks for."""+ monkeypatch.delenv("LLAMA_ARG_FLASH_ATTN", raising = False)+ _flash_attn_caps(monkeypatch, True)+ off = _runtime(qwen3_shaped_gguf, ["--flash-attn", "off"])+ real_read = ri.LlamaCppBackend._read_gguf_metadata++ def read_as_grok(self, path):+ real_read(self, path)+ self._architecture = "grok"++ monkeypatch.setattr(ri.LlamaCppBackend, "_read_gguf_metadata", read_as_grok)+ assert _runtime(qwen3_shaped_gguf) == offdiff --git a/studio/backend/tests/test_kv_cache_estimation.py b/studio/backend/tests/test_kv_cache_estimation.pyindex f62bcf7207c..871b027966e 100644--- a/studio/backend/tests/test_kv_cache_estimation.py+++ b/studio/backend/tests/test_kv_cache_estimation.py@@ -207,6 +207,12 @@ class TestGGUFParserNewFields:("_ssm_state_size", "ssm.state_size", 128),("_ssm_group_count", "ssm.group_count", 16),("_ssm_conv_kernel", "ssm.conv_kernel", 4),+ ("_feed_forward_length", "feed_forward_length", 12288),+ ("_expert_used_count", "expert_used_count", 8),+ ("_expert_feed_forward_length", "expert_feed_forward_length", 512),+ ("_expert_shared_feed_forward_length", "expert_shared_feed_forward_length", 512),+ ("_expert_shared_count", "expert_shared_count", 1),+ ("_embedding_length_per_layer_input", "embedding_length_per_layer_input", 256),],)def test_field_parsed(self, field, gguf_key, value):@@ -256,6 +262,14 @@ class TestGGUFParserNewFields:assert b._n_kv_heads == 8assert b._sliding_window_pattern == [True, True, True, True, True, False]+ def test_per_layer_feed_forward_length_keeps_the_widest(self):+ # Gemma 4 E2B stores one FFN width per layer; the compute buffer is set by+ # the widest.+ b = _backend_from_gguf(+ "gemma4", {"block_count": 3, "feed_forward_length": [6144, 12288, 6144]}+ )+ assert b._feed_forward_length == 12288+class TestArchSwaPatternDefaults:"""Bootstrap arch table fires when GGUF reports `sliding_window` butdiff --git a/studio/backend/tests/test_llama_cpp_placement.py b/studio/backend/tests/test_llama_cpp_placement.pyindex 05371d102b7..b1cb2ae41e5 100644--- a/studio/backend/tests/test_llama_cpp_placement.py+++ b/studio/backend/tests/test_llama_cpp_placement.py@@ -110,6 +110,32 @@ def _backend(tmp_path: Path, *, vulkan: bool, memory):return backend, gguf+# A synthetic compute buffer for the slot-reduction planner fixtures, per micro-batch+# token: a fixed base, plus a fixed cost for every slot past the first.+SLOT_COMPUTE_BASE_BYTES_PER_TOKEN = 92 * 1024+SLOT_COMPUTE_EXTRA_SLOT_BYTES_PER_TOKEN = 1116 * 1024+++def _install_slot_scaled_compute(backend):+ """Use a synthetic per-slot cost to exercise slot reduction on dense fixtures."""++ def compute(+ *,+ n_ubatch = None,+ n_parallel = 1,+ **_kwargs,+ ):+ ub = max(1, int(backend._DEFAULT_N_UBATCH if n_ubatch is None else n_ubatch))+ extra_slots = max(0, int(n_parallel) - 1)+ return ub * (+ SLOT_COMPUTE_BASE_BYTES_PER_TOKEN+ + extra_slots * SLOT_COMPUTE_EXTRA_SLOT_BYTES_PER_TOKEN+ )++ backend._estimate_compute_buffer_bytes = compute+ return backend++def _backend_non_vulkan(*args,vulkan = False,@@ -650,11 +676,7 @@ def test_auto_classifies_placement_on_the_device_flags_the_child_gets(tmp_path):def _hybrid_reserve_backend(tmp_path: Path, *, caps = None):- """A Hybrid Mamba target on one 24 GB card with the MTP-overhead math live.-- The drafter's own KV is stubbed away so the only moving term is the target's- recurrent rollback state, which is what the reserve has to keep charging.- """+ """Hybrid Mamba on a 24 GB card, with only target rollback state affecting MTP cost."""gb = 1024**3backend, gguf = _backend(tmp_path, vulkan = False, memory = [(0, 24_576, 24_576)])sidecar = tmp_path / "dflash-model-Q8_0.gguf"@@ -664,6 +686,7 @@ def _hybrid_reserve_backend(tmp_path: Path, *, caps = None):backend._compute_buffer_ctx_bytes = lambda *args, **kwargs: 0backend._estimate_compute_buffer_bytes = lambda **kwargs: 1backend._mtp_draft_kv_bytes = lambda *args, **kwargs: 0+ backend._mtp_draft_compute_bytes = lambda *args, **kwargs: 0backend._select_gpus = lambda *args, **kwargs: ([0], False)backend._select_gpus_split_aware = lambda *args, **kwargs: ([0], False)@@ -742,9 +765,9 @@ def _recorded_mtp_reserve_and_callbacks(backend, gguf, **load_kwargs):def test_a_cpu_pinned_drafter_still_pays_the_hybrid_target_rollback(tmp_path):# -ngld 0 moves the drafter's weights and KV to host memory, but the rollback- # snapshots live in the TARGET context, so they stay on the GPU. Releasing the- # whole reserve here undercounts them and the fit can pick a placement that- # spills.+ # snapshots and verification output rows live in the TARGET context, so they stay+ # on the GPU. Releasing the whole reserve here undercounts them and the fit can+ # pick a placement that spills.backend, gguf, sidecar = _hybrid_reserve_backend(tmp_path)charged = _recorded_mtp_reserve_std(@@ -756,9 +779,9 @@ def test_a_cpu_pinned_drafter_still_pays_the_hybrid_target_rollback(tmp_path):)# After the launch: the GGUF dims land when the load reads the metadata.- expected = backend._mamba_recurrent_state_bytes(n_parallel = 4) * 2- assert expected > 0- assert set(charged) == {expected}+ rollback = backend._mamba_recurrent_state_bytes(n_parallel = 4) * 2+ assert rollback > 0+ assert set(charged) == {rollback + backend._spec_verify_rows_bytes(4, 2)}def test_the_cpu_drafter_reserve_still_reprices_per_slot_candidate(tmp_path):@@ -783,6 +806,7 @@ def test_the_cpu_drafter_reserve_still_reprices_per_slot_candidate(tmp_path):for slots in (1, 2, 4):assert fn(8192, _np = slots, _n_ubatch = 512) == (backend._mamba_recurrent_state_bytes(n_parallel = slots) * 2+ + backend._spec_verify_rows_bytes(slots, 2))# Per-slot state, not per-token: context does not move it.assert fn(2048, _np = 4, _n_ubatch = 512) == fn(131072, _np = 4, _n_ubatch = 512)@@ -1125,10 +1149,10 @@ def test_a_busy_second_gpu_does_not_condemn_a_drafter_the_first_one_holds(tmp_padef test_a_cpu_offloaded_sidecar_releases_the_byte_accurate_reserve(tmp_path):"""-ngld 0 puts the drafter in host memory, and a separate sidecar displaces- the embedded head that mtp_overhead_fn was sized from, so nothing speculative- is GPU-resident. The flat fraction already stands down here; the byte-accurate- callback did not, so the fit went on charging GPU bytes for a drafter that- allocates none, cutting the context or taking --fit for them.+ the embedded head that mtp_overhead_fn was sized from, so only the target's+ verification rows stay GPU-resident. The flat fraction already stands down here;+ the byte-accurate callback did not, so the fit went on charging GPU bytes for a+ drafter that allocates none, cutting the context or taking --fit for them."""backend, gguf, sidecar = _tight_vram_backend(tmp_path, drafter_gb = 12.0)backend._nextn_predict_layers = 1@@ -1156,7 +1180,8 @@ def test_a_cpu_offloaded_sidecar_releases_the_byte_accurate_reserve(tmp_path):)assert charged, "the fit never ran, so this proves nothing"- assert set(charged) == {0}+ # One slot at the three-token DSpark depth.+ assert set(charged) == {backend._spec_verify_rows_bytes(1, 3)}def test_an_mla_model_keeps_the_reason_that_actually_dropped_its_drafter(tmp_path):@@ -3588,3 +3613,86 @@ def test_a_drafter_carrying_its_own_embeddings_still_reaches_the_command(tmp_patassert cmd[cmd.index("--model-draft") + 1] == str(drafter)assert backend.mtp_draft_path == str(drafter)assert backend.mtp_draft_suppressed_path is None+++def _recording_compute_backend(+ tmp_path,+ monkeypatch,+ *,+ build = 10909,+):+ """A dense backend on one 24 GB card whose compute terms are the real estimator,+ recording what the loader asks of them."""+ backend, gguf = _backend(tmp_path, vulkan = False, memory = [(0, 24_576, 24_576)])++ def read(_path):+ backend._architecture = "qwen3"+ backend._vocab_size = 151936+ backend._embedding_length = 4096+ backend._feed_forward_length = 12288+ backend._n_layers = 36+ backend._n_heads = 32+ backend._n_kv_heads = 8+ backend._kv_key_length = 128+ backend._kv_value_length = 128+ backend._context_length = 40960++ backend._read_gguf_metadata = read+ backend._get_gguf_size_bytes = lambda _path: 4 * 1024**3+ del backend._can_estimate_kv # the real one, now that the dims are set+ backend.probe_server_capabilities = lambda _binary = None: {+ "supports_kv_unified": True,+ "supports_flash_attn": True,+ "flash_attn_takes_value": True,+ }+ monkeypatch.setattr(+ LlamaCppBackend, "probe_build_number", classmethod(lambda cls, binary = None: build)+ )+ calls = {"ctx": [], "flat": [], "kv": []}+ real_ctx = backend._compute_buffer_ctx_bytes+ real_flat = backend._estimate_compute_buffer_bytes+ real_kv = backend._estimate_kv_cache_bytes++ def ctx(*args, **kwargs):+ calls["ctx"].append(kwargs.get("flash_attn", True))+ return real_ctx(*args, **kwargs)++ def flat(**kwargs):+ calls["flat"].append(backend._reserves_micro_batch_outputs)+ return real_flat(**kwargs)++ def kv(*args, **kwargs):+ calls["kv"].append(kwargs.get("flash_attn", True))+ return real_kv(*args, **kwargs)++ backend._compute_buffer_ctx_bytes = ctx+ backend._estimate_compute_buffer_bytes = flat+ backend._estimate_kv_cache_bytes = kv+ return backend, gguf, calls+++ "extra_args,expected",+ [([], True), (["--flash-attn", "off"], False), (["-fa", "off", "--flash-attn", "on"], True)],+)+def test_the_loader_prices_the_attention_mode_it_launches(+ tmp_path, monkeypatch, extra_args, expected+):+ """Compute follows the launch's attention mode; KV covers the no-flash retry."""+ backend, gguf, calls = _recording_compute_backend(tmp_path, monkeypatch)++ assert _launch(backend, gguf, n_ctx = 0, n_parallel = 4, extra_args = extra_args)["cmd"]++ assert calls["ctx"], "the fit never priced the context term"+ assert set(calls["ctx"]) == {expected}+ assert False in calls["kv"]++[email protected]("build,expected", [(9415, True), (10909, False), (None, False)])+def test_the_loader_prices_the_output_rows_of_its_build(tmp_path, monkeypatch, build, expected):+ backend, gguf, calls = _recording_compute_backend(tmp_path, monkeypatch, build = build)++ assert _launch(backend, gguf, n_ctx = 0, n_parallel = 4)["cmd"]++ assert calls["flat"], "the fit never priced the flat compute buffer"+ assert set(calls["flat"]) == {expected}diff --git a/studio/backend/tests/test_load_mode_fit_matrix.py b/studio/backend/tests/test_load_mode_fit_matrix.pyindex 385aeb4752c..337f1acbbb7 100644--- a/studio/backend/tests/test_load_mode_fit_matrix.py+++ b/studio/backend/tests/test_load_mode_fit_matrix.py@@ -1480,14 +1480,8 @@ def test_a_cpu_only_kv_cache_is_not_charged_twice(monkeypatch):def test_a_tensor_split_charges_the_replicated_compute_buffer():- """Checked at the source: reaching this call needs a real multi-GPU probe.-- _plan_tensor_parallel admits devices against, and reserves per device,- _estimate_compute_buffer_bytes(per_device_tensor = True). The fit has to charge- the same thing: the layer-mode lump is ONE device's buffer at the smaller rate,- and understating it on a pooled multi-GPU credit is the direction that claims a- fit that is not there.- """+ """Check that the fit charges the same per-device buffer as _plan_tensor_parallel.+ This source check avoids requiring a multi-GPU probe."""from core.inference.llama_cpp import LlamaCppBackend as Bimport inspect@@ -1508,26 +1502,18 @@ def test_a_tensor_split_charges_the_replicated_compute_buffer():) in compact-def test_the_tensor_rate_really_exceeds_the_layer_one():- """The under-charge the fix closes is not a rounding difference: at one slot- the layer figure drops the vocab-width output buffer entirely."""-- class _Dims:- # is_embedding_gguf is a read-only property on the real backend.- _vocab_size = 151936- _embedding_length = 5120- is_embedding_gguf = False- _DEFAULT_N_UBATCH = LlamaCppBackend._DEFAULT_N_UBATCH- _COMPUTE_BUFFER_SAFETY = LlamaCppBackend._COMPUTE_BUFFER_SAFETY-- layer = LlamaCppBackend._estimate_compute_buffer_bytes(- _Dims(), n_ubatch = 512, n_parallel = 1, per_device_tensor = False- )- tensor = LlamaCppBackend._estimate_compute_buffer_bytes(- _Dims(), n_ubatch = 512, n_parallel = 1, per_device_tensor = True- )- assert tensor > layer- # 4-GPU tensor split: what the fit used to charge vs what it now does.+def test_a_tensor_split_pays_the_buffer_on_every_device():+ """Each device of a tensor split reserves the whole single-GPU compute buffer+ (Qwen3 8B across two GPUs: 128 MiB on each at ubatch 512, 512 MiB at 2048), so+ the fit's per-device multiplication is the whole difference from a layer lump."""+ b = LlamaCppBackend()+ b._vocab_size = 151936+ b._embedding_length = 4096+ b._feed_forward_length = 12288+ layer = b._estimate_compute_buffer_bytes(n_ubatch = 2048, n_parallel = 1)+ tensor = b._estimate_compute_buffer_bytes(n_ubatch = 2048, n_parallel = 1, per_device_tensor = True)+ assert tensor == layer+ # 4-GPU tensor split: three more copies than the layer lump.assert 4 * tensor - layer > GIBdiff --git a/studio/backend/tests/test_memory_estimate.py b/studio/backend/tests/test_memory_estimate.pyindex 4b957320c24..ade01122918 100644--- a/studio/backend/tests/test_memory_estimate.py+++ b/studio/backend/tests/test_memory_estimate.py@@ -1383,6 +1383,10 @@ class TestDrafterAccounting:return 1.0 + (0.0 if pinned else drafter_bytes / 1024**3)monkeypatch.setattr(ri, "_gguf_resident_file_gb", _files)+ verify_rows = 7 * 1024 * 1024+ monkeypatch.setattr(+ ri.LlamaCppBackend, "_spec_verify_rows_bytes", lambda *a, **k: verify_rows+ )on_gpu = ri._gguf_memory_breakdown(config, gqa_gguf, n_ctx = 8192)assert on_gpu.drafter_runtime_bytes > 0# Default placement: all of it, so the row says nothing rather than guessing.@@ -1391,7 +1395,8 @@ class TestDrafterAccounting:on_cpu = ri._gguf_memory_breakdown(config, gqa_gguf, n_ctx = 8192, llama_extra_args = ["--spec-draft-ngl", "0"])- assert on_cpu.drafter_runtime_gpu_bytes == 0+ # Only the target's verification rows stay with the GPU target.+ assert on_cpu.drafter_runtime_gpu_bytes == verify_rows# --no-kv-offload moves the TARGET cache and leaves the drafter alone. A# boolean read off kv_on_gpu would call the whole term host-resident here,@@ -1687,7 +1692,12 @@ class TestSpeculativeModeTerms:probe._read_gguf_metadata(mla)return probe._estimate_kv_cache_bytes(n_ctx, "f16")- def test_a_dspark_drafter_is_not_charged_the_targets_context(self, mla, tmp_path, priced_files):+ def test_a_dspark_drafter_is_not_charged_the_targets_context(+ self, mla, tmp_path, priced_files, monkeypatch+ ):+ # The two drafters' compute buffers differ with their own headers; this pins+ # the cache terms.+ monkeypatch.setattr(ri.LlamaCppBackend, "_mtp_draft_compute_bytes", lambda *a, **k: 0)ctx = 131072mtp = ri._gguf_memory_breakdown(self._config(mla, tmp_path, "mtp"), mla, n_ctx = ctx)dspark = ri._gguf_memory_breakdown(self._config(mla, tmp_path, "dspark"), mla, n_ctx = ctx)@@ -1699,11 +1709,13 @@ class TestSpeculativeModeTerms:assert dspark.total_bytes < mtp.total_bytesdef test_a_gpu_drafter_is_charged_when_the_target_keeps_no_layers(- self, mla, tmp_path, priced_files+ self, mla, tmp_path, priced_files, monkeypatch):# A separate drafter does not inherit --gpu-layers: llama.cpp overwrites it# with the draft placement, whose default is auto. At --gpu-layers 0 the- # drafter is still on the GPU and still holds its cache there.+ # drafter is still on the GPU and still holds its cache there. Verification+ # rows follow the target and are pinned in their own test.+ monkeypatch.setattr(ri.LlamaCppBackend, "_spec_verify_rows_bytes", lambda *a, **k: 0)config = self._config(mla, tmp_path, "dspark")manual = dict(gpu_memory_mode = "manual", gpu_layers = 0, n_ctx = 131072)on_gpu = ri._gguf_memory_breakdown(config, mla, **manual)@@ -1719,8 +1731,9 @@ class TestSpeculativeModeTerms:)def test_the_target_side_of_the_reserve_stays_with_the_target(- self, mla, tmp_path, priced_files+ self, mla, tmp_path, priced_files, monkeypatch):+ monkeypatch.setattr(ri.LlamaCppBackend, "_spec_verify_rows_bytes", lambda *a, **k: 0)ctx = 131072config = self._config(mla, tmp_path, "mtp")copy_bytes = self._target_ctx_copy(mla, ctx)@@ -1750,6 +1763,25 @@ class TestSpeculativeModeTerms:# the loader reserves for the same launch.assert pinned.gpu_bytes - no_layers_pinned.gpu_bytes > copy_bytes+ def test_verification_rows_follow_the_targets_layers(+ self, mla, tmp_path, priced_files, monkeypatch+ ):+ """They are target compute: on the GPU with the target's layers, whichever+ device the drafter uses."""+ verify_rows = 7 * 1024 * 1024+ monkeypatch.setattr(+ ri.LlamaCppBackend, "_spec_verify_rows_bytes", lambda *a, **k: verify_rows+ )+ config = self._config(mla, tmp_path, "dspark")+ pin = ["--spec-draft-ngl", "0"]+ no_layers = dict(gpu_memory_mode = "manual", gpu_layers = 0)+ target_gpu = ri._gguf_memory_breakdown(config, mla, n_ctx = 8192, llama_extra_args = pin)+ target_cpu = ri._gguf_memory_breakdown(config, mla, n_ctx = 8192, **no_layers)+ assert target_gpu.drafter_runtime_gpu_bytes == verify_rows+ assert target_cpu.drafter_runtime_gpu_bytes == (+ target_cpu.drafter_runtime_bytes - verify_rows+ )+def test_extras_owning_the_spec_block_price_the_builds_depth(self, mla):# _build_speculative_flags returns without emitting a depth once the extras# name --spec-type, so neither the panel field nor Studio's 2/3 platform@@ -1854,9 +1886,15 @@ class TestLaunchShapedPricing:)assert full.kv_checkpoint_bytes == 0assert windowed.kv_checkpoint_bytes > 0- # Drafter runtime follows the cache proper; its checkpoint share is not estimated.- assert full.drafter_runtime_bytes == full.kv_bytes - full.kv_checkpoint_bytes- assert windowed.drafter_runtime_bytes == windowed.kv_bytes - windowed.kv_checkpoint_bytes+ # Same header on both sides, so the drafter's cache moves exactly as the target's+ # GPU cache does. By DIFFERENCE, not equality: the draft decode graph's floor+ # (_MTP_DRAFT_COMPUTE_BYTES) now rides on top of this figure and does not follow+ # --swa-full, so the two sides no longer match outright.+ assert full.drafter_runtime_bytes - windowed.drafter_runtime_bytes == (+ (full.kv_bytes - full.kv_checkpoint_bytes)+ - (windowed.kv_bytes - windowed.kv_checkpoint_bytes)+ )+ assert windowed.drafter_runtime_bytes > windowed.kv_bytes - windowed.kv_checkpoint_bytesdef test_a_cpu_device_selection_takes_the_weights_off_the_gpu(self, spec_config, swa):"""``--device none`` runs on the CPU whatever the layer count says.@@ -1897,8 +1935,9 @@ class TestLaunchShapedPricing:assert many.gpu_bytes == none.gpu_bytesdef test_one_card_is_priced_as_the_layer_load_it_launches(self, spec_config, swa):- # Tensor mode needs two usable GPUs. Below that load_model drops it, so pricing- # tensor charged per-device compute buffers for a launch that runs neither.+ # Tensor mode needs two usable GPUs. Below that load_model drops it, so the+ # panel prices the layer load. A tensor device holds the single-device compute+ # buffer, so on one card the two price the same.## The cache type is no longer part of this: #8939 removed the gate that rewrote# both axes to f16 on a tensor split, so a quantized KV now survives one and the@@ -1914,8 +1953,8 @@ class TestLaunchShapedPricing:assert downgraded.cache_type_kv == "q4_0"assert as_tensor.cache_type_kv == "q4_0"assert downgraded.kv_bytes == as_tensor.kv_bytes- assert downgraded.compute_bytes < as_tensor.compute_bytes- assert downgraded.total_bytes < as_tensor.total_bytes+ assert downgraded.compute_bytes == as_tensor.compute_bytes+ assert downgraded.total_bytes == as_tensor.total_bytesdef test_manual_auto_layers_is_priced_as_the_layer_load_it_launches(self, spec_config, swa):# Manual with Auto layers hands the budget to llama.cpp --fit, which load_modeldiff --git a/studio/backend/tests/test_mmproj_placement_policy.py b/studio/backend/tests/test_mmproj_placement_policy.pyindex 8f19047bf43..df4f00ee198 100644--- a/studio/backend/tests/test_mmproj_placement_policy.py+++ b/studio/backend/tests/test_mmproj_placement_policy.py@@ -1031,7 +1031,7 @@ _CC_PER_TOKEN = 1536 # 6 MiB at 4096, the rate the bundled estimator producesdef _split_rate_backend(tmp_path, *, memory, **kwargs):backend, gguf = _backend(tmp_path, memory = memory, **kwargs)backend._compute_buffer_ctx_bytes = (- lambda n_ctx, n_ubatch = None, cache_type_kv = None, *, layer_split = False: (+ lambda n_ctx, n_ubatch = None, cache_type_kv = None, *, layer_split = False, **_kw: (n_ctx * _CC_PER_TOKEN * (LlamaCppBackend._CTX_COMPUTE_SPLIT_MULT if layer_split else 1)))diff --git a/studio/backend/tests/test_mtp_mla_target_ctx.py b/studio/backend/tests/test_mtp_mla_target_ctx.pyindex 5c7e60e41f3..70b17065f7f 100644--- a/studio/backend/tests/test_mtp_mla_target_ctx.py+++ b/studio/backend/tests/test_mtp_mla_target_ctx.py@@ -109,6 +109,9 @@ def _make_mla_backend(b._kv_value_length_swa = Noneb._draft_backend_cache = Noneb._vocab_size = vocab+ # The speculative compute buffers ride on this reserve too; test_compute_buffer+ # prices them, and these cases pin the cache terms.+ b._mtp_draft_compute_bytes = lambda *args, **kwargs: 0return bdiff --git a/studio/backend/tests/test_mtp_reserve_note_and_unload_log.py b/studio/backend/tests/test_mtp_reserve_note_and_unload_log.pyindex c6242a23e9e..763b65a9bf3 100644--- a/studio/backend/tests/test_mtp_reserve_note_and_unload_log.py+++ b/studio/backend/tests/test_mtp_reserve_note_and_unload_log.py@@ -176,12 +176,9 @@ def test_an_unload_of_a_resident_model_still_logs_one_event(backend, monkeypatchdef test_the_real_estimator_ignores_both_axes_for_a_dense_embedded_head(backend):- """The premise the note now asks about, pinned against the estimator itself.-- Without this the note's tests would only prove it renders whatever a stand-in- tells it, and the claim that slots and ubatch really are inert on this branch- would rest on reading the code.- """+ """Slots and ubatch leave the dense draft cache unchanged but grow its compute cost."""+ real_compute = backend._mtp_draft_compute_bytes+ backend._mtp_draft_compute_bytes = lambda *args, **kwargs: 0backend._nextn_predict_layers = 1backend._n_kv_heads = 8backend._n_heads = 64@@ -213,3 +210,13 @@ def test_the_real_estimator_ignores_both_axes_for_a_dense_embedded_head(backend)8192, spec_draft_n_max = 4, n_parallel = 3, kv_unified = False, n_ubatch = 512)assert split != base++ # With the compute buffers back, a slot adds verify rows and the micro-batch the+ # draft context's activations, so the note has both to name.+ backend._mtp_draft_compute_bytes = real_compute+ backend._vocab_size = 151936+ backend._embedding_length = 4096+ backend._feed_forward_length = 12288+ moved = _reserve(2, 512)+ assert _reserve(3, 512) > moved+ assert _reserve(2, 2048) > moveddiff --git a/studio/backend/tests/test_mtp_vram_budget.py b/studio/backend/tests/test_mtp_vram_budget.pyindex 88ab0ee2b15..f7538c1ab21 100644--- a/studio/backend/tests/test_mtp_vram_budget.py+++ b/studio/backend/tests/test_mtp_vram_budget.py@@ -127,6 +127,9 @@ def _make_backend(b._kv_key_length_swa = Noneb._kv_value_length_swa = Noneb._draft_backend_cache = None+ # The speculative compute buffers ride on this reserve too; test_compute_buffer+ # prices them, and these cases pin the cache and rollback terms.+ b._mtp_draft_compute_bytes = lambda *args, **kwargs: 0return bdiff --git a/studio/backend/tests/test_slot_offload_fit.py b/studio/backend/tests/test_slot_offload_fit.pyindex bf8bd63231a..487220d813c 100644--- a/studio/backend/tests/test_slot_offload_fit.py+++ b/studio/backend/tests/test_slot_offload_fit.py@@ -20,6 +20,9 @@ from pathlib import Path_BACKEND_DIR = str(Path(__file__).resolve().parent.parent)if _BACKEND_DIR not in sys.path:sys.path.insert(0, _BACKEND_DIR)+_TESTS_DIR = str(Path(__file__).resolve().parent)+if _TESTS_DIR not in sys.path:+ sys.path.insert(0, _TESTS_DIR)_loggers_stub = _types.ModuleType("loggers")_loggers_stub.get_logger = lambda name: __import__("logging").getLogger(name)@@ -39,11 +42,15 @@ def _backend(kv_calls = None,):"""Backend with the dims the compute buffer reads; KV mocked to a fixed size so the- only slot-dependent term is the compute buffer (485 MiB/slot f32 output x 1.15)."""+ only slot-dependent term is the synthetic compute buffer from+ ``_install_slot_scaled_compute`` (558 MiB per extra slot at ubatch 512)."""+ from test_llama_cpp_placement import _install_slot_scaled_compute+b = LlamaCppBackend.__new__(LlamaCppBackend)b._vocab_size = vocabb._embedding_length = embdb._key_length_mla = None+ _install_slot_scaled_compute(b)def estimate(ctx,@@ -88,8 +95,8 @@ def _run(class TestSlotsThatFitOnGpu:- """Compute-buffer per slot (vocab 248320, embd 5120): cb(1)=46, cb(2)=604, cb(3)=1162,- cb(4)=1719 MiB. Single 24 GB card usable = 24576 - 0.03*24576 = 23839 MiB."""+ """Synthetic compute buffer per slot count: cb(1)=46, cb(2)=604, cb(3)=1162,+ cb(4)=1720 MiB. Single 24 GB card usable = 24576 - 0.03*24576 = 23839 MiB."""def test_reduces_to_largest_fitting_slot(self):# base+KV = 22500: par4 (24219) over 23839, par3 (23662) fits -> 3 slots on GPU.@@ -159,12 +166,8 @@ class TestSlotsThatFitOnGpu:assert all(call["swa_full"] is True for call in calls)def test_micro_batch_is_re_derived_per_candidate(self):- """llama-server raises --batch-size to max(slots, 2) and llama.cpp caps the- micro-batch against it, so a batch below the requested slot count shrinks as the- candidates do: n_batch=1 launches the 3-slot candidate at -b 3, not the 64 the- 4-slot request resolved to. At these dims that is 6.8 MiB of compute buffer- against 145.2, so pricing every candidate at the requested count's micro-batch- rejects a 3-slot fit that launches comfortably and cuts serving capacity for it."""+ """Recompute the batch floor and ubatch for each candidate slot count.+ Keeping the requested ubatch would reject the fitting three-slot candidate."""from core.inference.llama_cpp import _emitted_n_batch, _extra_args_n_ubatchdef ubatch_for_slots(slots: int):diff --git a/studio/backend/tests/test_slot_reduction_context_refit.py b/studio/backend/tests/test_slot_reduction_context_refit.pyindex 42f0156c81c..ee7aff84860 100644--- a/studio/backend/tests/test_slot_reduction_context_refit.py+++ b/studio/backend/tests/test_slot_reduction_context_refit.py@@ -4,7 +4,8 @@"""Regression tests for context fitting after serving-slot reduction.The synthetic plans use Qwen3.8-27B metadata and capture the generated command.-Each case names its speculative mode because unknown values resolve to Auto.+Cases specify speculation explicitly and use synthetic slot costs unless testing+the real estimator."""from __future__ import annotations@@ -20,7 +21,11 @@ if _TESTS_DIR not in sys.path:import pytest # noqa: E402import core.inference.llama_cpp as llama_cpp # noqa: E402-from test_llama_cpp_placement import _backend, _launch # noqa: E402+from test_llama_cpp_placement import ( # noqa: E402+ _backend,+ _install_slot_scaled_compute,+ _launch,+)MIB = 1024 * 1024NATIVE_CTX = 262144@@ -72,6 +77,23 @@ DENSE = {"_context_length": NATIVE_CTX,}+# Gemma 3 12B-shaped: five sliding-window layers in six.+SLIDING_WINDOW = {+ "_architecture": "gemma3",+ "_vocab_size": 262144,+ "_n_layers": 48,+ "_n_kv_heads": 8,+ "_n_heads": 16,+ "_embedding_length": 3840,+ "_feed_forward_length": 15360,+ "_kv_key_length": 256,+ "_kv_value_length": 256,+ "_key_length_mla": None,+ "_context_length": 131072,+ "_sliding_window": 1024,+ "_sliding_window_pattern": [i % 6 != 5 for i in range(48)],+}+def _plan(tmp_path,@@ -83,8 +105,10 @@ def _plan(n_ctx = 0,metadata = HYBRID,gpus = 1,+ slot_scaled_compute = True,):- """Return the generated placement plan. ``vram_mib`` may be a per-card sequence."""+ """Return the generated placement plan. ``vram_mib`` may be a per-card sequence.+ ``slot_scaled_compute`` False prices the compute buffer with the real estimator."""cards = list(vram_mib) if isinstance(vram_mib, (tuple, list)) else [vram_mib] * gpusmemory = [(i, mib, mib) for i, mib in enumerate(cards)]backend, gguf = _backend(tmp_path, vulkan = False, memory = memory)@@ -96,6 +120,8 @@ def _plan(backend._read_gguf_metadata = readbackend._get_gguf_size_bytes = lambda _path: weights_mib * MIBdel backend._can_estimate_kv # the real one, now that the dims are set+ if slot_scaled_compute:+ _install_slot_scaled_compute(backend)backend.probe_server_capabilities = lambda _binary = None: {"mtp_token": "draft-mtp","supports_ngram_mod": True,@@ -127,8 +153,8 @@ class TestTheLaunchedCountOwnsTheContext:"weights_mib,asked,slots,ctx",[(_KEEPS_THREE_MIB, 4, 3, 12_288),- (_KEEPS_TWO_MIB, 4, 2, 13_824),- (_KEEPS_TWO_MIB, 8, 2, 13_824), # the same answer from a larger ask+ (_KEEPS_TWO_MIB, 4, 2, 14_080),+ (_KEEPS_TWO_MIB, 8, 2, 14_080), # the same answer from a larger ask],)def test_auto_context_follows_the_reduction(self, tmp_path, weights_mib, asked, slots, ctx):@@ -155,22 +181,73 @@ class TestTheLaunchedCountOwnsTheContext:was published instead of the context the final slot count actually affords."""got = _plan(tmp_path, weights_mib = _KEEPS_TWO_MIB, n_parallel = 4, spec = "off")- assert got["ceiling"] == 13_824+ assert got["ceiling"] == 14_080assert got["ceiling"] != llama_cpp._FIT_MIN_CTXdef test_a_layer_split_across_two_cards_re_fits_the_same_way(self, tmp_path):"""The same invariant holds for a two-card layer split."""reduced = _plan(- tmp_path, weights_mib = 14_000, n_parallel = 4, spec = "off", vram_mib = 8_704, gpus = 2+ tmp_path, weights_mib = 13_800, n_parallel = 4, spec = "off", vram_mib = 8_704, gpus = 2)direct = _plan(- tmp_path, weights_mib = 14_000, n_parallel = 2, spec = "off", vram_mib = 8_704, gpus = 2+ tmp_path, weights_mib = 13_800, n_parallel = 2, spec = "off", vram_mib = 8_704, gpus = 2)assert (reduced["slots"], reduced["fit"]) == (2, "off")- assert reduced["ctx"] == reduced["ceiling"] == 8_448+ assert reduced["ctx"] == reduced["ceiling"] == 10_752assert (reduced["ctx"], reduced["ceiling"]) == (direct["ctx"], direct["ceiling"])+class TestTheRealEstimatorStillReduces:+ """Real pricing reduces slots for hybrid state costs and keeps slots on dense models."""++ @pytest.mark.parametrize("weights_mib,slots,ctx", [(10_400, 3, 9_472), (10_500, 2, 10_240)])+ def test_a_hybrid_load_reduces_and_re_fits(self, tmp_path, weights_mib, slots, ctx):+ got = _plan(+ tmp_path, weights_mib = weights_mib, n_parallel = 4, spec = "off", slot_scaled_compute = False+ )+ assert (got["slots"], got["fit"], got["ctx"]) == (slots, "off", ctx)+ direct = _plan(+ tmp_path,+ weights_mib = weights_mib,+ n_parallel = slots,+ spec = "off",+ slot_scaled_compute = False,+ )+ assert (direct["slots"], direct["ctx"]) == (slots, ctx)++ def test_a_sliding_window_split_reprices_its_masks_per_candidate(self, tmp_path):+ """A layer split's sliding-window masks scale with slots, so each candidate+ prices its own count rather than the one asked for."""+ plans = [+ _plan(+ tmp_path,+ weights_mib = 21_200,+ n_parallel = asked,+ spec = "off",+ metadata = SLIDING_WINDOW,+ gpus = 2,+ slot_scaled_compute = False,+ )+ for asked in (16, 1)+ ]+ assert [(p["slots"], p["fit"], p["ctx"]) for p in plans] == [(1, "off", 8_448)] * 2++ def test_a_dense_load_keeps_its_slots(self, tmp_path):+ plans = [+ _plan(+ tmp_path,+ weights_mib = 8_200,+ n_parallel = asked,+ spec = "off",+ metadata = DENSE,+ slot_scaled_compute = False,+ )+ for asked in (1, 4)+ ]+ assert [(p["slots"], p["fit"]) for p in plans] == [(1, "off"), (4, "off")]+ assert plans[0]["ctx"] == plans[1]["ctx"]++class TestTheRefitStaysOnTheCardsTheReductionChose:"""A 12 GiB primary next to a 1.25 GiB card: the reduced plan fits on the primaryalone, and only the small card can hold a longer context."""@@ -180,7 +257,7 @@ class TestTheRefitStaysOnTheCardsTheReductionChose:tmp_path, weights_mib = _KEEPS_TWO_MIB, n_parallel = 4, spec = "off", vram_mib = MIXED_CARDS)assert (got["slots"], got["fit"], got["devices"]) == (2, "off", "0")- assert got["ctx"] == 13_824+ assert got["ctx"] == 14_080def test_it_matches_a_request_started_at_the_final_count(self, tmp_path):reduced = _plan(@@ -198,8 +275,8 @@ class TestTheRefitStaysOnTheCardsTheReductionChose:tmp_path, weights_mib = _KEEPS_TWO_MIB, n_parallel = 4, spec = "off", vram_mib = MIXED_CARDS)alone = _plan(tmp_path, weights_mib = _KEEPS_TWO_MIB, n_parallel = 4, spec = "off")- assert mixed["ctx"] == alone["ctx"] == 13_824- assert (mixed["ceiling"], alone["ceiling"]) == (14_848, 13_824)+ assert mixed["ctx"] == alone["ctx"] == 14_080+ assert (mixed["ceiling"], alone["ceiling"]) == (14_592, 14_080)class TestAutoSpeculationStillDecidesBeforeTheReduction:@@ -207,7 +284,7 @@ class TestAutoSpeculationStillDecidesBeforeTheReduction:def test_a_direct_one_slot_request_drops_the_drafter_and_keeps_context(self, tmp_path):got = _plan(tmp_path, weights_mib = 10_200, n_parallel = 1, spec = "auto")- assert (got["slots"], got["ctx"], got["spec"]) == (1, 18_688, "ngram-mod")+ assert (got["slots"], got["ctx"], got["spec"]) == (1, 18_944, "ngram-mod")@pytest.mark.parametrize("asked", [4, 8])def test_a_reduced_request_still_carries_the_drafter_it_admitted(self, tmp_path, asked):@@ -230,7 +307,7 @@ class TestWhatMustNotMove:got = _plan(tmp_path, weights_mib = 8_400, n_parallel = 4, spec = "off", n_ctx = 32768)assert (got["ctx"], got["slots"], got["fit"]) == (32768, 2, "off")# The measured ceiling still follows the final slot count.- assert got["ceiling"] == 35_840+ assert got["ceiling"] == 36_096def test_an_explicit_context_that_forces_offload_is_unchanged(self, tmp_path):"""An explicit context that requires offload is unchanged."""@@ -246,7 +323,7 @@ class TestWhatMustNotMove:def test_a_count_that_needs_no_reduction_is_untouched(self, tmp_path):"""A plan that needs no slot reduction is unchanged."""got = _plan(tmp_path, weights_mib = 6_000, n_parallel = 4, spec = "off")- assert (got["slots"], got["fit"], got["ctx"]) == (4, "off", 51_200)+ assert (got["slots"], got["fit"], got["ctx"]) == (4, "off", 51_456)def test_weights_that_fit_nowhere_still_offload(self, tmp_path):"""Oversized weights still fall back to offload, at the Auto offload context.@@ -328,7 +405,7 @@ class TestTheReductionIsPricedAtTheFitFloor:"""monkeypatch.setattr(llama_cpp, "_AUTO_OFFLOAD_CTX", offload_ctx)got = _plan(tmp_path, weights_mib = _KEEPS_TWO_MIB, n_parallel = 4, spec = "off")- assert (got["slots"], got["fit"], got["ctx"], got["devices"]) == (2, "off", 13_824, "0")+ assert (got["slots"], got["fit"], got["ctx"], got["devices"]) == (2, "off", 14_080, "0")@pytest.mark.parametrize("weights_mib,metadata", RESCUED)@pytest.mark.parametrize("offload_ctx", _OFFLOAD_SWEEP[:2])diff --git a/studio/backend/tests/test_slot_refit_platform_matrix.py b/studio/backend/tests/test_slot_refit_platform_matrix.pyindex fd8fb167127..5012070a977 100644--- a/studio/backend/tests/test_slot_refit_platform_matrix.py+++ b/studio/backend/tests/test_slot_refit_platform_matrix.py@@ -29,7 +29,11 @@ import pytest # noqa: E402import core.inference.llama_cpp as llama_mod # noqa: E402from core.inference.llama_cpp import LlamaCppBackend # noqa: E402-from test_llama_cpp_placement import _backend, _launch # noqa: E402+from test_llama_cpp_placement import ( # noqa: E402+ _backend,+ _install_slot_scaled_compute,+ _launch,+)MIB = 1024 * 1024NATIVE_CTX = 262144@@ -150,6 +154,8 @@ def _plan(backend._read_gguf_metadata = readbackend._get_gguf_size_bytes = lambda _path: weights_mib * MIBdel backend._can_estimate_kv # the real one, now that the dims are set+ # The per-slot cost the re-fit trades against (see the helper).+ _install_slot_scaled_compute(backend)backend.probe_server_capabilities = lambda _binary = None: {"mtp_token": "draft-mtp","supports_ngram_mod": True,