Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

Studio: make the GGUF memory estimate match the compute buffers llama.cpp allocates (#11043)

oobabooga820d377f13f7
diff --git a/studio/backend/core/inference/llama_cpp.py b/studio/backend/core/inference/llama_cpp.py
index 6d32fb898e6..cfe11c45455 100644
--- a/studio/backend/core/inference/llama_cpp.py
+++ b/studio/backend/core/inference/llama_cpp.py
@@ -3896,6 +3896,14 @@ def _kv_bytes_per_elem(cache_type: Optional[str]) -> float:
}.get((cache_type or "f16").strip().lower(), 2.0)
+# Upper bound on any current tokenizer, for output rows when a truncated header drops
+# the token array. Above Llama 4 / Gemma 3 (256k), the widest shipping.
+_ASSUMED_MAX_VOCAB = 262144
+# Ceiling on the float32 activations a compute graph keeps per micro-batch token, for a
+# header that cannot be read: 4 * n_ff at n_ff = 65536, above Llama 3.1 405B's 53056.
+_ASSUMED_MAX_ACTIVATION_WIDTH = 262144
+
+
def _pad_kv_cells(cells: int) -> int:
return ((cells + 255) // 256) * 256
@@ -4689,6 +4697,24 @@ def _flash_attn_enabled_from_args(
return enabled
+def _planned_flash_attn(
+ extra_args: Optional[Iterable[str]],
+ supports_flash_attn: bool,
+ v_cache_type: Optional[str] = None,
+ architecture: Optional[str] = None,
+) -> bool:
+ """Flash attention as the launch command will resolve it.
+
+ The managed --flash-attn on follows the environment and precedes user extras, so
+ only extras can turn it off. A build without the flag gets neither the flag nor
+ the inherited LLAMA_ARG_FLASH_ATTN. A quantized V cache forces it on, except on
+ Grok, which llama.cpp always runs without it."""
+ if not supports_flash_attn or architecture == "grok":
+ return False
+ enabled = _flash_attn_enabled_from_args(extra_args, default = True, env = {})
+ return enabled or (v_cache_type or "f16").strip().lower() not in {"f16", "bf16", "f32"}
+
+
def _effective_spec_type(
extra_args: Optional[Iterable[str]], env: Optional[Mapping[str, str]] = None
) -> Optional[str]:
@@ -7032,8 +7058,17 @@ class LlamaCppBackend:
# llama_pooling_type from the GGUF; present and non-zero marks an embedding model.
self._pooling_type: Optional[int] = None
# For the compute-graph buffer estimate; vocab from the tokens array len.
+ # feed_forward_length is the widest layer when the GGUF stores one per layer.
self._feed_forward_length: Optional[int] = None
+ self._expert_used_count: Optional[int] = None
+ self._expert_feed_forward_length: Optional[int] = None
+ self._expert_shared_feed_forward_length: Optional[int] = None
+ self._expert_shared_count: Optional[int] = None
+ # Gemma 3n / Gemma 4 per-layer input embedding width.
+ self._embedding_length_per_layer_input: Optional[int] = None
self._vocab_size: Optional[int] = None
+ # Set from reserves_micro_batch_outputs() by the callers that know the binary.
+ self._reserves_micro_batch_outputs: bool = False
# Architecture-aware KV fields for 5-path estimation
self._kv_key_length: Optional[int] = None
self._kv_value_length: Optional[int] = None
@@ -8515,6 +8550,8 @@ class LlamaCppBackend:
_CAPABILITY_PROBE_RETRY_SECONDS = 30.0
_CAPABILITY_PROBE_RETRY_MAX_SECONDS = 600.0
_capability_cache: dict[tuple[str, int, int], dict[str, object]] = {}
+ # (bin_path, mtime_ns, size) -> build number from `llama-server --version`, or None.
+ _build_number_cache: dict[tuple[str, int, int], Optional[int]] = {}
_capability_retry_after: dict[tuple[str, int, int], float] = {}
_capability_retry_backoff: dict[tuple[str, int, int], float] = {}
_capability_cache_lock = threading.Lock()
@@ -8540,6 +8577,68 @@ class LlamaCppBackend:
return True
return bool(cls._FLASH_ATTN_ENUM_RE.search(help_text))
+ # ggml-org/llama.cpp#23861 capped the output rows a context reserves at the rows it
+ # can emit. Builds before it reserve a row for every micro-batch token.
+ _OUTPUT_ROWS_CAPPED_BUILD = 9460
+
+ @classmethod
+ def probe_build_number(cls, binary: Optional[str] = None) -> Optional[int]:
+ """The llama.cpp build number from ``llama-server --version``, cached per binary.
+
+ Current builds print ``version: X.Y.Z (build NNNN, commit ...)``, older ones
+ ``version: NNNN (commit)``. None when the binary is missing, the probe fails,
+ or the build reports 1 (a source build without tags)."""
+ bin_path = cls._exec_path_for_launch(binary or cls._find_llama_server_binary())
+ if not bin_path or not Path(bin_path).is_file():
+ return None
+ try:
+ binary_stat = Path(bin_path).stat()
+ cache_key = (bin_path, binary_stat.st_mtime_ns, binary_stat.st_size)
+ except OSError:
+ cache_key = (bin_path, 0, 0)
+ with cls._capability_cache_lock:
+ if cache_key in cls._build_number_cache:
+ return cls._build_number_cache[cache_key]
+ build: Optional[int] = None
+ try:
+ env = cls._llama_server_env_for_binary(bin_path)
+ if sys.platform == "darwin":
+ env["GGML_METAL_DEVICES"] = "0" # see probe_server_capabilities
+ result = subprocess.run(
+ [bin_path, "--version"],
+ capture_output = True,
+ text = True,
+ encoding = "utf-8",
+ errors = "replace",
+ timeout = 10,
+ check = False,
+ env = env,
+ **_windows_hidden_subprocess_kwargs(),
+ )
+ output = (result.stdout or "") + "\n" + (result.stderr or "")
+ match = re.search(r"version:[^\r\n]*\bbuild\s+(\d+)\b", output) or re.search(
+ r"version:\s*(\d+)\b(?!\.)", output
+ )
+ if match and int(match.group(1)) > 1:
+ build = int(match.group(1))
+ except Exception as exc:
+ logger.debug("llama-server --version probe failed: %s", exc)
+ with cls._capability_cache_lock:
+ cls._build_number_cache[cache_key] = build
+ return build
+
+ @classmethod
+ def reserves_micro_batch_outputs(cls, binary: Optional[str] = None) -> bool:
+ """Whether this llama-server reserves an output row per micro-batch token.
+
+ Known builds before b9460, including the older-macOS b9415 fallback, do.
+ Unknown builds are priced as current, matching Studio's prebuilts."""
+ try:
+ build = cls.probe_build_number(binary)
+ except Exception:
+ return False
+ return build is not None and build < cls._OUTPUT_ROWS_CAPPED_BUILD
+
_ADVERTISED_DEFAULT_RE = re.compile(r"\(default:\s*(-?\d+)")
@classmethod
@@ -14821,16 +14920,10 @@ class LlamaCppBackend:
reserve_bytes_fn: Callable[[int], int],
min_ctx: int = 8192,
) -> int:
- """Largest context in [``min_ctx``, ``ctx``] whose replicated per-device reserve
- still fits the SMALLEST card of a layer-split subset; 0 when even ``min_ctx``
- does not. The reserve shrinks with the context, so a subset
- ``_every_gpu_holds_reserve`` turns down at the pooled context usually serves a
- lower one: 40000 + 2500 MiB free holds 31488, not the 4096 a drop would leave.
- Bisects the reserve, which is monotone in context on every branch but
- proportional on only some: deepseek4's flat indexer term makes a closed form
- answer 43341 on a 4000 MiB card, whose reserve there is 6048 MiB. The MiB
- comparison mirrors ``_every_gpu_holds_reserve``, so that gate accepts what this
- returns."""
+ """Largest context in [min_ctx, ctx] whose reserve fits every selected GPU.
+
+ Bisect because reserves are monotone but may include fixed costs. Use the
+ same MiB comparison as _every_gpu_holds_reserve. Return 0 if min_ctx fails."""
values = list(usable_mib)
if not values or ctx < min_ctx:
return 0
@@ -15593,14 +15686,56 @@ class LlamaCppBackend:
target_recurrent_copies = 0
if target_rollback and spec_draft_n_max > 0:
target_recurrent_copies = self._rollback_state_bytes(n_parallel) * spec_draft_n_max
+ spec_compute = self._mtp_draft_compute_bytes(
+ n_ctx,
+ drafter_path = drafter_path,
+ n_ubatch = n_ubatch,
+ n_parallel = n_parallel,
+ spec_draft_n_max = spec_draft_n_max,
+ )
if draft_kv is None:
# KV unsized (exotic/remote drafter): still reserve known weights + any
# MLA target copy so a large config can't launch over budget (the small
# unsized draft KV rides in the cushion). Nothing known -> None, so the
# caller keeps the flat fallback.
total = weights + target_ctx_copy + target_recurrent_copies
- return total if total > 0 else None
- return draft_kv + weights + target_ctx_copy + target_recurrent_copies
+ return total + spec_compute if total > 0 else None
+ return draft_kv + weights + target_ctx_copy + target_recurrent_copies + spec_compute
+
+ def _mtp_draft_compute_bytes(
+ self,
+ n_ctx: int,
+ *,
+ drafter_path: Optional[str] = None,
+ n_ubatch: Optional[int] = None,
+ n_parallel: int = 1,
+ spec_draft_n_max: int = 0,
+ ) -> int:
+ """Draft mask and activations beyond the separately charged floor, plus
+ target verification rows. Use the drafter's dimensions when available,
+ otherwise the target's dimensions for an embedded head."""
+ verify_rows = self._spec_verify_rows_bytes(n_parallel, spec_draft_n_max)
+ if n_ctx <= 0:
+ return verify_rows
+ ub = max(1, int(self._DEFAULT_N_UBATCH if n_ubatch is None else n_ubatch))
+ width = self._compute_activation_width()
+ if drafter_path:
+ db = self._draft_backend_for(drafter_path)
+ draft_width = getattr(db, "_compute_activation_width", lambda: 0)()
+ if draft_width > 0:
+ width = draft_width + 2 * (getattr(self, "_embedding_length", None) or 0)
+ activations = int(width * ub * 4 * self._COMPUTE_BUFFER_SAFETY)
+ return verify_rows + max(0, ub * 2 * n_ctx + activations - self._MTP_DRAFT_COMPUTE_BYTES)
+
+ def _spec_verify_rows_bytes(self, n_parallel: int, spec_draft_n_max: int) -> int:
+ """Target output rows for verified draft tokens, wherever the drafter runs."""
+ return (
+ max(1, n_parallel)
+ * max(0, spec_draft_n_max)
+ * self._SPEC_VERIFY_ROW_COPIES
+ * (getattr(self, "_vocab_size", None) or _ASSUMED_MAX_VOCAB)
+ * 4
+ )
def _mtp_reserve_note(
self,
@@ -15667,7 +15802,7 @@ class LlamaCppBackend:
)
_DEFAULT_N_UBATCH = _DEFAULT_LLAMA_N_UBATCH
- _COMPUTE_BUFFER_SAFETY = 1.15 # upper-bound margin on the compute-buffer estimate
+ _COMPUTE_BUFFER_SAFETY = 1.10 # margin on the modelled activation width
# Soft VRAM the modeled terms omit; charged to the fit budget on tight tiers (#6682).
_CUDA_CONTEXT_RESERVE_BYTES = 320 * 1024 * 1024 # CUDA ctx + cuBLAS workspace (~330 MiB)
_MMPROJ_VRAM_SAFETY = 1.4 # mmproj worst-case buffer vs file size (runtime ~1.3x)
@@ -15683,30 +15818,10 @@ class LlamaCppBackend:
# load while over-reserving only costs context. Zero cost on any other model,
# where the recurrent state is nil.
_UNKNOWN_SPEC_DRAFT_N_MAX = 16
- # The flash-attn KQ mask + attention scratch grow ~linearly with context; the flat
- # _estimate_compute_buffer_bytes term only covers ctx -> 0. The per-token rate
- # depends on the KV cache type: a QUANTIZED cache (q8_0/q5/q4/iq4) needs a
- # context-sized dequant scratch that scales with n_embd, measured at 0.74-2.02 x
- # n_embd across Qwen3.5/3.6 (2B/4B/9B/27B) and Gemma-4 (12B/31B) at q8_0; an
- # f16/bf16/f32 cache skips the dequant and pays only the KQ mask, a flat n_ubatch*2
- # bytes per context token regardless of n_embd (measured 1024 B/tok on Qwen-9B and
- # Gemma-31B alike). So Qwen3.5-4B at 256k is 1.30 GiB at q8_0 vs 0.31 GiB at f16.
- # 2.25 covers the worst quantized case (Qwen3.5-4B, ~2.0x) plus the under-modeled
- # flat base; the mask safety covers the f16 base gap. Without this term, tight tiers
- # at extreme context over-pin and spill to CPU (the 3% cushion is only ~0.25 GiB on
- # an 8 GB card, far below the ~1-2.4 GiB quantized buffer at 256k): e.g. Qwen3.5-4B
- # Q4 at 256k needs ~8.5 GiB on a real 8 GB card (weights 2.4 + KV 4.3 + compute 1.3
- # + CUDA ctx) -> CPU spill; with this reserve the auto context caps to ~210k, fits.
- _CTX_COMPUTE_BYTES_PER_EMBD = 2.25 # quantized KV, regular attention (dequant scratch)
- _CTX_COMPUTE_BYTES_PER_EMBD_MLA = 1.25 # quantized KV, MLA (compressed attn: measured 0.94x)
- _CTX_COMPUTE_F16_MASK_SAFETY = 1.5 # f16/bf16/f32 KV: KQ mask only (n_ubatch*2 B/tok)
- # Per-device f16 ctx-linear rate steps 4x once the model spans >1 device under
- # `-sm layer`: pipeline parallelism sets the ggml scheduler's n_copies to
- # GGML_SCHED_MAX_COPIES == 4, and the [n_kv, n_ubatch] f16 KQ mask is a
- # GGML_TENSOR_FLAG_INPUT, so 1 live copy becomes 4. Hence a step on "is split", not
- # a ramp in device count: measured 2.00 B/tok/ubatch on 1 GPU and 8.00 on 2-8 GPUs,
- # architecture- and --parallel-independent. Charging 1x on a split under-reserves
- # 2.67x (= 8/(2*1.5)). llama.cpp declines it in _pipeline_parallel_disabled_by_args.
+ # Vocabulary-wide float32 copies per verified draft token (logits and sampling).
+ _SPEC_VERIFY_ROW_COPIES = 3
+ # Pipeline parallelism uses GGML_SCHED_MAX_COPIES input copies per device.
+ # This applies only when _pipeline_parallel_disabled_by_args is false.
_CTX_COMPUTE_SPLIT_MULT = 4
# DeepSeek-V4 (deepseek4): its lightning indexer + sparse attention reserve a large
# context-scaling compute buffer the rates above miss (present even with an f16
@@ -15728,39 +15843,132 @@ class LlamaCppBackend:
# path): measured 402.5 GiB reserve at 1M ctx pre-fix, ~402 KiB per token.
_INKLING_CTX_COMPUTE_DENSE_BYTES_PER_TOK = 402470 # per token at ub=512
+ def _compute_activation_width(self) -> int:
+ """Float32 activations the compute graph keeps live per micro-batch token.
+
+ Take the widest dense, expert or recurrent block, then add per-layer
+ embeddings. The 12 * n_embd floor covers attention and residual buffers.
+ Return 0 without embedding dimensions.
+ """
+
+ # getattr: a backend built with __new__ carries only the dims it was given.
+ def dim(name: str) -> int:
+ return getattr(self, name, None) or 0
+
+ n_embd = dim("_embedding_length")
+ if n_embd <= 0:
+ return 0
+
+ widths = [12 * n_embd]
+ if dim("_feed_forward_length") > 0:
+ widths.append(4 * dim("_feed_forward_length"))
+ n_used = dim("_expert_used_count")
+ # Granite 4.0 H writes its expert width as feed_forward_length.
+ n_ff_exp = dim("_expert_feed_forward_length") or (
+ dim("_feed_forward_length") if dim("_n_experts") else 0
+ )
+ if n_used > 0 and n_ff_exp > 0:
+ shared = dim("_expert_shared_feed_forward_length") or (
+ n_ff_exp * dim("_expert_shared_count")
+ )
+ widths.append(n_used * (2 * n_embd + 3 * n_ff_exp) + 3 * shared)
+ inner = dim("_ssm_inner_size")
+ if inner > 0:
+ conv_in = inner + 2 * (dim("_ssm_group_count") or 1) * dim("_ssm_state_size")
+ widths.append(4 * conv_in)
+ per_layer_input = dim("_embedding_length_per_layer_input") * dim("_n_layers")
+ return max(widths) + per_layer_input
+
def _estimate_compute_buffer_bytes(
self,
*,
n_ubatch: Optional[int] = None,
n_parallel: int = 1,
per_device_tensor: bool = False,
+ vocab_ceiling: Optional[int] = None,
) -> int:
- """Per-device compute-graph buffer (bytes) from GGUF dims: a vocab-width
- output buffer + activation scratch. Context-independent; scales with
- ``--parallel`` (serving slots). Tensor mode materializes it on every device.
- A slight upper bound over measured allocations; 0 when dims are missing."""
- n_vocab = self._vocab_size or 0
- n_embd = self._embedding_length or 0
- if n_vocab <= 0 or n_embd <= 0:
+ """Per-device compute-graph buffer (bytes) that does not grow with context.
+
+ Includes activations, output rows, sliding-window masks and recurrent state.
+ Since b9460 (#23861), output rows are capped at min(ubatch, slots); older
+ builds reserve ubatch rows. Extra verification rows belong to the MTP reserve.
+
+ Tensor splits use the same buffer per device, so per_device_tensor does not
+ change the result. Return 0 for missing dimensions; vocab_ceiling supplies
+ a fallback vocabulary size."""
+ width = self._compute_activation_width()
+ n_vocab = self._vocab_size or vocab_ceiling or 0
+ if width <= 0 or n_vocab <= 0:
return 0
ub = max(
1,
int(self._DEFAULT_N_UBATCH if n_ubatch is None else n_ubatch),
)
par = max(1, int(n_parallel))
- out_buffer = n_vocab * ub * 4 # f32 output/logits buffer
- act_scratch = 4 * n_embd * ub * 4 # a few resident hidden-width buffers
- if per_device_tensor:
- # Output + comm/staging materialized on every device, every slot.
- compute = 2 * act_scratch + out_buffer * par
+ activations = width * ub * 4
+ if self.is_embedding_gguf:
+ outputs = (self._embedding_length or 0) * ub * 4
+ elif getattr(self, "_reserves_micro_batch_outputs", False):
+ # Before ggml-org/llama.cpp#23861: a row for every micro-batch token, once.
+ # Measured on b9415 Vulkan: Gemma 4 E2B 515 MiB at ubatch 512, 137 on b10909.
+ outputs = n_vocab * ub * 4
else:
- # Each extra concurrent slot adds one output buffer (chat decode sizes
- # ~one logit row per slot). Embedding mode outputs the whole first
- # micro-batch too, so it pays for every slot rather than slots past one.
- # Matches measured chat {1:36,2:492,4:1388,8:3220} MiB.
- output_slots = par if self.is_embedding_gguf else max(0, par - 1)
- compute = act_scratch + out_buffer * output_slots
- return int(compute * self._COMPUTE_BUFFER_SAFETY)
+ outputs = n_vocab * min(ub, par) * 4
+ swa_mask = 0
+ sliding_window = self._effective_sliding_window()
+ if sliding_window > 0:
+ swa_mask = _pad_kv_cells(sliding_window * par + ub) * ub * 2
+ # A recurrent block works on every slot's state at once: Qwen3.5 4B's buffer
+ # grows 19.5 MiB from one slot to eight at the same micro-batch.
+ inner = getattr(self, "_ssm_inner_size", None) or 0
+ recurrent = 0
+ if inner > 0:
+ state = getattr(self, "_ssm_state_size", None) or 0
+ conv_in = inner + 2 * (getattr(self, "_ssm_group_count", None) or 1) * state
+ recurrent = par * (inner * state + 3 * conv_in) * 4
+ return int((activations + outputs + recurrent) * self._COMPUTE_BUFFER_SAFETY + swa_mask)
+
+ def _effective_sliding_window(self) -> int:
+ """The sliding window when some layer really slides, else 0. Phi-4-mini
+ declares a 262144 window over a 131072 context and no pattern, which bounds
+ nothing."""
+ window = getattr(self, "_sliding_window", None) or 0
+ if window <= 0:
+ return 0
+ pattern = getattr(self, "_sliding_window_pattern", None)
+ if pattern is not None:
+ return window if any(pattern) else 0
+ context = getattr(self, "_context_length", None) or 0
+ return window if not context or window < context else 0
+
+ def _attention_dequant_width(self) -> int:
+ """K+V elements per cell copied to f16 from a quantized cache.
+
+ Use the widest global layer: sliding-window layers do not scale with
+ context. MLA uses compressed K only. Fall back to n_embd without head dims."""
+ n_heads = getattr(self, "_n_heads", None)
+ n_kv = getattr(self, "_n_kv_heads", None) or n_heads
+ kv_lora_rank = getattr(self, "_kv_lora_rank", None)
+ key_len = getattr(self, "_kv_key_length", None)
+ val_len = getattr(self, "_kv_value_length", None)
+ if kv_lora_rank is not None:
+ rope_dim = getattr(self, "_key_length_mla", None) or 64
+ return (getattr(self, "_n_kv_heads", None) or 1) * (key_len or kv_lora_rank + rope_dim)
+ if key_len is None or val_len is None:
+ if not (n_heads and n_kv):
+ return getattr(self, "_embedding_length", None) or 0
+ return 2 * n_kv * self._legacy_head_dim()
+ by_layer = getattr(self, "_n_kv_heads_by_layer", None)
+ pattern = (
+ getattr(self, "_sliding_window_pattern", None)
+ if getattr(self, "_sliding_window", None)
+ else None
+ )
+ widest = 0
+ for i, heads in enumerate(by_layer or ()):
+ if not (pattern is not None and i < len(pattern) and pattern[i]):
+ widest = max(widest, heads)
+ return (widest or n_kv or 1) * (key_len + val_len)
def _compute_buffer_ctx_bytes(
self,
@@ -15769,21 +15977,18 @@ class LlamaCppBackend:
cache_type_kv: Optional[str] = None,
*,
layer_split: bool = False,
+ flash_attn: bool = True,
+ n_parallel: int = 1,
) -> int:
- """Context-linear growth of the per-device compute buffer (bytes), charged
- on top of the flat ``_estimate_compute_buffer_bytes``. The flash-attn KQ
- mask + attention scratch scale ~linearly with context and with the micro-
- batch; the flat term only covers ctx -> 0. A quantized KV cache adds a
- context-sized dequant scratch that scales with n_embd; f16/bf16/f32 pays only
- the KQ mask, a flat n_ubatch*2 bytes per context token. ``cache_type_kv`` None
- -> f16 (llama.cpp's default; env-set types are adopted by the caller via
- ``_env_main_cache_type_for_budget``). Returns 0 when dims are missing or
- ``n_ctx`` <= 0.
-
- ``layer_split`` adds the extra KQ-mask copies a multi-device split pays (see
- ``_CTX_COMPUTE_SPLIT_MULT``) on every architecture except deepseek4, whose own
- rate already subsumes them. Tensor mode passes False, having been measured
- separately at the single-device rate (see ``_plan_tensor_parallel``)."""
+ """Per-device context-dependent scratch, added to the flat compute buffer.
+
+ Includes the KQ mask and quantized-cache dequantization. Without flash
+ attention, also charge float32 scores and a float32 mask. Layer splits add
+ scheduler copies, with sliding-window masks sized by n_parallel; tensor
+ splits pass layer_split=False. DeepSeek and Inkling use separate rates.
+
+ None cache type means f16; callers resolve environment overrides.
+ Return 0 without n_embd or a positive context."""
n_embd = self._embedding_length or 0
if n_embd <= 0 or n_ctx <= 0:
return 0
@@ -15791,10 +15996,11 @@ class LlamaCppBackend:
1,
int(self._DEFAULT_N_UBATCH if n_ubatch is None else n_ubatch),
)
- # One KQ mask copy ([n_kv, n_ubatch] f16), n_embd-independent. Every rate below
- # is a single-GPU total containing exactly one, so a split adds the extra copies.
- mask_rate = ub * 2 * self._CTX_COMPUTE_F16_MASK_SAFETY
- split_extra = (self._CTX_COMPUTE_SPLIT_MULT - 1) * mask_rate if layer_split else 0.0
+ # One KQ mask copy ([n_kv, n_ubatch]), n_embd-independent: f16 under flash
+ # attention, f32 without it (build_attn_inp_kq_mask). Every rate below is a
+ # single-GPU total containing exactly one, so a split adds the extra copies.
+ mask_rate = ub * (2 if flash_attn else 4)
+ split_extra = (self._CTX_COMPUTE_SPLIT_MULT - 1) * mask_rate if layer_split else 0
if getattr(self, "_architecture", None) == "deepseek4":
# DSV4 indexer/CSA buffer (see constants): flat + linear, ub-scaled. Fires
# for any KV type -- the indexer scratch is present even with an f16 cache.
@@ -15820,23 +16026,27 @@ class LlamaCppBackend:
# Both paths allocate the mask, and 8192 has only ~1.5x headroom over the
# 5.6 KiB/tok banded measurement -- a split alone would overrun it.
return int(rate * n_ctx * ub_scale + split_extra * n_ctx)
+ per_tok = mask_rate
if _kv_bytes_per_elem(cache_type_kv) < 2.0:
- # Quantized cache: the dequant scratch dominates and scales with n_embd.
- # MLA (compressed KV) needs far less of it: measured 0.94 x n_embd on
- # GLM-5.2 and Kimi-K2.7 vs up to 2.02x on regular attention.
- ub_scale = ub / self._DEFAULT_N_UBATCH
- rate = (
- self._CTX_COMPUTE_BYTES_PER_EMBD_MLA
- if self._key_length_mla
- else self._CTX_COMPUTE_BYTES_PER_EMBD
- )
- per_tok = rate * n_embd * ub_scale
- else:
- # f16/bf16/f32: only the KQ mask.
- per_tok = mask_rate
- # Only the mask goes 1x -> 4x, not the dequant scratch, so ADD the extra copies:
- # max() would under-reserve the quantized path by the whole dequant scratch.
- return int((per_tok + split_extra) * n_ctx)
+ # Quantized cache: the f16 K/V copy, independent of the micro-batch.
+ per_tok += self._attention_dequant_width() * 2
+ if not flash_attn:
+ n_head = getattr(self, "_n_heads", None) or max(1, n_embd // 64)
+ per_tok += ub * n_head * 4
+ if getattr(self, "_kv_lora_rank", None) is not None:
+ # MLA decompresses a float32 copy of the latent cache to attend over it:
+ # GLM-4.7-Flash grows 44 KiB per cell at ubatch 512, 2 KiB past the scores.
+ per_tok += self._attention_dequant_width() * 4
+ # Scheduler copies replicate masks and hidden states, but not dequant scratch.
+ # Sliding-window copies use the same per-slot cell count as the flat buffer.
+ split_fixed = 0
+ if layer_split:
+ split_fixed = self._CTX_COMPUTE_SPLIT_MULT * n_embd * ub * 4
+ sliding_window = self._effective_sliding_window()
+ if sliding_window > 0:
+ swa_cells = _pad_kv_cells(sliding_window * max(1, int(n_parallel)) + ub)
+ split_fixed += (self._CTX_COMPUTE_SPLIT_MULT - 1) * swa_cells * mask_rate
+ return int((per_tok + split_extra) * n_ctx + split_fixed)
def _slots_that_fit_on_gpu(
self,
@@ -15854,6 +16064,7 @@ class LlamaCppBackend:
kv_unified: bool = True,
flash_attn: bool = True,
split_extra_bytes: int = 0,
+ split_extra_for_slots: Optional[Callable[[int], int]] = None,
ubatch_for_slots: Optional[Callable[[int], Optional[int]]] = None,
mtp_bytes_for_slots: Optional[Callable[[int, Optional[int]], int]] = None,
ctx_checkpoints: int = 0,
@@ -15886,7 +16097,8 @@ class LlamaCppBackend:
layers charges per slot. It takes the candidate micro-batch as well as the slot
count, since a reduced candidate lowers the batch floor and so the ubatch too;
pricing the reserve at the requested pair over-charged every candidate, so one that
- fits could be rejected and the load kept --fit."""
+ fits could be rejected and the load kept --fit. ``split_extra_for_slots(slots)``
+ re-prices the split step, whose sliding-window masks also scale with slots."""
for slots in range(n_parallel if include_requested else n_parallel - 1, 0, -1):
_ub = ubatch_for_slots(slots) if ubatch_for_slots else n_ubatch
cb = self._estimate_compute_buffer_bytes(
@@ -15916,7 +16128,9 @@ class LlamaCppBackend:
total_by_idx = total_by_idx,
per_device_overhead_bytes = per_device_overhead_bytes,
min_gpus = min_gpus,
- split_extra_bytes = split_extra_bytes,
+ split_extra_bytes = (
+ split_extra_for_slots(slots) if split_extra_for_slots else split_extra_bytes
+ ),
)
if not use_fit:
return gpu_indices, False, slots
@@ -16580,6 +16794,11 @@ class LlamaCppBackend:
self._pooling_type = None
self._gguf_path = gguf_path
self._feed_forward_length = None
+ self._expert_used_count = None
+ self._expert_feed_forward_length = None
+ self._expert_shared_feed_forward_length = None
+ self._expert_shared_count = None
+ self._embedding_length_per_layer_input = None
self._vocab_size = None
self._kv_key_length = None
self._kv_value_length = None
@@ -16700,6 +16919,11 @@ class LlamaCppBackend:
f"{arch}.embedding_length": "embedding_length",
f"{arch}.pooling_type": "pooling_type",
f"{arch}.feed_forward_length": "feed_forward_length",
+ f"{arch}.expert_used_count": "expert_used_count",
+ f"{arch}.expert_feed_forward_length": "expert_feed_forward_length",
+ f"{arch}.expert_shared_feed_forward_length": "expert_shared_feed_forward_length",
+ f"{arch}.expert_shared_count": "expert_shared_count",
+ f"{arch}.embedding_length_per_layer_input": "embedding_length_per_layer_input",
f"{arch}.attention.key_length": "kv_key_length",
f"{arch}.attention.value_length": "kv_value_length",
f"{arch}.attention.sliding_window": "sliding_window",
@@ -16762,6 +16986,8 @@ class LlamaCppBackend:
elif attr == "sliding_window_pattern" and val_a is not None:
self._sliding_window_pattern = [bool(x) for x in val_a]
sliding_window_pattern_period = None
+ elif attr == "feed_forward_length" and val_a:
+ self._feed_forward_length = max(int(x) for x in val_a)
else:
self._gguf_skip_value(f, vtype)
else:
@@ -22342,6 +22568,7 @@ class LlamaCppBackend:
raise LlamaServerNotFoundError(LLAMA_SERVER_NOT_FOUND_DETAIL)
server_caps = _launch_caps(binary)
+ self._reserves_micro_batch_outputs = self.reserves_micro_batch_outputs(binary)
# Outside ``self._lock`` so /unload, /cancel, /status aren't
# blocked. ``unload_model`` also records the kill, so the
@@ -22474,6 +22701,13 @@ class LlamaCppBackend:
None if _cache_type_from_env else cache_type_kv,
extra_args,
)
+ # Price compute for the launch; KV separately covers the no-flash retry.
+ _launch_flash_attn = _planned_flash_attn(
+ extra_args,
+ bool(server_caps.get("supports_flash_attn", True)),
+ _planned_cache_pair[1],
+ self._architecture,
+ )
# A user --split-mode in extras last-wins-overrides the toggle, and
# an inherited tensor LLAMA_ARG_SPLIT_MODE flips it on (the child
# would run tensor unbudgeted otherwise). The duplicate-load matchers
@@ -23262,7 +23496,11 @@ class LlamaCppBackend:
flash_attn = planned_flash_attn,
)
- def _cc_bytes(ctx: int, n_gpus: int = 1) -> int:
+ def _cc_bytes(
+ ctx: int,
+ n_gpus: int = 1,
+ slots: int = 0,
+ ) -> int:
# Context-linear compute-buffer growth (flash-attn KQ mask +
# attention scratch); the flat _compute_buffer_pipeline folded
# into model_size_fit only covers ctx -> 0. Charged per
@@ -23276,18 +23514,21 @@ class LlamaCppBackend:
# under-reserves ~(n-1)x it (e.g. Qwen3.5-397B on 3 GPUs). The
# per-device rate also steps up once split, unless llama.cpp
# declines the pipeline parallelism that causes the step.
+ # A nonzero ``slots`` prices that candidate count and its micro-batch.
return max(1, n_gpus) * self._compute_buffer_ctx_bytes(
ctx,
- _effective_ubatch,
+ _ubatch_for_slots(slots) if slots else _effective_ubatch,
_scratch_cache_type_kv,
layer_split = n_gpus > 1 and not _pipeline_parallel_off,
+ flash_attn = _launch_flash_attn,
+ n_parallel = slots or n_parallel,
)
- def _cc_split_extra(ctx: int) -> int:
+ def _cc_split_extra(ctx: int, slots: int = 0) -> int:
# Per-device step from the single-device rate to the split one,
# for the paths that must select GPUs before they know the
# count. 0 when llama.cpp declines pipeline parallelism.
- return max(0, _cc_bytes(ctx, 2) // 2 - _cc_bytes(ctx))
+ return max(0, _cc_bytes(ctx, 2, slots) // 2 - _cc_bytes(ctx, 1, slots))
# Layer-split compute buffer (one lump; tensor mode reserves it
# per device in _plan_tensor_parallel). Context-independent, so
@@ -23305,9 +23546,9 @@ class LlamaCppBackend:
)
# ...and the tensor-mode rate, reserved on EVERY selected device:
- # the f32 output buffer and the comm staging are replicated rather
- # than split, so the layer lump above is a fraction of what a tensor
- # split allocates. Same flat fallback when the dims are missing.
+ # each holds the whole buffer rather than a share of it, so the layer
+ # lump above is one device of what a tensor split allocates. Same flat
+ # fallback when the dims are missing.
_compute_buffer_tensor = self._estimate_compute_buffer_bytes(
n_ubatch = _effective_ubatch,
n_parallel = n_parallel,
@@ -23451,11 +23692,12 @@ class LlamaCppBackend:
# displaced does not run. The flat fraction below is gated on
# this; the byte-accurate callback was not, so the fit went on
# charging VRAM no drafter allocates.
- # One term survives: a recurrent target's rollback snapshots sit
- # in the TARGET context, so pinning the drafter to CPU does not
- # move them. Charge those alone. Flat in ctx (the
- # state is per-slot), hence the same _np/_n_ubatch keywords the
- # replaced callback takes, so _mtp_bytes can still re-price slots.
+ # Two terms survive: a recurrent target's rollback snapshots and
+ # the verification output rows sit in the TARGET context, so
+ # pinning the drafter to CPU does not move them. Charge those
+ # alone. Flat in ctx (both are per-slot), hence the same
+ # _np/_n_ubatch keywords the replaced callback takes, so
+ # _mtp_bytes can still re-price slots.
def _cpu_draft_target_state(
_ctx: int,
_np: int = n_parallel,
@@ -23463,9 +23705,10 @@ class LlamaCppBackend:
_n: int = _mtp_eff_n_max,
_rollback: bool = _target_rollback,
) -> int:
- if not _rollback or _n <= 0:
+ if _n <= 0:
return 0
- return self._rollback_state_bytes(_np) * _n
+ rollback = self._rollback_state_bytes(_np) * _n if _rollback else 0
+ return rollback + self._spec_verify_rows_bytes(_np, _n)
mtp_overhead_fn = (
_cpu_draft_target_state
@@ -24597,7 +24840,7 @@ class LlamaCppBackend:
swa_full = swa_full,
kv_unified = planned_kv_unified,
flash_attn = planned_flash_attn,
- split_extra_bytes = _cc_split_extra(_reduce_ctx),
+ split_extra_for_slots = lambda s: _cc_split_extra(_reduce_ctx, s),
ubatch_for_slots = _ubatch_for_slots,
mtp_bytes_for_slots = lambda s, ub: _mtp_bytes(_reduce_ctx, s, ub),
ctx_checkpoints = _fit_ctx_checkpoints,
diff --git a/studio/backend/routes/inference.py b/studio/backend/routes/inference.py
index ee169f95947..9ceac27c872 100644
--- a/studio/backend/routes/inference.py
+++ b/studio/backend/routes/inference.py
@@ -1613,12 +1613,12 @@ try:
_emitted_n_batch,
_extra_args_draft_device_pin,
_extra_args_n_ubatch,
- _flash_attn_enabled_from_args,
_hf_offline_if_unreachable,
_hf_offline_if_unreachable_for,
_kv_bytes_per_elem,
_kv_unified_from_args,
_metal_device_is_paravirtual,
+ _planned_flash_attn,
_planned_main_cache_types,
_planned_scratch_cache_type,
_swa_full_from_args_or_env,
@@ -1675,12 +1675,12 @@ except ImportError:
_emitted_n_batch,
_extra_args_draft_device_pin,
_extra_args_n_ubatch,
- _flash_attn_enabled_from_args,
_hf_offline_if_unreachable,
_hf_offline_if_unreachable_for,
_kv_bytes_per_elem,
_kv_unified_from_args,
_metal_device_is_paravirtual,
+ _planned_flash_attn,
_planned_main_cache_types,
_planned_scratch_cache_type,
_swa_full_from_args_or_env,
@@ -10494,10 +10494,6 @@ def _cached_repo_gguf_bytes(repo: str, hint: str = "") -> int:
return 0
-# Upper bound on any current tokenizer, used to rebuild the compute buffer when a
-# truncated header drops the token array. Above Llama 4 / Gemma 3 (256k), the widest shipping.
-_ASSUMED_MAX_VOCAB = 262144
-
# Canonical speculative modes whose launch opens no drafter file at all:
# _build_speculative_flags returns out of each before it reaches --model-draft.
_SPEC_MODES_WITHOUT_A_DRAFTER = frozenset({"off", "ngram", "ngram-simple"})
@@ -10641,7 +10637,7 @@ def _gguf_runtime_bytes(
over-reserves on purpose; a panel quoting a number to a user wants the other
one, since a smaller ``-c`` in the extras is the context the user gets."""
try:
- from core.inference.llama_cpp import _batch_ubatch_for_mmproj
+ from core.inference.llama_cpp import _ASSUMED_MAX_VOCAB, _batch_ubatch_for_mmproj
from core.inference.llama_cpp import effective_ctx_checkpoints_for_caps
from core.inference.llama_server_args import (
parse_ctx_override,
@@ -10651,6 +10647,14 @@ def _gguf_runtime_bytes(
probe = _probe_backend()
probe._model_identifier = model_identifier
probe._read_gguf_metadata(gguf_path)
+ # An older llama.cpp reserves an output row per micro-batch token; priced as
+ # the loader prices it. getattr: a stand-in backend has no probe.
+ try:
+ probe._reserves_micro_batch_outputs = bool(
+ getattr(LlamaCppBackend, "reserves_micro_batch_outputs", lambda: False)()
+ )
+ except Exception as _rows_exc:
+ logger.debug("llama-server build probe failed: %s", _rows_exc)
# Price the same batch sizes used by load_model.
n_batch, n_ubatch = _batch_ubatch_for_mmproj(
0 if is_diffusion else launch_required_ubatch,
@@ -10715,7 +10719,6 @@ def _gguf_runtime_bytes(
# llama-server b10632, Qwen3-0.6B at 32k with -ctk/-ctv q4_0: 2296 MiB reserved
# against 1008 MiB allocated, identical on the CPU and Vulkan builds.
planned_v_type = planned_cache_types[1]
- v_forces_flash_attn = planned_v_type not in {"f16", "bf16", "f32"}
# the loader raises --batch-size to max(slots, 2) before launch, and llama.cpp
# caps the micro-batch against it, so budget from the emitted value. Diffusion
# takes neither flag, and SWA metadata prices the KV against the micro-batch,
@@ -10787,24 +10790,23 @@ def _gguf_runtime_bytes(
# so the default here is ON, not off. The false arm pads variable-width V
# tensors to the model-wide maximum (_max_kv_value_width), which on an
# architecture whose global and SWA layers disagree about n_embd_v_gqa
- # inflates the whole cache and can call a load that fits an overflow. Resolved
- # the way the launch resolves it: the managed default, narrowed by the
- # capability probe, then llama.cpp's own env-then-last-wins-argv rule. A
- # quantized V still forces it on top of all that, because llama-context.cpp
- # turns it on itself rather than refusing the load.
- _fa_default = True
+ # inflates the whole cache and can call a load that fits an overflow.
+ # Resolved the way the launch resolves it, narrowed by the capability probe.
+ _fa_supported = True
try:
_fa_caps = LlamaCppBackend.probe_server_capabilities()
# Same ``found`` test as the checkpoint probe above: the defaults dict
# reports nothing supported, so keying on the flag alone would size every
# unprobed host as flash-attention-less.
if _fa_caps.get("found") and not _fa_caps.get("supports_flash_attn", True):
- _fa_default = False
+ _fa_supported = False
except Exception as _fa_exc:
logger.debug("flash-attention capability probe failed: %s", _fa_exc)
- flash_attn = (
- _flash_attn_enabled_from_args(llama_extra_args, default = _fa_default, env = os.environ)
- or v_forces_flash_attn
+ flash_attn = _planned_flash_attn(
+ llama_extra_args,
+ _fa_supported,
+ planned_v_type,
+ getattr(probe, "_architecture", None),
)
kv = probe._estimate_kv_cache_bytes(
ctx,
@@ -10854,38 +10856,13 @@ def _gguf_runtime_bytes(
devices = max(1, int(n_devices))
def _flat_buffer(per_device_tensor: bool) -> int:
- """Flat compute buffer, rebuilt when the header is short of a vocab size.
-
- _estimate_compute_buffer_bytes returns 0 when vocab_size or embedding_length is
- missing. Only the first is reachable: _vocab_size is set solely from the
- tokenizer.ggml.tokens array length, which a truncated header drops while keeping
- the dims. So rebuild from the real formula with a vocab ceiling rather than
- substituting the loader's flat reserve. The loader can afford that reserve
- because over-reserving there just shrinks the context; here it denies the load.
- """
- flat = probe._estimate_compute_buffer_bytes(
+ """Use a vocabulary ceiling when a truncated header omits the token array."""
+ return probe._estimate_compute_buffer_bytes(
n_ubatch = effective_ubatch,
n_parallel = slots,
per_device_tensor = per_device_tensor,
+ vocab_ceiling = _ASSUMED_MAX_VOCAB,
)
- if flat > 0:
- return flat
- # getattr: a bare backend double carries neither dims nor constants.
- n_embd = getattr(probe, "_embedding_length", None) or 0
- if n_embd <= 0:
- return 0 # nothing to rebuild from; the total is floored below instead
- ub = max(1, int(effective_ubatch or probe._DEFAULT_N_UBATCH))
- act_scratch = 4 * n_embd * ub * 4
- out_buffer = _ASSUMED_MAX_VOCAB * ub * 4
- output_slots = (
- slots if getattr(probe, "is_embedding_gguf", False) else max(0, slots - 1)
- )
- raw = (
- 2 * act_scratch + out_buffer * slots
- if per_device_tensor
- else act_scratch + out_buffer * output_slots
- )
- return int(raw * probe._COMPUTE_BUFFER_SAFETY)
if tensor_parallel:
# mirrors _plan_tensor_parallel: per-device buffer and ctx growth on every device
@@ -10910,6 +10887,8 @@ def _gguf_runtime_bytes(
effective_ubatch,
cache_type_for_scratch,
layer_split = devices > 1 and not pipeline_parallel_off,
+ flash_attn = flash_attn,
+ n_parallel = slots,
)
)
if compute <= 0:
@@ -10993,8 +10972,12 @@ def _remote_gguf_compute_reserve_gb(
``required_ubatch`` matches the post-download launch.
"""
- # remote dims are unreadable; only the kq mask, linear in ubatch x ctx, can be sized here
- from core.inference.llama_cpp import _batch_ubatch_for_mmproj
+ # remote dims are unreadable: the kq mask is sized exactly, the activations at a ceiling
+ from core.inference.llama_cpp import (
+ _ASSUMED_MAX_ACTIVATION_WIDTH,
+ _ASSUMED_MAX_VOCAB,
+ _batch_ubatch_for_mmproj,
+ )
from core.inference.llama_server_args import parse_ctx_override
n_batch, n_ubatch = _batch_ubatch_for_mmproj(
@@ -11041,33 +11024,25 @@ def _remote_gguf_compute_reserve_gb(
and not _pipeline_parallel_disabled_by_args(llama_extra_args, n_layers = None)
else 1
)
- mask_bytes = (
- budget_ctx
- * effective_ubatch
- * 2
- * LlamaCppBackend._CTX_COMPUTE_F16_MASK_SAFETY
- * devices
- * split_mult
- )
- # The mask is only the context-linear half. The flat half needs dims that are
- # unreadable remotely, but its dominant term needs only a vocab ceiling:
- # llama.cpp reserves n_vocab * ubatch * 4 per slot past the first (every slot
- # under tensor mode), so n_batch = n_ubatch = 32768 on two slots is ~32 GiB the
- # mask does not cover, and omitting it let the guard admit an uncached load that
- # then OOMs the training job. The activation scratch needs embedding_length and
- # stays uncharged: the small half, and over-reserving denies the load outright.
- # Scaled per device only in tensor mode, mirroring the local branch. The remote
- # header is unknown, so reserve as if it enables embeddings.
- _out_slots = max(1, n_parallel)
- out_buffer_bytes = (
- _ASSUMED_MAX_VOCAB
- * effective_ubatch
- * 4
- * _out_slots
- * (devices if tensor_parallel else 1)
+ mask_bytes = budget_ctx * effective_ubatch * 2 * devices * split_mult
+ # Unknown dimensions require an activation ceiling. Reserve output rows for the
+ # maximum assumed draft depth. Tensor mode replicates both on every device.
+ activation_bytes = _ASSUMED_MAX_ACTIVATION_WIDTH * effective_ubatch * 4
+ output_rows = min(
+ effective_ubatch,
+ max(1, n_parallel) * (1 + LlamaCppBackend._UNKNOWN_SPEC_DRAFT_N_MAX),
+ )
+ try:
+ if LlamaCppBackend.reserves_micro_batch_outputs():
+ output_rows = effective_ubatch
+ except Exception as _rows_exc:
+ logger.debug("llama-server build probe failed: %s", _rows_exc)
+ flat_bytes = (
+ (activation_bytes + _ASSUMED_MAX_VOCAB * output_rows * 4)
* LlamaCppBackend._COMPUTE_BUFFER_SAFETY
+ * (devices if tensor_parallel else 1)
)
- return (mask_bytes + out_buffer_bytes) / (1024**3)
+ return (mask_bytes + flat_bytes) / (1024**3)
return 0.0
@@ -12585,6 +12560,8 @@ def _gguf_memory_breakdown(
# The part of that reserve allocated in the TARGET's context rather than the
# drafter's, so it stays with the target when the two are placed apart.
target_spec_bytes = 0
+ # The verification output rows in that reserve: target compute, not target KV.
+ target_verify_bytes = 0
drafter_on_gpu = host_drafter_bytes == 0
charged_drafter = _charged_drafter_path(
config, host_drafter_bytes or gpu_drafter_bytes, extras = extras
@@ -12659,7 +12636,9 @@ def _gguf_memory_breakdown(
n_ubatch = runtime.n_ubatch,
flash_attn = bool(_draft_v_type) and _draft_v_type not in {"f16", "bf16", "f32"},
)
- drafter_runtime_bytes = (
+ # The draft context's decode graph is charged on top, as the loader charges it:
+ # the helper prices only what that graph grows beyond this floor.
+ drafter_runtime_bytes = getattr(probe, "_MTP_DRAFT_COMPUTE_BYTES", 0) + (
probe._estimate_mtp_overhead_bytes(
runtime.n_ctx,
spec_draft_n_max = draft_n_max,
@@ -12691,6 +12670,10 @@ def _gguf_memory_breakdown(
probe._mamba_recurrent_state_bytes(runtime.n_parallel) * draft_n_max
)
target_spec_bytes = min(target_spec_bytes, drafter_runtime_bytes)
+ target_verify_bytes = min(
+ probe._spec_verify_rows_bytes(runtime.n_parallel, draft_n_max),
+ drafter_runtime_bytes - target_spec_bytes,
+ )
layer_count = runtime.layer_count
gpu_fraction = _gguf_offloaded_layer_fraction(
@@ -12766,7 +12749,8 @@ def _gguf_memory_breakdown(
if gpu_fraction > 0.0:
# Compute buffers land on the devices running layers.
gpu_bytes += runtime.compute_bytes
- # Target-side spec terms are target KV allocations and follow the target cache. The
+ # Target-side spec KV follows the target cache, and its verification rows follow the
+ # target's layers like the other compute buffers. The
# drafter's own KV follows the drafter: a separate one does not inherit
# --gpu-layers (llama.cpp overwrites it with the draft placement, default auto), so
# at --gpu-layers 0 it is still on the GPU holding that cache. An embedded head is
@@ -12774,8 +12758,14 @@ def _gguf_memory_breakdown(
# drafter_on_gpu said yes even for a CPU-placed target -- but it is part of the
# target's tensors and takes the target's context, so it goes where the target goes.
_drafter_on_gpu = (gpu_fraction > 0.0) if embedded_mtp else drafter_on_gpu
- drafter_runtime_gpu_bytes = (target_spec_bytes if kv_on_gpu else 0) + (
- (drafter_runtime_bytes - target_spec_bytes) if _drafter_on_gpu else 0
+ drafter_runtime_gpu_bytes = (
+ (target_spec_bytes if kv_on_gpu else 0)
+ + (target_verify_bytes if gpu_fraction > 0.0 else 0)
+ + (
+ (drafter_runtime_bytes - target_spec_bytes - target_verify_bytes)
+ if _drafter_on_gpu
+ else 0
+ )
)
gpu_bytes += drafter_runtime_gpu_bytes
# The encoder's buffers sit with the projector, so --no-mmproj-offload takes them
diff --git a/studio/backend/routes/models.py b/studio/backend/routes/models.py
index 2ed8bb52100..fb5190b2430 100644
--- a/studio/backend/routes/models.py
+++ b/studio/backend/routes/models.py
@@ -4092,6 +4092,10 @@ async def get_kv_cache_estimate(
# GPU, 3 without) and the rollback state is multiplied by it. An explicit 0 is still honoured.
spec_draft_n_max = _effective_draft_n_max,
)
+ # Plus the draft decode graph's floor, which the helper leaves to
+ # the loader's soft overhead.
+ if spec is not None:
+ spec += be._MTP_DRAFT_COMPUTE_BYTES
except Exception as e:
logger.debug(f"mtp overhead estimate failed for '{repo_id}' {quant}: {e}")
diff --git a/studio/backend/tests/test_batch_sizes_per_load.py b/studio/backend/tests/test_batch_sizes_per_load.py
index de13abca9c2..d6a7c83fb78 100644
--- a/studio/backend/tests/test_batch_sizes_per_load.py
+++ b/studio/backend/tests/test_batch_sizes_per_load.py
@@ -403,12 +403,8 @@ def test_emitted_batch_clears_both_llama_server_floors(n_batch, n_parallel, expe
def test_budgets_use_the_raised_batch_not_the_requested_one():
- """llama.cpp caps the micro-batch against the batch it is GIVEN
- (cparams.n_ubatch = min(cparams.n_batch, n_ubatch or n_batch)), and the loader
- raises --batch-size to max(slots, 2) before launch. Budgeting from the requested
- value instead planned n_batch=1 / n_ubatch=64 / 64 slots at a micro-batch of 1
- and launched it at 64: ~2.2 GB of compute buffer the training guard never
- charged, and an auto-fit sized for a graph the server does not build."""
+ """Budget the micro-batch after the loader raises batch size to max(slots, 2).
+ A batch of 1 with 64 slots and ubatch 64 must be priced at ubatch 64."""
from unittest.mock import patch
from routes import inference as route
@@ -429,7 +425,17 @@ def test_budgets_use_the_raised_batch_not_the_requested_one():
"/x.gguf", 32768, n_parallel = 64, n_batch = 1, n_ubatch = 1
)
assert raised == pytest.approx(explicit, abs = 0.01)
- assert raised > unraised + 2.0
+ # 63 more micro-batch tokens of activations and KQ mask, and 63 more output rows.
+ probe = LlamaCppBackend()
+ _header_reader(**_QWEN3_8B)(probe, "/x.gguf")
+ grown = (
+ probe._estimate_compute_buffer_bytes(n_ubatch = 64, n_parallel = 64)
+ + probe._compute_buffer_ctx_bytes(32768, 64)
+ - probe._estimate_compute_buffer_bytes(n_ubatch = 1, n_parallel = 64)
+ - probe._compute_buffer_ctx_bytes(32768, 1)
+ ) / 1024**3
+ assert grown > 0.05
+ assert raised - unraised == pytest.approx(grown, abs = 0.001)
# the remote branch shares the floor: no dims, but the kq mask still grows
from types import SimpleNamespace
@@ -648,8 +654,9 @@ def test_the_recorded_micro_batch_is_derived_from_the_slots_that_launched():
and isinstance(node.func, ast.Name)
and node.func.id == "_ubatch_for_slots"
]
- # sizing pass, embedding slot clamp, fit-time reduction, then the post-launch record
- assert len(calls) == 4, f"expected four re-derivations, found {len(calls)}"
+ # sizing pass, embedding slot clamp, candidate split pricing, fit-time reduction, then
+ # the post-launch record
+ assert len(calls) == 5, f"expected five re-derivations, found {len(calls)}"
# the record must not reuse the sizing pass's value
compact = "".join(src.split())
assert "self._n_ubatch=max(0,int(self._DEFAULT_N_UBATCHif_launched_ubatchisNone" in compact
@@ -660,12 +667,27 @@ def test_the_recorded_micro_batch_is_derived_from_the_slots_that_launched():
)
+def test_the_guard_prices_the_output_rows_of_the_build_it_launches():
+ """An older llama.cpp (before ggml-org/llama.cpp#23861) reserves an output row for
+ every micro-batch token; the guard asks the binary rather than assuming the cap."""
+ from unittest.mock import patch
+
+ from routes import inference as route
+
+ def _gb(older):
+ with (
+ patch.object(LlamaCppBackend, "_read_gguf_metadata", _header_reader(**_QWEN3_8B)),
+ patch.object(LlamaCppBackend, "reserves_micro_batch_outputs", lambda *a, **k: older),
+ ):
+ return route._estimate_gguf_kv_gb("/x.gguf", 8192, n_parallel = 1, n_ubatch = 2048)
+
+ rows = (2048 - 1) * 151936 * 4 * LlamaCppBackend._COMPUTE_BUFFER_SAFETY / 1024**3
+ assert _gb(True) - _gb(False) == pytest.approx(rows, abs = 0.001)
+
+
def test_the_remote_guard_charges_the_flat_output_buffer():
- """The KQ mask is only the context-linear half of the compute buffer. llama.cpp also
- reserves n_vocab * ubatch * 4 per slot past the first, which is context-INdependent and
- dwarfs the mask at large settings: n_batch = n_ubatch = 32768 on two slots is ~32 GiB
- the mask never covers. Omitting it remotely let the coexistence guard admit an uncached
- load that then OOMs the training job it exists to protect."""
+ """Unknown headers still need an activation reserve beyond the KQ mask.
+ Activations scale with ubatch; extra slots add output rows."""
from types import SimpleNamespace
from unittest.mock import patch
@@ -698,11 +720,11 @@ def test_the_remote_guard_charges_the_flat_output_buffer():
# Unset fields still launch at llama.cpp's known default 512-token micro-batch.
assert blank_1 > 1.5
- assert blank_4 > blank_1 + 1.5
- # The remote header may enable embeddings, so the first output buffer is charged.
+ # Slots add output rows, far below a second activation reserve.
+ assert blank_1 < blank_4 < blank_1 + 0.5
+ # The activation ceiling at a 32768-token micro-batch.
assert big_1 > blank_1 + 30.0
- # 262144 * 32768 * 4 = 32 GiB for the second slot too.
- assert big_2 > big_1 + 30.0
+ assert big_1 < big_2 < big_1 + 1.0
# and it stays proportionate where the values are ordinary
assert typical_4 < 4.0
@@ -730,7 +752,7 @@ def test_the_remote_guard_charges_the_flat_output_buffer():
layer_1, layer_2 = _split(False, 1), _split(False, 2)
tensor_1, tensor_2 = _split(True, 1), _split(True, 2)
- # the second device adds only the ctx-linear mask, not another 32 GiB of logits
+ # the second device adds only the ctx-linear mask, not another 35 GiB of activations
assert layer_2 - layer_1 < 30.0
# tensor mode replicates the whole buffer on every card, so it does roughly double
assert tensor_2 > tensor_1 * 1.8
diff --git a/studio/backend/tests/test_chat_load_during_training.py b/studio/backend/tests/test_chat_load_during_training.py
index 25015777baf..b52624dad74 100644
--- a/studio/backend/tests/test_chat_load_during_training.py
+++ b/studio/backend/tests/test_chat_load_during_training.py
@@ -1306,12 +1306,7 @@ class TestValidateRefusesDuringTraining(unittest.TestCase):
class TestRemoteGgufComputeReserve(unittest.TestCase):
- """The compute reserve a remote GGUF estimate carries.
-
- Every other caller here patches it to zero to assert exact GB totals, so its arithmetic goes
- unasserted even though it decides whether a load is refused. These pin the shape, not the
- magnitude: retuning a safety factor keeps them passing, dropping a term does not.
- """
+ """Check the remote GGUF compute reserve without the other tests' zero-cost stub."""
@classmethod
def setUpClass(cls):
@@ -1329,27 +1324,56 @@ class TestRemoteGgufComputeReserve(unittest.TestCase):
with patch.dict(os.environ, env, clear = True):
return self.route._remote_gguf_compute_reserve_gb(max_seq_length = 4096, **kwargs)
- def test_a_single_slot_still_reserves_an_output_buffer(self):
- """llama-server allocates an output buffer for its one slot, but the old
- max(0, n_parallel - 1) count reserved nothing there.
-
- The total is spelled out absolutely rather than compared against a neighbouring call: the
- reserve is linear in slot count, so any two samples are one buffer apart under both the
- old formula and the new one, and only an absolute anchor sees the floor.
- """
- from core.inference.llama_cpp import LlamaCppBackend
+ def test_reserve_is_the_mask_plus_an_activation_ceiling(self):
+ """Charge the activation ceiling once per micro-batch and output rows per slot."""
+ from core.inference.llama_cpp import (
+ _ASSUMED_MAX_ACTIVATION_WIDTH,
+ _ASSUMED_MAX_VOCAB,
+ LlamaCppBackend,
+ )
ubatch = LlamaCppBackend._DEFAULT_N_UBATCH
- mask = 4096 * ubatch * 2 * LlamaCppBackend._CTX_COMPUTE_F16_MASK_SAFETY
- per_slot = (
- self.route._ASSUMED_MAX_VOCAB * ubatch * 4 * LlamaCppBackend._COMPUTE_BUFFER_SAFETY
+ mask = 4096 * ubatch * 2
+
+ def expected(slots):
+ rows = min(ubatch, slots * (1 + LlamaCppBackend._UNKNOWN_SPEC_DRAFT_N_MAX))
+ flat = _ASSUMED_MAX_ACTIVATION_WIDTH * ubatch * 4 + _ASSUMED_MAX_VOCAB * rows * 4
+ return (mask + flat * LlamaCppBackend._COMPUTE_BUFFER_SAFETY) / (1024**3)
+
+ self.assertAlmostEqual(self._reserve(n_parallel = 1), expected(1), places = 6)
+ self.assertAlmostEqual(self._reserve(n_parallel = 2), expected(2), places = 6)
+ # A second slot costs rows, far less than a second activation reserve.
+ self.assertLess(
+ self._reserve(n_parallel = 2) - self._reserve(n_parallel = 1),
+ _ASSUMED_MAX_ACTIVATION_WIDTH * ubatch * 4 / (1024**3) / 4,
)
- self.assertAlmostEqual(self._reserve(n_parallel = 1), (mask + per_slot) / (1024**3), places = 6)
- # One more buffer for a second slot, pinning the count as well as the floor.
+
+ def test_an_older_build_reserves_a_row_per_micro_batch_token(self):
+ """Before ggml-org/llama.cpp#23861 the output rows cover the whole micro-batch,
+ which the remote guard charges when the binary reports such a build."""
+ from core.inference.llama_cpp import _ASSUMED_MAX_VOCAB, LlamaCppBackend
+
+ ubatch = LlamaCppBackend._DEFAULT_N_UBATCH
+ with patch.object(LlamaCppBackend, "reserves_micro_batch_outputs", lambda *a, **k: False):
+ current = self._reserve(n_parallel = 1)
+ with patch.object(LlamaCppBackend, "reserves_micro_batch_outputs", lambda *a, **k: True):
+ older = self._reserve(n_parallel = 1)
+ rows = (ubatch - (1 + LlamaCppBackend._UNKNOWN_SPEC_DRAFT_N_MAX)) * _ASSUMED_MAX_VOCAB * 4
self.assertAlmostEqual(
- self._reserve(n_parallel = 2), (mask + 2 * per_slot) / (1024**3), places = 6
+ older - current, rows * LlamaCppBackend._COMPUTE_BUFFER_SAFETY / (1024**3), places = 6
)
+ def test_tensor_mode_replicates_the_whole_buffer_on_every_device(self):
+ """Tensor mode reserves the mask, activations and output rows on each device."""
+ from core.inference.llama_cpp import LlamaCppBackend
+ for older_build in (False, True):
+ with patch.object(
+ LlamaCppBackend, "reserves_micro_batch_outputs", lambda *a, _o = older_build, **k: _o
+ ):
+ single = self._reserve(n_parallel = 2)
+ tensor = self._reserve(n_parallel = 2, n_devices = 2, tensor_parallel = True)
+ self.assertAlmostEqual(tensor, 2 * single, places = 6)
+
def test_diffusion_reserves_nothing(self):
"""The default micro-batch is a llama-server notion: a diffusion estimate has no ubatch to
assume, and charging one would refuse loads that fit."""
@@ -2829,6 +2853,7 @@ class TestEstimateGgufRequiredGb(unittest.TestCase):
n_ubatch = None,
n_parallel = 1,
per_device_tensor = False,
+ vocab_ceiling = None,
):
seen["compute_n_ubatch"] = n_ubatch
return 0
@@ -2840,6 +2865,8 @@ class TestEstimateGgufRequiredGb(unittest.TestCase):
cache_type_kv = None,
*,
layer_split = False,
+ flash_attn = True,
+ n_parallel = 1,
):
return 0
diff --git a/studio/backend/tests/test_compute_buffer.py b/studio/backend/tests/test_compute_buffer.py
index 16381ece51b..727082ab887 100644
--- a/studio/backend/tests/test_compute_buffer.py
+++ b/studio/backend/tests/test_compute_buffer.py
@@ -1,10 +1,7 @@
# SPDX-License-Identifier: AGPL-3.0-only
# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
-"""Tests for ``_estimate_compute_buffer_bytes``: it scales with ``--parallel``,
-tensor exceeds pipeline, and it is a safe upper bound on the allocations measured
-on real hardware (Qwen3.6-27B-MTP: parallel 1/2/4/8 -> 36/492/1388/3220 MiB single
-GPU, ~600 MiB/device tensor). No GPU, subprocess, or GGUF I/O."""
+"""Compute-buffer estimates checked against measured allocations and synthetic layouts."""
from __future__ import annotations
@@ -85,6 +82,15 @@ def _backend(
return b
+def _pad(cells):
+ return ((cells + 255) // 256) * 256
+
+
+def _hidden_copies(embd, ub = 512):
+ """The hidden-state copies a layer-split device holds, one per scheduler copy."""
+ return LlamaCppBackend._CTX_COMPUTE_SPLIT_MULT * embd * ub * 4
+
+
def _backend_from_gguf_local(
n_layers = 80,
embd = 8192,
@@ -110,76 +116,271 @@ def _backend_from_gguf_local(
return b
-# Measured ground truth (MiB) the estimate must upper-bound.
-_PIPELINE_MEASURED = {1: 36, 2: 492, 4: 1388, 8: 3220}
-_TENSOR_MEASURED_PER_DEVICE = 600
-
-
-class TestSafeUpperBound:
- """The estimate must be >= every measured allocation (never under-reserve)."""
-
- @pytest.mark.parametrize("parallel,measured", sorted(_PIPELINE_MEASURED.items()))
- def test_pipeline_upper_bounds_measured(self, parallel, measured):
- est = _backend()._estimate_compute_buffer_bytes(n_parallel = parallel) / MIB
- assert est >= measured, f"under-reserved at parallel={parallel}: {est:.0f} < {measured}"
-
- @pytest.mark.parametrize("parallel,measured", sorted(_PIPELINE_MEASURED.items()))
- def test_pipeline_not_wildly_over(self, parallel, measured):
- # Stay within ~2x of measured so we don't waste context (the point of
- # replacing the flat reserve). parallel=1 is tiny in absolute terms.
- est = _backend()._estimate_compute_buffer_bytes(n_parallel = parallel) / MIB
- assert est <= max(measured * 2.0, 128)
-
- def test_tensor_upper_bounds_measured(self):
- est = _backend()._estimate_compute_buffer_bytes(n_parallel = 1, per_device_tensor = True) / MIB
- assert est >= _TENSOR_MEASURED_PER_DEVICE
-
- def test_tensor_far_below_old_flat_reserve(self):
- # The whole point: deterministic estimate << flat 5120 for this model.
- est = _backend()._estimate_compute_buffer_bytes(n_parallel = 1, per_device_tensor = True) / MIB
- assert est < LlamaCppBackend._TENSOR_PARALLEL_BUFFER_RESERVE_MIB
-
-
-class TestScaling:
- def test_grows_with_serving_slots(self):
- b = _backend()
- vals = [b._estimate_compute_buffer_bytes(n_parallel = p) for p in (1, 2, 4, 8)]
- assert vals == sorted(vals) and vals[0] < vals[-1]
-
- def test_parallel_1_is_small(self):
- # Single-token decode: a few tens of MiB, not gigabytes.
- est = _backend()._estimate_compute_buffer_bytes(n_parallel = 1) / MIB
- assert est < 128
+_GEMMA4_12B_PATTERN = ([True] * 5 + [False]) * 8
+
+# GGUF dims of models whose buffers were measured on the bundled llama.cpp b10909
+# (RTX 6000 Ada, --flash-attn on, --kv-unified above one slot).
+_DIMS = {
+ "Llama-3.2-1B": dict(
+ _vocab_size = 128256,
+ _embedding_length = 2048,
+ _feed_forward_length = 8192,
+ _n_layers = 16,
+ _n_heads = 32,
+ _n_kv_heads = 8,
+ _kv_key_length = 64,
+ _kv_value_length = 64,
+ ),
+ "Qwen3-8B": dict(
+ _vocab_size = 151936,
+ _embedding_length = 4096,
+ _feed_forward_length = 12288,
+ _n_layers = 36,
+ _n_heads = 32,
+ _n_kv_heads = 8,
+ _kv_key_length = 128,
+ _kv_value_length = 128,
+ ),
+ "Qwen3.5-4B": dict(
+ _vocab_size = 248320,
+ _embedding_length = 2560,
+ _feed_forward_length = 9216,
+ _n_layers = 32,
+ _n_heads = 16,
+ _n_kv_heads = 4,
+ _kv_key_length = 256,
+ _kv_value_length = 256,
+ _ssm_inner_size = 4096,
+ _ssm_state_size = 128,
+ _ssm_group_count = 16,
+ _full_attention_interval = 4,
+ ),
+ "Qwen3.6-35B-A3B": dict(
+ _vocab_size = 248320,
+ _embedding_length = 2048,
+ _expert_used_count = 8,
+ _expert_feed_forward_length = 512,
+ _expert_shared_feed_forward_length = 512,
+ _n_layers = 40,
+ _n_heads = 16,
+ _n_kv_heads = 2,
+ _kv_key_length = 256,
+ _kv_value_length = 256,
+ _ssm_inner_size = 4096,
+ _ssm_state_size = 128,
+ _ssm_group_count = 16,
+ _full_attention_interval = 4,
+ ),
+ "Gemma-4-12B": dict(
+ _vocab_size = 262144,
+ _embedding_length = 3840,
+ _feed_forward_length = 15360,
+ _n_layers = 48,
+ _n_heads = 16,
+ _n_kv_heads = 8,
+ _n_kv_heads_by_layer = [1 if not swa else 8 for swa in _GEMMA4_12B_PATTERN],
+ _kv_key_length = 512,
+ _kv_value_length = 512,
+ _kv_key_length_swa = 256,
+ _kv_value_length_swa = 256,
+ _sliding_window = 1024,
+ _sliding_window_pattern = _GEMMA4_12B_PATTERN,
+ ),
+ "Gemma-4-E4B": dict(
+ _vocab_size = 262144,
+ _embedding_length = 2560,
+ _feed_forward_length = 10240,
+ _embedding_length_per_layer_input = 256,
+ _n_layers = 42,
+ _n_heads = 8,
+ _n_kv_heads = 2,
+ _kv_key_length = 512,
+ _kv_value_length = 512,
+ _kv_key_length_swa = 256,
+ _kv_value_length_swa = 256,
+ _sliding_window = 512,
+ _sliding_window_pattern = ([True] * 5 + [False]) * 7,
+ ),
+ "GLM-4.7-Flash": dict(
+ _vocab_size = 154880,
+ _embedding_length = 2048,
+ _feed_forward_length = 10240,
+ _expert_used_count = 4,
+ _expert_feed_forward_length = 1536,
+ _expert_shared_count = 1,
+ _n_layers = 47,
+ _n_heads = 20,
+ _n_kv_heads = 1,
+ _kv_key_length = 576,
+ _kv_value_length = 512,
+ _kv_lora_rank = 512,
+ _key_length_mla = 256,
+ ),
+}
+
+
+def _measured(name):
+ b = LlamaCppBackend()
+ for attr, value in _DIMS[name].items():
+ setattr(b, attr, value)
+ return b
- def test_tensor_exceeds_pipeline_at_same_parallel(self):
- b = _backend()
- pipe = b._estimate_compute_buffer_bytes(n_parallel = 1)
- tens = b._estimate_compute_buffer_bytes(n_parallel = 1, per_device_tensor = True)
- assert tens > pipe
- def test_scales_with_vocab(self):
- small = _backend(vocab = 32000)._estimate_compute_buffer_bytes(n_parallel = 4)
- big = _backend(vocab = 256000)._estimate_compute_buffer_bytes(n_parallel = 4)
- assert big > small
+# llama-server's "CUDA0 compute buffer size" (MiB): (model, ubatch, slots, n_ctx, KV type).
+_MEASURED_COMPUTE = [
+ ("Llama-3.2-1B", 256, 4, 32768, "f16", 46.01),
+ ("Llama-3.2-1B", 2048, 4, 32768, "f16", 368.04),
+ ("Llama-3.2-1B", 512, 1, 32768, "f16", 92.01),
+ ("Llama-3.2-1B", 512, 8, 32768, "f16", 92.01),
+ ("Llama-3.2-1B", 512, 4, 131072, "f16", 188.01),
+ ("Llama-3.2-1B", 512, 4, 131072, "q8_0", 400.04),
+ ("Llama-3.2-1B", 2048, 4, 131072, "q8_0", 832.07),
+ ("Qwen3-8B", 512, 4, 32768, "f16", 128.01),
+ ("Qwen3-8B", 2048, 4, 131072, "f16", 896.04),
+ ("Qwen3-8B", 512, 4, 131072, "q8_0", 672.09),
+ ("Qwen3.5-4B", 512, 1, 32768, "f16", 96.02),
+ ("Qwen3.5-4B", 512, 8, 32768, "f16", 115.52),
+ ("Qwen3.5-4B", 2048, 4, 131072, "f16", 768.06),
+ ("Qwen3.5-4B", 2048, 4, 131072, "q8_0", 1224.34),
+ ("Qwen3.6-35B-A3B", 256, 4, 32768, "f16", 66.02),
+ ("Qwen3.6-35B-A3B", 2048, 1, 32768, "f16", 528.2),
+ ("Qwen3.6-35B-A3B", 512, 4, 131072, "q8_0", 436.31),
+ ("Gemma-4-12B", 512, 1, 32768, "f16", 154.52),
+ ("Gemma-4-12B", 1120, 4, 32768, "f16", 346.21),
+ ("Gemma-4-12B", 2048, 8, 32768, "f16", 652.07),
+ ("Gemma-4-12B", 2048, 4, 131072, "f16", 1020.07),
+ ("Gemma-4-12B", 512, 4, 131072, "q8_0", 436.8),
+ ("Gemma-4-E4B", 512, 4, 32768, "f16", 135.52),
+ ("Gemma-4-E4B", 2048, 4, 131072, "q8_0", 1317.35),
+ ("GLM-4.7-Flash", 512, 4, 32768, "f16", 110.01),
+ ("GLM-4.7-Flash", 2048, 4, 131072, "f16", 824.04),
+ ("GLM-4.7-Flash", 512, 4, 131072, "q8_0", 358.9),
+]
+
+
+def _priced_mib(b, ub, slots, ctx, kv):
+ flat = b._estimate_compute_buffer_bytes(n_ubatch = ub, n_parallel = slots)
+ return (flat + b._compute_buffer_ctx_bytes(ctx, ub, kv)) / MIB
+
+
+class TestMeasuredComputeBuffer:
+ """Flat plus context-linear must cover llama-server's own reserve without
+ inflating it: the vocab-wide rows the old formula charged per slot came to 16 GiB
+ for Gemma 4 12B at 8 slots and ubatch 2048, which reserves 652 MiB."""
+
+ @pytest.mark.parametrize("name,ub,slots,ctx,kv,measured", _MEASURED_COMPUTE)
+ def test_covers_the_measured_buffer(self, name, ub, slots, ctx, kv, measured):
+ est = _priced_mib(_measured(name), ub, slots, ctx, kv)
+ assert est >= measured, f"{name} ub{ub} p{slots} {ctx} {kv}: {est:.0f} < {measured}"
+
+ @pytest.mark.parametrize("name,ub,slots,ctx,kv,measured", _MEASURED_COMPUTE)
+ def test_stays_close_to_the_measured_buffer(self, name, ub, slots, ctx, kv, measured):
+ # The quantized scratch and the activations peak at different points of the
+ # graph but are charged together, the widest gap at a short context.
+ est = _priced_mib(_measured(name), ub, slots, ctx, kv)
+ assert est <= measured * (1.35 if kv == "f16" else 1.3), f"{name}: {est:.0f} vs {measured}"
+
+ def test_the_reported_gemma4_vision_raise(self):
+ # The micro-batch raise for the Gemma 4 projector, 512 -> 1120 at 4 slots and
+ # 262144 cells: llama-server's buffer grows 455 MiB, the old estimate 2.5 GiB.
+ b = _measured("Gemma-4-12B")
+ raise_mib = _priced_mib(b, 1120, 4, 262144, "f16") - _priced_mib(b, 512, 4, 262144, "f16")
+ assert 455 <= raise_mib <= 455 * 1.15
+
+
+class TestFlatBufferShape:
+ def test_slots_add_output_rows_not_activations(self):
+ b = _measured("Gemma-4-12B")
+ one = b._estimate_compute_buffer_bytes(n_ubatch = 2048, n_parallel = 1)
+ eight = b._estimate_compute_buffer_bytes(n_ubatch = 2048, n_parallel = 8)
+ rows = 7 * 262144 * 4
+ swa = (_pad(1024 * 8 + 2048) - _pad(1024 + 2048)) * 2048 * 2
+ assert eight - one == pytest.approx(
+ rows * LlamaCppBackend._COMPUTE_BUFFER_SAFETY + swa, rel = 1e-6
+ )
- def test_scales_with_ubatch(self):
- b = _backend()
- lo = b._estimate_compute_buffer_bytes(n_parallel = 4, n_ubatch = 256)
- hi = b._estimate_compute_buffer_bytes(n_parallel = 4, n_ubatch = 1024)
- assert hi > lo
-
- def test_embedding_mode_budgets_the_first_output_buffer(self):
- chat = _backend()._estimate_compute_buffer_bytes(n_parallel = 1, n_ubatch = 512)
- embedding = _backend(pooling_type = 2)._estimate_compute_buffer_bytes(
- n_parallel = 1, n_ubatch = 512
+ def test_scales_linearly_with_ubatch(self):
+ b = _measured("Qwen3-8B")
+ lo = b._estimate_compute_buffer_bytes(n_ubatch = 512, n_parallel = 1)
+ hi = b._estimate_compute_buffer_bytes(n_ubatch = 1024, n_parallel = 1)
+ rows = 151936 * 4 * LlamaCppBackend._COMPUTE_BUFFER_SAFETY
+ assert hi - rows == pytest.approx(2 * (lo - rows), rel = 1e-6)
+
+ def test_a_tensor_device_holds_the_single_device_buffer(self):
+ # Qwen3 8B and Gemma 4 12B split over two GPUs reserve their single-GPU
+ # buffer on each device, at ubatch 512 and 2048 alike.
+ for name in ("Qwen3-8B", "Gemma-4-12B"):
+ b = _measured(name)
+ assert b._estimate_compute_buffer_bytes(
+ n_ubatch = 2048, n_parallel = 4, per_device_tensor = True
+ ) == b._estimate_compute_buffer_bytes(n_ubatch = 2048, n_parallel = 4)
+
+ def test_embedding_rows_are_pooled_not_vocab_wide(self):
+ b = _backend(vocab = 30522, embd = 384, pooling_type = 1)
+ chat = _backend(vocab = 30522, embd = 384)
+ assert b._estimate_compute_buffer_bytes(n_ubatch = 512, n_parallel = 4) > (
+ chat._estimate_compute_buffer_bytes(n_ubatch = 512, n_parallel = 4)
+ )
+ # 512 pooled rows of 384 floats, far below 512 vocabulary rows.
+ assert b._estimate_compute_buffer_bytes(n_ubatch = 512, n_parallel = 4) < 30522 * 512 * 4
+
+ def test_widest_block_sets_the_width(self):
+ # Dense FFN, routed experts, and the delta-net convolution each can be widest.
+ assert _measured("Qwen3-8B")._compute_activation_width() == 4 * 12288
+ assert _measured("Qwen3.6-35B-A3B")._compute_activation_width() == (
+ 8 * (2 * 2048 + 3 * 512) + 3 * 512
)
- assert embedding > chat
+ ssm = _backend(embd = 2560)
+ ssm._ssm_inner_size, ssm._ssm_state_size, ssm._ssm_group_count = 4096, 128, 16
+ assert ssm._compute_activation_width() == 4 * (4096 + 2 * 16 * 128)
+ # Per-layer input embeddings ride on top of the widest block.
+ assert _measured("Gemma-4-E4B")._compute_activation_width() == 4 * 10240 + 256 * 42
+
+ def test_header_without_ffn_dims_uses_the_embedding_floor(self):
+ assert _backend(embd = 4096)._compute_activation_width() == 12 * 4096
+
+ def test_expert_width_falls_back_to_the_ffn_length(self):
+ # Granite 4.0 H writes its 512-wide experts as feed_forward_length.
+ b = _backend(vocab = 100352, embd = 1536)
+ b._n_experts, b._expert_used_count = 64, 6
+ b._feed_forward_length, b._expert_shared_feed_forward_length = 512, 1024
+ assert b._compute_activation_width() == 6 * (2 * 1536 + 3 * 512) + 3 * 1024
+
+ def test_granite_hybrid_is_covered(self):
+ # Held out from the fit: llama-server reserved 80.0 MiB at one slot and 95.3
+ # MiB at eight (ubatch 512, 32768 cells), the recurrent state per slot.
+ b = _backend(vocab = 100352, embd = 1536)
+ b._n_experts, b._expert_used_count = 64, 6
+ b._feed_forward_length, b._expert_shared_feed_forward_length = 512, 1024
+ b._ssm_inner_size, b._ssm_state_size, b._ssm_group_count = 3072, 128, 1
+ for slots, measured in ((1, 80.0), (8, 95.3)):
+ est = _priced_mib(b, 512, slots, 32768, "f16")
+ assert measured <= est <= measured * 1.3
+
+ def test_a_window_wider_than_the_context_does_not_slide(self):
+ # Phi-4-mini declares a 262144 window over a 131072 context and no pattern;
+ # pricing it as a sliding mask charged 12x its 92 MiB buffer.
+ b = _backend(vocab = 200064, embd = 3072)
+ b._feed_forward_length, b._context_length = 8192, 131072
+ b._sliding_window, b._sliding_window_pattern = 262144, None
+ assert b._effective_sliding_window() == 0
+ assert 92.0 <= _priced_mib(b, 512, 4, 32768, "f16") <= 92.0 * 1.3
+ b._sliding_window_pattern = [False] * 32
+ assert b._effective_sliding_window() == 0
+ b._sliding_window_pattern = [True, False] * 16
+ assert b._effective_sliding_window() == 262144
class TestFallback:
def test_zero_when_vocab_missing(self):
assert _backend(vocab = None)._estimate_compute_buffer_bytes(n_parallel = 4) == 0
+ def test_vocab_ceiling_prices_a_header_without_a_token_array(self):
+ b = _backend(vocab = None)
+ assert b._estimate_compute_buffer_bytes(n_parallel = 4, vocab_ceiling = 262144) == (
+ _backend(vocab = 262144)._estimate_compute_buffer_bytes(n_parallel = 4)
+ )
+
def test_zero_when_embd_missing(self):
assert _backend(embd = None)._estimate_compute_buffer_bytes(n_parallel = 4) == 0
@@ -192,147 +393,78 @@ class TestFallback:
assert gi == [0, 1] # both GPUs usable under the flat fallback
-class TestParallel1Default:
- """At Unsloth's default --parallel 1 the buffer is negligible in pipeline."""
-
- def test_default_n_parallel(self):
- est = _backend()._estimate_compute_buffer_bytes() / MIB
- assert est < 128
-
-
class TestContextLinearBuffer:
- """``_compute_buffer_ctx_bytes``: the flash-attn KQ-mask + attention scratch
- grow ~linearly with context; the flat estimate above only covers ctx -> 0.
- Measured slope (q8_0 KV, ubatch 512) was 0.74-2.02 x n_embd; 2 x n_embd is the
- worst-case upper bound the term must hold to."""
+ """``_compute_buffer_ctx_bytes``: the KQ mask, n_ubatch * 2 bytes per cell, plus
+ for a quantized cache an f16 copy of the widest global layer's K and V."""
- # (model, n_embd, ctx, measured CUDA0 compute buffer MiB at that ctx, q8_0/ub512)
+ # (model, n_embd, ctx, measured CUDA0 compute buffer MiB at that ctx, q8_0/ub512).
+ # From an earlier build; kept as upper bounds on a header with only n_embd.
_MEASURED = [
("Qwen3.5-2B", 2048, 262144, 796),
- ("Qwen3.5-4B", 2560, 262144, 1330), # worst slope, 2.02 x n_embd
+ ("Qwen3.5-4B", 2560, 262144, 1330),
("Qwen3.5-9B", 4096, 262144, 1336),
("Qwen3.6-27B", 5120, 262144, 1360),
("Gemma-4-31B", 5376, 262144, 2392),
]
def test_zero_by_default(self):
- # Omitted/zero ctx -> no term (keeps the flat callers unchanged).
assert _backend()._compute_buffer_ctx_bytes(0) == 0
def test_zero_when_embd_missing(self):
assert _backend(embd = None)._compute_buffer_ctx_bytes(262144) == 0
def test_grows_linearly_with_context(self):
- b = _backend(embd = 4096)
- a = b._compute_buffer_ctx_bytes(65536)
- d = b._compute_buffer_ctx_bytes(131072)
- assert d == pytest.approx(2 * a, rel = 1e-6)
-
- def test_scales_with_embd(self):
- # The quantized (dequant-scratch) rate scales with n_embd; f16 (mask) does not.
- small = _backend(embd = 2048)._compute_buffer_ctx_bytes(131072, cache_type_kv = "q8_0")
- big = _backend(embd = 5120)._compute_buffer_ctx_bytes(131072, cache_type_kv = "q8_0")
- assert big > small
-
- def test_scales_with_ubatch(self):
- b = _backend(embd = 4096)
- lo = b._compute_buffer_ctx_bytes(131072, n_ubatch = 256)
- hi = b._compute_buffer_ctx_bytes(131072, n_ubatch = 1024)
- assert hi > lo
+ b = _measured("Qwen3-8B")
+ for kv in ("f16", "q8_0"):
+ a = b._compute_buffer_ctx_bytes(65536, cache_type_kv = kv)
+ d = b._compute_buffer_ctx_bytes(131072, cache_type_kv = kv)
+ assert d == pytest.approx(2 * a, rel = 1e-6)
@pytest.mark.parametrize("name,embd,ctx,measured", _MEASURED)
def test_upper_bounds_measured_compute_growth(self, name, embd, ctx, measured):
- # flat term + context-linear term must cover the real (q8_0) buffer at full ctx.
b = _backend(embd = embd)
flat = b._estimate_compute_buffer_bytes(n_parallel = 1)
total = (flat + b._compute_buffer_ctx_bytes(ctx, cache_type_kv = "q8_0")) / MIB
assert total >= measured, f"{name}: under-reserved {total:.0f} < {measured}"
- def test_worst_case_rate_covers_two_x_embd(self):
- # >= 2 x n_embd bytes per context token at the default micro-batch (the worst
- # measured quantized slope, Qwen3.5-4B), so flat + term upper-bounds the buffer.
- embd = 4096
- b = _backend(embd = embd)
- per_tok = b._compute_buffer_ctx_bytes(100000, cache_type_kv = "q8_0") / 100000
- assert per_tok >= 2 * embd
-
class TestContextBufferKVQuant:
- """The context-linear rate depends on the KV cache type: a quantized cache adds a
- context-sized dequant scratch (heavy); f16/bf16/f32 only pays the KQ mask (light).
- Measured Qwen3.5-4B at 256k: 1.30 GiB (q8_0) vs 0.31 GiB (f16)."""
-
def test_quantized_heavier_than_f16(self):
- b = _backend(embd = 4096)
- q = b._compute_buffer_ctx_bytes(131072, cache_type_kv = "q8_0")
- f = b._compute_buffer_ctx_bytes(131072, cache_type_kv = "f16")
- assert q > f
+ b = _measured("Qwen3-8B")
+ assert b._compute_buffer_ctx_bytes(131072, cache_type_kv = "q8_0") > (
+ b._compute_buffer_ctx_bytes(131072, cache_type_kv = "f16")
+ )
def test_none_cache_type_is_f16(self):
- # None -> f16 (llama.cpp's default); the env-quantized case is covered by the
- # KV budget's f16 over-reservation, so we take the lighter mask-only rate.
- b = _backend(embd = 4096)
- assert b._compute_buffer_ctx_bytes(
- 131072, cache_type_kv = None
- ) == b._compute_buffer_ctx_bytes(131072, cache_type_kv = "f16")
+ b = _measured("Qwen3-8B")
+ assert b._compute_buffer_ctx_bytes(131072, cache_type_kv = None) == (
+ b._compute_buffer_ctx_bytes(131072, cache_type_kv = "f16")
+ )
@pytest.mark.parametrize("ct", ["f16", "bf16", "f32"])
- def test_unquantized_uses_mask_only_rate(self, ct):
- # f16/bf16/f32: KQ mask only, n_ubatch*2 B/tok, independent of n_embd.
- b_small = _backend(embd = 2048)
- b_big = _backend(embd = 8192)
- per_small = b_small._compute_buffer_ctx_bytes(100000, cache_type_kv = ct) / 100000
- per_big = b_big._compute_buffer_ctx_bytes(100000, cache_type_kv = ct) / 100000
- assert per_small == per_big # no n_embd scaling on the f16 path
- expected = 512 * 2 * LlamaCppBackend._CTX_COMPUTE_F16_MASK_SAFETY # ubatch 512
- assert per_small == pytest.approx(expected, rel = 1e-6)
+ def test_unquantized_pays_exactly_the_mask(self, ct):
+ # n_ubatch * 2 bytes per cell whatever the model, as measured on every family.
+ for name in _DIMS:
+ b = _measured(name)
+ for ub in (512, 2048):
+ assert b._compute_buffer_ctx_bytes(100000, ub, ct) == ub * 2 * 100000
@pytest.mark.parametrize("ct", ["q8_0", "q5_1", "q4_0", "iq4_nl"])
- def test_quantized_types_use_heavy_rate(self, ct):
- embd = 4096
- b = _backend(embd = embd)
- per_tok = b._compute_buffer_ctx_bytes(100000, cache_type_kv = ct) / 100000
- assert per_tok == pytest.approx(
- LlamaCppBackend._CTX_COMPUTE_BYTES_PER_EMBD * embd, rel = 1e-6
- )
-
- def test_f16_covers_measured_mask(self):
- # f16 buffer is ~mask only (~n_ubatch*2 B/tok); 0.5 x n_embd must cover the
- # measured Qwen3.5-4B f16 slope (~0.4 x n_embd = 0.31 GiB at 256k).
- b = _backend(embd = 2560) # Qwen3.5-4B
- est = b._compute_buffer_ctx_bytes(262144, cache_type_kv = "f16") / MIB
- assert est >= 320 # measured 0.31 GiB growth
-
+ def test_quantized_adds_the_widest_kv_copy_whatever_the_ubatch(self, ct):
+ b = _measured("Qwen3-8B")
+ for ub in (512, 2048):
+ extra = b._compute_buffer_ctx_bytes(100000, ub, ct) - ub * 2 * 100000
+ assert extra == 8 * (128 + 128) * 2 * 100000
-class TestContextBufferMLA:
- """MLA (compressed attention) needs a smaller quantized dequant scratch than
- regular attention: measured 0.94 x n_embd on GLM-5.2 and Kimi-K2.7 vs up to
- 2.02x on Qwen/Gemma. Charging the regular rate would badly over-reserve a tight
- multi-GPU MLA pin (per-device scaling multiplies the error)."""
+ def test_sliding_window_layers_do_not_set_the_rate(self):
+ # Gemma 4 12B: 8 KV heads on the sliding layers, 1 on the global ones.
+ assert _measured("Gemma-4-12B")._attention_dequant_width() == 1 * (512 + 512)
- def test_mla_lighter_than_regular(self):
- reg = _backend(embd = 6144, mla = None)._compute_buffer_ctx_bytes(262144, cache_type_kv = "q8_0")
- mla = _backend(embd = 6144, mla = 256)._compute_buffer_ctx_bytes(262144, cache_type_kv = "q8_0")
- assert mla < reg
-
- @pytest.mark.parametrize(
- "name,embd,ctx,measured",
- [
- ("GLM-5.2", 6144, 754688, 4141), # per-device compute MiB at q8_0
- ("Kimi-K2.7", 7168, 262144, 1690),
- ],
- )
- def test_mla_rate_covers_measured(self, name, embd, ctx, measured):
- b = _backend(embd = embd, mla = 256)
- est = b._compute_buffer_ctx_bytes(ctx, cache_type_kv = "q8_0") / MIB
- assert est >= measured, f"{name}: MLA under-reserved {est:.0f} < {measured}"
+ def test_mla_copies_the_compressed_key_only(self):
+ assert _measured("GLM-4.7-Flash")._attention_dequant_width() == 576
- def test_mla_not_wildly_over(self):
- # 1.25 x n_embd should stay within ~1.6x of the measured 0.94x (not 2.4x like
- # the regular 2.25 rate would), so a multi-GPU MLA pin keeps its context.
- b = _backend(embd = 6144, mla = 256)
- est = b._compute_buffer_ctx_bytes(754688, cache_type_kv = "q8_0") / MIB
- assert est <= 4141 * 1.7
+ def test_header_without_head_dims_falls_back_to_embd(self):
+ assert _backend(embd = 4096)._attention_dequant_width() == 4096
class TestContextBufferDSV4:
@@ -390,8 +522,7 @@ class TestContextBufferDSV4:
# Regression guard: a non-deepseek4 model keeps the mask-only f16 rate.
b = _backend(embd = 4096, arch = "llama")
per_tok = b._compute_buffer_ctx_bytes(100000, cache_type_kv = "f16") / 100000
- expected = 512 * 2 * LlamaCppBackend._CTX_COMPUTE_F16_MASK_SAFETY
- assert per_tok == pytest.approx(expected, rel = 1e-6)
+ assert per_tok == pytest.approx(512 * 2, rel = 1e-6)
class TestContextBufferLayerSplit:
@@ -426,11 +557,22 @@ class TestContextBufferLayerSplit:
131072, cache_type_kv = "f16"
) == b._compute_buffer_ctx_bytes(131072, cache_type_kv = "f16", layer_split = False)
- def test_split_is_exactly_the_multiplier(self):
+ def test_split_is_the_multiplier_plus_the_hidden_state_copies(self):
b = _backend(embd = 4096)
one = b._compute_buffer_ctx_bytes(131072, cache_type_kv = "f16")
many = b._compute_buffer_ctx_bytes(131072, cache_type_kv = "f16", layer_split = True)
- assert many == pytest.approx(one * LlamaCppBackend._CTX_COMPUTE_SPLIT_MULT, rel = 1e-9)
+ assert many == one * LlamaCppBackend._CTX_COMPUTE_SPLIT_MULT + _hidden_copies(4096)
+
+ def test_matches_the_measured_two_gpu_split(self):
+ # Qwen3 8B over an RTX 6000 Ada and an RTX 3090: 256.04 MiB on each device at
+ # 32768 cells and ubatch 512, 2560.16 MiB at 131072 cells and ubatch 2048.
+ b = _measured("Qwen3-8B")
+ for ctx, ub, measured in ((32768, 512, 256.04), (131072, 2048, 2560.16)):
+ per_device = (
+ b._estimate_compute_buffer_bytes(n_ubatch = ub, n_parallel = 1)
+ + b._compute_buffer_ctx_bytes(ctx, ub, "f16", layer_split = True)
+ ) / MIB
+ assert measured <= per_device <= measured * 1.1
@pytest.mark.parametrize("name,n_gpus,ub,measured", _MEASURED)
def test_upper_bounds_measured_per_device_rate(self, name, n_gpus, ub, measured):
@@ -445,20 +587,17 @@ class TestContextBufferLayerSplit:
assert per_tok >= measured * ub, f"{name} n={n_gpus}: {per_tok:.0f} < {measured * ub:.0f}"
@pytest.mark.parametrize("name,n_gpus,ub,measured", _MEASURED)
- def test_not_wildly_over_measured_per_device_rate(self, name, n_gpus, ub, measured):
- # The step is a correction, not extra headroom: still the 1.5 safety factor.
+ def test_charges_exactly_the_measured_per_device_rate(self, name, n_gpus, ub, measured):
+ # The mask is priced exactly; only the split's hidden-state copies sit on top.
b = _backend(embd = 4096)
- per_tok = (
- b._compute_buffer_ctx_bytes(
- 100000, n_ubatch = ub, cache_type_kv = "f16", layer_split = n_gpus > 1
- )
- / 100000
+ hidden = _hidden_copies(4096, ub) if n_gpus > 1 else 0
+ charged = b._compute_buffer_ctx_bytes(
+ 100000, n_ubatch = ub, cache_type_kv = "f16", layer_split = n_gpus > 1
)
- expected = measured * ub * LlamaCppBackend._CTX_COMPUTE_F16_MASK_SAFETY
- assert per_tok == pytest.approx(expected, rel = 1e-6)
+ assert charged - hidden == pytest.approx(measured * ub * 100000, rel = 1e-6)
def test_pre_fix_split_reserve_was_short(self):
- # The bug: without the step a split reserved 8/(2*1.5) = 2.67x too little.
+ # The bug: without the step a split reserved a quarter of its masks.
b = _backend(embd = 4096)
one = b._compute_buffer_ctx_bytes(1048576, cache_type_kv = "f16")
measured = self._RATE_SPLIT * 512 * 1048576
@@ -485,7 +624,7 @@ class TestContextBufferLayerSplit:
mask = b._compute_buffer_ctx_bytes(131072, cache_type_kv = "f16")
single = b._compute_buffer_ctx_bytes(131072, cache_type_kv = ct)
split = b._compute_buffer_ctx_bytes(131072, cache_type_kv = ct, layer_split = True)
- delta = (LlamaCppBackend._CTX_COMPUTE_SPLIT_MULT - 1) * mask
+ delta = (LlamaCppBackend._CTX_COMPUTE_SPLIT_MULT - 1) * mask + _hidden_copies(embd)
assert split == pytest.approx(single + delta, rel = 1e-9)
@pytest.mark.parametrize("embd,mla", [(2048, None), (2560, None), (8192, None), (7168, 576)])
@@ -520,9 +659,7 @@ class TestContextBufferLayerSplit:
131072, cache_type_kv = "f16", layer_split = True
) == b._compute_buffer_ctx_bytes(131072, cache_type_kv = "f16")
measured = TestContextBufferDSV4._MEASURED_1M_GIB * 1024**3 / 1048576
- split_masks = (LlamaCppBackend._CTX_COMPUTE_SPLIT_MULT - 1) * (
- 512 * 2 * LlamaCppBackend._CTX_COMPUTE_F16_MASK_SAFETY
- )
+ split_masks = (LlamaCppBackend._CTX_COMPUTE_SPLIT_MULT - 1) * 512 * 2
assert LlamaCppBackend._DSV4_CTX_COMPUTE_BYTES_PER_TOK - measured >= split_masks
@@ -554,9 +691,7 @@ class TestContextBufferInklingSplit:
@pytest.mark.parametrize("ct", ["f16", None, "q8_0"])
def test_split_adds_exactly_the_extra_mask_copies(self, ct):
- delta = (LlamaCppBackend._CTX_COMPUTE_SPLIT_MULT - 1) * (
- self._UB * 2 * LlamaCppBackend._CTX_COMPUTE_F16_MASK_SAFETY
- )
+ delta = (LlamaCppBackend._CTX_COMPUTE_SPLIT_MULT - 1) * self._UB * 2
assert self._rate(ct, True) == pytest.approx(self._rate(ct, False) + delta, rel = 1e-9)
def test_dense_fallback_delta_is_present_but_tiny(self):
@@ -595,7 +730,9 @@ class TestLayerSplitWiring:
indent = len(src[start]) - len(src[start].lstrip())
body = [src[start]]
for line in src[start + 1 :]:
- if line.strip() and len(line) - len(line.lstrip()) <= indent:
+ # A one-per-line signature closes with ") -> int:" at the def's indent.
+ closes_signature = line.lstrip().startswith(")")
+ if line.strip() and len(line) - len(line.lstrip()) <= indent and not closes_signature:
break
body.append(line)
return "\n".join(body)
@@ -956,12 +1093,12 @@ class TestPerDeviceSplitReserve:
assert card1_usable < reserve_mib # would OOM card 1 at load
def test_subset_is_capped_not_rejected_when_a_card_cannot_hold_its_reserve(self):
- # 1762.72 MiB usable on card 1 holds 31488, not the pooled 262144. Rejecting
+ # 1762.72 MiB usable on card 1 holds 30720, not the pooled 262144. Rejecting
# the subset instead drops auto to the 4096 fallback for no reason.
b = self._fit_backend()
gpus, totals = self._HETEROGENEOUS
assert self._drive(b, gpus, totals, 20_480, 262144, cap = False) == (None, 0)
- assert self._drive(b, gpus, totals, 20_480, 262144) == ([0, 1], 31_488)
+ assert self._drive(b, gpus, totals, 20_480, 262144) == ([0, 1], 30_720)
def test_homogeneous_gpus_are_unaffected(self):
b = self._fit_backend()
@@ -1029,14 +1166,15 @@ class TestPerDeviceReserveCap:
reserve = lambda c: (
(self._OH + b._compute_buffer_ctx_bytes(c, self._UB, "f16", layer_split = True)) / MIB
)
- assert reserve(31_488) == 1762.0 <= usable
- assert reserve(31_744) == 1768.0 > usable
+ # 1024 MiB overhead + 256 MiB of hidden-state copies + 4 masks at 1/64 MiB a cell.
+ assert reserve(30_720) == 1760.0 <= usable
+ assert reserve(30_976) == 1764.0 > usable
def test_capped_context_still_fits_the_pooled_budget(self):
b = self._fit_backend()
gpus, totals = self._HETEROGENEOUS
idx, ctx = self._drive(b, gpus, totals, 20_480, 262144)
- assert idx == [0, 1] and ctx == 31_488
+ assert idx == [0, 1] and ctx == 30_720
pool = sum(g[1] - 0.03 * totals[g[0]] for g in gpus)
cc = 2 * b._compute_buffer_ctx_bytes(ctx, self._UB, "f16", layer_split = True)
footprint = (20_480 * MIB + self._OH + b._estimate_kv_cache_bytes(ctx) + cc) / MIB
@@ -1087,7 +1225,7 @@ class TestPerDeviceReserveCap:
totals = {0: 49_152, 1: 24_576, 2: 24_576}
assert self._drive(b, gpus, totals, model_mib, 262144, cap = False) == (None, 0)
idx, ctx = self._drive(b, gpus, totals, model_mib, 262144)
- assert idx == [0, 1] and ctx == 31_488
+ assert idx == [0, 1] and ctx == 30_720
def test_helper_edges(self):
cap = LlamaCppBackend._cap_ctx_to_per_device_reserve
@@ -1248,7 +1386,7 @@ class TestSplitRateRecheckAfterSelection:
for node in ast.walk(ast.parse(textwrap.dedent(source)))
if isinstance(node, ast.Call)
for keyword in node.keywords
- if keyword.arg == "split_extra_bytes"
+ if keyword.arg in ("split_extra_bytes", "split_extra_for_slots")
]
# Projector floor pin, explicit-context pin, reduced-slot retry, per-candidate
# re-fit. A fifth has to come here and say which context it prices at.
@@ -1261,14 +1399,20 @@ class TestSplitRateRecheckAfterSelection:
assert len(wired) == 4, wired
# Each passes the step at a context of its own, so none is exempt and none
# hardcodes one: `_cc_split_extra(4096)` would not match.
+ # The reduced-slot search re-prices the step for each candidate slot count.
for expression in wired:
- assert re.fullmatch(r"_cc_split_extra\(\w+\)", expression), expression
+ assert re.fullmatch(
+ r"_cc_split_extra\(\w+\)|lambda s: _cc_split_extra\(\w+, s\)", expression
+ ), expression
assert "gpu_indices,use_fit=self._select_gpus_split_aware(" in load
# The step rides _cc_bytes' pipelining gate, so it is 0 when llama.cpp declines.
- assert "returnmax(0,_cc_bytes(ctx,2)//2-_cc_bytes(ctx))" in load
+ assert "returnmax(0,_cc_bytes(ctx,2,slots)//2-_cc_bytes(ctx,1,slots))" in load
slots = "".join(inspect.getsource(LlamaCppBackend._slots_that_fit_on_gpu).split())
assert "self._select_gpus_split_aware(" in slots
- assert "split_extra_bytes=split_extra_bytes," in slots
+ assert (
+ "split_extra_bytes=(split_extra_for_slots(slots)ifsplit_extra_for_slotselsesplit_extra_bytes),"
+ in slots
+ )
# ── The scratch rate keys off the LIGHTER axis ───────────────────────────────
@@ -1370,8 +1514,9 @@ class TestTensorFitPricesTheQuantizedAxis:
axis alone the planner advertises the full 262144; the quantized axis
cannot hold it."""
b = _backend_from_gguf_local()
- optimistic = self._plan(b, "f16", None, 2048)[0]
- honest = self._plan(b, "f16", "q4_0", 2048)[0]
+ # 36 GB cards, so the KV cache rather than the native length bounds the context.
+ optimistic = self._plan(b, "f16", None, 2048, per_gpu = 36_000)[0]
+ honest = self._plan(b, "f16", "q4_0", 2048, per_gpu = 36_000)[0]
assert honest < optimistic, (honest, optimistic)
# Still a real context, not the 2048 floor: the fix must not collapse the
@@ -1381,7 +1526,7 @@ class TestTensorFitPricesTheQuantizedAxis:
# pinning: the asymmetric pair pays f16 KV bytes on both axes (the heavier
# axis budgets storage) AND the full quantized dequant scratch. Both terms
# conservative is the point; neither one alone describes this launch.
- assert honest < self._plan(b, "q4_0", "q4_0", 2048)[0]
+ assert honest < self._plan(b, "q4_0", "q4_0", 2048, per_gpu = 36_000)[0]
def test_a_symmetric_request_is_unchanged(self):
"""Default and explicit scratch type agree when both axes match, so no
@@ -1390,3 +1535,171 @@ class TestTensorFitPricesTheQuantizedAxis:
for kv in ("f16", "q8_0", "q4_0"):
for ub in (512, 2048):
assert self._plan(b, kv, None, ub) == self._plan(b, kv, kv, ub)
+
+
+class TestSpeculativeComputeBuffers:
+ """Draft buffers must cover growth beyond the 224 MiB floor.
+ Measurements below are from Qwen3.8 27B's embedded head."""
+
+ def _qwen38(self):
+ b = _backend(vocab = 248320, embd = 5120)
+ b._feed_forward_length = 17408
+ return b
+
+ @pytest.mark.parametrize(
+ "ub,ctx,measured", [(512, 32768, 130.02), (1120, 32768, 284.41), (1120, 131072, 428.78)]
+ )
+ def test_floor_plus_growth_covers_the_draft_context(self, ub, ctx, measured):
+ b = self._qwen38()
+ charged = (
+ LlamaCppBackend._MTP_DRAFT_COMPUTE_BYTES + b._mtp_draft_compute_bytes(ctx, n_ubatch = ub)
+ ) / MIB
+ assert charged >= measured
+
+ def test_short_context_and_micro_batch_stay_on_the_floor(self):
+ assert self._qwen38()._mtp_draft_compute_bytes(4096, n_ubatch = 256) == 0
+
+ def test_growth_follows_the_mask_once_past_the_floor(self):
+ b = self._qwen38()
+ lo = b._mtp_draft_compute_bytes(131072, n_ubatch = 1120)
+ hi = b._mtp_draft_compute_bytes(262144, n_ubatch = 1120)
+ assert hi - lo == 1120 * 2 * 131072
+
+ def test_verify_rows_scale_with_slots_and_depth(self):
+ b = self._qwen38()
+ base = b._mtp_draft_compute_bytes(4096, n_ubatch = 256)
+ rows = b._mtp_draft_compute_bytes(4096, n_ubatch = 256, n_parallel = 4, spec_draft_n_max = 2)
+ assert rows - base == 4 * 2 * LlamaCppBackend._SPEC_VERIFY_ROW_COPIES * 248320 * 4
+
+ def test_the_reserve_carries_it(self):
+ b = self._qwen38()
+ b._n_layers, b._n_kv_heads, b._kv_key_length, b._kv_value_length = 65, 4, 256, 256
+ b._nextn_predict_layers = 1
+ kwargs = dict(n_parallel = 1, target_rollback = False, mtp_keeps_target_ctx = False)
+ short = b._estimate_mtp_overhead_bytes(4096, n_ubatch = 256, **kwargs)
+ long = b._estimate_mtp_overhead_bytes(262144, n_ubatch = 1120, **kwargs)
+ draft_kv = b._mtp_draft_kv_bytes(262144, n_ubatch = 1120) - b._mtp_draft_kv_bytes(
+ 4096, n_ubatch = 256
+ )
+ assert long - short == draft_kv + b._mtp_draft_compute_bytes(262144, n_ubatch = 1120)
+
+
+class TestFlashAttentionOff:
+ """Without flash attention llama.cpp materialises [n_kv, n_ubatch, n_head] float32
+ attention scores and builds the KQ mask as float32. Measured on b10909 (RTX 6000
+ Ada), f16 cache, llama-server's CUDA0 compute buffer in MiB."""
+
+ _MEASURED = [
+ ("Llama-3.2-1B", 512, 4, 32768, 2130.01),
+ ("Llama-3.2-1B", 2048, 4, 32768, 8520.02),
+ ("Llama-3.2-1B", 512, 4, 65536, 4242.01),
+ ("Qwen3-8B", 1120, 4, 32768, 4698.76),
+ ("Qwen3.5-4B", 512, 4, 65536, 2214.01),
+ ("Qwen3.6-35B-A3B", 2048, 4, 32768, 4504.18),
+ ("Gemma-4-12B", 1120, 4, 32768, 2540.8),
+ ("Gemma-4-12B", 512, 4, 65536, 2248.01),
+ ("Gemma-4-E4B", 512, 4, 65536, 1217.51),
+ ("GLM-4.7-Flash", 1120, 4, 32768, 3086.3),
+ ("GLM-4.7-Flash", 512, 4, 65536, 2818.9),
+ ]
+
+ @pytest.mark.parametrize("name,ub,slots,ctx,measured", _MEASURED)
+ def test_covers_the_measured_buffer(self, name, ub, slots, ctx, measured):
+ b = _measured(name)
+ est = (
+ b._estimate_compute_buffer_bytes(n_ubatch = ub, n_parallel = slots)
+ + b._compute_buffer_ctx_bytes(ctx, ub, "f16", flash_attn = False)
+ ) / MIB
+ assert measured <= est <= measured * 1.1, f"{name}: {est:.0f} vs {measured}"
+
+ def test_the_rate_is_the_score_matrix_plus_a_float32_mask(self):
+ b = _measured("Gemma-4-12B")
+ on = b._compute_buffer_ctx_bytes(100000, 1120, "f16")
+ off = b._compute_buffer_ctx_bytes(100000, 1120, "f16", flash_attn = False)
+ assert on == 1120 * 2 * 100000
+ assert off == 1120 * (4 + 16 * 4) * 100000
+
+
+class TestLayerSplitSlidingWindowSlots:
+ """The replicated sliding-window masks of a layer split hold the window for every
+ slot, as the single-device mask does. Measured per GPU across an RTX 6000 Ada and an
+ RTX 3090, Gemma 4 12B at ubatch 1120."""
+
+ @pytest.mark.parametrize("slots,ctx,measured", [(8, 65536, 971), (16, 131072, 1601)])
+ def test_covers_the_measured_split(self, slots, ctx, measured):
+ b = _measured("Gemma-4-12B")
+ per_device = (
+ b._estimate_compute_buffer_bytes(n_ubatch = 1120, n_parallel = slots)
+ + b._compute_buffer_ctx_bytes(ctx, 1120, "f16", layer_split = True, n_parallel = slots)
+ ) / MIB
+ one_slot_window = (
+ b._estimate_compute_buffer_bytes(n_ubatch = 1120, n_parallel = slots)
+ + b._compute_buffer_ctx_bytes(ctx, 1120, "f16", layer_split = True)
+ ) / MIB
+ assert one_slot_window < measured <= per_device <= measured * 1.1
+
+
+class TestOutputRowsByBuild:
+ """Before ggml-org/llama.cpp#23861 (b9460) a context reserved an output row for every
+ micro-batch token, once, whatever the slot count: on Vulkan, Gemma 4 E2B at ubatch
+ 512 reserved 515 MiB on b9415 and 137 MiB on b10909, the same at one slot or four."""
+
+ @pytest.mark.parametrize(
+ "output,build",
+ [
+ ("version: 0.4.0-dev (build 10909, commit 329b6160f)\nbuilt with GNU", 10909),
+ ("version: 9415 (06d26dfdf)\nbuilt with GNU 11.4.0", 9415),
+ ("version: 1 (unknown)\n", None),
+ ("no version here", None),
+ ],
+ )
+ def test_the_build_is_read_from_either_version_format(
+ self, tmp_path, monkeypatch, output, build
+ ):
+ import subprocess
+ import types
+
+ binary = tmp_path / "llama-server"
+ binary.write_bytes(b"")
+ monkeypatch.setattr(LlamaCppBackend, "_build_number_cache", {})
+ monkeypatch.setattr(
+ subprocess,
+ "run",
+ lambda *a, **k: types.SimpleNamespace(stdout = output, stderr = "", returncode = 0),
+ )
+ assert LlamaCppBackend.probe_build_number(str(binary)) == build
+
+ @pytest.mark.parametrize(
+ "build,legacy", [(9415, True), (9459, True), (9460, False), (None, False)]
+ )
+ def test_only_a_known_older_build_reserves_micro_batch_rows(self, monkeypatch, build, legacy):
+ monkeypatch.setattr(
+ LlamaCppBackend, "probe_build_number", classmethod(lambda cls, binary = None: build)
+ )
+ assert LlamaCppBackend.reserves_micro_batch_outputs() is legacy
+
+ def test_an_older_build_is_charged_a_row_per_micro_batch_token(self):
+ # Measured b9415 Vulkan compute buffers (MiB) at 32768 cells, one and four slots.
+ cases = [
+ (dict(vocab = 128256, embd = 2048, ff = 8192), 512, 254.50),
+ (dict(vocab = 151936, embd = 4096, ff = 12288), 2048, 1667.05),
+ (dict(vocab = 262144, embd = 1536, ff = 12288, ple = 256 * 35), 2048, 2704.08),
+ ]
+ for dims, ub, measured in cases:
+ b = _backend(vocab = dims["vocab"], embd = dims["embd"])
+ b._feed_forward_length = dims["ff"]
+ b._embedding_length_per_layer_input = 0
+ b._n_layers = 1
+ if "ple" in dims:
+ b._embedding_length_per_layer_input, b._n_layers = 256, 35
+ b._reserves_micro_batch_outputs = True
+ for slots in (1, 4):
+ est = (
+ b._estimate_compute_buffer_bytes(n_ubatch = ub, n_parallel = slots)
+ + b._compute_buffer_ctx_bytes(32768, ub, "f16")
+ ) / MIB
+ assert measured <= est <= measured * 1.5, (dims, slots, est)
+ b._reserves_micro_batch_outputs = False
+ assert b._estimate_compute_buffer_bytes(n_ubatch = ub, n_parallel = 1) < (
+ dims["vocab"] * ub * 4
+ )
diff --git a/studio/backend/tests/test_cuda_sm_gate.py b/studio/backend/tests/test_cuda_sm_gate.py
index ae586feafe8..eab730014dc 100644
--- a/studio/backend/tests/test_cuda_sm_gate.py
+++ b/studio/backend/tests/test_cuda_sm_gate.py
@@ -217,6 +217,10 @@ def _gated_backend(
monkeypatch.setattr(
LlamaCppBackend, "_is_unsloth_managed_binary", staticmethod(lambda _binary: True)
)
+ # The build probe runs `--version`, which the Popen patch below would count as a launch.
+ monkeypatch.setattr(
+ LlamaCppBackend, "probe_build_number", classmethod(lambda cls, binary = None: None)
+ )
backend = LlamaCppBackend()
backend._get_gpu_memory = lambda _binary = None, **_kw: []
backend._get_gpu_free_memory = lambda _binary = None, **_kw: []
diff --git a/studio/backend/tests/test_estimator_flash_attn_parity.py b/studio/backend/tests/test_estimator_flash_attn_parity.py
index d9c32b525b2..0d6be183208 100644
--- a/studio/backend/tests/test_estimator_flash_attn_parity.py
+++ b/studio/backend/tests/test_estimator_flash_attn_parity.py
@@ -336,3 +336,50 @@ def test_pass_through_adapters_are_charged_and_follow_the_base_placement(
assert (
missing is not None and missing.adapters_unsized
), "an unsizable adapter has to mark the total a floor rather than vanish"
+
+
+def _runtime(path, extras = None):
+ runtime = ri._gguf_runtime_bytes(path, 32768, extras, 1, None, False)
+ return runtime.kv_bytes, runtime.compute_bytes
+
+
+def _flash_attn_caps(monkeypatch, supported):
+ monkeypatch.setattr(
+ ri.LlamaCppBackend,
+ "probe_server_capabilities",
+ classmethod(lambda cls, binary = None: {"found": True, "supports_flash_attn": supported}),
+ )
+
+
+def test_flash_attention_follows_the_launch_argv_not_the_inherited_env(
+ monkeypatch, qwen3_shaped_gguf
+):
+ """The managed --flash-attn on is parsed after LLAMA_ARG_FLASH_ATTN, so only the
+ extras can turn it off; a build without the flag drops both flag and env."""
+ monkeypatch.delenv("LLAMA_ARG_FLASH_ATTN", raising = False)
+ _flash_attn_caps(monkeypatch, True)
+ on = _runtime(qwen3_shaped_gguf)
+ off = _runtime(qwen3_shaped_gguf, ["--flash-attn", "off"])
+ assert off[1] > on[1]
+
+ monkeypatch.setenv("LLAMA_ARG_FLASH_ATTN", "off")
+ assert _runtime(qwen3_shaped_gguf) == on
+
+ monkeypatch.setenv("LLAMA_ARG_FLASH_ATTN", "on")
+ _flash_attn_caps(monkeypatch, False)
+ assert _runtime(qwen3_shaped_gguf) == off
+
+
+def test_grok_is_priced_without_flash_attention(monkeypatch, qwen3_shaped_gguf):
+ """llama.cpp forces flash attention off for Grok whatever the launch asks for."""
+ monkeypatch.delenv("LLAMA_ARG_FLASH_ATTN", raising = False)
+ _flash_attn_caps(monkeypatch, True)
+ off = _runtime(qwen3_shaped_gguf, ["--flash-attn", "off"])
+ real_read = ri.LlamaCppBackend._read_gguf_metadata
+
+ def read_as_grok(self, path):
+ real_read(self, path)
+ self._architecture = "grok"
+
+ monkeypatch.setattr(ri.LlamaCppBackend, "_read_gguf_metadata", read_as_grok)
+ assert _runtime(qwen3_shaped_gguf) == off
diff --git a/studio/backend/tests/test_kv_cache_estimation.py b/studio/backend/tests/test_kv_cache_estimation.py
index f62bcf7207c..871b027966e 100644
--- a/studio/backend/tests/test_kv_cache_estimation.py
+++ b/studio/backend/tests/test_kv_cache_estimation.py
@@ -207,6 +207,12 @@ class TestGGUFParserNewFields:
("_ssm_state_size", "ssm.state_size", 128),
("_ssm_group_count", "ssm.group_count", 16),
("_ssm_conv_kernel", "ssm.conv_kernel", 4),
+ ("_feed_forward_length", "feed_forward_length", 12288),
+ ("_expert_used_count", "expert_used_count", 8),
+ ("_expert_feed_forward_length", "expert_feed_forward_length", 512),
+ ("_expert_shared_feed_forward_length", "expert_shared_feed_forward_length", 512),
+ ("_expert_shared_count", "expert_shared_count", 1),
+ ("_embedding_length_per_layer_input", "embedding_length_per_layer_input", 256),
],
)
def test_field_parsed(self, field, gguf_key, value):
@@ -256,6 +262,14 @@ class TestGGUFParserNewFields:
assert b._n_kv_heads == 8
assert b._sliding_window_pattern == [True, True, True, True, True, False]
+ def test_per_layer_feed_forward_length_keeps_the_widest(self):
+ # Gemma 4 E2B stores one FFN width per layer; the compute buffer is set by
+ # the widest.
+ b = _backend_from_gguf(
+ "gemma4", {"block_count": 3, "feed_forward_length": [6144, 12288, 6144]}
+ )
+ assert b._feed_forward_length == 12288
+
class TestArchSwaPatternDefaults:
"""Bootstrap arch table fires when GGUF reports `sliding_window` but
diff --git a/studio/backend/tests/test_llama_cpp_placement.py b/studio/backend/tests/test_llama_cpp_placement.py
index 05371d102b7..b1cb2ae41e5 100644
--- a/studio/backend/tests/test_llama_cpp_placement.py
+++ b/studio/backend/tests/test_llama_cpp_placement.py
@@ -110,6 +110,32 @@ def _backend(tmp_path: Path, *, vulkan: bool, memory):
return backend, gguf
+# A synthetic compute buffer for the slot-reduction planner fixtures, per micro-batch
+# token: a fixed base, plus a fixed cost for every slot past the first.
+SLOT_COMPUTE_BASE_BYTES_PER_TOKEN = 92 * 1024
+SLOT_COMPUTE_EXTRA_SLOT_BYTES_PER_TOKEN = 1116 * 1024
+
+
+def _install_slot_scaled_compute(backend):
+ """Use a synthetic per-slot cost to exercise slot reduction on dense fixtures."""
+
+ def compute(
+ *,
+ n_ubatch = None,
+ n_parallel = 1,
+ **_kwargs,
+ ):
+ ub = max(1, int(backend._DEFAULT_N_UBATCH if n_ubatch is None else n_ubatch))
+ extra_slots = max(0, int(n_parallel) - 1)
+ return ub * (
+ SLOT_COMPUTE_BASE_BYTES_PER_TOKEN
+ + extra_slots * SLOT_COMPUTE_EXTRA_SLOT_BYTES_PER_TOKEN
+ )
+
+ backend._estimate_compute_buffer_bytes = compute
+ return backend
+
+
def _backend_non_vulkan(
*args,
vulkan = False,
@@ -650,11 +676,7 @@ def test_auto_classifies_placement_on_the_device_flags_the_child_gets(tmp_path):
def _hybrid_reserve_backend(tmp_path: Path, *, caps = None):
- """A Hybrid Mamba target on one 24 GB card with the MTP-overhead math live.
-
- The drafter's own KV is stubbed away so the only moving term is the target's
- recurrent rollback state, which is what the reserve has to keep charging.
- """
+ """Hybrid Mamba on a 24 GB card, with only target rollback state affecting MTP cost."""
gb = 1024**3
backend, gguf = _backend(tmp_path, vulkan = False, memory = [(0, 24_576, 24_576)])
sidecar = tmp_path / "dflash-model-Q8_0.gguf"
@@ -664,6 +686,7 @@ def _hybrid_reserve_backend(tmp_path: Path, *, caps = None):
backend._compute_buffer_ctx_bytes = lambda *args, **kwargs: 0
backend._estimate_compute_buffer_bytes = lambda **kwargs: 1
backend._mtp_draft_kv_bytes = lambda *args, **kwargs: 0
+ backend._mtp_draft_compute_bytes = lambda *args, **kwargs: 0
backend._select_gpus = lambda *args, **kwargs: ([0], False)
backend._select_gpus_split_aware = lambda *args, **kwargs: ([0], False)
@@ -742,9 +765,9 @@ def _recorded_mtp_reserve_and_callbacks(backend, gguf, **load_kwargs):
def test_a_cpu_pinned_drafter_still_pays_the_hybrid_target_rollback(tmp_path):
# -ngld 0 moves the drafter's weights and KV to host memory, but the rollback
- # snapshots live in the TARGET context, so they stay on the GPU. Releasing the
- # whole reserve here undercounts them and the fit can pick a placement that
- # spills.
+ # snapshots and verification output rows live in the TARGET context, so they stay
+ # on the GPU. Releasing the whole reserve here undercounts them and the fit can
+ # pick a placement that spills.
backend, gguf, sidecar = _hybrid_reserve_backend(tmp_path)
charged = _recorded_mtp_reserve_std(
@@ -756,9 +779,9 @@ def test_a_cpu_pinned_drafter_still_pays_the_hybrid_target_rollback(tmp_path):
)
# After the launch: the GGUF dims land when the load reads the metadata.
- expected = backend._mamba_recurrent_state_bytes(n_parallel = 4) * 2
- assert expected > 0
- assert set(charged) == {expected}
+ rollback = backend._mamba_recurrent_state_bytes(n_parallel = 4) * 2
+ assert rollback > 0
+ assert set(charged) == {rollback + backend._spec_verify_rows_bytes(4, 2)}
def test_the_cpu_drafter_reserve_still_reprices_per_slot_candidate(tmp_path):
@@ -783,6 +806,7 @@ def test_the_cpu_drafter_reserve_still_reprices_per_slot_candidate(tmp_path):
for slots in (1, 2, 4):
assert fn(8192, _np = slots, _n_ubatch = 512) == (
backend._mamba_recurrent_state_bytes(n_parallel = slots) * 2
+ + backend._spec_verify_rows_bytes(slots, 2)
)
# Per-slot state, not per-token: context does not move it.
assert fn(2048, _np = 4, _n_ubatch = 512) == fn(131072, _np = 4, _n_ubatch = 512)
@@ -1125,10 +1149,10 @@ def test_a_busy_second_gpu_does_not_condemn_a_drafter_the_first_one_holds(tmp_pa
def test_a_cpu_offloaded_sidecar_releases_the_byte_accurate_reserve(tmp_path):
"""-ngld 0 puts the drafter in host memory, and a separate sidecar displaces
- the embedded head that mtp_overhead_fn was sized from, so nothing speculative
- is GPU-resident. The flat fraction already stands down here; the byte-accurate
- callback did not, so the fit went on charging GPU bytes for a drafter that
- allocates none, cutting the context or taking --fit for them.
+ the embedded head that mtp_overhead_fn was sized from, so only the target's
+ verification rows stay GPU-resident. The flat fraction already stands down here;
+ the byte-accurate callback did not, so the fit went on charging GPU bytes for a
+ drafter that allocates none, cutting the context or taking --fit for them.
"""
backend, gguf, sidecar = _tight_vram_backend(tmp_path, drafter_gb = 12.0)
backend._nextn_predict_layers = 1
@@ -1156,7 +1180,8 @@ def test_a_cpu_offloaded_sidecar_releases_the_byte_accurate_reserve(tmp_path):
)
assert charged, "the fit never ran, so this proves nothing"
- assert set(charged) == {0}
+ # One slot at the three-token DSpark depth.
+ assert set(charged) == {backend._spec_verify_rows_bytes(1, 3)}
def test_an_mla_model_keeps_the_reason_that_actually_dropped_its_drafter(tmp_path):
@@ -3588,3 +3613,86 @@ def test_a_drafter_carrying_its_own_embeddings_still_reaches_the_command(tmp_pat
assert cmd[cmd.index("--model-draft") + 1] == str(drafter)
assert backend.mtp_draft_path == str(drafter)
assert backend.mtp_draft_suppressed_path is None
+
+
+def _recording_compute_backend(
+ tmp_path,
+ monkeypatch,
+ *,
+ build = 10909,
+):
+ """A dense backend on one 24 GB card whose compute terms are the real estimator,
+ recording what the loader asks of them."""
+ backend, gguf = _backend(tmp_path, vulkan = False, memory = [(0, 24_576, 24_576)])
+
+ def read(_path):
+ backend._architecture = "qwen3"
+ backend._vocab_size = 151936
+ backend._embedding_length = 4096
+ backend._feed_forward_length = 12288
+ backend._n_layers = 36
+ backend._n_heads = 32
+ backend._n_kv_heads = 8
+ backend._kv_key_length = 128
+ backend._kv_value_length = 128
+ backend._context_length = 40960
+
+ backend._read_gguf_metadata = read
+ backend._get_gguf_size_bytes = lambda _path: 4 * 1024**3
+ del backend._can_estimate_kv # the real one, now that the dims are set
+ backend.probe_server_capabilities = lambda _binary = None: {
+ "supports_kv_unified": True,
+ "supports_flash_attn": True,
+ "flash_attn_takes_value": True,
+ }
+ monkeypatch.setattr(
+ LlamaCppBackend, "probe_build_number", classmethod(lambda cls, binary = None: build)
+ )
+ calls = {"ctx": [], "flat": [], "kv": []}
+ real_ctx = backend._compute_buffer_ctx_bytes
+ real_flat = backend._estimate_compute_buffer_bytes
+ real_kv = backend._estimate_kv_cache_bytes
+
+ def ctx(*args, **kwargs):
+ calls["ctx"].append(kwargs.get("flash_attn", True))
+ return real_ctx(*args, **kwargs)
+
+ def flat(**kwargs):
+ calls["flat"].append(backend._reserves_micro_batch_outputs)
+ return real_flat(**kwargs)
+
+ def kv(*args, **kwargs):
+ calls["kv"].append(kwargs.get("flash_attn", True))
+ return real_kv(*args, **kwargs)
+
+ backend._compute_buffer_ctx_bytes = ctx
+ backend._estimate_compute_buffer_bytes = flat
+ backend._estimate_kv_cache_bytes = kv
+ return backend, gguf, calls
+
+
+ "extra_args,expected",
+ [([], True), (["--flash-attn", "off"], False), (["-fa", "off", "--flash-attn", "on"], True)],
+)
+def test_the_loader_prices_the_attention_mode_it_launches(
+ tmp_path, monkeypatch, extra_args, expected
+):
+ """Compute follows the launch's attention mode; KV covers the no-flash retry."""
+ backend, gguf, calls = _recording_compute_backend(tmp_path, monkeypatch)
+
+ assert _launch(backend, gguf, n_ctx = 0, n_parallel = 4, extra_args = extra_args)["cmd"]
+
+ assert calls["ctx"], "the fit never priced the context term"
+ assert set(calls["ctx"]) == {expected}
+ assert False in calls["kv"]
+
+
[email protected]("build,expected", [(9415, True), (10909, False), (None, False)])
+def test_the_loader_prices_the_output_rows_of_its_build(tmp_path, monkeypatch, build, expected):
+ backend, gguf, calls = _recording_compute_backend(tmp_path, monkeypatch, build = build)
+
+ assert _launch(backend, gguf, n_ctx = 0, n_parallel = 4)["cmd"]
+
+ assert calls["flat"], "the fit never priced the flat compute buffer"
+ assert set(calls["flat"]) == {expected}
diff --git a/studio/backend/tests/test_load_mode_fit_matrix.py b/studio/backend/tests/test_load_mode_fit_matrix.py
index 385aeb4752c..337f1acbbb7 100644
--- a/studio/backend/tests/test_load_mode_fit_matrix.py
+++ b/studio/backend/tests/test_load_mode_fit_matrix.py
@@ -1480,14 +1480,8 @@ def test_a_cpu_only_kv_cache_is_not_charged_twice(monkeypatch):
def test_a_tensor_split_charges_the_replicated_compute_buffer():
- """Checked at the source: reaching this call needs a real multi-GPU probe.
-
- _plan_tensor_parallel admits devices against, and reserves per device,
- _estimate_compute_buffer_bytes(per_device_tensor = True). The fit has to charge
- the same thing: the layer-mode lump is ONE device's buffer at the smaller rate,
- and understating it on a pooled multi-GPU credit is the direction that claims a
- fit that is not there.
- """
+ """Check that the fit charges the same per-device buffer as _plan_tensor_parallel.
+ This source check avoids requiring a multi-GPU probe."""
from core.inference.llama_cpp import LlamaCppBackend as B
import inspect
@@ -1508,26 +1502,18 @@ def test_a_tensor_split_charges_the_replicated_compute_buffer():
) in compact
-def test_the_tensor_rate_really_exceeds_the_layer_one():
- """The under-charge the fix closes is not a rounding difference: at one slot
- the layer figure drops the vocab-width output buffer entirely."""
-
- class _Dims:
- # is_embedding_gguf is a read-only property on the real backend.
- _vocab_size = 151936
- _embedding_length = 5120
- is_embedding_gguf = False
- _DEFAULT_N_UBATCH = LlamaCppBackend._DEFAULT_N_UBATCH
- _COMPUTE_BUFFER_SAFETY = LlamaCppBackend._COMPUTE_BUFFER_SAFETY
-
- layer = LlamaCppBackend._estimate_compute_buffer_bytes(
- _Dims(), n_ubatch = 512, n_parallel = 1, per_device_tensor = False
- )
- tensor = LlamaCppBackend._estimate_compute_buffer_bytes(
- _Dims(), n_ubatch = 512, n_parallel = 1, per_device_tensor = True
- )
- assert tensor > layer
- # 4-GPU tensor split: what the fit used to charge vs what it now does.
+def test_a_tensor_split_pays_the_buffer_on_every_device():
+ """Each device of a tensor split reserves the whole single-GPU compute buffer
+ (Qwen3 8B across two GPUs: 128 MiB on each at ubatch 512, 512 MiB at 2048), so
+ the fit's per-device multiplication is the whole difference from a layer lump."""
+ b = LlamaCppBackend()
+ b._vocab_size = 151936
+ b._embedding_length = 4096
+ b._feed_forward_length = 12288
+ layer = b._estimate_compute_buffer_bytes(n_ubatch = 2048, n_parallel = 1)
+ tensor = b._estimate_compute_buffer_bytes(n_ubatch = 2048, n_parallel = 1, per_device_tensor = True)
+ assert tensor == layer
+ # 4-GPU tensor split: three more copies than the layer lump.
assert 4 * tensor - layer > GIB
diff --git a/studio/backend/tests/test_memory_estimate.py b/studio/backend/tests/test_memory_estimate.py
index 4b957320c24..ade01122918 100644
--- a/studio/backend/tests/test_memory_estimate.py
+++ b/studio/backend/tests/test_memory_estimate.py
@@ -1383,6 +1383,10 @@ class TestDrafterAccounting:
return 1.0 + (0.0 if pinned else drafter_bytes / 1024**3)
monkeypatch.setattr(ri, "_gguf_resident_file_gb", _files)
+ verify_rows = 7 * 1024 * 1024
+ monkeypatch.setattr(
+ ri.LlamaCppBackend, "_spec_verify_rows_bytes", lambda *a, **k: verify_rows
+ )
on_gpu = ri._gguf_memory_breakdown(config, gqa_gguf, n_ctx = 8192)
assert on_gpu.drafter_runtime_bytes > 0
# Default placement: all of it, so the row says nothing rather than guessing.
@@ -1391,7 +1395,8 @@ class TestDrafterAccounting:
on_cpu = ri._gguf_memory_breakdown(
config, gqa_gguf, n_ctx = 8192, llama_extra_args = ["--spec-draft-ngl", "0"]
)
- assert on_cpu.drafter_runtime_gpu_bytes == 0
+ # Only the target's verification rows stay with the GPU target.
+ assert on_cpu.drafter_runtime_gpu_bytes == verify_rows
# --no-kv-offload moves the TARGET cache and leaves the drafter alone. A
# boolean read off kv_on_gpu would call the whole term host-resident here,
@@ -1687,7 +1692,12 @@ class TestSpeculativeModeTerms:
probe._read_gguf_metadata(mla)
return probe._estimate_kv_cache_bytes(n_ctx, "f16")
- def test_a_dspark_drafter_is_not_charged_the_targets_context(self, mla, tmp_path, priced_files):
+ def test_a_dspark_drafter_is_not_charged_the_targets_context(
+ self, mla, tmp_path, priced_files, monkeypatch
+ ):
+ # The two drafters' compute buffers differ with their own headers; this pins
+ # the cache terms.
+ monkeypatch.setattr(ri.LlamaCppBackend, "_mtp_draft_compute_bytes", lambda *a, **k: 0)
ctx = 131072
mtp = ri._gguf_memory_breakdown(self._config(mla, tmp_path, "mtp"), mla, n_ctx = ctx)
dspark = ri._gguf_memory_breakdown(self._config(mla, tmp_path, "dspark"), mla, n_ctx = ctx)
@@ -1699,11 +1709,13 @@ class TestSpeculativeModeTerms:
assert dspark.total_bytes < mtp.total_bytes
def test_a_gpu_drafter_is_charged_when_the_target_keeps_no_layers(
- self, mla, tmp_path, priced_files
+ self, mla, tmp_path, priced_files, monkeypatch
):
# A separate drafter does not inherit --gpu-layers: llama.cpp overwrites it
# with the draft placement, whose default is auto. At --gpu-layers 0 the
- # drafter is still on the GPU and still holds its cache there.
+ # drafter is still on the GPU and still holds its cache there. Verification
+ # rows follow the target and are pinned in their own test.
+ monkeypatch.setattr(ri.LlamaCppBackend, "_spec_verify_rows_bytes", lambda *a, **k: 0)
config = self._config(mla, tmp_path, "dspark")
manual = dict(gpu_memory_mode = "manual", gpu_layers = 0, n_ctx = 131072)
on_gpu = ri._gguf_memory_breakdown(config, mla, **manual)
@@ -1719,8 +1731,9 @@ class TestSpeculativeModeTerms:
)
def test_the_target_side_of_the_reserve_stays_with_the_target(
- self, mla, tmp_path, priced_files
+ self, mla, tmp_path, priced_files, monkeypatch
):
+ monkeypatch.setattr(ri.LlamaCppBackend, "_spec_verify_rows_bytes", lambda *a, **k: 0)
ctx = 131072
config = self._config(mla, tmp_path, "mtp")
copy_bytes = self._target_ctx_copy(mla, ctx)
@@ -1750,6 +1763,25 @@ class TestSpeculativeModeTerms:
# the loader reserves for the same launch.
assert pinned.gpu_bytes - no_layers_pinned.gpu_bytes > copy_bytes
+ def test_verification_rows_follow_the_targets_layers(
+ self, mla, tmp_path, priced_files, monkeypatch
+ ):
+ """They are target compute: on the GPU with the target's layers, whichever
+ device the drafter uses."""
+ verify_rows = 7 * 1024 * 1024
+ monkeypatch.setattr(
+ ri.LlamaCppBackend, "_spec_verify_rows_bytes", lambda *a, **k: verify_rows
+ )
+ config = self._config(mla, tmp_path, "dspark")
+ pin = ["--spec-draft-ngl", "0"]
+ no_layers = dict(gpu_memory_mode = "manual", gpu_layers = 0)
+ target_gpu = ri._gguf_memory_breakdown(config, mla, n_ctx = 8192, llama_extra_args = pin)
+ target_cpu = ri._gguf_memory_breakdown(config, mla, n_ctx = 8192, **no_layers)
+ assert target_gpu.drafter_runtime_gpu_bytes == verify_rows
+ assert target_cpu.drafter_runtime_gpu_bytes == (
+ target_cpu.drafter_runtime_bytes - verify_rows
+ )
+
def test_extras_owning_the_spec_block_price_the_builds_depth(self, mla):
# _build_speculative_flags returns without emitting a depth once the extras
# name --spec-type, so neither the panel field nor Studio's 2/3 platform
@@ -1854,9 +1886,15 @@ class TestLaunchShapedPricing:
)
assert full.kv_checkpoint_bytes == 0
assert windowed.kv_checkpoint_bytes > 0
- # Drafter runtime follows the cache proper; its checkpoint share is not estimated.
- assert full.drafter_runtime_bytes == full.kv_bytes - full.kv_checkpoint_bytes
- assert windowed.drafter_runtime_bytes == windowed.kv_bytes - windowed.kv_checkpoint_bytes
+ # Same header on both sides, so the drafter's cache moves exactly as the target's
+ # GPU cache does. By DIFFERENCE, not equality: the draft decode graph's floor
+ # (_MTP_DRAFT_COMPUTE_BYTES) now rides on top of this figure and does not follow
+ # --swa-full, so the two sides no longer match outright.
+ assert full.drafter_runtime_bytes - windowed.drafter_runtime_bytes == (
+ (full.kv_bytes - full.kv_checkpoint_bytes)
+ - (windowed.kv_bytes - windowed.kv_checkpoint_bytes)
+ )
+ assert windowed.drafter_runtime_bytes > windowed.kv_bytes - windowed.kv_checkpoint_bytes
def test_a_cpu_device_selection_takes_the_weights_off_the_gpu(self, spec_config, swa):
"""``--device none`` runs on the CPU whatever the layer count says.
@@ -1897,8 +1935,9 @@ class TestLaunchShapedPricing:
assert many.gpu_bytes == none.gpu_bytes
def test_one_card_is_priced_as_the_layer_load_it_launches(self, spec_config, swa):
- # Tensor mode needs two usable GPUs. Below that load_model drops it, so pricing
- # tensor charged per-device compute buffers for a launch that runs neither.
+ # Tensor mode needs two usable GPUs. Below that load_model drops it, so the
+ # panel prices the layer load. A tensor device holds the single-device compute
+ # buffer, so on one card the two price the same.
#
# The cache type is no longer part of this: #8939 removed the gate that rewrote
# both axes to f16 on a tensor split, so a quantized KV now survives one and the
@@ -1914,8 +1953,8 @@ class TestLaunchShapedPricing:
assert downgraded.cache_type_kv == "q4_0"
assert as_tensor.cache_type_kv == "q4_0"
assert downgraded.kv_bytes == as_tensor.kv_bytes
- assert downgraded.compute_bytes < as_tensor.compute_bytes
- assert downgraded.total_bytes < as_tensor.total_bytes
+ assert downgraded.compute_bytes == as_tensor.compute_bytes
+ assert downgraded.total_bytes == as_tensor.total_bytes
def test_manual_auto_layers_is_priced_as_the_layer_load_it_launches(self, spec_config, swa):
# Manual with Auto layers hands the budget to llama.cpp --fit, which load_model
diff --git a/studio/backend/tests/test_mmproj_placement_policy.py b/studio/backend/tests/test_mmproj_placement_policy.py
index 8f19047bf43..df4f00ee198 100644
--- a/studio/backend/tests/test_mmproj_placement_policy.py
+++ b/studio/backend/tests/test_mmproj_placement_policy.py
@@ -1031,7 +1031,7 @@ _CC_PER_TOKEN = 1536 # 6 MiB at 4096, the rate the bundled estimator produces
def _split_rate_backend(tmp_path, *, memory, **kwargs):
backend, gguf = _backend(tmp_path, memory = memory, **kwargs)
backend._compute_buffer_ctx_bytes = (
- lambda n_ctx, n_ubatch = None, cache_type_kv = None, *, layer_split = False: (
+ lambda n_ctx, n_ubatch = None, cache_type_kv = None, *, layer_split = False, **_kw: (
n_ctx * _CC_PER_TOKEN * (LlamaCppBackend._CTX_COMPUTE_SPLIT_MULT if layer_split else 1)
)
)
diff --git a/studio/backend/tests/test_mtp_mla_target_ctx.py b/studio/backend/tests/test_mtp_mla_target_ctx.py
index 5c7e60e41f3..70b17065f7f 100644
--- a/studio/backend/tests/test_mtp_mla_target_ctx.py
+++ b/studio/backend/tests/test_mtp_mla_target_ctx.py
@@ -109,6 +109,9 @@ def _make_mla_backend(
b._kv_value_length_swa = None
b._draft_backend_cache = None
b._vocab_size = vocab
+ # The speculative compute buffers ride on this reserve too; test_compute_buffer
+ # prices them, and these cases pin the cache terms.
+ b._mtp_draft_compute_bytes = lambda *args, **kwargs: 0
return b
diff --git a/studio/backend/tests/test_mtp_reserve_note_and_unload_log.py b/studio/backend/tests/test_mtp_reserve_note_and_unload_log.py
index c6242a23e9e..763b65a9bf3 100644
--- a/studio/backend/tests/test_mtp_reserve_note_and_unload_log.py
+++ b/studio/backend/tests/test_mtp_reserve_note_and_unload_log.py
@@ -176,12 +176,9 @@ def test_an_unload_of_a_resident_model_still_logs_one_event(backend, monkeypatch
def test_the_real_estimator_ignores_both_axes_for_a_dense_embedded_head(backend):
- """The premise the note now asks about, pinned against the estimator itself.
-
- Without this the note's tests would only prove it renders whatever a stand-in
- tells it, and the claim that slots and ubatch really are inert on this branch
- would rest on reading the code.
- """
+ """Slots and ubatch leave the dense draft cache unchanged but grow its compute cost."""
+ real_compute = backend._mtp_draft_compute_bytes
+ backend._mtp_draft_compute_bytes = lambda *args, **kwargs: 0
backend._nextn_predict_layers = 1
backend._n_kv_heads = 8
backend._n_heads = 64
@@ -213,3 +210,13 @@ def test_the_real_estimator_ignores_both_axes_for_a_dense_embedded_head(backend)
8192, spec_draft_n_max = 4, n_parallel = 3, kv_unified = False, n_ubatch = 512
)
assert split != base
+
+ # With the compute buffers back, a slot adds verify rows and the micro-batch the
+ # draft context's activations, so the note has both to name.
+ backend._mtp_draft_compute_bytes = real_compute
+ backend._vocab_size = 151936
+ backend._embedding_length = 4096
+ backend._feed_forward_length = 12288
+ moved = _reserve(2, 512)
+ assert _reserve(3, 512) > moved
+ assert _reserve(2, 2048) > moved
diff --git a/studio/backend/tests/test_mtp_vram_budget.py b/studio/backend/tests/test_mtp_vram_budget.py
index 88ab0ee2b15..f7538c1ab21 100644
--- a/studio/backend/tests/test_mtp_vram_budget.py
+++ b/studio/backend/tests/test_mtp_vram_budget.py
@@ -127,6 +127,9 @@ def _make_backend(
b._kv_key_length_swa = None
b._kv_value_length_swa = None
b._draft_backend_cache = None
+ # The speculative compute buffers ride on this reserve too; test_compute_buffer
+ # prices them, and these cases pin the cache and rollback terms.
+ b._mtp_draft_compute_bytes = lambda *args, **kwargs: 0
return b
diff --git a/studio/backend/tests/test_slot_offload_fit.py b/studio/backend/tests/test_slot_offload_fit.py
index bf8bd63231a..487220d813c 100644
--- a/studio/backend/tests/test_slot_offload_fit.py
+++ b/studio/backend/tests/test_slot_offload_fit.py
@@ -20,6 +20,9 @@ from pathlib import Path
_BACKEND_DIR = str(Path(__file__).resolve().parent.parent)
if _BACKEND_DIR not in sys.path:
sys.path.insert(0, _BACKEND_DIR)
+_TESTS_DIR = str(Path(__file__).resolve().parent)
+if _TESTS_DIR not in sys.path:
+ sys.path.insert(0, _TESTS_DIR)
_loggers_stub = _types.ModuleType("loggers")
_loggers_stub.get_logger = lambda name: __import__("logging").getLogger(name)
@@ -39,11 +42,15 @@ def _backend(
kv_calls = None,
):
"""Backend with the dims the compute buffer reads; KV mocked to a fixed size so the
- only slot-dependent term is the compute buffer (485 MiB/slot f32 output x 1.15)."""
+ only slot-dependent term is the synthetic compute buffer from
+ ``_install_slot_scaled_compute`` (558 MiB per extra slot at ubatch 512)."""
+ from test_llama_cpp_placement import _install_slot_scaled_compute
+
b = LlamaCppBackend.__new__(LlamaCppBackend)
b._vocab_size = vocab
b._embedding_length = embd
b._key_length_mla = None
+ _install_slot_scaled_compute(b)
def estimate(
ctx,
@@ -88,8 +95,8 @@ def _run(
class TestSlotsThatFitOnGpu:
- """Compute-buffer per slot (vocab 248320, embd 5120): cb(1)=46, cb(2)=604, cb(3)=1162,
- cb(4)=1719 MiB. Single 24 GB card usable = 24576 - 0.03*24576 = 23839 MiB."""
+ """Synthetic compute buffer per slot count: cb(1)=46, cb(2)=604, cb(3)=1162,
+ cb(4)=1720 MiB. Single 24 GB card usable = 24576 - 0.03*24576 = 23839 MiB."""
def test_reduces_to_largest_fitting_slot(self):
# base+KV = 22500: par4 (24219) over 23839, par3 (23662) fits -> 3 slots on GPU.
@@ -159,12 +166,8 @@ class TestSlotsThatFitOnGpu:
assert all(call["swa_full"] is True for call in calls)
def test_micro_batch_is_re_derived_per_candidate(self):
- """llama-server raises --batch-size to max(slots, 2) and llama.cpp caps the
- micro-batch against it, so a batch below the requested slot count shrinks as the
- candidates do: n_batch=1 launches the 3-slot candidate at -b 3, not the 64 the
- 4-slot request resolved to. At these dims that is 6.8 MiB of compute buffer
- against 145.2, so pricing every candidate at the requested count's micro-batch
- rejects a 3-slot fit that launches comfortably and cuts serving capacity for it."""
+ """Recompute the batch floor and ubatch for each candidate slot count.
+ Keeping the requested ubatch would reject the fitting three-slot candidate."""
from core.inference.llama_cpp import _emitted_n_batch, _extra_args_n_ubatch
def ubatch_for_slots(slots: int):
diff --git a/studio/backend/tests/test_slot_reduction_context_refit.py b/studio/backend/tests/test_slot_reduction_context_refit.py
index 42f0156c81c..ee7aff84860 100644
--- a/studio/backend/tests/test_slot_reduction_context_refit.py
+++ b/studio/backend/tests/test_slot_reduction_context_refit.py
@@ -4,7 +4,8 @@
"""Regression tests for context fitting after serving-slot reduction.
The synthetic plans use Qwen3.8-27B metadata and capture the generated command.
-Each case names its speculative mode because unknown values resolve to Auto.
+Cases specify speculation explicitly and use synthetic slot costs unless testing
+the real estimator.
"""
from __future__ import annotations
@@ -20,7 +21,11 @@ if _TESTS_DIR not in sys.path:
import pytest # noqa: E402
import core.inference.llama_cpp as llama_cpp # noqa: E402
-from test_llama_cpp_placement import _backend, _launch # noqa: E402
+from test_llama_cpp_placement import ( # noqa: E402
+ _backend,
+ _install_slot_scaled_compute,
+ _launch,
+)
MIB = 1024 * 1024
NATIVE_CTX = 262144
@@ -72,6 +77,23 @@ DENSE = {
"_context_length": NATIVE_CTX,
}
+# Gemma 3 12B-shaped: five sliding-window layers in six.
+SLIDING_WINDOW = {
+ "_architecture": "gemma3",
+ "_vocab_size": 262144,
+ "_n_layers": 48,
+ "_n_kv_heads": 8,
+ "_n_heads": 16,
+ "_embedding_length": 3840,
+ "_feed_forward_length": 15360,
+ "_kv_key_length": 256,
+ "_kv_value_length": 256,
+ "_key_length_mla": None,
+ "_context_length": 131072,
+ "_sliding_window": 1024,
+ "_sliding_window_pattern": [i % 6 != 5 for i in range(48)],
+}
+
def _plan(
tmp_path,
@@ -83,8 +105,10 @@ def _plan(
n_ctx = 0,
metadata = HYBRID,
gpus = 1,
+ slot_scaled_compute = True,
):
- """Return the generated placement plan. ``vram_mib`` may be a per-card sequence."""
+ """Return the generated placement plan. ``vram_mib`` may be a per-card sequence.
+ ``slot_scaled_compute`` False prices the compute buffer with the real estimator."""
cards = list(vram_mib) if isinstance(vram_mib, (tuple, list)) else [vram_mib] * gpus
memory = [(i, mib, mib) for i, mib in enumerate(cards)]
backend, gguf = _backend(tmp_path, vulkan = False, memory = memory)
@@ -96,6 +120,8 @@ def _plan(
backend._read_gguf_metadata = read
backend._get_gguf_size_bytes = lambda _path: weights_mib * MIB
del backend._can_estimate_kv # the real one, now that the dims are set
+ if slot_scaled_compute:
+ _install_slot_scaled_compute(backend)
backend.probe_server_capabilities = lambda _binary = None: {
"mtp_token": "draft-mtp",
"supports_ngram_mod": True,
@@ -127,8 +153,8 @@ class TestTheLaunchedCountOwnsTheContext:
"weights_mib,asked,slots,ctx",
[
(_KEEPS_THREE_MIB, 4, 3, 12_288),
- (_KEEPS_TWO_MIB, 4, 2, 13_824),
- (_KEEPS_TWO_MIB, 8, 2, 13_824), # the same answer from a larger ask
+ (_KEEPS_TWO_MIB, 4, 2, 14_080),
+ (_KEEPS_TWO_MIB, 8, 2, 14_080), # the same answer from a larger ask
],
)
def test_auto_context_follows_the_reduction(self, tmp_path, weights_mib, asked, slots, ctx):
@@ -155,22 +181,73 @@ class TestTheLaunchedCountOwnsTheContext:
was published instead of the context the final slot count actually affords.
"""
got = _plan(tmp_path, weights_mib = _KEEPS_TWO_MIB, n_parallel = 4, spec = "off")
- assert got["ceiling"] == 13_824
+ assert got["ceiling"] == 14_080
assert got["ceiling"] != llama_cpp._FIT_MIN_CTX
def test_a_layer_split_across_two_cards_re_fits_the_same_way(self, tmp_path):
"""The same invariant holds for a two-card layer split."""
reduced = _plan(
- tmp_path, weights_mib = 14_000, n_parallel = 4, spec = "off", vram_mib = 8_704, gpus = 2
+ tmp_path, weights_mib = 13_800, n_parallel = 4, spec = "off", vram_mib = 8_704, gpus = 2
)
direct = _plan(
- tmp_path, weights_mib = 14_000, n_parallel = 2, spec = "off", vram_mib = 8_704, gpus = 2
+ tmp_path, weights_mib = 13_800, n_parallel = 2, spec = "off", vram_mib = 8_704, gpus = 2
)
assert (reduced["slots"], reduced["fit"]) == (2, "off")
- assert reduced["ctx"] == reduced["ceiling"] == 8_448
+ assert reduced["ctx"] == reduced["ceiling"] == 10_752
assert (reduced["ctx"], reduced["ceiling"]) == (direct["ctx"], direct["ceiling"])
+class TestTheRealEstimatorStillReduces:
+ """Real pricing reduces slots for hybrid state costs and keeps slots on dense models."""
+
+ @pytest.mark.parametrize("weights_mib,slots,ctx", [(10_400, 3, 9_472), (10_500, 2, 10_240)])
+ def test_a_hybrid_load_reduces_and_re_fits(self, tmp_path, weights_mib, slots, ctx):
+ got = _plan(
+ tmp_path, weights_mib = weights_mib, n_parallel = 4, spec = "off", slot_scaled_compute = False
+ )
+ assert (got["slots"], got["fit"], got["ctx"]) == (slots, "off", ctx)
+ direct = _plan(
+ tmp_path,
+ weights_mib = weights_mib,
+ n_parallel = slots,
+ spec = "off",
+ slot_scaled_compute = False,
+ )
+ assert (direct["slots"], direct["ctx"]) == (slots, ctx)
+
+ def test_a_sliding_window_split_reprices_its_masks_per_candidate(self, tmp_path):
+ """A layer split's sliding-window masks scale with slots, so each candidate
+ prices its own count rather than the one asked for."""
+ plans = [
+ _plan(
+ tmp_path,
+ weights_mib = 21_200,
+ n_parallel = asked,
+ spec = "off",
+ metadata = SLIDING_WINDOW,
+ gpus = 2,
+ slot_scaled_compute = False,
+ )
+ for asked in (16, 1)
+ ]
+ assert [(p["slots"], p["fit"], p["ctx"]) for p in plans] == [(1, "off", 8_448)] * 2
+
+ def test_a_dense_load_keeps_its_slots(self, tmp_path):
+ plans = [
+ _plan(
+ tmp_path,
+ weights_mib = 8_200,
+ n_parallel = asked,
+ spec = "off",
+ metadata = DENSE,
+ slot_scaled_compute = False,
+ )
+ for asked in (1, 4)
+ ]
+ assert [(p["slots"], p["fit"]) for p in plans] == [(1, "off"), (4, "off")]
+ assert plans[0]["ctx"] == plans[1]["ctx"]
+
+
class TestTheRefitStaysOnTheCardsTheReductionChose:
"""A 12 GiB primary next to a 1.25 GiB card: the reduced plan fits on the primary
alone, and only the small card can hold a longer context."""
@@ -180,7 +257,7 @@ class TestTheRefitStaysOnTheCardsTheReductionChose:
tmp_path, weights_mib = _KEEPS_TWO_MIB, n_parallel = 4, spec = "off", vram_mib = MIXED_CARDS
)
assert (got["slots"], got["fit"], got["devices"]) == (2, "off", "0")
- assert got["ctx"] == 13_824
+ assert got["ctx"] == 14_080
def test_it_matches_a_request_started_at_the_final_count(self, tmp_path):
reduced = _plan(
@@ -198,8 +275,8 @@ class TestTheRefitStaysOnTheCardsTheReductionChose:
tmp_path, weights_mib = _KEEPS_TWO_MIB, n_parallel = 4, spec = "off", vram_mib = MIXED_CARDS
)
alone = _plan(tmp_path, weights_mib = _KEEPS_TWO_MIB, n_parallel = 4, spec = "off")
- assert mixed["ctx"] == alone["ctx"] == 13_824
- assert (mixed["ceiling"], alone["ceiling"]) == (14_848, 13_824)
+ assert mixed["ctx"] == alone["ctx"] == 14_080
+ assert (mixed["ceiling"], alone["ceiling"]) == (14_592, 14_080)
class TestAutoSpeculationStillDecidesBeforeTheReduction:
@@ -207,7 +284,7 @@ class TestAutoSpeculationStillDecidesBeforeTheReduction:
def test_a_direct_one_slot_request_drops_the_drafter_and_keeps_context(self, tmp_path):
got = _plan(tmp_path, weights_mib = 10_200, n_parallel = 1, spec = "auto")
- assert (got["slots"], got["ctx"], got["spec"]) == (1, 18_688, "ngram-mod")
+ assert (got["slots"], got["ctx"], got["spec"]) == (1, 18_944, "ngram-mod")
@pytest.mark.parametrize("asked", [4, 8])
def test_a_reduced_request_still_carries_the_drafter_it_admitted(self, tmp_path, asked):
@@ -230,7 +307,7 @@ class TestWhatMustNotMove:
got = _plan(tmp_path, weights_mib = 8_400, n_parallel = 4, spec = "off", n_ctx = 32768)
assert (got["ctx"], got["slots"], got["fit"]) == (32768, 2, "off")
# The measured ceiling still follows the final slot count.
- assert got["ceiling"] == 35_840
+ assert got["ceiling"] == 36_096
def test_an_explicit_context_that_forces_offload_is_unchanged(self, tmp_path):
"""An explicit context that requires offload is unchanged."""
@@ -246,7 +323,7 @@ class TestWhatMustNotMove:
def test_a_count_that_needs_no_reduction_is_untouched(self, tmp_path):
"""A plan that needs no slot reduction is unchanged."""
got = _plan(tmp_path, weights_mib = 6_000, n_parallel = 4, spec = "off")
- assert (got["slots"], got["fit"], got["ctx"]) == (4, "off", 51_200)
+ assert (got["slots"], got["fit"], got["ctx"]) == (4, "off", 51_456)
def test_weights_that_fit_nowhere_still_offload(self, tmp_path):
"""Oversized weights still fall back to offload, at the Auto offload context.
@@ -328,7 +405,7 @@ class TestTheReductionIsPricedAtTheFitFloor:
"""
monkeypatch.setattr(llama_cpp, "_AUTO_OFFLOAD_CTX", offload_ctx)
got = _plan(tmp_path, weights_mib = _KEEPS_TWO_MIB, n_parallel = 4, spec = "off")
- assert (got["slots"], got["fit"], got["ctx"], got["devices"]) == (2, "off", 13_824, "0")
+ assert (got["slots"], got["fit"], got["ctx"], got["devices"]) == (2, "off", 14_080, "0")
@pytest.mark.parametrize("weights_mib,metadata", RESCUED)
@pytest.mark.parametrize("offload_ctx", _OFFLOAD_SWEEP[:2])
diff --git a/studio/backend/tests/test_slot_refit_platform_matrix.py b/studio/backend/tests/test_slot_refit_platform_matrix.py
index fd8fb167127..5012070a977 100644
--- a/studio/backend/tests/test_slot_refit_platform_matrix.py
+++ b/studio/backend/tests/test_slot_refit_platform_matrix.py
@@ -29,7 +29,11 @@ import pytest # noqa: E402
import core.inference.llama_cpp as llama_mod # noqa: E402
from core.inference.llama_cpp import LlamaCppBackend # noqa: E402
-from test_llama_cpp_placement import _backend, _launch # noqa: E402
+from test_llama_cpp_placement import ( # noqa: E402
+ _backend,
+ _install_slot_scaled_compute,
+ _launch,
+)
MIB = 1024 * 1024
NATIVE_CTX = 262144
@@ -150,6 +154,8 @@ def _plan(
backend._read_gguf_metadata = read
backend._get_gguf_size_bytes = lambda _path: weights_mib * MIB
del backend._can_estimate_kv # the real one, now that the dims are set
+ # The per-slot cost the re-fit trades against (see the helper).
+ _install_slot_scaled_compute(backend)
backend.probe_server_capabilities = lambda _binary = None: {
"mtp_token": "draft-mtp",
"supports_ngram_mod": True,