Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
# SPDX-License-Identifier: AGPL-3.0-only
# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
"""Compute-buffer estimates checked against measured allocations and synthetic layouts."""
from __future__ import annotations
import sys
import types as _types
from pathlib import Path
import pytest
_BACKEND_DIR = str(Path(__file__).resolve().parent.parent)
if _BACKEND_DIR not in sys.path:
sys.path.insert(0, _BACKEND_DIR)
_loggers_stub = _types.ModuleType("loggers")
_loggers_stub.get_logger = lambda name: __import__("logging").getLogger(name)
sys.modules.setdefault("loggers", _loggers_stub)
_structlog_stub = _types.ModuleType("structlog")
_structlog_stub.get_logger = lambda *a, **k: __import__("logging").getLogger("stub")
sys.modules.setdefault("structlog", _structlog_stub)
# Only stub httpx when missing: a stub leaks via sys.modules and breaks huggingface_hub imports.
try:
import httpx as _httpx_real # noqa: F401
except ImportError:
_httpx_stub = _types.ModuleType("httpx")
for _exc in (
"ConnectError",
"TimeoutException",
"ReadTimeout",
"ReadError",
"RemoteProtocolError",
"CloseError",
"HTTPError",
"RequestError",
):
setattr(_httpx_stub, _exc, type(_exc, (Exception,), {}))
_httpx_stub.Timeout = type("T", (), {"__init__": lambda s, *a, **k: None})
_httpx_stub.Response = type("Response", (), {})
_httpx_stub.Client = type(
"C",
(),
{
"__init__": lambda s, **kw: None,
"__enter__": lambda s: s,
"__exit__": lambda s, *a: None,
},
)
sys.modules["httpx"] = _httpx_stub
from core.inference.llama_cpp import (
_FIT_MIN_CTX,
_kv_bytes_per_elem,
_planned_main_cache_types,
_planned_scratch_cache_type,
LlamaCppBackend,
)
MIB = 1024 * 1024
GIB = 1024 * MIB
def _backend(
vocab = 248320,
embd = 5120,
mla = None,
arch = None,
pooling_type = None,
):
"""Backend with just the dims the compute-buffer estimate reads."""
b = LlamaCppBackend.__new__(LlamaCppBackend)
b._vocab_size = vocab
b._embedding_length = embd
b._key_length_mla = mla
b._architecture = arch
b._pooling_type = pooling_type
return b
def _pad(cells):
return ((cells + 255) // 256) * 256
def _hidden_copies(embd, ub = 512):
"""The hidden-state copies a layer-split device holds, one per scheduler copy."""
return LlamaCppBackend._CTX_COMPUTE_SPLIT_MULT * embd * ub * 4
def _backend_from_gguf_local(
n_layers = 80,
embd = 8192,
n_kv_heads = 8,
head_dim = 128,
):
"""Builds a real __init__ backend so every attribute the KV estimator reads has its default."""
b = LlamaCppBackend()
b._vocab_size = 248_320
b._embedding_length = embd
b._n_layers = n_layers
b._n_kv_heads = n_kv_heads
b._kv_key_length = head_dim
b._kv_value_length = head_dim
b._context_length = 262_144
assert b._can_estimate_kv()
return b
_GEMMA4_12B_PATTERN = ([True] * 5 + [False]) * 8
# Measured on bundled llama.cpp b10909 (RTX 6000 Ada, --flash-attn on, --kv-unified).
_DIMS = {
"Llama-3.2-1B": dict(
_vocab_size = 128256,
_embedding_length = 2048,
_feed_forward_length = 8192,
_n_layers = 16,
_n_heads = 32,
_n_kv_heads = 8,
_kv_key_length = 64,
_kv_value_length = 64,
),
"Qwen3-8B": dict(
_vocab_size = 151936,
_embedding_length = 4096,
_feed_forward_length = 12288,
_n_layers = 36,
_n_heads = 32,
_n_kv_heads = 8,
_kv_key_length = 128,
_kv_value_length = 128,
),
"Qwen3.5-4B": dict(
_vocab_size = 248320,
_embedding_length = 2560,
_feed_forward_length = 9216,
_n_layers = 32,
_n_heads = 16,
_n_kv_heads = 4,
_kv_key_length = 256,
_kv_value_length = 256,
_ssm_inner_size = 4096,
_ssm_state_size = 128,
_ssm_group_count = 16,
_full_attention_interval = 4,
),
"Qwen3.6-35B-A3B": dict(
_vocab_size = 248320,
_embedding_length = 2048,
_expert_used_count = 8,
_expert_feed_forward_length = 512,
_expert_shared_feed_forward_length = 512,
_n_layers = 40,
_n_heads = 16,
_n_kv_heads = 2,
_kv_key_length = 256,
_kv_value_length = 256,
_ssm_inner_size = 4096,
_ssm_state_size = 128,
_ssm_group_count = 16,
_full_attention_interval = 4,
),
"Gemma-4-12B": dict(
_vocab_size = 262144,
_embedding_length = 3840,
_feed_forward_length = 15360,
_n_layers = 48,
_n_heads = 16,
_n_kv_heads = 8,
_n_kv_heads_by_layer = [1 if not swa else 8 for swa in _GEMMA4_12B_PATTERN],
_kv_key_length = 512,
_kv_value_length = 512,
_kv_key_length_swa = 256,
_kv_value_length_swa = 256,
_sliding_window = 1024,
_sliding_window_pattern = _GEMMA4_12B_PATTERN,
),
"Gemma-4-E4B": dict(
_vocab_size = 262144,
_embedding_length = 2560,
_feed_forward_length = 10240,
_embedding_length_per_layer_input = 256,
_n_layers = 42,
_n_heads = 8,
_n_kv_heads = 2,
_kv_key_length = 512,
_kv_value_length = 512,
_kv_key_length_swa = 256,
_kv_value_length_swa = 256,
_sliding_window = 512,
_sliding_window_pattern = ([True] * 5 + [False]) * 7,
),
"GLM-4.7-Flash": dict(
_vocab_size = 154880,
_embedding_length = 2048,
_feed_forward_length = 10240,
_expert_used_count = 4,
_expert_feed_forward_length = 1536,
_expert_shared_count = 1,
_n_layers = 47,
_n_heads = 20,
_n_kv_heads = 1,
_kv_key_length = 576,
_kv_value_length = 512,
_kv_lora_rank = 512,
_key_length_mla = 256,
),
}
def _measured(name):
b = LlamaCppBackend()
for attr, value in _DIMS[name].items():
setattr(b, attr, value)
return b
_MEASURED_COMPUTE = [
("Llama-3.2-1B", 256, 4, 32768, "f16", 46.01),
("Llama-3.2-1B", 2048, 4, 32768, "f16", 368.04),
("Llama-3.2-1B", 512, 1, 32768, "f16", 92.01),
("Llama-3.2-1B", 512, 8, 32768, "f16", 92.01),
("Llama-3.2-1B", 512, 4, 131072, "f16", 188.01),
("Llama-3.2-1B", 512, 4, 131072, "q8_0", 400.04),
("Llama-3.2-1B", 2048, 4, 131072, "q8_0", 832.07),
("Qwen3-8B", 512, 4, 32768, "f16", 128.01),
("Qwen3-8B", 2048, 4, 131072, "f16", 896.04),
("Qwen3-8B", 512, 4, 131072, "q8_0", 672.09),
("Qwen3.5-4B", 512, 1, 32768, "f16", 96.02),
("Qwen3.5-4B", 512, 8, 32768, "f16", 115.52),
("Qwen3.5-4B", 2048, 4, 131072, "f16", 768.06),
("Qwen3.5-4B", 2048, 4, 131072, "q8_0", 1224.34),
("Qwen3.6-35B-A3B", 256, 4, 32768, "f16", 66.02),
("Qwen3.6-35B-A3B", 2048, 1, 32768, "f16", 528.2),
("Qwen3.6-35B-A3B", 512, 4, 131072, "q8_0", 436.31),
("Gemma-4-12B", 512, 1, 32768, "f16", 154.52),
("Gemma-4-12B", 1120, 4, 32768, "f16", 346.21),
("Gemma-4-12B", 2048, 8, 32768, "f16", 652.07),
("Gemma-4-12B", 2048, 4, 131072, "f16", 1020.07),
("Gemma-4-12B", 512, 4, 131072, "q8_0", 436.8),
("Gemma-4-E4B", 512, 4, 32768, "f16", 135.52),
("Gemma-4-E4B", 2048, 4, 131072, "q8_0", 1317.35),
("GLM-4.7-Flash", 512, 4, 32768, "f16", 110.01),
("GLM-4.7-Flash", 2048, 4, 131072, "f16", 824.04),
("GLM-4.7-Flash", 512, 4, 131072, "q8_0", 358.9),
]
def _priced_mib(b, ub, slots, ctx, kv):
flat = b._estimate_compute_buffer_bytes(n_ubatch = ub, n_parallel = slots)
return (flat + b._compute_buffer_ctx_bytes(ctx, ub, kv)) / MIB
class TestMeasuredComputeBuffer:
"""Flat plus context-linear must match llama-server's reserve, not the old per-slot vocab rows."""
@pytest.mark.parametrize("name,ub,slots,ctx,kv,measured", _MEASURED_COMPUTE)
def test_covers_the_measured_buffer(self, name, ub, slots, ctx, kv, measured):
est = _priced_mib(_measured(name), ub, slots, ctx, kv)
assert est >= measured, f"{name} ub{ub} p{slots} {ctx} {kv}: {est:.0f} < {measured}"
@pytest.mark.parametrize("name,ub,slots,ctx,kv,measured", _MEASURED_COMPUTE)
def test_stays_close_to_the_measured_buffer(self, name, ub, slots, ctx, kv, measured):
est = _priced_mib(_measured(name), ub, slots, ctx, kv)
assert est <= measured * (1.35 if kv == "f16" else 1.3), f"{name}: {est:.0f} vs {measured}"
def test_the_reported_gemma4_vision_raise(self):
b = _measured("Gemma-4-12B")
raise_mib = _priced_mib(b, 1120, 4, 262144, "f16") - _priced_mib(b, 512, 4, 262144, "f16")
assert 455 <= raise_mib <= 455 * 1.15
class TestFlatBufferShape:
def test_slots_add_output_rows_not_activations(self):
b = _measured("Gemma-4-12B")
one = b._estimate_compute_buffer_bytes(n_ubatch = 2048, n_parallel = 1)
eight = b._estimate_compute_buffer_bytes(n_ubatch = 2048, n_parallel = 8)
rows = 7 * 262144 * 4
swa = (_pad(1024 * 8 + 2048) - _pad(1024 + 2048)) * 2048 * 2
assert eight - one == pytest.approx(
rows * LlamaCppBackend._COMPUTE_BUFFER_SAFETY + swa, rel = 1e-6
)
def test_scales_linearly_with_ubatch(self):
b = _measured("Qwen3-8B")
lo = b._estimate_compute_buffer_bytes(n_ubatch = 512, n_parallel = 1)
hi = b._estimate_compute_buffer_bytes(n_ubatch = 1024, n_parallel = 1)
rows = 151936 * 4 * LlamaCppBackend._COMPUTE_BUFFER_SAFETY
assert hi - rows == pytest.approx(2 * (lo - rows), rel = 1e-6)
def test_a_tensor_device_holds_the_single_device_buffer(self):
for name in ("Qwen3-8B", "Gemma-4-12B"):
b = _measured(name)
assert b._estimate_compute_buffer_bytes(
n_ubatch = 2048, n_parallel = 4, per_device_tensor = True
) == b._estimate_compute_buffer_bytes(n_ubatch = 2048, n_parallel = 4)
def test_embedding_rows_are_pooled_not_vocab_wide(self):
b = _backend(vocab = 30522, embd = 384, pooling_type = 1)
chat = _backend(vocab = 30522, embd = 384)
assert b._estimate_compute_buffer_bytes(n_ubatch = 512, n_parallel = 4) > (
chat._estimate_compute_buffer_bytes(n_ubatch = 512, n_parallel = 4)
)
assert b._estimate_compute_buffer_bytes(n_ubatch = 512, n_parallel = 4) < 30522 * 512 * 4
def test_widest_block_sets_the_width(self):
assert _measured("Qwen3-8B")._compute_activation_width() == 4 * 12288
assert _measured("Qwen3.6-35B-A3B")._compute_activation_width() == (
8 * (2 * 2048 + 3 * 512) + 3 * 512
)
ssm = _backend(embd = 2560)
ssm._ssm_inner_size, ssm._ssm_state_size, ssm._ssm_group_count = 4096, 128, 16
assert ssm._compute_activation_width() == 4 * (4096 + 2 * 16 * 128)
assert _measured("Gemma-4-E4B")._compute_activation_width() == 4 * 10240 + 256 * 42
def test_header_without_ffn_dims_uses_the_embedding_floor(self):
assert _backend(embd = 4096)._compute_activation_width() == 12 * 4096
def test_expert_width_falls_back_to_the_ffn_length(self):
b = _backend(vocab = 100352, embd = 1536)
b._n_experts, b._expert_used_count = 64, 6
b._feed_forward_length, b._expert_shared_feed_forward_length = 512, 1024
assert b._compute_activation_width() == 6 * (2 * 1536 + 3 * 512) + 3 * 1024
def test_granite_hybrid_is_covered(self):
b = _backend(vocab = 100352, embd = 1536)
b._n_experts, b._expert_used_count = 64, 6
b._feed_forward_length, b._expert_shared_feed_forward_length = 512, 1024
b._ssm_inner_size, b._ssm_state_size, b._ssm_group_count = 3072, 128, 1
for slots, measured in ((1, 80.0), (8, 95.3)):
est = _priced_mib(b, 512, slots, 32768, "f16")
assert measured <= est <= measured * 1.3
def test_a_window_wider_than_the_context_does_not_slide(self):
b = _backend(vocab = 200064, embd = 3072)
b._feed_forward_length, b._context_length = 8192, 131072
b._sliding_window, b._sliding_window_pattern = 262144, None
assert b._effective_sliding_window() == 0
assert 92.0 <= _priced_mib(b, 512, 4, 32768, "f16") <= 92.0 * 1.3
b._sliding_window_pattern = [False] * 32
assert b._effective_sliding_window() == 0
b._sliding_window_pattern = [True, False] * 16
assert b._effective_sliding_window() == 262144
class TestFallback:
def test_zero_when_vocab_missing(self):
assert _backend(vocab = None)._estimate_compute_buffer_bytes(n_parallel = 4) == 0
def test_vocab_ceiling_prices_a_header_without_a_token_array(self):
b = _backend(vocab = None)
assert b._estimate_compute_buffer_bytes(n_parallel = 4, vocab_ceiling = 262144) == (
_backend(vocab = 262144)._estimate_compute_buffer_bytes(n_parallel = 4)
)
def test_zero_when_embd_missing(self):
assert _backend(embd = None)._estimate_compute_buffer_bytes(n_parallel = 4) == 0
def test_zero_lets_tensor_plan_use_flat_fallback(self):
b = _backend(vocab = None, embd = None)
b._n_layers = None
ec, mac, gi, ts = b._plan_tensor_parallel([(0, 48000), (1, 48000)], 8 * 1024**3, 8192)
assert gi == [0, 1]
class TestContextLinearBuffer:
"""``_compute_buffer_ctx_bytes``: the KQ mask, n_ubatch * 2 bytes per cell, plus
for a quantized cache an f16 copy of the widest global layer's K and V."""
# (model, n_embd, ctx, measured CUDA0 compute MiB, q8_0/ub512); older build, upper bounds.
_MEASURED = [
("Qwen3.5-2B", 2048, 262144, 796),
("Qwen3.5-4B", 2560, 262144, 1330),
("Qwen3.5-9B", 4096, 262144, 1336),
("Qwen3.6-27B", 5120, 262144, 1360),
("Gemma-4-31B", 5376, 262144, 2392),
]
def test_zero_by_default(self):
assert _backend()._compute_buffer_ctx_bytes(0) == 0
def test_zero_when_embd_missing(self):
assert _backend(embd = None)._compute_buffer_ctx_bytes(262144) == 0
def test_grows_linearly_with_context(self):
b = _measured("Qwen3-8B")
for kv in ("f16", "q8_0"):
a = b._compute_buffer_ctx_bytes(65536, cache_type_kv = kv)
d = b._compute_buffer_ctx_bytes(131072, cache_type_kv = kv)
assert d == pytest.approx(2 * a, rel = 1e-6)
@pytest.mark.parametrize("name,embd,ctx,measured", _MEASURED)
def test_upper_bounds_measured_compute_growth(self, name, embd, ctx, measured):
b = _backend(embd = embd)
flat = b._estimate_compute_buffer_bytes(n_parallel = 1)
total = (flat + b._compute_buffer_ctx_bytes(ctx, cache_type_kv = "q8_0")) / MIB
assert total >= measured, f"{name}: under-reserved {total:.0f} < {measured}"
class TestContextBufferKVQuant:
def test_quantized_heavier_than_f16(self):
b = _measured("Qwen3-8B")
assert b._compute_buffer_ctx_bytes(131072, cache_type_kv = "q8_0") > (
b._compute_buffer_ctx_bytes(131072, cache_type_kv = "f16")
)
def test_none_cache_type_is_f16(self):
b = _measured("Qwen3-8B")
assert b._compute_buffer_ctx_bytes(131072, cache_type_kv = None) == (
b._compute_buffer_ctx_bytes(131072, cache_type_kv = "f16")
)
@pytest.mark.parametrize("ct", ["f16", "bf16", "f32"])
def test_unquantized_pays_exactly_the_mask(self, ct):
for name in _DIMS:
b = _measured(name)
for ub in (512, 2048):
assert b._compute_buffer_ctx_bytes(100000, ub, ct) == ub * 2 * 100000
@pytest.mark.parametrize("ct", ["q8_0", "q5_1", "q4_0", "iq4_nl"])
def test_quantized_adds_the_widest_kv_copy_whatever_the_ubatch(self, ct):
b = _measured("Qwen3-8B")
for ub in (512, 2048):
extra = b._compute_buffer_ctx_bytes(100000, ub, ct) - ub * 2 * 100000
assert extra == 8 * (128 + 128) * 2 * 100000
def test_sliding_window_layers_do_not_set_the_rate(self):
assert _measured("Gemma-4-12B")._attention_dequant_width() == 1 * (512 + 512)
def test_mla_copies_the_compressed_key_only(self):
assert _measured("GLM-4.7-Flash")._attention_dequant_width() == 576
def test_header_without_head_dims_falls_back_to_embd(self):
assert _backend(embd = 4096)._attention_dequant_width() == 4096
class TestContextBufferDSV4:
"""DeepSeek-V4 reserves a lightning-indexer compute buffer that KQ-mask and MLA rates miss."""
_MEASURED_1M_GIB = 65.5
GIB = 1024**3
def test_covers_measured_1m_buffer(self):
b = _backend(embd = 4096, arch = "deepseek4")
gib = b._compute_buffer_ctx_bytes(1048576, cache_type_kv = "f16") / self.GIB
assert gib >= self._MEASURED_1M_GIB, f"under-reserved {gib:.1f} < {self._MEASURED_1M_GIB}"
def test_not_wildly_over_at_1m(self):
b = _backend(embd = 4096, arch = "deepseek4")
gib = b._compute_buffer_ctx_bytes(1048576, cache_type_kv = "f16") / self.GIB
assert gib <= self._MEASURED_1M_GIB * 1.3
def test_fires_for_f16_cache(self):
dsv4 = _backend(embd = 4096, arch = "deepseek4")._compute_buffer_ctx_bytes(
262144, cache_type_kv = "f16"
)
other = _backend(embd = 4096, arch = "qwen3")._compute_buffer_ctx_bytes(
262144, cache_type_kv = "f16"
)
assert dsv4 > 40 * other
def test_cache_type_independent(self):
b = _backend(embd = 4096, arch = "deepseek4")
assert b._compute_buffer_ctx_bytes(
262144, cache_type_kv = "f16"
) == b._compute_buffer_ctx_bytes(262144, cache_type_kv = "q8_0")
def test_flat_floor_at_small_ctx(self):
b = _backend(embd = 4096, arch = "deepseek4")
assert b._compute_buffer_ctx_bytes(16384, cache_type_kv = "f16") / self.GIB >= 2.0
def test_scales_with_context_and_ubatch(self):
b = _backend(embd = 4096, arch = "deepseek4")
assert b._compute_buffer_ctx_bytes(131072) > b._compute_buffer_ctx_bytes(65536)
assert b._compute_buffer_ctx_bytes(131072, n_ubatch = 1024) > b._compute_buffer_ctx_bytes(
131072, n_ubatch = 256
)
def test_non_dsv4_unchanged(self):
b = _backend(embd = 4096, arch = "llama")
per_tok = b._compute_buffer_ctx_bytes(100000, cache_type_kv = "f16") / 100000
assert per_tok == pytest.approx(512 * 2, rel = 1e-6)
class TestContextBufferDSV41:
"""DeepSeek-V4.1 has a sparse-attention graph: about 2 GiB flat, then superlinear but well below
DSV4."""
MIB = 1024**2
# (n_ctx, n_ubatch): measured MiB
_MEASURED = {
(16384, 512): 2102.26,
(65536, 512): 2198.26,
(262144, 512): 3094.26,
(1048576, 512): 11197.26,
(262144, 2048): 9751.02,
(1048576, 2048): 37143.02,
}
@pytest.mark.parametrize("ctx_ub", list(_MEASURED))
def test_covers_every_measurement(self, ctx_ub):
n_ctx, ub = ctx_ub
b = _backend(embd = 5120, arch = "deepseek41")
mib = b._compute_buffer_ctx_bytes(n_ctx, ub, "f16") / self.MIB
assert mib >= self._MEASURED[ctx_ub], f"under-reserved {mib:.0f} < {self._MEASURED[ctx_ub]}"
def test_not_the_dsv4_rate(self):
# Reusing deepseek4's 72000 B/tok would reserve ~6.6x the measured 1M buffer and
# cut the context the fit grants for nothing; stay within 1.2x of measured.
b = _backend(embd = 5120, arch = "deepseek41")
mib = b._compute_buffer_ctx_bytes(1048576, 512, "f16") / self.MIB
assert mib <= self._MEASURED[(1048576, 512)] * 1.2
def test_layer_split_adds_mask_copies(self):
b = _backend(embd = 5120, arch = "deepseek41")
one = b._compute_buffer_ctx_bytes(262144, 512, "f16")
split = b._compute_buffer_ctx_bytes(262144, 512, "f16", layer_split = True)
assert split - one == (LlamaCppBackend._CTX_COMPUTE_SPLIT_MULT - 1) * 512 * 2 * 262144
def test_flash_attention_off_charges_dense_scores(self):
b = _backend(embd = 5120, arch = "deepseek41")
b._n_heads = 64
b._sliding_window = 128
b._context_length = 1048576
n_ctx = 262144
on = b._compute_buffer_ctx_bytes(n_ctx, 512, "f16")
off = b._compute_buffer_ctx_bytes(n_ctx, 512, "f16", flash_attn = False)
scores = 2 * 512 * 64 * 4
# 768 = window 128 + ub 512, padded.
assert off - on == (scores + 512 * 2) * n_ctx + scores * 768
# Measured flash-off delta on a CPU reserve graph at these shapes.
assert off - on >= 64945 * 1024**2
class TestContextBufferLayerSplit:
"""The per-device f16 (KQ-mask) rate steps 4x once the model spans >1 device under
``-sm layer``: a step on "is split", not a ramp in device count. ``_MEASURED`` is
from llama.cpp's memory breakdown: compute-column ctx slope / n_ctx / n_ubatch."""
_RATE_SINGLE = 2.0
_RATE_SPLIT = 8.0
_MEASURED = [
("Qwen3.5-9B-MTP", 1, 512, 2.0),
("Qwen3.5-9B-MTP", 2, 512, 8.0),
("Qwen3.6-27B-MTP", 1, 512, 2.0),
("Qwen3.6-27B-MTP", 4, 512, 8.0),
("Qwen3.6-35B-A3B", 1, 2048, 2.0),
("Qwen3.6-35B-A3B", 2, 2048, 8.0),
("gemma-4-12b-it", 1, 512, 2.0),
("gemma-4-12b-it", 3, 512, 8.0),
("gemma-4-12b-it", 5, 512, 8.0),
("gemma-4-12b-it", 6, 512, 8.0),
("gemma-4-12b-it", 8, 512, 8.0),
("gemma-4-26B-A4B-it", 1, 512, 2.0),
("gemma-4-26B-A4B-it", 2, 512, 8.0),
("Kimi-K3", 4, 512, 8.0),
]
def test_default_is_single_device(self):
b = _backend(embd = 4096)
assert b._compute_buffer_ctx_bytes(
131072, cache_type_kv = "f16"
) == b._compute_buffer_ctx_bytes(131072, cache_type_kv = "f16", layer_split = False)
def test_split_is_the_multiplier_plus_the_hidden_state_copies(self):
b = _backend(embd = 4096)
one = b._compute_buffer_ctx_bytes(131072, cache_type_kv = "f16")
many = b._compute_buffer_ctx_bytes(131072, cache_type_kv = "f16", layer_split = True)
assert many == one * LlamaCppBackend._CTX_COMPUTE_SPLIT_MULT + _hidden_copies(4096)
def test_matches_the_measured_two_gpu_split(self):
b = _measured("Qwen3-8B")
for ctx, ub, measured in ((32768, 512, 256.04), (131072, 2048, 2560.16)):
per_device = (
b._estimate_compute_buffer_bytes(n_ubatch = ub, n_parallel = 1)
+ b._compute_buffer_ctx_bytes(ctx, ub, "f16", layer_split = True)
) / MIB
assert measured <= per_device <= measured * 1.1
@pytest.mark.parametrize("name,n_gpus,ub,measured", _MEASURED)
def test_upper_bounds_measured_per_device_rate(self, name, n_gpus, ub, measured):
b = _backend(embd = 4096)
per_tok = (
b._compute_buffer_ctx_bytes(
100000, n_ubatch = ub, cache_type_kv = "f16", layer_split = n_gpus > 1
)
/ 100000
)
assert per_tok >= measured * ub, f"{name} n={n_gpus}: {per_tok:.0f} < {measured * ub:.0f}"
@pytest.mark.parametrize("name,n_gpus,ub,measured", _MEASURED)
def test_charges_exactly_the_measured_per_device_rate(self, name, n_gpus, ub, measured):
b = _backend(embd = 4096)
hidden = _hidden_copies(4096, ub) if n_gpus > 1 else 0
charged = b._compute_buffer_ctx_bytes(
100000, n_ubatch = ub, cache_type_kv = "f16", layer_split = n_gpus > 1
)
assert charged - hidden == pytest.approx(measured * ub * 100000, rel = 1e-6)
def test_pre_fix_split_reserve_was_short(self):
b = _backend(embd = 4096)
one = b._compute_buffer_ctx_bytes(1048576, cache_type_kv = "f16")
measured = self._RATE_SPLIT * 512 * 1048576
assert one < measured
assert (
b._compute_buffer_ctx_bytes(1048576, cache_type_kv = "f16", layer_split = True) >= measured
)
def test_kimi_k3_1m_four_gpu_reserve(self):
b = _backend(embd = 7168, mla = 576)
gib = b._compute_buffer_ctx_bytes(1048576, cache_type_kv = "f16", layer_split = True) / (
1024**3
)
assert 4.0 <= gib <= 6.5
@pytest.mark.parametrize("ct", ["q8_0", "q4_0"])
@pytest.mark.parametrize("embd,mla", [(2048, None), (8192, None), (7168, 576)])
def test_quantized_adds_the_mask_delta(self, ct, embd, mla):
# The quantized rate is a single-GPU total; only the mask replicates (3 more on a split).
b = _backend(embd = embd, mla = mla)
mask = b._compute_buffer_ctx_bytes(131072, cache_type_kv = "f16")
single = b._compute_buffer_ctx_bytes(131072, cache_type_kv = ct)
split = b._compute_buffer_ctx_bytes(131072, cache_type_kv = ct, layer_split = True)
delta = (LlamaCppBackend._CTX_COMPUTE_SPLIT_MULT - 1) * mask + _hidden_copies(embd)
assert split == pytest.approx(single + delta, rel = 1e-9)
@pytest.mark.parametrize("embd,mla", [(2048, None), (2560, None), (8192, None), (7168, 576)])
def test_quantized_split_beats_the_old_max_floor(self, embd, mla):
b = _backend(embd = embd, mla = mla)
old_floor = max(
b._compute_buffer_ctx_bytes(262144, cache_type_kv = "q8_0"),
b._compute_buffer_ctx_bytes(262144, cache_type_kv = "f16", layer_split = True),
)
assert b._compute_buffer_ctx_bytes(262144, cache_type_kv = "q8_0", layer_split = True) > (
old_floor
)
def test_quantized_split_covers_measured_plus_mask(self):
b = _backend(embd = 2560)
ctx, ub = 262144, 512
mask_mib = 3 * ub * 2 * ctx / MIB
assert mask_mib == pytest.approx(768.0, rel = 1e-6)
split_mib = b._compute_buffer_ctx_bytes(ctx, ub, "q8_0", layer_split = True) / MIB
assert split_mib >= 1330 + mask_mib
def test_deepseek4_rate_unchanged(self):
b = _backend(embd = 4096, arch = "deepseek4")
assert b._compute_buffer_ctx_bytes(
131072, cache_type_kv = "f16", layer_split = True
) == b._compute_buffer_ctx_bytes(131072, cache_type_kv = "f16")
measured = TestContextBufferDSV4._MEASURED_1M_GIB * 1024**3 / 1048576
split_masks = (LlamaCppBackend._CTX_COMPUTE_SPLIT_MULT - 1) * 512 * 2
assert LlamaCppBackend._DSV4_CTX_COMPUTE_BYTES_PER_TOK - measured >= split_masks
class TestContextBufferInklingSplit:
"""Inkling's rates are single-device totals too, and the banded 8192 B/tok has only
~1.5x headroom over its 5.6 KiB/tok measurement, too little for a split's masks."""
_MEASURED_BANDED = 5734
_CTX = 1048576
_UB = 512
def _rate(
self,
ct,
layer_split,
ub = 512,
):
b = _backend(embd = 4096, arch = "inkling")
return (
b._compute_buffer_ctx_bytes(self._CTX, ub, cache_type_kv = ct, layer_split = layer_split)
/ self._CTX
)
def test_pre_fix_banded_split_reserve_was_short(self):
measured_split = self._MEASURED_BANDED + 3 * self._UB * 2
assert LlamaCppBackend._INKLING_CTX_COMPUTE_BYTES_PER_TOK < measured_split
assert self._rate("f16", True) >= measured_split
@pytest.mark.parametrize("ct", ["f16", None, "q8_0"])
def test_split_adds_exactly_the_extra_mask_copies(self, ct):
delta = (LlamaCppBackend._CTX_COMPUTE_SPLIT_MULT - 1) * self._UB * 2
assert self._rate(ct, True) == pytest.approx(self._rate(ct, False) + delta, rel = 1e-9)
def test_dense_fallback_delta_is_present_but_tiny(self):
single, split = self._rate("q8_0", False), self._rate("q8_0", True)
assert single < split <= single * 1.02
def test_split_delta_scales_with_ubatch(self):
assert self._rate("f16", True, ub = 1024) - self._rate("f16", False, ub = 1024) == (
pytest.approx(2 * (self._rate("f16", True) - self._rate("f16", False)), rel = 1e-9)
)
def test_single_device_rates_unchanged(self):
for ct in ("f16", "q8_0"):
assert self._rate(ct, False) == pytest.approx(
(
LlamaCppBackend._INKLING_CTX_COMPUTE_DENSE_BYTES_PER_TOK
if ct == "q8_0"
else LlamaCppBackend._INKLING_CTX_COMPUTE_BYTES_PER_TOK
),
rel = 1e-6,
)
class TestLayerSplitWiring:
"""``_cc_bytes`` in ``load_model`` learns the device count, so it is the one place
that can set ``layer_split``. Source-level: the closure needs a real load."""
def _cc_bytes_source(self):
import inspect
src = inspect.getsource(LlamaCppBackend.load_model).splitlines()
start = next(i for i, l in enumerate(src) if "def _cc_bytes(" in l)
indent = len(src[start]) - len(src[start].lstrip())
body = [src[start]]
for line in src[start + 1 :]:
closes_signature = line.lstrip().startswith(")")
if line.strip() and len(line) - len(line.lstrip()) <= indent and not closes_signature:
break
body.append(line)
return "\n".join(body)
def test_forwards_layer_split_from_device_count(self):
assert "layer_split = n_gpus > 1" in self._cc_bytes_source()
def test_still_scales_by_device_count(self):
assert "max(1, n_gpus) * self._compute_buffer_ctx_bytes" in self._cc_bytes_source()
class TestPipelineParallelPredicate:
"""Pipeline parallelism's 4x rate applies only with layer split, KV offload and no -ot overrides."""
def _off(
self,
args = None,
env = None,
n_layers = None,
):
from core.inference.llama_cpp import _pipeline_parallel_disabled_by_args
return _pipeline_parallel_disabled_by_args(args, env = env or {}, n_layers = n_layers)
def test_plain_launch_keeps_the_step(self):
assert self._off([]) is False
assert self._off(None) is False
assert self._off(["-c", "131072", "--parallel", "4"]) is False
@pytest.mark.parametrize("flag", ["-ot", "--override-tensor"])
def test_any_tensor_override_disables(self, flag):
assert self._off([flag, "zzz_matches_nothing=CUDA0"]) is True
@pytest.mark.parametrize("flag", ["-nkvo", "--no-kv-offload"])
def test_kv_offload_off_disables(self, flag):
assert self._off([flag]) is True
def test_env_tensor_override_disables(self):
assert self._off([], env = {"LLAMA_ARG_OVERRIDE_TENSOR": "exps=CPU"}) is True
assert self._off([], env = {"LLAMA_ARG_OVERRIDE_TENSOR": ""}) is False
@pytest.mark.parametrize("val", ["off", "0", "false", "disabled"])
def test_env_kv_offload_off_disables(self, val):
assert self._off([], env = {"LLAMA_ARG_KV_OFFLOAD": val}) is True
def test_env_kv_offload_on_keeps_the_step(self):
assert self._off([], env = {"LLAMA_ARG_KV_OFFLOAD": "1"}) is False
def test_unrelated_flags_do_not_disable(self):
assert self._off(["--kv-unified", "-fa", "on", "-ngl", "-1"]) is False
@pytest.mark.parametrize(
"flag", ["-otd", "--override-tensor-draft", "--spec-draft-override-tensor"]
)
def test_draft_override_does_not_disable(self, flag):
assert self._off([flag, "exps=CPU"]) is False
@pytest.mark.parametrize("flag", ["-kvo", "--kv-offload"])
def test_cli_kv_offload_reenable_beats_a_false_env(self, flag):
assert self._off([flag], env = {"LLAMA_ARG_KV_OFFLOAD": "0"}) is False
def test_last_kv_offload_flag_wins(self):
assert self._off(["-kvo", "-nkvo"]) is True
assert self._off(["-nkvo", "-kvo"]) is False
def test_kv_offload_env_junk_value_keeps_the_default(self):
assert self._off([], env = {"LLAMA_ARG_KV_OFFLOAD": "maybe"}) is False
@pytest.mark.parametrize("mode", ["none", "row", "NONE", " row "])
def test_non_layer_split_mode_disables(self, mode):
assert self._off(["-sm", mode]) is True
assert self._off([f"--split-mode={mode}"]) is True
def test_explicit_layer_split_mode_keeps_the_step(self):
assert self._off(["-sm", "layer"]) is False
assert self._off(["-sm", "row", "-sm", "layer"]) is False
def test_tensor_split_mode_keeps_the_step(self):
# The layer branch is an elif on tensor_parallel; a downgrade strips -sm and leaves it pipelined.
assert self._off(["-sm", "tensor"]) is False
def test_layer_branch_is_only_reached_after_the_flag_is_stripped(self):
import inspect
compact = "".join(inspect.getsource(LlamaCppBackend.load_model).split())
assert "iftensor_parallelandtp_gpus:" in compact
assert "extra_args=strip_split_mode_only(" in compact
assert "elifgpusandself._can_estimate_kv()andeffective_ctx>0:" in compact
def test_env_split_mode_is_ignored(self):
assert self._off([], env = {"LLAMA_ARG_SPLIT_MODE": "row"}) is False
def test_layer_path_scrubs_a_non_layer_split_mode_env(self):
import inspect
assert "self._reconcile_split_mode_env(env" in inspect.getsource(LlamaCppBackend.load_model)
compact = "".join(inspect.getsource(LlamaCppBackend._reconcile_split_mode_env).split())
assert 'if_inherited_smand_inherited_sm!="layer":' in compact
assert 'env.pop("LLAMA_ARG_SPLIT_MODE",None)' in compact
@pytest.mark.parametrize("flag", ["-cmoe", "--cpu-moe"])
def test_cpu_moe_disables(self, flag):
assert self._off([flag]) is True
@pytest.mark.parametrize("flag", ["-ncmoe", "--n-cpu-moe"])
def test_n_cpu_moe_disables(self, flag):
assert self._off([flag, "8"]) is True
assert self._off([f"{flag}=8"]) is True
@pytest.mark.parametrize("flag", ["-ncmoe", "--n-cpu-moe"])
def test_n_cpu_moe_zero_keeps_the_step(self, flag):
assert self._off([flag, "0"]) is False
def test_env_cpu_moe_disables(self):
assert self._off([], env = {"LLAMA_ARG_CPU_MOE": "1"}) is True
assert self._off([], env = {"LLAMA_ARG_CPU_MOE": "0"}) is False
assert self._off([], env = {"LLAMA_ARG_CPU_MOE": ""}) is False
def test_env_n_cpu_moe_disables(self):
assert self._off([], env = {"LLAMA_ARG_N_CPU_MOE": "4"}) is True
assert self._off([], env = {"LLAMA_ARG_N_CPU_MOE": "0"}) is False
assert self._off([], env = {"LLAMA_ARG_N_CPU_MOE": "not-a-number"}) is False
@pytest.mark.parametrize("flag", ["-ngl", "--gpu-layers", "--n-gpu-layers"])
def test_finite_gpu_layers_below_the_count_disables(self, flag):
assert self._off([flag, "1"], n_layers = 93) is True
assert self._off([f"{flag}=1"], n_layers = 93) is True
def test_all_layers_keeps_the_step(self):
assert self._off(["-ngl", "-1"], n_layers = 93) is False
def test_gpu_layers_above_the_count_keeps_the_step(self):
assert self._off(["-ngl", "999"], n_layers = 93) is False
def test_gpu_layers_at_the_boundary(self):
# Equal is off; one above keeps the step because block_count can undercount n_layer_all.
assert self._off(["-ngl", "93"], n_layers = 93) is True
assert self._off(["-ngl", "94"], n_layers = 93) is False
def test_zero_gpu_layers_disables(self):
assert self._off(["-ngl", "0"], n_layers = 93) is True
def test_unknown_layer_count_keeps_the_step(self):
assert self._off(["-ngl", "1"]) is False
assert self._off(["-ngl", "1"], n_layers = 0) is False
def test_last_gpu_layers_flag_wins(self):
assert self._off(["-ngl", "1", "--gpu-layers", "-1"], n_layers = 93) is False
assert self._off(["-ngl", "-1", "--gpu-layers", "1"], n_layers = 93) is True
def test_malformed_gpu_layers_keeps_the_step(self):
assert self._off(["-ngl", "abc"], n_layers = 93) is False
assert self._off(["-ngl", "-2"], n_layers = 93) is False
assert self._off(["-ngl"], n_layers = 93) is False
def test_wired_into_the_fit(self):
import inspect
src = inspect.getsource(LlamaCppBackend.load_model)
compact = "".join(src.split())
assert "_pipeline_parallel_disabled_by_args(extra_args,n_layers=self._n_layers)" in compact
assert "layer_split = n_gpus > 1 and not _pipeline_parallel_off" in src
assert src.index("_read_gguf_metadata(model_path)") < src.index(
"_pipeline_parallel_disabled_by_args("
)
class TestPerDeviceSplitReserve:
"""Per-device reserve is replicated on every GPU, so the auto-context check must gate each card."""
_OH = LlamaCppBackend._PIPELINE_PER_DEVICE_OVERHEAD_MIB * MIB
_UB = 2048
_HETEROGENEOUS = ([(0, 40_000), (1, 2_500)], {0: 49_152, 1: 24_576})
_HOMOGENEOUS = ([(0, 40_000), (1, 40_000)], {0: 49_152, 1: 49_152})
def _fit_backend(self, kv_per_tok = 20_000):
b = _backend(embd = 8192)
b._can_estimate_kv = lambda: True
b._estimate_kv_cache_bytes = lambda ctx, ct = None, **kw: max(0, ctx) * kv_per_tok
return b
def _drive(
self,
b,
gpus,
totals,
model_mib,
native_ctx,
min_gpus = 2,
enforce = True,
cap = True,
):
"""Mirrors load_model's auto-context subset loop; returns (gpu_indices, chosen_ctx)."""
frac = LlamaCppBackend._GPU_PIN_VRAM_FRACTION
model_bytes = model_mib * MIB
def usable(g):
return g[1] - (1.0 - frac) * totals[g[0]]
ranked = sorted(gpus, key = usable, reverse = True)
for n in range(min_gpus, len(ranked) + 1):
subset = ranked[:n]
pool = sum(max(0.0, usable(g)) for g in subset)
ms = model_bytes + (n - 1) * self._OH
cc = lambda c, n = n: (
n * b._compute_buffer_ctx_bytes(c, self._UB, "f16", layer_split = n > 1)
)
capped = b._fit_context_to_vram(
native_ctx,
pool,
ms,
"f16",
n_ubatch = self._UB,
compute_ctx_bytes_fn = cc,
budget_frac = 1.0,
total_mib = None,
)
if (ms + b._estimate_kv_cache_bytes(capped) + cc(capped)) / MIB > pool:
continue
usable_mib = [usable(g) for g in subset]
reserve_at = lambda c, n = n: (self._OH if n > 1 else 0) + cc(c, n) // n
if enforce and not LlamaCppBackend._every_gpu_holds_reserve(
usable_mib, reserve_at(capped)
):
if not cap:
continue
capped = LlamaCppBackend._cap_ctx_to_per_device_reserve(
capped, usable_mib, reserve_at
)
if capped <= 0:
continue
if (ms + b._estimate_kv_cache_bytes(capped) + cc(capped)) / MIB > pool:
continue
return sorted(idx for idx, _ in subset), capped
return None, 0
def _drive_reduced(
self,
b,
gpus,
totals,
model_mib,
min_gpus = 2,
enforce = True,
):
"""Mirrors the Auto offload loop: same pooled admission, so it needs the same per-device gate."""
frac = LlamaCppBackend._GPU_PIN_VRAM_FRACTION
ctx = _FIT_MIN_CTX
def usable(g):
return g[1] - (1.0 - frac) * totals[g[0]]
ranked = sorted(gpus, key = usable, reverse = True)
for n in range(min_gpus, len(ranked) + 1):
subset = ranked[:n]
pool = sum(max(0.0, usable(g)) for g in subset)
cc = n * b._compute_buffer_ctx_bytes(ctx, self._UB, "f16", layer_split = n > 1)
ms = model_mib * MIB + (n - 1) * self._OH
if (ms + b._estimate_kv_cache_bytes(ctx) + cc) / MIB > pool:
continue
if enforce and not LlamaCppBackend._every_gpu_holds_reserve(
(usable(g) for g in subset),
(self._OH if n > 1 else 0) + cc // n,
):
continue
return sorted(idx for idx, _ in subset), ctx
return None, 0
def _card_at_floor_reserve(self, b, total_mib, margin_mib):
"""Reserve uses the fit floor context, so a fixed context cannot straddle the gate."""
reserve_mib = (
self._OH + b._compute_buffer_ctx_bytes(_FIT_MIN_CTX, self._UB, "f16", layer_split = True)
) / MIB
return round(reserve_mib + margin_mib + 0.03 * total_mib)
def test_reduced_context_fallback_enforces_the_same_reserve(self):
b = self._fit_backend()
totals = {0: 49_152, 1: 24_576}
gpus = [(0, 40_000), (1, self._card_at_floor_reserve(b, totals[1], -1))]
assert self._drive_reduced(b, gpus, totals, 20_480, enforce = False) == (
[0, 1],
_FIT_MIN_CTX,
)
assert self._drive_reduced(b, gpus, totals, 20_480) == (None, 0)
def test_reduced_context_fallback_keeps_a_card_that_holds_it(self):
b = self._fit_backend()
totals = {0: 49_152, 1: 24_576}
gpus = [(0, 40_000), (1, self._card_at_floor_reserve(b, totals[1], +1))]
assert self._drive_reduced(b, gpus, totals, 20_480) == ([0, 1], _FIT_MIN_CTX)
def test_pooled_budget_hides_the_small_cards_shortfall(self):
b = self._fit_backend()
gpus, totals = self._HETEROGENEOUS
gpu_indices, ctx = self._drive(b, gpus, totals, 20_480, 262144, enforce = False)
assert gpu_indices == [0, 1] and ctx == 262144
reserve_mib = (
self._OH + b._compute_buffer_ctx_bytes(ctx, self._UB, "f16", layer_split = True)
) / MIB
card1_usable = 2_500 - 0.03 * 24_576
assert card1_usable < reserve_mib
def test_subset_is_capped_not_rejected_when_a_card_cannot_hold_its_reserve(self):
b = self._fit_backend()
gpus, totals = self._HETEROGENEOUS
assert self._drive(b, gpus, totals, 20_480, 262144, cap = False) == (None, 0)
assert self._drive(b, gpus, totals, 20_480, 262144) == ([0, 1], 30_720)
def test_homogeneous_gpus_are_unaffected(self):
b = self._fit_backend()
gpus, totals = self._HOMOGENEOUS
with_gate = self._drive(b, gpus, totals, 20_480, 262144)
without = self._drive(b, gpus, totals, 20_480, 262144, enforce = False)
assert with_gate == without == ([0, 1], 262144)
@pytest.mark.parametrize("model_mib", [8_192, 20_480, 30_720])
def test_no_op_when_the_loop_may_start_at_one_gpu(self, model_mib):
b = self._fit_backend()
for gpus, totals in (self._HETEROGENEOUS, self._HOMOGENEOUS):
with_gate = self._drive(b, gpus, totals, model_mib, 262144, min_gpus = 1)
without = self._drive(b, gpus, totals, model_mib, 262144, min_gpus = 1, enforce = False)
assert with_gate == without
assert with_gate[0] is not None
def test_reserve_check_rejects_only_the_short_card(self):
reserve = 3072 * MIB
assert LlamaCppBackend._every_gpu_holds_reserve([4000.0, 3072.0], reserve) is True
assert LlamaCppBackend._every_gpu_holds_reserve([40000.0, 3071.0], reserve) is False
assert LlamaCppBackend._every_gpu_holds_reserve([-10.0], reserve) is False
assert LlamaCppBackend._every_gpu_holds_reserve([], reserve) is False
def test_wired_into_the_auto_context_loop(self):
import inspect
compact = "".join(inspect.getsource(LlamaCppBackend.load_model).split())
# All four gates must hold every GPU's reserve, since pooled VRAM can hide the weakest card.
assert compact.count("ifnotself._every_gpu_holds_reserve(") == 4
assert "_usable_mib=[_gpu_usable(g,_cap_fraction)forginsubset]" in compact
# Gated on the chosen context, and only reachable after the pooled test.
assert "_usable_mib=[_gpu_usable(g,pin_fraction)forginsubset]" in compact
assert "(_gpu_usable(g,pin_fraction)forginsubset)," in compact
assert "+_cc_bytes(c,n)//n)" in compact
assert "+_cc_bytes(effective_ctx,n_gpus)//n_gpus," in compact
# The cap runs only on gate failure, and the pooled price is redone after it
# (the walk's body is a helper since #14492, so a rejected subset returns None).
assert (
compact.count(
"capped=self._cap_ctx_to_per_device_reserve("
"capped,_usable_mib,_reserve_at)ifcapped<=0:returnNone"
)
== 1
)
assert "kv=_kv_bytes(capped)footprint_mib=(_ms+kv+_mtp_bytes(capped)" in compact
class TestPerDeviceReserveCap:
"""Rejecting a subset outright costs context the smallest card could have held.
Reuses the gate class's fixtures; the cap only changes what "reject" means."""
_OH = TestPerDeviceSplitReserve._OH
_UB = TestPerDeviceSplitReserve._UB
_HETEROGENEOUS = TestPerDeviceSplitReserve._HETEROGENEOUS
_HOMOGENEOUS = TestPerDeviceSplitReserve._HOMOGENEOUS
_fit_backend = TestPerDeviceSplitReserve._fit_backend
_drive = TestPerDeviceSplitReserve._drive
def test_cap_is_exact_at_the_256_boundary(self):
b = self._fit_backend()
usable = 2_500 - 0.03 * 24_576
reserve = lambda c: (
(self._OH + b._compute_buffer_ctx_bytes(c, self._UB, "f16", layer_split = True)) / MIB
)
assert reserve(30_720) == 1760.0 <= usable
assert reserve(30_976) == 1764.0 > usable
def test_capped_context_still_fits_the_pooled_budget(self):
b = self._fit_backend()
gpus, totals = self._HETEROGENEOUS
idx, ctx = self._drive(b, gpus, totals, 20_480, 262144)
assert idx == [0, 1] and ctx == 30_720
pool = sum(g[1] - 0.03 * totals[g[0]] for g in gpus)
cc = 2 * b._compute_buffer_ctx_bytes(ctx, self._UB, "f16", layer_split = True)
footprint = (20_480 * MIB + self._OH + b._estimate_kv_cache_bytes(ctx) + cc) / MIB
assert footprint <= pool
def test_homogeneous_gpus_are_unaffected_by_the_cap(self):
b = self._fit_backend()
gpus, totals = self._HOMOGENEOUS
assert self._drive(b, gpus, totals, 20_480, 262144) == ([0, 1], 262144)
_card_at_floor_reserve = TestPerDeviceSplitReserve._card_at_floor_reserve
def test_cap_floors_at_the_fit_minimum_and_still_rejects_below_it(self):
b = self._fit_backend()
totals = {0: 49_152, 1: 24_576}
gpus = [(0, 40_000), (1, self._card_at_floor_reserve(b, totals[1], -1))]
assert self._drive(b, gpus, totals, 20_480, 262144) == (None, 0)
def test_cap_keeps_a_card_that_holds_exactly_the_fit_minimum(self):
b = self._fit_backend()
totals = {0: 49_152, 1: 24_576}
gpus = [(0, 40_000), (1, self._card_at_floor_reserve(b, totals[1], +1))]
assert self._drive(b, gpus, totals, 20_480, 262144) == ([0, 1], _FIT_MIN_CTX)
def test_flat_arch_term_is_not_rate_inverted(self):
b = _backend(embd = 7168, arch = "deepseek4")
reserve = lambda c: self._OH + b._compute_buffer_ctx_bytes(c, 512, "q8_0", layer_split = True)
cap = LlamaCppBackend._cap_ctx_to_per_device_reserve(200_000, [4000.0], reserve)
assert cap == 13_312
assert reserve(cap) / MIB <= 4000.0 < reserve(cap + 256) / MIB
assert reserve(43_341) / MIB > 4000.0
def test_flat_arch_below_the_floor_is_infeasible(self):
b = _backend(embd = 7168, arch = "deepseek4")
reserve = lambda c: self._OH + b._compute_buffer_ctx_bytes(c, 512, "q8_0", layer_split = True)
assert LlamaCppBackend._cap_ctx_to_per_device_reserve(200_000, [2500.0], reserve) == 0
@pytest.mark.parametrize("model_mib", [8_192, 16_384, 20_480, 30_720])
def test_capping_never_loses_to_continuing_with_more_gpus(self, model_mib):
b = self._fit_backend()
gpus = [(0, 40_000), (1, 2_500), (2, 2_400)]
totals = {0: 49_152, 1: 24_576, 2: 24_576}
assert self._drive(b, gpus, totals, model_mib, 262144, cap = False) == (None, 0)
idx, ctx = self._drive(b, gpus, totals, model_mib, 262144)
assert idx == [0, 1] and ctx == 30_720
def test_helper_edges(self):
cap = LlamaCppBackend._cap_ctx_to_per_device_reserve
assert cap(262144, [], lambda c: 0) == 0
assert cap(1024, [4000.0], lambda c: 0) == 0
assert cap(262144, [4000.0], lambda c: 0) == 262144
linear = lambda c: c * 16384
best = cap(262144, [2000.0], linear)
assert best == 128_000 and best % 256 == 0
assert linear(best) / MIB <= 2000.0 < linear(best + 256) / MIB
class TestSplitRateRecheckAfterSelection:
"""A multi-GPU answer must be re-priced at the split compute rate before use_fit is pinned False."""
_OH = LlamaCppBackend._PIPELINE_PER_DEVICE_OVERHEAD_MIB * MIB
_UB = 512
_CTX = 1048576
_FRAC = LlamaCppBackend._GPU_PIN_VRAM_FRACTION
_CARD = 24_576
def _cards(self, n):
return [(i, self._CARD) for i in range(n)], {i: self._CARD for i in range(n)}
def _pin(
self,
total_mib,
n_cards,
recheck = True,
min_gpus = 1,
ctx = None,
):
"""Mirror of the explicit-context branch: total_mib is its weights + KV + MTP
footprint, to which each path adds one single-device compute copy."""
b = _backend(embd = 4096)
ctx = ctx or self._CTX
cc1 = b._compute_buffer_ctx_bytes(ctx, self._UB, "f16")
cc_split = b._compute_buffer_ctx_bytes(ctx, self._UB, "f16", layer_split = True)
gpus, totals = self._cards(n_cards)
return LlamaCppBackend._select_gpus_split_aware(
total_mib * MIB + cc1,
gpus,
usable_fraction = self._FRAC,
total_by_idx = totals,
per_device_overhead_bytes = self._OH + cc1,
min_gpus = min_gpus,
split_extra_bytes = (cc_split - cc1) if recheck else 0,
)
def test_pre_fix_pinned_a_pair_that_cannot_hold_the_split_rate(self):
b = _backend(embd = 4096)
cc1 = b._compute_buffer_ctx_bytes(self._CTX, self._UB, "f16")
ccs = b._compute_buffer_ctx_bytes(self._CTX, self._UB, "f16", layer_split = True)
gpus, totals = self._cards(2)
assert LlamaCppBackend._select_gpus(
40_000 * MIB + cc1,
gpus,
usable_fraction = self._FRAC,
total_by_idx = totals,
per_device_overhead_bytes = self._OH + cc1,
) == ([0, 1], False)
pool = 2 * (self._CARD - (1.0 - self._FRAC) * self._CARD)
assert (40_000 * MIB + ccs + self._OH + ccs) / MIB > pool
def test_recheck_falls_back_to_fit_when_no_subset_holds_it(self):
assert self._pin(40_000, 2) == (None, True)
def test_recheck_widens_the_subset_when_a_card_is_spare(self):
assert self._pin(40_000, 3, recheck = False) == ([0, 1], False)
assert self._pin(40_000, 3) == ([0, 1, 2], False)
def test_single_gpu_pin_is_untouched(self):
assert self._pin(20_000, 2) == ([0], False)
assert self._pin(20_000, 2, recheck = False) == ([0], False)
def test_equal_cards_never_collapse_to_one_gpu(self):
for total in range(24_000, 46_000, 2_000):
gi, use_fit = self._pin(total, 4)
assert use_fit or (gi is not None and len(gi) >= 2)
def test_collapse_to_one_gpu_is_repriced_without_the_delta(self):
# A lone card is not a split and pays no delta.
gpus = [(0, 16 * 1024), (1, 3 * 1024)]
kw = dict(usable_fraction = 1.0, per_device_overhead_bytes = int(2.5 * GIB))
assert LlamaCppBackend._select_gpus(int(14 * GIB), gpus, min_gpus = 2, **kw) == (
[0, 1],
False,
)
assert LlamaCppBackend._select_gpus_split_aware(
int(14 * GIB), gpus, min_gpus = 2, split_extra_bytes = int(4.5 * GIB), **kw
) == ([0], False)
assert LlamaCppBackend._select_gpus(int(14 * GIB), gpus, min_gpus = 1, **kw) == (
[0],
False,
)
def test_reprice_still_reports_fit_when_no_single_card_holds_it(self):
assert LlamaCppBackend._select_gpus_split_aware(
int(30 * GIB),
[(0, 20 * 1024), (1, 20 * 1024)],
usable_fraction = 1.0,
per_device_overhead_bytes = int(1 * GIB),
min_gpus = 2,
split_extra_bytes = int(15 * GIB),
) == (None, True)
def test_zero_step_reduces_to_plain_selection(self):
b = _backend(embd = 4096)
cc1 = b._compute_buffer_ctx_bytes(self._CTX, self._UB, "f16")
gpus, totals = self._cards(3)
for total_mib in (20_000, 40_000, 90_000):
assert self._pin(total_mib, 3, recheck = False) == LlamaCppBackend._select_gpus(
total_mib * MIB + cc1,
gpus,
usable_fraction = self._FRAC,
total_by_idx = totals,
per_device_overhead_bytes = self._OH + cc1,
)
def test_small_context_is_unaffected(self):
assert self._pin(40_000, 2, ctx = 4096) == self._pin(40_000, 2, recheck = False, ctx = 4096)
def test_wired_into_every_call_site(self):
import ast
import inspect
import re
import textwrap
source = inspect.getsource(LlamaCppBackend.load_model)
load = "".join(source.split())
# Read call sites from the AST: names for each site's context are not the contract.
wired = [
ast.unparse(keyword.value)
for node in ast.walk(ast.parse(textwrap.dedent(source)))
if isinstance(node, ast.Call)
for keyword in node.keywords
if keyword.arg in ("split_extra_bytes", "split_extra_for_slots")
]
# Five sites; a sixth has to say which context it prices at.
assert len(wired) == 5, wired
for expression in wired:
assert re.fullmatch(
r"_cc_split_extra\(\w+\)|lambda s: _cc_split_extra\(\w+, s\)"
r"|_cc_split_extra\(\w+, cache_type='q8_0'\)",
expression,
), expression
assert "gpu_indices,use_fit=self._select_gpus_split_aware(" in load
assert (
"returnmax(0,_cc_bytes(ctx,2,slots,cache_type)//2-_cc_bytes(ctx,1,slots,cache_type),)"
in load
)
slots = "".join(inspect.getsource(LlamaCppBackend._slots_that_fit_on_gpu).split())
assert "self._select_gpus_split_aware(" in slots
assert (
"split_extra_bytes=(split_extra_for_slots(slots)ifsplit_extra_for_slotselsesplit_extra_bytes),"
in slots
)
# Scratch rate keys off the lighter KV axis; the budget's heavier scalar hides a q4_0 K.
class TestScratchTakesTheLighterAxis:
"""_planned_scratch_cache_type: KV bytes use the heavier axis, dequant scratch the lighter."""
@pytest.mark.parametrize(
"k,v,expected",
[
("f16", "f16", "f16"),
("q8_0", "q8_0", "q8_0"),
("q4_0", "f16", "q4_0"),
("f16", "q4_0", "q4_0"),
("q8_0", "q4_0", "q4_0"),
("f32", "q8_0", "q8_0"),
],
)
def test_it_picks_the_quantized_axis_whichever_side_it_is_on(self, k, v, expected):
extras = ["--cache-type-k", k, "--cache-type-v", v]
assert _planned_scratch_cache_type(None, extras) == expected
heavier = max(_planned_main_cache_types(None, extras), key = _kv_bytes_per_elem)
assert (heavier != expected) == (_kv_bytes_per_elem(k) != _kv_bytes_per_elem(v))
def test_a_managed_symmetric_request_leaves_both_terms_equal(self):
"""Unsloth emits one type on both axes, so a plain q8_0 load's fit must not move."""
for kv in ("f16", "q8_0", "q4_0", "iq4_nl"):
assert _planned_scratch_cache_type(kv, None) == kv
def test_extras_beat_the_managed_field_per_axis(self):
"""Extras are appended last and win per axis, so the scratch follows them, not the UI."""
assert _planned_scratch_cache_type("f16", ["--cache-type-k", "q4_0"]) == "q4_0"
def test_it_reads_the_inherited_env_when_nothing_else_sets_a_type(self):
env = {"LLAMA_ARG_CACHE_TYPE_K": "q4_0", "LLAMA_ARG_CACHE_TYPE_V": "f16"}
assert _planned_scratch_cache_type(None, None, env) == "q4_0"
@pytest.mark.parametrize("k,v", [("q4_0", "f16"), ("f16", "q4_0")])
def test_the_asymmetric_pair_selects_the_dequant_rate(self, k, v):
"""An asymmetric pair must price its quantized axis's dequant, not the KQ-mask floor."""
b = _backend()
extras = ["--cache-type-k", k, "--cache-type-v", v]
heavier = max(_planned_main_cache_types(None, extras), key = _kv_bytes_per_elem)
lighter = _planned_scratch_cache_type(None, extras)
heavy_rate = b._compute_buffer_ctx_bytes(131_072, 2048, heavier)
light_rate = b._compute_buffer_ctx_bytes(131_072, 2048, lighter)
assert light_rate > heavy_rate, (light_rate, heavy_rate)
assert light_rate == b._compute_buffer_ctx_bytes(131_072, 2048, "q4_0")
class TestTensorFitPricesTheQuantizedAxis:
"""The advertised context must not exceed what an honest per-axis price allows; tensor mode has
no --fit valve, so an optimistic context OOMs at startup instead of spilling."""
@staticmethod
def _plan(
b,
cache_type,
scratch_type,
ub,
ngpu = 4,
per_gpu = 48_000,
):
return b._plan_tensor_parallel(
gpus = [(i, per_gpu) for i in range(ngpu)],
model_size = 60 * 1024**3,
target_ctx = 262_144,
max_target_ctx = 262_144,
total_by_idx = {i: per_gpu for i in range(ngpu)},
cache_type_kv = cache_type,
scratch_cache_type_kv = scratch_type,
n_parallel = 1,
swa_full = False,
kv_unified = True,
n_ubatch = ub,
flash_attn = False,
)
def test_an_asymmetric_pair_is_capped_like_its_quantized_axis(self):
"""With -ctk q4_0 -ctv f16 at a raised micro-batch, the quantized axis caps the advertised
context."""
b = _backend_from_gguf_local()
optimistic = self._plan(b, "f16", None, 2048, per_gpu = 36_000)[0]
honest = self._plan(b, "f16", "q4_0", 2048, per_gpu = 36_000)[0]
assert honest < optimistic, (honest, optimistic)
assert honest > 2048, honest
# Below symmetric q4_0 on purpose: f16 KV bytes on both axes plus full dequant scratch.
assert honest < self._plan(b, "q4_0", "q4_0", 2048, per_gpu = 36_000)[0]
def test_a_symmetric_request_is_unchanged(self):
"""Default and explicit scratch agree when both axes match, so no existing load moves."""
b = _backend_from_gguf_local()
for kv in ("f16", "q8_0", "q4_0"):
for ub in (512, 2048):
assert self._plan(b, kv, None, ub) == self._plan(b, kv, kv, ub)
class TestSpeculativeComputeBuffers:
"""Draft buffers must cover growth past the 224 MiB floor; measured on Qwen3.8 27B's head."""
def _qwen38(self):
b = _backend(vocab = 248320, embd = 5120)
b._feed_forward_length = 17408
return b
@pytest.mark.parametrize(
"ub,ctx,measured", [(512, 32768, 130.02), (1120, 32768, 284.41), (1120, 131072, 428.78)]
)
def test_floor_plus_growth_covers_the_draft_context(self, ub, ctx, measured):
b = self._qwen38()
charged = (
LlamaCppBackend._MTP_DRAFT_COMPUTE_BYTES + b._mtp_draft_compute_bytes(ctx, n_ubatch = ub)
) / MIB
assert charged >= measured
def test_short_context_and_micro_batch_stay_on_the_floor(self):
assert self._qwen38()._mtp_draft_compute_bytes(4096, n_ubatch = 256) == 0
def test_growth_follows_the_mask_once_past_the_floor(self):
b = self._qwen38()
lo = b._mtp_draft_compute_bytes(131072, n_ubatch = 1120)
hi = b._mtp_draft_compute_bytes(262144, n_ubatch = 1120)
assert hi - lo == 1120 * 2 * 131072
def test_verify_rows_scale_with_slots_and_depth(self):
b = self._qwen38()
base = b._mtp_draft_compute_bytes(4096, n_ubatch = 256)
rows = b._mtp_draft_compute_bytes(4096, n_ubatch = 256, n_parallel = 4, spec_draft_n_max = 2)
assert rows - base == 4 * 2 * LlamaCppBackend._SPEC_VERIFY_ROW_COPIES * 248320 * 4
def test_the_reserve_carries_it(self):
b = self._qwen38()
b._n_layers, b._n_kv_heads, b._kv_key_length, b._kv_value_length = 65, 4, 256, 256
b._nextn_predict_layers = 1
kwargs = dict(n_parallel = 1, target_rollback = False, mtp_keeps_target_ctx = False)
short = b._estimate_mtp_overhead_bytes(4096, n_ubatch = 256, **kwargs)
long = b._estimate_mtp_overhead_bytes(262144, n_ubatch = 1120, **kwargs)
draft_kv = b._mtp_draft_kv_bytes(262144, n_ubatch = 1120) - b._mtp_draft_kv_bytes(
4096, n_ubatch = 256
)
assert long - short == draft_kv + b._mtp_draft_compute_bytes(262144, n_ubatch = 1120)
class TestFlashAttentionOff:
"""Without flash attention llama.cpp materialises [n_kv, n_ubatch, n_head] float32 scores."""
_MEASURED = [
("Llama-3.2-1B", 512, 4, 32768, 2130.01),
("Llama-3.2-1B", 2048, 4, 32768, 8520.02),
("Llama-3.2-1B", 512, 4, 65536, 4242.01),
("Qwen3-8B", 1120, 4, 32768, 4698.76),
("Qwen3.5-4B", 512, 4, 65536, 2214.01),
("Qwen3.6-35B-A3B", 2048, 4, 32768, 4504.18),
("Gemma-4-12B", 1120, 4, 32768, 2540.8),
("Gemma-4-12B", 512, 4, 65536, 2248.01),
("Gemma-4-E4B", 512, 4, 65536, 1217.51),
("GLM-4.7-Flash", 1120, 4, 32768, 3086.3),
("GLM-4.7-Flash", 512, 4, 65536, 2818.9),
]
@pytest.mark.parametrize("name,ub,slots,ctx,measured", _MEASURED)
def test_covers_the_measured_buffer(self, name, ub, slots, ctx, measured):
b = _measured(name)
est = (
b._estimate_compute_buffer_bytes(n_ubatch = ub, n_parallel = slots)
+ b._compute_buffer_ctx_bytes(ctx, ub, "f16", flash_attn = False)
) / MIB
assert measured <= est <= measured * 1.1, f"{name}: {est:.0f} vs {measured}"
def test_the_rate_is_the_score_matrix_plus_a_float32_mask(self):
b = _measured("Gemma-4-12B")
on = b._compute_buffer_ctx_bytes(100000, 1120, "f16")
off = b._compute_buffer_ctx_bytes(100000, 1120, "f16", flash_attn = False)
assert on == 1120 * 2 * 100000
assert off == 1120 * (4 + 16 * 4) * 100000
class TestLayerSplitSlidingWindowSlots:
"""A layer split replicates sliding-window masks per GPU, each holding the window for every slot."""
@pytest.mark.parametrize("slots,ctx,measured", [(8, 65536, 971), (16, 131072, 1601)])
def test_covers_the_measured_split(self, slots, ctx, measured):
b = _measured("Gemma-4-12B")
per_device = (
b._estimate_compute_buffer_bytes(n_ubatch = 1120, n_parallel = slots)
+ b._compute_buffer_ctx_bytes(ctx, 1120, "f16", layer_split = True, n_parallel = slots)
) / MIB
one_slot_window = (
b._estimate_compute_buffer_bytes(n_ubatch = 1120, n_parallel = slots)
+ b._compute_buffer_ctx_bytes(ctx, 1120, "f16", layer_split = True)
) / MIB
assert one_slot_window < measured <= per_device <= measured * 1.1
class TestOutputRowsByBuild:
"""Builds before b9460 reserved an output row per micro-batch token, whatever the slot count."""
@pytest.mark.parametrize(
"output,build",
[
("version: 0.4.0-dev (build 10909, commit 329b6160f)\nbuilt with GNU", 10909),
("version: 9415 (06d26dfdf)\nbuilt with GNU 11.4.0", 9415),
("version: 1 (unknown)\n", None),
("no version here", None),
],
)
def test_the_build_is_read_from_either_version_format(
self, tmp_path, monkeypatch, output, build
):
import subprocess
import types
binary = tmp_path / "llama-server"
binary.write_bytes(b"")
monkeypatch.setattr(LlamaCppBackend, "_build_number_cache", {})
monkeypatch.setattr(
subprocess,
"run",
lambda *a, **k: types.SimpleNamespace(stdout = output, stderr = "", returncode = 0),
)
assert LlamaCppBackend.probe_build_number(str(binary)) == build
@pytest.mark.parametrize(
"build,legacy", [(9415, True), (9459, True), (9460, False), (None, False)]
)
def test_only_a_known_older_build_reserves_micro_batch_rows(self, monkeypatch, build, legacy):
monkeypatch.setattr(
LlamaCppBackend, "probe_build_number", classmethod(lambda cls, binary = None: build)
)
assert LlamaCppBackend.reserves_micro_batch_outputs() is legacy
def test_an_older_build_is_charged_a_row_per_micro_batch_token(self):
# Measured b9415 Vulkan compute buffers (MiB) at 32768 cells, one and four slots.
cases = [
(dict(vocab = 128256, embd = 2048, ff = 8192), 512, 254.50),
(dict(vocab = 151936, embd = 4096, ff = 12288), 2048, 1667.05),
(dict(vocab = 262144, embd = 1536, ff = 12288, ple = 256 * 35), 2048, 2704.08),
]
for dims, ub, measured in cases:
b = _backend(vocab = dims["vocab"], embd = dims["embd"])
b._feed_forward_length = dims["ff"]
b._embedding_length_per_layer_input = 0
b._n_layers = 1
if "ple" in dims:
b._embedding_length_per_layer_input, b._n_layers = 256, 35
b._reserves_micro_batch_outputs = True
for slots in (1, 4):
est = (
b._estimate_compute_buffer_bytes(n_ubatch = ub, n_parallel = slots)
+ b._compute_buffer_ctx_bytes(32768, ub, "f16")
) / MIB
assert measured <= est <= measured * 1.5, (dims, slots, est)
b._reserves_micro_batch_outputs = False
assert b._estimate_compute_buffer_bytes(n_ubatch = ub, n_parallel = 1) < (
dims["vocab"] * ub * 4
)