Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

65.0 KB
Raw Blame
# SPDX-License-Identifier: AGPL-3.0-only
# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0

"""Compute-buffer estimates checked against measured allocations and synthetic layouts."""

from __future__ import annotations

import sys
import types as _types
from pathlib import Path

import pytest

_BACKEND_DIR = str(Path(__file__).resolve().parent.parent)
if _BACKEND_DIR not in sys.path:
    sys.path.insert(0, _BACKEND_DIR)

_loggers_stub = _types.ModuleType("loggers")
_loggers_stub.get_logger = lambda name: __import__("logging").getLogger(name)
sys.modules.setdefault("loggers", _loggers_stub)
_structlog_stub = _types.ModuleType("structlog")
_structlog_stub.get_logger = lambda *a, **k: __import__("logging").getLogger("stub")
sys.modules.setdefault("structlog", _structlog_stub)
# Only stub httpx when missing: a stub leaks via sys.modules and breaks huggingface_hub imports.
try:
    import httpx as _httpx_real  # noqa: F401
except ImportError:
    _httpx_stub = _types.ModuleType("httpx")
    for _exc in (
        "ConnectError",
        "TimeoutException",
        "ReadTimeout",
        "ReadError",
        "RemoteProtocolError",
        "CloseError",
        "HTTPError",
        "RequestError",
    ):
        setattr(_httpx_stub, _exc, type(_exc, (Exception,), {}))
    _httpx_stub.Timeout = type("T", (), {"__init__": lambda s, *a, **k: None})
    _httpx_stub.Response = type("Response", (), {})
    _httpx_stub.Client = type(
        "C",
        (),
        {
            "__init__": lambda s, **kw: None,
            "__enter__": lambda s: s,
            "__exit__": lambda s, *a: None,
        },
    )
    sys.modules["httpx"] = _httpx_stub

from core.inference.llama_cpp import (
    _FIT_MIN_CTX,
    _kv_bytes_per_elem,
    _planned_main_cache_types,
    _planned_scratch_cache_type,
    LlamaCppBackend,
)

MIB = 1024 * 1024
GIB = 1024 * MIB


def _backend(
    vocab = 248320,
    embd = 5120,
    mla = None,
    arch = None,
    pooling_type = None,
):
    """Backend with just the dims the compute-buffer estimate reads."""
    b = LlamaCppBackend.__new__(LlamaCppBackend)
    b._vocab_size = vocab
    b._embedding_length = embd
    b._key_length_mla = mla
    b._architecture = arch
    b._pooling_type = pooling_type
    return b


def _pad(cells):
    return ((cells + 255) // 256) * 256


def _hidden_copies(embd, ub = 512):
    """The hidden-state copies a layer-split device holds, one per scheduler copy."""
    return LlamaCppBackend._CTX_COMPUTE_SPLIT_MULT * embd * ub * 4


def _backend_from_gguf_local(
    n_layers = 80,
    embd = 8192,
    n_kv_heads = 8,
    head_dim = 128,
):
    """Builds a real __init__ backend so every attribute the KV estimator reads has its default."""
    b = LlamaCppBackend()
    b._vocab_size = 248_320
    b._embedding_length = embd
    b._n_layers = n_layers
    b._n_kv_heads = n_kv_heads
    b._kv_key_length = head_dim
    b._kv_value_length = head_dim
    b._context_length = 262_144
    assert b._can_estimate_kv()
    return b


_GEMMA4_12B_PATTERN = ([True] * 5 + [False]) * 8

# Measured on bundled llama.cpp b10909 (RTX 6000 Ada, --flash-attn on, --kv-unified).
_DIMS = {
    "Llama-3.2-1B": dict(
        _vocab_size = 128256,
        _embedding_length = 2048,
        _feed_forward_length = 8192,
        _n_layers = 16,
        _n_heads = 32,
        _n_kv_heads = 8,
        _kv_key_length = 64,
        _kv_value_length = 64,
    ),
    "Qwen3-8B": dict(
        _vocab_size = 151936,
        _embedding_length = 4096,
        _feed_forward_length = 12288,
        _n_layers = 36,
        _n_heads = 32,
        _n_kv_heads = 8,
        _kv_key_length = 128,
        _kv_value_length = 128,
    ),
    "Qwen3.5-4B": dict(
        _vocab_size = 248320,
        _embedding_length = 2560,
        _feed_forward_length = 9216,
        _n_layers = 32,
        _n_heads = 16,
        _n_kv_heads = 4,
        _kv_key_length = 256,
        _kv_value_length = 256,
        _ssm_inner_size = 4096,
        _ssm_state_size = 128,
        _ssm_group_count = 16,
        _full_attention_interval = 4,
    ),
    "Qwen3.6-35B-A3B": dict(
        _vocab_size = 248320,
        _embedding_length = 2048,
        _expert_used_count = 8,
        _expert_feed_forward_length = 512,
        _expert_shared_feed_forward_length = 512,
        _n_layers = 40,
        _n_heads = 16,
        _n_kv_heads = 2,
        _kv_key_length = 256,
        _kv_value_length = 256,
        _ssm_inner_size = 4096,
        _ssm_state_size = 128,
        _ssm_group_count = 16,
        _full_attention_interval = 4,
    ),
    "Gemma-4-12B": dict(
        _vocab_size = 262144,
        _embedding_length = 3840,
        _feed_forward_length = 15360,
        _n_layers = 48,
        _n_heads = 16,
        _n_kv_heads = 8,
        _n_kv_heads_by_layer = [1 if not swa else 8 for swa in _GEMMA4_12B_PATTERN],
        _kv_key_length = 512,
        _kv_value_length = 512,
        _kv_key_length_swa = 256,
        _kv_value_length_swa = 256,
        _sliding_window = 1024,
        _sliding_window_pattern = _GEMMA4_12B_PATTERN,
    ),
    "Gemma-4-E4B": dict(
        _vocab_size = 262144,
        _embedding_length = 2560,
        _feed_forward_length = 10240,
        _embedding_length_per_layer_input = 256,
        _n_layers = 42,
        _n_heads = 8,
        _n_kv_heads = 2,
        _kv_key_length = 512,
        _kv_value_length = 512,
        _kv_key_length_swa = 256,
        _kv_value_length_swa = 256,
        _sliding_window = 512,
        _sliding_window_pattern = ([True] * 5 + [False]) * 7,
    ),
    "GLM-4.7-Flash": dict(
        _vocab_size = 154880,
        _embedding_length = 2048,
        _feed_forward_length = 10240,
        _expert_used_count = 4,
        _expert_feed_forward_length = 1536,
        _expert_shared_count = 1,
        _n_layers = 47,
        _n_heads = 20,
        _n_kv_heads = 1,
        _kv_key_length = 576,
        _kv_value_length = 512,
        _kv_lora_rank = 512,
        _key_length_mla = 256,
    ),
}


def _measured(name):
    b = LlamaCppBackend()
    for attr, value in _DIMS[name].items():
        setattr(b, attr, value)
    return b


_MEASURED_COMPUTE = [
    ("Llama-3.2-1B", 256, 4, 32768, "f16", 46.01),
    ("Llama-3.2-1B", 2048, 4, 32768, "f16", 368.04),
    ("Llama-3.2-1B", 512, 1, 32768, "f16", 92.01),
    ("Llama-3.2-1B", 512, 8, 32768, "f16", 92.01),
    ("Llama-3.2-1B", 512, 4, 131072, "f16", 188.01),
    ("Llama-3.2-1B", 512, 4, 131072, "q8_0", 400.04),
    ("Llama-3.2-1B", 2048, 4, 131072, "q8_0", 832.07),
    ("Qwen3-8B", 512, 4, 32768, "f16", 128.01),
    ("Qwen3-8B", 2048, 4, 131072, "f16", 896.04),
    ("Qwen3-8B", 512, 4, 131072, "q8_0", 672.09),
    ("Qwen3.5-4B", 512, 1, 32768, "f16", 96.02),
    ("Qwen3.5-4B", 512, 8, 32768, "f16", 115.52),
    ("Qwen3.5-4B", 2048, 4, 131072, "f16", 768.06),
    ("Qwen3.5-4B", 2048, 4, 131072, "q8_0", 1224.34),
    ("Qwen3.6-35B-A3B", 256, 4, 32768, "f16", 66.02),
    ("Qwen3.6-35B-A3B", 2048, 1, 32768, "f16", 528.2),
    ("Qwen3.6-35B-A3B", 512, 4, 131072, "q8_0", 436.31),
    ("Gemma-4-12B", 512, 1, 32768, "f16", 154.52),
    ("Gemma-4-12B", 1120, 4, 32768, "f16", 346.21),
    ("Gemma-4-12B", 2048, 8, 32768, "f16", 652.07),
    ("Gemma-4-12B", 2048, 4, 131072, "f16", 1020.07),
    ("Gemma-4-12B", 512, 4, 131072, "q8_0", 436.8),
    ("Gemma-4-E4B", 512, 4, 32768, "f16", 135.52),
    ("Gemma-4-E4B", 2048, 4, 131072, "q8_0", 1317.35),
    ("GLM-4.7-Flash", 512, 4, 32768, "f16", 110.01),
    ("GLM-4.7-Flash", 2048, 4, 131072, "f16", 824.04),
    ("GLM-4.7-Flash", 512, 4, 131072, "q8_0", 358.9),
]


def _priced_mib(b, ub, slots, ctx, kv):
    flat = b._estimate_compute_buffer_bytes(n_ubatch = ub, n_parallel = slots)
    return (flat + b._compute_buffer_ctx_bytes(ctx, ub, kv)) / MIB


class TestMeasuredComputeBuffer:
    """Flat plus context-linear must match llama-server's reserve, not the old per-slot vocab rows."""

    @pytest.mark.parametrize("name,ub,slots,ctx,kv,measured", _MEASURED_COMPUTE)
    def test_covers_the_measured_buffer(self, name, ub, slots, ctx, kv, measured):
        est = _priced_mib(_measured(name), ub, slots, ctx, kv)
        assert est >= measured, f"{name} ub{ub} p{slots} {ctx} {kv}: {est:.0f} < {measured}"

    @pytest.mark.parametrize("name,ub,slots,ctx,kv,measured", _MEASURED_COMPUTE)
    def test_stays_close_to_the_measured_buffer(self, name, ub, slots, ctx, kv, measured):
        est = _priced_mib(_measured(name), ub, slots, ctx, kv)
        assert est <= measured * (1.35 if kv == "f16" else 1.3), f"{name}: {est:.0f} vs {measured}"

    def test_the_reported_gemma4_vision_raise(self):
        b = _measured("Gemma-4-12B")
        raise_mib = _priced_mib(b, 1120, 4, 262144, "f16") - _priced_mib(b, 512, 4, 262144, "f16")
        assert 455 <= raise_mib <= 455 * 1.15


class TestFlatBufferShape:
    def test_slots_add_output_rows_not_activations(self):
        b = _measured("Gemma-4-12B")
        one = b._estimate_compute_buffer_bytes(n_ubatch = 2048, n_parallel = 1)
        eight = b._estimate_compute_buffer_bytes(n_ubatch = 2048, n_parallel = 8)
        rows = 7 * 262144 * 4
        swa = (_pad(1024 * 8 + 2048) - _pad(1024 + 2048)) * 2048 * 2
        assert eight - one == pytest.approx(
            rows * LlamaCppBackend._COMPUTE_BUFFER_SAFETY + swa, rel = 1e-6
        )

    def test_scales_linearly_with_ubatch(self):
        b = _measured("Qwen3-8B")
        lo = b._estimate_compute_buffer_bytes(n_ubatch = 512, n_parallel = 1)
        hi = b._estimate_compute_buffer_bytes(n_ubatch = 1024, n_parallel = 1)
        rows = 151936 * 4 * LlamaCppBackend._COMPUTE_BUFFER_SAFETY
        assert hi - rows == pytest.approx(2 * (lo - rows), rel = 1e-6)

    def test_a_tensor_device_holds_the_single_device_buffer(self):
        for name in ("Qwen3-8B", "Gemma-4-12B"):
            b = _measured(name)
            assert b._estimate_compute_buffer_bytes(
                n_ubatch = 2048, n_parallel = 4, per_device_tensor = True
            ) == b._estimate_compute_buffer_bytes(n_ubatch = 2048, n_parallel = 4)

    def test_embedding_rows_are_pooled_not_vocab_wide(self):
        b = _backend(vocab = 30522, embd = 384, pooling_type = 1)
        chat = _backend(vocab = 30522, embd = 384)
        assert b._estimate_compute_buffer_bytes(n_ubatch = 512, n_parallel = 4) > (
            chat._estimate_compute_buffer_bytes(n_ubatch = 512, n_parallel = 4)
        )
        assert b._estimate_compute_buffer_bytes(n_ubatch = 512, n_parallel = 4) < 30522 * 512 * 4

    def test_widest_block_sets_the_width(self):
        assert _measured("Qwen3-8B")._compute_activation_width() == 4 * 12288
        assert _measured("Qwen3.6-35B-A3B")._compute_activation_width() == (
            8 * (2 * 2048 + 3 * 512) + 3 * 512
        )
        ssm = _backend(embd = 2560)
        ssm._ssm_inner_size, ssm._ssm_state_size, ssm._ssm_group_count = 4096, 128, 16
        assert ssm._compute_activation_width() == 4 * (4096 + 2 * 16 * 128)
        assert _measured("Gemma-4-E4B")._compute_activation_width() == 4 * 10240 + 256 * 42

    def test_header_without_ffn_dims_uses_the_embedding_floor(self):
        assert _backend(embd = 4096)._compute_activation_width() == 12 * 4096

    def test_expert_width_falls_back_to_the_ffn_length(self):
        b = _backend(vocab = 100352, embd = 1536)
        b._n_experts, b._expert_used_count = 64, 6
        b._feed_forward_length, b._expert_shared_feed_forward_length = 512, 1024
        assert b._compute_activation_width() == 6 * (2 * 1536 + 3 * 512) + 3 * 1024

    def test_granite_hybrid_is_covered(self):
        b = _backend(vocab = 100352, embd = 1536)
        b._n_experts, b._expert_used_count = 64, 6
        b._feed_forward_length, b._expert_shared_feed_forward_length = 512, 1024
        b._ssm_inner_size, b._ssm_state_size, b._ssm_group_count = 3072, 128, 1
        for slots, measured in ((1, 80.0), (8, 95.3)):
            est = _priced_mib(b, 512, slots, 32768, "f16")
            assert measured <= est <= measured * 1.3

    def test_a_window_wider_than_the_context_does_not_slide(self):
        b = _backend(vocab = 200064, embd = 3072)
        b._feed_forward_length, b._context_length = 8192, 131072
        b._sliding_window, b._sliding_window_pattern = 262144, None
        assert b._effective_sliding_window() == 0
        assert 92.0 <= _priced_mib(b, 512, 4, 32768, "f16") <= 92.0 * 1.3
        b._sliding_window_pattern = [False] * 32
        assert b._effective_sliding_window() == 0
        b._sliding_window_pattern = [True, False] * 16
        assert b._effective_sliding_window() == 262144


class TestFallback:
    def test_zero_when_vocab_missing(self):
        assert _backend(vocab = None)._estimate_compute_buffer_bytes(n_parallel = 4) == 0

    def test_vocab_ceiling_prices_a_header_without_a_token_array(self):
        b = _backend(vocab = None)
        assert b._estimate_compute_buffer_bytes(n_parallel = 4, vocab_ceiling = 262144) == (
            _backend(vocab = 262144)._estimate_compute_buffer_bytes(n_parallel = 4)
        )

    def test_zero_when_embd_missing(self):
        assert _backend(embd = None)._estimate_compute_buffer_bytes(n_parallel = 4) == 0

    def test_zero_lets_tensor_plan_use_flat_fallback(self):
        b = _backend(vocab = None, embd = None)
        b._n_layers = None
        ec, mac, gi, ts = b._plan_tensor_parallel([(0, 48000), (1, 48000)], 8 * 1024**3, 8192)
        assert gi == [0, 1]


class TestContextLinearBuffer:
    """``_compute_buffer_ctx_bytes``: the KQ mask, n_ubatch * 2 bytes per cell, plus
    for a quantized cache an f16 copy of the widest global layer's K and V."""

    # (model, n_embd, ctx, measured CUDA0 compute MiB, q8_0/ub512); older build, upper bounds.
    _MEASURED = [
        ("Qwen3.5-2B", 2048, 262144, 796),
        ("Qwen3.5-4B", 2560, 262144, 1330),
        ("Qwen3.5-9B", 4096, 262144, 1336),
        ("Qwen3.6-27B", 5120, 262144, 1360),
        ("Gemma-4-31B", 5376, 262144, 2392),
    ]

    def test_zero_by_default(self):
        assert _backend()._compute_buffer_ctx_bytes(0) == 0

    def test_zero_when_embd_missing(self):
        assert _backend(embd = None)._compute_buffer_ctx_bytes(262144) == 0

    def test_grows_linearly_with_context(self):
        b = _measured("Qwen3-8B")
        for kv in ("f16", "q8_0"):
            a = b._compute_buffer_ctx_bytes(65536, cache_type_kv = kv)
            d = b._compute_buffer_ctx_bytes(131072, cache_type_kv = kv)
            assert d == pytest.approx(2 * a, rel = 1e-6)

    @pytest.mark.parametrize("name,embd,ctx,measured", _MEASURED)
    def test_upper_bounds_measured_compute_growth(self, name, embd, ctx, measured):
        b = _backend(embd = embd)
        flat = b._estimate_compute_buffer_bytes(n_parallel = 1)
        total = (flat + b._compute_buffer_ctx_bytes(ctx, cache_type_kv = "q8_0")) / MIB
        assert total >= measured, f"{name}: under-reserved {total:.0f} < {measured}"


class TestContextBufferKVQuant:
    def test_quantized_heavier_than_f16(self):
        b = _measured("Qwen3-8B")
        assert b._compute_buffer_ctx_bytes(131072, cache_type_kv = "q8_0") > (
            b._compute_buffer_ctx_bytes(131072, cache_type_kv = "f16")
        )

    def test_none_cache_type_is_f16(self):
        b = _measured("Qwen3-8B")
        assert b._compute_buffer_ctx_bytes(131072, cache_type_kv = None) == (
            b._compute_buffer_ctx_bytes(131072, cache_type_kv = "f16")
        )

    @pytest.mark.parametrize("ct", ["f16", "bf16", "f32"])
    def test_unquantized_pays_exactly_the_mask(self, ct):
        for name in _DIMS:
            b = _measured(name)
            for ub in (512, 2048):
                assert b._compute_buffer_ctx_bytes(100000, ub, ct) == ub * 2 * 100000

    @pytest.mark.parametrize("ct", ["q8_0", "q5_1", "q4_0", "iq4_nl"])
    def test_quantized_adds_the_widest_kv_copy_whatever_the_ubatch(self, ct):
        b = _measured("Qwen3-8B")
        for ub in (512, 2048):
            extra = b._compute_buffer_ctx_bytes(100000, ub, ct) - ub * 2 * 100000
            assert extra == 8 * (128 + 128) * 2 * 100000

    def test_sliding_window_layers_do_not_set_the_rate(self):
        assert _measured("Gemma-4-12B")._attention_dequant_width() == 1 * (512 + 512)

    def test_mla_copies_the_compressed_key_only(self):
        assert _measured("GLM-4.7-Flash")._attention_dequant_width() == 576

    def test_header_without_head_dims_falls_back_to_embd(self):
        assert _backend(embd = 4096)._attention_dequant_width() == 4096


class TestContextBufferDSV4:
    """DeepSeek-V4 reserves a lightning-indexer compute buffer that KQ-mask and MLA rates miss."""

    _MEASURED_1M_GIB = 65.5
    GIB = 1024**3

    def test_covers_measured_1m_buffer(self):
        b = _backend(embd = 4096, arch = "deepseek4")
        gib = b._compute_buffer_ctx_bytes(1048576, cache_type_kv = "f16") / self.GIB
        assert gib >= self._MEASURED_1M_GIB, f"under-reserved {gib:.1f} < {self._MEASURED_1M_GIB}"

    def test_not_wildly_over_at_1m(self):
        b = _backend(embd = 4096, arch = "deepseek4")
        gib = b._compute_buffer_ctx_bytes(1048576, cache_type_kv = "f16") / self.GIB
        assert gib <= self._MEASURED_1M_GIB * 1.3

    def test_fires_for_f16_cache(self):
        dsv4 = _backend(embd = 4096, arch = "deepseek4")._compute_buffer_ctx_bytes(
            262144, cache_type_kv = "f16"
        )
        other = _backend(embd = 4096, arch = "qwen3")._compute_buffer_ctx_bytes(
            262144, cache_type_kv = "f16"
        )
        assert dsv4 > 40 * other

    def test_cache_type_independent(self):
        b = _backend(embd = 4096, arch = "deepseek4")
        assert b._compute_buffer_ctx_bytes(
            262144, cache_type_kv = "f16"
        ) == b._compute_buffer_ctx_bytes(262144, cache_type_kv = "q8_0")

    def test_flat_floor_at_small_ctx(self):
        b = _backend(embd = 4096, arch = "deepseek4")
        assert b._compute_buffer_ctx_bytes(16384, cache_type_kv = "f16") / self.GIB >= 2.0

    def test_scales_with_context_and_ubatch(self):
        b = _backend(embd = 4096, arch = "deepseek4")
        assert b._compute_buffer_ctx_bytes(131072) > b._compute_buffer_ctx_bytes(65536)
        assert b._compute_buffer_ctx_bytes(131072, n_ubatch = 1024) > b._compute_buffer_ctx_bytes(
            131072, n_ubatch = 256
        )

    def test_non_dsv4_unchanged(self):
        b = _backend(embd = 4096, arch = "llama")
        per_tok = b._compute_buffer_ctx_bytes(100000, cache_type_kv = "f16") / 100000
        assert per_tok == pytest.approx(512 * 2, rel = 1e-6)


class TestContextBufferDSV41:
    """DeepSeek-V4.1 has a sparse-attention graph: about 2 GiB flat, then superlinear but well below
    DSV4."""

    MIB = 1024**2
    # (n_ctx, n_ubatch): measured MiB
    _MEASURED = {
        (16384, 512): 2102.26,
        (65536, 512): 2198.26,
        (262144, 512): 3094.26,
        (1048576, 512): 11197.26,
        (262144, 2048): 9751.02,
        (1048576, 2048): 37143.02,
    }

    @pytest.mark.parametrize("ctx_ub", list(_MEASURED))
    def test_covers_every_measurement(self, ctx_ub):
        n_ctx, ub = ctx_ub
        b = _backend(embd = 5120, arch = "deepseek41")
        mib = b._compute_buffer_ctx_bytes(n_ctx, ub, "f16") / self.MIB
        assert mib >= self._MEASURED[ctx_ub], f"under-reserved {mib:.0f} < {self._MEASURED[ctx_ub]}"

    def test_not_the_dsv4_rate(self):
        # Reusing deepseek4's 72000 B/tok would reserve ~6.6x the measured 1M buffer and
        # cut the context the fit grants for nothing; stay within 1.2x of measured.
        b = _backend(embd = 5120, arch = "deepseek41")
        mib = b._compute_buffer_ctx_bytes(1048576, 512, "f16") / self.MIB
        assert mib <= self._MEASURED[(1048576, 512)] * 1.2

    def test_layer_split_adds_mask_copies(self):
        b = _backend(embd = 5120, arch = "deepseek41")
        one = b._compute_buffer_ctx_bytes(262144, 512, "f16")
        split = b._compute_buffer_ctx_bytes(262144, 512, "f16", layer_split = True)
        assert split - one == (LlamaCppBackend._CTX_COMPUTE_SPLIT_MULT - 1) * 512 * 2 * 262144

    def test_flash_attention_off_charges_dense_scores(self):
        b = _backend(embd = 5120, arch = "deepseek41")
        b._n_heads = 64
        b._sliding_window = 128
        b._context_length = 1048576
        n_ctx = 262144
        on = b._compute_buffer_ctx_bytes(n_ctx, 512, "f16")
        off = b._compute_buffer_ctx_bytes(n_ctx, 512, "f16", flash_attn = False)
        scores = 2 * 512 * 64 * 4
        # 768 = window 128 + ub 512, padded.
        assert off - on == (scores + 512 * 2) * n_ctx + scores * 768
        # Measured flash-off delta on a CPU reserve graph at these shapes.
        assert off - on >= 64945 * 1024**2


class TestContextBufferLayerSplit:
    """The per-device f16 (KQ-mask) rate steps 4x once the model spans >1 device under
    ``-sm layer``: a step on "is split", not a ramp in device count. ``_MEASURED`` is
    from llama.cpp's memory breakdown: compute-column ctx slope / n_ctx / n_ubatch."""

    _RATE_SINGLE = 2.0
    _RATE_SPLIT = 8.0
    _MEASURED = [
        ("Qwen3.5-9B-MTP", 1, 512, 2.0),
        ("Qwen3.5-9B-MTP", 2, 512, 8.0),
        ("Qwen3.6-27B-MTP", 1, 512, 2.0),
        ("Qwen3.6-27B-MTP", 4, 512, 8.0),
        ("Qwen3.6-35B-A3B", 1, 2048, 2.0),
        ("Qwen3.6-35B-A3B", 2, 2048, 8.0),
        ("gemma-4-12b-it", 1, 512, 2.0),
        ("gemma-4-12b-it", 3, 512, 8.0),
        ("gemma-4-12b-it", 5, 512, 8.0),
        ("gemma-4-12b-it", 6, 512, 8.0),
        ("gemma-4-12b-it", 8, 512, 8.0),
        ("gemma-4-26B-A4B-it", 1, 512, 2.0),
        ("gemma-4-26B-A4B-it", 2, 512, 8.0),
        ("Kimi-K3", 4, 512, 8.0),
    ]

    def test_default_is_single_device(self):
        b = _backend(embd = 4096)
        assert b._compute_buffer_ctx_bytes(
            131072, cache_type_kv = "f16"
        ) == b._compute_buffer_ctx_bytes(131072, cache_type_kv = "f16", layer_split = False)

    def test_split_is_the_multiplier_plus_the_hidden_state_copies(self):
        b = _backend(embd = 4096)
        one = b._compute_buffer_ctx_bytes(131072, cache_type_kv = "f16")
        many = b._compute_buffer_ctx_bytes(131072, cache_type_kv = "f16", layer_split = True)
        assert many == one * LlamaCppBackend._CTX_COMPUTE_SPLIT_MULT + _hidden_copies(4096)

    def test_matches_the_measured_two_gpu_split(self):
        b = _measured("Qwen3-8B")
        for ctx, ub, measured in ((32768, 512, 256.04), (131072, 2048, 2560.16)):
            per_device = (
                b._estimate_compute_buffer_bytes(n_ubatch = ub, n_parallel = 1)
                + b._compute_buffer_ctx_bytes(ctx, ub, "f16", layer_split = True)
            ) / MIB
            assert measured <= per_device <= measured * 1.1

    @pytest.mark.parametrize("name,n_gpus,ub,measured", _MEASURED)
    def test_upper_bounds_measured_per_device_rate(self, name, n_gpus, ub, measured):
        b = _backend(embd = 4096)
        per_tok = (
            b._compute_buffer_ctx_bytes(
                100000, n_ubatch = ub, cache_type_kv = "f16", layer_split = n_gpus > 1
            )
            / 100000
        )
        assert per_tok >= measured * ub, f"{name} n={n_gpus}: {per_tok:.0f} < {measured * ub:.0f}"

    @pytest.mark.parametrize("name,n_gpus,ub,measured", _MEASURED)
    def test_charges_exactly_the_measured_per_device_rate(self, name, n_gpus, ub, measured):
        b = _backend(embd = 4096)
        hidden = _hidden_copies(4096, ub) if n_gpus > 1 else 0
        charged = b._compute_buffer_ctx_bytes(
            100000, n_ubatch = ub, cache_type_kv = "f16", layer_split = n_gpus > 1
        )
        assert charged - hidden == pytest.approx(measured * ub * 100000, rel = 1e-6)

    def test_pre_fix_split_reserve_was_short(self):
        b = _backend(embd = 4096)
        one = b._compute_buffer_ctx_bytes(1048576, cache_type_kv = "f16")
        measured = self._RATE_SPLIT * 512 * 1048576
        assert one < measured
        assert (
            b._compute_buffer_ctx_bytes(1048576, cache_type_kv = "f16", layer_split = True) >= measured
        )

    def test_kimi_k3_1m_four_gpu_reserve(self):
        b = _backend(embd = 7168, mla = 576)
        gib = b._compute_buffer_ctx_bytes(1048576, cache_type_kv = "f16", layer_split = True) / (
            1024**3
        )
        assert 4.0 <= gib <= 6.5

    @pytest.mark.parametrize("ct", ["q8_0", "q4_0"])
    @pytest.mark.parametrize("embd,mla", [(2048, None), (8192, None), (7168, 576)])
    def test_quantized_adds_the_mask_delta(self, ct, embd, mla):
        # The quantized rate is a single-GPU total; only the mask replicates (3 more on a split).
        b = _backend(embd = embd, mla = mla)
        mask = b._compute_buffer_ctx_bytes(131072, cache_type_kv = "f16")
        single = b._compute_buffer_ctx_bytes(131072, cache_type_kv = ct)
        split = b._compute_buffer_ctx_bytes(131072, cache_type_kv = ct, layer_split = True)
        delta = (LlamaCppBackend._CTX_COMPUTE_SPLIT_MULT - 1) * mask + _hidden_copies(embd)
        assert split == pytest.approx(single + delta, rel = 1e-9)

    @pytest.mark.parametrize("embd,mla", [(2048, None), (2560, None), (8192, None), (7168, 576)])
    def test_quantized_split_beats_the_old_max_floor(self, embd, mla):
        b = _backend(embd = embd, mla = mla)
        old_floor = max(
            b._compute_buffer_ctx_bytes(262144, cache_type_kv = "q8_0"),
            b._compute_buffer_ctx_bytes(262144, cache_type_kv = "f16", layer_split = True),
        )
        assert b._compute_buffer_ctx_bytes(262144, cache_type_kv = "q8_0", layer_split = True) > (
            old_floor
        )

    def test_quantized_split_covers_measured_plus_mask(self):
        b = _backend(embd = 2560)
        ctx, ub = 262144, 512
        mask_mib = 3 * ub * 2 * ctx / MIB
        assert mask_mib == pytest.approx(768.0, rel = 1e-6)
        split_mib = b._compute_buffer_ctx_bytes(ctx, ub, "q8_0", layer_split = True) / MIB
        assert split_mib >= 1330 + mask_mib

    def test_deepseek4_rate_unchanged(self):
        b = _backend(embd = 4096, arch = "deepseek4")
        assert b._compute_buffer_ctx_bytes(
            131072, cache_type_kv = "f16", layer_split = True
        ) == b._compute_buffer_ctx_bytes(131072, cache_type_kv = "f16")
        measured = TestContextBufferDSV4._MEASURED_1M_GIB * 1024**3 / 1048576
        split_masks = (LlamaCppBackend._CTX_COMPUTE_SPLIT_MULT - 1) * 512 * 2
        assert LlamaCppBackend._DSV4_CTX_COMPUTE_BYTES_PER_TOK - measured >= split_masks


class TestContextBufferInklingSplit:
    """Inkling's rates are single-device totals too, and the banded 8192 B/tok has only
    ~1.5x headroom over its 5.6 KiB/tok measurement, too little for a split's masks."""

    _MEASURED_BANDED = 5734
    _CTX = 1048576
    _UB = 512

    def _rate(
        self,
        ct,
        layer_split,
        ub = 512,
    ):
        b = _backend(embd = 4096, arch = "inkling")
        return (
            b._compute_buffer_ctx_bytes(self._CTX, ub, cache_type_kv = ct, layer_split = layer_split)
            / self._CTX
        )

    def test_pre_fix_banded_split_reserve_was_short(self):
        measured_split = self._MEASURED_BANDED + 3 * self._UB * 2
        assert LlamaCppBackend._INKLING_CTX_COMPUTE_BYTES_PER_TOK < measured_split
        assert self._rate("f16", True) >= measured_split

    @pytest.mark.parametrize("ct", ["f16", None, "q8_0"])
    def test_split_adds_exactly_the_extra_mask_copies(self, ct):
        delta = (LlamaCppBackend._CTX_COMPUTE_SPLIT_MULT - 1) * self._UB * 2
        assert self._rate(ct, True) == pytest.approx(self._rate(ct, False) + delta, rel = 1e-9)

    def test_dense_fallback_delta_is_present_but_tiny(self):
        single, split = self._rate("q8_0", False), self._rate("q8_0", True)
        assert single < split <= single * 1.02

    def test_split_delta_scales_with_ubatch(self):
        assert self._rate("f16", True, ub = 1024) - self._rate("f16", False, ub = 1024) == (
            pytest.approx(2 * (self._rate("f16", True) - self._rate("f16", False)), rel = 1e-9)
        )

    def test_single_device_rates_unchanged(self):
        for ct in ("f16", "q8_0"):
            assert self._rate(ct, False) == pytest.approx(
                (
                    LlamaCppBackend._INKLING_CTX_COMPUTE_DENSE_BYTES_PER_TOK
                    if ct == "q8_0"
                    else LlamaCppBackend._INKLING_CTX_COMPUTE_BYTES_PER_TOK
                ),
                rel = 1e-6,
            )


class TestLayerSplitWiring:
    """``_cc_bytes`` in ``load_model`` learns the device count, so it is the one place
    that can set ``layer_split``. Source-level: the closure needs a real load."""

    def _cc_bytes_source(self):
        import inspect

        src = inspect.getsource(LlamaCppBackend.load_model).splitlines()
        start = next(i for i, l in enumerate(src) if "def _cc_bytes(" in l)
        indent = len(src[start]) - len(src[start].lstrip())
        body = [src[start]]
        for line in src[start + 1 :]:
            closes_signature = line.lstrip().startswith(")")
            if line.strip() and len(line) - len(line.lstrip()) <= indent and not closes_signature:
                break
            body.append(line)
        return "\n".join(body)

    def test_forwards_layer_split_from_device_count(self):
        assert "layer_split = n_gpus > 1" in self._cc_bytes_source()

    def test_still_scales_by_device_count(self):
        assert "max(1, n_gpus) * self._compute_buffer_ctx_bytes" in self._cc_bytes_source()


class TestPipelineParallelPredicate:
    """Pipeline parallelism's 4x rate applies only with layer split, KV offload and no -ot overrides."""

    def _off(
        self,
        args = None,
        env = None,
        n_layers = None,
    ):
        from core.inference.llama_cpp import _pipeline_parallel_disabled_by_args
        return _pipeline_parallel_disabled_by_args(args, env = env or {}, n_layers = n_layers)

    def test_plain_launch_keeps_the_step(self):
        assert self._off([]) is False
        assert self._off(None) is False
        assert self._off(["-c", "131072", "--parallel", "4"]) is False

    @pytest.mark.parametrize("flag", ["-ot", "--override-tensor"])
    def test_any_tensor_override_disables(self, flag):
        assert self._off([flag, "zzz_matches_nothing=CUDA0"]) is True

    @pytest.mark.parametrize("flag", ["-nkvo", "--no-kv-offload"])
    def test_kv_offload_off_disables(self, flag):
        assert self._off([flag]) is True

    def test_env_tensor_override_disables(self):
        assert self._off([], env = {"LLAMA_ARG_OVERRIDE_TENSOR": "exps=CPU"}) is True
        assert self._off([], env = {"LLAMA_ARG_OVERRIDE_TENSOR": ""}) is False

    @pytest.mark.parametrize("val", ["off", "0", "false", "disabled"])
    def test_env_kv_offload_off_disables(self, val):
        assert self._off([], env = {"LLAMA_ARG_KV_OFFLOAD": val}) is True

    def test_env_kv_offload_on_keeps_the_step(self):
        assert self._off([], env = {"LLAMA_ARG_KV_OFFLOAD": "1"}) is False

    def test_unrelated_flags_do_not_disable(self):
        assert self._off(["--kv-unified", "-fa", "on", "-ngl", "-1"]) is False

    @pytest.mark.parametrize(
        "flag", ["-otd", "--override-tensor-draft", "--spec-draft-override-tensor"]
    )
    def test_draft_override_does_not_disable(self, flag):
        assert self._off([flag, "exps=CPU"]) is False

    @pytest.mark.parametrize("flag", ["-kvo", "--kv-offload"])
    def test_cli_kv_offload_reenable_beats_a_false_env(self, flag):
        assert self._off([flag], env = {"LLAMA_ARG_KV_OFFLOAD": "0"}) is False

    def test_last_kv_offload_flag_wins(self):
        assert self._off(["-kvo", "-nkvo"]) is True
        assert self._off(["-nkvo", "-kvo"]) is False

    def test_kv_offload_env_junk_value_keeps_the_default(self):
        assert self._off([], env = {"LLAMA_ARG_KV_OFFLOAD": "maybe"}) is False

    @pytest.mark.parametrize("mode", ["none", "row", "NONE", " row "])
    def test_non_layer_split_mode_disables(self, mode):
        assert self._off(["-sm", mode]) is True
        assert self._off([f"--split-mode={mode}"]) is True

    def test_explicit_layer_split_mode_keeps_the_step(self):
        assert self._off(["-sm", "layer"]) is False
        assert self._off(["-sm", "row", "-sm", "layer"]) is False

    def test_tensor_split_mode_keeps_the_step(self):
        # The layer branch is an elif on tensor_parallel; a downgrade strips -sm and leaves it pipelined.
        assert self._off(["-sm", "tensor"]) is False

    def test_layer_branch_is_only_reached_after_the_flag_is_stripped(self):
        import inspect

        compact = "".join(inspect.getsource(LlamaCppBackend.load_model).split())
        assert "iftensor_parallelandtp_gpus:" in compact
        assert "extra_args=strip_split_mode_only(" in compact
        assert "elifgpusandself._can_estimate_kv()andeffective_ctx>0:" in compact

    def test_env_split_mode_is_ignored(self):
        assert self._off([], env = {"LLAMA_ARG_SPLIT_MODE": "row"}) is False

    def test_layer_path_scrubs_a_non_layer_split_mode_env(self):
        import inspect

        assert "self._reconcile_split_mode_env(env" in inspect.getsource(LlamaCppBackend.load_model)
        compact = "".join(inspect.getsource(LlamaCppBackend._reconcile_split_mode_env).split())
        assert 'if_inherited_smand_inherited_sm!="layer":' in compact
        assert 'env.pop("LLAMA_ARG_SPLIT_MODE",None)' in compact

    @pytest.mark.parametrize("flag", ["-cmoe", "--cpu-moe"])
    def test_cpu_moe_disables(self, flag):
        assert self._off([flag]) is True

    @pytest.mark.parametrize("flag", ["-ncmoe", "--n-cpu-moe"])
    def test_n_cpu_moe_disables(self, flag):
        assert self._off([flag, "8"]) is True
        assert self._off([f"{flag}=8"]) is True

    @pytest.mark.parametrize("flag", ["-ncmoe", "--n-cpu-moe"])
    def test_n_cpu_moe_zero_keeps_the_step(self, flag):
        assert self._off([flag, "0"]) is False

    def test_env_cpu_moe_disables(self):
        assert self._off([], env = {"LLAMA_ARG_CPU_MOE": "1"}) is True
        assert self._off([], env = {"LLAMA_ARG_CPU_MOE": "0"}) is False
        assert self._off([], env = {"LLAMA_ARG_CPU_MOE": ""}) is False

    def test_env_n_cpu_moe_disables(self):
        assert self._off([], env = {"LLAMA_ARG_N_CPU_MOE": "4"}) is True
        assert self._off([], env = {"LLAMA_ARG_N_CPU_MOE": "0"}) is False
        assert self._off([], env = {"LLAMA_ARG_N_CPU_MOE": "not-a-number"}) is False

    @pytest.mark.parametrize("flag", ["-ngl", "--gpu-layers", "--n-gpu-layers"])
    def test_finite_gpu_layers_below_the_count_disables(self, flag):
        assert self._off([flag, "1"], n_layers = 93) is True
        assert self._off([f"{flag}=1"], n_layers = 93) is True

    def test_all_layers_keeps_the_step(self):
        assert self._off(["-ngl", "-1"], n_layers = 93) is False

    def test_gpu_layers_above_the_count_keeps_the_step(self):
        assert self._off(["-ngl", "999"], n_layers = 93) is False

    def test_gpu_layers_at_the_boundary(self):
        # Equal is off; one above keeps the step because block_count can undercount n_layer_all.
        assert self._off(["-ngl", "93"], n_layers = 93) is True
        assert self._off(["-ngl", "94"], n_layers = 93) is False

    def test_zero_gpu_layers_disables(self):
        assert self._off(["-ngl", "0"], n_layers = 93) is True

    def test_unknown_layer_count_keeps_the_step(self):
        assert self._off(["-ngl", "1"]) is False
        assert self._off(["-ngl", "1"], n_layers = 0) is False

    def test_last_gpu_layers_flag_wins(self):
        assert self._off(["-ngl", "1", "--gpu-layers", "-1"], n_layers = 93) is False
        assert self._off(["-ngl", "-1", "--gpu-layers", "1"], n_layers = 93) is True

    def test_malformed_gpu_layers_keeps_the_step(self):
        assert self._off(["-ngl", "abc"], n_layers = 93) is False
        assert self._off(["-ngl", "-2"], n_layers = 93) is False
        assert self._off(["-ngl"], n_layers = 93) is False

    def test_wired_into_the_fit(self):
        import inspect

        src = inspect.getsource(LlamaCppBackend.load_model)
        compact = "".join(src.split())
        assert "_pipeline_parallel_disabled_by_args(extra_args,n_layers=self._n_layers)" in compact
        assert "layer_split = n_gpus > 1 and not _pipeline_parallel_off" in src
        assert src.index("_read_gguf_metadata(model_path)") < src.index(
            "_pipeline_parallel_disabled_by_args("
        )


class TestPerDeviceSplitReserve:
    """Per-device reserve is replicated on every GPU, so the auto-context check must gate each card."""

    _OH = LlamaCppBackend._PIPELINE_PER_DEVICE_OVERHEAD_MIB * MIB
    _UB = 2048
    _HETEROGENEOUS = ([(0, 40_000), (1, 2_500)], {0: 49_152, 1: 24_576})
    _HOMOGENEOUS = ([(0, 40_000), (1, 40_000)], {0: 49_152, 1: 49_152})

    def _fit_backend(self, kv_per_tok = 20_000):
        b = _backend(embd = 8192)
        b._can_estimate_kv = lambda: True
        b._estimate_kv_cache_bytes = lambda ctx, ct = None, **kw: max(0, ctx) * kv_per_tok
        return b

    def _drive(
        self,
        b,
        gpus,
        totals,
        model_mib,
        native_ctx,
        min_gpus = 2,
        enforce = True,
        cap = True,
    ):
        """Mirrors load_model's auto-context subset loop; returns (gpu_indices, chosen_ctx)."""
        frac = LlamaCppBackend._GPU_PIN_VRAM_FRACTION
        model_bytes = model_mib * MIB

        def usable(g):
            return g[1] - (1.0 - frac) * totals[g[0]]

        ranked = sorted(gpus, key = usable, reverse = True)
        for n in range(min_gpus, len(ranked) + 1):
            subset = ranked[:n]
            pool = sum(max(0.0, usable(g)) for g in subset)
            ms = model_bytes + (n - 1) * self._OH
            cc = lambda c, n = n: (
                n * b._compute_buffer_ctx_bytes(c, self._UB, "f16", layer_split = n > 1)
            )
            capped = b._fit_context_to_vram(
                native_ctx,
                pool,
                ms,
                "f16",
                n_ubatch = self._UB,
                compute_ctx_bytes_fn = cc,
                budget_frac = 1.0,
                total_mib = None,
            )
            if (ms + b._estimate_kv_cache_bytes(capped) + cc(capped)) / MIB > pool:
                continue
            usable_mib = [usable(g) for g in subset]
            reserve_at = lambda c, n = n: (self._OH if n > 1 else 0) + cc(c, n) // n
            if enforce and not LlamaCppBackend._every_gpu_holds_reserve(
                usable_mib, reserve_at(capped)
            ):
                if not cap:
                    continue
                capped = LlamaCppBackend._cap_ctx_to_per_device_reserve(
                    capped, usable_mib, reserve_at
                )
                if capped <= 0:
                    continue
                if (ms + b._estimate_kv_cache_bytes(capped) + cc(capped)) / MIB > pool:
                    continue
            return sorted(idx for idx, _ in subset), capped
        return None, 0

    def _drive_reduced(
        self,
        b,
        gpus,
        totals,
        model_mib,
        min_gpus = 2,
        enforce = True,
    ):
        """Mirrors the Auto offload loop: same pooled admission, so it needs the same per-device gate."""
        frac = LlamaCppBackend._GPU_PIN_VRAM_FRACTION
        ctx = _FIT_MIN_CTX

        def usable(g):
            return g[1] - (1.0 - frac) * totals[g[0]]

        ranked = sorted(gpus, key = usable, reverse = True)
        for n in range(min_gpus, len(ranked) + 1):
            subset = ranked[:n]
            pool = sum(max(0.0, usable(g)) for g in subset)
            cc = n * b._compute_buffer_ctx_bytes(ctx, self._UB, "f16", layer_split = n > 1)
            ms = model_mib * MIB + (n - 1) * self._OH
            if (ms + b._estimate_kv_cache_bytes(ctx) + cc) / MIB > pool:
                continue
            if enforce and not LlamaCppBackend._every_gpu_holds_reserve(
                (usable(g) for g in subset),
                (self._OH if n > 1 else 0) + cc // n,
            ):
                continue
            return sorted(idx for idx, _ in subset), ctx
        return None, 0

    def _card_at_floor_reserve(self, b, total_mib, margin_mib):
        """Reserve uses the fit floor context, so a fixed context cannot straddle the gate."""
        reserve_mib = (
            self._OH + b._compute_buffer_ctx_bytes(_FIT_MIN_CTX, self._UB, "f16", layer_split = True)
        ) / MIB
        return round(reserve_mib + margin_mib + 0.03 * total_mib)

    def test_reduced_context_fallback_enforces_the_same_reserve(self):
        b = self._fit_backend()
        totals = {0: 49_152, 1: 24_576}
        gpus = [(0, 40_000), (1, self._card_at_floor_reserve(b, totals[1], -1))]
        assert self._drive_reduced(b, gpus, totals, 20_480, enforce = False) == (
            [0, 1],
            _FIT_MIN_CTX,
        )
        assert self._drive_reduced(b, gpus, totals, 20_480) == (None, 0)

    def test_reduced_context_fallback_keeps_a_card_that_holds_it(self):
        b = self._fit_backend()
        totals = {0: 49_152, 1: 24_576}
        gpus = [(0, 40_000), (1, self._card_at_floor_reserve(b, totals[1], +1))]
        assert self._drive_reduced(b, gpus, totals, 20_480) == ([0, 1], _FIT_MIN_CTX)

    def test_pooled_budget_hides_the_small_cards_shortfall(self):
        b = self._fit_backend()
        gpus, totals = self._HETEROGENEOUS
        gpu_indices, ctx = self._drive(b, gpus, totals, 20_480, 262144, enforce = False)
        assert gpu_indices == [0, 1] and ctx == 262144
        reserve_mib = (
            self._OH + b._compute_buffer_ctx_bytes(ctx, self._UB, "f16", layer_split = True)
        ) / MIB
        card1_usable = 2_500 - 0.03 * 24_576
        assert card1_usable < reserve_mib

    def test_subset_is_capped_not_rejected_when_a_card_cannot_hold_its_reserve(self):
        b = self._fit_backend()
        gpus, totals = self._HETEROGENEOUS
        assert self._drive(b, gpus, totals, 20_480, 262144, cap = False) == (None, 0)
        assert self._drive(b, gpus, totals, 20_480, 262144) == ([0, 1], 30_720)

    def test_homogeneous_gpus_are_unaffected(self):
        b = self._fit_backend()
        gpus, totals = self._HOMOGENEOUS
        with_gate = self._drive(b, gpus, totals, 20_480, 262144)
        without = self._drive(b, gpus, totals, 20_480, 262144, enforce = False)
        assert with_gate == without == ([0, 1], 262144)

    @pytest.mark.parametrize("model_mib", [8_192, 20_480, 30_720])
    def test_no_op_when_the_loop_may_start_at_one_gpu(self, model_mib):
        b = self._fit_backend()
        for gpus, totals in (self._HETEROGENEOUS, self._HOMOGENEOUS):
            with_gate = self._drive(b, gpus, totals, model_mib, 262144, min_gpus = 1)
            without = self._drive(b, gpus, totals, model_mib, 262144, min_gpus = 1, enforce = False)
            assert with_gate == without
            assert with_gate[0] is not None

    def test_reserve_check_rejects_only_the_short_card(self):
        reserve = 3072 * MIB
        assert LlamaCppBackend._every_gpu_holds_reserve([4000.0, 3072.0], reserve) is True
        assert LlamaCppBackend._every_gpu_holds_reserve([40000.0, 3071.0], reserve) is False
        assert LlamaCppBackend._every_gpu_holds_reserve([-10.0], reserve) is False
        assert LlamaCppBackend._every_gpu_holds_reserve([], reserve) is False

    def test_wired_into_the_auto_context_loop(self):
        import inspect

        compact = "".join(inspect.getsource(LlamaCppBackend.load_model).split())
        # All four gates must hold every GPU's reserve, since pooled VRAM can hide the weakest card.
        assert compact.count("ifnotself._every_gpu_holds_reserve(") == 4
        assert "_usable_mib=[_gpu_usable(g,_cap_fraction)forginsubset]" in compact
        # Gated on the chosen context, and only reachable after the pooled test.
        assert "_usable_mib=[_gpu_usable(g,pin_fraction)forginsubset]" in compact
        assert "(_gpu_usable(g,pin_fraction)forginsubset)," in compact
        assert "+_cc_bytes(c,n)//n)" in compact
        assert "+_cc_bytes(effective_ctx,n_gpus)//n_gpus," in compact
        # The cap runs only on gate failure, and the pooled price is redone after it
        # (the walk's body is a helper since #14492, so a rejected subset returns None).
        assert (
            compact.count(
                "capped=self._cap_ctx_to_per_device_reserve("
                "capped,_usable_mib,_reserve_at)ifcapped<=0:returnNone"
            )
            == 1
        )
        assert "kv=_kv_bytes(capped)footprint_mib=(_ms+kv+_mtp_bytes(capped)" in compact


class TestPerDeviceReserveCap:
    """Rejecting a subset outright costs context the smallest card could have held.
    Reuses the gate class's fixtures; the cap only changes what "reject" means."""

    _OH = TestPerDeviceSplitReserve._OH
    _UB = TestPerDeviceSplitReserve._UB
    _HETEROGENEOUS = TestPerDeviceSplitReserve._HETEROGENEOUS
    _HOMOGENEOUS = TestPerDeviceSplitReserve._HOMOGENEOUS
    _fit_backend = TestPerDeviceSplitReserve._fit_backend
    _drive = TestPerDeviceSplitReserve._drive

    def test_cap_is_exact_at_the_256_boundary(self):
        b = self._fit_backend()
        usable = 2_500 - 0.03 * 24_576
        reserve = lambda c: (
            (self._OH + b._compute_buffer_ctx_bytes(c, self._UB, "f16", layer_split = True)) / MIB
        )
        assert reserve(30_720) == 1760.0 <= usable
        assert reserve(30_976) == 1764.0 > usable

    def test_capped_context_still_fits_the_pooled_budget(self):
        b = self._fit_backend()
        gpus, totals = self._HETEROGENEOUS
        idx, ctx = self._drive(b, gpus, totals, 20_480, 262144)
        assert idx == [0, 1] and ctx == 30_720
        pool = sum(g[1] - 0.03 * totals[g[0]] for g in gpus)
        cc = 2 * b._compute_buffer_ctx_bytes(ctx, self._UB, "f16", layer_split = True)
        footprint = (20_480 * MIB + self._OH + b._estimate_kv_cache_bytes(ctx) + cc) / MIB
        assert footprint <= pool

    def test_homogeneous_gpus_are_unaffected_by_the_cap(self):
        b = self._fit_backend()
        gpus, totals = self._HOMOGENEOUS
        assert self._drive(b, gpus, totals, 20_480, 262144) == ([0, 1], 262144)

    _card_at_floor_reserve = TestPerDeviceSplitReserve._card_at_floor_reserve

    def test_cap_floors_at_the_fit_minimum_and_still_rejects_below_it(self):
        b = self._fit_backend()
        totals = {0: 49_152, 1: 24_576}
        gpus = [(0, 40_000), (1, self._card_at_floor_reserve(b, totals[1], -1))]
        assert self._drive(b, gpus, totals, 20_480, 262144) == (None, 0)

    def test_cap_keeps_a_card_that_holds_exactly_the_fit_minimum(self):
        b = self._fit_backend()
        totals = {0: 49_152, 1: 24_576}
        gpus = [(0, 40_000), (1, self._card_at_floor_reserve(b, totals[1], +1))]
        assert self._drive(b, gpus, totals, 20_480, 262144) == ([0, 1], _FIT_MIN_CTX)

    def test_flat_arch_term_is_not_rate_inverted(self):
        b = _backend(embd = 7168, arch = "deepseek4")
        reserve = lambda c: self._OH + b._compute_buffer_ctx_bytes(c, 512, "q8_0", layer_split = True)
        cap = LlamaCppBackend._cap_ctx_to_per_device_reserve(200_000, [4000.0], reserve)
        assert cap == 13_312
        assert reserve(cap) / MIB <= 4000.0 < reserve(cap + 256) / MIB
        assert reserve(43_341) / MIB > 4000.0

    def test_flat_arch_below_the_floor_is_infeasible(self):
        b = _backend(embd = 7168, arch = "deepseek4")
        reserve = lambda c: self._OH + b._compute_buffer_ctx_bytes(c, 512, "q8_0", layer_split = True)
        assert LlamaCppBackend._cap_ctx_to_per_device_reserve(200_000, [2500.0], reserve) == 0

    @pytest.mark.parametrize("model_mib", [8_192, 16_384, 20_480, 30_720])
    def test_capping_never_loses_to_continuing_with_more_gpus(self, model_mib):
        b = self._fit_backend()
        gpus = [(0, 40_000), (1, 2_500), (2, 2_400)]
        totals = {0: 49_152, 1: 24_576, 2: 24_576}
        assert self._drive(b, gpus, totals, model_mib, 262144, cap = False) == (None, 0)
        idx, ctx = self._drive(b, gpus, totals, model_mib, 262144)
        assert idx == [0, 1] and ctx == 30_720

    def test_helper_edges(self):
        cap = LlamaCppBackend._cap_ctx_to_per_device_reserve
        assert cap(262144, [], lambda c: 0) == 0
        assert cap(1024, [4000.0], lambda c: 0) == 0
        assert cap(262144, [4000.0], lambda c: 0) == 262144
        linear = lambda c: c * 16384
        best = cap(262144, [2000.0], linear)
        assert best == 128_000 and best % 256 == 0
        assert linear(best) / MIB <= 2000.0 < linear(best + 256) / MIB


class TestSplitRateRecheckAfterSelection:
    """A multi-GPU answer must be re-priced at the split compute rate before use_fit is pinned False."""

    _OH = LlamaCppBackend._PIPELINE_PER_DEVICE_OVERHEAD_MIB * MIB
    _UB = 512
    _CTX = 1048576
    _FRAC = LlamaCppBackend._GPU_PIN_VRAM_FRACTION
    _CARD = 24_576

    def _cards(self, n):
        return [(i, self._CARD) for i in range(n)], {i: self._CARD for i in range(n)}

    def _pin(
        self,
        total_mib,
        n_cards,
        recheck = True,
        min_gpus = 1,
        ctx = None,
    ):
        """Mirror of the explicit-context branch: total_mib is its weights + KV + MTP
        footprint, to which each path adds one single-device compute copy."""
        b = _backend(embd = 4096)
        ctx = ctx or self._CTX
        cc1 = b._compute_buffer_ctx_bytes(ctx, self._UB, "f16")
        cc_split = b._compute_buffer_ctx_bytes(ctx, self._UB, "f16", layer_split = True)
        gpus, totals = self._cards(n_cards)
        return LlamaCppBackend._select_gpus_split_aware(
            total_mib * MIB + cc1,
            gpus,
            usable_fraction = self._FRAC,
            total_by_idx = totals,
            per_device_overhead_bytes = self._OH + cc1,
            min_gpus = min_gpus,
            split_extra_bytes = (cc_split - cc1) if recheck else 0,
        )

    def test_pre_fix_pinned_a_pair_that_cannot_hold_the_split_rate(self):
        b = _backend(embd = 4096)
        cc1 = b._compute_buffer_ctx_bytes(self._CTX, self._UB, "f16")
        ccs = b._compute_buffer_ctx_bytes(self._CTX, self._UB, "f16", layer_split = True)
        gpus, totals = self._cards(2)
        assert LlamaCppBackend._select_gpus(
            40_000 * MIB + cc1,
            gpus,
            usable_fraction = self._FRAC,
            total_by_idx = totals,
            per_device_overhead_bytes = self._OH + cc1,
        ) == ([0, 1], False)
        pool = 2 * (self._CARD - (1.0 - self._FRAC) * self._CARD)
        assert (40_000 * MIB + ccs + self._OH + ccs) / MIB > pool

    def test_recheck_falls_back_to_fit_when_no_subset_holds_it(self):
        assert self._pin(40_000, 2) == (None, True)

    def test_recheck_widens_the_subset_when_a_card_is_spare(self):
        assert self._pin(40_000, 3, recheck = False) == ([0, 1], False)
        assert self._pin(40_000, 3) == ([0, 1, 2], False)

    def test_single_gpu_pin_is_untouched(self):
        assert self._pin(20_000, 2) == ([0], False)
        assert self._pin(20_000, 2, recheck = False) == ([0], False)

    def test_equal_cards_never_collapse_to_one_gpu(self):
        for total in range(24_000, 46_000, 2_000):
            gi, use_fit = self._pin(total, 4)
            assert use_fit or (gi is not None and len(gi) >= 2)

    def test_collapse_to_one_gpu_is_repriced_without_the_delta(self):
        # A lone card is not a split and pays no delta.
        gpus = [(0, 16 * 1024), (1, 3 * 1024)]
        kw = dict(usable_fraction = 1.0, per_device_overhead_bytes = int(2.5 * GIB))
        assert LlamaCppBackend._select_gpus(int(14 * GIB), gpus, min_gpus = 2, **kw) == (
            [0, 1],
            False,
        )
        assert LlamaCppBackend._select_gpus_split_aware(
            int(14 * GIB), gpus, min_gpus = 2, split_extra_bytes = int(4.5 * GIB), **kw
        ) == ([0], False)
        assert LlamaCppBackend._select_gpus(int(14 * GIB), gpus, min_gpus = 1, **kw) == (
            [0],
            False,
        )

    def test_reprice_still_reports_fit_when_no_single_card_holds_it(self):
        assert LlamaCppBackend._select_gpus_split_aware(
            int(30 * GIB),
            [(0, 20 * 1024), (1, 20 * 1024)],
            usable_fraction = 1.0,
            per_device_overhead_bytes = int(1 * GIB),
            min_gpus = 2,
            split_extra_bytes = int(15 * GIB),
        ) == (None, True)

    def test_zero_step_reduces_to_plain_selection(self):
        b = _backend(embd = 4096)
        cc1 = b._compute_buffer_ctx_bytes(self._CTX, self._UB, "f16")
        gpus, totals = self._cards(3)
        for total_mib in (20_000, 40_000, 90_000):
            assert self._pin(total_mib, 3, recheck = False) == LlamaCppBackend._select_gpus(
                total_mib * MIB + cc1,
                gpus,
                usable_fraction = self._FRAC,
                total_by_idx = totals,
                per_device_overhead_bytes = self._OH + cc1,
            )

    def test_small_context_is_unaffected(self):
        assert self._pin(40_000, 2, ctx = 4096) == self._pin(40_000, 2, recheck = False, ctx = 4096)

    def test_wired_into_every_call_site(self):
        import ast
        import inspect
        import re
        import textwrap

        source = inspect.getsource(LlamaCppBackend.load_model)
        load = "".join(source.split())
        # Read call sites from the AST: names for each site's context are not the contract.
        wired = [
            ast.unparse(keyword.value)
            for node in ast.walk(ast.parse(textwrap.dedent(source)))
            if isinstance(node, ast.Call)
            for keyword in node.keywords
            if keyword.arg in ("split_extra_bytes", "split_extra_for_slots")
        ]
        # Five sites; a sixth has to say which context it prices at.
        assert len(wired) == 5, wired
        for expression in wired:
            assert re.fullmatch(
                r"_cc_split_extra\(\w+\)|lambda s: _cc_split_extra\(\w+, s\)"
                r"|_cc_split_extra\(\w+, cache_type='q8_0'\)",
                expression,
            ), expression
        assert "gpu_indices,use_fit=self._select_gpus_split_aware(" in load
        assert (
            "returnmax(0,_cc_bytes(ctx,2,slots,cache_type)//2-_cc_bytes(ctx,1,slots,cache_type),)"
            in load
        )
        slots = "".join(inspect.getsource(LlamaCppBackend._slots_that_fit_on_gpu).split())
        assert "self._select_gpus_split_aware(" in slots
        assert (
            "split_extra_bytes=(split_extra_for_slots(slots)ifsplit_extra_for_slotselsesplit_extra_bytes),"
            in slots
        )


# Scratch rate keys off the lighter KV axis; the budget's heavier scalar hides a q4_0 K.


class TestScratchTakesTheLighterAxis:
    """_planned_scratch_cache_type: KV bytes use the heavier axis, dequant scratch the lighter."""

    @pytest.mark.parametrize(
        "k,v,expected",
        [
            ("f16", "f16", "f16"),
            ("q8_0", "q8_0", "q8_0"),
            ("q4_0", "f16", "q4_0"),
            ("f16", "q4_0", "q4_0"),
            ("q8_0", "q4_0", "q4_0"),
            ("f32", "q8_0", "q8_0"),
        ],
    )
    def test_it_picks_the_quantized_axis_whichever_side_it_is_on(self, k, v, expected):
        extras = ["--cache-type-k", k, "--cache-type-v", v]
        assert _planned_scratch_cache_type(None, extras) == expected
        heavier = max(_planned_main_cache_types(None, extras), key = _kv_bytes_per_elem)
        assert (heavier != expected) == (_kv_bytes_per_elem(k) != _kv_bytes_per_elem(v))

    def test_a_managed_symmetric_request_leaves_both_terms_equal(self):
        """Unsloth emits one type on both axes, so a plain q8_0 load's fit must not move."""
        for kv in ("f16", "q8_0", "q4_0", "iq4_nl"):
            assert _planned_scratch_cache_type(kv, None) == kv

    def test_extras_beat_the_managed_field_per_axis(self):
        """Extras are appended last and win per axis, so the scratch follows them, not the UI."""
        assert _planned_scratch_cache_type("f16", ["--cache-type-k", "q4_0"]) == "q4_0"

    def test_it_reads_the_inherited_env_when_nothing_else_sets_a_type(self):
        env = {"LLAMA_ARG_CACHE_TYPE_K": "q4_0", "LLAMA_ARG_CACHE_TYPE_V": "f16"}
        assert _planned_scratch_cache_type(None, None, env) == "q4_0"

    @pytest.mark.parametrize("k,v", [("q4_0", "f16"), ("f16", "q4_0")])
    def test_the_asymmetric_pair_selects_the_dequant_rate(self, k, v):
        """An asymmetric pair must price its quantized axis's dequant, not the KQ-mask floor."""
        b = _backend()
        extras = ["--cache-type-k", k, "--cache-type-v", v]
        heavier = max(_planned_main_cache_types(None, extras), key = _kv_bytes_per_elem)
        lighter = _planned_scratch_cache_type(None, extras)

        heavy_rate = b._compute_buffer_ctx_bytes(131_072, 2048, heavier)
        light_rate = b._compute_buffer_ctx_bytes(131_072, 2048, lighter)

        assert light_rate > heavy_rate, (light_rate, heavy_rate)
        assert light_rate == b._compute_buffer_ctx_bytes(131_072, 2048, "q4_0")


class TestTensorFitPricesTheQuantizedAxis:
    """The advertised context must not exceed what an honest per-axis price allows; tensor mode has
    no --fit valve, so an optimistic context OOMs at startup instead of spilling."""

    @staticmethod
    def _plan(
        b,
        cache_type,
        scratch_type,
        ub,
        ngpu = 4,
        per_gpu = 48_000,
    ):
        return b._plan_tensor_parallel(
            gpus = [(i, per_gpu) for i in range(ngpu)],
            model_size = 60 * 1024**3,
            target_ctx = 262_144,
            max_target_ctx = 262_144,
            total_by_idx = {i: per_gpu for i in range(ngpu)},
            cache_type_kv = cache_type,
            scratch_cache_type_kv = scratch_type,
            n_parallel = 1,
            swa_full = False,
            kv_unified = True,
            n_ubatch = ub,
            flash_attn = False,
        )

    def test_an_asymmetric_pair_is_capped_like_its_quantized_axis(self):
        """With -ctk q4_0 -ctv f16 at a raised micro-batch, the quantized axis caps the advertised
        context."""
        b = _backend_from_gguf_local()
        optimistic = self._plan(b, "f16", None, 2048, per_gpu = 36_000)[0]
        honest = self._plan(b, "f16", "q4_0", 2048, per_gpu = 36_000)[0]

        assert honest < optimistic, (honest, optimistic)
        assert honest > 2048, honest
        # Below symmetric q4_0 on purpose: f16 KV bytes on both axes plus full dequant scratch.
        assert honest < self._plan(b, "q4_0", "q4_0", 2048, per_gpu = 36_000)[0]

    def test_a_symmetric_request_is_unchanged(self):
        """Default and explicit scratch agree when both axes match, so no existing load moves."""
        b = _backend_from_gguf_local()
        for kv in ("f16", "q8_0", "q4_0"):
            for ub in (512, 2048):
                assert self._plan(b, kv, None, ub) == self._plan(b, kv, kv, ub)


class TestSpeculativeComputeBuffers:
    """Draft buffers must cover growth past the 224 MiB floor; measured on Qwen3.8 27B's head."""

    def _qwen38(self):
        b = _backend(vocab = 248320, embd = 5120)
        b._feed_forward_length = 17408
        return b

    @pytest.mark.parametrize(
        "ub,ctx,measured", [(512, 32768, 130.02), (1120, 32768, 284.41), (1120, 131072, 428.78)]
    )
    def test_floor_plus_growth_covers_the_draft_context(self, ub, ctx, measured):
        b = self._qwen38()
        charged = (
            LlamaCppBackend._MTP_DRAFT_COMPUTE_BYTES + b._mtp_draft_compute_bytes(ctx, n_ubatch = ub)
        ) / MIB
        assert charged >= measured

    def test_short_context_and_micro_batch_stay_on_the_floor(self):
        assert self._qwen38()._mtp_draft_compute_bytes(4096, n_ubatch = 256) == 0

    def test_growth_follows_the_mask_once_past_the_floor(self):
        b = self._qwen38()
        lo = b._mtp_draft_compute_bytes(131072, n_ubatch = 1120)
        hi = b._mtp_draft_compute_bytes(262144, n_ubatch = 1120)
        assert hi - lo == 1120 * 2 * 131072

    def test_verify_rows_scale_with_slots_and_depth(self):
        b = self._qwen38()
        base = b._mtp_draft_compute_bytes(4096, n_ubatch = 256)
        rows = b._mtp_draft_compute_bytes(4096, n_ubatch = 256, n_parallel = 4, spec_draft_n_max = 2)
        assert rows - base == 4 * 2 * LlamaCppBackend._SPEC_VERIFY_ROW_COPIES * 248320 * 4

    def test_the_reserve_carries_it(self):
        b = self._qwen38()
        b._n_layers, b._n_kv_heads, b._kv_key_length, b._kv_value_length = 65, 4, 256, 256
        b._nextn_predict_layers = 1
        kwargs = dict(n_parallel = 1, target_rollback = False, mtp_keeps_target_ctx = False)
        short = b._estimate_mtp_overhead_bytes(4096, n_ubatch = 256, **kwargs)
        long = b._estimate_mtp_overhead_bytes(262144, n_ubatch = 1120, **kwargs)
        draft_kv = b._mtp_draft_kv_bytes(262144, n_ubatch = 1120) - b._mtp_draft_kv_bytes(
            4096, n_ubatch = 256
        )
        assert long - short == draft_kv + b._mtp_draft_compute_bytes(262144, n_ubatch = 1120)


class TestFlashAttentionOff:
    """Without flash attention llama.cpp materialises [n_kv, n_ubatch, n_head] float32 scores."""

    _MEASURED = [
        ("Llama-3.2-1B", 512, 4, 32768, 2130.01),
        ("Llama-3.2-1B", 2048, 4, 32768, 8520.02),
        ("Llama-3.2-1B", 512, 4, 65536, 4242.01),
        ("Qwen3-8B", 1120, 4, 32768, 4698.76),
        ("Qwen3.5-4B", 512, 4, 65536, 2214.01),
        ("Qwen3.6-35B-A3B", 2048, 4, 32768, 4504.18),
        ("Gemma-4-12B", 1120, 4, 32768, 2540.8),
        ("Gemma-4-12B", 512, 4, 65536, 2248.01),
        ("Gemma-4-E4B", 512, 4, 65536, 1217.51),
        ("GLM-4.7-Flash", 1120, 4, 32768, 3086.3),
        ("GLM-4.7-Flash", 512, 4, 65536, 2818.9),
    ]

    @pytest.mark.parametrize("name,ub,slots,ctx,measured", _MEASURED)
    def test_covers_the_measured_buffer(self, name, ub, slots, ctx, measured):
        b = _measured(name)
        est = (
            b._estimate_compute_buffer_bytes(n_ubatch = ub, n_parallel = slots)
            + b._compute_buffer_ctx_bytes(ctx, ub, "f16", flash_attn = False)
        ) / MIB
        assert measured <= est <= measured * 1.1, f"{name}: {est:.0f} vs {measured}"

    def test_the_rate_is_the_score_matrix_plus_a_float32_mask(self):
        b = _measured("Gemma-4-12B")
        on = b._compute_buffer_ctx_bytes(100000, 1120, "f16")
        off = b._compute_buffer_ctx_bytes(100000, 1120, "f16", flash_attn = False)
        assert on == 1120 * 2 * 100000
        assert off == 1120 * (4 + 16 * 4) * 100000


class TestLayerSplitSlidingWindowSlots:
    """A layer split replicates sliding-window masks per GPU, each holding the window for every slot."""

    @pytest.mark.parametrize("slots,ctx,measured", [(8, 65536, 971), (16, 131072, 1601)])
    def test_covers_the_measured_split(self, slots, ctx, measured):
        b = _measured("Gemma-4-12B")
        per_device = (
            b._estimate_compute_buffer_bytes(n_ubatch = 1120, n_parallel = slots)
            + b._compute_buffer_ctx_bytes(ctx, 1120, "f16", layer_split = True, n_parallel = slots)
        ) / MIB
        one_slot_window = (
            b._estimate_compute_buffer_bytes(n_ubatch = 1120, n_parallel = slots)
            + b._compute_buffer_ctx_bytes(ctx, 1120, "f16", layer_split = True)
        ) / MIB
        assert one_slot_window < measured <= per_device <= measured * 1.1


class TestOutputRowsByBuild:
    """Builds before b9460 reserved an output row per micro-batch token, whatever the slot count."""

    @pytest.mark.parametrize(
        "output,build",
        [
            ("version: 0.4.0-dev (build 10909, commit 329b6160f)\nbuilt with GNU", 10909),
            ("version: 9415 (06d26dfdf)\nbuilt with GNU 11.4.0", 9415),
            ("version: 1 (unknown)\n", None),
            ("no version here", None),
        ],
    )
    def test_the_build_is_read_from_either_version_format(
        self, tmp_path, monkeypatch, output, build
    ):
        import subprocess
        import types

        binary = tmp_path / "llama-server"
        binary.write_bytes(b"")
        monkeypatch.setattr(LlamaCppBackend, "_build_number_cache", {})
        monkeypatch.setattr(
            subprocess,
            "run",
            lambda *a, **k: types.SimpleNamespace(stdout = output, stderr = "", returncode = 0),
        )
        assert LlamaCppBackend.probe_build_number(str(binary)) == build

    @pytest.mark.parametrize(
        "build,legacy", [(9415, True), (9459, True), (9460, False), (None, False)]
    )
    def test_only_a_known_older_build_reserves_micro_batch_rows(self, monkeypatch, build, legacy):
        monkeypatch.setattr(
            LlamaCppBackend, "probe_build_number", classmethod(lambda cls, binary = None: build)
        )
        assert LlamaCppBackend.reserves_micro_batch_outputs() is legacy

    def test_an_older_build_is_charged_a_row_per_micro_batch_token(self):
        # Measured b9415 Vulkan compute buffers (MiB) at 32768 cells, one and four slots.
        cases = [
            (dict(vocab = 128256, embd = 2048, ff = 8192), 512, 254.50),
            (dict(vocab = 151936, embd = 4096, ff = 12288), 2048, 1667.05),
            (dict(vocab = 262144, embd = 1536, ff = 12288, ple = 256 * 35), 2048, 2704.08),
        ]
        for dims, ub, measured in cases:
            b = _backend(vocab = dims["vocab"], embd = dims["embd"])
            b._feed_forward_length = dims["ff"]
            b._embedding_length_per_layer_input = 0
            b._n_layers = 1
            if "ple" in dims:
                b._embedding_length_per_layer_input, b._n_layers = 256, 35
            b._reserves_micro_batch_outputs = True
            for slots in (1, 4):
                est = (
                    b._estimate_compute_buffer_bytes(n_ubatch = ub, n_parallel = slots)
                    + b._compute_buffer_ctx_bytes(32768, ub, "f16")
                ) / MIB
                assert measured <= est <= measured * 1.5, (dims, slots, est)
            b._reserves_micro_batch_outputs = False
            assert b._estimate_compute_buffer_bytes(n_ubatch = ub, n_parallel = 1) < (
                dims["vocab"] * ub * 4
            )