Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

50.7 KB
Raw Blame
# SPDX-License-Identifier: AGPL-3.0-only
# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0

"""The -ot planner must never emit -ngl or --no-kv-offload or charge token_embd to VRAM."""

from __future__ import annotations

import os
import re
from dataclasses import replace

import pytest

from core.inference.offload_layout import (
    LM_HEAD_PATTERN,
    BlockLayout,
    ModelLayout,
    _layout_from_reader,
    _layout_from_readers,
    layout_from_gguf,
    split_shard_paths,
    spill_pattern_for,
)
from core.inference.offload_cost_model import HostProfile
from core.inference.offload_planner import (
    _device_slots,
    _per_device_shortfall,
    ContextPolicy,
    Plan,
    PlanOptions,
    SpillOrder,
    all_resident_bytes,
    max_context_for,
    plan_placement,
    plan_to_args,
    resident_floor_bytes,
)

GIB = 1024**3
MIB = 1024**2

_MODELS = "/mnt/disks/unslothai/daniel3/workspace_11/temp/loadmode_sim"
Q2_PATH = f"{_MODELS}/qwen38/Qwen3.8-27B-UD-Q2_K_XL.gguf"
Q4_PATH = f"{_MODELS}/qwen38/Qwen3.8-27B-UD-Q4_K_XL.gguf"
MOE_PATH = f"{_MODELS}/qwen36moe/Qwen3.6-35B-A3B-UD-Q4_K_XL.gguf"

needs_gguf = pytest.mark.skipif(
    not os.path.exists(Q4_PATH), reason = "local GGUF fixtures not present"
)


# Byte totals measured on Qwen3.8-27B, so the ladder uses real arithmetic sans fixtures.


def _uniform_layout(
    *,
    n_blocks: int = 65,
    spillable_total: int,
    resident_total: int,
    lm_head: int,
    token_embd: int,
    kv_per_token: int = 65536,
    recurrent: int = 156_893_184,
    is_moe: bool = False,
    n_ctx_train: int = 262144,
) -> ModelLayout:
    per_spill, per_res = spillable_total // n_blocks, resident_total // n_blocks
    blocks = tuple(
        BlockLayout(index = i, spillable_bytes = per_spill, resident_bytes = per_res)
        for i in range(n_blocks)
    )
    return ModelLayout(
        arch = "qwen35moe" if is_moe else "qwen35",
        n_layers = 64,
        n_attention_layers = 16,
        blocks = blocks,
        lm_head_bytes = lm_head,
        token_embd_bytes = token_embd,
        kv_bytes_per_token_f16 = kv_per_token,
        recurrent_bytes = recurrent,
        n_ctx_train = n_ctx_train,
        is_moe = is_moe,
        complete = True,
    )


def q4_layout() -> ModelLayout:
    return _uniform_layout(
        spillable_total = 10_836_131_840,
        resident_total = 4_953_923_584,
        lm_head = 1_042_915_328,
        token_embd = 715_128_832,
    )


def q2_layout() -> ModelLayout:
    return _uniform_layout(
        spillable_total = 5_900_615_680,
        resident_total = 2_785_009_664,
        lm_head = 715_128_832,
        token_embd = 417_202_176,
    )


def uneven_layout() -> ModelLayout:
    """Four blocks with deliberately lopsided FFN, for selection policy."""
    sizes = [100 * MIB, 400 * MIB, 200 * MIB, 50 * MIB]
    blocks = tuple(
        BlockLayout(index = i, spillable_bytes = s, resident_bytes = 10 * MIB)
        for i, s in enumerate(sizes)
    )
    return ModelLayout(
        arch = "qwen35",
        n_layers = 4,
        n_attention_layers = 4,
        blocks = blocks,
        lm_head_bytes = 100 * MIB,
        token_embd_bytes = 50 * MIB,
        kv_bytes_per_token_f16 = 1024,
        recurrent_bytes = 0,
        n_ctx_train = 65536,
        complete = True,
    )


def test_a_load_that_fits_spills_nothing():
    plan = plan_placement(q2_layout(), [24 * GIB], 64 * GIB, 8192)
    assert plan.spilled_blocks == ()
    assert plan.spilled_lm_head is False
    assert plan.ot_patterns == ()
    assert plan.load_mode_none is True
    assert "fits in VRAM" in plan.reason


def test_a_load_that_does_not_fit_spills_ffn_and_keeps_the_cache():
    """The whole point: weights move, the cache does not."""
    plan = plan_placement(q4_layout(), [12 * GIB], 64 * GIB, 32768)
    assert plan.spilled_blocks, "should have spilled something"
    assert plan.spilled_lm_head is False, "lm_head is the LAST rung, not the first"
    args = plan_to_args(plan)
    assert "-ngl" not in args and "--n-gpu-layers" not in args
    assert "--no-kv-offload" not in args and "-nkvo" not in args


def test_lm_head_is_only_spilled_after_every_block():
    """43% of generation alone, 16% after FFN. Never take it first."""
    layout = q4_layout()
    plan = plan_placement(layout, [7 * GIB], 64 * GIB, 8192)
    assert plan.spilled_lm_head is True
    assert len(plan.spilled_blocks) == len(layout.blocks), "all blocks go first"


def test_partial_spill_takes_only_what_is_needed():
    layout = q4_layout()
    plan = plan_placement(layout, [16 * GIB], 64 * GIB, 8192)
    assert 0 < len(plan.spilled_blocks) < len(layout.blocks)


def test_a_load_that_cannot_fit_keeps_mmap_and_says_so():
    """mmap must stay when -ot is exhausted: it is what keeps an over-commit pageable."""
    plan = plan_placement(q4_layout(), [4 * GIB], 64 * GIB, 131072)
    assert plan.insufficient is True
    assert plan.changed is False
    assert plan.load_mode_none is False
    assert plan_to_args(plan) == []
    assert "smaller quant" in plan.reason


@pytest.mark.parametrize("vram", [4 * GIB, 8 * GIB, 12 * GIB, 16 * GIB, 24 * GIB])
@pytest.mark.parametrize("ctx", [4096, 32768, 131072])
def test_no_plan_ever_moves_the_kv_cache(vram, ctx):
    """-ngl and --no-kv-offload both cost 22x to 73x. Neither may ever appear."""
    for layout in (q2_layout(), q4_layout()):
        args = plan_to_args(plan_placement(layout, [vram], 64 * GIB, ctx))
        for banned in ("-ngl", "--gpu-layers", "--n-gpu-layers", "-nkvo", "--no-kv-offload"):
            assert banned not in args


def test_token_embd_is_never_charged_to_vram():
    """llama.cpp pins dev_input to the CPU, so the embedding is charged to host RAM, never VRAM."""
    small = q4_layout()
    big = ModelLayout(**{**small.__dict__, "token_embd_bytes": 4 * GIB})

    tight = [12 * GIB]
    a = plan_placement(small, tight, 64 * GIB, 8192)
    b = plan_placement(big, tight, 64 * GIB, 8192)

    assert a.vram_bytes == b.vram_bytes
    assert a.spilled_blocks == b.spilled_blocks
    assert b.host_bytes - a.host_bytes == 4 * GIB - small.token_embd_bytes


def test_load_mode_none_is_withheld_when_host_ram_cannot_hold_the_spill():
    """Disabling mmap without host RAM for the spill turns a pageable load into an OOM kill."""
    layout = q4_layout()
    roomy = plan_placement(layout, [12 * GIB], 64 * GIB, 8192)
    cramped = plan_placement(layout, [12 * GIB], 3 * GIB, 8192)
    assert roomy.load_mode_none is True
    assert cramped.load_mode_none is False


def test_unreadable_host_ram_keeps_mmap():
    plan = plan_placement(q4_layout(), [12 * GIB], None, 8192)
    assert plan.load_mode_none is False


def test_every_emitted_pattern_is_anchored():
    """Unanchored is a live trap: 'output\\.weight' also matches every
    blk.N.attn_output.weight, which silently moved 16 attention projections."""
    for layout in (q2_layout(), q4_layout()):
        for vram in (6 * GIB, 12 * GIB, 16 * GIB):
            plan = plan_placement(layout, [vram], 64 * GIB, 8192)
            for pattern in plan.ot_patterns:
                assert pattern.startswith("^") and pattern.endswith("$"), pattern


def test_the_lm_head_pattern_does_not_catch_attention_output():
    assert re.search(LM_HEAD_PATTERN, "output.weight")
    assert not re.search(LM_HEAD_PATTERN, "blk.3.attn_output.weight")
    assert re.search(r"output\.weight", "blk.3.attn_output.weight")


def test_the_dense_pattern_does_not_catch_the_router():
    pattern = spill_pattern_for(q4_layout())
    assert re.search(pattern, "blk.7.ffn_up.weight")
    assert re.search(pattern, "blk.7.ffn_gate.weight")
    assert not re.search(pattern, "blk.7.ffn_gate_inp.weight")
    assert not re.search(pattern, "blk.7.ffn_norm.weight")
    assert not re.search(pattern, "blk.7.attn_q.weight")


def test_the_moe_pattern_spills_experts_but_not_shared_experts():
    """Shared experts hit every token like a dense FFN; spilling them only costs bandwidth."""
    moe = _uniform_layout(
        n_blocks = 40,
        spillable_total = 19_671_285_760,
        resident_total = 1_597_483_520,
        lm_head = 540_000_000,
        token_embd = 540_000_000,
        kv_per_token = 20480,
        is_moe = True,
    )
    pattern = spill_pattern_for(moe)
    assert re.search(pattern, "blk.5.ffn_up_exps.weight")
    assert re.search(pattern, "blk.5.ffn_down_exps.weight")
    assert not re.search(pattern, "blk.5.ffn_up_shexp.weight")
    assert not re.search(pattern, "blk.5.ffn_gate_inp.weight")


def test_partial_spill_names_only_the_chosen_blocks():
    pattern = spill_pattern_for(q4_layout(), [3, 7, 11])
    assert re.search(pattern, "blk.3.ffn_up.weight")
    assert re.search(pattern, "blk.11.ffn_down.weight")
    assert not re.search(pattern, "blk.4.ffn_up.weight")
    assert not re.search(pattern, "blk.1.ffn_up.weight")


def test_a_full_spill_uses_the_compact_global_pattern():
    layout = q4_layout()
    plan = plan_placement(layout, [7 * GIB], 64 * GIB, 8192)
    assert plan.ot_patterns[0] == spill_pattern_for(layout, None)
    assert r"\d+" in plan.ot_patterns[0]


def test_plan_to_args_shape():
    layout = q4_layout()
    args = plan_to_args(plan_placement(layout, [12 * GIB], 64 * GIB, 8192))
    assert args.count("-ot") == 1
    assert args[args.index("-ot") + 1].endswith("=CPU")
    assert "--load-mode" in args and args[args.index("--load-mode") + 1] == "none"


def test_largest_first_minimises_overshoot():
    """Cover a small residual with a small block, not a big one crossed on every token."""
    layout = uneven_layout()
    floor = resident_floor_bytes(layout, 4096)
    budget = floor + layout.spillable_bytes - 40 * MIB + 1 * GIB
    plan = plan_placement(
        layout,
        [budget],
        64 * GIB,
        4096,
        opts = PlanOptions(
            overhead_bytes_per_device = GIB,
        ),
    )
    spilled = sum(b.spillable_bytes for b in layout.blocks if b.index in plan.spilled_blocks)
    assert spilled == 50 * MIB, "should take the 50 MiB block, not a bigger one"


def test_front_and_back_orders_pick_opposite_ends():
    layout = uneven_layout()
    floor = resident_floor_bytes(layout, 4096)
    budget = floor + layout.spillable_bytes - 40 * MIB + 1 * GIB
    front = plan_placement(
        layout,
        [budget],
        64 * GIB,
        4096,
        opts = PlanOptions(overhead_bytes_per_device = GIB, spill_order = SpillOrder.FRONT_FIRST),
    )
    back = plan_placement(
        layout,
        [budget],
        64 * GIB,
        4096,
        opts = PlanOptions(overhead_bytes_per_device = GIB, spill_order = SpillOrder.BACK_FIRST),
    )
    assert front.spilled_blocks == (0,)
    assert back.spilled_blocks == (3,)


def test_an_incomplete_layout_abstains():
    plan = plan_placement(ModelLayout(), [24 * GIB], 64 * GIB, 8192)
    assert plan.changed is False
    assert plan_to_args(plan) == []


def test_no_devices_abstains():
    assert plan_placement(q4_layout(), [], 64 * GIB, 8192).changed is False


def test_a_device_smaller_than_its_own_overhead_abstains():
    assert plan_placement(q4_layout(), [512 * MIB], 64 * GIB, 8192).changed is False


def test_every_device_pays_the_fixed_overhead():
    """Every split device pays a CUDA context and scratch, so two 8 GiB cards hold less than one 16 GiB."""
    layout = q4_layout()
    one_big = max_context_for(layout, [16 * GIB], spill_all_ffn = True)
    two_small = max_context_for(layout, [8 * GIB, 8 * GIB], spill_all_ffn = True)
    assert one_big > 0 and two_small > 0
    assert two_small < one_big


def test_a_partial_spill_across_two_gpus_abstains():
    """A partial spill across two GPUs must abstain: -ot can relieve one card and leave the other over."""
    layout = q4_layout()
    one_card = plan_placement(layout, [16 * GIB], 64 * GIB, 8192)
    assert one_card.spilled_blocks
    assert len(one_card.spilled_blocks) < len(layout.blocks), "partial, not everything"

    two_cards = plan_placement(layout, [8 * GIB, 8 * GIB], 64 * GIB, 8192)
    assert two_cards.spilled_blocks == ()
    assert two_cards.ot_patterns == ()
    assert two_cards.changed is False


def test_a_safe_partial_spill_across_two_gpus_is_planned():
    sizes = [GIB // 2, GIB // 2, GIB // 2, 2 * GIB]
    layout = ModelLayout(
        arch = "qwen35",
        n_layers = 4,
        n_attention_layers = 4,
        blocks = tuple(
            BlockLayout(index = i, spillable_bytes = size, resident_bytes = GIB // 10)
            for i, size in enumerate(sizes)
        ),
        lm_head_bytes = GIB // 10,
        token_embd_bytes = 0,
        kv_bytes_per_token_f16 = 0,
        recurrent_bytes = 0,
        n_ctx_train = 4096,
        complete = True,
    )
    opts = PlanOptions(
        overhead_bytes_per_device = GIB,
        pipeline_overhead_bytes = GIB,
        host_ram_headroom_bytes = 0,
    )

    plan = plan_placement(
        layout,
        [23 * GIB // 10, 22 * GIB // 10],
        64 * GIB,
        4096,
        opts = opts,
    )

    assert plan.changed is True
    assert plan.spilled_blocks == (0, 3)


def test_partial_spill_selection_covers_each_device_shortfall():
    sizes = [1200 * MIB, 0, 0, 600 * MIB, 600 * MIB]
    layout = ModelLayout(
        arch = "qwen35",
        n_layers = 5,
        n_attention_layers = 5,
        blocks = tuple(
            BlockLayout(index = i, spillable_bytes = size, resident_bytes = 0)
            for i, size in enumerate(sizes)
        ),
        lm_head_bytes = 0,
        token_embd_bytes = 0,
        kv_bytes_per_token_f16 = 0,
        recurrent_bytes = 0,
        n_ctx_train = 4096,
        complete = True,
    )
    plan = plan_placement(
        layout,
        [1300 * MIB, 100 * MIB],
        64 * GIB,
        4096,
        opts = PlanOptions(overhead_bytes_per_device = 0, host_ram_headroom_bytes = 0),
        split_weights_per_device = [1, 1],
    )

    assert plan.spilled_blocks == (3, 4)


def test_per_device_selection_cannot_drop_cache_remainder_from_the_pool():
    spillable = [17, 4, 14, 20, 3, 5, 17, 17]
    resident = [1, 5, 5, 1, 6, 6, 5, 0]
    layout = ModelLayout(
        arch = "qwen35",
        n_layers = 8,
        n_attention_layers = 8,
        blocks = tuple(
            BlockLayout(index = i, spillable_bytes = spill, resident_bytes = keep)
            for i, (spill, keep) in enumerate(zip(spillable, resident))
        ),
        lm_head_bytes = 0,
        token_embd_bytes = 0,
        other_resident_bytes = 8,
        kv_bytes_per_token_f16 = 3,
        recurrent_bytes = 0,
        n_ctx_train = 4096,
        complete = True,
    )
    opts = PlanOptions(
        overhead_bytes_per_device = 8,
        pipeline_overhead_bytes = 5,
        extra_resident_bytes = 4,
        host_ram_headroom_bytes = 0,
    )
    plan = plan_placement(
        layout,
        [90, 25],
        1024,
        5,
        opts = opts,
        split_weights_per_device = [56, 27],
    )

    assert not plan.changed or plan.vram_bytes <= 90


def test_output_device_shortfall_can_reach_the_lm_head_rung():
    layout = ModelLayout(
        arch = "qwen35",
        n_layers = 3,
        n_attention_layers = 3,
        blocks = tuple(
            BlockLayout(index = i, spillable_bytes = size, resident_bytes = 0)
            for i, size in enumerate([100, 100, 10])
        ),
        lm_head_bytes = 100,
        token_embd_bytes = 0,
        kv_bytes_per_token_f16 = 0,
        recurrent_bytes = 0,
        n_ctx_train = 4096,
        complete = True,
    )
    plan = plan_placement(
        layout,
        [100, 20],
        1024,
        1,
        opts = PlanOptions(overhead_bytes_per_device = 0, host_ram_headroom_bytes = 0),
        split_weights_per_device = [1, 1],
    )

    assert plan.spilled_blocks == (0,)
    assert plan.spilled_lm_head is True
    assert plan.vram_bytes <= 120


def test_a_full_spill_is_checked_per_device_not_assumed():
    """A full spill is checked per device: 65 rows over two equal cards is 33/32, not a pooled fit."""
    layout = _layout_from_reader(_StubReader(_shard_fields(), _shard_tensors(range(64))))
    half = _ALL_SPILL_VRAM // 2
    plan = plan_placement(layout, [half, half], 256 * GIB, 4096, opts = _NO_OVERHEAD)
    assert plan.changed is False
    assert plan.spilled_blocks == ()
    assert "device 0" in plan.reason

    one = plan_placement(layout, [_ALL_SPILL_VRAM], 256 * GIB, 4096, opts = _NO_OVERHEAD)
    assert len(one.spilled_blocks) == len(layout.blocks)


def test_a_full_spill_abstains_when_the_cache_layout_is_unknown():
    """A hybrid's recurrent layers are not recorded per layer, so a multi-card full spill must abstain."""
    layout = _layout_from_reader(_StubReader(_shard_fields(), _shard_tensors(range(64))))
    hybrid = replace(layout, recurrent_bytes = 8 * MIB)
    half = _ALL_SPILL_VRAM // 2
    plan = plan_placement(hybrid, [half, half], 256 * GIB, 4096, opts = _NO_OVERHEAD)
    assert plan.changed is False
    assert "recurrent state" in plan.reason

    one = plan_placement(hybrid, [_ALL_SPILL_VRAM], 256 * GIB, 4096, opts = _NO_OVERHEAD)
    assert len(one.spilled_blocks) == len(layout.blocks)


def test_the_row_split_matches_llama_cpp():
    """_device_slots mirrors llama-model.cpp:1439-1457: contiguous ranges sized in proportion to
    free VRAM, with the remainder rows landing on the earlier GPU."""
    assert [len(r) for r in _device_slots(65, [8 * GIB, 8 * GIB])] == [33, 32]
    assert [len(r) for r in _device_slots(65, [24 * GIB, 8 * GIB])] == [49, 16]
    assert [len(r) for r in _device_slots(65, [8 * GIB, 8 * GIB, 8 * GIB])] == [22, 22, 21]
    rows = _device_slots(65, [24 * GIB, 8 * GIB])
    assert rows[0] == list(range(0, 49)) and rows[1] == list(range(49, 65))
    assert _device_slots(65, [8 * GIB]) == [list(range(65))]
    assert _device_slots(4, [0, 0]) == [[0, 1, 2, 3], []]


def test_the_row_split_uses_llama_cpp_float32_boundaries():
    rows = _device_slots(353, [39407 * MIB, 12114 * MIB])
    assert len(rows[0]) == 270
    assert 270 in rows[1]


def test_a_float32_split_boundary_cannot_approve_an_oom():
    blocks = tuple(
        BlockLayout(
            index = i,
            spillable_bytes = 2 * GIB,
            resident_bytes = 2 * GIB if i == 270 else 0,
        )
        for i in range(352)
    )
    layout = ModelLayout(
        arch = "qwen35",
        n_layers = 352,
        n_attention_layers = 352,
        blocks = blocks,
        lm_head_bytes = 0,
        token_embd_bytes = 0,
        kv_bytes_per_token_f16 = 0,
        recurrent_bytes = 0,
        n_ctx_train = 4096,
        complete = True,
    )
    plan = plan_placement(
        layout,
        [2 * GIB, GIB],
        1024 * GIB,
        4096,
        opts = PlanOptions(overhead_bytes_per_device = 0, host_ram_headroom_bytes = 0),
        split_weights_per_device = [39407 * MIB, 12114 * MIB],
    )

    assert plan.changed is False
    assert "device 1" in plan.reason


def test_the_per_device_check_passes_when_the_shares_really_fit():
    """The check must not be a disguised "never plan on two GPUs". Cards sized so
    that each one's row share fits with room to spare return None -- no abstain
    reason -- for the same full spill the tight case rejects."""
    layout = _layout_from_reader(_StubReader(_shard_fields(), _shard_tensors(range(64))))
    spilled = {b.index for b in layout.blocks}
    tight = _ALL_SPILL_VRAM // 2
    assert (
        _per_device_shortfall(
            layout,
            _NO_OVERHEAD,
            4096,
            spilled,
            False,
            [tight, tight],
            quantised = False,
            kv_bytes_floor = 0,
        )
        is not None
    )
    roomy = _ALL_SPILL_VRAM
    assert (
        _per_device_shortfall(
            layout,
            _NO_OVERHEAD,
            4096,
            spilled,
            False,
            [roomy, roomy],
            quantised = False,
            kv_bytes_floor = 0,
        )
        is None
    )


def test_the_per_device_check_charges_each_secondary_pipeline_reserve():
    layout = uneven_layout()
    spilled = {b.index for b in layout.blocks}
    cache_per_layer = layout.kv_bytes(4096, 2) // layout.n_layers
    device_one_used = layout.blocks[3].resident_bytes + cache_per_layer + layout.lm_head_bytes
    pipeline_reserve = GIB
    opts = PlanOptions(overhead_bytes_per_device = 0, pipeline_overhead_bytes = pipeline_reserve)

    below = _per_device_shortfall(
        layout,
        opts,
        4096,
        spilled,
        False,
        [4 * GIB, device_one_used + pipeline_reserve - 1],
        quantised = False,
        kv_bytes_floor = 0,
        split_weights_per_device = [1, 1],
    )
    exact = _per_device_shortfall(
        layout,
        opts,
        4096,
        spilled,
        False,
        [4 * GIB, device_one_used + pipeline_reserve],
        quantised = False,
        kv_bytes_floor = 0,
        split_weights_per_device = [1, 1],
    )
    above = _per_device_shortfall(
        layout,
        opts,
        4096,
        spilled,
        False,
        [4 * GIB, device_one_used + pipeline_reserve + 1],
        quantised = False,
        kv_bytes_floor = 0,
        split_weights_per_device = [1, 1],
    )

    assert below is not None and "device 1" in below
    assert exact is None
    assert above is None


def test_an_empty_secondary_still_has_to_fit_its_fixed_reserves():
    layout = uneven_layout()
    spilled = {b.index for b in layout.blocks}
    pipeline_reserve = GIB
    opts = PlanOptions(overhead_bytes_per_device = 0, pipeline_overhead_bytes = pipeline_reserve)

    below = _per_device_shortfall(
        layout,
        opts,
        4096,
        spilled,
        True,
        [4 * GIB, pipeline_reserve - 1],
        quantised = False,
        kv_bytes_floor = 0,
        split_weights_per_device = [1000, 1],
    )
    exact = _per_device_shortfall(
        layout,
        opts,
        4096,
        spilled,
        True,
        [4 * GIB, pipeline_reserve],
        quantised = False,
        kv_bytes_floor = 0,
        split_weights_per_device = [1000, 1],
    )

    assert _device_slots(layout.n_layers + 1, [1000, 1])[1] == []
    assert below is not None and "device 1" in below
    assert exact is None


def test_multi_gpu_credit_sums():
    layout = q2_layout()
    assert (
        plan_placement(
            layout, [6 * GIB, 6 * GIB], 64 * GIB, 8192, opts = FIXED_OVERHEAD_OPTS
        ).spilled_blocks
        == ()
    )


def test_never_reduce_keeps_the_requested_context():
    layout = q4_layout()
    plan = plan_placement(layout, [12 * GIB], 64 * GIB, 65536)
    assert plan.n_ctx == 65536
    assert plan.spilled_blocks, "it pays with spill, not with the user's context"


def test_prefer_resident_shrinks_instead_of_spilling():
    layout = q4_layout()
    plan = plan_placement(
        layout,
        [18 * GIB],
        64 * GIB,
        65536,
        opts = PlanOptions(context_policy = ContextPolicy.PREFER_RESIDENT),
    )
    assert plan.spilled_blocks == ()
    assert plan.n_ctx < 65536
    assert "shrank context" in plan.reason


def test_prefer_resident_still_spills_when_even_min_ctx_will_not_fit():
    layout = q4_layout()
    plan = plan_placement(
        layout,
        [10 * GIB],
        64 * GIB,
        65536,
        opts = PlanOptions(context_policy = ContextPolicy.PREFER_RESIDENT),
    )
    assert plan.spilled_blocks, "shrinking cannot save this one, so spill"


def test_context_is_clamped_to_what_the_model_was_trained_on():
    layout = q4_layout()
    plan = plan_placement(layout, [24 * GIB], 128 * GIB, 999_999)
    assert plan.n_ctx == layout.n_ctx_train


def test_kv_quantisation_is_off_by_default():
    """35% slower generation, and only matched pairs are compiled."""
    plan = plan_placement(q4_layout(), [12 * GIB], 64 * GIB, 65536)
    assert plan.cache_type_k is None and plan.cache_type_v is None


def test_kv_quantisation_rescues_a_load_f16_cannot_fit():
    """The f16 KV cache alone overshoots 8 GiB usable at 64K; halving it makes the load fit."""
    layout = q4_layout()
    without = plan_placement(layout, [9 * GIB], 64 * GIB, 65536)
    assert without.insufficient is True

    with_quant = plan_placement(
        layout,
        [9 * GIB],
        64 * GIB,
        65536,
        opts = PlanOptions(allow_kv_quant = True),
    )
    assert with_quant.insufficient is False
    assert with_quant.cache_type_k == "q8_0"
    # Matched K/V pair: mixed types need GGML_CUDA_FA_ALL_QUANTS or fall back to CPU.
    assert with_quant.cache_type_k == with_quant.cache_type_v


def test_f16_is_preferred_when_it_fits_even_with_quant_allowed():
    """q8_0 costs 35% of generation, so it is a rescue, not a default."""
    plan = plan_placement(
        q2_layout(),
        [24 * GIB],
        64 * GIB,
        8192,
        opts = PlanOptions(allow_kv_quant = True),
    )
    assert plan.cache_type_k is None


# Independently computed table: max context with FFN spilled, f16 cache, 1 GiB overhead.
@pytest.mark.parametrize(
    "budget_gib,expected_k",
    [(8, 20), (10, 52), (12, 84), (16, 148), (20, 212)],
)
def test_q4_ffn_spilled_context_ladder(budget_gib, expected_k):
    got = max_context_for(
        q4_layout(), [budget_gib * GIB], spill_all_ffn = True, opts = FIXED_OVERHEAD_OPTS
    )
    assert expected_k * 1024 <= got < (expected_k + 1) * 1024, got


# Arithmetic against a stated overhead, not measurements; the constant is pinned by
# test_the_overhead_reserve_covers_the_measured_prefill_buffer.
FIXED_OVERHEAD_OPTS = PlanOptions(overhead_bytes_per_device = GIB)


@pytest.mark.parametrize("budget_gib,expected_k", [(6, 25), (8, 57), (12, 121), (20, 249)])
def test_q2_ffn_spilled_context_ladder(budget_gib, expected_k):
    got = max_context_for(
        q2_layout(), [budget_gib * GIB], spill_all_ffn = True, opts = FIXED_OVERHEAD_OPTS
    )
    assert expected_k * 1024 <= got < (expected_k + 1) * 1024, got


@pytest.mark.parametrize("budget_gib,expected_k", [(12, 33), (16, 97), (24, 225)])
def test_q2_fully_resident_context_ladder(budget_gib, expected_k):
    got = max_context_for(q2_layout(), [budget_gib * GIB], opts = FIXED_OVERHEAD_OPTS)
    assert expected_k * 1024 <= got < (expected_k + 1) * 1024, got


@pytest.mark.parametrize("budget_gib", [4, 6, 8, 10])
def test_q4_cannot_be_fully_resident_below_18_gib(budget_gib):
    assert max_context_for(q4_layout(), [budget_gib * GIB]) == 0


def test_the_ladder_never_regresses_as_vram_grows():
    """More VRAM must never mean more spill."""
    layout = q4_layout()
    counts = [
        len(
            plan_placement(
                layout, [g * GIB], 64 * GIB, 32768, opts = FIXED_OVERHEAD_OPTS
            ).spilled_blocks
        )
        for g in (8, 10, 12, 14, 16, 18, 20, 22, 24)
    ]
    assert counts == sorted(counts, reverse = True), counts


@needs_gguf
@pytest.mark.parametrize(
    "path,spillable_gib,resident_gib,lm_head_gib,embd_gib,moe",
    [
        (Q2_PATH, 5.291, 2.471, 0.666, 0.389, False),
        (Q4_PATH, 9.888, 4.491, 0.971, 0.666, False),
        (MOE_PATH, 18.320, 1.488, 0.503, 0.503, True),
    ],
)
def test_layout_matches_the_measured_buckets(
    path, spillable_gib, resident_gib, lm_head_gib, embd_gib, moe
):
    layout = layout_from_gguf(path)
    assert layout.complete
    assert layout.is_moe is moe
    assert layout.spillable_bytes / GIB == pytest.approx(spillable_gib, abs = 0.01)
    assert layout.block_resident_bytes / GIB == pytest.approx(resident_gib, abs = 0.01)
    assert layout.lm_head_bytes / GIB == pytest.approx(lm_head_gib, abs = 0.01)
    assert layout.token_embd_bytes / GIB == pytest.approx(embd_gib, abs = 0.01)


@needs_gguf
def test_the_hybrid_cache_is_priced_on_attention_layers_only():
    """qwen35 is 1-in-4 attention; an all-layers cache would be 4x too big and refuse fits."""
    layout = layout_from_gguf(Q4_PATH)
    assert layout.n_layers == 64
    assert layout.n_attention_layers == 16
    assert layout.kv_bytes_per_token_f16 == 16 * 4 * (256 + 256) * 2
    assert layout.recurrent_bytes / MIB == pytest.approx(149.6, abs = 1.0)


@needs_gguf
def test_a_bad_path_abstains_rather_than_raising():
    assert layout_from_gguf("/nonexistent/model.gguf").complete is False


class _StubField:
    def __init__(self, value):
        self._value = value

    def contents(self):
        return self._value


class _StubTensor:
    def __init__(self, name, n_bytes):
        self.name = name
        self.n_bytes = n_bytes


class _StubReader:
    """The handful of attributes ``_layout_from_reader`` touches."""

    def __init__(self, fields, tensors):
        self.fields = {k: _StubField(v) for k, v in fields.items()}
        self.tensors = tensors


def _shard_fields(**extra):
    base = {
        "general.architecture": "llama",
        "llama.block_count": 64,
        "llama.attention.head_count_kv": 8,
        "llama.attention.head_count": 64,
        "llama.embedding_length": 8192,
        "llama.attention.key_length": 128,
        "llama.attention.value_length": 128,
        "llama.context_length": 262144,
    }
    base.update(extra)
    return base


def _shard_tensors(indices):
    out = []
    for i in indices:
        out.append(_StubTensor(f"blk.{i}.ffn_up.weight", GIB // 8))
        out.append(_StubTensor(f"blk.{i}.attn_q.weight", MIB * 32))
    return out


def test_a_single_file_gguf_is_still_read():
    """The guard below must not catch an ordinary one-file model."""
    layout = _layout_from_reader(_StubReader(_shard_fields(), _shard_tensors(range(64))))
    assert layout.complete
    assert len(layout.blocks) == 64


def test_the_layout_reader_totals_every_tensor_from_a_real_gguf(tmp_path):
    """tensor_bytes includes every tensor while token_embd_bytes is embedding-only."""
    import numpy as np
    from gguf import GGUFWriter

    path = tmp_path / "embd.gguf"
    writer = GGUFWriter(str(path), "llama")
    for key, value in _shard_fields().items():
        if key.startswith("llama.") and isinstance(value, int):
            writer.add_uint32(key, 1 if key == "llama.block_count" else value)
    writer.add_tensor("output_norm.weight", np.zeros(16, dtype = np.float32))
    writer.add_tensor("token_embd.weight", np.zeros(1024, dtype = np.float32))
    writer.add_tensor("per_layer_token_embd.weight", np.zeros(4096, dtype = np.float32))
    writer.add_tensor("blk.0.attn_q.weight", np.zeros(256, dtype = np.float32))
    writer.add_tensor("blk.1.attn_q.weight", np.zeros(64, dtype = np.float32))
    writer.write_header_to_file()
    writer.write_kv_data_to_file()
    writer.write_tensors_to_file()
    writer.close()

    layout = layout_from_gguf(str(path))
    assert layout.complete
    assert layout.tensor_bytes == 4 * (16 + 1024 + 4096 + 256 + 64)
    assert layout.token_embd_bytes == 4 * 1024
    assert layout.per_layer_embd_bytes == 4 * 4096
    assert layout.other_resident_bytes == 4 * (16 + 1024)


@pytest.mark.parametrize("endianess", ["LITTLE", "BIG"])
def test_reading_the_layout_does_not_walk_the_vocabulary(tmp_path, monkeypatch, endianess):
    import numpy as np
    from gguf import GGUFEndian, GGUFReader, GGUFWriter

    vocab = 50_000
    path = tmp_path / "vocab.gguf"
    writer = GGUFWriter(str(path), "nemotron_h", endianess = GGUFEndian[endianess])
    writer.add_block_count(3)
    writer.add_head_count(8)
    writer.add_key_length(128)
    writer.add_value_length(128)
    # A per-layer array, so the numeric-array path must keep every element.
    writer.add_array("nemotron_h.attention.head_count_kv", [0, 4, 2])
    writer.add_token_list([f"tok{i}" for i in range(vocab)])
    writer.add_token_scores([0.0] * vocab)
    writer.add_token_types([1] * vocab)
    writer.add_tensor("token_embd.weight", np.zeros(1024, dtype = np.float32))
    writer.add_tensor("blk.0.attn_q.weight", np.zeros(256, dtype = np.float32))
    writer.write_header_to_file()
    writer.write_kv_data_to_file()
    writer.write_tensors_to_file()
    writer.close()

    reads = 0
    real_get = GGUFReader._get

    def counting_get(self, *args, **kwargs):
        nonlocal reads
        reads += 1
        return real_get(self, *args, **kwargs)

    monkeypatch.setattr(GGUFReader, "_get", counting_get)
    layout = layout_from_gguf(str(path))
    assert layout.complete
    assert layout.token_embd_bytes == 4 * 1024
    assert layout.n_attention_layers == 2
    assert layout.kv_bytes_per_token_f16 == (4 + 2) * (128 + 128) * 2
    assert reads < 1000


def test_token_embd_norm_is_not_an_input_embedding():
    """token_embd_norm follows the repeating layers rather than the input layer."""
    tensors = _shard_tensors(range(64)) + [
        _StubTensor("token_embd.weight", 400 * MIB),
        _StubTensor("token_embd_norm.weight", 4 * MIB),
        _StubTensor("output.weight", 400 * MIB),
    ]
    layout = _layout_from_reader(_StubReader(_shard_fields(), tensors))
    assert layout.token_embd_bytes == 400 * MIB
    assert layout.other_resident_bytes == 4 * MIB


def test_a_split_gguf_abstains_instead_of_planning_on_one_shard():
    """A split GGUF must abstain on one shard: undercounting the model is the optimistic direction."""
    partial = _StubReader(_shard_fields(**{"split.count": 4}), _shard_tensors(range(16)))
    assert _layout_from_reader(partial).complete is False

    whole = _layout_from_reader(_StubReader(_shard_fields(), _shard_tensors(range(64))))
    as_if_whole = _layout_from_reader(_StubReader(_shard_fields(), _shard_tensors(range(16))))
    assert as_if_whole.spillable_bytes * 4 == whole.spillable_bytes


def _dense_q4() -> ModelLayout:
    """Measured Qwen3.8-27B UD-Q4_K_XL buckets."""
    return _uniform_layout(
        spillable_total = int(10.092 * GIB),
        resident_total = int(4.614 * GIB),
        lm_head = int(0.971 * GIB),
        token_embd = int(0.666 * GIB),
    )


def test_a_unified_memory_host_never_spills():
    """Unified-memory hosts report RAM as VRAM, so a spill frees nothing on the device; abstain."""
    plan = plan_placement(
        _dense_q4(),
        [8 * GIB],
        64 * GIB,
        32768,
        opts = PlanOptions(host = HostProfile(unified_memory = True)),
    )
    assert plan.changed is False
    assert plan.ot_patterns == ()
    assert "unified memory" in plan.reason


def test_a_spilling_plan_reports_what_it_will_cost():
    """A plan that spills is not free, and the number has to travel with it."""
    tight = plan_placement(_dense_q4(), [8 * GIB], 64 * GIB, 32768, opts = FIXED_OVERHEAD_OPTS)
    roomy = plan_placement(_dense_q4(), [48 * GIB], 64 * GIB, 32768, opts = FIXED_OVERHEAD_OPTS)
    assert tight.spills_anything and tight.predicted_gen_penalty_ms > 0.0
    assert not roomy.spills_anything and roomy.predicted_gen_penalty_ms == 0.0


def test_a_small_host_is_predicted_to_suffer_more_for_the_same_spill():
    """Spilled decode runs on the CPU at batch 1 (ggml migrates an op only at batch >= 32), so the
    penalty tracks core count; a desktop must not be told the server story."""
    layout, vram, ram, ctx = _dense_q4(), [8 * GIB], 64 * GIB, 32768
    big = plan_placement(
        layout,
        vram,
        ram,
        ctx,
        opts = PlanOptions(overhead_bytes_per_device = GIB, host = HostProfile(threads = 192)),
    )
    small = plan_placement(
        layout,
        vram,
        ram,
        ctx,
        opts = PlanOptions(overhead_bytes_per_device = GIB, host = HostProfile(threads = 8)),
    )
    assert big.spilled_blocks == small.spilled_blocks, "same placement, different host"
    assert small.predicted_gen_penalty_ms > big.predicted_gen_penalty_ms * 2


def test_routed_experts_are_charged_less_than_a_dense_ffn_of_equal_size():
    """Only n_expert_used of n_expert experts are read per token, so MoE spills move fewer bytes."""
    dense = _dense_q4()
    moe = _uniform_layout(
        spillable_total = int(10.092 * GIB),
        resident_total = int(4.614 * GIB),
        lm_head = int(0.971 * GIB),
        token_embd = int(0.666 * GIB),
        is_moe = True,
    )
    moe = ModelLayout(**{**moe.__dict__, "n_expert": 256, "n_expert_used": 8})
    opts = PlanOptions(overhead_bytes_per_device = GIB, host = HostProfile(threads = 192))
    d = plan_placement(dense, [8 * GIB], 64 * GIB, 32768, opts = opts)
    m = plan_placement(moe, [8 * GIB], 64 * GIB, 32768, opts = opts)
    assert d.spilled_blocks == m.spilled_blocks, "same bytes spilled either way"
    assert m.predicted_gen_penalty_ms < d.predicted_gen_penalty_ms


@needs_gguf
def test_the_mtp_block_is_not_counted_as_spillable():
    """blk.<nextn> is not loaded without a draft, so an -ot pattern naming it frees nothing; do not
    count it as spillable, which would credit 209 MiB it can never free."""
    layout = layout_from_gguf(Q4_PATH)
    assert layout.n_layers == 64
    assert [b.index for b in layout.blocks] == list(range(64)), "no nextn block"
    assert all(b.index < layout.n_layers for b in layout.blocks)


def test_the_overhead_reserve_covers_the_measured_prefill_buffer():
    """The per-device reserve must cover the prefill compute buffer plus the CUDA primary context."""
    reserve = PlanOptions().overhead_bytes_per_device
    measured_prefill_buffer = int(594.16 * 1024 * 1024)
    inferred_cuda_context = GIB - measured_prefill_buffer
    assert reserve >= measured_prefill_buffer + inferred_cuda_context
    assert reserve > GIB, "1 GiB is the value that OOMed"
    assert reserve <= 2 * GIB


# Just too little VRAM, so every block spills and the compact pattern is emitted.
_NO_OVERHEAD = PlanOptions(overhead_bytes_per_device = 0)
_ALL_SPILL_VRAM = 3 * GIB + 64 * MIB


def _nextn_reader(nextn: int, total_blocks: int = 66):
    """block_count includes trailing nextn/MTP blocks: they are the MTP head, carrying real ffn weights."""
    fields = _shard_fields(
        **{"llama.block_count": total_blocks, "llama.nextn_predict_layers": nextn}
    )
    return _StubReader(fields, _shard_tensors(range(total_blocks)))


def _tied_reader(*, with_output: bool):
    """The 64-block stub plus a vocabulary matrix, tied or untied."""
    tensors = list(_shard_tensors(range(64)))
    tensors.append(_StubTensor("token_embd.weight", GIB))
    if with_output:
        tensors.append(_StubTensor("output.weight", GIB))
    return _StubReader(_shard_fields(), tensors)


def test_a_tied_embedding_gguf_still_charges_a_vocabulary_matrix_to_vram():
    """A tied output matrix is duplicated from token_embd on the output device, so it costs VRAM."""
    tied = _layout_from_reader(_tied_reader(with_output = False))
    untied = _layout_from_reader(_tied_reader(with_output = True))
    assert tied.complete and untied.complete
    assert tied.lm_head_bytes == 0, "there is no output.weight to spill"

    assert resident_floor_bytes(tied, 4096) == resident_floor_bytes(untied, 4096)
    assert all_resident_bytes(tied, 4096) == all_resident_bytes(untied, 4096)

    opts = PlanOptions(overhead_bytes_per_device = 0)
    tied_plan = plan_placement(tied, [10 * GIB], 256 * GIB, 4096, opts = opts)
    untied_plan = plan_placement(untied, [10 * GIB], 256 * GIB, 4096, opts = opts)
    assert tied_plan.spilled_blocks, "a partial spill, so lm_head is not in play"
    assert len(tied_plan.spilled_blocks) == len(untied_plan.spilled_blocks)

    # Tied duplicate keeps the name token_embd.weight, so LM_HEAD_PATTERN cannot move it.
    assert tied_plan.spilled_lm_head is False
    assert LM_HEAD_PATTERN not in tied_plan.ot_patterns

    assert tied_plan.host_bytes - untied_plan.host_bytes == 0


def test_excluded_mtp_block_bytes_are_kept_so_a_draft_can_be_charged():
    """Keep trailing MTP block bytes: --spec-type draft-mtp makes those blocks resident for the draft."""
    layout = _layout_from_reader(_nextn_reader(2))
    assert layout.has_excluded_blocks is True

    per_block = layout.blocks[0].spillable_bytes + layout.blocks[0].resident_bytes
    assert layout.excluded_block_bytes == 2 * per_block

    plain = _layout_from_reader(_StubReader(_shard_fields(), _shard_tensors(range(64))))
    assert plain.excluded_block_bytes == 0

    base = PlanOptions(overhead_bytes_per_device = 0)
    charged = PlanOptions(
        overhead_bytes_per_device = 0, extra_resident_bytes = layout.excluded_block_bytes
    )
    without = plan_placement(layout, [10 * GIB], 256 * GIB, 4096, opts = base)
    with_mtp = plan_placement(layout, [10 * GIB], 256 * GIB, 4096, opts = charged)
    assert len(with_mtp.spilled_blocks) > len(without.spilled_blocks)


def test_a_nextn_gguf_is_marked_as_having_excluded_blocks():
    with_mtp = _layout_from_reader(_nextn_reader(2))
    assert with_mtp.complete
    assert with_mtp.has_excluded_blocks is True
    assert [b.index for b in with_mtp.blocks] == list(range(64))

    plain = _layout_from_reader(_StubReader(_shard_fields(), _shard_tensors(range(64))))
    assert plain.has_excluded_blocks is False


def test_the_spill_pattern_never_reaches_an_excluded_mtp_block():
    """Spill patterns must not reach excluded MTP blocks; the unbounded ^blk form matched them."""
    layout = _layout_from_reader(_nextn_reader(2))
    plan = plan_placement(layout, [_ALL_SPILL_VRAM], 256 * GIB, 4096, opts = _NO_OVERHEAD)
    assert len(plan.spilled_blocks) == len(layout.blocks), "every block goes"
    assert len(plan.ot_patterns) >= 1

    pattern = re.compile(plan.ot_patterns[0])
    assert pattern.search("blk.0.ffn_up.weight"), "a target block still spills"
    assert pattern.search("blk.63.ffn_up.weight")
    for excluded in ("blk.64.ffn_up.weight", "blk.65.ffn_down.weight"):
        assert pattern.search(excluded) is None, excluded


def test_a_gguf_without_excluded_blocks_keeps_the_compact_pattern():
    """Bounded pattern only when blocks are excluded; else keep the compact global form."""
    layout = _layout_from_reader(_StubReader(_shard_fields(), _shard_tensors(range(64))))
    plan = plan_placement(layout, [_ALL_SPILL_VRAM], 256 * GIB, 4096, opts = _NO_OVERHEAD)
    assert len(plan.spilled_blocks) == len(layout.blocks)
    assert re.compile(plan.ot_patterns[0]).search("blk.999.ffn_up.weight")


def test_extra_resident_bytes_are_charged_against_the_pooled_budget():
    """GPU-resident bytes outside the layout (projector, MTP draft reserve) must shrink the budget."""
    layout = q4_layout()
    base = PlanOptions(overhead_bytes_per_device = GIB)
    charged = PlanOptions(overhead_bytes_per_device = GIB, extra_resident_bytes = 3 * GIB)

    without = plan_placement(layout, [16 * GIB], 128 * GIB, 8192, opts = base)
    with_extra = plan_placement(layout, [16 * GIB], 128 * GIB, 8192, opts = charged)

    assert without.spills_anything and with_extra.spills_anything
    assert len(with_extra.spilled_blocks) > len(without.spilled_blocks)
    assert max_context_for(layout, [16 * GIB], spill_all_ffn = True, opts = charged) < max_context_for(
        layout, [16 * GIB], spill_all_ffn = True, opts = base
    )


def test_row_ownership_is_modelled_on_raw_free_not_on_the_budget():
    """Row ownership must use raw free VRAM, as llama.cpp does, not the budget, or the split shifts."""
    layout = _layout_from_reader(_StubReader(_shard_fields(), _shard_tensors(range(64))))
    spilled = {b.index for b in layout.blocks}
    budgets = [2 * GIB, 2 * GIB]

    even = _per_device_shortfall(
        layout,
        _NO_OVERHEAD,
        4096,
        spilled,
        False,
        budgets,
        quantised = False,
        kv_bytes_floor = 0,
        split_weights_per_device = [8 * GIB, 8 * GIB],
    )
    lopsided = _per_device_shortfall(
        layout,
        _NO_OVERHEAD,
        4096,
        spilled,
        False,
        budgets,
        quantised = False,
        kv_bytes_floor = 0,
        split_weights_per_device = [1 * GIB, 15 * GIB],
    )
    assert even != lopsided, "the split weights have to reach _device_slots"
    assert "device 1" in (lopsided or ""), "the card drawing 60 of 65 rows is the one over"


def test_a_sliding_window_model_abstains_on_a_multi_gpu_split():
    """Sliding-window models abstain on multi-GPU: an even spread under-books the full-context cards."""
    layout = _layout_from_reader(_StubReader(_shard_fields(), _shard_tensors(range(64))))
    swa = replace(layout, has_swa = True)
    assert swa.n_attention_layers == swa.n_layers, "the earlier guard does NOT cover this"

    half = _ALL_SPILL_VRAM // 2
    plan = plan_placement(swa, [half, half], 256 * GIB, 4096, opts = _NO_OVERHEAD)
    assert plan.changed is False
    assert "sliding-window" in plan.reason

    one = plan_placement(swa, [_ALL_SPILL_VRAM], 256 * GIB, 4096, opts = _NO_OVERHEAD)
    assert len(one.spilled_blocks) == len(layout.blocks)


def _moe_reader(names):
    """A 2-block MoE GGUF whose expert tensors use ``names``."""
    fields = dict(_shard_fields())
    fields["llama.block_count"] = 2
    fields["llama.expert_count"] = 64
    fields["llama.expert_used_count"] = 8
    tensors = []
    for i in range(2):
        for n in names:
            tensors.append(_StubTensor(f"blk.{i}.{n}.weight", 400 * MIB))
        tensors.append(_StubTensor(f"blk.{i}.attn_q.weight", 10 * MIB))
    tensors.append(_StubTensor("token_embd.weight", 100 * MIB))
    tensors.append(_StubTensor("output.weight", 100 * MIB))
    return _StubReader(fields, tensors)


@pytest.mark.parametrize(
    "label,names,expect_mib",
    [
        ("split", ["ffn_up_exps", "ffn_gate_exps", "ffn_down_exps"], 1200),
        ("fused", ["ffn_gate_up_exps", "ffn_down_exps"], 800),
        ("chunked", ["ffn_up_chexps", "ffn_gate_chexps", "ffn_down_chexps"], 1200),
    ],
)
def test_every_expert_spelling_is_spillable(label, names, expect_mib):
    """Fused expert tensors like ffn_gate_up_exps are spillable experts, not only the split spelling."""
    layout = _layout_from_reader(_moe_reader(names))
    assert layout.is_moe
    assert layout.blocks[0].spillable_bytes == expect_mib * MIB

    pattern = spill_pattern_for(layout, None)
    for n in names:
        assert re.search(pattern, f"blk.0.{n}.weight"), f"{label}: {n} not matched"


def test_routed_latent_projections_are_never_spilled():
    """kimi-k3 ffn_routed_up/down look like experts but have no expert axis and plain
    GGML_OP_MUL_MAT, so every token hits them; never spill them."""
    layout = _layout_from_reader(_moe_reader(["ffn_routed_up", "ffn_routed_down"]))
    assert layout.blocks[0].spillable_bytes == 0
    pattern = spill_pattern_for(layout, None)
    assert not re.search(pattern, "blk.0.ffn_routed_up.weight")
    assert not re.search(pattern, "blk.0.ffn_routed_down.weight")


def _swa_layout(n_blocks = 64):
    """Every layer attends with an uneven per-layer cache n_attention_layers cannot express."""
    layout = _layout_from_reader(_StubReader(_shard_fields(), _shard_tensors(range(n_blocks))))
    return replace(layout, has_swa = True)


def test_a_per_layer_vector_replaces_the_sliding_window_abstain():
    """Gemma3 window layers cache a fraction of full context; without a per-layer vector the planner
    spreads cache evenly and refuses, with one it sees where the big caches land."""
    layout = _swa_layout()
    half = _ALL_SPILL_VRAM // 2
    spilled = {b.index for b in layout.blocks}

    without = _per_device_shortfall(
        layout,
        _NO_OVERHEAD,
        4096,
        spilled,
        False,
        [half, half],
        quantised = False,
        kv_bytes_floor = 0,
    )
    assert without is not None and "sliding-window" in without

    weights = [64 if (i % 5 == 0) else 1 for i in range(layout.n_layers)]
    with_vector = _per_device_shortfall(
        layout,
        _NO_OVERHEAD,
        4096,
        spilled,
        False,
        [half, half],
        quantised = False,
        kv_bytes_floor = 0,
        kv_layer_weights = weights,
    )
    assert with_vector is None or "sliding-window" not in with_vector


def test_the_vector_places_the_cache_it_does_not_resize_it():
    """Vector scaled to the priced total; only its shape moves cache between devices."""
    layout = _swa_layout()
    spilled = {b.index for b in layout.blocks}
    n = layout.n_layers
    budgets = [_ALL_SPILL_VRAM // 2, _ALL_SPILL_VRAM // 2]

    front = [1] * (n // 2) + [0] * (n - n // 2)
    back = [0] * (n // 2) + [1] * (n - n // 2)
    a = _per_device_shortfall(
        layout,
        _NO_OVERHEAD,
        32768,
        spilled,
        False,
        budgets,
        quantised = False,
        kv_bytes_floor = 8 * GIB,
        kv_layer_weights = front,
    )
    b = _per_device_shortfall(
        layout,
        _NO_OVERHEAD,
        32768,
        spilled,
        False,
        budgets,
        quantised = False,
        kv_bytes_floor = 8 * GIB,
        kv_layer_weights = back,
    )
    assert a is not None and b is not None
    assert "device 0" in a, "the front-loaded cache overflows the first card"
    assert "device 1" in b, "the back-loaded cache overflows the second"


def test_a_wrong_length_vector_is_ignored_rather_than_trusted():
    """A vector of the wrong length is not evidence: abstain, do not stretch it."""
    layout = _swa_layout()
    spilled = {b.index for b in layout.blocks}
    half = _ALL_SPILL_VRAM // 2
    got = _per_device_shortfall(
        layout,
        _NO_OVERHEAD,
        4096,
        spilled,
        False,
        [half, half],
        quantised = False,
        kv_bytes_floor = 0,
        kv_layer_weights = [1, 2, 3],
    )
    assert got is not None and "sliding-window" in got


def test_every_shard_handed_over_prices_the_whole_split():
    """Two readers summing to one layout, token_embd in its bucket whichever shard holds it."""
    first = _StubReader(
        _shard_fields(**{"split.count": 2, "split.no": 0}),
        _shard_tensors(range(32)) + [_StubTensor("output.weight", GIB)],
    )
    second = _StubReader(
        _shard_fields(**{"split.count": 2, "split.no": 1}),
        _shard_tensors(range(32, 64)) + [_StubTensor("token_embd.weight", 3 * GIB)],
    )
    layout = _layout_from_readers([first, second])
    assert layout.complete
    assert len(layout.blocks) == 64
    assert layout.token_embd_bytes == 3 * GIB
    assert layout.lm_head_bytes == GIB


def test_a_partial_shard_set_still_abstains():
    readers = [
        _StubReader(_shard_fields(**{"split.count": 3}), _shard_tensors(range(32))),
        _StubReader(_shard_fields(**{"split.count": 3}), _shard_tensors(range(32, 64))),
    ]
    assert not _layout_from_readers(readers).complete


def test_split_shard_paths_follow_llama_cpp_naming():
    paths = split_shard_paths("/m/model-Q4-00002-of-00003.gguf")
    assert paths == [
        "/m/model-Q4-00001-of-00003.gguf",
        "/m/model-Q4-00002-of-00003.gguf",
        "/m/model-Q4-00003-of-00003.gguf",
    ]
    assert split_shard_paths("/m/model-Q4.gguf") is None


def test_layout_from_gguf_reads_the_sibling_shards_only_when_asked(tmp_path):
    import numpy as np
    from gguf import GGUFWriter

    shards = [tmp_path / f"m-{i:05d}-of-00002.gguf" for i in (1, 2)]
    for shard, blocks in zip(shards, (range(32), range(32, 64))):
        writer = GGUFWriter(str(shard), "llama")
        for key, value in _shard_fields().items():
            if key.startswith("llama."):
                writer.add_uint32(key, value)
        writer.add_uint16("split.count", 2)
        for tensor in _shard_tensors(blocks):
            writer.add_tensor(tensor.name, np.zeros(4, dtype = np.float32))
        writer.write_header_to_file()
        writer.write_kv_data_to_file()
        writer.write_tensors_to_file()
        writer.close()

    assert not layout_from_gguf(str(shards[0])).complete
    whole = layout_from_gguf(str(shards[0]), all_shards = True)
    assert whole.complete and len(whole.blocks) == 64

    shards[1].unlink()
    assert not layout_from_gguf(str(shards[0]), all_shards = True).complete