Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
# SPDX-License-Identifier: AGPL-3.0-only
# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
"""The -ot planner must never emit -ngl or --no-kv-offload or charge token_embd to VRAM."""
from __future__ import annotations
import os
import re
from dataclasses import replace
import pytest
from core.inference.offload_layout import (
LM_HEAD_PATTERN,
BlockLayout,
ModelLayout,
_layout_from_reader,
_layout_from_readers,
layout_from_gguf,
split_shard_paths,
spill_pattern_for,
)
from core.inference.offload_cost_model import HostProfile
from core.inference.offload_planner import (
_device_slots,
_per_device_shortfall,
ContextPolicy,
Plan,
PlanOptions,
SpillOrder,
all_resident_bytes,
max_context_for,
plan_placement,
plan_to_args,
resident_floor_bytes,
)
GIB = 1024**3
MIB = 1024**2
_MODELS = "/mnt/disks/unslothai/daniel3/workspace_11/temp/loadmode_sim"
Q2_PATH = f"{_MODELS}/qwen38/Qwen3.8-27B-UD-Q2_K_XL.gguf"
Q4_PATH = f"{_MODELS}/qwen38/Qwen3.8-27B-UD-Q4_K_XL.gguf"
MOE_PATH = f"{_MODELS}/qwen36moe/Qwen3.6-35B-A3B-UD-Q4_K_XL.gguf"
needs_gguf = pytest.mark.skipif(
not os.path.exists(Q4_PATH), reason = "local GGUF fixtures not present"
)
# Byte totals measured on Qwen3.8-27B, so the ladder uses real arithmetic sans fixtures.
def _uniform_layout(
*,
n_blocks: int = 65,
spillable_total: int,
resident_total: int,
lm_head: int,
token_embd: int,
kv_per_token: int = 65536,
recurrent: int = 156_893_184,
is_moe: bool = False,
n_ctx_train: int = 262144,
) -> ModelLayout:
per_spill, per_res = spillable_total // n_blocks, resident_total // n_blocks
blocks = tuple(
BlockLayout(index = i, spillable_bytes = per_spill, resident_bytes = per_res)
for i in range(n_blocks)
)
return ModelLayout(
arch = "qwen35moe" if is_moe else "qwen35",
n_layers = 64,
n_attention_layers = 16,
blocks = blocks,
lm_head_bytes = lm_head,
token_embd_bytes = token_embd,
kv_bytes_per_token_f16 = kv_per_token,
recurrent_bytes = recurrent,
n_ctx_train = n_ctx_train,
is_moe = is_moe,
complete = True,
)
def q4_layout() -> ModelLayout:
return _uniform_layout(
spillable_total = 10_836_131_840,
resident_total = 4_953_923_584,
lm_head = 1_042_915_328,
token_embd = 715_128_832,
)
def q2_layout() -> ModelLayout:
return _uniform_layout(
spillable_total = 5_900_615_680,
resident_total = 2_785_009_664,
lm_head = 715_128_832,
token_embd = 417_202_176,
)
def uneven_layout() -> ModelLayout:
"""Four blocks with deliberately lopsided FFN, for selection policy."""
sizes = [100 * MIB, 400 * MIB, 200 * MIB, 50 * MIB]
blocks = tuple(
BlockLayout(index = i, spillable_bytes = s, resident_bytes = 10 * MIB)
for i, s in enumerate(sizes)
)
return ModelLayout(
arch = "qwen35",
n_layers = 4,
n_attention_layers = 4,
blocks = blocks,
lm_head_bytes = 100 * MIB,
token_embd_bytes = 50 * MIB,
kv_bytes_per_token_f16 = 1024,
recurrent_bytes = 0,
n_ctx_train = 65536,
complete = True,
)
def test_a_load_that_fits_spills_nothing():
plan = plan_placement(q2_layout(), [24 * GIB], 64 * GIB, 8192)
assert plan.spilled_blocks == ()
assert plan.spilled_lm_head is False
assert plan.ot_patterns == ()
assert plan.load_mode_none is True
assert "fits in VRAM" in plan.reason
def test_a_load_that_does_not_fit_spills_ffn_and_keeps_the_cache():
"""The whole point: weights move, the cache does not."""
plan = plan_placement(q4_layout(), [12 * GIB], 64 * GIB, 32768)
assert plan.spilled_blocks, "should have spilled something"
assert plan.spilled_lm_head is False, "lm_head is the LAST rung, not the first"
args = plan_to_args(plan)
assert "-ngl" not in args and "--n-gpu-layers" not in args
assert "--no-kv-offload" not in args and "-nkvo" not in args
def test_lm_head_is_only_spilled_after_every_block():
"""43% of generation alone, 16% after FFN. Never take it first."""
layout = q4_layout()
plan = plan_placement(layout, [7 * GIB], 64 * GIB, 8192)
assert plan.spilled_lm_head is True
assert len(plan.spilled_blocks) == len(layout.blocks), "all blocks go first"
def test_partial_spill_takes_only_what_is_needed():
layout = q4_layout()
plan = plan_placement(layout, [16 * GIB], 64 * GIB, 8192)
assert 0 < len(plan.spilled_blocks) < len(layout.blocks)
def test_a_load_that_cannot_fit_keeps_mmap_and_says_so():
"""mmap must stay when -ot is exhausted: it is what keeps an over-commit pageable."""
plan = plan_placement(q4_layout(), [4 * GIB], 64 * GIB, 131072)
assert plan.insufficient is True
assert plan.changed is False
assert plan.load_mode_none is False
assert plan_to_args(plan) == []
assert "smaller quant" in plan.reason
@pytest.mark.parametrize("vram", [4 * GIB, 8 * GIB, 12 * GIB, 16 * GIB, 24 * GIB])
@pytest.mark.parametrize("ctx", [4096, 32768, 131072])
def test_no_plan_ever_moves_the_kv_cache(vram, ctx):
"""-ngl and --no-kv-offload both cost 22x to 73x. Neither may ever appear."""
for layout in (q2_layout(), q4_layout()):
args = plan_to_args(plan_placement(layout, [vram], 64 * GIB, ctx))
for banned in ("-ngl", "--gpu-layers", "--n-gpu-layers", "-nkvo", "--no-kv-offload"):
assert banned not in args
def test_token_embd_is_never_charged_to_vram():
"""llama.cpp pins dev_input to the CPU, so the embedding is charged to host RAM, never VRAM."""
small = q4_layout()
big = ModelLayout(**{**small.__dict__, "token_embd_bytes": 4 * GIB})
tight = [12 * GIB]
a = plan_placement(small, tight, 64 * GIB, 8192)
b = plan_placement(big, tight, 64 * GIB, 8192)
assert a.vram_bytes == b.vram_bytes
assert a.spilled_blocks == b.spilled_blocks
assert b.host_bytes - a.host_bytes == 4 * GIB - small.token_embd_bytes
def test_load_mode_none_is_withheld_when_host_ram_cannot_hold_the_spill():
"""Disabling mmap without host RAM for the spill turns a pageable load into an OOM kill."""
layout = q4_layout()
roomy = plan_placement(layout, [12 * GIB], 64 * GIB, 8192)
cramped = plan_placement(layout, [12 * GIB], 3 * GIB, 8192)
assert roomy.load_mode_none is True
assert cramped.load_mode_none is False
def test_unreadable_host_ram_keeps_mmap():
plan = plan_placement(q4_layout(), [12 * GIB], None, 8192)
assert plan.load_mode_none is False
def test_every_emitted_pattern_is_anchored():
"""Unanchored is a live trap: 'output\\.weight' also matches every
blk.N.attn_output.weight, which silently moved 16 attention projections."""
for layout in (q2_layout(), q4_layout()):
for vram in (6 * GIB, 12 * GIB, 16 * GIB):
plan = plan_placement(layout, [vram], 64 * GIB, 8192)
for pattern in plan.ot_patterns:
assert pattern.startswith("^") and pattern.endswith("$"), pattern
def test_the_lm_head_pattern_does_not_catch_attention_output():
assert re.search(LM_HEAD_PATTERN, "output.weight")
assert not re.search(LM_HEAD_PATTERN, "blk.3.attn_output.weight")
assert re.search(r"output\.weight", "blk.3.attn_output.weight")
def test_the_dense_pattern_does_not_catch_the_router():
pattern = spill_pattern_for(q4_layout())
assert re.search(pattern, "blk.7.ffn_up.weight")
assert re.search(pattern, "blk.7.ffn_gate.weight")
assert not re.search(pattern, "blk.7.ffn_gate_inp.weight")
assert not re.search(pattern, "blk.7.ffn_norm.weight")
assert not re.search(pattern, "blk.7.attn_q.weight")
def test_the_moe_pattern_spills_experts_but_not_shared_experts():
"""Shared experts hit every token like a dense FFN; spilling them only costs bandwidth."""
moe = _uniform_layout(
n_blocks = 40,
spillable_total = 19_671_285_760,
resident_total = 1_597_483_520,
lm_head = 540_000_000,
token_embd = 540_000_000,
kv_per_token = 20480,
is_moe = True,
)
pattern = spill_pattern_for(moe)
assert re.search(pattern, "blk.5.ffn_up_exps.weight")
assert re.search(pattern, "blk.5.ffn_down_exps.weight")
assert not re.search(pattern, "blk.5.ffn_up_shexp.weight")
assert not re.search(pattern, "blk.5.ffn_gate_inp.weight")
def test_partial_spill_names_only_the_chosen_blocks():
pattern = spill_pattern_for(q4_layout(), [3, 7, 11])
assert re.search(pattern, "blk.3.ffn_up.weight")
assert re.search(pattern, "blk.11.ffn_down.weight")
assert not re.search(pattern, "blk.4.ffn_up.weight")
assert not re.search(pattern, "blk.1.ffn_up.weight")
def test_a_full_spill_uses_the_compact_global_pattern():
layout = q4_layout()
plan = plan_placement(layout, [7 * GIB], 64 * GIB, 8192)
assert plan.ot_patterns[0] == spill_pattern_for(layout, None)
assert r"\d+" in plan.ot_patterns[0]
def test_plan_to_args_shape():
layout = q4_layout()
args = plan_to_args(plan_placement(layout, [12 * GIB], 64 * GIB, 8192))
assert args.count("-ot") == 1
assert args[args.index("-ot") + 1].endswith("=CPU")
assert "--load-mode" in args and args[args.index("--load-mode") + 1] == "none"
def test_largest_first_minimises_overshoot():
"""Cover a small residual with a small block, not a big one crossed on every token."""
layout = uneven_layout()
floor = resident_floor_bytes(layout, 4096)
budget = floor + layout.spillable_bytes - 40 * MIB + 1 * GIB
plan = plan_placement(
layout,
[budget],
64 * GIB,
4096,
opts = PlanOptions(
overhead_bytes_per_device = GIB,
),
)
spilled = sum(b.spillable_bytes for b in layout.blocks if b.index in plan.spilled_blocks)
assert spilled == 50 * MIB, "should take the 50 MiB block, not a bigger one"
def test_front_and_back_orders_pick_opposite_ends():
layout = uneven_layout()
floor = resident_floor_bytes(layout, 4096)
budget = floor + layout.spillable_bytes - 40 * MIB + 1 * GIB
front = plan_placement(
layout,
[budget],
64 * GIB,
4096,
opts = PlanOptions(overhead_bytes_per_device = GIB, spill_order = SpillOrder.FRONT_FIRST),
)
back = plan_placement(
layout,
[budget],
64 * GIB,
4096,
opts = PlanOptions(overhead_bytes_per_device = GIB, spill_order = SpillOrder.BACK_FIRST),
)
assert front.spilled_blocks == (0,)
assert back.spilled_blocks == (3,)
def test_an_incomplete_layout_abstains():
plan = plan_placement(ModelLayout(), [24 * GIB], 64 * GIB, 8192)
assert plan.changed is False
assert plan_to_args(plan) == []
def test_no_devices_abstains():
assert plan_placement(q4_layout(), [], 64 * GIB, 8192).changed is False
def test_a_device_smaller_than_its_own_overhead_abstains():
assert plan_placement(q4_layout(), [512 * MIB], 64 * GIB, 8192).changed is False
def test_every_device_pays_the_fixed_overhead():
"""Every split device pays a CUDA context and scratch, so two 8 GiB cards hold less than one 16 GiB."""
layout = q4_layout()
one_big = max_context_for(layout, [16 * GIB], spill_all_ffn = True)
two_small = max_context_for(layout, [8 * GIB, 8 * GIB], spill_all_ffn = True)
assert one_big > 0 and two_small > 0
assert two_small < one_big
def test_a_partial_spill_across_two_gpus_abstains():
"""A partial spill across two GPUs must abstain: -ot can relieve one card and leave the other over."""
layout = q4_layout()
one_card = plan_placement(layout, [16 * GIB], 64 * GIB, 8192)
assert one_card.spilled_blocks
assert len(one_card.spilled_blocks) < len(layout.blocks), "partial, not everything"
two_cards = plan_placement(layout, [8 * GIB, 8 * GIB], 64 * GIB, 8192)
assert two_cards.spilled_blocks == ()
assert two_cards.ot_patterns == ()
assert two_cards.changed is False
def test_a_safe_partial_spill_across_two_gpus_is_planned():
sizes = [GIB // 2, GIB // 2, GIB // 2, 2 * GIB]
layout = ModelLayout(
arch = "qwen35",
n_layers = 4,
n_attention_layers = 4,
blocks = tuple(
BlockLayout(index = i, spillable_bytes = size, resident_bytes = GIB // 10)
for i, size in enumerate(sizes)
),
lm_head_bytes = GIB // 10,
token_embd_bytes = 0,
kv_bytes_per_token_f16 = 0,
recurrent_bytes = 0,
n_ctx_train = 4096,
complete = True,
)
opts = PlanOptions(
overhead_bytes_per_device = GIB,
pipeline_overhead_bytes = GIB,
host_ram_headroom_bytes = 0,
)
plan = plan_placement(
layout,
[23 * GIB // 10, 22 * GIB // 10],
64 * GIB,
4096,
opts = opts,
)
assert plan.changed is True
assert plan.spilled_blocks == (0, 3)
def test_partial_spill_selection_covers_each_device_shortfall():
sizes = [1200 * MIB, 0, 0, 600 * MIB, 600 * MIB]
layout = ModelLayout(
arch = "qwen35",
n_layers = 5,
n_attention_layers = 5,
blocks = tuple(
BlockLayout(index = i, spillable_bytes = size, resident_bytes = 0)
for i, size in enumerate(sizes)
),
lm_head_bytes = 0,
token_embd_bytes = 0,
kv_bytes_per_token_f16 = 0,
recurrent_bytes = 0,
n_ctx_train = 4096,
complete = True,
)
plan = plan_placement(
layout,
[1300 * MIB, 100 * MIB],
64 * GIB,
4096,
opts = PlanOptions(overhead_bytes_per_device = 0, host_ram_headroom_bytes = 0),
split_weights_per_device = [1, 1],
)
assert plan.spilled_blocks == (3, 4)
def test_per_device_selection_cannot_drop_cache_remainder_from_the_pool():
spillable = [17, 4, 14, 20, 3, 5, 17, 17]
resident = [1, 5, 5, 1, 6, 6, 5, 0]
layout = ModelLayout(
arch = "qwen35",
n_layers = 8,
n_attention_layers = 8,
blocks = tuple(
BlockLayout(index = i, spillable_bytes = spill, resident_bytes = keep)
for i, (spill, keep) in enumerate(zip(spillable, resident))
),
lm_head_bytes = 0,
token_embd_bytes = 0,
other_resident_bytes = 8,
kv_bytes_per_token_f16 = 3,
recurrent_bytes = 0,
n_ctx_train = 4096,
complete = True,
)
opts = PlanOptions(
overhead_bytes_per_device = 8,
pipeline_overhead_bytes = 5,
extra_resident_bytes = 4,
host_ram_headroom_bytes = 0,
)
plan = plan_placement(
layout,
[90, 25],
1024,
5,
opts = opts,
split_weights_per_device = [56, 27],
)
assert not plan.changed or plan.vram_bytes <= 90
def test_output_device_shortfall_can_reach_the_lm_head_rung():
layout = ModelLayout(
arch = "qwen35",
n_layers = 3,
n_attention_layers = 3,
blocks = tuple(
BlockLayout(index = i, spillable_bytes = size, resident_bytes = 0)
for i, size in enumerate([100, 100, 10])
),
lm_head_bytes = 100,
token_embd_bytes = 0,
kv_bytes_per_token_f16 = 0,
recurrent_bytes = 0,
n_ctx_train = 4096,
complete = True,
)
plan = plan_placement(
layout,
[100, 20],
1024,
1,
opts = PlanOptions(overhead_bytes_per_device = 0, host_ram_headroom_bytes = 0),
split_weights_per_device = [1, 1],
)
assert plan.spilled_blocks == (0,)
assert plan.spilled_lm_head is True
assert plan.vram_bytes <= 120
def test_a_full_spill_is_checked_per_device_not_assumed():
"""A full spill is checked per device: 65 rows over two equal cards is 33/32, not a pooled fit."""
layout = _layout_from_reader(_StubReader(_shard_fields(), _shard_tensors(range(64))))
half = _ALL_SPILL_VRAM // 2
plan = plan_placement(layout, [half, half], 256 * GIB, 4096, opts = _NO_OVERHEAD)
assert plan.changed is False
assert plan.spilled_blocks == ()
assert "device 0" in plan.reason
one = plan_placement(layout, [_ALL_SPILL_VRAM], 256 * GIB, 4096, opts = _NO_OVERHEAD)
assert len(one.spilled_blocks) == len(layout.blocks)
def test_a_full_spill_abstains_when_the_cache_layout_is_unknown():
"""A hybrid's recurrent layers are not recorded per layer, so a multi-card full spill must abstain."""
layout = _layout_from_reader(_StubReader(_shard_fields(), _shard_tensors(range(64))))
hybrid = replace(layout, recurrent_bytes = 8 * MIB)
half = _ALL_SPILL_VRAM // 2
plan = plan_placement(hybrid, [half, half], 256 * GIB, 4096, opts = _NO_OVERHEAD)
assert plan.changed is False
assert "recurrent state" in plan.reason
one = plan_placement(hybrid, [_ALL_SPILL_VRAM], 256 * GIB, 4096, opts = _NO_OVERHEAD)
assert len(one.spilled_blocks) == len(layout.blocks)
def test_the_row_split_matches_llama_cpp():
"""_device_slots mirrors llama-model.cpp:1439-1457: contiguous ranges sized in proportion to
free VRAM, with the remainder rows landing on the earlier GPU."""
assert [len(r) for r in _device_slots(65, [8 * GIB, 8 * GIB])] == [33, 32]
assert [len(r) for r in _device_slots(65, [24 * GIB, 8 * GIB])] == [49, 16]
assert [len(r) for r in _device_slots(65, [8 * GIB, 8 * GIB, 8 * GIB])] == [22, 22, 21]
rows = _device_slots(65, [24 * GIB, 8 * GIB])
assert rows[0] == list(range(0, 49)) and rows[1] == list(range(49, 65))
assert _device_slots(65, [8 * GIB]) == [list(range(65))]
assert _device_slots(4, [0, 0]) == [[0, 1, 2, 3], []]
def test_the_row_split_uses_llama_cpp_float32_boundaries():
rows = _device_slots(353, [39407 * MIB, 12114 * MIB])
assert len(rows[0]) == 270
assert 270 in rows[1]
def test_a_float32_split_boundary_cannot_approve_an_oom():
blocks = tuple(
BlockLayout(
index = i,
spillable_bytes = 2 * GIB,
resident_bytes = 2 * GIB if i == 270 else 0,
)
for i in range(352)
)
layout = ModelLayout(
arch = "qwen35",
n_layers = 352,
n_attention_layers = 352,
blocks = blocks,
lm_head_bytes = 0,
token_embd_bytes = 0,
kv_bytes_per_token_f16 = 0,
recurrent_bytes = 0,
n_ctx_train = 4096,
complete = True,
)
plan = plan_placement(
layout,
[2 * GIB, GIB],
1024 * GIB,
4096,
opts = PlanOptions(overhead_bytes_per_device = 0, host_ram_headroom_bytes = 0),
split_weights_per_device = [39407 * MIB, 12114 * MIB],
)
assert plan.changed is False
assert "device 1" in plan.reason
def test_the_per_device_check_passes_when_the_shares_really_fit():
"""The check must not be a disguised "never plan on two GPUs". Cards sized so
that each one's row share fits with room to spare return None -- no abstain
reason -- for the same full spill the tight case rejects."""
layout = _layout_from_reader(_StubReader(_shard_fields(), _shard_tensors(range(64))))
spilled = {b.index for b in layout.blocks}
tight = _ALL_SPILL_VRAM // 2
assert (
_per_device_shortfall(
layout,
_NO_OVERHEAD,
4096,
spilled,
False,
[tight, tight],
quantised = False,
kv_bytes_floor = 0,
)
is not None
)
roomy = _ALL_SPILL_VRAM
assert (
_per_device_shortfall(
layout,
_NO_OVERHEAD,
4096,
spilled,
False,
[roomy, roomy],
quantised = False,
kv_bytes_floor = 0,
)
is None
)
def test_the_per_device_check_charges_each_secondary_pipeline_reserve():
layout = uneven_layout()
spilled = {b.index for b in layout.blocks}
cache_per_layer = layout.kv_bytes(4096, 2) // layout.n_layers
device_one_used = layout.blocks[3].resident_bytes + cache_per_layer + layout.lm_head_bytes
pipeline_reserve = GIB
opts = PlanOptions(overhead_bytes_per_device = 0, pipeline_overhead_bytes = pipeline_reserve)
below = _per_device_shortfall(
layout,
opts,
4096,
spilled,
False,
[4 * GIB, device_one_used + pipeline_reserve - 1],
quantised = False,
kv_bytes_floor = 0,
split_weights_per_device = [1, 1],
)
exact = _per_device_shortfall(
layout,
opts,
4096,
spilled,
False,
[4 * GIB, device_one_used + pipeline_reserve],
quantised = False,
kv_bytes_floor = 0,
split_weights_per_device = [1, 1],
)
above = _per_device_shortfall(
layout,
opts,
4096,
spilled,
False,
[4 * GIB, device_one_used + pipeline_reserve + 1],
quantised = False,
kv_bytes_floor = 0,
split_weights_per_device = [1, 1],
)
assert below is not None and "device 1" in below
assert exact is None
assert above is None
def test_an_empty_secondary_still_has_to_fit_its_fixed_reserves():
layout = uneven_layout()
spilled = {b.index for b in layout.blocks}
pipeline_reserve = GIB
opts = PlanOptions(overhead_bytes_per_device = 0, pipeline_overhead_bytes = pipeline_reserve)
below = _per_device_shortfall(
layout,
opts,
4096,
spilled,
True,
[4 * GIB, pipeline_reserve - 1],
quantised = False,
kv_bytes_floor = 0,
split_weights_per_device = [1000, 1],
)
exact = _per_device_shortfall(
layout,
opts,
4096,
spilled,
True,
[4 * GIB, pipeline_reserve],
quantised = False,
kv_bytes_floor = 0,
split_weights_per_device = [1000, 1],
)
assert _device_slots(layout.n_layers + 1, [1000, 1])[1] == []
assert below is not None and "device 1" in below
assert exact is None
def test_multi_gpu_credit_sums():
layout = q2_layout()
assert (
plan_placement(
layout, [6 * GIB, 6 * GIB], 64 * GIB, 8192, opts = FIXED_OVERHEAD_OPTS
).spilled_blocks
== ()
)
def test_never_reduce_keeps_the_requested_context():
layout = q4_layout()
plan = plan_placement(layout, [12 * GIB], 64 * GIB, 65536)
assert plan.n_ctx == 65536
assert plan.spilled_blocks, "it pays with spill, not with the user's context"
def test_prefer_resident_shrinks_instead_of_spilling():
layout = q4_layout()
plan = plan_placement(
layout,
[18 * GIB],
64 * GIB,
65536,
opts = PlanOptions(context_policy = ContextPolicy.PREFER_RESIDENT),
)
assert plan.spilled_blocks == ()
assert plan.n_ctx < 65536
assert "shrank context" in plan.reason
def test_prefer_resident_still_spills_when_even_min_ctx_will_not_fit():
layout = q4_layout()
plan = plan_placement(
layout,
[10 * GIB],
64 * GIB,
65536,
opts = PlanOptions(context_policy = ContextPolicy.PREFER_RESIDENT),
)
assert plan.spilled_blocks, "shrinking cannot save this one, so spill"
def test_context_is_clamped_to_what_the_model_was_trained_on():
layout = q4_layout()
plan = plan_placement(layout, [24 * GIB], 128 * GIB, 999_999)
assert plan.n_ctx == layout.n_ctx_train
def test_kv_quantisation_is_off_by_default():
"""35% slower generation, and only matched pairs are compiled."""
plan = plan_placement(q4_layout(), [12 * GIB], 64 * GIB, 65536)
assert plan.cache_type_k is None and plan.cache_type_v is None
def test_kv_quantisation_rescues_a_load_f16_cannot_fit():
"""The f16 KV cache alone overshoots 8 GiB usable at 64K; halving it makes the load fit."""
layout = q4_layout()
without = plan_placement(layout, [9 * GIB], 64 * GIB, 65536)
assert without.insufficient is True
with_quant = plan_placement(
layout,
[9 * GIB],
64 * GIB,
65536,
opts = PlanOptions(allow_kv_quant = True),
)
assert with_quant.insufficient is False
assert with_quant.cache_type_k == "q8_0"
# Matched K/V pair: mixed types need GGML_CUDA_FA_ALL_QUANTS or fall back to CPU.
assert with_quant.cache_type_k == with_quant.cache_type_v
def test_f16_is_preferred_when_it_fits_even_with_quant_allowed():
"""q8_0 costs 35% of generation, so it is a rescue, not a default."""
plan = plan_placement(
q2_layout(),
[24 * GIB],
64 * GIB,
8192,
opts = PlanOptions(allow_kv_quant = True),
)
assert plan.cache_type_k is None
# Independently computed table: max context with FFN spilled, f16 cache, 1 GiB overhead.
@pytest.mark.parametrize(
"budget_gib,expected_k",
[(8, 20), (10, 52), (12, 84), (16, 148), (20, 212)],
)
def test_q4_ffn_spilled_context_ladder(budget_gib, expected_k):
got = max_context_for(
q4_layout(), [budget_gib * GIB], spill_all_ffn = True, opts = FIXED_OVERHEAD_OPTS
)
assert expected_k * 1024 <= got < (expected_k + 1) * 1024, got
# Arithmetic against a stated overhead, not measurements; the constant is pinned by
# test_the_overhead_reserve_covers_the_measured_prefill_buffer.
FIXED_OVERHEAD_OPTS = PlanOptions(overhead_bytes_per_device = GIB)
@pytest.mark.parametrize("budget_gib,expected_k", [(6, 25), (8, 57), (12, 121), (20, 249)])
def test_q2_ffn_spilled_context_ladder(budget_gib, expected_k):
got = max_context_for(
q2_layout(), [budget_gib * GIB], spill_all_ffn = True, opts = FIXED_OVERHEAD_OPTS
)
assert expected_k * 1024 <= got < (expected_k + 1) * 1024, got
@pytest.mark.parametrize("budget_gib,expected_k", [(12, 33), (16, 97), (24, 225)])
def test_q2_fully_resident_context_ladder(budget_gib, expected_k):
got = max_context_for(q2_layout(), [budget_gib * GIB], opts = FIXED_OVERHEAD_OPTS)
assert expected_k * 1024 <= got < (expected_k + 1) * 1024, got
@pytest.mark.parametrize("budget_gib", [4, 6, 8, 10])
def test_q4_cannot_be_fully_resident_below_18_gib(budget_gib):
assert max_context_for(q4_layout(), [budget_gib * GIB]) == 0
def test_the_ladder_never_regresses_as_vram_grows():
"""More VRAM must never mean more spill."""
layout = q4_layout()
counts = [
len(
plan_placement(
layout, [g * GIB], 64 * GIB, 32768, opts = FIXED_OVERHEAD_OPTS
).spilled_blocks
)
for g in (8, 10, 12, 14, 16, 18, 20, 22, 24)
]
assert counts == sorted(counts, reverse = True), counts
@needs_gguf
@pytest.mark.parametrize(
"path,spillable_gib,resident_gib,lm_head_gib,embd_gib,moe",
[
(Q2_PATH, 5.291, 2.471, 0.666, 0.389, False),
(Q4_PATH, 9.888, 4.491, 0.971, 0.666, False),
(MOE_PATH, 18.320, 1.488, 0.503, 0.503, True),
],
)
def test_layout_matches_the_measured_buckets(
path, spillable_gib, resident_gib, lm_head_gib, embd_gib, moe
):
layout = layout_from_gguf(path)
assert layout.complete
assert layout.is_moe is moe
assert layout.spillable_bytes / GIB == pytest.approx(spillable_gib, abs = 0.01)
assert layout.block_resident_bytes / GIB == pytest.approx(resident_gib, abs = 0.01)
assert layout.lm_head_bytes / GIB == pytest.approx(lm_head_gib, abs = 0.01)
assert layout.token_embd_bytes / GIB == pytest.approx(embd_gib, abs = 0.01)
@needs_gguf
def test_the_hybrid_cache_is_priced_on_attention_layers_only():
"""qwen35 is 1-in-4 attention; an all-layers cache would be 4x too big and refuse fits."""
layout = layout_from_gguf(Q4_PATH)
assert layout.n_layers == 64
assert layout.n_attention_layers == 16
assert layout.kv_bytes_per_token_f16 == 16 * 4 * (256 + 256) * 2
assert layout.recurrent_bytes / MIB == pytest.approx(149.6, abs = 1.0)
@needs_gguf
def test_a_bad_path_abstains_rather_than_raising():
assert layout_from_gguf("/nonexistent/model.gguf").complete is False
class _StubField:
def __init__(self, value):
self._value = value
def contents(self):
return self._value
class _StubTensor:
def __init__(self, name, n_bytes):
self.name = name
self.n_bytes = n_bytes
class _StubReader:
"""The handful of attributes ``_layout_from_reader`` touches."""
def __init__(self, fields, tensors):
self.fields = {k: _StubField(v) for k, v in fields.items()}
self.tensors = tensors
def _shard_fields(**extra):
base = {
"general.architecture": "llama",
"llama.block_count": 64,
"llama.attention.head_count_kv": 8,
"llama.attention.head_count": 64,
"llama.embedding_length": 8192,
"llama.attention.key_length": 128,
"llama.attention.value_length": 128,
"llama.context_length": 262144,
}
base.update(extra)
return base
def _shard_tensors(indices):
out = []
for i in indices:
out.append(_StubTensor(f"blk.{i}.ffn_up.weight", GIB // 8))
out.append(_StubTensor(f"blk.{i}.attn_q.weight", MIB * 32))
return out
def test_a_single_file_gguf_is_still_read():
"""The guard below must not catch an ordinary one-file model."""
layout = _layout_from_reader(_StubReader(_shard_fields(), _shard_tensors(range(64))))
assert layout.complete
assert len(layout.blocks) == 64
def test_the_layout_reader_totals_every_tensor_from_a_real_gguf(tmp_path):
"""tensor_bytes includes every tensor while token_embd_bytes is embedding-only."""
import numpy as np
from gguf import GGUFWriter
path = tmp_path / "embd.gguf"
writer = GGUFWriter(str(path), "llama")
for key, value in _shard_fields().items():
if key.startswith("llama.") and isinstance(value, int):
writer.add_uint32(key, 1 if key == "llama.block_count" else value)
writer.add_tensor("output_norm.weight", np.zeros(16, dtype = np.float32))
writer.add_tensor("token_embd.weight", np.zeros(1024, dtype = np.float32))
writer.add_tensor("per_layer_token_embd.weight", np.zeros(4096, dtype = np.float32))
writer.add_tensor("blk.0.attn_q.weight", np.zeros(256, dtype = np.float32))
writer.add_tensor("blk.1.attn_q.weight", np.zeros(64, dtype = np.float32))
writer.write_header_to_file()
writer.write_kv_data_to_file()
writer.write_tensors_to_file()
writer.close()
layout = layout_from_gguf(str(path))
assert layout.complete
assert layout.tensor_bytes == 4 * (16 + 1024 + 4096 + 256 + 64)
assert layout.token_embd_bytes == 4 * 1024
assert layout.per_layer_embd_bytes == 4 * 4096
assert layout.other_resident_bytes == 4 * (16 + 1024)
@pytest.mark.parametrize("endianess", ["LITTLE", "BIG"])
def test_reading_the_layout_does_not_walk_the_vocabulary(tmp_path, monkeypatch, endianess):
import numpy as np
from gguf import GGUFEndian, GGUFReader, GGUFWriter
vocab = 50_000
path = tmp_path / "vocab.gguf"
writer = GGUFWriter(str(path), "nemotron_h", endianess = GGUFEndian[endianess])
writer.add_block_count(3)
writer.add_head_count(8)
writer.add_key_length(128)
writer.add_value_length(128)
# A per-layer array, so the numeric-array path must keep every element.
writer.add_array("nemotron_h.attention.head_count_kv", [0, 4, 2])
writer.add_token_list([f"tok{i}" for i in range(vocab)])
writer.add_token_scores([0.0] * vocab)
writer.add_token_types([1] * vocab)
writer.add_tensor("token_embd.weight", np.zeros(1024, dtype = np.float32))
writer.add_tensor("blk.0.attn_q.weight", np.zeros(256, dtype = np.float32))
writer.write_header_to_file()
writer.write_kv_data_to_file()
writer.write_tensors_to_file()
writer.close()
reads = 0
real_get = GGUFReader._get
def counting_get(self, *args, **kwargs):
nonlocal reads
reads += 1
return real_get(self, *args, **kwargs)
monkeypatch.setattr(GGUFReader, "_get", counting_get)
layout = layout_from_gguf(str(path))
assert layout.complete
assert layout.token_embd_bytes == 4 * 1024
assert layout.n_attention_layers == 2
assert layout.kv_bytes_per_token_f16 == (4 + 2) * (128 + 128) * 2
assert reads < 1000
def test_token_embd_norm_is_not_an_input_embedding():
"""token_embd_norm follows the repeating layers rather than the input layer."""
tensors = _shard_tensors(range(64)) + [
_StubTensor("token_embd.weight", 400 * MIB),
_StubTensor("token_embd_norm.weight", 4 * MIB),
_StubTensor("output.weight", 400 * MIB),
]
layout = _layout_from_reader(_StubReader(_shard_fields(), tensors))
assert layout.token_embd_bytes == 400 * MIB
assert layout.other_resident_bytes == 4 * MIB
def test_a_split_gguf_abstains_instead_of_planning_on_one_shard():
"""A split GGUF must abstain on one shard: undercounting the model is the optimistic direction."""
partial = _StubReader(_shard_fields(**{"split.count": 4}), _shard_tensors(range(16)))
assert _layout_from_reader(partial).complete is False
whole = _layout_from_reader(_StubReader(_shard_fields(), _shard_tensors(range(64))))
as_if_whole = _layout_from_reader(_StubReader(_shard_fields(), _shard_tensors(range(16))))
assert as_if_whole.spillable_bytes * 4 == whole.spillable_bytes
def _dense_q4() -> ModelLayout:
"""Measured Qwen3.8-27B UD-Q4_K_XL buckets."""
return _uniform_layout(
spillable_total = int(10.092 * GIB),
resident_total = int(4.614 * GIB),
lm_head = int(0.971 * GIB),
token_embd = int(0.666 * GIB),
)
def test_a_unified_memory_host_never_spills():
"""Unified-memory hosts report RAM as VRAM, so a spill frees nothing on the device; abstain."""
plan = plan_placement(
_dense_q4(),
[8 * GIB],
64 * GIB,
32768,
opts = PlanOptions(host = HostProfile(unified_memory = True)),
)
assert plan.changed is False
assert plan.ot_patterns == ()
assert "unified memory" in plan.reason
def test_a_spilling_plan_reports_what_it_will_cost():
"""A plan that spills is not free, and the number has to travel with it."""
tight = plan_placement(_dense_q4(), [8 * GIB], 64 * GIB, 32768, opts = FIXED_OVERHEAD_OPTS)
roomy = plan_placement(_dense_q4(), [48 * GIB], 64 * GIB, 32768, opts = FIXED_OVERHEAD_OPTS)
assert tight.spills_anything and tight.predicted_gen_penalty_ms > 0.0
assert not roomy.spills_anything and roomy.predicted_gen_penalty_ms == 0.0
def test_a_small_host_is_predicted_to_suffer_more_for_the_same_spill():
"""Spilled decode runs on the CPU at batch 1 (ggml migrates an op only at batch >= 32), so the
penalty tracks core count; a desktop must not be told the server story."""
layout, vram, ram, ctx = _dense_q4(), [8 * GIB], 64 * GIB, 32768
big = plan_placement(
layout,
vram,
ram,
ctx,
opts = PlanOptions(overhead_bytes_per_device = GIB, host = HostProfile(threads = 192)),
)
small = plan_placement(
layout,
vram,
ram,
ctx,
opts = PlanOptions(overhead_bytes_per_device = GIB, host = HostProfile(threads = 8)),
)
assert big.spilled_blocks == small.spilled_blocks, "same placement, different host"
assert small.predicted_gen_penalty_ms > big.predicted_gen_penalty_ms * 2
def test_routed_experts_are_charged_less_than_a_dense_ffn_of_equal_size():
"""Only n_expert_used of n_expert experts are read per token, so MoE spills move fewer bytes."""
dense = _dense_q4()
moe = _uniform_layout(
spillable_total = int(10.092 * GIB),
resident_total = int(4.614 * GIB),
lm_head = int(0.971 * GIB),
token_embd = int(0.666 * GIB),
is_moe = True,
)
moe = ModelLayout(**{**moe.__dict__, "n_expert": 256, "n_expert_used": 8})
opts = PlanOptions(overhead_bytes_per_device = GIB, host = HostProfile(threads = 192))
d = plan_placement(dense, [8 * GIB], 64 * GIB, 32768, opts = opts)
m = plan_placement(moe, [8 * GIB], 64 * GIB, 32768, opts = opts)
assert d.spilled_blocks == m.spilled_blocks, "same bytes spilled either way"
assert m.predicted_gen_penalty_ms < d.predicted_gen_penalty_ms
@needs_gguf
def test_the_mtp_block_is_not_counted_as_spillable():
"""blk.<nextn> is not loaded without a draft, so an -ot pattern naming it frees nothing; do not
count it as spillable, which would credit 209 MiB it can never free."""
layout = layout_from_gguf(Q4_PATH)
assert layout.n_layers == 64
assert [b.index for b in layout.blocks] == list(range(64)), "no nextn block"
assert all(b.index < layout.n_layers for b in layout.blocks)
def test_the_overhead_reserve_covers_the_measured_prefill_buffer():
"""The per-device reserve must cover the prefill compute buffer plus the CUDA primary context."""
reserve = PlanOptions().overhead_bytes_per_device
measured_prefill_buffer = int(594.16 * 1024 * 1024)
inferred_cuda_context = GIB - measured_prefill_buffer
assert reserve >= measured_prefill_buffer + inferred_cuda_context
assert reserve > GIB, "1 GiB is the value that OOMed"
assert reserve <= 2 * GIB
# Just too little VRAM, so every block spills and the compact pattern is emitted.
_NO_OVERHEAD = PlanOptions(overhead_bytes_per_device = 0)
_ALL_SPILL_VRAM = 3 * GIB + 64 * MIB
def _nextn_reader(nextn: int, total_blocks: int = 66):
"""block_count includes trailing nextn/MTP blocks: they are the MTP head, carrying real ffn weights."""
fields = _shard_fields(
**{"llama.block_count": total_blocks, "llama.nextn_predict_layers": nextn}
)
return _StubReader(fields, _shard_tensors(range(total_blocks)))
def _tied_reader(*, with_output: bool):
"""The 64-block stub plus a vocabulary matrix, tied or untied."""
tensors = list(_shard_tensors(range(64)))
tensors.append(_StubTensor("token_embd.weight", GIB))
if with_output:
tensors.append(_StubTensor("output.weight", GIB))
return _StubReader(_shard_fields(), tensors)
def test_a_tied_embedding_gguf_still_charges_a_vocabulary_matrix_to_vram():
"""A tied output matrix is duplicated from token_embd on the output device, so it costs VRAM."""
tied = _layout_from_reader(_tied_reader(with_output = False))
untied = _layout_from_reader(_tied_reader(with_output = True))
assert tied.complete and untied.complete
assert tied.lm_head_bytes == 0, "there is no output.weight to spill"
assert resident_floor_bytes(tied, 4096) == resident_floor_bytes(untied, 4096)
assert all_resident_bytes(tied, 4096) == all_resident_bytes(untied, 4096)
opts = PlanOptions(overhead_bytes_per_device = 0)
tied_plan = plan_placement(tied, [10 * GIB], 256 * GIB, 4096, opts = opts)
untied_plan = plan_placement(untied, [10 * GIB], 256 * GIB, 4096, opts = opts)
assert tied_plan.spilled_blocks, "a partial spill, so lm_head is not in play"
assert len(tied_plan.spilled_blocks) == len(untied_plan.spilled_blocks)
# Tied duplicate keeps the name token_embd.weight, so LM_HEAD_PATTERN cannot move it.
assert tied_plan.spilled_lm_head is False
assert LM_HEAD_PATTERN not in tied_plan.ot_patterns
assert tied_plan.host_bytes - untied_plan.host_bytes == 0
def test_excluded_mtp_block_bytes_are_kept_so_a_draft_can_be_charged():
"""Keep trailing MTP block bytes: --spec-type draft-mtp makes those blocks resident for the draft."""
layout = _layout_from_reader(_nextn_reader(2))
assert layout.has_excluded_blocks is True
per_block = layout.blocks[0].spillable_bytes + layout.blocks[0].resident_bytes
assert layout.excluded_block_bytes == 2 * per_block
plain = _layout_from_reader(_StubReader(_shard_fields(), _shard_tensors(range(64))))
assert plain.excluded_block_bytes == 0
base = PlanOptions(overhead_bytes_per_device = 0)
charged = PlanOptions(
overhead_bytes_per_device = 0, extra_resident_bytes = layout.excluded_block_bytes
)
without = plan_placement(layout, [10 * GIB], 256 * GIB, 4096, opts = base)
with_mtp = plan_placement(layout, [10 * GIB], 256 * GIB, 4096, opts = charged)
assert len(with_mtp.spilled_blocks) > len(without.spilled_blocks)
def test_a_nextn_gguf_is_marked_as_having_excluded_blocks():
with_mtp = _layout_from_reader(_nextn_reader(2))
assert with_mtp.complete
assert with_mtp.has_excluded_blocks is True
assert [b.index for b in with_mtp.blocks] == list(range(64))
plain = _layout_from_reader(_StubReader(_shard_fields(), _shard_tensors(range(64))))
assert plain.has_excluded_blocks is False
def test_the_spill_pattern_never_reaches_an_excluded_mtp_block():
"""Spill patterns must not reach excluded MTP blocks; the unbounded ^blk form matched them."""
layout = _layout_from_reader(_nextn_reader(2))
plan = plan_placement(layout, [_ALL_SPILL_VRAM], 256 * GIB, 4096, opts = _NO_OVERHEAD)
assert len(plan.spilled_blocks) == len(layout.blocks), "every block goes"
assert len(plan.ot_patterns) >= 1
pattern = re.compile(plan.ot_patterns[0])
assert pattern.search("blk.0.ffn_up.weight"), "a target block still spills"
assert pattern.search("blk.63.ffn_up.weight")
for excluded in ("blk.64.ffn_up.weight", "blk.65.ffn_down.weight"):
assert pattern.search(excluded) is None, excluded
def test_a_gguf_without_excluded_blocks_keeps_the_compact_pattern():
"""Bounded pattern only when blocks are excluded; else keep the compact global form."""
layout = _layout_from_reader(_StubReader(_shard_fields(), _shard_tensors(range(64))))
plan = plan_placement(layout, [_ALL_SPILL_VRAM], 256 * GIB, 4096, opts = _NO_OVERHEAD)
assert len(plan.spilled_blocks) == len(layout.blocks)
assert re.compile(plan.ot_patterns[0]).search("blk.999.ffn_up.weight")
def test_extra_resident_bytes_are_charged_against_the_pooled_budget():
"""GPU-resident bytes outside the layout (projector, MTP draft reserve) must shrink the budget."""
layout = q4_layout()
base = PlanOptions(overhead_bytes_per_device = GIB)
charged = PlanOptions(overhead_bytes_per_device = GIB, extra_resident_bytes = 3 * GIB)
without = plan_placement(layout, [16 * GIB], 128 * GIB, 8192, opts = base)
with_extra = plan_placement(layout, [16 * GIB], 128 * GIB, 8192, opts = charged)
assert without.spills_anything and with_extra.spills_anything
assert len(with_extra.spilled_blocks) > len(without.spilled_blocks)
assert max_context_for(layout, [16 * GIB], spill_all_ffn = True, opts = charged) < max_context_for(
layout, [16 * GIB], spill_all_ffn = True, opts = base
)
def test_row_ownership_is_modelled_on_raw_free_not_on_the_budget():
"""Row ownership must use raw free VRAM, as llama.cpp does, not the budget, or the split shifts."""
layout = _layout_from_reader(_StubReader(_shard_fields(), _shard_tensors(range(64))))
spilled = {b.index for b in layout.blocks}
budgets = [2 * GIB, 2 * GIB]
even = _per_device_shortfall(
layout,
_NO_OVERHEAD,
4096,
spilled,
False,
budgets,
quantised = False,
kv_bytes_floor = 0,
split_weights_per_device = [8 * GIB, 8 * GIB],
)
lopsided = _per_device_shortfall(
layout,
_NO_OVERHEAD,
4096,
spilled,
False,
budgets,
quantised = False,
kv_bytes_floor = 0,
split_weights_per_device = [1 * GIB, 15 * GIB],
)
assert even != lopsided, "the split weights have to reach _device_slots"
assert "device 1" in (lopsided or ""), "the card drawing 60 of 65 rows is the one over"
def test_a_sliding_window_model_abstains_on_a_multi_gpu_split():
"""Sliding-window models abstain on multi-GPU: an even spread under-books the full-context cards."""
layout = _layout_from_reader(_StubReader(_shard_fields(), _shard_tensors(range(64))))
swa = replace(layout, has_swa = True)
assert swa.n_attention_layers == swa.n_layers, "the earlier guard does NOT cover this"
half = _ALL_SPILL_VRAM // 2
plan = plan_placement(swa, [half, half], 256 * GIB, 4096, opts = _NO_OVERHEAD)
assert plan.changed is False
assert "sliding-window" in plan.reason
one = plan_placement(swa, [_ALL_SPILL_VRAM], 256 * GIB, 4096, opts = _NO_OVERHEAD)
assert len(one.spilled_blocks) == len(layout.blocks)
def _moe_reader(names):
"""A 2-block MoE GGUF whose expert tensors use ``names``."""
fields = dict(_shard_fields())
fields["llama.block_count"] = 2
fields["llama.expert_count"] = 64
fields["llama.expert_used_count"] = 8
tensors = []
for i in range(2):
for n in names:
tensors.append(_StubTensor(f"blk.{i}.{n}.weight", 400 * MIB))
tensors.append(_StubTensor(f"blk.{i}.attn_q.weight", 10 * MIB))
tensors.append(_StubTensor("token_embd.weight", 100 * MIB))
tensors.append(_StubTensor("output.weight", 100 * MIB))
return _StubReader(fields, tensors)
@pytest.mark.parametrize(
"label,names,expect_mib",
[
("split", ["ffn_up_exps", "ffn_gate_exps", "ffn_down_exps"], 1200),
("fused", ["ffn_gate_up_exps", "ffn_down_exps"], 800),
("chunked", ["ffn_up_chexps", "ffn_gate_chexps", "ffn_down_chexps"], 1200),
],
)
def test_every_expert_spelling_is_spillable(label, names, expect_mib):
"""Fused expert tensors like ffn_gate_up_exps are spillable experts, not only the split spelling."""
layout = _layout_from_reader(_moe_reader(names))
assert layout.is_moe
assert layout.blocks[0].spillable_bytes == expect_mib * MIB
pattern = spill_pattern_for(layout, None)
for n in names:
assert re.search(pattern, f"blk.0.{n}.weight"), f"{label}: {n} not matched"
def test_routed_latent_projections_are_never_spilled():
"""kimi-k3 ffn_routed_up/down look like experts but have no expert axis and plain
GGML_OP_MUL_MAT, so every token hits them; never spill them."""
layout = _layout_from_reader(_moe_reader(["ffn_routed_up", "ffn_routed_down"]))
assert layout.blocks[0].spillable_bytes == 0
pattern = spill_pattern_for(layout, None)
assert not re.search(pattern, "blk.0.ffn_routed_up.weight")
assert not re.search(pattern, "blk.0.ffn_routed_down.weight")
def _swa_layout(n_blocks = 64):
"""Every layer attends with an uneven per-layer cache n_attention_layers cannot express."""
layout = _layout_from_reader(_StubReader(_shard_fields(), _shard_tensors(range(n_blocks))))
return replace(layout, has_swa = True)
def test_a_per_layer_vector_replaces_the_sliding_window_abstain():
"""Gemma3 window layers cache a fraction of full context; without a per-layer vector the planner
spreads cache evenly and refuses, with one it sees where the big caches land."""
layout = _swa_layout()
half = _ALL_SPILL_VRAM // 2
spilled = {b.index for b in layout.blocks}
without = _per_device_shortfall(
layout,
_NO_OVERHEAD,
4096,
spilled,
False,
[half, half],
quantised = False,
kv_bytes_floor = 0,
)
assert without is not None and "sliding-window" in without
weights = [64 if (i % 5 == 0) else 1 for i in range(layout.n_layers)]
with_vector = _per_device_shortfall(
layout,
_NO_OVERHEAD,
4096,
spilled,
False,
[half, half],
quantised = False,
kv_bytes_floor = 0,
kv_layer_weights = weights,
)
assert with_vector is None or "sliding-window" not in with_vector
def test_the_vector_places_the_cache_it_does_not_resize_it():
"""Vector scaled to the priced total; only its shape moves cache between devices."""
layout = _swa_layout()
spilled = {b.index for b in layout.blocks}
n = layout.n_layers
budgets = [_ALL_SPILL_VRAM // 2, _ALL_SPILL_VRAM // 2]
front = [1] * (n // 2) + [0] * (n - n // 2)
back = [0] * (n // 2) + [1] * (n - n // 2)
a = _per_device_shortfall(
layout,
_NO_OVERHEAD,
32768,
spilled,
False,
budgets,
quantised = False,
kv_bytes_floor = 8 * GIB,
kv_layer_weights = front,
)
b = _per_device_shortfall(
layout,
_NO_OVERHEAD,
32768,
spilled,
False,
budgets,
quantised = False,
kv_bytes_floor = 8 * GIB,
kv_layer_weights = back,
)
assert a is not None and b is not None
assert "device 0" in a, "the front-loaded cache overflows the first card"
assert "device 1" in b, "the back-loaded cache overflows the second"
def test_a_wrong_length_vector_is_ignored_rather_than_trusted():
"""A vector of the wrong length is not evidence: abstain, do not stretch it."""
layout = _swa_layout()
spilled = {b.index for b in layout.blocks}
half = _ALL_SPILL_VRAM // 2
got = _per_device_shortfall(
layout,
_NO_OVERHEAD,
4096,
spilled,
False,
[half, half],
quantised = False,
kv_bytes_floor = 0,
kv_layer_weights = [1, 2, 3],
)
assert got is not None and "sliding-window" in got
def test_every_shard_handed_over_prices_the_whole_split():
"""Two readers summing to one layout, token_embd in its bucket whichever shard holds it."""
first = _StubReader(
_shard_fields(**{"split.count": 2, "split.no": 0}),
_shard_tensors(range(32)) + [_StubTensor("output.weight", GIB)],
)
second = _StubReader(
_shard_fields(**{"split.count": 2, "split.no": 1}),
_shard_tensors(range(32, 64)) + [_StubTensor("token_embd.weight", 3 * GIB)],
)
layout = _layout_from_readers([first, second])
assert layout.complete
assert len(layout.blocks) == 64
assert layout.token_embd_bytes == 3 * GIB
assert layout.lm_head_bytes == GIB
def test_a_partial_shard_set_still_abstains():
readers = [
_StubReader(_shard_fields(**{"split.count": 3}), _shard_tensors(range(32))),
_StubReader(_shard_fields(**{"split.count": 3}), _shard_tensors(range(32, 64))),
]
assert not _layout_from_readers(readers).complete
def test_split_shard_paths_follow_llama_cpp_naming():
paths = split_shard_paths("/m/model-Q4-00002-of-00003.gguf")
assert paths == [
"/m/model-Q4-00001-of-00003.gguf",
"/m/model-Q4-00002-of-00003.gguf",
"/m/model-Q4-00003-of-00003.gguf",
]
assert split_shard_paths("/m/model-Q4.gguf") is None
def test_layout_from_gguf_reads_the_sibling_shards_only_when_asked(tmp_path):
import numpy as np
from gguf import GGUFWriter
shards = [tmp_path / f"m-{i:05d}-of-00002.gguf" for i in (1, 2)]
for shard, blocks in zip(shards, (range(32), range(32, 64))):
writer = GGUFWriter(str(shard), "llama")
for key, value in _shard_fields().items():
if key.startswith("llama."):
writer.add_uint32(key, value)
writer.add_uint16("split.count", 2)
for tensor in _shard_tensors(blocks):
writer.add_tensor(tensor.name, np.zeros(4, dtype = np.float32))
writer.write_header_to_file()
writer.write_kv_data_to_file()
writer.write_tensors_to_file()
writer.close()
assert not layout_from_gguf(str(shards[0])).complete
whole = layout_from_gguf(str(shards[0]), all_shards = True)
assert whole.complete and len(whole.blocks) == 64
shards[1].unlink()
assert not layout_from_gguf(str(shards[0]), all_shards = True).complete