Skip to content

FreeToken brings datacenter-scale model serving to your desktop. Run massive models locally, fast and efficiently.

4.5 KB
Raw Blame
"""Pool-factory generalization: hybrid linear x ANY paged family.

The old factory hard-required "linear + one GQA group" (Qwen3.5 GDN shape);
glm5_next is linear (KDA) x DSA. Checks the factory dispatch, the MLA/DSA
layer-id remap (34 KDA layers cost no latent slabs), the kpool pool selection
(gate slab), and the KV cost model's kpool double-count of the index slabs.
"""

from __future__ import annotations

import pytest
import torch

from freetoken.kvcache import create_kvcache_pool, resolve_pool_class
from freetoken.kvcache.dsa_pool import DSAKVCache, KpoolDSAKVCache
from freetoken.models.config import (
    FullAttentionGroupConfig,
    LinearGatedDeltaGroupConfig,
    ModelConfig,
    RotaryConfig,
)


@pytest.fixture(autouse=True)
def _single_rank_tp():
    from freetoken.distributed import set_tp_info, try_get_tp_info

    if try_get_tp_info() is None:
        set_tp_info(rank=0, size=1)


def _glm5_like_config(index_kpool=4, index_head_dim=128):
    n_layers = 12
    dsa_ids = tuple(range(3, n_layers, 4))  # 3, 7, 11
    kda_ids = tuple(i for i in range(n_layers) if i not in dsa_ids)
    rotary = RotaryConfig(head_dim=256, rotary_dim=0, max_position=4096, base=1e4, scaling=None)
    groups = (
        LinearGatedDeltaGroupConfig(
            name="linear", layer_ids=kda_ids,
            num_key_heads=4, num_value_heads=4, key_head_dim=128, value_head_dim=128,
            conv_kernel_dim=4, output_gate="sigmoid", variant="kda",
        ),
        FullAttentionGroupConfig(
            name="full", layer_ids=dsa_ids, num_kv_heads=1, head_dim=512,
            rotary_config=rotary, mla=True,
            index_head_dim=index_head_dim, num_index_layers=len(dsa_ids),
            index_ratio=index_kpool,
        ),
    )
    return ModelConfig(
        num_layers=n_layers, num_qo_heads=4, num_kv_heads=1, head_dim=512,
        hidden_size=256, vocab_size=1000, intermediate_size=512,
        rms_norm_eps=1e-5, rotary_config=rotary, hidden_act="silu",
        tie_word_embeddings=False, num_experts=8, num_experts_per_tok=2,
        moe_intermediate_size=64, norm_topk_prob=True, model_type="glm5_next",
        architectures=["Glm5NextForCausalLM"], moe_enabled=True,
        attention_groups=groups,
    )


def test_factory_builds_kpool_pool_with_layer_remap():
    cfg = _glm5_like_config()
    assert resolve_pool_class(cfg) is KpoolDSAKVCache

    pool = create_kvcache_pool(
        model_config=cfg, num_pages=4, page_size=64,
        dtype=torch.bfloat16, device=torch.device("cpu"), num_req_slots=5,
    )
    assert isinstance(pool, KpoolDSAKVCache)
    # Latent slabs back ONLY the 3 DSA layers (34-of-45 economy at real scale).
    assert pool._kv_buffer.shape[1] == 3
    # Global layer-id addressing: DSA layers resolve, KDA layers have no slab.
    for lid in (3, 7, 11):
        assert pool.latent_rows(lid).shape == (256, 512)
    with pytest.raises(KeyError):
        pool.latent_rows(0)  # a KDA layer
    # Shadow index slab: tokens/ratio rows + one scratch row per request slot.
    assert pool.index_k_cache(0).shape == (256 // 4 + 5, 128)
    assert pool.cmp_scratch_base == 256 // 4
    # kpool tail rings exist at [num_req_slots, ratio, head_dim] per indexer layer.
    assert pool.tail_k(0).shape == pool.tail_gate(0).shape
    assert pool.tail_k(0).shape == (5, 4, 128)


def test_factory_kpool1_builds_plain_dsa_pool():
    cfg = _glm5_like_config(index_kpool=1)
    assert resolve_pool_class(cfg) is DSAKVCache
    pool = create_kvcache_pool(
        model_config=cfg, num_pages=16, page_size=1,
        dtype=torch.bfloat16, device=torch.device("cpu"),
    )
    assert type(pool) is DSAKVCache


def test_cost_model_kpool_shadow_slab_quarter_cost():
    """The shadow slab stores one row per index_ratio tokens: the kpool spec's
    index bytes are 1/ratio of the plain DSA slab (rings/scratch are per-request
    and not part of the per-token price)."""
    from types import SimpleNamespace

    from freetoken.kvcache.base import spec_kv_bytes_per_token

    tp = SimpleNamespace(size=1)
    econf = SimpleNamespace(tp_info=tp, dtype=torch.bfloat16)
    (spec,) = [
        s for s in _glm5_like_config().kv_cache_group_specs() if s.num_layers > 0
    ]
    (spec1,) = [
        s
        for s in _glm5_like_config(index_kpool=1).kv_cache_group_specs()
        if s.num_layers > 0
    ]
    index_full = spec.index_head_dim * spec.num_index_layers * 2
    assert (
        spec_kv_bytes_per_token(spec1, econf) - spec_kv_bytes_per_token(spec, econf)
        == index_full - index_full // 4
    )