Skip to content

FreeToken brings datacenter-scale model serving to your desktop. Run massive models locally, fast and efficiently.

7.9 KB
Raw Blame
"""Glm5NextForCausalLM wiring smoke test (tiny random model, dense MLPs).

The per-op math is covered elsewhere (KDA kernels/op, kpool backend, mHC); this
test checks the ASSEMBLY: a 2-layer hybrid (KDA + DSA) model with mHC threading
runs prefill and decode through the real backends/pools, and the strongest
cache invariant holds -- decoding token T after prefilling [0, T) produces the
same logits as prefilling [0, T] outright (state handoff across the KDA
recurrent pool, the MLA latent pool, and the kpool indexer cache).
"""

from __future__ import annotations

from types import SimpleNamespace

import pytest
import torch

pytestmark = pytest.mark.skipif(not torch.cuda.is_available(), reason="needs CUDA")

HIDDEN, VOCAB = 64, 128
KDA_H, KDA_D = 2, 128  # KDA kernels specialize on D=128
IDX_H, IDX_D = 16, 64
LATENT = 32
DEV = "cuda"


def _hf_config():
    from freetoken.utils.hf import RawConfigShim

    text = {
        "hidden_size": HIDDEN, "intermediate_size": 96, "num_hidden_layers": 2,
        "num_attention_heads": 2, "vocab_size": VOCAB, "hidden_act": "silu",
        "rms_norm_eps": 1e-5, "max_position_embeddings": 4096,
        "tie_word_embeddings": False,
        "q_lora_rank": 48, "kv_lora_rank": LATENT, "qk_nope_head_dim": 32,
        "qk_rope_head_dim": 0, "v_head_dim": 32, "mla_use_nope": True,
        "index_n_heads": IDX_H, "index_head_dim": IDX_D, "index_topk": 32,
        "indexer_types": ["full", "full"], "indexer_rope_interleave": True,
        "index_kpool": 4, "index_kpool_compress": True,
        "index_kpool_always_select_tail": True,
        "linear_attn_config": {
            "num_heads": KDA_H, "head_dim": KDA_D,
            "short_conv_kernel_size": 4, "gate_lower_bound": -5.0,
        },
        "layer_types": ["linear_attention", "deepseek_sparse_attention"],
        "mlp_layer_types": ["dense", "dense"],  # no MoE machinery in this test
        "first_k_dense_replace": 2,
        "mhc": True, "hc_mult": 4, "hc_eps": 1e-6, "hc_sinkhorn_iters": 20,
        "n_routed_experts": 8, "num_experts_per_tok": 2, "n_shared_experts": 1,
        "moe_intermediate_size": 32, "norm_topk_prob": True,
        "routed_scaling_factor": 2.5, "scoring_func": "sigmoid",
        "n_group": 1, "topk_group": 1, "swiglu_limit": 10.0,
        "attention_bias": False, "model_type": "glm5_next_text",
    }
    return RawConfigShim({
        "architectures": ["Glm5NextForConditionalGeneration"],
        "model_type": "glm5_next", "text_config": text,
    })


@pytest.fixture()
def rig(monkeypatch):
    from freetoken.attention.dsa_indexer_kpool import Glm5NextDSABackend
    from freetoken.distributed import set_tp_info, try_get_tp_info
    from freetoken.kvcache.dsa_pool import KpoolDSAKVCache
    from freetoken.kvcache.linear_state_pool import LinearStatePool
    from freetoken.models.glm5_next.config import parse_config
    from freetoken.models.glm5_next.model import Glm5NextForCausalLM

    if try_get_tp_info() is None:
        set_tp_info(rank=0, size=1)
    config = parse_config(_hf_config())

    prev_dtype = torch.get_default_dtype()
    torch.set_default_dtype(torch.bfloat16)
    prev_dev = torch.get_default_device()
    torch.set_default_device(DEV)
    try:
        model = Glm5NextForCausalLM(config)
    finally:
        torch.set_default_dtype(prev_dtype)
        torch.set_default_device(prev_dev)

    # Random weights via the state-dict round trip (keeps shapes/dtypes honest).
    torch.manual_seed(0)
    sd = model.state_dict()
    rand = {}
    for k, v in sd.items():
        t = torch.randn(v.shape, dtype=torch.float32, device=DEV) * 0.05
        if k.endswith("norm.weight") or ".o_norm.weight" in k:
            t = t.abs() + 0.5
        rand[k] = t.to(v.dtype)
    model.load_state_dict(rand)

    kv = KpoolDSAKVCache(
        latent_dim=LATENT, num_layers=2, num_pages=4, page_size=64,
        dtype=torch.bfloat16, device=torch.device(DEV),
        index_head_dim=IDX_D, num_index_layers=1,
        index_ratio=4, num_req_slots=4,
    )
    page_table = torch.full((2, 256), -1, dtype=torch.int32, device=DEV)
    page_table[0] = torch.arange(256, dtype=torch.int32, device=DEV)
    linear_pool = LinearStatePool(
        config.linear_attention_group(), num_slots=4,
        dtype=torch.bfloat16, device=torch.device(DEV), tp_size=1,
    )

    ctx = SimpleNamespace(
        kv_cache=kv, page_table=page_table, linear_state_pool=linear_pool,
        attn_backend=None, batch=None,
    )
    for mod in (
        "freetoken.attention.dsa.get_global_ctx",
        "freetoken.models.glm5_next.kda.get_global_ctx",
        "freetoken.models.glm5_next.attention.get_global_ctx",
        "freetoken.models.glm5_next.model.get_global_ctx",
        "freetoken.layers.embedding.get_global_ctx",
    ):
        monkeypatch.setattr(mod, lambda: ctx)
    ctx.attn_backend = Glm5NextDSABackend(config)
    return model, ctx


def _req(device_len, cached_len):
    return SimpleNamespace(
        table_idx=0, device_len=device_len, extend_len=device_len - cached_len,
        cached_len=cached_len, linear_slot_idx=1, mamba_ping_pong=None,
    )


def _batch(ctx, ids, t0, phase):
    from freetoken.attention.linear import FLAMetadata

    t1 = t0 + len(ids)
    is_decode = phase == "decode"
    batch = SimpleNamespace(
        phase=phase,
        is_prefill=not is_decode, is_decode=is_decode, size=1,
        reqs=[_req(t1, t0)], padded_reqs=[_req(t1, t0)],
        input_ids=torch.tensor(ids, device=DEV),
        positions=torch.arange(t0, t1, device=DEV),
        out_loc=torch.arange(t0, t1, device=DEV),
        active_table_idx=torch.tensor([0], device=DEV) if is_decode else None,
        fla_metadata=FLAMetadata(
            cu_seqlens=torch.tensor([0, len(ids)], dtype=torch.int32, device=DEV),
            cache_indices=torch.tensor([1], dtype=torch.int32, device=DEV),
            has_initial_state=None if is_decode else torch.tensor([t0 > 0], device=DEV),
            fresh_state_indices=(
                None if (is_decode or t0 > 0)
                else torch.tensor([1], dtype=torch.int64, device=DEV)
            ),
        ),
        mm_embeds=None,
    )
    ctx.batch = batch
    ctx.attn_backend.prepare_metadata(batch)
    return batch


def _reset(ctx):
    ctx.linear_state_pool.reset(1)
    ctx.kv_cache._kv_buffer.zero_()
    ctx.kv_cache._index_k_buffer.zero_()
    ctx.kv_cache._tail_k.zero_()
    ctx.kv_cache._tail_gate.zero_()


def test_prefill_decode_consistency(rig):
    model, ctx = rig
    torch.manual_seed(1)
    total = 24
    ids = torch.randint(0, VOCAB, (total,)).tolist()

    # One-shot prefill over the full sequence: last-token logits per position
    # are only produced for the final token, so run it twice at different splits.
    _reset(ctx)
    _batch(ctx, ids, 0, "prefill")
    full_logits = model.forward()  # [1, VOCAB] logits of the last position
    assert full_logits.shape == (1, VOCAB)
    assert torch.isfinite(full_logits.float()).all()

    # Prefill [0, total-1) then decode the last token: must match the one-shot run.
    _reset(ctx)
    _batch(ctx, ids[:-1], 0, "prefill")
    model.forward()
    _batch(ctx, ids[-1:], total - 1, "decode")
    dec_logits = model.forward()
    err = (dec_logits.float() - full_logits.float()).abs().max().item()
    scale = full_logits.float().abs().max().item() + 1e-8
    assert err / scale < 3e-2, f"decode/prefill divergence: {err} (scale {scale})"


def test_chunked_prefill_consistency(rig):
    model, ctx = rig
    torch.manual_seed(2)
    total = 28  # split 16 + 12; chunk boundary pool-aligned (16 % 4 == 0)
    ids = torch.randint(0, VOCAB, (total,)).tolist()

    _reset(ctx)
    _batch(ctx, ids, 0, "prefill")
    full_logits = model.forward()

    _reset(ctx)
    _batch(ctx, ids[:16], 0, "prefill")
    model.forward()
    _batch(ctx, ids[16:], 16, "prefill")
    chunk_logits = model.forward()
    err = (chunk_logits.float() - full_logits.float()).abs().max().item()
    scale = full_logits.float().abs().max().item() + 1e-8
    assert err / scale < 3e-2, f"chunked/one-shot divergence: {err} (scale {scale})"