Skip to content

FreeToken brings datacenter-scale model serving to your desktop. Run massive models locally, fast and efficiently.

6.6 KB
Raw Blame
"""DSV41 indexer logits over packed fp4 index keys: full-range (Full mode) and candidate-restricted
(Reindex mode) against a torch reference on the dequantized keys, with causality from ``live``. The
row of a compressed position comes from the request's full-token locs (``locs[t * ratio] // ratio``);
in Full mode only the live columns are written."""

from __future__ import annotations

import pytest
import torch

from freetoken.kvcache.dsv4.v41_row_format import FP4_E8M0_B32

pytestmark = pytest.mark.skipif(not torch.cuda.is_available(), reason="needs CUDA")

D, H = 128, 32
ROWS = 4200


RATIO = 2


def _setup(b, s, t, seed):
    from freetoken.kernel.triton.dsv41.pack import pack_rows, unpack_rows

    torch.manual_seed(seed)
    keys = torch.randn(ROWS, D, device="cuda", dtype=torch.bfloat16)
    pool = pack_rows(keys, FP4_E8M0_B32)
    deq = unpack_rows(pool, FP4_E8M0_B32, D)
    q = torch.randn(b, s, H, D, device="cuda", dtype=torch.bfloat16)
    w = (torch.randn(b, s, H, device="cuda") * 0.3).bfloat16()
    # each request owns a random permutation of pool rows for its compressed positions; a few holes.
    # The kernel derives rows from full-token locs: compressed position p -> loc = row * RATIO (+ any
    # in-group offset) at locs[p * RATIO]; a hole is a negative loc.
    k_rows = torch.stack([torch.randperm(ROWS, device="cuda")[:t] for _ in range(b)]).to(torch.int32)
    k_rows[:, 5::11] = -1
    locs = torch.full((b, t * RATIO), -1, device="cuda", dtype=torch.int32)
    for i in range(b):
        for p in range(t):
            r = int(k_rows[i, p])
            if r >= 0:
                locs[i, p * RATIO : (p + 1) * RATIO] = torch.arange(r * RATIO, (r + 1) * RATIO, device="cuda", dtype=torch.int32)
    return pool, deq, q, w, k_rows, locs


def _reference(q, w, deq, k_rows, live, positions):
    """positions: [B, S, N] compressed positions (-1 empty) -> logits [B, S, N]."""
    b, s, n = positions.shape
    out = torch.full((b, s, n), float("-inf"), device="cuda")
    for i in range(b):
        for j in range(s):
            for c in range(n):
                p = int(positions[i, j, c])
                if p < 0 or p >= int(live[i, j]):
                    continue
                r = int(k_rows[i, p])
                if r < 0:
                    continue
                score = (q[i, j] @ deq[r]).relu() * w[i, j]
                out[i, j, c] = score.sum()
    return out


def test_full_range_matches_reference_with_causal_live():
    from freetoken.kernel.triton.dsv41.indexer import indexer_logits_packed

    b, s, t = 2, 5, 300
    pool, deq, q, w, k_rows, locs = _setup(b, s, t, seed=1)
    live = torch.tensor([[10, 64, 65, 200, 300], [0, 1, 128, 129, 250]], device="cuda", dtype=torch.int32)
    out = torch.full((b, s, t), 12345.0, device="cuda")  # a sentinel: the dead area must stay untouched
    got = indexer_logits_packed(q, w, pool, FP4_E8M0_B32, locs, RATIO, live, out=out)
    positions = torch.arange(t, device="cuda").view(1, 1, t).expand(b, s, t)
    want = _reference(q, w, deq, k_rows, live, positions)
    assert got.shape == (b, s, t)
    written = positions < live.unsqueeze(-1)
    touched = positions < (torch.div(live + 63, 64, rounding_mode="floor") * 64).unsqueeze(-1)  # whole live tiles
    assert (got[~touched] == 12345.0).all() and torch.isneginf(got[touched & ~written]).all()
    assert torch.equal(torch.isinf(got[written]), torch.isinf(want[written]))
    finite = written & ~torch.isinf(want)
    torch.testing.assert_close(got[finite], want[finite], atol=2e-2, rtol=2e-2)


def test_quantized_queries_preserve_reference_score_rounding():
    from freetoken.kernel.triton.dsv41.indexer import indexer_logits_packed
    from freetoken.kernel.triton.dsv41.pack import pack_rows, unpack_rows

    torch.manual_seed(23)
    t, s, topk = 4096, 32, 512
    pool = pack_rows(torch.randn(t, D, device="cuda", dtype=torch.bfloat16), FP4_E8M0_B32)
    keys = unpack_rows(pool, FP4_E8M0_B32, D)
    q = torch.randn(s * H, D, device="cuda", dtype=torch.bfloat16)
    q = unpack_rows(pack_rows(q, FP4_E8M0_B32), FP4_E8M0_B32, D).view(1, s, H, D)
    w = torch.randn(1, s, H, device="cuda", dtype=torch.bfloat16) * (D * H) ** -0.5
    locs = torch.arange(t, device="cuda", dtype=torch.int32).view(1, t)
    live = torch.full((1, s), t, device="cuda", dtype=torch.int32)
    got = indexer_logits_packed(q, w, pool, FP4_E8M0_B32, locs, 1, live)
    reference = torch.einsum("bshd,btd->bsht", q, keys.unsqueeze(0))
    reference = (reference.relu() * w.unsqueeze(-1)).sum(dim=2).float()
    torch.testing.assert_close(got, reference, atol=0, rtol=0)
    # BF16 scores can tie at the cutoff; compare selected scores, not arbitrary tie order.
    torch.testing.assert_close(got.topk(topk).values, reference.topk(topk).values, atol=0, rtol=0)


def test_full_range_worker_grid_covers_wide_histories(monkeypatch):
    """More live tiles than worker programs: each worker strides over its share."""
    from freetoken.kernel.triton.dsv41 import indexer as mod

    b, s, t = 1, 2, 4096  # 64 tiles
    pool, deq, q, w, k_rows, locs = _setup(b, s, t, seed=4)
    live = torch.tensor([[4096, 4000]], device="cuda", dtype=torch.int32)
    from types import SimpleNamespace

    properties = torch.cuda.get_device_properties(q.device)
    with monkeypatch.context() as patch:
        patch.setattr(torch.cuda, "get_device_properties", lambda _: SimpleNamespace(
            multi_processor_count=5, major=properties.major, minor=properties.minor,
        ))
        got = mod.indexer_logits_packed(q, w, pool, FP4_E8M0_B32, locs, RATIO, live)
    positions = torch.arange(t, device="cuda").view(1, 1, t).expand(b, s, t)
    want = _reference(q, w, deq, k_rows, live, positions)
    written = positions < live.unsqueeze(-1)
    assert torch.equal(torch.isinf(got[written]), torch.isinf(want[written]))
    finite = written & ~torch.isinf(want)
    torch.testing.assert_close(got[finite], want[finite], atol=2e-2, rtol=2e-2)


def test_candidate_pool_matches_reference():
    from freetoken.kernel.triton.dsv41.indexer import indexer_logits_packed

    b, s, t, nc = 2, 3, 400, 96
    pool, deq, q, w, k_rows, locs = _setup(b, s, t, seed=2)
    live = torch.tensor([[400, 350, 33], [1, 400, 200]], device="cuda", dtype=torch.int32)
    cand = torch.randint(0, t, (b, s, nc), device="cuda", dtype=torch.int32)
    cand[..., -10:] = -1  # empty tail of the pool
    got = indexer_logits_packed(q, w, pool, FP4_E8M0_B32, locs, RATIO, live, candidates=cand)
    want = _reference(q, w, deq, k_rows, live, cand)
    assert got.shape == (b, s, nc)
    assert torch.equal(torch.isinf(got), torch.isinf(want))
    finite = ~torch.isinf(want)
    torch.testing.assert_close(got[finite], want[finite], atol=2e-2, rtol=2e-2)