Skip to content

FreeToken brings datacenter-scale model serving to your desktop. Run massive models locally, fast and efficiently.

6.5 KB
Raw Blame
"""Flint contracts checked against independent tensor math and checkpoint layouts."""
from __future__ import annotations

import json
import os
from types import SimpleNamespace
from unittest.mock import patch

import pytest
import torch
import torch.nn.functional as F

from freetoken.models.flint.config import parse_config
from freetoken.models.flint.moe import FlintMoE


@pytest.fixture(autouse=True)
def _single_rank(monkeypatch):
    from freetoken.distributed import info
    monkeypatch.setattr(info, "_TP_INFO", info.DistributedInfo(rank=0, size=1))


def _hf(**overrides):
    return SimpleNamespace(**(dict(
        architectures=["FlintForCausalLM"], model_type="flint", num_hidden_layers=3,
        hidden_size=64, head_dim=128, num_attention_heads=4, num_key_value_heads=2,
        intermediate_size=128, moe_intermediate_size=64,
        shared_expert_intermediate_size=128, vocab_size=128, max_position_embeddings=2048,
        layer_types=["full_attention", "sliding_attention", "full_attention"],
        no_rope_layer_indices=[0, 2], mlp_only_layers=[0],
        hidden_act="silu", router_score_function="sigmoid", softmax_type="learnable",
        norm_topk_prob=True, num_experts=8, num_experts_per_tok=2,
        rms_norm_eps=1e-6, sliding_window=513, rope_theta=10000., tie_word_embeddings=False,
    ) | overrides))


@pytest.mark.skipif(not torch.cuda.is_available(), reason="fused CUDA router")
@pytest.mark.parametrize("renormalize", [False, True])
def test_sigmoid_routing_bias_affects_selection_only(renormalize):
    from freetoken.layers import LinearReplicated
    torch.manual_seed(42)
    block = FlintMoE.__new__(FlintMoE)
    block.top_k, block.renormalize = 6, renormalize
    block.gate = LinearReplicated(128, 256, has_bias=False)
    block.gate.weight = torch.randn(256, 128, device="cuda", dtype=torch.bfloat16) * .1
    block.gate.expert_bias = torch.randn(256, device="cuda") * .05
    x = torch.randn(32, 128, device="cuda", dtype=torch.bfloat16)
    weights, ids = block._route(x)
    logits = F.linear(x, block.gate.weight).float()
    scores = 1. / (1. + torch.exp(-logits))
    expected_ids = torch.argsort(scores + block.gate.expert_bias, dim=-1, descending=True)[:, :6]
    assert [set(row) for row in ids.tolist()] == [set(row) for row in expected_ids.tolist()]
    expected = scores.gather(1, ids.long())
    if renormalize:
        expected = expected / expected.sum(1, keepdim=True)
    torch.testing.assert_close(weights, expected, rtol=1e-6, atol=1e-7)
    assert ids.dtype == torch.int32


def test_weight_reader_merges_cross_shard_projections_and_preserves_sinks(tmp_path):
    from safetensors.torch import save_file
    from freetoken.models.flint.weight import iter_weights
    hf = _hf()
    (tmp_path / "config.json").write_text(json.dumps(vars(hf)))
    q = torch.randn(512, 64, dtype=torch.bfloat16)
    k = torch.randn(256, 64, dtype=torch.bfloat16)
    v = torch.randn(256, 64, dtype=torch.bfloat16)
    pre = "model.layers.0.self_attn"
    sink = torch.tensor([.125, .25, .5, 1.])
    save_file({f"{pre}.q_proj.weight": q, f"{pre}.softmax_offset": sink}, tmp_path / "a.safetensors")
    save_file({f"{pre}.k_proj.weight": k, f"{pre}.v_proj.weight": v}, tmp_path / "b.safetensors")
    weights = dict(iter_weights(str(tmp_path), torch.device("cpu"), include_moe_experts=False, include_non_moe=True))
    torch.testing.assert_close(weights[f"{pre}.qkv_proj.weight"], torch.cat([q, k, v]))
    torch.testing.assert_close(weights[f"{pre}.softmax_offset"], sink)
    for i in (1, 2):
        assert not weights[f"model.layers.{i}.mlp.gate.expert_bias"].any()


def test_gguf_explicit_end_of_generation_ids_are_not_lost():
    from freetoken.models.gguf.tokenizer import gguf_eos_token_ids
    tokens = ["x"] * 19
    for i, token in [(1, "<|end_of_text|>"), (3, "<|user|>"), (5, "<|observation|>"), (18, "<|developer|>")]:
        tokens[i] = token
    meta = {"tokenizer.ggml.tokens": tokens, "tokenizer.ggml.eos_token_id": 1,
            "tokenizer.ggml.eog_token_ids": [1, 3, 5, 18]}
    with patch("freetoken.models.gguf.tokenizer.load_gguf_metadata", return_value=meta):
        assert gguf_eos_token_ids("test.gguf") == {1, 3, 5, 18}


@pytest.mark.needs_weights
def test_official_gguf_matches_hf_tokenizer_and_config():
    from transformers import AutoTokenizer
    from freetoken.models.gguf.config import build_gguf_shim
    from freetoken.models.gguf.tokenizer import load_gguf_tokenizer, gguf_eos_token_ids
    from freetoken.models.flint.gguf import parse_gguf_config
    from freetoken.utils import cached_load_hf_config
    gguf_path = os.environ.get("FREETOKEN_FLINT_GGUF")
    hf_path = os.environ.get("FREETOKEN_FLINT_MODEL")
    if not gguf_path or not hf_path:
        pytest.skip("set FREETOKEN_FLINT_GGUF and FREETOKEN_FLINT_MODEL")
    hf = AutoTokenizer.from_pretrained(hf_path, local_files_only=True)
    gg = load_gguf_tokenizer(gguf_path)
    texts = [
        "HelloWorld camelCase XMLHTTPRequest don't we're I'LL",
        "1234567890 3.1415926535 -42 1e-10",
        "漢字かなカナ 中文测试 café e\u0301 👩‍💻",
        "def f(x):\n\treturn {'value': x + 1}\n\n# EOF   ",
        "<|user|>hello<|assistant|><think>check</think>42<|end_of_text|>",
    ]
    for text in texts:
        assert gg.encode(text, add_special_tokens=False) == hf.encode(text, add_special_tokens=False), text
    tools = [{"type": "function", "function": {"name": "read", "parameters": {
        "type": "object", "properties": {"path": {"type": "string"}}, "required": ["path"],
    }}}]
    messages = [
        {"role": "user", "content": "Read the file."},
        {"role": "assistant", "content": "", "reasoning_content": "Inspect it first.",
         "tool_calls": [{"type": "function", "function": {"name": "read", "arguments": {"path": "/tmp/a.py"}}}]},
        {"role": "tool", "content": "print(42)"},
    ]
    assert gg.apply_chat_template(messages, tools=tools, add_generation_prompt=True) == hf.apply_chat_template(
        messages, tools=tools, add_generation_prompt=True,
    )
    assert gguf_eos_token_ids(gguf_path) == {150001, 150003, 150005, 150018}
    cfg_hf = parse_config(cached_load_hf_config(hf_path))
    cfg_gg = parse_gguf_config(build_gguf_shim(gguf_path))
    for name in ("head_dim", "num_qo_heads", "num_kv_heads", "num_layers", "hidden_size",
                 "num_experts", "num_experts_per_tok", "moe_intermediate_size",
                 "shared_expert_intermediate_size", "first_k_dense_replace", "attention_groups"):
        assert getattr(cfg_hf, name) == getattr(cfg_gg, name), name