Skip to content

FreeToken brings datacenter-scale model serving to your desktop. Run massive models locally, fast and efficiently.

10.0 KB
Raw Blame
from __future__ import annotations

import re
from types import SimpleNamespace

import pytest
import torch


def test_shard_tensor_splits_vocab_with_ceil_partition():
    from freetoken.models.loader import shard_tensor

    value = torch.arange(5 * 2, dtype=torch.float32).reshape(5, 2)

    rank0 = shard_tensor(
        "model.embed_tokens.weight",
        value,
        rank=0,
        world_size=2,
        num_kv_heads=1,
    )
    rank1 = shard_tensor(
        "model.embed_tokens.weight",
        value,
        rank=1,
        world_size=2,
        num_kv_heads=1,
    )

    assert rank0.tolist() == value[:3].tolist()
    assert rank1.tolist() == value[3:5].tolist()


def test_iter_weight_files_reads_shards_from_an_indexed_subdirectory(tmp_path):
    """FP4-Local releases keep shards in tensors/; the root glob alone finds nothing."""
    import json
    import os
    from freetoken.models.loader import iter_weight_files

    sub = tmp_path / "tensors"
    sub.mkdir()
    for i in (1, 2):
        (sub / f"model-{i:05d}-of-00002.safetensors").write_bytes(b"")
    (tmp_path / "model.safetensors.index.json").write_text(
        json.dumps(
            {
                "weight_map": {
                    "model.embed_tokens.weight": "tensors/model-00001-of-00002.safetensors",
                    "lm_head.weight": "tensors/model-00002-of-00002.safetensors",
                }
            }
        )
    )

    files = iter_weight_files(str(tmp_path))

    assert len(files) == 2
    assert all(os.path.exists(f) for f in files)


def test_iter_weight_files_lets_the_index_decide_which_shards_load(tmp_path):
    """The index is authoritative: it may name root and subdirectory shards; an unlisted root file is not loaded."""
    import json
    from freetoken.models.loader import iter_weight_files

    root = tmp_path / "model-00001-of-00002.safetensors"
    root.write_bytes(b"")
    (tmp_path / "unlisted.safetensors").write_bytes(b"")
    sub = tmp_path / "tensors"
    sub.mkdir()
    (sub / "model-00002-of-00002.safetensors").write_bytes(b"")
    (tmp_path / "model.safetensors.index.json").write_text(
        json.dumps(
            {
                "weight_map": {
                    "model.embed_tokens.weight": "model-00001-of-00002.safetensors",
                    "lm_head.weight": "tensors/model-00002-of-00002.safetensors",
                }
            }
        )
    )

    files = iter_weight_files(str(tmp_path))

    assert files == [str(root), str(sub / "model-00002-of-00002.safetensors")]


def test_iter_weight_files_falls_back_to_root_shards_without_an_index(tmp_path):
    """No index: the root glob loads, excluding consolidated.safetensors unless it is the only shard."""
    from freetoken.models.loader import iter_weight_files

    a = tmp_path / "model-00001-of-00002.safetensors"
    b = tmp_path / "model-00002-of-00002.safetensors"
    consolidated = tmp_path / "consolidated.safetensors"
    for path in (a, b, consolidated):
        path.write_bytes(b"")

    assert sorted(iter_weight_files(str(tmp_path))) == sorted([str(a), str(b)])

    a.unlink()
    b.unlink()
    assert iter_weight_files(str(tmp_path)) == [str(consolidated)]


def test_iter_root_safetensor_files_from_index_keeps_only_root_index_shards(tmp_path):
    import json
    from freetoken.models.loader import iter_root_safetensor_files_from_index

    root_a = tmp_path / "model-00000-of-00002.safetensors"
    root_b = tmp_path / "model-00001-of-00002.safetensors"
    ignored = tmp_path / "consolidated.safetensors"
    metal = tmp_path / "metal" / "model.safetensors"
    original = tmp_path / "original" / "model.safetensors"
    for path in (root_a, root_b, ignored, metal, original):
        path.parent.mkdir(parents=True, exist_ok=True)
        path.write_bytes(b"")
    (tmp_path / "model.safetensors.index.json").write_text(
        json.dumps(
            {
                "weight_map": {
                    "model.embed_tokens.weight": root_a.name,
                    "lm_head.weight": root_b.name,
                    "bad": "metal/model.safetensors",
                }
            }
        ),
        encoding="utf-8",
    )

    files = iter_root_safetensor_files_from_index(str(tmp_path))

    assert files == [str(root_a), str(root_b)]


def test_iter_root_safetensor_files_from_index_rejects_subdirectory_model_path(tmp_path):
    import pytest
    from freetoken.models.loader import iter_root_safetensor_files_from_index

    subdir = tmp_path / "original"
    subdir.mkdir()

    with pytest.raises(ValueError, match="root GPT-OSS"):
        iter_root_safetensor_files_from_index(str(subdir))


def test_iter_root_safetensor_files_from_index_requires_root_shards(tmp_path):
    import pytest
    from freetoken.models.loader import iter_root_safetensor_files_from_index

    with pytest.raises(ValueError, match="No root GPT-OSS safetensors shards"):
        iter_root_safetensor_files_from_index(str(tmp_path))


def test_shard_tensor_replicates_kv_heads_when_tp_exceeds_kv_heads():
    from freetoken.models.loader import shard_tensor

    value = torch.arange(2 * 4, dtype=torch.float32).reshape(2, 4)

    rank0 = shard_tensor(
        "model.layers.0.self_attn.k_proj.weight",
        value,
        rank=0,
        world_size=4,
        num_kv_heads=2,
    )
    rank1 = shard_tensor(
        "model.layers.0.self_attn.k_proj.weight",
        value,
        rank=1,
        world_size=4,
        num_kv_heads=2,
    )
    rank2 = shard_tensor(
        "model.layers.0.self_attn.k_proj.weight",
        value,
        rank=2,
        world_size=4,
        num_kv_heads=2,
    )
    rank3 = shard_tensor(
        "model.layers.0.self_attn.k_proj.weight",
        value,
        rank=3,
        world_size=4,
        num_kv_heads=2,
    )

    assert rank0.tolist() == value[:1].tolist()
    assert rank1.tolist() == value[:1].tolist()
    assert rank2.tolist() == value[1:2].tolist()
    assert rank3.tolist() == value[1:2].tolist()


def test_merge_stream_buffers_qkv_until_all_slots_arrive():
    from freetoken.models.loader import MergeRule, iter_merged_tensors

    tensors = [
        ("model.layers.0.self_attn.q_proj.weight", torch.full((1, 2), 1.0)),
        ("model.layers.0.self_attn.v_proj.weight", torch.full((1, 2), 3.0)),
        ("model.layers.0.self_attn.k_proj.weight", torch.full((1, 2), 2.0)),
    ]
    rules = {
        ".q_proj": MergeRule(".qkv_proj", "q", ("q", "k", "v")),
        ".k_proj": MergeRule(".qkv_proj", "k", ("q", "k", "v")),
        ".v_proj": MergeRule(".qkv_proj", "v", ("q", "k", "v")),
    }

    merged = list(iter_merged_tensors(tensors, rules, model_name="test"))

    assert len(merged) == 1
    assert merged[0][0] == "model.layers.0.self_attn.qkv_proj.weight"
    assert merged[0][1].tolist() == [[1.0, 1.0], [2.0, 2.0], [3.0, 3.0]]


def test_iter_merged_tensors_reports_incomplete_merge_with_model_name():
    import pytest
    from freetoken.models.loader import MergeRule, iter_merged_tensors

    rules = {
        ".q_proj": MergeRule(".qkv_proj", "q", ("q", "k", "v")),
        ".k_proj": MergeRule(".qkv_proj", "k", ("q", "k", "v")),
        ".v_proj": MergeRule(".qkv_proj", "v", ("q", "k", "v")),
    }

    with pytest.raises(AssertionError, match="test.*Incomplete merge groups"):
        list(iter_merged_tensors([("x.q_proj.weight", torch.zeros(1, 1))], rules, model_name="test"))


def test_stack_expert_tensors_after_all_experts_arrive():
    from freetoken.models.loader import iter_stacked_experts

    expert_pattern = re.compile(
        r"^(?P<prefix>.+\.experts)\.(?P<idx>\d+)\.(?P<name>.+)$"
    )
    tensors = [
        ("model.layers.0.mlp.experts.1.gate_up_proj.weight", torch.full((1, 2), 11.0)),
        ("model.layers.0.mlp.experts.0.gate_up_proj.weight", torch.full((1, 2), 10.0)),
    ]

    packed = list(
        iter_stacked_experts(
            tensors,
            num_experts=2,
            model_name="qwen3_moe",
            expert_pattern=expert_pattern,
        )
    )

    assert len(packed) == 1
    assert packed[0][0] == "model.layers.0.mlp.experts.gate_up_proj"
    assert packed[0][1].shape == (2, 1, 2)
    assert packed[0][1][0].tolist() == [[10.0, 10.0]]
    assert packed[0][1][1].tolist() == [[11.0, 11.0]]


def test_stacked_expert_pieces_pair_each_layer_in_arrival_order():
    from freetoken.moe.expert_pieces import stacked_expert_pieces

    config = SimpleNamespace(num_layers=2, num_experts=2)
    tensors = [
        ("model.layers.1.mlp.experts.down_proj", torch.full((2, 4, 3), 11.0)),
        ("model.layers.0.mlp.experts.gate_up_proj", torch.full((2, 3, 4), 2.0)),
        ("model.layers.1.mlp.experts.gate_up_proj", torch.full((2, 3, 4), 3.0)),
        ("model.layers.0.mlp.experts.down_proj", torch.full((2, 4, 3), 10.0)),
    ]

    pieces = list(stacked_expert_pieces(tensors, config))

    assert [(layer, e0, e1) for layer, e0, e1, _ in pieces] == [(1, 0, 2), (0, 0, 2)]
    assert torch.equal(pieces[0][3]["gate_up"], torch.full((2, 3, 4), 3.0))
    assert torch.equal(pieces[0][3]["down"], torch.full((2, 4, 3), 11.0))
    assert torch.equal(pieces[1][3]["gate_up"], torch.full((2, 3, 4), 2.0))
    with pytest.raises(ValueError, match="Missing MoE expert source layers"):
        list(stacked_expert_pieces(tensors[:3], config))


def test_stacked_expert_pieces_skip_the_layers_of_other_formats():
    from freetoken.moe.expert_pieces import stacked_expert_pieces

    config = SimpleNamespace(num_layers=3, num_experts=2)
    tensors = [
        ("model.layers.0.mlp.experts.gate_up_proj", torch.full((2, 3, 4), 2.0)),
        ("model.layers.1.mlp.experts.0.gate_proj.weight_scale", torch.zeros(3, 1)),
        ("model.layers.0.mlp.experts.down_proj", torch.full((2, 4, 3), 10.0)),
        ("model.layers.2.mlp.experts.gate_up_proj", torch.full((2, 3, 4), 3.0)),
        ("model.layers.2.mlp.experts.down_proj", torch.full((2, 4, 3), 11.0)),
    ]

    pieces = list(stacked_expert_pieces(tensors, config, layers=[0]))

    assert [(layer, e0, e1) for layer, e0, e1, _ in pieces] == [(0, 0, 2)]
    with pytest.raises(ValueError, match="Unexpected expert weight key"):
        list(stacked_expert_pieces(tensors, config))