FreeToken brings datacenter-scale model serving to your desktop. Run massive models locally, fast and efficiently.
"""Hybrid decode's bandwidth-matched fetch split.
Covers the profile reader of --moe-hybrid-max-fetch auto, which turns the `ft bench bw`
row of each MoE layer kind into a fetch fraction.
"""
import json
import os
import pytest
from freetoken.moe.bench_profile import PROFILE_VERSION, default_profile_path, fetch_fraction, layer_kind_key, load_layer_kinds, locked_fetch_fraction
def test_layer_kind_rows_give_fetch_fractions(tmp_path):
bf16 = layer_kind_key("bf16", {}, 2048, 768, 8, 12)
gguf = layer_kind_key("gguf", {"gate_up_type": 12, "down_type": 14}, 2048, 768, 8, 12)
assert gguf == "gguf[down_type=14,gate_up_type=12]:2048x768:top8:t12"
prof = {
"version": PROFILE_VERSION,
"gpu": {"name": "FAKE GPU"},
"layer_kinds": {
bf16: {"cpu_moe_gbs": 100.0, "pcie_gather_gbs": 40.0},
# overlapped (contended) pair wins over the standalone numbers when present
gguf: {"cpu_moe_gbs": 100.0, "pcie_gather_gbs": 40.0, "cpu_moe_overlap_gbs": 90.0, "pcie_gather_overlap_gbs": 30.0},
},
}
path = tmp_path / "benchbw.json"
path.write_text(json.dumps(prof))
rows = load_layer_kinds(path=str(path))
# standalone fallback: full-contention assumption -> pcie / cpu
assert fetch_fraction(rows[bf16]) == pytest.approx(0.4)
# overlapped pair preferred: pcie_ov / (pcie_ov + cpu_ov)
assert fetch_fraction(rows[gguf]) == pytest.approx(0.25)
# a locked layer's fetch adds a memcpy at the CPU MoE rate before the gather: p / (2p + c), from the same pair
assert locked_fetch_fraction(rows[bf16]) == pytest.approx(40 / 180)
assert locked_fetch_fraction(rows[gguf]) == pytest.approx(30 / 150)
# another thread count is another kind
assert layer_kind_key("bf16", {}, 2048, 768, 8, 6) not in rows
# a profile from different hardware is ignored
assert load_layer_kinds(gpu_name="OTHER", path=str(path)) is None
# so is one of another version
path.write_text(json.dumps({**prof, "version": PROFILE_VERSION - 1}))
assert load_layer_kinds(path=str(path)) is None
def test_profile_lookup_prefers_the_gpu_uuid_file(tmp_path, monkeypatch):
monkeypatch.setenv("XDG_CACHE_HOME", str(tmp_path))
monkeypatch.delenv("FREETOKEN_BENCHBW_PATH", raising=False)
uuid = "GPU-2f3a9b1c-0000-1111-2222-333344445555"
def write(path, name, row):
os.makedirs(os.path.dirname(path), exist_ok=True)
with open(path, "w") as f:
json.dump({"version": PROFILE_VERSION, "gpu": {"name": name}, "layer_kinds": {"k": row}}, f)
# legacy single file only: used when the name matches, ignored otherwise
write(default_profile_path(), "FAKE GPU", "legacy")
assert load_layer_kinds(gpu_name="FAKE GPU", gpu_uuid=uuid) == {"k": "legacy"}
assert load_layer_kinds(gpu_name="OTHER", gpu_uuid=uuid) is None
# this card's own file wins over the legacy one
write(default_profile_path(uuid), "FAKE GPU", "card")
assert load_layer_kinds(gpu_name="FAKE GPU", gpu_uuid=uuid) == {"k": "card"}