FreeToken brings datacenter-scale model serving to your desktop. Run massive models locally, fast and efficiently.
"""Block-FP8 CPU MoE GEMV vs a float64 reference and FreeToken's Triton block-fp8 decode kernel.
Two things this is really guarding:
* the scale is indexed [row // 128][k // 128] and *multiplies*, despite upstream naming it ``weight_scale_inv``. Inverting it, or transposing the two axes, still produces plausible-looking output.
* the SIMD kernels decode e4m3 by reading each byte as fp16 bits. Subnormals carry too little weight in a full-range GEMV to show a decode error, so they get their own case.
"""
from __future__ import annotations
from types import SimpleNamespace
import pytest
import torch
BLK = 128
def _nb(n: int) -> int:
return (n + BLK - 1) // BLK
def _make_fp8_block_cache(L, E, H, I, seed=0):
from freetoken.kernel.aot_models import fp8_block_scale_pad
if torch.cuda.is_available():
from freetoken.kernel.pinned import alloc_pinned_tensor
else:
def alloc_pinned_tensor(*shape, dtype):
return torch.empty(*shape, dtype=dtype)
torch.manual_seed(seed)
S = L * E
def rows(OUT, IN):
w = alloc_pinned_tensor(S, OUT, IN, dtype=torch.float8_e4m3fn)
w.copy_((torch.randn(S, OUT, IN) * 6.0).to(torch.float8_e4m3fn))
s = alloc_pinned_tensor(
S, _nb(OUT), fp8_block_scale_pad(_nb(OUT), _nb(IN)), dtype=torch.bfloat16
)
s.copy_((0.01 + 0.02 * torch.rand_like(s)).to(torch.bfloat16))
return w, s
gu, gus = rows(2 * I, H)
dn, dns = rows(H, I)
return SimpleNamespace(
quant_format="fp8_block",
bank_sources={
"gate_up": list(gu.split(E)), "gate_up_scale": list(gus.split(E)),
"down": list(dn.split(E)), "down_scale": list(dns.split(E)),
},
num_layers=L,
num_experts=E,
decode_target="cpu",
cpu_executor=None,
)
def test_cpu_fp8_block_accepts_qwen38_padded_scale_bank():
from freetoken.kernel.aot_models import fp8_block_scale_pad
from freetoken.moe.cpu_executor import CpuMoeExecutor
H, I = 2560, 640
cache = _make_fp8_block_cache(1, 1, H, I)
assert cache.bank_sources["down_scale"][0].shape[1:] == (
_nb(H), fp8_block_scale_pad(_nb(H), _nb(I))
) == (20, 6)
ex = CpuMoeExecutor(
cache,
top_k=1,
activation="silu",
apply_router_weight_on_input=False,
num_threads=1,
max_tokens=1,
device=torch.device("cpu"),
)
hidden = torch.randn(1, H, dtype=torch.bfloat16)
io = ex._io
io["x"].copy_(hidden)
io["ids"].zero_()
io["w"].fill_(1.0)
ex._ext.run(0, 1)
cpu_out = io["y"][0].float().to(torch.float64)
banks = cache.bank_sources
gate_up = _reference_fp8_block_gemv(banks["gate_up"][0], banks["gate_up_scale"][0], hidden[0])
intermediate = torch.nn.functional.silu(gate_up[:I]) * gate_up[I:]
expected = _reference_fp8_block_gemv(banks["down"][0], banks["down_scale"][0], intermediate)
rel = (cpu_out - expected).abs().max() / (expected.abs().max() + 1e-6)
assert rel < 5e-3, f"max relative error {rel.item()}"
# every finite e4m3 code, and the subnormal ones (exp == 0) with both signs
_FINITE_CODES = [c for c in range(256) if c not in (0x7F, 0xFF)]
_SUBNORMAL_CODES = [c for c in range(256) if c & 0x78 == 0]
def _make_coded_fp8_block_cache(H, I, codes, seed=0):
from freetoken.kernel.aot_models import fp8_block_scale_pad
if torch.cuda.is_available():
from freetoken.kernel.pinned import alloc_pinned_tensor
else:
def alloc_pinned_tensor(*shape, dtype):
return torch.empty(*shape, dtype=dtype)
torch.manual_seed(seed)
pool = torch.tensor(codes, dtype=torch.uint8)
def rows(OUT, IN):
count = OUT * IN
picked = pool.repeat((count + pool.numel() - 1) // pool.numel())[:count]
picked = picked[torch.randperm(count)].reshape(1, OUT, IN)
weights = alloc_pinned_tensor(1, OUT, IN, dtype=torch.float8_e4m3fn)
weights.copy_(picked.view(torch.float8_e4m3fn))
scales = alloc_pinned_tensor(
1, _nb(OUT), fp8_block_scale_pad(_nb(OUT), _nb(IN)), dtype=torch.bfloat16
)
scales.copy_((0.01 + 0.09 * torch.rand_like(scales)).to(torch.bfloat16))
return weights, scales
gu, gus = rows(2 * I, H)
dn, dns = rows(H, I)
return SimpleNamespace(
quant_format="fp8_block",
bank_sources={
"gate_up": [gu], "gate_up_scale": [gus],
"down": [dn], "down_scale": [dns],
},
num_layers=1,
num_experts=1,
decode_target="cpu",
cpu_executor=None,
)
def _reference_fp8_block_gemv(weights, scales, x):
rows, cols = weights.shape[-2:]
scale = scales[0, :_nb(rows), :_nb(cols)].to(torch.float64)
scale = scale.repeat_interleave(BLK, 0).repeat_interleave(BLK, 1)[:rows, :cols]
return (weights[0].to(torch.float64) * scale) @ x.to(torch.float64)
@pytest.mark.parametrize("codes", ["finite", "subnormal"])
@pytest.mark.parametrize("isa", ["scalar", "avx2", "avx512"])
def test_cpu_fp8_block_matches_float64_reference(monkeypatch, isa, codes):
from freetoken.moe.cpu_executor import CpuMoeExecutor
H, I = 259, 257 # ragged K for both GEMVs; 257 down rows exercise many reductions
cache = _make_coded_fp8_block_cache(H, I, _FINITE_CODES if codes == "finite" else _SUBNORMAL_CODES, seed=20260904)
monkeypatch.setenv("FREETOKEN_CPU_MOE_ISA", isa)
ex = CpuMoeExecutor(
cache,
top_k=1,
activation="silu",
apply_router_weight_on_input=False,
num_threads=1,
max_tokens=1,
device=torch.device("cpu"),
)
tiers = {"scalar": 0, "avx2": 1, "avx512": 2}
if tiers[isa] > ("scalar", "avx2", "avx512f", "avx512bf16").index(ex.isa):
pytest.skip(f"requested {isa}, host/build selected {ex.isa}")
hidden = torch.randn(1, H, dtype=torch.bfloat16)
io = ex._io
io["x"].copy_(hidden)
io["ids"].zero_()
io["w"].fill_(1.0)
ex._ext.run(0, 1)
cpu_out = io["y"][0].float().to(torch.float64)
banks = cache.bank_sources
gate_up = _reference_fp8_block_gemv(banks["gate_up"][0], banks["gate_up_scale"][0], hidden[0])
intermediate = (torch.nn.functional.silu(gate_up[:I]) * gate_up[I:])
expected = _reference_fp8_block_gemv(banks["down"][0], banks["down_scale"][0], intermediate)
rel = (cpu_out - expected).abs().max() / (expected.abs().max() + 1e-6)
# The executor stores bf16; measured max relative error is 3.3e-3 (finite codes) and 1.2e-3 (subnormal codes).
# The bound leaves room for reduction order, not for a narrow accumulator.
assert rel < 5e-3, f"max relative error {rel.item()}"
@pytest.mark.parametrize("isa", ["scalar", "avx2", "avx512", "avx512bf16"])
def test_cpu_fp8_block_isa_override(monkeypatch, isa):
from freetoken.moe.cpu_executor import CpuMoeExecutor
cache = _make_fp8_block_cache(1, 1, 256, 128, seed=17)
monkeypatch.delenv("FREETOKEN_CPU_MOE_ISA", raising=False)
auto = CpuMoeExecutor(
cache, top_k=1, activation="silu", apply_router_weight_on_input=False,
num_threads=1, max_tokens=1, device=torch.device("cpu"),
)
names = ("scalar", "avx2", "avx512f", "avx512bf16")
auto_rank = names.index(auto.isa)
monkeypatch.setenv("FREETOKEN_CPU_MOE_ISA", isa)
forced = CpuMoeExecutor(
cache, top_k=1, activation="silu", apply_router_weight_on_input=False,
num_threads=1, max_tokens=1, device=torch.device("cpu"),
)
assert names.index(forced.isa) == min(names.index({
"scalar": "scalar", "avx2": "avx2", "avx512": "avx512f",
"avx512bf16": "avx512bf16",
}[isa]), auto_rank)
# FREETOKEN_CPU_MOE_ISA tier -> the fp8_block kernel it selects; the avx512bf16 tier selects the avx512f kernel, so it is not repeated here.
# The env var caps *down* only, so a tier the CPU or the build lacks selects a lower kernel, which turns into a skip instead of testing the same kernel twice.
_ISA_TIERS = {"scalar": "scalar", "avx2": "avx2", "avx512": "avx512f"}
@pytest.mark.skipif(not torch.cuda.is_available(), reason="needs CUDA")
@pytest.mark.parametrize("isa", list(_ISA_TIERS))
@pytest.mark.parametrize("bs", [1, 2, 5])
def test_cpu_fp8_block_matches_gpu_decode_kernel(bs, isa, monkeypatch):
from freetoken.moe.cpu_executor import CpuMoeExecutor
from freetoken.moe.fused_fp8_block import fused_experts_decode_fp8_block
L, E, H, I, top_k = 2, 8, 256, 128, 4
layer = 1
dev = torch.device("cuda")
cache = _make_fp8_block_cache(L, E, H, I, seed=bs)
# Set before construction: the executor latches its dot kernel there.
monkeypatch.setenv("FREETOKEN_CPU_MOE_ISA", isa)
ex = CpuMoeExecutor(
cache,
top_k=top_k,
activation="silu",
apply_router_weight_on_input=False,
num_threads=0,
max_tokens=bs,
device=dev,
)
if ex.isa != _ISA_TIERS[isa]:
pytest.skip(f"asked for {isa}, host/build selected {ex.isa}: not supported here")
hidden = torch.randn(bs, H, device=dev, dtype=torch.bfloat16)
ids = torch.stack([torch.randperm(E, device=dev)[:top_k] for _ in range(bs)]).to(torch.int32)
w = torch.rand(bs, top_k, device=dev, dtype=torch.float32)
cpu_out = ex.decode(layer, hidden, w, ids).float()
torch.cuda.synchronize()
gpu_out = fused_experts_decode_fp8_block(
hidden,
cache.bank_sources["gate_up"][layer].to(dev),
cache.bank_sources["gate_up_scale"][layer].to(dev),
cache.bank_sources["down"][layer].to(dev),
cache.bank_sources["down_scale"][layer].to(dev),
w, ids.clone(), "silu", False,
).float()
rel = (cpu_out - gpu_out).abs().max() / (gpu_out.abs().max() + 1e-6)
assert rel < 2e-2, f"bs={bs} isa={ex.isa} rel err {rel.item()}"
def test_fp16_bits_carry_every_e4m3_byte():
"""A byte b read as fp16 bits (b & 0x80) << 8 | (b & 0x7F) << 7 is its e4m3 value times 2^-8, which the SIMD kernels rely on."""
codes = torch.arange(256, dtype=torch.int32)
ref = codes.to(torch.uint8).view(torch.float8_e4m3fn).float()
bits = ((codes & 0x80) << 8) | ((codes & 0x7F) << 7)
got = torch.where(bits >= 0x8000, bits - 0x10000, bits).to(torch.int16).view(torch.float16).float() * 256
finite = torch.isfinite(ref)
assert torch.equal(got[finite], ref[finite])
assert torch.equal(got[~finite].abs(), torch.full((2,), 480.0)) # 0x7F / 0xFF, as e4m3_decode