FreeToken brings datacenter-scale model serving to your desktop. Run massive models locally, fast and efficiently.
"""Disk PLE backend, module level: store byte fidelity, on-disk layouts and errors, table vs GPU oracle, and the CUDA-graph sync protocol."""
from __future__ import annotations
import os
from pathlib import Path
import subprocess
import sys
import threading
from types import SimpleNamespace
import pytest
import torch
from freetoken.models.qwen4_exp.config import parse_config
from freetoken.models.qwen4_exp.ple import GpuResidentTable, NGramEmbedding
from .common import EOS, hash_constants, requires_cuda, toy_hf_config
from .test_ple import _meta
_row_store = pytest.importorskip("freetoken.kernel._row_store")
_KEY_PREFIX = "model.layers.1.ple.ple_embedding.ngram_embedding"
def _embedding() -> NGramEmbedding:
args = parse_config(toy_hf_config()).qwen4_args
emb = NGramEmbedding(args)
multipliers, sizes, offsets = hash_constants(args)
emb.layer_multipliers.copy_(multipliers)
emb.ngram_heads_vocab_sizes.copy_(sizes)
emb.ngram_heads_offsets.copy_(offsets)
return emb
def _bitwise_equal(got: torch.Tensor, want: torch.Tensor) -> bool:
# random table bytes include fp8 NaN encodings, and NaN != NaN under torch.equal
return torch.equal(got.view(torch.int16), want.view(torch.int16))
def _make_store(tmp_path, *, write=True, use_io_uring=True):
args = parse_config(toy_hf_config()).qwen4_args
multipliers, sizes, offsets = hash_constants(args)
total_rows = int(offsets[-1] + sizes[-1])
cols = args.ngram_head_dim
gen = torch.Generator().manual_seed(5)
table = torch.randint(0, 256, (total_rows, cols), dtype=torch.uint8, generator=gen)
path = tmp_path / "ple-table.bin"
if write:
path.write_bytes(table.numpy().tobytes())
store = _row_store.PleStore(
paths=[str(path)],
extent_file=[0],
extent_base=[0],
rows_per_extent=total_rows,
row_bytes=cols,
row_stride=cols,
multipliers=multipliers.tolist(),
head_vocab_sizes=sizes.tolist(),
head_offsets=offsets.tolist(),
eos_token_id=EOS,
use_io_uring=use_io_uring,
)
return store, table, args
def _fill(store, args, window, tokens):
ctx = torch.tensor([window[0], window[1], *tokens], dtype=torch.int64)
staging = torch.empty(len(tokens) * args.num_ngram_heads * args.ngram_head_dim, dtype=torch.uint8)
store.stage(ctx.data_ptr(), len(tokens), staging.data_ptr())
store.flush(0)
return staging
def _write_checkpoint(tmp_path, table, n_shards):
"""uint8 ``table`` bytes are stored as a scaled fp8 table, any other dtype as an unscaled table of that dtype."""
from safetensors.torch import save_file
fp8 = table.dtype is torch.uint8
per = table.shape[0] // n_shards
tensors = {
f"{_KEY_PREFIX}.shard_{i}.weight": table[i * per : (i + 1) * per].view(torch.float8_e4m3fn) if fp8 else table[i * per : (i + 1) * per].clone()
for i in range(n_shards)
}
if fp8:
tensors[f"{_KEY_PREFIX}.weight_scale"] = torch.tensor(0.03125, dtype=torch.bfloat16)
save_file(tensors, str(tmp_path / "model.safetensors"))
def _make_table(tmp_path, dtype=torch.float8_e4m3fn):
from freetoken.models.qwen4_exp.ple_disk import DiskRowTable, source_from_safetensors
args = parse_config(toy_hf_config()).qwen4_args
multipliers, sizes, offsets = hash_constants(args)
total_rows = int(offsets[-1] + sizes[-1])
gen = torch.Generator().manual_seed(9)
if dtype is torch.float8_e4m3fn:
table = torch.randint(0, 256, (total_rows, args.ngram_head_dim), dtype=torch.uint8, generator=gen)
else:
table = torch.randn((total_rows, args.ngram_head_dim), generator=gen).to(dtype)
n_shards = next(k for k in (4, 2, 1) if total_rows % k == 0)
_write_checkpoint(tmp_path, table, n_shards)
constants = {
"num_ngram_heads": args.num_ngram_heads,
"layer_multipliers": multipliers.tolist(),
"per_head_vocab_sizes": sizes.tolist(),
"per_head_offsets": offsets.tolist(),
"eos_token_id": EOS,
}
disk = DiskRowTable(source_from_safetensors(str(tmp_path)), constants)
oracle = GpuResidentTable(table.cuda().view(torch.float8_e4m3fn), scale=0.03125) if dtype is torch.float8_e4m3fn else GpuResidentTable(table.cuda())
return disk, oracle, args
def _decode_batch(history, token):
req = SimpleNamespace(
input_ids=torch.tensor(history, dtype=torch.int32),
device_len=len(history) + 1,
cached_len=len(history),
)
return SimpleNamespace(
is_decode=True,
input_ids=torch.tensor([token], dtype=torch.int32, device="cuda"),
reqs=[req],
size=1,
padded_size=1,
)
def test_store_stages_bitwise_rows(tmp_path):
store, table, args = _make_store(tmp_path)
emb = _embedding()
row = args.num_ngram_heads * args.ngram_head_dim
# full run with mid-sequence eos vs production row ids
seq = [3, 4, EOS, 5, EOS, EOS, 8, 9, 3, 4]
whole = _fill(store, args, (EOS, EOS), seq)
ids = emb.row_ids(_meta([seq], [[EOS, EOS]]))
assert torch.equal(whole, table[ids.reshape(-1)].reshape(-1)), "prefill vs oracle"
# decode = many 1-token stages; must reproduce the same bytes
parts, window = [], (EOS, EOS)
for t in seq:
parts.append(_fill(store, args, window, [t]))
window = (window[1], t)
assert torch.equal(torch.cat(parts), whole), "split stages vs one stage"
# several lanes merged into one flush stay independent
contexts = [(3, 4), (EOS, EOS), (7, EOS)]
ctx = torch.tensor([[o, nw, 9] for o, nw in contexts], dtype=torch.int64)
staging = torch.empty(3 * row, dtype=torch.uint8)
for i in range(len(contexts)):
store.stage(ctx.data_ptr() + 24 * i, 1, staging.data_ptr() + i * row)
store.flush(0)
for i, context in enumerate(contexts):
assert torch.equal(staging[i * row : (i + 1) * row], _fill(store, args, context, [9])), f"lane {i}"
# hundreds of deduped reads through the 64-deep pipeline
gen = torch.Generator().manual_seed(23)
big = torch.randint(0, EOS, (150,), generator=gen, dtype=torch.int64)
got = _fill(store, args, (EOS, EOS), big.tolist())
ids = emb.row_ids(_meta([big.tolist()], [[EOS, EOS]]))
assert torch.equal(got, table[ids.reshape(-1)].reshape(-1)), "pipeline vs oracle"
# flush signals the flag, even when nothing was staged
flag = torch.zeros(1, dtype=torch.int64)
store.flush(flag.data_ptr())
assert int(flag[0]) == 1, "empty flush must still signal"
def test_layouts_readers_and_errors(tmp_path):
# 4 extents in 2 files, out of order, unaligned junk between; the last extent ends at EOF
sizes, offsets = [500, 400, 300, 800], [0, 500, 900, 1200]
total, per, cols, eos = 2000, 500, 24, 90
gen = torch.Generator().manual_seed(11)
table = torch.randint(0, 256, (total, cols), dtype=torch.uint8, generator=gen)
shard = lambda i: table[i * per : (i + 1) * per].numpy().tobytes() # noqa: E731
nb = per * cols
flat = tmp_path / "flat.bin"
flat.write_bytes(table.numpy().tobytes())
fa, fb = tmp_path / "a.bin", tmp_path / "b.bin"
fa.write_bytes(b"j" * 1231 + shard(0) + b"k" * 77 + shard(2))
fb.write_bytes(shard(1) + b"m" * 4095 + shard(3))
kwargs = dict(
rows_per_extent=per, row_bytes=cols, row_stride=cols,
multipliers=[3, 5, 7], head_vocab_sizes=sizes, head_offsets=offsets,
eos_token_id=eos,
)
ref = _row_store.PleStore(
paths=[str(flat)], extent_file=[0, 0, 0, 0], extent_base=[0, nb, 2 * nb, 3 * nb], **kwargs
)
multi = _row_store.PleStore(
paths=[str(fa), str(fb)], extent_file=[0, 1, 0, 1],
extent_base=[1231, 0, 1231 + nb + 77, nb + 4095], **kwargs,
)
tokens = torch.randint(0, eos, (40,), generator=gen, dtype=torch.int64)
ctx = torch.cat((torch.tensor([eos, eos], dtype=torch.int64), tokens))
def run(store):
staging = torch.empty(40 * 4 * cols, dtype=torch.uint8)
store.stage(ctx.data_ptr(), 40, staging.data_ptr())
store.flush(0)
return staging
assert torch.equal(run(multi), run(ref)), "multi-extent vs flat"
# thread-pool fallback must produce the same bytes as io_uring
ring, _, args = _make_store(tmp_path)
pool, _, _ = _make_store(tmp_path, write=False, use_io_uring=False)
gen2 = torch.Generator().manual_seed(31)
seq = torch.randint(0, EOS, (150,), generator=gen2, dtype=torch.int64).tolist()
assert torch.equal(_fill(pool, args, (EOS, EOS), seq), _fill(ring, args, (EOS, EOS), seq)), "pool vs ring"
# geometry that exceeds the file is rejected at construction
del ring, pool
with open(tmp_path / "ple-table.bin", "r+b") as fh:
fh.truncate(1000)
with pytest.raises(Exception, match="extent needs"):
_make_store(tmp_path, write=False)
# checkpoint scan guards
from safetensors.torch import save_file
from freetoken.models.qwen4_exp.ple_disk import source_from_safetensors
save_file(
{f"{_KEY_PREFIX}.shard_0.weight": torch.zeros(8, 4, dtype=torch.uint8),
f"{_KEY_PREFIX}.weight_scale": torch.tensor(1.0, dtype=torch.bfloat16)},
str(tmp_path / "model.safetensors"),
)
with pytest.raises(ValueError, match="dtype"):
source_from_safetensors(str(tmp_path))
save_file(
{f"{_KEY_PREFIX}.shard_1.weight": torch.zeros(8, 4, dtype=torch.float8_e4m3fn),
f"{_KEY_PREFIX}.weight_scale": torch.tensor(1.0, dtype=torch.bfloat16)},
str(tmp_path / "model.safetensors"),
)
with pytest.raises(ValueError, match="contiguous"):
source_from_safetensors(str(tmp_path))
save_file(
{f"{_KEY_PREFIX}.shard_0.weight": torch.zeros(8, 4, dtype=torch.float8_e4m3fn),
f"{_KEY_PREFIX}.weight_scale": torch.tensor(1.0, dtype=torch.bfloat16)},
str(tmp_path / "model.safetensors"),
)
save_file(
{f"{_KEY_PREFIX}.shard_0.weight": torch.zeros(8, 4, dtype=torch.float8_e4m3fn)},
str(tmp_path / "model-2.safetensors"),
)
with pytest.raises(ValueError, match="duplicate"):
source_from_safetensors(str(tmp_path))
(tmp_path / "model-2.safetensors").unlink()
# truncated checkpoint: a contiguous shard prefix passes the scan, init rejects the row count
from freetoken.models.qwen4_exp.ple_disk import DiskRowTable
args = parse_config(toy_hf_config()).qwen4_args
multipliers, vocab, offs = hash_constants(args)
rows = int(offs[-1] + vocab[-1])
_write_checkpoint(tmp_path, torch.zeros(rows // 2, args.ngram_head_dim, dtype=torch.uint8), 1)
constants = {
"num_ngram_heads": args.num_ngram_heads, "layer_multipliers": multipliers.tolist(),
"per_head_vocab_sizes": vocab.tolist(), "per_head_offsets": offs.tolist(), "eos_token_id": EOS,
}
with pytest.raises(ValueError, match="hash addresses"):
DiskRowTable(source_from_safetensors(str(tmp_path)), constants)
@requires_cuda
@pytest.mark.parametrize("dtype", [torch.float8_e4m3fn, torch.bfloat16], ids=["fp8", "bf16"])
def test_disk_table_matches_oracle(tmp_path, dtype):
disk, oracle, args = _make_table(tmp_path, dtype)
emb = _embedding()
# prefill: two segments, one fresh and one mid-sequence window
seqs = [[3, 4, EOS, 5, 6, 8], [2, EOS, 11, 12, 13, 14]]
disk.fill([torch.tensor([EOS, EOS, *seqs[0]]), torch.tensor([21, 22, *seqs[1]])], graph=False)
row_ids = emb.row_ids(_meta(seqs, [[EOS, EOS], [21, 22]])).cuda()
assert _bitwise_equal(disk.lookup(row_ids), oracle.lookup(row_ids)), "prefill"
# decode steps with a rolling window, plus the out= contract
older, newer = 41, EOS
for token in (7, 9, 13):
disk.fill([torch.tensor([older, newer, token])], graph=False)
ids = emb.row_ids(_meta([[token]], [[older, newer]], decode=True)).cuda()
out = torch.empty((1, ids.shape[-1] * disk.head_dim), dtype=disk.dtype, device="cuda")
assert disk.lookup(ids, out) is out and _bitwise_equal(out, oracle.lookup(ids)), f"token {token}"
older, newer = newer, token
# the engine hook end to end: eager decode, then fresh + continuation prefill
disk.host_fill_batch(_decode_batch([3, 4, EOS, 5], 9), use_graph=False)
ids = emb.row_ids(_meta([[9]], [[EOS, 5]], decode=True)).cuda()
assert _bitwise_equal(disk.lookup(ids), oracle.lookup(ids)), "hook decode"
prompt = [3, 4, EOS, 5, 6, 8]
fresh = SimpleNamespace(input_ids=torch.tensor(prompt[:4], dtype=torch.int32), device_len=4, cached_len=0)
cont = SimpleNamespace(input_ids=torch.tensor(prompt, dtype=torch.int32), device_len=6, cached_len=4)
disk.host_fill_batch(SimpleNamespace(is_decode=False, padded_reqs=[fresh, cont]), use_graph=False)
ids = emb.row_ids(_meta([prompt[:4], prompt[4:]], [[EOS, EOS], [prompt[2], prompt[3]]])).cuda()
assert _bitwise_equal(disk.lookup(ids), oracle.lookup(ids)), "hook prefill"
@requires_cuda
def test_eager_fill_does_not_overwrite_an_in_flight_lookup(tmp_path):
"""The overlap scheduler fills batch k+1 while batch k's lookup copy is still queued behind the
layers before it (a sleep kernel here): batch k must read its own rows."""
disk, oracle, args = _make_table(tmp_path)
emb = _embedding()
steps = [(3, 4, 7), (4, 7, 9), (7, 9, 13), (9, 13, 5)]
ids = [emb.row_ids(_meta([[token]], [[older, newer]], decode=True)).cuda() for older, newer, token in steps]
got = []
for (older, newer, token), step_ids in zip(steps, ids):
disk.fill([torch.tensor([older, newer, token])], graph=False)
torch.cuda._sleep(50_000_000)
got.append(disk.lookup(step_ids))
torch.cuda.synchronize()
for i, (out, step_ids) in enumerate(zip(got, ids)):
assert _bitwise_equal(out, oracle.lookup(step_ids)), f"step {i} read another step's rows"
@requires_cuda
def test_graph_sync_protocol(tmp_path, monkeypatch):
disk, oracle, args = _make_table(tmp_path)
emb = _embedding()
rows = 1
row_ids = torch.zeros((rows, args.num_ngram_heads), dtype=torch.int64, device="cuda")
out = torch.empty((rows, args.num_ngram_heads * disk.head_dim), dtype=disk.dtype, device="cuda")
stream = torch.cuda.Stream()
stream.wait_stream(torch.cuda.current_stream())
with torch.cuda.stream(stream):
graph = torch.cuda.CUDAGraph()
with torch.cuda.graph(graph, stream=stream):
disk.lookup(row_ids, out)
torch.cuda.current_stream().wait_stream(stream)
# plain fill-then-replay
disk.fill([torch.tensor([3, 4, 7])], graph=True)
graph.replay()
torch.cuda.synchronize()
ids = emb.row_ids(_meta([[7]], [[3, 4]], decode=True)).cuda()
assert _bitwise_equal(out, oracle.lookup(ids)), "capture+replay"
if disk._wait_sync:
# launch first, fill after: an early WAIT pass would surface the previous step's bytes
older, newer = 3, 4
for token in (7, 9, 13):
graph.replay()
disk.fill([torch.tensor([older, newer, token])], graph=True)
torch.cuda.synchronize()
ids = emb.row_ids(_meta([[token]], [[older, newer]], decode=True)).cuda()
assert _bitwise_equal(out, oracle.lookup(ids)), f"wait-sync token {token}"
older, newer = newer, token
# the engine-shaped seam: replay inside the context, deferred fill on exit
with disk.forward_host_ctx(_decode_batch([3, 4], 7), use_graph=True):
graph.replay()
torch.cuda.synchronize()
ids = emb.row_ids(_meta([[7]], [[3, 4]], decode=True)).cuda()
assert _bitwise_equal(out, oracle.lookup(ids)), "forward_host_ctx deferred"
# gate mode: the hook fills inline and returns no deferred
monkeypatch.setenv("FREETOKEN_PLE_SYNC", "gate")
gated, _, _ = _make_table(tmp_path)
assert not gated._wait_sync
assert gated.host_fill_batch(_decode_batch([3, 4], 5), use_graph=True) is None
def _wait_sync_large_graph(tmp_path, kernels):
from freetoken.core import Batch, Context, Req, set_global_ctx
from freetoken.distributed import set_tp_info
from freetoken.engine.graph import GraphRunner
set_tp_info(0, 1)
set_global_ctx(Context(page_size=1))
disk, oracle, args = _make_table(tmp_path)
emb = _embedding()
row_ids = torch.zeros((8, args.num_ngram_heads), dtype=torch.int64, device="cuda")
pad = torch.zeros(256, device="cuda")
release = threading.Event()
fill = disk.fill
def delayed_fill(*args, **kwargs):
assert release.wait(timeout=5), "fill was not released"
return fill(*args, **kwargs)
def forward():
out = disk.lookup(row_ids)
if torch.cuda.is_current_stream_capturing():
for _ in range(kernels):
pad.add_(1)
return out
disk.fill = delayed_fill
req = Req(input_ids=torch.tensor([3, 4, 9], dtype=torch.int32), table_idx=0,
cached_len=2, output_len=3, uid=0, sampling_params=None, cache_handle=None)
attn = SimpleNamespace(init_capture_graph=lambda **kwargs: None,
prepare_for_capture=lambda batch: None, prepare_for_replay=lambda batch: None)
stream = torch.cuda.Stream()
stream.wait_stream(torch.cuda.current_stream())
with torch.cuda.stream(stream):
# Exercise the capture-time upload from #697 as well as the PLE dispatch hook.
runner = GraphRunner(stream=stream, device=torch.device("cuda"), forward=forward,
attn_backend=attn, cuda_graph_bs=[8], cuda_graph_max_bs=None,
free_memory=1024**3, max_seq_len=16, dummy_req=req, out_dtype=torch.float32)
batch = Batch(reqs=[req] * 8, phase="decode")
runner.pad_batch(batch)
batch.positions = torch.full((8,), 2, dtype=torch.int32, device="cuda")
batch.out_loc = torch.arange(8, dtype=torch.int32, device="cuda")
for step, token in enumerate((9, 13, 5)):
release.clear()
batch.input_ids = torch.full((8,), token, dtype=torch.int32, device="cuda")
with disk.forward_host_ctx(batch, use_graph=True):
# Keep the WAIT unsatisfied when the first replay starts.
assert int(disk._flag[0]) == 0
timer = threading.Timer(0.05, release.set)
timer.start()
try:
out = runner.replay(batch)
finally:
timer.join()
stream.synchronize()
ids = emb.row_ids(_meta([[token]] * 8, [[3, 4]] * 8, decode=True)).cuda()
assert _bitwise_equal(out.to(disk.dtype), oracle.lookup(ids))
assert torch.equal(pad, torch.full_like(pad, (step + 1) * kernels))
assert int(disk._flag[0]) == 0
runner.destroy_cuda_graphs()
@requires_cuda
def test_wait_sync_large_graph(tmp_path):
from freetoken.kernel.row_store import probe_wait_sync
if not probe_wait_sync("auto", torch.device("cuda")):
pytest.skip("stream memops unavailable")
root = Path(__file__).resolve().parents[3]
env = dict(os.environ, FREETOKEN_PLE_SYNC="wait")
env["PYTHONPATH"] = os.pathsep.join((str(root / "python"), str(root), env.get("PYTHONPATH", "")))
script = (
"from pathlib import Path; "
"from tests.models.qwen4_exp.test_ple_disk import _wait_sync_large_graph; "
f"_wait_sync_large_graph(Path({str(tmp_path)!r}), 16384)"
)
result = subprocess.run(
[sys.executable, "-c", script], cwd=root, env=env, capture_output=True, text=True, timeout=45,
)
assert result.returncode == 0, result.stdout + result.stderr