Skip to content

FreeToken brings datacenter-scale model serving to your desktop. Run massive models locally, fast and efficiently.

refactor(ple): extract a shared disk row store (#548)

Xiaoze Fanf5b9700cc103
diff --git a/python/freetoken/kernel/csrc/ple_store/ple_store_ext.cpp b/python/freetoken/kernel/csrc/row_store/row_store_ext.cpp
similarity index 83%
rename from python/freetoken/kernel/csrc/ple_store/ple_store_ext.cpp
rename to python/freetoken/kernel/csrc/row_store/row_store_ext.cpp
index 2d54d692..e33b18a8 100644
--- a/python/freetoken/kernel/csrc/ple_store/ple_store_ext.cpp
+++ b/python/freetoken/kernel/csrc/row_store/row_store_ext.cpp
@@ -1,7 +1,12 @@
-// Disk-backed PLE row store: rows read straight from the checkpoint's fp8 shard tensors
-// through an extent table (PleRowSource in ple_ssd.py). Engine-thread only, no locks.
-// Duplicate rows in one fill dedup into ONE batched read round; no RAM cache and no
-// per-sequence state. Hash reference: tests/models/qwen4_exp/test_ple_disk.py.
+// Disk-backed row store: fixed-width rows read straight from checkpoint shard tensors
+// through an extent table (row i of extent e at extent_base[e] + i * row_stride).
+// Engine-thread only, no locks. Duplicate rows in one fill dedup into ONE batched read
+// round; no RAM cache and no per-sequence state.
+//
+// RowStore -- hash-agnostic: stage_rows(row_ids) + flush(); callers supply row ids.
+// PleStore -- RowStore plus the Qwen3.8-Flash-Next PLE n-gram hash: stage(tokens).
+// Hash reference: tests/models/qwen4_exp/test_ple_disk.py.
+//
// Platform seams: TableFile (O_DIRECT+pread; Win: NO_BUFFERING), BatchReader (io_uring,
// pread-pool fallback = the portable shape), cumemop_* (dlopen libcuda; Win: nvcuda).
@@ -183,7 +188,9 @@ class ThreadPoolBatchReader final : public BatchReader {
ThreadPoolBatchReader() {
// these threads block on I/O, not compute, so the core count is only a default
unsigned n = std::max(1u, std::min(kReaderThreads, std::thread::hardware_concurrency()));
- if (const char *env = std::getenv("FREETOKEN_PLE_READER_THREADS")) {
+ const char *env = std::getenv("FREETOKEN_ROW_STORE_READER_THREADS");
+ if (env == nullptr) env = std::getenv("FREETOKEN_PLE_READER_THREADS"); // the PLE-era name
+ if (env != nullptr) {
// kBatchEntries is the submit depth, so threads past it never get a read
const int v = std::atoi(env);
if (v > 0) n = std::min((unsigned)v, kBatchEntries);
@@ -432,29 +439,24 @@ int64_t pos_mod(int64_t v, int64_t m) {
return r < 0 ? r + m : r;
}
-class PleStore {
+class RowStore {
struct Extent {
const TableFile *file;
int64_t base;
};
public:
- PleStore(std::vector<std::string> paths, std::vector<int64_t> extent_file,
+ RowStore(std::vector<std::string> paths, std::vector<int64_t> extent_file,
std::vector<int64_t> extent_base, int64_t rows_per_extent, int64_t row_bytes,
- int64_t row_stride, std::vector<int64_t> multipliers, std::vector<int64_t> head_vocab_sizes,
- std::vector<int64_t> head_offsets, int64_t eos_token_id, bool use_io_uring)
- : row_bytes_(row_bytes),
- row_stride_(row_stride),
- rows_per_extent_(rows_per_extent),
- mult_(std::move(multipliers)),
- sizes_(std::move(head_vocab_sizes)),
- offsets_(std::move(head_offsets)),
- eos_(eos_token_id) {
- if (mult_.size() != 3 || sizes_.size() != offsets_.size() || sizes_.empty())
- throw std::runtime_error("PLE hash geometry: want 3 multipliers and equal-length head tables");
+ int64_t row_stride, bool use_io_uring)
+ : row_bytes_(row_bytes), row_stride_(row_stride), rows_per_extent_(rows_per_extent) {
+ if (row_bytes_ <= 0 || row_stride_ < row_bytes_ || rows_per_extent_ <= 0)
+ throw std::runtime_error("row store geometry: want row_bytes > 0, row_stride >= row_bytes, rows_per_extent > 0");
if (row_bytes_ > kPage)
- throw std::runtime_error("PLE row_bytes " + std::to_string(row_bytes_) +
+ throw std::runtime_error("row_bytes " + std::to_string(row_bytes_) +
" exceeds a page; bounce slots assume one-page rows");
+ if (extent_file.size() != extent_base.size() || extent_file.empty())
+ throw std::runtime_error("row store geometry: extent_file and extent_base must be equal-length and non-empty");
for (const std::string &p : paths)
files_.push_back(std::make_unique<TableFile>(p));
const int64_t extent_bytes = (rows_per_extent_ - 1) * row_stride_ + row_bytes_;
@@ -472,36 +474,33 @@ class PleStore {
}
// reader first: a still-running read must not land in freed bounce memory
- ~PleStore() {
+ virtual ~RowStore() {
reader_.reset();
free(bounce_);
}
- PleStore(const PleStore &) = delete;
- PleStore &operator=(const PleStore &) = delete;
+ RowStore(const RowStore &) = delete;
+ RowStore &operator=(const RowStore &) = delete;
- // Row ids for the token at w[2] with context (w[0], w[1]); mirrors NGramEmbedding.row_ids incl. the eos barrier.
- void hash_rows(const int64_t *w, int64_t *rows) {
- const int64_t prev1 = w[1];
- const int64_t prev2 = prev1 == eos_ ? eos_ : w[0];
- const int64_t bigram = wrap_mul(w[2], mult_[0]) ^ wrap_mul(prev1, mult_[1]);
- const int64_t trigram = bigram ^ wrap_mul(prev2, mult_[2]);
- const size_t half = sizes_.size() / 2;
- for (size_t h = 0; h < sizes_.size(); h++)
- rows[h] = pos_mod(h < half ? bigram : trigram, sizes_[h]) + offsets_[h];
- }
+ int64_t row_bytes() const { return row_bytes_; }
+ int64_t total_rows() const { return (int64_t)extents_.size() * rows_per_extent_; }
- // Hash and queue one run: tokens_addr holds n+2 ids, the leading two are context. No I/O until flush().
- void stage(uintptr_t tokens_addr, int64_t n, uintptr_t staging_addr) {
- const int64_t *tokens = reinterpret_cast<const int64_t *>(tokens_addr);
+ // Queue n row ids (int64 at rows_addr); row i lands at staging_addr + i * dst_stride
+ // (dst_stride = 0 packs rows back to back). Out-of-range ids throw before any I/O is
+ // issued. No I/O until flush().
+ void stage_rows(uintptr_t rows_addr, int64_t n, uintptr_t staging_addr, int64_t dst_stride) {
+ const int64_t *rows = reinterpret_cast<const int64_t *>(rows_addr);
uint8_t *staging = reinterpret_cast<uint8_t *>(staging_addr);
- const size_t heads = sizes_.size();
- std::vector<int64_t> rows(heads);
+ if (dst_stride == 0) dst_stride = row_bytes_;
+ if (dst_stride < row_bytes_)
+ throw std::runtime_error("stage_rows: dst_stride below row_bytes would overlap rows");
+ const int64_t total = total_rows();
for (int64_t i = 0; i < n; i++) {
- hash_rows(tokens + i, rows.data());
- for (size_t h = 0; h < heads; h++)
- request_row(rows[h], staging + ((size_t)i * heads + h) * row_bytes_);
+ if (rows[i] < 0 || rows[i] >= total)
+ throw std::out_of_range("stage_rows: row id " + std::to_string(rows[i]) + " outside [0, " +
+ std::to_string(total) + ")");
}
+ for (int64_t i = 0; i < n; i++) request_row(rows[i], staging + (size_t)i * (size_t)dst_stride);
}
// One batched disk round for everything staged; signals even when nothing was.
@@ -519,15 +518,7 @@ class PleStore {
std::to_string(files_.size()) + " files";
}
- private:
- struct Pending {
- const TableFile *file;
- int64_t read_off;
- int64_t read_len;
- int64_t row_off; // row payload start inside the read buffer
- std::vector<uint8_t *> dsts;
- };
-
+ protected:
// Queue dst on this fill's pending batch; duplicate rows fan out from one read.
void request_row(int64_t row_id, uint8_t *dst) {
auto pit = pending_index_.find(row_id);
@@ -548,11 +539,20 @@ class PleStore {
pending_.push_back(std::move(p));
}
+ private:
+ struct Pending {
+ const TableFile *file;
+ int64_t read_off;
+ int64_t read_len;
+ int64_t row_off; // row payload start inside the read buffer
+ std::vector<uint8_t *> dsts;
+ };
+
// Read every pending row in reader-capacity batches and fan out the copies.
void flush_pending() {
if (pending_.empty()) return;
struct Cleanup {
- PleStore *s;
+ RowStore *s;
~Cleanup() {
s->pending_.clear();
s->pending_index_.clear();
@@ -586,8 +586,6 @@ class PleStore {
}
int64_t row_bytes_, row_stride_, rows_per_extent_;
- std::vector<int64_t> mult_, sizes_, offsets_;
- int64_t eos_;
std::vector<std::unique_ptr<TableFile>> files_;
std::vector<Extent> extents_;
std::unique_ptr<BatchReader> reader_;
@@ -597,10 +595,71 @@ class PleStore {
std::unordered_map<int64_t, size_t> pending_index_;
};
+// The Qwen3.8-Flash-Next PLE table: RowStore plus the checkpoint's n-gram hash, so a fill
+// takes raw token runs and never crosses the Python boundary per row.
+class PleStore final : public RowStore {
+ public:
+ PleStore(std::vector<std::string> paths, std::vector<int64_t> extent_file,
+ std::vector<int64_t> extent_base, int64_t rows_per_extent, int64_t row_bytes,
+ int64_t row_stride, std::vector<int64_t> multipliers, std::vector<int64_t> head_vocab_sizes,
+ std::vector<int64_t> head_offsets, int64_t eos_token_id, bool use_io_uring)
+ : RowStore(std::move(paths), std::move(extent_file), std::move(extent_base), rows_per_extent,
+ row_bytes, row_stride, use_io_uring),
+ mult_(std::move(multipliers)),
+ sizes_(std::move(head_vocab_sizes)),
+ offsets_(std::move(head_offsets)),
+ eos_(eos_token_id) {
+ if (mult_.size() != 3 || sizes_.size() != offsets_.size() || sizes_.empty())
+ throw std::runtime_error("PLE hash geometry: want 3 multipliers and equal-length head tables");
+ }
+
+ // Row ids for the token at w[2] with context (w[0], w[1]); mirrors NGramEmbedding.row_ids incl. the eos barrier.
+ void hash_rows(const int64_t *w, int64_t *rows) {
+ const int64_t prev1 = w[1];
+ const int64_t prev2 = prev1 == eos_ ? eos_ : w[0];
+ const int64_t bigram = wrap_mul(w[2], mult_[0]) ^ wrap_mul(prev1, mult_[1]);
+ const int64_t trigram = bigram ^ wrap_mul(prev2, mult_[2]);
+ const size_t half = sizes_.size() / 2;
+ for (size_t h = 0; h < sizes_.size(); h++)
+ rows[h] = pos_mod(h < half ? bigram : trigram, sizes_[h]) + offsets_[h];
+ }
+
+ // Hash and queue one run: tokens_addr holds n+2 ids, the leading two are context. No I/O until flush().
+ void stage(uintptr_t tokens_addr, int64_t n, uintptr_t staging_addr) {
+ const int64_t *tokens = reinterpret_cast<const int64_t *>(tokens_addr);
+ uint8_t *staging = reinterpret_cast<uint8_t *>(staging_addr);
+ const size_t heads = sizes_.size();
+ std::vector<int64_t> rows(heads);
+ for (int64_t i = 0; i < n; i++) {
+ hash_rows(tokens + i, rows.data());
+ for (size_t h = 0; h < heads; h++)
+ request_row(rows[h], staging + ((size_t)i * heads + h) * (size_t)row_bytes());
+ }
+ }
+
+ private:
+ std::vector<int64_t> mult_, sizes_, offsets_;
+ int64_t eos_;
+};
+
} // namespace
PYBIND11_MODULE(TORCH_EXTENSION_NAME, m) {
- py::class_<PleStore>(m, "PleStore")
+ py::class_<RowStore>(m, "RowStore")
+ .def(py::init<std::vector<std::string>, std::vector<int64_t>, std::vector<int64_t>,
+ int64_t, int64_t, int64_t, bool>(),
+ py::arg("paths"), py::arg("extent_file"), py::arg("extent_base"),
+ py::arg("rows_per_extent"), py::arg("row_bytes"), py::arg("row_stride"),
+ py::arg("use_io_uring") = true)
+ .def("stage_rows", &RowStore::stage_rows, py::arg("rows_addr"), py::arg("n"),
+ py::arg("staging_addr"), py::arg("dst_stride") = 0,
+ py::call_guard<py::gil_scoped_release>())
+ .def("flush", &RowStore::flush, py::arg("signal_addr") = 0,
+ py::call_guard<py::gil_scoped_release>())
+ .def("io_backend", &RowStore::io_backend)
+ .def_property_readonly("row_bytes", &RowStore::row_bytes)
+ .def_property_readonly("total_rows", &RowStore::total_rows);
+ py::class_<PleStore, RowStore>(m, "PleStore")
.def(py::init<std::vector<std::string>, std::vector<int64_t>, std::vector<int64_t>,
int64_t, int64_t, int64_t, std::vector<int64_t>,
std::vector<int64_t>, std::vector<int64_t>, int64_t, bool>(),
@@ -610,10 +669,7 @@ PYBIND11_MODULE(TORCH_EXTENSION_NAME, m) {
py::arg("head_vocab_sizes"), py::arg("head_offsets"), py::arg("eos_token_id"),
py::arg("use_io_uring") = true)
.def("stage", &PleStore::stage, py::arg("tokens_addr"), py::arg("n"),
- py::arg("staging_addr"), py::call_guard<py::gil_scoped_release>())
- .def("flush", &PleStore::flush, py::arg("signal_addr") = 0,
- py::call_guard<py::gil_scoped_release>())
- .def("io_backend", &PleStore::io_backend);
+ py::arg("staging_addr"), py::call_guard<py::gil_scoped_release>());
m.def("memop_write", &memop_write, py::arg("stream"), py::arg("addr"), py::arg("value"));
m.def("memop_wait_geq", &memop_wait_geq, py::arg("stream"), py::arg("addr"), py::arg("value"));
m.def("memop_wait_reset", &memop_wait_reset, py::arg("stream"), py::arg("flag_addr"));
diff --git a/python/freetoken/kernel/row_store.py b/python/freetoken/kernel/row_store.py
new file mode 100644
index 00000000..5b42479e
--- /dev/null
+++ b/python/freetoken/kernel/row_store.py
@@ -0,0 +1,54 @@
+"""Python face of the disk row store extension (``kernel/csrc/row_store``) and its CUDA-graph sync.
+
+``RowStore`` reads fixed-width rows straight out of checkpoint shard files (io_uring or a pread pool,
+O_DIRECT) into pinned staging by row id; ``PleStore`` adds PLE's n-gram hashing.
+
+Graph sync: a captured graph cannot wait for host I/O, so a captured ``lookup`` WAITs on a pinned
+flag through a stream memop (``wait_reset``) that the host ``signal``s once the fill landed; where
+the driver rejects memops in capture, the caller falls back to launch-gating (fill before replay).
+``probe_wait_sync`` decides which, once, at startup.
+"""
+
+from __future__ import annotations
+
+import torch
+
+from freetoken.kernel import _row_store
+from freetoken.kernel.pinned import alloc_pinned_tensor
+
+RowStore = _row_store.RowStore
+PleStore = _row_store.PleStore
+
+
+def probe_wait_sync(mode: str, device: torch.device) -> bool:
+ """``mode``: ``auto`` (probe), ``wait`` (require memops) or ``gate`` (never use them)."""
+ if mode not in ("auto", "wait", "gate"):
+ raise ValueError(f"unknown row-store sync mode {mode!r}; expected auto, wait or gate")
+ if mode == "gate":
+ return False
+ scratch = alloc_pinned_tensor(1, dtype=torch.int64)
+ scratch.zero_()
+ stream = torch.cuda.current_stream(device)
+ ok = (
+ _row_store.memop_write(stream.cuda_stream, scratch.data_ptr(), 7) == 0
+ and _row_store.memop_wait_geq(stream.cuda_stream, scratch.data_ptr(), 7) == 0
+ )
+ if ok:
+ stream.synchronize()
+ ok = int(scratch[0]) == 7
+ if mode == "wait" and not ok:
+ raise RuntimeError("wait-sync requested but stream memops are unavailable")
+ return ok
+
+
+def wait_reset(stream: torch.cuda.Stream, flag: torch.Tensor) -> None:
+ """Enqueue WAIT(flag >= 1) then flag := 0 on ``stream`` (capturable)."""
+ _row_store.memop_wait_reset(stream.cuda_stream, flag.data_ptr())
+
+
+def signal(flag: torch.Tensor) -> None:
+ """Host side: release a pending WAIT (a release-store of 1)."""
+ _row_store.signal_flag(flag.data_ptr())
+
+
+__all__ = ["RowStore", "PleStore", "probe_wait_sync", "wait_reset", "signal"]
diff --git a/python/freetoken/models/qwen4_exp/ple_disk.py b/python/freetoken/models/qwen4_exp/ple_disk.py
index cea62693..ec7da80d 100644
--- a/python/freetoken/models/qwen4_exp/ple_disk.py
+++ b/python/freetoken/models/qwen4_exp/ple_disk.py
@@ -112,7 +112,7 @@ class DiskRowTable:
max_extend_tokens: int = 8192,
dtype: torch.dtype = torch.bfloat16,
) -> None:
- from freetoken.kernel import _ple_store
+ from freetoken.kernel.row_store import PleStore
self.num_rows = source.total_rows
self.head_dim = source.row_bytes # fp8: one byte per element
@@ -128,7 +128,7 @@ class DiskRowTable:
raise ValueError(
f"PLE row source holds {source.total_rows} rows but the hash addresses {need}; incomplete checkpoint?"
)
- self._store = _ple_store.PleStore(
+ self._store = PleStore(
paths=list(source.paths),
extent_file=list(source.extent_file),
extent_base=list(source.extent_base),
@@ -155,7 +155,9 @@ class DiskRowTable:
self._eager_pinned.zero_() # the warmup prefill stages nothing and reads whatever sits here
self._eager_dev = torch.empty(eager_bytes, dtype=torch.uint8, device=self._device)
# probe picks flag-sync (graph WAITs at the consume, host fills then signals) or launch-gating
- self._wait_sync = self._probe_wait_sync(os.getenv(_SYNC_ENV, "auto"))
+ from freetoken.kernel.row_store import probe_wait_sync
+
+ self._wait_sync = probe_wait_sync(os.getenv(_SYNC_ENV, "auto"), self._device)
# one flag for all graphs: the readback event orders a fill after the previous graph, so signals never overlap
self._flag = alloc_pinned_tensor(1, dtype=torch.int64)
self._flag.zero_()
@@ -164,25 +166,6 @@ class DiskRowTable:
sync = "wait-sync" if self._wait_sync else "launch-gating"
logger.info_rank0(f"PLE disk backend: {self._store.io_backend()}, {sync}")
- def _probe_wait_sync(self, mode: str) -> bool:
- from freetoken.kernel import _ple_store
-
- if mode == "gate":
- return False
- scratch = alloc_pinned_tensor(1, dtype=torch.int64)
- scratch.zero_()
- stream = torch.cuda.current_stream(self._device)
- ok = (
- _ple_store.memop_write(stream.cuda_stream, scratch.data_ptr(), 7) == 0
- and _ple_store.memop_wait_geq(stream.cuda_stream, scratch.data_ptr(), 7) == 0
- )
- if ok:
- stream.synchronize()
- ok = int(scratch[0]) == 7
- if mode == "wait" and not ok:
- raise RuntimeError("FREETOKEN_PLE_SYNC=wait but stream memops are unavailable")
- return ok
-
# ---------------- host side (engine thread, before the forward launches) ----------------
def fill(self, runs: Sequence[torch.Tensor], *, graph: bool) -> None:
@@ -222,10 +205,10 @@ class DiskRowTable:
for r, t in zip(reqs, tokens)]
self.fill(runs, graph=True)
except BaseException:
- from freetoken.kernel import _ple_store
+ from freetoken.kernel.row_store import signal
# unblock the stream before surfacing; the step's output is discarded
- _ple_store.signal_flag(self._flag.data_ptr())
+ signal(self._flag)
raise
return _complete
@@ -260,11 +243,9 @@ class DiskRowTable:
rows = row_ids.shape[0]
capturing = torch.cuda.is_current_stream_capturing()
if capturing and self._wait_sync:
- from freetoken.kernel import _ple_store
+ from freetoken.kernel.row_store import wait_reset
- _ple_store.memop_wait_reset(
- torch.cuda.current_stream(self._device).cuda_stream, self._flag.data_ptr()
- )
+ wait_reset(torch.cuda.current_stream(self._device), self._flag)
pinned, dev = (
(self._graph_pinned, self._graph_dev) if capturing else (self._eager_pinned, self._eager_dev)
)
diff --git a/setup.py b/setup.py
index d8e705ab..7132eea0 100644
--- a/setup.py
+++ b/setup.py
@@ -64,12 +64,12 @@ setup(
libraries=["cudart"],
extra_compile_args=["-O3", "-std=c++17", "-pthread"],
),
- # --ple-backend disk row store; Linux-only until the TableFile/BatchReader seams grow Windows bodies
+ # disk-backed row store (PLE / Engram tables); Linux-only until the TableFile/BatchReader seams grow Windows bodies
*([
CppExtension(
- name="freetoken.kernel._ple_store",
+ name="freetoken.kernel._row_store",
sources=[
- "python/freetoken/kernel/csrc/ple_store/ple_store_ext.cpp",
+ "python/freetoken/kernel/csrc/row_store/row_store_ext.cpp",
],
extra_compile_args=["-O3", "-std=c++17"],
)
diff --git a/tests/kernels/test_row_store.py b/tests/kernels/test_row_store.py
new file mode 100644
index 00000000..d4a19502
--- /dev/null
+++ b/tests/kernels/test_row_store.py
@@ -0,0 +1,87 @@
+"""The hash-agnostic ``RowStore`` (``kernel/csrc/row_store``): byte-exact staged rows across extents and files,
+dedup of repeated ids, a destination stride, and bounds checks before any I/O. The PLE hash wrapper on top of
+it is covered by ``tests/models/qwen4_exp/test_ple_disk.py``."""
+
+from __future__ import annotations
+
+import pytest
+import torch
+
+_row_store = pytest.importorskip("freetoken.kernel._row_store")
+
+ROW_BYTES = 264 # deliberately unaligned rows exercise reads across page boundaries
+
+
+def _table(rows: int, seed: int) -> torch.Tensor:
+ gen = torch.Generator().manual_seed(seed)
+ return torch.randint(0, 256, (rows, ROW_BYTES), dtype=torch.uint8, generator=gen)
+
+
+def _stage(store, ids: torch.Tensor, dst_stride: int = 0) -> torch.Tensor:
+ stride = dst_stride or store.row_bytes
+ staging = torch.zeros(ids.numel() * stride, dtype=torch.uint8)
+ store.stage_rows(ids.data_ptr(), ids.numel(), staging.data_ptr(), dst_stride)
+ store.flush(0)
+ return staging.view(ids.numel(), stride)
+
+
[email protected]("use_io_uring", [True, False])
+def test_rows_come_back_bitwise_across_extents_and_files(tmp_path, use_io_uring):
+ # two files; file 0 carries extents 0 and 2 behind an unrelated header, file 1 carries extent 1
+ per_extent = 1000
+ table = _table(3 * per_extent, seed=1)
+ header = b"safetensors-like header, not a row" + bytes(4096 - 34)
+ f0 = tmp_path / "shard0.bin"
+ f1 = tmp_path / "shard1.bin"
+ f0.write_bytes(header + table[:per_extent].numpy().tobytes() + table[2 * per_extent :].numpy().tobytes())
+ f1.write_bytes(table[per_extent : 2 * per_extent].numpy().tobytes())
+ store = _row_store.RowStore(
+ paths=[str(f0), str(f1)],
+ extent_file=[0, 1, 0],
+ extent_base=[len(header), 0, len(header) + per_extent * ROW_BYTES],
+ rows_per_extent=per_extent,
+ row_bytes=ROW_BYTES,
+ row_stride=ROW_BYTES,
+ use_io_uring=use_io_uring,
+ )
+ assert store.total_rows == 3 * per_extent and store.row_bytes == ROW_BYTES
+
+ ids = torch.tensor([0, 999, 1000, 1999, 2000, 2999, 1234, 1234, 42, 0], dtype=torch.int64)
+ got = _stage(store, ids)
+ assert torch.equal(got, table[ids])
+
+ # a wider destination stride leaves the gap untouched
+ got = _stage(store, ids, dst_stride=ROW_BYTES + 40)
+ assert torch.equal(got[:, :ROW_BYTES], table[ids]) and int(got[:, ROW_BYTES:].sum()) == 0
+
+ # a fill larger than the reader's in-flight window pipelines through
+ many = torch.randint(0, 3 * per_extent, (5000,), dtype=torch.int64, generator=torch.Generator().manual_seed(2))
+ assert torch.equal(_stage(store, many), table[many])
+
+ # flushing nothing still signals the flag
+ flag = torch.zeros(1, dtype=torch.int64)
+ store.flush(flag.data_ptr())
+ assert int(flag[0]) == 1
+
+
+def test_geometry_and_bounds_are_checked_up_front(tmp_path):
+ table = _table(64, seed=3)
+ path = tmp_path / "t.bin"
+ path.write_bytes(table.numpy().tobytes())
+ make = lambda **kw: _row_store.RowStore(
+ paths=[str(path)], extent_file=[0], extent_base=[0], rows_per_extent=64,
+ row_bytes=ROW_BYTES, row_stride=ROW_BYTES, use_io_uring=False, **kw,
+ )
+ store = make()
+ ids = torch.tensor([3, 64], dtype=torch.int64)
+ staging = torch.zeros(2 * ROW_BYTES, dtype=torch.uint8)
+ with pytest.raises(IndexError):
+ store.stage_rows(ids.data_ptr(), 2, staging.data_ptr(), 0)
+ with pytest.raises(RuntimeError, match="overlap"):
+ store.stage_rows(ids[:1].data_ptr(), 1, staging.data_ptr(), ROW_BYTES - 1)
+ with pytest.raises(RuntimeError, match="extent needs"):
+ _row_store.RowStore(paths=[str(path)], extent_file=[0], extent_base=[0], rows_per_extent=65,
+ row_bytes=ROW_BYTES, row_stride=ROW_BYTES, use_io_uring=False)
+ with pytest.raises(RuntimeError, match="exceeds a page"):
+ _row_store.RowStore(paths=[str(path)], extent_file=[0], extent_base=[0], rows_per_extent=1,
+ row_bytes=4097, row_stride=4097, use_io_uring=False)
diff --git a/tests/models/qwen4_exp/test_ple_disk.py b/tests/models/qwen4_exp/test_ple_disk.py
index fa118073..1112210b 100644
--- a/tests/models/qwen4_exp/test_ple_disk.py
+++ b/tests/models/qwen4_exp/test_ple_disk.py
@@ -13,7 +13,7 @@ from freetoken.models.qwen4_exp.ple import GpuResidentTable, NGramEmbedding
from .common import EOS, hash_constants, requires_cuda, toy_hf_config
from .test_ple import _meta
-_ple_store = pytest.importorskip("freetoken.kernel._ple_store")
+_row_store = pytest.importorskip("freetoken.kernel._row_store")
_KEY_PREFIX = "model.layers.1.ple.ple_embedding.ngram_embedding"
@@ -43,7 +43,7 @@ def _make_store(tmp_path, *, write=True, use_io_uring=True):
path = tmp_path / "ple-table.bin"
if write:
path.write_bytes(table.numpy().tobytes())
- store = _ple_store.PleStore(
+ store = _row_store.PleStore(
paths=[str(path)],
extent_file=[0],
extent_base=[0],
@@ -175,10 +175,10 @@ def test_layouts_readers_and_errors(tmp_path):
multipliers=[3, 5, 7], head_vocab_sizes=sizes, head_offsets=offsets,
eos_token_id=eos,
)
- ref = _ple_store.PleStore(
+ ref = _row_store.PleStore(
paths=[str(flat)], extent_file=[0, 0, 0, 0], extent_base=[0, nb, 2 * nb, 3 * nb], **kwargs
)
- multi = _ple_store.PleStore(
+ multi = _row_store.PleStore(
paths=[str(fa), str(fb)], extent_file=[0, 1, 0, 1],
extent_base=[1231, 0, 1231 + nb + 77, nb + 4095], **kwargs,
)