FreeToken brings datacenter-scale model serving to your desktop. Run massive models locally, fast and efficiently.
refactor(ple): extract a shared disk row store (#548)
Xiaoze Fanf5b9700cc103
diff --git a/python/freetoken/kernel/csrc/ple_store/ple_store_ext.cpp b/python/freetoken/kernel/csrc/row_store/row_store_ext.cppsimilarity index 83%rename from python/freetoken/kernel/csrc/ple_store/ple_store_ext.cpprename to python/freetoken/kernel/csrc/row_store/row_store_ext.cppindex 2d54d692..e33b18a8 100644--- a/python/freetoken/kernel/csrc/ple_store/ple_store_ext.cpp+++ b/python/freetoken/kernel/csrc/row_store/row_store_ext.cpp@@ -1,7 +1,12 @@-// Disk-backed PLE row store: rows read straight from the checkpoint's fp8 shard tensors-// through an extent table (PleRowSource in ple_ssd.py). Engine-thread only, no locks.-// Duplicate rows in one fill dedup into ONE batched read round; no RAM cache and no-// per-sequence state. Hash reference: tests/models/qwen4_exp/test_ple_disk.py.+// Disk-backed row store: fixed-width rows read straight from checkpoint shard tensors+// through an extent table (row i of extent e at extent_base[e] + i * row_stride).+// Engine-thread only, no locks. Duplicate rows in one fill dedup into ONE batched read+// round; no RAM cache and no per-sequence state.+//+// RowStore -- hash-agnostic: stage_rows(row_ids) + flush(); callers supply row ids.+// PleStore -- RowStore plus the Qwen3.8-Flash-Next PLE n-gram hash: stage(tokens).+// Hash reference: tests/models/qwen4_exp/test_ple_disk.py.+//// Platform seams: TableFile (O_DIRECT+pread; Win: NO_BUFFERING), BatchReader (io_uring,// pread-pool fallback = the portable shape), cumemop_* (dlopen libcuda; Win: nvcuda).@@ -183,7 +188,9 @@ class ThreadPoolBatchReader final : public BatchReader {ThreadPoolBatchReader() {// these threads block on I/O, not compute, so the core count is only a defaultunsigned n = std::max(1u, std::min(kReaderThreads, std::thread::hardware_concurrency()));- if (const char *env = std::getenv("FREETOKEN_PLE_READER_THREADS")) {+ const char *env = std::getenv("FREETOKEN_ROW_STORE_READER_THREADS");+ if (env == nullptr) env = std::getenv("FREETOKEN_PLE_READER_THREADS"); // the PLE-era name+ if (env != nullptr) {// kBatchEntries is the submit depth, so threads past it never get a readconst int v = std::atoi(env);if (v > 0) n = std::min((unsigned)v, kBatchEntries);@@ -432,29 +439,24 @@ int64_t pos_mod(int64_t v, int64_t m) {return r < 0 ? r + m : r;}-class PleStore {+class RowStore {struct Extent {const TableFile *file;int64_t base;};public:- PleStore(std::vector<std::string> paths, std::vector<int64_t> extent_file,+ RowStore(std::vector<std::string> paths, std::vector<int64_t> extent_file,std::vector<int64_t> extent_base, int64_t rows_per_extent, int64_t row_bytes,- int64_t row_stride, std::vector<int64_t> multipliers, std::vector<int64_t> head_vocab_sizes,- std::vector<int64_t> head_offsets, int64_t eos_token_id, bool use_io_uring)- : row_bytes_(row_bytes),- row_stride_(row_stride),- rows_per_extent_(rows_per_extent),- mult_(std::move(multipliers)),- sizes_(std::move(head_vocab_sizes)),- offsets_(std::move(head_offsets)),- eos_(eos_token_id) {- if (mult_.size() != 3 || sizes_.size() != offsets_.size() || sizes_.empty())- throw std::runtime_error("PLE hash geometry: want 3 multipliers and equal-length head tables");+ int64_t row_stride, bool use_io_uring)+ : row_bytes_(row_bytes), row_stride_(row_stride), rows_per_extent_(rows_per_extent) {+ if (row_bytes_ <= 0 || row_stride_ < row_bytes_ || rows_per_extent_ <= 0)+ throw std::runtime_error("row store geometry: want row_bytes > 0, row_stride >= row_bytes, rows_per_extent > 0");if (row_bytes_ > kPage)- throw std::runtime_error("PLE row_bytes " + std::to_string(row_bytes_) ++ throw std::runtime_error("row_bytes " + std::to_string(row_bytes_) +" exceeds a page; bounce slots assume one-page rows");+ if (extent_file.size() != extent_base.size() || extent_file.empty())+ throw std::runtime_error("row store geometry: extent_file and extent_base must be equal-length and non-empty");for (const std::string &p : paths)files_.push_back(std::make_unique<TableFile>(p));const int64_t extent_bytes = (rows_per_extent_ - 1) * row_stride_ + row_bytes_;@@ -472,36 +474,33 @@ class PleStore {}// reader first: a still-running read must not land in freed bounce memory- ~PleStore() {+ virtual ~RowStore() {reader_.reset();free(bounce_);}- PleStore(const PleStore &) = delete;- PleStore &operator=(const PleStore &) = delete;+ RowStore(const RowStore &) = delete;+ RowStore &operator=(const RowStore &) = delete;- // Row ids for the token at w[2] with context (w[0], w[1]); mirrors NGramEmbedding.row_ids incl. the eos barrier.- void hash_rows(const int64_t *w, int64_t *rows) {- const int64_t prev1 = w[1];- const int64_t prev2 = prev1 == eos_ ? eos_ : w[0];- const int64_t bigram = wrap_mul(w[2], mult_[0]) ^ wrap_mul(prev1, mult_[1]);- const int64_t trigram = bigram ^ wrap_mul(prev2, mult_[2]);- const size_t half = sizes_.size() / 2;- for (size_t h = 0; h < sizes_.size(); h++)- rows[h] = pos_mod(h < half ? bigram : trigram, sizes_[h]) + offsets_[h];- }+ int64_t row_bytes() const { return row_bytes_; }+ int64_t total_rows() const { return (int64_t)extents_.size() * rows_per_extent_; }- // Hash and queue one run: tokens_addr holds n+2 ids, the leading two are context. No I/O until flush().- void stage(uintptr_t tokens_addr, int64_t n, uintptr_t staging_addr) {- const int64_t *tokens = reinterpret_cast<const int64_t *>(tokens_addr);+ // Queue n row ids (int64 at rows_addr); row i lands at staging_addr + i * dst_stride+ // (dst_stride = 0 packs rows back to back). Out-of-range ids throw before any I/O is+ // issued. No I/O until flush().+ void stage_rows(uintptr_t rows_addr, int64_t n, uintptr_t staging_addr, int64_t dst_stride) {+ const int64_t *rows = reinterpret_cast<const int64_t *>(rows_addr);uint8_t *staging = reinterpret_cast<uint8_t *>(staging_addr);- const size_t heads = sizes_.size();- std::vector<int64_t> rows(heads);+ if (dst_stride == 0) dst_stride = row_bytes_;+ if (dst_stride < row_bytes_)+ throw std::runtime_error("stage_rows: dst_stride below row_bytes would overlap rows");+ const int64_t total = total_rows();for (int64_t i = 0; i < n; i++) {- hash_rows(tokens + i, rows.data());- for (size_t h = 0; h < heads; h++)- request_row(rows[h], staging + ((size_t)i * heads + h) * row_bytes_);+ if (rows[i] < 0 || rows[i] >= total)+ throw std::out_of_range("stage_rows: row id " + std::to_string(rows[i]) + " outside [0, " ++ std::to_string(total) + ")");}+ for (int64_t i = 0; i < n; i++) request_row(rows[i], staging + (size_t)i * (size_t)dst_stride);}// One batched disk round for everything staged; signals even when nothing was.@@ -519,15 +518,7 @@ class PleStore {std::to_string(files_.size()) + " files";}- private:- struct Pending {- const TableFile *file;- int64_t read_off;- int64_t read_len;- int64_t row_off; // row payload start inside the read buffer- std::vector<uint8_t *> dsts;- };-+ protected:// Queue dst on this fill's pending batch; duplicate rows fan out from one read.void request_row(int64_t row_id, uint8_t *dst) {auto pit = pending_index_.find(row_id);@@ -548,11 +539,20 @@ class PleStore {pending_.push_back(std::move(p));}+ private:+ struct Pending {+ const TableFile *file;+ int64_t read_off;+ int64_t read_len;+ int64_t row_off; // row payload start inside the read buffer+ std::vector<uint8_t *> dsts;+ };+// Read every pending row in reader-capacity batches and fan out the copies.void flush_pending() {if (pending_.empty()) return;struct Cleanup {- PleStore *s;+ RowStore *s;~Cleanup() {s->pending_.clear();s->pending_index_.clear();@@ -586,8 +586,6 @@ class PleStore {}int64_t row_bytes_, row_stride_, rows_per_extent_;- std::vector<int64_t> mult_, sizes_, offsets_;- int64_t eos_;std::vector<std::unique_ptr<TableFile>> files_;std::vector<Extent> extents_;std::unique_ptr<BatchReader> reader_;@@ -597,10 +595,71 @@ class PleStore {std::unordered_map<int64_t, size_t> pending_index_;};+// The Qwen3.8-Flash-Next PLE table: RowStore plus the checkpoint's n-gram hash, so a fill+// takes raw token runs and never crosses the Python boundary per row.+class PleStore final : public RowStore {+ public:+ PleStore(std::vector<std::string> paths, std::vector<int64_t> extent_file,+ std::vector<int64_t> extent_base, int64_t rows_per_extent, int64_t row_bytes,+ int64_t row_stride, std::vector<int64_t> multipliers, std::vector<int64_t> head_vocab_sizes,+ std::vector<int64_t> head_offsets, int64_t eos_token_id, bool use_io_uring)+ : RowStore(std::move(paths), std::move(extent_file), std::move(extent_base), rows_per_extent,+ row_bytes, row_stride, use_io_uring),+ mult_(std::move(multipliers)),+ sizes_(std::move(head_vocab_sizes)),+ offsets_(std::move(head_offsets)),+ eos_(eos_token_id) {+ if (mult_.size() != 3 || sizes_.size() != offsets_.size() || sizes_.empty())+ throw std::runtime_error("PLE hash geometry: want 3 multipliers and equal-length head tables");+ }++ // Row ids for the token at w[2] with context (w[0], w[1]); mirrors NGramEmbedding.row_ids incl. the eos barrier.+ void hash_rows(const int64_t *w, int64_t *rows) {+ const int64_t prev1 = w[1];+ const int64_t prev2 = prev1 == eos_ ? eos_ : w[0];+ const int64_t bigram = wrap_mul(w[2], mult_[0]) ^ wrap_mul(prev1, mult_[1]);+ const int64_t trigram = bigram ^ wrap_mul(prev2, mult_[2]);+ const size_t half = sizes_.size() / 2;+ for (size_t h = 0; h < sizes_.size(); h++)+ rows[h] = pos_mod(h < half ? bigram : trigram, sizes_[h]) + offsets_[h];+ }++ // Hash and queue one run: tokens_addr holds n+2 ids, the leading two are context. No I/O until flush().+ void stage(uintptr_t tokens_addr, int64_t n, uintptr_t staging_addr) {+ const int64_t *tokens = reinterpret_cast<const int64_t *>(tokens_addr);+ uint8_t *staging = reinterpret_cast<uint8_t *>(staging_addr);+ const size_t heads = sizes_.size();+ std::vector<int64_t> rows(heads);+ for (int64_t i = 0; i < n; i++) {+ hash_rows(tokens + i, rows.data());+ for (size_t h = 0; h < heads; h++)+ request_row(rows[h], staging + ((size_t)i * heads + h) * (size_t)row_bytes());+ }+ }++ private:+ std::vector<int64_t> mult_, sizes_, offsets_;+ int64_t eos_;+};+} // namespacePYBIND11_MODULE(TORCH_EXTENSION_NAME, m) {- py::class_<PleStore>(m, "PleStore")+ py::class_<RowStore>(m, "RowStore")+ .def(py::init<std::vector<std::string>, std::vector<int64_t>, std::vector<int64_t>,+ int64_t, int64_t, int64_t, bool>(),+ py::arg("paths"), py::arg("extent_file"), py::arg("extent_base"),+ py::arg("rows_per_extent"), py::arg("row_bytes"), py::arg("row_stride"),+ py::arg("use_io_uring") = true)+ .def("stage_rows", &RowStore::stage_rows, py::arg("rows_addr"), py::arg("n"),+ py::arg("staging_addr"), py::arg("dst_stride") = 0,+ py::call_guard<py::gil_scoped_release>())+ .def("flush", &RowStore::flush, py::arg("signal_addr") = 0,+ py::call_guard<py::gil_scoped_release>())+ .def("io_backend", &RowStore::io_backend)+ .def_property_readonly("row_bytes", &RowStore::row_bytes)+ .def_property_readonly("total_rows", &RowStore::total_rows);+ py::class_<PleStore, RowStore>(m, "PleStore").def(py::init<std::vector<std::string>, std::vector<int64_t>, std::vector<int64_t>,int64_t, int64_t, int64_t, std::vector<int64_t>,std::vector<int64_t>, std::vector<int64_t>, int64_t, bool>(),@@ -610,10 +669,7 @@ PYBIND11_MODULE(TORCH_EXTENSION_NAME, m) {py::arg("head_vocab_sizes"), py::arg("head_offsets"), py::arg("eos_token_id"),py::arg("use_io_uring") = true).def("stage", &PleStore::stage, py::arg("tokens_addr"), py::arg("n"),- py::arg("staging_addr"), py::call_guard<py::gil_scoped_release>())- .def("flush", &PleStore::flush, py::arg("signal_addr") = 0,- py::call_guard<py::gil_scoped_release>())- .def("io_backend", &PleStore::io_backend);+ py::arg("staging_addr"), py::call_guard<py::gil_scoped_release>());m.def("memop_write", &memop_write, py::arg("stream"), py::arg("addr"), py::arg("value"));m.def("memop_wait_geq", &memop_wait_geq, py::arg("stream"), py::arg("addr"), py::arg("value"));m.def("memop_wait_reset", &memop_wait_reset, py::arg("stream"), py::arg("flag_addr"));diff --git a/python/freetoken/kernel/row_store.py b/python/freetoken/kernel/row_store.pynew file mode 100644index 00000000..5b42479e--- /dev/null+++ b/python/freetoken/kernel/row_store.py@@ -0,0 +1,54 @@+"""Python face of the disk row store extension (``kernel/csrc/row_store``) and its CUDA-graph sync.++``RowStore`` reads fixed-width rows straight out of checkpoint shard files (io_uring or a pread pool,+O_DIRECT) into pinned staging by row id; ``PleStore`` adds PLE's n-gram hashing.++Graph sync: a captured graph cannot wait for host I/O, so a captured ``lookup`` WAITs on a pinned+flag through a stream memop (``wait_reset``) that the host ``signal``s once the fill landed; where+the driver rejects memops in capture, the caller falls back to launch-gating (fill before replay).+``probe_wait_sync`` decides which, once, at startup.+"""++from __future__ import annotations++import torch++from freetoken.kernel import _row_store+from freetoken.kernel.pinned import alloc_pinned_tensor++RowStore = _row_store.RowStore+PleStore = _row_store.PleStore+++def probe_wait_sync(mode: str, device: torch.device) -> bool:+ """``mode``: ``auto`` (probe), ``wait`` (require memops) or ``gate`` (never use them)."""+ if mode not in ("auto", "wait", "gate"):+ raise ValueError(f"unknown row-store sync mode {mode!r}; expected auto, wait or gate")+ if mode == "gate":+ return False+ scratch = alloc_pinned_tensor(1, dtype=torch.int64)+ scratch.zero_()+ stream = torch.cuda.current_stream(device)+ ok = (+ _row_store.memop_write(stream.cuda_stream, scratch.data_ptr(), 7) == 0+ and _row_store.memop_wait_geq(stream.cuda_stream, scratch.data_ptr(), 7) == 0+ )+ if ok:+ stream.synchronize()+ ok = int(scratch[0]) == 7+ if mode == "wait" and not ok:+ raise RuntimeError("wait-sync requested but stream memops are unavailable")+ return ok+++def wait_reset(stream: torch.cuda.Stream, flag: torch.Tensor) -> None:+ """Enqueue WAIT(flag >= 1) then flag := 0 on ``stream`` (capturable)."""+ _row_store.memop_wait_reset(stream.cuda_stream, flag.data_ptr())+++def signal(flag: torch.Tensor) -> None:+ """Host side: release a pending WAIT (a release-store of 1)."""+ _row_store.signal_flag(flag.data_ptr())+++__all__ = ["RowStore", "PleStore", "probe_wait_sync", "wait_reset", "signal"]diff --git a/python/freetoken/models/qwen4_exp/ple_disk.py b/python/freetoken/models/qwen4_exp/ple_disk.pyindex cea62693..ec7da80d 100644--- a/python/freetoken/models/qwen4_exp/ple_disk.py+++ b/python/freetoken/models/qwen4_exp/ple_disk.py@@ -112,7 +112,7 @@ class DiskRowTable:max_extend_tokens: int = 8192,dtype: torch.dtype = torch.bfloat16,) -> None:- from freetoken.kernel import _ple_store+ from freetoken.kernel.row_store import PleStoreself.num_rows = source.total_rowsself.head_dim = source.row_bytes # fp8: one byte per element@@ -128,7 +128,7 @@ class DiskRowTable:raise ValueError(f"PLE row source holds {source.total_rows} rows but the hash addresses {need}; incomplete checkpoint?")- self._store = _ple_store.PleStore(+ self._store = PleStore(paths=list(source.paths),extent_file=list(source.extent_file),extent_base=list(source.extent_base),@@ -155,7 +155,9 @@ class DiskRowTable:self._eager_pinned.zero_() # the warmup prefill stages nothing and reads whatever sits hereself._eager_dev = torch.empty(eager_bytes, dtype=torch.uint8, device=self._device)# probe picks flag-sync (graph WAITs at the consume, host fills then signals) or launch-gating- self._wait_sync = self._probe_wait_sync(os.getenv(_SYNC_ENV, "auto"))+ from freetoken.kernel.row_store import probe_wait_sync++ self._wait_sync = probe_wait_sync(os.getenv(_SYNC_ENV, "auto"), self._device)# one flag for all graphs: the readback event orders a fill after the previous graph, so signals never overlapself._flag = alloc_pinned_tensor(1, dtype=torch.int64)self._flag.zero_()@@ -164,25 +166,6 @@ class DiskRowTable:sync = "wait-sync" if self._wait_sync else "launch-gating"logger.info_rank0(f"PLE disk backend: {self._store.io_backend()}, {sync}")- def _probe_wait_sync(self, mode: str) -> bool:- from freetoken.kernel import _ple_store-- if mode == "gate":- return False- scratch = alloc_pinned_tensor(1, dtype=torch.int64)- scratch.zero_()- stream = torch.cuda.current_stream(self._device)- ok = (- _ple_store.memop_write(stream.cuda_stream, scratch.data_ptr(), 7) == 0- and _ple_store.memop_wait_geq(stream.cuda_stream, scratch.data_ptr(), 7) == 0- )- if ok:- stream.synchronize()- ok = int(scratch[0]) == 7- if mode == "wait" and not ok:- raise RuntimeError("FREETOKEN_PLE_SYNC=wait but stream memops are unavailable")- return ok-# ---------------- host side (engine thread, before the forward launches) ----------------def fill(self, runs: Sequence[torch.Tensor], *, graph: bool) -> None:@@ -222,10 +205,10 @@ class DiskRowTable:for r, t in zip(reqs, tokens)]self.fill(runs, graph=True)except BaseException:- from freetoken.kernel import _ple_store+ from freetoken.kernel.row_store import signal# unblock the stream before surfacing; the step's output is discarded- _ple_store.signal_flag(self._flag.data_ptr())+ signal(self._flag)raisereturn _complete@@ -260,11 +243,9 @@ class DiskRowTable:rows = row_ids.shape[0]capturing = torch.cuda.is_current_stream_capturing()if capturing and self._wait_sync:- from freetoken.kernel import _ple_store+ from freetoken.kernel.row_store import wait_reset- _ple_store.memop_wait_reset(- torch.cuda.current_stream(self._device).cuda_stream, self._flag.data_ptr()- )+ wait_reset(torch.cuda.current_stream(self._device), self._flag)pinned, dev = ((self._graph_pinned, self._graph_dev) if capturing else (self._eager_pinned, self._eager_dev))diff --git a/setup.py b/setup.pyindex d8e705ab..7132eea0 100644--- a/setup.py+++ b/setup.py@@ -64,12 +64,12 @@ setup(libraries=["cudart"],extra_compile_args=["-O3", "-std=c++17", "-pthread"],),- # --ple-backend disk row store; Linux-only until the TableFile/BatchReader seams grow Windows bodies+ # disk-backed row store (PLE / Engram tables); Linux-only until the TableFile/BatchReader seams grow Windows bodies*([CppExtension(- name="freetoken.kernel._ple_store",+ name="freetoken.kernel._row_store",sources=[- "python/freetoken/kernel/csrc/ple_store/ple_store_ext.cpp",+ "python/freetoken/kernel/csrc/row_store/row_store_ext.cpp",],extra_compile_args=["-O3", "-std=c++17"],)diff --git a/tests/kernels/test_row_store.py b/tests/kernels/test_row_store.pynew file mode 100644index 00000000..d4a19502--- /dev/null+++ b/tests/kernels/test_row_store.py@@ -0,0 +1,87 @@+"""The hash-agnostic ``RowStore`` (``kernel/csrc/row_store``): byte-exact staged rows across extents and files,+dedup of repeated ids, a destination stride, and bounds checks before any I/O. The PLE hash wrapper on top of+it is covered by ``tests/models/qwen4_exp/test_ple_disk.py``."""++from __future__ import annotations++import pytest+import torch++_row_store = pytest.importorskip("freetoken.kernel._row_store")++ROW_BYTES = 264 # deliberately unaligned rows exercise reads across page boundaries+++def _table(rows: int, seed: int) -> torch.Tensor:+ gen = torch.Generator().manual_seed(seed)+ return torch.randint(0, 256, (rows, ROW_BYTES), dtype=torch.uint8, generator=gen)+++def _stage(store, ids: torch.Tensor, dst_stride: int = 0) -> torch.Tensor:+ stride = dst_stride or store.row_bytes+ staging = torch.zeros(ids.numel() * stride, dtype=torch.uint8)+ store.stage_rows(ids.data_ptr(), ids.numel(), staging.data_ptr(), dst_stride)+ store.flush(0)+ return staging.view(ids.numel(), stride)++[email protected]("use_io_uring", [True, False])+def test_rows_come_back_bitwise_across_extents_and_files(tmp_path, use_io_uring):+ # two files; file 0 carries extents 0 and 2 behind an unrelated header, file 1 carries extent 1+ per_extent = 1000+ table = _table(3 * per_extent, seed=1)+ header = b"safetensors-like header, not a row" + bytes(4096 - 34)+ f0 = tmp_path / "shard0.bin"+ f1 = tmp_path / "shard1.bin"+ f0.write_bytes(header + table[:per_extent].numpy().tobytes() + table[2 * per_extent :].numpy().tobytes())+ f1.write_bytes(table[per_extent : 2 * per_extent].numpy().tobytes())+ store = _row_store.RowStore(+ paths=[str(f0), str(f1)],+ extent_file=[0, 1, 0],+ extent_base=[len(header), 0, len(header) + per_extent * ROW_BYTES],+ rows_per_extent=per_extent,+ row_bytes=ROW_BYTES,+ row_stride=ROW_BYTES,+ use_io_uring=use_io_uring,+ )+ assert store.total_rows == 3 * per_extent and store.row_bytes == ROW_BYTES++ ids = torch.tensor([0, 999, 1000, 1999, 2000, 2999, 1234, 1234, 42, 0], dtype=torch.int64)+ got = _stage(store, ids)+ assert torch.equal(got, table[ids])++ # a wider destination stride leaves the gap untouched+ got = _stage(store, ids, dst_stride=ROW_BYTES + 40)+ assert torch.equal(got[:, :ROW_BYTES], table[ids]) and int(got[:, ROW_BYTES:].sum()) == 0++ # a fill larger than the reader's in-flight window pipelines through+ many = torch.randint(0, 3 * per_extent, (5000,), dtype=torch.int64, generator=torch.Generator().manual_seed(2))+ assert torch.equal(_stage(store, many), table[many])++ # flushing nothing still signals the flag+ flag = torch.zeros(1, dtype=torch.int64)+ store.flush(flag.data_ptr())+ assert int(flag[0]) == 1+++def test_geometry_and_bounds_are_checked_up_front(tmp_path):+ table = _table(64, seed=3)+ path = tmp_path / "t.bin"+ path.write_bytes(table.numpy().tobytes())+ make = lambda **kw: _row_store.RowStore(+ paths=[str(path)], extent_file=[0], extent_base=[0], rows_per_extent=64,+ row_bytes=ROW_BYTES, row_stride=ROW_BYTES, use_io_uring=False, **kw,+ )+ store = make()+ ids = torch.tensor([3, 64], dtype=torch.int64)+ staging = torch.zeros(2 * ROW_BYTES, dtype=torch.uint8)+ with pytest.raises(IndexError):+ store.stage_rows(ids.data_ptr(), 2, staging.data_ptr(), 0)+ with pytest.raises(RuntimeError, match="overlap"):+ store.stage_rows(ids[:1].data_ptr(), 1, staging.data_ptr(), ROW_BYTES - 1)+ with pytest.raises(RuntimeError, match="extent needs"):+ _row_store.RowStore(paths=[str(path)], extent_file=[0], extent_base=[0], rows_per_extent=65,+ row_bytes=ROW_BYTES, row_stride=ROW_BYTES, use_io_uring=False)+ with pytest.raises(RuntimeError, match="exceeds a page"):+ _row_store.RowStore(paths=[str(path)], extent_file=[0], extent_base=[0], rows_per_extent=1,+ row_bytes=4097, row_stride=4097, use_io_uring=False)diff --git a/tests/models/qwen4_exp/test_ple_disk.py b/tests/models/qwen4_exp/test_ple_disk.pyindex fa118073..1112210b 100644--- a/tests/models/qwen4_exp/test_ple_disk.py+++ b/tests/models/qwen4_exp/test_ple_disk.py@@ -13,7 +13,7 @@ from freetoken.models.qwen4_exp.ple import GpuResidentTable, NGramEmbeddingfrom .common import EOS, hash_constants, requires_cuda, toy_hf_configfrom .test_ple import _meta-_ple_store = pytest.importorskip("freetoken.kernel._ple_store")+_row_store = pytest.importorskip("freetoken.kernel._row_store")_KEY_PREFIX = "model.layers.1.ple.ple_embedding.ngram_embedding"@@ -43,7 +43,7 @@ def _make_store(tmp_path, *, write=True, use_io_uring=True):path = tmp_path / "ple-table.bin"if write:path.write_bytes(table.numpy().tobytes())- store = _ple_store.PleStore(+ store = _row_store.PleStore(paths=[str(path)],extent_file=[0],extent_base=[0],@@ -175,10 +175,10 @@ def test_layouts_readers_and_errors(tmp_path):multipliers=[3, 5, 7], head_vocab_sizes=sizes, head_offsets=offsets,eos_token_id=eos,)- ref = _ple_store.PleStore(+ ref = _row_store.PleStore(paths=[str(flat)], extent_file=[0, 0, 0, 0], extent_base=[0, nb, 2 * nb, 3 * nb], **kwargs)- multi = _ple_store.PleStore(+ multi = _row_store.PleStore(paths=[str(fa), str(fb)], extent_file=[0, 1, 0, 1],extent_base=[1231, 0, 1231 + nb + 77, nb + 4095], **kwargs,)