FreeToken brings datacenter-scale model serving to your desktop. Run massive models locally, fast and efficiently.
fix(ple): add rocm disk wait-sync support (#776)
LZ_QAQ32056cf4608b
diff --git a/python/freetoken/kernel/csrc/row_store/row_store_ext.cpp b/python/freetoken/kernel/csrc/row_store/row_store_ext.cppindex e33b18a8..109e9bdd 100644--- a/python/freetoken/kernel/csrc/row_store/row_store_ext.cpp+++ b/python/freetoken/kernel/csrc/row_store/row_store_ext.cpp@@ -8,10 +8,11 @@// Hash reference: tests/models/qwen4_exp/test_ple_disk.py.//// Platform seams: TableFile (O_DIRECT+pread; Win: NO_BUFFERING), BatchReader (io_uring,-// pread-pool fallback = the portable shape), cumemop_* (dlopen libcuda; Win: nvcuda).+// pread-pool fallback = the portable shape), stream memops (CUDA driver or HIP signal memory).#include <algorithm>#include <cerrno>+#include <chrono>#include <condition_variable>#include <cstdint>#include <cstdlib>@@ -41,6 +42,10 @@#include <torch/extension.h>+#if defined(__HIP_PLATFORM_AMD__) || defined(USE_ROCM)+#include <hip/hip_runtime.h>+#endif+namespace py = pybind11;namespace {@@ -63,6 +68,109 @@ uint8_t *page_aligned_alloc(size_t bytes) {return static_cast<uint8_t *>(p);}+#if defined(__HIP_PLATFORM_AMD__) || defined(USE_ROCM)++hipError_t alloc_signal(int64_t **out) {+ void *word = nullptr;+ // Like CPU MoE, HIP stream waits require signal memory, not a pinned staging allocation.+ const auto rc = hipExtMallocWithFlags(&word, sizeof(int64_t), hipMallocSignalMemory);+ if (rc == hipSuccess) {+ *out = static_cast<int64_t *>(word);+ release_store_i64(*out, 0);+ }+ return rc;+}++torch::Tensor alloc_signal_tensor() {+ int64_t *word = nullptr;+ const auto rc = alloc_signal(&word);+ TORCH_CHECK(rc == hipSuccess, "HIP signal allocation failed: ", hipGetErrorString(rc));+ return torch::from_blob(word, {1}, [](void *p) { (void)hipFree(p); },+ torch::TensorOptions().dtype(torch::kInt64).device(torch::kCPU));+}++int memop_write(uintptr_t stream, uintptr_t addr, uint64_t value) {+ return hipStreamWriteValue64(reinterpret_cast<hipStream_t>(stream),+ reinterpret_cast<void *>(addr), value, hipStreamWriteValueDefault);+}++int memop_wait_geq(uintptr_t stream, uintptr_t addr, uint64_t value) {+ return hipStreamWaitValue64(reinterpret_cast<hipStream_t>(stream),+ reinterpret_cast<void *>(addr), value, hipStreamWaitValueGte, UINT64_MAX);+}++// Capability failures must not poison gating fallback or hide a caller's earlier error.+struct ProbeErrorScope {+ ProbeErrorScope() {+ TORCH_CHECK(hipPeekAtLastError() == hipSuccess, "HIP error pending before row-store sync probe");+ }+ ~ProbeErrorScope() { (void)hipGetLastError(); }+};++bool capture_memops_supported() {+ ProbeErrorScope errors;+ int device = 0;+ int can_wait = 0;+ if (hipGetDevice(&device) != hipSuccess ||+ hipDeviceGetAttribute(&can_wait, hipDeviceAttributeCanUseStreamWaitValue, device) != hipSuccess || can_wait == 0)+ return false;++ int64_t *ready = nullptr;+ int64_t *done = nullptr;+ hipStream_t stream = nullptr;+ hipGraph_t graph = nullptr;+ hipGraphExec_t exec = nullptr;+ auto rc = alloc_signal(&ready);+ if (rc == hipSuccess) rc = alloc_signal(&done);+ if (rc == hipSuccess) rc = hipStreamCreateWithFlags(&stream, hipStreamNonBlocking);+ // A runtime that executes the WAIT during capture must not block the probe.+ if (rc == hipSuccess) release_store_i64(done, 1);+ if (rc == hipSuccess) rc = hipStreamBeginCapture(stream, hipStreamCaptureModeRelaxed);+ const bool started = rc == hipSuccess;+ if (rc == hipSuccess) rc = static_cast<hipError_t>(memop_write(+ reinterpret_cast<uintptr_t>(stream), reinterpret_cast<uintptr_t>(ready), 1));+ if (rc == hipSuccess) rc = static_cast<hipError_t>(memop_wait_geq(+ reinterpret_cast<uintptr_t>(stream), reinterpret_cast<uintptr_t>(done), 1));+ if (started) {+ const auto end_rc = hipStreamEndCapture(stream, &graph);+ if (rc == hipSuccess) rc = end_rc;+ }+ if (rc == hipSuccess) rc = hipStreamSynchronize(stream);+ size_t nodes = 0;+ if (rc == hipSuccess) rc = hipGraphGetNodes(graph, nullptr, &nodes);+ // Older HIP runtimes execute memops during capture without recording them.+ if (rc == hipSuccess && nodes != 2) rc = hipErrorNotSupported;+ if (rc == hipSuccess) rc = hipGraphInstantiate(&exec, graph, nullptr, nullptr, 0);+ bool served = false;+ if (rc == hipSuccess) {+ release_store_i64(ready, 0);+ release_store_i64(done, 0);+ std::thread responder([&] {+ const auto deadline = std::chrono::steady_clock::now() + std::chrono::seconds(2);+ while (__atomic_load_n(ready, __ATOMIC_ACQUIRE) == 0 && std::chrono::steady_clock::now() < deadline)+ std::this_thread::yield();+ served = __atomic_load_n(ready, __ATOMIC_ACQUIRE) == 1;+ // The graph never resets done, so even a launch arriving after the deadline is released.+ release_store_i64(done, 1);+ });+ rc = hipGraphLaunch(exec, stream);+ if (rc == hipSuccess) rc = hipStreamSynchronize(stream);+ responder.join();+ }+ const bool ok = rc == hipSuccess && served;+ if (exec != nullptr) (void)hipGraphExecDestroy(exec);+ if (graph != nullptr) (void)hipGraphDestroy(graph);+ if (stream != nullptr) {+ (void)hipStreamSynchronize(stream);+ (void)hipStreamDestroy(stream);+ }+ if (ready != nullptr) (void)hipFree(ready);+ if (done != nullptr) (void)hipFree(done);+ return ok;+}++#else+// Stream memops for the flag-sync fast path, resolved from the driver at runtime.using CuMemOp64Fn = int (*)(void *stream, unsigned long long addr, unsigned long long value,unsigned int flags);@@ -97,6 +205,8 @@ int memop_wait_geq(uintptr_t stream, uintptr_t addr, uint64_t value) {return g_cu_wait64(reinterpret_cast<void *>(stream), addr, value, kCuWaitValueGeq);}+#endif+// WAIT(>=1) then RESET: resetting first would race a fast host signal and deadlock the stream.void memop_wait_reset(uintptr_t stream, uintptr_t flag_addr) {if (memop_wait_geq(stream, flag_addr, 1) != 0 || memop_write(stream, flag_addr, 0) != 0)@@ -674,4 +784,8 @@ PYBIND11_MODULE(TORCH_EXTENSION_NAME, m) {m.def("memop_wait_geq", &memop_wait_geq, py::arg("stream"), py::arg("addr"), py::arg("value"));m.def("memop_wait_reset", &memop_wait_reset, py::arg("stream"), py::arg("flag_addr"));m.def("signal_flag", &signal_flag, py::arg("flag_addr"));+#if defined(__HIP_PLATFORM_AMD__) || defined(USE_ROCM)+ m.def("alloc_signal_tensor", &alloc_signal_tensor);+ m.def("capture_memops_supported", &capture_memops_supported);+#endif}diff --git a/python/freetoken/kernel/row_store.py b/python/freetoken/kernel/row_store.pyindex 5b42479e..09c74baa 100644--- a/python/freetoken/kernel/row_store.py+++ b/python/freetoken/kernel/row_store.py@@ -3,9 +3,10 @@``RowStore`` reads fixed-width rows straight out of checkpoint shard files (io_uring or a pread pool,O_DIRECT) into pinned staging by row id; ``PleStore`` adds PLE's n-gram hashing.-Graph sync: a captured graph cannot wait for host I/O, so a captured ``lookup`` WAITs on a pinned-flag through a stream memop (``wait_reset``) that the host ``signal``s once the fill landed; where-the driver rejects memops in capture, the caller falls back to launch-gating (fill before replay).+Graph sync: a captured graph cannot wait for host I/O, so a captured ``lookup`` WAITs on a host-visible+flag (pinned on CUDA, signal memory on HIP) through a stream memop (``wait_reset``). The host+``signal``s once the fill landed; where the driver rejects memops in capture, the caller falls back+to launch-gating (fill before replay).``probe_wait_sync`` decides which, once, at startup."""@@ -20,22 +21,32 @@ RowStore = _row_store.RowStorePleStore = _row_store.PleStore+def alloc_sync_flag() -> torch.Tensor:+ if torch.version.hip is not None:+ return _row_store.alloc_signal_tensor()+ return alloc_pinned_tensor(1, dtype=torch.int64).zero_()++def probe_wait_sync(mode: str, device: torch.device) -> bool:"""``mode``: ``auto`` (probe), ``wait`` (require memops) or ``gate`` (never use them)."""if mode not in ("auto", "wait", "gate"):raise ValueError(f"unknown row-store sync mode {mode!r}; expected auto, wait or gate")if mode == "gate":return False- scratch = alloc_pinned_tensor(1, dtype=torch.int64)- scratch.zero_()- stream = torch.cuda.current_stream(device)- ok = (- _row_store.memop_write(stream.cuda_stream, scratch.data_ptr(), 7) == 0- and _row_store.memop_wait_geq(stream.cuda_stream, scratch.data_ptr(), 7) == 0- )- if ok:- stream.synchronize()- ok = int(scratch[0]) == 7+ if torch.version.hip is not None:+ with torch.cuda.device(device):+ ok = _row_store.capture_memops_supported()+ else:+ scratch = alloc_pinned_tensor(1, dtype=torch.int64)+ scratch.zero_()+ stream = torch.cuda.current_stream(device)+ ok = (+ _row_store.memop_write(stream.cuda_stream, scratch.data_ptr(), 7) == 0+ and _row_store.memop_wait_geq(stream.cuda_stream, scratch.data_ptr(), 7) == 0+ )+ if ok:+ stream.synchronize()+ ok = int(scratch[0]) == 7if mode == "wait" and not ok:raise RuntimeError("wait-sync requested but stream memops are unavailable")return ok@@ -51,4 +62,4 @@ def signal(flag: torch.Tensor) -> None:_row_store.signal_flag(flag.data_ptr())-__all__ = ["RowStore", "PleStore", "probe_wait_sync", "wait_reset", "signal"]+__all__ = ["RowStore", "PleStore", "alloc_sync_flag", "probe_wait_sync", "wait_reset", "signal"]diff --git a/python/freetoken/models/deepseek_v41/engram_table.py b/python/freetoken/models/deepseek_v41/engram_table.pyindex 55450bbd..3bec75c3 100644--- a/python/freetoken/models/deepseek_v41/engram_table.py+++ b/python/freetoken/models/deepseek_v41/engram_table.py@@ -130,8 +130,7 @@ class EngramDiskTable:self._eager_slot = 0self._eager_dev = Noneself._grow_eager(max_extend_tokens)- self.flag = alloc_pinned_tensor(1, dtype=torch.int64)- self.flag.zero_()+ self.flag = Noneself.wait_sync = False # set by the host coordinator after its probedef _grow_eager(self, num_tokens: int) -> None:@@ -209,6 +208,11 @@ class EngramHost:self.wait_sync = probe_wait_sync(sync_mode, device)for t in self.tables:t.wait_sync = self.wait_sync+ if self.wait_sync:+ from freetoken.kernel.row_store import alloc_sync_flag++ with torch.cuda.device(device):+ t.flag = alloc_sync_flag()self._token_readback = alloc_pinned_tensor(min(t.max_graph_rows for t in self.tables), dtype=torch.int32)self._readback_event = torch.cuda.Event()self._worker = ThreadPoolExecutor(max_workers=1, thread_name_prefix="engram-fill")diff --git a/python/freetoken/models/qwen4_exp/ple_disk.py b/python/freetoken/models/qwen4_exp/ple_disk.pyindex 536853be..7ec067bd 100644--- a/python/freetoken/models/qwen4_exp/ple_disk.py+++ b/python/freetoken/models/qwen4_exp/ple_disk.py@@ -6,6 +6,7 @@ Hash windows are pure functions of ``req.input_ids`` + ``device_len`` (prefix hifrom __future__ import annotationsimport os+from concurrent.futures import Future, ThreadPoolExecutorfrom contextlib import contextmanagerfrom dataclasses import dataclassfrom typing import Sequence@@ -202,12 +203,58 @@ class DiskRowTable:self._wait_sync = probe_wait_sync(os.getenv(_SYNC_ENV, "auto"), self._device)# one flag for all graphs: the readback event orders a fill after the previous graph, so signals never overlap- self._flag = alloc_pinned_tensor(1, dtype=torch.int64)- self._flag.zero_()+ from freetoken.kernel.row_store import alloc_sync_flag++ self._flag = alloc_sync_flag() if self._wait_sync else Noneself._token_readback = alloc_pinned_tensor(max_graph_rows, dtype=torch.int32)self._readback_event = torch.cuda.Event()sync = "wait-sync" if self._wait_sync else "launch-gating"logger.info_rank0(f"PLE disk backend: {self._store.io_backend()}, {sync}")+ self._worker = (+ ThreadPoolExecutor(max_workers=1, thread_name_prefix="ple-fill")+ if self._wait_sync and torch.version.hip is not None else None+ )+ self._pending: Future | None = None+ self._failed = False++ def _fill_decode(self, histories: Sequence[Sequence[int]], bs: int) -> None:+ try:+ self._readback_event.synchronize()+ tokens = self._token_readback[:bs].to(torch.int64).tolist()+ runs = [torch.tensor([*history, t], dtype=torch.int64)+ for history, t in zip(histories, tokens)]+ self.fill(runs, graph=True)+ except BaseException:+ from freetoken.kernel.row_store import signal++ # unblock the stream before surfacing; the step's output is discarded+ signal(self._flag)+ raise++ def _start_fill(self, histories: Sequence[Sequence[int]], bs: int) -> None:+ @torch.inference_mode()+ def _complete() -> None:+ self._fill_decode(histories, bs)++ # Like Engram, release HIP's WAIT independently of a queue-blocked graph launch.+ self._pending = self._worker.submit(_complete)++ def _await_pending(self) -> None:+ if self._pending is not None:+ pending, self._pending = self._pending, None+ try:+ pending.result()+ except BaseException:+ self._failed = True+ if self._worker is not None:+ # An interrupted waiter must drain the fill before recording its consumed event.+ self._worker.shutdown(wait=True)+ raise++ def __del__(self) -> None:+ worker = getattr(self, "_worker", None)+ if worker is not None:+ worker.shutdown(wait=False)# ---------------- host side (engine thread, before the forward launches) ----------------@@ -237,7 +284,11 @@ class DiskRowTable:return [self.image_token_id if t >= MM_PAD_SHIFT_VALUE else t for t in _context(ids, position, self.eos_token_id)]def host_fill_batch(self, batch: Batch, use_graph: bool):- """Stage this batch's rows; returns the post-dispatch fill callable under flag-sync, else None."""+ """Stage this batch's rows; returns the deferred fill for wait-sync without a worker."""+ if torch.version.hip is not None:+ self._await_pending()+ if self._failed:+ raise RuntimeError("PLE disk table failed; recreate the engine before dispatch")eos = self.eos_token_idif batch.is_decode:reqs = list(batch.reqs)@@ -245,20 +296,13 @@ class DiskRowTable:bs = batch.padded_sizeself._token_readback[:bs].copy_(batch.input_ids, non_blocking=True)self._readback_event.record(torch.cuda.current_stream(self._device))+ histories = [self._ple_context(r.input_ids, r.device_len - 1) for r in reqs]+ if self._worker is not None:+ self._start_fill(histories, bs)+ return Nonedef _complete() -> None:- try:- self._readback_event.synchronize()- tokens = self._token_readback[:bs].to(torch.int64).tolist()- runs = [torch.tensor([*self._ple_context(r.input_ids, r.device_len - 1), t], dtype=torch.int64)- for r, t in zip(reqs, tokens)]- self.fill(runs, graph=True)- except BaseException:- from freetoken.kernel.row_store import signal-- # unblock the stream before surfacing; the step's output is discarded- signal(self._flag)- raise+ self._fill_decode(histories, bs)return _complete# launch-gating: this D2H is the step's readback and orders the fill after sampling@@ -279,8 +323,22 @@ class DiskRowTable:@contextmanagerdef forward_host_ctx(self, batch: Batch, use_graph: bool):- """Around one dispatch: stage on enter, run the deferred fill+signal on exit."""+ """Around one dispatch: stage on enter, finish the fill+signal on exit."""deferred = self.host_fill_batch(batch, use_graph)+ if self._worker is not None:+ try:+ yield+ except BaseException:+ self._failed = True+ raise+ finally:+ try:+ self._await_pending()+ finally:+ # The worker must finish before recording the event it waits on at the next fill.+ if use_graph:+ self._graph_consumed.record(torch.cuda.current_stream(self._device))+ returnyield# no try/finally: a failed launch leaves no WAIT pending, so the fill must not runif deferred is not None:diff --git a/setup.py b/setup.pyindex a836038b..31383256 100644--- a/setup.py+++ b/setup.py@@ -289,7 +289,11 @@ setup(sources=["python/freetoken/kernel/csrc/row_store/row_store_ext.cpp",],+ include_dirs=runtime_include_dirs if IS_ROCM else [],+ library_dirs=runtime_library_dirs if IS_ROCM else [],+ libraries=[runtime_lib] if IS_ROCM else [],extra_compile_args=["-O3", "-std=c++20"],+ extra_link_args=runtime_link_args if IS_ROCM else [],)] if sys.platform == "linux" else []),*ggml_cpu_modules,diff --git a/tests/models/deepseek_v41/test_engram.py b/tests/models/deepseek_v41/test_engram.pyindex dec852ec..51a83297 100644--- a/tests/models/deepseek_v41/test_engram.py+++ b/tests/models/deepseek_v41/test_engram.py@@ -101,6 +101,10 @@ def test_disk_table_matches_the_dequantized_shard(tmp_path):from freetoken.kernel.row_store import probe_wait_synctable.wait_sync = probe_wait_sync("auto", torch.device("cuda"))+ if table.wait_sync:+ from freetoken.kernel.row_store import alloc_sync_flag++ table.flag = alloc_sync_flag()dec_rows = hash.row_ids(ids[40 - 1 :], context=ids[: 40 - 1])[:, 0]out = torch.empty(1, hash.n_cols * 32, dtype=torch.bfloat16, device="cuda")graph = torch.cuda.CUDAGraph()diff --git a/tests/models/qwen4_exp/test_ple_disk.py b/tests/models/qwen4_exp/test_ple_disk.pyindex 83817207..f71c9e6a 100644--- a/tests/models/qwen4_exp/test_ple_disk.py+++ b/tests/models/qwen4_exp/test_ple_disk.py@@ -2,6 +2,11 @@from __future__ import annotations+import os+from pathlib import Path+import subprocess+import sys+import threadingfrom types import SimpleNamespaceimport pytest@@ -357,3 +362,85 @@ def test_graph_sync_protocol(tmp_path, monkeypatch):gated, _, _ = _make_table(tmp_path)assert not gated._wait_syncassert gated.host_fill_batch(_decode_batch([3, 4], 5), use_graph=True) is None+++def _wait_sync_large_graph(tmp_path, kernels):+ from freetoken.core import Batch, Context, Req, set_global_ctx+ from freetoken.distributed import set_tp_info+ from freetoken.engine.graph import GraphRunner++ set_tp_info(0, 1)+ set_global_ctx(Context(page_size=1))+ disk, oracle, args = _make_table(tmp_path)+ emb = _embedding()+ row_ids = torch.zeros((8, args.num_ngram_heads), dtype=torch.int64, device="cuda")+ pad = torch.zeros(256, device="cuda")+ release = threading.Event()+ fill = disk.fill++ def delayed_fill(*args, **kwargs):+ assert release.wait(timeout=5), "fill was not released"+ return fill(*args, **kwargs)++ def forward():+ out = disk.lookup(row_ids)+ if torch.cuda.is_current_stream_capturing():+ for _ in range(kernels):+ pad.add_(1)+ return out++ disk.fill = delayed_fill+ req = Req(input_ids=torch.tensor([3, 4, 9], dtype=torch.int32), table_idx=0,+ cached_len=2, output_len=3, uid=0, sampling_params=None, cache_handle=None)+ attn = SimpleNamespace(init_capture_graph=lambda **kwargs: None,+ prepare_for_capture=lambda batch: None, prepare_for_replay=lambda batch: None)+ stream = torch.cuda.Stream()+ stream.wait_stream(torch.cuda.current_stream())+ with torch.cuda.stream(stream):+ # Exercise the capture-time upload from #697 as well as the PLE dispatch hook.+ runner = GraphRunner(stream=stream, device=torch.device("cuda"), model=SimpleNamespace(forward=forward),+ attn_backend=attn, cuda_graph_bs=[8], cuda_graph_max_bs=None,+ free_memory=1024**3, max_seq_len=16,+ vocab_size=args.num_ngram_heads * disk.head_dim, dummy_req=req)+ batch = Batch(reqs=[req] * 8, phase="decode")+ runner.pad_batch(batch)+ batch.positions = torch.full((8,), 2, dtype=torch.int32, device="cuda")+ batch.out_loc = torch.arange(8, dtype=torch.int32, device="cuda")+ for step, token in enumerate((9, 13, 5)):+ release.clear()+ batch.input_ids = torch.full((8,), token, dtype=torch.int32, device="cuda")+ with disk.forward_host_ctx(batch, use_graph=True):+ # Keep the WAIT unsatisfied when the first replay starts.+ assert int(disk._flag[0]) == 0+ timer = threading.Timer(0.05, release.set)+ timer.start()+ try:+ out = runner.replay(batch)+ finally:+ timer.join()+ stream.synchronize()+ ids = emb.row_ids(_meta([[token]] * 8, [[3, 4]] * 8, decode=True)).cuda()+ assert _bitwise_equal(out.to(disk.dtype), oracle.lookup(ids))+ assert torch.equal(pad, torch.full_like(pad, (step + 1) * kernels))+ assert int(disk._flag[0]) == 0+ runner.destroy_cuda_graphs()+++@requires_cuda+def test_wait_sync_large_graph(tmp_path):+ from freetoken.kernel.row_store import probe_wait_sync++ if not probe_wait_sync("auto", torch.device("cuda")):+ pytest.skip("stream memops unavailable")+ root = Path(__file__).resolve().parents[3]+ env = dict(os.environ, FREETOKEN_PLE_SYNC="wait")+ env["PYTHONPATH"] = os.pathsep.join((str(root / "python"), str(root), env.get("PYTHONPATH", "")))+ script = (+ "from pathlib import Path; "+ "from tests.models.qwen4_exp.test_ple_disk import _wait_sync_large_graph; "+ f"_wait_sync_large_graph(Path({str(tmp_path)!r}), 16384)"+ )+ result = subprocess.run(+ [sys.executable, "-c", script], cwd=root, env=env, capture_output=True, text=True, timeout=45,+ )+ assert result.returncode == 0, result.stdout + result.stderr