Skip to content

FreeToken brings datacenter-scale model serving to your desktop. Run massive models locally, fast and efficiently.

fix(ple): fence disk staging against in-flight lookups (#588)

Xiaoze Fand3512b43affe
diff --git a/python/freetoken/models/qwen4_exp/ple_disk.py b/python/freetoken/models/qwen4_exp/ple_disk.py
index ec7da80d..69ee9fa9 100644
--- a/python/freetoken/models/qwen4_exp/ple_disk.py
+++ b/python/freetoken/models/qwen4_exp/ple_disk.py
@@ -150,9 +150,18 @@ class DiskRowTable:
self._graph_dev = torch.empty(
max_graph_rows * self._token_bytes, dtype=torch.uint8, device=self._device
)
+ # completes once the last graph launch that read ``_graph_pinned`` has run (recorded by the
+ # dispatch context after the step's fill: a record inside the capture would not be observable)
+ self._graph_consumed = torch.cuda.Event()
eager_bytes = max_extend_tokens * self._token_bytes
- self._eager_pinned = alloc_pinned_tensor(eager_bytes, dtype=torch.uint8)
- self._eager_pinned.zero_() # the warmup prefill stages nothing and reads whatever sits here
+ # the overlap scheduler fills batch k+1 while batch k's lookup copy may still be queued: the
+ # eager staging alternates two pinned buffers, and each is rewritten only once the H2D copy
+ # that last read it has run (``_eager_read[i]``, recorded right after that copy)
+ self._eager_pinned = [alloc_pinned_tensor(eager_bytes, dtype=torch.uint8) for _ in range(2)]
+ for buf in self._eager_pinned:
+ buf.zero_() # the warmup prefill stages nothing and reads whatever sits here
+ self._eager_read = [torch.cuda.Event(), torch.cuda.Event()]
+ self._eager_slot = 0
self._eager_dev = torch.empty(eager_bytes, dtype=torch.uint8, device=self._device)
# probe picks flag-sync (graph WAITs at the consume, host fills then signals) or launch-gating
from freetoken.kernel.row_store import probe_wait_sync
@@ -170,7 +179,13 @@ class DiskRowTable:
def fill(self, runs: Sequence[torch.Tensor], *, graph: bool) -> None:
"""Stage per-request token runs (two context ids, then the new tokens) in batch order."""
- pinned = self._graph_pinned if graph else self._eager_pinned
+ if graph:
+ self._graph_consumed.synchronize()
+ pinned = self._graph_pinned
+ else:
+ self._eager_slot ^= 1
+ self._eager_read[self._eager_slot].synchronize()
+ pinned = self._eager_pinned[self._eager_slot]
offset = 0
for run in runs:
self._store.stage(run.data_ptr(), run.numel() - 2, pinned.data_ptr() + offset * self._token_bytes)
@@ -236,21 +251,27 @@ class DiskRowTable:
# no try/finally: a failed launch leaves no WAIT pending, so the fill must not run
if deferred is not None:
deferred()
+ if use_graph:
+ # only after this step's fill: the next fill waits on this launch, not the fill on it
+ self._graph_consumed.record(torch.cuda.current_stream(self._device))
# ---------------- device side (PLETableBackend protocol) ----------------
def lookup(self, row_ids: torch.Tensor, out: torch.Tensor | None = None) -> torch.Tensor:
rows = row_ids.shape[0]
+ stream = torch.cuda.current_stream(self._device)
capturing = torch.cuda.is_current_stream_capturing()
if capturing and self._wait_sync:
from freetoken.kernel.row_store import wait_reset
- wait_reset(torch.cuda.current_stream(self._device), self._flag)
+ wait_reset(stream, self._flag)
pinned, dev = (
- (self._graph_pinned, self._graph_dev) if capturing else (self._eager_pinned, self._eager_dev)
+ (self._graph_pinned, self._graph_dev) if capturing else (self._eager_pinned[self._eager_slot], self._eager_dev)
)
nbytes = rows * self._token_bytes
dev[:nbytes].copy_(pinned[:nbytes], non_blocking=True)
+ if not capturing:
+ self._eager_read[self._eager_slot].record(stream)
values = dev[:nbytes].view(torch.float8_e4m3fn).to(self.dtype)
if self.scale != 1.0:
values = values * self.scale
diff --git a/tests/models/qwen4_exp/test_ple_disk.py b/tests/models/qwen4_exp/test_ple_disk.py
index 1112210b..ba9cd98c 100644
--- a/tests/models/qwen4_exp/test_ple_disk.py
+++ b/tests/models/qwen4_exp/test_ple_disk.py
@@ -287,6 +287,24 @@ def test_disk_table_matches_oracle(tmp_path):
assert _bitwise_equal(disk.lookup(ids), oracle.lookup(ids)), "hook prefill"
+@requires_cuda
+def test_eager_fill_does_not_overwrite_an_in_flight_lookup(tmp_path):
+ """The overlap scheduler fills batch k+1 while batch k's lookup copy is still queued behind the
+ layers before it (a sleep kernel here): batch k must read its own rows."""
+ disk, oracle, args = _make_table(tmp_path)
+ emb = _embedding()
+ steps = [(3, 4, 7), (4, 7, 9), (7, 9, 13), (9, 13, 5)]
+ ids = [emb.row_ids(_meta([[token]], [[older, newer]], decode=True)).cuda() for older, newer, token in steps]
+ got = []
+ for (older, newer, token), step_ids in zip(steps, ids):
+ disk.fill([torch.tensor([older, newer, token])], graph=False)
+ torch.cuda._sleep(50_000_000)
+ got.append(disk.lookup(step_ids))
+ torch.cuda.synchronize()
+ for i, (out, step_ids) in enumerate(zip(got, ids)):
+ assert _bitwise_equal(out, oracle.lookup(step_ids)), f"step {i} read another step's rows"
+
+
@requires_cuda
def test_graph_sync_protocol(tmp_path, monkeypatch):
disk, oracle, args = _make_table(tmp_path)