FreeToken brings datacenter-scale model serving to your desktop. Run massive models locally, fast and efficiently.
fix(ple): fence disk staging against in-flight lookups (#588)
Xiaoze Fand3512b43affe
diff --git a/python/freetoken/models/qwen4_exp/ple_disk.py b/python/freetoken/models/qwen4_exp/ple_disk.pyindex ec7da80d..69ee9fa9 100644--- a/python/freetoken/models/qwen4_exp/ple_disk.py+++ b/python/freetoken/models/qwen4_exp/ple_disk.py@@ -150,9 +150,18 @@ class DiskRowTable:self._graph_dev = torch.empty(max_graph_rows * self._token_bytes, dtype=torch.uint8, device=self._device)+ # completes once the last graph launch that read ``_graph_pinned`` has run (recorded by the+ # dispatch context after the step's fill: a record inside the capture would not be observable)+ self._graph_consumed = torch.cuda.Event()eager_bytes = max_extend_tokens * self._token_bytes- self._eager_pinned = alloc_pinned_tensor(eager_bytes, dtype=torch.uint8)- self._eager_pinned.zero_() # the warmup prefill stages nothing and reads whatever sits here+ # the overlap scheduler fills batch k+1 while batch k's lookup copy may still be queued: the+ # eager staging alternates two pinned buffers, and each is rewritten only once the H2D copy+ # that last read it has run (``_eager_read[i]``, recorded right after that copy)+ self._eager_pinned = [alloc_pinned_tensor(eager_bytes, dtype=torch.uint8) for _ in range(2)]+ for buf in self._eager_pinned:+ buf.zero_() # the warmup prefill stages nothing and reads whatever sits here+ self._eager_read = [torch.cuda.Event(), torch.cuda.Event()]+ self._eager_slot = 0self._eager_dev = torch.empty(eager_bytes, dtype=torch.uint8, device=self._device)# probe picks flag-sync (graph WAITs at the consume, host fills then signals) or launch-gatingfrom freetoken.kernel.row_store import probe_wait_sync@@ -170,7 +179,13 @@ class DiskRowTable:def fill(self, runs: Sequence[torch.Tensor], *, graph: bool) -> None:"""Stage per-request token runs (two context ids, then the new tokens) in batch order."""- pinned = self._graph_pinned if graph else self._eager_pinned+ if graph:+ self._graph_consumed.synchronize()+ pinned = self._graph_pinned+ else:+ self._eager_slot ^= 1+ self._eager_read[self._eager_slot].synchronize()+ pinned = self._eager_pinned[self._eager_slot]offset = 0for run in runs:self._store.stage(run.data_ptr(), run.numel() - 2, pinned.data_ptr() + offset * self._token_bytes)@@ -236,21 +251,27 @@ class DiskRowTable:# no try/finally: a failed launch leaves no WAIT pending, so the fill must not runif deferred is not None:deferred()+ if use_graph:+ # only after this step's fill: the next fill waits on this launch, not the fill on it+ self._graph_consumed.record(torch.cuda.current_stream(self._device))# ---------------- device side (PLETableBackend protocol) ----------------def lookup(self, row_ids: torch.Tensor, out: torch.Tensor | None = None) -> torch.Tensor:rows = row_ids.shape[0]+ stream = torch.cuda.current_stream(self._device)capturing = torch.cuda.is_current_stream_capturing()if capturing and self._wait_sync:from freetoken.kernel.row_store import wait_reset- wait_reset(torch.cuda.current_stream(self._device), self._flag)+ wait_reset(stream, self._flag)pinned, dev = (- (self._graph_pinned, self._graph_dev) if capturing else (self._eager_pinned, self._eager_dev)+ (self._graph_pinned, self._graph_dev) if capturing else (self._eager_pinned[self._eager_slot], self._eager_dev))nbytes = rows * self._token_bytesdev[:nbytes].copy_(pinned[:nbytes], non_blocking=True)+ if not capturing:+ self._eager_read[self._eager_slot].record(stream)values = dev[:nbytes].view(torch.float8_e4m3fn).to(self.dtype)if self.scale != 1.0:values = values * self.scalediff --git a/tests/models/qwen4_exp/test_ple_disk.py b/tests/models/qwen4_exp/test_ple_disk.pyindex 1112210b..ba9cd98c 100644--- a/tests/models/qwen4_exp/test_ple_disk.py+++ b/tests/models/qwen4_exp/test_ple_disk.py@@ -287,6 +287,24 @@ def test_disk_table_matches_oracle(tmp_path):assert _bitwise_equal(disk.lookup(ids), oracle.lookup(ids)), "hook prefill"+@requires_cuda+def test_eager_fill_does_not_overwrite_an_in_flight_lookup(tmp_path):+ """The overlap scheduler fills batch k+1 while batch k's lookup copy is still queued behind the+ layers before it (a sleep kernel here): batch k must read its own rows."""+ disk, oracle, args = _make_table(tmp_path)+ emb = _embedding()+ steps = [(3, 4, 7), (4, 7, 9), (7, 9, 13), (9, 13, 5)]+ ids = [emb.row_ids(_meta([[token]], [[older, newer]], decode=True)).cuda() for older, newer, token in steps]+ got = []+ for (older, newer, token), step_ids in zip(steps, ids):+ disk.fill([torch.tensor([older, newer, token])], graph=False)+ torch.cuda._sleep(50_000_000)+ got.append(disk.lookup(step_ids))+ torch.cuda.synchronize()+ for i, (out, step_ids) in enumerate(zip(got, ids)):+ assert _bitwise_equal(out, oracle.lookup(step_ids)), f"step {i} read another step's rows"++@requires_cudadef test_graph_sync_protocol(tmp_path, monkeypatch):disk, oracle, args = _make_table(tmp_path)