Skip to content

FreeToken brings datacenter-scale model serving to your desktop. Run massive models locally, fast and efficiently.

2.5 KB
Raw Blame
"""Python face of the disk row store extension (``kernel/csrc/row_store``) and its CUDA-graph sync.

``RowStore`` reads fixed-width rows straight out of checkpoint shard files (io_uring or a pread pool,
O_DIRECT) into pinned staging by row id; ``PleStore`` adds PLE's n-gram hashing.

Graph sync: a captured graph cannot wait for host I/O, so a captured ``lookup`` WAITs on a host-visible
flag (pinned on CUDA, signal memory on HIP) through a stream memop (``wait_reset``). The host
``signal``s once the fill landed; where the driver rejects memops in capture, the caller falls back
to launch-gating (fill before replay).
``probe_wait_sync`` decides which, once, at startup.
"""

from __future__ import annotations

import torch

from freetoken.kernel import _row_store
from freetoken.kernel.pinned import alloc_pinned_tensor

RowStore = _row_store.RowStore
PleStore = _row_store.PleStore


def alloc_sync_flag() -> torch.Tensor:
    if torch.version.hip is not None:
        return _row_store.alloc_signal_tensor()
    return alloc_pinned_tensor(1, dtype=torch.int64).zero_()


def probe_wait_sync(mode: str, device: torch.device) -> bool:
    """``mode``: ``auto`` (probe), ``wait`` (require memops) or ``gate`` (never use them)."""
    if mode not in ("auto", "wait", "gate"):
        raise ValueError(f"unknown row-store sync mode {mode!r}; expected auto, wait or gate")
    if mode == "gate":
        return False
    if torch.version.hip is not None:
        with torch.cuda.device(device):
            ok = _row_store.capture_memops_supported()
    else:
        scratch = alloc_pinned_tensor(1, dtype=torch.int64)
        scratch.zero_()
        stream = torch.cuda.current_stream(device)
        ok = (
            _row_store.memop_write(stream.cuda_stream, scratch.data_ptr(), 7) == 0
            and _row_store.memop_wait_geq(stream.cuda_stream, scratch.data_ptr(), 7) == 0
        )
        if ok:
            stream.synchronize()
            ok = int(scratch[0]) == 7
    if mode == "wait" and not ok:
        raise RuntimeError("wait-sync requested but stream memops are unavailable")
    return ok


def wait_reset(stream: torch.cuda.Stream, flag: torch.Tensor) -> None:
    """Enqueue WAIT(flag >= 1) then flag := 0 on ``stream`` (capturable)."""
    _row_store.memop_wait_reset(stream.cuda_stream, flag.data_ptr())


def signal(flag: torch.Tensor) -> None:
    """Host side: release a pending WAIT (a release-store of 1)."""
    _row_store.signal_flag(flag.data_ptr())


__all__ = ["RowStore", "PleStore", "alloc_sync_flag", "probe_wait_sync", "wait_reset", "signal"]