Skip to content

FreeToken brings datacenter-scale model serving to your desktop. Run massive models locally, fast and efficiently.

17.1 KB
Raw Blame
"""Disk-backed PLE table (--ple-backend disk): the C++ store hashes n-gram windows and batch-reads rows from the checkpoint's fp8 or bf16 shard tensors into pinned staging; the captured ``lookup`` is a fixed-shape H2D copy + dequant.

Hash windows are pure functions of ``req.input_ids`` + ``device_len`` (prefix hits, restores and COW forks need no bookkeeping); the decode input token lives device-side under overlap scheduling and is read back here.
"""

from __future__ import annotations

import os
from concurrent.futures import Future, ThreadPoolExecutor
from contextlib import contextmanager
from dataclasses import dataclass
from typing import Sequence

import safetensors
import torch

from freetoken.mm import MM_PAD_SHIFT_VALUE, restore_placeholder

from freetoken.core import Batch
from freetoken.kernel.pinned import alloc_pinned_tensor
from freetoken.utils import init_logger

from .weight import (
    _PLE_SCALE_SUFFIX,
    _PLE_SHARD_RE,
    _PLE_ST_DTYPES,
    _ple_table_files,
    _safetensors_header,
)

_IO_URING_ENV = "FREETOKEN_PLE_IO_URING"
_SYNC_ENV = "FREETOKEN_PLE_SYNC"  # auto | wait | gate

logger = init_logger(__name__)


def _context(ids: torch.Tensor, position: int, eos: int) -> list[int]:
    """The two token ids before ``position``; eos pads past the start."""
    return [int(ids[position - 2]) if position >= 2 else eos,
            int(ids[position - 1]) if position >= 1 else eos]


@dataclass(frozen=True)
class PleRowSource:
    """On-disk row layout: equal extents, row i of an extent at ``base + i * row_stride`` (a repacked flat file is one extent with its own stride)."""

    paths: list[str]
    extent_file: list[int]
    extent_base: list[int]
    rows_per_extent: int
    row_bytes: int
    row_stride: int
    scale: float
    # packed ggml rows (a GGUF table) of ``row_values`` values each; None for float rows of ``row_dtype``
    ggml_type: int | None = None
    row_values: int | None = None
    row_dtype: torch.dtype = torch.float8_e4m3fn

    @property
    def total_rows(self) -> int:
        return len(self.extent_base) * self.rows_per_extent


def source_from_safetensors(folder: str) -> PleRowSource:
    """Map the checkpoint's ``ngram_embedding.shard_<i>`` tensors in place: one extent per shard, no copy."""
    rows = cols = 0
    dtype: torch.dtype | None = None
    scale: torch.Tensor | None = None
    paths: list[str] = []
    path_idx: dict[str, int] = {}
    shards: dict[int, tuple[int, int]] = {}
    for path in _ple_table_files(folder):
        header, base = _safetensors_header(path)
        for key, meta in header.items():
            if key == "__metadata__":
                continue
            if key.endswith(_PLE_SCALE_SUFFIX):
                with safetensors.safe_open(path, framework="pt", device="cpu") as f:
                    scale = f.get_tensor(key).reshape(())
                continue
            match = _PLE_SHARD_RE.search(key)
            if match is None:
                continue
            shard_dtype = _PLE_ST_DTYPES.get(meta["dtype"])
            if shard_dtype is None or dtype not in (None, shard_dtype):
                raise ValueError(f"PLE shard {key} has dtype {meta['dtype']}, expected one of {sorted(_PLE_ST_DTYPES)} for every shard")
            dtype = shard_dtype
            if rows and tuple(meta["shape"]) != (rows, cols):
                raise ValueError(f"PLE shard {key} is {meta['shape']}, expected {[rows, cols]}")
            rows, cols = meta["shape"]
            if path not in path_idx:
                path_idx[path] = len(paths)
                paths.append(path)
            idx = int(match.group("shard"))
            if idx in shards:
                raise ValueError(f"duplicate PLE shard {idx} in {path}")
            shards[idx] = (path_idx[path], base + meta["data_offsets"][0])
    if sorted(shards) != list(range(len(shards))) or not shards:
        raise ValueError(f"PLE shard indices are not contiguous 0..N-1: {sorted(shards)[:8]}")
    if scale is None:
        if dtype is torch.float8_e4m3fn:
            raise ValueError("PLE table has no weight_scale")
        scale = torch.tensor(1.0)
    order = [shards[i] for i in range(len(shards))]
    row_bytes = cols * dtype.itemsize
    return PleRowSource(paths, [f for f, _ in order], [b for _, b in order], rows, row_bytes, row_bytes, float(scale),
                        row_values=cols, row_dtype=dtype)


def source_from_gguf(model_path: str) -> PleRowSource:
    """The GGUF ``per_layer_token_embd`` table in place: one extent of packed ggml rows."""
    from freetoken.layers.quantization.ggml_types import row_bytes

    from .gguf import ple_table_info

    info = ple_table_info(model_path)
    values, rows = info.ne
    nbytes = row_bytes(info.ggml_type, values)
    return PleRowSource([info.path], [0], [info.offset], rows, nbytes, nbytes, 1.0, info.ggml_type, values)


def resolve_row_source(model_path: str) -> PleRowSource:
    """Pick the row source for a checkpoint; the seam where a repacked format would plug in."""
    from freetoken.models.gguf.reader import gguf_config_source

    if (gguf_src := gguf_config_source(model_path)) is not None:
        return source_from_gguf(gguf_src)
    from freetoken.utils import download_hf_weight

    return source_from_safetensors(download_hf_weight(model_path))


class DiskRowTable:
    """``PLETableBackend`` whose rows are read from disk per fill (--ple-backend disk)."""

    def __init__(
        self,
        source: PleRowSource,
        hash_constants: dict,
        *,
        max_graph_rows: int = 256,
        max_extend_tokens: int = 8192,
        dtype: torch.dtype = torch.bfloat16,
    ) -> None:
        from freetoken.kernel.row_store import PleStore
        from freetoken.layers.quantization.ggml_types import float_dtype

        self.num_rows = source.total_rows
        self.head_dim = source.row_values or source.row_bytes
        self._ggml_type = source.ggml_type
        self._row_bytes = source.row_bytes
        # float rows (fp8, or F32 / F16 / BF16 in a GGUF) are only cast; quantized ggml rows go through ggml
        self._row_dtype = source.row_dtype if source.ggml_type is None else float_dtype(source.ggml_type)
        self.dtype = dtype
        self.heads = int(hash_constants["num_ngram_heads"])
        self.scale = source.scale
        self.eos_token_id = int(hash_constants["eos_token_id"])
        self.image_token_id = hash_constants.get("image_token_id")
        sizes = [int(x) for x in hash_constants["per_head_vocab_sizes"]]
        offsets = [int(x) for x in hash_constants["per_head_offsets"]]
        need = max(o + s for o, s in zip(offsets, sizes))
        if need > source.total_rows:
            raise ValueError(
                f"PLE row source holds {source.total_rows} rows but the hash addresses {need}; incomplete checkpoint?"
            )
        self._store = PleStore(
            paths=list(source.paths),
            extent_file=list(source.extent_file),
            extent_base=list(source.extent_base),
            rows_per_extent=source.rows_per_extent,
            row_bytes=source.row_bytes,
            row_stride=source.row_stride,
            multipliers=[int(x) for x in hash_constants["layer_multipliers"]],
            head_vocab_sizes=sizes,
            head_offsets=offsets,
            eos_token_id=self.eos_token_id,
            use_io_uring=os.getenv(_IO_URING_ENV, "1") != "0",
        )
        self._device = torch.device("cuda", torch.cuda.current_device())
        self._token_bytes = self.heads * self._row_bytes
        # allocated up front: pinned alloc inside stream capture is illegal; one replay consumes it at a time
        self._graph_pinned = alloc_pinned_tensor(max_graph_rows * self._token_bytes, dtype=torch.uint8)
        self._graph_pinned.zero_()  # padded decode lanes read whatever sits here
        # outlives any one graph: a cache rebuild recaptures against the same pointer
        self._graph_dev = torch.empty(
            max_graph_rows * self._token_bytes, dtype=torch.uint8, device=self._device
        )
        # completes once the last graph launch that read ``_graph_pinned`` has run (recorded by the
        # dispatch context after the step's fill: a record inside the capture would not be observable)
        self._graph_consumed = torch.cuda.Event()
        eager_bytes = max_extend_tokens * self._token_bytes
        # the overlap scheduler fills batch k+1 while batch k's lookup copy may still be queued: the
        # eager staging alternates two pinned buffers, and each is rewritten only once the H2D copy
        # that last read it has run (``_eager_read[i]``, recorded right after that copy)
        self._eager_pinned = [alloc_pinned_tensor(eager_bytes, dtype=torch.uint8) for _ in range(2)]
        for buf in self._eager_pinned:
            buf.zero_()  # the warmup prefill stages nothing and reads whatever sits here
        self._eager_read = [torch.cuda.Event(), torch.cuda.Event()]
        self._eager_slot = 0
        self._eager_dev = torch.empty(eager_bytes, dtype=torch.uint8, device=self._device)
        # probe picks flag-sync (graph WAITs at the consume, host fills then signals) or launch-gating
        from freetoken.kernel.row_store import probe_wait_sync

        self._wait_sync = probe_wait_sync(os.getenv(_SYNC_ENV, "auto"), self._device)
        # one flag for all graphs: the readback event orders a fill after the previous graph, so signals never overlap
        from freetoken.kernel.row_store import alloc_sync_flag

        self._flag = alloc_sync_flag() if self._wait_sync else None
        self._token_readback = alloc_pinned_tensor(max_graph_rows, dtype=torch.int32)
        self._readback_event = torch.cuda.Event()
        sync = "wait-sync" if self._wait_sync else "launch-gating"
        logger.info_rank0(f"PLE disk backend: {self._store.io_backend()}, {sync}")
        self._worker = (
            ThreadPoolExecutor(max_workers=1, thread_name_prefix="ple-fill")
            if self._wait_sync and torch.version.hip is not None else None
        )
        self._pending: Future | None = None
        self._failed = False

    def _fill_decode(self, histories: Sequence[Sequence[int]], bs: int) -> None:
        try:
            self._readback_event.synchronize()
            tokens = self._token_readback[:bs].to(torch.int64).tolist()
            runs = [torch.tensor([*history, t], dtype=torch.int64)
                    for history, t in zip(histories, tokens)]
            self.fill(runs, graph=True)
        except BaseException:
            from freetoken.kernel.row_store import signal

            # unblock the stream before surfacing; the step's output is discarded
            signal(self._flag)
            raise

    def _start_fill(self, histories: Sequence[Sequence[int]], bs: int) -> None:
        @torch.inference_mode()
        def _complete() -> None:
            self._fill_decode(histories, bs)

        # Like Engram, release HIP's WAIT independently of a queue-blocked graph launch.
        self._pending = self._worker.submit(_complete)

    def _await_pending(self) -> None:
        if self._pending is not None:
            pending, self._pending = self._pending, None
            try:
                pending.result()
            except BaseException:
                self._failed = True
                if self._worker is not None:
                    # An interrupted waiter must drain the fill before recording its consumed event.
                    self._worker.shutdown(wait=True)
                raise

    def __del__(self) -> None:
        worker = getattr(self, "_worker", None)
        if worker is not None:
            worker.shutdown(wait=False)

    # ---------------- host side (engine thread, before the forward launches) ----------------

    def fill(self, runs: Sequence[torch.Tensor], *, graph: bool) -> None:
        """Stage per-request token runs (two context ids, then the new tokens) in batch order."""
        if graph:
            self._graph_consumed.synchronize()
            pinned = self._graph_pinned
        else:
            self._eager_slot ^= 1
            self._eager_read[self._eager_slot].synchronize()
            pinned = self._eager_pinned[self._eager_slot]
        offset = 0
        for run in runs:
            self._store.stage(run.data_ptr(), run.numel() - 2, pinned.data_ptr() + offset * self._token_bytes)
            offset += run.numel() - 2
        self._store.flush(self._flag.data_ptr() if graph and self._wait_sync else 0)

    def _ple_ids(self, input_ids: torch.Tensor) -> torch.Tensor:
        if self.image_token_id is None:
            return input_ids
        return restore_placeholder(input_ids, self.image_token_id)

    def _ple_context(self, ids: torch.Tensor, position: int) -> list[int]:
        if self.image_token_id is None:
            return _context(ids, position, self.eos_token_id)
        return [self.image_token_id if t >= MM_PAD_SHIFT_VALUE else t for t in _context(ids, position, self.eos_token_id)]

    def host_fill_batch(self, batch: Batch, use_graph: bool):
        """Stage this batch's rows; returns the deferred fill for wait-sync without a worker."""
        if torch.version.hip is not None:
            self._await_pending()
            if self._failed:
                raise RuntimeError("PLE disk table failed; recreate the engine before dispatch")
        eos = self.eos_token_id
        if batch.is_decode:
            reqs = list(batch.reqs)
            if use_graph and self._wait_sync:
                bs = batch.padded_size
                self._token_readback[:bs].copy_(batch.input_ids, non_blocking=True)
                self._readback_event.record(torch.cuda.current_stream(self._device))
                histories = [self._ple_context(r.input_ids, r.device_len - 1) for r in reqs]
                if self._worker is not None:
                    self._start_fill(histories, bs)
                    return None

                def _complete() -> None:
                    self._fill_decode(histories, bs)

                return _complete
            # launch-gating: this D2H is the step's readback and orders the fill after sampling
            tokens = batch.input_ids.to("cpu").to(torch.int64).tolist()
            runs = [torch.tensor([*self._ple_context(r.input_ids, r.device_len - 1), t], dtype=torch.int64)
                    for r, t in zip(reqs, tokens)]
            self.fill(runs, graph=use_graph)
            return None
        runs = [
            torch.cat((
                torch.tensor(self._ple_context(req.input_ids, req.cached_len), dtype=torch.int64),
                self._ple_ids(req.input_ids[req.cached_len : req.device_len]).to(torch.int64),
            ))
            for req in batch.padded_reqs
        ]
        self.fill(runs, graph=False)
        return None

    @contextmanager
    def forward_host_ctx(self, batch: Batch, use_graph: bool):
        """Around one dispatch: stage on enter, finish the fill+signal on exit."""
        deferred = self.host_fill_batch(batch, use_graph)
        if self._worker is not None:
            try:
                yield
            except BaseException:
                self._failed = True
                raise
            finally:
                try:
                    self._await_pending()
                finally:
                    # The worker must finish before recording the event it waits on at the next fill.
                    if use_graph:
                        self._graph_consumed.record(torch.cuda.current_stream(self._device))
            return
        yield
        # no try/finally: a failed launch leaves no WAIT pending, so the fill must not run
        if deferred is not None:
            deferred()
        if use_graph:
            # only after this step's fill: the next fill waits on this launch, not the fill on it
            self._graph_consumed.record(torch.cuda.current_stream(self._device))

    # ---------------- device side (PLETableBackend protocol) ----------------

    def lookup(self, row_ids: torch.Tensor, out: torch.Tensor | None = None) -> torch.Tensor:
        rows = row_ids.shape[0]
        stream = torch.cuda.current_stream(self._device)
        capturing = torch.cuda.is_current_stream_capturing()
        if capturing and self._wait_sync:
            from freetoken.kernel.row_store import wait_reset

            wait_reset(stream, self._flag)
        pinned, dev = (
            (self._graph_pinned, self._graph_dev) if capturing else (self._eager_pinned[self._eager_slot], self._eager_dev)
        )
        nbytes = rows * self._token_bytes
        dev[:nbytes].copy_(pinned[:nbytes], non_blocking=True)
        if not capturing:
            self._eager_read[self._eager_slot].record(stream)
        if self._row_dtype is not None:
            # copy=True: rows already in self.dtype must not alias the staging buffer the next fill reuses
            values = dev[:nbytes].view(self._row_dtype).to(self.dtype, copy=True)
        else:
            from freetoken.kernel import ggml

            values = ggml.dequantize(dev[:nbytes].view(-1, self._row_bytes), self._ggml_type, self.head_dim, self.dtype)
        if self.scale != 1.0:
            values = values * self.scale
        values = values.view(*row_ids.shape[:-1], -1)
        if out is None:
            return values
        out.copy_(values)
        return out

    def prefetch(self, row_ids: torch.Tensor) -> None:
        return None