Skip to content

FreeToken brings datacenter-scale model serving to your desktop. Run massive models locally, fast and efficiently.

4.7 KB
Raw Blame
"""Block weights streamed from pinned host banks through two device staging buffers: block i computes from buffer i % 2 while the copy stream fills the other with block i + 1."""

from __future__ import annotations

from dataclasses import dataclass
from typing import Any, Iterator, List, Sequence

import torch

from freetoken.layers import BaseOP

_ALIGN = 256


@dataclass(frozen=True)
class _Slot:
    owner: Any
    attr: str
    offset: int
    nbytes: int
    dtype: torch.dtype
    shape: torch.Size

    def view(self, row: torch.Tensor) -> torch.Tensor:
        return row[self.offset : self.offset + self.nbytes].view(self.dtype).view(self.shape)


def _slots(block: BaseOP) -> tuple[List[_Slot], int]:
    """Every tensor attribute of the block (recursively), laid out back to back with 256-byte alignment."""
    slots: List[_Slot] = []
    offset = 0

    def walk(op: BaseOP) -> None:
        nonlocal offset
        for name, value in op.__dict__.items():
            if name.startswith("_"):
                continue
            if isinstance(value, torch.Tensor):
                nbytes = value.numel() * value.element_size()
                slots.append(_Slot(op, name, offset, nbytes, value.dtype, value.shape))
                offset += -(-nbytes // _ALIGN) * _ALIGN
            elif isinstance(value, BaseOP):
                walk(value)

    walk(block)
    return slots, offset


class BlockWeightStreamer:
    """Owns the host banks and the two staging buffers of a block stack; drives one forward with `blocks()`."""

    def __init__(self, blocks: Sequence[BaseOP], device: torch.device):
        self.device = device
        self._layouts = [_slots(b) for b in blocks]
        row_bytes = self._layouts[0][1]
        assert all(nbytes == row_bytes for _, nbytes in self._layouts), "streamed blocks must share one layout"
        self.bank = torch.empty((len(blocks), row_bytes), dtype=torch.uint8, pin_memory=True)
        self.staging = torch.empty((2, row_bytes), dtype=torch.uint8, device=device)
        for b, (slots, _) in enumerate(self._layouts):
            for s in slots:
                s.view(self.bank[b]).copy_(getattr(s.owner, s.attr))
        torch.cuda.synchronize(device)
        self._bind_all_to_host()
        self.copy_stream = torch.cuda.Stream(device=device)
        self.begin_event = torch.cuda.Event()
        self.ready_events = [torch.cuda.Event() for _ in range(2)]
        self.release_events = [torch.cuda.Event() for _ in range(2)]
        self._holder: List[int | None] = [None, None]
        self._has_release: List[bool] = [False, False]

    @property
    def device_bytes(self) -> int:
        return self.staging.numel()

    def _bind_all_to_host(self) -> None:
        # between forwards the attributes point at the pinned bank, so state_dict() stays complete and correct
        for b, (slots, _) in enumerate(self._layouts):
            for s in slots:
                setattr(s.owner, s.attr, s.view(self.bank[b]))

    def unstream(self) -> None:
        """Put every block's tensors back on the device as plain resident copies."""
        for b, (slots, _) in enumerate(self._layouts):
            for s in slots:
                setattr(s.owner, s.attr, s.view(self.bank[b]).to(self.device))
        torch.cuda.synchronize(self.device)

    def _prefetch(self, i: int) -> None:
        if i >= len(self._layouts):
            return
        buf = i % 2
        if self._holder[buf] == i:
            return
        with torch.cuda.stream(self.copy_stream):
            if self._has_release[buf]:
                self.copy_stream.wait_event(self.release_events[buf])
            self.staging[buf].copy_(self.bank[i], non_blocking=True)
            self.ready_events[buf].record(self.copy_stream)
        self._holder[buf] = i

    def blocks(self, ops: Sequence[BaseOP]) -> Iterator[tuple[int, BaseOP]]:
        """Yield (i, block) with block i's tensors bound to a filled staging buffer while block i + 1 streams in."""
        compute = torch.cuda.current_stream(self.device)
        # the copy stream must not overwrite staging that a still-running previous forward reads
        self.begin_event.record(compute)
        self.copy_stream.wait_event(self.begin_event)
        for i, op in enumerate(ops):
            buf = i % 2
            self._prefetch(i)
            self._prefetch(i + 1)
            compute.wait_event(self.ready_events[buf])
            slots, _ = self._layouts[i]
            for s in slots:
                setattr(s.owner, s.attr, s.view(self.staging[buf]))
            yield i, op
            self.release_events[buf].record(compute)
            self._has_release[buf] = True
            for s in slots:
                setattr(s.owner, s.attr, s.view(self.bank[i]))


__all__ = ["BlockWeightStreamer"]