Skip to content

FreeToken brings datacenter-scale model serving to your desktop. Run massive models locally, fast and efficiently.

3.2 KB
Raw Blame
"""Encoder embedding cache keyed by content hash; an entry lives while any request still has rows of it to gather."""

from __future__ import annotations

from dataclasses import dataclass, field
from typing import Dict

import torch

from freetoken.utils import init_logger

logger = init_logger(__name__)


@dataclass
class _Entry:
    tensor: torch.Tensor | None = None
    # uid -> embedding rows the request still has to gather, summed over its occurrences of the image
    remaining: Dict[int, int] = field(default_factory=dict)


class EncoderCache:
    def __init__(self, storage: str = "cpu") -> None:
        assert storage in ("cpu", "cuda")
        self._storage = storage
        self._entries: Dict[int, _Entry] = {}

    def register(self, item_hash: int, uid: int, rows: int) -> None:
        """Claim rows of the image for uid before any of its chunks run; a repeated image adds up."""
        if rows <= 0:
            return
        entry = self._entries.setdefault(item_hash, _Entry())
        entry.remaining[uid] = entry.remaining.get(uid, 0) + rows

    def has(self, item_hash: int) -> bool:
        entry = self._entries.get(item_hash)
        return entry is not None and entry.tensor is not None

    def put(self, item_hash: int, embedding: torch.Tensor) -> None:
        entry = self._entries.get(item_hash)
        assert entry is not None, f"image {item_hash} encoded before any request registered it"
        if entry.tensor is not None:
            return
        if self._storage == "cpu":
            stored = torch.empty(
                embedding.shape, dtype=embedding.dtype, device="cpu",
                pin_memory=torch.cuda.is_available(),
            )
            stored.copy_(embedding, non_blocking=True)
        else:
            stored = embedding
        entry.tensor = stored

    def get_slice(self, item_hash: int, lo: int, hi: int, device: torch.device) -> torch.Tensor:
        view = self._entries[item_hash].tensor[lo:hi]
        if view.device == device:
            return view
        return view.to(device, non_blocking=True)

    def consume(self, item_hash: int, uid: int, rows: int) -> None:
        """Account rows uid gathered; the last row of the last holder frees the entry."""
        entry = self._entries[item_hash]
        left = entry.remaining[uid] - rows
        assert left >= 0, f"request {uid} gathered more rows of image {item_hash} than it registered"
        if left:
            entry.remaining[uid] = left
            return
        del entry.remaining[uid]
        if not entry.remaining:
            del self._entries[item_hash]

    def release(self, uid: int, hashes: list[int]) -> None:
        """Drop uid's claims whatever it gathered (abort); entries nobody else holds are freed."""
        for h in hashes:
            entry = self._entries.get(h)
            if entry is None:
                continue
            entry.remaining.pop(uid, None)
            if not entry.remaining:
                del self._entries[h]

    def stats(self) -> tuple[int, int]:
        stored = [e.tensor for e in self._entries.values() if e.tensor is not None]
        return len(stored), sum(t.numel() * t.element_size() for t in stored)


__all__ = ["EncoderCache"]