FreeToken brings datacenter-scale model serving to your desktop. Run massive models locally, fast and efficiently.
from __future__ import annotations
from dataclasses import dataclass, field
from typing import Iterable, Set
from freetoken.core import Batch, Req
@dataclass
class DecodeManager:
page_size: int
# slots a decode step may hold past each request's committed length (provisional rows)
token_headroom: int = 0
running_reqs: Set[Req] = field(default_factory=set)
def filter_reqs(self, reqs: Iterable[Req]) -> None:
self.running_reqs = {req for req in self.running_reqs.union(reqs) if req.can_decode}
def remove_req(self, req: Req) -> None:
self.running_reqs.discard(req)
def abort_req(self, uid: int) -> Req | None:
for req in self.running_reqs:
if req.uid == uid:
self.running_reqs.remove(req)
return req
return None
@property
def inflight_tokens(self) -> int:
# each request's partly filled page, plus the provisional rows a decode step may add
tokens_reserved = (self.page_size - 1 + self.token_headroom) * len(self.running_reqs)
return sum(req.remain_len for req in self.running_reqs) + tokens_reserved
def schedule_next_batch(self) -> Batch | None:
if not self.runnable:
return None
return Batch(reqs=sorted(self.running_reqs, key=lambda req: req.uid), phase="decode")
@property
def runnable(self) -> bool:
return len(self.running_reqs) > 0