FreeToken brings datacenter-scale model serving to your desktop. Run massive models locally, fast and efficiently.
from __future__ import annotations
from typing import TYPE_CHECKING, Protocol
from freetoken.utils import Registry
if TYPE_CHECKING:
import torch
from freetoken.models import ModelConfig
from .base import (
BaseCacheHandle,
BaseKVCachePool,
BasePrefixCache,
MatchResult,
SizeInfo,
)
class CacheManagerCreator(Protocol):
def __call__(self, device: torch.device) -> BasePrefixCache: ...
SUPPORTED_CACHE_MANAGER = Registry[CacheManagerCreator]("Cache Manager")
def resolve_pool_class(model_config: ModelConfig) -> type[BaseKVCachePool]:
"""attn_type -> KV pool family, the dispatch shared by ``create_kv_pool`` and the
engine's pre-pool sizing calls (the classmethod cost/solve surface). Driven by the
group-spec walk (same source as the backend capability matrix); getattr fallbacks
cover duck-typed test configs that don't implement it."""
from freetoken.attention import AttnType
specs_fn = getattr(model_config, "kv_cache_group_specs", None)
if specs_fn is None:
if getattr(model_config, "dsv4_args", None) is not None:
from .dsv4.v4_pool import DSV4PagedKVCache
return DSV4PagedKVCache
from .mha_pool import MHAKVCache
return MHAKVCache
specs = list(specs_fn())
types = {spec.attn_type for spec in specs}
if AttnType.DSV4 in types:
from .dsv4.v4_pool import DSV4PagedKVCache
return DSV4PagedKVCache
if AttnType.DSV41 in types:
from .dsv4.v41_pool import DSV41PagedKVCache
return DSV41PagedKVCache
if AttnType.SWA in types:
from .hybrid_swa_pool import HybridSWAKVCache
return HybridSWAKVCache
if AttnType.DSA in types:
# kpool-compressed indexer (glm5_next): shadow slab + tail rings.
if any(s.attn_type == AttnType.DSA and s.index_ratio > 1 for s in specs):
from .dsa_pool import KpoolDSAKVCache
return KpoolDSAKVCache
from .dsa_pool import DSAKVCache
return DSAKVCache
if AttnType.MLA in types:
from .dsa_pool import MLAKVCache
return MLAKVCache
if AttnType.QSA in types:
from .qsa_pool import QSAKVCache
return QSAKVCache
if AttnType.BSA in types:
from .bsa_pool import BSAKVCache
return BSAKVCache
from .mha_pool import MHAKVCache
return MHAKVCache
def create_kv_pool(config, num_pages: int, device: torch.device, dtype: torch.dtype):
"""Build the engine's KV pool for ``num_pages`` USABLE pages (the dummy page and every
secondary tier -- window pool, index slab, state rings -- are derived here or inside
the pool). Single factory entry for all pool families, DSV4 included."""
from .dsv4.v4_cost_model import _dsv4_pool_sizes
from .hybrid_swa_pool import _naive_swa_num_tokens, _swa_paged_num_tokens
from .dsv4.v4_pool import DSV4PagedKVCache
model_config = config.model_config
if resolve_pool_class(model_config) is DSV4PagedKVCache:
# DSV4 is driven by the generic CacheManager over the shared page table; the pool is
# the only DSV4-specific piece (the swa_pool plug-in: window tier + cmp/idx/state
# shadows). Sizing reads dsv4_args, never the group spec.
pool = DSV4PagedKVCache(
sizes=_dsv4_pool_sizes(config, num_pages + 1), # +1 for dummy page
args=model_config.dsv4_args,
device=device,
dtype=dtype,
P=model_config.dsv4_args.window_size,
n_scratch=config.max_running_req + 1,
)
pool._init_paged_state(config.max_running_req, config.cache_type != "naive")
return pool
if getattr(model_config, "dsv41_args", None) is not None:
from .dsv4.v41_cost_model import _dsv41_pool_sizes
from .dsv4.v41_pool import DSV41PagedKVCache
# Same route as DSV4: the generic CacheManager over the shared page table; the pool is the
# swa_pool plug-in (window tier + per-source packed main / index pools + state rings).
pool = DSV41PagedKVCache(
sizes=_dsv41_pool_sizes(config, num_pages + 1), # +1 for dummy page
args=model_config.dsv41_args,
device=device,
dtype=dtype,
n_scratch=config.max_running_req + 1,
)
pool._init_paged_state(config.max_running_req, config.cache_type != "naive")
return pool
num_swa_tokens = None
# Both the naive and radix SWA paths share the global-paged swa pool; radix sizes it by
# ratio (cross-request reuse), naive by concurrency x window.
if model_config.has_swa_attention:
num_swa_tokens = (
_swa_paged_num_tokens(config, num_pages + 1)
if config.cache_type == "swa_radix"
else _naive_swa_num_tokens(config)
)
return create_kvcache_pool(
model_config=model_config,
num_pages=num_pages + 1, # +1 for dummy page
page_size=config.page_size,
num_swa_tokens=num_swa_tokens,
device=device,
dtype=dtype,
num_req_slots=config.max_running_req + 1, # + 1 for the dummy request row
)
def create_kvcache_pool(
model_config: ModelConfig,
num_pages: int,
page_size: int,
dtype: torch.dtype,
device: torch.device,
num_swa_tokens: int | None = None,
num_req_slots: int | None = None,
) -> BaseKVCachePool:
if model_config.has_swa_attention:
from .hybrid_swa_pool import HybridSWAKVCache
return HybridSWAKVCache(
groups=model_config.kv_cache_group_specs(),
num_layers=model_config.num_layers,
num_full_pages=num_pages,
page_size=page_size,
num_swa_tokens=num_swa_tokens,
device=device,
dtype=dtype,
)
from .mha_pool import MHAKVCache
# Hybrid linear-attention models only store paged KV for their non-linear
# layers; the linear layers keep a separate recurrent state (LinearStatePool).
# The linear group emits no paged spec, so the remaining paged spec(s) drive
# the dispatch below; the paged pool backs JUST those layers via a global-id
# -> dense-slot remap (layer_ids) so the (majority) linear layers cost no
# slabs.
layer_ids: tuple[int, ...] | None = None
kv_specs = [s for s in model_config.kv_cache_group_specs() if s.num_layers > 0]
if model_config.has_linear_attention:
assert len(kv_specs) == 1, (
f"hybrid-linear models support one paged-KV group, got "
f"{[s.name for s in kv_specs]}"
)
layer_ids = kv_specs[0].layer_ids
# Latent-KV MLA / GQA block-sparse models declare their geometry on the single
# paged spec; the same spec fields drive the KV cost model, so the factory and
# the budget can never disagree.
from freetoken.attention import AttnType as _AttnType
if len(kv_specs) == 1 and kv_specs[0].attn_type == _AttnType.BSA:
from .bsa_pool import BSAKVCache
spec = kv_specs[0]
assert layer_ids is None, "hybrid-linear x BSA has no pool support yet"
return BSAKVCache(
num_kv_heads=spec.num_kv_heads,
num_layers=model_config.num_layers,
head_dim=spec.head_dim,
num_pages=num_pages,
page_size=page_size,
dtype=dtype,
device=device,
index_head_dim=spec.index_head_dim,
num_index_layers=spec.num_index_layers,
)
# QSA (Qwen3.8-Flash-Next): the same GQA group, but it stores one index key per
# index_ratio tokens and adds per-request tiers sized by the concurrency, not the pages.
# layer_ids is mandatory here -- the model is hybrid-linear, and letting MHAKVCache back
# all num_layers would allocate K/V slabs for the GDN layers too.
if len(kv_specs) == 1 and kv_specs[0].attn_type == _AttnType.QSA:
from .qsa_pool import QSAKVCache
spec = kv_specs[0]
if num_req_slots is None:
raise ValueError("QSA pools need num_req_slots (max_running_req + 1)")
return QSAKVCache(
num_kv_heads=spec.num_kv_heads,
num_layers=model_config.num_layers,
head_dim=spec.head_dim,
num_pages=num_pages,
page_size=page_size,
dtype=dtype,
device=device,
index_head_dim=spec.index_head_dim,
num_index_layers=spec.num_index_layers,
index_ratio=spec.index_ratio,
num_req_slots=num_req_slots,
layer_ids=spec.layer_ids,
mrope=model_config.model_is_mrope,
)
if len(kv_specs) == 1 and kv_specs[0].mla:
from .dsa_pool import DSAKVCache, KpoolDSAKVCache, MLAKVCache
spec = kv_specs[0]
# With a layer remap the pool allocates len(layer_ids) slabs; without one
# it backs every model layer (all-MLA models, GLM-5.2).
num_layers = model_config.num_layers if layer_ids is None else len(layer_ids)
if spec.index_head_dim > 0 and spec.num_index_layers > 0:
common = dict(
latent_dim=spec.head_dim,
num_layers=num_layers,
num_pages=num_pages,
page_size=page_size,
dtype=dtype,
device=device,
index_head_dim=spec.index_head_dim,
num_index_layers=spec.num_index_layers,
layer_ids=layer_ids,
)
if spec.index_ratio > 1:
# kpool tail rings are keyed by Req.table_idx; + 1 covers the dummy request row.
if num_req_slots is None:
raise ValueError("kpool pools need num_req_slots (max_running_req + 1)")
return KpoolDSAKVCache(
**common,
index_ratio=spec.index_ratio,
num_req_slots=num_req_slots,
)
return DSAKVCache(**common)
return MLAKVCache(
latent_dim=spec.head_dim,
num_layers=num_layers,
num_pages=num_pages,
page_size=page_size,
dtype=dtype,
device=device,
layer_ids=layer_ids,
)
spec = kv_specs[0] if len(kv_specs) == 1 else None
return MHAKVCache(
num_kv_heads=spec.num_kv_heads if spec is not None else model_config.num_kv_heads,
num_pages=num_pages,
page_size=page_size,
num_layers=model_config.num_layers,
head_dim=spec.head_dim if spec is not None else model_config.head_dim,
device=device,
dtype=dtype,
layer_ids=layer_ids,
)
@SUPPORTED_CACHE_MANAGER.register("naive")
def create_naive_cache(device: torch.device, page_size: int | None = None):
from .naive_cache import NaivePrefixCache
return NaivePrefixCache(device=device) # naive has no page arithmetic
@SUPPORTED_CACHE_MANAGER.register("radix")
def create_radix_cache(device: torch.device, page_size: int | None = None):
from .radix_cache import RadixPrefixCache
return RadixPrefixCache(device=device, page_size=page_size)
# NOTE: "hybrid_radix" is NOT registered as a user-facing --cache-type. It is the internal
# materialization of "radix" for hybrid GDN models (cross-request GDN-state reuse), produced by
# _resolve_cache_type and built directly in CacheManager._make_prefix_cache (HybridRadixCache
# needs page_size). Users pick "radix" (the concept) or "naive"; the engine picks hybrid_radix.
def create_prefix_cache(
device: torch.device, type: str, page_size: int | None = None
) -> BasePrefixCache:
return SUPPORTED_CACHE_MANAGER[type](device, page_size=page_size)
__all__ = [
"create_kv_pool",
"create_kvcache_pool",
"create_prefix_cache",
"resolve_pool_class",
"BaseKVCachePool",
"BaseCacheHandle",
"BasePrefixCache",
"SizeInfo",
"MatchResult",
"SUPPORTED_CACHE_MANAGER",
]