FreeToken brings datacenter-scale model serving to your desktop. Run massive models locally, fast and efficiently.
from __future__ import annotations
import copy
from dataclasses import dataclass, field, replace
from functools import cached_property
from typing import TYPE_CHECKING, Any, List, Mapping
import torch
from freetoken.distributed import DistributedInfo
from freetoken.layers.quantization import set_quant_config
from freetoken.mm.config import ENCODER_SECTIONS, MultimodalConfig
from freetoken.models.register import EncoderSpec, ModelSpec, _load_attr, checkpoint_quant_config, get_model_spec
from freetoken.utils import cached_load_hf_config, init_logger
if TYPE_CHECKING:
from freetoken.models import ModelConfig
logger = init_logger(__name__)
@dataclass(frozen=True)
class EngineConfig:
model_path: str
tp_info: DistributedInfo
dtype: torch.dtype
# --hf-overrides: applied to the checkpoint config the model is built from (cached_load_hf_config)
hf_overrides: Mapping[str, Any] = field(default_factory=dict)
max_running_req: int = 4
attention_backend: str = "auto"
moe_strategy: str = "auto"
# old name of moe_strategy; __post_init__ folds it in
moe_backend: str | None = field(default=None, repr=False)
# --quant-backend: layer[.kind]=kernel entries, comma separated
quant_backend: str | None = None
# PLE table backend: "disk" (default) reads rows from the checkpoint files per fill, "pinned" preloads the table into page-locked host RAM.
ple_backend: str = "disk"
# --embed-weights: "gpu" (default) keeps the input embedding tables in VRAM; "host" keeps the untied ones in pinned host RAM, read zero-copy by the lookup.
embed_weights: str = "gpu"
# DeepSeek-V4.1 Decoder SWA Bounded Replay: "bounded" (default) runs the decoder layers on each
# prompt's last window; "exact" runs them on every token.
swa_decoder_replay: str = "bounded"
# Expert-bank host load (--expert-load): auto|serial|parallel. "auto" reads scattered
# experts in parallel but falls back to serial when free RAM can't cover the banks + the
# parallel reader's extra (non-reclaimable) whole-shard buffer; "serial" forces the
# low-memory reclaimable read; "parallel" forces the fast read.
expert_load: str = "auto"
moe_cache_size: int = 0
moe_cache_rate: float | None = None
moe_cache_auto: bool = False
# --moe-cache-mem: slot-cache budget in bytes, turned into a slot count once the expert banks are loaded
moe_cache_mem: int | None = None
# --moe-cache-scope: unified (one LRU pool per expert quant) or per-layer (one pool per layer)
moe_cache_scope: str = "unified"
# --moe-cache-split: how pools share the slots; equal or profile:<path> (by expert use from a profile file)
moe_cache_split: str = "equal"
kv_reserve_tokens: int = 8192 # KV floor for --moe-cache-auto; small by design (MoE-priority)
moe_cache_policy: str = "lru"
moe_prefill_overlap: bool = True
# Prefill hit/miss split: serve cache-resident experts D2D during prefill
# prefetch instead of re-streaming the full layer over PCIe. Needs CUDA >= 13.0
# (cudaMemcpyBatchAsync) or HIP (hipMemcpyBatchAsync);
# no-op unless moe_cache_size > 2 * num_experts.
moe_prefill_hit_d2d: bool = False
moe_collect_stats: bool = False # capture decode miss-rate counters into the cuda graph
# CPU MoE backend (--moe-strategy cpu): number of CPU worker threads computing
# the decode experts. 0 = auto (physical cores). Ignored by other backends.
moe_cpu_threads: int = 0
# Hybrid CPU/GPU decode (--moe-strategy offload only): which MoE layers decode on
# the CPU executor instead of the GPU offload/PCIe path. Spec is an explicit id
# list ("3,7,11"), a count ("8" -> 8 layers evenly strided across depth), or a
# fraction ("0.5"). None/"" = all layers on GPU (plain offload). --moe-strategy cpu
# already means all layers on CPU and ignores this.
moe_cpu_layers: str | None = None
# Hybrid MoE backend (--moe-strategy hybrid): max experts fetched over PCIe per
# (layer, decode step); the rest of that step's misses are computed on the CPU.
# -1 (default) = auto: fetch the benched pcie_bw/cpu_bw fraction of each step's
# misses so the PCIe fetch and the CPU compute finish together (perfect overlap);
# falls back to a fixed cap of 1 without a usable `ft bench bw` profile.
moe_hybrid_max_fetch: int = -1
# Hybrid with locked layers (banks the pin quota could not take, OS-locked instead): a locked layer keeps GPU
# slots of its own pools and fetches this share of each decode step's misses through pinned staging; the rest
# are computed on the CPU. -1 (default) = auto: per layer from its `ft bench bw` row (bench_profile.locked_fetch_fraction),
# a fixed cap of 1 without one. 0 = locked layers decode on the CPU only. moe_locked_max_fetch caps the fetches per
# (layer, step) and sizes the staging (that many rows of the widest expert, per bank).
moe_locked_fetch: float = -1.0
moe_locked_max_fetch: int = 8
cuda_graph_bs: List[int] | None = None
cuda_graph_max_bs: int | None = None
page_size: int = 1
memory_ratio: float = 0.9
# Hybrid GDN models default to the HybridRadixCache (cross-request GDN-state prefix reuse);
# `--cache-type naive` opts out. linear_state_cache_ratio sizes the GDN snapshot cache as
# ceil(ratio * max_running_req) extra slots.
linear_state_cache_ratio: float = 2.0
# Window/full ratio for the SWA radix cache (`--cache-type radix` on SWA models) and the DSV4
# window tier: the DEFAULT window-pool size = max(working-set floor, ratio x full-pool tokens).
# < 1.0 trades retained window-prefix capacity for memory savings; must be in (0, 1]. It is the
# DSV4 window/full ratio directly. Used only when swa_num_pages_override is None (a runtime
# rebuild can pin an absolute window instead).
swa_full_tokens_ratio: float = 0.2
# Absolute window-pool size in the pool's own pages (usable, dummy excluded); None -> use the
# ratio default above. A runtime cache rebuild sets this (num_swa_pages) to pin the window
# regardless of the full anchor; the ratio is the startup default and the fallback.
swa_num_pages_override: int | None = None
distributed_timeout: float = 60.0
use_dummy_weight: bool = False
use_pynccl: bool = True
max_seq_len_override: int | None = None
num_page_override: int | None = None # if not None, will override the number of pages
# KV capacity in tokens; resolved into num_page_override by _adjust_config once page_size
# is final. Mutually exclusive with num_page_override.
num_token_override: int | None = None
# Runtime knobs of the multimodal path; the architecture side (vision_config, mrope) lives in ModelConfig.
mm: MultimodalConfig = field(default_factory=MultimodalConfig)
def __post_init__(self):
if not (self.moe_locked_fetch == -1 or 0.0 <= self.moe_locked_fetch <= 1.0):
raise ValueError(f"moe_locked_fetch must be -1 (auto) or in [0, 1], got {self.moe_locked_fetch}")
if self.moe_locked_fetch > 0 and round(self.moe_locked_fetch * (1 << 16)) == 0:
# the ensure kernel takes the share in Q16: this one rounds to 0 and the layers would never fetch
raise ValueError(f"moe_locked_fetch {self.moe_locked_fetch} rounds to 0 in Q16; pass 0 to keep locked layers CPU-only")
if self.moe_locked_max_fetch < 1:
raise ValueError(f"moe_locked_max_fetch must be at least 1, got {self.moe_locked_max_fetch}")
if self.moe_backend is None:
return
if self.moe_strategy != "auto":
raise ValueError("moe_backend is the old name of moe_strategy; pass only moe_strategy")
logger.warning("EngineConfig.moe_backend is deprecated; use moe_strategy")
object.__setattr__(self, "moe_strategy", self.moe_backend)
object.__setattr__(self, "moe_backend", None)
@cached_property
def hf_config(self):
return cached_load_hf_config(self.model_path, self.hf_overrides)
@cached_property
def model_spec(self) -> ModelSpec:
return get_model_spec(self.hf_config.architectures[0])
@cached_property
def active_encoders(self) -> tuple[EncoderSpec, ...]:
"""The encoder towers this process builds: the family registers them, the checkpoint config carries their section, --mm-disable did not name them."""
return tuple(
e
for e in self.model_spec.encoders
if getattr(self.hf_config, e.config_key, None) is not None
and e.kind not in self.mm.disabled_encoders
)
@cached_property
def served_modalities(self) -> frozenset[str]:
"""Modalities this process accepts."""
return frozenset(m for e in self.active_encoders for m in e.modalities)
@cached_property
def model_config(self) -> ModelConfig:
# the parser sees no section for a tower this process does not build (for the vision tower that also means 1-D rope)
hf_config = copy.copy(self.hf_config)
built = {e.config_key for e in self.active_encoders}
for key in set(ENCODER_SECTIONS) | {e.config_key for e in self.model_spec.encoders}:
if key not in built and getattr(hf_config, key, None) is not None:
setattr(hf_config, key, None)
spec = self.model_spec
quant = checkpoint_quant_config(self.model_path, hf_config, spec)
set_quant_config(quant)
model_config = _load_attr(spec.module, spec.parse_config)(hf_config)
return replace(model_config, quant=quant)
@property
def max_seq_len(self) -> int:
if self.max_seq_len_override is not None:
return self.max_seq_len_override
return self.model_config.rotary_config.table_positions
@property
def max_forward_len(self) -> int:
return self.max_seq_len
@property
def decode_rows_per_req(self) -> int:
"""Query rows each running request takes per decode step; every per-request decode allowance derives from it."""
return 1
@property
def distributed_addr(self) -> str:
return "tcp://127.0.0.1:2333"