FreeToken brings datacenter-scale model serving to your desktop. Run massive models locally, fast and efficiently.
from __future__ import annotations
import collections
import contextlib
import errno
import gc
import math
import os
from datetime import timedelta
from typing import Any, Dict, Iterable, NamedTuple, Tuple
import torch
from freetoken.attention import AttnType, attention_backend_info, create_attention_backend
from freetoken.attention.linear import LinearAttnBackend
from freetoken.core import Batch, Context, Req, set_global_ctx
from freetoken.distributed import destroy_distributed, enable_pynccl_distributed, set_tp_info
from freetoken.gpu_select import gpu_identity
from freetoken.checkpoint.ftw import ftw_tensor_names, is_ftw_checkpoint
from freetoken.layers import OffloadMoELayer, VocabParallelEmbedding, iter_moe_layers, iter_ops, set_rope_device
from freetoken.layers.quantization import LayerKind, QuantBackend, QuantKind, finalize_quant, set_quant_backend
from freetoken.moe.offload_cache import iter_offload_moe_layers
from freetoken.mm.config import ENCODER_SECTIONS
from freetoken.models import create_model, load_weight
from freetoken.models.weight import ftw_lacks_vision
from freetoken.moe import is_offload_moe_strategy
from freetoken.moe.expert_banks import attach_resident_banks, load_expert_banks, pin_probe_enabled
from freetoken.moe.host_banks import PinFailed
from freetoken.moe.locked_fetch import attach_locked_fetch, check_locked_fetch_config, locked_pool_tags, settle_banks, staging_reserve
from freetoken.moe.offload_cache import OffloadMoeCache, attach_offload_moe_cache, bank_layouts
from freetoken.moe.pool_policy import group_layers, make_pools, pool_of_layer, resolve_profile, split_slots
from freetoken.utils import align_ceil, init_logger, is_sm90_family, is_sm100_family, mem_GB, torch_dtype
from .config import EngineConfig
from .graph import GraphRunner, get_free_memory
from .sample import BatchSamplingArgs, Sampler
from freetoken.kvcache import create_kv_pool, resolve_pool_class
from freetoken.kvcache.base import CacheRebuildRejected
from freetoken.kvcache.cache_status import _supports_swa_ratio
from freetoken.kvcache.linear_state_pool import (
_linear_pool_min_slots, _linear_pool_num_slots, state_pool_bytes,
)
logger = init_logger(__name__)
def _clamp_decode_batch_to_pools(config: EngineConfig, pool_sizes: list[int]) -> None:
"""Clamp the decode batch so every GPU-decoded layer's routed experts fit in its pool."""
num_experts = config.model_config.num_experts
small = [s for s in pool_sizes if s < num_experts]
if not small:
return
# a decode step routes every row of every request, decode_rows_per_req rows each
limit = min(small) // (config.model_config.num_experts_per_tok * config.decode_rows_per_req)
if config.max_running_req > limit:
logger.warning_rank0(
f"the smallest MoE slot pool has {min(small)} slots; lowering max_running_req "
f"{config.max_running_req} -> {limit} so a decode step fits in it"
)
object.__setattr__(config, "max_running_req", limit)
if config.cuda_graph_max_bs is None or config.cuda_graph_max_bs > limit:
object.__setattr__(config, "cuda_graph_max_bs", limit)
if config.cuda_graph_bs is not None:
object.__setattr__(config, "cuda_graph_bs", [bs for bs in config.cuda_graph_bs if bs <= limit])
def _require_offload_cache_size(cache_size: int, num_experts: int) -> None:
"""The offload MoE cache needs at least one slot per expert per layer. A too-small size
(e.g. a bare offload run with moe_cache_size unset and auto disabled) must fail loudly."""
if cache_size < num_experts:
raise ValueError(
f"moe_cache_size={cache_size} is too small: need at least num_experts={num_experts} "
f"slots. Pass --moe-cache-size/--moe-cache-rate, or use --moe-cache-auto "
f"(the default for offload/hybrid backends when no cache-sizing flag is given; "
f"--moe-strategy cpu always sizes its own fixed two-layer buffer and ignores "
f"cache-sizing flags)."
)
def _flashinfer_available() -> bool:
from freetoken.kernel.backend import is_flashinfer_installed
return is_flashinfer_installed()
def _sgl_flash_attn_available() -> bool:
from freetoken.kernel.backend import is_rocm
if is_rocm():
return False
try:
from sgl_kernel.flash_attn import flash_attn_with_kvcache # noqa: F401
except Exception as exc:
detail = next((line.strip() for line in str(exc).splitlines() if line.strip()), "")
logger.warning_rank0(
"sgl_kernel.flash_attn is unavailable; auto attention backend falls back to fi "
f"({type(exc).__name__}: {detail})"
)
return False
return True
def _startup_kv_budget(memory_ratio: float, init_free_memory: int, new_free_memory: int) -> int:
"""Bytes available to the KV pool at startup: ratio-scaled pre-load free memory minus
what the resident model consumed. Kept as a pure function so the composition with the
pool families' ``solve_num_pages`` stays CPU-testable."""
return int(memory_ratio * init_free_memory) - (init_free_memory - new_free_memory)
def _page_table_width(max_seq_len: int, page_size: int) -> int:
"""Column count for the page table. ``_write_page_table`` writes WHOLE trailing pages, so the
highest column touched is ``align_ceil(max_seq_len, page_size) - 1`` -- which the 32-alignment
alone does not cover once page_size > 32 (an unaligned --max-seq-len-override on DSV4's P=128
or trtllm's forced 64 would index past the row)."""
return align_ceil(align_ceil(max_seq_len, page_size), 32)
def _seq_limits(config, num_tokens: int) -> tuple[int, int]:
"""(longest request, page-table width). A decode step may widen a request by decode_rows_per_req - 1 provisional rows past its length, so requests get that much less and the table still covers the widened tail."""
limit = min(config.max_seq_len, num_tokens)
return limit - (config.decode_rows_per_req - 1), _page_table_width(limit, config.page_size)
def _required_attn_types(model_config) -> frozenset[AttnType]:
"""Backend-driving attention types of this model, from the group-spec walk
(single source shared with the pool factory and the KV cost model). getattr
fallbacks: duck-typed test configs may not implement the spec walk; for those,
dsv4_args marks DSV4 (the real config declares a DSV4 attention group)."""
specs_fn = getattr(model_config, "kv_cache_group_specs", None)
if specs_fn is None:
if getattr(model_config, "dsv4_args", None) is not None:
return frozenset({AttnType.DSV4})
return frozenset({AttnType.FULL})
types = frozenset(
spec.attn_type for spec in specs_fn() if spec.attn_type.backend_driven
)
return types or frozenset({AttnType.FULL})
def _backend_parts_serve(name: str, required: frozenset[AttnType]) -> bool:
return all(
required <= attention_backend_info(part).supported_types
for part in name.split(",")
)
def _backend_requirements_met(name: str) -> bool:
# flashinfer first across ALL parts: the sgl probe logs a "falls back to fi" warning,
# which would mislead when the candidate is about to fail on flashinfer anyway.
infos = [attention_backend_info(part) for part in name.split(",")]
if any(i.requires_flashinfer for i in infos) and not _flashinfer_available():
return False
if any(i.requires_sgl_kernel for i in infos) and not _sgl_flash_attn_available():
return False
if any(i.requires_sm100 for i in infos) and not is_sm100_family():
return False
return True
def _resolve_auto_attention_backend(required: frozenset[AttnType]) -> str:
"""First candidate (in per-type priority order) whose arch condition holds,
whose packages are installed, and whose every comma part serves ALL required
types. Reproduces the historical hardware tree for FULL-only models:
sm_100 -> trtllm, sm_90+sgl_kernel -> "fa,fi", flashinfer -> fi, else triton."""
candidates: list[tuple[str, bool]] = []
if AttnType.DSV4 in required:
candidates.append(("dsv4_sparse", True))
if AttnType.DSV41 in required:
candidates.append(("dsv41_sparse", True))
if required & {AttnType.MLA, AttnType.DSA}:
candidates.append(("dsa", True))
if AttnType.BSA in required:
candidates.append(("m3_sparse", True))
if AttnType.QSA in required:
candidates.append(("qsa_sparse", True))
if AttnType.SWA in required:
candidates.append(("triton", True))
if AttnType.FULL in required:
candidates += [
("trtllm", is_sm100_family()),
("fa,fi", is_sm90_family()),
("fi", True),
("triton", True),
]
for name, arch_ok in candidates:
if not arch_ok:
continue
if not _backend_parts_serve(name, required):
continue
if not _backend_requirements_met(name):
continue
return name
raise RuntimeError(
"No attention backend can serve attention types "
f"{sorted(t.value for t in required)} on this machine."
)
def _validate_attention_backend_choice(config, override, required: frozenset[AttnType]) -> None:
"""Config-time type x backend capability check for the resolved (or explicit)
backend string: every comma part must serve every required type and have its
packages/arch available. Replaces the per-model gates; in particular this is
where a DSV4 or MLA checkpoint rejects a generic backend before weights load,
and where a generic model rejects dsa/dsv4_sparse."""
from freetoken.attention import validate_attn_backend
from freetoken.kernel.backend import is_rocm
# Name membership first (ArgumentTypeError listing the supported names): the CLI already
# ran this, but the programmatic EngineConfig path reaches here unvalidated and would
# otherwise die on a bare KeyError from the info lookup below.
validate_attn_backend(config.attention_backend, allow_auto=False)
model_config = config.model_config
backend_parts = [p.strip() for p in config.attention_backend.split(",")]
for part in backend_parts:
info = attention_backend_info(part)
missing = required - info.supported_types
if missing:
valid = [
name
for name in (
"fa", "fi", "trtllm", "triton", "dsa", "dsv4_sparse", "m3_sparse",
"qsa_sparse",
)
if required <= attention_backend_info(name).supported_types
]
missing_names = "/".join(sorted(t.value for t in missing))
raise ValueError(
f"{getattr(model_config, 'model_type', 'model')} uses {missing_names} "
f"attention, which backend {part!r} does not support; valid backends: "
f"{', '.join(valid)} (or auto), got {config.attention_backend!r}."
)
if AttnType.SWA in required and not info.consumes_attn_spec:
# SWA models drive window/sinks/sm_scale through the per-call AttentionSpec;
# a backend that drops it would attend with the wrong window silently.
raise ValueError(
f"backend {part!r} does not consume the per-call AttentionSpec that "
f"SWA models require, got {config.attention_backend!r}."
)
# An explicitly-selected backend may require a package that isn't installed. Auto
# never resolves to one of these when its package is missing, so this only fires for
# explicit --attention-backend choices.
for part in backend_parts:
info = attention_backend_info(part)
if info.requires_flashinfer and not _flashinfer_available():
if is_rocm():
raise RuntimeError(
f"Attention backend {part!r} uses CUDA-only FlashInfer kernels "
"unavailable on ROCm; use --attention-backend auto or triton."
)
raise RuntimeError(
f"Attention backend {config.attention_backend!r} requires flashinfer, which is "
"not installed. Install it with `pip install 'freetoken[fi]'` (or "
"'freetoken[accel]'), or use --attention-backend triton."
)
if info.requires_sgl_kernel and not _sgl_flash_attn_available():
if is_rocm():
raise RuntimeError(
f"Attention backend {part!r} uses CUDA-only sgl_kernel kernels "
"unavailable on ROCm; use --attention-backend auto or triton."
)
raise RuntimeError(
f"Attention backend {config.attention_backend!r} requires sgl_kernel, which is "
"not installed. Install it with `pip install 'freetoken[sgl]'` (or "
"'freetoken[accel]'), or use --attention-backend triton."
)
if info.requires_sm100 and not is_sm100_family():
raise RuntimeError(
f"Attention backend {config.attention_backend!r} requires a compute capability "
"10.x GPU: flashinfer's trtllm-gen kernels ship sm_100a/103a cubins only. "
"Use --attention-backend fi (or triton) instead."
)
if required & {AttnType.MLA, AttnType.DSA}:
# Plain MLA/DSA runs on page_size 1; the kpool indexer layout needs 64.
_kpool_ratio = max(
(s.index_ratio for s in model_config.kv_cache_group_specs() if s.mla),
default=1,
)
want_page = 64 if _kpool_ratio > 1 else 1
if config.page_size != want_page:
logger.warning_rank0(
f"Page size {config.page_size} is auto-adjusted to {want_page} "
f"for latent-KV attention."
)
override("page_size", want_page)
for part in backend_parts:
info = attention_backend_info(part)
if info.page_sizes is not None and config.page_size not in info.page_sizes:
override("page_size", info.page_sizes[-1])
logger.warning_rank0(
f"Page size is overridden to {info.page_sizes[-1]} for the {part} backend"
)
def _host_embedding_keys(model, config: EngineConfig) -> frozenset[str]:
"""State-dict keys of the tables --embed-weights host keeps in pinned host RAM."""
if config.embed_weights != "host":
return frozenset()
if config.model_config.tie_word_embeddings:
raise ValueError(
"--embed-weights host: this model's embedding table is tied to its LM head, which reads the whole table "
"every step; serve it with --embed-weights gpu"
)
tables = {id(op.weight) for op in iter_ops(model) if type(op) is VocabParallelEmbedding}
keys = frozenset(key for key, t in model.state_dict().items() if id(t) in tables)
if not keys:
raise ValueError(
f"--embed-weights host: {type(model).__name__} has no plain embedding table to move (a DF11 table stays "
"on the GPU); serve it with --embed-weights gpu"
)
return keys
def _pinned_copy(key: str, t: torch.Tensor) -> torch.Tensor:
from freetoken.kernel.pinned import alloc_pinned_tensor
try:
host = alloc_pinned_tensor(*t.shape, dtype=t.dtype)
except RuntimeError as exc:
raise PinFailed(
f"cudaHostAlloc refused {t.nbytes / 2**20:.0f} MiB for {key} (--embed-weights host); "
"free host RAM or serve with --embed-weights gpu"
) from exc
host.copy_(t)
return host
def _make_dummy_weight_state_dict(
model_state: Dict[str, torch.Tensor],
*,
device: torch.device,
host_keys: frozenset[str] = frozenset(),
) -> Dict[str, torch.Tensor]:
state_dict: Dict[str, torch.Tensor] = {}
fp8_dtypes = (torch.float8_e4m3fn, torch.float8_e5m2)
for key, param in model_state.items():
if param.dtype in fp8_dtypes:
# torch.randn is not implemented for fp8; fill via a uint8 view with small
# codes (avoid NaN/inf fp8 encodings). Lets dummy-weight startup work for
# block-fp8 models (the dense fp8 linears are fp8 regardless of moe_strategy).
t = torch.empty(param.shape, dtype=param.dtype, device=device)
t.view(torch.uint8).random_(0, 16)
state_dict[key] = t
elif param.dtype.is_floating_point or param.dtype.is_complex:
state_dict[key] = torch.randn(param.shape, dtype=param.dtype, device=device)
elif param.dtype == torch.uint8 and key.endswith("weight_scale_inv"):
# MXFP8 e8m0 exponent codes: 127 encodes scale 1.0; zeros would collapse
# every scale to 2^-127 and zero the model. Scoped BY NAME: other uint8
# buffers are packed payloads whose bytes mean something else entirely
# (GGUF blocks embed fp16 scales -- 0x7F7F is fp16 NaN), so they
# keep the benign all-zeros fill below.
state_dict[key] = torch.full(param.shape, 127, dtype=param.dtype, device=device)
else:
state_dict[key] = torch.zeros(param.shape, dtype=param.dtype, device=device)
if key in host_keys:
state_dict[key] = _pinned_copy(key, state_dict[key])
return state_dict
class WeightLoadError(RuntimeError):
"""The checkpoint itself could not be read. Resource and config failures keep their own type."""
def _is_resource_failure(exc: Exception) -> bool:
if isinstance(exc, (torch.OutOfMemoryError, MemoryError, PinFailed)):
return True
# an anonymous mmap that does not fit raises ENOMEM, not MemoryError
if isinstance(exc, OSError) and exc.errno == errno.ENOMEM:
return True
# torch has no type for a failed CPU allocation
return isinstance(exc, RuntimeError) and "DefaultCPUAllocator" in str(exc)
@contextlib.contextmanager
def _weight_load_context():
"""Wrap a checkpoint read so the startup failure reason (log and /health) starts with WeightLoadError."""
try:
yield
except Exception as exc:
if _is_resource_failure(exc):
raise
raise WeightLoadError(f"{type(exc).__name__}: {exc}") from exc
def _materialize_loaded_weight_state_dict(
model_state: Dict[str, torch.Tensor],
weights: Iterable[Tuple[str, torch.Tensor]],
*,
device: torch.device,
host_keys: frozenset[str] = frozenset(),
) -> Dict[str, torch.Tensor]:
state_dict: Dict[str, torch.Tensor] = {}
for key, weight in weights:
expected = model_state.get(key)
if expected is None:
# NOTE: the quant scheme may declare no input_scale for a layer whose FTW still stores one
if key.endswith(".input_scale"):
continue
state_dict[key] = weight.to(device=device)
elif key in host_keys:
state_dict[key] = _pinned_copy(key, weight.to(dtype=expected.dtype))
else:
state_dict[key] = weight.to(device=device, dtype=expected.dtype)
return state_dict
class ForwardOutput(NamedTuple):
next_tokens_gpu: torch.Tensor
next_tokens_cpu: torch.Tensor
copy_done_event: torch.cuda.Event
class Engine:
def __init__(self, config: EngineConfig):
assert not torch.cuda.is_initialized()
set_tp_info(rank=config.tp_info.rank, size=config.tp_info.size)
set_quant_backend(_adjust_ftw_quant_backend(config.model_path, QuantBackend.parse(config.quant_backend)))
_ensure_expandable_segments() # before the first CUDA allocation below
from freetoken.gpu_select import bind_assigned_gpu
self.device = bind_assigned_gpu(config.tp_info.rank)
_adjust_config(config)
torch.manual_seed(42)
self.stream = torch.cuda.Stream()
torch.cuda.set_stream(self.stream)
self.dtype = config.dtype
self.config = config # retained for runtime cache rebuild (rebuild_runtime_cache)
# KV pool family fixed at construction from the model config: its classmethods own the
# page-token geometry and cost arithmetic the engine needs BEFORE the pool exists
# (num_pages sizing, --moe-cache-auto); the instance owns rebuild/validation after.
self._pool_cls = resolve_pool_class(config.model_config)
self.ctx = Context(config.page_size)
set_global_ctx(self.ctx)
self.tp_cpu_group = self._init_communication(config)
free_min, free_max = self._sync_get_memory()
init_free_memory = free_max # startup KV sizing keeps cross-rank MAX (unchanged)
self._baseline_free = free_min # rebuild baseline: cross-rank MIN, deterministic across ranks
logger.info_rank0(f"Free memory before loading model: {mem_GB(init_free_memory)}")
# ======================= Model initialization ========================
set_rope_device(self.device)
with torch.device("meta"), torch_dtype(config.dtype):
self.model = create_model(config.model_config)
self._load_weights(config)
if config.active_encoders:
from freetoken.models.blocks import SupportsMultimodal
if not isinstance(self.model, SupportsMultimodal):
raise TypeError(
f"{type(self.model).__name__} has encoders registered but lacks the SupportsMultimodal hooks; "
"run with --text-model-only"
)
# before the residency snapshot, so streamed blocks are not charged as resident weights
self.model.place_encoder_weights(config.mm.encoder_weights)
post_weights_free = self._sync_get_memory()[0]
self._weights_bytes = self._baseline_free - post_weights_free
# Pool-budget baseline for the desktop cache sliders: free VRAM after the weights are
# resident but before ANY runtime cache pool (MoE expert cache below, KV pages, GDN
# state) is allocated. This is the stable "if all free VRAM went to one pool" budget —
# unlike a query-time mem_get_info it doesn't drift with allocator caching, CUDA
# graphs, or other processes. Cross-rank MIN, deterministic across ranks.
self._post_weights_free = post_weights_free
self.moe_offload_cache = None
self.cpu_moe_executor = None
# Host-side auxiliary stores (--embed-weights host tables, qwen4_exp's pinned PLE table): after the weights so a
# load failure is not masked, before the MoE offload cache so the bank residency
# planning sees the pin quota the table already spent.
host_embeds = [op for op in iter_ops(self.model) if type(op) is VocabParallelEmbedding and not op.weight.is_cuda]
self._host_tables_bytes = sum(op.weight.nbytes for op in host_embeds)
if host_embeds:
logger.info_rank0(f"Embedding tables in pinned host RAM (--embed-weights host): {mem_GB(self._host_tables_bytes)}")
if hasattr(self.model, "load_host_tables"):
with _weight_load_context():
self._host_tables_bytes += int(self.model.load_host_tables(config) or 0)
if is_offload_moe_strategy(config.moe_strategy):
self._init_offload_moe_cache(config)
if hasattr(self.model, "prepare_for_runtime"):
self.model.prepare_for_runtime()
self.encoder_cache = None
self.mm_processor = None
if config.active_encoders:
from freetoken.mm.encoder_cache import EncoderCache
from freetoken.mm.processor import get_mm_processor
self.mm_processor = get_mm_processor(config.model_path, config.mm)
self.encoder_cache = EncoderCache(storage=config.mm.embed_cache_device)
logger.info_rank0(
f"Multimodal enabled: {type(self.mm_processor).__name__}, encoders "
f"{[e.kind for e in config.active_encoders]} on {config.mm.encoder_weights}, serving {sorted(config.served_modalities)}"
)
self._warmup_encoders()
elif any(getattr(config.hf_config, key, None) is not None for key in ENCODER_SECTIONS):
logger.info_rank0(
"Multimodal disabled: --text-model-only"
if config.mm.text_model_only
else "Multimodal disabled: --mm-disable"
if config.mm.disabled_encoders
else "Multimodal disabled: no encoder registered for this architecture"
)
# ======================= KV cache initialization ========================
new_free = self._sync_get_memory()[1]
# The engine measures the budget and settles the sibling GDN state pool's bytes
# off it; the KV pool family owns every geometry-specific formula behind the rest.
available_memory = _startup_kv_budget(config.memory_ratio, init_free_memory, new_free)
available_memory -= state_pool_bytes(config)
self.num_pages = self._pool_cls.solve_num_pages(config, available_memory)
num_tokens = self.num_pages * config.page_size
self.ctx.kv_cache = self.kv_cache = create_kv_pool(
config, self.num_pages, device=self.device, dtype=self.dtype
)
# ======================= Linear (GatedDeltaNet) state initialization ========================
linear_group = config.model_config.linear_attention_group()
if linear_group is not None:
from freetoken.kvcache.linear_state_pool import LinearStatePool
self.linear_state_pool = LinearStatePool(
group=linear_group,
num_slots=_linear_pool_num_slots(config),
dtype=self.dtype,
device=self.device,
tp_size=config.tp_info.size,
slot_states=config.model_config.slot_states,
)
self.ctx.linear_state_pool = self.linear_state_pool
else:
self.linear_state_pool = None
# ======================= Page table initialization ========================
# NOTE: 1. aligned to 128 bytes; 2. store raw locations instead of pages
self.max_seq_len, aligned_max_seq_len = _seq_limits(config, num_tokens)
self.ctx.page_table = self.page_table = torch.zeros( # + 1 for dummy request
(config.max_running_req + 1, aligned_max_seq_len),
dtype=torch.int32,
device=self.device,
)
# Pools routed by the shared table but deriving reads through their own mappings (DSV4)
# re-point here (and again on any table realloc). The graph-input snapshot that reads
# through them belongs to the attention backend, built later in init_capture_graph.
self.kv_cache.attach_page_table(self.page_table)
# ======================= Attention backend initialization ========================
self.ctx.attn_backend = self.attn_backend = create_attention_backend(
config.attention_backend, config.model_config
)
self.linear_backend = LinearAttnBackend(self.device) if self.linear_state_pool is not None else None
# ======================= Sampler initialization ========================
self.sampler = Sampler(self.device, config.model_config.vocab_size)
post_free_memory = self._sync_get_memory()[0]
logger.info_rank0(f"Free memory after initialization: {mem_GB(post_free_memory)}")
# ======================= Graph capture initialization ========================
self.dummy_req = Req(
input_ids=torch.tensor([0], dtype=torch.int32, device="cpu"),
table_idx=config.max_running_req,
cached_len=0,
output_len=1,
uid=-1,
sampling_params=None, # type: ignore
cache_handle=None, # type: ignore
)
# padded/dummy rows index the GDN padding slot (0) so gather/scatter hits scratch.
# Naive GDN keys state by table_idx, where slot 0 is a live request and the dummy's own row is the scratch.
if self.linear_state_pool is not None and config.cache_type == "hybrid_radix":
self.dummy_req.linear_slot_idx = self.linear_state_pool.padding_slot
self.page_table[self.dummy_req.table_idx].fill_(num_tokens) # point to dummy page
self.graph_runner = GraphRunner(
stream=self.stream,
device=self.device,
forward=self.model.forward,
attn_backend=self.attn_backend,
cuda_graph_bs=config.cuda_graph_bs,
cuda_graph_max_bs=config.cuda_graph_max_bs,
free_memory=init_free_memory,
max_seq_len=aligned_max_seq_len,
dummy_req=self.dummy_req,
moe_offload_cache=self.moe_offload_cache,
mrope=config.model_config.model_is_mrope,
out_dtype=torch.float32,
linear_backend=self.linear_backend,
)
if config.attention_backend.split(",")[0] == "triton":
# Prefill runs on the first comma part; warm its autotune cache.
self._warmup_prefill()
def _init_communication(self, config: EngineConfig) -> torch.distributed.ProcessGroup:
if config.tp_info.size == 1 or config.use_pynccl:
torch.distributed.init_process_group(
backend="gloo",
rank=config.tp_info.rank,
world_size=config.tp_info.size,
timeout=timedelta(seconds=config.distributed_timeout),
init_method=config.distributed_addr,
)
tp_cpu_group = torch.distributed.group.WORLD
assert tp_cpu_group is not None
max_bytes = (
config.max_forward_len * config.model_config.hidden_size * self.dtype.itemsize
)
enable_pynccl_distributed(config.tp_info, tp_cpu_group, max_bytes)
else:
torch.distributed.init_process_group(
backend="nccl",
rank=config.tp_info.rank,
world_size=config.tp_info.size,
timeout=timedelta(seconds=config.distributed_timeout),
init_method=config.distributed_addr,
)
tp_cpu_group = torch.distributed.new_group(backend="gloo")
assert tp_cpu_group is not None
return tp_cpu_group
def _load_weights(self, config: EngineConfig) -> None:
if config.active_encoders and not config.use_dummy_weight and ftw_lacks_vision(config.model_path):
raise ValueError(
f"{config.model_path} holds no vision encoder tensors: it was converted by a build before this "
"family served images. Reconvert it with `ft checkpoint`, add the encoder in place with "
"scripts/ftw_hotfix.py (docs/ftw-hotfix.md), or start with --text-model-only"
)
resident = [layer for layer in iter_moe_layers(self.model) if not isinstance(layer, OffloadMoELayer)]
if resident and not config.use_dummy_weight and is_ftw_checkpoint(config.model_path) and not ftw_tensor_names(config.model_path, "experts_bank"):
raise ValueError(f"{config.model_path} keeps its routed experts as dense weights (an older `ft checkpoint --moe-backend fused`); reconvert it with `ft checkpoint`")
host_keys = _host_embedding_keys(self.model, config)
with _weight_load_context():
self.model.load_state_dict(self._load_weight_state_dict(config, host_keys))
finalize_quant(self.model)
if resident:
self._load_resident_experts(config, resident)
def _load_weight_state_dict(self, config: EngineConfig, host_keys: frozenset[str]) -> Dict[str, torch.Tensor]:
model_state = self.model.state_dict()
if config.use_dummy_weight:
return _make_dummy_weight_state_dict(model_state, device=self.device, host_keys=host_keys)
# _materialize casts each loaded tensor to its model-param dtype (model_state), so
# models declaring per-tensor dtypes (e.g. DSV4's mixed fp8/fp32/bf16) are preserved;
# MoE models exclude routed experts (they load as expert banks, not dense weights).
return _materialize_loaded_weight_state_dict(
model_state,
load_weight(
config.model_path,
self.device,
include_moe_experts=not getattr(config.model_config, "is_moe", False),
include_vision=bool(config.active_encoders),
),
device=self.device,
host_keys=host_keys,
)
def _load_resident_experts(self, config: EngineConfig, layers) -> None:
with _weight_load_context():
banks = load_expert_banks(
config.model_path,
config.model_config,
method=layers[0].quant_method,
device=self.device,
dummy=config.use_dummy_weight,
parallel={"serial": False, "parallel": True}.get(config.expert_load, None),
resident=True,
layer_methods=[layer.quant_method for layer in sorted(layers, key=lambda layer: layer.layer_id)],
)
attach_resident_banks(layers, banks)
@torch.inference_mode()
def _warmup_encoders(self) -> None:
for item in self.mm_processor.dummy_items(self.dtype, self.device):
if item.modality in self.config.served_modalities:
self.model.encode(item)
torch.cuda.synchronize(self.device)
@torch.inference_mode()
def _run_mm_encoder(self, batch: Batch) -> None:
"""Encode the chunk's cache-miss items and gather its embedding rows into batch.mm_embeds, right before the LM forward."""
cache = self.encoder_cache
jobs = batch.mm_encoder_jobs or ()
# a job whose rows are not gathered this chunk would be encoded and freed unread
planned = {row[1] for row in batch.mm_gather_plan}
orphans = [item.hash for item in jobs if item.hash not in planned]
assert not orphans, f"encoder jobs without gather rows: {orphans}"
for item in jobs:
if not cache.has(item.hash):
if item.precomputed_embeddings is not None:
emb = item.precomputed_embeddings.to(self.device, non_blocking=True)
else:
emb = self.model.encode(item)
cache.put(item.hash, emb)
# cached now; free the ~MiB feature buffer
item.feature = None
item.precomputed_embeddings = None
parts = []
for uid, item_hash, lo, hi, _, _ in batch.mm_gather_plan:
parts.append(cache.get_slice(item_hash, lo, hi, self.device))
cache.consume(item_hash, uid, hi - lo)
if parts:
batch.mm_embeds = torch.cat([p.to(self.dtype) for p in parts], dim=0)
def _resolve_auto_moe_cache_size(self, config: EngineConfig, banks, method=None, max_slots=None) -> tuple[int, int, bool]:
"""Resolve --moe-cache-auto into (moe_cache_size, num_pages, prefill_overlap).
Pure glue over the Phase-1 budget policy; isolated here so it is unit-testable
without a GPU. Reused by the Phase-2 runtime rebuild.
"""
from freetoken.engine.cache_budget import expert_bytes_per_slot, resolve_moe_cache_auto
cache_per_page, fixed_cache_size, page_tokens, min_reserve = self._pool_cls.kv_cost(config)
fixed_cache_size += state_pool_bytes(config) # sibling GDN state pool, engine-summed
num_experts = config.model_config.num_experts
total_experts = config.model_config.num_moe_layers * num_experts
return resolve_moe_cache_auto(
baseline_free=self._baseline_free,
weights_bytes=self._weights_bytes,
memory_ratio=config.memory_ratio,
cache_per_page=cache_per_page,
fixed_cache_size=fixed_cache_size,
per_expert_bytes=expert_bytes_per_slot(banks.sources),
num_experts=num_experts,
total_experts=total_experts,
prefill_overlap=config.moe_prefill_overlap,
kv_reserve_tokens=max(config.kv_reserve_tokens, min_reserve),
page_size=page_tokens,
max_slots=max_slots if max_slots is not None or method is None else method.slot_limit(),
)
def _init_offload_moe_cache(self, config: EngineConfig) -> OffloadMoeCache:
from freetoken.moe.expert_banks import is_mixed
method = shared_offload_method(self.model)
num_moe_layers = config.model_config.num_moe_layers
# experts of several formats: one method per MoE layer, one slot pool per format
layer_methods = offload_layer_methods(self.model)
if not is_mixed(layer_methods):
layer_methods = None
elif config.moe_strategy == "cpu" or config.moe_cpu_layers:
raise ValueError(
"this model's experts come in several formats, which are served from per-format GPU slot pools; "
"they do not combine with --moe-strategy cpu or --moe-cpu-layers (use offload or hybrid)"
)
# Fast path: an FTW checkpoint loads its repacked banks directly.
# Slow path: load_expert_banks auto-picks parallel vs serial baseline by
# expert-tensor granularity. Both pin-after-fill.
# --expert-load: serial/parallel force the read; auto (None) lets load_expert_banks
# pick (parallel for scattered experts, with a low-RAM fallback to serial).
expert_parallel = {"serial": False, "parallel": True}.get(config.expert_load, None)
check_locked_fetch_config(config)
# --moe-cpu-layers auto without a manual pin budget: load unpinned, then pin layers until the driver refuses (settle_banks)
# a model the CPU executor cannot serve takes the estimate path, which rejects it before the load when layers must lock
probe = pin_probe_enabled(config, mixed_formats=layer_methods is not None) and _cpu_moe_executor_viable(config.model_config)
if probe:
from freetoken.moe.host_banks import HostResidency
requested_residency = [HostResidency.DEFERRED.value] * num_moe_layers
cpu_layer_ids, decode_target, split_residency = frozenset(), None, False
else:
reserved = self._host_tables_bytes + staging_reserve(config, lambda: _bank_bytes(config, method, layer_methods))
cpu_layer_ids = _resolve_cpu_layers(config, num_moe_layers, reserved=reserved, method=method)
_check_pin_budget(config, reserved=reserved, method=method, layer_methods=layer_methods)
decode_target, split_residency = self._residency_split(config, cpu_layer_ids, method, capped=_pin_budget_bytes(reserved) is not None)
requested_residency = None
if split_residency:
from freetoken.moe.host_banks import HostResidency
requested_residency = [
HostResidency.LOCKED.value if i in cpu_layer_ids
else HostResidency.PINNED.value
for i in range(config.model_config.num_moe_layers)
]
# read before the expert load, so a bad profile fails in seconds
expert_counts = resolve_profile(
config.moe_cache_split, config.model_path,
config.model_config.num_moe_layers, config.model_config.num_experts,
)
try:
with _weight_load_context():
banks = load_expert_banks(
config.model_path,
config.model_config,
method=method,
device=self.device,
dummy=config.use_dummy_weight,
parallel=expert_parallel,
layer_residency=requested_residency,
layer_methods=layer_methods,
)
except PinFailed as exc:
raise RuntimeError(f"{exc}; {_pin_hint(self._host_tables_bytes)}") from exc
banks, probed, staging = settle_banks(config, banks, probe=probe, has_locked=split_residency)
if probe:
cpu_layer_ids = probed
decode_target, split_residency = self._residency_split(config, cpu_layer_ids, method, capped=True)
from freetoken.engine.cache_budget import expert_bytes_per_slot, layer_slot_bytes
self._moe_layer_slot_bytes = layer_slot_bytes(banks.sources)
# hybrid: locked layers keep GPU slots of their own pools and fetch part of their misses through the pinned staging (moe/locked_fetch.py)
locked_fetch = staging is not None
layer_tags = locked_pool_tags(banks.layer_tags, cpu_layer_ids, num_moe_layers) if locked_fetch else banks.layer_tags
self._moe_pool_groups = group_layers(config.moe_cache_scope, bank_layouts(banks.sources, layer_tags), self._moe_layer_slot_bytes)
if locked_fetch:
# a locked refill costs a staged copy: place the locked pools after the pinned ones, which hold the prefill staging when they are big enough
self._moe_pool_groups.sort(key=lambda layers: layers[0] in cpu_layer_ids)
if len(self._moe_pool_groups) > 1 and cpu_layer_ids and not locked_fetch:
raise ValueError(
"several MoE slot pools (--moe-cache-scope per-layer, or experts in several quant formats) need every MoE "
"layer to decode from the GPU slot cache; they combine with --moe-cpu-layers only for locked layers under "
"--moe-strategy hybrid unless --moe-locked-fetch is 0"
)
self._moe_expert_counts = expert_counts if len(self._moe_pool_groups) > 1 else None
if len(self._moe_pool_groups) == 1 and config.moe_cache_split != "equal":
logger.warning_rank0(
f"--moe-cache-split {config.moe_cache_split} is ignored: all MoE layers share one slot pool "
f"(--moe-cache-scope per-layer gives each layer its own)"
)
self._moe_slot_limit = _slot_limit(layer_methods) if layer_methods is not None else (method.slot_limit() if method is not None else None)
# with GPU decode, a pool below num_experts slots bounds the batch; the startup split keeps room for one request's decode step
pools = self._moe_pools(config.decode_rows_per_req if decode_target == "gpu" else 1)
size_cap = sum(p.max_num_slots for p in pools)
if config.moe_cache_auto:
size, pages, overlap = self._resolve_auto_moe_cache_size(config, banks, method, max_slots=size_cap)
object.__setattr__(config, "moe_cache_size", size)
object.__setattr__(config, "moe_prefill_overlap", overlap)
if config.num_page_override is None:
# Honor the plan's KV half too: MoE slots and KV pages were solved
# against ONE budget (ratio x baseline - weights), so both must come
# from it. Re-solving pages later from a fresh free-memory reading
# double-counts everything allocated since the weights measurement
# (this expert cache, the CPU-executor GPU buffers, allocator
# slack) and goes negative whenever the expert fill is exact --
# a greedy fill leaves no headroom for the measurement delta.
object.__setattr__(config, "num_page_override", pages)
logger.info_rank0(
f"--moe-cache-auto resolved moe_cache_size={size} "
f"num_pages={pages} (prefill_overlap={overlap})"
)
elif config.moe_cache_mem is not None:
per_slot = expert_bytes_per_slot(banks.sources)
size = min(config.moe_cache_mem // per_slot, size_cap)
# per_slot is a mean over layers, so with pools of different slot sizes the split can exceed the budget: shrink until it fits
while size > 0 and self._moe_pool_bytes(split_slots(size, pools, self._moe_expert_counts)) > config.moe_cache_mem:
size -= 1
overlap = config.moe_prefill_overlap and size >= 2 * config.model_config.num_experts
object.__setattr__(config, "moe_cache_size", size)
object.__setattr__(config, "moe_prefill_overlap", overlap)
logger.info_rank0(
f"--moe-cache-mem {mem_GB(config.moe_cache_mem)} resolved moe_cache_size={size} "
f"({per_slot / 2**20:.2f} MiB per slot, prefill_overlap={overlap})"
)
_require_offload_cache_size(config.moe_cache_size, config.model_config.num_experts)
pool_sizes = split_slots(config.moe_cache_size, pools, self._moe_expert_counts)
if len(pools) > 1:
logger.info_rank0(
f"MoE slot pools: {config.moe_cache_scope}, {len(pools)} pools of {min(pool_sizes)}-{max(pool_sizes)} "
f"slots ({config.moe_cache_split})"
)
if decode_target == "gpu":
_clamp_decode_batch_to_pools(config, pool_sizes)
layout = layer_layouts = max_slots = None
if layer_methods is not None:
_check_layer_formats(banks, layer_methods)
layer_layouts = [m.layout() for m in layer_methods]
max_slots = self._moe_slot_limit
elif method is not None:
if banks.layer_formats is not None:
raise ValueError(
f"expert banks hold experts of several formats ({sorted(set(banks.layer_formats))}) but the model "
f"binds {method.kind} / {method.kernel.name} on every MoE layer; reconvert the checkpoint"
)
if banks.kind is not None and (banks.kind, banks.kernel) != (method.kind, method.kernel.name):
raise ValueError(
f"expert banks were packed for {banks.kind} / {banks.kernel} but the model "
f"binds {method.kind} / {method.kernel.name}; reconvert or select that kernel"
)
# GGUF layers keep their own ggml types, so the cache reads each layer's layout off its banks
layout = None if method.kind is QuantKind.GGUF else method.layout()
max_slots = method.slot_limit()
cache = OffloadMoeCache(
# Models with leading dense layers (GLM-4) only have experts on the MoE
# layers; num_moe_layers == num_layers when first_k_dense_replace == 0.
num_layers=config.model_config.num_moe_layers,
num_experts=config.model_config.num_experts,
cache_size=config.moe_cache_size,
device=self.device,
cache_policy=config.moe_cache_policy,
prefill_overlap=config.moe_prefill_overlap,
prefill_hit_d2d=config.moe_prefill_hit_d2d,
quant_format=banks.quant_format,
decode_target=decode_target,
hybrid_max_fetch=config.moe_hybrid_max_fetch,
layout=layout,
layer_layouts=layer_layouts,
max_slots=max_slots,
pool_of_layer=pool_of_layer(pools, config.model_config.num_moe_layers),
pool_sizes=pool_sizes,
)
# before set_bank_sources: the residency validation and the copy plan's skip of non-pinned layers key on the CPU-layer set
cache.cpu_layer_ids = cpu_layer_ids
if locked_fetch:
cache.locked_hybrid = cpu_layer_ids
cache.set_bank_sources(banks.sources, layer_residency=banks.layer_residency, layer_tags=layer_tags, row_tail_bytes=banks.row_tail_bytes)
cache.set_alphas(banks.gate_up_alpha, banks.down_alpha)
# Must be set before CUDA graph capture so the (device-side) accumulation ops are
# captured and re-run on every decode replay.
cache.collect_stats = config.moe_collect_stats
layers = attach_offload_moe_cache(self.model, cache)
assert len(layers) == config.model_config.num_moe_layers
if cache.decode_target in ("cpu", "hybrid"):
self._init_cpu_moe_executor(config, cache, layers)
if decode_target == "hybrid":
self._resolve_hybrid_fetch(config, cache)
if locked_fetch:
attach_locked_fetch(cache, config, staging, self._profile_layer_rows)
self.moe_offload_cache = cache
return cache
def _residency_split(self, config: EngineConfig, cpu_layer_ids, method, *, capped: bool):
"""(decode_target, split_residency) for a CPU-layer set; turns prefill overlap off when some layers stay unpinned."""
# the kernels were picked for model_config.decode_target; --moe-cpu-layers auto may still find that every bank fits the pin budget
decode_target = config.model_config.decode_target
if decode_target == "cpu" and not cpu_layer_ids:
decode_target = "gpu"
# split residency: where pinning is quota-capped (_pin_budget_bytes), pin only the GPU layers' banks and mlock the CPU layers'
# uncapped hosts keep every bank pinned (CPU decode reads them the same; overlap prefill stays on)
# not applied to plain --moe-strategy cpu; all-locked under a cap = --moe-strategy offload --moe-cpu-layers 1.0
split_residency = (
bool(cpu_layer_ids)
and config.moe_strategy in ("offload", "hybrid")
and capped
)
if config.moe_strategy == "cpu" and not split_residency:
# cpu mode pins every bank for the prefill double buffer; over the pin cap that dies in cudaHostRegister, so lock everything instead
from freetoken.moe.expert_banks import bank_bytes_estimate, ftw_bank_bytes
budget = _pin_budget_bytes(self._host_tables_bytes)
bank_bytes = None
if budget is not None:
bank_bytes = ftw_bank_bytes(config.model_path) or bank_bytes_estimate(config.model_config, method)
if bank_bytes and bank_bytes > budget:
split_residency = True
logger.info_rank0(
f"--moe-strategy cpu: banks {bank_bytes / 2**30:.2f} GiB exceed the "
f"pin budget; OS-locking all layers instead of pinning"
)
if split_residency and config.moe_prefill_overlap:
# locked (unregistered) layers cannot feed the async pinned H2D double buffer; their prefill is a synchronous pageable copy via materialize
logger.info_rank0(
"--moe-cpu-layers split residency: disabling MoE prefill overlap "
"(locked layers prefill via synchronous pageable copies)"
)
object.__setattr__(config, "moe_prefill_overlap", False)
return decode_target, split_residency
def _moe_pool_bytes(self, pool_sizes: list[int]) -> int:
return sum(s * self._moe_layer_slot_bytes[g[0]] for g, s in zip(self._moe_pool_groups, pool_sizes))
def _moe_pools(self, batch: int) -> list:
"""The slot pools, each with room for the routed experts of ``batch`` decode tokens."""
model_config = self.config.model_config
num_experts = model_config.num_experts
min_num_slots = min(num_experts, model_config.num_experts_per_tok * batch)
return make_pools(self._moe_pool_groups, num_experts, min_num_slots, self._moe_slot_limit)
def _resolve_hybrid_fetch(self, config: EngineConfig, cache) -> None:
"""Resolve --moe-hybrid-max-fetch -1 (auto) into a bandwidth-matched fetch fraction.
Perfect fetch/compute overlap wants fetched : cpu-computed misses = pcie_bw :
(cpu_bw - pcie_bw), i.e. fetching a pcie_bw / cpu_bw fraction of each decode
step's misses -- both sides then finish together instead of one idling. The
achieved bandwidths come from the cached `ft bench bw` profile, from the row of each
MoE layer's kind (format, options, expert shape, top_k, CPU worker threads); a layer
without a row keeps the fixed cap of 1.
"""
if config.moe_hybrid_max_fetch >= 0:
return # explicit fixed cap
from freetoken.moe.bench_profile import fetch_fraction
cache.hybrid_max_fetch = 1
layer_rows = self._profile_layer_rows()
if layer_rows is None:
logger.warning_rank0("--moe-hybrid-max-fetch auto: the MoE layers bind no expert method to look up in the `ft bench bw` profile; using a fixed fetch cap of 1")
return
executor = self.cpu_moe_executor
keys, labels, rows = layer_rows
fractions = [fetch_fraction(rows.get(key)) for key in keys]
cache.hybrid_fetch_fraction = [f or 0.0 for f in fractions]
# locked layers fetch the fetch_plan share instead; their own log line lists it
groups = collections.Counter((labels[l], fractions[l]) for l in range(len(keys)) if l not in cache.locked_hybrid)
fetched = [f"{f:.1%} on {n} {label} layers" for (label, f), n in groups.items() if f is not None]
if fetched:
logger.info_rank0(
"--moe-hybrid-max-fetch auto: each decode step fetches this share of a layer's expert misses over PCIe "
f"(benched PCIe/CPU bandwidth ratio) and computes the rest on the CPU: {'; '.join(fetched)}"
)
if missing := sorted({label for label, f in groups if f is None}):
capped = sum(n for (_, f), n in groups.items() if f is None)
logger.warning_rank0(
f"--moe-hybrid-max-fetch auto: no `ft bench bw` row for {capped} MoE layers ({', '.join(missing)}) at "
f"{executor.num_threads} CPU threads; using a fixed fetch cap of 1 there. `ft bench bw --model <checkpoint>` benches each layer kind"
)
benched_isas = {rows[key].get("cpu_moe_isa") for key, f in zip(keys, fractions) if f is not None} - {None}
if other := sorted(benched_isas - set(executor.isa.split(","))):
logger.warning_rank0(
f"--moe-hybrid-max-fetch auto: some `ft bench bw` rows were benched on CPU kernels {other}, but this executor runs "
f"{executor.isa}; using them anyway"
)
def _profile_layer_rows(self):
"""(row key, label) of each MoE layer's kind in the `ft bench bw` profile, and the profile's rows; None without expert methods."""
from freetoken.moe.bench_profile import layer_kind_key, load_layer_kinds
from freetoken.moe.cpu_executor import _format_label
methods = offload_layer_methods(self.model)
if methods is None:
return None
gpu_name, gpu_uuid = _profile_gpu(self.device.index)
rows = load_layer_kinds(gpu_name=gpu_name, gpu_uuid=gpu_uuid) or {}
keys, labels = [], []
for m in methods:
fmt, opts = m.cpu_spec()
c = m.cfg
keys.append(layer_kind_key(fmt, opts, c.hidden, c.local_intermediate, c.top_k, self.cpu_moe_executor.num_threads))
labels.append(f"{_format_label(fmt, tuple(sorted(opts.items())))} {c.hidden}x{c.local_intermediate}")
return keys, labels, rows
def _init_cpu_moe_executor(self, config: EngineConfig, cache, layers) -> None:
"""Build the persistent CPU MoE executor (decode-time expert compute).
Must run before CUDA graph capture: the worker pool has to be live for the
eager warmup forward, and the pinned IO block must be stable for the captured
nodes.
"""
from freetoken.moe.cpu_executor import CpuMoeExecutor
sample = layers[0]
required = ("top_k", "activation", "apply_router_weight_on_input")
if not all(hasattr(sample, attr) for attr in required):
raise NotImplementedError(
"CPU MoE backend is not yet supported for this model architecture "
f"(MoE layer {type(sample).__name__} is missing {required})."
)
# Decode batches never exceed max_running_req, but CUDA-graph padding can
# round a batch up to the largest captured size; cover both.
max_tokens = max(config.max_running_req, config.cuda_graph_max_bs or 0, *(config.cuda_graph_bs or []), 1)
max_tokens *= config.decode_rows_per_req
by_id = sorted(layers, key=lambda layer: layer.layer_id)
layer_formats = [layer.quant_method.cpu_spec() for layer in by_id]
executor = CpuMoeExecutor(
cache,
top_k=sample.top_k,
activation=sample.activation,
apply_router_weight_on_input=sample.apply_router_weight_on_input,
num_threads=config.moe_cpu_threads,
max_tokens=max_tokens,
device=self.device,
swiglu_alpha=float(sample.alpha),
swiglu_limit=sample.limit,
layer_formats=layer_formats,
)
cache.set_cpu_executor(executor)
self.cpu_moe_executor = executor
def _sync_get_memory(self) -> Tuple[int, int]:
"""Get the min and max free memory across TP ranks."""
torch.cuda.synchronize(self.device)
torch.cuda.empty_cache()
torch.cuda.reset_peak_memory_stats(self.device)
free_memory = get_free_memory(self.device)
free_mem_tensor = torch.tensor([free_memory, -free_memory], device="cpu", dtype=torch.int64)
torch.distributed.all_reduce(
free_mem_tensor, op=torch.distributed.ReduceOp.MIN, group=self.tp_cpu_group
)
min_free_memory = int(free_mem_tensor[0].item())
max_free_memory = -int(free_mem_tensor[1].item())
if max_free_memory - min_free_memory > 2 * 1024 * 1024 * 1024:
logger.error(
f"Memory across TP ranks are imbalanced:"
f" min {mem_GB(min_free_memory)}, max {mem_GB(max_free_memory)}"
)
raise RuntimeError("Memory across TP ranks are imbalanced")
return min_free_memory, max_free_memory
def _target_moe_and_expert_bytes(self, moe_cache_size: int | None, pool_sizes: list[int] | None) -> tuple[int, int]:
cache = self.moe_offload_cache
if cache is None:
return moe_cache_size or 0, 0
if moe_cache_size is None:
target_moe = cache.cache_size
nbytes = sum(c.numel() * c.element_size() for _, c in cache.banks)
else:
target_moe = moe_cache_size
nbytes = self._moe_pool_bytes(pool_sizes)
# pools of several bank layouts have different slot sizes; round up so target * bytes covers them
return target_moe, -(-nbytes // target_moe)
def _resize_kv_pool(self, config, num_pages: int, num_swa_pages: int | None) -> None:
# IN-PLACE, identity-preserving: the CacheManager's swa_pool reference, ctx.kv_cache and
# the model's per-access pool property all keep pointing at THIS pool, which frees its old
# buffers before allocating the new ones. mark_for_rebind re-binds per-bind scratch on the
# next forward (graph re-capture); the prefix tree + page bookkeeping reset is the
# scheduler's generic cache_manager.rebuild.
if self.kv_cache.needs_rebind_on_rebuild:
self.model.mark_for_rebind()
self.kv_cache.rebuild_from_config(config, num_pages, num_swa_pages=num_swa_pages)
self.num_pages = num_pages
def _refresh_seq_state(self, config) -> None:
num_tokens = self.num_pages * config.page_size
self.max_seq_len, aligned_max_seq_len = _seq_limits(config, num_tokens)
if aligned_max_seq_len != self.page_table.shape[1]:
# max_seq_len changed (e.g. KV grew past the startup token budget); the page table
# columns must track it or new requests would index out of bounds. The scheduler
# re-points its managers to engine.page_table on a num_pages rebuild.
self.ctx.page_table = self.page_table = torch.zeros(
(config.max_running_req + 1, aligned_max_seq_len),
dtype=torch.int32,
device=self.device,
)
self.page_table[self.dummy_req.table_idx].fill_(num_tokens)
self.kv_cache.attach_page_table(self.page_table)
@torch.inference_mode()
def rebuild_runtime_cache(
self,
*,
moe_cache_size: int | None = None,
num_pages: int | None = None,
num_mamba_slots: int | None = None,
num_swa_pages: int | None = None,
) -> None:
"""Idle-only in-place resize of the MoE slot cache, KV page pool, GDN (mamba) state pool,
and/or the window pool (num_swa_pages: an absolute pinned window), followed by CUDA-graph
re-capture. Does NOT reload weights or host expert banks. The caller (scheduler) must
guarantee no in-flight prefill/decode.
"""
config = self.config
if (moe_cache_size is None and num_pages is None and num_mamba_slots is None
and num_swa_pages is None):
return
# 0a. Geometry prevalidation BEFORE any destructive free. An invalid target (moe
# slots on a model with no offload cache, moe below num_experts / above the
# marlin cap, non-positive pages, or too few GDN slots to run) must reject
# recoverably with the old cache intact -- NOT after teardown, which would
# leave the server unable to serve. These checks are model-agnostic.
if moe_cache_size is not None:
if self.moe_offload_cache is None:
raise CacheRebuildRejected(
"moe_cache_size requested but this model has no MoE offload cache"
)
try:
# a rebuild does not lower max_running_req again, so with GPU decode every pool must hold a full batch; hybrid caps its fetches instead
batch = config.max_running_req * config.decode_rows_per_req if self.moe_offload_cache.decode_target == "gpu" else 1
moe_pool_sizes = split_slots(moe_cache_size, self._moe_pools(batch), self._moe_expert_counts)
self.moe_offload_cache.validate_rebuild(moe_cache_size, moe_pool_sizes)
except ValueError as e:
raise CacheRebuildRejected(str(e)) from e
if num_pages is not None and num_pages <= 0:
raise CacheRebuildRejected(f"num_pages must be positive, got {num_pages}")
if num_mamba_slots is not None:
if self.linear_state_pool is None:
raise CacheRebuildRejected(
"num_mamba_slots requested but this model has no GDN state pool"
)
# num_mamba_slots is the USABLE slot count (what the user sets and the status bar
# shows); the pool also reserves a padding sink (slot 0), so the physical pool is
# num_mamba_slots + 1. _linear_pool_min_slots is the physical floor -> usable - 1.
min_usable = _linear_pool_min_slots(config) - 1
if num_mamba_slots < min_usable:
raise CacheRebuildRejected(
f"num_mamba_slots {num_mamba_slots} is below the minimum {min_usable} "
f"(non-evictable working set for max_running_req={config.max_running_req}) "
f"needed to run; admission would deadlock"
)
if num_swa_pages is not None:
# An absolute window pin for the radix-SWA window pool (Gemma) or the DSV4 window tier;
# meaningless for dense/MHA models and the naive SWA path (concurrency x window).
if not _supports_swa_ratio(config):
raise CacheRebuildRejected(
"num_swa_pages requested but this model has no window pool "
"(needs DSV4 or a sliding-window model with --cache-type radix)"
)
if num_swa_pages <= 0:
raise CacheRebuildRejected(
f"num_swa_pages must be positive, got {num_swa_pages}"
)
# 0b. Pool-family budget fit-check BEFORE any destructive free: an unfit geometry
# must reject (recoverable) so the old caches stay intact and serving continues,
# rather than freeing and then OOMing into permanent failure. The engine supplies
# the memory account; the pool answers whether its target geometry fits.
target_moe, per_expert_bytes = self._target_moe_and_expert_bytes(
moe_cache_size, moe_pool_sizes if moe_cache_size is not None else None
)
# Price the sibling GDN state pool at ITS target (physical slots = usable + padding
# sink) and hand the bytes in -- the KV pool only budgets its own tiers.
target_mamba = (
num_mamba_slots + 1
if num_mamba_slots is not None
else (self.linear_state_pool.num_slots if self.linear_state_pool is not None else None)
)
self.kv_cache.validate_rebuild(
config, num_pages=num_pages,
num_swa_pages=num_swa_pages, target_moe=target_moe,
per_expert_bytes=per_expert_bytes, baseline_free=self._baseline_free,
weights_bytes=self._weights_bytes, current_num_pages=self.num_pages,
extra_fixed_bytes=(
state_pool_bytes(config, target_mamba) if target_mamba is not None else 0
),
extra_note=(
f", mamba={target_mamba - 1} slots" if target_mamba is not None else ""
),
)
torch.cuda.synchronize(self.device)
# Preserve the CUDA-graph batch-size set resolved at startup. The auto heuristic keys
# off free memory, which is far smaller now that the caches are resident (post-cache
# free << startup pre-load free), so re-deriving it here would silently drop large
# batch sizes after the first rebuild. Reusing the already-resolved list keeps the
# captured coverage identical (the fit-check above guarantees the graph headroom fits).
prior_graph_bs = self.graph_runner.graph_bs_list
# Point of no return for the scheduler's rollback logic: from here the live graphs and
# pools start being freed. A failure BEFORE this flag flips leaves the engine serving
# untouched (no rollback needed); after it, only a rebuild restores service.
self.rebuild_teardown_started = True
# 1. Tear down CUDA graphs + backend capture scratch (free-before-alloc).
self.attn_backend.reset_capture()
if self.linear_backend is not None:
self.linear_backend.reset_capture()
self.graph_runner.destroy_cuda_graphs()
# 2. Resize caches in place (each frees its old GPU tensors before allocating).
# Pin the new window first (validated above) so any KV-pool rebuild below sizes the window
# to it (_dsv4_pool_sizes / _swa_paged_num_tokens read config.swa_num_pages_override).
# frozen EngineConfig — mutate in place like the moe_cache_size path; `config.x = y` raises
# FrozenInstanceError, which here aborts the rebuild after the CUDA graphs are gone (→ 503).
if num_swa_pages is not None:
object.__setattr__(config, "swa_num_pages_override", num_swa_pages)
if moe_cache_size is not None:
assert self.moe_offload_cache is not None, "no MoE offload cache to resize"
self.moe_offload_cache.rebuild(moe_cache_size, moe_pool_sizes)
if num_pages is not None:
# sets self.num_pages (rebuilds KV + window)
self._resize_kv_pool(config, num_pages, num_swa_pages)
elif num_swa_pages is not None:
# Window-only change: no page-count change, but re-derive the window pool at the new
# pin against the CURRENT page count. This re-allocs the same-size full pool and
# the resized window, both inside the pool's own rebuild_from_config.
self._resize_kv_pool(config, self.num_pages, num_swa_pages)
if num_mamba_slots is not None:
# Reallocate the GDN state pool (frees old tensors first). Must sit between graph
# teardown and re-capture so the recaptured graphs bind the new state tensors.
# +1 for the reserved padding sink: num_mamba_slots is the usable count.
self.linear_state_pool.rebuild(num_mamba_slots + 1)
# 3. Refresh max_seq_len (+ generic page table) for the new token budget.
self._refresh_seq_state(config)
aligned_max_seq_len = self.page_table.shape[1]
# 4. Re-capture CUDA graphs against the new tensors (reset_capture above re-armed
# the backend; _sync_get_memory empties the cache so freed memory is reclaimed).
gc.collect()
free_min = self._sync_get_memory()[0]
self.graph_runner = GraphRunner(
stream=self.stream,
device=self.device,
forward=self.model.forward,
attn_backend=self.attn_backend,
cuda_graph_bs=prior_graph_bs, # reuse the startup-resolved set (see above)
cuda_graph_max_bs=config.cuda_graph_max_bs,
free_memory=free_min,
max_seq_len=aligned_max_seq_len,
dummy_req=self.dummy_req,
moe_offload_cache=self.moe_offload_cache,
mrope=config.model_config.model_is_mrope,
out_dtype=torch.float32,
linear_backend=self.linear_backend,
)
def forward_logits(self, batch: Batch) -> torch.Tensor:
"""The batch's logits from its captured graph, or eagerly when none fits."""
use_graph = self.graph_runner.can_use_cuda_graph(batch)
with self.ctx.forward_batch(batch), self.model.forward_host_ctx(batch, use_graph):
logits = self.graph_runner.replay(batch) if use_graph else self.model.forward()
if self.cpu_moe_executor is not None:
# One pinned read: surfaces a fired flag-handshake watchdog (dead coordinator
# -> stale expert outputs) as a loud error instead of silent corruption.
self.cpu_moe_executor.raise_if_unhealthy()
return logits
def forward_batch(self, batch: Batch, args: BatchSamplingArgs) -> ForwardOutput:
assert torch.cuda.current_stream() == self.stream
if batch.mm_gather_plan:
self._run_mm_encoder(batch)
logits = self.forward_logits(batch)
for req in batch.reqs:
req.complete_one()
batch_logits = logits[: batch.size]
next_tokens_gpu = self.sampler.sample(batch_logits, args).to(torch.int32)
next_tokens_cpu = next_tokens_gpu.to("cpu", non_blocking=True)
copy_done_event = torch.cuda.Event()
copy_done_event.record(self.stream)
return ForwardOutput(next_tokens_gpu, next_tokens_cpu, copy_done_event)
@torch.inference_mode()
def _warmup_prefill(self) -> None:
"""Compile the Triton prefill path before the first real request.
Decode CUDA graph capture warms the decode path, but the first prefill
can still pay Triton/cublas setup costs. Use the dummy request row and
restore it afterwards so padded decode graph replay keeps using the
dedicated dummy KV slot.
"""
if self.max_seq_len < 2:
return
warmup_lens = [min(80, self.max_seq_len)]
if self.max_seq_len >= 128:
warmup_lens.append(128)
warmup_lens = sorted({length for length in warmup_lens if length >= 2})
if not warmup_lens:
return
dummy_row = self.page_table[self.dummy_req.table_idx]
dummy_slot = int(dummy_row[0].item())
started = torch.cuda.Event(enable_timing=True)
ended = torch.cuda.Event(enable_timing=True)
started.record(self.stream)
try:
for length in warmup_lens:
dummy_row[:length] = torch.arange(
length, dtype=torch.int32, device=self.device
)
warm_req = Req(
input_ids=torch.zeros(length, dtype=torch.int32, device="cpu"),
table_idx=self.dummy_req.table_idx,
cached_len=0,
output_len=1,
uid=-1,
sampling_params=None, # type: ignore[arg-type]
cache_handle=None, # type: ignore[arg-type]
)
batch = Batch(reqs=[warm_req], phase="prefill")
batch.padded_reqs = batch.reqs
batch.input_ids = torch.zeros(length, dtype=torch.int32, device=self.device)
batch.positions = torch.arange(length, dtype=torch.int32, device=self.device)
if self.config.model_config.model_is_mrope:
batch.mrope_positions = (
batch.positions.unsqueeze(0).expand(3, -1).contiguous()
)
batch.out_loc = dummy_row[:length]
self.attn_backend.prepare_metadata(batch)
with self.ctx.forward_batch(batch):
self.model.forward()
finally:
dummy_row.fill_(dummy_slot)
if self.moe_offload_cache is not None:
self.moe_offload_cache.reset()
ended.record(self.stream)
torch.cuda.synchronize(self.device)
logger.info_rank0(
f"Prefill warmup complete for lengths {warmup_lens} "
f"in {started.elapsed_time(ended) / 1000.0:.3f} s"
)
def shutdown(self) -> None:
self.graph_runner.destroy_cuda_graphs()
torch.distributed.destroy_process_group()
destroy_distributed()
def _profile_gpu(index: "int | None" = None) -> Tuple[str | None, str | None]:
"""(name, uuid) of visible device ``index`` (default: the current, i.e. bound, device); (None, None) without CUDA."""
if not torch.cuda.is_available():
return None, None
ident = gpu_identity(torch.cuda.current_device() if index is None else index)
return ident["name"], ident["uuid"]
def _is_unified_memory_gpu(index: "int | None" = None) -> bool:
"""True when the GPU has no separate device memory (cudaDevAttrIntegrated): host
banks and the GPU slot cache are the same DRAM, so the offload family's pinned
staging + slot gather are DRAM-to-DRAM copies with no PCIe link to hide behind.
The attribute is reliable on true-UMA parts (Jetson, GB10/DGX Spark) but not on
C2C-linked discrete-HBM parts (GH200 reports integrated=0 despite coherent CPU
memory), and it has only been verified on GB10 so far --
FREETOKEN_UNIFIED_MEMORY=0/1 overrides the probe where the attribute lies."""
env = os.environ.get("FREETOKEN_UNIFIED_MEMORY")
if env is not None:
return env.strip().lower() not in {"0", "false", "no", "off"}
if not torch.cuda.is_available():
return False
try:
dev = torch.cuda.current_device() if index is None else index
return bool(torch.cuda.get_device_properties(dev).is_integrated)
except Exception:
return False
_RESIDENT_EXPERT_QUANTS = ("none", "fp8_block", "nvfp4", "gguf")
def _fused_resident_ok(model_config) -> bool:
"""Whether the resident ('fused') MoE path can hold this model's experts; DeepSeek-V4 experts stay offload."""
if getattr(model_config, "expert_quant", "none") not in _RESIDENT_EXPERT_QUANTS:
return False
return getattr(model_config, "moe_weight_format", None) in (None, "bf16", "mxfp4")
def _ensure_expandable_segments() -> None:
"""Default the CUDA allocator to expandable segments.
The motivating case is the offload prefill, which repeatedly dequantizes
variable-sized NVFP4 expert blocks to BF16 (a different size per layer as the
active-expert count varies). Under that alloc/free churn the default caching
allocator fragments badly -- reserved memory can balloon far past the actual peak
allocation (observed ~78GiB reserved for a <30GiB working set).
``expandable_segments`` lets freed regions of any size be reused, keeping
reserved ~= allocated, so it is applied to every run, not just offload ones.
Env vars are parsed once at import and ignored if set afterwards, so we apply the
setting via the runtime API instead. Must run before the first CUDA allocation (the
caller guarantees CUDA is not yet initialized). Any user-provided allocator config
is respected and left untouched.
"""
if os.environ.get("PYTORCH_ALLOC_CONF") or os.environ.get("PYTORCH_CUDA_ALLOC_CONF"):
return
try:
torch.cuda.memory._set_allocator_settings("expandable_segments:True")
except Exception as exc: # pragma: no cover - depends on torch build
logger.info_rank0(f"Could not enable expandable_segments ({exc}); continuing")
return
logger.info_rank0("Enabled expandable_segments (override via PYTORCH_ALLOC_CONF)")
def _resolve_cache_type(has_linear_attention: bool, requested: str) -> str:
# Hybrid GDN models default to the HybridRadixCache (snapshots GDN state at chunk
# boundaries -> cross-request prefix reuse). An explicit ``--cache-type naive`` opts out
# to the old no-reuse path (debugging / parity baseline / lower GDN-state memory).
if has_linear_attention:
return "naive" if requested == "naive" else "hybrid_radix"
return requested
def _adjust_dsv41_config(config: EngineConfig, override) -> None:
"""DeepSeek-V4.1 config reconciliation, the DSV4 policy with the replay knob: sync the
runtime into ``dsv41_args``, page_size = the window page P, radix -> swa_radix, decode graph
batches <= max_running_req. Unlike DSV4 the prefill chunk keeps ``max_extend_tokens`` (whole
window pages): at a 1M ceiling the window pool alone would admit ~40K-token chunks whose
activations (64 x 512 latent queries per token) do not fit next to the expert cache."""
model_config = config.model_config
args = model_config.dsv41_args
args.max_seq_len = config.max_seq_len
args.max_batch_size = config.max_running_req + 1 # +1 dummy
args.swa_decoder_replay = config.swa_decoder_replay
P = args.window_size
override("page_size", P)
logger.info_rank0(
f"DSV41 KV pages are {P}-token window pages; page_size set to {P}; SWA decoder replay: {args.swa_decoder_replay}"
+ ("" if args.swa_decoder_replay == "exact" else f" (prefix hits stop {P} tokens before the prompt end)")
)
if getattr(config, "cache_type", "radix") != "naive":
override("cache_type", "swa_radix")
current = getattr(config, "max_extend_tokens", None) # a SchedulerConfig field
if current is not None and (extend := max(P, current // P * P)) != current:
override("max_extend_tokens", extend)
mr = config.max_running_req
if config.cuda_graph_max_bs is not None and config.cuda_graph_max_bs > mr:
logger.warning_rank0(f"cuda_graph_max_bs {config.cuda_graph_max_bs} exceeds max_running_req {mr}; clamping (larger decode batches never occur).")
override("cuda_graph_max_bs", mr)
if config.cuda_graph_bs is not None:
kept = [bs for bs in config.cuda_graph_bs if bs <= mr]
if kept != list(config.cuda_graph_bs):
logger.warning_rank0(f"dropping cuda_graph_bs entries above max_running_req {mr}: {[bs for bs in config.cuda_graph_bs if bs > mr]}")
override("cuda_graph_bs", kept)
def _adjust_dsv4_config(config: EngineConfig, override) -> None:
"""DSV4 engine-config reconciliation at config-resolution time (before the pool exists).
Syncs the resolved runtime config into the opaque ``dsv4_args`` payload, sets
page_size to the window page P, forces single-chunk prefill, and clamps cuda_graph_bs/max_bs to
the DSV4 decode batch size.
"""
model_config = config.model_config
model_config.dsv4_args.max_seq_len = config.max_seq_len
model_config.dsv4_args.max_batch_size = config.max_running_req + 1 # +1 dummy
# config.swa_full_tokens_ratio is the DSV4 window/full ratio directly (default sizing);
# a runtime rebuild pins an absolute window via swa_num_pages_override instead.
# DSV4's KV page IS the P-token window page (window == radix reuse granularity == lcm of
# the compress ratios), so max_num_tokens = num_pages * page_size holds like every model.
P = model_config.dsv4_args.window_size
override("page_size", P)
logger.info_rank0(f"DSV4 KV pages are {P}-token window pages; page_size set to {P}")
# The generic CacheManager materializes DSV4 'radix' as the shared SWARadixCache (is_swa);
# 'naive' stays naive with the pool's swa currency riding swa_paged.
if getattr(config, "cache_type", "radix") != "naive":
override("cache_type", "swa_radix")
# 'radix' (SWARadixCache on the full-loc currency, carry-aware re-prefill) is the default and is
# honored, as is an explicit 'naive'. Don't let max_extend_tokens force a second chunk within
# one prompt (the pool's prefill_chunk_budget still chunks prompts larger than the window
# pool); prefill batches ragged (bs>=1), each segment resuming from its own cached_len.
if getattr(config, "max_extend_tokens", 0) < config.max_seq_len:
override("max_extend_tokens", config.max_seq_len)
# DSV4 decode batches at most max_running_req rows; its full-loc snapshot is sized to that,
# so a graph bs above it would exceed the backend's captured snapshot rows. Clamp any
# oversized explicit list / max_bs here (before GraphRunner ever sees it).
mr = config.max_running_req
if config.cuda_graph_max_bs is not None and config.cuda_graph_max_bs > mr:
logger.warning_rank0(
f"cuda_graph_max_bs {config.cuda_graph_max_bs} exceeds DSV4 max_running_req {mr}; "
"clamping to max_running_req (larger decode batches never occur)."
)
override("cuda_graph_max_bs", mr)
if config.cuda_graph_bs is not None:
kept = [bs for bs in config.cuda_graph_bs if bs <= mr]
if kept != list(config.cuda_graph_bs):
dropped = [bs for bs in config.cuda_graph_bs if bs > mr]
logger.warning_rank0(
f"dropping cuda_graph_bs entries {dropped} above DSV4 max_running_req {mr} "
"(larger decode batches never occur)."
)
override("cuda_graph_bs", kept)
def _parse_cpu_layers_spec(spec: str, num_moe_layers: int) -> frozenset[int]:
"""Parse ``--moe-cpu-layers``: an explicit MoE-layer id list (``"3,7,11"``), a count
(``"8"`` -> 8 layers evenly strided across depth), or a fraction (``"0.5"``). Ids are
indices into the MoE layers, ``[0, num_moe_layers)``."""
s = spec.strip()
if not s:
return frozenset()
if "," in s:
ids = {int(x) for x in s.split(",") if x.strip()}
for i in ids:
if not 0 <= i < num_moe_layers:
raise ValueError(
f"--moe-cpu-layers id {i} out of range [0, {num_moe_layers})"
)
return frozenset(ids)
if "." in s:
frac = float(s)
if not 0.0 <= frac <= 1.0:
raise ValueError(f"--moe-cpu-layers fraction {frac} must be in [0, 1]")
k = round(frac * num_moe_layers)
else:
k = int(s)
if not 0 <= k <= num_moe_layers:
raise ValueError(f"--moe-cpu-layers count {k} must be in [0, {num_moe_layers}]")
# k layers spread evenly across depth (frozenset dedups any rounding collisions;
# k == 0 yields an empty range, hence an empty set).
return frozenset(round(i * num_moe_layers / k) for i in range(k))
def _resolve_cpu_layers(config: EngineConfig, num_moe_layers: int, *, reserved: int = 0, method=None) -> frozenset[int]:
"""MoE layer ids whose decode runs on the CPU executor.
``--moe-strategy cpu`` -> every layer. ``--moe-strategy offload`` + ``--moe-cpu-layers``
-> the parsed subset, or the pin-budget pick for ``auto`` (the rest stay on the GPU offload/PCIe path). Otherwise none.
"""
if config.moe_strategy == "cpu":
return frozenset(range(num_moe_layers))
spec = config.moe_cpu_layers
if not spec or not is_offload_moe_strategy(config.moe_strategy):
return frozenset()
if spec.strip() == "auto":
return _auto_cpu_layers(config, num_moe_layers, reserved=reserved, method=method)
return _parse_cpu_layers_spec(spec, num_moe_layers)
def _decode_target(config: EngineConfig) -> str:
"""Where routed experts decode, from the flags alone: hybrid co-compute, the CPU executor for some or all layers, or the GPU slot cache."""
if config.moe_strategy == "hybrid":
return "hybrid"
if config.moe_strategy == "cpu" or (config.moe_cpu_layers and is_offload_moe_strategy(config.moe_strategy)):
return "cpu"
return "gpu"
# expert activations the CPU MoE executor supports (csrc ActKind)
_CPU_MOE_ACTS = (
"silu", "swish", "gelu", "gelu_tanh", "gelu_pytorch_tanh", "swigluoai",
"swiglu_clamp",
)
def _cpu_moe_executor_viable(model_config) -> bool:
"""Whether an automatic CPU-decode decision may target the CPU MoE executor.
A default boot must degrade to GPU offload instead of crashing in CpuMoeExecutor after the whole load; explicit cpu/hybrid/--moe-cpu-layers picks still fail loudly."""
from freetoken.moe.cpu_executor import compiled_extension_supports
try:
from freetoken.kernel import _cpu_moe
except ImportError:
return False
act = getattr(model_config, "hidden_act", "silu")
moe_wfmt = getattr(model_config, "moe_weight_format", None)
if act not in _CPU_MOE_ACTS and moe_wfmt != "mxfp4":
return False
if moe_wfmt != "mxfp4" and not compiled_extension_supports(act):
return False
expert_quant = getattr(model_config, "expert_quant", "none")
fmt = expert_quant if expert_quant != "none" else (moe_wfmt or "bf16")
if fmt == "gguf":
from freetoken.kernel import ggml_cpu
return fmt in _cpu_moe.weight_formats() and ggml_cpu.select() is not None
return fmt == "mxfp4" or fmt in _cpu_moe.weight_formats()
def _pin_budget_bytes(reserved: int = 0) -> int | None:
"""Bytes this process can still safely cudaHostRegister, or None when the platform does not cap pinning (plain Linux).
WSL's WDDM-backed CUDA caps pinning near half of RAM, shared across processes -- budget 40%. FREETOKEN_PIN_BUDGET_GB overrides anywhere. ``reserved`` subtracts host bytes already pinned outside the expert banks (qwen4_exp's PLE table)."""
if env := os.environ.get("FREETOKEN_PIN_BUDGET_GB"):
cap = int(float(env) * 2**30)
elif not hasattr(os, "uname") or "microsoft" not in os.uname().release.lower(): # WSL kernel tag
return None
else:
cap = int(os.sysconf("SC_PHYS_PAGES") * os.sysconf("SC_PAGE_SIZE") * 0.4)
return max(0, cap - reserved)
def _bank_bytes(config: EngineConfig, method=None, layer_methods=None) -> int | None:
from freetoken.moe.expert_banks import bank_bytes_estimate, ftw_bank_bytes
return ftw_bank_bytes(config.model_path) or bank_bytes_estimate(config.model_config, method, layer_methods)
def _pin_hint(reserved: int) -> str:
if _pin_budget_bytes(reserved) is None:
return "the expert banks need more page-locked host RAM than this host has; free host RAM or serve a smaller model"
return (
"pass --moe-cpu-layers auto to lock the layers over the pin budget for CPU decode, "
"or --moe-cpu-layers <count|fraction|ids> to choose them yourself"
)
def _check_pin_budget(config: EngineConfig, *, reserved: int, method=None, layer_methods=None) -> None:
"""Stop a plain offload boot whose banks exceed a known pin budget before any bank is read."""
if config.moe_cpu_layers or config.moe_strategy not in ("offload", "hybrid"):
return
budget = _pin_budget_bytes(reserved)
bank_bytes = _bank_bytes(config, method, layer_methods) if budget is not None else None
if bank_bytes and bank_bytes > budget:
raise ValueError(
f"expert banks need {bank_bytes / 2**30:.1f} GiB of pinned host RAM but the pin budget is "
f"{budget / 2**30:.1f} GiB (WSL caps CUDA pinning; FREETOKEN_PIN_BUDGET_GB overrides); {_pin_hint(reserved)}"
)
def _auto_cpu_layers(config: EngineConfig, num_moe_layers: int, *, reserved: int = 0, method=None) -> frozenset[int]:
"""Pick CPU (locked) MoE layers for ``--moe-cpu-layers auto``: none while the banks fit the pin budget.
Locks just enough head+tail layers: per-layer decode miss rates are U-shaped, so the ends are the cheapest to move off the slot cache."""
bank_bytes = _bank_bytes(config, method)
if not bank_bytes:
return frozenset()
budget = _pin_budget_bytes(reserved)
if budget is None or bank_bytes <= budget:
return frozenset()
if not _cpu_moe_executor_viable(config.model_config):
raise ValueError(
f"--moe-cpu-layers auto: banks {bank_bytes / 2**30:.2f} GiB exceed the "
f"pin budget {budget / 2**30:.2f} GiB, but the CPU MoE executor cannot "
f"serve this model (see --moe-strategy cpu requirements)"
)
n = min(num_moe_layers, math.ceil(num_moe_layers * (1 - budget / bank_bytes)))
head = (n + 1) // 2
ids = frozenset(range(head)) | frozenset(range(num_moe_layers - (n - head), num_moe_layers))
logger.info_rank0(
f"--moe-cpu-layers auto: banks {bank_bytes / 2**30:.2f} GiB > pin budget "
f"{budget / 2**30:.2f} GiB; locking {n} head+tail MoE layers for CPU decode "
f"({sorted(ids)})"
)
return ids
# MoE-only knobs and the value each resolves to on a dense model. moe_strategy is handled
# separately (its dense value is 'fused', but 'auto' resolves there without a warning).
_DENSE_MOE_SETTINGS = {
"moe_cache_size": 0,
"moe_cache_rate": None,
"moe_cache_auto": False,
"moe_cache_mem": None,
"moe_cache_scope": "unified",
"moe_cache_split": "equal",
"moe_cpu_layers": None,
"moe_cpu_threads": 0,
"moe_hybrid_max_fetch": -1,
"moe_locked_fetch": -1.0,
"moe_locked_max_fetch": 8,
"moe_prefill_overlap": True,
"moe_prefill_hit_d2d": False,
"expert_load": "auto",
}
def _adjust_ftw_quant_backend(model_path: str, quant_backend: QuantBackend) -> QuantBackend:
"""--quant-backend with an FTW checkpoint's packed expert kernel filled in where the flag leaves that table automatic.
An entry that names another kernel is refused here, before any bank is read."""
from freetoken.checkpoint.ftw import ftw_quant_format
from freetoken.moe.legacy_format import kind_kernel_for
with _weight_load_context():
fmt = ftw_quant_format(model_path) if model_path else None
if fmt is None:
return quant_backend
try:
kind, kernel = kind_kernel_for(fmt)
except KeyError:
return quant_backend
requested = quant_backend.select(LayerKind.MOE, kind)
if requested == kernel:
return quant_backend
if requested != "auto":
raise ValueError(
f"the FTW checkpoint's expert banks were packed for {kind} / {kernel} but --quant-backend asks for "
f"{requested}; drop the entry, or reconvert with ft checkpoint --quant-backend moe.{kind}={requested}"
)
return QuantBackend(quant_backend.items + (((LayerKind.MOE, kind), kernel),))
def offload_layer_methods(model) -> list | None:
"""The expert method of each offload MoE layer of ``model``, by MoE-layer id; None for models whose MoE layers carry none (GGUF)."""
layers = [l for l in iter_offload_moe_layers(model) if getattr(l, "quant_method", None) is not None]
if not layers:
return None
layers.sort(key=lambda layer: layer.layer_id)
assert [layer.layer_id for layer in layers] == list(range(len(layers))), [layer.layer_id for layer in layers]
return [layer.quant_method for layer in layers]
def shared_offload_method(model):
"""The expert method the offload MoE layers of ``model`` use, or None for models whose MoE layers carry none (GGUF).
With experts of several formats (a checkpoint that keeps some layers unquantized) this is the format of the most
layers; the cache then takes every layer's method (``offload_layer_methods``) and keeps one slot pool per format."""
from freetoken.moe.expert_banks import method_groups
methods = offload_layer_methods(model)
if methods is None:
return None
groups = method_groups(methods)
found = ", ".join(f"{m.kind} via {m.kernel.name}" + (f" on {len(ids)} layers" if len(groups) > 1 else "") for m, ids in groups)
logger.info_rank0(f"MoE experts: {found}")
return groups[0][0]
def _slot_limit(layer_methods) -> int | None:
"""The tightest slot cap of the layers' expert kernels (a pool never mixes kernels, but the limit is checked per pool against the largest)."""
limits = [limit for limit in (m.slot_limit() for m in layer_methods) if limit is not None]
return min(limits) if limits else None
def _check_layer_formats(banks, layer_methods) -> None:
from freetoken.moe.legacy_format import legacy_format_for
want = [legacy_format_for(m.kind, m.kernel.name) for m in layer_methods]
have = banks.layer_formats or [banks.quant_format] * len(want)
bad = [layer for layer, (w, h) in enumerate(zip(want, have)) if w != h]
if bad:
raise ValueError(
f"expert banks of MoE layers {bad[:8]} were packed as {[have[l] for l in bad[:8]]} but the model binds "
f"{[want[l] for l in bad[:8]]}; reconvert the checkpoint or select those kernels"
)
def _meta_model(config: EngineConfig, decode_target: str):
from freetoken.models import create_model
from freetoken.utils.torch_utils import torch_dtype
set_quant_backend(_adjust_ftw_quant_backend(config.model_path, QuantBackend.parse(config.quant_backend)))
object.__setattr__(config.model_config, "moe_strategy", config.moe_strategy)
object.__setattr__(config.model_config, "decode_target", decode_target)
with torch.device("meta"), torch_dtype(config.dtype):
return create_model(config.model_config)
def offload_expert_methods(config: EngineConfig) -> tuple:
"""``(method, layer_methods)`` of ``config``'s model, from a meta-device build: the most common expert method and,
for experts of several formats, one per MoE layer (else None).
For tools that load expert banks without an engine (the FTW converter); they pack for GPU decode."""
from freetoken.moe.expert_banks import is_mixed
model = _meta_model(config, "gpu")
layer_methods = offload_layer_methods(model)
return shared_offload_method(model), (layer_methods if is_mixed(layer_methods) else None)
def meta_offload_layer_methods(config: EngineConfig, decode_target: str) -> list | None:
"""The expert method of each offload MoE layer of ``config``'s model, from a meta-device build for ``decode_target``; see ``offload_layer_methods``."""
return offload_layer_methods(_meta_model(config, decode_target))
def offload_expert_method(config: EngineConfig):
"""The (most common) offload expert method of ``config``'s model; see ``offload_expert_methods``."""
return offload_expert_methods(config)[0]
def _adjust_config(config: EngineConfig):
def override(attr: str, value: Any): # this is dangerous, use with caution
object.__setattr__(config, attr, value)
model_config = config.model_config
single_stream_only = getattr(model_config, "single_stream_only", False)
is_dsv4 = getattr(model_config, "dsv4_args", None) is not None
is_dsv41 = getattr(model_config, "dsv41_args", None) is not None
has_swa_attention = getattr(model_config, "has_swa_attention", False)
has_linear_attention = getattr(model_config, "has_linear_attention", False)
is_moe = getattr(model_config, "is_moe", False)
expert_quant = getattr(model_config, "expert_quant", "none")
if not is_moe:
# A dense model has no routed experts: the MoE knobs are inert, and the offload family
# is worse than inert -- engine init would build an expert cache for a model that has
# none and abort startup (weights already resident) on an unrelated expert-source
# error. Drop them at this one choke point, which the CLI and the programmatic
# LLM(...) path both pass through. 'auto'/'fused' is the silent dense resolution;
# anything else was asked for explicitly, so report what is being ignored.
dropped = [
f"{name}={getattr(config, name)!r}"
for name, dense_value in _DENSE_MOE_SETTINGS.items()
if getattr(config, name, dense_value) != dense_value
]
if config.moe_strategy not in ("auto", "fused"):
dropped.insert(0, f"moe_strategy={config.moe_strategy!r}")
override("moe_strategy", "fused")
for name, dense_value in _DENSE_MOE_SETTINGS.items():
override(name, dense_value)
if dropped:
logger.warning_rank0(
f"{getattr(model_config, 'model_type', 'model')} is a dense model (no routed "
f"experts); ignoring MoE settings: {', '.join(dropped)}"
)
if single_stream_only:
# The model runs one sequence at a time: it collapses the batch to one row and the
# decode CUDA graph is captured at bs=1. Force the runtime knobs so the KV pool, page
# table and graph capture all stay bs=1.
if config.max_running_req != 1:
override("max_running_req", 1)
if config.cuda_graph_max_bs is None or config.cuda_graph_max_bs >= 1:
override("cuda_graph_bs", [1])
override("cuda_graph_max_bs", 1)
if config.cuda_graph_max_bs is None:
override("cuda_graph_max_bs", config.max_running_req)
if is_dsv4:
_adjust_dsv4_config(config, override)
if is_dsv41:
_adjust_dsv41_config(config, override)
if has_swa_attention:
# Both SWA cache paths use the global-paged swa pool (page_size==1 only for now).
if config.page_size != 1:
raise ValueError(
f"SWA models currently support only page_size=1, got {config.page_size}."
)
# naive keeps cache_type='naive' (NaivePrefixCache, no reuse) on the paged pool (==
# sglang SWAChunkCache); radix materializes as swa_radix (SWARadixCache, cross-request
# reuse == sglang SWARadixCache). Both allocate from the same swa pool + free out-of-window.
if getattr(config, "cache_type", "radix") != "naive":
if not 0.0 < config.swa_full_tokens_ratio <= 1.0:
raise ValueError(
f"swa_full_tokens_ratio must be in (0, 1], got {config.swa_full_tokens_ratio}"
)
override("cache_type", "swa_radix")
if has_linear_attention:
override(
"cache_type",
_resolve_cache_type(True, getattr(config, "cache_type", "radix")),
)
# Type x backend capability matrix: resolve auto from the per-type priority
# lists, then validate whatever is now selected (explicit or auto) -- every
# comma part must serve every required type, with packages/arch available.
required_attn_types = _required_attn_types(model_config)
_dtype = getattr(config, "dtype", None) # duck-typed test configs omit it
if (
required_attn_types & {AttnType.BSA, AttnType.QSA}
and _dtype is not None
and _dtype.itemsize != 2
):
# Reject at config time: the BSA/QSA pool's own assert only fires after the
# model is resident (and not at all under `python -O`).
raise ValueError(
f"--dtype {config.dtype}: block-sparse attention serves 16-bit "
"compute only (the index slab budgets 2 bytes/token); use bfloat16 "
"or float16."
)
if _dtype == torch.float16 and "mxfp8" in (
getattr(model_config, "attn_quant", "none"),
getattr(model_config, "dense_quant", "none"),
):
# The MXFP8 GEMV folds the pow2-descaled fp8 weight into the activation
# dtype; fp16's narrow exponent can overflow/flush what bf16 represents
# exactly, and the combination was never numerically validated.
raise ValueError(
"--dtype float16 with MXFP8 resident weights is unsupported (the "
"W8A16 fold is only validated exact in bfloat16); use bfloat16."
)
if _dtype == torch.float16 and getattr(getattr(model_config, "quant", None), "weight_scales", False):
# GGUF NVFP4 results are stored in the activation dtype before weight_scale_2 (~1e-4) multiplies them, which overflows fp16.
raise ValueError("--dtype float16 with NVFP4 GGUF weight scales is unsupported (the unscaled results overflow float16); use bfloat16.")
if config.attention_backend == "auto":
override(
"attention_backend",
_resolve_auto_attention_backend(required_attn_types),
)
logger.info_rank0(f"Auto-selected attention backend: {config.attention_backend}")
_validate_attention_backend_choice(config, override, required_attn_types)
if config.moe_cache_rate is not None:
total_experts = config.model_config.num_moe_layers * config.model_config.num_experts
override("moe_cache_size", math.ceil(total_experts * config.moe_cache_rate))
# The CPU MoE executor supports the silu/gelu family plus the clamped
# swigluoai (csrc ActKind; "gpt_oss_swiglu" rides inside the mxfp4 kernel and
# swigluoai the generic GEMV epilogue). A model with any other expert
# activation cannot decode on the CPU: reject an explicit cpu/hybrid pick at
# config time.
# hidden_act (the dense activation) stands proxy for the expert activation --
# true for every in-tree model. mxfp4 experts pass regardless: their act runs
# inside the mxfp4 kernel, not the generic epilogue.
_cpu_moe_act_ok = getattr(model_config, "hidden_act", "silu") in _CPU_MOE_ACTS or (
getattr(model_config, "moe_weight_format", None) == "mxfp4"
)
if (
is_moe
and not _cpu_moe_act_ok
and (config.moe_strategy in ("cpu", "hybrid") or config.moe_cpu_layers)
):
asked = (
f"--moe-cpu-layers={config.moe_cpu_layers!r}"
if config.moe_strategy not in ("cpu", "hybrid")
else f"--moe-strategy {config.moe_strategy!r}"
)
raise ValueError(
f"{asked}: the CPU MoE executor does not support this model's expert "
f"activation {getattr(model_config, 'hidden_act', None)!r}; drop the flag "
"and let every layer decode on the GPU offload path instead."
)
if is_moe and config.moe_strategy == "auto":
# A MoE model always defaults to the offload family: experts stream from pinned host
# banks into an auto-sized GPU slot cache, which is the only default that serves a model
# bigger than the GPU. The resident 'fused' path (every expert format but
# DeepSeek-V4's) is still reachable, but only when asked for explicitly
# -- auto never picks it, because nothing here knows whether the experts would fit in
# HBM and a wrong guess is a weight-load OOM rather than a slower-but-working run.
default_backend = "offload"
# Unified memory (GB10/DGX Spark, Jetson): there is no host/device boundary, so
# the offload family stages and gathers between two names for the same DRAM (on
# GB10 this added a measured ~130 s stall to every request, #369). Resident
# experts are the safe default here, not the risky one: the model and its banks
# page from the same pool, so the "wrong guess = weight-load OOM" rationale above
# does not apply. Only formats the resident path
# can actually hold take this branch; the rest stay on offload as before.
unified_memory = _is_unified_memory_gpu()
if unified_memory and _fused_resident_ok(model_config):
default_backend = "fused"
logger.info_rank0(
"Unified-memory GPU detected; auto-selecting 'fused' MoE strategy "
"(resident experts) instead of offload"
)
# the cache sizing flags get their own warning in the 'fused' block below
inert = [flag for flag, on in (
(f"--moe-cpu-layers={config.moe_cpu_layers}", config.moe_cpu_layers),
("--moe-prefill-hit-d2d", config.moe_prefill_hit_d2d),
("--disable-moe-prefill-overlap", not config.moe_prefill_overlap),
) if on]
if inert:
logger.warning_rank0(
f"Unified-memory GPU serves MoE experts resident; ignoring {', '.join(inert)} "
"(use --moe-strategy offload to keep the offload flags)"
)
override("moe_cpu_layers", None)
override("moe_strategy", default_backend)
logger.info_rank0(f"Auto-selected MoE strategy: {config.moe_strategy}")
if (
is_offload_moe_strategy(config.moe_strategy)
and config.moe_cache_size <= 0
and config.moe_cache_rate is None
and config.moe_cache_mem is None
and not getattr(config, "moe_cache_auto", False)
):
# args.py's "no sizing flag -> default --moe-cache-auto" only fires when the
# backend is already offload-family at *parse* time. A bare `ft serve <FTW MoE
# checkpoint>` (no --moe-strategy, no cache flags) still has moe_strategy=="auto" at
# parse time -- the auto -> offload/cpu/hybrid resolution above is the first point
# the concrete backend is known, so mirror the same default here: no sizing flag
# was given, so let the scheduler resolve the cache size from free VRAM instead of
# failing the _require_offload_cache_size guard with size=0.
override("moe_cache_auto", True)
logger.info_rank0(
"No MoE cache sizing flag given; defaulting to --moe-cache-auto for "
f"auto-selected strategy {config.moe_strategy!r}"
)
if (
is_moe
and config.moe_strategy == "offload"
and _is_unified_memory_gpu()
and _fused_resident_ok(model_config)
):
# An explicit offload pick is honored, but on unified memory the user is paying
# for copies between two names for the same DRAM; say so once at config time.
logger.warning_rank0(
"--moe-strategy offload on a unified-memory GPU: expert 'streaming' copies "
"DRAM to DRAM (there is no PCIe link to overlap it with). If the model fits, "
"--moe-strategy fused avoids the slot-cache machinery entirely."
)
if is_moe and config.moe_strategy == "fused":
# An explicit 'fused' keeps the experts resident, so there is no slot cache to size. The
# sizing flags no longer redirect the backend, so ignore them here and say so -- the
# geometry the user asked for is what runs. Report the flag actually passed: --moe-cache-
# rate was already folded into moe_cache_size above, and the sizing flags are mutually exclusive.
if config.moe_cache_rate:
inert = f"--moe-cache-rate={config.moe_cache_rate}"
elif config.moe_cache_size:
inert = f"--moe-cache-size={config.moe_cache_size}"
elif config.moe_cache_mem is not None:
inert = f"--moe-cache-mem={config.moe_cache_mem}"
elif getattr(config, "moe_cache_auto", False):
inert = "--moe-cache-auto"
else:
inert = None
if inert:
logger.warning_rank0(
f"MoE backend 'fused' keeps its experts resident; ignoring {inert} "
"(use --moe-strategy offload to serve experts from a slot cache)"
)
override("moe_cache_size", 0)
override("moe_cache_rate", None)
override("moe_cache_mem", None)
override("moe_cache_auto", False)
if is_moe and config.moe_strategy == "cpu":
# CPU-compute decode keeps experts in host RAM and computes them on the CPU;
# the GPU only holds the two-layer prefill double buffer. So the slot cache is
# fixed at exactly two expert layers (prefill overlap requires >= 2*num_experts)
# and --moe-cache-size / --moe-cache-auto / --moe-cache-rate do not apply.
num_experts = config.model_config.num_experts
if getattr(config, "moe_cache_auto", False):
override("moe_cache_auto", False)
override("moe_cache_mem", None)
override("moe_cache_size", 2 * num_experts)
override("moe_prefill_overlap", True)
logger.info_rank0(
f"MoE backend 'cpu': decode computes experts on CPU; GPU keeps a "
f"two-layer prefill buffer (moe_cache_size={2 * num_experts})"
)
if (
is_moe
and expert_quant not in _RESIDENT_EXPERT_QUANTS
and not is_offload_moe_strategy(config.moe_strategy)
):
raise ValueError(
f"{expert_quant} experts require --moe-strategy offload or cpu, "
f"got {config.moe_strategy!r}"
)
if is_moe and config.moe_cpu_layers and config.moe_strategy not in ("offload", "hybrid"):
# the layer split needs the offload host banks + slot cache; 'cpu' already runs every layer on CPU, 'fused' keeps experts resident on the GPU (no host banks)
raise ValueError(
"--moe-cpu-layers requires --moe-strategy offload or hybrid (got "
f"{config.moe_strategy!r}); use --moe-strategy cpu to run all layers on CPU"
)
if is_moe and config.moe_cpu_layers and config.moe_cpu_layers.strip() != "auto":
if not _parse_cpu_layers_spec(config.moe_cpu_layers, model_config.num_moe_layers):
override("moe_cpu_layers", None)
if is_moe:
object.__setattr__(model_config, "moe_strategy", config.moe_strategy)
object.__setattr__(model_config, "decode_target", _decode_target(config))
# Must stay LAST: page_size is only final here (_adjust_dsv4_config sets P=128, the
# TRTLLM block sets 64). Also covers the programmatic LLM(...) path that bypasses parse_args.
if config.num_token_override is not None:
if config.num_page_override is not None:
raise ValueError("--num-tokens and --num-pages are mutually exclusive")
if config.num_token_override % config.page_size != 0:
raise ValueError(
f"--num-tokens {config.num_token_override} is not a multiple of the resolved "
f"page size {config.page_size}; nearest valid values: "
f"{config.num_token_override // config.page_size * config.page_size} or "
f"{(config.num_token_override // config.page_size + 1) * config.page_size}"
)
override("num_page_override", config.num_token_override // config.page_size)
# The rope cos/sin table covers rotary_config.table_positions, and neither rope kernel
# bounds-checks the position it gathers with -- a longer ceiling reads past the table.
# DSV4 is exempt: it sizes its own table from the resolved max_seq_len (_adjust_dsv4_config).
rotary = getattr(model_config, "rotary_config", None)
seq_override = getattr(config, "max_seq_len_override", None)
if seq_override is not None and rotary is not None and not (is_dsv4 or is_dsv41):
if seq_override > rotary.table_positions:
raise ValueError(
f"--max-seq-len-override {seq_override} exceeds the model's "
f"rope table ({rotary.table_positions} positions). Serving past it would read "
"out of bounds; extend the rope with --hf-overrides (YaRN rope_parameters) instead."
)
# The startup ServerArgs dump is the *requested* config, printed in the frontend process
# before any of the resolution above ran -- so "moe_strategy='auto'" is all it can say. This
# is the one line that reports what actually runs, for every path (explicit backends never
# hit an "Auto-selected ..." log at all).
resolved = [
f"attention_backend={config.attention_backend!r}",
f"cache_type={getattr(config, 'cache_type', 'radix')!r}",
f"page_size={config.page_size}",
]
if is_moe:
resolved.insert(0, f"moe_strategy={config.moe_strategy!r}")
logger.info_rank0(f"Resolved config: {', '.join(resolved)}")