Skip to content

FreeToken brings datacenter-scale model serving to your desktop. Run massive models locally, fast and efficiently.

110 KB
Raw Blame
from __future__ import annotations

import collections
import contextlib
import errno
import gc
import math
import os
from datetime import timedelta
from typing import Any, Dict, Iterable, NamedTuple, Tuple

import torch
from freetoken.attention import AttnType, attention_backend_info, create_attention_backend
from freetoken.attention.linear import LinearAttnBackend
from freetoken.core import Batch, Context, Req, set_global_ctx
from freetoken.distributed import destroy_distributed, enable_pynccl_distributed, set_tp_info
from freetoken.gpu_select import gpu_identity
from freetoken.checkpoint.ftw import ftw_tensor_names, is_ftw_checkpoint
from freetoken.layers import OffloadMoELayer, VocabParallelEmbedding, iter_moe_layers, iter_ops, set_rope_device
from freetoken.layers.quantization import LayerKind, QuantBackend, QuantKind, finalize_quant, set_quant_backend
from freetoken.moe.offload_cache import iter_offload_moe_layers
from freetoken.mm.config import ENCODER_SECTIONS
from freetoken.models import create_model, load_weight
from freetoken.models.weight import ftw_lacks_vision
from freetoken.moe import is_offload_moe_strategy
from freetoken.moe.expert_banks import attach_resident_banks, load_expert_banks, pin_probe_enabled
from freetoken.moe.host_banks import PinFailed
from freetoken.moe.locked_fetch import attach_locked_fetch, check_locked_fetch_config, locked_pool_tags, settle_banks, staging_reserve
from freetoken.moe.offload_cache import OffloadMoeCache, attach_offload_moe_cache, bank_layouts
from freetoken.moe.pool_policy import group_layers, make_pools, pool_of_layer, resolve_profile, split_slots
from freetoken.utils import align_ceil, init_logger, is_sm90_family, is_sm100_family, mem_GB, torch_dtype

from .config import EngineConfig
from .graph import GraphRunner, get_free_memory
from .sample import BatchSamplingArgs, Sampler
from freetoken.kvcache import create_kv_pool, resolve_pool_class
from freetoken.kvcache.base import CacheRebuildRejected
from freetoken.kvcache.cache_status import _supports_swa_ratio
from freetoken.kvcache.linear_state_pool import (
    _linear_pool_min_slots, _linear_pool_num_slots, state_pool_bytes,
)

logger = init_logger(__name__)

def _clamp_decode_batch_to_pools(config: EngineConfig, pool_sizes: list[int]) -> None:
    """Clamp the decode batch so every GPU-decoded layer's routed experts fit in its pool."""
    num_experts = config.model_config.num_experts
    small = [s for s in pool_sizes if s < num_experts]
    if not small:
        return
    # a decode step routes every row of every request, decode_rows_per_req rows each
    limit = min(small) // (config.model_config.num_experts_per_tok * config.decode_rows_per_req)
    if config.max_running_req > limit:
        logger.warning_rank0(
            f"the smallest MoE slot pool has {min(small)} slots; lowering max_running_req "
            f"{config.max_running_req} -> {limit} so a decode step fits in it"
        )
        object.__setattr__(config, "max_running_req", limit)
    if config.cuda_graph_max_bs is None or config.cuda_graph_max_bs > limit:
        object.__setattr__(config, "cuda_graph_max_bs", limit)
    if config.cuda_graph_bs is not None:
        object.__setattr__(config, "cuda_graph_bs", [bs for bs in config.cuda_graph_bs if bs <= limit])


def _require_offload_cache_size(cache_size: int, num_experts: int) -> None:
    """The offload MoE cache needs at least one slot per expert per layer. A too-small size
    (e.g. a bare offload run with moe_cache_size unset and auto disabled) must fail loudly."""
    if cache_size < num_experts:
        raise ValueError(
            f"moe_cache_size={cache_size} is too small: need at least num_experts={num_experts} "
            f"slots. Pass --moe-cache-size/--moe-cache-rate, or use --moe-cache-auto "
            f"(the default for offload/hybrid backends when no cache-sizing flag is given; "
            f"--moe-strategy cpu always sizes its own fixed two-layer buffer and ignores "
            f"cache-sizing flags)."
        )


def _flashinfer_available() -> bool:
    from freetoken.kernel.backend import is_flashinfer_installed

    return is_flashinfer_installed()


def _sgl_flash_attn_available() -> bool:
    from freetoken.kernel.backend import is_rocm

    if is_rocm():
        return False
    try:
        from sgl_kernel.flash_attn import flash_attn_with_kvcache  # noqa: F401
    except Exception as exc:
        detail = next((line.strip() for line in str(exc).splitlines() if line.strip()), "")
        logger.warning_rank0(
            "sgl_kernel.flash_attn is unavailable; auto attention backend falls back to fi "
            f"({type(exc).__name__}: {detail})"
        )
        return False
    return True


def _startup_kv_budget(memory_ratio: float, init_free_memory: int, new_free_memory: int) -> int:
    """Bytes available to the KV pool at startup: ratio-scaled pre-load free memory minus
    what the resident model consumed. Kept as a pure function so the composition with the
    pool families' ``solve_num_pages`` stays CPU-testable."""
    return int(memory_ratio * init_free_memory) - (init_free_memory - new_free_memory)


def _page_table_width(max_seq_len: int, page_size: int) -> int:
    """Column count for the page table. ``_write_page_table`` writes WHOLE trailing pages, so the
    highest column touched is ``align_ceil(max_seq_len, page_size) - 1`` -- which the 32-alignment
    alone does not cover once page_size > 32 (an unaligned --max-seq-len-override on DSV4's P=128
    or trtllm's forced 64 would index past the row)."""
    return align_ceil(align_ceil(max_seq_len, page_size), 32)


def _seq_limits(config, num_tokens: int) -> tuple[int, int]:
    """(longest request, page-table width). A decode step may widen a request by decode_rows_per_req - 1 provisional rows past its length, so requests get that much less and the table still covers the widened tail."""
    limit = min(config.max_seq_len, num_tokens)
    return limit - (config.decode_rows_per_req - 1), _page_table_width(limit, config.page_size)


def _required_attn_types(model_config) -> frozenset[AttnType]:
    """Backend-driving attention types of this model, from the group-spec walk
    (single source shared with the pool factory and the KV cost model). getattr
    fallbacks: duck-typed test configs may not implement the spec walk; for those,
    dsv4_args marks DSV4 (the real config declares a DSV4 attention group)."""
    specs_fn = getattr(model_config, "kv_cache_group_specs", None)
    if specs_fn is None:
        if getattr(model_config, "dsv4_args", None) is not None:
            return frozenset({AttnType.DSV4})
        return frozenset({AttnType.FULL})
    types = frozenset(
        spec.attn_type for spec in specs_fn() if spec.attn_type.backend_driven
    )
    return types or frozenset({AttnType.FULL})


def _backend_parts_serve(name: str, required: frozenset[AttnType]) -> bool:
    return all(
        required <= attention_backend_info(part).supported_types
        for part in name.split(",")
    )


def _backend_requirements_met(name: str) -> bool:
    # flashinfer first across ALL parts: the sgl probe logs a "falls back to fi" warning,
    # which would mislead when the candidate is about to fail on flashinfer anyway.
    infos = [attention_backend_info(part) for part in name.split(",")]
    if any(i.requires_flashinfer for i in infos) and not _flashinfer_available():
        return False
    if any(i.requires_sgl_kernel for i in infos) and not _sgl_flash_attn_available():
        return False
    if any(i.requires_sm100 for i in infos) and not is_sm100_family():
        return False
    return True


def _resolve_auto_attention_backend(required: frozenset[AttnType]) -> str:
    """First candidate (in per-type priority order) whose arch condition holds,
    whose packages are installed, and whose every comma part serves ALL required
    types. Reproduces the historical hardware tree for FULL-only models:
    sm_100 -> trtllm, sm_90+sgl_kernel -> "fa,fi", flashinfer -> fi, else triton."""
    candidates: list[tuple[str, bool]] = []
    if AttnType.DSV4 in required:
        candidates.append(("dsv4_sparse", True))
    if AttnType.DSV41 in required:
        candidates.append(("dsv41_sparse", True))
    if required & {AttnType.MLA, AttnType.DSA}:
        candidates.append(("dsa", True))
    if AttnType.BSA in required:
        candidates.append(("m3_sparse", True))
    if AttnType.QSA in required:
        candidates.append(("qsa_sparse", True))
    if AttnType.SWA in required:
        candidates.append(("triton", True))
    if AttnType.FULL in required:
        candidates += [
            ("trtllm", is_sm100_family()),
            ("fa,fi", is_sm90_family()),
            ("fi", True),
            ("triton", True),
        ]
    for name, arch_ok in candidates:
        if not arch_ok:
            continue
        if not _backend_parts_serve(name, required):
            continue
        if not _backend_requirements_met(name):
            continue
        return name
    raise RuntimeError(
        "No attention backend can serve attention types "
        f"{sorted(t.value for t in required)} on this machine."
    )


def _validate_attention_backend_choice(config, override, required: frozenset[AttnType]) -> None:
    """Config-time type x backend capability check for the resolved (or explicit)
    backend string: every comma part must serve every required type and have its
    packages/arch available. Replaces the per-model gates; in particular this is
    where a DSV4 or MLA checkpoint rejects a generic backend before weights load,
    and where a generic model rejects dsa/dsv4_sparse."""
    from freetoken.attention import validate_attn_backend
    from freetoken.kernel.backend import is_rocm

    # Name membership first (ArgumentTypeError listing the supported names): the CLI already
    # ran this, but the programmatic EngineConfig path reaches here unvalidated and would
    # otherwise die on a bare KeyError from the info lookup below.
    validate_attn_backend(config.attention_backend, allow_auto=False)

    model_config = config.model_config
    backend_parts = [p.strip() for p in config.attention_backend.split(",")]
    for part in backend_parts:
        info = attention_backend_info(part)
        missing = required - info.supported_types
        if missing:
            valid = [
                name
                for name in (
                    "fa", "fi", "trtllm", "triton", "dsa", "dsv4_sparse", "m3_sparse",
                    "qsa_sparse",
                )
                if required <= attention_backend_info(name).supported_types
            ]
            missing_names = "/".join(sorted(t.value for t in missing))
            raise ValueError(
                f"{getattr(model_config, 'model_type', 'model')} uses {missing_names} "
                f"attention, which backend {part!r} does not support; valid backends: "
                f"{', '.join(valid)} (or auto), got {config.attention_backend!r}."
            )
        if AttnType.SWA in required and not info.consumes_attn_spec:
            # SWA models drive window/sinks/sm_scale through the per-call AttentionSpec;
            # a backend that drops it would attend with the wrong window silently.
            raise ValueError(
                f"backend {part!r} does not consume the per-call AttentionSpec that "
                f"SWA models require, got {config.attention_backend!r}."
            )

    # An explicitly-selected backend may require a package that isn't installed. Auto
    # never resolves to one of these when its package is missing, so this only fires for
    # explicit --attention-backend choices.
    for part in backend_parts:
        info = attention_backend_info(part)
        if info.requires_flashinfer and not _flashinfer_available():
            if is_rocm():
                raise RuntimeError(
                    f"Attention backend {part!r} uses CUDA-only FlashInfer kernels "
                    "unavailable on ROCm; use --attention-backend auto or triton."
                )
            raise RuntimeError(
                f"Attention backend {config.attention_backend!r} requires flashinfer, which is "
                "not installed. Install it with `pip install 'freetoken[fi]'` (or "
                "'freetoken[accel]'), or use --attention-backend triton."
            )
        if info.requires_sgl_kernel and not _sgl_flash_attn_available():
            if is_rocm():
                raise RuntimeError(
                    f"Attention backend {part!r} uses CUDA-only sgl_kernel kernels "
                    "unavailable on ROCm; use --attention-backend auto or triton."
                )
            raise RuntimeError(
                f"Attention backend {config.attention_backend!r} requires sgl_kernel, which is "
                "not installed. Install it with `pip install 'freetoken[sgl]'` (or "
                "'freetoken[accel]'), or use --attention-backend triton."
            )
        if info.requires_sm100 and not is_sm100_family():
            raise RuntimeError(
                f"Attention backend {config.attention_backend!r} requires a compute capability "
                "10.x GPU: flashinfer's trtllm-gen kernels ship sm_100a/103a cubins only. "
                "Use --attention-backend fi (or triton) instead."
            )

    if required & {AttnType.MLA, AttnType.DSA}:
        # Plain MLA/DSA runs on page_size 1; the kpool indexer layout needs 64.
        _kpool_ratio = max(
            (s.index_ratio for s in model_config.kv_cache_group_specs() if s.mla),
            default=1,
        )
        want_page = 64 if _kpool_ratio > 1 else 1
        if config.page_size != want_page:
            logger.warning_rank0(
                f"Page size {config.page_size} is auto-adjusted to {want_page} "
                f"for latent-KV attention."
            )
            override("page_size", want_page)

    for part in backend_parts:
        info = attention_backend_info(part)
        if info.page_sizes is not None and config.page_size not in info.page_sizes:
            override("page_size", info.page_sizes[-1])
            logger.warning_rank0(
                f"Page size is overridden to {info.page_sizes[-1]} for the {part} backend"
            )


def _host_embedding_keys(model, config: EngineConfig) -> frozenset[str]:
    """State-dict keys of the tables --embed-weights host keeps in pinned host RAM."""
    if config.embed_weights != "host":
        return frozenset()
    if config.model_config.tie_word_embeddings:
        raise ValueError(
            "--embed-weights host: this model's embedding table is tied to its LM head, which reads the whole table "
            "every step; serve it with --embed-weights gpu"
        )
    tables = {id(op.weight) for op in iter_ops(model) if type(op) is VocabParallelEmbedding}
    keys = frozenset(key for key, t in model.state_dict().items() if id(t) in tables)
    if not keys:
        raise ValueError(
            f"--embed-weights host: {type(model).__name__} has no plain embedding table to move (a DF11 table stays "
            "on the GPU); serve it with --embed-weights gpu"
        )
    return keys


def _pinned_copy(key: str, t: torch.Tensor) -> torch.Tensor:
    from freetoken.kernel.pinned import alloc_pinned_tensor

    try:
        host = alloc_pinned_tensor(*t.shape, dtype=t.dtype)
    except RuntimeError as exc:
        raise PinFailed(
            f"cudaHostAlloc refused {t.nbytes / 2**20:.0f} MiB for {key} (--embed-weights host); "
            "free host RAM or serve with --embed-weights gpu"
        ) from exc
    host.copy_(t)
    return host


def _make_dummy_weight_state_dict(
    model_state: Dict[str, torch.Tensor],
    *,
    device: torch.device,
    host_keys: frozenset[str] = frozenset(),
) -> Dict[str, torch.Tensor]:
    state_dict: Dict[str, torch.Tensor] = {}
    fp8_dtypes = (torch.float8_e4m3fn, torch.float8_e5m2)
    for key, param in model_state.items():
        if param.dtype in fp8_dtypes:
            # torch.randn is not implemented for fp8; fill via a uint8 view with small
            # codes (avoid NaN/inf fp8 encodings). Lets dummy-weight startup work for
            # block-fp8 models (the dense fp8 linears are fp8 regardless of moe_strategy).
            t = torch.empty(param.shape, dtype=param.dtype, device=device)
            t.view(torch.uint8).random_(0, 16)
            state_dict[key] = t
        elif param.dtype.is_floating_point or param.dtype.is_complex:
            state_dict[key] = torch.randn(param.shape, dtype=param.dtype, device=device)
        elif param.dtype == torch.uint8 and key.endswith("weight_scale_inv"):
            # MXFP8 e8m0 exponent codes: 127 encodes scale 1.0; zeros would collapse
            # every scale to 2^-127 and zero the model. Scoped BY NAME: other uint8
            # buffers are packed payloads whose bytes mean something else entirely
            # (GGUF blocks embed fp16 scales -- 0x7F7F is fp16 NaN), so they
            # keep the benign all-zeros fill below.
            state_dict[key] = torch.full(param.shape, 127, dtype=param.dtype, device=device)
        else:
            state_dict[key] = torch.zeros(param.shape, dtype=param.dtype, device=device)
        if key in host_keys:
            state_dict[key] = _pinned_copy(key, state_dict[key])
    return state_dict


class WeightLoadError(RuntimeError):
    """The checkpoint itself could not be read. Resource and config failures keep their own type."""


def _is_resource_failure(exc: Exception) -> bool:
    if isinstance(exc, (torch.OutOfMemoryError, MemoryError, PinFailed)):
        return True
    # an anonymous mmap that does not fit raises ENOMEM, not MemoryError
    if isinstance(exc, OSError) and exc.errno == errno.ENOMEM:
        return True
    # torch has no type for a failed CPU allocation
    return isinstance(exc, RuntimeError) and "DefaultCPUAllocator" in str(exc)


@contextlib.contextmanager
def _weight_load_context():
    """Wrap a checkpoint read so the startup failure reason (log and /health) starts with WeightLoadError."""
    try:
        yield
    except Exception as exc:
        if _is_resource_failure(exc):
            raise
        raise WeightLoadError(f"{type(exc).__name__}: {exc}") from exc


def _materialize_loaded_weight_state_dict(
    model_state: Dict[str, torch.Tensor],
    weights: Iterable[Tuple[str, torch.Tensor]],
    *,
    device: torch.device,
    host_keys: frozenset[str] = frozenset(),
) -> Dict[str, torch.Tensor]:
    state_dict: Dict[str, torch.Tensor] = {}
    for key, weight in weights:
        expected = model_state.get(key)
        if expected is None:
            # NOTE: the quant scheme may declare no input_scale for a layer whose FTW still stores one
            if key.endswith(".input_scale"):
                continue
            state_dict[key] = weight.to(device=device)
        elif key in host_keys:
            state_dict[key] = _pinned_copy(key, weight.to(dtype=expected.dtype))
        else:
            state_dict[key] = weight.to(device=device, dtype=expected.dtype)
    return state_dict


class ForwardOutput(NamedTuple):
    next_tokens_gpu: torch.Tensor
    next_tokens_cpu: torch.Tensor
    copy_done_event: torch.cuda.Event


class Engine:
    def __init__(self, config: EngineConfig):
        assert not torch.cuda.is_initialized()
        set_tp_info(rank=config.tp_info.rank, size=config.tp_info.size)
        set_quant_backend(_adjust_ftw_quant_backend(config.model_path, QuantBackend.parse(config.quant_backend)))
        _ensure_expandable_segments()  # before the first CUDA allocation below

        from freetoken.gpu_select import bind_assigned_gpu

        self.device = bind_assigned_gpu(config.tp_info.rank)
        _adjust_config(config)
        torch.manual_seed(42)
        self.stream = torch.cuda.Stream()
        torch.cuda.set_stream(self.stream)
        self.dtype = config.dtype
        self.config = config  # retained for runtime cache rebuild (rebuild_runtime_cache)
        # KV pool family fixed at construction from the model config: its classmethods own the
        # page-token geometry and cost arithmetic the engine needs BEFORE the pool exists
        # (num_pages sizing, --moe-cache-auto); the instance owns rebuild/validation after.
        self._pool_cls = resolve_pool_class(config.model_config)
        self.ctx = Context(config.page_size)
        set_global_ctx(self.ctx)

        self.tp_cpu_group = self._init_communication(config)
        free_min, free_max = self._sync_get_memory()
        init_free_memory = free_max  # startup KV sizing keeps cross-rank MAX (unchanged)
        self._baseline_free = free_min  # rebuild baseline: cross-rank MIN, deterministic across ranks
        logger.info_rank0(f"Free memory before loading model: {mem_GB(init_free_memory)}")

        # ======================= Model initialization ========================
        set_rope_device(self.device)
        with torch.device("meta"), torch_dtype(config.dtype):
            self.model = create_model(config.model_config)
        self._load_weights(config)
        if config.active_encoders:
            from freetoken.models.blocks import SupportsMultimodal

            if not isinstance(self.model, SupportsMultimodal):
                raise TypeError(
                    f"{type(self.model).__name__} has encoders registered but lacks the SupportsMultimodal hooks; "
                    "run with --text-model-only"
                )
            # before the residency snapshot, so streamed blocks are not charged as resident weights
            self.model.place_encoder_weights(config.mm.encoder_weights)
        post_weights_free = self._sync_get_memory()[0]
        self._weights_bytes = self._baseline_free - post_weights_free
        # Pool-budget baseline for the desktop cache sliders: free VRAM after the weights are
        # resident but before ANY runtime cache pool (MoE expert cache below, KV pages, GDN
        # state) is allocated. This is the stable "if all free VRAM went to one pool" budget —
        # unlike a query-time mem_get_info it doesn't drift with allocator caching, CUDA
        # graphs, or other processes. Cross-rank MIN, deterministic across ranks.
        self._post_weights_free = post_weights_free
        self.moe_offload_cache = None
        self.cpu_moe_executor = None
        # Host-side auxiliary stores (--embed-weights host tables, qwen4_exp's pinned PLE table): after the weights so a
        # load failure is not masked, before the MoE offload cache so the bank residency
        # planning sees the pin quota the table already spent.
        host_embeds = [op for op in iter_ops(self.model) if type(op) is VocabParallelEmbedding and not op.weight.is_cuda]
        self._host_tables_bytes = sum(op.weight.nbytes for op in host_embeds)
        if host_embeds:
            logger.info_rank0(f"Embedding tables in pinned host RAM (--embed-weights host): {mem_GB(self._host_tables_bytes)}")
        if hasattr(self.model, "load_host_tables"):
            with _weight_load_context():
                self._host_tables_bytes += int(self.model.load_host_tables(config) or 0)
        if is_offload_moe_strategy(config.moe_strategy):
            self._init_offload_moe_cache(config)
        if hasattr(self.model, "prepare_for_runtime"):
            self.model.prepare_for_runtime()
        self.encoder_cache = None
        self.mm_processor = None
        if config.active_encoders:
            from freetoken.mm.encoder_cache import EncoderCache
            from freetoken.mm.processor import get_mm_processor

            self.mm_processor = get_mm_processor(config.model_path, config.mm)
            self.encoder_cache = EncoderCache(storage=config.mm.embed_cache_device)
            logger.info_rank0(
                f"Multimodal enabled: {type(self.mm_processor).__name__}, encoders "
                f"{[e.kind for e in config.active_encoders]} on {config.mm.encoder_weights}, serving {sorted(config.served_modalities)}"
            )
            self._warmup_encoders()
        elif any(getattr(config.hf_config, key, None) is not None for key in ENCODER_SECTIONS):
            logger.info_rank0(
                "Multimodal disabled: --text-model-only"
                if config.mm.text_model_only
                else "Multimodal disabled: --mm-disable"
                if config.mm.disabled_encoders
                else "Multimodal disabled: no encoder registered for this architecture"
            )

        # ======================= KV cache initialization ========================
        new_free = self._sync_get_memory()[1]
        # The engine measures the budget and settles the sibling GDN state pool's bytes
        # off it; the KV pool family owns every geometry-specific formula behind the rest.
        available_memory = _startup_kv_budget(config.memory_ratio, init_free_memory, new_free)
        available_memory -= state_pool_bytes(config)
        self.num_pages = self._pool_cls.solve_num_pages(config, available_memory)
        num_tokens = self.num_pages * config.page_size
        self.ctx.kv_cache = self.kv_cache = create_kv_pool(
            config, self.num_pages, device=self.device, dtype=self.dtype
        )

        # ======================= Linear (GatedDeltaNet) state initialization ========================
        linear_group = config.model_config.linear_attention_group()
        if linear_group is not None:
            from freetoken.kvcache.linear_state_pool import LinearStatePool

            self.linear_state_pool = LinearStatePool(
                group=linear_group,
                num_slots=_linear_pool_num_slots(config),
                dtype=self.dtype,
                device=self.device,
                tp_size=config.tp_info.size,
                slot_states=config.model_config.slot_states,
            )
            self.ctx.linear_state_pool = self.linear_state_pool
        else:
            self.linear_state_pool = None

        # ======================= Page table initialization ========================
        # NOTE: 1. aligned to 128 bytes; 2. store raw locations instead of pages
        self.max_seq_len, aligned_max_seq_len = _seq_limits(config, num_tokens)
        self.ctx.page_table = self.page_table = torch.zeros(  # + 1 for dummy request
            (config.max_running_req + 1, aligned_max_seq_len),
            dtype=torch.int32,
            device=self.device,
        )
        # Pools routed by the shared table but deriving reads through their own mappings (DSV4)
        # re-point here (and again on any table realloc). The graph-input snapshot that reads
        # through them belongs to the attention backend, built later in init_capture_graph.
        self.kv_cache.attach_page_table(self.page_table)

        # ======================= Attention backend initialization ========================
        self.ctx.attn_backend = self.attn_backend = create_attention_backend(
            config.attention_backend, config.model_config
        )
        self.linear_backend = LinearAttnBackend(self.device) if self.linear_state_pool is not None else None

        # ======================= Sampler initialization ========================
        self.sampler = Sampler(self.device, config.model_config.vocab_size)

        post_free_memory = self._sync_get_memory()[0]
        logger.info_rank0(f"Free memory after initialization: {mem_GB(post_free_memory)}")

        # ======================= Graph capture initialization ========================
        self.dummy_req = Req(
            input_ids=torch.tensor([0], dtype=torch.int32, device="cpu"),
            table_idx=config.max_running_req,
            cached_len=0,
            output_len=1,
            uid=-1,
            sampling_params=None,  # type: ignore
            cache_handle=None,  # type: ignore
        )
        # padded/dummy rows index the GDN padding slot (0) so gather/scatter hits scratch.
        # Naive GDN keys state by table_idx, where slot 0 is a live request and the dummy's own row is the scratch.
        if self.linear_state_pool is not None and config.cache_type == "hybrid_radix":
            self.dummy_req.linear_slot_idx = self.linear_state_pool.padding_slot
        self.page_table[self.dummy_req.table_idx].fill_(num_tokens)  # point to dummy page
        self.graph_runner = GraphRunner(
            stream=self.stream,
            device=self.device,
            forward=self.model.forward,
            attn_backend=self.attn_backend,
            cuda_graph_bs=config.cuda_graph_bs,
            cuda_graph_max_bs=config.cuda_graph_max_bs,
            free_memory=init_free_memory,
            max_seq_len=aligned_max_seq_len,
            dummy_req=self.dummy_req,
            moe_offload_cache=self.moe_offload_cache,
            mrope=config.model_config.model_is_mrope,
            out_dtype=torch.float32,
            linear_backend=self.linear_backend,
        )
        if config.attention_backend.split(",")[0] == "triton":
            # Prefill runs on the first comma part; warm its autotune cache.
            self._warmup_prefill()

    def _init_communication(self, config: EngineConfig) -> torch.distributed.ProcessGroup:
        if config.tp_info.size == 1 or config.use_pynccl:
            torch.distributed.init_process_group(
                backend="gloo",
                rank=config.tp_info.rank,
                world_size=config.tp_info.size,
                timeout=timedelta(seconds=config.distributed_timeout),
                init_method=config.distributed_addr,
            )
            tp_cpu_group = torch.distributed.group.WORLD
            assert tp_cpu_group is not None
            max_bytes = (
                config.max_forward_len * config.model_config.hidden_size * self.dtype.itemsize
            )
            enable_pynccl_distributed(config.tp_info, tp_cpu_group, max_bytes)
        else:
            torch.distributed.init_process_group(
                backend="nccl",
                rank=config.tp_info.rank,
                world_size=config.tp_info.size,
                timeout=timedelta(seconds=config.distributed_timeout),
                init_method=config.distributed_addr,
            )
            tp_cpu_group = torch.distributed.new_group(backend="gloo")
            assert tp_cpu_group is not None
        return tp_cpu_group

    def _load_weights(self, config: EngineConfig) -> None:
        if config.active_encoders and not config.use_dummy_weight and ftw_lacks_vision(config.model_path):
            raise ValueError(
                f"{config.model_path} holds no vision encoder tensors: it was converted by a build before this "
                "family served images. Reconvert it with `ft checkpoint`, add the encoder in place with "
                "scripts/ftw_hotfix.py (docs/ftw-hotfix.md), or start with --text-model-only"
            )
        resident = [layer for layer in iter_moe_layers(self.model) if not isinstance(layer, OffloadMoELayer)]
        if resident and not config.use_dummy_weight and is_ftw_checkpoint(config.model_path) and not ftw_tensor_names(config.model_path, "experts_bank"):
            raise ValueError(f"{config.model_path} keeps its routed experts as dense weights (an older `ft checkpoint --moe-backend fused`); reconvert it with `ft checkpoint`")
        host_keys = _host_embedding_keys(self.model, config)
        with _weight_load_context():
            self.model.load_state_dict(self._load_weight_state_dict(config, host_keys))
        finalize_quant(self.model)
        if resident:
            self._load_resident_experts(config, resident)

    def _load_weight_state_dict(self, config: EngineConfig, host_keys: frozenset[str]) -> Dict[str, torch.Tensor]:
        model_state = self.model.state_dict()
        if config.use_dummy_weight:
            return _make_dummy_weight_state_dict(model_state, device=self.device, host_keys=host_keys)
        # _materialize casts each loaded tensor to its model-param dtype (model_state), so
        # models declaring per-tensor dtypes (e.g. DSV4's mixed fp8/fp32/bf16) are preserved;
        # MoE models exclude routed experts (they load as expert banks, not dense weights).
        return _materialize_loaded_weight_state_dict(
            model_state,
            load_weight(
                config.model_path,
                self.device,
                include_moe_experts=not getattr(config.model_config, "is_moe", False),
                include_vision=bool(config.active_encoders),
            ),
            device=self.device,
            host_keys=host_keys,
        )

    def _load_resident_experts(self, config: EngineConfig, layers) -> None:
        with _weight_load_context():
            banks = load_expert_banks(
                config.model_path,
                config.model_config,
                method=layers[0].quant_method,
                device=self.device,
                dummy=config.use_dummy_weight,
                parallel={"serial": False, "parallel": True}.get(config.expert_load, None),
                resident=True,
                layer_methods=[layer.quant_method for layer in sorted(layers, key=lambda layer: layer.layer_id)],
            )
        attach_resident_banks(layers, banks)

    @torch.inference_mode()
    def _warmup_encoders(self) -> None:
        for item in self.mm_processor.dummy_items(self.dtype, self.device):
            if item.modality in self.config.served_modalities:
                self.model.encode(item)
        torch.cuda.synchronize(self.device)

    @torch.inference_mode()
    def _run_mm_encoder(self, batch: Batch) -> None:
        """Encode the chunk's cache-miss items and gather its embedding rows into batch.mm_embeds, right before the LM forward."""
        cache = self.encoder_cache
        jobs = batch.mm_encoder_jobs or ()
        # a job whose rows are not gathered this chunk would be encoded and freed unread
        planned = {row[1] for row in batch.mm_gather_plan}
        orphans = [item.hash for item in jobs if item.hash not in planned]
        assert not orphans, f"encoder jobs without gather rows: {orphans}"
        for item in jobs:
            if not cache.has(item.hash):
                if item.precomputed_embeddings is not None:
                    emb = item.precomputed_embeddings.to(self.device, non_blocking=True)
                else:
                    emb = self.model.encode(item)
                cache.put(item.hash, emb)
            # cached now; free the ~MiB feature buffer
            item.feature = None
            item.precomputed_embeddings = None
        parts = []
        for uid, item_hash, lo, hi, _, _ in batch.mm_gather_plan:
            parts.append(cache.get_slice(item_hash, lo, hi, self.device))
            cache.consume(item_hash, uid, hi - lo)
        if parts:
            batch.mm_embeds = torch.cat([p.to(self.dtype) for p in parts], dim=0)

    def _resolve_auto_moe_cache_size(self, config: EngineConfig, banks, method=None, max_slots=None) -> tuple[int, int, bool]:
        """Resolve --moe-cache-auto into (moe_cache_size, num_pages, prefill_overlap).

        Pure glue over the Phase-1 budget policy; isolated here so it is unit-testable
        without a GPU. Reused by the Phase-2 runtime rebuild.
        """
        from freetoken.engine.cache_budget import expert_bytes_per_slot, resolve_moe_cache_auto

        cache_per_page, fixed_cache_size, page_tokens, min_reserve = self._pool_cls.kv_cost(config)
        fixed_cache_size += state_pool_bytes(config)  # sibling GDN state pool, engine-summed
        num_experts = config.model_config.num_experts
        total_experts = config.model_config.num_moe_layers * num_experts
        return resolve_moe_cache_auto(
            baseline_free=self._baseline_free,
            weights_bytes=self._weights_bytes,
            memory_ratio=config.memory_ratio,
            cache_per_page=cache_per_page,
            fixed_cache_size=fixed_cache_size,
            per_expert_bytes=expert_bytes_per_slot(banks.sources),
            num_experts=num_experts,
            total_experts=total_experts,
            prefill_overlap=config.moe_prefill_overlap,
            kv_reserve_tokens=max(config.kv_reserve_tokens, min_reserve),
            page_size=page_tokens,
            max_slots=max_slots if max_slots is not None or method is None else method.slot_limit(),
        )

    def _init_offload_moe_cache(self, config: EngineConfig) -> OffloadMoeCache:
        from freetoken.moe.expert_banks import is_mixed

        method = shared_offload_method(self.model)
        num_moe_layers = config.model_config.num_moe_layers
        # experts of several formats: one method per MoE layer, one slot pool per format
        layer_methods = offload_layer_methods(self.model)
        if not is_mixed(layer_methods):
            layer_methods = None
        elif config.moe_strategy == "cpu" or config.moe_cpu_layers:
            raise ValueError(
                "this model's experts come in several formats, which are served from per-format GPU slot pools; "
                "they do not combine with --moe-strategy cpu or --moe-cpu-layers (use offload or hybrid)"
            )
        # Fast path: an FTW checkpoint loads its repacked banks directly.
        # Slow path: load_expert_banks auto-picks parallel vs serial baseline by
        # expert-tensor granularity. Both pin-after-fill.
        # --expert-load: serial/parallel force the read; auto (None) lets load_expert_banks
        # pick (parallel for scattered experts, with a low-RAM fallback to serial).
        expert_parallel = {"serial": False, "parallel": True}.get(config.expert_load, None)
        check_locked_fetch_config(config)
        # --moe-cpu-layers auto without a manual pin budget: load unpinned, then pin layers until the driver refuses (settle_banks)
        # a model the CPU executor cannot serve takes the estimate path, which rejects it before the load when layers must lock
        probe = pin_probe_enabled(config, mixed_formats=layer_methods is not None) and _cpu_moe_executor_viable(config.model_config)
        if probe:
            from freetoken.moe.host_banks import HostResidency

            requested_residency = [HostResidency.DEFERRED.value] * num_moe_layers
            cpu_layer_ids, decode_target, split_residency = frozenset(), None, False
        else:
            reserved = self._host_tables_bytes + staging_reserve(config, lambda: _bank_bytes(config, method, layer_methods))
            cpu_layer_ids = _resolve_cpu_layers(config, num_moe_layers, reserved=reserved, method=method)
            _check_pin_budget(config, reserved=reserved, method=method, layer_methods=layer_methods)
            decode_target, split_residency = self._residency_split(config, cpu_layer_ids, method, capped=_pin_budget_bytes(reserved) is not None)
            requested_residency = None
            if split_residency:
                from freetoken.moe.host_banks import HostResidency

                requested_residency = [
                    HostResidency.LOCKED.value if i in cpu_layer_ids
                    else HostResidency.PINNED.value
                    for i in range(config.model_config.num_moe_layers)
                ]
        # read before the expert load, so a bad profile fails in seconds
        expert_counts = resolve_profile(
            config.moe_cache_split, config.model_path,
            config.model_config.num_moe_layers, config.model_config.num_experts,
        )
        try:
            with _weight_load_context():
                banks = load_expert_banks(
                    config.model_path,
                    config.model_config,
                    method=method,
                    device=self.device,
                    dummy=config.use_dummy_weight,
                    parallel=expert_parallel,
                    layer_residency=requested_residency,
                    layer_methods=layer_methods,
                )
        except PinFailed as exc:
            raise RuntimeError(f"{exc}; {_pin_hint(self._host_tables_bytes)}") from exc
        banks, probed, staging = settle_banks(config, banks, probe=probe, has_locked=split_residency)
        if probe:
            cpu_layer_ids = probed
            decode_target, split_residency = self._residency_split(config, cpu_layer_ids, method, capped=True)
        from freetoken.engine.cache_budget import expert_bytes_per_slot, layer_slot_bytes

        self._moe_layer_slot_bytes = layer_slot_bytes(banks.sources)
        # hybrid: locked layers keep GPU slots of their own pools and fetch part of their misses through the pinned staging (moe/locked_fetch.py)
        locked_fetch = staging is not None
        layer_tags = locked_pool_tags(banks.layer_tags, cpu_layer_ids, num_moe_layers) if locked_fetch else banks.layer_tags
        self._moe_pool_groups = group_layers(config.moe_cache_scope, bank_layouts(banks.sources, layer_tags), self._moe_layer_slot_bytes)
        if locked_fetch:
            # a locked refill costs a staged copy: place the locked pools after the pinned ones, which hold the prefill staging when they are big enough
            self._moe_pool_groups.sort(key=lambda layers: layers[0] in cpu_layer_ids)
        if len(self._moe_pool_groups) > 1 and cpu_layer_ids and not locked_fetch:
            raise ValueError(
                "several MoE slot pools (--moe-cache-scope per-layer, or experts in several quant formats) need every MoE "
                "layer to decode from the GPU slot cache; they combine with --moe-cpu-layers only for locked layers under "
                "--moe-strategy hybrid unless --moe-locked-fetch is 0"
            )
        self._moe_expert_counts = expert_counts if len(self._moe_pool_groups) > 1 else None
        if len(self._moe_pool_groups) == 1 and config.moe_cache_split != "equal":
            logger.warning_rank0(
                f"--moe-cache-split {config.moe_cache_split} is ignored: all MoE layers share one slot pool "
                f"(--moe-cache-scope per-layer gives each layer its own)"
            )
        self._moe_slot_limit = _slot_limit(layer_methods) if layer_methods is not None else (method.slot_limit() if method is not None else None)
        # with GPU decode, a pool below num_experts slots bounds the batch; the startup split keeps room for one request's decode step
        pools = self._moe_pools(config.decode_rows_per_req if decode_target == "gpu" else 1)
        size_cap = sum(p.max_num_slots for p in pools)
        if config.moe_cache_auto:
            size, pages, overlap = self._resolve_auto_moe_cache_size(config, banks, method, max_slots=size_cap)
            object.__setattr__(config, "moe_cache_size", size)
            object.__setattr__(config, "moe_prefill_overlap", overlap)
            if config.num_page_override is None:
                # Honor the plan's KV half too: MoE slots and KV pages were solved
                # against ONE budget (ratio x baseline - weights), so both must come
                # from it. Re-solving pages later from a fresh free-memory reading
                # double-counts everything allocated since the weights measurement
                # (this expert cache, the CPU-executor GPU buffers, allocator
                # slack) and goes negative whenever the expert fill is exact --
                # a greedy fill leaves no headroom for the measurement delta.
                object.__setattr__(config, "num_page_override", pages)
            logger.info_rank0(
                f"--moe-cache-auto resolved moe_cache_size={size} "
                f"num_pages={pages} (prefill_overlap={overlap})"
            )
        elif config.moe_cache_mem is not None:
            per_slot = expert_bytes_per_slot(banks.sources)
            size = min(config.moe_cache_mem // per_slot, size_cap)
            # per_slot is a mean over layers, so with pools of different slot sizes the split can exceed the budget: shrink until it fits
            while size > 0 and self._moe_pool_bytes(split_slots(size, pools, self._moe_expert_counts)) > config.moe_cache_mem:
                size -= 1
            overlap = config.moe_prefill_overlap and size >= 2 * config.model_config.num_experts
            object.__setattr__(config, "moe_cache_size", size)
            object.__setattr__(config, "moe_prefill_overlap", overlap)
            logger.info_rank0(
                f"--moe-cache-mem {mem_GB(config.moe_cache_mem)} resolved moe_cache_size={size} "
                f"({per_slot / 2**20:.2f} MiB per slot, prefill_overlap={overlap})"
            )
        _require_offload_cache_size(config.moe_cache_size, config.model_config.num_experts)
        pool_sizes = split_slots(config.moe_cache_size, pools, self._moe_expert_counts)
        if len(pools) > 1:
            logger.info_rank0(
                f"MoE slot pools: {config.moe_cache_scope}, {len(pools)} pools of {min(pool_sizes)}-{max(pool_sizes)} "
                f"slots ({config.moe_cache_split})"
            )
            if decode_target == "gpu":
                _clamp_decode_batch_to_pools(config, pool_sizes)
        layout = layer_layouts = max_slots = None
        if layer_methods is not None:
            _check_layer_formats(banks, layer_methods)
            layer_layouts = [m.layout() for m in layer_methods]
            max_slots = self._moe_slot_limit
        elif method is not None:
            if banks.layer_formats is not None:
                raise ValueError(
                    f"expert banks hold experts of several formats ({sorted(set(banks.layer_formats))}) but the model "
                    f"binds {method.kind} / {method.kernel.name} on every MoE layer; reconvert the checkpoint"
                )
            if banks.kind is not None and (banks.kind, banks.kernel) != (method.kind, method.kernel.name):
                raise ValueError(
                    f"expert banks were packed for {banks.kind} / {banks.kernel} but the model "
                    f"binds {method.kind} / {method.kernel.name}; reconvert or select that kernel"
                )
            # GGUF layers keep their own ggml types, so the cache reads each layer's layout off its banks
            layout = None if method.kind is QuantKind.GGUF else method.layout()
            max_slots = method.slot_limit()
        cache = OffloadMoeCache(
            # Models with leading dense layers (GLM-4) only have experts on the MoE
            # layers; num_moe_layers == num_layers when first_k_dense_replace == 0.
            num_layers=config.model_config.num_moe_layers,
            num_experts=config.model_config.num_experts,
            cache_size=config.moe_cache_size,
            device=self.device,
            cache_policy=config.moe_cache_policy,
            prefill_overlap=config.moe_prefill_overlap,
            prefill_hit_d2d=config.moe_prefill_hit_d2d,
            quant_format=banks.quant_format,
            decode_target=decode_target,
            hybrid_max_fetch=config.moe_hybrid_max_fetch,
            layout=layout,
            layer_layouts=layer_layouts,
            max_slots=max_slots,
            pool_of_layer=pool_of_layer(pools, config.model_config.num_moe_layers),
            pool_sizes=pool_sizes,
        )
        # before set_bank_sources: the residency validation and the copy plan's skip of non-pinned layers key on the CPU-layer set
        cache.cpu_layer_ids = cpu_layer_ids
        if locked_fetch:
            cache.locked_hybrid = cpu_layer_ids
        cache.set_bank_sources(banks.sources, layer_residency=banks.layer_residency, layer_tags=layer_tags, row_tail_bytes=banks.row_tail_bytes)
        cache.set_alphas(banks.gate_up_alpha, banks.down_alpha)
        # Must be set before CUDA graph capture so the (device-side) accumulation ops are
        # captured and re-run on every decode replay.
        cache.collect_stats = config.moe_collect_stats
        layers = attach_offload_moe_cache(self.model, cache)
        assert len(layers) == config.model_config.num_moe_layers
        if cache.decode_target in ("cpu", "hybrid"):
            self._init_cpu_moe_executor(config, cache, layers)
        if decode_target == "hybrid":
            self._resolve_hybrid_fetch(config, cache)
        if locked_fetch:
            attach_locked_fetch(cache, config, staging, self._profile_layer_rows)
        self.moe_offload_cache = cache
        return cache

    def _residency_split(self, config: EngineConfig, cpu_layer_ids, method, *, capped: bool):
        """(decode_target, split_residency) for a CPU-layer set; turns prefill overlap off when some layers stay unpinned."""
        # the kernels were picked for model_config.decode_target; --moe-cpu-layers auto may still find that every bank fits the pin budget
        decode_target = config.model_config.decode_target
        if decode_target == "cpu" and not cpu_layer_ids:
            decode_target = "gpu"
        # split residency: where pinning is quota-capped (_pin_budget_bytes), pin only the GPU layers' banks and mlock the CPU layers'
        # uncapped hosts keep every bank pinned (CPU decode reads them the same; overlap prefill stays on)
        # not applied to plain --moe-strategy cpu; all-locked under a cap = --moe-strategy offload --moe-cpu-layers 1.0
        split_residency = (
            bool(cpu_layer_ids)
            and config.moe_strategy in ("offload", "hybrid")
            and capped
        )
        if config.moe_strategy == "cpu" and not split_residency:
            # cpu mode pins every bank for the prefill double buffer; over the pin cap that dies in cudaHostRegister, so lock everything instead
            from freetoken.moe.expert_banks import bank_bytes_estimate, ftw_bank_bytes

            budget = _pin_budget_bytes(self._host_tables_bytes)
            bank_bytes = None
            if budget is not None:
                bank_bytes = ftw_bank_bytes(config.model_path) or bank_bytes_estimate(config.model_config, method)
            if bank_bytes and bank_bytes > budget:
                split_residency = True
                logger.info_rank0(
                    f"--moe-strategy cpu: banks {bank_bytes / 2**30:.2f} GiB exceed the "
                    f"pin budget; OS-locking all layers instead of pinning"
                )
        if split_residency and config.moe_prefill_overlap:
            # locked (unregistered) layers cannot feed the async pinned H2D double buffer; their prefill is a synchronous pageable copy via materialize
            logger.info_rank0(
                "--moe-cpu-layers split residency: disabling MoE prefill overlap "
                "(locked layers prefill via synchronous pageable copies)"
            )
            object.__setattr__(config, "moe_prefill_overlap", False)
        return decode_target, split_residency

    def _moe_pool_bytes(self, pool_sizes: list[int]) -> int:
        return sum(s * self._moe_layer_slot_bytes[g[0]] for g, s in zip(self._moe_pool_groups, pool_sizes))

    def _moe_pools(self, batch: int) -> list:
        """The slot pools, each with room for the routed experts of ``batch`` decode tokens."""
        model_config = self.config.model_config
        num_experts = model_config.num_experts
        min_num_slots = min(num_experts, model_config.num_experts_per_tok * batch)
        return make_pools(self._moe_pool_groups, num_experts, min_num_slots, self._moe_slot_limit)

    def _resolve_hybrid_fetch(self, config: EngineConfig, cache) -> None:
        """Resolve --moe-hybrid-max-fetch -1 (auto) into a bandwidth-matched fetch fraction.

        Perfect fetch/compute overlap wants fetched : cpu-computed misses = pcie_bw :
        (cpu_bw - pcie_bw), i.e. fetching a pcie_bw / cpu_bw fraction of each decode
        step's misses -- both sides then finish together instead of one idling. The
        achieved bandwidths come from the cached `ft bench bw` profile, from the row of each
        MoE layer's kind (format, options, expert shape, top_k, CPU worker threads); a layer
        without a row keeps the fixed cap of 1.
        """
        if config.moe_hybrid_max_fetch >= 0:
            return  # explicit fixed cap
        from freetoken.moe.bench_profile import fetch_fraction

        cache.hybrid_max_fetch = 1
        layer_rows = self._profile_layer_rows()
        if layer_rows is None:
            logger.warning_rank0("--moe-hybrid-max-fetch auto: the MoE layers bind no expert method to look up in the `ft bench bw` profile; using a fixed fetch cap of 1")
            return
        executor = self.cpu_moe_executor
        keys, labels, rows = layer_rows
        fractions = [fetch_fraction(rows.get(key)) for key in keys]
        cache.hybrid_fetch_fraction = [f or 0.0 for f in fractions]
        # locked layers fetch the fetch_plan share instead; their own log line lists it
        groups = collections.Counter((labels[l], fractions[l]) for l in range(len(keys)) if l not in cache.locked_hybrid)
        fetched = [f"{f:.1%} on {n} {label} layers" for (label, f), n in groups.items() if f is not None]
        if fetched:
            logger.info_rank0(
                "--moe-hybrid-max-fetch auto: each decode step fetches this share of a layer's expert misses over PCIe "
                f"(benched PCIe/CPU bandwidth ratio) and computes the rest on the CPU: {'; '.join(fetched)}"
            )
        if missing := sorted({label for label, f in groups if f is None}):
            capped = sum(n for (_, f), n in groups.items() if f is None)
            logger.warning_rank0(
                f"--moe-hybrid-max-fetch auto: no `ft bench bw` row for {capped} MoE layers ({', '.join(missing)}) at "
                f"{executor.num_threads} CPU threads; using a fixed fetch cap of 1 there. `ft bench bw --model <checkpoint>` benches each layer kind"
            )
        benched_isas = {rows[key].get("cpu_moe_isa") for key, f in zip(keys, fractions) if f is not None} - {None}
        if other := sorted(benched_isas - set(executor.isa.split(","))):
            logger.warning_rank0(
                f"--moe-hybrid-max-fetch auto: some `ft bench bw` rows were benched on CPU kernels {other}, but this executor runs "
                f"{executor.isa}; using them anyway"
            )

    def _profile_layer_rows(self):
        """(row key, label) of each MoE layer's kind in the `ft bench bw` profile, and the profile's rows; None without expert methods."""
        from freetoken.moe.bench_profile import layer_kind_key, load_layer_kinds
        from freetoken.moe.cpu_executor import _format_label

        methods = offload_layer_methods(self.model)
        if methods is None:
            return None
        gpu_name, gpu_uuid = _profile_gpu(self.device.index)
        rows = load_layer_kinds(gpu_name=gpu_name, gpu_uuid=gpu_uuid) or {}
        keys, labels = [], []
        for m in methods:
            fmt, opts = m.cpu_spec()
            c = m.cfg
            keys.append(layer_kind_key(fmt, opts, c.hidden, c.local_intermediate, c.top_k, self.cpu_moe_executor.num_threads))
            labels.append(f"{_format_label(fmt, tuple(sorted(opts.items())))} {c.hidden}x{c.local_intermediate}")
        return keys, labels, rows

    def _init_cpu_moe_executor(self, config: EngineConfig, cache, layers) -> None:
        """Build the persistent CPU MoE executor (decode-time expert compute).

        Must run before CUDA graph capture: the worker pool has to be live for the
        eager warmup forward, and the pinned IO block must be stable for the captured
        nodes.
        """
        from freetoken.moe.cpu_executor import CpuMoeExecutor

        sample = layers[0]
        required = ("top_k", "activation", "apply_router_weight_on_input")
        if not all(hasattr(sample, attr) for attr in required):
            raise NotImplementedError(
                "CPU MoE backend is not yet supported for this model architecture "
                f"(MoE layer {type(sample).__name__} is missing {required})."
            )
        # Decode batches never exceed max_running_req, but CUDA-graph padding can
        # round a batch up to the largest captured size; cover both.
        max_tokens = max(config.max_running_req, config.cuda_graph_max_bs or 0, *(config.cuda_graph_bs or []), 1)
        max_tokens *= config.decode_rows_per_req
        by_id = sorted(layers, key=lambda layer: layer.layer_id)
        layer_formats = [layer.quant_method.cpu_spec() for layer in by_id]
        executor = CpuMoeExecutor(
            cache,
            top_k=sample.top_k,
            activation=sample.activation,
            apply_router_weight_on_input=sample.apply_router_weight_on_input,
            num_threads=config.moe_cpu_threads,
            max_tokens=max_tokens,
            device=self.device,
            swiglu_alpha=float(sample.alpha),
            swiglu_limit=sample.limit,
            layer_formats=layer_formats,
        )
        cache.set_cpu_executor(executor)
        self.cpu_moe_executor = executor

    def _sync_get_memory(self) -> Tuple[int, int]:
        """Get the min and max free memory across TP ranks."""
        torch.cuda.synchronize(self.device)
        torch.cuda.empty_cache()
        torch.cuda.reset_peak_memory_stats(self.device)
        free_memory = get_free_memory(self.device)
        free_mem_tensor = torch.tensor([free_memory, -free_memory], device="cpu", dtype=torch.int64)
        torch.distributed.all_reduce(
            free_mem_tensor, op=torch.distributed.ReduceOp.MIN, group=self.tp_cpu_group
        )
        min_free_memory = int(free_mem_tensor[0].item())
        max_free_memory = -int(free_mem_tensor[1].item())
        if max_free_memory - min_free_memory > 2 * 1024 * 1024 * 1024:
            logger.error(
                f"Memory across TP ranks are imbalanced:"
                f" min {mem_GB(min_free_memory)}, max {mem_GB(max_free_memory)}"
            )
            raise RuntimeError("Memory across TP ranks are imbalanced")

        return min_free_memory, max_free_memory

    def _target_moe_and_expert_bytes(self, moe_cache_size: int | None, pool_sizes: list[int] | None) -> tuple[int, int]:
        cache = self.moe_offload_cache
        if cache is None:
            return moe_cache_size or 0, 0
        if moe_cache_size is None:
            target_moe = cache.cache_size
            nbytes = sum(c.numel() * c.element_size() for _, c in cache.banks)
        else:
            target_moe = moe_cache_size
            nbytes = self._moe_pool_bytes(pool_sizes)
        # pools of several bank layouts have different slot sizes; round up so target * bytes covers them
        return target_moe, -(-nbytes // target_moe)

    def _resize_kv_pool(self, config, num_pages: int, num_swa_pages: int | None) -> None:
        # IN-PLACE, identity-preserving: the CacheManager's swa_pool reference, ctx.kv_cache and
        # the model's per-access pool property all keep pointing at THIS pool, which frees its old
        # buffers before allocating the new ones. mark_for_rebind re-binds per-bind scratch on the
        # next forward (graph re-capture); the prefix tree + page bookkeeping reset is the
        # scheduler's generic cache_manager.rebuild.
        if self.kv_cache.needs_rebind_on_rebuild:
            self.model.mark_for_rebind()
        self.kv_cache.rebuild_from_config(config, num_pages, num_swa_pages=num_swa_pages)
        self.num_pages = num_pages

    def _refresh_seq_state(self, config) -> None:
        num_tokens = self.num_pages * config.page_size
        self.max_seq_len, aligned_max_seq_len = _seq_limits(config, num_tokens)
        if aligned_max_seq_len != self.page_table.shape[1]:
            # max_seq_len changed (e.g. KV grew past the startup token budget); the page table
            # columns must track it or new requests would index out of bounds. The scheduler
            # re-points its managers to engine.page_table on a num_pages rebuild.
            self.ctx.page_table = self.page_table = torch.zeros(
                (config.max_running_req + 1, aligned_max_seq_len),
                dtype=torch.int32,
                device=self.device,
            )
        self.page_table[self.dummy_req.table_idx].fill_(num_tokens)
        self.kv_cache.attach_page_table(self.page_table)

    @torch.inference_mode()
    def rebuild_runtime_cache(
        self,
        *,
        moe_cache_size: int | None = None,
        num_pages: int | None = None,
        num_mamba_slots: int | None = None,
        num_swa_pages: int | None = None,
    ) -> None:
        """Idle-only in-place resize of the MoE slot cache, KV page pool, GDN (mamba) state pool,
        and/or the window pool (num_swa_pages: an absolute pinned window), followed by CUDA-graph
        re-capture. Does NOT reload weights or host expert banks. The caller (scheduler) must
        guarantee no in-flight prefill/decode.
        """
        config = self.config
        if (moe_cache_size is None and num_pages is None and num_mamba_slots is None
                and num_swa_pages is None):
            return

        # 0a. Geometry prevalidation BEFORE any destructive free. An invalid target (moe
        #     slots on a model with no offload cache, moe below num_experts / above the
        #     marlin cap, non-positive pages, or too few GDN slots to run) must reject
        #     recoverably with the old cache intact -- NOT after teardown, which would
        #     leave the server unable to serve. These checks are model-agnostic.
        if moe_cache_size is not None:
            if self.moe_offload_cache is None:
                raise CacheRebuildRejected(
                    "moe_cache_size requested but this model has no MoE offload cache"
                )
            try:
                # a rebuild does not lower max_running_req again, so with GPU decode every pool must hold a full batch; hybrid caps its fetches instead
                batch = config.max_running_req * config.decode_rows_per_req if self.moe_offload_cache.decode_target == "gpu" else 1
                moe_pool_sizes = split_slots(moe_cache_size, self._moe_pools(batch), self._moe_expert_counts)
                self.moe_offload_cache.validate_rebuild(moe_cache_size, moe_pool_sizes)
            except ValueError as e:
                raise CacheRebuildRejected(str(e)) from e
        if num_pages is not None and num_pages <= 0:
            raise CacheRebuildRejected(f"num_pages must be positive, got {num_pages}")
        if num_mamba_slots is not None:
            if self.linear_state_pool is None:
                raise CacheRebuildRejected(
                    "num_mamba_slots requested but this model has no GDN state pool"
                )
            # num_mamba_slots is the USABLE slot count (what the user sets and the status bar
            # shows); the pool also reserves a padding sink (slot 0), so the physical pool is
            # num_mamba_slots + 1. _linear_pool_min_slots is the physical floor -> usable - 1.
            min_usable = _linear_pool_min_slots(config) - 1
            if num_mamba_slots < min_usable:
                raise CacheRebuildRejected(
                    f"num_mamba_slots {num_mamba_slots} is below the minimum {min_usable} "
                    f"(non-evictable working set for max_running_req={config.max_running_req}) "
                    f"needed to run; admission would deadlock"
                )
        if num_swa_pages is not None:
            # An absolute window pin for the radix-SWA window pool (Gemma) or the DSV4 window tier;
            # meaningless for dense/MHA models and the naive SWA path (concurrency x window).
            if not _supports_swa_ratio(config):
                raise CacheRebuildRejected(
                    "num_swa_pages requested but this model has no window pool "
                    "(needs DSV4 or a sliding-window model with --cache-type radix)"
                )
            if num_swa_pages <= 0:
                raise CacheRebuildRejected(
                    f"num_swa_pages must be positive, got {num_swa_pages}"
                )

        # 0b. Pool-family budget fit-check BEFORE any destructive free: an unfit geometry
        #     must reject (recoverable) so the old caches stay intact and serving continues,
        #     rather than freeing and then OOMing into permanent failure. The engine supplies
        #     the memory account; the pool answers whether its target geometry fits.
        target_moe, per_expert_bytes = self._target_moe_and_expert_bytes(
            moe_cache_size, moe_pool_sizes if moe_cache_size is not None else None
        )
        # Price the sibling GDN state pool at ITS target (physical slots = usable + padding
        # sink) and hand the bytes in -- the KV pool only budgets its own tiers.
        target_mamba = (
            num_mamba_slots + 1
            if num_mamba_slots is not None
            else (self.linear_state_pool.num_slots if self.linear_state_pool is not None else None)
        )
        self.kv_cache.validate_rebuild(
            config, num_pages=num_pages,
            num_swa_pages=num_swa_pages, target_moe=target_moe,
            per_expert_bytes=per_expert_bytes, baseline_free=self._baseline_free,
            weights_bytes=self._weights_bytes, current_num_pages=self.num_pages,
            extra_fixed_bytes=(
                state_pool_bytes(config, target_mamba) if target_mamba is not None else 0
            ),
            extra_note=(
                f", mamba={target_mamba - 1} slots" if target_mamba is not None else ""
            ),
        )

        torch.cuda.synchronize(self.device)
        # Preserve the CUDA-graph batch-size set resolved at startup. The auto heuristic keys
        # off free memory, which is far smaller now that the caches are resident (post-cache
        # free << startup pre-load free), so re-deriving it here would silently drop large
        # batch sizes after the first rebuild. Reusing the already-resolved list keeps the
        # captured coverage identical (the fit-check above guarantees the graph headroom fits).
        prior_graph_bs = self.graph_runner.graph_bs_list
        # Point of no return for the scheduler's rollback logic: from here the live graphs and
        # pools start being freed. A failure BEFORE this flag flips leaves the engine serving
        # untouched (no rollback needed); after it, only a rebuild restores service.
        self.rebuild_teardown_started = True
        # 1. Tear down CUDA graphs + backend capture scratch (free-before-alloc).
        self.attn_backend.reset_capture()
        if self.linear_backend is not None:
            self.linear_backend.reset_capture()
        self.graph_runner.destroy_cuda_graphs()
        # 2. Resize caches in place (each frees its old GPU tensors before allocating).
        # Pin the new window first (validated above) so any KV-pool rebuild below sizes the window
        # to it (_dsv4_pool_sizes / _swa_paged_num_tokens read config.swa_num_pages_override).
        # frozen EngineConfig — mutate in place like the moe_cache_size path; `config.x = y` raises
        # FrozenInstanceError, which here aborts the rebuild after the CUDA graphs are gone (→ 503).
        if num_swa_pages is not None:
            object.__setattr__(config, "swa_num_pages_override", num_swa_pages)
        if moe_cache_size is not None:
            assert self.moe_offload_cache is not None, "no MoE offload cache to resize"
            self.moe_offload_cache.rebuild(moe_cache_size, moe_pool_sizes)
        if num_pages is not None:
            # sets self.num_pages (rebuilds KV + window)
            self._resize_kv_pool(config, num_pages, num_swa_pages)
        elif num_swa_pages is not None:
            # Window-only change: no page-count change, but re-derive the window pool at the new
            # pin against the CURRENT page count. This re-allocs the same-size full pool and
            # the resized window, both inside the pool's own rebuild_from_config.
            self._resize_kv_pool(config, self.num_pages, num_swa_pages)
        if num_mamba_slots is not None:
            # Reallocate the GDN state pool (frees old tensors first). Must sit between graph
            # teardown and re-capture so the recaptured graphs bind the new state tensors.
            # +1 for the reserved padding sink: num_mamba_slots is the usable count.
            self.linear_state_pool.rebuild(num_mamba_slots + 1)
        # 3. Refresh max_seq_len (+ generic page table) for the new token budget.
        self._refresh_seq_state(config)
        aligned_max_seq_len = self.page_table.shape[1]
        # 4. Re-capture CUDA graphs against the new tensors (reset_capture above re-armed
        #    the backend; _sync_get_memory empties the cache so freed memory is reclaimed).
        gc.collect()
        free_min = self._sync_get_memory()[0]
        self.graph_runner = GraphRunner(
            stream=self.stream,
            device=self.device,
            forward=self.model.forward,
            attn_backend=self.attn_backend,
            cuda_graph_bs=prior_graph_bs,  # reuse the startup-resolved set (see above)
            cuda_graph_max_bs=config.cuda_graph_max_bs,
            free_memory=free_min,
            max_seq_len=aligned_max_seq_len,
            dummy_req=self.dummy_req,
            moe_offload_cache=self.moe_offload_cache,
            mrope=config.model_config.model_is_mrope,
            out_dtype=torch.float32,
            linear_backend=self.linear_backend,
        )

    def forward_logits(self, batch: Batch) -> torch.Tensor:
        """The batch's logits from its captured graph, or eagerly when none fits."""
        use_graph = self.graph_runner.can_use_cuda_graph(batch)
        with self.ctx.forward_batch(batch), self.model.forward_host_ctx(batch, use_graph):
            logits = self.graph_runner.replay(batch) if use_graph else self.model.forward()
        if self.cpu_moe_executor is not None:
            # One pinned read: surfaces a fired flag-handshake watchdog (dead coordinator
            # -> stale expert outputs) as a loud error instead of silent corruption.
            self.cpu_moe_executor.raise_if_unhealthy()
        return logits

    def forward_batch(self, batch: Batch, args: BatchSamplingArgs) -> ForwardOutput:
        assert torch.cuda.current_stream() == self.stream
        if batch.mm_gather_plan:
            self._run_mm_encoder(batch)
        logits = self.forward_logits(batch)

        for req in batch.reqs:
            req.complete_one()

        batch_logits = logits[: batch.size]
        next_tokens_gpu = self.sampler.sample(batch_logits, args).to(torch.int32)
        next_tokens_cpu = next_tokens_gpu.to("cpu", non_blocking=True)
        copy_done_event = torch.cuda.Event()
        copy_done_event.record(self.stream)
        return ForwardOutput(next_tokens_gpu, next_tokens_cpu, copy_done_event)

    @torch.inference_mode()
    def _warmup_prefill(self) -> None:
        """Compile the Triton prefill path before the first real request.

        Decode CUDA graph capture warms the decode path, but the first prefill
        can still pay Triton/cublas setup costs. Use the dummy request row and
        restore it afterwards so padded decode graph replay keeps using the
        dedicated dummy KV slot.
        """
        if self.max_seq_len < 2:
            return

        warmup_lens = [min(80, self.max_seq_len)]
        if self.max_seq_len >= 128:
            warmup_lens.append(128)
        warmup_lens = sorted({length for length in warmup_lens if length >= 2})
        if not warmup_lens:
            return

        dummy_row = self.page_table[self.dummy_req.table_idx]
        dummy_slot = int(dummy_row[0].item())
        started = torch.cuda.Event(enable_timing=True)
        ended = torch.cuda.Event(enable_timing=True)
        started.record(self.stream)
        try:
            for length in warmup_lens:
                dummy_row[:length] = torch.arange(
                    length, dtype=torch.int32, device=self.device
                )
                warm_req = Req(
                    input_ids=torch.zeros(length, dtype=torch.int32, device="cpu"),
                    table_idx=self.dummy_req.table_idx,
                    cached_len=0,
                    output_len=1,
                    uid=-1,
                    sampling_params=None,  # type: ignore[arg-type]
                    cache_handle=None,  # type: ignore[arg-type]
                )
                batch = Batch(reqs=[warm_req], phase="prefill")
                batch.padded_reqs = batch.reqs
                batch.input_ids = torch.zeros(length, dtype=torch.int32, device=self.device)
                batch.positions = torch.arange(length, dtype=torch.int32, device=self.device)
                if self.config.model_config.model_is_mrope:
                    batch.mrope_positions = (
                        batch.positions.unsqueeze(0).expand(3, -1).contiguous()
                    )
                batch.out_loc = dummy_row[:length]
                self.attn_backend.prepare_metadata(batch)
                with self.ctx.forward_batch(batch):
                    self.model.forward()
        finally:
            dummy_row.fill_(dummy_slot)
            if self.moe_offload_cache is not None:
                self.moe_offload_cache.reset()
        ended.record(self.stream)
        torch.cuda.synchronize(self.device)
        logger.info_rank0(
            f"Prefill warmup complete for lengths {warmup_lens} "
            f"in {started.elapsed_time(ended) / 1000.0:.3f} s"
        )

    def shutdown(self) -> None:
        self.graph_runner.destroy_cuda_graphs()
        torch.distributed.destroy_process_group()
        destroy_distributed()


def _profile_gpu(index: "int | None" = None) -> Tuple[str | None, str | None]:
    """(name, uuid) of visible device ``index`` (default: the current, i.e. bound, device); (None, None) without CUDA."""
    if not torch.cuda.is_available():
        return None, None
    ident = gpu_identity(torch.cuda.current_device() if index is None else index)
    return ident["name"], ident["uuid"]


def _is_unified_memory_gpu(index: "int | None" = None) -> bool:
    """True when the GPU has no separate device memory (cudaDevAttrIntegrated): host
    banks and the GPU slot cache are the same DRAM, so the offload family's pinned
    staging + slot gather are DRAM-to-DRAM copies with no PCIe link to hide behind.

    The attribute is reliable on true-UMA parts (Jetson, GB10/DGX Spark) but not on
    C2C-linked discrete-HBM parts (GH200 reports integrated=0 despite coherent CPU
    memory), and it has only been verified on GB10 so far --
    FREETOKEN_UNIFIED_MEMORY=0/1 overrides the probe where the attribute lies."""
    env = os.environ.get("FREETOKEN_UNIFIED_MEMORY")
    if env is not None:
        return env.strip().lower() not in {"0", "false", "no", "off"}
    if not torch.cuda.is_available():
        return False
    try:
        dev = torch.cuda.current_device() if index is None else index
        return bool(torch.cuda.get_device_properties(dev).is_integrated)
    except Exception:
        return False


_RESIDENT_EXPERT_QUANTS = ("none", "fp8_block", "nvfp4", "gguf")


def _fused_resident_ok(model_config) -> bool:
    """Whether the resident ('fused') MoE path can hold this model's experts; DeepSeek-V4 experts stay offload."""
    if getattr(model_config, "expert_quant", "none") not in _RESIDENT_EXPERT_QUANTS:
        return False
    return getattr(model_config, "moe_weight_format", None) in (None, "bf16", "mxfp4")


def _ensure_expandable_segments() -> None:
    """Default the CUDA allocator to expandable segments.

    The motivating case is the offload prefill, which repeatedly dequantizes
    variable-sized NVFP4 expert blocks to BF16 (a different size per layer as the
    active-expert count varies). Under that alloc/free churn the default caching
    allocator fragments badly -- reserved memory can balloon far past the actual peak
    allocation (observed ~78GiB reserved for a <30GiB working set).
    ``expandable_segments`` lets freed regions of any size be reused, keeping
    reserved ~= allocated, so it is applied to every run, not just offload ones.

    Env vars are parsed once at import and ignored if set afterwards, so we apply the
    setting via the runtime API instead. Must run before the first CUDA allocation (the
    caller guarantees CUDA is not yet initialized). Any user-provided allocator config
    is respected and left untouched.
    """
    if os.environ.get("PYTORCH_ALLOC_CONF") or os.environ.get("PYTORCH_CUDA_ALLOC_CONF"):
        return
    try:
        torch.cuda.memory._set_allocator_settings("expandable_segments:True")
    except Exception as exc:  # pragma: no cover - depends on torch build
        logger.info_rank0(f"Could not enable expandable_segments ({exc}); continuing")
        return
    logger.info_rank0("Enabled expandable_segments (override via PYTORCH_ALLOC_CONF)")


def _resolve_cache_type(has_linear_attention: bool, requested: str) -> str:
    # Hybrid GDN models default to the HybridRadixCache (snapshots GDN state at chunk
    # boundaries -> cross-request prefix reuse). An explicit ``--cache-type naive`` opts out
    # to the old no-reuse path (debugging / parity baseline / lower GDN-state memory).
    if has_linear_attention:
        return "naive" if requested == "naive" else "hybrid_radix"
    return requested


def _adjust_dsv41_config(config: EngineConfig, override) -> None:
    """DeepSeek-V4.1 config reconciliation, the DSV4 policy with the replay knob: sync the
    runtime into ``dsv41_args``, page_size = the window page P, radix -> swa_radix, decode graph
    batches <= max_running_req. Unlike DSV4 the prefill chunk keeps ``max_extend_tokens`` (whole
    window pages): at a 1M ceiling the window pool alone would admit ~40K-token chunks whose
    activations (64 x 512 latent queries per token) do not fit next to the expert cache."""
    model_config = config.model_config
    args = model_config.dsv41_args
    args.max_seq_len = config.max_seq_len
    args.max_batch_size = config.max_running_req + 1  # +1 dummy
    args.swa_decoder_replay = config.swa_decoder_replay
    P = args.window_size
    override("page_size", P)
    logger.info_rank0(
        f"DSV41 KV pages are {P}-token window pages; page_size set to {P}; SWA decoder replay: {args.swa_decoder_replay}"
        + ("" if args.swa_decoder_replay == "exact" else f" (prefix hits stop {P} tokens before the prompt end)")
    )
    if getattr(config, "cache_type", "radix") != "naive":
        override("cache_type", "swa_radix")
    current = getattr(config, "max_extend_tokens", None)  # a SchedulerConfig field
    if current is not None and (extend := max(P, current // P * P)) != current:
        override("max_extend_tokens", extend)
    mr = config.max_running_req
    if config.cuda_graph_max_bs is not None and config.cuda_graph_max_bs > mr:
        logger.warning_rank0(f"cuda_graph_max_bs {config.cuda_graph_max_bs} exceeds max_running_req {mr}; clamping (larger decode batches never occur).")
        override("cuda_graph_max_bs", mr)
    if config.cuda_graph_bs is not None:
        kept = [bs for bs in config.cuda_graph_bs if bs <= mr]
        if kept != list(config.cuda_graph_bs):
            logger.warning_rank0(f"dropping cuda_graph_bs entries above max_running_req {mr}: {[bs for bs in config.cuda_graph_bs if bs > mr]}")
            override("cuda_graph_bs", kept)


def _adjust_dsv4_config(config: EngineConfig, override) -> None:
    """DSV4 engine-config reconciliation at config-resolution time (before the pool exists).
    Syncs the resolved runtime config into the opaque ``dsv4_args`` payload, sets
    page_size to the window page P, forces single-chunk prefill, and clamps cuda_graph_bs/max_bs to
    the DSV4 decode batch size.
    """
    model_config = config.model_config
    model_config.dsv4_args.max_seq_len = config.max_seq_len
    model_config.dsv4_args.max_batch_size = config.max_running_req + 1  # +1 dummy
    # config.swa_full_tokens_ratio is the DSV4 window/full ratio directly (default sizing);
    # a runtime rebuild pins an absolute window via swa_num_pages_override instead.
    # DSV4's KV page IS the P-token window page (window == radix reuse granularity == lcm of
    # the compress ratios), so max_num_tokens = num_pages * page_size holds like every model.
    P = model_config.dsv4_args.window_size
    override("page_size", P)
    logger.info_rank0(f"DSV4 KV pages are {P}-token window pages; page_size set to {P}")
    # The generic CacheManager materializes DSV4 'radix' as the shared SWARadixCache (is_swa);
    # 'naive' stays naive with the pool's swa currency riding swa_paged.
    if getattr(config, "cache_type", "radix") != "naive":
        override("cache_type", "swa_radix")
    # 'radix' (SWARadixCache on the full-loc currency, carry-aware re-prefill) is the default and is
    # honored, as is an explicit 'naive'. Don't let max_extend_tokens force a second chunk within
    # one prompt (the pool's prefill_chunk_budget still chunks prompts larger than the window
    # pool); prefill batches ragged (bs>=1), each segment resuming from its own cached_len.
    if getattr(config, "max_extend_tokens", 0) < config.max_seq_len:
        override("max_extend_tokens", config.max_seq_len)

    # DSV4 decode batches at most max_running_req rows; its full-loc snapshot is sized to that,
    # so a graph bs above it would exceed the backend's captured snapshot rows. Clamp any
    # oversized explicit list / max_bs here (before GraphRunner ever sees it).
    mr = config.max_running_req
    if config.cuda_graph_max_bs is not None and config.cuda_graph_max_bs > mr:
        logger.warning_rank0(
            f"cuda_graph_max_bs {config.cuda_graph_max_bs} exceeds DSV4 max_running_req {mr}; "
            "clamping to max_running_req (larger decode batches never occur)."
        )
        override("cuda_graph_max_bs", mr)
    if config.cuda_graph_bs is not None:
        kept = [bs for bs in config.cuda_graph_bs if bs <= mr]
        if kept != list(config.cuda_graph_bs):
            dropped = [bs for bs in config.cuda_graph_bs if bs > mr]
            logger.warning_rank0(
                f"dropping cuda_graph_bs entries {dropped} above DSV4 max_running_req {mr} "
                "(larger decode batches never occur)."
            )
            override("cuda_graph_bs", kept)


def _parse_cpu_layers_spec(spec: str, num_moe_layers: int) -> frozenset[int]:
    """Parse ``--moe-cpu-layers``: an explicit MoE-layer id list (``"3,7,11"``), a count
    (``"8"`` -> 8 layers evenly strided across depth), or a fraction (``"0.5"``). Ids are
    indices into the MoE layers, ``[0, num_moe_layers)``."""
    s = spec.strip()
    if not s:
        return frozenset()
    if "," in s:
        ids = {int(x) for x in s.split(",") if x.strip()}
        for i in ids:
            if not 0 <= i < num_moe_layers:
                raise ValueError(
                    f"--moe-cpu-layers id {i} out of range [0, {num_moe_layers})"
                )
        return frozenset(ids)
    if "." in s:
        frac = float(s)
        if not 0.0 <= frac <= 1.0:
            raise ValueError(f"--moe-cpu-layers fraction {frac} must be in [0, 1]")
        k = round(frac * num_moe_layers)
    else:
        k = int(s)
        if not 0 <= k <= num_moe_layers:
            raise ValueError(f"--moe-cpu-layers count {k} must be in [0, {num_moe_layers}]")
    # k layers spread evenly across depth (frozenset dedups any rounding collisions;
    # k == 0 yields an empty range, hence an empty set).
    return frozenset(round(i * num_moe_layers / k) for i in range(k))


def _resolve_cpu_layers(config: EngineConfig, num_moe_layers: int, *, reserved: int = 0, method=None) -> frozenset[int]:
    """MoE layer ids whose decode runs on the CPU executor.

    ``--moe-strategy cpu`` -> every layer. ``--moe-strategy offload`` + ``--moe-cpu-layers``
    -> the parsed subset, or the pin-budget pick for ``auto`` (the rest stay on the GPU offload/PCIe path). Otherwise none.
    """
    if config.moe_strategy == "cpu":
        return frozenset(range(num_moe_layers))
    spec = config.moe_cpu_layers
    if not spec or not is_offload_moe_strategy(config.moe_strategy):
        return frozenset()
    if spec.strip() == "auto":
        return _auto_cpu_layers(config, num_moe_layers, reserved=reserved, method=method)
    return _parse_cpu_layers_spec(spec, num_moe_layers)


def _decode_target(config: EngineConfig) -> str:
    """Where routed experts decode, from the flags alone: hybrid co-compute, the CPU executor for some or all layers, or the GPU slot cache."""
    if config.moe_strategy == "hybrid":
        return "hybrid"
    if config.moe_strategy == "cpu" or (config.moe_cpu_layers and is_offload_moe_strategy(config.moe_strategy)):
        return "cpu"
    return "gpu"


# expert activations the CPU MoE executor supports (csrc ActKind)
_CPU_MOE_ACTS = (
    "silu", "swish", "gelu", "gelu_tanh", "gelu_pytorch_tanh", "swigluoai",
    "swiglu_clamp",
)


def _cpu_moe_executor_viable(model_config) -> bool:
    """Whether an automatic CPU-decode decision may target the CPU MoE executor.

    A default boot must degrade to GPU offload instead of crashing in CpuMoeExecutor after the whole load; explicit cpu/hybrid/--moe-cpu-layers picks still fail loudly."""
    from freetoken.moe.cpu_executor import compiled_extension_supports

    try:
        from freetoken.kernel import _cpu_moe
    except ImportError:
        return False
    act = getattr(model_config, "hidden_act", "silu")
    moe_wfmt = getattr(model_config, "moe_weight_format", None)
    if act not in _CPU_MOE_ACTS and moe_wfmt != "mxfp4":
        return False
    if moe_wfmt != "mxfp4" and not compiled_extension_supports(act):
        return False
    expert_quant = getattr(model_config, "expert_quant", "none")
    fmt = expert_quant if expert_quant != "none" else (moe_wfmt or "bf16")
    if fmt == "gguf":
        from freetoken.kernel import ggml_cpu

        return fmt in _cpu_moe.weight_formats() and ggml_cpu.select() is not None
    return fmt == "mxfp4" or fmt in _cpu_moe.weight_formats()


def _pin_budget_bytes(reserved: int = 0) -> int | None:
    """Bytes this process can still safely cudaHostRegister, or None when the platform does not cap pinning (plain Linux).

    WSL's WDDM-backed CUDA caps pinning near half of RAM, shared across processes -- budget 40%. FREETOKEN_PIN_BUDGET_GB overrides anywhere. ``reserved`` subtracts host bytes already pinned outside the expert banks (qwen4_exp's PLE table)."""
    if env := os.environ.get("FREETOKEN_PIN_BUDGET_GB"):
        cap = int(float(env) * 2**30)
    elif not hasattr(os, "uname") or "microsoft" not in os.uname().release.lower():  # WSL kernel tag
        return None
    else:
        cap = int(os.sysconf("SC_PHYS_PAGES") * os.sysconf("SC_PAGE_SIZE") * 0.4)
    return max(0, cap - reserved)


def _bank_bytes(config: EngineConfig, method=None, layer_methods=None) -> int | None:
    from freetoken.moe.expert_banks import bank_bytes_estimate, ftw_bank_bytes

    return ftw_bank_bytes(config.model_path) or bank_bytes_estimate(config.model_config, method, layer_methods)


def _pin_hint(reserved: int) -> str:
    if _pin_budget_bytes(reserved) is None:
        return "the expert banks need more page-locked host RAM than this host has; free host RAM or serve a smaller model"
    return (
        "pass --moe-cpu-layers auto to lock the layers over the pin budget for CPU decode, "
        "or --moe-cpu-layers <count|fraction|ids> to choose them yourself"
    )


def _check_pin_budget(config: EngineConfig, *, reserved: int, method=None, layer_methods=None) -> None:
    """Stop a plain offload boot whose banks exceed a known pin budget before any bank is read."""
    if config.moe_cpu_layers or config.moe_strategy not in ("offload", "hybrid"):
        return
    budget = _pin_budget_bytes(reserved)
    bank_bytes = _bank_bytes(config, method, layer_methods) if budget is not None else None
    if bank_bytes and bank_bytes > budget:
        raise ValueError(
            f"expert banks need {bank_bytes / 2**30:.1f} GiB of pinned host RAM but the pin budget is "
            f"{budget / 2**30:.1f} GiB (WSL caps CUDA pinning; FREETOKEN_PIN_BUDGET_GB overrides); {_pin_hint(reserved)}"
        )


def _auto_cpu_layers(config: EngineConfig, num_moe_layers: int, *, reserved: int = 0, method=None) -> frozenset[int]:
    """Pick CPU (locked) MoE layers for ``--moe-cpu-layers auto``: none while the banks fit the pin budget.

    Locks just enough head+tail layers: per-layer decode miss rates are U-shaped, so the ends are the cheapest to move off the slot cache."""
    bank_bytes = _bank_bytes(config, method)
    if not bank_bytes:
        return frozenset()
    budget = _pin_budget_bytes(reserved)
    if budget is None or bank_bytes <= budget:
        return frozenset()
    if not _cpu_moe_executor_viable(config.model_config):
        raise ValueError(
            f"--moe-cpu-layers auto: banks {bank_bytes / 2**30:.2f} GiB exceed the "
            f"pin budget {budget / 2**30:.2f} GiB, but the CPU MoE executor cannot "
            f"serve this model (see --moe-strategy cpu requirements)"
        )
    n = min(num_moe_layers, math.ceil(num_moe_layers * (1 - budget / bank_bytes)))
    head = (n + 1) // 2
    ids = frozenset(range(head)) | frozenset(range(num_moe_layers - (n - head), num_moe_layers))
    logger.info_rank0(
        f"--moe-cpu-layers auto: banks {bank_bytes / 2**30:.2f} GiB > pin budget "
        f"{budget / 2**30:.2f} GiB; locking {n} head+tail MoE layers for CPU decode "
        f"({sorted(ids)})"
    )
    return ids


# MoE-only knobs and the value each resolves to on a dense model. moe_strategy is handled
# separately (its dense value is 'fused', but 'auto' resolves there without a warning).
_DENSE_MOE_SETTINGS = {
    "moe_cache_size": 0,
    "moe_cache_rate": None,
    "moe_cache_auto": False,
    "moe_cache_mem": None,
    "moe_cache_scope": "unified",
    "moe_cache_split": "equal",
    "moe_cpu_layers": None,
    "moe_cpu_threads": 0,
    "moe_hybrid_max_fetch": -1,
    "moe_locked_fetch": -1.0,
    "moe_locked_max_fetch": 8,
    "moe_prefill_overlap": True,
    "moe_prefill_hit_d2d": False,
    "expert_load": "auto",
}


def _adjust_ftw_quant_backend(model_path: str, quant_backend: QuantBackend) -> QuantBackend:
    """--quant-backend with an FTW checkpoint's packed expert kernel filled in where the flag leaves that table automatic.

    An entry that names another kernel is refused here, before any bank is read."""
    from freetoken.checkpoint.ftw import ftw_quant_format
    from freetoken.moe.legacy_format import kind_kernel_for

    with _weight_load_context():
        fmt = ftw_quant_format(model_path) if model_path else None
    if fmt is None:
        return quant_backend
    try:
        kind, kernel = kind_kernel_for(fmt)
    except KeyError:
        return quant_backend
    requested = quant_backend.select(LayerKind.MOE, kind)
    if requested == kernel:
        return quant_backend
    if requested != "auto":
        raise ValueError(
            f"the FTW checkpoint's expert banks were packed for {kind} / {kernel} but --quant-backend asks for "
            f"{requested}; drop the entry, or reconvert with ft checkpoint --quant-backend moe.{kind}={requested}"
        )
    return QuantBackend(quant_backend.items + (((LayerKind.MOE, kind), kernel),))


def offload_layer_methods(model) -> list | None:
    """The expert method of each offload MoE layer of ``model``, by MoE-layer id; None for models whose MoE layers carry none (GGUF)."""
    layers = [l for l in iter_offload_moe_layers(model) if getattr(l, "quant_method", None) is not None]
    if not layers:
        return None
    layers.sort(key=lambda layer: layer.layer_id)
    assert [layer.layer_id for layer in layers] == list(range(len(layers))), [layer.layer_id for layer in layers]
    return [layer.quant_method for layer in layers]


def shared_offload_method(model):
    """The expert method the offload MoE layers of ``model`` use, or None for models whose MoE layers carry none (GGUF).

    With experts of several formats (a checkpoint that keeps some layers unquantized) this is the format of the most
    layers; the cache then takes every layer's method (``offload_layer_methods``) and keeps one slot pool per format."""
    from freetoken.moe.expert_banks import method_groups

    methods = offload_layer_methods(model)
    if methods is None:
        return None
    groups = method_groups(methods)
    found = ", ".join(f"{m.kind} via {m.kernel.name}" + (f" on {len(ids)} layers" if len(groups) > 1 else "") for m, ids in groups)
    logger.info_rank0(f"MoE experts: {found}")
    return groups[0][0]


def _slot_limit(layer_methods) -> int | None:
    """The tightest slot cap of the layers' expert kernels (a pool never mixes kernels, but the limit is checked per pool against the largest)."""
    limits = [limit for limit in (m.slot_limit() for m in layer_methods) if limit is not None]
    return min(limits) if limits else None


def _check_layer_formats(banks, layer_methods) -> None:
    from freetoken.moe.legacy_format import legacy_format_for

    want = [legacy_format_for(m.kind, m.kernel.name) for m in layer_methods]
    have = banks.layer_formats or [banks.quant_format] * len(want)
    bad = [layer for layer, (w, h) in enumerate(zip(want, have)) if w != h]
    if bad:
        raise ValueError(
            f"expert banks of MoE layers {bad[:8]} were packed as {[have[l] for l in bad[:8]]} but the model binds "
            f"{[want[l] for l in bad[:8]]}; reconvert the checkpoint or select those kernels"
        )


def _meta_model(config: EngineConfig, decode_target: str):
    from freetoken.models import create_model
    from freetoken.utils.torch_utils import torch_dtype

    set_quant_backend(_adjust_ftw_quant_backend(config.model_path, QuantBackend.parse(config.quant_backend)))
    object.__setattr__(config.model_config, "moe_strategy", config.moe_strategy)
    object.__setattr__(config.model_config, "decode_target", decode_target)
    with torch.device("meta"), torch_dtype(config.dtype):
        return create_model(config.model_config)


def offload_expert_methods(config: EngineConfig) -> tuple:
    """``(method, layer_methods)`` of ``config``'s model, from a meta-device build: the most common expert method and,
    for experts of several formats, one per MoE layer (else None).

    For tools that load expert banks without an engine (the FTW converter); they pack for GPU decode."""
    from freetoken.moe.expert_banks import is_mixed

    model = _meta_model(config, "gpu")
    layer_methods = offload_layer_methods(model)
    return shared_offload_method(model), (layer_methods if is_mixed(layer_methods) else None)


def meta_offload_layer_methods(config: EngineConfig, decode_target: str) -> list | None:
    """The expert method of each offload MoE layer of ``config``'s model, from a meta-device build for ``decode_target``; see ``offload_layer_methods``."""
    return offload_layer_methods(_meta_model(config, decode_target))


def offload_expert_method(config: EngineConfig):
    """The (most common) offload expert method of ``config``'s model; see ``offload_expert_methods``."""
    return offload_expert_methods(config)[0]


def _adjust_config(config: EngineConfig):
    def override(attr: str, value: Any):  # this is dangerous, use with caution
        object.__setattr__(config, attr, value)

    model_config = config.model_config
    single_stream_only = getattr(model_config, "single_stream_only", False)
    is_dsv4 = getattr(model_config, "dsv4_args", None) is not None
    is_dsv41 = getattr(model_config, "dsv41_args", None) is not None
    has_swa_attention = getattr(model_config, "has_swa_attention", False)
    has_linear_attention = getattr(model_config, "has_linear_attention", False)
    is_moe = getattr(model_config, "is_moe", False)
    expert_quant = getattr(model_config, "expert_quant", "none")

    if not is_moe:
        # A dense model has no routed experts: the MoE knobs are inert, and the offload family
        # is worse than inert -- engine init would build an expert cache for a model that has
        # none and abort startup (weights already resident) on an unrelated expert-source
        # error. Drop them at this one choke point, which the CLI and the programmatic
        # LLM(...) path both pass through. 'auto'/'fused' is the silent dense resolution;
        # anything else was asked for explicitly, so report what is being ignored.
        dropped = [
            f"{name}={getattr(config, name)!r}"
            for name, dense_value in _DENSE_MOE_SETTINGS.items()
            if getattr(config, name, dense_value) != dense_value
        ]
        if config.moe_strategy not in ("auto", "fused"):
            dropped.insert(0, f"moe_strategy={config.moe_strategy!r}")
        override("moe_strategy", "fused")
        for name, dense_value in _DENSE_MOE_SETTINGS.items():
            override(name, dense_value)
        if dropped:
            logger.warning_rank0(
                f"{getattr(model_config, 'model_type', 'model')} is a dense model (no routed "
                f"experts); ignoring MoE settings: {', '.join(dropped)}"
            )

    if single_stream_only:
        # The model runs one sequence at a time: it collapses the batch to one row and the
        # decode CUDA graph is captured at bs=1. Force the runtime knobs so the KV pool, page
        # table and graph capture all stay bs=1.
        if config.max_running_req != 1:
            override("max_running_req", 1)
        if config.cuda_graph_max_bs is None or config.cuda_graph_max_bs >= 1:
            override("cuda_graph_bs", [1])
            override("cuda_graph_max_bs", 1)

    if config.cuda_graph_max_bs is None:
        override("cuda_graph_max_bs", config.max_running_req)

    if is_dsv4:
        _adjust_dsv4_config(config, override)
    if is_dsv41:
        _adjust_dsv41_config(config, override)

    if has_swa_attention:
        # Both SWA cache paths use the global-paged swa pool (page_size==1 only for now).
        if config.page_size != 1:
            raise ValueError(
                f"SWA models currently support only page_size=1, got {config.page_size}."
            )
        # naive keeps cache_type='naive' (NaivePrefixCache, no reuse) on the paged pool (==
        # sglang SWAChunkCache); radix materializes as swa_radix (SWARadixCache, cross-request
        # reuse == sglang SWARadixCache). Both allocate from the same swa pool + free out-of-window.
        if getattr(config, "cache_type", "radix") != "naive":
            if not 0.0 < config.swa_full_tokens_ratio <= 1.0:
                raise ValueError(
                    f"swa_full_tokens_ratio must be in (0, 1], got {config.swa_full_tokens_ratio}"
                )
            override("cache_type", "swa_radix")

    if has_linear_attention:
        override(
            "cache_type",
            _resolve_cache_type(True, getattr(config, "cache_type", "radix")),
        )

    # Type x backend capability matrix: resolve auto from the per-type priority
    # lists, then validate whatever is now selected (explicit or auto) -- every
    # comma part must serve every required type, with packages/arch available.
    required_attn_types = _required_attn_types(model_config)
    _dtype = getattr(config, "dtype", None)  # duck-typed test configs omit it
    if (
        required_attn_types & {AttnType.BSA, AttnType.QSA}
        and _dtype is not None
        and _dtype.itemsize != 2
    ):
        # Reject at config time: the BSA/QSA pool's own assert only fires after the
        # model is resident (and not at all under `python -O`).
        raise ValueError(
            f"--dtype {config.dtype}: block-sparse attention serves 16-bit "
            "compute only (the index slab budgets 2 bytes/token); use bfloat16 "
            "or float16."
        )
    if _dtype == torch.float16 and "mxfp8" in (
        getattr(model_config, "attn_quant", "none"),
        getattr(model_config, "dense_quant", "none"),
    ):
        # The MXFP8 GEMV folds the pow2-descaled fp8 weight into the activation
        # dtype; fp16's narrow exponent can overflow/flush what bf16 represents
        # exactly, and the combination was never numerically validated.
        raise ValueError(
            "--dtype float16 with MXFP8 resident weights is unsupported (the "
            "W8A16 fold is only validated exact in bfloat16); use bfloat16."
        )
    if _dtype == torch.float16 and getattr(getattr(model_config, "quant", None), "weight_scales", False):
        # GGUF NVFP4 results are stored in the activation dtype before weight_scale_2 (~1e-4) multiplies them, which overflows fp16.
        raise ValueError("--dtype float16 with NVFP4 GGUF weight scales is unsupported (the unscaled results overflow float16); use bfloat16.")
    if config.attention_backend == "auto":
        override(
            "attention_backend",
            _resolve_auto_attention_backend(required_attn_types),
        )
        logger.info_rank0(f"Auto-selected attention backend: {config.attention_backend}")
    _validate_attention_backend_choice(config, override, required_attn_types)

    if config.moe_cache_rate is not None:
        total_experts = config.model_config.num_moe_layers * config.model_config.num_experts
        override("moe_cache_size", math.ceil(total_experts * config.moe_cache_rate))

    # The CPU MoE executor supports the silu/gelu family plus the clamped
    # swigluoai (csrc ActKind; "gpt_oss_swiglu" rides inside the mxfp4 kernel and
    # swigluoai the generic GEMV epilogue). A model with any other expert
    # activation cannot decode on the CPU: reject an explicit cpu/hybrid pick at
    # config time.
    # hidden_act (the dense activation) stands proxy for the expert activation --
    # true for every in-tree model. mxfp4 experts pass regardless: their act runs
    # inside the mxfp4 kernel, not the generic epilogue.
    _cpu_moe_act_ok = getattr(model_config, "hidden_act", "silu") in _CPU_MOE_ACTS or (
        getattr(model_config, "moe_weight_format", None) == "mxfp4"
    )
    if (
        is_moe
        and not _cpu_moe_act_ok
        and (config.moe_strategy in ("cpu", "hybrid") or config.moe_cpu_layers)
    ):
        asked = (
            f"--moe-cpu-layers={config.moe_cpu_layers!r}"
            if config.moe_strategy not in ("cpu", "hybrid")
            else f"--moe-strategy {config.moe_strategy!r}"
        )
        raise ValueError(
            f"{asked}: the CPU MoE executor does not support this model's expert "
            f"activation {getattr(model_config, 'hidden_act', None)!r}; drop the flag "
            "and let every layer decode on the GPU offload path instead."
        )

    if is_moe and config.moe_strategy == "auto":
        # A MoE model always defaults to the offload family: experts stream from pinned host
        # banks into an auto-sized GPU slot cache, which is the only default that serves a model
        # bigger than the GPU. The resident 'fused' path (every expert format but
        # DeepSeek-V4's) is still reachable, but only when asked for explicitly
        # -- auto never picks it, because nothing here knows whether the experts would fit in
        # HBM and a wrong guess is a weight-load OOM rather than a slower-but-working run.
        default_backend = "offload"
        # Unified memory (GB10/DGX Spark, Jetson): there is no host/device boundary, so
        # the offload family stages and gathers between two names for the same DRAM (on
        # GB10 this added a measured ~130 s stall to every request, #369). Resident
        # experts are the safe default here, not the risky one: the model and its banks
        # page from the same pool, so the "wrong guess = weight-load OOM" rationale above
        # does not apply. Only formats the resident path
        # can actually hold take this branch; the rest stay on offload as before.
        unified_memory = _is_unified_memory_gpu()
        if unified_memory and _fused_resident_ok(model_config):
            default_backend = "fused"
            logger.info_rank0(
                "Unified-memory GPU detected; auto-selecting 'fused' MoE strategy "
                "(resident experts) instead of offload"
            )
            # the cache sizing flags get their own warning in the 'fused' block below
            inert = [flag for flag, on in (
                (f"--moe-cpu-layers={config.moe_cpu_layers}", config.moe_cpu_layers),
                ("--moe-prefill-hit-d2d", config.moe_prefill_hit_d2d),
                ("--disable-moe-prefill-overlap", not config.moe_prefill_overlap),
            ) if on]
            if inert:
                logger.warning_rank0(
                    f"Unified-memory GPU serves MoE experts resident; ignoring {', '.join(inert)} "
                    "(use --moe-strategy offload to keep the offload flags)"
                )
                override("moe_cpu_layers", None)
        override("moe_strategy", default_backend)
        logger.info_rank0(f"Auto-selected MoE strategy: {config.moe_strategy}")

        if (
            is_offload_moe_strategy(config.moe_strategy)
            and config.moe_cache_size <= 0
            and config.moe_cache_rate is None
            and config.moe_cache_mem is None
            and not getattr(config, "moe_cache_auto", False)
        ):
            # args.py's "no sizing flag -> default --moe-cache-auto" only fires when the
            # backend is already offload-family at *parse* time. A bare `ft serve <FTW MoE
            # checkpoint>` (no --moe-strategy, no cache flags) still has moe_strategy=="auto" at
            # parse time -- the auto -> offload/cpu/hybrid resolution above is the first point
            # the concrete backend is known, so mirror the same default here: no sizing flag
            # was given, so let the scheduler resolve the cache size from free VRAM instead of
            # failing the _require_offload_cache_size guard with size=0.
            override("moe_cache_auto", True)
            logger.info_rank0(
                "No MoE cache sizing flag given; defaulting to --moe-cache-auto for "
                f"auto-selected strategy {config.moe_strategy!r}"
            )

    if (
        is_moe
        and config.moe_strategy == "offload"
        and _is_unified_memory_gpu()
        and _fused_resident_ok(model_config)
    ):
        # An explicit offload pick is honored, but on unified memory the user is paying
        # for copies between two names for the same DRAM; say so once at config time.
        logger.warning_rank0(
            "--moe-strategy offload on a unified-memory GPU: expert 'streaming' copies "
            "DRAM to DRAM (there is no PCIe link to overlap it with). If the model fits, "
            "--moe-strategy fused avoids the slot-cache machinery entirely."
        )

    if is_moe and config.moe_strategy == "fused":
        # An explicit 'fused' keeps the experts resident, so there is no slot cache to size. The
        # sizing flags no longer redirect the backend, so ignore them here and say so -- the
        # geometry the user asked for is what runs. Report the flag actually passed: --moe-cache-
        # rate was already folded into moe_cache_size above, and the sizing flags are mutually exclusive.
        if config.moe_cache_rate:
            inert = f"--moe-cache-rate={config.moe_cache_rate}"
        elif config.moe_cache_size:
            inert = f"--moe-cache-size={config.moe_cache_size}"
        elif config.moe_cache_mem is not None:
            inert = f"--moe-cache-mem={config.moe_cache_mem}"
        elif getattr(config, "moe_cache_auto", False):
            inert = "--moe-cache-auto"
        else:
            inert = None
        if inert:
            logger.warning_rank0(
                f"MoE backend 'fused' keeps its experts resident; ignoring {inert} "
                "(use --moe-strategy offload to serve experts from a slot cache)"
            )
            override("moe_cache_size", 0)
            override("moe_cache_rate", None)
            override("moe_cache_mem", None)
            override("moe_cache_auto", False)

    if is_moe and config.moe_strategy == "cpu":
        # CPU-compute decode keeps experts in host RAM and computes them on the CPU;
        # the GPU only holds the two-layer prefill double buffer. So the slot cache is
        # fixed at exactly two expert layers (prefill overlap requires >= 2*num_experts)
        # and --moe-cache-size / --moe-cache-auto / --moe-cache-rate do not apply.
        num_experts = config.model_config.num_experts
        if getattr(config, "moe_cache_auto", False):
            override("moe_cache_auto", False)
        override("moe_cache_mem", None)
        override("moe_cache_size", 2 * num_experts)
        override("moe_prefill_overlap", True)
        logger.info_rank0(
            f"MoE backend 'cpu': decode computes experts on CPU; GPU keeps a "
            f"two-layer prefill buffer (moe_cache_size={2 * num_experts})"
        )

    if (
        is_moe
        and expert_quant not in _RESIDENT_EXPERT_QUANTS
        and not is_offload_moe_strategy(config.moe_strategy)
    ):
        raise ValueError(
            f"{expert_quant} experts require --moe-strategy offload or cpu, "
            f"got {config.moe_strategy!r}"
        )

    if is_moe and config.moe_cpu_layers and config.moe_strategy not in ("offload", "hybrid"):
        # the layer split needs the offload host banks + slot cache; 'cpu' already runs every layer on CPU, 'fused' keeps experts resident on the GPU (no host banks)
        raise ValueError(
            "--moe-cpu-layers requires --moe-strategy offload or hybrid (got "
            f"{config.moe_strategy!r}); use --moe-strategy cpu to run all layers on CPU"
        )

    if is_moe and config.moe_cpu_layers and config.moe_cpu_layers.strip() != "auto":
        if not _parse_cpu_layers_spec(config.moe_cpu_layers, model_config.num_moe_layers):
            override("moe_cpu_layers", None)

    if is_moe:
        object.__setattr__(model_config, "moe_strategy", config.moe_strategy)
        object.__setattr__(model_config, "decode_target", _decode_target(config))

    # Must stay LAST: page_size is only final here (_adjust_dsv4_config sets P=128, the
    # TRTLLM block sets 64). Also covers the programmatic LLM(...) path that bypasses parse_args.
    if config.num_token_override is not None:
        if config.num_page_override is not None:
            raise ValueError("--num-tokens and --num-pages are mutually exclusive")
        if config.num_token_override % config.page_size != 0:
            raise ValueError(
                f"--num-tokens {config.num_token_override} is not a multiple of the resolved "
                f"page size {config.page_size}; nearest valid values: "
                f"{config.num_token_override // config.page_size * config.page_size} or "
                f"{(config.num_token_override // config.page_size + 1) * config.page_size}"
            )
        override("num_page_override", config.num_token_override // config.page_size)

    # The rope cos/sin table covers rotary_config.table_positions, and neither rope kernel
    # bounds-checks the position it gathers with -- a longer ceiling reads past the table.
    # DSV4 is exempt: it sizes its own table from the resolved max_seq_len (_adjust_dsv4_config).
    rotary = getattr(model_config, "rotary_config", None)
    seq_override = getattr(config, "max_seq_len_override", None)
    if seq_override is not None and rotary is not None and not (is_dsv4 or is_dsv41):
        if seq_override > rotary.table_positions:
            raise ValueError(
                f"--max-seq-len-override {seq_override} exceeds the model's "
                f"rope table ({rotary.table_positions} positions). Serving past it would read "
                "out of bounds; extend the rope with --hf-overrides (YaRN rope_parameters) instead."
            )

    # The startup ServerArgs dump is the *requested* config, printed in the frontend process
    # before any of the resolution above ran -- so "moe_strategy='auto'" is all it can say. This
    # is the one line that reports what actually runs, for every path (explicit backends never
    # hit an "Auto-selected ..." log at all).
    resolved = [
        f"attention_backend={config.attention_backend!r}",
        f"cache_type={getattr(config, 'cache_type', 'radix')!r}",
        f"page_size={config.page_size}",
    ]
    if is_moe:
        resolved.insert(0, f"moe_strategy={config.moe_strategy!r}")
    logger.info_rank0(f"Resolved config: {', '.join(resolved)}")