FreeToken brings datacenter-scale model serving to your desktop. Run massive models locally, fast and efficiently.
"""The MoE side: per-layer config, bank descriptions, the kernel ABC and the Method base."""
from __future__ import annotations
from abc import ABC, abstractmethod
from dataclasses import dataclass
from typing import Any, ClassVar
import torch
from ..method import QuantMethod
from ..scheme import QuantScheme
@dataclass(frozen=True)
class MoEConfig:
"""Everything an expert kernel may look at to accept or decline a layer and to size its banks."""
num_experts: int
hidden: int
intermediate: int
top_k: int
tp_rank: int = 0
tp_size: int = 1
scheme: QuantScheme | None = None
activation: str = "silu"
alpha: float = 1.0
beta: float = 0.0
limit: float | None = None
interleaved: bool = False
# dynamic fp8 activation quant block of a W4A8 expert kernel, set by the quant dialect; None = no activation quant
act_block: int | None = None
has_bias: bool = False
apply_router_weight_on_input: bool = False
strategy: str = "resident"
decode_target: str = "gpu"
dtype: torch.dtype = torch.bfloat16
@classmethod
def from_layer(cls, layer: Any, scheme: QuantScheme | None) -> "MoEConfig":
return cls(
num_experts=layer.num_experts,
hidden=layer.hidden_size,
intermediate=layer.intermediate_size,
top_k=layer.top_k,
tp_rank=layer.tp_rank,
tp_size=layer.tp_size,
scheme=scheme,
activation=layer.activation,
alpha=float(layer.alpha),
beta=float(layer.beta),
limit=layer.limit,
interleaved=bool(layer.interleaved),
has_bias=bool(layer.has_bias),
apply_router_weight_on_input=bool(layer.apply_router_weight_on_input),
strategy=layer.strategy,
decode_target=layer.decode_target,
dtype=torch.get_default_dtype(),
)
@property
def local_intermediate(self) -> int:
return self.intermediate // self.tp_size
@property
def plain_silu(self) -> bool:
return (self.activation, self.alpha, self.beta, self.limit) == ("silu", 1.0, 0.0, None)
@dataclass(frozen=True)
class BankSpec:
"""One bank of an expert layout, per expert (the E dim is prepended by the allocator)."""
shape: tuple[int, ...]
dtype: torch.dtype
# GPU-resident per-expert vector (marlin / b12x alphas) instead of a host bank
resident: bool = False
@dataclass
class ExpertView:
"""The weights an apply() call reads: layout roles -> [E, ...] (resident, materialized layer) or [S, ...] (slot cache)."""
tensors: dict[str, torch.Tensor]
slots: torch.Tensor | None = None
n: int | None = None
alphas: tuple[torch.Tensor, torch.Tensor] | None = None
def fused_piece(pieces: dict[str, torch.Tensor], role: str) -> torch.Tensor:
"""A gate_up-family piece, concatenating separate gate / up pieces along the row axis."""
if role in pieces:
return pieces[role]
suffix = role[len("gate_up"):]
return torch.cat([pieces["gate" + suffix], pieces["up" + suffix]], dim=1)
def global_rows(piece: torch.Tensor, rows: int) -> torch.Tensor:
"""Per-expert global scale as one fp32 value per output row (a scalar per expert is broadcast)."""
flat = piece.reshape(piece.shape[0], -1).to(torch.float32)
if flat.shape[1] == rows:
return flat
assert flat.shape[1] == 1, (flat.shape, rows)
return flat.expand(-1, rows)
def fused_global(pieces: dict[str, torch.Tensor], half_rows: int) -> torch.Tensor:
if "gate_up_global" in pieces:
return global_rows(pieces["gate_up_global"], 2 * half_rows)
return torch.cat([global_rows(pieces["gate_global"], half_rows), global_rows(pieces["up_global"], half_rows)], dim=1)
def limit_or_inf(layer: Any) -> float:
limit = getattr(layer, "limit", None)
return float("inf") if limit is None else float(limit)
def gated_epilogue_reason(cfg: "MoEConfig") -> str | None:
"""Why ``gated_act_and_mul`` cannot run ``cfg``'s activation quadruple, or None; the plain kinds have no alpha / limit and nothing takes beta or interleaved rows."""
from freetoken.layers import GATED_ACTIVATIONS
if cfg.activation not in GATED_ACTIVATIONS:
return f"no {cfg.activation!r} epilogue"
if cfg.interleaved:
return "the epilogue reads uninterleaved [gate; up] halves"
if cfg.beta != 0.0:
return f"the epilogue has no beta (got {cfg.beta})"
if cfg.activation in ("silu", "gelu", "gelu_tanh") and (cfg.alpha != 1.0 or cfg.limit is not None):
return f"{cfg.activation} takes no alpha / limit (got alpha={cfg.alpha}, limit={cfg.limit})"
return None
def is_resident(layer: Any) -> bool:
return layer.quant_method.cfg.strategy == "resident"
class MoEKernel(ABC):
"""One backend for one expert kind: bank layout, packing from checkpoint pieces, and the fused forward."""
name: ClassVar[str]
cpu_format: ClassVar[str | None] = None
max_slots: ClassVar[int | None] = None
# Whether ``apply`` treats a slot of -1 as an inactive route (zero output, no read of any slot).
# Hybrid decode hands the GPU a route split with such holes; kernels without this get the holes
# pointed at slot 0 with weight 0 (slot storage is zero-filled at allocation, so that is exact).
supports_inactive_slots: ClassVar[bool] = False
def unusable_reason(self, cfg: MoEConfig) -> str | None:
return None
def worth_it(self, cfg: MoEConfig) -> bool:
return True
def cpu_options(self, cfg: MoEConfig) -> dict[str, int]:
"""Integer options of ``cpu_format`` for these experts."""
return {}
def slot_limit(self, cfg: MoEConfig) -> int | None:
"""Most GPU cache slots the kernel can address for ``cfg``; None for no limit."""
return self.max_slots
def _common_reject(self, cfg: MoEConfig, *, tp_ok: bool, cpu_ok: bool, plain_silu_only: bool) -> str | None:
if not tp_ok and cfg.tp_size > 1:
return "TP > 1 is not supported for this expert format"
if not cpu_ok and cfg.decode_target != "gpu":
return "has no CPU executor format; decode must run on the GPU"
if cfg.has_bias:
return "kernel has no bias epilogue"
if plain_silu_only and not cfg.plain_silu:
return f"kernel is plain-silu only, experts use ({cfg.activation}, alpha={cfg.alpha}, beta={cfg.beta}, limit={cfg.limit})"
return None
@abstractmethod
def layout(self, cfg: MoEConfig) -> dict[str, BankSpec]: ...
@abstractmethod
def pack(self, pieces: dict[str, torch.Tensor], cfg: MoEConfig, out: dict[str, torch.Tensor]) -> dict[str, torch.Tensor]:
"""Write one batch of expert pieces into ``out`` rows; returns the GPU-resident per-expert values, if any."""
@abstractmethod
def apply(self, layer: Any, x: torch.Tensor, topk_weights: torch.Tensor, topk_ids: torch.Tensor, view: ExpertView, *, is_prefill: bool) -> torch.Tensor: ...
class MoEMethod(QuantMethod):
"""Declares resident experts for its kind; layout, pack and apply go to the kernel."""
def layout(self) -> dict[str, BankSpec]:
return self.kernel.layout(self.cfg)
def pack(self, pieces: dict[str, torch.Tensor], out: dict[str, torch.Tensor]) -> dict[str, torch.Tensor]:
return self.kernel.pack(pieces, self.cfg, out)
def slot_limit(self) -> int | None:
return self.kernel.slot_limit(self.cfg)
def apply(self, x: torch.Tensor, topk_weights: torch.Tensor, topk_ids: torch.Tensor, view: ExpertView, *, layer: Any, is_prefill: bool) -> torch.Tensor:
return self.kernel.apply(layer, x, topk_weights, topk_ids, view, is_prefill=is_prefill)
@property
def cpu_format(self) -> str | None:
return self.kernel.cpu_format
def cpu_spec(self) -> tuple[str | None, dict[str, int]]:
"""The CPU executor format and options of this layer's experts."""
return self.cpu_format, self.kernel.cpu_options(self.cfg)
def create_weights(self, layer: Any) -> None:
# resident experts stay out of the state dict; attach_resident_banks fills these after the dense weights load
layer._expert_banks = None
layer._expert_alphas = None
def finalize(self, layer: Any) -> None:
pass
def resident_view(self, layer: Any) -> ExpertView:
"""Layout roles -> the resident layer's tensors, for apply()."""
assert layer._expert_banks is not None, "resident expert banks were never loaded"
return ExpertView(layer._expert_banks, n=self.cfg.num_experts, alphas=layer._expert_alphas)