Skip to content

FreeToken brings datacenter-scale model serving to your desktop. Run massive models locally, fast and efficiently.

8.5 KB
Raw Blame
"""The MoE side: per-layer config, bank descriptions, the kernel ABC and the Method base."""

from __future__ import annotations

from abc import ABC, abstractmethod
from dataclasses import dataclass
from typing import Any, ClassVar

import torch

from ..method import QuantMethod
from ..scheme import QuantScheme


@dataclass(frozen=True)
class MoEConfig:
    """Everything an expert kernel may look at to accept or decline a layer and to size its banks."""

    num_experts: int
    hidden: int
    intermediate: int
    top_k: int
    tp_rank: int = 0
    tp_size: int = 1
    scheme: QuantScheme | None = None
    activation: str = "silu"
    alpha: float = 1.0
    beta: float = 0.0
    limit: float | None = None
    interleaved: bool = False
    # dynamic fp8 activation quant block of a W4A8 expert kernel, set by the quant dialect; None = no activation quant
    act_block: int | None = None
    has_bias: bool = False
    apply_router_weight_on_input: bool = False
    strategy: str = "resident"
    decode_target: str = "gpu"
    dtype: torch.dtype = torch.bfloat16

    @classmethod
    def from_layer(cls, layer: Any, scheme: QuantScheme | None) -> "MoEConfig":
        return cls(
            num_experts=layer.num_experts,
            hidden=layer.hidden_size,
            intermediate=layer.intermediate_size,
            top_k=layer.top_k,
            tp_rank=layer.tp_rank,
            tp_size=layer.tp_size,
            scheme=scheme,
            activation=layer.activation,
            alpha=float(layer.alpha),
            beta=float(layer.beta),
            limit=layer.limit,
            interleaved=bool(layer.interleaved),
            has_bias=bool(layer.has_bias),
            apply_router_weight_on_input=bool(layer.apply_router_weight_on_input),
            strategy=layer.strategy,
            decode_target=layer.decode_target,
            dtype=torch.get_default_dtype(),
        )

    @property
    def local_intermediate(self) -> int:
        return self.intermediate // self.tp_size

    @property
    def plain_silu(self) -> bool:
        return (self.activation, self.alpha, self.beta, self.limit) == ("silu", 1.0, 0.0, None)


@dataclass(frozen=True)
class BankSpec:
    """One bank of an expert layout, per expert (the E dim is prepended by the allocator)."""

    shape: tuple[int, ...]
    dtype: torch.dtype
    # GPU-resident per-expert vector (marlin / b12x alphas) instead of a host bank
    resident: bool = False


@dataclass
class ExpertView:
    """The weights an apply() call reads: layout roles -> [E, ...] (resident, materialized layer) or [S, ...] (slot cache)."""

    tensors: dict[str, torch.Tensor]
    slots: torch.Tensor | None = None
    n: int | None = None
    alphas: tuple[torch.Tensor, torch.Tensor] | None = None


def fused_piece(pieces: dict[str, torch.Tensor], role: str) -> torch.Tensor:
    """A gate_up-family piece, concatenating separate gate / up pieces along the row axis."""
    if role in pieces:
        return pieces[role]
    suffix = role[len("gate_up"):]
    return torch.cat([pieces["gate" + suffix], pieces["up" + suffix]], dim=1)


def global_rows(piece: torch.Tensor, rows: int) -> torch.Tensor:
    """Per-expert global scale as one fp32 value per output row (a scalar per expert is broadcast)."""
    flat = piece.reshape(piece.shape[0], -1).to(torch.float32)
    if flat.shape[1] == rows:
        return flat
    assert flat.shape[1] == 1, (flat.shape, rows)
    return flat.expand(-1, rows)


def fused_global(pieces: dict[str, torch.Tensor], half_rows: int) -> torch.Tensor:
    if "gate_up_global" in pieces:
        return global_rows(pieces["gate_up_global"], 2 * half_rows)
    return torch.cat([global_rows(pieces["gate_global"], half_rows), global_rows(pieces["up_global"], half_rows)], dim=1)


def limit_or_inf(layer: Any) -> float:
    limit = getattr(layer, "limit", None)
    return float("inf") if limit is None else float(limit)


def gated_epilogue_reason(cfg: "MoEConfig") -> str | None:
    """Why ``gated_act_and_mul`` cannot run ``cfg``'s activation quadruple, or None; the plain kinds have no alpha / limit and nothing takes beta or interleaved rows."""
    from freetoken.layers import GATED_ACTIVATIONS

    if cfg.activation not in GATED_ACTIVATIONS:
        return f"no {cfg.activation!r} epilogue"
    if cfg.interleaved:
        return "the epilogue reads uninterleaved [gate; up] halves"
    if cfg.beta != 0.0:
        return f"the epilogue has no beta (got {cfg.beta})"
    if cfg.activation in ("silu", "gelu", "gelu_tanh") and (cfg.alpha != 1.0 or cfg.limit is not None):
        return f"{cfg.activation} takes no alpha / limit (got alpha={cfg.alpha}, limit={cfg.limit})"
    return None


def is_resident(layer: Any) -> bool:
    return layer.quant_method.cfg.strategy == "resident"


class MoEKernel(ABC):
    """One backend for one expert kind: bank layout, packing from checkpoint pieces, and the fused forward."""

    name: ClassVar[str]
    cpu_format: ClassVar[str | None] = None
    max_slots: ClassVar[int | None] = None
    # Whether ``apply`` treats a slot of -1 as an inactive route (zero output, no read of any slot).
    # Hybrid decode hands the GPU a route split with such holes; kernels without this get the holes
    # pointed at slot 0 with weight 0 (slot storage is zero-filled at allocation, so that is exact).
    supports_inactive_slots: ClassVar[bool] = False

    def unusable_reason(self, cfg: MoEConfig) -> str | None:
        return None

    def worth_it(self, cfg: MoEConfig) -> bool:
        return True

    def cpu_options(self, cfg: MoEConfig) -> dict[str, int]:
        """Integer options of ``cpu_format`` for these experts."""
        return {}

    def slot_limit(self, cfg: MoEConfig) -> int | None:
        """Most GPU cache slots the kernel can address for ``cfg``; None for no limit."""
        return self.max_slots

    def _common_reject(self, cfg: MoEConfig, *, tp_ok: bool, cpu_ok: bool, plain_silu_only: bool) -> str | None:
        if not tp_ok and cfg.tp_size > 1:
            return "TP > 1 is not supported for this expert format"
        if not cpu_ok and cfg.decode_target != "gpu":
            return "has no CPU executor format; decode must run on the GPU"
        if cfg.has_bias:
            return "kernel has no bias epilogue"
        if plain_silu_only and not cfg.plain_silu:
            return f"kernel is plain-silu only, experts use ({cfg.activation}, alpha={cfg.alpha}, beta={cfg.beta}, limit={cfg.limit})"
        return None

    @abstractmethod
    def layout(self, cfg: MoEConfig) -> dict[str, BankSpec]: ...

    @abstractmethod
    def pack(self, pieces: dict[str, torch.Tensor], cfg: MoEConfig, out: dict[str, torch.Tensor]) -> dict[str, torch.Tensor]:
        """Write one batch of expert pieces into ``out`` rows; returns the GPU-resident per-expert values, if any."""

    @abstractmethod
    def apply(self, layer: Any, x: torch.Tensor, topk_weights: torch.Tensor, topk_ids: torch.Tensor, view: ExpertView, *, is_prefill: bool) -> torch.Tensor: ...


class MoEMethod(QuantMethod):
    """Declares resident experts for its kind; layout, pack and apply go to the kernel."""

    def layout(self) -> dict[str, BankSpec]:
        return self.kernel.layout(self.cfg)

    def pack(self, pieces: dict[str, torch.Tensor], out: dict[str, torch.Tensor]) -> dict[str, torch.Tensor]:
        return self.kernel.pack(pieces, self.cfg, out)

    def slot_limit(self) -> int | None:
        return self.kernel.slot_limit(self.cfg)

    def apply(self, x: torch.Tensor, topk_weights: torch.Tensor, topk_ids: torch.Tensor, view: ExpertView, *, layer: Any, is_prefill: bool) -> torch.Tensor:
        return self.kernel.apply(layer, x, topk_weights, topk_ids, view, is_prefill=is_prefill)

    @property
    def cpu_format(self) -> str | None:
        return self.kernel.cpu_format

    def cpu_spec(self) -> tuple[str | None, dict[str, int]]:
        """The CPU executor format and options of this layer's experts."""
        return self.cpu_format, self.kernel.cpu_options(self.cfg)

    def create_weights(self, layer: Any) -> None:
        # resident experts stay out of the state dict; attach_resident_banks fills these after the dense weights load
        layer._expert_banks = None
        layer._expert_alphas = None

    def finalize(self, layer: Any) -> None:
        pass

    def resident_view(self, layer: Any) -> ExpertView:
        """Layout roles -> the resident layer's tensors, for apply()."""
        assert layer._expert_banks is not None, "resident expert banks were never loaded"
        return ExpertView(layer._expert_banks, n=self.cfg.num_experts, alphas=layer._expert_alphas)