Skip to content

FreeToken brings datacenter-scale model serving to your desktop. Run massive models locally, fast and efficiently.

4.1 KB
Raw Blame
"""DeepSeek-V4.1 MoE block: sqrtsoftplus router with a selection-only correction bias, one shared
clamped-SwiGLU expert, and the routed MXFP4 experts on the shared offload cache.

Router (reference ``Gate``): ``scores = sqrt(softplus(x @ W^T))`` in fp32; the bias picks the experts,
the unbiased scores weight them (renormalized, ``+1e-20`` as in training), times ``route_scale``.
Image-span tokens select experts using the checkpoint's separate ``bias_vl``.
"""

from __future__ import annotations

from typing import TYPE_CHECKING, Tuple

import torch
from freetoken.kernel.triton.dsv4.bf16_linear import bf16_linear_fp32
from freetoken.kernel.triton.dsv4.router import router_select
from freetoken.kernel.triton.dsv4.swiglu import fused_swiglu
from freetoken.layers import BaseOP, LinearReplicated, make_moe_layer

from .args import DeepseekV41Args

if TYPE_CHECKING:
    from freetoken.models.config import ModelConfig

TopK = Tuple[torch.Tensor, torch.Tensor]


class Router(BaseOP):
    def __init__(self, args: DeepseekV41Args):
        self.topk = args.n_activated_experts
        self.score_func = args.score_func
        self.norm_topk_prob = args.norm_topk_prob
        self.route_scale = args.route_scale
        self.weight = torch.empty(args.n_routed_experts, args.dim, dtype=torch.bfloat16)
        self.bias = torch.empty(args.n_routed_experts, dtype=torch.float32)
        self._bias_vl = None  # bound to the visual stack's per-layer routing bias after loading

    def forward(self, x: torch.Tensor, *, image_mask: torch.Tensor | None = None) -> TopK:
        if image_mask is not None:
            assert self._bias_vl is not None, "image routing bias was not loaded"
        scores = bf16_linear_fp32(x, self.weight)
        return router_select(
            scores.view(-1, scores.shape[-1]), self.bias, self.topk,
            score_func=self.score_func, normalize=self.norm_topk_prob, route_scale=self.route_scale,
            alternate_bias=self._bias_vl, alternate_mask=image_mask,
        )


class SharedExpert(BaseOP):
    """The dense clamped-SwiGLU expert every token passes through (fp8 block-32 linears)."""

    def __init__(self, args: DeepseekV41Args, *, quant_config=None, prefix: str = ""):
        self.w1 = LinearReplicated(args.dim, args.moe_inter_dim, has_bias=False, quant_config=quant_config, prefix=f"{prefix}.w1")
        self.w2 = LinearReplicated(args.moe_inter_dim, args.dim, has_bias=False, quant_config=quant_config, prefix=f"{prefix}.w2")
        self.w3 = LinearReplicated(args.dim, args.moe_inter_dim, has_bias=False, quant_config=quant_config, prefix=f"{prefix}.w3")
        self.limit = args.swiglu_limit

    def forward(self, x: torch.Tensor) -> torch.Tensor:
        h = fused_swiglu(self.w1.forward(x), self.w3.forward(x), self.limit, x.dtype)
        return self.w2.forward(h)


class MoE(BaseOP):
    def __init__(self, config: ModelConfig, layer_id: int, *, prefix: str = ""):
        args: DeepseekV41Args = config.dsv41_args
        self.dim = args.dim
        self.gate = Router(args)
        self.shared_experts = SharedExpert(args, quant_config=config.quant, prefix=f"{prefix}.shared_experts")
        # every layer is MoE: the offload cache indexes experts by layer id directly
        self.experts = make_moe_layer(
            config,
            layer_id=layer_id,
            activation="swiglu_clamp",
            renormalize=args.norm_topk_prob,
            limit=args.swiglu_limit,
            quant_config=config.quant,
            prefix=f"{prefix}.experts",
        )

    def forward(self, x: torch.Tensor, *, image_mask: torch.Tensor | None = None) -> torch.Tensor:
        shape = x.shape
        x = x.view(-1, self.dim)
        weights, indices = self.gate.forward(x, image_mask=image_mask)
        # the shared expert runs before the routed experts (same stream, so it does not overlap the
        # hybrid decode's CPU work, whose submit follows it -- reordering that is follow-up work)
        shared = self.shared_experts.forward(x)
        routed = self.experts.routed_forward(x, weights, indices)
        return (routed + shared).view(shape)


__all__ = ["MoE", "Router", "SharedExpert"]