Skip to content

FreeToken brings datacenter-scale model serving to your desktop. Run massive models locally, fast and efficiently.

1.7 KB
Raw Blame
from __future__ import annotations

from dataclasses import replace

import torch

from freetoken.kernel.triton.moe_router import fused_topk_sigmoid
from freetoken.layers import BaseOP, LinearReplicated, make_moe_layer
from freetoken.models.blocks import GatedMLP
from freetoken.models.config import ModelConfig


class FlintMoE(BaseOP):
    def __init__(self, config: ModelConfig, layer_id: int, *, prefix: str = ""):
        self.top_k = config.num_experts_per_tok
        self.renormalize = config.norm_topk_prob
        self.gate = LinearReplicated(config.hidden_size, config.num_experts, has_bias=False)
        self.gate.expert_bias = torch.zeros(config.num_experts, dtype=torch.float32)
        self.experts = make_moe_layer(
            config, layer_id=layer_id - config.first_k_dense_replace,
            quant_config=config.quant, prefix=f"{prefix}.experts",
        )
        shared_config = replace(config, intermediate_size=config.shared_expert_intermediate_size)
        self.shared_expert = GatedMLP(shared_config, quant_config=config.quant, prefix=f"{prefix}.shared_expert")

    def _route(self, x: torch.Tensor) -> tuple[torch.Tensor, torch.Tensor]:
        # The reference projects in model dtype, then computes sigmoid in fp32.
        return fused_topk_sigmoid(
            self.gate.forward(x).float(), self.gate.expert_bias,
            self.top_k, self.renormalize,
        )

    def forward(self, x: torch.Tensor) -> torch.Tensor:
        weights, ids = self._route(x)
        # Some routed kernels overwrite their input; the shared branch needs it first.
        shared = self.shared_expert.forward(x)
        return self.experts.routed_forward(x, weights, ids) + shared