Skip to content

FreeToken brings datacenter-scale model serving to your desktop. Run massive models locally, fast and efficiently.

15.2 KB
Raw Blame
"""DeepSeek-V4.1-Flash model (engine-native port of ``inference/model.py``).

    embed -> hc_mult residual streams -> [Engram at 1, 14] -> 40 blocks (single-pass mHC around
    DSV41 attention and MoE) -> collapse with the last block's pre gates -> norm -> head

Single-pass mHC (``layers/mhc.py``): each sublayer boundary applies the PREVIOUS sublayer's post /
comb to the streams, predicts this sublayer's (pre, post, comb) from the updated streams, and mixes
the sublayer input with the pre gates the previous sublayer predicted. The chain state
``(y, residual, post, comb, pre)`` threads through the layer loop; layer 0's attention starts it
with a one-hot pre on stream 0 and no pending post.

Causal Encoder-Decoder: the decoder's kv source (layer 20) compresses ITS INPUT -- the final encoder
hidden state -- into the global KV every decoder layer reads. Under Decoder SWA Bounded Replay the
decoder layers run on each request's last ``window_size`` prompt tokens only (their sliding window
truncated at the start of that window, see the backend; a prefix hit stops before it, so those tokens
are always prefilled); layer 20 still publishes global KV for every prompt token.
``--swa-decoder-replay exact`` runs the decoder on every token (the reference numerics).

KV addressing is the attention backend's; pool buffers are read off the live pool per access, so a
runtime cache rebuild needs no unbind. Decode is batched and CUDA-graph safe (the DSV4 precedent).
"""

from __future__ import annotations

from typing import TYPE_CHECKING, List, NamedTuple

import torch
from freetoken.attention.dsv41_sparse import DSV41AttnMetadata, PrefillSegment
from freetoken.core import get_global_ctx
from freetoken.layers import BaseOP, OPList, ParallelLMHead, RMSNorm, VocabParallelEmbedding
from freetoken.layers.mhc import mhc_fused_post_pre_single_pass, mhc_mix_input, mhc_post
from freetoken.models.blocks import BaseLLMModel, embed_input_ids

from .args import DeepseekV41Args
from .attention import DSV41Attention, DecodeStepContext
from .engram import EngramLayer
from .moe import MoE

if TYPE_CHECKING:
    from freetoken.models.config import ModelConfig

POST_MULT = 2.0  # post gates are 2 * sigmoid (reference hc_split_sinkhorn)


class Streams(NamedTuple):
    """The single-pass mHC chain state between sublayers: the last sublayer output ``y`` with its
    not-yet-applied ``post`` / ``comb``, the residual streams ``[T, hc, dim]`` they apply to, and
    the ``pre`` gates ``[T, hc]`` the next sublayer mixes its input with."""

    y: torch.Tensor | None
    residual: torch.Tensor
    post: torch.Tensor | None
    comb: torch.Tensor | None
    pre: torch.Tensor

    def materialize(self) -> torch.Tensor:
        """The streams with the pending post applied."""
        if self.post is None:
            return self.residual
        return mhc_post(self.y, self.residual, self.post, self.comb)


class Block(BaseOP):
    def __init__(self, config: ModelConfig, layer_id: int, *, prefix: str = ""):
        args: DeepseekV41Args = config.dsv41_args
        self.layer_id = layer_id
        self.role = args.roles[layer_id]
        self.norm_eps = args.norm_eps
        self.hc_eps = args.hc_eps
        self.sinkhorn = args.hc_sinkhorn_iters
        hc, dim = args.hc_mult, args.dim
        mix = (2 + hc) * hc
        self.attn = DSV41Attention(args, self.role, quant_config=config.quant, prefix=f"{prefix}.attn")
        self.ffn = MoE(config, layer_id, prefix=f"{prefix}.ffn")
        self.attn_norm = RMSNorm(dim, args.norm_eps)
        self.ffn_norm = RMSNorm(dim, args.norm_eps)
        self.engram = EngramLayer(args, layer_id, quant_config=config.quant, prefix=f"{prefix}.engram") if layer_id in args.engram_layer_ids else None
        self.hc_attn_fn = torch.empty(mix, hc * dim, dtype=torch.float32)
        self.hc_ffn_fn = torch.empty(mix, hc * dim, dtype=torch.float32)
        self.hc_attn_base = torch.empty(mix, dtype=torch.float32)
        self.hc_ffn_base = torch.empty(mix, dtype=torch.float32)
        self.hc_attn_scale = torch.empty(3, dtype=torch.float32)
        self.hc_ffn_scale = torch.empty(3, dtype=torch.float32)

    def _boundary(self, s: Streams, fn, scale, base):
        residual, post, comb, pre_next, x = mhc_fused_post_pre_single_pass(
            s.y if s.y is not None else s.residual[:, 0], s.residual, s.post, s.comb, s.pre, fn, scale, base,
            self.norm_eps, self.hc_eps, POST_MULT, self.sinkhorn,
        )
        return residual, post, comb, pre_next, x

    def attn_input(self, s: Streams, *, image_mask: torch.Tensor | None = None):
        """Apply the pending post, predict the attention gates, mix and norm the attention input.
        Returns ``(residual, post, comb, pre_next, x)``."""
        if self.engram is not None:
            s = Streams(None, self.engram.forward(s.materialize(), image_mask=image_mask), None, None, s.pre)
        residual, post, comb, pre_next, x = self._boundary(s, self.hc_attn_fn, self.hc_attn_scale, self.hc_attn_base)
        return residual, post, comb, pre_next, self.attn_norm.forward(x)

    def ffn_step(self, y: torch.Tensor, residual, post, comb, pre, *, image_mask: torch.Tensor | None = None) -> Streams:
        """The FFN sublayer after the attention output ``y``: boundary, norm, MoE; returns the chain state."""
        residual, post, comb, pre_next, x = self._boundary(Streams(y, residual, post, comb, pre), self.hc_ffn_fn, self.hc_ffn_scale, self.hc_ffn_base)
        return Streams(self.ffn.forward(self.ffn_norm.forward(x), image_mask=image_mask), residual, post, comb, pre_next)

    def forward_prefill(self, s: Streams, segments: List[PrefillSegment], positions: torch.Tensor, image_mask: torch.Tensor | None = None) -> Streams:
        residual, post, comb, pre_next, x = self.attn_input(s, image_mask=image_mask)
        if self.attn.compressor is not None:
            self.attn.publish_prefill(x, segments)
        y = self.attn.forward_prefill(x, segments, positions)
        return self.ffn_step(y, residual, post, comb, pre_next, image_mask=image_mask)

    def forward_prefill_narrowed(
        self, s: Streams, segments: List[PrefillSegment], rows: torch.Tensor, query_segments: List[PrefillSegment],
        query_positions: torch.Tensor, image_mask: torch.Tensor | None = None, query_image_mask: torch.Tensor | None = None,
    ) -> Streams | None:
        """``forward_prefill`` with a row cut after the global KV is published: the attention input and
        ``publish_prefill`` cover every token of ``segments``, the attention and the FFN only ``rows``
        (tiled by ``query_segments``) -- the CED decoder's kv source under bounded replay. None when no
        query segment is left (every chunk in the batch ends before its prompt's last window)."""
        residual, post, comb, pre_next, x = self.attn_input(s, image_mask=image_mask)
        if self.attn.compressor is not None:
            self.attn.publish_prefill(x, segments)
        if not query_segments:
            return None
        y = self.attn.forward_prefill(x[rows], query_segments, query_positions)
        return self.ffn_step(y, residual[rows], post[rows], comb[rows], pre_next[rows], image_mask=query_image_mask)

    def forward_decode(self, s: Streams, pos: torch.Tensor, rows: torch.Tensor, dctx, cmp_stage_cap: int) -> Streams:
        residual, post, comb, pre_next, x = self.attn_input(s)
        y = self.attn.forward_decode(x, pos, rows, dctx, cmp_stage_cap)
        return self.ffn_step(y, residual, post, comb, pre_next)


class Transformer(BaseOP):
    def __init__(self, config: ModelConfig, *, prefix: str = "model"):
        args: DeepseekV41Args = config.dsv41_args
        self.args = args
        self.embed = VocabParallelEmbedding(num_embeddings=args.vocab_size, embedding_dim=args.dim)
        self.layers = OPList([Block(config, i, prefix=f"{prefix}.layers.{i}") for i in range(args.n_layers)])
        self.norm = RMSNorm(args.dim, args.norm_eps)

    def bind(self, device: torch.device) -> None:
        tables: dict = {}  # one RoPE table per distinct parameter set, shared by the layers using it
        for block in self.layers.op_list:
            block.attn.bind(device, tables)

    def _entry(self, input_ids: torch.Tensor) -> Streams:
        h = embed_input_ids(self.embed, input_ids, get_global_ctx().batch)
        residual = h.unsqueeze(1).expand(-1, self.args.hc_mult, -1).contiguous()
        pre = torch.zeros(h.shape[0], self.args.hc_mult, dtype=torch.float32, device=h.device)
        pre[:, 0] = 1.0
        return Streams(None, residual, None, None, pre)

    def _exit(self, s: Streams) -> torch.Tensor:
        return self.norm.forward(mhc_mix_input(s.materialize(), s.pre))

    def prefill(self, input_ids: torch.Tensor, positions: torch.Tensor, md: DSV41AttnMetadata) -> torch.Tensor:
        """Ragged prefill over the flat new-token stream; returns the hidden states the head reads (the
        decoder stream under bounded replay, where ``md.last_indices`` also point, plus the placeholder
        row ``md.decoder_pad`` asks for)."""
        s = self._entry(input_ids)
        dtype = s.residual.dtype
        image_mask = input_ids >= self.args.vocab_size if get_global_ctx().batch.mm_embeds is not None else None
        segments, dec_segments, rows = md.segments, md.decoder_segments, md.decoder_rows
        decoder_start = self.args.decoder_start_layer
        if rows is not None:
            dec_positions = positions[rows]
            dec_image_mask = None if image_mask is None else image_mask[rows]
        for block in self.layers.op_list:
            if rows is None or block.layer_id < decoder_start:
                s = block.forward_prefill(s, segments, positions, image_mask)
            elif block.layer_id == decoder_start:
                s = block.forward_prefill_narrowed(s, segments, rows, dec_segments, dec_positions, image_mask, dec_image_mask)
                if s is None:
                    break
            else:
                s = block.forward_prefill(s, dec_segments, dec_positions, dec_image_mask)
        pad = [torch.zeros(1, self.args.dim, dtype=dtype, device=input_ids.device)] if md.decoder_pad else []
        return torch.cat([self._exit(s)] + pad) if s is not None else pad[0]

    def decode(self, input_ids: torch.Tensor, pos: torch.Tensor, md: DSV41AttnMetadata, cmp_stage_cap: int) -> torch.Tensor:
        B = input_ids.shape[0]
        rows = torch.arange(B, device=input_ids.device)
        s = self._entry(input_ids)
        # the layer-invariant step context, resolved once (a capture records it once): ring slots and
        # window candidates of the shared pool -- and of the private rings when bounded replay keeps
        # the decoder's window KV per request -- plus each RoPE table at the step's positions
        dctx = DecodeStepContext.build([b.attn for b in self.layers.op_list], md, pos, rows, self.args.window_size, self.args.index_topk)
        for block in self.layers.op_list:
            s = block.forward_decode(s, pos, rows, dctx, cmp_stage_cap)
        return self._exit(s)


class DeepseekV41ForCausalLM(BaseLLMModel):
    def __init__(self, config: ModelConfig):
        self._config = config
        self._args: DeepseekV41Args = config.dsv41_args
        self.model = Transformer(config)
        self.head = ParallelLMHead(num_embeddings=self._args.vocab_size, embedding_dim=self._args.dim, quant_config=config.quant, prefix="head")
        self._bound = False
        if config.is_multimodal:
            from .vision import DeepseekV41Vision

            self.visual = DeepseekV41Vision(config.vision_config, config.hidden_size,
                                           num_layers=self._args.n_layers, num_experts=self._args.n_routed_experts)

    def encode(self, item):
        return self.visual.forward(item)

    def place_encoder_weights(self, mode: str) -> None:
        self.visual.place_weights(mode)

    def _ensure_bound(self) -> None:
        if not self._bound:
            self.model.bind(get_global_ctx().kv_cache.device)
            if self._config.is_multimodal:
                for i, block in enumerate(self.model.layers.op_list):
                    block.ffn.gate._bias_vl = self.visual.routing_bias[i]
            self._bound = True

    def mark_for_rebind(self) -> None:
        """A runtime pool rebuild changes nothing the model caches (rope tables are pool-independent),
        but the engine asks; re-bind is cheap and keeps the contract with the DSV4 sibling."""
        self._bound = False

    def engram_layers(self) -> List[EngramLayer]:
        return [b.engram for b in self.model.layers.op_list if b.engram is not None]

    def load_host_tables(self, engine_config) -> int:
        """Attach the Engram tables (streamed from the checkpoint shards; zeros for dummy weights)
        and take over ``forward_host_ctx`` so every dispatch stages its rows. Returns the pinned
        host bytes the engine reserves from its pin budget."""
        import os

        layers = self.engram_layers()
        if not layers:
            return 0
        device = torch.device("cuda", torch.cuda.current_device())
        if getattr(engine_config, "use_dummy_weight", False):
            from .engram import ZeroEngramTable

            for layer in layers:
                layer.attach_table(ZeroEngramTable(layer.width, device))
            return 0
        from freetoken.utils import download_hf_weight
        from transformers import AutoTokenizer

        from .engram import EngramHash, build_compressed_token_map
        from .engram_table import EngramDiskTable, EngramHost, engram_row_source

        folder = download_hf_weight(engine_config.model_path)
        token_map, compressed_vocab = build_compressed_token_map(AutoTokenizer.from_pretrained(folder, trust_remote_code=True))
        hash = EngramHash(self._args, token_map, compressed_vocab)
        graph_rows = max(engine_config.max_running_req, engine_config.cuda_graph_max_bs or 0, 1)
        tables = [
            EngramDiskTable(
                engram_row_source(folder, layer.layer_id), hash.n_cols, device,
                max_graph_rows=graph_rows,
                max_extend_tokens=engine_config.max_extend_tokens,
                use_io_uring=os.getenv("FREETOKEN_ENGRAM_IO_URING", "1") != "0",
            )
            for layer in layers
        ]
        for layer, table in zip(layers, tables):
            layer.attach_table(table)
        self._engram_host = EngramHost(hash, tables, device, sync_mode=os.getenv("FREETOKEN_ENGRAM_SYNC", "auto"))
        self.forward_host_ctx = self._engram_host.forward_host_ctx
        return sum(t._graph_pinned.numel() + sum(b.numel() for b in t._eager_pinned) for t in tables)

    def forward(self) -> torch.Tensor:
        self._ensure_bound()
        batch = get_global_ctx().batch
        md = batch.attn_metadata
        assert isinstance(md, DSV41AttnMetadata)
        if batch.is_prefill:
            hidden = self.model.prefill(batch.input_ids, batch.positions.long(), md)
        else:
            input_ids = batch.input_ids
            B = batch.padded_size
            pos = batch.positions.long().view(-1)[:B]
            if torch.cuda.is_current_stream_capturing():
                cmp_stage_cap = md.stage_width - 1
            else:
                cmp_stage_cap = int(pos.max().item())
            hidden = self.model.decode(input_ids.view(B), pos, md, cmp_stage_cap)
        return self.head.forward(hidden)


__all__ = ["DeepseekV41ForCausalLM", "Transformer", "Block", "Streams"]