Skip to content

FreeToken brings datacenter-scale model serving to your desktop. Run massive models locally, fast and efficiently.

5.6 KB
Raw Blame
"""DeepSeek ViT and 3x3 pixel-unshuffle aligner, with the shared encoder weight streamer."""

from __future__ import annotations

import torch
import torch.nn.functional as F
from freetoken.layers import BaseOP, LinearReplicated, OPList
from freetoken.models.weight_stream import BlockWeightStreamer


class VisionNorm(BaseOP):
    def __init__(self, dim: int):
        self.weight = torch.empty(dim, dtype=torch.float32)

    def forward(self, x: torch.Tensor) -> torch.Tensor:
        y = x.float()
        return (y * torch.rsqrt(y.square().mean(-1, keepdim=True) + 1e-6) * self.weight).to(x.dtype)


class VisionAttention(BaseOP):
    def __init__(self, vc):
        self.n_heads = vc.num_attention_heads
        self.head_dim = vc.hidden_size // self.n_heads
        self.wqkv = LinearReplicated(vc.hidden_size, 3 * vc.hidden_size, has_bias=True)
        self.wo = LinearReplicated(vc.hidden_size, vc.hidden_size, has_bias=True)

    def forward(self, x, cos, sin):
        n = x.shape[0]
        q, k, v = (t.reshape(n, self.n_heads, self.head_dim) for t in self.wqkv.forward(x).chunk(3, -1))
        def rotary(t):
            a, b = t.float().chunk(2, -1)
            return torch.cat((a * cos - b * sin, b * cos + a * sin), -1).to(t.dtype)
        q, k = rotary(q), rotary(k)
        q, k, v = (t.transpose(0, 1).unsqueeze(0) for t in (q, k, v))
        y = F.scaled_dot_product_attention(q, k, v)
        return self.wo.forward(y[0].transpose(0, 1).reshape(n, -1))


class VisionMLP(BaseOP):
    def __init__(self, vc):
        self.w1 = LinearReplicated(vc.hidden_size, 2 * vc.intermediate_size, has_bias=False)
        self.w2 = LinearReplicated(vc.intermediate_size, vc.hidden_size, has_bias=False)

    def forward(self, x):
        gate, up = self.w1.forward(x).chunk(2, -1)
        return self.w2.forward(F.silu(gate) * up)


class VisionBlock(BaseOP):
    def __init__(self, vc):
        self.norm1 = VisionNorm(vc.hidden_size)
        self.attn = VisionAttention(vc)
        self.norm2 = VisionNorm(vc.hidden_size)
        self.mlp = VisionMLP(vc)

    def forward(self, x, cos, sin):
        x = x + self.attn.forward(self.norm1.forward(x), cos, sin)
        return x + self.mlp.forward(self.norm2.forward(x))


class PatchEmbed(BaseOP):
    def __init__(self, vc):
        self.proj = LinearReplicated(3 * vc.patch_size**2, vc.hidden_size, has_bias=True)

    def forward(self, x):
        return self.proj.forward(x.flatten(1))


class VisionTransformer(BaseOP):
    def __init__(self, vc):
        self.patch_embed = PatchEmbed(vc)
        self.blocks = OPList([VisionBlock(vc) for _ in range(vc.num_hidden_layers)])
        self.norm = VisionNorm(vc.hidden_size)
        self._vc = vc
        self._streamer = None

    def place_weights(self, mode: str) -> None:
        if mode == "host" and self._streamer is None:
            self._streamer = BlockWeightStreamer(self.blocks.op_list, self.patch_embed.proj.weight.device)
        elif mode == "gpu" and self._streamer is not None:
            self._streamer.unstream()
            self._streamer = None
        elif mode not in ("gpu", "host"):
            raise ValueError(f"unknown vision weight placement {mode!r}")

    def forward(self, patches, h: int, w: int):
        x = self.patch_embed.forward(patches)
        dim = self._vc.hidden_size // self._vc.num_attention_heads // 2
        inv = 1.0 / (self._vc.rope_theta ** (torch.arange(0, dim, 2, dtype=torch.float32, device=x.device) / dim))
        rows = torch.arange(h, device=x.device)[:, None].expand(h, w)
        cols = torch.arange(w, device=x.device)[None, :].expand(h, w)
        angles = (torch.stack((rows, cols), -1).reshape(-1, 2, 1).float() * inv).flatten(1)
        cos, sin = angles.cos().unsqueeze(1), angles.sin().unsqueeze(1)
        blocks = enumerate(self.blocks.op_list) if self._streamer is None else self._streamer.blocks(self.blocks.op_list)
        for _, block in blocks:
            x = block.forward(x, cos, sin)
        return self.norm.forward(x)


class VisionAligner(BaseOP):
    def __init__(self, vc, dim: int):
        self.ratio = vc.downsample_ratio
        self.w1 = LinearReplicated(vc.hidden_size * self.ratio**2, dim, has_bias=True)
        self.w2 = LinearReplicated(dim, dim, has_bias=True)

    def forward(self, x, h: int, w: int):
        r = self.ratio
        x = x.view(h, w, -1).permute(2, 0, 1)
        x = F.pad(x, (0, -w % r, 0, -h % r))
        x = F.unfold(x.unsqueeze(0), r, stride=r).squeeze(0).T
        return self.w2.forward(F.gelu(self.w1.forward(x)))


class DeepseekV41Vision(BaseOP):
    def __init__(self, vc, dim: int, *, num_layers: int, num_experts: int):
        self.vision = VisionTransformer(vc)
        self.aligner = VisionAligner(vc, dim)
        self.image_start = torch.empty(dim)
        self.image_end = torch.empty(dim)
        self.image_newline = torch.empty(dim)
        # Keep all image-only parameters under visual, including FTW/text-only filtering.
        self.routing_bias = torch.empty(num_layers, num_experts, dtype=torch.float32)

    def place_weights(self, mode: str) -> None:
        self.vision.place_weights(mode)

    @torch.inference_mode()
    def forward(self, item):
        _, h, w = item.grid_thw
        patches = item.feature.to(device=self.image_start.device, dtype=self.image_start.dtype)
        x = self.aligner.forward(self.vision.forward(patches, h, w), h, w)
        r = self.aligner.ratio
        nh, nw = (h + r - 1) // r, (w + r - 1) // r
        out = x.new_empty(nh * (nw + 1) + 2, x.shape[-1])
        out[0], out[-1] = self.image_start, self.image_end
        rows = out[1:-1].view(nh, nw + 1, -1)
        rows[:, :nw] = x.view(nh, nw, -1)
        rows[:, nw] = self.image_newline
        return out