FreeToken brings datacenter-scale model serving to your desktop. Run massive models locally, fast and efficiently.
"""bf16 Linear: one kernel (torch), no scheme."""
from __future__ import annotations
from typing import Any
import torch
import torch.nn.functional as F
from ..registry import LayerKind, register_method
from ..scheme import QuantKind
from .base import LinearKernel, LinearMethod
class TorchLinearKernel(LinearKernel):
name = "torch"
def apply(self, layer: Any, x: torch.Tensor) -> torch.Tensor:
# an fp32 activation stream (DeepSeek-V4's compressors) upcasts the bf16 weight on the fly, as the reference does
w, b = layer.weight, layer.bias
if w.dtype != x.dtype:
w = w.to(x.dtype)
b = b.to(x.dtype) if b is not None else None
return F.linear(x, w, b)
@register_method(QuantKind.NONE, LayerKind.LINEAR)
class UnquantizedLinearMethod(LinearMethod):
candidates = (TorchLinearKernel,)
def create_weights(self, layer: Any) -> None:
g = self.cfg
layer.weight = torch.empty(g.out_features, g.in_features)