Skip to content

FreeToken brings datacenter-scale model serving to your desktop. Run massive models locally, fast and efficiently.

4.4 KB
Raw Blame
"""QuantScheme: the storage description and role set of one quantized weight, plus the per-kind builders."""

from __future__ import annotations

from dataclasses import dataclass
from enum import Enum
from typing import Iterable

import torch

# -1 selects the whole dimension: (-1, -1) one scale per tensor, (1, -1) one per output row, (1, 16) one per 16 input elements, (128, 128) a 2-D block
GroupShape = tuple[int, int]

class QuantKind(Enum):
    """The quantization kinds; one Method per (kind, layer kind)."""

    NONE = "none"
    FP8_TENSOR = "fp8_tensor"
    FP8_BLOCK = "fp8_block"
    MXFP8 = "mxfp8"
    NVFP4 = "nvfp4"
    MXFP4 = "mxfp4"
    GGUF = "gguf"

    def __str__(self) -> str:
        return self.value


FP8_BLOCK = 128  # DeepSeek-V3 style square block; V4.1 exports 32x32
FP8_BLOCK_SIZES = (32, 128)
NVFP4_GROUP = 16
MX_GROUP = 32


@dataclass(frozen=True)
class WeightDesc:
    elem: str
    group: GroupShape | None
    scale: str | None




@dataclass(frozen=True, eq=False)
class QuantScheme:
    """``roles`` are the canonical tensor names the kind's Method declares; which checkpoint tensors feed them is the dialect Config's business."""

    kind: QuantKind
    weight: WeightDesc
    roles: frozenset[str]

    def __init__(self, kind: QuantKind, weight: WeightDesc, roles: Iterable[str]):
        if not isinstance(kind, QuantKind):
            raise TypeError(f"kind must be a QuantKind, got {kind!r}")
        object.__setattr__(self, "kind", kind)
        object.__setattr__(self, "weight", weight)
        object.__setattr__(self, "roles", frozenset(roles))

    def _key(self):
        return (self.kind, self.weight, self.roles)

    def __eq__(self, other: object) -> bool:
        return isinstance(other, QuantScheme) and self._key() == other._key()

    def __hash__(self) -> int:
        return hash(self._key())

    def has(self, role: str) -> bool:
        return role in self.roles

    def __repr__(self) -> str:
        return f"QuantScheme({self.kind}, {self.weight}, roles={sorted(self.roles)})"


# ---------------------------------------------------------------------------
# Builders: one per kind; the dialect configs pick the variant their checkpoints carry
# ---------------------------------------------------------------------------


def fp8_tensor_scheme(scale: str, *, per_row: bool = False, input_scale: bool = False) -> QuantScheme:
    roles = {"weight", "weight_scale"} | ({"input_scale"} if input_scale else set())
    return QuantScheme(QuantKind.FP8_TENSOR, WeightDesc("e4m3", (1, -1) if per_row else (-1, -1), scale), roles)


def fp8_block_scheme(scale: str, block: int = FP8_BLOCK) -> QuantScheme:
    """Square ``block x block`` e4m3 weight blocks, one scale each; ``block`` is the checkpoint's ``weight_block_size``."""
    if block not in FP8_BLOCK_SIZES:
        raise NotImplementedError(f"fp8 block size {block} is not supported; one of {FP8_BLOCK_SIZES}")
    return QuantScheme(QuantKind.FP8_BLOCK, WeightDesc("e4m3", (block, block), scale), {"weight", "weight_scale_inv"})


def fp8_block_size(scheme: QuantScheme) -> int:
    """The square block edge of an FP8_BLOCK scheme (128 for DeepSeek-V3 style exports, 32 for DeepSeek-V4.1)."""
    assert scheme.kind is QuantKind.FP8_BLOCK, scheme
    return scheme.weight.group[0]


def mxfp8_scheme() -> QuantScheme:
    return QuantScheme(QuantKind.MXFP8, WeightDesc("e4m3", (1, MX_GROUP), "e8m0"), {"weight", "weight_scale_inv"})


def nvfp4_scheme(*, input_scale: bool) -> QuantScheme:
    roles = {"weight", "weight_scale", "weight_global"} | ({"input_scale"} if input_scale else set())
    return QuantScheme(QuantKind.NVFP4, WeightDesc("e2m1", (1, NVFP4_GROUP), "e4m3"), roles)


def mxfp4_scheme() -> QuantScheme:
    return QuantScheme(QuantKind.MXFP4, WeightDesc("e2m1", (1, MX_GROUP), "e8m0"), {"weight", "weight_scale"})


def gguf_scheme(types: Iterable[int], scaled: bool = False) -> QuantScheme:
    """Packed ggml blocks, one ggml type per fused part (``types`` follows the parts' order).

    ``scaled``: the NVFP4 tensors carry the fp32 ``<name>.scale`` (weight_scale_2) that llama.cpp's converter writes."""
    return QuantScheme(QuantKind.GGUF, WeightDesc("ggml:" + ",".join(str(int(t)) for t in types), None, "f32" if scaled else None), {"weight"})


def gguf_types(scheme: QuantScheme) -> tuple[int, ...]:
    assert scheme.kind is QuantKind.GGUF, scheme
    return tuple(int(t) for t in scheme.weight.elem.removeprefix("ggml:").split(","))