Skip to content

FreeToken brings datacenter-scale model serving to your desktop. Run massive models locally, fast and efficiently.

5.6 KB
Raw Blame
"""QuantConfig: reads one quantization_config dialect and answers what scheme each module has."""

from __future__ import annotations

from abc import ABC, abstractmethod
from dataclasses import dataclass
from typing import Any, ClassVar

from ..embedding import EmbeddingConfig
from ..linear import LinearConfig
from ..moe import MoEConfig
from ..names import Matcher, NameMap, name_set
from ..quant_backend import get_quant_backend
from ..registry import LayerKind, dialects, method_class, methods_for, register_dialect
from ..scheme import QuantKind, QuantScheme

LAYER_CONFIGS = {LayerKind.LINEAR: LinearConfig, LayerKind.MOE: MoEConfig, LayerKind.EMBEDDING: EmbeddingConfig}


def cfg_get(cfg: Any, key: str, default: Any = None) -> Any:
    if cfg is None:
        return default
    if isinstance(cfg, dict):
        return cfg.get(key, default)
    return getattr(cfg, key, default)


def quantization_config_of(hf_config: Any) -> dict[str, Any] | None:
    q = cfg_get(hf_config, "quantization_config")
    if q is None:
        q = cfg_get(cfg_get(hf_config, "text_config"), "quantization_config")
    if q is None:
        return None
    if isinstance(q, dict):
        return q
    if hasattr(q, "to_dict"):
        return dict(q.to_dict())
    return dict(vars(q))


@dataclass(frozen=True)
class Stored:
    """One checkpoint tensor behind a role: its suffix, other suffixes exports of the same dialect use for it, and whether it holds the quant-side scale whose reciprocal the layer wants."""

    name: str
    reciprocal: bool = False
    aliases: tuple[str, ...] = ()

    @property
    def names(self) -> tuple[str, ...]:
        return (self.name, *self.aliases)


class QuantConfig(ABC):
    dialect: ClassVar[str]
    # per kind the dialect exports, role -> the checkpoint tensor suffix (or Stored) that feeds it
    STORAGE: ClassVar[dict[QuantKind, dict[str, str | Stored]]]

    def __init__(self, name_map: NameMap | None = None, unquantized: tuple[str, ...] = ()):
        self.name_map = name_map or NameMap()
        # modules the family serves in bf16 regardless of the dialect (glob patterns on checkpoint names)
        self.unquantized: Matcher = name_set(tuple(unquantized))
        self._schemes: dict[str, QuantScheme | None] = {}

    @classmethod
    def claims(cls, q: dict[str, Any]) -> bool:
        """Whether this dialect reads the given ``quantization_config``."""
        return str(q.get("quant_method") or "").lower() == cls.dialect

    @abstractmethod
    def scheme_for_name(self, name: str) -> QuantScheme | None:
        """Scheme of one module by its checkpoint name; None means unquantized."""

    def scheme_for(self, prefix: str) -> QuantScheme | None:
        if prefix in self._schemes:
            return self._schemes[prefix]
        names = self.name_map.to_checkpoint(prefix)
        schemes = {None if self.unquantized(n) else self.scheme_for_name(n) for n in names}
        if len(schemes) != 1:
            raise ValueError(f"fused module {prefix!r} mixes quantization schemes across {names}: {schemes}")
        scheme = schemes.pop()
        self._schemes[prefix] = scheme
        return scheme

    def stored_tensors(self, kind: QuantKind) -> dict[str, Stored]:
        """role -> checkpoint tensor for every role the dialect stores for ``kind``."""
        return {role: entry if isinstance(entry, Stored) else Stored(entry) for role, entry in self.STORAGE[kind].items()}

    def storage(self, scheme: QuantScheme) -> dict[str, Stored]:
        """role -> checkpoint tensor for one scheme's tensors."""
        return {role: entry for role, entry in self.stored_tensors(scheme.kind).items() if scheme.has(role)}

    def get_quant_method(self, layer: Any, prefix: str):
        scheme = self.scheme_for(prefix)
        layer_kind = layer.quant_layer_kind
        kind = scheme.kind if scheme else QuantKind.NONE
        if layer_kind is LayerKind.EMBEDDING and kind not in methods_for(layer_kind):
            # a checkpoint that quantizes its linears stores the vocab table dense
            scheme, kind = None, QuantKind.NONE
        cls = method_class(kind, layer_kind)
        cfg = self.layer_config(layer, layer_kind, scheme)
        return cls(cfg, get_quant_backend().select(layer_kind, kind))

    def layer_config(self, layer: Any, layer_kind: LayerKind, scheme: QuantScheme | None) -> LinearConfig | MoEConfig:
        """The layer's Linear / MoE config; a dialect overrides it to add what only its checkpoint knows."""
        return LAYER_CONFIGS[layer_kind].from_layer(layer, scheme)

    @staticmethod
    def from_hf(
        hf_config: Any,
        *,
        name_map: NameMap | None = None,
        unquantized: tuple[str, ...] = (),
    ) -> "QuantConfig":
        """``unquantized`` lists the modules the family keeps bf16 when the checkpoint's config does not (DeepSeek-V4's compressors).

        ``hf_config`` comes from ``cached_load_hf_config``, which folds an old ModelOpt ``hf_quant_config.json`` into ``quantization_config``."""
        q = quantization_config_of(hf_config)
        if q is None:
            return NoQuantConfig(name_map, unquantized)
        for cls in dialects():
            if cls.claims(q):
                return cls(q, hf_config, name_map=name_map, unquantized=unquantized)
        raise NotImplementedError(f"quantization method {q.get('quant_method')!r} is not supported")


@register_dialect
class NoQuantConfig(QuantConfig):
    """No ``quantization_config``: every module is bf16."""

    dialect = "none"
    STORAGE: ClassVar[dict[QuantKind, dict[str, str | Stored]]] = {}

    @classmethod
    def claims(cls, q: dict[str, Any]) -> bool:
        return False

    def scheme_for_name(self, name: str) -> QuantScheme | None:
        return None