FreeToken brings datacenter-scale model serving to your desktop. Run massive models locally, fast and efficiently.
"""QuantConfig: reads one quantization_config dialect and answers what scheme each module has."""
from __future__ import annotations
from abc import ABC, abstractmethod
from dataclasses import dataclass
from typing import Any, ClassVar
from ..embedding import EmbeddingConfig
from ..linear import LinearConfig
from ..moe import MoEConfig
from ..names import Matcher, NameMap, name_set
from ..quant_backend import get_quant_backend
from ..registry import LayerKind, dialects, method_class, methods_for, register_dialect
from ..scheme import QuantKind, QuantScheme
LAYER_CONFIGS = {LayerKind.LINEAR: LinearConfig, LayerKind.MOE: MoEConfig, LayerKind.EMBEDDING: EmbeddingConfig}
def cfg_get(cfg: Any, key: str, default: Any = None) -> Any:
if cfg is None:
return default
if isinstance(cfg, dict):
return cfg.get(key, default)
return getattr(cfg, key, default)
def quantization_config_of(hf_config: Any) -> dict[str, Any] | None:
q = cfg_get(hf_config, "quantization_config")
if q is None:
q = cfg_get(cfg_get(hf_config, "text_config"), "quantization_config")
if q is None:
return None
if isinstance(q, dict):
return q
if hasattr(q, "to_dict"):
return dict(q.to_dict())
return dict(vars(q))
@dataclass(frozen=True)
class Stored:
"""One checkpoint tensor behind a role: its suffix, other suffixes exports of the same dialect use for it, and whether it holds the quant-side scale whose reciprocal the layer wants."""
name: str
reciprocal: bool = False
aliases: tuple[str, ...] = ()
@property
def names(self) -> tuple[str, ...]:
return (self.name, *self.aliases)
class QuantConfig(ABC):
dialect: ClassVar[str]
# per kind the dialect exports, role -> the checkpoint tensor suffix (or Stored) that feeds it
STORAGE: ClassVar[dict[QuantKind, dict[str, str | Stored]]]
def __init__(self, name_map: NameMap | None = None, unquantized: tuple[str, ...] = ()):
self.name_map = name_map or NameMap()
# modules the family serves in bf16 regardless of the dialect (glob patterns on checkpoint names)
self.unquantized: Matcher = name_set(tuple(unquantized))
self._schemes: dict[str, QuantScheme | None] = {}
@classmethod
def claims(cls, q: dict[str, Any]) -> bool:
"""Whether this dialect reads the given ``quantization_config``."""
return str(q.get("quant_method") or "").lower() == cls.dialect
@abstractmethod
def scheme_for_name(self, name: str) -> QuantScheme | None:
"""Scheme of one module by its checkpoint name; None means unquantized."""
def scheme_for(self, prefix: str) -> QuantScheme | None:
if prefix in self._schemes:
return self._schemes[prefix]
names = self.name_map.to_checkpoint(prefix)
schemes = {None if self.unquantized(n) else self.scheme_for_name(n) for n in names}
if len(schemes) != 1:
raise ValueError(f"fused module {prefix!r} mixes quantization schemes across {names}: {schemes}")
scheme = schemes.pop()
self._schemes[prefix] = scheme
return scheme
def stored_tensors(self, kind: QuantKind) -> dict[str, Stored]:
"""role -> checkpoint tensor for every role the dialect stores for ``kind``."""
return {role: entry if isinstance(entry, Stored) else Stored(entry) for role, entry in self.STORAGE[kind].items()}
def storage(self, scheme: QuantScheme) -> dict[str, Stored]:
"""role -> checkpoint tensor for one scheme's tensors."""
return {role: entry for role, entry in self.stored_tensors(scheme.kind).items() if scheme.has(role)}
def get_quant_method(self, layer: Any, prefix: str):
scheme = self.scheme_for(prefix)
layer_kind = layer.quant_layer_kind
kind = scheme.kind if scheme else QuantKind.NONE
if layer_kind is LayerKind.EMBEDDING and kind not in methods_for(layer_kind):
# a checkpoint that quantizes its linears stores the vocab table dense
scheme, kind = None, QuantKind.NONE
cls = method_class(kind, layer_kind)
cfg = self.layer_config(layer, layer_kind, scheme)
return cls(cfg, get_quant_backend().select(layer_kind, kind))
def layer_config(self, layer: Any, layer_kind: LayerKind, scheme: QuantScheme | None) -> LinearConfig | MoEConfig:
"""The layer's Linear / MoE config; a dialect overrides it to add what only its checkpoint knows."""
return LAYER_CONFIGS[layer_kind].from_layer(layer, scheme)
@staticmethod
def from_hf(
hf_config: Any,
*,
name_map: NameMap | None = None,
unquantized: tuple[str, ...] = (),
) -> "QuantConfig":
"""``unquantized`` lists the modules the family keeps bf16 when the checkpoint's config does not (DeepSeek-V4's compressors).
``hf_config`` comes from ``cached_load_hf_config``, which folds an old ModelOpt ``hf_quant_config.json`` into ``quantization_config``."""
q = quantization_config_of(hf_config)
if q is None:
return NoQuantConfig(name_map, unquantized)
for cls in dialects():
if cls.claims(q):
return cls(q, hf_config, name_map=name_map, unquantized=unquantized)
raise NotImplementedError(f"quantization method {q.get('quant_method')!r} is not supported")
@register_dialect
class NoQuantConfig(QuantConfig):
"""No ``quantization_config``: every module is bf16."""
dialect = "none"
STORAGE: ClassVar[dict[QuantKind, dict[str, str | Stored]]] = {}
@classmethod
def claims(cls, q: dict[str, Any]) -> bool:
return False
def scheme_for_name(self, name: str) -> QuantScheme | None:
return None