FreeToken brings datacenter-scale model serving to your desktop. Run massive models locally, fast and efficiently.
from __future__ import annotations
from typing import Any, ClassVar
from ..names import Matcher, ct_set
from ..registry import register_dialect
from ..scheme import QuantKind, QuantScheme
from ..scheme import fp8_block_scheme, fp8_tensor_scheme, nvfp4_scheme
from .base import QuantConfig, Stored
@register_dialect
class CompressedTensorsConfig(QuantConfig):
"""llm-compressor exports: ordered ``config_groups`` with ``targets`` (class names, names or ``re:``) minus ``ignore``."""
dialect = "compressed-tensors"
SCHEMES: ClassVar[dict[str, QuantScheme]] = {
"NVFP4": nvfp4_scheme(input_scale=False),
"NVFP4_LOCAL": nvfp4_scheme(input_scale=True),
"FP8_TENSOR_STATIC": fp8_tensor_scheme("bf16", input_scale=True),
"FP8_TENSOR_DYNAMIC": fp8_tensor_scheme("bf16"),
"FP8_CHANNEL": fp8_tensor_scheme("bf16", per_row=True),
"FP8_BLOCK": fp8_block_scheme("float"),
}
# the NVFP4 globals are the quant-side scales (vLLM: alpha = 1 / (input_global_scale * weight_global_scale))
STORAGE: ClassVar[dict[QuantKind, dict[str, str | Stored]]] = {
QuantKind.NVFP4: {
"weight": "weight_packed", "weight_scale": "weight_scale",
"weight_global": Stored("weight_global_scale", reciprocal=True), "input_scale": Stored("input_global_scale", reciprocal=True),
},
QuantKind.FP8_TENSOR: {"weight": "weight", "weight_scale": "weight_scale", "input_scale": "input_scale"},
QuantKind.FP8_BLOCK: {"weight": "weight", "weight_scale_inv": "weight_scale"},
}
def __init__(self, q: dict[str, Any], hf_config: Any = None, *, name_map=None, unquantized=()):
super().__init__(name_map, unquantized)
self.ignore = ct_set(tuple(q.get("ignore") or ()), class_names=False)
groups = q.get("config_groups") or {}
self.groups: list[tuple[Matcher, QuantScheme]] = [
(ct_set(tuple(spec.get("targets") or ()), class_names=True), self._scheme_of(spec)) for spec in groups.values()
]
def scheme_for_name(self, name: str) -> QuantScheme | None:
if self.ignore(name):
return None
for targets, scheme in self.groups:
if targets(name):
return scheme
return None
@classmethod
def _scheme_of(cls, spec: dict[str, Any]) -> QuantScheme:
w = spec.get("weights") or {}
act = spec.get("input_activations")
bits = int(w.get("num_bits") or 0)
wtype = str(w.get("type") or "").lower()
strategy = str(w.get("strategy") or "").lower()
group = int(w.get("group_size") or 0)
if bits == 4 and wtype == "float":
if strategy == "tensor_group" and group == 16:
return cls.SCHEMES["NVFP4_LOCAL" if act and act.get("dynamic") == "local" else "NVFP4"]
raise NotImplementedError(f"compressed-tensors 4-bit float scheme (strategy={strategy!r}, group_size={group}) is not supported; only NVFP4 is")
if bits == 8 and wtype == "float":
if strategy == "tensor":
return cls.SCHEMES["FP8_TENSOR_DYNAMIC" if act is None or act.get("dynamic") else "FP8_TENSOR_STATIC"]
if strategy == "channel":
return cls.SCHEMES["FP8_CHANNEL"]
if strategy == "block":
block = tuple(int(x) for x in (w.get("block_structure") or ()))
if block == (128, 128):
return cls.SCHEMES["FP8_BLOCK"]
raise NotImplementedError(f"compressed-tensors weight scheme {w} is not supported")