Skip to content

FreeToken brings datacenter-scale model serving to your desktop. Run massive models locally, fast and efficiently.

4.1 KB
Raw Blame
from __future__ import annotations

from typing import Any, ClassVar

from ..names import ancestors, name_set
from ..registry import register_dialect
from ..scheme import QuantKind, QuantScheme
from ..scheme import fp8_block_scheme, fp8_tensor_scheme, mxfp8_scheme, nvfp4_scheme
from .base import QuantConfig, Stored


@register_dialect
class ModelOptConfig(QuantConfig):
    """NVIDIA ModelOpt exports: one ``quant_algo`` for every Linear minus ``ignore``, or
    ``MIXED_PRECISION`` with a per-module ``quantized_layers`` allow-list."""

    dialect = "modelopt"

    SCHEMES: ClassVar[dict[str, QuantScheme]] = {
        "NVFP4": nvfp4_scheme(input_scale=True),
        "W4A16_NVFP4": nvfp4_scheme(input_scale=False),
        "FP8": fp8_tensor_scheme("fp32", input_scale=True),
        "FP8_PER_CHANNEL_PER_TOKEN": fp8_tensor_scheme("fp32", per_row=True),
        "FP8_PB_WO": fp8_block_scheme("fp32"),
        "MXFP8": mxfp8_scheme(),
    }
    STORAGE: ClassVar[dict[QuantKind, dict[str, str | Stored]]] = {
        QuantKind.FP8_TENSOR: {"weight": "weight", "weight_scale": "weight_scale", "input_scale": "input_scale"},
        QuantKind.FP8_BLOCK: {"weight": "weight", "weight_scale_inv": "weight_scale_inv"},
        # same e8m0 codes under either name: MiniMax-M3 exports weight_scale_inv, Qwen3.8-Flash-Next weight_scale
        QuantKind.MXFP8: {"weight": "weight", "weight_scale_inv": Stored("weight_scale_inv", aliases=("weight_scale",))},
        QuantKind.NVFP4: {"weight": "weight", "weight_scale": "weight_scale", "weight_global": "weight_scale_2", "input_scale": "input_scale"},
    }

    @classmethod
    def claims(cls, q: dict[str, Any]) -> bool:
        method = str(q.get("quant_method") or "").lower()
        return method == "modelopt" or (not method and bool(q.get("quant_algo")))

    def __init__(self, q: dict[str, Any], hf_config: Any = None, *, name_map=None, unquantized=()):
        super().__init__(name_map, unquantized)
        self.algo = str(q.get("quant_algo") or "").upper()
        # Inferact exports carry this key; false says the FP8 and NVFP4 layers store no input_scale, and an explicit value wins over the config_groups rule below
        self.with_input_scale = q.get("with_input_scale")
        groups = q.get("config_groups")
        # an export with no activation quantizer can still say NVFP4; every config group then has input_activations null (vLLM applies the same rule)
        if self.with_input_scale is None and self.algo == "NVFP4" and isinstance(groups, dict) and groups and all(isinstance(g, dict) and g.get("input_activations") is None for g in groups.values()):
            self.algo = "W4A16_NVFP4"
        self.ignore = name_set(tuple(q.get("ignore") or q.get("exclude_modules") or ()))
        layers = q.get("quantized_layers") or {}
        self.quantized_layers = {k: str((v or {}).get("quant_algo") or "").upper() for k, v in layers.items()} if isinstance(layers, dict) else {}
        if self.algo == "MIXED_PRECISION" and not self.quantized_layers:
            raise NotImplementedError("ModelOpt MIXED_PRECISION without quantized_layers in quantization_config")
        if self.algo != "MIXED_PRECISION":
            self._scheme_of(self.algo)

    def scheme_for_name(self, name: str) -> QuantScheme | None:
        if self.ignore(name):
            return None
        algo = self._module_algo(name)
        return None if algo is None else self._scheme_of(algo)

    def _module_algo(self, name: str) -> str | None:
        if self.algo != "MIXED_PRECISION":
            return self.algo
        for a in ancestors(name):
            algo = self.quantized_layers.get(a)
            if algo is not None:
                return algo
        return None

    def _scheme_of(self, algo: str) -> QuantScheme:
        if self.with_input_scale is False:
            if algo == "FP8":
                return fp8_tensor_scheme("fp32")  # ModelOpt has no algo name for fp8 without an activation scale
            if algo == "NVFP4":
                algo = "W4A16_NVFP4"
        try:
            return self.SCHEMES[algo]
        except KeyError:
            raise NotImplementedError(f"ModelOpt quant_algo {algo!r} is not supported") from None