FreeToken brings datacenter-scale model serving to your desktop. Run massive models locally, fast and efficiently.
from __future__ import annotations
from typing import Any, ClassVar
from ..names import name_set
from ..registry import register_dialect
from ..scheme import QuantKind, QuantScheme
from ..scheme import mxfp4_scheme
from .base import QuantConfig, Stored
@register_dialect
class Mxfp4Config(QuantConfig):
"""gpt-oss: the stacked ``experts`` containers are MXFP4 with biases, everything else bf16."""
dialect = "mxfp4"
SCHEME: ClassVar[QuantScheme] = mxfp4_scheme()
# the experts are stacked per layer (``gate_up_proj_blocks`` / ``_scales``), not per-Linear tensors; gpt_oss reads them itself
STORAGE: ClassVar[dict[QuantKind, dict[str, str | Stored]]] = {}
def __init__(self, q: dict[str, Any], hf_config: Any = None, *, name_map=None, unquantized=()):
super().__init__(name_map, unquantized)
self.not_convert = name_set(tuple(q.get("modules_to_not_convert") or ()))
def scheme_for_name(self, name: str) -> QuantScheme | None:
if not name.endswith(".experts") or self.not_convert(name):
return None
return self.SCHEME