Skip to content

PTQ-Level Efficiency, QAT-Level Accuracy | 训练后量化的效率,量化感知训练的精度

feat: add MXFP4/MXFP8 quantization support (llmc_compressor format) and related tests (#1865)

Xin Hedc5216c8a2f5
diff --git a/auto_round/compressors/model_free.py b/auto_round/compressors/model_free.py
index 9f18f5255..8717ab17c 100644
--- a/auto_round/compressors/model_free.py
+++ b/auto_round/compressors/model_free.py
@@ -24,39 +24,67 @@ The main entry point is the :class:`ModelFreeCompressor` class.
Supported schemes
-----------------
-Model-free mode currently supports **integer weight-only** quantization
-schemes packed in the ``auto_round:auto_gptq`` format only. Specifically:
+Model-free mode supports the following quantization families:
-* Preset names: ``W2A16``, ``W2A16G32``, ``W2A16G64``, ``W3A16``, ``W4A16``,
- ``W8A16``.
+**Integer weight-only** (packed in ``auto_round:auto_gptq`` format):
+
+* Preset names: ``W2A16``, ``W2A16G32``, ``W2A16G64``, ``W4A16``,
+ ``W4A16_MIXED``, ``W8A16``.
* Custom :class:`~auto_round.schemes.QuantizationScheme` instances with
- ``data_type="int"``, ``bits in {2, 3, 4, 8}``, ``act_bits >= 16``, and any
+ ``data_type="int"``, ``bits in {2, 4, 8}``, ``act_bits >= 16``, and any
symmetric / asymmetric configuration.
-Schemes that require special packing (FP8, MXFP4, NVFP4, GGUF, INT8_W8A8,
+**MXFP (Microscaling Floating Point)** (packed in ``mxfp4-pack-quantized`` or
+``mxfp8-quantized`` format, compatible with llm-compressor / compressed-tensors):
+
+* Preset names: ``MXFP4``, ``MXFP8``.
+* ``data_type="mx_fp"``, ``group_size=32``, ``bits in {4, 8}``.
+
+Schemes that require special packing (FP8, NVFP4, GGUF, INT8_W8A8,
BF16, FPW8A16, ...) are **not** supported in model-free mode and will raise
``ValueError``. Use the standard AutoRound flow for those.
+Output formats
+--------------
+* **INT schemes** → ``auto_round:auto_gptq`` packing format, ``quant_method="auto-round"``.
+* **MXFP schemes** → ``mxfp4-pack-quantized`` or ``mxfp8-quantized`` format,
+ ``quant_method="compressed-tensors"``, compatible with vLLM / llm-compressor.
+
Usage (CLI)
-----------
::
+ # Integer WOQ
auto_round facebook/opt-125m \\
--model_free \\
--scheme W4A16 \\
--output_dir int4-125m
+ # MXFP4
+ auto_round facebook/opt-125m \\
+ --model_free \\
+ --scheme MXFP4 \\
+ --output_dir mxfp4-125m
+
Usage (API)
-----------
::
from auto_round import AutoRound
+ # Integer WOQ
AutoRound(
model="facebook/opt-125m",
scheme="W4A16",
model_free=True,
).quantize_and_save("./int4-125m")
+
+ # MXFP4
+ AutoRound(
+ model="facebook/opt-125m",
+ scheme="MXFP4",
+ model_free=True,
+ ).quantize_and_save("./mxfp4-125m")
"""
from __future__ import annotations
@@ -74,6 +102,7 @@ from typing import Optional, Union
import torch
from auto_round import envs
+from auto_round.compressors.utils import is_mx_fp
from auto_round.logger import logger
from auto_round.schemes import PRESET_SCHEMES, QuantizationScheme, preset_name_to_scheme
from auto_round.utils.common import AUDIO_MM_KEYS, VISION_MM_KEYS, compress_layer_names, to_standard_regex
@@ -87,9 +116,9 @@ from auto_round.utils.missing_tensors import quantize_weight_rtn, split_fused_ex
# add "embed", "conv" in case of auto detection failure in _check_conv1d_and_embedding
_BLOCK_NAME_TO_IGNORE = ["shared_expert_gate.", ".gate.", "embed", "conv"]
-# Integer WOQ preset schemes that model-free mode can produce.
-# Other presets (FP8/MX/NV/GGUF/BF16/INT8_W8A8/FPW8A16) require different
-# packing kernels not implemented by ``quantize_weight_rtn``.
+# Preset schemes that model-free mode can produce.
+# INT presets use ``auto_round:auto_gptq`` packing; MXFP presets use
+# ``mxfp4-pack-quantized`` or ``mxfp8-quantized`` (compressed-tensors) packing.
#
# Note: ``W3A16`` (3-bit) is intentionally excluded. 3-bit packing requires
# in_features to be padded to a multiple of pack_factor=10, which the current
@@ -101,12 +130,17 @@ SUPPORTED_PRESET_SCHEMES: tuple[str, ...] = (
"W4A16",
"W4A16_MIXED",
"W8A16",
+ "MXFP4",
+ "MXFP8",
)
# Allowed ``bits`` values for integer WOQ.
# 3-bit is excluded — see note above.
_SUPPORTED_INT_BITS: tuple[int, ...] = (2, 4, 8)
+# Allowed ``bits`` values for MXFP weight quantization.
+_SUPPORTED_MXFP_BITS: tuple[int, ...] = (4, 8)
+
# Multimodal keywords kept in full precision by default.
_NONTEXT_KEYWORDS: tuple[str, ...] = VISION_MM_KEYS + AUDIO_MM_KEYS
@@ -449,6 +483,77 @@ class _PatternMatcher:
# ---------------------------------------------------------------------------
+def _quantize_weight_mxfp(
+ weight: torch.Tensor,
+ layer_name: str,
+ bits: int,
+ group_size: int,
+ data_type: str,
+ device: str = "cpu",
+) -> dict[str, torch.Tensor]:
+ """Quantize a 2D weight tensor to MXFP4 / MXFP8 and return packed outputs.
+
+ Reuses :func:`auto_round.data_type.mxfp.quant_mx` to derive the per-block
+ shared exponent (E8M0 scale), and :class:`auto_round.export.export_to_autoround.qlinear_fp.QuantLinear`
+ to perform the same packing as :func:`auto_round.export.export_to_llmcompressor.export_to_fp.pack_layer`.
+
+ Returns a dict with one of:
+ * MXFP8: ``{layer_name+'.weight': float8_e4m3fn, layer_name+'.weight_scale': uint8}``
+ * MXFP4: ``{layer_name+'.weight_packed': uint8, layer_name+'.weight_scale': uint8}``
+ """
+ import torch.nn as nn
+
+ from auto_round.data_type.mxfp import quant_mx
+ from auto_round.export.export_to_autoround.qlinear_fp import QuantLinear
+
+ if not is_mx_fp(data_type):
+ data_type = "mx_fp"
+
+ out_features, in_features = weight.shape
+ if in_features % group_size != 0:
+ raise ValueError(
+ f"in_features={in_features} for layer '{layer_name}' is not divisible "
+ f"by MXFP group_size={group_size}; cannot pack."
+ )
+
+ weight_dev = weight.to(device)
+ # quant_mx returns (qdq_tensor, shared_exp, None). We only need shared_exp
+ # (the per-block log2 scale). The element-wise rounding to the FP4/FP8 grid
+ # is performed inside QuantLinear.pack via dtype casts / pack_fp4_to_uint8.
+ _, shared_exp, _ = quant_mx(weight_dev, bits=bits, group_size=group_size, data_type=data_type)
+ # Reshape to (out_features, n_groups) so the on-disk weight_scale matches
+ # the llm-compressor convention (and QuantLinear's registered buffer shape).
+ shared_exp = shared_exp.reshape(out_features, in_features // group_size)
+
+ # Build a lightweight nn.Linear holding the original weight so we can
+ # delegate packing to the existing QuantLinear.pack implementation.
+ fake_linear = nn.Linear(in_features, out_features, bias=False)
+ with torch.no_grad():
+ fake_linear.weight = nn.Parameter(weight_dev, requires_grad=False)
+
+ qlayer = QuantLinear(
+ bits=bits,
+ group_size=group_size,
+ infeatures=in_features,
+ outfeatures=out_features,
+ bias=False,
+ data_type="mx_fp4" if bits == 4 else "mx_fp8e4m3",
+ sym=True,
+ act_bits=bits,
+ )
+ qlayer.pack(fake_linear, shared_exp, device=device)
+
+ if bits == 8:
+ return {
+ f"{layer_name}.weight": qlayer.weight.to("cpu"),
+ f"{layer_name}.weight_scale": qlayer.weight_scale.to("cpu"),
+ }
+ return {
+ f"{layer_name}.weight_packed": qlayer.weight_packed.to("cpu"),
+ f"{layer_name}.weight_scale": qlayer.weight_scale.to("cpu"),
+ }
+
+
def _quantize_single_tensor(
tensor_name: str,
tensor: torch.Tensor,
@@ -481,10 +586,29 @@ def _quantize_single_tensor(
bits = scheme["bits"]
group_size = scheme["group_size"]
sym = scheme.get("sym", True)
+ data_type = (scheme.get("data_type") or "int").lower()
if bits >= 16:
return layer_name, {tensor_name: tensor}, None, layer_name
+ # ---- MXFP path (MXFP4 / MXFP8) ----
+ if is_mx_fp(data_type):
+ try:
+ out = _quantize_weight_mxfp(
+ weight=tensor,
+ layer_name=layer_name,
+ bits=bits,
+ group_size=group_size,
+ data_type=data_type,
+ device=device,
+ )
+ logger.debug(f"Quantized (MXFP): {layer_name} (bits={bits}, group_size={group_size})")
+ return layer_name, out, layer_name, None
+ except Exception as e:
+ logger.warning(f"Failed to MXFP-quantize {layer_name}: {e}. Keeping original weight.")
+ return layer_name, {tensor_name: tensor}, None, layer_name
+
+ # ---- Integer WOQ path ----
try:
qweight, qzeros, scales = quantize_weight_rtn(
weight=tensor,
@@ -686,7 +810,28 @@ def _process_shard(
raw_tensors = {name: f.get_tensor(name) for name in f.keys()}
raw_tensors = split_fused_expert_tensors(raw_tensors)
+
+ # Preserve original tensors for ignored/skipped layers so that already-
+ # quantized weights (FP8, FP4-packed, etc.) are NOT dequantized.
+ # Check both ".weight" and ".weight_packed" so that layers whose primary
+ # tensor uses non-standard naming (e.g. already-quantized FP4-packed layers
+ # stored as ".weight_packed") are correctly captured.
+ preserved_prefixes: set[str] = set()
+ for tname in raw_tensors:
+ if (tname.endswith(".weight") or tname.endswith(".weight_packed") or tname.endswith(".qweight")) and (
+ matcher.should_ignore(tname) or matcher.should_skip(tname)
+ ):
+ preserved_prefixes.add(tname.rsplit(".", 1)[0])
+
+ preserved_tensors: dict[str, torch.Tensor] = {}
+ if preserved_prefixes:
+ for key in list(raw_tensors.keys()):
+ prefix = key.rsplit(".", 1)[0]
+ if prefix in preserved_prefixes:
+ preserved_tensors[key] = raw_tensors.pop(key)
+
raw_tensors = _dequant_fp8_tensors(raw_tensors, block_size=fp8_block_size)
+ raw_tensors.update(preserved_tensors)
for tensor_name in list(raw_tensors.keys()):
tensor = raw_tensors.pop(tensor_name)
@@ -710,6 +855,44 @@ def _process_shard(
# ---------------------------------------------------------------------------
+def _build_mxfp_quantization_config(
+ default_scheme: dict,
+ quantized_layers: list[str],
+ ignored_layers: list[str],
+) -> dict:
+ """Build a compressed-tensors / llm-compressor style quantization_config
+ dict for MXFP4 / MXFP8 model-free output.
+
+ Mirrors the per-group format produced by
+ :mod:`auto_round.export.export_to_llmcompressor.export_to_fp`.
+ """
+ from auto_round.export.export_to_llmcompressor.config import (
+ check_compressed_tensors_supported,
+ initialize_quantization,
+ )
+
+ check_compressed_tensors_supported(raise_error=True)
+
+ bits = default_scheme["bits"]
+ if bits not in _SUPPORTED_MXFP_BITS:
+ raise ValueError(f"Unsupported MXFP bits={bits} for model-free output.")
+
+ scheme_name = "MXFP4" if bits == 4 else "MXFP8"
+ fmt = "mxfp4-pack-quantized" if bits == 4 else "mxfp8-quantized"
+
+ # Default ignore list: any layer present in ignored_layers (deduped) that
+ # was NOT quantized.
+ ignore = list(dict.fromkeys(ignored_layers))
+ quant_set = set(quantized_layers)
+ ignore = [n for n in ignore if n not in quant_set]
+
+ qconfig = initialize_quantization(scheme=scheme_name, ignore=ignore)
+ qconfig = qconfig.to_dict()
+ qconfig["format"] = fmt
+ qconfig["provider"] = "auto-round"
+ return qconfig
+
+
def _build_quantization_config(
default_scheme: dict,
layer_config: dict,
@@ -719,6 +902,14 @@ def _build_quantization_config(
block_name_to_quantize: Optional[list[str]] = None,
) -> dict:
"""Build a quantization_config dict compatible with auto-round format."""
+ # MXFP (mx_fp) uses the llm-compressor / compressed-tensors style config.
+ if is_mx_fp((default_scheme.get("data_type") or "int").lower()):
+ return _build_mxfp_quantization_config(
+ default_scheme=default_scheme,
+ quantized_layers=quantized_layers,
+ ignored_layers=ignored_layers,
+ )
+
from auto_round.version import __version__
scheme_keys = [f.name for f in fields(QuantizationScheme)]
@@ -880,10 +1071,46 @@ def _validate_supported_scheme(
) -> None:
"""Raise ``ValueError`` if *scheme_obj* is not supported by model-free.
- Model-free only supports integer weight-only quantization (sym/asym),
- packed in the ``auto_round:auto_gptq`` format.
+ Model-free supports:
+
+ * Integer weight-only quantization (sym/asym), ``bits ∈ {2, 4, 8}``,
+ packed in the ``auto_round:auto_gptq`` format.
+ * MXFP weight quantization (``data_type='mx_fp'``), ``bits ∈ {4, 8}``,
+ ``group_size=32``, packed in ``mxfp4-pack-quantized`` / ``mxfp8-quantized``
+ format (compressed-tensors compatible).
"""
+ data_type = (scheme_obj.data_type or "int").lower()
+ bits = scheme_obj.bits
act_bits = scheme_obj.act_bits if scheme_obj.act_bits is not None else 16
+
+ # MXFP weight-only path: accept mx_fp data type with bits in {4, 8}.
+ # Activation quantization for MXFP is dynamic at inference time, so the
+ # weight-only RTN path here is independent of act_bits.
+ if is_mx_fp(data_type):
+ # Restrict to the two explicitly supported MXFP presets when a string
+ # name is provided. Variants such as MXFP4_RCEIL / MXFP8_RCEIL use a
+ # different activation format; silently mapping them to "MXFP4" /
+ # "MXFP8" in the output config would misrepresent the requested scheme.
+ if isinstance(scheme_input, str) and scheme_input not in ("MXFP4", "MXFP8"):
+ raise ValueError(
+ f"Model-free mode only supports MXFP preset names 'MXFP4' and 'MXFP8', "
+ f"but got '{scheme_input}'. "
+ f"Supported preset schemes: {list(SUPPORTED_PRESET_SCHEMES)}."
+ )
+ if bits is None or bits not in _SUPPORTED_MXFP_BITS:
+ raise ValueError(
+ f"Model-free mode supports MXFP bits in {_SUPPORTED_MXFP_BITS}, "
+ f"but '{scheme_input}' requests bits={bits}. "
+ f"Supported preset schemes: {list(SUPPORTED_PRESET_SCHEMES)}."
+ )
+ group_size = scheme_obj.group_size
+ if group_size not in (None, 32):
+ raise ValueError(
+ f"Model-free mode supports MXFP only with group_size=32, "
+ f"but '{scheme_input}' requests group_size={group_size}."
+ )
+ return
+
if act_bits < 16:
raise ValueError(
f"Model-free mode only supports weight-only quantization (WOQ) schemes "
@@ -891,17 +1118,15 @@ def _validate_supported_scheme(
f"Supported preset schemes: {list(SUPPORTED_PRESET_SCHEMES)}."
)
- data_type = (scheme_obj.data_type or "int").lower()
if data_type != "int":
raise ValueError(
f"Model-free mode only supports integer weight quantization "
- f"(data_type='int'), but '{scheme_input}' has data_type='{data_type}'. "
- f"FP8 / MXFP / NVFP / GGUF / BF16 schemes require the standard "
- f"AutoRound flow. Supported preset schemes: "
+ f"(data_type='int') or MXFP (data_type='mx_fp'), but '{scheme_input}' "
+ f"has data_type='{data_type}'. FP8 / NVFP / GGUF / BF16 schemes require "
+ f"the standard AutoRound flow. Supported preset schemes: "
f"{list(SUPPORTED_PRESET_SCHEMES)}."
)
- bits = scheme_obj.bits
if bits is None or bits not in _SUPPORTED_INT_BITS:
raise ValueError(
f"Model-free mode supports bits in {_SUPPORTED_INT_BITS}, "
@@ -920,7 +1145,7 @@ def is_model_free_supported_scheme(
"""
try:
scheme_obj = _apply_scheme_overrides(scheme, scheme_overrides)
- _validate_supported_scheme(scheme_obj, scheme_obj)
+ _validate_supported_scheme(scheme_obj, scheme)
return True
except (ValueError, TypeError):
return False
@@ -945,14 +1170,19 @@ class _ModelFreeCompressorCore:
Args:
model_name_or_path: HuggingFace model ID or local directory path.
output_dir: Directory to save the quantized model.
- scheme: Quantization scheme name (e.g. ``"W4A16"``) or a
- :class:`QuantizationScheme` instance.
+ scheme: Quantization scheme name (e.g. ``"W4A16"``, ``"MXFP4"``,
+ ``"MXFP8"``) or a :class:`QuantizationScheme` instance.
layer_config: Per-layer quantization overrides. Keys are layer
names or regex patterns; values are dicts with ``bits``,
``group_size``, ``sym`` etc.
ignore_layers: Comma-separated list of layer name patterns to keep
- in full precision.
- format: Output format (only ``"auto_round"`` is supported).
+ in full precision. Ignored layers that are already quantized
+ (e.g. FP8) are preserved in their original format.
+ format: Output format. Supported: ``"auto_round"``,
+ ``"auto_round:auto_gptq"``, ``"llm_compressor"``,
+ ``"auto_round:llm_compressor"``. The packing format is
+ auto-selected based on the scheme (INT→auto_gptq,
+ MXFP→compressed-tensors).
device: Device for quantization computation (``"cpu"`` or
``"cuda"``).
quant_lm_head: If True, quantize ``lm_head`` as well. By default
@@ -963,7 +1193,12 @@ class _ModelFreeCompressorCore:
modules are kept in full precision.
"""
- SUPPORTED_FORMATS: tuple[str, ...] = ("auto_round",)
+ SUPPORTED_FORMATS: tuple[str, ...] = (
+ "auto_round",
+ "auto_round:auto_gptq",
+ "llm_compressor",
+ "auto_round:llm_compressor",
+ )
def __init__(
self,
@@ -1388,9 +1623,18 @@ class _ModelFreeCompressorCore:
self._detect_fp8_source()
self._discover_shards()
+ # Determine the output packing format based on scheme data type
+ data_type = (self.default_scheme.get("data_type") or "int").lower()
+ if is_mx_fp(data_type):
+ bits = self.default_scheme.get("bits", 4)
+ packing_format = "mxfp4-pack-quantized" if bits == 4 else "mxfp8-quantized"
+ else:
+ packing_format = "auto_round:auto_gptq"
+
logger.info(
f"Model-free quantization: {self.model_name_or_path}\n"
f" Scheme: {self.scheme_obj}\n"
+ f" Packing format: {packing_format}\n"
f" Output: {self.output_dir}\n"
f" Shards: {len(self.shard_names)}\n"
f" Streaming download: {self.is_streaming}\n"
@@ -1610,7 +1854,19 @@ class ModelFreeCompressor(_ModelFreeCompressorCore):
**kwargs,
):
"""Quantize and save — AutoRound compressor entry point."""
- if format not in ["auto_round", "auto_round:auto_gptq"]:
+ # Accept the standard auto_round formats.
+ _accepted_formats = {
+ "auto_round",
+ "auto_round:auto_gptq",
+ }
+ # MXFP only supports the llm_compressor format.
+ if self.scheme_input in ["MXFP4", "MXFP8"]:
+ _accepted_formats = ["llm_compressor"]
+ if format not in _accepted_formats:
+ logger.warning(
+ f"Format '{format}' is not supported by model-free mode for scheme '{self.scheme_input}'; "
+ f"falling back to the regular AutoRound flow."
+ )
return self._fallback_to_quantize_and_save(output_dir=output_dir, format=format, inplace=inplace, **kwargs)
# Apply user scheme overrides before running
diff --git a/docs/step_by_step.md b/docs/step_by_step.md
index ad3b33922..cea28ea1d 100644
--- a/docs/step_by_step.md
+++ b/docs/step_by_step.md
@@ -525,7 +525,7 @@ ar.quantize_and_save(output_dir, format="auto_round")
Model-free mode performs RTN WOQ quantization **without loading the full model into memory**. It downloads safetensors files directly, quantizes each Linear weight tensor shard-by-shard, and saves the packed result. This is useful when you want fast, no-calibration quantization with minimal resource requirements.
-> **Auto-enabled by default.** As of v0.13, when you pass `--iters 0 --disable_opt_rtn` together with a supported INT WOQ scheme, the CLI automatically takes the model-free path. This is **bit-exactly equivalent** to the regular `--iters 0 --disable_opt_rtn` flow but uses far less memory. Use `--disable_model_free` to opt out and force the original flow.
+> **Auto-enabled by default.** As of v0.13, when you pass `--iters 0 --disable_opt_rtn` together with a supported INT WOQ or MXFP scheme, the CLI automatically takes the model-free path. This is **bit-exactly equivalent** to the regular `--iters 0 --disable_opt_rtn` flow but uses far less memory. Use `--disable_model_free` to opt out and force the original flow.
**Key features:**
- **No model object required** – only `config.json` and safetensors files are needed
@@ -539,7 +539,9 @@ Model-free mode performs RTN WOQ quantization **without loading the full model i
**Supported schemes**
-Model-free mode currently supports the following **integer weight-only** preset schemes (packed in the `auto_round:auto_gptq` format):
+Model-free mode supports the following preset schemes:
+
+**Integer weight-only** (packed in `auto_round:auto_gptq` format):
| Preset | Bits | Group size | Sym |
| --- | --- | --- | --- |
@@ -554,7 +556,14 @@ All of the above presets also support **asymmetric quantization** (`sym=False`)
You can also pass a custom `QuantizationScheme(bits=N, group_size=G, sym=True/False, data_type="int", act_bits=16)` with `bits ∈ {2, 4, 8}` and any group_size / sym configuration.
-Schemes that require special packing kernels (`W3A16`, `FPW8A16`, `BF16`, `MXFP4`, `MXFP8`, `MXINT4`, `NVFP4`, `FP8_BLOCK`, `FP8_STATIC`, `INT8_W8A8`, `GGUF:*`, ...) are **not** supported in model-free mode and will raise `ValueError`. Use the regular AutoRound flow for those.
+**MXFP (Microscaling Floating Point)** (packed in `mxfp4-pack-quantized` / `mxfp8-quantized` format, compatible with compressed-tensors / vLLM):
+
+| Preset | Bits | Group size | Format |
+| --- | --- | --- | --- |
+| `MXFP4` | 4 | 32 | mxfp4-pack-quantized |
+| `MXFP8` | 8 | 32 | mxfp8-quantized |
+
+Schemes that require special packing kernels (`W3A16`, `FPW8A16`, `BF16`, `MXINT4`, `NVFP4`, `FP8_BLOCK`, `FP8_STATIC`, `INT8_W8A8`, `GGUF:*`, ...) are **not** supported in model-free mode and will raise `ValueError`. Use the regular AutoRound flow for those.
#### CLI Usage
@@ -586,6 +595,18 @@ auto_round meta-llama/Llama-3.2-1B-Instruct \
--layer_config "{k_proj:{bits:8},v_proj:{bits:8}}" \
--ignore_layers "mlp" \
--output_dir ./int4-llama
+
+# MXFP4 quantization
+auto_round meta-llama/Llama-3.2-1B-Instruct \
+ --model_free \
+ --scheme MXFP4 \
+ --output_dir ./mxfp4-llama
+
+# MXFP8 quantization
+auto_round meta-llama/Llama-3.2-1B-Instruct \
+ --model_free \
+ --scheme MXFP8 \
+ --output_dir ./mxfp8-llama
```
#### API Usage
@@ -605,7 +626,7 @@ AutoRound(
).quantize_and_save("./int4-llama")
```
-> **Note:** Model-free mode only supports the `auto_round` output format and uses RTN (no calibration data, no iterative tuning). For higher-quality quantization or schemes outside the supported list, use the standard AutoRound flow.
+> **Note:** Model-free mode uses RTN (no calibration data, no iterative tuning). INT schemes output in `auto_round:auto_gptq` format; MXFP schemes output in compressed-tensors format (`mxfp4-pack-quantized` / `mxfp8-quantized`). For higher-quality quantization or schemes outside the supported list, use the standard AutoRound flow.
</details>
diff --git a/docs/step_by_step_CN.md b/docs/step_by_step_CN.md
index 8677fe185..7da4e43a7 100644
--- a/docs/step_by_step_CN.md
+++ b/docs/step_by_step_CN.md
@@ -522,7 +522,7 @@ ar.quantize_and_save(output_dir, format="auto_round")
免模型架构量化模式(Model-Free Mode)可以**无需将完整模型加载到内存中**即可执行 RTN WOQ 量化。它直接下载 safetensors 文件,逐分片地对每个 Linear 权重张量进行量化并保存打包结果。当您需要快速、无标定数据的量化且资源有限时,该模式非常实用。
-> **默认自动启用。** 自 v0.13 起,当您同时传入 `--iters 0 --disable_opt_rtn` 与一个受支持的 INT WOQ scheme 时,CLI 会自动走免模型路径。该路径与原始 `--iters 0 --disable_opt_rtn` 流程**位级(bit-exact)等价**,但内存占用大幅降低。如需关闭自动路由、强制使用原始流程,可加 `--disable_model_free`。
+> **默认自动启用。** 自 v0.13 起,当您同时传入 `--iters 0 --disable_opt_rtn` 与一个受支持的 INT WOQ 或 MXFP scheme 时,CLI 会自动走免模型路径。该路径与原始 `--iters 0 --disable_opt_rtn` 流程**位级(bit-exact)等价**,但内存占用大幅降低。如需关闭自动路由、强制使用原始流程,可加 `--disable_model_free`。
**主要特性:**
- **无需模型对象** — 仅需 `config.json` 和 safetensors 文件
@@ -536,7 +536,9 @@ ar.quantize_and_save(output_dir, format="auto_round")
**支持的 Scheme**
-免模型模式当前支持以下整数权重量化预设(均使用 `auto_round:auto_gptq` 打包格式):
+免模型模式支持以下量化预设:
+
+**整数权重量化**(使用 `auto_round:auto_gptq` 打包格式):
| Preset | Bits | Group size | Sym |
| --- | --- | --- | --- |
@@ -551,7 +553,14 @@ ar.quantize_and_save(output_dir, format="auto_round")
也可以传入自定义的 `QuantizationScheme(bits=N, group_size=G, sym=True/False, data_type="int", act_bits=16)`,其中 `bits ∈ {2, 4, 8}`,group_size / sym 可任意设置。
-需要特殊打包内核的 scheme(`W3A16`、`FPW8A16`、`BF16`、`MXFP4`、`MXFP8`、`MXINT4`、`NVFP4`、`FP8_BLOCK`、`FP8_STATIC`、`INT8_W8A8`、`GGUF:*` 等)**不被支持**,传入会抛 `ValueError`。这些请使用标准 AutoRound 流程。
+**MXFP(微缩放浮点)**(使用 `mxfp4-pack-quantized` / `mxfp8-quantized` 格式,兼容 compressed-tensors / vLLM):
+
+| Preset | Bits | Group size | 格式 |
+| --- | --- | --- | --- |
+| `MXFP4` | 4 | 32 | mxfp4-pack-quantized |
+| `MXFP8` | 8 | 32 | mxfp8-quantized |
+
+需要特殊打包内核的 scheme(`W3A16`、`FPW8A16`、`BF16`、`MXINT4`、`NVFP4`、`FP8_BLOCK`、`FP8_STATIC`、`INT8_W8A8`、`GGUF:*` 等)**不被支持**,传入会抛 `ValueError`。这些请使用标准 AutoRound 流程。
#### 命令行用法
@@ -583,6 +592,18 @@ auto_round meta-llama/Llama-3.2-1B-Instruct \
--layer_config "{k_proj:{bits:8},v_proj:{bits:8}}" \
--ignore_layers "mlp" \
--output_dir ./int4-llama
+
+# MXFP4 量化
+auto_round meta-llama/Llama-3.2-1B-Instruct \
+ --model_free \
+ --scheme MXFP4 \
+ --output_dir ./mxfp4-llama
+
+# MXFP8 量化
+auto_round meta-llama/Llama-3.2-1B-Instruct \
+ --model_free \
+ --scheme MXFP8 \
+ --output_dir ./mxfp8-llama
```
#### API 用法
@@ -602,7 +623,7 @@ AutoRound(
).quantize_and_save("./int4-llama")
```
-> **注意:** 免模型量化模式仅支持 `auto_round` 输出格式,并使用 RTN(无标定数据、无迭代调优)。如需更高质量的量化结果或使用受支持列表外的 scheme,请使用标准 AutoRound 流程。
+> **注意:** 免模型量化模式使用 RTN(无标定数据、无迭代调优)。INT scheme 输出为 `auto_round:auto_gptq` 格式;MXFP scheme 输出为 compressed-tensors 格式(`mxfp4-pack-quantized` / `mxfp8-quantized`)。如需更高质量的量化结果或使用受支持列表外的 scheme,请使用标准 AutoRound 流程。
</details>
diff --git a/test/test_cpu/models/test_moe_model.py b/test/test_cpu/models/test_moe_model.py
index 9c25b1d81..1d25ba719 100644
--- a/test/test_cpu/models/test_moe_model.py
+++ b/test/test_cpu/models/test_moe_model.py
@@ -90,8 +90,7 @@ def test_llama4(tiny_llama4_model_path):
shutil.rmtree(output_dir, ignore_errors=True)
-def test_qwen3_vl_moe_mxfp(tiny_qwen3_vl_moe_model_path):
- output_dir = "/tmp/test_quantized_qwen3_vl_moe"
+def test_qwen3_vl_moe_mxfp(tiny_qwen3_vl_moe_model_path, tmp_path):
autoround = AutoRound(
tiny_qwen3_vl_moe_model_path,
scheme="MXFP4",
@@ -101,14 +100,13 @@ def test_qwen3_vl_moe_mxfp(tiny_qwen3_vl_moe_model_path):
disable_opt_rtn=True,
ignore_layers="self_attn,lm_head, mlp.gate",
)
- quantized_model, quantized_model_path = autoround.quantize_and_save(format="auto_round", output_dir=output_dir)
+ quantized_model, quantized_model_path = autoround.quantize_and_save(format="auto_round", output_dir=tmp_path)
assert quantized_model is not None, "Quantized model should not be None."
loaded_model = Qwen3VLMoeForConditionalGeneration.from_pretrained(quantized_model_path, device_map="cpu")
inp = torch.randint(0, 100, (1, 32))
with torch.inference_mode():
loaded_out = loaded_model(inp)
- shutil.rmtree(output_dir, ignore_errors=True)
class _FakeMoELinear(nn.Module):
diff --git a/test/test_cpu/quantization/test_model_free.py b/test/test_cpu/quantization/test_model_free.py
index 736b6c910..daa82c617 100644
--- a/test/test_cpu/quantization/test_model_free.py
+++ b/test/test_cpu/quantization/test_model_free.py
@@ -28,11 +28,14 @@ from auto_round.compressors.model_free import (
_ModelFreeCompressorCore,
_PatternMatcher,
_process_shard,
+ _quantize_weight_mxfp,
get_predefined_ignore_layers_from_config,
is_model_free_supported_scheme,
)
from auto_round.schemes import QuantizationScheme
+from ...envs import require_compressed_tensors
+
# ---------------------------------------------------------------------------
# Helpers
# ---------------------------------------------------------------------------
@@ -254,6 +257,25 @@ class TestFP8Source:
output, quantized, _ = _process_shard(shard_path, _DEFAULT_SCHEME, {}, [], device="cpu", fp8_block_size=None)
assert "layer" in quantized and "layer.qweight" in output
+ def test_ignored_layer_preserves_original_fp8(self, tmp_path):
+ """Ignored layers keep their original quantized tensors (no dequant)."""
+ shard_path = str(tmp_path / "shard.safetensors")
+ w_fp8 = torch.randn(64, 128, dtype=torch.bfloat16).to(torch.float8_e4m3fn)
+ scale = torch.tensor(0.5)
+ save_file(
+ {"lm_head.weight": w_fp8, "lm_head.weight_scale_inv": scale, "layer.weight": torch.randn(64, 128)},
+ shard_path,
+ )
+ output, quantized, ignored = _process_shard(
+ shard_path, _DEFAULT_SCHEME, {}, ["lm_head"], device="cpu", fp8_block_size=None
+ )
+ # lm_head should be ignored and kept in original FP8 format
+ assert "lm_head" in ignored
+ assert output["lm_head.weight"].dtype == torch.float8_e4m3fn
+ assert "lm_head.weight_scale_inv" in output
+ # non-ignored layer should be quantized normally
+ assert "layer" in quantized
+
# ===========================================================================
# End-to-end ModelFreeQuantize
@@ -305,22 +327,114 @@ class TestModelFreeQuantize:
assert qc["sym"] is False and qc["group_size"] == 64
+# ===========================================================================
+# MXFP4 / MXFP8 model-free quantization
+# ===========================================================================
+
+
+class TestModelFreeMXFP:
+ """End-to-end tests for MXFP4/MXFP8 model-free quantization."""
+
+ def test_quantize_weight_mxfp4_shapes(self):
+ w = torch.randn(64, 128, dtype=torch.bfloat16)
+ out = _quantize_weight_mxfp(w, "layer", bits=4, group_size=32, data_type="mx_fp")
+ assert out["layer.weight_packed"].shape == (64, 64) # in_features / 2
+ assert out["layer.weight_packed"].dtype == torch.uint8
+ assert out["layer.weight_scale"].shape == (64, 4) # in_features / group_size
+ assert out["layer.weight_scale"].dtype == torch.uint8
+
+ def test_quantize_weight_mxfp8_shapes(self):
+ w = torch.randn(64, 128, dtype=torch.bfloat16)
+ out = _quantize_weight_mxfp(w, "layer", bits=8, group_size=32, data_type="mx_fp")
+ assert out["layer.weight"].shape == (64, 128)
+ assert out["layer.weight"].dtype == torch.float8_e4m3fn
+ assert out["layer.weight_scale"].shape == (64, 4)
+ assert out["layer.weight_scale"].dtype == torch.uint8
+
+ @require_compressed_tensors
+ @pytest.mark.parametrize("scheme,fmt", [("MXFP4", "mxfp4-pack-quantized"), ("MXFP8", "mxfp8-quantized")])
+ def test_e2e_mxfp(self, tmp_path, scheme, fmt):
+ tensors = {
+ "model.layers.0.self_attn.q_proj.weight": torch.randn(128, 128),
+ "model.layers.0.fc1.weight": torch.randn(512, 128),
+ "lm_head.weight": torch.randn(1000, 128),
+ }
+ model_dir = _make_model_dir(tmp_path, _LLAMA_CFG, tensors)
+ output_dir = str(tmp_path / "output")
+ _ModelFreeCompressorCore(model_name_or_path=model_dir, output_dir=output_dir, scheme=scheme).run()
+ qc = _read_qconfig(output_dir)
+ assert qc["format"] == fmt
+ assert qc["quant_method"] == "compressed-tensors"
+ assert "lm_head" in qc["ignore"]
+ keys = _read_output_keys(output_dir)
+ # MXFP4 produces weight_packed, MXFP8 produces weight
+ if scheme == "MXFP4":
+ assert "model.layers.0.fc1.weight_packed" in keys
+ else:
+ assert "model.layers.0.fc1.weight" in keys
+ assert "model.layers.0.fc1.weight_scale" in keys
+ # lm_head stays full precision
+ assert "lm_head.weight" in keys
+ assert "lm_head.weight_packed" not in keys
+
+ @require_compressed_tensors
+ def test_mxfp4_via_autoround_api(self, tmp_path):
+ tensors = {"model.layers.0.fc1.weight": torch.randn(128, 128)}
+ model_dir = _make_model_dir(tmp_path, _LLAMA_CFG, tensors)
+ output_dir = str(tmp_path / "output")
+ AutoRound(model=model_dir, scheme="MXFP4", model_free=True).quantize_and_save(
+ output_dir, format="llm_compressor"
+ )
+ qc = _read_qconfig(output_dir)
+ assert qc["format"] == "mxfp4-pack-quantized"
+
+ @require_compressed_tensors
+ def test_process_shard_mxfp(self, tmp_path):
+ shard_path = str(tmp_path / "shard.safetensors")
+ save_file({"layer.fc1.weight": torch.randn(64, 128)}, shard_path)
+ scheme = {"bits": 4, "group_size": 32, "sym": True, "data_type": "mx_fp"}
+ output, quantized, _ = _process_shard(shard_path, scheme, {}, [])
+ assert "layer.fc1" in quantized
+ assert "layer.fc1.weight_packed" in output
+ assert "layer.fc1.weight_scale" in output
+
+
# ===========================================================================
# Scheme validation
# ===========================================================================
-_SUPPORTED = ["W2A16", "W2A16G32", "W2A16G64", "W4A16", "W4A16_MIXED", "W8A16"]
-_UNSUPPORTED = ["W3A16", "FPW8A16", "BF16", "MXFP4", "MXFP8", "MXINT4", "NVFP4", "FP8_BLOCK", "FP8_STATIC", "INT8_W8A8"]
+_SUPPORTED = ["W2A16", "W2A16G32", "W2A16G64", "W4A16", "W4A16_MIXED", "W8A16", "MXFP4", "MXFP8"]
+_UNSUPPORTED = [
+ "W3A16",
+ "FPW8A16",
+ "BF16",
+ "MXINT4",
+ "NVFP4",
+ "FP8_BLOCK",
+ "FP8_STATIC",
+ "INT8_W8A8",
+ "MXFP4_RCEIL",
+ "MXFP8_RCEIL",
+]
class TestSchemeValidation:
@pytest.mark.parametrize("name", _SUPPORTED)
def test_supported(self, tmp_path, name):
+ if name.startswith("MXFP"):
+ pytest.importorskip("compressed_tensors", reason="test requires compressed-tensors")
+ format = "llm_compressor"
+ else:
+ format = "auto_round"
model_dir = _make_model_dir(tmp_path, _LLAMA_CFG, {"model.layers.0.mlp.fc1.weight": torch.randn(64, 128)})
out = str(tmp_path / f"out_{name}")
- AutoRound(model=model_dir, scheme=name, model_free=True).quantize_and_save(out)
- assert "model.layers.0.mlp.fc1.qweight" in _read_output_keys(out)
+ AutoRound(model=model_dir, scheme=name, model_free=True).quantize_and_save(out, format=format)
+ keys = _read_output_keys(out)
+ if name.startswith("MXFP"):
+ assert "model.layers.0.mlp.fc1.weight_scale" in keys
+ else:
+ assert "model.layers.0.mlp.fc1.qweight" in keys
@pytest.mark.parametrize("name", _UNSUPPORTED)
def test_unsupported_raises(self, tmp_path, name):