Skip to content

PTQ-Level Efficiency, QAT-Level Accuracy | 训练后量化的效率,量化感知训练的精度

feat: add support for WOQ fake quantization in model-free mode and update… (#2329)

Xin Hebdc6047564c1
diff --git a/auto_round/cli/main.py b/auto_round/cli/main.py
index 75c76a278..0e601547f 100644
--- a/auto_round/cli/main.py
+++ b/auto_round/cli/main.py
@@ -236,7 +236,7 @@ def start(recipe="default", argv=None):
parser = build_quantize_parser(prog="auto_round quantize")
args = parser.parse_args(argv)
- args._api_format = args.format if format_was_explicit else None
+ args._api_format = args.format if format_was_explicit or args.model_free else None
# Apply recipe defaults for fields the user didn't set
for key, value in recipe_defaults.items():
diff --git a/auto_round/compressors/model_free.py b/auto_round/compressors/model_free.py
index f54765477..ece7cd102 100644
--- a/auto_round/compressors/model_free.py
+++ b/auto_round/compressors/model_free.py
@@ -26,7 +26,8 @@ Supported schemes
-----------------
Model-free mode supports the following quantization families:
-**Integer weight-only** (packed in ``auto_round:auto_gptq`` format):
+**Integer weight-only** (packed in ``auto_round:auto_gptq`` format, or QDQ
+weights with ``fake`` format):
* Preset names: ``W2A16``, ``W2A16G32``, ``W2A16G64``, ``W4A16``,
``W4A16_MIXED``, ``W8A16``.
@@ -51,7 +52,8 @@ BF16, FPW8A16, ...) are **not** supported in model-free mode and will raise
Output formats
--------------
-* **INT schemes** → ``auto_round:auto_gptq`` packing format, ``quant_method="auto-round"``.
+* **INT schemes** → ``auto_round:auto_gptq`` packing format, ``quant_method="auto-round"``;
+ use ``fake`` for high-precision QDQ weights without quantization metadata.
* **MXFP schemes** → ``mxfp4-pack-quantized`` or ``mxfp8-quantized`` format,
``quant_method="compressed-tensors"``, compatible with vLLM / llm-compressor.
* **NVFP4_E5M3** → AutoRound format with packed ``.weight_packed`` and
@@ -1295,6 +1297,11 @@ class _ModelFreeCompressorCore:
self._remove_stale_quantization_config_files()
_remove_quantization_configs(self.config)
+ if self.format == "fake":
+ with open(os.path.join(self._quant_output_dir, "config.json"), "w") as f:
+ json.dump(self.config, f, indent=2)
+ return
+
self.config["quantization_config"] = quantization_config
with open(os.path.join(self._quant_output_dir, "config.json"), "w") as f:
json.dump(self.config, f, indent=2)
@@ -1413,7 +1420,7 @@ class _ModelFreeCompressorCore:
elif data_type == _NVFP4_E5M3_DATA_TYPE:
packing_format = "fake" if self.format == "fake" else "auto_round:llm_compressor_nvfp4_e5m3"
else:
- packing_format = "auto_round:auto_gptq"
+ packing_format = "fake" if self.format == "fake" else "auto_round:auto_gptq"
if is_mx_fp(data_type) or _layer_config_has_mxfp(self.layer_config):
if not self.disable_opt_rtn:
logger.info(
@@ -1825,10 +1832,7 @@ class ModelFreeCompressor(_ModelFreeCompressorCore):
self._resolve_auto_scheme()
# Accept the standard auto_round formats.
- _accepted_formats = {
- "auto_round",
- "auto_round:auto_gptq",
- }
+ _accepted_formats = {"fake", "auto_round", "auto_round:auto_gptq"}
# MXFP supports both llm_compressor (compressed-tensors) and auto_round formats.
# The only difference is the quantization_config metadata; on-disk weights are identical.
normalized_scheme = (
diff --git a/auto_round/eval/evaluation.py b/auto_round/eval/evaluation.py
index 9145ce7d5..9bc850be2 100644
--- a/auto_round/eval/evaluation.py
+++ b/auto_round/eval/evaluation.py
@@ -516,8 +516,20 @@ def run_model_evaluation(model, tokenizer, autoround, folders, formats, args):
if model is None:
return
else:
- eval_model_dtype = get_model_dtype(args.eval_model_dtype, "auto")
- model = prepare_model_for_eval(model, args.device_map, eval_model_dtype)
+ if model is None:
+ # Model-free mode: load model from the saved output directory
+ from transformers import AutoModelForCausalLM, AutoTokenizer
+
+ eval_model_dtype = get_model_dtype(args.eval_model_dtype, "auto")
+ model = AutoModelForCausalLM.from_pretrained(
+ eval_folder, device_map=args.device_map, torch_dtype=eval_model_dtype
+ )
+ model.eval()
+ if tokenizer is None:
+ tokenizer = AutoTokenizer.from_pretrained(eval_folder)
+ else:
+ eval_model_dtype = get_model_dtype(args.eval_model_dtype, "auto")
+ model = prepare_model_for_eval(model, args.device_map, eval_model_dtype)
# Evaluate with model instance
evaluate_with_model_instance(model, tokenizer, device_str, args)
diff --git a/auto_round/utils/model_free_utils.py b/auto_round/utils/model_free_utils.py
index 6cf82a2da..5a96f9efa 100644
--- a/auto_round/utils/model_free_utils.py
+++ b/auto_round/utils/model_free_utils.py
@@ -1256,6 +1256,29 @@ def _quantize_weight_nvfp4_e5m3(
return {f"{layer_name}.weight": qdq_weight.to(dtype=weight.dtype, device="cpu")}
+def _quantize_weight_int_fake(
+ weight: torch.Tensor,
+ layer_name: str,
+ bits: int,
+ group_size: int,
+ sym: bool,
+ device: str = "cpu",
+ disable_opt_rtn: bool = True,
+) -> dict[str, torch.Tensor]:
+ """Fake-quantize an INT WOQ weight tensor and return its QDQ weight."""
+ from auto_round.data_type.utils import get_quant_func
+
+ original_dtype = weight.dtype
+ original_in_features = weight.shape[1]
+ weight_dev = weight.to(device=device, dtype=torch.float32)
+ if original_in_features % group_size:
+ weight_dev = torch.nn.functional.pad(weight_dev, (0, group_size - original_in_features % group_size))
+
+ quant_func, _ = get_quant_func("int", bits, sym=sym, disable_opt_rtn=disable_opt_rtn, iters=0)
+ qdq_weight, _, _ = quant_func(weight_dev, bits=bits, group_size=group_size)
+ return {f"{layer_name}.weight": qdq_weight[:, :original_in_features].to(dtype=original_dtype, device="cpu")}
+
+
def _pack_weight_nvfp4_e5m3(
weight: torch.Tensor,
layer_name: str,
@@ -1393,6 +1416,18 @@ def _quantize_single_tensor(
# opt_rtn is always disabled for integer WOQ in model-free mode because
# the scale search does not improve accuracy for INT quantization here.
try:
+ if scheme.get("_output_format") == "fake":
+ out = _quantize_weight_int_fake(
+ weight=tensor,
+ layer_name=layer_name,
+ bits=bits,
+ group_size=group_size,
+ sym=sym,
+ device=device,
+ )
+ logger.debug(f"Fake-quantized: {layer_name} (bits={bits}, group_size={group_size}, sym={sym})")
+ return layer_name, out, layer_name, None
+
qweight, qzeros, scales = quantize_func(
weight=tensor,
bits=bits,
diff --git a/test/unit/test_cpu/quantization/test_model_free.py b/test/unit/test_cpu/quantization/test_model_free.py
index 2263e07a6..cf2ec25b5 100644
--- a/test/unit/test_cpu/quantization/test_model_free.py
+++ b/test/unit/test_cpu/quantization/test_model_free.py
@@ -456,6 +456,49 @@ def test_nvfp4_e5m3_model_free_fake_quantization():
assert not is_model_free_supported_scheme("NVFP4+")
+def test_int_model_free_fake_quantization():
+ shard_path = "int-fake-shard.safetensors"
+ weight = torch.randn(8, 32)
+ try:
+ save_file({"layer.fc.weight": weight}, shard_path)
+ output, quantized, _ = _process_shard(
+ shard_path,
+ {**_DEFAULT_SCHEME, "_output_format": "fake"},
+ {},
+ [],
+ )
+ finally:
+ if os.path.exists(shard_path):
+ os.remove(shard_path)
+
+ assert quantized == ["layer.fc"]
+ assert set(output) == {"layer.fc.weight"}
+ assert output["layer.fc.weight"].shape == weight.shape
+ assert output["layer.fc.weight"].dtype == weight.dtype
+ assert not torch.equal(output["layer.fc.weight"], weight)
+
+
+def test_int_model_free_fake_export_has_no_quantization_config(tmp_path):
+ tensors = {"model.layers.0.self_attn.q_proj.weight": torch.randn(32, 32)}
+ model_dir = _make_model_dir(tmp_path, _LLAMA_CFG, tensors)
+ output_dir = str(tmp_path / "output")
+ os.makedirs(output_dir)
+ with open(os.path.join(output_dir, "quantization_config.json"), "w") as f:
+ json.dump({"stale": True}, f)
+
+ compressor = _ModelFreeCompressorCore(
+ model_name_or_path=model_dir, output_dir=output_dir, scheme="W4A16", format="fake"
+ )
+ compressor.run()
+
+ output_keys = _read_output_keys(output_dir)
+ assert "model.layers.0.self_attn.q_proj.weight" in output_keys
+ assert "model.layers.0.self_attn.q_proj.qweight" not in output_keys
+ assert not os.path.exists(os.path.join(output_dir, "quantization_config.json"))
+ with open(os.path.join(output_dir, "config.json")) as f:
+ assert "quantization_config" not in json.load(f)
+
+
def test_nvfp4_e5m3_model_free_end_to_end(tmp_path):
tensors = {
"model.layers.0.self_attn.q_proj.weight": torch.randn(32, 32),
@@ -1093,6 +1136,16 @@ class TestSchemeValidation:
class TestCliAutoRouting:
+ def test_model_free_uses_auto_round_format_by_default(self, monkeypatch):
+ from auto_round.cli import main as cli_main
+
+ captured = {}
+ monkeypatch.setattr(cli_main, "tune", lambda args: captured.setdefault("format", args._api_format))
+
+ cli_main.start(argv=["--model", "dummy", "--model_free"])
+
+ assert captured["format"] == "auto_round"
+
def test_auto_routes(self, tmp_path):
model_dir = _make_model_dir(tmp_path, _LLAMA_CFG, {"layer.weight": torch.randn(64, 128)})
out_dir = str(tmp_path / "out")