PTQ-Level Efficiency, QAT-Level Accuracy | 训练后量化的效率,量化感知训练的精度
feat: add support for WOQ fake quantization in model-free mode and update… (#2329)
Xin Hebdc6047564c1
diff --git a/auto_round/cli/main.py b/auto_round/cli/main.pyindex 75c76a278..0e601547f 100644--- a/auto_round/cli/main.py+++ b/auto_round/cli/main.py@@ -236,7 +236,7 @@ def start(recipe="default", argv=None):parser = build_quantize_parser(prog="auto_round quantize")args = parser.parse_args(argv)- args._api_format = args.format if format_was_explicit else None+ args._api_format = args.format if format_was_explicit or args.model_free else None# Apply recipe defaults for fields the user didn't setfor key, value in recipe_defaults.items():diff --git a/auto_round/compressors/model_free.py b/auto_round/compressors/model_free.pyindex f54765477..ece7cd102 100644--- a/auto_round/compressors/model_free.py+++ b/auto_round/compressors/model_free.py@@ -26,7 +26,8 @@ Supported schemes-----------------Model-free mode supports the following quantization families:-**Integer weight-only** (packed in ``auto_round:auto_gptq`` format):+**Integer weight-only** (packed in ``auto_round:auto_gptq`` format, or QDQ+weights with ``fake`` format):* Preset names: ``W2A16``, ``W2A16G32``, ``W2A16G64``, ``W4A16``,``W4A16_MIXED``, ``W8A16``.@@ -51,7 +52,8 @@ BF16, FPW8A16, ...) are **not** supported in model-free mode and will raiseOutput formats---------------* **INT schemes** → ``auto_round:auto_gptq`` packing format, ``quant_method="auto-round"``.+* **INT schemes** → ``auto_round:auto_gptq`` packing format, ``quant_method="auto-round"``;+ use ``fake`` for high-precision QDQ weights without quantization metadata.* **MXFP schemes** → ``mxfp4-pack-quantized`` or ``mxfp8-quantized`` format,``quant_method="compressed-tensors"``, compatible with vLLM / llm-compressor.* **NVFP4_E5M3** → AutoRound format with packed ``.weight_packed`` and@@ -1295,6 +1297,11 @@ class _ModelFreeCompressorCore:self._remove_stale_quantization_config_files()_remove_quantization_configs(self.config)+ if self.format == "fake":+ with open(os.path.join(self._quant_output_dir, "config.json"), "w") as f:+ json.dump(self.config, f, indent=2)+ return+self.config["quantization_config"] = quantization_configwith open(os.path.join(self._quant_output_dir, "config.json"), "w") as f:json.dump(self.config, f, indent=2)@@ -1413,7 +1420,7 @@ class _ModelFreeCompressorCore:elif data_type == _NVFP4_E5M3_DATA_TYPE:packing_format = "fake" if self.format == "fake" else "auto_round:llm_compressor_nvfp4_e5m3"else:- packing_format = "auto_round:auto_gptq"+ packing_format = "fake" if self.format == "fake" else "auto_round:auto_gptq"if is_mx_fp(data_type) or _layer_config_has_mxfp(self.layer_config):if not self.disable_opt_rtn:logger.info(@@ -1825,10 +1832,7 @@ class ModelFreeCompressor(_ModelFreeCompressorCore):self._resolve_auto_scheme()# Accept the standard auto_round formats.- _accepted_formats = {- "auto_round",- "auto_round:auto_gptq",- }+ _accepted_formats = {"fake", "auto_round", "auto_round:auto_gptq"}# MXFP supports both llm_compressor (compressed-tensors) and auto_round formats.# The only difference is the quantization_config metadata; on-disk weights are identical.normalized_scheme = (diff --git a/auto_round/eval/evaluation.py b/auto_round/eval/evaluation.pyindex 9145ce7d5..9bc850be2 100644--- a/auto_round/eval/evaluation.py+++ b/auto_round/eval/evaluation.py@@ -516,8 +516,20 @@ def run_model_evaluation(model, tokenizer, autoround, folders, formats, args):if model is None:returnelse:- eval_model_dtype = get_model_dtype(args.eval_model_dtype, "auto")- model = prepare_model_for_eval(model, args.device_map, eval_model_dtype)+ if model is None:+ # Model-free mode: load model from the saved output directory+ from transformers import AutoModelForCausalLM, AutoTokenizer++ eval_model_dtype = get_model_dtype(args.eval_model_dtype, "auto")+ model = AutoModelForCausalLM.from_pretrained(+ eval_folder, device_map=args.device_map, torch_dtype=eval_model_dtype+ )+ model.eval()+ if tokenizer is None:+ tokenizer = AutoTokenizer.from_pretrained(eval_folder)+ else:+ eval_model_dtype = get_model_dtype(args.eval_model_dtype, "auto")+ model = prepare_model_for_eval(model, args.device_map, eval_model_dtype)# Evaluate with model instanceevaluate_with_model_instance(model, tokenizer, device_str, args)diff --git a/auto_round/utils/model_free_utils.py b/auto_round/utils/model_free_utils.pyindex 6cf82a2da..5a96f9efa 100644--- a/auto_round/utils/model_free_utils.py+++ b/auto_round/utils/model_free_utils.py@@ -1256,6 +1256,29 @@ def _quantize_weight_nvfp4_e5m3(return {f"{layer_name}.weight": qdq_weight.to(dtype=weight.dtype, device="cpu")}+def _quantize_weight_int_fake(+ weight: torch.Tensor,+ layer_name: str,+ bits: int,+ group_size: int,+ sym: bool,+ device: str = "cpu",+ disable_opt_rtn: bool = True,+) -> dict[str, torch.Tensor]:+ """Fake-quantize an INT WOQ weight tensor and return its QDQ weight."""+ from auto_round.data_type.utils import get_quant_func++ original_dtype = weight.dtype+ original_in_features = weight.shape[1]+ weight_dev = weight.to(device=device, dtype=torch.float32)+ if original_in_features % group_size:+ weight_dev = torch.nn.functional.pad(weight_dev, (0, group_size - original_in_features % group_size))++ quant_func, _ = get_quant_func("int", bits, sym=sym, disable_opt_rtn=disable_opt_rtn, iters=0)+ qdq_weight, _, _ = quant_func(weight_dev, bits=bits, group_size=group_size)+ return {f"{layer_name}.weight": qdq_weight[:, :original_in_features].to(dtype=original_dtype, device="cpu")}++def _pack_weight_nvfp4_e5m3(weight: torch.Tensor,layer_name: str,@@ -1393,6 +1416,18 @@ def _quantize_single_tensor(# opt_rtn is always disabled for integer WOQ in model-free mode because# the scale search does not improve accuracy for INT quantization here.try:+ if scheme.get("_output_format") == "fake":+ out = _quantize_weight_int_fake(+ weight=tensor,+ layer_name=layer_name,+ bits=bits,+ group_size=group_size,+ sym=sym,+ device=device,+ )+ logger.debug(f"Fake-quantized: {layer_name} (bits={bits}, group_size={group_size}, sym={sym})")+ return layer_name, out, layer_name, None+qweight, qzeros, scales = quantize_func(weight=tensor,bits=bits,diff --git a/test/unit/test_cpu/quantization/test_model_free.py b/test/unit/test_cpu/quantization/test_model_free.pyindex 2263e07a6..cf2ec25b5 100644--- a/test/unit/test_cpu/quantization/test_model_free.py+++ b/test/unit/test_cpu/quantization/test_model_free.py@@ -456,6 +456,49 @@ def test_nvfp4_e5m3_model_free_fake_quantization():assert not is_model_free_supported_scheme("NVFP4+")+def test_int_model_free_fake_quantization():+ shard_path = "int-fake-shard.safetensors"+ weight = torch.randn(8, 32)+ try:+ save_file({"layer.fc.weight": weight}, shard_path)+ output, quantized, _ = _process_shard(+ shard_path,+ {**_DEFAULT_SCHEME, "_output_format": "fake"},+ {},+ [],+ )+ finally:+ if os.path.exists(shard_path):+ os.remove(shard_path)++ assert quantized == ["layer.fc"]+ assert set(output) == {"layer.fc.weight"}+ assert output["layer.fc.weight"].shape == weight.shape+ assert output["layer.fc.weight"].dtype == weight.dtype+ assert not torch.equal(output["layer.fc.weight"], weight)+++def test_int_model_free_fake_export_has_no_quantization_config(tmp_path):+ tensors = {"model.layers.0.self_attn.q_proj.weight": torch.randn(32, 32)}+ model_dir = _make_model_dir(tmp_path, _LLAMA_CFG, tensors)+ output_dir = str(tmp_path / "output")+ os.makedirs(output_dir)+ with open(os.path.join(output_dir, "quantization_config.json"), "w") as f:+ json.dump({"stale": True}, f)++ compressor = _ModelFreeCompressorCore(+ model_name_or_path=model_dir, output_dir=output_dir, scheme="W4A16", format="fake"+ )+ compressor.run()++ output_keys = _read_output_keys(output_dir)+ assert "model.layers.0.self_attn.q_proj.weight" in output_keys+ assert "model.layers.0.self_attn.q_proj.qweight" not in output_keys+ assert not os.path.exists(os.path.join(output_dir, "quantization_config.json"))+ with open(os.path.join(output_dir, "config.json")) as f:+ assert "quantization_config" not in json.load(f)++def test_nvfp4_e5m3_model_free_end_to_end(tmp_path):tensors = {"model.layers.0.self_attn.q_proj.weight": torch.randn(32, 32),@@ -1093,6 +1136,16 @@ class TestSchemeValidation:class TestCliAutoRouting:+ def test_model_free_uses_auto_round_format_by_default(self, monkeypatch):+ from auto_round.cli import main as cli_main++ captured = {}+ monkeypatch.setattr(cli_main, "tune", lambda args: captured.setdefault("format", args._api_format))++ cli_main.start(argv=["--model", "dummy", "--model_free"])++ assert captured["format"] == "auto_round"+def test_auto_routes(self, tmp_path):model_dir = _make_model_dir(tmp_path, _LLAMA_CFG, {"layer.weight": torch.randn(64, 128)})out_dir = str(tmp_path / "out")