PTQ-Level Efficiency, QAT-Level Accuracy | 训练后量化的效率,量化感知训练的精度
[bug fix] model free: fix copy folder of diffusion model, fix unresponsive disable_model_free, and cuda UT (#1809)
Xin He2c915bbddafb
diff --git a/README.md b/README.mdindex ed97a394f..4bd63c71f 100644--- a/README.md+++ b/README.md@@ -39,6 +39,8 @@ See our papers [SignRoundV1](https://arxiv.org/pdf/2309.05516) and [SignRoundV2]## 🆕 What's New+* [2026/05] **Model free** quantization is available, `--iters 0 --disable_opt_rtn` will now default to using the model-free approach: [Doc](https://github.com/intel/auto-round/blob/main/docs/step_by_step.md#model-free-mode).+* [2026/03] **Block-wise FP8** quantization is available via `--scheme FP8_BLOCK --iters 0 --disable_opt_rtn`.* [2026/03] **MTP layer quantization** has been supported in this [PR](https://github.com/intel/auto-round/pull/1526)diff --git a/README_CN.md b/README_CN.mdindex a1e3879ac..3b263380f 100644--- a/README_CN.md+++ b/README_CN.md@@ -35,6 +35,8 @@ AutoRound 是专为大语言模型(LLMs)和视觉-语言模型(VLMs)设## 🆕 最新进展+* [2026/05] 现已支持**免模型量化**模式, `--iters 0 --disable_opt_rtn`现在会默认启用免模型方式: [文档](https://github.com/intel/auto-round/blob/main/docs/step_by_step_CN.md#%E5%85%8D%E6%A8%A1%E5%9E%8B%E9%87%8F%E5%8C%96%E6%A8%A1%E5%BC%8F).+* [2026/03] 支持**Block-wise FP8**,只需要将scheme改成 `--scheme FP8_BLOCK`.为了更快的量化,我们推荐使用RTN模式,即·--iters 0 --disable_opt_rtn·* [2026/03] 支持**MTP层量化**diff --git a/auto_round/__main__.py b/auto_round/__main__.pyindex 09385061d..dc22f80ee 100644--- a/auto_round/__main__.py+++ b/auto_round/__main__.py@@ -611,69 +611,6 @@ def tune(args):if "marlin" in args.format and args.asym is True:raise RuntimeError("marlin backend only supports sym quantization, please remove --asym")- # ======================= Model-Free Mode =======================- # The model-free path is now integrated into AutoRound itself. We only- # need to forward the relevant flags; AutoRound handles auto-routing- # (when iters=0 + disable_opt_rtn + supported scheme) and explicit- # ``--model_free``. Layer config / ignore-layers / output-dir handling- # for model-free still needs special treatment because the model is- # never loaded here.- explicit_model_free = bool(getattr(args, "model_free", False))- from auto_round.compressors.model_free import is_model_free_supported_scheme-- auto_model_free = (- not explicit_model_free- and not getattr(args, "disable_model_free", False)- and getattr(args, "iters", None) == 0- and getattr(args, "disable_opt_rtn", None) is True- and is_model_free_supported_scheme(args.scheme, vars(args))- and (- str(getattr(args, "format", "auto_round") or "auto_round").lower().replace(" ", "").split(",")[0]- == "auto_round"- or format.startswith("auto_round")- )- )-- if explicit_model_free or auto_model_free:- scheme = args.scheme.upper()- if scheme not in PRESET_SCHEMES:- raise ValueError(f"{scheme} is not supported. Only {list(PRESET_SCHEMES.keys())} are supported")- if not is_model_free_supported_scheme(scheme, vars(args)) and not explicit_model_free:- logger.info(- f"Auto-routing to model-free is skipped: scheme '{scheme}' is not in "- f"the model-free allowlist. Falling back to the regular AutoRound flow."- )- else:- layer_config = {}- if args.layer_config:- layer_config = parse_layer_config_arg(args.layer_config)-- model_name = args.model.rstrip("/")- output_dir = args.output_dir- if output_dir == "./tmp_autoround" and model_name.split("/")[-1].strip(".") != "":- s = preset_name_to_scheme(scheme)- suffix = f"g{s.group_size}" if s.group_size > 0 else f"a{s.act_bits}"- output_dir = os.path.join(args.output_dir, model_name.split("/")[-1] + f"-w{s.bits}{suffix}")-- from auto_round import AutoRound-- ar = AutoRound(- model_name,- scheme=scheme,- sym=getattr(args, "sym", None),- group_size=getattr(args, "group_size", None),- iters=0,- disable_opt_rtn=True,- model_free=True,- layer_config=layer_config,- ignore_layers=args.ignore_layers,- quant_lm_head=getattr(args, "quant_lm_head", False),- quant_nontext_module=getattr(args, "quant_nontext_module", False),- device_map=args.device_map,- )- ar.quantize_and_save(output_dir=output_dir, format=args.format) # pylint: disable=E1101- return-device_str, use_auto_mapping = get_device_and_parallelism(args.device_map)if args.enable_torch_compile:@@ -827,6 +764,8 @@ def tune(args):momentum=args.momentum,trust_remote_code=not args.disable_trust_remote_code,rotation_config=rot_config,+ model_free=args.model_free,+ disable_model_free=args.disable_model_free,)# ======================= Quantize and save model =======================@@ -834,8 +773,6 @@ def tune(args):# BaseCompressor._get_export_dir(), so we only need to pass the base output_dir.model, folders = autoround.quantize_and_save(args.output_dir, format=args.format) # pylint: disable=E1101tokenizer = autoround.tokenizer # pylint: disable=E1101-- model.eval()clear_memory()# ======================= Model evaluation =======================diff --git a/auto_round/compressors/model_free.py b/auto_round/compressors/model_free.pyindex 6a22e2f4f..3010a5514 100644--- a/auto_round/compressors/model_free.py+++ b/auto_round/compressors/model_free.py@@ -1329,17 +1329,19 @@ class _ModelFreeCompressorCore:returnif self.is_diffusion_model:- # For diffusion models, copy only root-level non-weight metadata files- # (model_index.json, tokenizer files, etc.) to output_dir.- # Sub-components other than transformer (vae, scheduler, …) are skipped- # per the "only quantize transformer" policy; the quantized transformer- # component is already written to output_dir/transformer/ by the pipeline.+ # For diffusion models, copy root-level metadata files and+ # sub-component directories (vae, scheduler, tokenizer, …) to+ # output_dir. The quantized transformer component is already+ # written to output_dir/transformer/ by the pipeline, so+ # copytree's ``not os.path.exists(dst)`` guard prevents+ # overwriting it.for fname in os.listdir(self.diffusion_root_dir):src = os.path.join(self.diffusion_root_dir, fname)dst = os.path.join(self.output_dir, fname)if os.path.isdir(src):- continue- if os.path.isfile(src) and not os.path.exists(dst):+ if not os.path.exists(dst):+ shutil.copytree(src, dst)+ elif os.path.isfile(src) and not os.path.exists(dst):shutil.copy2(src, dst)returndiff --git a/auto_round/eval/evaluation.py b/auto_round/eval/evaluation.pyindex 9aad08223..5933bbc0d 100644--- a/auto_round/eval/evaluation.py+++ b/auto_round/eval/evaluation.py@@ -410,6 +410,9 @@ def run_model_evaluation(model, tokenizer, autoround, folders, formats, device_s"""from auto_round.utils import get_library_version, get_model_dtype, logger+ if model is not None:+ model.eval()+# Handle diffusion models separatelyif getattr(autoround, "diffusion", False):evaluate_diffusion_model(args, autoround=autoround, model=model)diff --git a/test/test_cpu/quantization/test_model_free.py b/test/test_cpu/quantization/test_model_free.pyindex 1cfdd082d..736b6c910 100644--- a/test/test_cpu/quantization/test_model_free.py+++ b/test/test_cpu/quantization/test_model_free.py@@ -408,3 +408,158 @@ class TestPredefinedIgnoreLayers:"shared_head","layers.45",]+++# ===========================================================================+# Copy metadata & subfolder handling+# ===========================================================================+++def _make_diffusion_model_dir(tmp_path, transformer_config, transformer_tensors):+ """Create a minimal diffusion model directory layout.++ Layout::++ root/+ model_index.json+ transformer/+ config.json+ model.safetensors+ vae/+ config.json+ scheduler/+ scheduler_config.json+ tokenizer/+ tokenizer.json+ nested/+ vocab.txt+ """+ root_dir = str(tmp_path / "diffusion_model")+ os.makedirs(root_dir, exist_ok=True)+ with open(os.path.join(root_dir, "model_index.json"), "w") as f:+ json.dump({"_class_name": "FluxPipeline"}, f)++ # transformer component with weights+ transformer_dir = os.path.join(root_dir, "transformer")+ os.makedirs(transformer_dir, exist_ok=True)+ with open(os.path.join(transformer_dir, "config.json"), "w") as f:+ json.dump(transformer_config, f)+ save_file(transformer_tensors, os.path.join(transformer_dir, "model.safetensors"))++ # vae component+ vae_dir = os.path.join(root_dir, "vae")+ os.makedirs(vae_dir, exist_ok=True)+ with open(os.path.join(vae_dir, "config.json"), "w") as f:+ json.dump({"_class_name": "AutoencoderKL"}, f)++ # scheduler component+ sched_dir = os.path.join(root_dir, "scheduler")+ os.makedirs(sched_dir, exist_ok=True)+ with open(os.path.join(sched_dir, "scheduler_config.json"), "w") as f:+ json.dump({"_class_name": "FlowMatchEulerDiscreteScheduler"}, f)++ # tokenizer component with nested subdir+ tok_dir = os.path.join(root_dir, "tokenizer")+ os.makedirs(os.path.join(tok_dir, "nested"), exist_ok=True)+ with open(os.path.join(tok_dir, "tokenizer.json"), "w") as f:+ json.dump({"type": "BPE"}, f)+ with open(os.path.join(tok_dir, "nested", "vocab.txt"), "w") as f:+ f.write("hello\nworld\n")++ return root_dir+++_TRANSFORMER_CONFIG = {+ "architectures": ["FluxTransformer2DModel"],+ "model_type": "flux",+ "hidden_size": 128,+ "num_hidden_layers": 1,+}++_TRANSFORMER_TENSORS = {+ "transformer_blocks.0.attn.to_q.weight": torch.randn(128, 128),+ "transformer_blocks.0.attn.to_k.weight": torch.randn(128, 128),+ "transformer_blocks.0.ff.net.0.proj.weight": torch.randn(512, 128),+ "transformer_blocks.0.ff.net.2.weight": torch.randn(128, 512),+}+++class TestCopyMetadataSubfolders:+ """Tests for _copy_metadata_files including subdirectory handling."""++ def test_non_diffusion_copies_subfolders(self, tmp_path):+ """Non-diffusion model: subdirectories should be copied to output."""+ model_dir = _make_model_dir(tmp_path, _SIMPLE_CONFIG, _SIMPLE_TENSORS)+ # Add a subdirectory with files+ tokenizer_dir = os.path.join(model_dir, "tokenizer")+ os.makedirs(os.path.join(tokenizer_dir, "nested"), exist_ok=True)+ with open(os.path.join(tokenizer_dir, "tokenizer.json"), "w") as f:+ json.dump({"type": "BPE"}, f)+ with open(os.path.join(tokenizer_dir, "nested", "vocab.txt"), "w") as f:+ f.write("hello\nworld\n")++ output_dir = str(tmp_path / "output")+ core = _ModelFreeCompressorCore(+ model_name_or_path=model_dir,+ output_dir=output_dir,+ scheme="W4A16",+ )+ core.run()++ assert os.path.isdir(os.path.join(output_dir, "tokenizer"))+ assert os.path.isfile(os.path.join(output_dir, "tokenizer", "tokenizer.json"))+ assert os.path.isdir(os.path.join(output_dir, "tokenizer", "nested"))+ assert os.path.isfile(os.path.join(output_dir, "tokenizer", "nested", "vocab.txt"))++ def test_diffusion_copies_subfolders(self, tmp_path):+ """Diffusion model: non-transformer subdirectories should be copied."""+ root_dir = _make_diffusion_model_dir(tmp_path, _TRANSFORMER_CONFIG, _TRANSFORMER_TENSORS)+ output_dir = str(tmp_path / "output")++ core = _ModelFreeCompressorCore(+ model_name_or_path=root_dir,+ output_dir=output_dir,+ scheme="W4A16",+ )+ core.run()++ # Non-transformer subdirs must be present+ assert os.path.isdir(os.path.join(output_dir, "vae"))+ assert os.path.isfile(os.path.join(output_dir, "vae", "config.json"))+ assert os.path.isdir(os.path.join(output_dir, "scheduler"))+ assert os.path.isfile(os.path.join(output_dir, "scheduler", "scheduler_config.json"))+ assert os.path.isdir(os.path.join(output_dir, "tokenizer"))+ assert os.path.isfile(os.path.join(output_dir, "tokenizer", "tokenizer.json"))+ assert os.path.isfile(os.path.join(output_dir, "tokenizer", "nested", "vocab.txt"))+ # Root-level file+ assert os.path.isfile(os.path.join(output_dir, "model_index.json"))+ # Quantized transformer must also exist+ assert os.path.isdir(os.path.join(output_dir, "transformer"))+ assert os.path.isfile(os.path.join(output_dir, "transformer", "config.json"))++ def test_diffusion_does_not_overwrite_quantized_transformer(self, tmp_path):+ """Copying subfolders must not overwrite the quantized transformer."""+ root_dir = _make_diffusion_model_dir(tmp_path, _TRANSFORMER_CONFIG, _TRANSFORMER_TENSORS)+ output_dir = str(tmp_path / "output")++ core = _ModelFreeCompressorCore(+ model_name_or_path=root_dir,+ output_dir=output_dir,+ scheme="W4A16",+ )+ core.run()++ # The output transformer/ should contain quantized weights, not+ # the original model.safetensors from the source.+ transformer_out = os.path.join(output_dir, "transformer")+ out_files = os.listdir(transformer_out)+ # The original single shard would have been renamed to model.safetensors+ # by _write_index_file; confirm it has quantized tensor names.+ keys = set()+ for f in out_files:+ if f.endswith(".safetensors"):+ with safe_open(os.path.join(transformer_out, f), framework="pt") as sf:+ keys.update(sf.keys())+ assert any(+ k.endswith(".qweight") for k in keys+ ), f"Quantized transformer should contain .qweight tensors, got: {keys}"diff --git a/test/test_cuda/models/test_get_block_name.py b/test/test_cuda/models/test_get_block_name.pyindex 0933573a0..e16417db1 100644--- a/test/test_cuda/models/test_get_block_name.py+++ b/test/test_cuda/models/test_get_block_name.py@@ -165,15 +165,15 @@ class TestAutoRound:model_name = get_model_path("google/gemma-3-12b-it")tmp_path = "./tmp/tiny_gemma3" # fixture:clean_tmp_model_folder will clean itsave_tiny_model(- model_name, tmp_path, from_config=True+ model_name, tmp_path, from_config=True, num_layers=2) # make sure the model is downloaded before the test, to avoid timeout in the testmodel = Gemma3ForConditionalGeneration.from_pretrained(tmp_path, torch_dtype="auto", trust_remote_code=True)block_names = get_block_names(model)- self.check_block_names(block_names, ["model.language_model.layers"], [48])+ self.check_block_names(block_names, ["model.language_model.layers"], [2])block_names = get_block_names(model, quant_vision=True)self.check_block_names(- block_names, ["model.vision_tower.encoder.layers", "model.language_model.layers"], [27, 48]+ block_names, ["model.vision_tower.encoder.layers", "model.language_model.layers"], [2, 2])assert not is_pure_text_model(model)