Skip to content

PTQ-Level Efficiency, QAT-Level Accuracy | 训练后量化的效率,量化感知训练的精度

[bug fix] model free: fix copy folder of diffusion model, fix unresponsive disable_model_free, and cuda UT (#1809)

Xin He2c915bbddafb
diff --git a/README.md b/README.md
index ed97a394f..4bd63c71f 100644
--- a/README.md
+++ b/README.md
@@ -39,6 +39,8 @@ See our papers [SignRoundV1](https://arxiv.org/pdf/2309.05516) and [SignRoundV2]
## 🆕 What's New
+* [2026/05] **Model free** quantization is available, `--iters 0 --disable_opt_rtn` will now default to using the model-free approach: [Doc](https://github.com/intel/auto-round/blob/main/docs/step_by_step.md#model-free-mode).
+
* [2026/03] **Block-wise FP8** quantization is available via `--scheme FP8_BLOCK --iters 0 --disable_opt_rtn`.
* [2026/03] **MTP layer quantization** has been supported in this [PR](https://github.com/intel/auto-round/pull/1526)
diff --git a/README_CN.md b/README_CN.md
index a1e3879ac..3b263380f 100644
--- a/README_CN.md
+++ b/README_CN.md
@@ -35,6 +35,8 @@ AutoRound 是专为大语言模型(LLMs)和视觉-语言模型(VLMs)设
## 🆕 最新进展
+* [2026/05] 现已支持**免模型量化**模式, `--iters 0 --disable_opt_rtn`现在会默认启用免模型方式: [文档](https://github.com/intel/auto-round/blob/main/docs/step_by_step_CN.md#%E5%85%8D%E6%A8%A1%E5%9E%8B%E9%87%8F%E5%8C%96%E6%A8%A1%E5%BC%8F).
+
* [2026/03] 支持**Block-wise FP8**,只需要将scheme改成 `--scheme FP8_BLOCK`.为了更快的量化,我们推荐使用RTN模式,即·--iters 0 --disable_opt_rtn·
* [2026/03] 支持**MTP层量化**
diff --git a/auto_round/__main__.py b/auto_round/__main__.py
index 09385061d..dc22f80ee 100644
--- a/auto_round/__main__.py
+++ b/auto_round/__main__.py
@@ -611,69 +611,6 @@ def tune(args):
if "marlin" in args.format and args.asym is True:
raise RuntimeError("marlin backend only supports sym quantization, please remove --asym")
- # ======================= Model-Free Mode =======================
- # The model-free path is now integrated into AutoRound itself. We only
- # need to forward the relevant flags; AutoRound handles auto-routing
- # (when iters=0 + disable_opt_rtn + supported scheme) and explicit
- # ``--model_free``. Layer config / ignore-layers / output-dir handling
- # for model-free still needs special treatment because the model is
- # never loaded here.
- explicit_model_free = bool(getattr(args, "model_free", False))
- from auto_round.compressors.model_free import is_model_free_supported_scheme
-
- auto_model_free = (
- not explicit_model_free
- and not getattr(args, "disable_model_free", False)
- and getattr(args, "iters", None) == 0
- and getattr(args, "disable_opt_rtn", None) is True
- and is_model_free_supported_scheme(args.scheme, vars(args))
- and (
- str(getattr(args, "format", "auto_round") or "auto_round").lower().replace(" ", "").split(",")[0]
- == "auto_round"
- or format.startswith("auto_round")
- )
- )
-
- if explicit_model_free or auto_model_free:
- scheme = args.scheme.upper()
- if scheme not in PRESET_SCHEMES:
- raise ValueError(f"{scheme} is not supported. Only {list(PRESET_SCHEMES.keys())} are supported")
- if not is_model_free_supported_scheme(scheme, vars(args)) and not explicit_model_free:
- logger.info(
- f"Auto-routing to model-free is skipped: scheme '{scheme}' is not in "
- f"the model-free allowlist. Falling back to the regular AutoRound flow."
- )
- else:
- layer_config = {}
- if args.layer_config:
- layer_config = parse_layer_config_arg(args.layer_config)
-
- model_name = args.model.rstrip("/")
- output_dir = args.output_dir
- if output_dir == "./tmp_autoround" and model_name.split("/")[-1].strip(".") != "":
- s = preset_name_to_scheme(scheme)
- suffix = f"g{s.group_size}" if s.group_size > 0 else f"a{s.act_bits}"
- output_dir = os.path.join(args.output_dir, model_name.split("/")[-1] + f"-w{s.bits}{suffix}")
-
- from auto_round import AutoRound
-
- ar = AutoRound(
- model_name,
- scheme=scheme,
- sym=getattr(args, "sym", None),
- group_size=getattr(args, "group_size", None),
- iters=0,
- disable_opt_rtn=True,
- model_free=True,
- layer_config=layer_config,
- ignore_layers=args.ignore_layers,
- quant_lm_head=getattr(args, "quant_lm_head", False),
- quant_nontext_module=getattr(args, "quant_nontext_module", False),
- device_map=args.device_map,
- )
- ar.quantize_and_save(output_dir=output_dir, format=args.format) # pylint: disable=E1101
- return
-
device_str, use_auto_mapping = get_device_and_parallelism(args.device_map)
if args.enable_torch_compile:
@@ -827,6 +764,8 @@ def tune(args):
momentum=args.momentum,
trust_remote_code=not args.disable_trust_remote_code,
rotation_config=rot_config,
+ model_free=args.model_free,
+ disable_model_free=args.disable_model_free,
)
# ======================= Quantize and save model =======================
@@ -834,8 +773,6 @@ def tune(args):
# BaseCompressor._get_export_dir(), so we only need to pass the base output_dir.
model, folders = autoround.quantize_and_save(args.output_dir, format=args.format) # pylint: disable=E1101
tokenizer = autoround.tokenizer # pylint: disable=E1101
-
- model.eval()
clear_memory()
# ======================= Model evaluation =======================
diff --git a/auto_round/compressors/model_free.py b/auto_round/compressors/model_free.py
index 6a22e2f4f..3010a5514 100644
--- a/auto_round/compressors/model_free.py
+++ b/auto_round/compressors/model_free.py
@@ -1329,17 +1329,19 @@ class _ModelFreeCompressorCore:
return
if self.is_diffusion_model:
- # For diffusion models, copy only root-level non-weight metadata files
- # (model_index.json, tokenizer files, etc.) to output_dir.
- # Sub-components other than transformer (vae, scheduler, …) are skipped
- # per the "only quantize transformer" policy; the quantized transformer
- # component is already written to output_dir/transformer/ by the pipeline.
+ # For diffusion models, copy root-level metadata files and
+ # sub-component directories (vae, scheduler, tokenizer, …) to
+ # output_dir. The quantized transformer component is already
+ # written to output_dir/transformer/ by the pipeline, so
+ # copytree's ``not os.path.exists(dst)`` guard prevents
+ # overwriting it.
for fname in os.listdir(self.diffusion_root_dir):
src = os.path.join(self.diffusion_root_dir, fname)
dst = os.path.join(self.output_dir, fname)
if os.path.isdir(src):
- continue
- if os.path.isfile(src) and not os.path.exists(dst):
+ if not os.path.exists(dst):
+ shutil.copytree(src, dst)
+ elif os.path.isfile(src) and not os.path.exists(dst):
shutil.copy2(src, dst)
return
diff --git a/auto_round/eval/evaluation.py b/auto_round/eval/evaluation.py
index 9aad08223..5933bbc0d 100644
--- a/auto_round/eval/evaluation.py
+++ b/auto_round/eval/evaluation.py
@@ -410,6 +410,9 @@ def run_model_evaluation(model, tokenizer, autoround, folders, formats, device_s
"""
from auto_round.utils import get_library_version, get_model_dtype, logger
+ if model is not None:
+ model.eval()
+
# Handle diffusion models separately
if getattr(autoround, "diffusion", False):
evaluate_diffusion_model(args, autoround=autoround, model=model)
diff --git a/test/test_cpu/quantization/test_model_free.py b/test/test_cpu/quantization/test_model_free.py
index 1cfdd082d..736b6c910 100644
--- a/test/test_cpu/quantization/test_model_free.py
+++ b/test/test_cpu/quantization/test_model_free.py
@@ -408,3 +408,158 @@ class TestPredefinedIgnoreLayers:
"shared_head",
"layers.45",
]
+
+
+# ===========================================================================
+# Copy metadata & subfolder handling
+# ===========================================================================
+
+
+def _make_diffusion_model_dir(tmp_path, transformer_config, transformer_tensors):
+ """Create a minimal diffusion model directory layout.
+
+ Layout::
+
+ root/
+ model_index.json
+ transformer/
+ config.json
+ model.safetensors
+ vae/
+ config.json
+ scheduler/
+ scheduler_config.json
+ tokenizer/
+ tokenizer.json
+ nested/
+ vocab.txt
+ """
+ root_dir = str(tmp_path / "diffusion_model")
+ os.makedirs(root_dir, exist_ok=True)
+ with open(os.path.join(root_dir, "model_index.json"), "w") as f:
+ json.dump({"_class_name": "FluxPipeline"}, f)
+
+ # transformer component with weights
+ transformer_dir = os.path.join(root_dir, "transformer")
+ os.makedirs(transformer_dir, exist_ok=True)
+ with open(os.path.join(transformer_dir, "config.json"), "w") as f:
+ json.dump(transformer_config, f)
+ save_file(transformer_tensors, os.path.join(transformer_dir, "model.safetensors"))
+
+ # vae component
+ vae_dir = os.path.join(root_dir, "vae")
+ os.makedirs(vae_dir, exist_ok=True)
+ with open(os.path.join(vae_dir, "config.json"), "w") as f:
+ json.dump({"_class_name": "AutoencoderKL"}, f)
+
+ # scheduler component
+ sched_dir = os.path.join(root_dir, "scheduler")
+ os.makedirs(sched_dir, exist_ok=True)
+ with open(os.path.join(sched_dir, "scheduler_config.json"), "w") as f:
+ json.dump({"_class_name": "FlowMatchEulerDiscreteScheduler"}, f)
+
+ # tokenizer component with nested subdir
+ tok_dir = os.path.join(root_dir, "tokenizer")
+ os.makedirs(os.path.join(tok_dir, "nested"), exist_ok=True)
+ with open(os.path.join(tok_dir, "tokenizer.json"), "w") as f:
+ json.dump({"type": "BPE"}, f)
+ with open(os.path.join(tok_dir, "nested", "vocab.txt"), "w") as f:
+ f.write("hello\nworld\n")
+
+ return root_dir
+
+
+_TRANSFORMER_CONFIG = {
+ "architectures": ["FluxTransformer2DModel"],
+ "model_type": "flux",
+ "hidden_size": 128,
+ "num_hidden_layers": 1,
+}
+
+_TRANSFORMER_TENSORS = {
+ "transformer_blocks.0.attn.to_q.weight": torch.randn(128, 128),
+ "transformer_blocks.0.attn.to_k.weight": torch.randn(128, 128),
+ "transformer_blocks.0.ff.net.0.proj.weight": torch.randn(512, 128),
+ "transformer_blocks.0.ff.net.2.weight": torch.randn(128, 512),
+}
+
+
+class TestCopyMetadataSubfolders:
+ """Tests for _copy_metadata_files including subdirectory handling."""
+
+ def test_non_diffusion_copies_subfolders(self, tmp_path):
+ """Non-diffusion model: subdirectories should be copied to output."""
+ model_dir = _make_model_dir(tmp_path, _SIMPLE_CONFIG, _SIMPLE_TENSORS)
+ # Add a subdirectory with files
+ tokenizer_dir = os.path.join(model_dir, "tokenizer")
+ os.makedirs(os.path.join(tokenizer_dir, "nested"), exist_ok=True)
+ with open(os.path.join(tokenizer_dir, "tokenizer.json"), "w") as f:
+ json.dump({"type": "BPE"}, f)
+ with open(os.path.join(tokenizer_dir, "nested", "vocab.txt"), "w") as f:
+ f.write("hello\nworld\n")
+
+ output_dir = str(tmp_path / "output")
+ core = _ModelFreeCompressorCore(
+ model_name_or_path=model_dir,
+ output_dir=output_dir,
+ scheme="W4A16",
+ )
+ core.run()
+
+ assert os.path.isdir(os.path.join(output_dir, "tokenizer"))
+ assert os.path.isfile(os.path.join(output_dir, "tokenizer", "tokenizer.json"))
+ assert os.path.isdir(os.path.join(output_dir, "tokenizer", "nested"))
+ assert os.path.isfile(os.path.join(output_dir, "tokenizer", "nested", "vocab.txt"))
+
+ def test_diffusion_copies_subfolders(self, tmp_path):
+ """Diffusion model: non-transformer subdirectories should be copied."""
+ root_dir = _make_diffusion_model_dir(tmp_path, _TRANSFORMER_CONFIG, _TRANSFORMER_TENSORS)
+ output_dir = str(tmp_path / "output")
+
+ core = _ModelFreeCompressorCore(
+ model_name_or_path=root_dir,
+ output_dir=output_dir,
+ scheme="W4A16",
+ )
+ core.run()
+
+ # Non-transformer subdirs must be present
+ assert os.path.isdir(os.path.join(output_dir, "vae"))
+ assert os.path.isfile(os.path.join(output_dir, "vae", "config.json"))
+ assert os.path.isdir(os.path.join(output_dir, "scheduler"))
+ assert os.path.isfile(os.path.join(output_dir, "scheduler", "scheduler_config.json"))
+ assert os.path.isdir(os.path.join(output_dir, "tokenizer"))
+ assert os.path.isfile(os.path.join(output_dir, "tokenizer", "tokenizer.json"))
+ assert os.path.isfile(os.path.join(output_dir, "tokenizer", "nested", "vocab.txt"))
+ # Root-level file
+ assert os.path.isfile(os.path.join(output_dir, "model_index.json"))
+ # Quantized transformer must also exist
+ assert os.path.isdir(os.path.join(output_dir, "transformer"))
+ assert os.path.isfile(os.path.join(output_dir, "transformer", "config.json"))
+
+ def test_diffusion_does_not_overwrite_quantized_transformer(self, tmp_path):
+ """Copying subfolders must not overwrite the quantized transformer."""
+ root_dir = _make_diffusion_model_dir(tmp_path, _TRANSFORMER_CONFIG, _TRANSFORMER_TENSORS)
+ output_dir = str(tmp_path / "output")
+
+ core = _ModelFreeCompressorCore(
+ model_name_or_path=root_dir,
+ output_dir=output_dir,
+ scheme="W4A16",
+ )
+ core.run()
+
+ # The output transformer/ should contain quantized weights, not
+ # the original model.safetensors from the source.
+ transformer_out = os.path.join(output_dir, "transformer")
+ out_files = os.listdir(transformer_out)
+ # The original single shard would have been renamed to model.safetensors
+ # by _write_index_file; confirm it has quantized tensor names.
+ keys = set()
+ for f in out_files:
+ if f.endswith(".safetensors"):
+ with safe_open(os.path.join(transformer_out, f), framework="pt") as sf:
+ keys.update(sf.keys())
+ assert any(
+ k.endswith(".qweight") for k in keys
+ ), f"Quantized transformer should contain .qweight tensors, got: {keys}"
diff --git a/test/test_cuda/models/test_get_block_name.py b/test/test_cuda/models/test_get_block_name.py
index 0933573a0..e16417db1 100644
--- a/test/test_cuda/models/test_get_block_name.py
+++ b/test/test_cuda/models/test_get_block_name.py
@@ -165,15 +165,15 @@ class TestAutoRound:
model_name = get_model_path("google/gemma-3-12b-it")
tmp_path = "./tmp/tiny_gemma3" # fixture:clean_tmp_model_folder will clean it
save_tiny_model(
- model_name, tmp_path, from_config=True
+ model_name, tmp_path, from_config=True, num_layers=2
) # make sure the model is downloaded before the test, to avoid timeout in the test
model = Gemma3ForConditionalGeneration.from_pretrained(tmp_path, torch_dtype="auto", trust_remote_code=True)
block_names = get_block_names(model)
- self.check_block_names(block_names, ["model.language_model.layers"], [48])
+ self.check_block_names(block_names, ["model.language_model.layers"], [2])
block_names = get_block_names(model, quant_vision=True)
self.check_block_names(
- block_names, ["model.vision_tower.encoder.layers", "model.language_model.layers"], [27, 48]
+ block_names, ["model.vision_tower.encoder.layers", "model.language_model.layers"], [2, 2]
)
assert not is_pure_text_model(model)