PTQ-Level Efficiency, QAT-Level Accuracy | 训练后量化的效率,量化感知训练的精度
fix save_quantized log conflict (#1845)
Weiwei27761988adec
diff --git a/auto_round/algorithms/quantization/awq/quantizer.py b/auto_round/algorithms/quantization/awq/quantizer.pyindex f6aca2f75..e88f67cf1 100644--- a/auto_round/algorithms/quantization/awq/quantizer.py+++ b/auto_round/algorithms/quantization/awq/quantizer.py@@ -61,6 +61,7 @@ from auto_round.utils import (set_module,)from auto_round.wrapper import WrapperLinear+from auto_round.wrapper import WrapperMultiblock as _WrapperMultiblockclass AWQQuantizer(BaseQuantizers):@@ -90,6 +91,19 @@ class AWQQuantizer(BaseQuantizers):# Fail fast: validate scheme at construction timeself._check_scheme_compatibility()+ def bind(self, compressor) -> None:+ """Wire shared state and validate compressor settings for AWQ compatibility."""+ super().bind(compressor)+ # Check for unsupported compressor-level args+ nblocks = getattr(compressor, "nblocks", 1)+ if nblocks > 1:+ logger.warning(+ f"AWQ does not support nblocks > 1 (got nblocks={nblocks}). "+ f"AWQ smoothing resolves activation hooks and mappings per single block prefix. "+ f"Falling back to nblocks=1."+ )+ compressor.nblocks = 1+def __setattr__(self, name, value):"""Trigger model-level AWQ setup when compress_context is assigned."""super().__setattr__(name, value)@@ -397,6 +411,13 @@ class AWQQuantizer(BaseQuantizers):reference_output: FP16 block output (kept for interface consistency).**kwargs: Must include 'block_name' for mapping lookup."""+ if isinstance(block, _WrapperMultiblock):+ raise ValueError(+ "AWQ does not support nblocks > 1 (multi-block quantization). "+ "Each block must be quantized individually. "+ "Please set nblocks=1 when using algorithm='awq'."+ )+block_name = kwargs.get("block_name")if block_name is None:# Infer block_name from resolved mappings by matching the block's modulesdiff --git a/auto_round/export/export_to_autogptq/export.py b/auto_round/export/export_to_autogptq/export.pyindex dca82caeb..127b3f36b 100644--- a/auto_round/export/export_to_autogptq/export.py+++ b/auto_round/export/export_to_autogptq/export.py@@ -48,6 +48,7 @@ from tqdm import tqdmfrom auto_round.export.utils import (filter_quantization_config,get_autogptq_packing_qlinear,+ is_immediate_saving_mode,release_layer_safely,resolve_pipeline_export_layout,save_model,@@ -205,14 +206,15 @@ def save_quantized_as_autogptq(safe_serialization = kwargs.get("safe_serialization", True)# --- Save metadata (tokenizer, processor, etc.) ---+ immediate_saving = is_immediate_saving_mode(model, serialization_dict)processor_output_dir = output_dirmodel_output_dir = output_dirif output_dir:model_output_dir, processor_output_dir, _ = resolve_pipeline_export_layout(model, output_dir)if output_dir:- # if os.path.exists(output_dir):- # logger.info(f"{output_dir} already exists, may cause overwrite conflicts.")+ if os.path.exists(output_dir) and not immediate_saving:+ logger.warning(f"{output_dir} already exists, this may cause model conflict")for comp in (tokenizer, processor, image_processor):if comp is not None and hasattr(comp, "save_pretrained"):comp.save_pretrained(processor_output_dir)@@ -311,6 +313,11 @@ def save_quantized_as_autogptq(dtype = torch.float16 ##force dtype to fp16save_model(- model, model_output_dir, safe_serialization=safe_serialization, dtype=dtype, config_file="quantize_config.json"+ model,+ model_output_dir,+ safe_serialization=safe_serialization,+ dtype=dtype,+ config_file="quantize_config.json",+ immediate_saving=immediate_saving,)return modeldiff --git a/auto_round/export/export_to_autoround/export.py b/auto_round/export/export_to_autoround/export.pyindex a394a67d8..e93534857 100644--- a/auto_round/export/export_to_autoround/export.py+++ b/auto_round/export/export_to_autoround/export.py@@ -31,6 +31,7 @@ from auto_round.export.export_to_autoround.utils import check_neq_configfrom auto_round.export.utils import (filter_quantization_config,get_autogptq_packing_qlinear,+ is_immediate_saving_mode,release_layer_safely,resolve_pipeline_export_layout,save_model,@@ -338,8 +339,9 @@ def save_quantized_as_autoround(if output_dir is None:model.tokenizer = tokenizerreturn model- # if os.path.exists(output_dir):- # logger.info(f"{output_dir} already exists, this may cause model conflict")+ immediate_saving = is_immediate_saving_mode(model, serialization_dict)+ if os.path.exists(output_dir) and not immediate_saving:+ logger.warning(f"{output_dir} already exists, this may cause model conflict")model_output_dir = output_dirprocessor_output_dir = output_dirif output_dir:@@ -358,7 +360,9 @@ def save_quantized_as_autoround(dtype = torch.float16 ## awq vllm kernel only supports float16 on cudaelse:dtype = None- save_model(model, model_output_dir, safe_serialization=safe_serialization, dtype=dtype)+ save_model(+ model, model_output_dir, safe_serialization=safe_serialization, dtype=dtype, immediate_saving=immediate_saving+ )# Save rotation config to config.json for load-time reconstructionif hasattr(model, "_rotation_config"):diff --git a/auto_round/export/export_to_autoround/export_to_fp8.py b/auto_round/export/export_to_autoround/export_to_fp8.pyindex b242091ea..467df50ef 100644--- a/auto_round/export/export_to_autoround/export_to_fp8.py+++ b/auto_round/export/export_to_autoround/export_to_fp8.py@@ -24,7 +24,12 @@ from tqdm import tqdmfrom auto_round.data_type.utils import reshape_pad_tensor_by_group_size, revert_tensor_by_padfrom auto_round.export.export_to_autoround.utils import check_neq_config-from auto_round.export.utils import filter_quantization_config, release_layer_safely, save_model+from auto_round.export.utils import (+ filter_quantization_config,+ is_immediate_saving_mode,+ release_layer_safely,+ save_model,+)from auto_round.logger import loggerfrom auto_round.schemes import QuantizationSchemefrom auto_round.utils import (@@ -272,8 +277,9 @@ def save_quantized_as_autoround(if output_dir is None:model.tokenizer = tokenizerreturn model- # if os.path.exists(output_dir):- # logger.warning(f"{output_dir} already exists, this may cause model conflict")+ immediate_saving = is_immediate_saving_mode(model, serialization_dict)+ if os.path.exists(output_dir) and not immediate_saving:+ logger.warning(f"{output_dir} already exists, this may cause model conflict")if tokenizer is not None:tokenizer.save_pretrained(output_dir)@@ -287,7 +293,7 @@ def save_quantized_as_autoround(dtype = torch.float16 ## awq kernel only supports float16 on cudaelse:dtype = None- save_model(model, output_dir, safe_serialization=safe_serialization, dtype=dtype)+ save_model(model, output_dir, safe_serialization=safe_serialization, dtype=dtype, immediate_saving=immediate_saving)# Save rotation config to config.json for load-time reconstructionif hasattr(model, "_rotation_config"):diff --git a/auto_round/export/export_to_autoround/export_to_nvfp_mx.py b/auto_round/export/export_to_autoround/export_to_nvfp_mx.pyindex 8b491e768..c9607ae40 100644--- a/auto_round/export/export_to_autoround/export_to_nvfp_mx.py+++ b/auto_round/export/export_to_autoround/export_to_nvfp_mx.py@@ -26,7 +26,12 @@ from tqdm import tqdmfrom auto_round.compressors.utils import is_mx_fp, is_nv_fpfrom auto_round.export.export_to_autoround.utils import check_neq_config-from auto_round.export.utils import filter_quantization_config, release_layer_safely, save_model+from auto_round.export.utils import (+ filter_quantization_config,+ is_immediate_saving_mode,+ release_layer_safely,+ save_model,+)from auto_round.logger import loggerfrom auto_round.schemes import QuantizationSchemefrom auto_round.utils import (@@ -261,7 +266,8 @@ def save_quantized_as_fp(if output_dir is None:model.tokenizer = tokenizerreturn model- if os.path.exists(output_dir):+ immediate_saving = is_immediate_saving_mode(model, serialization_dict)+ if os.path.exists(output_dir) and not immediate_saving:logger.warning(f"{output_dir} already exists, this may cause model conflict")if tokenizer is not None:tokenizer.save_pretrained(output_dir)@@ -272,7 +278,7 @@ def save_quantized_as_fp(image_processor.save_pretrained(output_dir)dtype = None- save_model(model, output_dir, safe_serialization=safe_serialization, dtype=dtype)+ save_model(model, output_dir, safe_serialization=safe_serialization, dtype=dtype, immediate_saving=immediate_saving)# Save rotation config to config.json for load-time reconstructionif hasattr(model, "_rotation_config"):diff --git a/auto_round/export/export_to_awq/export.py b/auto_round/export/export_to_awq/export.pyindex cc178097a..a6a7fff7a 100644--- a/auto_round/export/export_to_awq/export.py+++ b/auto_round/export/export_to_awq/export.py@@ -30,7 +30,12 @@ import torch.nn as nnfrom tqdm import tqdmfrom auto_round.export.export_to_awq.utils import WQLinear_GEMM-from auto_round.export.utils import filter_quantization_config, release_layer_safely, save_model+from auto_round.export.utils import (+ filter_quantization_config,+ is_immediate_saving_mode,+ release_layer_safely,+ save_model,+)from auto_round.logger import loggerfrom auto_round.utils import (INNER_SUPPORTED_LAYER_TYPES,@@ -154,7 +159,8 @@ def save_quantized_as_autoawq(processor = kwargs.get("processor", None)image_processor = kwargs.get("image_processor", None)- if output_dir is not None and os.path.exists(output_dir):+ immediate_saving = is_immediate_saving_mode(model, serialization_dict)+ if output_dir is not None and os.path.exists(output_dir) and not immediate_saving:logger.warning(f"{output_dir} already exists, this may cause model conflict")logger.info("Saving quantized model to auto_awq format")@@ -210,6 +216,12 @@ def save_quantized_as_autoawq(# Force torch_dtype in the saved config to fp16 for vLLM CUDA kernel compatibility;# this does not change in-memory model weights.dtype = torch.float16- save_model(compressed_model, output_dir, safe_serialization=safe_serialization, dtype=dtype)+ save_model(+ compressed_model,+ output_dir,+ safe_serialization=safe_serialization,+ dtype=dtype,+ immediate_saving=immediate_saving,+ )return compressed_modeldiff --git a/auto_round/export/export_to_llmcompressor/export.py b/auto_round/export/export_to_llmcompressor/export.pyindex fbabfcfb0..794a6e051 100644--- a/auto_round/export/export_to_llmcompressor/export.py+++ b/auto_round/export/export_to_llmcompressor/export.py@@ -18,7 +18,7 @@ from typing import Callable, Unionimport torch-from auto_round.export.utils import save_model+from auto_round.export.utils import is_immediate_saving_mode, save_modelfrom auto_round.logger import loggerfrom auto_round.utils import (SUPPORTED_LAYER_TYPES,@@ -202,8 +202,9 @@ def save_quantized_as_llmcompressor(safe_serialization = kwargs.get("safe_serialization", True)processor = kwargs.get("processor", None)- # if output_dir is not None and os.path.exists(output_dir):- # logger.warning(f"{output_dir} already exists, this may cause model conflict")+ immediate_saving = is_immediate_saving_mode(model, serialization_dict)+ if output_dir is not None and os.path.exists(output_dir) and not immediate_saving:+ logger.warning(f"{output_dir} already exists, this may cause model conflict")if not inplace:model = copy.deepcopy(model.to("cpu"))@@ -229,7 +230,7 @@ def save_quantized_as_llmcompressor(# save model.config, model.state_dict()model.config.save_pretrained(output_dir)- save_model(model, output_dir, safe_serialization=safe_serialization)+ save_model(model, output_dir, safe_serialization=safe_serialization, immediate_saving=immediate_saving)try:copy_python_files_from_model_cache(model, output_dir)diff --git a/auto_round/export/export_to_llmcompressor/export_to_fp.py b/auto_round/export/export_to_llmcompressor/export_to_fp.pyindex b18d9b149..432610d54 100644--- a/auto_round/export/export_to_llmcompressor/export_to_fp.py+++ b/auto_round/export/export_to_llmcompressor/export_to_fp.py@@ -28,7 +28,12 @@ from auto_round.compressors.utils import is_mx_fp, is_nv_fpfrom auto_round.export.export_to_autoround.qlinear_fp import QuantLinearfrom auto_round.export.export_to_llmcompressor.config import initialize_quantizationfrom auto_round.export.export_to_llmcompressor.utils import generate_ignore_regex_list-from auto_round.export.utils import filter_quantization_config, release_layer_safely, save_model+from auto_round.export.utils import (+ filter_quantization_config,+ is_immediate_saving_mode,+ release_layer_safely,+ save_model,+)from auto_round.logger import loggerfrom auto_round.utils import (SUPPORTED_LAYER_TYPES,@@ -307,7 +312,8 @@ def save_quantized_as_fp(if output_dir is None:model.tokenizer = tokenizerreturn model- if os.path.exists(output_dir):+ immediate_saving = is_immediate_saving_mode(model, serialization_dict)+ if os.path.exists(output_dir) and not immediate_saving:logger.warning(f"{output_dir} already exists, this may cause model conflict")if tokenizer is not None:tokenizer.save_pretrained(output_dir)@@ -316,6 +322,6 @@ def save_quantized_as_fp(processor.save_pretrained(output_dir)dtype = None- save_model(model, output_dir, safe_serialization=safe_serialization, dtype=dtype)+ save_model(model, output_dir, safe_serialization=safe_serialization, dtype=dtype, immediate_saving=immediate_saving)return modeldiff --git a/auto_round/export/export_to_llmcompressor/export_to_static_fp.py b/auto_round/export/export_to_llmcompressor/export_to_static_fp.pyindex c220dfe8a..0754f6df4 100644--- a/auto_round/export/export_to_llmcompressor/export_to_static_fp.py+++ b/auto_round/export/export_to_llmcompressor/export_to_static_fp.py@@ -25,7 +25,7 @@ from tqdm import tqdmfrom auto_round.data_type.utils import reshape_pad_tensor_by_group_size, revert_tensor_by_padfrom auto_round.export.export_to_autoround.export_to_fp8 import FP8QLinearfrom auto_round.export.export_to_llmcompressor.config import check_compressed_tensors_supported-from auto_round.export.utils import save_model+from auto_round.export.utils import is_immediate_saving_mode, save_modelfrom auto_round.utils import (SUPPORTED_LAYER_TYPES,check_start_with_block_name,@@ -261,7 +261,8 @@ def save_quantized_as_static_fp(if output_dir is None:return model- if os.path.exists(output_dir):+ immediate_saving = is_immediate_saving_mode(model, serialization_dict)+ if os.path.exists(output_dir) and not immediate_saving:logger.warning(f"{output_dir} already exists, this may cause model conflict")if tokenizer is not None:@@ -274,6 +275,6 @@ def save_quantized_as_static_fp(image_processor.save_pretrained(output_dir)dtype = None- save_model(model, output_dir, safe_serialization=safe_serialization, dtype=dtype)+ save_model(model, output_dir, safe_serialization=safe_serialization, dtype=dtype, immediate_saving=immediate_saving)return modeldiff --git a/auto_round/export/utils.py b/auto_round/export/utils.pyindex eaa773fd0..c809f0b0b 100644--- a/auto_round/export/utils.py+++ b/auto_round/export/utils.py@@ -51,6 +51,25 @@ def _state_dict_has_meta_tensor(model: nn.Module) -> bool:return False+def is_immediate_saving_mode(model: nn.Module, serialization_dict: dict = None) -> bool:+ """Determine if the model was saved via ShardWriter (immediate saving mode).++ Resolution order:+ 1. CompressContext singleton (authoritative at export time).+ 2. Meta-tensor heuristics (ShardWriter offloads weights to meta device).+ """+ from auto_round.context.compress import CompressContext++ if CompressContext in CompressContext._instances:+ if CompressContext._instances[CompressContext].is_immediate_saving:+ return True+ if unsupported_meta_device(model):+ return True+ if _state_dict_has_meta_tensor(model):+ return True+ return False++def is_local_pipeline_model_dir(model_dir: str) -> bool:if not model_dir or not os.path.isdir(model_dir):return False@@ -198,6 +217,7 @@ def save_model(safe_serialization: bool = True,dtype=None,config_file="quantization_config.json",+ immediate_saving: bool = False,):"""Save model state dict and configs.@@ -217,31 +237,27 @@ def save_model(</Tip>safe_serialization (`bool`, defaults to `True`):Whether to save the model using `safetensors` or the traditional PyTorch way (that uses `pickle`).+ immediate_saving (`bool`, defaults to `False`):+ When True, weights were already saved by ShardWriter during quantization.+ Only configs and metadata will be saved; weight saving is skipped."""os.makedirs(save_dir, exist_ok=True)- if unsupported_meta_device(model):+ if immediate_saving:+ logger.info("Immediate saving mode: weights already saved by ShardWriter, saving configs only.")_save_model_configs(model, save_dir)else:- has_meta_tensor = _state_dict_has_meta_tensor(model)- if has_meta_tensor:- logger.info(- "Detected meta tensors in state_dict after shard-based saving; skipping model.save_pretrained and "- "saving configs only."- )+ try:+ model.save_pretrained(save_dir, max_shard_size=max_shard_size, safe_serialization=safe_serialization)+ except (KeyError, TypeError) as e:+ # Some third-party configs fail during config serialization in save_pretrained.+ # Fall back to saving weights separately + config without diff.+ logger.warning("model.save_pretrained failed (%s), falling back to manual save.", e)+ from safetensors.torch import save_file++ state_dict = model.state_dict()+ save_file(state_dict, os.path.join(save_dir, "model.safetensors"))_save_model_configs(model, save_dir)- else:- try:- model.save_pretrained(save_dir, max_shard_size=max_shard_size, safe_serialization=safe_serialization)- except (KeyError, TypeError) as e:- # Some third-party configs fail during config serialization in save_pretrained.- # Fall back to saving weights separately + config without diff.- logger.warning("model.save_pretrained failed (%s), falling back to manual save.", e)- from safetensors.torch import save_file-- state_dict = model.state_dict()- save_file(state_dict, os.path.join(save_dir, "model.safetensors"))- _save_model_configs(model, save_dir)source_dir = _resolve_model_source_dir(model)diff --git a/docs/step_by_step.md b/docs/step_by_step.mdindex 06c905ec8..562b88d0e 100644--- a/docs/step_by_step.md+++ b/docs/step_by_step.md@@ -356,12 +356,15 @@ ar.quantize_and_save(output_dir, format="auto_round:llm_compressor")### AutoScheme-AutoScheme provides an automatic algorithm to generate adaptive mixed bits/data-type quantization recipes. For some accuracy result, please refer this doc [here](./auto_scheme_acc.md)+AutoScheme automatically generates adaptive mixed-bit and mixed-data-type quantization recipes. For accuracy results, see [AutoScheme Accuracy Report](./auto_scheme_acc.md).-**Please note that mixed data types are supported during tuning, but cannot be exported to real models at this time..**+**Note:** Mixed-data-types are supported during tuning, but cannot be exported to real models at this time.#### CLI Usage-use `iters=200`for tuning.++- **`--iters 0`**: RTN. Fast (seconds to minutes).+- **`--iters 200`**: Tuning-aware scheme selection. More accurate but much slower.+~~~bashauto_round \--model_name $model_name \diff --git a/docs/step_by_step_CN.md b/docs/step_by_step_CN.mdindex d74dcd486..c5a07fb79 100644--- a/docs/step_by_step_CN.md+++ b/docs/step_by_step_CN.md@@ -353,12 +353,16 @@ ar.quantize_and_save(output_dir, format="auto_round:llm_compressor")### AutoScheme 自动混合精度量化方案-AutoScheme 采用自动化算法,可生成 **自适应的混合精度与数据类型** 的量化方案(mixed bits/data type quantization recipes)。相关测试结果请参考[《AutoScheme精度报告》](./auto_scheme_acc.md)。-**注意**:混合数据类型方案在训练阶段可用,但当前版本还不支持导出至实际模型。+AutoScheme 自动生成自适应的混合比特/混合数据类型量化方案。精度测试结果请参考 [AutoScheme 精度报告](./auto_scheme_acc.md)。++**说明:** 混合数据类型支持调优,但目前无法将其导出到实际模型中。#### 命令行用法-如需启动训练建议设置 `iters=200`++- **`--iters 0`**:基于 RTN 的 量化方案,速度快(秒到分钟级)。+- **`--iters 200`**:调优感知的量化方案,更精确但慢很多。+~~~bashauto_round \--model_name $model_name \diff --git a/test/test_cpu/export/test_export.py b/test/test_cpu/export/test_export.pyindex 4f4c0d177..3b6c411eb 100644--- a/test/test_cpu/export/test_export.py+++ b/test/test_cpu/export/test_export.py@@ -613,3 +613,50 @@ def test_fp8_autoround_export_packs_serially(tiny_opt_model_path, tmp_path, monkautoround.save_quantized(output_dir=tmp_path, format="auto_round")with safe_open(os.path.join(tmp_path, "model.safetensors"), framework="pt") as f:assert "model.decoder.layers.0.self_attn.k_proj.weight_scale" in f.keys()++[email protected]("low_cpu_mem_usage", [True, False])+def test_immediate_saving_mode(tiny_opt_model_path, tmp_path, low_cpu_mem_usage, caplog):+ """Verify that immediate_saving (triggered by low_cpu_mem_usage) produces a complete model output."""+ import logging++ output_dir = str(tmp_path / "output")+ with caplog.at_level(logging.DEBUG, logger="auto_round"):+ autoround = AutoRound(+ tiny_opt_model_path,+ scheme="MXFP4",+ iters=2,+ seqlen=2,+ nsamples=2,+ low_cpu_mem_usage=low_cpu_mem_usage,+ )+ _, quantized_model_path = autoround.quantize_and_save(output_dir=output_dir, format="llm_compressor")++ # No spurious "already exists" warning should be emitted+ conflict_messages = [r.message for r in caplog.records if "already exists" in r.message]+ assert len(conflict_messages) == 0, f"Unexpected conflict warnings: {conflict_messages}"++ # All essential files must exist regardless of immediate_saving mode+ assert os.path.exists(os.path.join(quantized_model_path, "config.json")), "config.json missing"+ assert os.path.exists(+ os.path.join(quantized_model_path, "quantization_config.json")+ ), "quantization_config.json missing"++ # Exactly 1 safetensors shard for this tiny model+ safetensor_files = [f for f in os.listdir(quantized_model_path) if f.endswith(".safetensors")]+ assert len(safetensor_files) == 1, f"Expected 1 safetensors file, got {len(safetensor_files)}: {safetensor_files}"++ # Tokenizer files must be present+ assert os.path.exists(os.path.join(quantized_model_path, "tokenizer_config.json")), "tokenizer_config.json missing"++ # Total file count: config.json, quantization_config.json, model.safetensors,+ # generation_config.json, tokenizer.json, tokenizer_config.json = 6+ all_files = os.listdir(quantized_model_path)+ assert len(all_files) == 6, f"Expected 6 files, got {len(all_files)}: {sorted(all_files)}"++ # Verify weights are loadable and non-empty+ from safetensors import safe_open++ with safe_open(os.path.join(quantized_model_path, safetensor_files[0]), framework="pt") as f:+ keys = f.keys()+ assert len(keys) > 0, "Safetensors file has no tensors"