Skip to content

PTQ-Level Efficiency, QAT-Level Accuracy | 训练后量化的效率,量化感知训练的精度

fix save_quantized log conflict (#1845)

Weiwei27761988adec
diff --git a/auto_round/algorithms/quantization/awq/quantizer.py b/auto_round/algorithms/quantization/awq/quantizer.py
index f6aca2f75..e88f67cf1 100644
--- a/auto_round/algorithms/quantization/awq/quantizer.py
+++ b/auto_round/algorithms/quantization/awq/quantizer.py
@@ -61,6 +61,7 @@ from auto_round.utils import (
set_module,
)
from auto_round.wrapper import WrapperLinear
+from auto_round.wrapper import WrapperMultiblock as _WrapperMultiblock
class AWQQuantizer(BaseQuantizers):
@@ -90,6 +91,19 @@ class AWQQuantizer(BaseQuantizers):
# Fail fast: validate scheme at construction time
self._check_scheme_compatibility()
+ def bind(self, compressor) -> None:
+ """Wire shared state and validate compressor settings for AWQ compatibility."""
+ super().bind(compressor)
+ # Check for unsupported compressor-level args
+ nblocks = getattr(compressor, "nblocks", 1)
+ if nblocks > 1:
+ logger.warning(
+ f"AWQ does not support nblocks > 1 (got nblocks={nblocks}). "
+ f"AWQ smoothing resolves activation hooks and mappings per single block prefix. "
+ f"Falling back to nblocks=1."
+ )
+ compressor.nblocks = 1
+
def __setattr__(self, name, value):
"""Trigger model-level AWQ setup when compress_context is assigned."""
super().__setattr__(name, value)
@@ -397,6 +411,13 @@ class AWQQuantizer(BaseQuantizers):
reference_output: FP16 block output (kept for interface consistency).
**kwargs: Must include 'block_name' for mapping lookup.
"""
+ if isinstance(block, _WrapperMultiblock):
+ raise ValueError(
+ "AWQ does not support nblocks > 1 (multi-block quantization). "
+ "Each block must be quantized individually. "
+ "Please set nblocks=1 when using algorithm='awq'."
+ )
+
block_name = kwargs.get("block_name")
if block_name is None:
# Infer block_name from resolved mappings by matching the block's modules
diff --git a/auto_round/export/export_to_autogptq/export.py b/auto_round/export/export_to_autogptq/export.py
index dca82caeb..127b3f36b 100644
--- a/auto_round/export/export_to_autogptq/export.py
+++ b/auto_round/export/export_to_autogptq/export.py
@@ -48,6 +48,7 @@ from tqdm import tqdm
from auto_round.export.utils import (
filter_quantization_config,
get_autogptq_packing_qlinear,
+ is_immediate_saving_mode,
release_layer_safely,
resolve_pipeline_export_layout,
save_model,
@@ -205,14 +206,15 @@ def save_quantized_as_autogptq(
safe_serialization = kwargs.get("safe_serialization", True)
# --- Save metadata (tokenizer, processor, etc.) ---
+ immediate_saving = is_immediate_saving_mode(model, serialization_dict)
processor_output_dir = output_dir
model_output_dir = output_dir
if output_dir:
model_output_dir, processor_output_dir, _ = resolve_pipeline_export_layout(model, output_dir)
if output_dir:
- # if os.path.exists(output_dir):
- # logger.info(f"{output_dir} already exists, may cause overwrite conflicts.")
+ if os.path.exists(output_dir) and not immediate_saving:
+ logger.warning(f"{output_dir} already exists, this may cause model conflict")
for comp in (tokenizer, processor, image_processor):
if comp is not None and hasattr(comp, "save_pretrained"):
comp.save_pretrained(processor_output_dir)
@@ -311,6 +313,11 @@ def save_quantized_as_autogptq(
dtype = torch.float16 ##force dtype to fp16
save_model(
- model, model_output_dir, safe_serialization=safe_serialization, dtype=dtype, config_file="quantize_config.json"
+ model,
+ model_output_dir,
+ safe_serialization=safe_serialization,
+ dtype=dtype,
+ config_file="quantize_config.json",
+ immediate_saving=immediate_saving,
)
return model
diff --git a/auto_round/export/export_to_autoround/export.py b/auto_round/export/export_to_autoround/export.py
index a394a67d8..e93534857 100644
--- a/auto_round/export/export_to_autoround/export.py
+++ b/auto_round/export/export_to_autoround/export.py
@@ -31,6 +31,7 @@ from auto_round.export.export_to_autoround.utils import check_neq_config
from auto_round.export.utils import (
filter_quantization_config,
get_autogptq_packing_qlinear,
+ is_immediate_saving_mode,
release_layer_safely,
resolve_pipeline_export_layout,
save_model,
@@ -338,8 +339,9 @@ def save_quantized_as_autoround(
if output_dir is None:
model.tokenizer = tokenizer
return model
- # if os.path.exists(output_dir):
- # logger.info(f"{output_dir} already exists, this may cause model conflict")
+ immediate_saving = is_immediate_saving_mode(model, serialization_dict)
+ if os.path.exists(output_dir) and not immediate_saving:
+ logger.warning(f"{output_dir} already exists, this may cause model conflict")
model_output_dir = output_dir
processor_output_dir = output_dir
if output_dir:
@@ -358,7 +360,9 @@ def save_quantized_as_autoround(
dtype = torch.float16 ## awq vllm kernel only supports float16 on cuda
else:
dtype = None
- save_model(model, model_output_dir, safe_serialization=safe_serialization, dtype=dtype)
+ save_model(
+ model, model_output_dir, safe_serialization=safe_serialization, dtype=dtype, immediate_saving=immediate_saving
+ )
# Save rotation config to config.json for load-time reconstruction
if hasattr(model, "_rotation_config"):
diff --git a/auto_round/export/export_to_autoround/export_to_fp8.py b/auto_round/export/export_to_autoround/export_to_fp8.py
index b242091ea..467df50ef 100644
--- a/auto_round/export/export_to_autoround/export_to_fp8.py
+++ b/auto_round/export/export_to_autoround/export_to_fp8.py
@@ -24,7 +24,12 @@ from tqdm import tqdm
from auto_round.data_type.utils import reshape_pad_tensor_by_group_size, revert_tensor_by_pad
from auto_round.export.export_to_autoround.utils import check_neq_config
-from auto_round.export.utils import filter_quantization_config, release_layer_safely, save_model
+from auto_round.export.utils import (
+ filter_quantization_config,
+ is_immediate_saving_mode,
+ release_layer_safely,
+ save_model,
+)
from auto_round.logger import logger
from auto_round.schemes import QuantizationScheme
from auto_round.utils import (
@@ -272,8 +277,9 @@ def save_quantized_as_autoround(
if output_dir is None:
model.tokenizer = tokenizer
return model
- # if os.path.exists(output_dir):
- # logger.warning(f"{output_dir} already exists, this may cause model conflict")
+ immediate_saving = is_immediate_saving_mode(model, serialization_dict)
+ if os.path.exists(output_dir) and not immediate_saving:
+ logger.warning(f"{output_dir} already exists, this may cause model conflict")
if tokenizer is not None:
tokenizer.save_pretrained(output_dir)
@@ -287,7 +293,7 @@ def save_quantized_as_autoround(
dtype = torch.float16 ## awq kernel only supports float16 on cuda
else:
dtype = None
- save_model(model, output_dir, safe_serialization=safe_serialization, dtype=dtype)
+ save_model(model, output_dir, safe_serialization=safe_serialization, dtype=dtype, immediate_saving=immediate_saving)
# Save rotation config to config.json for load-time reconstruction
if hasattr(model, "_rotation_config"):
diff --git a/auto_round/export/export_to_autoround/export_to_nvfp_mx.py b/auto_round/export/export_to_autoround/export_to_nvfp_mx.py
index 8b491e768..c9607ae40 100644
--- a/auto_round/export/export_to_autoround/export_to_nvfp_mx.py
+++ b/auto_round/export/export_to_autoround/export_to_nvfp_mx.py
@@ -26,7 +26,12 @@ from tqdm import tqdm
from auto_round.compressors.utils import is_mx_fp, is_nv_fp
from auto_round.export.export_to_autoround.utils import check_neq_config
-from auto_round.export.utils import filter_quantization_config, release_layer_safely, save_model
+from auto_round.export.utils import (
+ filter_quantization_config,
+ is_immediate_saving_mode,
+ release_layer_safely,
+ save_model,
+)
from auto_round.logger import logger
from auto_round.schemes import QuantizationScheme
from auto_round.utils import (
@@ -261,7 +266,8 @@ def save_quantized_as_fp(
if output_dir is None:
model.tokenizer = tokenizer
return model
- if os.path.exists(output_dir):
+ immediate_saving = is_immediate_saving_mode(model, serialization_dict)
+ if os.path.exists(output_dir) and not immediate_saving:
logger.warning(f"{output_dir} already exists, this may cause model conflict")
if tokenizer is not None:
tokenizer.save_pretrained(output_dir)
@@ -272,7 +278,7 @@ def save_quantized_as_fp(
image_processor.save_pretrained(output_dir)
dtype = None
- save_model(model, output_dir, safe_serialization=safe_serialization, dtype=dtype)
+ save_model(model, output_dir, safe_serialization=safe_serialization, dtype=dtype, immediate_saving=immediate_saving)
# Save rotation config to config.json for load-time reconstruction
if hasattr(model, "_rotation_config"):
diff --git a/auto_round/export/export_to_awq/export.py b/auto_round/export/export_to_awq/export.py
index cc178097a..a6a7fff7a 100644
--- a/auto_round/export/export_to_awq/export.py
+++ b/auto_round/export/export_to_awq/export.py
@@ -30,7 +30,12 @@ import torch.nn as nn
from tqdm import tqdm
from auto_round.export.export_to_awq.utils import WQLinear_GEMM
-from auto_round.export.utils import filter_quantization_config, release_layer_safely, save_model
+from auto_round.export.utils import (
+ filter_quantization_config,
+ is_immediate_saving_mode,
+ release_layer_safely,
+ save_model,
+)
from auto_round.logger import logger
from auto_round.utils import (
INNER_SUPPORTED_LAYER_TYPES,
@@ -154,7 +159,8 @@ def save_quantized_as_autoawq(
processor = kwargs.get("processor", None)
image_processor = kwargs.get("image_processor", None)
- if output_dir is not None and os.path.exists(output_dir):
+ immediate_saving = is_immediate_saving_mode(model, serialization_dict)
+ if output_dir is not None and os.path.exists(output_dir) and not immediate_saving:
logger.warning(f"{output_dir} already exists, this may cause model conflict")
logger.info("Saving quantized model to auto_awq format")
@@ -210,6 +216,12 @@ def save_quantized_as_autoawq(
# Force torch_dtype in the saved config to fp16 for vLLM CUDA kernel compatibility;
# this does not change in-memory model weights.
dtype = torch.float16
- save_model(compressed_model, output_dir, safe_serialization=safe_serialization, dtype=dtype)
+ save_model(
+ compressed_model,
+ output_dir,
+ safe_serialization=safe_serialization,
+ dtype=dtype,
+ immediate_saving=immediate_saving,
+ )
return compressed_model
diff --git a/auto_round/export/export_to_llmcompressor/export.py b/auto_round/export/export_to_llmcompressor/export.py
index fbabfcfb0..794a6e051 100644
--- a/auto_round/export/export_to_llmcompressor/export.py
+++ b/auto_round/export/export_to_llmcompressor/export.py
@@ -18,7 +18,7 @@ from typing import Callable, Union
import torch
-from auto_round.export.utils import save_model
+from auto_round.export.utils import is_immediate_saving_mode, save_model
from auto_round.logger import logger
from auto_round.utils import (
SUPPORTED_LAYER_TYPES,
@@ -202,8 +202,9 @@ def save_quantized_as_llmcompressor(
safe_serialization = kwargs.get("safe_serialization", True)
processor = kwargs.get("processor", None)
- # if output_dir is not None and os.path.exists(output_dir):
- # logger.warning(f"{output_dir} already exists, this may cause model conflict")
+ immediate_saving = is_immediate_saving_mode(model, serialization_dict)
+ if output_dir is not None and os.path.exists(output_dir) and not immediate_saving:
+ logger.warning(f"{output_dir} already exists, this may cause model conflict")
if not inplace:
model = copy.deepcopy(model.to("cpu"))
@@ -229,7 +230,7 @@ def save_quantized_as_llmcompressor(
# save model.config, model.state_dict()
model.config.save_pretrained(output_dir)
- save_model(model, output_dir, safe_serialization=safe_serialization)
+ save_model(model, output_dir, safe_serialization=safe_serialization, immediate_saving=immediate_saving)
try:
copy_python_files_from_model_cache(model, output_dir)
diff --git a/auto_round/export/export_to_llmcompressor/export_to_fp.py b/auto_round/export/export_to_llmcompressor/export_to_fp.py
index b18d9b149..432610d54 100644
--- a/auto_round/export/export_to_llmcompressor/export_to_fp.py
+++ b/auto_round/export/export_to_llmcompressor/export_to_fp.py
@@ -28,7 +28,12 @@ from auto_round.compressors.utils import is_mx_fp, is_nv_fp
from auto_round.export.export_to_autoround.qlinear_fp import QuantLinear
from auto_round.export.export_to_llmcompressor.config import initialize_quantization
from auto_round.export.export_to_llmcompressor.utils import generate_ignore_regex_list
-from auto_round.export.utils import filter_quantization_config, release_layer_safely, save_model
+from auto_round.export.utils import (
+ filter_quantization_config,
+ is_immediate_saving_mode,
+ release_layer_safely,
+ save_model,
+)
from auto_round.logger import logger
from auto_round.utils import (
SUPPORTED_LAYER_TYPES,
@@ -307,7 +312,8 @@ def save_quantized_as_fp(
if output_dir is None:
model.tokenizer = tokenizer
return model
- if os.path.exists(output_dir):
+ immediate_saving = is_immediate_saving_mode(model, serialization_dict)
+ if os.path.exists(output_dir) and not immediate_saving:
logger.warning(f"{output_dir} already exists, this may cause model conflict")
if tokenizer is not None:
tokenizer.save_pretrained(output_dir)
@@ -316,6 +322,6 @@ def save_quantized_as_fp(
processor.save_pretrained(output_dir)
dtype = None
- save_model(model, output_dir, safe_serialization=safe_serialization, dtype=dtype)
+ save_model(model, output_dir, safe_serialization=safe_serialization, dtype=dtype, immediate_saving=immediate_saving)
return model
diff --git a/auto_round/export/export_to_llmcompressor/export_to_static_fp.py b/auto_round/export/export_to_llmcompressor/export_to_static_fp.py
index c220dfe8a..0754f6df4 100644
--- a/auto_round/export/export_to_llmcompressor/export_to_static_fp.py
+++ b/auto_round/export/export_to_llmcompressor/export_to_static_fp.py
@@ -25,7 +25,7 @@ from tqdm import tqdm
from auto_round.data_type.utils import reshape_pad_tensor_by_group_size, revert_tensor_by_pad
from auto_round.export.export_to_autoround.export_to_fp8 import FP8QLinear
from auto_round.export.export_to_llmcompressor.config import check_compressed_tensors_supported
-from auto_round.export.utils import save_model
+from auto_round.export.utils import is_immediate_saving_mode, save_model
from auto_round.utils import (
SUPPORTED_LAYER_TYPES,
check_start_with_block_name,
@@ -261,7 +261,8 @@ def save_quantized_as_static_fp(
if output_dir is None:
return model
- if os.path.exists(output_dir):
+ immediate_saving = is_immediate_saving_mode(model, serialization_dict)
+ if os.path.exists(output_dir) and not immediate_saving:
logger.warning(f"{output_dir} already exists, this may cause model conflict")
if tokenizer is not None:
@@ -274,6 +275,6 @@ def save_quantized_as_static_fp(
image_processor.save_pretrained(output_dir)
dtype = None
- save_model(model, output_dir, safe_serialization=safe_serialization, dtype=dtype)
+ save_model(model, output_dir, safe_serialization=safe_serialization, dtype=dtype, immediate_saving=immediate_saving)
return model
diff --git a/auto_round/export/utils.py b/auto_round/export/utils.py
index eaa773fd0..c809f0b0b 100644
--- a/auto_round/export/utils.py
+++ b/auto_round/export/utils.py
@@ -51,6 +51,25 @@ def _state_dict_has_meta_tensor(model: nn.Module) -> bool:
return False
+def is_immediate_saving_mode(model: nn.Module, serialization_dict: dict = None) -> bool:
+ """Determine if the model was saved via ShardWriter (immediate saving mode).
+
+ Resolution order:
+ 1. CompressContext singleton (authoritative at export time).
+ 2. Meta-tensor heuristics (ShardWriter offloads weights to meta device).
+ """
+ from auto_round.context.compress import CompressContext
+
+ if CompressContext in CompressContext._instances:
+ if CompressContext._instances[CompressContext].is_immediate_saving:
+ return True
+ if unsupported_meta_device(model):
+ return True
+ if _state_dict_has_meta_tensor(model):
+ return True
+ return False
+
+
def is_local_pipeline_model_dir(model_dir: str) -> bool:
if not model_dir or not os.path.isdir(model_dir):
return False
@@ -198,6 +217,7 @@ def save_model(
safe_serialization: bool = True,
dtype=None,
config_file="quantization_config.json",
+ immediate_saving: bool = False,
):
"""Save model state dict and configs.
@@ -217,31 +237,27 @@ def save_model(
</Tip>
safe_serialization (`bool`, defaults to `True`):
Whether to save the model using `safetensors` or the traditional PyTorch way (that uses `pickle`).
+ immediate_saving (`bool`, defaults to `False`):
+ When True, weights were already saved by ShardWriter during quantization.
+ Only configs and metadata will be saved; weight saving is skipped.
"""
os.makedirs(save_dir, exist_ok=True)
- if unsupported_meta_device(model):
+ if immediate_saving:
+ logger.info("Immediate saving mode: weights already saved by ShardWriter, saving configs only.")
_save_model_configs(model, save_dir)
else:
- has_meta_tensor = _state_dict_has_meta_tensor(model)
- if has_meta_tensor:
- logger.info(
- "Detected meta tensors in state_dict after shard-based saving; skipping model.save_pretrained and "
- "saving configs only."
- )
+ try:
+ model.save_pretrained(save_dir, max_shard_size=max_shard_size, safe_serialization=safe_serialization)
+ except (KeyError, TypeError) as e:
+ # Some third-party configs fail during config serialization in save_pretrained.
+ # Fall back to saving weights separately + config without diff.
+ logger.warning("model.save_pretrained failed (%s), falling back to manual save.", e)
+ from safetensors.torch import save_file
+
+ state_dict = model.state_dict()
+ save_file(state_dict, os.path.join(save_dir, "model.safetensors"))
_save_model_configs(model, save_dir)
- else:
- try:
- model.save_pretrained(save_dir, max_shard_size=max_shard_size, safe_serialization=safe_serialization)
- except (KeyError, TypeError) as e:
- # Some third-party configs fail during config serialization in save_pretrained.
- # Fall back to saving weights separately + config without diff.
- logger.warning("model.save_pretrained failed (%s), falling back to manual save.", e)
- from safetensors.torch import save_file
-
- state_dict = model.state_dict()
- save_file(state_dict, os.path.join(save_dir, "model.safetensors"))
- _save_model_configs(model, save_dir)
source_dir = _resolve_model_source_dir(model)
diff --git a/docs/step_by_step.md b/docs/step_by_step.md
index 06c905ec8..562b88d0e 100644
--- a/docs/step_by_step.md
+++ b/docs/step_by_step.md
@@ -356,12 +356,15 @@ ar.quantize_and_save(output_dir, format="auto_round:llm_compressor")
### AutoScheme
-AutoScheme provides an automatic algorithm to generate adaptive mixed bits/data-type quantization recipes. For some accuracy result, please refer this doc [here](./auto_scheme_acc.md)
+AutoScheme automatically generates adaptive mixed-bit and mixed-data-type quantization recipes. For accuracy results, see [AutoScheme Accuracy Report](./auto_scheme_acc.md).
-**Please note that mixed data types are supported during tuning, but cannot be exported to real models at this time..**
+**Note:** Mixed-data-types are supported during tuning, but cannot be exported to real models at this time.
#### CLI Usage
-use `iters=200`for tuning.
+
+- **`--iters 0`**: RTN. Fast (seconds to minutes).
+- **`--iters 200`**: Tuning-aware scheme selection. More accurate but much slower.
+
~~~bash
auto_round \
--model_name $model_name \
diff --git a/docs/step_by_step_CN.md b/docs/step_by_step_CN.md
index d74dcd486..c5a07fb79 100644
--- a/docs/step_by_step_CN.md
+++ b/docs/step_by_step_CN.md
@@ -353,12 +353,16 @@ ar.quantize_and_save(output_dir, format="auto_round:llm_compressor")
### AutoScheme 自动混合精度量化方案
-AutoScheme 采用自动化算法,可生成 **自适应的混合精度与数据类型** 的量化方案(mixed bits/data type quantization recipes)。相关测试结果请参考[《AutoScheme精度报告》](./auto_scheme_acc.md)。
-**注意**:混合数据类型方案在训练阶段可用,但当前版本还不支持导出至实际模型。
+AutoScheme 自动生成自适应的混合比特/混合数据类型量化方案。精度测试结果请参考 [AutoScheme 精度报告](./auto_scheme_acc.md)。
+
+**说明:** 混合数据类型支持调优,但目前无法将其导出到实际模型中。
#### 命令行用法
-如需启动训练建议设置 `iters=200`
+
+- **`--iters 0`**:基于 RTN 的 量化方案,速度快(秒到分钟级)。
+- **`--iters 200`**:调优感知的量化方案,更精确但慢很多。
+
~~~bash
auto_round \
--model_name $model_name \
diff --git a/test/test_cpu/export/test_export.py b/test/test_cpu/export/test_export.py
index 4f4c0d177..3b6c411eb 100644
--- a/test/test_cpu/export/test_export.py
+++ b/test/test_cpu/export/test_export.py
@@ -613,3 +613,50 @@ def test_fp8_autoround_export_packs_serially(tiny_opt_model_path, tmp_path, monk
autoround.save_quantized(output_dir=tmp_path, format="auto_round")
with safe_open(os.path.join(tmp_path, "model.safetensors"), framework="pt") as f:
assert "model.decoder.layers.0.self_attn.k_proj.weight_scale" in f.keys()
+
+
[email protected]("low_cpu_mem_usage", [True, False])
+def test_immediate_saving_mode(tiny_opt_model_path, tmp_path, low_cpu_mem_usage, caplog):
+ """Verify that immediate_saving (triggered by low_cpu_mem_usage) produces a complete model output."""
+ import logging
+
+ output_dir = str(tmp_path / "output")
+ with caplog.at_level(logging.DEBUG, logger="auto_round"):
+ autoround = AutoRound(
+ tiny_opt_model_path,
+ scheme="MXFP4",
+ iters=2,
+ seqlen=2,
+ nsamples=2,
+ low_cpu_mem_usage=low_cpu_mem_usage,
+ )
+ _, quantized_model_path = autoround.quantize_and_save(output_dir=output_dir, format="llm_compressor")
+
+ # No spurious "already exists" warning should be emitted
+ conflict_messages = [r.message for r in caplog.records if "already exists" in r.message]
+ assert len(conflict_messages) == 0, f"Unexpected conflict warnings: {conflict_messages}"
+
+ # All essential files must exist regardless of immediate_saving mode
+ assert os.path.exists(os.path.join(quantized_model_path, "config.json")), "config.json missing"
+ assert os.path.exists(
+ os.path.join(quantized_model_path, "quantization_config.json")
+ ), "quantization_config.json missing"
+
+ # Exactly 1 safetensors shard for this tiny model
+ safetensor_files = [f for f in os.listdir(quantized_model_path) if f.endswith(".safetensors")]
+ assert len(safetensor_files) == 1, f"Expected 1 safetensors file, got {len(safetensor_files)}: {safetensor_files}"
+
+ # Tokenizer files must be present
+ assert os.path.exists(os.path.join(quantized_model_path, "tokenizer_config.json")), "tokenizer_config.json missing"
+
+ # Total file count: config.json, quantization_config.json, model.safetensors,
+ # generation_config.json, tokenizer.json, tokenizer_config.json = 6
+ all_files = os.listdir(quantized_model_path)
+ assert len(all_files) == 6, f"Expected 6 files, got {len(all_files)}: {sorted(all_files)}"
+
+ # Verify weights are loadable and non-empty
+ from safetensors import safe_open
+
+ with safe_open(os.path.join(quantized_model_path, safetensor_files[0]), framework="pt") as f:
+ keys = f.keys()
+ assert len(keys) > 0, "Safetensors file has no tensors"