Skip to content

PTQ-Level Efficiency, QAT-Level Accuracy | 训练后量化的效率,量化感知训练的精度

dynamic map checkpoint naming based on model objective. (#1840)

Xin He6c76487589f1
diff --git a/auto_round/inference/convert_model.py b/auto_round/inference/convert_model.py
index f8cdea026..7d19b1e3e 100644
--- a/auto_round/inference/convert_model.py
+++ b/auto_round/inference/convert_model.py
@@ -287,13 +287,28 @@ def get_layer_config(model, quantization_config):
# Determine the quantization block list
checkpoint_conversion_mapping = get_checkpoint_conversion_mapping(model)
+
+ # Determine whether to apply the conversion mapping.
+ # If the model's module paths match the source patterns of the mapping, the model is
+ # a composite model (e.g., VLM loaded via AutoModelForImageTextToText) whose paths are
+ # already in checkpoint namespace — remapping would incorrectly alter them.
+ # Only when the model is loaded as a text sub-model (e.g., via AutoModelForCausalLM)
+ # do its paths differ from checkpoint namespace and require remapping.
+ _should_remap = bool(checkpoint_conversion_mapping) and not any(
+ re.match(src, name) for name, _ in model.named_modules() for src in checkpoint_conversion_mapping
+ )
+
quant_block_list = getattr(quantization_config, "quant_block_list", None)
if quant_block_list is not None:
# Handle nested list format: [[block1, block2, ...], ...] -> [prefix1, ...]
if quant_block_list and isinstance(quant_block_list[0], (list, tuple)):
for i in range(len(quant_block_list)):
- quant_block_list[i] = apply_checkpoint_conversion_mapping(
- os.path.commonprefix(quant_block_list[i]).rstrip("."), checkpoint_conversion_mapping
+ quant_block_list[i] = (
+ apply_checkpoint_conversion_mapping(
+ os.path.commonprefix(quant_block_list[i]).rstrip("."), checkpoint_conversion_mapping
+ )
+ if _should_remap
+ else os.path.commonprefix(quant_block_list[i]).rstrip(".")
)
elif quant_block_list is None:
to_quant_block_names = getattr(quantization_config, "block_name_to_quantize", None) # Prioritize this parameter
@@ -311,10 +326,11 @@ def get_layer_config(model, quantization_config):
# Speed up the matching
for i in range(len(quant_block_list)):
quant_block_list[i] = os.path.commonprefix(quant_block_list[i]).rstrip(".")
- for i in range(len(quant_block_list)):
- quant_block_list[i] = apply_checkpoint_conversion_mapping(
- quant_block_list[i], checkpoint_conversion_mapping
- )
+ if _should_remap:
+ for i in range(len(quant_block_list)):
+ quant_block_list[i] = apply_checkpoint_conversion_mapping(
+ quant_block_list[i], checkpoint_conversion_mapping
+ )
# Get layer names that will be quantized
layer_names = []
@@ -328,7 +344,7 @@ def get_layer_config(model, quantization_config):
extra_config = getattr(quantization_config, "extra_config", {})
# Remap extra_config keys using conversion mapping (e.g. composite VLM paths to text sub-model paths)
- if checkpoint_conversion_mapping and extra_config:
+ if _should_remap and extra_config:
remapped_extra_config = {}
for key, value in extra_config.items():
new_key = apply_checkpoint_conversion_mapping(key, checkpoint_conversion_mapping)
diff --git a/test/helpers.py b/test/helpers.py
index e16b758ab..fea5fdfc0 100644
--- a/test/helpers.py
+++ b/test/helpers.py
@@ -63,7 +63,14 @@ def eval_generated_prompt(
def evaluate_accuracy(
- model_or_save_dir, tokenizer=None, task="lambada_openai", threshold=0.25, batch_size="auto", limit=None, device=None
+ model_or_save_dir,
+ tokenizer=None,
+ task="lambada_openai",
+ threshold=0.25,
+ batch_size="auto",
+ limit=None,
+ device=None,
+ model_type="hf",
):
"""Helper function to evaluate model accuracy on a given task.
@@ -86,7 +93,9 @@ def evaluate_accuracy(
if isinstance(model_or_save_dir, str):
# save_dir mode
model_args = f"pretrained={model_or_save_dir}"
- result = simple_evaluate(model="hf", model_args=model_args, tasks=task, batch_size=batch_size, device=device)
+ result = simple_evaluate(
+ model=model_type, model_args=model_args, tasks=task, batch_size=batch_size, limit=limit, device=device
+ )
else:
# model object mode
if tokenizer is None:
diff --git a/test/test_cuda/export/test_fp8_format.py b/test/test_cuda/export/test_fp8_format.py
index 0d832e372..7758cf0db 100644
--- a/test/test_cuda/export/test_fp8_format.py
+++ b/test/test_cuda/export/test_fp8_format.py
@@ -43,5 +43,7 @@ class TestAutoRoundBlockFP:
assert list(tmp_layer.weight_scale_inv.shape) == [16, 8]
assert compressed_model.config.quantization_config["quant_method"] == "fp8"
assert compressed_model.config.quantization_config["weight_block_size"] == (128, 128)
- if is_cuda_support_fp8():
- eval_generated_prompt(quantized_model_path, device="cuda")
+ # TODO: open below test after this issue is fixed.
+ # https://github.com/huggingface/transformers/issues/46209
+ # if is_cuda_support_fp8():
+ # eval_generated_prompt(quantized_model_path, device="cuda")
diff --git a/test/test_cuda/integrations/test_huggingface.py b/test/test_cuda/integrations/test_huggingface.py
index e027977d8..bd1324393 100644
--- a/test/test_cuda/integrations/test_huggingface.py
+++ b/test/test_cuda/integrations/test_huggingface.py
@@ -6,5 +6,15 @@ model_name_or_path = "Intel/Qwen3.5-2B-int4-AutoRound"
@pytest.mark.skip_ci(reason="Requires downloading a large model from HuggingFace")
-def test_evaluate_accuracy():
- evaluate_accuracy(model_name_or_path, threshold=0.5)
+def test_hf():
+ evaluate_accuracy(
+ model_name_or_path,
+ threshold=0.4,
+ model_type="hf",
+ limit=100,
+ )
+
+
[email protected]_ci(reason="Requires downloading a large model from HuggingFace")
+def test_hf_multimodal():
+ evaluate_accuracy(model_name_or_path, threshold=0.4, batch_size=8, model_type="hf-multimodal", limit=100)