Skip to content

PTQ-Level Efficiency, QAT-Level Accuracy | 训练后量化的效率,量化感知训练的精度

Add handle_generation_config function to manage model generation_config saving failure (#1448)

Xin He32c1e3c109ef
diff --git a/auto_round/export/export_to_llmcompressor/export.py b/auto_round/export/export_to_llmcompressor/export.py
index 47e545ff9..6ea8eca1d 100644
--- a/auto_round/export/export_to_llmcompressor/export.py
+++ b/auto_round/export/export_to_llmcompressor/export.py
@@ -166,12 +166,7 @@ def save_quantized_as_llmcompressor(
model.config.quantization_config = quantization_config.to_dict()
model.config.save_pretrained(output_dir)
- try:
- save_model(model, output_dir, safe_serialization=safe_serialization)
- except ValueError as e:
- if hasattr(model, "generation_config"):
- setattr(model.generation_config, "do_sample", True)
- save_model(model, output_dir, safe_serialization=safe_serialization)
+ save_model(model, output_dir, safe_serialization=safe_serialization)
try:
copy_python_files_from_model_cache(model, output_dir)
diff --git a/auto_round/export/utils.py b/auto_round/export/utils.py
index aec990b37..42ae86d5f 100644
--- a/auto_round/export/utils.py
+++ b/auto_round/export/utils.py
@@ -53,16 +53,9 @@ def save_model(
model.config.save_pretrained(save_dir)
if hasattr(model, "generation_config") and model.generation_config is not None:
- if hasattr(model, "generation_config"):
- setattr(model.generation_config, "do_sample", True)
model.generation_config.save_pretrained(save_dir)
else:
- try:
- model.save_pretrained(save_dir, max_shard_size=max_shard_size, safe_serialization=safe_serialization)
- except ValueError as e:
- if hasattr(model, "generation_config"):
- setattr(model.generation_config, "do_sample", True)
- model.save_pretrained(save_dir, max_shard_size=max_shard_size, safe_serialization=safe_serialization)
+ model.save_pretrained(save_dir, max_shard_size=max_shard_size, safe_serialization=safe_serialization)
config_path = os.path.join(save_dir, "config.json")
if dtype is not None and dtype != model.dtype and os.path.exists(os.path.join(save_dir, "config.json")):
diff --git a/auto_round/utils/model.py b/auto_round/utils/model.py
index baceddecd..f17398fcc 100644
--- a/auto_round/utils/model.py
+++ b/auto_round/utils/model.py
@@ -333,6 +333,7 @@ def llm_load_model(
model = model.eval()
check_and_mark_quantized_module(model)
+ handle_generation_config(model)
model = _to_model_dtype(model, model_dtype)
return model, tokenizer
@@ -482,6 +483,7 @@ def mllm_load_model(
model = model.eval()
check_and_mark_quantized_module(model)
+ handle_generation_config(model)
model = _to_model_dtype(model, model_dtype)
return model, processor, tokenizer, image_processor
@@ -1510,3 +1512,14 @@ def is_separate_tensor(model: torch.nn.Module, tensor_name: str) -> bool:
return True
else:
return False
+
+
+def handle_generation_config(model: torch.nn.Module):
+ if hasattr(model, "generation_config"):
+ generation_config = model.generation_config
+ if hasattr(generation_config, "top_p") and generation_config.top_p != 1.0:
+ model.generation_config.do_sample = True
+ if hasattr(generation_config, "top_k") and generation_config.top_k != 0:
+ model.generation_config.do_sample = True
+ if hasattr(generation_config, "temperature") and generation_config.temperature != 1.0:
+ model.generation_config.do_sample = True