PTQ-Level Efficiency, QAT-Level Accuracy | 训练后量化的效率,量化感知训练的精度
Add handle_generation_config function to manage model generation_config saving failure (#1448)
Xin He32c1e3c109ef
diff --git a/auto_round/export/export_to_llmcompressor/export.py b/auto_round/export/export_to_llmcompressor/export.pyindex 47e545ff9..6ea8eca1d 100644--- a/auto_round/export/export_to_llmcompressor/export.py+++ b/auto_round/export/export_to_llmcompressor/export.py@@ -166,12 +166,7 @@ def save_quantized_as_llmcompressor(model.config.quantization_config = quantization_config.to_dict()model.config.save_pretrained(output_dir)- try:- save_model(model, output_dir, safe_serialization=safe_serialization)- except ValueError as e:- if hasattr(model, "generation_config"):- setattr(model.generation_config, "do_sample", True)- save_model(model, output_dir, safe_serialization=safe_serialization)+ save_model(model, output_dir, safe_serialization=safe_serialization)try:copy_python_files_from_model_cache(model, output_dir)diff --git a/auto_round/export/utils.py b/auto_round/export/utils.pyindex aec990b37..42ae86d5f 100644--- a/auto_round/export/utils.py+++ b/auto_round/export/utils.py@@ -53,16 +53,9 @@ def save_model(model.config.save_pretrained(save_dir)if hasattr(model, "generation_config") and model.generation_config is not None:- if hasattr(model, "generation_config"):- setattr(model.generation_config, "do_sample", True)model.generation_config.save_pretrained(save_dir)else:- try:- model.save_pretrained(save_dir, max_shard_size=max_shard_size, safe_serialization=safe_serialization)- except ValueError as e:- if hasattr(model, "generation_config"):- setattr(model.generation_config, "do_sample", True)- model.save_pretrained(save_dir, max_shard_size=max_shard_size, safe_serialization=safe_serialization)+ model.save_pretrained(save_dir, max_shard_size=max_shard_size, safe_serialization=safe_serialization)config_path = os.path.join(save_dir, "config.json")if dtype is not None and dtype != model.dtype and os.path.exists(os.path.join(save_dir, "config.json")):diff --git a/auto_round/utils/model.py b/auto_round/utils/model.pyindex baceddecd..f17398fcc 100644--- a/auto_round/utils/model.py+++ b/auto_round/utils/model.py@@ -333,6 +333,7 @@ def llm_load_model(model = model.eval()check_and_mark_quantized_module(model)+ handle_generation_config(model)model = _to_model_dtype(model, model_dtype)return model, tokenizer@@ -482,6 +483,7 @@ def mllm_load_model(model = model.eval()check_and_mark_quantized_module(model)+ handle_generation_config(model)model = _to_model_dtype(model, model_dtype)return model, processor, tokenizer, image_processor@@ -1510,3 +1512,14 @@ def is_separate_tensor(model: torch.nn.Module, tensor_name: str) -> bool:return Trueelse:return False+++def handle_generation_config(model: torch.nn.Module):+ if hasattr(model, "generation_config"):+ generation_config = model.generation_config+ if hasattr(generation_config, "top_p") and generation_config.top_p != 1.0:+ model.generation_config.do_sample = True+ if hasattr(generation_config, "top_k") and generation_config.top_k != 0:+ model.generation_config.do_sample = True+ if hasattr(generation_config, "temperature") and generation_config.temperature != 1.0:+ model.generation_config.do_sample = True