PTQ-Level Efficiency, QAT-Level Accuracy | 训练后量化的效率,量化感知训练的精度
fix awq exporting bugs (#365)
Wenhua Chengd080ea0af5f5
diff --git a/README.md b/README.mdindex 8ee725206..e9afcffa5 100644--- a/README.md+++ b/README.md@@ -296,7 +296,7 @@ in [Gaudi Guide](https://docs.habana.ai/en/latest/).from transformers import AutoModelForCausalLM, AutoTokenizerfrom auto_round import AutoRoundConfig-backend = "auto" ##cpu, hpu, cuda, cuda:marlin(supported in auto_round>0.3.1 and 'pip install -v gptqmodel --no-build-isolation')+backend = "auto" ##cpu, hpu, cudaquantization_config = AutoRoundConfig(backend=backend)diff --git a/auto_round/export/export_to_awq/export.py b/auto_round/export/export_to_awq/export.pyindex b82297c60..1921de446 100644--- a/auto_round/export/export_to_awq/export.py+++ b/auto_round/export/export_to_awq/export.py@@ -113,8 +113,8 @@ def save_quantized_as_autoawq(output_dir, inplace=True, **kwargs):quantization_config["modules_to_not_convert"] = modules_to_not_convert##check module quantized in block, this may have bug for mixed precision quantization- if hasattr(model, "config"):- model.config.quantization_config = quantization_config+ if hasattr(compressed_model, "config"):+ compressed_model.config.quantization_config = quantization_configsave(compressed_model, output_dir, safe_serialization=True)return compressed_modeldiff --git a/auto_round/utils.py b/auto_round/utils.pyindex cce826468..fc4189ef3 100644--- a/auto_round/utils.py+++ b/auto_round/utils.py@@ -1074,7 +1074,7 @@ def set_layer_config_by_fp_layers(model, fp_layers):if fp_layer in all_layer_names:not_to_quantized_layers.append(fp_layer)continue- if fp_layer[-1] != ".":+ if fp_layer[-1].isdigit():fp_layer = fp_layer + "." ##ticky settingfor name in all_layer_names:if fp_layer in name: