PTQ-Level Efficiency, QAT-Level Accuracy | 训练后量化的效率,量化感知训练的精度
fixed typos
wenhuach21ee975a2236bb
diff --git a/README.md b/README.mdindex 915a8f690..8c7a915dc 100644--- a/README.md+++ b/README.md@@ -33,19 +33,21 @@ pip install -r requirements.txt```## Uasage```python+from transformers import AutoModelForCausalLM, AutoTokenizerfrom auto_round import AutoRound-model_name = "facebook/opt-125m"+model_name = "facebook/opt-125m"model = AutoModelForCausalLM.from_pretrained(model_name, low_cpu_mem_usage=True, torch_dtype="auto", trust_remote_code=True)tokenizer = AutoTokenizer.from_pretrained(model_name, trust_remote_code=True)-autoround = AutoRound(model, tokenizer, num_bits=4, group_size=128, scheme="asym")++autoround = AutoRound(model, tokenizer, bits=4, group_size=128, scheme="asym")fake_qdq_model,weight_config = autoround.quantize() ##scale,zp info are saved in weight config dict## export to gpu# packed_folder = "./tmp_autoround_packed"-# autoround.export_to_autogptq("packed_folder")+# autoround.export_to_autogptq(packed_folder)```### Detailed Hyperparametersdiff --git a/auto_round/autoround.py b/auto_round/autoround.pyindex 766bde3db..4a5c95146 100644--- a/auto_round/autoround.py+++ b/auto_round/autoround.py@@ -1373,6 +1373,9 @@ class AutoRound(object):"""Export the model to autogptq format to easily leverage cuda kernel"""+ if not self.quantized:+ logger.warning("please run autoround.quantize first")+ returnmodel = copy.deepcopy(self.model.to("cpu")) ##TODO avoid this deepcopyfrom auto_gptq.modeling._utils import pack_model@@ -1457,6 +1460,7 @@ class AutoRound(object):end_time = time.time()cost_time = end_time - start_timelogger.info(f"quantization tuning time {cost_time}")+ self.quantized = True##self.export_to_autogptq("test_export")return self.model, self.weight_config