Skip to content

PTQ-Level Efficiency, QAT-Level Accuracy | 训练后量化的效率,量化感知训练的精度

fixed typos

wenhuach21ee975a2236bb
diff --git a/README.md b/README.md
index 915a8f690..8c7a915dc 100644
--- a/README.md
+++ b/README.md
@@ -33,19 +33,21 @@ pip install -r requirements.txt
```
## Uasage
```python
+from transformers import AutoModelForCausalLM, AutoTokenizer
from auto_round import AutoRound
-model_name = "facebook/opt-125m"
+model_name = "facebook/opt-125m"
model = AutoModelForCausalLM.from_pretrained(
model_name, low_cpu_mem_usage=True, torch_dtype="auto", trust_remote_code=True
)
tokenizer = AutoTokenizer.from_pretrained(model_name, trust_remote_code=True)
-autoround = AutoRound(model, tokenizer, num_bits=4, group_size=128, scheme="asym")
+
+autoround = AutoRound(model, tokenizer, bits=4, group_size=128, scheme="asym")
fake_qdq_model,weight_config = autoround.quantize() ##scale,zp info are saved in weight config dict
## export to gpu
# packed_folder = "./tmp_autoround_packed"
-# autoround.export_to_autogptq("packed_folder")
+# autoround.export_to_autogptq(packed_folder)
```
### Detailed Hyperparameters
diff --git a/auto_round/autoround.py b/auto_round/autoround.py
index 766bde3db..4a5c95146 100644
--- a/auto_round/autoround.py
+++ b/auto_round/autoround.py
@@ -1373,6 +1373,9 @@ class AutoRound(object):
"""
Export the model to autogptq format to easily leverage cuda kernel
"""
+ if not self.quantized:
+ logger.warning("please run autoround.quantize first")
+ return
model = copy.deepcopy(self.model.to("cpu")) ##TODO avoid this deepcopy
from auto_gptq.modeling._utils import pack_model
@@ -1457,6 +1460,7 @@ class AutoRound(object):
end_time = time.time()
cost_time = end_time - start_time
logger.info(f"quantization tuning time {cost_time}")
+ self.quantized = True
##self.export_to_autogptq("test_export")
return self.model, self.weight_config