Skip to content

PTQ-Level Efficiency, QAT-Level Accuracy | 训练后量化的效率,量化感知训练的精度

update readme

wenhuach219f0db7eaa648
diff --git a/README.md b/README.md
index d3a42a3bd..8a5a00a3f 100644
--- a/README.md
+++ b/README.md
@@ -24,51 +24,64 @@ python setup.py install
## Usage
-### On Intel Gaudi2
+
+
+### On CPU/GPU
```python
from transformers import AutoModelForCausalLM, AutoTokenizer
+from auto_round import AutoRound
-model_name = "bigscience/bloom-560m"
+model_name = "meta-llama/Llama-2-7b-hf"
model = AutoModelForCausalLM.from_pretrained(
model_name, low_cpu_mem_usage=True, torch_dtype="auto", trust_remote_code=True)
tokenizer = AutoTokenizer.from_pretrained(model_name, trust_remote_code=True)
bits, group_size, scheme = 4, 128, "asym"
-
-# need to load model first, then import
-from auto_round import AutoRound
-autoround = AutoRound(model, tokenizer, bits=bits, group_size=group_size, scheme=scheme,
- device="hpu", scale_dtype="bf16", amp=False)
+autoround = AutoRound(model, tokenizer, bits=bits, group_size=group_size, scheme=scheme)
autoround.quantize()
-# Intel CPU Inference, Currently, llama, bloom, and mistral are supported.
-output_dir = "/path/to/quantized_model"
-autoround.export(output_dir)
+# Intel CPU Inference, For now only support llama, mistral and gpt-j.
# then follow ITREX(https://github.com/intel/intel-extension-for-transformers/tree/main/intel_extension_for_transformers/llm/runtime/neural_speed) to load the model and do inference
+# currently please install neural-speed (https://github.com/intel/neural-speed) from source
+output_dir = "./tmp_autoround"
+autoround.export(output_dir)
+from intel_extension_for_transformers.transformers import AutoModelForCausalLM, WeightOnlyQuantConfig
+
+woq_config = WeightOnlyQuantConfig(group_size=group_size, scheme=scheme, use_autoround=True) ##only supports 4 bits currently
+prompt = "Once upon a time, a little girl"
+
+tokenizer = AutoTokenizer.from_pretrained(model_name, trust_remote_code=True)
+inputs = tokenizer(prompt, return_tensors="pt").input_ids
+model = AutoModelForCausalLM.from_pretrained(model_name, quantization_config=woq_config, trust_remote_code=True,device="cpu")
+outputs = model.generate(inputs, max_new_tokens=30)
```
-### On CPU/GPU
+
+### Tuning on Intel Gaudi2
```python
from transformers import AutoModelForCausalLM, AutoTokenizer
-from auto_round import AutoRound
-model_name = "bigscience/bloom-560m"
+model_name = "meta-llama/Llama-2-7b-hf"
model = AutoModelForCausalLM.from_pretrained(
model_name, low_cpu_mem_usage=True, torch_dtype="auto", trust_remote_code=True)
tokenizer = AutoTokenizer.from_pretrained(model_name, trust_remote_code=True)
bits, group_size, scheme = 4, 128, "asym"
-autoround = AutoRound(model, tokenizer, bits=bits, group_size=group_size, scheme=scheme)
+
+# need to load model first, then import
+from auto_round import AutoRound
+autoround = AutoRound(model, tokenizer, bits=bits, group_size=group_size, scheme=scheme,
+ device="hpu", scale_dtype="bf16", amp=False)
autoround.quantize()
# Intel CPU Inference, Currently, llama, bloom, and mistral are supported.
output_dir = "/path/to/quantized_model"
autoround.export(output_dir)
# then follow ITREX(https://github.com/intel/intel-extension-for-transformers/tree/main/intel_extension_for_transformers/llm/runtime/neural_speed) to load the model and do inference
-
```
+
<details>
<summary>Detailed Hyperparameters</summary>