Skip to content

PTQ-Level Efficiency, QAT-Level Accuracy | 训练后量化的效率,量化感知训练的精度

add example for gaudi2. (#24)

lkkafcd62f11739
diff --git a/README.md b/README.md
index eb8516e7d..ca793d669 100644
--- a/README.md
+++ b/README.md
@@ -49,16 +49,17 @@ else:
```python
from transformers import AutoModelForCausalLM, AutoTokenizer
+import torch
model_name = "meta-llama/Llama-2-7b-hf"
-model = AutoModelForCausalLM.from_pretrained(model_name, torch_dtype="auto", trust_remote_code=True)
+model = AutoModelForCausalLM.from_pretrained(model_name, torch_dtype=torch.bfloat16, trust_remote_code=True)
tokenizer = AutoTokenizer.from_pretrained(model_name, trust_remote_code=True)
bits, group_size, scheme = 4, 128, "asym"
# need to load model first, then import
from auto_round import AutoRound
-autoround = AutoRound(model, tokenizer, bits=bits, group_size=group_size, scheme=scheme, device="hpu", amp=False)
+autoround = AutoRound(model, tokenizer, bits=bits, group_size=group_size, scheme=scheme, device="hpu", amp=True)
autoround.quantize()
```
diff --git a/auto_round/autoround.py b/auto_round/autoround.py
index cf044c520..429403f67 100644
--- a/auto_round/autoround.py
+++ b/auto_round/autoround.py
@@ -37,6 +37,10 @@ from .utils import (
set_module,
)
+if is_hpu_available:
+ import habana_frameworks.torch.core as htcore # pylint: disable=E0401
+ import habana_frameworks.torch.hpu as hthpu # pylint: disable=E0401
+
class SaveInputs:
"""Cache the inputs of the first block."""
diff --git a/examples/language-modeling/README.md b/examples/language-modeling/README.md
index e5daec3e4..742783761 100644
--- a/examples/language-modeling/README.md
+++ b/examples/language-modeling/README.md
@@ -57,6 +57,11 @@ Include the flag `--adam`. Note that AdamW is less effective than Sign gradient
- **Running the Original SignRound:**
```bash
CUDA_VISIBLE_DEVICES=0 python3 main.py --model_name facebook/opt-125m --amp --bits 4 --group_size -1 --iters 400 --lr 0.0025 --minmax_lr 0.0025
+```
+
+- **Running on Intel Gaudi2**
+```bash
+bash run_autoround_on_gaudi.sh
```
`--enable_minmax_tuning` is strongly recommended
diff --git a/examples/language-modeling/main.py b/examples/language-modeling/main.py
index b4b224b12..d4733402a 100644
--- a/examples/language-modeling/main.py
+++ b/examples/language-modeling/main.py
@@ -3,9 +3,6 @@ import sys
sys.path.insert(0, '../..')
-from auto_round import (AutoRound,
- AutoAdamRound)
-
parser = argparse.ArgumentParser()
import torch
import os
@@ -150,24 +147,31 @@ if __name__ == '__main__':
if model_name[-1] == "/":
model_name = model_name[:-1]
print(model_name, flush=True)
-
+ torch_dtype = "auto"
if args.device == "cpu":
device_str = "cpu"
+ elif args.device == "hpu":
+ device_str = "hpu"
+ torch_dtype = torch.bfloat16
else:
device_str = f"cuda:{int(args.device)}"
+
torch_device = torch.device(device_str)
is_glm = bool(re.search("chatglm", model_name.lower()))
is_llava = bool(re.search("llava", model_name.lower()))
if is_llava:
from transformers import LlavaForConditionalGeneration
- model = LlavaForConditionalGeneration.from_pretrained(model_name, low_cpu_mem_usage=True, torch_dtype="auto")
+ model = LlavaForConditionalGeneration.from_pretrained(model_name, low_cpu_mem_usage=True, torch_dtype=torch_dtype)
elif is_glm:
model = AutoModel.from_pretrained(model_name, trust_remote_code=True)
else:
model = AutoModelForCausalLM.from_pretrained(
- model_name, low_cpu_mem_usage=True, torch_dtype="auto", trust_remote_code=True
+ model_name, low_cpu_mem_usage=True, torch_dtype=torch_dtype, trust_remote_code=True
)
+
+ from auto_round import (AutoRound,
+ AutoAdamRound)
model = model.eval()
# align with GPTQ to eval ppl
if "opt" in model_name:
diff --git a/examples/language-modeling/run_autoround_on_gaudi.sh b/examples/language-modeling/run_autoround_on_gaudi.sh
new file mode 100644
index 000000000..1a063964a
--- /dev/null
+++ b/examples/language-modeling/run_autoround_on_gaudi.sh
@@ -0,0 +1,16 @@
+#!/bin/bash
+set -x
+model_name="Intel/neural-chat-7b-v3-3"
+eval_bs=1
+
+python3 main.py \
+ --amp \
+ --model_name $model_name \
+ --device "hpu" \
+ --group_size 128 \
+ --bits 4 \
+ --enable_minmax_tuning \
+ --use_quant_input \
+ --eval_bs $eval_bs \
+ --output_dir "./tmp_autoround"
+