PTQ-Level Efficiency, QAT-Level Accuracy | 训练后量化的效率,量化感知训练的精度
add example for gaudi2. (#24)
lkkafcd62f11739
diff --git a/README.md b/README.mdindex eb8516e7d..ca793d669 100644--- a/README.md+++ b/README.md@@ -49,16 +49,17 @@ else:```pythonfrom transformers import AutoModelForCausalLM, AutoTokenizer+import torchmodel_name = "meta-llama/Llama-2-7b-hf"-model = AutoModelForCausalLM.from_pretrained(model_name, torch_dtype="auto", trust_remote_code=True)+model = AutoModelForCausalLM.from_pretrained(model_name, torch_dtype=torch.bfloat16, trust_remote_code=True)tokenizer = AutoTokenizer.from_pretrained(model_name, trust_remote_code=True)bits, group_size, scheme = 4, 128, "asym"# need to load model first, then importfrom auto_round import AutoRound-autoround = AutoRound(model, tokenizer, bits=bits, group_size=group_size, scheme=scheme, device="hpu", amp=False)+autoround = AutoRound(model, tokenizer, bits=bits, group_size=group_size, scheme=scheme, device="hpu", amp=True)autoround.quantize()```diff --git a/auto_round/autoround.py b/auto_round/autoround.pyindex cf044c520..429403f67 100644--- a/auto_round/autoround.py+++ b/auto_round/autoround.py@@ -37,6 +37,10 @@ from .utils import (set_module,)+if is_hpu_available:+ import habana_frameworks.torch.core as htcore # pylint: disable=E0401+ import habana_frameworks.torch.hpu as hthpu # pylint: disable=E0401+class SaveInputs:"""Cache the inputs of the first block."""diff --git a/examples/language-modeling/README.md b/examples/language-modeling/README.mdindex e5daec3e4..742783761 100644--- a/examples/language-modeling/README.md+++ b/examples/language-modeling/README.md@@ -57,6 +57,11 @@ Include the flag `--adam`. Note that AdamW is less effective than Sign gradient- **Running the Original SignRound:**```bashCUDA_VISIBLE_DEVICES=0 python3 main.py --model_name facebook/opt-125m --amp --bits 4 --group_size -1 --iters 400 --lr 0.0025 --minmax_lr 0.0025+```++- **Running on Intel Gaudi2**+```bash+bash run_autoround_on_gaudi.sh````--enable_minmax_tuning` is strongly recommendeddiff --git a/examples/language-modeling/main.py b/examples/language-modeling/main.pyindex b4b224b12..d4733402a 100644--- a/examples/language-modeling/main.py+++ b/examples/language-modeling/main.py@@ -3,9 +3,6 @@ import syssys.path.insert(0, '../..')-from auto_round import (AutoRound,- AutoAdamRound)-parser = argparse.ArgumentParser()import torchimport os@@ -150,24 +147,31 @@ if __name__ == '__main__':if model_name[-1] == "/":model_name = model_name[:-1]print(model_name, flush=True)-+ torch_dtype = "auto"if args.device == "cpu":device_str = "cpu"+ elif args.device == "hpu":+ device_str = "hpu"+ torch_dtype = torch.bfloat16else:device_str = f"cuda:{int(args.device)}"+torch_device = torch.device(device_str)is_glm = bool(re.search("chatglm", model_name.lower()))is_llava = bool(re.search("llava", model_name.lower()))if is_llava:from transformers import LlavaForConditionalGeneration- model = LlavaForConditionalGeneration.from_pretrained(model_name, low_cpu_mem_usage=True, torch_dtype="auto")+ model = LlavaForConditionalGeneration.from_pretrained(model_name, low_cpu_mem_usage=True, torch_dtype=torch_dtype)elif is_glm:model = AutoModel.from_pretrained(model_name, trust_remote_code=True)else:model = AutoModelForCausalLM.from_pretrained(- model_name, low_cpu_mem_usage=True, torch_dtype="auto", trust_remote_code=True+ model_name, low_cpu_mem_usage=True, torch_dtype=torch_dtype, trust_remote_code=True)++ from auto_round import (AutoRound,+ AutoAdamRound)model = model.eval()# align with GPTQ to eval pplif "opt" in model_name:diff --git a/examples/language-modeling/run_autoround_on_gaudi.sh b/examples/language-modeling/run_autoround_on_gaudi.shnew file mode 100644index 000000000..1a063964a--- /dev/null+++ b/examples/language-modeling/run_autoround_on_gaudi.sh@@ -0,0 +1,16 @@+#!/bin/bash+set -x+model_name="Intel/neural-chat-7b-v3-3"+eval_bs=1++python3 main.py \+ --amp \+ --model_name $model_name \+ --device "hpu" \+ --group_size 128 \+ --bits 4 \+ --enable_minmax_tuning \+ --use_quant_input \+ --eval_bs $eval_bs \+ --output_dir "./tmp_autoround"+