PTQ-Level Efficiency, QAT-Level Accuracy | 训练后量化的效率,量化感知训练的精度
support gaudi2. (#7)
lkk3fc34ed91e09
diff --git a/README.md b/README.mdindex de9deca4d..18f777471 100644--- a/README.md+++ b/README.md@@ -22,6 +22,31 @@ pip install -r requirements.txtpython setup.py install```## Usage+++### On Intel Gaudi2++```python+from transformers import AutoModelForCausalLM, AutoTokenizer++model_name = "facebook/opt-125m"+model = AutoModelForCausalLM.from_pretrained(+ model_name, low_cpu_mem_usage=True, torch_dtype="auto", trust_remote_code=True+ )+tokenizer = AutoTokenizer.from_pretrained(model_name, trust_remote_code=True)+bits, group_size, scheme = 4, 128, "asym"++# need load model first, them import+from auto_round import AutoRound++autoround = AutoRound(model, tokenizer, bits=bits, group_size=group_size, scheme=scheme,+ device="hpu", scale_dtype="bf16", amp=False)+autoround.quantize()++```++### On GPU+```pythonfrom transformers import AutoModelForCausalLM, AutoTokenizerfrom auto_round import AutoRounddiff --git a/auto_round/autoround.py b/auto_round/autoround.pyindex 6653889ff..7d7ab1945 100644--- a/auto_round/autoround.py+++ b/auto_round/autoround.py@@ -33,6 +33,8 @@ from .utils import (quant_weight, set_module, get_module, get_block_names, blockget_dataloader, get_scale_shape, move_input_to_device, check_is_cpu, collect_round_v,collect_minmax_scale, get_batch_dim)+from .device_utils import *+class SaveInputs:"""Cache the inputs of the first block."""@@ -641,6 +643,8 @@ class AutoRound(object):"""scale_loss = loss * 1000scale_loss.backward()+ if is_hpu_available:+ htcore.mark_step()return scale_lossdef step(self, scaler, optimizer, lr_schedule):@@ -655,6 +659,9 @@ class AutoRound(object):None"""optimizer.step()+ # for hpu+ if is_hpu_available:+ htcore.mark_step()optimizer.zero_grad()lr_schedule.step()@@ -1186,12 +1193,12 @@ class AutoOPTRound(AutoRound):def scale_loss_and_backward(self, scaler, loss):if scaler is not None:- scale_loss = scaler.scale(loss)- scale_loss.backward()- return scale_loss- else:- loss.backward()- return loss+ loss = scaler.scale(loss)++ loss.backward()+ if is_hpu_available:+ htcore.mark_step()+ return lossdef step(self, scaler, optimizer, lr_schedule):if scaler is not None:@@ -1203,6 +1210,8 @@ class AutoOPTRound(AutoRound):optimizer.step()optimizer.zero_grad()lr_schedule.step()+ if is_hpu_available:+ htcore.mark_step()class AutoAdamRound(AutoOPTRound):diff --git a/auto_round/device_utils.py b/auto_round/device_utils.pynew file mode 100644index 000000000..2436aea0e--- /dev/null+++ b/auto_round/device_utils.py@@ -0,0 +1,33 @@++# Copyright (c) 2023 Intel Corporation+#+# Licensed under the Apache License, Version 2.0 (the "License");+# you may not use this file except in compliance with the License.+# You may obtain a copy of the License at+#+# http://www.apache.org/licenses/LICENSE-2.0+#+# Unless required by applicable law or agreed to in writing, software+# distributed under the License is distributed on an "AS IS" BASIS,+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+# See the License for the specific language governing permissions and+# limitations under the License.+++def is_optimum_habana_available():+ import importlib+ from transformers.utils.import_utils import is_optimum_available++ return is_optimum_available() and importlib.util.find_spec("optimum.habana") != None++try:+ import habana_frameworks.torch.hpu as hthpu+ import habana_frameworks.torch.core as htcore+ if is_optimum_habana_available():+ is_hpu_available = True+ else:+ print("Should install optimum-habana when the environment has habana frameworks")+ is_hpu_available = False+except ImportError:+ is_hpu_available = False+