Skip to content

PTQ-Level Efficiency, QAT-Level Accuracy | 训练后量化的效率,量化感知训练的精度

support gaudi2. (#7)

lkk3fc34ed91e09
diff --git a/README.md b/README.md
index de9deca4d..18f777471 100644
--- a/README.md
+++ b/README.md
@@ -22,6 +22,31 @@ pip install -r requirements.txt
python setup.py install
```
## Usage
+
+
+### On Intel Gaudi2
+
+```python
+from transformers import AutoModelForCausalLM, AutoTokenizer
+
+model_name = "facebook/opt-125m"
+model = AutoModelForCausalLM.from_pretrained(
+ model_name, low_cpu_mem_usage=True, torch_dtype="auto", trust_remote_code=True
+ )
+tokenizer = AutoTokenizer.from_pretrained(model_name, trust_remote_code=True)
+bits, group_size, scheme = 4, 128, "asym"
+
+# need load model first, them import
+from auto_round import AutoRound
+
+autoround = AutoRound(model, tokenizer, bits=bits, group_size=group_size, scheme=scheme,
+ device="hpu", scale_dtype="bf16", amp=False)
+autoround.quantize()
+
+```
+
+### On GPU
+
```python
from transformers import AutoModelForCausalLM, AutoTokenizer
from auto_round import AutoRound
diff --git a/auto_round/autoround.py b/auto_round/autoround.py
index 6653889ff..7d7ab1945 100644
--- a/auto_round/autoround.py
+++ b/auto_round/autoround.py
@@ -33,6 +33,8 @@ from .utils import (quant_weight, set_module, get_module, get_block_names, block
get_dataloader, get_scale_shape, move_input_to_device, check_is_cpu, collect_round_v,
collect_minmax_scale, get_batch_dim)
+from .device_utils import *
+
class SaveInputs:
"""Cache the inputs of the first block."""
@@ -641,6 +643,8 @@ class AutoRound(object):
"""
scale_loss = loss * 1000
scale_loss.backward()
+ if is_hpu_available:
+ htcore.mark_step()
return scale_loss
def step(self, scaler, optimizer, lr_schedule):
@@ -655,6 +659,9 @@ class AutoRound(object):
None
"""
optimizer.step()
+ # for hpu
+ if is_hpu_available:
+ htcore.mark_step()
optimizer.zero_grad()
lr_schedule.step()
@@ -1186,12 +1193,12 @@ class AutoOPTRound(AutoRound):
def scale_loss_and_backward(self, scaler, loss):
if scaler is not None:
- scale_loss = scaler.scale(loss)
- scale_loss.backward()
- return scale_loss
- else:
- loss.backward()
- return loss
+ loss = scaler.scale(loss)
+
+ loss.backward()
+ if is_hpu_available:
+ htcore.mark_step()
+ return loss
def step(self, scaler, optimizer, lr_schedule):
if scaler is not None:
@@ -1203,6 +1210,8 @@ class AutoOPTRound(AutoRound):
optimizer.step()
optimizer.zero_grad()
lr_schedule.step()
+ if is_hpu_available:
+ htcore.mark_step()
class AutoAdamRound(AutoOPTRound):
diff --git a/auto_round/device_utils.py b/auto_round/device_utils.py
new file mode 100644
index 000000000..2436aea0e
--- /dev/null
+++ b/auto_round/device_utils.py
@@ -0,0 +1,33 @@
+
+# Copyright (c) 2023 Intel Corporation
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+
+def is_optimum_habana_available():
+ import importlib
+ from transformers.utils.import_utils import is_optimum_available
+
+ return is_optimum_available() and importlib.util.find_spec("optimum.habana") != None
+
+try:
+ import habana_frameworks.torch.hpu as hthpu
+ import habana_frameworks.torch.core as htcore
+ if is_optimum_habana_available():
+ is_hpu_available = True
+ else:
+ print("Should install optimum-habana when the environment has habana frameworks")
+ is_hpu_available = False
+except ImportError:
+ is_hpu_available = False
+