Skip to content

PTQ-Level Efficiency, QAT-Level Accuracy | 训练后量化的效率,量化感知训练的精度

Fix hpu autocast (#64)

wenhuach21c7d2fa84ce6c
diff --git a/README.md b/README.md
index 053e0dc2f..9becb2c12 100644
--- a/README.md
+++ b/README.md
@@ -47,7 +47,7 @@ tokenizer = AutoTokenizer.from_pretrained(model_name, trust_remote_code=True)
from auto_round import AutoRound
bits, group_size, sym = 4, 128, False
-## The device will be detected automatically, or you can specify it.
+##device:Optional["auto", None, "hpu", "cpu", "cuda"]
autoround = AutoRound(model, tokenizer, bits=bits, group_size=group_size, sym=sym, device=None)
autoround.quantize()
output_dir = "./tmp_autoround"
diff --git a/auto_round/autoround.py b/auto_round/autoround.py
index 509e5b334..6f96506d7 100644
--- a/auto_round/autoround.py
+++ b/auto_round/autoround.py
@@ -445,7 +445,7 @@ class AutoRound(object):
self.amp_dtype = torch.float16
if self.model.dtype != torch.float32:
self.amp_dtype = self.model.dtype
- if self.device == "cpu" or self.device == "hpu":
+ if self.device == "cpu" or "hpu" in self.device:
self.amp_dtype = torch.bfloat16
if self.amp:
if self.device == "cpu" and not CpuInfo().bf16:
@@ -879,7 +879,7 @@ class AutoRound(object):
block, current_input_ids, current_input_others, self.amp, self.amp_dtype, device
)
if self.amp and not check_is_cpu(device):
- with autocast(device_type="cuda", dtype=self.amp_dtype):
+ with autocast(device_type=device.split(":")[0], dtype=self.amp_dtype):
loss = mse_loss(output_q, current_output) # pylint: disable=not-callable
else:
loss = mse_loss( # pylint: disable=not-callable
diff --git a/auto_round/utils.py b/auto_round/utils.py
index ba5f7bc2e..e31bf95bb 100644
--- a/auto_round/utils.py
+++ b/auto_round/utils.py
@@ -473,23 +473,17 @@ def block_forward(block, input_ids, input_others, amp=False, amp_dtype=torch.flo
alibi = input_others["alibi"]
if alibi is not None:
alibi = alibi.reshape(-1, alibi.shape[2], alibi.shape[3])
- if amp and not check_is_cpu(device):
- with autocast(device_type="cuda", dtype=amp_dtype): # pragma: no cover
+ if amp:
+ with autocast(device_type=device.split(":")[0], dtype=amp_dtype): # pragma: no cover
output = block(
input_ids, attention_mask=attention_mask, alibi=alibi
) ##TODO is this correct for all models with alibi?
- elif amp and check_is_cpu(device):
- with torch.autocast(device_type="cpu", dtype=torch.bfloat16):
- output = block(input_ids, attention_mask=attention_mask, alibi=alibi)
else:
output = block(input_ids, attention_mask=attention_mask, alibi=alibi)
else:
input_tuple = input_others.pop("positional_inputs", None)
- if amp and not check_is_cpu(device):
- with autocast(device_type="cuda", dtype=amp_dtype): # pragma: no cover
- output = block.forward(input_ids, *input_tuple, **input_others)
- elif amp and check_is_cpu(device):
- with torch.autocast(device_type="cpu", dtype=torch.bfloat16):
+ if amp:
+ with autocast(device_type=device.split(":")[0], dtype=amp_dtype): # pragma: no cover
output = block.forward(input_ids, *input_tuple, **input_others)
else:
output = block.forward(input_ids, *input_tuple, **input_others)
diff --git a/examples/code-generation/main.py b/examples/code-generation/main.py
index 46b7c9a6a..b04345006 100644
--- a/examples/code-generation/main.py
+++ b/examples/code-generation/main.py
@@ -127,7 +127,7 @@ if __name__ == '__main__':
device_str = detect_device(args.device)
torch_dtype = "auto"
- if device_str == "hpu":
+ if "hpu" in device_str:
torch_dtype = torch.bfloat16
torch_device = torch.device(device_str)
is_glm = bool(re.search("chatglm", model_name.lower()))
diff --git a/examples/language-modeling/main.py b/examples/language-modeling/main.py
index 256fe241c..5e3a65409 100644
--- a/examples/language-modeling/main.py
+++ b/examples/language-modeling/main.py
@@ -175,7 +175,7 @@ if __name__ == '__main__':
device_str = detect_device(args.device)
torch_dtype = "auto"
- if device_str == "hpu":
+ if "hpu" in device_str:
torch_dtype = torch.bfloat16
torch_device = torch.device(device_str)
diff --git a/examples/language-modeling/main_customized_data.py b/examples/language-modeling/main_customized_data.py
index 798894aa0..7ea020e73 100644
--- a/examples/language-modeling/main_customized_data.py
+++ b/examples/language-modeling/main_customized_data.py
@@ -165,7 +165,7 @@ if __name__ == '__main__':
device_str = detect_device(args.device)
torch_dtype = "auto"
- if device_str == "hpu":
+ if "hpu" in device_str:
torch_dtype = torch.bfloat16
torch_device = torch.device(device_str)