PTQ-Level Efficiency, QAT-Level Accuracy | 训练后量化的效率,量化感知训练的精度
Fix hpu autocast (#64)
wenhuach21c7d2fa84ce6c
diff --git a/README.md b/README.mdindex 053e0dc2f..9becb2c12 100644--- a/README.md+++ b/README.md@@ -47,7 +47,7 @@ tokenizer = AutoTokenizer.from_pretrained(model_name, trust_remote_code=True)from auto_round import AutoRoundbits, group_size, sym = 4, 128, False-## The device will be detected automatically, or you can specify it.+##device:Optional["auto", None, "hpu", "cpu", "cuda"]autoround = AutoRound(model, tokenizer, bits=bits, group_size=group_size, sym=sym, device=None)autoround.quantize()output_dir = "./tmp_autoround"diff --git a/auto_round/autoround.py b/auto_round/autoround.pyindex 509e5b334..6f96506d7 100644--- a/auto_round/autoround.py+++ b/auto_round/autoround.py@@ -445,7 +445,7 @@ class AutoRound(object):self.amp_dtype = torch.float16if self.model.dtype != torch.float32:self.amp_dtype = self.model.dtype- if self.device == "cpu" or self.device == "hpu":+ if self.device == "cpu" or "hpu" in self.device:self.amp_dtype = torch.bfloat16if self.amp:if self.device == "cpu" and not CpuInfo().bf16:@@ -879,7 +879,7 @@ class AutoRound(object):block, current_input_ids, current_input_others, self.amp, self.amp_dtype, device)if self.amp and not check_is_cpu(device):- with autocast(device_type="cuda", dtype=self.amp_dtype):+ with autocast(device_type=device.split(":")[0], dtype=self.amp_dtype):loss = mse_loss(output_q, current_output) # pylint: disable=not-callableelse:loss = mse_loss( # pylint: disable=not-callablediff --git a/auto_round/utils.py b/auto_round/utils.pyindex ba5f7bc2e..e31bf95bb 100644--- a/auto_round/utils.py+++ b/auto_round/utils.py@@ -473,23 +473,17 @@ def block_forward(block, input_ids, input_others, amp=False, amp_dtype=torch.floalibi = input_others["alibi"]if alibi is not None:alibi = alibi.reshape(-1, alibi.shape[2], alibi.shape[3])- if amp and not check_is_cpu(device):- with autocast(device_type="cuda", dtype=amp_dtype): # pragma: no cover+ if amp:+ with autocast(device_type=device.split(":")[0], dtype=amp_dtype): # pragma: no coveroutput = block(input_ids, attention_mask=attention_mask, alibi=alibi) ##TODO is this correct for all models with alibi?- elif amp and check_is_cpu(device):- with torch.autocast(device_type="cpu", dtype=torch.bfloat16):- output = block(input_ids, attention_mask=attention_mask, alibi=alibi)else:output = block(input_ids, attention_mask=attention_mask, alibi=alibi)else:input_tuple = input_others.pop("positional_inputs", None)- if amp and not check_is_cpu(device):- with autocast(device_type="cuda", dtype=amp_dtype): # pragma: no cover- output = block.forward(input_ids, *input_tuple, **input_others)- elif amp and check_is_cpu(device):- with torch.autocast(device_type="cpu", dtype=torch.bfloat16):+ if amp:+ with autocast(device_type=device.split(":")[0], dtype=amp_dtype): # pragma: no coveroutput = block.forward(input_ids, *input_tuple, **input_others)else:output = block.forward(input_ids, *input_tuple, **input_others)diff --git a/examples/code-generation/main.py b/examples/code-generation/main.pyindex 46b7c9a6a..b04345006 100644--- a/examples/code-generation/main.py+++ b/examples/code-generation/main.py@@ -127,7 +127,7 @@ if __name__ == '__main__':device_str = detect_device(args.device)torch_dtype = "auto"- if device_str == "hpu":+ if "hpu" in device_str:torch_dtype = torch.bfloat16torch_device = torch.device(device_str)is_glm = bool(re.search("chatglm", model_name.lower()))diff --git a/examples/language-modeling/main.py b/examples/language-modeling/main.pyindex 256fe241c..5e3a65409 100644--- a/examples/language-modeling/main.py+++ b/examples/language-modeling/main.py@@ -175,7 +175,7 @@ if __name__ == '__main__':device_str = detect_device(args.device)torch_dtype = "auto"- if device_str == "hpu":+ if "hpu" in device_str:torch_dtype = torch.bfloat16torch_device = torch.device(device_str)diff --git a/examples/language-modeling/main_customized_data.py b/examples/language-modeling/main_customized_data.pyindex 798894aa0..7ea020e73 100644--- a/examples/language-modeling/main_customized_data.py+++ b/examples/language-modeling/main_customized_data.py@@ -165,7 +165,7 @@ if __name__ == '__main__':device_str = detect_device(args.device)torch_dtype = "auto"- if device_str == "hpu":+ if "hpu" in device_str:torch_dtype = torch.bfloat16torch_device = torch.device(device_str)