PTQ-Level Efficiency, QAT-Level Accuracy | 训练后量化的效率,量化感知训练的精度
add check of mixed device (#190)
n1ck-guo61cf9eef4a3c
diff --git a/README.md b/README.mdindex 953e742cf..f14a08a79 100644--- a/README.md+++ b/README.md@@ -62,7 +62,7 @@ quantization in some scenarios, especially at lower bits,like 2. Please save quafrom transformers import AutoModelForCausalLM, AutoTokenizermodel_name = "facebook/opt-125m"-model = AutoModelForCausalLM.from_pretrained(model_name, device_map="auto")+model = AutoModelForCausalLM.from_pretrained(model_name)tokenizer = AutoTokenizer.from_pretrained(model_name)from auto_round import AutoRound@@ -146,7 +146,7 @@ from intel_extension_for_transformers.transformers import AutoModelForCausalLMfrom transformers import AutoTokenizerquantized_model_path = "./tmp_autoround"-model = AutoModelForCausalLM.from_pretrained(quantized_model_path, device_map="auto")+model = AutoModelForCausalLM.from_pretrained(quantized_model_path)tokenizer = AutoTokenizer.from_pretrained(quantized_model_path)text = "There is a girl who likes adventure,"inputs = tokenizer(text, return_tensors="pt").to(model.device)@@ -177,7 +177,7 @@ from auto_round.auto_quantizer import AutoHfQuantizerimport habana_frameworks.torch.core as htcoreimport habana_frameworks.torch.hpu as hthpuquantized_model_path = "./tmp_autoround"-model = AutoModelForCausalLM.from_pretrained(quantized_model_path, device_map="auto").to('hpu').to(torch.float32)+model = AutoModelForCausalLM.from_pretrained(quantized_model_path).to('hpu').to(torch.float32)tokenizer = AutoTokenizer.from_pretrained(quantized_model_path)text = "There is a girl who likes adventure,"inputs = tokenizer(text, return_tensors="pt").to(model.device)diff --git a/auto_round/autoround.py b/auto_round/autoround.pyindex 5b4b30a9e..8622be756 100644--- a/auto_round/autoround.py+++ b/auto_round/autoround.py@@ -43,6 +43,7 @@ from .utils import (to_dtype,get_layer_names_in_block,mv_module_from_gpu,+ unsupport_meta_device,)from .low_cpu_mem.utils import get_layers_before_block@@ -149,6 +150,10 @@ class AutoRound(object):self.quantized = Falseself.model_orig_dtype = model.dtypeself.low_cpu_mem_usage = low_cpu_mem_usage+ assert not unsupport_meta_device(model), (+ "autoround does not support for params on meta device by transformers` interfaces,"+ "please do not using device_map='auto' in model loading, "+ "or follow examples/language-modeling/main.py to enable low_cpu_mem_usage")self.model = model.eval()self.model = mv_module_from_gpu(self.model, self.low_cpu_mem_usage)self.amp = ampdiff --git a/auto_round/utils.py b/auto_round/utils.pyindex 2d6bd52fa..11c69ac19 100644--- a/auto_round/utils.py+++ b/auto_round/utils.py@@ -135,6 +135,30 @@ def get_scale_shape(weight, group_size):return shape+def unsupport_meta_device(model):+ """Checks if the model is a valid model for auto_round.++ Args:+ model: The model to be checked.++ Returns:+ bool: True if the model is valid, False otherwise.+ """+ target_device = None+ for param in model.parameters():+ if target_device is None:+ target_device = param.device+ if param.device != target_device:+ if param.device.type == 'meta' or target_device.type == 'meta':+ return True+ if target_device.type == 'meta':+ if hasattr(model, "path"):+ return False+ else:+ return True+ return False++def to_device(input, device=torch.device("cpu")):"""Moves input data to the specified device.diff --git a/test/test_autoround.py b/test/test_autoround.pyindex 0d454ed59..6c11b3010 100644--- a/test/test_autoround.py+++ b/test/test_autoround.py@@ -226,6 +226,22 @@ class TestAutoRound(unittest.TestCase):act_bits=4,)autoround.quantize()++ def test_auto_device_map(self):+ bits, group_size, sym = 4, 128, False+ model_name = "facebook/opt-125m"+ model = AutoModelForCausalLM.from_pretrained(model_name, torch_dtype="auto", trust_remote_code=True, device_map='auto')+ autoround = AutoRound(+ model,+ self.tokenizer,+ bits=bits,+ group_size=group_size,+ sym=sym,+ iters=2,+ seqlen=2,+ dataset=self.llm_dataloader,+ )+ autoround.quantize()