Skip to content

PTQ-Level Efficiency, QAT-Level Accuracy | 训练后量化的效率,量化感知训练的精度

add check of mixed device (#190)

n1ck-guo61cf9eef4a3c
diff --git a/README.md b/README.md
index 953e742cf..f14a08a79 100644
--- a/README.md
+++ b/README.md
@@ -62,7 +62,7 @@ quantization in some scenarios, especially at lower bits,like 2. Please save qua
from transformers import AutoModelForCausalLM, AutoTokenizer
model_name = "facebook/opt-125m"
-model = AutoModelForCausalLM.from_pretrained(model_name, device_map="auto")
+model = AutoModelForCausalLM.from_pretrained(model_name)
tokenizer = AutoTokenizer.from_pretrained(model_name)
from auto_round import AutoRound
@@ -146,7 +146,7 @@ from intel_extension_for_transformers.transformers import AutoModelForCausalLM
from transformers import AutoTokenizer
quantized_model_path = "./tmp_autoround"
-model = AutoModelForCausalLM.from_pretrained(quantized_model_path, device_map="auto")
+model = AutoModelForCausalLM.from_pretrained(quantized_model_path)
tokenizer = AutoTokenizer.from_pretrained(quantized_model_path)
text = "There is a girl who likes adventure,"
inputs = tokenizer(text, return_tensors="pt").to(model.device)
@@ -177,7 +177,7 @@ from auto_round.auto_quantizer import AutoHfQuantizer
import habana_frameworks.torch.core as htcore
import habana_frameworks.torch.hpu as hthpu
quantized_model_path = "./tmp_autoround"
-model = AutoModelForCausalLM.from_pretrained(quantized_model_path, device_map="auto").to('hpu').to(torch.float32)
+model = AutoModelForCausalLM.from_pretrained(quantized_model_path).to('hpu').to(torch.float32)
tokenizer = AutoTokenizer.from_pretrained(quantized_model_path)
text = "There is a girl who likes adventure,"
inputs = tokenizer(text, return_tensors="pt").to(model.device)
diff --git a/auto_round/autoround.py b/auto_round/autoround.py
index 5b4b30a9e..8622be756 100644
--- a/auto_round/autoround.py
+++ b/auto_round/autoround.py
@@ -43,6 +43,7 @@ from .utils import (
to_dtype,
get_layer_names_in_block,
mv_module_from_gpu,
+ unsupport_meta_device,
)
from .low_cpu_mem.utils import get_layers_before_block
@@ -149,6 +150,10 @@ class AutoRound(object):
self.quantized = False
self.model_orig_dtype = model.dtype
self.low_cpu_mem_usage = low_cpu_mem_usage
+ assert not unsupport_meta_device(model), (
+ "autoround does not support for params on meta device by transformers` interfaces,"
+ "please do not using device_map='auto' in model loading, "
+ "or follow examples/language-modeling/main.py to enable low_cpu_mem_usage")
self.model = model.eval()
self.model = mv_module_from_gpu(self.model, self.low_cpu_mem_usage)
self.amp = amp
diff --git a/auto_round/utils.py b/auto_round/utils.py
index 2d6bd52fa..11c69ac19 100644
--- a/auto_round/utils.py
+++ b/auto_round/utils.py
@@ -135,6 +135,30 @@ def get_scale_shape(weight, group_size):
return shape
+def unsupport_meta_device(model):
+ """Checks if the model is a valid model for auto_round.
+
+ Args:
+ model: The model to be checked.
+
+ Returns:
+ bool: True if the model is valid, False otherwise.
+ """
+ target_device = None
+ for param in model.parameters():
+ if target_device is None:
+ target_device = param.device
+ if param.device != target_device:
+ if param.device.type == 'meta' or target_device.type == 'meta':
+ return True
+ if target_device.type == 'meta':
+ if hasattr(model, "path"):
+ return False
+ else:
+ return True
+ return False
+
+
def to_device(input, device=torch.device("cpu")):
"""Moves input data to the specified device.
diff --git a/test/test_autoround.py b/test/test_autoround.py
index 0d454ed59..6c11b3010 100644
--- a/test/test_autoround.py
+++ b/test/test_autoround.py
@@ -226,6 +226,22 @@ class TestAutoRound(unittest.TestCase):
act_bits=4,
)
autoround.quantize()
+
+ def test_auto_device_map(self):
+ bits, group_size, sym = 4, 128, False
+ model_name = "facebook/opt-125m"
+ model = AutoModelForCausalLM.from_pretrained(model_name, torch_dtype="auto", trust_remote_code=True, device_map='auto')
+ autoround = AutoRound(
+ model,
+ self.tokenizer,
+ bits=bits,
+ group_size=group_size,
+ sym=sym,
+ iters=2,
+ seqlen=2,
+ dataset=self.llm_dataloader,
+ )
+ autoround.quantize()