Skip to content

PTQ-Level Efficiency, QAT-Level Accuracy | 训练后量化的效率,量化感知训练的精度

fix device_map setting (#854)

Tang Kaihui4bb944fd8848
diff --git a/auto_round/compressors/base.py b/auto_round/compressors/base.py
index 9aa4592bd..e1f3e7bf2 100644
--- a/auto_round/compressors/base.py
+++ b/auto_round/compressors/base.py
@@ -249,25 +249,6 @@ class BaseCompressor(object):
else:
torch.use_deterministic_algorithms(True, warn_only=True)
- if device is not None:
- logger.warning("`device` is deprecated, please use `device_map` instead")
-
- if device_map is None:
- device_map = 0
-
- # Set device, must place after model loading
- self._set_device(device_map)
-
- if (isinstance(device_map, dict) and device_map) or device_map == "auto":
- self.device_map = device_map
- elif isinstance(device_map, str) and "," in device_map:
- device_map = device_map.replace(" ", "") # Remove any spaces
- self.device_list = [int(dev) for dev in device_map.split(",") if dev.isdigit()]
- self.device_map = "auto"
- else:
- self.device_map = None
- self._set_device_map_in_blocks(self.device_map)
-
# Model related
self.quantized = False
if isinstance(model, str):
@@ -289,6 +270,25 @@ class BaseCompressor(object):
self.tokenizer = tokenizer
self.shared_cache_keys = get_shared_keys(self.model)
+ if device is not None:
+ logger.warning("`device` is deprecated, please use `device_map` instead")
+
+ if device_map is None:
+ device_map = 0
+
+ # Set device, must place after model loading
+ self._set_device(device_map)
+
+ if (isinstance(device_map, dict) and device_map) or device_map == "auto":
+ self.device_map = device_map
+ elif isinstance(device_map, str) and "," in device_map:
+ device_map = device_map.replace(" ", "") # Remove any spaces
+ self.device_list = [int(dev) for dev in device_map.split(",") if dev.isdigit()]
+ self.device_map = "auto"
+ else:
+ self.device_map = None
+ self._set_device_map_in_blocks(self.device_map)
+
not_quantize_layer_names = get_fp_layer_names(self.model, fp_layers)
if len(not_quantize_layer_names) > 0:
logger.info(f"{not_quantize_layer_names} will not be quantized.")
diff --git a/test/test_cpu/test_autoround.py b/test/test_cpu/test_autoround.py
index 9857cdc0d..c2ef29c59 100644
--- a/test/test_cpu/test_autoround.py
+++ b/test/test_cpu/test_autoround.py
@@ -335,6 +335,37 @@ class TestAutoRound(unittest.TestCase):
)
autoround.quantize()
+ def test_device_map_dict(self):
+ bits, group_size, sym = 4, 128, False
+ device_map = {".*": "cpu"}
+ autoround = AutoRound(
+ self.model,
+ self.tokenizer,
+ bits=bits,
+ group_size=group_size,
+ sym=sym,
+ iters=2,
+ seqlen=2,
+ dataset=self.llm_dataloader,
+ device_map=device_map,
+ )
+ autoround.quantize()
+
+ # test model_name
+ model_name = "/tf_dataset/auto_round/models/facebook/opt-125m"
+ autoround = AutoRound(
+ model_name,
+ self.tokenizer,
+ bits=bits,
+ group_size=group_size,
+ sym=sym,
+ iters=2,
+ seqlen=2,
+ dataset=self.llm_dataloader,
+ device_map=device_map,
+ )
+ autoround.quantize()
+
def test_fp32(self):
bits, group_size, sym = 4, 128, False
model_name = "/tf_dataset/auto_round/models/facebook/opt-125m"
diff --git a/test/test_cuda/test_multiple_card.py b/test/test_cuda/test_multiple_card.py
index e398b07c6..ad33f071b 100644
--- a/test/test_cuda/test_multiple_card.py
+++ b/test/test_cuda/test_multiple_card.py
@@ -209,6 +209,39 @@ class TestAutoRound(unittest.TestCase):
del model
torch.cuda.empty_cache()
+ @multi_card
+ def test_device_map_dict(self):
+ device_map = {".*q_proj": "0", ".*k_proj": "cuda:1", "v_proj": 1, ".*up_proj": "1"}
+ bits, group_size, sym = 4, 128, False
+ model_name = "/models/opt-125m"
+ model = AutoModelForCausalLM.from_pretrained(model_name, torch_dtype=torch.float16, device_map="auto")
+ tokenizer = AutoTokenizer.from_pretrained(model_name)
+ autoround = AutoRound(
+ model,
+ tokenizer,
+ bits=bits,
+ group_size=group_size,
+ sym=sym,
+ iters=2,
+ seqlen=2,
+ device_map=device_map,
+ )
+ autoround.quantize()
+
+ # test model_name
+ model_name = "/models/opt-125m"
+ autoround = AutoRound(
+ model_name,
+ tokenizer,
+ bits=bits,
+ group_size=group_size,
+ sym=sym,
+ iters=2,
+ seqlen=2,
+ device_map=device_map,
+ )
+ autoround.quantize()
+
@multi_card
@require_greater_than_050
def test_device_map_for_triton(self):