PTQ-Level Efficiency, QAT-Level Accuracy | 训练后量化的效率,量化感知训练的精度
fix device_map setting (#854)
Tang Kaihui4bb944fd8848
diff --git a/auto_round/compressors/base.py b/auto_round/compressors/base.pyindex 9aa4592bd..e1f3e7bf2 100644--- a/auto_round/compressors/base.py+++ b/auto_round/compressors/base.py@@ -249,25 +249,6 @@ class BaseCompressor(object):else:torch.use_deterministic_algorithms(True, warn_only=True)- if device is not None:- logger.warning("`device` is deprecated, please use `device_map` instead")-- if device_map is None:- device_map = 0-- # Set device, must place after model loading- self._set_device(device_map)-- if (isinstance(device_map, dict) and device_map) or device_map == "auto":- self.device_map = device_map- elif isinstance(device_map, str) and "," in device_map:- device_map = device_map.replace(" ", "") # Remove any spaces- self.device_list = [int(dev) for dev in device_map.split(",") if dev.isdigit()]- self.device_map = "auto"- else:- self.device_map = None- self._set_device_map_in_blocks(self.device_map)-# Model relatedself.quantized = Falseif isinstance(model, str):@@ -289,6 +270,25 @@ class BaseCompressor(object):self.tokenizer = tokenizerself.shared_cache_keys = get_shared_keys(self.model)+ if device is not None:+ logger.warning("`device` is deprecated, please use `device_map` instead")++ if device_map is None:+ device_map = 0++ # Set device, must place after model loading+ self._set_device(device_map)++ if (isinstance(device_map, dict) and device_map) or device_map == "auto":+ self.device_map = device_map+ elif isinstance(device_map, str) and "," in device_map:+ device_map = device_map.replace(" ", "") # Remove any spaces+ self.device_list = [int(dev) for dev in device_map.split(",") if dev.isdigit()]+ self.device_map = "auto"+ else:+ self.device_map = None+ self._set_device_map_in_blocks(self.device_map)+not_quantize_layer_names = get_fp_layer_names(self.model, fp_layers)if len(not_quantize_layer_names) > 0:logger.info(f"{not_quantize_layer_names} will not be quantized.")diff --git a/test/test_cpu/test_autoround.py b/test/test_cpu/test_autoround.pyindex 9857cdc0d..c2ef29c59 100644--- a/test/test_cpu/test_autoround.py+++ b/test/test_cpu/test_autoround.py@@ -335,6 +335,37 @@ class TestAutoRound(unittest.TestCase):)autoround.quantize()+ def test_device_map_dict(self):+ bits, group_size, sym = 4, 128, False+ device_map = {".*": "cpu"}+ autoround = AutoRound(+ self.model,+ self.tokenizer,+ bits=bits,+ group_size=group_size,+ sym=sym,+ iters=2,+ seqlen=2,+ dataset=self.llm_dataloader,+ device_map=device_map,+ )+ autoround.quantize()++ # test model_name+ model_name = "/tf_dataset/auto_round/models/facebook/opt-125m"+ autoround = AutoRound(+ model_name,+ self.tokenizer,+ bits=bits,+ group_size=group_size,+ sym=sym,+ iters=2,+ seqlen=2,+ dataset=self.llm_dataloader,+ device_map=device_map,+ )+ autoround.quantize()+def test_fp32(self):bits, group_size, sym = 4, 128, Falsemodel_name = "/tf_dataset/auto_round/models/facebook/opt-125m"diff --git a/test/test_cuda/test_multiple_card.py b/test/test_cuda/test_multiple_card.pyindex e398b07c6..ad33f071b 100644--- a/test/test_cuda/test_multiple_card.py+++ b/test/test_cuda/test_multiple_card.py@@ -209,6 +209,39 @@ class TestAutoRound(unittest.TestCase):del modeltorch.cuda.empty_cache()+ @multi_card+ def test_device_map_dict(self):+ device_map = {".*q_proj": "0", ".*k_proj": "cuda:1", "v_proj": 1, ".*up_proj": "1"}+ bits, group_size, sym = 4, 128, False+ model_name = "/models/opt-125m"+ model = AutoModelForCausalLM.from_pretrained(model_name, torch_dtype=torch.float16, device_map="auto")+ tokenizer = AutoTokenizer.from_pretrained(model_name)+ autoround = AutoRound(+ model,+ tokenizer,+ bits=bits,+ group_size=group_size,+ sym=sym,+ iters=2,+ seqlen=2,+ device_map=device_map,+ )+ autoround.quantize()++ # test model_name+ model_name = "/models/opt-125m"+ autoround = AutoRound(+ model_name,+ tokenizer,+ bits=bits,+ group_size=group_size,+ sym=sym,+ iters=2,+ seqlen=2,+ device_map=device_map,+ )+ autoround.quantize()+@multi_card@require_greater_than_050def test_device_map_for_triton(self):