PTQ-Level Efficiency, QAT-Level Accuracy | 训练后量化的效率,量化感知训练的精度
fix cuda hanging issue during packing (#669)
Weiwei7863f69dd235
diff --git a/auto_round/export/export_to_autogptq/export.py b/auto_round/export/export_to_autogptq/export.pyindex c4dd1c59d..c9688436e 100644--- a/auto_round/export/export_to_autogptq/export.py+++ b/auto_round/export/export_to_autogptq/export.py@@ -170,7 +170,7 @@ def save_quantized_as_autogptq(output_dir, inplace=True, backend="auto_gptq:exlllayer_config = kwargs["layer_config"]names = list(layer_config.keys())max_workers = 1- if not torch.cuda.is_available() or not torch.xpu.is_available():+ if not torch.cuda.is_available() and not torch.xpu.is_available():max_workers = 2 ## 2 with cuda packing will cause hang occasionallywith ThreadPoolExecutor(max_workers=max_workers) as executor:@@ -240,3 +240,4 @@ def save(model: torch.nn.Module, save_dir: str, max_shard_size: str = "5GB", safjson.dump(model.config.quantization_config, f, indent=2)+diff --git a/auto_round/export/export_to_autoround/export.py b/auto_round/export/export_to_autoround/export.pyindex 62d434811..cf8fe66c9 100644--- a/auto_round/export/export_to_autoround/export.py+++ b/auto_round/export/export_to_autoround/export.py@@ -307,7 +307,7 @@ def save_quantized_as_autoround(output_dir, inplace=True, backend="auto_round:exquantization_config["extra_config"] = extra_confignames = list(layer_config.keys())max_workers = 1- if not torch.cuda.is_available() or not torch.xpu.is_available():+ if not torch.cuda.is_available() and not torch.xpu.is_available():max_workers = 2 ## 2 with cuda packing will cause hang occasionallywith ThreadPoolExecutor(max_workers=max_workers) as executor:with tqdm(total=len(names), leave=True) as pbar:@@ -380,3 +380,4 @@ def save(model: nn.Module, save_dir: str, max_shard_size: str = "5GB", safe_seriwith open(os.path.join(save_dir, config_file), "w", encoding="utf-8") as f:json.dump(model.config.quantization_config, f, indent=2)+diff --git a/auto_round/export/export_to_awq/export.py b/auto_round/export/export_to_awq/export.pyindex 8d18020a4..9f04458dd 100644--- a/auto_round/export/export_to_awq/export.py+++ b/auto_round/export/export_to_awq/export.py@@ -102,7 +102,7 @@ def save_quantized_as_autoawq(output_dir, inplace=True, **kwargs):backend = Nonemax_workers = 1- if not torch.cuda.is_available() or not torch.xpu.is_available():+ if not torch.cuda.is_available() and not torch.xpu.is_available():max_workers = 2 ## 2 with cuda packing will cause hang occasionallywith ThreadPoolExecutor(max_workers=max_workers) as executor:with tqdm(total=len(names), leave=True) as pbar:@@ -178,3 +178,4 @@ def save(model: nn.Module, save_dir: str, max_shard_size: str = "5GB", safe_seriif hasattr(model, "config") and hasattr(model.config, "quantization_config"):with open(os.path.join(save_dir, config_file), "w", encoding="utf-8") as f:json.dump(model.config.quantization_config, f, indent=2)+diff --git a/test/test_cuda/test_2_3bits.py b/test/test_cuda/test_2_3bits.pyindex c82d58765..7b1f3d31b 100644--- a/test/test_cuda/test_2_3bits.py+++ b/test/test_cuda/test_2_3bits.py@@ -63,8 +63,7 @@ class TestAutoRound(unittest.TestCase):tokenizer = AutoTokenizer.from_pretrained(model_name)bits, sym = 3, Falseautoround = AutoRound(model, tokenizer, bits=bits, sym=sym)- autoround.quantize()- autoround.save_quantized(self.save_dir, format="auto_round", inplace=False)+ autoround.quantize_and_save(self.save_dir, format="auto_round", inplace=False)model_args = f"pretrained={self.save_dir}"res = simple_evaluate(model="hf", model_args=model_args,# tasks="arc_easy",@@ -127,3 +126,4 @@ class TestAutoRound(unittest.TestCase):if __name__ == "__main__":unittest.main()+