Skip to content

PTQ-Level Efficiency, QAT-Level Accuracy | 训练后量化的效率,量化感知训练的精度

fix cuda hanging issue during packing (#669)

Weiwei7863f69dd235
diff --git a/auto_round/export/export_to_autogptq/export.py b/auto_round/export/export_to_autogptq/export.py
index c4dd1c59d..c9688436e 100644
--- a/auto_round/export/export_to_autogptq/export.py
+++ b/auto_round/export/export_to_autogptq/export.py
@@ -170,7 +170,7 @@ def save_quantized_as_autogptq(output_dir, inplace=True, backend="auto_gptq:exll
layer_config = kwargs["layer_config"]
names = list(layer_config.keys())
max_workers = 1
- if not torch.cuda.is_available() or not torch.xpu.is_available():
+ if not torch.cuda.is_available() and not torch.xpu.is_available():
max_workers = 2 ## 2 with cuda packing will cause hang occasionally
with ThreadPoolExecutor(max_workers=max_workers) as executor:
@@ -240,3 +240,4 @@ def save(model: torch.nn.Module, save_dir: str, max_shard_size: str = "5GB", saf
json.dump(model.config.quantization_config, f, indent=2)
+
diff --git a/auto_round/export/export_to_autoround/export.py b/auto_round/export/export_to_autoround/export.py
index 62d434811..cf8fe66c9 100644
--- a/auto_round/export/export_to_autoround/export.py
+++ b/auto_round/export/export_to_autoround/export.py
@@ -307,7 +307,7 @@ def save_quantized_as_autoround(output_dir, inplace=True, backend="auto_round:ex
quantization_config["extra_config"] = extra_config
names = list(layer_config.keys())
max_workers = 1
- if not torch.cuda.is_available() or not torch.xpu.is_available():
+ if not torch.cuda.is_available() and not torch.xpu.is_available():
max_workers = 2 ## 2 with cuda packing will cause hang occasionally
with ThreadPoolExecutor(max_workers=max_workers) as executor:
with tqdm(total=len(names), leave=True) as pbar:
@@ -380,3 +380,4 @@ def save(model: nn.Module, save_dir: str, max_shard_size: str = "5GB", safe_seri
with open(os.path.join(save_dir, config_file), "w", encoding="utf-8") as f:
json.dump(model.config.quantization_config, f, indent=2)
+
diff --git a/auto_round/export/export_to_awq/export.py b/auto_round/export/export_to_awq/export.py
index 8d18020a4..9f04458dd 100644
--- a/auto_round/export/export_to_awq/export.py
+++ b/auto_round/export/export_to_awq/export.py
@@ -102,7 +102,7 @@ def save_quantized_as_autoawq(output_dir, inplace=True, **kwargs):
backend = None
max_workers = 1
- if not torch.cuda.is_available() or not torch.xpu.is_available():
+ if not torch.cuda.is_available() and not torch.xpu.is_available():
max_workers = 2 ## 2 with cuda packing will cause hang occasionally
with ThreadPoolExecutor(max_workers=max_workers) as executor:
with tqdm(total=len(names), leave=True) as pbar:
@@ -178,3 +178,4 @@ def save(model: nn.Module, save_dir: str, max_shard_size: str = "5GB", safe_seri
if hasattr(model, "config") and hasattr(model.config, "quantization_config"):
with open(os.path.join(save_dir, config_file), "w", encoding="utf-8") as f:
json.dump(model.config.quantization_config, f, indent=2)
+
diff --git a/test/test_cuda/test_2_3bits.py b/test/test_cuda/test_2_3bits.py
index c82d58765..7b1f3d31b 100644
--- a/test/test_cuda/test_2_3bits.py
+++ b/test/test_cuda/test_2_3bits.py
@@ -63,8 +63,7 @@ class TestAutoRound(unittest.TestCase):
tokenizer = AutoTokenizer.from_pretrained(model_name)
bits, sym = 3, False
autoround = AutoRound(model, tokenizer, bits=bits, sym=sym)
- autoround.quantize()
- autoround.save_quantized(self.save_dir, format="auto_round", inplace=False)
+ autoround.quantize_and_save(self.save_dir, format="auto_round", inplace=False)
model_args = f"pretrained={self.save_dir}"
res = simple_evaluate(model="hf", model_args=model_args,
# tasks="arc_easy",
@@ -127,3 +126,4 @@ class TestAutoRound(unittest.TestCase):
if __name__ == "__main__":
unittest.main()
+