Skip to content

PTQ-Level Efficiency, QAT-Level Accuracy | 训练后量化的效率,量化感知训练的精度

Remove threaded packing from exporters (#1719)

Yi Liu85a619260aa0
diff --git a/auto_round/export/export_to_autogptq/export.py b/auto_round/export/export_to_autogptq/export.py
index 98fde7b73..5594b5979 100644
--- a/auto_round/export/export_to_autogptq/export.py
+++ b/auto_round/export/export_to_autogptq/export.py
@@ -16,12 +16,9 @@ import copy
import inspect
import json
import os
-from concurrent.futures import ThreadPoolExecutor
from dataclasses import fields
from typing import Any, Callable, Dict, Union
-import threadpoolctl as tctl
-
# MIT License
#
# Copyright (c) 2023 潘其威(William)
@@ -295,21 +292,9 @@ def save_quantized_as_autogptq(
model = copy.deepcopy(model.to("cpu"))
names = list(layer_config.keys())
- max_workers = 1
- if not torch.cuda.is_available() and not torch.xpu.is_available():
- max_workers = 2 ## 2 with cuda packing will cause hang occasionally
if not unsupported_meta_device(model):
- with ThreadPoolExecutor(max_workers=max_workers) as executor:
- with tqdm(total=len(names), leave=True) as pbar:
-
- def wrapper(name):
- pbar.set_description(f"packing {name}")
- with tctl.threadpool_limits(limits=1):
- pack_layer(name, model, backend, device)
- pbar.update(1)
-
- for _ in executor.map(wrapper, names):
- pass
+ for name in tqdm(names, desc="packing", leave=True):
+ pack_layer(name, model, backend, device)
if output_dir is None:
return model
quantization_config["lm_head"] = lm_head_quantized
diff --git a/auto_round/export/export_to_autoround/export.py b/auto_round/export/export_to_autoround/export.py
index 372638e99..60613b7e8 100644
--- a/auto_round/export/export_to_autoround/export.py
+++ b/auto_round/export/export_to_autoround/export.py
@@ -18,12 +18,10 @@ import functools
import inspect
import json
import os
-from concurrent.futures import ThreadPoolExecutor
from dataclasses import fields
from enum import Enum
from typing import Callable, Union
-import threadpoolctl as tctl
import torch
import torch.nn as nn
import transformers
@@ -309,21 +307,9 @@ def save_quantized_as_autoround(
quantization_config["extra_config"] = extra_config
names = list(layer_config.keys())
- max_workers = 1
- if not torch.cuda.is_available() and not torch.xpu.is_available():
- max_workers = 2 ## 2 with cuda packing will cause hang occasionally
if not unsupported_meta_device(model):
- with ThreadPoolExecutor(max_workers=max_workers) as executor:
- with tqdm(total=len(names), leave=True) as pbar:
-
- def wrapper(name):
- pbar.set_description(f"packing {name}")
- with tctl.threadpool_limits(limits=1):
- pack_layer(name, model, backend, device)
- pbar.update(1)
-
- for _ in executor.map(wrapper, names):
- pass
+ for name in tqdm(names, desc="packing", leave=True):
+ pack_layer(name, model, backend, device)
filter_quantization_config(quantization_config)
if hasattr(model, "config"):
model.config.quantization_config = quantization_config
diff --git a/auto_round/export/export_to_autoround/export_to_fp8.py b/auto_round/export/export_to_autoround/export_to_fp8.py
index 38af5b7b5..90c228583 100644
--- a/auto_round/export/export_to_autoround/export_to_fp8.py
+++ b/auto_round/export/export_to_autoround/export_to_fp8.py
@@ -15,11 +15,9 @@
import copy
import json
import os
-from concurrent.futures import ThreadPoolExecutor
from dataclasses import fields
from typing import Callable, Union
-import threadpoolctl as tctl
import torch
import transformers
from tqdm import tqdm
@@ -240,20 +238,8 @@ def save_quantized_as_autoround(
if len(extra_config) > 0:
quantization_config["extra_config"] = extra_config
names = list(layer_config.keys())
- max_workers = 1
- if not torch.cuda.is_available() and not torch.xpu.is_available():
- max_workers = 2 ## 2 with cuda packing will cause hang occasionally
- with ThreadPoolExecutor(max_workers=max_workers) as executor:
- with tqdm(total=len(names), leave=True) as pbar:
-
- def wrapper(name):
- pbar.set_description(f"packing {name}")
- with tctl.threadpool_limits(limits=1):
- pack_layer(name, model, serialization_dict.get("data_type", "fp8"), device)
- pbar.update(1)
-
- for _ in executor.map(wrapper, names):
- pass
+ for name in tqdm(names, desc="packing", leave=True):
+ pack_layer(name, model, serialization_dict.get("data_type", "fp8"), device)
regex_config = quantization_config.pop("regex_config")
if regex_config is not None:
for name in regex_config.keys():
diff --git a/auto_round/export/export_to_autoround/export_to_nvfp_mx.py b/auto_round/export/export_to_autoround/export_to_nvfp_mx.py
index 81ffb3e51..506a6a85d 100644
--- a/auto_round/export/export_to_autoround/export_to_nvfp_mx.py
+++ b/auto_round/export/export_to_autoround/export_to_nvfp_mx.py
@@ -16,11 +16,9 @@ import copy
import inspect
import json
import os
-from concurrent.futures import ThreadPoolExecutor
from dataclasses import fields
from typing import Callable, Union
-import threadpoolctl as tctl
import torch
import torch.nn as nn
import transformers
@@ -237,20 +235,8 @@ def save_quantized_as_fp(
if len(extra_config) > 0:
quantization_config["extra_config"] = extra_config
names = list(layer_config.keys())
- max_workers = 1
- if not torch.cuda.is_available() and not torch.xpu.is_available():
- max_workers = 2 ## 2 with cuda packing will cause hang occasionally
- with ThreadPoolExecutor(max_workers=max_workers) as executor:
- with tqdm(total=len(names), leave=True) as pbar:
-
- def wrapper(name):
- pbar.set_description(f"packing {name}")
- with tctl.threadpool_limits(limits=1):
- pack_layer(name, model, backend, device)
- pbar.update(1)
-
- for _ in executor.map(wrapper, names):
- pass
+ for name in tqdm(names, desc="packing", leave=True):
+ pack_layer(name, model, backend, device)
filter_quantization_config(quantization_config)
if hasattr(model, "config"):
diff --git a/auto_round/export/export_to_awq/export.py b/auto_round/export/export_to_awq/export.py
index 1fedf3c60..cc178097a 100644
--- a/auto_round/export/export_to_awq/export.py
+++ b/auto_round/export/export_to_awq/export.py
@@ -23,10 +23,8 @@
import copy
import json
import os
-from concurrent.futures import ThreadPoolExecutor
from typing import Callable, Union
-import threadpoolctl as tctl
import torch
import torch.nn as nn
from tqdm import tqdm
@@ -179,21 +177,9 @@ def save_quantized_as_autoawq(
names = list(layer_config.keys())
backend = None
- max_workers = 1
- if not torch.cuda.is_available() and not torch.xpu.is_available():
- max_workers = 2 ## 2 with cuda packing will cause hang occasionally
if not unsupported_meta_device(model):
- with ThreadPoolExecutor(max_workers=max_workers) as executor:
- with tqdm(total=len(names), leave=True) as pbar:
-
- def wrapper(name):
- pbar.set_description(f"packing {name}")
- with tctl.threadpool_limits(limits=1):
- pack_layer(name, compressed_model, backend, device)
- pbar.update(1)
-
- for _ in executor.map(wrapper, names):
- pass
+ for name in tqdm(names, desc="packing", leave=True):
+ pack_layer(name, compressed_model, backend, device)
if output_dir is None:
return model
diff --git a/auto_round/export/export_to_llmcompressor/export_to_fp.py b/auto_round/export/export_to_llmcompressor/export_to_fp.py
index 165903396..b33bbb951 100644
--- a/auto_round/export/export_to_llmcompressor/export_to_fp.py
+++ b/auto_round/export/export_to_llmcompressor/export_to_fp.py
@@ -17,10 +17,8 @@ import inspect
import json
import os
import sys
-from concurrent.futures import ThreadPoolExecutor
from typing import Callable, Union
-import threadpoolctl as tctl
import torch
import torch.nn as nn
import transformers
@@ -264,21 +262,9 @@ def save_quantized_as_fp(
update_fused_layer_global_scales(module, base_name="input")
names = list(layer_config.keys())
- max_workers = 1
- if not torch.cuda.is_available() or not torch.xpu.is_available():
- max_workers = 2 ## 2 with cuda packing will cause hang occasionally
if not unsupported_meta_device(model):
- with ThreadPoolExecutor(max_workers=max_workers) as executor:
- with tqdm(total=len(names), leave=True) as pbar:
-
- def wrapper(name):
- pbar.set_description(f"packing {name}")
- with tctl.threadpool_limits(limits=1):
- pack_layer(name, model, device)
- pbar.update(1)
-
- for _ in executor.map(wrapper, names):
- pass
+ for name in tqdm(names, desc="packing", leave=True):
+ pack_layer(name, model, device)
ignore = generate_ignore_regex_list(regex_config=regex_config, layer_config=layer_config)
lm_head_name = get_lm_head_name(model)
diff --git a/auto_round/export/export_to_llmcompressor/export_to_static_fp.py b/auto_round/export/export_to_llmcompressor/export_to_static_fp.py
index 6c534f8fc..c733c4510 100644
--- a/auto_round/export/export_to_llmcompressor/export_to_static_fp.py
+++ b/auto_round/export/export_to_llmcompressor/export_to_static_fp.py
@@ -16,10 +16,8 @@ import copy
import json
import os
import sys
-from concurrent.futures import ThreadPoolExecutor
from typing import Callable, Union
-import threadpoolctl as tctl
import torch
import transformers
from tqdm import tqdm
@@ -36,7 +34,6 @@ from auto_round.utils import (
get_module,
get_packing_device,
is_gaudi2,
- is_hpex_available,
logger,
set_module,
unsupported_meta_device,
@@ -151,26 +148,9 @@ def save_quantized_as_static_fp(
image_processor = kwargs.get("image_processor", None)
names = list(layer_config.keys())
- max_workers = 1
- if not torch.cuda.is_available() and not torch.xpu.is_available():
- max_workers = 2 ## 2 with cuda packing will cause hang occasionally
if not unsupported_meta_device(model):
-
- if is_hpex_available(): # packing will cause hang occasionally on hpu
- for name in tqdm(names, total=len(names), leave=True, desc="packing"):
- pack_layer(name, model, serialization_dict.get("data_type", "fp8"), device)
- else:
- with ThreadPoolExecutor(max_workers=max_workers) as executor:
- with tqdm(total=len(names), leave=True) as pbar:
-
- def wrapper(name):
- pbar.set_description(f"packing {name}")
- with tctl.threadpool_limits(limits=1):
- pack_layer(name, model, serialization_dict.get("data_type", "fp8"), device)
- pbar.update(1)
-
- for _ in executor.map(wrapper, names):
- pass
+ for name in tqdm(names, desc="packing", leave=True):
+ pack_layer(name, model, serialization_dict.get("data_type", "fp8"), device)
# Get llm-compressor format config
check_compressed_tensors_supported()
diff --git a/requirements-hpu.txt b/requirements-hpu.txt
index d2704fd1a..0fbe967a4 100644
--- a/requirements-hpu.txt
+++ b/requirements-hpu.txt
@@ -7,4 +7,3 @@ tqdm
packaging
pillow
transformers
-threadpoolctl
diff --git a/requirements.txt b/requirements.txt
index b7abf7181..1bb587b64 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -3,7 +3,6 @@ accelerate
datasets
numpy
py-cpuinfo
-threadpoolctl
torch
tqdm
transformers>=4.38
diff --git a/test/helpers.py b/test/helpers.py
index fcf62b116..8c85d39c5 100644
--- a/test/helpers.py
+++ b/test/helpers.py
@@ -1,3 +1,4 @@
+import concurrent.futures
import copy
import os
import re
@@ -14,6 +15,18 @@ from auto_round.utils import detect_device, diffusion_load_model, get_attr, llm_
transformers_version = version.parse(transformers.__version__)
+def _raise_threaded_packing(*args, **kwargs):
+ raise AssertionError("Packing should not create a thread pool or call threadpoolctl.")
+
+
+def forbid_threaded_packing(monkeypatch, module):
+ monkeypatch.setattr(concurrent.futures, "ThreadPoolExecutor", _raise_threaded_packing)
+ monkeypatch.setattr(module, "ThreadPoolExecutor", _raise_threaded_packing, raising=False)
+ tctl = getattr(module, "tctl", None)
+ if tctl is not None:
+ monkeypatch.setattr(tctl, "threadpool_limits", _raise_threaded_packing)
+
+
def generate_prompt(model_obj_or_str, tokenizer=None, text="The capital of France is,", max_new_tokens=10, device=None):
"""Generate text using a model and tokenizer.
diff --git a/test/test_cpu/export/test_export.py b/test/test_cpu/export/test_export.py
index a7655ff2e..0f4ec3ce9 100644
--- a/test/test_cpu/export/test_export.py
+++ b/test/test_cpu/export/test_export.py
@@ -8,8 +8,12 @@ from packaging import version
from transformers import AutoModelForCausalLM, AutoRoundConfig, AutoTokenizer
from auto_round import AutoRound
+from auto_round.export.export_to_autogptq import export as autogptq_export
+from auto_round.export.export_to_autoround import export as autoround_export
+from auto_round.export.export_to_autoround import export_to_fp8 as autoround_fp8_export
+from auto_round.export.export_to_awq import export as awq_export
-from ...helpers import get_model_path, opt_name_or_path, transformers_version
+from ...helpers import forbid_threaded_packing, get_model_path, opt_name_or_path, transformers_version
def _get_folder_size(path: str) -> float:
@@ -552,3 +556,46 @@ class TestAutoRound:
len(input_scale_keys) == 0
), f"Expected no input_scale for weight-only {scheme}, but found: {input_scale_keys[:5]}"
shutil.rmtree(quantized_model_path, ignore_errors=True)
+
+
+ "format_name,export_module,sym",
+ [
+ ("auto_gptq", autogptq_export, False),
+ ("auto_awq", awq_export, False),
+ ("auto_round", autoround_export, True),
+ ],
+)
+def test_weight_only_exports_pack_serially(tiny_opt_model_path, tmp_path, monkeypatch, format_name, export_module, sym):
+ autoround = AutoRound(
+ tiny_opt_model_path,
+ bits=4,
+ group_size=128,
+ sym=sym,
+ iters=0,
+ disable_opt_rtn=True,
+ )
+ autoround.quantize()
+ forbid_threaded_packing(monkeypatch, export_module)
+ autoround.save_quantized(output_dir=tmp_path, inplace=False, format=format_name)
+ assert os.path.exists(os.path.join(tmp_path, "config.json"))
+
+
+def test_fp8_autoround_export_packs_serially(tiny_opt_model_path, tmp_path, monkeypatch):
+ from safetensors import safe_open
+
+ autoround = AutoRound(
+ tiny_opt_model_path,
+ bits=8,
+ group_size=-1,
+ iters=0,
+ scheme="FP8_STATIC",
+ nsamples=2,
+ seqlen=2,
+ static_kv_dtype="fp8",
+ )
+ autoround.quantize()
+ forbid_threaded_packing(monkeypatch, autoround_fp8_export)
+ autoround.save_quantized(output_dir=tmp_path, format="auto_round")
+ with safe_open(os.path.join(tmp_path, "model.safetensors"), framework="pt") as f:
+ assert "model.decoder.layers.0.self_attn.k_proj.weight_scale" in f.keys()
diff --git a/test/test_cpu/export/test_llmc_format.py b/test/test_cpu/export/test_llmc_format.py
index c6114c0df..2a6a0a5ec 100644
--- a/test/test_cpu/export/test_llmc_format.py
+++ b/test/test_cpu/export/test_llmc_format.py
@@ -7,8 +7,10 @@ import transformers
from transformers import AutoModelForCausalLM, AutoTokenizer
from auto_round import AutoRound, AutoScheme
+from auto_round.export.export_to_llmcompressor import export_to_fp as llmc_fp_export
+from auto_round.export.export_to_llmcompressor import export_to_static_fp as llmc_static_fp_export
-from ...helpers import get_model_path, opt_name_or_path
+from ...helpers import forbid_threaded_packing, get_model_path, opt_name_or_path
class TestLLMC:
@@ -156,3 +158,31 @@ class TestLLMC:
and quantization_config["config_groups"]["group_1"]["format"] == "mxfp4-pack-quantized"
and quantization_config["ignore"] == ["lm_head"]
), f"Invalid mixed precision quantization configuration: {quantization_config}"
+
+
+def test_llmcompressor_static_fp_export_packs_serially(tiny_opt_model_path, tmp_path, monkeypatch):
+ autoround = AutoRound(
+ tiny_opt_model_path,
+ scheme="FP8_STATIC",
+ seqlen=8,
+ nsamples=2,
+ iters=0,
+ )
+ autoround.quantize()
+ forbid_threaded_packing(monkeypatch, llmc_static_fp_export)
+ autoround.save_quantized(tmp_path, format="llm_compressor")
+ assert os.path.exists(os.path.join(tmp_path, "config.json"))
+
+
+def test_llmcompressor_mxfp8_export_packs_serially(tmp_path, monkeypatch):
+ autoround = AutoRound(
+ model=opt_name_or_path,
+ iters=0,
+ disable_opt_rtn=True,
+ scheme="mxfp8",
+ )
+ autoround.quantize()
+ forbid_threaded_packing(monkeypatch, llmc_fp_export)
+ compressed_model = autoround.save_quantized(output_dir=tmp_path, format="llm_compressor")
+ tmp_layer = compressed_model.model.decoder.layers[1].self_attn.q_proj
+ assert hasattr(tmp_layer, "weight_scale")
diff --git a/test/test_cpu/quantization/test_mxfp_nvfp.py b/test/test_cpu/quantization/test_mxfp_nvfp.py
index 2f92f73f8..d0c15ccf8 100644
--- a/test/test_cpu/quantization/test_mxfp_nvfp.py
+++ b/test/test_cpu/quantization/test_mxfp_nvfp.py
@@ -8,8 +8,9 @@ from packaging import version
from transformers import AutoModelForCausalLM, AutoRoundConfig, AutoTokenizer
from auto_round import AutoRound
+from auto_round.export.export_to_autoround import export_to_nvfp_mx as autoround_nvfp_mx_export
-from ...helpers import is_model_outputs_similar, transformers_version
+from ...helpers import forbid_threaded_packing, is_model_outputs_similar, transformers_version
def _get_folder_size(path: str) -> float:
@@ -289,7 +290,7 @@ class TestAutoRoundFP:
and tmp_layer.weight_scale.shape[0] == 768
), "Illegal NVFP4 packing name or data_type or shape"
- def test_nvfp4_autoround_save_quantized(self, tiny_opt_model_path, dataloader):
+ def test_nvfp4_autoround_save_quantized(self, tiny_opt_model_path, dataloader, monkeypatch):
model_name = tiny_opt_model_path
from transformers import AutoConfig
@@ -303,6 +304,7 @@ class TestAutoRoundFP:
)
quantized_model_path = self.save_dir
autoround.quantize()
+ forbid_threaded_packing(monkeypatch, autoround_nvfp_mx_export)
compressed_model = autoround.save_quantized(output_dir=quantized_model_path, format="auto_round")
tmp_layer = compressed_model.model.decoder.layers[1].self_attn.q_proj
assert (