PTQ-Level Efficiency, QAT-Level Accuracy | 训练后量化的效率,量化感知训练的精度
Remove threaded packing from exporters (#1719)
Yi Liu85a619260aa0
diff --git a/auto_round/export/export_to_autogptq/export.py b/auto_round/export/export_to_autogptq/export.pyindex 98fde7b73..5594b5979 100644--- a/auto_round/export/export_to_autogptq/export.py+++ b/auto_round/export/export_to_autogptq/export.py@@ -16,12 +16,9 @@ import copyimport inspectimport jsonimport os-from concurrent.futures import ThreadPoolExecutorfrom dataclasses import fieldsfrom typing import Any, Callable, Dict, Union-import threadpoolctl as tctl-# MIT License## Copyright (c) 2023 潘其威(William)@@ -295,21 +292,9 @@ def save_quantized_as_autogptq(model = copy.deepcopy(model.to("cpu"))names = list(layer_config.keys())- max_workers = 1- if not torch.cuda.is_available() and not torch.xpu.is_available():- max_workers = 2 ## 2 with cuda packing will cause hang occasionallyif not unsupported_meta_device(model):- with ThreadPoolExecutor(max_workers=max_workers) as executor:- with tqdm(total=len(names), leave=True) as pbar:-- def wrapper(name):- pbar.set_description(f"packing {name}")- with tctl.threadpool_limits(limits=1):- pack_layer(name, model, backend, device)- pbar.update(1)-- for _ in executor.map(wrapper, names):- pass+ for name in tqdm(names, desc="packing", leave=True):+ pack_layer(name, model, backend, device)if output_dir is None:return modelquantization_config["lm_head"] = lm_head_quantizeddiff --git a/auto_round/export/export_to_autoround/export.py b/auto_round/export/export_to_autoround/export.pyindex 372638e99..60613b7e8 100644--- a/auto_round/export/export_to_autoround/export.py+++ b/auto_round/export/export_to_autoround/export.py@@ -18,12 +18,10 @@ import functoolsimport inspectimport jsonimport os-from concurrent.futures import ThreadPoolExecutorfrom dataclasses import fieldsfrom enum import Enumfrom typing import Callable, Union-import threadpoolctl as tctlimport torchimport torch.nn as nnimport transformers@@ -309,21 +307,9 @@ def save_quantized_as_autoround(quantization_config["extra_config"] = extra_confignames = list(layer_config.keys())- max_workers = 1- if not torch.cuda.is_available() and not torch.xpu.is_available():- max_workers = 2 ## 2 with cuda packing will cause hang occasionallyif not unsupported_meta_device(model):- with ThreadPoolExecutor(max_workers=max_workers) as executor:- with tqdm(total=len(names), leave=True) as pbar:-- def wrapper(name):- pbar.set_description(f"packing {name}")- with tctl.threadpool_limits(limits=1):- pack_layer(name, model, backend, device)- pbar.update(1)-- for _ in executor.map(wrapper, names):- pass+ for name in tqdm(names, desc="packing", leave=True):+ pack_layer(name, model, backend, device)filter_quantization_config(quantization_config)if hasattr(model, "config"):model.config.quantization_config = quantization_configdiff --git a/auto_round/export/export_to_autoround/export_to_fp8.py b/auto_round/export/export_to_autoround/export_to_fp8.pyindex 38af5b7b5..90c228583 100644--- a/auto_round/export/export_to_autoround/export_to_fp8.py+++ b/auto_round/export/export_to_autoround/export_to_fp8.py@@ -15,11 +15,9 @@import copyimport jsonimport os-from concurrent.futures import ThreadPoolExecutorfrom dataclasses import fieldsfrom typing import Callable, Union-import threadpoolctl as tctlimport torchimport transformersfrom tqdm import tqdm@@ -240,20 +238,8 @@ def save_quantized_as_autoround(if len(extra_config) > 0:quantization_config["extra_config"] = extra_confignames = list(layer_config.keys())- max_workers = 1- if not torch.cuda.is_available() and not torch.xpu.is_available():- max_workers = 2 ## 2 with cuda packing will cause hang occasionally- with ThreadPoolExecutor(max_workers=max_workers) as executor:- with tqdm(total=len(names), leave=True) as pbar:-- def wrapper(name):- pbar.set_description(f"packing {name}")- with tctl.threadpool_limits(limits=1):- pack_layer(name, model, serialization_dict.get("data_type", "fp8"), device)- pbar.update(1)-- for _ in executor.map(wrapper, names):- pass+ for name in tqdm(names, desc="packing", leave=True):+ pack_layer(name, model, serialization_dict.get("data_type", "fp8"), device)regex_config = quantization_config.pop("regex_config")if regex_config is not None:for name in regex_config.keys():diff --git a/auto_round/export/export_to_autoround/export_to_nvfp_mx.py b/auto_round/export/export_to_autoround/export_to_nvfp_mx.pyindex 81ffb3e51..506a6a85d 100644--- a/auto_round/export/export_to_autoround/export_to_nvfp_mx.py+++ b/auto_round/export/export_to_autoround/export_to_nvfp_mx.py@@ -16,11 +16,9 @@ import copyimport inspectimport jsonimport os-from concurrent.futures import ThreadPoolExecutorfrom dataclasses import fieldsfrom typing import Callable, Union-import threadpoolctl as tctlimport torchimport torch.nn as nnimport transformers@@ -237,20 +235,8 @@ def save_quantized_as_fp(if len(extra_config) > 0:quantization_config["extra_config"] = extra_confignames = list(layer_config.keys())- max_workers = 1- if not torch.cuda.is_available() and not torch.xpu.is_available():- max_workers = 2 ## 2 with cuda packing will cause hang occasionally- with ThreadPoolExecutor(max_workers=max_workers) as executor:- with tqdm(total=len(names), leave=True) as pbar:-- def wrapper(name):- pbar.set_description(f"packing {name}")- with tctl.threadpool_limits(limits=1):- pack_layer(name, model, backend, device)- pbar.update(1)-- for _ in executor.map(wrapper, names):- pass+ for name in tqdm(names, desc="packing", leave=True):+ pack_layer(name, model, backend, device)filter_quantization_config(quantization_config)if hasattr(model, "config"):diff --git a/auto_round/export/export_to_awq/export.py b/auto_round/export/export_to_awq/export.pyindex 1fedf3c60..cc178097a 100644--- a/auto_round/export/export_to_awq/export.py+++ b/auto_round/export/export_to_awq/export.py@@ -23,10 +23,8 @@import copyimport jsonimport os-from concurrent.futures import ThreadPoolExecutorfrom typing import Callable, Union-import threadpoolctl as tctlimport torchimport torch.nn as nnfrom tqdm import tqdm@@ -179,21 +177,9 @@ def save_quantized_as_autoawq(names = list(layer_config.keys())backend = None- max_workers = 1- if not torch.cuda.is_available() and not torch.xpu.is_available():- max_workers = 2 ## 2 with cuda packing will cause hang occasionallyif not unsupported_meta_device(model):- with ThreadPoolExecutor(max_workers=max_workers) as executor:- with tqdm(total=len(names), leave=True) as pbar:-- def wrapper(name):- pbar.set_description(f"packing {name}")- with tctl.threadpool_limits(limits=1):- pack_layer(name, compressed_model, backend, device)- pbar.update(1)-- for _ in executor.map(wrapper, names):- pass+ for name in tqdm(names, desc="packing", leave=True):+ pack_layer(name, compressed_model, backend, device)if output_dir is None:return modeldiff --git a/auto_round/export/export_to_llmcompressor/export_to_fp.py b/auto_round/export/export_to_llmcompressor/export_to_fp.pyindex 165903396..b33bbb951 100644--- a/auto_round/export/export_to_llmcompressor/export_to_fp.py+++ b/auto_round/export/export_to_llmcompressor/export_to_fp.py@@ -17,10 +17,8 @@ import inspectimport jsonimport osimport sys-from concurrent.futures import ThreadPoolExecutorfrom typing import Callable, Union-import threadpoolctl as tctlimport torchimport torch.nn as nnimport transformers@@ -264,21 +262,9 @@ def save_quantized_as_fp(update_fused_layer_global_scales(module, base_name="input")names = list(layer_config.keys())- max_workers = 1- if not torch.cuda.is_available() or not torch.xpu.is_available():- max_workers = 2 ## 2 with cuda packing will cause hang occasionallyif not unsupported_meta_device(model):- with ThreadPoolExecutor(max_workers=max_workers) as executor:- with tqdm(total=len(names), leave=True) as pbar:-- def wrapper(name):- pbar.set_description(f"packing {name}")- with tctl.threadpool_limits(limits=1):- pack_layer(name, model, device)- pbar.update(1)-- for _ in executor.map(wrapper, names):- pass+ for name in tqdm(names, desc="packing", leave=True):+ pack_layer(name, model, device)ignore = generate_ignore_regex_list(regex_config=regex_config, layer_config=layer_config)lm_head_name = get_lm_head_name(model)diff --git a/auto_round/export/export_to_llmcompressor/export_to_static_fp.py b/auto_round/export/export_to_llmcompressor/export_to_static_fp.pyindex 6c534f8fc..c733c4510 100644--- a/auto_round/export/export_to_llmcompressor/export_to_static_fp.py+++ b/auto_round/export/export_to_llmcompressor/export_to_static_fp.py@@ -16,10 +16,8 @@ import copyimport jsonimport osimport sys-from concurrent.futures import ThreadPoolExecutorfrom typing import Callable, Union-import threadpoolctl as tctlimport torchimport transformersfrom tqdm import tqdm@@ -36,7 +34,6 @@ from auto_round.utils import (get_module,get_packing_device,is_gaudi2,- is_hpex_available,logger,set_module,unsupported_meta_device,@@ -151,26 +148,9 @@ def save_quantized_as_static_fp(image_processor = kwargs.get("image_processor", None)names = list(layer_config.keys())- max_workers = 1- if not torch.cuda.is_available() and not torch.xpu.is_available():- max_workers = 2 ## 2 with cuda packing will cause hang occasionallyif not unsupported_meta_device(model):-- if is_hpex_available(): # packing will cause hang occasionally on hpu- for name in tqdm(names, total=len(names), leave=True, desc="packing"):- pack_layer(name, model, serialization_dict.get("data_type", "fp8"), device)- else:- with ThreadPoolExecutor(max_workers=max_workers) as executor:- with tqdm(total=len(names), leave=True) as pbar:-- def wrapper(name):- pbar.set_description(f"packing {name}")- with tctl.threadpool_limits(limits=1):- pack_layer(name, model, serialization_dict.get("data_type", "fp8"), device)- pbar.update(1)-- for _ in executor.map(wrapper, names):- pass+ for name in tqdm(names, desc="packing", leave=True):+ pack_layer(name, model, serialization_dict.get("data_type", "fp8"), device)# Get llm-compressor format configcheck_compressed_tensors_supported()diff --git a/requirements-hpu.txt b/requirements-hpu.txtindex d2704fd1a..0fbe967a4 100644--- a/requirements-hpu.txt+++ b/requirements-hpu.txt@@ -7,4 +7,3 @@ tqdmpackagingpillowtransformers-threadpoolctldiff --git a/requirements.txt b/requirements.txtindex b7abf7181..1bb587b64 100644--- a/requirements.txt+++ b/requirements.txt@@ -3,7 +3,6 @@ acceleratedatasetsnumpypy-cpuinfo-threadpoolctltorchtqdmtransformers>=4.38diff --git a/test/helpers.py b/test/helpers.pyindex fcf62b116..8c85d39c5 100644--- a/test/helpers.py+++ b/test/helpers.py@@ -1,3 +1,4 @@+import concurrent.futuresimport copyimport osimport re@@ -14,6 +15,18 @@ from auto_round.utils import detect_device, diffusion_load_model, get_attr, llm_transformers_version = version.parse(transformers.__version__)+def _raise_threaded_packing(*args, **kwargs):+ raise AssertionError("Packing should not create a thread pool or call threadpoolctl.")+++def forbid_threaded_packing(monkeypatch, module):+ monkeypatch.setattr(concurrent.futures, "ThreadPoolExecutor", _raise_threaded_packing)+ monkeypatch.setattr(module, "ThreadPoolExecutor", _raise_threaded_packing, raising=False)+ tctl = getattr(module, "tctl", None)+ if tctl is not None:+ monkeypatch.setattr(tctl, "threadpool_limits", _raise_threaded_packing)++def generate_prompt(model_obj_or_str, tokenizer=None, text="The capital of France is,", max_new_tokens=10, device=None):"""Generate text using a model and tokenizer.diff --git a/test/test_cpu/export/test_export.py b/test/test_cpu/export/test_export.pyindex a7655ff2e..0f4ec3ce9 100644--- a/test/test_cpu/export/test_export.py+++ b/test/test_cpu/export/test_export.py@@ -8,8 +8,12 @@ from packaging import versionfrom transformers import AutoModelForCausalLM, AutoRoundConfig, AutoTokenizerfrom auto_round import AutoRound+from auto_round.export.export_to_autogptq import export as autogptq_export+from auto_round.export.export_to_autoround import export as autoround_export+from auto_round.export.export_to_autoround import export_to_fp8 as autoround_fp8_export+from auto_round.export.export_to_awq import export as awq_export-from ...helpers import get_model_path, opt_name_or_path, transformers_version+from ...helpers import forbid_threaded_packing, get_model_path, opt_name_or_path, transformers_versiondef _get_folder_size(path: str) -> float:@@ -552,3 +556,46 @@ class TestAutoRound:len(input_scale_keys) == 0), f"Expected no input_scale for weight-only {scheme}, but found: {input_scale_keys[:5]}"shutil.rmtree(quantized_model_path, ignore_errors=True)+++ "format_name,export_module,sym",+ [+ ("auto_gptq", autogptq_export, False),+ ("auto_awq", awq_export, False),+ ("auto_round", autoround_export, True),+ ],+)+def test_weight_only_exports_pack_serially(tiny_opt_model_path, tmp_path, monkeypatch, format_name, export_module, sym):+ autoround = AutoRound(+ tiny_opt_model_path,+ bits=4,+ group_size=128,+ sym=sym,+ iters=0,+ disable_opt_rtn=True,+ )+ autoround.quantize()+ forbid_threaded_packing(monkeypatch, export_module)+ autoround.save_quantized(output_dir=tmp_path, inplace=False, format=format_name)+ assert os.path.exists(os.path.join(tmp_path, "config.json"))+++def test_fp8_autoround_export_packs_serially(tiny_opt_model_path, tmp_path, monkeypatch):+ from safetensors import safe_open++ autoround = AutoRound(+ tiny_opt_model_path,+ bits=8,+ group_size=-1,+ iters=0,+ scheme="FP8_STATIC",+ nsamples=2,+ seqlen=2,+ static_kv_dtype="fp8",+ )+ autoround.quantize()+ forbid_threaded_packing(monkeypatch, autoround_fp8_export)+ autoround.save_quantized(output_dir=tmp_path, format="auto_round")+ with safe_open(os.path.join(tmp_path, "model.safetensors"), framework="pt") as f:+ assert "model.decoder.layers.0.self_attn.k_proj.weight_scale" in f.keys()diff --git a/test/test_cpu/export/test_llmc_format.py b/test/test_cpu/export/test_llmc_format.pyindex c6114c0df..2a6a0a5ec 100644--- a/test/test_cpu/export/test_llmc_format.py+++ b/test/test_cpu/export/test_llmc_format.py@@ -7,8 +7,10 @@ import transformersfrom transformers import AutoModelForCausalLM, AutoTokenizerfrom auto_round import AutoRound, AutoScheme+from auto_round.export.export_to_llmcompressor import export_to_fp as llmc_fp_export+from auto_round.export.export_to_llmcompressor import export_to_static_fp as llmc_static_fp_export-from ...helpers import get_model_path, opt_name_or_path+from ...helpers import forbid_threaded_packing, get_model_path, opt_name_or_pathclass TestLLMC:@@ -156,3 +158,31 @@ class TestLLMC:and quantization_config["config_groups"]["group_1"]["format"] == "mxfp4-pack-quantized"and quantization_config["ignore"] == ["lm_head"]), f"Invalid mixed precision quantization configuration: {quantization_config}"+++def test_llmcompressor_static_fp_export_packs_serially(tiny_opt_model_path, tmp_path, monkeypatch):+ autoround = AutoRound(+ tiny_opt_model_path,+ scheme="FP8_STATIC",+ seqlen=8,+ nsamples=2,+ iters=0,+ )+ autoround.quantize()+ forbid_threaded_packing(monkeypatch, llmc_static_fp_export)+ autoround.save_quantized(tmp_path, format="llm_compressor")+ assert os.path.exists(os.path.join(tmp_path, "config.json"))+++def test_llmcompressor_mxfp8_export_packs_serially(tmp_path, monkeypatch):+ autoround = AutoRound(+ model=opt_name_or_path,+ iters=0,+ disable_opt_rtn=True,+ scheme="mxfp8",+ )+ autoround.quantize()+ forbid_threaded_packing(monkeypatch, llmc_fp_export)+ compressed_model = autoround.save_quantized(output_dir=tmp_path, format="llm_compressor")+ tmp_layer = compressed_model.model.decoder.layers[1].self_attn.q_proj+ assert hasattr(tmp_layer, "weight_scale")diff --git a/test/test_cpu/quantization/test_mxfp_nvfp.py b/test/test_cpu/quantization/test_mxfp_nvfp.pyindex 2f92f73f8..d0c15ccf8 100644--- a/test/test_cpu/quantization/test_mxfp_nvfp.py+++ b/test/test_cpu/quantization/test_mxfp_nvfp.py@@ -8,8 +8,9 @@ from packaging import versionfrom transformers import AutoModelForCausalLM, AutoRoundConfig, AutoTokenizerfrom auto_round import AutoRound+from auto_round.export.export_to_autoround import export_to_nvfp_mx as autoround_nvfp_mx_export-from ...helpers import is_model_outputs_similar, transformers_version+from ...helpers import forbid_threaded_packing, is_model_outputs_similar, transformers_versiondef _get_folder_size(path: str) -> float:@@ -289,7 +290,7 @@ class TestAutoRoundFP:and tmp_layer.weight_scale.shape[0] == 768), "Illegal NVFP4 packing name or data_type or shape"- def test_nvfp4_autoround_save_quantized(self, tiny_opt_model_path, dataloader):+ def test_nvfp4_autoround_save_quantized(self, tiny_opt_model_path, dataloader, monkeypatch):model_name = tiny_opt_model_pathfrom transformers import AutoConfig@@ -303,6 +304,7 @@ class TestAutoRoundFP:)quantized_model_path = self.save_dirautoround.quantize()+ forbid_threaded_packing(monkeypatch, autoround_nvfp_mx_export)compressed_model = autoround.save_quantized(output_dir=quantized_model_path, format="auto_round")tmp_layer = compressed_model.model.decoder.layers[1].self_attn.q_projassert (