PTQ-Level Efficiency, QAT-Level Accuracy | 训练后量化的效率,量化感知训练的精度
simply CpuInfo class (#715)
Wenhua Chengc731b65cf964
diff --git a/auto_round/eval/evaluation.py b/auto_round/eval/evaluation.pyindex 7ba4fb8d1..f426a8a11 100644--- a/auto_round/eval/evaluation.py+++ b/auto_round/eval/evaluation.py@@ -53,7 +53,7 @@ def simple_evaluate(**kwargs):try:- from auto_round import AutoRoundConfig+ from transformers import AutoRoundConfigexcept:from auto_round.inference.auto_quantizer import AutoHfQuantizerdiff --git a/auto_round/mllm/eval.py b/auto_round/mllm/eval.pyindex db2f7882e..4d3f5e8e8 100644--- a/auto_round/mllm/eval.py+++ b/auto_round/mllm/eval.py@@ -101,7 +101,7 @@ def mllm_eval(ignore: bool = False,):try:- from auto_round import AutoRoundConfig+ from transformers import AutoRoundConfigexcept:from auto_round.inference.auto_quantizer import AutoHfQuantizerdiff --git a/auto_round/script/llm.py b/auto_round/script/llm.pyindex 1b042d31e..0245a7f72 100644--- a/auto_round/script/llm.py+++ b/auto_round/script/llm.py@@ -773,7 +773,6 @@ def eval_task_by_task(from lm_eval.models.huggingface import HFLMfrom transformers import AutoModelForCausalLM, AutoTokenizer- from auto_round import AutoRoundConfig # pylint: disable=E0611from auto_round.utils import loggerif batch_size is None:diff --git a/auto_round/utils.py b/auto_round/utils.pyindex f1d160757..dc738c122 100644--- a/auto_round/utils.py+++ b/auto_round/utils.py@@ -584,65 +584,22 @@ class CpuInfo(object):def __init__(self):"""Get whether the cpu numerical format is bf16, the number of sockets, cores and cores per socket."""self._bf16 = False- self._vnni = Falseinfo = cpuinfo.get_cpu_info()if "arch" in info and "X86" in info["arch"]:cpuid = cpuinfo.CPUID()max_extension_support = cpuid.get_max_extension_support()if max_extension_support >= 7:- ecx = cpuid._run_asm(- b"\x31\xc9", # xor ecx, ecx- b"\xb8\x07\x00\x00\x00" b"\x0f\xa2" b"\x89\xc8" b"\xc3", # mov eax, 7 # cpuid # mov ax, cx # ret- )- self._vnni = bool(ecx & (1 << 11))eax = cpuid._run_asm(b"\xb9\x01\x00\x00\x00", # mov ecx, 1b"\xb8\x07\x00\x00\x00" b"\x0f\xa2" b"\xc3", # mov eax, 7 # cpuid # ret)self._bf16 = bool(eax & (1 << 5))- if "arch" in info and "ARM" in info["arch"]: # pragma: no cover- self._sockets = 1- else:- self._sockets = self.get_number_of_sockets()- self._cores = psutil.cpu_count(logical=False)- self._cores_per_socket = int(self._cores / self._sockets)@propertydef bf16(self):"""Get whether it is bf16."""return self._bf16- @property- def vnni(self):- """Get whether it is vnni."""- return self._vnni-- @property- def cores_per_socket(self):- """Get the cores per socket."""- return self._cores_per_socket-- def get_number_of_sockets(self) -> int:- """Get number of sockets in platform."""- cmd = "cat /proc/cpuinfo | grep 'physical id' | sort -u | wc -l"- if psutil.WINDOWS:- cmd = r'wmic cpu get DeviceID | C:\Windows\System32\find.exe /C "CPU"'- elif psutil.MACOS: # pragma: no cover- cmd = "sysctl -n machdep.cpu.core_count"-- with subprocess.Popen(- args=cmd,- shell=True,- stdout=subprocess.PIPE,- stderr=subprocess.STDOUT,- universal_newlines=False,- ) as proc:- proc.wait()- if proc.stdout:- for line in proc.stdout:- return int(line.decode("utf-8", errors="ignore").strip())- return 0-def is_local_path(path):"""Checks if a given path exists locally.diff --git a/docs/step_by_step.md b/docs/step_by_step.mdindex 2d6cf5997..9a65c84bb 100644--- a/docs/step_by_step.md+++ b/docs/step_by_step.md@@ -566,8 +566,7 @@ The backend may not always be the most suitable for certain devices.You can specify your preferred backend such as "ipex" for CPU and Intel GPU, "marlin/exllamav2/triton" for CUDA, according to your needs or hardware compatibility. Please note that additional corresponding libraries may be required.```python-from transformers import AutoModelForCausalLM, AutoTokenizer-from auto_round import AutoRoundConfig+from transformers import AutoModelForCausalLM, AutoTokenizer, AutoRoundConfigmodel_name = "OPEA/Qwen2.5-1.5B-Instruct-int4-sym-inc"quantization_config = AutoRoundConfig(backend="ipex")@@ -598,8 +597,7 @@ print(tokenizer.decode(model.generate(**inputs, max_new_tokens=50, do_sample=FalMost GPTQ/AWQ models can be converted to the AutoRound format for better compatibility and support with Intel devices. Please note that the quantization config will be changed if the model is serialized.```python-from transformers import AutoModelForCausalLM, AutoTokenizer-from auto_round import AutoRoundConfig+from transformers import AutoModelForCausalLM, AutoTokenizer, AutoRoundConfigmodel_name = "ybelkada/opt-125m-gptq-4bit"quantization_config = AutoRoundConfig()diff --git a/test/test_cpu/test_autoround.py b/test/test_cpu/test_autoround.pyindex 84cf50f7e..afbd19ae0 100644--- a/test/test_cpu/test_autoround.py+++ b/test/test_cpu/test_autoround.py@@ -596,15 +596,15 @@ class TestAutoRound(unittest.TestCase):)autoround.quantize()- def test_fp8_model_input_rtn_generation(self):- model_name = "Qwen/Qwen3-0.6B-FP8"- ar = AutoRound(model=model_name, iters=0)- ar.quantize_and_save(output_dir=self.save_folder)- model = AutoModelForCausalLM.from_pretrained(self.save_folder, torch_dtype="auto", trust_remote_code=True)- tokenizer = AutoTokenizer.from_pretrained(self.save_folder)- text = "There is a girl who likes adventure,"- inputs = tokenizer(text, return_tensors="pt").to(model.device)- print(tokenizer.decode(model.generate(**inputs, max_new_tokens=50)[0]))+ # def test_fp8_model_input_rtn_generation(self):+ # model_name = "Qwen/Qwen3-0.6B-FP8"+ # ar = AutoRound(model=model_name, iters=0)+ # ar.quantize_and_save(output_dir=self.save_folder)+ # model = AutoModelForCausalLM.from_pretrained(self.save_folder, torch_dtype="auto", trust_remote_code=True)+ # tokenizer = AutoTokenizer.from_pretrained(self.save_folder)+ # text = "There is a girl who likes adventure,"+ # inputs = tokenizer(text, return_tensors="pt").to(model.device)+ # print(tokenizer.decode(model.generate(**inputs, max_new_tokens=50)[0]))if __name__ == "__main__":diff --git a/test/test_cuda/test_auto_round_format.py b/test/test_cuda/test_auto_round_format.pyindex f5c6ae5df..67c0f9b02 100644--- a/test/test_cuda/test_auto_round_format.py+++ b/test/test_cuda/test_auto_round_format.py@@ -199,7 +199,7 @@ class TestAutoRound(unittest.TestCase):autoround.quantize_and_save(output_dir=quantized_model_path)- from auto_round import AutoRoundConfig+ from transformers import AutoRoundConfigquantization_config = AutoRoundConfig(backend="ipex_gptq")