Skip to content

PTQ-Level Efficiency, QAT-Level Accuracy | 训练后量化的效率,量化感知训练的精度

simply CpuInfo class (#715)

Wenhua Chengc731b65cf964
diff --git a/auto_round/eval/evaluation.py b/auto_round/eval/evaluation.py
index 7ba4fb8d1..f426a8a11 100644
--- a/auto_round/eval/evaluation.py
+++ b/auto_round/eval/evaluation.py
@@ -53,7 +53,7 @@ def simple_evaluate(
**kwargs
):
try:
- from auto_round import AutoRoundConfig
+ from transformers import AutoRoundConfig
except:
from auto_round.inference.auto_quantizer import AutoHfQuantizer
diff --git a/auto_round/mllm/eval.py b/auto_round/mllm/eval.py
index db2f7882e..4d3f5e8e8 100644
--- a/auto_round/mllm/eval.py
+++ b/auto_round/mllm/eval.py
@@ -101,7 +101,7 @@ def mllm_eval(
ignore: bool = False,
):
try:
- from auto_round import AutoRoundConfig
+ from transformers import AutoRoundConfig
except:
from auto_round.inference.auto_quantizer import AutoHfQuantizer
diff --git a/auto_round/script/llm.py b/auto_round/script/llm.py
index 1b042d31e..0245a7f72 100644
--- a/auto_round/script/llm.py
+++ b/auto_round/script/llm.py
@@ -773,7 +773,6 @@ def eval_task_by_task(
from lm_eval.models.huggingface import HFLM
from transformers import AutoModelForCausalLM, AutoTokenizer
- from auto_round import AutoRoundConfig # pylint: disable=E0611
from auto_round.utils import logger
if batch_size is None:
diff --git a/auto_round/utils.py b/auto_round/utils.py
index f1d160757..dc738c122 100644
--- a/auto_round/utils.py
+++ b/auto_round/utils.py
@@ -584,65 +584,22 @@ class CpuInfo(object):
def __init__(self):
"""Get whether the cpu numerical format is bf16, the number of sockets, cores and cores per socket."""
self._bf16 = False
- self._vnni = False
info = cpuinfo.get_cpu_info()
if "arch" in info and "X86" in info["arch"]:
cpuid = cpuinfo.CPUID()
max_extension_support = cpuid.get_max_extension_support()
if max_extension_support >= 7:
- ecx = cpuid._run_asm(
- b"\x31\xc9", # xor ecx, ecx
- b"\xb8\x07\x00\x00\x00" b"\x0f\xa2" b"\x89\xc8" b"\xc3", # mov eax, 7 # cpuid # mov ax, cx # ret
- )
- self._vnni = bool(ecx & (1 << 11))
eax = cpuid._run_asm(
b"\xb9\x01\x00\x00\x00", # mov ecx, 1
b"\xb8\x07\x00\x00\x00" b"\x0f\xa2" b"\xc3", # mov eax, 7 # cpuid # ret
)
self._bf16 = bool(eax & (1 << 5))
- if "arch" in info and "ARM" in info["arch"]: # pragma: no cover
- self._sockets = 1
- else:
- self._sockets = self.get_number_of_sockets()
- self._cores = psutil.cpu_count(logical=False)
- self._cores_per_socket = int(self._cores / self._sockets)
@property
def bf16(self):
"""Get whether it is bf16."""
return self._bf16
- @property
- def vnni(self):
- """Get whether it is vnni."""
- return self._vnni
-
- @property
- def cores_per_socket(self):
- """Get the cores per socket."""
- return self._cores_per_socket
-
- def get_number_of_sockets(self) -> int:
- """Get number of sockets in platform."""
- cmd = "cat /proc/cpuinfo | grep 'physical id' | sort -u | wc -l"
- if psutil.WINDOWS:
- cmd = r'wmic cpu get DeviceID | C:\Windows\System32\find.exe /C "CPU"'
- elif psutil.MACOS: # pragma: no cover
- cmd = "sysctl -n machdep.cpu.core_count"
-
- with subprocess.Popen(
- args=cmd,
- shell=True,
- stdout=subprocess.PIPE,
- stderr=subprocess.STDOUT,
- universal_newlines=False,
- ) as proc:
- proc.wait()
- if proc.stdout:
- for line in proc.stdout:
- return int(line.decode("utf-8", errors="ignore").strip())
- return 0
-
def is_local_path(path):
"""Checks if a given path exists locally.
diff --git a/docs/step_by_step.md b/docs/step_by_step.md
index 2d6cf5997..9a65c84bb 100644
--- a/docs/step_by_step.md
+++ b/docs/step_by_step.md
@@ -566,8 +566,7 @@ The backend may not always be the most suitable for certain devices.
You can specify your preferred backend such as "ipex" for CPU and Intel GPU, "marlin/exllamav2/triton" for CUDA, according to your needs or hardware compatibility. Please note that additional corresponding libraries may be required.
```python
-from transformers import AutoModelForCausalLM, AutoTokenizer
-from auto_round import AutoRoundConfig
+from transformers import AutoModelForCausalLM, AutoTokenizer, AutoRoundConfig
model_name = "OPEA/Qwen2.5-1.5B-Instruct-int4-sym-inc"
quantization_config = AutoRoundConfig(backend="ipex")
@@ -598,8 +597,7 @@ print(tokenizer.decode(model.generate(**inputs, max_new_tokens=50, do_sample=Fal
Most GPTQ/AWQ models can be converted to the AutoRound format for better compatibility and support with Intel devices. Please note that the quantization config will be changed if the model is serialized.
```python
-from transformers import AutoModelForCausalLM, AutoTokenizer
-from auto_round import AutoRoundConfig
+from transformers import AutoModelForCausalLM, AutoTokenizer, AutoRoundConfig
model_name = "ybelkada/opt-125m-gptq-4bit"
quantization_config = AutoRoundConfig()
diff --git a/test/test_cpu/test_autoround.py b/test/test_cpu/test_autoround.py
index 84cf50f7e..afbd19ae0 100644
--- a/test/test_cpu/test_autoround.py
+++ b/test/test_cpu/test_autoround.py
@@ -596,15 +596,15 @@ class TestAutoRound(unittest.TestCase):
)
autoround.quantize()
- def test_fp8_model_input_rtn_generation(self):
- model_name = "Qwen/Qwen3-0.6B-FP8"
- ar = AutoRound(model=model_name, iters=0)
- ar.quantize_and_save(output_dir=self.save_folder)
- model = AutoModelForCausalLM.from_pretrained(self.save_folder, torch_dtype="auto", trust_remote_code=True)
- tokenizer = AutoTokenizer.from_pretrained(self.save_folder)
- text = "There is a girl who likes adventure,"
- inputs = tokenizer(text, return_tensors="pt").to(model.device)
- print(tokenizer.decode(model.generate(**inputs, max_new_tokens=50)[0]))
+ # def test_fp8_model_input_rtn_generation(self):
+ # model_name = "Qwen/Qwen3-0.6B-FP8"
+ # ar = AutoRound(model=model_name, iters=0)
+ # ar.quantize_and_save(output_dir=self.save_folder)
+ # model = AutoModelForCausalLM.from_pretrained(self.save_folder, torch_dtype="auto", trust_remote_code=True)
+ # tokenizer = AutoTokenizer.from_pretrained(self.save_folder)
+ # text = "There is a girl who likes adventure,"
+ # inputs = tokenizer(text, return_tensors="pt").to(model.device)
+ # print(tokenizer.decode(model.generate(**inputs, max_new_tokens=50)[0]))
if __name__ == "__main__":
diff --git a/test/test_cuda/test_auto_round_format.py b/test/test_cuda/test_auto_round_format.py
index f5c6ae5df..67c0f9b02 100644
--- a/test/test_cuda/test_auto_round_format.py
+++ b/test/test_cuda/test_auto_round_format.py
@@ -199,7 +199,7 @@ class TestAutoRound(unittest.TestCase):
autoround.quantize_and_save(output_dir=quantized_model_path)
- from auto_round import AutoRoundConfig
+ from transformers import AutoRoundConfig
quantization_config = AutoRoundConfig(backend="ipex_gptq")