Skip to content

PTQ-Level Efficiency, QAT-Level Accuracy | 训练后量化的效率,量化感知训练的精度

itrex requires torch<2.7 (#548)

Sun, Xuehao2e842a2fa7d7
diff --git a/.azure-pipelines/scripts/ut/run_ut.sh b/.azure-pipelines/scripts/ut/run_ut.sh
index 94f4291c0..91509b257 100644
--- a/.azure-pipelines/scripts/ut/run_ut.sh
+++ b/.azure-pipelines/scripts/ut/run_ut.sh
@@ -3,6 +3,8 @@ set -xe
# install requirements
echo "set up UT env..."
+export TQDM_MININTERVAL=60
+export TQDM_POSITION=-1
pip install pytest-cov pytest-html
pip install -r /auto-round/test/requirements.txt
pip list
diff --git a/.azure-pipelines/template/ut-template.yml b/.azure-pipelines/template/ut-template.yml
index 86dd4af1a..9a18eea29 100644
--- a/.azure-pipelines/template/ut-template.yml
+++ b/.azure-pipelines/template/ut-template.yml
@@ -35,8 +35,8 @@ steps:
- ${{ if eq(parameters.imageSource, 'build') }}:
- script: |
docker exec ${{ parameters.utContainerName }} bash -c "cd /auto-round \
- && pip install torch==2.6.0 --index-url https://download.pytorch.org/whl/cpu \
- && pip install intel-extension-for-pytorch==2.6.0 \
+ && pip install torch==2.7.0 --index-url https://download.pytorch.org/whl/cpu \
+ && pip install intel-extension-for-pytorch==2.7.0 \
&& pip install .[cpu] \
&& pip list"
displayName: "Env Setup"
diff --git a/.azure-pipelines/unit-test.yml b/.azure-pipelines/unit-test.yml
index e07303ee0..f46867ebe 100644
--- a/.azure-pipelines/unit-test.yml
+++ b/.azure-pipelines/unit-test.yml
@@ -14,6 +14,8 @@ pr:
- setup.py
- requirements.txt
- .azure-pipelines/scripts/ut
+ - .azure-pipelines/unit-test.yml
+ - .azure-pipelines/template/ut-template.yml
exclude:
- test/test*hpu*
- "*.md"
diff --git a/auto_round/inference/backend.py b/auto_round/inference/backend.py
index 727964003..7fd935d21 100644
--- a/auto_round/inference/backend.py
+++ b/auto_round/inference/backend.py
@@ -193,7 +193,7 @@ BackendInfos['qbits'] = BackendInfo(device=["cpu"], sym=[True, False],
alias=["itrex", "qbits"],
dtype=["float16", "bfloat16"],
convertable_format=["int32"],
- requirements=["intel-extension-for-transformers"])
+ requirements=["intel-extension-for-transformers", "torch<2.7.0"])
BackendInfos['qbits_zp'] = BackendInfo(device=["cpu"], sym=[True, False],
packing_format="qbits_zp",
@@ -203,7 +203,7 @@ BackendInfos['qbits_zp'] = BackendInfo(device=["cpu"], sym=[True, False],
feature_checks=[],
alias=["itrex", "qbits"],
convertable_format=["int32_zp"],
- requirements=["intel-extension-for-transformers"]
+ requirements=["intel-extension-for-transformers","torch<2.7.0"]
)
BackendInfos['auto_round:qbits_awq'] = BackendInfo(device=["cpu"], sym=[True, False], ## for awq, not robust
diff --git a/test/test_block_names.py b/test/test_block_names.py
index 8e86e62f2..7c618b685 100644
--- a/test/test_block_names.py
+++ b/test/test_block_names.py
@@ -147,7 +147,7 @@ class TestQuantizationBlocks(unittest.TestCase):
from auto_round.utils import get_block_names, validate_modules
llm_block_names = get_block_names(self.model)
validate_modules(llm_block_names)
- bits, group_size, sym, batch_size = 4, 128, False, 20
+ bits, group_size, sym, batch_size = 4, 128, True, 20
autoround = AutoRound(
self.model,
self.tokenizer,
diff --git a/test/test_conv1d.py b/test/test_conv1d.py
index 62b505b48..4f45fcbdb 100644
--- a/test/test_conv1d.py
+++ b/test/test_conv1d.py
@@ -33,7 +33,7 @@ class TestQuantizationConv1d(unittest.TestCase):
def test_quant(self):
self.model = AutoModelForCausalLM.from_pretrained(self.model_name, torch_dtype="auto", trust_remote_code=True)
- bits, group_size, sym = 4, 128, False
+ bits, group_size, sym = 4, 128, True
from auto_round import AutoRoundConfig
autoround = AutoRound(
self.model,
diff --git a/test/test_export.py b/test/test_export.py
index 3bde40491..55cecd37b 100644
--- a/test/test_export.py
+++ b/test/test_export.py
@@ -56,41 +56,8 @@ class TestAutoRound(unittest.TestCase):
print(tokenizer.decode(model.generate(**inputs, max_new_tokens=50)[0]))
shutil.rmtree("./saved", ignore_errors=True)
- # def test_autogptq_marlin_format(self):
- #
- # if not torch.cuda.is_available():
- # return
- # try:
- # import auto_gptq
- # except:
- # return
- # bits, group_size, sym = 4, 128, True
- # autoround = AutoRound(
- # self.model,
- # self.tokenizer,
- # bits=bits,
- # group_size=group_size,
- # sym=sym,
- # iters=2,
- # seqlen=2,
- # dataset=self.llm_dataloader,
- # )
- # autoround.quantize()
- # quantized_model_path = "./saved"
- #
- # autoround.save_quantized(output_dir=quantized_model_path, inplace=False, format="auto_gptq")
- #
- # from auto_gptq import AutoGPTQForCausalLM
- # model = AutoGPTQForCausalLM.from_quantized(quantized_model_path, device_map="auto", use_safetensors=True,
- # use_marlin=True)
- # tokenizer = AutoTokenizer.from_pretrained(quantized_model_path)
- # text = "There is a girl who likes adventure,"
- # inputs = tokenizer(text, return_tensors="pt").to(model.device)
- # print(tokenizer.decode(model.generate(**inputs, max_new_tokens=50)[0]))
- # shutil.rmtree("./saved", ignore_errors=True)
-
def test_autoround_format(self):
- bits, group_size, sym = 4, 128, False
+ bits, group_size, sym = 4, 128, True
autoround = AutoRound(
self.model,
self.tokenizer,
@@ -105,10 +72,7 @@ class TestAutoRound(unittest.TestCase):
quantized_model_path = "./saved"
autoround.save_quantized(output_dir=quantized_model_path, inplace=False, format="auto_round")
- try:
- import intel_extension_for_transformers
- except:
- return
+
device = "auto" ##cpu, hpu, cuda
from auto_round import AutoRoundConfig
diff --git a/test/test_generation.py b/test/test_generation.py
index 1fa849764..e2a140970 100644
--- a/test/test_generation.py
+++ b/test/test_generation.py
@@ -34,40 +34,6 @@ class TestAutoRoundFormatGeneration(unittest.TestCase):
shutil.rmtree(self.save_folder, ignore_errors=True)
shutil.rmtree("runs", ignore_errors=True)
-
- def test_4bits_asym(self):
- bits = 4
- group_size = 128
- sym = False
- autoround = AutoRound(
- self.model,
- self.tokenizer,
- bits=bits,
- group_size=group_size,
- sym=sym,
- iters=1,
- seqlen=2,
- dataset=self.llm_dataloader,
- )
- quantized_model_path = self.save_folder
-
- autoround.quantize_and_save(output_dir=quantized_model_path, format="auto_round", inplace=False)
-
- from auto_round import AutoRoundConfig
- quantization_config = AutoRoundConfig(
- backend="auto"
- )
- model = AutoModelForCausalLM.from_pretrained(quantized_model_path,
- device_map="cpu", quantization_config=quantization_config)
- tokenizer = AutoTokenizer.from_pretrained(quantized_model_path)
- text = "My name is "
- inputs = tokenizer(text, return_tensors="pt").to(model.device)
- res = tokenizer.decode(model.generate(**inputs, max_new_tokens=50)[0])
- print(res)
- assert ("!!!" not in res)
-
- shutil.rmtree(self.save_folder, ignore_errors=True)
-
def test_4bits_sym(self):
bits = 4
group_size = 128
@@ -109,62 +75,8 @@ class TestAutoRoundFormatGeneration(unittest.TestCase):
print(res)
assert ("!!!" not in res)
-
- from auto_round import AutoRoundConfig
- quantization_config = AutoRoundConfig(
- backend="itrex"
- )
- model = AutoModelForCausalLM.from_pretrained(quantized_model_path,
- device_map="cpu", quantization_config=quantization_config)
- tokenizer = AutoTokenizer.from_pretrained(quantized_model_path)
- text = "My name is "
- inputs = tokenizer(text, return_tensors="pt").to(model.device)
- res = tokenizer.decode(model.generate(**inputs, max_new_tokens=50)[0])
- print(res)
- assert ("!!!" not in res)
-
- model = AutoModelForCausalLM.from_pretrained(quantized_model_path,
- device_map="cpu", quantization_config=quantization_config,
- torch_dtype=torch.float16)
- tokenizer = AutoTokenizer.from_pretrained(quantized_model_path)
- text = "There is a girl who likes adventure,"
- inputs = tokenizer(text, return_tensors="pt").to(model.device)
- res = tokenizer.decode(model.generate(**inputs, max_new_tokens=50)[0])
- print(res)
- assert ("!!!" not in res)
- shutil.rmtree(self.save_folder, ignore_errors=True)
-
- def test_autoround_asym(self):
- for bits in [2, 4, 8]:
- model = AutoModelForCausalLM.from_pretrained(self.model_name, torch_dtype="auto", trust_remote_code=True)
- tokenizer = AutoTokenizer.from_pretrained(self.model_name, trust_remote_code=True)
- bits, group_size, sym = bits, 128, False
- autoround = AutoRound(
- model,
- tokenizer,
- bits=bits,
- group_size=group_size,
- sym=sym,
- iters=2,
- seqlen=2,
- dataset=self.llm_dataloader,
- )
- quantized_model_path = "./saved"
-
- autoround.quantize_and_save(output_dir=quantized_model_path, format="auto_round")
-
- model = AutoModelForCausalLM.from_pretrained(quantized_model_path, device_map="auto",
- trust_remote_code=True)
- tokenizer = AutoTokenizer.from_pretrained(quantized_model_path)
- text = "There is a girl who likes adventure,"
- inputs = tokenizer(text, return_tensors="pt").to(model.device)
- res = tokenizer.decode(model.generate(**inputs, max_new_tokens=50)[0])
- print(res)
- assert ("!!!" not in res)
- shutil.rmtree(self.save_folder, ignore_errors=True)
-
def test_autoround_sym(self):
- for bits in [2, 4, 8]:
+ for bits in [4]:
model = AutoModelForCausalLM.from_pretrained(self.model_name, torch_dtype="auto", trust_remote_code=True)
tokenizer = AutoTokenizer.from_pretrained(self.model_name, trust_remote_code=True)
bits, group_size, sym = bits, 128, True
@@ -192,38 +104,3 @@ class TestAutoRoundFormatGeneration(unittest.TestCase):
assert ("!!!" not in res)
shutil.rmtree(self.save_folder, ignore_errors=True)
-
- def test_mixed_precision(self):
- model = AutoModelForCausalLM.from_pretrained(self.model_name, torch_dtype="auto", trust_remote_code=True)
- tokenizer = AutoTokenizer.from_pretrained(self.model_name, trust_remote_code=True)
- layer_config = {}
-
- layer_config["model.decoder.layers.0.self_attn.k_proj"] = {"bits": 8}
- layer_config["model.decoder.layers.6.self_attn.out_proj"] = {"bits": 2, "group_size": 32}
- bits, group_size, sym = 4, 128, True
- autoround = AutoRound(
- model,
- tokenizer,
- bits=bits,
- group_size=group_size,
- iters=1,
- nsamples=1,
- sym=sym,
- layer_config=layer_config
- )
- quantized_model_path = self.save_folder
- autoround.quantize_and_save(output_dir=quantized_model_path, format="auto_round")
-
- model = AutoModelForCausalLM.from_pretrained(
- self.save_folder,
- torch_dtype=torch.float16,
- device_map="cpu",
- )
- tokenizer = AutoTokenizer.from_pretrained(self.save_folder)
- text = "There is a girl who likes adventure,"
- inputs = tokenizer(text, return_tensors="pt").to(model.device)
- res = tokenizer.decode(model.generate(**inputs, max_new_tokens=50)[0])
- print(res)
- assert ("!!!" not in res)
- shutil.rmtree(self.save_folder, ignore_errors=True)
-
diff --git a/test/test_load_awq_gptq.py b/test/test_load_awq_gptq.py
index 71e7e9a01..6db06c03a 100644
--- a/test/test_load_awq_gptq.py
+++ b/test/test_load_awq_gptq.py
@@ -41,44 +41,13 @@ class TestAutoRound(unittest.TestCase):
def tearDownClass(self):
shutil.rmtree("runs", ignore_errors=True)
- def test_load_gptq_model_4bits(self):
- model_name = "ybelkada/opt-125m-gptq-4bit"
- quantization_config = AutoRoundConfig()
- model = AutoModelForCausalLM.from_pretrained(model_name, torch_dtype="auto", trust_remote_code=True,
- device_map="cpu",
- quantization_config=quantization_config)
- tokenizer = AutoTokenizer.from_pretrained(model_name, trust_remote_code=True)
- self.model_infer(model, tokenizer)
-
- def test_load_gptq_model_8bits(self):
- model_name = "acloudfan/opt-125m-gptq-8bit"
- quantization_config = AutoRoundConfig()
- model = AutoModelForCausalLM.from_pretrained(model_name, torch_dtype="auto", trust_remote_code=True,
- device_map="cpu",
- quantization_config=quantization_config)
- tokenizer = AutoTokenizer.from_pretrained(model_name, trust_remote_code=True)
- self.model_infer(model, tokenizer)
-
-
- def test_load_gptq_model_2bits(self):
- model_name = "LucasSantiago257/gemma-2b-2bits-gptq"
- quantization_config = AutoRoundConfig()
- model = AutoModelForCausalLM.from_pretrained(model_name, torch_dtype="auto", trust_remote_code=True,
- device_map="cpu",
- quantization_config=quantization_config)
- tokenizer = AutoTokenizer.from_pretrained(model_name, trust_remote_code=True)
- self.model_infer(model, tokenizer)
-
-
-
-
def test_load_gptq_no_dummy_gidx_model(self):
model_name = "ModelCloud/Llama-3.2-1B-Instruct-gptqmodel-4bit-vortex-v1"
quantization_config = AutoRoundConfig()
with self.assertRaises(NotImplementedError) as cm:
model = AutoModelForCausalLM.from_pretrained(model_name, torch_dtype="auto", trust_remote_code=True,
- device_map="cpu",
- quantization_config=quantization_config)
+ device_map="cpu",
+ quantization_config=quantization_config)
def test_load_awq(self):
model_name = "casperhansen/opt-125m-awq"
@@ -88,6 +57,3 @@ class TestAutoRound(unittest.TestCase):
quantization_config=quantization_config)
tokenizer = AutoTokenizer.from_pretrained(model_name, trust_remote_code=True)
self.model_infer(model, tokenizer)
-
-
-
diff --git a/test_cuda/test_auto_round_format.py b/test_cuda/test_auto_round_format.py
index 37f8c2a20..9cc2b0a0c 100644
--- a/test_cuda/test_auto_round_format.py
+++ b/test_cuda/test_auto_round_format.py
@@ -75,6 +75,35 @@ class TestAutoRound(unittest.TestCase):
shutil.rmtree(self.save_folder, ignore_errors=True)
shutil.rmtree("runs", ignore_errors=True)
+ def test_autoround_asym(self):
+ for bits in [2, 4, 8]:
+ model = AutoModelForCausalLM.from_pretrained(self.model_name, torch_dtype="auto", trust_remote_code=True)
+ tokenizer = AutoTokenizer.from_pretrained(self.model_name, trust_remote_code=True)
+ bits, group_size, sym = bits, 128, False
+ autoround = AutoRound(
+ model,
+ tokenizer,
+ bits=bits,
+ group_size=group_size,
+ sym=sym,
+ iters=2,
+ seqlen=2,
+ dataset=self.llm_dataloader,
+ )
+ quantized_model_path = self.save_folder
+
+ autoround.quantize_and_save(output_dir=quantized_model_path, format="auto_round")
+
+ model = AutoModelForCausalLM.from_pretrained(quantized_model_path, device_map="auto",
+ trust_remote_code=True)
+ tokenizer = AutoTokenizer.from_pretrained(quantized_model_path)
+ text = "There is a girl who likes adventure,"
+ inputs = tokenizer(text, return_tensors="pt").to(model.device)
+ res = tokenizer.decode(model.generate(**inputs, max_new_tokens=50)[0])
+ print(res)
+ assert ("!!!" not in res)
+ shutil.rmtree(self.save_folder, ignore_errors=True)
+
def test_mixed_precision(self):
model = AutoModelForCausalLM.from_pretrained(self.model_name, torch_dtype="auto", trust_remote_code=True)
tokenizer = AutoTokenizer.from_pretrained(self.model_name, trust_remote_code=True)
@@ -109,7 +138,6 @@ class TestAutoRound(unittest.TestCase):
print(result['results']['lambada_openai']['acc,none'])
self.assertGreater(result['results']['lambada_openai']['acc,none'], 0.32)
-
def test_awq_backend(self):
model = AutoModelForCausalLM.from_pretrained(self.model_name, torch_dtype="auto", trust_remote_code=True)
tokenizer = AutoTokenizer.from_pretrained(self.model_name, trust_remote_code=True)
@@ -292,7 +320,7 @@ class TestAutoRound(unittest.TestCase):
self.model_infer(model, tokenizer)
def test_autoround_asym(self):
- for bits in [2,3,4,8]:
+ for bits in [2, 3, 4, 8]:
model = AutoModelForCausalLM.from_pretrained(self.model_name, torch_dtype="auto", trust_remote_code=True)
tokenizer = AutoTokenizer.from_pretrained(self.model_name, trust_remote_code=True)
bits, group_size, sym = bits, 128, False
diff --git a/test_cuda/test_conv1d.py b/test_cuda/test_conv1d.py
index a407a4aef..941e13240 100644
--- a/test_cuda/test_conv1d.py
+++ b/test_cuda/test_conv1d.py
@@ -33,7 +33,7 @@ class TestQuantizationConv1d(unittest.TestCase):
def test_quant(self):
self.model = AutoModelForCausalLM.from_pretrained(self.model_name, torch_dtype="auto", trust_remote_code=True)
- bits, group_size, sym = 4, 128, False
+ bits, group_size, sym = 4, 128, True
from auto_round import AutoRoundConfig
autoround = AutoRound(
self.model,
diff --git a/test_cuda/test_export.py b/test_cuda/test_export.py
index 74c0cafab..db68fa0dd 100644
--- a/test_cuda/test_export.py
+++ b/test_cuda/test_export.py
@@ -145,7 +145,7 @@ class TestAutoRound(unittest.TestCase):
def test_autoround_format(self):
model = AutoModelForCausalLM.from_pretrained(self.model_name, torch_dtype="auto", trust_remote_code=True)
tokenizer = AutoTokenizer.from_pretrained(self.model_name, trust_remote_code=True)
- bits, group_size, sym = 4, 128, False
+ bits, group_size, sym = 4, 128, True
autoround = AutoRound(
model,
tokenizer,
diff --git a/test_cuda/test_qbits.py b/test_cuda/test_qbits.py
new file mode 100644
index 000000000..3f1f1d3fe
--- /dev/null
+++ b/test_cuda/test_qbits.py
@@ -0,0 +1,130 @@
+import shutil
+import sys
+import unittest
+
+sys.path.insert(0, "..")
+
+from transformers import AutoModelForCausalLM, AutoTokenizer
+
+from auto_round import AutoRoundConfig, AutoRound
+
+
+class TestAutoRound(unittest.TestCase):
+ def model_infer(self, model, tokenizer):
+ prompts = [
+ "Hello,my name is",
+ # "The president of the United States is",
+ # "The capital of France is",
+ # "The future of AI is",
+ ]
+
+ inputs = tokenizer(prompts, return_tensors="pt", padding=False, truncation=True)
+
+ outputs = model.generate(
+ input_ids=inputs["input_ids"].to(model.device),
+ attention_mask=inputs["attention_mask"].to(model.device),
+ do_sample=False, ## change this to follow official usage
+ max_new_tokens=5
+ )
+ generated_ids = [
+ output_ids[len(input_ids):] for input_ids, output_ids in zip(inputs["input_ids"], outputs)
+ ]
+
+ decoded_outputs = tokenizer.batch_decode(generated_ids, skip_special_tokens=True)
+
+ for i, prompt in enumerate(prompts):
+ print(f"Prompt: {prompt}")
+ print(f"Generated: {decoded_outputs[i]}")
+ print("-" * 50)
+
+ @classmethod
+ def tearDownClass(self):
+ shutil.rmtree("runs", ignore_errors=True)
+
+
+ ## require torch 2.6
+ def test_load_gptq_model_8bits(self):
+ model_name = "acloudfan/opt-125m-gptq-8bit"
+ quantization_config = AutoRoundConfig()
+ model = AutoModelForCausalLM.from_pretrained(model_name, torch_dtype="auto", trust_remote_code=True,
+ device_map="cpu",
+ quantization_config=quantization_config)
+ tokenizer = AutoTokenizer.from_pretrained(model_name, trust_remote_code=True)
+ self.model_infer(model, tokenizer)
+
+
+ def test_load_gptq_model_2bits(self):
+ model_name = "LucasSantiago257/gemma-2b-2bits-gptq"
+ quantization_config = AutoRoundConfig()
+ model = AutoModelForCausalLM.from_pretrained(model_name, torch_dtype="auto", trust_remote_code=True,
+ device_map="cpu",
+ quantization_config=quantization_config)
+ tokenizer = AutoTokenizer.from_pretrained(model_name, trust_remote_code=True)
+ self.model_infer(model, tokenizer)
+
+
+ def test_mixed_precision(self):
+ model = AutoModelForCausalLM.from_pretrained(self.model_name, torch_dtype="auto", trust_remote_code=True)
+ tokenizer = AutoTokenizer.from_pretrained(self.model_name, trust_remote_code=True)
+ layer_config = {}
+
+ layer_config["model.decoder.layers.0.self_attn.k_proj"] = {"bits": 8}
+ layer_config["model.decoder.layers.6.self_attn.out_proj"] = {"bits": 2, "group_size": 32}
+ bits, group_size, sym = 4, 128, True
+ from auto_round import AutoRound
+ import torch
+ autoround = AutoRound(
+ model,
+ tokenizer,
+ bits=bits,
+ group_size=group_size,
+ iters=1,
+ nsamples=1,
+ sym=sym,
+ layer_config=layer_config
+ )
+ quantized_model_path = self.save_folder
+ autoround.quantize_and_save(output_dir=quantized_model_path, format="auto_round")
+
+ model = AutoModelForCausalLM.from_pretrained(
+ self.save_folder,
+ torch_dtype=torch.float16,
+ device_map="cpu",
+ )
+ tokenizer = AutoTokenizer.from_pretrained(self.save_folder)
+ text = "There is a girl who likes adventure,"
+ inputs = tokenizer(text, return_tensors="pt").to(model.device)
+ res = tokenizer.decode(model.generate(**inputs, max_new_tokens=50)[0])
+ print(res)
+ assert ("!!!" not in res)
+ shutil.rmtree(self.save_folder, ignore_errors=True)
+
+ def test_autoround_sym(self):
+ for bits in [4]:
+ model = AutoModelForCausalLM.from_pretrained(self.model_name, torch_dtype="auto", trust_remote_code=True)
+ tokenizer = AutoTokenizer.from_pretrained(self.model_name, trust_remote_code=True)
+ bits, group_size, sym = bits, 128, True
+ autoround = AutoRound(
+ model,
+ tokenizer,
+ bits=bits,
+ group_size=group_size,
+ sym=sym,
+ iters=2,
+ seqlen=2
+ )
+ quantized_model_path = "./saved"
+
+ autoround.quantize_and_save(output_dir=quantized_model_path, format="auto_round")
+
+ model = AutoModelForCausalLM.from_pretrained(quantized_model_path, device_map="auto",
+ trust_remote_code=True)
+ tokenizer = AutoTokenizer.from_pretrained(quantized_model_path)
+ text = "There is a girl who likes adventure,"
+ inputs = tokenizer(text, return_tensors="pt").to(model.device)
+ res = tokenizer.decode(model.generate(**inputs, max_new_tokens=50)[0])
+ print(res)
+ assert ("!!!" not in res)
+ shutil.rmtree(self.save_folder, ignore_errors=True)
+
+