PTQ-Level Efficiency, QAT-Level Accuracy | 训练后量化的效率,量化感知训练的精度
itrex requires torch<2.7 (#548)
Sun, Xuehao2e842a2fa7d7
diff --git a/.azure-pipelines/scripts/ut/run_ut.sh b/.azure-pipelines/scripts/ut/run_ut.shindex 94f4291c0..91509b257 100644--- a/.azure-pipelines/scripts/ut/run_ut.sh+++ b/.azure-pipelines/scripts/ut/run_ut.sh@@ -3,6 +3,8 @@ set -xe# install requirementsecho "set up UT env..."+export TQDM_MININTERVAL=60+export TQDM_POSITION=-1pip install pytest-cov pytest-htmlpip install -r /auto-round/test/requirements.txtpip listdiff --git a/.azure-pipelines/template/ut-template.yml b/.azure-pipelines/template/ut-template.ymlindex 86dd4af1a..9a18eea29 100644--- a/.azure-pipelines/template/ut-template.yml+++ b/.azure-pipelines/template/ut-template.yml@@ -35,8 +35,8 @@ steps:- ${{ if eq(parameters.imageSource, 'build') }}:- script: |docker exec ${{ parameters.utContainerName }} bash -c "cd /auto-round \- && pip install torch==2.6.0 --index-url https://download.pytorch.org/whl/cpu \- && pip install intel-extension-for-pytorch==2.6.0 \+ && pip install torch==2.7.0 --index-url https://download.pytorch.org/whl/cpu \+ && pip install intel-extension-for-pytorch==2.7.0 \&& pip install .[cpu] \&& pip list"displayName: "Env Setup"diff --git a/.azure-pipelines/unit-test.yml b/.azure-pipelines/unit-test.ymlindex e07303ee0..f46867ebe 100644--- a/.azure-pipelines/unit-test.yml+++ b/.azure-pipelines/unit-test.yml@@ -14,6 +14,8 @@ pr:- setup.py- requirements.txt- .azure-pipelines/scripts/ut+ - .azure-pipelines/unit-test.yml+ - .azure-pipelines/template/ut-template.ymlexclude:- test/test*hpu*- "*.md"diff --git a/auto_round/inference/backend.py b/auto_round/inference/backend.pyindex 727964003..7fd935d21 100644--- a/auto_round/inference/backend.py+++ b/auto_round/inference/backend.py@@ -193,7 +193,7 @@ BackendInfos['qbits'] = BackendInfo(device=["cpu"], sym=[True, False],alias=["itrex", "qbits"],dtype=["float16", "bfloat16"],convertable_format=["int32"],- requirements=["intel-extension-for-transformers"])+ requirements=["intel-extension-for-transformers", "torch<2.7.0"])BackendInfos['qbits_zp'] = BackendInfo(device=["cpu"], sym=[True, False],packing_format="qbits_zp",@@ -203,7 +203,7 @@ BackendInfos['qbits_zp'] = BackendInfo(device=["cpu"], sym=[True, False],feature_checks=[],alias=["itrex", "qbits"],convertable_format=["int32_zp"],- requirements=["intel-extension-for-transformers"]+ requirements=["intel-extension-for-transformers","torch<2.7.0"])BackendInfos['auto_round:qbits_awq'] = BackendInfo(device=["cpu"], sym=[True, False], ## for awq, not robustdiff --git a/test/test_block_names.py b/test/test_block_names.pyindex 8e86e62f2..7c618b685 100644--- a/test/test_block_names.py+++ b/test/test_block_names.py@@ -147,7 +147,7 @@ class TestQuantizationBlocks(unittest.TestCase):from auto_round.utils import get_block_names, validate_modulesllm_block_names = get_block_names(self.model)validate_modules(llm_block_names)- bits, group_size, sym, batch_size = 4, 128, False, 20+ bits, group_size, sym, batch_size = 4, 128, True, 20autoround = AutoRound(self.model,self.tokenizer,diff --git a/test/test_conv1d.py b/test/test_conv1d.pyindex 62b505b48..4f45fcbdb 100644--- a/test/test_conv1d.py+++ b/test/test_conv1d.py@@ -33,7 +33,7 @@ class TestQuantizationConv1d(unittest.TestCase):def test_quant(self):self.model = AutoModelForCausalLM.from_pretrained(self.model_name, torch_dtype="auto", trust_remote_code=True)- bits, group_size, sym = 4, 128, False+ bits, group_size, sym = 4, 128, Truefrom auto_round import AutoRoundConfigautoround = AutoRound(self.model,diff --git a/test/test_export.py b/test/test_export.pyindex 3bde40491..55cecd37b 100644--- a/test/test_export.py+++ b/test/test_export.py@@ -56,41 +56,8 @@ class TestAutoRound(unittest.TestCase):print(tokenizer.decode(model.generate(**inputs, max_new_tokens=50)[0]))shutil.rmtree("./saved", ignore_errors=True)- # def test_autogptq_marlin_format(self):- #- # if not torch.cuda.is_available():- # return- # try:- # import auto_gptq- # except:- # return- # bits, group_size, sym = 4, 128, True- # autoround = AutoRound(- # self.model,- # self.tokenizer,- # bits=bits,- # group_size=group_size,- # sym=sym,- # iters=2,- # seqlen=2,- # dataset=self.llm_dataloader,- # )- # autoround.quantize()- # quantized_model_path = "./saved"- #- # autoround.save_quantized(output_dir=quantized_model_path, inplace=False, format="auto_gptq")- #- # from auto_gptq import AutoGPTQForCausalLM- # model = AutoGPTQForCausalLM.from_quantized(quantized_model_path, device_map="auto", use_safetensors=True,- # use_marlin=True)- # tokenizer = AutoTokenizer.from_pretrained(quantized_model_path)- # text = "There is a girl who likes adventure,"- # inputs = tokenizer(text, return_tensors="pt").to(model.device)- # print(tokenizer.decode(model.generate(**inputs, max_new_tokens=50)[0]))- # shutil.rmtree("./saved", ignore_errors=True)-def test_autoround_format(self):- bits, group_size, sym = 4, 128, False+ bits, group_size, sym = 4, 128, Trueautoround = AutoRound(self.model,self.tokenizer,@@ -105,10 +72,7 @@ class TestAutoRound(unittest.TestCase):quantized_model_path = "./saved"autoround.save_quantized(output_dir=quantized_model_path, inplace=False, format="auto_round")- try:- import intel_extension_for_transformers- except:- return+device = "auto" ##cpu, hpu, cudafrom auto_round import AutoRoundConfigdiff --git a/test/test_generation.py b/test/test_generation.pyindex 1fa849764..e2a140970 100644--- a/test/test_generation.py+++ b/test/test_generation.py@@ -34,40 +34,6 @@ class TestAutoRoundFormatGeneration(unittest.TestCase):shutil.rmtree(self.save_folder, ignore_errors=True)shutil.rmtree("runs", ignore_errors=True)-- def test_4bits_asym(self):- bits = 4- group_size = 128- sym = False- autoround = AutoRound(- self.model,- self.tokenizer,- bits=bits,- group_size=group_size,- sym=sym,- iters=1,- seqlen=2,- dataset=self.llm_dataloader,- )- quantized_model_path = self.save_folder-- autoround.quantize_and_save(output_dir=quantized_model_path, format="auto_round", inplace=False)-- from auto_round import AutoRoundConfig- quantization_config = AutoRoundConfig(- backend="auto"- )- model = AutoModelForCausalLM.from_pretrained(quantized_model_path,- device_map="cpu", quantization_config=quantization_config)- tokenizer = AutoTokenizer.from_pretrained(quantized_model_path)- text = "My name is "- inputs = tokenizer(text, return_tensors="pt").to(model.device)- res = tokenizer.decode(model.generate(**inputs, max_new_tokens=50)[0])- print(res)- assert ("!!!" not in res)-- shutil.rmtree(self.save_folder, ignore_errors=True)-def test_4bits_sym(self):bits = 4group_size = 128@@ -109,62 +75,8 @@ class TestAutoRoundFormatGeneration(unittest.TestCase):print(res)assert ("!!!" not in res)-- from auto_round import AutoRoundConfig- quantization_config = AutoRoundConfig(- backend="itrex"- )- model = AutoModelForCausalLM.from_pretrained(quantized_model_path,- device_map="cpu", quantization_config=quantization_config)- tokenizer = AutoTokenizer.from_pretrained(quantized_model_path)- text = "My name is "- inputs = tokenizer(text, return_tensors="pt").to(model.device)- res = tokenizer.decode(model.generate(**inputs, max_new_tokens=50)[0])- print(res)- assert ("!!!" not in res)-- model = AutoModelForCausalLM.from_pretrained(quantized_model_path,- device_map="cpu", quantization_config=quantization_config,- torch_dtype=torch.float16)- tokenizer = AutoTokenizer.from_pretrained(quantized_model_path)- text = "There is a girl who likes adventure,"- inputs = tokenizer(text, return_tensors="pt").to(model.device)- res = tokenizer.decode(model.generate(**inputs, max_new_tokens=50)[0])- print(res)- assert ("!!!" not in res)- shutil.rmtree(self.save_folder, ignore_errors=True)-- def test_autoround_asym(self):- for bits in [2, 4, 8]:- model = AutoModelForCausalLM.from_pretrained(self.model_name, torch_dtype="auto", trust_remote_code=True)- tokenizer = AutoTokenizer.from_pretrained(self.model_name, trust_remote_code=True)- bits, group_size, sym = bits, 128, False- autoround = AutoRound(- model,- tokenizer,- bits=bits,- group_size=group_size,- sym=sym,- iters=2,- seqlen=2,- dataset=self.llm_dataloader,- )- quantized_model_path = "./saved"-- autoround.quantize_and_save(output_dir=quantized_model_path, format="auto_round")-- model = AutoModelForCausalLM.from_pretrained(quantized_model_path, device_map="auto",- trust_remote_code=True)- tokenizer = AutoTokenizer.from_pretrained(quantized_model_path)- text = "There is a girl who likes adventure,"- inputs = tokenizer(text, return_tensors="pt").to(model.device)- res = tokenizer.decode(model.generate(**inputs, max_new_tokens=50)[0])- print(res)- assert ("!!!" not in res)- shutil.rmtree(self.save_folder, ignore_errors=True)-def test_autoround_sym(self):- for bits in [2, 4, 8]:+ for bits in [4]:model = AutoModelForCausalLM.from_pretrained(self.model_name, torch_dtype="auto", trust_remote_code=True)tokenizer = AutoTokenizer.from_pretrained(self.model_name, trust_remote_code=True)bits, group_size, sym = bits, 128, True@@ -192,38 +104,3 @@ class TestAutoRoundFormatGeneration(unittest.TestCase):assert ("!!!" not in res)shutil.rmtree(self.save_folder, ignore_errors=True)-- def test_mixed_precision(self):- model = AutoModelForCausalLM.from_pretrained(self.model_name, torch_dtype="auto", trust_remote_code=True)- tokenizer = AutoTokenizer.from_pretrained(self.model_name, trust_remote_code=True)- layer_config = {}-- layer_config["model.decoder.layers.0.self_attn.k_proj"] = {"bits": 8}- layer_config["model.decoder.layers.6.self_attn.out_proj"] = {"bits": 2, "group_size": 32}- bits, group_size, sym = 4, 128, True- autoround = AutoRound(- model,- tokenizer,- bits=bits,- group_size=group_size,- iters=1,- nsamples=1,- sym=sym,- layer_config=layer_config- )- quantized_model_path = self.save_folder- autoround.quantize_and_save(output_dir=quantized_model_path, format="auto_round")-- model = AutoModelForCausalLM.from_pretrained(- self.save_folder,- torch_dtype=torch.float16,- device_map="cpu",- )- tokenizer = AutoTokenizer.from_pretrained(self.save_folder)- text = "There is a girl who likes adventure,"- inputs = tokenizer(text, return_tensors="pt").to(model.device)- res = tokenizer.decode(model.generate(**inputs, max_new_tokens=50)[0])- print(res)- assert ("!!!" not in res)- shutil.rmtree(self.save_folder, ignore_errors=True)-diff --git a/test/test_load_awq_gptq.py b/test/test_load_awq_gptq.pyindex 71e7e9a01..6db06c03a 100644--- a/test/test_load_awq_gptq.py+++ b/test/test_load_awq_gptq.py@@ -41,44 +41,13 @@ class TestAutoRound(unittest.TestCase):def tearDownClass(self):shutil.rmtree("runs", ignore_errors=True)- def test_load_gptq_model_4bits(self):- model_name = "ybelkada/opt-125m-gptq-4bit"- quantization_config = AutoRoundConfig()- model = AutoModelForCausalLM.from_pretrained(model_name, torch_dtype="auto", trust_remote_code=True,- device_map="cpu",- quantization_config=quantization_config)- tokenizer = AutoTokenizer.from_pretrained(model_name, trust_remote_code=True)- self.model_infer(model, tokenizer)-- def test_load_gptq_model_8bits(self):- model_name = "acloudfan/opt-125m-gptq-8bit"- quantization_config = AutoRoundConfig()- model = AutoModelForCausalLM.from_pretrained(model_name, torch_dtype="auto", trust_remote_code=True,- device_map="cpu",- quantization_config=quantization_config)- tokenizer = AutoTokenizer.from_pretrained(model_name, trust_remote_code=True)- self.model_infer(model, tokenizer)--- def test_load_gptq_model_2bits(self):- model_name = "LucasSantiago257/gemma-2b-2bits-gptq"- quantization_config = AutoRoundConfig()- model = AutoModelForCausalLM.from_pretrained(model_name, torch_dtype="auto", trust_remote_code=True,- device_map="cpu",- quantization_config=quantization_config)- tokenizer = AutoTokenizer.from_pretrained(model_name, trust_remote_code=True)- self.model_infer(model, tokenizer)----def test_load_gptq_no_dummy_gidx_model(self):model_name = "ModelCloud/Llama-3.2-1B-Instruct-gptqmodel-4bit-vortex-v1"quantization_config = AutoRoundConfig()with self.assertRaises(NotImplementedError) as cm:model = AutoModelForCausalLM.from_pretrained(model_name, torch_dtype="auto", trust_remote_code=True,- device_map="cpu",- quantization_config=quantization_config)+ device_map="cpu",+ quantization_config=quantization_config)def test_load_awq(self):model_name = "casperhansen/opt-125m-awq"@@ -88,6 +57,3 @@ class TestAutoRound(unittest.TestCase):quantization_config=quantization_config)tokenizer = AutoTokenizer.from_pretrained(model_name, trust_remote_code=True)self.model_infer(model, tokenizer)---diff --git a/test_cuda/test_auto_round_format.py b/test_cuda/test_auto_round_format.pyindex 37f8c2a20..9cc2b0a0c 100644--- a/test_cuda/test_auto_round_format.py+++ b/test_cuda/test_auto_round_format.py@@ -75,6 +75,35 @@ class TestAutoRound(unittest.TestCase):shutil.rmtree(self.save_folder, ignore_errors=True)shutil.rmtree("runs", ignore_errors=True)+ def test_autoround_asym(self):+ for bits in [2, 4, 8]:+ model = AutoModelForCausalLM.from_pretrained(self.model_name, torch_dtype="auto", trust_remote_code=True)+ tokenizer = AutoTokenizer.from_pretrained(self.model_name, trust_remote_code=True)+ bits, group_size, sym = bits, 128, False+ autoround = AutoRound(+ model,+ tokenizer,+ bits=bits,+ group_size=group_size,+ sym=sym,+ iters=2,+ seqlen=2,+ dataset=self.llm_dataloader,+ )+ quantized_model_path = self.save_folder++ autoround.quantize_and_save(output_dir=quantized_model_path, format="auto_round")++ model = AutoModelForCausalLM.from_pretrained(quantized_model_path, device_map="auto",+ trust_remote_code=True)+ tokenizer = AutoTokenizer.from_pretrained(quantized_model_path)+ text = "There is a girl who likes adventure,"+ inputs = tokenizer(text, return_tensors="pt").to(model.device)+ res = tokenizer.decode(model.generate(**inputs, max_new_tokens=50)[0])+ print(res)+ assert ("!!!" not in res)+ shutil.rmtree(self.save_folder, ignore_errors=True)+def test_mixed_precision(self):model = AutoModelForCausalLM.from_pretrained(self.model_name, torch_dtype="auto", trust_remote_code=True)tokenizer = AutoTokenizer.from_pretrained(self.model_name, trust_remote_code=True)@@ -109,7 +138,6 @@ class TestAutoRound(unittest.TestCase):print(result['results']['lambada_openai']['acc,none'])self.assertGreater(result['results']['lambada_openai']['acc,none'], 0.32)-def test_awq_backend(self):model = AutoModelForCausalLM.from_pretrained(self.model_name, torch_dtype="auto", trust_remote_code=True)tokenizer = AutoTokenizer.from_pretrained(self.model_name, trust_remote_code=True)@@ -292,7 +320,7 @@ class TestAutoRound(unittest.TestCase):self.model_infer(model, tokenizer)def test_autoround_asym(self):- for bits in [2,3,4,8]:+ for bits in [2, 3, 4, 8]:model = AutoModelForCausalLM.from_pretrained(self.model_name, torch_dtype="auto", trust_remote_code=True)tokenizer = AutoTokenizer.from_pretrained(self.model_name, trust_remote_code=True)bits, group_size, sym = bits, 128, Falsediff --git a/test_cuda/test_conv1d.py b/test_cuda/test_conv1d.pyindex a407a4aef..941e13240 100644--- a/test_cuda/test_conv1d.py+++ b/test_cuda/test_conv1d.py@@ -33,7 +33,7 @@ class TestQuantizationConv1d(unittest.TestCase):def test_quant(self):self.model = AutoModelForCausalLM.from_pretrained(self.model_name, torch_dtype="auto", trust_remote_code=True)- bits, group_size, sym = 4, 128, False+ bits, group_size, sym = 4, 128, Truefrom auto_round import AutoRoundConfigautoround = AutoRound(self.model,diff --git a/test_cuda/test_export.py b/test_cuda/test_export.pyindex 74c0cafab..db68fa0dd 100644--- a/test_cuda/test_export.py+++ b/test_cuda/test_export.py@@ -145,7 +145,7 @@ class TestAutoRound(unittest.TestCase):def test_autoround_format(self):model = AutoModelForCausalLM.from_pretrained(self.model_name, torch_dtype="auto", trust_remote_code=True)tokenizer = AutoTokenizer.from_pretrained(self.model_name, trust_remote_code=True)- bits, group_size, sym = 4, 128, False+ bits, group_size, sym = 4, 128, Trueautoround = AutoRound(model,tokenizer,diff --git a/test_cuda/test_qbits.py b/test_cuda/test_qbits.pynew file mode 100644index 000000000..3f1f1d3fe--- /dev/null+++ b/test_cuda/test_qbits.py@@ -0,0 +1,130 @@+import shutil+import sys+import unittest++sys.path.insert(0, "..")++from transformers import AutoModelForCausalLM, AutoTokenizer++from auto_round import AutoRoundConfig, AutoRound+++class TestAutoRound(unittest.TestCase):+ def model_infer(self, model, tokenizer):+ prompts = [+ "Hello,my name is",+ # "The president of the United States is",+ # "The capital of France is",+ # "The future of AI is",+ ]++ inputs = tokenizer(prompts, return_tensors="pt", padding=False, truncation=True)++ outputs = model.generate(+ input_ids=inputs["input_ids"].to(model.device),+ attention_mask=inputs["attention_mask"].to(model.device),+ do_sample=False, ## change this to follow official usage+ max_new_tokens=5+ )+ generated_ids = [+ output_ids[len(input_ids):] for input_ids, output_ids in zip(inputs["input_ids"], outputs)+ ]++ decoded_outputs = tokenizer.batch_decode(generated_ids, skip_special_tokens=True)++ for i, prompt in enumerate(prompts):+ print(f"Prompt: {prompt}")+ print(f"Generated: {decoded_outputs[i]}")+ print("-" * 50)++ @classmethod+ def tearDownClass(self):+ shutil.rmtree("runs", ignore_errors=True)+++ ## require torch 2.6+ def test_load_gptq_model_8bits(self):+ model_name = "acloudfan/opt-125m-gptq-8bit"+ quantization_config = AutoRoundConfig()+ model = AutoModelForCausalLM.from_pretrained(model_name, torch_dtype="auto", trust_remote_code=True,+ device_map="cpu",+ quantization_config=quantization_config)+ tokenizer = AutoTokenizer.from_pretrained(model_name, trust_remote_code=True)+ self.model_infer(model, tokenizer)+++ def test_load_gptq_model_2bits(self):+ model_name = "LucasSantiago257/gemma-2b-2bits-gptq"+ quantization_config = AutoRoundConfig()+ model = AutoModelForCausalLM.from_pretrained(model_name, torch_dtype="auto", trust_remote_code=True,+ device_map="cpu",+ quantization_config=quantization_config)+ tokenizer = AutoTokenizer.from_pretrained(model_name, trust_remote_code=True)+ self.model_infer(model, tokenizer)+++ def test_mixed_precision(self):+ model = AutoModelForCausalLM.from_pretrained(self.model_name, torch_dtype="auto", trust_remote_code=True)+ tokenizer = AutoTokenizer.from_pretrained(self.model_name, trust_remote_code=True)+ layer_config = {}++ layer_config["model.decoder.layers.0.self_attn.k_proj"] = {"bits": 8}+ layer_config["model.decoder.layers.6.self_attn.out_proj"] = {"bits": 2, "group_size": 32}+ bits, group_size, sym = 4, 128, True+ from auto_round import AutoRound+ import torch+ autoround = AutoRound(+ model,+ tokenizer,+ bits=bits,+ group_size=group_size,+ iters=1,+ nsamples=1,+ sym=sym,+ layer_config=layer_config+ )+ quantized_model_path = self.save_folder+ autoround.quantize_and_save(output_dir=quantized_model_path, format="auto_round")++ model = AutoModelForCausalLM.from_pretrained(+ self.save_folder,+ torch_dtype=torch.float16,+ device_map="cpu",+ )+ tokenizer = AutoTokenizer.from_pretrained(self.save_folder)+ text = "There is a girl who likes adventure,"+ inputs = tokenizer(text, return_tensors="pt").to(model.device)+ res = tokenizer.decode(model.generate(**inputs, max_new_tokens=50)[0])+ print(res)+ assert ("!!!" not in res)+ shutil.rmtree(self.save_folder, ignore_errors=True)++ def test_autoround_sym(self):+ for bits in [4]:+ model = AutoModelForCausalLM.from_pretrained(self.model_name, torch_dtype="auto", trust_remote_code=True)+ tokenizer = AutoTokenizer.from_pretrained(self.model_name, trust_remote_code=True)+ bits, group_size, sym = bits, 128, True+ autoround = AutoRound(+ model,+ tokenizer,+ bits=bits,+ group_size=group_size,+ sym=sym,+ iters=2,+ seqlen=2+ )+ quantized_model_path = "./saved"++ autoround.quantize_and_save(output_dir=quantized_model_path, format="auto_round")++ model = AutoModelForCausalLM.from_pretrained(quantized_model_path, device_map="auto",+ trust_remote_code=True)+ tokenizer = AutoTokenizer.from_pretrained(quantized_model_path)+ text = "There is a girl who likes adventure,"+ inputs = tokenizer(text, return_tensors="pt").to(model.device)+ res = tokenizer.decode(model.generate(**inputs, max_new_tokens=50)[0])+ print(res)+ assert ("!!!" not in res)+ shutil.rmtree(self.save_folder, ignore_errors=True)++