PTQ-Level Efficiency, QAT-Level Accuracy | 训练后量化的效率,量化感知训练的精度
change quantization method name and made several refinements (#500)
Wenhua Cheng0328e5b1ce70
diff --git a/auto_round/export/export_to_autoround/export.py b/auto_round/export/export_to_autoround/export.pyindex 5c86d1fd7..a481290ca 100644--- a/auto_round/export/export_to_autoround/export.py+++ b/auto_round/export/export_to_autoround/export.py@@ -259,10 +259,10 @@ def save_quantized_as_autoround(output_dir, inplace=True, backend="auto_round:exlayer_config = kwargs["layer_config"]quantization_config = kwargs["serialization_dict"]- quantization_config["quant_method"] = "intel/auto-round"+ quantization_config["quant_method"] = "auto-round"quantization_config["backend"] = backend- if quantization_config["bits"]==3:+ if quantization_config["bits"] == 3:backend = "auto_round:auto_gptq"tokenizer = kwargs.get("tokenizer", None)diff --git a/auto_round/inference/auto_quantizer.py b/auto_round/inference/auto_quantizer.pyindex 8dc62efb3..cc78965ad 100644--- a/auto_round/inference/auto_quantizer.py+++ b/auto_round/inference/auto_quantizer.py@@ -180,8 +180,8 @@ class AutoHfQuantizer:@staticmethoddef supports_quant_method(quantization_config_dict):from transformers.quantizers.auto import AUTO_QUANTIZATION_CONFIG_MAPPING- AUTO_QUANTIZATION_CONFIG_MAPPING['intel/auto-round'] = AutoRoundConfig- AUTO_QUANTIZATION_CONFIG_MAPPING['intel/auto_round'] = AutoRoundConfig+ AUTO_QUANTIZATION_CONFIG_MAPPING['auto-round'] = AutoRoundConfig+ AUTO_QUANTIZATION_CONFIG_MAPPING['auto_round'] = AutoRoundConfigquant_method = quantization_config_dict.get("quant_method", None)if quantization_config_dict.get("load_in_8bit", False) or quantization_config_dict.get("load_in_4bit", False):suffix = "_4bit" if quantization_config_dict.get("load_in_4bit", False) else "_8bit"@@ -203,7 +203,7 @@ class AutoHfQuantizer:class AutoRoundQuantizationMethod(str, Enum):- AutoRound = "intel/auto-round"+ AutoRound = "auto-round"@dataclassdiff --git a/auto_round/inference/backend.py b/auto_round/inference/backend.pyindex 4a4020426..479858f5a 100644--- a/auto_round/inference/backend.py+++ b/auto_round/inference/backend.py@@ -679,10 +679,13 @@ def process_requirement(requirements: list):gptqmodel_requirements = Noneother_requirements = []for requirement in requirements:- if "gptqmodel" in requirement:- gptqmodel_requirements = requirement- else:- other_requirements.append(requirement)+ try:+ require_version(requirement)+ except:+ if "gptqmodel" in requirement:+ gptqmodel_requirements = requirement+ else:+ other_requirements.append(requirement)infos = []diff --git a/auto_round/inference/convert_model.py b/auto_round/inference/convert_model.pyindex 5024d2f27..e6070fdd8 100644--- a/auto_round/inference/convert_model.py+++ b/auto_round/inference/convert_model.py@@ -207,6 +207,16 @@ def _replace_by_quant_layers(module: nn.Module, layer_configs, target_backend, tdevices = BackendInfos[layer_backend_must].deviceif target_device not in devices:raise ValueError(f"{target_backend} does not support {target_device}, please change device or backend")+ ##check requirements+ requirements = BackendInfos[layer_backend_must].requirements+ requirements_info = process_requirement(requirements)+ if len(requirements_info) > 0:+ extra_info = ""+ for index,req in enumerate(requirements_info):+ extra_info += (f"`{req}`")+ if index != len(requirements_info)-1:+ extra_info += " and "+ raise ImportError(f"{target_backend} requires the follow libraries. Please install them via {extra_info}")target_backend = target_backend or orig_backend # Default to original backend if not specified@@ -398,7 +408,7 @@ def convert_hf_model(model: nn.Module, target_device="cpu"):if hasattr(quantization_config, "desc_act") and quantization_config.desc_act == True:##check static_groupif (hasattr(quantization_config, "static_groups") and not quantization_config.static_groups) or (- not hasattr(quantization_config, "static_groups")):+ not hasattr(quantization_config, "static_groups")):raise NotImplementedError("This GPTQ model may contain a non-dummy g_idx, which is not yet supported by AutoRound")@@ -439,8 +449,12 @@ def convert_hf_model(model: nn.Module, target_device="cpu"):extra_info = inforequirements = BackendInfos[best_backend].requirementsrequirements_info = process_requirement(requirements)- for req in requirements_info:- extra_info += (f"`{req}`") + " "+ for index,req in enumerate(requirements_info):+ extra_info += (f"`{req}`")+ if index != len(requirements_info) - 1:+ extra_info += " and "++logger.warning(extra_info)return model, used_backendsdiff --git a/auto_round_extension/qbits/qlinear_qbits.py b/auto_round_extension/qbits/qlinear_qbits.pyindex 0ef746688..aee384f32 100644--- a/auto_round_extension/qbits/qlinear_qbits.py+++ b/auto_round_extension/qbits/qlinear_qbits.py@@ -93,14 +93,15 @@ class QuantLinear(nn.Module):def req_check(self):torch_version = str(torch.__version__)if QBITS_AVAILABLE:- import intel_extension_for_transformers- itrex_version = str(intel_extension_for_transformers.__version__)- version_match_map = {"1.4": "2.2.0+cpu",- "1.4.1": "2.2.0+cpu", "1.4.2": "2.3.0+cpu"}- if itrex_version in version_match_map:- if torch_version != version_match_map[itrex_version]:- logger.warning(- f"Please install torch {version_match_map[itrex_version]} by command 'pip install torch=={version_match_map[itrex_version]} --extra-index-url https://download.pytorch.org/whl/cpu' as Intel Extension for Transformers {itrex_version} is not compatible with current torch.")+ pass+ # import intel_extension_for_transformers+ # itrex_version = str(intel_extension_for_transformers.__version__)+ # version_match_map = {"1.4": "2.2.0+cpu",+ # "1.4.1": "2.2.0+cpu", "1.4.2": "2.3.0+cpu"}+ # if itrex_version in version_match_map:+ # if torch_version != version_match_map[itrex_version]:+ # logger.warning(+ # f"Please install torch {version_match_map[itrex_version]} by command 'pip install torch=={version_match_map[itrex_version]} --extra-index-url https://download.pytorch.org/whl/cpu' as Intel Extension for Transformers {itrex_version} is not compatible with current torch.")else:logger.error("Please install Intel Extension for Transformers by running 'pip install intel-extension-for-transformers' as qbits linear requirements checking fail. ")diff --git a/auto_round_extension/qbits/qlinear_qbits_gptq.py b/auto_round_extension/qbits/qlinear_qbits_gptq.pyindex 0afca0288..8a1211955 100644--- a/auto_round_extension/qbits/qlinear_qbits_gptq.py+++ b/auto_round_extension/qbits/qlinear_qbits_gptq.py@@ -93,14 +93,15 @@ class QuantLinear(nn.Module):def req_check(self):torch_version = str(torch.__version__)if QBITS_AVAILABLE:- import intel_extension_for_transformers- itrex_version = str(intel_extension_for_transformers.__version__)- version_match_map = {"1.4": "2.2.0+cpu",- "1.4.1": "2.2.0+cpu", "1.4.2": "2.3.0+cpu"}- if itrex_version in version_match_map:- if torch_version != version_match_map[itrex_version]:- logger.warning(- f"Please install torch {version_match_map[itrex_version]} by command 'pip install torch=={version_match_map[itrex_version]} --extra-index-url https://download.pytorch.org/whl/cpu' as Intel Extension for Transformers {itrex_version} is not compatible with current torch.")+ pass+ # import intel_extension_for_transformers+ # itrex_version = str(intel_extension_for_transformers.__version__)+ # version_match_map = {"1.4": "2.2.0+cpu",+ # "1.4.1": "2.2.0+cpu", "1.4.2": "2.3.0+cpu"}+ # if itrex_version in version_match_map:+ # if torch_version != version_match_map[itrex_version]:+ # logger.warning(+ # f"Please install torch {version_match_map[itrex_version]} by command 'pip install torch=={version_match_map[itrex_version]} --extra-index-url https://download.pytorch.org/whl/cpu' as Intel Extension for Transformers {itrex_version} is not compatible with current torch.")else:logger.error("Please install Intel Extension for Transformers by running 'pip install intel-extension-for-transformers' as qbits linear requirements checking fail. ")diff --git a/test/test_generation.py b/test/test_generation.pyindex a4afc278f..1fa849764 100644--- a/test/test_generation.py+++ b/test/test_generation.py@@ -192,3 +192,38 @@ class TestAutoRoundFormatGeneration(unittest.TestCase):assert ("!!!" not in res)shutil.rmtree(self.save_folder, ignore_errors=True)++ def test_mixed_precision(self):+ model = AutoModelForCausalLM.from_pretrained(self.model_name, torch_dtype="auto", trust_remote_code=True)+ tokenizer = AutoTokenizer.from_pretrained(self.model_name, trust_remote_code=True)+ layer_config = {}++ layer_config["model.decoder.layers.0.self_attn.k_proj"] = {"bits": 8}+ layer_config["model.decoder.layers.6.self_attn.out_proj"] = {"bits": 2, "group_size": 32}+ bits, group_size, sym = 4, 128, True+ autoround = AutoRound(+ model,+ tokenizer,+ bits=bits,+ group_size=group_size,+ iters=1,+ nsamples=1,+ sym=sym,+ layer_config=layer_config+ )+ quantized_model_path = self.save_folder+ autoround.quantize_and_save(output_dir=quantized_model_path, format="auto_round")++ model = AutoModelForCausalLM.from_pretrained(+ self.save_folder,+ torch_dtype=torch.float16,+ device_map="cpu",+ )+ tokenizer = AutoTokenizer.from_pretrained(self.save_folder)+ text = "There is a girl who likes adventure,"+ inputs = tokenizer(text, return_tensors="pt").to(model.device)+ res = tokenizer.decode(model.generate(**inputs, max_new_tokens=50)[0])+ print(res)+ assert ("!!!" not in res)+ shutil.rmtree(self.save_folder, ignore_errors=True)+diff --git a/test_cuda/test_auto_round_format.py b/test_cuda/test_auto_round_format.pyindex e2f0929d9..6f8861b22 100644--- a/test_cuda/test_auto_round_format.py+++ b/test_cuda/test_auto_round_format.py@@ -26,8 +26,7 @@ class LLMDataLoader:class TestAutoRound(unittest.TestCase):@classmethoddef setUpClass(self):- self.model_name = "/models/opt-125m"- ##self.model_name = "/data5/wenhuach/Meta-Llama-3.1-8B-Instruct-int4-sym-inc"+ self.model_name = "facebook/opt-125m"self.llm_dataloader = LLMDataLoader()self.save_folder = "./saved"@@ -76,196 +75,40 @@ class TestAutoRound(unittest.TestCase):shutil.rmtree(self.save_folder, ignore_errors=True)shutil.rmtree("runs", ignore_errors=True)- def test_device_map(self):- from transformers import AutoModelForCausalLM, AutoTokenizer- model_name = "/data5/wenhuach/Meta-Llama-3.1-8B-Instruct-int4-sym-inc"-- device_map = {}- for i in range(0, 32):- key = f"model.layers.{str(i)}"- device_map[key] = "cuda"- device_map["model.layers.1"] = "cpu"- device_map["model.layers.2"] = "cpu"- device_map["model.layers.3"] = "cpu"- device_map["model.layers.4"] = "cpu"- device_map["model.layers.5"] = "cpu"- device_map["model.layers.6"] = "cpu"- device_map["model.layers.27"] = "cpu"- device_map["model.layers.28"] = "cpu"- device_map["model.layers.30"] = "cpu"- device_map["model.layers.31"] = "cpu"- device_map["lm_head"] = "cuda"- device_map["model.norm"] = "cuda"- device_map["model.rotary_emb"] = "cuda"- device_map["model.embed_tokens"] = "cuda"-- device_map1 = {}- for i in range(0, 32):- key = f"model.layers.{str(i)}"- device_map1[key] = "cuda"- device_map1["model.layers.1"] = "cuda:1"- device_map1["model.layers.2"] = "cuda:1"- device_map1["model.layers.3"] = "cuda:1"- device_map1["model.layers.4"] = "cuda:1"- device_map1["model.layers.5"] = "cuda:1"- device_map1["model.layers.6"] = "cuda:1"- device_map1["model.layers.27"] = "cuda:1"- device_map1["model.layers.28"] = "cuda:1"- device_map1["model.layers.30"] = "cuda:1"- device_map1["model.layers.31"] = "cuda:1"- device_map1["lm_head"] = "cuda:1"- device_map1["model.norm"] = "cuda"- device_map1["model.rotary_emb"] = "cuda"- device_map1["model.embed_tokens"] = "cuda"-- for tmp_device_map in [device_map1, device_map, None, 0, "balanced", "balanced_low_0", "sequential", "cpu",- "cuda:0", "cuda", "auto",- ]:- model = AutoModelForCausalLM.from_pretrained(- model_name,- torch_dtype="auto",- device_map=tmp_device_map- )-- tokenizer = AutoTokenizer.from_pretrained(model_name)-- prompts = [- "Hello,my name is",- # "The president of the United States is",- # "The capital of France is",- # "The future of AI is",- ]-- texts = []- for prompt in prompts:- messages = [- {"role": "user", "content": prompt}- ]- text = tokenizer.apply_chat_template(- messages,- tokenize=False,- add_generation_prompt=True- )- texts.append(text)-- inputs = tokenizer(texts, return_tensors="pt", padding=False, truncation=True)-- outputs = model.generate(- input_ids=inputs["input_ids"].to(model.device),- attention_mask=inputs["attention_mask"].to(model.device),- do_sample=False, ## change this to follow official usage- max_new_tokens=5- )- generated_ids = [- output_ids[len(input_ids):] for input_ids, output_ids in zip(inputs["input_ids"], outputs)- ]-- decoded_outputs = tokenizer.batch_decode(generated_ids, skip_special_tokens=True)-- for i, prompt in enumerate(prompts):- print(f"Prompt: {prompt}")- print(f"Generated: {decoded_outputs[i]}")- print("-" * 50)- model = None- del model- torch.cuda.empty_cache()-- def test_device_map_for_triton(self):- from transformers import AutoModelForCausalLM, AutoTokenizer- model_name = "/data5/wenhuach/Meta-Llama-3.1-8B-Instruct-int4-sym-inc"-- device_map = {}- for i in range(0, 32):- key = f"model.layers.{str(i)}"- device_map[key] = "cuda"- device_map["model.layers.1"] = "cpu"- device_map["model.layers.2"] = "cpu"- device_map["model.layers.3"] = "cpu"- device_map["model.layers.4"] = "cpu"- device_map["model.layers.5"] = "cpu"- device_map["model.layers.6"] = "cpu"- device_map["model.layers.27"] = "cpu"- device_map["model.layers.28"] = "cpu"- device_map["model.layers.30"] = "cpu"- device_map["model.layers.31"] = "cpu"- device_map["lm_head"] = "cuda"- device_map["model.norm"] = "cuda"- device_map["model.rotary_emb"] = "cuda"- device_map["model.embed_tokens"] = "cuda"-- device_map1 = {}- for i in range(0, 32):- key = f"model.layers.{str(i)}"- device_map1[key] = "cuda"- device_map1["model.layers.1"] = "cuda:1"- device_map1["model.layers.2"] = "cuda:1"- device_map1["model.layers.3"] = "cuda:1"- device_map1["model.layers.4"] = "cuda:1"- device_map1["model.layers.5"] = "cuda:1"- device_map1["model.layers.6"] = "cuda:1"- device_map1["model.layers.27"] = "cuda:1"- device_map1["model.layers.28"] = "cuda:1"- device_map1["model.layers.30"] = "cuda:1"- device_map1["model.layers.31"] = "cuda:1"- device_map1["lm_head"] = "cuda:1"- device_map1["model.norm"] = "cuda"- device_map1["model.rotary_emb"] = "cuda"- device_map1["model.embed_tokens"] = "cuda"- from auto_round import AutoRoundConfig- quantization_config = AutoRoundConfig(backend="tritonv2")-- for tmp_device_map in [device_map1, 0, "balanced", "balanced_low_0", "sequential",- "cuda:0", "cuda", "auto",- ]:- model = AutoModelForCausalLM.from_pretrained(- model_name,- torch_dtype="auto",- device_map=tmp_device_map,- quantization_config=quantization_config- )+ def test_mixed_precision(self):+ model = AutoModelForCausalLM.from_pretrained(self.model_name, torch_dtype="auto", trust_remote_code=True)+ tokenizer = AutoTokenizer.from_pretrained(self.model_name, trust_remote_code=True)+ layer_config = {}- tokenizer = AutoTokenizer.from_pretrained(model_name)-- prompts = [- "Hello,my name is",- # "The president of the United States is",- # "The capital of France is",- # "The future of AI is",- ]-- texts = []- for prompt in prompts:- messages = [- {"role": "user", "content": prompt}- ]- text = tokenizer.apply_chat_template(- messages,- tokenize=False,- add_generation_prompt=True- )- texts.append(text)-- inputs = tokenizer(texts, return_tensors="pt", padding=False, truncation=True)-- outputs = model.generate(- input_ids=inputs["input_ids"].to(model.device),- attention_mask=inputs["attention_mask"].to(model.device),- do_sample=False, ## change this to follow official usage- max_new_tokens=5- )- generated_ids = [- output_ids[len(input_ids):] for input_ids, output_ids in zip(inputs["input_ids"], outputs)- ]+ layer_config["model.decoder.layers.0.self_attn.k_proj"] = {"bits": 8}+ layer_config["model.decoder.layers.2.self_attn.q_proj"] = {"bits": 3,+ "group_size": 64} ## 3bits when using asym will have some issue+ layer_config["model.decoder.layers.6.self_attn.out_proj"] = {"bits": 2, "group_size": 32}+ bits, group_size, sym = 4, 128, True+ autoround = AutoRound(+ model,+ tokenizer,+ bits=bits,+ group_size=group_size,+ sym=sym,+ layer_config=layer_config+ )+ quantized_model_path = self.save_folder+ autoround.quantize_and_save(output_dir=quantized_model_path, format="auto_round")+ quantization_config = AutoRoundConfig(backend="auto")+ model = AutoModelForCausalLM.from_pretrained(+ self.save_folder,+ torch_dtype=torch.float16,+ device_map="auto",+ quantization_config=quantization_config+ )- decoded_outputs = tokenizer.batch_decode(generated_ids, skip_special_tokens=True)+ tokenizer = AutoTokenizer.from_pretrained(self.save_folder)+ self.model_infer(model, tokenizer)+ result = simple_evaluate_user_model(model, tokenizer, batch_size=16, tasks="lambada_openai")+ print(result['results']['lambada_openai']['acc,none'])+ self.assertGreater(result['results']['lambada_openai']['acc,none'], 0.32)- for i, prompt in enumerate(prompts):- print(f"Prompt: {prompt}")- print(f"Generated: {decoded_outputs[i]}")- print("-" * 50)- model = None- del model- torch.cuda.empty_cache()def test_awq_backend(self):model = AutoModelForCausalLM.from_pretrained(self.model_name, torch_dtype="auto", trust_remote_code=True)@@ -307,6 +150,7 @@ class TestAutoRound(unittest.TestCase):tokenizer = AutoTokenizer.from_pretrained(self.save_folder)self.model_infer(model, tokenizer)+ shutil.rmtree(self.save_folder, ignore_errors=True)def test_tritonv2_bf16(self):model_name = "/data5/wenhuach/Meta-Llama-3.1-8B-Instruct-int4-sym-inc"@@ -409,11 +253,11 @@ class TestAutoRound(unittest.TestCase):shutil.rmtree("./saved", ignore_errors=True)- def test_autoround_asym(self):+ def test_autoround_sym(self):for bits in [2, 3, 4, 8]:model = AutoModelForCausalLM.from_pretrained(self.model_name, torch_dtype="auto", trust_remote_code=True)tokenizer = AutoTokenizer.from_pretrained(self.model_name, trust_remote_code=True)- bits, group_size, sym = bits, 128, False+ bits, group_size, sym = bits, 128, Trueautoround = AutoRound(model,tokenizer,@@ -436,12 +280,22 @@ class TestAutoRound(unittest.TestCase):res = tokenizer.decode(model.generate(**inputs, max_new_tokens=50)[0])print(res)assert ("!!!" not in res)+ shutil.rmtree(self.save_folder, ignore_errors=True)- def test_autoround_sym(self):- for bits in [2, 3, 4, 8]:+ def test_load_gptq_model_3bits(self):+ model_name = "LucasSantiago257/gemma-2b-2bits-gptq"+ quantization_config = AutoRoundConfig()+ model = AutoModelForCausalLM.from_pretrained(model_name, torch_dtype="auto", trust_remote_code=True,+ device_map="auto",+ quantization_config=quantization_config)+ tokenizer = AutoTokenizer.from_pretrained(model_name, trust_remote_code=True)+ self.model_infer(model, tokenizer)++ def test_autoround_asym(self):+ for bits in [2,3,4,8]:model = AutoModelForCausalLM.from_pretrained(self.model_name, torch_dtype="auto", trust_remote_code=True)tokenizer = AutoTokenizer.from_pretrained(self.model_name, trust_remote_code=True)- bits, group_size, sym = bits, 128, True+ bits, group_size, sym = bits, 128, Falseautoround = AutoRound(model,tokenizer,@@ -452,11 +306,11 @@ class TestAutoRound(unittest.TestCase):seqlen=2,dataset=self.llm_dataloader,)- quantized_model_path = "./saved"+ quantized_model_path = self.save_folderautoround.quantize_and_save(output_dir=quantized_model_path, format="auto_round")- model = AutoModelForCausalLM.from_pretrained(quantized_model_path, device_map="auto",+ model = AutoModelForCausalLM.from_pretrained(quantized_model_path, device_map="cuda:0",trust_remote_code=True)tokenizer = AutoTokenizer.from_pretrained(quantized_model_path)text = "There is a girl who likes adventure,"@@ -464,15 +318,7 @@ class TestAutoRound(unittest.TestCase):res = tokenizer.decode(model.generate(**inputs, max_new_tokens=50)[0])print(res)assert ("!!!" not in res)-- def test_load_gptq_model_3bits(self):- model_name = "LucasSantiago257/gemma-2b-2bits-gptq"- quantization_config = AutoRoundConfig()- model = AutoModelForCausalLM.from_pretrained(model_name, torch_dtype="auto", trust_remote_code=True,- device_map="auto",- quantization_config=quantization_config)- tokenizer = AutoTokenizer.from_pretrained(model_name, trust_remote_code=True)- self.model_infer(model, tokenizer)+ shutil.rmtree(self.save_folder, ignore_errors=True)if __name__ == "__main__":diff --git a/test_cuda/test_multiple_card.py b/test_cuda/test_multiple_card.pynew file mode 100644index 000000000..83d153122--- /dev/null+++ b/test_cuda/test_multiple_card.py@@ -0,0 +1,292 @@+import shutil+import sys+import unittest+from auto_round.eval.evaluation import simple_evaluate+from lm_eval.utils import make_table # pylint: disable=E0401++sys.path.insert(0, "..")+from auto_round import AutoRound++import torch+from transformers import AutoModelForCausalLM, AutoTokenizer+import re+++def get_accuracy(data):+ match = re.search(r'\|acc\s+\|[↑↓]\s+\|\s+([\d.]+)\|', data)++ if match:+ accuracy = float(match.group(1))+ return accuracy+ else:+ return 0.0+++# import os+# os.environ["CUDA_VISIBLE_DEVICES"] = "0,1"+class TestAutoRound(unittest.TestCase):+ @classmethod+ def setUpClass(self):+ self.save_dir = "./saved"+ self.tasks = "lambada_openai"++ @classmethod+ def tearDownClass(self):+ shutil.rmtree(self.save_dir, ignore_errors=True)+ shutil.rmtree("runs", ignore_errors=True)++ def test_device_map(self):+ model_name = "/models/Qwen2-0.5B-Instruct"+ model = AutoModelForCausalLM.from_pretrained(model_name, torch_dtype=torch.float16)+ tokenizer = AutoTokenizer.from_pretrained(model_name)+ device_map = {".*q_proj": '0', ".*k_proj": "cuda:1", "v_proj": 1, ".*up_proj": "cpu"}+ autoround = AutoRound(model, tokenizer, iters=2, device_map=device_map, nsamples=7,seqlen=32)+ autoround.quantize()++ def test_device_map_str(self):+ model_name = "/models/Qwen2-0.5B-Instruct"+ model = AutoModelForCausalLM.from_pretrained(model_name, torch_dtype=torch.float16)+ tokenizer = AutoTokenizer.from_pretrained(model_name)+ device_map = '.*q_proj:0,.*k_proj:cuda:0,v_proj:1,.*up_proj:1'+ autoround = AutoRound(model, tokenizer,device_map=device_map)+ autoround.quantize()+ autoround.save_quantized(self.save_dir, format="auto_round", inplace=False)+ model_args = f"pretrained={self.save_dir}"+ res = simple_evaluate(model="hf", model_args=model_args,+ tasks=self.tasks,+ batch_size="auto")+ res = make_table(res)+ accuracy = get_accuracy(res)+ print(accuracy)+ assert accuracy > 0.45 ##0.4786+ shutil.rmtree("./saved", ignore_errors=True)++ def test_layer_norm(self):+ model_name = "/models/opt-125m"+ model = AutoModelForCausalLM.from_pretrained(model_name, torch_dtype=torch.float16)+ tokenizer = AutoTokenizer.from_pretrained(model_name)+ device_map = {"norm": "cuda:1"}+ autoround = AutoRound(model, tokenizer, iters=2, device_map=device_map, nsamples=7, seqlen=32,+ enable_norm_bias_tuning=True)+ autoround.quantize()+++ def test_rms_norm(self):+ model_name = "/models/Qwen2-0.5B-Instruct"+ model = AutoModelForCausalLM.from_pretrained(model_name, torch_dtype=torch.float16)+ tokenizer = AutoTokenizer.from_pretrained(model_name)+ device_map = {"norm": "cuda:1"}+ autoround = AutoRound(model, tokenizer, iters=2, device_map=device_map, nsamples=7, seqlen=32,+ enable_norm_bias_tuning=True)+ autoround.quantize()++ def test_act_quantization(self):+ model_name = "/models/Qwen2-0.5B-Instruct"+ model = AutoModelForCausalLM.from_pretrained(model_name, torch_dtype=torch.float16)+ tokenizer = AutoTokenizer.from_pretrained(model_name)+ device_map = {".*q_proj": '0', ".*k_proj": "cuda:1", "v_proj": 1, ".*up_proj": "1"}+ autoround = AutoRound(model, tokenizer, iters=2, device_map=device_map, nsamples=7,seqlen=32,act_bits=4,act_dynamic=False)+ autoround.quantize()++ def test_lm_head(self):+ model_name = "/models/Qwen2.5-7B-Instruct"+ model = AutoModelForCausalLM.from_pretrained(model_name, torch_dtype=torch.float16)+ tokenizer = AutoTokenizer.from_pretrained(model_name)+ device_map = {".*q_proj": '0', ".*k_proj": "cuda:1", "v_proj": 1, ".*up_proj": "1","lm_head":1}+ layer_config={"lm_head": {"bits": 4}}+ autoround = AutoRound(model, tokenizer, iters=2, device_map=device_map, nsamples=7, seqlen=32,+ enable_norm_bias_tuning=True,layer_config=layer_config)+ autoround.quantize()++ def test_device_map(self):+ from transformers import AutoModelForCausalLM, AutoTokenizer+ model_name = "/data5/wenhuach/Meta-Llama-3.1-8B-Instruct-int4-sym-inc"++ device_map = {}+ for i in range(0, 32):+ key = f"model.layers.{str(i)}"+ device_map[key] = "cuda"+ device_map["model.layers.1"] = "cpu"+ device_map["model.layers.2"] = "cpu"+ device_map["model.layers.3"] = "cpu"+ device_map["model.layers.4"] = "cpu"+ device_map["model.layers.5"] = "cpu"+ device_map["model.layers.6"] = "cpu"+ device_map["model.layers.27"] = "cpu"+ device_map["model.layers.28"] = "cpu"+ device_map["model.layers.30"] = "cpu"+ device_map["model.layers.31"] = "cpu"+ device_map["lm_head"] = "cuda"+ device_map["model.norm"] = "cuda"+ device_map["model.rotary_emb"] = "cuda"+ device_map["model.embed_tokens"] = "cuda"++ device_map1 = {}+ for i in range(0, 32):+ key = f"model.layers.{str(i)}"+ device_map1[key] = "cuda"+ device_map1["model.layers.1"] = "cuda:1"+ device_map1["model.layers.2"] = "cuda:1"+ device_map1["model.layers.3"] = "cuda:1"+ device_map1["model.layers.4"] = "cuda:1"+ device_map1["model.layers.5"] = "cuda:1"+ device_map1["model.layers.6"] = "cuda:1"+ device_map1["model.layers.27"] = "cuda:1"+ device_map1["model.layers.28"] = "cuda:1"+ device_map1["model.layers.30"] = "cuda:1"+ device_map1["model.layers.31"] = "cuda:1"+ device_map1["lm_head"] = "cuda:1"+ device_map1["model.norm"] = "cuda"+ device_map1["model.rotary_emb"] = "cuda"+ device_map1["model.embed_tokens"] = "cuda"++ for tmp_device_map in [device_map1, device_map, None, 0, "balanced", "balanced_low_0", "sequential", "cpu",+ "cuda:0", "cuda", "auto",+ ]:+ model = AutoModelForCausalLM.from_pretrained(+ model_name,+ torch_dtype="auto",+ device_map=tmp_device_map+ )++ tokenizer = AutoTokenizer.from_pretrained(model_name)++ prompts = [+ "Hello,my name is",+ # "The president of the United States is",+ # "The capital of France is",+ # "The future of AI is",+ ]++ texts = []+ for prompt in prompts:+ messages = [+ {"role": "user", "content": prompt}+ ]+ text = tokenizer.apply_chat_template(+ messages,+ tokenize=False,+ add_generation_prompt=True+ )+ texts.append(text)++ inputs = tokenizer(texts, return_tensors="pt", padding=False, truncation=True)++ outputs = model.generate(+ input_ids=inputs["input_ids"].to(model.device),+ attention_mask=inputs["attention_mask"].to(model.device),+ do_sample=False, ## change this to follow official usage+ max_new_tokens=5+ )+ generated_ids = [+ output_ids[len(input_ids):] for input_ids, output_ids in zip(inputs["input_ids"], outputs)+ ]++ decoded_outputs = tokenizer.batch_decode(generated_ids, skip_special_tokens=True)++ for i, prompt in enumerate(prompts):+ print(f"Prompt: {prompt}")+ print(f"Generated: {decoded_outputs[i]}")+ print("-" * 50)+ model = None+ del model+ torch.cuda.empty_cache()++ def test_device_map_for_triton(self):+ from transformers import AutoModelForCausalLM, AutoTokenizer+ model_name = "/data5/wenhuach/Meta-Llama-3.1-8B-Instruct-int4-sym-inc"++ device_map = {}+ for i in range(0, 32):+ key = f"model.layers.{str(i)}"+ device_map[key] = "cuda"+ device_map["model.layers.1"] = "cpu"+ device_map["model.layers.2"] = "cpu"+ device_map["model.layers.3"] = "cpu"+ device_map["model.layers.4"] = "cpu"+ device_map["model.layers.5"] = "cpu"+ device_map["model.layers.6"] = "cpu"+ device_map["model.layers.27"] = "cpu"+ device_map["model.layers.28"] = "cpu"+ device_map["model.layers.30"] = "cpu"+ device_map["model.layers.31"] = "cpu"+ device_map["lm_head"] = "cuda"+ device_map["model.norm"] = "cuda"+ device_map["model.rotary_emb"] = "cuda"+ device_map["model.embed_tokens"] = "cuda"++ device_map1 = {}+ for i in range(0, 32):+ key = f"model.layers.{str(i)}"+ device_map1[key] = "cuda"+ device_map1["model.layers.1"] = "cuda:1"+ device_map1["model.layers.2"] = "cuda:1"+ device_map1["model.layers.3"] = "cuda:1"+ device_map1["model.layers.4"] = "cuda:1"+ device_map1["model.layers.5"] = "cuda:1"+ device_map1["model.layers.6"] = "cuda:1"+ device_map1["model.layers.27"] = "cuda:1"+ device_map1["model.layers.28"] = "cuda:1"+ device_map1["model.layers.30"] = "cuda:1"+ device_map1["model.layers.31"] = "cuda:1"+ device_map1["lm_head"] = "cuda:1"+ device_map1["model.norm"] = "cuda"+ device_map1["model.rotary_emb"] = "cuda"+ device_map1["model.embed_tokens"] = "cuda"+ from auto_round import AutoRoundConfig+ quantization_config = AutoRoundConfig(backend="tritonv2")++ for tmp_device_map in [device_map1, 0, "balanced", "balanced_low_0", "sequential",+ "cuda:0", "cuda", "auto",+ ]:+ model = AutoModelForCausalLM.from_pretrained(+ model_name,+ torch_dtype="auto",+ device_map=tmp_device_map,+ quantization_config=quantization_config+ )++ tokenizer = AutoTokenizer.from_pretrained(model_name)++ prompts = [+ "Hello,my name is",+ # "The president of the United States is",+ # "The capital of France is",+ # "The future of AI is",+ ]++ texts = []+ for prompt in prompts:+ messages = [+ {"role": "user", "content": prompt}+ ]+ text = tokenizer.apply_chat_template(+ messages,+ tokenize=False,+ add_generation_prompt=True+ )+ texts.append(text)++ inputs = tokenizer(texts, return_tensors="pt", padding=False, truncation=True)++ outputs = model.generate(+ input_ids=inputs["input_ids"].to(model.device),+ attention_mask=inputs["attention_mask"].to(model.device),+ do_sample=False, ## change this to follow official usage+ max_new_tokens=5+ )+ generated_ids = [+ output_ids[len(input_ids):] for input_ids, output_ids in zip(inputs["input_ids"], outputs)+ ]++ decoded_outputs = tokenizer.batch_decode(generated_ids, skip_special_tokens=True)++ for i, prompt in enumerate(prompts):+ print(f"Prompt: {prompt}")+ print(f"Generated: {decoded_outputs[i]}")+ print("-" * 50)+ model = None+ del model+ torch.cuda.empty_cache()++diff --git a/test_cuda/test_multiple_card_tuning.py b/test_cuda/test_multiple_card_tuning.pydeleted file mode 100644index 67254cd71..000000000--- a/test_cuda/test_multiple_card_tuning.py+++ /dev/null@@ -1,101 +0,0 @@-import shutil-import sys-import unittest-from auto_round.eval.evaluation import simple_evaluate-from lm_eval.utils import make_table # pylint: disable=E0401--sys.path.insert(0, "..")-from auto_round import AutoRound--import torch-from transformers import AutoModelForCausalLM, AutoTokenizer-import re---def get_accuracy(data):- match = re.search(r'\|acc\s+\|[↑↓]\s+\|\s+([\d.]+)\|', data)-- if match:- accuracy = float(match.group(1))- return accuracy- else:- return 0.0---# import os-# os.environ["CUDA_VISIBLE_DEVICES"] = "0,1"-class TestAutoRound(unittest.TestCase):- @classmethod- def setUpClass(self):- self.save_dir = "./saved"- self.tasks = "lambada_openai"-- @classmethod- def tearDownClass(self):- shutil.rmtree(self.save_dir, ignore_errors=True)- shutil.rmtree("runs", ignore_errors=True)-- def test_device_map(self):- model_name = "/models/Qwen2-0.5B-Instruct"- model = AutoModelForCausalLM.from_pretrained(model_name, torch_dtype=torch.float16)- tokenizer = AutoTokenizer.from_pretrained(model_name)- device_map = {".*q_proj": '0', ".*k_proj": "cuda:1", "v_proj": 1, ".*up_proj": "cpu"}- autoround = AutoRound(model, tokenizer, iters=2, device_map=device_map, nsamples=7,seqlen=32)- autoround.quantize()-- def test_device_map_str(self):- model_name = "/models/Qwen2-0.5B-Instruct"- model = AutoModelForCausalLM.from_pretrained(model_name, torch_dtype=torch.float16)- tokenizer = AutoTokenizer.from_pretrained(model_name)- device_map = '.*q_proj:0,.*k_proj:cuda:0,v_proj:1,.*up_proj:1'- autoround = AutoRound(model, tokenizer,device_map=device_map)- autoround.quantize()- autoround.save_quantized(self.save_dir, format="auto_round", inplace=False)- model_args = f"pretrained={self.save_dir}"- res = simple_evaluate(model="hf", model_args=model_args,- tasks=self.tasks,- batch_size="auto")- res = make_table(res)- accuracy = get_accuracy(res)- print(accuracy)- assert accuracy > 0.45 ##0.4786- shutil.rmtree("./saved", ignore_errors=True)-- def test_layer_norm(self):- model_name = "/models/opt-125m"- model = AutoModelForCausalLM.from_pretrained(model_name, torch_dtype=torch.float16)- tokenizer = AutoTokenizer.from_pretrained(model_name)- device_map = {"norm": "cuda:1"}- autoround = AutoRound(model, tokenizer, iters=2, device_map=device_map, nsamples=7, seqlen=32,- enable_norm_bias_tuning=True)- autoround.quantize()--- def test_rms_norm(self):- model_name = "/models/Qwen2-0.5B-Instruct"- model = AutoModelForCausalLM.from_pretrained(model_name, torch_dtype=torch.float16)- tokenizer = AutoTokenizer.from_pretrained(model_name)- device_map = {"norm": "cuda:1"}- autoround = AutoRound(model, tokenizer, iters=2, device_map=device_map, nsamples=7, seqlen=32,- enable_norm_bias_tuning=True)- autoround.quantize()-- def test_act_quantization(self):- model_name = "/models/Qwen2-0.5B-Instruct"- model = AutoModelForCausalLM.from_pretrained(model_name, torch_dtype=torch.float16)- tokenizer = AutoTokenizer.from_pretrained(model_name)- device_map = {".*q_proj": '0', ".*k_proj": "cuda:1", "v_proj": 1, ".*up_proj": "1"}- autoround = AutoRound(model, tokenizer, iters=2, device_map=device_map, nsamples=7,seqlen=32,act_bits=4,act_dynamic=False)- autoround.quantize()-- def test_lm_head(self):- model_name = "/models/Qwen2.5-7B-Instruct"- model = AutoModelForCausalLM.from_pretrained(model_name, torch_dtype=torch.float16)- tokenizer = AutoTokenizer.from_pretrained(model_name)- device_map = {".*q_proj": '0', ".*k_proj": "cuda:1", "v_proj": 1, ".*up_proj": "1","lm_head":1}- layer_config={"lm_head": {"bits": 4}}- autoround = AutoRound(model, tokenizer, iters=2, device_map=device_map, nsamples=7, seqlen=32,- enable_norm_bias_tuning=True,layer_config=layer_config)- autoround.quantize()--