Skip to content

PTQ-Level Efficiency, QAT-Level Accuracy | 训练后量化的效率,量化感知训练的精度

change quantization method name and made several refinements (#500)

Wenhua Cheng0328e5b1ce70
diff --git a/auto_round/export/export_to_autoround/export.py b/auto_round/export/export_to_autoround/export.py
index 5c86d1fd7..a481290ca 100644
--- a/auto_round/export/export_to_autoround/export.py
+++ b/auto_round/export/export_to_autoround/export.py
@@ -259,10 +259,10 @@ def save_quantized_as_autoround(output_dir, inplace=True, backend="auto_round:ex
layer_config = kwargs["layer_config"]
quantization_config = kwargs["serialization_dict"]
- quantization_config["quant_method"] = "intel/auto-round"
+ quantization_config["quant_method"] = "auto-round"
quantization_config["backend"] = backend
- if quantization_config["bits"]==3:
+ if quantization_config["bits"] == 3:
backend = "auto_round:auto_gptq"
tokenizer = kwargs.get("tokenizer", None)
diff --git a/auto_round/inference/auto_quantizer.py b/auto_round/inference/auto_quantizer.py
index 8dc62efb3..cc78965ad 100644
--- a/auto_round/inference/auto_quantizer.py
+++ b/auto_round/inference/auto_quantizer.py
@@ -180,8 +180,8 @@ class AutoHfQuantizer:
@staticmethod
def supports_quant_method(quantization_config_dict):
from transformers.quantizers.auto import AUTO_QUANTIZATION_CONFIG_MAPPING
- AUTO_QUANTIZATION_CONFIG_MAPPING['intel/auto-round'] = AutoRoundConfig
- AUTO_QUANTIZATION_CONFIG_MAPPING['intel/auto_round'] = AutoRoundConfig
+ AUTO_QUANTIZATION_CONFIG_MAPPING['auto-round'] = AutoRoundConfig
+ AUTO_QUANTIZATION_CONFIG_MAPPING['auto_round'] = AutoRoundConfig
quant_method = quantization_config_dict.get("quant_method", None)
if quantization_config_dict.get("load_in_8bit", False) or quantization_config_dict.get("load_in_4bit", False):
suffix = "_4bit" if quantization_config_dict.get("load_in_4bit", False) else "_8bit"
@@ -203,7 +203,7 @@ class AutoHfQuantizer:
class AutoRoundQuantizationMethod(str, Enum):
- AutoRound = "intel/auto-round"
+ AutoRound = "auto-round"
@dataclass
diff --git a/auto_round/inference/backend.py b/auto_round/inference/backend.py
index 4a4020426..479858f5a 100644
--- a/auto_round/inference/backend.py
+++ b/auto_round/inference/backend.py
@@ -679,10 +679,13 @@ def process_requirement(requirements: list):
gptqmodel_requirements = None
other_requirements = []
for requirement in requirements:
- if "gptqmodel" in requirement:
- gptqmodel_requirements = requirement
- else:
- other_requirements.append(requirement)
+ try:
+ require_version(requirement)
+ except:
+ if "gptqmodel" in requirement:
+ gptqmodel_requirements = requirement
+ else:
+ other_requirements.append(requirement)
infos = []
diff --git a/auto_round/inference/convert_model.py b/auto_round/inference/convert_model.py
index 5024d2f27..e6070fdd8 100644
--- a/auto_round/inference/convert_model.py
+++ b/auto_round/inference/convert_model.py
@@ -207,6 +207,16 @@ def _replace_by_quant_layers(module: nn.Module, layer_configs, target_backend, t
devices = BackendInfos[layer_backend_must].device
if target_device not in devices:
raise ValueError(f"{target_backend} does not support {target_device}, please change device or backend")
+ ##check requirements
+ requirements = BackendInfos[layer_backend_must].requirements
+ requirements_info = process_requirement(requirements)
+ if len(requirements_info) > 0:
+ extra_info = ""
+ for index,req in enumerate(requirements_info):
+ extra_info += (f"`{req}`")
+ if index != len(requirements_info)-1:
+ extra_info += " and "
+ raise ImportError(f"{target_backend} requires the follow libraries. Please install them via {extra_info}")
target_backend = target_backend or orig_backend # Default to original backend if not specified
@@ -398,7 +408,7 @@ def convert_hf_model(model: nn.Module, target_device="cpu"):
if hasattr(quantization_config, "desc_act") and quantization_config.desc_act == True:
##check static_group
if (hasattr(quantization_config, "static_groups") and not quantization_config.static_groups) or (
- not hasattr(quantization_config, "static_groups")):
+ not hasattr(quantization_config, "static_groups")):
raise NotImplementedError(
"This GPTQ model may contain a non-dummy g_idx, which is not yet supported by AutoRound")
@@ -439,8 +449,12 @@ def convert_hf_model(model: nn.Module, target_device="cpu"):
extra_info = info
requirements = BackendInfos[best_backend].requirements
requirements_info = process_requirement(requirements)
- for req in requirements_info:
- extra_info += (f"`{req}`") + " "
+ for index,req in enumerate(requirements_info):
+ extra_info += (f"`{req}`")
+ if index != len(requirements_info) - 1:
+ extra_info += " and "
+
+
logger.warning(extra_info)
return model, used_backends
diff --git a/auto_round_extension/qbits/qlinear_qbits.py b/auto_round_extension/qbits/qlinear_qbits.py
index 0ef746688..aee384f32 100644
--- a/auto_round_extension/qbits/qlinear_qbits.py
+++ b/auto_round_extension/qbits/qlinear_qbits.py
@@ -93,14 +93,15 @@ class QuantLinear(nn.Module):
def req_check(self):
torch_version = str(torch.__version__)
if QBITS_AVAILABLE:
- import intel_extension_for_transformers
- itrex_version = str(intel_extension_for_transformers.__version__)
- version_match_map = {"1.4": "2.2.0+cpu",
- "1.4.1": "2.2.0+cpu", "1.4.2": "2.3.0+cpu"}
- if itrex_version in version_match_map:
- if torch_version != version_match_map[itrex_version]:
- logger.warning(
- f"Please install torch {version_match_map[itrex_version]} by command 'pip install torch=={version_match_map[itrex_version]} --extra-index-url https://download.pytorch.org/whl/cpu' as Intel Extension for Transformers {itrex_version} is not compatible with current torch.")
+ pass
+ # import intel_extension_for_transformers
+ # itrex_version = str(intel_extension_for_transformers.__version__)
+ # version_match_map = {"1.4": "2.2.0+cpu",
+ # "1.4.1": "2.2.0+cpu", "1.4.2": "2.3.0+cpu"}
+ # if itrex_version in version_match_map:
+ # if torch_version != version_match_map[itrex_version]:
+ # logger.warning(
+ # f"Please install torch {version_match_map[itrex_version]} by command 'pip install torch=={version_match_map[itrex_version]} --extra-index-url https://download.pytorch.org/whl/cpu' as Intel Extension for Transformers {itrex_version} is not compatible with current torch.")
else:
logger.error(
"Please install Intel Extension for Transformers by running 'pip install intel-extension-for-transformers' as qbits linear requirements checking fail. ")
diff --git a/auto_round_extension/qbits/qlinear_qbits_gptq.py b/auto_round_extension/qbits/qlinear_qbits_gptq.py
index 0afca0288..8a1211955 100644
--- a/auto_round_extension/qbits/qlinear_qbits_gptq.py
+++ b/auto_round_extension/qbits/qlinear_qbits_gptq.py
@@ -93,14 +93,15 @@ class QuantLinear(nn.Module):
def req_check(self):
torch_version = str(torch.__version__)
if QBITS_AVAILABLE:
- import intel_extension_for_transformers
- itrex_version = str(intel_extension_for_transformers.__version__)
- version_match_map = {"1.4": "2.2.0+cpu",
- "1.4.1": "2.2.0+cpu", "1.4.2": "2.3.0+cpu"}
- if itrex_version in version_match_map:
- if torch_version != version_match_map[itrex_version]:
- logger.warning(
- f"Please install torch {version_match_map[itrex_version]} by command 'pip install torch=={version_match_map[itrex_version]} --extra-index-url https://download.pytorch.org/whl/cpu' as Intel Extension for Transformers {itrex_version} is not compatible with current torch.")
+ pass
+ # import intel_extension_for_transformers
+ # itrex_version = str(intel_extension_for_transformers.__version__)
+ # version_match_map = {"1.4": "2.2.0+cpu",
+ # "1.4.1": "2.2.0+cpu", "1.4.2": "2.3.0+cpu"}
+ # if itrex_version in version_match_map:
+ # if torch_version != version_match_map[itrex_version]:
+ # logger.warning(
+ # f"Please install torch {version_match_map[itrex_version]} by command 'pip install torch=={version_match_map[itrex_version]} --extra-index-url https://download.pytorch.org/whl/cpu' as Intel Extension for Transformers {itrex_version} is not compatible with current torch.")
else:
logger.error(
"Please install Intel Extension for Transformers by running 'pip install intel-extension-for-transformers' as qbits linear requirements checking fail. ")
diff --git a/test/test_generation.py b/test/test_generation.py
index a4afc278f..1fa849764 100644
--- a/test/test_generation.py
+++ b/test/test_generation.py
@@ -192,3 +192,38 @@ class TestAutoRoundFormatGeneration(unittest.TestCase):
assert ("!!!" not in res)
shutil.rmtree(self.save_folder, ignore_errors=True)
+
+ def test_mixed_precision(self):
+ model = AutoModelForCausalLM.from_pretrained(self.model_name, torch_dtype="auto", trust_remote_code=True)
+ tokenizer = AutoTokenizer.from_pretrained(self.model_name, trust_remote_code=True)
+ layer_config = {}
+
+ layer_config["model.decoder.layers.0.self_attn.k_proj"] = {"bits": 8}
+ layer_config["model.decoder.layers.6.self_attn.out_proj"] = {"bits": 2, "group_size": 32}
+ bits, group_size, sym = 4, 128, True
+ autoround = AutoRound(
+ model,
+ tokenizer,
+ bits=bits,
+ group_size=group_size,
+ iters=1,
+ nsamples=1,
+ sym=sym,
+ layer_config=layer_config
+ )
+ quantized_model_path = self.save_folder
+ autoround.quantize_and_save(output_dir=quantized_model_path, format="auto_round")
+
+ model = AutoModelForCausalLM.from_pretrained(
+ self.save_folder,
+ torch_dtype=torch.float16,
+ device_map="cpu",
+ )
+ tokenizer = AutoTokenizer.from_pretrained(self.save_folder)
+ text = "There is a girl who likes adventure,"
+ inputs = tokenizer(text, return_tensors="pt").to(model.device)
+ res = tokenizer.decode(model.generate(**inputs, max_new_tokens=50)[0])
+ print(res)
+ assert ("!!!" not in res)
+ shutil.rmtree(self.save_folder, ignore_errors=True)
+
diff --git a/test_cuda/test_auto_round_format.py b/test_cuda/test_auto_round_format.py
index e2f0929d9..6f8861b22 100644
--- a/test_cuda/test_auto_round_format.py
+++ b/test_cuda/test_auto_round_format.py
@@ -26,8 +26,7 @@ class LLMDataLoader:
class TestAutoRound(unittest.TestCase):
@classmethod
def setUpClass(self):
- self.model_name = "/models/opt-125m"
- ##self.model_name = "/data5/wenhuach/Meta-Llama-3.1-8B-Instruct-int4-sym-inc"
+ self.model_name = "facebook/opt-125m"
self.llm_dataloader = LLMDataLoader()
self.save_folder = "./saved"
@@ -76,196 +75,40 @@ class TestAutoRound(unittest.TestCase):
shutil.rmtree(self.save_folder, ignore_errors=True)
shutil.rmtree("runs", ignore_errors=True)
- def test_device_map(self):
- from transformers import AutoModelForCausalLM, AutoTokenizer
- model_name = "/data5/wenhuach/Meta-Llama-3.1-8B-Instruct-int4-sym-inc"
-
- device_map = {}
- for i in range(0, 32):
- key = f"model.layers.{str(i)}"
- device_map[key] = "cuda"
- device_map["model.layers.1"] = "cpu"
- device_map["model.layers.2"] = "cpu"
- device_map["model.layers.3"] = "cpu"
- device_map["model.layers.4"] = "cpu"
- device_map["model.layers.5"] = "cpu"
- device_map["model.layers.6"] = "cpu"
- device_map["model.layers.27"] = "cpu"
- device_map["model.layers.28"] = "cpu"
- device_map["model.layers.30"] = "cpu"
- device_map["model.layers.31"] = "cpu"
- device_map["lm_head"] = "cuda"
- device_map["model.norm"] = "cuda"
- device_map["model.rotary_emb"] = "cuda"
- device_map["model.embed_tokens"] = "cuda"
-
- device_map1 = {}
- for i in range(0, 32):
- key = f"model.layers.{str(i)}"
- device_map1[key] = "cuda"
- device_map1["model.layers.1"] = "cuda:1"
- device_map1["model.layers.2"] = "cuda:1"
- device_map1["model.layers.3"] = "cuda:1"
- device_map1["model.layers.4"] = "cuda:1"
- device_map1["model.layers.5"] = "cuda:1"
- device_map1["model.layers.6"] = "cuda:1"
- device_map1["model.layers.27"] = "cuda:1"
- device_map1["model.layers.28"] = "cuda:1"
- device_map1["model.layers.30"] = "cuda:1"
- device_map1["model.layers.31"] = "cuda:1"
- device_map1["lm_head"] = "cuda:1"
- device_map1["model.norm"] = "cuda"
- device_map1["model.rotary_emb"] = "cuda"
- device_map1["model.embed_tokens"] = "cuda"
-
- for tmp_device_map in [device_map1, device_map, None, 0, "balanced", "balanced_low_0", "sequential", "cpu",
- "cuda:0", "cuda", "auto",
- ]:
- model = AutoModelForCausalLM.from_pretrained(
- model_name,
- torch_dtype="auto",
- device_map=tmp_device_map
- )
-
- tokenizer = AutoTokenizer.from_pretrained(model_name)
-
- prompts = [
- "Hello,my name is",
- # "The president of the United States is",
- # "The capital of France is",
- # "The future of AI is",
- ]
-
- texts = []
- for prompt in prompts:
- messages = [
- {"role": "user", "content": prompt}
- ]
- text = tokenizer.apply_chat_template(
- messages,
- tokenize=False,
- add_generation_prompt=True
- )
- texts.append(text)
-
- inputs = tokenizer(texts, return_tensors="pt", padding=False, truncation=True)
-
- outputs = model.generate(
- input_ids=inputs["input_ids"].to(model.device),
- attention_mask=inputs["attention_mask"].to(model.device),
- do_sample=False, ## change this to follow official usage
- max_new_tokens=5
- )
- generated_ids = [
- output_ids[len(input_ids):] for input_ids, output_ids in zip(inputs["input_ids"], outputs)
- ]
-
- decoded_outputs = tokenizer.batch_decode(generated_ids, skip_special_tokens=True)
-
- for i, prompt in enumerate(prompts):
- print(f"Prompt: {prompt}")
- print(f"Generated: {decoded_outputs[i]}")
- print("-" * 50)
- model = None
- del model
- torch.cuda.empty_cache()
-
- def test_device_map_for_triton(self):
- from transformers import AutoModelForCausalLM, AutoTokenizer
- model_name = "/data5/wenhuach/Meta-Llama-3.1-8B-Instruct-int4-sym-inc"
-
- device_map = {}
- for i in range(0, 32):
- key = f"model.layers.{str(i)}"
- device_map[key] = "cuda"
- device_map["model.layers.1"] = "cpu"
- device_map["model.layers.2"] = "cpu"
- device_map["model.layers.3"] = "cpu"
- device_map["model.layers.4"] = "cpu"
- device_map["model.layers.5"] = "cpu"
- device_map["model.layers.6"] = "cpu"
- device_map["model.layers.27"] = "cpu"
- device_map["model.layers.28"] = "cpu"
- device_map["model.layers.30"] = "cpu"
- device_map["model.layers.31"] = "cpu"
- device_map["lm_head"] = "cuda"
- device_map["model.norm"] = "cuda"
- device_map["model.rotary_emb"] = "cuda"
- device_map["model.embed_tokens"] = "cuda"
-
- device_map1 = {}
- for i in range(0, 32):
- key = f"model.layers.{str(i)}"
- device_map1[key] = "cuda"
- device_map1["model.layers.1"] = "cuda:1"
- device_map1["model.layers.2"] = "cuda:1"
- device_map1["model.layers.3"] = "cuda:1"
- device_map1["model.layers.4"] = "cuda:1"
- device_map1["model.layers.5"] = "cuda:1"
- device_map1["model.layers.6"] = "cuda:1"
- device_map1["model.layers.27"] = "cuda:1"
- device_map1["model.layers.28"] = "cuda:1"
- device_map1["model.layers.30"] = "cuda:1"
- device_map1["model.layers.31"] = "cuda:1"
- device_map1["lm_head"] = "cuda:1"
- device_map1["model.norm"] = "cuda"
- device_map1["model.rotary_emb"] = "cuda"
- device_map1["model.embed_tokens"] = "cuda"
- from auto_round import AutoRoundConfig
- quantization_config = AutoRoundConfig(backend="tritonv2")
-
- for tmp_device_map in [device_map1, 0, "balanced", "balanced_low_0", "sequential",
- "cuda:0", "cuda", "auto",
- ]:
- model = AutoModelForCausalLM.from_pretrained(
- model_name,
- torch_dtype="auto",
- device_map=tmp_device_map,
- quantization_config=quantization_config
- )
+ def test_mixed_precision(self):
+ model = AutoModelForCausalLM.from_pretrained(self.model_name, torch_dtype="auto", trust_remote_code=True)
+ tokenizer = AutoTokenizer.from_pretrained(self.model_name, trust_remote_code=True)
+ layer_config = {}
- tokenizer = AutoTokenizer.from_pretrained(model_name)
-
- prompts = [
- "Hello,my name is",
- # "The president of the United States is",
- # "The capital of France is",
- # "The future of AI is",
- ]
-
- texts = []
- for prompt in prompts:
- messages = [
- {"role": "user", "content": prompt}
- ]
- text = tokenizer.apply_chat_template(
- messages,
- tokenize=False,
- add_generation_prompt=True
- )
- texts.append(text)
-
- inputs = tokenizer(texts, return_tensors="pt", padding=False, truncation=True)
-
- outputs = model.generate(
- input_ids=inputs["input_ids"].to(model.device),
- attention_mask=inputs["attention_mask"].to(model.device),
- do_sample=False, ## change this to follow official usage
- max_new_tokens=5
- )
- generated_ids = [
- output_ids[len(input_ids):] for input_ids, output_ids in zip(inputs["input_ids"], outputs)
- ]
+ layer_config["model.decoder.layers.0.self_attn.k_proj"] = {"bits": 8}
+ layer_config["model.decoder.layers.2.self_attn.q_proj"] = {"bits": 3,
+ "group_size": 64} ## 3bits when using asym will have some issue
+ layer_config["model.decoder.layers.6.self_attn.out_proj"] = {"bits": 2, "group_size": 32}
+ bits, group_size, sym = 4, 128, True
+ autoround = AutoRound(
+ model,
+ tokenizer,
+ bits=bits,
+ group_size=group_size,
+ sym=sym,
+ layer_config=layer_config
+ )
+ quantized_model_path = self.save_folder
+ autoround.quantize_and_save(output_dir=quantized_model_path, format="auto_round")
+ quantization_config = AutoRoundConfig(backend="auto")
+ model = AutoModelForCausalLM.from_pretrained(
+ self.save_folder,
+ torch_dtype=torch.float16,
+ device_map="auto",
+ quantization_config=quantization_config
+ )
- decoded_outputs = tokenizer.batch_decode(generated_ids, skip_special_tokens=True)
+ tokenizer = AutoTokenizer.from_pretrained(self.save_folder)
+ self.model_infer(model, tokenizer)
+ result = simple_evaluate_user_model(model, tokenizer, batch_size=16, tasks="lambada_openai")
+ print(result['results']['lambada_openai']['acc,none'])
+ self.assertGreater(result['results']['lambada_openai']['acc,none'], 0.32)
- for i, prompt in enumerate(prompts):
- print(f"Prompt: {prompt}")
- print(f"Generated: {decoded_outputs[i]}")
- print("-" * 50)
- model = None
- del model
- torch.cuda.empty_cache()
def test_awq_backend(self):
model = AutoModelForCausalLM.from_pretrained(self.model_name, torch_dtype="auto", trust_remote_code=True)
@@ -307,6 +150,7 @@ class TestAutoRound(unittest.TestCase):
tokenizer = AutoTokenizer.from_pretrained(self.save_folder)
self.model_infer(model, tokenizer)
+ shutil.rmtree(self.save_folder, ignore_errors=True)
def test_tritonv2_bf16(self):
model_name = "/data5/wenhuach/Meta-Llama-3.1-8B-Instruct-int4-sym-inc"
@@ -409,11 +253,11 @@ class TestAutoRound(unittest.TestCase):
shutil.rmtree("./saved", ignore_errors=True)
- def test_autoround_asym(self):
+ def test_autoround_sym(self):
for bits in [2, 3, 4, 8]:
model = AutoModelForCausalLM.from_pretrained(self.model_name, torch_dtype="auto", trust_remote_code=True)
tokenizer = AutoTokenizer.from_pretrained(self.model_name, trust_remote_code=True)
- bits, group_size, sym = bits, 128, False
+ bits, group_size, sym = bits, 128, True
autoround = AutoRound(
model,
tokenizer,
@@ -436,12 +280,22 @@ class TestAutoRound(unittest.TestCase):
res = tokenizer.decode(model.generate(**inputs, max_new_tokens=50)[0])
print(res)
assert ("!!!" not in res)
+ shutil.rmtree(self.save_folder, ignore_errors=True)
- def test_autoround_sym(self):
- for bits in [2, 3, 4, 8]:
+ def test_load_gptq_model_3bits(self):
+ model_name = "LucasSantiago257/gemma-2b-2bits-gptq"
+ quantization_config = AutoRoundConfig()
+ model = AutoModelForCausalLM.from_pretrained(model_name, torch_dtype="auto", trust_remote_code=True,
+ device_map="auto",
+ quantization_config=quantization_config)
+ tokenizer = AutoTokenizer.from_pretrained(model_name, trust_remote_code=True)
+ self.model_infer(model, tokenizer)
+
+ def test_autoround_asym(self):
+ for bits in [2,3,4,8]:
model = AutoModelForCausalLM.from_pretrained(self.model_name, torch_dtype="auto", trust_remote_code=True)
tokenizer = AutoTokenizer.from_pretrained(self.model_name, trust_remote_code=True)
- bits, group_size, sym = bits, 128, True
+ bits, group_size, sym = bits, 128, False
autoround = AutoRound(
model,
tokenizer,
@@ -452,11 +306,11 @@ class TestAutoRound(unittest.TestCase):
seqlen=2,
dataset=self.llm_dataloader,
)
- quantized_model_path = "./saved"
+ quantized_model_path = self.save_folder
autoround.quantize_and_save(output_dir=quantized_model_path, format="auto_round")
- model = AutoModelForCausalLM.from_pretrained(quantized_model_path, device_map="auto",
+ model = AutoModelForCausalLM.from_pretrained(quantized_model_path, device_map="cuda:0",
trust_remote_code=True)
tokenizer = AutoTokenizer.from_pretrained(quantized_model_path)
text = "There is a girl who likes adventure,"
@@ -464,15 +318,7 @@ class TestAutoRound(unittest.TestCase):
res = tokenizer.decode(model.generate(**inputs, max_new_tokens=50)[0])
print(res)
assert ("!!!" not in res)
-
- def test_load_gptq_model_3bits(self):
- model_name = "LucasSantiago257/gemma-2b-2bits-gptq"
- quantization_config = AutoRoundConfig()
- model = AutoModelForCausalLM.from_pretrained(model_name, torch_dtype="auto", trust_remote_code=True,
- device_map="auto",
- quantization_config=quantization_config)
- tokenizer = AutoTokenizer.from_pretrained(model_name, trust_remote_code=True)
- self.model_infer(model, tokenizer)
+ shutil.rmtree(self.save_folder, ignore_errors=True)
if __name__ == "__main__":
diff --git a/test_cuda/test_multiple_card.py b/test_cuda/test_multiple_card.py
new file mode 100644
index 000000000..83d153122
--- /dev/null
+++ b/test_cuda/test_multiple_card.py
@@ -0,0 +1,292 @@
+import shutil
+import sys
+import unittest
+from auto_round.eval.evaluation import simple_evaluate
+from lm_eval.utils import make_table # pylint: disable=E0401
+
+sys.path.insert(0, "..")
+from auto_round import AutoRound
+
+import torch
+from transformers import AutoModelForCausalLM, AutoTokenizer
+import re
+
+
+def get_accuracy(data):
+ match = re.search(r'\|acc\s+\|[↑↓]\s+\|\s+([\d.]+)\|', data)
+
+ if match:
+ accuracy = float(match.group(1))
+ return accuracy
+ else:
+ return 0.0
+
+
+# import os
+# os.environ["CUDA_VISIBLE_DEVICES"] = "0,1"
+class TestAutoRound(unittest.TestCase):
+ @classmethod
+ def setUpClass(self):
+ self.save_dir = "./saved"
+ self.tasks = "lambada_openai"
+
+ @classmethod
+ def tearDownClass(self):
+ shutil.rmtree(self.save_dir, ignore_errors=True)
+ shutil.rmtree("runs", ignore_errors=True)
+
+ def test_device_map(self):
+ model_name = "/models/Qwen2-0.5B-Instruct"
+ model = AutoModelForCausalLM.from_pretrained(model_name, torch_dtype=torch.float16)
+ tokenizer = AutoTokenizer.from_pretrained(model_name)
+ device_map = {".*q_proj": '0', ".*k_proj": "cuda:1", "v_proj": 1, ".*up_proj": "cpu"}
+ autoround = AutoRound(model, tokenizer, iters=2, device_map=device_map, nsamples=7,seqlen=32)
+ autoround.quantize()
+
+ def test_device_map_str(self):
+ model_name = "/models/Qwen2-0.5B-Instruct"
+ model = AutoModelForCausalLM.from_pretrained(model_name, torch_dtype=torch.float16)
+ tokenizer = AutoTokenizer.from_pretrained(model_name)
+ device_map = '.*q_proj:0,.*k_proj:cuda:0,v_proj:1,.*up_proj:1'
+ autoround = AutoRound(model, tokenizer,device_map=device_map)
+ autoround.quantize()
+ autoround.save_quantized(self.save_dir, format="auto_round", inplace=False)
+ model_args = f"pretrained={self.save_dir}"
+ res = simple_evaluate(model="hf", model_args=model_args,
+ tasks=self.tasks,
+ batch_size="auto")
+ res = make_table(res)
+ accuracy = get_accuracy(res)
+ print(accuracy)
+ assert accuracy > 0.45 ##0.4786
+ shutil.rmtree("./saved", ignore_errors=True)
+
+ def test_layer_norm(self):
+ model_name = "/models/opt-125m"
+ model = AutoModelForCausalLM.from_pretrained(model_name, torch_dtype=torch.float16)
+ tokenizer = AutoTokenizer.from_pretrained(model_name)
+ device_map = {"norm": "cuda:1"}
+ autoround = AutoRound(model, tokenizer, iters=2, device_map=device_map, nsamples=7, seqlen=32,
+ enable_norm_bias_tuning=True)
+ autoround.quantize()
+
+
+ def test_rms_norm(self):
+ model_name = "/models/Qwen2-0.5B-Instruct"
+ model = AutoModelForCausalLM.from_pretrained(model_name, torch_dtype=torch.float16)
+ tokenizer = AutoTokenizer.from_pretrained(model_name)
+ device_map = {"norm": "cuda:1"}
+ autoround = AutoRound(model, tokenizer, iters=2, device_map=device_map, nsamples=7, seqlen=32,
+ enable_norm_bias_tuning=True)
+ autoround.quantize()
+
+ def test_act_quantization(self):
+ model_name = "/models/Qwen2-0.5B-Instruct"
+ model = AutoModelForCausalLM.from_pretrained(model_name, torch_dtype=torch.float16)
+ tokenizer = AutoTokenizer.from_pretrained(model_name)
+ device_map = {".*q_proj": '0', ".*k_proj": "cuda:1", "v_proj": 1, ".*up_proj": "1"}
+ autoround = AutoRound(model, tokenizer, iters=2, device_map=device_map, nsamples=7,seqlen=32,act_bits=4,act_dynamic=False)
+ autoround.quantize()
+
+ def test_lm_head(self):
+ model_name = "/models/Qwen2.5-7B-Instruct"
+ model = AutoModelForCausalLM.from_pretrained(model_name, torch_dtype=torch.float16)
+ tokenizer = AutoTokenizer.from_pretrained(model_name)
+ device_map = {".*q_proj": '0', ".*k_proj": "cuda:1", "v_proj": 1, ".*up_proj": "1","lm_head":1}
+ layer_config={"lm_head": {"bits": 4}}
+ autoround = AutoRound(model, tokenizer, iters=2, device_map=device_map, nsamples=7, seqlen=32,
+ enable_norm_bias_tuning=True,layer_config=layer_config)
+ autoround.quantize()
+
+ def test_device_map(self):
+ from transformers import AutoModelForCausalLM, AutoTokenizer
+ model_name = "/data5/wenhuach/Meta-Llama-3.1-8B-Instruct-int4-sym-inc"
+
+ device_map = {}
+ for i in range(0, 32):
+ key = f"model.layers.{str(i)}"
+ device_map[key] = "cuda"
+ device_map["model.layers.1"] = "cpu"
+ device_map["model.layers.2"] = "cpu"
+ device_map["model.layers.3"] = "cpu"
+ device_map["model.layers.4"] = "cpu"
+ device_map["model.layers.5"] = "cpu"
+ device_map["model.layers.6"] = "cpu"
+ device_map["model.layers.27"] = "cpu"
+ device_map["model.layers.28"] = "cpu"
+ device_map["model.layers.30"] = "cpu"
+ device_map["model.layers.31"] = "cpu"
+ device_map["lm_head"] = "cuda"
+ device_map["model.norm"] = "cuda"
+ device_map["model.rotary_emb"] = "cuda"
+ device_map["model.embed_tokens"] = "cuda"
+
+ device_map1 = {}
+ for i in range(0, 32):
+ key = f"model.layers.{str(i)}"
+ device_map1[key] = "cuda"
+ device_map1["model.layers.1"] = "cuda:1"
+ device_map1["model.layers.2"] = "cuda:1"
+ device_map1["model.layers.3"] = "cuda:1"
+ device_map1["model.layers.4"] = "cuda:1"
+ device_map1["model.layers.5"] = "cuda:1"
+ device_map1["model.layers.6"] = "cuda:1"
+ device_map1["model.layers.27"] = "cuda:1"
+ device_map1["model.layers.28"] = "cuda:1"
+ device_map1["model.layers.30"] = "cuda:1"
+ device_map1["model.layers.31"] = "cuda:1"
+ device_map1["lm_head"] = "cuda:1"
+ device_map1["model.norm"] = "cuda"
+ device_map1["model.rotary_emb"] = "cuda"
+ device_map1["model.embed_tokens"] = "cuda"
+
+ for tmp_device_map in [device_map1, device_map, None, 0, "balanced", "balanced_low_0", "sequential", "cpu",
+ "cuda:0", "cuda", "auto",
+ ]:
+ model = AutoModelForCausalLM.from_pretrained(
+ model_name,
+ torch_dtype="auto",
+ device_map=tmp_device_map
+ )
+
+ tokenizer = AutoTokenizer.from_pretrained(model_name)
+
+ prompts = [
+ "Hello,my name is",
+ # "The president of the United States is",
+ # "The capital of France is",
+ # "The future of AI is",
+ ]
+
+ texts = []
+ for prompt in prompts:
+ messages = [
+ {"role": "user", "content": prompt}
+ ]
+ text = tokenizer.apply_chat_template(
+ messages,
+ tokenize=False,
+ add_generation_prompt=True
+ )
+ texts.append(text)
+
+ inputs = tokenizer(texts, return_tensors="pt", padding=False, truncation=True)
+
+ outputs = model.generate(
+ input_ids=inputs["input_ids"].to(model.device),
+ attention_mask=inputs["attention_mask"].to(model.device),
+ do_sample=False, ## change this to follow official usage
+ max_new_tokens=5
+ )
+ generated_ids = [
+ output_ids[len(input_ids):] for input_ids, output_ids in zip(inputs["input_ids"], outputs)
+ ]
+
+ decoded_outputs = tokenizer.batch_decode(generated_ids, skip_special_tokens=True)
+
+ for i, prompt in enumerate(prompts):
+ print(f"Prompt: {prompt}")
+ print(f"Generated: {decoded_outputs[i]}")
+ print("-" * 50)
+ model = None
+ del model
+ torch.cuda.empty_cache()
+
+ def test_device_map_for_triton(self):
+ from transformers import AutoModelForCausalLM, AutoTokenizer
+ model_name = "/data5/wenhuach/Meta-Llama-3.1-8B-Instruct-int4-sym-inc"
+
+ device_map = {}
+ for i in range(0, 32):
+ key = f"model.layers.{str(i)}"
+ device_map[key] = "cuda"
+ device_map["model.layers.1"] = "cpu"
+ device_map["model.layers.2"] = "cpu"
+ device_map["model.layers.3"] = "cpu"
+ device_map["model.layers.4"] = "cpu"
+ device_map["model.layers.5"] = "cpu"
+ device_map["model.layers.6"] = "cpu"
+ device_map["model.layers.27"] = "cpu"
+ device_map["model.layers.28"] = "cpu"
+ device_map["model.layers.30"] = "cpu"
+ device_map["model.layers.31"] = "cpu"
+ device_map["lm_head"] = "cuda"
+ device_map["model.norm"] = "cuda"
+ device_map["model.rotary_emb"] = "cuda"
+ device_map["model.embed_tokens"] = "cuda"
+
+ device_map1 = {}
+ for i in range(0, 32):
+ key = f"model.layers.{str(i)}"
+ device_map1[key] = "cuda"
+ device_map1["model.layers.1"] = "cuda:1"
+ device_map1["model.layers.2"] = "cuda:1"
+ device_map1["model.layers.3"] = "cuda:1"
+ device_map1["model.layers.4"] = "cuda:1"
+ device_map1["model.layers.5"] = "cuda:1"
+ device_map1["model.layers.6"] = "cuda:1"
+ device_map1["model.layers.27"] = "cuda:1"
+ device_map1["model.layers.28"] = "cuda:1"
+ device_map1["model.layers.30"] = "cuda:1"
+ device_map1["model.layers.31"] = "cuda:1"
+ device_map1["lm_head"] = "cuda:1"
+ device_map1["model.norm"] = "cuda"
+ device_map1["model.rotary_emb"] = "cuda"
+ device_map1["model.embed_tokens"] = "cuda"
+ from auto_round import AutoRoundConfig
+ quantization_config = AutoRoundConfig(backend="tritonv2")
+
+ for tmp_device_map in [device_map1, 0, "balanced", "balanced_low_0", "sequential",
+ "cuda:0", "cuda", "auto",
+ ]:
+ model = AutoModelForCausalLM.from_pretrained(
+ model_name,
+ torch_dtype="auto",
+ device_map=tmp_device_map,
+ quantization_config=quantization_config
+ )
+
+ tokenizer = AutoTokenizer.from_pretrained(model_name)
+
+ prompts = [
+ "Hello,my name is",
+ # "The president of the United States is",
+ # "The capital of France is",
+ # "The future of AI is",
+ ]
+
+ texts = []
+ for prompt in prompts:
+ messages = [
+ {"role": "user", "content": prompt}
+ ]
+ text = tokenizer.apply_chat_template(
+ messages,
+ tokenize=False,
+ add_generation_prompt=True
+ )
+ texts.append(text)
+
+ inputs = tokenizer(texts, return_tensors="pt", padding=False, truncation=True)
+
+ outputs = model.generate(
+ input_ids=inputs["input_ids"].to(model.device),
+ attention_mask=inputs["attention_mask"].to(model.device),
+ do_sample=False, ## change this to follow official usage
+ max_new_tokens=5
+ )
+ generated_ids = [
+ output_ids[len(input_ids):] for input_ids, output_ids in zip(inputs["input_ids"], outputs)
+ ]
+
+ decoded_outputs = tokenizer.batch_decode(generated_ids, skip_special_tokens=True)
+
+ for i, prompt in enumerate(prompts):
+ print(f"Prompt: {prompt}")
+ print(f"Generated: {decoded_outputs[i]}")
+ print("-" * 50)
+ model = None
+ del model
+ torch.cuda.empty_cache()
+
+
diff --git a/test_cuda/test_multiple_card_tuning.py b/test_cuda/test_multiple_card_tuning.py
deleted file mode 100644
index 67254cd71..000000000
--- a/test_cuda/test_multiple_card_tuning.py
+++ /dev/null
@@ -1,101 +0,0 @@
-import shutil
-import sys
-import unittest
-from auto_round.eval.evaluation import simple_evaluate
-from lm_eval.utils import make_table # pylint: disable=E0401
-
-sys.path.insert(0, "..")
-from auto_round import AutoRound
-
-import torch
-from transformers import AutoModelForCausalLM, AutoTokenizer
-import re
-
-
-def get_accuracy(data):
- match = re.search(r'\|acc\s+\|[↑↓]\s+\|\s+([\d.]+)\|', data)
-
- if match:
- accuracy = float(match.group(1))
- return accuracy
- else:
- return 0.0
-
-
-# import os
-# os.environ["CUDA_VISIBLE_DEVICES"] = "0,1"
-class TestAutoRound(unittest.TestCase):
- @classmethod
- def setUpClass(self):
- self.save_dir = "./saved"
- self.tasks = "lambada_openai"
-
- @classmethod
- def tearDownClass(self):
- shutil.rmtree(self.save_dir, ignore_errors=True)
- shutil.rmtree("runs", ignore_errors=True)
-
- def test_device_map(self):
- model_name = "/models/Qwen2-0.5B-Instruct"
- model = AutoModelForCausalLM.from_pretrained(model_name, torch_dtype=torch.float16)
- tokenizer = AutoTokenizer.from_pretrained(model_name)
- device_map = {".*q_proj": '0', ".*k_proj": "cuda:1", "v_proj": 1, ".*up_proj": "cpu"}
- autoround = AutoRound(model, tokenizer, iters=2, device_map=device_map, nsamples=7,seqlen=32)
- autoround.quantize()
-
- def test_device_map_str(self):
- model_name = "/models/Qwen2-0.5B-Instruct"
- model = AutoModelForCausalLM.from_pretrained(model_name, torch_dtype=torch.float16)
- tokenizer = AutoTokenizer.from_pretrained(model_name)
- device_map = '.*q_proj:0,.*k_proj:cuda:0,v_proj:1,.*up_proj:1'
- autoround = AutoRound(model, tokenizer,device_map=device_map)
- autoround.quantize()
- autoround.save_quantized(self.save_dir, format="auto_round", inplace=False)
- model_args = f"pretrained={self.save_dir}"
- res = simple_evaluate(model="hf", model_args=model_args,
- tasks=self.tasks,
- batch_size="auto")
- res = make_table(res)
- accuracy = get_accuracy(res)
- print(accuracy)
- assert accuracy > 0.45 ##0.4786
- shutil.rmtree("./saved", ignore_errors=True)
-
- def test_layer_norm(self):
- model_name = "/models/opt-125m"
- model = AutoModelForCausalLM.from_pretrained(model_name, torch_dtype=torch.float16)
- tokenizer = AutoTokenizer.from_pretrained(model_name)
- device_map = {"norm": "cuda:1"}
- autoround = AutoRound(model, tokenizer, iters=2, device_map=device_map, nsamples=7, seqlen=32,
- enable_norm_bias_tuning=True)
- autoround.quantize()
-
-
- def test_rms_norm(self):
- model_name = "/models/Qwen2-0.5B-Instruct"
- model = AutoModelForCausalLM.from_pretrained(model_name, torch_dtype=torch.float16)
- tokenizer = AutoTokenizer.from_pretrained(model_name)
- device_map = {"norm": "cuda:1"}
- autoround = AutoRound(model, tokenizer, iters=2, device_map=device_map, nsamples=7, seqlen=32,
- enable_norm_bias_tuning=True)
- autoround.quantize()
-
- def test_act_quantization(self):
- model_name = "/models/Qwen2-0.5B-Instruct"
- model = AutoModelForCausalLM.from_pretrained(model_name, torch_dtype=torch.float16)
- tokenizer = AutoTokenizer.from_pretrained(model_name)
- device_map = {".*q_proj": '0', ".*k_proj": "cuda:1", "v_proj": 1, ".*up_proj": "1"}
- autoround = AutoRound(model, tokenizer, iters=2, device_map=device_map, nsamples=7,seqlen=32,act_bits=4,act_dynamic=False)
- autoround.quantize()
-
- def test_lm_head(self):
- model_name = "/models/Qwen2.5-7B-Instruct"
- model = AutoModelForCausalLM.from_pretrained(model_name, torch_dtype=torch.float16)
- tokenizer = AutoTokenizer.from_pretrained(model_name)
- device_map = {".*q_proj": '0', ".*k_proj": "cuda:1", "v_proj": 1, ".*up_proj": "1","lm_head":1}
- layer_config={"lm_head": {"bits": 4}}
- autoround = AutoRound(model, tokenizer, iters=2, device_map=device_map, nsamples=7, seqlen=32,
- enable_norm_bias_tuning=True,layer_config=layer_config)
- autoround.quantize()
-
-