Skip to content

PTQ-Level Efficiency, QAT-Level Accuracy | 训练后量化的效率,量化感知训练的精度

support rtn via iters==0 (#510)

Wenhua Cheng58ca5837c6ff
diff --git a/auto_round/autoround.py b/auto_round/autoround.py
index 711ba3bb4..5a4e64689 100644
--- a/auto_round/autoround.py
+++ b/auto_round/autoround.py
@@ -54,7 +54,7 @@ from .utils import (
find_matching_blocks, is_debug_mode,
TORCH_VERSION_AT_LEAST_2_6,
supported_layer_types,
- get_layer_features,
+ get_layer_features, set_module,
)
from .low_cpu_mem.utils import get_layers_before_block
@@ -192,10 +192,13 @@ class AutoRound(object):
self.nblocks = nblocks
self.dataset = dataset
self.iters = iters
- if self.iters <= 0:
- logger.warning("iters must be positive, reset it to 200")
+ if self.iters < 0:
+ logger.warning("`iters` must be non-negative, reset it to 200")
self.iters = 200
- self.lr = lr or (1.0 / self.iters) ##must after iter setting
+ if self.iters == 0:
+ self.lr = 5e-3
+ else:
+ self.lr = lr or (1.0 / self.iters) ##must after iter setting
self.minmax_lr = minmax_lr or self.lr
self.data_type = data_type
@@ -360,7 +363,7 @@ class AutoRound(object):
self.super_bits = gguf_config["super_bits"] if self.super_bits is None else self.super_bits
self.super_group_size = gguf_config["super_group_size"] \
if self.super_group_size is None else self.super_group_size
-
+
def check_configs(self):
"""Checks if the configurations are valid.
@@ -375,7 +378,7 @@ class AutoRound(object):
assert self.act_group_size == -1 or self.act_group_size >= 1, \
"only supports positive group_size or -1(per channel)"
assert self.batch_size > 0, "batch size must be positive"
- assert self.iters > 0, "iters must be positive"
+ assert self.iters >= 0, "iters must be non-negative"
assert self.seqlen > 0, "seqlen must be positive"
assert self.nblocks > 0, "nblocks must be positive"
assert self.gradient_accumulate_steps > 0, "gradient accumulate step must be positive"
@@ -475,7 +478,7 @@ class AutoRound(object):
for index in range(len(formats)):
format = formats[index]
if "auto_round" in format:
- if (self.sym and ("gptq" not in format and "awq" not in format)) or self.bits==3:
+ if (self.sym and ("gptq" not in format and "awq" not in format)) or self.bits == 3:
format = format.replace('auto_round', 'auto_round:auto_gptq')
formats[index] = format
@@ -506,11 +509,40 @@ class AutoRound(object):
save_format_ = format.replace(":", "-").replace("_", "-")
save_folder = os.path.join(output_dir, save_format_) if len(formats) > 1 else output_dir
self.save_quantized(save_folder, format=format, inplace=inplace, **kwargs)
-
+
folders.append(save_folder)
return model, folders
+ @torch.inference_mode
+ def quantize_rtn(self):
+ if self.amp:
+ self.model.to(self.amp_dtype)
+ self.model.to("cpu")
+ all_to_quantized_module_names = []
+ for n, m in self.model.named_modules():
+ if check_to_quantized(m):
+ all_to_quantized_module_names.append(n)
+ pbar = tqdm(all_to_quantized_module_names)
+
+ for name in pbar:
+ pbar.set_description(f"Quantizing {name}")
+ m = get_module(self.model, name)
+
+ m.to(self.device)
+ m = WrapperLinear(m, enable_minmax_tuning=False, enable_norm_bias_tuning=False, enable_round_tuning=False)
+ m = m.unwrapper({})
+ m.to("cpu")
+ if self.is_packing_immediate:
+ from auto_round.export import PACKING_LAYER_WITH_FORMAT
+ if check_to_quantized(m):
+ target_backend = self.formats[0].split(":")[0] if ":" in self.formats[0] else self.formats[0]
+ PACKING_LAYER_WITH_FORMAT[target_backend](n, self.model, self.formats[0])
+ else:
+ set_module(self.model, name, m)
+ self.quantized = True
+ return self.model, self.layer_config
+
def quantize(self):
"""Quantize the model and return the quantized model along with layer configurations.
the entry of AutoRound.
@@ -518,6 +550,8 @@ class AutoRound(object):
Returns:
The quantized model and layer configurations.
"""
+ if self.iters == 0:
+ return self.quantize_rtn()
if bool(self.quant_block_list):
all_blocks = self.quant_block_list
@@ -585,7 +619,7 @@ class AutoRound(object):
quantized_layers.append(n)
else:
unquantized_layers.append(n)
- elif hasattr(m, "scales") or hasattr(m, "scale"): ##packing_immediately
+ elif hasattr(m, "scales") or hasattr(m, "scale"): ##packing_immediately
quantized_layers.append(n)
summary_info = (
f"Summary: quantized {len(quantized_layers)}/{len(quantized_layers) + len(unquantized_layers)} in the model"
diff --git a/auto_round/inference/backend.py b/auto_round/inference/backend.py
index 479858f5a..1642de32a 100644
--- a/auto_round/inference/backend.py
+++ b/auto_round/inference/backend.py
@@ -113,7 +113,7 @@ BackendInfos['auto_gptq:tritonv2'] = BackendInfo(device=["cuda"], sym=[True, Fal
BackendInfos['auto_gptq:cuda'] = BackendInfo(device=["cuda"], sym=[True, False],
packing_format="triton_zp",
bits=[2, 3, 4, 8], group_size=None,
- priority=1, feature_checks=[feature_multiply_checker_32],
+ priority=0, feature_checks=[feature_multiply_checker_32],
alias=["auto_gptq:cuda"],
dtype=["float16"],
convertable_format=["triton_zp"],
diff --git a/auto_round/inference/convert_model.py b/auto_round/inference/convert_model.py
index b39a293eb..67a8b7310 100644
--- a/auto_round/inference/convert_model.py
+++ b/auto_round/inference/convert_model.py
@@ -470,7 +470,7 @@ def post_init(model, used_backends):
for l in data_types[1:]:
common &= set(l)
common = list(common)
- if str(model.dtype).split('.')[-1] not in common:
+ if len(common)>0 and str(model.dtype).split('.')[-1] not in common:
if common[0] == "float16":
model = model.to(torch.float16)
logger.warning("force model to float16")
diff --git a/auto_round/utils.py b/auto_round/utils.py
index 9f0d58a95..4a0c82b8e 100644
--- a/auto_round/utils.py
+++ b/auto_round/utils.py
@@ -522,10 +522,10 @@ def check_to_quantized(config):
False otherwise.
"""
if isinstance(config, dict):
- bits = int(config.get("bits", 4))
+ bits = int(config.get("bits", 16))
act_bits = int(config.get("act_bits", 16))
else:
- bits = int(config.bits) if hasattr(config, "bits") else 4
+ bits = int(config.bits) if hasattr(config, "bits") else 16
act_bits = int(config.act_bits) if hasattr(config, "act_bits") else 16
return bits <= 8 or act_bits <= 8
diff --git a/auto_round/wrapper.py b/auto_round/wrapper.py
index 4e1a5d8a7..a624d9f21 100644
--- a/auto_round/wrapper.py
+++ b/auto_round/wrapper.py
@@ -58,7 +58,8 @@ class WrapperLinear(torch.nn.Module):
device (str): Device on which to run computations (e.g., 'cpu' or 'cuda').
"""
- def __init__(self, orig_layer, enable_minmax_tuning=True, enable_norm_bias_tuning=False, device='cpu', **kwargs):
+ def __init__(self, orig_layer, enable_minmax_tuning=True, enable_norm_bias_tuning=False, device='cpu',
+ enable_round_tuning=True, **kwargs):
"""Initializes the WrapperLinear module.
Args:
@@ -72,6 +73,7 @@ class WrapperLinear(torch.nn.Module):
self.output_device = device
self.device = self.orig_layer.tuning_device if hasattr(self.orig_layer, "tuning_device") else device
self.enable_minmax_tuning = enable_minmax_tuning
+ self.enable_round_tuning = enable_round_tuning
self.enable_norm_bias_tuning = enable_norm_bias_tuning and (orig_layer.bias is not None)
self.enable_act_quant = self.orig_layer.act_bits <= 8 or self._check_act_quantization(
self.orig_layer.act_data_type)
@@ -108,7 +110,7 @@ class WrapperLinear(torch.nn.Module):
weight_reshape = reshape_and_pad_tensor(orig_weight.data, orig_layer.group_size)
self.weight_min = torch.clamp(weight_reshape.min(1)[0], max=0)
self.weight_max = torch.clamp(weight_reshape.max(1)[0], min=0)
- self._init_params("value", p_dtype, weight_reshape.shape, 0, True)
+ self._init_params("value", p_dtype, weight_reshape.shape, 0, self.enable_round_tuning)
# Min-max scale initialization
shape = get_scale_shape(orig_weight, orig_layer.group_size)
self._init_params("min_scale", p_dtype, shape, 1.0, self.enable_minmax_tuning)
@@ -166,7 +168,7 @@ class WrapperLinear(torch.nn.Module):
weight = self.orig_layer.get_weight().to(self.device)
if isinstance(self.orig_layer, transformers.modeling_utils.Conv1D):
weight = weight.t()
-
+
quant_kwargs = {}
if hasattr(self.orig_layer, "super_bits"):
quant_kwargs["super_bits"] = self.orig_layer.super_bits
@@ -185,7 +187,7 @@ class WrapperLinear(torch.nn.Module):
data_type=self.data_type,
q_scale_thresh=self.q_scale_thresh,
**quant_kwargs
- )
+ )
weight_q = weight_q.to(weight.dtype)
if isinstance(self.orig_layer, transformers.modeling_utils.Conv1D):
weight_q = weight_q.t()
@@ -257,12 +259,12 @@ class WrapperLinear(torch.nn.Module):
else:
name = "w_" + key
setattr(self.orig_layer, name, attr_dict[key].to("cpu"))
-
+
if isinstance(scale, dict):
_set_dict_attr(scale, "scale")
else:
self.orig_layer.scale = scale.reshape(shape[0], -1).to("cpu")
-
+
if zp is not None:
if isinstance(zp, dict):
_set_dict_attr(zp, "zp")
diff --git a/test/_test_helpers.py b/test/_test_helpers.py
index ac753a2eb..7d848c728 100644
--- a/test/_test_helpers.py
+++ b/test/_test_helpers.py
@@ -7,3 +7,45 @@ def is_pytest_mode_compile():
def is_pytest_mode_lazy():
return pytest.mode == "lazy"
+
+
+def model_infer(model, tokenizer, apply_chat_template=False):
+ prompts = [
+ "Hello,my name is",
+ # "The president of the United States is",
+ # "The capital of France is",
+ # "The future of AI is",
+ ]
+ if apply_chat_template:
+ texts = []
+ for prompt in prompts:
+ messages = [
+ {"role": "user", "content": prompt}
+ ]
+ text = tokenizer.apply_chat_template(
+ messages,
+ tokenize=False,
+ add_generation_prompt=True
+ )
+ texts.append(text)
+ prompts = texts
+
+ inputs = tokenizer(prompts, return_tensors="pt", padding=False, truncation=True)
+
+ outputs = model.generate(
+ input_ids=inputs["input_ids"].to(model.device),
+ attention_mask=inputs["attention_mask"].to(model.device),
+ do_sample=False, ## change this to follow official usage
+ max_new_tokens=5
+ )
+ generated_ids = [
+ output_ids[len(input_ids):] for input_ids, output_ids in zip(inputs["input_ids"], outputs)
+ ]
+
+ decoded_outputs = tokenizer.batch_decode(generated_ids, skip_special_tokens=True)
+
+ for i, prompt in enumerate(prompts):
+ print(f"Prompt: {prompt}")
+ print(f"Generated: {decoded_outputs[i]}")
+ print("-" * 50)
+ return decoded_outputs[0]
diff --git a/test/test_autoround.py b/test/test_autoround.py
index b7afae1db..58105e973 100644
--- a/test/test_autoround.py
+++ b/test/test_autoround.py
@@ -3,13 +3,15 @@ import shutil
import sys
import unittest
+from auto_round.eval.evaluation import simple_evaluate_user_model
+
sys.path.insert(0, "..")
import torch
import transformers
from transformers import AutoModelForCausalLM, AutoTokenizer
from auto_round import AutoRound
-
+from _test_helpers import model_infer
class LLMDataLoader:
def __init__(self):
@@ -27,20 +29,21 @@ class TestAutoRound(unittest.TestCase):
self.model = AutoModelForCausalLM.from_pretrained(model_name, torch_dtype="auto", trust_remote_code=True)
self.tokenizer = AutoTokenizer.from_pretrained(model_name, trust_remote_code=True)
self.llm_dataloader = LLMDataLoader()
+ self.save_folder = "./saved"
@classmethod
def tearDownClass(self):
- shutil.rmtree("./saved", ignore_errors=True)
+ shutil.rmtree(self.save_folder, ignore_errors=True)
shutil.rmtree("runs", ignore_errors=True)
def test_remove_whole_block(self):
- layer_config={"model.decoder.layers.0.self_attn.k_proj": {"bits": 32},
- "model.decoder.layers.0.self_attn.v_proj": {"bits": 32},
- "model.decoder.layers.0.self_attn.q_proj": {"bits": 32},
- "model.decoder.layers.0.self_attn.out_proj": {"bits": 32},
- "model.decoder.layers.0.fc1": {"bits": 32},
- "model.decoder.layers.0.fc2": {"bits": 32},
- }
+ layer_config = {"model.decoder.layers.0.self_attn.k_proj": {"bits": 32},
+ "model.decoder.layers.0.self_attn.v_proj": {"bits": 32},
+ "model.decoder.layers.0.self_attn.q_proj": {"bits": 32},
+ "model.decoder.layers.0.self_attn.out_proj": {"bits": 32},
+ "model.decoder.layers.0.fc1": {"bits": 32},
+ "model.decoder.layers.0.fc2": {"bits": 32},
+ }
bits, group_size, sym = 4, 128, False
autoround = AutoRound(
self.model,
@@ -83,8 +86,6 @@ class TestAutoRound(unittest.TestCase):
)
autoround.quantize()
-
-
def test_mx_fp4(self):
bits, group_size, sym = 4, 32, False
autoround = AutoRound(
@@ -101,7 +102,7 @@ class TestAutoRound(unittest.TestCase):
autoround.quantize()
def test_nsample(self):
- autoround= AutoRound(
+ autoround = AutoRound(
self.model,
self.tokenizer,
bits=4,
@@ -237,6 +238,7 @@ class TestAutoRound(unittest.TestCase):
dataset=self.llm_dataloader,
)
autoround.quantize()
+
#
def test_signround(self):
bits, group_size, sym = 4, -1, False
@@ -290,7 +292,8 @@ class TestAutoRound(unittest.TestCase):
def test_auto_device_map(self):
bits, group_size, sym = 4, 128, False
model_name = "facebook/opt-125m"
- model = AutoModelForCausalLM.from_pretrained(model_name, torch_dtype="auto", trust_remote_code=True, device_map='auto')
+ model = AutoModelForCausalLM.from_pretrained(model_name, torch_dtype="auto", trust_remote_code=True,
+ device_map='auto')
autoround = AutoRound(
model,
self.tokenizer,
@@ -306,7 +309,8 @@ class TestAutoRound(unittest.TestCase):
def test_fp32(self):
bits, group_size, sym = 4, 128, False
model_name = "facebook/opt-125m"
- model = AutoModelForCausalLM.from_pretrained(model_name, torch_dtype=torch.float32, trust_remote_code=True, device_map='auto')
+ model = AutoModelForCausalLM.from_pretrained(model_name, torch_dtype=torch.float32, trust_remote_code=True,
+ device_map='auto')
autoround = AutoRound(
model,
self.tokenizer,
@@ -320,7 +324,6 @@ class TestAutoRound(unittest.TestCase):
)
autoround.quantize()
-
def test_fallback_layers(self):
bits, group_size, sym = 4, 128, True
model_name = "facebook/opt-125m"
@@ -340,7 +343,7 @@ class TestAutoRound(unittest.TestCase):
layer_config=layer_config
)
autoround.quantize()
- quantized_model_path = "./saved"
+ quantized_model_path = self.save_folder
autoround.save_quantized(output_dir=quantized_model_path, format="auto_round", inplace=True)
from auto_round import AutoRoundConfig
@@ -349,15 +352,18 @@ class TestAutoRound(unittest.TestCase):
)
model = AutoModelForCausalLM.from_pretrained(quantized_model_path,
- device_map='cpu',quantization_config=quantization_config)
+ device_map='cpu', quantization_config=quantization_config)
tokenizer = AutoTokenizer.from_pretrained(quantized_model_path)
text = "There is a girl who likes adventure,"
inputs = tokenizer(text, return_tensors="pt").to(model.device)
res = tokenizer.decode(model.generate(**inputs, max_new_tokens=1)[0])
+ shutil.rmtree(self.save_folder, ignore_errors=True)
+
def test_tensor_reshape(self):
model_name = "facebook/opt-125m"
- model = AutoModelForCausalLM.from_pretrained(model_name, torch_dtype=torch.float32, trust_remote_code=True, device_map='auto')
+ model = AutoModelForCausalLM.from_pretrained(model_name, torch_dtype=torch.float32, trust_remote_code=True,
+ device_map='auto')
bits, group_size, sym = 4, 100, False
autoround = AutoRound(
self.model,
@@ -415,13 +421,42 @@ class TestAutoRound(unittest.TestCase):
# Inference: Generation of the output
generated_ids = model.generate(**inputs, max_new_tokens=1)
generated_ids_trimmed = [
- out_ids[len(in_ids) :] for in_ids, out_ids in zip(inputs.input_ids, generated_ids)
+ out_ids[len(in_ids):] for in_ids, out_ids in zip(inputs.input_ids, generated_ids)
]
output_text = processor.batch_decode(
generated_ids_trimmed, skip_special_tokens=True, clean_up_tokenization_spaces=False
)
print(output_text)
+ def test_rtn(self):
+ model_name = "facebook/opt-125m"
+ model = AutoModelForCausalLM.from_pretrained(model_name, torch_dtype="auto", trust_remote_code=True)
+ tokenizer = AutoTokenizer.from_pretrained(model_name, trust_remote_code=True)
+
+ bits, group_size, sym = 4, 128, True
+ autoround = AutoRound(
+ model,
+ tokenizer,
+ bits=bits,
+ group_size=group_size,
+ sym=sym,
+ iters=1,
+ nsamples=1
+
+ )
+ quantized_model_path = self.save_folder
+ autoround.quantize_and_save(output_dir=quantized_model_path, format="auto_round")
+ from auto_round import AutoRoundConfig
+ model = AutoModelForCausalLM.from_pretrained(
+ self.save_folder,
+ torch_dtype=torch.float16,
+ device_map="auto",
+ )
+
+ tokenizer = AutoTokenizer.from_pretrained(self.save_folder)
+ model_infer(model, tokenizer)
+ shutil.rmtree(self.save_folder)
+
if __name__ == "__main__":
unittest.main()
diff --git a/test/test_basic_usage.py b/test/test_basic_usage.py
index e468d173c..695271bec 100644
--- a/test/test_basic_usage.py
+++ b/test/test_basic_usage.py
@@ -30,46 +30,46 @@ class TestAutoRoundCmd(unittest.TestCase):
if res > 0 or res == -1:
assert False, "cmd line test fail, please have a check"
- # res = os.system(
- # f"cd .. && {python_path} -m auto_round --model 'facebook/opt-125m' --seqlen 8 --iter 1 --nsamples 1 --eval_task_by_task --tasks openbookqa --bs 32"
- # )
- # if res > 0 or res == -1:
- # assert False, "cmd line test fail, please have a check"
- #
- # res = os.system(
- # f"cd .. && {python_path} -c 'from auto_round.__main__ import run_light; run_light()' --seqlen 8 --iter 2 --nsamples 8 --output_dir ./saved --tasks lambada_openai")
- # if res > 0 or res == -1:
- # assert False, "cmd line test fail, please have a check"
- #
- # # test mllm script
- # # test auto_round_mllm help
- # res = os.system(
- # f"cd .. && {python_path} -m auto_round --mllm -h")
- # if res > 0 or res == -1:
- # assert False, "cmd line test fail, please have a check"
- #
- # # test auto_round_mllm --eval help
- # res = os.system(
- # f"cd .. && {python_path} -m auto_round --mllm --eval -h")
- # if res > 0 or res == -1:
- # assert False, "cmd line test fail, please have a check"
- #
- # # test auto_round_mllm --lmms help
- # res = os.system(
- # f"cd .. && {python_path} -m auto_round --mllm --lmms -h")
- # if res > 0 or res == -1:
- # assert False, "cmd line test fail, please have a check"
- #
- # res = os.system(
- # f"cd .. && {python_path} -m auto_round --mllm --iter 2 --nsamples 10 --seqlen 32 --format auto_round --output_dir ./saved")
- # if res > 0 or res == -1:
- # assert False, "cmd line test fail, please have a check"
- #
- # res = os.system(
- # f"cd .. && {python_path} -m auto_round --mllm --iter 2 --nsamples 10 --seqlen 256 --format auto_round"
- # " --quant_nontext_module --output_dir ./saved ")
- # if res > 0 or res == -1:
- # assert False, "cmd line test fail, please have a check"
+ res = os.system(
+ f"cd .. && {python_path} -m auto_round --model 'facebook/opt-125m' --seqlen 8 --iter 1 --nsamples 1 --eval_task_by_task --tasks openbookqa --bs 32"
+ )
+ if res > 0 or res == -1:
+ assert False, "cmd line test fail, please have a check"
+
+ res = os.system(
+ f"cd .. && {python_path} -c 'from auto_round.__main__ import run_light; run_light()' --seqlen 8 --iter 2 --nsamples 8 --output_dir ./saved --tasks lambada_openai")
+ if res > 0 or res == -1:
+ assert False, "cmd line test fail, please have a check"
+
+ # test mllm script
+ # test auto_round_mllm help
+ res = os.system(
+ f"cd .. && {python_path} -m auto_round --mllm -h")
+ if res > 0 or res == -1:
+ assert False, "cmd line test fail, please have a check"
+
+ # test auto_round_mllm --eval help
+ res = os.system(
+ f"cd .. && {python_path} -m auto_round --mllm --eval -h")
+ if res > 0 or res == -1:
+ assert False, "cmd line test fail, please have a check"
+
+ # test auto_round_mllm --lmms help
+ res = os.system(
+ f"cd .. && {python_path} -m auto_round --mllm --lmms -h")
+ if res > 0 or res == -1:
+ assert False, "cmd line test fail, please have a check"
+
+ res = os.system(
+ f"cd .. && {python_path} -m auto_round --mllm --iter 2 --nsamples 10 --seqlen 32 --format auto_round --output_dir ./saved")
+ if res > 0 or res == -1:
+ assert False, "cmd line test fail, please have a check"
+
+ res = os.system(
+ f"cd .. && {python_path} -m auto_round --mllm --iter 2 --nsamples 10 --seqlen 256 --format auto_round"
+ " --quant_nontext_module --output_dir ./saved ")
+ if res > 0 or res == -1:
+ assert False, "cmd line test fail, please have a check"
if __name__ == "__main__":
diff --git a/test/test_conv1d.py b/test/test_conv1d.py
index 0d010051d..62b505b48 100644
--- a/test/test_conv1d.py
+++ b/test/test_conv1d.py
@@ -8,6 +8,7 @@ import torch
from transformers import AutoModelForCausalLM, AutoTokenizer
from auto_round import AutoRound
+from _test_helpers import model_infer
class LLMDataLoader:
def __init__(self):
self.batch_size = 1
@@ -33,6 +34,7 @@ class TestQuantizationConv1d(unittest.TestCase):
def test_quant(self):
self.model = AutoModelForCausalLM.from_pretrained(self.model_name, torch_dtype="auto", trust_remote_code=True)
bits, group_size, sym = 4, 128, False
+ from auto_round import AutoRoundConfig
autoround = AutoRound(
self.model,
self.tokenizer,
@@ -46,12 +48,12 @@ class TestQuantizationConv1d(unittest.TestCase):
)
autoround.quantize()
- try:
- import auto_gptq
- except:
- return
autoround.save_quantized("./saved")
+ model = AutoModelForCausalLM.from_pretrained("./saved", device_map="cpu", trust_remote_code=True)
+ model_infer(model, self.tokenizer)
+
+
if __name__ == "__main__":
unittest.main()
diff --git a/test_cuda/_test_helpers.py b/test_cuda/_test_helpers.py
new file mode 100644
index 000000000..3c499c499
--- /dev/null
+++ b/test_cuda/_test_helpers.py
@@ -0,0 +1,41 @@
+
+def model_infer(model, tokenizer, apply_chat_template=False):
+ prompts = [
+ "Hello,my name is",
+ # "The president of the United States is",
+ # "The capital of France is",
+ # "The future of AI is",
+ ]
+ if apply_chat_template:
+ texts = []
+ for prompt in prompts:
+ messages = [
+ {"role": "user", "content": prompt}
+ ]
+ text = tokenizer.apply_chat_template(
+ messages,
+ tokenize=False,
+ add_generation_prompt=True
+ )
+ texts.append(text)
+ prompts = texts
+
+ inputs = tokenizer(prompts, return_tensors="pt", padding=False, truncation=True)
+
+ outputs = model.generate(
+ input_ids=inputs["input_ids"].to(model.device),
+ attention_mask=inputs["attention_mask"].to(model.device),
+ do_sample=False, ## change this to follow official usage
+ max_new_tokens=5
+ )
+ generated_ids = [
+ output_ids[len(input_ids):] for input_ids, output_ids in zip(inputs["input_ids"], outputs)
+ ]
+
+ decoded_outputs = tokenizer.batch_decode(generated_ids, skip_special_tokens=True)
+
+ for i, prompt in enumerate(prompts):
+ print(f"Prompt: {prompt}")
+ print(f"Generated: {decoded_outputs[i]}")
+ print("-" * 50)
+ return decoded_outputs[0]
diff --git a/test_cuda/test_conv1d.py b/test_cuda/test_conv1d.py
new file mode 100644
index 000000000..a407a4aef
--- /dev/null
+++ b/test_cuda/test_conv1d.py
@@ -0,0 +1,59 @@
+import copy
+import shutil
+import sys
+import unittest
+
+sys.path.insert(0, "..")
+import torch
+from transformers import AutoModelForCausalLM, AutoTokenizer
+
+from auto_round import AutoRound
+from _test_helpers import model_infer
+class LLMDataLoader:
+ def __init__(self):
+ self.batch_size = 1
+
+ def __iter__(self):
+ for i in range(2):
+ yield torch.ones([1, 10], dtype=torch.long)
+
+
+class TestQuantizationConv1d(unittest.TestCase):
+ @classmethod
+ def setUpClass(self):
+ self.model_name = "MBZUAI/LaMini-GPT-124M"
+ self.tokenizer = AutoTokenizer.from_pretrained(self.model_name, trust_remote_code=True)
+ self.llm_dataloader = LLMDataLoader()
+
+ @classmethod
+ def tearDownClass(self):
+ shutil.rmtree("./saved", ignore_errors=True)
+ shutil.rmtree("runs", ignore_errors=True)
+
+
+ def test_quant(self):
+ self.model = AutoModelForCausalLM.from_pretrained(self.model_name, torch_dtype="auto", trust_remote_code=True)
+ bits, group_size, sym = 4, 128, False
+ from auto_round import AutoRoundConfig
+ autoround = AutoRound(
+ self.model,
+ self.tokenizer,
+ bits=bits,
+ group_size=group_size,
+ sym=sym,
+ iters=2,
+ seqlen=2,
+ dataset=self.llm_dataloader,
+
+ )
+
+ autoround.quantize()
+ autoround.save_quantized("./saved")
+
+ model = AutoModelForCausalLM.from_pretrained("./saved", device_map="cuda", trust_remote_code=True)
+ model_infer(model, self.tokenizer)
+
+
+
+if __name__ == "__main__":
+ unittest.main()