PTQ-Level Efficiency, QAT-Level Accuracy | 训练后量化的效率,量化感知训练的精度
support rtn via iters==0 (#510)
Wenhua Cheng58ca5837c6ff
diff --git a/auto_round/autoround.py b/auto_round/autoround.pyindex 711ba3bb4..5a4e64689 100644--- a/auto_round/autoround.py+++ b/auto_round/autoround.py@@ -54,7 +54,7 @@ from .utils import (find_matching_blocks, is_debug_mode,TORCH_VERSION_AT_LEAST_2_6,supported_layer_types,- get_layer_features,+ get_layer_features, set_module,)from .low_cpu_mem.utils import get_layers_before_block@@ -192,10 +192,13 @@ class AutoRound(object):self.nblocks = nblocksself.dataset = datasetself.iters = iters- if self.iters <= 0:- logger.warning("iters must be positive, reset it to 200")+ if self.iters < 0:+ logger.warning("`iters` must be non-negative, reset it to 200")self.iters = 200- self.lr = lr or (1.0 / self.iters) ##must after iter setting+ if self.iters == 0:+ self.lr = 5e-3+ else:+ self.lr = lr or (1.0 / self.iters) ##must after iter settingself.minmax_lr = minmax_lr or self.lrself.data_type = data_type@@ -360,7 +363,7 @@ class AutoRound(object):self.super_bits = gguf_config["super_bits"] if self.super_bits is None else self.super_bitsself.super_group_size = gguf_config["super_group_size"] \if self.super_group_size is None else self.super_group_size-+def check_configs(self):"""Checks if the configurations are valid.@@ -375,7 +378,7 @@ class AutoRound(object):assert self.act_group_size == -1 or self.act_group_size >= 1, \"only supports positive group_size or -1(per channel)"assert self.batch_size > 0, "batch size must be positive"- assert self.iters > 0, "iters must be positive"+ assert self.iters >= 0, "iters must be non-negative"assert self.seqlen > 0, "seqlen must be positive"assert self.nblocks > 0, "nblocks must be positive"assert self.gradient_accumulate_steps > 0, "gradient accumulate step must be positive"@@ -475,7 +478,7 @@ class AutoRound(object):for index in range(len(formats)):format = formats[index]if "auto_round" in format:- if (self.sym and ("gptq" not in format and "awq" not in format)) or self.bits==3:+ if (self.sym and ("gptq" not in format and "awq" not in format)) or self.bits == 3:format = format.replace('auto_round', 'auto_round:auto_gptq')formats[index] = format@@ -506,11 +509,40 @@ class AutoRound(object):save_format_ = format.replace(":", "-").replace("_", "-")save_folder = os.path.join(output_dir, save_format_) if len(formats) > 1 else output_dirself.save_quantized(save_folder, format=format, inplace=inplace, **kwargs)-+folders.append(save_folder)return model, folders+ @torch.inference_mode+ def quantize_rtn(self):+ if self.amp:+ self.model.to(self.amp_dtype)+ self.model.to("cpu")+ all_to_quantized_module_names = []+ for n, m in self.model.named_modules():+ if check_to_quantized(m):+ all_to_quantized_module_names.append(n)+ pbar = tqdm(all_to_quantized_module_names)++ for name in pbar:+ pbar.set_description(f"Quantizing {name}")+ m = get_module(self.model, name)++ m.to(self.device)+ m = WrapperLinear(m, enable_minmax_tuning=False, enable_norm_bias_tuning=False, enable_round_tuning=False)+ m = m.unwrapper({})+ m.to("cpu")+ if self.is_packing_immediate:+ from auto_round.export import PACKING_LAYER_WITH_FORMAT+ if check_to_quantized(m):+ target_backend = self.formats[0].split(":")[0] if ":" in self.formats[0] else self.formats[0]+ PACKING_LAYER_WITH_FORMAT[target_backend](n, self.model, self.formats[0])+ else:+ set_module(self.model, name, m)+ self.quantized = True+ return self.model, self.layer_config+def quantize(self):"""Quantize the model and return the quantized model along with layer configurations.the entry of AutoRound.@@ -518,6 +550,8 @@ class AutoRound(object):Returns:The quantized model and layer configurations."""+ if self.iters == 0:+ return self.quantize_rtn()if bool(self.quant_block_list):all_blocks = self.quant_block_list@@ -585,7 +619,7 @@ class AutoRound(object):quantized_layers.append(n)else:unquantized_layers.append(n)- elif hasattr(m, "scales") or hasattr(m, "scale"): ##packing_immediately+ elif hasattr(m, "scales") or hasattr(m, "scale"): ##packing_immediatelyquantized_layers.append(n)summary_info = (f"Summary: quantized {len(quantized_layers)}/{len(quantized_layers) + len(unquantized_layers)} in the model"diff --git a/auto_round/inference/backend.py b/auto_round/inference/backend.pyindex 479858f5a..1642de32a 100644--- a/auto_round/inference/backend.py+++ b/auto_round/inference/backend.py@@ -113,7 +113,7 @@ BackendInfos['auto_gptq:tritonv2'] = BackendInfo(device=["cuda"], sym=[True, FalBackendInfos['auto_gptq:cuda'] = BackendInfo(device=["cuda"], sym=[True, False],packing_format="triton_zp",bits=[2, 3, 4, 8], group_size=None,- priority=1, feature_checks=[feature_multiply_checker_32],+ priority=0, feature_checks=[feature_multiply_checker_32],alias=["auto_gptq:cuda"],dtype=["float16"],convertable_format=["triton_zp"],diff --git a/auto_round/inference/convert_model.py b/auto_round/inference/convert_model.pyindex b39a293eb..67a8b7310 100644--- a/auto_round/inference/convert_model.py+++ b/auto_round/inference/convert_model.py@@ -470,7 +470,7 @@ def post_init(model, used_backends):for l in data_types[1:]:common &= set(l)common = list(common)- if str(model.dtype).split('.')[-1] not in common:+ if len(common)>0 and str(model.dtype).split('.')[-1] not in common:if common[0] == "float16":model = model.to(torch.float16)logger.warning("force model to float16")diff --git a/auto_round/utils.py b/auto_round/utils.pyindex 9f0d58a95..4a0c82b8e 100644--- a/auto_round/utils.py+++ b/auto_round/utils.py@@ -522,10 +522,10 @@ def check_to_quantized(config):False otherwise."""if isinstance(config, dict):- bits = int(config.get("bits", 4))+ bits = int(config.get("bits", 16))act_bits = int(config.get("act_bits", 16))else:- bits = int(config.bits) if hasattr(config, "bits") else 4+ bits = int(config.bits) if hasattr(config, "bits") else 16act_bits = int(config.act_bits) if hasattr(config, "act_bits") else 16return bits <= 8 or act_bits <= 8diff --git a/auto_round/wrapper.py b/auto_round/wrapper.pyindex 4e1a5d8a7..a624d9f21 100644--- a/auto_round/wrapper.py+++ b/auto_round/wrapper.py@@ -58,7 +58,8 @@ class WrapperLinear(torch.nn.Module):device (str): Device on which to run computations (e.g., 'cpu' or 'cuda')."""- def __init__(self, orig_layer, enable_minmax_tuning=True, enable_norm_bias_tuning=False, device='cpu', **kwargs):+ def __init__(self, orig_layer, enable_minmax_tuning=True, enable_norm_bias_tuning=False, device='cpu',+ enable_round_tuning=True, **kwargs):"""Initializes the WrapperLinear module.Args:@@ -72,6 +73,7 @@ class WrapperLinear(torch.nn.Module):self.output_device = deviceself.device = self.orig_layer.tuning_device if hasattr(self.orig_layer, "tuning_device") else deviceself.enable_minmax_tuning = enable_minmax_tuning+ self.enable_round_tuning = enable_round_tuningself.enable_norm_bias_tuning = enable_norm_bias_tuning and (orig_layer.bias is not None)self.enable_act_quant = self.orig_layer.act_bits <= 8 or self._check_act_quantization(self.orig_layer.act_data_type)@@ -108,7 +110,7 @@ class WrapperLinear(torch.nn.Module):weight_reshape = reshape_and_pad_tensor(orig_weight.data, orig_layer.group_size)self.weight_min = torch.clamp(weight_reshape.min(1)[0], max=0)self.weight_max = torch.clamp(weight_reshape.max(1)[0], min=0)- self._init_params("value", p_dtype, weight_reshape.shape, 0, True)+ self._init_params("value", p_dtype, weight_reshape.shape, 0, self.enable_round_tuning)# Min-max scale initializationshape = get_scale_shape(orig_weight, orig_layer.group_size)self._init_params("min_scale", p_dtype, shape, 1.0, self.enable_minmax_tuning)@@ -166,7 +168,7 @@ class WrapperLinear(torch.nn.Module):weight = self.orig_layer.get_weight().to(self.device)if isinstance(self.orig_layer, transformers.modeling_utils.Conv1D):weight = weight.t()-+quant_kwargs = {}if hasattr(self.orig_layer, "super_bits"):quant_kwargs["super_bits"] = self.orig_layer.super_bits@@ -185,7 +187,7 @@ class WrapperLinear(torch.nn.Module):data_type=self.data_type,q_scale_thresh=self.q_scale_thresh,**quant_kwargs- )+ )weight_q = weight_q.to(weight.dtype)if isinstance(self.orig_layer, transformers.modeling_utils.Conv1D):weight_q = weight_q.t()@@ -257,12 +259,12 @@ class WrapperLinear(torch.nn.Module):else:name = "w_" + keysetattr(self.orig_layer, name, attr_dict[key].to("cpu"))-+if isinstance(scale, dict):_set_dict_attr(scale, "scale")else:self.orig_layer.scale = scale.reshape(shape[0], -1).to("cpu")-+if zp is not None:if isinstance(zp, dict):_set_dict_attr(zp, "zp")diff --git a/test/_test_helpers.py b/test/_test_helpers.pyindex ac753a2eb..7d848c728 100644--- a/test/_test_helpers.py+++ b/test/_test_helpers.py@@ -7,3 +7,45 @@ def is_pytest_mode_compile():def is_pytest_mode_lazy():return pytest.mode == "lazy"+++def model_infer(model, tokenizer, apply_chat_template=False):+ prompts = [+ "Hello,my name is",+ # "The president of the United States is",+ # "The capital of France is",+ # "The future of AI is",+ ]+ if apply_chat_template:+ texts = []+ for prompt in prompts:+ messages = [+ {"role": "user", "content": prompt}+ ]+ text = tokenizer.apply_chat_template(+ messages,+ tokenize=False,+ add_generation_prompt=True+ )+ texts.append(text)+ prompts = texts++ inputs = tokenizer(prompts, return_tensors="pt", padding=False, truncation=True)++ outputs = model.generate(+ input_ids=inputs["input_ids"].to(model.device),+ attention_mask=inputs["attention_mask"].to(model.device),+ do_sample=False, ## change this to follow official usage+ max_new_tokens=5+ )+ generated_ids = [+ output_ids[len(input_ids):] for input_ids, output_ids in zip(inputs["input_ids"], outputs)+ ]++ decoded_outputs = tokenizer.batch_decode(generated_ids, skip_special_tokens=True)++ for i, prompt in enumerate(prompts):+ print(f"Prompt: {prompt}")+ print(f"Generated: {decoded_outputs[i]}")+ print("-" * 50)+ return decoded_outputs[0]diff --git a/test/test_autoround.py b/test/test_autoround.pyindex b7afae1db..58105e973 100644--- a/test/test_autoround.py+++ b/test/test_autoround.py@@ -3,13 +3,15 @@ import shutilimport sysimport unittest+from auto_round.eval.evaluation import simple_evaluate_user_model+sys.path.insert(0, "..")import torchimport transformersfrom transformers import AutoModelForCausalLM, AutoTokenizerfrom auto_round import AutoRound-+from _test_helpers import model_inferclass LLMDataLoader:def __init__(self):@@ -27,20 +29,21 @@ class TestAutoRound(unittest.TestCase):self.model = AutoModelForCausalLM.from_pretrained(model_name, torch_dtype="auto", trust_remote_code=True)self.tokenizer = AutoTokenizer.from_pretrained(model_name, trust_remote_code=True)self.llm_dataloader = LLMDataLoader()+ self.save_folder = "./saved"@classmethoddef tearDownClass(self):- shutil.rmtree("./saved", ignore_errors=True)+ shutil.rmtree(self.save_folder, ignore_errors=True)shutil.rmtree("runs", ignore_errors=True)def test_remove_whole_block(self):- layer_config={"model.decoder.layers.0.self_attn.k_proj": {"bits": 32},- "model.decoder.layers.0.self_attn.v_proj": {"bits": 32},- "model.decoder.layers.0.self_attn.q_proj": {"bits": 32},- "model.decoder.layers.0.self_attn.out_proj": {"bits": 32},- "model.decoder.layers.0.fc1": {"bits": 32},- "model.decoder.layers.0.fc2": {"bits": 32},- }+ layer_config = {"model.decoder.layers.0.self_attn.k_proj": {"bits": 32},+ "model.decoder.layers.0.self_attn.v_proj": {"bits": 32},+ "model.decoder.layers.0.self_attn.q_proj": {"bits": 32},+ "model.decoder.layers.0.self_attn.out_proj": {"bits": 32},+ "model.decoder.layers.0.fc1": {"bits": 32},+ "model.decoder.layers.0.fc2": {"bits": 32},+ }bits, group_size, sym = 4, 128, Falseautoround = AutoRound(self.model,@@ -83,8 +86,6 @@ class TestAutoRound(unittest.TestCase):)autoround.quantize()--def test_mx_fp4(self):bits, group_size, sym = 4, 32, Falseautoround = AutoRound(@@ -101,7 +102,7 @@ class TestAutoRound(unittest.TestCase):autoround.quantize()def test_nsample(self):- autoround= AutoRound(+ autoround = AutoRound(self.model,self.tokenizer,bits=4,@@ -237,6 +238,7 @@ class TestAutoRound(unittest.TestCase):dataset=self.llm_dataloader,)autoround.quantize()+#def test_signround(self):bits, group_size, sym = 4, -1, False@@ -290,7 +292,8 @@ class TestAutoRound(unittest.TestCase):def test_auto_device_map(self):bits, group_size, sym = 4, 128, Falsemodel_name = "facebook/opt-125m"- model = AutoModelForCausalLM.from_pretrained(model_name, torch_dtype="auto", trust_remote_code=True, device_map='auto')+ model = AutoModelForCausalLM.from_pretrained(model_name, torch_dtype="auto", trust_remote_code=True,+ device_map='auto')autoround = AutoRound(model,self.tokenizer,@@ -306,7 +309,8 @@ class TestAutoRound(unittest.TestCase):def test_fp32(self):bits, group_size, sym = 4, 128, Falsemodel_name = "facebook/opt-125m"- model = AutoModelForCausalLM.from_pretrained(model_name, torch_dtype=torch.float32, trust_remote_code=True, device_map='auto')+ model = AutoModelForCausalLM.from_pretrained(model_name, torch_dtype=torch.float32, trust_remote_code=True,+ device_map='auto')autoround = AutoRound(model,self.tokenizer,@@ -320,7 +324,6 @@ class TestAutoRound(unittest.TestCase):)autoround.quantize()-def test_fallback_layers(self):bits, group_size, sym = 4, 128, Truemodel_name = "facebook/opt-125m"@@ -340,7 +343,7 @@ class TestAutoRound(unittest.TestCase):layer_config=layer_config)autoround.quantize()- quantized_model_path = "./saved"+ quantized_model_path = self.save_folderautoround.save_quantized(output_dir=quantized_model_path, format="auto_round", inplace=True)from auto_round import AutoRoundConfig@@ -349,15 +352,18 @@ class TestAutoRound(unittest.TestCase):)model = AutoModelForCausalLM.from_pretrained(quantized_model_path,- device_map='cpu',quantization_config=quantization_config)+ device_map='cpu', quantization_config=quantization_config)tokenizer = AutoTokenizer.from_pretrained(quantized_model_path)text = "There is a girl who likes adventure,"inputs = tokenizer(text, return_tensors="pt").to(model.device)res = tokenizer.decode(model.generate(**inputs, max_new_tokens=1)[0])+ shutil.rmtree(self.save_folder, ignore_errors=True)+def test_tensor_reshape(self):model_name = "facebook/opt-125m"- model = AutoModelForCausalLM.from_pretrained(model_name, torch_dtype=torch.float32, trust_remote_code=True, device_map='auto')+ model = AutoModelForCausalLM.from_pretrained(model_name, torch_dtype=torch.float32, trust_remote_code=True,+ device_map='auto')bits, group_size, sym = 4, 100, Falseautoround = AutoRound(self.model,@@ -415,13 +421,42 @@ class TestAutoRound(unittest.TestCase):# Inference: Generation of the outputgenerated_ids = model.generate(**inputs, max_new_tokens=1)generated_ids_trimmed = [- out_ids[len(in_ids) :] for in_ids, out_ids in zip(inputs.input_ids, generated_ids)+ out_ids[len(in_ids):] for in_ids, out_ids in zip(inputs.input_ids, generated_ids)]output_text = processor.batch_decode(generated_ids_trimmed, skip_special_tokens=True, clean_up_tokenization_spaces=False)print(output_text)+ def test_rtn(self):+ model_name = "facebook/opt-125m"+ model = AutoModelForCausalLM.from_pretrained(model_name, torch_dtype="auto", trust_remote_code=True)+ tokenizer = AutoTokenizer.from_pretrained(model_name, trust_remote_code=True)++ bits, group_size, sym = 4, 128, True+ autoround = AutoRound(+ model,+ tokenizer,+ bits=bits,+ group_size=group_size,+ sym=sym,+ iters=1,+ nsamples=1++ )+ quantized_model_path = self.save_folder+ autoround.quantize_and_save(output_dir=quantized_model_path, format="auto_round")+ from auto_round import AutoRoundConfig+ model = AutoModelForCausalLM.from_pretrained(+ self.save_folder,+ torch_dtype=torch.float16,+ device_map="auto",+ )++ tokenizer = AutoTokenizer.from_pretrained(self.save_folder)+ model_infer(model, tokenizer)+ shutil.rmtree(self.save_folder)+if __name__ == "__main__":unittest.main()diff --git a/test/test_basic_usage.py b/test/test_basic_usage.pyindex e468d173c..695271bec 100644--- a/test/test_basic_usage.py+++ b/test/test_basic_usage.py@@ -30,46 +30,46 @@ class TestAutoRoundCmd(unittest.TestCase):if res > 0 or res == -1:assert False, "cmd line test fail, please have a check"- # res = os.system(- # f"cd .. && {python_path} -m auto_round --model 'facebook/opt-125m' --seqlen 8 --iter 1 --nsamples 1 --eval_task_by_task --tasks openbookqa --bs 32"- # )- # if res > 0 or res == -1:- # assert False, "cmd line test fail, please have a check"- #- # res = os.system(- # f"cd .. && {python_path} -c 'from auto_round.__main__ import run_light; run_light()' --seqlen 8 --iter 2 --nsamples 8 --output_dir ./saved --tasks lambada_openai")- # if res > 0 or res == -1:- # assert False, "cmd line test fail, please have a check"- #- # # test mllm script- # # test auto_round_mllm help- # res = os.system(- # f"cd .. && {python_path} -m auto_round --mllm -h")- # if res > 0 or res == -1:- # assert False, "cmd line test fail, please have a check"- #- # # test auto_round_mllm --eval help- # res = os.system(- # f"cd .. && {python_path} -m auto_round --mllm --eval -h")- # if res > 0 or res == -1:- # assert False, "cmd line test fail, please have a check"- #- # # test auto_round_mllm --lmms help- # res = os.system(- # f"cd .. && {python_path} -m auto_round --mllm --lmms -h")- # if res > 0 or res == -1:- # assert False, "cmd line test fail, please have a check"- #- # res = os.system(- # f"cd .. && {python_path} -m auto_round --mllm --iter 2 --nsamples 10 --seqlen 32 --format auto_round --output_dir ./saved")- # if res > 0 or res == -1:- # assert False, "cmd line test fail, please have a check"- #- # res = os.system(- # f"cd .. && {python_path} -m auto_round --mllm --iter 2 --nsamples 10 --seqlen 256 --format auto_round"- # " --quant_nontext_module --output_dir ./saved ")- # if res > 0 or res == -1:- # assert False, "cmd line test fail, please have a check"+ res = os.system(+ f"cd .. && {python_path} -m auto_round --model 'facebook/opt-125m' --seqlen 8 --iter 1 --nsamples 1 --eval_task_by_task --tasks openbookqa --bs 32"+ )+ if res > 0 or res == -1:+ assert False, "cmd line test fail, please have a check"++ res = os.system(+ f"cd .. && {python_path} -c 'from auto_round.__main__ import run_light; run_light()' --seqlen 8 --iter 2 --nsamples 8 --output_dir ./saved --tasks lambada_openai")+ if res > 0 or res == -1:+ assert False, "cmd line test fail, please have a check"++ # test mllm script+ # test auto_round_mllm help+ res = os.system(+ f"cd .. && {python_path} -m auto_round --mllm -h")+ if res > 0 or res == -1:+ assert False, "cmd line test fail, please have a check"++ # test auto_round_mllm --eval help+ res = os.system(+ f"cd .. && {python_path} -m auto_round --mllm --eval -h")+ if res > 0 or res == -1:+ assert False, "cmd line test fail, please have a check"++ # test auto_round_mllm --lmms help+ res = os.system(+ f"cd .. && {python_path} -m auto_round --mllm --lmms -h")+ if res > 0 or res == -1:+ assert False, "cmd line test fail, please have a check"++ res = os.system(+ f"cd .. && {python_path} -m auto_round --mllm --iter 2 --nsamples 10 --seqlen 32 --format auto_round --output_dir ./saved")+ if res > 0 or res == -1:+ assert False, "cmd line test fail, please have a check"++ res = os.system(+ f"cd .. && {python_path} -m auto_round --mllm --iter 2 --nsamples 10 --seqlen 256 --format auto_round"+ " --quant_nontext_module --output_dir ./saved ")+ if res > 0 or res == -1:+ assert False, "cmd line test fail, please have a check"if __name__ == "__main__":diff --git a/test/test_conv1d.py b/test/test_conv1d.pyindex 0d010051d..62b505b48 100644--- a/test/test_conv1d.py+++ b/test/test_conv1d.py@@ -8,6 +8,7 @@ import torchfrom transformers import AutoModelForCausalLM, AutoTokenizerfrom auto_round import AutoRound+from _test_helpers import model_inferclass LLMDataLoader:def __init__(self):self.batch_size = 1@@ -33,6 +34,7 @@ class TestQuantizationConv1d(unittest.TestCase):def test_quant(self):self.model = AutoModelForCausalLM.from_pretrained(self.model_name, torch_dtype="auto", trust_remote_code=True)bits, group_size, sym = 4, 128, False+ from auto_round import AutoRoundConfigautoround = AutoRound(self.model,self.tokenizer,@@ -46,12 +48,12 @@ class TestQuantizationConv1d(unittest.TestCase):)autoround.quantize()- try:- import auto_gptq- except:- returnautoround.save_quantized("./saved")+ model = AutoModelForCausalLM.from_pretrained("./saved", device_map="cpu", trust_remote_code=True)+ model_infer(model, self.tokenizer)++if __name__ == "__main__":unittest.main()diff --git a/test_cuda/_test_helpers.py b/test_cuda/_test_helpers.pynew file mode 100644index 000000000..3c499c499--- /dev/null+++ b/test_cuda/_test_helpers.py@@ -0,0 +1,41 @@++def model_infer(model, tokenizer, apply_chat_template=False):+ prompts = [+ "Hello,my name is",+ # "The president of the United States is",+ # "The capital of France is",+ # "The future of AI is",+ ]+ if apply_chat_template:+ texts = []+ for prompt in prompts:+ messages = [+ {"role": "user", "content": prompt}+ ]+ text = tokenizer.apply_chat_template(+ messages,+ tokenize=False,+ add_generation_prompt=True+ )+ texts.append(text)+ prompts = texts++ inputs = tokenizer(prompts, return_tensors="pt", padding=False, truncation=True)++ outputs = model.generate(+ input_ids=inputs["input_ids"].to(model.device),+ attention_mask=inputs["attention_mask"].to(model.device),+ do_sample=False, ## change this to follow official usage+ max_new_tokens=5+ )+ generated_ids = [+ output_ids[len(input_ids):] for input_ids, output_ids in zip(inputs["input_ids"], outputs)+ ]++ decoded_outputs = tokenizer.batch_decode(generated_ids, skip_special_tokens=True)++ for i, prompt in enumerate(prompts):+ print(f"Prompt: {prompt}")+ print(f"Generated: {decoded_outputs[i]}")+ print("-" * 50)+ return decoded_outputs[0]diff --git a/test_cuda/test_conv1d.py b/test_cuda/test_conv1d.pynew file mode 100644index 000000000..a407a4aef--- /dev/null+++ b/test_cuda/test_conv1d.py@@ -0,0 +1,59 @@+import copy+import shutil+import sys+import unittest++sys.path.insert(0, "..")+import torch+from transformers import AutoModelForCausalLM, AutoTokenizer++from auto_round import AutoRound+from _test_helpers import model_infer+class LLMDataLoader:+ def __init__(self):+ self.batch_size = 1++ def __iter__(self):+ for i in range(2):+ yield torch.ones([1, 10], dtype=torch.long)+++class TestQuantizationConv1d(unittest.TestCase):+ @classmethod+ def setUpClass(self):+ self.model_name = "MBZUAI/LaMini-GPT-124M"+ self.tokenizer = AutoTokenizer.from_pretrained(self.model_name, trust_remote_code=True)+ self.llm_dataloader = LLMDataLoader()++ @classmethod+ def tearDownClass(self):+ shutil.rmtree("./saved", ignore_errors=True)+ shutil.rmtree("runs", ignore_errors=True)+++ def test_quant(self):+ self.model = AutoModelForCausalLM.from_pretrained(self.model_name, torch_dtype="auto", trust_remote_code=True)+ bits, group_size, sym = 4, 128, False+ from auto_round import AutoRoundConfig+ autoround = AutoRound(+ self.model,+ self.tokenizer,+ bits=bits,+ group_size=group_size,+ sym=sym,+ iters=2,+ seqlen=2,+ dataset=self.llm_dataloader,++ )++ autoround.quantize()+ autoround.save_quantized("./saved")++ model = AutoModelForCausalLM.from_pretrained("./saved", device_map="cuda", trust_remote_code=True)+ model_infer(model, self.tokenizer)++++if __name__ == "__main__":+ unittest.main()