PTQ-Level Efficiency, QAT-Level Accuracy | 训练后量化的效率,量化感知训练的精度
support naive multi-card tuning (#415)
Wenhua Cheng09f1ab49127c
diff --git a/auto_round/autoround.py b/auto_round/autoround.pyindex 2df9befd1..813856eb8 100644--- a/auto_round/autoround.py+++ b/auto_round/autoround.py@@ -13,6 +13,8 @@# limitations under the License.import os+import re+import torchimport transformersimport copy@@ -115,6 +117,7 @@ class AutoRound(object):block's layer names to be quantized.enable_norm_bias_tuning (bool): Whether to enable fast norm/layer_bias tuningenable_torch_compile (bool): Whether to enable torch compile to optimize quant_block/layer, torch>=2.6 True.+ device_map (str|dict): device map for each blockReturns:The quantized model."""@@ -157,6 +160,7 @@ class AutoRound(object):to_quant_block_names: Union[str, list] = None,enable_norm_bias_tuning: bool = False,enable_torch_compile: bool = None,+ device_map: Union[str, dict] = None,**kwargs,):self.quantized = False@@ -203,7 +207,6 @@ class AutoRound(object):all_blocks = get_block_names(model)self.quant_block_list = find_matching_blocks(model, all_blocks, self.to_quant_block_names)self.cache_device = torch.device("cpu") if self.low_gpu_mem_usage else self.device-##activationself.act_group_size = act_group_size if not (act_group_size is None) else self.group_size@@ -233,9 +236,63 @@ class AutoRound(object):logger.info("Optimum Habana is available, import htcore explicitly.")import habana_frameworks.torch.core as htcore # pylint: disable=E0401import habana_frameworks.torch.hpu as hthpu # pylint: disable=E0401]+ self.device_map = device_map++ self.set_device_map_in_blocks(self.device_map)self.set_layerwise_config(self.layer_config) ##better place in the end+ def set_device_map_in_blocks(self, device_map):+ """Sets the device map for specific blocks in the model.++ Args:+ device_map (Union[str, dict]): A mapping of module names to devices.+ If provided as a string, it should be in the format+ "module_name:device,module_name:device". Devices can be integers+ (GPU IDs) or strings (e.g., 'cpu', 'cuda:0').+ """+ if self.device_map is None or len(self.device_map) == 0:+ self.device_map = None+ if not device_map:+ return+ if isinstance(device_map, str):+ device_map = device_map.replace(" ", "")+ infos = device_map.split(",")+ device_map_dict = {}+ for info in infos:+ index = info.find(':')+ key = info[:index]+ value = info[index + 1:]+ device_map_dict[key] = value+ device_map = device_map_dict++ names = [n for n, m in self.model.named_modules() if len(list(m.children())) == 0]++ for key, device in device_map.items():+ if isinstance(device, str) and device.isdigit():+ device = int(device)+ device = detect_device(device)+ try:+ module = get_module(self.model, key)+ module.tuning_device = device+ except:+ matching_names = [name for name in names if re.match(key, name)]+ if len(matching_names) > 0:+ for name in matching_names:+ self._set_device_for_matching_module(name, device)+ else:+ for name in names:+ if key in name:+ self._set_device_for_matching_module(name, device)++ def _set_device_for_matching_module(self, name, device):+ module = get_module(self.model, name)+ if hasattr(module, "tuning_device") and module.tuning_device != device:+ logger.warning(+ f"Multiple devices have been set for layer {name}, keeping original device {module.tuning_device}")+ else:+ module.tuning_device = device+def check_configs(self):"""Checks if the configurations are valid.@@ -861,6 +918,9 @@ class AutoRound(object):"""logger.info(f"quantizing layer {layer_name}")layer = get_module(self.model, layer_name)+ if hasattr(layer, "tuning_device"):+ device = layer.tuning_device+layer = layer.to(device)for i in range(len(inputs)):inputs[i] = inputs[i].to(layer.weight.dtype)@@ -970,7 +1030,7 @@ class AutoRound(object):unwrapper_layer(self.model, wrapper_linear, layer_name, best_params)mv_module_from_gpu(layer, self.low_cpu_mem_usage)dump_info = f"quantized {layer_name}, loss iter 0: {init_loss:.6f} -> iter {best_iter}: {last_loss:.6f}"- logger.debug(dump_info)+ logger.info(dump_info)def register_act_max_hook(self, model):def get_act_max_hook(module, input, output):@@ -1002,6 +1062,15 @@ class AutoRound(object):Returns:Tuple: (q_outputs, output) if self.enable_quanted_input is True, else (None, output)"""+ if self.device_map is not None:+ from accelerate import dispatch_model+ for n, m in block.named_modules():+ if len(list(m.children())) != 0 or not hasattr(m, "tuning_device"):+ continue+ from accelerate.hooks import AlignDevicesHook, add_hook_to_module+ hook = AlignDevicesHook(m.tuning_device, io_same_device=True)+ add_hook_to_module(m, hook, True)+if q_input is None:hook_handles = self.register_act_max_hook(block)@@ -1023,6 +1092,10 @@ class AutoRound(object):handle.remove()if q_input is not None:+ if input_ids is not q_input:+ clear_memory(input_ids)+ else:+ clear_memory()input_ids = q_inputquantized_layer_names, unquantized_layer_names = wrapper_block(@@ -1050,7 +1123,7 @@ class AutoRound(object):f"quantized {len(quantized_layer_names)}/{(len(quantized_layer_names) + len(unquantized_layer_names))} "f"layers in the block")- logger.debug(dump_info)+ logger.info(dump_info)return output, outputif self.lr_scheduler is None:@@ -1141,7 +1214,7 @@ class AutoRound(object):f"quantized {len(quantized_layer_names)}/{(len(quantized_layer_names) + len(unquantized_layer_names))} "f"layers in the block, loss iter 0: {init_loss:.6f} -> iter {best_iter}: {last_loss:.6f}")- logger.debug(dump_info)+ logger.info(dump_info)if len(unquantized_layer_names) != 0:logger.info(f"{unquantized_layer_names} have not been quantized")with torch.no_grad():@@ -1154,12 +1227,18 @@ class AutoRound(object):block, input_ids, input_others, self.batch_size * self.infer_bs_coeff, device,cache_device=self.cache_device)+ if self.device_map is not None:+ accelerate.hooks.remove_hook_from_submodules(+ block)mv_module_from_gpu(block, self.low_cpu_mem_usage)clear_memory(input_ids)return q_outputs, outputelse:+ if self.device_map is not None:+ accelerate.hooks.remove_hook_from_submodules(+ block)mv_module_from_gpu(block, self.low_cpu_mem_usage)clear_memory(input_ids)return None, output@@ -1538,6 +1617,7 @@ class AutoRoundOPT(AutoRound):to_quant_block_names: Union[str, list] = None,enable_norm_bias_tuning: bool = False,enable_torch_compile: bool = None,+ device_map: Union[str, dict] = None,optimizer="AdamW",**kwargs,):@@ -1578,6 +1658,7 @@ class AutoRoundOPT(AutoRound):to_quant_block_names=to_quant_block_names,enable_norm_bias_tuning=enable_norm_bias_tuning,enable_torch_compile=enable_torch_compile,+ device_map=device_map,**kwargs,)@@ -1710,6 +1791,7 @@ class AutoRoundAdam(AutoRoundOPT):to_quant_block_names: Union[str, list] = None,enable_norm_bias_tuning: bool = False,enable_torch_compile: bool = None,+ device_map: Union[str, dict] = None,optimizer="AdamW",**kwargs,):@@ -1750,7 +1832,7 @@ class AutoRoundAdam(AutoRoundOPT):to_quant_block_names=to_quant_block_names,enable_norm_bias_tuning=enable_norm_bias_tuning,enable_torch_compile=enable_torch_compile,+ device_map=device_map,optimizer=optimizer,**kwargs,)-diff --git a/auto_round/quantizer.py b/auto_round/quantizer.pyindex 3f1eeb884..f44746086 100644--- a/auto_round/quantizer.py+++ b/auto_round/quantizer.py@@ -69,7 +69,8 @@ class WrapperLinear(torch.nn.Module):"""super(WrapperLinear, self).__init__()self.orig_layer = orig_layer- self.device = device+ self.output_device = device+ self.device = self.orig_layer.tuning_device if hasattr(self.orig_layer, "tuning_device") else deviceself.enable_minmax_tuning = enable_minmax_tuningself.enable_norm_bias_tuning = enable_norm_bias_tuning and (orig_layer.bias is not None)self.enable_act_quant = self.orig_layer.act_bits <= 8@@ -293,6 +294,7 @@ class WrapperLinear(torch.nn.Module):Returns:torch.Tensor: Output tensor after applying the wrapped layer."""+ x = x.to(self.device)weight_q, _, _ = self._qdq_weight(self.value, self.min_scale, self.max_scale)if self.enable_act_quant:@@ -306,7 +308,8 @@ class WrapperLinear(torch.nn.Module):if self.enable_norm_bias_tuning:bias, _, _ = self._qdq_bias(bias, self.bias_v)- return self.orig_forward(x, weight_q, bias)+ output = self.orig_forward(x, weight_q, bias).to(self.output_device)+ return outputclass WrapperWALayer(torch.nn.Module):@@ -339,7 +342,8 @@ class WrapperLayerNorm(torch.nn.Module):self.orig_layer = orig_layerself.bits = bitself.group_size = group_size- self.device = device+ self.device = self.orig_layer.tuning_device if hasattr(self.orig_layer, "tuning_device") else device+ self.output_device = deviceweight_dtype = torch.float32self.q_scale_thresh = 1e-5self.v = torch.nn.Parameter(@@ -362,11 +366,13 @@ class WrapperLayerNorm(torch.nn.Module):return self.orig_layerdef forward(self, input):+ input = input.to(self.device)weight_q, _, _ = self.quant_func(self.orig_layer.weight, self.bits, self.group_size,self.v, q_scale_thresh=self.q_scale_thresh)import torch.nn.functional as Freturn F.layer_norm(- input, self.orig_layer.normalized_shape, weight_q, self.orig_layer.bias, self.orig_layer.eps)+ input, self.orig_layer.normalized_shape, weight_q, self.orig_layer.bias, self.orig_layer.eps).to(+ self.output_device)class WrapperLlamaNorm(torch.nn.Module):@@ -382,7 +388,8 @@ class WrapperLlamaNorm(torch.nn.Module):self.orig_layer = orig_layerself.bits = bitself.group_size = group_size- self.device = device+ self.device = self.orig_layer.tuning_device if hasattr(self.orig_layer, "tuning_device") else device+ self.output_device = deviceweight_dtype = torch.float32self.q_scale_thresh = 1e-5self.v = torch.nn.Parameter(@@ -405,13 +412,14 @@ class WrapperLlamaNorm(torch.nn.Module):return self.orig_layerdef forward(self, hidden_states):+ hidden_states = hidden_states.to(self.device)weight_q, _, _ = self.quant_func(self.orig_layer.weight, self.bits, self.group_size,self.v, q_scale_thresh=self.q_scale_thresh)input_dtype = hidden_states.dtypehidden_states = hidden_states.to(torch.float32)variance = hidden_states.pow(2).mean(-1, keepdim=True)hidden_states = hidden_states * torch.rsqrt(variance + self.orig_layer.variance_epsilon)- return weight_q * hidden_states.to(input_dtype)+ return (weight_q * hidden_states.to(input_dtype)).to(self.output_device)norm_mapping = {}@@ -518,4 +526,3 @@ def unwrapper_block(block, best_params):best_param = Noneorig_layer = m.unwrapper(best_param)set_module(block, n, orig_layer)-diff --git a/auto_round/script/llm.py b/auto_round/script/llm.pyindex ba7267a10..7dfe86e94 100644--- a/auto_round/script/llm.py+++ b/auto_round/script/llm.py@@ -49,10 +49,10 @@ class BasicArgumentParser(argparse.ArgumentParser):self.add_argument("--device", "--devices", default="0", type=str,help="the device to be used for tuning. "- "Currently, device settings support CPU, GPU, and HPU."- "The default is set to cuda:0,"- "allowing for automatic detection and switch to HPU or CPU."- "set --device 0,1,2 to use multiple cards.")+ "Currently, device settings support CPU, GPU, and HPU."+ "The default is set to cuda:0,"+ "allowing for automatic detection and switch to HPU or CPU."+ "set --device 0,1,2 to use multiple cards.")self.add_argument("--asym", action='store_true',help="whether to use asym quantization")@@ -159,6 +159,10 @@ class BasicArgumentParser(argparse.ArgumentParser):self.add_argument("--disable_act_dynamic", action='store_true',help="activation static quantization")+ self.add_argument("--device_map", default=None, type=str,+ help="device_map for block in tuning phase")++class EvalArgumentParser(argparse.ArgumentParser):def __init__(self, *args, **kwargs):super().__init__(*args, **kwargs)@@ -166,10 +170,10 @@ class EvalArgumentParser(argparse.ArgumentParser):help="model name or path")self.add_argument("--device", "--devices", default="0", type=str,help="the device to be used for tuning. "- "Currently, device settings support CPU, GPU, and HPU."- "The default is set to cuda:0,"- "allowing for automatic detection and switch to HPU or CPU."- "set --device 0,1,2 to use multiple cards.")+ "Currently, device settings support CPU, GPU, and HPU."+ "The default is set to cuda:0,"+ "allowing for automatic detection and switch to HPU or CPU."+ "set --device 0,1,2 to use multiple cards.")self.add_argument("--tasks",default="lambada_openai,hellaswag,winogrande,piqa,mmlu,wikitext,truthfulqa_mc1," \"openbookqa,boolq,arc_easy,arc_challenge",@@ -178,7 +182,7 @@ class EvalArgumentParser(argparse.ArgumentParser):help="whether to disable trust_remote_code")self.add_argument("--eval_bs", "--bs", "--batch_size", default=None, type=int,help="batch size in evaluation")-+def setup_parser():parser = BasicArgumentParser()@@ -254,6 +258,7 @@ def setup_eval_parser():args = parser.parse_args()return args+def _gguf_args_check(args):from auto_round.utils import logger@@ -275,7 +280,7 @@ def _gguf_args_check(args):formats = args.format.lower().replace(' ', '').split(",")for format in _GGUF_CONFIG:if format in formats:- unsupport_list, reset_list = [],[]+ unsupport_list, reset_list = [], []gguf_config = _GGUF_CONFIG[format]for k, v in gguf_config.items():if getattr(args, k) != v:@@ -291,9 +296,10 @@ def _gguf_args_check(args):logger.error(f"format {format} not support for {', '.join(unsupport_list)},"f" reset to {', '.join(reset_list)}.")logger.info(f"export format {format}, sym = {not args.asym}, group_size = {args.group_size}")-+return args+def tune(args):import transformers@@ -453,7 +459,6 @@ def tune(args):##TODO gptq could support some mixed precision configlogger.warning(f"mixed precision exporting does not support {format} currently")-lm_head_layer_name = "lm_head"for n, _ in model.named_modules():lm_head_layer_name = n@@ -479,7 +484,8 @@ def tune(args):if "auto_awq" in args.format:from auto_round.utils import check_awq_gemm_compatibility- awq_supported, info = check_awq_gemm_compatibility(model,args.bits,args.group_size, not args.asym, layer_config)+ awq_supported, info = check_awq_gemm_compatibility(model, args.bits, args.group_size, not args.asym,+ layer_config)if not awq_supported:logger.warning(f"The AutoAWQ format may not be supported due to {info}")@@ -494,7 +500,8 @@ def tune(args):low_cpu_mem_usage=low_cpu_mem_usage, data_type=args.data_type,enable_norm_bias_tuning=args.enable_norm_bias_tuning, not_use_best_mse=args.not_use_best_mse,to_quant_block_names=args.to_quant_block_names, enable_torch_compile=args.enable_torch_compile,- act_data_type=args.act_data_type, act_dynamic=not args.disable_act_dynamic)+ act_data_type=args.act_data_type, act_dynamic=not args.disable_act_dynamic,+ device_map=args.device_map)model, _ = autoround.quantize()model_name = args.model.rstrip("/")if args.low_cpu_mem_mode == 1 or args.low_cpu_mem_mode == 2:@@ -556,6 +563,7 @@ def tune(args):batch_size=args.eval_bs)print(make_table(res))+def _eval_init(args):devices = args.device.replace(" ", "").split(',')parallelism = False@@ -596,6 +604,7 @@ def _eval_init(args):return tasks, model_args, device_str+def eval(args):from auto_round.eval.evaluation import simple_evaluate@@ -613,7 +622,7 @@ def eval(args):def eval_sequence(args):- from auto_round.eval.evaluation import simple_evaluate+ from auto_round.eval.evaluation import simple_evaluatetasks, model_args, device_str = _eval_init(args)from lm_eval.utils import make_table # pylint: disable=E0401@@ -627,8 +636,8 @@ def eval_sequence(args):device=device_str,batch_size=args.eval_bs)if not res_all:- res_all = res+ res_all = reselse:for key in res_keys:res_all[key].update(res[key])- print(make_table(res_all))\ No newline at end of file+ print(make_table(res_all))diff --git a/auto_round/script/mllm.py b/auto_round/script/mllm.pyindex 06e6397ab..883cb9a6a 100644--- a/auto_round/script/mllm.py+++ b/auto_round/script/mllm.py@@ -163,6 +163,9 @@ class BasicArgumentParser(argparse.ArgumentParser):self.add_argument("--to_quant_block_names", default=None, type=str,help="Names of quantitative blocks, please use commas to separate them.")+ self.add_argument("--device_map", default=None, type=str,+ help="device_map for block in tuning phase")+def setup_parser():parser = BasicArgumentParser()@@ -282,8 +285,8 @@ def tune(args):# load_modelprocessor, image_processor = None, Noneif "deepseek-vl2" in model_name.lower():- from deepseek_vl2.models import DeepseekVLV2Processor, DeepseekVLV2ForCausalLM # pylint: disable=E0401- processor = DeepseekVLV2Processor.from_pretrained(model_name)+ from deepseek_vl2.models import DeepseekVLV2Processor, DeepseekVLV2ForCausalLM # pylint: disable=E0401+ processor = DeepseekVLV2Processor.from_pretrained(model_name)tokenizer = processor.tokenizermodel: DeepseekVLV2ForCausalLM = AutoModelForCausalLM.from_pretrained(model_name, trust_remote_code=not args.disable_trust_remote_code, torch_dtype=torch_dtype,@@ -401,10 +404,11 @@ def tune(args):if "--truncation" not in sys.argv:args.truncation = None-+if "auto_awq" in args.format:from auto_round.utils import check_awq_gemm_compatibility- awq_supported, info = check_awq_gemm_compatibility(model,args.bits,args.group_size, not args.asym, layer_config)+ awq_supported, info = check_awq_gemm_compatibility(model, args.bits, args.group_size, not args.asym,+ layer_config)if not awq_supported:logger.warning(f"The AutoAWQ format may not be supported due to {info}")@@ -418,7 +422,8 @@ def tune(args):scale_dtype=args.scale_dtype, layer_config=layer_config, template=args.template,enable_minmax_tuning=not args.disable_minmax_tuning, act_bits=args.act_bits,quant_nontext_module=args.quant_nontext_module, not_use_best_mse=args.not_use_best_mse,- to_quant_block_names=args.to_quant_block_names, enable_torch_compile=args.enable_torch_compile)+ to_quant_block_names=args.to_quant_block_names, enable_torch_compile=args.enable_torch_compile,+ device_map=args.device_map)model, _ = autoround.quantize()model.eval()@@ -527,4 +532,3 @@ def lmms_eval(args):apply_chat_template=False,)return results-diff --git a/auto_round/utils.py b/auto_round/utils.pyindex 7ba339449..a182a53ec 100644--- a/auto_round/utils.py+++ b/auto_round/utils.py@@ -574,14 +574,14 @@ def detect_device(device=None):if device is None or device == "auto":if torch.cuda.is_available():device = torch.device("cuda")- logger.info("Using GPU device")+ # logger.info("Using GPU device")elif is_optimum_habana_available(): # pragma: no coverdevice = torch.device("hpu")- logger.info("Using HPU device")+ # logger.info("Using HPU device")# Use CPU as a fallbackelse:device = torch.device("cpu")- logger.info("Using CPU device")+ # logger.info("Using CPU device")if dev_idx is not None and str(device) != "cpu":device = str(device) + f":{dev_idx}"return str(device)@@ -943,6 +943,9 @@ def get_autogptq_packing_qlinear(backend, bits=4, group_size=128, sym=False):def _clear_memory_for_cpu_and_cuda(tensor=None):+ if isinstance(tensor, list):+ for i in range(len(tensor)):+ tensor[i] = Noneif tensor is not None:del tensorgc.collect()diff --git a/test_cuda/test_multiple_card_tuning.py b/test_cuda/test_multiple_card_tuning.pynew file mode 100644index 000000000..67254cd71--- /dev/null+++ b/test_cuda/test_multiple_card_tuning.py@@ -0,0 +1,101 @@+import shutil+import sys+import unittest+from auto_round.eval.evaluation import simple_evaluate+from lm_eval.utils import make_table # pylint: disable=E0401++sys.path.insert(0, "..")+from auto_round import AutoRound++import torch+from transformers import AutoModelForCausalLM, AutoTokenizer+import re+++def get_accuracy(data):+ match = re.search(r'\|acc\s+\|[↑↓]\s+\|\s+([\d.]+)\|', data)++ if match:+ accuracy = float(match.group(1))+ return accuracy+ else:+ return 0.0+++# import os+# os.environ["CUDA_VISIBLE_DEVICES"] = "0,1"+class TestAutoRound(unittest.TestCase):+ @classmethod+ def setUpClass(self):+ self.save_dir = "./saved"+ self.tasks = "lambada_openai"++ @classmethod+ def tearDownClass(self):+ shutil.rmtree(self.save_dir, ignore_errors=True)+ shutil.rmtree("runs", ignore_errors=True)++ def test_device_map(self):+ model_name = "/models/Qwen2-0.5B-Instruct"+ model = AutoModelForCausalLM.from_pretrained(model_name, torch_dtype=torch.float16)+ tokenizer = AutoTokenizer.from_pretrained(model_name)+ device_map = {".*q_proj": '0', ".*k_proj": "cuda:1", "v_proj": 1, ".*up_proj": "cpu"}+ autoround = AutoRound(model, tokenizer, iters=2, device_map=device_map, nsamples=7,seqlen=32)+ autoround.quantize()++ def test_device_map_str(self):+ model_name = "/models/Qwen2-0.5B-Instruct"+ model = AutoModelForCausalLM.from_pretrained(model_name, torch_dtype=torch.float16)+ tokenizer = AutoTokenizer.from_pretrained(model_name)+ device_map = '.*q_proj:0,.*k_proj:cuda:0,v_proj:1,.*up_proj:1'+ autoround = AutoRound(model, tokenizer,device_map=device_map)+ autoround.quantize()+ autoround.save_quantized(self.save_dir, format="auto_round", inplace=False)+ model_args = f"pretrained={self.save_dir}"+ res = simple_evaluate(model="hf", model_args=model_args,+ tasks=self.tasks,+ batch_size="auto")+ res = make_table(res)+ accuracy = get_accuracy(res)+ print(accuracy)+ assert accuracy > 0.45 ##0.4786+ shutil.rmtree("./saved", ignore_errors=True)++ def test_layer_norm(self):+ model_name = "/models/opt-125m"+ model = AutoModelForCausalLM.from_pretrained(model_name, torch_dtype=torch.float16)+ tokenizer = AutoTokenizer.from_pretrained(model_name)+ device_map = {"norm": "cuda:1"}+ autoround = AutoRound(model, tokenizer, iters=2, device_map=device_map, nsamples=7, seqlen=32,+ enable_norm_bias_tuning=True)+ autoround.quantize()+++ def test_rms_norm(self):+ model_name = "/models/Qwen2-0.5B-Instruct"+ model = AutoModelForCausalLM.from_pretrained(model_name, torch_dtype=torch.float16)+ tokenizer = AutoTokenizer.from_pretrained(model_name)+ device_map = {"norm": "cuda:1"}+ autoround = AutoRound(model, tokenizer, iters=2, device_map=device_map, nsamples=7, seqlen=32,+ enable_norm_bias_tuning=True)+ autoround.quantize()++ def test_act_quantization(self):+ model_name = "/models/Qwen2-0.5B-Instruct"+ model = AutoModelForCausalLM.from_pretrained(model_name, torch_dtype=torch.float16)+ tokenizer = AutoTokenizer.from_pretrained(model_name)+ device_map = {".*q_proj": '0', ".*k_proj": "cuda:1", "v_proj": 1, ".*up_proj": "1"}+ autoround = AutoRound(model, tokenizer, iters=2, device_map=device_map, nsamples=7,seqlen=32,act_bits=4,act_dynamic=False)+ autoround.quantize()++ def test_lm_head(self):+ model_name = "/models/Qwen2.5-7B-Instruct"+ model = AutoModelForCausalLM.from_pretrained(model_name, torch_dtype=torch.float16)+ tokenizer = AutoTokenizer.from_pretrained(model_name)+ device_map = {".*q_proj": '0', ".*k_proj": "cuda:1", "v_proj": 1, ".*up_proj": "1","lm_head":1}+ layer_config={"lm_head": {"bits": 4}}+ autoround = AutoRound(model, tokenizer, iters=2, device_map=device_map, nsamples=7, seqlen=32,+ enable_norm_bias_tuning=True,layer_config=layer_config)+ autoround.quantize()++