Skip to content

PTQ-Level Efficiency, QAT-Level Accuracy | 训练后量化的效率,量化感知训练的精度

support naive multi-card tuning (#415)

Wenhua Cheng09f1ab49127c
diff --git a/auto_round/autoround.py b/auto_round/autoround.py
index 2df9befd1..813856eb8 100644
--- a/auto_round/autoround.py
+++ b/auto_round/autoround.py
@@ -13,6 +13,8 @@
# limitations under the License.
import os
+import re
+
import torch
import transformers
import copy
@@ -115,6 +117,7 @@ class AutoRound(object):
block's layer names to be quantized.
enable_norm_bias_tuning (bool): Whether to enable fast norm/layer_bias tuning
enable_torch_compile (bool): Whether to enable torch compile to optimize quant_block/layer, torch>=2.6 True.
+ device_map (str|dict): device map for each block
Returns:
The quantized model.
"""
@@ -157,6 +160,7 @@ class AutoRound(object):
to_quant_block_names: Union[str, list] = None,
enable_norm_bias_tuning: bool = False,
enable_torch_compile: bool = None,
+ device_map: Union[str, dict] = None,
**kwargs,
):
self.quantized = False
@@ -203,7 +207,6 @@ class AutoRound(object):
all_blocks = get_block_names(model)
self.quant_block_list = find_matching_blocks(model, all_blocks, self.to_quant_block_names)
self.cache_device = torch.device("cpu") if self.low_gpu_mem_usage else self.device
-
##activation
self.act_group_size = act_group_size if not (act_group_size is None) else self.group_size
@@ -233,9 +236,63 @@ class AutoRound(object):
logger.info("Optimum Habana is available, import htcore explicitly.")
import habana_frameworks.torch.core as htcore # pylint: disable=E0401
import habana_frameworks.torch.hpu as hthpu # pylint: disable=E0401]
+ self.device_map = device_map
+
+ self.set_device_map_in_blocks(self.device_map)
self.set_layerwise_config(self.layer_config) ##better place in the end
+ def set_device_map_in_blocks(self, device_map):
+ """Sets the device map for specific blocks in the model.
+
+ Args:
+ device_map (Union[str, dict]): A mapping of module names to devices.
+ If provided as a string, it should be in the format
+ "module_name:device,module_name:device". Devices can be integers
+ (GPU IDs) or strings (e.g., 'cpu', 'cuda:0').
+ """
+ if self.device_map is None or len(self.device_map) == 0:
+ self.device_map = None
+ if not device_map:
+ return
+ if isinstance(device_map, str):
+ device_map = device_map.replace(" ", "")
+ infos = device_map.split(",")
+ device_map_dict = {}
+ for info in infos:
+ index = info.find(':')
+ key = info[:index]
+ value = info[index + 1:]
+ device_map_dict[key] = value
+ device_map = device_map_dict
+
+ names = [n for n, m in self.model.named_modules() if len(list(m.children())) == 0]
+
+ for key, device in device_map.items():
+ if isinstance(device, str) and device.isdigit():
+ device = int(device)
+ device = detect_device(device)
+ try:
+ module = get_module(self.model, key)
+ module.tuning_device = device
+ except:
+ matching_names = [name for name in names if re.match(key, name)]
+ if len(matching_names) > 0:
+ for name in matching_names:
+ self._set_device_for_matching_module(name, device)
+ else:
+ for name in names:
+ if key in name:
+ self._set_device_for_matching_module(name, device)
+
+ def _set_device_for_matching_module(self, name, device):
+ module = get_module(self.model, name)
+ if hasattr(module, "tuning_device") and module.tuning_device != device:
+ logger.warning(
+ f"Multiple devices have been set for layer {name}, keeping original device {module.tuning_device}")
+ else:
+ module.tuning_device = device
+
def check_configs(self):
"""Checks if the configurations are valid.
@@ -861,6 +918,9 @@ class AutoRound(object):
"""
logger.info(f"quantizing layer {layer_name}")
layer = get_module(self.model, layer_name)
+ if hasattr(layer, "tuning_device"):
+ device = layer.tuning_device
+
layer = layer.to(device)
for i in range(len(inputs)):
inputs[i] = inputs[i].to(layer.weight.dtype)
@@ -970,7 +1030,7 @@ class AutoRound(object):
unwrapper_layer(self.model, wrapper_linear, layer_name, best_params)
mv_module_from_gpu(layer, self.low_cpu_mem_usage)
dump_info = f"quantized {layer_name}, loss iter 0: {init_loss:.6f} -> iter {best_iter}: {last_loss:.6f}"
- logger.debug(dump_info)
+ logger.info(dump_info)
def register_act_max_hook(self, model):
def get_act_max_hook(module, input, output):
@@ -1002,6 +1062,15 @@ class AutoRound(object):
Returns:
Tuple: (q_outputs, output) if self.enable_quanted_input is True, else (None, output)
"""
+ if self.device_map is not None:
+ from accelerate import dispatch_model
+ for n, m in block.named_modules():
+ if len(list(m.children())) != 0 or not hasattr(m, "tuning_device"):
+ continue
+ from accelerate.hooks import AlignDevicesHook, add_hook_to_module
+ hook = AlignDevicesHook(m.tuning_device, io_same_device=True)
+ add_hook_to_module(m, hook, True)
+
if q_input is None:
hook_handles = self.register_act_max_hook(block)
@@ -1023,6 +1092,10 @@ class AutoRound(object):
handle.remove()
if q_input is not None:
+ if input_ids is not q_input:
+ clear_memory(input_ids)
+ else:
+ clear_memory()
input_ids = q_input
quantized_layer_names, unquantized_layer_names = wrapper_block(
@@ -1050,7 +1123,7 @@ class AutoRound(object):
f"quantized {len(quantized_layer_names)}/{(len(quantized_layer_names) + len(unquantized_layer_names))} "
f"layers in the block"
)
- logger.debug(dump_info)
+ logger.info(dump_info)
return output, output
if self.lr_scheduler is None:
@@ -1141,7 +1214,7 @@ class AutoRound(object):
f"quantized {len(quantized_layer_names)}/{(len(quantized_layer_names) + len(unquantized_layer_names))} "
f"layers in the block, loss iter 0: {init_loss:.6f} -> iter {best_iter}: {last_loss:.6f}"
)
- logger.debug(dump_info)
+ logger.info(dump_info)
if len(unquantized_layer_names) != 0:
logger.info(f"{unquantized_layer_names} have not been quantized")
with torch.no_grad():
@@ -1154,12 +1227,18 @@ class AutoRound(object):
block, input_ids, input_others, self.batch_size * self.infer_bs_coeff, device,
cache_device=self.cache_device
)
+ if self.device_map is not None:
+ accelerate.hooks.remove_hook_from_submodules(
+ block)
mv_module_from_gpu(block, self.low_cpu_mem_usage)
clear_memory(input_ids)
return q_outputs, output
else:
+ if self.device_map is not None:
+ accelerate.hooks.remove_hook_from_submodules(
+ block)
mv_module_from_gpu(block, self.low_cpu_mem_usage)
clear_memory(input_ids)
return None, output
@@ -1538,6 +1617,7 @@ class AutoRoundOPT(AutoRound):
to_quant_block_names: Union[str, list] = None,
enable_norm_bias_tuning: bool = False,
enable_torch_compile: bool = None,
+ device_map: Union[str, dict] = None,
optimizer="AdamW",
**kwargs,
):
@@ -1578,6 +1658,7 @@ class AutoRoundOPT(AutoRound):
to_quant_block_names=to_quant_block_names,
enable_norm_bias_tuning=enable_norm_bias_tuning,
enable_torch_compile=enable_torch_compile,
+ device_map=device_map,
**kwargs,
)
@@ -1710,6 +1791,7 @@ class AutoRoundAdam(AutoRoundOPT):
to_quant_block_names: Union[str, list] = None,
enable_norm_bias_tuning: bool = False,
enable_torch_compile: bool = None,
+ device_map: Union[str, dict] = None,
optimizer="AdamW",
**kwargs,
):
@@ -1750,7 +1832,7 @@ class AutoRoundAdam(AutoRoundOPT):
to_quant_block_names=to_quant_block_names,
enable_norm_bias_tuning=enable_norm_bias_tuning,
enable_torch_compile=enable_torch_compile,
+ device_map=device_map,
optimizer=optimizer,
**kwargs,
)
-
diff --git a/auto_round/quantizer.py b/auto_round/quantizer.py
index 3f1eeb884..f44746086 100644
--- a/auto_round/quantizer.py
+++ b/auto_round/quantizer.py
@@ -69,7 +69,8 @@ class WrapperLinear(torch.nn.Module):
"""
super(WrapperLinear, self).__init__()
self.orig_layer = orig_layer
- self.device = device
+ self.output_device = device
+ self.device = self.orig_layer.tuning_device if hasattr(self.orig_layer, "tuning_device") else device
self.enable_minmax_tuning = enable_minmax_tuning
self.enable_norm_bias_tuning = enable_norm_bias_tuning and (orig_layer.bias is not None)
self.enable_act_quant = self.orig_layer.act_bits <= 8
@@ -293,6 +294,7 @@ class WrapperLinear(torch.nn.Module):
Returns:
torch.Tensor: Output tensor after applying the wrapped layer.
"""
+ x = x.to(self.device)
weight_q, _, _ = self._qdq_weight(self.value, self.min_scale, self.max_scale)
if self.enable_act_quant:
@@ -306,7 +308,8 @@ class WrapperLinear(torch.nn.Module):
if self.enable_norm_bias_tuning:
bias, _, _ = self._qdq_bias(bias, self.bias_v)
- return self.orig_forward(x, weight_q, bias)
+ output = self.orig_forward(x, weight_q, bias).to(self.output_device)
+ return output
class WrapperWALayer(torch.nn.Module):
@@ -339,7 +342,8 @@ class WrapperLayerNorm(torch.nn.Module):
self.orig_layer = orig_layer
self.bits = bit
self.group_size = group_size
- self.device = device
+ self.device = self.orig_layer.tuning_device if hasattr(self.orig_layer, "tuning_device") else device
+ self.output_device = device
weight_dtype = torch.float32
self.q_scale_thresh = 1e-5
self.v = torch.nn.Parameter(
@@ -362,11 +366,13 @@ class WrapperLayerNorm(torch.nn.Module):
return self.orig_layer
def forward(self, input):
+ input = input.to(self.device)
weight_q, _, _ = self.quant_func(self.orig_layer.weight, self.bits, self.group_size,
self.v, q_scale_thresh=self.q_scale_thresh)
import torch.nn.functional as F
return F.layer_norm(
- input, self.orig_layer.normalized_shape, weight_q, self.orig_layer.bias, self.orig_layer.eps)
+ input, self.orig_layer.normalized_shape, weight_q, self.orig_layer.bias, self.orig_layer.eps).to(
+ self.output_device)
class WrapperLlamaNorm(torch.nn.Module):
@@ -382,7 +388,8 @@ class WrapperLlamaNorm(torch.nn.Module):
self.orig_layer = orig_layer
self.bits = bit
self.group_size = group_size
- self.device = device
+ self.device = self.orig_layer.tuning_device if hasattr(self.orig_layer, "tuning_device") else device
+ self.output_device = device
weight_dtype = torch.float32
self.q_scale_thresh = 1e-5
self.v = torch.nn.Parameter(
@@ -405,13 +412,14 @@ class WrapperLlamaNorm(torch.nn.Module):
return self.orig_layer
def forward(self, hidden_states):
+ hidden_states = hidden_states.to(self.device)
weight_q, _, _ = self.quant_func(self.orig_layer.weight, self.bits, self.group_size,
self.v, q_scale_thresh=self.q_scale_thresh)
input_dtype = hidden_states.dtype
hidden_states = hidden_states.to(torch.float32)
variance = hidden_states.pow(2).mean(-1, keepdim=True)
hidden_states = hidden_states * torch.rsqrt(variance + self.orig_layer.variance_epsilon)
- return weight_q * hidden_states.to(input_dtype)
+ return (weight_q * hidden_states.to(input_dtype)).to(self.output_device)
norm_mapping = {}
@@ -518,4 +526,3 @@ def unwrapper_block(block, best_params):
best_param = None
orig_layer = m.unwrapper(best_param)
set_module(block, n, orig_layer)
-
diff --git a/auto_round/script/llm.py b/auto_round/script/llm.py
index ba7267a10..7dfe86e94 100644
--- a/auto_round/script/llm.py
+++ b/auto_round/script/llm.py
@@ -49,10 +49,10 @@ class BasicArgumentParser(argparse.ArgumentParser):
self.add_argument("--device", "--devices", default="0", type=str,
help="the device to be used for tuning. "
- "Currently, device settings support CPU, GPU, and HPU."
- "The default is set to cuda:0,"
- "allowing for automatic detection and switch to HPU or CPU."
- "set --device 0,1,2 to use multiple cards.")
+ "Currently, device settings support CPU, GPU, and HPU."
+ "The default is set to cuda:0,"
+ "allowing for automatic detection and switch to HPU or CPU."
+ "set --device 0,1,2 to use multiple cards.")
self.add_argument("--asym", action='store_true',
help="whether to use asym quantization")
@@ -159,6 +159,10 @@ class BasicArgumentParser(argparse.ArgumentParser):
self.add_argument("--disable_act_dynamic", action='store_true',
help="activation static quantization")
+ self.add_argument("--device_map", default=None, type=str,
+ help="device_map for block in tuning phase")
+
+
class EvalArgumentParser(argparse.ArgumentParser):
def __init__(self, *args, **kwargs):
super().__init__(*args, **kwargs)
@@ -166,10 +170,10 @@ class EvalArgumentParser(argparse.ArgumentParser):
help="model name or path")
self.add_argument("--device", "--devices", default="0", type=str,
help="the device to be used for tuning. "
- "Currently, device settings support CPU, GPU, and HPU."
- "The default is set to cuda:0,"
- "allowing for automatic detection and switch to HPU or CPU."
- "set --device 0,1,2 to use multiple cards.")
+ "Currently, device settings support CPU, GPU, and HPU."
+ "The default is set to cuda:0,"
+ "allowing for automatic detection and switch to HPU or CPU."
+ "set --device 0,1,2 to use multiple cards.")
self.add_argument("--tasks",
default="lambada_openai,hellaswag,winogrande,piqa,mmlu,wikitext,truthfulqa_mc1," \
"openbookqa,boolq,arc_easy,arc_challenge",
@@ -178,7 +182,7 @@ class EvalArgumentParser(argparse.ArgumentParser):
help="whether to disable trust_remote_code")
self.add_argument("--eval_bs", "--bs", "--batch_size", default=None, type=int,
help="batch size in evaluation")
-
+
def setup_parser():
parser = BasicArgumentParser()
@@ -254,6 +258,7 @@ def setup_eval_parser():
args = parser.parse_args()
return args
+
def _gguf_args_check(args):
from auto_round.utils import logger
@@ -275,7 +280,7 @@ def _gguf_args_check(args):
formats = args.format.lower().replace(' ', '').split(",")
for format in _GGUF_CONFIG:
if format in formats:
- unsupport_list, reset_list = [],[]
+ unsupport_list, reset_list = [], []
gguf_config = _GGUF_CONFIG[format]
for k, v in gguf_config.items():
if getattr(args, k) != v:
@@ -291,9 +296,10 @@ def _gguf_args_check(args):
logger.error(f"format {format} not support for {', '.join(unsupport_list)},"
f" reset to {', '.join(reset_list)}.")
logger.info(f"export format {format}, sym = {not args.asym}, group_size = {args.group_size}")
-
+
return args
+
def tune(args):
import transformers
@@ -453,7 +459,6 @@ def tune(args):
##TODO gptq could support some mixed precision config
logger.warning(f"mixed precision exporting does not support {format} currently")
-
lm_head_layer_name = "lm_head"
for n, _ in model.named_modules():
lm_head_layer_name = n
@@ -479,7 +484,8 @@ def tune(args):
if "auto_awq" in args.format:
from auto_round.utils import check_awq_gemm_compatibility
- awq_supported, info = check_awq_gemm_compatibility(model,args.bits,args.group_size, not args.asym, layer_config)
+ awq_supported, info = check_awq_gemm_compatibility(model, args.bits, args.group_size, not args.asym,
+ layer_config)
if not awq_supported:
logger.warning(f"The AutoAWQ format may not be supported due to {info}")
@@ -494,7 +500,8 @@ def tune(args):
low_cpu_mem_usage=low_cpu_mem_usage, data_type=args.data_type,
enable_norm_bias_tuning=args.enable_norm_bias_tuning, not_use_best_mse=args.not_use_best_mse,
to_quant_block_names=args.to_quant_block_names, enable_torch_compile=args.enable_torch_compile,
- act_data_type=args.act_data_type, act_dynamic=not args.disable_act_dynamic)
+ act_data_type=args.act_data_type, act_dynamic=not args.disable_act_dynamic,
+ device_map=args.device_map)
model, _ = autoround.quantize()
model_name = args.model.rstrip("/")
if args.low_cpu_mem_mode == 1 or args.low_cpu_mem_mode == 2:
@@ -556,6 +563,7 @@ def tune(args):
batch_size=args.eval_bs)
print(make_table(res))
+
def _eval_init(args):
devices = args.device.replace(" ", "").split(',')
parallelism = False
@@ -596,6 +604,7 @@ def _eval_init(args):
return tasks, model_args, device_str
+
def eval(args):
from auto_round.eval.evaluation import simple_evaluate
@@ -613,7 +622,7 @@ def eval(args):
def eval_sequence(args):
- from auto_round.eval.evaluation import simple_evaluate
+ from auto_round.eval.evaluation import simple_evaluate
tasks, model_args, device_str = _eval_init(args)
from lm_eval.utils import make_table # pylint: disable=E0401
@@ -627,8 +636,8 @@ def eval_sequence(args):
device=device_str,
batch_size=args.eval_bs)
if not res_all:
- res_all = res
+ res_all = res
else:
for key in res_keys:
res_all[key].update(res[key])
- print(make_table(res_all))
\ No newline at end of file
+ print(make_table(res_all))
diff --git a/auto_round/script/mllm.py b/auto_round/script/mllm.py
index 06e6397ab..883cb9a6a 100644
--- a/auto_round/script/mllm.py
+++ b/auto_round/script/mllm.py
@@ -163,6 +163,9 @@ class BasicArgumentParser(argparse.ArgumentParser):
self.add_argument("--to_quant_block_names", default=None, type=str,
help="Names of quantitative blocks, please use commas to separate them.")
+ self.add_argument("--device_map", default=None, type=str,
+ help="device_map for block in tuning phase")
+
def setup_parser():
parser = BasicArgumentParser()
@@ -282,8 +285,8 @@ def tune(args):
# load_model
processor, image_processor = None, None
if "deepseek-vl2" in model_name.lower():
- from deepseek_vl2.models import DeepseekVLV2Processor, DeepseekVLV2ForCausalLM # pylint: disable=E0401
- processor = DeepseekVLV2Processor.from_pretrained(model_name)
+ from deepseek_vl2.models import DeepseekVLV2Processor, DeepseekVLV2ForCausalLM # pylint: disable=E0401
+ processor = DeepseekVLV2Processor.from_pretrained(model_name)
tokenizer = processor.tokenizer
model: DeepseekVLV2ForCausalLM = AutoModelForCausalLM.from_pretrained(
model_name, trust_remote_code=not args.disable_trust_remote_code, torch_dtype=torch_dtype,
@@ -401,10 +404,11 @@ def tune(args):
if "--truncation" not in sys.argv:
args.truncation = None
-
+
if "auto_awq" in args.format:
from auto_round.utils import check_awq_gemm_compatibility
- awq_supported, info = check_awq_gemm_compatibility(model,args.bits,args.group_size, not args.asym, layer_config)
+ awq_supported, info = check_awq_gemm_compatibility(model, args.bits, args.group_size, not args.asym,
+ layer_config)
if not awq_supported:
logger.warning(f"The AutoAWQ format may not be supported due to {info}")
@@ -418,7 +422,8 @@ def tune(args):
scale_dtype=args.scale_dtype, layer_config=layer_config, template=args.template,
enable_minmax_tuning=not args.disable_minmax_tuning, act_bits=args.act_bits,
quant_nontext_module=args.quant_nontext_module, not_use_best_mse=args.not_use_best_mse,
- to_quant_block_names=args.to_quant_block_names, enable_torch_compile=args.enable_torch_compile)
+ to_quant_block_names=args.to_quant_block_names, enable_torch_compile=args.enable_torch_compile,
+ device_map=args.device_map)
model, _ = autoround.quantize()
model.eval()
@@ -527,4 +532,3 @@ def lmms_eval(args):
apply_chat_template=False,
)
return results
-
diff --git a/auto_round/utils.py b/auto_round/utils.py
index 7ba339449..a182a53ec 100644
--- a/auto_round/utils.py
+++ b/auto_round/utils.py
@@ -574,14 +574,14 @@ def detect_device(device=None):
if device is None or device == "auto":
if torch.cuda.is_available():
device = torch.device("cuda")
- logger.info("Using GPU device")
+ # logger.info("Using GPU device")
elif is_optimum_habana_available(): # pragma: no cover
device = torch.device("hpu")
- logger.info("Using HPU device")
+ # logger.info("Using HPU device")
# Use CPU as a fallback
else:
device = torch.device("cpu")
- logger.info("Using CPU device")
+ # logger.info("Using CPU device")
if dev_idx is not None and str(device) != "cpu":
device = str(device) + f":{dev_idx}"
return str(device)
@@ -943,6 +943,9 @@ def get_autogptq_packing_qlinear(backend, bits=4, group_size=128, sym=False):
def _clear_memory_for_cpu_and_cuda(tensor=None):
+ if isinstance(tensor, list):
+ for i in range(len(tensor)):
+ tensor[i] = None
if tensor is not None:
del tensor
gc.collect()
diff --git a/test_cuda/test_multiple_card_tuning.py b/test_cuda/test_multiple_card_tuning.py
new file mode 100644
index 000000000..67254cd71
--- /dev/null
+++ b/test_cuda/test_multiple_card_tuning.py
@@ -0,0 +1,101 @@
+import shutil
+import sys
+import unittest
+from auto_round.eval.evaluation import simple_evaluate
+from lm_eval.utils import make_table # pylint: disable=E0401
+
+sys.path.insert(0, "..")
+from auto_round import AutoRound
+
+import torch
+from transformers import AutoModelForCausalLM, AutoTokenizer
+import re
+
+
+def get_accuracy(data):
+ match = re.search(r'\|acc\s+\|[↑↓]\s+\|\s+([\d.]+)\|', data)
+
+ if match:
+ accuracy = float(match.group(1))
+ return accuracy
+ else:
+ return 0.0
+
+
+# import os
+# os.environ["CUDA_VISIBLE_DEVICES"] = "0,1"
+class TestAutoRound(unittest.TestCase):
+ @classmethod
+ def setUpClass(self):
+ self.save_dir = "./saved"
+ self.tasks = "lambada_openai"
+
+ @classmethod
+ def tearDownClass(self):
+ shutil.rmtree(self.save_dir, ignore_errors=True)
+ shutil.rmtree("runs", ignore_errors=True)
+
+ def test_device_map(self):
+ model_name = "/models/Qwen2-0.5B-Instruct"
+ model = AutoModelForCausalLM.from_pretrained(model_name, torch_dtype=torch.float16)
+ tokenizer = AutoTokenizer.from_pretrained(model_name)
+ device_map = {".*q_proj": '0', ".*k_proj": "cuda:1", "v_proj": 1, ".*up_proj": "cpu"}
+ autoround = AutoRound(model, tokenizer, iters=2, device_map=device_map, nsamples=7,seqlen=32)
+ autoround.quantize()
+
+ def test_device_map_str(self):
+ model_name = "/models/Qwen2-0.5B-Instruct"
+ model = AutoModelForCausalLM.from_pretrained(model_name, torch_dtype=torch.float16)
+ tokenizer = AutoTokenizer.from_pretrained(model_name)
+ device_map = '.*q_proj:0,.*k_proj:cuda:0,v_proj:1,.*up_proj:1'
+ autoround = AutoRound(model, tokenizer,device_map=device_map)
+ autoround.quantize()
+ autoround.save_quantized(self.save_dir, format="auto_round", inplace=False)
+ model_args = f"pretrained={self.save_dir}"
+ res = simple_evaluate(model="hf", model_args=model_args,
+ tasks=self.tasks,
+ batch_size="auto")
+ res = make_table(res)
+ accuracy = get_accuracy(res)
+ print(accuracy)
+ assert accuracy > 0.45 ##0.4786
+ shutil.rmtree("./saved", ignore_errors=True)
+
+ def test_layer_norm(self):
+ model_name = "/models/opt-125m"
+ model = AutoModelForCausalLM.from_pretrained(model_name, torch_dtype=torch.float16)
+ tokenizer = AutoTokenizer.from_pretrained(model_name)
+ device_map = {"norm": "cuda:1"}
+ autoround = AutoRound(model, tokenizer, iters=2, device_map=device_map, nsamples=7, seqlen=32,
+ enable_norm_bias_tuning=True)
+ autoround.quantize()
+
+
+ def test_rms_norm(self):
+ model_name = "/models/Qwen2-0.5B-Instruct"
+ model = AutoModelForCausalLM.from_pretrained(model_name, torch_dtype=torch.float16)
+ tokenizer = AutoTokenizer.from_pretrained(model_name)
+ device_map = {"norm": "cuda:1"}
+ autoround = AutoRound(model, tokenizer, iters=2, device_map=device_map, nsamples=7, seqlen=32,
+ enable_norm_bias_tuning=True)
+ autoround.quantize()
+
+ def test_act_quantization(self):
+ model_name = "/models/Qwen2-0.5B-Instruct"
+ model = AutoModelForCausalLM.from_pretrained(model_name, torch_dtype=torch.float16)
+ tokenizer = AutoTokenizer.from_pretrained(model_name)
+ device_map = {".*q_proj": '0', ".*k_proj": "cuda:1", "v_proj": 1, ".*up_proj": "1"}
+ autoround = AutoRound(model, tokenizer, iters=2, device_map=device_map, nsamples=7,seqlen=32,act_bits=4,act_dynamic=False)
+ autoround.quantize()
+
+ def test_lm_head(self):
+ model_name = "/models/Qwen2.5-7B-Instruct"
+ model = AutoModelForCausalLM.from_pretrained(model_name, torch_dtype=torch.float16)
+ tokenizer = AutoTokenizer.from_pretrained(model_name)
+ device_map = {".*q_proj": '0', ".*k_proj": "cuda:1", "v_proj": 1, ".*up_proj": "1","lm_head":1}
+ layer_config={"lm_head": {"bits": 4}}
+ autoround = AutoRound(model, tokenizer, iters=2, device_map=device_map, nsamples=7, seqlen=32,
+ enable_norm_bias_tuning=True,layer_config=layer_config)
+ autoround.quantize()
+
+