PTQ-Level Efficiency, QAT-Level Accuracy | 训练后量化的效率,量化感知训练的精度
refine code and fix activation quantization eval regression (#762)
Wenhua Cheng305923bc808a
diff --git a/auto_round/autoround.py b/auto_round/autoround.pyindex fed33df34..1df45bf74 100644--- a/auto_round/autoround.py+++ b/auto_round/autoround.py@@ -19,7 +19,7 @@ import reimport sysimport timeimport traceback-from typing import Any, Union+from typing import Any, Callable, Unionimport accelerateimport torch@@ -307,7 +307,7 @@ class AutoRound(object):self.not_use_best_mse = not_use_best_mseself.dynamic_max_gap = dynamic_max_gapself.lr_scheduler = lr_scheduler- self.optimizer = self.get_optimizer(None)+ self.optimizer = self._get_optimizer(None)self.super_bits = super_bitsself.super_group_size = super_group_sizeself.disable_opt_rtn = disable_opt_rtn@@ -345,7 +345,7 @@ class AutoRound(object):self.device_map = device_mapself._set_device_map_in_blocks(self.device_map)self.scale_dtype = convert_dtype_str2torch(scale_dtype)- self.set_amp_dtype()+ self._set_amp_dtype()self.cache_device = torch.device("cpu") if self.low_gpu_mem_usage else self.deviceif self.act_bits <= 8 and self.amp_dtype == torch.float16:logger.warning("Force to use bf16 to for quantization tuning when enabling activation quantization")@@ -537,10 +537,11 @@ class AutoRound(object):if self.nsamples < self.gradient_accumulate_steps * self.batch_size:if self.batch_size > self.nsamples:- logger.warning(- f"Reset `batch_size` to {self.nsamples} as `nsamples`({self.nsamples})"- f" is smaller than batch_size({self.batch_size})"- )+ if self.iters > 0: # GGUF should log this warning, but we don't know the format here+ logger.warning(+ f"Reset `batch_size` to {self.nsamples} as `nsamples`({self.nsamples})"+ f" is smaller than batch_size({self.batch_size})"+ )self.batch_size = self.nsamplesif self.gradient_accumulate_steps > self.nsamples // self.batch_size:self.gradient_accumulate_steps = self.nsamples // self.batch_size@@ -751,7 +752,9 @@ class AutoRound(object):return format- def quantize_and_save(self, output_dir: str = "tmp_autoround", format: str = "auto_round", inplace=True, **kwargs):+ def quantize_and_save(+ self, output_dir: str = "tmp_autoround", format: str = "auto_round", inplace: bool = True, **kwargs+ ) -> tuple[torch.nn.Module, dict[str, Any]]:"""Quantizes the model and saves it in the specified format(s).This function checks the validity of the requested format(s), quantizes@@ -805,14 +808,14 @@ class AutoRound(object):" We recommend exporting to either the AutoAWQ format ( only 4 bits) or ""the AutoRound format(2/3/4/8 bits).")- save_folder = self.get_save_folder_name(format)+ save_folder = self._get_save_folder_name(format)self.save_quantized(save_folder, format=format, inplace=inplace, **kwargs)folders.append(save_folder)return model, folders- def get_save_folder_name(self, format_str: str) -> str:+ def _get_save_folder_name(self, format_str: str) -> str:"""Generates the save folder name based on the provided format string.If there are multiple formats to handle, the function creates a subfolder@@ -835,7 +838,7 @@ class AutoRound(object):return self.orig_output_dir@torch.inference_mode()- def quantize_embedding_layer(self):+ def _quantize_embedding_layer(self):"""Quantizes embedding layers in the model according to the configuration.This method iterates through all modules in the model, identifies embedding@@ -879,7 +882,7 @@ class AutoRound(object):except RuntimeError as e:cuda_error_msg = traceback.format_exc()try:- logger.info("out of VRAM, falling back to CPU.")+ logger.info("out of VRAM, falling back to CPU")weight, scale, zp = quant_func(module.weight.to("cpu"),**{@@ -912,7 +915,7 @@ class AutoRound(object):return is_quantized- def quant_rtn_with_imatrix(self, all_to_quantized_module_names: list[str]) -> None:+ def _quant_rtn_with_imatrix(self, all_to_quantized_module_names: list[str]) -> None:"""Performs RTN quantization using input activation statistics (imatrix).This method accumulates per-channel second-moment activation statistics (imatrix)@@ -926,7 +929,7 @@ class AutoRound(object):Returns:None"""- logger.info("start to compute imatrix for GGUF quantization.")+ logger.info("start to compute imatrix for GGUF quantization")# Load datasetfrom auto_round.calib_dataset import get_dataloader@@ -1015,7 +1018,7 @@ class AutoRound(object):cnt = 1for name in pbar:pbar.set_description(f"Quantizing {name}")- self.quantize_layer_via_rtn(name)+ self._quantize_layer_via_rtn(name)if cnt % clear_mem_freq == 0:clear_memory()cnt = 1@@ -1030,7 +1033,7 @@ class AutoRound(object):logger.warning("Out of VRAM, falling back to blockwise quantization. Accuracy may degrade.")model = model.to("cpu")clear_memory()- self.quantize_via_rtn_blockwise(all_to_quantized_module_names)+ self._quantize_via_rtn_blockwise(all_to_quantized_module_names)except RuntimeError as e:cuda_error_msg = traceback.format_exc()try:@@ -1048,7 +1051,7 @@ class AutoRound(object):orig_device = self.deviceself.device = "cpu"- self.quantize_via_rtn_blockwise(all_to_quantized_module_names)+ self._quantize_via_rtn_blockwise(all_to_quantized_module_names)self.device = orig_deviceexcept Exception as e:logger.error(cuda_error_msg)@@ -1058,11 +1061,7 @@ class AutoRound(object):for hook in hooks:hook.remove()- # Move back to CPU and free memory- model.to("cpu")- clear_memory()-- def check_need_to_quantize_lm_head_embedding(self) -> bool:+ def _check_need_to_quantize_lm_head_embedding(self) -> bool:"""Checks if LM head and embedding layers need quantization for GGUF format.This function inspects the current model's formats and determines whether@@ -1142,7 +1141,7 @@ class AutoRound(object):setattr(get_module(self.model, layer_name), "act_bits", act_bits)setattr(get_module(self.model, layer_name), "scale_dtype", scale_dtype)- def quantize_layer_via_rtn(self, name: str) -> None:+ def _quantize_layer_via_rtn(self, name: str) -> None:"""Quantizes a layer using RTN (Round-To-Nearest) if available.This function attempts to quantize a layer by switching its data type to a@@ -1218,7 +1217,7 @@ class AutoRound(object):if has_gguf:from auto_round.export.export_to_gguf.export import pack_gguf_layer- output_dir = self.get_save_folder_name(self.formats[0])+ output_dir = self._get_save_folder_name(self.formats[0])model_type = ModelType.MMPROJ if self.vlm else ModelType.TEXTpack_gguf_layer(name,@@ -1240,7 +1239,7 @@ class AutoRound(object):set_module(self.model, name, m)@torch.inference_mode()- def quantize_rtn(self) -> tuple[torch.nn.Module, dict[str, Any]]:+ def _quantize_rtn(self) -> tuple[torch.nn.Module, dict[str, Any]]:"""Quantize all modules in the model using RTN (Round-To-Nearest) strategy.If the target format includes GGUF with `k`, and optimized RTN is enabled,@@ -1271,17 +1270,17 @@ class AutoRound(object):has_gguf_k = any("gguf" in fmt and "k" in fmt for fmt in getattr(self, "formats", []))- self.quantize_embedding_layer()+ self._quantize_embedding_layer()self.model.to("cpu")if has_gguf_k and not self.disable_opt_rtn:- self.quant_rtn_with_imatrix(all_to_quantized_module_names)+ self._quant_rtn_with_imatrix(all_to_quantized_module_names)elif self.act_bits <= 8 and check_need_act_calibration(self.act_dynamic, self.act_data_type): ##TODO, mixed datatype has bug- hook_handles = self.register_act_max_hook(self.model)+ hook_handles = self._register_act_max_hook(self.model)try:- self.quantize_via_rtn_blockwise(all_to_quantized_module_names)+ self._quantize_via_rtn_blockwise(all_to_quantized_module_names)except RuntimeError as e:logger.warning("Fallback to CPU. Consider using more GPUs via `--device 0,1,2,3`.")self.model = self.model.to("cpu")@@ -1292,7 +1291,7 @@ class AutoRound(object):accelerate.hooks.remove_hook_from_submodules(self.model)orig_device = self.deviceself.device = "cpu"- self.quantize_via_rtn_blockwise(all_to_quantized_module_names)+ self._quantize_via_rtn_blockwise(all_to_quantized_module_names)self.device = orig_devicefor handle in hook_handles:handle.remove()@@ -1305,7 +1304,7 @@ class AutoRound(object):cnt = 1for name in pbar:pbar.set_description(f"Quantizing {name}")- self.quantize_layer_via_rtn(name)+ self._quantize_layer_via_rtn(name)if cnt % clear_mem_freq == 0:clear_memory()cnt = 1@@ -1316,7 +1315,7 @@ class AutoRound(object):self.quantized = Truereturn self.model, self.layer_config- def quantize_via_rtn_blockwise(self, all_to_quantized_module_names: list[str]) -> None:+ def _quantize_via_rtn_blockwise(self, all_to_quantized_module_names: list[str]) -> None:"""Quantize model layers block by block using cached inputs and imatrix.Args:@@ -1330,7 +1329,7 @@ class AutoRound(object):all_first_block_names = [block[0] for block in all_blocks]if self.act_bits < 16 and not self.act_dynamic:- layer_names = self.get_quantized_layer_names_outside_blocks()+ layer_names = self._get_quantized_layer_names_outside_blocks()if len(layer_names) > 0:logger.warning("quantize layers outside blocks for static activation quantizaiton"@@ -1393,7 +1392,7 @@ class AutoRound(object):add_hook_to_module(m, hook, True)else:block = block.to(self.device)- input_ids = self.get_block_outputs(+ input_ids = self._get_block_outputs(block,input_ids,input_others,@@ -1409,14 +1408,14 @@ class AutoRound(object):) or is_static_afp8(self):from auto_round.utils import set_amax_for_all_moe_layers- # enable moe experts act_max automatic generation for linears+ # enable moe experts act_max automatic generation for Linearset_amax_for_all_moe_layers(block, attr_name="act_max")# Normalize imatrix and quantize layersfor _, m in block.named_modules():if hasattr(m, "imatrix"):m.imatrix /= m.imatrix_cntif hasattr(m, "tmp_name") and m.tmp_name in all_to_quantized_module_names:- self.quantize_layer_via_rtn(m.tmp_name)+ self._quantize_layer_via_rtn(m.tmp_name)all_to_quantized_module_names.remove(m.tmp_name)mv_module_from_gpu(block, self.low_cpu_mem_usage)@@ -1430,13 +1429,13 @@ class AutoRound(object):clear_mem_freq = 1# Process remaining layers not in blocksfor name in all_to_quantized_module_names:- self.quantize_layer_via_rtn(name)+ self._quantize_layer_via_rtn(name)if cnt % clear_mem_freq == 0:clear_memory()cnt = 1cnt += 1- def quantize(self):+ def quantize(self) -> tuple[torch.nn.Module, dict[str, Any]]:"""Quantize the model and return the quantized model along with layer configurations.The entry of AutoRound.Returns:The quantized model and layer configurations.@@ -1444,7 +1443,7 @@ class AutoRound(object):for n, m in self.model.named_modules():m.tmp_name = nself._check_compatibility()- self.has_qlayer_outside_block = self.set_layerwise_config(self.layer_config)+ self.has_qlayer_outside_block = self._set_layerwise_config(self.layer_config)if not hasattr(self, "formats"):logger.warning("this API is deprecated, please use `quantize_and_save` instead")else:@@ -1478,7 +1477,7 @@ class AutoRound(object):):self.is_packing_immediate = Trueif self.iters == 0:- return self.quantize_rtn()+ return self._quantize_rtn()if bool(self.quant_block_list):all_blocks = self.quant_block_list@@ -1492,7 +1491,7 @@ class AutoRound(object):if self.amp:self.model = self.model.to(self.amp_dtype)- layer_names = self.get_quantized_layer_names_outside_blocks()+ layer_names = self._get_quantized_layer_names_outside_blocks()self.start_time = time.time()all_first_block_names = [block[0] for block in all_blocks]if len(layer_names) > 0:@@ -1502,7 +1501,7 @@ class AutoRound(object):else:logger.info("start to cache block inputs")all_inputs = self.try_cache_inter_data_gpucpu(all_first_block_names, self.nsamples, layer_names=layer_names)- is_quantized_embedding = self.quantize_embedding_layer()+ is_quantized_embedding = self._quantize_embedding_layer()all_q_inputs = Noneif is_quantized_embedding:all_inputs = copy.deepcopy(self.inputs)@@ -1547,7 +1546,7 @@ class AutoRound(object):self.batch_size = total_sampleslogger.warning(f"force the train batch size to {total_samples}")- self.quantize_blocks(+ self._quantize_blocks(self.model,inputs,block_names,@@ -1562,7 +1561,7 @@ class AutoRound(object):f"but got {len(self.formats)} formats.")- self.quant_layers(layer_names, all_inputs) ##TODO pack layer immediately+ self._quantize_layers(layer_names, all_inputs) ##TODO pack layer immediatelyif hasattr(self.model, "is_fp8"):for n, m in self.model.named_modules():@@ -1595,7 +1594,7 @@ class AutoRound(object):self.quantized = Truereturn self.model, self.layer_config- def quant_layers(self, layer_names, layer_inputs):+ def _quantize_layers(self, layer_names: list, layer_inputs: dict) -> None:"""Quantizes specified layers based on inputs and configuration.Args:@@ -1660,9 +1659,9 @@ class AutoRound(object):self.model = mv_module_from_gpu(self.model, self.low_cpu_mem_usage)clear_memory()if self.enable_torch_compile:- quant_layer = compile_func(self.quant_layer, self.device)+ quant_layer = compile_func(self._quantize_layer, self.device)else:- quant_layer = self.quant_layer+ quant_layer = self._quantize_layerfor layer_name in layer_names:layer_input = layer_inputs[layer_name]layer_input = to_device(layer_input, self.cache_device)@@ -1672,7 +1671,7 @@ class AutoRound(object):del layer_inputclear_memory(q_layer_input)- def set_layerwise_config(self, layer_config):+ def _set_layerwise_config(self, layer_config: dict) -> bool:"""Sets the layer-wise configuration based on the provided `layer_config`.By default, only quantize layers in blocks.@@ -1786,7 +1785,7 @@ class AutoRound(object):# Apply the configuration to the corresponding layer in the modelfor key in keys:setattr(m, key, layer_config[n][key])- need_to_quantize_lm_head = self.check_need_to_quantize_lm_head_embedding()+ need_to_quantize_lm_head = self._check_need_to_quantize_lm_head_embedding()if need_to_quantize_lm_head:has_qlayer_outside_block = True@@ -1794,7 +1793,16 @@ class AutoRound(object):return has_qlayer_outside_block@torch.no_grad()- def get_block_outputs(self, block, input_ids, input_others, bs, device, cache_device, save_output=True):+ def _get_block_outputs(+ self,+ block: torch.nn.Module,+ input_ids: torch.Tensor,+ input_others: torch.Tensor,+ bs: int,+ device: Union[str, torch.device],+ cache_device: Union[str, torch.device],+ save_output: bool = True,+ ):"""Compute the output of a given block of the model for a given input.Args:@@ -1815,7 +1823,7 @@ class AutoRound(object):for i in range(0, nsamples, bs):end_index = min(nsamples, i + bs)indices = torch.arange(i, end_index).to(torch.long)- tmp_input_ids, tmp_input_others = AutoRound.sampling_inputs(+ tmp_input_ids, tmp_input_others = AutoRound._sampling_inputs(input_ids, input_others, indices, self.seqlen, self.batch_dim, share_cache_keys=self.shared_cache_keys)tmp_output = block_forward(block, tmp_input_ids, tmp_input_others, self.amp, self.amp_dtype, device).to(@@ -2046,7 +2054,7 @@ class AutoRound(object):return res@torch.no_grad()- def get_block_forward_func(self, name):+ def _get_block_forward_func(self, name: str) -> Callable:"""Gets the forward function.Args:@@ -2192,13 +2200,15 @@ class AutoRound(object):for n, m in self.model.named_modules():if n in self.to_cached_layers and not isinstance(m, tuple(self.supported_types)): ##blockm.orig_forward = m.forward- m.forward = partial(self.get_block_forward_func(n), m)+ m.forward = partial(self._get_block_forward_func(n), m)elif n in self.to_cached_layers: ##linear layer or conv1d layerhook_func = self._get_cache_data_hook_for_layer(n)hook_handle = m.register_forward_hook(hook_func)self.hook_handles.append(hook_handle)- def quant_layer(self, layer_name, inputs, q_inputs=None, device=torch.device("cpu")):+ def _quantize_layer(+ self, layer_name: str, inputs: torch.Tensor, q_inputs: torch.Tensor = None, device: str = "cpu"+ ):"""Quantize a specific layer of the model using the provided inputs.Args:@@ -2255,7 +2265,7 @@ class AutoRound(object):last_best_iter = 0best_loss = torch.finfo(torch.float).maxmse_loss = torch.nn.MSELoss().to(device)- scaler = self.get_scaler() # pylint: disable=assignment-from-none+ scaler = self._get_scaler() # pylint: disable=assignment-from-noneinit_loss = None# best_v, best_min_scale, best_max_scale = torch.tensor(0), torch.tensor(1.0), torch.tensor(1.0)gradient_accumulate_steps = self.batch_size ##Force to low gpu@@ -2306,7 +2316,7 @@ class AutoRound(object):)total_loss += loss.item() / num_elm- self.scale_loss_and_backward(scaler, loss)+ self._scale_loss_and_backward(scaler, loss)if i == 0:init_loss = total_loss@@ -2321,7 +2331,7 @@ class AutoRound(object):if not self.not_use_best_mse:if 0 < self.dynamic_max_gap <= i - last_best_iter:break- self.step(scaler, optimizer, lr_schedule)+ self._step(scaler, optimizer, lr_schedule)last_loss = total_lossbest_iter = self.iters@@ -2334,7 +2344,7 @@ class AutoRound(object):dump_info = f"quantized {layer_name}, loss iter 0: {init_loss:.6f} -> iter {best_iter}: {last_loss:.6f}"logger.info(dump_info)- def register_act_max_hook(self, model):+ def _register_act_max_hook(self, model):def get_act_max_hook(module, input, output):if isinstance(input, (tuple, list)):input = input[0]@@ -2380,7 +2390,14 @@ class AutoRound(object):continuereturn hook_handles- def quantize_block(self, block, input_ids, input_others, q_input=None, device=torch.device("cpu")):+ def _quantize_block(+ self,+ block: torch.nn.Module,+ input_ids: list[torch.Tensor],+ input_others: dict,+ q_input: Union[None, torch.Tensor] = None,+ device: Union[str, torch.device] = "cpu",+ ):"""Quantize the weights of a given block of the model.Args:@@ -2411,21 +2428,21 @@ class AutoRound(object):add_hook_to_module(m, hook, True)if q_input is None:- hook_handles = self.register_act_max_hook(block)+ hook_handles = self._register_act_max_hook(block)- output = self.get_block_outputs(+ output = self._get_block_outputs(block, input_ids, input_others, self.batch_size * self.infer_bs_coeff, device, self.cache_device)for handle in hook_handles:handle.remove()else:- output = self.get_block_outputs(+ output = self._get_block_outputs(block, input_ids, input_others, self.batch_size * self.infer_bs_coeff, device, self.cache_device)- hook_handles = self.register_act_max_hook(block)+ hook_handles = self._register_act_max_hook(block)if hook_handles:- self.get_block_outputs(+ self._get_block_outputs(block,q_input,input_others,@@ -2502,7 +2519,7 @@ class AutoRound(object):if self.gradient_accumulate_steps != 1:mse_reduction = "sum"mse_loss = torch.nn.MSELoss(reduction=mse_reduction).to(device)- scaler = self.get_scaler() # pylint: disable=assignment-from-none+ scaler = self._get_scaler() # pylint: disable=assignment-from-noneinit_loss = Nonebest_params = {}total_loss = 0@@ -2517,7 +2534,7 @@ class AutoRound(object):for tmp_step in range(self.gradient_accumulate_steps):indices = whole_indices[tmp_step * self.batch_size : (tmp_step + 1) * self.batch_size]- current_input_ids, current_input_others = AutoRound.sampling_inputs(+ current_input_ids, current_input_others = AutoRound._sampling_inputs(input_ids,input_others,indices,@@ -2542,7 +2559,7 @@ class AutoRound(object):)total_loss += loss.item() / num_elm- self.scale_loss_and_backward(scaler, loss)+ self._scale_loss_and_backward(scaler, loss)if i == 0:init_loss = total_loss@@ -2560,7 +2577,7 @@ class AutoRound(object):if not self.not_use_best_mse:if 0 < self.dynamic_max_gap <= i - last_best_iter:break- self.step(scaler, optimizer, lr_schedule)+ self._step(scaler, optimizer, lr_schedule)last_loss = total_lossbest_iter = self.iters@@ -2584,7 +2601,7 @@ class AutoRound(object):if self.low_cpu_mem_usage:block = block.to(device)clear_memory()- q_outputs = self.get_block_outputs(+ q_outputs = self._get_block_outputs(block,input_ids,input_others,@@ -2606,8 +2623,15 @@ class AutoRound(object):clear_memory(input_ids)return None, output- def quantize_blocks(- self, model: torch.nn.Module, inputs, block_names, q_input=None, nblocks=1, device="cpu", pbar=None+ def _quantize_blocks(+ self,+ model: torch.nn.Module,+ inputs: dict,+ block_names: list,+ q_input: torch.Tensor = None,+ nblocks: int = 1,+ device: str = "cpu",+ pbar: tqdm = None,):"""Quantize and dequantize the weights of the specified blocks in the model.@@ -2663,13 +2687,13 @@ class AutoRound(object):else:logger.info("using algorithm extension for quantization.")except (ImportError, ModuleNotFoundError):- quantize_block = self.quantize_block+ quantize_block = self._quantize_blockif self.enable_torch_compile:quantize_block = compile_func(quantize_block, device)else:quantize_block = quantize_blockelse:- quantize_block = self.quantize_block+ quantize_block = self._quantize_blockif self.enable_torch_compile:quantize_block = compile_func(quantize_block, device)@@ -2703,27 +2727,28 @@ class AutoRound(object):from auto_round.export import PACKING_LAYER_WITH_FORMATfor _, tmp_m in m.named_modules():- if hasattr(tmp_m, "bits") and check_to_quantized(tmp_m):- target_backend = self.formats[0].split(":")[0] if ":" in self.formats[0] else self.formats[0]- has_gguf = any("gguf" in format_ for format_ in self.formats)- if has_gguf:- from auto_round.export.export_to_gguf.export import pack_gguf_layer-- output_dir = self.get_save_folder_name(self.formats[0])- model_type = ModelType.MMPROJ if self.vlm else ModelType.TEXT- pack_gguf_layer(- tmp_m.tmp_name,- self.model,- self.formats[0],- output_dir,- self.layer_config,- self.tokenizer,- processor=self.processor if hasattr(self, "processor") else None,- image_processor=self.image_processor if hasattr(self, "image_processor") else None,- model_type=model_type,- )- else:- PACKING_LAYER_WITH_FORMAT[target_backend](tmp_m.tmp_name, self.model, self.formats[0])+ if not (hasattr(tmp_m, "bits") and check_to_quantized(tmp_m)):+ continue+ target_backend = self.formats[0].split(":")[0] if ":" in self.formats[0] else self.formats[0]+ has_gguf = any("gguf" in format_ for format_ in self.formats)+ if has_gguf:+ from auto_round.export.export_to_gguf.export import pack_gguf_layer++ output_dir = self._get_save_folder_name(self.formats[0])+ model_type = ModelType.MMPROJ if self.vlm else ModelType.TEXT+ pack_gguf_layer(+ tmp_m.tmp_name,+ self.model,+ self.formats[0],+ output_dir,+ self.layer_config,+ self.tokenizer,+ processor=self.processor if hasattr(self, "processor") else None,+ image_processor=self.image_processor if hasattr(self, "image_processor") else None,+ model_type=model_type,+ )+ else:+ PACKING_LAYER_WITH_FORMAT[target_backend](tmp_m.tmp_name, self.model, self.formats[0])pbar.set_description("Quantizing done")pbar.update(1)pbar.close()@@ -2739,7 +2764,9 @@ class AutoRound(object):clear_memory()- def save_quantized(self, output_dir=None, format="auto_round", inplace=True, **kwargs):+ def save_quantized(+ self, output_dir: str = None, format: str = "auto_round", inplace: bool = True, **kwargs+ ) -> torch.nn.Module:"""Save the quantized model to the specified output directory in the specified format.Args:@@ -2832,7 +2859,7 @@ class AutoRound(object):)return compressed_model- def get_quantized_layer_names_outside_blocks(self):+ def _get_quantized_layer_names_outside_blocks(self) -> list:"""Gets the names of quantized layers outside blocks in the model.Returns:@@ -2856,7 +2883,8 @@ class AutoRound(object):return layer_names- def set_amp_dtype(self):+ def _set_amp_dtype(self) -> None:+ """Sets the automatic mixed precision (AMP) data type for the model based on the device and configuration."""self.amp_dtype = torch.float16if self.model.dtype != torch.float32:self.amp_dtype = self.model.dtype@@ -2876,7 +2904,7 @@ class AutoRound(object):self.amp_dtype = torch.float32self.model = self.model.to(torch.float32)- def get_optimizer(self, optimizer):+ def _get_optimizer(self, optimizer: Any):"""Returns the specified optimizer. In SignRound, we fix the optimizer.Args:@@ -2887,11 +2915,11 @@ class AutoRound(object):"""return SignSGD- def get_scaler(self):+ def _get_scaler(self):"""Returns scaler, in SignRound, no need to use scaler."""return None- def scale_loss_and_backward(self, scaler, loss):+ def _scale_loss_and_backward(self, scaler: Any, loss: torch.Tensor) -> torch.Tensor:"""Scales the loss and performs backward pass.Args:@@ -2907,7 +2935,7 @@ class AutoRound(object):htcore.mark_step()return scale_loss- def step(self, scaler, optimizer, lr_schedule):+ def _step(self, scaler: Any, optimizer: Any, lr_schedule: Any):"""Performs a step in the optimization process.Args:@@ -2927,7 +2955,15 @@ class AutoRound(object):@classmethod@torch.no_grad()- def sampling_inputs(cls, input_ids, input_others, indices, seqlen, batch_dim=0, share_cache_keys=()):+ def _sampling_inputs(+ cls,+ input_ids: list[torch.Tensor],+ input_others: dict,+ indices: list[int],+ seqlen: int,+ batch_dim: int = 0,+ share_cache_keys: tuple = (),+ ):"""Samples inputs based on the given indices and sequence length.Args:@@ -3105,9 +3141,9 @@ class AutoRoundAdam(AutoRound):**kwargs,)- self.optimizer = self.get_optimizer(optimizer)+ self.optimizer = self._get_optimizer(optimizer)- def get_optimizer(self, optimizer):+ def _get_optimizer(self, optimizer):if optimizer is None:optimizer = torch.optim.AdamWelif isinstance(optimizer, str):@@ -3116,7 +3152,7 @@ class AutoRoundAdam(AutoRound):optimizer = optimizerreturn optimizer- def get_scaler(self):+ def _get_scaler(self):scaler = Noneif self.amp and not check_is_cpu(self.device):from torch.cuda.amp import GradScaler@@ -3124,7 +3160,7 @@ class AutoRoundAdam(AutoRound):scaler = GradScaler(init_scale=1024, growth_interval=100000)return scaler- def scale_loss_and_backward(self, scaler, loss):+ def _scale_loss_and_backward(self, scaler, loss):if scaler is not None:loss = scaler.scale(loss)@@ -3133,7 +3169,7 @@ class AutoRoundAdam(AutoRound):htcore.mark_step()return loss- def step(self, scaler, optimizer, lr_schedule):+ def _step(self, scaler, optimizer, lr_schedule):if scaler is not None:scaler.step(optimizer)optimizer.zero_grad()diff --git a/auto_round/inference/backend.py b/auto_round/inference/backend.pyindex ea66a0389..1569e8005 100644--- a/auto_round/inference/backend.py+++ b/auto_round/inference/backend.py@@ -816,8 +816,8 @@ def process_requirement(requirements: list, target_device="cuda", logger_level="except:missing_requirements.append(req)- gptq_req = next((req for req in missing_requirements if "gptqmodel" in req), None)- other_reqs = [req for req in missing_requirements if "gptqmodel" not in req]+ gptq_req = next((f'"{req}"' for req in missing_requirements if "gptqmodel" in req), None)+ other_reqs = [f'"{req}"' for req in missing_requirements if "gptqmodel" not in req]pip_cmds = build_pip_commands(gptq_req, other_reqs)if not pip_cmds:diff --git a/auto_round/script/llm.py b/auto_round/script/llm.pyindex 4b7cc641c..3caefcb85 100644--- a/auto_round/script/llm.py+++ b/auto_round/script/llm.py@@ -634,7 +634,11 @@ def tune(args):eval_model_dtype = get_model_dtype(args.eval_model_dtype, "auto")tmp_act_bits = infer_bits_by_data_type(args.act_data_type)- if tmp_act_bits is not None and (args.act_bits <= 8 or tmp_act_bits <= 8 or eval_gguf_model):+ if tmp_act_bits is not None:+ act_bits = tmp_act_bits+ else:+ act_bits = args.act_bits+ if act_bits <= 8 or eval_gguf_model:if eval_gguf_model:# for file in os.listdir(eval_folder):# gguf_file = filediff --git a/auto_round/special_model_handler.py b/auto_round/special_model_handler.pyindex cf0bb90a5..e6d48567e 100644--- a/auto_round/special_model_handler.py+++ b/auto_round/special_model_handler.py@@ -11,7 +11,7 @@# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.# See the License for the specific language governing permissions and# limitations under the License.-+from auto_round.utils import loggermllms_with_limited_bs = ("llava", "qwen2_vl", "phi3_v", "mllama") # Limitations on batch_size@@ -129,7 +129,7 @@ def _handle_moe_model(model):parent = model.get_submodule(parent)setattr(parent, child, new_module)- print("warning, Llama4 experts are converted, the quantized model can not run on transformers.")+ logger.warning("Llama4 experts are converted, the quantized model can not run on transformers.")return model@@ -192,7 +192,7 @@ def check_mllm_model_batch(model, batch_size, gradient_accumulate_steps=1):for key in mllms_with_limited_bs:if hasattr(model, "config") and key in model.config.model_type and batch_size != 1:accumulate_steps = batch_size * gradient_accumulate_steps- print(+ logger.warning("To avoid the tensor concat mismatch problem, modified parameters to "f"batch_size=1. As an alternative, set the gradient_accumulate_steps={accumulate_steps}")diff --git a/auto_round/utils.py b/auto_round/utils.pyindex 74999c624..267f56005 100644--- a/auto_round/utils.py+++ b/auto_round/utils.py@@ -24,7 +24,7 @@ import sysfrom collections import UserDictfrom enum import Enumfrom functools import lru_cache-from typing import Any, Tuple, Union+from typing import Any, Callable, Tuple, Unionimport cpuinfoimport torch@@ -33,11 +33,9 @@ from packaging import versionfrom torch.amp import autocastfrom auto_round.export.export_to_gguf.config import GGML_QUANT_SIZES, GGUF_CONFIG, GGUF_INNER_CONFIG, QK_K, ModelType-from auto_round.special_model_handler import SPECIAL_MULTIMODAL_BLOCK, SPECIAL_SHARED_CACHE_KEYSSHARED_CACHE_KEYS = ("position_ids", "cache_position", "position_embeddings")-deepspeed_exists = Falseif importlib.util.find_spec("deepspeed"): # check if deepspeed is installeddeepspeed_exists = True@@ -421,6 +419,7 @@ def get_block_names(model, quant_vision=False):Returns:block_names: A list whose elements are list of block's layer names"""+ from auto_round.special_model_handler import SPECIAL_MULTIMODAL_BLOCKdef _get_llm_block_names(model):block_names = []@@ -952,6 +951,7 @@ TORCH_VERSION_AT_LEAST_2_6 = torch_version_at_least("2.6.0")TORCH_VERSION_AT_LEAST_2_5 = torch_version_at_least("2.5.0")TORCH_VERSION_AT_LEAST_2_4 = torch_version_at_least("2.4.0")+# Note on HPU usage:# There are two modes available for enabling auto-round on HPU:# 1. Compile Mode@@ -979,7 +979,10 @@ def compile_func_on_cuda_or_cpu(func):return torch.compile(func)-def compile_func(fun, device):+def compile_func(+ fun: Union[torch.nn.Module, Callable], device: Union[torch.nn.Module, Callable]+) -> Union[torch.nn.Module, Callable]:+ """Compile function on the specified device."""if "hpu" in str(device):return compile_func_on_hpu(fun) ## use auto by defaultelse:@@ -1481,7 +1484,6 @@ def mllm_load_model(model_dtype=None,**kwargs,):-import transformersfrom huggingface_hub import HfApi, HfFileSystem, hf_hub_downloadfrom transformers import AutoModel, AutoModelForCausalLM, AutoProcessor, AutoTokenizer@@ -1668,6 +1670,8 @@ def get_shared_keys(model):Returns:tuple: tuple of shared keys."""+ from auto_round.special_model_handler import SPECIAL_SHARED_CACHE_KEYS+shared_keys = SHARED_CACHE_KEYSshared_keys += SPECIAL_SHARED_CACHE_KEYS.get(model.__class__.__name__, ())return shared_keys@@ -2201,7 +2205,6 @@ def unpad_weight(weight: torch.Tensor, original_M: int, original_N: int, keep_fidef pad_block_fp8_weight_naive(weight: torch.Tensor, weight_scale: torch.Tensor, block_size: list) -> Tuple[torch.Tensor, int, int]:-assert len(block_size) == 2block_size_m, block_size_n = block_sizediff --git a/auto_round/wrapper.py b/auto_round/wrapper.pyindex 74efad368..9afc36fe9 100644--- a/auto_round/wrapper.py+++ b/auto_round/wrapper.py@@ -145,6 +145,7 @@ class WrapperLinear(torch.nn.Module):self._init_params("max_scale", p_dtype, shape, 1.0, (self.enable_minmax_tuning and self.orig_layer.bits < 16))self.weight_quant_func, self.data_type = get_quant_func(orig_layer.data_type, orig_layer.bits, orig_layer.sym)+if self.enable_act_quant:self.act_quant_func, self.act_data_type = get_quant_func(orig_layer.act_data_type, orig_layer.act_bits, orig_layer.act_symdiff --git a/auto_round_extension/torch/qlinear_torch.py b/auto_round_extension/torch/qlinear_torch.pyindex 1a1e0c8aa..f5e32507e 100644--- a/auto_round_extension/torch/qlinear_torch.py+++ b/auto_round_extension/torch/qlinear_torch.py@@ -86,7 +86,8 @@ class QuantLinear(nn.Module):def post_init(self):pass- def pack(self, linear, scales, zeros, g_idx=None):+ # @torch.compile() ## cpu side has bug+ def pack_248_bits(self, linear, scales, zeros, g_idx=None):scales_t = scales.t().contiguous()if linear.bias is not None:self.bias = linear.bias.clone().half()@@ -115,119 +116,166 @@ class QuantLinear(nn.Module):del repeat_scales- if self.bits in [2, 4, 8]:- intweight = intweight.reshape(-1, intweight.shape[1] // 32 * self.bits, 32 // self.bits)- order_map = torch.arange(0, 32 // self.bits, device=device) * self.bits- intweight = intweight.to(torch.int32)- intweight = intweight << order_map- intweight = torch.sum(intweight, dim=-1)-- intweight = intweight.t().contiguous().to(torch.int32)- self.qweight = intweight.to("cpu")- elif self.bits == 3:- intweight = intweight.t().contiguous()- intweight = intweight.cpu().numpy().astype(np.uint32)- i = 0- row = 0- qweight = torch.zeros((intweight.shape[0] // 32 * self.bits, intweight.shape[1]), dtype=torch.int32)- while row < qweight.shape[0]:- packed_weight = torch.tensor(intweight[i : i + 10]).to(dtype=torch.int32).t()- shifts = torch.arange(0, 10) * self.bits- shifted = packed_weight << shifts- qweight[row] |= shifted.sum(dim=-1)- i += 10- qweight[row] |= intweight[i] << 30- row += 1- qweight[row] |= (intweight[i] >> 2) & 1- i += 1- packed_weight = torch.tensor(intweight[i : i + 10]).to(dtype=torch.int32).t()- shifts = torch.arange(0, 10) * self.bits + 1- shifted = packed_weight << shifts- qweight[row] |= shifted.sum(dim=-1)- i += 10- qweight[row] |= intweight[i] << 31- row += 1- qweight[row] |= (intweight[i] >> 1) & 0x3- i += 1- packed_weight = torch.tensor(intweight[i : i + 10]).to(dtype=torch.int32).t()- shifts = torch.arange(0, 10) * self.bits + 2- shifted = packed_weight << shifts- qweight[row] |= shifted.sum(dim=-1)- i += 10- row += 1+ intweight = intweight.reshape(-1, intweight.shape[1] // 32 * self.bits, 32 // self.bits)+ order_map = torch.arange(0, 32 // self.bits, device=device) * self.bits+ intweight = intweight.to(torch.int32)+ intweight = intweight << order_map+ intweight = torch.sum(intweight, dim=-1)- self.qweight = qweight.cpu()+ intweight = intweight.t().contiguous().to(torch.int32)+ self.qweight = intweight.to("cpu")if isinstance(zeros, torch.Tensor):zeros = zeros.t().contiguous()- zeros = zeros.numpy().astype(np.uint32)- qzeros = torch.zeros((zeros.shape[0], zeros.shape[1] // 32 * self.bits), dtype=torch.int32)+ zeros -= 1+ # zeros = zeros.numpy().astype(np.uint32)+ qzeros = torch.zeros(+ (zeros.shape[0], zeros.shape[1] // 32 * self.bits), device=self.device, dtype=torch.int32+ )i = 0col = 0while col < qzeros.shape[1]:- if self.bits in [2, 4, 8]:- packed_zeros = torch.tensor(zeros[:, i : i + (32 // self.bits)]).to(dtype=torch.int32)- shifts = torch.arange(0, (32 // self.bits)) * self.bits- shifted = packed_zeros << shifts- qzeros[:, col] |= shifted.sum(dim=-1)- i += 32 // self.bits- col += 1- elif self.bits == 3:- packed_zeros = torch.tensor(zeros[:, i : i + 10]).to(dtype=torch.int32)- shifts = torch.arange(0, 10) * self.bits- shifted = packed_zeros << shifts- qzeros[:, col] = shifted.sum(dim=-1)- i += 10- qzeros[:, col] |= zeros[:, i] << 30- col += 1- qzeros[:, col] |= (zeros[:, i] >> 2) & 1- i += 1- packed_zeros = torch.tensor(zeros[:, i : i + 10]).to(dtype=torch.int32)- shifts = torch.arange(0, 10) * self.bits + 1- shifted = packed_zeros << shifts- qzeros[:, col] |= shifted.sum(dim=-1)- i += 10- qzeros[:, col] |= zeros[:, i] << 31- col += 1- qzeros[:, col] |= (zeros[:, i] >> 1) & 0x3- i += 1- packed_zeros = torch.tensor(zeros[:, i : i + 10]).to(dtype=torch.int32)- shifts = torch.arange(0, 10) * self.bits + 2- shifted = packed_zeros << shifts- qzeros[:, col] |= shifted.sum(dim=-1)- i += 10- col += 1+ packed_zeros = torch.tensor(zeros[:, i : i + (32 // self.bits)]).to(dtype=torch.int32)+ shifts = torch.arange(0, (32 // self.bits)) * self.bits+ shifted = packed_zeros << shifts+ qzeros[:, col] |= shifted.sum(dim=-1)+ i += 32 // self.bits+ col += 1+ self.qzeros = qzeros.cpu()+ else:+ zeros -= 1+ shape = scales_t.shape+ value = 0+ for j in range(0, (32 // self.bits)):+ value |= zeros << (self.bits * j)+ qzeros = torch.ones((shape[0], shape[1] // 32 * self.bits), dtype=torch.int32) * value+ self.qzeros = qzeros.cpu()++ # @torch.compile()+ def pack_3bits(self, linear, scales, zeros, g_idx=None):+ scales_t = scales.t().contiguous()+ if linear.bias is not None:+ self.bias = linear.bias.clone().half()+ self.scales = scales_t.clone().half()+ device = "cpu"+ if torch.cuda.is_available():+ device = "cuda:0"+ elif torch.xpu.is_available():+ device = "xpu:0"+ W = linear.weight.data.to(device).clone()+ if isinstance(linear, nn.Conv2d):+ W = W.flatten(1)+ if isinstance(linear, transformers.pytorch_utils.Conv1D):+ W = W.t()++ repeat_scales = scales.to(device).repeat_interleave(self.group_size, 1)+ if isinstance(zeros, torch.Tensor):+ repeat_zeros = zeros.to(device).repeat_interleave(self.group_size, 1)+ intweight = torch.round(W.to(device) / repeat_scales[:, : W.shape[1]] + repeat_zeros[:, : W.shape[1]]).to(+ torch.int32+ )+ else:+ repeat_zeros = zeros+ intweight = torch.round(W.to(device) / repeat_scales[:, : W.shape[1]] + repeat_zeros).to(torch.int32)++ del repeat_scales++ intweight = intweight.t().contiguous().to(torch.int32)+ i = 0+ row = 0+ qweight = torch.zeros(+ (intweight.shape[0] // 32 * self.bits, intweight.shape[1]), dtype=torch.int32, device=device+ )+ while row < qweight.shape[0]:+ packed_weight = (intweight[i : i + 10]).to(dtype=torch.int32).t()+ shifts = torch.arange(0, 10).to(device) * self.bits+ shifted = packed_weight << shifts+ qweight[row] |= shifted.sum(dim=-1)+ i += 10+ qweight[row] |= intweight[i] << 30+ row += 1+ qweight[row] |= (intweight[i] >> 2) & 1+ i += 1+ packed_weight = (intweight[i : i + 10]).to(dtype=torch.int32).t()+ shifts = torch.arange(0, 10).to(device) * self.bits + 1+ shifted = packed_weight << shifts+ qweight[row] |= shifted.sum(dim=-1)+ i += 10+ qweight[row] |= intweight[i] << 31+ row += 1+ qweight[row] |= (intweight[i] >> 1) & 0x3+ i += 1+ packed_weight = (intweight[i : i + 10]).to(dtype=torch.int32).t()+ shifts = torch.arange(0, 10).to(device) * self.bits + 2+ shifted = packed_weight << shifts+ qweight[row] |= shifted.sum(dim=-1)++ i += 10+ row += 1++ self.qweight = qweight.cpu()++ if isinstance(zeros, torch.Tensor):+ zeros = zeros.t().contiguous().to(device).to(torch.int32)+ # zeros = zeros.numpy().astype(np.uint32)+ qzeros = torch.zeros((zeros.shape[0], zeros.shape[1] // 32 * self.bits), device=device, dtype=torch.int32)+ i = 0+ col = 0+ while col < qzeros.shape[1]:+ packed_zeros = (zeros[:, i : i + 10]).to(dtype=torch.int32)+ shifts = torch.arange(0, 10).to(device) * self.bits+ shifted = packed_zeros << shifts+ qzeros[:, col] = shifted.sum(dim=-1)+ i += 10+ qzeros[:, col] |= zeros[:, i] << 30+ col += 1+ qzeros[:, col] |= (zeros[:, i] >> 2) & 1+ i += 1+ # packed_zeros = (zeros[:, i: i + 10]).to(dtype=torch.int32)+ shifts = torch.arange(0, 10).to(device) * self.bits + 1+ shifted = packed_zeros << shifts+ qzeros[:, col] |= shifted.sum(dim=-1)+ i += 10+ qzeros[:, col] |= zeros[:, i] << 31+ col += 1+ qzeros[:, col] |= (zeros[:, i] >> 1) & 0x3+ i += 1+ # packed_zeros = (zeros[:, i: i + 10]).to(dtype=torch.int32)+ shifts = torch.arange(0, 10).to(device) * self.bits + 2+ shifted = packed_zeros << shifts+ qzeros[:, col] |= shifted.sum(dim=-1)+ i += 10+ col += 1+ self.qzeros = qzeros.cpu()+ else:+ shape = scales_t.shape[0], scales_t.shape[1] // 32 * self.bits+ qzeros = torch.zeros(shape, dtype=torch.int32)+ zero_val = zeros+ total_cols = shape[1]+ # Precompute shifts for 3-bit packing+ shifts0 = torch.arange(0, 10, dtype=torch.int32) * self.bits+ shifts1 = shifts0 + 1+ shifts2 = shifts0 + 2+ # Compute packed pattern parts+ part0 = (zero_val << shifts0).sum().to(torch.int32) | (zero_val << 30)+ part1 = (zero_val << shifts1).sum().to(torch.int32) | (zero_val << 31) | ((zero_val >> 2) & 1)+ part2 = (zero_val << shifts2).sum().to(torch.int32) | ((zero_val >> 1) & 0x3)+ pattern = torch.tensor([part0, part1, part2], dtype=torch.int32)+ # Tile pattern across all columns+ repeats = (total_cols + 2) // 3+ full_row = pattern.repeat(repeats)[:total_cols]+ # Broadcast across rows+ qzeros[:] = full_row.unsqueeze(0)self.qzeros = qzeros.cpu()++ def pack(self, linear, scales, zeros, g_idx=None):+ if self.bits in [2, 4, 8]:+ return self.pack_248_bits(linear, scales, zeros, g_idx)+ elif self.bits in [3]:+ return self.pack_3bits(linear, scales, zeros, g_idx)else:- if self.bits in [2, 4, 8]:- shape = scales_t.shape- value = 0- for j in range(0, (32 // self.bits)):- value |= zeros << (self.bits * j)- qzeros = np.ones((shape[0], shape[1] // 32 * self.bits), dtype=np.uint32) * value- qzeros = qzeros.astype(np.int32)- self.qzeros = torch.from_numpy(qzeros)- else: ## bits == 3- shape = scales_t.shape[0], scales_t.shape[1] // 32 * self.bits- qzeros = torch.zeros(shape, dtype=torch.int32)- zero_val = zeros- total_cols = shape[1]- # Precompute shifts for 3-bit packing- shifts0 = torch.arange(0, 10, dtype=torch.int32) * self.bits- shifts1 = shifts0 + 1- shifts2 = shifts0 + 2- # Compute packed pattern parts- part0 = (zero_val << shifts0).sum().to(torch.int32) | (zero_val << 30)- part1 = (zero_val << shifts1).sum().to(torch.int32) | (zero_val << 31) | ((zero_val >> 2) & 1)- part2 = (zero_val << shifts2).sum().to(torch.int32) | ((zero_val >> 1) & 0x3)- pattern = torch.tensor([part0, part1, part2], dtype=torch.int32)- # Tile pattern across all columns- repeats = (total_cols + 2) // 3- full_row = pattern.repeat(repeats)[:total_cols]- # Broadcast across rows- qzeros[:] = full_row.unsqueeze(0)- self.qzeros = qzeros.cpu()+ raise ValueError("Only 2,3,4,8 bits are supported.")def forward(self, x):out_shape = x.shape[:-1] + (self.outfeatures,)diff --git a/auto_round_extension/torch/qlinear_torch_zp.py b/auto_round_extension/torch/qlinear_torch_zp.pyindex 74485f93a..1ddbb363e 100644--- a/auto_round_extension/torch/qlinear_torch_zp.py+++ b/auto_round_extension/torch/qlinear_torch_zp.py@@ -87,7 +87,8 @@ class QuantLinear(nn.Module):def post_init(self):pass- def pack(self, linear, scales, zeros, g_idx=None):+ # @torch.compile()+ def pack_248_bits(self, linear, scales, zeros, g_idx=None):scales_t = scales.t().contiguous()if linear.bias is not None:self.bias = linear.bias.clone().half()@@ -116,120 +117,164 @@ class QuantLinear(nn.Module):del repeat_scales- if self.bits in [2, 4, 8]:- intweight = intweight.reshape(-1, intweight.shape[1] // 32 * self.bits, 32 // self.bits)- order_map = torch.arange(0, 32 // self.bits, device=device) * self.bits- intweight = intweight.to(torch.int32)- intweight = intweight << order_map- intweight = torch.sum(intweight, dim=-1)-- intweight = intweight.t().contiguous().to(torch.int32)- self.qweight = intweight.to("cpu")- else: ## bits == 3- intweight = intweight.t().contiguous()- intweight = intweight.cpu().numpy().astype(np.uint32)+ intweight = intweight.reshape(-1, intweight.shape[1] // 32 * self.bits, 32 // self.bits)+ order_map = torch.arange(0, 32 // self.bits, device=device) * self.bits+ intweight = intweight.to(torch.int32)+ intweight = intweight << order_map+ intweight = torch.sum(intweight, dim=-1)++ intweight = intweight.t().contiguous().to(torch.int32)+ self.qweight = intweight.to("cpu")++ if isinstance(zeros, torch.Tensor):+ zeros = zeros.t().contiguous().to(device)+ zeros -= 1+ # zeros = zeros.numpy().astype(np.uint32)+ qzeros = torch.zeros((zeros.shape[0], zeros.shape[1] // 32 * self.bits), device=device, dtype=torch.int32)i = 0- row = 0- qweight = torch.zeros((intweight.shape[0] // 32 * self.bits, intweight.shape[1]), dtype=torch.int32)- while row < qweight.shape[0]:- packed_weight = torch.tensor(intweight[i : i + 10]).to(dtype=torch.int32).t()- shifts = torch.arange(0, 10) * self.bits- shifted = packed_weight << shifts- qweight[row] |= shifted.sum(dim=-1)- i += 10- qweight[row] |= intweight[i] << 30- row += 1- qweight[row] |= (intweight[i] >> 2) & 1- i += 1- packed_weight = torch.tensor(intweight[i : i + 10]).to(dtype=torch.int32).t()- shifts = torch.arange(0, 10) * self.bits + 1- shifted = packed_weight << shifts- qweight[row] |= shifted.sum(dim=-1)- i += 10- qweight[row] |= intweight[i] << 31- row += 1- qweight[row] |= (intweight[i] >> 1) & 0x3- i += 1- packed_weight = torch.tensor(intweight[i : i + 10]).to(dtype=torch.int32).t()- shifts = torch.arange(0, 10) * self.bits + 2- shifted = packed_weight << shifts- qweight[row] |= shifted.sum(dim=-1)- i += 10- row += 1+ col = 0+ while col < qzeros.shape[1]:+ packed_zeros = torch.tensor(zeros[:, i : i + (32 // self.bits)]).to(dtype=torch.int32)+ shifts = torch.arange(0, (32 // self.bits)) * self.bits+ shifted = packed_zeros << shifts+ qzeros[:, col] |= shifted.sum(dim=-1)+ i += 32 // self.bits+ col += 1+ self.qzeros = qzeros.cpu()+ else:+ zeros -= 1+ shape = scales_t.shape+ value = 0+ for j in range(0, (32 // self.bits)):+ value |= zeros << (self.bits * j)+ qzeros = torch.ones((shape[0], shape[1] // 32 * self.bits), dtype=torch.int32) * value+ self.qzeros = qzeros.cpu()- self.qweight = qweight.cpu()+ def pack_3bits(self, linear, scales, zeros, g_idx=None):+ scales_t = scales.t().contiguous()+ if linear.bias is not None:+ self.bias = linear.bias.clone().half()+ self.scales = scales_t.clone().half()+ device = "cpu"+ if torch.cuda.is_available():+ device = "cuda:0"+ elif torch.xpu.is_available():+ device = "xpu:0"+ W = linear.weight.data.to(device).clone()+ if isinstance(linear, nn.Conv2d):+ W = W.flatten(1)+ if isinstance(linear, transformers.pytorch_utils.Conv1D):+ W = W.t()++ repeat_scales = scales.to(device).repeat_interleave(self.group_size, 1)if isinstance(zeros, torch.Tensor):- zeros = zeros.t().contiguous()+ repeat_zeros = zeros.to(device).repeat_interleave(self.group_size, 1)+ intweight = torch.round(W.to(device) / repeat_scales[:, : W.shape[1]] + repeat_zeros[:, : W.shape[1]]).to(+ torch.int32+ )+ else:+ repeat_zeros = zeros+ intweight = torch.round(W.to(device) / repeat_scales[:, : W.shape[1]] + repeat_zeros).to(torch.int32)++ del repeat_scales++ intweight = intweight.t().contiguous().to(torch.int32)+ i = 0+ row = 0+ qweight = torch.zeros(+ (intweight.shape[0] // 32 * self.bits, intweight.shape[1]), dtype=torch.int32, device=device+ )+ while row < qweight.shape[0]:+ packed_weight = (intweight[i : i + 10]).to(dtype=torch.int32).t()+ shifts = torch.arange(0, 10).to(device) * self.bits+ shifted = packed_weight << shifts+ qweight[row] |= shifted.sum(dim=-1)+ i += 10+ qweight[row] |= intweight[i] << 30+ row += 1+ qweight[row] |= (intweight[i] >> 2) & 1+ i += 1+ packed_weight = (intweight[i : i + 10]).to(dtype=torch.int32).t()+ shifts = torch.arange(0, 10).to(device) * self.bits + 1+ shifted = packed_weight << shifts+ qweight[row] |= shifted.sum(dim=-1)+ i += 10+ qweight[row] |= intweight[i] << 31+ row += 1+ qweight[row] |= (intweight[i] >> 1) & 0x3+ i += 1+ packed_weight = (intweight[i : i + 10]).to(dtype=torch.int32).t()+ shifts = torch.arange(0, 10).to(device) * self.bits + 2+ shifted = packed_weight << shifts+ qweight[row] |= shifted.sum(dim=-1)++ i += 10+ row += 1++ self.qweight = qweight.cpu()++ if isinstance(zeros, torch.Tensor):+ zeros = zeros.t().contiguous().to(torch.int32).to(device)zeros -= 1- zeros = zeros.numpy().astype(np.uint32)- qzeros = torch.zeros((zeros.shape[0], zeros.shape[1] // 32 * self.bits), dtype=torch.int32)+ qzeros = torch.zeros((zeros.shape[0], zeros.shape[1] // 32 * self.bits), device=device, dtype=torch.int32)i = 0col = 0while col < qzeros.shape[1]:- if self.bits in [2, 4, 8]:- packed_zeros = torch.tensor(zeros[:, i : i + (32 // self.bits)]).to(dtype=torch.int32)- shifts = torch.arange(0, (32 // self.bits)) * self.bits- shifted = packed_zeros << shifts- qzeros[:, col] |= shifted.sum(dim=-1)- i += 32 // self.bits- col += 1- else: ## bits == 3- packed_zeros = torch.tensor(zeros[:, i : i + 10]).to(dtype=torch.int32)- shifts = torch.arange(0, 10) * self.bits- shifted = packed_zeros << shifts- qzeros[:, col] = shifted.sum(dim=-1)- i += 10- qzeros[:, col] |= zeros[:, i] << 30- col += 1- qzeros[:, col] |= (zeros[:, i] >> 2) & 1- i += 1- packed_zeros = torch.tensor(zeros[:, i : i + 10]).to(dtype=torch.int32)- shifts = torch.arange(0, 10) * self.bits + 1- shifted = packed_zeros << shifts- qzeros[:, col] |= shifted.sum(dim=-1)- i += 10- qzeros[:, col] |= zeros[:, i] << 31- col += 1- qzeros[:, col] |= (zeros[:, i] >> 1) & 0x3- i += 1- packed_zeros = torch.tensor(zeros[:, i : i + 10]).to(dtype=torch.int32)- shifts = torch.arange(0, 10) * self.bits + 2- shifted = packed_zeros << shifts- qzeros[:, col] |= shifted.sum(dim=-1)- i += 10- col += 1+ packed_zeros = torch.tensor(zeros[:, i : i + 10]).to(dtype=torch.int32)+ shifts = torch.arange(0, 10).to(device) * self.bits+ shifted = packed_zeros << shifts+ qzeros[:, col] = shifted.sum(dim=-1)+ i += 10+ qzeros[:, col] |= zeros[:, i] << 30+ col += 1+ qzeros[:, col] |= (zeros[:, i] >> 2) & 1+ i += 1+ packed_zeros = torch.tensor(zeros[:, i : i + 10]).to(dtype=torch.int32)+ shifts = torch.arange(0, 10).to(device) * self.bits + 1+ shifted = packed_zeros << shifts+ qzeros[:, col] |= shifted.sum(dim=-1)+ i += 10+ qzeros[:, col] |= zeros[:, i] << 31+ col += 1+ qzeros[:, col] |= (zeros[:, i] >> 1) & 0x3+ i += 1+ packed_zeros = torch.tensor(zeros[:, i : i + 10]).to(dtype=torch.int32)+ shifts = torch.arange(0, 10).to(device) * self.bits + 2+ shifted = packed_zeros << shifts+ qzeros[:, col] |= shifted.sum(dim=-1)+ i += 10+ col += 1self.qzeros = qzeros.cpu()else:zeros -= 1- if self.bits in [2, 4, 8]:- shape = scales_t.shape- value = 0- for j in range(0, (32 // self.bits)):- value |= zeros << (self.bits * j)- qzeros = np.ones((shape[0], shape[1] // 32 * self.bits), dtype=np.uint32) * value- qzeros = qzeros.astype(np.int32)- self.qzeros = torch.from_numpy(qzeros)- else: ## bits == 3- shape = scales_t.shape[0], scales_t.shape[1] // 32 * self.bits- qzeros = torch.zeros(shape, dtype=torch.int32)- zero_val = zeros- total_cols = shape[1]- # Precompute shifts for 3-bit packing- shifts0 = torch.arange(0, 10, dtype=torch.int32) * self.bits- shifts1 = shifts0 + 1- shifts2 = shifts0 + 2- # Compute packed pattern parts- part0 = (zero_val << shifts0).sum().to(torch.int32) | (zero_val << 30)- part1 = (zero_val << shifts1).sum().to(torch.int32) | (zero_val << 31) | ((zero_val >> 2) & 1)- part2 = (zero_val << shifts2).sum().to(torch.int32) | ((zero_val >> 1) & 0x3)- pattern = torch.tensor([part0, part1, part2], dtype=torch.int32)- # Tile pattern across all columns- repeats = (total_cols + 2) // 3- full_row = pattern.repeat(repeats)[:total_cols]- # Broadcast across rows- qzeros[:] = full_row.unsqueeze(0)- self.qzeros = qzeros.cpu()+ shape = scales_t.shape[0], scales_t.shape[1] // 32 * self.bits+ qzeros = torch.zeros(shape, dtype=torch.int32)+ zero_val = zeros+ total_cols = shape[1]+ # Precompute shifts for 3-bit packing+ shifts0 = torch.arange(0, 10, dtype=torch.int32) * self.bits+ shifts1 = shifts0 + 1+ shifts2 = shifts0 + 2+ # Compute packed pattern parts+ part0 = (zero_val << shifts0).sum().to(torch.int32) | (zero_val << 30)+ part1 = (zero_val << shifts1).sum().to(torch.int32) | (zero_val << 31) | ((zero_val >> 2) & 1)+ part2 = (zero_val << shifts2).sum().to(torch.int32) | ((zero_val >> 1) & 0x3)+ pattern = torch.tensor([part0, part1, part2], dtype=torch.int32)+ # Tile pattern across all columns+ repeats = (total_cols + 2) // 3+ full_row = pattern.repeat(repeats)[:total_cols]+ # Broadcast across rows+ qzeros[:] = full_row.unsqueeze(0)+ self.qzeros = qzeros.cpu()++ def pack(self, linear, scales, zeros, g_idx=None):+ if self.bits in [2, 4, 8]:+ return self.pack_248_bits(linear, scales, zeros, g_idx)+ elif self.bits in [3]:+ return self.pack_3bits(linear, scales, zeros, g_idx)+ else:+ raise ValueError("Only 2,3,4,8 bits are supported.")def forward(self, x):out_shape = x.shape[:-1] + (self.outfeatures,)diff --git a/test/test_cpu/test_auto_round_hpu_only.py b/test/test_cpu/test_auto_round_hpu_only.pyindex a6fdb433a..fdcee80d6 100644--- a/test/test_cpu/test_auto_round_hpu_only.py+++ b/test/test_cpu/test_auto_round_hpu_only.py@@ -74,6 +74,7 @@ def test_w4a8(data_type):data_type=data_type,act_data_type="fp8_sym",act_bits=8,+ nsamples=1,act_dynamic=False,)q_model, qconfig = autoround.quantize()diff --git a/test/test_cpu/test_autoround.py b/test/test_cpu/test_autoround.pyindex 4d55869cd..86de61341 100644--- a/test/test_cpu/test_autoround.py+++ b/test/test_cpu/test_autoround.py@@ -111,21 +111,6 @@ class TestAutoRound(unittest.TestCase):)autoround.quantize()- def test_nsample(self):- autoround = AutoRound(- self.model,- self.tokenizer,- bits=4,- group_size=128,- seqlen=2,- nsamples=3,- batch_size=3,- iters=2,- dataset=self.llm_dataloader,- gradient_accumulate_steps=4,- )- autoround.quantize()-def test_default(self):bits, group_size, sym = 4, 128, Falseautoround = AutoRound(@@ -148,20 +133,6 @@ class TestAutoRound(unittest.TestCase):if torch.cuda.is_available():autoround.save_quantized(output_dir="./saved", inplace=False)- def test_sym(self):- bits, group_size, sym = 4, 128, True- autoround = AutoRound(- self.model,- self.tokenizer,- bits=bits,- group_size=group_size,- sym=sym,- iters=2,- seqlen=10,- dataset=self.llm_dataloader,- )- autoround.quantize()-def test_w4g1(self):bits, group_size, sym = 4, -1, Trueautoround = AutoRound(@@ -356,7 +327,6 @@ class TestAutoRound(unittest.TestCase):autoround.quantize()def test_tensor_reshape(self):- model_name = "facebook/opt-125m"bits, group_size, sym = 4, 100, Falseautoround = AutoRound(self.model,diff --git a/test/test_cpu/test_autoround_export_to_itrex.py b/test/test_cpu/test_autoround_export_to_itrex.pyindex 90415f240..09db9f6d4 100644--- a/test/test_cpu/test_autoround_export_to_itrex.py+++ b/test/test_cpu/test_autoround_export_to_itrex.py@@ -56,7 +56,7 @@ class TestAutoroundExport(unittest.TestCase):model = copy.deepcopy(self.gptj)out1 = model(self.lm_input)round = AutoRound- optq_1 = round(model, self.tokenizer, nsamples=20, amp=False, seqlen=10, iters=10, enable_torch_compile=False)+ optq_1 = round(model, self.tokenizer, nsamples=1, amp=False, seqlen=10, iters=10, enable_torch_compile=False)q_model, layer_config1 = optq_1.quantize() ##compile modelfrom auto_round.export.export_to_itrex import pack_model@@ -72,7 +72,7 @@ class TestAutoroundExport(unittest.TestCase):model = copy.deepcopy(self.gptj)out6 = model(self.lm_input)- optq_2 = round(model, self.tokenizer, device="cpu", nsamples=20, seqlen=10)+ optq_2 = round(model, self.tokenizer, device="cpu", nsamples=1, seqlen=10)q_model, layer_config2 = optq_2.quantize()compressed_model = pack_model(model=q_model, layer_config=layer_config2, inplace=False)compressed_model = compressed_model.to(torch.float32)@@ -96,7 +96,7 @@ class TestAutoroundExport(unittest.TestCase):model = copy.deepcopy(self.gptj)out1 = model(self.lm_input)round = AutoRound- optq_1 = round(model, self.tokenizer, nsamples=20, amp=False, seqlen=10, iters=10, enable_torch_compile=False)+ optq_1 = round(model, self.tokenizer, nsamples=1, amp=False, seqlen=10, iters=10, enable_torch_compile=False)q_model, layer_config1 = optq_1.quantize()from auto_round.export.export_to_itrex import pack_modeldiff --git a/test/test_cpu/test_basic_usage.py b/test/test_cpu/test_basic_usage.pyindex 93266c2b7..02db860fe 100644--- a/test/test_cpu/test_basic_usage.py+++ b/test/test_cpu/test_basic_usage.py@@ -56,13 +56,13 @@ class TestAutoRoundCmd(unittest.TestCase):assert False, "cmd line test fail, please have a check"res = os.system(- f"cd ../.. && {python_path} -m auto_round --mllm --model Qwen/Qwen2-VL-2B-Instruct --iter 2 --nsamples 10 --seqlen 32 --format auto_round --output_dir ./saved"+ f"cd ../.. && {python_path} -m auto_round --mllm --model Qwen/Qwen2-VL-2B-Instruct --iter 2 --nsamples 2 --seqlen 32 --format auto_round --output_dir ./saved")if res > 0 or res == -1:assert False, "cmd line test fail, please have a check"res = os.system(- f"cd ../.. && {python_path} -m auto_round --mllm --iter 2 --nsamples 10 --model Qwen/Qwen2-VL-2B-Instruct --seqlen 256 --format auto_round"+ f"cd ../.. && {python_path} -m auto_round --mllm --iter 2 --nsamples 2 --model Qwen/Qwen2-VL-2B-Instruct --seqlen 32 --format auto_round"" --quant_nontext_module --output_dir ./saved ")if res > 0 or res == -1:diff --git a/test/test_cpu/test_block_names.py b/test/test_cpu/test_block_names.pyindex 403382580..cf3506816 100644--- a/test/test_cpu/test_block_names.py+++ b/test/test_cpu/test_block_names.py@@ -217,24 +217,6 @@ class TestQuantizationBlocks(unittest.TestCase):self.assertTrue(block_name == block_name_2)self.assertTrue(len(block_name_2) == 1)self.assertTrue("model.layers.23" == block_name_2[0][-1])- ##tokenizer = AutoTokenizer.from_pretrained(model_name)- # python_path = sys.executable- # res = os.system(- # f"cd ../.. && CUDA_VISIBLE_DEVICES=0 {python_path} -m auto_round --model {model_name} --iter 1 --nsamples 1 --format auto_round --output_dir test/saved --disable_eval")- # if res > 0 or res == -1:- # assert False, "cmd line test fail, please have a check"- #- # quantized_model_path = "../saved"- #- # from auto_round import AutoHfQuantizer- # model = AutoModelForCausalLM.from_pretrained(quantized_model_path, device_map="auto")- # tokenizer = AutoTokenizer.from_pretrained(quantized_model_path)- # text = "There is a girl who likes adventure,"- # inputs = tokenizer(text, return_tensors="pt").to(model.device)- # print(tokenizer.decode(model.generate(**inputs, max_new_tokens=50)[0]))- # shutil.rmtree("./saved", ignore_errors=True)- # quant_config = model.config.quantization_config- # assert quant_config.to_quant_block_names is not Noneif __name__ == "__main__":diff --git a/test/test_cpu/test_calib_dataset.py b/test/test_cpu/test_calib_dataset.pyindex ce201048d..6746ff69c 100644--- a/test/test_cpu/test_calib_dataset.py+++ b/test/test_cpu/test_calib_dataset.py@@ -77,7 +77,15 @@ class TestLocalCalibDataset(unittest.TestCase):dataset = "NeelNanda/pile-10k:apply_chat_template:system_prompt=''"bits, group_size, sym = 4, 128, Trueautoround = AutoRound(- model, tokenizer, bits=bits, group_size=group_size, sym=sym, iters=2, seqlen=128, dataset=dataset+ model,+ tokenizer,+ bits=bits,+ group_size=group_size,+ sym=sym,+ iters=2,+ seqlen=128,+ dataset=dataset,+ nsamples=1,)autoround.quantize()@@ -85,7 +93,15 @@ class TestLocalCalibDataset(unittest.TestCase):dataset = "NeelNanda/pile-10k" + "," + "madao33/new-title-chinese" + "," + "mbpp"bits, group_size, sym = 4, 128, Trueautoround = AutoRound(- self.model, self.tokenizer, bits=bits, group_size=group_size, sym=sym, iters=2, seqlen=128, dataset=dataset+ self.model,+ self.tokenizer,+ bits=bits,+ group_size=group_size,+ sym=sym,+ iters=2,+ seqlen=128,+ dataset=dataset,+ nsamples=1,)autoround.quantize()@@ -93,7 +109,15 @@ class TestLocalCalibDataset(unittest.TestCase):dataset = "NeelNanda/pile-10k:num=256,mbpp:num=256"bits, group_size, sym = 4, 128, Trueautoround = AutoRound(- self.model, self.tokenizer, bits=bits, group_size=group_size, sym=sym, iters=2, seqlen=128, dataset=dataset+ self.model,+ self.tokenizer,+ bits=bits,+ group_size=group_size,+ sym=sym,+ iters=2,+ seqlen=128,+ dataset=dataset,+ nsamples=1,)autoround.quantize()