PTQ-Level Efficiency, QAT-Level Accuracy | 训练后量化的效率,量化感知训练的精度
Revert "[STEP 1] refact output formats" (#1157)
Heng Guoc91b404ee3f9
diff --git a/auto_round/__main__.py b/auto_round/__main__.pyindex 8541f1c3b..94e3ad163 100644--- a/auto_round/__main__.py+++ b/auto_round/__main__.py@@ -140,9 +140,7 @@ class BasicArgumentParser(argparse.ArgumentParser):"--format",default="auto_round",type=str,- help="Output format for the quantized model."- "'auto_round' is the recommended format"- "use command `auto_round list format` to show all supported formats with support scheme.",+ help="Output format for the quantized model." "'auto_round' is the recommended format",)basic.add_argument("--output_dir",@@ -464,25 +462,14 @@ class BasicArgumentParser(argparse.ArgumentParser):)-def list_item():- args = argparse.ArgumentParser()- args.add_argument("item", type=str, help="item to list, e.g., format")- args = args.parse_args()- if args.item == "format":- from auto_round.formats import OutputFormat-- print("AutoRound supported output formats and quantization scheme:")- print(OutputFormat.get_support_matrix())---def start(recipe="default"):+def setup_parser(recipe="default"):recipe = RECIPES[recipe]parser = BasicArgumentParser()args = parser.parse_args()for k, v in recipe.items():if getattr(args, k) is None:setattr(args, k, v)- tune(args)+ return argsdef tune(args):@@ -894,33 +881,27 @@ def run_eval():def run():- if "list" in sys.argv or "--list" in sys.argv:- if "list" in sys.argv:- sys.argv.remove("list")- if "--list" in sys.argv:- sys.argv.remove("--list")- list_item()- exit()- if "--eval" in sys.argv or "eval" in sys.argv:- if "--eval" in sys.argv:- sys.argv.remove("--eval")- if "eval" in sys.argv:- sys.argv.remove("eval")+ if "--eval" in sys.argv:+ sys.argv.remove("--eval")run_eval()else:- start()+ args = setup_parser()+ tune(args)def run_best():- start("best")+ args = setup_parser("best")+ tune(args)def run_light():- start("light")+ args = setup_parser("light")+ tune(args)def run_fast():- start("fast")+ args = setup_parser("fast")+ tune(args)if __name__ == "__main__":diff --git a/auto_round/compressors/base.py b/auto_round/compressors/base.pyindex 69f3be72d..4a860ae10 100644--- a/auto_round/compressors/base.py+++ b/auto_round/compressors/base.py@@ -40,6 +40,7 @@ from auto_round.compressors.utils import (check_skippable_keywords,collect_best_params,get_shared_keys,+ gguf_args_check,immediate_saving,infer_bits_by_data_type,init_cache,@@ -53,8 +54,8 @@ from auto_round.compressors.utils import ()from auto_round.data_type import QUANT_FUNC_WITH_DTYPEfrom auto_round.data_type.utils import reshape_pad_tensor_by_group_size+from auto_round.export.export_to_autoround import AutoRoundFormatfrom auto_round.export.export_to_gguf.config import GGUF_INNER_CONFIG, ModelType-from auto_round.formats import OutputFormat, get_formatsfrom auto_round.logger import loggerfrom auto_round.schemes import (SPECIAL_SCHEMES,@@ -68,6 +69,7 @@ from auto_round.special_model_handler import _handle_moe_modelfrom auto_round.utils import (INNER_SUPPORTED_LAYER_TYPES,SUPPORTED_DTYPES,+ SUPPORTED_FORMATS,SUPPORTED_LAYER_TYPES,TORCH_VERSION_AT_LEAST_2_6,CpuInfo,@@ -734,6 +736,35 @@ class BaseCompressor(object):def _check_compatibility(self) -> None:"""Checks compatibility of the configurations and model."""+ # Check gguf and others+ has_gguf = False+ if hasattr(self, "formats"):+ has_besides_gguf = False+ for format_ in self.formats:+ if "gguf" in format_:+ has_gguf = True+ elif format_ != "fake":+ has_besides_gguf = True+ if has_gguf and has_besides_gguf:+ raise ValueError("GGUF format is not compatible with other formats, please choose only one of them")+ if has_gguf:+ from transformers.utils.versions import require_version++ require_version(+ "sentencepiece",+ "GGUF format requires SentencePiece to be installed. "+ "Please install it with `pip install sentencepiece`",+ )+ if has_gguf and self.iters != 0 and self.bits != 3 and not self.enable_alg_ext:+ logger.warning(+ "`iters=0` is recommended when exporting to current GGUF format"+ " or add `enable_alg_ext` for better accuracy with much more tuning cost."+ " Please refer to https://github.com/intel/auto-round/tree/main/docs/gguf_alg_ext_acc.md"+ " for the accuracy results."+ )+ elif self.bits >= 8 and self.iters != 0:+ logger.warning("`iters=0` is recommended for bits>=8")+if (self.seqlen is not Noneand hasattr(self.model, "config")@@ -758,6 +789,230 @@ class BaseCompressor(object):if self.group_size == 0 and "fp8" not in self.data_type:logger.warning("`group_size==0` is not supported for data_type other than fp8 ")+ def _parse_format_to_list(self, format: str) -> list:+ """Parses the format string into a list of formats.++ This method checks the requested format(s) against the model's+ quantization settings and adjusts them if necessary. It ensures that+ the formats are compatible with the model's data type, bit width,+ and activation quantization settings.++ Args:+ format (str): The requested format(s) for quantization, separated by commas.++ Returns:+ list: A list of validated and updated formats.+ """++ # Remove duplicates from formats list+ def remove_duplicates(lst):+ seen = set()+ return [x for x in lst if not (x in seen or seen.add(x))]++ formats = format.replace("q*_", f"q{self.bits}_").replace(" ", "").split(",")+ formats = remove_duplicates(formats) # need the keep origin order++ gguf_format_name = get_gguf_scheme(self.scheme)++ if gguf_format_name:+ for i in range(len(formats)):+ if gguf_format_name.lower().endswith("mixed"):+ gguf_format_name = gguf_format_name.lower().replace("_mixed", "_s")+ if formats[i] != "fake" and formats[i] != gguf_format_name.lower():+ logger.warning(+ f"reset format {formats[i]} to {gguf_format_name.lower()} "+ f"since scheme {gguf_format_name} can only be exported to format {gguf_format_name.lower()}"+ )+ formats[i] = gguf_format_name.lower()++ gguf_args_check(self, formats, model_type=ModelType.TEXT)+ if self.mllm:+ gguf_args_check(self, formats, model_type=ModelType.MMPROJ)++ for f in formats:+ if f.startswith("gguf"):+ self.scheme = f.upper()+ break++ for format_ in formats:+ if format_ not in SUPPORTED_FORMATS:+ logger.error(f"Unsupported format {format_}, please choose from {SUPPORTED_FORMATS}")+ exit(-1)+ if self.scale_dtype != torch.float32:+ only_gguf = True+ for format_ in formats:+ if not ("gguf" in format_ or "fake" in format_):+ only_gguf = False+ break+ if len(formats) == 1 and "fake" == formats[0]:+ only_gguf = False+ if only_gguf:+ self.scale_dtype = torch.float32+ logger.info("change `scale_dtype` to `torch.float32`")++ # Adjust format settings based on compatibility+ for index in range(len(formats)):+ format = formats[index]+ if format == "auto_round":+ if self.sym and "int" in self.data_type:+ format = "auto_round:auto_gptq"+ elif self.bits == 4 and not self.sym and "int" in self.data_type:+ enable_awq = all(+ config["bits"] == self.bits or config["bits"] >= 16 for config in self.layer_config.values()+ )+ if enable_awq:+ format = "auto_round:auto_awq"+ elif is_nv_fp(self.data_type) or is_mx_fp(self.data_type):+ format = f"auto_round:{self.data_type}"+ elif is_static_wfp8afp8(self): # static wfp8afp8+ format = f"auto_round:{AutoRoundFormat.FP8_STATIC.value}"+ elif self.data_type.startswith("fp") and self.bits == 8 and self.act_bits >= 16: # woq fp8+ format = f"auto_round:{AutoRoundFormat.FP8.value}"+ elif self.act_bits < 16:+ raise ValueError(+ "AutoRound format does not support exporting "+ "for the current quantization configuration, "+ "please change to `fake` format for research purpose"+ )+ formats[index] = format+ elif format == "llm_compressor":+ from auto_round.export.export_to_llmcompressor import check_compressed_tensors_supported++ if is_nv_fp(self.data_type) or is_mx_fp(self.data_type):+ check_compressed_tensors_supported()+ format = format.replace("llm_compressor", f"llm_compressor:{self.data_type}")+ formats[index] = format+ elif is_static_wfp8afp8(self):+ format = f"llm_compressor:{AutoRoundFormat.FP8_STATIC.value}"+ formats[index] = format+ if self.act_group_size != 0:+ logger.warning(+ f"scheme FP8_STATIC export to llm_compressor format only support for act_group_size 0,"+ f" ,but got act_group_size={self.act_group_size}, reset = 0"+ )+ self.act_group_size = 0+ if self.group_size > 0:+ logger.warning(+ f"please note that group_size={self.group_size}"+ " may not be supported for llm_compressor format, and cannot be loaded in llm_compressor"+ )+ elif not is_wfp8afp8(self):+ logger.error(+ "Currently, the llm_compressor format only supports MXFP/NVFP/FP8. "+ "Please change format to fake or auto_round etc."+ )+ elif "auto_awq" in format:+ from auto_round.compressors.utils import check_awq_gemm_compatibility++ awq_supported, info = check_awq_gemm_compatibility(+ self.model, self.bits, self.group_size, self.sym, self.layer_config+ )+ if not awq_supported:+ logger.warning(f"The AutoAWQ format may not be supported due to {info}")+ else:+ if (is_nv_fp(self.data_type) or is_mx_fp(self.data_type)) and format != "fake":+ logger.warning(f"nv_fp and mx_fp dtypes are not supported for export format: {format}")++ formats = remove_duplicates(formats)+ for i in range(len(formats)):+ formats[i] = self._check_supported_format(formats[i])+ formats = remove_duplicates(formats)+ return formats++ def _check_supported_format(self, format: str) -> bool:+ """Checks if the specified format is supported.++ This method validates the requested format against the model's bit width,+ group size, symmetry, and activation quantization settings. It raises an+ error if the format is incompatible with the current model configuration.++ Args:+ format (str): The requested format for quantization.++ Returns:+ bool: True if the format is supported, False otherwise.+ """+ if format == "fake":+ return format+ format = format.replace("q*_", f"q{self.bits}_")++ # format check for fp8+ w_fp8 = self.data_type.startswith("fp") and self.bits == 8+ act_fp8 = self.act_data_type.startswith("fp") and self.act_bits == 8+ if (w_fp8 or act_fp8) and re.search("^auto_round|^llm_compressor", format) is None:+ error_msg = (+ f"is only supported to export auto_round or llm_compressor format," f" but got {format}, please check."+ )+ error_msg = ("act_data_type<fp8> " + error_msg) if act_fp8 else error_msg+ error_msg = ("data_type<fp8> " + error_msg) if w_fp8 else error_msg+ logger.error(error_msg)+ sys.exit(-1)++ # Only support to export afp8/nv_fp/mx_fp+ if self.act_bits <= 8:+ if not is_standard_fp(self.act_data_type) or self.act_dynamic:+ if "llm_compressor" in format:+ if (is_nv_fp(self.act_data_type) and "static_gs" in self.act_data_type) or (+ is_mx_fp(self.act_data_type)+ ):+ return format+ bits, group_size, sym, act_bits = 8, -1, True, 8+ assert (+ self.bits == bits+ and self.group_size == group_size+ and self.sym == sym+ and self.act_bits == act_bits+ and self.act_dynamic+ ), (+ f"Currently only support to export llm_compressor format for sym dynamic quantized"+ f" W{self.bits}A{self.act_bits} model with group_size={group_size},"+ f" but got bits={self.bits}, group_size={self.group_size}, sym={self.sym},"+ f" act_bits={self.act_bits}"+ )+ elif "auto_round" in format and (+ is_mx_fp(self.act_data_type) or (is_nv_fp(self.act_data_type) and "static_gs" in self.act_data_type)+ ):+ pass+ elif format != "fake":+ logger.warning(+ "Currently only support to export auto_round format quantized model"+ " with fp8, mx_fp and nv_fp4 dtype activation for activation quantization."+ " Change format to fake and save."+ )+ format = "fake"+ else:+ if format not in [+ "auto_round",+ f"auto_round:{AutoRoundFormat.FP8_STATIC.value}",+ f"llm_compressor:{AutoRoundFormat.FP8_STATIC.value}",+ "auto_round:llm_compressor",+ ]:+ logger.warning(+ f"Currently only support to export auto_round or fake format for static W{self.bits}AFP8 model,"+ f" change format {format} to auto_round"+ )+ if is_static_wfp8afp8(self):+ format = f"auto_round:{AutoRoundFormat.FP8_STATIC.value}"+ else:+ format = f"auto_round:{AutoRoundFormat.FP8.value}"+ if (+ self.act_group_size != 0+ and not self.act_dynamic+ and format == f"auto_round:{AutoRoundFormat.FP8.value}"+ ):+ logger.warning(+ f"Please note that quantize activation with act_group_size={self.act_group_size}"+ " may result in failure to export or import normally."+ )+ if re.search(r"q\d_k", format) and not self.data_type.endswith("_dq"):+ logger.error(+ f"datatype<{self.data_type}> not support to export {format} format."+ " Please change export format or `data_type`."+ )+ sys.exit(-1)++ return format+def quantize_and_save(self, output_dir: str = "tmp_autoround", format: str = "auto_round", inplace: bool = True, **kwargs) -> tuple[torch.nn.Module, dict[str, Any]]:@@ -787,7 +1042,7 @@ class BaseCompressor(object):self.orig_output_dir = output_dir# check and update the format based on the current configuration- format_list = get_formats(format, self)+ format_list = self._parse_format_to_list(format)self.formats = format_list# If multiple formats are specified, enforce inplace=False@@ -810,12 +1065,24 @@ class BaseCompressor(object):else:model, _ = self.quantize()# Save the quantized model in the specified format_list- model, folders = self.save_quantized(output_dir, format=format, inplace=inplace, return_folders=True, **kwargs)+ folders = []+ for format in format_list:+ if "gptq" in format and not self.sym:+ logger.warning(+ "The asymmetrical kernel of the GPTQ format may result in a noticeable accuracy drop,"+ " particularly for 2-bit quantization and smaller models."+ " We recommend exporting to either the AutoAWQ format ( only 4 bits) or "+ "the AutoRound format(2/3/4/8 bits)."+ )+ save_folder = self._get_save_folder_name(format)+ self.save_quantized(save_folder, format=format, inplace=inplace, **kwargs)++ folders.append(save_folder)memory_monitor.log_summary()return model, folders- def _get_save_folder_name(self, format: OutputFormat) -> str:+ def _get_save_folder_name(self, format_str: str) -> str:"""Generates the save folder name based on the provided format string.If there are multiple formats to handle, the function creates a subfolder@@ -829,7 +1096,7 @@ class BaseCompressor(object):str: The path to the folder where results should be saved."""# Replace special characters to make the folder name filesystem-safe- sanitized_format = format.get_backend_name().replace(":", "-").replace("_", "-")+ sanitized_format = format_str.replace(":", "-").replace("_", "-")# Use a subfolder only if there are multiple formatsif len(self.formats) > 1:@@ -1040,7 +1307,7 @@ class BaseCompressor(object):set_module(self.model, name, m)tuning_device = m.tuning_device if hasattr(m, "tuning_device") else self.device# Step 1: Try quantization on GPU first, fall back to CPU if OOM- if self.immediate_packing and self.iters == 0 and self.formats[0].is_gguf() and not self.disable_opt_rtn:+ if self.immediate_packing and self.iters == 0 and "gguf" in self.formats[0] and not self.disable_opt_rtn:m = m.to(tuning_device)m.scale = Nonem.zp = None@@ -1100,8 +1367,8 @@ class BaseCompressor(object):returnfrom auto_round.export import PACKING_LAYER_WITH_FORMAT- target_backend = self.formats[0].output_format- has_gguf = any(fmt.is_gguf() for fmt in self.formats)+ target_backend = self.formats[0].split(":")[0] if ":" in self.formats[0] else self.formats[0]+ has_gguf = any("gguf" in fmt for fmt in self.formats)if has_gguf:from auto_round.export.export_to_gguf.export import pack_gguf_layer@@ -1111,7 +1378,7 @@ class BaseCompressor(object):pack_gguf_layer(name,self.model,- self.formats[0].get_backend_name(),+ self.formats[0],output_dir,self.layer_config,self.tokenizer,@@ -1121,9 +1388,7 @@ class BaseCompressor(object):device=self.device,)else:- PACKING_LAYER_WITH_FORMAT[target_backend](- name, self.model, self.formats[0].get_backend_name(), device=self.device- )+ PACKING_LAYER_WITH_FORMAT[target_backend](name, self.model, self.formats[0], device=self.device)@torch.inference_mode()def _quantize_rtn(self) -> tuple[torch.nn.Module, dict[str, Any]]:@@ -1158,7 +1423,7 @@ class BaseCompressor(object):update_fused_layer_global_scales(module)logger.info("Finished updating fused layer global scales.")- if not (any(fmt.is_gguf() for fmt in getattr(self, "formats", [])) or self.super_bits is not None):+ if not (any("gguf" in fmt for fmt in getattr(self, "formats", [])) or self.super_bits is not None):self._quantize_embedding_layer() # leave to gguf itself to handleself.model.to("cpu")@@ -1168,8 +1433,7 @@ class BaseCompressor(object):enable_imatrix = Falseif not self.disable_opt_rtn:has_gguf_k = (- any(fmt.is_gguf() and "k" in fmt.output_format for fmt in getattr(self, "formats", []))- or self.super_bits is not None+ any("gguf" in fmt and "k" in fmt for fmt in getattr(self, "formats", [])) or self.super_bits is not None)if has_gguf_k:enable_imatrix = True@@ -1408,11 +1672,20 @@ class BaseCompressor(object):else:# Determine if immediate packing is requiredformats = self.formats- if len(formats) == 1 and not formats[0].is_fake() and self.inplace:+ if (+ len(formats) == 1+ and (+ "awq" in formats[0]+ or "gptq" in formats[0]+ or "auto_round" in formats[0]+ or "gguf" in formats[0]+ or "llm_compressor" in formats[0]+ )+ and self.inplace+ ):self.immediate_packing = True- if not formats[0].is_gguf() and self.low_cpu_mem_usage:+ if "gguf" not in formats[0] and self.low_cpu_mem_usage:self.immediate_saving = True-if self.immediate_saving and "int" not in self.data_type:logger.warning("immediate_saving is only supported for int quantization, set to False")self.immediate_saving = False@@ -1589,7 +1862,7 @@ class BaseCompressor(object):has_gguf = Falseif hasattr(self, "formats"):- has_gguf = any(format_.is_gguf() for format_ in self.formats)+ has_gguf = any("gguf" in format_ for format_ in self.formats)if has_gguf and self.immediate_packing:enable_quanted_input = False@@ -2860,12 +3133,7 @@ class BaseCompressor(object):clear_memory(device_list=self.device_list)def save_quantized(- self,- output_dir: str = None,- format: Union[str, list[OutputFormat]] = "auto_round",- inplace: bool = True,- return_folders=False,- **kwargs,+ self, output_dir: str = None, format: str = "auto_round", inplace: bool = True, **kwargs) -> torch.nn.Module:"""Save the quantized model to the specified output directory in the specified format.@@ -2878,118 +3146,122 @@ class BaseCompressor(object):Returns:object: The compressed model object."""- self.orig_output_dir = output_dir- if isinstance(format, str):- formats = get_formats(format, self)- if not hasattr(self, "formats"):- self.formats = formats+ format = self._check_supported_format(format)if not self.quantized:logger.warning("please run autoround.quantize first")return- folders = []- for format in formats:- save_folder = self._get_save_folder_name(format)- if format.is_fake(): # TODO fix act quantization later- self.model = self.model.to("cpu")- self.model.save_pretrained(output_dir)- if self.tokenizer is not None and hasattr(self.tokenizer, "save_pretrained"):- self.tokenizer.save_pretrained(output_dir)- processor = kwargs.get("processor", None)- if processor is not None:- processor.save_pretrained(output_dir)- try:- copy_python_files_from_model_cache(self.model, output_dir)- except Exception as e:- logger.warning("Skipping source model Python file copy due to error: %s", e)- compressed_model = self.model- continue- if self.act_bits <= 8 and format.is_fake():- logger.warning(- "Support for exporting activation quantization is limited. "- "Please ensure that your configuration is supported."- )- from auto_round.export import EXPORT_FORMAT-- backend = format.get_backend_name()- output_format = format.output_format- if output_format not in EXPORT_FORMAT:- raise ValueError(f"export format only supports {EXPORT_FORMAT.keys()}, but got {output_format}")- save_quantized_as_format = EXPORT_FORMAT.get(output_format)- serialization_keys = [- "bits",- "group_size",- "sym",- "data_type",- "enable_quanted_input",- "enable_minmax_tuning",- "seqlen",- "batch_size",- "scale_dtype",- "lr",- "minmax_lr",- "gradient_accumulate_steps",- "iters",- "amp",- "nsamples",- "low_gpu_mem_usage",- "to_quant_block_names",- "enable_norm_bias_tuning",- "act_bits",- "act_group_size",- "act_sym",- "act_dynamic",- "act_data_type",- "super_bits",- "super_group_size",- "regex_config",- "static_kv_dtype",- "static_attention_dtype",- ]- if isinstance(self.dataset, str):- serialization_keys.append("dataset")- serialization_dict = {}- for key in serialization_keys:- serialization_dict[key] = getattr(self, key)- from auto_round.version import __version__-- serialization_dict["autoround_version"] = __version__- if "scale_dtype" in serialization_dict.keys():- serialization_dict["scale_dtype"] = str(serialization_dict["scale_dtype"])- compressed_model = save_quantized_as_format( # TODO refine the code- save_folder,- model=self.model,- layer_config=self.layer_config,- inplace=inplace,- bits=self.bits,- act_bits=self.act_bits,- group_size=self.group_size,- sym=self.sym,- iters=self.iters,- lr=self.lr,- minmax_lr=self.minmax_lr,- enable_minmax_tuning=self.enable_minmax_tuning,- enable_quanted_input=self.enable_quanted_input,- scale_dtype=self.scale_dtype,- tokenizer=self.tokenizer,- supported_types=self.supported_types,- data_type=self.data_type,- act_data_type=self.act_data_type,- serialization_dict=serialization_dict,- backend=backend,- to_quant_block_names=self.to_quant_block_names,- quant_block_list=self.quant_block_list,- device=self.device,- static_kv_dtype=self.static_kv_dtype,- static_attention_dtype=self.static_attention_dtype,- **kwargs,+ if format == "fake" or format == "qdq": # TODO fix act quantization later+ self.model = self.model.to("cpu")+ self.model.save_pretrained(output_dir)+ if self.tokenizer is not None and hasattr(self.tokenizer, "save_pretrained"):+ self.tokenizer.save_pretrained(output_dir)+ processor = kwargs.get("processor", None)+ if processor is not None:+ processor.save_pretrained(output_dir)+ try:+ copy_python_files_from_model_cache(self.model, output_dir)+ except Exception as e:+ logger.warning("Skipping source model Python file copy due to error: %s", e)+ return+ if self.act_bits <= 8 and format == "qdq":+ logger.warning(+ "Support for exporting activation quantization is limited. "+ "Please ensure that your configuration is supported.")- folders.append(save_folder)-- if return_folders:- return compressed_model, folders- else:- return compressed_model+ # if format == "llm_compressor" and (is_nv_fp(self.data_type) or is_mx_fp(self.data_type)):+ # format = format.replace("llm_compressor", f"llm_compressor:{self.data_type}")+ if format == "llm_compressor" and (is_nv_fp(self.data_type) or is_mx_fp(self.data_type)):+ format = format.replace("llm_compressor", f"llm_compressor:{self.data_type}")+ if format == "llm_compressor" and is_static_wfp8afp8(self):+ format = format.replace("llm_compressor", "llm_compressor:{AutoRoundFormat.FP8_STATIC.value}")++ from auto_round.export import EXPORT_FORMAT++ backend = format+ format = format.split(":")[0]+ if format not in EXPORT_FORMAT:+ logger.error(f"export format only supports {EXPORT_FORMAT.keys()}")+ raise ValueError(f"export format only supports {EXPORT_FORMAT.keys()}, but got {format}")+ save_quantized_as_format = EXPORT_FORMAT.get(format)+ if "gptq" in format and not self.sym:+ logger.warning(+ "the asymmetrical kernel of the GPTQ format may result in a noticeable accuracy drop,"+ " particularly for 2-bit quantization and smaller models."+ " We recommend exporting to either the AutoAWQ format ( only 4 bits) or "+ "the AutoRound format(2/3/4/8 bits)."+ )+ if "awq" in format and not self.bits == 4:+ raise ValueError("The AWQ format only supports W4 quantization ")+ serialization_keys = [+ "bits",+ "group_size",+ "sym",+ "data_type",+ "enable_quanted_input",+ "enable_minmax_tuning",+ "seqlen",+ "batch_size",+ "scale_dtype",+ "lr",+ "minmax_lr",+ "gradient_accumulate_steps",+ "iters",+ "amp",+ "nsamples",+ "low_gpu_mem_usage",+ "to_quant_block_names",+ "enable_norm_bias_tuning",+ "act_bits",+ "act_group_size",+ "act_sym",+ "act_dynamic",+ "act_data_type",+ "super_bits",+ "super_group_size",+ "regex_config",+ "static_kv_dtype",+ "static_attention_dtype",+ ]+ if isinstance(self.dataset, str):+ serialization_keys.append("dataset")+ serialization_dict = {}+ for key in serialization_keys:+ serialization_dict[key] = getattr(self, key)+ from auto_round.version import __version__++ serialization_dict["autoround_version"] = __version__+ if "scale_dtype" in serialization_dict.keys():+ serialization_dict["scale_dtype"] = str(serialization_dict["scale_dtype"])+ compressed_model = save_quantized_as_format( # TODO refine the code+ output_dir,+ model=self.model,+ layer_config=self.layer_config,+ inplace=inplace,+ bits=self.bits,+ act_bits=self.act_bits,+ group_size=self.group_size,+ sym=self.sym,+ iters=self.iters,+ lr=self.lr,+ minmax_lr=self.minmax_lr,+ enable_minmax_tuning=self.enable_minmax_tuning,+ enable_quanted_input=self.enable_quanted_input,+ scale_dtype=self.scale_dtype,+ tokenizer=self.tokenizer,+ supported_types=self.supported_types,+ data_type=self.data_type,+ act_data_type=self.act_data_type,+ serialization_dict=serialization_dict,+ backend=backend,+ to_quant_block_names=self.to_quant_block_names,+ quant_block_list=self.quant_block_list,+ device=self.device,+ static_kv_dtype=self.static_kv_dtype,+ static_attention_dtype=self.static_attention_dtype,+ **kwargs,+ )+ return compressed_modeldef _get_quantized_layer_names_outside_blocks(self) -> list:"""Gets the names of quantized layers outside blocks in the model.diff --git a/auto_round/compressors/utils.py b/auto_round/compressors/utils.pyindex 268cbed5b..dd85f4ebd 100644--- a/auto_round/compressors/utils.py+++ b/auto_round/compressors/utils.py@@ -164,6 +164,48 @@ def check_need_act_calibration(return False+def check_awq_gemm_compatibility(model, bits, group_size, sym, layer_configs=None):+ """Checks if a model is compatible with the AutoAWQ GEMM kernel.++ Args:+ model: The model object to evaluate, typically a PyTorch model.+ bits (int): The number of bits for quantization (must be 4 for compatibility).+ group_size (int): The group size for quantization.+ sym (bool): Whether symmetric quantization is used (not utilized in the current function logic).+ layer_configs (dict, optional): A dictionary mapping layer names to configurations, where each+ configuration can specify a custom number of bits for the layer.++ Returns:+ tuple: A tuple containing:+ - bool: `True` if the model is compatible, `False` otherwise.+ - str: An error message describing why the model is incompatible, or an empty string if compatible.+ """+ from auto_round.utils.model import get_layer_names_in_block, get_module++ if bits != 4:+ return False, "AutoAWQ GEMM kernel only supports 4 bits"+ for n, m in model.named_modules():+ if type(m) == transformers.pytorch_utils.Conv1D:+ return False, "AutoAWQ GEMM kernel does not support conv1d"++ layer_names = get_layer_names_in_block(model)+ for layer_name in layer_names:+ if (+ layer_configs is not None+ and layer_name in layer_configs.keys()+ and layer_configs[layer_name].get("bits", bits) > 8+ ):+ continue++ layer = get_module(model, layer_name)+ if layer.in_features % group_size != 0:+ return False, f"Layer {layer_name} in_features is not multiple of group_size {group_size}"+ if layer.out_features % (32 // bits) != 0:+ return False, f"Layer {layer_name} out_features is not multiple of 32 // bits"++ return True, ""++def collect_best_params(block, cache_device="cpu"):"""Collect the best parameters from the block to the specified device."""params = {}@@ -471,6 +513,117 @@ def gguf_type_fallback(gguf_type: str) -> str:return gguf_type+def gguf_args_check(args_or_ar, formats: list[str] = None, model_type=ModelType.TEXT):+ import argparse++ from auto_round.export.export_to_gguf.convert import download_convert_file+ from auto_round.logger import logger+ from auto_round.utils.model import download_or_get_path, get_gguf_architecture++ formats = sorted(formats, key=lambda x: len(x))+ export_gguf = False+ for f in formats:+ if f.startswith("gguf"):+ export_gguf = True++ if f.startswith("gguf") and f not in GGUF_CONFIG:+ logger.error(f"{f} is not supported, please check.")++ redownload = False+ if export_gguf:+ try:+ from auto_round.export.export_to_gguf.convert_hf_to_gguf import ( # pylint: disable=E0401+ ModelBase,+ ModelType,+ get_model_architecture,+ )++ if isinstance(args_or_ar.model, str):+ model_path = args_or_ar.model+ else:+ model_path = args_or_ar.model.name_or_path+ if not os.path.isdir(model_path):+ model_path = download_or_get_path(model_path, args_or_ar.platform)+ model_architecture = get_gguf_architecture(model_path, model_type=ModelType.TEXT)+ if model_architecture not in ModelBase._model_classes[ModelType.TEXT]:+ logger.warning(+ f"Current version of gguf export does not support for {model_architecture},"+ " will re-download dependency file. Please restart the task."+ )+ redownload = True+ except ModuleNotFoundError as e:+ if "convert_hf_to_gguf" in str(e):+ logger.warning("GGUF export dependency file is not found, download from github.")+ redownload = True+ except AttributeError as e:+ raise ImportError(+ "Please use the latest gguf-py, you can use the following command to install it:\n"+ "git clone https://github.com/ggml-org/llama.cpp.git && cd llama.cpp/gguf-py &&"+ " pip install . sentencepiece"+ )+ download_convert_file(redownload)++ try:+ from auto_round.export.export_to_gguf.convert_hf_to_gguf import ( # pylint: disable=E0401+ ModelBase,+ ModelType,+ )+ except ImportError as e:+ raise ImportError(+ "Please use the latest gguf-py, you can use the following command to install it:\n"+ "git clone https://github.com/ggml-org/llama.cpp.git && cd llama.cpp/gguf-py &&"+ " pip install . sentencepiece"+ )+ if isinstance(args_or_ar.model, str):+ model_path = args_or_ar.model+ else:+ model_path = args_or_ar.model.name_or_path+ if not os.path.isdir(model_path):+ model_path = download_or_get_path(model_path, args_or_ar.platform)+ model_architecture = get_gguf_architecture(model_path, model_type=ModelType.TEXT)+ if model_architecture not in ModelBase._model_classes[ModelType.TEXT]:+ logger.error(f"Model {model_architecture} is not supported to export gguf format.")+ sys.exit(1)++ pattern = re.compile(r"q\d_k")+ pre_dq_format = ""+ unsupported_list, reset_list = [], []+ for format in GGUF_CONFIG:+ if format in formats:+ if format == "q6_k_s":+ logger.warning("Please note that q6_k_s is q6_k.")++ if re.search(pattern, format):+ if pre_dq_format and re.search(pattern, format).group() not in pre_dq_format:+ logger.error(f"Cannot export {pre_dq_format} and {format} at the same time.")+ sys.exit(-1)+ else:+ pre_dq_format = format++ unsupported_list, reset_list = [], []+ gguf_config = GGUF_CONFIG[format]+ for k, v in gguf_config.items():+ if not hasattr(args_or_ar, k):+ continue+ if k == "data_type":+ if re.search(r"q\d_1", format) and len(formats) > 1:+ v = "int"+ if k == "sym" and isinstance(args_or_ar, argparse.Namespace):+ k = "asym"+ v = not v+ if getattr(args_or_ar, k) != v:+ unsupported_list.append(f"{k}={getattr(args_or_ar, k)}")+ reset_list.append(f"{k}={v}")+ setattr(args_or_ar, k, v)+ if len(unsupported_list) > 0:+ logger.info(+ f"format {format} does not support for {', '.join(unsupported_list)},"+ f" reset to {', '.join(reset_list)}."+ )+ # Removed obsolete commented-out block for improved readability and maintainability.+ return args_or_ar++def get_gguf_qtype_by_layer_config(layer_config):import gguf # pylint: disable=E0401@@ -957,6 +1110,7 @@ def immediate_saving(rounder: object, m: torch.nn.Module, name: str = None, lastwrites shard index, renames shard files, copies source files, and releases temporary state."""import json+ import osfrom collections import OrderedDictfrom auto_round.utils import clear_memory, get_modulediff --git a/auto_round/export/export_to_autoround/__init__.py b/auto_round/export/export_to_autoround/__init__.pyindex 0c036d831..6cdcd5aed 100644--- a/auto_round/export/export_to_autoround/__init__.py+++ b/auto_round/export/export_to_autoround/__init__.py@@ -12,4 +12,4 @@# See the License for the specific language governing permissions and# limitations under the License.-from .export import save_quantized_as_autoround+from .export import save_quantized_as_autoround, AutoRoundFormatdiff --git a/auto_round/export/export_to_autoround/export.py b/auto_round/export/export_to_autoround/export.pyindex 2dfac16ef..ccc7780b3 100644--- a/auto_round/export/export_to_autoround/export.py+++ b/auto_round/export/export_to_autoround/export.py@@ -35,7 +35,6 @@ from auto_round.export.utils import (release_layer_safely,save_model,)-from auto_round.formats import AutoRoundExportFormatfrom auto_round.logger import loggerfrom auto_round.schemes import QuantizationSchemefrom auto_round.utils import (@@ -51,6 +50,16 @@ from auto_round.utils import ()+class AutoRoundFormat(str, Enum):+ # Weight: FP8, per-channel, may be extended to per-tensor in future+ # Activation: FP8, per-tensor+ FP8_STATIC = "fp8_static"+ MXFP8 = "mxfp8"+ MXFP4 = "mxfp4"+ NVFP4 = "nvfp4"+ FP8 = "fp8"++def dynamic_import_quant_linear_for_packing(backend, bits, group_size, sym, act_bits=16):"""Dynamically imports and returns the appropriate QuantLinear class based on the specified backend and parameters.@@ -161,14 +170,14 @@ def pack_layer(layer_name, model, backend, device=None):return pack_layer(layer_name, model, backend, device)if (- backend == f"auto_round:{AutoRoundExportFormat.FP8.value}"- or backend == f"auto_round:{AutoRoundExportFormat.FP8_STATIC.value}"+ backend == f"auto_round:{AutoRoundFormat.FP8.value}"+ or backend == f"auto_round:{AutoRoundFormat.FP8_STATIC.value}"):from auto_round.export.export_to_autoround.export_to_fp8 import pack_layerreturn pack_layer(layer_name, model, backend, device)- if backend in ["auto_round:llm_compressor", f"auto_round:llm_compressor:{AutoRoundExportFormat.FP8_STATIC.value}"]:+ if backend == "auto_round:llm_compressor":from auto_round.export.export_to_llmcompressor.export_to_static_fp import pack_layerreturn pack_layer(layer_name, model, backend, device)@@ -278,7 +287,7 @@ def save_quantized_as_autoround(output_dir, inplace=True, backend="auto_round:exreturn save_quantized_as_fp(output_dir, inplace=inplace, backend="auto_round:llm_compressor", **kwargs)- if backend in ["auto_round:llm_compressor", f"auto_round:llm_compressor:{AutoRoundExportFormat.FP8_STATIC.value}"]:+ if backend == "auto_round:llm_compressor":from auto_round.export.export_to_llmcompressor.export_to_static_fp import save_quantized_as_static_fpreturn save_quantized_as_static_fp(output_dir, inplace=inplace, backend="auto_round:llm_compressor", **kwargs)@@ -292,7 +301,7 @@ def save_quantized_as_autoround(output_dir, inplace=True, backend="auto_round:exif ((kwargs.get("sym") is None or kwargs.get("sym"))and ("gptq" not in backend and "awq" not in backend)- and (AutoRoundExportFormat.FP8_STATIC.value not in backend)+ and (AutoRoundFormat.FP8_STATIC.value not in backend)):backend = backend.replace("auto_round", "auto_round:auto_gptq")diff --git a/auto_round/formats.py b/auto_round/formats.pydeleted file mode 100644index 16592320a..000000000--- a/auto_round/formats.py+++ /dev/null@@ -1,654 +0,0 @@-# Copyright (c) 2025 Intel Corporation-#-# Licensed under the Apache License, Version 2.0 (the "License");-# you may not use this file except in compliance with the License.-# You may obtain a copy of the License at-#-# http://www.apache.org/licenses/LICENSE-2.0-#-# Unless required by applicable law or agreed to in writing, software-# distributed under the License is distributed on an "AS IS" BASIS,-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.-# See the License for the specific language governing permissions and-# limitations under the License.--from __future__ import annotations--import copy-import os-import re-import sys-from dataclasses import asdict-from enum import Enum-from typing import TYPE_CHECKING, Callable, Union--import torch-import transformers--from auto_round.compressors.utils import (- is_mx_fp,- is_nv_fp,- is_standard_fp,- is_static_wfp8afp8,- is_wfp8afp8,-)-from auto_round.export.export_to_gguf.config import ModelType-from auto_round.schemes import (- PRESET_SCHEMES,- QuantizationScheme,- get_gguf_scheme,-)-from auto_round.utils import SUPPORTED_FORMATS, logger---class AutoRoundExportFormat(str, Enum):- # Weight: FP8, per-channel, may be extended to per-tensor in future- # Activation: FP8, per-tensor- FP8_STATIC = "fp8_static"- MXFP8 = "mxfp8"- MXFP4 = "mxfp4"- NVFP4 = "nvfp4"- FP8 = "fp8"- MX_FP = "mx_fp"- NV_FP = "nv_fp"- MX_FP_RCEIL = "mx_fp_rceil"- NV_FP4_WITH_STATIC_GS = "nv_fp4_with_static_gs"---if TYPE_CHECKING:- from auto_round.compressors.base import BaseCompressor---def _check_compatibility(formats: list[str], ar: BaseCompressor):- if (- any(["gguf" in f.lower() for f in formats])- and len([f for f in formats if f.lower() != "fake" and not f.lower().startswith("gguf")]) > 1- ):- raise ValueError(- f"GGUF format is not compatible with other formats, but got {formats}, please choose only one of them"- )- gguf_format_name = get_gguf_scheme(ar.scheme)- if gguf_format_name:- if gguf_format_name.lower().endswith("mixed"):- gguf_format_name = gguf_format_name.lower().replace("_mixed", "_s")- if any([f.lower() not in ["fake", gguf_format_name.lower()] for f in formats]):- tmp_format_name = gguf_format_name.lower() if "fake" not in formats else f"{gguf_format_name.lower()},fake"- logger.warning(- f"reset format {','.join(formats)} to {tmp_format_name} "- f"since scheme {gguf_format_name} can only be exported to format {gguf_format_name.lower()} or fake"- )- formats = tmp_format_name.split(",")- return formats---def get_formats(- format: str,- ar: BaseCompressor,-) -> list[OutputFormat]:- """Get the list of OutputFormat instances based on the provided name."""-- def remove_duplicates(lst):- seen = set()- return [x for x in lst if not (x in seen or seen.add(x))]-- formats = format.replace("q*_", f"q{ar.bits}_").replace(" ", "").split(",")- formats = remove_duplicates(formats) # need the keep origin order-- formats = _check_compatibility(formats, ar)-- formats = remove_duplicates(formats)-- for i in range(len(formats)):- if formats[i].startswith("gguf:"):- formats[i] = GGUFFormat(formats[i], ar)- elif formats[i] not in OutputFormat._format_list:- raise KeyError(f"Unsupported format {formats[i]}, please choose from {SUPPORTED_FORMATS}")- else:- formats[i] = OutputFormat._format_list[formats[i]](formats[i], ar)-- new_format = formats[i].check_and_reset_format(ar)- if new_format is not None:- if new_format not in format:- formats[i] = OutputFormat._format_list[new_format](new_format, ar)- else:- formats[i] = None-- formats = [fmt for fmt in formats if fmt is not None]-- if len(formats) == 1 and formats[0].is_gguf and ar.scale_dtype != torch.float32:- ar.scale_dtype = torch.float32- logger.info("change `scale_dtype` to `torch.float32` for gguf format")-- return formats---def _check_divisible_by_32(ar):- from auto_round.schemes import preset_name_to_scheme-- if isinstance(ar.scheme, str):- default_dict = asdict(preset_name_to_scheme(ar.scheme.upper()))- else:- default_dict = asdict(ar.scheme)- if default_dict["data_type"] == "int" and default_dict["act_bits"] >= 16:- for n, m in ar.model.named_modules():- if type(m) in ar.supported_types or m.__class__.__name__ in ar.inner_supported_types:- if m.weight.shape[0] % 32 or m.weight.shape[1] % 32:- if ar.layer_config is None:- ar.layer_config = {}- if ar.layer_config.get(n) is not None and ar.layer_config[n]["bits"] >= 16:- continue- ar.layer_config.setdefault(n, copy.deepcopy(default_dict))- ar.layer_config[n].update({"bits": 16, "data_type": "fp", "fixed_by_user": True})- logger.warning_once(f"{n} skipped quantization (shape not divisible by 32).")---class OutputFormat:- """ "Base class for different output formats.-- format: determines which method from export module to use for exporting.- For example, auto_round, gguf, llmcompressor etc.- backend: determines the specific export process within the format.- For example, auto_round:fp8_static, auto_round:auto_awq etc.- """-- support_schemes: list = []- _format_list: dict[str, OutputFormat] = {}- format_name = "base"-- def __init__(self, format: str, ar: BaseCompressor):- """Initialize the OutputFormat class."""- self.output_format = format- self.backend = None-- if not self.is_fake() and not self.is_support_scheme(ar.scheme):- logger.error(- f"Currently, the {self.format_name} format only supports {self.support_schemes}, "- f"but got scheme {ar.scheme}, please change to fake or auto_round etc."- )- exit(-1)-- @classmethod- def register(cls, *names: str) -> Callable[[OutputFormat], OutputFormat]:- assert names-- def func(output_format: OutputFormat) -> OutputFormat:- for name in names:- cls._format_list[name] = output_format- return output_format-- return func-- @classmethod- def get_support_matrix(cls: OutputFormat) -> str:- output_str = ""- for k, v in cls._format_list.items():- if k == "fake":- support_scheme = "All schemes"- else:- support_scheme = ", ".join(v.support_schemes).rstrip(",")- output_str += f"\x1b[31;1m{k}\x1b[0m support scheme:\n\t{support_scheme}\n"- return output_str-- def get_backend_name(self) -> str:- if self.backend is None:- return self.output_format-- # auto_round:llm_compressor:fp8_static- if self.backend.backend is not None:- return f"{self.output_format}:{self.backend.get_backend_name()}"- # auto_round:auto_awq, auto_round:auto_gptq- elif self.backend.get_backend_name() in self._format_list:- return f"{self.output_format}:{self.backend.get_backend_name()}"- # auto_round:fp8_static, llm_compressor:fp8_static- else:- return self.backend.get_backend_name()-- @classmethod- def is_support_scheme(cls: OutputFormat, scheme: Union[str, QuantizationScheme]) -> bool:- if isinstance(scheme, str) and scheme.upper() in cls.support_schemes:- return True- if isinstance(scheme, QuantizationScheme):- return True- return False-- def check_and_reset_format(self, ar: BaseCompressor) -> str:- if self.backend is not None:- new_format = self.backend.check_and_reset_format(ar)- self.backend = OutputFormat._format_list[new_format](new_format, ar) if new_format else self.backend-- w_fp8 = ar.data_type.startswith("fp") and ar.bits == 8- act_fp8 = ar.act_data_type.startswith("fp") and ar.act_bits == 8- if w_fp8 or act_fp8:- error_msg = (- f"is only supported to export auto_round or llm_compressor format,"- f" but got {self.format_name}, please check."- )- error_msg = ("act_data_type<fp8> " + error_msg) if act_fp8 else error_msg- error_msg = ("data_type<fp8> " + error_msg) if w_fp8 else error_msg- logger.error(error_msg)- sys.exit(-1)-- if ar.act_bits <= 8 and (not is_standard_fp(ar.act_data_type) or ar.act_dynamic):- logger.warning(- f"{self.format_name} format not support for current activation quantization configuration,"- " reset to fake format and save."- )- return "fake"-- return None-- def is_gguf(self) -> bool:- return "gguf" in self.output_format-- def is_fake(self) -> bool:- return self.output_format == "fake"-- def is_gptq(self) -> bool:- return "gptq" in self.output_format or (self.backend is not None and self.backend.is_gptq())-- def is_awq(self) -> bool:- return "awq" in self.output_format or (self.backend is not None and self.backend.is_awq())-- def is_llm_compressor(self) -> bool:- return "llm_compressor" in self.output_format or (self.backend is not None and self.backend.is_llm_compressor())--[email protected]("fake")-class FakeFormat(OutputFormat):- support_schemes = None- format_name = "fake"-- def check_and_reset_format(self, ar: BaseCompressor) -> str:- return None--[email protected]("llm_compressor", "llmcompressor")-class LLMCompressorFormat(OutputFormat):- support_schemes = ["MXFP4", "MXFP8", "NVFP4", "FPW8A16", "FP8_STATIC"]- format_name = "llm_compressor"-- def __init__(self, format, ar):- if not self.is_support_scheme(ar.scheme):- logger.error(- f"Currently, the llm_compressor format only supports {self.support_schemes}, "- f"but got scheme {ar.scheme}, please change to fake or auto_round etc."- )- exit(-1)- if format.startswith("llm_compressor"):- self.output_format = format- self.backend = None- if is_nv_fp(ar.data_type) or is_mx_fp(ar.data_type):- from auto_round.export.export_to_llmcompressor import check_compressed_tensors_supported-- check_compressed_tensors_supported()- self.backend = LLMCompressorFormat(ar.data_type, ar)- elif is_static_wfp8afp8(ar):- self.backend = LLMCompressorFormat(AutoRoundExportFormat.FP8_STATIC.value, ar)- if ar.act_group_size != 0:- logger.warning(- f"scheme FP8_STATIC export to llm_compressor format only support for act_group_size 0,"- f" ,but got act_group_size={ar.act_group_size}, reset = 0"- )- ar.act_group_size = 0- if ar.group_size > 0:- logger.warning(- f"please note that group_size={ar.group_size}"- " may not be supported for llm_compressor format, and cannot be loaded in llm_compressor"- )- else:- if format.upper() not in list(AutoRoundExportFormat.__members__.keys()):- raise KeyError(f"Unsupported backend format llm_compressor:{format}, please check")- self.output_format = f"llm_compressor:{format}"- self.backend = None-- def check_and_reset_format(self, ar: BaseCompressor) -> str:- if self.backend is not None:- new_format = self.backend.check_and_reset_format(ar)- self.backend = OutputFormat._format_list[new_format](new_format, ar) if new_format else self.backend-- if ar.act_bits <= 8 and (not is_standard_fp(ar.act_data_type) or ar.act_dynamic):- if (is_nv_fp(ar.act_data_type) and "static_gs" in ar.act_data_type) or (is_mx_fp(ar.act_data_type)):- return None- else:- bits, group_size, sym, act_bits = 8, -1, True, 8- assert (- ar.bits == bits- and ar.group_size == group_size- and ar.sym == sym- and ar.act_bits == act_bits- and ar.act_dynamic- ), (- f"Currently only support to export llm_compressor format for sym dynamic quantized"- f" W{ar.bits}A{ar.act_bits} model with group_size={group_size},"- f" but got bits={ar.bits}, group_size={ar.group_size}, sym={ar.sym},"- f" act_bits={ar.act_bits}"- )- return None- return None--[email protected]("auto_gptq", "gptqmodel")-class AutoGPTQFormat(OutputFormat):- support_schemes = ["W4A16", "W2A16", "W3A16", "W8A16", "BF16", "W2A16G64", "W2A16G32"]- format_name = "auto_gptq"-- def check_and_reset_format(self, ar):- if not ar.sym:- logger.warning(- "the asymmetrical kernel of the GPTQ format may result in a noticeable accuracy drop,"- " particularly for 2-bit quantization and smaller models."- " We recommend exporting to either the AutoAWQ format ( only 4 bits) or "- "the AutoRound format(2/3/4/8 bits)."- )- if self.backend is None:- _check_divisible_by_32(ar)- return super().check_and_reset_format(ar)--[email protected]("auto_awq")-class AutoAWQFormat(OutputFormat):- support_schemes = ["W4A16", "W2A16", "W3A16", "W8A16", "BF16", "W2A16G64", "W2A16G32"]- format_name = "auto_awq"-- @staticmethod- def check_awq_gemm_compatibility(model, bits, group_size, sym, layer_configs=None):- """Checks if a model is compatible with the AutoAWQ GEMM kernel.-- Args:- model: The model object to evaluate, typically a PyTorch model.- bits (int): The number of bits for quantization (must be 4 for compatibility).- group_size (int): The group size for quantization.- sym (bool): Whether symmetric quantization is used (not utilized in the current function logic).- layer_configs (dict, optional): A dictionary mapping layer names to configurations, where each- configuration can specify a custom number of bits for the layer.-- Returns:- tuple: A tuple containing:- - bool: `True` if the model is compatible, `False` otherwise.- - str: An error message describing why the model is incompatible, or an empty string if compatible.- """- from auto_round.utils.model import get_layer_names_in_block, get_module-- if bits != 4:- return False, "AutoAWQ GEMM kernel only supports 4 bits"- for n, m in model.named_modules():- if type(m) == transformers.pytorch_utils.Conv1D:- return False, "AutoAWQ GEMM kernel does not support conv1d"-- layer_names = get_layer_names_in_block(model)- for layer_name in layer_names:- if (- layer_configs is not None- and layer_name in layer_configs.keys()- and layer_configs[layer_name].get("bits", bits) > 8- ):- continue-- layer = get_module(model, layer_name)- if layer.in_features % group_size != 0:- return False, f"Layer {layer_name} in_features is not multiple of group_size {group_size}"- if layer.out_features % (32 // bits) != 0:- return False, f"Layer {layer_name} out_features is not multiple of 32 // bits"-- return True, ""-- def check_and_reset_format(self, ar):- awq_supported, info = self.check_awq_gemm_compatibility(- ar.model, ar.bits, ar.group_size, ar.sym, ar.layer_config- )- if not awq_supported:- logger.warning(f"The AutoAWQ format may not be supported due to {info}")- if ar.bits != 4:- raise ValueError("The AWQ format only supports W4 quantization ")-- if self.backend is None:- _check_divisible_by_32(ar)-- return super().check_and_reset_format(ar)--[email protected]("itrex")[email protected]("itrex_xpu")-class ITREXFormat(OutputFormat):- support_schemes = ["W4A16", "W2A16", "W3A16", "W8A16", "BF16", "W2A16G64", "W2A16G32"]- format_name = "itrex"--[email protected]("gguf")-class GGUFFormat(OutputFormat):- support_schemes = [- "GGUF:Q4_0",- "GGUF:Q4_1",- "GGUF:Q5_0",- "GGUF:Q5_1",- "GGUF:Q2_K_S",- "GGUF:Q3_K_S",- "GGUF:Q3_K_M",- "GGUF:Q3_K_L",- "GGUF:Q4_K_S",- "GGUF:Q4_K_M",- "GGUF:Q5_K_S",- "GGUF:Q5_K_M",- "GGUF:Q6_K",- "GGUF:Q8_0",- ]- format_name = "gguf"-- def __init__(self, format: str, ar: BaseCompressor):- if format.startswith("gguf:"):- self.gguf_args_check(ar, format, model_type=ModelType.TEXT)- if ar.mllm:- self.gguf_args_check(ar, format, model_type=ModelType.MMPROJ)- ar.scheme = format.upper()-- self.output_format = "gguf"- self.backend_cls = GGUFFormat- self.backend = GGUFFormat(format.split(":")[-1], ar)- else:- self.output_format = f"gguf:{format}"- self.backend = None-- def check_and_reset_format(self, ar):- if ar.iters != 0 and ar.bits != 3 and not ar.enable_alg_ext:- logger.warning_once(- "`iters=0` is recommended when exporting to current GGUF format"- " or add `enable_alg_ext` for better accuracy with much more tuning cost."- " Please refer to https://github.com/intel/auto-round/tree/main/docs/gguf_alg_ext_acc.md"- " for the accuracy results."- )- elif ar.bits >= 8 and ar.iters != 0:- logger.warning_once("`iters=0` is recommended for bits>=8")-- return super().check_and_reset_format(ar)-- @staticmethod- def gguf_args_check(args_or_ar, formats: Union[str, list[str]] = None, model_type=ModelType.TEXT):- import argparse-- from auto_round.export.export_to_gguf.config import GGUF_CONFIG- from auto_round.export.export_to_gguf.convert import download_convert_file- from auto_round.logger import logger- from auto_round.utils.model import download_or_get_path, get_gguf_architecture-- formats = [formats] if isinstance(formats, str) else formats- formats = sorted(formats, key=lambda x: len(x))- export_gguf = False- for f in formats:- if f.startswith("gguf"):- export_gguf = True-- if f.startswith("gguf") and f not in GGUF_CONFIG:- logger.error(f"{f} is not supported, please check.")-- redownload = False- if export_gguf:- try:- from auto_round.export.export_to_gguf.convert_hf_to_gguf import ( # pylint: disable=E0401- ModelBase,- ModelType,- get_model_architecture,- )-- if isinstance(args_or_ar.model, str):- model_path = args_or_ar.model- else:- model_path = args_or_ar.model.name_or_path- if not os.path.isdir(model_path):- model_path = download_or_get_path(model_path, args_or_ar.platform)- model_architecture = get_gguf_architecture(model_path, model_type=ModelType.TEXT)- if model_architecture not in ModelBase._model_classes[ModelType.TEXT]:- logger.warning(- f"Current version of gguf export does not support for {model_architecture},"- " will re-download dependency file. Please restart the task."- )- redownload = True- except ModuleNotFoundError as e:- if "convert_hf_to_gguf" in str(e):- logger.warning("GGUF export dependency file is not found, download from github.")- redownload = True- except AttributeError as e:- raise ImportError(- "Please use the latest gguf-py, you can use the following command to install it:\n"- "git clone https://github.com/ggml-org/llama.cpp.git && cd llama.cpp/gguf-py"- " && pip install . sentencepiece"- )- download_convert_file(redownload)-- try:- from auto_round.export.export_to_gguf.convert_hf_to_gguf import ( # pylint: disable=E0401- ModelBase,- ModelType,- )- except ImportError as e:- raise ImportError(- "Please use the latest gguf-py, you can use the following command to install it:\n"- "git clone https://github.com/ggml-org/llama.cpp.git && cd llama.cpp/gguf-py"- " && pip install . sentencepiece"- )- if isinstance(args_or_ar.model, str):- model_path = args_or_ar.model- else:- model_path = args_or_ar.model.name_or_path- if not os.path.isdir(model_path):- model_path = download_or_get_path(model_path, args_or_ar.platform)- model_architecture = get_gguf_architecture(model_path, model_type=ModelType.TEXT)- if model_architecture not in ModelBase._model_classes[ModelType.TEXT]:- logger.error(f"Model {model_architecture} is not supported to export gguf format.")- sys.exit(1)-- pattern = re.compile(r"q\d_k")- pre_dq_format = ""- unsupported_list, reset_list = [], []- for format in GGUF_CONFIG:- if format in formats:- if format == "q6_k_s":- logger.warning("Please note that q6_k_s is q6_k.")-- if re.search(pattern, format):- if pre_dq_format and re.search(pattern, format).group() not in pre_dq_format:- logger.error(f"Cannot export {pre_dq_format} and {format} at the same time.")- sys.exit(-1)- else:- pre_dq_format = format-- unsupported_list, reset_list = [], []- gguf_config = GGUF_CONFIG[format]- for k, v in gguf_config.items():- if not hasattr(args_or_ar, k):- continue- if k == "data_type":- if re.search(r"q\d_1", format) and len(formats) > 1:- v = "int"- if k == "sym" and isinstance(args_or_ar, argparse.Namespace):- k = "asym"- v = not v- if getattr(args_or_ar, k) != v:- unsupported_list.append(f"{k}={getattr(args_or_ar, k)}")- reset_list.append(f"{k}={v}")- setattr(args_or_ar, k, v)- if len(unsupported_list) > 0:- logger.info(- f"format {format} does not support for {', '.join(unsupported_list)},"- f" reset to {', '.join(reset_list)}."- )- # Removed obsolete commented-out block for improved readability and maintainability.- return args_or_ar--[email protected]("auto_round")[email protected]("auto_round:auto_awq")[email protected]("auto_round:llm_compressor")[email protected]("auto_round:gptqmodel", "auto_round:auto_gptq")-class AutoRoundFormat(OutputFormat):- support_schemes = [- "W4A16",- "W2A16",- "W3A16",- "W8A16",- "MXFP4",- "MXFP8",- "NVFP4",- "FPW8A16",- "W2A16G64",- "W2A16G32",- "FP8_STATIC",- "BF16",- ]- format_name = "auto_round"-- def __init__(self, format: str, ar: BaseCompressor):- self.output_format = "auto_round"- self.backend = None-- if format == "auto_round":- if ar.sym and "int" in ar.data_type:- self.backend = AutoGPTQFormat("auto_gptq", ar)- elif ar.bits == 4 and not ar.sym and "int" in ar.data_type:- enable_awq = all(- config["bits"] == ar.bits or config["bits"] >= 16 for config in ar.layer_config.values()- )- if enable_awq:- self.backend = AutoAWQFormat("auto_awq", ar)- elif is_nv_fp(ar.data_type) or is_mx_fp(ar.data_type):- self.backend = AutoRoundFormat(ar.data_type, ar)- elif is_static_wfp8afp8(ar): # static wfp8afp8- self.backend = AutoRoundFormat(AutoRoundExportFormat.FP8_STATIC.value, ar)- elif ar.data_type.startswith("fp") and ar.bits == 8 and ar.act_bits >= 16: # woq fp8- self.backend = AutoRoundFormat(AutoRoundExportFormat.FP8.value, ar)- elif ar.act_bits < 16:- raise ValueError(- "AutoRound format does not support exporting "- "for the current quantization configuration, "- "please change to `fake` format for research purpose"- )- elif not format.startswith("auto_round"):- if format.upper() not in list(AutoRoundExportFormat.__members__.keys()):- raise KeyError(f"Unsupported backend format auto_round:{format}, please check")- self.output_format = f"auto_round:{format}"- self.backend = None- else:- backend = format.split(":")[1] if ":" in format else None- self.backend = self._format_list.get(backend)(backend, ar) if backend else None-- if self.backend is not None:- self.support_schemes = self.backend.support_schemes-- def check_and_reset_format(self, ar):- if self.backend is not None:- new_format = self.backend.check_and_reset_format(ar)- self.backend = OutputFormat._format_list[new_format](new_format, ar) if new_format else self.backend-- if ar.act_bits <= 8:- if is_standard_fp(ar.act_data_type) and not ar.act_dynamic:- if (- ar.act_group_size != 0- and not ar.act_dynamic- and self.get_backend_name() == f"auto_round:{AutoRoundExportFormat.FP8.value}"- ):- logger.warning(- f"Please note that quantize activation with act_group_size={ar.act_group_size}"- " may result in failure to export or import normally."- )- if self.backend is None:- _check_divisible_by_32(ar)- return Nonediff --git a/auto_round/inference/convert_model.py b/auto_round/inference/convert_model.pyindex 4cd47d7e5..f4cd1c527 100644--- a/auto_round/inference/convert_model.py+++ b/auto_round/inference/convert_model.py@@ -20,7 +20,7 @@ import torch.nn as nnfrom tqdm import tqdmfrom transformers.pytorch_utils import Conv1D-from auto_round.formats import AutoRoundExportFormat+from auto_round.export.export_to_autoround import AutoRoundFormatfrom auto_round.inference.backend import (BackendInfos,dynamic_import_inference_linear,@@ -414,10 +414,10 @@ def _create_quant_layer(layer, layer_backend, config, in_features, out_features)bias=bias,)elif (- AutoRoundExportFormat.FP8_STATIC.value in layer_backend- or AutoRoundExportFormat.MXFP8.value in layer_backend- or AutoRoundExportFormat.MXFP4.value in layer_backend- or AutoRoundExportFormat.NVFP4.value in layer_backend+ AutoRoundFormat.FP8_STATIC.value in layer_backend+ or AutoRoundFormat.MXFP8.value in layer_backend+ or AutoRoundFormat.MXFP4.value in layer_backend+ or AutoRoundFormat.NVFP4.value in layer_backend):return QuantLinear.from_original(config, layer)diff --git a/auto_round/special_model_handler.py b/auto_round/special_model_handler.pyindex ef688634f..fe3865b85 100644--- a/auto_round/special_model_handler.py+++ b/auto_round/special_model_handler.py@@ -12,7 +12,6 @@# See the License for the specific language governing permissions and# limitations under the License.import auto_round.modelling as auto_round_modelling-from auto_round.formats import OutputFormatfrom auto_round.utils import LazyImport, logger, unsupported_meta_devicemllms_with_limited_bs = ("llava", "qwen2_vl", "phi3_v", "mllama") # Limitations on batch_size@@ -68,8 +67,8 @@ def _handle_special_model(model):return model-def _handle_moe_model(model, formats: list[OutputFormat] = None):- if formats is not None and any([format_.is_gguf() for format_ in formats]):+def _handle_moe_model(model, formats=None):+ if formats is not None and any(["gguf" in format_ for format_ in formats]):return modelif hasattr(model.config, "model_type") and model.config.model_type in CONVERT_EXPERT_TO_LINEAR_MODELS:from tqdm import tqdmdiff --git a/auto_round_extension/vllm_ext/moe_impl_mxfp4.py b/auto_round_extension/vllm_ext/moe_impl_mxfp4.pyindex 75e298acd..0ff5241ab 100644--- a/auto_round_extension/vllm_ext/moe_impl_mxfp4.py+++ b/auto_round_extension/vllm_ext/moe_impl_mxfp4.py@@ -283,6 +283,7 @@ class AutoRoundMoEMethodMXFp4Impl(AutoRoundMoEMethod):return revert_bias+ # breakpoint()if self.has_bias:if envs.VLLM_AR_POST_PROCESS_GPTOSS:w13_bias_swapped = revert_interleaved_bias(layer.w13_bias)diff --git a/test/test_cpu/test_export.py b/test/test_cpu/test_export.pyindex d5faa737a..57b30354b 100644--- a/test/test_cpu/test_export.py+++ b/test/test_cpu/test_export.py@@ -402,32 +402,6 @@ class TestAutoRound(unittest.TestCase):print(res)shutil.rmtree(quantized_model_path, ignore_errors=True)- def test_export_format(self):- from auto_round.formats import get_formats-- autoround = AutoRound(- self.model_name,- scheme="FP8_STATIC",- )- format_list = get_formats("auto_round, llm_compressor, auto_round:llm_compressor", autoround)- self.assertEqual(len(format_list), 3)- self.assertEqual(format_list[0].output_format, "auto_round")- self.assertEqual(format_list[0].get_backend_name(), "auto_round:fp8_static")- self.assertEqual(format_list[1].output_format, "llm_compressor")- self.assertEqual(format_list[1].get_backend_name(), "llm_compressor:fp8_static")- self.assertEqual(format_list[2].output_format, "auto_round")- self.assertEqual(format_list[2].get_backend_name(), "auto_round:llm_compressor:fp8_static")-- autoround = AutoRound(- self.model_name,- scheme="W4A16",- )- format_list = get_formats("auto_round:auto_awq, auto_gptq", autoround)- self.assertEqual(format_list[0].output_format, "auto_round")- self.assertEqual(format_list[0].get_backend_name(), "auto_round:auto_awq")- self.assertEqual(format_list[1].output_format, "auto_gptq")- self.assertEqual(format_list[1].get_backend_name(), "auto_gptq")-if __name__ == "__main__":unittest.main()diff --git a/test/test_cpu/test_mx_quant_linear.py b/test/test_cpu/test_mx_quant_linear.pyindex c2e9a3c00..e8e18c3bb 100644--- a/test/test_cpu/test_mx_quant_linear.py+++ b/test/test_cpu/test_mx_quant_linear.py@@ -3,14 +3,14 @@ import torchfrom auto_round.data_type.utils import get_quant_funcfrom auto_round.experimental import qmodules as ar_qmodules+from auto_round.export.export_to_autoround import AutoRoundFormatfrom auto_round.export.export_to_autoround.qlinear_fp import QuantLinear as _MXFPLinear-from auto_round.formats import AutoRoundExportFormatfrom auto_round.schemes import PRESET_SCHEMES-mx_schemes = [AutoRoundExportFormat.MXFP8.value, AutoRoundExportFormat.MXFP4.value]+mx_schemes = [AutoRoundFormat.MXFP8.value, AutoRoundFormat.MXFP4.value]QMODULE_MAPPING = {- AutoRoundExportFormat.MXFP8.value: ar_qmodules.MXFP8QuantLinear,- AutoRoundExportFormat.MXFP4.value: ar_qmodules.MXFP4QuantLinear,+ AutoRoundFormat.MXFP8.value: ar_qmodules.MXFP8QuantLinear,+ AutoRoundFormat.MXFP4.value: ar_qmodules.MXFP4QuantLinear,}diff --git a/test/test_cpu/test_mxfp_save_load.py b/test/test_cpu/test_mxfp_save_load.pyindex 5331949a0..aca5c7592 100644--- a/test/test_cpu/test_mxfp_save_load.py+++ b/test/test_cpu/test_mxfp_save_load.py@@ -9,22 +9,22 @@ from transformers.models.qwen2.modeling_qwen2 import Qwen2ForCausalLMfrom auto_round import AutoRoundfrom auto_round import schemes as ar_schemesfrom auto_round.experimental import qmodules as ar_qmodules+from auto_round.export.export_to_autoround import AutoRoundFormatfrom auto_round.export.export_to_autoround import qlinear_fp as ar_qlinear_fp-from auto_round.formats import AutoRoundExportFormatfrom auto_round.inference.backend import MX_TENSOR_DATA_TYPESfrom auto_round.testing_utils import has_moduletesting_scheme_name_lst = [- AutoRoundExportFormat.MXFP8.value,- AutoRoundExportFormat.MXFP4.value,+ AutoRoundFormat.MXFP8.value,+ AutoRoundFormat.MXFP4.value,]QMODULE_MAPPING = {- AutoRoundExportFormat.MXFP8.value: ar_qmodules.MXFP8QuantLinear,- AutoRoundExportFormat.MXFP4.value: ar_qmodules.MXFP4QuantLinear,+ AutoRoundFormat.MXFP8.value: ar_qmodules.MXFP8QuantLinear,+ AutoRoundFormat.MXFP4.value: ar_qmodules.MXFP4QuantLinear,}SCHEMES_MAPPING = {- AutoRoundExportFormat.MXFP8.value: ar_schemes.MXFP8,- AutoRoundExportFormat.MXFP4.value: ar_schemes.MXFP4,+ AutoRoundFormat.MXFP8.value: ar_schemes.MXFP8,+ AutoRoundFormat.MXFP4.value: ar_schemes.MXFP4,}diff --git a/test/test_cpu/test_nvfp4_quant_linear.py b/test/test_cpu/test_nvfp4_quant_linear.pyindex 80a289b70..0a42f009a 100644--- a/test/test_cpu/test_nvfp4_quant_linear.py+++ b/test/test_cpu/test_nvfp4_quant_linear.py@@ -4,12 +4,12 @@ import torchfrom auto_round.data_type.nvfp import calculate_gparamfrom auto_round.data_type.utils import get_quant_funcfrom auto_round.experimental import qmodules as ar_qmodules+from auto_round.export.export_to_autoround import AutoRoundFormatfrom auto_round.export.export_to_autoround.qlinear_fp import QuantLinear as _FPLinear-from auto_round.formats import AutoRoundExportFormatfrom auto_round.schemes import PRESET_SCHEMESQMODULE_MAPPING = {- AutoRoundExportFormat.NVFP4.value: ar_qmodules.NVFP4QuantLinear,+ AutoRoundFormat.NVFP4.value: ar_qmodules.NVFP4QuantLinear,}@@ -26,7 +26,7 @@ def fixed_seed():# (Optional) cleanup or reset after test[email protected]("scheme", [AutoRoundExportFormat.NVFP4.value])[email protected]("scheme", [AutoRoundFormat.NVFP4.value])@torch.inference_mode()def test_nvfp4_quantlinear_from_original_and_forward(scheme):"""diff --git a/test/test_cuda/test_mxfp_and_nvfp_quant.py b/test/test_cuda/test_mxfp_and_nvfp_quant.pyindex 4954b2320..0dc43b093 100644--- a/test/test_cuda/test_mxfp_and_nvfp_quant.py+++ b/test/test_cuda/test_mxfp_and_nvfp_quant.py@@ -8,19 +8,15 @@ from transformers import AutoModelForCausalLM, AutoTokenizerfrom auto_round import AutoRoundfrom auto_round import schemes as ar_schemesfrom auto_round.experimental import qmodules as ar_qmodules+from auto_round.export.export_to_autoround import AutoRoundFormatfrom auto_round.export.export_to_autoround import qlinear_fp as ar_qlinear_fp-from auto_round.formats import AutoRoundExportFormatfrom auto_round.testing_utils import has_module-testing_schemes = [- AutoRoundExportFormat.MXFP8.value,- AutoRoundExportFormat.MXFP4.value,- AutoRoundExportFormat.NVFP4.value,-]+testing_schemes = [AutoRoundFormat.MXFP8.value, AutoRoundFormat.MXFP4.value, AutoRoundFormat.NVFP4.value]QMODULE_MAPPING = {- AutoRoundExportFormat.MXFP8.value: ar_qmodules.MXFP8QuantLinear,- AutoRoundExportFormat.MXFP4.value: ar_qmodules.MXFP4QuantLinear,- AutoRoundExportFormat.NVFP4.value: ar_qmodules.NVFP4QuantLinear,+ AutoRoundFormat.MXFP8.value: ar_qmodules.MXFP8QuantLinear,+ AutoRoundFormat.MXFP4.value: ar_qmodules.MXFP4QuantLinear,+ AutoRoundFormat.NVFP4.value: ar_qmodules.NVFP4QuantLinear,}