PTQ-Level Efficiency, QAT-Level Accuracy | 训练后量化的效率,量化感知训练的精度
[STEP 1] refact output formats (#906)
Heng Guocf99bb3be0c8
diff --git a/auto_round/__main__.py b/auto_round/__main__.pyindex 94e3ad163..8541f1c3b 100644--- a/auto_round/__main__.py+++ b/auto_round/__main__.py@@ -140,7 +140,9 @@ class BasicArgumentParser(argparse.ArgumentParser):"--format",default="auto_round",type=str,- help="Output format for the quantized model." "'auto_round' is the recommended format",+ help="Output format for the quantized model."+ "'auto_round' is the recommended format"+ "use command `auto_round list format` to show all supported formats with support scheme.",)basic.add_argument("--output_dir",@@ -462,14 +464,25 @@ class BasicArgumentParser(argparse.ArgumentParser):)-def setup_parser(recipe="default"):+def list_item():+ args = argparse.ArgumentParser()+ args.add_argument("item", type=str, help="item to list, e.g., format")+ args = args.parse_args()+ if args.item == "format":+ from auto_round.formats import OutputFormat++ print("AutoRound supported output formats and quantization scheme:")+ print(OutputFormat.get_support_matrix())+++def start(recipe="default"):recipe = RECIPES[recipe]parser = BasicArgumentParser()args = parser.parse_args()for k, v in recipe.items():if getattr(args, k) is None:setattr(args, k, v)- return args+ tune(args)def tune(args):@@ -881,27 +894,33 @@ def run_eval():def run():- if "--eval" in sys.argv:- sys.argv.remove("--eval")+ if "list" in sys.argv or "--list" in sys.argv:+ if "list" in sys.argv:+ sys.argv.remove("list")+ if "--list" in sys.argv:+ sys.argv.remove("--list")+ list_item()+ exit()+ if "--eval" in sys.argv or "eval" in sys.argv:+ if "--eval" in sys.argv:+ sys.argv.remove("--eval")+ if "eval" in sys.argv:+ sys.argv.remove("eval")run_eval()else:- args = setup_parser()- tune(args)+ start()def run_best():- args = setup_parser("best")- tune(args)+ start("best")def run_light():- args = setup_parser("light")- tune(args)+ start("light")def run_fast():- args = setup_parser("fast")- tune(args)+ start("fast")if __name__ == "__main__":diff --git a/auto_round/compressors/base.py b/auto_round/compressors/base.pyindex 4a860ae10..69f3be72d 100644--- a/auto_round/compressors/base.py+++ b/auto_round/compressors/base.py@@ -40,7 +40,6 @@ from auto_round.compressors.utils import (check_skippable_keywords,collect_best_params,get_shared_keys,- gguf_args_check,immediate_saving,infer_bits_by_data_type,init_cache,@@ -54,8 +53,8 @@ from auto_round.compressors.utils import ()from auto_round.data_type import QUANT_FUNC_WITH_DTYPEfrom auto_round.data_type.utils import reshape_pad_tensor_by_group_size-from auto_round.export.export_to_autoround import AutoRoundFormatfrom auto_round.export.export_to_gguf.config import GGUF_INNER_CONFIG, ModelType+from auto_round.formats import OutputFormat, get_formatsfrom auto_round.logger import loggerfrom auto_round.schemes import (SPECIAL_SCHEMES,@@ -69,7 +68,6 @@ from auto_round.special_model_handler import _handle_moe_modelfrom auto_round.utils import (INNER_SUPPORTED_LAYER_TYPES,SUPPORTED_DTYPES,- SUPPORTED_FORMATS,SUPPORTED_LAYER_TYPES,TORCH_VERSION_AT_LEAST_2_6,CpuInfo,@@ -736,35 +734,6 @@ class BaseCompressor(object):def _check_compatibility(self) -> None:"""Checks compatibility of the configurations and model."""- # Check gguf and others- has_gguf = False- if hasattr(self, "formats"):- has_besides_gguf = False- for format_ in self.formats:- if "gguf" in format_:- has_gguf = True- elif format_ != "fake":- has_besides_gguf = True- if has_gguf and has_besides_gguf:- raise ValueError("GGUF format is not compatible with other formats, please choose only one of them")- if has_gguf:- from transformers.utils.versions import require_version-- require_version(- "sentencepiece",- "GGUF format requires SentencePiece to be installed. "- "Please install it with `pip install sentencepiece`",- )- if has_gguf and self.iters != 0 and self.bits != 3 and not self.enable_alg_ext:- logger.warning(- "`iters=0` is recommended when exporting to current GGUF format"- " or add `enable_alg_ext` for better accuracy with much more tuning cost."- " Please refer to https://github.com/intel/auto-round/tree/main/docs/gguf_alg_ext_acc.md"- " for the accuracy results."- )- elif self.bits >= 8 and self.iters != 0:- logger.warning("`iters=0` is recommended for bits>=8")-if (self.seqlen is not Noneand hasattr(self.model, "config")@@ -789,230 +758,6 @@ class BaseCompressor(object):if self.group_size == 0 and "fp8" not in self.data_type:logger.warning("`group_size==0` is not supported for data_type other than fp8 ")- def _parse_format_to_list(self, format: str) -> list:- """Parses the format string into a list of formats.-- This method checks the requested format(s) against the model's- quantization settings and adjusts them if necessary. It ensures that- the formats are compatible with the model's data type, bit width,- and activation quantization settings.-- Args:- format (str): The requested format(s) for quantization, separated by commas.-- Returns:- list: A list of validated and updated formats.- """-- # Remove duplicates from formats list- def remove_duplicates(lst):- seen = set()- return [x for x in lst if not (x in seen or seen.add(x))]-- formats = format.replace("q*_", f"q{self.bits}_").replace(" ", "").split(",")- formats = remove_duplicates(formats) # need the keep origin order-- gguf_format_name = get_gguf_scheme(self.scheme)-- if gguf_format_name:- for i in range(len(formats)):- if gguf_format_name.lower().endswith("mixed"):- gguf_format_name = gguf_format_name.lower().replace("_mixed", "_s")- if formats[i] != "fake" and formats[i] != gguf_format_name.lower():- logger.warning(- f"reset format {formats[i]} to {gguf_format_name.lower()} "- f"since scheme {gguf_format_name} can only be exported to format {gguf_format_name.lower()}"- )- formats[i] = gguf_format_name.lower()-- gguf_args_check(self, formats, model_type=ModelType.TEXT)- if self.mllm:- gguf_args_check(self, formats, model_type=ModelType.MMPROJ)-- for f in formats:- if f.startswith("gguf"):- self.scheme = f.upper()- break-- for format_ in formats:- if format_ not in SUPPORTED_FORMATS:- logger.error(f"Unsupported format {format_}, please choose from {SUPPORTED_FORMATS}")- exit(-1)- if self.scale_dtype != torch.float32:- only_gguf = True- for format_ in formats:- if not ("gguf" in format_ or "fake" in format_):- only_gguf = False- break- if len(formats) == 1 and "fake" == formats[0]:- only_gguf = False- if only_gguf:- self.scale_dtype = torch.float32- logger.info("change `scale_dtype` to `torch.float32`")-- # Adjust format settings based on compatibility- for index in range(len(formats)):- format = formats[index]- if format == "auto_round":- if self.sym and "int" in self.data_type:- format = "auto_round:auto_gptq"- elif self.bits == 4 and not self.sym and "int" in self.data_type:- enable_awq = all(- config["bits"] == self.bits or config["bits"] >= 16 for config in self.layer_config.values()- )- if enable_awq:- format = "auto_round:auto_awq"- elif is_nv_fp(self.data_type) or is_mx_fp(self.data_type):- format = f"auto_round:{self.data_type}"- elif is_static_wfp8afp8(self): # static wfp8afp8- format = f"auto_round:{AutoRoundFormat.FP8_STATIC.value}"- elif self.data_type.startswith("fp") and self.bits == 8 and self.act_bits >= 16: # woq fp8- format = f"auto_round:{AutoRoundFormat.FP8.value}"- elif self.act_bits < 16:- raise ValueError(- "AutoRound format does not support exporting "- "for the current quantization configuration, "- "please change to `fake` format for research purpose"- )- formats[index] = format- elif format == "llm_compressor":- from auto_round.export.export_to_llmcompressor import check_compressed_tensors_supported-- if is_nv_fp(self.data_type) or is_mx_fp(self.data_type):- check_compressed_tensors_supported()- format = format.replace("llm_compressor", f"llm_compressor:{self.data_type}")- formats[index] = format- elif is_static_wfp8afp8(self):- format = f"llm_compressor:{AutoRoundFormat.FP8_STATIC.value}"- formats[index] = format- if self.act_group_size != 0:- logger.warning(- f"scheme FP8_STATIC export to llm_compressor format only support for act_group_size 0,"- f" ,but got act_group_size={self.act_group_size}, reset = 0"- )- self.act_group_size = 0- if self.group_size > 0:- logger.warning(- f"please note that group_size={self.group_size}"- " may not be supported for llm_compressor format, and cannot be loaded in llm_compressor"- )- elif not is_wfp8afp8(self):- logger.error(- "Currently, the llm_compressor format only supports MXFP/NVFP/FP8. "- "Please change format to fake or auto_round etc."- )- elif "auto_awq" in format:- from auto_round.compressors.utils import check_awq_gemm_compatibility-- awq_supported, info = check_awq_gemm_compatibility(- self.model, self.bits, self.group_size, self.sym, self.layer_config- )- if not awq_supported:- logger.warning(f"The AutoAWQ format may not be supported due to {info}")- else:- if (is_nv_fp(self.data_type) or is_mx_fp(self.data_type)) and format != "fake":- logger.warning(f"nv_fp and mx_fp dtypes are not supported for export format: {format}")-- formats = remove_duplicates(formats)- for i in range(len(formats)):- formats[i] = self._check_supported_format(formats[i])- formats = remove_duplicates(formats)- return formats-- def _check_supported_format(self, format: str) -> bool:- """Checks if the specified format is supported.-- This method validates the requested format against the model's bit width,- group size, symmetry, and activation quantization settings. It raises an- error if the format is incompatible with the current model configuration.-- Args:- format (str): The requested format for quantization.-- Returns:- bool: True if the format is supported, False otherwise.- """- if format == "fake":- return format- format = format.replace("q*_", f"q{self.bits}_")-- # format check for fp8- w_fp8 = self.data_type.startswith("fp") and self.bits == 8- act_fp8 = self.act_data_type.startswith("fp") and self.act_bits == 8- if (w_fp8 or act_fp8) and re.search("^auto_round|^llm_compressor", format) is None:- error_msg = (- f"is only supported to export auto_round or llm_compressor format," f" but got {format}, please check."- )- error_msg = ("act_data_type<fp8> " + error_msg) if act_fp8 else error_msg- error_msg = ("data_type<fp8> " + error_msg) if w_fp8 else error_msg- logger.error(error_msg)- sys.exit(-1)-- # Only support to export afp8/nv_fp/mx_fp- if self.act_bits <= 8:- if not is_standard_fp(self.act_data_type) or self.act_dynamic:- if "llm_compressor" in format:- if (is_nv_fp(self.act_data_type) and "static_gs" in self.act_data_type) or (- is_mx_fp(self.act_data_type)- ):- return format- bits, group_size, sym, act_bits = 8, -1, True, 8- assert (- self.bits == bits- and self.group_size == group_size- and self.sym == sym- and self.act_bits == act_bits- and self.act_dynamic- ), (- f"Currently only support to export llm_compressor format for sym dynamic quantized"- f" W{self.bits}A{self.act_bits} model with group_size={group_size},"- f" but got bits={self.bits}, group_size={self.group_size}, sym={self.sym},"- f" act_bits={self.act_bits}"- )- elif "auto_round" in format and (- is_mx_fp(self.act_data_type) or (is_nv_fp(self.act_data_type) and "static_gs" in self.act_data_type)- ):- pass- elif format != "fake":- logger.warning(- "Currently only support to export auto_round format quantized model"- " with fp8, mx_fp and nv_fp4 dtype activation for activation quantization."- " Change format to fake and save."- )- format = "fake"- else:- if format not in [- "auto_round",- f"auto_round:{AutoRoundFormat.FP8_STATIC.value}",- f"llm_compressor:{AutoRoundFormat.FP8_STATIC.value}",- "auto_round:llm_compressor",- ]:- logger.warning(- f"Currently only support to export auto_round or fake format for static W{self.bits}AFP8 model,"- f" change format {format} to auto_round"- )- if is_static_wfp8afp8(self):- format = f"auto_round:{AutoRoundFormat.FP8_STATIC.value}"- else:- format = f"auto_round:{AutoRoundFormat.FP8.value}"- if (- self.act_group_size != 0- and not self.act_dynamic- and format == f"auto_round:{AutoRoundFormat.FP8.value}"- ):- logger.warning(- f"Please note that quantize activation with act_group_size={self.act_group_size}"- " may result in failure to export or import normally."- )- if re.search(r"q\d_k", format) and not self.data_type.endswith("_dq"):- logger.error(- f"datatype<{self.data_type}> not support to export {format} format."- " Please change export format or `data_type`."- )- sys.exit(-1)-- return format-def quantize_and_save(self, output_dir: str = "tmp_autoround", format: str = "auto_round", inplace: bool = True, **kwargs) -> tuple[torch.nn.Module, dict[str, Any]]:@@ -1042,7 +787,7 @@ class BaseCompressor(object):self.orig_output_dir = output_dir# check and update the format based on the current configuration- format_list = self._parse_format_to_list(format)+ format_list = get_formats(format, self)self.formats = format_list# If multiple formats are specified, enforce inplace=False@@ -1065,24 +810,12 @@ class BaseCompressor(object):else:model, _ = self.quantize()# Save the quantized model in the specified format_list- folders = []- for format in format_list:- if "gptq" in format and not self.sym:- logger.warning(- "The asymmetrical kernel of the GPTQ format may result in a noticeable accuracy drop,"- " particularly for 2-bit quantization and smaller models."- " We recommend exporting to either the AutoAWQ format ( only 4 bits) or "- "the AutoRound format(2/3/4/8 bits)."- )- save_folder = self._get_save_folder_name(format)- self.save_quantized(save_folder, format=format, inplace=inplace, **kwargs)-- folders.append(save_folder)+ model, folders = self.save_quantized(output_dir, format=format, inplace=inplace, return_folders=True, **kwargs)memory_monitor.log_summary()return model, folders- def _get_save_folder_name(self, format_str: str) -> str:+ def _get_save_folder_name(self, format: OutputFormat) -> str:"""Generates the save folder name based on the provided format string.If there are multiple formats to handle, the function creates a subfolder@@ -1096,7 +829,7 @@ class BaseCompressor(object):str: The path to the folder where results should be saved."""# Replace special characters to make the folder name filesystem-safe- sanitized_format = format_str.replace(":", "-").replace("_", "-")+ sanitized_format = format.get_backend_name().replace(":", "-").replace("_", "-")# Use a subfolder only if there are multiple formatsif len(self.formats) > 1:@@ -1307,7 +1040,7 @@ class BaseCompressor(object):set_module(self.model, name, m)tuning_device = m.tuning_device if hasattr(m, "tuning_device") else self.device# Step 1: Try quantization on GPU first, fall back to CPU if OOM- if self.immediate_packing and self.iters == 0 and "gguf" in self.formats[0] and not self.disable_opt_rtn:+ if self.immediate_packing and self.iters == 0 and self.formats[0].is_gguf() and not self.disable_opt_rtn:m = m.to(tuning_device)m.scale = Nonem.zp = None@@ -1367,8 +1100,8 @@ class BaseCompressor(object):returnfrom auto_round.export import PACKING_LAYER_WITH_FORMAT- target_backend = self.formats[0].split(":")[0] if ":" in self.formats[0] else self.formats[0]- has_gguf = any("gguf" in fmt for fmt in self.formats)+ target_backend = self.formats[0].output_format+ has_gguf = any(fmt.is_gguf() for fmt in self.formats)if has_gguf:from auto_round.export.export_to_gguf.export import pack_gguf_layer@@ -1378,7 +1111,7 @@ class BaseCompressor(object):pack_gguf_layer(name,self.model,- self.formats[0],+ self.formats[0].get_backend_name(),output_dir,self.layer_config,self.tokenizer,@@ -1388,7 +1121,9 @@ class BaseCompressor(object):device=self.device,)else:- PACKING_LAYER_WITH_FORMAT[target_backend](name, self.model, self.formats[0], device=self.device)+ PACKING_LAYER_WITH_FORMAT[target_backend](+ name, self.model, self.formats[0].get_backend_name(), device=self.device+ )@torch.inference_mode()def _quantize_rtn(self) -> tuple[torch.nn.Module, dict[str, Any]]:@@ -1423,7 +1158,7 @@ class BaseCompressor(object):update_fused_layer_global_scales(module)logger.info("Finished updating fused layer global scales.")- if not (any("gguf" in fmt for fmt in getattr(self, "formats", [])) or self.super_bits is not None):+ if not (any(fmt.is_gguf() for fmt in getattr(self, "formats", [])) or self.super_bits is not None):self._quantize_embedding_layer() # leave to gguf itself to handleself.model.to("cpu")@@ -1433,7 +1168,8 @@ class BaseCompressor(object):enable_imatrix = Falseif not self.disable_opt_rtn:has_gguf_k = (- any("gguf" in fmt and "k" in fmt for fmt in getattr(self, "formats", [])) or self.super_bits is not None+ any(fmt.is_gguf() and "k" in fmt.output_format for fmt in getattr(self, "formats", []))+ or self.super_bits is not None)if has_gguf_k:enable_imatrix = True@@ -1672,20 +1408,11 @@ class BaseCompressor(object):else:# Determine if immediate packing is requiredformats = self.formats- if (- len(formats) == 1- and (- "awq" in formats[0]- or "gptq" in formats[0]- or "auto_round" in formats[0]- or "gguf" in formats[0]- or "llm_compressor" in formats[0]- )- and self.inplace- ):+ if len(formats) == 1 and not formats[0].is_fake() and self.inplace:self.immediate_packing = True- if "gguf" not in formats[0] and self.low_cpu_mem_usage:+ if not formats[0].is_gguf() and self.low_cpu_mem_usage:self.immediate_saving = True+if self.immediate_saving and "int" not in self.data_type:logger.warning("immediate_saving is only supported for int quantization, set to False")self.immediate_saving = False@@ -1862,7 +1589,7 @@ class BaseCompressor(object):has_gguf = Falseif hasattr(self, "formats"):- has_gguf = any("gguf" in format_ for format_ in self.formats)+ has_gguf = any(format_.is_gguf() for format_ in self.formats)if has_gguf and self.immediate_packing:enable_quanted_input = False@@ -3133,7 +2860,12 @@ class BaseCompressor(object):clear_memory(device_list=self.device_list)def save_quantized(- self, output_dir: str = None, format: str = "auto_round", inplace: bool = True, **kwargs+ self,+ output_dir: str = None,+ format: Union[str, list[OutputFormat]] = "auto_round",+ inplace: bool = True,+ return_folders=False,+ **kwargs,) -> torch.nn.Module:"""Save the quantized model to the specified output directory in the specified format.@@ -3146,122 +2878,118 @@ class BaseCompressor(object):Returns:object: The compressed model object."""- format = self._check_supported_format(format)+ self.orig_output_dir = output_dir+ if isinstance(format, str):+ formats = get_formats(format, self)+ if not hasattr(self, "formats"):+ self.formats = formatsif not self.quantized:logger.warning("please run autoround.quantize first")return- if format == "fake" or format == "qdq": # TODO fix act quantization later- self.model = self.model.to("cpu")- self.model.save_pretrained(output_dir)- if self.tokenizer is not None and hasattr(self.tokenizer, "save_pretrained"):- self.tokenizer.save_pretrained(output_dir)- processor = kwargs.get("processor", None)- if processor is not None:- processor.save_pretrained(output_dir)- try:- copy_python_files_from_model_cache(self.model, output_dir)- except Exception as e:- logger.warning("Skipping source model Python file copy due to error: %s", e)- return- if self.act_bits <= 8 and format == "qdq":- logger.warning(- "Support for exporting activation quantization is limited. "- "Please ensure that your configuration is supported."- )- # if format == "llm_compressor" and (is_nv_fp(self.data_type) or is_mx_fp(self.data_type)):- # format = format.replace("llm_compressor", f"llm_compressor:{self.data_type}")- if format == "llm_compressor" and (is_nv_fp(self.data_type) or is_mx_fp(self.data_type)):- format = format.replace("llm_compressor", f"llm_compressor:{self.data_type}")- if format == "llm_compressor" and is_static_wfp8afp8(self):- format = format.replace("llm_compressor", "llm_compressor:{AutoRoundFormat.FP8_STATIC.value}")-- from auto_round.export import EXPORT_FORMAT-- backend = format- format = format.split(":")[0]- if format not in EXPORT_FORMAT:- logger.error(f"export format only supports {EXPORT_FORMAT.keys()}")- raise ValueError(f"export format only supports {EXPORT_FORMAT.keys()}, but got {format}")- save_quantized_as_format = EXPORT_FORMAT.get(format)- if "gptq" in format and not self.sym:- logger.warning(- "the asymmetrical kernel of the GPTQ format may result in a noticeable accuracy drop,"- " particularly for 2-bit quantization and smaller models."- " We recommend exporting to either the AutoAWQ format ( only 4 bits) or "- "the AutoRound format(2/3/4/8 bits)."+ folders = []+ for format in formats:+ save_folder = self._get_save_folder_name(format)+ if format.is_fake(): # TODO fix act quantization later+ self.model = self.model.to("cpu")+ self.model.save_pretrained(output_dir)+ if self.tokenizer is not None and hasattr(self.tokenizer, "save_pretrained"):+ self.tokenizer.save_pretrained(output_dir)+ processor = kwargs.get("processor", None)+ if processor is not None:+ processor.save_pretrained(output_dir)+ try:+ copy_python_files_from_model_cache(self.model, output_dir)+ except Exception as e:+ logger.warning("Skipping source model Python file copy due to error: %s", e)+ compressed_model = self.model+ continue+ if self.act_bits <= 8 and format.is_fake():+ logger.warning(+ "Support for exporting activation quantization is limited. "+ "Please ensure that your configuration is supported."+ )+ from auto_round.export import EXPORT_FORMAT++ backend = format.get_backend_name()+ output_format = format.output_format+ if output_format not in EXPORT_FORMAT:+ raise ValueError(f"export format only supports {EXPORT_FORMAT.keys()}, but got {output_format}")+ save_quantized_as_format = EXPORT_FORMAT.get(output_format)+ serialization_keys = [+ "bits",+ "group_size",+ "sym",+ "data_type",+ "enable_quanted_input",+ "enable_minmax_tuning",+ "seqlen",+ "batch_size",+ "scale_dtype",+ "lr",+ "minmax_lr",+ "gradient_accumulate_steps",+ "iters",+ "amp",+ "nsamples",+ "low_gpu_mem_usage",+ "to_quant_block_names",+ "enable_norm_bias_tuning",+ "act_bits",+ "act_group_size",+ "act_sym",+ "act_dynamic",+ "act_data_type",+ "super_bits",+ "super_group_size",+ "regex_config",+ "static_kv_dtype",+ "static_attention_dtype",+ ]+ if isinstance(self.dataset, str):+ serialization_keys.append("dataset")+ serialization_dict = {}+ for key in serialization_keys:+ serialization_dict[key] = getattr(self, key)+ from auto_round.version import __version__++ serialization_dict["autoround_version"] = __version__+ if "scale_dtype" in serialization_dict.keys():+ serialization_dict["scale_dtype"] = str(serialization_dict["scale_dtype"])+ compressed_model = save_quantized_as_format( # TODO refine the code+ save_folder,+ model=self.model,+ layer_config=self.layer_config,+ inplace=inplace,+ bits=self.bits,+ act_bits=self.act_bits,+ group_size=self.group_size,+ sym=self.sym,+ iters=self.iters,+ lr=self.lr,+ minmax_lr=self.minmax_lr,+ enable_minmax_tuning=self.enable_minmax_tuning,+ enable_quanted_input=self.enable_quanted_input,+ scale_dtype=self.scale_dtype,+ tokenizer=self.tokenizer,+ supported_types=self.supported_types,+ data_type=self.data_type,+ act_data_type=self.act_data_type,+ serialization_dict=serialization_dict,+ backend=backend,+ to_quant_block_names=self.to_quant_block_names,+ quant_block_list=self.quant_block_list,+ device=self.device,+ static_kv_dtype=self.static_kv_dtype,+ static_attention_dtype=self.static_attention_dtype,+ **kwargs,)- if "awq" in format and not self.bits == 4:- raise ValueError("The AWQ format only supports W4 quantization ")- serialization_keys = [- "bits",- "group_size",- "sym",- "data_type",- "enable_quanted_input",- "enable_minmax_tuning",- "seqlen",- "batch_size",- "scale_dtype",- "lr",- "minmax_lr",- "gradient_accumulate_steps",- "iters",- "amp",- "nsamples",- "low_gpu_mem_usage",- "to_quant_block_names",- "enable_norm_bias_tuning",- "act_bits",- "act_group_size",- "act_sym",- "act_dynamic",- "act_data_type",- "super_bits",- "super_group_size",- "regex_config",- "static_kv_dtype",- "static_attention_dtype",- ]- if isinstance(self.dataset, str):- serialization_keys.append("dataset")- serialization_dict = {}- for key in serialization_keys:- serialization_dict[key] = getattr(self, key)- from auto_round.version import __version__-- serialization_dict["autoround_version"] = __version__- if "scale_dtype" in serialization_dict.keys():- serialization_dict["scale_dtype"] = str(serialization_dict["scale_dtype"])- compressed_model = save_quantized_as_format( # TODO refine the code- output_dir,- model=self.model,- layer_config=self.layer_config,- inplace=inplace,- bits=self.bits,- act_bits=self.act_bits,- group_size=self.group_size,- sym=self.sym,- iters=self.iters,- lr=self.lr,- minmax_lr=self.minmax_lr,- enable_minmax_tuning=self.enable_minmax_tuning,- enable_quanted_input=self.enable_quanted_input,- scale_dtype=self.scale_dtype,- tokenizer=self.tokenizer,- supported_types=self.supported_types,- data_type=self.data_type,- act_data_type=self.act_data_type,- serialization_dict=serialization_dict,- backend=backend,- to_quant_block_names=self.to_quant_block_names,- quant_block_list=self.quant_block_list,- device=self.device,- static_kv_dtype=self.static_kv_dtype,- static_attention_dtype=self.static_attention_dtype,- **kwargs,- )- return compressed_model+ folders.append(save_folder)++ if return_folders:+ return compressed_model, folders+ else:+ return compressed_modeldef _get_quantized_layer_names_outside_blocks(self) -> list:"""Gets the names of quantized layers outside blocks in the model.diff --git a/auto_round/compressors/utils.py b/auto_round/compressors/utils.pyindex dd85f4ebd..268cbed5b 100644--- a/auto_round/compressors/utils.py+++ b/auto_round/compressors/utils.py@@ -164,48 +164,6 @@ def check_need_act_calibration(return False-def check_awq_gemm_compatibility(model, bits, group_size, sym, layer_configs=None):- """Checks if a model is compatible with the AutoAWQ GEMM kernel.-- Args:- model: The model object to evaluate, typically a PyTorch model.- bits (int): The number of bits for quantization (must be 4 for compatibility).- group_size (int): The group size for quantization.- sym (bool): Whether symmetric quantization is used (not utilized in the current function logic).- layer_configs (dict, optional): A dictionary mapping layer names to configurations, where each- configuration can specify a custom number of bits for the layer.-- Returns:- tuple: A tuple containing:- - bool: `True` if the model is compatible, `False` otherwise.- - str: An error message describing why the model is incompatible, or an empty string if compatible.- """- from auto_round.utils.model import get_layer_names_in_block, get_module-- if bits != 4:- return False, "AutoAWQ GEMM kernel only supports 4 bits"- for n, m in model.named_modules():- if type(m) == transformers.pytorch_utils.Conv1D:- return False, "AutoAWQ GEMM kernel does not support conv1d"-- layer_names = get_layer_names_in_block(model)- for layer_name in layer_names:- if (- layer_configs is not None- and layer_name in layer_configs.keys()- and layer_configs[layer_name].get("bits", bits) > 8- ):- continue-- layer = get_module(model, layer_name)- if layer.in_features % group_size != 0:- return False, f"Layer {layer_name} in_features is not multiple of group_size {group_size}"- if layer.out_features % (32 // bits) != 0:- return False, f"Layer {layer_name} out_features is not multiple of 32 // bits"-- return True, ""--def collect_best_params(block, cache_device="cpu"):"""Collect the best parameters from the block to the specified device."""params = {}@@ -513,117 +471,6 @@ def gguf_type_fallback(gguf_type: str) -> str:return gguf_type-def gguf_args_check(args_or_ar, formats: list[str] = None, model_type=ModelType.TEXT):- import argparse-- from auto_round.export.export_to_gguf.convert import download_convert_file- from auto_round.logger import logger- from auto_round.utils.model import download_or_get_path, get_gguf_architecture-- formats = sorted(formats, key=lambda x: len(x))- export_gguf = False- for f in formats:- if f.startswith("gguf"):- export_gguf = True-- if f.startswith("gguf") and f not in GGUF_CONFIG:- logger.error(f"{f} is not supported, please check.")-- redownload = False- if export_gguf:- try:- from auto_round.export.export_to_gguf.convert_hf_to_gguf import ( # pylint: disable=E0401- ModelBase,- ModelType,- get_model_architecture,- )-- if isinstance(args_or_ar.model, str):- model_path = args_or_ar.model- else:- model_path = args_or_ar.model.name_or_path- if not os.path.isdir(model_path):- model_path = download_or_get_path(model_path, args_or_ar.platform)- model_architecture = get_gguf_architecture(model_path, model_type=ModelType.TEXT)- if model_architecture not in ModelBase._model_classes[ModelType.TEXT]:- logger.warning(- f"Current version of gguf export does not support for {model_architecture},"- " will re-download dependency file. Please restart the task."- )- redownload = True- except ModuleNotFoundError as e:- if "convert_hf_to_gguf" in str(e):- logger.warning("GGUF export dependency file is not found, download from github.")- redownload = True- except AttributeError as e:- raise ImportError(- "Please use the latest gguf-py, you can use the following command to install it:\n"- "git clone https://github.com/ggml-org/llama.cpp.git && cd llama.cpp/gguf-py &&"- " pip install . sentencepiece"- )- download_convert_file(redownload)-- try:- from auto_round.export.export_to_gguf.convert_hf_to_gguf import ( # pylint: disable=E0401- ModelBase,- ModelType,- )- except ImportError as e:- raise ImportError(- "Please use the latest gguf-py, you can use the following command to install it:\n"- "git clone https://github.com/ggml-org/llama.cpp.git && cd llama.cpp/gguf-py &&"- " pip install . sentencepiece"- )- if isinstance(args_or_ar.model, str):- model_path = args_or_ar.model- else:- model_path = args_or_ar.model.name_or_path- if not os.path.isdir(model_path):- model_path = download_or_get_path(model_path, args_or_ar.platform)- model_architecture = get_gguf_architecture(model_path, model_type=ModelType.TEXT)- if model_architecture not in ModelBase._model_classes[ModelType.TEXT]:- logger.error(f"Model {model_architecture} is not supported to export gguf format.")- sys.exit(1)-- pattern = re.compile(r"q\d_k")- pre_dq_format = ""- unsupported_list, reset_list = [], []- for format in GGUF_CONFIG:- if format in formats:- if format == "q6_k_s":- logger.warning("Please note that q6_k_s is q6_k.")-- if re.search(pattern, format):- if pre_dq_format and re.search(pattern, format).group() not in pre_dq_format:- logger.error(f"Cannot export {pre_dq_format} and {format} at the same time.")- sys.exit(-1)- else:- pre_dq_format = format-- unsupported_list, reset_list = [], []- gguf_config = GGUF_CONFIG[format]- for k, v in gguf_config.items():- if not hasattr(args_or_ar, k):- continue- if k == "data_type":- if re.search(r"q\d_1", format) and len(formats) > 1:- v = "int"- if k == "sym" and isinstance(args_or_ar, argparse.Namespace):- k = "asym"- v = not v- if getattr(args_or_ar, k) != v:- unsupported_list.append(f"{k}={getattr(args_or_ar, k)}")- reset_list.append(f"{k}={v}")- setattr(args_or_ar, k, v)- if len(unsupported_list) > 0:- logger.info(- f"format {format} does not support for {', '.join(unsupported_list)},"- f" reset to {', '.join(reset_list)}."- )- # Removed obsolete commented-out block for improved readability and maintainability.- return args_or_ar--def get_gguf_qtype_by_layer_config(layer_config):import gguf # pylint: disable=E0401@@ -1110,7 +957,6 @@ def immediate_saving(rounder: object, m: torch.nn.Module, name: str = None, lastwrites shard index, renames shard files, copies source files, and releases temporary state."""import json- import osfrom collections import OrderedDictfrom auto_round.utils import clear_memory, get_modulediff --git a/auto_round/export/export_to_autoround/__init__.py b/auto_round/export/export_to_autoround/__init__.pyindex 6cdcd5aed..0c036d831 100644--- a/auto_round/export/export_to_autoround/__init__.py+++ b/auto_round/export/export_to_autoround/__init__.py@@ -12,4 +12,4 @@# See the License for the specific language governing permissions and# limitations under the License.-from .export import save_quantized_as_autoround, AutoRoundFormat+from .export import save_quantized_as_autorounddiff --git a/auto_round/export/export_to_autoround/export.py b/auto_round/export/export_to_autoround/export.pyindex ccc7780b3..2dfac16ef 100644--- a/auto_round/export/export_to_autoround/export.py+++ b/auto_round/export/export_to_autoround/export.py@@ -35,6 +35,7 @@ from auto_round.export.utils import (release_layer_safely,save_model,)+from auto_round.formats import AutoRoundExportFormatfrom auto_round.logger import loggerfrom auto_round.schemes import QuantizationSchemefrom auto_round.utils import (@@ -50,16 +51,6 @@ from auto_round.utils import ()-class AutoRoundFormat(str, Enum):- # Weight: FP8, per-channel, may be extended to per-tensor in future- # Activation: FP8, per-tensor- FP8_STATIC = "fp8_static"- MXFP8 = "mxfp8"- MXFP4 = "mxfp4"- NVFP4 = "nvfp4"- FP8 = "fp8"--def dynamic_import_quant_linear_for_packing(backend, bits, group_size, sym, act_bits=16):"""Dynamically imports and returns the appropriate QuantLinear class based on the specified backend and parameters.@@ -170,14 +161,14 @@ def pack_layer(layer_name, model, backend, device=None):return pack_layer(layer_name, model, backend, device)if (- backend == f"auto_round:{AutoRoundFormat.FP8.value}"- or backend == f"auto_round:{AutoRoundFormat.FP8_STATIC.value}"+ backend == f"auto_round:{AutoRoundExportFormat.FP8.value}"+ or backend == f"auto_round:{AutoRoundExportFormat.FP8_STATIC.value}"):from auto_round.export.export_to_autoround.export_to_fp8 import pack_layerreturn pack_layer(layer_name, model, backend, device)- if backend == "auto_round:llm_compressor":+ if backend in ["auto_round:llm_compressor", f"auto_round:llm_compressor:{AutoRoundExportFormat.FP8_STATIC.value}"]:from auto_round.export.export_to_llmcompressor.export_to_static_fp import pack_layerreturn pack_layer(layer_name, model, backend, device)@@ -287,7 +278,7 @@ def save_quantized_as_autoround(output_dir, inplace=True, backend="auto_round:exreturn save_quantized_as_fp(output_dir, inplace=inplace, backend="auto_round:llm_compressor", **kwargs)- if backend == "auto_round:llm_compressor":+ if backend in ["auto_round:llm_compressor", f"auto_round:llm_compressor:{AutoRoundExportFormat.FP8_STATIC.value}"]:from auto_round.export.export_to_llmcompressor.export_to_static_fp import save_quantized_as_static_fpreturn save_quantized_as_static_fp(output_dir, inplace=inplace, backend="auto_round:llm_compressor", **kwargs)@@ -301,7 +292,7 @@ def save_quantized_as_autoround(output_dir, inplace=True, backend="auto_round:exif ((kwargs.get("sym") is None or kwargs.get("sym"))and ("gptq" not in backend and "awq" not in backend)- and (AutoRoundFormat.FP8_STATIC.value not in backend)+ and (AutoRoundExportFormat.FP8_STATIC.value not in backend)):backend = backend.replace("auto_round", "auto_round:auto_gptq")diff --git a/auto_round/formats.py b/auto_round/formats.pynew file mode 100644index 000000000..16592320a--- /dev/null+++ b/auto_round/formats.py@@ -0,0 +1,654 @@+# Copyright (c) 2025 Intel Corporation+#+# Licensed under the Apache License, Version 2.0 (the "License");+# you may not use this file except in compliance with the License.+# You may obtain a copy of the License at+#+# http://www.apache.org/licenses/LICENSE-2.0+#+# Unless required by applicable law or agreed to in writing, software+# distributed under the License is distributed on an "AS IS" BASIS,+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+# See the License for the specific language governing permissions and+# limitations under the License.++from __future__ import annotations++import copy+import os+import re+import sys+from dataclasses import asdict+from enum import Enum+from typing import TYPE_CHECKING, Callable, Union++import torch+import transformers++from auto_round.compressors.utils import (+ is_mx_fp,+ is_nv_fp,+ is_standard_fp,+ is_static_wfp8afp8,+ is_wfp8afp8,+)+from auto_round.export.export_to_gguf.config import ModelType+from auto_round.schemes import (+ PRESET_SCHEMES,+ QuantizationScheme,+ get_gguf_scheme,+)+from auto_round.utils import SUPPORTED_FORMATS, logger+++class AutoRoundExportFormat(str, Enum):+ # Weight: FP8, per-channel, may be extended to per-tensor in future+ # Activation: FP8, per-tensor+ FP8_STATIC = "fp8_static"+ MXFP8 = "mxfp8"+ MXFP4 = "mxfp4"+ NVFP4 = "nvfp4"+ FP8 = "fp8"+ MX_FP = "mx_fp"+ NV_FP = "nv_fp"+ MX_FP_RCEIL = "mx_fp_rceil"+ NV_FP4_WITH_STATIC_GS = "nv_fp4_with_static_gs"+++if TYPE_CHECKING:+ from auto_round.compressors.base import BaseCompressor+++def _check_compatibility(formats: list[str], ar: BaseCompressor):+ if (+ any(["gguf" in f.lower() for f in formats])+ and len([f for f in formats if f.lower() != "fake" and not f.lower().startswith("gguf")]) > 1+ ):+ raise ValueError(+ f"GGUF format is not compatible with other formats, but got {formats}, please choose only one of them"+ )+ gguf_format_name = get_gguf_scheme(ar.scheme)+ if gguf_format_name:+ if gguf_format_name.lower().endswith("mixed"):+ gguf_format_name = gguf_format_name.lower().replace("_mixed", "_s")+ if any([f.lower() not in ["fake", gguf_format_name.lower()] for f in formats]):+ tmp_format_name = gguf_format_name.lower() if "fake" not in formats else f"{gguf_format_name.lower()},fake"+ logger.warning(+ f"reset format {','.join(formats)} to {tmp_format_name} "+ f"since scheme {gguf_format_name} can only be exported to format {gguf_format_name.lower()} or fake"+ )+ formats = tmp_format_name.split(",")+ return formats+++def get_formats(+ format: str,+ ar: BaseCompressor,+) -> list[OutputFormat]:+ """Get the list of OutputFormat instances based on the provided name."""++ def remove_duplicates(lst):+ seen = set()+ return [x for x in lst if not (x in seen or seen.add(x))]++ formats = format.replace("q*_", f"q{ar.bits}_").replace(" ", "").split(",")+ formats = remove_duplicates(formats) # need the keep origin order++ formats = _check_compatibility(formats, ar)++ formats = remove_duplicates(formats)++ for i in range(len(formats)):+ if formats[i].startswith("gguf:"):+ formats[i] = GGUFFormat(formats[i], ar)+ elif formats[i] not in OutputFormat._format_list:+ raise KeyError(f"Unsupported format {formats[i]}, please choose from {SUPPORTED_FORMATS}")+ else:+ formats[i] = OutputFormat._format_list[formats[i]](formats[i], ar)++ new_format = formats[i].check_and_reset_format(ar)+ if new_format is not None:+ if new_format not in format:+ formats[i] = OutputFormat._format_list[new_format](new_format, ar)+ else:+ formats[i] = None++ formats = [fmt for fmt in formats if fmt is not None]++ if len(formats) == 1 and formats[0].is_gguf and ar.scale_dtype != torch.float32:+ ar.scale_dtype = torch.float32+ logger.info("change `scale_dtype` to `torch.float32` for gguf format")++ return formats+++def _check_divisible_by_32(ar):+ from auto_round.schemes import preset_name_to_scheme++ if isinstance(ar.scheme, str):+ default_dict = asdict(preset_name_to_scheme(ar.scheme.upper()))+ else:+ default_dict = asdict(ar.scheme)+ if default_dict["data_type"] == "int" and default_dict["act_bits"] >= 16:+ for n, m in ar.model.named_modules():+ if type(m) in ar.supported_types or m.__class__.__name__ in ar.inner_supported_types:+ if m.weight.shape[0] % 32 or m.weight.shape[1] % 32:+ if ar.layer_config is None:+ ar.layer_config = {}+ if ar.layer_config.get(n) is not None and ar.layer_config[n]["bits"] >= 16:+ continue+ ar.layer_config.setdefault(n, copy.deepcopy(default_dict))+ ar.layer_config[n].update({"bits": 16, "data_type": "fp", "fixed_by_user": True})+ logger.warning_once(f"{n} skipped quantization (shape not divisible by 32).")+++class OutputFormat:+ """ "Base class for different output formats.++ format: determines which method from export module to use for exporting.+ For example, auto_round, gguf, llmcompressor etc.+ backend: determines the specific export process within the format.+ For example, auto_round:fp8_static, auto_round:auto_awq etc.+ """++ support_schemes: list = []+ _format_list: dict[str, OutputFormat] = {}+ format_name = "base"++ def __init__(self, format: str, ar: BaseCompressor):+ """Initialize the OutputFormat class."""+ self.output_format = format+ self.backend = None++ if not self.is_fake() and not self.is_support_scheme(ar.scheme):+ logger.error(+ f"Currently, the {self.format_name} format only supports {self.support_schemes}, "+ f"but got scheme {ar.scheme}, please change to fake or auto_round etc."+ )+ exit(-1)++ @classmethod+ def register(cls, *names: str) -> Callable[[OutputFormat], OutputFormat]:+ assert names++ def func(output_format: OutputFormat) -> OutputFormat:+ for name in names:+ cls._format_list[name] = output_format+ return output_format++ return func++ @classmethod+ def get_support_matrix(cls: OutputFormat) -> str:+ output_str = ""+ for k, v in cls._format_list.items():+ if k == "fake":+ support_scheme = "All schemes"+ else:+ support_scheme = ", ".join(v.support_schemes).rstrip(",")+ output_str += f"\x1b[31;1m{k}\x1b[0m support scheme:\n\t{support_scheme}\n"+ return output_str++ def get_backend_name(self) -> str:+ if self.backend is None:+ return self.output_format++ # auto_round:llm_compressor:fp8_static+ if self.backend.backend is not None:+ return f"{self.output_format}:{self.backend.get_backend_name()}"+ # auto_round:auto_awq, auto_round:auto_gptq+ elif self.backend.get_backend_name() in self._format_list:+ return f"{self.output_format}:{self.backend.get_backend_name()}"+ # auto_round:fp8_static, llm_compressor:fp8_static+ else:+ return self.backend.get_backend_name()++ @classmethod+ def is_support_scheme(cls: OutputFormat, scheme: Union[str, QuantizationScheme]) -> bool:+ if isinstance(scheme, str) and scheme.upper() in cls.support_schemes:+ return True+ if isinstance(scheme, QuantizationScheme):+ return True+ return False++ def check_and_reset_format(self, ar: BaseCompressor) -> str:+ if self.backend is not None:+ new_format = self.backend.check_and_reset_format(ar)+ self.backend = OutputFormat._format_list[new_format](new_format, ar) if new_format else self.backend++ w_fp8 = ar.data_type.startswith("fp") and ar.bits == 8+ act_fp8 = ar.act_data_type.startswith("fp") and ar.act_bits == 8+ if w_fp8 or act_fp8:+ error_msg = (+ f"is only supported to export auto_round or llm_compressor format,"+ f" but got {self.format_name}, please check."+ )+ error_msg = ("act_data_type<fp8> " + error_msg) if act_fp8 else error_msg+ error_msg = ("data_type<fp8> " + error_msg) if w_fp8 else error_msg+ logger.error(error_msg)+ sys.exit(-1)++ if ar.act_bits <= 8 and (not is_standard_fp(ar.act_data_type) or ar.act_dynamic):+ logger.warning(+ f"{self.format_name} format not support for current activation quantization configuration,"+ " reset to fake format and save."+ )+ return "fake"++ return None++ def is_gguf(self) -> bool:+ return "gguf" in self.output_format++ def is_fake(self) -> bool:+ return self.output_format == "fake"++ def is_gptq(self) -> bool:+ return "gptq" in self.output_format or (self.backend is not None and self.backend.is_gptq())++ def is_awq(self) -> bool:+ return "awq" in self.output_format or (self.backend is not None and self.backend.is_awq())++ def is_llm_compressor(self) -> bool:+ return "llm_compressor" in self.output_format or (self.backend is not None and self.backend.is_llm_compressor())++[email protected]("fake")+class FakeFormat(OutputFormat):+ support_schemes = None+ format_name = "fake"++ def check_and_reset_format(self, ar: BaseCompressor) -> str:+ return None++[email protected]("llm_compressor", "llmcompressor")+class LLMCompressorFormat(OutputFormat):+ support_schemes = ["MXFP4", "MXFP8", "NVFP4", "FPW8A16", "FP8_STATIC"]+ format_name = "llm_compressor"++ def __init__(self, format, ar):+ if not self.is_support_scheme(ar.scheme):+ logger.error(+ f"Currently, the llm_compressor format only supports {self.support_schemes}, "+ f"but got scheme {ar.scheme}, please change to fake or auto_round etc."+ )+ exit(-1)+ if format.startswith("llm_compressor"):+ self.output_format = format+ self.backend = None+ if is_nv_fp(ar.data_type) or is_mx_fp(ar.data_type):+ from auto_round.export.export_to_llmcompressor import check_compressed_tensors_supported++ check_compressed_tensors_supported()+ self.backend = LLMCompressorFormat(ar.data_type, ar)+ elif is_static_wfp8afp8(ar):+ self.backend = LLMCompressorFormat(AutoRoundExportFormat.FP8_STATIC.value, ar)+ if ar.act_group_size != 0:+ logger.warning(+ f"scheme FP8_STATIC export to llm_compressor format only support for act_group_size 0,"+ f" ,but got act_group_size={ar.act_group_size}, reset = 0"+ )+ ar.act_group_size = 0+ if ar.group_size > 0:+ logger.warning(+ f"please note that group_size={ar.group_size}"+ " may not be supported for llm_compressor format, and cannot be loaded in llm_compressor"+ )+ else:+ if format.upper() not in list(AutoRoundExportFormat.__members__.keys()):+ raise KeyError(f"Unsupported backend format llm_compressor:{format}, please check")+ self.output_format = f"llm_compressor:{format}"+ self.backend = None++ def check_and_reset_format(self, ar: BaseCompressor) -> str:+ if self.backend is not None:+ new_format = self.backend.check_and_reset_format(ar)+ self.backend = OutputFormat._format_list[new_format](new_format, ar) if new_format else self.backend++ if ar.act_bits <= 8 and (not is_standard_fp(ar.act_data_type) or ar.act_dynamic):+ if (is_nv_fp(ar.act_data_type) and "static_gs" in ar.act_data_type) or (is_mx_fp(ar.act_data_type)):+ return None+ else:+ bits, group_size, sym, act_bits = 8, -1, True, 8+ assert (+ ar.bits == bits+ and ar.group_size == group_size+ and ar.sym == sym+ and ar.act_bits == act_bits+ and ar.act_dynamic+ ), (+ f"Currently only support to export llm_compressor format for sym dynamic quantized"+ f" W{ar.bits}A{ar.act_bits} model with group_size={group_size},"+ f" but got bits={ar.bits}, group_size={ar.group_size}, sym={ar.sym},"+ f" act_bits={ar.act_bits}"+ )+ return None+ return None++[email protected]("auto_gptq", "gptqmodel")+class AutoGPTQFormat(OutputFormat):+ support_schemes = ["W4A16", "W2A16", "W3A16", "W8A16", "BF16", "W2A16G64", "W2A16G32"]+ format_name = "auto_gptq"++ def check_and_reset_format(self, ar):+ if not ar.sym:+ logger.warning(+ "the asymmetrical kernel of the GPTQ format may result in a noticeable accuracy drop,"+ " particularly for 2-bit quantization and smaller models."+ " We recommend exporting to either the AutoAWQ format ( only 4 bits) or "+ "the AutoRound format(2/3/4/8 bits)."+ )+ if self.backend is None:+ _check_divisible_by_32(ar)+ return super().check_and_reset_format(ar)++[email protected]("auto_awq")+class AutoAWQFormat(OutputFormat):+ support_schemes = ["W4A16", "W2A16", "W3A16", "W8A16", "BF16", "W2A16G64", "W2A16G32"]+ format_name = "auto_awq"++ @staticmethod+ def check_awq_gemm_compatibility(model, bits, group_size, sym, layer_configs=None):+ """Checks if a model is compatible with the AutoAWQ GEMM kernel.++ Args:+ model: The model object to evaluate, typically a PyTorch model.+ bits (int): The number of bits for quantization (must be 4 for compatibility).+ group_size (int): The group size for quantization.+ sym (bool): Whether symmetric quantization is used (not utilized in the current function logic).+ layer_configs (dict, optional): A dictionary mapping layer names to configurations, where each+ configuration can specify a custom number of bits for the layer.++ Returns:+ tuple: A tuple containing:+ - bool: `True` if the model is compatible, `False` otherwise.+ - str: An error message describing why the model is incompatible, or an empty string if compatible.+ """+ from auto_round.utils.model import get_layer_names_in_block, get_module++ if bits != 4:+ return False, "AutoAWQ GEMM kernel only supports 4 bits"+ for n, m in model.named_modules():+ if type(m) == transformers.pytorch_utils.Conv1D:+ return False, "AutoAWQ GEMM kernel does not support conv1d"++ layer_names = get_layer_names_in_block(model)+ for layer_name in layer_names:+ if (+ layer_configs is not None+ and layer_name in layer_configs.keys()+ and layer_configs[layer_name].get("bits", bits) > 8+ ):+ continue++ layer = get_module(model, layer_name)+ if layer.in_features % group_size != 0:+ return False, f"Layer {layer_name} in_features is not multiple of group_size {group_size}"+ if layer.out_features % (32 // bits) != 0:+ return False, f"Layer {layer_name} out_features is not multiple of 32 // bits"++ return True, ""++ def check_and_reset_format(self, ar):+ awq_supported, info = self.check_awq_gemm_compatibility(+ ar.model, ar.bits, ar.group_size, ar.sym, ar.layer_config+ )+ if not awq_supported:+ logger.warning(f"The AutoAWQ format may not be supported due to {info}")+ if ar.bits != 4:+ raise ValueError("The AWQ format only supports W4 quantization ")++ if self.backend is None:+ _check_divisible_by_32(ar)++ return super().check_and_reset_format(ar)++[email protected]("itrex")[email protected]("itrex_xpu")+class ITREXFormat(OutputFormat):+ support_schemes = ["W4A16", "W2A16", "W3A16", "W8A16", "BF16", "W2A16G64", "W2A16G32"]+ format_name = "itrex"++[email protected]("gguf")+class GGUFFormat(OutputFormat):+ support_schemes = [+ "GGUF:Q4_0",+ "GGUF:Q4_1",+ "GGUF:Q5_0",+ "GGUF:Q5_1",+ "GGUF:Q2_K_S",+ "GGUF:Q3_K_S",+ "GGUF:Q3_K_M",+ "GGUF:Q3_K_L",+ "GGUF:Q4_K_S",+ "GGUF:Q4_K_M",+ "GGUF:Q5_K_S",+ "GGUF:Q5_K_M",+ "GGUF:Q6_K",+ "GGUF:Q8_0",+ ]+ format_name = "gguf"++ def __init__(self, format: str, ar: BaseCompressor):+ if format.startswith("gguf:"):+ self.gguf_args_check(ar, format, model_type=ModelType.TEXT)+ if ar.mllm:+ self.gguf_args_check(ar, format, model_type=ModelType.MMPROJ)+ ar.scheme = format.upper()++ self.output_format = "gguf"+ self.backend_cls = GGUFFormat+ self.backend = GGUFFormat(format.split(":")[-1], ar)+ else:+ self.output_format = f"gguf:{format}"+ self.backend = None++ def check_and_reset_format(self, ar):+ if ar.iters != 0 and ar.bits != 3 and not ar.enable_alg_ext:+ logger.warning_once(+ "`iters=0` is recommended when exporting to current GGUF format"+ " or add `enable_alg_ext` for better accuracy with much more tuning cost."+ " Please refer to https://github.com/intel/auto-round/tree/main/docs/gguf_alg_ext_acc.md"+ " for the accuracy results."+ )+ elif ar.bits >= 8 and ar.iters != 0:+ logger.warning_once("`iters=0` is recommended for bits>=8")++ return super().check_and_reset_format(ar)++ @staticmethod+ def gguf_args_check(args_or_ar, formats: Union[str, list[str]] = None, model_type=ModelType.TEXT):+ import argparse++ from auto_round.export.export_to_gguf.config import GGUF_CONFIG+ from auto_round.export.export_to_gguf.convert import download_convert_file+ from auto_round.logger import logger+ from auto_round.utils.model import download_or_get_path, get_gguf_architecture++ formats = [formats] if isinstance(formats, str) else formats+ formats = sorted(formats, key=lambda x: len(x))+ export_gguf = False+ for f in formats:+ if f.startswith("gguf"):+ export_gguf = True++ if f.startswith("gguf") and f not in GGUF_CONFIG:+ logger.error(f"{f} is not supported, please check.")++ redownload = False+ if export_gguf:+ try:+ from auto_round.export.export_to_gguf.convert_hf_to_gguf import ( # pylint: disable=E0401+ ModelBase,+ ModelType,+ get_model_architecture,+ )++ if isinstance(args_or_ar.model, str):+ model_path = args_or_ar.model+ else:+ model_path = args_or_ar.model.name_or_path+ if not os.path.isdir(model_path):+ model_path = download_or_get_path(model_path, args_or_ar.platform)+ model_architecture = get_gguf_architecture(model_path, model_type=ModelType.TEXT)+ if model_architecture not in ModelBase._model_classes[ModelType.TEXT]:+ logger.warning(+ f"Current version of gguf export does not support for {model_architecture},"+ " will re-download dependency file. Please restart the task."+ )+ redownload = True+ except ModuleNotFoundError as e:+ if "convert_hf_to_gguf" in str(e):+ logger.warning("GGUF export dependency file is not found, download from github.")+ redownload = True+ except AttributeError as e:+ raise ImportError(+ "Please use the latest gguf-py, you can use the following command to install it:\n"+ "git clone https://github.com/ggml-org/llama.cpp.git && cd llama.cpp/gguf-py"+ " && pip install . sentencepiece"+ )+ download_convert_file(redownload)++ try:+ from auto_round.export.export_to_gguf.convert_hf_to_gguf import ( # pylint: disable=E0401+ ModelBase,+ ModelType,+ )+ except ImportError as e:+ raise ImportError(+ "Please use the latest gguf-py, you can use the following command to install it:\n"+ "git clone https://github.com/ggml-org/llama.cpp.git && cd llama.cpp/gguf-py"+ " && pip install . sentencepiece"+ )+ if isinstance(args_or_ar.model, str):+ model_path = args_or_ar.model+ else:+ model_path = args_or_ar.model.name_or_path+ if not os.path.isdir(model_path):+ model_path = download_or_get_path(model_path, args_or_ar.platform)+ model_architecture = get_gguf_architecture(model_path, model_type=ModelType.TEXT)+ if model_architecture not in ModelBase._model_classes[ModelType.TEXT]:+ logger.error(f"Model {model_architecture} is not supported to export gguf format.")+ sys.exit(1)++ pattern = re.compile(r"q\d_k")+ pre_dq_format = ""+ unsupported_list, reset_list = [], []+ for format in GGUF_CONFIG:+ if format in formats:+ if format == "q6_k_s":+ logger.warning("Please note that q6_k_s is q6_k.")++ if re.search(pattern, format):+ if pre_dq_format and re.search(pattern, format).group() not in pre_dq_format:+ logger.error(f"Cannot export {pre_dq_format} and {format} at the same time.")+ sys.exit(-1)+ else:+ pre_dq_format = format++ unsupported_list, reset_list = [], []+ gguf_config = GGUF_CONFIG[format]+ for k, v in gguf_config.items():+ if not hasattr(args_or_ar, k):+ continue+ if k == "data_type":+ if re.search(r"q\d_1", format) and len(formats) > 1:+ v = "int"+ if k == "sym" and isinstance(args_or_ar, argparse.Namespace):+ k = "asym"+ v = not v+ if getattr(args_or_ar, k) != v:+ unsupported_list.append(f"{k}={getattr(args_or_ar, k)}")+ reset_list.append(f"{k}={v}")+ setattr(args_or_ar, k, v)+ if len(unsupported_list) > 0:+ logger.info(+ f"format {format} does not support for {', '.join(unsupported_list)},"+ f" reset to {', '.join(reset_list)}."+ )+ # Removed obsolete commented-out block for improved readability and maintainability.+ return args_or_ar++[email protected]("auto_round")[email protected]("auto_round:auto_awq")[email protected]("auto_round:llm_compressor")[email protected]("auto_round:gptqmodel", "auto_round:auto_gptq")+class AutoRoundFormat(OutputFormat):+ support_schemes = [+ "W4A16",+ "W2A16",+ "W3A16",+ "W8A16",+ "MXFP4",+ "MXFP8",+ "NVFP4",+ "FPW8A16",+ "W2A16G64",+ "W2A16G32",+ "FP8_STATIC",+ "BF16",+ ]+ format_name = "auto_round"++ def __init__(self, format: str, ar: BaseCompressor):+ self.output_format = "auto_round"+ self.backend = None++ if format == "auto_round":+ if ar.sym and "int" in ar.data_type:+ self.backend = AutoGPTQFormat("auto_gptq", ar)+ elif ar.bits == 4 and not ar.sym and "int" in ar.data_type:+ enable_awq = all(+ config["bits"] == ar.bits or config["bits"] >= 16 for config in ar.layer_config.values()+ )+ if enable_awq:+ self.backend = AutoAWQFormat("auto_awq", ar)+ elif is_nv_fp(ar.data_type) or is_mx_fp(ar.data_type):+ self.backend = AutoRoundFormat(ar.data_type, ar)+ elif is_static_wfp8afp8(ar): # static wfp8afp8+ self.backend = AutoRoundFormat(AutoRoundExportFormat.FP8_STATIC.value, ar)+ elif ar.data_type.startswith("fp") and ar.bits == 8 and ar.act_bits >= 16: # woq fp8+ self.backend = AutoRoundFormat(AutoRoundExportFormat.FP8.value, ar)+ elif ar.act_bits < 16:+ raise ValueError(+ "AutoRound format does not support exporting "+ "for the current quantization configuration, "+ "please change to `fake` format for research purpose"+ )+ elif not format.startswith("auto_round"):+ if format.upper() not in list(AutoRoundExportFormat.__members__.keys()):+ raise KeyError(f"Unsupported backend format auto_round:{format}, please check")+ self.output_format = f"auto_round:{format}"+ self.backend = None+ else:+ backend = format.split(":")[1] if ":" in format else None+ self.backend = self._format_list.get(backend)(backend, ar) if backend else None++ if self.backend is not None:+ self.support_schemes = self.backend.support_schemes++ def check_and_reset_format(self, ar):+ if self.backend is not None:+ new_format = self.backend.check_and_reset_format(ar)+ self.backend = OutputFormat._format_list[new_format](new_format, ar) if new_format else self.backend++ if ar.act_bits <= 8:+ if is_standard_fp(ar.act_data_type) and not ar.act_dynamic:+ if (+ ar.act_group_size != 0+ and not ar.act_dynamic+ and self.get_backend_name() == f"auto_round:{AutoRoundExportFormat.FP8.value}"+ ):+ logger.warning(+ f"Please note that quantize activation with act_group_size={ar.act_group_size}"+ " may result in failure to export or import normally."+ )+ if self.backend is None:+ _check_divisible_by_32(ar)+ return Nonediff --git a/auto_round/inference/convert_model.py b/auto_round/inference/convert_model.pyindex f4cd1c527..4cd47d7e5 100644--- a/auto_round/inference/convert_model.py+++ b/auto_round/inference/convert_model.py@@ -20,7 +20,7 @@ import torch.nn as nnfrom tqdm import tqdmfrom transformers.pytorch_utils import Conv1D-from auto_round.export.export_to_autoround import AutoRoundFormat+from auto_round.formats import AutoRoundExportFormatfrom auto_round.inference.backend import (BackendInfos,dynamic_import_inference_linear,@@ -414,10 +414,10 @@ def _create_quant_layer(layer, layer_backend, config, in_features, out_features)bias=bias,)elif (- AutoRoundFormat.FP8_STATIC.value in layer_backend- or AutoRoundFormat.MXFP8.value in layer_backend- or AutoRoundFormat.MXFP4.value in layer_backend- or AutoRoundFormat.NVFP4.value in layer_backend+ AutoRoundExportFormat.FP8_STATIC.value in layer_backend+ or AutoRoundExportFormat.MXFP8.value in layer_backend+ or AutoRoundExportFormat.MXFP4.value in layer_backend+ or AutoRoundExportFormat.NVFP4.value in layer_backend):return QuantLinear.from_original(config, layer)diff --git a/auto_round/special_model_handler.py b/auto_round/special_model_handler.pyindex fe3865b85..ef688634f 100644--- a/auto_round/special_model_handler.py+++ b/auto_round/special_model_handler.py@@ -12,6 +12,7 @@# See the License for the specific language governing permissions and# limitations under the License.import auto_round.modelling as auto_round_modelling+from auto_round.formats import OutputFormatfrom auto_round.utils import LazyImport, logger, unsupported_meta_devicemllms_with_limited_bs = ("llava", "qwen2_vl", "phi3_v", "mllama") # Limitations on batch_size@@ -67,8 +68,8 @@ def _handle_special_model(model):return model-def _handle_moe_model(model, formats=None):- if formats is not None and any(["gguf" in format_ for format_ in formats]):+def _handle_moe_model(model, formats: list[OutputFormat] = None):+ if formats is not None and any([format_.is_gguf() for format_ in formats]):return modelif hasattr(model.config, "model_type") and model.config.model_type in CONVERT_EXPERT_TO_LINEAR_MODELS:from tqdm import tqdmdiff --git a/auto_round_extension/vllm_ext/moe_impl_mxfp4.py b/auto_round_extension/vllm_ext/moe_impl_mxfp4.pyindex 0ff5241ab..75e298acd 100644--- a/auto_round_extension/vllm_ext/moe_impl_mxfp4.py+++ b/auto_round_extension/vllm_ext/moe_impl_mxfp4.py@@ -283,7 +283,6 @@ class AutoRoundMoEMethodMXFp4Impl(AutoRoundMoEMethod):return revert_bias- # breakpoint()if self.has_bias:if envs.VLLM_AR_POST_PROCESS_GPTOSS:w13_bias_swapped = revert_interleaved_bias(layer.w13_bias)diff --git a/test/test_cpu/test_export.py b/test/test_cpu/test_export.pyindex 57b30354b..d5faa737a 100644--- a/test/test_cpu/test_export.py+++ b/test/test_cpu/test_export.py@@ -402,6 +402,32 @@ class TestAutoRound(unittest.TestCase):print(res)shutil.rmtree(quantized_model_path, ignore_errors=True)+ def test_export_format(self):+ from auto_round.formats import get_formats++ autoround = AutoRound(+ self.model_name,+ scheme="FP8_STATIC",+ )+ format_list = get_formats("auto_round, llm_compressor, auto_round:llm_compressor", autoround)+ self.assertEqual(len(format_list), 3)+ self.assertEqual(format_list[0].output_format, "auto_round")+ self.assertEqual(format_list[0].get_backend_name(), "auto_round:fp8_static")+ self.assertEqual(format_list[1].output_format, "llm_compressor")+ self.assertEqual(format_list[1].get_backend_name(), "llm_compressor:fp8_static")+ self.assertEqual(format_list[2].output_format, "auto_round")+ self.assertEqual(format_list[2].get_backend_name(), "auto_round:llm_compressor:fp8_static")++ autoround = AutoRound(+ self.model_name,+ scheme="W4A16",+ )+ format_list = get_formats("auto_round:auto_awq, auto_gptq", autoround)+ self.assertEqual(format_list[0].output_format, "auto_round")+ self.assertEqual(format_list[0].get_backend_name(), "auto_round:auto_awq")+ self.assertEqual(format_list[1].output_format, "auto_gptq")+ self.assertEqual(format_list[1].get_backend_name(), "auto_gptq")+if __name__ == "__main__":unittest.main()diff --git a/test/test_cpu/test_mx_quant_linear.py b/test/test_cpu/test_mx_quant_linear.pyindex e8e18c3bb..c2e9a3c00 100644--- a/test/test_cpu/test_mx_quant_linear.py+++ b/test/test_cpu/test_mx_quant_linear.py@@ -3,14 +3,14 @@ import torchfrom auto_round.data_type.utils import get_quant_funcfrom auto_round.experimental import qmodules as ar_qmodules-from auto_round.export.export_to_autoround import AutoRoundFormatfrom auto_round.export.export_to_autoround.qlinear_fp import QuantLinear as _MXFPLinear+from auto_round.formats import AutoRoundExportFormatfrom auto_round.schemes import PRESET_SCHEMES-mx_schemes = [AutoRoundFormat.MXFP8.value, AutoRoundFormat.MXFP4.value]+mx_schemes = [AutoRoundExportFormat.MXFP8.value, AutoRoundExportFormat.MXFP4.value]QMODULE_MAPPING = {- AutoRoundFormat.MXFP8.value: ar_qmodules.MXFP8QuantLinear,- AutoRoundFormat.MXFP4.value: ar_qmodules.MXFP4QuantLinear,+ AutoRoundExportFormat.MXFP8.value: ar_qmodules.MXFP8QuantLinear,+ AutoRoundExportFormat.MXFP4.value: ar_qmodules.MXFP4QuantLinear,}diff --git a/test/test_cpu/test_mxfp_save_load.py b/test/test_cpu/test_mxfp_save_load.pyindex aca5c7592..5331949a0 100644--- a/test/test_cpu/test_mxfp_save_load.py+++ b/test/test_cpu/test_mxfp_save_load.py@@ -9,22 +9,22 @@ from transformers.models.qwen2.modeling_qwen2 import Qwen2ForCausalLMfrom auto_round import AutoRoundfrom auto_round import schemes as ar_schemesfrom auto_round.experimental import qmodules as ar_qmodules-from auto_round.export.export_to_autoround import AutoRoundFormatfrom auto_round.export.export_to_autoround import qlinear_fp as ar_qlinear_fp+from auto_round.formats import AutoRoundExportFormatfrom auto_round.inference.backend import MX_TENSOR_DATA_TYPESfrom auto_round.testing_utils import has_moduletesting_scheme_name_lst = [- AutoRoundFormat.MXFP8.value,- AutoRoundFormat.MXFP4.value,+ AutoRoundExportFormat.MXFP8.value,+ AutoRoundExportFormat.MXFP4.value,]QMODULE_MAPPING = {- AutoRoundFormat.MXFP8.value: ar_qmodules.MXFP8QuantLinear,- AutoRoundFormat.MXFP4.value: ar_qmodules.MXFP4QuantLinear,+ AutoRoundExportFormat.MXFP8.value: ar_qmodules.MXFP8QuantLinear,+ AutoRoundExportFormat.MXFP4.value: ar_qmodules.MXFP4QuantLinear,}SCHEMES_MAPPING = {- AutoRoundFormat.MXFP8.value: ar_schemes.MXFP8,- AutoRoundFormat.MXFP4.value: ar_schemes.MXFP4,+ AutoRoundExportFormat.MXFP8.value: ar_schemes.MXFP8,+ AutoRoundExportFormat.MXFP4.value: ar_schemes.MXFP4,}diff --git a/test/test_cpu/test_nvfp4_quant_linear.py b/test/test_cpu/test_nvfp4_quant_linear.pyindex 0a42f009a..80a289b70 100644--- a/test/test_cpu/test_nvfp4_quant_linear.py+++ b/test/test_cpu/test_nvfp4_quant_linear.py@@ -4,12 +4,12 @@ import torchfrom auto_round.data_type.nvfp import calculate_gparamfrom auto_round.data_type.utils import get_quant_funcfrom auto_round.experimental import qmodules as ar_qmodules-from auto_round.export.export_to_autoround import AutoRoundFormatfrom auto_round.export.export_to_autoround.qlinear_fp import QuantLinear as _FPLinear+from auto_round.formats import AutoRoundExportFormatfrom auto_round.schemes import PRESET_SCHEMESQMODULE_MAPPING = {- AutoRoundFormat.NVFP4.value: ar_qmodules.NVFP4QuantLinear,+ AutoRoundExportFormat.NVFP4.value: ar_qmodules.NVFP4QuantLinear,}@@ -26,7 +26,7 @@ def fixed_seed():# (Optional) cleanup or reset after test[email protected]("scheme", [AutoRoundFormat.NVFP4.value])[email protected]("scheme", [AutoRoundExportFormat.NVFP4.value])@torch.inference_mode()def test_nvfp4_quantlinear_from_original_and_forward(scheme):"""diff --git a/test/test_cuda/test_mxfp_and_nvfp_quant.py b/test/test_cuda/test_mxfp_and_nvfp_quant.pyindex 0dc43b093..4954b2320 100644--- a/test/test_cuda/test_mxfp_and_nvfp_quant.py+++ b/test/test_cuda/test_mxfp_and_nvfp_quant.py@@ -8,15 +8,19 @@ from transformers import AutoModelForCausalLM, AutoTokenizerfrom auto_round import AutoRoundfrom auto_round import schemes as ar_schemesfrom auto_round.experimental import qmodules as ar_qmodules-from auto_round.export.export_to_autoround import AutoRoundFormatfrom auto_round.export.export_to_autoround import qlinear_fp as ar_qlinear_fp+from auto_round.formats import AutoRoundExportFormatfrom auto_round.testing_utils import has_module-testing_schemes = [AutoRoundFormat.MXFP8.value, AutoRoundFormat.MXFP4.value, AutoRoundFormat.NVFP4.value]+testing_schemes = [+ AutoRoundExportFormat.MXFP8.value,+ AutoRoundExportFormat.MXFP4.value,+ AutoRoundExportFormat.NVFP4.value,+]QMODULE_MAPPING = {- AutoRoundFormat.MXFP8.value: ar_qmodules.MXFP8QuantLinear,- AutoRoundFormat.MXFP4.value: ar_qmodules.MXFP4QuantLinear,- AutoRoundFormat.NVFP4.value: ar_qmodules.NVFP4QuantLinear,+ AutoRoundExportFormat.MXFP8.value: ar_qmodules.MXFP8QuantLinear,+ AutoRoundExportFormat.MXFP4.value: ar_qmodules.MXFP4QuantLinear,+ AutoRoundExportFormat.NVFP4.value: ar_qmodules.NVFP4QuantLinear,}