PTQ-Level Efficiency, QAT-Level Accuracy | 训练后量化的效率,量化感知训练的精度
Fix llm-compressor export (#820)
Tang Kaihui40aed0641bd5
diff --git a/auto_round/export/export_to_llmcompressor/export.py b/auto_round/export/export_to_llmcompressor/export.pyindex 65864cb16..f55ac280c 100644--- a/auto_round/export/export_to_llmcompressor/export.py+++ b/auto_round/export/export_to_llmcompressor/export.py@@ -12,6 +12,7 @@# See the License for the specific language governing permissions and# limitations under the License.+import copyimport osimport torch@@ -75,10 +76,34 @@ def pack_layer(layer_name, model, backend, device=None):@torch.no_grad()-def save_quantized_as_llmcompressor(output_dir, **kwargs):+def save_quantized_as_llmcompressor(output_dir: str, inplace: bool = True, **kwargs) -> torch.nn.Module:+ """+ Save a quantized model in the LLM-Compressor format.++ This function saves a quantized model, including its configuration, state dictionary,+ tokenizer, and processor, in the specified output directory. It supports inplace+ modification of the model or creating a deepcopy for saving. Currently, only NVFP+ and MXFP backends are supported for specific quantization formats.++ Args:+ output_dir (str): The directory where the quantized model will be saved.+ inplace (bool, optional): If True, modifies the model in place. Otherwise, creates a deepcopy of the model.+ Default is True.+ **kwargs: Additional arguments, including:+ - model (torch.nn.Module): The model to be quantized and saved.+ - backend (str): The backend framework used for quantization.+ - tokenizer: The tokenizer associated with the model.+ - processor: The processor associated with the model.+ - safe_serialization (bool): Whether to use safe serialization when saving+ the model. Default is True.++ Returns:+ torch.nn.Module: The quantized model that was saved.+ """+backend = kwargs.get("backend", None)if is_nv_fp(backend) or is_mx_fp(backend):- return save_quantized_as_fp(output_dir, **kwargs)+ return save_quantized_as_fp(output_dir, inplace=inplace, **kwargs)model = kwargs.get("model", None)safe_serialization = kwargs.get("safe_serialization", True)@@ -86,6 +111,8 @@ def save_quantized_as_llmcompressor(output_dir, **kwargs):processor = kwargs.get("processor", None)if output_dir is not None and os.path.exists(output_dir):logger.warning(f"{output_dir} already exists, this may cause model conflict")+ if not inplace:+ model = copy.deepcopy(model.to("cpu"))# save tokenizer, processorif output_dir is not None and tokenizer is not None and hasattr(tokenizer, "save_pretrained"):@@ -125,3 +152,5 @@ def save_quantized_as_llmcompressor(output_dir, **kwargs):copy_python_files_from_model_cache(model, output_dir)except Exception as e:logger.warning("Skipping source model Python file copy due to error: %s", e)++ return model