PTQ-Level Efficiency, QAT-Level Accuracy | 训练后量化的效率,量化感知训练的精度
Support diffusion model saving (#1519)
Wang, Mengnia0ad7780c7a0
diff --git a/auto_round/compressors/diffusion/compressor.py b/auto_round/compressors/diffusion/compressor.pyindex 50e807320..26f72c73b 100644--- a/auto_round/compressors/diffusion/compressor.py+++ b/auto_round/compressors/diffusion/compressor.py@@ -12,6 +12,7 @@# See the License for the specific language governing permissions and# limitations under the License.+import osfrom collections import defaultdictfrom copy import deepcopyfrom typing import Union@@ -22,6 +23,7 @@ from tqdm import tqdmfrom auto_round.compressors.base import BaseCompressorfrom auto_round.compressors.diffusion.dataset import get_diffusion_dataloaderfrom auto_round.compressors.utils import block_forward+from auto_round.formats import OutputFormatfrom auto_round.logger import loggerfrom auto_round.schemes import QuantizationSchemefrom auto_round.utils import (@@ -112,13 +114,7 @@ class DiffusionCompressor(BaseCompressor):device_map = 0self._set_device(device_map)- if isinstance(model, str):- pipe, model = diffusion_load_model(model, platform=platform, device=self.device, model_dtype=model_dtype)- elif isinstance(model, pipeline_utils.DiffusionPipeline):- pipe = model- model = pipe.transformer- else:- raise ValueError(f"Only support str or DiffusionPipeline class for model, but get {type(model)}")+ pipe, model = diffusion_load_model(model, platform=platform, device=self.device, model_dtype=model_dtype)self.model = modelself.pipe = pipe@@ -373,6 +369,33 @@ class DiffusionCompressor(BaseCompressor):# torch.cuda.empty_cache()+ def _get_save_folder_name(self, format: OutputFormat) -> str:+ """Generates the save folder name based on the provided format string.++ If there are multiple formats to handle, the function creates a subfolder+ named after the format string with special characters replaced. If there's+ only one format, it returns the original output directory directly.++ Args:+ format_str (str): The format identifier (e.g., 'gguf:q2_k_s').++ Returns:+ str: The path to the folder where results should be saved.+ """+ # Replace special characters to make the folder name filesystem-safe+ sanitized_format = format.get_backend_name().replace(":", "-").replace("_", "-")++ # Use a subfolder only if there are multiple formats+ if len(self.formats) > 1:+ return (+ os.path.join(self.orig_output_dir, sanitized_format, "transformer")+ if self.is_immediate_saving+ else os.path.join(self.orig_output_dir, sanitized_format, "transformer")+ )++ # if use is_immediate_saving, we need to save model in self.orig_output_dir/transformer folder+ return os.path.join(self.orig_output_dir, "transformer") if self.is_immediate_saving else self.orig_output_dir+def save_quantized(self, output_dir=None, format="auto_round", inplace=True, **kwargs):"""Save the quantized model to the specified output directory in the specified format.@@ -385,5 +408,27 @@ class DiffusionCompressor(BaseCompressor):Returns:object: The compressed model object."""- compressed_model = super().save_quantized(output_dir=output_dir, format=format, inplace=inplace, **kwargs)+ if output_dir is None:+ return super().save_quantized(output_dir, format=format, inplace=inplace, **kwargs)++ compressed_model = None+ for name in self.pipe.components.keys():+ val = getattr(self.pipe, name)+ sub_module_path = (+ os.path.join(output_dir, name) if os.path.basename(os.path.normpath(output_dir)) != name else output_dir+ )+ if (+ hasattr(val, "config")+ and hasattr(val.config, "_name_or_path")+ and val.config._name_or_path == self.model.config._name_or_path+ ):+ compressed_model = super().save_quantized(+ output_dir=sub_module_path if not self.is_immediate_saving else output_dir,+ format=format,+ inplace=inplace,+ **kwargs,+ )+ elif val is not None and hasattr(val, "save_pretrained"):+ val.save_pretrained(sub_module_path)+ self.pipe.config.save_pretrained(output_dir)return compressed_modeldiff --git a/auto_round/eval/evaluation.py b/auto_round/eval/evaluation.pyindex f962ae995..d4c4ec917 100644--- a/auto_round/eval/evaluation.py+++ b/auto_round/eval/evaluation.py@@ -100,10 +100,15 @@ def evaluate_diffusion_model(args, autoround=None, model=None, pipe=None):import torch- from auto_round.utils import detect_device, get_model_dtype, logger+ from auto_round.utils import detect_device, get_model_dtype, logger, unsupported_meta_device# Prepare inference pipelineif pipe is None:+ if model is not None and unsupported_meta_device(model):+ logger.error(+ "Quantized model is meta and diffusers doesn't support loading auto-round quantized model now. Exit."+ )+ exit(0)pipe = autoround.pipepipe.to(model.dtype)pipe.transformer = modeldiff --git a/auto_round/utils/model.py b/auto_round/utils/model.pyindex 368f83886..0e8bb5938 100644--- a/auto_round/utils/model.py+++ b/auto_round/utils/model.py@@ -527,6 +527,8 @@ def diffusion_load_model(model_dtype: str = None,**kwargs,):+ from functools import partial+from auto_round.utils.common import LazyImportfrom auto_round.utils.device import get_device_and_parallelism@@ -543,12 +545,68 @@ def diffusion_load_model(torch_dtype = torch.bfloat16pipelines = LazyImport("diffusers.pipelines")+ if isinstance(pretrained_model_name_or_path, str):+ if torch_dtype == "auto":+ torch_dtype = {}+ model_index = os.path.join(pretrained_model_name_or_path, "model_index.json")+ with open(model_index, "r", encoding="utf-8") as file:+ config = json.load(file)+ for k, v in config.items():+ component_folder = os.path.join(pretrained_model_name_or_path, k)+ if isinstance(v, list) and os.path.exists(os.path.join(component_folder, "config.json")):+ component_folder = os.path.join(pretrained_model_name_or_path, k)+ with open(os.path.join(component_folder, "config.json"), "r", encoding="utf-8") as file:+ component_config = json.load(file)+ torch_dtype[k] = component_config.get("torch_dtype", "auto")++ pipe = pipelines.auto_pipeline.AutoPipelineForText2Image.from_pretrained(+ pretrained_model_name_or_path, torch_dtype=torch_dtype+ )+ pipe_config = pipe.load_config(pretrained_model_name_or_path)++ elif isinstance(pretrained_model_name_or_path, pipelines.pipeline_utils.DiffusionPipeline):+ pipe = pretrained_model_name_or_path+ pipe_config = pipe.load_config(pipe.config["_name_or_path"])++ else:+ raise ValueError(+ f"Only support str or DiffusionPipeline class for model, but get {type(pretrained_model_name_or_path)}"+ )++ # add missing key+ for k, v in pipe_config.items():+ if k not in pipe.config:+ pipe.config[k] = v- pipe = pipelines.auto_pipeline.AutoPipelineForText2Image.from_pretrained(- pretrained_model_name_or_path, torch_dtype=torch_dtype- )pipe = _to_model_dtype(pipe, model_dtype)model = pipe.transformer++ def config_save_pretrained(config, file_name, save_directory):+ if os.path.isfile(save_directory):+ raise AssertionError(f"Provided path ({save_directory}) should be a directory, not a file")+ os.makedirs(save_directory, exist_ok=True)+ output_config_file = os.path.join(save_directory, file_name)++ config_dict = dict(config)+ if file_name == "config.json" and hasattr(model.config, "quantization_config"):+ config_dict["quantization_config"] = model.config.quantization_config++ with open(output_config_file, "w", encoding="utf-8") as writer:+ writer.write(json.dumps(config_dict, indent=2, sort_keys=True) + "\n")++ # meta model uses model.config.save_pretrained for config saving+ setattr(model.config, "save_pretrained", partial(config_save_pretrained, model.config, "config.json"))+ setattr(pipe.config, "save_pretrained", partial(config_save_pretrained, pipe.config, "model_index.json"))++ def model_save_pretrained(model, save_directory, **kwargs):+ super(model.__class__, model).save_pretrained(save_directory, **kwargs)+ if hasattr(model.config, "quantization_config"):+ model.config["quantization_config"] = model.config.quantization_config+ with open(os.path.join(save_directory, "config.json"), "w", encoding="utf-8") as writer:+ writer.write(json.dumps(dict(model.config), indent=2, sort_keys=True) + "\n")++ # non-meta model uses model.save_pretrained for model and config saving+ setattr(model, "save_pretrained", partial(model_save_pretrained, model))return pipe, model.to(device)diff --git a/test/test_cpu/models/test_diffusion.py b/test/test_cpu/models/test_diffusion.pyindex ca495500a..4439131f4 100644--- a/test/test_cpu/models/test_diffusion.py+++ b/test/test_cpu/models/test_diffusion.py@@ -1,6 +1,8 @@+import osimport shutilimport pytest+import torchfrom packaging import versionfrom auto_round import AutoRound@@ -16,11 +18,32 @@ def setup_flux():from diffusers import AutoPipelineForText2Imagemodel_name = flux_name_or_path- pipe = AutoPipelineForText2Image.from_pretrained(model_name)+ # use bf16 to reduce the saved model size+ pipe = AutoPipelineForText2Image.from_pretrained(model_name, torch_dtype=torch.bfloat16)output_dir = "./tmp/test_quantized_flux"return pipe, output_dir+ transformers_version >= version.parse("5.0.0"),+ reason="cannot import name 'MT5Tokenizer' from 'transformers', https://github.com/huggingface/diffusers/issues/13035",+)+def test_flux_saving(setup_flux):+ pipe, output_dir = setup_flux+ autoround = AutoRound(+ pipe,+ tokenizer=None,+ scheme="W4A16",+ iters=0,+ num_inference_steps=2,+ disable_opt_rtn=True,+ )+ autoround.quantize_and_save(output_dir)+ assert os.path.exists(os.path.join(output_dir, "model_index.json"))+ assert os.path.exists(os.path.join(output_dir, "transformer", "quantization_config.json"))+ shutil.rmtree(output_dir, ignore_errors=True)++@pytest.mark.skipif(transformers_version >= version.parse("5.0.0"),reason="cannot import name 'MT5Tokenizer' from 'transformers', https://github.com/huggingface/diffusers/issues/13035",