PTQ-Level Efficiency, QAT-Level Accuracy | 训练后量化的效率,量化感知训练的精度
enable --eval for diffusion model (#1522)
Xin Heeec1bed686c8
diff --git a/auto_round/compressors/diffusion/eval.py b/auto_round/compressors/diffusion/eval.pyindex 226054f4f..2e4ff600a 100644--- a/auto_round/compressors/diffusion/eval.py+++ b/auto_round/compressors/diffusion/eval.py@@ -70,6 +70,7 @@ def diffusion_eval(image_save_dir,batch_size,gen_kwargs,+ limit=None,):if (not importlib.util.find_spec("clip")@@ -82,6 +83,7 @@ def diffusion_eval(dataloader, _, _ = get_diffusion_dataloader(prompt_file, nsamples=-1, bs=batch_size)prompt_list = []image_list = []+ num_samples = 0for image_ids, prompts in dataloader:prompt_list.extend(prompts)@@ -95,6 +97,9 @@ def diffusion_eval(continuenew_ids.append(image_id)new_prompts.append(prompts[idx])+ num_samples += 1+ if limit is not None and num_samples >= limit > 0:+ breakif len(new_prompts) == 0:continue@@ -103,6 +108,9 @@ def diffusion_eval(for idx, image_id in enumerate(new_ids):output.images[idx].save(os.path.join(image_save_dir, str(image_id) + ".png"))+ if limit is not None and num_samples >= limit > 0:+ break+result = {}for metric in metrics:result.update(metric_map[metric](prompt_list, image_list, pipe.device))diff --git a/auto_round/eval/eval_cli.py b/auto_round/eval/eval_cli.pyindex c1ccf2b40..d9ff900b1 100644--- a/auto_round/eval/eval_cli.py+++ b/auto_round/eval/eval_cli.py@@ -24,6 +24,7 @@ from auto_round.utils import (get_device_and_parallelism,get_device_str,get_model_dtype,+ is_diffusion_model,set_cuda_visible_devices,)@@ -113,6 +114,59 @@ class EvalArgumentParser(argparse.ArgumentParser):help="(for vllm) Custom vllm arguments in format: 'arg1=value1,arg2=value2'. ""Example: 'tensor_parallel_size=2,gpu_memory_utilization=0.9'",)+ # Diffusion model specific arguments+ diffusion_args = self.add_argument_group("diffusion model arguments")+ diffusion_args.add_argument(+ "--prompt_file",+ default=None,+ type=str,+ help="File containing prompts for evaluation, one per line. "+ "Use this for batch evaluation with multiple prompts.",+ )+ diffusion_args.add_argument(+ "--prompt",+ default=None,+ type=str,+ help="Single prompt for quick testing. " "Overrides prompt_file if both are specified.",+ )+ diffusion_args.add_argument(+ "--metrics",+ "--metric",+ default="clip",+ help="Evaluation metrics for generated images. "+ "'clip': CLIP score measuring text-image alignment. "+ "'clip-iqa': CLIP-based image quality assessment. "+ "'imagereward': Learned metric for image quality.",+ )+ diffusion_args.add_argument(+ "--image_save_dir",+ default="./tmp_image_save",+ type=str,+ help="Directory to save generated images during evaluation. " "Useful for visual inspection of results.",+ )+ diffusion_args.add_argument(+ "--guidance_scale",+ default=7.5,+ type=float,+ help="Classifier-free guidance scale for diffusion models. "+ "Higher values (7-20) make the model follow the prompt more closely. "+ "Lower values give more creative/random results.",+ )+ diffusion_args.add_argument(+ "--num_inference_steps",+ default=50,+ type=int,+ help="Number of denoising steps in the diffusion process. "+ "More steps (50-100) usually give better quality but take longer. "+ "Fewer steps (10-30) are faster but lower quality.",+ )+ diffusion_args.add_argument(+ "--generator_seed",+ default=None,+ type=int,+ help="Random seed for image generation reproducibility. "+ "Using the same seed produces identical results across runs.",+ )def _eval_init(tasks, model_path, device, disable_trust_remote_code=False, dtype="auto"):@@ -134,6 +188,13 @@ def eval(args):"lm_eval>=0.4.2", "lm-eval is required for evaluation, please install it with `pip install 'lm-eval>=0.4.2'`")+ if is_diffusion_model(args.model):+ from auto_round.eval.evaluation import evaluate_diffusion_model+ from auto_round.utils import diffusion_load_model++ pipe, _ = diffusion_load_model(args.model)+ evaluate_diffusion_model(args, pipe=pipe)+ returnif args.eval_backend == "vllm":assert isinstance(args.model, str), "vllm evaluation only supports model name or path."eval_with_vllm(args)diff --git a/auto_round/eval/evaluation.py b/auto_round/eval/evaluation.pyindex 27a1fe746..f962ae995 100644--- a/auto_round/eval/evaluation.py+++ b/auto_round/eval/evaluation.py@@ -85,23 +85,28 @@ def simple_evaluate()-def evaluate_diffusion_model(autoround, model, args):+def evaluate_diffusion_model(args, autoround=None, model=None, pipe=None):"""Evaluate diffusion models.Args:- autoround: AutoRound instance- model: Quantized modelargs: Command line arguments+ autoround: AutoRound instance (option 1)+ model: Diffusion model instance (option 1)+ pipe: Diffusion pipeline instance (option 2)"""+ if pipe is None and (autoround is None or model is None):+ raise ValueError("Either 'pipe' must be provided, or both 'autoround' and 'model' must be provided.")+import torchfrom auto_round.utils import detect_device, get_model_dtype, logger# Prepare inference pipeline- pipe = autoround.pipe- pipe.to(model.dtype)- pipe.transformer = model+ if pipe is None:+ pipe = autoround.pipe+ pipe.to(model.dtype)+ pipe.transformer = modeldevice_str = detect_device(args.device_map if hasattr(args, "device_map") else "0")pipe = pipe.to(device_str)@@ -134,11 +139,11 @@ def evaluate_diffusion_model(autoround, model, args):logger.info(f"Image generated with prompt {args.prompt} is saved as {save_path}")# Batch prompt evaluation- if args.prompt_file is not None:+ elif args.prompt_file is not None:from auto_round.compressors.diffusion import diffusion_evalmetrics = args.metrics.split(",")- diffusion_eval(pipe, args.prompt_file, metrics, args.image_save_dir, 1, gen_kwargs)+ diffusion_eval(pipe, args.prompt_file, metrics, args.image_save_dir, 1, gen_kwargs, args.limit)def load_gguf_model_for_eval(eval_folder, formats, args):@@ -375,7 +380,7 @@ def run_model_evaluation(model, tokenizer, autoround, folders, formats, device_s# Handle diffusion models separatelyif getattr(autoround, "diffusion", False):- evaluate_diffusion_model(autoround, model, args)+ evaluate_diffusion_model(args, autoround=autoround, model=model)return# Check if evaluation is needed for language models