PTQ-Level Efficiency, QAT-Level Accuracy | 训练后量化的效率,量化感知训练的精度
fix: recognize Modular Diffusers pipelines as diffusion models (#2120)
Nikita Davidchuk0d76712e7b83
diff --git a/auto_round/utils/model.py b/auto_round/utils/model.pyindex 12c21c58c..e018ca364 100644--- a/auto_round/utils/model.py+++ b/auto_round/utils/model.py@@ -1011,8 +1011,17 @@ def diffusion_load_model(return load_cosmos3_diffusion(pretrained_model_name_or_path, device_str)pipelines = LazyImport("diffusers.pipelines")+ modular_pipeline_cls = _get_modular_pipeline_class()if isinstance(pretrained_model_name_or_path, str):model_index = os.path.join(pretrained_model_name_or_path, "model_index.json")+ if not os.path.exists(model_index) and os.path.exists(+ os.path.join(pretrained_model_name_or_path, MODULAR_PIPELINE_INDEX_NAME)+ ):+ raise NotImplementedError(+ f"{pretrained_model_name_or_path} is a Modular Diffusers pipeline "+ f"({MODULAR_PIPELINE_INDEX_NAME}), which auto_round cannot assemble from a path yet. "+ "Build the ModularPipeline yourself and pass the pipeline object as `model` instead."+ )with open(model_index, "r", encoding="utf-8") as file:config = json.load(file)@@ -1035,13 +1044,19 @@ def diffusion_load_model()pipe_config = pipe.load_config(pretrained_model_name_or_path)- elif isinstance(pretrained_model_name_or_path, pipelines.pipeline_utils.DiffusionPipeline):+ elif isinstance(pretrained_model_name_or_path, pipelines.pipeline_utils.DiffusionPipeline) or (+ modular_pipeline_cls is not None and isinstance(pretrained_model_name_or_path, modular_pipeline_cls)+ ):pipe = pretrained_model_name_or_path- pipe_config = pipe.load_config(pipe.config["_name_or_path"])+ # a pipeline assembled in-process, as Modular Diffusers ones typically are,+ # has no _name_or_path to reload the on-disk index from+ name_or_path = pipe.config.get("_name_or_path", None)+ pipe_config = pipe.load_config(name_or_path) if name_or_path is not None else {}else:raise ValueError(- f"Only support str or DiffusionPipeline class for model, but get {type(pretrained_model_name_or_path)}"+ f"Only support str, DiffusionPipeline or ModularPipeline class for model, "+ f"but get {type(pretrained_model_name_or_path)}")# add missing key@@ -1324,8 +1339,9 @@ def is_mllm_model(model_or_path: Union[str, torch.nn.Module], platform: str = Nomodel_path = get_model_name_or_path(model_or_path)- # Fast path: return cached result for already-seen paths- if model_path in _is_mllm_model_cache:+ # Path-less in-process objects must be inspected independently rather than+ # sharing a cache entry under None.+ if model_path and model_path in _is_mllm_model_cache:return _is_mllm_model_cache[model_path]# Check model_type exclusion: some models have multimodal components@@ -1344,7 +1360,10 @@ def is_mllm_model(model_or_path: Union[str, torch.nn.Module], platform: str = No# For dummy model, model_path could be "".# Only try to download if the path looks like a HF repo id (not a local filesystem path).# Skip download for absolute paths or relative paths that contain current/parent dir markers.- _is_local_path = os.path.isabs(model_path) or model_path.startswith("./") or model_path.startswith("../")+ # model_path is None for a model or pipeline built in-process, which has no name or path+ _is_local_path = isinstance(model_path, str) and (+ os.path.isabs(model_path) or model_path.startswith("./") or model_path.startswith("../")+ )if model_path and not os.path.isdir(model_path) and not _is_local_path:model_path = download_or_get_path(model_path, platform=platform)@@ -1370,7 +1389,8 @@ def is_mllm_model(model_or_path: Union[str, torch.nn.Module], platform: str = No# Cache by the original path key (model_path may have been resolved above)original_key = get_model_name_or_path(model_or_path)- _is_mllm_model_cache[original_key] = result+ if original_key:+ _is_mllm_model_cache[original_key] = resultreturn result@@ -1387,6 +1407,48 @@ def is_gguf_model(model_path: Union[str, torch.nn.Module]) -> bool:return is_gguf_file+## ModularPipeline.config_name, spelled out to keep the diffusers import lazy+MODULAR_PIPELINE_INDEX_NAME = "modular_model_index.json"+++def _find_pipeline_index_file(model_dir_or_repo: str) -> Optional[str]:+ """Return the pipeline index file of a diffusers directory or repo, if it has one.++ Standard pipelines ship ``model_index.json``, Modular Diffusers pipelines ship+ ``modular_model_index.json`` instead.+ """+ index_names = ("model_index.json", MODULAR_PIPELINE_INDEX_NAME)++ if os.path.isdir(model_dir_or_repo):+ for name in index_names:+ index_file = os.path.join(model_dir_or_repo, name)+ if os.path.exists(index_file):+ check_diffusers_installed()+ return index_file+ return None++ from huggingface_hub import hf_hub_download++ for name in index_names:+ try:+ index_file = hf_hub_download(model_dir_or_repo, name)+ check_diffusers_installed()+ return index_file+ except Exception as e:+ logger.debug(f"No {name} found in {model_dir_or_repo}: {e}")+ return None+++def _get_modular_pipeline_class():+ """Return ModularPipeline when supported by the installed Diffusers version."""+ try:+ from diffusers.modular_pipelines import ModularPipeline++ return ModularPipeline+ except (ImportError, AttributeError):+ return None++def is_diffusion_model(model_or_path: Union[str, object], trust_remote_code: bool = True) -> bool:from auto_round.utils.common import LazyImport@@ -1411,25 +1473,14 @@ def is_diffusion_model(model_or_path: Union[str, object], trust_remote_code: boologger.warning(f"Failed to load config for {model_or_path}, trying to check model_index.json for diffusion pipeline.")- index_file = None- if not os.path.isdir(model_or_path):- try:- from huggingface_hub import hf_hub_download-- index_file = hf_hub_download(model_or_path, "model_index.json")- check_diffusers_installed()- except Exception as e:- print(e)- index_file = None-- elif os.path.exists(os.path.join(model_or_path, "model_index.json")):- check_diffusers_installed()- index_file = os.path.join(model_or_path, "model_index.json")- return index_file is not None+ return _find_pipeline_index_file(model_or_path) is not Noneelif not isinstance(model_or_path, torch.nn.Module):check_diffusers_installed()pipeline_utils = LazyImport("diffusers.pipelines.pipeline_utils")- return isinstance(model_or_path, pipeline_utils.DiffusionPipeline)+ if isinstance(model_or_path, pipeline_utils.DiffusionPipeline):+ return True+ modular_pipeline_cls = _get_modular_pipeline_class()+ return modular_pipeline_cls is not None and isinstance(model_or_path, modular_pipeline_cls)else:return Falsediff --git a/test/unit/test_cpu/models/test_diffusion.py b/test/unit/test_cpu/models/test_diffusion.pyindex bb2b077fa..5db877744 100644--- a/test/unit/test_cpu/models/test_diffusion.py+++ b/test/unit/test_cpu/models/test_diffusion.py@@ -172,6 +172,58 @@ def test_flux(setup_flux):shutil.rmtree(output_dir, ignore_errors=True)+def _build_empty_modular_pipeline():+ """A ModularPipeline with no components, so nothing is downloaded or loaded."""+ from diffusers.modular_pipelines import SequentialPipelineBlocks++ class EmptyBlocks(SequentialPipelineBlocks):+ block_classes = []+ block_names = []++ return EmptyBlocks().init_pipeline()+++def test_modular_pipeline_is_detected_as_diffusion():+ """A ModularPipeline is not a DiffusionPipeline, but it is still a diffusion model."""+ pytest.importorskip("diffusers.modular_pipelines")++ from auto_round.utils.model import detect_model_type, is_diffusion_model, is_mllm_model++ pipe = _build_empty_modular_pipeline()++ assert is_mllm_model(pipe) is False+ assert is_diffusion_model(pipe) is True+ assert detect_model_type(pipe) == "diffusion"+++def test_pathless_models_do_not_share_mllm_cache_entry():+ """Each in-process model without a path must be inspected independently."""+ from auto_round.utils.model import _is_mllm_model_cache, is_mllm_model++ class VisionModel(torch.nn.Module):+ def __init__(self):+ super().__init__()+ self.vision_model = torch.nn.Linear(1, 1)++ _is_mllm_model_cache.clear()+ assert is_mllm_model(torch.nn.Linear(1, 1)) is False+ assert is_mllm_model(VisionModel()) is True+ assert None not in _is_mllm_model_cache+++def test_modular_model_index_dir_is_detected_as_diffusion(tmp_path):+ """Modular Diffusers ships modular_model_index.json instead of model_index.json."""+ pytest.importorskip("diffusers.modular_pipelines")++ from auto_round.utils.model import diffusion_load_model, is_diffusion_model++ (tmp_path / "modular_model_index.json").write_text("{}", encoding="utf-8")++ assert is_diffusion_model(str(tmp_path)) is True+ with pytest.raises(NotImplementedError, match="Modular Diffusers"):+ diffusion_load_model(str(tmp_path))++# def test_flux_calib(setup_flux):# pipe, output_dir = setup_flux# autoround = AutoRound(