PTQ-Level Efficiency, QAT-Level Accuracy | 训练后量化的效率,量化感知训练的精度
support for model scope (#957)
Heng Guo2448dcf2b864
diff --git a/auto_round/__main__.py b/auto_round/__main__.pyindex 844f366bb..98c2e2cba 100644--- a/auto_round/__main__.py+++ b/auto_round/__main__.py@@ -45,6 +45,12 @@ class BasicArgumentParser(argparse.ArgumentParser):help="Path to the pre-trained model or model identifier from huggingface.co/models. ""Examples: 'facebook/opt-125m', 'bert-base-uncased', or local path like '/path/to/model'",)+ basic.add_argument(+ "--platform",+ default="hf",+ help="Platform to load the pre-trained model. Options: [hf, model_scope]."+ " hf stands for huggingface and model_scope stands for model scope.",+ )basic.add_argument("--scheme",default="W4A16",@@ -566,6 +572,7 @@ def tune(args):autoround: BaseCompressor = AutoRound(model=model_name,+ platform=args.platform,scheme=scheme,dataset=args.dataset,iters=args.iters,diff --git a/auto_round/autoround.py b/auto_round/autoround.pyindex cceb0668a..ff06ec851 100644--- a/auto_round/autoround.py+++ b/auto_round/autoround.py@@ -43,6 +43,7 @@ class AutoRound:Attributes:model (torch.nn.Module): The loaded PyTorch model in eval mode.tokenizer: Tokenizer used to prepare input text for calibration/tuning.+ platform (str): The platform to load pretrained moded, options: ["hf", "model_scope"]bits (int): Weight quantization bits.group_size (int): Per-group size for weight quantization.sym (bool): Whether to use symmetric weight quantization.@@ -67,6 +68,7 @@ class AutoRound:cls,model: Union[torch.nn.Module, str],tokenizer=None,+ platform: str = "hf",scheme: Union[str, dict, QuantizationScheme, AutoScheme] = "W4A16",layer_config: dict[str, Union[str, dict, QuantizationScheme]] = None,dataset: Union[str, list, tuple, torch.utils.data.DataLoader] = "NeelNanda/pile-10k",@@ -146,7 +148,7 @@ class AutoRound:"""model_cls = []- if (extra_config and not extra_config.mllm_config.is_default()) or is_mllm_model(model):+ if (extra_config and not extra_config.mllm_config.is_default()) or is_mllm_model(model, platform=platform):logger.info("using MLLM mode for multimodal model.")model_cls.append(MLLMCompressor)if extra_config:@@ -170,6 +172,7 @@ class AutoRound:ar = dynamic_compressor(model=model,tokenizer=tokenizer,+ platform=platform,scheme=scheme,layer_config=layer_config,dataset=dataset,@@ -314,6 +317,7 @@ class AutoRoundLLM(LLMCompressor):self,model: Union[torch.nn.Module, str],tokenizer=None,+ platform: str = "hf",scheme: Union[str, dict, QuantizationScheme] = "W4A16",layer_config: dict[str, Union[str, dict, QuantizationScheme]] = None,dataset: Union[str, list, tuple, torch.utils.data.DataLoader] = "NeelNanda/pile-10k",@@ -331,6 +335,7 @@ class AutoRoundLLM(LLMCompressor):super().__init__(model=model,tokenizer=tokenizer,+ platform=platform,scheme=scheme,layer_config=layer_config,dataset=dataset,@@ -354,6 +359,7 @@ class AutoRoundAdam(AdamCompressor):Args:model: The PyTorch model to be quantized.tokenizer: An optional tokenizer for processing input data.+ platform (str): The platform to load pretrained moded, options: ["hf", "model_scope"]scheme (str| dict | QuantizationScheme ): A preset scheme that defines the quantization configurationsbits (int): Number of bits for quantization (default is 4).group_size (int): Size of the quantization group (default is 128).@@ -413,6 +419,7 @@ class AutoRoundAdam(AdamCompressor):self,model: Union[torch.nn.Module, str],tokenizer=None,+ platform: str = "hf",scheme: Union[str, dict, QuantizationScheme] = "W4A16",layer_config: dict[str, Union[str, dict, QuantizationScheme]] = None,dataset: Union[str, list, tuple, torch.utils.data.DataLoader] = "NeelNanda/pile-10k",@@ -431,6 +438,7 @@ class AutoRoundAdam(AdamCompressor):super().__init__(model=model,tokenizer=tokenizer,+ platform=platform,scheme=scheme,layer_config=layer_config,batch_size=batch_size,@@ -455,6 +463,7 @@ class AutoRoundMLLM(MLLMCompressor):Args:model: The PyTorch model to be quantized.tokenizer: An optional tokenizer for processing input data.+ platform (str): The platform to load pretrained moded, options: ["hf", "model_scope"]processor: Any multi-modal model will require an object to encode ordecode the data that groups several modalities (among text, vision and audio).image_processor: Image processor for special model like llava.@@ -513,6 +522,7 @@ class AutoRoundMLLM(MLLMCompressor):self,model: Union[torch.nn.Module, str],tokenizer=None,+ platform: str = "hf",processor=None,image_processor=None,scheme: Union[str, dict, QuantizationScheme] = "W4A16",@@ -533,6 +543,7 @@ class AutoRoundMLLM(MLLMCompressor):super().__init__(model=model,tokenizer=tokenizer,+ platform=platform,processor=processor,image_processor=image_processor,scheme=scheme,@@ -559,6 +570,7 @@ class AutoRoundDiffusion(DiffusionCompressor):Args:model: The PyTorch model to be quantized.tokenizer: An optional tokenizer for processing input data, is not used for diffusion models.+ platform (str): The platform to load pretrained moded, options: ["hf", "model_scope"]guidance_scale (float): Control how much the image generation process follows the text prompt.The more it is, the more closely it follows the prompt (default is 7.5).num_inference_steps (int): The reference number of denoising steps (default is 50).diff --git a/auto_round/compressors/adam.py b/auto_round/compressors/adam.pyindex 4606eab3a..fb79cf39a 100644--- a/auto_round/compressors/adam.py+++ b/auto_round/compressors/adam.py@@ -27,6 +27,7 @@ class AdamCompressor(BaseCompressor):Args:model: The PyTorch model to be quantized.tokenizer: An optional tokenizer for processing input data.+ platform (str): The platform to load pretrained moded, options: ["hf", "model_scope"]scheme (str| dict | QuantizationScheme ): A preset scheme that defines the quantization configurationsbits (int): Number of bits for quantization (default is 4).group_size (int): Size of the quantization group (default is 128).@@ -86,6 +87,7 @@ class AdamCompressor(BaseCompressor):self,model: Union[torch.nn.Module, str],tokenizer=None,+ platform="hf",scheme: Union[str, dict, QuantizationScheme] = "W4A16",layer_config: dict[str, Union[str, dict, QuantizationScheme]] = None,dataset: Union[str, list, tuple, torch.utils.data.DataLoader] = "NeelNanda/pile-10k",@@ -104,6 +106,7 @@ class AdamCompressor(BaseCompressor):super(AdamCompressor, self).__init__(model=model,tokenizer=tokenizer,+ platform=platform,scheme=scheme,layer_config=layer_config,batch_size=batch_size,diff --git a/auto_round/compressors/base.py b/auto_round/compressors/base.pyindex fe0939c1f..602b566a2 100644--- a/auto_round/compressors/base.py+++ b/auto_round/compressors/base.py@@ -30,6 +30,7 @@ from torch import autocastfrom tqdm import tqdmfrom transformers import set_seed+from auto_round import envsfrom auto_round.auto_scheme.gen_auto_scheme import AutoSchemefrom auto_round.compressors.utils import (block_forward,@@ -105,6 +106,7 @@ class BaseCompressor(object):Attributes:model (torch.nn.Module): The loaded PyTorch model in eval mode.tokenizer: Tokenizer used to prepare input text for calibration/tuning.+ platform (str): The platform to load pretrained moded, options: ["hf", "model_scope"]bits (int): Weight quantization bits.group_size (int): Per-group size for weight quantization.sym (bool): Whether to use symmetric weight quantization.@@ -129,6 +131,7 @@ class BaseCompressor(object):self,model: Union[torch.nn.Module, str],tokenizer=None,+ platform="hf",scheme: Union[str, dict, QuantizationScheme, AutoScheme] = "W4A16",layer_config: dict[str, Union[str, dict, QuantizationScheme]] = None,dataset: Union[str, list, tuple, torch.utils.data.DataLoader] = "NeelNanda/pile-10k",@@ -228,6 +231,10 @@ class BaseCompressor(object):device = kwargs.pop("device", None)# Scale factor for RAM usage per parameter.mem_per_param_scale = kwargs.pop("mem_per_param_scale", None)++ if envs.AR_USE_MODELSCOPE:+ platform = "model_scope"+ self.platform = platformself.quant_lm_head = kwargs.pop("quant_lm_head", False)self.mllm = kwargs.pop("mllm") if "mllm" in kwargs else Falseself.diffusion = kwargs.pop("diffusion") if "diffusion" in kwargs else False@@ -259,6 +266,7 @@ class BaseCompressor(object):if isinstance(model, str):model, tokenizer = llm_load_model(model,+ platform=platform,device="cpu", # always load cpu first)elif tokenizer is None and not self.diffusion and iters > 0:diff --git a/auto_round/compressors/diffusion/compressor.py b/auto_round/compressors/diffusion/compressor.pyindex cb69e6a88..904767602 100644--- a/auto_round/compressors/diffusion/compressor.py+++ b/auto_round/compressors/diffusion/compressor.py@@ -47,6 +47,7 @@ class DiffusionCompressor(BaseCompressor):Args:model: The PyTorch model to be quantized.tokenizer: An optional tokenizer for processing input data, is not used for diffusion models.+ platform (str): The platform to load pretrained moded, options: ["hf", "model_scope"]guidance_scale (float): Control how much the image generation process follows the text prompt.The more it is, the more closely it follows the prompt (default is 7.5).num_inference_steps (int): The reference number of denoising steps (default is 50).@@ -81,6 +82,7 @@ class DiffusionCompressor(BaseCompressor):self,model: Union[object, str],tokenizer=None,+ platform: str = "hf",guidance_scale: float = 7.5,num_inference_steps: int = 50,generator_seed: int = None,@@ -110,7 +112,7 @@ class DiffusionCompressor(BaseCompressor):self._set_device(device_map)if isinstance(model, str):- pipe, model = diffusion_load_model(model, device=self.device)+ pipe, model = diffusion_load_model(model, platform=platform, device=self.device)elif isinstance(model, pipeline_utils.DiffusionPipeline):pipe = modelmodel = pipe.transformer@@ -145,6 +147,7 @@ class DiffusionCompressor(BaseCompressor):super(DiffusionCompressor, self).__init__(model=model,tokenizer=None,+ platform=platform,scheme=scheme,layer_config=layer_config,dataset=dataset,diff --git a/auto_round/compressors/mllm/compressor.py b/auto_round/compressors/mllm/compressor.pyindex b5586a4bb..2cfa457b7 100644--- a/auto_round/compressors/mllm/compressor.py+++ b/auto_round/compressors/mllm/compressor.py@@ -87,6 +87,7 @@ class MLLMCompressor(BaseCompressor):Args:model: The PyTorch model to be quantized.tokenizer: An optional tokenizer for processing input data.+ platform (str): The platform to load pretrained moded, options: ["hf", "model_scope"]processor: Any multi-modal model will require an object to encode ordecode the data that groups several modalities (among text, vision and audio).image_processor: Image processor for special model like llava.@@ -145,6 +146,7 @@ class MLLMCompressor(BaseCompressor):self,model: Union[torch.nn.Module, str],tokenizer=None,+ platform: str = "hf",processor=None,image_processor=None,scheme: Union[str, dict, QuantizationScheme] = "W4A16",@@ -171,7 +173,7 @@ class MLLMCompressor(BaseCompressor):self._set_device(device_map)if isinstance(model, str):- model, processor, tokenizer, image_processor = mllm_load_model(model, device=self.device)+ model, processor, tokenizer, image_processor = mllm_load_model(model, platform=platform, device=self.device)self.model = modelquant_nontext_module = self._check_quant_nontext(layer_config, quant_nontext_module)@@ -258,6 +260,7 @@ class MLLMCompressor(BaseCompressor):super(MLLMCompressor, self).__init__(model=model,tokenizer=tokenizer,+ platform=platform,scheme=scheme,layer_config=layer_config,dataset=dataset,@@ -374,6 +377,7 @@ class MLLMCompressor(BaseCompressor):continuetry:if isinstance(data_new, torch.Tensor):+ data_new = data_new.to(self.model.device)self.model(data_new)elif isinstance(data_new, tuple) or isinstance(data_new, list):self.model(*data_new)diff --git a/auto_round/compressors/utils.py b/auto_round/compressors/utils.pyindex 8a8fe602c..79e8b7133 100644--- a/auto_round/compressors/utils.py+++ b/auto_round/compressors/utils.py@@ -480,7 +480,7 @@ def gguf_args_check(args_or_ar, formats: list[str] = None, model_type=ModelType.from auto_round.export.export_to_gguf.convert import download_convert_filefrom auto_round.logger import logger- from auto_round.utils.model import download_hf_model, get_gguf_architecture+ from auto_round.utils.model import download_or_get_path, get_gguf_architectureformats = sorted(formats, key=lambda x: len(x))export_gguf = False@@ -505,7 +505,7 @@ def gguf_args_check(args_or_ar, formats: list[str] = None, model_type=ModelType.else:model_path = args_or_ar.model.name_or_pathif not os.path.isdir(model_path):- model_path = download_hf_model(model_path)+ model_path = download_or_get_path(model_path, args_or_ar.platform)model_architecture = get_gguf_architecture(model_path, model_type=ModelType.TEXT)if model_architecture not in ModelBase._model_classes[ModelType.TEXT]:logger.warning(@@ -539,7 +539,7 @@ def gguf_args_check(args_or_ar, formats: list[str] = None, model_type=ModelType.else:model_path = args_or_ar.model.name_or_pathif not os.path.isdir(model_path):- model_path = download_hf_model(model_path)+ model_path = download_or_get_path(model_path, args_or_ar.platform)model_architecture = get_gguf_architecture(model_path, model_type=ModelType.TEXT)if model_architecture not in ModelBase._model_classes[ModelType.TEXT]:logger.error(f"Model {model_architecture} is not supported to export gguf format.")diff --git a/auto_round/envs.py b/auto_round/envs.pyindex ae0d79f1d..3d9b0dfa3 100644--- a/auto_round/envs.py+++ b/auto_round/envs.py@@ -18,10 +18,12 @@ from typing import TYPE_CHECKING, Any, Callable, Optionalif TYPE_CHECKING:AR_LOG_LEVEL: str = "INFO"+ AR_USE_MODELSCOPE: bool = "False"environment_variables: dict[str, Callable[[], Any]] = {# this is used for configuring the default logging level"AR_LOG_LEVEL": lambda: os.getenv("AR_LOG_LEVEL", "INFO").upper(),+ "AR_USE_MODELSCOPE": lambda: os.getenv("AR_USE_MODELSCOPE", "False").lower() in ["1", "true"],}@@ -41,3 +43,30 @@ def is_set(name: str):if name in environment_variables:return name in os.environraise AttributeError(f"module {__name__!r} has no attribute {name!r}")+++def set_config(**kwargs):+ """+ Set configuration values for environment variables.++ Args:+ **kwargs: Keyword arguments where keys are environment variable names+ and values are the desired values to set.++ Example:+ set_config(AR_LOG_LEVEL="DEBUG", AR_USE_MODELSCOPE=True)+ """+ for key, value in kwargs.items():+ if key in environment_variables:+ # Convert value to appropriate string format+ if key == "AR_USE_MODELSCOPE":+ # Handle boolean values for AR_USE_MODELSCOPE+ str_value = "true" if value in [True, "True", "true", "1", 1] else "false"+ else:+ # For other variables, convert to string+ str_value = str(value)++ # Set the environment variable+ os.environ[key] = str_value+ else:+ raise AttributeError(f"module {__name__!r} has no attribute {key!r}")diff --git a/auto_round/export/export_to_gguf/convert.py b/auto_round/export/export_to_gguf/convert.pyindex 03981db61..3ac31932d 100644--- a/auto_round/export/export_to_gguf/convert.py+++ b/auto_round/export/export_to_gguf/convert.py@@ -167,11 +167,11 @@ def get_tensors(cls) -> Iterator[tuple[str, Tensor]]:from safetensors import safe_openfrom auto_round.export.export_to_gguf.special_handle import get_tensor_from_file- from auto_round.utils import download_hf_model+ from auto_round.utils import download_or_get_pathdir_path = cls.model.name_or_pathif not os.path.isdir(dir_path):- dir_path = download_hf_model(dir_path)+ dir_path = download_or_get_path(dir_path)INDEX_FILE = "model.safetensors.index.json"if INDEX_FILE in os.listdir(dir_path):with open(os.path.join(dir_path, INDEX_FILE)) as f:diff --git a/auto_round/export/export_to_gguf/export.py b/auto_round/export/export_to_gguf/export.pyindex 526056263..8633a2a50 100644--- a/auto_round/export/export_to_gguf/export.py+++ b/auto_round/export/export_to_gguf/export.py@@ -28,7 +28,7 @@ from auto_round.utils import (LazyImport,check_to_quantized,clear_memory,- download_hf_model,+ download_or_get_path,flatten_list,get_block_names,get_gguf_architecture,@@ -77,7 +77,7 @@ def create_model_class(tmp_work_dir = model.name_or_pathos.makedirs(output_dir, exist_ok=True)if not os.path.isdir(tmp_work_dir):- tmp_work_dir = download_hf_model(tmp_work_dir)+ tmp_work_dir = download_or_get_path(tmp_work_dir)with torch.inference_mode():model_architecture = get_gguf_architecture(tmp_work_dir, model_type=model_type)try:diff --git a/auto_round/export/export_to_gguf/special_handle.py b/auto_round/export/export_to_gguf/special_handle.pyindex f8a463e9b..cb15af7a1 100644--- a/auto_round/export/export_to_gguf/special_handle.py+++ b/auto_round/export/export_to_gguf/special_handle.py@@ -20,7 +20,7 @@ import torchfrom safetensors import safe_openfrom torch import Tensor-from auto_round.utils import download_hf_model+from auto_round.utils import download_or_get_pathdef handle_special_model(cls, model_architecture):@@ -32,7 +32,7 @@ def handle_special_model(cls, model_architecture):def get_tensor_from_file(dir_path, tensor_name):if not os.path.isdir(dir_path):- dir_path = download_hf_model(dir_path)+ dir_path = download_or_get_path(dir_path)INDEX_FILE = "model.safetensors.index.json"# get filenameif INDEX_FILE in os.listdir(dir_path):diff --git a/auto_round/utils/model.py b/auto_round/utils/model.pyindex 7a1c66de8..427ec7f9c 100644--- a/auto_round/utils/model.py+++ b/auto_round/utils/model.py@@ -23,6 +23,7 @@ from typing import Unionimport torchimport transformers+from auto_round import envsfrom auto_round.export.export_to_gguf.config import ModelTypefrom auto_round.logger import loggerfrom auto_round.schemes import QuantizationScheme@@ -148,6 +149,39 @@ def check_start_with_block_name(name: str, block_name_to_quantize: list):return False+def download_or_get_path(repo_id: str, platform: str = None) -> str:+ from auto_round import envs++ if platform is None:+ if envs.AR_USE_MODELSCOPE:+ platform = "model_scope"+ else:+ platform = "hf"++ if platform == "model_scope":+ return download_modelscope_model(repo_id)+ else:+ return download_hf_model(repo_id)+++def download_modelscope_model(repo_id: str, local_dir: str = None, cache_dir: str = None):+ from modelscope.utils.file_utils import get_modelscope_cache_dir # pylint: disable=E0401++ system_cache = cache_dir if cache_dir is not None else get_modelscope_cache_dir()+ if local_dir:+ directory = os.path.abspath(local_dir)+ elif cache_dir:+ directory = os.path.join(system_cache, *repo_id.split("/"))+ else:+ directory = os.path.join(system_cache, "models", *repo_id.split("/"))+ if os.path.exists(directory):+ return directory+ else:+ from modelscope.hub.snapshot_download import snapshot_download # pylint: disable=E0401++ return snapshot_download(repo_id)++def download_hf_model(repo_id, cache_dir=None, repo_type=None, revision=None):"""Download hugging face model from hf hub."""from huggingface_hub.constants import DEFAULT_REVISION, HUGGINGFACE_HUB_CACHE@@ -180,13 +214,23 @@ def download_hf_model(repo_id, cache_dir=None, repo_type=None, revision=None):def llm_load_model(- pretrained_model_name_or_path,- trust_remote_code=True,- model_dtype=None,- device="cpu",+ pretrained_model_name_or_path: str,+ platform: str = "hf",+ trust_remote_code: bool = True,+ model_dtype: str = None,+ device: str = "cpu",**kwargs,):- from transformers import AutoModel, AutoModelForCausalLM, AutoTokenizer+ assert platform.lower() in [+ "hf",+ "model_scope",+ ], "current only support hf or model_scope platform to load pretrained model."+ if platform.lower() == "model_scope" and not envs.AR_USE_MODELSCOPE:+ envs.set_config(AR_USE_MODELSCOPE=True)+ if platform == "model_scope":+ from modelscope import AutoModel, AutoModelForCausalLM, AutoTokenizer # pylint: disable=E0401+ else:+ from transformers import AutoModel, AutoModelForCausalLM, AutoTokenizerfrom auto_round.utils.device import (_use_hpu_compile_mode,@@ -254,16 +298,32 @@ def llm_load_model(def mllm_load_model(- pretrained_model_name_or_path,- device="cpu",- torch_dtype="auto",- use_auto_mapping=True,- trust_remote_code=True,- model_dtype=None,+ pretrained_model_name_or_path: str,+ platform: str = "hf",+ device: str = "cpu",+ torch_dtype: str = "auto",+ use_auto_mapping: bool = True,+ trust_remote_code: bool = True,+ model_dtype: str = None,**kwargs,):- import transformers- from transformers import AutoModel, AutoModelForCausalLM, AutoProcessor, AutoTokenizer+ assert platform.lower() in [+ "hf",+ "model_scope",+ ], "current only support hf or model_scope platform to load pretrained model."+ if platform.lower() == "model_scope" and not envs.AR_USE_MODELSCOPE:+ envs.set_config(AR_USE_MODELSCOPE=True)++ if platform == "model_scope":+ import modelscope # pylint: disable=E0401+ from modelscope import AutoModel, AutoModelForCausalLM, AutoProcessor, AutoTokenizer # pylint: disable=E0401++ base_lib = modelscope+ else:+ import transformers+ from transformers import AutoModel, AutoModelForCausalLM, AutoProcessor, AutoTokenizer++ base_lib = transformersfrom auto_round.utils.device import get_device_and_parallelism, set_fake_cuda_device_capability@@ -322,11 +382,11 @@ def mllm_load_model()else:if architectures.endswith("Model") and hasattr(- transformers, n := architectures.replace("Model", "ForConditionalGeneration")+ base_lib, n := architectures.replace("Model", "ForConditionalGeneration")):- cls = getattr(transformers, n)- elif hasattr(transformers, architectures):- cls = getattr(transformers, architectures)+ cls = getattr(base_lib, n)+ elif hasattr(base_lib, architectures):+ cls = getattr(base_lib, architectures)else:cls = AutoModelForCausalLMtry:@@ -365,7 +425,10 @@ def mllm_load_model(pretrained_model_name_or_path, trust_remote_code=trust_remote_code)try:- from transformers import AutoImageProcessor+ if platform == "model_scope":+ from modelscope import AutoImageProcessor # pylint: disable=E0401+ else:+ from transformers import AutoImageProcessorimage_processor = AutoImageProcessor.from_pretrained(pretrained_model_name_or_path, trust_remote_code=trust_remote_code@@ -382,6 +445,7 @@ def mllm_load_model(def diffusion_load_model(pretrained_model_name_or_path: str,+ platform: str = "hf",device: Union[str, torch.device] = "cpu",torch_dtype: Union[str, torch.dtype] = "auto",use_auto_mapping: bool = False,@@ -392,6 +456,11 @@ def diffusion_load_model(from auto_round.utils.common import LazyImportfrom auto_round.utils.device import get_device_and_parallelism+ if platform != "hf":+ raise NotImplementedError(+ f"auto_round current only support hf as platform for diffusion model, but get {platform}"+ )+device_str, use_auto_mapping = get_device_and_parallelism(device)torch_dtype = "auto"if device_str is not None and "hpu" in device_str:@@ -425,7 +494,7 @@ def is_pure_text_model(model):return True-def is_mllm_model(model_or_path: Union[str, torch.nn.Module]):+def is_mllm_model(model_or_path: Union[str, torch.nn.Module], platform: str = None):MM_KEYS = ["multi_modal_projector","vision_tower",@@ -446,7 +515,7 @@ def is_mllm_model(model_or_path: Union[str, torch.nn.Module]):model_path = model_or_path if isinstance(model_or_path, str) else model_or_path.name_or_pathif not os.path.isdir(model_path):- model_path = download_hf_model(model_path)+ model_path = download_or_get_path(model_path, platform=platform)if isinstance(model_path, str):if os.path.exists(os.path.join(model_path, "preprocessor_config.json")):diff --git a/test/test_cpu/test_model_scope.py b/test/test_cpu/test_model_scope.pynew file mode 100644index 000000000..0663bcf12--- /dev/null+++ b/test/test_cpu/test_model_scope.py@@ -0,0 +1,51 @@+import copy+import shutil+import sys+import unittest++sys.path.insert(0, "../..")++import torch++from auto_round import AutoRound+++class LLMDataLoader:+ def __init__(self):+ self.batch_size = 1++ def __iter__(self):+ for i in range(3):+ yield torch.ones([1, 10], dtype=torch.long)+++class TestModelScope(unittest.TestCase):+ @classmethod+ def setUpClass(self):+ self.saved_path = "./saved"+ self.dataset = LLMDataLoader()++ @classmethod+ def tearDownClass(self):+ shutil.rmtree("./saved", ignore_errors=True)+ shutil.rmtree("runs", ignore_errors=True)++ return super().tearDownClass()++ def test_llm(self):+ model_name = "Qwen/Qwen2.5-0.5B-Instruct"+ autoround = AutoRound(+ model_name, platform="model_scope", scheme="w4a16", iters=0, seqlen=2, dataset=self.dataset+ )+ autoround.quantize_and_save()++ def test_mllm(self):+ model_name = "Qwen/Qwen2-VL-2B-Instruct"+ autoround = AutoRound(+ model_name, platform="model_scope", scheme="w4a16", iters=0, seqlen=2, dataset=self.dataset, batch_size=2+ )+ autoround.quantize_and_save(self.saved_path)+++if __name__ == "__main__":+ unittest.main()