Skip to content

PTQ-Level Efficiency, QAT-Level Accuracy | 训练后量化的效率,量化感知训练的精度

support for model scope (#957)

Heng Guo2448dcf2b864
diff --git a/auto_round/__main__.py b/auto_round/__main__.py
index 844f366bb..98c2e2cba 100644
--- a/auto_round/__main__.py
+++ b/auto_round/__main__.py
@@ -45,6 +45,12 @@ class BasicArgumentParser(argparse.ArgumentParser):
help="Path to the pre-trained model or model identifier from huggingface.co/models. "
"Examples: 'facebook/opt-125m', 'bert-base-uncased', or local path like '/path/to/model'",
)
+ basic.add_argument(
+ "--platform",
+ default="hf",
+ help="Platform to load the pre-trained model. Options: [hf, model_scope]."
+ " hf stands for huggingface and model_scope stands for model scope.",
+ )
basic.add_argument(
"--scheme",
default="W4A16",
@@ -566,6 +572,7 @@ def tune(args):
autoround: BaseCompressor = AutoRound(
model=model_name,
+ platform=args.platform,
scheme=scheme,
dataset=args.dataset,
iters=args.iters,
diff --git a/auto_round/autoround.py b/auto_round/autoround.py
index cceb0668a..ff06ec851 100644
--- a/auto_round/autoround.py
+++ b/auto_round/autoround.py
@@ -43,6 +43,7 @@ class AutoRound:
Attributes:
model (torch.nn.Module): The loaded PyTorch model in eval mode.
tokenizer: Tokenizer used to prepare input text for calibration/tuning.
+ platform (str): The platform to load pretrained moded, options: ["hf", "model_scope"]
bits (int): Weight quantization bits.
group_size (int): Per-group size for weight quantization.
sym (bool): Whether to use symmetric weight quantization.
@@ -67,6 +68,7 @@ class AutoRound:
cls,
model: Union[torch.nn.Module, str],
tokenizer=None,
+ platform: str = "hf",
scheme: Union[str, dict, QuantizationScheme, AutoScheme] = "W4A16",
layer_config: dict[str, Union[str, dict, QuantizationScheme]] = None,
dataset: Union[str, list, tuple, torch.utils.data.DataLoader] = "NeelNanda/pile-10k",
@@ -146,7 +148,7 @@ class AutoRound:
"""
model_cls = []
- if (extra_config and not extra_config.mllm_config.is_default()) or is_mllm_model(model):
+ if (extra_config and not extra_config.mllm_config.is_default()) or is_mllm_model(model, platform=platform):
logger.info("using MLLM mode for multimodal model.")
model_cls.append(MLLMCompressor)
if extra_config:
@@ -170,6 +172,7 @@ class AutoRound:
ar = dynamic_compressor(
model=model,
tokenizer=tokenizer,
+ platform=platform,
scheme=scheme,
layer_config=layer_config,
dataset=dataset,
@@ -314,6 +317,7 @@ class AutoRoundLLM(LLMCompressor):
self,
model: Union[torch.nn.Module, str],
tokenizer=None,
+ platform: str = "hf",
scheme: Union[str, dict, QuantizationScheme] = "W4A16",
layer_config: dict[str, Union[str, dict, QuantizationScheme]] = None,
dataset: Union[str, list, tuple, torch.utils.data.DataLoader] = "NeelNanda/pile-10k",
@@ -331,6 +335,7 @@ class AutoRoundLLM(LLMCompressor):
super().__init__(
model=model,
tokenizer=tokenizer,
+ platform=platform,
scheme=scheme,
layer_config=layer_config,
dataset=dataset,
@@ -354,6 +359,7 @@ class AutoRoundAdam(AdamCompressor):
Args:
model: The PyTorch model to be quantized.
tokenizer: An optional tokenizer for processing input data.
+ platform (str): The platform to load pretrained moded, options: ["hf", "model_scope"]
scheme (str| dict | QuantizationScheme ): A preset scheme that defines the quantization configurations
bits (int): Number of bits for quantization (default is 4).
group_size (int): Size of the quantization group (default is 128).
@@ -413,6 +419,7 @@ class AutoRoundAdam(AdamCompressor):
self,
model: Union[torch.nn.Module, str],
tokenizer=None,
+ platform: str = "hf",
scheme: Union[str, dict, QuantizationScheme] = "W4A16",
layer_config: dict[str, Union[str, dict, QuantizationScheme]] = None,
dataset: Union[str, list, tuple, torch.utils.data.DataLoader] = "NeelNanda/pile-10k",
@@ -431,6 +438,7 @@ class AutoRoundAdam(AdamCompressor):
super().__init__(
model=model,
tokenizer=tokenizer,
+ platform=platform,
scheme=scheme,
layer_config=layer_config,
batch_size=batch_size,
@@ -455,6 +463,7 @@ class AutoRoundMLLM(MLLMCompressor):
Args:
model: The PyTorch model to be quantized.
tokenizer: An optional tokenizer for processing input data.
+ platform (str): The platform to load pretrained moded, options: ["hf", "model_scope"]
processor: Any multi-modal model will require an object to encode or
decode the data that groups several modalities (among text, vision and audio).
image_processor: Image processor for special model like llava.
@@ -513,6 +522,7 @@ class AutoRoundMLLM(MLLMCompressor):
self,
model: Union[torch.nn.Module, str],
tokenizer=None,
+ platform: str = "hf",
processor=None,
image_processor=None,
scheme: Union[str, dict, QuantizationScheme] = "W4A16",
@@ -533,6 +543,7 @@ class AutoRoundMLLM(MLLMCompressor):
super().__init__(
model=model,
tokenizer=tokenizer,
+ platform=platform,
processor=processor,
image_processor=image_processor,
scheme=scheme,
@@ -559,6 +570,7 @@ class AutoRoundDiffusion(DiffusionCompressor):
Args:
model: The PyTorch model to be quantized.
tokenizer: An optional tokenizer for processing input data, is not used for diffusion models.
+ platform (str): The platform to load pretrained moded, options: ["hf", "model_scope"]
guidance_scale (float): Control how much the image generation process follows the text prompt.
The more it is, the more closely it follows the prompt (default is 7.5).
num_inference_steps (int): The reference number of denoising steps (default is 50).
diff --git a/auto_round/compressors/adam.py b/auto_round/compressors/adam.py
index 4606eab3a..fb79cf39a 100644
--- a/auto_round/compressors/adam.py
+++ b/auto_round/compressors/adam.py
@@ -27,6 +27,7 @@ class AdamCompressor(BaseCompressor):
Args:
model: The PyTorch model to be quantized.
tokenizer: An optional tokenizer for processing input data.
+ platform (str): The platform to load pretrained moded, options: ["hf", "model_scope"]
scheme (str| dict | QuantizationScheme ): A preset scheme that defines the quantization configurations
bits (int): Number of bits for quantization (default is 4).
group_size (int): Size of the quantization group (default is 128).
@@ -86,6 +87,7 @@ class AdamCompressor(BaseCompressor):
self,
model: Union[torch.nn.Module, str],
tokenizer=None,
+ platform="hf",
scheme: Union[str, dict, QuantizationScheme] = "W4A16",
layer_config: dict[str, Union[str, dict, QuantizationScheme]] = None,
dataset: Union[str, list, tuple, torch.utils.data.DataLoader] = "NeelNanda/pile-10k",
@@ -104,6 +106,7 @@ class AdamCompressor(BaseCompressor):
super(AdamCompressor, self).__init__(
model=model,
tokenizer=tokenizer,
+ platform=platform,
scheme=scheme,
layer_config=layer_config,
batch_size=batch_size,
diff --git a/auto_round/compressors/base.py b/auto_round/compressors/base.py
index fe0939c1f..602b566a2 100644
--- a/auto_round/compressors/base.py
+++ b/auto_round/compressors/base.py
@@ -30,6 +30,7 @@ from torch import autocast
from tqdm import tqdm
from transformers import set_seed
+from auto_round import envs
from auto_round.auto_scheme.gen_auto_scheme import AutoScheme
from auto_round.compressors.utils import (
block_forward,
@@ -105,6 +106,7 @@ class BaseCompressor(object):
Attributes:
model (torch.nn.Module): The loaded PyTorch model in eval mode.
tokenizer: Tokenizer used to prepare input text for calibration/tuning.
+ platform (str): The platform to load pretrained moded, options: ["hf", "model_scope"]
bits (int): Weight quantization bits.
group_size (int): Per-group size for weight quantization.
sym (bool): Whether to use symmetric weight quantization.
@@ -129,6 +131,7 @@ class BaseCompressor(object):
self,
model: Union[torch.nn.Module, str],
tokenizer=None,
+ platform="hf",
scheme: Union[str, dict, QuantizationScheme, AutoScheme] = "W4A16",
layer_config: dict[str, Union[str, dict, QuantizationScheme]] = None,
dataset: Union[str, list, tuple, torch.utils.data.DataLoader] = "NeelNanda/pile-10k",
@@ -228,6 +231,10 @@ class BaseCompressor(object):
device = kwargs.pop("device", None)
# Scale factor for RAM usage per parameter.
mem_per_param_scale = kwargs.pop("mem_per_param_scale", None)
+
+ if envs.AR_USE_MODELSCOPE:
+ platform = "model_scope"
+ self.platform = platform
self.quant_lm_head = kwargs.pop("quant_lm_head", False)
self.mllm = kwargs.pop("mllm") if "mllm" in kwargs else False
self.diffusion = kwargs.pop("diffusion") if "diffusion" in kwargs else False
@@ -259,6 +266,7 @@ class BaseCompressor(object):
if isinstance(model, str):
model, tokenizer = llm_load_model(
model,
+ platform=platform,
device="cpu", # always load cpu first
)
elif tokenizer is None and not self.diffusion and iters > 0:
diff --git a/auto_round/compressors/diffusion/compressor.py b/auto_round/compressors/diffusion/compressor.py
index cb69e6a88..904767602 100644
--- a/auto_round/compressors/diffusion/compressor.py
+++ b/auto_round/compressors/diffusion/compressor.py
@@ -47,6 +47,7 @@ class DiffusionCompressor(BaseCompressor):
Args:
model: The PyTorch model to be quantized.
tokenizer: An optional tokenizer for processing input data, is not used for diffusion models.
+ platform (str): The platform to load pretrained moded, options: ["hf", "model_scope"]
guidance_scale (float): Control how much the image generation process follows the text prompt.
The more it is, the more closely it follows the prompt (default is 7.5).
num_inference_steps (int): The reference number of denoising steps (default is 50).
@@ -81,6 +82,7 @@ class DiffusionCompressor(BaseCompressor):
self,
model: Union[object, str],
tokenizer=None,
+ platform: str = "hf",
guidance_scale: float = 7.5,
num_inference_steps: int = 50,
generator_seed: int = None,
@@ -110,7 +112,7 @@ class DiffusionCompressor(BaseCompressor):
self._set_device(device_map)
if isinstance(model, str):
- pipe, model = diffusion_load_model(model, device=self.device)
+ pipe, model = diffusion_load_model(model, platform=platform, device=self.device)
elif isinstance(model, pipeline_utils.DiffusionPipeline):
pipe = model
model = pipe.transformer
@@ -145,6 +147,7 @@ class DiffusionCompressor(BaseCompressor):
super(DiffusionCompressor, self).__init__(
model=model,
tokenizer=None,
+ platform=platform,
scheme=scheme,
layer_config=layer_config,
dataset=dataset,
diff --git a/auto_round/compressors/mllm/compressor.py b/auto_round/compressors/mllm/compressor.py
index b5586a4bb..2cfa457b7 100644
--- a/auto_round/compressors/mllm/compressor.py
+++ b/auto_round/compressors/mllm/compressor.py
@@ -87,6 +87,7 @@ class MLLMCompressor(BaseCompressor):
Args:
model: The PyTorch model to be quantized.
tokenizer: An optional tokenizer for processing input data.
+ platform (str): The platform to load pretrained moded, options: ["hf", "model_scope"]
processor: Any multi-modal model will require an object to encode or
decode the data that groups several modalities (among text, vision and audio).
image_processor: Image processor for special model like llava.
@@ -145,6 +146,7 @@ class MLLMCompressor(BaseCompressor):
self,
model: Union[torch.nn.Module, str],
tokenizer=None,
+ platform: str = "hf",
processor=None,
image_processor=None,
scheme: Union[str, dict, QuantizationScheme] = "W4A16",
@@ -171,7 +173,7 @@ class MLLMCompressor(BaseCompressor):
self._set_device(device_map)
if isinstance(model, str):
- model, processor, tokenizer, image_processor = mllm_load_model(model, device=self.device)
+ model, processor, tokenizer, image_processor = mllm_load_model(model, platform=platform, device=self.device)
self.model = model
quant_nontext_module = self._check_quant_nontext(layer_config, quant_nontext_module)
@@ -258,6 +260,7 @@ class MLLMCompressor(BaseCompressor):
super(MLLMCompressor, self).__init__(
model=model,
tokenizer=tokenizer,
+ platform=platform,
scheme=scheme,
layer_config=layer_config,
dataset=dataset,
@@ -374,6 +377,7 @@ class MLLMCompressor(BaseCompressor):
continue
try:
if isinstance(data_new, torch.Tensor):
+ data_new = data_new.to(self.model.device)
self.model(data_new)
elif isinstance(data_new, tuple) or isinstance(data_new, list):
self.model(*data_new)
diff --git a/auto_round/compressors/utils.py b/auto_round/compressors/utils.py
index 8a8fe602c..79e8b7133 100644
--- a/auto_round/compressors/utils.py
+++ b/auto_round/compressors/utils.py
@@ -480,7 +480,7 @@ def gguf_args_check(args_or_ar, formats: list[str] = None, model_type=ModelType.
from auto_round.export.export_to_gguf.convert import download_convert_file
from auto_round.logger import logger
- from auto_round.utils.model import download_hf_model, get_gguf_architecture
+ from auto_round.utils.model import download_or_get_path, get_gguf_architecture
formats = sorted(formats, key=lambda x: len(x))
export_gguf = False
@@ -505,7 +505,7 @@ def gguf_args_check(args_or_ar, formats: list[str] = None, model_type=ModelType.
else:
model_path = args_or_ar.model.name_or_path
if not os.path.isdir(model_path):
- model_path = download_hf_model(model_path)
+ model_path = download_or_get_path(model_path, args_or_ar.platform)
model_architecture = get_gguf_architecture(model_path, model_type=ModelType.TEXT)
if model_architecture not in ModelBase._model_classes[ModelType.TEXT]:
logger.warning(
@@ -539,7 +539,7 @@ def gguf_args_check(args_or_ar, formats: list[str] = None, model_type=ModelType.
else:
model_path = args_or_ar.model.name_or_path
if not os.path.isdir(model_path):
- model_path = download_hf_model(model_path)
+ model_path = download_or_get_path(model_path, args_or_ar.platform)
model_architecture = get_gguf_architecture(model_path, model_type=ModelType.TEXT)
if model_architecture not in ModelBase._model_classes[ModelType.TEXT]:
logger.error(f"Model {model_architecture} is not supported to export gguf format.")
diff --git a/auto_round/envs.py b/auto_round/envs.py
index ae0d79f1d..3d9b0dfa3 100644
--- a/auto_round/envs.py
+++ b/auto_round/envs.py
@@ -18,10 +18,12 @@ from typing import TYPE_CHECKING, Any, Callable, Optional
if TYPE_CHECKING:
AR_LOG_LEVEL: str = "INFO"
+ AR_USE_MODELSCOPE: bool = "False"
environment_variables: dict[str, Callable[[], Any]] = {
# this is used for configuring the default logging level
"AR_LOG_LEVEL": lambda: os.getenv("AR_LOG_LEVEL", "INFO").upper(),
+ "AR_USE_MODELSCOPE": lambda: os.getenv("AR_USE_MODELSCOPE", "False").lower() in ["1", "true"],
}
@@ -41,3 +43,30 @@ def is_set(name: str):
if name in environment_variables:
return name in os.environ
raise AttributeError(f"module {__name__!r} has no attribute {name!r}")
+
+
+def set_config(**kwargs):
+ """
+ Set configuration values for environment variables.
+
+ Args:
+ **kwargs: Keyword arguments where keys are environment variable names
+ and values are the desired values to set.
+
+ Example:
+ set_config(AR_LOG_LEVEL="DEBUG", AR_USE_MODELSCOPE=True)
+ """
+ for key, value in kwargs.items():
+ if key in environment_variables:
+ # Convert value to appropriate string format
+ if key == "AR_USE_MODELSCOPE":
+ # Handle boolean values for AR_USE_MODELSCOPE
+ str_value = "true" if value in [True, "True", "true", "1", 1] else "false"
+ else:
+ # For other variables, convert to string
+ str_value = str(value)
+
+ # Set the environment variable
+ os.environ[key] = str_value
+ else:
+ raise AttributeError(f"module {__name__!r} has no attribute {key!r}")
diff --git a/auto_round/export/export_to_gguf/convert.py b/auto_round/export/export_to_gguf/convert.py
index 03981db61..3ac31932d 100644
--- a/auto_round/export/export_to_gguf/convert.py
+++ b/auto_round/export/export_to_gguf/convert.py
@@ -167,11 +167,11 @@ def get_tensors(cls) -> Iterator[tuple[str, Tensor]]:
from safetensors import safe_open
from auto_round.export.export_to_gguf.special_handle import get_tensor_from_file
- from auto_round.utils import download_hf_model
+ from auto_round.utils import download_or_get_path
dir_path = cls.model.name_or_path
if not os.path.isdir(dir_path):
- dir_path = download_hf_model(dir_path)
+ dir_path = download_or_get_path(dir_path)
INDEX_FILE = "model.safetensors.index.json"
if INDEX_FILE in os.listdir(dir_path):
with open(os.path.join(dir_path, INDEX_FILE)) as f:
diff --git a/auto_round/export/export_to_gguf/export.py b/auto_round/export/export_to_gguf/export.py
index 526056263..8633a2a50 100644
--- a/auto_round/export/export_to_gguf/export.py
+++ b/auto_round/export/export_to_gguf/export.py
@@ -28,7 +28,7 @@ from auto_round.utils import (
LazyImport,
check_to_quantized,
clear_memory,
- download_hf_model,
+ download_or_get_path,
flatten_list,
get_block_names,
get_gguf_architecture,
@@ -77,7 +77,7 @@ def create_model_class(
tmp_work_dir = model.name_or_path
os.makedirs(output_dir, exist_ok=True)
if not os.path.isdir(tmp_work_dir):
- tmp_work_dir = download_hf_model(tmp_work_dir)
+ tmp_work_dir = download_or_get_path(tmp_work_dir)
with torch.inference_mode():
model_architecture = get_gguf_architecture(tmp_work_dir, model_type=model_type)
try:
diff --git a/auto_round/export/export_to_gguf/special_handle.py b/auto_round/export/export_to_gguf/special_handle.py
index f8a463e9b..cb15af7a1 100644
--- a/auto_round/export/export_to_gguf/special_handle.py
+++ b/auto_round/export/export_to_gguf/special_handle.py
@@ -20,7 +20,7 @@ import torch
from safetensors import safe_open
from torch import Tensor
-from auto_round.utils import download_hf_model
+from auto_round.utils import download_or_get_path
def handle_special_model(cls, model_architecture):
@@ -32,7 +32,7 @@ def handle_special_model(cls, model_architecture):
def get_tensor_from_file(dir_path, tensor_name):
if not os.path.isdir(dir_path):
- dir_path = download_hf_model(dir_path)
+ dir_path = download_or_get_path(dir_path)
INDEX_FILE = "model.safetensors.index.json"
# get filename
if INDEX_FILE in os.listdir(dir_path):
diff --git a/auto_round/utils/model.py b/auto_round/utils/model.py
index 7a1c66de8..427ec7f9c 100644
--- a/auto_round/utils/model.py
+++ b/auto_round/utils/model.py
@@ -23,6 +23,7 @@ from typing import Union
import torch
import transformers
+from auto_round import envs
from auto_round.export.export_to_gguf.config import ModelType
from auto_round.logger import logger
from auto_round.schemes import QuantizationScheme
@@ -148,6 +149,39 @@ def check_start_with_block_name(name: str, block_name_to_quantize: list):
return False
+def download_or_get_path(repo_id: str, platform: str = None) -> str:
+ from auto_round import envs
+
+ if platform is None:
+ if envs.AR_USE_MODELSCOPE:
+ platform = "model_scope"
+ else:
+ platform = "hf"
+
+ if platform == "model_scope":
+ return download_modelscope_model(repo_id)
+ else:
+ return download_hf_model(repo_id)
+
+
+def download_modelscope_model(repo_id: str, local_dir: str = None, cache_dir: str = None):
+ from modelscope.utils.file_utils import get_modelscope_cache_dir # pylint: disable=E0401
+
+ system_cache = cache_dir if cache_dir is not None else get_modelscope_cache_dir()
+ if local_dir:
+ directory = os.path.abspath(local_dir)
+ elif cache_dir:
+ directory = os.path.join(system_cache, *repo_id.split("/"))
+ else:
+ directory = os.path.join(system_cache, "models", *repo_id.split("/"))
+ if os.path.exists(directory):
+ return directory
+ else:
+ from modelscope.hub.snapshot_download import snapshot_download # pylint: disable=E0401
+
+ return snapshot_download(repo_id)
+
+
def download_hf_model(repo_id, cache_dir=None, repo_type=None, revision=None):
"""Download hugging face model from hf hub."""
from huggingface_hub.constants import DEFAULT_REVISION, HUGGINGFACE_HUB_CACHE
@@ -180,13 +214,23 @@ def download_hf_model(repo_id, cache_dir=None, repo_type=None, revision=None):
def llm_load_model(
- pretrained_model_name_or_path,
- trust_remote_code=True,
- model_dtype=None,
- device="cpu",
+ pretrained_model_name_or_path: str,
+ platform: str = "hf",
+ trust_remote_code: bool = True,
+ model_dtype: str = None,
+ device: str = "cpu",
**kwargs,
):
- from transformers import AutoModel, AutoModelForCausalLM, AutoTokenizer
+ assert platform.lower() in [
+ "hf",
+ "model_scope",
+ ], "current only support hf or model_scope platform to load pretrained model."
+ if platform.lower() == "model_scope" and not envs.AR_USE_MODELSCOPE:
+ envs.set_config(AR_USE_MODELSCOPE=True)
+ if platform == "model_scope":
+ from modelscope import AutoModel, AutoModelForCausalLM, AutoTokenizer # pylint: disable=E0401
+ else:
+ from transformers import AutoModel, AutoModelForCausalLM, AutoTokenizer
from auto_round.utils.device import (
_use_hpu_compile_mode,
@@ -254,16 +298,32 @@ def llm_load_model(
def mllm_load_model(
- pretrained_model_name_or_path,
- device="cpu",
- torch_dtype="auto",
- use_auto_mapping=True,
- trust_remote_code=True,
- model_dtype=None,
+ pretrained_model_name_or_path: str,
+ platform: str = "hf",
+ device: str = "cpu",
+ torch_dtype: str = "auto",
+ use_auto_mapping: bool = True,
+ trust_remote_code: bool = True,
+ model_dtype: str = None,
**kwargs,
):
- import transformers
- from transformers import AutoModel, AutoModelForCausalLM, AutoProcessor, AutoTokenizer
+ assert platform.lower() in [
+ "hf",
+ "model_scope",
+ ], "current only support hf or model_scope platform to load pretrained model."
+ if platform.lower() == "model_scope" and not envs.AR_USE_MODELSCOPE:
+ envs.set_config(AR_USE_MODELSCOPE=True)
+
+ if platform == "model_scope":
+ import modelscope # pylint: disable=E0401
+ from modelscope import AutoModel, AutoModelForCausalLM, AutoProcessor, AutoTokenizer # pylint: disable=E0401
+
+ base_lib = modelscope
+ else:
+ import transformers
+ from transformers import AutoModel, AutoModelForCausalLM, AutoProcessor, AutoTokenizer
+
+ base_lib = transformers
from auto_round.utils.device import get_device_and_parallelism, set_fake_cuda_device_capability
@@ -322,11 +382,11 @@ def mllm_load_model(
)
else:
if architectures.endswith("Model") and hasattr(
- transformers, n := architectures.replace("Model", "ForConditionalGeneration")
+ base_lib, n := architectures.replace("Model", "ForConditionalGeneration")
):
- cls = getattr(transformers, n)
- elif hasattr(transformers, architectures):
- cls = getattr(transformers, architectures)
+ cls = getattr(base_lib, n)
+ elif hasattr(base_lib, architectures):
+ cls = getattr(base_lib, architectures)
else:
cls = AutoModelForCausalLM
try:
@@ -365,7 +425,10 @@ def mllm_load_model(
pretrained_model_name_or_path, trust_remote_code=trust_remote_code
)
try:
- from transformers import AutoImageProcessor
+ if platform == "model_scope":
+ from modelscope import AutoImageProcessor # pylint: disable=E0401
+ else:
+ from transformers import AutoImageProcessor
image_processor = AutoImageProcessor.from_pretrained(
pretrained_model_name_or_path, trust_remote_code=trust_remote_code
@@ -382,6 +445,7 @@ def mllm_load_model(
def diffusion_load_model(
pretrained_model_name_or_path: str,
+ platform: str = "hf",
device: Union[str, torch.device] = "cpu",
torch_dtype: Union[str, torch.dtype] = "auto",
use_auto_mapping: bool = False,
@@ -392,6 +456,11 @@ def diffusion_load_model(
from auto_round.utils.common import LazyImport
from auto_round.utils.device import get_device_and_parallelism
+ if platform != "hf":
+ raise NotImplementedError(
+ f"auto_round current only support hf as platform for diffusion model, but get {platform}"
+ )
+
device_str, use_auto_mapping = get_device_and_parallelism(device)
torch_dtype = "auto"
if device_str is not None and "hpu" in device_str:
@@ -425,7 +494,7 @@ def is_pure_text_model(model):
return True
-def is_mllm_model(model_or_path: Union[str, torch.nn.Module]):
+def is_mllm_model(model_or_path: Union[str, torch.nn.Module], platform: str = None):
MM_KEYS = [
"multi_modal_projector",
"vision_tower",
@@ -446,7 +515,7 @@ def is_mllm_model(model_or_path: Union[str, torch.nn.Module]):
model_path = model_or_path if isinstance(model_or_path, str) else model_or_path.name_or_path
if not os.path.isdir(model_path):
- model_path = download_hf_model(model_path)
+ model_path = download_or_get_path(model_path, platform=platform)
if isinstance(model_path, str):
if os.path.exists(os.path.join(model_path, "preprocessor_config.json")):
diff --git a/test/test_cpu/test_model_scope.py b/test/test_cpu/test_model_scope.py
new file mode 100644
index 000000000..0663bcf12
--- /dev/null
+++ b/test/test_cpu/test_model_scope.py
@@ -0,0 +1,51 @@
+import copy
+import shutil
+import sys
+import unittest
+
+sys.path.insert(0, "../..")
+
+import torch
+
+from auto_round import AutoRound
+
+
+class LLMDataLoader:
+ def __init__(self):
+ self.batch_size = 1
+
+ def __iter__(self):
+ for i in range(3):
+ yield torch.ones([1, 10], dtype=torch.long)
+
+
+class TestModelScope(unittest.TestCase):
+ @classmethod
+ def setUpClass(self):
+ self.saved_path = "./saved"
+ self.dataset = LLMDataLoader()
+
+ @classmethod
+ def tearDownClass(self):
+ shutil.rmtree("./saved", ignore_errors=True)
+ shutil.rmtree("runs", ignore_errors=True)
+
+ return super().tearDownClass()
+
+ def test_llm(self):
+ model_name = "Qwen/Qwen2.5-0.5B-Instruct"
+ autoround = AutoRound(
+ model_name, platform="model_scope", scheme="w4a16", iters=0, seqlen=2, dataset=self.dataset
+ )
+ autoround.quantize_and_save()
+
+ def test_mllm(self):
+ model_name = "Qwen/Qwen2-VL-2B-Instruct"
+ autoround = AutoRound(
+ model_name, platform="model_scope", scheme="w4a16", iters=0, seqlen=2, dataset=self.dataset, batch_size=2
+ )
+ autoround.quantize_and_save(self.saved_path)
+
+
+if __name__ == "__main__":
+ unittest.main()