Skip to content

PTQ-Level Efficiency, QAT-Level Accuracy | 训练后量化的效率,量化感知训练的精度

api support for fp8 model and mllm api support load from str (#752)

Heng Guofcbcaa48de52
diff --git a/README.md b/README.md
index a23785a0a..819167905 100644
--- a/README.md
+++ b/README.md
@@ -300,7 +300,7 @@ for output in outputs:
### Transformers (CPU/Intel GPU/Gaudi/CUDA)
-AutoRound support 10+ backends an automatically selects the best available backend based on the installed libraries and prompts the user to
+AutoRound support 10+ backends and automatically selects the best available backend based on the installed libraries and prompts the user to
install additional libraries when a better backend is found.
**Please avoid manually moving the quantized model to a different device** (e.g., model.to('cpu')) during inference, as
diff --git a/auto_round/autoround.py b/auto_round/autoround.py
index 15ee357b5..aedcd860d 100644
--- a/auto_round/autoround.py
+++ b/auto_round/autoround.py
@@ -40,6 +40,7 @@ from auto_round.utils import (
CpuInfo,
_gguf_args_check,
block_forward,
+ check_and_mark_fp8_model,
check_is_cpu,
check_need_act_calibration,
check_seqlen_compatible,
@@ -223,11 +224,31 @@ class AutoRound(object):
os.environ["CUBLAS_WORKSPACE_CONFIG"] = ":4096:8"
torch.use_deterministic_algorithms(True, warn_only=False)
+ # Model related
+ self.quantized = False
+ if isinstance(model, str):
+ model, tokenizer, low_cpu_mem_usage = llm_load_model(
+ model, device=device, low_cpu_mem_mode=low_cpu_mem_usage
+ )
+ elif tokenizer is None and iters > 0:
+ raise ValueError("A tokenizer must be set for non-str model input")
+ self.low_cpu_mem_usage = bool(low_cpu_mem_usage)
+ if unsupport_meta_device(model):
+ raise RuntimeError(
+ "AutoRound does not support parameters on meta device. "
+ "Please use more GPUs by setting `--device 0,1,2,3` or just place the model on CPU."
+ )
+ check_and_mark_fp8_model(model)
+ model = _handle_moe_model(model)
+ self.model = model.eval()
+ self.tokenizer = tokenizer
+ self.shared_cache_keys = get_shared_keys(self.model)
+
# Set quantization configs
self.bits = bits
self.data_type = data_type
tmp_bits = infer_bits_by_data_type(self.data_type)
- if tmp_bits < 16 and tmp_bits != bits:
+ if tmp_bits is not None and tmp_bits < 16 and tmp_bits != bits:
logger.warning(f"'data_type' do not match the specified 'bits' setting. Resetting 'bits' to {tmp_bits}.")
self.bits = tmp_bits
self.group_size = group_size
@@ -248,7 +269,7 @@ class AutoRound(object):
else:
self.act_data_type = "float"
tmp_act_bits = infer_bits_by_data_type(self.act_data_type)
- if tmp_act_bits < 16 and tmp_act_bits != self.act_bits:
+ if tmp_act_bits is not None and tmp_act_bits < 16 and tmp_act_bits != self.act_bits:
self.act_bits = tmp_act_bits
logger.warning(
f"`act_data_type` do not"
@@ -1711,14 +1732,14 @@ class AutoRound(object):
layer_config[n]["fixed_by_user"] = True
# Check dtype and act_dtype
tmp_bits = infer_bits_by_data_type(layer_config[n]["data_type"])
- if tmp_bits != layer_config[n]["bits"]:
+ if tmp_bits is not None and tmp_bits != layer_config[n]["bits"]:
logger.warning(
f"'data_type' do not match the specified 'bits' setting for {n}."
f" Resetting 'bits' to {tmp_bits}."
)
layer_config[n]["bits"] = tmp_bits
tmp_act_bits = infer_bits_by_data_type(layer_config[n]["act_data_type"])
- if tmp_act_bits != layer_config[n]["act_bits"]:
+ if tmp_act_bits is not None and tmp_act_bits != layer_config[n]["act_bits"]:
logger.warning(
f"'act_data_type' do not match the specified 'act_bits' setting for {n}."
f" Resetting 'act_bits' to {tmp_act_bits}."
diff --git a/auto_round/mllm/autoround_mllm.py b/auto_round/mllm/autoround_mllm.py
index 67b64e113..90532aa21 100644
--- a/auto_round/mllm/autoround_mllm.py
+++ b/auto_round/mllm/autoround_mllm.py
@@ -18,15 +18,16 @@ from typing import Union
import torch
from tqdm import tqdm
+from auto_round.autoround import AutoRound
+from auto_round.low_cpu_mem.utils import get_layers_before_block
+from auto_round.mllm.mllm_dataset import get_mllm_dataloader
+from auto_round.mllm.template import Template, get_template
from auto_round.special_model_handler import (
NOT_SUPPORT_ONLY_TEXT_MODELS,
SUPPORT_ONLY_TEXT_MODELS,
_handle_special_model,
)
-
-from ..autoround import AutoRound
-from ..low_cpu_mem.utils import get_layers_before_block
-from ..utils import (
+from auto_round.utils import (
check_to_quantized,
clear_memory,
detect_device,
@@ -38,8 +39,6 @@ from ..utils import (
to_device,
to_dtype,
)
-from .mllm_dataset import get_mllm_dataloader
-from .template import Template, get_template
def _only_text_test(model, tokenizer, device, model_type):
@@ -129,8 +128,8 @@ class AutoRoundMLLM(AutoRound):
def __init__(
self,
- model: torch.nn.Module,
- tokenizer,
+ model: Union[torch.nn.Module, str],
+ tokenizer=None,
processor=None,
image_processor=None,
bits: int = 4,
@@ -139,7 +138,7 @@ class AutoRoundMLLM(AutoRound):
layer_config: dict = None,
batch_size: int = 8,
amp: bool = True,
- device: str = None,
+ device: Union[str, torch.device, int] = 0,
lr_scheduler=None,
dataset: Union[str, list, tuple, torch.utils.data.DataLoader] = None,
extra_data_dir: str = None,
@@ -173,6 +172,9 @@ class AutoRoundMLLM(AutoRound):
model_kwargs: dict = None,
**kwargs,
):
+ if isinstance(model, str):
+ model, processor, tokenizer, image_processor = mllm_load_model(model, device=device)
+
quant_nontext_module = self._check_quant_nontext(layer_config, quant_nontext_module)
all_blocks = get_block_names(model, quant_nontext_module)
self.quant_block_list = find_matching_blocks(model, all_blocks, to_quant_block_names)
diff --git a/auto_round/script/llm.py b/auto_round/script/llm.py
index 76177e5bb..b24bc5bd6 100644
--- a/auto_round/script/llm.py
+++ b/auto_round/script/llm.py
@@ -628,7 +628,8 @@ def tune(args):
import time
eval_model_dtype = get_model_dtype(args.eval_model_dtype, "auto")
- if args.act_bits <= 8 or infer_bits_by_data_type(args.act_data_type) <= 8 or eval_gguf_model:
+ tmp_act_bits = infer_bits_by_data_type(args.act_data_type)
+ if tmp_act_bits is not None and (args.act_bits <= 8 or tmp_act_bits <= 8 or eval_gguf_model):
if eval_gguf_model:
# for file in os.listdir(eval_folder):
# gguf_file = file
diff --git a/auto_round/utils.py b/auto_round/utils.py
index 635648299..40fbcc5de 100644
--- a/auto_round/utils.py
+++ b/auto_round/utils.py
@@ -97,7 +97,7 @@ def infer_bits_by_data_type(data_type: str):
return int(suc_2str)
if str.isdigit(data_type[len(supported_dtype)]):
return int(data_type[len(supported_dtype)])
- return 16
+ return None
@lru_cache(None)
@@ -1116,15 +1116,15 @@ def check_awq_gemm_compatibility(model, bits, group_size, sym, layer_configs=Non
return True, ""
-def get_device_and_parallelism(device):
+def get_device_and_parallelism(device: Union[str, torch.device, int]) -> Tuple[str, bool]:
from auto_round.utils import detect_device
if isinstance(device, str):
devices = device.replace(" ", "").split(",")
elif isinstance(device, int):
- devices = str(device)
+ devices = [str(device)]
else:
- devices = device
+ devices = [device]
if all(s.isdigit() for s in devices) and len(devices) > 1 and torch.cuda.is_available():
device = "cuda"
parallelism = True
diff --git a/test/test_cpu/test_mllm.py b/test/test_cpu/test_mllm.py
index cd9fb25b4..0fd102407 100644
--- a/test/test_cpu/test_mllm.py
+++ b/test/test_cpu/test_mllm.py
@@ -41,16 +41,9 @@ class TestAutoRoundMLLM(unittest.TestCase):
return super().tearDownClass()
def test_tune(self):
- tokenizer = AutoTokenizer.from_pretrained(self.model_name)
- processor = AutoProcessor.from_pretrained(self.model_name, trust_remote_code=True)
- model = Qwen2VLForConditionalGeneration.from_pretrained(
- self.model_name, trust_remote_code=True, device_map="auto"
- )
bits, group_size = 4, 128
autoround = AutoRoundMLLM(
- model,
- tokenizer,
- processor=processor,
+ model=self.model_name,
bits=bits,
group_size=group_size,
nsamples=1,