Skip to content

PTQ-Level Efficiency, QAT-Level Accuracy | 训练后量化的效率,量化感知训练的精度

patch 1 for mllm (#298)

n1ck-guo73861bb76bd8
diff --git a/auto_round/autoround.py b/auto_round/autoround.py
index 2401a732b..621e8b877 100644
--- a/auto_round/autoround.py
+++ b/auto_round/autoround.py
@@ -55,7 +55,6 @@ from .utils import (
)
from .low_cpu_mem.utils import get_layers_before_block
-
class AutoRound(object):
"""For more information, please refer to Cheng, Wenhua, et al. "Optimize weight rounding via signed gradient descent
for the quantization of llms." arXiv preprint arXiv:2309.05516 (2023).
diff --git a/auto_round/mllm/autoround_mllm.py b/auto_round/mllm/autoround_mllm.py
index 2aebd7565..fa20a04c0 100644
--- a/auto_round/mllm/autoround_mllm.py
+++ b/auto_round/mllm/autoround_mllm.py
@@ -13,6 +13,7 @@
# limitations under the License.
from typing import Optional, Union
+from tqdm import tqdm
import torch
@@ -27,6 +28,8 @@ from .template import get_template, Template
from .mllm_dataset import get_mllm_dataloader
from ..low_cpu_mem.utils import get_layers_before_block
from ..special_model_handler import check_mllm_model_batch
+
+
class AutoRoundMLLM(AutoRound):
"""Class for automatic rounding-based quantization with MLLMs.
@@ -74,6 +77,7 @@ class AutoRoundMLLM(AutoRound):
self,
model,
tokenizer,
+ image_processor = None,
bits: int = 4,
group_size: int = 128,
sym: bool = False,
@@ -83,7 +87,7 @@ class AutoRoundMLLM(AutoRound):
device: str = None,
lr_scheduler=None,
dataset: Union[str, list, tuple, torch.utils.data.DataLoader] = None,
- extra_data_dir: Union[str, torch.utils.data.DataLoader] = None,
+ extra_data_dir: str = None,
template: Union[str, Template] = None,
quant_nontext_module: bool = False,
enable_quanted_input: bool = True,
@@ -115,13 +119,12 @@ class AutoRoundMLLM(AutoRound):
quant_block_list = get_multimodal_block_names(model, quant_nontext_module)
self.extra_data_dir = extra_data_dir
self.quant_nontext_module = quant_nontext_module
- self.template = template
- if self.template is None:
- self.template = get_template(model.config.model_type)
+ self.image_processor = image_processor
+ self.template = template if template is not None else model.config.model_type
+ self.template = get_template(
+ self.template, model=model, tokenizer=tokenizer, image_processor=image_processor)
assert dataset is not None, "dataset should not be None"
batch_size, gradient_accumulate_steps = check_mllm_model_batch(model, batch_size, gradient_accumulate_steps)
- if isinstance(dataset, str):
- dataset = get_mllm_dataloader(self.template, model, tokenizer, dataset, extra_data_dir, seqlen, batch_size)
super(AutoRoundMLLM, self).__init__(
model=model,
@@ -177,7 +180,16 @@ class AutoRoundMLLM(AutoRound):
if isinstance(self.dataset, str):
dataset = self.dataset.replace(" ", "")
self.dataloader = get_mllm_dataloader(
- self.template, self.model, self.tokenizer, dataset, self.extra_data_dir, self.seqlen, bs)
+ template=self.template,
+ model=self.model,
+ tokenizer=self.tokenizer,
+ image_processor=self.image_processor,
+ dataset=dataset,
+ extra_data_dir=self.extra_data_dir,
+ seqlen=self.seqlen,
+ bs=bs,
+ seed=self.seed
+ )
else:
self.dataloader = self.dataset
total_cnt = 0
@@ -187,7 +199,7 @@ class AutoRoundMLLM(AutoRound):
for n, m in embed_layers:
m = m.to(self.device)
- for data in self.dataloader:
+ for data in tqdm(self.dataloader, desc="calib", total=nsamples-1):
if data is None:
continue
if isinstance(data, torch.Tensor):
@@ -199,8 +211,6 @@ class AutoRoundMLLM(AutoRound):
exit()
# data = self.template._encode(data)
data = self.template.processor.get_input(
- self.model,
- self.tokenizer,
text=data,
images=None,
max_length=self.seqlen,
@@ -215,14 +225,9 @@ class AutoRoundMLLM(AutoRound):
if isinstance(text, dict):
text = [text]
input_text = self.template._encode(text)
- image = None
- if "image" in data:
- image = self.template.processor.image_processor(data["image"])
data = self.template.processor.get_input(
- self.model,
- self.tokenizer,
text=input_text,
- images=image,
+ images=data["image"],
max_length=self.seqlen,
squeeze=False,
)
@@ -243,9 +248,6 @@ class AutoRoundMLLM(AutoRound):
data_new[key] = to_dtype(data_new[key], self.model.dtype)
input_ids = data_new["input_ids"]
- if input_ids.shape[-1] < self.seqlen:
- continue
-
try:
if isinstance(data_new, torch.Tensor):
self.model(data_new)
diff --git a/auto_round/mllm/eval.py b/auto_round/mllm/eval.py
index ec1b2b118..58f8ba182 100644
--- a/auto_round/mllm/eval.py
+++ b/auto_round/mllm/eval.py
@@ -27,6 +27,7 @@
# limitations under the License.
import os
+import time
import json
from functools import partial
@@ -84,6 +85,10 @@ def mllm_eval(
model = None
if data_store_dir is not None:
+ if not os.path.exists(data_store_dir):
+ oldmask = os.umask(000)
+ os.makedirs(data_store_dir, mode=0o777)
+ os.umask(oldmask)
os.environ['LMUData'] = data_store_dir
model_name = pretrained_model_name_or_path
@@ -119,7 +124,10 @@ def mllm_eval(
pred_root = os.path.join(work_dir, model_name)
os.makedirs(pred_root, exist_ok=True)
+ st = time.time()
+ rt_file = open(f'{pred_root}/{model_name}_eval_cost.txt', 'w')
for dataset_name in dataset:
+ task_st = time.time()
try:
dataset_kwargs = {}
if dataset_name in ['MMLongBench_DOC', 'DUDE', 'DUDE_MINI', 'SLIDEVQA', 'SLIDEVQA_MINI']:
@@ -269,8 +277,10 @@ def mllm_eval(
logger.info('\n' + tabulate.tabulate(eval_results))
except:
logger.info(eval_results.to_string())
-
+ rt_file.write('%s cost: %.4fs\n' % (dataset_name, time.time() - task_st))
except Exception as e:
logger.exception(f'Model {model_name} x Dataset {dataset_name} combination failed: {e}, '
'skipping this combination.')
continue
+ rt_file.write('%d tasks cost: %.4fs\n' % (len(dataset), time.time() - st))
+ rt_file.close()
diff --git a/auto_round/mllm/mllm_dataset.py b/auto_round/mllm/mllm_dataset.py
index 423d70e0d..cf319fbf0 100644
--- a/auto_round/mllm/mllm_dataset.py
+++ b/auto_round/mllm/mllm_dataset.py
@@ -19,9 +19,11 @@ from typing import Dict
import torch
from torch.utils.data import Dataset, DataLoader
+from transformers import set_seed
from .utils import _extract_data_dir
from .template import Template
+from ..utils import logger
MLLM_DATASET : Dict[str, Dataset] = {}
@@ -44,9 +46,17 @@ def register_dataset(name):
return register
-@register_dataset("llava")
+
+@register_dataset("liuhaotian/llava")
class LlavaDataset(Dataset):
"""Dataset for supervised fine-tuning."""
+ BASE_LLAVA_URL = "https://huggingface.co/datasets/liuhaotian/LLaVA-Instruct-150K/resolve/main/"
+ LLAVA_DATASET = {
+ "llava_conv_58k": BASE_LLAVA_URL + "conversation_58k.json?download=true",
+ "llava_instruct_80k": BASE_LLAVA_URL + "llava_instruct_80k.json?download=true",
+ "llava_instruct_150k": BASE_LLAVA_URL + "llava_instruct_150k.json?download=true",
+ }
+ _COCO_DATA_URL = "http://images.cocodataset.org/train2017/"
def __init__(
self,
@@ -54,8 +64,8 @@ class LlavaDataset(Dataset):
model,
tokenzier,
dataset_path,
- extra_data_dir,
- max_length,
+ extra_data_dir=None,
+ seqlen=None,
padding=True,
truncation=True,
) -> None:
@@ -64,13 +74,44 @@ class LlavaDataset(Dataset):
self.model_type = template.model_type
self.template = template
self.tokenizer = tokenzier
- self.questions = json.load(open(dataset_path, "r"))
+ if os.path.exists(dataset_path):
+ logger.info(f'use dataset {dataset_path}, loading from disk...')
+ self.questions = json.load(open(dataset_path, "r"))
+ else:
+ import requests
+ dataset_name = dataset_path.split('/')[-1]
+ if dataset_name in self.LLAVA_DATASET:
+ logger.info(f'use dataset {dataset_name}, downloading ...')
+ self.questions = requests.get(self.LLAVA_DATASET[dataset_name], stream=True).json()
+ else:
+ raise KeyError(f"{dataset_path} is not support, we support {self.LLAVA_DATASET.keys()}.")
+ self.seqlen = seqlen
+ self.questions = self.check(self.questions, seqlen)
self.padding = padding
self.truncation = truncation
self.extra_data_dir = extra_data_dir
- self.max_length = max_length
self.role_mapping = {"human": "user", "gpt": "assistant"}
self.cached_data_dict = {}
+
+ self.image_fold = None
+ if extra_data_dir is not None:
+ image_fold = _extract_data_dir(self.extra_data_dir)
+ if isinstance(image_fold, dict):
+ image_fold = image_fold['image']
+ self.image_fold = image_fold
+
+
+ @staticmethod
+ def check(questions, seqlen):
+ new_questions = []
+ for source in questions:
+ text_lenght = 0
+ for text in source['conversations']:
+ text_lenght += len(text['value'].split(' '))
+ if text_lenght >= seqlen:
+ new_questions.append(source)
+ assert len(new_questions) > 0, f"no data with length greater than {seqlen}, please check"
+ return new_questions
def __len__(self):
@@ -81,28 +122,30 @@ class LlavaDataset(Dataset):
return self.cached_data_dict[i]
text = self.questions[i]["conversations"]
- text = self.covert_conversations(text)
+ if self.template.model_type != "llava":
+ text = self.covert_conversations(text)
- text = self.template._encode(text)
- if self.extra_data_dir is not None:
- image_fold = _extract_data_dir(self.extra_data_dir)
- if isinstance(image_fold, dict):
- image_fold = image_fold['image']
+ if self.image_fold is not None:
image_path = os.path.join(
- image_fold, os.path.basename(self.questions[i]["image"]))
+ self.image_fold, os.path.basename(self.questions[i]["image"]))
else:
image_path = self.questions[i]["image"]
- image = self.template.processor.image_processor(image_path)
+ if not os.path.exists(image_path):
+ image_path = self._COCO_DATA_URL + self.questions[i]["image"].split('/')[-1]
+ # image = self.template.processor.image_processor(image_path)
+ text = self.template._encode(text)
+
+ max_length = self.seqlen
+ truncation_strategy = "text"
ret = self.template.processor.get_input(
- self.model,
- self.tokenizer,
text=text,
- images=image,
+ images=image_path,
padding=self.padding,
truncation=self.truncation,
return_tensors="pt",
- max_length = self.max_length
+ max_length = max_length,
+ truncation_strategy=truncation_strategy
)
self.cached_data_dict[i] = ret
return ret
@@ -111,8 +154,9 @@ class LlavaDataset(Dataset):
new_data = []
for d in data:
content = d["value"]
- for old, new in self.template.replace_tokens:
- content = content.replace(old, new)
+ if self.template.replace_tokens is not None:
+ for old, new in self.template.replace_tokens:
+ content = content.replace(old, new)
new_data.append({
"role": self.role_mapping.get(d["from"], d["from"]),
"content": content
@@ -124,12 +168,14 @@ def get_mllm_dataloader(
template,
model,
tokenizer,
- dataset,
- extra_data_dir,
+ image_processor=None,
+ dataset="liuhaotian/llava_conv_58k",
+ extra_data_dir=None,
seqlen=512,
bs=1,
split=None,
apply_template=None,
+ seed=42,
):
"""Generate a DataLoader for calibration using specified parameters.
@@ -148,13 +194,19 @@ def get_mllm_dataloader(
Returns:
DataLoader: The DataLoader for the calibrated datasets.
"""
- assert isinstance(template, Template)
+ if isinstance(template, str):
+ from .template import get_template
+ template = get_template(template, model=model, tokenizer=tokenizer, image_processor=image_processor)
if isinstance(dataset, str):
if os.path.isfile(dataset):
- dataset = MLLM_DATASET['llava'](
+ dataset = MLLM_DATASET['liuhaotian/llava'](
+ template, model, tokenizer, dataset, extra_data_dir,
+ seqlen=min(seqlen, tokenizer.model_max_length))
+ elif "liuhaotian/llava" in dataset:
+ dataset = MLLM_DATASET["liuhaotian/llava"](
template, model, tokenizer, dataset, extra_data_dir,
- max_length=min(seqlen, tokenizer.model_max_length))
+ seqlen=min(seqlen, tokenizer.model_max_length))
else:
from datasets import load_dataset
from ..calib_dataset import get_tokenizer_function
@@ -163,8 +215,10 @@ def get_mllm_dataloader(
dataset = dataset.map(tokenizer_function, batched=True)
+ set_seed(seed)
dataloader_params = {
"batch_size": bs,
+ "shuffle": True,
"collate_fn": dataset.template.processor.data_collator
}
diff --git a/auto_round/mllm/processor.py b/auto_round/mllm/processor.py
index eb6305fcf..3b285e015 100644
--- a/auto_round/mllm/processor.py
+++ b/auto_round/mllm/processor.py
@@ -29,39 +29,66 @@ def regist_processor(name):
class BasicProcessor:
def __init__(self):
pass
+
+ def post_init(self, model, tokenizer, image_processor=None, **kwargs):
+ self.model = model
+ self.tokenizer = tokenizer
+ if image_processor is not None:
+ self.image_processor = image_processor
+ else:
+ self.image_processor = self.default_image_processor
def get_input(
self,
- model,
- tokenizer,
text,
images,
- padding=True,
- truncation=True,
return_tensors="pt",
- max_length=None,
squeeze=True,
+ max_length=None,
+ truncation_strategy="text",
**kwargs):
+
+ if isinstance(text, list):
+ for message in text:
+ if not ("role" in message and "content" in message):
+ raise ValueError(
+ "When passing chat dicts as input, each dict must have a 'role' and 'content' key.")
+ continue_final_message = text[-1]["role"] == "assistant"
- if max_length:
- token_length = len(tokenizer(text).input_ids)
- if token_length < max_length:
- if tokenizer.pad_token:
- text += tokenizer.pad_token * (max_length - token_length)
- else:
- text = tokenizer.decode(tokenizer(text).input_ids[:max_length])
+ # add_generation_prompt=True will add <|im_start|>assistant to the end
+ try:
+ text = self.tokenizer.apply_chat_template(
+ text, tokenize=False, add_generation_prompt=not continue_final_message,
+ continue_final_message=continue_final_message,)
+ except:
+ raise NotImplementedError("current not support for non-instructed model.")
+
+ if text == '':
+ raise NotImplementedError("current not support for non-instructed model.")
+
+ if images is not None:
+ images = self.image_processor(images)
+
+ if truncation_strategy == "text" and max_length is not None:
+ text = text[:max_length]
- ret = tokenizer.processor(
+ ret = self.tokenizer.processor(
text=text,
images=images,
- padding=padding,
- truncation=truncation,
return_tensors=return_tensors,
# videos = None
)
- if squeeze:
+ if truncation_strategy == "token" and max_length:
+ seqlen = ret['input_ids'].shape[-1]
for key in ret:
- ret[key] = ret[key][0]
+ shape_ = ret[key].shape
+ if len(shape_) == 2 and shape_[-1] == seqlen:
+ ret[key] = ret[key][:,:max_length]
+ elif len(shape_) == 4 and shape_[1] == seqlen:
+ ret[key] = ret[key][:,:max_length]
+
+ if squeeze:
+ ret = self.squeeze_result(ret)
return ret
@staticmethod
@@ -69,57 +96,41 @@ class BasicProcessor:
return default_data_collator(batch)
@staticmethod
- def image_processor(image_path_or_url):
+ def default_image_processor(image_path_or_url):
return fetch_image(image_path_or_url)
+
+ @staticmethod
+ def squeeze_result(ret):
+ for key in ret:
+ ret[key] = ret[key][0]
+ return ret
@regist_processor("qwen2_vl")
class Qwen2VLProcessor(BasicProcessor):
- def get_input(
- self,
- model,
- tokenizer,
- text,
- images,
- padding=True,
- truncation=True,
- return_tensors="pt",
- max_length=None,
- squeeze=True,
- **kwargs):
-
- if max_length:
- token_length = len(tokenizer(text).input_ids)
- if token_length < max_length:
- if tokenizer.pad_token:
- text += tokenizer.pad_token * (max_length - token_length)
- else:
- text = tokenizer.decode(tokenizer(text).input_ids[:max_length])
-
- ret = tokenizer.processor(
- text=text,
- images=images,
- padding=padding,
- truncation=truncation,
- return_tensors=return_tensors,
- # videos = None
- )
- if squeeze:
- for key in ret:
- if key == "pixel_values":
+ @staticmethod
+ def squeeze_result(ret):
+ for key in ret:
+ if key == "pixel_values":
continue
- ret[key] = ret[key][0]
+ ret[key] = ret[key][0]
return ret
-
@regist_processor("cogvlm2")
class CogVLM2Processor(BasicProcessor):
def get_input(
- self, model, tokenizer, text, images, max_length=2048,
- padding=True, truncation=True, squeeze=True, **kwargs):
+ self, text, images,
+ squeeze=True, **kwargs):
+
+ if images is not None:
+ images = self.image_processor(images)
+
padding_len = 2303
+ max_length = 0
max_length += padding_len
- input_data = model.build_conversation_input_ids(
- tokenizer,
+ truncation = True
+ padding = False
+ input_data = self.model.build_conversation_input_ids(
+ self.tokenizer,
query=text,
history=None,
images=[images],
@@ -177,5 +188,51 @@ class CogVLM2Processor(BasicProcessor):
return batched_data
@staticmethod
- def image_processor(image_path_or_url):
- return fetch_image(image_path_or_url).convert('RGB')
\ No newline at end of file
+ def default_image_processor(image_path_or_url):
+ return fetch_image(image_path_or_url).convert('RGB')
+
+
+from ..utils import LazyImport
+llava_train = LazyImport("llava.train.train")
+
+@regist_processor("llava")
+class LlavaProcessor(BasicProcessor):
+ def post_init(self, model, tokenizer, image_processor=None, **kwargs):
+ self.model = model
+ self.tokenizer = tokenizer
+ assert image_processor is not None, "for llava model, image_processor should not be None"
+ self.image_processor = image_processor
+ self.collator_func = llava_train.DataCollatorForSupervisedDataset(tokenizer=self.tokenizer)
+
+
+ def get_input(
+ self, text, images,max_length=None,
+ squeeze=True, truncation_strategy="text", **kwargs):
+
+ if images is not None:
+ images = fetch_image(images).convert('RGB')
+ images = self.image_processor.preprocess(images, return_tensors='pt')['pixel_values'][0]
+
+ class DataArgs:
+ is_multimodal = True
+ mm_use_im_start_end = False
+
+ if truncation_strategy == "text" and max_length is not None:
+ text = text[:max_length]
+
+ input_data = llava_train.preprocess_multimodal([text], DataArgs())
+ ret = llava_train.preprocess(input_data, self.tokenizer, has_image=(images is not None))
+
+ if truncation_strategy == "token" and max_length:
+ seqlen = ret['input_ids'].shape[-1]
+ for key in ret:
+ if ret[key].shape[-1] == seqlen:
+ ret[key] = ret[key][:,:max_length]
+ if squeeze:
+ ret = self.squeeze_result(ret)
+ ret['image'] = images
+
+ return ret
+
+ def data_collator(self, batch):
+ return self.collator_func(batch)
\ No newline at end of file
diff --git a/auto_round/mllm/template.py b/auto_round/mllm/template.py
index 3478f1a10..6d477f04f 100644
--- a/auto_round/mllm/template.py
+++ b/auto_round/mllm/template.py
@@ -19,8 +19,6 @@ from typing import Dict, Optional, List, Union, Sequence
from enum import Enum, unique
from ..utils import logger
-
-
from .processor import BasicProcessor, PROCESSORS
TEMPLATES: Dict[str, "Template"] = {}
@@ -50,26 +48,30 @@ class Template:
format_separator: str
default_system: str
replace_tokens: List[tuple]
+ add_special_token: bool
processor: "BasicProcessor"
def _encode(self, sources):
"""Encodes formatted inputs to pairs of token ids."""
- element = ""
- for i, source in enumerate(sources):
- if i == 0:
- element += fill_content(self.format_system, content=self.default_system)
- # if i > 0 and i % 2 ==0:
- # element += fill_content(self.format_separator)
-
- if source['role'] == Role.USER.value:
- element += fill_content(self.format_user, content=source["content"])
- elif source['role'] == Role.ASSISTANT.value:
- element += fill_content(self.format_assistant, content=source["content"])
- elif source['role'] == Role.OBSERVATION.value:
- element += fill_content(self.format_observation, content=source["content"])
- elif source['role'] == Role.FUNCTION.value:
- element += fill_content(self.format_function, content=source["content"])
- return element
+ if self.add_special_token:
+ element = ""
+ for i, source in enumerate(sources):
+ if i == 0:
+ element += fill_content(self.format_system, content=self.default_system)
+ # if i > 0 and i % 2 ==0:
+ # element += fill_content(self.format_separator)
+
+ if source['role'] == Role.USER.value:
+ element += fill_content(self.format_user, content=source["content"])
+ elif source['role'] == Role.ASSISTANT.value:
+ element += fill_content(self.format_assistant, content=source["content"])
+ elif source['role'] == Role.OBSERVATION.value:
+ element += fill_content(self.format_observation, content=source["content"])
+ elif source['role'] == Role.FUNCTION.value:
+ element += fill_content(self.format_function, content=source["content"])
+ return element
+ else:
+ return sources
def _register_template(
@@ -82,7 +84,8 @@ def _register_template(
format_separator: Optional[str] = None,
default_system: str = "",
replace_tokens: List[tuple] = None,
- processor: "BasicProcessor" = PROCESSORS["basic"]
+ add_special_token: Optional[bool] = True,
+ processor: "BasicProcessor" = PROCESSORS["basic"],
):
"""Registers a chat template."""
template_class = Template
@@ -102,6 +105,7 @@ def _register_template(
format_separator = format_separator or default_format_separator,
default_system = default_system,
replace_tokens = replace_tokens,
+ add_special_token = add_special_token,
processor = processor()
)
return TEMPLATES[model_type]
@@ -137,7 +141,7 @@ def _load_preset_template():
_load_preset_template()
-def get_template(template_or_path: str):
+def get_template(template_or_path: str, model=None, tokenizer=None, image_processor=None):
"""Get template by template name or from a json file.
Args:
@@ -155,4 +159,7 @@ def get_template(template_or_path: str):
else:
logger.warning(f"Unable to recognize {template_or_path}, using default template instead.")
template = TEMPLATES["default"]
+
+ template.processor.post_init(model=model, tokenizer=tokenizer, image_processor=image_processor)
+
return template
\ No newline at end of file
diff --git a/auto_round/mllm/templates/default.json b/auto_round/mllm/templates/default.json
index c16844529..51ec37f34 100644
--- a/auto_round/mllm/templates/default.json
+++ b/auto_round/mllm/templates/default.json
@@ -8,5 +8,6 @@
"format_separator": "\n",
"default_system": "You are a helpful assistant.",
"replace_tokens": null,
- "processor": "basic"
+ "processor": "basic",
+ "add_special_token" : false
}
\ No newline at end of file
diff --git a/auto_round/mllm/templates/llava.json b/auto_round/mllm/templates/llava.json
new file mode 100644
index 000000000..13d5a6b49
--- /dev/null
+++ b/auto_round/mllm/templates/llava.json
@@ -0,0 +1,6 @@
+{
+ "model_type": "llava",
+ "replace_tokens": null,
+ "processor": "llava",
+ "add_special_token" : false
+}
\ No newline at end of file
diff --git a/auto_round/mllm/templates/phi3_v.json b/auto_round/mllm/templates/phi3_v.json
new file mode 100644
index 000000000..09949ff47
--- /dev/null
+++ b/auto_round/mllm/templates/phi3_v.json
@@ -0,0 +1,6 @@
+{
+ "model_type": "phi3_v",
+ "replace_tokens": ["<image>", "<|image_1|>"],
+ "processor": "basic",
+ "add_special_token" : false
+}
\ No newline at end of file
diff --git a/auto_round/mllm/utils.py b/auto_round/mllm/utils.py
index ce3bcae1f..118ac1974 100644
--- a/auto_round/mllm/utils.py
+++ b/auto_round/mllm/utils.py
@@ -24,7 +24,7 @@ from PIL import Image # pylint: disable=E0401
def _extract_data_dir(dir_path: str):
if os.path.isdir(dir_path):
return dir_path
- else:
+ elif "=" in dir_path:
result = {}
dir_path = dir_path.split(",")
for _path in dir_path:
@@ -32,6 +32,8 @@ def _extract_data_dir(dir_path: str):
if k in ['image', 'video', 'audio']:
result[k] = v
return result
+ else:
+ raise TypeError("incorrect input of extra_data_dir, please use auto_round --help for more details.")
def fetch_image(path_or_url):
diff --git a/auto_round/script/mllm.py b/auto_round/script/mllm.py
index 4690ccab9..54a25f3a2 100644
--- a/auto_round/script/mllm.py
+++ b/auto_round/script/mllm.py
@@ -30,7 +30,7 @@ class BasicArgumentParser(argparse.ArgumentParser):
def __init__(self, *args, **kwargs):
super().__init__(*args, **kwargs)
self.add_argument("--model", "--model_name", "--model_name_or_path",
- default="facebook/opt-125m",
+ default="Qwen/Qwen2-VL-2B-Instruct",
help="model name or path")
self.add_argument('--eval', action='store_true',
@@ -50,8 +50,10 @@ class BasicArgumentParser(argparse.ArgumentParser):
self.add_argument("--asym", action='store_true',
help="whether to use asym quantization")
- self.add_argument("--dataset", type=str, default=None,
- help="the dataset for quantization training. It can be a custom one.")
+ self.add_argument("--dataset", type=str, default="liuhaotian/llava_conv_58k",
+ help="the dataset for quantization training."
+ " current support llava_conv_58k,llava_instruct_80k "
+ "It can be a custom one.")
self.add_argument("--lr", default=None, type=float,
help="learning rate, if None, it will be set to 1.0/iters automatically")
@@ -65,7 +67,7 @@ class BasicArgumentParser(argparse.ArgumentParser):
self.add_argument("--adam", action='store_true',
help="whether to use adam optimizer instead of SignSGD")
- self.add_argument("--gradient_accumulate_steps", default=1, type=int,
+ self.add_argument("--gradient_accumulate_steps", default=4, type=int,
help="gradient accumulate steps")
self.add_argument("--nblocks", default=1, type=int,
@@ -136,17 +138,19 @@ class BasicArgumentParser(argparse.ArgumentParser):
self.add_argument("--quant_nontext_module", action='store_true',
help="whether to quantize non-text module, e.g. vision component")
- self.add_argument("--extra_data_dir", default="", type=str,
+ self.add_argument("--extra_data_dir", default=None, type=str,
help="dataset dir for storing images/audio/videos. "
"Can be a dir path or multiple dir path with format as "
"'image=path_to_image,video=path_to_video,audio=path_to_audio'"
- "By default, it will search in the relative path.")
+ "By default, it will search in the relative path, "
+ "and if not find, will automatic download.")
self.add_argument("--template", default=None, type=str,
help="the template for building training dataset. It can be a custom one.")
## ======================= VLM eval=======================
- self.add_argument("--tasks", type=str, default="COCO_VAL",
+ self.add_argument("--tasks", type=str,
+ default="MMBench_DEV_EN_V11,ScienceQA_VAL,TextVQA_VAL,POPE",
help="eval tasks for VLMEvalKit.")
# Args that only apply to Video Dataset
self.add_argument("--nframe", type=int, default=8,
@@ -189,7 +193,7 @@ def setup_parser():
parser.add_argument("--iters", "--iter", default=200, type=int,
help=" iters")
- parser.add_argument("--seqlen", "--seq_len", default=2048, type=int,
+ parser.add_argument("--seqlen", "--seq_len", default=256, type=int,
help="sequence length")
parser.add_argument("--nsamples", default=128, type=int,
@@ -250,24 +254,30 @@ def tune(args):
torch_dtype = torch.bfloat16
# load_model
- config = AutoConfig.from_pretrained(model_name, trust_remote_code=not args.disable_trust_remote_code)
- tokenizer = AutoTokenizer.from_pretrained(model_name)
- processor = AutoProcessor.from_pretrained(model_name, trust_remote_code=not args.disable_trust_remote_code)
- tokenizer.processor = processor
- model_type = config.model_type
- if "qwen2_vl" in model_type:
- from transformers import Qwen2VLForConditionalGeneration
- cls = Qwen2VLForConditionalGeneration
- elif "mllama" in model_type:
- from transformers import MllamaForConditionalGeneration
- cls = MllamaForConditionalGeneration
+ processor, image_processor = None, None
+ if "llava" in model_name:
+ from llava.model.builder import load_pretrained_model # pylint: disable=E0401
+ tokenizer, model, image_processor, _ = load_pretrained_model(model_name, model_base=None, model_name=model_name,
+ torch_dtype=torch_dtype)
+ model_type = "llava"
else:
- cls = AutoModelForCausalLM
-
+ config = AutoConfig.from_pretrained(model_name, trust_remote_code=not args.disable_trust_remote_code)
+ tokenizer = AutoTokenizer.from_pretrained(model_name)
+ processor = AutoProcessor.from_pretrained(model_name, trust_remote_code=not args.disable_trust_remote_code)
+ tokenizer.processor = processor
+ model_type = config.model_type
+ if "qwen2_vl" in model_type:
+ from transformers import Qwen2VLForConditionalGeneration
+ cls = Qwen2VLForConditionalGeneration
+ elif "mllama" in model_type:
+ from transformers import MllamaForConditionalGeneration
+ cls = MllamaForConditionalGeneration
+ else:
+ cls = AutoModelForCausalLM
+
model = cls.from_pretrained(
- model_name, trust_remote_code=not args.disable_trust_remote_code, torch_dtype=torch_dtype,
+ model_name,trust_remote_code=not args.disable_trust_remote_code, torch_dtype=torch_dtype,
device_map="auto" if use_auto_mapping else None)
-
if "cogvlm2" in model_name:
model.config.model_type = "cogvlm2"
@@ -276,6 +286,12 @@ def tune(args):
model = model.eval()
seqlen = args.seqlen
+ if args.model_dtype != None:
+ if args.model_dtype == "float16" or args.model_dtype == "fp16":
+ model = model.to(torch.float16)
+ if args.model_dtype == "bfloat16" or args.model_dtype == "bfp16":
+ model = model.to(torch.bfloat16)
+
round = AutoRoundMLLM
layer_config = {}
for n, m in model.named_modules():
@@ -310,14 +326,15 @@ def tune(args):
if args.quant_lm_head and args.low_gpu_mem_usage:
print(f"warning, low_gpu_mem_usage=False is strongly recommended if the whole model could be loaded to "
f"gpu")
-
- autoround = round(model, tokenizer, dataset=args.dataset, extra_data_dir=args.extra_data_dir,
- bits=args.bits, group_size=args.group_size, sym=not args.asym,
- batch_size=args.batch_size, seqlen=seqlen, nblocks=args.nblocks, iters=args.iters,
- lr=args.lr, minmax_lr=args.minmax_lr, enable_quanted_input=not args.disable_quanted_input,
- amp=not args.disable_amp, nsamples=args.nsamples, low_gpu_mem_usage=args.low_gpu_mem_usage,
+
+ autoround = round(model, tokenizer, image_processor=image_processor, dataset=args.dataset,
+ extra_data_dir=args.extra_data_dir, bits=args.bits, group_size=args.group_size,
+ sym=not args.asym, batch_size=args.batch_size, seqlen=seqlen, nblocks=args.nblocks,
+ iters=args.iters, lr=args.lr, minmax_lr=args.minmax_lr, amp=not args.disable_amp,
+ enable_quanted_input=not args.disable_quanted_input,
+ nsamples=args.nsamples, low_gpu_mem_usage=args.low_gpu_mem_usage,
device=device_str, seed=args.seed, gradient_accumulate_steps=args.gradient_accumulate_steps,
- scale_dtype=args.scale_dtype, layer_config=layer_config,
+ scale_dtype=args.scale_dtype, layer_config=layer_config, template=args.template,
enable_minmax_tuning=not args.disable_minmax_tuning, act_bits=args.act_bits,
quant_nontext_module=args.quant_nontext_module, not_use_best_mse=args.not_use_best_mse)
model, _ = autoround.quantize()
@@ -335,12 +352,18 @@ def tune(args):
inplace = False if len(format_list) > 1 else True
for format_ in format_list:
eval_folder = f'{export_dir}-{format_}'
- autoround.save_quantized(eval_folder, format=format_, inplace=inplace, processor=processor)
+ if processor is not None and not hasattr(processor, "chat_template"):
+ processor.chat_template = None
+ safe_serialization = True
+ if "phi3_v" in model_type:
+ safe_serialization = False
+ autoround.save_quantized(
+ eval_folder, format=format_, inplace=inplace, processor=processor, safe_serialization=safe_serialization)
def eval(args):
if isinstance(args.tasks, str):
- args.tasks = args.tasks.split(',')
+ args.tasks = args.tasks.replace(' ', '').split(',')
from auto_round.mllm import mllm_eval
mllm_eval(
args.model,
diff --git a/test/test_basic_usage.py b/test/test_basic_usage.py
index 0ee08c9b6..9664b2eb3 100644
--- a/test/test_basic_usage.py
+++ b/test/test_basic_usage.py
@@ -3,6 +3,8 @@ import shutil
import sys
import unittest
+sys.path.insert(0, '..')
+
class TestAutoRoundCmd(unittest.TestCase):
@classmethod
@@ -13,11 +15,39 @@ class TestAutoRoundCmd(unittest.TestCase):
def tearDownClass(self):
shutil.rmtree("./saved", ignore_errors=True)
shutil.rmtree("runs", ignore_errors=True)
-
+
def test_auto_round_cmd(self):
python_path = sys.executable
+ # test llm script
+ res = os.system(
+ f"cd .. && {python_path} -m auto_round -h")
+ if res > 0 or res == -1:
+ assert False, "cmd line test fail, please have a check"
+
+ res = os.system(
+ f"cd .. && {python_path} -m auto_round --model 'facebook/opt-125m' --iter 2 --nsamples 1 --format auto_gptq,auto_round --disable_eval --output_dir ./saved")
+ if res > 0 or res == -1:
+ assert False, "cmd line test fail, please have a check"
+
+
+ # test mllm script
+ res = os.system(
+ f"cd .. && {python_path} -m auto_round --mllm -h")
+ if res > 0 or res == -1:
+ assert False, "cmd line test fail, please have a check"
+
res = os.system(
- f"{python_path} ../auto_round/__main__.py --model 'facebook/opt-125m' --iter 2 --nsamples 1 --format auto_gptq,auto_round --disable_eval --output_dir ./saved")
+ f"cd .. && {python_path} -m auto_round --mllm --iter 2 --nsamples 10 --format auto_round --output_dir ./saved")
if res > 0 or res == -1:
- assert False, "cmd line test fail, please have acheck"
+ assert False, "cmd line test fail, please have a check"
+
+ res = os.system(
+ f"cd .. && {python_path} -m auto_round --mllm --iter 2 --nsamples 10 --format auto_round"
+ " --quant_nontext_module --output_dir ./saved")
+ if res > 0 or res == -1:
+ assert False, "cmd line test fail, please have a check"
+
+
+if __name__ == "__main__":
+ unittest.main()
\ No newline at end of file