Skip to content

PTQ-Level Efficiency, QAT-Level Accuracy | 训练后量化的效率,量化感知训练的精度

support for llava-based hf model (#568)

Heng Guo91487375d260
diff --git a/README.md b/README.md
index 76f984d62..8fdaf12e0 100644
--- a/README.md
+++ b/README.md
@@ -471,6 +471,7 @@ model architecture or kernel limitations.
|--------------------------------|---------------------|----------------------|------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|
| allenai/Molmo | pile | X | [Molmo-7B-D-0924-int4-sym](https://huggingface.co/OPEA/Molmo-7B-D-0924-int4-sym-inc), [Molmo-72B-0924-int4-sym-gptq](https://huggingface.co/OPEA/Molmo-72B-0924-int4-sym-gptq-inc), [Molmo-72B-0924-int4-sym](https://huggingface.co/OPEA/Molmo-72B-0924-int4-sym-inc) |
| deepseek-ai/deepseek-vl2 | pile/llava | √ | [deepseek-vl2-int4-sym-gptq](https://huggingface.co/OPEA/deepseek-vl2-int4-sym-gptq-inc) |
+| fancyfeast/llama-joycaption-beta-one-hf-llava | pile | X | [fancyfeast/llama-joycaption-beta-one-hf-llava](https://huggingface.co/fancyfeast/llama-joycaption-beta-one-hf-llava)
| google/gemma-3 | pile/llava | √ | [gemma-3-12b-it-AutoRound-gguf-q4-0](https://huggingface.co/OPEA/gemma-3-12b-it-AutoRound-gguf-q4-0), [gemma-3-27b-it-AutoRound-gguf-q4-0](https://huggingface.co/OPEA/gemma-3-27b-it-AutoRound-gguf-q4-0), [gemma-3-12b-it-int4-AutoRound](https://huggingface.co/OPEA/gemma-3-12b-it-int4-AutoRound), [gemma-3-27b-it-int4-AutoRound](https://huggingface.co/OPEA/gemma-3-27b-it-int4-AutoRound) |
| HuggingFaceTB/SmolVLM | pile/llava | √ | [SmolVLM-Instruct-int4-sym](https://huggingface.co/OPEA/SmolVLM-Instruct-int4-sym-inc) |
| ibm-granite/granite-vision-3.2 | pile/llava | - | |
diff --git a/auto_round/mllm/autoround_mllm.py b/auto_round/mllm/autoround_mllm.py
index 200a33274..c3dded9f4 100644
--- a/auto_round/mllm/autoround_mllm.py
+++ b/auto_round/mllm/autoround_mllm.py
@@ -12,7 +12,7 @@
# See the License for the specific language governing permissions and
# limitations under the License.
-from typing import Optional, Union
+from typing import Union
from tqdm import tqdm
from copy import deepcopy
@@ -174,10 +174,18 @@ class AutoRoundMLLM(AutoRound):
self.quant_nontext_module = quant_nontext_module
self.processor = processor
self.image_processor = image_processor
+ from transformers import PreTrainedModel
+ if model.config.model_type == "llava" and isinstance(model, PreTrainedModel):
+ template = "default"
self.template = template if template is not None else model.config.model_type
if not isinstance(dataset, torch.utils.data.DataLoader):
self.template = get_template(
- self.template, model=model, tokenizer=tokenizer, processor=processor, image_processor=image_processor)
+ self.template,
+ model=model,
+ tokenizer=tokenizer,
+ processor=processor,
+ image_processor=image_processor,
+ use_rtn=iters == 0)
dataset = self.template.default_dataset if dataset is None else dataset
model = _handle_special_model(model)
diff --git a/auto_round/mllm/processor.py b/auto_round/mllm/processor.py
index 06a4102b7..a29efb3b6 100644
--- a/auto_round/mllm/processor.py
+++ b/auto_round/mllm/processor.py
@@ -48,7 +48,7 @@ class BasicProcessor:
def __init__(self):
pass
- def post_init(self, model, tokenizer, processor=None, image_processor=None, **kwargs):
+ def post_init(self, model, tokenizer, processor=None, image_processor=None, use_rtn=False, **kwargs):
self.model = model
self.tokenizer = tokenizer
self.processor = processor
@@ -56,6 +56,8 @@ class BasicProcessor:
self.image_processor = image_processor
else:
self.image_processor = self.default_image_processor
+ self.use_rtn = use_rtn
+ self.check_image_processor()
def get_input(
self,
@@ -78,6 +80,10 @@ class BasicProcessor:
for key in ret:
ret[key] = ret[key][0]
return ret
+
+ def check_image_processor(self):
+ if not self.use_rtn and self.image_processor is None:
+ raise ValueError("image processor should not be None.")
@register_processor("hf")
@@ -87,15 +93,6 @@ class HFProcessor(BasicProcessor):
def __init__(self):
self.process_func = self._process_v1
- def post_init(self, model, tokenizer, processor=None, image_processor=None, **kwargs):
- self.model = model
- self.tokenizer = tokenizer
- self.processor = processor
- if image_processor is not None:
- self.image_processor = image_processor
- else:
- self.image_processor = self.default_image_processor
-
def _process_v1(self, messages, image):
"""support models: Qwen2-VL, gemma-3, granite-vision-3.2, Aria"""
conversation = []
@@ -270,12 +267,12 @@ llava_train = LazyImport("llava.train.train")
@register_processor("llava")
class LlavaProcessor(BasicProcessor):
- def post_init(self, model, tokenizer, image_processor=None, **kwargs):
+ def post_init(self, model, tokenizer, image_processor=None, use_rtn=False, **kwargs):
self.model = model
self.tokenizer = tokenizer
- assert image_processor is not None, "for llava model, image_processor should not be None"
self.image_processor = image_processor
self.collator_func = llava_train.DataCollatorForSupervisedDataset(tokenizer=self.tokenizer)
+ self.check_image_processor()
def get_input(
self, text, images, max_length=None,
diff --git a/auto_round/mllm/template.py b/auto_round/mllm/template.py
index 62e1a2225..88ba62593 100644
--- a/auto_round/mllm/template.py
+++ b/auto_round/mllm/template.py
@@ -15,7 +15,7 @@
import os
import json
from dataclasses import dataclass
-from typing import Dict, Optional, List, Union, Sequence
+from typing import Dict, Optional, List
from enum import Enum, unique
from ..utils import logger
@@ -152,7 +152,8 @@ def _load_preset_template():
_load_preset_template()
-def get_template(template_or_path: str, model=None, tokenizer=None, processor=None, image_processor=None):
+def get_template(
+ template_or_path: str, model=None, tokenizer=None, processor=None, image_processor=None, use_rtn=False):
"""Get template by template name or from a json file.
Args:
@@ -172,6 +173,7 @@ def get_template(template_or_path: str, model=None, tokenizer=None, processor=No
template = TEMPLATES["default"]
template.model_type = template_or_path
- template.processor.post_init(model=model, tokenizer=tokenizer, processor=processor, image_processor=image_processor)
+ template.processor.post_init(
+ model=model, tokenizer=tokenizer, processor=processor, image_processor=image_processor, use_rtn=use_rtn)
return template
diff --git a/auto_round/utils.py b/auto_round/utils.py
index e61ba480a..cd63b3831 100644
--- a/auto_round/utils.py
+++ b/auto_round/utils.py
@@ -1364,7 +1364,10 @@ def mllm_load_model(
model_name=pretrained_model_name_or_path,
torch_dtype=torch_dtype)
else:
- if hasattr(transformers, architectures):
+ if architectures.endswith("Model") \
+ and hasattr(transformers, n := architectures.replace("Model", "ForConditionalGeneration")):
+ cls = getattr(transformers, n)
+ elif hasattr(transformers, architectures):
cls = getattr(transformers, architectures)
else:
cls = AutoModelForCausalLM
diff --git a/test/test_mllm.py b/test/test_mllm.py
index fd6d996b5..f92cdf0d7 100644
--- a/test/test_mllm.py
+++ b/test/test_mllm.py
@@ -79,7 +79,7 @@ class TestAutoRoundMLLM(unittest.TestCase):
model = Qwen2VLForConditionalGeneration.from_pretrained(
self.model_name, trust_remote_code=True, device_map="auto")
to_quant_block_names = 'visual.*12,layers.0,model.layers.*9'
- target_blocks = [['visual.blocks.12'], ['model.layers.0', 'model.layers.9', 'model.layers.19']]
+ target_blocks = [['model.visual.blocks.12'], ['model.language_model.layers.0', 'model.language_model.layers.9', 'model.language_model.layers.19']]
all_blocks = get_block_names(model, quant_vision=True)
blocks = find_matching_blocks(model, all_blocks, to_quant_block_names)
assert target_blocks == blocks