PTQ-Level Efficiency, QAT-Level Accuracy | 训练后量化的效率,量化感知训练的精度
support for llava-based hf model (#568)
Heng Guo91487375d260
diff --git a/README.md b/README.mdindex 76f984d62..8fdaf12e0 100644--- a/README.md+++ b/README.md@@ -471,6 +471,7 @@ model architecture or kernel limitations.|--------------------------------|---------------------|----------------------|------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|| allenai/Molmo | pile | X | [Molmo-7B-D-0924-int4-sym](https://huggingface.co/OPEA/Molmo-7B-D-0924-int4-sym-inc), [Molmo-72B-0924-int4-sym-gptq](https://huggingface.co/OPEA/Molmo-72B-0924-int4-sym-gptq-inc), [Molmo-72B-0924-int4-sym](https://huggingface.co/OPEA/Molmo-72B-0924-int4-sym-inc) || deepseek-ai/deepseek-vl2 | pile/llava | √ | [deepseek-vl2-int4-sym-gptq](https://huggingface.co/OPEA/deepseek-vl2-int4-sym-gptq-inc) |+| fancyfeast/llama-joycaption-beta-one-hf-llava | pile | X | [fancyfeast/llama-joycaption-beta-one-hf-llava](https://huggingface.co/fancyfeast/llama-joycaption-beta-one-hf-llava)| google/gemma-3 | pile/llava | √ | [gemma-3-12b-it-AutoRound-gguf-q4-0](https://huggingface.co/OPEA/gemma-3-12b-it-AutoRound-gguf-q4-0), [gemma-3-27b-it-AutoRound-gguf-q4-0](https://huggingface.co/OPEA/gemma-3-27b-it-AutoRound-gguf-q4-0), [gemma-3-12b-it-int4-AutoRound](https://huggingface.co/OPEA/gemma-3-12b-it-int4-AutoRound), [gemma-3-27b-it-int4-AutoRound](https://huggingface.co/OPEA/gemma-3-27b-it-int4-AutoRound) || HuggingFaceTB/SmolVLM | pile/llava | √ | [SmolVLM-Instruct-int4-sym](https://huggingface.co/OPEA/SmolVLM-Instruct-int4-sym-inc) || ibm-granite/granite-vision-3.2 | pile/llava | - | |diff --git a/auto_round/mllm/autoround_mllm.py b/auto_round/mllm/autoround_mllm.pyindex 200a33274..c3dded9f4 100644--- a/auto_round/mllm/autoround_mllm.py+++ b/auto_round/mllm/autoround_mllm.py@@ -12,7 +12,7 @@# See the License for the specific language governing permissions and# limitations under the License.-from typing import Optional, Union+from typing import Unionfrom tqdm import tqdmfrom copy import deepcopy@@ -174,10 +174,18 @@ class AutoRoundMLLM(AutoRound):self.quant_nontext_module = quant_nontext_moduleself.processor = processorself.image_processor = image_processor+ from transformers import PreTrainedModel+ if model.config.model_type == "llava" and isinstance(model, PreTrainedModel):+ template = "default"self.template = template if template is not None else model.config.model_typeif not isinstance(dataset, torch.utils.data.DataLoader):self.template = get_template(- self.template, model=model, tokenizer=tokenizer, processor=processor, image_processor=image_processor)+ self.template,+ model=model,+ tokenizer=tokenizer,+ processor=processor,+ image_processor=image_processor,+ use_rtn=iters == 0)dataset = self.template.default_dataset if dataset is None else datasetmodel = _handle_special_model(model)diff --git a/auto_round/mllm/processor.py b/auto_round/mllm/processor.pyindex 06a4102b7..a29efb3b6 100644--- a/auto_round/mllm/processor.py+++ b/auto_round/mllm/processor.py@@ -48,7 +48,7 @@ class BasicProcessor:def __init__(self):pass- def post_init(self, model, tokenizer, processor=None, image_processor=None, **kwargs):+ def post_init(self, model, tokenizer, processor=None, image_processor=None, use_rtn=False, **kwargs):self.model = modelself.tokenizer = tokenizerself.processor = processor@@ -56,6 +56,8 @@ class BasicProcessor:self.image_processor = image_processorelse:self.image_processor = self.default_image_processor+ self.use_rtn = use_rtn+ self.check_image_processor()def get_input(self,@@ -78,6 +80,10 @@ class BasicProcessor:for key in ret:ret[key] = ret[key][0]return ret++ def check_image_processor(self):+ if not self.use_rtn and self.image_processor is None:+ raise ValueError("image processor should not be None.")@register_processor("hf")@@ -87,15 +93,6 @@ class HFProcessor(BasicProcessor):def __init__(self):self.process_func = self._process_v1- def post_init(self, model, tokenizer, processor=None, image_processor=None, **kwargs):- self.model = model- self.tokenizer = tokenizer- self.processor = processor- if image_processor is not None:- self.image_processor = image_processor- else:- self.image_processor = self.default_image_processor-def _process_v1(self, messages, image):"""support models: Qwen2-VL, gemma-3, granite-vision-3.2, Aria"""conversation = []@@ -270,12 +267,12 @@ llava_train = LazyImport("llava.train.train")@register_processor("llava")class LlavaProcessor(BasicProcessor):- def post_init(self, model, tokenizer, image_processor=None, **kwargs):+ def post_init(self, model, tokenizer, image_processor=None, use_rtn=False, **kwargs):self.model = modelself.tokenizer = tokenizer- assert image_processor is not None, "for llava model, image_processor should not be None"self.image_processor = image_processorself.collator_func = llava_train.DataCollatorForSupervisedDataset(tokenizer=self.tokenizer)+ self.check_image_processor()def get_input(self, text, images, max_length=None,diff --git a/auto_round/mllm/template.py b/auto_round/mllm/template.pyindex 62e1a2225..88ba62593 100644--- a/auto_round/mllm/template.py+++ b/auto_round/mllm/template.py@@ -15,7 +15,7 @@import osimport jsonfrom dataclasses import dataclass-from typing import Dict, Optional, List, Union, Sequence+from typing import Dict, Optional, Listfrom enum import Enum, uniquefrom ..utils import logger@@ -152,7 +152,8 @@ def _load_preset_template():_load_preset_template()-def get_template(template_or_path: str, model=None, tokenizer=None, processor=None, image_processor=None):+def get_template(+ template_or_path: str, model=None, tokenizer=None, processor=None, image_processor=None, use_rtn=False):"""Get template by template name or from a json file.Args:@@ -172,6 +173,7 @@ def get_template(template_or_path: str, model=None, tokenizer=None, processor=Notemplate = TEMPLATES["default"]template.model_type = template_or_path- template.processor.post_init(model=model, tokenizer=tokenizer, processor=processor, image_processor=image_processor)+ template.processor.post_init(+ model=model, tokenizer=tokenizer, processor=processor, image_processor=image_processor, use_rtn=use_rtn)return templatediff --git a/auto_round/utils.py b/auto_round/utils.pyindex e61ba480a..cd63b3831 100644--- a/auto_round/utils.py+++ b/auto_round/utils.py@@ -1364,7 +1364,10 @@ def mllm_load_model(model_name=pretrained_model_name_or_path,torch_dtype=torch_dtype)else:- if hasattr(transformers, architectures):+ if architectures.endswith("Model") \+ and hasattr(transformers, n := architectures.replace("Model", "ForConditionalGeneration")):+ cls = getattr(transformers, n)+ elif hasattr(transformers, architectures):cls = getattr(transformers, architectures)else:cls = AutoModelForCausalLMdiff --git a/test/test_mllm.py b/test/test_mllm.pyindex fd6d996b5..f92cdf0d7 100644--- a/test/test_mllm.py+++ b/test/test_mllm.py@@ -79,7 +79,7 @@ class TestAutoRoundMLLM(unittest.TestCase):model = Qwen2VLForConditionalGeneration.from_pretrained(self.model_name, trust_remote_code=True, device_map="auto")to_quant_block_names = 'visual.*12,layers.0,model.layers.*9'- target_blocks = [['visual.blocks.12'], ['model.layers.0', 'model.layers.9', 'model.layers.19']]+ target_blocks = [['model.visual.blocks.12'], ['model.language_model.layers.0', 'model.language_model.layers.9', 'model.language_model.layers.19']]all_blocks = get_block_names(model, quant_vision=True)blocks = find_matching_blocks(model, all_blocks, to_quant_block_names)assert target_blocks == blocks