Skip to content

PTQ-Level Efficiency, QAT-Level Accuracy | 训练后量化的效率,量化感知训练的精度

support_llava_hf_vlm_example (#381)

WeiweiZhang1e6f89d196b5f
diff --git a/auto_round/script/mllm.py b/auto_round/script/mllm.py
index 89e5b23a3..634518bec 100644
--- a/auto_round/script/mllm.py
+++ b/auto_round/script/mllm.py
@@ -282,18 +282,21 @@ def tune(args):
# load_model
processor, image_processor = None, None
- if "llava" in model_name:
- from llava.model.builder import load_pretrained_model # pylint: disable=E0401
+ config = AutoConfig.from_pretrained(model_name, trust_remote_code=not args.disable_trust_remote_code)
+ if "llava" in model_name and config.architectures[0] != "LlavaForConditionalGeneration":
+ from llava.model.builder import load_pretrained_model # pylint: disable=E0401
tokenizer, model, image_processor, _ = load_pretrained_model(
model_name, model_base=None, model_name=model_name,
torch_dtype=torch_dtype)
model_type = "llava"
else:
- config = AutoConfig.from_pretrained(model_name, trust_remote_code=not args.disable_trust_remote_code)
- tokenizer = AutoTokenizer.from_pretrained(model_name)
+ tokenizer = AutoTokenizer.from_pretrained(model_name, trust_remote_code=not args.disable_trust_remote_code)
processor = AutoProcessor.from_pretrained(model_name, trust_remote_code=not args.disable_trust_remote_code)
model_type = config.model_type
- if "qwen2_vl" in model_type:
+ if "llava" in model_type:
+ from transformers import LlavaForConditionalGeneration
+ cls = LlavaForConditionalGeneration
+ elif "qwen2_vl" in model_type:
from transformers import Qwen2VLForConditionalGeneration
cls = Qwen2VLForConditionalGeneration
elif "mllama" in model_type:
@@ -511,3 +514,4 @@ def lmms_eval(args):
apply_chat_template=False,
)
return results
+