Skip to content

PTQ-Level Efficiency, QAT-Level Accuracy | 训练后量化的效率,量化感知训练的精度

fix critical bug of get_multimodal_block_names (#509)

Heng Guoee890b2cf1a2
diff --git a/README.md b/README.md
index 37e4872d9..33048db91 100644
--- a/README.md
+++ b/README.md
@@ -224,7 +224,7 @@ autoround.quantize_and_save(output_dir, format='auto_round')
</details>
### API Usage for VLMs
-
+If you encounter issues during quantization, try setting iters=0 (to enable RTN) and use group_size=32 for better results.
<details>
<summary>Click to expand</summary>
diff --git a/auto_round/export/export_to_awq/export.py b/auto_round/export/export_to_awq/export.py
index 4759ae01e..a55c4f863 100644
--- a/auto_round/export/export_to_awq/export.py
+++ b/auto_round/export/export_to_awq/export.py
@@ -27,7 +27,7 @@ import torch.nn as nn
from auto_round.utils import (logger, get_module,
set_module,
check_to_quantized,
- get_multimodal_block_names,
+ get_block_names,
extract_block_names_to_str, supported_layer_types)
import copy
import json
@@ -84,7 +84,7 @@ def save_quantized_as_autoawq(output_dir, inplace=True, **kwargs):
if processor is not None:
processor.save_pretrained(output_dir)
# mllm models
- all_blocks = get_multimodal_block_names(model, quant_vision=True)
+ all_blocks = get_block_names(model, quant_vision=True)
all_block_names = extract_block_names_to_str(all_blocks)
all_block_names = all_block_names.split(',')
to_quant_block_names = to_quant_block_names.split(',')
diff --git a/auto_round/inference/convert_model.py b/auto_round/inference/convert_model.py
index 67a8b7310..417276939 100644
--- a/auto_round/inference/convert_model.py
+++ b/auto_round/inference/convert_model.py
@@ -21,11 +21,13 @@ import torch.nn as nn
from transformers.pytorch_utils import Conv1D
-from auto_round.utils import (get_module, set_module, is_hpu_supported, get_multimodal_block_names,
- find_matching_blocks, get_layer_names_in_block, check_to_quantized)
+from auto_round.utils import (
+ get_module, set_module, is_hpu_supported, get_block_names, find_matching_blocks,
+ get_layer_names_in_block, check_to_quantized)
-from auto_round.inference.backend import (get_layer_backend, dynamic_import_inference_linear,
- find_backend, BackendInfos, get_highest_priority_backend, process_requirement)
+from auto_round.inference.backend import (
+ get_layer_backend, dynamic_import_inference_linear, find_backend, BackendInfos, get_highest_priority_backend,
+ process_requirement)
logger = getLogger(__name__)
@@ -231,7 +233,7 @@ def get_layer_config(model, quantization_config):
]
else:
# Find matching blocks if no explicit names are provided
- all_blocks = get_multimodal_block_names(model, quant_vision=True)
+ all_blocks = get_block_names(model, quant_vision=True)
quant_block_list = find_matching_blocks(model, all_blocks, to_quant_block_names)
# Get layer names that will be quantized
diff --git a/auto_round/mllm/autoround_mllm.py b/auto_round/mllm/autoround_mllm.py
index 1427df4d7..e0ba63586 100644
--- a/auto_round/mllm/autoround_mllm.py
+++ b/auto_round/mllm/autoround_mllm.py
@@ -23,7 +23,7 @@ from ..utils import (
detect_device,
to_device,
to_dtype,
- get_multimodal_block_names,
+ get_block_names,
find_matching_blocks,
extract_block_names_to_str,
clear_memory
@@ -163,7 +163,7 @@ class AutoRoundMLLM(AutoRound):
enable_torch_compile: bool = False,
**kwargs,
):
- all_blocks = get_multimodal_block_names(model, quant_nontext_module)
+ all_blocks = get_block_names(model, quant_nontext_module)
self.quant_block_list = find_matching_blocks(model, all_blocks, to_quant_block_names)
if to_quant_block_names is None:
to_quant_block_names = extract_block_names_to_str(self.quant_block_list)
diff --git a/auto_round/script/mllm.py b/auto_round/script/mllm.py
index 6fb6aa1b4..4c74a1732 100644
--- a/auto_round/script/mllm.py
+++ b/auto_round/script/mllm.py
@@ -145,7 +145,7 @@ class BasicArgumentParser(argparse.ArgumentParser):
choices=["fp16", "float16", "bf16", "bfloat16", "fp32", "float32"],
help="force to convert the dtype, some backends supports fp16 dtype better")
- self.add_argument("--act_bits", default=32, type=int, help="activation bits")
+ self.add_argument("--act_bits", default=16, type=int, help="activation bits")
self.add_argument("--fp_layers", default="", type=str, help="layers to maintain original data type")
diff --git a/auto_round/utils.py b/auto_round/utils.py
index 4a0c82b8e..9e1b64511 100644
--- a/auto_round/utils.py
+++ b/auto_round/utils.py
@@ -380,7 +380,7 @@ def find_matching_blocks(model, all_blocks, to_quant_block_names):
return target_blocks
-def get_block_names(model):
+def get_block_names(model, quant_vision=False):
"""Get the block names for transformers-like networks.
Args:
@@ -389,43 +389,43 @@ def get_block_names(model):
Returns:
block_names: A list whose elements are list of block's layer names
"""
- block_names = []
- target_modules = []
- for n, m in model.named_modules():
- if hasattr(type(m), "__name__") and "ModuleList" in type(m).__name__:
- target_modules.append((n, m))
- break ## only find the first modulelist, may be not robust
- for i, target_m in enumerate(target_modules):
- block_names.append([])
- for n, m in target_m[1].named_children():
- block_names[i].append(target_m[0] + "." + n)
- return block_names
-
-
-def get_multimodal_block_names(model, quant_vision=False):
- """Get the multimodal model block names for transformers-like networks.
-
- Args:
- model: The model.
-
- Returns:
- block_names: A list whose elements are list of block's layer names
- """
- if hasattr(model, "config") and model.config.model_type in SPECIAL_MULTIMODAL_BLOCK.keys():
- return SPECIAL_MULTIMODAL_BLOCK.get(model.config.model_type)(model, quant_vision=quant_vision)
- block_names = []
- target_modules = []
- vison_blocks_tuple = ("vision", "visual",)
- for n, m in model.named_modules():
- if hasattr(type(m), "__name__") and "ModuleList" in type(m).__name__:
- if quant_vision or all(key not in n.lower() for key in (vison_blocks_tuple)):
+ def _get_llm_block_names(model):
+ block_names = []
+ target_modules = []
+ for n, m in model.named_modules():
+ if hasattr(type(m), "__name__") and "ModuleList" in type(m).__name__:
target_modules.append((n, m))
- validate_modules(target_modules, quant_vision, vison_blocks_tuple)
- for i, target_m in enumerate(target_modules):
- block_names.append([])
- for n, m in target_m[1].named_children():
- block_names[i].append(target_m[0] + "." + n)
- return block_names
+ break ## only find the first modulelist, may be not robust
+ for i, target_m in enumerate(target_modules):
+ block_names.append([])
+ for n, m in target_m[1].named_children():
+ block_names[i].append(target_m[0] + "." + n)
+ return block_names
+
+ def _get_vlm_block_names(model, quant_vision=False):
+ if hasattr(model, "config") and model.config.model_type in SPECIAL_MULTIMODAL_BLOCK.keys():
+ return SPECIAL_MULTIMODAL_BLOCK.get(model.config.model_type)(model, quant_vision=quant_vision)
+ block_names = []
+ target_modules = []
+ vision_blocks_tuple = ("vision", "visual", "image", "img")
+ last_block_name = ""
+ for n, m in model.named_modules():
+ if hasattr(type(m), "__name__") and "ModuleList" in type(m).__name__:
+ if quant_vision or all(key not in n.lower() for key in (vision_blocks_tuple)):
+ if last_block_name and last_block_name in n:
+ continue
+ target_modules.append((n, m))
+ last_block_name = n
+ for i, target_m in enumerate(target_modules):
+ block_names.append([])
+ for n, m in target_m[1].named_children():
+ block_names[i].append(target_m[0] + "." + n)
+ return block_names
+
+ if quant_vision or not is_pure_text_model(model):
+ return _get_vlm_block_names(model, quant_vision=quant_vision)
+ else:
+ return _get_llm_block_names(model)
def collect_best_params(block):
@@ -1265,3 +1265,19 @@ def _gguf_args_check(args):
return args
+def is_pure_text_model(model):
+ """verify on: phi-3.5, Mistral-Small-3.1, gemma-3, qwen2-vl, """
+ if hasattr(model,"config") and hasattr(model.config, "vision_config"):
+ return False
+ if hasattr(model.__class__, "main_input_name") and model.__class__.main_input_name != "input_ids":
+ return False
+ for module in model.modules():
+ if hasattr(module.__class__, "main_input_name") and module.__class__.main_input_name != "input_ids":
+ return False
+ if "vision" in str(module.__class__).lower():
+ return False
+ if "image" in str(module.__class__).lower():
+ return False
+ if "img" in str(module.__class__).lower():
+ return False
+ return True
diff --git a/test/test_autoopt.py b/test/test_autoopt.py
index 6dcb65543..3ea4565d7 100644
--- a/test/test_autoopt.py
+++ b/test/test_autoopt.py
@@ -51,8 +51,8 @@ class TestAutoRound(unittest.TestCase):
def test_Adam(self):
bits, group_size, sym = 4, 128, False
- from auto_round.utils import get_multimodal_block_names
- llm_block_names = get_multimodal_block_names(self.model, quant_vision=True)
+ from auto_round.utils import get_block_names
+ llm_block_names = get_block_names(self.model, quant_vision=True)
bits, group_size, sym, batch_size = 4, 128, False, 20
adamround = AutoRoundAdam(
self.model,
diff --git a/test/test_block_names.py b/test/test_block_names.py
index 0a106de6c..c98885e50 100644
--- a/test/test_block_names.py
+++ b/test/test_block_names.py
@@ -1,3 +1,4 @@
+import os
import shutil
import sys
import unittest
@@ -5,7 +6,7 @@ sys.path.insert(0, ".")
sys.path.insert(0, "..")
import torch
import torch.nn as nn
-from transformers import AutoModelForCausalLM, AutoTokenizer
+from transformers import AutoModelForCausalLM, AutoTokenizer, AutoConfig
from auto_round import AutoRound
class LLMDataLoader:
@@ -119,14 +120,11 @@ class TestQuantizationBlocks(unittest.TestCase):
experts_per_group = 2
self.llm_dataloader = LLMDataLoader(input_size)
self.model = NestedMoEModel(input_size, hidden_size, num_groups, experts_per_group)
- from auto_round.utils import get_multimodal_block_names, get_block_names
+ from auto_round.utils import get_block_names
llm_block_names = get_block_names(self.model)
all_block_names = []
- try:
- all_block_names = get_multimodal_block_names(self.model, quant_vision=True)
- except:
- pass
- assert len(llm_block_names) != len(all_block_names)
+ all_block_names = get_block_names(self.model, quant_vision=True)
+ assert len(llm_block_names) == len(all_block_names)
def test_multimodal_quant(self):
@@ -136,10 +134,10 @@ class TestQuantizationBlocks(unittest.TestCase):
text_input_size = 10
hidden_size = 10
self.model = SimpleMultimodalModel(text_input_size, image_input_size, hidden_size, num_text_encoders, num_image_encoders)
- from auto_round.utils import get_multimodal_block_names, get_block_names
+ from auto_round.utils import get_block_names
llm_block_names = get_block_names(self.model)
- block_names_wo_vision = get_multimodal_block_names(self.model, quant_vision=False)
- block_names_with_vision = get_multimodal_block_names(self.model, quant_vision=True)
+ block_names_wo_vision = get_block_names(self.model, quant_vision=False)
+ block_names_with_vision = get_block_names(self.model, quant_vision=True)
assert block_names_wo_vision == llm_block_names
assert len(block_names_wo_vision) != (block_names_with_vision)
@@ -163,25 +161,50 @@ class TestQuantizationBlocks(unittest.TestCase):
to_quant_block_names=llm_block_names
)
autoround.quantize()
- try:
- import auto_gptq
- except:
- return
- if not torch.cuda.is_available():
- return
+
quantized_model_path = "./saved"
autoround.save_quantized(quantized_model_path, inplace=False, safe_serialization=False, format="auto_round")
-
+ from auto_round import AutoRoundConfig
model = AutoModelForCausalLM.from_pretrained(quantized_model_path, device_map="auto")
tokenizer = AutoTokenizer.from_pretrained(quantized_model_path)
text = "There is a girl who likes adventure,"
inputs = tokenizer(text, return_tensors="pt").to(model.device)
print(tokenizer.decode(model.generate(**inputs, max_new_tokens=50)[0]))
- shutil.rmtree("./saved", ignore_errors=True)
quant_config = model.config.quantization_config
assert quant_config.to_quant_block_names is not None
+ shutil.rmtree("./saved", ignore_errors=True)
-
+ def test_moe(self):
+ from auto_round.utils import get_block_names
+ model_name = "Qwen/Qwen1.5-MoE-A2.7B"
+ # config = AutoConfig.from_pretrained(model_name)
+ model = AutoModelForCausalLM.from_pretrained(model_name)
+
+
+ block_name = get_block_names(model)
+ block_name_2 = get_block_names(model, quant_vision=True)
+ self.assertTrue(block_name == block_name_2)
+ self.assertTrue(len(block_name_2) == 1)
+ self.assertTrue('model.layers.23' == block_name_2[0][-1])
+ ##tokenizer = AutoTokenizer.from_pretrained(model_name)
+ # python_path = sys.executable
+ # res = os.system(
+ # f"cd .. && CUDA_VISIBLE_DEVICES=0 {python_path} -m auto_round --model {model_name} --iter 1 --nsamples 1 --format auto_round --output_dir test/saved --disable_eval")
+ # if res > 0 or res == -1:
+ # assert False, "cmd line test fail, please have a check"
+ #
+ # quantized_model_path = "../saved"
+ #
+ # from auto_round import AutoHfQuantizer
+ # model = AutoModelForCausalLM.from_pretrained(quantized_model_path, device_map="auto")
+ # tokenizer = AutoTokenizer.from_pretrained(quantized_model_path)
+ # text = "There is a girl who likes adventure,"
+ # inputs = tokenizer(text, return_tensors="pt").to(model.device)
+ # print(tokenizer.decode(model.generate(**inputs, max_new_tokens=50)[0]))
+ # shutil.rmtree("./saved", ignore_errors=True)
+ # quant_config = model.config.quantization_config
+ # assert quant_config.to_quant_block_names is not None
+
if __name__ == "__main__":
diff --git a/test/test_mllm.py b/test/test_mllm.py
index 2845c1392..68b27752e 100644
--- a/test/test_mllm.py
+++ b/test/test_mllm.py
@@ -69,14 +69,14 @@ class TestAutoRoundMLLM(unittest.TestCase):
autoround.save_quantized("./saved/", format="auto_round", inplace=True)
def test_quant_block_names(self):
- from auto_round.utils import get_multimodal_block_names,find_matching_blocks
+ from auto_round.utils import get_block_names,find_matching_blocks
tokenizer = AutoTokenizer.from_pretrained(self.model_name)
processor = AutoProcessor.from_pretrained(self.model_name, trust_remote_code=True)
model = Qwen2VLForConditionalGeneration.from_pretrained(
self.model_name, trust_remote_code=True, device_map="auto")
to_quant_block_names = 'visual.*12,layers.0,model.layers.*9'
target_blocks = [['visual.blocks.12'], ['model.layers.0', 'model.layers.9', 'model.layers.19']]
- all_blocks = get_multimodal_block_names(model, quant_vision=True)
+ all_blocks = get_block_names(model, quant_vision=True)
blocks = find_matching_blocks(model, all_blocks, to_quant_block_names)
assert target_blocks == blocks
@@ -102,6 +102,16 @@ class TestAutoRoundMLLM(unittest.TestCase):
nsamples=2,
batch_size=1, iters=2, dataset=dataset, seqlen=1)
autoround.quantize()
+
+ def test_pure_text_model_check(self):
+ from transformers import AutoModelForCausalLM
+ from auto_round.utils import is_pure_text_model
+ model = Qwen2VLForConditionalGeneration.from_pretrained(
+ self.model_name, trust_remote_code=True, device_map="auto")
+ self.assertFalse(is_pure_text_model(model))
+ model = AutoModelForCausalLM.from_pretrained("facebook/opt-125m", trust_remote_code=True)
+ self.assertTrue(is_pure_text_model(model))
+
if __name__ == "__main__":
unittest.main()
diff --git a/test_cuda/test_get_block_name.py b/test_cuda/test_get_block_name.py
new file mode 100644
index 000000000..f077a65d6
--- /dev/null
+++ b/test_cuda/test_get_block_name.py
@@ -0,0 +1,175 @@
+import copy
+import shutil
+import sys
+import unittest
+
+sys.path.insert(0, "..")
+import torch
+import transformers
+from transformers import AutoModelForCausalLM, AutoTokenizer, Qwen2VLForConditionalGeneration, AutoModelForVision2Seq, \
+ Gemma3ForConditionalGeneration, Mistral3ForConditionalGeneration
+from auto_round.utils import get_block_names, is_pure_text_model
+from auto_round import AutoRound
+
+
+class TestAutoRound(unittest.TestCase):
+ @classmethod
+ def setUpClass(self):
+ pass
+
+ @classmethod
+ def tearDownClass(self):
+ shutil.rmtree("runs", ignore_errors=True)
+
+ def check_block_names(self, block_names, prefixs=[], n_layers=[]):
+ for i, block_name in enumerate(block_names):
+ prefix = prefixs[i]
+ n_layer = n_layers[i]
+ expected_block_names = [prefix + "." + str(i) for i in range(n_layer)]
+ assert block_name == expected_block_names
+
+ def test_glm4(self):
+ model_name = "/models/glm-4-9b-chat"
+ model = AutoModelForCausalLM.from_pretrained(model_name, torch_dtype="auto", trust_remote_code=True)
+ block_names = get_block_names(model)
+ self.check_block_names(block_names, ["transformer.encoder.layers"], [40])
+ assert is_pure_text_model(model) == True
+
+ def test_opt_125m(self):
+ model_name = "/models/opt-125m"
+ model = AutoModelForCausalLM.from_pretrained(model_name, torch_dtype="auto", trust_remote_code=True)
+ block_names = get_block_names(model)
+ self.check_block_names(block_names, ["model.decoder.layers"], [12])
+
+ assert is_pure_text_model(model) == True
+
+ def test_Qwen(self):
+ model_name = "/models/Qwen2.5-7B-Instruct"
+ model = AutoModelForCausalLM.from_pretrained(model_name, torch_dtype="auto", trust_remote_code=True)
+ block_names = get_block_names(model)
+ self.check_block_names(block_names, ["model.layers"], [28])
+ assert is_pure_text_model(model) == True
+
+ def test_phi4(self):
+ model_name = "/models/phi-4"
+ model = AutoModelForCausalLM.from_pretrained(model_name, torch_dtype="auto", trust_remote_code=True)
+ block_names = get_block_names(model)
+ self.check_block_names(block_names, ["model.layers"], [40])
+ assert is_pure_text_model(model) == True
+
+ def test_llama3(self):
+ model_name = "/models/Meta-Llama-3.1-8B-Instruct"
+ model = AutoModelForCausalLM.from_pretrained(model_name, torch_dtype="auto", trust_remote_code=True)
+ block_names = get_block_names(model)
+ self.check_block_names(block_names, ["model.layers"], [32])
+ assert is_pure_text_model(model) == True
+
+ def test_mixtral(self):
+ model_name = "/models/Mixtral-8x7B-Instruct-v0.1"
+ model = AutoModelForCausalLM.from_pretrained(model_name, torch_dtype="auto", trust_remote_code=True)
+ block_names = get_block_names(model)
+ self.check_block_names(block_names, ["model.layers"], [32])
+ assert is_pure_text_model(model) == True
+
+ def test_falcon(self):
+ model_name = "/models/Falcon3-7B-Instruct"
+ model = AutoModelForCausalLM.from_pretrained(model_name, torch_dtype="auto", trust_remote_code=True)
+ block_names = get_block_names(model)
+ self.check_block_names(block_names, ["model.layers"], [28])
+ assert is_pure_text_model(model) == True
+
+ def test_orca(self):
+ model_name = "/models/Orca-2-7b"
+ model = AutoModelForCausalLM.from_pretrained(model_name, torch_dtype="auto", trust_remote_code=True)
+ block_names = get_block_names(model)
+ self.check_block_names(block_names, ["model.layers"], [32])
+ assert is_pure_text_model(model) == True
+
+ def test_OLMo(self):
+ model_name = "/models/OLMo-2-1124-7B-Instruct"
+ model = AutoModelForCausalLM.from_pretrained(model_name, torch_dtype="auto", trust_remote_code=True)
+ block_names = get_block_names(model)
+ self.check_block_names(block_names, ["model.layers"], [32])
+ assert is_pure_text_model(model) == True
+
+ def test_Qwen2VL(self):
+ model_name = "/models/Qwen2-VL-2B-Instruct"
+ model = Qwen2VLForConditionalGeneration.from_pretrained(model_name, torch_dtype="auto", trust_remote_code=True)
+ block_names = get_block_names(model)
+ self.check_block_names(block_names, ["model.layers"], [28])
+
+ block_names = get_block_names(model, quant_vision=True)
+ self.check_block_names(block_names, ["visual.blocks", "model.layers"], [32, 28])
+ assert is_pure_text_model(model) == False
+
+ def test_Llama32(self):
+ model_name = "/models/Llama-3.2-11B-Vision-Instruct"
+ model = AutoModelForVision2Seq.from_pretrained(model_name, torch_dtype="auto", trust_remote_code=True)
+ block_names = get_block_names(model)
+ self.check_block_names(block_names, ["language_model.model.layers"], [40])
+
+ block_names = get_block_names(model, quant_vision=True)
+ self.check_block_names(block_names,
+ ["vision_model.transformer.layers", "vision_model.global_transformer.layers",
+ "language_model.model.layers"], [32, 8, 40])
+
+ assert is_pure_text_model(model) == False
+
+ def test_SmolVLM(self):
+ model_name = "/models/SmolVLM-Instruct"
+ model = AutoModelForVision2Seq.from_pretrained(model_name, torch_dtype="auto", trust_remote_code=True)
+ block_names = get_block_names(model)
+ self.check_block_names(block_names, ["model.text_model.layers"], [24])
+
+ block_names = get_block_names(model, quant_vision=True)
+ self.check_block_names(block_names, ["model.vision_model.encoder.layers", "model.text_model.layers"], [27, 24])
+ assert is_pure_text_model(model) == False
+
+ def test_glm_4v(self):
+ model_name = "/models/glm-4v-9b"
+ model = AutoModelForCausalLM.from_pretrained(model_name, torch_dtype="auto", trust_remote_code=True)
+ block_names = get_block_names(model)
+ self.check_block_names(block_names, ["transformer.encoder.layers"], [40])
+
+ block_names = get_block_names(model, quant_vision=True)
+ self.check_block_names(block_names, ["transformer.encoder.layers", "transformer.vision.transformer.layers"],
+ [40, 63])
+ assert is_pure_text_model(model) == False
+
+ def test_gemma3(self):
+ model_name = "/models/gemma-3-12b-it"
+ model = Gemma3ForConditionalGeneration.from_pretrained(model_name, torch_dtype="auto", trust_remote_code=True)
+ block_names = get_block_names(model)
+ self.check_block_names(block_names, ["language_model.model.layers"], [48])
+
+ block_names = get_block_names(model, quant_vision=True)
+ self.check_block_names(block_names, ["vision_tower.vision_model.encoder.layers", "language_model.model.layers"],
+ [27, 48])
+ assert is_pure_text_model(model) == False
+
+ def test_Mistral3(self):
+ model_name = "/models/Mistral-Small-3.1-24B-Instruct-2503"
+ model = Mistral3ForConditionalGeneration.from_pretrained(model_name, torch_dtype="auto", trust_remote_code=True)
+ block_names = get_block_names(model)
+ self.check_block_names(block_names, ["language_model.model.layers"], [40])
+
+ block_names = get_block_names(model, quant_vision=True)
+ self.check_block_names(block_names, ["vision_tower.transformer.layers", "language_model.model.layers"],
+ [24, 40])
+ assert is_pure_text_model(model) == False
+
+ def test_Molmo(self):
+ model_name = "/models/Molmo-7B-D-0924"
+ model = AutoModelForCausalLM.from_pretrained(model_name, torch_dtype="auto", trust_remote_code=True)
+ block_names = get_block_names(model)
+ self.check_block_names(block_names, ["model.transformer.blocks"], [28])
+
+ block_names = get_block_names(model, quant_vision=True)
+ self.check_block_names(block_names,
+ ["model.transformer.blocks", "model.vision_backbone.image_vit.transformer.resblocks"],
+ [28, 23])
+ assert is_pure_text_model(model) == False
+
+
+if __name__ == "__main__":
+ unittest.main()