PTQ-Level Efficiency, QAT-Level Accuracy | 训练后量化的效率,量化感知训练的精度
[GGUF] using quant_nontext_module to control whether quant vision model (#1317)
Heng Guo24e43976900a
diff --git a/auto_round/compressors/base.py b/auto_round/compressors/base.pyindex fe2d2d092..c2c8bf436 100644--- a/auto_round/compressors/base.py+++ b/auto_round/compressors/base.py@@ -1198,11 +1198,8 @@ class BaseCompressor(object):model=self.model,device=self.device,output_dir=self._get_save_folder_name(self.formats[0]),- mllm=self.mllm,layer_config=self.layer_config,tokenizer=self.tokenizer,- processor=self.processor if hasattr(self, "processor") else None,- image_processor=self.image_processor if hasattr(self, "image_processor") else None,)@torch.inference_mode()diff --git a/auto_round/compressors/mllm/compressor.py b/auto_round/compressors/mllm/compressor.pyindex 462df089a..730f8e205 100644--- a/auto_round/compressors/mllm/compressor.py+++ b/auto_round/compressors/mllm/compressor.py@@ -183,13 +183,18 @@ class MLLMCompressor(BaseCompressor):self.model = modelquant_nontext_module = self._check_quant_nontext(layer_config, quant_nontext_module)- if quant_nontext_module:- from transformers.utils.versions import require_version-- require_version(- "pillow",- "pillow is required for quantizing non-text modules, please install it with `pip install pillow`",- )+ if quant_nontext_module and iters > 0:+ import importlib.util++ missing_libs = []+ for require_lib in ["pillow", "torchvision"]:+ if importlib.util.find_spec(require_lib) is None:+ missing_libs.append(require_lib)+ if len(missing_libs) > 0:+ logger.error(+ f"{', '.join(missing_libs)} are required for quantizing non-text modules,"+ f" please install them with `pip install {' '.join(missing_libs)}`",+ )all_blocks = get_block_names(model, quant_nontext_module)self.quant_block_list = find_matching_blocks(model, all_blocks, to_quant_block_names)if to_quant_block_names is None:@@ -453,7 +458,12 @@ class MLLMCompressor(BaseCompressor):if self.processor is not None and not hasattr(self.processor, "chat_template"):self.processor.chat_template = Nonecompressed_model = super().save_quantized(- output_dir=output_dir, format=format, inplace=inplace, processor=self.processor, **kwargs+ output_dir=output_dir,+ format=format,+ inplace=inplace,+ processor=self.processor,+ quant_nontext_module=self.quant_nontext_module if hasattr(self, "quant_nontext_module") else False,+ **kwargs,)return compressed_model@@ -467,3 +477,19 @@ class MLLMCompressor(BaseCompressor):if vlm_key in layer_name and check_to_quantized(layer_config[layer_name]):return Truereturn quant_nontext_module++ def _immediate_pack(self, name: str):+ if not self.is_immediate_packing: # pylint: disable=E1101+ return+ self.formats[0].immediate_pack(+ name=name,+ model=self.model,+ device=self.device,+ output_dir=self._get_save_folder_name(self.formats[0]),+ mllm=self.mllm,+ layer_config=self.layer_config,+ tokenizer=self.tokenizer,+ processor=self.processor if hasattr(self, "processor") else None,+ image_processor=self.image_processor if hasattr(self, "image_processor") else None,+ quant_nontext_module=self.quant_nontext_module if hasattr(self, "quant_nontext_module") else False,+ )diff --git a/auto_round/export/export_to_gguf/convert.py b/auto_round/export/export_to_gguf/convert.pyindex 0e190184f..5b9252fe2 100644--- a/auto_round/export/export_to_gguf/convert.py+++ b/auto_round/export/export_to_gguf/convert.py@@ -85,7 +85,9 @@ def download_convert_file(redownload=False):f.write(response.text)-def wrapper_model_instance(model_instance, model, layer_config, low_cpu_mem_usage=False, device=None):+def wrapper_model_instance(+ model_instance, model, layer_config, low_cpu_mem_usage=False, device=None, quant_nontext_module=False+):if model_instance.model_arch == gguf.MODEL_ARCH.MMPROJ and model_instance.fname_out.is_dir():model_instance.fname_out = model_instance.fname_out / "mmproj-model.gguf"model_instance.model = model@@ -96,6 +98,7 @@ def wrapper_model_instance(model_instance, model, layer_config, low_cpu_mem_usagmodel_instance.prepare_tensors = partial(prepare_tensors, model_instance)model_instance.device = device+ model_instance.quant_nontext_module = quant_nontext_modulereturn model_instance@@ -528,6 +531,9 @@ def prepare_tensors(cls):elif data_qtype == gguf.GGMLQuantizationType.Q6_K:data_qtype = gguf.GGMLQuantizationType.Q8_0+ if cls.model_arch == gguf.MODEL_ARCH.MMPROJ and cls.quant_nontext_module is False:+ data_qtype = gguf.GGMLQuantizationType.F32+from auto_round.export.export_to_gguf.config import GGML_QUANT_SIZESif data_qtype.name.lower() in GGML_QUANT_SIZES:diff --git a/auto_round/export/export_to_gguf/export.py b/auto_round/export/export_to_gguf/export.pyindex 4fa88ffb1..57d510ac4 100644--- a/auto_round/export/export_to_gguf/export.py+++ b/auto_round/export/export_to_gguf/export.py@@ -74,6 +74,7 @@ def create_model_class(low_cpu_mem_usage=False,model_type=convert_hf_to_gguf.ModelType.TEXT,device="cpu",+ quant_nontext_module: bool = False,):tmp_work_dir = model.name_or_pathos.makedirs(output_dir, exist_ok=True)@@ -118,7 +119,12 @@ def create_model_class(small_first_shard=False,)model_instance = wrapper_model_instance(- model_instance, model=model, layer_config=layer_config, low_cpu_mem_usage=low_cpu_mem_usage, device=device+ model_instance,+ model=model,+ layer_config=layer_config,+ low_cpu_mem_usage=low_cpu_mem_usage,+ device=device,+ quant_nontext_module=quant_nontext_module,)model_instance = handle_special_model(model_instance, model_architecture)return model_instance@@ -136,6 +142,7 @@ def pack_gguf_layer(image_processor=None,model_type=convert_hf_to_gguf.ModelType.TEXT,device="cpu",+ quant_nontext_module=False,):"""Export the model to gguf format."""global gguf_model_instance_global@@ -153,6 +160,7 @@ def pack_gguf_layer(low_cpu_mem_usage=True,model_type=convert_hf_to_gguf.ModelType.TEXT,device=device,+ quant_nontext_module=quant_nontext_module,)]if model_type == convert_hf_to_gguf.ModelType.MMPROJ:@@ -165,6 +173,7 @@ def pack_gguf_layer(low_cpu_mem_usage=True,model_type=convert_hf_to_gguf.ModelType.MMPROJ,device=device,+ quant_nontext_module=quant_nontext_module,))@@ -215,7 +224,14 @@ def pack_gguf_layer(@torch.inference_mode()def save_quantized_as_gguf(- output_dir, model=None, backend="gguf:q4_0", layer_config=None, mllm=False, device="cpu", **kwargs+ output_dir,+ model=None,+ backend="gguf:q4_0",+ layer_config=None,+ mllm=False,+ device="cpu",+ quant_nontext_module=False,+ **kwargs,):"""Export the model to gguf format."""st = time.time()@@ -224,7 +240,13 @@ def save_quantized_as_gguf(if "gguf_model_instance_global" not in globals():gguf_model_instance_global = [create_model_class(- output_dir, model, layer_config, backend, model_type=convert_hf_to_gguf.ModelType.TEXT, device=device+ output_dir,+ model,+ layer_config,+ backend,+ model_type=convert_hf_to_gguf.ModelType.TEXT,+ device=device,+ quant_nontext_module=quant_nontext_module,)]if mllm:@@ -236,6 +258,7 @@ def save_quantized_as_gguf(backend,model_type=convert_hf_to_gguf.ModelType.MMPROJ,device=device,+ quant_nontext_module=quant_nontext_module,))diff --git a/auto_round/formats.py b/auto_round/formats.pyindex a81b34bb7..b8e384c65 100644--- a/auto_round/formats.py+++ b/auto_round/formats.py@@ -43,6 +43,8 @@ from auto_round.utils import (SUPPORTED_FORMATS,check_to_quantized,copy_python_files_from_model_cache,+ find_matching_blocks,+ get_block_names,get_module,logger,)@@ -647,6 +649,10 @@ class GGUFFormat(OutputFormat):elif ar.bits >= 8 and ar.iters != 0:logger.warning_once("`iters=0` is recommended for bits>=8")+ if getattr(ar, "quant_nontext_module", False):+ # for gguf export, leave vl model for gguf itself+ all_blocks = get_block_names(ar.model, False)+ ar.quant_block_list = find_matching_blocks(ar.model, all_blocks, None)return super().check_and_reset_format(ar)def pack_layer(@@ -661,6 +667,7 @@ class GGUFFormat(OutputFormat):image_processor=None,model_type=ModelType.TEXT,device="cpu",+ quant_nontext_module=False,):from auto_round.export.export_to_gguf.export import pack_gguf_layer@@ -675,6 +682,7 @@ class GGUFFormat(OutputFormat):image_processor,model_type,device,+ quant_nontext_module,)def save_quantized(@@ -826,6 +834,7 @@ class GGUFFormat(OutputFormat):tokenizer=None,processor=None,image_processor=None,+ quant_nontext_module: bool = False,**kwargs,):m = get_module(model, name)@@ -843,6 +852,7 @@ class GGUFFormat(OutputFormat):image_processor=image_processor,model_type=model_type,device=device,+ quant_nontext_module=quant_nontext_module,)diff --git a/test/test_cpu/export/test_gguf_format.py b/test/test_cpu/export/test_gguf_format.pyindex 6f736eb98..2e3cde6f0 100644--- a/test/test_cpu/export/test_gguf_format.py+++ b/test/test_cpu/export/test_gguf_format.py@@ -201,6 +201,7 @@ class TestGGUF:iters=0,nsamples=8,disable_opt_rtn=True,+ quant_nontext_module=True,)quantized_model_path = "./saved"autoround.quantize_and_save(output_dir=quantized_model_path, format="gguf:q4_0")@@ -214,6 +215,32 @@ class TestGGUF:shutil.rmtree("./saved", ignore_errors=True)shutil.rmtree(tiny_model_path, ignore_errors=True)+ def test_vlm_gguf_wo_quant_nontext_module(self):+ from ...helpers import save_tiny_model++ model_name = get_model_path("Qwen/Qwen2-VL-2B-Instruct")+ tiny_model_path = save_tiny_model(model_name, "./tmp/tiny_qwen_vl_model_path", num_layers=3, is_mllm=True)+ from auto_round import AutoRoundMLLM++ autoround = AutoRoundMLLM(+ tiny_model_path,+ iters=0,+ nsamples=8,+ disable_opt_rtn=True,+ quant_nontext_module=False,+ )+ quantized_model_path = "./saved"+ autoround.quantize_and_save(output_dir=quantized_model_path, format="gguf:q4_0")+ assert "mmproj-model.gguf" in os.listdir("./saved")+ for file_name in os.listdir(quantized_model_path):+ file_size = os.path.getsize(os.path.join(quantized_model_path, file_name)) / 1024**2+ if file_name == "mmproj-model.gguf":+ assert abs(file_size - 361) < 5.0+ else:+ assert abs(file_size - 264) < 5.0+ shutil.rmtree("./saved", ignore_errors=True)+ shutil.rmtree(tiny_model_path, ignore_errors=True)+def test_qtype_setting(self):# Qwen2.5-0.5B-Instruct no output, token_embed q6_k fallbakc to q8_0 336M# Qwen3-0.6B output q6_k, token_embed q4_0 448Mdiff --git a/test/test_cuda/export/test_gguf.py b/test/test_cuda/export/test_gguf.pyindex 454a2c6ce..09e7451c3 100644--- a/test/test_cuda/export/test_gguf.py+++ b/test/test_cuda/export/test_gguf.py@@ -175,6 +175,7 @@ class TestAutoRound:nsamples=32,iters=0,disable_opt_rtn=True,+ quant_nontext_module=True,)quantized_model_path = "./saved"autoround.quantize_and_save(output_dir=quantized_model_path, format="gguf:q4_k_m")