PTQ-Level Efficiency, QAT-Level Accuracy | 训练后量化的效率,量化感知训练的精度
[mllm] support longcat_next (#1637)
Xin He0821817be6be
diff --git a/auto_round/compressors/base.py b/auto_round/compressors/base.pyindex ffe9205ab..11e30f448 100644--- a/auto_round/compressors/base.py+++ b/auto_round/compressors/base.py@@ -92,6 +92,7 @@ from auto_round.utils import (get_lm_head_name,get_module,global_state,+ hook_ngram_embeddings_on_cpu,htcore,is_auto_device_mapping,is_debug_mode,@@ -2212,8 +2213,15 @@ class BaseCompressor(object):self.model(*data_new, **kwargs)else:self.model(**data_new, **kwargs)- except NotImplementedError:- pass+ except NotImplementedError as error:+ error_msg = str(error)+ # Raise NotImplementedError to fallback to CUDA device+ if "flash_attn::" in error_msg and "CPU" in error_msg:+ raise NotImplementedError(+ "Could not run 'flash_attn::_flash_attn_varlen_forward' with arguments from the 'CPU' backend."+ )+ else:+ passexcept RuntimeError as error:error_msg = str(error)if "The expanded size of the tensor" in str(error_msg) and "must match the existing size" in error_msg:@@ -2261,6 +2269,9 @@ class BaseCompressor(object):layer_names = []if layer_names is None:layer_names = []++ calibrate_on_cpu = False+ cannot_calibrate_on_cpu = Falseif self.low_gpu_mem_usage or (len(block_names) == 1and len(layer_names) == 0@@ -2268,8 +2279,19 @@ class BaseCompressor(object):and (last_cache_name is None or last_cache_name in block_names)):# low_gpu_mem_usage or calibrate only the embedding layer, which is also very fast on CPU- all_inputs = self.cache_inter_data(block_names, nsamples, layer_names=[], last_cache_name=last_cache_name)- else:+ calibrate_on_cpu = True+ try:+ all_inputs = self.cache_inter_data(+ block_names, nsamples, layer_names=[], last_cache_name=last_cache_name+ )+ except NotImplementedError as error:+ error_msg = str(error)+ if "flash_attn::" in error_msg and "CPU" in error_msg:+ cannot_calibrate_on_cpu = True # fallback to GPU when flash attention is not supported on CPU+ else:+ raise error++ if not calibrate_on_cpu or cannot_calibrate_on_cpu:try:if any(p.device.type == "meta" for p in self.model.parameters()):materialize_model_(self.model)@@ -2314,7 +2336,8 @@ class BaseCompressor(object):"No non-CPU device available in accelerate's reported memory. ""Falling back to CPU caching.")-+ # Keep ngram_embeddings on CPU+ has_ngram_embeddings, raw_ngram_embeddings = hook_ngram_embeddings_on_cpu(self.model)new_max_memory = get_balanced_memory(self.model,max_memory=new_max_memory,@@ -2331,8 +2354,9 @@ class BaseCompressor(object):)try:-self.model = dispatch_model(self.model, device_map=device_map)+ if has_ngram_embeddings:+ self.model.model.ngram_embeddings = raw_ngram_embeddingsexcept ValueError as e:if "offload_dir" in e.__str__():logger.warning(@@ -2345,7 +2369,6 @@ class BaseCompressor(object):else:raiseelse:-self.model = self.model.to(self.device)all_inputs = self.cache_inter_data(@@ -2354,7 +2377,9 @@ class BaseCompressor(object):if hasattr(self.model, "hf_device_map") and len(self.model.hf_device_map) > 1:accelerate.hooks.remove_hook_from_submodules(self.model)- except torch.OutOfMemoryError:+ except torch.OutOfMemoryError as e:+ if cannot_calibrate_on_cpu:+ raise ecuda_error_msg = traceback.format_exc()try:logger.info("switch to cpu to cache block inputs")@@ -2415,13 +2440,16 @@ class BaseCompressor(object):calib_bs = self.batch_sizeself.hook_handles = []self._replace_forward()- self.calib(nsamples, calib_bs)- self._recover_forward()+ try:+ self.calib(nsamples, calib_bs)+ finally:+ # Use finally to recover_forward and delattr in case of that+ # self.calib raises NotImplementedError, such as: flash_attn on CPU.+ self._recover_forward()+ for attr in ("last_cache_name", "_cache_target_set", "_cache_seen_targets", "to_cached_layers"):+ if hasattr(self, attr):+ delattr(self, attr)res = self.inputs- del self.last_cache_name- del self._cache_target_set- del self._cache_seen_targets- del self.to_cached_layersif tmp_dtype is not None:self.model = self.model.to(tmp_dtype)@@ -2613,7 +2641,6 @@ class BaseCompressor(object):def _replace_forward(self):"""Replaces the forward function."""-for n, m in self.model.named_modules():if n in self.to_cached_layers and type(m) not in self.supported_types: ##blockm.orig_forward = m.forwarddiff --git a/auto_round/compressors/mllm/processor.py b/auto_round/compressors/mllm/processor.pyindex b42962cbb..ffd6ae387 100644--- a/auto_round/compressors/mllm/processor.py+++ b/auto_round/compressors/mllm/processor.py@@ -29,6 +29,7 @@ Support Matrix"""import os+import sysfrom datetime import datetime, timedeltaimport torch@@ -138,7 +139,7 @@ class HFProcessor(BasicProcessor):conversation[-1]["content"].append({"image": image, "type": "image"})else:conversation.append({"role": content["role"], "content": content["content"]})- if hasattr(self.processor, "chat_template"):+ if hasattr(self.processor, "chat_template") and self.processor.chat_template is not None:text = self.processor.apply_chat_template(conversation, add_generation_prompt=True, tokenize=False, return_dict=False)@@ -165,7 +166,7 @@ class HFProcessor(BasicProcessor):max_length=None,truncation=False,truncation_strategy="text",- **kwargs+ **kwargs,):if isinstance(text, list):@@ -197,6 +198,105 @@ class Qwen2VLProcessor(HFProcessor):return ret+@register_processor("longcat_next")+class LongCatNextProcessor(BasicProcessor):+ """Processor for meituan-longcat/LongCat-Next multimodal models.++ LongCat-Next supports text, image, and audio inputs. Images are referenced+ in the conversation text via ``<longcat_img_start>URL<longcat_img_end>``+ tags. The HuggingFace ``processor`` returns a tuple of+ ``(text_inputs, visual_inputs, audio_inputs)`` instead of a single dict,+ so this class unpacks them into a flat dict suitable for ``model.forward()``.+ """++ IMAGE_TOKEN = "<image>"+ LONGCAT_IMG_START = "<longcat_img_start>"+ LONGCAT_IMG_END = "<longcat_img_end>"++ def post_init(self, model, tokenizer, processor=None, image_processor=None, use_rtn=False, **kwargs):+ assert tokenizer is not None, "tokenizer should not be None"+ assert processor is not None, "processor should not be None"+ self.model = model+ self.tokenizer = tokenizer+ self.tokenizer.fix_mistral_regex = True+ self.processor = processor+ if image_processor is not None:+ self.image_processor = image_processor+ else:+ self.image_processor = self.default_image_processor+ self.use_rtn = use_rtn+ # LongCat-Next get_input() relies on the HF processor output directly and+ # does not use self.image_processor in the current input path. Do not+ # enforce image_processor availability here so text-only calibration can+ # still proceed when AutoImageProcessor loading is unavailable.++ # build generation_config from model file because the code is on hub.+ model_module = sys.modules[self.model.__module__]+ GenerationConfig = model_module.GenerationConfig+ LongcatNextForCausalLMGenerationStatus = model_module.LongcatNextForCausalLMGenerationStatus++ self.visual_generation_config = GenerationConfig(**self.model.generation_config.visual_generation_config)+ self.audio_generation_config = GenerationConfig(**self.model.generation_config.audio_generation_config)+ self.multimodal_generation_status = LongcatNextForCausalLMGenerationStatus(+ self.visual_generation_config, self.audio_generation_config+ )++ def get_input(self, text, images, squeeze=True, max_length=None, truncation=False, **kwargs):+ if isinstance(text, list):+ # text is a list of message dicts (conversation format)+ messages = []+ for content in text:+ msg = {"role": content["role"], "content": content["content"]}+ if self.IMAGE_TOKEN in content["content"] and images is not None:+ # Replace generic <image> token with LongCat image tags wrapping the URL+ msg["content"] = content["content"].replace(+ self.IMAGE_TOKEN,+ f"{self.LONGCAT_IMG_START}{images}{self.LONGCAT_IMG_END}",+ )+ messages.append(msg)++ text_input = self.tokenizer.apply_chat_template(messages, tokenize=False, add_generation_prompt=True)+ else:+ # Plain string input+ if max_length is not None:+ text_input = self.tokenizer.decode(self.tokenizer(text).input_ids[:max_length])+ else:+ text_input = text++ # LongCat processor returns (text_inputs, visual_inputs, audio_inputs)+ text_inputs, visual_inputs, audio_inputs = self.processor(text=text_input, return_tensors="pt")++ ret = {}+ for key in text_inputs:+ ret[key] = text_inputs[key]+ if visual_inputs is not None:+ ret["visual_inputs"] = visual_inputs+ if audio_inputs is not None:+ ret["audio_inputs"] = audio_inputs+ ret["multimodal_generation_status"] = self.multimodal_generation_status+ ret["visual_generation_config"] = self.visual_generation_config++ return ret++ @staticmethod+ def data_collator(batch):+ if len(batch) == 1:+ return batch[0]++ batched_data = {}+ for key in batch[0].keys():+ values = [item[key] for item in batch]+ if isinstance(values[0], torch.Tensor):+ try:+ batched_data[key] = torch.stack(values)+ except (RuntimeError, TypeError):+ batched_data[key] = values+ else:+ batched_data[key] = values++ return batched_data++@register_processor("qwen2_5_omni")class Qwen2_5OmniProcessor(HFProcessor):"""Processor for Qwen2.5-Omni multimodal models.@@ -438,7 +538,7 @@ class Mistral3Processor(BasicProcessor):max_length=None,truncation=False,truncation_strategy="text",- **kwargs+ **kwargs,):from mistral_common.protocol.instruct.request import ChatCompletionRequest # pylint: disable=E0401diff --git a/auto_round/compressors/mllm/template.py b/auto_round/compressors/mllm/template.pyindex 541dfcef9..78abc6424 100644--- a/auto_round/compressors/mllm/template.py+++ b/auto_round/compressors/mllm/template.py@@ -127,6 +127,7 @@ _register_template("deepseek_vl_v2", default_dataset="NeelNanda/pile-10k", proce_register_template("mistral3", default_dataset="NeelNanda/pile-10k", processor=PROCESSORS["hf"])_register_template("mistral3_2", default_dataset="liuhaotian/llava", processor=PROCESSORS["mistral3_2"])_register_template("gemma3", default_dataset="NeelNanda/pile-10k", processor=PROCESSORS["hf"])+_register_template("longcat_next", default_dataset="liuhaotian/llava", processor=PROCESSORS["longcat_next"])def load_template(path: str):diff --git a/auto_round/compressors/shard_writer.py b/auto_round/compressors/shard_writer.pyindex 3e0f63c47..f061a92b9 100644--- a/auto_round/compressors/shard_writer.py+++ b/auto_round/compressors/shard_writer.py@@ -131,6 +131,27 @@ class ShardWriter:self.current_shard_tensors[name] = tensorself.current_shard_size += t_size+ def _handle_tied_weights(self):+ """+ Detects tied weights in the current shard and ensures they are only saved once.+ This is done by tracking storage pointers of tensors and skipping duplicates.+ """+ from collections import defaultdict++ storage_map = set()+ filtered_tensors = {}++ for name, tensor in self.current_shard_tensors.items():+ if not isinstance(tensor, torch.Tensor):+ filtered_tensors[name] = tensor+ continue++ ptr = tensor.untyped_storage().data_ptr()+ if ptr not in storage_map:+ storage_map.add(ptr)+ filtered_tensors[name] = tensor+ self.current_shard_tensors = filtered_tensors+def _flush_shard(self):if not self.current_shard_tensors:return@@ -138,6 +159,7 @@ class ShardWriter:self.shard_counter += 1tmp_name = f"model-shard-{self.shard_counter:05d}.{self.shard_suffix}"tmp_path = os.path.join(self.output_dir, tmp_name)+ self._handle_tied_weights()if self.use_safetensors:from safetensors.torch import save_filediff --git a/auto_round/utils/model.py b/auto_round/utils/model.pyindex e60e595ec..85f101cce 100644--- a/auto_round/utils/model.py+++ b/auto_round/utils/model.py@@ -36,6 +36,8 @@ from auto_round.utils.weight_handler import (is_quantized_input_module,)+FIX_MISTRAL_REGEX_MODEL_TYPE_LIST = ["longcat_next"]+def clean_module_parameter(submodule: torch.nn.Module, param_name: str) -> None:"""This function is recommended to be used instead of module.weight = None.@@ -599,6 +601,7 @@ def mllm_load_model(tokenizer = AutoTokenizer.from_pretrained(pretrained_model_name_or_path,trust_remote_code=trust_remote_code,+ fix_mistral_regex=True if model_type in FIX_MISTRAL_REGEX_MODEL_TYPE_LIST else False,**processor_load_kwargs,)processor = AutoProcessor.from_pretrained(@@ -874,7 +877,8 @@ def get_block_names(model, quant_vision=False):block_names[i].append(target_m[0] + "." + n)return block_names- def _get_vlm_block_names(model, quant_vision=False):+ def _get_vlm_block_names(model, quant_vision=False, ignore_audio=True):+ # Since calibration dataset doesn't contain audio data, audio-related blocks will be ignored by default.if (hasattr(model, "config")and hasattr(model.config, "model_type")@@ -884,10 +888,13 @@ def get_block_names(model, quant_vision=False):block_names = []target_modules = []vision_blocks_tuple = ("vision", "visual", "image", "img")+ audio_blocks_tuple = ("audio", "speech", "wav", "waveform")target_modules = _search_block("", model)for i, target_m in enumerate(target_modules):if quant_vision or all(key not in target_m[0].lower() for key in (vision_blocks_tuple)):+ if ignore_audio and any(key in target_m[0].lower() for key in audio_blocks_tuple):+ continueblock_names.append([])for n, m in target_m[1].named_children():block_names[-1].append(target_m[0] + "." + n)@@ -1897,3 +1904,22 @@ def wrap_block_forward_positional_to_kwargs(base_hook):return base_hook(m, hidden_states, *positional_inputs, **kwargs)return forward+++def hook_ngram_embeddings_on_cpu(model):+ has_ngram_embeddings = hasattr(model, "model") and hasattr(model.model, "ngram_embeddings")+ if has_ngram_embeddings:+ raw_ngram_embeddings = model.model.ngram_embeddings++ def hook_input_output_device_for_cpu_module(module):+ from accelerate.hooks import AlignDevicesHook, add_hook_to_module++ hook = AlignDevicesHook(+ io_same_device=True,+ execution_device="cpu",+ )++ add_hook_to_module(module, hook)++ hook_input_output_device_for_cpu_module(raw_ngram_embeddings)+ return has_ngram_embeddings, raw_ngram_embeddings if has_ngram_embeddings else None