PTQ-Level Efficiency, QAT-Level Accuracy | 训练后量化的效率,量化感知训练的精度
support vlm models for gguf format
Heng Guoa2f3f016cd8f
diff --git a/README.md b/README.mdindex 046b23a22..68da05f8e 100644--- a/README.md+++ b/README.md@@ -34,17 +34,7 @@ and [fbaldassarri](https://huggingface.co/fbaldassarri).## What's New-[//]: # (* [2025.07] AutoRound now offers experimental support for the widely used **GGUF** format. We currently recommend using)--[//]: # ( RTN mode (--iters 0) for all the bits except 3. A more advanced algorithm tailored for some specific configurations is likely)--[//]: # ( to be introduced in the upcoming release. Example models are)--[//]: # ( available on the Intel Hugging Face space, including)--[//]: # ( [Intel/Qwen3-235B-A22B-q2ks-mixed-AutoRound-inc-v0](https://huggingface.co/Intel/Qwen3-235B-A22B-q2ks-mixed-AutoRound-inc-v0))--[//]: # ( and [Intel/DeepSeek-R1-0528-q2ks-mixed-AutoRound-inc-v0](https://huggingface.co/Intel/DeepSeek-R1-0528-q2ks-mixed-AutoRound-inc-v0))+* [2025.07] AutoRound now offers experimental support for **GGUF** format, and recommends using RTN mode (--iters 0) for all bits other than 3 bits. A more advanced algorithm tailored for specific configurations may be available in upcoming release. Example models: [Intel/Qwen3-235B-A22B-q2ks-mixed-AutoRound-inc-v0](https://huggingface.co/Intel/Qwen3-235B-A22B-q2ks-mixed-AutoRound-inc-v0) and [Intel/DeepSeek-R1-0528-q2ks-mixed-AutoRound-inc-v0](https://huggingface.co/Intel/DeepSeek-R1-0528-q2ks-mixed-AutoRound-inc-v0).* [2025.05] AutoRound provides some recipes for **DeepSeek-R1-0528**, please referto [DeepSeek-R1-0528-int2-mixed-sym-inc](https://huggingface.co/Intel/DeepSeek-R1-0528-int2-mixed-sym-inc), [DeepSeek-R1-0528-int4-sym-gptq-inc](https://huggingface.co/Intel/DeepSeek-R1-0528-int4-gptq-inc-auto-round)and [DeepSeek-R1-0528-int4-asym-awq-inc](https://huggingface.co/Intel/DeepSeek-R1-0528-int4-awq-inc-auto-round) fordiff --git a/auto_round/autoround.py b/auto_round/autoround.pyindex b52f7bc0d..8acb21d00 100644--- a/auto_round/autoround.py+++ b/auto_round/autoround.py@@ -25,7 +25,7 @@ from torch import autocastfrom tqdm import tqdmimport accelerate-from auto_round.export.export_to_gguf.config import GGUF_CONFIG, GGUF_INNER_CONFIG+from auto_round.export.export_to_gguf.config import GGUF_CONFIG, GGUF_INNER_CONFIG, ModelTypefrom auto_round.wrapper import WrapperMultiblock, wrapper_block, unwrapper_block, WrapperLinear, unwrapper_layerfrom auto_round.utils import (CpuInfo,@@ -173,6 +173,7 @@ class AutoRound(object):model_kwargs: dict = None,**kwargs,):+ self.vlm = kwargs.pop("vlm") if "vlm" in kwargs else Falseif kwargs:logger.warning(f"unrecognized keys {list(kwargs.keys())} were passed. Please check them.")self.quantized = False@@ -260,6 +261,7 @@ class AutoRound(object):self.disable_opt_rtn = disable_opt_rtn+torch.set_printoptions(precision=3, sci_mode=True)self.check_configs()if self.act_bits <= 8 and self.amp_dtype == torch.float16:@@ -515,7 +517,9 @@ class AutoRound(object):Returns:list: A list of validated and updated formats."""- _gguf_args_check(self, format)+ _gguf_args_check(self, format, model_type=ModelType.TEXT)+ if self.vlm:+ _gguf_args_check(self, format, model_type=ModelType.MMPROJ)formats = format.replace("q*_", f"q{self.bits}_").replace(' ', '').split(',')from auto_round.utils import SUPPORTED_FORMATS@@ -947,7 +951,8 @@ class AutoRound(object):if not tie_word_embeddings:lm_head_name: str = get_lm_head_name(self.model)config: dict[str, Any] = GGUF_CONFIG[GGUF_CONFIG[target_format]["lm_head"]]- check_fixed_by_user = self.layer_config[lm_head_name].get("fixed_by_user", False)+ check_fixed_by_user = self.layer_config[lm_head_name].get(+ "fixed_by_user", False) if lm_head_name in self.layer_config else Noneself._apply_config_to_layer(lm_head_name, config, check_fixed_by_user=check_fixed_by_user)return True@@ -1060,10 +1065,17 @@ class AutoRound(object):if has_gguf:from auto_round.export.export_to_gguf.export import pack_gguf_layeroutput_dir = self.get_save_folder_name(self.formats[0])+ model_type = ModelType.MMPROJ if self.vlm else ModelType.TEXTpack_gguf_layer(- name, self.model, self.formats[0],- output_dir, self.layer_config, self.tokenizer- )+ name,+ self.model,+ self.formats[0],+ output_dir,+ self.layer_config,+ self.tokenizer,+ processor=self.processor if hasattr(self, "processor") else None,+ image_processor=self.image_processor if hasattr(self, "image_processor") else None,+ model_type=model_type)else:PACKING_LAYER_WITH_FORMAT[target_backend](name, self.model, self.formats[0]@@ -1220,8 +1232,11 @@ class AutoRound(object):if len(self.formats) == 1 and self.formats[0] == "fake":only_gguf = Falseif only_gguf:- self.layer_config, gguf_format_config = get_layer_config_by_gguf_format(self.layer_config, self.formats,- self.model)+ self.layer_config, gguf_format_config = get_layer_config_by_gguf_format(+ self.layer_config, self.formats, self.model, model_type=ModelType.TEXT)+ if self.vlm:+ self.layer_config, gguf_format_config = get_layer_config_by_gguf_format(+ self.layer_config, self.formats, self.model, model_type=ModelType.MMPROJ)# Determine if immediate packing is requiredformats = self.formatsif (len(formats) == 1 and@@ -2339,8 +2354,17 @@ class AutoRound(object):if has_gguf:from auto_round.export.export_to_gguf.export import pack_gguf_layeroutput_dir = self.get_save_folder_name(self.formats[0])- pack_gguf_layer(tmp_m.tmp_name, self.model, self.formats[0], output_dir, self.layer_config,- self.tokenizer)+ model_type = ModelType.MMPROJ if self.vlm else ModelType.TEXT+ pack_gguf_layer(+ tmp_m.tmp_name,+ self.model,+ self.formats[0],+ output_dir,+ self.layer_config,+ self.tokenizer,+ processor=self.processor if hasattr(self, "processor") else None,+ image_processor=self.image_processor if hasattr(self, "image_processor") else None,+ model_type=model_type)else:PACKING_LAYER_WITH_FORMAT[target_backend](tmp_m.tmp_name, self.model, self.formats[0])pbar.set_description(f"Quantizing done")diff --git a/auto_round/data_type/nvfp.py b/auto_round/data_type/nvfp.pyindex 76c4eb92d..d1293e899 100644--- a/auto_round/data_type/nvfp.py+++ b/auto_round/data_type/nvfp.py@@ -51,8 +51,8 @@ def get_reciprocal(x):FLOAT4_E2M1_MAX = 6.0-FLOAT8_E4M3_MAX = torch.finfo(torch.float8_e4m3fn).max-FLOAT8_E4M3_MIN = torch.finfo(torch.float8_e4m3fn).min+FLOAT8_E4M3_MAX = torch.finfo(torch.float8_e4m3fn).max if hasattr(torch, "float8_e4m3fn") else 448+FLOAT8_E4M3_MIN = torch.finfo(torch.float8_e4m3fn).min if hasattr(torch, "float8_e4m3fn") else -448def ref_nvfp4_quant(x, global_scale, block_size=16, v=0):diff --git a/auto_round/export/export_to_gguf/config.py b/auto_round/export/export_to_gguf/config.pyindex 484c5f033..d2b6a821e 100644--- a/auto_round/export/export_to_gguf/config.py+++ b/auto_round/export/export_to_gguf/config.py@@ -11,6 +11,11 @@# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.# See the License for the specific language governing permissions and# limitations under the License.+from enum import IntEnum++class ModelType(IntEnum):+ TEXT = 1+ MMPROJ = 2GGUF_INNER_CONFIG = {}diff --git a/auto_round/export/export_to_gguf/convert.py b/auto_round/export/export_to_gguf/convert.pyindex 4b074d762..669f81d22 100644--- a/auto_round/export/export_to_gguf/convert.py+++ b/auto_round/export/export_to_gguf/convert.py@@ -542,6 +542,8 @@ class ModelBase(OriModel):dry_run: bool = False,small_first_shard: bool = False,hparams: dict[str, Any] | None = None):+ if self.model_arch == gguf.MODEL_ARCH.MMPROJ and fname_out.is_dir():+ fname_out = fname_out / "mmproj-model.gguf"self.model = modelself.layer_config = layer_configself.low_cpu_mem_usage = self._need_low_cpu_mem(low_cpu_mem_usage)@@ -618,8 +620,14 @@ class ModelBase(OriModel):data_torch, data_qtype.name.lower(), scale, zp, wmin=wmin, d_scale=d_scale, d_wmin=d_wmin, device=device)return data- def _quant_data(self, data_torch, data_qtype, name, bid):+ def _quant_data(self, data_torch, data_qtype, name, modify_name, bid):suffix = '.weight'+ if torch.cuda.is_available():+ device = "cuda"+ elif torch.xpu.is_available():+ device = "xpu"+ else:+ device = "cpu"if suffix in name:layer_name = name[:-len(suffix)]module = get_module(self.model, layer_name)@@ -630,17 +638,12 @@ class ModelBase(OriModel):if hasattr(module, attr) and getattr(module, attr) is not None:attr_tensor = getattr(module, attr)ori_shape = attr_tensor.shape- attr_tensor = self.modify_tensors(attr_tensor.reshape(bs, -1), name, bid)[0][1]+ # attr_tensor = self.modify_tensors(attr_tensor.reshape(bs, -1), modify_name, bid)[0][1]+ attr_tensor = self.modify_tensors(attr_tensor, modify_name, bid)[0][1]attr_tensor = attr_tensor.reshape(ori_shape)setattr(module, attr, attr_tensor)scale = module.scale if hasattr(module, "scale") else Nonezp = module.zp if hasattr(module, "zp") else None- if torch.cuda.is_available():- device = "cuda"- elif torch.xpu.is_available():- device = "xpu"- else:- device = "cpu"data_torch = data_torch.to(torch.float32)scale = scale.to(torch.float32) if isinstance(scale, torch.Tensor) else scalezp = zp.to(torch.float32) if isinstance(zp, torch.Tensor) else zp@@ -670,20 +673,20 @@ class ModelBase(OriModel):data_qtype = gguf.GGMLQuantizationType.F32 ##FP16 has issues at inferencedata = data_torch.to(torch.float32).squeeze().cpu().numpy()else:+ # for Llama-4# if data_torch.dtype == torch.float32:# data_qtype = gguf.GGMLQuantizationType.F32# else:# data_qtype = gguf.GGMLQuantizationType.F16# data = data_torch.squeeze().cpu().numpy()- data_qtype = gguf.GGMLQuantizationType.F32- data = data_torch.to(torch.float32).squeeze().cpu().numpy()+ # data_qtype = gguf.GGMLQuantizationType.F32+ # data = data_torch.to(torch.float32).squeeze().cpu().numpy()+ data = ggml_quant(data_torch, data_qtype.name.lower(), device=device)return data, data_qtypedef get_qtype_by_layer_config(self, layer_config, name, data_qtype):name = name[:-len('.weight')]if name not in layer_config or layer_config[name]['bits'] >= 16:- if isinstance(data_qtype, bool):- return gguf.GGMLQuantizationType.F32return data_qtypebits = layer_config[name]['bits']super_bits = layer_config[name]["super_bits"]@@ -738,6 +741,42 @@ class ModelBase(OriModel):# raise ValueError(f"Unknown file type: {data_qtype}")# return data_qtype+ def _special_name_handle(self, name):+ # for Qwen2VL+ def remove_prefix(name, key_list):+ for key in key_list:+ if key in name and not name.startswith(key):+ name = ".".join(name.split(".")[1:])+ break+ return name+ if self.model_arch == gguf.MODEL_ARCH.QWEN2VL:+ name = name.replace("language_model.", "")+ visual_keys = ["thinker", "visual", "audio", "talker", "token2wav"]+ name = remove_prefix(name, visual_keys)+ if isinstance(self, Qwen2VLVisionModel):+ if "visual." in name and not name.startswith("visual."):+ name = ".".join(name.split(".")[1:])++ # for gemma3+ if self.model_arch == gguf.MODEL_ARCH.GEMMA3 or isinstance(self, Gemma3VisionModel):+ visual_keys = ["multi_modal_projector", "vision_tower", "multimodal_projector", "vision_model"]+ name = remove_prefix(name, visual_keys)++ # for LlavaForConditionalGeneration+ if self.model_arch == gguf.MODEL_ARCH.LLAMA:+ name = name.replace("language_model.", "")+ # name = name.replace("model.", "")+ if isinstance(self, LlavaVisionModel):+ visual_keys = ["multi_modal_projector", "vision_tower"]+ name = remove_prefix(name, visual_keys)++ # for InternVisionModel+ if isinstance(self, InternVisionModel):+ visual_keys = ["vision_model"]+ name = remove_prefix(name, visual_keys)++ return name+def prepare_tensors(self):max_name_len = max(len(s) for _, s in self.tensor_map.mapping.values()) + len(".weight,")@@ -768,7 +807,9 @@ class ModelBase(OriModel):breakclean_weight_list = []- for new_name, data_torch in (self.modify_tensors(data_torch, name, bid)):++ modify_name = self._special_name_handle(name)+ for new_name, data_torch in (self.modify_tensors(data_torch, modify_name, bid)):data = data_torch.squeeze().cpu().numpy()# if data ends up empty, it means data_torch was a scalar tensor -> restore@@ -813,40 +854,53 @@ class ModelBase(OriModel):name = self.get_moe_name(name, new_name)clean_weight_list.append(name)# get data_qtype by layer_config- data_qtype = self.get_qtype_by_layer_config(self.layer_config, name, data_qtype)+ if isinstance(data_qtype, bool):+ data_qtype = self.get_qtype_by_layer_config(self.layer_config, name, data_qtype)# # No override (data_qtype is False), or wants to be quantized (data_qtype is True)- # if isinstance(data_qtype, bool):- # if self.ftype == gguf.LlamaFileType.ALL_F32:- # data_qtype = gguf.GGMLQuantizationType.F32- # elif self.ftype == gguf.LlamaFileType.MOSTLY_F16:- # data_qtype = gguf.GGMLQuantizationType.F16- # elif self.ftype == gguf.LlamaFileType.MOSTLY_BF16:- # data_qtype = gguf.GGMLQuantizationType.BF16- # elif self.ftype == gguf.LlamaFileType.MOSTLY_Q8_0:- # data_qtype = gguf.GGMLQuantizationType.Q8_0- # elif self.ftype == gguf.LlamaFileType.MOSTLY_Q4_0:- # data_qtype = gguf.GGMLQuantizationType.Q4_0- # elif self.ftype == gguf.LlamaFileType.MOSTLY_Q4_1:- # data_qtype = gguf.GGMLQuantizationType.Q4_1- # elif self.ftype == gguf.LlamaFileType.MOSTLY_Q5_0:- # data_qtype = gguf.GGMLQuantizationType.Q5_0- # elif self.ftype == gguf.LlamaFileType.MOSTLY_Q5_1:- # data_qtype = gguf.GGMLQuantizationType.Q5_1- # elif self.ftype == gguf.LlamaFileType.MOSTLY_Q2_K_S:- # data_qtype = gguf.GGMLQuantizationType.Q2_K- # elif self.ftype == gguf.LlamaFileType.MOSTLY_Q3_K_S:- # data_qtype = gguf.GGMLQuantizationType.Q3_K- # elif self.ftype == gguf.LlamaFileType.MOSTLY_Q4_K_S:- # data_qtype = gguf.GGMLQuantizationType.Q4_K- # elif self.ftype == gguf.LlamaFileType.MOSTLY_Q4_K_M:- # data_qtype = gguf.GGMLQuantizationType.Q4_K- # elif self.ftype == gguf.LlamaFileType.MOSTLY_Q5_K_S:- # data_qtype = gguf.GGMLQuantizationType.Q5_K- # elif self.ftype == gguf.LlamaFileType.MOSTLY_Q6_K:- # data_qtype = gguf.GGMLQuantizationType.Q6_K- # else:- # raise ValueError(f"Unknown file type: {self.ftype.name}")+ if isinstance(data_qtype, bool):+ if self.ftype == gguf.LlamaFileType.ALL_F32:+ data_qtype = gguf.GGMLQuantizationType.F32+ elif self.ftype == gguf.LlamaFileType.MOSTLY_F16:+ data_qtype = gguf.GGMLQuantizationType.F16+ elif self.ftype == gguf.LlamaFileType.MOSTLY_BF16:+ data_qtype = gguf.GGMLQuantizationType.BF16+ elif self.ftype == gguf.LlamaFileType.MOSTLY_Q8_0:+ data_qtype = gguf.GGMLQuantizationType.Q8_0+ elif self.ftype == gguf.LlamaFileType.MOSTLY_Q4_0:+ data_qtype = gguf.GGMLQuantizationType.Q4_0+ elif self.ftype == gguf.LlamaFileType.MOSTLY_Q4_1:+ data_qtype = gguf.GGMLQuantizationType.Q4_1+ elif self.ftype == gguf.LlamaFileType.MOSTLY_Q5_0:+ data_qtype = gguf.GGMLQuantizationType.Q5_0+ elif self.ftype == gguf.LlamaFileType.MOSTLY_Q5_1:+ data_qtype = gguf.GGMLQuantizationType.Q5_1+ elif self.ftype == gguf.LlamaFileType.MOSTLY_Q2_K_S or \+ self.ftype == gguf.LlamaFileType.MOSTLY_Q2_K:+ data_qtype = gguf.GGMLQuantizationType.Q2_K+ elif self.ftype == gguf.LlamaFileType.MOSTLY_Q3_K_S or \+ self.ftype == gguf.LlamaFileType.MOSTLY_Q3_K_M or \+ self.ftype == gguf.LlamaFileType.MOSTLY_Q3_K_L:+ data_qtype = gguf.GGMLQuantizationType.Q3_K+ elif self.ftype == gguf.LlamaFileType.MOSTLY_Q4_K_S or \+ self.ftype == gguf.LlamaFileType.MOSTLY_Q4_K_M:+ data_qtype = gguf.GGMLQuantizationType.Q4_K+ elif self.ftype == gguf.LlamaFileType.MOSTLY_Q5_K_S \+ or self.ftype == gguf.LlamaFileType.MOSTLY_Q5_K_M:+ data_qtype = gguf.GGMLQuantizationType.Q5_K+ elif self.ftype == gguf.LlamaFileType.MOSTLY_Q6_K:+ data_qtype = gguf.GGMLQuantizationType.Q6_K+ else:+ raise ValueError(f"Unknown file type: {self.ftype.name}")++ if data_qtype.name.endswith("_K") and data_torch.shape[-1] % 256 != 0:+ if data_qtype in [gguf.GGMLQuantizationType.Q2_K, gguf.GGMLQuantizationType.Q3_K,+ gguf.GGMLQuantizationType.Q4_K]:+ data_qtype = gguf.GGMLQuantizationType.Q5_0+ elif data_qtype == gguf.GGMLQuantizationType.Q5_K:+ data_qtype = gguf.GGMLQuantizationType.Q5_0+ elif data_qtype == gguf.GGMLQuantizationType.Q6_K:+ data_qtype = gguf.GGMLQuantizationType.Q8_0if isinstance(data_qtype, bool) or data_qtype in [gguf.GGMLQuantizationType.F16, gguf.GGMLQuantizationType.BF16, gguf.GGMLQuantizationType.F32@@ -877,7 +931,7 @@ class ModelBase(OriModel):k_b, v_b = torch.split(kv_b, [qk_nope_head_dim, v_head_dim], dim=1)k_b = k_b.transpose(1, 2)- name_kb = name.replace("kv_b_proj", "k_b_proj")+ name_kb = modify_name.replace("kv_b_proj", "k_b_proj")if new_name == self.map_tensor_name(name_kb):attr_list[attr] = k_belse:@@ -885,7 +939,7 @@ class ModelBase(OriModel):data = self._quant_data_with_args(data_torch, data_qtype, **attr_list)# for MOE model- elif len(data_torch.shape) == 3:+ elif len(data_torch.shape) == 3 and len(re.findall("\d+", name)) == 2:new_data = []for idx, arr in enumerate(data_torch):arr_name = name.split('.')@@ -893,12 +947,12 @@ class ModelBase(OriModel):if arr_name[i].isdecimal() and int(arr_name[i]) == (data_torch.shape[0] - 1):arr_name[i] = str(idx)arr_name = ".".join(arr_name)- arr, data_qtype = self._quant_data(arr, data_qtype, arr_name, bid)+ arr, data_qtype = self._quant_data(arr, data_qtype, arr_name, modify_name, bid)new_data.append(arr)data = np.array(new_data)del new_dataelse:- data, data_qtype = self._quant_data(data_torch, data_qtype, name, bid)+ data, data_qtype = self._quant_data(data_torch, data_qtype, name, modify_name, bid)shape = gguf.quant_shape_from_byte_shape(data.shape, data_qtype) if data.dtype == np.uint8 else data.shape@@ -917,6 +971,13 @@ class ModelBase(OriModel):if self.low_cpu_mem_usage:for weight_name in clean_weight_list:module = get_module(self.model, ".".join(weight_name.split(".")[:-1]))+ for key in ["scale", "zp", "d_scale", "wmin", "d_wmin", "imatrix"]:+ if hasattr(module, key):+ setattr(module, key, None)+ if hasattr(module, "w_" + key):+ setattr(module, "w_" + key, None)+ if self.model_arch == gguf.MODEL_ARCH.LLAMA and "embed_tokens.weight" in weight_name:+ continueclean_module_parameter(module, weight_name.split(".")[-1])@@ -1062,10 +1123,10 @@ class TextModel(ModelBase):)seems_special = seems_special or (token_text.startswith("<|") and token_text.endswith("|>"))- seems_special = seems_special or (token_text.startswith("<|") and token_text.endswith("|>"))+ seems_special = seems_special or (token_text.startswith("<|") and token_text.endswith("|>"))# TODO: should these be marked as UNUSED instead? (maybe not)- seems_special = seems_special or (token_text.startswith("<unused") and token_text.endswith(">"))+ seems_special = seems_special or (token_text.startswith("<unused") and token_text.endswith(">"))return seems_special@@ -1109,7 +1170,7 @@ class TextModel(ModelBase):else:# NOTE: this was added for Gemma.# Encoding and decoding the tokens above isn't sufficient for this case.- token = token.replace(b"\xe2\x96\x81".decode("utf-8"), " ")+ token = token.replace(b"\xe2\x96\x81".decode("utf-8"), " ")toktypes.append(gguf.TokenType.USER_DEFINED)else:toktypes.append(gguf.TokenType.NORMAL)@@ -3283,6 +3344,7 @@ class Qwen2VLModel(TextModel):del bid # unusedif name.startswith("thinker."):name = name.replace("thinker.", "")+if name.startswith("visual") or name.startswith("audio") or \name.startswith("talker") or name.startswith("token2wav"):# skip multimodal tensors@@ -6630,7 +6692,8 @@ class ChatGLMModel(TextModel):def modify_tensors(self, data_torch: Tensor, name: str, bid: int | None) -> Iterable[tuple[str, Tensor]]:del bid # unused- if name.endswith(".rotary_pos_emb.inv_freq") or name.startswith("model.vision."):+ if name.endswith(".rotary_pos_emb.inv_freq") or name.startswith("model.vision.") \+ or name.startswith("transformers.vision."):return []name = name.removeprefix("transformer.")diff --git a/auto_round/export/export_to_gguf/export.py b/auto_round/export/export_to_gguf/export.pyindex b75fe51f9..6b92695e7 100644--- a/auto_round/export/export_to_gguf/export.py+++ b/auto_round/export/export_to_gguf/export.py@@ -18,9 +18,11 @@ import shutilimport torchfrom pathlib import Pathimport time-from auto_round.export.export_to_gguf.convert import ModelBase+from auto_round.export.export_to_gguf.convert import ModelBase, ModelType, get_model_architecturefrom auto_round.utils import logger, LazyImport, get_block_names, flatten_list, check_to_quantized, get_module+TMP_DIR_NAME = "tmp_dir"+gguf = LazyImport("gguf")FTYPE_MAP: dict[str, gguf.LlamaFileType] = {@@ -46,25 +48,27 @@ FTYPE_MAP: dict[str, gguf.LlamaFileType] = {}-def create_model_class(output_dir, model, layer_config, backend="gguf:q4_0", low_cpu_mem_usage=False):- tmp_work_dir = Path(os.path.join(output_dir, 'tmp_dir'))+def create_model_class(+ output_dir, model, layer_config, backend="gguf:q4_0", low_cpu_mem_usage=False, model_type=ModelType.TEXT):+ tmp_work_dir = Path(os.path.join(output_dir, TMP_DIR_NAME))with torch.inference_mode():hparams = ModelBase.load_hparams(tmp_work_dir)- if "architectures" in hparams:- model_architecture = hparams["architectures"][0]- else:- model_architecture = type(model).__name__- if model_architecture not in ModelBase._model_classes:- if model_architecture.replace("CausalLM", "ConditionalGeneration") in ModelBase._model_classes:- model_architecture = model_architecture.replace("CausalLM", "ConditionalGeneration")- elif model_architecture.replace("ConditionalGeneration", "CausalLM") in ModelBase._model_classes:- model_architecture = model_architecture.replace("ConditionalGeneration", "CausalLM")+ model_architecture = get_model_architecture(hparams=hparams, model_type=model_type)+ # if "architectures" in hparams:+ # model_architecture = hparams["architectures"][0]+ # else:+ # model_architecture = type(model).__name__+ # if model_architecture not in ModelBase._model_classes:+ # if model_architecture.replace("CausalLM", "ConditionalGeneration") in ModelBase._model_classes:+ # model_architecture = model_architecture.replace("CausalLM", "ConditionalGeneration")+ # elif model_architecture.replace("ConditionalGeneration", "CausalLM") in ModelBase._model_classes:+ # model_architecture = model_architecture.replace("ConditionalGeneration", "CausalLM")try:- model_class = ModelBase.from_model_architecture(model_architecture)+ model_class = ModelBase.from_model_architecture(model_architecture, model_type=model_type)except NotImplementedError:logger.error(f"Model {model_architecture} is not supported")sys.exit(1)- model_class = ModelBase.from_model_architecture(model_architecture)+ model_class = ModelBase.from_model_architecture(model_architecture, model_type=model_type)model_name = model.name_or_path.split('/')if len(model_name[-1]) == 0:model_name = model_name[-2]@@ -93,22 +97,40 @@ def create_model_class(output_dir, model, layer_config, backend="gguf:q4_0", low@torch.inference_mode()-def pack_gguf_layer(name, model, backend, output_dir, layer_config, tokenizer):+def pack_gguf_layer(+ name,+ model,+ backend,+ output_dir,+ layer_config,+ tokenizer,+ processor=None,+ image_processor=None,+ model_type=ModelType.TEXT):"""Export the model to gguf format."""global gguf_model_instance_global- tmp_work_dir = Path(os.path.join(output_dir, 'tmp_dir'))+ tmp_work_dir = Path(os.path.join(output_dir, TMP_DIR_NAME))if output_dir is not None and os.path.exists(output_dir) and not os.path.exists(tmp_work_dir):logger.warning_once(f"{output_dir} already exists, this may cause model conflict")- tmp_work_dir = Path(os.path.join(output_dir, 'tmp_dir'))+ tmp_work_dir = Path(os.path.join(output_dir, TMP_DIR_NAME))if "gguf_model_instance_global" not in globals():config = model.config+ config.save_pretrained(tmp_work_dir)if tokenizer is not None:tokenizer.save_pretrained(tmp_work_dir)- config.save_pretrained(tmp_work_dir)-- gguf_model_instance_global = create_model_class(output_dir, model, layer_config, backend,- low_cpu_mem_usage=True)-+ if processor is not None:+ processor.save_pretrained(tmp_work_dir)+ if image_processor is not None:+ image_processor.save_pretrained(tmp_work_dir)++ gguf_model_instance_global = [+ create_model_class(+ output_dir, model, layer_config, backend, low_cpu_mem_usage=True, model_type=ModelType.TEXT)+ ]+ if model_type == ModelType.MMPROJ:+ gguf_model_instance_global.append(+ create_model_class(+ output_dir, model, layer_config, backend, low_cpu_mem_usage=True, model_type=ModelType.MMPROJ))if not hasattr(model, "last_layer_name_to_block_name"):block_name_to_last_layer_name = {}block_names = get_block_names(model, quant_vision=True)@@ -127,8 +149,9 @@ def pack_gguf_layer(name, model, backend, output_dir, layer_config, tokenizer):model.last_layer_name_to_block_name = last_layer_name_to_block_nameif name in model.last_layer_name_to_block_name:##packing block- gguf_model_instance_global.current_packing_block = model.last_layer_name_to_block_name[name]- gguf_model_instance_global.prepare_tensors()+ for gguf_model in gguf_model_instance_global:+ gguf_model.current_packing_block = model.last_layer_name_to_block_name[name]+ gguf_model.prepare_tensors()block = get_module(model, model.last_layer_name_to_block_name[name])for n, m in block.named_modules():@@ -140,13 +163,14 @@ def pack_gguf_layer(name, model, backend, output_dir, layer_config, tokenizer):gc.collect()model.last_layer_name_to_block_name.pop(name)if len(model.last_layer_name_to_block_name) == 0:- gguf_model_instance_global.current_packing_block = None+ for gguf_model in gguf_model_instance_global:+ gguf_model.current_packing_block = None@torch.inference_mode()-def save_quantized_as_gguf(output_dir, backend="gguf:q4_0", layer_config=None, **kwargs):+def save_quantized_as_gguf(output_dir, backend="gguf:q4_0", layer_config=None, vlm=False, **kwargs):"""Export the model to gguf format."""- tmp_work_dir = Path(os.path.join(output_dir, 'tmp_dir'))+ tmp_work_dir = Path(os.path.join(output_dir, TMP_DIR_NAME))if output_dir is not None and os.path.exists(output_dir) and not os.path.exists(tmp_work_dir):logger.warning(f"{output_dir} already exists, this may cause model conflict")@@ -155,22 +179,30 @@ def save_quantized_as_gguf(output_dir, backend="gguf:q4_0", layer_config=None, *model = kwargs["model"]if "gguf_model_instance_global" not in globals():- tokenizer = kwargs.get("tokenizer", None)config = model.config- if tokenizer is not None:- tokenizer.save_pretrained(tmp_work_dir)config.save_pretrained(tmp_work_dir)-- config = model.config+ tokenizer = kwargs.get("tokenizer", None)if tokenizer is not None:tokenizer.save_pretrained(tmp_work_dir)- config.save_pretrained(tmp_work_dir)-- gguf_model_instance_global = create_model_class(output_dir, model, layer_config, backend)-- gguf_model_instance_global.write()- rt = time.time() - st- logger.info(f"Model successfully exported to {gguf_model_instance_global.fname_out}, running time={rt}")+ processor = kwargs.get("processor", None)+ if processor is not None:+ processor.save_pretrained(tmp_work_dir)+ image_processor = kwargs.get("image_processor", None)+ if image_processor is not None:+ image_processor.save_pretrained(tmp_work_dir)+++ gguf_model_instance_global = [+ create_model_class(output_dir, model, layer_config, backend, model_type=ModelType.TEXT)+ ]+ if vlm:+ gguf_model_instance_global.append(+ create_model_class(output_dir, model, layer_config, backend, model_type=ModelType.MMPROJ))++ for gguf_model in gguf_model_instance_global:+ gguf_model.write()+ rt = time.time() - st+ logger.info(f"Model successfully exported to {gguf_model.fname_out}, running time={rt}")del gguf_model_instance_globalshutil.rmtree(tmp_work_dir, ignore_errors=True)diff --git a/auto_round/export/export_to_gguf/packing.py b/auto_round/export/export_to_gguf/packing.pyindex 80c2b1abd..e83a0c7f6 100644--- a/auto_round/export/export_to_gguf/packing.py+++ b/auto_round/export/export_to_gguf/packing.py@@ -52,8 +52,20 @@ def ggml_quant(n_blocks = data.nelement() // block_sizeblocks = data.reshape((n_blocks, block_size))quant_func = GGML_QUANT_TYPE[ggml_type]- new_data = quant_func(- blocks, scale, zp=zp, wmin=wmin, d_scale=d_scale, d_wmin=d_wmin, imatrix=imatrix, original=original)+ try:+ new_data = quant_func(+ blocks, scale, zp=zp, wmin=wmin, d_scale=d_scale, d_wmin=d_wmin, imatrix=imatrix, original=original)+ except Exception:+ device = "cpu"+ blocks = blocks.to(device)+ scale = scale.to(device) if scale is not None else scale+ zp = zp.to(device) if zp is not None else zp+ wmin = wmin.to(device) if wmin is not None else wmin+ d_scale = d_scale.to(device) if d_scale is not None else d_scale+ d_wmin = d_wmin.to(device) if d_wmin is not None else d_wmin+ new_data = quant_func(+ blocks, scale, zp=zp, wmin=wmin, d_scale=d_scale, d_wmin=d_wmin, imatrix=imatrix, original=original)+assert new_data.shape[-1] == type_sizenew_data = new_data.reshape(*shape[:-1], shape[-1] // block_size * type_size)diff --git a/auto_round/mllm/autoround_mllm.py b/auto_round/mllm/autoround_mllm.pyindex c3dded9f4..5ab25e92d 100644--- a/auto_round/mllm/autoround_mllm.py+++ b/auto_round/mllm/autoround_mllm.py@@ -36,6 +36,7 @@ from .mllm_dataset import get_mllm_dataloaderfrom ..low_cpu_mem.utils import get_layers_before_block+def _only_text_test(model, tokenizer, device, model_type):"""Test if the model whether can use text-only datasets."""@@ -267,6 +268,7 @@ class AutoRoundMLLM(AutoRound):to_quant_block_names=self.to_quant_block_names,enable_norm_bias_tuning=enable_norm_bias_tuning,enable_torch_compile=enable_torch_compile,+ vlm=True,**kwargs,)diff --git a/auto_round/mllm/processor.py b/auto_round/mllm/processor.pyindex a29efb3b6..a01ac0847 100644--- a/auto_round/mllm/processor.py+++ b/auto_round/mllm/processor.py@@ -136,7 +136,8 @@ class HFProcessor(BasicProcessor):messages, tokenize=False, add_generation_prompt=not continue_final_message,continue_final_message=continue_final_message, )if image is not None:- image = self.image_processor(image)+ image = self.default_image_processor(image)+ # image = self.image_processor(image)ret = self.processor(text=text, images=image, return_tensors="pt")return retdiff --git a/auto_round/script/mllm.py b/auto_round/script/mllm.pyindex 5e3fdea8f..c25c6180b 100644--- a/auto_round/script/mllm.py+++ b/auto_round/script/mllm.py@@ -195,6 +195,9 @@ class BasicArgumentParser(argparse.ArgumentParser):self.add_argument("--device_map", default=None, type=str, help="device_map for block in tuning phase")+ self.add_argument("--disable_opt_rtn", action='store_true',+ help="whether to disable optimization of the RTN mode(iters=0) (default is False).")+def setup_parser():parser = BasicArgumentParser()@@ -287,16 +290,12 @@ def tune(args):if args.format is None:args.format = "auto_round"- supported_formats = [- "auto_round", "auto_round:auto_gptq", "auto_round:auto_awq", "auto_awq", "gguf:q4_0", "gguf:q4_1", "fake"- ]- if not args.quant_nontext_module:- supported_formats.extend(["auto_gptq"])formats = args.format.replace(' ', '').split(",")+ from auto_round.utils import SUPPORTED_FORMATSfor format in formats:- if format not in supported_formats:- raise ValueError(f"{format} is not supported, we only support {supported_formats}")+ if format not in SUPPORTED_FORMATS:+ raise ValueError(f"{format} is not supported, we only support {SUPPORTED_FORMATS}")##must set this before import torchset_cuda_visible_devices(args.device)@@ -387,7 +386,7 @@ def tune(args):layer_config[lm_head_layer_name] = {"bits": args.bits}for format in formats:if "auto_round" not in format and "fake" not in format:- auto_round_formats = [s for s in supported_formats if s.startswith("auto_round")]+ auto_round_formats = [s for s in SUPPORTED_FORMATS if s.startswith("auto_round")]raise ValueError(f"{format} is not supported for lm-head quantization, please change to {auto_round_formats}")@@ -449,28 +448,31 @@ def tune(args):device_map=args.device_map,model_kwargs=model_kwargs,data_type=args.data_type,+ disable_opt_rtn=args.disable_opt_rtn,)- model, _ = autoround.quantize()++ model_name = args.model.rstrip("/")++ if model_name.split('/')[-1].strip('.') == "" and "gguf" not in args.format:+ export_dir = os.path.join(args.output_dir, f"w{autoround.bits}g{autoround.group_size}")+ elif model_name.split('/')[-1].strip('.') == "" and "gguf" in args.format:+ export_dir = args.output_dir+ elif model_name.split('./')[-1].strip('./') != "" and "gguf" in args.format:+ export_dir = os.path.join(args.output_dir,+ model_name.split('/')[-1] + "-gguf")+ else:+ export_dir = os.path.join(args.output_dir,+ model_name.split('/')[-1] + f"-w{autoround.bits}g{autoround.group_size}")++ model, folders = autoround.quantize_and_save(export_dir, format=args.format)++ if args.low_cpu_mem_mode == 1 or args.low_cpu_mem_mode == 2:+ import shutil+ shutil.rmtree(args.low_cpu_mem_tmp_dir, ignore_errors=True)model.eval()clear_memory()- if model_name.split('/')[-1].strip('.') == "":- export_dir = os.path.join(args.output_dir, f"w{args.bits}g{args.group_size}")- else:- export_dir = os.path.join(args.output_dir, model_name.split('/')[-1] + f"-w{args.bits}g{args.group_size}")-- format_list = args.format.replace(' ', '').split(',')- inplace = False if len(format_list) > 1 else True- for format_ in format_list:- save_format_ = format_.replace(":", "-").replace("_", "-")- eval_folder = os.path.join(export_dir, save_format_) if len(formats) > 1 else export_dir- safe_serialization = True- if hasattr(autoround.model, "config") and hasattr(autoround.model.config, "model_type"):- if "phi3_v" in autoround.model.config.model_type:- safe_serialization = False- autoround.save_quantized(eval_folder, format=format_, inplace=inplace, safe_serialization=safe_serialization)-def eval(args):set_cuda_visible_devices(args.device)diff --git a/auto_round/special_model_handler.py b/auto_round/special_model_handler.pyindex f602f9b3f..6d9a0ec95 100644--- a/auto_round/special_model_handler.py+++ b/auto_round/special_model_handler.py@@ -22,7 +22,9 @@ SUPPORT_ONLY_TEXT_MODELS = ["qwen2_vl","deepseek_vl_v2","chatglm",- "idefics3"+ "idefics3",+ "llama4",+ "internvl_chat"]SPECIAL_SHARED_CACHE_KEYS = {diff --git a/auto_round/utils.py b/auto_round/utils.pyindex be55e6585..be03caeb5 100644--- a/auto_round/utils.py+++ b/auto_round/utils.py@@ -29,7 +29,7 @@ from packaging import versionimport gcfrom auto_round.special_model_handler import SPECIAL_MULTIMODAL_BLOCK, SPECIAL_SHARED_CACHE_KEYSimport transformers-from auto_round.export.export_to_gguf.config import GGUF_CONFIG, GGML_QUANT_SIZES, GGUF_INNER_CONFIG, QK_K+from auto_round.export.export_to_gguf.config import GGUF_CONFIG, GGML_QUANT_SIZES, GGUF_INNER_CONFIG, QK_K, ModelTypeSHARED_CACHE_KEYS = ("position_ids", "cache_position", "position_embeddings")@@ -1170,9 +1170,8 @@ def get_layer_features(layer):return None, None # Unsupported layer type-def _gguf_args_check(args_or_ar, format_str=None):+def _gguf_args_check(args_or_ar, format_str=None, model_type=ModelType.TEXT):from auto_round.utils import logger- from auto_round.export.export_to_gguf.config import GGUF_CONFIGimport argparseif format_str is None:@@ -1191,7 +1190,7 @@ def _gguf_args_check(args_or_ar, format_str=None):for format in GGUF_CONFIG:if format in formats:if format == "q6_k_s":- logger.warning("Please not that q6_k_s is q6_k.")+ logger.warning("Please note that q6_k_s is q6_k.")try:from auto_round.export.export_to_gguf.convert import ModelBaseexcept:@@ -1211,7 +1210,7 @@ def _gguf_args_check(args_or_ar, format_str=None):hparams = ModelBase.load_hparams(Path(args_or_ar.model))model_architecture = hparams["architectures"][0]try:- model_class = ModelBase.from_model_architecture(model_architecture)+ model_class = ModelBase.from_model_architecture(model_architecture, model_type=model_type)except NotImplementedError:logger.error(f"Model {model_architecture} is not supported to export GGUF format")sys.exit(1)@@ -1332,7 +1331,7 @@ def mllm_load_model(**kwargs):import jsonimport transformers- from transformers import AutoProcessor, AutoTokenizer, AutoModelForCausalLM+ from transformers import AutoProcessor, AutoTokenizer, AutoModelForCausalLM, AutoModelfrom huggingface_hub import HfApi, hf_hub_download, HfFileSystemif os.path.isdir(pretrained_model_name_or_path):@@ -1395,6 +1394,12 @@ def mllm_load_model(pretrained_model_name_or_path, trust_remote_code=trust_remote_code)processor = AutoProcessor.from_pretrained(pretrained_model_name_or_path, trust_remote_code=trust_remote_code)+ try:+ from transformers import AutoImageProcessor+ image_processor = AutoImageProcessor.from_pretrained(+ pretrained_model_name_or_path, trust_remote_code=trust_remote_code)+ except Exception as e:+ passmodel = model.eval()model = _to_model_dtype(model, model_dtype)@@ -1606,30 +1611,28 @@ def _gguf_type_fallback(gguf_type):return gguf_type##https://github.com/ggml-org/llama.cpp/blob/9e31bec4fd53634c9e5b04650488a09a055f5dab/src/llama-quant.cpp#L129-def get_layer_config_by_gguf_format(layer_config, gguf_format, model):+def get_layer_config_by_gguf_format(layer_config, gguf_format, model, model_type=ModelType.TEXT):# TODO: support for other format latertarget_gguf_format = next((fmt for fmt in gguf_format if fmt != "fake"), None)import gguf # pylint: disable=E0401- from auto_round.export.export_to_gguf.convert import ModelBase- if model.config.architectures is not None:- model_architecture = model.config.architectures[0]- else:- model_architecture = type(model).__name__- if model_architecture not in ModelBase._model_classes:- if model_architecture.replace("CausalLM", "ConditionalGeneration") in ModelBase._model_classes:- model_architecture = model_architecture.replace("CausalLM", "ConditionalGeneration")- elif model_architecture.replace("ConditionalGeneration", "CausalLM") in ModelBase._model_classes:- model_architecture = model_architecture.replace("ConditionalGeneration", "CausalLM")+ from auto_round.export.export_to_gguf.convert import ModelBase, get_model_architecture+ model_architecture = get_model_architecture(hparams=model.config.to_dict(), model_type=model_type)try:- model_class = ModelBase.from_model_architecture(model_architecture)+ model_class = ModelBase.from_model_architecture(model_architecture, model_type=model_type)except NotImplementedError:return layer_config, {}n_layer = Nonefor name in ["n_layers", "num_hidden_layers", "n_layer", "num_layers"]:+ sub_attr = "text_config" if model_type == ModelType.TEXT else "vision_config"if hasattr(model.config, name):n_layer = getattr(model.config, name)+ break+ if hasattr(model.config, sub_attr):+ if hasattr(getattr(model.config, sub_attr), name):+ n_layer = getattr(getattr(model.config, sub_attr), name)+ breakif n_layer is None:return layer_config, {}@@ -1830,10 +1833,10 @@ def get_layer_config_by_gguf_format(layer_config, gguf_format, model):and 'Deepseek' in model.config.architectures[0]:fallback = False- # calc if need fallback+ # calc if need fallbackqk_nope_head_dim = model.config.qk_nope_head_dimkv_b_shape = get_module(model, layer_name).weight.shape-+if qk_nope_head_dim < QK_K or qk_nope_head_dim % QK_K != 0 \or kv_b_shape[-1] < QK_K or kv_b_shape[-1] % QK_K != 0:fallback = True@@ -1909,9 +1912,11 @@ def flatten_list(nested_list):return flatteneddef clean_module_parameter(submodule, parameter):+ if submodule is None:+ returnis_buffer = parameter in submodule._bufferswith torch.no_grad():if is_buffer:submodule._buffers[parameter] = Noneelse:- submodule._parameters[parameter] = None\ No newline at end of file+ submodule._parameters[parameter] = Nonediff --git a/test/test_cpu/test_gguf_format.py b/test/test_cpu/test_gguf_format.pyindex 4f33b7204..ae5cfba5a 100644--- a/test/test_cpu/test_gguf_format.py+++ b/test/test_cpu/test_gguf_format.py@@ -238,7 +238,7 @@ class TestGGUF(unittest.TestCase):def test_q4_k_m(self):- model_name = "Qwen/Qwen2.5-7B-Instruct"+ model_name = "Qwen/Qwen2.5-1.5B-Instruct"model = AutoModelForCausalLM.from_pretrained(model_name, torch_dtype="auto", trust_remote_code=True)tokenizer = AutoTokenizer.from_pretrained(model_name, trust_remote_code=True)layer_config = {@@ -257,7 +257,7 @@ class TestGGUF(unittest.TestCase):disable_opt_rtn=True)quantized_model_path = "./saved"- autoround.quantize_and_save(output_dir=quantized_model_path, format="gguf:q4_k_m")+ autoround.quantize_and_save(output_dir=quantized_model_path, format="gguf:q4_k_m,fake")self.assertEqual(autoround.layer_config["model.layers.11.self_attn.v_proj"]["super_group_size"], 16)self.assertEqual(autoround.layer_config["model.layers.11.self_attn.v_proj"]["data_type"], "int_sym_dq")self.assertEqual(autoround.layer_config["model.layers.7.self_attn.v_proj"]["data_type"], "int_asym_dq")@@ -269,27 +269,57 @@ class TestGGUF(unittest.TestCase):self.assertEqual(autoround.model.model.layers[10].mlp.gate_proj.bits, 8)self.assertEqual(autoround.layer_config['model.layers.10.mlp.gate_proj']['mostly'], "gguf:q8_0")shutil.rmtree("./saved", ignore_errors=True)++ autoround = AutoRound(+ model,+ tokenizer,+ iters=0,+ nsamples=1,+ seqlen=128,+ disable_opt_rtn=False+ )+ quantized_model_path = "./saved"+ autoround.quantize_and_save(output_dir=quantized_model_path, format="gguf:q4_k_m,fake")+ shutil.rmtree("./saved", ignore_errors=True)def test_all_format(self):- from auto_round.export.export_to_gguf.config import GGUF_CONFIG- model_name = "Qwen/Qwen2.5-7B-Instruct"+ model_name = "Qwen/Qwen2.5-1.5B-Instruct"python_path = sys.executable- for gguf_format in GGUF_CONFIG.keys():+ for gguf_format in ["gguf:q4_0", "gguf:q4_1", "gguf:q4_k_m", "gguf:q6_k"]:res = os.system(f"cd ../.. && {python_path} -m auto_round --model {model_name} "- f" --bs 16 --iters 1 --nsamples 1 --format {gguf_format}"+ f" --bs 16 --iters 1 --nsamples 1 --seqlen 16 --format {gguf_format}")if res > 0 or res == -1:assert False, "cmd line test fail, please have a check"shutil.rmtree("../../tmp_autoround", ignore_errors=True)res = os.system(- f"cd ../.. && {python_path} -m auto_round --model {model_name} "- f" --bs 16 --iters 0 --nsamples 1 --format fake,{gguf_format}"+ f"cd ../.. && {python_path} -m auto_round --model {model_name}"+ f" --bs 16 --iters 0 --nsamples 1 --seqlen 16 --format fake,{gguf_format}")if res > 0 or res == -1:assert False, "cmd line test fail, please have a check"shutil.rmtree("../../tmp_autoround", ignore_errors=True)++ def test_vlm_gguf(self):+ model_name = "Qwen/Qwen2-VL-2B-Instruct"+ from auto_round.mllm.autoround_mllm import AutoRoundMLLM+ from auto_round.utils import mllm_load_model+ model, processor, tokenizer, image_processor = mllm_load_model(model_name)+ autoround = AutoRoundMLLM(+ model,+ tokenizer=tokenizer,+ processor=processor,+ image_processor=image_processor,+ iters=0,+ )+ quantized_model_path = "./saved"+ autoround.quantize_and_save(output_dir=quantized_model_path, format="gguf:q4_0")+ self.assertTrue("mmproj-model.gguf" in os.listdir("./saved"))+ file_size = os.path.getsize("./saved/Qwen2-VL-2B-Instruct-1.5B-Q4_0.gguf") / 1024 ** 2+ self.assertAlmostEqual(file_size, 892, delta=1.0)+ shutil.rmtree("./saved", ignore_errors=True)if __name__ == "__main__":unittest.main()diff --git a/test/test_cuda/test_gguf.py b/test/test_cuda/test_gguf.pyindex 05e212e7e..d6cd50391 100644--- a/test/test_cuda/test_gguf.py+++ b/test/test_cuda/test_gguf.py@@ -171,11 +171,13 @@ class TestAutoRound(unittest.TestCase):self.assertGreater(result['results']['piqa']['acc,none'], 0.55)shutil.rmtree("./saved", ignore_errors=True)+ @require_ggufdef test_all_format(self):from auto_round.export.export_to_gguf.config import GGUF_CONFIGpython_path = sys.executable- for model_name in ["/models/Qwen3-8B/", "/models/Llama-3.2-3B/"]:+ for model_name in ["/models/Qwen3-8B/", "/models/Llama-3.2-3B/", "/models/Meta-Llama-3.1-8B-Instruct"]:for gguf_format in GGUF_CONFIG.keys():+ print(model_name, gguf_format)res = os.system(f"cd ../.. && {python_path} -m auto_round --model {model_name} "f" --bs 16 --iters 1 --nsamples 1 --format fake,{gguf_format}"@@ -192,5 +194,73 @@ class TestAutoRound(unittest.TestCase):assert False, "cmd line test fail, please have a check"shutil.rmtree("../../tmp_autoround", ignore_errors=True)+ @require_gguf+ def test_vlm_gguf(self):+ model_name = "/models/Qwen2.5-VL-7B-Instruct"+ from auto_round.mllm.autoround_mllm import AutoRoundMLLM+ from auto_round.utils import mllm_load_model+ model, processor, tokenizer, image_processor = mllm_load_model(model_name)+ autoround = AutoRoundMLLM(+ model,+ tokenizer=tokenizer,+ processor=processor,+ image_processor=image_processor,+ device="auto",+ iters=0,+ )+ quantized_model_path = "./saved"+ autoround.quantize_and_save(output_dir=quantized_model_path, format="gguf:q4_0")+ self.assertTrue("mmproj-model.gguf" in os.listdir("./saved"))+ file_size = os.path.getsize("./saved/Qwen2.5-VL-7B-Instruct-7.6B-Q4_0.gguf") / 1024 ** 2+ self.assertAlmostEqual(file_size, 4226, delta=1.0)+ file_size = os.path.getsize("./saved/mmproj-model.gguf") / 1024 ** 2+ self.assertAlmostEqual(file_size, 2578, delta=1.0)+ shutil.rmtree("./saved", ignore_errors=True)++ model_name = "/models/gemma-3-12b-it"+ from auto_round.mllm.autoround_mllm import AutoRoundMLLM+ from auto_round.utils import mllm_load_model+ model, processor, tokenizer, image_processor = mllm_load_model(model_name)+ autoround = AutoRoundMLLM(+ model,+ tokenizer=tokenizer,+ processor=processor,+ image_processor=image_processor,+ device="auto",+ nsamples=32,+ iters=1,+ )+ quantized_model_path = "./saved"+ autoround.quantize_and_save(output_dir=quantized_model_path, format="gguf:q4_k_m")+ self.assertTrue("mmproj-model.gguf" in os.listdir("./saved"))+ file_size = os.path.getsize("./saved/gemma-3-12b-it-12B-Q4_K_M.gguf") / 1024 ** 2+ self.assertAlmostEqual(file_size, 6568, delta=1.0)+ file_size = os.path.getsize("./saved/mmproj-model.gguf") / 1024 ** 2+ self.assertAlmostEqual(file_size, 1599, delta=1.0)+ shutil.rmtree("./saved", ignore_errors=True)++ @require_gguf+ def test_llama_4(self):+ model_name = "/dataset/Llama-4-Scout-17B-16E-Instruct/"+ from auto_round.mllm.autoround_mllm import AutoRoundMLLM+ from auto_round.utils import mllm_load_model+ model, processor, tokenizer, image_processor = mllm_load_model(model_name)+ autoround = AutoRoundMLLM(+ model,+ tokenizer=tokenizer,+ processor=processor,+ image_processor=image_processor,+ device="auto",+ iters=0,+ )+ quantized_model_path = "/dataset/Llam-4-test"+ autoround.quantize_and_save(output_dir=quantized_model_path, format="gguf:q4_0")+ self.assertTrue("mmproj-model.gguf" in os.listdir(quantized_model_path))+ file_size = os.path.getsize(os.path.join(quantized_model_path, "Llama-4-Scout-17B-16E-Instruct-16x17B-Q4_0.gguf")) / 1024 ** 2+ self.assertAlmostEqual(file_size, 58093.62, delta=1.0)+ file_size = os.path.getsize(os.path.join(quantized_model_path, "mmproj-model.gguf")) / 1024 ** 2+ self.assertAlmostEqual(file_size, 3323.52, delta=1.0)+ shutil.rmtree(quantized_model_path, ignore_errors=True)+if __name__ == "__main__":unittest.main()\ No newline at end of file