Skip to content

PTQ-Level Efficiency, QAT-Level Accuracy | 训练后量化的效率,量化感知训练的精度

support vlm models for gguf format

Heng Guoa2f3f016cd8f
diff --git a/README.md b/README.md
index 046b23a22..68da05f8e 100644
--- a/README.md
+++ b/README.md
@@ -34,17 +34,7 @@ and [fbaldassarri](https://huggingface.co/fbaldassarri).
## What's New
-[//]: # (* [2025.07] AutoRound now offers experimental support for the widely used **GGUF** format. We currently recommend using)
-
-[//]: # ( RTN mode (--iters 0) for all the bits except 3. A more advanced algorithm tailored for some specific configurations is likely)
-
-[//]: # ( to be introduced in the upcoming release. Example models are)
-
-[//]: # ( available on the Intel Hugging Face space, including)
-
-[//]: # ( [Intel/Qwen3-235B-A22B-q2ks-mixed-AutoRound-inc-v0](https://huggingface.co/Intel/Qwen3-235B-A22B-q2ks-mixed-AutoRound-inc-v0))
-
-[//]: # ( and [Intel/DeepSeek-R1-0528-q2ks-mixed-AutoRound-inc-v0](https://huggingface.co/Intel/DeepSeek-R1-0528-q2ks-mixed-AutoRound-inc-v0))
+* [2025.07] AutoRound now offers experimental support for **GGUF** format, and recommends using RTN mode (--iters 0) for all bits other than 3 bits. A more advanced algorithm tailored for specific configurations may be available in upcoming release. Example models: [Intel/Qwen3-235B-A22B-q2ks-mixed-AutoRound-inc-v0](https://huggingface.co/Intel/Qwen3-235B-A22B-q2ks-mixed-AutoRound-inc-v0) and [Intel/DeepSeek-R1-0528-q2ks-mixed-AutoRound-inc-v0](https://huggingface.co/Intel/DeepSeek-R1-0528-q2ks-mixed-AutoRound-inc-v0).
* [2025.05] AutoRound provides some recipes for **DeepSeek-R1-0528**, please refer
to [DeepSeek-R1-0528-int2-mixed-sym-inc](https://huggingface.co/Intel/DeepSeek-R1-0528-int2-mixed-sym-inc), [DeepSeek-R1-0528-int4-sym-gptq-inc](https://huggingface.co/Intel/DeepSeek-R1-0528-int4-gptq-inc-auto-round)
and [DeepSeek-R1-0528-int4-asym-awq-inc](https://huggingface.co/Intel/DeepSeek-R1-0528-int4-awq-inc-auto-round) for
diff --git a/auto_round/autoround.py b/auto_round/autoround.py
index b52f7bc0d..8acb21d00 100644
--- a/auto_round/autoround.py
+++ b/auto_round/autoround.py
@@ -25,7 +25,7 @@ from torch import autocast
from tqdm import tqdm
import accelerate
-from auto_round.export.export_to_gguf.config import GGUF_CONFIG, GGUF_INNER_CONFIG
+from auto_round.export.export_to_gguf.config import GGUF_CONFIG, GGUF_INNER_CONFIG, ModelType
from auto_round.wrapper import WrapperMultiblock, wrapper_block, unwrapper_block, WrapperLinear, unwrapper_layer
from auto_round.utils import (
CpuInfo,
@@ -173,6 +173,7 @@ class AutoRound(object):
model_kwargs: dict = None,
**kwargs,
):
+ self.vlm = kwargs.pop("vlm") if "vlm" in kwargs else False
if kwargs:
logger.warning(f"unrecognized keys {list(kwargs.keys())} were passed. Please check them.")
self.quantized = False
@@ -260,6 +261,7 @@ class AutoRound(object):
self.disable_opt_rtn = disable_opt_rtn
+
torch.set_printoptions(precision=3, sci_mode=True)
self.check_configs()
if self.act_bits <= 8 and self.amp_dtype == torch.float16:
@@ -515,7 +517,9 @@ class AutoRound(object):
Returns:
list: A list of validated and updated formats.
"""
- _gguf_args_check(self, format)
+ _gguf_args_check(self, format, model_type=ModelType.TEXT)
+ if self.vlm:
+ _gguf_args_check(self, format, model_type=ModelType.MMPROJ)
formats = format.replace("q*_", f"q{self.bits}_").replace(' ', '').split(',')
from auto_round.utils import SUPPORTED_FORMATS
@@ -947,7 +951,8 @@ class AutoRound(object):
if not tie_word_embeddings:
lm_head_name: str = get_lm_head_name(self.model)
config: dict[str, Any] = GGUF_CONFIG[GGUF_CONFIG[target_format]["lm_head"]]
- check_fixed_by_user = self.layer_config[lm_head_name].get("fixed_by_user", False)
+ check_fixed_by_user = self.layer_config[lm_head_name].get(
+ "fixed_by_user", False) if lm_head_name in self.layer_config else None
self._apply_config_to_layer(lm_head_name, config, check_fixed_by_user=check_fixed_by_user)
return True
@@ -1060,10 +1065,17 @@ class AutoRound(object):
if has_gguf:
from auto_round.export.export_to_gguf.export import pack_gguf_layer
output_dir = self.get_save_folder_name(self.formats[0])
+ model_type = ModelType.MMPROJ if self.vlm else ModelType.TEXT
pack_gguf_layer(
- name, self.model, self.formats[0],
- output_dir, self.layer_config, self.tokenizer
- )
+ name,
+ self.model,
+ self.formats[0],
+ output_dir,
+ self.layer_config,
+ self.tokenizer,
+ processor=self.processor if hasattr(self, "processor") else None,
+ image_processor=self.image_processor if hasattr(self, "image_processor") else None,
+ model_type=model_type)
else:
PACKING_LAYER_WITH_FORMAT[target_backend](
name, self.model, self.formats[0]
@@ -1220,8 +1232,11 @@ class AutoRound(object):
if len(self.formats) == 1 and self.formats[0] == "fake":
only_gguf = False
if only_gguf:
- self.layer_config, gguf_format_config = get_layer_config_by_gguf_format(self.layer_config, self.formats,
- self.model)
+ self.layer_config, gguf_format_config = get_layer_config_by_gguf_format(
+ self.layer_config, self.formats, self.model, model_type=ModelType.TEXT)
+ if self.vlm:
+ self.layer_config, gguf_format_config = get_layer_config_by_gguf_format(
+ self.layer_config, self.formats, self.model, model_type=ModelType.MMPROJ)
# Determine if immediate packing is required
formats = self.formats
if (len(formats) == 1 and
@@ -2339,8 +2354,17 @@ class AutoRound(object):
if has_gguf:
from auto_round.export.export_to_gguf.export import pack_gguf_layer
output_dir = self.get_save_folder_name(self.formats[0])
- pack_gguf_layer(tmp_m.tmp_name, self.model, self.formats[0], output_dir, self.layer_config,
- self.tokenizer)
+ model_type = ModelType.MMPROJ if self.vlm else ModelType.TEXT
+ pack_gguf_layer(
+ tmp_m.tmp_name,
+ self.model,
+ self.formats[0],
+ output_dir,
+ self.layer_config,
+ self.tokenizer,
+ processor=self.processor if hasattr(self, "processor") else None,
+ image_processor=self.image_processor if hasattr(self, "image_processor") else None,
+ model_type=model_type)
else:
PACKING_LAYER_WITH_FORMAT[target_backend](tmp_m.tmp_name, self.model, self.formats[0])
pbar.set_description(f"Quantizing done")
diff --git a/auto_round/data_type/nvfp.py b/auto_round/data_type/nvfp.py
index 76c4eb92d..d1293e899 100644
--- a/auto_round/data_type/nvfp.py
+++ b/auto_round/data_type/nvfp.py
@@ -51,8 +51,8 @@ def get_reciprocal(x):
FLOAT4_E2M1_MAX = 6.0
-FLOAT8_E4M3_MAX = torch.finfo(torch.float8_e4m3fn).max
-FLOAT8_E4M3_MIN = torch.finfo(torch.float8_e4m3fn).min
+FLOAT8_E4M3_MAX = torch.finfo(torch.float8_e4m3fn).max if hasattr(torch, "float8_e4m3fn") else 448
+FLOAT8_E4M3_MIN = torch.finfo(torch.float8_e4m3fn).min if hasattr(torch, "float8_e4m3fn") else -448
def ref_nvfp4_quant(x, global_scale, block_size=16, v=0):
diff --git a/auto_round/export/export_to_gguf/config.py b/auto_round/export/export_to_gguf/config.py
index 484c5f033..d2b6a821e 100644
--- a/auto_round/export/export_to_gguf/config.py
+++ b/auto_round/export/export_to_gguf/config.py
@@ -11,6 +11,11 @@
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
+from enum import IntEnum
+
+class ModelType(IntEnum):
+ TEXT = 1
+ MMPROJ = 2
GGUF_INNER_CONFIG = {}
diff --git a/auto_round/export/export_to_gguf/convert.py b/auto_round/export/export_to_gguf/convert.py
index 4b074d762..669f81d22 100644
--- a/auto_round/export/export_to_gguf/convert.py
+++ b/auto_round/export/export_to_gguf/convert.py
@@ -542,6 +542,8 @@ class ModelBase(OriModel):
dry_run: bool = False,
small_first_shard: bool = False,
hparams: dict[str, Any] | None = None):
+ if self.model_arch == gguf.MODEL_ARCH.MMPROJ and fname_out.is_dir():
+ fname_out = fname_out / "mmproj-model.gguf"
self.model = model
self.layer_config = layer_config
self.low_cpu_mem_usage = self._need_low_cpu_mem(low_cpu_mem_usage)
@@ -618,8 +620,14 @@ class ModelBase(OriModel):
data_torch, data_qtype.name.lower(), scale, zp, wmin=wmin, d_scale=d_scale, d_wmin=d_wmin, device=device)
return data
- def _quant_data(self, data_torch, data_qtype, name, bid):
+ def _quant_data(self, data_torch, data_qtype, name, modify_name, bid):
suffix = '.weight'
+ if torch.cuda.is_available():
+ device = "cuda"
+ elif torch.xpu.is_available():
+ device = "xpu"
+ else:
+ device = "cpu"
if suffix in name:
layer_name = name[:-len(suffix)]
module = get_module(self.model, layer_name)
@@ -630,17 +638,12 @@ class ModelBase(OriModel):
if hasattr(module, attr) and getattr(module, attr) is not None:
attr_tensor = getattr(module, attr)
ori_shape = attr_tensor.shape
- attr_tensor = self.modify_tensors(attr_tensor.reshape(bs, -1), name, bid)[0][1]
+ # attr_tensor = self.modify_tensors(attr_tensor.reshape(bs, -1), modify_name, bid)[0][1]
+ attr_tensor = self.modify_tensors(attr_tensor, modify_name, bid)[0][1]
attr_tensor = attr_tensor.reshape(ori_shape)
setattr(module, attr, attr_tensor)
scale = module.scale if hasattr(module, "scale") else None
zp = module.zp if hasattr(module, "zp") else None
- if torch.cuda.is_available():
- device = "cuda"
- elif torch.xpu.is_available():
- device = "xpu"
- else:
- device = "cpu"
data_torch = data_torch.to(torch.float32)
scale = scale.to(torch.float32) if isinstance(scale, torch.Tensor) else scale
zp = zp.to(torch.float32) if isinstance(zp, torch.Tensor) else zp
@@ -670,20 +673,20 @@ class ModelBase(OriModel):
data_qtype = gguf.GGMLQuantizationType.F32 ##FP16 has issues at inference
data = data_torch.to(torch.float32).squeeze().cpu().numpy()
else:
+ # for Llama-4
# if data_torch.dtype == torch.float32:
# data_qtype = gguf.GGMLQuantizationType.F32
# else:
# data_qtype = gguf.GGMLQuantizationType.F16
# data = data_torch.squeeze().cpu().numpy()
- data_qtype = gguf.GGMLQuantizationType.F32
- data = data_torch.to(torch.float32).squeeze().cpu().numpy()
+ # data_qtype = gguf.GGMLQuantizationType.F32
+ # data = data_torch.to(torch.float32).squeeze().cpu().numpy()
+ data = ggml_quant(data_torch, data_qtype.name.lower(), device=device)
return data, data_qtype
def get_qtype_by_layer_config(self, layer_config, name, data_qtype):
name = name[:-len('.weight')]
if name not in layer_config or layer_config[name]['bits'] >= 16:
- if isinstance(data_qtype, bool):
- return gguf.GGMLQuantizationType.F32
return data_qtype
bits = layer_config[name]['bits']
super_bits = layer_config[name]["super_bits"]
@@ -738,6 +741,42 @@ class ModelBase(OriModel):
# raise ValueError(f"Unknown file type: {data_qtype}")
# return data_qtype
+ def _special_name_handle(self, name):
+ # for Qwen2VL
+ def remove_prefix(name, key_list):
+ for key in key_list:
+ if key in name and not name.startswith(key):
+ name = ".".join(name.split(".")[1:])
+ break
+ return name
+ if self.model_arch == gguf.MODEL_ARCH.QWEN2VL:
+ name = name.replace("language_model.", "")
+ visual_keys = ["thinker", "visual", "audio", "talker", "token2wav"]
+ name = remove_prefix(name, visual_keys)
+ if isinstance(self, Qwen2VLVisionModel):
+ if "visual." in name and not name.startswith("visual."):
+ name = ".".join(name.split(".")[1:])
+
+ # for gemma3
+ if self.model_arch == gguf.MODEL_ARCH.GEMMA3 or isinstance(self, Gemma3VisionModel):
+ visual_keys = ["multi_modal_projector", "vision_tower", "multimodal_projector", "vision_model"]
+ name = remove_prefix(name, visual_keys)
+
+ # for LlavaForConditionalGeneration
+ if self.model_arch == gguf.MODEL_ARCH.LLAMA:
+ name = name.replace("language_model.", "")
+ # name = name.replace("model.", "")
+ if isinstance(self, LlavaVisionModel):
+ visual_keys = ["multi_modal_projector", "vision_tower"]
+ name = remove_prefix(name, visual_keys)
+
+ # for InternVisionModel
+ if isinstance(self, InternVisionModel):
+ visual_keys = ["vision_model"]
+ name = remove_prefix(name, visual_keys)
+
+ return name
+
def prepare_tensors(self):
max_name_len = max(len(s) for _, s in self.tensor_map.mapping.values()) + len(".weight,")
@@ -768,7 +807,9 @@ class ModelBase(OriModel):
break
clean_weight_list = []
- for new_name, data_torch in (self.modify_tensors(data_torch, name, bid)):
+
+ modify_name = self._special_name_handle(name)
+ for new_name, data_torch in (self.modify_tensors(data_torch, modify_name, bid)):
data = data_torch.squeeze().cpu().numpy()
# if data ends up empty, it means data_torch was a scalar tensor -> restore
@@ -813,40 +854,53 @@ class ModelBase(OriModel):
name = self.get_moe_name(name, new_name)
clean_weight_list.append(name)
# get data_qtype by layer_config
- data_qtype = self.get_qtype_by_layer_config(self.layer_config, name, data_qtype)
+ if isinstance(data_qtype, bool):
+ data_qtype = self.get_qtype_by_layer_config(self.layer_config, name, data_qtype)
# # No override (data_qtype is False), or wants to be quantized (data_qtype is True)
- # if isinstance(data_qtype, bool):
- # if self.ftype == gguf.LlamaFileType.ALL_F32:
- # data_qtype = gguf.GGMLQuantizationType.F32
- # elif self.ftype == gguf.LlamaFileType.MOSTLY_F16:
- # data_qtype = gguf.GGMLQuantizationType.F16
- # elif self.ftype == gguf.LlamaFileType.MOSTLY_BF16:
- # data_qtype = gguf.GGMLQuantizationType.BF16
- # elif self.ftype == gguf.LlamaFileType.MOSTLY_Q8_0:
- # data_qtype = gguf.GGMLQuantizationType.Q8_0
- # elif self.ftype == gguf.LlamaFileType.MOSTLY_Q4_0:
- # data_qtype = gguf.GGMLQuantizationType.Q4_0
- # elif self.ftype == gguf.LlamaFileType.MOSTLY_Q4_1:
- # data_qtype = gguf.GGMLQuantizationType.Q4_1
- # elif self.ftype == gguf.LlamaFileType.MOSTLY_Q5_0:
- # data_qtype = gguf.GGMLQuantizationType.Q5_0
- # elif self.ftype == gguf.LlamaFileType.MOSTLY_Q5_1:
- # data_qtype = gguf.GGMLQuantizationType.Q5_1
- # elif self.ftype == gguf.LlamaFileType.MOSTLY_Q2_K_S:
- # data_qtype = gguf.GGMLQuantizationType.Q2_K
- # elif self.ftype == gguf.LlamaFileType.MOSTLY_Q3_K_S:
- # data_qtype = gguf.GGMLQuantizationType.Q3_K
- # elif self.ftype == gguf.LlamaFileType.MOSTLY_Q4_K_S:
- # data_qtype = gguf.GGMLQuantizationType.Q4_K
- # elif self.ftype == gguf.LlamaFileType.MOSTLY_Q4_K_M:
- # data_qtype = gguf.GGMLQuantizationType.Q4_K
- # elif self.ftype == gguf.LlamaFileType.MOSTLY_Q5_K_S:
- # data_qtype = gguf.GGMLQuantizationType.Q5_K
- # elif self.ftype == gguf.LlamaFileType.MOSTLY_Q6_K:
- # data_qtype = gguf.GGMLQuantizationType.Q6_K
- # else:
- # raise ValueError(f"Unknown file type: {self.ftype.name}")
+ if isinstance(data_qtype, bool):
+ if self.ftype == gguf.LlamaFileType.ALL_F32:
+ data_qtype = gguf.GGMLQuantizationType.F32
+ elif self.ftype == gguf.LlamaFileType.MOSTLY_F16:
+ data_qtype = gguf.GGMLQuantizationType.F16
+ elif self.ftype == gguf.LlamaFileType.MOSTLY_BF16:
+ data_qtype = gguf.GGMLQuantizationType.BF16
+ elif self.ftype == gguf.LlamaFileType.MOSTLY_Q8_0:
+ data_qtype = gguf.GGMLQuantizationType.Q8_0
+ elif self.ftype == gguf.LlamaFileType.MOSTLY_Q4_0:
+ data_qtype = gguf.GGMLQuantizationType.Q4_0
+ elif self.ftype == gguf.LlamaFileType.MOSTLY_Q4_1:
+ data_qtype = gguf.GGMLQuantizationType.Q4_1
+ elif self.ftype == gguf.LlamaFileType.MOSTLY_Q5_0:
+ data_qtype = gguf.GGMLQuantizationType.Q5_0
+ elif self.ftype == gguf.LlamaFileType.MOSTLY_Q5_1:
+ data_qtype = gguf.GGMLQuantizationType.Q5_1
+ elif self.ftype == gguf.LlamaFileType.MOSTLY_Q2_K_S or \
+ self.ftype == gguf.LlamaFileType.MOSTLY_Q2_K:
+ data_qtype = gguf.GGMLQuantizationType.Q2_K
+ elif self.ftype == gguf.LlamaFileType.MOSTLY_Q3_K_S or \
+ self.ftype == gguf.LlamaFileType.MOSTLY_Q3_K_M or \
+ self.ftype == gguf.LlamaFileType.MOSTLY_Q3_K_L:
+ data_qtype = gguf.GGMLQuantizationType.Q3_K
+ elif self.ftype == gguf.LlamaFileType.MOSTLY_Q4_K_S or \
+ self.ftype == gguf.LlamaFileType.MOSTLY_Q4_K_M:
+ data_qtype = gguf.GGMLQuantizationType.Q4_K
+ elif self.ftype == gguf.LlamaFileType.MOSTLY_Q5_K_S \
+ or self.ftype == gguf.LlamaFileType.MOSTLY_Q5_K_M:
+ data_qtype = gguf.GGMLQuantizationType.Q5_K
+ elif self.ftype == gguf.LlamaFileType.MOSTLY_Q6_K:
+ data_qtype = gguf.GGMLQuantizationType.Q6_K
+ else:
+ raise ValueError(f"Unknown file type: {self.ftype.name}")
+
+ if data_qtype.name.endswith("_K") and data_torch.shape[-1] % 256 != 0:
+ if data_qtype in [gguf.GGMLQuantizationType.Q2_K, gguf.GGMLQuantizationType.Q3_K,
+ gguf.GGMLQuantizationType.Q4_K]:
+ data_qtype = gguf.GGMLQuantizationType.Q5_0
+ elif data_qtype == gguf.GGMLQuantizationType.Q5_K:
+ data_qtype = gguf.GGMLQuantizationType.Q5_0
+ elif data_qtype == gguf.GGMLQuantizationType.Q6_K:
+ data_qtype = gguf.GGMLQuantizationType.Q8_0
if isinstance(data_qtype, bool) or data_qtype in [
gguf.GGMLQuantizationType.F16, gguf.GGMLQuantizationType.BF16, gguf.GGMLQuantizationType.F32
@@ -877,7 +931,7 @@ class ModelBase(OriModel):
k_b, v_b = torch.split(kv_b, [qk_nope_head_dim, v_head_dim], dim=1)
k_b = k_b.transpose(1, 2)
- name_kb = name.replace("kv_b_proj", "k_b_proj")
+ name_kb = modify_name.replace("kv_b_proj", "k_b_proj")
if new_name == self.map_tensor_name(name_kb):
attr_list[attr] = k_b
else:
@@ -885,7 +939,7 @@ class ModelBase(OriModel):
data = self._quant_data_with_args(data_torch, data_qtype, **attr_list)
# for MOE model
- elif len(data_torch.shape) == 3:
+ elif len(data_torch.shape) == 3 and len(re.findall("\d+", name)) == 2:
new_data = []
for idx, arr in enumerate(data_torch):
arr_name = name.split('.')
@@ -893,12 +947,12 @@ class ModelBase(OriModel):
if arr_name[i].isdecimal() and int(arr_name[i]) == (data_torch.shape[0] - 1):
arr_name[i] = str(idx)
arr_name = ".".join(arr_name)
- arr, data_qtype = self._quant_data(arr, data_qtype, arr_name, bid)
+ arr, data_qtype = self._quant_data(arr, data_qtype, arr_name, modify_name, bid)
new_data.append(arr)
data = np.array(new_data)
del new_data
else:
- data, data_qtype = self._quant_data(data_torch, data_qtype, name, bid)
+ data, data_qtype = self._quant_data(data_torch, data_qtype, name, modify_name, bid)
shape = gguf.quant_shape_from_byte_shape(
data.shape, data_qtype) if data.dtype == np.uint8 else data.shape
@@ -917,6 +971,13 @@ class ModelBase(OriModel):
if self.low_cpu_mem_usage:
for weight_name in clean_weight_list:
module = get_module(self.model, ".".join(weight_name.split(".")[:-1]))
+ for key in ["scale", "zp", "d_scale", "wmin", "d_wmin", "imatrix"]:
+ if hasattr(module, key):
+ setattr(module, key, None)
+ if hasattr(module, "w_" + key):
+ setattr(module, "w_" + key, None)
+ if self.model_arch == gguf.MODEL_ARCH.LLAMA and "embed_tokens.weight" in weight_name:
+ continue
clean_module_parameter(module, weight_name.split(".")[-1])
@@ -1062,10 +1123,10 @@ class TextModel(ModelBase):
)
seems_special = seems_special or (token_text.startswith("<|") and token_text.endswith("|>"))
- seems_special = seems_special or (token_text.startswith("<|") and token_text.endswith("|>"))
+ seems_special = seems_special or (token_text.startswith("<|") and token_text.endswith("|>"))
# TODO: should these be marked as UNUSED instead? (maybe not)
- seems_special = seems_special or (token_text.startswith("<unused") and token_text.endswith(">"))
+ seems_special = seems_special or (token_text.startswith("<unused") and token_text.endswith(">"))
return seems_special
@@ -1109,7 +1170,7 @@ class TextModel(ModelBase):
else:
# NOTE: this was added for Gemma.
# Encoding and decoding the tokens above isn't sufficient for this case.
- token = token.replace(b"\xe2\x96\x81".decode("utf-8"), " ")
+ token = token.replace(b"\xe2\x96\x81".decode("utf-8"), " ")
toktypes.append(gguf.TokenType.USER_DEFINED)
else:
toktypes.append(gguf.TokenType.NORMAL)
@@ -3283,6 +3344,7 @@ class Qwen2VLModel(TextModel):
del bid # unused
if name.startswith("thinker."):
name = name.replace("thinker.", "")
+
if name.startswith("visual") or name.startswith("audio") or \
name.startswith("talker") or name.startswith("token2wav"):
# skip multimodal tensors
@@ -6630,7 +6692,8 @@ class ChatGLMModel(TextModel):
def modify_tensors(self, data_torch: Tensor, name: str, bid: int | None) -> Iterable[tuple[str, Tensor]]:
del bid # unused
- if name.endswith(".rotary_pos_emb.inv_freq") or name.startswith("model.vision."):
+ if name.endswith(".rotary_pos_emb.inv_freq") or name.startswith("model.vision.") \
+ or name.startswith("transformers.vision."):
return []
name = name.removeprefix("transformer.")
diff --git a/auto_round/export/export_to_gguf/export.py b/auto_round/export/export_to_gguf/export.py
index b75fe51f9..6b92695e7 100644
--- a/auto_round/export/export_to_gguf/export.py
+++ b/auto_round/export/export_to_gguf/export.py
@@ -18,9 +18,11 @@ import shutil
import torch
from pathlib import Path
import time
-from auto_round.export.export_to_gguf.convert import ModelBase
+from auto_round.export.export_to_gguf.convert import ModelBase, ModelType, get_model_architecture
from auto_round.utils import logger, LazyImport, get_block_names, flatten_list, check_to_quantized, get_module
+TMP_DIR_NAME = "tmp_dir"
+
gguf = LazyImport("gguf")
FTYPE_MAP: dict[str, gguf.LlamaFileType] = {
@@ -46,25 +48,27 @@ FTYPE_MAP: dict[str, gguf.LlamaFileType] = {
}
-def create_model_class(output_dir, model, layer_config, backend="gguf:q4_0", low_cpu_mem_usage=False):
- tmp_work_dir = Path(os.path.join(output_dir, 'tmp_dir'))
+def create_model_class(
+ output_dir, model, layer_config, backend="gguf:q4_0", low_cpu_mem_usage=False, model_type=ModelType.TEXT):
+ tmp_work_dir = Path(os.path.join(output_dir, TMP_DIR_NAME))
with torch.inference_mode():
hparams = ModelBase.load_hparams(tmp_work_dir)
- if "architectures" in hparams:
- model_architecture = hparams["architectures"][0]
- else:
- model_architecture = type(model).__name__
- if model_architecture not in ModelBase._model_classes:
- if model_architecture.replace("CausalLM", "ConditionalGeneration") in ModelBase._model_classes:
- model_architecture = model_architecture.replace("CausalLM", "ConditionalGeneration")
- elif model_architecture.replace("ConditionalGeneration", "CausalLM") in ModelBase._model_classes:
- model_architecture = model_architecture.replace("ConditionalGeneration", "CausalLM")
+ model_architecture = get_model_architecture(hparams=hparams, model_type=model_type)
+ # if "architectures" in hparams:
+ # model_architecture = hparams["architectures"][0]
+ # else:
+ # model_architecture = type(model).__name__
+ # if model_architecture not in ModelBase._model_classes:
+ # if model_architecture.replace("CausalLM", "ConditionalGeneration") in ModelBase._model_classes:
+ # model_architecture = model_architecture.replace("CausalLM", "ConditionalGeneration")
+ # elif model_architecture.replace("ConditionalGeneration", "CausalLM") in ModelBase._model_classes:
+ # model_architecture = model_architecture.replace("ConditionalGeneration", "CausalLM")
try:
- model_class = ModelBase.from_model_architecture(model_architecture)
+ model_class = ModelBase.from_model_architecture(model_architecture, model_type=model_type)
except NotImplementedError:
logger.error(f"Model {model_architecture} is not supported")
sys.exit(1)
- model_class = ModelBase.from_model_architecture(model_architecture)
+ model_class = ModelBase.from_model_architecture(model_architecture, model_type=model_type)
model_name = model.name_or_path.split('/')
if len(model_name[-1]) == 0:
model_name = model_name[-2]
@@ -93,22 +97,40 @@ def create_model_class(output_dir, model, layer_config, backend="gguf:q4_0", low
@torch.inference_mode()
-def pack_gguf_layer(name, model, backend, output_dir, layer_config, tokenizer):
+def pack_gguf_layer(
+ name,
+ model,
+ backend,
+ output_dir,
+ layer_config,
+ tokenizer,
+ processor=None,
+ image_processor=None,
+ model_type=ModelType.TEXT):
"""Export the model to gguf format."""
global gguf_model_instance_global
- tmp_work_dir = Path(os.path.join(output_dir, 'tmp_dir'))
+ tmp_work_dir = Path(os.path.join(output_dir, TMP_DIR_NAME))
if output_dir is not None and os.path.exists(output_dir) and not os.path.exists(tmp_work_dir):
logger.warning_once(f"{output_dir} already exists, this may cause model conflict")
- tmp_work_dir = Path(os.path.join(output_dir, 'tmp_dir'))
+ tmp_work_dir = Path(os.path.join(output_dir, TMP_DIR_NAME))
if "gguf_model_instance_global" not in globals():
config = model.config
+ config.save_pretrained(tmp_work_dir)
if tokenizer is not None:
tokenizer.save_pretrained(tmp_work_dir)
- config.save_pretrained(tmp_work_dir)
-
- gguf_model_instance_global = create_model_class(output_dir, model, layer_config, backend,
- low_cpu_mem_usage=True)
-
+ if processor is not None:
+ processor.save_pretrained(tmp_work_dir)
+ if image_processor is not None:
+ image_processor.save_pretrained(tmp_work_dir)
+
+ gguf_model_instance_global = [
+ create_model_class(
+ output_dir, model, layer_config, backend, low_cpu_mem_usage=True, model_type=ModelType.TEXT)
+ ]
+ if model_type == ModelType.MMPROJ:
+ gguf_model_instance_global.append(
+ create_model_class(
+ output_dir, model, layer_config, backend, low_cpu_mem_usage=True, model_type=ModelType.MMPROJ))
if not hasattr(model, "last_layer_name_to_block_name"):
block_name_to_last_layer_name = {}
block_names = get_block_names(model, quant_vision=True)
@@ -127,8 +149,9 @@ def pack_gguf_layer(name, model, backend, output_dir, layer_config, tokenizer):
model.last_layer_name_to_block_name = last_layer_name_to_block_name
if name in model.last_layer_name_to_block_name:
##packing block
- gguf_model_instance_global.current_packing_block = model.last_layer_name_to_block_name[name]
- gguf_model_instance_global.prepare_tensors()
+ for gguf_model in gguf_model_instance_global:
+ gguf_model.current_packing_block = model.last_layer_name_to_block_name[name]
+ gguf_model.prepare_tensors()
block = get_module(model, model.last_layer_name_to_block_name[name])
for n, m in block.named_modules():
@@ -140,13 +163,14 @@ def pack_gguf_layer(name, model, backend, output_dir, layer_config, tokenizer):
gc.collect()
model.last_layer_name_to_block_name.pop(name)
if len(model.last_layer_name_to_block_name) == 0:
- gguf_model_instance_global.current_packing_block = None
+ for gguf_model in gguf_model_instance_global:
+ gguf_model.current_packing_block = None
@torch.inference_mode()
-def save_quantized_as_gguf(output_dir, backend="gguf:q4_0", layer_config=None, **kwargs):
+def save_quantized_as_gguf(output_dir, backend="gguf:q4_0", layer_config=None, vlm=False, **kwargs):
"""Export the model to gguf format."""
- tmp_work_dir = Path(os.path.join(output_dir, 'tmp_dir'))
+ tmp_work_dir = Path(os.path.join(output_dir, TMP_DIR_NAME))
if output_dir is not None and os.path.exists(output_dir) and not os.path.exists(tmp_work_dir):
logger.warning(f"{output_dir} already exists, this may cause model conflict")
@@ -155,22 +179,30 @@ def save_quantized_as_gguf(output_dir, backend="gguf:q4_0", layer_config=None, *
model = kwargs["model"]
if "gguf_model_instance_global" not in globals():
- tokenizer = kwargs.get("tokenizer", None)
config = model.config
- if tokenizer is not None:
- tokenizer.save_pretrained(tmp_work_dir)
config.save_pretrained(tmp_work_dir)
-
- config = model.config
+ tokenizer = kwargs.get("tokenizer", None)
if tokenizer is not None:
tokenizer.save_pretrained(tmp_work_dir)
- config.save_pretrained(tmp_work_dir)
-
- gguf_model_instance_global = create_model_class(output_dir, model, layer_config, backend)
-
- gguf_model_instance_global.write()
- rt = time.time() - st
- logger.info(f"Model successfully exported to {gguf_model_instance_global.fname_out}, running time={rt}")
+ processor = kwargs.get("processor", None)
+ if processor is not None:
+ processor.save_pretrained(tmp_work_dir)
+ image_processor = kwargs.get("image_processor", None)
+ if image_processor is not None:
+ image_processor.save_pretrained(tmp_work_dir)
+
+
+ gguf_model_instance_global = [
+ create_model_class(output_dir, model, layer_config, backend, model_type=ModelType.TEXT)
+ ]
+ if vlm:
+ gguf_model_instance_global.append(
+ create_model_class(output_dir, model, layer_config, backend, model_type=ModelType.MMPROJ))
+
+ for gguf_model in gguf_model_instance_global:
+ gguf_model.write()
+ rt = time.time() - st
+ logger.info(f"Model successfully exported to {gguf_model.fname_out}, running time={rt}")
del gguf_model_instance_global
shutil.rmtree(tmp_work_dir, ignore_errors=True)
diff --git a/auto_round/export/export_to_gguf/packing.py b/auto_round/export/export_to_gguf/packing.py
index 80c2b1abd..e83a0c7f6 100644
--- a/auto_round/export/export_to_gguf/packing.py
+++ b/auto_round/export/export_to_gguf/packing.py
@@ -52,8 +52,20 @@ def ggml_quant(
n_blocks = data.nelement() // block_size
blocks = data.reshape((n_blocks, block_size))
quant_func = GGML_QUANT_TYPE[ggml_type]
- new_data = quant_func(
- blocks, scale, zp=zp, wmin=wmin, d_scale=d_scale, d_wmin=d_wmin, imatrix=imatrix, original=original)
+ try:
+ new_data = quant_func(
+ blocks, scale, zp=zp, wmin=wmin, d_scale=d_scale, d_wmin=d_wmin, imatrix=imatrix, original=original)
+ except Exception:
+ device = "cpu"
+ blocks = blocks.to(device)
+ scale = scale.to(device) if scale is not None else scale
+ zp = zp.to(device) if zp is not None else zp
+ wmin = wmin.to(device) if wmin is not None else wmin
+ d_scale = d_scale.to(device) if d_scale is not None else d_scale
+ d_wmin = d_wmin.to(device) if d_wmin is not None else d_wmin
+ new_data = quant_func(
+ blocks, scale, zp=zp, wmin=wmin, d_scale=d_scale, d_wmin=d_wmin, imatrix=imatrix, original=original)
+
assert new_data.shape[-1] == type_size
new_data = new_data.reshape(*shape[:-1], shape[-1] // block_size * type_size)
diff --git a/auto_round/mllm/autoround_mllm.py b/auto_round/mllm/autoround_mllm.py
index c3dded9f4..5ab25e92d 100644
--- a/auto_round/mllm/autoround_mllm.py
+++ b/auto_round/mllm/autoround_mllm.py
@@ -36,6 +36,7 @@ from .mllm_dataset import get_mllm_dataloader
from ..low_cpu_mem.utils import get_layers_before_block
+
def _only_text_test(model, tokenizer, device, model_type):
"""Test if the model whether can use text-only datasets."""
@@ -267,6 +268,7 @@ class AutoRoundMLLM(AutoRound):
to_quant_block_names=self.to_quant_block_names,
enable_norm_bias_tuning=enable_norm_bias_tuning,
enable_torch_compile=enable_torch_compile,
+ vlm=True,
**kwargs,
)
diff --git a/auto_round/mllm/processor.py b/auto_round/mllm/processor.py
index a29efb3b6..a01ac0847 100644
--- a/auto_round/mllm/processor.py
+++ b/auto_round/mllm/processor.py
@@ -136,7 +136,8 @@ class HFProcessor(BasicProcessor):
messages, tokenize=False, add_generation_prompt=not continue_final_message,
continue_final_message=continue_final_message, )
if image is not None:
- image = self.image_processor(image)
+ image = self.default_image_processor(image)
+ # image = self.image_processor(image)
ret = self.processor(
text=text, images=image, return_tensors="pt")
return ret
diff --git a/auto_round/script/mllm.py b/auto_round/script/mllm.py
index 5e3fdea8f..c25c6180b 100644
--- a/auto_round/script/mllm.py
+++ b/auto_round/script/mllm.py
@@ -195,6 +195,9 @@ class BasicArgumentParser(argparse.ArgumentParser):
self.add_argument("--device_map", default=None, type=str, help="device_map for block in tuning phase")
+ self.add_argument("--disable_opt_rtn", action='store_true',
+ help="whether to disable optimization of the RTN mode(iters=0) (default is False).")
+
def setup_parser():
parser = BasicArgumentParser()
@@ -287,16 +290,12 @@ def tune(args):
if args.format is None:
args.format = "auto_round"
- supported_formats = [
- "auto_round", "auto_round:auto_gptq", "auto_round:auto_awq", "auto_awq", "gguf:q4_0", "gguf:q4_1", "fake"
- ]
- if not args.quant_nontext_module:
- supported_formats.extend(["auto_gptq"])
formats = args.format.replace(' ', '').split(",")
+ from auto_round.utils import SUPPORTED_FORMATS
for format in formats:
- if format not in supported_formats:
- raise ValueError(f"{format} is not supported, we only support {supported_formats}")
+ if format not in SUPPORTED_FORMATS:
+ raise ValueError(f"{format} is not supported, we only support {SUPPORTED_FORMATS}")
##must set this before import torch
set_cuda_visible_devices(args.device)
@@ -387,7 +386,7 @@ def tune(args):
layer_config[lm_head_layer_name] = {"bits": args.bits}
for format in formats:
if "auto_round" not in format and "fake" not in format:
- auto_round_formats = [s for s in supported_formats if s.startswith("auto_round")]
+ auto_round_formats = [s for s in SUPPORTED_FORMATS if s.startswith("auto_round")]
raise ValueError(
f"{format} is not supported for lm-head quantization, please change to {auto_round_formats}")
@@ -449,28 +448,31 @@ def tune(args):
device_map=args.device_map,
model_kwargs=model_kwargs,
data_type=args.data_type,
+ disable_opt_rtn=args.disable_opt_rtn,
)
- model, _ = autoround.quantize()
+
+ model_name = args.model.rstrip("/")
+
+ if model_name.split('/')[-1].strip('.') == "" and "gguf" not in args.format:
+ export_dir = os.path.join(args.output_dir, f"w{autoround.bits}g{autoround.group_size}")
+ elif model_name.split('/')[-1].strip('.') == "" and "gguf" in args.format:
+ export_dir = args.output_dir
+ elif model_name.split('./')[-1].strip('./') != "" and "gguf" in args.format:
+ export_dir = os.path.join(args.output_dir,
+ model_name.split('/')[-1] + "-gguf")
+ else:
+ export_dir = os.path.join(args.output_dir,
+ model_name.split('/')[-1] + f"-w{autoround.bits}g{autoround.group_size}")
+
+ model, folders = autoround.quantize_and_save(export_dir, format=args.format)
+
+ if args.low_cpu_mem_mode == 1 or args.low_cpu_mem_mode == 2:
+ import shutil
+ shutil.rmtree(args.low_cpu_mem_tmp_dir, ignore_errors=True)
model.eval()
clear_memory()
- if model_name.split('/')[-1].strip('.') == "":
- export_dir = os.path.join(args.output_dir, f"w{args.bits}g{args.group_size}")
- else:
- export_dir = os.path.join(args.output_dir, model_name.split('/')[-1] + f"-w{args.bits}g{args.group_size}")
-
- format_list = args.format.replace(' ', '').split(',')
- inplace = False if len(format_list) > 1 else True
- for format_ in format_list:
- save_format_ = format_.replace(":", "-").replace("_", "-")
- eval_folder = os.path.join(export_dir, save_format_) if len(formats) > 1 else export_dir
- safe_serialization = True
- if hasattr(autoround.model, "config") and hasattr(autoround.model.config, "model_type"):
- if "phi3_v" in autoround.model.config.model_type:
- safe_serialization = False
- autoround.save_quantized(eval_folder, format=format_, inplace=inplace, safe_serialization=safe_serialization)
-
def eval(args):
set_cuda_visible_devices(args.device)
diff --git a/auto_round/special_model_handler.py b/auto_round/special_model_handler.py
index f602f9b3f..6d9a0ec95 100644
--- a/auto_round/special_model_handler.py
+++ b/auto_round/special_model_handler.py
@@ -22,7 +22,9 @@ SUPPORT_ONLY_TEXT_MODELS = [
"qwen2_vl",
"deepseek_vl_v2",
"chatglm",
- "idefics3"
+ "idefics3",
+ "llama4",
+ "internvl_chat"
]
SPECIAL_SHARED_CACHE_KEYS = {
diff --git a/auto_round/utils.py b/auto_round/utils.py
index be55e6585..be03caeb5 100644
--- a/auto_round/utils.py
+++ b/auto_round/utils.py
@@ -29,7 +29,7 @@ from packaging import version
import gc
from auto_round.special_model_handler import SPECIAL_MULTIMODAL_BLOCK, SPECIAL_SHARED_CACHE_KEYS
import transformers
-from auto_round.export.export_to_gguf.config import GGUF_CONFIG, GGML_QUANT_SIZES, GGUF_INNER_CONFIG, QK_K
+from auto_round.export.export_to_gguf.config import GGUF_CONFIG, GGML_QUANT_SIZES, GGUF_INNER_CONFIG, QK_K, ModelType
SHARED_CACHE_KEYS = ("position_ids", "cache_position", "position_embeddings")
@@ -1170,9 +1170,8 @@ def get_layer_features(layer):
return None, None # Unsupported layer type
-def _gguf_args_check(args_or_ar, format_str=None):
+def _gguf_args_check(args_or_ar, format_str=None, model_type=ModelType.TEXT):
from auto_round.utils import logger
- from auto_round.export.export_to_gguf.config import GGUF_CONFIG
import argparse
if format_str is None:
@@ -1191,7 +1190,7 @@ def _gguf_args_check(args_or_ar, format_str=None):
for format in GGUF_CONFIG:
if format in formats:
if format == "q6_k_s":
- logger.warning("Please not that q6_k_s is q6_k.")
+ logger.warning("Please note that q6_k_s is q6_k.")
try:
from auto_round.export.export_to_gguf.convert import ModelBase
except:
@@ -1211,7 +1210,7 @@ def _gguf_args_check(args_or_ar, format_str=None):
hparams = ModelBase.load_hparams(Path(args_or_ar.model))
model_architecture = hparams["architectures"][0]
try:
- model_class = ModelBase.from_model_architecture(model_architecture)
+ model_class = ModelBase.from_model_architecture(model_architecture, model_type=model_type)
except NotImplementedError:
logger.error(f"Model {model_architecture} is not supported to export GGUF format")
sys.exit(1)
@@ -1332,7 +1331,7 @@ def mllm_load_model(
**kwargs):
import json
import transformers
- from transformers import AutoProcessor, AutoTokenizer, AutoModelForCausalLM
+ from transformers import AutoProcessor, AutoTokenizer, AutoModelForCausalLM, AutoModel
from huggingface_hub import HfApi, hf_hub_download, HfFileSystem
if os.path.isdir(pretrained_model_name_or_path):
@@ -1395,6 +1394,12 @@ def mllm_load_model(
pretrained_model_name_or_path, trust_remote_code=trust_remote_code)
processor = AutoProcessor.from_pretrained(
pretrained_model_name_or_path, trust_remote_code=trust_remote_code)
+ try:
+ from transformers import AutoImageProcessor
+ image_processor = AutoImageProcessor.from_pretrained(
+ pretrained_model_name_or_path, trust_remote_code=trust_remote_code)
+ except Exception as e:
+ pass
model = model.eval()
model = _to_model_dtype(model, model_dtype)
@@ -1606,30 +1611,28 @@ def _gguf_type_fallback(gguf_type):
return gguf_type
##https://github.com/ggml-org/llama.cpp/blob/9e31bec4fd53634c9e5b04650488a09a055f5dab/src/llama-quant.cpp#L129
-def get_layer_config_by_gguf_format(layer_config, gguf_format, model):
+def get_layer_config_by_gguf_format(layer_config, gguf_format, model, model_type=ModelType.TEXT):
# TODO: support for other format later
target_gguf_format = next((fmt for fmt in gguf_format if fmt != "fake"), None)
import gguf # pylint: disable=E0401
- from auto_round.export.export_to_gguf.convert import ModelBase
- if model.config.architectures is not None:
- model_architecture = model.config.architectures[0]
- else:
- model_architecture = type(model).__name__
- if model_architecture not in ModelBase._model_classes:
- if model_architecture.replace("CausalLM", "ConditionalGeneration") in ModelBase._model_classes:
- model_architecture = model_architecture.replace("CausalLM", "ConditionalGeneration")
- elif model_architecture.replace("ConditionalGeneration", "CausalLM") in ModelBase._model_classes:
- model_architecture = model_architecture.replace("ConditionalGeneration", "CausalLM")
+ from auto_round.export.export_to_gguf.convert import ModelBase, get_model_architecture
+ model_architecture = get_model_architecture(hparams=model.config.to_dict(), model_type=model_type)
try:
- model_class = ModelBase.from_model_architecture(model_architecture)
+ model_class = ModelBase.from_model_architecture(model_architecture, model_type=model_type)
except NotImplementedError:
return layer_config, {}
n_layer = None
for name in ["n_layers", "num_hidden_layers", "n_layer", "num_layers"]:
+ sub_attr = "text_config" if model_type == ModelType.TEXT else "vision_config"
if hasattr(model.config, name):
n_layer = getattr(model.config, name)
+ break
+ if hasattr(model.config, sub_attr):
+ if hasattr(getattr(model.config, sub_attr), name):
+ n_layer = getattr(getattr(model.config, sub_attr), name)
+ break
if n_layer is None:
return layer_config, {}
@@ -1830,10 +1833,10 @@ def get_layer_config_by_gguf_format(layer_config, gguf_format, model):
and 'Deepseek' in model.config.architectures[0]:
fallback = False
- # calc if need fallback
+ # calc if need fallback
qk_nope_head_dim = model.config.qk_nope_head_dim
kv_b_shape = get_module(model, layer_name).weight.shape
-
+
if qk_nope_head_dim < QK_K or qk_nope_head_dim % QK_K != 0 \
or kv_b_shape[-1] < QK_K or kv_b_shape[-1] % QK_K != 0:
fallback = True
@@ -1909,9 +1912,11 @@ def flatten_list(nested_list):
return flattened
def clean_module_parameter(submodule, parameter):
+ if submodule is None:
+ return
is_buffer = parameter in submodule._buffers
with torch.no_grad():
if is_buffer:
submodule._buffers[parameter] = None
else:
- submodule._parameters[parameter] = None
\ No newline at end of file
+ submodule._parameters[parameter] = None
diff --git a/test/test_cpu/test_gguf_format.py b/test/test_cpu/test_gguf_format.py
index 4f33b7204..ae5cfba5a 100644
--- a/test/test_cpu/test_gguf_format.py
+++ b/test/test_cpu/test_gguf_format.py
@@ -238,7 +238,7 @@ class TestGGUF(unittest.TestCase):
def test_q4_k_m(self):
- model_name = "Qwen/Qwen2.5-7B-Instruct"
+ model_name = "Qwen/Qwen2.5-1.5B-Instruct"
model = AutoModelForCausalLM.from_pretrained(model_name, torch_dtype="auto", trust_remote_code=True)
tokenizer = AutoTokenizer.from_pretrained(model_name, trust_remote_code=True)
layer_config = {
@@ -257,7 +257,7 @@ class TestGGUF(unittest.TestCase):
disable_opt_rtn=True
)
quantized_model_path = "./saved"
- autoround.quantize_and_save(output_dir=quantized_model_path, format="gguf:q4_k_m")
+ autoround.quantize_and_save(output_dir=quantized_model_path, format="gguf:q4_k_m,fake")
self.assertEqual(autoround.layer_config["model.layers.11.self_attn.v_proj"]["super_group_size"], 16)
self.assertEqual(autoround.layer_config["model.layers.11.self_attn.v_proj"]["data_type"], "int_sym_dq")
self.assertEqual(autoround.layer_config["model.layers.7.self_attn.v_proj"]["data_type"], "int_asym_dq")
@@ -269,27 +269,57 @@ class TestGGUF(unittest.TestCase):
self.assertEqual(autoround.model.model.layers[10].mlp.gate_proj.bits, 8)
self.assertEqual(autoround.layer_config['model.layers.10.mlp.gate_proj']['mostly'], "gguf:q8_0")
shutil.rmtree("./saved", ignore_errors=True)
+
+ autoround = AutoRound(
+ model,
+ tokenizer,
+ iters=0,
+ nsamples=1,
+ seqlen=128,
+ disable_opt_rtn=False
+ )
+ quantized_model_path = "./saved"
+ autoround.quantize_and_save(output_dir=quantized_model_path, format="gguf:q4_k_m,fake")
+ shutil.rmtree("./saved", ignore_errors=True)
def test_all_format(self):
- from auto_round.export.export_to_gguf.config import GGUF_CONFIG
- model_name = "Qwen/Qwen2.5-7B-Instruct"
+ model_name = "Qwen/Qwen2.5-1.5B-Instruct"
python_path = sys.executable
- for gguf_format in GGUF_CONFIG.keys():
+ for gguf_format in ["gguf:q4_0", "gguf:q4_1", "gguf:q4_k_m", "gguf:q6_k"]:
res = os.system(
f"cd ../.. && {python_path} -m auto_round --model {model_name} "
- f" --bs 16 --iters 1 --nsamples 1 --format {gguf_format}"
+ f" --bs 16 --iters 1 --nsamples 1 --seqlen 16 --format {gguf_format}"
)
if res > 0 or res == -1:
assert False, "cmd line test fail, please have a check"
shutil.rmtree("../../tmp_autoround", ignore_errors=True)
res = os.system(
- f"cd ../.. && {python_path} -m auto_round --model {model_name} "
- f" --bs 16 --iters 0 --nsamples 1 --format fake,{gguf_format}"
+ f"cd ../.. && {python_path} -m auto_round --model {model_name}"
+ f" --bs 16 --iters 0 --nsamples 1 --seqlen 16 --format fake,{gguf_format}"
)
if res > 0 or res == -1:
assert False, "cmd line test fail, please have a check"
shutil.rmtree("../../tmp_autoround", ignore_errors=True)
+
+ def test_vlm_gguf(self):
+ model_name = "Qwen/Qwen2-VL-2B-Instruct"
+ from auto_round.mllm.autoround_mllm import AutoRoundMLLM
+ from auto_round.utils import mllm_load_model
+ model, processor, tokenizer, image_processor = mllm_load_model(model_name)
+ autoround = AutoRoundMLLM(
+ model,
+ tokenizer=tokenizer,
+ processor=processor,
+ image_processor=image_processor,
+ iters=0,
+ )
+ quantized_model_path = "./saved"
+ autoround.quantize_and_save(output_dir=quantized_model_path, format="gguf:q4_0")
+ self.assertTrue("mmproj-model.gguf" in os.listdir("./saved"))
+ file_size = os.path.getsize("./saved/Qwen2-VL-2B-Instruct-1.5B-Q4_0.gguf") / 1024 ** 2
+ self.assertAlmostEqual(file_size, 892, delta=1.0)
+ shutil.rmtree("./saved", ignore_errors=True)
if __name__ == "__main__":
unittest.main()
diff --git a/test/test_cuda/test_gguf.py b/test/test_cuda/test_gguf.py
index 05e212e7e..d6cd50391 100644
--- a/test/test_cuda/test_gguf.py
+++ b/test/test_cuda/test_gguf.py
@@ -171,11 +171,13 @@ class TestAutoRound(unittest.TestCase):
self.assertGreater(result['results']['piqa']['acc,none'], 0.55)
shutil.rmtree("./saved", ignore_errors=True)
+ @require_gguf
def test_all_format(self):
from auto_round.export.export_to_gguf.config import GGUF_CONFIG
python_path = sys.executable
- for model_name in ["/models/Qwen3-8B/", "/models/Llama-3.2-3B/"]:
+ for model_name in ["/models/Qwen3-8B/", "/models/Llama-3.2-3B/", "/models/Meta-Llama-3.1-8B-Instruct"]:
for gguf_format in GGUF_CONFIG.keys():
+ print(model_name, gguf_format)
res = os.system(
f"cd ../.. && {python_path} -m auto_round --model {model_name} "
f" --bs 16 --iters 1 --nsamples 1 --format fake,{gguf_format}"
@@ -192,5 +194,73 @@ class TestAutoRound(unittest.TestCase):
assert False, "cmd line test fail, please have a check"
shutil.rmtree("../../tmp_autoround", ignore_errors=True)
+ @require_gguf
+ def test_vlm_gguf(self):
+ model_name = "/models/Qwen2.5-VL-7B-Instruct"
+ from auto_round.mllm.autoround_mllm import AutoRoundMLLM
+ from auto_round.utils import mllm_load_model
+ model, processor, tokenizer, image_processor = mllm_load_model(model_name)
+ autoround = AutoRoundMLLM(
+ model,
+ tokenizer=tokenizer,
+ processor=processor,
+ image_processor=image_processor,
+ device="auto",
+ iters=0,
+ )
+ quantized_model_path = "./saved"
+ autoround.quantize_and_save(output_dir=quantized_model_path, format="gguf:q4_0")
+ self.assertTrue("mmproj-model.gguf" in os.listdir("./saved"))
+ file_size = os.path.getsize("./saved/Qwen2.5-VL-7B-Instruct-7.6B-Q4_0.gguf") / 1024 ** 2
+ self.assertAlmostEqual(file_size, 4226, delta=1.0)
+ file_size = os.path.getsize("./saved/mmproj-model.gguf") / 1024 ** 2
+ self.assertAlmostEqual(file_size, 2578, delta=1.0)
+ shutil.rmtree("./saved", ignore_errors=True)
+
+ model_name = "/models/gemma-3-12b-it"
+ from auto_round.mllm.autoround_mllm import AutoRoundMLLM
+ from auto_round.utils import mllm_load_model
+ model, processor, tokenizer, image_processor = mllm_load_model(model_name)
+ autoround = AutoRoundMLLM(
+ model,
+ tokenizer=tokenizer,
+ processor=processor,
+ image_processor=image_processor,
+ device="auto",
+ nsamples=32,
+ iters=1,
+ )
+ quantized_model_path = "./saved"
+ autoround.quantize_and_save(output_dir=quantized_model_path, format="gguf:q4_k_m")
+ self.assertTrue("mmproj-model.gguf" in os.listdir("./saved"))
+ file_size = os.path.getsize("./saved/gemma-3-12b-it-12B-Q4_K_M.gguf") / 1024 ** 2
+ self.assertAlmostEqual(file_size, 6568, delta=1.0)
+ file_size = os.path.getsize("./saved/mmproj-model.gguf") / 1024 ** 2
+ self.assertAlmostEqual(file_size, 1599, delta=1.0)
+ shutil.rmtree("./saved", ignore_errors=True)
+
+ @require_gguf
+ def test_llama_4(self):
+ model_name = "/dataset/Llama-4-Scout-17B-16E-Instruct/"
+ from auto_round.mllm.autoround_mllm import AutoRoundMLLM
+ from auto_round.utils import mllm_load_model
+ model, processor, tokenizer, image_processor = mllm_load_model(model_name)
+ autoround = AutoRoundMLLM(
+ model,
+ tokenizer=tokenizer,
+ processor=processor,
+ image_processor=image_processor,
+ device="auto",
+ iters=0,
+ )
+ quantized_model_path = "/dataset/Llam-4-test"
+ autoround.quantize_and_save(output_dir=quantized_model_path, format="gguf:q4_0")
+ self.assertTrue("mmproj-model.gguf" in os.listdir(quantized_model_path))
+ file_size = os.path.getsize(os.path.join(quantized_model_path, "Llama-4-Scout-17B-16E-Instruct-16x17B-Q4_0.gguf")) / 1024 ** 2
+ self.assertAlmostEqual(file_size, 58093.62, delta=1.0)
+ file_size = os.path.getsize(os.path.join(quantized_model_path, "mmproj-model.gguf")) / 1024 ** 2
+ self.assertAlmostEqual(file_size, 3323.52, delta=1.0)
+ shutil.rmtree(quantized_model_path, ignore_errors=True)
+
if __name__ == "__main__":
unittest.main()
\ No newline at end of file