PTQ-Level Efficiency, QAT-Level Accuracy | 训练后量化的效率,量化感知训练的精度
GGUF format add support for MoE models with non-linear expert layers. (#1244)
Heng Guoc7d6aee671dc
diff --git a/auto_round/compressors/base.py b/auto_round/compressors/base.pyindex 5605393e6..d473e1bf3 100644--- a/auto_round/compressors/base.py+++ b/auto_round/compressors/base.py@@ -58,7 +58,6 @@ from auto_round.export.export_to_gguf.config import GGUF_INNER_CONFIG, ModelTypefrom auto_round.formats import OutputFormat, get_formatsfrom auto_round.logger import loggerfrom auto_round.schemes import (- SPECIAL_SCHEMES,QuantizationScheme,_handle_special_schemes,get_gguf_scheme,@@ -610,8 +609,7 @@ class BaseCompressor(object):scheme = scheme.strip("'\" ")res = schemescheme = scheme.upper()- if scheme in SPECIAL_SCHEMES:- self.layer_config = _handle_special_schemes(scheme, self.layer_config, self.model)+ self.layer_config = _handle_special_schemes(scheme, self.layer_config, self.model)scheme = asdict(preset_name_to_scheme(scheme))scheme_keys = [f.name for f in fields(QuantizationScheme)]for key in scheme_keys:diff --git a/auto_round/export/export_to_gguf/config.py b/auto_round/export/export_to_gguf/config.pyindex d4e53ed5b..56f2af953 100644--- a/auto_round/export/export_to_gguf/config.py+++ b/auto_round/export/export_to_gguf/config.py@@ -193,6 +193,7 @@ GGUF_CONFIG["gguf:q8_0"]["mostly"] = "gguf:q8_0"# GGUF_CONFIG["gguf:fp16"]["mostly"]= "gguf:fp16"# GGUF_CONFIG["gguf:bf16"] = GGUF_INNER_CONFIG["gguf:fp16"]# GGUF_CONFIG["gguf:bf16"]["mostly"]= "gguf:bf16"+GGUF_CONFIG["gguf:q2_k_mixed"] = GGUF_INNER_CONFIG["gguf:q2_k"]QK_K = 256diff --git a/auto_round/export/export_to_gguf/convert.py b/auto_round/export/export_to_gguf/convert.pyindex 1b7e0735d..ed99b1485 100644--- a/auto_round/export/export_to_gguf/convert.py+++ b/auto_round/export/export_to_gguf/convert.py@@ -229,51 +229,45 @@ def _quant_data(cls, data_torch, data_qtype, name, modify_name, new_name, bid, d"""suffix = ".weight"device = data_torch.device if device is None else device- if suffix in name:++ if name.endswith(suffix):layer_name = name[: -len(suffix)]- module = get_module(cls.model, layer_name)- kwargs = {- "scale": None,- "zp": None,- "d_scale": None,- "d_wmin": None,- "wmin": None,- "imatrix": None,- }- if hasattr(module, "scale"):+ else:+ layer_name = name+ module = get_module(cls.model, layer_name)+ kwargs = {+ "scale": None,+ "zp": None,+ "d_scale": None,+ "d_wmin": None,+ "wmin": None,+ "imatrix": None,+ }+ # support for MOE model with cls eexperts not linear+ # if hasattr(module, "scale") or ("exps" in new_name and len(data_torch.shape) == 3):+ for attr in ["scale", "zp", "w_d_scale", "w_d_wmin", "w_wmin"]:+ if hasattr(module, attr) and getattr(module, attr) is not None:+ attr_tensor = getattr(module, attr)+ if not isinstance(attr_tensor, torch.Tensor):+ continueif hasattr(cls, "permute"):bs = module.weight.shape[0]- for attr in ["scale", "zp", "w_d_scale", "w_d_wmin", "w_wmin"]:- if hasattr(module, attr) and getattr(module, attr) is not None:- attr_tensor = getattr(module, attr)- if not isinstance(attr_tensor, torch.Tensor):- continue- attr_tensors_dict = dict(cls.modify_tensors(attr_tensor.reshape(bs, -1), modify_name, bid))- attr_tensor = attr_tensors_dict[new_name]- if attr in kwargs:- kwargs[attr] = attr_tensor.to(torch.float32)- else:- kwargs[attr.replace("w_", "")] = attr_tensor.to(torch.float32)- data_torch = data_torch.to(torch.float32)+ attr_tensors_dict = dict(cls.modify_tensors(attr_tensor.reshape(bs, -1), modify_name, bid))+ attr_tensor = attr_tensors_dict[new_name]+ if attr in kwargs:+ kwargs[attr] = attr_tensor.to(torch.float32)+ else:+ kwargs[attr.replace("w_", "")] = attr_tensor.to(torch.float32)+ data_torch = data_torch.to(torch.float32)- data = ggml_quant(data_torch, data_qtype.name.lower(), device=device, **kwargs)- else:- # if data_torch.dtype ==torch.float32:- # data_qtype = gguf.GGMLQuantizationType.F32- # else:- # data_qtype = gguf.GGMLQuantizationType.F16- data_qtype = gguf.GGMLQuantizationType.F32 ##FP16 has issues at inference- data = data_torch.to(torch.float32).squeeze().cpu().numpy()- else:- # for Llama-4- # if data_torch.dtype == torch.float32:- # data_qtype = gguf.GGMLQuantizationType.F32- # else:- # data_qtype = gguf.GGMLQuantizationType.F16- # data = data_torch.squeeze().cpu().numpy()- # data_qtype = gguf.GGMLQuantizationType.F32- # data = data_torch.to(torch.float32).squeeze().cpu().numpy()- data = ggml_quant(data_torch, data_qtype.name.lower(), device=device)+ data = ggml_quant(data_torch, data_qtype.name.lower(), device=device, **kwargs)+ # else:+ # # if data_torch.dtype ==torch.float32:+ # # data_qtype = gguf.GGMLQuantizationType.F32+ # # else:+ # # data_qtype = gguf.GGMLQuantizationType.F16+ # data_qtype = gguf.GGMLQuantizationType.F32 ##FP16 has issues at inference+ # data = data_torch.to(torch.float32).squeeze().cpu().numpy()return data, data_qtype@@ -419,7 +413,9 @@ def prepare_tensors(cls):breakif skip:continue- data = data_torch.squeeze()+ # sync with new version of gguf+ # data = data_torch.squeeze()+ data = data_torchn_dims = len(data.shape)data_qtype: gguf.GGMLQuantizationType | bool = cls.tensor_force_quant(name, new_name, bid, n_dims)@@ -529,17 +525,30 @@ def prepare_tensors(cls):elif data_qtype == gguf.GGMLQuantizationType.Q6_K:data_qtype = gguf.GGMLQuantizationType.Q8_0+ from auto_round.export.export_to_gguf.config import GGML_QUANT_SIZES++ if data_qtype.name.lower() in GGML_QUANT_SIZES:+ block_size, type_size = GGML_QUANT_SIZES[data_qtype.name.lower()]+ if data_torch.shape[-1] % block_size != 0:+ logger.warning(+ f"{new_name}: Can't quantize tensor with shape {data_torch.shape} to {data_qtype.name},"+ " fallback to F16"+ )+ data_qtype = gguf.GGMLQuantizationType.F16+if isinstance(data_qtype, bool) or data_qtype in [gguf.GGMLQuantizationType.F16,gguf.GGMLQuantizationType.BF16,gguf.GGMLQuantizationType.F32,]:- data = data_torch.squeeze().cpu().numpy()+ # sync with new version of gguf+ # data = data_torch.squeeze().cpu().numpy()# if data ends up empty, it means data_torch was a scalar tensor -> restore- if len(data.shape) == 0:+ if len(data_torch.shape) == 0:data = data_torch.numpy()try:+ data = data_torch.cpu().numpy()data = gguf.quants.quantize(data, data_qtype)except gguf.QuantError as e:logger.warning("%s, %s", e, "falling back to F16")diff --git a/auto_round/export/export_to_gguf/packing.py b/auto_round/export/export_to_gguf/packing.pyindex 9a48cfa0d..c64066932 100644--- a/auto_round/export/export_to_gguf/packing.py+++ b/auto_round/export/export_to_gguf/packing.py@@ -22,6 +22,7 @@ GGML_QUANT_TYPE = {}def register_qtype(name):+def register(cls):GGML_QUANT_TYPE[name] = clsreturn cls@@ -109,7 +110,6 @@ def ggml_quant(else:new_data = np.concatenate(results, axis=0)new_data = new_data.reshape(*shape[:-1], shape[-1] // block_size * type_size) # Check shape correctness- new_data = new_data.reshape(*shape[:-1], -1)return new_datadiff --git a/auto_round/formats.py b/auto_round/formats.pyindex b2f172f40..15d8b34a6 100644--- a/auto_round/formats.py+++ b/auto_round/formats.py@@ -76,8 +76,8 @@ def _check_compatibility(formats: list[str], ar: BaseCompressor):)gguf_format_name = get_gguf_scheme(ar.scheme)if gguf_format_name:- if gguf_format_name.lower().endswith("mixed"):- gguf_format_name = gguf_format_name.lower().replace("_mixed", "_s")+ # if gguf_format_name.lower().endswith("mixed"):+ # gguf_format_name = gguf_format_name.lower().replace("_mixed", "_s")if any([f.lower() not in ["fake", gguf_format_name.lower()] for f in formats]):tmp_format_name = gguf_format_name.lower() if "fake" not in formats else f"{gguf_format_name.lower()},fake"logger.warning(@@ -98,7 +98,7 @@ def get_formats(seen = set()return [x for x in lst if not (x in seen or seen.add(x))]- formats = format.replace("q*_", f"q{ar.bits}_").replace(" ", "").split(",")+ formats = format.lower().replace("q*_", f"q{ar.bits}_").replace(" ", "").split(",")formats = remove_duplicates(formats) # need the keep origin orderformats = _check_compatibility(formats, ar)@@ -650,6 +650,7 @@ class GGUFFormat(OutputFormat):"GGUF:Q5_K_M","GGUF:Q6_K","GGUF:Q8_0",+ "GGUF:Q2_K_MIXED",]format_name = "gguf"@@ -658,13 +659,22 @@ class GGUFFormat(OutputFormat):self.gguf_args_check(ar, format, model_type=ModelType.TEXT)if ar.mllm:self.gguf_args_check(ar, format, model_type=ModelType.MMPROJ)- ar.scheme = format.upper()self.output_format = "gguf"self.backend_cls = GGUFFormatself.backend = GGUFFormat(format.split(":")[-1], ar)else:- self.output_format = f"gguf:{format}"+ scheme = ar.scheme+ gguf_format = f"gguf:{format.lower()}"+ if format.lower().endswith("_mixed"):+ from auto_round.schemes import _handle_special_schemes++ ar.layer_config = _handle_special_schemes(scheme, ar.layer_config, ar.model)+ gguf_format = gguf_format.lower().replace("_mixed", "_s")+ if isinstance(scheme, str) and scheme.lower() != gguf_format:+ logger.warning(f"reset scheme {scheme.lower()} to {gguf_format} for gguf format export")+ ar.scheme = gguf_format+ self.output_format = gguf_formatself.backend = Noneself.mllm = ar.mllmdiff --git a/auto_round/schemes.py b/auto_round/schemes.pyindex 0ca7710e0..95078742c 100644--- a/auto_round/schemes.py+++ b/auto_round/schemes.py@@ -295,16 +295,13 @@ for key, val in GGUF_CONFIG.items():value.pop("lm_head", None)PRESET_SCHEMES[key.upper()] = QuantizationScheme.from_dict(value)-SPECIAL_SCHEMES = {"GGUF:Q2_K_MIXED": PRESET_SCHEMES["GGUF:Q2_K_S"]}-PRESET_SCHEMES.update(SPECIAL_SCHEMES)-def _handle_special_schemes(scheme_name: str, layer_config: dict, model: torch.nn.Module) -> dict:"""handle special schemes, like GGUF:Q2_K_MIXED.Provide some special auto_round recipes."""- if scheme_name == "GGUF:Q2_K_MIXED":+ if scheme_name.lower() == "gguf:q2_k_mixed":for n, m in model.named_modules():if n in layer_config:continuediff --git a/docs/step_by_step.md b/docs/step_by_step.mdindex 06e65c56a..654d55d50 100644--- a/docs/step_by_step.md+++ b/docs/step_by_step.md@@ -149,12 +149,14 @@ adopted within the community, **only 4-bits quantization is supported**. Please**LLM-Compressor Format**: **NVFP4, MXFP4(kernel in WIP), MXFP8 are supported**. Please set `--format llm_compressor`#### Format and scheme support matrix+> Italics indicates the absence of a kernel or the presence of only an inefficient/reference kernel.+|export format | supported scheme ||--------------|------------------|-|**auto_round** | W4A16, W2A16, W3A16, W8A16, MXFP4, MXFP8, NVFP4, FPW8A16, W2A16G64, W2A16G32, FP8_STATIC, BF16|+|**auto_round** | W4A16, W2A16, W3A16, W8A16, *MXFP4*, *MXFP8*, NVFP4, FPW8A16, W2A16G64, W2A16G32, *FP8_STATIC*, BF16||**auto_awq / auto_round:auto_awq** | W4A16||**auto_gptq / auto_round:auto_gptq / auto_round:gptqmodel**|W4A16, W2A16, W3A16, W8A16, BF16, W2A16G64, W2A16G32|-|**llm_compressor / auto_round:llm_compressor** | MXFP4, MXFP8, NVFP4, FPW8A16, FP8_STATIC |+|**llm_compressor / auto_round:llm_compressor** | *MXFP4*, *MXFP8*, NVFP4, FPW8A16, *FP8_STATIC* ||**gguf** | GGUF:Q4_0, GGUF:Q4_1, GGUF:Q5_0, GGUF:Q5_1, GGUF:Q2_K_S, GGUF:Q3_K_S, GGUF:Q3_K_M, GGUF:Q3_K_L, GGUF:Q4_K_S, GGUF:Q4_K_M, GGUF:Q5_K_S, GGUF:Q5_K_M, GGUF:Q6_K, GGUF:Q8_0 ||**fake** | all scheme|### Hardware Compatibilitydiff --git a/test/helpers.py b/test/helpers.pyindex 6bd5ad30c..e8630e1fa 100644--- a/test/helpers.py+++ b/test/helpers.py@@ -70,6 +70,18 @@ def get_tiny_model(model_name_or_path, num_layers=2, is_mllm=False, **kwargs):if hasattr(model.config, "num_hidden_layers"):model.config.num_hidden_layers = num_layers+ if hasattr(model.config, "text_config"):+ n_block_keys = ["n_layers", "num_hidden_layers", "n_layer", "num_layers", "depth", "encoder_layers"]+ for key in n_block_keys:+ if hasattr(model.config.text_config, key):+ setattr(model.config.text_config, key, num_layers)+ if hasattr(model.config.text_config, "layer_types"):+ model.config.text_config.layer_types = model.config.text_config.layer_types[:num_layers]+ if hasattr(model.config, "vision_config"):+ n_block_keys = ["n_layers", "num_hidden_layers", "n_layer", "num_layers", "depth", "encoder_layers"]+ for key in n_block_keys:+ if hasattr(model.config.vision_config, key):+ setattr(model.config.vision_config, key, num_layers)if hasattr(model.config, "layer_types"):model.config.layer_types = model.config.layer_types[:num_layers]diff --git a/test/test_cpu/export/test_gguf_format.py b/test/test_cpu/export/test_gguf_format.pyindex 7304f22c7..feeddcd0f 100644--- a/test/test_cpu/export/test_gguf_format.py+++ b/test/test_cpu/export/test_gguf_format.py@@ -189,19 +189,26 @@ class TestGGUF:assert False, "cmd line test fail, please have a check"shutil.rmtree("../../tmp_autoround", ignore_errors=True)+ res = os.system(+ f"PYTHONPATH='AUTO_ROUND_PATH:$PYTHONPATH' {python_path} -m auto_round --model {model_name}"+ f" --bs 16 --iters 0 --nsamples 1 --seqlen 16 --format gguf:q2_k_mixed"+ )+ if res > 0 or res == -1:+ assert False, "cmd line test fail, please have a check"+ shutil.rmtree("../../tmp_autoround", ignore_errors=True)+def test_vlm_gguf(self):+ from ...helpers import save_tiny_model+model_name = get_model_path("Qwen/Qwen2-VL-2B-Instruct")+ tiny_model_path = save_tiny_model(model_name, "./tmp/tiny_qwen_vl_model_path", num_layers=3, is_mllm=True)from auto_round import AutoRoundMLLM- from auto_round.utils import mllm_load_model- model, processor, tokenizer, image_processor = mllm_load_model(model_name)autoround = AutoRoundMLLM(- model,- tokenizer=tokenizer,- processor=processor,- image_processor=image_processor,+ tiny_model_path,iters=0,nsamples=8,+ disable_opt_rtn=True,)quantized_model_path = "./saved"autoround.quantize_and_save(output_dir=quantized_model_path, format="gguf:q4_0")@@ -209,10 +216,11 @@ class TestGGUF:for file_name in os.listdir(quantized_model_path):file_size = os.path.getsize(os.path.join(quantized_model_path, file_name)) / 1024**2if file_name == "mmproj-model.gguf":- assert abs(file_size - 2537) < 5.0+ assert abs(file_size - 56) < 5.0else:- assert abs(file_size - 892) < 5.0+ assert abs(file_size - 264) < 5.0shutil.rmtree("./saved", ignore_errors=True)+ shutil.rmtree(tiny_model_path, ignore_errors=True)def test_qtype_setting(self):# Qwen2.5-0.5B-Instruct no output, token_embed q6_k fallbakc to q8_0 336Mdiff --git a/test/test_cuda/export/test_gguf.py b/test/test_cuda/export/test_gguf.pyindex 4146cb938..a6089a086 100644--- a/test/test_cuda/export/test_gguf.py+++ b/test/test_cuda/export/test_gguf.py@@ -140,46 +140,52 @@ class TestAutoRound:shutil.rmtree(self.save_dir, ignore_errors=True)@require_gguf- def test_vlm_gguf(self):- model_name = "/models/Qwen2-VL-2B-Instruct"- from auto_round import AutoRoundMLLM- from auto_round.utils import mllm_load_model+ def test_special_model(self):+ from ...helpers import save_tiny_model- model, processor, tokenizer, image_processor = mllm_load_model(model_name)- autoround = AutoRoundMLLM(- model,- tokenizer=tokenizer,- processor=processor,- image_processor=image_processor,- device="auto",+ model_name = get_model_path("ibm-granite/granite-4.0-h-tiny")+ tiny_model_path = save_tiny_model(model_name, "tiny_model_path", num_layers=2)+ from auto_round import AutoRound++ autoround = AutoRound(+ tiny_model_path,iters=0,+ nsamples=8,+ disable_opt_rtn=True,)quantized_model_path = "./saved"autoround.quantize_and_save(output_dir=quantized_model_path, format="gguf:q4_0")- assert "mmproj-model.gguf" in os.listdir("./saved")- file_size = os.path.getsize("./saved/Qwen2-VL-2B-Instruct-Q4_0.gguf") / 1024**2- assert abs(file_size - 894) < 5.0- file_size = os.path.getsize("./saved/mmproj-model.gguf") / 1024**2- assert abs(file_size - 2580) < 5.0+ file_name = os.listdir(quantized_model_path)[0]+ file_size = os.path.getsize(os.path.join(quantized_model_path, file_name)) / 1024**2+ assert abs(file_size - 307) < 5.0shutil.rmtree("./saved", ignore_errors=True)+ shutil.rmtree(tiny_model_path, ignore_errors=True)- model_name = "/models/gemma-3-12b-it"+ @require_gguf+ def test_vlm_gguf(self):+ from ...helpers import save_tiny_model- model, processor, tokenizer, image_processor = mllm_load_model(model_name)- autoround = AutoRoundMLLM(- model,- tokenizer=tokenizer,- processor=processor,- image_processor=image_processor,+ model_name = "/models/gemma-3-4b-it"+ tiny_model_path = save_tiny_model(model_name, "tiny_model_path", num_layers=3, is_mllm=True)+ from auto_round import AutoRound++ autoround = AutoRound(+ tiny_model_path,device="auto",nsamples=32,iters=0,+ disable_opt_rtn=True,)quantized_model_path = "./saved"autoround.quantize_and_save(output_dir=quantized_model_path, format="gguf:q4_k_m")assert "mmproj-model.gguf" in os.listdir("./saved")- file_size = os.path.getsize("./saved/gemma-3-12B-it-Q4_K_M.gguf") / 1024**2- assert abs(file_size - 6568) < 5.0- file_size = os.path.getsize("./saved/mmproj-model.gguf") / 1024**2- assert abs(file_size - 1599) < 5.0+ for file in os.listdir("./saved"):+ print(f"{file}: {os.path.getsize(os.path.join('./saved', file)) / 1024**2} MB")+ file_size = os.path.getsize(os.path.join("./saved", file)) / 1024**2+ if "mmproj-model.gguf" in file:+ assert abs(file_size - 75) < 5.0+ else:+ assert abs(file_size - 690) < 5.0+shutil.rmtree(quantized_model_path, ignore_errors=True)+ shutil.rmtree(tiny_model_path, ignore_errors=True)