PTQ-Level Efficiency, QAT-Level Accuracy | 训练后量化的效率,量化感知训练的精度
fix lots of gguf critic bugs and support imatrix in rtn mode (#595)
Wenhua Chengf9921cdf5837
diff --git a/README.md b/README.mdindex 9d33b4bd4..6b4e92d0a 100644--- a/README.md+++ b/README.md@@ -277,10 +277,7 @@ models. Besides, recently 3 bits may have some accuracy issues in Transformers.**AutoAWQ Format**: This format is well-suited for asymmetric 4-bit quantization on CUDA devices and is widelyadopted within the community, **only 4-bits quantization is supported**.-**GGUF** Format: This format is well-suited for CPU devices and is widely adopted by the community. Mixed-bit-configs like `q4_k_m` are not supported yet. Please note: In contrast to the official implementation, AutoRound does not-quantize the embedding layer or the LM head layer by default.-+**GGUF** Format: Experimental feature. This format is well-suited for CPU devices and is widely adopted by the community.### Quantization CostsTesting was conducted on the Nvidia A100 80G using the nightly version of PyTorch 2.6.0.dev20241029+cu124. Please notediff --git a/auto_round/__init__.py b/auto_round/__init__.pyindex f492c5cf9..af0aa1834 100644--- a/auto_round/__init__.py+++ b/auto_round/__init__.py@@ -11,8 +11,8 @@# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.# See the License for the specific language governing permissions and# limitations under the License.-from .autoround import AutoRound, AutoRoundAdam, AutoRoundOPT-from .mllm import AutoRoundMLLM+from auto_round.autoround import AutoRound, AutoRoundAdam, AutoRoundOPT+from auto_round.mllm import AutoRoundMLLMfrom auto_round.utils import LazyImportdef __getattr__(name):diff --git a/auto_round/autoround.py b/auto_round/autoround.pyindex ebc39a8ac..0ceef40c1 100644--- a/auto_round/autoround.py+++ b/auto_round/autoround.py@@ -24,8 +24,10 @@ from transformers import set_seedfrom torch import autocastfrom tqdm import tqdmimport accelerate-from .wrapper import WrapperMultiblock, wrapper_block, unwrapper_block, WrapperLinear, unwrapper_layer-from .utils import (++from auto_round.export.export_to_gguf.config import GGUF_CONFIG, GGUF_INNER_CONFIG+from auto_round.wrapper import WrapperMultiblock, wrapper_block, unwrapper_block, WrapperLinear, unwrapper_layer+from auto_round.utils import (CpuInfo,block_forward,check_is_cpu,@@ -54,7 +56,7 @@ from .utils import (init_cache, check_skippable_keywords, get_shared_keys, SUPPORTED_DTYPES, infer_bits_by_data_type,_gguf_args_check,check_seqlen_compatible,- get_layer_config_by_gguf_format+ get_layer_config_by_gguf_format, get_lm_head_name)from .low_cpu_mem.utils import get_layers_before_block@@ -120,6 +122,7 @@ class AutoRound(object):enable_norm_bias_tuning (bool): Whether to enable fast norm/layer_bias tuningenable_torch_compile (bool): Whether to enable torch compile to optimize quant_block/layer (default it False).device_map (str|dict): device map for each block+ disable_opt_rtn (bool): Whether to disable optimization of the RTN mode(iters=0) (default is False).Returns:The quantized model."""@@ -153,7 +156,7 @@ class AutoRound(object):not_use_best_mse: bool = False,dynamic_max_gap: int = -1,data_type: str = "int",- scale_dtype: str = "fp16",+ scale_dtype: Union[str, torch.dtype] = "fp16",act_bits: int = 16,act_group_size: int = None,act_sym: bool = None,@@ -165,6 +168,7 @@ class AutoRound(object):device_map: Union[str, dict] = None,super_bits: int = None,super_group_size: int = None,+ disable_opt_rtn: bool = False,model_kwargs: dict = None,**kwargs,):@@ -253,6 +257,8 @@ class AutoRound(object):self.super_bits = super_bitsself.super_group_size = super_group_size+ self.disable_opt_rtn = disable_opt_rtn+torch.set_printoptions(precision=3, sci_mode=True)self.check_configs()if self.act_bits <= 8 and self.amp_dtype == torch.float16:@@ -322,7 +328,6 @@ class AutoRound(object):"super_group_size"]- self.has_qlayer_outside_block = self.set_layerwise_config(self.layer_config) ##better place in the endself.shared_cache_keys = get_shared_keys(self.model)def set_device_map_in_blocks(self, device_map):@@ -445,6 +450,25 @@ class AutoRound(object):def _check_compatibility(self):# Check the legitimacy of seqlen++ ##check gguf and others+ has_gguf = False+ if hasattr(self, "formats"):+ has_besides_gguf = False+ for format_ in self.formats:+ if "gguf" in format_:+ has_gguf = True+ elif format_ != "fake":+ has_besides_gguf = True+ if has_gguf and has_besides_gguf:+ raise ValueError("gguf format is not compatible with other formats, please choose only one of them")+ if has_gguf and self.iters!=0:+ logger.warning(+ "We recommend setting `iters=0` when exporting to GGUF format,"+ " as we have optimized the RTN method for this case."+ " We will release new algorithms for certain configurations in the future."+ )+if self.seqlen is not None and hasattr(self.model, "config") and \hasattr(self.model.config, "max_position_embeddings"):if self.model.config.max_position_embeddings < self.seqlen:@@ -486,16 +510,22 @@ class AutoRound(object):"""# Validate and process the specified formats_gguf_args_check(self, format)- if "gguf" in format:- self.layer_config, gguf_format_config = get_layer_config_by_gguf_format(self.layer_config, format,- self.model)- self.has_qlayer_outside_block = self.set_layerwise_config(self.layer_config)+formats = format.replace("q*_", f"q{self.bits}_").replace(' ', '').split(',')from auto_round.utils import SUPPORTED_FORMATSfor format_ in formats:if format_ not in SUPPORTED_FORMATS:logger.error(f"Unsupported format {format_}, please choose from {SUPPORTED_FORMATS}")exit(-1)+ if self.scale_dtype != torch.float32:+ only_gguf = True+ for format_ in formats:+ if not ("gguf" in format_ or "fake" in format_):+ only_gguf = False+ break+ if only_gguf:+ self.scale_dtype = torch.float32+ logger.info(f"change `scale_dtype` to `torch.float32`")# only support to export afp8if self.act_bits <= 8:@@ -518,15 +548,9 @@ class AutoRound(object):# If multiple formats are specified, enforce inplace=Falseif len(formats) > 1:inplace = False- inplace = kwargs.get("inplace", inplace)+ self.inplace = kwargs.get("inplace", inplace)kwargs.pop("inplace", None)- # Determine if immediate packing is required- if (len(formats) == 1 and- ("awq" in formats[0] or "gptq" in formats[0] or "auto_round" in formats[0]) and- not self.has_qlayer_outside_block and inplace): # TODO: Support more formats- self.is_packing_immediate = True-# Adjust format settings based on compatibilityfor index in range(len(formats)):format = formats[index]@@ -574,25 +598,228 @@ class AutoRound(object):return model, folders+ @torch.inference_mode+ def quantize_embedding_layer(self):+ for n, m in self.model.named_modules():+ if not isinstance(m, torch.nn.Embedding) or n not in self.layer_config:+ continue++ config = self.layer_config[n]+ if not check_to_quantized(config):+ continue++ layer = m+ layer_name = n+ config["scale_dtype"] = self.scale_dtype++ from auto_round.data_type import QUANT_FUNC_WITH_DTYPE+ dtype = config["data_type"]+ if dtype not in QUANT_FUNC_WITH_DTYPE:+ if config['sym']:+ dtype = dtype + "_sym"+ else:+ dtype = dtype + "_asym"++ if not self.disable_opt_rtn and "rtn_" + dtype in QUANT_FUNC_WITH_DTYPE:+ dtype = "rtn_" + dtype+ quant_func = QUANT_FUNC_WITH_DTYPE[dtype]++ try:+ weight, scale, zp = quant_func(layer.weight.to(self.device), **{k: config[k] for k in+ ["bits", "group_size", "super_bits",+ "super_group_size", "scale_dtype"]})+ except RuntimeError as e:+ if "CUDA out of memory" in str(e) or "MODULE:PT_DEVMEM" in str(e):+ logger.info("out of vram, fallback to cpu")+ weight, scale, zp = quant_func(layer.weight.to("cpu"), **{k: config[k] for k in+ ["bits", "group_size", "super_bits",+ "super_group_size", "scale_dtype"]})+ else:+ raise+ layer.weight.data.copy_(weight.cpu())++ for name, val in zip(["scale", "zp"], [scale, zp]):+ if isinstance(val, dict):+ for k, v in val.items():+ setattr(layer, k if k == "scale" else "w_" + k, v.cpu())+ else:+ setattr(layer, name, val.cpu())++ self.layer_config.setdefault(layer_name, {}).update(config)+ clear_memory()+ return True++ def get_imatrix(self):+ ##TODO switch to blockwise way when oom+ logger.info("start to get imatrix for gguf quantization")+ from .calib_dataset import get_dataloader+ if isinstance(self.dataset, str):+ dataset = self.dataset.replace(" ", "") ##remove all whitespaces++ self.dataloader = get_dataloader(+ self.tokenizer,+ self.seqlen,+ dataset,+ self.seed,+ self.batch_size,+ self.nsamples,+ )+ else:+ self.dataloader = self.dataset+ model = self.model+ if hasattr(self.model, "hf_device_map") and len(self.model.hf_device_map) > 1:+ from accelerate.big_modeling import dispatch_model+ dispatch_model(model, model.hf_device_map)+ else:+ model = self.model.to(self.device)++ def register_act_hook(model):+ def get_act_max_hook(module, input, output):+ if isinstance(input, (tuple, list)):+ input = input[0]+ if not hasattr(module, "imatrix"):+ module.imatrix = (torch.sum(input.reshape(-1, input.shape[-1]).to(torch.float32) ** 2, dim=0)).to(+ torch.float32)+ else:+ module.imatrix = module.imatrix + (+ torch.sum(input.reshape(-1, input.shape[-1]).to(torch.float32) ** 2, dim=0)).to(torch.float32)++ hook_handles = []++ for n, m in model.named_modules():+ if isinstance(m, self.supported_types) and check_to_quantized(m):+ hook = m.register_forward_hook(get_act_max_hook)+ hook_handles.append(hook)+ return hook_handles++ hooks = register_act_hook(self.model)++ try:+ cnt = 0+ for data in self.dataloader:+ cnt += data["input_ids"].shape[0]+ data = to_device(data, self.model.device)+ model(**data)+ if cnt > self.nsamples:+ break+ except:+ logger.warning("out of vram, fallback to cpu, pleaser use more gpus by setting `--device 0,1,2,3`")+ model = self.model.to("cpu")+ if hasattr(self.model, "hf_device_map") and len(self.model.hf_device_map) > 1:+ accelerate.hooks.remove_hook_from_submodules(+ self.model) ##self.model.hf_device_map has not been changed+ cnt = 0+ for data in self.dataloader:+ cnt += data["input_ids"].shape[0]+ data = to_device(data, self.model.device)+ model(**data)+ if cnt > self.nsamples:+ break++ for hook in hooks:+ hook.remove()+ for n, m in model.named_modules():+ if hasattr(m, "imatrix"):+ m.imatrix /= cnt++ model.to("cpu")+ clear_memory()++ def check_need_to_quantize_lm_head_embedding(self):+ if not hasattr(self, "formats"):+ return False+ has_gguf = False+ for format_ in self.formats:+ if "gguf" in format_:+ has_gguf = True+ if not has_gguf:+ return False++ formats = [f for f in self.formats if "fake" not in f]+ if not (len(formats) == 1 and "gguf" in formats[0]):+ raise NotImplementedError("Only one GGUF format can be set at a time.")+ target_gguf_format = formats[0]+ tie_word_embeddings = True+ if hasattr(self.model, "config") and hasattr(self.model.config, "tie_word_embeddings"):+ tie_word_embeddings = self.model.config.tie_word_embeddings+ for n, m in self.model.named_modules():+ if isinstance(m, torch.nn.Embedding):+ embedding_name = n+ key = "lm_head" if tie_word_embeddings else "embedding"+ config = GGUF_INNER_CONFIG[GGUF_CONFIG[target_gguf_format][key]]+ act_bits = 16+ scale_dtype = self.scale_dtype+ keys = ["bits", "group_size", "super_bits", "super_group_size", "data_type", "sym"]+ self.layer_config[embedding_name] = {}+ for key in keys:+ self.layer_config[embedding_name][key] = getattr(config, "get")(key)+ setattr(get_module(self.model, embedding_name), key, config.get(key))+ self.layer_config[embedding_name]["act_bits"] = act_bits+ self.layer_config[embedding_name]["scale_dtype"] = scale_dtype+ setattr(get_module(self.model, embedding_name), "act_bits", act_bits)+ setattr(get_module(self.model, embedding_name), "scale_dtype", scale_dtype)++ if not tie_word_embeddings:+ # TODO: If the user has set this explicitly, do not override++ lm_head_name = get_lm_head_name(self.model)+ config = GGUF_CONFIG[GGUF_CONFIG[formats[0]]["lm_head"]]++ act_bits = 16+ scale_dtype = self.scale_dtype+ keys = ["bits", "group_size", "super_bits", "super_group_size", "data_type", "sym"]+ for key in keys:+ self.layer_config[lm_head_name][key] = getattr(config, "get")(key)+ setattr(get_module(self.model, lm_head_name), key, config.get(key))+ self.layer_config[lm_head_name]["act_bits"] = act_bits+ self.layer_config[lm_head_name]["scale_dtype"] = scale_dtype++ setattr(get_module(self.model, lm_head_name), "act_bits", act_bits)+ setattr(get_module(self.model, lm_head_name), "scale_dtype", scale_dtype)+ return True+ return False+@torch.inference_modedef quantize_rtn(self):if self.amp:self.model.to(self.amp_dtype)self.model.to("cpu")all_to_quantized_module_names = []+for n, m in self.model.named_modules():if check_to_quantized(m):all_to_quantized_module_names.append(n)+ if hasattr(self, "formats"):+ has_gguf_k = False+ for format_ in self.formats:+ if "gguf" in format_ and "k" in format_:+ has_gguf_k = True+ if has_gguf_k:+ self.get_imatrix()+ self.quantize_embedding_layer()pbar = tqdm(all_to_quantized_module_names)for name in pbar:pbar.set_description(f"Quantizing {name}")m = get_module(self.model, name)-- m.to(self.device)- m = WrapperLinear(m, enable_minmax_tuning=False, enable_norm_bias_tuning=False, enable_round_tuning=False)- m = m.unwrapper({})- m.to("cpu")+ if not self.disable_opt_rtn and not (m.data_type.startswith("rtn_")): ## use rtn version first+ from auto_round.data_type import QUANT_FUNC_WITH_DTYPE+ if "rtn_" + m.data_type in QUANT_FUNC_WITH_DTYPE:+ m.data_type = "rtn_" + m.data_type+ self.layer_config[name]["data_type"] = m.data_type+ try:+ m.to(self.device)+ m = WrapperLinear(m, enable_minmax_tuning=False, enable_norm_bias_tuning=False,+ enable_round_tuning=False)+ m = m.unwrapper({})+ m.to("cpu")+ except RuntimeError as e:+ if "CUDA out of memory" in str(e) or "MODULE:PT_DEVMEM" in str(e):+ logger.warning("out of vram, fallback to cpu")+ m.to("cpu")+ m = WrapperLinear(m, enable_minmax_tuning=False, enable_norm_bias_tuning=False,+ enable_round_tuning=False)+ m = m.unwrapper({})if self.low_gpu_mem_usage:clear_memory()if self.is_packing_immediate:@@ -618,6 +845,26 @@ class AutoRound(object):## TODO add common checkself._check_compatibility()+ self.has_qlayer_outside_block = self.set_layerwise_config(self.layer_config)+ if not hasattr(self, "formats"):+ logger.warning("this API is deprecated, please use `quantize_and_save` instead")+ else:+ only_gguf = True+ for format_ in self.formats:+ if not ("gguf" in format_ or "fake" in format_):+ only_gguf = False+ break+ if len(self.formats) == 1 and self.formats[0] == "fake":+ only_gguf = False+ if only_gguf:+ self.layer_config, gguf_format_config = get_layer_config_by_gguf_format(self.layer_config, self.formats,+ self.model)+ # Determine if immediate packing is required+ formats = self.formats+ if (len(formats) == 1 and+ ("awq" in formats[0] or "gptq" in formats[0] or "auto_round" in formats[0]) and+ not self.has_qlayer_outside_block and self.inplace): # TODO: Support more formats+ self.is_packing_immediate = Trueif self.iters == 0:return self.quantize_rtn()@@ -636,8 +883,19 @@ class AutoRound(object):layer_names = self.get_quantized_layer_names_outside_blocks()self.start_time = time.time()all_first_block_names = [block[0] for block in all_blocks]- logger.info("start to cache block inputs")+ if len(layer_names) > 0:+ logger.info("Starting to cache block inputs. This may be slow due to external block layers: %s",+ layer_names)+ else:+ logger.info("start to cache block inputs")all_inputs = self.try_cache_inter_data_gpucpu(all_first_block_names, self.nsamples, layer_names=layer_names)+ is_quantized_embedding = self.quantize_embedding_layer()+ all_q_inputs = None+ if is_quantized_embedding:+ all_inputs = copy.deepcopy(self.inputs)+ clear_memory(self.inputs)+ all_q_inputs = self.try_cache_inter_data_gpucpu(all_first_block_names, self.nsamples,+ layer_names=layer_names)if hasattr(self.model, "hf_device_map") and len(self.model.hf_device_map) > 1:accelerate.hooks.remove_hook_from_submodules(self.model) ##self.model.hf_device_map has not been changedself.model = mv_module_from_gpu(self.model, self.low_cpu_mem_usage)@@ -647,12 +905,19 @@ class AutoRound(object):for block_names in all_blocks:inputs = all_inputs[block_names[0]]all_inputs.pop(block_names[0])+ q_inputs = None+ if all_q_inputs is not None:+ q_inputs = all_q_inputs[block_names[0]]+ all_q_inputs.pop(block_names[0])keys = inputs.keys()input_id_str = [key for key in keys if key.startswith('hidden_state')]if len(input_id_str) != 1:raise RuntimeError(f"hidden_states arg mismatch error,""please raise an issue in https://github.com/intel/auto-round/issues")inputs["input_ids"] = inputs.pop(input_id_str[0], None)+ if q_inputs is not None:+ q_inputs["input_ids"] = q_inputs.pop(input_id_str[0], None)+clear_memory(self.inputs)if "input_ids" in inputs.keys():@@ -665,6 +930,7 @@ class AutoRound(object):self.model,inputs,block_names,+ q_input=q_inputs["input_ids"] if q_inputs is not None else None,nblocks=self.nblocks,device=self.device,pbar=pbar@@ -713,6 +979,24 @@ class AutoRound(object):None"""##TODO currently we take all the layers outside blocks as post block layers which is not optimal+ ## if there is no input for layer, we use rtn+ for layer_name in copy.deepcopy(layer_names):+ if layer_name not in layer_inputs:+ logger.info(f"using rtn to quantize {layer_name}")+ from auto_round.data_type import QUANT_FUNC_WITH_DTYPE++ layer = get_module(self.model, layer_name)+ if "rtn_" + layer.data_type in QUANT_FUNC_WITH_DTYPE:+ layer.data_type = "rtn_" + layer.data_type+ logger.info("using optimized rtn method for quantizing %s", layer_name)+ self.layer_config[layer_name]["data_type"] = layer.data_type+ layer.to(self.device)+ wrapper_layer = WrapperLinear(layer, enable_round_tuning=False, enable_minmax_tuning=False,+ enable_norm_bias_tuning=False, device=self.device)+ new_layer = wrapper_layer.unwrapper({})+ set_module(self.model, layer_name, new_layer)+ layer.cpu()+ layer_names.remove(layer_name)if len(layer_names) == 0:returnq_layer_inputs = None@@ -723,6 +1007,9 @@ class AutoRound(object):dispatch_model(self.model, self.model.hf_device_map)if enable_quanted_input:+ logger.info(+ "starting to cache layer inputs for %s as `enable_quanted_input` is enable, this may be quite slow ",+ layer_names)q_layer_inputs = self.try_cache_inter_data_gpucpu([], self.nsamples, layer_names=layer_names)if hasattr(self.model, "hf_device_map") and len(self.model.hf_device_map) > 1:accelerate.hooks.remove_hook_from_submodules(@@ -761,7 +1048,20 @@ class AutoRound(object):##process regex in layer_configall_supported_layer_names = []# List of configuration keys- keys = self.serialization_keys+ keys = [+ "bits",+ "group_size",+ "sym",+ "data_type",+ "scale_dtype",+ "act_bits",+ "act_group_size",+ "act_sym",+ "act_dynamic",+ "act_data_type",+ "super_bits",+ "super_group_size"+ ]for n, m in self.model.named_modules():# Delete previous configuration to avoid conflicts with prior tuning@@ -834,6 +1134,9 @@ class AutoRound(object):# Apply the configuration to the corresponding layer in the modelfor key in keys:setattr(m, key, layer_config[n][key])+ need_to_quantize_lm_head = self.check_need_to_quantize_lm_head_embedding()+ if need_to_quantize_lm_head:+ has_qlayer_outside_block = True# Return whether there are quantized layers outside the blocksreturn has_qlayer_outside_block@@ -1013,18 +1316,17 @@ class AutoRound(object):clear_memory()except RuntimeError as e:if "CUDA out of memory" in str(e) or "MODULE:PT_DEVMEM" in str(e):+ self.no_enough_vram_for_w_model = Truelogger.info("switch to cpu to cache block inputs")- if (("lm_head" in self.layer_config and self.layer_config["lm_head"]["bits"] < 16) or+ if (self.has_qlayer_outside_block orself.__class__.__name__ == "AutoRoundMLLM"):- logger.warning(f"we strongly recommend using additional CUDA/HPU devices,e.g. "- f"set `--device '0,1'` in our cmd line usage or "- f"load the model with `device_mapping=auto`,"- f" for optimal performance during calibration "- f"Otherwise, the process may be significantly slower.")+ logger.warning("We strongly recommend using more GPUs."+ " Otherwise, some layers may fall back to `rtn` mode, which can affect accuracy.")self.model = mv_module_from_gpu(self.model, self.low_cpu_mem_usage)clear_memory()+ ## Important change after v0.51, on cpu, we use rtn mode for layers in layer_namesall_inputs = self.cache_inter_data(- block_names, nsamples, layer_names=layer_names, last_cache_name=last_cache_name+ block_names, nsamples, layer_names=[], last_cache_name=last_cache_name)else:raise@@ -1569,6 +1871,7 @@ class AutoRound(object):model: torch.nn.Module,inputs,block_names,+ q_input=None,nblocks=1,device="cpu",pbar=None@@ -1585,7 +1888,6 @@ class AutoRound(object):Returns:None"""- q_input = Noneclear_memory()for n, m in model.named_parameters():m.requires_grad_(False)@@ -1699,7 +2001,7 @@ class AutoRound(object):if re.search(r"q\d_k", format) and not self.data_type.endswith("_dq"):logger.error(f"datatype<{self.data_type}> not support to export {format} format."- " Please change export format or data_type."+ " Please change export format or `data_type`.")sys.exit(-1)@@ -2012,6 +2314,7 @@ class AutoRoundOPT(AutoRound):optimizer="AdamW",super_bits: int = None,super_group_size: int = None,+ disable_opt_rtn: bool = False,**kwargs,):super(AutoRoundOPT, self).__init__(@@ -2190,6 +2493,7 @@ class AutoRoundAdam(AutoRoundOPT):optimizer="AdamW",super_bits: int = None,super_group_size: int = None,+ disable_opt_rtn: bool = False,**kwargs,):super(AutoRoundAdam, self).__init__(diff --git a/auto_round/data_type/__init__.py b/auto_round/data_type/__init__.pyindex 44c2e9368..5502fc659 100644--- a/auto_round/data_type/__init__.py+++ b/auto_round/data_type/__init__.py@@ -19,4 +19,5 @@ from auto_round.data_type.register import QUANT_FUNC_WITH_DTYPEimport auto_round.data_type.w4fp8from auto_round.data_type.utils import get_quant_funcimport auto_round.data_type.nvfp+import auto_round.data_type.ggufdiff --git a/auto_round/data_type/gguf.py b/auto_round/data_type/gguf.pynew file mode 100644index 000000000..b999c81ed--- /dev/null+++ b/auto_round/data_type/gguf.py@@ -0,0 +1,518 @@+# Copyright (c) 2025 Intel Corporation+#+# Licensed under the Apache License, Version 2.0 (the "License");+# you may not use this file except in compliance with the License.+# You may obtain a copy of the License at+#+# http://www.apache.org/licenses/LICENSE-2.0+#+# Unless required by applicable law or agreed to in writing, software+# distributed under the License is distributed on an "AS IS" BASIS,+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+# See the License for the specific language governing permissions and+# limitations under the License.++import torch+from auto_round.data_type.utils import round_ste, reshape_pad_tensor_by_group_size, revert_tensor_by_pad+from auto_round.data_type.register import register_dtype+++@register_dtype("int_sym_dq")+def quant_tensor_sym_dq(+ tensor,+ bits=4,+ group_size=-1,+ v=0,+ min_scale=1.0,+ max_scale=1.0,+ scale_dtype=torch.float16,+ tensor_min=None,+ tensor_max=None,+ q_scale_thresh=1e-5,+ super_group_size=16,+ super_bits=6,+ **kwargs):+ """Quantize and de-quantize tensor symmetrically. full range, credict goes to llamacpp community++ Args:+ tensor: Tensor containing the tensor to be quantized+ bits: Number of bits for quantization (e.g., 2, 3, 4, 8)+ group_size: Number of elements to share scale for quantization+ v: Rounding value perturbation+ min_scale: Minimum scale coefficient for tensor+ max_scale: Maximum scale coefficient for tensor+ tensor_min (Tensor, optional): Minimum tensor value for quantization. Defaults to None.+ tensor_max (Tensor, optional): Maximum tensor value for quantization. Defaults to None.+ scale_dtype: dtype of the quantized scale,as most kernels only support FP16 or FP32, while this value is import+ q_scale_thresh: clip the quantized scale's magnitude to this value to improve the numerical stability++ Returns:+ Quantized and de-quantized tensor, scale, zero-point+ """+ tensor, orig_shape, pad_len = reshape_pad_tensor_by_group_size(tensor, group_size)++ maxq = 2 ** (bits - 1)+ if tensor_min is None or tensor_max is None:+ wmin_tmp = torch.clamp(tensor.min(-1)[0], max=0)+ wmax_tmp = torch.clamp(tensor.max(-1)[0], min=0)+ else:+ wmin_tmp = tensor_min+ wmax_tmp = tensor_max++ wmin_abs = -(wmin_tmp * min_scale) # pylint: disable=E1130+ wmax_abs = wmax_tmp * max_scale+ max_v = (2 * (wmax_abs < wmin_abs).int() - 1) * torch.max(wmax_abs, wmin_abs)+ scale = (max_v / maxq).to(scale_dtype)+ scale = scale.view(-1, super_group_size)++ # conduct double quant+ scale, d_scale = double_quant_tensor_sym(scale, super_bits)++ scale = scale.view(-1, 1)+ zp = torch.full_like(scale, maxq) # pylint: disable=E1130+ int_w = torch.where(scale != 0, round_ste(tensor / scale + v), 0)+ q = torch.clamp(int_w + zp, 0, 2 ** bits - 1)+ qdq_result = (scale * (q - zp)).to(tensor.dtype)+ qdq_result = revert_tensor_by_pad(qdq_result, orig_shape=orig_shape, pad_len=pad_len)+ return qdq_result, {"scale": scale, "d_scale": d_scale}, zp+++@register_dtype("int_asym_float_zp")+def quant_tensor_asym_float_zp(+ tensor,+ bits=4,+ group_size=-1,+ v=0,+ min_scale=1.0,+ max_scale=1.0,+ scale_dtype=torch.float16,+ tensor_min=None,+ tensor_max=None,+ q_scale_thresh=1e-5,+ **kwargs):+ """Quantize and de-quantize tensor asymmetrically.++ Args:+ tensor: Tensor containing the tensor to be quantized+ bits: Number of bits for quantization (e.g., 2, 3, 4, 8)+ group_size: Number of elements to share scale for quantization+ v: Rounding value perturbation+ min_scale: Minimum scale coefficient for tensor+ max_scale: Maximum scale coefficient for tensor+ tensor_min (Tensor, optional): Minimum tensor value for quantization. Defaults to None.+ tensor_max (Tensor, optional): Maximum tensor value for quantization. Defaults to None.+ scale_dtype: dtype of the quantized scale,as most kernels only support FP16 or FP32, while this value is import+ q_scale_thresh: clip the quantized scale's magnitude to this value to improve the numerical stability++ Returns:+ Quantized and de-quantized tensor, scale, zero-point+ """+ tensor, orig_shape, pad_len = reshape_pad_tensor_by_group_size(tensor, group_size)+ maxq = 2 ** bits - 1+ if tensor_min is None or tensor_max is None:+ wmin_tmp = torch.clamp(tensor.min(-1)[0], max=0)+ wmax_tmp = torch.clamp(tensor.max(-1)[0], min=0)+ else:+ wmin_tmp = tensor_min+ wmax_tmp = tensor_max+ if isinstance(min_scale, torch.Tensor):+ wmin = wmin_tmp * min_scale+ wmax = wmax_tmp * max_scale+ else:+ wmin = wmin_tmp+ wmax = wmax_tmp+ scale = ((wmax - wmin) / maxq).to(scale_dtype)+ scale = torch.clamp(scale, min=q_scale_thresh)+ zp = -wmin / scale # pylint: disable=E1130+ scale = scale.unsqueeze(dim=-1)+ zp = zp.unsqueeze(dim=-1)+ int_w = round_ste(tensor / scale + v)+ q = torch.clamp(int_w + zp, 0, maxq)+ qdq_result = (scale * (q - zp)).to(tensor.dtype)+ qdq_result = revert_tensor_by_pad(qdq_result, orig_shape=orig_shape, pad_len=pad_len)+ return qdq_result, scale, zp+++## the values should be positive+def double_quant_tensor(tensor, bits):+ tensor = tensor.to(torch.float32) # Ensure tensor is in float32 for precision+ maxq = 2 ** bits - 1+ wmax = torch.clamp(tensor.max(-1)[0], min=0)+ scale = wmax / maxq+ scale = scale.view(-1, 1)+ inverse_scale = torch.where(scale == 0, 0, 1 / scale)+ qdq_tensor = torch.clamp(round_ste(tensor * inverse_scale), max=maxq) * scale+ return qdq_tensor, scale+++def double_quant_tensor_sym(tensor, bits):+ tensor = tensor.to(torch.float32) # Ensure tensor is in float32 for precision+ maxq = 2 ** (bits - 1)+ imax = abs(tensor).argmax(axis=-1, keepdims=True)+ wmax = torch.take_along_dim(tensor, imax, dim=-1)+ scale = wmax / -maxq+ inverse_scale = torch.where(scale == 0, 0, 1 / scale)+ qdq_tensor = (tensor * inverse_scale).clip(-maxq, maxq - 1) * scale+ return qdq_tensor, scale+++def make_qp_quants(nmax, data, quant_weights):+ data = data.to(torch.float32)+ quant_weights = quant_weights.to(torch.float32)+ group_max = torch.max(data, dim=-1, keepdim=True)[0]+ scale = group_max / nmax+ iscale = torch.where(scale == 0, 0, 1 / scale)++ L = torch.round(iscale * data)+ diffs = data - scale * L+ best_mse = torch.sum(quant_weights * diffs * diffs)++ for _is in range(-4, 5):+ if _is == 0:+ continue+ scale_is = group_max / (0.1 * _is + nmax)+ iscale_is = torch.where(scale == 0, 0, 1 / scale_is)++ tmp_L = torch.round(iscale_is * data).clip(max=nmax)+ diffs = data - scale_is * tmp_L+ mse = torch.sum(quant_weights * diffs * diffs)++ replace_idx = mse < best_mse+ best_mse[replace_idx] = mse[replace_idx]+ iscale[replace_idx] = iscale_is[replace_idx]++ L = torch.round(iscale * data).clip(max=nmax)+ sumlx = torch.sum(quant_weights * data * L, dim=-1)+ suml2 = torch.sum(quant_weights * L * L, dim=-1)++ for _ in range(5):+ n_changed = 0+ for i in range(data.shape[-1]):+ slx = sumlx - quant_weights[:, i] * data[:, i] * L[:, i]+ sl2 = suml2 - quant_weights[:, i] * L[:, i] * L[:, i]+ replace_idx = (slx > 0) & (sl2 > 0)+ new_L = torch.round(data[:, i] * sl2 / slx).clip(max=nmax)+ replace_idx &= new_L != L[:, i]+ slx[replace_idx] += quant_weights[:, i][replace_idx] * data[:, i][replace_idx] * new_L[replace_idx]+ sl2[replace_idx] += quant_weights[:, i][replace_idx] * new_L[replace_idx] * new_L[replace_idx]++ replace_idx &= slx * slx * suml2 > sumlx * sumlx * sl2+ L[:, i][replace_idx] = new_L[replace_idx]+ sumlx[replace_idx] = slx[replace_idx]+ suml2[replace_idx] = sl2[replace_idx]+ n_changed = replace_idx.sum()+ if n_changed == 0:+ break++ return sumlx / suml2, L+++@register_dtype("int_asym_dq")+def quant_tensor_asym_dq(tensor, bits=4, group_size=-1, v=0, min_scale=1.0, max_scale=1.0, scale_dtype=torch.float16,+ tensor_min=None, tensor_max=None, q_scale_thresh=1e-5, super_group_size=8, super_bits=6,+ **kwargs):+ """Quantize and de-quantize tensor asymmetrically.++ Args:+ tensor: Tensor containing the tensor to be quantized+ bits: Number of bits for quantization (e.g., 2, 3, 4, 8)+ group_size: Number of elements to share scale for quantization+ v: Rounding value perturbation+ min_scale: Minimum scale coefficient for tensor+ max_scale: Maximum scale coefficient for tensor+ tensor_min (Tensor, optional): Minimum tensor value for quantization. Defaults to None.+ tensor_max (Tensor, optional): Maximum tensor value for quantization. Defaults to None.+ scale_dtype: dtype of the quantized scale,as most kernels only support FP16 or FP32, while this value is import+ q_scale_thresh: clip the quantized scale's magnitude to this value to improve the numerical stability++ Returns:+ Quantized and de-quantized tensor, scale, zero-point+ """+ tensor, orig_shape, pad_len = reshape_pad_tensor_by_group_size(tensor, group_size)+ maxq = 2 ** bits - 1+ if tensor_min is None or tensor_max is None:+ wmin_tmp = torch.clamp(tensor.min(-1)[0], max=0)+ wmax_tmp = torch.clamp(tensor.max(-1)[0], min=0)+ else:+ wmin_tmp = tensor_min+ wmax_tmp = tensor_max+ if isinstance(min_scale, torch.Tensor):+ wmin = wmin_tmp * min_scale+ wmax = wmax_tmp * max_scale+ else:+ wmin = wmin_tmp+ wmax = wmax_tmp+ scale = ((wmax - wmin) / maxq).to(scale_dtype)+ scale = torch.clamp(scale, min=q_scale_thresh)+ scale = scale.view(-1, super_group_size)+ wmin_m = -wmin # pylint: disable=E1130+ wmin_m = wmin_m.view(-1, super_group_size)++ ##conduct double quant+ scale, d_scale = double_quant_tensor(scale, super_bits)+ wmin_m, d_wmin_m = double_quant_tensor(wmin_m, super_bits)++ scale = scale.view(-1, 1)+ scale = torch.clamp(scale, q_scale_thresh)+ d_scale = torch.clamp(d_scale, q_scale_thresh)+ d_wmin_m = torch.clamp(d_wmin_m, q_scale_thresh)+ wmin_m = wmin_m.view(-1, 1)++ int_w = round_ste((tensor + wmin_m) / scale + v)+ q = torch.clamp(int_w, 0, maxq)+ qdq_result = (scale * q - wmin_m).to(tensor.dtype)+ qdq_result = revert_tensor_by_pad(qdq_result, orig_shape=orig_shape, pad_len=pad_len)++ # zp = round_ste(wmin_m / scale) # remove this later+ return qdq_result, {"scale": scale, "d_scale": d_scale}, {"wmin_m": wmin_m, "d_wmin_m": d_wmin_m}+++@register_dtype("rtn_int_asym_dq")+def quant_tensor_gguf_asym_dq(+ tensor,+ bits=4,+ group_size=-1,+ v=0,+ min_scale=1.0,+ max_scale=1.0,+ scale_dtype=torch.float16,+ tensor_min=None,+ tensor_max=None,+ q_scale_thresh=1e-5,+ super_group_size=8,+ super_bits=6,+ imatrix=None,+ **kwargs):+ """Quantizes and dequantizes a tensor using asymmetric integer quantization for formats like Q2_K, Q4_K, and Q5_K.+ Only fit for iters 0++ Args:+ tensor (torch.Tensor): Input tensor to quantize.+ bits (int): Number of bits for quantization.+ group_size (int): Group size for per-group quantization.+ v (float): Perturbation added before rounding.+ min_scale (float): Minimum allowed scale value.+ max_scale (float): Maximum allowed scale value.+ scale_dtype (torch.dtype): Data type for quantized scale.+ tensor_min (torch.Tensor, optional): Minimum values for the tensor groups.+ tensor_max (torch.Tensor, optional): Maximum values for the tensor groups.+ q_scale_thresh (float): Threshold to clamp the quantized scale.+ super_group_size (int): Number of groups to bundle for secondary quantization.+ super_bits (int): Number of bits used in secondary quantization.+ imatrix (torch.Tensor, optional): Importance matrix for weighted quantization.++ Returns:+ Tuple: (Quantized-dequantized tensor, scale dictionary, zero-point dictionary)+ """+ orig_dtype = tensor.dtype+ tensor, orig_shape, pad_len = reshape_pad_tensor_by_group_size(tensor, group_size)+ if bits not in [2, 4, 5]:+ raise ValueError(f"bits={bits} not supported by gguf_int_asym_dq")+ maxq = 2 ** bits - 1+ QK_K = 256+ quant_weights = None+ if imatrix is None or (imatrix is not None and torch.sum(imatrix)==0):+ search_kwargs = {+ 2: {"rmin": -0.5, "rdelta": 0.1, "nstep": 15, "use_mad": True},+ 4: {"rmin": -1, "rdelta": 0.1, "nstep": 20, "use_mad": False},+ 5: {"rmin": -0.5, "rdelta": 0.1, "nstep": 15, "use_mad": False}+ }+ if bits == 2:+ quant_weights = torch.abs(tensor)+ elif bits == 4 or bits == 5:+ sigma2 = torch.sum(tensor ** 2, dim=-1, keepdim=True) / 32 ##Note 32 is different from QK_K+ av_x = torch.sqrt(sigma2)+ quant_weights = torch.sqrt(av_x + tensor * tensor)+ params = search_kwargs[bits]+ scale, wmin_m = iterative_wls_quant_search(+ tensor, bits=bits, rrmin=params["rmin"], rdelta=params["rdelta"], nstep=params["nstep"],+ use_mad=params["use_mad"], weights=quant_weights+ )+ scale= scale.reshape(-1, super_group_size)+ wmin = wmin_m.reshape(-1, super_group_size)+ scale, d_scale = double_quant_tensor(scale, super_bits)+ wmin_m, d_wmin_m = double_quant_tensor(wmin, super_bits)+ wmin_m = wmin_m.view(-1, 1)+ scale = scale.view(-1, 1)+ else:+ imatrix = imatrix.to(tensor.device)+ search_kwargs = {+ 2: {"rmin": -0.9, "rdelta": 0.05, "nstep": 36, "use_mad": False},+ 4: {"rmin": -0.9, "rdelta": 0.05, "nstep": 36, "use_mad": False},+ 5: {"rmin": -0.9, "rdelta": 0.05, "nstep": 36, "use_mad": False},+ }+ if bits == 2:+ sigma2 = torch.sum(tensor ** 2, dim=-1, keepdim=True) / QK_K+ imatrix = imatrix.reshape(1, -1).expand(tensor.numel() // imatrix.numel(), -1).reshape(tensor.shape)+ quant_weights = imatrix * torch.sqrt(sigma2 + tensor * tensor)+ elif bits == 4 or bits == 5:+ sigma2 = 2 * torch.sum(tensor ** 2, dim=-1, keepdim=True) / QK_K+ av_x = torch.sqrt(sigma2)+ imatrix = imatrix.reshape(1, -1).expand(tensor.numel() // imatrix.numel(), -1).reshape(tensor.shape)+ quant_weights = imatrix * torch.sqrt(av_x + tensor * tensor)+ weights = imatrix.reshape(1, -1)+ weights = weights.expand(tensor.numel() // weights.numel(), -1)+ quant_weights = weights.reshape(tensor.shape)++ params = search_kwargs[bits]++ scale, wmin_m_0 = iterative_wls_quant_search(+ tensor, bits=bits, rrmin=params["rmin"], rdelta=params["rdelta"], nstep=params["nstep"],+ use_mad=params["use_mad"], weights=quant_weights+ )++ nmax = 2 ** super_bits - 1+ scale = scale.reshape(-1, super_group_size)+ wmin_m = wmin_m_0.reshape(-1, super_group_size)+ sum_quant_weights = quant_weights.sum(-1, keepdim=True).reshape(-1, super_group_size)++ d_scale, q_scale = make_qp_quants(nmax, scale, sum_quant_weights)+ d_wmin_m, q_wmin_m = make_qp_quants(nmax, wmin_m, sum_quant_weights)+ d_scale = d_scale.unsqueeze(-1)+ d_wmin_m = d_wmin_m.unsqueeze(-1)+ scale = (d_scale * q_scale).view(-1, 1)+ wmin_m = (d_wmin_m * q_wmin_m).view(-1, 1)+ inverse_scale = torch.where(scale == 0, 0, 1 / scale)++ int_w = torch.clamp(round_ste((tensor + wmin_m) * inverse_scale + v), 0, maxq)+ qdq_result = (scale * int_w - wmin_m).to(orig_dtype)+ qdq_result = revert_tensor_by_pad(qdq_result, orig_shape=orig_shape, pad_len=pad_len)+ return qdq_result, {"scale": scale, "d_scale": d_scale}, {"wmin_m": wmin_m, "d_wmin_m": d_wmin_m}+++def iterative_wls_quant_search(data, bits=4, rrmin=-1.0, rdelta=0.1, nstep=20, use_mad=False, weights=None):+ """Performs iterative weighted least squares quantization search.++ Args:+ data (torch.Tensor): Input tensor to quantize.+ bits (int): Number of quantization bits.+ rrmin (float): Initial range scaling factor.+ rdelta (float): Step size for range scaling.+ nstep (int): Number of search steps.+ use_mad (bool): Whether to use mean absolute deviation instead of squared error.+ weights (torch.Tensor): Weight matrix for each element.++ Returns:+ Tuple: (Optimal scale tensor, optimal minimum value tensor)+ """+ dtype = torch.bfloat16+ data = data.to(dtype)+ maxq = 2 ** bits - 1+ minq = 0+ weights = 1.0 if weights is None else weights.to(dtype)++ rmin = torch.min(data, dim=1, keepdim=True)[0]+ rmax = torch.max(data, dim=1, keepdim=True)[0]++ sum_w = torch.sum(weights, dim=1, keepdim=True)+ sum_x = torch.sum(weights * data, dim=1, keepdim=True)++ scale = 1 / ((maxq - minq) / (rmax - rmin + 1e-8))+ quant_data = torch.clamp(torch.round((maxq - minq) / (rmax - rmin + 1e-8) * (data - rmin)), minq, maxq)+ diff = scale * quant_data + rmin - data+ best_mad = torch.sum(weights * (torch.abs(diff) if use_mad else diff ** 2), dim=1, keepdim=True)++ for is_ in range(nstep):+ factor = rrmin + rdelta * is_ + maxq - minq+ iscale_new = factor / (rmax - rmin + 1e-8)+ quant_data_new = torch.clamp(torch.round(iscale_new * (data - rmin)), minq, maxq)++ mul_weights_quant_data = weights * quant_data_new+ sum_l = torch.sum(mul_weights_quant_data, dim=1, keepdim=True)+ sum_l2 = torch.sum(mul_weights_quant_data * quant_data_new, dim=1, keepdim=True)+ sum_xl = torch.sum(mul_weights_quant_data * data, dim=1, keepdim=True)++ D = sum_w * sum_l2 - sum_l ** 2+ this_scale = (sum_w * sum_xl - sum_x * sum_l) / D+ this_min = (sum_l2 * sum_x - sum_l * sum_xl) / D++ quant_data = torch.clamp(torch.round((1 / this_scale) * (data - this_min)), minq, maxq)+ diff = this_scale * quant_data + this_min - data+ mad = torch.sum(weights * (torch.abs(diff) if use_mad else diff ** 2), dim=1, keepdim=True)++ idx_to_replace = torch.where((mad < best_mad) & (D > 0))[0]+ best_mad[idx_to_replace] = mad[idx_to_replace]+ scale[idx_to_replace] = this_scale[idx_to_replace]+ rmin[idx_to_replace] = this_min[idx_to_replace]++ return scale.to(torch.float32), -rmin.to(torch.float32)+++@register_dtype("rtn_int_sym_dq")+def quant_tensor_gguf_sym_dq(+ tensor,+ bits=4,+ group_size=-1,+ v=0,+ min_scale=1.0,+ max_scale=1.0,+ scale_dtype=torch.float16,+ tensor_min=None,+ tensor_max=None,+ q_scale_thresh=1e-5,+ super_group_size=16,+ super_bits=6,+ imatrix=None,+ **kwargs):+ """Quantize and de-quantize tensor asymmetrically. For Q3_K, Q6_K.++ Args:+ tensor: Tensor containing the tensor to be quantized+ bits: Number of bits for quantization (e.g., 2, 3, 4, 8)+ group_size: Number of elements to share scale for quantization+ v: Rounding value perturbation+ min_scale: Minimum scale coefficient for tensor+ max_scale: Maximum scale coefficient for tensor+ tensor_min (Tensor, optional): Minimum tensor value for quantization. Defaults to None.+ tensor_max (Tensor, optional): Maximum tensor value for quantization. Defaults to None.+ scale_dtype: dtype of the quantized scale,as most kernels only support FP16 or FP32, while this value is import+ q_scale_thresh: clip the quantized scale's magnitude to this value to improve the numerical stability++ Returns:+ Quantized and de-quantized tensor, scale, zero-point+ """+ from auto_round.export.export_to_gguf.config import QK_K, K_SCALE_SIZE, GGML_QUANT_SIZES+ from auto_round.export.export_to_gguf.packing import make_q3_quants, make_qx_quants++ if bits not in [3, 6]:+ raise KeyError(f"bits={bits} is not supported by gguf_int_sym_dq, please check.")++ tensor, orig_shape, pad_len = reshape_pad_tensor_by_group_size(tensor, group_size)+ maxq = 2 ** (bits - 1)+ ggml_type = f"q{bits}_k"+ block_size, type_size = GGML_QUANT_SIZES[ggml_type]+ orig_dtype = tensor.dtype++ tensor = tensor.to(torch.float32)+ n_blocks = tensor.nelement() // block_size+ # (nb, 16, 16)+ tensor = tensor.reshape(n_blocks, super_group_size, QK_K // super_group_size)+ if imatrix is None or (imatrix is not None and torch.sum(imatrix)==0):+ if bits == 3:+ scale, int_w = make_q3_quants(tensor, bits=bits, do_rmse=True)+ elif bits == 6:+ scale, int_w = make_qx_quants(tensor, bits=bits, rmse_type=1, qw=None)+ else:+ imatrix = imatrix.to(tensor.device)+ if bits == 3:+ sigma2 = 2 * torch.sum(tensor ** 2, dim=-1, keepdim=True) / QK_K+ imatrix = imatrix.reshape(1, -1).expand(tensor.numel() // imatrix.numel(), -1).reshape(tensor.shape)+ quant_weights = imatrix * torch.sqrt(sigma2 + tensor * tensor)+ scale, int_w = make_qx_quants(tensor, bits=bits, rmse_type=1, qw=quant_weights)+ elif bits == 6:+ weights = imatrix.reshape(1, -1)+ weights = weights.expand(tensor.numel() // weights.numel(), -1)+ quant_weights = weights.reshape(tensor.shape)+ scale, int_w = make_qx_quants(tensor, bits=bits, rmse_type=1, qw=quant_weights)++ # conduct double quant+ scale, d_scale = double_quant_tensor_sym(scale, super_bits)++ replace_ids = scale != 0+ scale = scale.unsqueeze(-1)+ zp = torch.full_like(scale, maxq) # pylint: disable=E1130+ int_w[replace_ids] = (torch.round(+ tensor[replace_ids] / scale[replace_ids]).clip(-maxq, maxq - 1) + maxq).to(torch.uint8)+ qdq_result = (scale * (int_w - zp)).to(orig_dtype)+ qdq_result = revert_tensor_by_pad(qdq_result, orig_shape=orig_shape, pad_len=pad_len)+ return qdq_result, {"scale": scale, "d_scale": d_scale}, zpdiff --git a/auto_round/data_type/int.py b/auto_round/data_type/int.pyindex 32c87f9b3..5367b6925 100644--- a/auto_round/data_type/int.py+++ b/auto_round/data_type/int.py@@ -13,7 +13,7 @@# limitations under the License.import torch-from .utils import round_ste, reshape_pad_tensor_by_group_size, revert_tensor_by_pad+from auto_round.data_type.utils import round_ste, reshape_pad_tensor_by_group_size, revert_tensor_by_padfrom auto_round.data_type.register import register_dtype@@ -63,282 +63,6 @@ def quant_tensor_sym(tensor, bits=4, group_size=-1, v=0, min_scale=1.0, max_scalreturn qdq_result, scale, zp-## the values should be positive-def double_quant_tensor(tensor, bits, q_scale_thresh):- maxq = 2 ** bits - 1- wmax = torch.clamp(tensor.max(-1)[0], min=0)- scale = torch.clamp(wmax / maxq, q_scale_thresh)- scale = scale.view(-1, 1)- qdq_tensor = torch.clamp(round_ste(tensor / scale), max=maxq) * scale- return qdq_tensor, scale--def double_quant_tensor_sym(tensor, bits, q_scale_thresh):- maxq = 2 ** (bits - 1)- imax = abs(tensor).argmax(axis=-1, keepdims=True)- wmax = torch.take_along_dim(tensor, imax, axis=-1)- scale = wmax / -maxq- qdq_tensor = torch.where(scale != 0, round_ste(tensor / scale), 0).clip(-maxq, maxq -1) * scale- return qdq_tensor, scale--@register_dtype("int_sym_dq")-def quant_tensor_sym_dq(- tensor,- bits=4,- group_size=-1,- v=0,- min_scale=1.0,- max_scale=1.0,- scale_dtype=torch.float16,- tensor_min=None,- tensor_max=None,- q_scale_thresh=1e-5,- super_group_size=16,- super_bits=6,- **kwargs):- """Quantize and de-quantize tensor symmetrically. full range, credict goes to llamacpp community-- Args:- tensor: Tensor containing the tensor to be quantized- bits: Number of bits for quantization (e.g., 2, 3, 4, 8)- group_size: Number of elements to share scale for quantization- v: Rounding value perturbation- min_scale: Minimum scale coefficient for tensor- max_scale: Maximum scale coefficient for tensor- tensor_min (Tensor, optional): Minimum tensor value for quantization. Defaults to None.- tensor_max (Tensor, optional): Maximum tensor value for quantization. Defaults to None.- scale_dtype: dtype of the quantized scale,as most kernels only support FP16 or FP32, while this value is import- q_scale_thresh: clip the quantized scale's magnitude to this value to improve the numerical stability-- Returns:- Quantized and de-quantized tensor, scale, zero-point- """- tensor, orig_shape, pad_len = reshape_pad_tensor_by_group_size(tensor, group_size)-- maxq = 2 ** (bits - 1)- if tensor_min is None or tensor_max is None:- wmin_tmp = torch.clamp(tensor.min(-1)[0], max=0)- wmax_tmp = torch.clamp(tensor.max(-1)[0], min=0)- else:- wmin_tmp = tensor_min- wmax_tmp = tensor_max-- wmin_abs = -(wmin_tmp * min_scale) # pylint: disable=E1130- wmax_abs = wmax_tmp * max_scale- max_v = (2 * (wmax_abs < wmin_abs).int() - 1) * torch.max(wmax_abs, wmin_abs)- scale = (max_v / maxq).to(scale_dtype)- scale = scale.view(-1, super_group_size)-- #conduct double quant- scale, d_scale = double_quant_tensor_sym(scale, super_bits, q_scale_thresh)-- scale = scale.view(-1, 1)- zp = torch.full_like(scale, maxq) # pylint: disable=E1130- int_w = torch.where(scale !=0, round_ste(tensor / scale + v), 0)- q = torch.clamp(int_w + zp, 0, 2 ** bits - 1)- qdq_result = (scale * (q - zp)).to(tensor.dtype)- qdq_result = revert_tensor_by_pad(qdq_result, orig_shape=orig_shape, pad_len=pad_len)- return qdq_result, {"scale": scale, "d_scale": d_scale}, zp--@register_dtype("int_asym_dq")-def quant_tensor_asym_dq(tensor, bits=4, group_size=-1, v=0, min_scale=1.0, max_scale=1.0, scale_dtype=torch.float16,- tensor_min=None, tensor_max=None, q_scale_thresh=1e-5, super_group_size=8, super_bits=6,- **kwargs):- """Quantize and de-quantize tensor asymmetrically.-- Args:- tensor: Tensor containing the tensor to be quantized- bits: Number of bits for quantization (e.g., 2, 3, 4, 8)- group_size: Number of elements to share scale for quantization- v: Rounding value perturbation- min_scale: Minimum scale coefficient for tensor- max_scale: Maximum scale coefficient for tensor- tensor_min (Tensor, optional): Minimum tensor value for quantization. Defaults to None.- tensor_max (Tensor, optional): Maximum tensor value for quantization. Defaults to None.- scale_dtype: dtype of the quantized scale,as most kernels only support FP16 or FP32, while this value is import- q_scale_thresh: clip the quantized scale's magnitude to this value to improve the numerical stability-- Returns:- Quantized and de-quantized tensor, scale, zero-point- """- tensor, orig_shape, pad_len = reshape_pad_tensor_by_group_size(tensor, group_size)-- maxq = 2 ** bits - 1- if tensor_min is None or tensor_max is None:- wmin_tmp = torch.clamp(tensor.min(-1)[0], max=0)- wmax_tmp = torch.clamp(tensor.max(-1)[0], min=0)- else:- wmin_tmp = tensor_min- wmax_tmp = tensor_max- if isinstance(min_scale, torch.Tensor):- wmin = wmin_tmp * min_scale- wmax = wmax_tmp * max_scale- else:- wmin = wmin_tmp- wmax = wmax_tmp- scale = ((wmax - wmin) / maxq).to(scale_dtype)- scale = torch.clamp(scale, min=q_scale_thresh)- scale = scale.view(-1, super_group_size)- wmin_m = -wmin # pylint: disable=E1130- wmin_m = wmin_m.view(-1, super_group_size)-- ##conduct double quant- scale, d_scale = double_quant_tensor(scale, super_bits, q_scale_thresh)- wmin_m, d_wmin_m = double_quant_tensor(wmin_m, super_bits, q_scale_thresh)-- scale = scale.view(-1, 1)- scale = torch.clamp(scale, q_scale_thresh)- wmin_m = wmin_m.view(-1, 1)-- int_w = round_ste((tensor + wmin_m) / scale + v)- q = torch.clamp(int_w, 0, maxq)- qdq_result = (scale * q - wmin_m).to(tensor.dtype)- qdq_result = revert_tensor_by_pad(qdq_result, orig_shape=orig_shape, pad_len=pad_len)- # zp = round_ste(wmin_m / scale) # remove this later- return qdq_result, {"scale": scale, "d_scale": d_scale}, {"wmin_m": wmin_m, "d_wmin_m": d_wmin_m}---@register_dtype("gguf_int_asym_dq")-def quant_tensor_gguf_asym_dq(- tensor,- bits=4,- group_size=-1,- v=0,- min_scale=1.0,- max_scale=1.0,- scale_dtype=torch.float16,- tensor_min=None,- tensor_max=None,- q_scale_thresh=1e-5,- super_group_size=8,- super_bits=6,- **kwargs):- """Quantize and de-quantize tensor asymmetrically. For Q2_K, Q4_K, Q5_K.-- Args:- tensor: Tensor containing the tensor to be quantized- bits: Number of bits for quantization (e.g., 2, 3, 4, 8)- group_size: Number of elements to share scale for quantization- v: Rounding value perturbation- min_scale: Minimum scale coefficient for tensor- max_scale: Maximum scale coefficient for tensor- tensor_min (Tensor, optional): Minimum tensor value for quantization. Defaults to None.- tensor_max (Tensor, optional): Maximum tensor value for quantization. Defaults to None.- scale_dtype: dtype of the quantized scale,as most kernels only support FP16 or FP32, while this value is import- q_scale_thresh: clip the quantized scale's magnitude to this value to improve the numerical stability-- Returns:- Quantized and de-quantized tensor, scale, zero-point- """- from auto_round.export.export_to_gguf.utils import QK_K, K_SCALE_SIZE, GGML_QUANT_SIZES- from auto_round.export.export_to_gguf.quant_gpu import make_qkx2_quants--- tensor, orig_shape, pad_len = reshape_pad_tensor_by_group_size(tensor, group_size)-- search_kwargs = {- 2: {"rmin": -0.5, "rdelta": 0.1, "nstep": 15, "use_mad": True},- 4: {"rmin": -1, "rdelta": 0.1, "nstep": 20, "use_mad": False},- 5: {"rmin": -0.5, "rdelta": 0.1, "nstep": 15, "use_mad": False}- }- if bits not in search_kwargs:- raise KeyError(f"bits={bits} is not supported by gguf_int_asym_dq, please check.")-- maxq = 2 ** bits - 1- ggml_type = f"q{bits}_k"- block_size, type_size = GGML_QUANT_SIZES[ggml_type]- orig_dtype = tensor.dtype-- tensor = tensor.to(torch.float32)- n_blocks = tensor.nelement() // block_size- # q2_k (nb, 16, 16) q4_k/q5_k: (nb, 8, 32)- tensor = tensor.reshape(n_blocks, super_group_size, QK_K // super_group_size)-- scale, int_w, wmin_m = make_qkx2_quants(tensor, bits=bits, **search_kwargs[bits])-- ##conduct double quant- scale, d_scale = double_quant_tensor(scale, super_bits, q_scale_thresh)- wmin_m, d_wmin_m = double_quant_tensor(wmin_m, super_bits, q_scale_thresh)-- replace_ids = scale != 0- scale = scale.unsqueeze(-1)- wmin_m = wmin_m.unsqueeze(-1)- int_w[replace_ids] = torch.round(- (tensor[replace_ids] + wmin_m[replace_ids]) / scale[replace_ids]).clip(0, maxq).to(- torch.uint8)- qdq_result = (scale*int_w - wmin_m).to(orig_dtype)- qdq_result = revert_tensor_by_pad(qdq_result, orig_shape=orig_shape, pad_len=pad_len)- # zp = round_ste(wmin_m / scale) # remove this later- scale = scale.view(-1, 1)- wmin_m = wmin_m.view(-1, 1)- return qdq_result, {"scale": scale, "d_scale": d_scale}, {"wmin_m": wmin_m, "d_wmin_m": d_wmin_m}---@register_dtype("gguf_int_sym_dq")-def quant_tensor_gguf_sym_dq(- tensor,- bits=4,- group_size=-1,- v=0,- min_scale=1.0,- max_scale=1.0,- scale_dtype=torch.float16,- tensor_min=None,- tensor_max=None,- q_scale_thresh=1e-5,- super_group_size=16,- super_bits=6,- **kwargs):- """Quantize and de-quantize tensor asymmetrically. For Q3_K, Q6_K.-- Args:- tensor: Tensor containing the tensor to be quantized- bits: Number of bits for quantization (e.g., 2, 3, 4, 8)- group_size: Number of elements to share scale for quantization- v: Rounding value perturbation- min_scale: Minimum scale coefficient for tensor- max_scale: Maximum scale coefficient for tensor- tensor_min (Tensor, optional): Minimum tensor value for quantization. Defaults to None.- tensor_max (Tensor, optional): Maximum tensor value for quantization. Defaults to None.- scale_dtype: dtype of the quantized scale,as most kernels only support FP16 or FP32, while this value is import- q_scale_thresh: clip the quantized scale's magnitude to this value to improve the numerical stability-- Returns:- Quantized and de-quantized tensor, scale, zero-point- """- from auto_round.export.export_to_gguf.utils import QK_K, K_SCALE_SIZE, GGML_QUANT_SIZES- from auto_round.export.export_to_gguf.quant_gpu import make_q3_quants, make_qx_quant-- if bits not in [3, 6]:- raise KeyError(f"bits={bits} is not supported by gguf_int_sym_dq, please check.")-- tensor, orig_shape, pad_len = reshape_pad_tensor_by_group_size(tensor, group_size)- maxq = 2 ** (bits - 1)- ggml_type = f"q{bits}_k"- block_size, type_size = GGML_QUANT_SIZES[ggml_type]- orig_dtype = tensor.dtype-- tensor = tensor.to(torch.float32)- n_blocks = tensor.nelement() // block_size- # (nb, 16, 16)- tensor = tensor.reshape(n_blocks, super_group_size, QK_K // super_group_size)-- if bits == 3:- scale, int_w = make_q3_quants(tensor, bits=3, do_rmse=True)- else:- scale, int_w = make_qx_quant(tensor, bits=6, rmse_type=1, qw=None)-- #conduct double quant- scale, d_scale = double_quant_tensor_sym(scale, super_bits, q_scale_thresh)-- replace_ids = scale != 0- scale = scale.unsqueeze(-1)- zp = torch.full_like(scale, maxq) # pylint: disable=E1130- int_w[replace_ids] = (torch.round(- tensor[replace_ids] / scale[replace_ids]).clip(-maxq, maxq - 1) + maxq).to(torch.uint8)- qdq_result = (scale * (int_w - zp)).to(orig_dtype)- qdq_result = revert_tensor_by_pad(qdq_result, orig_shape=orig_shape, pad_len=pad_len)- return qdq_result, {"scale": scale, "d_scale": d_scale}, zp-@register_dtype("int_asym")def quant_tensor_asym(tensor, bits=4, group_size=-1, v=0, min_scale=1.0, max_scale=1.0, scale_dtype=torch.float16,tensor_min=None, tensor_max=None, q_scale_thresh=1e-5, **kwargs):@@ -385,62 +109,6 @@ def quant_tensor_asym(tensor, bits=4, group_size=-1, v=0, min_scale=1.0, max_scareturn qdq_result, scale, zp-@register_dtype("int_asym_float_zp")-def quant_tensor_asym_float_zp(- tensor,- bits=4,- group_size=-1,- v=0,- min_scale=1.0,- max_scale=1.0,- scale_dtype=torch.float16,- tensor_min=None,- tensor_max=None,- q_scale_thresh=1e-5,- **kwargs):- """Quantize and de-quantize tensor asymmetrically.-- Args:- tensor: Tensor containing the tensor to be quantized- bits: Number of bits for quantization (e.g., 2, 3, 4, 8)- group_size: Number of elements to share scale for quantization- v: Rounding value perturbation- min_scale: Minimum scale coefficient for tensor- max_scale: Maximum scale coefficient for tensor- tensor_min (Tensor, optional): Minimum tensor value for quantization. Defaults to None.- tensor_max (Tensor, optional): Maximum tensor value for quantization. Defaults to None.- scale_dtype: dtype of the quantized scale,as most kernels only support FP16 or FP32, while this value is import- q_scale_thresh: clip the quantized scale's magnitude to this value to improve the numerical stability-- Returns:- Quantized and de-quantized tensor, scale, zero-point- """- tensor, orig_shape, pad_len = reshape_pad_tensor_by_group_size(tensor, group_size)- maxq = 2 ** bits - 1- if tensor_min is None or tensor_max is None:- wmin_tmp = torch.clamp(tensor.min(-1)[0], max=0)- wmax_tmp = torch.clamp(tensor.max(-1)[0], min=0)- else:- wmin_tmp = tensor_min- wmax_tmp = tensor_max- if isinstance(min_scale, torch.Tensor):- wmin = wmin_tmp * min_scale- wmax = wmax_tmp * max_scale- else:- wmin = wmin_tmp- wmax = wmax_tmp- scale = ((wmax - wmin) / maxq).to(scale_dtype)- scale = torch.clamp(scale, min=q_scale_thresh)- zp = -wmin / scale # pylint: disable=E1130- scale = scale.unsqueeze(dim=-1)- zp = zp.unsqueeze(dim=-1)- int_w = round_ste(tensor / scale + v)- q = torch.clamp(int_w + zp, 0, maxq)- qdq_result = (scale * (q - zp)).to(tensor.dtype)- qdq_result = revert_tensor_by_pad(qdq_result, orig_shape=orig_shape, pad_len=pad_len)- return qdq_result, scale, zp--@register_dtype("int_sym_gptq")def quant_tensor_sym_gptq(tensor, bits=4, group_size=-1, v=0, min_scale=1.0, max_scale=1.0, scale_dtype=torch.float16,tensor_min=None,diff --git a/auto_round/export/export_to_gguf/config.py b/auto_round/export/export_to_gguf/config.pyindex f82853f32..d227cfa9d 100644--- a/auto_round/export/export_to_gguf/config.py+++ b/auto_round/export/export_to_gguf/config.py@@ -12,77 +12,199 @@# See the License for the specific language governing permissions and# limitations under the License.-GGUF_CONFIG = {}+GGUF_INNER_CONFIG = {}-GGUF_CONFIG["gguf:q4_0"] = {"bits": 4, "act_bits": 16, "group_size": 32, "sym": True, "data_type": "int"}+GGUF_INNER_CONFIG["gguf:q4_0"] = {+ "bits": 4,+ "act_bits": 16,+ "group_size": 32,+ "sym": True,+ "data_type": "int",+ "embedding": "gguf:q4_0",+ "lm_head": "gguf:q6_k_s",+ "super_bits": None,+ "super_group_size": None,+}-GGUF_CONFIG["gguf:q4_1"] = {+GGUF_INNER_CONFIG["gguf:q4_1"] = {"bits": 4,"act_bits": 16,"group_size": 32,"sym": False,- "data_type": "int_asym_float_zp"+ "data_type": "int_asym_float_zp",+ "embedding": "gguf:q4_1",+ "lm_head": "gguf:q6_k_s",+ "super_bits": None,+ "super_group_size": None,}-GGUF_CONFIG["gguf:q5_0"] = {"bits": 5, "act_bits": 16, "group_size": 32, "sym": True, "data_type": "int"}+GGUF_INNER_CONFIG["gguf:q5_0"] = {+ "bits": 5,+ "act_bits": 16,+ "group_size": 32,+ "sym": True,+ "data_type": "int",+ "embedding": "gguf:q5_0",+ "lm_head": "gguf:q6_k_s",+ "super_bits": None,+ "super_group_size": None,+}-GGUF_CONFIG["gguf:q5_1"] = {+GGUF_INNER_CONFIG["gguf:q5_1"] = {"bits": 5,"act_bits": 16,"group_size": 32,"sym": False,- "data_type": "int_asym_float_zp"+ "data_type": "int_asym_float_zp",+ "embedding": "gguf:q5_1",+ "lm_head": "gguf:q6_k_s",+ "super_bits": None,+ "super_group_size": None,}-GGUF_CONFIG["gguf:q8_0"] = {"bits": 8, "act_bits": 16, "group_size": 32, "sym": True, "data_type": "int"}-+GGUF_INNER_CONFIG["gguf:q8_0"] = {+ "bits": 8,+ "act_bits": 16,+ "group_size": 32,+ "sym": True,+ "data_type": "int",+ "embedding": "gguf:q8_0",+ "lm_head": "gguf:q8_0",+ "super_bits": None,+ "super_group_size": None,+}-GGUF_CONFIG["gguf:q2_k_s"] = {+GGUF_INNER_CONFIG["gguf:q2_k"] = {"bits": 2,"act_bits": 16,"super_group_size": 16,"super_bits": 4,"group_size": 16,"sym": False,- "data_type": "int_asym_dq"+ "data_type": "int_asym_dq",+ "embedding": "gguf:q2_k",+ "lm_head": "gguf:q6_k"}-GGUF_CONFIG["gguf:q3_k_s"] = {+GGUF_INNER_CONFIG["gguf:q3_k"] = {"bits": 3,"act_bits": 16,"super_group_size": 16,"super_bits": 6,"group_size": 16,"sym": True,- "data_type": "int_sym_dq"+ "data_type": "int_sym_dq",+ "embedding": "gguf:q3_k",+ "lm_head": "gguf:q6_k"}-GGUF_CONFIG["gguf:q4_k_m"] = GGUF_CONFIG["gguf:q4_k_s"] = {+GGUF_INNER_CONFIG["gguf:q4_k"] = {"bits": 4,"act_bits": 16,"super_group_size": 8,"super_bits": 6,"group_size": 32,"sym": False,- "data_type": "int_asym_dq"+ "data_type": "int_asym_dq",+ "embedding": "gguf:q4_k",+ "lm_head": "gguf:q6_k"}-GGUF_CONFIG["gguf:q5_k_s"] = {+GGUF_INNER_CONFIG["gguf:q5_k"] = {"bits": 5,"act_bits": 16,"super_group_size": 8,"super_bits": 6,"group_size": 32,"sym": False,- "data_type": "int_asym_dq"+ "data_type": "int_asym_dq",+ "embedding": "gguf:q5_k",+ "lm_head": "gguf:q6_k"}-GGUF_CONFIG["gguf:q6_k"] = GGUF_CONFIG["gguf:q6_k_s"] = {+GGUF_INNER_CONFIG["gguf:q6_k"] = {"bits": 6,"act_bits": 16,"super_group_size": 16,"super_bits": 8,"group_size": 16,"sym": True,- "data_type": "int_sym_dq"+ "data_type": "int_sym_dq",+ "embedding": "gguf:q6_k",+ "lm_head": "gguf:q6_k"}++GGUF_INNER_CONFIG["gguf:bf16"]=GGUF_INNER_CONFIG["gguf:fp16"] = {+ "bits": 16,+ "act_bits": 16,+ "super_group_size": None,+ "super_bits": None,+ "group_size": None,+ "sym": True,+ "data_type": "int_sym_dq",+ "embedding": None,+ "lm_head": None,+}++++GGUF_CONFIG = {}+GGUF_CONFIG["gguf:q4_0"] = GGUF_INNER_CONFIG["gguf:q4_0"]+GGUF_CONFIG["gguf:q4_0"]["mostly"] = "gguf:q4_0"+GGUF_CONFIG["gguf:q4_1"] = GGUF_INNER_CONFIG["gguf:q4_1"]+GGUF_CONFIG["gguf:q4_1"]["mostly"]= "gguf:q4_1"+GGUF_CONFIG["gguf:q5_0"] = GGUF_INNER_CONFIG["gguf:q5_0"]+GGUF_CONFIG["gguf:q5_0"]["mostly"]= "gguf:q5_0"+GGUF_CONFIG["gguf:q5_1"] = GGUF_INNER_CONFIG["gguf:q5_1"]+GGUF_CONFIG["gguf:q5_1"]["mostly"] = "gguf:q5_1"+GGUF_CONFIG["gguf:q2_k"] = GGUF_INNER_CONFIG["gguf:q2_k"]+GGUF_CONFIG["gguf:q2_k"]["mostly"] = "gguf:q2_k"+GGUF_CONFIG["gguf:q2_k_s"] = GGUF_INNER_CONFIG["gguf:q2_k"]+GGUF_CONFIG["gguf:q2_k_s"]["mostly"]= "gguf:q2_k"+# GGUF_CONFIG["gguf:q3_k"] = GGUF_INNER_CONFIG["gguf:q3_k"]+# GGUF_CONFIG["gguf:q3_k"]["mostly"] = "gguf:q3_k"+GGUF_CONFIG["gguf:q3_k_s"] = GGUF_INNER_CONFIG["gguf:q3_k"]+GGUF_CONFIG["gguf:q3_k_s"]["mostly"] = "gguf:q3_k"+GGUF_CONFIG["gguf:q3_k_m"] = GGUF_INNER_CONFIG["gguf:q3_k"]+GGUF_CONFIG["gguf:q3_k_m"]["mostly"] = "gguf:q3_k"+GGUF_CONFIG["gguf:q3_k_l"] = GGUF_INNER_CONFIG["gguf:q3_k"]+GGUF_CONFIG["gguf:q3_k_l"]["mostly"]= "gguf:q3_k"+# GGUF_CONFIG["gguf:q4_k"] = GGUF_INNER_CONFIG["gguf:q4_k"]+# GGUF_CONFIG["gguf:q4_k"]["mostly"]= "gguf:q4_k"+GGUF_CONFIG["gguf:q4_k_s"] = GGUF_INNER_CONFIG["gguf:q4_k"]+GGUF_CONFIG["gguf:q4_k_s"]["mostly"]= "gguf:q4_k"+GGUF_CONFIG["gguf:q4_k_m"] = GGUF_INNER_CONFIG["gguf:q4_k"]+GGUF_CONFIG["gguf:q4_k_m"]["mostly"] = "gguf:q4_k"+GGUF_CONFIG["gguf:q5_k"] = GGUF_INNER_CONFIG["gguf:q5_k"]+GGUF_CONFIG["gguf:q5_k"]["mostly"]= "gguf:q5_k"+GGUF_CONFIG["gguf:q5_k_s"] = GGUF_INNER_CONFIG["gguf:q5_k"]+GGUF_CONFIG["gguf:q5_k_s"]["mostly"] = "gguf:q5_k"+GGUF_CONFIG["gguf:q5_k_m"] = GGUF_INNER_CONFIG["gguf:q5_k"]+GGUF_CONFIG["gguf:q5_k_m"]["mostly"] = "gguf:q5_k"+GGUF_CONFIG["gguf:q6_k"] = GGUF_INNER_CONFIG["gguf:q6_k"]+GGUF_CONFIG["gguf:q6_k"]["mostly"]= "gguf:q6_k"+GGUF_CONFIG["gguf:q8_0"] = GGUF_INNER_CONFIG["gguf:q8_0"]+GGUF_CONFIG["gguf:q8_0"]["mostly"]= "gguf:q8_0"+# GGUF_CONFIG["gguf:fp16"] = GGUF_INNER_CONFIG["gguf:fp16"]+# GGUF_CONFIG["gguf:fp16"]["mostly"]= "gguf:fp16"+# GGUF_CONFIG["gguf:bf16"] = GGUF_INNER_CONFIG["gguf:fp16"]+# GGUF_CONFIG["gguf:bf16"]["mostly"]= "gguf:bf16"++++QK_K = 256+K_SCALE_SIZE = 12+GGML_QUANT_SIZES = {+ "bf16": (1, 2),+ "q4_0": (32, 2 + 16),+ "q4_1": (32, 2 + 2 + 16),+ "q5_0": (32, 2 + 4 + 16),+ "q5_1": (32, 2 + 2 + 4 + 16),+ "q8_0": (32, 2 + 32),+ "q2_k": (256, 2 + 2 + QK_K//16 + QK_K//4),+ "q3_k": (256, 2 + QK_K // 4 + QK_K // 8 + 12),+ "q4_k": (256, 2 + 2 + QK_K//2 + 12),+ "q5_k": (256, 2 + 2 + QK_K // 2 + QK_K // 8 + 12),+ "q6_k": (256, 2 + QK_K // 2 + QK_K // 4 + QK_K // 16),+ "q8_k": (256, 4 + QK_K + QK_K // 8)+}\ No newline at end of filediff --git a/auto_round/export/export_to_gguf/convert.py b/auto_round/export/export_to_gguf/convert.pyindex 085a91a36..a979cf0b8 100644--- a/auto_round/export/export_to_gguf/convert.py+++ b/auto_round/export/export_to_gguf/convert.py@@ -52,10 +52,8 @@ import mathimport numpy as npimport torch-# autoround importfrom auto_round.utils import logger, LazyImport-from .quant_cpu import ggml_quant_cpu-from .quant_gpu import ggml_quant_gpu+from .packing import ggml_quant_gpugguf = LazyImport("gguf")@@ -65,6 +63,7 @@ if TYPE_CHECKING:if 'NO_LOCAL_GGUF' not in os.environ:sys.path.insert(1, str(Path(__file__).parent / 'gguf-py'))+###### MODEL DEFINITIONS ######class SentencePieceTokenTypes(IntEnum):@@ -189,7 +188,7 @@ class OriModel:def set_vocab(self):self._set_vocab_gpt2()- def get_tensors(self) -> Iterator[tuple[str, Tensor]]: # pylint: disable=E0202+ def get_tensors(self) -> Iterator[tuple[str, Tensor]]: # pylint: disable=E0202tensor_names_from_parts: set[str] = set()index_name = "model.safetensors" if self.is_safetensors else "pytorch_model.bin"@@ -650,7 +649,7 @@ class OriModel:# we will use this unique identifier to write a "tokenizer.ggml.pre" entry in the GGUF file which we can# use in llama.cpp to implement the same pre-tokenizer- chktxt = '\n \n\n \n\n\n \t \t\t \t\n \n \n \n \n🚀 (normal) 😶\u200d🌫️ (multiple emojis concatenated) ✅ 🦙🦙 3 33 333 3333 33333 333333 3333333 33333333 3.3 3..3 3...3 កាន់តែពិសេសអាច😁 ?我想在apple工作1314151天~ ------======= нещо на Български \'\'\'\'\'\'```````""""......!!!!!!?????? I\'ve been \'told he\'s there, \'RE you sure? \'M not sure I\'ll make it, \'D you like some tea? We\'Ve a\'lL' # pylint: disable=C0301+ chktxt = '\n \n\n \n\n\n \t \t\t \t\n \n \n \n \n🚀 (normal) 😶\u200d🌫️ (multiple emojis concatenated) ✅ 🦙🦙 3 33 333 3333 33333 333333 3333333 33333333 3.3 3..3 3...3 កាន់តែពិសេសអាច😁 ?我想在apple工作1314151天~ ------======= нещо на Български \'\'\'\'\'\'```````""""......!!!!!!?????? I\'ve been \'told he\'s there, \'RE you sure? \'M not sure I\'ll make it, \'D you like some tea? We\'Ve a\'lL' # pylint: disable=C0301chktok = tokenizer.encode(chktxt)chkhsh = sha256(str(chktok).encode()).hexdigest()@@ -733,7 +732,7 @@ class OriModel:# ref: https://huggingface.co/jinaai/jina-embeddings-v2-base-coderes = "jina-v2-code"if chkhsh == "b6e8e1518dc4305be2fe39c313ed643381c4da5db34a98f6a04c093f8afbe99b" \- or chkhsh == "81d72c7348a9f0ebe86f23298d37debe0a5e71149e29bd283904c02262b27516":+ or chkhsh == "81d72c7348a9f0ebe86f23298d37debe0a5e71149e29bd283904c02262b27516":# ref: https://huggingface.co/THUDM/glm-4-9b-chatres = "chatglm-bpe"if chkhsh == "7fc505bd3104ca1083b150b17d088b59534ede9bde81f0dd2090967d7fe52cee":@@ -1135,7 +1134,7 @@ class Model(OriModel):def get_tensors(self) -> Iterator[tuple[str, Tensor]]:for name, tensor in self.model.named_parameters():yield name, tensor-+def _quant_data(self, data_torch, data_qtype, name, bid):from auto_round.utils import get_modulesuffix = '.weight'@@ -1144,7 +1143,7 @@ class Model(OriModel):module = get_module(self.model, layer_name)if hasattr(module, "scale"):if hasattr(self, "permute"):- bs = module.scale.shape[0]+ bs = module.weight.shape[0]for attr in ["scale", "zp", "w_d_scale", "w_d_wmin_m", "w_wmin_m"]:if hasattr(module, attr) and getattr(module, attr) is not None:attr_tensor = getattr(module, attr)@@ -1154,30 +1153,43 @@ class Model(OriModel):setattr(module, attr, attr_tensor)scale = module.scalezp = module.zp if hasattr(module, "zp") else None- ggml_quant = ggml_quant_gpu if torch.cuda.is_available() else ggml_quant_cpu- # ggml_quant = ggml_quant_cpu-+ if torch.cuda.is_available():+ device = "cuda"+ elif torch.xpu.is_available():+ device = "xpu"+ else:+ device = "cpu"if data_qtype.name.lower().endswith("_k"):d_scale = module.w_d_scale.to(torch.float32)d_wmin_m = module.w_d_wmin_m.to(torch.float32) if hasattr(module, "w_d_wmin_m") else Nonewmin_m = module.w_wmin_m.to(torch.float32) if hasattr(module, "w_wmin_m") else None- data = ggml_quant(+ data = ggml_quant_gpu(data_torch,data_qtype.name.lower(),scale,zp,wmin_m=wmin_m,d_scale=d_scale,- d_wmin_m=d_wmin_m+ d_wmin_m=d_wmin_m,+ device=device)else:- data = ggml_quant(data_torch, data_qtype.name.lower(), scale, zp)+ data = ggml_quant_gpu(data_torch, data_qtype.name.lower(), scale, zp, device=device)else:- data_qtype = gguf.GGMLQuantizationType.F32- data = data_torch.squeeze().cpu().numpy()+ # if data_torch.dtype ==torch.float32:+ # data_qtype = gguf.GGMLQuantizationType.F32+ # else:+ # data_qtype = gguf.GGMLQuantizationType.F16+ data_qtype = gguf.GGMLQuantizationType.F32 ##FP16 has issues at inference+ data = data_torch.to(torch.float32).squeeze().cpu().numpy()else:+ # if data_torch.dtype == torch.float32:+ # data_qtype = gguf.GGMLQuantizationType.F32+ # else:+ # data_qtype = gguf.GGMLQuantizationType.F16+ # data = data_torch.squeeze().cpu().numpy()data_qtype = gguf.GGMLQuantizationType.F32- data = data_torch.squeeze().cpu().numpy()+ data = data_torch.to(torch.float32).squeeze().cpu().numpy()return data, data_qtypedef get_qtype_by_layer_config(self, layer_config, name, data_qtype):@@ -1185,31 +1197,57 @@ class Model(OriModel):if name not in layer_config or layer_config[name]['bits'] >= 16:return data_qtypebits = layer_config[name]['bits']- data_qtype = re.sub(r"Q\d", f"Q{bits}", self.ftype.name)-- if data_qtype == "MOSTLY_Q8_0":- data_qtype = gguf.GGMLQuantizationType.Q8_0- elif data_qtype == "MOSTLY_Q4_0":- data_qtype = gguf.GGMLQuantizationType.Q4_0- elif data_qtype == "MOSTLY_Q4_1":- data_qtype = gguf.GGMLQuantizationType.Q4_1- elif data_qtype == "MOSTLY_Q5_0":- data_qtype = gguf.GGMLQuantizationType.Q5_0- elif data_qtype == "MOSTLY_Q5_1":- data_qtype = gguf.GGMLQuantizationType.Q5_1- elif data_qtype.startswith("MOSTLY_Q2_K"):- data_qtype = gguf.GGMLQuantizationType.Q2_K- elif data_qtype.startswith("MOSTLY_Q3_K"):- data_qtype = gguf.GGMLQuantizationType.Q3_K- elif data_qtype.startswith("MOSTLY_Q4_K"):- data_qtype = gguf.GGMLQuantizationType.Q4_K- elif data_qtype.startswith("MOSTLY_Q5_K"):- data_qtype = gguf.GGMLQuantizationType.Q5_K- elif data_qtype.startswith("MOSTLY_Q6_K"):- data_qtype = gguf.GGMLQuantizationType.Q6_K- else:- raise ValueError(f"Unknown file type: {data_qtype}")- return data_qtype+ super_bits = layer_config[name]["super_bits"]+ sym = layer_config[name]["sym"]+ if bits == 2:+ return gguf.GGMLQuantizationType.Q2_K+ if bits == 3:+ return gguf.GGMLQuantizationType.Q3_K+ if bits == 4:+ if super_bits is not None:+ return gguf.GGMLQuantizationType.Q4_K+ if super_bits is None and sym:+ return gguf.GGMLQuantizationType.Q4_0+ if super_bits is None and not sym:+ return gguf.GGMLQuantizationType.Q4_1+ if bits == 5:+ if super_bits is not None:+ return gguf.GGMLQuantizationType.Q5_K+ if super_bits is None and sym:+ return gguf.GGMLQuantizationType.Q5_0+ if super_bits is None and not sym:+ return gguf.GGMLQuantizationType.Q5_1+ if bits == 6:+ return gguf.GGMLQuantizationType.Q6_K+ if bits == 8:+ return gguf.GGMLQuantizationType.Q8_0+ raise ValueError(f"Unknown file type: {data_qtype}")++ # data_qtype = re.sub(r"Q\d", f"Q{bits}", self.ftype.name)+ #+ # if data_qtype == "MOSTLY_Q8_0":+ # data_qtype = gguf.GGMLQuantizationType.Q8_0+ # elif data_qtype == "MOSTLY_Q4_0":+ # data_qtype = gguf.GGMLQuantizationType.Q4_0+ # elif data_qtype == "MOSTLY_Q4_1":+ # data_qtype = gguf.GGMLQuantizationType.Q4_1+ # elif data_qtype == "MOSTLY_Q5_0":+ # data_qtype = gguf.GGMLQuantizationType.Q5_0+ # elif data_qtype == "MOSTLY_Q5_1":+ # data_qtype = gguf.GGMLQuantizationType.Q5_1+ # elif data_qtype.startswith("MOSTLY_Q2_K"):+ # data_qtype = gguf.GGMLQuantizationType.Q2_K+ # elif data_qtype.startswith("MOSTLY_Q3_K"):+ # data_qtype = gguf.GGMLQuantizationType.Q3_K+ # elif data_qtype.startswith("MOSTLY_Q4_K"):+ # data_qtype = gguf.GGMLQuantizationType.Q4_K+ # elif data_qtype.startswith("MOSTLY_Q5_K"):+ # data_qtype = gguf.GGMLQuantizationType.Q5_K+ # elif data_qtype.startswith("MOSTLY_Q6_K"):+ # data_qtype = gguf.GGMLQuantizationType.Q6_K+ # else:+ # raise ValueError(f"Unknown file type: {data_qtype}")+ # return data_qtypedef prepare_tensors(self):max_name_len = max(len(s) for _, s in self.tensor_map.mapping.values()) + len(".weight,")@@ -1272,65 +1310,68 @@ class Model(OriModel):# TODO: use Q4_K and Q6_Kdata_qtype = gguf.GGMLQuantizationType.F16# data_qtype = gguf.GGMLQuantizationType.Q8_0 # llama.cpp:llama_tensor_get_type-+# get data_qtype by layer_configdata_qtype = self.get_qtype_by_layer_config(self.layer_config, name, data_qtype)- # No override (data_qtype is False), or wants to be quantized (data_qtype is True)- if isinstance(data_qtype, bool):- if self.ftype == gguf.LlamaFileType.ALL_F32:- data_qtype = gguf.GGMLQuantizationType.F32- elif self.ftype == gguf.LlamaFileType.MOSTLY_F16:+ # # No override (data_qtype is False), or wants to be quantized (data_qtype is True)+ # if isinstance(data_qtype, bool):+ # if self.ftype == gguf.LlamaFileType.ALL_F32:+ # data_qtype = gguf.GGMLQuantizationType.F32+ # elif self.ftype == gguf.LlamaFileType.MOSTLY_F16:+ # data_qtype = gguf.GGMLQuantizationType.F16+ # elif self.ftype == gguf.LlamaFileType.MOSTLY_BF16:+ # data_qtype = gguf.GGMLQuantizationType.BF16+ # elif self.ftype == gguf.LlamaFileType.MOSTLY_Q8_0:+ # data_qtype = gguf.GGMLQuantizationType.Q8_0+ # elif self.ftype == gguf.LlamaFileType.MOSTLY_Q4_0:+ # data_qtype = gguf.GGMLQuantizationType.Q4_0+ # elif self.ftype == gguf.LlamaFileType.MOSTLY_Q4_1:+ # data_qtype = gguf.GGMLQuantizationType.Q4_1+ # elif self.ftype == gguf.LlamaFileType.MOSTLY_Q5_0:+ # data_qtype = gguf.GGMLQuantizationType.Q5_0+ # elif self.ftype == gguf.LlamaFileType.MOSTLY_Q5_1:+ # data_qtype = gguf.GGMLQuantizationType.Q5_1+ # elif self.ftype == gguf.LlamaFileType.MOSTLY_Q2_K_S:+ # data_qtype = gguf.GGMLQuantizationType.Q2_K+ # elif self.ftype == gguf.LlamaFileType.MOSTLY_Q3_K_S:+ # data_qtype = gguf.GGMLQuantizationType.Q3_K+ # elif self.ftype == gguf.LlamaFileType.MOSTLY_Q4_K_S:+ # data_qtype = gguf.GGMLQuantizationType.Q4_K+ # elif self.ftype == gguf.LlamaFileType.MOSTLY_Q4_K_M:+ # data_qtype = gguf.GGMLQuantizationType.Q4_K+ # elif self.ftype == gguf.LlamaFileType.MOSTLY_Q5_K_S:+ # data_qtype = gguf.GGMLQuantizationType.Q5_K+ # elif self.ftype == gguf.LlamaFileType.MOSTLY_Q6_K:+ # data_qtype = gguf.GGMLQuantizationType.Q6_K+ # else:+ # raise ValueError(f"Unknown file type: {self.ftype.name}")++ if isinstance(data_qtype, bool) or data_qtype in [gguf.GGMLQuantizationType.F16,+ gguf.GGMLQuantizationType.BF16,+ gguf.GGMLQuantizationType.F32]:+ try:+ data = gguf.quants.quantize(data, data_qtype)+ except gguf.QuantError as e:+ logger.warning("%s, %s", e, "falling back to F16")data_qtype = gguf.GGMLQuantizationType.F16- elif self.ftype == gguf.LlamaFileType.MOSTLY_BF16:- data_qtype = gguf.GGMLQuantizationType.BF16- elif self.ftype == gguf.LlamaFileType.MOSTLY_Q8_0:- data_qtype = gguf.GGMLQuantizationType.Q8_0- elif self.ftype == gguf.LlamaFileType.MOSTLY_Q4_0:- data_qtype = gguf.GGMLQuantizationType.Q4_0- elif self.ftype == gguf.LlamaFileType.MOSTLY_Q4_1:- data_qtype = gguf.GGMLQuantizationType.Q4_1- elif self.ftype == gguf.LlamaFileType.MOSTLY_Q5_0:- data_qtype = gguf.GGMLQuantizationType.Q5_0- elif self.ftype == gguf.LlamaFileType.MOSTLY_Q5_1:- data_qtype = gguf.GGMLQuantizationType.Q5_1- elif self.ftype == gguf.LlamaFileType.MOSTLY_Q2_K_S:- data_qtype = gguf.GGMLQuantizationType.Q2_K- elif self.ftype == gguf.LlamaFileType.MOSTLY_Q3_K_S:- data_qtype = gguf.GGMLQuantizationType.Q3_K- elif self.ftype == gguf.LlamaFileType.MOSTLY_Q4_K_S:- data_qtype = gguf.GGMLQuantizationType.Q4_K- elif self.ftype == gguf.LlamaFileType.MOSTLY_Q4_K_M:- data_qtype = gguf.GGMLQuantizationType.Q4_K- elif self.ftype == gguf.LlamaFileType.MOSTLY_Q5_K_S:- data_qtype = gguf.GGMLQuantizationType.Q5_K- elif self.ftype == gguf.LlamaFileType.MOSTLY_Q6_K:- data_qtype = gguf.GGMLQuantizationType.Q6_K- else:- raise ValueError(f"Unknown file type: {self.ftype.name}")-- # try:- # data = gguf.quants.quantize(data, data_qtype)- # except gguf.QuantError as e:- # logger.warning("%s, %s", e, "falling back to F16")- # data_qtype = gguf.GGMLQuantizationType.F16- # data = gguf.quants.quantize(data, data_qtype)-- # for MOE model- if len(data_torch.shape) == 3:- new_data = []- for idx, arr in enumerate(data_torch):- arr_name = name.split('.')- for i in range(len(arr_name) - 1, -1, -1):- if arr_name[i].isdecimal() and int(arr_name[i]) == (data_torch.shape[0] - 1):- arr_name[i] = str(idx)- arr_name = ".".join(arr_name)- arr, data_qtype = self._quant_data(arr, data_qtype, name, bid)- new_data.append(arr)- data = np.array(new_data)- del new_data+ data = gguf.quants.quantize(data, data_qtype)else:- data, data_qtype = self._quant_data(data_torch, data_qtype, name, bid)+ # for MOE model+ if len(data_torch.shape) == 3:+ new_data = []+ for idx, arr in enumerate(data_torch):+ arr_name = name.split('.')+ for i in range(len(arr_name) - 1, -1, -1):+ if arr_name[i].isdecimal() and int(arr_name[i]) == (data_torch.shape[0] - 1):+ arr_name[i] = str(idx)+ arr_name = ".".join(arr_name)+ arr, data_qtype = self._quant_data(arr, data_qtype, arr_name, bid)+ new_data.append(arr)+ data = np.array(new_data)+ del new_data+ else:+ data, data_qtype = self._quant_data(data_torch, data_qtype, name, bid)shape = gguf.quant_shape_from_byte_shape(data.shape, data_qtype) if data.dtype == np.uint8 else data.shape@@ -1358,7 +1399,7 @@ class GPTNeoXModel(Model):self.gguf_writer.add_block_count(block_count)self.gguf_writer.add_feed_forward_length(self.hparams["intermediate_size"])self.gguf_writer.add_rope_dimension_count(- int(self.hparams["rotary_pct"] * (self.hparams["hidden_size"] // self.hparams["num_attention_heads"])),)+ int(self.hparams["rotary_pct"] * (self.hparams["hidden_size"] // self.hparams["num_attention_heads"])), )self.gguf_writer.add_head_count(self.hparams["num_attention_heads"])self.gguf_writer.add_parallel_residual(self.hparams.get("use_parallel_residual", True))self.gguf_writer.add_layer_norm_eps(self.hparams["layer_norm_eps"])@@ -1599,18 +1640,18 @@ class BaichuanModel(Model):*weights.shape[1:]).swapaxes(1, 2).reshape(weights.shape))def _reverse_hf_permute_part(- self,- weights: Tensor,- n_part: int,- n_head: int,- n_head_kv: int | None = None,+ self,+ weights: Tensor,+ n_part: int,+ n_head: int,+ n_head_kv: int | None = None,) -> Tensor:r = weights.shape[0] // 3- return self._reverse_hf_permute(weights[r * n_part:r*n_part + r, ...], n_head, n_head_kv)+ return self._reverse_hf_permute(weights[r * n_part:r * n_part + r, ...], n_head, n_head_kv)def _reverse_hf_part(self, weights: Tensor, n_part: int) -> Tensor:r = weights.shape[0] // 3- return weights[r * n_part:r*n_part + r, ...]+ return weights[r * n_part:r * n_part + r, ...]@Model.register("XverseForCausalLM")@@ -1763,7 +1804,7 @@ class FalconModel(Model):n_head_kv = self.find_hparam(["num_kv_heads", "n_head_kv"], optional=True) or 1head_dim = self.hparams["hidden_size"] // n_head- qkv = data_torch.view(n_head_kv, n_head//n_head_kv + 2, head_dim, head_dim * n_head)+ qkv = data_torch.view(n_head_kv, n_head // n_head_kv + 2, head_dim, head_dim * n_head)q = qkv[:, :-2].reshape(n_head * head_dim, head_dim * n_head)k = qkv[:, [-2]].reshape(n_head_kv * head_dim, head_dim * n_head)v = qkv[:, [-1]].reshape(n_head_kv * head_dim, head_dim * n_head)@@ -1810,7 +1851,7 @@ class RefactModel(Model):inner_dim = 4 * hidden_dimhidden_dim = int(2 * inner_dim / 3)multiple_of = 256- ff_dim = multiple_of * ((hidden_dim+multiple_of-1) // multiple_of)+ ff_dim = multiple_of * ((hidden_dim + multiple_of - 1) // multiple_of)block_count = self.hparams["n_layer"]@@ -1830,7 +1871,7 @@ class RefactModel(Model):inner_dim = 4 * hidden_dimhidden_dim = int(2 * inner_dim / 3)multiple_of = 256- ff_dim = multiple_of * ((hidden_dim+multiple_of-1) // multiple_of)+ ff_dim = multiple_of * ((hidden_dim + multiple_of - 1) // multiple_of)n_head = self.hparams["n_head"]n_head_kv = 1head_dim = self.hparams["n_embd"] // n_head@@ -2057,7 +2098,7 @@ class LlamaModel(Model):if rope_scaling.get("rope_type", '').lower() == "llama3":base = self.hparams.get("rope_theta", 10000.0)dim = self.hparams.get("head_dim", self.hparams["hidden_size"] // self.hparams["num_attention_heads"])- freqs = 1.0 / (base**(torch.arange(0, dim, 2, dtype=torch.float32) / dim))+ freqs = 1.0 / (base ** (torch.arange(0, dim, 2, dtype=torch.float32) / dim))factor = rope_scaling.get("factor", 8.0)low_freq_factor = rope_scaling.get("low_freq_factor", 1.0)@@ -2076,8 +2117,8 @@ class LlamaModel(Model):elif wavelen > low_freq_wavelen:rope_factors.append(factor)else:- smooth = (old_context_len/wavelen - low_freq_factor) / (high_freq_factor-low_freq_factor)- rope_factors.append(1 / ((1-smooth) / factor + smooth))+ smooth = (old_context_len / wavelen - low_freq_factor) / (high_freq_factor - low_freq_factor)+ rope_factors.append(1 / ((1 - smooth) / factor + smooth))yield (self.format_tensor_name(gguf.MODEL_TENSOR.ROPE_FREQS),@@ -2176,7 +2217,7 @@ class DeciModel(Model):# DeciLM-specific codeif n % k == 0:return n- return n + k - (n%k)+ return n + k - (n % k)def __init__(self, *args, **kwargs):super().__init__(*args, **kwargs)@@ -2306,7 +2347,7 @@ class DeciModel(Model):if rope_scaling.get("rope_type", '').lower() == "llama3":base = self.hparams.get("rope_theta", 10000.0)dim = self.hparams.get("head_dim", self.hparams["hidden_size"] // self.hparams["num_attention_heads"])- freqs = 1.0 / (base**(torch.arange(0, dim, 2, dtype=torch.float32) / dim))+ freqs = 1.0 / (base ** (torch.arange(0, dim, 2, dtype=torch.float32) / dim))factor = rope_scaling.get("factor", 8.0)low_freq_factor = rope_scaling.get("low_freq_factor", 1.0)@@ -2325,8 +2366,8 @@ class DeciModel(Model):elif wavelen > low_freq_wavelen:rope_factors.append(factor)else:- smooth = (old_context_len/wavelen - low_freq_factor) / (high_freq_factor-low_freq_factor)- rope_factors.append(1 / ((1-smooth) / factor + smooth))+ smooth = (old_context_len / wavelen - low_freq_factor) / (high_freq_factor - low_freq_factor)+ rope_factors.append(1 / ((1 - smooth) / factor + smooth))yield (self.format_tensor_name(gguf.MODEL_TENSOR.ROPE_FREQS),@@ -2360,13 +2401,13 @@ class BitnetModel(Model):new_name = self.map_tensor_name(name)if any(self.match_model_tensor_name(new_name, key, bid) for key in [- gguf.MODEL_TENSOR.ATTN_Q,- gguf.MODEL_TENSOR.ATTN_K,- gguf.MODEL_TENSOR.ATTN_V,- gguf.MODEL_TENSOR.ATTN_OUT,- gguf.MODEL_TENSOR.FFN_UP,- gguf.MODEL_TENSOR.FFN_DOWN,- gguf.MODEL_TENSOR.FFN_GATE,+ gguf.MODEL_TENSOR.ATTN_Q,+ gguf.MODEL_TENSOR.ATTN_K,+ gguf.MODEL_TENSOR.ATTN_V,+ gguf.MODEL_TENSOR.ATTN_OUT,+ gguf.MODEL_TENSOR.FFN_UP,+ gguf.MODEL_TENSOR.FFN_DOWN,+ gguf.MODEL_TENSOR.FFN_GATE,]):# transform weight into 1/0/-1 (in fp32)data_torch = self.weight_quant(data_torch)@@ -2489,7 +2530,7 @@ class DbrxModel(Model):return [(new_name, data_torch)]def tensor_force_quant(self, name: str, new_name: str, bid: int | None, n_dims: int) \- -> gguf.GGMLQuantizationType | bool:+ -> gguf.GGMLQuantizationType | bool:del name, new_name, bid # unusedreturn n_dims > 1@@ -2504,7 +2545,7 @@ class MiniCPMModel(Model):embedding_scale = float(self.hparams["scale_emb"])self.gguf_writer.add_embedding_scale(embedding_scale)logger.info(f"gguf: (minicpm) embedding_scale = {embedding_scale}")- residual_scale = self.hparams["scale_depth"] / self.hparams["num_hidden_layers"]**0.5+ residual_scale = self.hparams["scale_depth"] / self.hparams["num_hidden_layers"] ** 0.5self.gguf_writer.add_residual_scale(residual_scale)logger.info(f"gguf: (minicpm) residual_scale = {residual_scale}")logit_scale = self.hparams["hidden_size"] / self.hparams["dim_model_base"]@@ -2703,8 +2744,8 @@ class WavTokenizerDecModel(Model):if \name.endswith("codebook.cluster_size") or \- name.endswith("codebook.embed_avg") or \- name.endswith("codebook.inited"):+ name.endswith("codebook.embed_avg") or \+ name.endswith("codebook.inited"):logger.debug(f"Skipping {name!r}")return []@@ -3672,11 +3713,11 @@ class XLMRobertaModel(BertModel):tokens = [b'<s>', b'<pad>', b'</s>', b'<unk>'] + tokens[3:-1]scores = [0.0, 0.0, 0.0, 0.0] + scores[3:-1]toktypes = [- SentencePieceTokenTypes.CONTROL,- SentencePieceTokenTypes.CONTROL,- SentencePieceTokenTypes.CONTROL,- SentencePieceTokenTypes.UNKNOWN,- ] + toktypes[3:-1]+ SentencePieceTokenTypes.CONTROL,+ SentencePieceTokenTypes.CONTROL,+ SentencePieceTokenTypes.CONTROL,+ SentencePieceTokenTypes.UNKNOWN,+ ] + toktypes[3:-1]self.gguf_writer.add_tokenizer_model("t5")self.gguf_writer.add_tokenizer_pre("default")@@ -3739,7 +3780,7 @@ class GemmaModel(Model):self.gguf_writer.add_feed_forward_length(hparams["intermediate_size"])self.gguf_writer.add_head_count(hparams["num_attention_heads"])self.gguf_writer.add_head_count_kv(self.hparams["num_key_value_heads"] \- if "num_key_value_heads" in hparams else hparams["num_attention_heads"])+ if "num_key_value_heads" in hparams else hparams["num_attention_heads"])self.gguf_writer.add_layer_norm_rms_eps(self.hparams["rms_norm_eps"])self.gguf_writer.add_key_length(hparams["head_dim"])self.gguf_writer.add_value_length(hparams["head_dim"])@@ -3779,7 +3820,7 @@ class Gemma2Model(Model):self.gguf_writer.add_feed_forward_length(hparams["intermediate_size"])self.gguf_writer.add_head_count(hparams["num_attention_heads"])self.gguf_writer.add_head_count_kv(self.hparams["num_key_value_heads"] \- if "num_key_value_heads" in hparams else hparams["num_attention_heads"])+ if "num_key_value_heads" in hparams else hparams["num_attention_heads"])self.gguf_writer.add_layer_norm_rms_eps(self.hparams["rms_norm_eps"])self.gguf_writer.add_key_length(hparams["head_dim"])self.gguf_writer.add_value_length(hparams["head_dim"])@@ -3927,7 +3968,7 @@ class Rwkv6Model(Model):new_name += ".weight"if new_name.endswith("time_mix_w1.weight") \- or new_name.endswith("time_mix_decay_w1.weight") or new_name.endswith("time_mix_decay_w2.weight"):+ or new_name.endswith("time_mix_decay_w1.weight") or new_name.endswith("time_mix_decay_w2.weight"):data_torch = data_torch.transpose(0, 1)if new_name.endswith("time_mix_w2.weight"):@@ -3940,7 +3981,7 @@ class Rwkv6Model(Model):rescale_every_n_layers = self.hparams["rescale_every"]if rescale_every_n_layers > 0:if new_name.endswith("time_mix_output.weight") or new_name.endswith("channel_mix_value.weight"):- data_torch = data_torch.div_(2**int(bid // rescale_every_n_layers))+ data_torch = data_torch.div_(2 ** int(bid // rescale_every_n_layers))except KeyError:pass@@ -3955,7 +3996,7 @@ class Rwkv6Model(Model):for i in ["w", "k", "v", "r", "g"]):new_name = f"blk.{bid}.time_mix_lerp_fused.weight"data = torch.stack([self.lerp_weights[bid][f"blk.{bid}.time_mix_lerp_{i}.weight"].unsqueeze(0) \- for i in ["w", "k", "v", "r", "g"]], dim=0).unsqueeze(1)+ for i in ["w", "k", "v", "r", "g"]], dim=0).unsqueeze(1)yield (new_name, data)return@@ -4024,7 +4065,7 @@ class Rwkv7Model(Model):self._set_vocab_rwkv_world()def calc_lora_rank(self, hidden_size, exponent, multiplier):- return max(1, round(hidden_size**exponent * multiplier / 32)) * 32+ return max(1, round(hidden_size ** exponent * multiplier / 32)) * 32def set_gguf_parameters(self):block_count = self.hparams["num_hidden_layers"]@@ -4041,23 +4082,30 @@ class Rwkv7Model(Model):# ICLR: In-Context-Learning-Ratetry:lora_rank_decay = self.hparams["lora_rank_decay"] if self.hparams["lora_rank_decay"] \- is not None else self.calc_lora_rank(hidden_size, 0.5, 1.8)+ is not None else self.calc_lora_rank(hidden_size, 0.5,+ 1.8)lora_rank_iclr = self.hparams["lora_rank_iclr"] if self.hparams["lora_rank_iclr"] \- is not None else self.calc_lora_rank(hidden_size, 0.5, 1.8)+ is not None else self.calc_lora_rank(hidden_size, 0.5,+ 1.8)lora_rank_value_residual_mix = self.hparams["lora_rank_value_residual_mix"] \if self.hparams["lora_rank_value_residual_mix"] is not None \- else self.calc_lora_rank(hidden_size, 0.5, 1.3)+ else self.calc_lora_rank(hidden_size, 0.5, 1.3)lora_rank_gate = self.hparams["lora_rank_gate"] if self.hparams["lora_rank_gate"] \- is not None else self.calc_lora_rank(hidden_size, 0.8, 0.6)+ is not None else self.calc_lora_rank(hidden_size, 0.8,+ 0.6)except KeyError:lora_rank_decay = self.hparams["decay_low_rank_dim"] if self.hparams["decay_low_rank_dim"] \- is not None else self.calc_lora_rank(hidden_size, 0.5, 1.8)+ is not None else self.calc_lora_rank(hidden_size,+ 0.5, 1.8)lora_rank_iclr = self.hparams["a_low_rank_dim"] if self.hparams["a_low_rank_dim"] \- is not None else self.calc_lora_rank(hidden_size, 0.5, 1.8)+ is not None else self.calc_lora_rank(hidden_size, 0.5,+ 1.8)lora_rank_value_residual_mix = self.hparams["v_low_rank_dim"] if self.hparams["v_low_rank_dim"] \- is not None else self.calc_lora_rank(hidden_size, 0.5, 1.3)+ is not None else self.calc_lora_rank(+ hidden_size, 0.5, 1.3)lora_rank_gate = self.hparams["gate_low_rank_dim"] if self.hparams["gate_low_rank_dim"] \- is not None else self.calc_lora_rank(hidden_size, 0.8, 0.6)+ is not None else self.calc_lora_rank(hidden_size, 0.8,+ 0.6)# RWKV isn't context limitedself.gguf_writer.add_context_length(1048576)@@ -4126,14 +4174,14 @@ class Rwkv7Model(Model):new_name += ".weight"if self.lora_needs_transpose and any(new_name.endswith(t) for t in [- "time_mix_w1.weight",- "time_mix_w2.weight",- "time_mix_a1.weight",- "time_mix_a2.weight",- "time_mix_v1.weight",- "time_mix_v2.weight",- "time_mix_g1.weight",- "time_mix_g2.weight",+ "time_mix_w1.weight",+ "time_mix_w2.weight",+ "time_mix_a1.weight",+ "time_mix_a2.weight",+ "time_mix_v1.weight",+ "time_mix_v2.weight",+ "time_mix_g1.weight",+ "time_mix_g2.weight",]):data_torch = data_torch.transpose(0, 1)@@ -4226,7 +4274,7 @@ class MambaModel(Model):# Fail early for models which don't have a block expansion factor of 2assert d_inner == 2 * d_model- self.gguf_writer.add_context_length(2**20) # arbitrary value; for those who use the default+ self.gguf_writer.add_context_length(2 ** 20) # arbitrary value; for those who use the defaultself.gguf_writer.add_embedding_length(d_model)self.gguf_writer.add_feed_forward_length(0) # unused, but seemingly required when loadingself.gguf_writer.add_head_count(0) # unused, but seemingly required when loading@@ -4299,7 +4347,7 @@ class Cohere2Model(Model):rotary_pct = self.hparams["rotary_pct"]hidden_size = self.hparams["hidden_size"]num_attention_heads = self.hparams["num_attention_heads"]- self.gguf_writer.add_rope_dimension_count(int(rotary_pct * (hidden_size//num_attention_heads)))+ self.gguf_writer.add_rope_dimension_count(int(rotary_pct * (hidden_size // num_attention_heads)))self.gguf_writer.add_rope_scaling_type(gguf.RopeScalingType.NONE)@@ -4449,7 +4497,7 @@ class OpenELMModel(Model):@staticmethoddef _make_divisible(v: float | int, divisor: int) -> int:- new_v = max(divisor, int(v + divisor/2) // divisor * divisor)+ new_v = max(divisor, int(v + divisor / 2) // divisor * divisor)# Make sure that round down does not go down by more than 10%.if new_v < 0.9 * v:new_v += divisor@@ -4866,7 +4914,7 @@ class DeepseekV2Model(Model):v_head_dim = self.hparams["v_head_dim"]qk_nope_head_dim = self.hparams["qk_nope_head_dim"]- assert data_torch.shape[0] == n_head_kv * (v_head_dim+qk_nope_head_dim)+ assert data_torch.shape[0] == n_head_kv * (v_head_dim + qk_nope_head_dim)kv_b = data_torch.view(n_head_kv, v_head_dim + qk_nope_head_dim, data_torch.shape[-1])k_b, v_b = torch.split(kv_b, [qk_nope_head_dim, v_head_dim], dim=1)@@ -5241,7 +5289,7 @@ class JaisModel(Model):# Some other models has max_alibi_bias spelled out explicitly in the hyperparams,# but Jais's PyTorch model simply precalculates the slope values and places them# in relative_pes.slopes- n_head_closest_log2 = 2**math.floor(math.log2(self.hparams["n_head"]))+ n_head_closest_log2 = 2 ** math.floor(math.log2(self.hparams["n_head"]))first_val = float(data_torch[0].item())self.max_alibi_bias = -round(math.log2(first_val) * n_head_closest_log2)@@ -5519,7 +5567,7 @@ class ExaoneModel(Model):if rope_scaling.get("rope_type", '').lower() == "llama3":base = self.hparams.get("rope_theta", 10000.0)dim = self.hparams.get("head_dim", self.hparams["hidden_size"] // self.hparams["num_attention_heads"])- freqs = 1.0 / (base**(torch.arange(0, dim, 2, dtype=torch.float32) / dim))+ freqs = 1.0 / (base ** (torch.arange(0, dim, 2, dtype=torch.float32) / dim))factor = rope_scaling.get("factor", 8.0)low_freq_factor = rope_scaling.get("low_freq_factor", 1.0)@@ -5538,8 +5586,8 @@ class ExaoneModel(Model):elif wavelen > low_freq_wavelen:rope_factors.append(factor)else:- smooth = (old_context_len/wavelen - low_freq_factor) / (high_freq_factor-low_freq_factor)- rope_factors.append(1 / ((1-smooth) / factor + smooth))+ smooth = (old_context_len / wavelen - low_freq_factor) / (high_freq_factor - low_freq_factor)+ rope_factors.append(1 / ((1 - smooth) / factor + smooth))yield (self.format_tensor_name(gguf.MODEL_TENSOR.ROPE_FREQS),@@ -5836,8 +5884,8 @@ def parse_args() -> argparse.Namespace:choices=["f32", "f16", "bf16", "q8_0", "tq1_0", "tq2_0", "auto"],default="f16",help="output format - use f32 for float32, f16 for float16, bf16 for bfloat16, q8_0 for Q8_0, tq1_0 or"- " tq2_0 for ternary, and auto for the highest-fidelity 16-bit float type depending on the first loaded"- " tensor type",+ " tq2_0 for ternary, and auto for the highest-fidelity 16-bit float type depending on the first loaded"+ " tensor type",)parser.add_argument("--bigendian",@@ -5898,10 +5946,10 @@ def parse_args() -> argparse.Namespace:"--remote",action="store_true",help="(Experimental) Read safetensors file remotely without downloading to disk."- " Config and tokenizer files will still be downloaded. To use this feature, you need"- " to specify Hugging Face model repo name instead of a local directory. For example:"- " 'HuggingFaceTB/SmolLM2-1.7B-Instruct'. Note: To access gated repo, set HF_TOKEN "- "environment variable to your Hugging Face token.",+ " Config and tokenizer files will still be downloaded. To use this feature, you need"+ " to specify Hugging Face model repo name instead of a local directory. For example:"+ " 'HuggingFaceTB/SmolLM2-1.7B-Instruct'. Note: To access gated repo, set HF_TOKEN "+ "environment variable to your Hugging Face token.",)args = parser.parse_args()@@ -6017,4 +6065,4 @@ def main() -> None:if __name__ == '__main__':- main()\ No newline at end of file+ main()diff --git a/auto_round/export/export_to_gguf/quant_gpu.py b/auto_round/export/export_to_gguf/packing.pysimilarity index 71%rename from auto_round/export/export_to_gguf/quant_gpu.pyrename to auto_round/export/export_to_gguf/packing.pyindex 99da6eb4e..a2b25e2d4 100644--- a/auto_round/export/export_to_gguf/quant_gpu.py+++ b/auto_round/export/export_to_gguf/packing.py@@ -15,13 +15,12 @@import torchimport numpy as np-from .utils import QK_K, K_SCALE_SIZE, GGML_QUANT_SIZES+from auto_round.export.export_to_gguf.config import QK_K, K_SCALE_SIZE, GGML_QUANT_SIZESGGML_QUANT_TYPE = {}def register_qtype(name):-def register(cls):GGML_QUANT_TYPE[name] = clsreturn cls@@ -29,7 +28,8 @@ def register_qtype(name):return register-def ggml_quant_gpu(data, ggml_type, scale=None, zp=None, wmin_m=None, d_scale=None, d_wmin_m=None, device="cuda"):+def ggml_quant_gpu(data, ggml_type, scale=None, zp=None, wmin_m=None, d_scale=None, d_wmin_m=None, imatrix=None,+ device="cuda"):block_size, type_size = GGML_QUANT_SIZES[ggml_type]data = data.to(torch.float32).to(device)@@ -42,39 +42,39 @@ def ggml_quant_gpu(data, ggml_type, scale=None, zp=None, wmin_m=None, d_scale=Noshape = data.shapen_blocks = data.nelement() // block_sizeblocks = data.reshape((n_blocks, block_size))-quant_func = GGML_QUANT_TYPE[ggml_type]- new_data = quant_func(blocks, scale, zp, wmin_m=wmin_m, d_scale=d_scale, d_wmin_m=d_wmin_m)+ new_data = quant_func(blocks, scale, zp, wmin_m=wmin_m, d_scale=d_scale, d_wmin_m=d_wmin_m, imatrix=imatrix)assert new_data.shape[-1] == type_sizenew_data = new_data.reshape(*shape[:-1], shape[-1] // block_size * type_size)+ new_data = new_data.reshape(*shape[:-1], -1)return new_datadef torch_roundf(n):a = torch.abs(n)floored = torch.floor(a)- b = floored + torch.floor(2 * (a-floored))+ b = floored + torch.floor(2 * (a - floored))return torch.sign(n) * b-def make_qx_quant(data, bits, rmse_type=0, qw = None):+def make_qx_quants(data, bits, rmse_type=0, qw=None):nmax = pow(2, bits - 1)imax = abs(data).argmax(axis=-1, keepdims=True)- group_max = torch.take_along_dim(data, imax, axis=-1)+ group_max = torch.take_along_dim(data, imax, dim=-1)iscales = torch.where(group_max != 0, -nmax / group_max, 0)if rmse_type == 0:- L = (torch.round(iscales * data).clip(-nmax, nmax-1) + nmax).to(torch.uint8)+ L = (torch.round(iscales * data).clip(-nmax, nmax - 1) + nmax).to(torch.uint8)scales = torch.where(iscales != 0, 1 / iscales, 0).reshape(iscales.shape[:2])return scales, L-+return_early = Falseif rmse_type < 0:rmse_type = -rmse_typereturn_early = True-- L = torch.round(iscales * data).clip(-nmax, nmax-1)++ L = torch.round(iscales * data).clip(-nmax, nmax - 1)if qw is not None:w = qwelif rmse_type == 1:@@ -85,9 +85,9 @@ def make_qx_quant(data, bits, rmse_type=0, qw = None):w = torch.abs(data)else:w = torch.sqrt(torch.abs(data))- sumlx = torch.sum(w * data * L, axis=-1)- suml2 = torch.sum(w * L * L, axis=-1)- scales = torch.where(suml2 !=0, sumlx / suml2, 0)+ sumlx = torch.sum(w * data * L, dim=-1)+ suml2 = torch.sum(w * L * L, dim=-1)+ scales = torch.where(suml2 != 0, sumlx / suml2, 0)if return_early:iscales_inv = torch.where(iscales != 0, 1 / iscales, 0).reshape(iscales.shape[:2])scales = torch.where(suml2 > 0, 0.5 * (scales + iscales_inv), iscales_inv)@@ -99,10 +99,10 @@ def make_qx_quant(data, bits, rmse_type=0, qw = None):continueiscales = torch.where(group_max != 0, -(nmax + -0.1 * _is) / nmax, 0)tmp_L = torch.round(iscales * data).clip(-nmax, nmax - 1)- sumlx = torch.sum(w * data * L, axis=-1)- suml2 = torch.sum(w * L * L, axis=-1)+ sumlx = torch.sum(w * data * L, dim=-1)+ suml2 = torch.sum(w * L * L, dim=-1)- replace_id = (suml2 > 0) & (sumlx*sumlx > best*suml2)+ replace_id = (suml2 > 0) & (sumlx * sumlx > best * suml2)L[replace_id] = tmp_L[replace_id]scales[replace_id] = sumlx[replace_id] / suml2[replace_id]best[replace_id] = scales[replace_id] * sumlx[replace_id]@@ -114,13 +114,13 @@ def make_qx_quant(data, bits, rmse_type=0, qw = None):def make_q3_quants(data, bits, do_rmse=False):nmax = pow(2, bits - 1)imax = abs(data).argmax(axis=-1, keepdims=True)- group_max = torch.take_along_dim(data, imax, axis=-1)+ group_max = torch.take_along_dim(data, imax, dim=-1)iscale = torch.where(group_max != 0, -nmax / group_max, 0)if do_rmse:L = torch.round(iscale * data).clip(-nmax, nmax - 1)w = torch.pow(data, 2)- sumlx = torch.sum(w * data * L, axis=-1)[0]- suml2 = torch.sum(w * L * L, axis=-1)[0]+ sumlx = torch.sum(w * data * L, dim=-1)[0]+ suml2 = torch.sum(w * L * L, dim=-1)[0]for itry in range(5):# w = np.power(data, 2)@@ -128,9 +128,9 @@ def make_q3_quants(data, bits, do_rmse=False):sumlx = sumlx.unsqueeze(-1)if len(suml2.shape) == 2:suml2 = suml2.unsqueeze(-1)- slx = sumlx - w*data*L+ slx = sumlx - w * data * Lrepalce_idx = slx > 0- sl2 = suml2 - w*L*L+ sl2 = suml2 - w * L * Lnew_L = torch.round(data * sl2 / slx).clip(-nmax, nmax - 1)tmp_repalce_idx = repalce_idx & (new_L != L)slx[tmp_repalce_idx] += w[tmp_repalce_idx] * data[tmp_repalce_idx] * new_L[tmp_repalce_idx]@@ -146,7 +146,7 @@ def make_q3_quants(data, bits, do_rmse=False):return scales, L.to(torch.uint8)-def make_qkx2_quants(data, bits, rmin=-1, rdelta=0.1, nstep=20, use_mad=False, weights=None):+def make_qkx2_quants(data, bits, weights=None, rmin=-1.0, rdelta=0.1, nstep=20, use_mad=False):nmax = pow(2, bits) - 1# data shape (nb, 8, 32) for Q4_K, (nb, 16, 16) for Q2_Kif len(data.shape) == 2:@@ -158,58 +158,58 @@ def make_qkx2_quants(data, bits, rmin=-1, rdelta=0.1, nstep=20, use_mad=False, wraise NotImplementedError(f"bits = {bits} is not supported")data = data.reshape(data_shape)if weights is None:- sum_x2 = torch.sum(torch.pow(data, 2), axis=-1, keepdims=True)+ sum_x2 = torch.sum(torch.pow(data, 2), dim=-1, keepdim=True)if bits == 2:av_x = 0else:av_x = torch.sqrt(sum_x2 / data.shape[-1])weights = torch.abs(data) + av_x- group_min = torch.min(data, axis=-1, keepdims=True)[0]- group_max = torch.max(data, axis=-1, keepdims=True)[0]+ group_min = torch.min(data, dim=-1, keepdim=True)[0]+ group_max = torch.max(data, dim=-1, keepdim=True)[0]the_mins = -group_min- sum_w = torch.sum(weights, axis=-1, keepdims=True)- sum_x = torch.sum(weights * data, axis=-1, keepdims=True)+ sum_w = torch.sum(weights, dim=-1, keepdim=True)+ sum_x = torch.sum(weights * data, dim=-1, keepdim=True)group_min[group_min > 0] = 0- scale = (group_max-group_min) / nmax+ scale = (group_max - group_min) / nmaxiscale = torch.where(scale == 0, 0, 1 / scale)- l_values = torch.round(iscale * (data-group_min))+ l_values = torch.round(iscale * (data - group_min))L = torch.clip(l_values, 0, nmax).to(torch.uint8)- diffs = scale*L + group_min - data- diffs = torch.abs(diffs) if use_mad else diffs**2- best_mad = torch.sum(weights * diffs, axis=-1, keepdims=True)+ diffs = scale * L + group_min - data+ diffs = torch.abs(diffs) if use_mad else diffs ** 2+ best_mad = torch.sum(weights * diffs, dim=-1, keepdim=True)if nstep < 1:return scale.reshape(scale.shape[:2]), L, the_mins.reshape(the_mins.shape[:2])for step in range(nstep):- new_scale = (group_max-group_min) / (rmin + rdelta*step + nmax)+ new_scale = (group_max - group_min) / (rmin + rdelta * step + nmax)new_iscale = torch.where(new_scale == 0, 0, 1 / new_scale)- l_values = torch.round(new_iscale * (data-group_min))+ l_values = torch.round(new_iscale * (data - group_min))Laux = torch.clip(l_values, 0, nmax).to(torch.uint8)- sum_l = torch.sum(weights * Laux, axis=-1, keepdims=True)- sum_l2 = torch.sum(weights * Laux**2, axis=-1, keepdims=True)- sum_xl = torch.sum(weights * Laux * data, axis=-1, keepdims=True)+ sum_l = torch.sum(weights * Laux, dim=-1, keepdim=True)+ sum_l2 = torch.sum(weights * Laux ** 2, dim=-1, keepdim=True)+ sum_xl = torch.sum(weights * Laux * data, dim=-1, keepdim=True)- D = sum_w*sum_l2 - sum_l*sum_l+ D = sum_w * sum_l2 - sum_l * sum_lreplace_idx = D > 0- this_scale = (sum_w*sum_xl - sum_x*sum_l) / D- this_min = (sum_l2*sum_x - sum_l*sum_xl) / D+ this_scale = (sum_w * sum_xl - sum_x * sum_l) / D+ this_min = (sum_l2 * sum_x - sum_l * sum_xl) / Dthis_min[this_min > 0] = 0this_scale[this_min > 0] = (sum_xl / sum_l2)[this_min > 0]- diffs = this_scale*Laux + this_min - data- diffs = torch.abs(diffs) if use_mad else diffs**2- mad = torch.sum(weights * diffs, axis=-1, keepdims=True)+ diffs = this_scale * Laux + this_min - data+ diffs = torch.abs(diffs) if use_mad else diffs ** 2+ mad = torch.sum(weights * diffs, dim=-1, keepdim=True)replace_idx &= mad < best_madbest_mad[replace_idx] = mad[replace_idx]@@ -221,6 +221,59 @@ def make_qkx2_quants(data, bits, rmin=-1, rdelta=0.1, nstep=20, use_mad=False, wreturn scale.reshape(scale.shape[:2]), L, the_mins.reshape(the_mins.shape[:2])+def make_qkx3_quants(data, bits, weights, rmin=-1.0, rdelta=0.1, nstep=20, use_mad=False):+ return make_qkx2_quants(data, bits, weights, rmin=rmin, rdelta=rdelta, nstep=nstep, use_mad=use_mad)+++def make_qp_quants(nmax, data, quant_weights):+ group_max = torch.max(data, dim=-1, keepdim=True)[0]+ scale = group_max / nmax+ iscale = torch.where(scale == 0, 0, 1 / scale)++ L = torch.round(iscale * data)+ diffs = data - scale * L+ best_mse = torch.sum(quant_weights * diffs * diffs)++ for _is in range(-4, 5):+ if _is == 0:+ continue+ scale_is = group_max / (0.1 * _is + nmax)+ iscale_is = torch.where(scale == 0, 0, 1 / scale_is)++ tmp_L = torch.round(iscale_is * data).clip(max=nmax)+ diffs = data - scale_is * tmp_L+ mse = torch.sum(quant_weights * diffs * diffs)++ replace_idx = mse < best_mse+ best_mse[replace_idx] = mse[replace_idx]+ iscale[replace_idx] = iscale_is[replace_idx]++ L = torch.round(iscale * data).clip(max=nmax)+ sumlx = torch.sum(quant_weights * data * L, dim=-1)+ suml2 = torch.sum(quant_weights * L * L, dim=-1)++ for _ in range(5):+ n_changed = 0+ for i in range(data.shape[-1]):+ slx = sumlx - quant_weights[:, i] * data[:, i] * L[:, i]+ sl2 = suml2 - quant_weights[:, i] * L[:, i] * L[:, i]+ replace_idx = (slx > 0) & (sl2 > 0)+ new_L = torch.round(data[:, i] * sl2 / slx).clip(max=nmax)+ replace_idx &= new_L != L[:, i]+ slx[replace_idx] += quant_weights[:, i][replace_idx] * data[:, i][replace_idx] * new_L[replace_idx]+ sl2[replace_idx] += quant_weights[:, i][replace_idx] * new_L[replace_idx] * new_L[replace_idx]++ replace_idx &= slx * slx * suml2 > sumlx * sumlx * sl2+ L[:, i][replace_idx] = new_L[replace_idx]+ sumlx[replace_idx] = slx[replace_idx]+ suml2[replace_idx] = sl2[replace_idx]+ n_changed = replace_idx.sum()+ if n_changed == 0:+ break++ return sumlx / suml2, L++@register_qtype("bf16")def bf16_quant_block(blocks, scale=None, zp=None, **kwargs):n = blocks.view(torch.uint32)@@ -241,7 +294,7 @@ def q4_0_quant_block(blocks, scale=None, zp=None, **kwargs):d = max / -8id = torch.where(d == 0, 0, 1 / d)- qs = torch.trunc(blocks.to(torch.float64) * id.to(torch.float64) + 8.5).to(torch.uint8).clip(0, 15)+ qs = torch.trunc(blocks.to(torch.float64) * id.to(torch.float64) + 8.5).clip(0, 15).to(torch.uint8)n_blocks = blocks.shape[0]block_size = GGML_QUANT_SIZES["q4_0"][0]@@ -261,10 +314,10 @@ def q4_1_quant_block(blocks, scale=None, zp=None, **kwargs):else:max = blocks.max(axis=-1, keepdims=True)[0]min = blocks.min(axis=-1, keepdims=True)[0]- d = (max-min) / 15+ d = (max - min) / 15id = torch.where(d == 0, 0, 1 / d)- qs = torch.trunc((blocks-min) * id + 0.5).to(torch.uint8).clip(0, 15)+ qs = torch.trunc((blocks - min) * id + 0.5).clip(0, 15).to(torch.uint8)n_blocks = blocks.shape[0]block_size = GGML_QUANT_SIZES["q4_1"][0]@@ -282,7 +335,7 @@ def q5_0_quant_block(blocks: np.array, scale=None, zp=None, **kwargs):d = scale.reshape((-1, 1))else:imax = abs(blocks).argmax(axis=-1, keepdims=True)- max = torch.take_along_dim(blocks, imax, axis=-1)[0]+ max = torch.take_along_dim(blocks, imax, dim=-1)[0]d = max / -16id = torch.where(d == 0, 0, 1 / d)@@ -290,7 +343,7 @@ def q5_0_quant_block(blocks: np.array, scale=None, zp=None, **kwargs):block_size = GGML_QUANT_SIZES["q5_0"][0]# FIXME: Q5_0's reference rounding is cursed and depends on FMA- q = torch.trunc(blocks.to(torch.float64) * id.to(torch.float64) + 16.5).to(torch.uint8).clip(0, 31).cpu().numpy()+ q = torch.trunc(blocks.to(torch.float64) * id.to(torch.float64) + 16.5).clip(0, 31).to(torch.uint8).cpu().numpy()qs = q.reshape((n_blocks, 2, block_size // 2))qs = (qs[..., 0, :] & np.uint8(0x0F)) | (qs[..., 1, :] << np.uint8(4))@@ -310,13 +363,13 @@ def q5_1_quant_block(blocks: np.array, scale=None, zp=None, **kwargs):else:max = blocks.max(axis=-1, keepdims=True)[0]min = blocks.min(axis=-1, keepdims=True)[0]- d = (max-min) / 31+ d = (max - min) / 31n_blocks = blocks.shape[0]block_size = GGML_QUANT_SIZES["q5_1"][0]id = torch.where(d == 0, 0, 1 / d)- q = torch.trunc((blocks-min) * id + 0.5).to(torch.uint8).clip(0, 31).cpu().numpy()+ q = torch.trunc((blocks - min) * id + 0.5).clip(0, 31).to(torch.uint8).cpu().numpy()qs = q.reshape((n_blocks, 2, block_size // 2))qs = (qs[..., 0, :] & np.uint8(0x0F)) | (qs[..., 1, :] << np.uint8(4))@@ -334,10 +387,10 @@ def q8_0_quant_block(blocks, scale=None, zp=None, **kwargs) -> np.ndarray:if scale is not None:d = scale.reshape((-1, 1))else:- d = torch.abs(blocks).max(axis=1, keepdims=True)[0] / 127+ d = torch.abs(blocks).max(dim=1, keepdim=True)[0] / 127id = torch.where(d == 0, 0, 1 / d)- qs = torch_roundf(blocks * id)+ qs = torch.clip(torch_roundf(blocks * id), -128, 127)# (n_blocks, 2)d = d.cpu().numpy().astype(np.float16).view(np.uint8)@@ -348,10 +401,8 @@ def q8_0_quant_block(blocks, scale=None, zp=None, **kwargs) -> np.ndarray:@register_qtype("q2_k")-def q2_k_quant_block(blocks, scale=None, zp=None, wmin_m=None, d_scale=None, d_wmin_m=None):+def q2_k_quant_block(blocks, scale=None, zp=None, wmin_m=None, d_scale=None, d_wmin_m=None, **kwargs):nb = blocks.shape[0]- output_scale = np.empty((nb, 16), dtype=np.uint8)- output_qs = np.empty((nb, QK_K // 16 // 4, 16), dtype=np.uint8)blocks = blocks.reshape((nb, QK_K // 16, 16)) # (nb, 16, 16)@@ -362,23 +413,25 @@ def q2_k_quant_block(blocks, scale=None, zp=None, wmin_m=None, d_scale=None, d_woutput_dmin = d_wmin_m.reshape(-1, 1).to(torch.float32)inv_scales = torch.where(output_d == 0, 0, 1 / output_d)inv_mins = torch.where(d_wmin_m == 0, 0, 1 / output_dmin)- max_scales = torch.max(scales, axis=-1, keepdims=True)[0] # (nb, 1)- max_mins = torch.max(mins, axis=-1, keepdims=True)[0] # (nb, 1)- all_L = torch.round((blocks + mins.reshape(*mins.shape, 1)) / scales.reshape(*scales.shape, 1)).to(torch.uint8)+ max_scales = torch.max(scales, dim=-1, keepdim=True)[0] # (nb, 1)+ max_mins = torch.max(mins, dim=-1, keepdim=True)[0] # (nb, 1)+ tmp_inverse_scale = torch.where(scales == 0, 0, 1.0 / scales)+ all_L = torch.clip(+ torch.round((blocks + mins.reshape(*mins.shape, 1)) / tmp_inverse_scale.reshape(*scales.shape, 1)), 0,+ 3).to(torch.uint8)else:scales, all_L, mins = make_qkx2_quants(blocks, bits=2, rmin=-0.5, rdelta=0.1, nstep=15, use_mad=True)- max_scales = torch.max(scales, axis=-1, keepdims=True)[0]- max_mins = torch.max(mins, axis=-1, keepdims=True)[0]+ max_scales = torch.max(scales, dim=-1, keepdim=True)[0]+ max_mins = torch.max(mins, dim=-1, keepdim=True)[0]inv_scales = torch.where(max_scales > 0, 15. / max_scales, 0)inv_mins = torch.where(max_mins > 0, 15. / max_mins, 0)replace_ids = (max_scales > 0).squeeze()output_scale = torch.zeros_like(scales).to(torch.uint8)- output_scale[replace_ids] = torch.round(inv_scales * scales).to(torch.uint8)+ output_scale[replace_ids] = torch.round(inv_scales * scales).clip(0, 15).to(torch.uint8)replace_ids = (max_mins > 0).squeeze()- output_scale[replace_ids] |= torch.round(inv_mins * mins).to(torch.uint8) << 4-+ output_scale[replace_ids] |= torch.round(inv_mins * mins).clip(0, 15).to(torch.uint8) << 4if d_scale is None:output_d = torch.where(max_scales > 0, max_scales / 15, 0)if d_wmin_m is None:@@ -389,11 +442,13 @@ def q2_k_quant_block(blocks, scale=None, zp=None, wmin_m=None, d_scale=None, d_wreplace_ids = d_tmp != 0all_L[replace_ids] = torch.round(- (blocks[replace_ids] + dm_tmp[replace_ids].unsqueeze(-1)) / d_tmp[replace_ids].unsqueeze(-1)).to(torch.uint8)- all_L = np.clip(all_L.cpu().numpy().astype(np.uint8), 0, 3)+ (blocks[replace_ids] + dm_tmp[replace_ids].unsqueeze(-1)) / d_tmp[replace_ids].unsqueeze(-1)).clip(0, 3).to(+ torch.uint8)+ all_L = np.clip(all_L.cpu().numpy(), 0, 3).astype(np.uint8)output_scale = output_scale.cpu().numpy()- output_qs = all_L[:, ::4] | (all_L[:, 1::4] << 2) | (all_L[:, 2::4] << 4) | (all_L[:, 3::4] << 6)+ all_L = all_L.reshape(-1, 4, 32)+ output_qs = all_L[:, 0, :] | (all_L[:, 1, :] << 2) | (all_L[:, 2, :] << 4) | (all_L[:, 3, :] << 6)output_d = output_d.cpu().numpy()output_d = output_d.reshape(-1, 1).astype(np.float16).view(np.uint8)output_dmin = output_dmin.cpu().numpy()@@ -405,7 +460,7 @@ def q2_k_quant_block(blocks, scale=None, zp=None, wmin_m=None, d_scale=None, d_w@register_qtype("q3_k")-def q3_k_quant_block(blocks: np.array, scale=None, zp=None, wmin_m=None, d_scale=None, d_wmin_m=None):+def q3_k_quant_block(blocks: np.array, scale=None, zp=None, wmin_m=None, d_scale=None, d_wmin_m=None, **kwargs):nb = blocks.shape[0]blocks = blocks.reshape(nb, QK_K // 16, 16)@@ -419,8 +474,8 @@ def q3_k_quant_block(blocks: np.array, scale=None, zp=None, wmin_m=None, d_scaleall_L = all_L.to(torch.uint8)else:scales, all_L = make_q3_quants(blocks, bits=3, do_rmse=True)- imax = abs(scales).argmax(axis=-1, keepdims=True)- max_scales = torch.take_along_dim(scales, imax, axis=-1)+ imax = abs(scales).argmax(dim=-1, keepdim=True)+ max_scales = torch.take_along_dim(scales, imax, dim=-1)iscales = torch.where(max_scales != 0, -32 / max_scales, 0)output_d = torch.where(iscales != 0, 1 / iscales, 0)@@ -439,11 +494,11 @@ def q3_k_quant_block(blocks: np.array, scale=None, zp=None, wmin_m=None, d_scaleall_L = all_L.cpu().numpy()output_hmask = np.bitwise_or.reduce(- all_L.reshape(nb, 8, 32) >> 2 << np.arange(8, dtype=np.uint8).reshape(1, 8, 1), axis=1) # pylint: disable=E1121+ all_L.reshape(nb, 8, 32) >> 2 << np.arange(8, dtype=np.uint8).reshape(1, 8, 1), axis=1) # pylint: disable=E1121all_L = np.where(all_L > 3, all_L - 4, all_L)output_qs = np.bitwise_or.reduce(- all_L.reshape(nb, 2, 4, 32) << np.array([0, 2, 4, 6]).reshape(1, 1, 4, 1), axis=2) # pylint: disable=E1121+ all_L.reshape(nb, 2, 4, 32) << np.array([0, 2, 4, 6]).reshape(1, 1, 4, 1), axis=2) # pylint: disable=E1121output_qs = output_qs.reshape(nb, 64).astype(np.uint8)output_d = output_d.cpu().numpy().reshape(-1, 1).astype(np.float16).view(np.uint8)@@ -452,7 +507,7 @@ def q3_k_quant_block(blocks: np.array, scale=None, zp=None, wmin_m=None, d_scale@register_qtype("q4_k")-def q4_k_quant_block(blocks, scale=None, zp=None, wmin_m=None, d_scale=None, d_wmin_m=None):+def q4_k_quant_block(blocks, scale=None, zp=None, wmin_m=None, d_scale=None, d_wmin_m=None, **kwargs):nb = blocks.shape[0]blocks = blocks.reshape((nb, QK_K // 32, 32))@@ -466,18 +521,19 @@ def q4_k_quant_block(blocks, scale=None, zp=None, wmin_m=None, d_scale=None, d_woutput_dmin = d_wmin_m.reshape(-1, 1).to(torch.float32)inv_scales = torch.where(output_d == 0, 0, 1 / output_d)inv_mins = torch.where(d_wmin_m == 0, 0, 1 / output_dmin)- max_scales = torch.max(scales, axis=-1, keepdims=True)[0] # (nb, 1)- max_mins = torch.max(mins, axis=-1, keepdims=True)[0] # (nb, 1)- all_L = torch.round((blocks + mins.reshape(*mins.shape, 1)) / scales.reshape(*scales.shape, 1)).to(torch.uint8)+ max_scales = torch.max(scales, dim=-1, keepdim=True)[0] # (nb, 1)+ max_mins = torch.max(mins, dim=-1, keepdim=True)[0] # (nb, 1)+ all_L = torch.round((blocks + mins.reshape(*mins.shape, 1)) / scales.reshape(*scales.shape, 1)).clip(0, 15).to(+ torch.uint8)else:scales, all_L, mins = make_qkx2_quants(blocks, bits=4, rmin=-1, rdelta=0.1, nstep=20, use_mad=False)- max_scales = torch.max(scales, axis=-1, keepdims=True)[0]- max_mins = torch.max(mins, axis=-1, keepdims=True)[0]+ max_scales = torch.max(scales, dim=-1, keepdim=True)[0]+ max_mins = torch.max(mins, dim=-1, keepdim=True)[0]inv_scales = torch.where(max_scales > 0, 63. / max_scales, 0)inv_mins = torch.where(max_mins > 0, 63. / max_mins, 0)- q_scales = torch.round(inv_scales * scales)- q_mins = torch.round(inv_mins * mins)+ q_scales = torch.round(inv_scales * scales).clip(0, 63)+ q_mins = torch.round(inv_mins * mins).clip(0, 63)if d_scale is None:output_d = max_scales / 63@@ -486,7 +542,6 @@ def q4_k_quant_block(blocks, scale=None, zp=None, wmin_m=None, d_scale=None, d_wd_tmp = output_d * q_scalesdm_tmp = output_dmin * q_mins-q_scales = q_scales.cpu().numpy().astype(np.uint8)q_mins = q_mins.cpu().numpy().astype(np.uint8)output_scale[:, :4] = q_scales[:, :4]@@ -498,8 +553,9 @@ def q4_k_quant_block(blocks, scale=None, zp=None, wmin_m=None, d_scale=None, d_wreplace_ids = d_tmp != 0all_L[replace_ids] = torch.round(- (blocks[replace_ids] + dm_tmp[replace_ids].unsqueeze(-1)) / d_tmp[replace_ids].unsqueeze(-1)).to(torch.uint8)- all_L = np.clip(all_L.cpu().numpy().astype(np.uint8), 0, 15)+ (blocks[replace_ids] + dm_tmp[replace_ids].unsqueeze(-1)) / d_tmp[replace_ids].unsqueeze(-1)).clip(0, 15).to(+ torch.uint8)+ all_L = np.clip(all_L.cpu().numpy(), 0, 15).astype(np.uint8)output_qs = all_L[:, ::2] | (all_L[:, 1::2] << 4)output_d = output_d.cpu().numpy()@@ -512,15 +568,13 @@ def q4_k_quant_block(blocks, scale=None, zp=None, wmin_m=None, d_scale=None, d_wreturn np.concatenate([output_d, output_dmin, output_scale, output_qs], axis=-1)+@register_qtype("q5_k")-def q5_k_quant_block(blocks, scale=None, zp=None, wmin_m=None, d_scale=None, d_wmin_m=None):+def q5_k_quant_block(blocks, scale=None, zp=None, wmin_m=None, d_scale=None, d_wmin_m=None, **kwargs):nb = blocks.shape[0]blocks = blocks.reshape((nb, QK_K // 32, 32))output_scale = np.empty((nb, K_SCALE_SIZE), dtype=np.uint8)- output_qs = np.empty((nb, QK_K // 64, 32), dtype=np.uint8)- output_qh = np.empty((nb, QK_K // 8), dtype=np.uint8)-if scale is not None:scales = scale.reshape(-1, QK_K // 32)mins = wmin_m.reshape(-1, QK_K // 32)@@ -528,13 +582,14 @@ def q5_k_quant_block(blocks, scale=None, zp=None, wmin_m=None, d_scale=None, d_woutput_dmin = d_wmin_m.reshape(-1, 1).to(torch.float32)inv_scales = torch.where(output_d == 0, 0, 1 / output_d)inv_mins = torch.where(d_wmin_m == 0, 0, 1 / output_dmin)- max_scales = torch.max(scales, axis=-1, keepdims=True)[0] # (nb, 1)- max_mins = torch.max(mins, axis=-1, keepdims=True)[0] # (nb, 1)- all_L = torch.round((blocks + mins.reshape(*mins.shape, 1)) / scales.reshape(*scales.shape, 1)).to(torch.uint8)+ max_scales = torch.max(scales, dim=-1, keepdim=True)[0] # (nb, 1)+ max_mins = torch.max(mins, dim=-1, keepdim=True)[0] # (nb, 1)+ all_L = torch.round((blocks + mins.reshape(*mins.shape, 1)) / scales.reshape(*scales.shape, 1)).clip(0, 31).to(+ torch.uint8)else:scales, all_L, mins = make_qkx2_quants(blocks, bits=5, rmin=-0.5, rdelta=0.1, nstep=15, use_mad=False)- max_scales = torch.max(scales, axis=-1, keepdims=True)[0]- max_mins = torch.max(mins, axis=-1, keepdims=True)[0]+ max_scales = torch.max(scales, dim=-1, keepdim=True)[0]+ max_mins = torch.max(mins, dim=-1, keepdim=True)[0]inv_scales = torch.where(max_scales > 0, 63. / max_scales, 0)inv_mins = torch.where(max_mins > 0, 63. / max_mins, 0)@@ -560,12 +615,13 @@ def q5_k_quant_block(blocks, scale=None, zp=None, wmin_m=None, d_scale=None, d_wreplace_ids = d_tmp != 0all_L[replace_ids] = torch.round(- (blocks[replace_ids] + dm_tmp[replace_ids].unsqueeze(-1)) / d_tmp[replace_ids].unsqueeze(-1)).to(torch.uint8)+ (blocks[replace_ids] + dm_tmp[replace_ids].unsqueeze(-1)) \+ / d_tmp[replace_ids].unsqueeze(-1)).clip(0, 31).to(torch.uint8)all_L = np.clip(all_L.cpu().numpy().astype(np.uint8), 0, 31) # (nb, 8, 32)output_qs = all_L[:, ::2] | (all_L[:, 1::2] << 4)output_qh = np.bitwise_or.reduce(- all_L >> 4 << np.arange(8, dtype=np.uint8).reshape(1, 8, 1), axis=1).astype(np.uint8) # pylint: disable=E1121+ all_L >> 4 << np.arange(8, dtype=np.uint8).reshape(1, 8, 1), axis=1).astype(np.uint8) # pylint: disable=E1121output_d = output_d.cpu().numpy()output_d = output_d.reshape(-1, 1).astype(np.float16).view(np.uint8)@@ -577,7 +633,7 @@ def q5_k_quant_block(blocks, scale=None, zp=None, wmin_m=None, d_scale=None, d_w@register_qtype("q6_k")-def q6_k_quant_block(blocks: np.array, scale=None, zp=None, wmin_m=None, d_scale=None, d_wmin_m=None):+def q6_k_quant_block(blocks: np.array, scale=None, zp=None, wmin_m=None, d_scale=None, d_wmin_m=None, **kwargs):nb = blocks.shape[0]blocks = blocks.reshape((nb, QK_K // 16, 16))@@ -586,11 +642,11 @@ def q6_k_quant_block(blocks: np.array, scale=None, zp=None, wmin_m=None, d_scalescales = scale.reshape(-1, QK_K // 16)output_d = d_scale.reshape(-1, 1).to(torch.float32)iscales = torch.where(output_d == 0, 0, 1 / output_d)- all_L = (torch.round(blocks / scales.reshape(*scales.shape, 1)).clip(-32, 31) + 32).to(torch.uint8)+ all_L = torch.round(blocks / scales.reshape(*scales.shape, 1)).clip(0, 63).to(torch.uint8)else:- scales, all_L = make_qx_quant(blocks, bits=6, rmse_type=1, qw=None)- imax = abs(scales).argmax(axis=-1, keepdims=True)- max_scales = torch.take_along_dim(scales, imax, axis=-1)+ scales, all_L = make_qx_quants(blocks, bits=6, rmse_type=1, qw=None)+ imax = abs(scales).argmax(dim=-1, keepdim=True)+ max_scales = torch.take_along_dim(scales, imax, dim=-1)iscales = torch.where(max_scales != 0, -128 / max_scales, 0)output_d = torch.where(iscales != 0, 1 / iscales, -1)@@ -604,11 +660,11 @@ def q6_k_quant_block(blocks: np.array, scale=None, zp=None, wmin_m=None, d_scaletmp_L = all_L.reshape(nb, 4, 64) & 0xFoutput_ql = (tmp_L[:, ::2] | (tmp_L[:, 1::2] << 4)).reshape(nb, QK_K // 2)output_qh = np.bitwise_or.reduce(- (all_L >> 4).reshape(nb, 2, 4, 32) << np.array([0, 2, 4, 6]).reshape(1, 1, 4, 1), # pylint: disable=E1121- axis=2).reshape(nb, QK_K // 4).astype(np.uint8) # pylint: disable=E1121+ (all_L >> 4).reshape(nb, 2, 4, 32) << np.array([0, 2, 4, 6]).reshape(1, 1, 4, 1), # pylint: disable=E1121+ axis=2).reshape(nb, QK_K // 4).astype(np.uint8) # pylint: disable=E1121output_d = output_d.cpu().numpy().reshape(-1, 1).astype(np.float16).view(np.uint8)# [ql, qh, scale, d]output_scale = output_scale.cpu().numpy().view(np.uint8)- return np.concatenate([output_ql, output_qh, output_scale, output_d], axis=-1)\ No newline at end of file+ return np.concatenate([output_ql, output_qh, output_scale, output_d], axis=-1)diff --git a/auto_round/export/export_to_gguf/quant_cpu.py b/auto_round/export/export_to_gguf/quant_cpu.pydeleted file mode 100644index f4e097b35..000000000--- a/auto_round/export/export_to_gguf/quant_cpu.py+++ /dev/null@@ -1,618 +0,0 @@-# Copyright (c) 2024 Intel Corporation-#-# Licensed under the Apache License, Version 2.0 (the "License");-# you may not use this file except in compliance with the License.-# You may obtain a copy of the License at-#-# http://www.apache.org/licenses/LICENSE-2.0-#-# Unless required by applicable law or agreed to in writing, software-# distributed under the License is distributed on an "AS IS" BASIS,-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.-# See the License for the specific language governing permissions and-# limitations under the License.--import numpy as np-from concurrent.futures import ProcessPoolExecutor-from .utils import QK_K, K_SCALE_SIZE, GGML_QUANT_SIZES--GGML_QUANT_TYPE = {}---def register_qtype(name):- def register(cls):- GGML_QUANT_TYPE[name] = cls- return cls-- return register---def ggml_quant_cpu(data, ggml_type, scale=None, zp=None, wmin_m=None, d_scale=None, d_wmin_m=None):- import torch- data = data.squeeze().cpu().numpy() if isinstance(data, torch.Tensor) else data- scale = scale.numpy() if isinstance(scale, torch.Tensor) else scale- zp = zp.numpy() if isinstance(zp, torch.Tensor) else zp- wmin_m = wmin_m.numpy() if isinstance(wmin_m, torch.Tensor) else wmin_m- d_scale = d_scale.numpy() if isinstance(d_scale, torch.Tensor) else d_scale- d_wmin_m = d_wmin_m.numpy() if isinstance(d_wmin_m, torch.Tensor) else d_wmin_m-- block_size, type_size = GGML_QUANT_SIZES[ggml_type]-- data = data.astype(np.float32, copy=False)- shape = data.shape- n_blocks = data.size // block_size- blocks = data.reshape((n_blocks, block_size))-- quant_func = GGML_QUANT_TYPE[ggml_type]- new_data = quant_func(blocks, scale, zp, wmin_m=wmin_m, d_scale=d_scale, d_wmin_m=d_wmin_m)-- assert new_data.dtype == np.uint8- assert new_data.shape[-1] == type_size- new_data = new_data.reshape(*shape[:-1], shape[-1] // block_size * type_size)- return new_data---def np_roundf(n: np.ndarray) -> np.ndarray:- a = abs(n)- floored = np.floor(a)- b = floored + np.floor(2 * (a-floored))- return np.sign(n) * b---def make_qx_quant(data, bits, rmse_type=0, qw = None):- nmax = pow(2, bits - 1)- imax = abs(data).argmax(axis=-1, keepdims=True)- group_max = np.take_along_axis(data, imax, axis=-1)- iscales = np.where(group_max != 0, -nmax / group_max, 0)-- if rmse_type == 0:- L = (np.round(iscales * data).clip(-nmax, nmax-1) + nmax).astype(np.uint8)- scales = np.where(iscales != 0, 1 / iscales, 0).reshape(iscales.shape[:2])- return scales, L-- return_early = False- if rmse_type < 0:- rmse_type = -rmse_type- return_early = True-- L = np.round(iscales * data).clip(-nmax, nmax-1)- if qw is not None:- w = qw- elif rmse_type == 1:- w = data ** 2- elif rmse_type == 2:- w = 1- elif rmse_type == 3:- w = np.fabs(data)- else:- w = np.sqrt(np.fabs(data))- sumlx = np.sum(w * data * L, axis=-1)- suml2 = np.sum(w * L * L, axis=-1)- scales = np.where(suml2 !=0, sumlx / suml2, 0)- if return_early:- iscales_inv = np.where(iscales != 0, 1 / iscales, 0).reshape(iscales.shape[:2])- scales = np.where(suml2 > 0, 0.5 * (scales + iscales_inv), iscales_inv)- L = (L + nmax).astype(np.uint8)- return scales, L- best = scales * sumlx- for _is in range(-9, 10):- if _is == 0:- continue- iscales = np.where(group_max != 0, -(nmax + -0.1 * _is) / nmax, 0)- tmp_L = np.round(iscales * data).clip(-nmax, nmax - 1)- sumlx = np.sum(w * data * L, axis=-1)- suml2 = np.sum(w * L * L, axis=-1)-- replace_id = (suml2 > 0) & (sumlx*sumlx > best*suml2)- L[replace_id] = tmp_L[replace_id]- scales[replace_id] = sumlx[replace_id] / suml2[replace_id]- best[replace_id] = scales[replace_id] * sumlx[replace_id]-- L = (L + nmax).astype(np.uint8)- return scales, L---def make_q3_quants(data, bits, do_rmse=False):- # data shape (nb, 16, 16)- nmax = pow(2, bits - 1)- imax = abs(data).argmax(axis=-1, keepdims=True)- group_max = np.take_along_axis(data, imax, axis=-1)- iscale = np.where(group_max != 0, -nmax / group_max, 0)- if do_rmse:- L = np.round(iscale * data).clip(-nmax, nmax - 1)- w = np.power(data, 2)- sumlx = np.sum(w * data * L, axis=-1)- suml2 = np.sum(w * L * L, axis=-1)-- for itry in range(5):- # w = np.power(data, 2)- if len(sumlx.shape) == 2:- sumlx = sumlx.reshape(*sumlx.shape, 1)- if len(suml2.shape) == 2:- suml2 = suml2.reshape(*suml2.shape, 1)- slx = sumlx - w*data*L- repalce_idx = slx > 0- sl2 = suml2 - w*L*L- tmp_L = np.where(slx > 0, data * sl2 / slx, 0)- new_L = np.round(tmp_L).clip(-nmax, nmax - 1)- tmp_repalce_idx = repalce_idx & (new_L != L)- slx[tmp_repalce_idx] += w[tmp_repalce_idx] * data[tmp_repalce_idx] * new_L[tmp_repalce_idx]- sl2[tmp_repalce_idx] += w[tmp_repalce_idx] * new_L[tmp_repalce_idx] * new_L[tmp_repalce_idx]- repalce_idx &= (sl2 > 0) & (slx * slx * suml2 > sumlx * sumlx * sl2)-- L[repalce_idx] = new_L[repalce_idx]- sumlx = slx- suml2 = sl2-- L = np.round(iscale * data).clip(-nmax, nmax - 1) + nmax- L = L.astype(np.uint8)- scales = np.where(iscale != 0, 1 / iscale, 0).reshape(iscale.shape[:2])- return scales, L---def make_qkx2_quants(data, bits, rmin=-1, rdelta=0.1, nstep=20, use_mad=False, weights=None):- # data shape (nb, 8, 32) for Q4_K, (nb, 16, 16) for Q2_K- nmax = pow(2, bits) - 1- if len(data.shape) == 2:- if bits in [4, 5]:- data_shape = (-1, 8, 32)- elif bits in [2]:- data_shape = (-1, 16, 16)- else:- raise NotImplementedError(f"bits = {bits} is not supported")- data = data.reshape(data_shape)- if weights is None:- sum_x2 = np.sum(np.pow(data, 2), axis=-1, keepdims=True)- if bits == 2:- av_x = 0- else:- av_x = np.sqrt(sum_x2 / data.shape[-1])- weights = np.abs(data) + av_x- group_min = np.min(data, axis=-1, keepdims=True)- group_max = np.max(data, axis=-1, keepdims=True)-- the_mins = -group_min-- sum_w = np.sum(weights, axis=-1, keepdims=True)- sum_x = np.sum(weights * data, axis=-1, keepdims=True)-- group_min[group_min > 0] = 0-- scale = (group_max-group_min) / nmax- iscale = np.where(scale == 0, 0, 1 / scale)-- l_values = np.round(iscale * (data-group_min))- L = np.clip(l_values, 0, nmax).astype(np.uint8)-- diffs = scale*L + group_min - data- diffs = np.abs(diffs) if use_mad else diffs**2- best_mad = np.sum(weights * diffs, axis=-1, keepdims=True)-- if nstep < 1:- return scale.reshape(scale.shape[:2]), L, the_mins.reshape(the_mins.shape[:2])-- for step in range(nstep):- new_scale = (group_max-group_min) / (rmin + rdelta*step + nmax)- new_iscale = np.where(new_scale == 0, 0, 1 / new_scale)-- l_values = np.round(new_iscale * (data-group_min))- Laux = np.clip(l_values, 0, nmax).astype(np.uint8)-- sum_l = np.sum(weights * Laux, axis=-1, keepdims=True)- sum_l2 = np.sum(weights * Laux**2, axis=-1, keepdims=True)- sum_xl = np.sum(weights * Laux * data, axis=-1, keepdims=True)-- D = sum_w*sum_l2 - sum_l*sum_l- replace_idx = D > 0-- this_scale = (sum_w*sum_xl - sum_x*sum_l) / D- this_min = (sum_l2*sum_x - sum_l*sum_xl) / D- this_min[this_min > 0] = 0- this_scale[this_min > 0] = (sum_xl / sum_l2)[this_min > 0]-- diffs = this_scale*Laux + this_min - data- diffs = np.abs(diffs) if use_mad else diffs**2- mad = np.sum(weights * diffs, axis=-1, keepdims=True)-- replace_idx &= mad < best_mad- best_mad[replace_idx] = mad[replace_idx]- L[replace_idx.reshape(replace_idx.shape[:2])] = Laux[replace_idx.reshape(replace_idx.shape[:2])]- scale[replace_idx] = this_scale[replace_idx]- group_min[replace_idx] = this_min[replace_idx]-- the_mins = -group_min- return scale.reshape(scale.shape[:2]), L, the_mins.reshape(the_mins.shape[:2])---@register_qtype("bf16")-def bf16_quant_block(blocks: np.array, scale=None, zp=None, **kwargs):- n = blocks.view(np.uint32)- # force nan to quiet- n = np.where((n & 0x7fffffff) > 0x7f800000, (n & np.uint32(0xffff0000)) | np.uint32(64 << 16), n)- # round to nearest even- n = (np.uint64(n) + (0x7fff + ((n >> 16) & 1))) >> 16- return n.astype(np.uint16).view(np.uint8)---@register_qtype("q4_0")-def q4_0_quant_block(blocks: np.array, scale=None, zp=None, **kwargs):- if scale is not None:- d = scale.reshape((-1, 1))- else:- imax = abs(blocks).argmax(axis=-1, keepdims=True)- max = np.take_along_axis(blocks, imax, axis=-1)- d = max / -8- with np.errstate(divide="ignore"):- id = np.where(d == 0, 0, 1 / d)-- qs = np.trunc((np.float64(blocks) * np.float64(id)) + np.float64(8.5),- dtype=np.float32).astype(np.uint8).clip(0, 15)-- n_blocks = blocks.shape[0]- block_size = GGML_QUANT_SIZES["q4_0"][0]- qs = qs.reshape((n_blocks, 2, block_size // 2))- qs = qs[..., 0, :] | (qs[..., 1, :] << np.uint8(4))-- d = d.astype(np.float16).view(np.uint8)-- return np.concatenate([d, qs], axis=-1)---@register_qtype("q4_1")-def q4_1_quant_block(blocks: np.array, scale=None, zp=None, **kwargs):- if scale is not None:- d = scale.reshape((-1, 1))- min = zp.reshape((-1, 1)) * d * -1- else:- max = blocks.max(axis=-1, keepdims=True)- min = blocks.min(axis=-1, keepdims=True)- d = (max-min) / 15- with np.errstate(divide="ignore"):- id = np.where(d == 0, 0, 1 / d)-- qs = np.trunc((blocks-min) * id + np.float32(0.5), dtype=np.float32).astype(np.uint8).clip(0, 15)-- n_blocks = blocks.shape[0]- block_size = GGML_QUANT_SIZES["q4_1"][0]- qs = qs.reshape((n_blocks, 2, block_size // 2))- qs = qs[..., 0, :] | (qs[..., 1, :] << np.uint8(4))-- d = d.astype(np.float16).view(np.uint8)- m = min.astype(np.float16).view(np.uint8)- return np.concatenate([d, m, qs], axis=-1)---@register_qtype("q5_0")-def q5_0_quant_block(blocks: np.array, scale=None, zp=None, **kwargs):- if scale is not None:- d = scale.reshape((-1, 1))- else:- imax = abs(blocks).argmax(axis=-1, keepdims=True)- max = np.take_along_axis(blocks, imax, axis=-1)- d = max / -16-- n_blocks = blocks.shape[0]- block_size = GGML_QUANT_SIZES["q5_0"][0]- with np.errstate(divide="ignore"):- id = np.where(d == 0, 0, 1 / d)- # FIXME: Q5_0's reference rounding is cursed and depends on FMA- q = np.trunc((np.float64(blocks) * np.float64(id)) + np.float64(16.5),- dtype=np.float32).astype(np.uint8).clip(0, 31)-- qs = q.reshape((n_blocks, 2, block_size // 2))- qs = (qs[..., 0, :] & np.uint8(0x0F)) | (qs[..., 1, :] << np.uint8(4))-- qh = np.packbits(q.reshape((n_blocks, 1, 32)) >> np.uint8(4), axis=-1, bitorder="little").reshape(n_blocks, 4)-- d = d.astype(np.float16).view(np.uint8)-- return np.concatenate([d, qh, qs], axis=-1)---@register_qtype("q5_1")-def q5_1_quant_block(blocks: np.array, scale=None, zp=None, **kwargs):- if scale is not None:- d = scale.reshape((-1, 1))- min = zp.reshape((-1, 1)) * d * -1- else:- max = blocks.max(axis=-1, keepdims=True)- min = blocks.min(axis=-1, keepdims=True)- d = (max-min) / 31-- n_blocks = blocks.shape[0]- block_size = GGML_QUANT_SIZES["q5_1"][0]-- with np.errstate(divide="ignore"):- id = np.where(d == 0, 0, 1 / d)- q = np.trunc((blocks-min) * id + np.float32(0.5), dtype=np.float32).astype(np.uint8).clip(0, 31)-- qs = q.reshape((n_blocks, 2, block_size // 2))- qs = (qs[..., 0, :] & np.uint8(0x0F)) | (qs[..., 1, :] << np.uint8(4))-- qh = np.packbits(q.reshape((n_blocks, 1, 32)) >> np.uint8(4), axis=-1, bitorder="little").reshape(n_blocks, 4)-- d = d.astype(np.float16).view(np.uint8)- m = min.astype(np.float16).view(np.uint8)-- return np.concatenate([d, m, qh, qs], axis=-1)---@register_qtype("q8_0")-def q8_0_quant_block(blocks: np.array, scale=None, zp=None) -> np.ndarray:- if scale is not None:- d = scale.reshape((-1, 1))- else:- d = abs(blocks).max(axis=1, keepdims=True) / 127- with np.errstate(divide="ignore"):- id = np.where(d == 0, 0, 1 / d)- qs = np_roundf(blocks * id)-- # (n_blocks, 2)- d = d.astype(np.float16).view(np.uint8)- # (n_blocks, block_size)- qs = qs.astype(np.int8).view(np.uint8)-- return np.concatenate([d, qs], axis=1)---@register_qtype("q2_k")-def q2_k_quant_block(blocks: np.array, scale=None, zp=None, wmin_m=None, d_scale=None, d_wmin_m=None):- nb = blocks.shape[0]- output_scale = np.empty((nb, 16), dtype=np.uint8)- output_qs = np.empty((nb, QK_K // 16 // 4, 16), dtype=np.uint8)-- blocks = blocks.reshape((nb, QK_K // 16, 16)) # (nb, 16, 16)-- if scale is not None:- scales = scale.reshape((-1, QK_K // 16))- mins = wmin_m.reshape((-1, QK_K // 16))- output_d = d_scale.reshape(-1, 1).astype(np.float32)- output_dmin = d_wmin_m.reshape(-1, 1).astype(np.float32)- inv_scales = np.where(output_d == 0, 0, 1 / output_d)- inv_mins = np.where(d_wmin_m == 0, 0, 1 / output_dmin)- max_scales = np.max(scales, axis=-1, keepdims=True) # (nb, 1)- max_mins = np.max(mins, axis=-1, keepdims=True) # (nb, 1)- all_L = np.round((blocks + mins.reshape(*mins.shape, 1)) / scales.reshape(*scales.shape, 1)).astype(np.uint8)- else:- scales, all_L, mins = make_qkx2_quants(blocks, bits=2, rmin=-0.5, rdelta=0.1, nstep=15, use_mad=True)- max_scales = np.max(scales, axis=-1, keepdims=True)- max_mins = np.max(mins, axis=-1, keepdims=True)- inv_scales = np.where(max_scales > 0, 15. / max_scales, 0)- inv_mins = np.where(max_mins > 0, 15. / max_mins, 0)-- replace_ids = (max_scales > 0).reshape(-1)- output_scale = np.zeros_like(scales).astype(np.uint8)- output_scale[replace_ids] = np.round(inv_scales * scales).astype(np.uint8)-- replace_ids = (max_mins > 0).reshape(-1)- output_scale[replace_ids] |= np.round(inv_mins * mins).astype(np.uint8) << 4-- if d_scale is None:- output_d = np.where(max_scales > 0, max_scales / 15, 0)- if d_wmin_m is None:- output_dmin = np.where(max_mins > 0, max_mins / 15., 0)-- d_tmp = output_d * (output_scale & 0xF)- dm_tmp = output_dmin * (output_scale >> 4)-- replace_ids = d_tmp != 0- all_L[replace_ids] = np.round(- (blocks[replace_ids] + dm_tmp[replace_ids].reshape(-1, 1)) / d_tmp[replace_ids].reshape(-1, 1)).astype(- np.uint8)- all_L = np.clip(all_L, 0, 3)-- output_qs = all_L[:, ::4] | (all_L[:, 1::4] << 2) | (all_L[:, 2::4] << 4) | (all_L[:, 3::4] << 6)- output_d = output_d.reshape(-1, 1).astype(np.float16).view(np.uint8)- output_dmin = output_dmin.reshape(-1, 1).astype(np.float16).view(np.uint8)- output_qs = output_qs.reshape((nb, QK_K // 4))-- # [scale, qs, d, dmin]- return np.concatenate([output_scale, output_qs, output_d, output_dmin], axis=-1)---@register_qtype("q3_k")-def q3_k_quant_block(blocks: np.array, scale=None, zp=None, wmin_m=None, d_scale=None, d_wmin_m=None):- nb = blocks.shape[0]- blocks = blocks.reshape((nb, QK_K // 16, 16))-- output_scale = np.empty((nb, K_SCALE_SIZE), dtype=np.uint8)-- if scale is not None:- scales = scale.reshape(-1, QK_K // 16)- output_d = d_scale.reshape(-1, 1).astype(np.float32)- iscales = np.where(output_d == 0, 0, 1 / output_d)- all_L = np.round(blocks / scales.reshape(*scales.shape, 1)).clip(-4, 3) + 4- all_L = all_L.astype(np.uint8)- else:- scales, all_L = make_q3_quants(blocks, bits=3, do_rmse=False)- imax = abs(scales).argmax(axis=-1, keepdims=True)- max_scales = np.take_along_axis(scales, imax, axis=-1)- iscales = np.where(max_scales != 0, -32 / max_scales, 0)- output_d = np.where(iscales != 0, 1 / iscales, 0)-- q_scales = np.round(iscales * scales).clip(-32, 31) + 32- q_scales = q_scales.astype(np.uint8)- output_scale[:, :8] = (q_scales[:, :8] & 0xF) | ((q_scales[:, 8:] & 0xF) << 4)- hmask = q_scales >> 4- output_scale[:, 8:] = hmask[:, :4] | hmask[:, 4:8] << 2 | hmask[:, 8:12] << 4 | hmask[:, 12:] << 6--- # lscales, hscales = np.hsplit(output_scale, [8])- # lscales = lscales.reshape((nb, 1, 8)) >> np.array([0, 4], dtype=np.uint8).reshape((1, 2, 1))- # lscales = lscales.reshape((nb, 16))- # hscales = hscales.reshape((nb, 1, 4)) >> np.array([0, 2, 4, 6], dtype=np.uint8).reshape((1, 4, 1))- # hscales = hscales.reshape((nb, 16))- # sc = (lscales & np.uint8(0x0F)) | ((hscales & np.uint8(0x03)) << np.uint8(4))- # sc = (sc.astype(np.int8) - np.int8(32)).astype(np.float32)- sc = np.round(iscales * scales).clip(-32, 31)-- d = output_d.astype(np.float32) * sc- replace_idx = (d != 0)-- all_L[replace_idx] = np.round(blocks[replace_idx] / d[replace_idx].reshape(-1, 1)).clip(-4, 3) + 4-- output_hmask = np.bitwise_or.reduce(- all_L.reshape(nb, 8, 32) >> 2 << np.arange(8, dtype=np.uint8).reshape(1, 8, 1), axis=1) # pylint: disable=E1121- all_L = np.where(all_L > 3, all_L - 4, all_L)-- output_qs = np.bitwise_or.reduce(- all_L.reshape(nb, 2, 4, 32) << np.array([0, 2, 4, 6]).reshape(1, 1, 4, 1), axis=2) # pylint: disable=E1121-- output_qs = output_qs.reshape(nb, 64).astype(np.uint8)- output_d = output_d.reshape(-1, 1).astype(np.float16).view(np.uint8)- # [hmask, qs, scale, d]- return np.concatenate([output_hmask, output_qs, output_scale, output_d], axis=-1)---@register_qtype("q4_k")-def q4_k_quant_block(blocks: np.array, scale=None, zp=None, wmin_m=None, d_scale=None, d_wmin_m=None):- nb = blocks.shape[0]- blocks = blocks.reshape(nb, QK_K // 32, 32) # (nb, 8, 32)-- output_scale = np.empty((nb, K_SCALE_SIZE), dtype=np.uint8)- output_qs = np.empty((nb, QK_K // 64, 32), dtype=np.uint8)-- if scale is not None:- scales = scale.reshape(-1, QK_K // 32)- mins = wmin_m.reshape(-1, QK_K // 32)- output_d = d_scale.reshape(-1, 1).astype(np.float32)- output_dmin = d_wmin_m.reshape(-1, 1).astype(np.float32)- inv_scales = np.where(output_d == 0, 0, 1 / output_d)- inv_mins = np.where(d_wmin_m == 0, 0, 1 / output_dmin)- max_scales = np.max(scales, axis=-1, keepdims=True) # (nb, 1)- max_mins = np.max(mins, axis=-1, keepdims=True) # (nb, 1)- all_L = np.round((blocks + mins.reshape(*mins.shape, 1)) / scales.reshape(*scales.shape, 1)).astype(np.uint8)- else:- scales, all_L, mins = make_qkx2_quants(blocks, bits=4, rmin=-1, rdelta=0.1, nstep=20, use_mad=False)- max_scales = np.max(scales, axis=-1, keepdims=True)- max_mins = np.max(mins, axis=-1, keepdims=True)- inv_scales = np.where(max_scales > 0, 63. / max_scales, 0)- inv_mins = np.where(max_mins > 0, 63. / max_mins, 0)-- q_scales = np.round(inv_scales * scales).astype(np.uint8).clip(0, 63)- q_mins = np.round(inv_mins * mins).astype(np.uint8).clip(0, 63)-- output_scale[:, :4] = q_scales[:, :4]- output_scale[:, 4:8] = q_mins[:, :4]-- output_scale[:, 8:] = (q_scales[:, 4:] & 0xF) | ((q_mins[:, 4:] & 0xF) << 4)- output_scale[:, :4] |= ((q_scales[:, 4:] >> 4) << 6)- output_scale[:, 4:8] |= ((q_mins[:, 4:] >> 4) << 6)-- if d_scale is None:- output_d = max_scales / 63- if d_wmin_m is None:- output_dmin = max_mins / 63-- d_tmp = output_d * q_scales- dm_tmp = output_dmin * q_mins-- replace_ids = d_tmp != 0- all_L[replace_ids] = np.round(- (blocks[replace_ids] + dm_tmp[replace_ids].reshape(-1, 1)) / d_tmp[replace_ids].reshape(-1, 1)).astype(- np.uint8)- all_L = np.clip(all_L, 0, 15)- output_qs = all_L[:, ::2] | (all_L[:, 1::2] << 4)-- output_d = output_d.reshape(-1, 1).astype(np.float16).view(np.uint8)- output_dmin = output_dmin.reshape(-1, 1).astype(np.float16).view(np.uint8)- output_qs = output_qs.reshape(nb, QK_K // 2)-- # [d, dmin, scale, qs]- return np.concatenate([output_d, output_dmin, output_scale, output_qs], axis=-1)---@register_qtype("q5_k")-def q5_k_quant_block(blocks: np.array, scale=None, zp=None, wmin_m=None, d_scale=None, d_wmin_m=None):- nb = blocks.shape[0]- blocks = blocks.reshape((nb, QK_K // 32, 32))-- output_scale = np.empty((nb, K_SCALE_SIZE), dtype=np.uint8)- output_qs = np.empty((nb, QK_K // 64, 32), dtype=np.uint8)- output_qh = np.empty((nb, QK_K // 8), dtype=np.uint8)-- if scale is not None:- scales = scale.reshape(-1, QK_K // 32)- mins = wmin_m.reshape(-1, QK_K // 32)- output_d = d_scale.reshape(-1, 1).astype(np.float32)- output_dmin = d_wmin_m.reshape(-1, 1).astype(np.float32)- inv_scales = np.where(output_d == 0, 0, 1 / output_d)- inv_mins = np.where(d_wmin_m == 0, 0, 1 / output_dmin)- max_scales = np.max(scales, axis=-1, keepdims=True) # (nb, 1)- max_mins = np.max(mins, axis=-1, keepdims=True) # (nb, 1)- all_L = np.round((blocks + mins.reshape(*mins.shape, 1)) / scales.reshape(*scales.shape, 1)).astype(np.uint8)- else:- scales, all_L, mins = make_qkx2_quants(blocks, bits=5, rmin=-0.5, rdelta=0.1, nstep=15, use_mad=False)- max_scales = np.max(scales, axis=-1, keepdims=True)- max_mins = np.max(mins, axis=-1, keepdims=True)- inv_scales = np.where(max_scales > 0, 63. / max_scales, 0)- inv_mins = np.where(max_mins > 0, 63. / max_mins, 0)-- q_scales = np.round(inv_scales * scales).astype(np.uint8).clip(0, 63)- q_mins = np.round(inv_mins * mins).astype(np.uint8).clip(0, 63)-- output_scale[:, :4] = q_scales[:, :4]- output_scale[:, 4:8] = q_mins[:, :4]-- output_scale[:, 8:] = (q_scales[:, 4:] & 0xF) | ((q_mins[:, 4:] & 0xF) << 4)- output_scale[:, :4] |= ((q_scales[:, 4:] >> 4) << 6)- output_scale[:, 4:8] |= ((q_mins[:, 4:] >> 4) << 6)-- if d_scale is None:- output_d = max_scales / 63- if d_wmin_m is None:- output_dmin = max_mins / 63-- d_tmp = output_d * q_scales- dm_tmp = output_dmin * q_mins-- replace_ids = d_tmp != 0- all_L[replace_ids] = np.round(- (blocks[replace_ids] + dm_tmp[replace_ids].reshape(-1, 1)) / d_tmp[replace_ids].reshape(-1, 1)).astype(- np.uint8)- all_L = np.clip(all_L, 0, 31) # (nb, 8, 32)-- output_qs = all_L[:, ::2] | (all_L[:, 1::2] << 4)- output_qh = np.bitwise_or.reduce(- all_L >> 4 << np.arange(8, dtype=np.uint8).reshape(1, 8, 1), axis=1).astype(np.uint8) # pylint: disable=E1121-- output_d = output_d.reshape(-1, 1).astype(np.float16).view(np.uint8)- output_dmin = output_dmin.reshape(-1, 1).astype(np.float16).view(np.uint8)- output_qs = output_qs.reshape(nb, QK_K // 2)- # [d, dmin, scale, qh, qs]- return np.concatenate([output_d, output_dmin, output_scale, output_qh, output_qs], axis=-1)--@register_qtype("q6_k")-def q6_k_quant_block(blocks: np.array, scale=None, zp=None, wmin_m=None, d_scale=None, d_wmin_m=None):- nb = blocks.shape[0]- blocks = blocks.reshape((nb, QK_K // 16, 16))-- if scale is not None:- scales = scale.reshape(-1, QK_K // 16)- output_d = d_scale.reshape(-1, 1).astype(np.float32)- iscales = np.where(output_d == 0, 0, 1 / output_d)- all_L = (np.round(blocks / scales.reshape(*scales.shape, 1)).clip(-32, 31) + 32).astype(np.uint8)- else:- scales, all_L = make_qx_quant(blocks, bits=6, rmse_type=1, qw=None)- imax = abs(scales).argmax(axis=-1, keepdims=True)- max_scales = np.take_along_axis(scales, imax, axis=-1)- iscales = np.where(max_scales != 0, -128 / max_scales, 0)- output_d = np.where(iscales != 0, 1 / iscales, -1)-- output_scale = np.round(iscales * scales).clip(max=127).astype(np.int8)- d_tmp = output_d * output_scale.astype(np.float32)- replace_ids = d_tmp != 0- all_L[replace_ids] = np.round(blocks[replace_ids] / d_tmp[replace_ids].reshape(-1, 1) + 32) \- .clip(0, 63).astype(np.uint8)-- tmp_L = all_L.reshape(nb, 4, 64) & 0xF- output_ql = (tmp_L[:, ::2] | (tmp_L[:, 1::2] << 4)).reshape(nb, QK_K // 2)- output_qh = np.bitwise_or.reduce(- (all_L >> 4).reshape(nb, 2, 4, 32) << np.array([0, 2, 4, 6]).reshape(1, 1, 4, 1), # pylint: disable=E1121- axis=2).reshape(nb, QK_K // 4).astype(np.uint8) # pylint: disable=E1121-- output_d = output_d.reshape(-1, 1).astype(np.float16).view(np.uint8)-- # [ql, qh, scale, d]- output_scale = output_scale.view(np.uint8)- return np.concatenate([output_ql, output_qh, output_scale, output_d], axis=-1)diff --git a/auto_round/export/export_to_gguf/utils.py b/auto_round/export/export_to_gguf/utils.pydeleted file mode 100644index 99ab13c7d..000000000--- a/auto_round/export/export_to_gguf/utils.py+++ /dev/null@@ -1,30 +0,0 @@-# Copyright (c) 2025 Intel Corporation-#-# Licensed under the Apache License, Version 2.0 (the "License");-# you may not use this file except in compliance with the License.-# You may obtain a copy of the License at-#-# http://www.apache.org/licenses/LICENSE-2.0-#-# Unless required by applicable law or agreed to in writing, software-# distributed under the License is distributed on an "AS IS" BASIS,-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.-# See the License for the specific language governing permissions and-# limitations under the License.--QK_K = 256-K_SCALE_SIZE = 12-GGML_QUANT_SIZES = {- "bf16": (1, 2),- "q4_0": (32, 2 + 16),- "q4_1": (32, 2 + 2 + 16),- "q5_0": (32, 2 + 4 + 16),- "q5_1": (32, 2 + 2 + 4 + 16),- "q8_0": (32, 2 + 32),- "q2_k": (256, 2 + 2 + QK_K//16 + QK_K//4),- "q3_k": (256, 2 + QK_K // 4 + QK_K // 8 + 12),- "q4_k": (256, 2 + 2 + QK_K//2 + 12),- "q5_k": (256, 2 + 2 + QK_K // 2 + QK_K // 8 + 12),- "q6_k": (256, 2 + QK_K // 2 + QK_K // 4 + QK_K // 16),- "q8_k": (256, 4 + QK_K + QK_K // 8)-}\ No newline at end of filediff --git a/auto_round/script/llm.py b/auto_round/script/llm.pyindex 190bcffa0..f7fc3af4e 100644--- a/auto_round/script/llm.py+++ b/auto_round/script/llm.py@@ -200,6 +200,10 @@ class BasicArgumentParser(argparse.ArgumentParser):type=str,help="the torch_dytpe to load the model for evaluation.")+ self.add_argument("--disable_opt_rtn", action='store_true',+ help="whether to disable optimization of the RTN mode(iters=0) (default is False).")++class EvalArgumentParser(argparse.ArgumentParser):def __init__(self, *args, **kwargs):@@ -343,7 +347,6 @@ def tune(args):if args.format is None:args.format = "auto_round"-formats = args.format.lower().replace(' ', '').split(",")from auto_round.utils import SUPPORTED_FORMATSfor format in formats:@@ -485,13 +488,21 @@ def tune(args):device_map=args.device_map,super_group_size=args.super_group_size,super_bits=args.super_bits,+ disable_opt_rtn=args.disable_opt_rtn,)model_name = args.model.rstrip("/")- if model_name.split('/')[-1].strip('.') == "":- export_dir = os.path.join(args.output_dir, f"w{args.bits}g{args.group_size}")++ if model_name.split('/')[-1].strip('.') == "" and "gguf" not in args.format:+ export_dir = os.path.join(args.output_dir, f"w{autoround.bits}g{autoround.group_size}")+ elif model_name.split('/')[-1].strip('.') == "" and "gguf" in args.format:+ export_dir = args.output_dir+ elif model_name.split('./')[-1].strip('./') != "" and "gguf" in args.format:+ export_dir = os.path.join(args.output_dir,+ model_name.split('/')[-1] + "-gguf")else:- export_dir = os.path.join(args.output_dir, model_name.split('/')[-1] + f"-w{args.bits}g{args.group_size}")+ export_dir = os.path.join(args.output_dir,+ model_name.split('/')[-1] + f"-w{autoround.bits}g{autoround.group_size}")model, folders = autoround.quantize_and_save(export_dir, format=args.format)@@ -645,7 +656,7 @@ def eval(args):model.eval()st = time.time()res = simple_evaluate_user_model(- model, tokenizer, tasks=tasks, batch_size=batch_size, device=device_str)+ model, tokenizer, tasks=tasks, batch_size=batch_size, device=device_str)print(make_table(res))print("evaluation running time=%ds" % (time.time() - st))else:@@ -706,7 +717,7 @@ def eval_task_by_task(model = AutoModelForCausalLM.from_pretrained(model, gguf_file=gguf_file, device_map="auto", torch_dtype=eval_model_dtype)model.eval()- parallelism=False+ parallelism = Falsehflm = HFLM(pretrained=model,tokenizer=tokenizer,@@ -716,7 +727,7 @@ def eval_task_by_task(parallelize=parallelism,trust_remote_code=trust_remote_code,dtype=eval_model_dtype- )+ )if isinstance(tasks, str):tasks = tasks.replace(" ", "").split(",")@@ -737,7 +748,7 @@ def eval_task_by_task(ori_batch_sizes = hflm.batch_sizes if hflm.batch_sizes else {"0": 64}try:for k, v in hflm.batch_sizes.items():- hflm.batch_sizes[k] = max(v // 2, 1)+ hflm.batch_sizes[k] = max(v // 2, 1)logger.warning(f"Out of memory, reset batch_size to {hflm.batch_sizes} and re-try.")res = lm_simple_evaluate(@@ -759,5 +770,3 @@ def eval_task_by_task(print(make_table(res_all))print("total eval time:", time.time() - st)--diff --git a/auto_round/utils.py b/auto_round/utils.pyindex 0516c70f0..a9b085307 100644--- a/auto_round/utils.py+++ b/auto_round/utils.py@@ -27,12 +27,13 @@ from torch.amp import autocastfrom functools import lru_cachefrom packaging import versionimport gc-from .special_model_handler import SPECIAL_MULTIMODAL_BLOCK, SPECIAL_SHARED_CACHE_KEYS+from auto_round.special_model_handler import SPECIAL_MULTIMODAL_BLOCK, SPECIAL_SHARED_CACHE_KEYSimport transformers-from auto_round.export.export_to_gguf.config import GGUF_CONFIG+from auto_round.export.export_to_gguf.config import GGUF_CONFIG, GGML_QUANT_SIZES, GGUF_INNER_CONFIGSHARED_CACHE_KEYS = ("position_ids", "cache_position", "position_embeddings")+class SupportedFormats:def __init__(self):self._support_format = (@@ -45,11 +46,13 @@ class SupportedFormats:return True if key in self._support_list else Falsedef __str__(self):- return "(%s)" % ', '.join(self._support_format + ("gguf:q*_0", "gguf:q*_1", "gguf:q*_k_s"))+ ##return "(%s)" % ', '.join(self._support_format + ("gguf:q*_0", "gguf:q*_1", "gguf:q*_k_s"))+ return "(%s)" % ', '.join(self._support_list)def __getitem__(self, key):return self._support_list[key]+SUPPORTED_FORMATS = SupportedFormats()SUPPORTED_LAYER_TYPES = (torch.nn.Linear, transformers.pytorch_utils.Conv1D)@@ -1191,7 +1194,6 @@ def _gguf_args_check(args_or_ar, format_str=None):raise ImportError(f"Please use the latest gguf-py for {format}, you can use the following command to install it:\n""git clone https://github.com/ggml-org/llama.cpp.git && cd llama.cpp/gguf-py && pip install .")- sys.exit(-1)if re.search(pattern, format):if pre_dq_format and re.search(pattern, format).group() not in pre_dq_format:logger.error(f"Cannot export {pre_dq_format} and {format} at the same time.")@@ -1227,8 +1229,6 @@ def _gguf_args_check(args_or_ar, format_str=None):if k == "data_type":if re.search("q\d_1", format) and len(formats) > 1:v = "int"- if re.search("q\d_k", format) and args_or_ar.iters == 0:- v = f"gguf_{v}"if k == "sym" and isinstance(args_or_ar, argparse.Namespace):k = "asym"v = not v@@ -1237,7 +1237,7 @@ def _gguf_args_check(args_or_ar, format_str=None):reset_list.append(f"{k}={v}")setattr(args_or_ar, k, v)if len(unsupport_list) > 0:- logger.error(+ logger.warning(f"format {format} does not support for {', '.join(unsupport_list)},"f" reset to {', '.join(reset_list)}.")if not isinstance(args_or_ar, argparse.Namespace) and len(unsupport_list) > 0:@@ -1247,7 +1247,6 @@ def _gguf_args_check(args_or_ar, format_str=None):for k in args_or_ar.layer_config[layer_name]:if hasattr(args_or_ar, k):args_or_ar.layer_config[layer_name][k] = getattr(args_or_ar, k)- args_or_ar.has_qlayer_outside_block = args_or_ar.set_layerwise_config(args_or_ar.layer_config)return args_or_ar@@ -1370,7 +1369,7 @@ def mllm_load_model(torch_dtype=torch_dtype)else:if architectures.endswith("Model") \- and hasattr(transformers, n := architectures.replace("Model", "ForConditionalGeneration")):+ and hasattr(transformers, n := architectures.replace("Model", "ForConditionalGeneration")):cls = getattr(transformers, n)elif hasattr(transformers, architectures):cls = getattr(transformers, architectures)@@ -1481,6 +1480,7 @@ def get_model_dtype(model_dtype, default="auto"):model_dtype = defaultreturn model_dtype+def str2bool(v):import argparseif isinstance(v, bool):@@ -1492,6 +1492,7 @@ def str2bool(v):else:raise argparse.ArgumentTypeError('Boolean value expected.')+def filter_quantization_config(quantization_config):default_dict = {"amp": True, "batch_size": 8, "data_type": int, "dataset": "NeelNanda/pile-10k","enable_minmax_tuning": True, "enable_norm_bias_tuning": False, "enable_quanted_input": True,@@ -1517,7 +1518,6 @@ def filter_quantization_config(quantization_config):quantization_config.pop("act_group_size", None)-def check_start_with_block_name(name: str, block_name_to_quantize: list):"""Checks if the given layer name starts with any of the block names to be quantized.@@ -1552,13 +1552,15 @@ def check_seqlen_compatible(input_seqlen, tokenizer=None, model=None):model_config = model.configif hasattr(model_config, 'max_position_embeddings') and input_seqlen > model_config.max_position_embeddings:raise ValueError(f"seqlen({input_seqlen}) exceeds model.config.max_position_embeddings(" \- f"{model_config.max_position_embeddings}). Please lowering '--seqlen'")+ f"{model_config.max_position_embeddings}). Please lowering '--seqlen'")if tokenizer is not None and hasattr(tokenizer, 'model_max_length') and input_seqlen > tokenizer.model_max_length:raise ValueError(f"seqlen({input_seqlen}) exceeds tokenizer.model_max_length({tokenizer.model_max_length}). " \- "Please oncider Consider lowering the '--seqlen' or increasing tokenizer.model_max_length.")+ "Please oncider Consider lowering the '--seqlen' or increasing tokenizer.model_max_length.")+def _use_more_bits(i_layer: int, n_layer: int):- return (i_layer < n_layer // 8) or (i_layer >= 7 * n_layer // 8) or ((i_layer - n_layer//8) % 3 == 2)+ return (i_layer < n_layer // 8) or (i_layer >= 7 * n_layer // 8) or ((i_layer - n_layer // 8) % 3 == 2)+def _get_digital_in_layer_name(layer_name):pattern = re.compile("([a-zA-Z]+\.){1,}(\d+)")@@ -1568,14 +1570,14 @@ def _get_digital_in_layer_name(layer_name):else:return None++##https://github.com/ggml-org/llama.cpp/blob/9e31bec4fd53634c9e5b04650488a09a055f5dab/src/llama-quant.cpp#L129def get_layer_config_by_gguf_format(layer_config, gguf_format, model):# TODO: support for other format later- if "q4_k_m" not in gguf_format:- return layer_config, {}+ target_gguf_format = next((fmt for fmt in gguf_format if fmt != "fake"), None)- import gguf # pylint: disable=E0401+ import gguf # pylint: disable=E0401from auto_round.export.export_to_gguf.convert import Model- from auto_round.export.export_to_gguf.config import GGUF_CONFIGmodel_architecture = model.config.architectures[0]try:model_class = Model.from_model_architecture(model_architecture)@@ -1591,62 +1593,223 @@ def get_layer_config_by_gguf_format(layer_config, gguf_format, model):tensor_map = gguf.get_tensor_name_map(model_class.model_arch, n_layer)- def _set_config(config, gguf_config):- for k, v in gguf_config.items():- if k in config:- if k == "data_type" and config[k].startswith("gguf_"):- config[k] = "gguf_" + v- else:- config[k] = v+ def _set_config(config, target_config):+ for k, v in target_config.items():+ if isinstance(config, dict):+ config[k] = v+ else:+ setattr(config, k, v)return config-+gguf_format_config = {}- for layer_name, config in layer_config.items():- if config["bits"] >= 16:+ lm_head_name = get_lm_head_name(model)+ inner_gguf_format = GGUF_CONFIG[target_gguf_format]["mostly"]+ # ggml_type = getattr(gguf.GGMLQuantizationType,inner_gguf_format.split(":")[-1].upper())+ block_size = GGML_QUANT_SIZES[inner_gguf_format.split(":")[-1].lower()][0]+ tie_word_embeddings = True+ if hasattr(model, "config") and hasattr(model.config, "tie_word_embeddings"):+ tie_word_embeddings = model.config.tie_word_embeddings++ n_gqa = 1+ if hasattr(model, "config") and hasattr(model.config, "num_attention_heads") and hasattr(model.config,+ "num_key_value_heads"):+ n_gqa = model.config.num_attention_heads // model.config.num_key_value_heads+ n_expert = 0+ for name in ["num_experts", "num_local_experts", "n_routed_experts"]:+ if hasattr(model.config, name):+ n_expert = getattr(model.config, name)++ i_attention_wv = 0+ i_ffn_down = 0+ layer_config_copy = copy.deepcopy(layer_config)++ for layer_name, config in layer_config_copy.items():+ if not check_to_quantized(config):continue- gguf_format_config[layer_name] = gguf_format+ new_type = GGUF_CONFIG[target_gguf_format]["mostly"]+ layer = get_module(model, layer_name)+ if isinstance(layer, transformers.pytorch_utils.Conv1D):+ input_features = layer.weight.shape[0]+ else:+ input_features = layer.weight.shape[-1]i_layer = _get_digital_in_layer_name(layer_name)- if i_layer is None:- continue+gguf_name = tensor_map.get_name(layer_name)- if gguf_name is None:- continue- # attn_v- if "attn_v" in gguf_name:- if _use_more_bits(i_layer, n_layer):- # q6_k- config = _set_config(config, GGUF_CONFIG["gguf:q6_k"])- gguf_format_config[layer_name] = "gguf:q6_k"+ if lm_head_name is not None and layer_name == lm_head_name and not tie_word_embeddings:+ if gguf.MODEL_ARCH.FALCON == model_class.model_arch or input_features % block_size != 0:+ new_type = "gguf:q8_0"+ elif new_type != "gguf:q8_0":+ new_type = "gguf:q6_k"+ elif lm_head_name is not None and layer_name == lm_head_name and tie_word_embeddings:+ pass+ elif isinstance(layer, torch.nn.Embedding):+ pass++ # attn_v+ elif "attn_v" in gguf_name:+ if target_gguf_format == "gguf:q2_k":+ new_type = "gguf:q4_k" if n_gqa >= 4 else "gguf:q3_k"+ elif target_gguf_format == "gguf:q2_k_s" and n_gqa >= 4:+ new_type = "gguf:q4_k"+ elif target_gguf_format == "gguf:q3_k_m":+ new_type = "gguf:q5_k" if i_attention_wv < 2 else "gguf:q4_k"+ elif target_gguf_format == "gguf:q3_k_l":+ new_type = "gguf:q5_k"+ elif (target_gguf_format == "gguf:q4_k_m" or target_gguf_format == "gguf:q5_k_m") and _use_more_bits(+ i_layer, n_layer):+ new_type = "gguf:q6_k"+ elif target_gguf_format == "gguf:q4_k_s" and i_attention_wv < 4:+ new_type = "gguf:q5_k"+ ##TODO check which models are be grouped into to LLM_TYPE_70B+ # if (qs.model.type == LLM_TYPE_70B) {+ # // In the 70B model we have 8 heads sharing the same attn_v weights.+ # As a result, the attn_v.weight tensor is+ # // 8x smaller compared to attn_q.weight.Hence, we can get a nice boost in quantization accuracy with+ # // nearly negligible increase in model size by quantizing this tensor with more bits:+ # if+ # (new_type == GGML_TYPE_Q3_K | | new_type == GGML_TYPE_Q4_K)+ # new_type = GGML_TYPE_Q5_K;+ # }+ if n_expert == 8:+ new_type = "gguf:q8_k"+ i_attention_wv += 1++ elif "attn_k" in gguf_name:+ if n_expert == 8:+ new_type = "gguf:q8_0"# ffn_downif "ffn_down" in gguf_name:- if gguf.MODEL_ARCH.FALCON == model_class.model_arch:- if i_layer < n_layer // 16:- format = "gguf:q6_k"- elif _use_more_bits(i_layer, n_layer):- format = "gguf:q5_k_s"+ if target_gguf_format == "gguf:q2_k":+ new_type = "gguf:q3_k"+ elif target_gguf_format == "gguf:q2_k_s":+ if i_layer < n_layer / 8:+ new_type = "gguf:q4_k"+ elif target_gguf_format == "gguf:q3_k_m":+ if i_layer < n_layer / 16:+ new_type = "gguf:q5_k"+ elif gguf.MODEL_ARCH.FALCON == model_class.model_arch or _use_more_bits(i_layer, n_layer):+ new_type = "gguf:q4_k"else:- format = "gguf:q4_k_s"- config = _set_config(config, GGUF_CONFIG[format])- gguf_format_config[layer_name] = format- else:- if _use_more_bits(i_layer, n_layer):- config = _set_config(config, GGUF_CONFIG["gguf:q6_k"])- gguf_format_config[layer_name] = "gguf:q6_k"+ new_type = "gguf:q3_k"+ elif target_gguf_format == "gguf:q3_k_l":+ if gguf.MODEL_ARCH.FALCON == model_class.model_arch:+ new_type = "gguf:q4_k"+ else:+ new_type = "gguf:q5_k"+ elif target_gguf_format == "gguf:q4_k_m":+ if gguf.MODEL_ARCH.FALCON == model_class.model_arch:+ if i_layer < n_layer // 16:+ new_type = "gguf:q6_k"+ elif _use_more_bits(i_layer, n_layer):+ new_type = "gguf:q5_k"+ else:+ new_type = "gguf:q4_k"+ else:+ if _use_more_bits(i_layer, n_layer):+ new_type = "gguf:q6_k"+ elif target_gguf_format == "gguf:q5_k_m" and _use_more_bits(i_layer, n_layer):+ new_type = "gguf:q6_k"+ elif (target_gguf_format == "gguf:q4_k_s" and+ model_class.model_arch != gguf.MODEL_ARCH.FALCON and i_layer < n_layer / 8):+ new_type = "gguf:q5_k"+ elif (target_gguf_format == "gguf:q4_0" or target_gguf_format == "gguf:q5_0") and i_layer < n_layer / 8:+ if target_gguf_format == "gguf:q4_0":+ new_type = "gguf:q4_1"+ else:+ new_type = "gguf:q5_1"+ i_ffn_down += 1# attn_output- if "attn_output" in gguf_name and gguf.MODEL_ARCH.FALCON != model_class.model_arch:- n_export = 0- for name in ["num_experts", "num_local_experts", "n_routed_experts"]:- if hasattr(model.config, name):- n_export = getattr(model.config, name)- if n_export == 8:- config = _set_config(config, GGUF_CONFIG["gguf:q5_k_s"])- gguf_format_config[layer_name] = "gguf:q5_k_s"-+ elif "attn_output" in gguf_name:+ if gguf.MODEL_ARCH.FALCON != model_class.model_arch:+ if n_expert == 8:+ if target_gguf_format in ("gguf:q2_k", "gguf:q3_k_s", "gguf:q3_k_m", "gguf:q4_k_s", "gguf:q4_k_m",+ "gguf:q5_k"):+ new_type = "gguf:q5_k"+ elif target_gguf_format == "gguf:q2_k":+ new_type = "gguf:q3_k"+ elif target_gguf_format == "gguf:q3_k_m":+ new_type = "gguf:q4_k"+ elif target_gguf_format == "gguf:q3_k_l":+ new_type = "gguf:q5_k"+ else:+ if target_gguf_format == "gguf:q3_k_l":+ new_type = "gguf:q4_k"# attn_qkv- if "attn_qkv" in gguf_name:- config = _set_config(config, GGUF_CONFIG["gguf:q5_k_s"])-- layer_config[layer_name] = config- return layer_config, gguf_format_config\ No newline at end of file+ elif "attn_qkv" in gguf_name:+ if target_gguf_format in ("gguf:q3_k_m", "gguf:q3_k_l"):+ new_type = "gguf:q4_k"+ elif target_gguf_format == "gguf:q4_k_m":+ new_type = "gguf:q5_k"+ elif target_gguf_format == "gguf:q5_k_m":+ new_type = "gguf:q5_k"+ if input_features % block_size != 0:+ if new_type in ("gguf:q2_k", "gguf:q3_k", "gguf:q4_k"):+ new_type = "gguf:q5_0"+ elif new_type == "gguf:q5_k":+ new_type = "gguf:q5_0"+ elif new_type == "gguf:q6_k":+ new_type = "gguf:q8_0"+ new_block_size = GGML_QUANT_SIZES[new_type.split(":")[-1].lower()][0]+ if input_features % new_block_size != 0:+ new_type = "gguf:bf16"+ logger.warning(+ f"fallback {layer_name} to {new_type}, "+ f"because input_features({input_features}) % block_size({block_size}) != 0")++ target_config = GGUF_INNER_CONFIG[new_type]++ _set_config(layer_config[layer_name], target_config)+ _set_config(layer, target_config)+ gguf_format_config[layer_name] = new_type++ return layer_config, gguf_format_config+++def get_lm_head_name(model):+ block_names = get_block_names(model, True)+ last_name = None+ for n, m in model.named_modules():+ if any(m.children()):+ continue+ last_name = n+ for l in block_names:+ if last_name in l:+ last_name = None+ break+ return last_name+++def get_gguf_qtype_by_layer_config(layer_config):+ import gguf # pylint: disable=E0401+ if layer_config['bits'] >= 16:+ return None+ bits = layer_config['bits']+ super_bits = layer_config.get("super_bits", None)+ sym = layer_config["sym"]+ group_size = layer_config.get("group_size", None)+ super_group_size = layer_config.get("super_group_size", None)+ if bits == 2 and super_bits == 4 and not sym and group_size == 16 and super_group_size == 16:+ return gguf.GGMLQuantizationType.Q2_K+ if bits == 3 and super_bits == 6 and sym and group_size == 16 and super_group_size == 16:+ return gguf.GGMLQuantizationType.Q3_K+ if bits == 4:+ if super_bits is not None and super_bits == 6 and not sym and group_size == 32 and super_group_size == 8:+ return gguf.GGMLQuantizationType.Q4_K+ if super_bits is None and sym and group_size == 32:+ return gguf.GGMLQuantizationType.Q4_0+ if super_bits is None and not sym and group_size == 32:+ return gguf.GGMLQuantizationType.Q4_1+ if bits == 5:+ if super_bits == 6 and not sym and group_size == 32 and super_group_size == 8:+ return gguf.GGMLQuantizationType.Q5_K+ if super_bits is None and sym and group_size == 32:+ return gguf.GGMLQuantizationType.Q5_0+ if super_bits is None and not sym and group_size == 32:+ return gguf.GGMLQuantizationType.Q5_1+ if bits == 6 and super_bits == 8 and group_size == 16 and super_group_size == 16:+ return gguf.GGMLQuantizationType.Q6_K+ if bits == 8 and sym and group_size == 32:+ return gguf.GGMLQuantizationType.Q8_0+ raise ValueError(f"Unknown layer config")diff --git a/auto_round/wrapper.py b/auto_round/wrapper.pyindex 703c2b1f5..f85a99970 100644--- a/auto_round/wrapper.py+++ b/auto_round/wrapper.py@@ -80,8 +80,6 @@ class WrapperLinear(torch.nn.Module):self._init_tuning_params_and_quant_func()self.orig_forward = self.linear_forward if isinstance(self.orig_layer, torch.nn.Linear) else self.conv1d_forward--def _init_tuning_params_and_quant_func(self):"""Initializes tuning parameters and quantization functions.@@ -173,6 +171,7 @@ class WrapperLinear(torch.nn.Module):tensor_max=self.weight_max,data_type=self.data_type,q_scale_thresh=self.q_scale_thresh,+ imatrix=self.orig_layer.imatrix if hasattr(self.orig_layer, "imatrix") else None,**quant_kwargs)weight_q = weight_q.to(weight.dtype)@@ -231,7 +230,8 @@ class WrapperLinear(torch.nn.Module):self.orig_layer.to(self.device)##unwrapper weightqdq_weight, scale, zp = self._qdq_weight(v, min_scale, max_scale)-+ if hasattr(self.orig_layer, "imatrix"):+ self.orig_layer.imatrix = Noneself.orig_layer.weight.data.copy_(qdq_weight)self.orig_layer.weight.grad = Nonediff --git a/docs/step_by_step.md b/docs/step_by_step.mdindex 04992e0af..876a9dac1 100644--- a/docs/step_by_step.md+++ b/docs/step_by_step.md@@ -242,10 +242,8 @@ autoround.quantize_and_save(output_dir, format='auto_round')```### GGUF format--This format is well-suited for CPU devices and is widely adopted by the community. Mixed bits-configs like q4_k_m have not been supported yet. Please note: In contrast to the official implementation, AutoRound does not quantize the embedding layer or the LM head layer by default.-+Experimental feature. This format is well-suited for CPU devices and is widely adopted by the community.+This format is well-suited for CPU devices and is widely adopted by the community.```pythonfrom transformers import AutoModelForCausalLM, AutoTokenizerfrom auto_round import AutoRounddiff --git a/test/test_cpu/test_autoround.py b/test/test_cpu/test_autoround.pyindex 1e05a191a..261471f20 100644--- a/test/test_cpu/test_autoround.py+++ b/test/test_cpu/test_autoround.py@@ -575,6 +575,7 @@ class TestAutoRound(unittest.TestCase):dataset=self.llm_dataloader,layer_config=layer_config)+ autoround.quantize()if __name__ == "__main__":diff --git a/test/test_cpu/test_gguf_format.py b/test/test_cpu/test_gguf_format.pyindex 54213f731..dbeb8a3bf 100644--- a/test/test_cpu/test_gguf_format.py+++ b/test/test_cpu/test_gguf_format.py@@ -31,12 +31,12 @@ class TestGGUF(unittest.TestCase):def tearDownClass(self):shutil.rmtree("./saved", ignore_errors=True)shutil.rmtree("runs", ignore_errors=True)-+def test_basic_usage(self):python_path = sys.executableres = os.system(- f"cd ../.. && {python_path} -m auto_round --model {self.model_name} --eval_task_by_task"- f" --tasks piqa --bs 16 --iters 1 --nsamples 1 --format fake,gguf:q4_0"+ f"cd ../.. && {python_path} -m auto_round --model {self.model_name} "+ f" --bs 16 --iters 1 --nsamples 1 --format fake,gguf:q4_0")if res > 0 or res == -1:assert False, "cmd line test fail, please have a check"@@ -44,7 +44,7 @@ class TestGGUF(unittest.TestCase):res = os.system(f"cd ../.. && {python_path} -m auto_round --model {self.model_name}"- f" --tasks piqa,openbookqa --bs 16 --iters 1 --nsamples 1 --format fake,gguf:q4_0"+ f" --bs 16 --iters 1 --nsamples 1 --format fake,gguf:q4_0")if res > 0 or res == -1:assert False, "cmd line test fail, please have a check"@@ -71,12 +71,12 @@ class TestGGUF(unittest.TestCase):inputs = self.tokenizer(text, return_tensors="pt").to(model.device)print(self.tokenizer.decode(model.generate(**inputs, max_new_tokens=10)[0]))- from auto_round.eval.evaluation import simple_evaluate_user_model- result = simple_evaluate_user_model(model, self.tokenizer, batch_size=16, tasks="openbookqa", eval_model_dtype="bf16")- # 0.246- self.assertGreater(result['results']['openbookqa']['acc,none'], 0.23)+ # from auto_round.eval.evaluation import simple_evaluate_user_model+ # result = simple_evaluate_user_model(model, self.tokenizer, batch_size=16, tasks="openbookqa", eval_model_dtype="bf16")+ # # 0.246+ # self.assertGreater(result['results']['openbookqa']['acc,none'], 0.23)shutil.rmtree("./saved", ignore_errors=True)-+def test_q4_1(self):bits, group_size, sym = 4, 32, Falseautoround = AutoRound(@@ -97,22 +97,22 @@ class TestGGUF(unittest.TestCase):inputs = self.tokenizer(text, return_tensors="pt").to(model.device)print(self.tokenizer.decode(model.generate(**inputs, max_new_tokens=10)[0]))- from auto_round.eval.evaluation import simple_evaluate_user_model- result = simple_evaluate_user_model(model, self.tokenizer, batch_size=16, tasks="openbookqa", eval_model_dtype="bf16")- # 0.23- self.assertGreater(result['results']['openbookqa']['acc,none'], 0.22)+ # from auto_round.eval.evaluation import simple_evaluate_user_model+ # result = simple_evaluate_user_model(model, self.tokenizer, batch_size=16, tasks="openbookqa", eval_model_dtype="bf16")+ # # 0.23+ # self.assertGreater(result['results']['openbookqa']['acc,none'], 0.22)shutil.rmtree("./saved", ignore_errors=True)-+def test_func(self):bits, group_size, sym = 4, 128, Trueautoround = AutoRound(self.model,self.tokenizer,- bits=bits,- group_size=group_size,- sym=sym,- iters=0,- data_type="int"+ # bits=bits,+ # group_size=group_size,+ # sym=sym,+ iters=1,+ # data_type="int")quantized_model_path = "./saved"autoround.quantize_and_save(output_dir=quantized_model_path, inplace=False, format="gguf:q*_1")@@ -124,8 +124,8 @@ class TestGGUF(unittest.TestCase):inputs = self.tokenizer(text, return_tensors="pt").to(model.device)print(self.tokenizer.decode(model.generate(**inputs, max_new_tokens=10)[0]))shutil.rmtree("./saved", ignore_errors=True)-- model_name = "Qwen/Qwen2.5-1.5B-Instruct"++ model_name = "Qwen/Qwen2.5-1.5B-Instruct"model = AutoModelForCausalLM.from_pretrained(model_name, torch_dtype="auto", trust_remote_code=True)autoround = AutoRound(model,@@ -133,22 +133,22 @@ class TestGGUF(unittest.TestCase):bits=3,group_size=16,sym=True,- iters=0,+ iters=1,data_type="int_sym_dq",super_group_size=16,super_bits=6)quantized_model_path = "./saved"- autoround.quantize_and_save(output_dir=quantized_model_path, inplace=False, format="gguf:q*_k_s")- from auto_round.eval.evaluation import simple_evaluate_user_model- gguf_file = os.listdir("saved")[0]- model = AutoModelForCausalLM.from_pretrained(quantized_model_path, gguf_file=gguf_file, device_map="auto")- result = simple_evaluate_user_model(model, self.tokenizer, batch_size=16, tasks="lambada_openai", eval_model_dtype="bf16")- self.assertGreater(result['results']['lambada_openai']['acc,none'], 0.5)+ # autoround.quantize_and_save(output_dir=quantized_model_path, inplace=False, format="gguf:q*_k_s")+ # from auto_round.eval.evaluation import simple_evaluate_user_model+ # gguf_file = os.listdir("saved")[0]+ # model = AutoModelForCausalLM.from_pretrained(quantized_model_path, gguf_file=gguf_file, device_map="auto")+ # result = simple_evaluate_user_model(model, self.tokenizer, batch_size=16, tasks="lambada_openai", eval_model_dtype="bf16")+ # self.assertGreater(result['results']['lambada_openai']['acc,none'], 0.5)shutil.rmtree("./saved", ignore_errors=True)-+def test_q5_k(self):- model_name = "Qwen/Qwen2.5-1.5B-Instruct"+ model_name = "Qwen/Qwen2.5-1.5B-Instruct"model = AutoModelForCausalLM.from_pretrained(model_name, torch_dtype="auto", trust_remote_code=True)autoround = AutoRound(model,@@ -156,7 +156,7 @@ class TestGGUF(unittest.TestCase):bits=5,group_size=32,sym=False,- iters=0,+ iters=1,data_type="int_asym_dq",super_group_size=8,super_bits=6@@ -169,9 +169,9 @@ class TestGGUF(unittest.TestCase):inputs = self.tokenizer(text, return_tensors="pt").to(model.device)print(self.tokenizer.decode(model.generate(**inputs, max_new_tokens=10)[0]))shutil.rmtree("./saved", ignore_errors=True)-+def test_q6_k(self):- model_name = "Qwen/Qwen2.5-1.5B-Instruct"+ model_name = "Qwen/Qwen2.5-1.5B-Instruct"model = AutoModelForCausalLM.from_pretrained(model_name, torch_dtype="auto", trust_remote_code=True)autoround = AutoRound(model,@@ -179,7 +179,7 @@ class TestGGUF(unittest.TestCase):bits=6,group_size=16,sym=True,- iters=0,+ iters=1,data_type="int_sym_dq",super_group_size=16,super_bits=8@@ -192,9 +192,9 @@ class TestGGUF(unittest.TestCase):inputs = self.tokenizer(text, return_tensors="pt").to(model.device)print(self.tokenizer.decode(model.generate(**inputs, max_new_tokens=10)[0]))shutil.rmtree("./saved", ignore_errors=True)-+def test_gguf_baseline(self):- model_name = "Qwen/Qwen2.5-1.5B-Instruct"+ model_name = "Qwen/Qwen2.5-1.5B-Instruct"model = AutoModelForCausalLM.from_pretrained(model_name, torch_dtype="auto", trust_remote_code=True)autoround = AutoRound(model,@@ -203,9 +203,10 @@ class TestGGUF(unittest.TestCase):group_size=16,sym=True,iters=0,- data_type="gguf_int_sym_dq",+ data_type="rtn_int_sym_dq",super_group_size=16,- super_bits=6+ super_bits=6,+ disable_opt_rtn=True,)quantized_model_path = "./saved"autoround.quantize_and_save(output_dir=quantized_model_path, inplace=False, format="fake")@@ -214,7 +215,7 @@ class TestGGUF(unittest.TestCase):inputs = self.tokenizer(text, return_tensors="pt").to(model.device)print(self.tokenizer.decode(model.generate(**inputs, max_new_tokens=10)[0]))shutil.rmtree("./saved", ignore_errors=True)-+autoround = AutoRound(model,self.tokenizer,@@ -224,7 +225,8 @@ class TestGGUF(unittest.TestCase):iters=0,data_type="int_asym_dq",super_group_size=8,- super_bits=6+ super_bits=6,+ disable_opt_rtn=True,)quantized_model_path = "./saved"autoround.quantize_and_save(output_dir=quantized_model_path, inplace=False, format="gguf:q5_k_s,fake")@@ -233,7 +235,7 @@ class TestGGUF(unittest.TestCase):inputs = self.tokenizer(text, return_tensors="pt").to(model.device)print(self.tokenizer.decode(model.generate(**inputs, max_new_tokens=10)[0]))shutil.rmtree("./saved", ignore_errors=True)- self.assertTrue(autoround.data_type == "gguf_int_asym_dq")+if __name__ == "__main__":unittest.main()