LLM inference in C/C++
C++ ggml
convert : add FP8 to Q8 conversion (#23250)
ynankanic5229087a515
diff --git a/conversion/base.py b/conversion/base.pyindex e1c15a9580..9cddd1340f 100644--- a/conversion/base.py+++ b/conversion/base.py@@ -119,7 +119,8 @@ class ModelBase:small_first_shard: bool = False, hparams: dict[str, Any] | None = None, remote_hf_model_id: str | None = None,disable_mistral_community_chat_template: bool = False,sentence_transformers_dense_modules: bool = False,- fuse_gate_up_exps: bool = False):+ fuse_gate_up_exps: bool = False,+ fp8_as_q8: bool = False):if type(self) is ModelBase or \type(self) is TextModel or \type(self) is MmprojModel:@@ -148,6 +149,8 @@ class ModelBase:self.dir_model_card = dir_model # overridden in convert_lora_to_gguf.pyself._is_nvfp4 = Falseself._is_mxfp4 = False+ self._fp8_as_q8 = fp8_as_q8+ self._fp8_dequantized: set[str] = set()# Apply heuristics to figure out typical tensor encoding based on first tensor's dtype# NOTE: can't use field "torch_dtype" in config.json, because some finetunes lie.@@ -429,6 +432,8 @@ class ModelBase:s = self.model_tensors[name]self.model_tensors[weight_name] = lambda w=w, s=s, bs=block_size: dequant_simple(w(), s(), bs)tensors_to_remove.append(name)+ if self._fp8_as_q8:+ self._fp8_dequantized.add(weight_name)if name.endswith(".activation_scale"): # unusedtensors_to_remove.append(name)if name.endswith("_activation_scale"): # Mistral-Small-4-119B-2602, unused@@ -440,6 +445,8 @@ class ModelBase:s = self.model_tensors[name]self.model_tensors[weight_name] = lambda w=w, s=s, bs=block_size: dequant_simple(w(), s(), bs)tensors_to_remove.append(name)+ if self._fp8_as_q8:+ self._fp8_dequantized.add(weight_name)if name.endswith(".qscale_act"):tensors_to_remove.append(name)elif quant_method == "gptq":@@ -483,6 +490,11 @@ class ModelBase:strategy = weight_config.get("strategy")assert strategy == "channel" or strategy == "block"assert weight_config.get("group_size") is None # didn't find a model using this yet+ is_fp8 = (+ quant_format == "float-quantized"+ and weight_config.get("type") == "float"+ and weight_config.get("num_bits") == 8+ )for name in self.model_tensors.keys():if name.endswith(".weight_scale"):weight_name = name.removesuffix("_scale")@@ -490,6 +502,8 @@ class ModelBase:s = self.model_tensors[name]self.model_tensors[weight_name] = lambda w=w, s=s: dequant_simple(w(), s(), block_size)tensors_to_remove.append(name)+ if self._fp8_as_q8 and is_fp8:+ self._fp8_dequantized.add(weight_name)elif quant_format == "pack-quantized":assert weight_config.get("strategy") == "group"assert weight_config.get("type", "int") == "int"@@ -524,10 +538,18 @@ class ModelBase:for name in self.model_tensors.keys():if name.endswith(".weight_scale"):weight_name = name.removesuffix("_scale")+ if weight_name not in self.model_tensors:+ tensors_to_remove.append(name)+ continuew = self.model_tensors[weight_name]s = self.model_tensors[name]+ is_fp8_weight = False+ if self._fp8_as_q8:+ is_fp8_weight = w().dtype in (torch.float8_e4m3fn, torch.float8_e5m2)self.model_tensors[weight_name] = lambda w=w, s=s: dequant_simple(w(), s(), None)tensors_to_remove.append(name)+ if is_fp8_weight:+ self._fp8_dequantized.add(weight_name)if name.endswith((".input_scale", ".k_scale", ".v_scale")):tensors_to_remove.append(name)elif quant_method is not None:@@ -615,8 +637,10 @@ class ModelBase:return [(new_name, data_torch)]def tensor_force_quant(self, name: str, new_name: str, bid: int | None, n_dims: int) -> gguf.GGMLQuantizationType | bool:- del name, new_name, bid, n_dims # unused-+ del new_name, bid # unused+ # Force FP8-original tensors to Q8_0 when requested; Q8_0 is faster than F16/BF16.+ if self._fp8_as_q8 and name in self._fp8_dequantized and n_dims >= 2:+ return gguf.GGMLQuantizationType.Q8_0return False# some models need extra generated tensors (like rope_freqs)@@ -791,7 +815,7 @@ class ModelBase:if quant_algo != "NVFP4":if nvfp4_compressed_tensors:quant_algo = "NVFP4"- elif any(v.get("quant_algo") == "NVFP4" for v in quant_layers.values() if isinstance(v, dict)):+ elif any(str(v.get("quant_algo")).endswith("NVFP4") for v in quant_layers.values() if isinstance(v, dict)):quant_algo = "NVFP4"self._is_nvfp4 = quant_algo == "NVFP4"@@ -2417,10 +2441,9 @@ class MmprojModel(ModelBase):raise KeyError(f"could not find any of: {keys}")def tensor_force_quant(self, name, new_name, bid, n_dims):- del bid, name, n_dims # unusedif ".patch_embd.weight" in new_name or ".patch_merger.weight" in new_name:return gguf.GGMLQuantizationType.F16 if self.ftype == gguf.LlamaFileType.MOSTLY_F16 else gguf.GGMLQuantizationType.F32- return False+ return super().tensor_force_quant(name, new_name, bid, n_dims)class LazyTorchTensor(gguf.LazyBase):diff --git a/convert_hf_to_gguf.py b/convert_hf_to_gguf.pyindex 1d18a1bf91..8552755356 100755--- a/convert_hf_to_gguf.py+++ b/convert_hf_to_gguf.py@@ -148,6 +148,10 @@ def parse_args() -> argparse.Namespace:"--fuse-gate-up-exps", action="store_true",help="Fuse gate_exps and up_exps tensors into a single gate_up_exps tensor for MoE models.",)+ parser.add_argument(+ "--fp8-as-q8", action="store_true",+ help="Store tensors dequantized from FP8 as Q8_0 instead of BF16/F16.",+ )args = parser.parse_args()if not args.print_supported_models and args.model is None:@@ -264,7 +268,8 @@ def main() -> None:small_first_shard=args.no_tensor_first_split,remote_hf_model_id=hf_repo_id, disable_mistral_community_chat_template=disable_mistral_community_chat_template,sentence_transformers_dense_modules=args.sentence_transformers_dense_modules,- fuse_gate_up_exps=args.fuse_gate_up_exps+ fuse_gate_up_exps=args.fuse_gate_up_exps,+ fp8_as_q8=args.fp8_as_q8,)if args.vocab_only: