LLM inference in C/C++
C++ ggml
convert: add option to create separate dspark GGUF (#26452)
Aman Gupta3581ba0cf591
diff --git a/conversion/__init__.py b/conversion/__init__.pyindex 1a47b851a0..534f9e309a 100644--- a/conversion/__init__.py+++ b/conversion/__init__.py@@ -55,6 +55,7 @@ TEXT_MODEL_MAP: dict[str, str] = {"DFlashDraftModel": "qwen","Qwen3DSparkModel": "qwen","DeepseekV4ForCausalLM": "deepseek",+ "DeepseekV4DSparkModel": "deepseek","DistilBertForMaskedLM": "bert","DistilBertForSequenceClassification": "bert","DistilBertModel": "bert",diff --git a/conversion/deepseek.py b/conversion/deepseek.pyindex 0bf69be3be..5b69e23437 100644--- a/conversion/deepseek.py+++ b/conversion/deepseek.py@@ -620,7 +620,8 @@ class DeepseekV4Model(TextModel):self.gguf_writer.add_hyper_connection_sinkhorn_iterations(hparams["hc_sinkhorn_iters"])self.gguf_writer.add_hyper_connection_epsilon(hparams["hc_eps"])self.gguf_writer.add_hash_layer_count(hparams["num_hash_layers"])- self.gguf_writer.add_embedding_length_out(hparams["hidden_size"] * hparams["hc_mult"])+ if self.model_arch == gguf.MODEL_ARCH.DEEPSEEK4:+ self.gguf_writer.add_embedding_length_out(hparams["hidden_size"] * hparams["hc_mult"])if self.mtp_only and (num_nextn_predict_layers := hparams.get("num_nextn_predict_layers", 0)) > 0:self.gguf_writer.add_nextn_predict_layers(num_nextn_predict_layers)@@ -878,3 +879,105 @@ class DeepseekV4Model(TextModel):super().prepare_tensors()self._is_mxfp4 = Trueself.ftype = gguf.LlamaFileType.MOSTLY_MXFP4_MOE++[email protected]("DeepseekV4DSparkModel")+class DeepseekV4DSparkModel(DeepseekV4Model):+ model_arch = gguf.MODEL_ARCH.DFLASH++ _DSPARK_ROOT_MAP: dict[str, tuple[gguf.MODEL_TENSOR, str]] = {+ "main_proj.weight": (gguf.MODEL_TENSOR.FC, ".weight"),+ "main_norm.weight": (gguf.MODEL_TENSOR.ENC_OUTPUT_NORM, ".weight"),+ "markov_head.markov_w1.weight": (gguf.MODEL_TENSOR.DSPARK_MARKOV_W1, ".weight"),+ "markov_head.markov_w2.weight": (gguf.MODEL_TENSOR.DSPARK_MARKOV_W2, ".weight"),+ "confidence_head.proj.weight": (gguf.MODEL_TENSOR.DSPARK_CONF_PROJ, ".weight"),+ }++ def __init__(self, *args, **kwargs):+ super().__init__(*args, **kwargs)++ self.block_count = 1 + max(+ int(match.group(1)) for name in self.model_tensors+ if (match := re.match(r"layers\.(\d+)\.", name))+ )+ self.tensor_map = gguf.get_tensor_name_map(self.model_arch, self.block_count)++ self.hparams["compress_ratios"] = [0] * self.block_count+ self.hparams["num_hash_layers"] = 0++ def index_tensors(self, remote_hf_model_id: str | None = None) -> dict[str, Callable[[], Tensor]]:+ if remote_hf_model_id is None:+ return super().index_tensors()++ with open(self.dir_model / "model.safetensors.index.json", "r", encoding="utf-8") as f:+ weight_map = json.load(f)["weight_map"]++ part_names = sorted({+ part_name for name, part_name in weight_map.items()+ if name.startswith("mtp.")+ })+ tensors: dict[str, Callable[[], Tensor]] = {}++ for part_name in part_names:+ from huggingface_hub import hf_hub_download++ logger.info("gguf: caching remote DSpark part '%s'", part_name)+ part_path = Path(hf_hub_download(repo_id=remote_hf_model_id, filename=part_name))+ with gguf.utility.SafetensorsLocal(part_path) as model_part:+ for name in model_part:+ data = model_part[name]+ data_gen = lambda data=data: LazyTorchTensor.from_local_tensor(data) # noqa: E731+ if titem := self.filter_tensors((name, data_gen)):+ tensor_name, tensor_gen = titem+ tensors[tensor_name] = tensor_gen++ return tensors++ @classmethod+ def filter_tensors(cls, item: tuple[str, Callable[[], Tensor]]) -> tuple[str, Callable[[], Tensor]] | None:+ name, gen = item+ if not name.startswith("mtp."):+ return None+ return super().filter_tensors((cls._rekey_mtp_tensor_name(name), gen))++ @staticmethod+ def _rekey_mtp_tensor_name(name: str) -> str:+ match = re.match(r"mtp\.(\d+)\.(.+)$", name)+ if match is None:+ raise ValueError(f"Unexpected DSpark tensor {name!r}")++ stage, rest = match.group(1), match.group(2)+ root_names = (+ "main_proj.scale",+ "norm.weight",+ "hc_head_fn",+ "hc_head_base",+ "hc_head_scale",+ )+ if rest in DeepseekV4DSparkModel._DSPARK_ROOT_MAP or rest in root_names:+ return rest+ return f"layers.{stage}.{rest}"++ def _map_dsv4_tensor_name(self, name: str, bid: int | None) -> tuple[gguf.MODEL_TENSOR, str]:+ if name in self._DSPARK_ROOT_MAP:+ return self._DSPARK_ROOT_MAP[name]+ return super()._map_dsv4_tensor_name(name, bid)++ def set_vocab(self):+ if self.target_model_dir is None:+ raise ValueError("DeepSeek-V4 DSpark requires --target-model-dir with the target tokenizer")++ original_dir = self.dir_model+ try:+ self.dir_model = self.target_model_dir+ super().set_vocab()+ finally:+ self.dir_model = original_dir++ self.gguf_writer.add_mask_token_id(self.hparams["dspark_noise_token_id"])++ def set_gguf_parameters(self):+ super().set_gguf_parameters()++ self.gguf_writer.add_block_size(self.hparams["dspark_block_size"])+ self.gguf_writer.add_target_layers([layer + 1 for layer in self.hparams["dspark_target_layer_ids"]])diff --git a/convert_hf_to_gguf.py b/convert_hf_to_gguf.pyindex 2c5e62a16f..78ad26c656 100755--- a/convert_hf_to_gguf.py+++ b/convert_hf_to_gguf.py@@ -122,8 +122,12 @@ def parse_args() -> argparse.Namespace:help="Export only the multi-token prediction (MTP) head as a separate GGUF, suitable for use as a speculative draft. An 'mtp-' prefix will be added to the output file name.",)parser.add_argument(- "--no-mtp", action="store_true",- help="Exclude the multi-token prediction (MTP) head from the converted GGUF. Pair with --mtp on a second run to publish trunk and MTP as two files. Note: the split form duplicates embeddings, but even though the bundled default is more space-efficient overall, this allows differing quantization which may be more performant.",+ "--no-nextn", "--no-mtp", dest="no_mtp", action="store_true",+ help="Exclude NextN speculative draft tensors from the converted GGUF. Pair with --mtp or --dspark on a second run to publish target and draft as two files.",+ )+ parser.add_argument(+ "--dspark", action="store_true",+ help="Export only the DeepSeek-V4 DSpark draft tensors as a separate GGUF.",)parser.add_argument("--mistral-format", action="store_true",@@ -254,13 +258,20 @@ def main() -> None:from conversion.mistral import MistralModelmodel_class = MistralModel- if args.mtp and args.no_mtp:- logger.error("--mtp and --no-mtp are mutually exclusive")+ if sum((args.mtp, args.no_mtp, args.dspark)) > 1:+ logger.error("--mtp, --no-nextn, and --dspark are mutually exclusive")sys.exit(1)+ if args.dspark:+ if is_mistral_format or model_architecture != "DeepseekV4ForCausalLM":+ logger.error("--dspark is only supported for DeepseekV4ForCausalLM")+ sys.exit(1)+ from conversion.deepseek import DeepseekV4DSparkModel+ model_class = DeepseekV4DSparkModel+if args.mtp or args.no_mtp:if not model_class.supports_mtp_export:- logger.error("--mtp / --no-mtp are not supported for %s", model_architecture)+ logger.error("--mtp / --no-nextn are not supported for %s", model_architecture)sys.exit(1)if args.no_mtp:model_class.no_mtp = Truediff --git a/gguf-py/gguf/constants.py b/gguf-py/gguf/constants.pyindex 978231091c..a0e55781ba 100644--- a/gguf-py/gguf/constants.py+++ b/gguf-py/gguf/constants.py@@ -4383,10 +4383,35 @@ MODEL_TENSORS: dict[MODEL_ARCH, list[MODEL_TENSOR]] = {MODEL_TENSOR.ATTN_OUT,MODEL_TENSOR.ATTN_Q_NORM,MODEL_TENSOR.ATTN_K_NORM,+ MODEL_TENSOR.ATTN_SINKS,+ MODEL_TENSOR.ATTN_Q_A,+ MODEL_TENSOR.ATTN_Q_B,+ MODEL_TENSOR.ATTN_Q_A_NORM,+ MODEL_TENSOR.ATTN_KV,+ MODEL_TENSOR.ATTN_KV_NORM,+ MODEL_TENSOR.ATTN_OUT_A,+ MODEL_TENSOR.ATTN_OUT_B,+ MODEL_TENSOR.HC_ATTN_FN,+ MODEL_TENSOR.HC_ATTN_BASE,+ MODEL_TENSOR.HC_ATTN_SCALE,+ MODEL_TENSOR.HC_FFN_FN,+ MODEL_TENSOR.HC_FFN_BASE,+ MODEL_TENSOR.HC_FFN_SCALE,+ MODEL_TENSOR.HC_HEAD_FN,+ MODEL_TENSOR.HC_HEAD_BASE,+ MODEL_TENSOR.HC_HEAD_SCALE,MODEL_TENSOR.FFN_NORM,MODEL_TENSOR.FFN_GATE,MODEL_TENSOR.FFN_DOWN,MODEL_TENSOR.FFN_UP,+ MODEL_TENSOR.FFN_GATE_INP,+ MODEL_TENSOR.FFN_EXP_PROBS_B,+ MODEL_TENSOR.FFN_GATE_EXP,+ MODEL_TENSOR.FFN_DOWN_EXP,+ MODEL_TENSOR.FFN_UP_EXP,+ MODEL_TENSOR.FFN_GATE_SHEXP,+ MODEL_TENSOR.FFN_DOWN_SHEXP,+ MODEL_TENSOR.FFN_UP_SHEXP,MODEL_TENSOR.FC,MODEL_TENSOR.ENC_OUTPUT_NORM,# optional DSpark heads