LLM inference in C/C++
C++ ggml
spec : support DFlash for HunyuanOCR (#28890)
wendadawen828fdf282e19
diff --git a/conversion/hunyuan.py b/conversion/hunyuan.pyindex ee1a106545..58dee2ec8b 100644--- a/conversion/hunyuan.py+++ b/conversion/hunyuan.py@@ -159,32 +159,14 @@ class HunYuanMoEModel(TextModel):class HunYuanModel(TextModel):model_arch = gguf.MODEL_ARCH.HUNYUAN_DENSE- def _get_eod_token_id(self) -> int | None:- """Get the actual end-of-generation token from config (eod_token_id)."""- return self.hparams.get("eod_token_id")-- def _get_eot_token_id(self) -> int | None:- """Get the end-of-turn token from generation_config.json.- This is the first entry in eos_token_id when it's a list."""- gen_cfg_path = self.dir_model / "generation_config.json"- if gen_cfg_path.is_file():- with open(gen_cfg_path, encoding="utf-8") as f:- gen_cfg = json.load(f)- eos = gen_cfg.get("eos_token_id")- if isinstance(eos, list) and len(eos) >= 2:- return eos[0]- return None-- def _fix_special_tokens(self):- """Fix EOS/EOT tokens that are incorrect in upstream configs."""- eod_id = self._get_eod_token_id()- if eod_id is not None:- self.gguf_writer.add_eos_token_id(eod_id)- eot_id = self._get_eot_token_id()- if eot_id is not None:- self.gguf_writer.add_eot_token_id(eot_id)-def set_vocab(self):+ # Also called by draft models (e.g. DFlash), with dir_model pointing at+ # the target model.+ config = ModelBase.load_hparams(self.dir_model, self.is_mistral_format)+ config = {**config, **config.get("text_config", {})}+ self.hparams["pad_token_id"] = config.get("pad_token_id")+ self.hparams["eod_token_id"] = config.get("eod_token_id")+if (self.dir_model / "tokenizer.json").is_file():tokens, toktypes, tokpre = self.get_vocab_base()self.gguf_writer.add_tokenizer_model("gpt2")@@ -199,7 +181,6 @@ class HunYuanModel(TextModel):token_types = ('bos', 'eos', 'unk', 'sep', 'cls', 'mask')special_vocab = gguf.SpecialVocab(self.dir_model, load_merges=True, special_token_types=token_types)special_vocab.add_to_gguf(self.gguf_writer)- self._fix_special_tokens()else:from transformers import AutoTokenizertokenizer = AutoTokenizer.from_pretrained(self.dir_model, trust_remote_code=True)@@ -251,7 +232,18 @@ class HunYuanModel(TextModel):# FIX for BOS token: Overwrite incorrect id read from config.jsonif self.hparams['hidden_size'] == 4096:self.gguf_writer.add_bos_token_id(127958) # only for 7b dense, fix <|bos|> token- self._fix_special_tokens()++ # Fix EOS/EOT tokens that are incorrect in upstream configs.+ eod_id = self.hparams.get("eod_token_id")+ if eod_id is not None:+ self.gguf_writer.add_eos_token_id(eod_id)++ gen_cfg = self.dir_model / "generation_config.json"+ if gen_cfg.is_file():+ with open(gen_cfg, encoding="utf-8") as f:+ eos = json.load(f).get("eos_token_id")+ if isinstance(eos, list) and len(eos) >= 2:+ self.gguf_writer.add_eot_token_id(eos[0])def set_gguf_parameters(self):# Some HunYuanVL variants set num_experts=1 (not real MoE);diff --git a/src/models/hunyuan-vl.cpp b/src/models/hunyuan-vl.cppindex da9bb74de7..18b6eaf8cd 100644--- a/src/models/hunyuan-vl.cpp+++ b/src/models/hunyuan-vl.cpp@@ -83,6 +83,8 @@ llama_model_hunyuan_vl::graph::graph(const llama_model & model, const llm_graph_ggml_tensor * inp_out_ids = build_inp_out_ids();for (int il = 0; il < n_layer; ++il) {+ res->t_layer_inp[il] = inpL;+ggml_tensor * inpSA = inpL;// norm