LLM inference in C/C++
C++ ggml
hybrid dna tokenizer fixes (#30825)
Kashif Rasul4c3c5810f87e
diff --git a/conversion/base.py b/conversion/base.pyindex 14a52f01d4..dec1d43222 100644--- a/conversion/base.py+++ b/conversion/base.py@@ -10,6 +10,7 @@ import jsonimport osimport reimport sys+import tempfilefrom enum import IntEnumfrom pathlib import Pathfrom hashlib import sha256@@ -2169,20 +2170,34 @@ class TextModel(ModelBase):special_vocab = gguf.SpecialVocab(self.dir_model, load_merges=True)special_vocab.add_to_gguf(self.gguf_writer)+ def _is_hybriddna(self) -> bool:+ tokenizer_config_file = self.dir_model / "tokenizer_config.json"+ if not tokenizer_config_file.is_file():+ return False+ with open(tokenizer_config_file, "r", encoding="utf-8") as f:+ return json.load(f).get("tokenizer_class") == "HybridDNATokenizer"+def _set_vocab_hybriddna(self):from transformers import AutoTokenizertokenizer = AutoTokenizer.from_pretrained(self.dir_model, trust_remote_code=True)- vocab_size = self.hparams.get("vocab_size", len(tokenizer.vocab)) # ty: ignore[unresolved-attribute]- assert max(tokenizer.vocab.values()) < vocab_size # ty: ignore[unresolved-attribute]+ vocab = tokenizer.get_vocab()+ vocab_size = self.hparams.get("vocab_size", len(vocab))+ assert max(vocab.values()) < vocab_size- reverse_vocab = {id_: encoded_tok for encoded_tok, id_ in tokenizer.vocab.items()} # ty: ignore[unresolved-attribute]- # k-mers can share text with a base-vocab BPE token (e.g. CCCCCC) and get- # dropped by get_vocab(); a reserved marker suffix (U+E000) keeps each- # k-mer's own id (llama.cpp strips it on detokenization)- for kmer in tokenizer.kmers: # ty: ignore[unresolved-attribute]+ reverse_vocab = {id_: encoded_tok for encoded_tok, id_ in vocab.items()}+ # k-mers can share text with a base-vocab BPE token (e.g. CCCCCC) and get dropped by get_vocab()+ # a reserved marker suffix (U+E000) keeps each k-mer's own id (llama.cpp strips it on detokenization)+ # tail="pad_n" (Carbon-14B) adds N-padded tail tokens ("CT" -> "CTNNNN") that get the same marker+ # llama.cpp detects them to pad with N and strips the padding on detokenization, tail="char" is unsupported+ tail = getattr(tokenizer, "tail", "pad_a")+ if tokenizer.k != 6 or tail not in ("pad_a", "pad_n"):+ raise NotImplementedError(f"HybridDNATokenizer with k={tokenizer.k}, tail={tail!r}")+ for kmer in [*tokenizer.kmers, *getattr(tokenizer, "npad_tokens", [])]:reverse_vocab[tokenizer.dna_token_to_id[kmer]] = kmer + "\ue000" # ty: ignore[unresolved-attribute]- added_vocab = tokenizer.get_added_vocab() # ty: ignore[unresolved-attribute]- added_tokens_decoder = tokenizer.added_tokens_decoder # ty: ignore[unresolved-attribute]+ # older HybridDNATokenizer versions (Carbon-500M/3B/8B) do not register the base tokenizer's added tokens (<|im_start|>, <think>, ...) as their own+ # but they are still encoded as special tokens: take their types from the base tokenizer+ base_tokenizer = getattr(tokenizer, "_base_tokenizer", tokenizer)+ added_tokens_decoder = {**base_tokenizer.added_tokens_decoder, **tokenizer.added_tokens_decoder} # ty: ignore[unresolved-attribute]tokens: list[str] = []toktypes: list[int] = []@@ -2192,7 +2207,7 @@ class TextModel(ModelBase):toktypes.append(gguf.TokenType.UNUSED)else:token: str = reverse_vocab[i]- if token in added_vocab:+ if i in added_tokens_decoder:if added_tokens_decoder[i].special or self.does_token_look_special(token):toktypes.append(gguf.TokenType.CONTROL)else:@@ -2204,10 +2219,19 @@ class TextModel(ModelBase):tokpre = self.get_vocab_base_pre(tokenizer)self.gguf_writer.add_tokenizer_model("hybriddna")self.gguf_writer.add_tokenizer_pre(tokpre)+ self.gguf_writer.add_dna_auto_tags(getattr(tokenizer, "auto_dna_tags", False))self.gguf_writer.add_token_list(tokens)self.gguf_writer.add_token_types(toktypes)special_vocab = gguf.SpecialVocab(self.dir_model, load_merges=True)+ if not special_vocab.merges:+ # Carbon-500M/3B/8B ship only tokenizer.py, no tokenizer.json+ # take the merges and special token ids from the base BPE tokenizer, keeping the model's own chat template (if any)+ with tempfile.TemporaryDirectory() as tmp:+ base_tokenizer.save_pretrained(tmp)+ base_special_vocab = gguf.SpecialVocab(Path(tmp), load_merges=True)+ base_special_vocab.chat_template = special_vocab.chat_template+ special_vocab = base_special_vocabspecial_vocab.add_to_gguf(self.gguf_writer)def _set_vocab_qwen(self):diff --git a/conversion/llama.py b/conversion/llama.pyindex 41d8c23092..b04f54d419 100644--- a/conversion/llama.py+++ b/conversion/llama.py@@ -129,8 +129,9 @@ class LlamaModel(TextModel):tokenizer_config_json = json.load(f)if (add_prefix_space := tokenizer_config_json.get("add_prefix_space")) is not None:self.gguf_writer.add_add_space_prefix(add_prefix_space)- if tokenizer_config_json.get("tokenizer_class") == "HybridDNATokenizer":- return self._set_vocab_hybriddna()++ if self._is_hybriddna():+ return self._set_vocab_hybriddna()try:self._set_vocab_sentencepiece()diff --git a/conversion/qwen.py b/conversion/qwen.pyindex 0c0d0c1edc..e1013dd531 100644--- a/conversion/qwen.py+++ b/conversion/qwen.py@@ -59,6 +59,10 @@ class Qwen2Model(TextModel):model_arch = gguf.MODEL_ARCH.QWEN2def set_vocab(self):+ # HuggingFaceBio/Carbon-14B: Qwen3 with the HybridDNATokenizer (Qwen3 BPE plus DNA 6-mers)+ if self._is_hybriddna():+ self._set_vocab_hybriddna()+ returntry:self._set_vocab_sentencepiece()except FileNotFoundError:diff --git a/gguf-py/gguf/constants.py b/gguf-py/gguf/constants.pyindex e098efae42..8c19e22dab 100644--- a/gguf-py/gguf/constants.py+++ b/gguf-py/gguf/constants.py@@ -366,6 +366,7 @@ class Keys:ADD_BOS = "tokenizer.ggml.add_bos_token"ADD_EOS = "tokenizer.ggml.add_eos_token"ADD_SEP = "tokenizer.ggml.add_sep_token"+ DNA_AUTO_TAGS = "tokenizer.ggml.dna_auto_tags"ADD_PREFIX = "tokenizer.ggml.add_space_prefix"REMOVE_EXTRA_WS = "tokenizer.ggml.remove_extra_whitespaces"PRECOMPILED_CHARSMAP = "tokenizer.ggml.precompiled_charsmap"diff --git a/gguf-py/gguf/gguf_writer.py b/gguf-py/gguf/gguf_writer.pyindex 93ec95eb64..7becd74af9 100644--- a/gguf-py/gguf/gguf_writer.py+++ b/gguf-py/gguf/gguf_writer.py@@ -1314,6 +1314,9 @@ class GGUFWriter:def add_add_sep_token(self, value: bool) -> None:self.add_bool(Keys.Tokenizer.ADD_SEP, value)+ def add_dna_auto_tags(self, value: bool) -> None:+ self.add_bool(Keys.Tokenizer.DNA_AUTO_TAGS, value)+def add_add_space_prefix(self, value: bool) -> None:self.add_bool(Keys.Tokenizer.ADD_PREFIX, value)diff --git a/src/llama-arch.cpp b/src/llama-arch.cppindex ffa35bdea7..961011a2c7 100644--- a/src/llama-arch.cpp+++ b/src/llama-arch.cpp@@ -430,6 +430,7 @@ static const std::map<llm_kv, const char *> LLM_KV_NAMES = {{ LLM_KV_TOKENIZER_ADD_BOS, "tokenizer.ggml.add_bos_token" },{ LLM_KV_TOKENIZER_ADD_EOS, "tokenizer.ggml.add_eos_token" },{ LLM_KV_TOKENIZER_ADD_SEP, "tokenizer.ggml.add_sep_token" },+ { LLM_KV_TOKENIZER_DNA_AUTO_TAGS, "tokenizer.ggml.dna_auto_tags" },{ LLM_KV_TOKENIZER_ADD_PREFIX, "tokenizer.ggml.add_space_prefix" },{ LLM_KV_TOKENIZER_REMOVE_EXTRA_WS, "tokenizer.ggml.remove_extra_whitespaces" },{ LLM_KV_TOKENIZER_PRECOMPILED_CHARSMAP, "tokenizer.ggml.precompiled_charsmap" },diff --git a/src/llama-arch.h b/src/llama-arch.hindex 14ce29ab32..3ea86c601d 100644--- a/src/llama-arch.h+++ b/src/llama-arch.h@@ -394,6 +394,7 @@ enum llm_kv {LLM_KV_TOKENIZER_ADD_BOS,LLM_KV_TOKENIZER_ADD_EOS,LLM_KV_TOKENIZER_ADD_SEP,+ LLM_KV_TOKENIZER_DNA_AUTO_TAGS,LLM_KV_TOKENIZER_ADD_PREFIX,LLM_KV_TOKENIZER_REMOVE_EXTRA_WS,LLM_KV_TOKENIZER_PRECOMPILED_CHARSMAP,diff --git a/src/llama-vocab.cpp b/src/llama-vocab.cppindex 05d2527dc0..a237a4b044 100644--- a/src/llama-vocab.cpp+++ b/src/llama-vocab.cpp@@ -1776,16 +1776,15 @@ private:// text at loadstatic const std::string dna_kmer_marker = "\xee\x80\x80";+// HybridDNATokenizer (HuggingFaceBio/Carbon-*): text inside <dna>...</dna> is cut into fixed 6-mers, everything else goes through the base BPE+// Mirrors HybridDNATokenizer.encode, with optional tagging of untagged DNA (auto_dna_tags, Carbon-14B) and tail="pad_n" vocab, whose dedicated tokens pad a trailing partial k-mer with N instead of A ("CT" -> "CTNNNN")struct llm_tokenizer_hybriddna_session : llm_tokenizer_bpe_session {llm_tokenizer_hybriddna_session(const llama_vocab & vocab, const llm_tokenizer_bpe & tokenizer) : llm_tokenizer_bpe_session{vocab, tokenizer}, vocab{vocab} {}void tokenize(const std::string & text, std::vector<llama_token> & output) override {- static const std::string open_tag = "<dna>";- static const std::string close_tag = "</dna>";-- const auto dna_begin_id = vocab.text_to_token(open_tag);- const auto dna_end_id = vocab.text_to_token(close_tag);- const auto dna_oov_id = vocab.text_to_token("<oov>");+ dna_begin_id = vocab.text_to_token(open_tag);+ dna_end_id = vocab.text_to_token(close_tag);+ dna_oov_id = vocab.text_to_token("<oov>");// Fall back to plain BPE if the DNA pieces aren't in the vocab.if (dna_begin_id == LLAMA_TOKEN_NULL || dna_end_id == LLAMA_TOKEN_NULL || dna_oov_id == LLAMA_TOKEN_NULL) {@@ -1793,63 +1792,187 @@ struct llm_tokenizer_hybriddna_session : llm_tokenizer_bpe_session {return;}- const size_t k = 6;- size_t pos = 0;+ pad_char = vocab.text_to_token("A" + std::string(k - 1, 'N') + dna_kmer_marker) != LLAMA_TOKEN_NULL ? 'N' : 'A';+ // split on explicit tags as HybridDNATokenizer._split_by_dna_tags does; only the text outside them is auto-tagged+ size_t pos = 0;while (pos < text.size()) {const size_t start = text.find(open_tag, pos);- if (start == std::string::npos) {- if (pos < text.size()) {- llm_tokenizer_bpe_session::tokenize(text.substr(pos), output);- }+ const size_t end = text.find(close_tag, pos);++ if (start == std::string::npos && end == std::string::npos) {+ tokenize_text(text.substr(pos), output);break;}- if (start > pos) {- llm_tokenizer_bpe_session::tokenize(text.substr(pos, start - pos), output);+ if (start == std::string::npos || end < start) {+ // as in _split_by_dna_tags, text before a </dna> without a matching <dna> is DNA (no begin token)+ tokenize_dna(text.substr(pos, end - pos), false, true, output);+ pos = end + close_tag.size();+ continue;}- output.push_back(dna_begin_id);- const size_t content_start = start + open_tag.size();- const size_t end = text.find(close_tag, content_start);- const size_t content_end = (end == std::string::npos) ? text.size() : end;-- emit_dna_kmers(text.substr(content_start, content_end - content_start), k, dna_oov_id, output);+ tokenize_text(text.substr(pos, start - pos), output);+ const size_t content_start = start + open_tag.size();if (end == std::string::npos) {+ // an unterminated <dna> runs to the end of the text (no end token)+ tokenize_dna(text.substr(content_start), true, false, output);break;}- output.push_back(dna_end_id);+ tokenize_dna(text.substr(content_start, end - content_start), true, true, output);pos = end + close_tag.size();}}private:- void emit_dna_kmers(const std::string & raw, size_t k, llama_token oov_id, std::vector<llama_token> & output) {- std::string seq = raw;- for (char & c : seq) {- if (c >= 'a' && c <= 'z') {- c = char(c - 32);+ static bool is_letter(char c) { return (c >= 'A' && c <= 'Z') || (c >= 'a' && c <= 'z'); }+ static bool is_acgt (char c) { return c == 'A' || c == 'C' || c == 'G' || c == 'T'; }+ static char to_upper (char c) { return c >= 'a' && c <= 'z' ? char(c - 32) : c; }++ // IUPAC nucleotide codes, either case (HybridDNATokenizer.IUPAC_RUN_RE)+ static bool is_iupac(char c) {+ c = to_upper(c);+ return std::string("ACGTRYSWKMBDHVN").find(c) != std::string::npos;+ }++ // HybridDNATokenizer._normalize_spaced_dna: join displays like "ACGTA CGTAC ..." when they hold at least twelve bases+ // (at least six chunks of 1-5 uppercase ACGT separated by spaces or tabs, not touching a letter)+ static std::string normalize_spaced_dna(const std::string & text) {+ const auto is_blank = [](char c) { return c == ' ' || c == '\t'; };+ std::string out;+ size_t p = 0;+ while (p < text.size()) {+ if (is_acgt(text[p]) && (p == 0 || !is_letter(text[p - 1]))) {+ size_t match_end = std::string::npos;+ size_t chunk = p;+ for (int n_chunks = 1; ; n_chunks++) {+ size_t chunk_end = chunk;+ while (chunk_end < text.size() && chunk_end - chunk < 5 && is_acgt(text[chunk_end])) {+ chunk_end++;+ }+ if (chunk_end == chunk) {+ break;+ }+ if (n_chunks >= 6 && (chunk_end == text.size() || !is_letter(text[chunk_end]))) {+ match_end = chunk_end; // the regex is greedy, so the last valid end wins+ }+ size_t next = chunk_end;+ while (next < text.size() && is_blank(text[next])) {+ next++;+ }+ if (next == chunk_end) {+ break;+ }+ chunk = next;+ }+ if (match_end != std::string::npos) {+ std::string seq;+ for (size_t i = p; i < match_end; i++) {+ if (!is_blank(text[i])) {+ seq += text[i];+ }+ }+ out += seq.size() >= 12 ? seq : text.substr(p, match_end - p);+ p = match_end;+ continue;+ }}+ out += text[p++];}+ return out;+ }- // k-mers carry the reserved marker suffix; a non-ACGT k-mer simply- // isn't in the vocab and falls back to <oov>- auto kmer_token = [&](const std::string & kmer) {- const auto tok = vocab.text_to_token(kmer + dna_kmer_marker);- return tok != LLAMA_TOKEN_NULL ? tok : oov_id;+ // text outside explicit tags: with auto_dna_tags, tag DNA runs as HybridDNATokenizer._auto_tag_text_segment does+ // (pure ACGT runs of at least 6 bases, or runs of at least 12 with at most 20% IUPAC ambiguity codes)+ void tokenize_text(const std::string & raw, std::vector<llama_token> & output) {+ auto bpe = [&](const std::string & piece) {+ if (!piece.empty()) {+ llm_tokenizer_bpe_session::tokenize(piece, output);+ }};+ if (!vocab.get_dna_auto_tags()) {+ bpe(raw);+ return;+ }+ const std::string text = normalize_spaced_dna(raw);+ size_t prev = 0;size_t i = 0;- for (; i + k <= seq.size(); i += k) {- output.push_back(kmer_token(seq.substr(i, k)));+ while (i < text.size()) {+ if (!is_iupac(text[i])) {+ i++;+ continue;+ }+ size_t j = i;+ size_t n_ambiguous = 0;+ for (; j < text.size() && is_iupac(text[j]); j++) {+ n_ambiguous += !is_acgt(to_upper(text[j]));+ }+ const size_t len = j - i;+ if ((n_ambiguous == 0 && len >= 6) || (n_ambiguous > 0 && len >= 12 && 5 * n_ambiguous <= len)) {+ bpe(text.substr(prev, i - prev));+ tokenize_dna(text.substr(i, len), true, true, output);+ prev = j;+ }+ i = j;+ }+ bpe(text.substr(prev));+ }++ // HybridDNATokenizer._process_dna_sequence: strip, uppercase, then one token per k bases+ // a k-mer with anything but ACGT is <oov>, and a trailing partial k-mer is right-padded with pad_char+ void tokenize_dna(const std::string & content, bool has_start, bool has_end, std::vector<llama_token> & output) {+ if (has_start) {+ output.push_back(dna_begin_id);+ }++ auto cpts = unicode_cpts_from_utf8(content);+ size_t first = 0;+ size_t last = cpts.size();+ while (first < last && unicode_cpt_flags_from_cpt(cpts[first]).is_whitespace) {+ first++;+ }+ while (last > first && unicode_cpt_flags_from_cpt(cpts[last - 1]).is_whitespace) {+ last--;+ }++ // k-mers carry the reserved marker suffix, so they never resolve to a base-vocab token+ auto kmer_token = [&](size_t from, size_t to) {+ std::string kmer;+ for (size_t i = from; i < to; i++) {+ const uint32_t c = cpts[i] < 128 ? (uint32_t) to_upper((char) cpts[i]) : cpts[i];+ if (c > 127 || !is_acgt((char) c)) {+ return dna_oov_id;+ }+ kmer += (char) c;+ }+ kmer.append(k - kmer.size(), pad_char);+ const auto tok = vocab.text_to_token(kmer + dna_kmer_marker);+ return tok != LLAMA_TOKEN_NULL ? tok : dna_oov_id;+ };++ size_t i = first;+ for (; i + k <= last; i += k) {+ output.push_back(kmer_token(i, i + k));+ }+ if (i < last) {+ output.push_back(kmer_token(i, last));}- if (i < seq.size()) {- std::string kmer = seq.substr(i);- kmer.append(k - kmer.size(), 'A');- output.push_back(kmer_token(kmer));++ if (has_end) {+ output.push_back(dna_end_id);}}+ const std::string open_tag = "<dna>";+ const std::string close_tag = "</dna>";+ const size_t k = 6;++ llama_token dna_begin_id = LLAMA_TOKEN_NULL;+ llama_token dna_end_id = LLAMA_TOKEN_NULL;+ llama_token dna_oov_id = LLAMA_TOKEN_NULL;+ char pad_char = 'A';+const llama_vocab & vocab;};@@ -1957,6 +2080,7 @@ struct llama_vocab::impl {bool add_bos = false;bool add_eos = false;bool add_sep = false;+ bool dna_auto_tags = false; // hybriddna: tag untagged DNA runs before tokenizingbool ignore_merges = false;bool clean_spaces = false; // clean_up_tokenization_spacesbool remove_extra_whitespaces = false;@@ -2689,6 +2813,12 @@ void llama_vocab::impl::load(llama_model_loader & ml, const LLM_KV & kv) {if (text.size() > dna_kmer_marker.size()&& text.compare(text.size() - dna_kmer_marker.size(), dna_kmer_marker.size(), dna_kmer_marker) == 0) {text.erase(text.size() - dna_kmer_marker.size());+ // tail="pad_n": "CTNNNN" detokenizes to its real bases "CT"+ // full k-mers are ACGT only, so only N-padded tail tokens end in N+ const size_t last_base = text.find_last_not_of('N');+ if (last_base != std::string::npos) {+ text.erase(last_base + 1);+ }}}}@@ -2781,6 +2911,9 @@ void llama_vocab::impl::load(llama_model_loader & ml, const LLM_KV & kv) {if (ml.get_key(LLM_KV_TOKENIZER_ADD_SEP, temp, false)) {add_sep = temp;}+ if (ml.get_key(LLM_KV_TOKENIZER_DNA_AUTO_TAGS, temp, false)) {+ dna_auto_tags = temp;+ }// workaround for Gemma 4// ref: https://github.com/ggml-org/llama.cpp/pull/21500@@ -4312,6 +4445,10 @@ bool llama_vocab::get_add_sep() const {return pimpl->add_sep;}+bool llama_vocab::get_dna_auto_tags() const {+ return pimpl->dna_auto_tags;+}+bool llama_vocab::get_ignore_merges() const {return pimpl->ignore_merges;}diff --git a/src/llama-vocab.h b/src/llama-vocab.hindex 898a23ae6e..415acd22bf 100644--- a/src/llama-vocab.h+++ b/src/llama-vocab.h@@ -149,6 +149,7 @@ struct llama_vocab {bool get_add_bos () const;bool get_add_eos () const;bool get_add_sep () const;+ bool get_dna_auto_tags () const;bool get_ignore_merges () const;bool get_clean_spaces () const;bool get_remove_extra_whitespaces () const;