LLM inference in C/C++
C++ ggml
vocab : validate default special token ids (#26506)
Sigbjørn Skjærete8e06f78e253
diff --git a/src/llama-vocab.cpp b/src/llama-vocab.cppindex 443cd46408..10032a8c64 100644--- a/src/llama-vocab.cpp+++ b/src/llama-vocab.cpp@@ -2532,6 +2532,12 @@ void llama_vocab::impl::load(llama_model_loader & ml, const LLM_KV & kv) {const std::string & key = kv(std::get<0>(it));int32_t & id = std::get<1>(it);+ if (id >= 0 && static_cast<size_t>(id) >= id_to_token.size()) {+ LLAMA_LOG_WARN("%s: default special token '%s' = %d out of vocab range, disabling\n",+ __func__, key.c_str(), id);+ id = LLAMA_TOKEN_NULL;+ }+uint32_t new_id;if (!ml.get_key(std::get<0>(it), new_id, false)) {continue;