Skip to content

LLM inference in C/C++

C++ ggml

llama: various bug fixes (#26051)

Xuan-Son Nguyen298219f985b0
diff --git a/src/llama-grammar.cpp b/src/llama-grammar.cpp
index badcbfd0fb..363644464b 100644
--- a/src/llama-grammar.cpp
+++ b/src/llama-grammar.cpp
@@ -1139,6 +1139,18 @@ struct llama_grammar * llama_grammar_init_impl(
vec_rules[i].push_back({LLAMA_GRETYPE_END, 0});
}
+ // Validate that all rule references point to valid rules
+ for (size_t i = 0; i < n_rules; i++) {
+ for (const auto & elem : vec_rules[i]) {
+ if (elem.type == LLAMA_GRETYPE_RULE_REF) {
+ if (elem.value >= n_rules || vec_rules[elem.value].empty()) {
+ LLAMA_LOG_ERROR("invalid grammar: rule %zu references undefined rule %u\n", i, elem.value);
+ return nullptr;
+ }
+ }
+ }
+ }
+
// Check for left recursion
std::vector<bool> rules_visited(n_rules);
std::vector<bool> rules_in_progress(n_rules);
diff --git a/src/llama-kv-cache.cpp b/src/llama-kv-cache.cpp
index e70583e641..65a01e00e5 100644
--- a/src/llama-kv-cache.cpp
+++ b/src/llama-kv-cache.cpp
@@ -2054,7 +2054,12 @@ void llama_kv_cache::state_read(llama_io_read_i & io, llama_seq_id seq_id, llama
bool res = true;
res = res && state_read_meta(io, strm, cell_count, sinfo, seq_id);
- res = res && state_read_data(io, strm, cell_count, sinfo);
+
+ try {
+ res = res && state_read_data(io, strm, cell_count, sinfo);
+ } catch (...) {
+ res = false;
+ }
if (!res) {
if (seq_id == -1) {
diff --git a/src/llama-memory-recurrent.cpp b/src/llama-memory-recurrent.cpp
index 3d6c6db876..ef82eb976c 100644
--- a/src/llama-memory-recurrent.cpp
+++ b/src/llama-memory-recurrent.cpp
@@ -819,7 +819,12 @@ void llama_memory_recurrent::state_read(llama_io_read_i & io, llama_seq_id seq_i
bool res = true;
res = res && state_read_meta(io, cell_count, seq_id);
- res = res && state_read_data(io, cell_count);
+
+ try {
+ res = res && state_read_data(io, cell_count);
+ } catch (...) {
+ res = false;
+ }
if (!res) {
if (seq_id == -1) {
diff --git a/src/llama-model.cpp b/src/llama-model.cpp
index c9b290e99f..ecb44f3259 100644
--- a/src/llama-model.cpp
+++ b/src/llama-model.cpp
@@ -1083,6 +1083,7 @@ void llama_model_base::load_hparams(llama_model_loader & ml) {
ml.get_key(LLM_KV_ATTENTION_CAUSAL, hparams.causal_attn, false);
ml.get_key(LLM_KV_POOLING_TYPE, hparams.pooling_type, false);
ml.get_key(LLM_KV_BLOCK_COUNT, hparams.n_layer_all);
+ GGML_ASSERT(hparams.n_layer_all > 0 && hparams.n_layer_all <= LLAMA_MAX_LAYERS);
ml.get_key(LLM_KV_EXPERT_COUNT, hparams.n_expert, false);
ml.get_key(LLM_KV_EXPERT_USED_COUNT, hparams.n_expert_used, false);
ml.get_key(LLM_KV_EXPERT_GROUP_COUNT, hparams.n_expert_groups, false);
diff --git a/src/llama-quant.cpp b/src/llama-quant.cpp
index afecb76946..caf7733a5b 100644
--- a/src/llama-quant.cpp
+++ b/src/llama-quant.cpp
@@ -1355,6 +1355,7 @@ llama_model * llama_quant_model_from_metadata(const llama_quant_model_desc * des
model->hparams.n_embd_head_k_full = desc->n_embd_head_k;
model->hparams.n_embd_head_v_full = desc->n_embd_head_v;
model->hparams.n_layer_all = desc->n_layer;
+ GGML_ASSERT(desc->n_layer > 0 && desc->n_layer <= LLAMA_MAX_LAYERS);
model->hparams.n_expert = desc->n_expert;
for (uint32_t i = 0; i < desc->n_layer; i++) {
diff --git a/src/llama-sampler.cpp b/src/llama-sampler.cpp
index 2370e91a14..6520e4181e 100644
--- a/src/llama-sampler.cpp
+++ b/src/llama-sampler.cpp
@@ -263,6 +263,10 @@ static void llama_log_softmax(float * array, size_t size) {
*/
static void llama_sampler_temp_impl(llama_token_data_array * cur_p, float temp) {
+ if (cur_p->size == 0) {
+ return;
+ }
+
if (temp <= 0.0f) {
// find the token with the highest logit and set the rest to -inf
size_t max_i = 0;
diff --git a/src/llama-vocab.cpp b/src/llama-vocab.cpp
index d926eee6ec..7b312d1d88 100644
--- a/src/llama-vocab.cpp
+++ b/src/llama-vocab.cpp
@@ -1331,6 +1331,9 @@ struct llm_tokenizer_rwkv_session {
token_id = node->value;
token_length = position + 1;
}
+ if (position + 1 >= text.size()) {
+ break;
+ }
node = node->traverse(text[++position]);
}
@@ -2865,6 +2868,11 @@ void llama_vocab::impl::load(llama_model_loader & ml, const LLM_KV & kv) {
LLAMA_LOG_INFO("%s: printing all EOG tokens:\n", __func__);
for (auto tid : special_eog_ids) {
+ if (tid < 0 || tid >= (llama_token) id_to_token.size()) {
+ LLAMA_LOG_WARN("%s: EOG token id %d is out of range (vocab size %zu), skipping\n",
+ __func__, tid, id_to_token.size());
+ continue;
+ }
auto & text = id_to_token[tid].text;
LLAMA_LOG_INFO("%s: - %d ('%s')\n", __func__, tid, text.c_str());
@@ -2899,6 +2907,9 @@ void llama_vocab::impl::load(llama_model_loader & ml, const LLM_KV & kv) {
llama_token s_id = LLAMA_TOKEN_NULL;
for (auto tid : special_eog_ids) {
+ if (tid < 0 || tid >= (llama_token) id_to_token.size()) {
+ continue;
+ }
const auto & text = id_to_token[tid].text;
if (text == "<|tool_response>") {
has_tool_response = true;
@@ -4028,7 +4039,11 @@ int llama_vocab::find_bpe_rank(const std::string & token_left, const std::string
}
std::vector<std::string> llama_vocab::get_bpe_merges() const {
- std::vector<std::string> result(pimpl->bpe_ranks.size());
+ int max_rank = -1;
+ for (const auto & pair : pimpl->bpe_ranks) {
+ max_rank = std::max(max_rank, pair.second);
+ }
+ std::vector<std::string> result(max_rank + 1);
for (const auto & pair : pimpl->bpe_ranks) {
result[pair.second] = pair.first.first + " " + pair.first.second;