LLM inference in C/C++
C++ ggml
llama: various bug fixes (#26051)
Xuan-Son Nguyen298219f985b0
diff --git a/src/llama-grammar.cpp b/src/llama-grammar.cppindex badcbfd0fb..363644464b 100644--- a/src/llama-grammar.cpp+++ b/src/llama-grammar.cpp@@ -1139,6 +1139,18 @@ struct llama_grammar * llama_grammar_init_impl(vec_rules[i].push_back({LLAMA_GRETYPE_END, 0});}+ // Validate that all rule references point to valid rules+ for (size_t i = 0; i < n_rules; i++) {+ for (const auto & elem : vec_rules[i]) {+ if (elem.type == LLAMA_GRETYPE_RULE_REF) {+ if (elem.value >= n_rules || vec_rules[elem.value].empty()) {+ LLAMA_LOG_ERROR("invalid grammar: rule %zu references undefined rule %u\n", i, elem.value);+ return nullptr;+ }+ }+ }+ }+// Check for left recursionstd::vector<bool> rules_visited(n_rules);std::vector<bool> rules_in_progress(n_rules);diff --git a/src/llama-kv-cache.cpp b/src/llama-kv-cache.cppindex e70583e641..65a01e00e5 100644--- a/src/llama-kv-cache.cpp+++ b/src/llama-kv-cache.cpp@@ -2054,7 +2054,12 @@ void llama_kv_cache::state_read(llama_io_read_i & io, llama_seq_id seq_id, llamabool res = true;res = res && state_read_meta(io, strm, cell_count, sinfo, seq_id);- res = res && state_read_data(io, strm, cell_count, sinfo);++ try {+ res = res && state_read_data(io, strm, cell_count, sinfo);+ } catch (...) {+ res = false;+ }if (!res) {if (seq_id == -1) {diff --git a/src/llama-memory-recurrent.cpp b/src/llama-memory-recurrent.cppindex 3d6c6db876..ef82eb976c 100644--- a/src/llama-memory-recurrent.cpp+++ b/src/llama-memory-recurrent.cpp@@ -819,7 +819,12 @@ void llama_memory_recurrent::state_read(llama_io_read_i & io, llama_seq_id seq_ibool res = true;res = res && state_read_meta(io, cell_count, seq_id);- res = res && state_read_data(io, cell_count);++ try {+ res = res && state_read_data(io, cell_count);+ } catch (...) {+ res = false;+ }if (!res) {if (seq_id == -1) {diff --git a/src/llama-model.cpp b/src/llama-model.cppindex c9b290e99f..ecb44f3259 100644--- a/src/llama-model.cpp+++ b/src/llama-model.cpp@@ -1083,6 +1083,7 @@ void llama_model_base::load_hparams(llama_model_loader & ml) {ml.get_key(LLM_KV_ATTENTION_CAUSAL, hparams.causal_attn, false);ml.get_key(LLM_KV_POOLING_TYPE, hparams.pooling_type, false);ml.get_key(LLM_KV_BLOCK_COUNT, hparams.n_layer_all);+ GGML_ASSERT(hparams.n_layer_all > 0 && hparams.n_layer_all <= LLAMA_MAX_LAYERS);ml.get_key(LLM_KV_EXPERT_COUNT, hparams.n_expert, false);ml.get_key(LLM_KV_EXPERT_USED_COUNT, hparams.n_expert_used, false);ml.get_key(LLM_KV_EXPERT_GROUP_COUNT, hparams.n_expert_groups, false);diff --git a/src/llama-quant.cpp b/src/llama-quant.cppindex afecb76946..caf7733a5b 100644--- a/src/llama-quant.cpp+++ b/src/llama-quant.cpp@@ -1355,6 +1355,7 @@ llama_model * llama_quant_model_from_metadata(const llama_quant_model_desc * desmodel->hparams.n_embd_head_k_full = desc->n_embd_head_k;model->hparams.n_embd_head_v_full = desc->n_embd_head_v;model->hparams.n_layer_all = desc->n_layer;+ GGML_ASSERT(desc->n_layer > 0 && desc->n_layer <= LLAMA_MAX_LAYERS);model->hparams.n_expert = desc->n_expert;for (uint32_t i = 0; i < desc->n_layer; i++) {diff --git a/src/llama-sampler.cpp b/src/llama-sampler.cppindex 2370e91a14..6520e4181e 100644--- a/src/llama-sampler.cpp+++ b/src/llama-sampler.cpp@@ -263,6 +263,10 @@ static void llama_log_softmax(float * array, size_t size) {*/static void llama_sampler_temp_impl(llama_token_data_array * cur_p, float temp) {+ if (cur_p->size == 0) {+ return;+ }+if (temp <= 0.0f) {// find the token with the highest logit and set the rest to -infsize_t max_i = 0;diff --git a/src/llama-vocab.cpp b/src/llama-vocab.cppindex d926eee6ec..7b312d1d88 100644--- a/src/llama-vocab.cpp+++ b/src/llama-vocab.cpp@@ -1331,6 +1331,9 @@ struct llm_tokenizer_rwkv_session {token_id = node->value;token_length = position + 1;}+ if (position + 1 >= text.size()) {+ break;+ }node = node->traverse(text[++position]);}@@ -2865,6 +2868,11 @@ void llama_vocab::impl::load(llama_model_loader & ml, const LLM_KV & kv) {LLAMA_LOG_INFO("%s: printing all EOG tokens:\n", __func__);for (auto tid : special_eog_ids) {+ if (tid < 0 || tid >= (llama_token) id_to_token.size()) {+ LLAMA_LOG_WARN("%s: EOG token id %d is out of range (vocab size %zu), skipping\n",+ __func__, tid, id_to_token.size());+ continue;+ }auto & text = id_to_token[tid].text;LLAMA_LOG_INFO("%s: - %d ('%s')\n", __func__, tid, text.c_str());@@ -2899,6 +2907,9 @@ void llama_vocab::impl::load(llama_model_loader & ml, const LLM_KV & kv) {llama_token s_id = LLAMA_TOKEN_NULL;for (auto tid : special_eog_ids) {+ if (tid < 0 || tid >= (llama_token) id_to_token.size()) {+ continue;+ }const auto & text = id_to_token[tid].text;if (text == "<|tool_response>") {has_tool_response = true;@@ -4028,7 +4039,11 @@ int llama_vocab::find_bpe_rank(const std::string & token_left, const std::string}std::vector<std::string> llama_vocab::get_bpe_merges() const {- std::vector<std::string> result(pimpl->bpe_ranks.size());+ int max_rank = -1;+ for (const auto & pair : pimpl->bpe_ranks) {+ max_rank = std::max(max_rank, pair.second);+ }+ std::vector<std::string> result(max_rank + 1);for (const auto & pair : pimpl->bpe_ranks) {result[pair.second] = pair.first.first + " " + pair.first.second;