LLM inference in C/C++
C++ ggml
common : read ngram cache parts by reference instead of copying (#30061)
hayder13b80489204c
diff --git a/common/ngram-cache.cpp b/common/ngram-cache.cppindex dce54b3647..419a0ebc98 100644--- a/common/ngram-cache.cpp+++ b/common/ngram-cache.cpp@@ -68,7 +68,7 @@ static llama_token try_draft(common_ngram_cache & nc_static, const common_ngramif (part_static_it == nc_static.end()) {return LLAMA_TOKEN_NULL;}- const common_ngram_cache_part part_static = part_static_it->second;+ const common_ngram_cache_part & part_static = part_static_it->second;int max_count_static = 0;int sum_count_static = 0;@@ -96,7 +96,7 @@ static llama_token try_draft(common_ngram_cache & nc_static, const common_ngram// Try to draft a token from primary cache (context/dynamic), validate with static cache:static llama_token try_draft(- common_ngram_cache & nc_primary, const std::vector<common_ngram> & ngrams_primary, common_ngram_cache_part & part_static,+ common_ngram_cache & nc_primary, const std::vector<common_ngram> & ngrams_primary, const common_ngram_cache_part & part_static,const int * min_sample_size, const int * min_percent) {llama_token drafted_token = LLAMA_TOKEN_NULL;@@ -108,7 +108,7 @@ static llama_token try_draft(if (part_primary_it == nc_primary.end()) {continue;}- const common_ngram_cache_part part_primary = part_primary_it->second;+ const common_ngram_cache_part & part_primary = part_primary_it->second;int max_count_primary = 0;int max_count_static = 0;@@ -118,7 +118,7 @@ static llama_token try_draft(for (std::pair<llama_token, int> token_count_primary : part_primary) {const llama_token token = token_count_primary.first;- common_ngram_cache_part::iterator token_count_static_it = part_static.find(token);+ common_ngram_cache_part::const_iterator token_count_static_it = part_static.find(token);const int32_t count_primary = token_count_primary.second;const int32_t count_static = token_count_static_it != part_static.end() ? 100*token_count_static_it->second : 1;@@ -162,11 +162,9 @@ void common_ngram_cache_draft(for (int j = ngram_start_static; j < ngram_start_static + LLAMA_NGRAM_STATIC; ++j) {ngram_static.tokens[j-ngram_start_static] = get_token(inp, draft, j);}- common_ngram_cache::iterator part_static_it = nc_static.find(ngram_static);- common_ngram_cache_part part_static;- if (part_static_it != nc_static.end()) {- part_static = part_static_it->second;- }+ static const common_ngram_cache_part part_static_empty;+ common_ngram_cache::const_iterator part_static_it = nc_static.find(ngram_static);+ const common_ngram_cache_part & part_static = part_static_it != nc_static.end() ? part_static_it->second : part_static_empty;// cd = context + dynamicstd::vector<common_ngram> ngrams_cd;