LLM inference in C/C++
C++ ggml
spec : save the dynamic ngram cache file (#22055)
Peter Sideris83c2d012a273
diff --git a/common/speculative.cpp b/common/speculative.cppindex d8afd40954..5a0b5b5123 100644--- a/common/speculative.cpp+++ b/common/speculative.cpp@@ -16,6 +16,7 @@#include <cassert>#include <cmath>#include <cstring>+#include <fstream>#include <iomanip>#include <map>#include <cinttypes>@@ -2152,9 +2153,6 @@ struct common_speculative_impl_ngram_cache : public common_speculative_impl {uint16_t n_draft;- bool save_dynamic;- bool save_static;-struct seq_info {size_t cache_size = 0; // number of tokens in n-gram cache@@ -2168,54 +2166,72 @@ struct common_speculative_impl_ngram_cache : public common_speculative_impl {common_speculative_impl_ngram_cache(const common_params_speculative & params,uint32_t n_seq,- uint16_t n_draft,- const std::string & path_static,- const std::string & path_dynamic,- bool save_dynamic,- bool save_static)+ uint16_t n_draft): common_speculative_impl(COMMON_SPECULATIVE_TYPE_NGRAM_CACHE, n_seq, n_draft), params(params.ngram_cache), n_draft(n_draft)- , save_dynamic(save_dynamic)- , save_static(save_static){+ const auto & ngram = params.ngram_cache;+SPC_TRC("%s", "adding speculative implementation 'ngram-cache'\n");SPC_TRC("- n_draft=%d, cache_static=%s, cache_dynamic=%s\n",n_draft,- path_static.empty() ? "none" : path_static.c_str(),- path_dynamic.empty() ? "none" : path_dynamic.c_str());+ ngram.lookup_cache_static.empty() ? "none" : ngram.lookup_cache_static.c_str(),+ ngram.lookup_cache_dynamic.empty() ? "none" : ngram.lookup_cache_dynamic.c_str());sinfos.resize(n_seq);- if (!path_static.empty()) {+ if (!ngram.lookup_cache_static.empty()) {try {- auto ngram_cache_static = common_ngram_cache_load(path_static);+ auto ngram_cache_static = common_ngram_cache_load(ngram.lookup_cache_static);for (auto & sinfo : sinfos) {sinfo.ngram_cache_static = ngram_cache_static;}- } catch (...) {- SPC_ERR("failed to open static lookup cache: %s", path_static.c_str());+ } catch (const std::ifstream::failure &) {+ SPC_ERR("failed to open static lookup cache: %s\n", ngram.lookup_cache_static.c_str());GGML_ABORT("Couldn't read static lookup cache");}}- if (!path_dynamic.empty()) {+ if (!ngram.lookup_cache_dynamic.empty()) {try {- auto ngram_cache_dynamic = common_ngram_cache_load(path_dynamic);+ auto ngram_cache_dynamic = common_ngram_cache_load(ngram.lookup_cache_dynamic);for (auto & sinfo : sinfos) {sinfo.ngram_cache_dynamic = ngram_cache_dynamic;}- } catch (...) {- SPC_ERR("failed to open dynamic lookup cache: %s", path_dynamic.c_str());- GGML_ABORT("Couldn't read dynamic lookup cache");+ } catch (const std::ifstream::failure &) {+ std::error_code ec;+ const bool exists = std::filesystem::exists(ngram.lookup_cache_dynamic, ec);+ if (!exists && !ec) {+ // missing file is created on destruction+ SPC_WRN("dynamic lookup cache file does not exist: %s, starting with empty cache\n",+ ngram.lookup_cache_dynamic.c_str());+ } else {+ SPC_ERR("failed to open dynamic lookup cache: %s\n", ngram.lookup_cache_dynamic.c_str());+ GGML_ABORT("Couldn't read dynamic lookup cache");+ }}}}- void begin(llama_seq_id /*seq_id*/, const llama_tokens & /*prompt*/) override {- // noop+ ~common_speculative_impl_ngram_cache() override {+ if (params.lookup_cache_dynamic.empty() || sinfos.empty()) {+ return;+ }++ auto & dynamic_cache = sinfos.front().ngram_cache_dynamic;+ for (auto & sinfo : sinfos) {+ common_ngram_cache_merge(dynamic_cache, sinfo.ngram_cache_context);+ }++ SPC_INF("saving dynamic lookup cache at %s\n", params.lookup_cache_dynamic.c_str());+ common_ngram_cache_save(dynamic_cache, params.lookup_cache_dynamic);+ }++ void begin(llama_seq_id seq_id, const llama_tokens & /*prompt*/) override {+ sinfos[seq_id].cache_size = 0;}void draft_one(@@ -2226,22 +2242,6 @@ struct common_speculative_impl_ngram_cache : public common_speculative_impl {const auto & prompt = *dparams.prompt;- if (sinfo.cache_size < prompt.size() + 1) {- llama_tokens tokens_new;- tokens_new.reserve(prompt.size() + 1 - sinfo.cache_size);- for (size_t j = sinfo.cache_size; j < prompt.size(); ++j) {- tokens_new.push_back(prompt[j]);- }- tokens_new.push_back(dparams.id_last); // add the last token-- // Update context ngram cache with new dparams.prompt:- common_ngram_cache_update(- sinfo.ngram_cache_context,- LLAMA_NGRAM_MIN, LLAMA_NGRAM_MAX,- tokens_new, tokens_new.size(), false);- sinfo.cache_size = prompt.size() + 1;- }-llama_tokens inp;inp.reserve(prompt.size() + 1);for (size_t j = 0; j < prompt.size(); ++j) {@@ -2249,6 +2249,16 @@ struct common_speculative_impl_ngram_cache : public common_speculative_impl {}inp.push_back(dparams.id_last);+ if (sinfo.cache_size < inp.size()) {+ // Update context ngram cache with the full history so n-grams+ // ending at the new tokens have their preceding context+ common_ngram_cache_update(+ sinfo.ngram_cache_context,+ LLAMA_NGRAM_MIN, LLAMA_NGRAM_MAX,+ inp, inp.size() - sinfo.cache_size, false);+ sinfo.cache_size = inp.size();+ }+result.push_back(dparams.id_last);common_ngram_cache_draft(@@ -2309,20 +2319,12 @@ static common_ngram_map get_common_ngram_map(return common_ngram_map(size_key, size_value, key_only, min_hits);}-static common_speculative_impl_ngram_cache create_state_ngram_cache(+static std::unique_ptr<common_speculative_impl_ngram_cache> create_state_ngram_cache(const common_speculative_config & config,- uint32_t n_seq,- const std::string & path_static,- const std::string & path_dynamic) {+ uint32_t n_seq) {uint16_t n_draft = 8; // TODO get from config?- // TODO bool param in common/common.h to set save_static/save_dynamic?- bool save_static = false;- bool save_dynamic = false;-- common_speculative_impl_ngram_cache state(config.params, n_seq, n_draft, path_static, path_dynamic, save_static, save_dynamic);-- return state;+ return std::make_unique<common_speculative_impl_ngram_cache>(config.params, n_seq, n_draft);}std::string common_speculative_type_name_str(const std::vector<common_speculative_type> & types) {@@ -2822,11 +2824,7 @@ common_speculative * common_speculative_init(common_params_speculative & params,break;}case COMMON_SPECULATIVE_TYPE_NGRAM_CACHE: {- auto state = create_state_ngram_cache(- config, n_seq,- params.ngram_cache.lookup_cache_static,- params.ngram_cache.lookup_cache_dynamic);- impls.push_back(std::make_unique<common_speculative_impl_ngram_cache>(state));+ impls.push_back(create_state_ngram_cache(config, n_seq));break;}default: