LLM inference in C/C++
C++ ggml
llama: add token ID tracking to KV cell (#27762)
Xuan-Son Nguyen925e1179947e
diff --git a/include/llama.h b/include/llama.hindex a04177f9f7..84313cfd74 100644--- a/include/llama.h+++ b/include/llama.h@@ -43,10 +43,10 @@#define LLAMA_FILE_MAGIC_GGSQ 0x67677371u // 'ggsq'#define LLAMA_SESSION_MAGIC LLAMA_FILE_MAGIC_GGSN-#define LLAMA_SESSION_VERSION 9+#define LLAMA_SESSION_VERSION 10#define LLAMA_STATE_SEQ_MAGIC LLAMA_FILE_MAGIC_GGSQ-#define LLAMA_STATE_SEQ_VERSION 2+#define LLAMA_STATE_SEQ_VERSION 3#ifdef __cplusplusextern "C" {diff --git a/src/llama-kv-cache.cpp b/src/llama-kv-cache.cppindex ec0f5a7531..383bf83199 100644--- a/src/llama-kv-cache.cpp+++ b/src/llama-kv-cache.cpp@@ -12,6 +12,7 @@#include <limits>#include <map>#include <stdexcept>+#include <unordered_map>static bool ggml_is_power_of_2(int n) {return (n & (n - 1)) == 0;@@ -1128,11 +1129,18 @@ void llama_kv_cache::apply_ubatch(const slot_info & sinfo, const llama_ubatch &cells.pos_set(idx, ubatch.pos[i]);- if (ubatch.is_pos_2d()) {- llama_kv_cell_ext ext {- /*.x =*/ ubatch.pos[i + ubatch.n_tokens*2],- /*.y =*/ ubatch.pos[i + ubatch.n_tokens],- };+ if (ubatch.is_pos_2d() || ubatch.token) {+ llama_kv_cell_ext ext;++ if (ubatch.is_pos_2d()) {+ ext.x = ubatch.pos[i + ubatch.n_tokens*2];+ ext.y = ubatch.pos[i + ubatch.n_tokens];+ }++ if (ubatch.token) {+ ext.tok = ubatch.token[i];+ }+cells.ext_set(idx, ext);}@@ -1805,6 +1813,69 @@ void llama_kv_cache::set_input_v_rot(ggml_tensor * dst) const {memcpy(dst->data, attn_rot_hadamard.at(n_rot).data(), ggml_nbytes(dst));}+bool llama_kv_cache::has_cell_ext() const {+ return hparams.n_pos_per_embd() > 1;+}++void llama_kv_cache::get_prev_tokens(const llama_ubatch & ubatch, uint32_t n, std::vector<llama_token> & res) const {+ const uint32_t n_tokens = ubatch.n_tokens;++ res.clear();+ res.resize(n_tokens*n, LLAMA_TOKEN_NULL);++ if (n == 0) {+ return;+ }++ // note: apply_ubatch() has already stored the current ubatch+ // the window below thus covers tokens of this very ubatch as well, which is what we want+ llama_pos p_min = std::numeric_limits<llama_pos>::max();+ llama_pos p_max = std::numeric_limits<llama_pos>::min();++ std::bitset<LLAMA_MAX_SEQ> seqs;++ for (uint32_t i = 0; i < n_tokens; ++i) {+ p_min = std::min(p_min, ubatch.pos[i]);+ p_max = std::max(p_max, ubatch.pos[i]);+ }++ for (uint32_t s = 0; s < ubatch.n_seqs_unq; ++s) {+ seqs.set(ubatch.seq_id_unq[s]);+ }++ // (seq_id, pos) -> token, for every cell that could be a predecessor of a ubatch token+ std::unordered_map<uint64_t, llama_token> hist;++ const auto key = [](llama_seq_id seq_id, llama_pos pos) {+ return ((uint64_t) seq_id << 32) | (uint32_t) pos;+ };++ for (uint32_t s = 0; s < n_stream; ++s) {+ v_cells[s].for_each_token_in(seqs, p_min - (llama_pos) n, p_max,+ [&](llama_seq_id seq_id, llama_pos pos, llama_token tok) {+ hist[key(seq_id, pos)] = tok;+ });+ }++ for (uint32_t i = 0; i < n_tokens; ++i) {+ // TODO: a token that belongs to more than one sequence has an ambiguous history.+ // the n-gram architectures have to reject such batches+ const llama_seq_id seq_id = ubatch.seq_id[i][0];++ for (uint32_t j = 0; j < n; ++j) {+ const llama_pos p = ubatch.pos[i] - (llama_pos) (n - j);+ if (p < 0) {+ continue;+ }++ const auto it = hist.find(key(seq_id, p));+ if (it != hist.end()) {+ res[i*n + j] = it->second;+ }+ }+ }+}+size_t llama_kv_cache::total_size() const {size_t size = 0;@@ -2106,7 +2177,7 @@ void llama_kv_cache::state_write_meta(llama_io_write_i & io, const cell_ranges_tio.write(&pos, sizeof(pos));io.write(&n_seq_id, sizeof(n_seq_id));- if (hparams.n_pos_per_embd() > 1) {+ if (has_cell_ext()) {const llama_kv_cell_ext ext = cells.ext_get(i);io.write(&ext, sizeof(ext));}@@ -2243,12 +2314,17 @@ bool llama_kv_cache::state_read_meta(llama_io_read_i & io, uint32_t strm, uint32return false;}- if (hparams.n_pos_per_embd() > 1) {+ if (has_cell_ext()) {llama_kv_cell_ext ext;io.read(&ext, sizeof(ext));- ubatch.pos[i + ubatch.n_tokens] = ext.y;- ubatch.pos[i + ubatch.n_tokens*2] = ext.x;+ if (hparams.n_pos_per_embd() > 1) {+ ubatch.pos[i + ubatch.n_tokens] = ext.y;+ ubatch.pos[i + ubatch.n_tokens*2] = ext.x;+ }++ // apply_ubatch() below restores ext.tok from the ubatch tokens+ ubatch.token[i] = ext.tok;}// read the sequence id, but directly discard it - we will use dest_seq_id instead@@ -2268,7 +2344,8 @@ bool llama_kv_cache::state_read_meta(llama_io_read_i & io, uint32_t strm, uint32return false;}- // TODO: we cannot yet restore llama_kv_cell_ext as the apply_ubatch() does not support it yet+ // note: apply_ubatch() rebuilds llama_kv_cell_ext from the ubatch+ // only ext.tok and the M-RoPE 2D position round-trip through it// see: https://github.com/ggml-org/llama.cpp/pull/16825#issuecomment-3460868350apply_ubatch(sinfo, ubatch);@@ -2301,7 +2378,7 @@ bool llama_kv_cache::state_read_meta(llama_io_read_i & io, uint32_t strm, uint32cells.pos_set(i, pos);- if (hparams.n_pos_per_embd() > 1) {+ if (has_cell_ext()) {llama_kv_cell_ext ext;io.read(&ext, sizeof(ext));cells.ext_set(i, ext);@@ -2652,3 +2729,7 @@ void llama_kv_cache_context::set_input_k_rot(ggml_tensor * dst) const {void llama_kv_cache_context::set_input_v_rot(ggml_tensor * dst) const {kv->set_input_v_rot(dst);}++void llama_kv_cache_context::get_prev_tokens(const llama_ubatch & ubatch, uint32_t n, std::vector<llama_token> & res) const {+ kv->get_prev_tokens(ubatch, n, res);+}diff --git a/src/llama-kv-cache.h b/src/llama-kv-cache.hindex 6cb6dbd2f9..9b225fae31 100644--- a/src/llama-kv-cache.h+++ b/src/llama-kv-cache.h@@ -219,6 +219,14 @@ public:void set_input_k_rot(ggml_tensor * dst) const;void set_input_v_rot(ggml_tensor * dst) const;+ // true if llama_kv_cell_ext holds information that has to survive a state save/restore+ bool has_cell_ext() const;++ // for every token of the ubatch, the ids of the n tokens that precede it in its sequence+ // entries with no matching cell are set to LLAMA_TOKEN_NULL+ // note: used by n-gram input embeddings+ void get_prev_tokens(const llama_ubatch & ubatch, uint32_t n, std::vector<llama_token> & res) const;+private:const llama_model & model;const llama_hparams & hparams;@@ -401,6 +409,9 @@ public:void set_input_k_rot(ggml_tensor * dst) const;void set_input_v_rot(ggml_tensor * dst) const;+ // see llama_kv_cache::get_prev_tokens()+ void get_prev_tokens(const llama_ubatch & ubatch, uint32_t n, std::vector<llama_token> & res) const;+private:llama_memory_status status;diff --git a/src/llama-kv-cells.h b/src/llama-kv-cells.hindex fddd31a0b2..5167c037db 100644--- a/src/llama-kv-cells.h+++ b/src/llama-kv-cells.h@@ -15,6 +15,10 @@ struct llama_kv_cell_ext {llama_pos x = 0;llama_pos y = 0;+ // when tok = LLAMA_TOKEN_NULL when the cell is produced by embedding input (i.e. multimodal)+ // use case: n-gram embeddings hash+ llama_token tok = LLAMA_TOKEN_NULL;+// return true if the current 2D spatial position is greater than otherbool is_2d_gt(llama_pos ox, llama_pos oy) const {return (y > oy) || (y == oy && x > ox);@@ -23,7 +27,7 @@ struct llama_kv_cell_ext {void reset() {static_assert(std::is_trivially_copyable_v<llama_kv_cell_ext>);- memset(this, 0, sizeof(*this));+ *this = llama_kv_cell_ext{};}};@@ -305,6 +309,29 @@ public:return seq[i].test(seq_id);}+ // gather the token ids of the cells in `seqs` with position in [p0, p1)+ // the callback receives (seq_id, pos, token) for every such (cell, seq) pair+ // note: used by n-gram input embeddings to recover the tokens preceding a ubatch+ template<typename F>+ void for_each_token_in(const std::bitset<LLAMA_MAX_SEQ> & seqs, llama_pos p0, llama_pos p1, F && f) const {+ for (const auto & i : used) {+ if (pos[i] < p0 || pos[i] >= p1) {+ continue;+ }++ const auto m = seq[i] & seqs;+ if (m.none()) {+ continue;+ }++ for (llama_seq_id s = 0; s < LLAMA_MAX_SEQ; ++s) {+ if (m.test(s)) {+ f(s, pos[i], ext[i].tok);+ }+ }+ }+ }+// note: call only if the cell is not empty and the seq_id is not in the cellvoid seq_add(uint32_t i, llama_seq_id seq_id) {assert(i < pos.size());