LLM inference in C/C++
C++ ggml
kv-cells: look up the n-gram history in the sequence position index (#28040)
Pascalb356fa262464
diff --git a/src/llama-kv-cache.cpp b/src/llama-kv-cache.cppindex fd7ce0bb6e..3e4a4d56f9 100644--- a/src/llama-kv-cache.cpp+++ b/src/llama-kv-cache.cpp@@ -6,7 +6,6 @@#include "llama-context.h"#include <algorithm>-#include <array>#include <cassert>#include <cmath>#include <cstring>@@ -1836,58 +1835,10 @@ void llama_kv_cache::get_prev_tokens(const llama_ubatch & ubatch, uint32_t n, streturn;}- // note: apply_ubatch() has already stored the current ubatch- // the window below thus covers tokens of this very ubatch as well, which is what we want- llama_pos p_min = std::numeric_limits<llama_pos>::max();- llama_pos p_max = std::numeric_limits<llama_pos>::min();-- std::bitset<LLAMA_MAX_SEQ> seqs;-- for (uint32_t i = 0; i < n_tokens; ++i) {- p_min = std::min(p_min, ubatch.pos[i]);- p_max = std::max(p_max, ubatch.pos[i]);- }-- for (uint32_t s = 0; s < ubatch.n_seqs_unq; ++s) {- seqs.set(ubatch.seq_id_unq[s]);- }-- const llama_pos w0 = p_min - (llama_pos) n;-- // (seq_id, pos) -> token, for every cell that could be a predecessor of a ubatch token- std::unordered_map<uint64_t, llama_token> hist;-- const auto key = [](llama_seq_id seq_id, llama_pos pos) {- return ((uint64_t) seq_id << 32) | (uint32_t) pos;- };-- // handle M-RoPE gaps: multiple tokens share the same temporal pos- // TODO @ngxson : improve this in the future- std::array<std::pair<llama_pos, llama_token>, LLAMA_MAX_SEQ> below;- below.fill({ -1, LLAMA_TOKEN_NULL });-- for (uint32_t s = 0; s < n_stream; ++s) {- // p_max inclusive: an embd token looks up cells at its own (shared) position- v_cells[s].for_each_token_in(seqs, 0, p_max + 1,- [&](llama_seq_id seq_id, llama_pos pos, llama_token tok) {- if (pos >= w0) {- hist[key(seq_id, pos)] = tok;- } else if (pos > below[seq_id].first) {- below[seq_id] = { pos, tok };- }- });- }-- // the token at pos p, or the nearest earlier one when p falls in an M-RoPE gap- const auto lookup = [&](llama_seq_id seq_id, llama_pos p) -> llama_token {- for (llama_pos q = p; q >= w0; --q) {- const auto it = hist.find(key(seq_id, q));- if (it != hist.end()) {- return it->second;- }- }- return below[seq_id].second;- };+ // note: apply_ubatch() has already stored the current ubatch, so the cells cover the tokens+ // of this very ubatch as well, which is what we want+ // the nearest cell at or before a position also resolves M-RoPE gaps, where multiple tokens+ // share the same temporal pos// an embd (multimodal) ubatch can repeat one position for a whole image, so positions// do not encode the token order; resolve its predecessors by ubatch order instead@@ -1925,7 +1876,7 @@ void llama_kv_cache::get_prev_tokens(const llama_ubatch & ubatch, uint32_t n, stcontinue;}- res[i*n + j] = lookup(seq_id, p);+ res[i*n + j] = v_cells[seq_to_stream[seq_id]].seq_pos_tok_le(seq_id, p);}}}diff --git a/src/llama-kv-cells.h b/src/llama-kv-cells.hindex e9adffc098..5d567a6ed0 100644--- a/src/llama-kv-cells.h+++ b/src/llama-kv-cells.h@@ -6,7 +6,7 @@#include <bitset>#include <cassert>#include <cstring>-#include <map>+#include <limits>#include <set>#include <vector>@@ -248,7 +248,7 @@ public:assert(seq_id >= 0);seq[i].reset(seq_id);- seq_pos_dec(seq_id, pos[i]);+ seq_pos_dec(seq_id, i);if (seq[i].none()) {pos[i] = -1;@@ -272,7 +272,7 @@ public:seq[i].reset();seq[i].set(seq_id);- seq_pos_inc(seq_id, pos[i]);+ seq_pos_inc(seq_id, i);return false;}@@ -318,28 +318,22 @@ public:return seq[i].test(seq_id);}- // gather the token ids of the cells in `seqs` with position in [p0, p1)- // the callback receives (seq_id, pos, token) for every such (cell, seq) pair+ // the token of the cell of sequence seq_id at the largest position <= p+ // when several cells share that position, the one with the highest index wins+ // return LLAMA_TOKEN_NULL if the sequence has no cell at or before p// note: used by n-gram input embeddings to recover the tokens preceding a ubatch- template<typename F>- void for_each_token_in(const std::bitset<LLAMA_MAX_SEQ> & seqs, llama_pos p0, llama_pos p1, F && f) const {- for (const auto & i : used) {- if (pos[i] < p0 || pos[i] >= p1) {- continue;- }-- const auto m = seq[i] & seqs;+ llama_token seq_pos_tok_le(llama_seq_id seq_id, llama_pos p) const {+ assert(seq_id >= 0);+ assert(seq_id < LLAMA_MAX_SEQ);- // a cell carries a handful of sequences at most, out of LLAMA_MAX_SEQ- size_t left = m.count();+ const auto & sp = seq_pos[seq_id];- for (llama_seq_id s = 0; left > 0 && s < (llama_seq_id) LLAMA_MAX_SEQ; ++s) {- if (m.test(s)) {- f(s, pos[i], ext[i].tok);- --left;- }- }+ auto it = sp.upper_bound({ p, std::numeric_limits<uint32_t>::max() });+ if (it == sp.begin()) {+ return LLAMA_TOKEN_NULL;}++ return ext[(--it)->second].tok;}// note: call only if the cell is not empty and the seq_id is not in the cell@@ -349,7 +343,7 @@ public:assert(!seq[i].test(seq_id));seq[i].set(seq_id);- seq_pos_inc(seq_id, pos[i]);+ seq_pos_inc(seq_id, i);}// return the sequence id of this cell@@ -376,8 +370,6 @@ public:return -1;}- assert(seq_pos[seq_id].begin()->second > 0);-return seq_pos[seq_id].begin()->first;}@@ -391,8 +383,6 @@ public:return -1;}- assert(seq_pos[seq_id].rbegin()->second > 0);-return seq_pos[seq_id].rbegin()->first;}@@ -523,36 +513,33 @@ private:// the bitset seq[i] tells us which sequences are currently occupying the i-th cellstd::vector<seq_set_t> seq;- // the set seq_pos[s][p] tells us how many times the position p is currently present for sequence s- // if the position p is not present, seq_pos[s][p] is not set+ // the set seq_pos[s] holds one (pos, cell) pair per cell that carries sequence s, ordered by position// this way seq_pos[s].begin() and seq_pos[s].rbegin() give us the min/max positions currently in the cache+ // and upper_bound() on a position finds the nearest cell of the sequence in logarithmic time//- // note that we cannot a use an std::set because in some cases a position can occur more than once for the same seq:+ // the cell index is part of the key because a position can occur more than once for the same seq:// - during performing a cache reuse via (rm + add)// - some vision models have input embeddings with repeating positions//- std::map<llama_pos, int> seq_pos[LLAMA_MAX_SEQ];+ std::set<std::pair<llama_pos, uint32_t>> seq_pos[LLAMA_MAX_SEQ];// helper functions for updating `seq_pos`, once cell at a time:- void seq_pos_dec(llama_seq_id s, llama_pos p) {- auto it = seq_pos[s].find(p);- assert(it != seq_pos[s].end());-- if (--it->second == 0) {- seq_pos[s].erase(it);- }+ void seq_pos_dec(llama_seq_id s, uint32_t i) {+ const auto n = seq_pos[s].erase({ pos[i], i });+ assert(n == 1);+ GGML_UNUSED(n);}- void seq_pos_inc(llama_seq_id s, llama_pos p) {- seq_pos[s][p]++;+ void seq_pos_inc(llama_seq_id s, uint32_t i) {+ seq_pos[s].insert({ pos[i], i });}// remove cell ivoid seq_pos_rm(uint32_t i) {for (int s = 0; s < LLAMA_MAX_SEQ; ++s) {if (seq[i].test(s)) {- seq_pos_dec(s, pos[i]);+ seq_pos_dec(s, i);}}}@@ -561,7 +548,7 @@ private:void seq_pos_add(uint32_t i) {for (int s = 0; s < LLAMA_MAX_SEQ; ++s) {if (seq[i].test(s)) {- seq_pos_inc(s, pos[i]);+ seq_pos_inc(s, i);}}}