LLM inference in C/C++
C++ ggml
llama-batch: add n_keep_tail in split_equal for recurrent models (#25278)
Aman Gupta230ea9d21432
diff --git a/src/llama-batch.cpp b/src/llama-batch.cppindex 6bf76939cd..5436717c42 100644--- a/src/llama-batch.cpp+++ b/src/llama-batch.cpp@@ -505,7 +505,7 @@ llama_ubatch llama_batch_allocr::split_simple(uint32_t n_ubatch) {return ubatch_add(idxs, idxs.size(), false);}-llama_ubatch llama_batch_allocr::split_equal(uint32_t n_ubatch, bool sequential) {+llama_ubatch llama_batch_allocr::split_equal(uint32_t n_ubatch, bool sequential, uint32_t n_keep_tail) {if (sequential && has_cpl) {LLAMA_LOG_ERROR("%s: sequential split is not supported when there are coupled sequences in the input batch (you may need to use the -kvu flag)\n", __func__);@@ -548,7 +548,7 @@ llama_ubatch llama_batch_allocr::split_equal(uint32_t n_ubatch, bool sequential)}}- const uint32_t n_seqs = cur_seq_set.size();+ uint32_t n_seqs = cur_seq_set.size();// we are doneif (n_seqs == 0) {@@ -569,7 +569,7 @@ llama_ubatch llama_batch_allocr::split_equal(uint32_t n_ubatch, bool sequential)std::vector<idx_vec_t> idxs_per_seq(n_seqs);while (true) {- // we can only add new n_seq_tokens tokens if all the sequence sets have at least one more unused token and+ // we can only add new n_seq_tokens tokens if all the sequence sets have at least 1 more unused tokens and// if we haven't reached n_ubatchbool can_expand = true;@@ -600,6 +600,72 @@ llama_ubatch llama_batch_allocr::split_equal(uint32_t n_ubatch, bool sequential)}}+ // if n_keep_tail > 0, keep only the seqs that either finish in this ubatch or have at least+ // n_keep_tail tokens remaining for a future ubatch, so that the trailing n_keep_tail tokens+ // of each seq are never split across ubatches+ if (n_keep_tail > 0) {+ GGML_ASSERT(n_ubatch > n_keep_tail);++ auto n_remaining = [&](uint32_t s) {+ return (uint32_t) (seq_set_map[cur_seq_set[s]].size() - cur_idx[s]);+ };++ // keep the longest prefix of seqs that satisfy the constraint, to preserve sequential seq ids+ uint32_t n_keep = 0;+ while (n_keep < n_seqs) {+ const uint32_t remaining = n_remaining(n_keep);++ if (remaining != 0 && remaining < n_keep_tail) {+ break;+ }++ n_keep++;+ }++ // all seqs violate the constraint - resolve the first one directly and emit it alone+ if (n_keep == 0) {+ auto & idxs = idxs_per_seq[0];++ const auto & seq_idxs = seq_set_map[cur_seq_set[0]];++ if (idxs.size() + n_remaining(0) <= n_ubatch) {+ // extend the seq to completion+ while (n_remaining(0) > 0) {+ const int32_t idx = seq_idxs[cur_idx[0]];++ idxs.push_back(idx);++ used[idx] = true;+ ++n_used;++ ++cur_idx[0];+ }+ } else {+ // truncate the seq so that at least n_keep_tail tokens remain+ while (n_remaining(0) < n_keep_tail) {+ used[idxs.back()] = false;+ --n_used;++ idxs.pop_back();++ --cur_idx[0];+ }+ }++ n_keep = 1;+ }++ // return the tokens of the deferred seqs back to the pool+ for (uint32_t s = n_keep; s < n_seqs; ++s) {+ for (const int32_t idx : idxs_per_seq[s]) {+ used[idx] = false;+ --n_used;+ }+ }++ n_seqs = n_keep;+ }+// concat the per-sequence-set listsstd::vector<int32_t> idxs;@@ -814,7 +880,7 @@ void llama_batch_allocr::ubatch_print(const llama_ubatch & ubatch, int debug) {LLAMA_LOG_DEBUG("%s: output = %p\n", __func__, (void *) ubatch.output);LLAMA_LOG_DEBUG("%s: n_outputs = %d\n", __func__, n_outputs);- if (debug > 1) {+ if (debug > 0) {int seq_id_max = 0;for (uint32_t i = 0; i < ubatch.n_tokens; ++i) {for (int s = 0; s < ubatch.n_seq_id[i]; ++s) {diff --git a/src/llama-batch.h b/src/llama-batch.hindex f77520e86c..a3d1889d4a 100644--- a/src/llama-batch.h+++ b/src/llama-batch.h@@ -104,7 +104,8 @@ public:// make ubatches of equal-length sequences sets// if sequential == true, the tokens in the ubatch will have increasing sequential sequence ids- llama_ubatch split_equal(uint32_t n_ubatch, bool sequential);+ // n_keep_tail = minimum trailing tokens of a seq that must land in the same ubatch+ llama_ubatch split_equal(uint32_t n_ubatch, bool sequential, uint32_t n_keep_tail);// sequence-set-wise split - each ubatch contains a single sequence-setllama_ubatch split_seq(uint32_t n_ubatch);diff --git a/src/llama-kv-cache-dsa.cpp b/src/llama-kv-cache-dsa.cppindex 916ab65375..241c50365a 100644--- a/src/llama-kv-cache-dsa.cpp+++ b/src/llama-kv-cache-dsa.cpp@@ -113,7 +113,7 @@ llama_memory_context_ptr llama_kv_cache_dsa::init_batch(std::vector<llama_ubatch> ubatches;while (true) {- auto ubatch = n_stream == 1 ? balloc.split_simple(n_ubatch) : balloc.split_equal(n_ubatch, true);+ auto ubatch = n_stream == 1 ? balloc.split_simple(n_ubatch) : balloc.split_equal(n_ubatch, true, 0);if (ubatch.n_tokens == 0) {break;diff --git a/src/llama-kv-cache-dsv4.cpp b/src/llama-kv-cache-dsv4.cppindex 3a698d719e..9fccf347ed 100644--- a/src/llama-kv-cache-dsv4.cpp+++ b/src/llama-kv-cache-dsv4.cpp@@ -1110,7 +1110,7 @@ llama_memory_context_ptr llama_kv_cache_dsv4::init_batch(if (has_coupled) {ubatch = balloc.split_seq(n_ubatch);} else {- ubatch = balloc.split_equal(n_ubatch, raw_per_seq || comp_per_seq);+ ubatch = balloc.split_equal(n_ubatch, raw_per_seq || comp_per_seq, 0);}if (ubatch.n_tokens == 0) {diff --git a/src/llama-kv-cache-iswa.cpp b/src/llama-kv-cache-iswa.cppindex 2fcf238d91..e91866469a 100644--- a/src/llama-kv-cache-iswa.cpp+++ b/src/llama-kv-cache-iswa.cpp@@ -206,7 +206,7 @@ llama_memory_context_ptr llama_kv_cache_iswa::init_batch(llama_batch_allocr & bastd::vector<llama_ubatch> ubatches;while (true) {- auto ubatch = balloc.split_equal(n_ubatch, !unified);+ auto ubatch = balloc.split_equal(n_ubatch, !unified, 0);if (ubatch.n_tokens == 0) {break;diff --git a/src/llama-kv-cache.cpp b/src/llama-kv-cache.cppindex 680de5144a..e70583e641 100644--- a/src/llama-kv-cache.cpp+++ b/src/llama-kv-cache.cpp@@ -706,7 +706,7 @@ llama_memory_context_ptr llama_kv_cache::init_batch(std::vector<llama_ubatch> ubatches;while (true) {- auto ubatch = n_stream == 1 ? balloc.split_simple(n_ubatch) : balloc.split_equal(n_ubatch, true);+ auto ubatch = n_stream == 1 ? balloc.split_simple(n_ubatch) : balloc.split_equal(n_ubatch, true, 0);if (ubatch.n_tokens == 0) {break;diff --git a/src/llama-memory-hybrid-iswa.cpp b/src/llama-memory-hybrid-iswa.cppindex c7d4bcd413..06f7fd5428 100644--- a/src/llama-memory-hybrid-iswa.cpp+++ b/src/llama-memory-hybrid-iswa.cpp@@ -77,15 +77,15 @@ llama_memory_context_ptr llama_memory_hybrid_iswa::init_batch(llama_batch_allocr// if all tokens are output, split by sequenceubatch = balloc.split_seq(n_ubatch);} else {- if (mem_recr->n_rs_seq > 0) {- // [TAG_RECURRENT_ROLLBACK_SPLITS]- // TODO: recurrent state rollback does not support equal splits- ubatch = balloc.split_seq(n_ubatch);- } else {- // Use non-sequential split when KV cache is unified (needed for hellaswag/winogrande/multiple-choice)- const bool unified = (mem_attn->get_base()->get_n_stream() == 1);- ubatch = balloc.split_equal(n_ubatch, !unified);- }+ // Use non-sequential split when KV cache is unified (needed for hellaswag/winogrande/multiple-choice)+ const bool unified = (mem_attn->get_base()->get_n_stream() == 1);++ // [TAG_RECURRENT_ROLLBACK_SPLITS]+ // the trailing (1 + n_rs_seq) tokens of each seq must stay in the same ubatch+ // so that the rollback snapshots remain valid+ const uint32_t n_rs_seq = mem_recr->n_rs_seq;++ ubatch = balloc.split_equal(n_ubatch, !unified, n_rs_seq > 0 ? n_rs_seq + 1 : 0);}if (ubatch.n_tokens == 0) {diff --git a/src/llama-memory-hybrid.cpp b/src/llama-memory-hybrid.cppindex f2d49cbce5..42c7381a9e 100644--- a/src/llama-memory-hybrid.cpp+++ b/src/llama-memory-hybrid.cpp@@ -78,15 +78,15 @@ llama_memory_context_ptr llama_memory_hybrid::init_batch(llama_batch_allocr & ba// if all tokens are output, split by sequenceubatch = balloc.split_seq(n_ubatch);} else {- if (mem_recr->n_rs_seq > 0) {- // [TAG_RECURRENT_ROLLBACK_SPLITS]- // TODO: recurrent state rollback does not support equal splits- ubatch = balloc.split_seq(n_ubatch);- } else {- // Use non-sequential split when KV cache is unified (needed for hellaswag/winogrande/multiple-choice)- const bool unified = (mem_attn->get_n_stream() == 1);- ubatch = balloc.split_equal(n_ubatch, !unified);- }+ // Use non-sequential split when KV cache is unified (needed for hellaswag/winogrande/multiple-choice)+ const bool unified = (mem_attn->get_n_stream() == 1);++ // [TAG_RECURRENT_ROLLBACK_SPLITS]+ // the trailing (1 + n_rs_seq) tokens of each seq must stay in the same ubatch+ // so that the rollback snapshots remain valid+ const uint32_t n_rs_seq = mem_recr->n_rs_seq;++ ubatch = balloc.split_equal(n_ubatch, !unified, n_rs_seq > 0 ? n_rs_seq + 1 : 0);}if (ubatch.n_tokens == 0) {diff --git a/src/llama-memory-recurrent.cpp b/src/llama-memory-recurrent.cppindex 6a4892fb47..3d6c6db876 100644--- a/src/llama-memory-recurrent.cpp+++ b/src/llama-memory-recurrent.cpp@@ -416,15 +416,12 @@ llama_memory_context_ptr llama_memory_recurrent::init_batch(llama_batch_allocr &// if all tokens are output, split by sequenceubatch = balloc.split_seq(n_ubatch);} else {- if (n_rs_seq > 0) {- // [TAG_RECURRENT_ROLLBACK_SPLITS]- // TODO: recurrent state rollback does not support equal splits- ubatch = balloc.split_seq(n_ubatch);- } else {- // TODO: non-sequential equal split can be done if using unified KV cache- // for simplicity, we always use sequential equal split for now- ubatch = balloc.split_equal(n_ubatch, true);- }+ // TODO: non-sequential equal split can be done if using unified KV cache+ // for simplicity, we always use sequential equal split for now+ // [TAG_RECURRENT_ROLLBACK_SPLITS]+ // the trailing (1 + n_rs_seq) tokens of each seq must stay in the same ubatch+ // so that the rollback snapshots remain valid+ ubatch = balloc.split_equal(n_ubatch, true, n_rs_seq > 0 ? n_rs_seq + 1 : 0);}if (ubatch.n_tokens == 0) {diff --git a/src/models/delta-net-base.cpp b/src/models/delta-net-base.cppindex ad9ce77140..cf5e380950 100644--- a/src/models/delta-net-base.cpp+++ b/src/models/delta-net-base.cpp@@ -496,8 +496,8 @@ ggml_tensor * llm_build_delta_net_base::build_conv_state(ggml_build_forward_expand(gf, ggml_cpy(ctx0, conv_state_last, conv_state_update));} else {// [TAG_RECURRENT_ROLLBACK_SPLITS]- // TODO: this logic incorrectly assumes that the last (n_rs_seq + 1) tokens of a sequence in a batch are- // inside the same ubatch. currently with `split_equal()` this is not correct+ // this logic assumes that the last (n_rs_seq + 1) tokens of a sequence in a batch are inside+ // the same ubatch, which `split_equal()` guarantees via its n_keep_tail argumentconst int64_t K = (int64_t) cparams.n_rs_seq + 1;