Skip to content

LLM inference in C/C++

C++ ggml

llama-batch: add n_keep_tail in split_equal for recurrent models (#25278)

Aman Gupta230ea9d21432
diff --git a/src/llama-batch.cpp b/src/llama-batch.cpp
index 6bf76939cd..5436717c42 100644
--- a/src/llama-batch.cpp
+++ b/src/llama-batch.cpp
@@ -505,7 +505,7 @@ llama_ubatch llama_batch_allocr::split_simple(uint32_t n_ubatch) {
return ubatch_add(idxs, idxs.size(), false);
}
-llama_ubatch llama_batch_allocr::split_equal(uint32_t n_ubatch, bool sequential) {
+llama_ubatch llama_batch_allocr::split_equal(uint32_t n_ubatch, bool sequential, uint32_t n_keep_tail) {
if (sequential && has_cpl) {
LLAMA_LOG_ERROR("%s: sequential split is not supported when there are coupled sequences in the input batch (you may need to use the -kvu flag)\n", __func__);
@@ -548,7 +548,7 @@ llama_ubatch llama_batch_allocr::split_equal(uint32_t n_ubatch, bool sequential)
}
}
- const uint32_t n_seqs = cur_seq_set.size();
+ uint32_t n_seqs = cur_seq_set.size();
// we are done
if (n_seqs == 0) {
@@ -569,7 +569,7 @@ llama_ubatch llama_batch_allocr::split_equal(uint32_t n_ubatch, bool sequential)
std::vector<idx_vec_t> idxs_per_seq(n_seqs);
while (true) {
- // we can only add new n_seq_tokens tokens if all the sequence sets have at least one more unused token and
+ // we can only add new n_seq_tokens tokens if all the sequence sets have at least 1 more unused tokens and
// if we haven't reached n_ubatch
bool can_expand = true;
@@ -600,6 +600,72 @@ llama_ubatch llama_batch_allocr::split_equal(uint32_t n_ubatch, bool sequential)
}
}
+ // if n_keep_tail > 0, keep only the seqs that either finish in this ubatch or have at least
+ // n_keep_tail tokens remaining for a future ubatch, so that the trailing n_keep_tail tokens
+ // of each seq are never split across ubatches
+ if (n_keep_tail > 0) {
+ GGML_ASSERT(n_ubatch > n_keep_tail);
+
+ auto n_remaining = [&](uint32_t s) {
+ return (uint32_t) (seq_set_map[cur_seq_set[s]].size() - cur_idx[s]);
+ };
+
+ // keep the longest prefix of seqs that satisfy the constraint, to preserve sequential seq ids
+ uint32_t n_keep = 0;
+ while (n_keep < n_seqs) {
+ const uint32_t remaining = n_remaining(n_keep);
+
+ if (remaining != 0 && remaining < n_keep_tail) {
+ break;
+ }
+
+ n_keep++;
+ }
+
+ // all seqs violate the constraint - resolve the first one directly and emit it alone
+ if (n_keep == 0) {
+ auto & idxs = idxs_per_seq[0];
+
+ const auto & seq_idxs = seq_set_map[cur_seq_set[0]];
+
+ if (idxs.size() + n_remaining(0) <= n_ubatch) {
+ // extend the seq to completion
+ while (n_remaining(0) > 0) {
+ const int32_t idx = seq_idxs[cur_idx[0]];
+
+ idxs.push_back(idx);
+
+ used[idx] = true;
+ ++n_used;
+
+ ++cur_idx[0];
+ }
+ } else {
+ // truncate the seq so that at least n_keep_tail tokens remain
+ while (n_remaining(0) < n_keep_tail) {
+ used[idxs.back()] = false;
+ --n_used;
+
+ idxs.pop_back();
+
+ --cur_idx[0];
+ }
+ }
+
+ n_keep = 1;
+ }
+
+ // return the tokens of the deferred seqs back to the pool
+ for (uint32_t s = n_keep; s < n_seqs; ++s) {
+ for (const int32_t idx : idxs_per_seq[s]) {
+ used[idx] = false;
+ --n_used;
+ }
+ }
+
+ n_seqs = n_keep;
+ }
+
// concat the per-sequence-set lists
std::vector<int32_t> idxs;
@@ -814,7 +880,7 @@ void llama_batch_allocr::ubatch_print(const llama_ubatch & ubatch, int debug) {
LLAMA_LOG_DEBUG("%s: output = %p\n", __func__, (void *) ubatch.output);
LLAMA_LOG_DEBUG("%s: n_outputs = %d\n", __func__, n_outputs);
- if (debug > 1) {
+ if (debug > 0) {
int seq_id_max = 0;
for (uint32_t i = 0; i < ubatch.n_tokens; ++i) {
for (int s = 0; s < ubatch.n_seq_id[i]; ++s) {
diff --git a/src/llama-batch.h b/src/llama-batch.h
index f77520e86c..a3d1889d4a 100644
--- a/src/llama-batch.h
+++ b/src/llama-batch.h
@@ -104,7 +104,8 @@ public:
// make ubatches of equal-length sequences sets
// if sequential == true, the tokens in the ubatch will have increasing sequential sequence ids
- llama_ubatch split_equal(uint32_t n_ubatch, bool sequential);
+ // n_keep_tail = minimum trailing tokens of a seq that must land in the same ubatch
+ llama_ubatch split_equal(uint32_t n_ubatch, bool sequential, uint32_t n_keep_tail);
// sequence-set-wise split - each ubatch contains a single sequence-set
llama_ubatch split_seq(uint32_t n_ubatch);
diff --git a/src/llama-kv-cache-dsa.cpp b/src/llama-kv-cache-dsa.cpp
index 916ab65375..241c50365a 100644
--- a/src/llama-kv-cache-dsa.cpp
+++ b/src/llama-kv-cache-dsa.cpp
@@ -113,7 +113,7 @@ llama_memory_context_ptr llama_kv_cache_dsa::init_batch(
std::vector<llama_ubatch> ubatches;
while (true) {
- auto ubatch = n_stream == 1 ? balloc.split_simple(n_ubatch) : balloc.split_equal(n_ubatch, true);
+ auto ubatch = n_stream == 1 ? balloc.split_simple(n_ubatch) : balloc.split_equal(n_ubatch, true, 0);
if (ubatch.n_tokens == 0) {
break;
diff --git a/src/llama-kv-cache-dsv4.cpp b/src/llama-kv-cache-dsv4.cpp
index 3a698d719e..9fccf347ed 100644
--- a/src/llama-kv-cache-dsv4.cpp
+++ b/src/llama-kv-cache-dsv4.cpp
@@ -1110,7 +1110,7 @@ llama_memory_context_ptr llama_kv_cache_dsv4::init_batch(
if (has_coupled) {
ubatch = balloc.split_seq(n_ubatch);
} else {
- ubatch = balloc.split_equal(n_ubatch, raw_per_seq || comp_per_seq);
+ ubatch = balloc.split_equal(n_ubatch, raw_per_seq || comp_per_seq, 0);
}
if (ubatch.n_tokens == 0) {
diff --git a/src/llama-kv-cache-iswa.cpp b/src/llama-kv-cache-iswa.cpp
index 2fcf238d91..e91866469a 100644
--- a/src/llama-kv-cache-iswa.cpp
+++ b/src/llama-kv-cache-iswa.cpp
@@ -206,7 +206,7 @@ llama_memory_context_ptr llama_kv_cache_iswa::init_batch(llama_batch_allocr & ba
std::vector<llama_ubatch> ubatches;
while (true) {
- auto ubatch = balloc.split_equal(n_ubatch, !unified);
+ auto ubatch = balloc.split_equal(n_ubatch, !unified, 0);
if (ubatch.n_tokens == 0) {
break;
diff --git a/src/llama-kv-cache.cpp b/src/llama-kv-cache.cpp
index 680de5144a..e70583e641 100644
--- a/src/llama-kv-cache.cpp
+++ b/src/llama-kv-cache.cpp
@@ -706,7 +706,7 @@ llama_memory_context_ptr llama_kv_cache::init_batch(
std::vector<llama_ubatch> ubatches;
while (true) {
- auto ubatch = n_stream == 1 ? balloc.split_simple(n_ubatch) : balloc.split_equal(n_ubatch, true);
+ auto ubatch = n_stream == 1 ? balloc.split_simple(n_ubatch) : balloc.split_equal(n_ubatch, true, 0);
if (ubatch.n_tokens == 0) {
break;
diff --git a/src/llama-memory-hybrid-iswa.cpp b/src/llama-memory-hybrid-iswa.cpp
index c7d4bcd413..06f7fd5428 100644
--- a/src/llama-memory-hybrid-iswa.cpp
+++ b/src/llama-memory-hybrid-iswa.cpp
@@ -77,15 +77,15 @@ llama_memory_context_ptr llama_memory_hybrid_iswa::init_batch(llama_batch_allocr
// if all tokens are output, split by sequence
ubatch = balloc.split_seq(n_ubatch);
} else {
- if (mem_recr->n_rs_seq > 0) {
- // [TAG_RECURRENT_ROLLBACK_SPLITS]
- // TODO: recurrent state rollback does not support equal splits
- ubatch = balloc.split_seq(n_ubatch);
- } else {
- // Use non-sequential split when KV cache is unified (needed for hellaswag/winogrande/multiple-choice)
- const bool unified = (mem_attn->get_base()->get_n_stream() == 1);
- ubatch = balloc.split_equal(n_ubatch, !unified);
- }
+ // Use non-sequential split when KV cache is unified (needed for hellaswag/winogrande/multiple-choice)
+ const bool unified = (mem_attn->get_base()->get_n_stream() == 1);
+
+ // [TAG_RECURRENT_ROLLBACK_SPLITS]
+ // the trailing (1 + n_rs_seq) tokens of each seq must stay in the same ubatch
+ // so that the rollback snapshots remain valid
+ const uint32_t n_rs_seq = mem_recr->n_rs_seq;
+
+ ubatch = balloc.split_equal(n_ubatch, !unified, n_rs_seq > 0 ? n_rs_seq + 1 : 0);
}
if (ubatch.n_tokens == 0) {
diff --git a/src/llama-memory-hybrid.cpp b/src/llama-memory-hybrid.cpp
index f2d49cbce5..42c7381a9e 100644
--- a/src/llama-memory-hybrid.cpp
+++ b/src/llama-memory-hybrid.cpp
@@ -78,15 +78,15 @@ llama_memory_context_ptr llama_memory_hybrid::init_batch(llama_batch_allocr & ba
// if all tokens are output, split by sequence
ubatch = balloc.split_seq(n_ubatch);
} else {
- if (mem_recr->n_rs_seq > 0) {
- // [TAG_RECURRENT_ROLLBACK_SPLITS]
- // TODO: recurrent state rollback does not support equal splits
- ubatch = balloc.split_seq(n_ubatch);
- } else {
- // Use non-sequential split when KV cache is unified (needed for hellaswag/winogrande/multiple-choice)
- const bool unified = (mem_attn->get_n_stream() == 1);
- ubatch = balloc.split_equal(n_ubatch, !unified);
- }
+ // Use non-sequential split when KV cache is unified (needed for hellaswag/winogrande/multiple-choice)
+ const bool unified = (mem_attn->get_n_stream() == 1);
+
+ // [TAG_RECURRENT_ROLLBACK_SPLITS]
+ // the trailing (1 + n_rs_seq) tokens of each seq must stay in the same ubatch
+ // so that the rollback snapshots remain valid
+ const uint32_t n_rs_seq = mem_recr->n_rs_seq;
+
+ ubatch = balloc.split_equal(n_ubatch, !unified, n_rs_seq > 0 ? n_rs_seq + 1 : 0);
}
if (ubatch.n_tokens == 0) {
diff --git a/src/llama-memory-recurrent.cpp b/src/llama-memory-recurrent.cpp
index 6a4892fb47..3d6c6db876 100644
--- a/src/llama-memory-recurrent.cpp
+++ b/src/llama-memory-recurrent.cpp
@@ -416,15 +416,12 @@ llama_memory_context_ptr llama_memory_recurrent::init_batch(llama_batch_allocr &
// if all tokens are output, split by sequence
ubatch = balloc.split_seq(n_ubatch);
} else {
- if (n_rs_seq > 0) {
- // [TAG_RECURRENT_ROLLBACK_SPLITS]
- // TODO: recurrent state rollback does not support equal splits
- ubatch = balloc.split_seq(n_ubatch);
- } else {
- // TODO: non-sequential equal split can be done if using unified KV cache
- // for simplicity, we always use sequential equal split for now
- ubatch = balloc.split_equal(n_ubatch, true);
- }
+ // TODO: non-sequential equal split can be done if using unified KV cache
+ // for simplicity, we always use sequential equal split for now
+ // [TAG_RECURRENT_ROLLBACK_SPLITS]
+ // the trailing (1 + n_rs_seq) tokens of each seq must stay in the same ubatch
+ // so that the rollback snapshots remain valid
+ ubatch = balloc.split_equal(n_ubatch, true, n_rs_seq > 0 ? n_rs_seq + 1 : 0);
}
if (ubatch.n_tokens == 0) {
diff --git a/src/models/delta-net-base.cpp b/src/models/delta-net-base.cpp
index ad9ce77140..cf5e380950 100644
--- a/src/models/delta-net-base.cpp
+++ b/src/models/delta-net-base.cpp
@@ -496,8 +496,8 @@ ggml_tensor * llm_build_delta_net_base::build_conv_state(
ggml_build_forward_expand(gf, ggml_cpy(ctx0, conv_state_last, conv_state_update));
} else {
// [TAG_RECURRENT_ROLLBACK_SPLITS]
- // TODO: this logic incorrectly assumes that the last (n_rs_seq + 1) tokens of a sequence in a batch are
- // inside the same ubatch. currently with `split_equal()` this is not correct
+ // this logic assumes that the last (n_rs_seq + 1) tokens of a sequence in a batch are inside
+ // the same ubatch, which `split_equal()` guarantees via its n_keep_tail argument
const int64_t K = (int64_t) cparams.n_rs_seq + 1;