Skip to content

LLM inference in C/C++

C++ ggml

llama : clamp kpool re-pool bound to existing pools (#29805)

Georgi Gerganov81e39ad34368
diff --git a/src/llama-memory-hybrid-idx.cpp b/src/llama-memory-hybrid-idx.cpp
index de64a37000..3b43df92d3 100644
--- a/src/llama-memory-hybrid-idx.cpp
+++ b/src/llama-memory-hybrid-idx.cpp
@@ -762,10 +762,12 @@ void llama_memory_hybrid_idx_context::kpool_build_state(const llama_ubatch & uba
}
}
- // a ubatch touches at most t_s/kpool + 1 pools of a sequence with t_s tokens: pad to that bound so the
- // graph keeps its shape as the count moves, e.g. between 0 and n_seq while several sequences decode
+ // a ubatch touches at most t_s/kpool + 1 pools per sequence, pad to that bound so the graph keeps its shape
+ // as the count moves; reserve sizes the list for every pool the cache can hold, so never pad past n_pool_max
+ const auto * idx = mem->get_mem_idx();
+ const uint32_t n_pool_max = idx->get_size() / kpool * idx->get_n_seq_max();
const uint32_t bound = ubatch.n_tokens/kpool + ubatch.n_seqs_unq;
- st.n_new_g = std::max({st.n_new, 1u, std::min(bound, kpool_pad(st.n_pool_real) - 1)});
+ st.n_new_g = std::max({st.n_new, 1u, std::min({bound, kpool_pad(st.n_pool_real) - 1, n_pool_max})});
}
const llama_memory_hybrid_idx_context::kpool_state & llama_memory_hybrid_idx_context::kpool_cur() const {
diff --git a/tests/test-recurrent-state-rollback.cpp b/tests/test-recurrent-state-rollback.cpp
index f8eda55c8b..cac432e3e4 100644
--- a/tests/test-recurrent-state-rollback.cpp
+++ b/tests/test-recurrent-state-rollback.cpp
@@ -34,10 +34,10 @@ static const char * test_status_str(test_status status) {
return "";
}
-static bool decode_tokens(llama_context * ctx, const std::vector<llama_token> & tokens, uint32_t count) {
+static bool decode_tokens(llama_context * ctx, const std::vector<llama_token> & tokens) {
common_batch batch(ctx);
- for (uint32_t pos = 0; pos < count; ++pos) {
- batch.add(tokens[pos], pos, 0, pos + 1 == count);
+ for (uint32_t pos = 0; pos < tokens.size(); ++pos) {
+ batch.add(tokens[pos], pos, 0, pos + 1 == tokens.size());
}
return llama_process(ctx, LLAMA_PROCESS_TYPE_DECODE, batch.get()) == 0;
}
@@ -74,7 +74,7 @@ static llama_context_ptr init_ctx(llama_model * model, llama_context_params cpar
// Use a full ubatch so buffer discovery preserves prefill allocation sizes.
const uint32_t n_tokens = llama_n_ubatch(ctx.get());
- if (!decode_tokens(ctx.get(), std::vector<llama_token>(n_tokens, 0), n_tokens)) {
+ if (!decode_tokens(ctx.get(), std::vector<llama_token>(n_tokens, 0))) {
return nullptr;
}
llama_synchronize(ctx.get());
@@ -347,7 +347,7 @@ static test_status test_rollback(const common_params & params, llama_model * mod
// Decode the full prompt on the source, then roll back three positions.
// Replaying them crosses DSV4's ratio-4 compressor boundary.
// Rollback leaves the recurrent memory in a snapshot state (rs_idx != 0).
- if (!decode_tokens(ctx_src.get(), tokens, n_tokens)) {
+ if (!decode_tokens(ctx_src.get(), tokens)) {
LOG_ERR("%s: failed to decode prompt\n", __func__);
return test_status::FAIL;
}
@@ -432,7 +432,7 @@ static test_status test_rollback(const common_params & params, llama_model * mod
t = 0;
}
}
- if (!decode_tokens(ctx_dirty.get(), noise, n_tokens)) {
+ if (!decode_tokens(ctx_dirty.get(), noise)) {
LOG_ERR("%s: dirty prompt decode failed\n", __func__);
return test_status::FAIL;
}