LLM inference in C/C++
C++ ggml
model: fix vllm d3 multi seq case (#30408)
Xuan-Son Nguyencbe28bb3cb07
diff --git a/src/llama-batch.h b/src/llama-batch.hindex 8882c9941e..e1e43979b5 100644--- a/src/llama-batch.h+++ b/src/llama-batch.h@@ -186,6 +186,7 @@ public:llama_ubatch split_equal(uint32_t n_ubatch, bool sequential, uint32_t n_keep_tail);// sequence-set-wise split - each ubatch contains a single sequence-set+ // TODO: extend to support multiple _full_ sequences in the ubatchllama_ubatch split_seq(uint32_t n_ubatch);// a helper method for creating a well-defined ubatch of tokensdiff --git a/src/llama-context.cpp b/src/llama-context.cppindex 23170d8a37..231b34dc28 100644--- a/src/llama-context.cpp+++ b/src/llama-context.cpp@@ -1838,6 +1838,8 @@ int llama_context::decode(const llama_batch_ext & batch_inp) {// when computing embeddings, all tokens are outputconst bool output_all = cparams.embeddings;+ // non-causal attention needs the whole sequence in one ubatch+ const bool embd_all = output_all || !cparams.causal_attn;const bool has_samplers = !sampling.samplers.empty();const uint32_t n_seq_max = cparams.kv_unified ? LLAMA_MAX_SEQ : cparams.n_seq_max;@@ -1916,7 +1918,7 @@ int llama_context::decode(const llama_batch_ext & batch_inp) {llama_memory_context_ptr mctx;while (true) {- mctx = memory->init_batch(*balloc, cparams.n_ubatch, output_all);+ mctx = memory->init_batch(*balloc, cparams.n_ubatch, embd_all);if (!mctx) {return -2;}diff --git a/src/llama-kv-cache-iswa.cpp b/src/llama-kv-cache-iswa.cppindex e91866469a..a6ce547e64 100644--- a/src/llama-kv-cache-iswa.cpp+++ b/src/llama-kv-cache-iswa.cpp@@ -157,8 +157,6 @@ std::map<ggml_backend_buffer_type_t, size_t> llama_kv_cache_iswa::memory_breakdo}llama_memory_context_ptr llama_kv_cache_iswa::init_batch(llama_batch_allocr & balloc, uint32_t n_ubatch, bool embd_all) {- GGML_UNUSED(embd_all);-// first try simple splitdo {if (!unified) {@@ -206,7 +204,7 @@ llama_memory_context_ptr llama_kv_cache_iswa::init_batch(llama_batch_allocr & bastd::vector<llama_ubatch> ubatches;while (true) {- auto ubatch = balloc.split_equal(n_ubatch, !unified, 0);+ auto ubatch = embd_all ? balloc.split_seq(n_ubatch) : balloc.split_equal(n_ubatch, !unified, 0);if (ubatch.n_tokens == 0) {break;diff --git a/src/llama-kv-cache.cpp b/src/llama-kv-cache.cppindex 463510caa7..804c90532c 100644--- a/src/llama-kv-cache.cpp+++ b/src/llama-kv-cache.cpp@@ -721,14 +721,12 @@ llama_memory_context_ptr llama_kv_cache::init_batch(llama_batch_allocr & balloc,uint32_t n_ubatch,bool embd_all) {- GGML_UNUSED(embd_all);-do {balloc.split_reset();std::vector<llama_ubatch> ubatches;while (true) {- auto ubatch = n_stream == 1 ? balloc.split_simple(n_ubatch) : balloc.split_equal(n_ubatch, true, 0);+ auto ubatch = n_stream == 1 ? balloc.split_simple(n_ubatch) : embd_all ? balloc.split_seq(n_ubatch) : balloc.split_equal(n_ubatch, true, 0);if (ubatch.n_tokens == 0) {break;diff --git a/src/llama-memory.h b/src/llama-memory.hindex db82539664..b33c7aee41 100644--- a/src/llama-memory.h+++ b/src/llama-memory.h@@ -85,6 +85,7 @@ struct llama_memory_i {// split the input batch into a set of ubatches and verify that they can fit into the cache// return a context object containing the ubatches and memory state required to process them// check the llama_memory_context_i::get_status() for the result+ // embd_all: keep all the tokens of a sequence in one ubatch (all tokens are output, or the attention is non-causal)virtual llama_memory_context_ptr init_batch(llama_batch_allocr & balloc,uint32_t n_ubatch,