LLM inference in C/C++
C++ ggml
server : allow RANK pooling batch splitting for causal LLM rerankers (ie. Qwen3 and Qwen3-VL) (#28876)
Tim Wang4da6337767f9
diff --git a/include/llama.h b/include/llama.hindex 1805ed0559..ce454df527 100644--- a/include/llama.h+++ b/include/llama.h@@ -1108,6 +1108,9 @@ extern "C" {// If set to true, the model will only attend to the past tokensLLAMA_API void llama_set_causal_attn(struct llama_context * ctx, bool causal_attn);+ // Returns whether the context is currently using causal attention+ LLAMA_API bool llama_get_causal_attn(const struct llama_context * ctx);+// Set whether the model is in warmup mode or not// If true, all model tensors are activated during llama_decode() to load and cache their weights.//diff --git a/src/llama-context.cpp b/src/llama-context.cppindex 99e55da687..27b9a38d72 100644--- a/src/llama-context.cpp+++ b/src/llama-context.cpp@@ -1259,6 +1259,10 @@ void llama_context::set_causal_attn(bool value) {sched_need_reserve = true;}+bool llama_context::get_causal_attn() const {+ return cparams.causal_attn;+}+void llama_context::set_warmup(bool value) {LLAMA_LOG_DEBUG("%s: value = %d\n", __func__, value);@@ -3933,6 +3937,10 @@ void llama_set_causal_attn(llama_context * ctx, bool causal_attn) {ctx->set_causal_attn(causal_attn);}+bool llama_get_causal_attn(const llama_context * ctx) {+ return ctx->get_causal_attn();+}+void llama_set_warmup(llama_context * ctx, bool warmup) {ctx->set_warmup(warmup);}diff --git a/src/llama-context.h b/src/llama-context.hindex b403b099b7..35a7071eda 100644--- a/src/llama-context.h+++ b/src/llama-context.h@@ -103,6 +103,8 @@ struct llama_context {const llama_token * get_sampled_candidates_ith(int32_t idx);size_t get_sampled_candidates_count(int32_t idx);+ bool get_causal_attn() const;+void attach_threadpool(ggml_threadpool_t threadpool,ggml_threadpool_t threadpool_batch);diff --git a/src/llama-graph.cpp b/src/llama-graph.cppindex 0b3bab6123..a806126efa 100644--- a/src/llama-graph.cpp+++ b/src/llama-graph.cpp@@ -297,7 +297,7 @@ void llm_graph_input_cls::set_input(const llama_ubatch * ubatch) {const bool last = (cparams.pooling_type == LLAMA_POOLING_TYPE_LAST ||- (cparams.pooling_type == LLAMA_POOLING_TYPE_RANK && (arch == LLM_ARCH_QWEN3 || arch == LLM_ARCH_QWEN3VL)) // qwen3 reranking & embedding models use last token+ (cparams.pooling_type == LLAMA_POOLING_TYPE_RANK && cparams.causal_attn));for (int i = 0; i < n_tokens; ++i) {diff --git a/tools/server/server-context.cpp b/tools/server/server-context.cppindex e95fb63ab3..611e82a6af 100644--- a/tools/server/server-context.cpp+++ b/tools/server/server-context.cpp@@ -435,15 +435,26 @@ struct server_slot {return task->need_embd();}- // if the context does not have a memory module then all embeddings have to be computed within a single ubatch- // also we cannot split if the pooling would require any past tokens- // (MTP supports splitting — uses task->need_embd() not need_embd())bool can_split() const {GGML_ASSERT(task);-- return- !task->need_embd() ||- (llama_get_memory(ctx_tgt) && llama_pooling_type(ctx_tgt) == LLAMA_POOLING_TYPE_LAST);+ // MTP supports splitting - uses task->need_embd() not need_embd()+ if (!task->need_embd()) {+ return true;+ }+ // if the context does not have a memory module then all embeddings have to be computed within a single ubatch+ if (!llama_get_memory(ctx_tgt)) {+ return false;+ }+ // context can be chunked/split if the pooling type is LAST+ const auto pooling = llama_pooling_type(ctx_tgt);+ if (pooling == LLAMA_POOLING_TYPE_LAST) {+ return true;+ }+ // causal rerankers read the last token and have a KV cache, so they can also be chunked/split.+ if (pooling == LLAMA_POOLING_TYPE_RANK && llama_get_causal_attn(ctx_tgt)) {+ return true;+ }+ return false;}bool can_batch_with(server_slot & other_slot) const {