Skip to content

LLM inference in C/C++

C++ ggml

server : allow RANK pooling batch splitting for causal LLM rerankers (ie. Qwen3 and Qwen3-VL) (#28876)

Tim Wang4da6337767f9
diff --git a/include/llama.h b/include/llama.h
index 1805ed0559..ce454df527 100644
--- a/include/llama.h
+++ b/include/llama.h
@@ -1108,6 +1108,9 @@ extern "C" {
// If set to true, the model will only attend to the past tokens
LLAMA_API void llama_set_causal_attn(struct llama_context * ctx, bool causal_attn);
+ // Returns whether the context is currently using causal attention
+ LLAMA_API bool llama_get_causal_attn(const struct llama_context * ctx);
+
// Set whether the model is in warmup mode or not
// If true, all model tensors are activated during llama_decode() to load and cache their weights.
//
diff --git a/src/llama-context.cpp b/src/llama-context.cpp
index 99e55da687..27b9a38d72 100644
--- a/src/llama-context.cpp
+++ b/src/llama-context.cpp
@@ -1259,6 +1259,10 @@ void llama_context::set_causal_attn(bool value) {
sched_need_reserve = true;
}
+bool llama_context::get_causal_attn() const {
+ return cparams.causal_attn;
+}
+
void llama_context::set_warmup(bool value) {
LLAMA_LOG_DEBUG("%s: value = %d\n", __func__, value);
@@ -3933,6 +3937,10 @@ void llama_set_causal_attn(llama_context * ctx, bool causal_attn) {
ctx->set_causal_attn(causal_attn);
}
+bool llama_get_causal_attn(const llama_context * ctx) {
+ return ctx->get_causal_attn();
+}
+
void llama_set_warmup(llama_context * ctx, bool warmup) {
ctx->set_warmup(warmup);
}
diff --git a/src/llama-context.h b/src/llama-context.h
index b403b099b7..35a7071eda 100644
--- a/src/llama-context.h
+++ b/src/llama-context.h
@@ -103,6 +103,8 @@ struct llama_context {
const llama_token * get_sampled_candidates_ith(int32_t idx);
size_t get_sampled_candidates_count(int32_t idx);
+ bool get_causal_attn() const;
+
void attach_threadpool(
ggml_threadpool_t threadpool,
ggml_threadpool_t threadpool_batch);
diff --git a/src/llama-graph.cpp b/src/llama-graph.cpp
index 0b3bab6123..a806126efa 100644
--- a/src/llama-graph.cpp
+++ b/src/llama-graph.cpp
@@ -297,7 +297,7 @@ void llm_graph_input_cls::set_input(const llama_ubatch * ubatch) {
const bool last = (
cparams.pooling_type == LLAMA_POOLING_TYPE_LAST ||
- (cparams.pooling_type == LLAMA_POOLING_TYPE_RANK && (arch == LLM_ARCH_QWEN3 || arch == LLM_ARCH_QWEN3VL)) // qwen3 reranking & embedding models use last token
+ (cparams.pooling_type == LLAMA_POOLING_TYPE_RANK && cparams.causal_attn)
);
for (int i = 0; i < n_tokens; ++i) {
diff --git a/tools/server/server-context.cpp b/tools/server/server-context.cpp
index e95fb63ab3..611e82a6af 100644
--- a/tools/server/server-context.cpp
+++ b/tools/server/server-context.cpp
@@ -435,15 +435,26 @@ struct server_slot {
return task->need_embd();
}
- // if the context does not have a memory module then all embeddings have to be computed within a single ubatch
- // also we cannot split if the pooling would require any past tokens
- // (MTP supports splitting — uses task->need_embd() not need_embd())
bool can_split() const {
GGML_ASSERT(task);
-
- return
- !task->need_embd() ||
- (llama_get_memory(ctx_tgt) && llama_pooling_type(ctx_tgt) == LLAMA_POOLING_TYPE_LAST);
+ // MTP supports splitting - uses task->need_embd() not need_embd()
+ if (!task->need_embd()) {
+ return true;
+ }
+ // if the context does not have a memory module then all embeddings have to be computed within a single ubatch
+ if (!llama_get_memory(ctx_tgt)) {
+ return false;
+ }
+ // context can be chunked/split if the pooling type is LAST
+ const auto pooling = llama_pooling_type(ctx_tgt);
+ if (pooling == LLAMA_POOLING_TYPE_LAST) {
+ return true;
+ }
+ // causal rerankers read the last token and have a KV cache, so they can also be chunked/split.
+ if (pooling == LLAMA_POOLING_TYPE_RANK && llama_get_causal_attn(ctx_tgt)) {
+ return true;
+ }
+ return false;
}
bool can_batch_with(server_slot & other_slot) const {