LLM inference in C/C++
C++ ggml
server : support typed content input for RERANK endpoint (#29625)
Tim Wanga5d22276be84
diff --git a/tools/server/README.md b/tools/server/README.mdindex 71310d9030..56d1ea5fe8 100644--- a/tools/server/README.md+++ b/tools/server/README.md@@ -771,37 +771,81 @@ This endpoint also supports multimodal embeddings. See the documentation for the### POST `/reranking`: Rerank documents according to a given query+> [!IMPORTANT]+>+> This endpoint is **not** OAI-compatible.Similar to https://jina.ai/reranker/ but might change in the future.-Requires a reranker model (such as [bge-reranker-v2-m3](https://huggingface.co/BAAI/bge-reranker-v2-m3)) and the `--embedding --pooling rank` options.--*Options:*+Requires a reranker model (such as [bge-reranker-v2-m3](https://huggingface.co/BAAI/bge-reranker-v2-m3)) and the `--embedding --pooling rank` or `--reranking` options.-`query`: The query against which the documents will be ranked.+`query` and each `documents`/`texts` element accept a plain string, or+(for multimodal rerankers such as Qwen3-VL-Reranker) an object of the form+`{"text": string | [strings], "image": url | data-uri | raw-base64 | [urls]}`.+All keys are optional; a plain string is treated as `{"text": "..."}`.-`documents`: An array strings representing the documents to be ranked.+Options: `top_n` (default: all), `model` (echoed in the Jina response), `return_text` (TEI only).*Aliases:*- `/rerank`- `/v1/rerank`- `/v1/reranking`-*Examples:*+The format is selected by which key carries the documents: `texts` (TEI, same input shape as Jina,+set `return_text: true` to echo document text) or `documents` (Jina, multimodal supported).++**Text only (TEI format)** - `query` + `texts`; set `return_text: true` to echo document text:```shellcurl http://127.0.0.1:8012/v1/rerank \-H "Content-Type: application/json" \-d '{- "model": "some-model",- "query": "What is panda?",- "top_n": 3,- "documents": [- "hi",- "it is a bear",- "The giant panda (Ailuropoda melanoleuca), sometimes called a panda bear or simply panda, is a bear species endemic to China."- ]+ "query": "What is machine learning?",+ "texts": [+ "Machine learning is a subset of artificial intelligence.",+ "A python is a large snake."+ ],+ "return_text": true}' | jq```+**Multimodal (Jina Format)** - mix text and images in `query` and/or `documents`:++```shell+curl http://127.0.0.1:8012/v1/rerank \+ -H "Content-Type: application/json" \+ -d '{+ "model": "Qwen/Qwen3-VL-Reranker-2B",+ "query": {+ "text": "A city skyline with tall buildings and a river",+ "image": "data:image/jpeg;base64,<base64>"+ },+ "documents": [+ "a plain string document is also valid",+ {+ "text": "A photograph of the Boston skyline at dusk",+ "image": "https://example.com/skyline.jpg"+ },+ {+ "text": "caption for the two-figure spread",+ "image": ["https://example.com/fig1.png", "data:image/jpeg;base64,<base64>"]+ }+ ],+ "top_n": 2+ }' | jq+```++Jina format returns a wrapped object with `model`, `object`, `usage` (`prompt_tokens` is the sum+of `tokens_evaluated` across all scored pairs), and a `results` array of `{index, relevance_score}`.+TEI format returns a bare array of `{index, score}` (+ `text` when `return_text: true`).+Both are sorted descending by score and truncated to `top_n`; the score is the match probability in `[0, 1]`.++Images never appear in any response shape.++> **NOTE:** **Input size limit:** A bidirectional-encoder reranker (e.g. `bge-reranker-v2-m3`) must fit the whole prompt within+`--ubatch-size`; if the input is too large to process, increase `--ubatch-size`.+Causal decoder rerankers (e.g. `Qwen3-VL-Reranker`) have no such limitation.+++### POST `/tts`: Generate speech audio from textReturns raw audio bytes (`audio/wav` by default) rather than JSON. For more info, see [tts/README.md](../tts/README.md)diff --git a/tools/server/server-common.cpp b/tools/server/server-common.cppindex baaca58eb3..4f1acba1a7 100644--- a/tools/server/server-common.cpp+++ b/tools/server/server-common.cpp@@ -1265,7 +1265,6 @@ void handle_media(out_files.push_back(decoded_data);}}-// load media files from an OAI content array, then replace each media part with a media marker text partstatic void oaicompat_content_load_media(json & content, const server_chat_params & opt, std::vector<raw_buffer> & out_files) {for (auto & p : content) {@@ -1336,6 +1335,7 @@ server_tokens tokenize_oai_content_array(const llama_vocab * vocab, mtmd_contextif (files.empty()) {return server_tokens(common_tokenize(vocab, prompt, add_special, parse_special), false);}+return process_mtmd_prompt(mctx, prompt, files, init_opt);}@@ -1951,18 +1951,49 @@ server_tokens format_prompt_rerank(mtmd_context * mctx,const std::string & query,const std::string & doc,+ const std::vector<raw_buffer> & query_files,+ const std::vector<raw_buffer> & doc_files,const mtmd_helper_init_opt & init_opt) {server_tokens result = {};+ // media must be spliced through the mtmd pipeline+ if ((!query_files.empty() || !doc_files.empty()) && mctx == nullptr) {+ throw std::invalid_argument("multimodal rerank input requires --mmproj");+ }+const char * rerank_prompt = llama_model_chat_template(model, "rerank");if (rerank_prompt != nullptr) {+ // multimodal: append one media marker per image (arrays already flattened by caller)+ std::string q = query;+ for (size_t i = 0; i < query_files.size(); i++) {+ q += get_media_marker();+ }+ std::string d = doc;+ for (size_t i = 0; i < doc_files.size(); i++) {+ d += get_media_marker();+ }+std::string prompt = rerank_prompt;- string_replace_all(prompt, "{query}" , query);- string_replace_all(prompt, "{document}", doc );- server_tokens tokens = tokenize_input_subprompt(vocab, mctx, prompt, false, true, init_opt);- result.push_back(tokens);+ string_replace_all(prompt, "{query}" , q);+ string_replace_all(prompt, "{document}", d );++ if (query_files.empty() && doc_files.empty()) {+ server_tokens tokens = tokenize_input_subprompt(vocab, mctx, prompt, false, true, init_opt);+ result.push_back(tokens);+ } else {+ // process_mtmd_prompt returns a complete mtmd server_tokens; return it directly+ // (push_back into a non-mtmd result would hit the NULL placeholder tokens)+ std::vector<raw_buffer> files;+ files.insert(files.end(), query_files.begin(), query_files.end());+ files.insert(files.end(), doc_files.begin(), doc_files.end());+ return process_mtmd_prompt(mctx, prompt, files, init_opt);+ }} else {+ if (!query_files.empty() || !doc_files.empty()) {+ throw std::invalid_argument("multimodal rerank input requires a model with a rerank chat template");+ }+// Get EOS token - use SEP token as fallback if EOS is not availableserver_tokens query_tokens = tokenize_input_subprompt(vocab, mctx, query, false, false, init_opt);server_tokens doc_tokens = tokenize_input_subprompt(vocab, mctx, doc, false, false, init_opt);diff --git a/tools/server/server-common.h b/tools/server/server-common.hindex 538e008e94..8bbff7726c 100644--- a/tools/server/server-common.h+++ b/tools/server/server-common.h@@ -303,8 +303,11 @@ server_tokens process_mtmd_prompt(mtmd_context * mctx,const std::string & prompt,const std::vector<raw_buffer> & files,- const mtmd_helper_init_opt & init_opt,- bool is_placeholder = false);+ const mtmd_helper_init_opt & init_opt,+ bool is_placeholder = false);++// decode one media url (http/file/data-uri) into out_files; throws on failure+void handle_media(std::vector<raw_buffer> & out_files, const std::string & url, const std::string & media_path);/*** break the input "prompt" object into multiple prompt if needed, then tokenize them@@ -590,12 +593,16 @@ llama_tokens format_prompt_infill(const llama_tokens & tokens_prompt);// format rerank task: [BOS]query[EOS][SEP]doc[EOS].+// query_files/doc_files carry decoded media for multimodal (Qwen) inputs; when non-empty the+// prompt is built with get_media_marker() sentinels and spliced via process_mtmd_prompt.server_tokens format_prompt_rerank(const struct llama_model * model,const struct llama_vocab * vocab,mtmd_context * mctx,const std::string & query,const std::string & doc,+ const std::vector<raw_buffer> & query_files,+ const std::vector<raw_buffer> & doc_files,const mtmd_helper_init_opt & init_opt);// simple implementation of a pipediff --git a/tools/server/server-context.cpp b/tools/server/server-context.cppindex 296893f7a1..a0acf16559 100644--- a/tools/server/server-context.cpp+++ b/tools/server/server-context.cpp@@ -6284,83 +6284,7 @@ void server_routes::init_routes() {};this->post_rerank = [this](const server_http_req & req) {- auto res = create_response();- if (!params.embedding || params.pooling_type != LLAMA_POOLING_TYPE_RANK) {- res->error(format_error_response("This server does not support reranking. Start it with `--reranking`", ERROR_TYPE_NOT_SUPPORTED));- return res;- }-- const json body = json::parse(req.body);-- // if true, use TEI API format, otherwise use Jina API format- // Jina: https://jina.ai/reranker/- // TEI: https://huggingface.github.io/text-embeddings-inference/#/Text%20Embeddings%20Inference/rerank- bool is_tei_format = body.contains("texts");-- json query;- if (body.count("query") == 1) {- query = body.at("query");- if (!query.is_string()) {- res->error(format_error_response("\"query\" must be a string", ERROR_TYPE_INVALID_REQUEST));- return res;- }- } else {- res->error(format_error_response("\"query\" must be provided", ERROR_TYPE_INVALID_REQUEST));- return res;- }-- std::vector<std::string> documents = json_value(body, "documents",- json_value(body, "texts", std::vector<std::string>()));- if (documents.empty()) {- res->error(format_error_response("\"documents\" must be a non-empty string array", ERROR_TYPE_INVALID_REQUEST));- return res;- }-- int top_n = json_value(body, "top_n", (int)documents.size());-- // create and queue the task- json responses = json::array();- auto & rd = res->rd;- {- std::vector<server_task> tasks;- tasks.reserve(documents.size());- for (size_t i = 0; i < documents.size(); i++) {- auto tmp = format_prompt_rerank(ctx_server.model_tgt, ctx_server.vocab, ctx_server.mctx, query, documents[i], ctx_server.init_opt);- server_task task = server_task(SERVER_TASK_TYPE_RERANK);- task.id = rd.get_new_id();- task.tokens = std::move(tmp);- tasks.push_back(std::move(task));- }- rd.post_tasks(std::move(tasks));- }-- // wait for the results- auto all_results = rd.wait_for_all(req.should_stop);-- // collect results- if (all_results.is_terminated) {- return res; // connection is closed- } else if (all_results.error) {- res->error(all_results.error->to_json());- return res;- } else {- for (auto & res : all_results.results) {- GGML_ASSERT(dynamic_cast<server_task_result_rerank*>(res.get()) != nullptr);- responses.push_back(res->to_json());- }- }-- // write JSON response- json root = format_response_rerank(- body,- meta->model_name,- responses,- is_tei_format,- documents,- top_n);-- res->ok(root);- return res;+ return handle_rerank_impl(req);};this->post_tts = [this](const server_http_req & req) {@@ -6822,6 +6746,7 @@ std::unique_ptr<server_res_generator> server_routes::handle_embeddings_impl(consreturn tokenize_input_subprompt(ctx_server.vocab, ctx_server.mctx, p, true, true, ctx_server.init_opt);};+ // dispatch to the appropriate tokenizer based on the shape of the inputstd::vector<server_tokens> tokenized_prompts;if (prompt.is_array() && !json_is_array_and_contains_numbers(prompt)) {for (const auto & p : prompt) {@@ -6895,6 +6820,154 @@ std::unique_ptr<server_res_generator> server_routes::handle_embeddings_impl(consreturn res;}+std::unique_ptr<server_res_generator> server_routes::handle_rerank_impl(const server_http_req & req) {+ auto res = create_response();+ if (!params.embedding || params.pooling_type != LLAMA_POOLING_TYPE_RANK) {+ res->error(format_error_response("This server does not support reranking. Start it with `--reranking`", ERROR_TYPE_NOT_SUPPORTED));+ return res;+ }++ const json body = json::parse(req.body);++ // if true, use TEI API format, otherwise use Jina API format+ // Jina: https://jina.ai/reranker/+ // TEI: https://huggingface.github.io/text-embeddings-inference/#/Text%20Embeddings%20Inference/rerank+ bool is_tei_format = body.contains("texts");++ // expand one rerank input item: plain string or {"text": str|[str], "image": url|data-uri|raw-base64|[urls]}+ auto parse_item = [&](const json & item, std::string & out_text, std::vector<raw_buffer> & out_files) {+ if (item.is_string()) {+ out_text = item.get<std::string>();+ return;+ }+ if (!item.is_object()) {+ throw std::invalid_argument("rerank input must be a string or an object of the form {\"text\": ..., \"image\": ...}");+ }+ if (item.contains("video") || item.contains("fps") || item.contains("max_frames")) {+ throw std::invalid_argument("video/fps/max_frames are not supported for rerank input");+ }+ json text = item.value("text", json());+ if (text.is_string()) {+ out_text += text.get<std::string>();+ } else if (text.is_array()) {+ for (const auto & t : text) {+ if (!t.is_string()) {+ throw std::invalid_argument("\"text\" array elements must be strings");+ }+ out_text += t.get<std::string>();+ }+ } else if (!text.is_null()) {+ throw std::invalid_argument("\"text\" must be a string or an array of strings");+ }+ json image = item.value("image", json());+ auto add_image = [&](const std::string & url) {+ // marker is appended by format_prompt_rerank, one per decoded file+ try {+ handle_media(out_files, url, params.media_path);+ } catch (const std::runtime_error & e) {+ throw std::invalid_argument(e.what());+ }+ };+ if (image.is_string()) {+ add_image(image.get<std::string>());+ } else if (image.is_array()) {+ for (const auto & u : image) {+ if (!u.is_string()) {+ throw std::invalid_argument("\"image\" array elements must be strings");+ }+ add_image(u.get<std::string>());+ }+ } else if (!image.is_null()) {+ throw std::invalid_argument("\"image\" must be a string or an array of strings");+ }+ };++ // Parse query+ std::string query_str;+ std::vector<raw_buffer> query_files;+ if (body.count("query") == 1) {+ try {+ parse_item(body.at("query"), query_str, query_files);+ } catch (const std::invalid_argument & e) {+ res->error(format_error_response(e.what(), ERROR_TYPE_INVALID_REQUEST));+ return res;+ }+ } else {+ res->error(format_error_response("\"query\" must be provided", ERROR_TYPE_INVALID_REQUEST));+ return res;+ }++ // Resolve Jina/TEI format ambiguity+ json documents_json = json_value(body, "documents",+ json_value(body, "texts", json::array()));+ if (!documents_json.is_array() || documents_json.empty()) {+ res->error(format_error_response("\"documents\" must be a non-empty array of strings or objects", ERROR_TYPE_INVALID_REQUEST));+ return res;+ }++ // Parse documents+ std::vector<std::string> documents(documents_json.size());+ std::vector<std::vector<raw_buffer>> document_files(documents_json.size());+ for (size_t i = 0; i < documents_json.size(); i++) {+ try {+ parse_item(documents_json[i], documents[i], document_files[i]);+ } catch (const std::invalid_argument & e) {+ res->error(format_error_response(e.what(), ERROR_TYPE_INVALID_REQUEST));+ return res;+ }+ }++ int top_n = json_value(body, "top_n", (int)documents.size());+ if (top_n < 0) {+ res->error(format_error_response("\"top_n\" must be >= 0", ERROR_TYPE_INVALID_REQUEST));+ return res;+ }++ // create and queue the task+ json responses = json::array();+ auto & rd = res->rd;+ {+ std::vector<server_task> tasks;+ tasks.reserve(documents.size());+ for (size_t i = 0; i < documents.size(); i++) {+ auto tmp = format_prompt_rerank(ctx_server.model_tgt, ctx_server.vocab, ctx_server.mctx, query_str, documents[i], query_files, document_files[i], ctx_server.init_opt);+ server_task task = server_task(SERVER_TASK_TYPE_RERANK);+ task.id = rd.get_new_id();+ task.tokens = std::move(tmp);+ tasks.push_back(std::move(task));+ }+ rd.post_tasks(std::move(tasks));+ }++ // wait for the results+ auto all_results = rd.wait_for_all(req.should_stop);++ // collect results+ if (all_results.is_terminated) {+ return res; // connection is closed+ } else if (all_results.error) {+ res->error(all_results.error->to_json());+ return res;+ } else {+ for (auto & res : all_results.results) {+ GGML_ASSERT(dynamic_cast<server_task_result_rerank*>(res.get()) != nullptr);+ responses.push_back(res->to_json());+ }+ }++ // write JSON response+ json root = format_response_rerank(+ body,+ meta->model_name,+ responses,+ is_tei_format,+ documents,+ top_n);++ res->ok(root);+ return res;+}+std::unique_ptr<server_res_generator> server_routes::handle_count_tokens(const server_http_req & req, task_response_type res_type) {auto res = create_response();std::vector<raw_buffer> files;diff --git a/tools/server/server-context.h b/tools/server/server-context.hindex a579b0ecad..4a8e327fcd 100644--- a/tools/server/server-context.h+++ b/tools/server/server-context.h@@ -177,6 +177,7 @@ private:std::unique_ptr<server_res_generator> handle_slots_restore(const server_http_req & req, int id_slot);std::unique_ptr<server_res_generator> handle_slots_erase(const server_http_req &, int id_slot);std::unique_ptr<server_res_generator> handle_embeddings_impl(const server_http_req & req, task_response_type res_type);+ std::unique_ptr<server_res_generator> handle_rerank_impl(const server_http_req & req);std::unique_ptr<server_res_generator> handle_count_tokens(const server_http_req & req, task_response_type res_type);// using unique_ptr to allow late initialization of const