LLM inference in C/C++
C++ ggml
server: Substitute rerank placeholder from the original template only. (#30864)
pzhu184f65360246
diff --git a/tests/test-server-component.cpp b/tests/test-server-component.cppindex aa353e190b..18e0cdc2fe 100644--- a/tests/test-server-component.cpp+++ b/tests/test-server-component.cpp@@ -435,6 +435,35 @@ static void test_decision_escape(testing & t) {});}+static void test_rerank_template(testing & t) {+ struct test_case {+ const char * name;+ std::string tmpl;+ std::string query;+ std::string doc;+ std::string expected;+ };+ const test_case cases[] = {+ {"plain inputs", "Q: {query}\nD: {document}", "query", "doc", "Q: query\nD: doc"},+ {"document placeholder in query", "Q: {query}\nD: {document}", "Explain {document}.", "doc", "Q: Explain {document}.\nD: doc"},+ {"query placeholder in query", "{query}|{document}", "{query}", "doc", "{query}|doc"},+ {"placeholders in both inputs", "{query}|{document}", "{query} {document}", "{query} {document}", "{query} {document}|{query} {document}"},+ {"repeated placeholders", "{query}|{document}|{query}|{document}", "{document}", "doc", "{document}|doc|{document}|doc"},+ {"adjacent reversed placeholders", "{document}{query}", "query", "doc", "docquery"},+ {"query only", "{query}", "{document}", "doc", "{document}"},+ {"document only", "{document}", "query", "{query}", "{query}"},+ {"no placeholders", "fixed", "{document}", "{query}", "fixed"},+ {"unknown and incomplete placeholders", "{unknown}|{query}|{document}|{query", "query", "doc", "{unknown}|query|doc|{query"},+ {"nested braces", "{{query}}|{{document}}", "query", "doc", "{query}|{doc}"},+ {"placeholder formed at boundary", "{query}ument}", "{doc", "doc", "{document}"},+ {"empty inputs", "Q: {query}\nD: {document}", "", "", "Q: \nD: "},+ {"empty template", "", "query", "doc", ""},+ };+ for (const auto & test : cases) {+ t.assert_equal(test.name, test.expected, format_rerank_template(test.tmpl, test.query, test.doc));+ }+}+int main(int argc, char ** argv) {testing t;@@ -454,6 +483,7 @@ int main(int argc, char ** argv) {t.test("lora_helpers", test_lora_helpers);t.test("server_pipe", test_server_pipe);t.test("decision_escape", test_decision_escape);+ t.test("rerank_template", test_rerank_template);return t.summary();}diff --git a/tools/server/server-common.cpp b/tools/server/server-common.cppindex 4f1acba1a7..10e04d370a 100644--- a/tools/server/server-common.cpp+++ b/tools/server/server-common.cpp@@ -1945,6 +1945,26 @@ llama_tokens format_prompt_infill(return embd_inp;}+std::string format_rerank_template(+ const std::string & tmpl,+ const std::string & query,+ const std::string & doc) {+ std::string prompt;+ prompt.reserve(tmpl.size());+ for (size_t pos = 0; pos < tmpl.size();) {+ if (tmpl.compare(pos, 7, "{query}") == 0) {+ prompt += query;+ pos += 7;+ } else if (tmpl.compare(pos, 10, "{document}") == 0) {+ prompt += doc;+ pos += 10;+ } else {+ prompt += tmpl[pos++];+ }+ }+ return prompt;+}+server_tokens format_prompt_rerank(const struct llama_model * model,const struct llama_vocab * vocab,@@ -1974,9 +1994,7 @@ server_tokens format_prompt_rerank(d += get_media_marker();}- std::string prompt = rerank_prompt;- string_replace_all(prompt, "{query}" , q);- string_replace_all(prompt, "{document}", d );+ std::string prompt = format_rerank_template(rerank_prompt, q, d);if (query_files.empty() && doc_files.empty()) {server_tokens tokens = tokenize_input_subprompt(vocab, mctx, prompt, false, true, init_opt);diff --git a/tools/server/server-common.h b/tools/server/server-common.hindex 8bbff7726c..09ffa9a56f 100644--- a/tools/server/server-common.h+++ b/tools/server/server-common.h@@ -592,6 +592,11 @@ llama_tokens format_prompt_infill(const bool spm_infill,const llama_tokens & tokens_prompt);+std::string format_rerank_template(+ const std::string & tmpl,+ const std::string & query,+ const std::string & doc);+// format rerank task: [BOS]query[EOS][SEP]doc[EOS].// query_files/doc_files carry decoded media for multimodal (Qwen) inputs; when non-empty the// prompt is built with get_media_marker() sentinels and spliced via process_mtmd_prompt.