LLM inference in C/C++
C++ ggml
chat : add qwen3 specialized parser (#26252)
Aldehir Rojasf5919bf458ef
diff --git a/common/chat-peg-parser.cpp b/common/chat-peg-parser.cppindex f786f5ff23..1910b4f1e1 100644--- a/common/chat-peg-parser.cpp+++ b/common/chat-peg-parser.cpp@@ -6,6 +6,9 @@#include <nlohmann/json.hpp>+#include <cstdint>+#include <functional>+using ordered_json = nlohmann::ordered_json;static std::string_view trim_trailing_space(std::string_view sv, int max = -1) {@@ -235,6 +238,43 @@ common_peg_parser common_chat_peg_builder::tag_with_safe_content(const std::strireturn zero_or_more(choice({ p, content_chunk }));}+common_peg_parser common_chat_peg_builder::permute(const std::string & rule_prefix,+ const std::vector<common_peg_parser> & parsers) {+ if (parsers.empty()) {+ return eps();+ }++ if (parsers.size() == 1 || parsers.size() > COMMON_CHAT_MAX_PERMUTE) {+ return sequence(parsers);+ }++ std::map<uint32_t, common_peg_parser> rules;+ std::function<common_peg_parser(uint32_t)> remaining_of;++ remaining_of = [&](uint32_t remaining) -> common_peg_parser {+ if (remaining == 0) {+ return eps();+ }++ auto cached = rules.find(remaining);+ if (cached != rules.end()) {+ return cached->second;+ }++ auto alternatives = choice();+ for (size_t i = 0; i < parsers.size(); i++) {+ const uint32_t bit = 1u << i;+ if (remaining & bit) {+ alternatives |= parsers[i] + remaining_of(remaining & ~bit);+ }+ }++ return rules.emplace(remaining, rule(rule_prefix + "-" + std::to_string(remaining), alternatives)).first->second;+ };++ return remaining_of((1u << parsers.size()) - 1);+}+std::string & common_chat_peg_mapper::args_target() {return (current_tool && !current_tool->name.empty()) ? current_tool->arguments : args_buffer;}diff --git a/common/chat-peg-parser.h b/common/chat-peg-parser.hindex cd14f2c117..5d764dbaa0 100644--- a/common/chat-peg-parser.h+++ b/common/chat-peg-parser.h@@ -55,6 +55,8 @@ class common_chat_peg_minimax_m3_mapper : public common_chat_peg_mapper {struct content_structure;struct tool_call_structure;+constexpr size_t COMMON_CHAT_MAX_PERMUTE = 6;+class common_chat_peg_builder : public common_peg_parser_builder {public:// Tag constants (from former common_chat_peg_base_builder)@@ -105,6 +107,9 @@ class common_chat_peg_builder : public common_peg_parser_builder {common_peg_parser tool_arg_json_value(const common_peg_parser & p) { return tag(TOOL_ARG_VALUE, p); }+ // Matches every parser exactly once, in any order.+ common_peg_parser permute(const std::string & rule_prefix, const std::vector<common_peg_parser> & parsers);+// Return a parser that parses the prefix of a string, up to a given delimiter.common_peg_parser prefix(const std::string & s, const std::string & delimiter = {});diff --git a/common/chat.cpp b/common/chat.cppindex 0e06fa591f..f7431a6f13 100644--- a/common/chat.cpp+++ b/common/chat.cpp@@ -1110,6 +1110,172 @@ static common_chat_params common_chat_params_init_ministral_3(const common_chat_return data;}+static common_chat_params common_chat_params_init_qwen3_coder(const common_chat_template & tmpl,+ const autoparser::generation_params & inputs) {+ common_chat_params data;++ const std::string GEN_PREFIX = "<|im_start|>assistant\n";++ data.prompt = common_chat_template_direct_apply_impl(tmpl, inputs);+ data.generation_prompt = common_chat_template_generation_prompt_impl(tmpl, inputs);+ data.format = COMMON_CHAT_FORMAT_PEG_NATIVE;++ auto supports_reasoning = tmpl.source().find("<think>") != std::string::npos;++ data.supports_thinking = supports_reasoning;+ data.preserved_tokens = {+ "<tool_call>",+ "</tool_call>",+ };++ if (supports_reasoning) {+ data.thinking_start_tag = "<think>";+ // Support both </think> and <tool_call> as reasoning end sequences.+ // <function= is omitted, as it is a workaround for Qwen3-Coder which is not a thinking model+ data.thinking_end_tags = { "</think>", "<tool_call>" };+ data.preserved_tokens.insert(data.preserved_tokens.end(), { "<think>", "</think>" });+ }++ data.message_delimiters = {+ { COMMON_CHAT_ROLE_ASSISTANT, "<|im_start|>assistant" },+ { COMMON_CHAT_ROLE_TOOL, "<|im_start|>user\n<tool_response>" }, // Qwen3-Coder, Qwen3.5, Nemotron Nano 3+ { COMMON_CHAT_ROLE_TOOL, "<|im_start|>tool_response" }, // StepFun-3.5-Flash+ { COMMON_CHAT_ROLE_USER, "<|im_start|>user" },+ { COMMON_CHAT_ROLE_SYSTEM, "<|im_start|>system" },+ };++ auto has_tools = inputs.tools.is_array() && !inputs.tools.empty();+ auto has_response_format = inputs.json_schema.is_object() && !inputs.json_schema.empty();+ auto extract_reasoning = inputs.reasoning_format != COMMON_REASONING_FORMAT_NONE;+ auto include_grammar = has_response_format || (has_tools && inputs.tool_choice != COMMON_CHAT_TOOL_CHOICE_NONE);++ if (inputs.has_continuation()) {+ const auto & msg = inputs.continue_msg;++ data.generation_prompt = GEN_PREFIX;+ if (supports_reasoning) {+ data.generation_prompt += "<think>\n" + msg.reasoning_content;+ if (inputs.continue_final_message == COMMON_CHAT_CONTINUATION_CONTENT) {+ data.generation_prompt += "\n</think>\n\n";+ }+ }+ if (inputs.continue_final_message == COMMON_CHAT_CONTINUATION_CONTENT) {+ data.generation_prompt += msg.render_content();+ }++ data.prompt += data.generation_prompt;+ }++ auto parser = build_chat_peg_parser([&](common_chat_peg_builder & p) {+ auto generation_prompt = p.literal(GEN_PREFIX);++ auto reasoning = p.eps();+ if (supports_reasoning && extract_reasoning) {+ reasoning = p.optional("<think>" + p.space() ++ p.reasoning(p.until_one_of({ "</think>", "<tool_call>" })) ++ (p.literal("</think>") | p.peek(p.literal("<tool_call>"))));+ }++ // Response format parser+ if (has_response_format) {+ return generation_prompt + (reasoning << p.content(p.schema(p.json(), "response-format", inputs.json_schema)));+ }++ // Tool call parser+ if (has_tools && inputs.tool_choice != COMMON_CHAT_TOOL_CHOICE_NONE) {+ auto arg_close = p.tool_arg_close(p.literal("\n</parameter>\n"));+ auto arg_string = p.rule("xml-arg-string",+ p.ac(p.tool_arg_string_value(p.until("\n</parameter>\n")) + arg_close, "\n</parameter>\n"));++ auto tool_choice = p.choice();+ foreach_function(inputs.tools, [&](const json & tool) {+ const auto & function = tool.at("function");+ std::string name = function.at("name");+ auto parameters = function.contains("parameters") ? function.at("parameters") : json::object();++ auto schema_info = common_schema_info();+ schema_info.resolve_refs(parameters);++ std::vector<common_peg_parser> required_args;+ std::vector<common_peg_parser> optional_args;++ foreach_parameter(function, [&](const std::string & param_name, const json & param_schema, bool is_required) {+ auto rule_name = "tool-" + name + "-arg-" + param_name;++ auto arg_open = p.tool_arg_open("<parameter=" + p.tool_arg_name(p.literal(param_name)) + ">\n");++ auto arg_value = schema_info.resolves_to_string(param_schema) ?+ arg_string :+ p.tool_arg_json_value(p.schema(p.json(), rule_name + "-schema", param_schema)) + arg_close;++ auto arg_rule = p.rule(rule_name, p.tool_arg(arg_open + arg_value));++ (is_required ? required_args : optional_args).push_back(arg_rule);+ });++ // Accept required arguments in any order, as Qwen does not always adhere to the+ // order provided.+ auto args = p.permute("tool-" + name + "-args", required_args);+ if (!optional_args.empty()) {+ args = args + p.zero_or_more(p.choice(optional_args));+ }++ auto func = p.tool(p.tool_open("<function=" + p.tool_name(p.literal(name)) + ">\n") ++ p.tool_args(args) ++ p.tool_close(p.literal("</function>\n")));++ tool_choice |= p.rule("tool-" + name, func);+ });++ auto min_calls = inputs.tool_choice == COMMON_CHAT_TOOL_CHOICE_REQUIRED ? 1 : 0;++ // Qwen3-Coder models may occasionally omit the <tool_call> token.+ auto tool_call_body = tool_choice + "</tool_call>" + p.space();+ auto tool_call_first = p.rule("tool-call-first", p.optional(p.literal("<tool_call>\n")) + tool_call_body);+ auto tool_call = p.rule("tool-call", "<tool_call>\n" + tool_call_body);++ auto calls = inputs.parallel_tool_calls ? tool_call_first + p.zero_or_more(tool_call) : tool_call_first;+ auto tool_calls = p.trigger_rule("tool-call-root", p.repeat(calls, min_calls, 1));++ return generation_prompt ++ (reasoning << p.content(p.until_one_of({ "<tool_call>", "<function=" })) << tool_calls);+ }++ // Content only parser+ return generation_prompt + (reasoning << p.content(p.rest()));+ });++ data.parser = parser.save();++ if (include_grammar) {+ data.grammar_lazy = has_tools && inputs.tool_choice == COMMON_CHAT_TOOL_CHOICE_AUTO;++ data.grammar = build_grammar([&](const common_grammar_builder & builder) {+ foreach_function(inputs.tools, [&](const json & tool) {+ const auto & function = tool.at("function");+ auto schema = function.contains("parameters") ? function.at("parameters") : json::object();+ builder.resolve_refs(schema);+ });+ if (has_response_format) {+ auto schema = inputs.json_schema;+ builder.resolve_refs(schema);+ }+ parser.build_grammar(builder, data.grammar_lazy);+ });++ if (data.grammar_lazy) {+ data.grammar_triggers = {+ { COMMON_GRAMMAR_TRIGGER_TYPE_WORD, "<tool_call>" },+ // Trigger on "<function" and not "<function=" because the trailing "=" is part of+ // the token with the function name e.g. "=read"+ { COMMON_GRAMMAR_TRIGGER_TYPE_WORD, "<function" },+ };+ }+ }++ return data;+}+static common_chat_params common_chat_params_init_gpt_oss(const common_chat_template & tmpl,const autoparser::generation_params & inputs) {common_chat_params data;@@ -3012,6 +3178,14 @@ std::optional<common_chat_params> common_chat_try_specialized_template(return common_chat_params_init_minicpm5(tmpl, params);}+ // Qwen3-Coder XML tool calls, also used by Nemotron Nano 3, Qwen3.5 and StepFun-3.5-Flash+ if (src.find("<tool_call>") != std::string::npos &&+ src.find("<function=") != std::string::npos &&+ src.find("<parameter=") != std::string::npos) {+ LOG_DBG("Using specialized template: Qwen3-Coder\n");+ return common_chat_params_init_qwen3_coder(tmpl, params);+ }+return std::nullopt;}diff --git a/tests/test-chat-peg-parser.cpp b/tests/test-chat-peg-parser.cppindex 908b13fd0c..3ab7a67b6a 100644--- a/tests/test-chat-peg-parser.cpp+++ b/tests/test-chat-peg-parser.cpp@@ -8,6 +8,7 @@#include <iostream>#include <numeric>+#include <regex>#include <string>#include "nlohmann/json.hpp"@@ -21,6 +22,7 @@ static void test_example_qwen3_non_coder(testing & t);static void test_command7_parser_compare(testing & t);static void test_prefix_tool_names(testing & t);static void test_tagged_peg_parser(testing & t);+static void test_permute(testing & t);int main(int argc, char * argv[]) {testing t(std::cout);@@ -39,6 +41,7 @@ int main(int argc, char * argv[]) {t.test("comparison", test_command7_parser_compare);t.test("prefix tool names", test_prefix_tool_names);t.test("tagged peg parser", test_tagged_peg_parser);+ t.test("permute", test_permute);return t.summary();}@@ -981,3 +984,103 @@ static void test_tagged_peg_parser(testing & t) {t.assert_equal("fun_post should be '>'", ">", result.tags["fun_post"]);});}++static void test_permute(testing & t) {+ auto accepts = [](const common_peg_arena & parser, const std::string & input) {+ common_peg_parse_context ctx(input);+ return parser.parse(ctx).success();+ };++ auto gbnf_of = [](const common_peg_arena & parser) {+ return build_grammar([&](const common_grammar_builder & builder) { parser.build_grammar(builder); });+ };++ auto assert_gbnf_equal = [](testing & t, const std::string & expected, const std::string & actual) {+ static const std::regex leading_ws_re = std::regex(R"((^|\n)\s+)");+ t.assert_equal("gbnf are equal", std::regex_replace(expected, leading_ws_re, "$1"), actual);+ };++ auto count_rules = [](const std::string & gbnf, const std::string & prefix) {+ size_t count = 0;+ for (const auto & line : string_split<std::string>(gbnf, '\n')) {+ if (line.rfind(prefix, 0) == 0) {+ count++;+ }+ }+ return count;+ };++ t.test("accepts every ordering", [&](testing & t) {+ auto parser = build_chat_peg_parser([](common_chat_peg_builder & p) {+ return p.permute("abc", { p.literal("a"), p.literal("b"), p.literal("c") }) + p.end();+ });++ for (const std::string input : { "abc", "acb", "bac", "bca", "cab", "cba" }) {+ t.assert_true("accepts " + input, accepts(parser, input));+ }+ });++ t.test("single element", [&](testing & t) {+ auto parser = build_chat_peg_parser([](common_chat_peg_builder & p) {+ return p.permute("a", { p.literal("a") }) + p.end();+ });++ t.assert_true("accepts a", accepts(parser, "a"));+ t.assert_true("rejects aa", !accepts(parser, "aa"));+ });++ t.test("grammar left-factorizes shared tails", [&](testing & t) {+ auto parser = build_chat_peg_parser([](common_chat_peg_builder & p) {+ return p.permute("abc", { p.literal("a"), p.literal("b"), p.literal("c") }) + p.end();+ });++ // Every rule is one remaining subset, keyed by bitmask: abc-3 is {a,b}, abc-7 is {a,b,c}.+ // Each subset is emitted once and shared by every branch that leads into it.+ assert_gbnf_equal(t, R"""(+ abc-1 ::= "a"+ abc-2 ::= "b"+ abc-3 ::= "a" abc-2 | "b" abc-1+ abc-4 ::= "c"+ abc-5 ::= "a" abc-4 | "c" abc-1+ abc-6 ::= "b" abc-4 | "c" abc-2+ abc-7 ::= "a" abc-6 | "b" abc-5 | "c" abc-3+ root ::= abc-7+ space ::= | " " | "\n"{1,2} [ \t]{0,20}+ )""", gbnf_of(parser));+ });++ t.test("grammar emits one rule per remaining subset", [&](testing & t) {+ auto parser = build_chat_peg_parser([](common_chat_peg_builder & p) {+ return p.permute("abcd", { p.literal("a"), p.literal("b"), p.literal("c"), p.literal("d") }) + p.end();+ });++ // 2^4 - 1 non-empty subsets, one rule each - not the 4! = 24 orderings.+ t.assert_equal("permute rule count", 15u, count_rules(gbnf_of(parser), "abcd-"));+ });++ t.test("grammar emits no rules for a single element", [&](testing & t) {+ auto parser = build_chat_peg_parser([](common_chat_peg_builder & p) {+ return p.permute("a", { p.literal("a") }) + p.end();+ });++ assert_gbnf_equal(t, R"""(+ root ::= "a"+ space ::= | " " | "\n"{1,2} [ \t]{0,20}+ )""", gbnf_of(parser));+ });++ t.test("grammar falls back to the given order when too large", [&](testing & t) {+ auto parser = build_chat_peg_parser([](common_chat_peg_builder & p) {+ std::vector<common_peg_parser> parsers;+ for (size_t i = 0; i <= COMMON_CHAT_MAX_PERMUTE; i++) {+ parsers.push_back(p.literal(std::string(1, (char) ('a' + i))));+ }+ return p.permute("big", parsers) + p.end();+ });++ assert_gbnf_equal(t, R"""(+ root ::= "a" "b" "c" "d" "e" "f" "g"+ space ::= | " " | "\n"{1,2} [ \t]{0,20}+ )""", gbnf_of(parser));+ });+}diff --git a/tests/test-chat.cpp b/tests/test-chat.cppindex ef02fdde57..8d571d369e 100644--- a/tests/test-chat.cpp+++ b/tests/test-chat.cpp@@ -2278,46 +2278,39 @@ static void test_template_output_peg_parsers(bool detailed_debug) {.expect_content(R"({"amount": 123.45, "date": "2025-12-03"})").run();- // tool call segment in reasoning+ // a tool call ends the prefilled thinking block, with or without a closing </think>tst.test(- "Let's call a tool: <tool_call>\n"- "<function=python>\n"- "<parameter=code>\n"- "def hello():\n"- " print(\"Not the real call!\")\n"- "\n"- "hello()\n"+ "<tool_call>\n"+ "<function=run_in_terminal>\n"+ "<parameter=command>\n"+ "pwd\n""</parameter>\n""</function>\n"- "</tool_call>\n</think>\n\n"+ "</tool_call>")+ .enable_thinking(true)+ .reasoning_format(COMMON_REASONING_FORMAT_AUTO)+ .tools({ run_in_terminal_tool })+ .expect_tool_calls({+ { "run_in_terminal", R"({"command": "pwd"})", {} },+ })+ .run();++ // ...including after the model has thought about it+ tst.test(+ "Need to inspect the current directory.\n""<tool_call>\n"- "<function=python>\n"- "<parameter=code>\n"- "def hello():\n"- " print(\"Hello, world!\")\n"- "\n"- "hello()\n"+ "<function=run_in_terminal>\n"+ "<parameter=command>\n"+ "pwd\n""</parameter>\n""</function>\n""</tool_call>").enable_thinking(true).reasoning_format(COMMON_REASONING_FORMAT_AUTO)- .tools({- python_tool- })- .expect_reasoning(- "Let's call a tool: <tool_call>\n"- "<function=python>\n"- "<parameter=code>\n"- "def hello():\n"- " print(\"Not the real call!\")\n"- "\n"- "hello()\n"- "</parameter>\n"- "</function>\n"- "</tool_call>")+ .tools({ run_in_terminal_tool })+ .expect_reasoning("Need to inspect the current directory.").expect_tool_calls({- { "python", "{\"code\": \"def hello():\\n print(\\\"Hello, world!\\\")\\n\\nhello()\"}", {} },+ { "run_in_terminal", R"({"command": "pwd"})", {} },}).run();@@ -2461,17 +2454,6 @@ static void test_template_output_peg_parsers(bool detailed_debug) {}).run();- tst.test(- "I might call <tool_call> later, but I am still thinking.\n"- "</think>\n\n"- "Final answer without tools.")- .reasoning_format(COMMON_REASONING_FORMAT_AUTO)- .enable_thinking(true)- .tools({ run_in_terminal_tool })- .expect_reasoning("I might call <tool_call> later, but I am still thinking.")- .expect_content("Final answer without tools.")- .run();-// Continuation teststst.test("world!\nWhat's up?").reasoning_format(COMMON_REASONING_FORMAT_AUTO)@@ -2776,49 +2758,6 @@ static void test_template_output_peg_parsers(bool detailed_debug) {.expect_content(R"({"amount": 123.45, "date": "2025-12-03"})").run();- // tool call segment in reasoning- tst.test(- "Let's call a tool: <tool_call>\n"- "<function=python>\n"- "<parameter=code>\n"- "def hello():\n"- " print(\"Not the real call!\")\n"- "\n"- "hello()\n"- "</parameter>\n"- "</function>\n"- "</tool_call>\n</think>\n"- "<tool_call>\n"- "<function=python>\n"- "<parameter=code>\n"- "def hello():\n"- " print(\"Hello, world!\")\n"- "\n"- "hello()\n"- "</parameter>\n"- "</function>\n"- "</tool_call>\n"- )- .enable_thinking(true)- .reasoning_format(COMMON_REASONING_FORMAT_AUTO)- .tools({- python_tool- })- .expect_reasoning("Let's call a tool: <tool_call>\n"- "<function=python>\n"- "<parameter=code>\n"- "def hello():\n"- " print(\"Not the real call!\")\n"- "\n"- "hello()\n"- "</parameter>\n"- "</function>\n"- "</tool_call>\n")- .expect_tool_calls({- { "python", "{\"code\": \"def hello():\\n print(\\\"Hello, world!\\\")\\n\\nhello()\"}", {} },- })- .run();-// Continuation teststst.test("world!\nWhat's up?").reasoning_format(COMMON_REASONING_FORMAT_AUTO)@@ -3572,6 +3511,61 @@ static void test_template_output_peg_parsers(bool detailed_debug) {.expect_reconstruction().run();+ // Some models skip the opening <tool_call> and go straight to <function=>+ tst.test(+ "<function=special_function>\n"+ "<parameter=arg1>\n"+ "1\n"+ "</parameter>\n"+ "</function>\n"+ "</tool_call>")+ .tools({ special_function_tool })+ .expect(message_assist_call)+ .run();++ tst.test(+ "Let me call it.\n"+ "<function=special_function>\n"+ "<parameter=arg1>\n"+ "1\n"+ "</parameter>\n"+ "</function>\n"+ "</tool_call>")+ .tools({ special_function_tool })+ .expect_content("Let me call it.\n")+ .expect_tool_calls({+ { "special_function", R"({"arg1": 1})", {} },+ })+ .run();++ // Only the first call may omit it, the rest keep the </tool_call>\n<tool_call> separator+ tst.test(+ "<function=special_function>\n"+ "<parameter=arg1>\n"+ "1\n"+ "</parameter>\n"+ "</function>\n"+ "</tool_call>\n"+ "<tool_call>\n"+ "<function=special_function_with_opt>\n"+ "<parameter=arg1>\n"+ "1\n"+ "</parameter>\n"+ "<parameter=arg2>\n"+ "2\n"+ "</parameter>\n"+ "</function>\n"+ "</tool_call>")+ .parallel_tool_calls(true)+ .tools({+ special_function_tool, special_function_tool_with_optional_param+ })+ .expect_tool_calls({+ { "special_function", R"({"arg1": 1})", {} },+ { "special_function_with_opt", R"({"arg1": 1, "arg2": 2})", {} },+ })+ .run();+tst.test("<tool_call>\n""<function=special_function>\n"@@ -3680,6 +3674,37 @@ static void test_template_output_peg_parsers(bool detailed_debug) {.expect_reconstruction().run();+ // Test flexible required argument ordering (required args still come first, in any order)+ tst.test(+ "<tool_call>\n"+ "<function=edit>\n"+ "<parameter=newString>\n#include\n</parameter>\n"+ "<parameter=filename>\nfoo.c\n</parameter>\n"+ "<parameter=oldString>\n#iclunde\n</parameter>\n"+ "</function>\n"+ "</tool_call>")+ .tools({ edit_tool })+ .expect_tool_calls({+ { "edit", R"({"newString": "#include", "filename": "foo.c", "oldString": "#iclunde"})", {} },+ })+ .expect_reconstruction()+ .run();++ tst.test(+ "<tool_call>\n"+ "<function=tool_2req_4opt>\n"+ "<parameter=req2>\n42\n</parameter>\n"+ "<parameter=req1>\nhello\n</parameter>\n"+ "<parameter=opt2>\n200\n</parameter>\n"+ "</function>\n"+ "</tool_call>")+ .tools({ tool_2req_4opt })+ .expect_tool_calls({+ { "tool_2req_4opt", R"({"req2": 42, "req1": "hello", "opt2": 200})", {} },+ })+ .expect_reconstruction()+ .run();+// Test flexible optional argument ordering (2 required + 4 optional, reversed optional order)tst.test("<tool_call>\n"