LLM inference in C/C++
C++ ggml
chat : implement minicpm5 parser (#24889)
Aldehir Rojasc818263f2a5d
diff --git a/common/chat.cpp b/common/chat.cppindex 0cee80434e..a38c62f5fe 100644--- a/common/chat.cpp+++ b/common/chat.cpp@@ -2376,6 +2376,149 @@ static void func_args_not_string(json & messages) {}+// MiniCPM5 format:+// - Reasoning: <think>{reasoning}</think> (optional)+// - Tool calls: <function name="foo"><param name="bar">value</param></function>+static common_chat_params common_chat_params_init_minicpm5(const common_chat_template & tmpl,+ const autoparser::generation_params & inputs) {+ common_chat_params data;++ data.prompt = common_chat_template_direct_apply_impl(tmpl, inputs);+ data.generation_prompt = common_chat_template_generation_prompt_impl(tmpl, inputs);+ data.format = COMMON_CHAT_FORMAT_PEG_NATIVE;+ data.supports_thinking = true;+ data.preserved_tokens = {+ "<function",+ "<param",+ "</function>",+ "</param>",+ "<think>",+ "</think>",+ };++ data.thinking_start_tag = "<think>";+ data.thinking_end_tag = "</think>";++ data.message_delimiters = {+ { COMMON_CHAT_ROLE_ASSISTANT, "<|im_start|>assistant" },+ { COMMON_CHAT_ROLE_TOOL, "<|im_start|>user\n<tool_response>" },+ { COMMON_CHAT_ROLE_USER, "<|im_start|>user" },+ { COMMON_CHAT_ROLE_SYSTEM, "<|im_start|>system" },+ };++ auto has_tools = inputs.tools.is_array() && !inputs.tools.empty();+ auto has_response_format = inputs.json_schema.is_object() && !inputs.json_schema.empty();+ auto extract_reasoning = inputs.reasoning_format != COMMON_REASONING_FORMAT_NONE;+ auto include_grammar = has_response_format || (has_tools && inputs.tool_choice != COMMON_CHAT_TOOL_CHOICE_NONE);++ if (inputs.has_continuation()) {+ const auto & msg = inputs.continue_msg;++ data.generation_prompt = "<|im_start|>assistant\n<think>\n" + msg.reasoning_content;+ if (inputs.continue_final_message == COMMON_CHAT_CONTINUATION_CONTENT) {+ data.generation_prompt += "\n</think>\n\n" + msg.render_content();+ }++ data.prompt += data.generation_prompt;+ }++ auto parser = build_chat_peg_parser([&](common_chat_peg_builder & p) {+ auto generation_prompt = p.literal("<|im_start|>assistant\n");++ auto reasoning = p.eps();+ if (extract_reasoning) {+ reasoning = ("<think>" << p.reasoning(p.until("</think>")) << "</think>") + p.space();+ }++ // Response format parser+ if (has_response_format) {+ return generation_prompt + reasoning + p.content(p.schema(p.json(), "response-format", inputs.json_schema));+ }++ if (has_tools && inputs.tool_choice != COMMON_CHAT_TOOL_CHOICE_NONE) {+ // CDATA lets a value carry characters that would otherwise close the tag (e.g.+ // </param>); capture the inner text only, excluding the CDATA markers.+ auto string_value = p.choice({+ p.literal("<![CDATA[") + p.ac(p.tool_arg_string_value(p.until("]]>")) + p.literal("]]>"), "]]>") + p.tool_arg_close(p.literal("</param>")),+ p.negate(p.literal("< {+ const auto & function = tool.at("function");+ const std::string name = function.at("name");+ auto params = function.contains("parameters") ? function.at("parameters") : json::object();++ auto args = p.eps();+ if (params.contains("properties") && params.at("properties").is_object() && !params.at("properties").empty()) {+ auto schema_info = common_schema_info();+ schema_info.resolve_refs(params);++ auto arg_choice = p.choice();+ for (const auto & [prop_name, prop_schema] : params.at("properties").items()) {+ auto value_parser = p.eps();+ if (schema_info.resolves_to_string(prop_schema)) {+ value_parser = string_value;+ } else {+ value_parser = p.tool_arg_json_value(+ p.schema(p.json(), "tool-" + name + "-arg-" + prop_name + "-schema", prop_schema, false)+ ) + p.tool_arg_close(p.literal("</param>"));+ }++ auto arg_rule = p.tool_arg(+ p.tool_arg_open(p.literal("<param name=\"") + p.tool_arg_name(p.literal(prop_name)) + p.literal("\">")) ++ value_parser+ );++ arg_choice |= arg_rule;+ }+ args = p.zero_or_more(arg_choice + p.space());+ }++ auto tool_parser = p.tool(+ p.tool_open(p.literal("<function name=\"") + p.tool_name(p.literal(name)) + p.literal("\">"))+ << p.tool_args(args)+ << p.tool_close(p.literal("</function>")));++ tool_choice |= p.rule("tool-" + name, tool_parser);+ });++ auto max_calls = inputs.parallel_tool_calls ? -1 : 1;+ auto tool_calls = p.trigger_rule("tool-call", p.repeat(tool_choice + p.space(), 1, max_calls));++ auto content = p.content(p.until("<function"));++ return generation_prompt + reasoning + content + tool_calls + p.end();+ }++ return generation_prompt + reasoning + p.content(p.rest()) + p.end();+ });++ data.parser = parser.save();++ if (include_grammar) {+ data.grammar_lazy = !(has_response_format || (has_tools && inputs.tool_choice == COMMON_CHAT_TOOL_CHOICE_REQUIRED));+ data.grammar = build_grammar([&](const common_grammar_builder & builder) {+ foreach_function(inputs.tools, [&](const json & tool) {+ const auto & function = tool.at("function");+ auto schema = function.contains("parameters") ? function.at("parameters") : json::object();+ builder.resolve_refs(schema);+ });+ if (has_response_format) {+ auto schema = inputs.json_schema;+ builder.resolve_refs(schema);+ }+ parser.build_grammar(builder, data.grammar_lazy);+ });++ data.grammar_triggers = {+ { COMMON_GRAMMAR_TRIGGER_TYPE_WORD, "<function" },+ };+ }++ return data;+}+static json common_chat_extra_context() {json ctx = json::object();std::chrono::system_clock::time_point now = std::chrono::system_clock::now();@@ -2468,6 +2611,14 @@ std::optional<common_chat_params> common_chat_try_specialized_template(return common_chat_params_init_gemma4(tmpl, params);}+ // MiniCPM5 - XML tool calls with <function name="..."><param name="...">...</param></function>+ if (src.find("Tool usage guidelines:") != std::string::npos &&+ src.find("<function name=\"") != std::string::npos &&+ src.find("<param name=\"") != std::string::npos) {+ LOG_DBG("Using specialized template: MiniCPM5\n");+ return common_chat_params_init_minicpm5(tmpl, params);+ }+return std::nullopt;}diff --git a/common/jinja/value.cpp b/common/jinja/value.cppindex cd6a36956c..5055ae9ac1 100644--- a/common/jinja/value.cpp+++ b/common/jinja/value.cpp@@ -1108,6 +1108,50 @@ const func_builtins & value_array_t::get_builtins() const {std::reverse(arr.begin(), arr.end());return is_val<value_tuple>(val) ? mk_val<value_tuple>(std::move(arr)) : mk_val<value_array>(std::move(arr));}},+ {"min", [](const func_args & args) -> value {+ args.ensure_count(1, 4);+ args.ensure_vals<value_array>();+ value val_case = args.get_kwarg_or_pos("case_sensitive", 1);+ value attribute = args.get_kwarg_or_pos("attribute", 2);+ if (!attribute->is_undefined()) {+ throw not_implemented_exception("min: attribute not implemented");+ }+ // FIXME: min is currently always case sensitive+ (void) val_case;+ const auto & arr = args.get_pos(0)->as_array();+ if (arr.empty()) {+ return mk_val<value_undefined>();+ }+ value result = arr[0];+ for (size_t i = 1; i < arr.size(); ++i) {+ if (value_compare(arr[i], result, value_compare_op::lt)) {+ result = arr[i];+ }+ }+ return result;+ }},+ {"max", [](const func_args & args) -> value {+ args.ensure_count(1, 4);+ args.ensure_vals<value_array>();+ value val_case = args.get_kwarg_or_pos("case_sensitive", 1);+ value attribute = args.get_kwarg_or_pos("attribute", 2);+ if (!attribute->is_undefined()) {+ throw not_implemented_exception("max: attribute not implemented");+ }+ // FIXME: max is currently always case sensitive+ (void) val_case;+ const auto & arr = args.get_pos(0)->as_array();+ if (arr.empty()) {+ return mk_val<value_undefined>();+ }+ value result = arr[0];+ for (size_t i = 1; i < arr.size(); ++i) {+ if (value_compare(arr[i], result, value_compare_op::gt)) {+ result = arr[i];+ }+ }+ return result;+ }},{"unique", array_unique_not_implemented},};return builtins;diff --git a/models/templates/openbmb-MiniCPM5-1B.jinja b/models/templates/openbmb-MiniCPM5-1B.jinjanew file mode 100644index 0000000000..cb2934c459--- /dev/null+++ b/models/templates/openbmb-MiniCPM5-1B.jinja@@ -0,0 +1,179 @@+{{- bos_token }}{%- if tools %}+ {%- set tool_definitions %}+ {{- "# Tools\n\nYou are provided with function signatures within <tools></tools> XML tags:\n<tools>" }}+ {%- for tool in tools %}+ {{- "\n" }}+ {{- tool | tojson(ensure_ascii=False) }}+ {%- endfor %}+ {{- '\n</tools>\n\nTool usage guidelines:\n- You may call zero or more functions. If no function calls are needed, just answer normally and do not include any <function ... </function>.\n- When calling a function, return an XML object within <function ... </function> using:\n<function name="function-name"><param name="param-name">param-value</param></function>\n- param-value may be multi-line. If it contains <, & or newline characters, wrap it in a CDATA block: <param name="param-name"><![CDATA[...multi-line value...]]></param>' }}+ {%- endset %}++ {{- '<|im_start|>system\n' }}+ {%- if messages[0].role == 'system' %}+ {%- if '<tool_def_sep>' in messages[0].content %}+ {{- messages[0].content.replace('<tool_def_sep>', tool_definitions) }}+ {%- else %}+ {{- messages[0].content + '\n\n' + tool_definitions }}+ {%- endif %}+ {%- else %}+ {{- tool_definitions.lstrip() }}+ {%- endif %}+ {{- '<|im_end|>\n' }}+{%- else %}+ {%- if messages[0].role == 'system' %}+ {{- '<|im_start|>system\n' + messages[0].content + '<|im_end|>\n' }}+ {%- endif %}+{%- endif %}+{%- set ns = namespace(multi_step_tool=true, last_query_index=messages|length - 1) %}+{%- for message in messages[::-1] %}+ {%- set index = (messages|length - 1) - loop.index0 %}+ {%- if ns.multi_step_tool and message.role == "user" and message.content is string and not(message.content.startswith('<tool_response>') and message.content.endswith('</tool_response>')) %}+ {%- set ns.multi_step_tool = false %}+ {%- set ns.last_query_index = index %}+ {%- endif %}+{%- endfor %}+{%- for message in messages %}+ {%- if message.content is string %}+ {%- set content = message.content %}+ {%- else %}+ {%- set content = '' %}+ {%- endif %}+ {%- if (message.role == "user") or (message.role == "system" and not loop.first) %}+ {{- '<|im_start|>' + message.role + '\n' + content + '<|im_end|>' + '\n' }}+ {%- elif message.role == "assistant" %}+ {%- set reasoning_content = '' %}+ {%- if message.reasoning_content is string %}+ {%- set reasoning_content = message.reasoning_content %}+ {%- else %}+ {%- if '</think>' in content %}+ {%- set reasoning_content = content.split('</think>')[0].rstrip('\n').split('<think>')[-1].lstrip('\n') %}+ {%- set content = content.split('</think>')[-1].lstrip('\n') %}+ {%- endif %}+ {%- endif %}++ {%- if message.tool_calls %}+ {%- set content_parts = content.split('<tool_sep>') %}+ {%- set processed_content = content_parts[0] %}+ {%- set tool_calls_count = message.tool_calls|length %}+ {%- set tool_sep_count = content_parts|length - 1 %}+ {%- set min_count = [tool_calls_count, tool_sep_count]|min %}++ {%- for i in range(1, content_parts|length) %}+ {%- set tool_index = i - 1 %}+ {%- if tool_index < tool_calls_count %}+ {%- set tool_call = message.tool_calls[tool_index] %}+ {%- if tool_call.function %}+ {%- set tool_call = tool_call.function %}+ {%- endif %}+ {%- set single_tool_xml %}+ {{- '<function name="' ~ tool_call.name ~ '">' }}+ {%- if tool_call.arguments %}+ {%- set args_dict = tool_call.arguments %}+ {%- for param_name, param_value in args_dict.items() %}+ {{- '<param name="' ~ param_name ~ '">' }}+ {%- if param_value is string and ('<' in param_value or '&' in param_value or '\n' in param_value) %}+ {{- '<![CDATA[' + param_value + ']]>' }}+ {%- else %}+ {{- param_value }}+ {%- endif %}+ {{- '</param>' }}+ {%- endfor %}+ {%- endif %}+ {{- '</function>' }}+ {%- endset %}+ {%- set processed_content = processed_content + single_tool_xml + content_parts[i] %}+ {%- else %}+ {%- set processed_content = processed_content + content_parts[i] %}+ {%- endif %}+ {%- endfor %}++ {%- if tool_calls_count > tool_sep_count %}+ {%- for remaining_index in range(tool_sep_count, tool_calls_count) %}+ {%- set tool_call = message.tool_calls[remaining_index] %}+ {%- if tool_call.function %}+ {%- set tool_call = tool_call.function %}+ {%- endif %}+ {%- set remaining_tool_xml %}+ {{- '<function name="' ~ tool_call.name ~ '">' }}+ {%- if tool_call.arguments %}+ {%- set args_dict = tool_call.arguments %}+ {%- for param_name, param_value in args_dict.items() %}+ {{- '<param name="' ~ param_name ~ '">' }}+ {%- if param_value is string and ('<' in param_value or '&' in param_value or '\n' in param_value) %}+ {{- '<![CDATA[' + param_value + ']]>' }}+ {%- else %}+ {{- param_value }}+ {%- endif %}+ {{- '</param>' }}+ {%- endfor %}+ {%- endif %}+ {{- '</function>' }}+ {%- endset %}+ {%- set processed_content = processed_content + remaining_tool_xml %}+ {%- endfor %}+ {%- endif %}++ {%- set content = processed_content %}+ {%- endif %}++ {%- if loop.index0 > ns.last_query_index %}+ {%- if reasoning_content %}+ {{- '<|im_start|>' + message.role + '\n<think>\n' + reasoning_content.strip('\n') + '\n</think>\n\n' + content.lstrip('\n') }}+ {%- else %}+ {{- '<|im_start|>' + message.role + '\n' + content }}+ {%- endif %}+ {%- else %}+ {{- '<|im_start|>' + message.role + '\n' + content }}+ {%- endif %}++ {%- if message.tool_calls and not has_tool_sep %}+ {%- for tool_call in message.tool_calls %}+ {%- if (loop.first and content) or (not loop.first) %}+ {{- '\n' }}+ {%- endif %}+ {%- if tool_call.function %}+ {%- set tool_call = tool_call.function %}+ {%- endif %}+ {{- '<function name="' ~ tool_call.name ~ '">' }}+ {%- if tool_call.arguments %}+ {%- set args_dict = tool_call.arguments %}+ {%- for param_name, param_value in args_dict.items() %}+ {{- '<param name="' ~ param_name ~ '">' }}+ {%- if param_value is string and ('<' in param_value or '&' in param_value or '\n' in param_value) %}+ {{- '<![CDATA[' + param_value + ']]>' }}+ {%- else %}+ {{- param_value }}+ {%- endif %}+ {{- '</param>' }}+ {%- endfor %}+ {%- endif %}+ {{- '</function>' }}+ {%- endfor %}+ {%- endif %}+ {{- '<|im_end|>\n' }}+ {%- elif message.role == "tool" %}+ {%- if loop.first or (messages[loop.index0 - 1].role != "tool") %}+ {{- '<|im_start|>user' }}+ {%- endif %}+ {{- '\n<tool_response>\n' }}+ {%- if message.content is string %}+ {{- content }}+ {%- else %}+ {{- message.content | tojson(ensure_ascii=False) }}+ {%- endif %}+ {{- '\n</tool_response>' }}+ {%- if loop.last or (messages[loop.index0 + 1].role != "tool") %}+ {{- '<|im_end|>\n' }}+ {%- endif %}+ {%- endif %}+{%- endfor %}+{%- if add_generation_prompt %}+ {{- '<|im_start|>assistant\n' }}+ {%- if enable_thinking is defined %}+ {%- if enable_thinking is false %}+ {{- '<think>\n\n</think>\n\n' }}+ {%- elif enable_thinking is true %}+ {{- '<think>\n' }}+ {%- endif %}+ {%- endif %}+{%- endif %}diff --git a/tests/test-chat.cpp b/tests/test-chat.cppindex c38aed8cfe..5f71e5da6e 100644--- a/tests/test-chat.cpp+++ b/tests/test-chat.cpp@@ -5593,6 +5593,77 @@ static void test_template_output_peg_parsers(bool detailed_debug) {.expect_content("Hello, world!\nWhat's up?").run();}++ // MiniCPM5 - XML tool calls with <function name="..."><param name="...">...</param></function>+ {+ auto tst = peg_tester("models/templates/openbmb-MiniCPM5-1B.jinja", detailed_debug);++ tst.test("Hello, world!\nWhat's up?")+ .enable_thinking(false)+ .reasoning_format(COMMON_REASONING_FORMAT_AUTO)+ .expect(message_assist)+ .run();++ tst.test(R"(<function name="python"><param name="code">print('Hello, World!')</param></function>)")+ .enable_thinking(false)+ .reasoning_format(COMMON_REASONING_FORMAT_AUTO)+ .tools({ python_tool })+ .expect_tool_calls({ { "python", R"#({"code": "print('Hello, World!')"})#", {} } })+ .run();++ tst.test(R"(<function name="empty_args"></function>)")+ .enable_thinking(false)+ .reasoning_format(COMMON_REASONING_FORMAT_AUTO)+ .tools({ empty_args_tool })+ .expect(simple_assist_msg("", "", "empty_args", "{}"))+ .run();++ tst.test(R"(<function name="python"><param name="code">print('x')</param></function>)")+ .enable_thinking(false)+ .reasoning_format(COMMON_REASONING_FORMAT_AUTO)+ .parallel_tool_calls(true)+ .tools({ python_tool })+ .expect_tool_calls({ { "python", R"#({"code": "print('x')"})#", {} } })+ .run();++ // CDATA lets a string value carry characters that would otherwise close the tag.+ tst.test(R"(<function name="html"><param name="markup"><![CDATA[<a href="/x">hi</a> </param>]]></param></function>)")+ .enable_thinking(false)+ .reasoning_format(COMMON_REASONING_FORMAT_AUTO)+ .tools({ html_tool })+ .expect_tool_calls({ { "html", R"#({"markup": "<a href=\"/x\">hi</a> </param>"})#", {} } })+ .run();++ tst.test(R"(I'm thinking</think><function name="python"><param name="code">print('hey')</param></function>)")+ .enable_thinking(true)+ .reasoning_format(COMMON_REASONING_FORMAT_AUTO)+ .tools({ python_tool })+ .expect_reasoning("I'm thinking")+ .expect_tool_calls({ { "python", R"#({"code": "print('hey')"})#", {} } })+ .run();++ tst.test(R"(<function name="python"><param name="code">print('x')</param></function>+<function name="python"><param name="code">print('y')</param></function>)")+ .enable_thinking(false)+ .reasoning_format(COMMON_REASONING_FORMAT_AUTO)+ .parallel_tool_calls(true)+ .tools({ python_tool })+ .expect_tool_calls({+ { "python", R"#({"code": "print('x')"})#", {} },+ { "python", R"#({"code": "print('y')"})#", {} },+ })+ .run();++ tst.test(" thinking</think>Hello, world!\nWhat's up?")+ .enable_thinking(true)+ .reasoning_format(COMMON_REASONING_FORMAT_AUTO)+ .messages({ message_user, message_assist_prefill_reasoning })+ .add_generation_prompt(false)+ .continue_final_message(COMMON_CHAT_CONTINUATION_REASONING)+ .expect_reasoning("I'm thinking")+ .expect_content("Hello, world!\nWhat's up?")+ .run();+ }}static void test_template_generation_prompt() {@@ -5740,6 +5811,13 @@ static void test_template_generation_prompt() {check(tmpls, continuation_content(), "<|Assistant|><think>I'm thinking</think>Hello, ");check(tmpls, continuation_reasoning(), "<|Assistant|><think>I'm");}++ {+ auto tmpls = read_templates("models/templates/openbmb-MiniCPM5-1B.jinja");+ check(tmpls, basic(), "<|im_start|>assistant\n<think>\n");+ check(tmpls, continuation_content(), "<|im_start|>assistant\n<think>\nI'm thinking\n</think>\n\nHello, ");+ check(tmpls, continuation_reasoning(), "<|im_start|>assistant\n<think>\nI'm");+ }}// Test the developer role to system workaround with a simple mock templatediff --git a/tests/test-jinja.cpp b/tests/test-jinja.cppindex 81bbcd55a4..d8d1892a91 100644--- a/tests/test-jinja.cpp+++ b/tests/test-jinja.cpp@@ -1584,6 +1584,36 @@ static void test_array_methods(testing & t) {"6");+ test_template(t, "array|min",+ "{{ [tool_calls_count, tool_sep_count]|min }}",+ {{"tool_calls_count", 2}, {"tool_sep_count", 1}},+ "1"+ );++ test_template(t, "array|max",+ "{{ [tool_calls_count, tool_sep_count]|max }}",+ {{"tool_calls_count", 2}, {"tool_sep_count", 1}},+ "2"+ );++ test_template(t, "array|min attribute",+ "{{ items|min(attribute='x') }}",+ {{"items", json::array({+ json({{"x", 2}}),+ json({{"x", 1}}),+ })}},+ "{'x': 1}"+ );++ test_template(t, "array|max attribute",+ "{{ items|max(attribute='x') }}",+ {{"items", json::array({+ json({{"x", 2}}),+ json({{"x", 1}}),+ })}},+ "{'x': 2}"+ );+// not used by any chat templates// test_template(t, "array.insert()",// "{% set _ = arr.insert(1, 'x') %}{{ arr|join(',') }}",