LLM inference in C/C++
C++ ggml
chat : split specialized parsers into common/parsers (#27764)
Piotr Wilkin (ilintar)895c045fd104
diff --git a/common/CMakeLists.txt b/common/CMakeLists.txtindex 36f1e0cd50..1506bf6479 100644--- a/common/CMakeLists.txt+++ b/common/CMakeLists.txt@@ -53,7 +53,10 @@ endif()set(TARGET llama-common)+include(parsers/sources.cmake)+add_library(${TARGET}+ ${LLAMA_CHAT_PARSERS_SOURCES}arg.cpparg.hbase64.hppdiff --git a/common/chat.cpp b/common/chat.cppindex 743ecde0a7..faf27f7867 100644--- a/common/chat.cpp+++ b/common/chat.cpp@@ -8,6 +8,7 @@#include "json-schema-to-grammar.h"#include "json.h"#include "log.h"+#include "parsers/parsers.h"#include "jinja/value.h"#include "jinja/runtime.h"@@ -717,13 +718,6 @@ bool common_chat_templates_was_explicit(const struct common_chat_templates * tmpreturn tmpls->has_explicit_template;}-// LFM2 format detection: template uses <|tool_list_start|>[...]<|tool_list_end|> around the tool list-// and <|tool_call_start|>[...]<|tool_call_end|> around each tool call-static bool is_lfm2_template(const std::string & src) {- return src.find("<|tool_list_start|>") != std::string::npos &&- src.find("<|tool_list_end|>") != std::string::npos;-}-common_chat_prompt_preset common_chat_get_asr_prompt(const common_chat_templates * chat_templates) {common_chat_prompt_preset asr_preset;asr_preset.system = "";@@ -898,42 +892,12 @@ common_reasoning_format common_reasoning_format_from_name(const std::string & fothrow std::runtime_error("Unknown reasoning format: " + format);}-static void foreach_function(const json & tools, const std::function<void(const json &)> & fn) {- for (const auto & tool : tools) {- if (!tool.contains("type") || tool.at("type") != "function" || !tool.contains("function")) {- LOG_INF("Skipping tool without function: %s", tool.dump(2).c_str());- continue;- }- fn(tool);- }-}--static void foreach_parameter(const json & function,- const std::function<void(const std::string &, const json &, bool)> & fn) {- if (!function.contains("parameters") || !function.at("parameters").is_object()) {- return;- }- const auto & params = function.at("parameters");- if (!params.contains("properties") || !params.at("properties").is_object()) {- return;- }- const auto & props = params.at("properties");- std::set<std::string> required;- if (params.contains("required") && params.at("required").is_array()) {- required = params.at("required").get<std::set<std::string>>();- }- for (const auto & [name, prop] : props.items()) {- bool is_required = (required.find(name) != required.end());- fn(name, prop, is_required);- }-}--static std::string common_chat_template_direct_apply_impl(+std::string common_chat_template_direct_apply_impl(const common_chat_template & tmpl,const autoparser::generation_params & inputs,- const std::optional<json> & messages_override = std::nullopt,- const std::optional<json> & tools_override = std::nullopt,- const std::optional<json> & additional_context = std::nullopt) {+ const std::optional<json> & messages_override,+ const std::optional<json> & tools_override,+ const std::optional<json> & additional_context) {jinja::context ctx(tmpl.source());// messages_override is already built for this template, do not touch its content parts@@ -997,12 +961,12 @@ std::string common_chat_template_direct_apply(return common_chat_template_direct_apply_impl(tmpl, inputs, std::nullopt, std::nullopt, std::nullopt);}-static std::string common_chat_template_generation_prompt_impl(+std::string common_chat_template_generation_prompt_impl(const common_chat_template & tmpl,const autoparser::generation_params & inputs,- const std::optional<json> & messages_override = std::nullopt,- const std::optional<json> & tools_override = std::nullopt,- const std::optional<json> & additional_context = std::nullopt) {+ const std::optional<json> & messages_override,+ const std::optional<json> & tools_override,+ const std::optional<json> & additional_context) {autoparser::generation_params params = inputs;params.add_generation_prompt = false;@@ -1025,2448 +989,82 @@ std::string common_chat_template_generation_prompt(return common_chat_template_generation_prompt_impl(tmpl, inputs, std::nullopt, std::nullopt, std::nullopt);}-static common_chat_params common_chat_params_init_ministral_3(const common_chat_template & tmpl,- const autoparser::generation_params & inputs) {- common_chat_params data;-- // Build up messages to follow the format: https://huggingface.co/mistralai/Ministral-3-14B-Reasoning-2512/blob/main/chat_template.jinja- auto adjusted_messages = json::array();- for (const auto & msg : inputs.messages) {- auto role = msg.value("role", "");- if (role != "system" && role != "assistant") {- // Only adjust system and assistant messages. Interestingly, the system message may contain thinking.- adjusted_messages.push_back(msg);- continue;- }-- auto content = json::array();-- // If message contains `reasoning_content`, add it as a block of type `thinking`- if (msg.contains("reasoning_content") && msg.at("reasoning_content").is_string()) {- content.push_back({- { "type", "thinking" },- { "thinking", msg.at("reasoning_content").get<std::string>() },- });- }+namespace workaround {- // If message contains `content`, add it as a block of type `text`- if (msg.contains("content")) {- if (msg.at("content").is_string()) {- content.push_back({- { "type", "text" },- { "text", msg.at("content").get<std::string>() },- });- } else if (msg.at("content").is_array()) {- auto blocks = msg.at("content");- content.insert(blocks);+static void map_developer_role_to_system(json & messages) {+ for (auto & message : messages) {+ if (message.contains("role")) {+ if (message["role"] == "developer") {+ message["role"] = "system";}}-- auto adjusted = msg;- adjusted["content"] = content;- adjusted.erase("reasoning_content");- adjusted_messages.push_back(adjusted);- }-- auto has_tools = inputs.tools.is_array() && !inputs.tools.empty();- auto has_response_format = inputs.json_schema.is_object() && !inputs.json_schema.empty();- auto extract_reasoning = inputs.reasoning_format != COMMON_REASONING_FORMAT_NONE;- auto include_grammar = true;-- data.supports_thinking = true;- data.thinking_start_tag = "[THINK]";- data.thinking_end_tags = {"[/THINK]"};- data.prompt = common_chat_template_direct_apply_impl(tmpl, inputs, /* messages_override = */ adjusted_messages);- data.generation_prompt = common_chat_template_generation_prompt_impl(tmpl, inputs, /* messages_override = */ adjusted_messages);- data.format = COMMON_CHAT_FORMAT_PEG_NATIVE;- data.preserved_tokens = {- "[THINK]",- "[/THINK]",- "[TOOL_CALLS]",- "[ARGS]",- };-- if (inputs.has_continuation()) {- const auto & msg = inputs.continue_msg;-- data.generation_prompt = "[THINK]" + msg.reasoning_content;- if (inputs.continue_final_message == COMMON_CHAT_CONTINUATION_CONTENT) {- data.generation_prompt += "[/THINK]" + msg.render_content();- }-- data.prompt += data.generation_prompt;}+}- auto parser = build_chat_peg_parser([&](common_chat_peg_builder & p) {- auto generation_prompt = p.eps();- auto reasoning =- extract_reasoning ? p.optional("[THINK]" + p.reasoning(p.until("[/THINK]")) + "[/THINK]") : p.eps();-- // Response format parser- if (has_response_format) {- // Ministral wants to emit json surrounded by code fences- return generation_prompt + (reasoning << "```json" << p.content(p.schema(p.json(), "response-format", inputs.json_schema)) << "```");- }-- // Tool call parser- if (has_tools && inputs.tool_choice != COMMON_CHAT_TOOL_CHOICE_NONE) {- auto tool_choice = p.choice();- foreach_function(inputs.tools, [&](const json & tool) {- const auto & function = tool.at("function");- std::string name = function.at("name");- const auto & schema = function.at("parameters");-- tool_choice |=- p.rule("tool-" + name, p.tool_open(p.tool_name(p.literal(name)) + "[ARGS]") +- p.tool_args(p.schema(p.json(), "tool-" + name + "-schema", schema)));- });-- auto min_calls = inputs.tool_choice == COMMON_CHAT_TOOL_CHOICE_REQUIRED ? 1 : 0;- auto max_calls = inputs.parallel_tool_calls ? -1 : 1;- auto tool_calls = p.trigger_rule("tool-call", p.repeat("[TOOL_CALLS]" + tool_choice, min_calls, max_calls));- return generation_prompt + (reasoning << p.content(p.until("[TOOL_CALLS]")) << tool_calls);+// if first message is system and template does not support it, merge it with next message+static void system_message_not_supported(json & messages) {+ if (!messages.empty() && messages.front().at("role") == "system") {+ if (messages.size() > 1) {+ LOG_DBG("Merging system prompt into next message\n");+ auto & first_msg = messages.front();+ auto & second_msg = messages[1];+ second_msg["content"] = first_msg.at("content").get<std::string>()+ + "\n" + second_msg.at("content").get<std::string>();+ messages.erase(0);+ } else {+ LOG_WRN("Removing system prompt due to template not supporting system role\n");+ messages.erase(0);}-- // Content only parser- include_grammar = false;- return generation_prompt + (reasoning << p.content(p.rest()));- });-- data.parser = parser.save();-- if (include_grammar) {- data.grammar_lazy = has_tools && inputs.tool_choice == COMMON_CHAT_TOOL_CHOICE_AUTO;-- data.grammar = build_grammar([&](const common_grammar_builder & builder) {- foreach_function(inputs.tools, [&](const json & tool) {- const auto & function = tool.at("function");- auto schema = function.at("parameters");- builder.resolve_refs(schema);- });- if (has_response_format) {- auto schema = inputs.json_schema;- builder.resolve_refs(schema);- }- parser.build_grammar(builder, data.grammar_lazy);- });-- data.grammar_triggers = {- { COMMON_GRAMMAR_TRIGGER_TYPE_WORD, "[TOOL_CALLS]" }- };}-- return data;}-static common_chat_params common_chat_params_init_qwen3_coder(const common_chat_template & tmpl,- const autoparser::generation_params & inputs) {- common_chat_params data;-- const std::string GEN_PREFIX = "<|im_start|>assistant\n";-- data.prompt = common_chat_template_direct_apply_impl(tmpl, inputs);- data.generation_prompt = common_chat_template_generation_prompt_impl(tmpl, inputs);- data.format = COMMON_CHAT_FORMAT_PEG_NATIVE;-- auto supports_reasoning = tmpl.source().find("<think>") != std::string::npos;-- data.supports_thinking = supports_reasoning;- data.preserved_tokens = {- "<tool_call>",- "</tool_call>",- };-- auto is_qwen3_coder = !supports_reasoning;-- if (supports_reasoning) {- data.thinking_start_tag = "<think>";- // Support both </think> and <tool_call> as reasoning end sequences.- // <function= is omitted, as it is a workaround for Qwen3-Coder which is not a thinking model- data.thinking_end_tags = { "</think>", "<tool_call>" };- data.preserved_tokens.insert(data.preserved_tokens.end(), { "<think>", "</think>" });- }-- data.message_delimiters = {- { COMMON_CHAT_ROLE_ASSISTANT, "<|im_start|>assistant" },- { COMMON_CHAT_ROLE_TOOL, "<|im_start|>user\n<tool_response>" }, // Qwen3-Coder, Qwen3.5, Nemotron Nano 3- { COMMON_CHAT_ROLE_TOOL, "<|im_start|>tool_response" }, // StepFun-3.5-Flash- { COMMON_CHAT_ROLE_USER, "<|im_start|>user" },- { COMMON_CHAT_ROLE_SYSTEM, "<|im_start|>system" },- };-- auto has_tools = inputs.tools.is_array() && !inputs.tools.empty();- auto has_response_format = inputs.json_schema.is_object() && !inputs.json_schema.empty();- auto extract_reasoning = inputs.reasoning_format != COMMON_REASONING_FORMAT_NONE;- auto include_grammar = has_response_format || (has_tools && inputs.tool_choice != COMMON_CHAT_TOOL_CHOICE_NONE);-- if (inputs.has_continuation()) {- const auto & msg = inputs.continue_msg;-- data.generation_prompt = GEN_PREFIX;- if (supports_reasoning) {- data.generation_prompt += "<think>\n" + msg.reasoning_content;- if (inputs.continue_final_message == COMMON_CHAT_CONTINUATION_CONTENT) {- data.generation_prompt += "\n</think>\n\n";- }- }- if (inputs.continue_final_message == COMMON_CHAT_CONTINUATION_CONTENT) {- data.generation_prompt += msg.render_content();+static void requires_non_null_content(json & messages) {+ GGML_ASSERT(messages.is_array());+ for (auto & message : messages) {+ if (message.contains("tool_calls") && !message.contains("content")) {+ message["content"] = "";}-- data.prompt += data.generation_prompt;- }-- std::vector<std::string> tool_call_starts = { "<tool_call>" };-- if (is_qwen3_coder) {- // Match complete <function=name> opener for Qwen3-Coder models that occasionally omit the- // starting <tool_call>. The model may hallucinate a tool name, but it is preferable over- // constraining on <function which may occur in valid content generation, e.g. #include <functional>- foreach_function(inputs.tools, [&](const json & tool) {- const std::string name = tool.at("function").at("name");- tool_call_starts.push_back("<function=" + name + ">");- });}+}- auto parser = build_chat_peg_parser([&](common_chat_peg_builder & p) {- auto generation_prompt = p.literal(GEN_PREFIX);-- auto reasoning = p.eps();- if (supports_reasoning && extract_reasoning) {- reasoning = p.optional("<think>" + p.space() +- p.reasoning(p.until_one_of({ "</think>", "<tool_call>" })) +- (p.literal("</think>") | p.peek(p.literal("<tool_call>"))));- }-- // Response format parser- if (has_response_format) {- return generation_prompt + (reasoning << p.content(p.schema(p.json(), "response-format", inputs.json_schema)));- }-- // Tool call parser- if (has_tools && inputs.tool_choice != COMMON_CHAT_TOOL_CHOICE_NONE) {- auto arg_close = p.tool_arg_close(p.literal("\n</parameter>\n"));- auto arg_string = p.rule("xml-arg-string",- p.ac(p.tool_arg_string_value(p.until("\n</parameter>\n")) + arg_close, "\n</parameter>\n"));-- auto tool_choice = p.choice();- foreach_function(inputs.tools, [&](const json & tool) {- const auto & function = tool.at("function");- std::string name = function.at("name");- auto parameters = function.contains("parameters") ? function.at("parameters") : json::object();-- auto schema_info = common_schema_info();- schema_info.resolve_refs(parameters);-- std::vector<common_peg_parser> required_args;- std::vector<common_peg_parser> optional_args;-- foreach_parameter(function, [&](const std::string & param_name, const json & param_schema, bool is_required) {- auto rule_name = "tool-" + name + "-arg-" + param_name;-- auto arg_open = p.tool_arg_open("<parameter=" + p.tool_arg_name(p.literal(param_name)) + ">\n");-- auto arg_value = schema_info.resolves_to_string(param_schema) ?- arg_string :- p.tool_arg_json_value(p.schema(p.json(), rule_name + "-schema", param_schema)) + arg_close;-- auto arg_rule = p.rule(rule_name, p.tool_arg(arg_open + arg_value));-- (is_required ? required_args : optional_args).push_back(arg_rule);- });-- // Accept required arguments in any order, as Qwen does not always adhere to the- // order provided.- auto args = p.permute("tool-" + name + "-args", required_args);- if (!optional_args.empty()) {- args = args + p.zero_or_more(p.choice(optional_args));+static void func_args_not_string(json & messages) {+ GGML_ASSERT(messages.is_array());+ for (auto & message : messages) {+ if (message.contains("tool_calls")) {+ for (auto & tool_call : message["tool_calls"]) {+ if (tool_call.contains("function") && tool_call["function"].contains("arguments")) {+ auto & args = tool_call["function"]["arguments"];+ if (args.is_string()) {+ try {+ args = json::parse(args.get<std::string>());+ } catch (const std::exception & e) {+ throw std::runtime_error("Failed to parse tool call arguments as JSON: " + std::string(e.what()));+ }+ }}-- auto func = p.tool(p.tool_open("<function=" + p.tool_name(p.literal(name)) + ">\n") +- p.tool_args(args) +- p.tool_close(p.literal("</function>\n")));-- tool_choice |= p.rule("tool-" + name, func);- });-- auto min_calls = inputs.tool_choice == COMMON_CHAT_TOOL_CHOICE_REQUIRED ? 1 : 0;-- auto tool_call_body = tool_choice + "</tool_call>" + p.space();- auto tool_call = p.rule("tool-call", "<tool_call>\n" + tool_call_body);-- // Qwen3-Coder models may occasionally omit the <tool_call> token.- auto tool_call_first = is_qwen3_coder ?- p.rule("tool-call-first", p.optional(p.literal("<tool_call>\n")) + tool_call_body) :- tool_call;-- auto calls = inputs.parallel_tool_calls ? tool_call_first + p.zero_or_more(tool_call) : tool_call_first;- auto tool_calls = p.trigger_rule("tool-call-root", p.repeat(calls, min_calls, 1));-- return generation_prompt +- (reasoning << p.content(p.until_one_of(tool_call_starts)) << tool_calls);- }-- // Content only parser- return generation_prompt + (reasoning << p.content(p.rest()));- });-- data.parser = parser.save();-- if (include_grammar) {- data.grammar_lazy = has_tools && inputs.tool_choice == COMMON_CHAT_TOOL_CHOICE_AUTO;-- data.grammar = build_grammar([&](const common_grammar_builder & builder) {- foreach_function(inputs.tools, [&](const json & tool) {- const auto & function = tool.at("function");- auto schema = function.contains("parameters") ? function.at("parameters") : json::object();- builder.resolve_refs(schema);- });- if (has_response_format) {- auto schema = inputs.json_schema;- builder.resolve_refs(schema);- }- parser.build_grammar(builder, data.grammar_lazy);- });-- if (data.grammar_lazy) {- for (const auto & start : tool_call_starts) {- data.grammar_triggers.push_back({ COMMON_GRAMMAR_TRIGGER_TYPE_WORD, start });}}}-- return data;}-static common_chat_params common_chat_params_init_gpt_oss(const common_chat_template & tmpl,- const autoparser::generation_params & inputs) {- common_chat_params data;-- // Copy reasoning to the "thinking" field as expected by the gpt-oss template- auto adjusted_messages = json::array();- for (auto msg : inputs.messages) {- if (msg.contains("reasoning_content") && msg.at("reasoning_content").is_string()) {- msg["thinking"] = msg.at("reasoning_content");- if (msg.contains("tool_calls") && msg.at("tool_calls").is_array() && !msg.at("tool_calls").empty()) {- msg.erase("content");- }- }- adjusted_messages.push_back(msg);- }-- auto prompt = common_chat_template_direct_apply_impl(tmpl, inputs, /* messages_override= */ adjusted_messages);-- // Check if we need to replace the return token with end token during- // inference and without generation prompt. For more details see:- // https://github.com/ggml-org/llama.cpp/issues/15417- if (inputs.is_inference && !inputs.add_generation_prompt) {- static constexpr std::string_view return_token = "<|return|>";- static constexpr std::string_view end_token = "<|end|>";- if (size_t pos = prompt.rfind(return_token); pos != std::string::npos) {- prompt.replace(pos, return_token.length(), end_token);- }- }-- data.prompt = prompt;- data.generation_prompt = common_chat_template_generation_prompt_impl(tmpl, inputs, /* messages_override= */ adjusted_messages);- data.message_delimiters = {- { COMMON_CHAT_ROLE_ASSISTANT, "<|start|>assistant" },- { COMMON_CHAT_ROLE_USER, "<|start|>user" },- { COMMON_CHAT_ROLE_SYSTEM, "<|start|>developer" },- { COMMON_CHAT_ROLE_SYSTEM, "<|start|>system" },- { COMMON_CHAT_ROLE_TOOL, "<|start|>functions" },- };-- data.format = COMMON_CHAT_FORMAT_PEG_NATIVE;- data.supports_thinking = true;-- data.thinking_start_tag = "<|channel|>analysis<|message|>";- data.thinking_end_tags = {"<|end|>"};-- // These special tokens are required to parse properly, so we include them- // even if parse_tool_calls is false.- data.preserved_tokens = {- "<|channel|>", "<|constrain|>", "<|message|>", "<|start|>", "<|end|>",- };-- // Adjust prompt for continuation- if (inputs.has_continuation()) {- const auto & msg = inputs.continue_msg;-- data.generation_prompt = "<|start|>assistant<|channel|>analysis<|message|>" + msg.reasoning_content;- if (inputs.continue_final_message == COMMON_CHAT_CONTINUATION_CONTENT) {- data.generation_prompt += "<|end|><|start|>assistant<|channel|>final<|message|>" + msg.render_content();- }-- data.prompt += data.generation_prompt;- }-- auto has_tools = inputs.tools.is_array() && !inputs.tools.empty();- auto has_response_format = !inputs.json_schema.is_null() && inputs.json_schema.is_object();- auto include_grammar = has_response_format || (has_tools && inputs.tool_choice != COMMON_CHAT_TOOL_CHOICE_NONE);- auto extract_reasoning = inputs.reasoning_format != COMMON_REASONING_FORMAT_NONE;-- auto parser = build_chat_peg_parser([&](common_chat_peg_builder & p) {- auto start = p.rule("start", p.literal("<|start|>assistant"));- auto end = p.rule("end", p.literal("<|end|>"));- auto content = p.rule("message-content", p.until("<|end|>"));- auto channel = p.literal("<|channel|>") + (p.literal("commentary") | p.literal("analysis"));- auto constrain_type = p.chars("[A-Za-z0-9_-]", 1, -1);-- // Occasionally, gpt-oss-20b will prefix channels with this commentary- auto stray_commentary = p.optional(p.literal("<|channel|>commentary") + p.optional(p.literal(" to=assistant")));- auto start_analysis = stray_commentary + p.literal("<|channel|>analysis<|message|>");-- if (extract_reasoning) {- p.rule("analysis", start_analysis + p.reasoning(content) + end);- } else {- p.rule("analysis", p.content(start_analysis + content + end));- }-- auto analysis = p.ref("analysis");- auto preamble = p.rule("preamble", p.literal("<|channel|>commentary<|message|>") + p.content(content) + end);- auto final_msg = p.rule("final", stray_commentary + p.literal("<|channel|>final<|message|>") + p.content(content));-- // Consume any unsolicited tool calls, e.g. builtin functions- auto unsolicited = p.rule("unsolicited", p.atomic(p.optional(channel) + p.literal(" to=") + content + end));-- auto any = p.rule("any", preamble | analysis);-- if (has_response_format) {- auto constraint = p.optional(p.space() + p.optional(p.literal("<|constrain|>")) + constrain_type);- auto response_format = p.rule("response-format",- p.literal("<|channel|>final") + constraint + p.literal("<|message|>") +- p.content(p.schema(p.json(), "response-format-schema", inputs.json_schema)));-- return p.zero_or_more(start + analysis) + start + response_format;- }-- if (has_tools && inputs.tool_choice != COMMON_CHAT_TOOL_CHOICE_NONE) {- auto tool_choice = p.choice();-- foreach_function(inputs.tools, [&](const json & tool) {- const auto & function = tool.at("function");- std::string name = function.at("name");- const auto & params = function.at("parameters");-- auto func_name = p.literal(" to=functions.") + p.tool_name(p.literal(name));- auto constraint = p.optional(p.space() + p.optional(p.literal("<|constrain|>")) + constrain_type);- auto args = p.tool_args(p.schema(p.json(), "tool-" + name + "-schema", params));-- // recipient in role header- // <|start|>assistant to=functions.NAME<|channel|>(commentary|analysis)[constraint]<|message|>ARGS- auto tool_in_role = p.tool(p.tool_open(func_name + channel + constraint + p.literal("<|message|>")) + args);-- // recipient in channel header- // <|channel|>(commentary|analysis) to=functions.NAME[constraint]<|message|>ARGS- auto tool_in_channel = p.tool(p.tool_open(channel + func_name + constraint + p.literal("<|message|>")) + args);-- tool_choice |= p.rule("tool-" + name, tool_in_role | tool_in_channel);- });-- auto tool_call = p.trigger_rule("tool-call", tool_choice);-- if (inputs.tool_choice == COMMON_CHAT_TOOL_CHOICE_REQUIRED) {- return p.zero_or_more(start + any) + start + tool_call;- }-- return p.zero_or_more(start + any) + start + (tool_call | final_msg);- }-- return p.zero_or_more(start + any) + start + (final_msg | unsolicited);- });-- data.parser = parser.save();-- if (include_grammar) {- data.grammar_lazy = !(has_response_format || (has_tools && inputs.tool_choice == COMMON_CHAT_TOOL_CHOICE_REQUIRED));- data.grammar = build_grammar([&](const common_grammar_builder & builder) {- foreach_function(inputs.tools, [&](const json & tool) {- const auto & function = tool.at("function");- auto schema = function.at("parameters");- builder.resolve_refs(schema);- });- if (has_response_format) {- auto schema = inputs.json_schema;- builder.resolve_refs(schema);+// Trim leading/trailing whitespace from message contents before rendering. This+// has to run on the messages (not on the rendered JSON) because templates with+// string-only content caps concatenate typed content parts into a single string+// during rendering, after which the per-part whitespace can no longer be reached.+// Both the plain string content and the text of typed content parts are trimmed.+static void trim_all_content(std::vector<common_chat_msg> & messages) {+ for (auto & message : messages) {+ message.content = trim_whitespace(message.content);+ message.reasoning_content = trim_whitespace(message.reasoning_content);+ for (auto & part : message.content_parts) {+ if (part.type == "text") {+ part.text = trim_whitespace(part.text);}- parser.build_grammar(builder, data.grammar_lazy);- });-- data.grammar_triggers = {- { COMMON_GRAMMAR_TRIGGER_TYPE_PATTERN, "^\\s+to$" },- { COMMON_GRAMMAR_TRIGGER_TYPE_PATTERN, "^<\\|channel\\|>(?:commentary|analysis)\\s+to=functions$" },- { COMMON_GRAMMAR_TRIGGER_TYPE_PATTERN, "<\\|start\\|>assistant(\\s+to)" },- { COMMON_GRAMMAR_TRIGGER_TYPE_PATTERN, "<\\|start\\|>assistant(<\\|channel\\|>(?:commentary|analysis)\\s+to)" }- };- }-- return data;-}--static common_chat_params common_chat_params_init_gemma4(const common_chat_template & tmpl,- const autoparser::generation_params & inputs) {- common_chat_params data;-- data.prompt = common_chat_template_direct_apply_impl(tmpl, inputs);- data.generation_prompt = common_chat_template_generation_prompt_impl(tmpl, inputs);-- if (inputs.add_generation_prompt && string_ends_with(data.prompt, "<turn|>\n")) {- // This may happen if the model generates content + tool_call, the- // template does not add the model's next turn and confuses the model- // from emitting its proper reasoning token sequence.- data.generation_prompt = "<|turn>model\n";- data.prompt += data.generation_prompt;- }-- data.message_delimiters = {- { COMMON_CHAT_ROLE_USER, "<|turn>user" },- { COMMON_CHAT_ROLE_ASSISTANT, "<|turn>model" },- };-- data.format = COMMON_CHAT_FORMAT_PEG_GEMMA4;- data.supports_thinking = true;- data.thinking_start_tag = "<|channel>thought";- data.thinking_end_tags = {"<channel|>"};-- data.preserved_tokens = {- "<|channel>",- "<channel|>",- "<|tool_call>",- "<tool_call|>",- "<|turn>",- };-- if (inputs.has_continuation()) {- const auto & msg = inputs.continue_msg;-- data.generation_prompt = string_ends_with(data.prompt, "<turn|>\n") ? "<|turn>model\n" : "";- data.generation_prompt += "<|channel>thought\n" + msg.reasoning_content;- if (inputs.continue_final_message == COMMON_CHAT_CONTINUATION_CONTENT) {- data.generation_prompt += "<channel|>" + msg.render_content();- }-- data.prompt += data.generation_prompt;- }-- auto has_tools = inputs.tools.is_array() && !inputs.tools.empty();- auto has_response_format = !inputs.json_schema.is_null() && inputs.json_schema.is_object();- auto include_grammar = has_response_format || (has_tools && inputs.tool_choice != COMMON_CHAT_TOOL_CHOICE_NONE);- auto extract_reasoning = inputs.reasoning_format != COMMON_REASONING_FORMAT_NONE;-- auto parser = build_chat_peg_parser([&](common_chat_peg_builder & p) {- auto start = p.rule("start", p.optional(p.literal("<|turn>model\n")));-- if (extract_reasoning) {- p.rule("thought", p.literal("<|channel>thought") + p.space() + p.reasoning(p.until("<channel|>")) + p.literal("<channel|>"));- } else {- p.rule("thought", p.content(p.literal("<|channel>thought") + p.space() + p.until("<channel|>") + p.literal("<channel|>")));- }-- auto consume_empty_channels = p.gbnf(p.zero_or_more(p.literal("<|channel>") + p.negate(p.literal("thought"))), "");- auto thought = (p.peek(p.literal("<|channel>")) + consume_empty_channels + p.ref("thought")) | p.negate(p.literal("<|channel>"));-- if (has_response_format) {- auto response_format = p.literal("```json") <<- p.content(p.schema(p.json(), "response-format-schema", inputs.json_schema)) <<- p.literal("```");- return start + p.optional(thought) + response_format;- }-- if (has_tools && inputs.tool_choice != COMMON_CHAT_TOOL_CHOICE_NONE) {- // Gemma4 tool calling syntax- // Rules should match traversal logic in gemma4_to_json()- p.rule("gemma4-string-content", p.until("<|\"|>"));- p.rule("gemma4-string", p.literal("<|\"|>") + p.ref("gemma4-string-content") + p.literal("<|\"|>"));- p.rule("gemma4-bool", p.json_bool());- p.rule("gemma4-null", p.json_null());- p.rule("gemma4-number", p.json_number());- p.rule("gemma4-dict-key", p.rule("gemma4-dict-key-name", p.chars("[^:}]", 1, -1)) + p.literal(":"));- p.rule("gemma4-dict-kv", p.ref("gemma4-dict-key") + p.space() + p.ref("gemma4-value"));- p.rule("gemma4-dict", [&]() {- auto ws = p.space();- auto member = p.ref("gemma4-dict-kv");- auto members = p.sequence({member, p.zero_or_more(p.sequence({p.literal(","), ws, member}))});- return p.sequence({- p.literal("{"), ws,- p.choice({p.literal("}"), p.sequence({members, ws, p.literal("}")})})- });- });- p.rule("gemma4-array", [&]() {- auto ws = p.space();- auto value = p.ref("gemma4-value");- auto elements = p.sequence({value, p.zero_or_more(p.sequence({p.literal(","), ws, value}))});- return p.sequence({- p.literal("["), ws,- p.choice({p.literal("]"), p.sequence({elements, ws, p.literal("]")})})- });- });- p.rule("gemma4-value", [&]() {- return p.choice({- p.ref("gemma4-string"), p.ref("gemma4-dict"), p.ref("gemma4-array"),- p.ref("gemma4-number"), p.ref("gemma4-bool"), p.ref("gemma4-null")- });- });-- auto tool_choice = p.choice();-- foreach_function(inputs.tools, [&](const json & tool) {- const auto & function = tool.at("function");- std::string name = function.at("name");- // TODO @aldehir : need to extend json-schema-to-grammar to produce more than JSON rules- // const auto & params = function.at("parameters");-- tool_choice |= p.rule("tool-" + name, p.tool(p.sequence({- p.tool_open(p.tool_name(p.literal(name)) + p.peek(p.literal("{"))),- p.tool_args(p.ref("gemma4-dict")),- })));- });-- auto tool_call = p.trigger_rule("tool-call", p.repeat(- "<|tool_call>call:" + tool_choice + "<tool_call|>",- /* min = */ inputs.tool_choice == COMMON_CHAT_TOOL_CHOICE_REQUIRED ? 1 : 0,- /* max = */ inputs.parallel_tool_calls ? -1 : 1- ));-- auto scan_to_toolcall = p.rule("scan-to-toolcall", p.until("<|tool_call>"));- auto content = p.rule("content", p.content(p.until_one_of({"<|channel>", "<channel|>", "<|tool_call>"})));- auto message = p.rule("message", thought + content);- return start + p.zero_or_more(message) + scan_to_toolcall + tool_call;}-- // Gemma 4 may emit an extra <|channel>thought\n<channel|> at the end of the content. It may- // also emit a single trailing <channel|> token. Consume all complete reasoning blocks and- // then stop at the first unmatched <channel|> token.- auto content = p.rule("content", p.content(p.until_one_of({"<|channel>", "<channel|>"})));- auto message = p.rule("message", thought + content);- return start + p.one_or_more(message);- });-- data.parser = parser.save();-- if (include_grammar) {- data.grammar_lazy = !(has_response_format || (has_tools && inputs.tool_choice == COMMON_CHAT_TOOL_CHOICE_REQUIRED));- data.grammar = build_grammar([&](const common_grammar_builder & builder) {- foreach_function(inputs.tools, [&](const json & tool) {- const auto & function = tool.at("function");- auto schema = function.at("parameters");- builder.resolve_refs(schema);- });- if (has_response_format) {- auto schema = inputs.json_schema;- builder.resolve_refs(schema);- }- parser.build_grammar(builder, data.grammar_lazy);- });-- data.grammar_triggers = {- { COMMON_GRAMMAR_TRIGGER_TYPE_WORD, "<|tool_call>" },- };}-- return data;}-// Functionary v3.2 - uses recipient-based format: >>>recipient\n{content}-static common_chat_params common_chat_params_init_functionary_v3_2(const common_chat_template & tmpl,- const autoparser::generation_params & inputs) {- common_chat_params data;-- data.prompt = common_chat_template_direct_apply_impl(tmpl, inputs);- data.generation_prompt = common_chat_template_generation_prompt_impl(tmpl, inputs);- data.format = COMMON_CHAT_FORMAT_PEG_NATIVE;- data.preserved_tokens = {- ">>>all",- };-- auto has_tools = inputs.tools.is_array() && !inputs.tools.empty();- auto include_grammar = has_tools && inputs.tool_choice != COMMON_CHAT_TOOL_CHOICE_NONE;-- if (inputs.has_continuation()) {- const auto & msg = inputs.continue_msg;- data.generation_prompt = "<|start_header_id|>assistant<|end_header_id|>\n\n>>>all\n" + msg.render_content();- data.prompt += data.generation_prompt;- }-- auto parser = build_chat_peg_parser([&](common_chat_peg_builder & p) {- // Functionary v3.2 format:- // - Normal content: >>>all\n{content}- // - Tool calls: >>>function_name\n{json_args}- // Generation prompt ends with ">>>" so model outputs recipient immediately-- // Build content parser for >>>all\n{content}- // When tools are present, content stops before the next ">>>" (tool call)- // When no tools, content goes until end- auto content_until_tool = p.literal("all\n") + p.content(p.until(">>>"));- auto content_until_end = p.literal("all\n") + p.content(p.rest());- auto generation_prompt = p.literal("<|start_header_id|>assistant<|end_header_id|>\n\n>>>");-- // If no tools or tool_choice is NONE, just parse content- if (!has_tools || inputs.tool_choice == COMMON_CHAT_TOOL_CHOICE_NONE) {- // When no tools, just match the prefix and capture everything after- return generation_prompt + content_until_end + p.end();- }-- // Build tool call parsers for each available function- auto tool_choice = p.choice();- foreach_function(inputs.tools, [&](const json & tool) {- const auto & function = tool.at("function");- std::string name = function.at("name");- const auto & schema = function.at("parameters");-- // Tool format: >>>function_name\n{json_args}- auto tool_parser = p.tool(- p.tool_open(p.tool_name(p.literal(name)) + p.literal("\n")) +- p.tool_args(p.schema(p.json(), "tool-" + name + "-schema", schema))- );-- tool_choice |= p.rule("tool-" + name, tool_parser);- });-- auto content_only = content_until_end;- auto tools_only = p.trigger_rule("tools", p.one_or_more(tool_choice));- auto content_and_tools = content_until_tool + tools_only;-- auto ret = p.eps();- if (inputs.tool_choice == COMMON_CHAT_TOOL_CHOICE_REQUIRED) {- if (inputs.parallel_tool_calls) {- ret = p.choice({ content_and_tools, tools_only }) + p.end();- } else {- ret = p.choice({ content_until_tool + tool_choice, tools_only }) + p.end();- }- } else if (inputs.parallel_tool_calls) {- ret = p.choice({ content_and_tools, content_only, tools_only }) + p.end();- } else {- auto content_and_tool = content_until_tool + tool_choice;- ret = p.choice({ content_and_tool, content_only, tool_choice }) + p.end();- }- return generation_prompt + ret;- });-- data.parser = parser.save();-- if (include_grammar) {- data.grammar_lazy = inputs.tool_choice == COMMON_CHAT_TOOL_CHOICE_AUTO;-- data.grammar = build_grammar([&](const common_grammar_builder & builder) {- foreach_function(inputs.tools, [&](const json & tool) {- const auto & function = tool.at("function");- auto schema = function.at("parameters");- builder.resolve_refs(schema);- });- parser.build_grammar(builder, data.grammar_lazy);- });-- // Grammar trigger for when the model starts outputting a tool call- // (after the initial ">>>" in the generation prompt but recipient other than "all")- data.grammar_triggers = {- { COMMON_GRAMMAR_TRIGGER_TYPE_PATTERN, ">>>(?!all)" }- };- }-- return data;-}--// Kimi K2 Thinking - uses unique tool call ID format: functions.<name>:<index>-// The ID contains both the function name and an incrementing counter-static common_chat_params common_chat_params_init_kimi_k2(const common_chat_template & tmpl,- const autoparser::generation_params & inputs) {- common_chat_params data;-- data.prompt = common_chat_template_direct_apply_impl(tmpl, inputs);- data.generation_prompt = common_chat_template_generation_prompt_impl(tmpl, inputs);- data.format = COMMON_CHAT_FORMAT_PEG_NATIVE;- data.supports_thinking = true;- data.preserved_tokens = {- "<|tool_calls_section_begin|>",- "<|tool_calls_section_end|>",- "<|tool_call_begin|>",- "<|tool_call_argument_begin|>",- "<|tool_call_end|>",- "<think>",- "</think>",- };-- auto has_tools = inputs.tools.is_array() && !inputs.tools.empty();- auto extract_reasoning = inputs.reasoning_format != COMMON_REASONING_FORMAT_NONE;- auto include_grammar = has_tools && inputs.tool_choice != COMMON_CHAT_TOOL_CHOICE_NONE;-- const std::string SECTION_BEGIN = "<|tool_calls_section_begin|>";- const std::string SECTION_END = "<|tool_calls_section_end|>";- const std::string CALL_BEGIN = "<|tool_call_begin|>";- const std::string ARGS_BEGIN = "<|tool_call_argument_begin|>";- const std::string CALL_END = "<|tool_call_end|>";-- const std::string THINK_START = "<think>";- const std::string THINK_END = "</think>";- const std::string GEN_PROMPT = "<|im_assistant|>assistant<|im_middle|>";-- data.thinking_start_tag = THINK_START;- data.thinking_end_tags = {THINK_END};-- if (inputs.has_continuation()) {- const auto & msg = inputs.continue_msg;-- data.generation_prompt = GEN_PROMPT + THINK_START + msg.reasoning_content;- if (inputs.continue_final_message == COMMON_CHAT_CONTINUATION_CONTENT) {- data.generation_prompt += THINK_END + msg.render_content();- }-- data.prompt += data.generation_prompt;- }-- auto parser = build_chat_peg_parser([&](common_chat_peg_builder & p) {- // Kimi K2 Thinking format:- // - Reasoning: <think>{reasoning}</think>- // - Content: text after reasoning- // - Tool calls section:- // <|tool_calls_section_begin|>- // <|tool_call_begin|>functions.<name>:<index><|tool_call_argument_begin|>{json_args}<|tool_call_end|>- // ...- // <|tool_calls_section_end|>- // The ID format is: functions.<function_name>:<counter> where counter is 0, 1, 2, ...-- // Tool call markers- auto end = p.end();-- // Note: this model is CRAZY. It can diverge from its supposed tool calling pattern in so many ways it's not funny.- // For example, it can call tools at the end of reasoning without closing reasoning...- auto reasoning = extract_reasoning ? p.optional(THINK_START + p.reasoning(- p.until_one_of({ THINK_END, "<|tool_calls_section_begin|>", "<|tool_call_begin|>" })) +- p.optional(p.literal(THINK_END))) : p.eps();- auto generation_prompt = p.literal(GEN_PROMPT);--- // Content only parser (no tools)- if (!has_tools || inputs.tool_choice == COMMON_CHAT_TOOL_CHOICE_NONE) {- return generation_prompt + reasoning + p.content(p.rest()) + end;- }-- // Build tool call parsers for each available function- // The ID format is: functions.<name>:<index>- // We need to match: functions.<name>:<digits>- auto tool_choice = p.choice();- foreach_function(inputs.tools, [&](const json & tool) {- const auto & function = tool.at("function");- std::string name = function.at("name");- const auto & schema = function.at("parameters");-- // Match: functions.<name>:<digits>- // Capture the full call id (functions.<name>:<digits>) using tool_id tag- auto tool_id = p.tool_id(p.literal("functions.") + p.tool_name(p.literal(name)) + p.literal(":") + p.chars("[0-9]", 1, -1));- auto tool_parser = p.tool(- p.tool_open(tool_id + p.literal(ARGS_BEGIN)) +- p.tool_args(p.schema(p.json(), "tool-" + name + "-schema", schema)) +- p.tool_close(p.optional((p.literal(CALL_END))))- );-- tool_choice |= p.rule("tool-" + name, tool_parser);- });-- // Tool calls section: <|tool_calls_section_begin|> tool_calls <|tool_calls_section_end|>- auto min_calls = inputs.tool_choice == COMMON_CHAT_TOOL_CHOICE_REQUIRED ? 1 : 0;- auto max_calls = inputs.parallel_tool_calls ? -1 : 1;- // Use trigger_rule so grammar generator knows where to start generating rules- auto tool_calls = p.rule("tool-calls",- p.optional(p.literal(SECTION_BEGIN)) +- p.trigger_rule("tool-call", p.repeat(CALL_BEGIN + tool_choice, min_calls, max_calls) +- p.optional(p.literal(SECTION_END)))- );-- auto content_before_tools = p.content(p.until_one_of({ SECTION_BEGIN, CALL_BEGIN }));-- return generation_prompt + reasoning + content_before_tools + tool_calls + end;- });-- data.parser = parser.save();-- if (include_grammar) {- data.grammar_lazy = inputs.tool_choice == COMMON_CHAT_TOOL_CHOICE_AUTO;- data.grammar = build_grammar([&](const common_grammar_builder & builder) {- foreach_function(inputs.tools, [&](const json & tool) {- const auto & function = tool.at("function");- auto schema = function.at("parameters");- builder.resolve_refs(schema);- });- parser.build_grammar(builder, data.grammar_lazy);- });-- data.grammar_triggers = {- { COMMON_GRAMMAR_TRIGGER_TYPE_WORD, "<|tool_call_begin|>" }- };- }-- return data;-}--// LFM2/LFM2.5 parser. Tool calls are almost Python-style and parallel-capable-// (except dotted names and JSON literals true/false/null).-// Always wrapped in <|tool_call_start|>[name(args)]<|tool_call_end|> with optional <think> reasoning.-// tool_list_tokens preserves LFM2 system tool-list markers.-static common_chat_params common_chat_params_init_lfm2(const common_chat_template & tmpl,- const autoparser::generation_params & inputs,- bool tool_list_tokens) {- common_chat_params data;-- const std::string TOOL_CALL_START = "<|tool_call_start|>";- const std::string TOOL_CALL_END = "<|tool_call_end|>";- const std::string TOOL_LIST_START = "<|tool_list_start|>";- const std::string TOOL_LIST_END = "<|tool_list_end|>";- const std::string THINK_START = "<think>";- const std::string THINK_END = "</think>";- const std::string GEN_PROMPT = "<|im_start|>assistant\n";-- // Copy reasoning to the "thinking" field the template expects- auto adjusted_messages = json::array();- for (auto msg : inputs.messages) {- if (msg.contains("reasoning_content") && msg.at("reasoning_content").is_string()) {- msg["thinking"] = msg.at("reasoning_content");- }- adjusted_messages.push_back(msg);- }-- data.prompt = common_chat_template_direct_apply_impl(tmpl, inputs, adjusted_messages);- data.generation_prompt = common_chat_template_generation_prompt_impl(tmpl, inputs, adjusted_messages);- data.format = COMMON_CHAT_FORMAT_PEG_NATIVE;- data.supports_thinking = true;- data.preserved_tokens = { TOOL_CALL_START, TOOL_CALL_END, THINK_START, THINK_END };- if (tool_list_tokens) {- data.preserved_tokens.push_back(TOOL_LIST_START);- data.preserved_tokens.push_back(TOOL_LIST_END);- }-- data.thinking_start_tag = THINK_START;- data.thinking_end_tags = {THINK_END};-- auto has_tools = inputs.tools.is_array() && !inputs.tools.empty();- auto has_response_format = !inputs.json_schema.is_null() && inputs.json_schema.is_object();- // Gate by reasoning format and whether the template supports <think>- auto extract_reasoning = inputs.reasoning_format != COMMON_REASONING_FORMAT_NONE &&- tmpl.source().find(THINK_START) != std::string::npos;- auto include_grammar = has_response_format || (has_tools && inputs.tool_choice != COMMON_CHAT_TOOL_CHOICE_NONE);-- if (inputs.has_continuation()) {- const auto & msg = inputs.continue_msg;-- data.generation_prompt = GEN_PROMPT + THINK_START + msg.reasoning_content;- if (inputs.continue_final_message == COMMON_CHAT_CONTINUATION_CONTENT) {- data.generation_prompt += THINK_END + msg.render_content();- }-- data.prompt += data.generation_prompt;- }-- auto parser = build_chat_peg_parser([&](common_chat_peg_builder & p) {- auto generation_prompt = p.literal(GEN_PROMPT);- auto end = p.end();-- auto reasoning = p.eps();- if (extract_reasoning) {- reasoning = p.optional(THINK_START + p.reasoning(p.until(THINK_END)) + THINK_END);- }-- if (!has_tools || inputs.tool_choice == COMMON_CHAT_TOOL_CHOICE_NONE) {- if (has_response_format) {- auto response_format = p.content(p.schema(p.json(), "response-format-schema", inputs.json_schema));- return generation_prompt + reasoning + response_format + end;- }- return generation_prompt + reasoning + p.content(p.rest()) + end;- }- auto tool_calls = p.rule("tool-calls",- p.trigger_rule("tool-call",- p.literal(TOOL_CALL_START) +- p.python_style_tool_calls(inputs.tools, inputs.parallel_tool_calls, /* allow_json_literals = */ true) +- p.literal(TOOL_CALL_END)- )- );-- auto content = p.content(p.until(TOOL_CALL_START));-- return generation_prompt + reasoning + content + tool_calls + end;- });-- data.parser = parser.save();-- if (include_grammar) {- data.grammar_lazy = !(has_response_format || (has_tools && inputs.tool_choice == COMMON_CHAT_TOOL_CHOICE_REQUIRED));- data.grammar = build_grammar([&](const common_grammar_builder & builder) {- foreach_function(inputs.tools, [&](const json & tool) {- const auto & function = tool.at("function");- auto schema = function.at("parameters");- builder.resolve_refs(schema);- });- if (has_response_format) {- auto schema = inputs.json_schema;- builder.resolve_refs(schema);- }- parser.build_grammar(builder, data.grammar_lazy);- });-- data.grammar_triggers = {- { COMMON_GRAMMAR_TRIGGER_TYPE_WORD, TOOL_CALL_START }- };- }-- return data;-}--static common_chat_params common_chat_params_init_gigachat_v3(- const common_chat_template & tmpl,- const autoparser::generation_params & inputs) {-- common_chat_params data;-- data.prompt = common_chat_template_direct_apply_impl(tmpl, inputs);- data.generation_prompt = common_chat_template_generation_prompt_impl(tmpl, inputs);- data.format = COMMON_CHAT_FORMAT_PEG_NATIVE;- data.supports_thinking = false;- data.preserved_tokens = {- "<|message_sep|>\n\n",- "<|role_sep|>\n",- };-- if (inputs.has_continuation()) {- const auto & msg = inputs.continue_msg;- data.generation_prompt = "assistant<|role_sep|>\n" + msg.render_content();- data.prompt += data.generation_prompt;- }-- auto has_tools = inputs.tools.is_array() && !inputs.tools.empty();- auto include_grammar = has_tools && inputs.tool_choice != COMMON_CHAT_TOOL_CHOICE_NONE;- const auto *tool_call_start_prefix = "<|message_sep|>\n\nfunction call<|role_sep|>\n";-- auto parser = build_chat_peg_parser([&](common_chat_peg_builder & p) {- auto ret = p.eps();- if (has_tools && inputs.tool_choice != COMMON_CHAT_TOOL_CHOICE_NONE) {- // Build a choice of all available tools- auto tool_choice = p.choice();- for (const auto & tool : inputs.tools) {- const auto & function = tool.at("function");- std::string name = function.at("name");- const auto & schema = function.at("parameters");-- auto tool_name = p.json_member("name", "\"" + p.tool_name(p.literal(name)) + "\"");- auto tool_args = p.json_member("arguments", p.tool_args(p.schema(p.json(), "tool-" + name + "-schema", schema)));-- auto tool_open = p.tool_open(p.literal("{") << tool_name);-- tool_choice |= p.rule("tool-" + name, tool_open << "," << tool_args << "}");- }-- // Define the tool call structure- auto min_calls = inputs.tool_choice == COMMON_CHAT_TOOL_CHOICE_REQUIRED ? 1 : 0;- auto max_calls = 1; // parallel toolcalls are not supported- auto tool_call = p.rule("tool-call", p.literal(tool_call_start_prefix) + tool_choice);- auto tool_calls = p.trigger_rule("tool-call-root", p.repeat(tool_call, /* min = */ min_calls, /* max = */ max_calls));-- ret = p.content(p.until("<|message_sep|>\n\n")) << tool_calls;- } else {- // Content only parser- include_grammar = false;- ret = p.content(p.rest());- }-- return p.literal("assistant<|role_sep|>\n") + ret;- });-- data.parser = parser.save();-- if (include_grammar) {- data.grammar_lazy = has_tools && inputs.tool_choice == COMMON_CHAT_TOOL_CHOICE_AUTO;-- data.grammar = build_grammar([&](const common_grammar_builder & builder) {- foreach_function(inputs.tools, [&](const json & tool) {- const auto & function = tool.at("function");- auto schema = function.at("parameters");- builder.resolve_refs(schema);- });- parser.build_grammar(builder, data.grammar_lazy);- });-- data.grammar_triggers = {- {COMMON_GRAMMAR_TRIGGER_TYPE_WORD, tool_call_start_prefix}- };- }- return data;-}--// The DeepSeek V4 reference implementation renders consecutive tool results into a single-// user block, ordered by the tool call order of the preceding assistant message (matched-// by tool call id) rather than by the order they appear in the conversation.-static json deepseek_v4_sort_tool_results(const json & messages) {- json adjusted = messages;- std::map<std::string, size_t> call_order;-- for (size_t i = 0; i < adjusted.size();) {- const auto & msg = adjusted[i];- const auto role = msg.value("role", "");-- if (role == "assistant" && msg.contains("tool_calls") &&- msg.at("tool_calls").is_array() && !msg.at("tool_calls").empty()) {- call_order.clear();- const auto & tool_calls = msg.at("tool_calls");- for (size_t idx = 0; idx < tool_calls.size(); idx++) {- auto id = tool_calls[idx].value("id", "");- if (!id.empty()) {- call_order[id] = idx;- }- }- i++;- continue;- }-- if (role != "user" && role != "tool") {- i++;- continue;- }-- // collect a maximal run of user/tool messages - they render into one user block- std::vector<size_t> tool_positions;- size_t run_end = i;- for (; run_end < adjusted.size(); run_end++) {- const auto r = adjusted[run_end].value("role", "");- if (r == "tool") {- tool_positions.push_back(run_end);- } else if (r != "user") {- break;- }- }-- if (tool_positions.size() > 1 && !call_order.empty()) {- std::vector<json> results;- results.reserve(tool_positions.size());- for (auto pos : tool_positions) {- results.push_back(adjusted[pos]);- }- std::stable_sort(results.begin(), results.end(), [&](const json & a, const json & b) {- const auto order = [&](const json & m) {- auto it = call_order.find(m.value("tool_call_id", ""));- return it == call_order.end() ? (size_t) 0 : it->second;- };- return order(a) < order(b);- });- for (size_t k = 0; k < tool_positions.size(); k++) {- adjusted[tool_positions[k]] = std::move(results[k]);- }- }-- i = run_end;- }-- return adjusted;-}--static common_chat_params common_chat_params_init_deepseek_v3_2(const common_chat_template & tmpl,- const autoparser::generation_params & inputs) {- common_chat_params data;-- // V4 uses the same DSML markup as V3.2, but names the tool call block "tool_calls"- // instead of "function_calls", renders tool results in tool call order and its- // non-thinking generation prompt ends with a bare </think> instead of an empty- // <think></think> pair.- const bool is_v4 = tmpl.source().find("function_calls") == std::string::npos;-- std::optional<json> adjusted_messages;- if (is_v4) {- adjusted_messages = deepseek_v4_sort_tool_results(inputs.messages);- }-- auto has_tools = inputs.tools.is_array() && !inputs.tools.empty();- auto has_response_format = !inputs.json_schema.is_null() && inputs.json_schema.is_object();- auto extract_reasoning = inputs.reasoning_format != COMMON_REASONING_FORMAT_NONE;- auto include_grammar = has_response_format || (has_tools && inputs.tool_choice != COMMON_CHAT_TOOL_CHOICE_NONE);-- std::optional<json> additional_context;- if (is_v4 && has_response_format) {- additional_context = json{ { "response_format", inputs.json_schema } };- }-- const std::string DSML = "|DSML|";- const std::string THINK_START = "<think>";- const std::string THINK_END = "</think>";- const std::string TC_BLOCK = is_v4 ? "tool_calls" : "function_calls";- const std::string FC_START = "<" + DSML + TC_BLOCK + ">";- const std::string FC_END = "</" + DSML + TC_BLOCK + ">";- const std::string INVOKE_START = "<" + DSML + "invoke";- const std::string INVOKE_END = "</" + DSML + "invoke>";- const std::string PARAM_START = "<" + DSML + "parameter";- const std::string PARAM_END = "</" + DSML + "parameter>";- const std::string GEN_PROMPT = "<|Assistant|>";- const std::string TC_SEPARATOR = "\n\n";-- data.prompt = common_chat_template_direct_apply_impl(- tmpl, inputs, adjusted_messages, std::nullopt, additional_context);- data.generation_prompt = common_chat_template_generation_prompt_impl(- tmpl, inputs, adjusted_messages, std::nullopt, additional_context);- data.format = COMMON_CHAT_FORMAT_PEG_NATIVE;- data.supports_thinking = true;- data.thinking_start_tag = THINK_START;- data.thinking_end_tags = {THINK_END, FC_START};- data.preserved_tokens = {- DSML,- THINK_START,- THINK_END,- };-- if (inputs.has_continuation()) {- const auto & msg = inputs.continue_msg;-- if (is_v4 && msg.reasoning_content.empty()) {- data.generation_prompt = GEN_PROMPT + THINK_END;- if (inputs.continue_final_message == COMMON_CHAT_CONTINUATION_CONTENT) {- data.generation_prompt += msg.render_content();- }- } else {- data.generation_prompt = GEN_PROMPT + THINK_START + msg.reasoning_content;- if (inputs.continue_final_message == COMMON_CHAT_CONTINUATION_CONTENT) {- data.generation_prompt += THINK_END + msg.render_content();- }- }-- data.prompt += data.generation_prompt;- }-- bool require_tools = inputs.tool_choice == COMMON_CHAT_TOOL_CHOICE_REQUIRED;- bool has_tool_calls = has_tools && inputs.tool_choice != COMMON_CHAT_TOOL_CHOICE_NONE;-- auto parser = build_chat_peg_parser([&](common_chat_peg_builder & p) {- auto generation_prompt = p.literal(GEN_PROMPT);- auto end = p.end();-- // build tool call section first since we might need it in reasoning- auto tool_choice = p.choice();- if (has_tool_calls) {- foreach_function(inputs.tools, [&](const json & tool) {- const auto & function = tool.at("function");- std::string name = function.at("name");- auto params = function.contains("parameters") ? function.at("parameters") : json::object();- const auto & props = params.contains("properties") ? params.at("properties") : json::object();-- std::set<std::string> required;- if (params.contains("required")) {- required = params.at("required").get<std::set<std::string>>();- }-- auto schema_info = common_schema_info();- schema_info.resolve_refs(params);-- std::vector<common_peg_parser> required_parsers;- std::vector<common_peg_parser> optional_parsers;- for (const auto & [param_name, param_schema] : props.items()) {- bool is_required = required.find(param_name) != required.end();- bool is_string = schema_info.resolves_to_string(param_schema);-- auto arg = p.tool_arg(- p.tool_arg_open(p.literal(PARAM_START + " name=\"") + p.tool_arg_name(p.literal(param_name)) +- p.literal("\" string=\"" + std::string(is_string ? "true" : "false") + "\">")) +- (is_string ?- p.tool_arg_string_value(p.until(PARAM_END)) :- p.tool_arg_json_value(p.schema(p.json(), "tool-" + name + "-arg-" + param_name + "-schema",- param_schema, false))) +- p.tool_arg_close(p.literal(PARAM_END)));-- auto named_arg = p.rule("tool-" + name + "-arg-" + param_name, arg);- if (is_required) {- required_parsers.push_back(named_arg);- } else {- optional_parsers.push_back(named_arg);- }- }-- common_peg_parser args_seq = p.eps();- for (size_t i = 0; i < required_parsers.size(); i++) {- if (i > 0) {- args_seq = args_seq + p.space();- }- args_seq = args_seq + required_parsers[i];- }-- if (!optional_parsers.empty()) {- common_peg_parser any_opt = p.choice();- for (const auto & opt : optional_parsers) {- any_opt |= opt;- }- args_seq = args_seq + p.repeat(p.space() + any_opt, 0, -1);- }-- common_peg_parser invoke_body = args_seq;- auto func_parser = p.tool(p.tool_open(p.literal(INVOKE_START + " name=\"") +- p.tool_name(p.literal(name)) + p.literal("\">\n")) +- invoke_body + p.space() + p.tool_close(p.literal(INVOKE_END)));-- tool_choice |= p.rule("tool-" + name, func_parser);- });- }-- common_peg_parser tool_calls = p.eps();- if (inputs.parallel_tool_calls) {- tool_calls = p.trigger_rule("tool-call",- p.literal(FC_START) + p.space() + tool_choice +- p.zero_or_more(p.space() + tool_choice) + p.space() + p.literal(FC_END));- } else {- tool_calls = p.trigger_rule("tool-call",- p.literal(FC_START) + p.space() + tool_choice + p.space() + p.literal(FC_END));- }-- auto reasoning = p.eps();- auto reasoning_with_tc = p.eps();- auto obligatory_tool_calls = tool_calls;- bool allow_reasoning_with_tc = false;-- if (!require_tools) {- tool_calls = p.optional(tool_calls);- }-- if (extract_reasoning && inputs.enable_thinking) {- reasoning = p.optional(THINK_START + p.reasoning(p.until(THINK_END)) + THINK_END);- reasoning_with_tc = THINK_START +- p.reasoning(p.until_one_of({ TC_SEPARATOR + FC_START, FC_START, THINK_END })) +- p.space() + obligatory_tool_calls;- allow_reasoning_with_tc = true;- } else if (extract_reasoning) {- // Thinking disabled but reasoning extraction requested: the generation prompt- // contains an empty <think></think> pair (V3.2) or a bare </think> (V4) that- // must still be consumed.- reasoning = is_v4- ? p.optional(p.literal(THINK_END))- : p.optional(p.literal(THINK_START) + p.until(THINK_END) + p.literal(THINK_END));- }-- if (has_response_format) {- auto response_format = p.rule("response-format",- p.literal("```json") + p.space() +- p.content(p.schema(p.json(), "response-format-schema", inputs.json_schema)) +- p.space() + p.literal("```"));- return generation_prompt + reasoning + response_format + end;- }-- if (!has_tool_calls) {- return generation_prompt + reasoning + p.content(p.rest()) + end;- }-- auto content_before_tools = p.negate(p.literal(THINK_START)) +- p.content(p.until_one_of({ TC_SEPARATOR + FC_START, FC_START })) +- p.space();- return allow_reasoning_with_tc ? generation_prompt + (reasoning_with_tc | (reasoning + content_before_tools + tool_calls)) + end :- generation_prompt + reasoning + content_before_tools + tool_calls + end;- });-- data.parser = parser.save();-- if (include_grammar) {- data.grammar_lazy = has_tools && !require_tools;- data.grammar = build_grammar([&](const common_grammar_builder & builder) {- foreach_function(inputs.tools, [&](const json & tool) {- const auto & function = tool.at("function");- auto schema = function.contains("parameters") ? function.at("parameters") : json::object();- builder.resolve_refs(schema);- });- if (has_response_format) {- auto schema = inputs.json_schema;- builder.resolve_refs(schema);- }- parser.build_grammar(builder, data.grammar_lazy);- });-- data.grammar_triggers = {- { COMMON_GRAMMAR_TRIGGER_TYPE_WORD, FC_START },- };- }-- return data;-}--// Kimi K3 - XTML tagged format, built by open_tag/close_tag macros:-// open_tag(t, attrs) = <|open|>t k="v"...<|sep|> close_tag(t) = <|close|>t<|sep|>-// assistant := [think] [response] [tools] close_tag(message) <|end_of_msg|>-// the generation prompt already opens the think (or response) section, so the-// section opener is optional here - same as Kimi K2 Thinking-static common_chat_params common_chat_params_init_kimi_k3(const common_chat_template & tmpl,- const autoparser::generation_params & inputs) {- common_chat_params data;-- data.prompt = common_chat_template_direct_apply_impl(tmpl, inputs);- data.generation_prompt = common_chat_template_generation_prompt_impl(tmpl, inputs);- data.format = COMMON_CHAT_FORMAT_PEG_NATIVE;- data.supports_thinking = true;-- const std::string SEP = "<|sep|>";- const std::string MSG_START = "<|open|>message role=\"assistant\"<|sep|>";- const std::string THINK_START = "<|open|>think<|sep|>";- const std::string THINK_END = "<|close|>think<|sep|>";- const std::string RESP_START = "<|open|>response<|sep|>";- const std::string RESP_END = "<|close|>response<|sep|>";- const std::string TOOLS_START = "<|open|>tools<|sep|>";- const std::string TOOLS_END = "<|close|>tools<|sep|>";- const std::string CALL_START = "<|open|>call tool=\"";- const std::string CALL_END = "<|close|>call<|sep|>";- const std::string ARG_START = "<|open|>argument key=\"";- const std::string ARG_END = "<|close|>argument<|sep|>";- const std::string MSG_END = "<|close|>message<|sep|>";- const std::string EOM_TOKEN = "<|end_of_msg|>";-- // only the markers are special tokens. tag names ("think", "response", ...) are- // normal tokens and must not be preserved, or prose with those words is broken- data.preserved_tokens = {- "<|open|>",- "<|close|>",- "<|sep|>",- "<|end_of_msg|>",- };-- data.thinking_start_tag = THINK_START;- data.thinking_end_tags = { THINK_END };-- // per-role message-start delimiters. user/assistant messages only have the role- // attribute, so the full opener is used. system and tool messages have more- // attributes, so those delimiters stop after the closing quote of the role- data.message_delimiters = {- { COMMON_CHAT_ROLE_ASSISTANT, "<|open|>message role=\"assistant\"<|sep|>" },- { COMMON_CHAT_ROLE_USER, "<|open|>message role=\"user\"<|sep|>" },- { COMMON_CHAT_ROLE_TOOL, "<|open|>message role=\"tool\"" },- { COMMON_CHAT_ROLE_SYSTEM, "<|open|>message role=\"system\"" },- };-- auto has_tools = inputs.tools.is_array() && !inputs.tools.empty();- auto extract_reasoning = inputs.reasoning_format != COMMON_REASONING_FORMAT_NONE;- auto include_grammar = has_tools && inputs.tool_choice != COMMON_CHAT_TOOL_CHOICE_NONE;-- if (inputs.has_continuation()) {- const auto & msg = inputs.continue_msg;-- data.generation_prompt = MSG_START + THINK_START + msg.reasoning_content;- if (inputs.continue_final_message == COMMON_CHAT_CONTINUATION_CONTENT) {- data.generation_prompt += THINK_END + RESP_START + msg.render_content();- }-- data.prompt += data.generation_prompt;- }-- auto parser = build_chat_peg_parser([&](common_chat_peg_builder & p) {- auto end = p.end();-- auto start = p.optional(p.literal(MSG_START));-- // the think section is always consumed, even with reasoning extraction off:- // the generation prompt ends with open_tag('think'), so it is always present.- // reasoning stops at its own closer, or at the response opener if the model- // skips the closer- auto think_body = extract_reasoning ? p.reasoning(p.until_one_of({ THINK_END, RESP_START })) :- p.content(p.until_one_of({ THINK_END, RESP_START }));-- auto reasoning = p.optional(p.optional(p.literal(THINK_START)) + think_body +- p.optional(p.literal(THINK_END)));-- // content runs to the response closer, or to the next section if truncated- auto response = p.optional(p.literal(RESP_START)) +- p.content(p.until_one_of({ RESP_END, TOOLS_START, MSG_END })) +- p.optional(p.literal(RESP_END));-- // the EOG token after the message closer reaches the parser as text,- // so it must be consumed or the parse stays incomplete- auto trailer = p.optional(p.literal(MSG_END)) + p.optional(p.literal(EOM_TOKEN));-- if (!has_tools || inputs.tool_choice == COMMON_CHAT_TOOL_CHOICE_NONE) {- return start + reasoning + response + trailer + end;- }-- auto tool_choices = p.choice();- foreach_function(inputs.tools, [&](const json & tool) {- const auto & function = tool.at("function");- std::string name = function.at("name");- const json schema = function.contains("parameters") ? function.at("parameters") : json::object();-- // arguments come one tag per key, with the JSON type in a type="..."- // attribute. the type is taken from the tool schema instead, as it tells- // us if the value is JSON or a literal string- auto args = p.eps();- if (schema.contains("properties") && !schema.at("properties").empty()) {- auto arg_choices = p.choice();- for (const auto & prop : schema.at("properties").items()) {- const std::string & key = prop.key();-- std::string type = "string";- if (prop.value().is_object() && prop.value().contains("type") &&- prop.value().at("type").is_string()) {- type = prop.value().at("type").get<std::string>();- }-- auto value = type == "string" ? p.tool_arg_string_value(p.until(ARG_END)) :- p.tool_arg_value(p.until(ARG_END));-- // skip the trailing type="..." attribute: anything up to <|sep|>- arg_choices |= p.rule("kimi-k3-arg-" + name + "-" + key,- p.tool_arg(p.tool_arg_open(p.literal(ARG_START)) +- p.tool_arg_name(p.literal(key)) + p.literal("\"") +- p.until(SEP) + p.literal(SEP) + value +- p.tool_arg_close(p.literal(ARG_END))));- }- args = p.zero_or_more(arg_choices);- }-- // skip the trailing index="N" attribute the same way- auto call = p.tool(p.tool_open(p.literal(CALL_START) + p.tool_name(p.literal(name)) + p.literal("\"") +- p.until(SEP) + p.literal(SEP)) +- p.tool_args(args) + p.tool_close(p.literal(CALL_END)));-- tool_choices |= p.rule("kimi-k3-tool-" + name, call);- });-- // all calls go inside one tools section, then the message is closed. the- // message closer is part of the trigger rule, or else the lazy grammar- // rejects it once tool calls have started- auto tools_section =- p.trigger_rule("kimi-k3-tool-call", p.literal(TOOLS_START) + p.one_or_more(tool_choices) +- p.literal(TOOLS_END) + p.optional(p.literal(MSG_END)) +- p.optional(p.literal(EOM_TOKEN)));-- auto tools = inputs.tool_choice == COMMON_CHAT_TOOL_CHOICE_REQUIRED ? tools_section :- p.optional(tools_section);-- return start + reasoning + response + tools + trailer + end;- });-- data.parser = parser.save();-- if (include_grammar) {- data.grammar_lazy = inputs.tool_choice != COMMON_CHAT_TOOL_CHOICE_REQUIRED;- data.grammar = build_grammar([&](const common_grammar_builder & builder) {- foreach_function(inputs.tools, [&](const json & tool) {- const auto & function = tool.at("function");- if (function.contains("parameters")) {- auto schema = function.at("parameters");- builder.resolve_refs(schema);- }- });- parser.build_grammar(builder, data.grammar_lazy);- });-- data.grammar_triggers = {- { COMMON_GRAMMAR_TRIGGER_TYPE_WORD, TOOLS_START },- };- }-- return data;-}--// Cohere2 MoE (a.k.a. "North Code") parser.-//-// The assistant turn is fully marker-wrapped:-// <|START_OF_TURN_TOKEN|><|CHATBOT_TOKEN|>-// <|START_THINKING|>{reasoning}<|END_THINKING|>-// then EITHER content: <|START_TEXT|>{content}<|END_TEXT|>-// OR tool calls: <|START_ACTION|>[-// {"tool_call_id": "0", "tool_name": "f", "parameters": {...}}, ...-// ]<|END_ACTION|>-// <|END_OF_TURN_TOKEN|>-//-// The generation prompt forces a leading <|START_THINKING|> (when reasoning is enabled, which is-// the template default), so the model's output continues from *inside* the thinking block. The-// parser literal therefore only covers the stable <|START_OF_TURN_TOKEN|><|CHATBOT_TOKEN|> prefix-// and the reasoning rule consumes the <|START_THINKING|> ... <|END_THINKING|> markers itself,-// regardless of whether they came from the generation prompt or the generated text.-static common_chat_params common_chat_params_init_cohere2moe(const common_chat_template & tmpl,- const autoparser::generation_params & inputs) {- common_chat_params data;-- const std::string TURN_START = "<|START_OF_TURN_TOKEN|>";- const std::string TURN_END = "<|END_OF_TURN_TOKEN|>";- const std::string CHATBOT = "<|CHATBOT_TOKEN|>";- const std::string USER = "<|USER_TOKEN|>";- const std::string SYSTEM = "<|SYSTEM_TOKEN|>";- const std::string THINK_START = "<|START_THINKING|>";- const std::string THINK_END = "<|END_THINKING|>";- const std::string TEXT_START = "<|START_TEXT|>";- const std::string TEXT_END = "<|END_TEXT|>";- const std::string ACTION_START = "<|START_ACTION|>";- const std::string ACTION_END = "<|END_ACTION|>";- const std::string RESULT_START = "<|START_TOOL_RESULT|>";- const std::string RESULT_END = "<|END_TOOL_RESULT|>";-- // Stable prefix of the generation prompt that precedes the (forced) <|START_THINKING|> marker.- const std::string GEN_PREFIX = TURN_START + CHATBOT;-- data.prompt = common_chat_template_direct_apply_impl(tmpl, inputs);- data.generation_prompt = common_chat_template_generation_prompt_impl(tmpl, inputs);- data.format = COMMON_CHAT_FORMAT_PEG_NATIVE;- data.supports_thinking = true;- data.thinking_start_tag = THINK_START;- data.thinking_end_tags = {THINK_END};- data.preserved_tokens = {- TURN_START, TURN_END, CHATBOT, USER, SYSTEM,- THINK_START, THINK_END,- TEXT_START, TEXT_END,- ACTION_START, ACTION_END,- RESULT_START, RESULT_END,- };-- // Declare per-role message delimiters. Tool results are rendered with the- // system token followed by <|START_TOOL_RESULT|>, so the "tool" delimiter must be listed before- // the plain "system" one (it is a strict superset, and the role split tries delimiters in order).- data.message_delimiters = {- { COMMON_CHAT_ROLE_ASSISTANT, GEN_PREFIX },- { COMMON_CHAT_ROLE_USER, TURN_START + USER },- { COMMON_CHAT_ROLE_TOOL, TURN_START + SYSTEM + RESULT_START },- { COMMON_CHAT_ROLE_SYSTEM, TURN_START + SYSTEM },- };-- auto has_tools = inputs.tools.is_array() && !inputs.tools.empty();- auto has_response_format = inputs.json_schema.is_object() && !inputs.json_schema.empty();- auto extract_reasoning = inputs.reasoning_format != COMMON_REASONING_FORMAT_NONE;- auto include_grammar = has_response_format || (has_tools && inputs.tool_choice != COMMON_CHAT_TOOL_CHOICE_NONE);-- if (inputs.has_continuation()) {- const auto & msg = inputs.continue_msg;-- data.generation_prompt = GEN_PREFIX + THINK_START + msg.reasoning_content;- if (inputs.continue_final_message == COMMON_CHAT_CONTINUATION_CONTENT) {- data.generation_prompt += THINK_END + TEXT_START + msg.render_content();- }-- data.prompt += data.generation_prompt;- }-- auto parser = build_chat_peg_parser([&](common_chat_peg_builder & p) {- auto generation_prompt = p.literal(GEN_PREFIX);- auto end = p.end();-- // The thinking block is always present (the generation prompt forces <|START_THINKING|>).- // When extracting reasoning, capture its body; otherwise keep the whole block (markers- // included) inline as content, matching reasoning_format=NONE conventions.- common_peg_parser reasoning = p.eps();- if (extract_reasoning) {- reasoning = p.optional(p.literal(THINK_START) +- p.reasoning(p.until_one_of({ THINK_END, TEXT_START, ACTION_START })) +- p.optional(p.literal(THINK_END)));- } else {- reasoning = p.optional(p.content(p.literal(THINK_START) +- p.until_one_of({ THINK_END, TEXT_START, ACTION_START }) +- p.optional(p.literal(THINK_END))));- }-- auto text_content = has_response_format- ? p.literal(TEXT_START) +- p.content(p.schema(p.json(), "response-format-schema", inputs.json_schema)) +- p.optional(p.literal(TEXT_END))- : p.literal(TEXT_START) + p.content(p.until(TEXT_END)) + p.optional(p.literal(TEXT_END));-- if (!has_tools || inputs.tool_choice == COMMON_CHAT_TOOL_CHOICE_NONE) {- return generation_prompt + reasoning + text_content + p.optional(p.literal(TURN_END)) + end;- }-- auto require_tools = inputs.tool_choice == COMMON_CHAT_TOOL_CHOICE_REQUIRED;-- // <|START_ACTION|>[ {"tool_call_id": "0", "tool_name": "f", "parameters": {...}}, ... ]<|END_ACTION|>- auto tool_calls = p.standard_json_tools(ACTION_START, ACTION_END, inputs.tools, inputs.parallel_tool_calls,- /* force_tool_calls = */ true,- /* name_key = */ "tool_name",- /* args_key = */ "parameters",- /* array_wrapped = */ true,- /* function_is_key = */ false,- /* call_id_key = */ "",- /* gen_call_id_key = */ "tool_call_id",- /* parameters_order = */ { "tool_call_id", "tool_name", "parameters" });-- // Content and tool calls are mutually exclusive in this format.- common_peg_parser body = require_tools ? tool_calls : p.choice({ tool_calls, text_content });-- return generation_prompt + reasoning + body + p.optional(p.literal(TURN_END)) + end;- });-- data.parser = parser.save();-- if (include_grammar) {- data.grammar_lazy = !has_response_format && inputs.tool_choice == COMMON_CHAT_TOOL_CHOICE_AUTO;- data.grammar = build_grammar([&](const common_grammar_builder & builder) {- foreach_function(inputs.tools, [&](const json & tool) {- const auto & function = tool.at("function");- auto schema = function.at("parameters");- builder.resolve_refs(schema);- });- if (has_response_format) {- auto schema = inputs.json_schema;- builder.resolve_refs(schema);- }- parser.build_grammar(builder, data.grammar_lazy);- });-- data.grammar_triggers = {- { COMMON_GRAMMAR_TRIGGER_TYPE_WORD, ACTION_START }- };- }-- return data;-}--static common_chat_params common_chat_params_init_minimax_m3(const common_chat_template & tmpl,- const autoparser::generation_params & inputs) {- common_chat_params data;-- data.prompt = common_chat_template_direct_apply_impl(tmpl, inputs);- data.generation_prompt = common_chat_template_generation_prompt_impl(tmpl, inputs);- data.format = COMMON_CHAT_FORMAT_PEG_MINIMAX_M3;- data.supports_thinking = true;- data.thinking_start_tag = "<mm:think>";- data.thinking_end_tags = {"</mm:think>"};-- // M3 prefixes every tool tag with the namespace token "]<]minimax[>[";- // params use the parameter name as the tag (<file_path>...</file_path>).- const std::string NS = "]<]minimax[>[";- const std::string THINK_START = "<mm:think>";- const std::string THINK_END = "</mm:think>";- const std::string FC_START = NS + "<tool_call>";- const std::string FC_END = NS + "</tool_call>";- const std::string INVOKE_END = NS + "</invoke>";-- data.preserved_tokens = {- NS,- "<tool_call>",- "</tool_call>",- THINK_START,- THINK_END,- };-- data.message_delimiters = {- { COMMON_CHAT_ROLE_ASSISTANT, "]~b]ai" },- { COMMON_CHAT_ROLE_USER, "]~b]user" },- { COMMON_CHAT_ROLE_TOOL, "]~b]tool" },- { COMMON_CHAT_ROLE_SYSTEM, "]~b]developer" },- { COMMON_CHAT_ROLE_SYSTEM, "]~b]system" },- };-- auto has_tools = inputs.tools.is_array() && !inputs.tools.empty();- auto has_response_format = !inputs.json_schema.is_null() && inputs.json_schema.is_object();- auto extract_reasoning = inputs.reasoning_format != COMMON_REASONING_FORMAT_NONE;- auto include_grammar = has_response_format || (has_tools && inputs.tool_choice != COMMON_CHAT_TOOL_CHOICE_NONE);-- const std::string GEN_PROMPT = data.generation_prompt;-- using mm3 = common_chat_peg_minimax_m3_mapper;-- if (inputs.has_continuation()) {- const auto & msg = inputs.continue_msg;-- data.generation_prompt = GEN_PROMPT + THINK_START + msg.reasoning_content;- if (inputs.continue_final_message == COMMON_CHAT_CONTINUATION_CONTENT) {- data.generation_prompt += THINK_END + msg.render_content();- }-- data.prompt += data.generation_prompt;- }-- auto parser = build_chat_peg_parser([&](common_chat_peg_builder & p) {- auto generation_prompt = p.prefix(GEN_PROMPT, THINK_START);- auto end = p.end();-- auto reasoning = p.eps();- if (extract_reasoning) {- auto block = inputs.enable_thinking- ? p.literal(THINK_START) + p.space() +- p.ac(p.reasoning(p.until(THINK_END)) + p.literal(THINK_END), THINK_END)- : p.literal(THINK_START) + p.ac(p.until(THINK_END) + p.literal(THINK_END), THINK_END);-- // A turn without reasoning is prefixed with a bare </mm:think>, written either by the- // generation prompt (thinking_mode = "disabled") or by the model itself.- reasoning = p.optional(p.choice({ block, p.literal(THINK_END) }));- }-- if (has_response_format) {- auto response_format = p.rule("response-format",- p.literal("```json") + p.space() +- p.content(p.schema(p.json(), "response-format-schema", inputs.json_schema)) +- p.space() + p.literal("```"));- return generation_prompt + reasoning + response_format + end;- }-- if (!has_tools || inputs.tool_choice == COMMON_CHAT_TOOL_CHOICE_NONE) {- return generation_prompt + reasoning + p.content(p.rest()) + end;- }-- auto alternatives_of = [](const json & schema) -> std::optional<json> {- for (const auto * keyword : { "oneOf", "anyOf" }) {- if (schema.contains(keyword) && schema.at(keyword).is_array() && !schema.at(keyword).empty()) {- return schema.at(keyword);- }- }- return std::nullopt;- };-- auto tool_choice = p.choice();- foreach_function(inputs.tools, [&](const json & tool) {- const auto & function = tool.at("function");- std::string name = function.at("name");- auto params = function.contains("parameters") ? function.at("parameters") : json::object();-- auto schema_info = common_schema_info();- schema_info.resolve_refs(params);-- // The template expands argument values recursively in XML (see the to_xml() macro)- std::function<common_peg_parser(const json &, const std::string &, const std::string &)> value_of;- std::function<common_peg_parser(const json &, const std::string &)> members_of;-- auto element_of = [&](const std::string & tag, const json & schema, const std::string & rule_name) {- const std::string close = NS + "</" + tag + ">";- return p.rule(rule_name,- p.tool_arg(- p.tool_arg_open(- p.literal(NS + "<") +- p.tool_arg_name(p.literal(tag)) +- p.literal(">")) +- value_of(schema, rule_name, close)));- };-- value_of = [&](const json & schema,- const std::string & rule_name,- const std::string & close) -> common_peg_parser {- auto close_tag = p.tool_arg_close(p.literal(close));-- // A string accepts anything, so a union with a string alternative is a string- if (schema_info.resolves_to_string(schema)) {- return p.ac(p.tool_arg_string_value(p.until(close)) + close_tag, close);- }-- if (auto alternatives = alternatives_of(schema)) {- std::vector<common_peg_parser> choices;-- size_t index = 0;- for (const auto & alternative : *alternatives) {- const std::string alt_name = rule_name + "-" + std::to_string(index++);-- // There is a risk that this breaks streaming deltas, but that's a risk we- // assume to provide tool arg streaming.- choices.push_back(value_of(alternative, alt_name, close));- }-- return p.choice(choices);- }-- const std::string type = schema.contains("type") && schema.at("type").is_string()- ? schema.at("type").get<std::string>()- : "";-- if (type == "object" && schema.contains("properties")) {- return p.tag(mm3::TOOL_ARG_OBJECT, members_of(schema, rule_name)) + p.space() + close_tag;- }-- if (type == "array" && schema.contains("items")) {- const std::string item_close = NS + "</item>";- auto item = p.rule(rule_name + "-item",- p.tag(mm3::TOOL_ARG_ITEM,- p.literal(NS + "<item>") +- value_of(schema.at("items"), rule_name + "-item", item_close)));- return p.tag(mm3::TOOL_ARG_ARRAY, p.repeat(p.space() + item, 0, -1)) + p.space() + close_tag;- }-- return p.tool_arg_json_value(p.schema(p.json(), rule_name + "-schema", schema, false)) + close_tag;- };-- // Required properties in schema order, then any number of optional ones in any order.- members_of = [&](const json & schema, const std::string & rule_prefix) -> common_peg_parser {- const auto & props = schema.at("properties");-- std::set<std::string> required;- if (schema.contains("required")) {- required = schema.at("required").get<std::set<std::string>>();- }-- std::vector<common_peg_parser> required_elements;- std::vector<common_peg_parser> optional_elements;- for (const auto & [key, key_schema] : props.items()) {- auto element = element_of(key, key_schema, rule_prefix + "-" + key);- if (required.find(key) != required.end()) {- required_elements.push_back(element);- } else {- optional_elements.push_back(element);- }- }-- common_peg_parser members = p.eps();- for (size_t i = 0; i < required_elements.size(); i++) {- if (i > 0) {- members = members + p.space();- }- members = members + required_elements[i];- }-- if (!optional_elements.empty()) {- common_peg_parser any_optional = p.choice();- for (const auto & element : optional_elements) {- any_optional |= element;- }- members = members + p.repeat(p.space() + any_optional, 0, -1);- }-- return members;- };-- common_peg_parser invoke_body =- params.contains("properties") ? members_of(params, "tool-" + name + "-arg") : p.eps();-- auto func_parser = p.tool(- p.tool_open(p.literal(NS + "<invoke name=\"") +- p.tool_name(p.literal(name)) + p.literal("\">")) +- p.space() + invoke_body + p.space() +- p.tool_close(p.literal(INVOKE_END)));-- tool_choice |= p.rule("tool-" + name, func_parser);- });-- auto require_tools = inputs.tool_choice == COMMON_CHAT_TOOL_CHOICE_REQUIRED;-- common_peg_parser tool_calls = p.eps();- if (inputs.parallel_tool_calls) {- tool_calls = p.trigger_rule("tool-call",- p.literal(FC_START) + p.space() + tool_choice +- p.zero_or_more(p.space() + tool_choice) + p.space() + p.literal(FC_END));- } else {- tool_calls = p.trigger_rule("tool-call",- p.literal(FC_START) + p.space() + tool_choice + p.space() + p.literal(FC_END));- }-- if (!require_tools) {- tool_calls = p.optional(tool_calls);- }-- auto content_before_tools = p.content(p.until(FC_START));- return generation_prompt + reasoning + content_before_tools + tool_calls + end;- });-- data.parser = parser.save();-- if (include_grammar) {- data.grammar_lazy = !(has_response_format || (has_tools && inputs.tool_choice == COMMON_CHAT_TOOL_CHOICE_REQUIRED));- data.grammar = build_grammar([&](const common_grammar_builder & builder) {- foreach_function(inputs.tools, [&](const json & tool) {- const auto & function = tool.at("function");- auto schema = function.contains("parameters") ? function.at("parameters") : json::object();- builder.resolve_refs(schema);- });- if (has_response_format) {- auto schema = inputs.json_schema;- builder.resolve_refs(schema);- }- parser.build_grammar(builder, data.grammar_lazy);- });-- data.grammar_triggers = {- { COMMON_GRAMMAR_TRIGGER_TYPE_WORD, FC_START },- };- }-- return data;-}--namespace workaround {--static void map_developer_role_to_system(json & messages) {- for (auto & message : messages) {- if (message.contains("role")) {- if (message["role"] == "developer") {- message["role"] = "system";- }- }- }-}---// if first message is system and template does not support it, merge it with next message-static void system_message_not_supported(json & messages) {- if (!messages.empty() && messages.front().at("role") == "system") {- if (messages.size() > 1) {- LOG_DBG("Merging system prompt into next message\n");- auto & first_msg = messages.front();- auto & second_msg = messages[1];- second_msg["content"] = first_msg.at("content").get<std::string>()- + "\n" + second_msg.at("content").get<std::string>();- messages.erase(0);- } else {- LOG_WRN("Removing system prompt due to template not supporting system role\n");- messages.erase(0);- }- }-}--static void requires_non_null_content(json & messages) {- GGML_ASSERT(messages.is_array());- for (auto & message : messages) {- if (message.contains("tool_calls") && !message.contains("content")) {- message["content"] = "";- }- }-}--// Gemma4 uses a custom tool_responses field instead of role:tool messages.-//-// This will transform a sequence of messages:-// assistant(tool_call+) -> tool+ -> assistant(content)-//-// Into a single assistant message containing a tool_responses field:-// assistant(content + tool_call + tool_responses)-//-// This is necessary for the Gemma4 chat template to properly format the prompt.-// See https://ai.google.dev/gemma/docs/core/prompt-formatting-gemma4-struct gemma4_model_turn_builder {- json & messages;- size_t pos;- json tool_calls = json::array();- json tool_responses = json::array();- json content;- json reasoning_content;-- gemma4_model_turn_builder(json & msgs, size_t pos) : messages(msgs), pos(pos) {}-- void collect() {- // Collect the first assistant message- auto & msg = messages[pos];- if (msg.contains("reasoning_content") && msg.at("reasoning_content").is_string()) {- // According to the prompt formatting guide, we need to preserve reasoning_content- // between function calls. The current chat templates do not support this, but we will do it anyway.- reasoning_content = msg.at("reasoning_content");- }- for (auto & tc : msg.at("tool_calls")) {- tool_calls.push_back(tc);- }- pos++;-- // Collect tool call results- while (pos < messages.size() && messages[pos].value("role", "") == "tool") {- collect_result(messages[pos]);- pos++;- }-- // Check if the next assistant message is the final message- if (pos < messages.size() && messages[pos].value("role", "") == "assistant") {- auto & next = messages[pos];- if (!has_tool_calls(next) && has_content(next)) {- content = next.at("content");- pos++;- }- }- }-- void collect_result(const json & curr) {- json response;- if (curr.contains("content")) {- const auto & content = curr.at("content");- if (content.is_string()) {- // Try to parse the content as JSON; fall back to raw string- try {- response = json::parse(content.get<std::string>());- } catch (...) {- response = content;- }- } else {- response = content;- }- }-- std::string name;-- // Match name with corresponding tool call- size_t idx = tool_responses.size();- if (idx < tool_calls.size()) {- auto & tc = tool_calls[idx];- if (tc.contains("function")) {- name = tc.at("function").value("name", "");- }- }-- // Fallback to the tool call id- if (name.empty()) {- name = curr.value("tool_call_id", "");- }-- tool_responses.push_back({{"name", name}, {"response", response}});- }-- json build() {- collect();-- json msg = {- {"role", "assistant"},- {"tool_calls", tool_calls},- };- if (!tool_responses.empty()) {- msg["tool_responses"] = tool_responses;- }- if (!content.is_null()) {- msg["content"] = content;- }- if (!reasoning_content.is_null()) {- msg["reasoning_content"] = reasoning_content;- }- return msg;- }-- static bool has_content(const json & msg) {- if (!msg.contains("content") || msg.at("content").is_null()) {- return false;- }- const auto & content = msg.at("content");- if (content.is_string() && !content.get<std::string>().empty()) {- return true;- }- if (content.is_array() && !content.empty()) {- return true;- }- return false;- }-- static bool has_tool_calls(const json & msg) {- return msg.contains("tool_calls") && msg.at("tool_calls").is_array() && !msg.at("tool_calls").empty();- }-};--static void convert_tool_responses_gemma4(json & messages) {- json result = json::array();- size_t i = 0;-- while (i < messages.size()) {- auto & msg = messages[i];-- if (msg.value("role", "") != "assistant" || !msg.contains("tool_calls") ||- !msg.at("tool_calls").is_array() || msg.at("tool_calls").empty()) {- result.push_back(msg);- i++;- continue;- }-- gemma4_model_turn_builder builder(messages, i);- result.push_back(builder.build());- i = builder.pos;- }-- messages = result;-}--static void func_args_not_string(json & messages) {- GGML_ASSERT(messages.is_array());- for (auto & message : messages) {- if (message.contains("tool_calls")) {- for (auto & tool_call : message["tool_calls"]) {- if (tool_call.contains("function") && tool_call["function"].contains("arguments")) {- auto & args = tool_call["function"]["arguments"];- if (args.is_string()) {- try {- args = json::parse(args.get<std::string>());- } catch (const std::exception & e) {- throw std::runtime_error("Failed to parse tool call arguments as JSON: " + std::string(e.what()));- }- }- }- }- }- }-}--// Trim leading/trailing whitespace from message contents before rendering. This-// has to run on the messages (not on the rendered JSON) because templates with-// string-only content caps concatenate typed content parts into a single string-// during rendering, after which the per-part whitespace can no longer be reached.-// Both the plain string content and the text of typed content parts are trimmed.-static void trim_all_content(std::vector<common_chat_msg> & messages) {- for (auto & message : messages) {- message.content = trim_whitespace(message.content);- message.reasoning_content = trim_whitespace(message.reasoning_content);- for (auto & part : message.content_parts) {- if (part.type == "text") {- part.text = trim_whitespace(part.text);- }- }- }-}--}--// MiniCPM5 format:-// - Reasoning: <think>{reasoning}</think> (optional)-// - Tool calls: <function name="foo"><param name="bar">value</param></function>-static common_chat_params common_chat_params_init_minicpm5(const common_chat_template & tmpl,- const autoparser::generation_params & inputs) {- common_chat_params data;-- data.prompt = common_chat_template_direct_apply_impl(tmpl, inputs);- data.generation_prompt = common_chat_template_generation_prompt_impl(tmpl, inputs);- data.format = COMMON_CHAT_FORMAT_PEG_NATIVE;- data.supports_thinking = true;- data.preserved_tokens = {- "<function",- "<param",- "</function>",- "</param>",- "<think>",- "</think>",- };-- data.thinking_start_tag = "<think>";- data.thinking_end_tags = {"</think>"};-- data.message_delimiters = {- { COMMON_CHAT_ROLE_ASSISTANT, "<|im_start|>assistant" },- { COMMON_CHAT_ROLE_TOOL, "<|im_start|>user\n<tool_response>" },- { COMMON_CHAT_ROLE_USER, "<|im_start|>user" },- { COMMON_CHAT_ROLE_SYSTEM, "<|im_start|>system" },- };-- auto has_tools = inputs.tools.is_array() && !inputs.tools.empty();- auto has_response_format = inputs.json_schema.is_object() && !inputs.json_schema.empty();- auto extract_reasoning = inputs.reasoning_format != COMMON_REASONING_FORMAT_NONE;- auto include_grammar = has_response_format || (has_tools && inputs.tool_choice != COMMON_CHAT_TOOL_CHOICE_NONE);-- if (inputs.has_continuation()) {- const auto & msg = inputs.continue_msg;-- data.generation_prompt = "<|im_start|>assistant\n<think>\n" + msg.reasoning_content;- if (inputs.continue_final_message == COMMON_CHAT_CONTINUATION_CONTENT) {- data.generation_prompt += "\n</think>\n\n" + msg.render_content();- }-- data.prompt += data.generation_prompt;- }-- auto parser = build_chat_peg_parser([&](common_chat_peg_builder & p) {- auto generation_prompt = p.literal("<|im_start|>assistant\n");-- auto reasoning = p.eps();- if (extract_reasoning) {- reasoning = ("<think>" << p.reasoning(p.until("</think>")) << "</think>") + p.space();- }-- // Response format parser- if (has_response_format) {- return generation_prompt + reasoning + p.content(p.schema(p.json(), "response-format", inputs.json_schema));- }-- if (has_tools && inputs.tool_choice != COMMON_CHAT_TOOL_CHOICE_NONE) {- // CDATA lets a value carry characters that would otherwise close the tag (e.g.- // </param>); capture the inner text only, excluding the CDATA markers.- auto string_value = p.choice({- p.literal("<![CDATA[") + p.ac(p.tool_arg_string_value(p.until("]]>")) + p.literal("]]>"), "]]>") + p.tool_arg_close(p.literal("</param>")),- p.negate(p.literal("< {- const auto & function = tool.at("function");- const std::string name = function.at("name");- auto params = function.contains("parameters") ? function.at("parameters") : json::object();-- auto args = p.eps();- if (params.contains("properties") && params.at("properties").is_object() && !params.at("properties").empty()) {- auto schema_info = common_schema_info();- schema_info.resolve_refs(params);-- auto arg_choice = p.choice();- for (const auto & [prop_name, prop_schema] : params.at("properties").items()) {- auto value_parser = p.eps();- if (schema_info.resolves_to_string(prop_schema)) {- value_parser = string_value;- } else {- value_parser = p.tool_arg_json_value(- p.schema(p.json(), "tool-" + name + "-arg-" + prop_name + "-schema", prop_schema, false)- ) + p.tool_arg_close(p.literal("</param>"));- }-- auto arg_rule = p.tool_arg(- p.tool_arg_open(p.literal("<param name=\"") + p.tool_arg_name(p.literal(prop_name)) + p.literal("\">")) +- value_parser- );-- arg_choice |= arg_rule;- }- args = p.zero_or_more(arg_choice + p.space());- }-- auto tool_parser = p.tool(- p.tool_open(p.literal("<function name=\"") + p.tool_name(p.literal(name)) + p.literal("\">"))- << p.tool_args(args)- << p.tool_close(p.literal("</function>")));-- tool_choice |= p.rule("tool-" + name, tool_parser);- });-- auto max_calls = inputs.parallel_tool_calls ? -1 : 1;- auto tool_calls = p.trigger_rule("tool-call", p.repeat(tool_choice + p.space(), 1, max_calls));-- auto content = p.content(p.until("<function"));-- return generation_prompt + reasoning + content + tool_calls + p.end();- }-- return generation_prompt + reasoning + p.content(p.rest()) + p.end();- });-- data.parser = parser.save();-- if (include_grammar) {- data.grammar_lazy = !(has_response_format || (has_tools && inputs.tool_choice == COMMON_CHAT_TOOL_CHOICE_REQUIRED));- data.grammar = build_grammar([&](const common_grammar_builder & builder) {- foreach_function(inputs.tools, [&](const json & tool) {- const auto & function = tool.at("function");- auto schema = function.contains("parameters") ? function.at("parameters") : json::object();- builder.resolve_refs(schema);- });- if (has_response_format) {- auto schema = inputs.json_schema;- builder.resolve_refs(schema);- }- parser.build_grammar(builder, data.grammar_lazy);- });-- data.grammar_triggers = {- { COMMON_GRAMMAR_TRIGGER_TYPE_WORD, "<function" },- };- }-- return data;-}--// An assistant turn is rendered as one or more messages, each-// "<|start|>assistant to=<recipient><|message|>{content}{END}" where END is-// <|eom|> (more messages follow) or <|eot|> (end of turn):-// - chain-of-thought: to=self, terminated by <|eom|>-// - final answer: to=user, terminated by <|eot|>-// The generation prompt is just "<|start|>assistant"; the model emits its own-// " to=...<|message|>".-static common_chat_params common_chat_params_init_muse_glimmer(const common_chat_template & tmpl,- const autoparser::generation_params & inputs) {- common_chat_params data;-- data.prompt = common_chat_template_direct_apply_impl(tmpl, inputs);- data.generation_prompt = "<|start|>assistant";- data.format = COMMON_CHAT_FORMAT_PEG_NATIVE;- data.supports_thinking = true;-- data.preserved_tokens = {- "<|start|>", "<|message|>", "<|eom|>", "<|eot|>",- // ATEM tool-call markup emitted on " to=<tool>" turns.- "<atem:function_calls>", "<atem:invoke", "<atem:parameter", "</atem:parameter>",- "</atem:invoke>", "</atem:function_calls>",- };-- data.message_delimiters = {- { COMMON_CHAT_ROLE_ASSISTANT, "<|start|>assistant" },- { COMMON_CHAT_ROLE_USER, "<|start|>user" },- { COMMON_CHAT_ROLE_SYSTEM, "<|start|>system" },- { COMMON_CHAT_ROLE_TOOL, "<|start|>tool" },- };-- if (inputs.has_continuation()) {- const auto & msg = inputs.continue_msg;-- data.generation_prompt = "<|start|>assistant to=self<|message|>" + msg.reasoning_content;- if (inputs.continue_final_message == COMMON_CHAT_CONTINUATION_CONTENT) {- data.generation_prompt += "<|eom|><|start|>assistant to=user<|message|>" + msg.render_content();- }-- data.prompt += data.generation_prompt;- }-- auto extract_reasoning = inputs.reasoning_format != COMMON_REASONING_FORMAT_NONE;-- auto has_tools = inputs.tools.is_array() && !inputs.tools.empty();- // Constrained grammar whenever tools are offered.- auto include_grammar = has_tools && inputs.tool_choice != COMMON_CHAT_TOOL_CHOICE_NONE;-- auto parser = build_chat_peg_parser([&](common_chat_peg_builder & p) {- auto start = p.rule("start", p.literal("<|start|>assistant"));-- if (!extract_reasoning && !include_grammar) {- return start + p.content(p.rest());- }-- if (extract_reasoning) {- p.rule("analysis", p.literal(" to=self<|message|>") + p.reasoning(p.until("<|eom|>")) + p.literal("<|eom|>"));- } else {- p.rule("analysis", p.literal(" to=self<|message|>") + p.content(p.until("<|eom|>")) + p.literal("<|eom|>"));- }- auto analysis = p.ref("analysis");-- auto recipient = p.optional(p.literal(" to=user"));- auto final_msg = p.rule("final", recipient + p.literal("<|message|>") +- p.content(p.until_one_of({ "<|eot|>", "<|eom|>" })));-- if (has_tools && inputs.tool_choice != COMMON_CHAT_TOOL_CHOICE_NONE) {- auto string_value = p.ac(- p.tool_arg_string_value(p.until("</atem:parameter>")) + p.tool_arg_close(p.literal("</atem:parameter>")),- "</atem:parameter>");-- auto tool_choice = p.choice();- foreach_function(inputs.tools, [&](const json & tool) {- const auto & function = tool.at("function");- const std::string name = function.at("name");- auto params = function.contains("parameters") ? function.at("parameters") : json::object();-- auto args = p.eps();- if (params.contains("properties") && params.at("properties").is_object() && !params.at("properties").empty()) {- auto schema_info = common_schema_info();- schema_info.resolve_refs(params);-- auto arg_choice = p.choice();- for (const auto & [prop_name, prop_schema] : params.at("properties").items()) {- auto value_parser = p.eps();- if (schema_info.resolves_to_string(prop_schema)) {- value_parser = string_value;- } else {- value_parser = p.tool_arg_json_value(- p.schema(p.json(), "tool-" + name + "-arg-" + prop_name + "-schema", prop_schema, false))- + p.tool_arg_close(p.literal("</atem:parameter>"));- }-- auto arg_rule = p.tool_arg(- p.tool_arg_open(p.literal("<atem:parameter name=\"") + p.tool_arg_name(p.literal(prop_name)) + p.literal("\">")) +- value_parser);-- arg_choice |= arg_rule;- }- args = p.zero_or_more(arg_choice + p.space());- }-- auto tool_parser = p.tool(- p.tool_open(p.literal(" to=") + p.until("<|message|>") +- p.literal("<|message|><atem:function_calls>") + p.space() +- p.literal("<atem:invoke name=\"") + p.tool_name(p.literal(name)) + p.literal("\">") + p.space())- << p.tool_args(args)- << p.tool_close(p.literal("</atem:invoke>") + p.space() + p.literal("</atem:function_calls>")));-- tool_choice |= p.rule("tool-" + name, tool_parser);- });-- auto tool_calls = inputs.parallel_tool_calls- ? p.trigger_rule("tool-call", tool_choice + p.zero_or_more(p.literal("<|eom|>") + start + tool_choice))- : p.trigger_rule("tool-call", tool_choice);--- if (inputs.tool_choice == COMMON_CHAT_TOOL_CHOICE_REQUIRED) {- return p.zero_or_more(start + analysis) + start + tool_calls;- }- auto trailing_calls = p.optional(p.literal("<|eom|>") + start + tool_calls);- return p.zero_or_more(start + analysis) + start + (tool_calls | (final_msg + trailing_calls));- }-- return p.zero_or_more(start + analysis) + start + final_msg;- });-- data.parser = parser.save();-- if (include_grammar) {- data.grammar_lazy = inputs.tool_choice != COMMON_CHAT_TOOL_CHOICE_REQUIRED;- data.grammar = build_grammar([&](const common_grammar_builder & builder) {- foreach_function(inputs.tools, [&](const json & tool) {- const auto & function = tool.at("function");- auto schema = function.contains("parameters") ? function.at("parameters") : json::object();- builder.resolve_refs(schema);- });- parser.build_grammar(builder, data.grammar_lazy);- });- data.grammar_triggers = {- { COMMON_GRAMMAR_TRIGGER_TYPE_PATTERN,- "<\\|start\\|>assistant( to=(?!self<\\|message\\|>)(?!user<\\|message\\|>)[^<]*?<\\|message\\|>)" },- };- }-- return data;}static json common_chat_extra_context() {diff --git a/common/parsers/cohere2moe.cpp b/common/parsers/cohere2moe.cppnew file mode 100644index 0000000000..46a2a01baa--- /dev/null+++ b/common/parsers/cohere2moe.cpp@@ -0,0 +1,150 @@+#include "parsers.h"++// Cohere2 MoE (a.k.a. "North Code") parser.+//+// The assistant turn is fully marker-wrapped:+// <|START_OF_TURN_TOKEN|><|CHATBOT_TOKEN|>+// <|START_THINKING|>{reasoning}<|END_THINKING|>+// then EITHER content: <|START_TEXT|>{content}<|END_TEXT|>+// OR tool calls: <|START_ACTION|>[+// {"tool_call_id": "0", "tool_name": "f", "parameters": {...}}, ...+// ]<|END_ACTION|>+// <|END_OF_TURN_TOKEN|>+//+// The generation prompt forces a leading <|START_THINKING|> (when reasoning is enabled, which is+// the template default), so the model's output continues from *inside* the thinking block. The+// parser literal therefore only covers the stable <|START_OF_TURN_TOKEN|><|CHATBOT_TOKEN|> prefix+// and the reasoning rule consumes the <|START_THINKING|> ... <|END_THINKING|> markers itself,+// regardless of whether they came from the generation prompt or the generated text.+common_chat_params common_chat_params_init_cohere2moe(const common_chat_template & tmpl,+ const autoparser::generation_params & inputs) {+ common_chat_params data;++ const std::string TURN_START = "<|START_OF_TURN_TOKEN|>";+ const std::string TURN_END = "<|END_OF_TURN_TOKEN|>";+ const std::string CHATBOT = "<|CHATBOT_TOKEN|>";+ const std::string USER = "<|USER_TOKEN|>";+ const std::string SYSTEM = "<|SYSTEM_TOKEN|>";+ const std::string THINK_START = "<|START_THINKING|>";+ const std::string THINK_END = "<|END_THINKING|>";+ const std::string TEXT_START = "<|START_TEXT|>";+ const std::string TEXT_END = "<|END_TEXT|>";+ const std::string ACTION_START = "<|START_ACTION|>";+ const std::string ACTION_END = "<|END_ACTION|>";+ const std::string RESULT_START = "<|START_TOOL_RESULT|>";+ const std::string RESULT_END = "<|END_TOOL_RESULT|>";++ // Stable prefix of the generation prompt that precedes the (forced) <|START_THINKING|> marker.+ const std::string GEN_PREFIX = TURN_START + CHATBOT;++ data.prompt = common_chat_template_direct_apply_impl(tmpl, inputs);+ data.generation_prompt = common_chat_template_generation_prompt_impl(tmpl, inputs);+ data.format = COMMON_CHAT_FORMAT_PEG_NATIVE;+ data.supports_thinking = true;+ data.thinking_start_tag = THINK_START;+ data.thinking_end_tags = {THINK_END};+ data.preserved_tokens = {+ TURN_START, TURN_END, CHATBOT, USER, SYSTEM,+ THINK_START, THINK_END,+ TEXT_START, TEXT_END,+ ACTION_START, ACTION_END,+ RESULT_START, RESULT_END,+ };++ // Declare per-role message delimiters. Tool results are rendered with the+ // system token followed by <|START_TOOL_RESULT|>, so the "tool" delimiter must be listed before+ // the plain "system" one (it is a strict superset, and the role split tries delimiters in order).+ data.message_delimiters = {+ { COMMON_CHAT_ROLE_ASSISTANT, GEN_PREFIX },+ { COMMON_CHAT_ROLE_USER, TURN_START + USER },+ { COMMON_CHAT_ROLE_TOOL, TURN_START + SYSTEM + RESULT_START },+ { COMMON_CHAT_ROLE_SYSTEM, TURN_START + SYSTEM },+ };++ auto has_tools = inputs.tools.is_array() && !inputs.tools.empty();+ auto has_response_format = inputs.json_schema.is_object() && !inputs.json_schema.empty();+ auto extract_reasoning = inputs.reasoning_format != COMMON_REASONING_FORMAT_NONE;+ auto include_grammar = has_response_format || (has_tools && inputs.tool_choice != COMMON_CHAT_TOOL_CHOICE_NONE);++ if (inputs.has_continuation()) {+ const auto & msg = inputs.continue_msg;++ data.generation_prompt = GEN_PREFIX + THINK_START + msg.reasoning_content;+ if (inputs.continue_final_message == COMMON_CHAT_CONTINUATION_CONTENT) {+ data.generation_prompt += THINK_END + TEXT_START + msg.render_content();+ }++ data.prompt += data.generation_prompt;+ }++ auto parser = build_chat_peg_parser([&](common_chat_peg_builder & p) {+ auto generation_prompt = p.literal(GEN_PREFIX);+ auto end = p.end();++ // The thinking block is always present (the generation prompt forces <|START_THINKING|>).+ // When extracting reasoning, capture its body; otherwise keep the whole block (markers+ // included) inline as content, matching reasoning_format=NONE conventions.+ common_peg_parser reasoning = p.eps();+ if (extract_reasoning) {+ reasoning = p.optional(p.literal(THINK_START) ++ p.reasoning(p.until_one_of({ THINK_END, TEXT_START, ACTION_START })) ++ p.optional(p.literal(THINK_END)));+ } else {+ reasoning = p.optional(p.content(p.literal(THINK_START) ++ p.until_one_of({ THINK_END, TEXT_START, ACTION_START }) ++ p.optional(p.literal(THINK_END))));+ }++ auto text_content = has_response_format+ ? p.literal(TEXT_START) ++ p.content(p.schema(p.json(), "response-format-schema", inputs.json_schema)) ++ p.optional(p.literal(TEXT_END))+ : p.literal(TEXT_START) + p.content(p.until(TEXT_END)) + p.optional(p.literal(TEXT_END));++ if (!has_tools || inputs.tool_choice == COMMON_CHAT_TOOL_CHOICE_NONE) {+ return generation_prompt + reasoning + text_content + p.optional(p.literal(TURN_END)) + end;+ }++ auto require_tools = inputs.tool_choice == COMMON_CHAT_TOOL_CHOICE_REQUIRED;++ // <|START_ACTION|>[ {"tool_call_id": "0", "tool_name": "f", "parameters": {...}}, ... ]<|END_ACTION|>+ auto tool_calls = p.standard_json_tools(ACTION_START, ACTION_END, inputs.tools, inputs.parallel_tool_calls,+ /* force_tool_calls = */ true,+ /* name_key = */ "tool_name",+ /* args_key = */ "parameters",+ /* array_wrapped = */ true,+ /* function_is_key = */ false,+ /* call_id_key = */ "",+ /* gen_call_id_key = */ "tool_call_id",+ /* parameters_order = */ { "tool_call_id", "tool_name", "parameters" });++ // Content and tool calls are mutually exclusive in this format.+ common_peg_parser body = require_tools ? tool_calls : p.choice({ tool_calls, text_content });++ return generation_prompt + reasoning + body + p.optional(p.literal(TURN_END)) + end;+ });++ data.parser = parser.save();++ if (include_grammar) {+ data.grammar_lazy = !has_response_format && inputs.tool_choice == COMMON_CHAT_TOOL_CHOICE_AUTO;+ data.grammar = build_grammar([&](const common_grammar_builder & builder) {+ foreach_function(inputs.tools, [&](const json & tool) {+ const auto & function = tool.at("function");+ auto schema = function.at("parameters");+ builder.resolve_refs(schema);+ });+ if (has_response_format) {+ auto schema = inputs.json_schema;+ builder.resolve_refs(schema);+ }+ parser.build_grammar(builder, data.grammar_lazy);+ });++ data.grammar_triggers = {+ { COMMON_GRAMMAR_TRIGGER_TYPE_WORD, ACTION_START }+ };+ }++ return data;+}diff --git a/common/parsers/deepseek.cpp b/common/parsers/deepseek.cppnew file mode 100644index 0000000000..5e25817272--- /dev/null+++ b/common/parsers/deepseek.cpp@@ -0,0 +1,287 @@+#include "parsers.h"++// The DeepSeek V4 reference implementation renders consecutive tool results into a single+// user block, ordered by the tool call order of the preceding assistant message (matched+// by tool call id) rather than by the order they appear in the conversation.+static json deepseek_v4_sort_tool_results(const json & messages) {+ json adjusted = messages;+ std::map<std::string, size_t> call_order;++ for (size_t i = 0; i < adjusted.size();) {+ const auto & msg = adjusted[i];+ const auto role = msg.value("role", "");++ if (role == "assistant" && msg.contains("tool_calls") &&+ msg.at("tool_calls").is_array() && !msg.at("tool_calls").empty()) {+ call_order.clear();+ const auto & tool_calls = msg.at("tool_calls");+ for (size_t idx = 0; idx < tool_calls.size(); idx++) {+ auto id = tool_calls[idx].value("id", "");+ if (!id.empty()) {+ call_order[id] = idx;+ }+ }+ i++;+ continue;+ }++ if (role != "user" && role != "tool") {+ i++;+ continue;+ }++ // collect a maximal run of user/tool messages - they render into one user block+ std::vector<size_t> tool_positions;+ size_t run_end = i;+ for (; run_end < adjusted.size(); run_end++) {+ const auto r = adjusted[run_end].value("role", "");+ if (r == "tool") {+ tool_positions.push_back(run_end);+ } else if (r != "user") {+ break;+ }+ }++ if (tool_positions.size() > 1 && !call_order.empty()) {+ std::vector<json> results;+ results.reserve(tool_positions.size());+ for (auto pos : tool_positions) {+ results.push_back(adjusted[pos]);+ }+ std::stable_sort(results.begin(), results.end(), [&](const json & a, const json & b) {+ const auto order = [&](const json & m) {+ auto it = call_order.find(m.value("tool_call_id", ""));+ return it == call_order.end() ? (size_t) 0 : it->second;+ };+ return order(a) < order(b);+ });+ for (size_t k = 0; k < tool_positions.size(); k++) {+ adjusted[tool_positions[k]] = std::move(results[k]);+ }+ }++ i = run_end;+ }++ return adjusted;+}++common_chat_params common_chat_params_init_deepseek_v3_2(const common_chat_template & tmpl,+ const autoparser::generation_params & inputs) {+ common_chat_params data;++ // V4 uses the same DSML markup as V3.2, but names the tool call block "tool_calls"+ // instead of "function_calls", renders tool results in tool call order and its+ // non-thinking generation prompt ends with a bare </think> instead of an empty+ // <think></think> pair.+ const bool is_v4 = tmpl.source().find("function_calls") == std::string::npos;++ std::optional<json> adjusted_messages;+ if (is_v4) {+ adjusted_messages = deepseek_v4_sort_tool_results(inputs.messages);+ }++ auto has_tools = inputs.tools.is_array() && !inputs.tools.empty();+ auto has_response_format = !inputs.json_schema.is_null() && inputs.json_schema.is_object();+ auto extract_reasoning = inputs.reasoning_format != COMMON_REASONING_FORMAT_NONE;+ auto include_grammar = has_response_format || (has_tools && inputs.tool_choice != COMMON_CHAT_TOOL_CHOICE_NONE);++ std::optional<json> additional_context;+ if (is_v4 && has_response_format) {+ additional_context = json{ { "response_format", inputs.json_schema } };+ }++ const std::string DSML = "|DSML|";+ const std::string THINK_START = "<think>";+ const std::string THINK_END = "</think>";+ const std::string TC_BLOCK = is_v4 ? "tool_calls" : "function_calls";+ const std::string FC_START = "<" + DSML + TC_BLOCK + ">";+ const std::string FC_END = "</" + DSML + TC_BLOCK + ">";+ const std::string INVOKE_START = "<" + DSML + "invoke";+ const std::string INVOKE_END = "</" + DSML + "invoke>";+ const std::string PARAM_START = "<" + DSML + "parameter";+ const std::string PARAM_END = "</" + DSML + "parameter>";+ const std::string GEN_PROMPT = "<|Assistant|>";+ const std::string TC_SEPARATOR = "\n\n";++ data.prompt = common_chat_template_direct_apply_impl(+ tmpl, inputs, adjusted_messages, std::nullopt, additional_context);+ data.generation_prompt = common_chat_template_generation_prompt_impl(+ tmpl, inputs, adjusted_messages, std::nullopt, additional_context);+ data.format = COMMON_CHAT_FORMAT_PEG_NATIVE;+ data.supports_thinking = true;+ data.thinking_start_tag = THINK_START;+ data.thinking_end_tags = {THINK_END, FC_START};+ data.preserved_tokens = {+ DSML,+ THINK_START,+ THINK_END,+ };++ if (inputs.has_continuation()) {+ const auto & msg = inputs.continue_msg;++ if (is_v4 && msg.reasoning_content.empty()) {+ data.generation_prompt = GEN_PROMPT + THINK_END;+ if (inputs.continue_final_message == COMMON_CHAT_CONTINUATION_CONTENT) {+ data.generation_prompt += msg.render_content();+ }+ } else {+ data.generation_prompt = GEN_PROMPT + THINK_START + msg.reasoning_content;+ if (inputs.continue_final_message == COMMON_CHAT_CONTINUATION_CONTENT) {+ data.generation_prompt += THINK_END + msg.render_content();+ }+ }++ data.prompt += data.generation_prompt;+ }++ bool require_tools = inputs.tool_choice == COMMON_CHAT_TOOL_CHOICE_REQUIRED;+ bool has_tool_calls = has_tools && inputs.tool_choice != COMMON_CHAT_TOOL_CHOICE_NONE;++ auto parser = build_chat_peg_parser([&](common_chat_peg_builder & p) {+ auto generation_prompt = p.literal(GEN_PROMPT);+ auto end = p.end();++ // build tool call section first since we might need it in reasoning+ auto tool_choice = p.choice();+ if (has_tool_calls) {+ foreach_function(inputs.tools, [&](const json & tool) {+ const auto & function = tool.at("function");+ std::string name = function.at("name");+ auto params = function.contains("parameters") ? function.at("parameters") : json::object();+ const auto & props = params.contains("properties") ? params.at("properties") : json::object();++ std::set<std::string> required;+ if (params.contains("required")) {+ required = params.at("required").get<std::set<std::string>>();+ }++ auto schema_info = common_schema_info();+ schema_info.resolve_refs(params);++ std::vector<common_peg_parser> required_parsers;+ std::vector<common_peg_parser> optional_parsers;+ for (const auto & [param_name, param_schema] : props.items()) {+ bool is_required = required.find(param_name) != required.end();+ bool is_string = schema_info.resolves_to_string(param_schema);++ auto arg = p.tool_arg(+ p.tool_arg_open(p.literal(PARAM_START + " name=\"") + p.tool_arg_name(p.literal(param_name)) ++ p.literal("\" string=\"" + std::string(is_string ? "true" : "false") + "\">")) ++ (is_string ?+ p.tool_arg_string_value(p.until(PARAM_END)) :+ p.tool_arg_json_value(p.schema(p.json(), "tool-" + name + "-arg-" + param_name + "-schema",+ param_schema, false))) ++ p.tool_arg_close(p.literal(PARAM_END)));++ auto named_arg = p.rule("tool-" + name + "-arg-" + param_name, arg);+ if (is_required) {+ required_parsers.push_back(named_arg);+ } else {+ optional_parsers.push_back(named_arg);+ }+ }++ common_peg_parser args_seq = p.eps();+ for (size_t i = 0; i < required_parsers.size(); i++) {+ if (i > 0) {+ args_seq = args_seq + p.space();+ }+ args_seq = args_seq + required_parsers[i];+ }++ if (!optional_parsers.empty()) {+ common_peg_parser any_opt = p.choice();+ for (const auto & opt : optional_parsers) {+ any_opt |= opt;+ }+ args_seq = args_seq + p.repeat(p.space() + any_opt, 0, -1);+ }++ common_peg_parser invoke_body = args_seq;+ auto func_parser = p.tool(p.tool_open(p.literal(INVOKE_START + " name=\"") ++ p.tool_name(p.literal(name)) + p.literal("\">\n")) ++ invoke_body + p.space() + p.tool_close(p.literal(INVOKE_END)));++ tool_choice |= p.rule("tool-" + name, func_parser);+ });+ }++ common_peg_parser tool_calls = p.eps();+ if (inputs.parallel_tool_calls) {+ tool_calls = p.trigger_rule("tool-call",+ p.literal(FC_START) + p.space() + tool_choice ++ p.zero_or_more(p.space() + tool_choice) + p.space() + p.literal(FC_END));+ } else {+ tool_calls = p.trigger_rule("tool-call",+ p.literal(FC_START) + p.space() + tool_choice + p.space() + p.literal(FC_END));+ }++ auto reasoning = p.eps();+ auto reasoning_with_tc = p.eps();+ auto obligatory_tool_calls = tool_calls;+ bool allow_reasoning_with_tc = false;++ if (!require_tools) {+ tool_calls = p.optional(tool_calls);+ }++ if (extract_reasoning && inputs.enable_thinking) {+ reasoning = p.optional(THINK_START + p.reasoning(p.until(THINK_END)) + THINK_END);+ reasoning_with_tc = THINK_START ++ p.reasoning(p.until_one_of({ TC_SEPARATOR + FC_START, FC_START, THINK_END })) ++ p.space() + obligatory_tool_calls;+ allow_reasoning_with_tc = true;+ } else if (extract_reasoning) {+ // Thinking disabled but reasoning extraction requested: the generation prompt+ // contains an empty <think></think> pair (V3.2) or a bare </think> (V4) that+ // must still be consumed.+ reasoning = is_v4+ ? p.optional(p.literal(THINK_END))+ : p.optional(p.literal(THINK_START) + p.until(THINK_END) + p.literal(THINK_END));+ }++ if (has_response_format) {+ auto response_format = p.rule("response-format",+ p.literal("```json") + p.space() ++ p.content(p.schema(p.json(), "response-format-schema", inputs.json_schema)) ++ p.space() + p.literal("```"));+ return generation_prompt + reasoning + response_format + end;+ }++ if (!has_tool_calls) {+ return generation_prompt + reasoning + p.content(p.rest()) + end;+ }++ auto content_before_tools = p.negate(p.literal(THINK_START)) ++ p.content(p.until_one_of({ TC_SEPARATOR + FC_START, FC_START })) ++ p.space();+ return allow_reasoning_with_tc ? generation_prompt + (reasoning_with_tc | (reasoning + content_before_tools + tool_calls)) + end :+ generation_prompt + reasoning + content_before_tools + tool_calls + end;+ });++ data.parser = parser.save();++ if (include_grammar) {+ data.grammar_lazy = has_tools && !require_tools;+ data.grammar = build_grammar([&](const common_grammar_builder & builder) {+ foreach_function(inputs.tools, [&](const json & tool) {+ const auto & function = tool.at("function");+ auto schema = function.contains("parameters") ? function.at("parameters") : json::object();+ builder.resolve_refs(schema);+ });+ if (has_response_format) {+ auto schema = inputs.json_schema;+ builder.resolve_refs(schema);+ }+ parser.build_grammar(builder, data.grammar_lazy);+ });++ data.grammar_triggers = {+ { COMMON_GRAMMAR_TRIGGER_TYPE_WORD, FC_START },+ };+ }++ return data;+}diff --git a/common/parsers/functionary-v3-2.cpp b/common/parsers/functionary-v3-2.cppnew file mode 100644index 0000000000..349b8065ac--- /dev/null+++ b/common/parsers/functionary-v3-2.cpp@@ -0,0 +1,101 @@+#include "parsers.h"++// Functionary v3.2 - uses recipient-based format: >>>recipient\n{content}+common_chat_params common_chat_params_init_functionary_v3_2(const common_chat_template & tmpl,+ const autoparser::generation_params & inputs) {+ common_chat_params data;++ data.prompt = common_chat_template_direct_apply_impl(tmpl, inputs);+ data.generation_prompt = common_chat_template_generation_prompt_impl(tmpl, inputs);+ data.format = COMMON_CHAT_FORMAT_PEG_NATIVE;+ data.preserved_tokens = {+ ">>>all",+ };++ auto has_tools = inputs.tools.is_array() && !inputs.tools.empty();+ auto include_grammar = has_tools && inputs.tool_choice != COMMON_CHAT_TOOL_CHOICE_NONE;++ if (inputs.has_continuation()) {+ const auto & msg = inputs.continue_msg;+ data.generation_prompt = "<|start_header_id|>assistant<|end_header_id|>\n\n>>>all\n" + msg.render_content();+ data.prompt += data.generation_prompt;+ }++ auto parser = build_chat_peg_parser([&](common_chat_peg_builder & p) {+ // Functionary v3.2 format:+ // - Normal content: >>>all\n{content}+ // - Tool calls: >>>function_name\n{json_args}+ // Generation prompt ends with ">>>" so model outputs recipient immediately++ // Build content parser for >>>all\n{content}+ // When tools are present, content stops before the next ">>>" (tool call)+ // When no tools, content goes until end+ auto content_until_tool = p.literal("all\n") + p.content(p.until(">>>"));+ auto content_until_end = p.literal("all\n") + p.content(p.rest());+ auto generation_prompt = p.literal("<|start_header_id|>assistant<|end_header_id|>\n\n>>>");++ // If no tools or tool_choice is NONE, just parse content+ if (!has_tools || inputs.tool_choice == COMMON_CHAT_TOOL_CHOICE_NONE) {+ // When no tools, just match the prefix and capture everything after+ return generation_prompt + content_until_end + p.end();+ }++ // Build tool call parsers for each available function+ auto tool_choice = p.choice();+ foreach_function(inputs.tools, [&](const json & tool) {+ const auto & function = tool.at("function");+ std::string name = function.at("name");+ const auto & schema = function.at("parameters");++ // Tool format: >>>function_name\n{json_args}+ auto tool_parser = p.tool(+ p.tool_open(p.tool_name(p.literal(name)) + p.literal("\n")) ++ p.tool_args(p.schema(p.json(), "tool-" + name + "-schema", schema))+ );++ tool_choice |= p.rule("tool-" + name, tool_parser);+ });++ auto content_only = content_until_end;+ auto tools_only = p.trigger_rule("tools", p.one_or_more(tool_choice));+ auto content_and_tools = content_until_tool + tools_only;++ auto ret = p.eps();+ if (inputs.tool_choice == COMMON_CHAT_TOOL_CHOICE_REQUIRED) {+ if (inputs.parallel_tool_calls) {+ ret = p.choice({ content_and_tools, tools_only }) + p.end();+ } else {+ ret = p.choice({ content_until_tool + tool_choice, tools_only }) + p.end();+ }+ } else if (inputs.parallel_tool_calls) {+ ret = p.choice({ content_and_tools, content_only, tools_only }) + p.end();+ } else {+ auto content_and_tool = content_until_tool + tool_choice;+ ret = p.choice({ content_and_tool, content_only, tool_choice }) + p.end();+ }+ return generation_prompt + ret;+ });++ data.parser = parser.save();++ if (include_grammar) {+ data.grammar_lazy = inputs.tool_choice == COMMON_CHAT_TOOL_CHOICE_AUTO;++ data.grammar = build_grammar([&](const common_grammar_builder & builder) {+ foreach_function(inputs.tools, [&](const json & tool) {+ const auto & function = tool.at("function");+ auto schema = function.at("parameters");+ builder.resolve_refs(schema);+ });+ parser.build_grammar(builder, data.grammar_lazy);+ });++ // Grammar trigger for when the model starts outputting a tool call+ // (after the initial ">>>" in the generation prompt but recipient other than "all")+ data.grammar_triggers = {+ { COMMON_GRAMMAR_TRIGGER_TYPE_PATTERN, ">>>(?!all)" }+ };+ }++ return data;+}diff --git a/common/parsers/gemma4.cpp b/common/parsers/gemma4.cppnew file mode 100644index 0000000000..041523acb8--- /dev/null+++ b/common/parsers/gemma4.cpp@@ -0,0 +1,312 @@+#include "parsers.h"++namespace workaround {++// Gemma4 uses a custom tool_responses field instead of role:tool messages.+//+// This will transform a sequence of messages:+// assistant(tool_call+) -> tool+ -> assistant(content)+//+// Into a single assistant message containing a tool_responses field:+// assistant(content + tool_call + tool_responses)+//+// This is necessary for the Gemma4 chat template to properly format the prompt.+// See https://ai.google.dev/gemma/docs/core/prompt-formatting-gemma4+struct gemma4_model_turn_builder {+ json & messages;+ size_t pos;+ json tool_calls = json::array();+ json tool_responses = json::array();+ json content;+ json reasoning_content;++ gemma4_model_turn_builder(json & msgs, size_t pos) : messages(msgs), pos(pos) {}++ void collect() {+ // Collect the first assistant message+ auto & msg = messages[pos];+ if (msg.contains("reasoning_content") && msg.at("reasoning_content").is_string()) {+ // According to the prompt formatting guide, we need to preserve reasoning_content+ // between function calls. The current chat templates do not support this, but we will do it anyway.+ reasoning_content = msg.at("reasoning_content");+ }+ for (auto & tc : msg.at("tool_calls")) {+ tool_calls.push_back(tc);+ }+ pos++;++ // Collect tool call results+ while (pos < messages.size() && messages[pos].value("role", "") == "tool") {+ collect_result(messages[pos]);+ pos++;+ }++ // Check if the next assistant message is the final message+ if (pos < messages.size() && messages[pos].value("role", "") == "assistant") {+ auto & next = messages[pos];+ if (!has_tool_calls(next) && has_content(next)) {+ content = next.at("content");+ pos++;+ }+ }+ }++ void collect_result(const json & curr) {+ json response;+ if (curr.contains("content")) {+ const auto & content = curr.at("content");+ if (content.is_string()) {+ // Try to parse the content as JSON; fall back to raw string+ try {+ response = json::parse(content.get<std::string>());+ } catch (...) {+ response = content;+ }+ } else {+ response = content;+ }+ }++ std::string name;++ // Match name with corresponding tool call+ size_t idx = tool_responses.size();+ if (idx < tool_calls.size()) {+ auto & tc = tool_calls[idx];+ if (tc.contains("function")) {+ name = tc.at("function").value("name", "");+ }+ }++ // Fallback to the tool call id+ if (name.empty()) {+ name = curr.value("tool_call_id", "");+ }++ tool_responses.push_back({{"name", name}, {"response", response}});+ }++ json build() {+ collect();++ json msg = {+ {"role", "assistant"},+ {"tool_calls", tool_calls},+ };+ if (!tool_responses.empty()) {+ msg["tool_responses"] = tool_responses;+ }+ if (!content.is_null()) {+ msg["content"] = content;+ }+ if (!reasoning_content.is_null()) {+ msg["reasoning_content"] = reasoning_content;+ }+ return msg;+ }++ static bool has_content(const json & msg) {+ if (!msg.contains("content") || msg.at("content").is_null()) {+ return false;+ }+ const auto & content = msg.at("content");+ if (content.is_string() && !content.get<std::string>().empty()) {+ return true;+ }+ if (content.is_array() && !content.empty()) {+ return true;+ }+ return false;+ }++ static bool has_tool_calls(const json & msg) {+ return msg.contains("tool_calls") && msg.at("tool_calls").is_array() && !msg.at("tool_calls").empty();+ }+};++void convert_tool_responses_gemma4(json & messages) {+ json result = json::array();+ size_t i = 0;++ while (i < messages.size()) {+ auto & msg = messages[i];++ if (msg.value("role", "") != "assistant" || !msg.contains("tool_calls") ||+ !msg.at("tool_calls").is_array() || msg.at("tool_calls").empty()) {+ result.push_back(msg);+ i++;+ continue;+ }++ gemma4_model_turn_builder builder(messages, i);+ result.push_back(builder.build());+ i = builder.pos;+ }++ messages = result;+}++}++common_chat_params common_chat_params_init_gemma4(const common_chat_template & tmpl,+ const autoparser::generation_params & inputs) {+ common_chat_params data;++ data.prompt = common_chat_template_direct_apply_impl(tmpl, inputs);+ data.generation_prompt = common_chat_template_generation_prompt_impl(tmpl, inputs);++ if (inputs.add_generation_prompt && string_ends_with(data.prompt, "<turn|>\n")) {+ // This may happen if the model generates content + tool_call, the+ // template does not add the model's next turn and confuses the model+ // from emitting its proper reasoning token sequence.+ data.generation_prompt = "<|turn>model\n";+ data.prompt += data.generation_prompt;+ }++ data.message_delimiters = {+ { COMMON_CHAT_ROLE_USER, "<|turn>user" },+ { COMMON_CHAT_ROLE_ASSISTANT, "<|turn>model" },+ };++ data.format = COMMON_CHAT_FORMAT_PEG_GEMMA4;+ data.supports_thinking = true;+ data.thinking_start_tag = "<|channel>thought";+ data.thinking_end_tags = {"<channel|>"};++ data.preserved_tokens = {+ "<|channel>",+ "<channel|>",+ "<|tool_call>",+ "<tool_call|>",+ "<|turn>",+ };++ if (inputs.has_continuation()) {+ const auto & msg = inputs.continue_msg;++ data.generation_prompt = string_ends_with(data.prompt, "<turn|>\n") ? "<|turn>model\n" : "";+ data.generation_prompt += "<|channel>thought\n" + msg.reasoning_content;+ if (inputs.continue_final_message == COMMON_CHAT_CONTINUATION_CONTENT) {+ data.generation_prompt += "<channel|>" + msg.render_content();+ }++ data.prompt += data.generation_prompt;+ }++ auto has_tools = inputs.tools.is_array() && !inputs.tools.empty();+ auto has_response_format = !inputs.json_schema.is_null() && inputs.json_schema.is_object();+ auto include_grammar = has_response_format || (has_tools && inputs.tool_choice != COMMON_CHAT_TOOL_CHOICE_NONE);+ auto extract_reasoning = inputs.reasoning_format != COMMON_REASONING_FORMAT_NONE;++ auto parser = build_chat_peg_parser([&](common_chat_peg_builder & p) {+ auto start = p.rule("start", p.optional(p.literal("<|turn>model\n")));++ if (extract_reasoning) {+ p.rule("thought", p.literal("<|channel>thought") + p.space() + p.reasoning(p.until("<channel|>")) + p.literal("<channel|>"));+ } else {+ p.rule("thought", p.content(p.literal("<|channel>thought") + p.space() + p.until("<channel|>") + p.literal("<channel|>")));+ }++ auto consume_empty_channels = p.gbnf(p.zero_or_more(p.literal("<|channel>") + p.negate(p.literal("thought"))), "");+ auto thought = (p.peek(p.literal("<|channel>")) + consume_empty_channels + p.ref("thought")) | p.negate(p.literal("<|channel>"));++ if (has_response_format) {+ auto response_format = p.literal("```json") <<+ p.content(p.schema(p.json(), "response-format-schema", inputs.json_schema)) <<+ p.literal("```");+ return start + p.optional(thought) + response_format;+ }++ if (has_tools && inputs.tool_choice != COMMON_CHAT_TOOL_CHOICE_NONE) {+ // Gemma4 tool calling syntax+ // Rules should match traversal logic in gemma4_to_json()+ p.rule("gemma4-string-content", p.until("<|\"|>"));+ p.rule("gemma4-string", p.literal("<|\"|>") + p.ref("gemma4-string-content") + p.literal("<|\"|>"));+ p.rule("gemma4-bool", p.json_bool());+ p.rule("gemma4-null", p.json_null());+ p.rule("gemma4-number", p.json_number());+ p.rule("gemma4-dict-key", p.rule("gemma4-dict-key-name", p.chars("[^:}]", 1, -1)) + p.literal(":"));+ p.rule("gemma4-dict-kv", p.ref("gemma4-dict-key") + p.space() + p.ref("gemma4-value"));+ p.rule("gemma4-dict", [&]() {+ auto ws = p.space();+ auto member = p.ref("gemma4-dict-kv");+ auto members = p.sequence({member, p.zero_or_more(p.sequence({p.literal(","), ws, member}))});+ return p.sequence({+ p.literal("{"), ws,+ p.choice({p.literal("}"), p.sequence({members, ws, p.literal("}")})})+ });+ });+ p.rule("gemma4-array", [&]() {+ auto ws = p.space();+ auto value = p.ref("gemma4-value");+ auto elements = p.sequence({value, p.zero_or_more(p.sequence({p.literal(","), ws, value}))});+ return p.sequence({+ p.literal("["), ws,+ p.choice({p.literal("]"), p.sequence({elements, ws, p.literal("]")})})+ });+ });+ p.rule("gemma4-value", [&]() {+ return p.choice({+ p.ref("gemma4-string"), p.ref("gemma4-dict"), p.ref("gemma4-array"),+ p.ref("gemma4-number"), p.ref("gemma4-bool"), p.ref("gemma4-null")+ });+ });++ auto tool_choice = p.choice();++ foreach_function(inputs.tools, [&](const json & tool) {+ const auto & function = tool.at("function");+ std::string name = function.at("name");+ // TODO @aldehir : need to extend json-schema-to-grammar to produce more than JSON rules+ // const auto & params = function.at("parameters");++ tool_choice |= p.rule("tool-" + name, p.tool(p.sequence({+ p.tool_open(p.tool_name(p.literal(name)) + p.peek(p.literal("{"))),+ p.tool_args(p.ref("gemma4-dict")),+ })));+ });++ auto tool_call = p.trigger_rule("tool-call", p.repeat(+ "<|tool_call>call:" + tool_choice + "<tool_call|>",+ /* min = */ inputs.tool_choice == COMMON_CHAT_TOOL_CHOICE_REQUIRED ? 1 : 0,+ /* max = */ inputs.parallel_tool_calls ? -1 : 1+ ));++ auto scan_to_toolcall = p.rule("scan-to-toolcall", p.until("<|tool_call>"));+ auto content = p.rule("content", p.content(p.until_one_of({"<|channel>", "<channel|>", "<|tool_call>"})));+ auto message = p.rule("message", thought + content);+ return start + p.zero_or_more(message) + scan_to_toolcall + tool_call;+ }++ // Gemma 4 may emit an extra <|channel>thought\n<channel|> at the end of the content. It may+ // also emit a single trailing <channel|> token. Consume all complete reasoning blocks and+ // then stop at the first unmatched <channel|> token.+ auto content = p.rule("content", p.content(p.until_one_of({"<|channel>", "<channel|>"})));+ auto message = p.rule("message", thought + content);+ return start + p.one_or_more(message);+ });++ data.parser = parser.save();++ if (include_grammar) {+ data.grammar_lazy = !(has_response_format || (has_tools && inputs.tool_choice == COMMON_CHAT_TOOL_CHOICE_REQUIRED));+ data.grammar = build_grammar([&](const common_grammar_builder & builder) {+ foreach_function(inputs.tools, [&](const json & tool) {+ const auto & function = tool.at("function");+ auto schema = function.at("parameters");+ builder.resolve_refs(schema);+ });+ if (has_response_format) {+ auto schema = inputs.json_schema;+ builder.resolve_refs(schema);+ }+ parser.build_grammar(builder, data.grammar_lazy);+ });++ data.grammar_triggers = {+ { COMMON_GRAMMAR_TRIGGER_TYPE_WORD, "<|tool_call>" },+ };+ }++ return data;+}diff --git a/common/parsers/gigachat-v3.cpp b/common/parsers/gigachat-v3.cppnew file mode 100644index 0000000000..41da5554ac--- /dev/null+++ b/common/parsers/gigachat-v3.cpp@@ -0,0 +1,81 @@+#include "parsers.h"++common_chat_params common_chat_params_init_gigachat_v3(+ const common_chat_template & tmpl,+ const autoparser::generation_params & inputs) {++ common_chat_params data;++ data.prompt = common_chat_template_direct_apply_impl(tmpl, inputs);+ data.generation_prompt = common_chat_template_generation_prompt_impl(tmpl, inputs);+ data.format = COMMON_CHAT_FORMAT_PEG_NATIVE;+ data.supports_thinking = false;+ data.preserved_tokens = {+ "<|message_sep|>\n\n",+ "<|role_sep|>\n",+ };++ if (inputs.has_continuation()) {+ const auto & msg = inputs.continue_msg;+ data.generation_prompt = "assistant<|role_sep|>\n" + msg.render_content();+ data.prompt += data.generation_prompt;+ }++ auto has_tools = inputs.tools.is_array() && !inputs.tools.empty();+ auto include_grammar = has_tools && inputs.tool_choice != COMMON_CHAT_TOOL_CHOICE_NONE;+ const auto *tool_call_start_prefix = "<|message_sep|>\n\nfunction call<|role_sep|>\n";++ auto parser = build_chat_peg_parser([&](common_chat_peg_builder & p) {+ auto ret = p.eps();+ if (has_tools && inputs.tool_choice != COMMON_CHAT_TOOL_CHOICE_NONE) {+ // Build a choice of all available tools+ auto tool_choice = p.choice();+ for (const auto & tool : inputs.tools) {+ const auto & function = tool.at("function");+ std::string name = function.at("name");+ const auto & schema = function.at("parameters");++ auto tool_name = p.json_member("name", "\"" + p.tool_name(p.literal(name)) + "\"");+ auto tool_args = p.json_member("arguments", p.tool_args(p.schema(p.json(), "tool-" + name + "-schema", schema)));++ auto tool_open = p.tool_open(p.literal("{") << tool_name);++ tool_choice |= p.rule("tool-" + name, tool_open << "," << tool_args << "}");+ }++ // Define the tool call structure+ auto min_calls = inputs.tool_choice == COMMON_CHAT_TOOL_CHOICE_REQUIRED ? 1 : 0;+ auto max_calls = 1; // parallel toolcalls are not supported+ auto tool_call = p.rule("tool-call", p.literal(tool_call_start_prefix) + tool_choice);+ auto tool_calls = p.trigger_rule("tool-call-root", p.repeat(tool_call, /* min = */ min_calls, /* max = */ max_calls));++ ret = p.content(p.until("<|message_sep|>\n\n")) << tool_calls;+ } else {+ // Content only parser+ include_grammar = false;+ ret = p.content(p.rest());+ }++ return p.literal("assistant<|role_sep|>\n") + ret;+ });++ data.parser = parser.save();++ if (include_grammar) {+ data.grammar_lazy = has_tools && inputs.tool_choice == COMMON_CHAT_TOOL_CHOICE_AUTO;++ data.grammar = build_grammar([&](const common_grammar_builder & builder) {+ foreach_function(inputs.tools, [&](const json & tool) {+ const auto & function = tool.at("function");+ auto schema = function.at("parameters");+ builder.resolve_refs(schema);+ });+ parser.build_grammar(builder, data.grammar_lazy);+ });++ data.grammar_triggers = {+ {COMMON_GRAMMAR_TRIGGER_TYPE_WORD, tool_call_start_prefix}+ };+ }+ return data;+}diff --git a/common/parsers/gpt-oss.cpp b/common/parsers/gpt-oss.cppnew file mode 100644index 0000000000..d7dbfbfb57--- /dev/null+++ b/common/parsers/gpt-oss.cpp@@ -0,0 +1,167 @@+#include "parsers.h"++common_chat_params common_chat_params_init_gpt_oss(const common_chat_template & tmpl,+ const autoparser::generation_params & inputs) {+ common_chat_params data;++ // Copy reasoning to the "thinking" field as expected by the gpt-oss template+ auto adjusted_messages = json::array();+ for (auto msg : inputs.messages) {+ if (msg.contains("reasoning_content") && msg.at("reasoning_content").is_string()) {+ msg["thinking"] = msg.at("reasoning_content");+ if (msg.contains("tool_calls") && msg.at("tool_calls").is_array() && !msg.at("tool_calls").empty()) {+ msg.erase("content");+ }+ }+ adjusted_messages.push_back(msg);+ }++ auto prompt = common_chat_template_direct_apply_impl(tmpl, inputs, /* messages_override= */ adjusted_messages);++ // Check if we need to replace the return token with end token during+ // inference and without generation prompt. For more details see:+ // https://github.com/ggml-org/llama.cpp/issues/15417+ if (inputs.is_inference && !inputs.add_generation_prompt) {+ static constexpr std::string_view return_token = "<|return|>";+ static constexpr std::string_view end_token = "<|end|>";+ if (size_t pos = prompt.rfind(return_token); pos != std::string::npos) {+ prompt.replace(pos, return_token.length(), end_token);+ }+ }++ data.prompt = prompt;+ data.generation_prompt = common_chat_template_generation_prompt_impl(tmpl, inputs, /* messages_override= */ adjusted_messages);+ data.message_delimiters = {+ { COMMON_CHAT_ROLE_ASSISTANT, "<|start|>assistant" },+ { COMMON_CHAT_ROLE_USER, "<|start|>user" },+ { COMMON_CHAT_ROLE_SYSTEM, "<|start|>developer" },+ { COMMON_CHAT_ROLE_SYSTEM, "<|start|>system" },+ { COMMON_CHAT_ROLE_TOOL, "<|start|>functions" },+ };++ data.format = COMMON_CHAT_FORMAT_PEG_NATIVE;+ data.supports_thinking = true;++ data.thinking_start_tag = "<|channel|>analysis<|message|>";+ data.thinking_end_tags = {"<|end|>"};++ // These special tokens are required to parse properly, so we include them+ // even if parse_tool_calls is false.+ data.preserved_tokens = {+ "<|channel|>", "<|constrain|>", "<|message|>", "<|start|>", "<|end|>",+ };++ // Adjust prompt for continuation+ if (inputs.has_continuation()) {+ const auto & msg = inputs.continue_msg;++ data.generation_prompt = "<|start|>assistant<|channel|>analysis<|message|>" + msg.reasoning_content;+ if (inputs.continue_final_message == COMMON_CHAT_CONTINUATION_CONTENT) {+ data.generation_prompt += "<|end|><|start|>assistant<|channel|>final<|message|>" + msg.render_content();+ }++ data.prompt += data.generation_prompt;+ }++ auto has_tools = inputs.tools.is_array() && !inputs.tools.empty();+ auto has_response_format = !inputs.json_schema.is_null() && inputs.json_schema.is_object();+ auto include_grammar = has_response_format || (has_tools && inputs.tool_choice != COMMON_CHAT_TOOL_CHOICE_NONE);+ auto extract_reasoning = inputs.reasoning_format != COMMON_REASONING_FORMAT_NONE;++ auto parser = build_chat_peg_parser([&](common_chat_peg_builder & p) {+ auto start = p.rule("start", p.literal("<|start|>assistant"));+ auto end = p.rule("end", p.literal("<|end|>"));+ auto content = p.rule("message-content", p.until("<|end|>"));+ auto channel = p.literal("<|channel|>") + (p.literal("commentary") | p.literal("analysis"));+ auto constrain_type = p.chars("[A-Za-z0-9_-]", 1, -1);++ // Occasionally, gpt-oss-20b will prefix channels with this commentary+ auto stray_commentary = p.optional(p.literal("<|channel|>commentary") + p.optional(p.literal(" to=assistant")));+ auto start_analysis = stray_commentary + p.literal("<|channel|>analysis<|message|>");++ if (extract_reasoning) {+ p.rule("analysis", start_analysis + p.reasoning(content) + end);+ } else {+ p.rule("analysis", p.content(start_analysis + content + end));+ }++ auto analysis = p.ref("analysis");+ auto preamble = p.rule("preamble", p.literal("<|channel|>commentary<|message|>") + p.content(content) + end);+ auto final_msg = p.rule("final", stray_commentary + p.literal("<|channel|>final<|message|>") + p.content(content));++ // Consume any unsolicited tool calls, e.g. builtin functions+ auto unsolicited = p.rule("unsolicited", p.atomic(p.optional(channel) + p.literal(" to=") + content + end));++ auto any = p.rule("any", preamble | analysis);++ if (has_response_format) {+ auto constraint = p.optional(p.space() + p.optional(p.literal("<|constrain|>")) + constrain_type);+ auto response_format = p.rule("response-format",+ p.literal("<|channel|>final") + constraint + p.literal("<|message|>") ++ p.content(p.schema(p.json(), "response-format-schema", inputs.json_schema)));++ return p.zero_or_more(start + analysis) + start + response_format;+ }++ if (has_tools && inputs.tool_choice != COMMON_CHAT_TOOL_CHOICE_NONE) {+ auto tool_choice = p.choice();++ foreach_function(inputs.tools, [&](const json & tool) {+ const auto & function = tool.at("function");+ std::string name = function.at("name");+ const auto & params = function.at("parameters");++ auto func_name = p.literal(" to=functions.") + p.tool_name(p.literal(name));+ auto constraint = p.optional(p.space() + p.optional(p.literal("<|constrain|>")) + constrain_type);+ auto args = p.tool_args(p.schema(p.json(), "tool-" + name + "-schema", params));++ // recipient in role header+ // <|start|>assistant to=functions.NAME<|channel|>(commentary|analysis)[constraint]<|message|>ARGS+ auto tool_in_role = p.tool(p.tool_open(func_name + channel + constraint + p.literal("<|message|>")) + args);++ // recipient in channel header+ // <|channel|>(commentary|analysis) to=functions.NAME[constraint]<|message|>ARGS+ auto tool_in_channel = p.tool(p.tool_open(channel + func_name + constraint + p.literal("<|message|>")) + args);++ tool_choice |= p.rule("tool-" + name, tool_in_role | tool_in_channel);+ });++ auto tool_call = p.trigger_rule("tool-call", tool_choice);++ if (inputs.tool_choice == COMMON_CHAT_TOOL_CHOICE_REQUIRED) {+ return p.zero_or_more(start + any) + start + tool_call;+ }++ return p.zero_or_more(start + any) + start + (tool_call | final_msg);+ }++ return p.zero_or_more(start + any) + start + (final_msg | unsolicited);+ });++ data.parser = parser.save();++ if (include_grammar) {+ data.grammar_lazy = !(has_response_format || (has_tools && inputs.tool_choice == COMMON_CHAT_TOOL_CHOICE_REQUIRED));+ data.grammar = build_grammar([&](const common_grammar_builder & builder) {+ foreach_function(inputs.tools, [&](const json & tool) {+ const auto & function = tool.at("function");+ auto schema = function.at("parameters");+ builder.resolve_refs(schema);+ });+ if (has_response_format) {+ auto schema = inputs.json_schema;+ builder.resolve_refs(schema);+ }+ parser.build_grammar(builder, data.grammar_lazy);+ });++ data.grammar_triggers = {+ { COMMON_GRAMMAR_TRIGGER_TYPE_PATTERN, "^\\s+to$" },+ { COMMON_GRAMMAR_TRIGGER_TYPE_PATTERN, "^<\\|channel\\|>(?:commentary|analysis)\\s+to=functions$" },+ { COMMON_GRAMMAR_TRIGGER_TYPE_PATTERN, "<\\|start\\|>assistant(\\s+to)" },+ { COMMON_GRAMMAR_TRIGGER_TYPE_PATTERN, "<\\|start\\|>assistant(<\\|channel\\|>(?:commentary|analysis)\\s+to)" }+ };+ }++ return data;+}diff --git a/common/parsers/kimi-k2.cpp b/common/parsers/kimi-k2.cppnew file mode 100644index 0000000000..57f6bfdcb6--- /dev/null+++ b/common/parsers/kimi-k2.cpp@@ -0,0 +1,133 @@+#include "parsers.h"++// Kimi K2 Thinking - uses unique tool call ID format: functions.<name>:<index>+// The ID contains both the function name and an incrementing counter+common_chat_params common_chat_params_init_kimi_k2(const common_chat_template & tmpl,+ const autoparser::generation_params & inputs) {+ common_chat_params data;++ data.prompt = common_chat_template_direct_apply_impl(tmpl, inputs);+ data.generation_prompt = common_chat_template_generation_prompt_impl(tmpl, inputs);+ data.format = COMMON_CHAT_FORMAT_PEG_NATIVE;+ data.supports_thinking = true;+ data.preserved_tokens = {+ "<|tool_calls_section_begin|>",+ "<|tool_calls_section_end|>",+ "<|tool_call_begin|>",+ "<|tool_call_argument_begin|>",+ "<|tool_call_end|>",+ "<think>",+ "</think>",+ };++ auto has_tools = inputs.tools.is_array() && !inputs.tools.empty();+ auto extract_reasoning = inputs.reasoning_format != COMMON_REASONING_FORMAT_NONE;+ auto include_grammar = has_tools && inputs.tool_choice != COMMON_CHAT_TOOL_CHOICE_NONE;++ const std::string SECTION_BEGIN = "<|tool_calls_section_begin|>";+ const std::string SECTION_END = "<|tool_calls_section_end|>";+ const std::string CALL_BEGIN = "<|tool_call_begin|>";+ const std::string ARGS_BEGIN = "<|tool_call_argument_begin|>";+ const std::string CALL_END = "<|tool_call_end|>";++ const std::string THINK_START = "<think>";+ const std::string THINK_END = "</think>";+ const std::string GEN_PROMPT = "<|im_assistant|>assistant<|im_middle|>";++ data.thinking_start_tag = THINK_START;+ data.thinking_end_tags = {THINK_END};++ if (inputs.has_continuation()) {+ const auto & msg = inputs.continue_msg;++ data.generation_prompt = GEN_PROMPT + THINK_START + msg.reasoning_content;+ if (inputs.continue_final_message == COMMON_CHAT_CONTINUATION_CONTENT) {+ data.generation_prompt += THINK_END + msg.render_content();+ }++ data.prompt += data.generation_prompt;+ }++ auto parser = build_chat_peg_parser([&](common_chat_peg_builder & p) {+ // Kimi K2 Thinking format:+ // - Reasoning: <think>{reasoning}</think>+ // - Content: text after reasoning+ // - Tool calls section:+ // <|tool_calls_section_begin|>+ // <|tool_call_begin|>functions.<name>:<index><|tool_call_argument_begin|>{json_args}<|tool_call_end|>+ // ...+ // <|tool_calls_section_end|>+ // The ID format is: functions.<function_name>:<counter> where counter is 0, 1, 2, ...++ // Tool call markers+ auto end = p.end();++ // Note: this model is CRAZY. It can diverge from its supposed tool calling pattern in so many ways it's not funny.+ // For example, it can call tools at the end of reasoning without closing reasoning...+ auto reasoning = extract_reasoning ? p.optional(THINK_START + p.reasoning(+ p.until_one_of({ THINK_END, "<|tool_calls_section_begin|>", "<|tool_call_begin|>" })) ++ p.optional(p.literal(THINK_END))) : p.eps();+ auto generation_prompt = p.literal(GEN_PROMPT);+++ // Content only parser (no tools)+ if (!has_tools || inputs.tool_choice == COMMON_CHAT_TOOL_CHOICE_NONE) {+ return generation_prompt + reasoning + p.content(p.rest()) + end;+ }++ // Build tool call parsers for each available function+ // The ID format is: functions.<name>:<index>+ // We need to match: functions.<name>:<digits>+ auto tool_choice = p.choice();+ foreach_function(inputs.tools, [&](const json & tool) {+ const auto & function = tool.at("function");+ std::string name = function.at("name");+ const auto & schema = function.at("parameters");++ // Match: functions.<name>:<digits>+ // Capture the full call id (functions.<name>:<digits>) using tool_id tag+ auto tool_id = p.tool_id(p.literal("functions.") + p.tool_name(p.literal(name)) + p.literal(":") + p.chars("[0-9]", 1, -1));+ auto tool_parser = p.tool(+ p.tool_open(tool_id + p.literal(ARGS_BEGIN)) ++ p.tool_args(p.schema(p.json(), "tool-" + name + "-schema", schema)) ++ p.tool_close(p.optional((p.literal(CALL_END))))+ );++ tool_choice |= p.rule("tool-" + name, tool_parser);+ });++ // Tool calls section: <|tool_calls_section_begin|> tool_calls <|tool_calls_section_end|>+ auto min_calls = inputs.tool_choice == COMMON_CHAT_TOOL_CHOICE_REQUIRED ? 1 : 0;+ auto max_calls = inputs.parallel_tool_calls ? -1 : 1;+ // Use trigger_rule so grammar generator knows where to start generating rules+ auto tool_calls = p.rule("tool-calls",+ p.optional(p.literal(SECTION_BEGIN)) ++ p.trigger_rule("tool-call", p.repeat(CALL_BEGIN + tool_choice, min_calls, max_calls) ++ p.optional(p.literal(SECTION_END)))+ );++ auto content_before_tools = p.content(p.until_one_of({ SECTION_BEGIN, CALL_BEGIN }));++ return generation_prompt + reasoning + content_before_tools + tool_calls + end;+ });++ data.parser = parser.save();++ if (include_grammar) {+ data.grammar_lazy = inputs.tool_choice == COMMON_CHAT_TOOL_CHOICE_AUTO;+ data.grammar = build_grammar([&](const common_grammar_builder & builder) {+ foreach_function(inputs.tools, [&](const json & tool) {+ const auto & function = tool.at("function");+ auto schema = function.at("parameters");+ builder.resolve_refs(schema);+ });+ parser.build_grammar(builder, data.grammar_lazy);+ });++ data.grammar_triggers = {+ { COMMON_GRAMMAR_TRIGGER_TYPE_WORD, "<|tool_call_begin|>" }+ };+ }++ return data;+}diff --git a/common/parsers/kimi-k3.cpp b/common/parsers/kimi-k3.cppnew file mode 100644index 0000000000..56a49903f7--- /dev/null+++ b/common/parsers/kimi-k3.cpp@@ -0,0 +1,174 @@+#include "parsers.h"++// Kimi K3 - XTML tagged format, built by open_tag/close_tag macros:+// open_tag(t, attrs) = <|open|>t k="v"...<|sep|> close_tag(t) = <|close|>t<|sep|>+// assistant := [think] [response] [tools] close_tag(message) <|end_of_msg|>+// the generation prompt already opens the think (or response) section, so the+// section opener is optional here - same as Kimi K2 Thinking+common_chat_params common_chat_params_init_kimi_k3(const common_chat_template & tmpl,+ const autoparser::generation_params & inputs) {+ common_chat_params data;++ data.prompt = common_chat_template_direct_apply_impl(tmpl, inputs);+ data.generation_prompt = common_chat_template_generation_prompt_impl(tmpl, inputs);+ data.format = COMMON_CHAT_FORMAT_PEG_NATIVE;+ data.supports_thinking = true;++ const std::string SEP = "<|sep|>";+ const std::string MSG_START = "<|open|>message role=\"assistant\"<|sep|>";+ const std::string THINK_START = "<|open|>think<|sep|>";+ const std::string THINK_END = "<|close|>think<|sep|>";+ const std::string RESP_START = "<|open|>response<|sep|>";+ const std::string RESP_END = "<|close|>response<|sep|>";+ const std::string TOOLS_START = "<|open|>tools<|sep|>";+ const std::string TOOLS_END = "<|close|>tools<|sep|>";+ const std::string CALL_START = "<|open|>call tool=\"";+ const std::string CALL_END = "<|close|>call<|sep|>";+ const std::string ARG_START = "<|open|>argument key=\"";+ const std::string ARG_END = "<|close|>argument<|sep|>";+ const std::string MSG_END = "<|close|>message<|sep|>";+ const std::string EOM_TOKEN = "<|end_of_msg|>";++ // only the markers are special tokens. tag names ("think", "response", ...) are+ // normal tokens and must not be preserved, or prose with those words is broken+ data.preserved_tokens = {+ "<|open|>",+ "<|close|>",+ "<|sep|>",+ "<|end_of_msg|>",+ };++ data.thinking_start_tag = THINK_START;+ data.thinking_end_tags = { THINK_END };++ // per-role message-start delimiters. user/assistant messages only have the role+ // attribute, so the full opener is used. system and tool messages have more+ // attributes, so those delimiters stop after the closing quote of the role+ data.message_delimiters = {+ { COMMON_CHAT_ROLE_ASSISTANT, "<|open|>message role=\"assistant\"<|sep|>" },+ { COMMON_CHAT_ROLE_USER, "<|open|>message role=\"user\"<|sep|>" },+ { COMMON_CHAT_ROLE_TOOL, "<|open|>message role=\"tool\"" },+ { COMMON_CHAT_ROLE_SYSTEM, "<|open|>message role=\"system\"" },+ };++ auto has_tools = inputs.tools.is_array() && !inputs.tools.empty();+ auto extract_reasoning = inputs.reasoning_format != COMMON_REASONING_FORMAT_NONE;+ auto include_grammar = has_tools && inputs.tool_choice != COMMON_CHAT_TOOL_CHOICE_NONE;++ if (inputs.has_continuation()) {+ const auto & msg = inputs.continue_msg;++ data.generation_prompt = MSG_START + THINK_START + msg.reasoning_content;+ if (inputs.continue_final_message == COMMON_CHAT_CONTINUATION_CONTENT) {+ data.generation_prompt += THINK_END + RESP_START + msg.render_content();+ }++ data.prompt += data.generation_prompt;+ }++ auto parser = build_chat_peg_parser([&](common_chat_peg_builder & p) {+ auto end = p.end();++ auto start = p.optional(p.literal(MSG_START));++ // the think section is always consumed, even with reasoning extraction off:+ // the generation prompt ends with open_tag('think'), so it is always present.+ // reasoning stops at its own closer, or at the response opener if the model+ // skips the closer+ auto think_body = extract_reasoning ? p.reasoning(p.until_one_of({ THINK_END, RESP_START })) :+ p.content(p.until_one_of({ THINK_END, RESP_START }));++ auto reasoning = p.optional(p.optional(p.literal(THINK_START)) + think_body ++ p.optional(p.literal(THINK_END)));++ // content runs to the response closer, or to the next section if truncated+ auto response = p.optional(p.literal(RESP_START)) ++ p.content(p.until_one_of({ RESP_END, TOOLS_START, MSG_END })) ++ p.optional(p.literal(RESP_END));++ // the EOG token after the message closer reaches the parser as text,+ // so it must be consumed or the parse stays incomplete+ auto trailer = p.optional(p.literal(MSG_END)) + p.optional(p.literal(EOM_TOKEN));++ if (!has_tools || inputs.tool_choice == COMMON_CHAT_TOOL_CHOICE_NONE) {+ return start + reasoning + response + trailer + end;+ }++ auto tool_choices = p.choice();+ foreach_function(inputs.tools, [&](const json & tool) {+ const auto & function = tool.at("function");+ std::string name = function.at("name");+ const json schema = function.contains("parameters") ? function.at("parameters") : json::object();++ // arguments come one tag per key, with the JSON type in a type="..."+ // attribute. the type is taken from the tool schema instead, as it tells+ // us if the value is JSON or a literal string+ auto args = p.eps();+ if (schema.contains("properties") && !schema.at("properties").empty()) {+ auto arg_choices = p.choice();+ for (const auto & prop : schema.at("properties").items()) {+ const std::string & key = prop.key();++ std::string type = "string";+ if (prop.value().is_object() && prop.value().contains("type") &&+ prop.value().at("type").is_string()) {+ type = prop.value().at("type").get<std::string>();+ }++ auto value = type == "string" ? p.tool_arg_string_value(p.until(ARG_END)) :+ p.tool_arg_value(p.until(ARG_END));++ // skip the trailing type="..." attribute: anything up to <|sep|>+ arg_choices |= p.rule("kimi-k3-arg-" + name + "-" + key,+ p.tool_arg(p.tool_arg_open(p.literal(ARG_START)) ++ p.tool_arg_name(p.literal(key)) + p.literal("\"") ++ p.until(SEP) + p.literal(SEP) + value ++ p.tool_arg_close(p.literal(ARG_END))));+ }+ args = p.zero_or_more(arg_choices);+ }++ // skip the trailing index="N" attribute the same way+ auto call = p.tool(p.tool_open(p.literal(CALL_START) + p.tool_name(p.literal(name)) + p.literal("\"") ++ p.until(SEP) + p.literal(SEP)) ++ p.tool_args(args) + p.tool_close(p.literal(CALL_END)));++ tool_choices |= p.rule("kimi-k3-tool-" + name, call);+ });++ // all calls go inside one tools section, then the message is closed. the+ // message closer is part of the trigger rule, or else the lazy grammar+ // rejects it once tool calls have started+ auto tools_section =+ p.trigger_rule("kimi-k3-tool-call", p.literal(TOOLS_START) + p.one_or_more(tool_choices) ++ p.literal(TOOLS_END) + p.optional(p.literal(MSG_END)) ++ p.optional(p.literal(EOM_TOKEN)));++ auto tools = inputs.tool_choice == COMMON_CHAT_TOOL_CHOICE_REQUIRED ? tools_section :+ p.optional(tools_section);++ return start + reasoning + response + tools + trailer + end;+ });++ data.parser = parser.save();++ if (include_grammar) {+ data.grammar_lazy = inputs.tool_choice != COMMON_CHAT_TOOL_CHOICE_REQUIRED;+ data.grammar = build_grammar([&](const common_grammar_builder & builder) {+ foreach_function(inputs.tools, [&](const json & tool) {+ const auto & function = tool.at("function");+ if (function.contains("parameters")) {+ auto schema = function.at("parameters");+ builder.resolve_refs(schema);+ }+ });+ parser.build_grammar(builder, data.grammar_lazy);+ });++ data.grammar_triggers = {+ { COMMON_GRAMMAR_TRIGGER_TYPE_WORD, TOOLS_START },+ };+ }++ return data;+}diff --git a/common/parsers/lfm2.cpp b/common/parsers/lfm2.cppnew file mode 100644index 0000000000..4514f908b9--- /dev/null+++ b/common/parsers/lfm2.cpp@@ -0,0 +1,119 @@+#include "parsers.h"++// LFM2 format detection: template uses <|tool_list_start|>[...]<|tool_list_end|> around the tool list+// and <|tool_call_start|>[...]<|tool_call_end|> around each tool call+bool is_lfm2_template(const std::string & src) {+ return src.find("<|tool_list_start|>") != std::string::npos &&+ src.find("<|tool_list_end|>") != std::string::npos;+}++// LFM2/LFM2.5 parser. Tool calls are almost Python-style and parallel-capable+// (except dotted names and JSON literals true/false/null).+// Always wrapped in <|tool_call_start|>[name(args)]<|tool_call_end|> with optional <think> reasoning.+// tool_list_tokens preserves LFM2 system tool-list markers.+common_chat_params common_chat_params_init_lfm2(const common_chat_template & tmpl,+ const autoparser::generation_params & inputs,+ bool tool_list_tokens) {+ common_chat_params data;++ const std::string TOOL_CALL_START = "<|tool_call_start|>";+ const std::string TOOL_CALL_END = "<|tool_call_end|>";+ const std::string TOOL_LIST_START = "<|tool_list_start|>";+ const std::string TOOL_LIST_END = "<|tool_list_end|>";+ const std::string THINK_START = "<think>";+ const std::string THINK_END = "</think>";+ const std::string GEN_PROMPT = "<|im_start|>assistant\n";++ // Copy reasoning to the "thinking" field the template expects+ auto adjusted_messages = json::array();+ for (auto msg : inputs.messages) {+ if (msg.contains("reasoning_content") && msg.at("reasoning_content").is_string()) {+ msg["thinking"] = msg.at("reasoning_content");+ }+ adjusted_messages.push_back(msg);+ }++ data.prompt = common_chat_template_direct_apply_impl(tmpl, inputs, adjusted_messages);+ data.generation_prompt = common_chat_template_generation_prompt_impl(tmpl, inputs, adjusted_messages);+ data.format = COMMON_CHAT_FORMAT_PEG_NATIVE;+ data.supports_thinking = true;+ data.preserved_tokens = { TOOL_CALL_START, TOOL_CALL_END, THINK_START, THINK_END };+ if (tool_list_tokens) {+ data.preserved_tokens.push_back(TOOL_LIST_START);+ data.preserved_tokens.push_back(TOOL_LIST_END);+ }++ data.thinking_start_tag = THINK_START;+ data.thinking_end_tags = {THINK_END};++ auto has_tools = inputs.tools.is_array() && !inputs.tools.empty();+ auto has_response_format = !inputs.json_schema.is_null() && inputs.json_schema.is_object();+ // Gate by reasoning format and whether the template supports <think>+ auto extract_reasoning = inputs.reasoning_format != COMMON_REASONING_FORMAT_NONE &&+ tmpl.source().find(THINK_START) != std::string::npos;+ auto include_grammar = has_response_format || (has_tools && inputs.tool_choice != COMMON_CHAT_TOOL_CHOICE_NONE);++ if (inputs.has_continuation()) {+ const auto & msg = inputs.continue_msg;++ data.generation_prompt = GEN_PROMPT + THINK_START + msg.reasoning_content;+ if (inputs.continue_final_message == COMMON_CHAT_CONTINUATION_CONTENT) {+ data.generation_prompt += THINK_END + msg.render_content();+ }++ data.prompt += data.generation_prompt;+ }++ auto parser = build_chat_peg_parser([&](common_chat_peg_builder & p) {+ auto generation_prompt = p.literal(GEN_PROMPT);+ auto end = p.end();++ auto reasoning = p.eps();+ if (extract_reasoning) {+ reasoning = p.optional(THINK_START + p.reasoning(p.until(THINK_END)) + THINK_END);+ }++ if (!has_tools || inputs.tool_choice == COMMON_CHAT_TOOL_CHOICE_NONE) {+ if (has_response_format) {+ auto response_format = p.content(p.schema(p.json(), "response-format-schema", inputs.json_schema));+ return generation_prompt + reasoning + response_format + end;+ }+ return generation_prompt + reasoning + p.content(p.rest()) + end;+ }+ auto tool_calls = p.rule("tool-calls",+ p.trigger_rule("tool-call",+ p.literal(TOOL_CALL_START) ++ p.python_style_tool_calls(inputs.tools, inputs.parallel_tool_calls, /* allow_json_literals = */ true) ++ p.literal(TOOL_CALL_END)+ )+ );++ auto content = p.content(p.until(TOOL_CALL_START));++ return generation_prompt + reasoning + content + tool_calls + end;+ });++ data.parser = parser.save();++ if (include_grammar) {+ data.grammar_lazy = !(has_response_format || (has_tools && inputs.tool_choice == COMMON_CHAT_TOOL_CHOICE_REQUIRED));+ data.grammar = build_grammar([&](const common_grammar_builder & builder) {+ foreach_function(inputs.tools, [&](const json & tool) {+ const auto & function = tool.at("function");+ auto schema = function.at("parameters");+ builder.resolve_refs(schema);+ });+ if (has_response_format) {+ auto schema = inputs.json_schema;+ builder.resolve_refs(schema);+ }+ parser.build_grammar(builder, data.grammar_lazy);+ });++ data.grammar_triggers = {+ { COMMON_GRAMMAR_TRIGGER_TYPE_WORD, TOOL_CALL_START }+ };+ }++ return data;+}diff --git a/common/parsers/minicpm5.cpp b/common/parsers/minicpm5.cppnew file mode 100644index 0000000000..e6e0abf066--- /dev/null+++ b/common/parsers/minicpm5.cpp@@ -0,0 +1,144 @@+#include "parsers.h"++// MiniCPM5 format:+// - Reasoning: <think>{reasoning}</think> (optional)+// - Tool calls: <function name="foo"><param name="bar">value</param></function>+common_chat_params common_chat_params_init_minicpm5(const common_chat_template & tmpl,+ const autoparser::generation_params & inputs) {+ common_chat_params data;++ data.prompt = common_chat_template_direct_apply_impl(tmpl, inputs);+ data.generation_prompt = common_chat_template_generation_prompt_impl(tmpl, inputs);+ data.format = COMMON_CHAT_FORMAT_PEG_NATIVE;+ data.supports_thinking = true;+ data.preserved_tokens = {+ "<function",+ "<param",+ "</function>",+ "</param>",+ "<think>",+ "</think>",+ };++ data.thinking_start_tag = "<think>";+ data.thinking_end_tags = {"</think>"};++ data.message_delimiters = {+ { COMMON_CHAT_ROLE_ASSISTANT, "<|im_start|>assistant" },+ { COMMON_CHAT_ROLE_TOOL, "<|im_start|>user\n<tool_response>" },+ { COMMON_CHAT_ROLE_USER, "<|im_start|>user" },+ { COMMON_CHAT_ROLE_SYSTEM, "<|im_start|>system" },+ };++ auto has_tools = inputs.tools.is_array() && !inputs.tools.empty();+ auto has_response_format = inputs.json_schema.is_object() && !inputs.json_schema.empty();+ auto extract_reasoning = inputs.reasoning_format != COMMON_REASONING_FORMAT_NONE;+ auto include_grammar = has_response_format || (has_tools && inputs.tool_choice != COMMON_CHAT_TOOL_CHOICE_NONE);++ if (inputs.has_continuation()) {+ const auto & msg = inputs.continue_msg;++ data.generation_prompt = "<|im_start|>assistant\n<think>\n" + msg.reasoning_content;+ if (inputs.continue_final_message == COMMON_CHAT_CONTINUATION_CONTENT) {+ data.generation_prompt += "\n</think>\n\n" + msg.render_content();+ }++ data.prompt += data.generation_prompt;+ }++ auto parser = build_chat_peg_parser([&](common_chat_peg_builder & p) {+ auto generation_prompt = p.literal("<|im_start|>assistant\n");++ auto reasoning = p.eps();+ if (extract_reasoning) {+ reasoning = ("<think>" << p.reasoning(p.until("</think>")) << "</think>") + p.space();+ }++ // Response format parser+ if (has_response_format) {+ return generation_prompt + reasoning + p.content(p.schema(p.json(), "response-format", inputs.json_schema));+ }++ if (has_tools && inputs.tool_choice != COMMON_CHAT_TOOL_CHOICE_NONE) {+ // CDATA lets a value carry characters that would otherwise close the tag (e.g.+ // </param>); capture the inner text only, excluding the CDATA markers.+ auto string_value = p.choice({+ p.literal("<![CDATA[") + p.ac(p.tool_arg_string_value(p.until("]]>")) + p.literal("]]>"), "]]>") + p.tool_arg_close(p.literal("</param>")),+ p.negate(p.literal("< {+ const auto & function = tool.at("function");+ const std::string name = function.at("name");+ auto params = function.contains("parameters") ? function.at("parameters") : json::object();++ auto args = p.eps();+ if (params.contains("properties") && params.at("properties").is_object() && !params.at("properties").empty()) {+ auto schema_info = common_schema_info();+ schema_info.resolve_refs(params);++ auto arg_choice = p.choice();+ for (const auto & [prop_name, prop_schema] : params.at("properties").items()) {+ auto value_parser = p.eps();+ if (schema_info.resolves_to_string(prop_schema)) {+ value_parser = string_value;+ } else {+ value_parser = p.tool_arg_json_value(+ p.schema(p.json(), "tool-" + name + "-arg-" + prop_name + "-schema", prop_schema, false)+ ) + p.tool_arg_close(p.literal("</param>"));+ }++ auto arg_rule = p.tool_arg(+ p.tool_arg_open(p.literal("<param name=\"") + p.tool_arg_name(p.literal(prop_name)) + p.literal("\">")) ++ value_parser+ );++ arg_choice |= arg_rule;+ }+ args = p.zero_or_more(arg_choice + p.space());+ }++ auto tool_parser = p.tool(+ p.tool_open(p.literal("<function name=\"") + p.tool_name(p.literal(name)) + p.literal("\">"))+ << p.tool_args(args)+ << p.tool_close(p.literal("</function>")));++ tool_choice |= p.rule("tool-" + name, tool_parser);+ });++ auto max_calls = inputs.parallel_tool_calls ? -1 : 1;+ auto tool_calls = p.trigger_rule("tool-call", p.repeat(tool_choice + p.space(), 1, max_calls));++ auto content = p.content(p.until("<function"));++ return generation_prompt + reasoning + content + tool_calls + p.end();+ }++ return generation_prompt + reasoning + p.content(p.rest()) + p.end();+ });++ data.parser = parser.save();++ if (include_grammar) {+ data.grammar_lazy = !(has_response_format || (has_tools && inputs.tool_choice == COMMON_CHAT_TOOL_CHOICE_REQUIRED));+ data.grammar = build_grammar([&](const common_grammar_builder & builder) {+ foreach_function(inputs.tools, [&](const json & tool) {+ const auto & function = tool.at("function");+ auto schema = function.contains("parameters") ? function.at("parameters") : json::object();+ builder.resolve_refs(schema);+ });+ if (has_response_format) {+ auto schema = inputs.json_schema;+ builder.resolve_refs(schema);+ }+ parser.build_grammar(builder, data.grammar_lazy);+ });++ data.grammar_triggers = {+ { COMMON_GRAMMAR_TRIGGER_TYPE_WORD, "<function" },+ };+ }++ return data;+}diff --git a/common/parsers/minimax-m3.cpp b/common/parsers/minimax-m3.cppnew file mode 100644index 0000000000..ff23ea153c--- /dev/null+++ b/common/parsers/minimax-m3.cpp@@ -0,0 +1,259 @@+#include "parsers.h"++common_chat_params common_chat_params_init_minimax_m3(const common_chat_template & tmpl,+ const autoparser::generation_params & inputs) {+ common_chat_params data;++ data.prompt = common_chat_template_direct_apply_impl(tmpl, inputs);+ data.generation_prompt = common_chat_template_generation_prompt_impl(tmpl, inputs);+ data.format = COMMON_CHAT_FORMAT_PEG_MINIMAX_M3;+ data.supports_thinking = true;+ data.thinking_start_tag = "<mm:think>";+ data.thinking_end_tags = {"</mm:think>"};++ // M3 prefixes every tool tag with the namespace token "]<]minimax[>[";+ // params use the parameter name as the tag (<file_path>...</file_path>).+ const std::string NS = "]<]minimax[>[";+ const std::string THINK_START = "<mm:think>";+ const std::string THINK_END = "</mm:think>";+ const std::string FC_START = NS + "<tool_call>";+ const std::string FC_END = NS + "</tool_call>";+ const std::string INVOKE_END = NS + "</invoke>";++ data.preserved_tokens = {+ NS,+ "<tool_call>",+ "</tool_call>",+ THINK_START,+ THINK_END,+ };++ data.message_delimiters = {+ { COMMON_CHAT_ROLE_ASSISTANT, "]~b]ai" },+ { COMMON_CHAT_ROLE_USER, "]~b]user" },+ { COMMON_CHAT_ROLE_TOOL, "]~b]tool" },+ { COMMON_CHAT_ROLE_SYSTEM, "]~b]developer" },+ { COMMON_CHAT_ROLE_SYSTEM, "]~b]system" },+ };++ auto has_tools = inputs.tools.is_array() && !inputs.tools.empty();+ auto has_response_format = !inputs.json_schema.is_null() && inputs.json_schema.is_object();+ auto extract_reasoning = inputs.reasoning_format != COMMON_REASONING_FORMAT_NONE;+ auto include_grammar = has_response_format || (has_tools && inputs.tool_choice != COMMON_CHAT_TOOL_CHOICE_NONE);++ const std::string GEN_PROMPT = data.generation_prompt;++ using mm3 = common_chat_peg_minimax_m3_mapper;++ if (inputs.has_continuation()) {+ const auto & msg = inputs.continue_msg;++ data.generation_prompt = GEN_PROMPT + THINK_START + msg.reasoning_content;+ if (inputs.continue_final_message == COMMON_CHAT_CONTINUATION_CONTENT) {+ data.generation_prompt += THINK_END + msg.render_content();+ }++ data.prompt += data.generation_prompt;+ }++ auto parser = build_chat_peg_parser([&](common_chat_peg_builder & p) {+ auto generation_prompt = p.prefix(GEN_PROMPT, THINK_START);+ auto end = p.end();++ auto reasoning = p.eps();+ if (extract_reasoning) {+ auto block = inputs.enable_thinking+ ? p.literal(THINK_START) + p.space() ++ p.ac(p.reasoning(p.until(THINK_END)) + p.literal(THINK_END), THINK_END)+ : p.literal(THINK_START) + p.ac(p.until(THINK_END) + p.literal(THINK_END), THINK_END);++ // A turn without reasoning is prefixed with a bare </mm:think>, written either by the+ // generation prompt (thinking_mode = "disabled") or by the model itself.+ reasoning = p.optional(p.choice({ block, p.literal(THINK_END) }));+ }++ if (has_response_format) {+ auto response_format = p.rule("response-format",+ p.literal("```json") + p.space() ++ p.content(p.schema(p.json(), "response-format-schema", inputs.json_schema)) ++ p.space() + p.literal("```"));+ return generation_prompt + reasoning + response_format + end;+ }++ if (!has_tools || inputs.tool_choice == COMMON_CHAT_TOOL_CHOICE_NONE) {+ return generation_prompt + reasoning + p.content(p.rest()) + end;+ }++ auto alternatives_of = [](const json & schema) -> std::optional<json> {+ for (const auto * keyword : { "oneOf", "anyOf" }) {+ if (schema.contains(keyword) && schema.at(keyword).is_array() && !schema.at(keyword).empty()) {+ return schema.at(keyword);+ }+ }+ return std::nullopt;+ };++ auto tool_choice = p.choice();+ foreach_function(inputs.tools, [&](const json & tool) {+ const auto & function = tool.at("function");+ std::string name = function.at("name");+ auto params = function.contains("parameters") ? function.at("parameters") : json::object();++ auto schema_info = common_schema_info();+ schema_info.resolve_refs(params);++ // The template expands argument values recursively in XML (see the to_xml() macro)+ std::function<common_peg_parser(const json &, const std::string &, const std::string &)> value_of;+ std::function<common_peg_parser(const json &, const std::string &)> members_of;++ auto element_of = [&](const std::string & tag, const json & schema, const std::string & rule_name) {+ const std::string close = NS + "</" + tag + ">";+ return p.rule(rule_name,+ p.tool_arg(+ p.tool_arg_open(+ p.literal(NS + "<") ++ p.tool_arg_name(p.literal(tag)) ++ p.literal(">")) ++ value_of(schema, rule_name, close)));+ };++ value_of = [&](const json & schema,+ const std::string & rule_name,+ const std::string & close) -> common_peg_parser {+ auto close_tag = p.tool_arg_close(p.literal(close));++ // A string accepts anything, so a union with a string alternative is a string+ if (schema_info.resolves_to_string(schema)) {+ return p.ac(p.tool_arg_string_value(p.until(close)) + close_tag, close);+ }++ if (auto alternatives = alternatives_of(schema)) {+ std::vector<common_peg_parser> choices;++ size_t index = 0;+ for (const auto & alternative : *alternatives) {+ const std::string alt_name = rule_name + "-" + std::to_string(index++);++ // There is a risk that this breaks streaming deltas, but that's a risk we+ // assume to provide tool arg streaming.+ choices.push_back(value_of(alternative, alt_name, close));+ }++ return p.choice(choices);+ }++ const std::string type = schema.contains("type") && schema.at("type").is_string()+ ? schema.at("type").get<std::string>()+ : "";++ if (type == "object" && schema.contains("properties")) {+ return p.tag(mm3::TOOL_ARG_OBJECT, members_of(schema, rule_name)) + p.space() + close_tag;+ }++ if (type == "array" && schema.contains("items")) {+ const std::string item_close = NS + "</item>";+ auto item = p.rule(rule_name + "-item",+ p.tag(mm3::TOOL_ARG_ITEM,+ p.literal(NS + "<item>") ++ value_of(schema.at("items"), rule_name + "-item", item_close)));+ return p.tag(mm3::TOOL_ARG_ARRAY, p.repeat(p.space() + item, 0, -1)) + p.space() + close_tag;+ }++ return p.tool_arg_json_value(p.schema(p.json(), rule_name + "-schema", schema, false)) + close_tag;+ };++ // Required properties in schema order, then any number of optional ones in any order.+ members_of = [&](const json & schema, const std::string & rule_prefix) -> common_peg_parser {+ const auto & props = schema.at("properties");++ std::set<std::string> required;+ if (schema.contains("required")) {+ required = schema.at("required").get<std::set<std::string>>();+ }++ std::vector<common_peg_parser> required_elements;+ std::vector<common_peg_parser> optional_elements;+ for (const auto & [key, key_schema] : props.items()) {+ auto element = element_of(key, key_schema, rule_prefix + "-" + key);+ if (required.find(key) != required.end()) {+ required_elements.push_back(element);+ } else {+ optional_elements.push_back(element);+ }+ }++ common_peg_parser members = p.eps();+ for (size_t i = 0; i < required_elements.size(); i++) {+ if (i > 0) {+ members = members + p.space();+ }+ members = members + required_elements[i];+ }++ if (!optional_elements.empty()) {+ common_peg_parser any_optional = p.choice();+ for (const auto & element : optional_elements) {+ any_optional |= element;+ }+ members = members + p.repeat(p.space() + any_optional, 0, -1);+ }++ return members;+ };++ common_peg_parser invoke_body =+ params.contains("properties") ? members_of(params, "tool-" + name + "-arg") : p.eps();++ auto func_parser = p.tool(+ p.tool_open(p.literal(NS + "<invoke name=\"") ++ p.tool_name(p.literal(name)) + p.literal("\">")) ++ p.space() + invoke_body + p.space() ++ p.tool_close(p.literal(INVOKE_END)));++ tool_choice |= p.rule("tool-" + name, func_parser);+ });++ auto require_tools = inputs.tool_choice == COMMON_CHAT_TOOL_CHOICE_REQUIRED;++ common_peg_parser tool_calls = p.eps();+ if (inputs.parallel_tool_calls) {+ tool_calls = p.trigger_rule("tool-call",+ p.literal(FC_START) + p.space() + tool_choice ++ p.zero_or_more(p.space() + tool_choice) + p.space() + p.literal(FC_END));+ } else {+ tool_calls = p.trigger_rule("tool-call",+ p.literal(FC_START) + p.space() + tool_choice + p.space() + p.literal(FC_END));+ }++ if (!require_tools) {+ tool_calls = p.optional(tool_calls);+ }++ auto content_before_tools = p.content(p.until(FC_START));+ return generation_prompt + reasoning + content_before_tools + tool_calls + end;+ });++ data.parser = parser.save();++ if (include_grammar) {+ data.grammar_lazy = !(has_response_format || (has_tools && inputs.tool_choice == COMMON_CHAT_TOOL_CHOICE_REQUIRED));+ data.grammar = build_grammar([&](const common_grammar_builder & builder) {+ foreach_function(inputs.tools, [&](const json & tool) {+ const auto & function = tool.at("function");+ auto schema = function.contains("parameters") ? function.at("parameters") : json::object();+ builder.resolve_refs(schema);+ });+ if (has_response_format) {+ auto schema = inputs.json_schema;+ builder.resolve_refs(schema);+ }+ parser.build_grammar(builder, data.grammar_lazy);+ });++ data.grammar_triggers = {+ { COMMON_GRAMMAR_TRIGGER_TYPE_WORD, FC_START },+ };+ }++ return data;+}diff --git a/common/parsers/ministral3.cpp b/common/parsers/ministral3.cppnew file mode 100644index 0000000000..075f14dbc1--- /dev/null+++ b/common/parsers/ministral3.cpp@@ -0,0 +1,135 @@+#include "parsers.h"++common_chat_params common_chat_params_init_ministral_3(const common_chat_template & tmpl,+ const autoparser::generation_params & inputs) {+ common_chat_params data;++ // Build up messages to follow the format: https://huggingface.co/mistralai/Ministral-3-14B-Reasoning-2512/blob/main/chat_template.jinja+ auto adjusted_messages = json::array();+ for (const auto & msg : inputs.messages) {+ auto role = msg.value("role", "");+ if (role != "system" && role != "assistant") {+ // Only adjust system and assistant messages. Interestingly, the system message may contain thinking.+ adjusted_messages.push_back(msg);+ continue;+ }++ auto content = json::array();++ // If message contains `reasoning_content`, add it as a block of type `thinking`+ if (msg.contains("reasoning_content") && msg.at("reasoning_content").is_string()) {+ content.push_back({+ { "type", "thinking" },+ { "thinking", msg.at("reasoning_content").get<std::string>() },+ });+ }++ // If message contains `content`, add it as a block of type `text`+ if (msg.contains("content")) {+ if (msg.at("content").is_string()) {+ content.push_back({+ { "type", "text" },+ { "text", msg.at("content").get<std::string>() },+ });+ } else if (msg.at("content").is_array()) {+ auto blocks = msg.at("content");+ content.insert(blocks);+ }+ }++ auto adjusted = msg;+ adjusted["content"] = content;+ adjusted.erase("reasoning_content");+ adjusted_messages.push_back(adjusted);+ }++ auto has_tools = inputs.tools.is_array() && !inputs.tools.empty();+ auto has_response_format = inputs.json_schema.is_object() && !inputs.json_schema.empty();+ auto extract_reasoning = inputs.reasoning_format != COMMON_REASONING_FORMAT_NONE;+ auto include_grammar = true;++ data.supports_thinking = true;+ data.thinking_start_tag = "[THINK]";+ data.thinking_end_tags = {"[/THINK]"};+ data.prompt = common_chat_template_direct_apply_impl(tmpl, inputs, /* messages_override = */ adjusted_messages);+ data.generation_prompt = common_chat_template_generation_prompt_impl(tmpl, inputs, /* messages_override = */ adjusted_messages);+ data.format = COMMON_CHAT_FORMAT_PEG_NATIVE;+ data.preserved_tokens = {+ "[THINK]",+ "[/THINK]",+ "[TOOL_CALLS]",+ "[ARGS]",+ };++ if (inputs.has_continuation()) {+ const auto & msg = inputs.continue_msg;++ data.generation_prompt = "[THINK]" + msg.reasoning_content;+ if (inputs.continue_final_message == COMMON_CHAT_CONTINUATION_CONTENT) {+ data.generation_prompt += "[/THINK]" + msg.render_content();+ }++ data.prompt += data.generation_prompt;+ }++ auto parser = build_chat_peg_parser([&](common_chat_peg_builder & p) {+ auto generation_prompt = p.eps();+ auto reasoning =+ extract_reasoning ? p.optional("[THINK]" + p.reasoning(p.until("[/THINK]")) + "[/THINK]") : p.eps();++ // Response format parser+ if (has_response_format) {+ // Ministral wants to emit json surrounded by code fences+ return generation_prompt + (reasoning << "```json" << p.content(p.schema(p.json(), "response-format", inputs.json_schema)) << "```");+ }++ // Tool call parser+ if (has_tools && inputs.tool_choice != COMMON_CHAT_TOOL_CHOICE_NONE) {+ auto tool_choice = p.choice();+ foreach_function(inputs.tools, [&](const json & tool) {+ const auto & function = tool.at("function");+ std::string name = function.at("name");+ const auto & schema = function.at("parameters");++ tool_choice |=+ p.rule("tool-" + name, p.tool_open(p.tool_name(p.literal(name)) + "[ARGS]") ++ p.tool_args(p.schema(p.json(), "tool-" + name + "-schema", schema)));+ });++ auto min_calls = inputs.tool_choice == COMMON_CHAT_TOOL_CHOICE_REQUIRED ? 1 : 0;+ auto max_calls = inputs.parallel_tool_calls ? -1 : 1;+ auto tool_calls = p.trigger_rule("tool-call", p.repeat("[TOOL_CALLS]" + tool_choice, min_calls, max_calls));++ return generation_prompt + (reasoning << p.content(p.until("[TOOL_CALLS]")) << tool_calls);+ }++ // Content only parser+ include_grammar = false;+ return generation_prompt + (reasoning << p.content(p.rest()));+ });++ data.parser = parser.save();++ if (include_grammar) {+ data.grammar_lazy = has_tools && inputs.tool_choice == COMMON_CHAT_TOOL_CHOICE_AUTO;++ data.grammar = build_grammar([&](const common_grammar_builder & builder) {+ foreach_function(inputs.tools, [&](const json & tool) {+ const auto & function = tool.at("function");+ auto schema = function.at("parameters");+ builder.resolve_refs(schema);+ });+ if (has_response_format) {+ auto schema = inputs.json_schema;+ builder.resolve_refs(schema);+ }+ parser.build_grammar(builder, data.grammar_lazy);+ });++ data.grammar_triggers = {+ { COMMON_GRAMMAR_TRIGGER_TYPE_WORD, "[TOOL_CALLS]" }+ };+ }++ return data;+}diff --git a/common/parsers/muse-glimmer.cpp b/common/parsers/muse-glimmer.cppnew file mode 100644index 0000000000..7f4dfcd511--- /dev/null+++ b/common/parsers/muse-glimmer.cpp@@ -0,0 +1,148 @@+#include "parsers.h"++// An assistant turn is rendered as one or more messages, each+// "<|start|>assistant to=<recipient><|message|>{content}{END}" where END is+// <|eom|> (more messages follow) or <|eot|> (end of turn):+// - chain-of-thought: to=self, terminated by <|eom|>+// - final answer: to=user, terminated by <|eot|>+// The generation prompt is just "<|start|>assistant"; the model emits its own+// " to=...<|message|>".+common_chat_params common_chat_params_init_muse_glimmer(const common_chat_template & tmpl,+ const autoparser::generation_params & inputs) {+ common_chat_params data;++ data.prompt = common_chat_template_direct_apply_impl(tmpl, inputs);+ data.generation_prompt = "<|start|>assistant";+ data.format = COMMON_CHAT_FORMAT_PEG_NATIVE;+ data.supports_thinking = true;++ data.preserved_tokens = {+ "<|start|>", "<|message|>", "<|eom|>", "<|eot|>",+ // ATEM tool-call markup emitted on " to=<tool>" turns.+ "<atem:function_calls>", "<atem:invoke", "<atem:parameter", "</atem:parameter>",+ "</atem:invoke>", "</atem:function_calls>",+ };++ data.message_delimiters = {+ { COMMON_CHAT_ROLE_ASSISTANT, "<|start|>assistant" },+ { COMMON_CHAT_ROLE_USER, "<|start|>user" },+ { COMMON_CHAT_ROLE_SYSTEM, "<|start|>system" },+ { COMMON_CHAT_ROLE_TOOL, "<|start|>tool" },+ };++ if (inputs.has_continuation()) {+ const auto & msg = inputs.continue_msg;++ data.generation_prompt = "<|start|>assistant to=self<|message|>" + msg.reasoning_content;+ if (inputs.continue_final_message == COMMON_CHAT_CONTINUATION_CONTENT) {+ data.generation_prompt += "<|eom|><|start|>assistant to=user<|message|>" + msg.render_content();+ }++ data.prompt += data.generation_prompt;+ }++ auto extract_reasoning = inputs.reasoning_format != COMMON_REASONING_FORMAT_NONE;++ auto has_tools = inputs.tools.is_array() && !inputs.tools.empty();+ // Constrained grammar whenever tools are offered.+ auto include_grammar = has_tools && inputs.tool_choice != COMMON_CHAT_TOOL_CHOICE_NONE;++ auto parser = build_chat_peg_parser([&](common_chat_peg_builder & p) {+ auto start = p.rule("start", p.literal("<|start|>assistant"));++ if (!extract_reasoning && !include_grammar) {+ return start + p.content(p.rest());+ }++ if (extract_reasoning) {+ p.rule("analysis", p.literal(" to=self<|message|>") + p.reasoning(p.until("<|eom|>")) + p.literal("<|eom|>"));+ } else {+ p.rule("analysis", p.literal(" to=self<|message|>") + p.content(p.until("<|eom|>")) + p.literal("<|eom|>"));+ }+ auto analysis = p.ref("analysis");++ auto recipient = p.optional(p.literal(" to=user"));+ auto final_msg = p.rule("final", recipient + p.literal("<|message|>") ++ p.content(p.until_one_of({ "<|eot|>", "<|eom|>" })));++ if (has_tools && inputs.tool_choice != COMMON_CHAT_TOOL_CHOICE_NONE) {+ auto string_value = p.ac(+ p.tool_arg_string_value(p.until("</atem:parameter>")) + p.tool_arg_close(p.literal("</atem:parameter>")),+ "</atem:parameter>");++ auto tool_choice = p.choice();+ foreach_function(inputs.tools, [&](const json & tool) {+ const auto & function = tool.at("function");+ const std::string name = function.at("name");+ auto params = function.contains("parameters") ? function.at("parameters") : json::object();++ auto args = p.eps();+ if (params.contains("properties") && params.at("properties").is_object() && !params.at("properties").empty()) {+ auto schema_info = common_schema_info();+ schema_info.resolve_refs(params);++ auto arg_choice = p.choice();+ for (const auto & [prop_name, prop_schema] : params.at("properties").items()) {+ auto value_parser = p.eps();+ if (schema_info.resolves_to_string(prop_schema)) {+ value_parser = string_value;+ } else {+ value_parser = p.tool_arg_json_value(+ p.schema(p.json(), "tool-" + name + "-arg-" + prop_name + "-schema", prop_schema, false))+ + p.tool_arg_close(p.literal("</atem:parameter>"));+ }++ auto arg_rule = p.tool_arg(+ p.tool_arg_open(p.literal("<atem:parameter name=\"") + p.tool_arg_name(p.literal(prop_name)) + p.literal("\">")) ++ value_parser);++ arg_choice |= arg_rule;+ }+ args = p.zero_or_more(arg_choice + p.space());+ }++ auto tool_parser = p.tool(+ p.tool_open(p.literal(" to=") + p.until("<|message|>") ++ p.literal("<|message|><atem:function_calls>") + p.space() ++ p.literal("<atem:invoke name=\"") + p.tool_name(p.literal(name)) + p.literal("\">") + p.space())+ << p.tool_args(args)+ << p.tool_close(p.literal("</atem:invoke>") + p.space() + p.literal("</atem:function_calls>")));++ tool_choice |= p.rule("tool-" + name, tool_parser);+ });++ auto tool_calls = inputs.parallel_tool_calls+ ? p.trigger_rule("tool-call", tool_choice + p.zero_or_more(p.literal("<|eom|>") + start + tool_choice))+ : p.trigger_rule("tool-call", tool_choice);+++ if (inputs.tool_choice == COMMON_CHAT_TOOL_CHOICE_REQUIRED) {+ return p.zero_or_more(start + analysis) + start + tool_calls;+ }+ auto trailing_calls = p.optional(p.literal("<|eom|>") + start + tool_calls);+ return p.zero_or_more(start + analysis) + start + (tool_calls | (final_msg + trailing_calls));+ }++ return p.zero_or_more(start + analysis) + start + final_msg;+ });++ data.parser = parser.save();++ if (include_grammar) {+ data.grammar_lazy = inputs.tool_choice != COMMON_CHAT_TOOL_CHOICE_REQUIRED;+ data.grammar = build_grammar([&](const common_grammar_builder & builder) {+ foreach_function(inputs.tools, [&](const json & tool) {+ const auto & function = tool.at("function");+ auto schema = function.contains("parameters") ? function.at("parameters") : json::object();+ builder.resolve_refs(schema);+ });+ parser.build_grammar(builder, data.grammar_lazy);+ });+ data.grammar_triggers = {+ { COMMON_GRAMMAR_TRIGGER_TYPE_PATTERN,+ "<\\|start\\|>assistant( to=(?!self<\\|message\\|>)(?!user<\\|message\\|>)[^<]*?<\\|message\\|>)" },+ };+ }++ return data;+}diff --git a/common/parsers/parsers.cpp b/common/parsers/parsers.cppnew file mode 100644index 0000000000..0a4d5cfbb5--- /dev/null+++ b/common/parsers/parsers.cpp@@ -0,0 +1,34 @@+#include "parsers.h"++#include "log.h"++#include <set>++void foreach_function(const json & tools, const std::function<void(const json &)> & fn) {+ for (const auto & tool : tools) {+ if (!tool.contains("type") || tool.at("type") != "function" || !tool.contains("function")) {+ LOG_INF("Skipping tool without function: %s", tool.dump(2).c_str());+ continue;+ }+ fn(tool);+ }+}++void foreach_parameter(const json & function, const std::function<void(const std::string &, const json &, bool)> & fn) {+ if (!function.contains("parameters") || !function.at("parameters").is_object()) {+ return;+ }+ const auto & params = function.at("parameters");+ if (!params.contains("properties") || !params.at("properties").is_object()) {+ return;+ }+ const auto & props = params.at("properties");+ std::set<std::string> required;+ if (params.contains("required") && params.at("required").is_array()) {+ required = params.at("required").get<std::set<std::string>>();+ }+ for (const auto & [name, prop] : props.items()) {+ bool is_required = (required.find(name) != required.end());+ fn(name, prop, is_required);+ }+}diff --git a/common/parsers/parsers.h b/common/parsers/parsers.hnew file mode 100644index 0000000000..7898f00071--- /dev/null+++ b/common/parsers/parsers.h@@ -0,0 +1,77 @@+#pragma once++#include "chat.h"+#include "chat-auto-parser.h"+#include "chat-auto-parser-helpers.h"+#include "chat-peg-parser.h"+#include "common.h"+#include "ggml.h"+#include "json-schema-to-grammar.h"+#include "json.h"++#include <functional>+#include <optional>+#include <set>+#include <string>+#include <vector>++using json = common_json;++// iterate over the function tools of an OpenAI-style tools array+void foreach_function(const json & tools, const std::function<void(const json &)> & fn);++// iterate over the parameters of a function tool, flagging the ones listed as required+void foreach_parameter(const json & function, const std::function<void(const std::string &, const json &, bool)> & fn);++// render a template; the override arguments let a parser feed in messages, tools or context it has rewritten+std::string common_chat_template_direct_apply_impl(+ const common_chat_template & tmpl,+ const autoparser::generation_params & inputs,+ const std::optional<json> & messages_override = std::nullopt,+ const std::optional<json> & tools_override = std::nullopt,+ const std::optional<json> & additional_context = std::nullopt);++// the suffix a template appends when add_generation_prompt is set+std::string common_chat_template_generation_prompt_impl(+ const common_chat_template & tmpl,+ const autoparser::generation_params & inputs,+ const std::optional<json> & messages_override = std::nullopt,+ const std::optional<json> & tools_override = std::nullopt,+ const std::optional<json> & additional_context = std::nullopt);++bool is_lfm2_template(const std::string & src);++namespace workaround {++void convert_tool_responses_gemma4(json & messages);++}++common_chat_params common_chat_params_init_cohere2moe(const common_chat_template & tmpl, const autoparser::generation_params & inputs);++common_chat_params common_chat_params_init_deepseek_v3_2(const common_chat_template & tmpl, const autoparser::generation_params & inputs);++common_chat_params common_chat_params_init_functionary_v3_2(const common_chat_template & tmpl, const autoparser::generation_params & inputs);++common_chat_params common_chat_params_init_gemma4(const common_chat_template & tmpl, const autoparser::generation_params & inputs);++common_chat_params common_chat_params_init_gigachat_v3(const common_chat_template & tmpl, const autoparser::generation_params & inputs);++common_chat_params common_chat_params_init_gpt_oss(const common_chat_template & tmpl, const autoparser::generation_params & inputs);++common_chat_params common_chat_params_init_kimi_k2(const common_chat_template & tmpl, const autoparser::generation_params & inputs);++common_chat_params common_chat_params_init_kimi_k3(const common_chat_template & tmpl, const autoparser::generation_params & inputs);++// tool_list_tokens preserves the LFM2 system tool-list markers; LFM2.5 renders without them+common_chat_params common_chat_params_init_lfm2(const common_chat_template & tmpl, const autoparser::generation_params & inputs, bool tool_list_tokens);++common_chat_params common_chat_params_init_minicpm5(const common_chat_template & tmpl, const autoparser::generation_params & inputs);++common_chat_params common_chat_params_init_minimax_m3(const common_chat_template & tmpl, const autoparser::generation_params & inputs);++common_chat_params common_chat_params_init_ministral_3(const common_chat_template & tmpl, const autoparser::generation_params & inputs);++common_chat_params common_chat_params_init_muse_glimmer(const common_chat_template & tmpl, const autoparser::generation_params & inputs);++common_chat_params common_chat_params_init_qwen3_coder(const common_chat_template & tmpl, const autoparser::generation_params & inputs);diff --git a/common/parsers/qwen3-coder.cpp b/common/parsers/qwen3-coder.cppnew file mode 100644index 0000000000..8a1e521370--- /dev/null+++ b/common/parsers/qwen3-coder.cpp@@ -0,0 +1,181 @@+#include "parsers.h"++common_chat_params common_chat_params_init_qwen3_coder(const common_chat_template & tmpl,+ const autoparser::generation_params & inputs) {+ common_chat_params data;++ const std::string GEN_PREFIX = "<|im_start|>assistant\n";++ data.prompt = common_chat_template_direct_apply_impl(tmpl, inputs);+ data.generation_prompt = common_chat_template_generation_prompt_impl(tmpl, inputs);+ data.format = COMMON_CHAT_FORMAT_PEG_NATIVE;++ auto supports_reasoning = tmpl.source().find("<think>") != std::string::npos;++ data.supports_thinking = supports_reasoning;+ data.preserved_tokens = {+ "<tool_call>",+ "</tool_call>",+ };++ auto is_qwen3_coder = !supports_reasoning;++ if (supports_reasoning) {+ data.thinking_start_tag = "<think>";+ // Support both </think> and <tool_call> as reasoning end sequences.+ // <function= is omitted, as it is a workaround for Qwen3-Coder which is not a thinking model+ data.thinking_end_tags = { "</think>", "<tool_call>" };+ data.preserved_tokens.insert(data.preserved_tokens.end(), { "<think>", "</think>" });+ }++ data.message_delimiters = {+ { COMMON_CHAT_ROLE_ASSISTANT, "<|im_start|>assistant" },+ { COMMON_CHAT_ROLE_TOOL, "<|im_start|>user\n<tool_response>" }, // Qwen3-Coder, Qwen3.5, Nemotron Nano 3+ { COMMON_CHAT_ROLE_TOOL, "<|im_start|>tool_response" }, // StepFun-3.5-Flash+ { COMMON_CHAT_ROLE_USER, "<|im_start|>user" },+ { COMMON_CHAT_ROLE_SYSTEM, "<|im_start|>system" },+ };++ auto has_tools = inputs.tools.is_array() && !inputs.tools.empty();+ auto has_response_format = inputs.json_schema.is_object() && !inputs.json_schema.empty();+ auto extract_reasoning = inputs.reasoning_format != COMMON_REASONING_FORMAT_NONE;+ auto include_grammar = has_response_format || (has_tools && inputs.tool_choice != COMMON_CHAT_TOOL_CHOICE_NONE);++ if (inputs.has_continuation()) {+ const auto & msg = inputs.continue_msg;++ data.generation_prompt = GEN_PREFIX;+ if (supports_reasoning) {+ data.generation_prompt += "<think>\n" + msg.reasoning_content;+ if (inputs.continue_final_message == COMMON_CHAT_CONTINUATION_CONTENT) {+ data.generation_prompt += "\n</think>\n\n";+ }+ }+ if (inputs.continue_final_message == COMMON_CHAT_CONTINUATION_CONTENT) {+ data.generation_prompt += msg.render_content();+ }++ data.prompt += data.generation_prompt;+ }++ std::vector<std::string> tool_call_starts = { "<tool_call>" };++ if (is_qwen3_coder) {+ // Match complete <function=name> opener for Qwen3-Coder models that occasionally omit the+ // starting <tool_call>. The model may hallucinate a tool name, but it is preferable over+ // constraining on <function which may occur in valid content generation, e.g. #include <functional>+ foreach_function(inputs.tools, [&](const json & tool) {+ const std::string name = tool.at("function").at("name");+ tool_call_starts.push_back("<function=" + name + ">");+ });+ }++ auto parser = build_chat_peg_parser([&](common_chat_peg_builder & p) {+ auto generation_prompt = p.literal(GEN_PREFIX);++ auto reasoning = p.eps();+ if (supports_reasoning && extract_reasoning) {+ reasoning = p.optional("<think>" + p.space() ++ p.reasoning(p.until_one_of({ "</think>", "<tool_call>" })) ++ (p.literal("</think>") | p.peek(p.literal("<tool_call>"))));+ }++ // Response format parser+ if (has_response_format) {+ return generation_prompt + (reasoning << p.content(p.schema(p.json(), "response-format", inputs.json_schema)));+ }++ // Tool call parser+ if (has_tools && inputs.tool_choice != COMMON_CHAT_TOOL_CHOICE_NONE) {+ auto arg_close = p.tool_arg_close(p.literal("\n</parameter>\n"));+ auto arg_string = p.rule("xml-arg-string",+ p.ac(p.tool_arg_string_value(p.until("\n</parameter>\n")) + arg_close, "\n</parameter>\n"));++ auto tool_choice = p.choice();+ foreach_function(inputs.tools, [&](const json & tool) {+ const auto & function = tool.at("function");+ std::string name = function.at("name");+ auto parameters = function.contains("parameters") ? function.at("parameters") : json::object();++ auto schema_info = common_schema_info();+ schema_info.resolve_refs(parameters);++ std::vector<common_peg_parser> required_args;+ std::vector<common_peg_parser> optional_args;++ foreach_parameter(function, [&](const std::string & param_name, const json & param_schema, bool is_required) {+ auto rule_name = "tool-" + name + "-arg-" + param_name;++ auto arg_open = p.tool_arg_open("<parameter=" + p.tool_arg_name(p.literal(param_name)) + ">\n");++ auto arg_value = schema_info.resolves_to_string(param_schema) ?+ arg_string :+ p.tool_arg_json_value(p.schema(p.json(), rule_name + "-schema", param_schema)) + arg_close;++ auto arg_rule = p.rule(rule_name, p.tool_arg(arg_open + arg_value));++ (is_required ? required_args : optional_args).push_back(arg_rule);+ });++ // Accept required arguments in any order, as Qwen does not always adhere to the+ // order provided.+ auto args = p.permute("tool-" + name + "-args", required_args);+ if (!optional_args.empty()) {+ args = args + p.zero_or_more(p.choice(optional_args));+ }++ auto func = p.tool(p.tool_open("<function=" + p.tool_name(p.literal(name)) + ">\n") ++ p.tool_args(args) ++ p.tool_close(p.literal("</function>\n")));++ tool_choice |= p.rule("tool-" + name, func);+ });++ auto min_calls = inputs.tool_choice == COMMON_CHAT_TOOL_CHOICE_REQUIRED ? 1 : 0;++ auto tool_call_body = tool_choice + "</tool_call>" + p.space();+ auto tool_call = p.rule("tool-call", "<tool_call>\n" + tool_call_body);++ // Qwen3-Coder models may occasionally omit the <tool_call> token.+ auto tool_call_first = is_qwen3_coder ?+ p.rule("tool-call-first", p.optional(p.literal("<tool_call>\n")) + tool_call_body) :+ tool_call;++ auto calls = inputs.parallel_tool_calls ? tool_call_first + p.zero_or_more(tool_call) : tool_call_first;+ auto tool_calls = p.trigger_rule("tool-call-root", p.repeat(calls, min_calls, 1));++ return generation_prompt ++ (reasoning << p.content(p.until_one_of(tool_call_starts)) << tool_calls);+ }++ // Content only parser+ return generation_prompt + (reasoning << p.content(p.rest()));+ });++ data.parser = parser.save();++ if (include_grammar) {+ data.grammar_lazy = has_tools && inputs.tool_choice == COMMON_CHAT_TOOL_CHOICE_AUTO;++ data.grammar = build_grammar([&](const common_grammar_builder & builder) {+ foreach_function(inputs.tools, [&](const json & tool) {+ const auto & function = tool.at("function");+ auto schema = function.contains("parameters") ? function.at("parameters") : json::object();+ builder.resolve_refs(schema);+ });+ if (has_response_format) {+ auto schema = inputs.json_schema;+ builder.resolve_refs(schema);+ }+ parser.build_grammar(builder, data.grammar_lazy);+ });++ if (data.grammar_lazy) {+ for (const auto & start : tool_call_starts) {+ data.grammar_triggers.push_back({ COMMON_GRAMMAR_TRIGGER_TYPE_WORD, start });+ }+ }+ }++ return data;+}diff --git a/common/parsers/sources.cmake b/common/parsers/sources.cmakenew file mode 100644index 0000000000..9d7fb0992a--- /dev/null+++ b/common/parsers/sources.cmake@@ -0,0 +1,20 @@+# Specialized chat template parsers, listed explicitly so that adding or removing one re-runs CMake instead of leaving an incremental build stale.++set(LLAMA_CHAT_PARSERS_SOURCES+ ${CMAKE_CURRENT_LIST_DIR}/parsers.cpp+ ${CMAKE_CURRENT_LIST_DIR}/parsers.h+ ${CMAKE_CURRENT_LIST_DIR}/cohere2moe.cpp+ ${CMAKE_CURRENT_LIST_DIR}/deepseek.cpp+ ${CMAKE_CURRENT_LIST_DIR}/functionary-v3-2.cpp+ ${CMAKE_CURRENT_LIST_DIR}/gemma4.cpp+ ${CMAKE_CURRENT_LIST_DIR}/gigachat-v3.cpp+ ${CMAKE_CURRENT_LIST_DIR}/gpt-oss.cpp+ ${CMAKE_CURRENT_LIST_DIR}/kimi-k2.cpp+ ${CMAKE_CURRENT_LIST_DIR}/kimi-k3.cpp+ ${CMAKE_CURRENT_LIST_DIR}/lfm2.cpp+ ${CMAKE_CURRENT_LIST_DIR}/minicpm5.cpp+ ${CMAKE_CURRENT_LIST_DIR}/minimax-m3.cpp+ ${CMAKE_CURRENT_LIST_DIR}/ministral3.cpp+ ${CMAKE_CURRENT_LIST_DIR}/muse-glimmer.cpp+ ${CMAKE_CURRENT_LIST_DIR}/qwen3-coder.cpp+)