LLM inference in C/C++
C++ ggml
Fix DeepSeek4 crafted template (#25414)
Piotr Wilkin (ilintar)f534da26e4ab
diff --git a/common/chat.cpp b/common/chat.cppindex 22d2ee4a2a..c441021b97 100644--- a/common/chat.cpp+++ b/common/chat.cpp@@ -15,11 +15,13 @@#include "nlohmann/json.hpp"+#include <algorithm>#include <cstdio>#include <cstdlib>#include <ctime>#include <exception>#include <functional>+#include <map>#include <optional>#include <sstream>@@ -1855,12 +1857,89 @@ static common_chat_params common_chat_params_init_gigachat_v3(return data;}+// The DeepSeek V4 reference implementation renders consecutive tool results into a single+// user block, ordered by the tool call order of the preceding assistant message (matched+// by tool call id) rather than by the order they appear in the conversation.+static json deepseek_v4_sort_tool_results(const json & messages) {+ json adjusted = messages;+ std::map<std::string, size_t> call_order;++ for (size_t i = 0; i < adjusted.size();) {+ const auto & msg = adjusted[i];+ const auto role = msg.value("role", "");++ if (role == "assistant" && msg.contains("tool_calls") &&+ msg.at("tool_calls").is_array() && !msg.at("tool_calls").empty()) {+ call_order.clear();+ const auto & tool_calls = msg.at("tool_calls");+ for (size_t idx = 0; idx < tool_calls.size(); idx++) {+ auto id = tool_calls[idx].value("id", "");+ if (!id.empty()) {+ call_order[id] = idx;+ }+ }+ i++;+ continue;+ }++ if (role != "user" && role != "tool") {+ i++;+ continue;+ }++ // collect a maximal run of user/tool messages - they render into one user block+ std::vector<size_t> tool_positions;+ size_t run_end = i;+ for (; run_end < adjusted.size(); run_end++) {+ const auto r = adjusted[run_end].value("role", "");+ if (r == "tool") {+ tool_positions.push_back(run_end);+ } else if (r != "user") {+ break;+ }+ }++ if (tool_positions.size() > 1 && !call_order.empty()) {+ std::vector<json> results;+ results.reserve(tool_positions.size());+ for (auto pos : tool_positions) {+ results.push_back(adjusted[pos]);+ }+ std::stable_sort(results.begin(), results.end(), [&](const json & a, const json & b) {+ const auto order = [&](const json & m) {+ auto it = call_order.find(m.value("tool_call_id", ""));+ return it == call_order.end() ? (size_t) 0 : it->second;+ };+ return order(a) < order(b);+ });+ for (size_t k = 0; k < tool_positions.size(); k++) {+ adjusted[tool_positions[k]] = std::move(results[k]);+ }+ }++ i = run_end;+ }++ return adjusted;+}+static common_chat_params common_chat_params_init_deepseek_v3_2(const common_chat_template & tmpl,const autoparser::generation_params & inputs) {common_chat_params data;- data.prompt = common_chat_template_direct_apply_impl(tmpl, inputs);- data.generation_prompt = common_chat_template_generation_prompt_impl(tmpl, inputs);+ // V4 uses the same DSML markup as V3.2, but names the tool call block "tool_calls"+ // instead of "function_calls", renders tool results in tool call order and its+ // non-thinking generation prompt ends with a bare </think> instead of an empty+ // <think></think> pair.+ const bool is_v4 = tmpl.source().find("function_calls") == std::string::npos;++ std::optional<json> adjusted_messages;+ if (is_v4) {+ adjusted_messages = deepseek_v4_sort_tool_results(inputs.messages);+ }++ data.prompt = common_chat_template_direct_apply_impl(tmpl, inputs, adjusted_messages);+ data.generation_prompt = common_chat_template_generation_prompt_impl(tmpl, inputs, adjusted_messages);data.format = COMMON_CHAT_FORMAT_PEG_NATIVE;data.supports_thinking = true;data.thinking_start_tag = "<think>";@@ -1879,8 +1958,9 @@ static common_chat_params common_chat_params_init_deepseek_v3_2(const common_chaconst std::string DSML = "|DSML|";const std::string THINK_START = "<think>";const std::string THINK_END = "</think>";- const std::string FC_START = "<" + DSML + "function_calls>";- const std::string FC_END = "</" + DSML + "function_calls>";+ const std::string TC_BLOCK = is_v4 ? "tool_calls" : "function_calls";+ const std::string FC_START = "<" + DSML + TC_BLOCK + ">";+ const std::string FC_END = "</" + DSML + TC_BLOCK + ">";const std::string INVOKE_START = "<" + DSML + "invoke";const std::string INVOKE_END = "</" + DSML + "invoke>";const std::string PARAM_START = "<" + DSML + "parameter";@@ -1907,8 +1987,11 @@ static common_chat_params common_chat_params_init_deepseek_v3_2(const common_chareasoning = p.optional(THINK_START + p.reasoning(p.until(THINK_END)) + THINK_END);} else if (extract_reasoning) {// Thinking disabled but reasoning extraction requested: the generation prompt- // contains an empty <think></think> pair that must still be consumed.- reasoning = p.optional(p.literal(THINK_START) + p.until(THINK_END) + p.literal(THINK_END));+ // contains an empty <think></think> pair (V3.2) or a bare </think> (V4) that+ // must still be consumed.+ reasoning = is_v4+ ? p.optional(p.literal(THINK_END))+ : p.optional(p.literal(THINK_START) + p.until(THINK_END) + p.literal(THINK_END));}if (has_response_format) {@@ -2612,12 +2695,14 @@ std::optional<common_chat_params> common_chat_try_specialized_template(return common_chat_params_init_gigachat_v3(tmpl, params);}- // DeepSeek V3.2 format detection: template defines dsml_token and uses it for tool calls.+ // DeepSeek V3.2/V4 format detection: template defines dsml_token and uses it for tool calls.// The template source contains the token as a variable assignment, not as a literal in markup.+ // V3.2 names the tool call block "function_calls", V4 names it "tool_calls".if (src.find("dsml_token") != std::string::npos &&- src.find("function_calls") != std::string::npos &&- src.find("DSML") != std::string::npos) {- LOG_DBG("Using specialized template: DeepSeek V3.2\n");+ src.find("DSML") != std::string::npos &&+ (src.find("function_calls") != std::string::npos ||+ src.find("tool_calls") != std::string::npos)) {+ LOG_DBG("Using specialized template: DeepSeek V3.2/V4\n");return common_chat_params_init_deepseek_v3_2(tmpl, params);}diff --git a/common/jinja/caps.cpp b/common/jinja/caps.cppindex ae378ebd4f..0d9336c0d9 100644--- a/common/jinja/caps.cpp+++ b/common/jinja/caps.cpp@@ -23,6 +23,7 @@ void caps_apply_preserve_reasoning(jinja::context & ctx, bool enabled) {ctx.set_val("preserve_thinking", mk_val<value_bool>(enabled));ctx.set_val("clear_thinking", mk_val<value_bool>(!enabled));ctx.set_val("truncate_history_thinking", mk_val<value_bool>(!enabled));+ ctx.set_val("drop_reasoning", mk_val<value_bool>(!enabled));}static void caps_try_execute(jinja::program & prog,diff --git a/models/templates/deepseek-ai-DeepSeek-V4.jinja b/models/templates/deepseek-ai-DeepSeek-V4.jinjaindex f19f787b1b..ad1ff8ce23 100644--- a/models/templates/deepseek-ai-DeepSeek-V4.jinja+++ b/models/templates/deepseek-ai-DeepSeek-V4.jinja@@ -8,12 +8,15 @@{%- set thinking = false -%}{%- endif -%}{%- endif -%}+{%- if not drop_thinking is defined -%}+ {%- set drop_thinking = false -%}+{%- endif -%}{%- set dsml_token = '|DSML|' -%}{%- set thinking_start_token = '<think>' -%}{%- set thinking_end_token = '</think>' -%}{%- set tools_header = '## Tools\n\nYou have access to a set of tools to help answer the user\'s question. You can invoke tools by writing a "<' + dsml_token + 'tool_calls>" block like the following:\n\n<' + dsml_token + 'tool_calls>\n<' + dsml_token + 'invoke name="$TOOL_NAME">\n<' + dsml_token + 'parameter name="$PARAMETER_NAME" string="true|false">$PARAMETER_VALUE</' + dsml_token + 'parameter>\n...\n</' + dsml_token + 'invoke>\n<' + dsml_token + 'invoke name="$TOOL_NAME2">\n...\n</' + dsml_token + 'invoke>\n</' + dsml_token + 'tool_calls>\n\nString parameters should be specified as is and set `string="true"`. For all other types (numbers, booleans, arrays, objects), pass the value in JSON format and set `string="false"`.\n\nIf thinking_mode is enabled (triggered by ' + thinking_start_token + '), you MUST output your complete reasoning inside ' + thinking_start_token + '...' + thinking_end_token + ' BEFORE any tool calls or final response.\n\nOtherwise, output directly after ' + thinking_end_token + ' with tool calls or final response.\n\n### Available Tool Schemas\n\n' -%}{%- set tools_footer = '\nYou MUST strictly follow the above defined tool name and parameter schemas to invoke tool calls.\n' -%}-{%- set ns = namespace(system_prompt='', is_first_sp=true) -%}+{%- set ns = namespace(system_prompt='', is_first_sp=true, has_tool_calls=false) -%}{%- for message in messages -%}{%- if message['role'] == 'system' -%}{%- if ns.is_first_sp -%}@@ -46,6 +49,11 @@{%- endif -%}{%- endfor -%}{%- set state = namespace(in_user=false) -%}+{%- for message in messages -%}+ {%- if message['role'] == 'tool' -%}+ {%- set ns.has_tool_calls = true -%}+ {%- endif -%}+{%- endfor -%}{%- for message in messages -%}{%- if message['role'] == 'user' or message['role'] == 'developer' -%}{%- if state.in_user -%}@@ -67,7 +75,8 @@{%- set state.in_user = false -%}{{- '<|Assistant|>' -}}{%- set is_after_last_user = loop.index0 > last_user_idx.value -%}- {%- if is_after_last_user and thinking -%}+ {%- set retain_reasoning = (not drop_thinking) or (is_after_last_user or ns.has_tool_calls) -%}+ {%- if retain_reasoning and thinking -%}{{- thinking_start_token -}}{%- if message['reasoning_content'] is defined and message['reasoning_content'] -%}{{- message['reasoning_content'] -}}diff --git a/tests/test-chat.cpp b/tests/test-chat.cppindex 93685ec8ff..28b28ca0f4 100644--- a/tests/test-chat.cpp+++ b/tests/test-chat.cpp@@ -109,6 +109,15 @@ static void assert_contains(const std::string & haystack, const std::string & ne}}+static void assert_not_contains(const std::string & haystack, const std::string & needle) {+ if (haystack.find(needle) != std::string::npos) {+ LOG_ERR("Expected NOT to contain: %s\n", needle.c_str());+ LOG_ERR("Actual: %s\n", haystack.c_str());+ common_log_flush(common_log_main());+ throw std::runtime_error("Test failed");+ }+}+static void assert_ends_with(const std::string & str, const std::string & suffix) {if (str.size() < suffix.size() ||str.compare(str.size() - suffix.size(), suffix.size(), suffix) != 0) {@@ -4016,6 +4025,132 @@ static void test_template_output_peg_parsers(bool detailed_debug) {.run();}+ // DeepSeek V4 tests - same DSML markup as V3.2, but the tool call block is named+ // "tool_calls" and the non-thinking generation prompt ends in a bare </think>+ // instead of an empty <think></think> pair.+ {+ auto tst = peg_tester("models/templates/deepseek-ai-DeepSeek-V4.jinja", detailed_debug);++ // Pure content (non-thinking mode; generation prompt ends with </think>)+ tst.test("Hello, world!\nWhat's up?")+ .enable_thinking(false)+ .reasoning_format(COMMON_REASONING_FORMAT_DEEPSEEK)+ .expect(message_assist)+ .run();++ // Thinking + content+ tst.test("I'm\nthinking</think>Hello, world!\nWhat's up?")+ .enable_thinking(true)+ .reasoning_format(COMMON_REASONING_FORMAT_DEEPSEEK)+ .expect(message_assist_thoughts)+ .run();++ // Thinking + tool call (single, string param)+ tst.test(+ "Let me check the time</think>\n\n"+ "<|DSML|tool_calls>\n"+ "<|DSML|invoke name=\"get_time\">\n"+ "<|DSML|parameter name=\"city\" string=\"true\">Tokyo</|DSML|parameter>\n"+ "</|DSML|invoke>\n"+ "</|DSML|tool_calls>")+ .enable_thinking(true)+ .reasoning_format(COMMON_REASONING_FORMAT_DEEPSEEK)+ .tools({ get_time_tool })+ .expect(message_with_tool_calls_and_reasoning("get_time", R"({"city": "Tokyo"})", "Let me check the time"))+ .run();++ // Tool call without reasoning (non-thinking mode), integer param (string="false")+ tst.test(+ "<|DSML|tool_calls>\n"+ "<|DSML|invoke name=\"special_function\">\n"+ "<|DSML|parameter name=\"arg1\" string=\"false\">1</|DSML|parameter>\n"+ "</|DSML|invoke>\n"+ "</|DSML|tool_calls>")+ .enable_thinking(false)+ .reasoning_format(COMMON_REASONING_FORMAT_DEEPSEEK)+ .tools({ special_function_tool })+ .expect(message_assist_call)+ .run();++ // Multiple parallel tool calls with reasoning+ tst.test(+ "Calling both</think>\n\n"+ "<|DSML|tool_calls>\n"+ "<|DSML|invoke name=\"get_time\">\n"+ "<|DSML|parameter name=\"city\" string=\"true\">Paris</|DSML|parameter>\n"+ "</|DSML|invoke>\n"+ "<|DSML|invoke name=\"get_weather\">\n"+ "<|DSML|parameter name=\"city\" string=\"true\">Paris</|DSML|parameter>\n"+ "</|DSML|invoke>\n"+ "</|DSML|tool_calls>")+ .enable_thinking(true)+ .reasoning_format(COMMON_REASONING_FORMAT_DEEPSEEK)+ .parallel_tool_calls(true)+ .tools({ get_time_tool, get_weather_tool })+ .expect(message_with_reasoning_content_and_multiple_tool_calls(+ "Calling both", "",+ { { "get_time", R"({"city": "Paris"})" }, { "get_weather", R"({"city": "Paris"})" } }))+ .run();++ // Tool call with content before tool calls+ tst.test(+ "Thinking about it</think>"+ "Let me call the function.\n\n"+ "<|DSML|tool_calls>\n"+ "<|DSML|invoke name=\"special_function\">\n"+ "<|DSML|parameter name=\"arg1\" string=\"false\">1</|DSML|parameter>\n"+ "</|DSML|invoke>\n"+ "</|DSML|tool_calls>")+ .enable_thinking(true)+ .reasoning_format(COMMON_REASONING_FORMAT_DEEPSEEK)+ .tools({ special_function_tool })+ .expect_reasoning("Thinking about it")+ .expect_content("Let me call the function.")+ .expect_tool_calls({+ { "special_function", R"({"arg1": 1})", {} },+ })+ .run();++ // Tool call with multiple params (mixed types)+ tst.test(+ "Multi-arg call</think>\n\n"+ "<|DSML|tool_calls>\n"+ "<|DSML|invoke name=\"magic_int\">\n"+ "<|DSML|parameter name=\"ref\" string=\"false\">42</|DSML|parameter>\n"+ "<|DSML|parameter name=\"name\" string=\"true\">foo bar</|DSML|parameter>\n"+ "</|DSML|invoke>\n"+ "</|DSML|tool_calls>")+ .enable_thinking(true)+ .reasoning_format(COMMON_REASONING_FORMAT_DEEPSEEK)+ .tools({ magic_int_tool })+ .expect_reasoning("Multi-arg call")+ .expect_tool_calls({+ { "magic_int", R"({"ref": 42, "name": "foo bar"})", {} },+ })+ .run();++ // Continuation tests+ tst.test("world!\nWhat's up?")+ .reasoning_format(COMMON_REASONING_FORMAT_DEEPSEEK)+ .enable_thinking(true)+ .messages({ message_user, message_assist_prefill_content })+ .add_generation_prompt(false)+ .continue_final_message(COMMON_CHAT_CONTINUATION_CONTENT)+ .expect_reasoning("I'm thinking")+ .expect_content("Hello, world!\nWhat's up?")+ .run();++ tst.test(" thinking</think>Hello, world!\nWhat's up?")+ .reasoning_format(COMMON_REASONING_FORMAT_DEEPSEEK)+ .enable_thinking(true)+ .messages({ message_user, message_assist_prefill_reasoning })+ .add_generation_prompt(false)+ .continue_final_message(COMMON_CHAT_CONTINUATION_REASONING)+ .expect_reasoning("I'm thinking")+ .expect_content("Hello, world!\nWhat's up?")+ .run();+ }+// GLM-4.6 tests - format: <tool_call>function_name\n<arg_key>...</arg_key>\n<arg_value>...</arg_value>\n</tool_call>{auto tst = peg_tester("models/templates/GLM-4.6.jinja", detailed_debug);@@ -5918,6 +6053,144 @@ static void test_developer_role_to_system_workaround() {}}+// Verify reasoning-trace retention rules in the DeepSeek-V4 template:+// all traces are retained unless drop_thinking is true AND the conversation+// has no tool calls, in which case only the last (after-final-user) trace is+// kept and earlier ones are dropped.+static void test_deepseek_v4_thinking_retention() {+ LOG_DBG("%s\n", __func__);++ auto tmpls = read_templates("models/templates/deepseek-ai-DeepSeek-V4.jinja");++ common_chat_msg user_q1; user_q1.role = "user"; user_q1.content = "Question 1";+ common_chat_msg user_q2; user_q2.role = "user"; user_q2.content = "Question 2";+ common_chat_msg asst_a1 = simple_assist_msg("Answer 1", "thinking A1");+ common_chat_msg asst_a2 = simple_assist_msg("Answer 2", "thinking A2");++ common_chat_msg tool_assist = message_with_tool_calls("special_function", "{\"arg1\": 1}");+ common_chat_msg tool_result; tool_result.role = "tool";+ tool_result.tool_name = "special_function"; tool_result.tool_call_id = "0"; tool_result.content = "result";++ // The template uses U+FF5C as the role separator and literal think tags+ // for the reasoning block.+ const std::string asst_marker = "<\xef\xbd\x9c" "Assistant" "\xef\xbd\x9c>";+ // Built via concatenation so the thinking tokens are not interpreted by+ // tooling processing this source file.+ const std::string think_start = "<" "think" ">";+ const std::string think_end = "</" "think" ">";++ const std::string think_a1 = asst_marker + think_start + "thinking A1" + think_end;+ const std::string think_a2 = asst_marker + think_start + "thinking A2" + think_end;+ const std::string asst_no_think = asst_marker + think_end;++ auto render = [&](const std::vector<common_chat_msg> & messages, bool drop_thinking) {+ common_chat_templates_inputs inputs;+ inputs.messages = messages;+ inputs.add_generation_prompt = false;+ inputs.chat_template_kwargs["thinking"] = "true";+ inputs.chat_template_kwargs["drop_thinking"] = drop_thinking ? "true" : "false";+ return common_chat_templates_apply(tmpls.get(), inputs).prompt;+ };++ // No tools, drop_thinking=false: all reasoning is retained.+ {+ auto prompt = render({ user_q1, asst_a1, user_q2, asst_a2 }, /* drop_thinking = */ false);+ assert_contains(prompt, think_a1);+ assert_contains(prompt, think_a2);+ }++ // No tools, drop_thinking=true: only the last reasoning trace is kept,+ // earlier ones are dropped (the assistant block emits just the end token).+ {+ auto prompt = render({ user_q1, asst_a1, user_q2, asst_a2 }, /* drop_thinking = */ true);+ assert_not_contains(prompt, think_a1);+ assert_contains(prompt, think_a2);+ // The dropped assistant turn still opens with the marker + bare end token.+ assert_contains(prompt, asst_no_think + "Answer 1");+ }++ // Single assistant turn, drop_thinking=true: the only trace is the last+ // one, so it must be retained even with drop_thinking set.+ {+ auto prompt = render({ user_q1, asst_a1 }, /* drop_thinking = */ true);+ assert_contains(prompt, think_a1);+ }++ // Single assistant turn, drop_thinking=false: reasoning is retained.+ {+ auto prompt = render({ user_q1, asst_a1 }, /* drop_thinking = */ false);+ assert_contains(prompt, think_a1);+ }++ // With tool calls, drop_thinking=true: tool presence forces all reasoning+ // to be retained, including the pre-tool-call trace.+ {+ auto prompt = render({ user_q1, asst_a1, user_q2, tool_assist, tool_result, asst_a2 },+ /* drop_thinking = */ true);+ assert_contains(prompt, think_a1);+ assert_contains(prompt, think_a2);+ }++ // With tool calls, drop_thinking=false: all reasoning retained.+ {+ auto prompt = render({ user_q1, asst_a1, user_q2, tool_assist, tool_result, asst_a2 },+ /* drop_thinking = */ false);+ assert_contains(prompt, think_a1);+ assert_contains(prompt, think_a2);+ }+}++// Verify that consecutive tool results are rendered in the tool call order of the+// preceding assistant message (matched by tool call id), as required by the reference+// DeepSeek-V4 implementation.+static void test_deepseek_v4_tool_result_ordering() {+ LOG_DBG("%s\n", __func__);++ auto tmpls = read_templates("models/templates/deepseek-ai-DeepSeek-V4.jinja");++ common_chat_msg user_q; user_q.role = "user"; user_q.content = "Question";++ common_chat_msg assist_calls;+ assist_calls.role = "assistant";+ assist_calls.tool_calls.push_back({ "get_time", "{\"city\": \"Paris\"}", "call_1" });+ assist_calls.tool_calls.push_back({ "get_weather", "{\"city\": \"Paris\"}", "call_2" });++ common_chat_msg time_result; time_result.role = "tool";+ time_result.tool_name = "get_time"; time_result.tool_call_id = "call_1"; time_result.content = "12:00";+ common_chat_msg weather_result; weather_result.role = "tool";+ weather_result.tool_name = "get_weather"; weather_result.tool_call_id = "call_2"; weather_result.content = "sunny";++ auto render = [&](const std::vector<common_chat_msg> & messages) {+ common_chat_templates_inputs inputs;+ inputs.messages = messages;+ inputs.add_generation_prompt = false;+ return common_chat_templates_apply(tmpls.get(), inputs).prompt;+ };++ // Results sent out of order are reordered to match the tool call order.+ {+ auto prompt = render({ user_q, assist_calls, weather_result, time_result });+ assert_contains(prompt, "<tool_result>12:00</tool_result>\n\n<tool_result>sunny</tool_result>");+ }++ // Results already in call order stay put.+ {+ auto prompt = render({ user_q, assist_calls, time_result, weather_result });+ assert_contains(prompt, "<tool_result>12:00</tool_result>\n\n<tool_result>sunny</tool_result>");+ }++ // Without tool call ids there is nothing to match against; order is preserved.+ {+ auto no_id_calls = assist_calls;+ no_id_calls.tool_calls[0].id = "";+ no_id_calls.tool_calls[1].id = "";+ auto no_id_weather = weather_result; no_id_weather.tool_call_id = "";+ auto no_id_time = time_result; no_id_time.tool_call_id = "";+ auto prompt = render({ user_q, no_id_calls, no_id_weather, no_id_time });+ assert_contains(prompt, "<tool_result>sunny</tool_result>\n\n<tool_result>12:00</tool_result>");+ }+}+static void test_reasoning_budget_tokens_per_request() {LOG_DBG("%s\n", __func__);// Use Qwen3 template which has <think>...</think> reasoning markers.@@ -6139,6 +6412,8 @@ int main(int argc, char ** argv) {test_tools_oaicompat_json_conversion();test_convert_responses_to_chatcmpl();test_developer_role_to_system_workaround();+ test_deepseek_v4_thinking_retention();+ test_deepseek_v4_tool_result_ordering();test_template_generation_prompt();test_reasoning_budget_tokens_per_request();test_reasoning_budget_message_per_request();