LLM inference in C/C++
C++ ggml
chat: trim messages sent to StepFun parser (fixes long reasoning loops) (#25238)
Piotr Wilkin (ilintar)2d973636e292
diff --git a/common/chat.cpp b/common/chat.cppindex 6da59f4dbd..22d2ee4a2a 100644--- a/common/chat.cpp+++ b/common/chat.cpp@@ -2378,6 +2378,23 @@ static void func_args_not_string(json & messages) {}}+// Trim leading/trailing whitespace from message contents before rendering. This+// has to run on the messages (not on the rendered JSON) because templates with+// string-only content caps concatenate typed content parts into a single string+// during rendering, after which the per-part whitespace can no longer be reached.+// Both the plain string content and the text of typed content parts are trimmed.+static void trim_all_content(std::vector<common_chat_msg> & messages) {+ for (auto & message : messages) {+ message.content = trim_whitespace(message.content);+ message.reasoning_content = trim_whitespace(message.reasoning_content);+ for (auto & part : message.content_parts) {+ if (part.type == "text") {+ part.text = trim_whitespace(part.text);+ }+ }+ }+}+}// MiniCPM5 format:@@ -2634,7 +2651,16 @@ static common_chat_params common_chat_templates_apply_jinja(const struct common_params.tools.is_array() && tmpls->template_tool_use ? *tmpls->template_tool_use : *tmpls->template_default;const auto & src = tmpl.source();const auto & caps = tmpl.original_caps();- params.messages = render_message_to_json(inputs.messages, tmpl.original_caps());+ std::vector<common_chat_msg> trimmed_messages;+ const std::vector<common_chat_msg> * messages_to_render = &inputs.messages;+ if (src.find("You have access to the following functions in JSONSchema format") != std::string::npos) {+ // StepFun: trim message contents (including typed content parts) before rendering,+ // otherwise leftover whitespace drives the model into reasoning loops (issue #24181)+ trimmed_messages = inputs.messages;+ workaround::trim_all_content(trimmed_messages);+ messages_to_render = &trimmed_messages;+ }+ params.messages = render_message_to_json(*messages_to_render, tmpl.original_caps());params.tool_choice = inputs.tool_choice;params.reasoning_format = inputs.reasoning_format;params.enable_thinking = inputs.enable_thinking;diff --git a/models/templates/stepfun-ai-Step-3.5-Flash.jinja b/models/templates/stepfun-ai-Step-3.5-Flash.jinjadeleted file mode 100644index c09ea497da..0000000000--- a/models/templates/stepfun-ai-Step-3.5-Flash.jinja+++ /dev/null@@ -1,80 +0,0 @@-{% macro render_content(content) %}{% if content is none %}{{- '' }}{% elif content is string %}{{- content }}{% elif content is mapping %}{{- content['value'] if 'value' in content else content['text'] }}{% elif content is iterable %}{% for item in content %}{% if item.type == 'text' %}{{- item['value'] if 'value' in item else item['text'] }}{% elif item.type == 'image' %}<im_patch>{% endif %}{% endfor %}{% endif %}{% endmacro %}-{{bos_token}}{%- if tools %}- {{- '<|im_start|>system\n' }}- {%- if messages[0].role == 'system' %}- {{- render_content(messages[0].content) + '\n\n' }}- {%- endif %}- {{- "# Tools\n\nYou have access to the following functions in JSONSchema format:\n\n<tools>" }}- {%- for tool in tools %}- {{- "\n" }}- {{- tool | tojson(ensure_ascii=False) }}- {%- endfor %}- {{- "\n</tools>\n\nIf you choose to call a function ONLY reply in the following format with NO suffix:\n\n<tool_call>\n<function=example_function_name>\n<parameter=example_parameter_1>\nvalue_1\n</parameter>\n<parameter=example_parameter_2>\nThis is the value for the second parameter\nthat can span\nmultiple lines\n</parameter>\n</function>\n</tool_call>\n\n<IMPORTANT>\nReminder:\n- Function calls MUST follow the specified format: an inner <function=...>\n...\n</function> block must be nested within <tool_call>\n...\n</tool_call> XML tags\n- Required parameters MUST be specified\n</IMPORTANT><|im_end|>\n" }}-{%- else %}- {%- if messages[0].role == 'system' %}- {{- '<|im_start|>system\n' + render_content(messages[0].content) + '<|im_end|>\n' }}- {%- endif %}-{%- endif %}-{%- set ns = namespace(multi_step_tool=true, last_query_index=messages|length - 1) %}-{%- for message in messages[::-1] %}- {%- set index = (messages|length - 1) - loop.index0 %}- {%- if ns.multi_step_tool and message.role == "user" and render_content(message.content) is string and not(render_content(message.content).startswith('<tool_response>') and render_content(message.content).endswith('</tool_response>')) %}- {%- set ns.multi_step_tool = false %}- {%- set ns.last_query_index = index %}- {%- endif %}-{%- endfor %}-{%- for message in messages %}- {%- set content = render_content(message.content) %}- {%- if (message.role == "user") or (message.role == "system" and not loop.first) %}- {%- set role_name = 'observation' if (message.role == "system" and not loop.first and message.name == 'observation') else message.role %}- {{- '<|im_start|>' + role_name + '\n' + content + '<|im_end|>' + '\n' }}- {%- elif message.role == "assistant" %}- {%- if message.reasoning_content is string %}- {%- set reasoning_content = render_content(message.reasoning_content) %}- {%- else %}- {%- if '</think>' in content %}- {%- set reasoning_content = content.split('</think>')[0].rstrip('\n').split('<think>')[-1].lstrip('\n') %}- {%- set content = content.split('</think>')[-1].lstrip('\n') %}- {%- else %}- {%- set reasoning_content = '' %}- {%- endif %}- {%- endif %}- {%- if loop.index0 > ns.last_query_index %}- {{- '<|im_start|>' + message.role + '\n<think>\n' + reasoning_content + '\n</think>\n' + content }}- {%- else %}- {{- '<|im_start|>' + message.role + '\n' + content }}- {%- endif %}- {%- if message.tool_calls %}- {%- for tool_call in message.tool_calls %}- {%- if tool_call.function is defined %}- {%- set tool_call = tool_call.function %}- {%- endif %}- {{- '<tool_call>\n<function=' + tool_call.name + '>\n' }}- {%- if tool_call.arguments is defined %}- {%- set arguments = tool_call.arguments %}- {%- for args_name, args_value in arguments|items %}- {{- '<parameter=' + args_name + '>\n' }}- {%- set args_value = args_value | tojson(ensure_ascii=False) | safe if args_value is mapping or (args_value is sequence and args_value is not string) else args_value | string %}- {{- args_value }}- {{- '\n</parameter>\n' }}- {%- endfor %}- {%- endif %}- {{- '</function>\n</tool_call>' }}- {%- endfor %}- {%- endif %}- {{- '<|im_end|>\n' }}- {%- elif message.role == "tool" %}- {%- if loop.first or (messages[loop.index0 - 1].role != "tool") %}- {{- '<|im_start|>tool_response\n' }}- {%- endif %}- {{- '<tool_response>' }}- {{- content }}- {{- '</tool_response>' }}- {%- if loop.last or (messages[loop.index0 + 1].role != "tool") %}- {{- '<|im_end|>\n' }}- {%- endif %}- {%- endif %}-{%- endfor %}-{%- if add_generation_prompt %}- {{- '<|im_start|>assistant\n<think>\n' }}-{%- endif %}diff --git a/tests/test-chat-auto-parser.cpp b/tests/test-chat-auto-parser.cppindex 5cc1057532..d15fdd2c02 100644--- a/tests/test-chat-auto-parser.cpp+++ b/tests/test-chat-auto-parser.cpp@@ -1887,7 +1887,6 @@ static void test_role_markers_all_templates(testing & t) {{ "Qwen-Qwen3-0.6B.jinja", "<|im_start|>user", "<|im_start|>assistant" },{ "Qwen-QwQ-32B.jinja", "<|im_start|>user", "<|im_start|>assistant" },{ "StepFun3.5-Flash.jinja", "<|im_start|>user", "<|im_start|>assistant" },- { "stepfun-ai-Step-3.5-Flash.jinja", "<|im_start|>user", "<|im_start|>assistant" },// DeepSeek family{ "deepseek-ai-DeepSeek-R1-Distill-Llama-8B.jinja", "<|User|>", "<|Assistant|>" },diff --git a/tests/test-chat.cpp b/tests/test-chat.cppindex 5f71e5da6e..e1e0a59e6d 100644--- a/tests/test-chat.cpp+++ b/tests/test-chat.cpp@@ -3155,6 +3155,59 @@ static void test_template_output_peg_parsers(bool detailed_debug) {}}}++ {+ // StepFun trimming regression test (see https://github.com/ggml-org/llama.cpp/pull/25238)+ auto tmpls = read_templates("models/templates/StepFun3.5-Flash.jinja");++ common_chat_msg message_chatbot = simple_assist_msg("Let me check.\n\n", "I am thinking.\n\n");++ {+ common_chat_templates_inputs inputs;+ inputs.messages = { message_chatbot };+ inputs.add_generation_prompt = true;++ auto params = common_chat_templates_apply(tmpls.get(), inputs);++ if (params.prompt.find("Let me check.\n\n") != std::string::npos) {+ throw std::runtime_error("StepFun 3.5: content not trimmed");+ }++ if (params.prompt.find("I am thinking.\n\n") != std::string::npos) {+ throw std::runtime_error("StepFun 3.5: reasoning_content not trimmed");+ }+ }++ {+ // Trimming must also reach typed (text) content parts, not just string content+ // (see https://github.com/ggml-org/llama.cpp/pull/25238)+ common_chat_msg message_parts;+ message_parts.role = "user";+ message_parts.content_parts = {+ { /* .type = */ "text", /* .text = */ "First part.\n\n" },+ { /* .type = */ "media_marker", /* .text = */ "<__media__>" },+ { /* .type = */ "text", /* .text = */ "Second part.\n\n" },+ };++ common_chat_templates_inputs inputs;+ inputs.messages = { message_parts };+ inputs.add_generation_prompt = true;++ auto params = common_chat_templates_apply(tmpls.get(), inputs);++ if (params.prompt.find("First part.\n\n") != std::string::npos ||+ params.prompt.find("Second part.\n\n") != std::string::npos) {+ throw std::runtime_error("StepFun 3.5: text content parts not trimmed");+ }++ // the trimmed text itself must still be present+ if (params.prompt.find("First part.") == std::string::npos ||+ params.prompt.find("Second part.") == std::string::npos) {+ throw std::runtime_error("StepFun 3.5: text content parts missing after trim");+ }+ }+ }+}{