Skip to content

armgit / uzu

Public Imported from GitHub
0

A high-performance inference engine for AI models

handle grammars with triggers (#1110)

odilf417e109db0d0
diff --git a/crates/legacy/hanashi/configs/rendering/gemma-4.json b/crates/legacy/hanashi/configs/rendering/gemma-4.json
index 6ca0ee6d0..9d99a7334 100644
--- a/crates/legacy/hanashi/configs/rendering/gemma-4.json
+++ b/crates/legacy/hanashi/configs/rendering/gemma-4.json
@@ -1,4 +1,5 @@
{
+ "grammar_trigger": "<channel|>",
"jinja": {
"template": "{%- macro format_parameters(properties, required) -%}\n {%- set standard_keys = ['description', 'type', 'properties', 'required', 'nullable'] -%}\n {%- set ns = namespace(found_first=false) -%}\n {%- for key, value in properties | dictsort -%}\n {%- set add_comma = false -%}\n {%- if key not in standard_keys -%}\n {%- if ns.found_first %},{% endif -%}\n {%- set ns.found_first = true -%}\n {{ key }}:{\n {%- if value['description'] -%}\n description:<|\"|>{{ value['description'] }}<|\"|>\n {%- set add_comma = true -%}\n {%- endif -%}\n {%- if value['nullable'] %}\n {%- if add_comma %},{%- else -%} {%- set add_comma = true -%} {% endif -%}\n nullable:true\n {%- endif -%}\n {%- if value['type'] | upper == 'STRING' -%}\n {%- if value['enum'] -%}\n {%- if add_comma %},{%- else -%} {%- set add_comma = true -%} {% endif -%}\n enum:{{ format_argument(value['enum']) }}\n {%- endif -%}\n {%- elif value['type'] | upper == 'OBJECT' -%}\n ,properties:{\n {%- if value['properties'] is defined and value['properties'] is mapping -%}\n {{- format_parameters(value['properties'], value['required'] | default([])) -}}\n {%- elif value is mapping -%}\n {{- format_parameters(value, value['required'] | default([])) -}}\n {%- endif -%}\n }\n {%- if value['required'] -%}\n ,required:[\n {%- for item in value['required'] | default([]) -%}\n <|\"|>{{- item -}}<|\"|>\n {%- if not loop.last %},{% endif -%}\n {%- endfor -%}\n ]\n {%- endif -%}\n {%- elif value['type'] | upper == 'ARRAY' -%}\n {%- if value['items'] is mapping and value['items'] -%}\n ,items:{\n {%- set ns_items = namespace(found_first=false) -%}\n {%- for item_key, item_value in value['items'] | dictsort -%}\n {%- if item_value is not none -%}\n {%- if ns_items.found_first %},{% endif -%}\n {%- set ns_items.found_first = true -%}\n {%- if item_key == 'properties' -%}\n properties:{\n {%- if item_value is mapping -%}\n {{- format_parameters(item_value, value['items']['required'] | default([])) -}}\n {%- endif -%}\n }\n {%- elif item_key == 'required' -%}\n required:[\n {%- for req_item in item_value -%}\n <|\"|>{{- req_item -}}<|\"|>\n {%- if not loop.last %},{% endif -%}\n {%- endfor -%}\n ]\n {%- elif item_key == 'type' -%}\n {%- if item_value is string -%}\n type:{{ format_argument(item_value | upper) }}\n {%- else -%}\n type:{{ format_argument(item_value | map('upper') | list) }}\n {%- endif -%}\n {%- else -%}\n {{ item_key }}:{{ format_argument(item_value) }}\n {%- endif -%}\n {%- endif -%}\n {%- endfor -%}\n }\n {%- endif -%}\n {%- endif -%}\n {%- if add_comma %},{%- else -%} {%- set add_comma = true -%} {% endif -%}\n type:<|\"|>{{ value['type'] | upper }}<|\"|>}\n {%- endif -%}\n {%- endfor -%}\n{%- endmacro -%}\n{%- macro format_function_declaration(tool_data) -%}\n declaration:{{- tool_data['function']['name'] -}}{description:<|\"|>{{- tool_data['function']['description'] -}}<|\"|>\n {%- set params = tool_data['function']['parameters'] -%}\n {%- if params -%}\n ,parameters:{\n {%- if params['properties'] -%}\n properties:{ {{- format_parameters(params['properties'], params['required']) -}} },\n {%- endif -%}\n {%- if params['required'] -%}\n required:[\n {%- for item in params['required'] -%}\n <|\"|>{{- item -}}<|\"|>\n {{- ',' if not loop.last -}}\n {%- endfor -%}\n ],\n {%- endif -%}\n {%- if params['type'] -%}\n type:<|\"|>{{- params['type'] | upper -}}<|\"|>}\n {%- endif -%}\n {%- endif -%}\n {%- if 'response' in tool_data['function'] -%}\n {%- set response_declaration = tool_data['function']['response'] -%}\n ,response:{\n {%- if response_declaration['description'] -%}\n description:<|\"|>{{- response_declaration['description'] -}}<|\"|>,\n {%- endif -%}\n {%- if response_declaration['type'] | upper == 'OBJECT' -%}\n type:<|\"|>{{- response_declaration['type'] | upper -}}<|\"|>}\n {%- endif -%}\n {%- endif -%}\n }\n{%- endmacro -%}\n{%- macro format_argument(argument, escape_keys=True) -%}\n {%- if argument is string -%}\n {{- '<|\"|>' + argument + '<|\"|>' -}}\n {%- elif argument is boolean -%}\n {{- 'true' if argument else 'false' -}}\n {%- elif argument is mapping -%}\n {{- '{' -}}\n {%- set ns = namespace(found_first=false) -%}\n {%- for key, value in argument | dictsort -%}\n {%- if ns.found_first %},{% endif -%}\n {%- set ns.found_first = true -%}\n {%- if escape_keys -%}\n {{- '<|\"|>' + key + '<|\"|>' -}}\n {%- else -%}\n {{- key -}}\n {%- endif -%}\n :{{- format_argument(value, escape_keys=escape_keys) -}}\n {%- endfor -%}\n {{- '}' -}}\n {%- elif argument is sequence -%}\n {{- '[' -}}\n {%- for item in argument -%}\n {{- format_argument(item, escape_keys=escape_keys) -}}\n {%- if not loop.last %},{% endif -%}\n {%- endfor -%}\n {{- ']' -}}\n {%- else -%}\n {{- argument -}}\n {%- endif -%}\n{%- endmacro -%}\n{%- macro strip_thinking(text) -%}\n {%- set ns = namespace(result='') -%}\n {%- for part in text.split('<channel|>') -%}\n {%- if '<|channel>' in part -%}\n {%- set ns.result = ns.result + part.split('<|channel>')[0] -%}\n {%- else -%}\n {%- set ns.result = ns.result + part -%}\n {%- endif -%}\n {%- endfor -%}\n {{- ns.result | trim -}}\n{%- endmacro -%}\n\n{%- set ns = namespace(prev_message_type=None) -%}\n{%- set loop_messages = messages -%}\n{{ bos_token }}\n{#- Handle System/Tool Definitions Block -#}\n{%- if (enable_thinking is defined and enable_thinking) or tools or messages[0]['role'] in ['system', 'developer'] -%}\n {{- '<|turn>system\\n' -}}\n\n {#- Inject Thinking token at the very top of the FIRST system turn -#}\n {%- if enable_thinking is defined and enable_thinking -%}\n {{- '<|think|>' -}}\n {%- set ns.prev_message_type = 'think' -%}\n {%- endif -%}\n\n {%- if messages[0]['role'] in ['system', 'developer'] -%}\n {{- messages[0]['content'] | trim -}}\n {%- set loop_messages = messages[1:] -%}\n {%- endif -%}\n\n {%- if tools -%}\n {%- for tool in tools %}\n {{- '<|tool>' -}}\n {{- format_function_declaration(tool) | trim -}}\n {{- '<tool|>' -}}\n {%- endfor %}\n {%- set ns.prev_message_type = 'tool' -%}\n {%- endif -%}\n\n {{- '<turn|>\\n' -}}\n{%- endif %}\n\n{#- Loop through messages -#}\n{%- for message in loop_messages -%}\n {%- set ns.prev_message_type = None -%}\n {%- set role = 'model' if message['role'] == 'assistant' else message['role'] -%}\n {{- '<|turn>' + role + '\\n' }}\n\n {%- if message['tool_calls'] -%}\n {%- for tool_call in message['tool_calls'] -%}\n {%- set function = tool_call['function'] -%}\n {{- '<|tool_call>call:' + function['name'] + '{' -}}\n {%- if function['arguments'] is mapping -%}\n {%- set ns_args = namespace(found_first=false) -%}\n {%- for key, value in function['arguments'] | dictsort -%}\n {%- if ns_args.found_first %},{% endif -%}\n {%- set ns_args.found_first = true -%}\n {{- key -}}:{{- format_argument(value, escape_keys=False) -}}\n {%- endfor -%}\n {%- elif function['arguments'] is string -%}\n {{- function['arguments'] -}}\n {%- endif -%}\n {{- '}<tool_call|>' -}}\n {%- endfor -%}\n {%- set ns.prev_message_type = 'tool_call' -%}\n {%- endif -%}\n\n {%- if message['tool_responses'] -%}\n {#- Tool Response handling -#}\n {%- for tool_response in message['tool_responses'] -%}\n {{- '<|tool_response>' -}}\n {%- if tool_response['response'] is mapping -%}\n {{- 'response:' + tool_response['name'] | default('unknown') + '{' -}}\n {%- for key, value in tool_response['response'] | dictsort -%}\n {{- key -}}:{{- format_argument(value, escape_keys=False) -}}\n {%- if not loop.last %},{% endif -%}\n {%- endfor -%}\n {{- '}' -}}\n {%- else -%}\n {{- 'response:' + tool_response['name'] | default('unknown') + '{value:' + format_argument(tool_response['response'], escape_keys=False) + '}' -}}\n {%- endif -%}\n {{- '<tool_response|>' -}}\n {%- endfor -%}\n {%- set ns.prev_message_type = 'tool_response' -%}\n {%- endif -%}\n\n {%- if message['content'] is string -%}\n {%- if role == 'model' -%}\n {{- strip_thinking(message['content']) -}}\n {%- else -%}\n {{- message['content'] | trim -}}\n {%- endif -%}\n {%- elif message['content'] is sequence -%}\n {%- for item in message['content'] -%}\n {%- if item['type'] == 'text' -%}\n {%- if role == 'model' -%}\n {{- strip_thinking(item['text']) -}}\n {%- else -%}\n {{- item['text'] | trim -}}\n {%- endif -%}\n {%- elif item['type'] == 'image' -%}\n {{- '\\n\\n<|image|>\\n\\n' -}}\n {%- set ns.prev_message_type = 'image' -%}\n {%- elif item['type'] == 'audio' -%}\n {{- '<|audio|>' -}}\n {%- set ns.prev_message_type = 'audio' -%}\n {%- elif item['type'] == 'video' -%}\n {{- '\\n\\n<|video|>\\n\\n' -}}\n {%- set ns.prev_message_type = 'video' -%}\n {%- endif -%}\n {%- endfor -%}\n {%- endif -%}\n\n {%- if not (message['tool_responses'] and not message['content']) -%}\n {{- '<turn|>\\n' -}}\n {%- endif -%}\n{%- endfor -%}\n\n{%- if add_generation_prompt -%}\n {%- if ns.prev_message_type != 'tool_response' -%}\n {{- '<|turn>model\\n' -}}\n {%- endif -%}\n{%- endif -%}",
"required_functions": [],
diff --git a/crates/legacy/hanashi/configs/rendering/gpt-oss.json b/crates/legacy/hanashi/configs/rendering/gpt-oss.json
index 86e0bf111..d283ed737 100644
--- a/crates/legacy/hanashi/configs/rendering/gpt-oss.json
+++ b/crates/legacy/hanashi/configs/rendering/gpt-oss.json
@@ -1,4 +1,5 @@
{
+ "grammar_trigger": "<|channel|>final<|message|>",
"default_reasoning_effort": "medium",
"jinja": {
"template": "{#-\n In addition to the normal inputs of `messages` and `tools`, this template also accepts the\n following kwargs:\n - \"builtin_tools\": A list, can contain \"browser\" and/or \"python\".\n - \"model_identity\": A string that optionally describes the model identity.\n - \"reasoning_effort\": A string that describes the reasoning effort, defaults to \"medium\".\n #}\n\n{#- Tool Definition Rendering ============================================== #}\n{%- macro render_typescript_type(param_spec, required_params, is_nullable=false) -%}\n {%- if param_spec.type == \"array\" -%}\n {%- if param_spec['items'] -%}\n {%- if param_spec['items']['type'] == \"string\" -%}\n {{- \"string[]\" }}\n {%- elif param_spec['items']['type'] == \"number\" -%}\n {{- \"number[]\" }}\n {%- elif param_spec['items']['type'] == \"integer\" -%}\n {{- \"number[]\" }}\n {%- elif param_spec['items']['type'] == \"boolean\" -%}\n {{- \"boolean[]\" }}\n {%- else -%}\n {%- set inner_type = render_typescript_type(param_spec['items'], required_params) -%}\n {%- if inner_type == \"object | object\" or inner_type|length > 50 -%}\n {{- \"any[]\" }}\n {%- else -%}\n {{- inner_type + \"[]\" }}\n {%- endif -%}\n {%- endif -%}\n {%- if param_spec.nullable -%}\n {{- \" | null\" }}\n {%- endif -%}\n {%- else -%}\n {{- \"any[]\" }}\n {%- if param_spec.nullable -%}\n {{- \" | null\" }}\n {%- endif -%}\n {%- endif -%}\n {%- elif param_spec.type is defined and param_spec.type is iterable and param_spec.type is not string and param_spec.type is not mapping and param_spec.type[0] is defined -%}\n {#- Handle array of types like [\"object\", \"object\"] from Union[dict, list] #}\n {%- if param_spec.type | length > 1 -%}\n {{- param_spec.type | join(\" | \") }}\n {%- else -%}\n {{- param_spec.type[0] }}\n {%- endif -%}\n {%- elif param_spec.oneOf -%}\n {#- Handle oneOf schemas - check for complex unions and fallback to any #}\n {%- set has_object_variants = false -%}\n {%- for variant in param_spec.oneOf -%}\n {%- if variant.type == \"object\" -%}\n {%- set has_object_variants = true -%}\n {%- endif -%}\n {%- endfor -%}\n {%- if has_object_variants and param_spec.oneOf|length > 1 -%}\n {{- \"any\" }}\n {%- else -%}\n {%- for variant in param_spec.oneOf -%}\n {{- render_typescript_type(variant, required_params) -}}\n {%- if variant.description %}\n {{- \"// \" + variant.description }}\n {%- endif -%}\n {%- if variant.default is defined %}\n {{ \"// default: \" + variant.default|tojson }}\n {%- endif -%}\n {%- if not loop.last %}\n {{- \" | \" }}\n {% endif -%}\n {%- endfor -%}\n {%- endif -%}\n {%- elif param_spec.type == \"string\" -%}\n {%- if param_spec.enum -%}\n {{- '\"' + param_spec.enum|join('\" | \"') + '\"' -}}\n {%- else -%}\n {{- \"string\" }}\n {%- if param_spec.nullable %}\n {{- \" | null\" }}\n {%- endif -%}\n {%- endif -%}\n {%- elif param_spec.type == \"number\" -%}\n {{- \"number\" }}\n {%- elif param_spec.type == \"integer\" -%}\n {{- \"number\" }}\n {%- elif param_spec.type == \"boolean\" -%}\n {{- \"boolean\" }}\n\n {%- elif param_spec.type == \"object\" -%}\n {%- if param_spec.properties -%}\n {{- \"{\\n\" }}\n {%- for prop_name, prop_spec in param_spec.properties.items() -%}\n {{- prop_name -}}\n {%- if prop_name not in (param_spec.required or []) -%}\n {{- \"?\" }}\n {%- endif -%}\n {{- \": \" }}\n {{ render_typescript_type(prop_spec, param_spec.required or []) }}\n {%- if not loop.last -%}\n {{-\", \" }}\n {%- endif -%}\n {%- endfor -%}\n {{- \"}\" }}\n {%- else -%}\n {{- \"object\" }}\n {%- endif -%}\n {%- else -%}\n {{- \"any\" }}\n {%- endif -%}\n{%- endmacro -%}\n\n{%- macro render_tool_namespace(namespace_name, tools) -%}\n {{- \"## \" + namespace_name + \"\\n\\n\" }}\n {{- \"namespace \" + namespace_name + \" {\\n\\n\" }}\n {%- for tool in tools %}\n {%- set tool = tool.function %}\n {{- \"// \" + tool.description + \"\\n\" }}\n {{- \"type \"+ tool.name + \" = \" }}\n {%- if tool.parameters and tool.parameters.properties %}\n {{- \"(_: {\\n\" }}\n {%- for param_name, param_spec in tool.parameters.properties.items() %}\n {%- if param_spec.description %}\n {{- \"// \" + param_spec.description + \"\\n\" }}\n {%- endif %}\n {{- param_name }}\n {%- if param_name not in (tool.parameters.required or []) -%}\n {{- \"?\" }}\n {%- endif -%}\n {{- \": \" }}\n {{- render_typescript_type(param_spec, tool.parameters.required or []) }}\n {%- if param_spec.default is defined -%}\n {%- if param_spec.enum %}\n {{- \", // default: \" + param_spec.default }}\n {%- elif param_spec.oneOf %}\n {{- \"// default: \" + param_spec.default }}\n {%- else %}\n {{- \", // default: \" + param_spec.default|tojson }}\n {%- endif -%}\n {%- endif -%}\n {%- if not loop.last %}\n {{- \",\\n\" }}\n {%- else %}\n {{- \",\\n\" }}\n {%- endif -%}\n {%- endfor %}\n {{- \"}) => any;\\n\\n\" }}\n {%- else -%}\n {{- \"() => any;\\n\\n\" }}\n {%- endif -%}\n {%- endfor %}\n {{- \"} // namespace \" + namespace_name }}\n{%- endmacro -%}\n\n{%- macro render_builtin_tools(browser_tool, python_tool) -%}\n {%- if browser_tool %}\n {{- \"## browser\\n\\n\" }}\n {{- \"// Tool for browsing.\\n\" }}\n {{- \"// The `cursor` appears in brackets before each browsing display: `[{cursor}]`.\\n\" }}\n {{- \"// Cite information from the tool using the following format:\\n\" }}\n {{- \"// `【{cursor}†L{line_start}(-L{line_end})?】`, for example: `【6†L9-L11】` or `【8†L3】`.\\n\" }}\n {{- \"// Do not quote more than 10 words directly from the tool output.\\n\" }}\n {{- \"// sources=web (default: web)\\n\" }}\n {{- \"namespace browser {\\n\\n\" }}\n {{- \"// Searches for information related to `query` and displays `topn` results.\\n\" }}\n {{- \"type search = (_: {\\n\" }}\n {{- \"query: string,\\n\" }}\n {{- \"topn?: number, // default: 10\\n\" }}\n {{- \"source?: string,\\n\" }}\n {{- \"}) => any;\\n\\n\" }}\n {{- \"// Opens the link `id` from the page indicated by `cursor` starting at line number `loc`, showing `num_lines` lines.\\n\" }}\n {{- \"// Valid link ids are displayed with the formatting: `【{id}†.*】`.\\n\" }}\n {{- \"// If `cursor` is not provided, the most recent page is implied.\\n\" }}\n {{- \"// If `id` is a string, it is treated as a fully qualified URL associated with `source`.\\n\" }}\n {{- \"// If `loc` is not provided, the viewport will be positioned at the beginning of the document or centered on the most relevant passage, if available.\\n\" }}\n {{- \"// Use this function without `id` to scroll to a new location of an opened page.\\n\" }}\n {{- \"type open = (_: {\\n\" }}\n {{- \"id?: number | string, // default: -1\\n\" }}\n {{- \"cursor?: number, // default: -1\\n\" }}\n {{- \"loc?: number, // default: -1\\n\" }}\n {{- \"num_lines?: number, // default: -1\\n\" }}\n {{- \"view_source?: boolean, // default: false\\n\" }}\n {{- \"source?: string,\\n\" }}\n {{- \"}) => any;\\n\\n\" }}\n {{- \"// Finds exact matches of `pattern` in the current page, or the page given by `cursor`.\\n\" }}\n {{- \"type find = (_: {\\n\" }}\n {{- \"pattern: string,\\n\" }}\n {{- \"cursor?: number, // default: -1\\n\" }}\n {{- \"}) => any;\\n\\n\" }}\n {{- \"} // namespace browser\\n\\n\" }}\n {%- endif -%}\n\n {%- if python_tool %}\n {{- \"## python\\n\\n\" }}\n {{- \"Use this tool to execute Python code in your chain of thought. The code will not be shown to the user. This tool should be used for internal reasoning, but not for code that is intended to be visible to the user (e.g. when creating plots, tables, or files).\\n\\n\" }}\n {{- \"When you send a message containing Python code to python, it will be executed in a stateful Jupyter notebook environment. python will respond with the output of the execution or time out after 120.0 seconds. The drive at '/mnt/data' can be used to save and persist user files. Internet access for this session is UNKNOWN. Depends on the cluster.\\n\\n\" }}\n {%- endif -%}\n{%- endmacro -%}\n\n{#- System Message Construction ============================================ #}\n{%- macro build_system_message() -%}\n {%- if model_identity is not defined %}\n {%- set model_identity = \"You are ChatGPT, a large language model trained by OpenAI.\" %}\n {%- endif %}\n {{- model_identity + \"\\n\" }}\n {{- \"Knowledge cutoff: 2024-06\\n\" }}\n {{- \"Current date: \" + strftime_now(\"%Y-%m-%d\") + \"\\n\\n\" }}\n {%- if reasoning_effort is not defined %}\n {%- set reasoning_effort = \"medium\" %}\n {%- endif %}\n {{- \"Reasoning: \" + reasoning_effort + \"\\n\\n\" }}\n {%- if builtin_tools %}\n {{- \"# Tools\\n\\n\" }}\n {%- set available_builtin_tools = namespace(browser=false, python=false) %}\n {%- for tool in builtin_tools %}\n {%- if tool == \"browser\" %}\n {%- set available_builtin_tools.browser = true %}\n {%- elif tool == \"python\" %}\n {%- set available_builtin_tools.python = true %}\n {%- endif %}\n {%- endfor %}\n {{- render_builtin_tools(available_builtin_tools.browser, available_builtin_tools.python) }}\n {%- endif -%}\n {{- \"# Valid channels: analysis, commentary, final. Channel must be included for every message.\" }}\n {%- if tools -%}\n {{- \"\\nCalls to these tools must go to the commentary channel: 'functions'.\" }}\n {%- endif -%}\n{%- endmacro -%}\n\n{#- Main Template Logic ================================================= #}\n{#- Set defaults #}\n\n{#- Render system message #}\n{{- \"<|start|>system<|message|>\" }}\n{{- build_system_message() }}\n{{- \"<|end|>\" }}\n\n{#- Extract developer message #}\n{%- if messages[0].role == \"developer\" or messages[0].role == \"system\" %}\n {%- set developer_message = messages[0].content %}\n {%- set loop_messages = messages[1:] %}\n{%- else %}\n {%- set developer_message = \"\" %}\n {%- set loop_messages = messages %}\n{%- endif %}\n\n{#- Render developer message #}\n{%- if developer_message or tools %}\n {{- \"<|start|>developer<|message|>\" }}\n {%- if developer_message %}\n {{- \"# Instructions\\n\\n\" }}\n {{- developer_message }}\n {{- \"\\n\\n\" }}\n {%- endif %}\n {%- if tools -%}\n {{- \"# Tools\\n\\n\" }}\n {{- render_tool_namespace(\"functions\", tools) }}\n {%- endif -%}\n {{- \"<|end|>\" }}\n{%- endif %}\n\n{#- Render messages #}\n{%- set last_tool_call = namespace(name=none) %}\n{%- for message in loop_messages -%}\n {#- At this point only assistant/user/tool messages should remain #}\n {%- if message.role == 'assistant' -%}\n {#- Checks to ensure the messages are being passed in the format we expect #}\n {%- if \"content\" in message %}\n {%- if \"<|channel|>analysis<|message|>\" in message.content or \"<|channel|>final<|message|>\" in message.content %}\n {{- raise_exception(\"You have passed a message containing <|channel|> tags in the content field. Instead of doing this, you should pass analysis messages (the string between '<|message|>' and '<|end|>') in the 'thinking' field, and final messages (the string between '<|message|>' and '<|end|>') in the 'content' field.\") }}\n {%- endif %}\n {%- endif %}\n {%- if \"thinking\" in message %}\n {%- if \"<|channel|>analysis<|message|>\" in message.thinking or \"<|channel|>final<|message|>\" in message.thinking %}\n {{- raise_exception(\"You have passed a message containing <|channel|> tags in the thinking field. Instead of doing this, you should pass analysis messages (the string between '<|message|>' and '<|end|>') in the 'thinking' field, and final messages (the string between '<|message|>' and '<|end|>') in the 'content' field.\") }}\n {%- endif %}\n {%- endif %}\n {%- if \"tool_calls\" in message %}\n {#- We need very careful handling here - we want to drop the tool call analysis message if the model #}\n {#- has output a later <|final|> message, but otherwise we want to retain it. This is the only case #}\n {#- when we render CoT/analysis messages in inference. #}\n {%- set future_final_message = namespace(found=false) %}\n {%- for future_message in loop_messages[loop.index:] %}\n {%- if future_message.role == 'assistant' and \"tool_calls\" not in future_message %}\n {%- set future_final_message.found = true %}\n {%- endif %}\n {%- endfor %}\n {#- We assume max 1 tool call per message, and so we infer the tool call name #}\n {#- in \"tool\" messages from the most recent assistant tool call name #}\n {%- set tool_call = message.tool_calls[0] %}\n {%- if tool_call.function %}\n {%- set tool_call = tool_call.function %}\n {%- endif %}\n {%- if message.content and message.thinking %}\n {{- raise_exception(\"Cannot pass both content and thinking in an assistant message with tool calls! Put the analysis message in one or the other, but not both.\") }}\n {%- elif message.content and not future_final_message.found %}\n {{- \"<|start|>assistant<|channel|>analysis<|message|>\" + message.content + \"<|end|>\" }}\n {%- elif message.thinking and not future_final_message.found %}\n {{- \"<|start|>assistant<|channel|>analysis<|message|>\" + message.thinking + \"<|end|>\" }}\n {%- endif %}\n {{- \"<|start|>assistant to=\" }}\n {{- \"functions.\" + tool_call.name + \"<|channel|>commentary \" }}\n {{- (tool_call.content_type if tool_call.content_type is defined else \"json\") + \"<|message|>\" }}\n {{- tool_call.arguments|tojson }}\n {{- \"<|call|>\" }}\n {%- set last_tool_call.name = tool_call.name %}\n {%- elif loop.last and not add_generation_prompt %}\n {#- Only render the CoT if the final turn is an assistant turn and add_generation_prompt is false #}\n {#- This is a situation that should only occur in training, never in inference. #}\n {%- if \"thinking\" in message %}\n {{- \"<|start|>assistant<|channel|>analysis<|message|>\" + message.thinking + \"<|end|>\" }}\n {%- endif %}\n {#- <|return|> indicates the end of generation, but <|end|> does not #}\n {#- <|return|> should never be an input to the model, but we include it as the final token #}\n {#- when training, so the model learns to emit it. #}\n {{- \"<|start|>assistant<|channel|>final<|message|>\" + message.content + \"<|return|>\" }}\n {%- else %}\n {#- CoT is dropped during all previous turns, so we never render it for inference #}\n {{- \"<|start|>assistant<|channel|>final<|message|>\" + message.content + \"<|end|>\" }}\n {%- set last_tool_call.name = none %}\n {%- endif %}\n {%- elif message.role == 'tool' -%}\n {%- if last_tool_call.name is none %}\n {{- raise_exception(\"Message has tool role, but there was no previous assistant message with a tool call!\") }}\n {%- endif %}\n {{- \"<|start|>functions.\" + last_tool_call.name }}\n {{- \" to=assistant<|channel|>commentary<|message|>\" + message.content|tojson + \"<|end|>\" }}\n {%- elif message.role == 'user' -%}\n {{- \"<|start|>user<|message|>\" + message.content + \"<|end|>\" }}\n {%- endif -%}\n{%- endfor -%}\n\n{#- Generation prompt #}\n{%- if add_generation_prompt -%}\n<|start|>assistant\n{%- endif -%}",
diff --git a/crates/legacy/hanashi/configs/rendering/lfm2.5-thinking.json b/crates/legacy/hanashi/configs/rendering/lfm2.5-thinking.json
index 76e1fb755..35fcec23e 100644
--- a/crates/legacy/hanashi/configs/rendering/lfm2.5-thinking.json
+++ b/crates/legacy/hanashi/configs/rendering/lfm2.5-thinking.json
@@ -1,4 +1,5 @@
{
+ "grammar_trigger": "</think>",
"jinja": {
"template": "{{- bos_token -}}\n{%- set keep_past_thinking = keep_past_thinking | default(false) -%}\n{%- set ns = namespace(system_prompt=\"\") -%}\n{%- if messages[0][\"role\"] == \"system\" -%}\n {%- set ns.system_prompt = messages[0][\"content\"] -%}\n {%- set messages = messages[1:] -%}\n{%- endif -%}\n{%- if tools -%}\n {%- set ns.system_prompt = ns.system_prompt + (\"\\n\" if ns.system_prompt else \"\") + \"List of tools: [\" -%}\n {%- for tool in tools -%}\n {%- if tool is not string -%}\n {%- set tool = tool | tojson -%}\n {%- endif -%}\n {%- set ns.system_prompt = ns.system_prompt + tool -%}\n {%- if not loop.last -%}\n {%- set ns.system_prompt = ns.system_prompt + \", \" -%}\n {%- endif -%}\n {%- endfor -%}\n {%- set ns.system_prompt = ns.system_prompt + \"]\" -%}\n{%- endif -%}\n{%- if ns.system_prompt -%}\n {{- \"<|im_start|>system\\n\" + ns.system_prompt + \"<|im_end|>\\n\" -}}\n{%- endif -%}\n{%- set ns.last_assistant_index = -1 -%}\n{%- for message in messages -%}\n {%- if message[\"role\"] == \"assistant\" -%}\n {%- set ns.last_assistant_index = loop.index0 -%}\n {%- endif -%}\n{%- endfor -%}\n{%- for message in messages -%}\n {{- \"<|im_start|>\" + message[\"role\"] + \"\\n\" -}}\n {%- set content = message[\"content\"] -%}\n {%- if content is not string -%}\n {%- set content = content | tojson -%}\n {%- endif -%}\n {%- if message[\"role\"] == \"assistant\" and not keep_past_thinking and loop.index0 != ns.last_assistant_index -%}\n {%- if \"</think>\" in content -%}\n {%- set content = content.split(\"</think>\")[-1] | trim -%}\n {%- endif -%}\n {%- endif -%}\n {{- content + \"<|im_end|>\\n\" -}}\n{%- endfor -%}\n{%- if add_generation_prompt -%}\n {{- \"<|im_start|>assistant\\n\" -}}\n{%- endif -%}",
"required_functions": [],
diff --git a/crates/legacy/hanashi/configs/rendering/muse-glimmer.json b/crates/legacy/hanashi/configs/rendering/muse-glimmer.json
index 4d02410b2..84f08da74 100644
--- a/crates/legacy/hanashi/configs/rendering/muse-glimmer.json
+++ b/crates/legacy/hanashi/configs/rendering/muse-glimmer.json
@@ -1,4 +1,5 @@
{
+ "grammar_trigger": " to=user<|message|>",
"jinja": {
"template": "{%- macro render_content(content) -%}{%- if content is string -%}{{- content -}}{%- elif content is not none -%}{%- for part in content -%}{%- if part['type'] == 'image' -%}{{- '<|patch|>' -}}{%- elif part['type'] == 'video' -%}{{- '<|video|>' -}}{%- elif part['type'] == 'text' -%}{{- part['text'] -}}{%- endif -%}{%- endfor -%}{%- endif -%}{%- endmacro -%}{%- macro render_atem(tc) -%}{%- set tc_fn = tc.function if tc.function is defined else tc -%}{%- set args = tc_fn.arguments -%}{%- if args is not mapping -%}{{- raise_exception('Onyx ATEM chat template requires tool_call.function.arguments to be a dict (mapping); a JSON string cannot be parsed in the HF jinja sandbox.') -}}{%- endif -%}{{- '<atem:function_calls>\\n<atem:invoke name=\"' + tc_fn.name + '\">\\n' -}}{%- for k, v in args.items() -%}{{- '<atem:parameter name=\"' + k + '\">' -}}{%- if v is boolean -%}{%- if v -%}true{%- else -%}false{%- endif -%}{%- elif v is none -%}null{%- elif v is mapping or (v is iterable and v is not string) -%}{{- v | tojson -}}{%- else -%}{{- v -}}{%- endif -%}{{- '</atem:parameter>\\n' -}}{%- endfor -%}{{- '</atem:invoke>\\n</atem:function_calls>' -}}{%- endmacro -%}{%- macro render_tool_defs(tools) -%}{{- 'In this environment you have access to a set of tools you can use to answer the user\\'s question.\\n\\n' -}}{{- 'You can invoke a function by writing a \"<atem:function_calls>\" block like the following:\\n' -}}{{- '<atem:function_calls>\\n<atem:invoke name=\"$FUNCTION_NAME\">\\n<atem:parameter name=\"$PARAMETER_NAME\">$PARAMETER_VALUE</atem:parameter>\\n...\\n</atem:invoke>\\n</atem:function_calls>\\n\\n' -}}{{- 'String and scalar parameters should be specified as is, while lists and objects should use JSON format. Note that spaces for string values are not stripped. The output is not expected to be valid XML and is parsed with regular expressions.\\n' -}}{{- 'Here are the functions available in JSONSchema format:\\n' -}}{{- '// Tool metadata\\n' -}}{%- set nsns = namespace(seen=[]) -%}{%- for tool in tools -%}{%- set fn = tool.function if tool.function is defined else tool -%}{%- set tns = fn.name.split('.')[0] -%}{%- if tns not in nsns.seen -%}{%- set nsns.seen = nsns.seen + [tns] -%}{%- endif -%}{%- endfor -%}{%- set nd = tool_namespace_descriptions if tool_namespace_descriptions is defined else {} -%}{%- for tns in nsns.seen -%}{{- '{\"name\": ' + (tns | tojson) + ', \"description\": ' + ((nd[tns] if tns in nd else '') | tojson) + '}\\n' -}}{%- endfor -%}{{- '// Function schemas' -}}{%- for tool in tools -%}{%- set fn = tool.function if tool.function is defined else tool -%}{{- '\\n{\"name\": ' + (fn.name | tojson) + ', \"description\": ' + (fn.description | tojson) + ', \"parameters\": ' + (fn.parameters | tojson) + '}' -}}{%- endfor -%}{{- '\\n\\nHere\\'s an example of how to call a function in the tool set:\\n' -}}{{- '(If the tool namespace is not specified, invoke the function directly as `example_function_name` rather than `example_tool_name.example_function_name`)\\n\\n' -}}{{- 'to=example_tool_name.example_function_name\\n\\n' -}}{{- '<atem:function_calls>\\n<atem:invoke name=\"example_tool_name.example_function_name\">\\n' -}}{{- '<atem:parameter name=\"example_parameter_1\">value_1</atem:parameter>\\n' -}}{{- '<atem:parameter name=\"example_parameter_2\">This is the value for the second parameter\\nthat can span\\n\"multiple\" lines\\n</atem:parameter>\\n' -}}{{- '</atem:invoke>\\n</atem:function_calls>' -}}{%- endmacro -%}{%- macro render_reasoning() -%}{%- set rs = reasoning_strength if reasoning_strength is defined and reasoning_strength else 'high' -%}{{- 'Reasoning strength: ' + rs + '.' -}}{%- endmacro -%}{%- macro render_system_meta(tools) -%}{%- set rns = namespace(recipients=['\"self\"'], nslist=[]) -%}{%- if tools -%}{%- for tool in tools -%}{%- set fn = tool.function if tool.function is defined else tool -%}{%- set tns = fn.name.split('.')[0] -%}{%- if tns not in rns.nslist -%}{%- set rns.nslist = rns.nslist + [tns] -%}{%- endif -%}{%- endfor -%}{%- for tns in rns.nslist -%}{%- set rns.recipients = rns.recipients + ['\"' + tns + '.*\"'] -%}{%- endfor -%}{%- endif -%}{%- set rns.recipients = rns.recipients + ['\"user\"'] -%}{{- '# Valid recipients: ' + rns.recipients | join(', ') + '.' -}}{%- endmacro -%}{{- bos_token -}}{%- set ns = namespace(has_system=false) -%}{%- for m in messages -%}{%- if m['role'] == 'system' -%}{%- set ns.has_system = true -%}{%- endif -%}{%- endfor -%}{%- if not ns.has_system -%}{{- '<|start|>system<|message|>You are a helpful AI assistant.' -}}{%- set kc = knowledge_cutoff if knowledge_cutoff is defined and knowledge_cutoff else '2026-01-04' -%}{{- '\\nKnowledge cutoff: ' + kc + '.' -}}{%- if current_date is defined and current_date -%}{{- '\\nCurrent date: ' + current_date + '.' -}}{%- elif strftime_now is defined -%}{{- '\\nCurrent date: ' + strftime_now('%Y-%m-%d') + '.' -}}{%- endif -%}{{- '\\n\\n' -}}{{- render_reasoning() -}}{%- if tools -%}{{- '\\n\\n' -}}{{- render_tool_defs(tools) -}}{%- endif -%}{{- '\\n\\n' -}}{{- render_system_meta(tools) -}}{{- '<|eot|>' -}}{%- endif -%}{%- for message in messages -%}{%- set role = message['role'] -%}{%- set end_token = '<|eom|>' if (not loop.last and messages[loop.index0 + 1]['role'] == role) else '<|eot|>' -%}{%- if role == 'system' -%}{{- '<|start|>system<|message|>' -}}{{- render_content(message['content']) -}}{{- '\\n\\n' -}}{{- render_reasoning() -}}{%- if tools -%}{{- '\\n\\n' -}}{{- render_tool_defs(tools) -}}{%- endif -%}{{- '\\n\\n' -}}{{- render_system_meta(tools) -}}{{- '<|eot|>' -}}{%- elif role == 'user' -%}{{- '<|start|>user<|message|>' -}}{{- render_content(message['content']) -}}{{- '<|eot|>' -}}{%- elif role == 'tool' -%}{%- set tname = message.get('name') -%}{%- set tcontent = message['content'] -%}{%- if tcontent is mapping and tcontent.get('name') is not none -%}{%- set tname = tcontent.get('name') -%}{%- set tcontent = tcontent.get('response') -%}{%- if tcontent is mapping and (tcontent | length) == 1 and tcontent.get('value') is not none -%}{%- set tcontent = tcontent.get('value') -%}{%- endif -%}{%- endif -%}{{- '<|start|>tool ' + tname + '<|message|><tool_output name=\"' + tname + '\">\\n' -}}{%- if tcontent is string -%}{{- tcontent -}}{%- else -%}{{- tcontent | tojson -}}{%- endif -%}{{- '\\n</tool_output><|eot|>' -}}{%- elif role == 'assistant' -%}{%- if message.get('reasoning_content') -%}{{- '<|start|>assistant to=self<|message|>' + message['reasoning_content'] + '<|eom|>' -}}{%- endif -%}{%- if message.get('tool_calls') -%}{%- for tc in message['tool_calls'] -%}{%- set tc_fn = tc.function if tc.function is defined else tc -%}{{- '<|start|>assistant to=' + tc_fn.name + '<|message|>' -}}{{- render_atem(tc) -}}{%- if loop.last -%}{{- end_token -}}{%- else -%}{{- '<|eom|>' -}}{%- endif -%}{%- endfor -%}{%- else -%}{%- set recipient = message.get('recipient') or 'user' -%}{%- set end_turn = message.get('end_turn') -%}{%- if end_turn is none -%}{%- set end_turn = not (recipient and recipient != 'user') -%}{%- endif -%}{{- '<|start|>assistant' -}}{%- if recipient -%}{{- ' to=' + recipient -}}{%- endif -%}{{- '<|message|>' -}}{{- render_content(message['content']) -}}{{- ('<|eot|>' if end_turn else '<|eom|>') -}}{%- endif -%}{%- endif -%}{%- endfor -%}{%- if add_generation_prompt -%}{{- '<|start|>assistant' -}}{%- endif -%}",
"required_functions": [
diff --git a/crates/legacy/hanashi/configs/rendering/qwen3-thinking.json b/crates/legacy/hanashi/configs/rendering/qwen3-thinking.json
index dff3b6bd2..3aacf1d61 100644
--- a/crates/legacy/hanashi/configs/rendering/qwen3-thinking.json
+++ b/crates/legacy/hanashi/configs/rendering/qwen3-thinking.json
@@ -1,4 +1,5 @@
{
+ "grammar_trigger": "</think>",
"jinja": {
"template": "{%- if tools %}\n {{- '<|im_start|>system\\n' }}\n {%- if messages[0].role == 'system' %}\n {{- messages[0].content + '\\n\\n' }}\n {%- endif %}\n {{- \"# Tools\\n\\nYou may call one or more functions to assist with the user query.\\n\\nYou are provided with function signatures within <tools></tools> XML tags:\\n<tools>\" }}\n {%- for tool in tools %}\n {{- \"\\n\" }}\n {{- tool | tojson }}\n {%- endfor %}\n {{- \"\\n</tools>\\n\\nFor each function call, return a json object with function name and arguments within <tool_call></tool_call> XML tags:\\n<tool_call>\\n{\\\"name\\\": <function-name>, \\\"arguments\\\": <args-json-object>}\\n</tool_call><|im_end|>\\n\" }}\n{%- else %}\n {%- if messages[0].role == 'system' %}\n {{- '<|im_start|>system\\n' + messages[0].content + '<|im_end|>\\n' }}\n {%- endif %}\n{%- endif %}\n{%- set ns = namespace(multi_step_tool=true, last_query_index=messages|length - 1) %}\n{%- for message in messages[::-1] %}\n {%- set index = (messages|length - 1) - loop.index0 %}\n {%- if ns.multi_step_tool and message.role == \"user\" and message.content is string and not(message.content.startswith('<tool_response>') and message.content.endswith('</tool_response>')) %}\n {%- set ns.multi_step_tool = false %}\n {%- set ns.last_query_index = index %}\n {%- endif %}\n{%- endfor %}\n{%- for message in messages %}\n {%- if message.content is string %}\n {%- set content = message.content %}\n {%- elif message.content is mapping %}\n {%- set content = message.content | tojson %}\n {%- else %}\n {%- set content = '' %}\n {%- endif %}\n {%- if (message.role == \"user\") or (message.role == \"system\" and not loop.first) %}\n {{- '<|im_start|>' + message.role + '\\n' + content + '<|im_end|>' + '\\n' }}\n {%- elif message.role == \"assistant\" %}\n {%- set reasoning_content = '' %}\n {%- if message.reasoning_content is string %}\n {%- set reasoning_content = message.reasoning_content %}\n {%- else %}\n {%- if '</think>' in content %}\n {%- set reasoning_content = content.split('</think>')[0].rstrip('\\n').split('<think>')[-1].lstrip('\\n') %}\n {%- set content = content.split('</think>')[-1].lstrip('\\n') %}\n {%- endif %}\n {%- endif %}\n {%- if loop.index0 > ns.last_query_index %}\n {%- if loop.last or (not loop.last and reasoning_content) %}\n {{- '<|im_start|>' + message.role + '\\n<think>\\n' + reasoning_content.strip('\\n') + '\\n</think>\\n\\n' + content.lstrip('\\n') }}\n {%- else %}\n {{- '<|im_start|>' + message.role + '\\n' + content }}\n {%- endif %}\n {%- else %}\n {{- '<|im_start|>' + message.role + '\\n' + content }}\n {%- endif %}\n {%- if message.tool_calls %}\n {%- for tool_call in message.tool_calls %}\n {%- if (loop.first and content) or (not loop.first) %}\n {{- '\\n' }}\n {%- endif %}\n {%- if tool_call.function %}\n {%- set tool_call = tool_call.function %}\n {%- endif %}\n {{- '<tool_call>\\n{\"name\": \"' }}\n {{- tool_call.name }}\n {{- '\", \"arguments\": ' }}\n {%- if tool_call.arguments is string %}\n {{- tool_call.arguments }}\n {%- else %}\n {{- tool_call.arguments | tojson }}\n {%- endif %}\n {{- '}\\n</tool_call>' }}\n {%- endfor %}\n {%- endif %}\n {{- '<|im_end|>\\n' }}\n {%- elif message.role == \"tool\" %}\n {%- if loop.first or (messages[loop.index0 - 1].role != \"tool\") %}\n {{- '<|im_start|>user' }}\n {%- endif %}\n {{- '\\n<tool_response>\\n' }}\n {{- content }}\n {{- '\\n</tool_response>' }}\n {%- if loop.last or (messages[loop.index0 + 1].role != \"tool\") %}\n {{- '<|im_end|>\\n' }}\n {%- endif %}\n {%- endif %}\n{%- endfor %}\n{%- if add_generation_prompt %}\n {{- '<|im_start|>assistant\\n<think>\\n' }}\n{%- endif %}",
"required_functions": [],
diff --git a/crates/legacy/hanashi/configs/rendering/qwen3.5.json b/crates/legacy/hanashi/configs/rendering/qwen3.5.json
index 2ee55d9d2..23be2d2ac 100644
--- a/crates/legacy/hanashi/configs/rendering/qwen3.5.json
+++ b/crates/legacy/hanashi/configs/rendering/qwen3.5.json
@@ -1,4 +1,5 @@
{
+ "grammar_trigger": "</think>",
"jinja": {
"template": "{%- set image_count = namespace(value=0) %}\n{%- set video_count = namespace(value=0) %}\n{%- macro render_content(content, do_vision_count, is_system_content=false) %}\n {%- if content is string %}\n {{- content }}\n {%- elif content is mapping %}\n {{- content | tojson }}\n {%- elif content is iterable and content is not mapping %}\n {%- for item in content %}\n {%- if 'image' in item or 'image_url' in item or item.type == 'image' %}\n {%- if is_system_content %}\n {{- raise_exception('System message cannot contain images.') }}\n {%- endif %}\n {%- if do_vision_count %}\n {%- set image_count.value = image_count.value + 1 %}\n {%- endif %}\n {%- if add_vision_id %}\n {{- 'Picture ' ~ image_count.value ~ ': ' }}\n {%- endif %}\n {{- '<|vision_start|><|image_pad|><|vision_end|>' }}\n {%- elif 'video' in item or item.type == 'video' %}\n {%- if is_system_content %}\n {{- raise_exception('System message cannot contain videos.') }}\n {%- endif %}\n {%- if do_vision_count %}\n {%- set video_count.value = video_count.value + 1 %}\n {%- endif %}\n {%- if add_vision_id %}\n {{- 'Video ' ~ video_count.value ~ ': ' }}\n {%- endif %}\n {{- '<|vision_start|><|video_pad|><|vision_end|>' }}\n {%- elif 'text' in item %}\n {{- item.text }}\n {%- else %}\n {{- raise_exception('Unexpected item type in content.') }}\n {%- endif %}\n {%- endfor %}\n {%- elif content is none or content is undefined %}\n {{- '' }}\n {%- else %}\n {{- content | tojson }}\n {%- endif %}\n{%- endmacro %}\n{%- if not messages %}\n {{- raise_exception('No messages provided.') }}\n{%- endif %}\n{%- if tools and tools is iterable and tools is not mapping %}\n {{- '<|im_start|>system\\n' }}\n {{- \"# Tools\\n\\nYou have access to the following functions:\\n\\n<tools>\" }}\n {%- for tool in tools %}\n {{- \"\\n\" }}\n {{- tool | tojson }}\n {%- endfor %}\n {{- \"\\n</tools>\" }}\n {{- '\\n\\nIf you choose to call a function ONLY reply in the following format with NO suffix:\\n\\n<tool_call>\\n<function=example_function_name>\\n<parameter=example_parameter_1>\\nvalue_1\\n</parameter>\\n<parameter=example_parameter_2>\\nThis is the value for the second parameter\\nthat can span\\nmultiple lines\\n</parameter>\\n</function>\\n</tool_call>\\n\\n<IMPORTANT>\\nReminder:\\n- Function calls MUST follow the specified format: an inner <function=...></function> block must be nested within <tool_call></tool_call> XML tags\\n- Required parameters MUST be specified\\n- You may provide optional reasoning for your function call in natural language BEFORE the function call, but NOT after\\n- If there is no function call available, answer the question like normal with your current knowledge and do not tell the user about function calls\\n</IMPORTANT>' }}\n {%- if messages[0].role == 'system' %}\n {%- set content = render_content(messages[0].content, false, true)|trim %}\n {%- if content %}\n {{- '\\n\\n' + content }}\n {%- endif %}\n {%- endif %}\n {{- '<|im_end|>\\n' }}\n{%- else %}\n {%- if messages[0].role == 'system' %}\n {%- set content = render_content(messages[0].content, false, true)|trim %}\n {{- '<|im_start|>system\\n' + content + '<|im_end|>\\n' }}\n {%- endif %}\n{%- endif %}\n{%- set ns = namespace(multi_step_tool=true, last_query_index=messages|length - 1) %}\n{%- for message in messages[::-1] %}\n {%- set index = (messages|length - 1) - loop.index0 %}\n {%- if ns.multi_step_tool and message.role == \"user\" %}\n {%- set content = render_content(message.content, false)|trim %}\n {%- if not(content.startswith('<tool_response>') and content.endswith('</tool_response>')) %}\n {%- set ns.multi_step_tool = false %}\n {%- set ns.last_query_index = index %}\n {%- endif %}\n {%- endif %}\n{%- endfor %}\n{%- if ns.multi_step_tool %}\n {{- raise_exception('No user query found in messages.') }}\n{%- endif %}\n{%- for message in messages %}\n {%- set content = render_content(message.content, true)|trim %}\n {%- if message.role == \"system\" %}\n {%- if not loop.first %}\n {{- raise_exception('System message must be at the beginning.') }}\n {%- endif %}\n {%- elif message.role == \"user\" %}\n {{- '<|im_start|>' + message.role + '\\n' + content + '<|im_end|>' + '\\n' }}\n {%- elif message.role == \"assistant\" %}\n {%- set reasoning_content = '' %}\n {%- if message.reasoning_content is string %}\n {%- set reasoning_content = message.reasoning_content %}\n {%- else %}\n {%- if '</think>' in content %}\n {%- set reasoning_content = content.split('</think>')[0].rstrip('\\n').split('<think>')[-1].lstrip('\\n') %}\n {%- set content = content.split('</think>')[-1].lstrip('\\n') %}\n {%- endif %}\n {%- endif %}\n {%- set reasoning_content = reasoning_content|trim %}\n {{- '<|im_start|>' + message.role + '\\n<think>\\n' + reasoning_content + '\\n</think>\\n\\n' + content }}\n {%- if message.tool_calls and message.tool_calls is iterable and message.tool_calls is not mapping %}\n {%- for tool_call in message.tool_calls %}\n {%- if tool_call.function is defined %}\n {%- set tool_call = tool_call.function %}\n {%- endif %}\n {%- if loop.first %}\n {%- if content|trim %}\n {{- '\\n\\n<tool_call>\\n<function=' + tool_call.name + '>\\n' }}\n {%- else %}\n {{- '<tool_call>\\n<function=' + tool_call.name + '>\\n' }}\n {%- endif %}\n {%- else %}\n {{- '\\n<tool_call>\\n<function=' + tool_call.name + '>\\n' }}\n {%- endif %}\n {%- if tool_call.arguments is defined %}\n {%- for args_name, args_value in tool_call.arguments|items %}\n {{- '<parameter=' + args_name + '>\\n' }}\n {%- set args_value = args_value | tojson | safe if args_value is mapping or (args_value is sequence and args_value is not string) else args_value | string %}\n {{- args_value }}\n {{- '\\n</parameter>\\n' }}\n {%- endfor %}\n {%- endif %}\n {{- '</function>\\n</tool_call>' }}\n {%- endfor %}\n {%- endif %}\n {{- '<|im_end|>\\n' }}\n {%- elif message.role == \"tool\" %}\n {%- if loop.previtem and loop.previtem.role != \"tool\" %}\n {{- '<|im_start|>user' }}\n {%- endif %}\n {{- '\\n<tool_response>\\n' }}\n {{- content }}\n {{- '\\n</tool_response>' }}\n {%- if not loop.last and loop.nextitem.role != \"tool\" %}\n {{- '<|im_end|>\\n' }}\n {%- elif loop.last %}\n {{- '<|im_end|>\\n' }}\n {%- endif %}\n {%- else %}\n {{- raise_exception('Unexpected message role.') }}\n {%- endif %}\n{%- endfor %}\n{%- if add_generation_prompt %}\n {{- '<|im_start|>assistant\\n' }}\n {%- if enable_thinking is defined and enable_thinking is true %}\n {{- '<think>\\n' }}\n {%- else %}\n {{- '<think>\\n\\n</think>\\n\\n' }}\n {%- endif %}\n{%- endif %}",
"required_functions": [],
diff --git a/crates/legacy/hanashi/configs/rendering/qwen3.6.json b/crates/legacy/hanashi/configs/rendering/qwen3.6.json
index 4cbfd4b86..c83ee4098 100644
--- a/crates/legacy/hanashi/configs/rendering/qwen3.6.json
+++ b/crates/legacy/hanashi/configs/rendering/qwen3.6.json
@@ -1,4 +1,5 @@
{
+ "grammar_trigger": "</think>",
"jinja": {
"template": "{%- set image_count = namespace(value=0) %}\n{%- set video_count = namespace(value=0) %}\n{%- macro render_content(content, do_vision_count, is_system_content=false) %}\n {%- if content is string %}\n {{- content }}\n {%- elif content is mapping %}\n {{- content | tojson }}\n {%- elif content is iterable and content is not mapping %}\n {%- for item in content %}\n {%- if 'image' in item or 'image_url' in item or item.type == 'image' %}\n {%- if is_system_content %}\n {{- raise_exception('System message cannot contain images.') }}\n {%- endif %}\n {%- if do_vision_count %}\n {%- set image_count.value = image_count.value + 1 %}\n {%- endif %}\n {%- if add_vision_id %}\n {{- 'Picture ' ~ image_count.value ~ ': ' }}\n {%- endif %}\n {{- '<|vision_start|><|image_pad|><|vision_end|>' }}\n {%- elif 'video' in item or item.type == 'video' %}\n {%- if is_system_content %}\n {{- raise_exception('System message cannot contain videos.') }}\n {%- endif %}\n {%- if do_vision_count %}\n {%- set video_count.value = video_count.value + 1 %}\n {%- endif %}\n {%- if add_vision_id %}\n {{- 'Video ' ~ video_count.value ~ ': ' }}\n {%- endif %}\n {{- '<|vision_start|><|video_pad|><|vision_end|>' }}\n {%- elif 'text' in item %}\n {{- item.text }}\n {%- else %}\n {{- raise_exception('Unexpected item type in content.') }}\n {%- endif %}\n {%- endfor %}\n {%- elif content is none or content is undefined %}\n {{- '' }}\n {%- else %}\n {{- raise_exception('Unexpected content type.') }}\n {%- endif %}\n{%- endmacro %}\n{%- if not messages %}\n {{- raise_exception('No messages provided.') }}\n{%- endif %}\n{%- if tools and tools is iterable and tools is not mapping %}\n {{- '<|im_start|>system\\n' }}\n {{- \"# Tools\\n\\nYou have access to the following functions:\\n\\n<tools>\" }}\n {%- for tool in tools %}\n {{- \"\\n\" }}\n {{- tool | tojson }}\n {%- endfor %}\n {{- \"\\n</tools>\" }}\n {{- '\\n\\nIf you choose to call a function ONLY reply in the following format with NO suffix:\\n\\n<tool_call>\\n<function=example_function_name>\\n<parameter=example_parameter_1>\\nvalue_1\\n</parameter>\\n<parameter=example_parameter_2>\\nThis is the value for the second parameter\\nthat can span\\nmultiple lines\\n</parameter>\\n</function>\\n</tool_call>\\n\\n<IMPORTANT>\\nReminder:\\n- Function calls MUST follow the specified format: an inner <function=...></function> block must be nested within <tool_call></tool_call> XML tags\\n- Required parameters MUST be specified\\n- You may provide optional reasoning for your function call in natural language BEFORE the function call, but NOT after\\n- If there is no function call available, answer the question like normal with your current knowledge and do not tell the user about function calls\\n</IMPORTANT>' }}\n {%- if messages[0].role == 'system' %}\n {%- set content = render_content(messages[0].content, false, true)|trim %}\n {%- if content %}\n {{- '\\n\\n' + content }}\n {%- endif %}\n {%- endif %}\n {{- '<|im_end|>\\n' }}\n{%- else %}\n {%- if messages[0].role == 'system' %}\n {%- set content = render_content(messages[0].content, false, true)|trim %}\n {{- '<|im_start|>system\\n' + content + '<|im_end|>\\n' }}\n {%- endif %}\n{%- endif %}\n{%- set ns = namespace(multi_step_tool=true, last_query_index=messages|length - 1) %}\n{%- for message in messages[::-1] %}\n {%- set index = (messages|length - 1) - loop.index0 %}\n {%- if ns.multi_step_tool and message.role == \"user\" %}\n {%- set content = render_content(message.content, false)|trim %}\n {%- if not(content.startswith('<tool_response>') and content.endswith('</tool_response>')) %}\n {%- set ns.multi_step_tool = false %}\n {%- set ns.last_query_index = index %}\n {%- endif %}\n {%- endif %}\n{%- endfor %}\n{%- if ns.multi_step_tool %}\n {{- raise_exception('No user query found in messages.') }}\n{%- endif %}\n{%- for message in messages %}\n {%- set content = render_content(message.content, true)|trim %}\n {%- if message.role == \"system\" %}\n {%- if not loop.first %}\n {{- raise_exception('System message must be at the beginning.') }}\n {%- endif %}\n {%- elif message.role == \"user\" %}\n {{- '<|im_start|>' + message.role + '\\n' + content + '<|im_end|>' + '\\n' }}\n {%- elif message.role == \"assistant\" %}\n {%- set reasoning_content = '' %}\n {%- if message.reasoning_content is string %}\n {%- set reasoning_content = message.reasoning_content %}\n {%- else %}\n {%- if '</think>' in content %}\n {%- set reasoning_content = content.split('</think>')[0].rstrip('\\n').split('<think>')[-1].lstrip('\\n') %}\n {%- set content = content.split('</think>')[-1].lstrip('\\n') %}\n {%- endif %}\n {%- endif %}\n {%- set reasoning_content = reasoning_content|trim %}\n {%- if loop.index0 > ns.last_query_index %}\n {{- '<|im_start|>' + message.role + '\\n<think>\\n' + reasoning_content + '\\n</think>\\n\\n' + content }}\n {%- else %}\n {{- '<|im_start|>' + message.role + '\\n' + content }}\n {%- endif %}\n {%- if message.tool_calls and message.tool_calls is iterable and message.tool_calls is not mapping %}\n {%- for tool_call in message.tool_calls %}\n {%- if tool_call.function is defined %}\n {%- set tool_call = tool_call.function %}\n {%- endif %}\n {%- if loop.first %}\n {%- if content|trim %}\n {{- '\\n\\n<tool_call>\\n<function=' + tool_call.name + '>\\n' }}\n {%- else %}\n {{- '<tool_call>\\n<function=' + tool_call.name + '>\\n' }}\n {%- endif %}\n {%- else %}\n {{- '\\n<tool_call>\\n<function=' + tool_call.name + '>\\n' }}\n {%- endif %}\n {%- if tool_call.arguments is defined %}\n {%- for args_name, args_value in tool_call.arguments|items %}\n {{- '<parameter=' + args_name + '>\\n' }}\n {%- set args_value = args_value | tojson | safe if args_value is mapping or (args_value is sequence and args_value is not string) else args_value | string %}\n {{- args_value }}\n {{- '\\n</parameter>\\n' }}\n {%- endfor %}\n {%- endif %}\n {{- '</function>\\n</tool_call>' }}\n {%- endfor %}\n {%- endif %}\n {{- '<|im_end|>\\n' }}\n {%- elif message.role == \"tool\" %}\n {%- if loop.previtem and loop.previtem.role != \"tool\" %}\n {{- '<|im_start|>user' }}\n {%- endif %}\n {{- '\\n<tool_response>\\n' }}\n {{- content }}\n {{- '\\n</tool_response>' }}\n {%- if not loop.last and loop.nextitem.role != \"tool\" %}\n {{- '<|im_end|>\\n' }}\n {%- elif loop.last %}\n {{- '<|im_end|>\\n' }}\n {%- endif %}\n {%- else %}\n {{- raise_exception('Unexpected message role.') }}\n {%- endif %}\n{%- endfor %}\n{%- if add_generation_prompt %}\n {{- '<|im_start|>assistant\\n' }}\n {%- if enable_thinking is defined and enable_thinking is true %}\n {{- '<think>\\n' }}\n {%- else %}\n {{- '<think>\\n\\n</think>\\n\\n' }}\n {%- endif %}\n{%- endif %}",
"required_functions": [],
diff --git a/crates/legacy/hanashi/configs/rendering/qwen3.8.json b/crates/legacy/hanashi/configs/rendering/qwen3.8.json
index 456fe80d9..37d516b72 100644
--- a/crates/legacy/hanashi/configs/rendering/qwen3.8.json
+++ b/crates/legacy/hanashi/configs/rendering/qwen3.8.json
@@ -1,4 +1,5 @@
{
+ "grammar_trigger": "</think>",
"jinja": {
"template": "{%- set image_count = namespace(value=0) %}\n{%- set video_count = namespace(value=0) %}\n{%- macro render_content(content, do_vision_count, is_system_content=false) %}\n {%- if content is string %}\n {{- content }}\n {%- elif content is iterable and content is not mapping %}\n {%- for item in content %}\n {%- if 'image' in item or 'image_url' in item or item.type == 'image' %}\n {%- if is_system_content %}\n {{- raise_exception('System message cannot contain images.') }}\n {%- endif %}\n {%- if do_vision_count %}\n {%- set image_count.value = image_count.value + 1 %}\n {%- endif %}\n {%- if add_vision_id %}\n {{- 'Picture ' ~ image_count.value ~ ': ' }}\n {%- endif %}\n {{- '<|vision_start|><|image_pad|><|vision_end|>' }}\n {%- elif 'video' in item or item.type == 'video' %}\n {%- if is_system_content %}\n {{- raise_exception('System message cannot contain videos.') }}\n {%- endif %}\n {%- if do_vision_count %}\n {%- set video_count.value = video_count.value + 1 %}\n {%- endif %}\n {%- if add_vision_id %}\n {{- 'Video ' ~ video_count.value ~ ': ' }}\n {%- endif %}\n {{- '<|vision_start|><|video_pad|><|vision_end|>' }}\n {%- elif 'text' in item %}\n {{- item.text }}\n {%- else %}\n {{- raise_exception('Unexpected item type in content.') }}\n {%- endif %}\n {%- endfor %}\n {%- elif content is none or content is undefined %}\n {{- '' }}\n {%- else %}\n {{- raise_exception('Unexpected content type.') }}\n {%- endif %}\n{%- endmacro %}\n{%- if not messages %}\n {{- raise_exception('No messages provided.') }}\n{%- endif %}\n{%- set reasoning_instructions = '' %}\n{%- if enable_thinking is undefined or enable_thinking is true %}\n {%- set resolved_reasoning_effort = reasoning_effort|default('xhigh') %}\n {%- if resolved_reasoning_effort not in ('xhigh', 'medium', 'low') %}\n {{- raise_exception('Unexpected reasoning effort ' ~ reasoning_effort ~ '. Supported types are xhigh (default), medium, and low.') }}\n {%- endif %}\n {%- if resolved_reasoning_effort == 'xhigh' %}\n {%- set reasoning_instructions = 'Reasoning effort is set to xhigh. Please think carefully through the task, validate key assumptions, consider plausible alternatives, and prioritize correctness, consistency, and clarity in the final answer.' %}\n {%- elif resolved_reasoning_effort == 'low' %}\n {%- set reasoning_instructions = 'Reasoning effort is set to low. Keep your thinking brief and focused, moving directly to the conclusion without unnecessary elaboration.' %}\n {%- endif %}\n{%- endif %}\n{%- if tools and tools is iterable and tools is not mapping %}\n {{- '<|im_start|>system\\n' }}\n {%- if reasoning_instructions %}\n {{- reasoning_instructions + '\\n\\n' }}\n {%- endif %}\n {{- \"# Tools\\n\\nYou have access to the following functions:\\n\\n<tools>\" }}\n {%- for tool in tools %}\n {{- \"\\n\" }}\n {{- tool | tojson }}\n {%- endfor %}\n {{- \"\\n</tools>\" }}\n {{- '\\n\\nIf you choose to call a function ONLY reply in the following format with NO suffix:\\n\\n<tool_call>\\n<function=example_function_name>\\n<parameter=example_parameter_1>\\nvalue_1\\n</parameter>\\n<parameter=example_parameter_2>\\nThis is the value for the second parameter\\nthat can span\\nmultiple lines\\n</parameter>\\n</function>\\n</tool_call>\\n\\n<IMPORTANT>\\nReminder:\\n- Function calls MUST follow the specified format: an inner <function=...></function> block must be nested within <tool_call></tool_call> XML tags\\n- Required parameters MUST be specified\\n- You may provide optional reasoning for your function call in natural language BEFORE the function call, but NOT after\\n- If there is no function call available, answer the question like normal with your current knowledge and do not tell the user about function calls\\n</IMPORTANT>' }}\n {%- if messages[0].role == 'system' %}\n {%- set content = render_content(messages[0].content, false, true)|trim %}\n {%- if content %}\n {{- '\\n\\n' + content }}\n {%- endif %}\n {%- endif %}\n {{- '<|im_end|>\\n' }}\n{%- else %}\n {%- if messages[0].role == 'system' %}\n {%- set content = render_content(messages[0].content, false, true)|trim %}\n {%- if content %}\n {{- '<|im_start|>system\\n' + (reasoning_instructions + '\\n\\n' if reasoning_instructions else '') + content + '<|im_end|>\\n' }}\n {%- elif reasoning_instructions %}\n {{- '<|im_start|>system\\n' + reasoning_instructions + '<|im_end|>\\n' }}\n {%- endif %}\n {%- elif reasoning_instructions %}\n {{- '<|im_start|>system\\n' + reasoning_instructions + '<|im_end|>\\n' }}\n {%- endif %}\n{%- endif %}\n{%- set ns = namespace(multi_step_tool=true, last_query_index=messages|length - 1) %}\n{%- for message in messages[::-1] %}\n {%- set index = (messages|length - 1) - loop.index0 %}\n {%- if ns.multi_step_tool and message.role == \"user\" %}\n {%- set content = render_content(message.content, false)|trim %}\n {%- if not(content.startswith('<tool_response>') and content.endswith('</tool_response>')) %}\n {%- set ns.multi_step_tool = false %}\n {%- set ns.last_query_index = index %}\n {%- endif %}\n {%- endif %}\n{%- endfor %}\n{%- if ns.multi_step_tool %}\n {{- raise_exception('No user query found in messages.') }}\n{%- endif %}\n{%- for message in messages %}\n {%- set content = render_content(message.content, true)|trim %}\n {%- if message.role == \"system\" %}\n {%- if not loop.first %}\n {{- raise_exception('System message must be at the beginning.') }}\n {%- endif %}\n {%- elif message.role == \"user\" %}\n {{- '<|im_start|>' + message.role + '\\n' + content + '<|im_end|>' + '\\n' }}\n {%- elif message.role == \"assistant\" %}\n {%- set reasoning_content = '' %}\n {%- if message.reasoning_content is string %}\n {%- set reasoning_content = message.reasoning_content %}\n {%- endif %}\n {%- set reasoning_content = reasoning_content|trim %}\n {%- if preserve_thinking is undefined or preserve_thinking is true or loop.index0 > ns.last_query_index %}\n {{- '<|im_start|>' + message.role + '\\n<think>\\n' + reasoning_content + '\\n</think>\\n\\n' + content }}\n {%- else %}\n {{- '<|im_start|>' + message.role + '\\n' + content }}\n {%- endif %}\n {%- if message.tool_calls and message.tool_calls is iterable and message.tool_calls is not mapping %}\n {%- for tool_call in message.tool_calls %}\n {%- if tool_call.function is defined %}\n {%- set tool_call = tool_call.function %}\n {%- endif %}\n {%- if loop.first %}\n {%- if content|trim %}\n {{- '\\n\\n<tool_call>\\n<function=' + tool_call.name + '>\\n' }}\n {%- else %}\n {{- '<tool_call>\\n<function=' + tool_call.name + '>\\n' }}\n {%- endif %}\n {%- else %}\n {{- '\\n<tool_call>\\n<function=' + tool_call.name + '>\\n' }}\n {%- endif %}\n {%- if tool_call.arguments is defined and tool_call.arguments != '' %}\n {%- for args_name, args_value in tool_call.arguments|items %}\n {{- '<parameter=' + args_name + '>\\n' }}\n {%- set args_value = args_value | string if args_value is string else args_value | tojson | safe %}\n {{- args_value }}\n {{- '\\n</parameter>\\n' }}\n {%- endfor %}\n {%- endif %}\n {{- '</function>\\n</tool_call>' }}\n {%- endfor %}\n {%- endif %}\n {{- '<|im_end|>\\n' }}\n {%- elif message.role == \"tool\" %}\n {%- if loop.previtem and loop.previtem.role != \"tool\" %}\n {{- '<|im_start|>user' }}\n {%- endif %}\n {{- '\\n<tool_response>\\n' }}\n {{- content }}\n {{- '\\n</tool_response>' }}\n {%- if not loop.last and loop.nextitem.role != \"tool\" %}\n {{- '<|im_end|>\\n' }}\n {%- elif loop.last %}\n {{- '<|im_end|>\\n' }}\n {%- endif %}\n {%- else %}\n {{- raise_exception('Unexpected message role.') }}\n {%- endif %}\n{%- endfor %}\n{%- if add_generation_prompt %}\n {{- '<|im_start|>assistant\\n' }}\n {%- if enable_thinking is defined and enable_thinking is false %}\n {{- '<think>\\n\\n</think>\\n\\n' }}\n {%- else %}\n {{- '<think>\\n' }}\n {%- endif %}\n{%- endif %}",
"required_functions": [],
diff --git a/crates/legacy/hanashi/configs/rendering/qwen3.json b/crates/legacy/hanashi/configs/rendering/qwen3.json
index a03f39d93..a74f815cf 100644
--- a/crates/legacy/hanashi/configs/rendering/qwen3.json
+++ b/crates/legacy/hanashi/configs/rendering/qwen3.json
@@ -1,4 +1,5 @@
{
+ "grammar_trigger": "</think>",
"jinja": {
"template": "{%- if tools %}\n {{- '<|im_start|>system\\n' }}\n {%- if messages[0].role == 'system' %}\n {{- messages[0].content + '\\n\\n' }}\n {%- endif %}\n {{- \"# Tools\\n\\nYou may call one or more functions to assist with the user query.\\n\\nYou are provided with function signatures within <tools></tools> XML tags:\\n<tools>\" }}\n {%- for tool in tools %}\n {{- \"\\n\" }}\n {{- tool | tojson }}\n {%- endfor %}\n {{- \"\\n</tools>\\n\\nFor each function call, return a json object with function name and arguments within <tool_call></tool_call> XML tags:\\n<tool_call>\\n{\\\"name\\\": <function-name>, \\\"arguments\\\": <args-json-object>}\\n</tool_call><|im_end|>\\n\" }}\n{%- else %}\n {%- if messages[0].role == 'system' %}\n {{- '<|im_start|>system\\n' + messages[0].content + '<|im_end|>\\n' }}\n {%- endif %}\n{%- endif %}\n{%- set ns = namespace(multi_step_tool=true, last_query_index=messages|length - 1) %}\n{%- for message in messages[::-1] %}\n {%- set index = (messages|length - 1) - loop.index0 %}\n {%- if ns.multi_step_tool and message.role == \"user\" and message.content is string and not(message.content.startswith('<tool_response>') and message.content.endswith('</tool_response>')) %}\n {%- set ns.multi_step_tool = false %}\n {%- set ns.last_query_index = index %}\n {%- endif %}\n{%- endfor %}\n{%- for message in messages %}\n {%- if message.content is string %}\n {%- set content = message.content %}\n {%- elif message.content is mapping %}\n {%- set content = message.content | tojson %}\n {%- else %}\n {%- set content = '' %}\n {%- endif %}\n {%- if (message.role == \"user\") or (message.role == \"system\" and not loop.first) %}\n {{- '<|im_start|>' + message.role + '\\n' + content + '<|im_end|>' + '\\n' }}\n {%- elif message.role == \"assistant\" %}\n {%- set reasoning_content = '' %}\n {%- if message.reasoning_content is string %}\n {%- set reasoning_content = message.reasoning_content %}\n {%- else %}\n {%- if '</think>' in content %}\n {%- set reasoning_content = content.split('</think>')[0].rstrip('\\n').split('<think>')[-1].lstrip('\\n') %}\n {%- set content = content.split('</think>')[-1].lstrip('\\n') %}\n {%- endif %}\n {%- endif %}\n {%- if loop.index0 > ns.last_query_index %}\n {%- if loop.last or (not loop.last and reasoning_content) %}\n {{- '<|im_start|>' + message.role + '\\n<think>\\n' + reasoning_content.strip('\\n') + '\\n</think>\\n\\n' + content.lstrip('\\n') }}\n {%- else %}\n {{- '<|im_start|>' + message.role + '\\n' + content }}\n {%- endif %}\n {%- else %}\n {{- '<|im_start|>' + message.role + '\\n' + content }}\n {%- endif %}\n {%- if message.tool_calls %}\n {%- for tool_call in message.tool_calls %}\n {%- if (loop.first and content) or (not loop.first) %}\n {{- '\\n' }}\n {%- endif %}\n {%- if tool_call.function %}\n {%- set tool_call = tool_call.function %}\n {%- endif %}\n {{- '<tool_call>\\n{\"name\": \"' }}\n {{- tool_call.name }}\n {{- '\", \"arguments\": ' }}\n {%- if tool_call.arguments is string %}\n {{- tool_call.arguments }}\n {%- else %}\n {{- tool_call.arguments | tojson }}\n {%- endif %}\n {{- '}\\n</tool_call>' }}\n {%- endfor %}\n {%- endif %}\n {{- '<|im_end|>\\n' }}\n {%- elif message.role == \"tool\" %}\n {%- if loop.first or (messages[loop.index0 - 1].role != \"tool\") %}\n {{- '<|im_start|>user' }}\n {%- endif %}\n {{- '\\n<tool_response>\\n' }}\n {{- content }}\n {{- '\\n</tool_response>' }}\n {%- if loop.last or (messages[loop.index0 + 1].role != \"tool\") %}\n {{- '<|im_end|>\\n' }}\n {%- endif %}\n {%- endif %}\n{%- endfor %}\n{%- if add_generation_prompt %}\n {{- '<|im_start|>assistant\\n' }}\n {%- if enable_thinking is defined and enable_thinking is false %}\n {{- '<think>\\n\\n</think>\\n\\n' }}\n {%- endif %}\n{%- endif %}",
"required_functions": [],
diff --git a/crates/legacy/hanashi/src/chat/hanashi/mod.rs b/crates/legacy/hanashi/src/chat/hanashi/mod.rs
index 734ffb6fc..7a1ec01d1 100644
--- a/crates/legacy/hanashi/src/chat/hanashi/mod.rs
+++ b/crates/legacy/hanashi/src/chat/hanashi/mod.rs
@@ -46,6 +46,10 @@ pub struct HanashiEncodingImpl {
}
impl HanashiEncodingImpl {
+ pub fn grammar_trigger(&self) -> Option<&str> {
+ self.config.rendering.grammar_trigger(&self.state.messages)
+ }
+
pub fn new(
config: HanashiConfig,
tokenizer: Arc<Tokenizer>,
diff --git a/crates/legacy/hanashi/src/chat/hanashi/renderer/config/mod.rs b/crates/legacy/hanashi/src/chat/hanashi/renderer/config/mod.rs
index 194fde38a..2e9c30b3b 100644
--- a/crates/legacy/hanashi/src/chat/hanashi/renderer/config/mod.rs
+++ b/crates/legacy/hanashi/src/chat/hanashi/renderer/config/mod.rs
@@ -7,7 +7,7 @@ pub use jinja_function::JinjaFunction;
use serde::{Deserialize, Serialize};
use shoji::types::{
basic::ReasoningEffort,
- session::chat::{ChatContentBlockType, ChatRole},
+ session::chat::{ChatContentBlock, ChatContentBlockType, ChatMessage, ChatRole},
};
use crate::chat::hanashi::messages::{
@@ -20,12 +20,47 @@ pub struct RendererConfig {
pub jinja: JinjaConfig,
pub canonization: CanonicalConfig,
pub rendering: IndexMap<ChatRole, RenderedConfig>,
+ #[serde(default, skip_serializing_if = "Option::is_none")]
+ pub grammar_trigger: Option<String>,
// Needed when the default lives inside the template rather than its field mapping.
#[serde(default, skip_serializing_if = "Option::is_none")]
pub default_reasoning_effort: Option<ReasoningEffort>,
}
impl RendererConfig {
+ pub fn grammar_trigger(
+ &self,
+ messages: &[ChatMessage],
+ ) -> Option<&str> {
+ let effort = messages.iter().find_map(|message| {
+ let role = self.rendering.get(&message.role)?;
+ let has_reasoning_control = role.context.values().any(|field| {
+ matches!(
+ &field.config,
+ FieldConfig::Unique {
+ block: ChatContentBlockType::ReasoningEffort,
+ ..
+ }
+ )
+ });
+ if !has_reasoning_control {
+ return None;
+ }
+ message.content.iter().find_map(|block| match block {
+ ChatContentBlock::ReasoningEffort {
+ value,
+ } => Some(*value),
+ _ => None,
+ })
+ });
+ let effort =
+ effort.filter(|effort| *effort != ReasoningEffort::Default).or_else(|| self.default_reasoning_effort());
+ if effort == Some(ReasoningEffort::Disabled) {
+ return None;
+ }
+ self.grammar_trigger.as_deref()
+ }
+
pub fn default_reasoning_effort(&self) -> Option<ReasoningEffort> {
if let Some(effort) = self.default_reasoning_effort {
return Some(effort);
@@ -130,6 +165,77 @@ mod tests {
use super::*;
use crate::chat::hanashi::config::HanashiConfig;
+ #[test]
+ fn bundled_grammar_triggers_identify_answer_boundaries() {
+ for (config, expected) in [
+ (HanashiConfig::Gemma4, Some("<channel|>")),
+ (HanashiConfig::GptOss, Some("<|channel|>final<|message|>")),
+ (HanashiConfig::MuseGlimmer, Some(" to=user<|message|>")),
+ (HanashiConfig::Lfm25Thinking, Some("</think>")),
+ (HanashiConfig::Qwen3, Some("</think>")),
+ (HanashiConfig::Qwen3Thinking, Some("</think>")),
+ (HanashiConfig::Qwen35, Some("</think>")),
+ (HanashiConfig::Qwen36, Some("</think>")),
+ (HanashiConfig::Qwen38, Some("</think>")),
+ (HanashiConfig::FunctionGemma, None),
+ (HanashiConfig::Gemma3, None),
+ (HanashiConfig::Lfm2, None),
+ (HanashiConfig::Lfm25Instruct, None),
+ (HanashiConfig::Llama32, None),
+ (HanashiConfig::Qwen3Instruct, None),
+ ] {
+ assert_eq!(config.resolve().unwrap().rendering.grammar_trigger(&[]), expected, "{config:?}");
+ }
+ }
+
+ #[test]
+ fn grammar_trigger_tracks_reasoning_controls() {
+ for config in [
+ HanashiConfig::Gemma4,
+ HanashiConfig::Qwen3,
+ HanashiConfig::Qwen35,
+ HanashiConfig::Qwen36,
+ HanashiConfig::Qwen38,
+ ] {
+ let config = config.resolve().unwrap().rendering;
+ let disabled = [ChatMessage::system().with_reasoning_effort(ReasoningEffort::Disabled)];
+ assert_eq!(config.grammar_trigger(&disabled), None);
+ let default = [ChatMessage::system().with_reasoning_effort(ReasoningEffort::Default)];
+ assert_eq!(config.grammar_trigger(&default), config.grammar_trigger.as_deref());
+ let uncontrolled = [ChatMessage::user().with_reasoning_effort(ReasoningEffort::Disabled)];
+ assert_eq!(config.grammar_trigger(&uncontrolled), config.grammar_trigger.as_deref());
+ }
+ }
+
+ #[test]
+ fn grammar_trigger_respects_a_disabled_default() {
+ let mut config = HanashiConfig::Gemma4.resolve().unwrap().rendering;
+ config.default_reasoning_effort = Some(ReasoningEffort::Disabled);
+ assert_eq!(config.grammar_trigger(&[]), None);
+ assert_eq!(
+ config.grammar_trigger(&[ChatMessage::system().with_reasoning_effort(ReasoningEffort::Default)]),
+ None
+ );
+ assert_eq!(
+ config.grammar_trigger(&[ChatMessage::system().with_reasoning_effort(ReasoningEffort::High)]),
+ Some("<channel|>")
+ );
+ }
+
+ #[test]
+ fn custom_grammar_trigger_serializes_and_is_optional() {
+ let mut json: serde_json::Value =
+ serde_json::from_str(include_str!(concat!(env!("CARGO_MANIFEST_DIR"), "/configs/rendering/gemma-4.json")))
+ .unwrap();
+ json["grammar_trigger"] = serde_json::json!("<answer>");
+ let config: RendererConfig = serde_json::from_value(json.clone()).unwrap();
+ assert_eq!(config.grammar_trigger(&[]), Some("<answer>"));
+ assert_eq!(serde_json::to_value(&config).unwrap()["grammar_trigger"], "<answer>");
+ json.as_object_mut().unwrap().remove("grammar_trigger");
+ let config: RendererConfig = serde_json::from_value(json).unwrap();
+ assert_eq!(config.grammar_trigger(&[]), None);
+ }
+
#[test]
fn omitted_default_and_disabled_do_not_prove_the_template_default_is_off() {
let mut config = HanashiConfig::Qwen35.resolve().unwrap().rendering;
diff --git a/crates/legacy/hanashi/src/chat/harmony/mod.rs b/crates/legacy/hanashi/src/chat/harmony/mod.rs
index c13637517..de883677e 100644
--- a/crates/legacy/hanashi/src/chat/harmony/mod.rs
+++ b/crates/legacy/hanashi/src/chat/harmony/mod.rs
@@ -72,6 +72,10 @@ pub struct HarmonyEncodingImpl {
}
impl HarmonyEncodingImpl {
+ pub fn grammar_trigger(&self) -> Option<&str> {
+ Some("<|channel|>final<|message|>")
+ }
+
pub fn new(
config: HarmonyConfig,
tokenizer_location: TokenizerLocation,
diff --git a/crates/legacy/hanashi/src/chat/mod.rs b/crates/legacy/hanashi/src/chat/mod.rs
index 774d407f1..86bf7b61d 100644
--- a/crates/legacy/hanashi/src/chat/mod.rs
+++ b/crates/legacy/hanashi/src/chat/mod.rs
@@ -41,6 +41,10 @@ pub enum Encoding {
}
impl Encoding {
+ pub fn grammar_trigger(&self) -> Option<&str> {
+ dispatch!(infallible self, grammar_trigger)
+ }
+
pub fn try_append(
&mut self,
messages: &[ChatMessage],
diff --git a/crates/legacy/nagare/src/chat/token.rs b/crates/legacy/nagare/src/chat/token.rs
index a310b529b..313278a92 100644
--- a/crates/legacy/nagare/src/chat/token.rs
+++ b/crates/legacy/nagare/src/chat/token.rs
@@ -15,7 +15,8 @@ use shoji::{
Error,
chat_message::{Output, ToolCallState},
chat_token::{
- Backend, Instance as ChatTokenBackendInstance, StreamInput, StreamOutput, TokenStreamMetrics,
+ Backend, Instance as ChatTokenBackendInstance, StreamConfig, StreamInput, StreamOutput,
+ TokenStreamMetrics,
},
},
},
@@ -138,7 +139,11 @@ impl Session {
}
let time_prefill_start = Instant::now();
let cancel_token = cancel_token.child_token();
- let stream = instance.stream(&self.input_tokens, self.state.as_mut(), config.clone(), cancel_token.clone());
+ let stream_config = StreamConfig {
+ reply: config.clone(),
+ grammar_trigger: self.encoding.grammar_trigger().map(str::to_owned),
+ };
+ let stream = instance.stream(&self.input_tokens, self.state.as_mut(), stream_config, cancel_token.clone());
let output_tokens_start = self.encoding.state().tokens.len();
let stream_state = StreamingState {
diff --git a/crates/legacy/nagare/src/chat/tool_cancellation_tests.rs b/crates/legacy/nagare/src/chat/tool_cancellation_tests.rs
index 5aaad744c..792ebb58b 100644
--- a/crates/legacy/nagare/src/chat/tool_cancellation_tests.rs
+++ b/crates/legacy/nagare/src/chat/tool_cancellation_tests.rs
@@ -8,7 +8,7 @@ use hanashi::chat::{EncodingConfig, hanashi::config::HanashiConfig};
use shoji::{
traits::backend::{
Error, Instance as BackendInstance, InstanceStream, NoMetricsStream, State,
- chat_token::{Instance as TokenInstance, StreamInput, StreamOutput, TokenStreamMetrics},
+ chat_token::{Instance as TokenInstance, StreamConfig, StreamInput, StreamOutput, TokenStreamMetrics},
},
types::{
basic::SamplingParameters,
@@ -31,7 +31,7 @@ struct FunctionGemma {
}
impl BackendInstance for FunctionGemma {
- type StreamConfig = ChatReplyConfig;
+ type StreamConfig = StreamConfig;
type StreamInput = StreamInput;
type StreamOutput = StreamOutput;
type StreamMetrics = Option<TokenStreamMetrics>;
@@ -47,7 +47,7 @@ impl BackendInstance for FunctionGemma {
&'a self,
input: &'a StreamInput,
_state: &'a mut dyn State,
- _config: ChatReplyConfig,
+ _config: StreamConfig,
_cancel_token: CancellationToken,
) -> Pin<Box<dyn InstanceStream<Item = Result<StreamOutput, Error>, Metrics = Self::StreamMetrics> + Send + 'a>>
{
diff --git a/crates/legacy/shoji/src/traits/backend/chat_token.rs b/crates/legacy/shoji/src/traits/backend/chat_token.rs
index 40961794c..a686abcce 100644
--- a/crates/legacy/shoji/src/traits/backend/chat_token.rs
+++ b/crates/legacy/shoji/src/traits/backend/chat_token.rs
@@ -25,6 +25,12 @@ pub struct TokenStreamMetrics {
pub num_tokens_returned: usize,
}
+pub struct StreamConfig {
+ pub reply: ChatReplyConfig,
+ /// Generated marker after which grammar constraints apply; absent means immediate enforcement.
+ pub grammar_trigger: Option<String>,
+}
+
pub type StreamInput = Vec<u64>;
pub type StreamOutput = TokenStreamOutput;
pub type StreamMetrics = Option<TokenStreamMetrics>;
@@ -39,7 +45,7 @@ pub trait Backend: Send + Sync {
pub trait Instance:
InstanceTrait<
- StreamConfig = ChatReplyConfig,
+ StreamConfig = StreamConfig,
StreamInput = StreamInput,
StreamOutput = StreamOutput,
StreamMetrics = StreamMetrics,
diff --git a/crates/legacy/shoji/src/types/session/classification/codec.rs b/crates/legacy/shoji/src/types/session/classification/codec.rs
index 15ba61993..a48948fc7 100644
--- a/crates/legacy/shoji/src/types/session/classification/codec.rs
+++ b/crates/legacy/shoji/src/types/session/classification/codec.rs
@@ -15,6 +15,5 @@ pub struct ChatTokenCodecConfig {
pub assistant_role_name: String,
pub eos_token: Option<String>,
pub bos_token: Option<String>,
- pub end_of_thinking_tag: Option<String>,
pub default_system_prompt: Option<String>,
}
diff --git a/crates/legacy/uzu/src/engine/bridge/chat_token_backend.rs b/crates/legacy/uzu/src/engine/bridge/chat_token_backend.rs
index 00c5e1c15..075b5b880 100644
--- a/crates/legacy/uzu/src/engine/bridge/chat_token_backend.rs
+++ b/crates/legacy/uzu/src/engine/bridge/chat_token_backend.rs
@@ -13,15 +13,15 @@ use shoji::{
backend::{
Error as ShojiBackendError, Instance as BackendInstance, InstanceStream, NoMetricsStream,
chat_token::{
- Instance as ChatTokenBackendInstance, StreamInput as ChatTokenStreamInput,
- StreamMetrics as ChatTokenStreamMetrics, StreamOutput as ChatTokenStreamOutput, TokenStreamMetrics,
- TokenStreamOutput,
+ Instance as ChatTokenBackendInstance, StreamConfig as ChatTokenStreamConfig,
+ StreamInput as ChatTokenStreamInput, StreamMetrics as ChatTokenStreamMetrics,
+ StreamOutput as ChatTokenStreamOutput, TokenStreamMetrics, TokenStreamOutput,
},
},
},
types::{
basic::{BackendError, SamplingParameters, SamplingSeed},
- session::chat::{ChatConfig, ChatReplyConfig},
+ session::chat::ChatConfig,
},
};
use tokenizers::Tokenizer;
@@ -47,7 +47,6 @@ pub struct UzuChatTokenBackendInstance<B: Backend> {
stop_token_ids: Vec<i32>,
max_context_length: Option<u32>,
sampling_defaults: SamplingParameters,
- grammar_trigger_token_sequence: Option<Vec<u64>>,
}
impl<B: Backend> UzuChatTokenBackendInstance<B> {
@@ -70,8 +69,6 @@ impl<B: Backend> UzuChatTokenBackendInstance<B> {
suffix_repetition_length: generation_config.suffix_repetition_length.map(|value| value as i64),
};
let max_context_length = get_max_context_length(&model, config.context_length.clone());
- let grammar_trigger_token_sequence =
- grammar_trigger_token_sequence(model.end_of_thinking_tag(), model.tokenizer());
Ok(Self {
engine,
@@ -80,13 +77,12 @@ impl<B: Backend> UzuChatTokenBackendInstance<B> {
stop_token_ids,
max_context_length,
sampling_defaults,
- grammar_trigger_token_sequence,
})
}
}
impl<B: Backend> BackendInstance for UzuChatTokenBackendInstance<B> {
- type StreamConfig = ChatReplyConfig;
+ type StreamConfig = ChatTokenStreamConfig;
type StreamInput = ChatTokenStreamInput;
type StreamOutput = ChatTokenStreamOutput;
type StreamMetrics = ChatTokenStreamMetrics;
@@ -121,11 +117,17 @@ impl<B: Backend> BackendInstance for UzuChatTokenBackendInstance<B> {
> {
let state = (state as &mut dyn Any).downcast_mut::<UzuChatTokenBackendInstanceState<B>>().unwrap();
+ let ChatTokenStreamConfig {
+ reply: config,
+ grammar_trigger,
+ } = config;
let token_limit = config.token_limit.map(|count| count as usize);
let grammar = if let Some(grammar_config) = config.grammar {
+ let trigger_token_sequence =
+ grammar_trigger_token_sequence(grammar_trigger.as_deref(), self.model.tokenizer());
let trigger_token_sequence = grammar_trigger_token_sequence_for_prompt(
- self.grammar_trigger_token_sequence.as_deref(),
+ trigger_token_sequence.as_deref(),
input,
self.model.tokenizer(),
);
diff --git a/crates/legacy/uzu/src/engine/bridge/helpers.rs b/crates/legacy/uzu/src/engine/bridge/helpers.rs
index 7988dd114..cc010714c 100644
--- a/crates/legacy/uzu/src/engine/bridge/helpers.rs
+++ b/crates/legacy/uzu/src/engine/bridge/helpers.rs
@@ -69,17 +69,13 @@ pub fn get_grammar(
UzuGrammar::new(&config, tokenizer, trigger_token_sequence, Some(stop_token_ids))
}
-/// The token sequence of the model's end-of-thinking tag. A reasoning model
-/// opens every reply with reasoning tokens that no grammar can accept, so the
-/// grammar must stay disengaged until this sequence has been generated.
-/// Leading whitespace-only tokens are excluded because BPE can merge them with
-/// preceding whitespace in both the prompt and generated reasoning. Tokens that
-/// fuse whitespace with marker text retain their original encoding.
+/// Tokenizes a grammar trigger without leading whitespace-only tokens.
+/// Tokens combining whitespace with marker text retain their original encoding.
pub fn grammar_trigger_token_sequence(
- end_of_thinking_tag: Option<&str>,
+ trigger: Option<&str>,
tokenizer: &Tokenizer,
) -> Option<Vec<u64>> {
- let tag = end_of_thinking_tag?;
+ let tag = trigger?;
if tag.trim().is_empty() {
return None;
}
@@ -380,6 +376,38 @@ mod tests {
}
}
+ #[test]
+ fn gemma_codec_grammar_waits_for_the_closing_channel_marker() {
+ use hanashi::chat::hanashi::config::HanashiConfig;
+ use shoji::types::{basic::ReasoningEffort, session::chat::ChatMessage};
+
+ let config = HanashiConfig::Gemma4.resolve().unwrap().rendering;
+ let mut tokenizer = bpe_tokenizer();
+ tokenizer.add_special_tokens(&[AddedToken::from("<|channel>", true), AddedToken::from("<channel|>", true)]);
+ let opening = tokenizer.token_to_id("<|channel>").unwrap() as u64;
+ let closing = tokenizer.token_to_id("<channel|>").unwrap() as u64;
+ let trigger = grammar_trigger_token_sequence(config.grammar_trigger(&[]), &tokenizer);
+ assert_eq!(trigger, Some(vec![closing]));
+
+ for grammar_config in grammars() {
+ let mut grammar = get_grammar(grammar_config, &tokenizer, &[12], trigger.clone()).unwrap();
+ grammar.accept_token(opening).unwrap();
+ assert!(!grammar.next_bitmask(&mut [0]));
+ grammar.accept_token(5).unwrap();
+ assert!(!grammar.next_bitmask(&mut [0]));
+ grammar.accept_token(closing).unwrap();
+ assert!(grammar.next_bitmask(&mut [0]));
+ }
+
+ let disabled = [ChatMessage::system().with_reasoning_effort(ReasoningEffort::Disabled)];
+ let trigger = grammar_trigger_token_sequence(config.grammar_trigger(&disabled), &tokenizer);
+ assert_eq!(trigger, None);
+ for grammar_config in grammars() {
+ let mut grammar = get_grammar(grammar_config, &tokenizer, &[12], trigger.clone()).unwrap();
+ assert!(grammar.next_bitmask(&mut [0]));
+ }
+ }
+
#[test]
fn grammar_trigger_ignores_missing_or_whitespace_only_tags() {
let tokenizer = bpe_tokenizer();
diff --git a/crates/uzu-engine/src/config/token_codec/chat_codec.rs b/crates/uzu-engine/src/config/token_codec/chat_codec.rs
index 7f66df493..7638f7a17 100644
--- a/crates/uzu-engine/src/config/token_codec/chat_codec.rs
+++ b/crates/uzu-engine/src/config/token_codec/chat_codec.rs
@@ -31,9 +31,3 @@ pub struct ChatCodecConfig {
#[serde(default)] // TODO: ??? inconsistency in converted models in registry workaround, remove when resolved
pub reasoning_config: Option<ReasoningConfig>,
}
-
-impl ChatCodecConfig {
- pub fn end_of_thinking_tag(&self) -> Option<&str> {
- self.response_format.as_ref()?.prefixes.last().map(String::as_str)
- }
-}
diff --git a/crates/uzu-engine/src/config/token_codec/mod.rs b/crates/uzu-engine/src/config/token_codec/mod.rs
index 2a0ed693f..b6d4d9bea 100644
--- a/crates/uzu-engine/src/config/token_codec/mod.rs
+++ b/crates/uzu-engine/src/config/token_codec/mod.rs
@@ -21,7 +21,6 @@ impl From<&AnyTokenCodecConfig> for ShojiTokenCodecConfig {
assistant_role_name: config.assistant_role_name.clone(),
eos_token: config.eos_token.clone(),
bos_token: config.bos_token.clone(),
- end_of_thinking_tag: config.end_of_thinking_tag().map(str::to_owned),
default_system_prompt: config.default_system_prompt.clone(),
}),
AnyTokenCodecConfig::RawTextCodecConfig(_) => ShojiTokenCodecConfig::RawText,
diff --git a/crates/uzu-engine/src/engine/language_model/mod.rs b/crates/uzu-engine/src/engine/language_model/mod.rs
index e4ea16421..baa2525cc 100644
--- a/crates/uzu-engine/src/engine/language_model/mod.rs
+++ b/crates/uzu-engine/src/engine/language_model/mod.rs
@@ -6,7 +6,7 @@ use tokenizers::Tokenizer;
use crate::{
backends::common::{Backend, Context, DeviceCapabilities, Kernels, kernel::ContextRingUpdateKernel},
- config::{model::language_model::LanguageModelConfig, token_codec::AnyTokenCodecConfig},
+ config::model::language_model::LanguageModelConfig,
data_type::DataType,
encodable_block::{
decoder::{Decoder, DecoderError},
@@ -31,7 +31,6 @@ pub struct LanguageModel<B: Backend> {
sampling: Sampling<B>,
context_ring_update: <B::Kernels as Kernels>::ContextRingUpdateKernel,
generation_config: GenerationConfig,
- end_of_thinking_tag: Option<String>,
tokenizer: Arc<Tokenizer>,
vocab_size: usize,
}
@@ -98,13 +97,6 @@ impl<B: Backend> Engine<B> {
weight_loader.assert_all_tensors_read()?;
let generation_config = config.generation_config;
- let end_of_thinking_tag = match &config.token_codec_config {
- AnyTokenCodecConfig::ChatCodecConfig(token_codec_config) => {
- token_codec_config.end_of_thinking_tag().map(str::to_owned)
- },
- AnyTokenCodecConfig::RawTextCodecConfig(_) => None,
- };
-
let vocab_size = config.decoder_config.vocab_size as usize;
Ok(LanguageModel {
@@ -114,7 +106,6 @@ impl<B: Backend> Engine<B> {
sampling,
context_ring_update,
generation_config,
- end_of_thinking_tag,
tokenizer,
vocab_size,
})
@@ -165,10 +156,6 @@ impl<B: Backend> LanguageModel<B> {
&self.generation_config
}
- pub fn end_of_thinking_tag(&self) -> Option<&str> {
- self.end_of_thinking_tag.as_deref()
- }
-
pub fn tokenizer(&self) -> &Arc<Tokenizer> {
&self.tokenizer
}
diff --git a/scripts/benchmarks/engine-uzu/src/engine.rs b/scripts/benchmarks/engine-uzu/src/engine.rs
index 53b0eda6a..d52e7a8fe 100644
--- a/scripts/benchmarks/engine-uzu/src/engine.rs
+++ b/scripts/benchmarks/engine-uzu/src/engine.rs
@@ -8,7 +8,7 @@ use tokio_util::sync::CancellationToken;
use uzu::{
engine::{Engine, EngineConfig},
session::chat::ChatInstanceKind,
- traits::backend::chat_token::{Instance, TokenStreamOutput},
+ traits::backend::chat_token::{Instance, StreamConfig, TokenStreamOutput},
types::{
basic::{SamplingMethod, ToolCall, ToolDescription, ToolFunction, ToolNamespace},
session::chat::{ChatConfig, ChatContentBlock, ChatMessage, ChatReplyConfig, ChatRole},
@@ -89,6 +89,10 @@ impl UzuEngine {
let mut first_token = None;
let started = Instant::now();
+ let config = StreamConfig {
+ reply: config,
+ grammar_trigger: None,
+ };
let mut stream = self.instance.stream(tokens, state.as_mut(), config, CancellationToken::new());
loop {
let Some(event) = stream.next().await else {