LLM inference in C/C++
C++ ggml
cohere2 moe template parser: enforce JSON schema for text responses if a response schema is provided (#26018)
Matt Thompson8f5ab832ca7d
diff --git a/common/chat.cpp b/common/chat.cppindex c441021b97..2b0b40ccbe 100644--- a/common/chat.cpp+++ b/common/chat.cpp@@ -2179,9 +2179,10 @@ static common_chat_params common_chat_params_init_cohere2moe(const common_chat_t{ COMMON_CHAT_ROLE_SYSTEM, TURN_START + SYSTEM },};- auto has_tools = inputs.tools.is_array() && !inputs.tools.empty();- auto extract_reasoning = inputs.reasoning_format != COMMON_REASONING_FORMAT_NONE;- auto include_grammar = has_tools && inputs.tool_choice != COMMON_CHAT_TOOL_CHOICE_NONE;+ auto has_tools = inputs.tools.is_array() && !inputs.tools.empty();+ auto has_response_format = inputs.json_schema.is_object() && !inputs.json_schema.empty();+ auto extract_reasoning = inputs.reasoning_format != COMMON_REASONING_FORMAT_NONE;+ auto include_grammar = has_response_format || (has_tools && inputs.tool_choice != COMMON_CHAT_TOOL_CHOICE_NONE);if (inputs.has_continuation()) {const auto & msg = inputs.continue_msg;@@ -2212,7 +2213,11 @@ static common_chat_params common_chat_params_init_cohere2moe(const common_chat_tp.optional(p.literal(THINK_END))));}- auto text_content = p.literal(TEXT_START) + p.content(p.until(TEXT_END)) + p.optional(p.literal(TEXT_END));+ auto text_content = has_response_format+ ? p.literal(TEXT_START) ++ p.content(p.schema(p.json(), "response-format-schema", inputs.json_schema)) ++ p.optional(p.literal(TEXT_END))+ : p.literal(TEXT_START) + p.content(p.until(TEXT_END)) + p.optional(p.literal(TEXT_END));if (!has_tools || inputs.tool_choice == COMMON_CHAT_TOOL_CHOICE_NONE) {return generation_prompt + reasoning + text_content + p.optional(p.literal(TURN_END)) + end;@@ -2240,13 +2245,17 @@ static common_chat_params common_chat_params_init_cohere2moe(const common_chat_tdata.parser = parser.save();if (include_grammar) {- data.grammar_lazy = inputs.tool_choice == COMMON_CHAT_TOOL_CHOICE_AUTO;+ data.grammar_lazy = !has_response_format && inputs.tool_choice == COMMON_CHAT_TOOL_CHOICE_AUTO;data.grammar = build_grammar([&](const common_grammar_builder & builder) {foreach_function(inputs.tools, [&](const json & tool) {const auto & function = tool.at("function");auto schema = function.at("parameters");builder.resolve_refs(schema);});+ if (has_response_format) {+ auto schema = inputs.json_schema;+ builder.resolve_refs(schema);+ }parser.build_grammar(builder, data.grammar_lazy);});diff --git a/tests/test-chat.cpp b/tests/test-chat.cppindex 28b28ca0f4..371a305fc3 100644--- a/tests/test-chat.cpp+++ b/tests/test-chat.cpp@@ -2849,6 +2849,17 @@ static void test_template_output_peg_parsers(bool detailed_debug) {.expect(message_assist).run();+ // JSON output schema+ tst.test(+ "I need to output the invoice details in JSON<|END_THINKING|>"+ "<|START_TEXT|>{\"amount\": 123.45, \"date\": \"2025-12-03\"}<|END_TEXT|>")+ .reasoning_format(COMMON_REASONING_FORMAT_DEEPSEEK)+ .json_schema(invoice_schema)+ .tools({ special_function_tool })+ .expect_reasoning("I need to output the invoice details in JSON")+ .expect_content(R"({"amount": 123.45, "date": "2025-12-03"})")+ .run();+// Single tool call with reasoning.tst.test("I'm\nthinking<|END_THINKING|>"