LLM inference in C/C++
C++ ggml
chat: harden peg-native tool call parsing (#24329)
Pascal581e8eca8b41
diff --git a/common/chat-auto-parser-generator.cpp b/common/chat-auto-parser-generator.cppindex 6f825a131b..37ca55c8df 100644--- a/common/chat-auto-parser-generator.cpp+++ b/common/chat-auto-parser-generator.cpp@@ -103,6 +103,10 @@ common_chat_params peg_generator::generate_parser(const common_chat_template &data.grammar_triggers = {{ COMMON_GRAMMAR_TRIGGER_TYPE_WORD, trigger_marker }};+ if (autoparser.tools.format.openai_wrapper_trigger) {+ // model emits the OpenAI function wrapper, trigger on it+ data.grammar_triggers.push_back({ COMMON_GRAMMAR_TRIGGER_TYPE_WORD, "{\"type\": \"function\"," });+ }}}@@ -224,13 +228,13 @@ common_peg_parser analyze_tools::build_tool_parser_json_native(parser_build_contauto single_tool_parser = p.standard_json_tools(format.per_call_start, format.per_call_end, inputs.tools, inputs.parallel_tool_calls,inputs.tool_choice == COMMON_CHAT_TOOL_CHOICE_REQUIRED, name_field, args_field, format.tools_array_wrapped,- format.fun_name_is_key, format.id_field, format.gen_id_field, format.parameter_order);+ format.fun_name_is_key, format.id_field, format.gen_id_field, format.parameter_order, format.openai_wrapper_trigger);tools_parser = p.trigger_rule("tool-calls", p.one_or_more(single_tool_parser + p.space()));} else {tools_parser = p.standard_json_tools(format.section_start, format.section_end, inputs.tools, inputs.parallel_tool_calls,inputs.tool_choice == COMMON_CHAT_TOOL_CHOICE_REQUIRED, name_field, args_field, format.tools_array_wrapped,- format.fun_name_is_key, format.id_field, format.gen_id_field, format.parameter_order);+ format.fun_name_is_key, format.id_field, format.gen_id_field, format.parameter_order, format.openai_wrapper_trigger);}// Handle content wrappers if presentdiff --git a/common/chat-auto-parser.h b/common/chat-auto-parser.hindex 7858f6572f..9e8113f244 100644--- a/common/chat-auto-parser.h+++ b/common/chat-auto-parser.h@@ -181,6 +181,7 @@ struct tool_format_analysis {bool fun_name_is_key = false; // In JSON format function name is JSON key, i.e. { "<funname>": { ... arguments ... } }bool tools_array_wrapped = false; // Tool calls wrapped in JSON array [...]+ bool openai_wrapper_trigger = false; // model emits the OpenAI function wrapper, trigger on itstd::string function_field = "function";std::string name_field = "name";diff --git a/common/chat-diff-analyzer.cpp b/common/chat-diff-analyzer.cppindex ecd9c807c1..b166ee5a18 100644--- a/common/chat-diff-analyzer.cpp+++ b/common/chat-diff-analyzer.cpp@@ -165,6 +165,14 @@ static std::vector<std::function<void(const common_chat_template & tmpl, autoparLOG_DBG(ANSI_ORANGE "[Patch: Apriel 1.6]\n" ANSI_RESET);}},+ // template uses the JSON {name, parameters} tool instruction, emits the OpenAI function wrapper+ [](const common_chat_template & tmpl, autoparser & analysis) -> void {+ if (tmpl.src.find("Respond in the format {\"name\": function name") != std::string::npos &&+ tmpl.src.find("Do not use variables.") != std::string::npos) {+ analysis.tools.format.openai_wrapper_trigger = true;+ LOG_DBG(ANSI_ORANGE "[Patch: JSON name/parameters tool instruction]\n" ANSI_RESET);+ }+ },});diff --git a/common/chat-peg-parser.cpp b/common/chat-peg-parser.cppindex 23b5b38412..a3aa765d1c 100644--- a/common/chat-peg-parser.cpp+++ b/common/chat-peg-parser.cpp@@ -745,7 +745,8 @@ common_peg_parser common_chat_peg_builder::build_json_tools_flat_keys(const std::string & effective_args_key,const std::string & call_id_key,const std::string & gen_call_id_key,- const std::vector<std::string> & parameters_order) {+ const std::vector<std::string> & parameters_order,+ bool accept_openai_wrapper) {auto tool_choices = choice();auto name_key_parser = literal("\"" + effective_name_key + "\"");@@ -807,7 +808,13 @@ common_peg_parser common_chat_peg_builder::build_json_tools_flat_keys(return idx_a < idx_b;});- auto ordered_body = tool_open(literal("{")) + space();+ // accept an optional leading "type": "function" field when the model emits the OpenAI wrapper+ common_peg_parser type_field = eps();+ if (accept_openai_wrapper) {+ type_field = optional(literal("\"type\"") + space() + literal(":") + space() ++ literal("\"function\"") + space() + literal(",") + space());+ }+ auto ordered_body = tool_open(literal("{")) + space() + type_field;for (size_t i = 0; i < parser_pairs.size(); i++) {ordered_body = ordered_body + parser_pairs[i].first;if (i < parser_pairs.size() - 1) {@@ -870,7 +877,8 @@ common_peg_parser common_chat_peg_builder::standard_json_tools(bool function_is_key,const std::string & call_id_key,const std::string & gen_call_id_key,- const std::vector<std::string> & parameters_order) {+ const std::vector<std::string> & parameters_order,+ bool accept_openai_wrapper) {if (!tools.is_array() || tools.empty()) {return eps();}@@ -888,7 +896,7 @@ common_peg_parser common_chat_peg_builder::standard_json_tools(if (!name_spec.first.empty() || !args_spec.first.empty()) {tool_choices = build_json_tools_nested_keys(tools, effective_name_key, effective_args_key, call_id_key, gen_call_id_key);} else {- tool_choices = build_json_tools_flat_keys(tools, effective_name_key, effective_args_key, call_id_key, gen_call_id_key, parameters_order);+ tool_choices = build_json_tools_flat_keys(tools, effective_name_key, effective_args_key, call_id_key, gen_call_id_key, parameters_order, accept_openai_wrapper);}}diff --git a/common/chat-peg-parser.h b/common/chat-peg-parser.hindex a4643fbea8..b3ffd7de2d 100644--- a/common/chat-peg-parser.h+++ b/common/chat-peg-parser.h@@ -120,7 +120,8 @@ class common_chat_peg_builder : public common_peg_parser_builder {bool function_is_key = false,const std::string & call_id_key = "",const std::string & gen_call_id_key = "",- const std::vector<std::string> & parameters_order = {});+ const std::vector<std::string> & parameters_order = {},+ bool accept_openai_wrapper = false);// Legacy-compatible helper for building XML/tagged style tool calls// Used by tests and manual parsers@@ -157,7 +158,8 @@ class common_chat_peg_builder : public common_peg_parser_builder {const std::string & effective_args_key,const std::string & call_id_key,const std::string & gen_call_id_key,- const std::vector<std::string> & parameters_order);+ const std::vector<std::string> & parameters_order,+ bool accept_openai_wrapper);};inline common_peg_arena build_chat_peg_parser(diff --git a/common/chat.cpp b/common/chat.cppindex bad53e8b5a..05c2e85bee 100644--- a/common/chat.cpp+++ b/common/chat.cpp@@ -2678,8 +2678,10 @@ common_chat_msg common_chat_peg_parse(const common_peg_arena & src_pars}return msg;}- throw std::runtime_error(std::string("Failed to parse input at pos ") + std::to_string(result.end) + ": " +- effective_input.substr(result.end));+ LOG_WRN("%s: unparsed %s output: %s\n", __func__, common_chat_format_name(params.format),+ effective_input.substr(result.end).c_str());+ throw std::runtime_error(std::string("The model produced output that does not match the expected ") ++ common_chat_format_name(params.format) + " format");}common_chat_msg msg;