Skip to content

LLM inference in C/C++

C++ ggml

chat: harden peg-native tool call parsing (#24329)

Pascal581e8eca8b41
diff --git a/common/chat-auto-parser-generator.cpp b/common/chat-auto-parser-generator.cpp
index 6f825a131b..37ca55c8df 100644
--- a/common/chat-auto-parser-generator.cpp
+++ b/common/chat-auto-parser-generator.cpp
@@ -103,6 +103,10 @@ common_chat_params peg_generator::generate_parser(const common_chat_template &
data.grammar_triggers = {
{ COMMON_GRAMMAR_TRIGGER_TYPE_WORD, trigger_marker }
};
+ if (autoparser.tools.format.openai_wrapper_trigger) {
+ // model emits the OpenAI function wrapper, trigger on it
+ data.grammar_triggers.push_back({ COMMON_GRAMMAR_TRIGGER_TYPE_WORD, "{\"type\": \"function\"," });
+ }
}
}
@@ -224,13 +228,13 @@ common_peg_parser analyze_tools::build_tool_parser_json_native(parser_build_cont
auto single_tool_parser = p.standard_json_tools(
format.per_call_start, format.per_call_end, inputs.tools, inputs.parallel_tool_calls,
inputs.tool_choice == COMMON_CHAT_TOOL_CHOICE_REQUIRED, name_field, args_field, format.tools_array_wrapped,
- format.fun_name_is_key, format.id_field, format.gen_id_field, format.parameter_order);
+ format.fun_name_is_key, format.id_field, format.gen_id_field, format.parameter_order, format.openai_wrapper_trigger);
tools_parser = p.trigger_rule("tool-calls", p.one_or_more(single_tool_parser + p.space()));
} else {
tools_parser = p.standard_json_tools(
format.section_start, format.section_end, inputs.tools, inputs.parallel_tool_calls,
inputs.tool_choice == COMMON_CHAT_TOOL_CHOICE_REQUIRED, name_field, args_field, format.tools_array_wrapped,
- format.fun_name_is_key, format.id_field, format.gen_id_field, format.parameter_order);
+ format.fun_name_is_key, format.id_field, format.gen_id_field, format.parameter_order, format.openai_wrapper_trigger);
}
// Handle content wrappers if present
diff --git a/common/chat-auto-parser.h b/common/chat-auto-parser.h
index 7858f6572f..9e8113f244 100644
--- a/common/chat-auto-parser.h
+++ b/common/chat-auto-parser.h
@@ -181,6 +181,7 @@ struct tool_format_analysis {
bool fun_name_is_key = false; // In JSON format function name is JSON key, i.e. { "<funname>": { ... arguments ... } }
bool tools_array_wrapped = false; // Tool calls wrapped in JSON array [...]
+ bool openai_wrapper_trigger = false; // model emits the OpenAI function wrapper, trigger on it
std::string function_field = "function";
std::string name_field = "name";
diff --git a/common/chat-diff-analyzer.cpp b/common/chat-diff-analyzer.cpp
index ecd9c807c1..b166ee5a18 100644
--- a/common/chat-diff-analyzer.cpp
+++ b/common/chat-diff-analyzer.cpp
@@ -165,6 +165,14 @@ static std::vector<std::function<void(const common_chat_template & tmpl, autopar
LOG_DBG(ANSI_ORANGE "[Patch: Apriel 1.6]\n" ANSI_RESET);
}
},
+ // template uses the JSON {name, parameters} tool instruction, emits the OpenAI function wrapper
+ [](const common_chat_template & tmpl, autoparser & analysis) -> void {
+ if (tmpl.src.find("Respond in the format {\"name\": function name") != std::string::npos &&
+ tmpl.src.find("Do not use variables.") != std::string::npos) {
+ analysis.tools.format.openai_wrapper_trigger = true;
+ LOG_DBG(ANSI_ORANGE "[Patch: JSON name/parameters tool instruction]\n" ANSI_RESET);
+ }
+ },
});
diff --git a/common/chat-peg-parser.cpp b/common/chat-peg-parser.cpp
index 23b5b38412..a3aa765d1c 100644
--- a/common/chat-peg-parser.cpp
+++ b/common/chat-peg-parser.cpp
@@ -745,7 +745,8 @@ common_peg_parser common_chat_peg_builder::build_json_tools_flat_keys(
const std::string & effective_args_key,
const std::string & call_id_key,
const std::string & gen_call_id_key,
- const std::vector<std::string> & parameters_order) {
+ const std::vector<std::string> & parameters_order,
+ bool accept_openai_wrapper) {
auto tool_choices = choice();
auto name_key_parser = literal("\"" + effective_name_key + "\"");
@@ -807,7 +808,13 @@ common_peg_parser common_chat_peg_builder::build_json_tools_flat_keys(
return idx_a < idx_b;
});
- auto ordered_body = tool_open(literal("{")) + space();
+ // accept an optional leading "type": "function" field when the model emits the OpenAI wrapper
+ common_peg_parser type_field = eps();
+ if (accept_openai_wrapper) {
+ type_field = optional(literal("\"type\"") + space() + literal(":") + space() +
+ literal("\"function\"") + space() + literal(",") + space());
+ }
+ auto ordered_body = tool_open(literal("{")) + space() + type_field;
for (size_t i = 0; i < parser_pairs.size(); i++) {
ordered_body = ordered_body + parser_pairs[i].first;
if (i < parser_pairs.size() - 1) {
@@ -870,7 +877,8 @@ common_peg_parser common_chat_peg_builder::standard_json_tools(
bool function_is_key,
const std::string & call_id_key,
const std::string & gen_call_id_key,
- const std::vector<std::string> & parameters_order) {
+ const std::vector<std::string> & parameters_order,
+ bool accept_openai_wrapper) {
if (!tools.is_array() || tools.empty()) {
return eps();
}
@@ -888,7 +896,7 @@ common_peg_parser common_chat_peg_builder::standard_json_tools(
if (!name_spec.first.empty() || !args_spec.first.empty()) {
tool_choices = build_json_tools_nested_keys(tools, effective_name_key, effective_args_key, call_id_key, gen_call_id_key);
} else {
- tool_choices = build_json_tools_flat_keys(tools, effective_name_key, effective_args_key, call_id_key, gen_call_id_key, parameters_order);
+ tool_choices = build_json_tools_flat_keys(tools, effective_name_key, effective_args_key, call_id_key, gen_call_id_key, parameters_order, accept_openai_wrapper);
}
}
diff --git a/common/chat-peg-parser.h b/common/chat-peg-parser.h
index a4643fbea8..b3ffd7de2d 100644
--- a/common/chat-peg-parser.h
+++ b/common/chat-peg-parser.h
@@ -120,7 +120,8 @@ class common_chat_peg_builder : public common_peg_parser_builder {
bool function_is_key = false,
const std::string & call_id_key = "",
const std::string & gen_call_id_key = "",
- const std::vector<std::string> & parameters_order = {});
+ const std::vector<std::string> & parameters_order = {},
+ bool accept_openai_wrapper = false);
// Legacy-compatible helper for building XML/tagged style tool calls
// Used by tests and manual parsers
@@ -157,7 +158,8 @@ class common_chat_peg_builder : public common_peg_parser_builder {
const std::string & effective_args_key,
const std::string & call_id_key,
const std::string & gen_call_id_key,
- const std::vector<std::string> & parameters_order);
+ const std::vector<std::string> & parameters_order,
+ bool accept_openai_wrapper);
};
inline common_peg_arena build_chat_peg_parser(
diff --git a/common/chat.cpp b/common/chat.cpp
index bad53e8b5a..05c2e85bee 100644
--- a/common/chat.cpp
+++ b/common/chat.cpp
@@ -2678,8 +2678,10 @@ common_chat_msg common_chat_peg_parse(const common_peg_arena & src_pars
}
return msg;
}
- throw std::runtime_error(std::string("Failed to parse input at pos ") + std::to_string(result.end) + ": " +
- effective_input.substr(result.end));
+ LOG_WRN("%s: unparsed %s output: %s\n", __func__, common_chat_format_name(params.format),
+ effective_input.substr(result.end).c_str());
+ throw std::runtime_error(std::string("The model produced output that does not match the expected ") +
+ common_chat_format_name(params.format) + " format");
}
common_chat_msg msg;