Skip to content

LLM inference in C/C++

C++ ggml

server: support input_image in function_call_output (#20663) (#22575)

Empressia4098fdc92246
diff --git a/tools/server/server-chat.cpp b/tools/server/server-chat.cpp
index a6fe3c6ba6..a3a4ea152f 100644
--- a/tools/server/server-chat.cpp
+++ b/tools/server/server-chat.cpp
@@ -203,10 +203,24 @@ json server_chat_convert_responses_to_chatcmpl(const json & response_body) {
} else {
json chatcmpl_outputs = item.at("output");
for (json & chatcmpl_output : chatcmpl_outputs) {
- if (!chatcmpl_output.contains("type") || chatcmpl_output.at("type") != "input_text") {
- throw std::invalid_argument("Output of tool call should be 'Input text'");
+ if (!chatcmpl_output.contains("type")) {
+ throw std::invalid_argument("Output of tool call missing 'type' field");
+ }
+ const auto type = chatcmpl_output.at("type");
+ if (type != "input_text" && type != "input_image") {
+ throw std::invalid_argument("Output of tool call should be 'Input text' or 'Input image'");
+ }
+ if (type == "input_text") {
+ chatcmpl_output["type"] = "text";
+ } else if (type == "input_image") {
+ if (!chatcmpl_output.contains("image_url")) {
+ throw std::invalid_argument("'image_url' is required");
+ }
+ chatcmpl_output["type"] = "image_url";
+ chatcmpl_output["image_url"] = json {
+ {"url", chatcmpl_output.at("image_url")}
+ };
}
- chatcmpl_output["type"] = "text";
}
chatcmpl_messages.push_back(json {
{"content", chatcmpl_outputs},