LLM inference in C/C++
C++ ggml
server : accept data: URLs for input_video and input_audio (#27735)
Abhiram9cffdcc80158
diff --git a/tools/server/server-common.cpp b/tools/server/server-common.cppindex c30955e89f..2ac98b6fdd 100644--- a/tools/server/server-common.cpp+++ b/tools/server/server-common.cpp@@ -1062,8 +1062,7 @@ json oaicompat_completion_params_parse(const json & body) {static void handle_media(std::vector<raw_buffer> & out_files,const std::string & url,- const std::string & media_path,- bool accept_base64_uri) {+ const std::string & media_path) {if (!media_path.empty()) {// should already be enforced by arg.cpp, but checking just in caseGGML_ASSERT(media_path.back() == DIRECTORY_SEPARATOR);@@ -1104,15 +1103,17 @@ static void handle_media(data.assign((std::istreambuf_iterator<char>(file)), std::istreambuf_iterator<char>());out_files.push_back(data);- } else if (accept_base64_uri && string_starts_with(url, "data:")) {- // try to decode base64 image+ } else if (string_starts_with(url, "data:")) {+ // try to decode base64 image, video, or audiostd::vector<std::string> parts = string_split<std::string>(url, /*separator*/ ',');if (parts.size() != 2) {- throw std::runtime_error("Invalid uri-encoded base64 value");- } else if (!string_starts_with(parts[0], "data:image/")) {- throw std::runtime_error("Invalid uri format: " + parts[0]);+ throw std::invalid_argument("Invalid uri-encoded base64 value");+ } else if (!string_starts_with(parts[0], "data:image/")+ && !string_starts_with(parts[0], "data:video/")+ && !string_starts_with(parts[0], "data:audio/")) {+ throw std::invalid_argument("Invalid uri format: " + parts[0]);} else if (!string_ends_with(parts[0], "base64")) {- throw std::runtime_error("uri must be base64 encoded");+ throw std::invalid_argument("uri must be base64 encoded");} else {auto base64_data = parts[1];auto decoded_data = base64_decode(base64_data);@@ -1219,7 +1220,7 @@ json oaicompat_chat_params_parse(json image_url = json_value(p, "image_url", json::object());std::string url = json_value(image_url, "url", std::string());- handle_media(out_files, url, opt.media_path, true);+ handle_media(out_files, url, opt.media_path);p["type"] = "media_marker";p["text"] = get_media_marker();@@ -1234,7 +1235,7 @@ json oaicompat_chat_params_parse(json input_audio = json_value(p, "input_audio", json::object());std::string url = json_value(input_audio, "data",json_value(input_audio, "url", std::string()));- handle_media(out_files, url, opt.media_path, false);+ handle_media(out_files, url, opt.media_path);p["type"] = "media_marker";p["text"] = get_media_marker();@@ -1248,7 +1249,7 @@ json oaicompat_chat_params_parse(json input_video = json_value(p, "input_video", json::object());std::string url = json_value(input_video, "data",json_value(input_video, "url", std::string()));- handle_media(out_files, url, opt.media_path, false);+ handle_media(out_files, url, opt.media_path);p["type"] = "media_marker";p["text"] = get_media_marker();diff --git a/tools/server/tests/unit/test_vision_api.py b/tools/server/tests/unit/test_vision_api.pyindex 8b01c5372c..3bf868e662 100644--- a/tools/server/tests/unit/test_vision_api.py+++ b/tools/server/tests/unit/test_vision_api.py@@ -71,6 +71,7 @@ def test_v1_models_supports_multimodal_capability():("What is this:\n", "malformed", False, None),("What is this:\n", "https://google.com/404", False, None), # non-existent image("What is this:\n", "https://ggml.ai", False, None), # non-image data+ ("What is this:\n", "data:text/html;base64,aGVsbG8=", False, None), # unsupported data uri mime# TODO @ngxson : test with multiple images, no images and with audio])