Skip to content

LLM inference in C/C++

C++ ggml

server : accept data: URLs for input_video and input_audio (#27735)

Abhiram9cffdcc80158
diff --git a/tools/server/server-common.cpp b/tools/server/server-common.cpp
index c30955e89f..2ac98b6fdd 100644
--- a/tools/server/server-common.cpp
+++ b/tools/server/server-common.cpp
@@ -1062,8 +1062,7 @@ json oaicompat_completion_params_parse(const json & body) {
static void handle_media(
std::vector<raw_buffer> & out_files,
const std::string & url,
- const std::string & media_path,
- bool accept_base64_uri) {
+ const std::string & media_path) {
if (!media_path.empty()) {
// should already be enforced by arg.cpp, but checking just in case
GGML_ASSERT(media_path.back() == DIRECTORY_SEPARATOR);
@@ -1104,15 +1103,17 @@ static void handle_media(
data.assign((std::istreambuf_iterator<char>(file)), std::istreambuf_iterator<char>());
out_files.push_back(data);
- } else if (accept_base64_uri && string_starts_with(url, "data:")) {
- // try to decode base64 image
+ } else if (string_starts_with(url, "data:")) {
+ // try to decode base64 image, video, or audio
std::vector<std::string> parts = string_split<std::string>(url, /*separator*/ ',');
if (parts.size() != 2) {
- throw std::runtime_error("Invalid uri-encoded base64 value");
- } else if (!string_starts_with(parts[0], "data:image/")) {
- throw std::runtime_error("Invalid uri format: " + parts[0]);
+ throw std::invalid_argument("Invalid uri-encoded base64 value");
+ } else if (!string_starts_with(parts[0], "data:image/")
+ && !string_starts_with(parts[0], "data:video/")
+ && !string_starts_with(parts[0], "data:audio/")) {
+ throw std::invalid_argument("Invalid uri format: " + parts[0]);
} else if (!string_ends_with(parts[0], "base64")) {
- throw std::runtime_error("uri must be base64 encoded");
+ throw std::invalid_argument("uri must be base64 encoded");
} else {
auto base64_data = parts[1];
auto decoded_data = base64_decode(base64_data);
@@ -1219,7 +1220,7 @@ json oaicompat_chat_params_parse(
json image_url = json_value(p, "image_url", json::object());
std::string url = json_value(image_url, "url", std::string());
- handle_media(out_files, url, opt.media_path, true);
+ handle_media(out_files, url, opt.media_path);
p["type"] = "media_marker";
p["text"] = get_media_marker();
@@ -1234,7 +1235,7 @@ json oaicompat_chat_params_parse(
json input_audio = json_value(p, "input_audio", json::object());
std::string url = json_value(input_audio, "data",
json_value(input_audio, "url", std::string()));
- handle_media(out_files, url, opt.media_path, false);
+ handle_media(out_files, url, opt.media_path);
p["type"] = "media_marker";
p["text"] = get_media_marker();
@@ -1248,7 +1249,7 @@ json oaicompat_chat_params_parse(
json input_video = json_value(p, "input_video", json::object());
std::string url = json_value(input_video, "data",
json_value(input_video, "url", std::string()));
- handle_media(out_files, url, opt.media_path, false);
+ handle_media(out_files, url, opt.media_path);
p["type"] = "media_marker";
p["text"] = get_media_marker();
diff --git a/tools/server/tests/unit/test_vision_api.py b/tools/server/tests/unit/test_vision_api.py
index 8b01c5372c..3bf868e662 100644
--- a/tools/server/tests/unit/test_vision_api.py
+++ b/tools/server/tests/unit/test_vision_api.py
@@ -71,6 +71,7 @@ def test_v1_models_supports_multimodal_capability():
("What is this:\n", "malformed", False, None),
("What is this:\n", "https://google.com/404", False, None), # non-existent image
("What is this:\n", "https://ggml.ai", False, None), # non-image data
+ ("What is this:\n", "data:text/html;base64,aGVsbG8=", False, None), # unsupported data uri mime
# TODO @ngxson : test with multiple images, no images and with audio
]
)