Skip to content

LLM inference in C/C++

C++ ggml

mtmd: support webp via ffmpeg (#27520)

Xuan-Son Nguyend775b8967a46
diff --git a/tools/mtmd/mtmd-helper.cpp b/tools/mtmd/mtmd-helper.cpp
index 8b8e5f5f66..f719323c0b 100644
--- a/tools/mtmd/mtmd-helper.cpp
+++ b/tools/mtmd/mtmd-helper.cpp
@@ -358,6 +358,15 @@ static bool decode_audio_from_buf(const unsigned char * buf_in, size_t len, int
} // namespace audio_helpers
+static bool is_webp_file(const unsigned char * buf, size_t len) {
+ // WEBP ref: https://developers.google.com/speed/webp/docs/riff_container
+ return len >= 12 && memcmp(buf, "RIFF", 4) == 0 && memcmp(buf + 8, "WEBP", 4) == 0;
+}
+
+#ifdef MTMD_VIDEO
+static mtmd_bitmap * decode_webp_with_ffmpeg(mtmd_context * mctx, const unsigned char * buf, size_t len, bool placeholder);
+#endif
+
mtmd_helper_bitmap_wrapper mtmd_helper_bitmap_init_from_buf(mtmd_context * ctx, const unsigned char * buf, size_t len, bool placeholder) {
// calculate the hash if needed
std::string id;
@@ -397,6 +406,19 @@ mtmd_helper_bitmap_wrapper mtmd_helper_bitmap_init_from_buf(mtmd_context * ctx,
// otherwise, fallthrough to video decoding (if supported)
}
+#ifdef MTMD_VIDEO
+ // stb_image does not support webp; decode it with ffmpeg as a single frame
+ if (!result && is_webp_file(buf, len)) {
+ result = decode_webp_with_ffmpeg(ctx, buf, len, placeholder);
+ if (!result) {
+ LOG_ERR("%s: failed to decode webp buffer\n", __func__);
+ return {nullptr, nullptr};
+ }
+ mtmd_bitmap_set_id(result, id.empty() ? nullptr : id.c_str());
+ return {result, nullptr};
+ }
+#endif
+
// last try: load as video
#ifdef MTMD_VIDEO
if (!result) {
@@ -820,6 +842,33 @@ static std::string video_resolve_bin(const char * bin_dir, const char * name) {
return result;
}
+#ifdef MTMD_VIDEO
+static mtmd_bitmap * decode_webp_with_ffmpeg(mtmd_context * mctx, const unsigned char * buf, size_t len, bool placeholder) {
+ auto params = mtmd_helper_video_init_params_default();
+ mtmd_helper_video vctx;
+ vctx.mctx = mctx;
+ vctx.input_buf.assign(buf, buf + len);
+ vctx.ffmpeg_bin = video_resolve_bin(params.ffmpeg_bin_dir, "ffmpeg");
+ vctx.ffprobe_bin = video_resolve_bin(params.ffmpeg_bin_dir, "ffprobe");
+ if (!vctx.probe(0.0f)) {
+ return nullptr;
+ }
+ if (placeholder) {
+ return mtmd_bitmap_init(vctx.info.width, vctx.info.height, nullptr);
+ }
+ // still image: the fps filter would output no frame, so disable it
+ vctx.fps_target = 0.0f;
+ if (!vctx.start_ffmpeg(0.0f)) {
+ return nullptr;
+ }
+ mtmd_bitmap * frame = vctx.read_next_frame();
+ if (frame) {
+ mtmd_bitmap_set_mergeable(frame, false);
+ }
+ return frame;
+}
+#endif
+
mtmd_helper_video * mtmd_helper_video_init(
mtmd_context * mctx,
const char * path,
diff --git a/tools/mtmd/mtmd-helper.h b/tools/mtmd/mtmd-helper.h
index 5c6b924197..58dfb15250 100644
--- a/tools/mtmd/mtmd-helper.h
+++ b/tools/mtmd/mtmd-helper.h
@@ -45,6 +45,7 @@ MTMD_API struct mtmd_helper_bitmap_wrapper mtmd_helper_bitmap_init_from_file(mtm
// helper function to construct a mtmd_bitmap from a buffer containing a file
// supported formats:
// image: formats supported by stb_image: jpg, png, bmp, gif, etc.
+// webp is decoded via ffmpeg, requires MTMD_VIDEO build with ffmpeg in PATH
// audio: formats supported by miniaudio: wav, mp3, flac
// note:
// - for now, video input is only supported via C++ helper functions