LLM inference in C/C++
C++ ggml
mtmd: support webp via ffmpeg (#27520)
Xuan-Son Nguyend775b8967a46
diff --git a/tools/mtmd/mtmd-helper.cpp b/tools/mtmd/mtmd-helper.cppindex 8b8e5f5f66..f719323c0b 100644--- a/tools/mtmd/mtmd-helper.cpp+++ b/tools/mtmd/mtmd-helper.cpp@@ -358,6 +358,15 @@ static bool decode_audio_from_buf(const unsigned char * buf_in, size_t len, int} // namespace audio_helpers+static bool is_webp_file(const unsigned char * buf, size_t len) {+ // WEBP ref: https://developers.google.com/speed/webp/docs/riff_container+ return len >= 12 && memcmp(buf, "RIFF", 4) == 0 && memcmp(buf + 8, "WEBP", 4) == 0;+}++#ifdef MTMD_VIDEO+static mtmd_bitmap * decode_webp_with_ffmpeg(mtmd_context * mctx, const unsigned char * buf, size_t len, bool placeholder);+#endif+mtmd_helper_bitmap_wrapper mtmd_helper_bitmap_init_from_buf(mtmd_context * ctx, const unsigned char * buf, size_t len, bool placeholder) {// calculate the hash if neededstd::string id;@@ -397,6 +406,19 @@ mtmd_helper_bitmap_wrapper mtmd_helper_bitmap_init_from_buf(mtmd_context * ctx,// otherwise, fallthrough to video decoding (if supported)}+#ifdef MTMD_VIDEO+ // stb_image does not support webp; decode it with ffmpeg as a single frame+ if (!result && is_webp_file(buf, len)) {+ result = decode_webp_with_ffmpeg(ctx, buf, len, placeholder);+ if (!result) {+ LOG_ERR("%s: failed to decode webp buffer\n", __func__);+ return {nullptr, nullptr};+ }+ mtmd_bitmap_set_id(result, id.empty() ? nullptr : id.c_str());+ return {result, nullptr};+ }+#endif+// last try: load as video#ifdef MTMD_VIDEOif (!result) {@@ -820,6 +842,33 @@ static std::string video_resolve_bin(const char * bin_dir, const char * name) {return result;}+#ifdef MTMD_VIDEO+static mtmd_bitmap * decode_webp_with_ffmpeg(mtmd_context * mctx, const unsigned char * buf, size_t len, bool placeholder) {+ auto params = mtmd_helper_video_init_params_default();+ mtmd_helper_video vctx;+ vctx.mctx = mctx;+ vctx.input_buf.assign(buf, buf + len);+ vctx.ffmpeg_bin = video_resolve_bin(params.ffmpeg_bin_dir, "ffmpeg");+ vctx.ffprobe_bin = video_resolve_bin(params.ffmpeg_bin_dir, "ffprobe");+ if (!vctx.probe(0.0f)) {+ return nullptr;+ }+ if (placeholder) {+ return mtmd_bitmap_init(vctx.info.width, vctx.info.height, nullptr);+ }+ // still image: the fps filter would output no frame, so disable it+ vctx.fps_target = 0.0f;+ if (!vctx.start_ffmpeg(0.0f)) {+ return nullptr;+ }+ mtmd_bitmap * frame = vctx.read_next_frame();+ if (frame) {+ mtmd_bitmap_set_mergeable(frame, false);+ }+ return frame;+}+#endif+mtmd_helper_video * mtmd_helper_video_init(mtmd_context * mctx,const char * path,diff --git a/tools/mtmd/mtmd-helper.h b/tools/mtmd/mtmd-helper.hindex 5c6b924197..58dfb15250 100644--- a/tools/mtmd/mtmd-helper.h+++ b/tools/mtmd/mtmd-helper.h@@ -45,6 +45,7 @@ MTMD_API struct mtmd_helper_bitmap_wrapper mtmd_helper_bitmap_init_from_file(mtm// helper function to construct a mtmd_bitmap from a buffer containing a file// supported formats:// image: formats supported by stb_image: jpg, png, bmp, gif, etc.+// webp is decoded via ffmpeg, requires MTMD_VIDEO build with ffmpeg in PATH// audio: formats supported by miniaudio: wav, mp3, flac// note:// - for now, video input is only supported via C++ helper functions