LLM inference in C/C++
C++ ggml
mtmd: fix mel preprocessor in LFM2 audio (#29403)
Yuri Khrustalevfcc891545b0f
diff --git a/tests/test-mtmd-impl.cpp b/tests/test-mtmd-impl.cppindex 2ec6b23915..5321f2ed1e 100644--- a/tests/test-mtmd-impl.cpp+++ b/tests/test-mtmd-impl.cpp@@ -1,8 +1,10 @@#include "testing.h"+#include "mtmd-audio.h"#include "mtmd-image.h"#include "mtmd-internal.h"+#include <cmath>#include <iostream>#include <stdexcept>#include <string>@@ -137,6 +139,54 @@ MAKE_TEST(test_temporal_merge_grouping) {}}+//+// mtmd_audio+//++MAKE_TEST(test_audio_preprocessor_conformer) {+ clip_hparams hparams;+ hparams.n_mel_bins = 128;+ hparams.audio_sample_rate = 16000;+ hparams.audio_n_fft = 512;+ hparams.audio_window_len = 400;+ hparams.audio_hop_len = 160;++ // 0.4 s of tones, 0.2 s of silence, 0.4 s of a quiet tone+ const double pi = 3.14159265358979323846;+ const int sr = hparams.audio_sample_rate;+ std::vector<float> samples(sr, 0.0f);+ for (int i = 0; i < sr; i++) {+ const double ts = (double) i / sr;+ if (i < 0.4 * sr) {+ samples[i] = (float) (0.3 * std::sin(2 * pi * 300 * ts) + 0.2 * std::sin(2 * pi * 1200 * ts) + 0.1 * std::sin(2 * pi * 3500 * ts));+ } else if (i >= 0.6 * sr) {+ samples[i] = (float) (1e-3 * std::sin(2 * pi * 800 * ts));+ }+ }++ mtmd_audio_preprocessor_conformer preproc(hparams);+ preproc.initialize();+ std::vector<mtmd_audio_mel> mels;+ if (!t.assert_true("preprocess", preproc.preprocess(samples.data(), samples.size(), mels))) {+ return;+ }+ const mtmd_audio_mel & mel = mels[0];+ t.assert_equal("n_len", (int64_t) 101, mel.n_len);++ // reference values from NeMo AudioToMelSpectrogramPreprocessor (liquid-audio 1.3.0)+ // { mel bin, frame, value }+ const std::vector<std::tuple<int, int, float>> cases = {+ { 8, 0, 3.6104f }, // tones+ { 24, 50, -0.6371f }, // silence+ { 24, 60, -0.0476f }, // silence -> quiet tone+ { 100, 20, -0.0913f }, // tones, high bin+ };+ for (const auto & [bin, frame, expected] : cases) {+ const float actual = mel.data[bin * mel.n_len + frame];+ t.assert_true("mel[" + std::to_string(bin) + "][" + std::to_string(frame) + "] = " + std::to_string(actual) + ", expected " + std::to_string(expected), std::fabs(actual - expected) < 1e-3f);+ }+}+//// main//diff --git a/tools/mtmd/mtmd-audio.cpp b/tools/mtmd/mtmd-audio.cppindex c25bf4ec89..6c8b194413 100644--- a/tools/mtmd/mtmd-audio.cpp+++ b/tools/mtmd/mtmd-audio.cpp@@ -283,6 +283,8 @@ struct filter_params {bool norm_per_feature = false;bool use_magnitude = false; // |X| instead of |X|^2float mel_floor = 5.960464477539063e-08f;+ bool mel_floor_add = false; // log(x + floor) instead of log(max(x, floor))+ bool std_eps_after_sqrt = false; // std + eps instead of sqrt(var + eps)};static void log_mel_spectrogram_worker_thread(int ith,@@ -347,7 +349,7 @@ static void log_mel_spectrogram_worker_thread(int ith,for (; k < n_fft_bins; k++) {sum += fft_out[k] * filters.data[(size_t)j * n_fft_bins + k];}- sum = std::max(sum, (double)params.mel_floor);+ sum = params.mel_floor_add ? sum + (double)params.mel_floor : std::max(sum, (double)params.mel_floor);sum = params.use_natural_log? log(sum): log10(sum);@@ -491,7 +493,7 @@ static bool log_mel_spectrogram(var += value * value;}var /= effective_n_len - 1; // unbiased- const double mstd = std::sqrt(var + 1e-5);+ const double mstd = params.std_eps_after_sqrt ? std::sqrt(var) + 1e-5 : std::sqrt(var + 1e-5);for (int64_t j = 0; j < effective_n_len; ++j) {auto &value = out.data[(size_t)i * out.n_len + j];@@ -949,7 +951,8 @@ bool mtmd_audio_preprocessor_qwen3tts_spk::preprocess(const float *void mtmd_audio_preprocessor_conformer::initialize() {cache.fill_sin_cos_table(hparams.audio_n_fft);- cache.fill_hann_window(hparams.audio_window_len, true);+ // NeMo uses a symmetric window: torch.hann_window(periodic=False)+ cache.fill_hann_window(hparams.audio_window_len, false);cache.fill_mel_filterbank_matrix(hparams.n_mel_bins, hparams.audio_n_fft, hparams.audio_sample_rate);}@@ -971,6 +974,8 @@ bool mtmd_audio_preprocessor_conformer::preprocess(const float *params.preemph = 0.97f;params.use_natural_log = true;params.norm_per_feature = true;+ params.mel_floor_add = true;+ params.std_eps_after_sqrt = true;// make sure the cache is initializedGGML_ASSERT(!cache.sin_vals.empty());diff --git a/tools/mtmd/mtmd-audio.h b/tools/mtmd/mtmd-audio.hindex 4a15fe6d4a..36db50526f 100644--- a/tools/mtmd/mtmd-audio.h+++ b/tools/mtmd/mtmd-audio.h@@ -54,6 +54,7 @@ struct mtmd_audio_preprocessor {const clip_hparams & hparams;mtmd_audio_preprocessor(const clip_ctx * ctx): hparams(*clip_get_hparams(ctx)) {}+ mtmd_audio_preprocessor(const clip_hparams & hparams): hparams(hparams) {}virtual ~mtmd_audio_preprocessor() = default;virtual void initialize() = 0; // NOT thread-safe@@ -71,6 +72,7 @@ struct mtmd_audio_preprocessor_whisper : mtmd_audio_preprocessor {struct mtmd_audio_preprocessor_conformer : mtmd_audio_preprocessor {mtmd_audio_preprocessor_conformer(const clip_ctx * ctx) : mtmd_audio_preprocessor(ctx) {}+ mtmd_audio_preprocessor_conformer(const clip_hparams & hparams) : mtmd_audio_preprocessor(hparams) {}void initialize() override;bool preprocess(const float * samples, size_t n_samples, std::vector<mtmd_audio_mel> & output) const override;