LLM inference in C/C++
C++ ggml
mtmd: add lanczos resize method [no release] (#26341)
Xuan-Son Nguyen5f55650a78f9
diff --git a/tools/mtmd/clip-model.h b/tools/mtmd/clip-model.hindex 146eabce23..fec2b01802 100644--- a/tools/mtmd/clip-model.h+++ b/tools/mtmd/clip-model.h@@ -33,7 +33,7 @@ enum resize_algo {RESIZE_ALGO_BILINEAR, // stretch to target resolutionRESIZE_ALGO_BICUBIC, // center-crop when aspect ratio doesn't matchRESIZE_ALGO_BICUBIC_PILLOW,- // RESIZE_ALGO_LANCZOS, // TODO+ RESIZE_ALGO_LANCZOS,};// Padding style for img_tool::resizediff --git a/tools/mtmd/mtmd-image.cpp b/tools/mtmd/mtmd-image.cppindex 36cd463b20..72d35fce69 100644--- a/tools/mtmd/mtmd-image.cpp+++ b/tools/mtmd/mtmd-image.cpp@@ -68,6 +68,9 @@ struct img_tool {case RESIZE_ALGO_BICUBIC_PILLOW:resize_bicubic_pillow(src, dst, target_resolution.width, target_resolution.height);break;+ case RESIZE_ALGO_LANCZOS:+ resize_lanczos_pillow(src, dst, target_resolution.width, target_resolution.height);+ break;default:throw std::runtime_error("Unsupported resize algorithm");}@@ -97,6 +100,9 @@ struct img_tool {case RESIZE_ALGO_BICUBIC_PILLOW:resize_bicubic_pillow(src, resized_image, new_width, new_height);break;+ case RESIZE_ALGO_LANCZOS:+ resize_lanczos_pillow(src, resized_image, new_width, new_height);+ break;default:throw std::runtime_error("Unsupported resize algorithm");}@@ -337,22 +343,50 @@ private:}}- // Bicubic resize function using Pillow's ImagingResample algorithm+ // Pillow-compatible separable resampling (Bicubic and Lanczos)// Adapted from https://github.com/python-pillow/Pillow/blob/main/src/libImaging/Resample.c//- // Key Difference with resize_bicubic:- // 1. Uses separable filtering: horizontal pass followed by vertical pass+ // Key properties:+ // 1. Separable filtering: horizontal pass followed by vertical pass// 2. Pre-computes normalized filter coefficients for each output pixel- // 3. Applies convolution using fixed-point integer arithmetic for performance+ // 3. Fixed-point integer arithmetic (22 fractional bits) for speed and determinismstatic bool resize_bicubic_pillow(const clip_image_u8 & img, clip_image_u8 & dst, int target_width, int target_height) {+ return resize_pillow(img, dst, target_width, target_height, /*use_lanczos=*/false);+ }++ // Lanczos-3 (support radius 3), matches Pillow's Image.LANCZOS+ static bool resize_lanczos_pillow(const clip_image_u8 & img, clip_image_u8 & dst, int target_width, int target_height) {+ return resize_pillow(img, dst, target_width, target_height, /*use_lanczos=*/true);+ }++ static bool resize_pillow(+ const clip_image_u8 & img,+ clip_image_u8 & dst,+ int target_width,+ int target_height,+ bool use_lanczos) {// Fixed-point precision: 22 bits = 32 (int32_t) - 8 (uint8_t pixels) - 2 (headroom for accumulation)// This allows encoding fractional weights as integers: weight * 2^22const int PRECISION_BITS = 32 - 8 - 2;- // Bicubic filter function with a = -0.5 (Note that GGML/PyTorch takes a = -0.75)+ // Resample filter: Lanczos-3 (support [-3, 3]) or bicubic with a = -0.5 (support [-2, 2])+ // Note: GGML/PyTorch bicubic uses a = -0.75, Pillow uses a = -0.5// Returns filter weight for distance x from pixel center- // Support: [-2, 2], meaning the filter influences pixels within 2 units of distance- auto bicubic_filter = [](double x) -> double {+ auto resample_filter = [use_lanczos](double x) -> double {+ if (use_lanczos) {+ if (-3.0 <= x && x < 3.0) {+ auto sinc = [](double v) {+ if (v == 0.0) {+ return 1.0;+ }+ const double pi_v = v * 3.141592653589793238462643383279502884;+ return std::sin(pi_v) / pi_v;+ };+ return sinc(x) * sinc(x / 3.0);+ }+ return 0.0;+ }+constexpr double a = -0.5;if (x < 0.0) {x = -x;@@ -366,8 +400,8 @@ private:return 0.0; // Zero outside [-2, 2]};- // Filter support radius: bicubic extends 2 pixels in each direction- constexpr double filter_support = 2.0;+ // Filter support radius: 2 for bicubic, 3 for lanczos+ const double filter_support = use_lanczos ? 3.0 : 2.0;// Clipping function for 8-bit valuesauto clip8 = [](int val) -> uint8_t {@@ -434,7 +468,7 @@ private:// Compute filter weights for each contributing input pixelfor (x = 0; x < xmax; x++) {// Distance from input pixel center to output pixel center in input space- double w = bicubic_filter((x + xmin - center + 0.5) * ss);+ double w = resample_filter((x + xmin - center + 0.5) * ss);pre_weights[xx * ksize + x] = w;ww += w; // Accumulate for normalization}@@ -463,6 +497,12 @@ private:const double fxp_scale = std::ldexp(1.0, PRECISION_BITS); // 1.0 * 2^PRECISION_BITSfor (int i = 0; i < outSize * ksize; i++) {+ if (use_lanczos) {+ // Pillow adds +/- 0.5 then truncates toward zero; std::round would round twice+ const double rounded = pre_weights[i] * fxp_scale + (pre_weights[i] < 0 ? -0.5 : 0.5);+ weights[i] = static_cast<int32_t>(rounded);+ continue;+ }double tmp_val = pre_weights[i] * fxp_scale;if (pre_weights[i] < 0) {tmp_val -= 0.5;