LLM inference in C/C++
C++ ggml
mtmd: add various sanity checks (#29276)
Xuan-Son Nguyen73c941b11165
diff --git a/tools/mtmd/clip.cpp b/tools/mtmd/clip.cppindex 1783c7ddfd..feceb7ff76 100644--- a/tools/mtmd/clip.cpp+++ b/tools/mtmd/clip.cpp@@ -1408,9 +1408,14 @@ struct clip_model_loader {}// Load the vision/audio feature layer indices if they are explicitly provided- // NOTE: gguf conversions should standardize the values of the vision feature layer to- // be non-negative, since we use -1 to mark values as unset here.+ // NOTE: gguf conversions should standardize the values of the vision feature layer to be non-negative, since we use -1 to mark values as unset here.get_arr_int(string_format(KEY_FEATURE_LAYERS, prefix), hparams.feature_layers, false);+ for (const auto & v : hparams.feature_layers) {+ if (v > (int) hparams.n_layer) {+ throw std::runtime_error(string_format("%s: feature layer index %d is out of range (n_layer: %d)",+ __func__, v, hparams.n_layer));+ }+ }// model-specific paramsswitch (model.proj_type) {@@ -1456,7 +1461,12 @@ struct clip_model_loader {std::vector<int> wa_layer_indexes_vec;get_arr_int(KEY_WIN_ATTN_LAYER_INDEXES, wa_layer_indexes_vec, false);if (!wa_layer_indexes_vec.empty()) {- hparams.insert_layer_id = wa_layer_indexes_vec[0];+ const int insert_lid = wa_layer_indexes_vec[0];+ if (insert_lid < 0 || insert_lid >= (int) hparams.n_layer) {+ throw std::runtime_error(string_format("%s: layer index %d is out of range (n_layer: %d)",+ __func__, insert_lid, hparams.n_layer));+ }+ hparams.insert_layer_id = insert_lid;}} break;case PROJECTOR_TYPE_INTERNVL:@@ -3226,6 +3236,7 @@ struct clip_model_loader {model.pos_embed = get_tensor(string_format(TN_SAM_POS_EMBD, "weight"));model.patch_embed_proj_w = get_tensor(string_format(TN_SAM_PATCH_EMBD, "weight"));model.patch_embed_proj_b = get_tensor(string_format(TN_SAM_PATCH_EMBD, "bias"));+ model.n_sam_layers = hparams.sam_n_layer;model.sam_layers.resize(model.n_sam_layers);for (int il = 0; il < model.n_sam_layers; ++il) {auto & layer = model.sam_layers[il];@@ -4652,8 +4663,10 @@ bool clip_encode(struct clip_ctx * ctx, struct clip_encode_params * params) {// -> https://huggingface.co/HuggingFaceM4/siglip-so400m-14-980-flash-attn2-navit// -> https://huggingface.co/HuggingFaceM4/siglip-so400m-14-980-flash-attn2-navit/blob/d66538faeba44480d0bfaa42145eef26f9423199/modeling_siglip.py#L316std::vector<int32_t> positions(pos_h * pos_w);- int bucket_coords_h[1024];- int bucket_coords_w[1024];+ // note: sized by the actual patch counts; a tall/wide image produces more+ // than 1024 patches per side and a fixed [1024] array would be overrun+ std::vector<int> bucket_coords_h(pos_h);+ std::vector<int> bucket_coords_w(pos_w);for (int i = 0; i < pos_h; i++){bucket_coords_h[i] = std::floor(70.0*i/pos_h);}@@ -4696,8 +4709,8 @@ bool clip_encode(struct clip_ctx * ctx, struct clip_encode_params * params) {// SigLIP position buckets (same as resampler path)std::vector<int32_t> positions(pos_h * pos_w);- int bucket_coords_h[1024];- int bucket_coords_w[1024];+ std::vector<int> bucket_coords_h(pos_h);+ std::vector<int> bucket_coords_w(pos_w);for (int i = 0; i < pos_h; i++){bucket_coords_h[i] = std::floor(70.0*i/pos_h);}diff --git a/tools/mtmd/mtmd-image.cpp b/tools/mtmd/mtmd-image.cppindex c11d35c87d..9b2e98862d 100644--- a/tools/mtmd/mtmd-image.cpp+++ b/tools/mtmd/mtmd-image.cpp@@ -294,8 +294,8 @@ private:support = filter_support * filterscale; // Widen filter when downsamplingksize = static_cast<int>(std::ceil(support)) * 2 + 1; // Total pixels in kernel- std::vector<double> pre_weights(outSize * ksize); // Temporary weights- bounds.resize(outSize * 2);+ std::vector<double> pre_weights((size_t) outSize * ksize); // Temporary weights+ bounds.resize((size_t) outSize * 2);// For each output pixel, compute its filter coefficients@@ -322,20 +322,20 @@ private:for (x = 0; x < xmax; x++) {// Distance from input pixel center to output pixel center in input spacedouble w = resample_filter((x + xmin - center + 0.5) * ss);- pre_weights[xx * ksize + x] = w;+ pre_weights[(size_t) xx * ksize + x] = w;ww += w; // Accumulate for normalization}// Normalize weights to sum to 1.0 (preserves brightness)for (x = 0; x < xmax; x++) {if (ww != 0.0) {- pre_weights[xx * ksize + x] /= ww;+ pre_weights[(size_t) xx * ksize + x] /= ww;}}// Zero-pad remaining kernel positionsfor (; x < ksize; x++) {- pre_weights[xx * ksize + x] = 0;+ pre_weights[(size_t) xx * ksize + x] = 0;}// Store input pixel range for this output pixel@@ -345,11 +345,11 @@ private:// Convert floating-point coefficients to fixed-point integers// Formula: int32 = round(float * 2^PRECISION_BITS)- weights.resize(outSize * ksize);+ weights.resize((size_t) outSize * ksize);const double fxp_scale = std::ldexp(1.0, PRECISION_BITS); // 1.0 * 2^PRECISION_BITS- for (int i = 0; i < outSize * ksize; i++) {+ for (size_t i = 0; i < (size_t) outSize * ksize; i++) {// Pillow adds +/- 0.5 then truncates toward zero; std::round would round twiceconst double rounded = pre_weights[i] * fxp_scale + (pre_weights[i] < 0 ? -0.5 : 0.5);weights[i] = static_cast<int32_t>(rounded);@@ -442,6 +442,12 @@ private:const int src_width = img.get_size().width;const int src_height = img.get_size().height;+ // sanity check on the target size+ if (target_width <= 0 || target_width > 65536 || target_height <= 0 || target_height > 65536) {+ throw std::runtime_error("resize target " + std::to_string(target_width) + "x" ++ std::to_string(target_height) + " is out of range (max 65536)");+ }+bool need_horizontal = (target_width != src_width);bool need_vertical = (target_height != src_height);