LLM inference in C/C++
C++ ggml
fix: OOB reads in UGM tokenizer (precompiled_charsmap handling) (#18750)
hourhl4a7ee3126d52
diff --git a/src/llama-vocab.cpp b/src/llama-vocab.cppindex 6e78a3f6c0..fdd447147d 100644--- a/src/llama-vocab.cpp+++ b/src/llama-vocab.cpp@@ -887,9 +887,6 @@ struct llm_tokenizer_ugm : llm_tokenizer {// blob containing XOR-compressed compact double array (XCDA) entriesuint32_t xcda_blob_size = *(const uint32_t *) &precompiled_charsmap[0];charsmap_offset += sizeof(xcda_blob_size);- if (xcda_blob_size + charsmap_offset >= precompiled_charsmap.size()) {- throw std::runtime_error("Index out of array bounds in precompiled charsmap!");- }// Next xcda_blob_size bytes contain entries of XOR-compressed compact// double array (XCDA). Each entry is bit-packed into a 32-bit integer.@@ -1205,7 +1202,15 @@ private:throw std::runtime_error("Index out of array bounds in precompiled charsmap!");}const char * prefix_replacement = &(tokenizer.prefix_replacements)[longest_prefix_offset];- return { prefix_replacement, strlen(prefix_replacement), longest_prefix_length };+ size_t max_len = tokenizer.prefix_replacements_size - longest_prefix_offset;+ size_t repl_len = 0;+ while (repl_len < max_len && prefix_replacement[repl_len] != '\0') {+ repl_len++;+ }+ if (repl_len == max_len) {+ throw std::runtime_error("Unterminated string in precompiled charsmap!");+ }+ return { prefix_replacement, repl_len, longest_prefix_length };}// check if the input prefix contains a valid sequence of UTF-8 code units@@ -2018,11 +2023,18 @@ void llama_vocab::impl::load(llama_model_loader & ml, const LLM_KV & kv) {const size_t n_precompiled_charsmap = gguf_get_arr_n(ctx, precompiled_charsmap_keyidx);const char * pc = (const char *) gguf_get_arr_data(ctx, precompiled_charsmap_keyidx);precompiled_charsmap.assign(pc, pc + n_precompiled_charsmap);-#if defined(__BYTE_ORDER__) && defined(__ORDER_BIG_ENDIAN__) && __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__- // correct endianness of data in precompiled_charsmap binary blob+ if (precompiled_charsmap.size() < sizeof(uint32_t)) {+ throw std::runtime_error("precompiled_charsmap too small for xcda_blob_size header!");+ }uint32_t * xcda_blob_size = (uint32_t *) &precompiled_charsmap[0];+#if defined(__BYTE_ORDER__) && defined(__ORDER_BIG_ENDIAN__) && __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__*xcda_blob_size = __builtin_bswap32(*xcda_blob_size);- assert(*xcda_blob_size + sizeof(uint32_t) < n_precompiled_charsmap);+#endif+ if (*xcda_blob_size + sizeof(uint32_t) >= precompiled_charsmap.size()) {+ throw std::runtime_error("Index out of array bounds in precompiled charsmap!");+ }+#if defined(__BYTE_ORDER__) && defined(__ORDER_BIG_ENDIAN__) && __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__+ // correct endianness of data in precompiled_charsmap binary blobsize_t xcda_array_size = *xcda_blob_size / sizeof(uint32_t);uint32_t * xcda_array = (uint32_t *) &precompiled_charsmap[sizeof(uint32_t)];for (size_t i = 0; i < xcda_array_size; ++i) {