LLM inference in C/C++
C++ ggml
gguf : reject empty metadata keys (#24917)
Rohit Maheshd72bfa38f77b
diff --git a/ggml/src/gguf.cpp b/ggml/src/gguf.cppindex 5e19861825..c3ffa1a134 100644--- a/ggml/src/gguf.cpp+++ b/ggml/src/gguf.cpp@@ -557,6 +557,10 @@ static struct gguf_context * gguf_init_from_reader(const struct gguf_reader & grGGML_LOG_ERROR("%s: encountered bad_alloc error while reading key %" PRIi64 "\n", __func__, i);ok = false;}+ if (ok && key.empty()) {+ GGML_LOG_ERROR("%s: key %" PRIi64 " is empty\n", __func__, i);+ ok = false;+ }for (size_t j = 0; ok && j < ctx->kv.size(); ++j) {if (key == ctx->kv[j].key) {GGML_LOG_ERROR("%s: duplicate key '%s' for tensors %zu and %" PRIi64 " \n", __func__, key.c_str(), j, i);diff --git a/tests/test-gguf.cpp b/tests/test-gguf.cppindex 1ae468fbd6..ddb1b4d948 100644--- a/tests/test-gguf.cpp+++ b/tests/test-gguf.cpp@@ -26,6 +26,7 @@ enum handcrafted_file_type {HANDCRAFTED_HEADER_EMPTY = 800,HANDCRAFTED_KV_BAD_KEY_SIZE = 10 + offset_has_kv,+ HANDCRAFTED_KV_EMPTY_KEY = 15 + offset_has_kv,HANDCRAFTED_KV_BAD_TYPE = 20 + offset_has_kv,// HANDCRAFTED_KV_BAD_VALUE_SIZE = 30 + offset_has_kv, // removed because it can result in allocations > 1 TB (default sanitizer limit)HANDCRAFTED_KV_DUPLICATE_KEY = 40 + offset_has_kv,@@ -64,6 +65,7 @@ static std::string handcrafted_file_type_name(const enum handcrafted_file_type hcase HANDCRAFTED_HEADER_EMPTY: return "HEADER_EMPTY";case HANDCRAFTED_KV_BAD_KEY_SIZE: return "KV_BAD_KEY_SIZE";+ case HANDCRAFTED_KV_EMPTY_KEY: return "KV_EMPTY_KEY";case HANDCRAFTED_KV_BAD_TYPE: return "KV_BAD_TYPE";case HANDCRAFTED_KV_DUPLICATE_KEY: return "KV_DUPLICATE_KEY";case HANDCRAFTED_KV_BAD_ALIGN: return "KV_BAD_ALIGN";@@ -284,7 +286,9 @@ static FILE * get_handcrafted_file(const unsigned int seed, const enum handcraftconst enum gguf_type type = gguf_type(hft == HANDCRAFTED_KV_BAD_TYPE ? GGUF_TYPE_COUNT : kv_types[i].first);const enum gguf_type type_arr = gguf_type(hft == HANDCRAFTED_KV_BAD_TYPE ? GGUF_TYPE_COUNT : kv_types[i].second);- const std::string key = "my_key_" + std::to_string((hft == HANDCRAFTED_KV_DUPLICATE_KEY ? i/2 : i));+ const std::string key = hft == HANDCRAFTED_KV_EMPTY_KEY+ ? ""+ : "my_key_" + std::to_string((hft == HANDCRAFTED_KV_DUPLICATE_KEY ? i/2 : i));if (hft == HANDCRAFTED_KV_BAD_KEY_SIZE) {const uint64_t n = -1;@@ -732,6 +736,7 @@ static std::pair<int, int> test_handcrafted_file(const unsigned int seed) {HANDCRAFTED_HEADER_EMPTY,HANDCRAFTED_KV_BAD_KEY_SIZE,+ HANDCRAFTED_KV_EMPTY_KEY,HANDCRAFTED_KV_BAD_TYPE,HANDCRAFTED_KV_DUPLICATE_KEY,HANDCRAFTED_KV_BAD_ALIGN,