LLM inference in C/C++
C++ ggml
mtmd: use sha256 for input hashing (#27274)
Xuan-Son Nguyened1c3a20f5e3
diff --git a/CMakeLists.txt b/CMakeLists.txtindex 4e1fd45dd1..19f14e0d18 100644--- a/CMakeLists.txt+++ b/CMakeLists.txt@@ -224,10 +224,12 @@ add_subdirectory(src)# utils, programs, examples and tests#+# mtmd needs this even when common is not built+add_subdirectory(vendor/hash)+if (LLAMA_BUILD_COMMON)add_subdirectory(common)add_subdirectory(vendor/cpp-httplib)- add_subdirectory(vendor/hash)endif()if (LLAMA_BUILD_COMMON AND LLAMA_BUILD_TESTS AND NOT CMAKE_JS_VERSION)diff --git a/examples/gguf-hash/gguf-hash.cpp b/examples/gguf-hash/gguf-hash.cppindex 331de301ff..43de6300d9 100644--- a/examples/gguf-hash/gguf-hash.cpp+++ b/examples/gguf-hash/gguf-hash.cpp@@ -18,13 +18,16 @@ extern "C" {#endif#include "xxhash/xxhash.h"-#include "sha1/sha1.h"#include "sha256/sha256.h"#ifdef __cplusplus}#endif+// sha1 is compiled as C++ and lives in a namespace, see scripts/sync_vendor.py+#include "sha1/sha1.h"+using namespace vendor_hash;+// uuid.uuid5(uuid.NAMESPACE_URL, 'en.wikipedia.org/wiki/Llama.cpp')#define UUID_NAMESPACE_LLAMA_CPP "ef001206-dadc-5f6d-a15f-3359e577d4e5"diff --git a/scripts/sync_vendor.py b/scripts/sync_vendor.pyindex 3ab62f34c1..18a94e1c69 100755--- a/scripts/sync_vendor.py+++ b/scripts/sync_vendor.py@@ -56,6 +56,44 @@ patches = {' && (defined(_MSC_VER) && (_MSC_VER >= 1000) || !defined(_MSC_VER)) /* >= C11 */\n')],+ # sha1 exports a bare "SHA1" symbol, which clashes with the boringssl one at link time.+ # we compile it as C++ (see vendor/hash/CMakeLists.txt) and put it in a namespace.+ "vendor/hash/sha1/sha1.h": [+ (+ '#if defined(__cplusplus)\n'+ 'extern "C" {\n'+ '#endif\n',++ 'namespace vendor_hash {\n'+ ),+ (+ '#if defined(__cplusplus)\n'+ '}\n'+ '#endif\n',++ '} // namespace vendor_hash\n'+ ),+ ],++ "vendor/hash/sha1/sha1.c": [+ (+ '#include "sha1.h"\n',++ '#include "sha1.h"\n'+ '\n'+ 'namespace vendor_hash {\n'+ ),+ (+ ' SHA1Final((unsigned char *)hash_out, &ctx);\n'+ '}\n',++ ' SHA1Final((unsigned char *)hash_out, &ctx);\n'+ '}\n'+ '\n'+ '} // namespace vendor_hash\n'+ ),+ ],+# silence a maybe-uninitialized warning"vendor/hash/sha256/sha256.c": [(" uint32_t W[16];\n",diff --git a/tools/mtmd/CMakeLists.txt b/tools/mtmd/CMakeLists.txtindex 769a44e0b7..db758395ff 100644--- a/tools/mtmd/CMakeLists.txt+++ b/tools/mtmd/CMakeLists.txt@@ -78,7 +78,7 @@ set_target_properties(mtmd PROPERTIES)target_link_libraries (mtmd PUBLIC ggml llama)-target_link_libraries (mtmd PRIVATE Threads::Threads)+target_link_libraries (mtmd PRIVATE Threads::Threads vendor-hash)target_include_directories(mtmd PUBLIC .)target_include_directories(mtmd PRIVATE ../..)target_include_directories(mtmd PRIVATE ../../vendor)diff --git a/tools/mtmd/mtmd-helper.cpp b/tools/mtmd/mtmd-helper.cppindex d77c939664..bce8e38cc3 100644--- a/tools/mtmd/mtmd-helper.cpp+++ b/tools/mtmd/mtmd-helper.cpp@@ -12,6 +12,8 @@#include "mtmd-helper-common.h"#include "llama.h"+#include "hash.h"+#include <algorithm>#include <cinttypes>#include <vector>@@ -356,25 +358,14 @@ static bool decode_audio_from_buf(const unsigned char * buf_in, size_t len, int} // namespace audio_helpers-// Computes FNV-1a hash of the data-static std::string fnv_hash(const uint8_t * data, size_t len) {- const uint64_t fnv_prime = 0x100000001b3ULL;- uint64_t hash = 0xcbf29ce484222325ULL;-- for (size_t i = 0; i < len; ++i) {- hash ^= data[i];- hash *= fnv_prime;- }- return std::to_string(hash);-}-mtmd_helper_bitmap_wrapper mtmd_helper_bitmap_init_from_buf(mtmd_context * ctx, const unsigned char * buf, size_t len, bool placeholder) {// calculate the hash if neededstd::string id;mtmd_bitmap * result = nullptr;if (!placeholder) {- id = fnv_hash(buf, len);+ // use sha256 to prevent cache poisoning+ id = hash_sha256_hex(buf, len);}if (audio_helpers::is_audio_file((const char *)buf, len)) {diff --git a/tools/mtmd/mtmd-helper.h b/tools/mtmd/mtmd-helper.hindex 832f7171ac..5c6b924197 100644--- a/tools/mtmd/mtmd-helper.h+++ b/tools/mtmd/mtmd-helper.h@@ -49,7 +49,7 @@ MTMD_API struct mtmd_helper_bitmap_wrapper mtmd_helper_bitmap_init_from_file(mtm// note:// - for now, video input is only supported via C++ helper functions// - audio files will be auto-detected based on magic bytes-// - output bitmap will have FNV hash as the ID+// - output bitmap will have SHA-256 hash (hex string) as the ID// returns nullptr on failure// this function is thread-safeMTMD_API struct mtmd_helper_bitmap_wrapper mtmd_helper_bitmap_init_from_buf(mtmd_context * ctx, const unsigned char * buf, size_t len, bool placeholder);diff --git a/tools/ui/embed.cpp b/tools/ui/embed.cppindex 914d51fa1d..b76c9047f2 100644--- a/tools/ui/embed.cpp+++ b/tools/ui/embed.cpp@@ -259,6 +259,8 @@ int main(int argc, char ** argv) {}cpp += fmt("static const unsigned char asset_%d_data[] = {", i);append_bytes_hex(cpp, bytes);++ // note: this is a simple hash for cache busting, not a cryptographic hash; fnv is enough hereconst auto hash = fnv_hash(bytes.data(), bytes.size());cpp += fmt("};\nstatic const std::size_t asset_%d_size = %zu;\n",diff --git a/vendor/hash/CMakeLists.txt b/vendor/hash/CMakeLists.txtindex 122a6419f9..efdf58e63f 100644--- a/vendor/hash/CMakeLists.txt+++ b/vendor/hash/CMakeLists.txt@@ -4,18 +4,30 @@ llama_add_compile_flags()set(CMAKE_POSITION_INDEPENDENT_CODE ON)-add_library(${TARGET} STATIC+set(VENDOR_SRCSxxhash/xxhash.csha1/sha1.csha256/sha256.c)-# disable warnings in 3rd party code+add_library(${TARGET} STATIC+ hash.cpp+ hash.h+ ${VENDOR_SRCS}+)++target_compile_features(${TARGET} PRIVATE cxx_std_17)++# disable warnings in 3rd party code, but keep them for hash.cppif (CMAKE_C_COMPILER_ID STREQUAL "MSVC")- target_compile_options(${TARGET} PRIVATE /w)+ set(NO_WARN_FLAG /w)else()- target_compile_options(${TARGET} PRIVATE -w)+ set(NO_WARN_FLAG -w)endif()+set_source_files_properties(${VENDOR_SRCS} PROPERTIES COMPILE_OPTIONS ${NO_WARN_FLAG})++# sha1 lives in a namespace to avoid a clash with boringssl, see scripts/sync_vendor.py+set_source_files_properties(sha1/sha1.c PROPERTIES LANGUAGE CXX)# sha256.c includes "rotate-bits/rotate-bits.h", so consumers get this dir tootarget_include_directories(${TARGET} PUBLIC .)diff --git a/vendor/hash/hash.cpp b/vendor/hash/hash.cppnew file mode 100644index 0000000000..6493716bf7--- /dev/null+++ b/vendor/hash/hash.cpp@@ -0,0 +1,23 @@+#include "hash.h"++extern "C" {+#include "sha256/sha256.h"+}++static std::string to_hex(const unsigned char * digest, size_t len) {+ static const char hex[] = "0123456789abcdef";++ std::string out;+ out.reserve(2*len);+ for (size_t i = 0; i < len; ++i) {+ out += hex[digest[i] >> 4];+ out += hex[digest[i] & 0xf];+ }+ return out;+}++std::string hash_sha256_hex(const void * data, size_t len) {+ unsigned char digest[SHA256_DIGEST_SIZE];+ sha256_hash(digest, (const unsigned char *) data, len);+ return to_hex(digest, SHA256_DIGEST_SIZE);+}diff --git a/vendor/hash/hash.h b/vendor/hash/hash.hnew file mode 100644index 0000000000..1298b4196b--- /dev/null+++ b/vendor/hash/hash.h@@ -0,0 +1,9 @@+#pragma once++// C++ wrapper for the vendored hash functions++#include <cstddef>+#include <string>++// returns the SHA-256 digest as a lowercase hex string+std::string hash_sha256_hex(const void * data, size_t len);diff --git a/vendor/hash/sha1/sha1.c b/vendor/hash/sha1/sha1.cindex 76cd6ca338..4d84340d43 100644--- a/vendor/hash/sha1/sha1.c+++ b/vendor/hash/sha1/sha1.c@@ -25,6 +25,8 @@ A million repetitions of "a"#include "sha1.h"+namespace vendor_hash {+#define rol(value, bits) (((value) << (bits)) | ((value) >> (32 - (bits))))@@ -293,3 +295,5 @@ void SHA1(SHA1Final((unsigned char *)hash_out, &ctx);}+} // namespace vendor_hash+diff --git a/vendor/hash/sha1/sha1.h b/vendor/hash/sha1/sha1.hindex f492009c97..4ec5df0f4f 100644--- a/vendor/hash/sha1/sha1.h+++ b/vendor/hash/sha1/sha1.h@@ -9,9 +9,7 @@#include "stdint.h"-#if defined(__cplusplus)-extern "C" {-#endif+namespace vendor_hash {typedef struct{@@ -45,8 +43,6 @@ void SHA1(const char *str,uint32_t len);-#if defined(__cplusplus)-}-#endif+} // namespace vendor_hash#endif /* SHA1_H */