Skip to content

LLM inference in C/C++

C++ ggml

mtmd: use sha256 for input hashing (#27274)

Xuan-Son Nguyened1c3a20f5e3
diff --git a/CMakeLists.txt b/CMakeLists.txt
index 4e1fd45dd1..19f14e0d18 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -224,10 +224,12 @@ add_subdirectory(src)
# utils, programs, examples and tests
#
+# mtmd needs this even when common is not built
+add_subdirectory(vendor/hash)
+
if (LLAMA_BUILD_COMMON)
add_subdirectory(common)
add_subdirectory(vendor/cpp-httplib)
- add_subdirectory(vendor/hash)
endif()
if (LLAMA_BUILD_COMMON AND LLAMA_BUILD_TESTS AND NOT CMAKE_JS_VERSION)
diff --git a/examples/gguf-hash/gguf-hash.cpp b/examples/gguf-hash/gguf-hash.cpp
index 331de301ff..43de6300d9 100644
--- a/examples/gguf-hash/gguf-hash.cpp
+++ b/examples/gguf-hash/gguf-hash.cpp
@@ -18,13 +18,16 @@ extern "C" {
#endif
#include "xxhash/xxhash.h"
-#include "sha1/sha1.h"
#include "sha256/sha256.h"
#ifdef __cplusplus
}
#endif
+// sha1 is compiled as C++ and lives in a namespace, see scripts/sync_vendor.py
+#include "sha1/sha1.h"
+using namespace vendor_hash;
+
// uuid.uuid5(uuid.NAMESPACE_URL, 'en.wikipedia.org/wiki/Llama.cpp')
#define UUID_NAMESPACE_LLAMA_CPP "ef001206-dadc-5f6d-a15f-3359e577d4e5"
diff --git a/scripts/sync_vendor.py b/scripts/sync_vendor.py
index 3ab62f34c1..18a94e1c69 100755
--- a/scripts/sync_vendor.py
+++ b/scripts/sync_vendor.py
@@ -56,6 +56,44 @@ patches = {
' && (defined(_MSC_VER) && (_MSC_VER >= 1000) || !defined(_MSC_VER)) /* >= C11 */\n'
)],
+ # sha1 exports a bare "SHA1" symbol, which clashes with the boringssl one at link time.
+ # we compile it as C++ (see vendor/hash/CMakeLists.txt) and put it in a namespace.
+ "vendor/hash/sha1/sha1.h": [
+ (
+ '#if defined(__cplusplus)\n'
+ 'extern "C" {\n'
+ '#endif\n',
+
+ 'namespace vendor_hash {\n'
+ ),
+ (
+ '#if defined(__cplusplus)\n'
+ '}\n'
+ '#endif\n',
+
+ '} // namespace vendor_hash\n'
+ ),
+ ],
+
+ "vendor/hash/sha1/sha1.c": [
+ (
+ '#include "sha1.h"\n',
+
+ '#include "sha1.h"\n'
+ '\n'
+ 'namespace vendor_hash {\n'
+ ),
+ (
+ ' SHA1Final((unsigned char *)hash_out, &ctx);\n'
+ '}\n',
+
+ ' SHA1Final((unsigned char *)hash_out, &ctx);\n'
+ '}\n'
+ '\n'
+ '} // namespace vendor_hash\n'
+ ),
+ ],
+
# silence a maybe-uninitialized warning
"vendor/hash/sha256/sha256.c": [(
" uint32_t W[16];\n",
diff --git a/tools/mtmd/CMakeLists.txt b/tools/mtmd/CMakeLists.txt
index 769a44e0b7..db758395ff 100644
--- a/tools/mtmd/CMakeLists.txt
+++ b/tools/mtmd/CMakeLists.txt
@@ -78,7 +78,7 @@ set_target_properties(mtmd PROPERTIES
)
target_link_libraries (mtmd PUBLIC ggml llama)
-target_link_libraries (mtmd PRIVATE Threads::Threads)
+target_link_libraries (mtmd PRIVATE Threads::Threads vendor-hash)
target_include_directories(mtmd PUBLIC .)
target_include_directories(mtmd PRIVATE ../..)
target_include_directories(mtmd PRIVATE ../../vendor)
diff --git a/tools/mtmd/mtmd-helper.cpp b/tools/mtmd/mtmd-helper.cpp
index d77c939664..bce8e38cc3 100644
--- a/tools/mtmd/mtmd-helper.cpp
+++ b/tools/mtmd/mtmd-helper.cpp
@@ -12,6 +12,8 @@
#include "mtmd-helper-common.h"
#include "llama.h"
+#include "hash.h"
+
#include <algorithm>
#include <cinttypes>
#include <vector>
@@ -356,25 +358,14 @@ static bool decode_audio_from_buf(const unsigned char * buf_in, size_t len, int
} // namespace audio_helpers
-// Computes FNV-1a hash of the data
-static std::string fnv_hash(const uint8_t * data, size_t len) {
- const uint64_t fnv_prime = 0x100000001b3ULL;
- uint64_t hash = 0xcbf29ce484222325ULL;
-
- for (size_t i = 0; i < len; ++i) {
- hash ^= data[i];
- hash *= fnv_prime;
- }
- return std::to_string(hash);
-}
-
mtmd_helper_bitmap_wrapper mtmd_helper_bitmap_init_from_buf(mtmd_context * ctx, const unsigned char * buf, size_t len, bool placeholder) {
// calculate the hash if needed
std::string id;
mtmd_bitmap * result = nullptr;
if (!placeholder) {
- id = fnv_hash(buf, len);
+ // use sha256 to prevent cache poisoning
+ id = hash_sha256_hex(buf, len);
}
if (audio_helpers::is_audio_file((const char *)buf, len)) {
diff --git a/tools/mtmd/mtmd-helper.h b/tools/mtmd/mtmd-helper.h
index 832f7171ac..5c6b924197 100644
--- a/tools/mtmd/mtmd-helper.h
+++ b/tools/mtmd/mtmd-helper.h
@@ -49,7 +49,7 @@ MTMD_API struct mtmd_helper_bitmap_wrapper mtmd_helper_bitmap_init_from_file(mtm
// note:
// - for now, video input is only supported via C++ helper functions
// - audio files will be auto-detected based on magic bytes
-// - output bitmap will have FNV hash as the ID
+// - output bitmap will have SHA-256 hash (hex string) as the ID
// returns nullptr on failure
// this function is thread-safe
MTMD_API struct mtmd_helper_bitmap_wrapper mtmd_helper_bitmap_init_from_buf(mtmd_context * ctx, const unsigned char * buf, size_t len, bool placeholder);
diff --git a/tools/ui/embed.cpp b/tools/ui/embed.cpp
index 914d51fa1d..b76c9047f2 100644
--- a/tools/ui/embed.cpp
+++ b/tools/ui/embed.cpp
@@ -259,6 +259,8 @@ int main(int argc, char ** argv) {
}
cpp += fmt("static const unsigned char asset_%d_data[] = {", i);
append_bytes_hex(cpp, bytes);
+
+ // note: this is a simple hash for cache busting, not a cryptographic hash; fnv is enough here
const auto hash = fnv_hash(bytes.data(), bytes.size());
cpp += fmt("};\nstatic const std::size_t asset_%d_size = %zu;\n",
diff --git a/vendor/hash/CMakeLists.txt b/vendor/hash/CMakeLists.txt
index 122a6419f9..efdf58e63f 100644
--- a/vendor/hash/CMakeLists.txt
+++ b/vendor/hash/CMakeLists.txt
@@ -4,18 +4,30 @@ llama_add_compile_flags()
set(CMAKE_POSITION_INDEPENDENT_CODE ON)
-add_library(${TARGET} STATIC
+set(VENDOR_SRCS
xxhash/xxhash.c
sha1/sha1.c
sha256/sha256.c
)
-# disable warnings in 3rd party code
+add_library(${TARGET} STATIC
+ hash.cpp
+ hash.h
+ ${VENDOR_SRCS}
+)
+
+target_compile_features(${TARGET} PRIVATE cxx_std_17)
+
+# disable warnings in 3rd party code, but keep them for hash.cpp
if (CMAKE_C_COMPILER_ID STREQUAL "MSVC")
- target_compile_options(${TARGET} PRIVATE /w)
+ set(NO_WARN_FLAG /w)
else()
- target_compile_options(${TARGET} PRIVATE -w)
+ set(NO_WARN_FLAG -w)
endif()
+set_source_files_properties(${VENDOR_SRCS} PROPERTIES COMPILE_OPTIONS ${NO_WARN_FLAG})
+
+# sha1 lives in a namespace to avoid a clash with boringssl, see scripts/sync_vendor.py
+set_source_files_properties(sha1/sha1.c PROPERTIES LANGUAGE CXX)
# sha256.c includes "rotate-bits/rotate-bits.h", so consumers get this dir too
target_include_directories(${TARGET} PUBLIC .)
diff --git a/vendor/hash/hash.cpp b/vendor/hash/hash.cpp
new file mode 100644
index 0000000000..6493716bf7
--- /dev/null
+++ b/vendor/hash/hash.cpp
@@ -0,0 +1,23 @@
+#include "hash.h"
+
+extern "C" {
+#include "sha256/sha256.h"
+}
+
+static std::string to_hex(const unsigned char * digest, size_t len) {
+ static const char hex[] = "0123456789abcdef";
+
+ std::string out;
+ out.reserve(2*len);
+ for (size_t i = 0; i < len; ++i) {
+ out += hex[digest[i] >> 4];
+ out += hex[digest[i] & 0xf];
+ }
+ return out;
+}
+
+std::string hash_sha256_hex(const void * data, size_t len) {
+ unsigned char digest[SHA256_DIGEST_SIZE];
+ sha256_hash(digest, (const unsigned char *) data, len);
+ return to_hex(digest, SHA256_DIGEST_SIZE);
+}
diff --git a/vendor/hash/hash.h b/vendor/hash/hash.h
new file mode 100644
index 0000000000..1298b4196b
--- /dev/null
+++ b/vendor/hash/hash.h
@@ -0,0 +1,9 @@
+#pragma once
+
+// C++ wrapper for the vendored hash functions
+
+#include <cstddef>
+#include <string>
+
+// returns the SHA-256 digest as a lowercase hex string
+std::string hash_sha256_hex(const void * data, size_t len);
diff --git a/vendor/hash/sha1/sha1.c b/vendor/hash/sha1/sha1.c
index 76cd6ca338..4d84340d43 100644
--- a/vendor/hash/sha1/sha1.c
+++ b/vendor/hash/sha1/sha1.c
@@ -25,6 +25,8 @@ A million repetitions of "a"
#include "sha1.h"
+namespace vendor_hash {
+
#define rol(value, bits) (((value) << (bits)) | ((value) >> (32 - (bits))))
@@ -293,3 +295,5 @@ void SHA1(
SHA1Final((unsigned char *)hash_out, &ctx);
}
+} // namespace vendor_hash
+
diff --git a/vendor/hash/sha1/sha1.h b/vendor/hash/sha1/sha1.h
index f492009c97..4ec5df0f4f 100644
--- a/vendor/hash/sha1/sha1.h
+++ b/vendor/hash/sha1/sha1.h
@@ -9,9 +9,7 @@
#include "stdint.h"
-#if defined(__cplusplus)
-extern "C" {
-#endif
+namespace vendor_hash {
typedef struct
{
@@ -45,8 +43,6 @@ void SHA1(
const char *str,
uint32_t len);
-#if defined(__cplusplus)
-}
-#endif
+} // namespace vendor_hash
#endif /* SHA1_H */