LLM inference in C/C++
C++ ggml
vendor: move hash to vendor (#27262)
Xuan-Son Nguyen39be55c97e70
diff --git a/CMakeLists.txt b/CMakeLists.txtindex e0ebb3ccb1..4e1fd45dd1 100644--- a/CMakeLists.txt+++ b/CMakeLists.txt@@ -227,6 +227,7 @@ add_subdirectory(src)if (LLAMA_BUILD_COMMON)add_subdirectory(common)add_subdirectory(vendor/cpp-httplib)+ add_subdirectory(vendor/hash)endif()if (LLAMA_BUILD_COMMON AND LLAMA_BUILD_TESTS AND NOT CMAKE_JS_VERSION)diff --git a/examples/gguf-hash/CMakeLists.txt b/examples/gguf-hash/CMakeLists.txtindex 15c5c68c6f..2542074fbc 100644--- a/examples/gguf-hash/CMakeLists.txt+++ b/examples/gguf-hash/CMakeLists.txt@@ -2,21 +2,5 @@ set(TARGET llama-gguf-hash)add_executable(${TARGET} gguf-hash.cpp)install(TARGETS ${TARGET} RUNTIME)-# clibs dependencies-include_directories(deps/)--add_library(xxhash OBJECT deps/xxhash/xxhash.c deps/xxhash/xxhash.h)-target_link_libraries(${TARGET} PRIVATE xxhash)--add_library(sha1 OBJECT deps/sha1/sha1.c deps/sha1/sha1.h)-target_link_libraries(${TARGET} PRIVATE sha1)-if (NOT MSVC)- # disable warnings in 3rd party code- target_compile_options(sha1 PRIVATE -w)-endif()--add_library(sha256 OBJECT deps/sha256/sha256.c deps/sha256/sha256.h)-target_link_libraries(${TARGET} PRIVATE sha256)--target_link_libraries(${TARGET} PRIVATE ggml ${CMAKE_THREAD_LIBS_INIT})+target_link_libraries(${TARGET} PRIVATE vendor-hash ggml ${CMAKE_THREAD_LIBS_INIT})target_compile_features(${TARGET} PRIVATE cxx_std_17)diff --git a/examples/gguf-hash/deps/rotate-bits/package.json b/examples/gguf-hash/deps/rotate-bits/package.jsondeleted file mode 100644index 74c0bef68d..0000000000--- a/examples/gguf-hash/deps/rotate-bits/package.json+++ /dev/null@@ -1,13 +0,0 @@-{- "name": "rotate-bits",- "version": "0.1.1",- "repo": "jb55/rotate-bits.h",- "description": "rotate bits",- "keywords": ["rotl", "rotr"],- "src": ["rotate-bits.h"],- "license": "Public Domain",- "development": {- "thlorenz/tap.c": "*"- }-}-diff --git a/examples/gguf-hash/deps/sha1/package.json b/examples/gguf-hash/deps/sha1/package.jsondeleted file mode 100644index 6a5843dd1e..0000000000--- a/examples/gguf-hash/deps/sha1/package.json+++ /dev/null@@ -1,9 +0,0 @@-{- "name": "sha1",- "version": "0.0.1",- "repo": "clibs/sha1",- "description": "sha1 hash algorithm",- "keywords": ["sha1", "hash"],- "license": "public domain",- "src": ["sha1.c", "sha1.h"]-}diff --git a/examples/gguf-hash/deps/sha256/package.json b/examples/gguf-hash/deps/sha256/package.jsondeleted file mode 100644index b92a041273..0000000000--- a/examples/gguf-hash/deps/sha256/package.json+++ /dev/null@@ -1,15 +0,0 @@-{- "name": "sha256",- "version": "0.0.2",- "repo": "jb55/sha256.c",- "description": "sha256 in c",- "keywords": ["sha256", "sha2"],- "src": ["sha256.c", "sha256.h"],- "dependencies": {- "jb55/rotate-bits.h": "0.1.1"- },- "development": {- "thlorenz/tap.c": "*"- }-}-diff --git a/examples/gguf-hash/deps/xxhash/clib.json b/examples/gguf-hash/deps/xxhash/clib.jsondeleted file mode 100644index 242343c5d9..0000000000--- a/examples/gguf-hash/deps/xxhash/clib.json+++ /dev/null@@ -1,12 +0,0 @@-{- "name": "xxhash",- "version": "0.8.2",- "repo": "Cyan4973/xxhash",- "description": "Extremely fast non-cryptographic hash algorithm",- "keywords": ["xxhash", "hashing"],- "license": "BSD-2-Clause",- "src": [- "xxhash.c",- "xxhash.h"- ]-}diff --git a/scripts/sync_vendor.py b/scripts/sync_vendor.pyindex 4ecf50f908..3ab62f34c1 100755--- a/scripts/sync_vendor.py+++ b/scripts/sync_vendor.py@@ -7,6 +7,12 @@ import subprocessHTTPLIB_VERSION = "refs/tags/v0.53.1"+# used by examples/gguf-hash, these repos have no release tag, so we pin a commit+XXHASH_COMMIT = "9f465f1ea932d6ad9a26cd77496311ffa544cd68"+SHA1_COMMIT = "e1e2536fcf6a8f9703be8c85d58724b408552287"+SHA256_COMMIT = "5e637272c13f200872d55ff579f7e2ab6c3f252f"+ROTATE_BITS_COMMIT = "27e784942f67db44abf2115c6638e735b579acd1"+vendor = {"https://github.com/nlohmann/json/releases/latest/download/json.hpp": "vendor/nlohmann/json.hpp","https://github.com/nlohmann/json/releases/latest/download/json_fwd.hpp": "vendor/nlohmann/json_fwd.hpp",@@ -22,12 +28,57 @@ vendor = {f"https://raw.githubusercontent.com/yhirose/cpp-httplib/{HTTPLIB_VERSION}/LICENSE": "vendor/cpp-httplib/LICENSE","https://raw.githubusercontent.com/sheredom/subprocess.h/9ce0d701b6fb10f8f8c4445edd31e7c60a1237e3/subprocess.h": "vendor/sheredom/subprocess.h",++ f"https://raw.githubusercontent.com/Cyan4973/xxHash/{XXHASH_COMMIT}/xxhash.c": "vendor/hash/xxhash/xxhash.c",+ f"https://raw.githubusercontent.com/Cyan4973/xxHash/{XXHASH_COMMIT}/xxhash.h": "vendor/hash/xxhash/xxhash.h",+ f"https://raw.githubusercontent.com/Cyan4973/xxHash/{XXHASH_COMMIT}/LICENSE": "vendor/hash/xxhash/LICENSE",++ # clibs/sha1 ships no license file, the source header says public domain+ f"https://raw.githubusercontent.com/clibs/sha1/{SHA1_COMMIT}/sha1.c": "vendor/hash/sha1/sha1.c",+ f"https://raw.githubusercontent.com/clibs/sha1/{SHA1_COMMIT}/sha1.h": "vendor/hash/sha1/sha1.h",++ f"https://raw.githubusercontent.com/jb55/sha256.c/{SHA256_COMMIT}/sha256.c": "vendor/hash/sha256/sha256.c",+ f"https://raw.githubusercontent.com/jb55/sha256.c/{SHA256_COMMIT}/sha256.h": "vendor/hash/sha256/sha256.h",+ f"https://raw.githubusercontent.com/jb55/sha256.c/{SHA256_COMMIT}/LICENSE": "vendor/hash/sha256/LICENSE",++ f"https://raw.githubusercontent.com/jb55/rotate-bits.h/{ROTATE_BITS_COMMIT}/rotate-bits.h": "vendor/hash/rotate-bits/rotate-bits.h",+ f"https://raw.githubusercontent.com/jb55/rotate-bits.h/{ROTATE_BITS_COMMIT}/LICENSE.md": "vendor/hash/rotate-bits/LICENSE.md",+}++# local changes kept on top of the upstream sources+patches = {+ "vendor/hash/xxhash/xxhash.h": [(+ '#if defined(__STDC_VERSION__) && (__STDC_VERSION__ >= 201112L) /* >= C11 */\n',+ '/* Windows SDK under 10.0.22000 is missing stdalign.h so we add a check\n'+ ' before allowing the windows compiler to use the C11 form.\n'+ ' Reference: https://github.com/Cyan4973/xxHash/issues/955 */\n'+ '#if defined(__STDC_VERSION__) && (__STDC_VERSION__ >= 201112L) \\\n'+ ' && (defined(_MSC_VER) && (_MSC_VER >= 1000) || !defined(_MSC_VER)) /* >= C11 */\n'+ )],++ # silence a maybe-uninitialized warning+ "vendor/hash/sha256/sha256.c": [(+ " uint32_t W[16];\n",+ " uint32_t W[16] = {0};\n"+ )],}for url, filename in vendor.items():print(f"downloading {url} to {filename}") # noqa: NP100urllib.request.urlretrieve(url, filename)+for filename, replacements in patches.items():+ print(f"patching {filename}") # noqa: NP100+ with open(filename, "r", encoding="utf-8", newline="") as f:+ content = f.read()+ for old, new in replacements:+ if content.count(old) != 1:+ print(f"Error: cannot apply patch on {filename}, upstream code has changed") # noqa: NP100+ sys.exit(1)+ content = content.replace(old, new)+ with open(filename, "w", encoding="utf-8", newline="") as f:+ f.write(content)+print("Splitting httplib.h...") # noqa: NP100try:subprocess.check_call([diff --git a/vendor/hash/CMakeLists.txt b/vendor/hash/CMakeLists.txtnew file mode 100644index 0000000000..122a6419f9--- /dev/null+++ b/vendor/hash/CMakeLists.txt@@ -0,0 +1,21 @@+set(TARGET vendor-hash)++llama_add_compile_flags()++set(CMAKE_POSITION_INDEPENDENT_CODE ON)++add_library(${TARGET} STATIC+ xxhash/xxhash.c+ sha1/sha1.c+ sha256/sha256.c+)++# disable warnings in 3rd party code+if (CMAKE_C_COMPILER_ID STREQUAL "MSVC")+ target_compile_options(${TARGET} PRIVATE /w)+else()+ target_compile_options(${TARGET} PRIVATE -w)+endif()++# sha256.c includes "rotate-bits/rotate-bits.h", so consumers get this dir too+target_include_directories(${TARGET} PUBLIC .)diff --git a/vendor/hash/rotate-bits/LICENSE.md b/vendor/hash/rotate-bits/LICENSE.mdnew file mode 100644index 0000000000..4a07c57f26--- /dev/null+++ b/vendor/hash/rotate-bits/LICENSE.md@@ -0,0 +1,21 @@+MIT License++Copyright (c) 2021 William Casarin <[email protected]>++Permission is hereby granted, free of charge, to any person obtaining a copy+of this software and associated documentation files (the "Software"), to deal+in the Software without restriction, including without limitation the rights+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell+copies of the Software, and to permit persons to whom the Software is+furnished to do so, subject to the following conditions:++The above copyright notice and this permission notice shall be included in all+copies or substantial portions of the Software.++THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE+SOFTWARE.diff --git a/examples/gguf-hash/deps/rotate-bits/rotate-bits.h b/vendor/hash/rotate-bits/rotate-bits.hsimilarity index 100%rename from examples/gguf-hash/deps/rotate-bits/rotate-bits.hrename to vendor/hash/rotate-bits/rotate-bits.hdiff --git a/examples/gguf-hash/deps/sha1/sha1.c b/vendor/hash/sha1/sha1.csimilarity index 100%rename from examples/gguf-hash/deps/sha1/sha1.crename to vendor/hash/sha1/sha1.cdiff --git a/examples/gguf-hash/deps/sha1/sha1.h b/vendor/hash/sha1/sha1.hsimilarity index 100%rename from examples/gguf-hash/deps/sha1/sha1.hrename to vendor/hash/sha1/sha1.hdiff --git a/vendor/hash/sha256/LICENSE b/vendor/hash/sha256/LICENSEnew file mode 100644index 0000000000..950b19483a--- /dev/null+++ b/vendor/hash/sha256/LICENSE@@ -0,0 +1 @@+2010-06-11 : Igor Pavlov : Public domaindiff --git a/examples/gguf-hash/deps/sha256/sha256.c b/vendor/hash/sha256/sha256.csimilarity index 100%rename from examples/gguf-hash/deps/sha256/sha256.crename to vendor/hash/sha256/sha256.cdiff --git a/examples/gguf-hash/deps/sha256/sha256.h b/vendor/hash/sha256/sha256.hsimilarity index 100%rename from examples/gguf-hash/deps/sha256/sha256.hrename to vendor/hash/sha256/sha256.hdiff --git a/vendor/hash/xxhash/LICENSE b/vendor/hash/xxhash/LICENSEnew file mode 100644index 0000000000..e4c5da7234--- /dev/null+++ b/vendor/hash/xxhash/LICENSE@@ -0,0 +1,26 @@+xxHash Library+Copyright (c) 2012-2021 Yann Collet+All rights reserved.++BSD 2-Clause License (https://www.opensource.org/licenses/bsd-license.php)++Redistribution and use in source and binary forms, with or without modification,+are permitted provided that the following conditions are met:++* Redistributions of source code must retain the above copyright notice, this+ list of conditions and the following disclaimer.++* Redistributions in binary form must reproduce the above copyright notice, this+ list of conditions and the following disclaimer in the documentation and/or+ other materials provided with the distribution.++THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND+ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED+WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE+DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR+ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES+(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;+LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON+ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT+(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS+SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.diff --git a/examples/gguf-hash/deps/xxhash/xxhash.c b/vendor/hash/xxhash/xxhash.csimilarity index 100%rename from examples/gguf-hash/deps/xxhash/xxhash.crename to vendor/hash/xxhash/xxhash.cdiff --git a/examples/gguf-hash/deps/xxhash/xxhash.h b/vendor/hash/xxhash/xxhash.hsimilarity index 100%rename from examples/gguf-hash/deps/xxhash/xxhash.hrename to vendor/hash/xxhash/xxhash.h