Open Source alternative to Algolia + Pinecone and an Easier-to-Use alternative to ElasticSearch ⚡ 🔍 ✨ Fast, typo tolerant, in-memory fuzzy Search Engine for building delightful search experiences
C++ algoliadatastoreelasticsearchenterprise-searchfacetingfull-text-searchfuzzy-searchgeosearchin-memoryinstantsearchmerchandisingpineconesearchsearch-enginesemantic-searchsimilarity-searchsite-searchsynonymstypo-tolerancevector-search
Force use of jemalloc via static linking.
kishorenc1ab9b6509874
diff --git a/CMakeLists.txt b/CMakeLists.txtindex b5afc665..5f3ffce2 100644--- a/CMakeLists.txt+++ b/CMakeLists.txt@@ -22,6 +22,8 @@ if (APPLE)# Prefer brew installated librariesset(OPENSSL_ROOT_DIR /usr/local/opt/[email protected])+ set(ENV{OPENSSL_ROOT_DIR} /usr/local/opt/[email protected])+set(SNAPPY_ROOT_DIR /usr/local/opt/snappy)set(ZLIB_ROOT /usr/local/opt/zlib)set(CMAKE_PREFIX_PATH /usr/local/opt/curl-openssl /usr/local/opt/icu4c)@@ -33,8 +35,17 @@ IF(WIN32 OR MSVC)SET(CMAKE_FIND_LIBRARY_SUFFIXES .lib .a)ELSE()SET(CMAKE_FIND_LIBRARY_SUFFIXES .a)+ set(ENV{CMAKE_FIND_LIBRARY_SUFFIXES} ".a")ENDIF()+include(cmake/For.cmake)+include(cmake/H2O.cmake)+include(cmake/RocksDB.cmake)+include(cmake/GoogleTest.cmake)+include(cmake/TestResources.cmake)+include(cmake/Iconv.cmake)+include(cmake/Jemalloc.cmake)+FIND_PACKAGE(OpenSSL 1.1.1 REQUIRED)FIND_PACKAGE(Snappy REQUIRED)FIND_PACKAGE(ZLIB REQUIRED)@@ -47,13 +58,6 @@ FIND_PACKAGE(glog REQUIRED)message("OpenSSL library: ${OPENSSL_LIBRARIES}")-include(cmake/For.cmake)-include(cmake/H2O.cmake)-include(cmake/RocksDB.cmake)-include(cmake/GoogleTest.cmake)-include(cmake/TestResources.cmake)-include(cmake/Iconv.cmake)-FILE(GLOB SRC_FILES src/*.cpp)FILE(GLOB TEST_FILES test/*.cpp)@@ -68,52 +72,28 @@ include_directories(${DEP_ROOT_DIR}/${ROCKSDB_NAME}/include)include_directories(${DEP_ROOT_DIR}/${ICONV_NAME}/include)include_directories(${DEP_ROOT_DIR}/${BRPC_NAME}/include)include_directories(${DEP_ROOT_DIR}/${BRAFT_NAME}/include)--# Write dependency include directories to a file-file(WRITE ${DEP_ROOT_DIR}/includes.txt "")-file(APPEND ${DEP_ROOT_DIR}/includes.txt "${DEP_ROOT_DIR}/${FOR_NAME}")-file(APPEND ${DEP_ROOT_DIR}/includes.txt " ${DEP_ROOT_DIR}/${H2O_NAME}/include")-file(APPEND ${DEP_ROOT_DIR}/includes.txt " ${DEP_ROOT_DIR}/${ROCKSDB_NAME}/include")-file(APPEND ${DEP_ROOT_DIR}/includes.txt " ${DEP_ROOT_DIR}/${ICONV_NAME}/include")-file(APPEND ${DEP_ROOT_DIR}/includes.txt " ${DEP_ROOT_DIR}/${BRPC_NAME}/include")-file(APPEND ${DEP_ROOT_DIR}/includes.txt " ${DEP_ROOT_DIR}/${BRAFT_NAME}/include")+include_directories(${DEP_ROOT_DIR}/${JEMALLOC_NAME}/include/jemalloc)link_directories(${DEP_ROOT_DIR}/${GTEST_NAME}/googletest/build)link_directories(${DEP_ROOT_DIR}/${FOR_NAME})link_directories(${DEP_ROOT_DIR}/${H2O_NAME}/build)link_directories(${DEP_ROOT_DIR}/${ROCKSDB_NAME})link_directories(${DEP_ROOT_DIR}/${ICONV_NAME}/lib/.libs)-if (APPLE)-link_directories(${DEP_ROOT_DIR}/${BRPC_NAME}/lib)-link_directories(${DEP_ROOT_DIR}/${BRAFT_NAME}/lib)-endif()+link_directories(${DEP_ROOT_DIR}/${JEMALLOC_NAME}/lib)-# Write dependency libraries to a file-file(WRITE ${DEP_ROOT_DIR}/libs.txt "")-file(APPEND ${DEP_ROOT_DIR}/libs.txt " ${DEP_ROOT_DIR}/${FOR_NAME}")-file(APPEND ${DEP_ROOT_DIR}/libs.txt " ${DEP_ROOT_DIR}/${H2O_NAME}/build")-file(APPEND ${DEP_ROOT_DIR}/libs.txt " ${DEP_ROOT_DIR}/${ROCKSDB_NAME}")-file(APPEND ${DEP_ROOT_DIR}/libs.txt " ${DEP_ROOT_DIR}/${ICONV_NAME}/lib/.libs")-file(APPEND ${DEP_ROOT_DIR}/libs.txt " ${DEP_ROOT_DIR}/${BRPC_NAME}/lib")-file(APPEND ${DEP_ROOT_DIR}/libs.txt " ${DEP_ROOT_DIR}/${BRAFT_NAME}/lib")+set(JEMALLOC_ROOT_DIR "${DEP_ROOT_DIR}/${JEMALLOC_NAME}")+FIND_PACKAGE(Jemalloc REQUIRED)add_executable(typesense-server ${SRC_FILES} src/main/typesense_server.cpp)add_executable(search ${SRC_FILES} src/main/main.cpp)add_executable(benchmark ${SRC_FILES} src/main/benchmark.cpp)add_executable(typesense-test ${SRC_FILES} ${TEST_FILES})-add_library(typesense-core STATIC ${SRC_FILES})-target_compile_definitions(typesense-server PRIVATETYPESENSE_VERSION="${TYPESENSE_VERSION}")-target_compile_definitions(- typesense-core PRIVATE- TYPESENSE_VERSION="${TYPESENSE_VERSION}"-)-target_compile_definitions(benchmark PRIVATETYPESENSE_VERSION="${TYPESENSE_VERSION}"@@ -133,7 +113,7 @@ target_compile_definitions(set(ROCKSDB_LIBS rocksdb ${SNAPPY_LIBRARIES} ${ZLIB_LIBRARIES})if (APPLE)- set(STD_LIB -static-libstdc++) # https://stackoverflow.com/a/26543140/131050 (can't statically link libgcc on Mac)+ set(STD_LIB "") # https://stackoverflow.com/a/26543140/131050 (can't statically link libgcc on Mac)FIND_PACKAGE(ngHTTP2 REQUIRED)set(CURL_LIBRARIES ${NGHTTP2_LIBRARIES} ${CURL_LIBRARIES}) # nghttp2 was not being statically built@@ -150,9 +130,9 @@ set(ICU_ALL_LIBRARIES ${ICU_I18N_LIBRARIES} ${ICU_LIBRARIES} ${ICU_DATA_LIBRARIEset(CORE_LIBS h2o-evloop iconv ${CURL_LIBRARIES} for ${ICU_ALL_LIBRARIES} ${G3LOGGER_LIBRARIES}${ROCKSDB_LIBS} ${OPENSSL_LIBRARIES} pthread dl ${STD_LIB})-set(CORE_LIBS braft brpc ${LevelDB_LIBRARIES} glog ${CORE_LIBS} ${GFLAGS_LIBRARIES} ${PROTOBUF_LIBRARIES} ${SYSTEM_LIBS})+set(CORE_LIBS ${JEMALLOC_LIBRARIES} braft brpc ${LevelDB_LIBRARIES} glog+ ${CORE_LIBS} ${GFLAGS_LIBRARIES} ${PROTOBUF_LIBRARIES} ${SYSTEM_LIBS})-target_link_libraries(typesense-core ${CORE_LIBS})target_link_libraries(typesense-server ${CORE_LIBS})target_link_libraries(search ${CORE_LIBS})target_link_libraries(benchmark ${CORE_LIBS})diff --git a/build.sh b/build.shindex c31288f4..26d8c00b 100755--- a/build.sh+++ b/build.sh@@ -31,16 +31,3 @@ if [[ "$@" == *"--package-binary"* ]]; thentar -cvzf $PROJECT_DIR/$BUILD_DIR/$RELEASE_NAME.tar.gz -C $PROJECT_DIR/$BUILD_DIR typesense-server typesense-server.md5.txtecho "Built binary successfully: $PROJECT_DIR/$BUILD_DIR/$RELEASE_NAME.tar.gz"fi--if [[ "$@" == *"--package-libs"* ]]; then- OS_FAMILY=$(echo `uname` | awk '{print tolower($0)}')- RELEASE_NAME=typesense-server-libs-$TYPESENSE_VERSION-$OS_FAMILY-amd64- LIBS=`cat $PROJECT_DIR/external-$SYSTEM_NAME/libs.txt`- TAR_PATHS=""- for lib in $LIBS; do- TAR_PATHS="$TAR_PATHS -C $lib `ls $lib/*.a | xargs basename`"- done-- TAR_PATHS="$TAR_PATHS -C $PROJECT_DIR/$BUILD_DIR `ls $PROJECT_DIR/$BUILD_DIR/*.a | xargs basename`"- tar -cvzf $PROJECT_DIR/$BUILD_DIR/$RELEASE_NAME.tar.gz $TAR_PATHS-fidiff --git a/cmake/Jemalloc.cmake b/cmake/Jemalloc.cmakenew file mode 100644index 00000000..2747acb0--- /dev/null+++ b/cmake/Jemalloc.cmake@@ -0,0 +1,34 @@+# Download and build Jemalloc++set(JEMALLOC_VERSION 5.2.1)+set(JEMALLOC_NAME jemalloc-${JEMALLOC_VERSION})+set(JEMALLOC_TAR_PATH ${DEP_ROOT_DIR}/${JEMALLOC_NAME}.tar.bz2)++if(NOT EXISTS ${JEMALLOC_TAR_PATH})+ message(STATUS "Downloading ${JEMALLOC_NAME}...")+ file(DOWNLOAD https://github.com/jemalloc/jemalloc/releases/download/${JEMALLOC_VERSION}/${JEMALLOC_NAME}.tar.bz2+ ${JEMALLOC_TAR_PATH})+endif()++if(NOT EXISTS ${DEP_ROOT_DIR}/${JEMALLOC_NAME})+ message(STATUS "Extracting jemalloc...")+ execute_process(COMMAND ${CMAKE_COMMAND} -E tar xzf ${JEMALLOC_TAR_PATH} WORKING_DIRECTORY ${DEP_ROOT_DIR})+endif()++if(NOT EXISTS ${DEP_ROOT_DIR}/${JEMALLOC_NAME}/Makefile)+ message("Configuring jemalloc locally...")+ # Builds with "--with-jemalloc-prefix=je_" on OSX+ execute_process(COMMAND ./configure WORKING_DIRECTORY ${DEP_ROOT_DIR}/${JEMALLOC_NAME} RESULT_VARIABLE JEMALLOC_CONFIGURE)+ if(NOT JEMALLOC_CONFIGURE EQUAL 0)+ message(FATAL_ERROR "${JEMALLOC_NAME} configure failed!")+ message("${RESULT_VARIABLE}")+ endif()+endif()++if(NOT EXISTS ${DEP_ROOT_DIR}/${JEMALLOC_NAME}/lib/libjemalloc.a)+ message("Building jemalloc locally...")+ execute_process(COMMAND make "build_lib_static" WORKING_DIRECTORY ${DEP_ROOT_DIR}/${JEMALLOC_NAME})+ if(NOT EXISTS ${DEP_ROOT_DIR}/${JEMALLOC_NAME}/lib/libjemalloc.a)+ message(FATAL_ERROR "${JEMALLOC_NAME} build failed!")+ endif()+endif()diff --git a/cmake/Modules/FindJemalloc.cmake b/cmake/Modules/FindJemalloc.cmakenew file mode 100644index 00000000..f27b3b36--- /dev/null+++ b/cmake/Modules/FindJemalloc.cmake@@ -0,0 +1,38 @@+# Tries to find Jemalloc headers and libraries.+#+# Usage of this module as follows:+#+# find_package(jemalloc)+#+# Variables used by this module, they can change the default behaviour and need+# to be set before calling find_package:+#+# JEMALLOC_ROOT_DIR Set this variable to the root installation of+# Jemalloc if the module has problems finding+# the proper installation path.+#+# Variables defined by this module:+#+# JEMALLOC_FOUND System has Jemalloc libs/headers+# JEMALLOC_LIBRARIES The Jemalloc libraries+# JEMALLOC_INCLUDE_DIR The location of Jemalloc headers++find_path(JEMALLOC_INCLUDE_DIR+ NAMES jemalloc.h+ HINTS ${JEMALLOC_ROOT_DIR}/include/jemalloc+ NO_DEFAULT_PATH)++find_library(JEMALLOC_LIBRARIES+ NAMES jemalloc+ HINTS ${JEMALLOC_ROOT_DIR}/lib+ NO_DEFAULT_PATH)++include(FindPackageHandleStandardArgs)+find_package_handle_standard_args(jemalloc DEFAULT_MSG+ JEMALLOC_LIBRARIES+ JEMALLOC_INCLUDE_DIR)++mark_as_advanced(+ JEMALLOC_ROOT_DIR+ JEMALLOC_LIBRARIES+ JEMALLOC_INCLUDE_DIR)diff --git a/docker-build.sh b/docker-build.shindex e2ab7d71..89dcaeb6 100755--- a/docker-build.sh+++ b/docker-build.sh@@ -46,18 +46,6 @@ if [[ "$@" == *"--package-binary"* ]]; thenecho "Built binary successfully: $PROJECT_DIR/$BUILD_DIR/$RELEASE_NAME.tar.gz"fi-if [[ "$@" == *"--package-libs"* ]]; then- OS_FAMILY=linux- RELEASE_NAME=typesense-server-libs-$TYPESENSE_VERSION-$OS_FAMILY-amd64- LIBS=`cat $PROJECT_DIR/external-$SYSTEM_NAME/libs.txt`- TAR_PATHS=""- for lib in $LIBS; do- TAR_PATHS="$TAR_PATHS -C $PROJECT_DIR/../$lib `ls $PROJECT_DIR/../$lib/*.a | xargs basename`"- done-- TAR_PATHS="$TAR_PATHS -C $PROJECT_DIR/$BUILD_DIR `ls $PROJECT_DIR/$BUILD_DIR/*.a | xargs basename`"- tar -cvzf $PROJECT_DIR/$BUILD_DIR/$RELEASE_NAME.tar.gz $TAR_PATHS-fi##if [[ "$@" == *"--create-deb-upload"* ]]; then# docker run -it -v $PROJECT_DIR:/typesense typesense/typesense-development:23-JUNE-2020-1 cmake -DTYPESENSE_VERSION=$TYPESENSE_VERSION \diff --git a/include/cvt.h b/include/cvt.hnew file mode 100644index 00000000..ca0840cb--- /dev/null+++ b/include/cvt.h@@ -0,0 +1,109 @@+/* Compact Variable Trie+================================================================================================================+ ates, at, as, bet, to, tk++ [ * ]+ ⁄ | \+ a b t+ / \ \ /\+ s t etØ k o+ / / \ / \+ Ø esØ Ø Ø Ø+++ BASIC DESIGN+ ============++ * All siblings in the same block.+ * Pointer to ONLY FIRST child node of each sibling.+ * Each sibling node's children represented by their character/byte+ * For root there are no siblings, so pointer only to `a` child.++ ROOT -> [0|0|PTRA][3][a][b][t]+ PTRA -> [0|PTRS][2|PTRE][4|PTRK[2][s][t][2][e][t][2][k][o]+ PTRS -> [0|PTRØ][1|PTRT][1][Ø][2][e][Ø]+ PTRT -> [0|L_PTRE][3|PTRØ][3][e][s][Ø][1][Ø] (path compression)+ PTRØ -> [0|LEAF]++ [TYPE+OFFSET][PTR_1]..[NUM_CHILDREN][A][B]..[X]++ 2 bytes for type+offset+ 6 bytes for address+ 1 byte for num_children+ x bytes for bytes++ Actual offset to the Nth node's children: (8*NUM_CHILDREN) + N + offset++ if num_children >= 32:+ Use bitset to represent children present+ Read 32 bytes and do bitset operations to extract matched index+ else:+ Use array to represent children+ Read `num_children` bytes and do sequential search++ Removal of [be]++ 1. Realloc contents of PTR1 by removing "e" from the nodes list+ 2. Free PTR3+ 3. Realloc contents of ROOT by removing "b" from the nodes list++*/++#include <cstdint>+#include <cstddef>+++++class CVTrie {+private:+ size_t size;++ // [TYPE+OFFSET][PTR_1]... (8 bytes each)+ // [NUM_PREFIX][A][B]..[X] (upto 33 bytes)+ uint8_t* root;++ const uintptr_t PTR_MASK = ~(1ULL << 48ULL);++public:++ uint8_t* get_ptr(const uint8_t* tagged_ptr) {+ // Unfortunately, right shifting of signed integer for sign extension is implementation-defined+ // but works on all major compilers+ return (uint8_t*)( (intptr_t)((uintptr_t)tagged_ptr << 16ULL) >> 16ULL );+ }++ uint8_t* tag_ptr(const uint8_t* ptr, const uint64_t data) {+ return (uint8_t*)(((uintptr_t)ptr & PTR_MASK) | (data << 48ULL));+ }++ uint16_t get_data(const uint16_t* ptr) {+ return (uintptr_t)(ptr) >> 56ULL;;+ }++ void add(const unsigned char* key, const size_t length, void* value) {+ // If the key exists, augment the node, otherwise insert a new node++ size_t num_siblings = 1;+ uint8_t* buf = root;+ size_t key_index = 0;++ if(root == nullptr) {+ // trie is empty+ root = new uint8_t[8+1+1];++ }++ while(true) {+ // for each sibling+ for(auto sindex = 0; sindex < num_siblings; sindex++) {+ unsigned char c = key[key_index];+++ }+ }+ }++};++diff --git a/include/store.h b/include/store.hindex ffb96b8a..20bb5529 100644--- a/include/store.h+++ b/include/store.h@@ -186,7 +186,7 @@ public:return Option<std::vector<std::string>*>(updates);}- rocksdb::unique_ptr<rocksdb::TransactionLogIterator> iter;+ std::unique_ptr<rocksdb::TransactionLogIterator> iter;rocksdb::Status status = db->GetUpdatesSince(seq_number, &iter);if(!status.ok()) {diff --git a/src/main/typesense_server.cpp b/src/main/typesense_server.cppindex fd9ed722..788fde93 100644--- a/src/main/typesense_server.cpp+++ b/src/main/typesense_server.cpp@@ -2,6 +2,16 @@#include "core_api.h"#include "config.h"+extern "C" {+#include "jemalloc.h"+}++#ifdef __APPLE__+extern "C" {+ extern void je_zone_register();+}+#endif+void master_server_routes() {// collection managementserver->post("/collections", post_create_collection);@@ -56,6 +66,15 @@ void replica_server_routes() {}int main(int argc, char **argv) {+ #ifdef __APPLE__+ // On OS X, je_zone_register registers jemalloc with the system allocator.+ // We have to force the presence of these symbols on macOS by explicitly calling this method.+ // See these issues:+ // - https://github.com/jemalloc/jemalloc/issues/708+ // - https://github.com/ClickHouse/ClickHouse/pull/11897+ je_zone_register();+ #endif+Config config;cmdline::parser options;diff --git a/src/typesense_server_utils.cpp b/src/typesense_server_utils.cppindex ce1db3db..de7e3155 100644--- a/src/typesense_server_utils.cpp+++ b/src/typesense_server_utils.cpp@@ -1,7 +1,6 @@#include <curl/curl.h>-#include <sys/stat.h>-#include <gflags/gflags.h>+#include <dlfcn.h>#include <brpc/controller.h>#include <brpc/server.h>#include <braft/raft.h>@@ -18,6 +17,21 @@HttpServer* server;std::atomic<bool> quit_raft_service;+extern "C" {+ typedef int (*mallctl_t)(const char *name, void *oldp, size_t *oldlenp, void *newp, size_t newlen);+}++bool using_jemalloc() {+ // On OSX, jemalloc API is prefixed with "je_"+ mallctl_t mallctl;+#ifdef __APPLE__+ mallctl = (mallctl_t) ::dlsym(RTLD_DEFAULT, "je_mallctl");+#else+ mallctl = (mallctl_t) ::dlsym(RTLD_DEFAULT, "mallctl");+#endif+ return (mallctl != nullptr);+}+void catch_interrupt(int sig) {LOG(INFO) << "Stopping Typesense server...";signal(sig, SIG_IGN); // ignore for now as we want to shut down elegantly@@ -292,6 +306,13 @@ int start_raft_server(ReplicationState& replication_state, const std::string& stint run_server(const Config & config, const std::string & version, void (*master_server_routes)()) {LOG(INFO) << "Starting Typesense " << version << std::flush;++ if(using_jemalloc()) {+ LOG(INFO) << "Typesense is using jemalloc.";+ } else {+ LOG(WARNING) << "Typesense is NOT using jemalloc.";+ }+quit_raft_service = false;if(!directory_exists(config.get_data_dir())) {diff --git a/test/art_test.cpp b/test/art_test.cppindex 6ede5656..b843978a 100644--- a/test/art_test.cpp+++ b/test/art_test.cpp@@ -194,7 +194,7 @@ TEST(ArtTest, test_art_insert_delete) {// Delete, should get lineno backart_values* values = (art_values*)art_delete(&t, (unsigned char*)buf, len);EXPECT_EQ(line, values->ids.at(0));- free(values);+ delete values;// Check the sizeASSERT_TRUE(art_size(&t) == nlines - line);