Skip to content

Open Source alternative to Algolia + Pinecone and an Easier-to-Use alternative to ElasticSearch ⚡ 🔍 ✨ Fast, typo tolerant, in-memory fuzzy Search Engine for building delightful search experiences

Force use of jemalloc via static linking.

kishorenc1ab9b6509874
diff --git a/CMakeLists.txt b/CMakeLists.txt
index b5afc665..5f3ffce2 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -22,6 +22,8 @@ if (APPLE)
# Prefer brew installated libraries
set(OPENSSL_ROOT_DIR /usr/local/opt/[email protected])
+ set(ENV{OPENSSL_ROOT_DIR} /usr/local/opt/[email protected])
+
set(SNAPPY_ROOT_DIR /usr/local/opt/snappy)
set(ZLIB_ROOT /usr/local/opt/zlib)
set(CMAKE_PREFIX_PATH /usr/local/opt/curl-openssl /usr/local/opt/icu4c)
@@ -33,8 +35,17 @@ IF(WIN32 OR MSVC)
SET(CMAKE_FIND_LIBRARY_SUFFIXES .lib .a)
ELSE()
SET(CMAKE_FIND_LIBRARY_SUFFIXES .a)
+ set(ENV{CMAKE_FIND_LIBRARY_SUFFIXES} ".a")
ENDIF()
+include(cmake/For.cmake)
+include(cmake/H2O.cmake)
+include(cmake/RocksDB.cmake)
+include(cmake/GoogleTest.cmake)
+include(cmake/TestResources.cmake)
+include(cmake/Iconv.cmake)
+include(cmake/Jemalloc.cmake)
+
FIND_PACKAGE(OpenSSL 1.1.1 REQUIRED)
FIND_PACKAGE(Snappy REQUIRED)
FIND_PACKAGE(ZLIB REQUIRED)
@@ -47,13 +58,6 @@ FIND_PACKAGE(glog REQUIRED)
message("OpenSSL library: ${OPENSSL_LIBRARIES}")
-include(cmake/For.cmake)
-include(cmake/H2O.cmake)
-include(cmake/RocksDB.cmake)
-include(cmake/GoogleTest.cmake)
-include(cmake/TestResources.cmake)
-include(cmake/Iconv.cmake)
-
FILE(GLOB SRC_FILES src/*.cpp)
FILE(GLOB TEST_FILES test/*.cpp)
@@ -68,52 +72,28 @@ include_directories(${DEP_ROOT_DIR}/${ROCKSDB_NAME}/include)
include_directories(${DEP_ROOT_DIR}/${ICONV_NAME}/include)
include_directories(${DEP_ROOT_DIR}/${BRPC_NAME}/include)
include_directories(${DEP_ROOT_DIR}/${BRAFT_NAME}/include)
-
-# Write dependency include directories to a file
-file(WRITE ${DEP_ROOT_DIR}/includes.txt "")
-file(APPEND ${DEP_ROOT_DIR}/includes.txt "${DEP_ROOT_DIR}/${FOR_NAME}")
-file(APPEND ${DEP_ROOT_DIR}/includes.txt " ${DEP_ROOT_DIR}/${H2O_NAME}/include")
-file(APPEND ${DEP_ROOT_DIR}/includes.txt " ${DEP_ROOT_DIR}/${ROCKSDB_NAME}/include")
-file(APPEND ${DEP_ROOT_DIR}/includes.txt " ${DEP_ROOT_DIR}/${ICONV_NAME}/include")
-file(APPEND ${DEP_ROOT_DIR}/includes.txt " ${DEP_ROOT_DIR}/${BRPC_NAME}/include")
-file(APPEND ${DEP_ROOT_DIR}/includes.txt " ${DEP_ROOT_DIR}/${BRAFT_NAME}/include")
+include_directories(${DEP_ROOT_DIR}/${JEMALLOC_NAME}/include/jemalloc)
link_directories(${DEP_ROOT_DIR}/${GTEST_NAME}/googletest/build)
link_directories(${DEP_ROOT_DIR}/${FOR_NAME})
link_directories(${DEP_ROOT_DIR}/${H2O_NAME}/build)
link_directories(${DEP_ROOT_DIR}/${ROCKSDB_NAME})
link_directories(${DEP_ROOT_DIR}/${ICONV_NAME}/lib/.libs)
-if (APPLE)
-link_directories(${DEP_ROOT_DIR}/${BRPC_NAME}/lib)
-link_directories(${DEP_ROOT_DIR}/${BRAFT_NAME}/lib)
-endif()
+link_directories(${DEP_ROOT_DIR}/${JEMALLOC_NAME}/lib)
-# Write dependency libraries to a file
-file(WRITE ${DEP_ROOT_DIR}/libs.txt "")
-file(APPEND ${DEP_ROOT_DIR}/libs.txt " ${DEP_ROOT_DIR}/${FOR_NAME}")
-file(APPEND ${DEP_ROOT_DIR}/libs.txt " ${DEP_ROOT_DIR}/${H2O_NAME}/build")
-file(APPEND ${DEP_ROOT_DIR}/libs.txt " ${DEP_ROOT_DIR}/${ROCKSDB_NAME}")
-file(APPEND ${DEP_ROOT_DIR}/libs.txt " ${DEP_ROOT_DIR}/${ICONV_NAME}/lib/.libs")
-file(APPEND ${DEP_ROOT_DIR}/libs.txt " ${DEP_ROOT_DIR}/${BRPC_NAME}/lib")
-file(APPEND ${DEP_ROOT_DIR}/libs.txt " ${DEP_ROOT_DIR}/${BRAFT_NAME}/lib")
+set(JEMALLOC_ROOT_DIR "${DEP_ROOT_DIR}/${JEMALLOC_NAME}")
+FIND_PACKAGE(Jemalloc REQUIRED)
add_executable(typesense-server ${SRC_FILES} src/main/typesense_server.cpp)
add_executable(search ${SRC_FILES} src/main/main.cpp)
add_executable(benchmark ${SRC_FILES} src/main/benchmark.cpp)
add_executable(typesense-test ${SRC_FILES} ${TEST_FILES})
-add_library(typesense-core STATIC ${SRC_FILES})
-
target_compile_definitions(
typesense-server PRIVATE
TYPESENSE_VERSION="${TYPESENSE_VERSION}"
)
-target_compile_definitions(
- typesense-core PRIVATE
- TYPESENSE_VERSION="${TYPESENSE_VERSION}"
-)
-
target_compile_definitions(
benchmark PRIVATE
TYPESENSE_VERSION="${TYPESENSE_VERSION}"
@@ -133,7 +113,7 @@ target_compile_definitions(
set(ROCKSDB_LIBS rocksdb ${SNAPPY_LIBRARIES} ${ZLIB_LIBRARIES})
if (APPLE)
- set(STD_LIB -static-libstdc++) # https://stackoverflow.com/a/26543140/131050 (can't statically link libgcc on Mac)
+ set(STD_LIB "") # https://stackoverflow.com/a/26543140/131050 (can't statically link libgcc on Mac)
FIND_PACKAGE(ngHTTP2 REQUIRED)
set(CURL_LIBRARIES ${NGHTTP2_LIBRARIES} ${CURL_LIBRARIES}) # nghttp2 was not being statically built
@@ -150,9 +130,9 @@ set(ICU_ALL_LIBRARIES ${ICU_I18N_LIBRARIES} ${ICU_LIBRARIES} ${ICU_DATA_LIBRARIE
set(CORE_LIBS h2o-evloop iconv ${CURL_LIBRARIES} for ${ICU_ALL_LIBRARIES} ${G3LOGGER_LIBRARIES}
${ROCKSDB_LIBS} ${OPENSSL_LIBRARIES} pthread dl ${STD_LIB})
-set(CORE_LIBS braft brpc ${LevelDB_LIBRARIES} glog ${CORE_LIBS} ${GFLAGS_LIBRARIES} ${PROTOBUF_LIBRARIES} ${SYSTEM_LIBS})
+set(CORE_LIBS ${JEMALLOC_LIBRARIES} braft brpc ${LevelDB_LIBRARIES} glog
+ ${CORE_LIBS} ${GFLAGS_LIBRARIES} ${PROTOBUF_LIBRARIES} ${SYSTEM_LIBS})
-target_link_libraries(typesense-core ${CORE_LIBS})
target_link_libraries(typesense-server ${CORE_LIBS})
target_link_libraries(search ${CORE_LIBS})
target_link_libraries(benchmark ${CORE_LIBS})
diff --git a/build.sh b/build.sh
index c31288f4..26d8c00b 100755
--- a/build.sh
+++ b/build.sh
@@ -31,16 +31,3 @@ if [[ "$@" == *"--package-binary"* ]]; then
tar -cvzf $PROJECT_DIR/$BUILD_DIR/$RELEASE_NAME.tar.gz -C $PROJECT_DIR/$BUILD_DIR typesense-server typesense-server.md5.txt
echo "Built binary successfully: $PROJECT_DIR/$BUILD_DIR/$RELEASE_NAME.tar.gz"
fi
-
-if [[ "$@" == *"--package-libs"* ]]; then
- OS_FAMILY=$(echo `uname` | awk '{print tolower($0)}')
- RELEASE_NAME=typesense-server-libs-$TYPESENSE_VERSION-$OS_FAMILY-amd64
- LIBS=`cat $PROJECT_DIR/external-$SYSTEM_NAME/libs.txt`
- TAR_PATHS=""
- for lib in $LIBS; do
- TAR_PATHS="$TAR_PATHS -C $lib `ls $lib/*.a | xargs basename`"
- done
-
- TAR_PATHS="$TAR_PATHS -C $PROJECT_DIR/$BUILD_DIR `ls $PROJECT_DIR/$BUILD_DIR/*.a | xargs basename`"
- tar -cvzf $PROJECT_DIR/$BUILD_DIR/$RELEASE_NAME.tar.gz $TAR_PATHS
-fi
diff --git a/cmake/Jemalloc.cmake b/cmake/Jemalloc.cmake
new file mode 100644
index 00000000..2747acb0
--- /dev/null
+++ b/cmake/Jemalloc.cmake
@@ -0,0 +1,34 @@
+# Download and build Jemalloc
+
+set(JEMALLOC_VERSION 5.2.1)
+set(JEMALLOC_NAME jemalloc-${JEMALLOC_VERSION})
+set(JEMALLOC_TAR_PATH ${DEP_ROOT_DIR}/${JEMALLOC_NAME}.tar.bz2)
+
+if(NOT EXISTS ${JEMALLOC_TAR_PATH})
+ message(STATUS "Downloading ${JEMALLOC_NAME}...")
+ file(DOWNLOAD https://github.com/jemalloc/jemalloc/releases/download/${JEMALLOC_VERSION}/${JEMALLOC_NAME}.tar.bz2
+ ${JEMALLOC_TAR_PATH})
+endif()
+
+if(NOT EXISTS ${DEP_ROOT_DIR}/${JEMALLOC_NAME})
+ message(STATUS "Extracting jemalloc...")
+ execute_process(COMMAND ${CMAKE_COMMAND} -E tar xzf ${JEMALLOC_TAR_PATH} WORKING_DIRECTORY ${DEP_ROOT_DIR})
+endif()
+
+if(NOT EXISTS ${DEP_ROOT_DIR}/${JEMALLOC_NAME}/Makefile)
+ message("Configuring jemalloc locally...")
+ # Builds with "--with-jemalloc-prefix=je_" on OSX
+ execute_process(COMMAND ./configure WORKING_DIRECTORY ${DEP_ROOT_DIR}/${JEMALLOC_NAME} RESULT_VARIABLE JEMALLOC_CONFIGURE)
+ if(NOT JEMALLOC_CONFIGURE EQUAL 0)
+ message(FATAL_ERROR "${JEMALLOC_NAME} configure failed!")
+ message("${RESULT_VARIABLE}")
+ endif()
+endif()
+
+if(NOT EXISTS ${DEP_ROOT_DIR}/${JEMALLOC_NAME}/lib/libjemalloc.a)
+ message("Building jemalloc locally...")
+ execute_process(COMMAND make "build_lib_static" WORKING_DIRECTORY ${DEP_ROOT_DIR}/${JEMALLOC_NAME})
+ if(NOT EXISTS ${DEP_ROOT_DIR}/${JEMALLOC_NAME}/lib/libjemalloc.a)
+ message(FATAL_ERROR "${JEMALLOC_NAME} build failed!")
+ endif()
+endif()
diff --git a/cmake/Modules/FindJemalloc.cmake b/cmake/Modules/FindJemalloc.cmake
new file mode 100644
index 00000000..f27b3b36
--- /dev/null
+++ b/cmake/Modules/FindJemalloc.cmake
@@ -0,0 +1,38 @@
+# Tries to find Jemalloc headers and libraries.
+#
+# Usage of this module as follows:
+#
+# find_package(jemalloc)
+#
+# Variables used by this module, they can change the default behaviour and need
+# to be set before calling find_package:
+#
+# JEMALLOC_ROOT_DIR Set this variable to the root installation of
+# Jemalloc if the module has problems finding
+# the proper installation path.
+#
+# Variables defined by this module:
+#
+# JEMALLOC_FOUND System has Jemalloc libs/headers
+# JEMALLOC_LIBRARIES The Jemalloc libraries
+# JEMALLOC_INCLUDE_DIR The location of Jemalloc headers
+
+find_path(JEMALLOC_INCLUDE_DIR
+ NAMES jemalloc.h
+ HINTS ${JEMALLOC_ROOT_DIR}/include/jemalloc
+ NO_DEFAULT_PATH)
+
+find_library(JEMALLOC_LIBRARIES
+ NAMES jemalloc
+ HINTS ${JEMALLOC_ROOT_DIR}/lib
+ NO_DEFAULT_PATH)
+
+include(FindPackageHandleStandardArgs)
+find_package_handle_standard_args(jemalloc DEFAULT_MSG
+ JEMALLOC_LIBRARIES
+ JEMALLOC_INCLUDE_DIR)
+
+mark_as_advanced(
+ JEMALLOC_ROOT_DIR
+ JEMALLOC_LIBRARIES
+ JEMALLOC_INCLUDE_DIR)
diff --git a/docker-build.sh b/docker-build.sh
index e2ab7d71..89dcaeb6 100755
--- a/docker-build.sh
+++ b/docker-build.sh
@@ -46,18 +46,6 @@ if [[ "$@" == *"--package-binary"* ]]; then
echo "Built binary successfully: $PROJECT_DIR/$BUILD_DIR/$RELEASE_NAME.tar.gz"
fi
-if [[ "$@" == *"--package-libs"* ]]; then
- OS_FAMILY=linux
- RELEASE_NAME=typesense-server-libs-$TYPESENSE_VERSION-$OS_FAMILY-amd64
- LIBS=`cat $PROJECT_DIR/external-$SYSTEM_NAME/libs.txt`
- TAR_PATHS=""
- for lib in $LIBS; do
- TAR_PATHS="$TAR_PATHS -C $PROJECT_DIR/../$lib `ls $PROJECT_DIR/../$lib/*.a | xargs basename`"
- done
-
- TAR_PATHS="$TAR_PATHS -C $PROJECT_DIR/$BUILD_DIR `ls $PROJECT_DIR/$BUILD_DIR/*.a | xargs basename`"
- tar -cvzf $PROJECT_DIR/$BUILD_DIR/$RELEASE_NAME.tar.gz $TAR_PATHS
-fi
#
#if [[ "$@" == *"--create-deb-upload"* ]]; then
# docker run -it -v $PROJECT_DIR:/typesense typesense/typesense-development:23-JUNE-2020-1 cmake -DTYPESENSE_VERSION=$TYPESENSE_VERSION \
diff --git a/include/cvt.h b/include/cvt.h
new file mode 100644
index 00000000..ca0840cb
--- /dev/null
+++ b/include/cvt.h
@@ -0,0 +1,109 @@
+/* Compact Variable Trie
+================================================================================================================
+ ates, at, as, bet, to, tk
+
+ [ * ]
+ ⁄ | \
+ a b t
+ / \ \ /\
+ s t etØ k o
+ / / \ / \
+ Ø esØ Ø Ø Ø
+
+
+ BASIC DESIGN
+ ============
+
+ * All siblings in the same block.
+ * Pointer to ONLY FIRST child node of each sibling.
+ * Each sibling node's children represented by their character/byte
+ * For root there are no siblings, so pointer only to `a` child.
+
+ ROOT -> [0|0|PTRA][3][a][b][t]
+ PTRA -> [0|PTRS][2|PTRE][4|PTRK[2][s][t][2][e][t][2][k][o]
+ PTRS -> [0|PTRØ][1|PTRT][1][Ø][2][e][Ø]
+ PTRT -> [0|L_PTRE][3|PTRØ][3][e][s][Ø][1][Ø] (path compression)
+ PTRØ -> [0|LEAF]
+
+ [TYPE+OFFSET][PTR_1]..[NUM_CHILDREN][A][B]..[X]
+
+ 2 bytes for type+offset
+ 6 bytes for address
+ 1 byte for num_children
+ x bytes for bytes
+
+ Actual offset to the Nth node's children: (8*NUM_CHILDREN) + N + offset
+
+ if num_children >= 32:
+ Use bitset to represent children present
+ Read 32 bytes and do bitset operations to extract matched index
+ else:
+ Use array to represent children
+ Read `num_children` bytes and do sequential search
+
+ Removal of [be]
+
+ 1. Realloc contents of PTR1 by removing "e" from the nodes list
+ 2. Free PTR3
+ 3. Realloc contents of ROOT by removing "b" from the nodes list
+
+*/
+
+#include <cstdint>
+#include <cstddef>
+
+
+
+
+class CVTrie {
+private:
+ size_t size;
+
+ // [TYPE+OFFSET][PTR_1]... (8 bytes each)
+ // [NUM_PREFIX][A][B]..[X] (upto 33 bytes)
+ uint8_t* root;
+
+ const uintptr_t PTR_MASK = ~(1ULL << 48ULL);
+
+public:
+
+ uint8_t* get_ptr(const uint8_t* tagged_ptr) {
+ // Unfortunately, right shifting of signed integer for sign extension is implementation-defined
+ // but works on all major compilers
+ return (uint8_t*)( (intptr_t)((uintptr_t)tagged_ptr << 16ULL) >> 16ULL );
+ }
+
+ uint8_t* tag_ptr(const uint8_t* ptr, const uint64_t data) {
+ return (uint8_t*)(((uintptr_t)ptr & PTR_MASK) | (data << 48ULL));
+ }
+
+ uint16_t get_data(const uint16_t* ptr) {
+ return (uintptr_t)(ptr) >> 56ULL;;
+ }
+
+ void add(const unsigned char* key, const size_t length, void* value) {
+ // If the key exists, augment the node, otherwise insert a new node
+
+ size_t num_siblings = 1;
+ uint8_t* buf = root;
+ size_t key_index = 0;
+
+ if(root == nullptr) {
+ // trie is empty
+ root = new uint8_t[8+1+1];
+
+ }
+
+ while(true) {
+ // for each sibling
+ for(auto sindex = 0; sindex < num_siblings; sindex++) {
+ unsigned char c = key[key_index];
+
+
+ }
+ }
+ }
+
+};
+
+
diff --git a/include/store.h b/include/store.h
index ffb96b8a..20bb5529 100644
--- a/include/store.h
+++ b/include/store.h
@@ -186,7 +186,7 @@ public:
return Option<std::vector<std::string>*>(updates);
}
- rocksdb::unique_ptr<rocksdb::TransactionLogIterator> iter;
+ std::unique_ptr<rocksdb::TransactionLogIterator> iter;
rocksdb::Status status = db->GetUpdatesSince(seq_number, &iter);
if(!status.ok()) {
diff --git a/src/main/typesense_server.cpp b/src/main/typesense_server.cpp
index fd9ed722..788fde93 100644
--- a/src/main/typesense_server.cpp
+++ b/src/main/typesense_server.cpp
@@ -2,6 +2,16 @@
#include "core_api.h"
#include "config.h"
+extern "C" {
+#include "jemalloc.h"
+}
+
+#ifdef __APPLE__
+extern "C" {
+ extern void je_zone_register();
+}
+#endif
+
void master_server_routes() {
// collection management
server->post("/collections", post_create_collection);
@@ -56,6 +66,15 @@ void replica_server_routes() {
}
int main(int argc, char **argv) {
+ #ifdef __APPLE__
+ // On OS X, je_zone_register registers jemalloc with the system allocator.
+ // We have to force the presence of these symbols on macOS by explicitly calling this method.
+ // See these issues:
+ // - https://github.com/jemalloc/jemalloc/issues/708
+ // - https://github.com/ClickHouse/ClickHouse/pull/11897
+ je_zone_register();
+ #endif
+
Config config;
cmdline::parser options;
diff --git a/src/typesense_server_utils.cpp b/src/typesense_server_utils.cpp
index ce1db3db..de7e3155 100644
--- a/src/typesense_server_utils.cpp
+++ b/src/typesense_server_utils.cpp
@@ -1,7 +1,6 @@
#include <curl/curl.h>
-#include <sys/stat.h>
-
#include <gflags/gflags.h>
+#include <dlfcn.h>
#include <brpc/controller.h>
#include <brpc/server.h>
#include <braft/raft.h>
@@ -18,6 +17,21 @@
HttpServer* server;
std::atomic<bool> quit_raft_service;
+extern "C" {
+ typedef int (*mallctl_t)(const char *name, void *oldp, size_t *oldlenp, void *newp, size_t newlen);
+}
+
+bool using_jemalloc() {
+ // On OSX, jemalloc API is prefixed with "je_"
+ mallctl_t mallctl;
+#ifdef __APPLE__
+ mallctl = (mallctl_t) ::dlsym(RTLD_DEFAULT, "je_mallctl");
+#else
+ mallctl = (mallctl_t) ::dlsym(RTLD_DEFAULT, "mallctl");
+#endif
+ return (mallctl != nullptr);
+}
+
void catch_interrupt(int sig) {
LOG(INFO) << "Stopping Typesense server...";
signal(sig, SIG_IGN); // ignore for now as we want to shut down elegantly
@@ -292,6 +306,13 @@ int start_raft_server(ReplicationState& replication_state, const std::string& st
int run_server(const Config & config, const std::string & version, void (*master_server_routes)()) {
LOG(INFO) << "Starting Typesense " << version << std::flush;
+
+ if(using_jemalloc()) {
+ LOG(INFO) << "Typesense is using jemalloc.";
+ } else {
+ LOG(WARNING) << "Typesense is NOT using jemalloc.";
+ }
+
quit_raft_service = false;
if(!directory_exists(config.get_data_dir())) {
diff --git a/test/art_test.cpp b/test/art_test.cpp
index 6ede5656..b843978a 100644
--- a/test/art_test.cpp
+++ b/test/art_test.cpp
@@ -194,7 +194,7 @@ TEST(ArtTest, test_art_insert_delete) {
// Delete, should get lineno back
art_values* values = (art_values*)art_delete(&t, (unsigned char*)buf, len);
EXPECT_EQ(line, values->ids.at(0));
- free(values);
+ delete values;
// Check the size
ASSERT_TRUE(art_size(&t) == nlines - line);