Open Source alternative to Algolia + Pinecone and an Easier-to-Use alternative to ElasticSearch ⚡ 🔍 ✨ Fast, typo tolerant, in-memory fuzzy Search Engine for building delightful search experiences
C++ algoliadatastoreelasticsearchenterprise-searchfacetingfull-text-searchfuzzy-searchgeosearchin-memoryinstantsearchmerchandisingpineconesearchsearch-enginesemantic-searchsimilarity-searchsite-searchsynonymstypo-tolerancevector-search
Patch RocksDB build to ensure that it links against only specific libraries.
Kishore Nallan8850f8b16969
diff --git a/CMakeLists.txt b/CMakeLists.txtindex eb1343d7..0330e582 100644--- a/CMakeLists.txt+++ b/CMakeLists.txt@@ -5,6 +5,7 @@ set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -std=c++11 -O2")set(CMAKE_CXX_FLAGS_DEBUG "${CMAKE_CXX_FLAGS_DEBUG} -std=c++11 -O0")# set(CMAKE_EXE_LINKER_FLAGS "--verbose")set(DEP_ROOT_DIR ${CMAKE_SOURCE_DIR}/external-${CMAKE_SYSTEM_NAME})+set(CMAKE_MODULE_PATH ${CMAKE_MODULE_PATH} "${CMAKE_SOURCE_DIR}/cmake/Modules/")if(NOT EXISTS ${DEP_ROOT_DIR})file(MAKE_DIRECTORY ${DEP_ROOT_DIR})@@ -13,10 +14,35 @@ endif()if (APPLE)add_definitions(-DGTEST_USE_OWN_TR1_TUPLE)add_definitions(-D__GLIBCXX__)- set(ENV{OPENSSL_ROOT_DIR} /usr/local/opt/openssl) # For Brew++ # For Brew+ set(ENV{OPENSSL_ROOT_DIR} /usr/local/opt/openssl)+ set(SNAPPY_ROOT_DIR /usr/local/opt/snappy)+ set(ZLIB_ROOT /usr/local/opt/zlib)endif (APPLE)-FIND_PACKAGE(OpenSSL REQUIRED)+# Ensure that we only use the static versions of libraries to link against+# You can't statically link a shared library (or dynamically link a static library)+IF(WIN32 OR MSVC)+ SET(CMAKE_FIND_LIBRARY_SUFFIXES .lib .a)+ELSE()+ SET(CMAKE_FIND_LIBRARY_SUFFIXES .a)+ENDIF()++function(find_static_library LIB_NAME OUT)+ find_library(FOUND_${LIB_NAME} ${LIB_NAME})+ message("FINDING: ${LIB_NAME} and got ${FOUND_${LIB_NAME}}")++ if(NOT FOUND_${LIB_NAME})+ message(FATAL_ERROR "Unable to find static library ${LIB_NAME}")+ endif()++ set(${OUT} ${FOUND_${LIB_NAME}} PARENT_SCOPE)+endfunction()++FIND_PACKAGE(OpenSSL 1.0.2 REQUIRED)+FIND_PACKAGE(Snappy REQUIRED)+FIND_PACKAGE(ZLIB REQUIRED)include(cmake/For.cmake)include(cmake/H2O.cmake)@@ -42,15 +68,15 @@ link_directories(${DEP_ROOT_DIR}/${ROCKSDB_NAME})add_executable(typesense-server ${SRC_FILES} src/main/server.cpp)add_executable(search ${SRC_FILES} src/main/main.cpp)add_executable(benchmark ${SRC_FILES} src/main/benchmark.cpp)-add_executable(typesense_test test/array_test.cpp test/sorted_array_test.cpp test/art_test.cpp- test/collection_test.cpp test/collection_manager_test.cpp test/topster_test.cpp ${HEADER_FILES} ${SRC_FILES})+add_executable(typesense_test ${SRC_FILES} test/array_test.cpp test/sorted_array_test.cpp test/art_test.cpp+ test/collection_test.cpp test/collection_manager_test.cpp test/topster_test.cpp)target_compile_definitions(typesense-server PRIVATE ROOT_DIR="${CMAKE_SOURCE_DIR}/")target_compile_definitions(search PRIVATE ROOT_DIR="${CMAKE_SOURCE_DIR}/")target_compile_definitions(benchmark PRIVATE ROOT_DIR="${CMAKE_SOURCE_DIR}/")target_compile_definitions(typesense_test PRIVATE ROOT_DIR="${CMAKE_SOURCE_DIR}/")-target_link_libraries(typesense-server for h2o-evloop pthread rocksdb ${OPENSSL_LIBRARIES})-target_link_libraries(search for pthread rocksdb)-target_link_libraries(benchmark for pthread rocksdb)-target_link_libraries(typesense_test pthread for rocksdb gtest gtest_main)+target_link_libraries(typesense-server for h2o-evloop pthread rocksdb ${SNAPPY_LIBRARIES} ${ZLIB_LIBRARIES} ${OPENSSL_LIBRARIES} dl)+target_link_libraries(search for pthread rocksdb ${SNAPPY_LIBRARIES} ${ZLIB_LIBRARIES})+target_link_libraries(benchmark for pthread rocksdb ${SNAPPY_LIBRARIES} ${ZLIB_LIBRARIES})+target_link_libraries(typesense_test pthread for rocksdb ${SNAPPY_LIBRARIES} ${ZLIB_LIBRARIES} gtest gtest_main)diff --git a/build.sh b/build.shindex 02df235a..5be536bb 100755--- a/build.sh+++ b/build.sh@@ -1,17 +1,10 @@#!/bin/sh-set -o xtrace-PROJECT_DIR=`dirname $0`if [ "$1" = "--clean" ]; thenrm -rf $PROJECT_DIR/buildmkdir $PROJECT_DIR/build-- # rm -rf $PROJECT_DIR/external- # DOCKER_NO_CACHE=" --no-cache"fi-docker build --file $PROJECT_DIR/docker/development.Dockerfile --tag wreally/typesense-development $DOCKER_NO_CACHE $PROJECT_DIR/docker--./dockcross --image wreally/typesense-development cmake -H$PROJECT_DIR -B$PROJECT_DIR/build-./dockcross --image wreally/typesense-development make -C $PROJECT_DIR/build\ No newline at end of file+cmake -H$PROJECT_DIR -B$PROJECT_DIR/build+make -C $PROJECT_DIR/build\ No newline at end of filediff --git a/cmake/Modules/FindSnappy.cmake b/cmake/Modules/FindSnappy.cmakenew file mode 100644index 00000000..99b43c6b--- /dev/null+++ b/cmake/Modules/FindSnappy.cmake@@ -0,0 +1,54 @@+#+# Licensed to the Apache Software Foundation (ASF) under one+# or more contributor license agreements. See the NOTICE file+# distributed with this work for additional information+# regarding copyright ownership. The ASF licenses this file+# to you under the Apache License, Version 2.0 (the+# "License"); you may not use this file except in compliance+# with the License. You may obtain a copy of the License at+#+# http://www.apache.org/licenses/LICENSE-2.0+#+# Unless required by applicable law or agreed to in writing,+# software distributed under the License is distributed on an+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY+# KIND, either express or implied. See the License for the+# specific language governing permissions and limitations+# under the License.+#+# Tries to find Snappy headers and libraries.+#+# Usage of this module as follows:+#+# find_package(Snappy)+#+# Variables used by this module, they can change the default behaviour and need+# to be set before calling find_package:+#+# SNAPPY_ROOT_DIR Set this variable to the root installation of+# Snappy if the module has problems finding+# the proper installation path.+#+# Variables defined by this module:+#+# SNAPPY_FOUND System has Snappy libs/headers+# SNAPPY_LIBRARIES The Snappy libraries+# SNAPPY_INCLUDE_DIR The location of Snappy headers++find_path(SNAPPY_INCLUDE_DIR+ NAMES snappy.h+ HINTS ${SNAPPY_ROOT_DIR}/include)++find_library(SNAPPY_LIBRARIES+ NAMES snappy+ HINTS ${SNAPPY_ROOT_DIR}/lib)++include(FindPackageHandleStandardArgs)+find_package_handle_standard_args(Snappy DEFAULT_MSG+ SNAPPY_LIBRARIES+ SNAPPY_INCLUDE_DIR)++mark_as_advanced(+ SNAPPY_ROOT_DIR+ SNAPPY_LIBRARIES+ SNAPPY_INCLUDE_DIR)\ No newline at end of filediff --git a/cmake/RocksDB.cmake b/cmake/RocksDB.cmakeindex 53e31f1f..b026df18 100644--- a/cmake/RocksDB.cmake+++ b/cmake/RocksDB.cmake@@ -1,6 +1,6 @@# Download and build RocksDB-set(ROCKSDB_VERSION 4.11.2)+set(ROCKSDB_VERSION 4.13.5)set(ROCKSDB_NAME rocksdb-${ROCKSDB_VERSION})set(ROCKSDB_TAR_PATH ${DEP_ROOT_DIR}/${ROCKSDB_NAME}.tar.gz)@@ -14,9 +14,13 @@ if(NOT EXISTS ${DEP_ROOT_DIR}/${ROCKSDB_NAME})execute_process(COMMAND ${CMAKE_COMMAND} -E tar xvzf ${ROCKSDB_TAR_PATH} WORKING_DIRECTORY ${DEP_ROOT_DIR}/)endif()-if(NOT EXISTS ${DEP_ROOT_DIR}/${ROCKSDB_NAME}/librocksdb.so)+# Use patched build detect platform script to control what libraries RocksDB links against+file(COPY ${CMAKE_SOURCE_DIR}/cmake/patches/build_detect_platform DESTINATION+ ${DEP_ROOT_DIR}/${ROCKSDB_NAME}/build_tools)++if(NOT EXISTS ${DEP_ROOT_DIR}/${ROCKSDB_NAME}/librocksdb.a)message("Building ${ROCKSDB_NAME} locally...")- execute_process(COMMAND make "shared_lib" WORKING_DIRECTORY ${DEP_ROOT_DIR}/${ROCKSDB_NAME}/+ execute_process(COMMAND make "static_lib" WORKING_DIRECTORY ${DEP_ROOT_DIR}/${ROCKSDB_NAME}/RESULT_VARIABLE ROCKSDB_BUILD)if(NOT ROCKSDB_BUILD EQUAL 0)message(FATAL_ERROR "${ROCKSDB_NAME} build failed!")diff --git a/cmake/TestResources.cmake b/cmake/TestResources.cmakeindex 761917b6..1d1c1284 100644--- a/cmake/TestResources.cmake+++ b/cmake/TestResources.cmake@@ -1,11 +1,12 @@# Download test resourcesset(ART_VERSION bbbf588bca55bce095538ee8ca8b422904baebc5)-set(ART_WORDS_PATH /tmp/typesense_test/words.txt)-set(ART_UUID_PATH /tmp/typesense_test/uuid.txt)+set(TEST_RESOURCES_DIR ${CMAKE_SOURCE_DIR}/build/test_resources)+set(ART_WORDS_PATH ${TEST_RESOURCES_DIR}/words.txt)+set(ART_UUID_PATH ${TEST_RESOURCES_DIR}/uuid.txt)-if(NOT EXISTS /tmp/typesense_test)- file(MAKE_DIRECTORY /tmp/typesense_test)+if(NOT EXISTS ${TEST_RESOURCES_DIR})+ file(MAKE_DIRECTORY ${TEST_RESOURCES_DIR})endif()if(NOT EXISTS ${ART_WORDS_PATH})diff --git a/cmake/patches/build_detect_platform b/cmake/patches/build_detect_platformindex 7fdeeba1..dcc3a06e 100755--- a/cmake/patches/build_detect_platform+++ b/cmake/patches/build_detect_platform@@ -215,30 +215,6 @@ EOFJAVA_LDFLAGS="$JAVA_LDFLAGS -lsnappy"fi- # Test whether gflags library is installed- # http://gflags.github.io/gflags/- # check if the namespace is gflags- $CXX $CFLAGS -x c++ - -o /dev/null 2>/dev/null << EOF- #include <gflags/gflags.h>- using namespace gflags;- int main() {}-EOF- if [ "$?" = 0 ]; then- COMMON_FLAGS="$COMMON_FLAGS -DGFLAGS=gflags"- PLATFORM_LDFLAGS="$PLATFORM_LDFLAGS -lgflags"- else- # check if namespace is google- $CXX $CFLAGS -x c++ - -o /dev/null 2>/dev/null << EOF- #include <gflags/gflags.h>- using namespace google;- int main() {}-EOF- if [ "$?" = 0 ]; then- COMMON_FLAGS="$COMMON_FLAGS -DGFLAGS=google"- PLATFORM_LDFLAGS="$PLATFORM_LDFLAGS -lgflags"- fi- fi-# Test whether zlib library is installed$CXX $CFLAGS $COMMON_FLAGS -x c++ - -o /dev/null 2>/dev/null <<EOF#include <zlib.h>@@ -250,78 +226,6 @@ EOFJAVA_LDFLAGS="$JAVA_LDFLAGS -lz"fi- # Test whether bzip library is installed- $CXX $CFLAGS $COMMON_FLAGS -x c++ - -o /dev/null 2>/dev/null <<EOF- #include <bzlib.h>- int main() {}-EOF- if [ "$?" = 0 ]; then- COMMON_FLAGS="$COMMON_FLAGS -DBZIP2"- PLATFORM_LDFLAGS="$PLATFORM_LDFLAGS -lbz2"- JAVA_LDFLAGS="$JAVA_LDFLAGS -lbz2"- fi-- # Test whether lz4 library is installed- $CXX $CFLAGS $COMMON_FLAGS -x c++ - -o /dev/null 2>/dev/null <<EOF- #include <lz4.h>- #include <lz4hc.h>- int main() {}-EOF- if [ "$?" = 0 ]; then- COMMON_FLAGS="$COMMON_FLAGS -DLZ4"- PLATFORM_LDFLAGS="$PLATFORM_LDFLAGS -llz4"- JAVA_LDFLAGS="$JAVA_LDFLAGS -llz4"- fi-- # Test whether zstd library is installed- $CXX $CFLAGS $COMMON_FLAGS -x c++ - -o /dev/null 2>/dev/null <<EOF- #include <zstd.h>- int main() {}-EOF- if [ "$?" = 0 ]; then- COMMON_FLAGS="$COMMON_FLAGS -DZSTD"- PLATFORM_LDFLAGS="$PLATFORM_LDFLAGS -lzstd"- JAVA_LDFLAGS="$JAVA_LDFLAGS -lzstd"- fi-- # Test whether numa is available- $CXX $CFLAGS -x c++ - -o /dev/null -lnuma 2>/dev/null <<EOF- #include <numa.h>- #include <numaif.h>- int main() {}-EOF- if [ "$?" = 0 ]; then- COMMON_FLAGS="$COMMON_FLAGS -DNUMA"- PLATFORM_LDFLAGS="$PLATFORM_LDFLAGS -lnuma"- JAVA_LDFLAGS="$JAVA_LDFLAGS -lnuma"- fi-- # Test whether tbb is available- $CXX $CFLAGS $LDFLAGS -x c++ - -o /dev/null -ltbb 2>/dev/null <<EOF- #include <tbb/tbb.h>- int main() {}-EOF- if [ "$?" = 0 ]; then- COMMON_FLAGS="$COMMON_FLAGS -DTBB"- PLATFORM_LDFLAGS="$PLATFORM_LDFLAGS -ltbb"- JAVA_LDFLAGS="$JAVA_LDFLAGS -ltbb"- fi-- # Test whether jemalloc is available- if echo 'int main() {}' | $CXX $CFLAGS -x c++ - -o /dev/null -ljemalloc \- 2>/dev/null; then- PLATFORM_LDFLAGS="$PLATFORM_LDFLAGS -ljemalloc"- JAVA_LDFLAGS="$JAVA_LDFLAGS -ljemalloc"- JEMALLOC=1- else- # jemalloc is not available. Let's try tcmalloc- if echo 'int main() {}' | $CXX $CFLAGS -x c++ - -o /dev/null \- -ltcmalloc 2>/dev/null; then- PLATFORM_LDFLAGS="$PLATFORM_LDFLAGS -ltcmalloc"- JAVA_LDFLAGS="$JAVA_LDFLAGS -ltcmalloc"- fi- fi-# Test whether malloc_usable_size is available$CXX $CFLAGS -x c++ - -o /dev/null 2>/dev/null <<EOF#include <malloc.h>@@ -346,33 +250,6 @@ EOFCOMMON_FLAGS="$COMMON_FLAGS -DROCKSDB_PTHREAD_ADAPTIVE_MUTEX"fi- # Test whether backtrace is available- $CXX $CFLAGS -x c++ - -o /dev/null 2>/dev/null <<EOF- #include <execinfo.h>>- int main() {- void* frames[1];- backtrace_symbols(frames, backtrace(frames, 1));- return 0;- }-EOF- if [ "$?" = 0 ]; then- COMMON_FLAGS="$COMMON_FLAGS -DROCKSDB_BACKTRACE"- else- # Test whether execinfo library is installed- $CXX $CFLAGS -lexecinfo -x c++ - -o /dev/null 2>/dev/null <<EOF- #include <execinfo.h>- int main() {- void* frames[1];- backtrace_symbols(frames, backtrace(frames, 1));- }-EOF- if [ "$?" = 0 ]; then- COMMON_FLAGS="$COMMON_FLAGS -DROCKSDB_BACKTRACE"- PLATFORM_LDFLAGS="$PLATFORM_LDFLAGS -lexecinfo"- JAVA_LDFLAGS="$JAVA_LDFLAGS -lexecinfo"- fi- fi-# Test if -pg is supported$CXX $CFLAGS -pg -x c++ - -o /dev/null 2>/dev/null <<EOFint main() {diff --git a/docker-build.sh b/docker-build.shnew file mode 100755index 00000000..02df235a--- /dev/null+++ b/docker-build.sh@@ -0,0 +1,17 @@+#!/bin/sh+set -o xtrace++PROJECT_DIR=`dirname $0`++if [ "$1" = "--clean" ]; then+ rm -rf $PROJECT_DIR/build+ mkdir $PROJECT_DIR/build++ # rm -rf $PROJECT_DIR/external+ # DOCKER_NO_CACHE=" --no-cache"+fi++docker build --file $PROJECT_DIR/docker/development.Dockerfile --tag wreally/typesense-development $DOCKER_NO_CACHE $PROJECT_DIR/docker++./dockcross --image wreally/typesense-development cmake -H$PROJECT_DIR -B$PROJECT_DIR/build+./dockcross --image wreally/typesense-development make -C $PROJECT_DIR/build\ No newline at end of filediff --git a/docker/development.Dockerfile b/docker/development.Dockerfileindex d0f81329..33e47d69 100644--- a/docker/development.Dockerfile+++ b/docker/development.Dockerfile@@ -1,4 +1,10 @@FROM dockcross/linux-x64-RUN apt-get update && apt-get install -y zlib1g-dev \- libuv0.10-dev+# Required for OpenSSL 1.0.2++RUN echo "deb http://ftp.debian.org/debian jessie-backports main" > /etc/apt/sources.list.d/jessie-backport.list+RUN apt-get update++RUN apt-get install -y -t jessie-backports openssl+RUN apt-get install -y -t jessie-backports libssl-dev+RUN apt-get install -y zlib1g-dev+RUN apt-get install -y libsnappy-dev\ No newline at end of filediff --git a/include/store.h b/include/store.hindex 05cb4489..fc58e821 100644--- a/include/store.h+++ b/include/store.h@@ -51,6 +51,9 @@ public:// open DBrocksdb::Status s = rocksdb::DB::Open(options, state_dir_path, &db);+ if(!s.ok()) {+ std::cerr << s.ToString() << std::endl;+ }assert(s.ok());}diff --git a/test/art_test.cpp b/test/art_test.cppindex dd8f5704..ecdbdcd7 100644--- a/test/art_test.cpp+++ b/test/art_test.cpp@@ -6,6 +6,9 @@#include <gtest/gtest.h>#include <art.h>+#define words_file_path std::string(std::string(ROOT_DIR)+"/build/test_resources/words.txt").c_str()+#define uuid_file_path std::string(std::string(ROOT_DIR)+"/build/test_resources/uuid.txt").c_str()+art_document get_document(uint32_t id) {art_document document;document.score = (uint16_t) id;@@ -35,8 +38,7 @@ TEST(ArtTest, test_art_insert) {size_t len;char buf[512];FILE *f;- char *path_to_resources = std::getenv("TEST_RESOURCES");- f = fopen("/tmp/typesense_test/words.txt", "r");+ f = fopen(words_file_path, "r");uintptr_t line = 1;while (fgets(buf, sizeof buf, f)) {@@ -116,7 +118,7 @@ TEST(ArtTest, test_art_insert_search) {int len;char buf[512];- FILE *f = fopen("/tmp/typesense_test/words.txt", "r");+ FILE *f = fopen(words_file_path, "r");uintptr_t line = 1;while (fgets(buf, sizeof buf, f)) {@@ -161,7 +163,7 @@ TEST(ArtTest, test_art_insert_delete) {int len;char buf[512];- FILE *f = fopen("/tmp/typesense_test/words.txt", "r");+ FILE *f = fopen(words_file_path, "r");uintptr_t line = 1, nlines;while (fgets(buf, sizeof buf, f)) {@@ -223,7 +225,7 @@ TEST(ArtTest, test_art_insert_iter) {int len;char buf[512];- FILE *f = fopen("/tmp/typesense_test/words.txt", "r");+ FILE *f = fopen(words_file_path, "r");uint64_t xor_mask = 0;uintptr_t line = 1, nlines;@@ -389,7 +391,7 @@ TEST(ArtTest, test_art_insert_search_uuid) {int len;char buf[512];- FILE *f = fopen("/tmp/typesense_test/uuid.txt", "r");+ FILE *f = fopen(uuid_file_path, "r");uintptr_t line = 1;while (fgets(buf, sizeof buf, f)) {@@ -613,7 +615,7 @@ TEST(ArtTest, test_art_fuzzy_search) {int len;char buf[512];- FILE *f = fopen("/tmp/typesense_test/words.txt", "r");+ FILE *f = fopen(words_file_path, "r");uint64_t xor_mask = 0;uintptr_t line = 1, nlines;diff --git a/test/topster_test.cpp b/test/topster_test.cppindex cddd3174..582a159e 100644--- a/test/topster_test.cpp+++ b/test/topster_test.cpp@@ -28,7 +28,7 @@ TEST(TopsterTest, StoreMaxValuesWithoutRepetition) {topster.sort();- std::vector<uint64_t> ids = {4, 1, 8, 5, 9};+ std::vector<uint64_t> ids = {4, 1, 5, 8, 9};for(int i = 0; i < topster.size; i++) {EXPECT_EQ(ids[i], topster.getKeyAt(i));