Skip to content

Open Source alternative to Algolia + Pinecone and an Easier-to-Use alternative to ElasticSearch ⚡ 🔍 ✨ Fast, typo tolerant, in-memory fuzzy Search Engine for building delightful search experiences

Patch RocksDB build to ensure that it links against only specific libraries.

Kishore Nallan8850f8b16969
diff --git a/CMakeLists.txt b/CMakeLists.txt
index eb1343d7..0330e582 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -5,6 +5,7 @@ set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -std=c++11 -O2")
set(CMAKE_CXX_FLAGS_DEBUG "${CMAKE_CXX_FLAGS_DEBUG} -std=c++11 -O0")
# set(CMAKE_EXE_LINKER_FLAGS "--verbose")
set(DEP_ROOT_DIR ${CMAKE_SOURCE_DIR}/external-${CMAKE_SYSTEM_NAME})
+set(CMAKE_MODULE_PATH ${CMAKE_MODULE_PATH} "${CMAKE_SOURCE_DIR}/cmake/Modules/")
if(NOT EXISTS ${DEP_ROOT_DIR})
file(MAKE_DIRECTORY ${DEP_ROOT_DIR})
@@ -13,10 +14,35 @@ endif()
if (APPLE)
add_definitions(-DGTEST_USE_OWN_TR1_TUPLE)
add_definitions(-D__GLIBCXX__)
- set(ENV{OPENSSL_ROOT_DIR} /usr/local/opt/openssl) # For Brew
+
+ # For Brew
+ set(ENV{OPENSSL_ROOT_DIR} /usr/local/opt/openssl)
+ set(SNAPPY_ROOT_DIR /usr/local/opt/snappy)
+ set(ZLIB_ROOT /usr/local/opt/zlib)
endif (APPLE)
-FIND_PACKAGE(OpenSSL REQUIRED)
+# Ensure that we only use the static versions of libraries to link against
+# You can't statically link a shared library (or dynamically link a static library)
+IF(WIN32 OR MSVC)
+ SET(CMAKE_FIND_LIBRARY_SUFFIXES .lib .a)
+ELSE()
+ SET(CMAKE_FIND_LIBRARY_SUFFIXES .a)
+ENDIF()
+
+function(find_static_library LIB_NAME OUT)
+ find_library(FOUND_${LIB_NAME} ${LIB_NAME})
+ message("FINDING: ${LIB_NAME} and got ${FOUND_${LIB_NAME}}")
+
+ if(NOT FOUND_${LIB_NAME})
+ message(FATAL_ERROR "Unable to find static library ${LIB_NAME}")
+ endif()
+
+ set(${OUT} ${FOUND_${LIB_NAME}} PARENT_SCOPE)
+endfunction()
+
+FIND_PACKAGE(OpenSSL 1.0.2 REQUIRED)
+FIND_PACKAGE(Snappy REQUIRED)
+FIND_PACKAGE(ZLIB REQUIRED)
include(cmake/For.cmake)
include(cmake/H2O.cmake)
@@ -42,15 +68,15 @@ link_directories(${DEP_ROOT_DIR}/${ROCKSDB_NAME})
add_executable(typesense-server ${SRC_FILES} src/main/server.cpp)
add_executable(search ${SRC_FILES} src/main/main.cpp)
add_executable(benchmark ${SRC_FILES} src/main/benchmark.cpp)
-add_executable(typesense_test test/array_test.cpp test/sorted_array_test.cpp test/art_test.cpp
- test/collection_test.cpp test/collection_manager_test.cpp test/topster_test.cpp ${HEADER_FILES} ${SRC_FILES})
+add_executable(typesense_test ${SRC_FILES} test/array_test.cpp test/sorted_array_test.cpp test/art_test.cpp
+ test/collection_test.cpp test/collection_manager_test.cpp test/topster_test.cpp)
target_compile_definitions(typesense-server PRIVATE ROOT_DIR="${CMAKE_SOURCE_DIR}/")
target_compile_definitions(search PRIVATE ROOT_DIR="${CMAKE_SOURCE_DIR}/")
target_compile_definitions(benchmark PRIVATE ROOT_DIR="${CMAKE_SOURCE_DIR}/")
target_compile_definitions(typesense_test PRIVATE ROOT_DIR="${CMAKE_SOURCE_DIR}/")
-target_link_libraries(typesense-server for h2o-evloop pthread rocksdb ${OPENSSL_LIBRARIES})
-target_link_libraries(search for pthread rocksdb)
-target_link_libraries(benchmark for pthread rocksdb)
-target_link_libraries(typesense_test pthread for rocksdb gtest gtest_main)
+target_link_libraries(typesense-server for h2o-evloop pthread rocksdb ${SNAPPY_LIBRARIES} ${ZLIB_LIBRARIES} ${OPENSSL_LIBRARIES} dl)
+target_link_libraries(search for pthread rocksdb ${SNAPPY_LIBRARIES} ${ZLIB_LIBRARIES})
+target_link_libraries(benchmark for pthread rocksdb ${SNAPPY_LIBRARIES} ${ZLIB_LIBRARIES})
+target_link_libraries(typesense_test pthread for rocksdb ${SNAPPY_LIBRARIES} ${ZLIB_LIBRARIES} gtest gtest_main)
diff --git a/build.sh b/build.sh
index 02df235a..5be536bb 100755
--- a/build.sh
+++ b/build.sh
@@ -1,17 +1,10 @@
#!/bin/sh
-set -o xtrace
-
PROJECT_DIR=`dirname $0`
if [ "$1" = "--clean" ]; then
rm -rf $PROJECT_DIR/build
mkdir $PROJECT_DIR/build
-
- # rm -rf $PROJECT_DIR/external
- # DOCKER_NO_CACHE=" --no-cache"
fi
-docker build --file $PROJECT_DIR/docker/development.Dockerfile --tag wreally/typesense-development $DOCKER_NO_CACHE $PROJECT_DIR/docker
-
-./dockcross --image wreally/typesense-development cmake -H$PROJECT_DIR -B$PROJECT_DIR/build
-./dockcross --image wreally/typesense-development make -C $PROJECT_DIR/build
\ No newline at end of file
+cmake -H$PROJECT_DIR -B$PROJECT_DIR/build
+make -C $PROJECT_DIR/build
\ No newline at end of file
diff --git a/cmake/Modules/FindSnappy.cmake b/cmake/Modules/FindSnappy.cmake
new file mode 100644
index 00000000..99b43c6b
--- /dev/null
+++ b/cmake/Modules/FindSnappy.cmake
@@ -0,0 +1,54 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+# Tries to find Snappy headers and libraries.
+#
+# Usage of this module as follows:
+#
+# find_package(Snappy)
+#
+# Variables used by this module, they can change the default behaviour and need
+# to be set before calling find_package:
+#
+# SNAPPY_ROOT_DIR Set this variable to the root installation of
+# Snappy if the module has problems finding
+# the proper installation path.
+#
+# Variables defined by this module:
+#
+# SNAPPY_FOUND System has Snappy libs/headers
+# SNAPPY_LIBRARIES The Snappy libraries
+# SNAPPY_INCLUDE_DIR The location of Snappy headers
+
+find_path(SNAPPY_INCLUDE_DIR
+ NAMES snappy.h
+ HINTS ${SNAPPY_ROOT_DIR}/include)
+
+find_library(SNAPPY_LIBRARIES
+ NAMES snappy
+ HINTS ${SNAPPY_ROOT_DIR}/lib)
+
+include(FindPackageHandleStandardArgs)
+find_package_handle_standard_args(Snappy DEFAULT_MSG
+ SNAPPY_LIBRARIES
+ SNAPPY_INCLUDE_DIR)
+
+mark_as_advanced(
+ SNAPPY_ROOT_DIR
+ SNAPPY_LIBRARIES
+ SNAPPY_INCLUDE_DIR)
\ No newline at end of file
diff --git a/cmake/RocksDB.cmake b/cmake/RocksDB.cmake
index 53e31f1f..b026df18 100644
--- a/cmake/RocksDB.cmake
+++ b/cmake/RocksDB.cmake
@@ -1,6 +1,6 @@
# Download and build RocksDB
-set(ROCKSDB_VERSION 4.11.2)
+set(ROCKSDB_VERSION 4.13.5)
set(ROCKSDB_NAME rocksdb-${ROCKSDB_VERSION})
set(ROCKSDB_TAR_PATH ${DEP_ROOT_DIR}/${ROCKSDB_NAME}.tar.gz)
@@ -14,9 +14,13 @@ if(NOT EXISTS ${DEP_ROOT_DIR}/${ROCKSDB_NAME})
execute_process(COMMAND ${CMAKE_COMMAND} -E tar xvzf ${ROCKSDB_TAR_PATH} WORKING_DIRECTORY ${DEP_ROOT_DIR}/)
endif()
-if(NOT EXISTS ${DEP_ROOT_DIR}/${ROCKSDB_NAME}/librocksdb.so)
+# Use patched build detect platform script to control what libraries RocksDB links against
+file(COPY ${CMAKE_SOURCE_DIR}/cmake/patches/build_detect_platform DESTINATION
+ ${DEP_ROOT_DIR}/${ROCKSDB_NAME}/build_tools)
+
+if(NOT EXISTS ${DEP_ROOT_DIR}/${ROCKSDB_NAME}/librocksdb.a)
message("Building ${ROCKSDB_NAME} locally...")
- execute_process(COMMAND make "shared_lib" WORKING_DIRECTORY ${DEP_ROOT_DIR}/${ROCKSDB_NAME}/
+ execute_process(COMMAND make "static_lib" WORKING_DIRECTORY ${DEP_ROOT_DIR}/${ROCKSDB_NAME}/
RESULT_VARIABLE ROCKSDB_BUILD)
if(NOT ROCKSDB_BUILD EQUAL 0)
message(FATAL_ERROR "${ROCKSDB_NAME} build failed!")
diff --git a/cmake/TestResources.cmake b/cmake/TestResources.cmake
index 761917b6..1d1c1284 100644
--- a/cmake/TestResources.cmake
+++ b/cmake/TestResources.cmake
@@ -1,11 +1,12 @@
# Download test resources
set(ART_VERSION bbbf588bca55bce095538ee8ca8b422904baebc5)
-set(ART_WORDS_PATH /tmp/typesense_test/words.txt)
-set(ART_UUID_PATH /tmp/typesense_test/uuid.txt)
+set(TEST_RESOURCES_DIR ${CMAKE_SOURCE_DIR}/build/test_resources)
+set(ART_WORDS_PATH ${TEST_RESOURCES_DIR}/words.txt)
+set(ART_UUID_PATH ${TEST_RESOURCES_DIR}/uuid.txt)
-if(NOT EXISTS /tmp/typesense_test)
- file(MAKE_DIRECTORY /tmp/typesense_test)
+if(NOT EXISTS ${TEST_RESOURCES_DIR})
+ file(MAKE_DIRECTORY ${TEST_RESOURCES_DIR})
endif()
if(NOT EXISTS ${ART_WORDS_PATH})
diff --git a/cmake/patches/build_detect_platform b/cmake/patches/build_detect_platform
index 7fdeeba1..dcc3a06e 100755
--- a/cmake/patches/build_detect_platform
+++ b/cmake/patches/build_detect_platform
@@ -215,30 +215,6 @@ EOF
JAVA_LDFLAGS="$JAVA_LDFLAGS -lsnappy"
fi
- # Test whether gflags library is installed
- # http://gflags.github.io/gflags/
- # check if the namespace is gflags
- $CXX $CFLAGS -x c++ - -o /dev/null 2>/dev/null << EOF
- #include <gflags/gflags.h>
- using namespace gflags;
- int main() {}
-EOF
- if [ "$?" = 0 ]; then
- COMMON_FLAGS="$COMMON_FLAGS -DGFLAGS=gflags"
- PLATFORM_LDFLAGS="$PLATFORM_LDFLAGS -lgflags"
- else
- # check if namespace is google
- $CXX $CFLAGS -x c++ - -o /dev/null 2>/dev/null << EOF
- #include <gflags/gflags.h>
- using namespace google;
- int main() {}
-EOF
- if [ "$?" = 0 ]; then
- COMMON_FLAGS="$COMMON_FLAGS -DGFLAGS=google"
- PLATFORM_LDFLAGS="$PLATFORM_LDFLAGS -lgflags"
- fi
- fi
-
# Test whether zlib library is installed
$CXX $CFLAGS $COMMON_FLAGS -x c++ - -o /dev/null 2>/dev/null <<EOF
#include <zlib.h>
@@ -250,78 +226,6 @@ EOF
JAVA_LDFLAGS="$JAVA_LDFLAGS -lz"
fi
- # Test whether bzip library is installed
- $CXX $CFLAGS $COMMON_FLAGS -x c++ - -o /dev/null 2>/dev/null <<EOF
- #include <bzlib.h>
- int main() {}
-EOF
- if [ "$?" = 0 ]; then
- COMMON_FLAGS="$COMMON_FLAGS -DBZIP2"
- PLATFORM_LDFLAGS="$PLATFORM_LDFLAGS -lbz2"
- JAVA_LDFLAGS="$JAVA_LDFLAGS -lbz2"
- fi
-
- # Test whether lz4 library is installed
- $CXX $CFLAGS $COMMON_FLAGS -x c++ - -o /dev/null 2>/dev/null <<EOF
- #include <lz4.h>
- #include <lz4hc.h>
- int main() {}
-EOF
- if [ "$?" = 0 ]; then
- COMMON_FLAGS="$COMMON_FLAGS -DLZ4"
- PLATFORM_LDFLAGS="$PLATFORM_LDFLAGS -llz4"
- JAVA_LDFLAGS="$JAVA_LDFLAGS -llz4"
- fi
-
- # Test whether zstd library is installed
- $CXX $CFLAGS $COMMON_FLAGS -x c++ - -o /dev/null 2>/dev/null <<EOF
- #include <zstd.h>
- int main() {}
-EOF
- if [ "$?" = 0 ]; then
- COMMON_FLAGS="$COMMON_FLAGS -DZSTD"
- PLATFORM_LDFLAGS="$PLATFORM_LDFLAGS -lzstd"
- JAVA_LDFLAGS="$JAVA_LDFLAGS -lzstd"
- fi
-
- # Test whether numa is available
- $CXX $CFLAGS -x c++ - -o /dev/null -lnuma 2>/dev/null <<EOF
- #include <numa.h>
- #include <numaif.h>
- int main() {}
-EOF
- if [ "$?" = 0 ]; then
- COMMON_FLAGS="$COMMON_FLAGS -DNUMA"
- PLATFORM_LDFLAGS="$PLATFORM_LDFLAGS -lnuma"
- JAVA_LDFLAGS="$JAVA_LDFLAGS -lnuma"
- fi
-
- # Test whether tbb is available
- $CXX $CFLAGS $LDFLAGS -x c++ - -o /dev/null -ltbb 2>/dev/null <<EOF
- #include <tbb/tbb.h>
- int main() {}
-EOF
- if [ "$?" = 0 ]; then
- COMMON_FLAGS="$COMMON_FLAGS -DTBB"
- PLATFORM_LDFLAGS="$PLATFORM_LDFLAGS -ltbb"
- JAVA_LDFLAGS="$JAVA_LDFLAGS -ltbb"
- fi
-
- # Test whether jemalloc is available
- if echo 'int main() {}' | $CXX $CFLAGS -x c++ - -o /dev/null -ljemalloc \
- 2>/dev/null; then
- PLATFORM_LDFLAGS="$PLATFORM_LDFLAGS -ljemalloc"
- JAVA_LDFLAGS="$JAVA_LDFLAGS -ljemalloc"
- JEMALLOC=1
- else
- # jemalloc is not available. Let's try tcmalloc
- if echo 'int main() {}' | $CXX $CFLAGS -x c++ - -o /dev/null \
- -ltcmalloc 2>/dev/null; then
- PLATFORM_LDFLAGS="$PLATFORM_LDFLAGS -ltcmalloc"
- JAVA_LDFLAGS="$JAVA_LDFLAGS -ltcmalloc"
- fi
- fi
-
# Test whether malloc_usable_size is available
$CXX $CFLAGS -x c++ - -o /dev/null 2>/dev/null <<EOF
#include <malloc.h>
@@ -346,33 +250,6 @@ EOF
COMMON_FLAGS="$COMMON_FLAGS -DROCKSDB_PTHREAD_ADAPTIVE_MUTEX"
fi
- # Test whether backtrace is available
- $CXX $CFLAGS -x c++ - -o /dev/null 2>/dev/null <<EOF
- #include <execinfo.h>>
- int main() {
- void* frames[1];
- backtrace_symbols(frames, backtrace(frames, 1));
- return 0;
- }
-EOF
- if [ "$?" = 0 ]; then
- COMMON_FLAGS="$COMMON_FLAGS -DROCKSDB_BACKTRACE"
- else
- # Test whether execinfo library is installed
- $CXX $CFLAGS -lexecinfo -x c++ - -o /dev/null 2>/dev/null <<EOF
- #include <execinfo.h>
- int main() {
- void* frames[1];
- backtrace_symbols(frames, backtrace(frames, 1));
- }
-EOF
- if [ "$?" = 0 ]; then
- COMMON_FLAGS="$COMMON_FLAGS -DROCKSDB_BACKTRACE"
- PLATFORM_LDFLAGS="$PLATFORM_LDFLAGS -lexecinfo"
- JAVA_LDFLAGS="$JAVA_LDFLAGS -lexecinfo"
- fi
- fi
-
# Test if -pg is supported
$CXX $CFLAGS -pg -x c++ - -o /dev/null 2>/dev/null <<EOF
int main() {
diff --git a/docker-build.sh b/docker-build.sh
new file mode 100755
index 00000000..02df235a
--- /dev/null
+++ b/docker-build.sh
@@ -0,0 +1,17 @@
+#!/bin/sh
+set -o xtrace
+
+PROJECT_DIR=`dirname $0`
+
+if [ "$1" = "--clean" ]; then
+ rm -rf $PROJECT_DIR/build
+ mkdir $PROJECT_DIR/build
+
+ # rm -rf $PROJECT_DIR/external
+ # DOCKER_NO_CACHE=" --no-cache"
+fi
+
+docker build --file $PROJECT_DIR/docker/development.Dockerfile --tag wreally/typesense-development $DOCKER_NO_CACHE $PROJECT_DIR/docker
+
+./dockcross --image wreally/typesense-development cmake -H$PROJECT_DIR -B$PROJECT_DIR/build
+./dockcross --image wreally/typesense-development make -C $PROJECT_DIR/build
\ No newline at end of file
diff --git a/docker/development.Dockerfile b/docker/development.Dockerfile
index d0f81329..33e47d69 100644
--- a/docker/development.Dockerfile
+++ b/docker/development.Dockerfile
@@ -1,4 +1,10 @@
FROM dockcross/linux-x64
-RUN apt-get update && apt-get install -y zlib1g-dev \
- libuv0.10-dev
+# Required for OpenSSL 1.0.2+
+RUN echo "deb http://ftp.debian.org/debian jessie-backports main" > /etc/apt/sources.list.d/jessie-backport.list
+RUN apt-get update
+
+RUN apt-get install -y -t jessie-backports openssl
+RUN apt-get install -y -t jessie-backports libssl-dev
+RUN apt-get install -y zlib1g-dev
+RUN apt-get install -y libsnappy-dev
\ No newline at end of file
diff --git a/include/store.h b/include/store.h
index 05cb4489..fc58e821 100644
--- a/include/store.h
+++ b/include/store.h
@@ -51,6 +51,9 @@ public:
// open DB
rocksdb::Status s = rocksdb::DB::Open(options, state_dir_path, &db);
+ if(!s.ok()) {
+ std::cerr << s.ToString() << std::endl;
+ }
assert(s.ok());
}
diff --git a/test/art_test.cpp b/test/art_test.cpp
index dd8f5704..ecdbdcd7 100644
--- a/test/art_test.cpp
+++ b/test/art_test.cpp
@@ -6,6 +6,9 @@
#include <gtest/gtest.h>
#include <art.h>
+#define words_file_path std::string(std::string(ROOT_DIR)+"/build/test_resources/words.txt").c_str()
+#define uuid_file_path std::string(std::string(ROOT_DIR)+"/build/test_resources/uuid.txt").c_str()
+
art_document get_document(uint32_t id) {
art_document document;
document.score = (uint16_t) id;
@@ -35,8 +38,7 @@ TEST(ArtTest, test_art_insert) {
size_t len;
char buf[512];
FILE *f;
- char *path_to_resources = std::getenv("TEST_RESOURCES");
- f = fopen("/tmp/typesense_test/words.txt", "r");
+ f = fopen(words_file_path, "r");
uintptr_t line = 1;
while (fgets(buf, sizeof buf, f)) {
@@ -116,7 +118,7 @@ TEST(ArtTest, test_art_insert_search) {
int len;
char buf[512];
- FILE *f = fopen("/tmp/typesense_test/words.txt", "r");
+ FILE *f = fopen(words_file_path, "r");
uintptr_t line = 1;
while (fgets(buf, sizeof buf, f)) {
@@ -161,7 +163,7 @@ TEST(ArtTest, test_art_insert_delete) {
int len;
char buf[512];
- FILE *f = fopen("/tmp/typesense_test/words.txt", "r");
+ FILE *f = fopen(words_file_path, "r");
uintptr_t line = 1, nlines;
while (fgets(buf, sizeof buf, f)) {
@@ -223,7 +225,7 @@ TEST(ArtTest, test_art_insert_iter) {
int len;
char buf[512];
- FILE *f = fopen("/tmp/typesense_test/words.txt", "r");
+ FILE *f = fopen(words_file_path, "r");
uint64_t xor_mask = 0;
uintptr_t line = 1, nlines;
@@ -389,7 +391,7 @@ TEST(ArtTest, test_art_insert_search_uuid) {
int len;
char buf[512];
- FILE *f = fopen("/tmp/typesense_test/uuid.txt", "r");
+ FILE *f = fopen(uuid_file_path, "r");
uintptr_t line = 1;
while (fgets(buf, sizeof buf, f)) {
@@ -613,7 +615,7 @@ TEST(ArtTest, test_art_fuzzy_search) {
int len;
char buf[512];
- FILE *f = fopen("/tmp/typesense_test/words.txt", "r");
+ FILE *f = fopen(words_file_path, "r");
uint64_t xor_mask = 0;
uintptr_t line = 1, nlines;
diff --git a/test/topster_test.cpp b/test/topster_test.cpp
index cddd3174..582a159e 100644
--- a/test/topster_test.cpp
+++ b/test/topster_test.cpp
@@ -28,7 +28,7 @@ TEST(TopsterTest, StoreMaxValuesWithoutRepetition) {
topster.sort();
- std::vector<uint64_t> ids = {4, 1, 8, 5, 9};
+ std::vector<uint64_t> ids = {4, 1, 5, 8, 9};
for(int i = 0; i < topster.size; i++) {
EXPECT_EQ(ids[i], topster.getKeyAt(i));