Open Source alternative to Algolia + Pinecone and an Easier-to-Use alternative to ElasticSearch ⚡ 🔍 ✨ Fast, typo tolerant, in-memory fuzzy Search Engine for building delightful search experiences
C++ algoliadatastoreelasticsearchenterprise-searchfacetingfull-text-searchfuzzy-searchgeosearchin-memoryinstantsearchmerchandisingpineconesearchsearch-enginesemantic-searchsimilarity-searchsite-searchsynonymstypo-tolerancevector-search
Build RocksDB and H2O also as part of the build process.
Kishore Nallanee68da6f537b
diff --git a/.gitmodules b/.gitmodulesdeleted file mode 100644index 911f1f5c..00000000--- a/.gitmodules+++ /dev/null@@ -1,4 +0,0 @@-[submodule "external/libfor"]- path = external/libfor- url = [email protected]:cruppstahl/libfor.git- ignore = dirty\ No newline at end of filediff --git a/CMakeLists.txt b/CMakeLists.txtindex 65a928bb..3025c583 100644--- a/CMakeLists.txt+++ b/CMakeLists.txt@@ -8,41 +8,34 @@ if(NOT EXISTS ${CMAKE_SOURCE_DIR}/external/)file(MAKE_DIRECTORY ${CMAKE_SOURCE_DIR}/external)endif()-include(GoogleTest.cmake)-include(For.cmake)+include(cmake/GoogleTest.cmake)+include(cmake/For.cmake)+include(cmake/H2O.cmake)+include(cmake/RocksDB.cmake)if (APPLE)add_definitions(-DGTEST_USE_OWN_TR1_TUPLE)add_definitions(-D__GLIBCXX__)- include_directories(/usr/local/opt/openssl/include) # For openssl installed with brewendif (APPLE)-# FIXME: Need to use find_package() to locate the correct openssl library-FILE(GLOB HEADER_FILES include/*)include_directories(include)include_directories(${CMAKE_SOURCE_DIR}/external/${FOR_NAME})-include_directories(${CMAKE_SOURCE_DIR}/external/googletest-release-1.8.0/googletest/include)+include_directories(${CMAKE_SOURCE_DIR}/external/${GTEST_NAME}/googletest/include)+include_directories(${CMAKE_SOURCE_DIR}/external/${H2O_NAME}/include)+include_directories(${CMAKE_SOURCE_DIR}/external/${ROCKSDB_NAME}/include)-link_directories(${CMAKE_SOURCE_DIR}/external/googletest-release-1.8.0/googletest/build)+link_directories(${CMAKE_SOURCE_DIR}/external/${GTEST_NAME}/googletest/build)+link_directories(${CMAKE_SOURCE_DIR}/external/${FOR_NAME})+link_directories(${CMAKE_SOURCE_DIR}/external/${H2O_NAME}/build)+link_directories(${CMAKE_SOURCE_DIR}/external/${H2O_NAME}/build/libressl-build/lib)+link_directories(${CMAKE_SOURCE_DIR}/external/${ROCKSDB_NAME})add_executable(typesense_test test/forarray_test.cpp)add_executable(search ${HEADER_FILES} src/art.cpp src/intersection.cpp src/main.cpp src/collection.cpp)add_executable(typesense-server ${HEADER_FILES} src/art.cpp src/intersection.cpp src/collection.cpp src/server.cpp)-target_link_libraries(typesense_test- ${CMAKE_SOURCE_DIR}/external/${FOR_NAME}/libfor.a- gtest gtest_main-)--target_link_libraries(search ${CMAKE_SOURCE_DIR}/external/${FOR_NAME}/libfor.a rocksdb)-target_link_libraries(typesense-server- ${CMAKE_SOURCE_DIR}/external/${FOR_NAME}/libfor.a- curl h2o-evloop pthread rocksdb- /usr/local/opt/openssl/lib/libssl.a- /usr/local/opt/openssl/lib/libcrypto.a)--#add_dependencies(typesense_test libfor)-#add_dependencies(search libfor)-#add_dependencies(typesense-server libfor)\ No newline at end of file+target_link_libraries(typesense_test for gtest gtest_main)+target_link_libraries(search for pthread rocksdb)+target_link_libraries(typesense-server for curl h2o-evloop pthread rocksdb ssl crypto)\ No newline at end of filediff --git a/README.md b/README.mdindex dc33f90c..f2decb9b 100644--- a/README.md+++ b/README.md@@ -7,34 +7,9 @@ Typesense is an open source search engine for building a delightful search exper- **Blazing fast:** Meticulously designed and optimized for speed- **Simple and delightful:** Simple API, delightful out-of-the-box experience-## Dependencies--* [RocksDB](https://github.com/facebook/rocksdb)-* [libfor](https://github.com/cruppstahl/libfor/)-* [h2o](https://github.com/h2o/h2o)-## Building Typesense from source-First, clone the git repository with `--recursive` option.--### OS X--Install the following Homebrew dependencies:--```-brew install cmake-brew install openssl-brew install h2o-brew install rocksdb-```--Building libfor:--```-make -C external/libfor-```--Finally, build Typesense:+Please ensure that you have `cmake` available on your system. All other dependencies are built automatically.```./build.shdiff --git a/For.cmake b/cmake/For.cmakesimilarity index 100%rename from For.cmakerename to cmake/For.cmakediff --git a/GoogleTest.cmake b/cmake/GoogleTest.cmakesimilarity index 69%rename from GoogleTest.cmakerename to cmake/GoogleTest.cmakeindex b2d681fc..6b2af49e 100644--- a/GoogleTest.cmake+++ b/cmake/GoogleTest.cmake@@ -6,16 +6,16 @@ set(GTEST_TAR_PATH ${CMAKE_SOURCE_DIR}/external/${GTEST_NAME}.tar.gz)if(NOT EXISTS ${CMAKE_SOURCE_DIR}/external/${GTEST_NAME})message(STATUS "Downloading and extracting Google Test...")- file(DOWNLOAD https://github.com/google/googletest/archive/release-1.8.0.tar.gz ${GTEST_TAR_PATH})+ file(DOWNLOAD https://github.com/google/googletest/archive/release-${GTEST_VERSION}.tar.gz ${GTEST_TAR_PATH})execute_process(COMMAND ${CMAKE_COMMAND} -E tar xvzf ${GTEST_TAR_PATH} WORKING_DIRECTORY ${CMAKE_SOURCE_DIR}/external/)endif()-if(NOT EXISTS ${CMAKE_SOURCE_DIR}/external/googletest-release-1.8.0/googletest/build)+if(NOT EXISTS ${CMAKE_SOURCE_DIR}/external/googletest-release-${GTEST_VERSION}/googletest/build)message("Configuring Google Test...")- file(MAKE_DIRECTORY ${CMAKE_SOURCE_DIR}/external/googletest-release-1.8.0/googletest/build)+ file(MAKE_DIRECTORY ${CMAKE_SOURCE_DIR}/external/googletest-release-${GTEST_VERSION}/googletest/build)execute_process(COMMAND ${CMAKE_COMMAND}- "-H${CMAKE_SOURCE_DIR}/external/googletest-release-1.8.0/googletest"- "-B${CMAKE_SOURCE_DIR}/external/googletest-release-1.8.0/googletest/build"+ "-H${CMAKE_SOURCE_DIR}/external/googletest-release-${GTEST_VERSION}/googletest"+ "-B${CMAKE_SOURCE_DIR}/external/googletest-release-${GTEST_VERSION}/googletest/build"RESULT_VARIABLEGOOGLETEST_CONFIGURE)if(NOT GOOGLETEST_CONFIGURE EQUAL 0)@@ -24,7 +24,7 @@ if(NOT EXISTS ${CMAKE_SOURCE_DIR}/external/googletest-release-1.8.0/googletest/bmessage("Building Google Test locally...")execute_process(COMMAND ${CMAKE_COMMAND} --build- "${CMAKE_SOURCE_DIR}/external/googletest-release-1.8.0/googletest/build"+ "${CMAKE_SOURCE_DIR}/external/googletest-release-${GTEST_VERSION}/googletest/build"RESULT_VARIABLEGOOGLETEST_BUILD)if(NOT GOOGLETEST_BUILD EQUAL 0)diff --git a/cmake/H2O.cmake b/cmake/H2O.cmakenew file mode 100644index 00000000..30731351--- /dev/null+++ b/cmake/H2O.cmake@@ -0,0 +1,34 @@+# Download and build gtest++set(H2O_VERSION 2.0.4)+set(H2O_NAME h2o-${H2O_VERSION})+set(H2O_TAR_PATH ${CMAKE_SOURCE_DIR}/external/${H2O_NAME}.tar.gz)++if(NOT EXISTS ${CMAKE_SOURCE_DIR}/external/${H2O_NAME})+ message(STATUS "Downloading and extracting ${H2O_NAME}...")+ file(DOWNLOAD https://github.com/h2o/h2o/archive/v2.0.4.tar.gz ${H2O_TAR_PATH})+ execute_process(COMMAND ${CMAKE_COMMAND} -E tar xvzf ${H2O_TAR_PATH} WORKING_DIRECTORY ${CMAKE_SOURCE_DIR}/external/)+endif()++if(NOT EXISTS ${CMAKE_SOURCE_DIR}/external/${H2O_NAME}/build)+ message("Configuring ${H2O_NAME}...")+ file(MAKE_DIRECTORY ${CMAKE_SOURCE_DIR}/external/${H2O_NAME}/build)+ execute_process(COMMAND ${CMAKE_COMMAND}+ "-DWITH_BUNDLED_SSL=on"+ "-H${CMAKE_SOURCE_DIR}/external/${H2O_NAME}"+ "-B${CMAKE_SOURCE_DIR}/external/${H2O_NAME}/build"+ RESULT_VARIABLE+ H2O_CONFIGURE)+ if(NOT H2O_CONFIGURE EQUAL 0)+ message(FATAL_ERROR "${H2O_NAME} configure failed!")+ endif()++ message("Building ${H2O_NAME} locally...")+ execute_process(COMMAND ${CMAKE_COMMAND} --build+ "${CMAKE_SOURCE_DIR}/external/${H2O_NAME}/build"+ RESULT_VARIABLE+ H2O_BUILD)+ if(NOT H2O_BUILD EQUAL 0)+ message(FATAL_ERROR "${H2O_NAME} build failed!")+ endif()+endif()\ No newline at end of filediff --git a/cmake/RocksDB.cmake b/cmake/RocksDB.cmakenew file mode 100644index 00000000..4936e200--- /dev/null+++ b/cmake/RocksDB.cmake@@ -0,0 +1,16 @@+# Download and build RocksDB++set(ROCKSDB_VERSION 4.11.2)+set(ROCKSDB_NAME rocksdb-${ROCKSDB_VERSION})+set(ROCKSDB_TAR_PATH ${CMAKE_SOURCE_DIR}/external/${ROCKSDB_NAME}.tar.gz)++if(NOT EXISTS ${CMAKE_SOURCE_DIR}/external/${ROCKSDB_NAME})+ message(STATUS "Downloading and extracting ${ROCKSDB_NAME}...")+ file(DOWNLOAD https://github.com/facebook/rocksdb/archive/v${ROCKSDB_VERSION}.tar.gz ${ROCKSDB_TAR_PATH})+ execute_process(COMMAND ${CMAKE_COMMAND} -E tar xvzf ${ROCKSDB_TAR_PATH} WORKING_DIRECTORY ${CMAKE_SOURCE_DIR}/external/)+endif()++if(NOT EXISTS ${CMAKE_SOURCE_DIR}/external/${ROCKSDB_NAME}/libfor.a)+ message("Building ${ROCKSDB_NAME} locally...")+ execute_process(COMMAND make "static_lib" WORKING_DIRECTORY ${CMAKE_SOURCE_DIR}/external/${ROCKSDB_NAME}/)+endif()\ No newline at end of filediff --git a/include/forarray.h b/include/forarray.hindex 90121f82..d6821a49 100644--- a/include/forarray.h+++ b/include/forarray.h@@ -3,6 +3,7 @@#include <stdio.h>#include <cstdlib>#include <for.h>+#include <cstring>#include <limits>#include <iostream>@@ -46,7 +47,7 @@ public:}static inline uint32_t required_bits(const uint32_t v) {- return v == 0 ? 0 : 32 - __builtin_clz(v);+ return (uint32_t) (v == 0 ? 0 : 32 - __builtin_clz(v));}uint32_t inline sorted_append_size_required(uint32_t value, uint32_t new_length) {diff --git a/src/art.cpp b/src/art.cppindex 6178cc43..d19e06df 100644--- a/src/art.cpp+++ b/src/art.cpp@@ -5,6 +5,9 @@#include <emmintrin.h>#include <assert.h>#include <art.h>+#include <functional>+#include <chrono>+#include <algorithm>#include <iostream>#include <limits>#include <queue>diff --git a/src/collection.cpp b/src/collection.cppindex 4cb99438..c8a32c5c 100644--- a/src/collection.cpp+++ b/src/collection.cpp@@ -2,6 +2,7 @@#include <iostream>#include <numeric>+#include <chrono>#include <topster.h>#include <intersection.h>#include <match_score.h>diff --git a/src/main.cpp b/src/main.cppindex 3f92b37c..aae50e08 100644--- a/src/main.cpp+++ b/src/main.cpp@@ -3,6 +3,7 @@#include <fstream>#include <vector>#include <numeric>+#include <chrono>#include <art.h>#include <unordered_map>#include "string_utils.h"