Open Source alternative to Algolia + Pinecone and an Easier-to-Use alternative to ElasticSearch ⚡ 🔍 ✨ Fast, typo tolerant, in-memory fuzzy Search Engine for building delightful search experiences
C++ algoliadatastoreelasticsearchenterprise-searchfacetingfull-text-searchfuzzy-searchgeosearchin-memoryinstantsearchmerchandisingpineconesearchsearch-enginesemantic-searchsimilarity-searchsite-searchsynonymstypo-tolerancevector-search
Build lib along with executable.
Kishore Nallanea883b4beca0
diff --git a/.circleci/config.yml b/.circleci/config.ymlindex c10bb713..55d0708b 100644--- a/.circleci/config.yml+++ b/.circleci/config.yml@@ -29,4 +29,4 @@ jobs:destination: build- run:name: test- command: $PROJECT_DIR/build/typesense_test+ command: $PROJECT_DIR/build/typesense-testdiff --git a/CMakeLists.txt b/CMakeLists.txtindex e610688f..6d18acda 100644--- a/CMakeLists.txt+++ b/CMakeLists.txt@@ -6,6 +6,10 @@ set(CMAKE_CXX_FLAGS_DEBUG "${CMAKE_CXX_FLAGS_DEBUG} -Wall -Wextra -Wno-unused-paset(DEP_ROOT_DIR ${CMAKE_SOURCE_DIR}/external-${CMAKE_SYSTEM_NAME})set(CMAKE_MODULE_PATH ${CMAKE_MODULE_PATH} "${CMAKE_SOURCE_DIR}/cmake/Modules/")+# will be overridden from command line during build+set(TYPESENSE_VERSION "nightly" CACHE STRING "")+set(BUILD_DEPS "yes" CACHE STRING "")+if(NOT EXISTS ${DEP_ROOT_DIR})file(MAKE_DIRECTORY ${DEP_ROOT_DIR})endif()@@ -54,27 +58,46 @@ include_directories(${DEP_ROOT_DIR}/${H2O_NAME}/include)include_directories(${DEP_ROOT_DIR}/${ROCKSDB_NAME}/include)include_directories(${DEP_ROOT_DIR}/${G3LOG_NAME}/src)+# Write dependency include directories to a file+file(WRITE ${DEP_ROOT_DIR}/includes.txt "")+file(APPEND ${DEP_ROOT_DIR}/includes.txt "${DEP_ROOT_DIR}/${FOR_NAME}")+file(APPEND ${DEP_ROOT_DIR}/includes.txt " ${DEP_ROOT_DIR}/${H2O_NAME}/include")+file(APPEND ${DEP_ROOT_DIR}/includes.txt " ${DEP_ROOT_DIR}/${ROCKSDB_NAME}/include")+file(APPEND ${DEP_ROOT_DIR}/includes.txt " ${DEP_ROOT_DIR}/${G3LOG_NAME}/src")+link_directories(${DEP_ROOT_DIR}/${GTEST_NAME}/googletest/build)link_directories(${DEP_ROOT_DIR}/${FOR_NAME})link_directories(${DEP_ROOT_DIR}/${H2O_NAME}/build)link_directories(${DEP_ROOT_DIR}/${ROCKSDB_NAME})link_directories(${DEP_ROOT_DIR}/${G3LOG_NAME}/build)+# Write dependency libraries to a file+file(WRITE ${DEP_ROOT_DIR}/libs.txt "")+file(APPEND ${DEP_ROOT_DIR}/libs.txt "${DEP_ROOT_DIR}/${FOR_NAME}")+file(APPEND ${DEP_ROOT_DIR}/libs.txt " ${DEP_ROOT_DIR}/${H2O_NAME}/build")+file(APPEND ${DEP_ROOT_DIR}/libs.txt " ${DEP_ROOT_DIR}/${ROCKSDB_NAME}")+file(APPEND ${DEP_ROOT_DIR}/libs.txt " ${DEP_ROOT_DIR}/${G3LOG_NAME}/build")+add_executable(typesense-server ${SRC_FILES} src/main/typesense_server.cpp)add_executable(search ${SRC_FILES} src/main/main.cpp)add_executable(benchmark ${SRC_FILES} src/main/benchmark.cpp)-add_executable(typesense_test ${SRC_FILES} test/main.cpp test/array_test.cpp test/sorted_array_test.cpp test/art_test.cpp+add_executable(typesense-test ${SRC_FILES} test/main.cpp test/array_test.cpp test/sorted_array_test.cpp test/art_test.cpptest/collection_test.cpp test/collection_manager_test.cpptest/topster_test.cpp test/match_score_test.cpp test/store_test.cpp test/array_utils_test.cpptest/string_utils_test.cpp)-set(TYPESENSE_VERSION "nightly" CACHE STRING "") # will be overridden from command line during a release build+add_library(typesense-core STATIC ${SRC_FILES})target_compile_definitions(typesense-server PRIVATETYPESENSE_VERSION="${TYPESENSE_VERSION}")+target_compile_definitions(+ typesense-core PRIVATE+ TYPESENSE_VERSION="${TYPESENSE_VERSION}"+)+target_compile_definitions(benchmark PRIVATETYPESENSE_VERSION="${TYPESENSE_VERSION}"@@ -86,7 +109,7 @@ target_compile_definitions()target_compile_definitions(- typesense_test PRIVATE+ typesense-test PRIVATEROOT_DIR="${CMAKE_SOURCE_DIR}/"TYPESENSE_VERSION="${TYPESENSE_VERSION}")@@ -107,8 +130,11 @@ if(NOT APPLE)endif()set(ICU_ALL_LIBRARIES ${ICU_I18N_LIBRARIES} ${ICU_LIBRARIES} ${ICU_DATA_LIBRARIES})--target_link_libraries(typesense-server h2o-evloop for ${ICU_ALL_LIBRARIES} ${G3LOGGER_LIBRARIES} pthread ${CURL_LIBRARIES} ${ROCKSDB_LIBS} ${OPENSSL_LIBRARIES} dl ${STD_LIB})-target_link_libraries(search for ${ICU_ALL_LIBRARIES} ${G3LOGGER_LIBRARIES} pthread h2o-evloop ${CURL_LIBRARIES} ${ROCKSDB_LIBS} ${OPENSSL_LIBRARIES} dl ${STD_LIB})-target_link_libraries(benchmark for ${ICU_ALL_LIBRARIES} ${G3LOGGER_LIBRARIES} pthread ${CURL_LIBRARIES} h2o-evloop ${ROCKSDB_LIBS} ${OPENSSL_LIBRARIES} dl ${STD_LIB})-target_link_libraries(typesense_test h2o-evloop ${ICU_ALL_LIBRARIES} ${OPENSSL_LIBRARIES} pthread for ${G3LOGGER_LIBRARIES} ${ROCKSDB_LIBS} gtest gtest_main dl ${STD_LIB})+set(CORE_LIBS h2o-evloop ${CURL_LIBRARIES} for ${ICU_ALL_LIBRARIES} ${G3LOGGER_LIBRARIES} pthread+ ${ROCKSDB_LIBS} ${OPENSSL_LIBRARIES} dl ${STD_LIB})++target_link_libraries(typesense-core ${CORE_LIBS})+target_link_libraries(typesense-server ${CORE_LIBS})+target_link_libraries(search ${CORE_LIBS})+target_link_libraries(benchmark ${CORE_LIBS})+target_link_libraries(typesense-test ${CORE_LIBS} gtest gtest_main)diff --git a/TODO.md b/TODO.mdindex 7065f3e4..77542dad 100644--- a/TODO.md+++ b/TODO.md@@ -97,6 +97,7 @@- ~~Have a LOG(ERROR) level~~- ~~Handle SIGTERM which is sent when process is killed~~- ~~Use snappy compression for storage~~+- Jemalloc- Exact search- NOT operator support- Log operationsdiff --git a/build.sh b/build.shindex c4093998..75ee9dcf 100755--- a/build.sh+++ b/build.sh@@ -29,4 +29,17 @@ if [[ "$@" == *"--create-binary"* ]]; thenprintf `md5sum $PROJECT_DIR/build/typesense-server | cut -b-32` > $PROJECT_DIR/build/typesense-server.md5.txttar -cvzf $PROJECT_DIR/build/$RELEASE_NAME.tar.gz -C $PROJECT_DIR/build typesense-server typesense-server.md5.txtecho "Built binary successfully: $PROJECT_DIR/build/$RELEASE_NAME.tar.gz"-fi\ No newline at end of file+fi++if [[ "$@" == *"--package-libs"* ]]; then+ OS_FAMILY=$(echo `uname` | awk '{print tolower($0)}')+ RELEASE_NAME=typesense-server-libs-$TYPESENSE_VERSION-$OS_FAMILY-amd64+ LIBS=`cat $PROJECT_DIR/external-$SYSTEM_NAME/libs.txt`+ TAR_PATHS=""+ for lib in $LIBS; do+ TAR_PATHS="$TAR_PATHS -C $lib `ls $lib/*.a | xargs basename`"+ done++ TAR_PATHS="$TAR_PATHS -C $PROJECT_DIR/build `ls $PROJECT_DIR/build/*.a | xargs basename`"+ tar -cvzf $PROJECT_DIR/build/$RELEASE_NAME.tar.gz $TAR_PATHS+fidiff --git a/cmake/For.cmake b/cmake/For.cmakeindex c7fc962b..bc21fbfa 100644--- a/cmake/For.cmake+++ b/cmake/For.cmake@@ -14,7 +14,7 @@ if(NOT EXISTS ${DEP_ROOT_DIR}/${FOR_NAME})execute_process(COMMAND ${CMAKE_COMMAND} -E tar xzf ${FOR_TAR_PATH} WORKING_DIRECTORY ${DEP_ROOT_DIR})endif()-if(NOT EXISTS ${DEP_ROOT_DIR}/${FOR_NAME}/libfor.a)+if(NOT EXISTS ${DEP_ROOT_DIR}/${FOR_NAME}/libfor.a AND BUILD_DEPS STREQUAL "yes")message("Building libfor locally...")execute_process(COMMAND make WORKING_DIRECTORY ${DEP_ROOT_DIR}/${FOR_NAME}/RESULT_VARIABLE FOR_BUILD)diff --git a/cmake/GoogleTest.cmake b/cmake/GoogleTest.cmakeindex 6dd358a2..3986adb9 100644--- a/cmake/GoogleTest.cmake+++ b/cmake/GoogleTest.cmake@@ -26,12 +26,14 @@ if(NOT EXISTS ${DEP_ROOT_DIR}/googletest-release-${GTEST_VERSION}/googletest/buimessage(FATAL_ERROR "Google Test Configure failed!")endif()- message("Building Google Test locally...")- execute_process(COMMAND ${CMAKE_COMMAND} --build- "${DEP_ROOT_DIR}/googletest-release-${GTEST_VERSION}/googletest/build"- RESULT_VARIABLE- GOOGLETEST_BUILD)- if(NOT GOOGLETEST_BUILD EQUAL 0)- message(FATAL_ERROR "Google Test build failed!")+ if(BUILD_DEPS STREQUAL "yes")+ message("Building Google Test locally...")+ execute_process(COMMAND ${CMAKE_COMMAND} --build+ "${DEP_ROOT_DIR}/googletest-release-${GTEST_VERSION}/googletest/build"+ RESULT_VARIABLE+ GOOGLETEST_BUILD)+ if(NOT GOOGLETEST_BUILD EQUAL 0)+ message(FATAL_ERROR "Google Test build failed!")+ endif()endif()-endif()\ No newline at end of file+endif()diff --git a/cmake/H2O.cmake b/cmake/H2O.cmakeindex 056d10c5..3938ae3c 100644--- a/cmake/H2O.cmake+++ b/cmake/H2O.cmake@@ -28,12 +28,14 @@ if(NOT EXISTS ${DEP_ROOT_DIR}/${H2O_NAME}/build/h2o)message(FATAL_ERROR "${H2O_NAME} configure failed!")endif()- message("Building ${H2O_NAME} locally...")- execute_process(COMMAND ${CMAKE_COMMAND} --build- "${DEP_ROOT_DIR}/${H2O_NAME}/build"- RESULT_VARIABLE- H2O_BUILD)- if(NOT H2O_BUILD EQUAL 0)- message(FATAL_ERROR "${H2O_NAME} build failed!")+ if(BUILD_DEPS STREQUAL "yes")+ message("Building ${H2O_NAME} locally...")+ execute_process(COMMAND ${CMAKE_COMMAND} --build+ "${DEP_ROOT_DIR}/${H2O_NAME}/build"+ RESULT_VARIABLE+ H2O_BUILD)+ if(NOT H2O_BUILD EQUAL 0)+ message(FATAL_ERROR "${H2O_NAME} build failed!")+ endif()endif()-endif()\ No newline at end of file+endif()diff --git a/cmake/RocksDB.cmake b/cmake/RocksDB.cmakeindex 44fcfcbe..a93adbb6 100644--- a/cmake/RocksDB.cmake+++ b/cmake/RocksDB.cmake@@ -14,7 +14,7 @@ if(NOT EXISTS ${DEP_ROOT_DIR}/${ROCKSDB_NAME})execute_process(COMMAND ${CMAKE_COMMAND} -E tar xzf ${ROCKSDB_TAR_PATH} WORKING_DIRECTORY ${DEP_ROOT_DIR}/)endif()-if(NOT EXISTS ${DEP_ROOT_DIR}/${ROCKSDB_NAME}/librocksdb.a)+if(NOT EXISTS ${DEP_ROOT_DIR}/${ROCKSDB_NAME}/librocksdb.a AND BUILD_DEPS STREQUAL "yes")message("Building ${ROCKSDB_NAME} locally...")set(ENV{PORTABLE} 1)set(ENV{DEBUG_LEVEL} 0)@@ -34,8 +34,8 @@ if(NOT EXISTS ${DEP_ROOT_DIR}/${ROCKSDB_NAME}/librocksdb.a)execute_process(COMMAND make "clean" WORKING_DIRECTORY ${DEP_ROOT_DIR}/${ROCKSDB_NAME}/)execute_process(COMMAND make "static_lib" WORKING_DIRECTORY ${DEP_ROOT_DIR}/${ROCKSDB_NAME}/- RESULT_VARIABLE ROCKSDB_BUILD)+ RESULT_VARIABLE ROCKSDB_BUILD)if(NOT ROCKSDB_BUILD EQUAL 0)message(FATAL_ERROR "${ROCKSDB_NAME} build failed!")endif()-endif()\ No newline at end of file+endif()diff --git a/cmake/TestResources.cmake b/cmake/TestResources.cmakeindex 1d1c1284..6eac8048 100644--- a/cmake/TestResources.cmake+++ b/cmake/TestResources.cmake@@ -9,12 +9,12 @@ if(NOT EXISTS ${TEST_RESOURCES_DIR})file(MAKE_DIRECTORY ${TEST_RESOURCES_DIR})endif()-if(NOT EXISTS ${ART_WORDS_PATH})+if(NOT EXISTS ${ART_WORDS_PATH} AND BUILD_DEPS STREQUAL "yes")message(STATUS "Downloading test resource - words.txt")file(DOWNLOAD https://raw.githubusercontent.com/kishorenc/libart/${ART_VERSION}/tests/words.txt ${ART_WORDS_PATH})endif()-if(NOT EXISTS ${ART_UUID_PATH})+if(NOT EXISTS ${ART_UUID_PATH} AND BUILD_DEPS STREQUAL "yes")message(STATUS "Downloading test resource - uuid.txt")file(DOWNLOAD https://raw.githubusercontent.com/kishorenc/libart/${ART_VERSION}/tests/uuid.txt ${ART_UUID_PATH})-endif()\ No newline at end of file+endif()diff --git a/cmake/g3log.cmake b/cmake/g3log.cmakeindex e48542b8..80842f82 100644--- a/cmake/g3log.cmake+++ b/cmake/g3log.cmake@@ -30,12 +30,14 @@ if(NOT EXISTS ${DEP_ROOT_DIR}/${G3LOG_NAME}/build)message(FATAL_ERROR "G3log Configure failed!")endif()- message("Building G3log locally...")- execute_process(COMMAND ${CMAKE_COMMAND} --build- "${DEP_ROOT_DIR}/${G3LOG_NAME}/build"- RESULT_VARIABLE- G3LOG_BUILD)- if(NOT G3LOG_BUILD EQUAL 0)- message(FATAL_ERROR "G3log build failed!")+ if(BUILD_DEPS STREQUAL "yes")+ message("Building G3log locally...")+ execute_process(COMMAND ${CMAKE_COMMAND} --build+ "${DEP_ROOT_DIR}/${G3LOG_NAME}/build"+ RESULT_VARIABLE+ G3LOG_BUILD)+ if(NOT G3LOG_BUILD EQUAL 0)+ message(FATAL_ERROR "G3log build failed!")+ endif()endif()-endif()\ No newline at end of file+endif()diff --git a/docker-build.sh b/docker-build.shindex b0baa7b6..7bdb2f33 100755--- a/docker-build.sh+++ b/docker-build.sh@@ -44,4 +44,17 @@ if [[ "$@" == *"--create-binary"* ]]; thenecho "Built binary successfully: $PROJECT_DIR/build/$RELEASE_NAME.tar.gz"fi-echo "Done... quitting."\ No newline at end of file+if [[ "$@" == *"--package-libs"* ]]; then+ OS_FAMILY=linux+ RELEASE_NAME=typesense-server-libs-$TYPESENSE_VERSION-$OS_FAMILY-amd64+ LIBS=`cat $PROJECT_DIR/external-Linux/libs.txt`+ TAR_PATHS=""+ for lib in $LIBS; do+ TAR_PATHS="$TAR_PATHS -C $PROJECT_DIR/../$lib `ls $PROJECT_DIR/../$lib/*.a | xargs basename`"+ done++ TAR_PATHS="$TAR_PATHS -C $PROJECT_DIR/build `ls $PROJECT_DIR/build/*.a | xargs basename`"+ tar -cvzf $PROJECT_DIR/build/$RELEASE_NAME.tar.gz $TAR_PATHS+fi++echo "Done... quitting."diff --git a/include/api.h b/include/core_api.hsimilarity index 100%rename from include/api.hrename to include/core_api.hdiff --git a/src/api.cpp b/src/core_api.cppsimilarity index 99%rename from src/api.cpprename to src/core_api.cppindex 93263c73..6e3158c0 100644--- a/src/api.cpp+++ b/src/core_api.cpp@@ -2,7 +2,7 @@#include <chrono>#include <thread>#include <sys/resource.h>-#include "api.h"+#include "core_api.h"#include "string_utils.h"#include "collection.h"#include "collection_manager.h"diff --git a/src/main/typesense_server.cpp b/src/main/typesense_server.cppindex 08fe1b45..3095ac34 100644--- a/src/main/typesense_server.cpp+++ b/src/main/typesense_server.cpp@@ -7,7 +7,7 @@#include <sys/stat.h>#include <cmdline.h>#include "http_server.h"-#include "api.h"+#include "core_api.h"#include "string_utils.h"#include "replicator.h"#include "logger.h"