Skip to content

Open Source alternative to Algolia + Pinecone and an Easier-to-Use alternative to ElasticSearch ⚡ 🔍 ✨ Fast, typo tolerant, in-memory fuzzy Search Engine for building delightful search experiences

Build lib along with executable.

Kishore Nallanea883b4beca0
diff --git a/.circleci/config.yml b/.circleci/config.yml
index c10bb713..55d0708b 100644
--- a/.circleci/config.yml
+++ b/.circleci/config.yml
@@ -29,4 +29,4 @@ jobs:
destination: build
- run:
name: test
- command: $PROJECT_DIR/build/typesense_test
+ command: $PROJECT_DIR/build/typesense-test
diff --git a/CMakeLists.txt b/CMakeLists.txt
index e610688f..6d18acda 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -6,6 +6,10 @@ set(CMAKE_CXX_FLAGS_DEBUG "${CMAKE_CXX_FLAGS_DEBUG} -Wall -Wextra -Wno-unused-pa
set(DEP_ROOT_DIR ${CMAKE_SOURCE_DIR}/external-${CMAKE_SYSTEM_NAME})
set(CMAKE_MODULE_PATH ${CMAKE_MODULE_PATH} "${CMAKE_SOURCE_DIR}/cmake/Modules/")
+# will be overridden from command line during build
+set(TYPESENSE_VERSION "nightly" CACHE STRING "")
+set(BUILD_DEPS "yes" CACHE STRING "")
+
if(NOT EXISTS ${DEP_ROOT_DIR})
file(MAKE_DIRECTORY ${DEP_ROOT_DIR})
endif()
@@ -54,27 +58,46 @@ include_directories(${DEP_ROOT_DIR}/${H2O_NAME}/include)
include_directories(${DEP_ROOT_DIR}/${ROCKSDB_NAME}/include)
include_directories(${DEP_ROOT_DIR}/${G3LOG_NAME}/src)
+# Write dependency include directories to a file
+file(WRITE ${DEP_ROOT_DIR}/includes.txt "")
+file(APPEND ${DEP_ROOT_DIR}/includes.txt "${DEP_ROOT_DIR}/${FOR_NAME}")
+file(APPEND ${DEP_ROOT_DIR}/includes.txt " ${DEP_ROOT_DIR}/${H2O_NAME}/include")
+file(APPEND ${DEP_ROOT_DIR}/includes.txt " ${DEP_ROOT_DIR}/${ROCKSDB_NAME}/include")
+file(APPEND ${DEP_ROOT_DIR}/includes.txt " ${DEP_ROOT_DIR}/${G3LOG_NAME}/src")
+
link_directories(${DEP_ROOT_DIR}/${GTEST_NAME}/googletest/build)
link_directories(${DEP_ROOT_DIR}/${FOR_NAME})
link_directories(${DEP_ROOT_DIR}/${H2O_NAME}/build)
link_directories(${DEP_ROOT_DIR}/${ROCKSDB_NAME})
link_directories(${DEP_ROOT_DIR}/${G3LOG_NAME}/build)
+# Write dependency libraries to a file
+file(WRITE ${DEP_ROOT_DIR}/libs.txt "")
+file(APPEND ${DEP_ROOT_DIR}/libs.txt "${DEP_ROOT_DIR}/${FOR_NAME}")
+file(APPEND ${DEP_ROOT_DIR}/libs.txt " ${DEP_ROOT_DIR}/${H2O_NAME}/build")
+file(APPEND ${DEP_ROOT_DIR}/libs.txt " ${DEP_ROOT_DIR}/${ROCKSDB_NAME}")
+file(APPEND ${DEP_ROOT_DIR}/libs.txt " ${DEP_ROOT_DIR}/${G3LOG_NAME}/build")
+
add_executable(typesense-server ${SRC_FILES} src/main/typesense_server.cpp)
add_executable(search ${SRC_FILES} src/main/main.cpp)
add_executable(benchmark ${SRC_FILES} src/main/benchmark.cpp)
-add_executable(typesense_test ${SRC_FILES} test/main.cpp test/array_test.cpp test/sorted_array_test.cpp test/art_test.cpp
+add_executable(typesense-test ${SRC_FILES} test/main.cpp test/array_test.cpp test/sorted_array_test.cpp test/art_test.cpp
test/collection_test.cpp test/collection_manager_test.cpp
test/topster_test.cpp test/match_score_test.cpp test/store_test.cpp test/array_utils_test.cpp
test/string_utils_test.cpp)
-set(TYPESENSE_VERSION "nightly" CACHE STRING "") # will be overridden from command line during a release build
+add_library(typesense-core STATIC ${SRC_FILES})
target_compile_definitions(
typesense-server PRIVATE
TYPESENSE_VERSION="${TYPESENSE_VERSION}"
)
+target_compile_definitions(
+ typesense-core PRIVATE
+ TYPESENSE_VERSION="${TYPESENSE_VERSION}"
+)
+
target_compile_definitions(
benchmark PRIVATE
TYPESENSE_VERSION="${TYPESENSE_VERSION}"
@@ -86,7 +109,7 @@ target_compile_definitions(
)
target_compile_definitions(
- typesense_test PRIVATE
+ typesense-test PRIVATE
ROOT_DIR="${CMAKE_SOURCE_DIR}/"
TYPESENSE_VERSION="${TYPESENSE_VERSION}"
)
@@ -107,8 +130,11 @@ if(NOT APPLE)
endif()
set(ICU_ALL_LIBRARIES ${ICU_I18N_LIBRARIES} ${ICU_LIBRARIES} ${ICU_DATA_LIBRARIES})
-
-target_link_libraries(typesense-server h2o-evloop for ${ICU_ALL_LIBRARIES} ${G3LOGGER_LIBRARIES} pthread ${CURL_LIBRARIES} ${ROCKSDB_LIBS} ${OPENSSL_LIBRARIES} dl ${STD_LIB})
-target_link_libraries(search for ${ICU_ALL_LIBRARIES} ${G3LOGGER_LIBRARIES} pthread h2o-evloop ${CURL_LIBRARIES} ${ROCKSDB_LIBS} ${OPENSSL_LIBRARIES} dl ${STD_LIB})
-target_link_libraries(benchmark for ${ICU_ALL_LIBRARIES} ${G3LOGGER_LIBRARIES} pthread ${CURL_LIBRARIES} h2o-evloop ${ROCKSDB_LIBS} ${OPENSSL_LIBRARIES} dl ${STD_LIB})
-target_link_libraries(typesense_test h2o-evloop ${ICU_ALL_LIBRARIES} ${OPENSSL_LIBRARIES} pthread for ${G3LOGGER_LIBRARIES} ${ROCKSDB_LIBS} gtest gtest_main dl ${STD_LIB})
+set(CORE_LIBS h2o-evloop ${CURL_LIBRARIES} for ${ICU_ALL_LIBRARIES} ${G3LOGGER_LIBRARIES} pthread
+ ${ROCKSDB_LIBS} ${OPENSSL_LIBRARIES} dl ${STD_LIB})
+
+target_link_libraries(typesense-core ${CORE_LIBS})
+target_link_libraries(typesense-server ${CORE_LIBS})
+target_link_libraries(search ${CORE_LIBS})
+target_link_libraries(benchmark ${CORE_LIBS})
+target_link_libraries(typesense-test ${CORE_LIBS} gtest gtest_main)
diff --git a/TODO.md b/TODO.md
index 7065f3e4..77542dad 100644
--- a/TODO.md
+++ b/TODO.md
@@ -97,6 +97,7 @@
- ~~Have a LOG(ERROR) level~~
- ~~Handle SIGTERM which is sent when process is killed~~
- ~~Use snappy compression for storage~~
+- Jemalloc
- Exact search
- NOT operator support
- Log operations
diff --git a/build.sh b/build.sh
index c4093998..75ee9dcf 100755
--- a/build.sh
+++ b/build.sh
@@ -29,4 +29,17 @@ if [[ "$@" == *"--create-binary"* ]]; then
printf `md5sum $PROJECT_DIR/build/typesense-server | cut -b-32` > $PROJECT_DIR/build/typesense-server.md5.txt
tar -cvzf $PROJECT_DIR/build/$RELEASE_NAME.tar.gz -C $PROJECT_DIR/build typesense-server typesense-server.md5.txt
echo "Built binary successfully: $PROJECT_DIR/build/$RELEASE_NAME.tar.gz"
-fi
\ No newline at end of file
+fi
+
+if [[ "$@" == *"--package-libs"* ]]; then
+ OS_FAMILY=$(echo `uname` | awk '{print tolower($0)}')
+ RELEASE_NAME=typesense-server-libs-$TYPESENSE_VERSION-$OS_FAMILY-amd64
+ LIBS=`cat $PROJECT_DIR/external-$SYSTEM_NAME/libs.txt`
+ TAR_PATHS=""
+ for lib in $LIBS; do
+ TAR_PATHS="$TAR_PATHS -C $lib `ls $lib/*.a | xargs basename`"
+ done
+
+ TAR_PATHS="$TAR_PATHS -C $PROJECT_DIR/build `ls $PROJECT_DIR/build/*.a | xargs basename`"
+ tar -cvzf $PROJECT_DIR/build/$RELEASE_NAME.tar.gz $TAR_PATHS
+fi
diff --git a/cmake/For.cmake b/cmake/For.cmake
index c7fc962b..bc21fbfa 100644
--- a/cmake/For.cmake
+++ b/cmake/For.cmake
@@ -14,7 +14,7 @@ if(NOT EXISTS ${DEP_ROOT_DIR}/${FOR_NAME})
execute_process(COMMAND ${CMAKE_COMMAND} -E tar xzf ${FOR_TAR_PATH} WORKING_DIRECTORY ${DEP_ROOT_DIR})
endif()
-if(NOT EXISTS ${DEP_ROOT_DIR}/${FOR_NAME}/libfor.a)
+if(NOT EXISTS ${DEP_ROOT_DIR}/${FOR_NAME}/libfor.a AND BUILD_DEPS STREQUAL "yes")
message("Building libfor locally...")
execute_process(COMMAND make WORKING_DIRECTORY ${DEP_ROOT_DIR}/${FOR_NAME}/
RESULT_VARIABLE FOR_BUILD)
diff --git a/cmake/GoogleTest.cmake b/cmake/GoogleTest.cmake
index 6dd358a2..3986adb9 100644
--- a/cmake/GoogleTest.cmake
+++ b/cmake/GoogleTest.cmake
@@ -26,12 +26,14 @@ if(NOT EXISTS ${DEP_ROOT_DIR}/googletest-release-${GTEST_VERSION}/googletest/bui
message(FATAL_ERROR "Google Test Configure failed!")
endif()
- message("Building Google Test locally...")
- execute_process(COMMAND ${CMAKE_COMMAND} --build
- "${DEP_ROOT_DIR}/googletest-release-${GTEST_VERSION}/googletest/build"
- RESULT_VARIABLE
- GOOGLETEST_BUILD)
- if(NOT GOOGLETEST_BUILD EQUAL 0)
- message(FATAL_ERROR "Google Test build failed!")
+ if(BUILD_DEPS STREQUAL "yes")
+ message("Building Google Test locally...")
+ execute_process(COMMAND ${CMAKE_COMMAND} --build
+ "${DEP_ROOT_DIR}/googletest-release-${GTEST_VERSION}/googletest/build"
+ RESULT_VARIABLE
+ GOOGLETEST_BUILD)
+ if(NOT GOOGLETEST_BUILD EQUAL 0)
+ message(FATAL_ERROR "Google Test build failed!")
+ endif()
endif()
-endif()
\ No newline at end of file
+endif()
diff --git a/cmake/H2O.cmake b/cmake/H2O.cmake
index 056d10c5..3938ae3c 100644
--- a/cmake/H2O.cmake
+++ b/cmake/H2O.cmake
@@ -28,12 +28,14 @@ if(NOT EXISTS ${DEP_ROOT_DIR}/${H2O_NAME}/build/h2o)
message(FATAL_ERROR "${H2O_NAME} configure failed!")
endif()
- message("Building ${H2O_NAME} locally...")
- execute_process(COMMAND ${CMAKE_COMMAND} --build
- "${DEP_ROOT_DIR}/${H2O_NAME}/build"
- RESULT_VARIABLE
- H2O_BUILD)
- if(NOT H2O_BUILD EQUAL 0)
- message(FATAL_ERROR "${H2O_NAME} build failed!")
+ if(BUILD_DEPS STREQUAL "yes")
+ message("Building ${H2O_NAME} locally...")
+ execute_process(COMMAND ${CMAKE_COMMAND} --build
+ "${DEP_ROOT_DIR}/${H2O_NAME}/build"
+ RESULT_VARIABLE
+ H2O_BUILD)
+ if(NOT H2O_BUILD EQUAL 0)
+ message(FATAL_ERROR "${H2O_NAME} build failed!")
+ endif()
endif()
-endif()
\ No newline at end of file
+endif()
diff --git a/cmake/RocksDB.cmake b/cmake/RocksDB.cmake
index 44fcfcbe..a93adbb6 100644
--- a/cmake/RocksDB.cmake
+++ b/cmake/RocksDB.cmake
@@ -14,7 +14,7 @@ if(NOT EXISTS ${DEP_ROOT_DIR}/${ROCKSDB_NAME})
execute_process(COMMAND ${CMAKE_COMMAND} -E tar xzf ${ROCKSDB_TAR_PATH} WORKING_DIRECTORY ${DEP_ROOT_DIR}/)
endif()
-if(NOT EXISTS ${DEP_ROOT_DIR}/${ROCKSDB_NAME}/librocksdb.a)
+if(NOT EXISTS ${DEP_ROOT_DIR}/${ROCKSDB_NAME}/librocksdb.a AND BUILD_DEPS STREQUAL "yes")
message("Building ${ROCKSDB_NAME} locally...")
set(ENV{PORTABLE} 1)
set(ENV{DEBUG_LEVEL} 0)
@@ -34,8 +34,8 @@ if(NOT EXISTS ${DEP_ROOT_DIR}/${ROCKSDB_NAME}/librocksdb.a)
execute_process(COMMAND make "clean" WORKING_DIRECTORY ${DEP_ROOT_DIR}/${ROCKSDB_NAME}/)
execute_process(COMMAND make "static_lib" WORKING_DIRECTORY ${DEP_ROOT_DIR}/${ROCKSDB_NAME}/
- RESULT_VARIABLE ROCKSDB_BUILD)
+ RESULT_VARIABLE ROCKSDB_BUILD)
if(NOT ROCKSDB_BUILD EQUAL 0)
message(FATAL_ERROR "${ROCKSDB_NAME} build failed!")
endif()
-endif()
\ No newline at end of file
+endif()
diff --git a/cmake/TestResources.cmake b/cmake/TestResources.cmake
index 1d1c1284..6eac8048 100644
--- a/cmake/TestResources.cmake
+++ b/cmake/TestResources.cmake
@@ -9,12 +9,12 @@ if(NOT EXISTS ${TEST_RESOURCES_DIR})
file(MAKE_DIRECTORY ${TEST_RESOURCES_DIR})
endif()
-if(NOT EXISTS ${ART_WORDS_PATH})
+if(NOT EXISTS ${ART_WORDS_PATH} AND BUILD_DEPS STREQUAL "yes")
message(STATUS "Downloading test resource - words.txt")
file(DOWNLOAD https://raw.githubusercontent.com/kishorenc/libart/${ART_VERSION}/tests/words.txt ${ART_WORDS_PATH})
endif()
-if(NOT EXISTS ${ART_UUID_PATH})
+if(NOT EXISTS ${ART_UUID_PATH} AND BUILD_DEPS STREQUAL "yes")
message(STATUS "Downloading test resource - uuid.txt")
file(DOWNLOAD https://raw.githubusercontent.com/kishorenc/libart/${ART_VERSION}/tests/uuid.txt ${ART_UUID_PATH})
-endif()
\ No newline at end of file
+endif()
diff --git a/cmake/g3log.cmake b/cmake/g3log.cmake
index e48542b8..80842f82 100644
--- a/cmake/g3log.cmake
+++ b/cmake/g3log.cmake
@@ -30,12 +30,14 @@ if(NOT EXISTS ${DEP_ROOT_DIR}/${G3LOG_NAME}/build)
message(FATAL_ERROR "G3log Configure failed!")
endif()
- message("Building G3log locally...")
- execute_process(COMMAND ${CMAKE_COMMAND} --build
- "${DEP_ROOT_DIR}/${G3LOG_NAME}/build"
- RESULT_VARIABLE
- G3LOG_BUILD)
- if(NOT G3LOG_BUILD EQUAL 0)
- message(FATAL_ERROR "G3log build failed!")
+ if(BUILD_DEPS STREQUAL "yes")
+ message("Building G3log locally...")
+ execute_process(COMMAND ${CMAKE_COMMAND} --build
+ "${DEP_ROOT_DIR}/${G3LOG_NAME}/build"
+ RESULT_VARIABLE
+ G3LOG_BUILD)
+ if(NOT G3LOG_BUILD EQUAL 0)
+ message(FATAL_ERROR "G3log build failed!")
+ endif()
endif()
-endif()
\ No newline at end of file
+endif()
diff --git a/docker-build.sh b/docker-build.sh
index b0baa7b6..7bdb2f33 100755
--- a/docker-build.sh
+++ b/docker-build.sh
@@ -44,4 +44,17 @@ if [[ "$@" == *"--create-binary"* ]]; then
echo "Built binary successfully: $PROJECT_DIR/build/$RELEASE_NAME.tar.gz"
fi
-echo "Done... quitting."
\ No newline at end of file
+if [[ "$@" == *"--package-libs"* ]]; then
+ OS_FAMILY=linux
+ RELEASE_NAME=typesense-server-libs-$TYPESENSE_VERSION-$OS_FAMILY-amd64
+ LIBS=`cat $PROJECT_DIR/external-Linux/libs.txt`
+ TAR_PATHS=""
+ for lib in $LIBS; do
+ TAR_PATHS="$TAR_PATHS -C $PROJECT_DIR/../$lib `ls $PROJECT_DIR/../$lib/*.a | xargs basename`"
+ done
+
+ TAR_PATHS="$TAR_PATHS -C $PROJECT_DIR/build `ls $PROJECT_DIR/build/*.a | xargs basename`"
+ tar -cvzf $PROJECT_DIR/build/$RELEASE_NAME.tar.gz $TAR_PATHS
+fi
+
+echo "Done... quitting."
diff --git a/include/api.h b/include/core_api.h
similarity index 100%
rename from include/api.h
rename to include/core_api.h
diff --git a/src/api.cpp b/src/core_api.cpp
similarity index 99%
rename from src/api.cpp
rename to src/core_api.cpp
index 93263c73..6e3158c0 100644
--- a/src/api.cpp
+++ b/src/core_api.cpp
@@ -2,7 +2,7 @@
#include <chrono>
#include <thread>
#include <sys/resource.h>
-#include "api.h"
+#include "core_api.h"
#include "string_utils.h"
#include "collection.h"
#include "collection_manager.h"
diff --git a/src/main/typesense_server.cpp b/src/main/typesense_server.cpp
index 08fe1b45..3095ac34 100644
--- a/src/main/typesense_server.cpp
+++ b/src/main/typesense_server.cpp
@@ -7,7 +7,7 @@
#include <sys/stat.h>
#include <cmdline.h>
#include "http_server.h"
-#include "api.h"
+#include "core_api.h"
#include "string_utils.h"
#include "replicator.h"
#include "logger.h"