Skip to content

Open Source alternative to Algolia + Pinecone and an Easier-to-Use alternative to ElasticSearch ⚡ 🔍 ✨ Fast, typo tolerant, in-memory fuzzy Search Engine for building delightful search experiences

Build RocksDB and H2O also as part of the build process.

Kishore Nallanee68da6f537b
diff --git a/.gitmodules b/.gitmodules
deleted file mode 100644
index 911f1f5c..00000000
--- a/.gitmodules
+++ /dev/null
@@ -1,4 +0,0 @@
-[submodule "external/libfor"]
- path = external/libfor
- url = [email protected]:cruppstahl/libfor.git
- ignore = dirty
\ No newline at end of file
diff --git a/CMakeLists.txt b/CMakeLists.txt
index 65a928bb..3025c583 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -8,41 +8,34 @@ if(NOT EXISTS ${CMAKE_SOURCE_DIR}/external/)
file(MAKE_DIRECTORY ${CMAKE_SOURCE_DIR}/external)
endif()
-include(GoogleTest.cmake)
-include(For.cmake)
+include(cmake/GoogleTest.cmake)
+include(cmake/For.cmake)
+include(cmake/H2O.cmake)
+include(cmake/RocksDB.cmake)
if (APPLE)
add_definitions(-DGTEST_USE_OWN_TR1_TUPLE)
add_definitions(-D__GLIBCXX__)
- include_directories(/usr/local/opt/openssl/include) # For openssl installed with brew
endif (APPLE)
-# FIXME: Need to use find_package() to locate the correct openssl library
-
FILE(GLOB HEADER_FILES include/*)
include_directories(include)
include_directories(${CMAKE_SOURCE_DIR}/external/${FOR_NAME})
-include_directories(${CMAKE_SOURCE_DIR}/external/googletest-release-1.8.0/googletest/include)
+include_directories(${CMAKE_SOURCE_DIR}/external/${GTEST_NAME}/googletest/include)
+include_directories(${CMAKE_SOURCE_DIR}/external/${H2O_NAME}/include)
+include_directories(${CMAKE_SOURCE_DIR}/external/${ROCKSDB_NAME}/include)
-link_directories(${CMAKE_SOURCE_DIR}/external/googletest-release-1.8.0/googletest/build)
+link_directories(${CMAKE_SOURCE_DIR}/external/${GTEST_NAME}/googletest/build)
+link_directories(${CMAKE_SOURCE_DIR}/external/${FOR_NAME})
+link_directories(${CMAKE_SOURCE_DIR}/external/${H2O_NAME}/build)
+link_directories(${CMAKE_SOURCE_DIR}/external/${H2O_NAME}/build/libressl-build/lib)
+link_directories(${CMAKE_SOURCE_DIR}/external/${ROCKSDB_NAME})
add_executable(typesense_test test/forarray_test.cpp)
add_executable(search ${HEADER_FILES} src/art.cpp src/intersection.cpp src/main.cpp src/collection.cpp)
add_executable(typesense-server ${HEADER_FILES} src/art.cpp src/intersection.cpp src/collection.cpp src/server.cpp)
-target_link_libraries(typesense_test
- ${CMAKE_SOURCE_DIR}/external/${FOR_NAME}/libfor.a
- gtest gtest_main
-)
-
-target_link_libraries(search ${CMAKE_SOURCE_DIR}/external/${FOR_NAME}/libfor.a rocksdb)
-target_link_libraries(typesense-server
- ${CMAKE_SOURCE_DIR}/external/${FOR_NAME}/libfor.a
- curl h2o-evloop pthread rocksdb
- /usr/local/opt/openssl/lib/libssl.a
- /usr/local/opt/openssl/lib/libcrypto.a)
-
-#add_dependencies(typesense_test libfor)
-#add_dependencies(search libfor)
-#add_dependencies(typesense-server libfor)
\ No newline at end of file
+target_link_libraries(typesense_test for gtest gtest_main)
+target_link_libraries(search for pthread rocksdb)
+target_link_libraries(typesense-server for curl h2o-evloop pthread rocksdb ssl crypto)
\ No newline at end of file
diff --git a/README.md b/README.md
index dc33f90c..f2decb9b 100644
--- a/README.md
+++ b/README.md
@@ -7,34 +7,9 @@ Typesense is an open source search engine for building a delightful search exper
- **Blazing fast:** Meticulously designed and optimized for speed
- **Simple and delightful:** Simple API, delightful out-of-the-box experience
-## Dependencies
-
-* [RocksDB](https://github.com/facebook/rocksdb)
-* [libfor](https://github.com/cruppstahl/libfor/)
-* [h2o](https://github.com/h2o/h2o)
-
## Building Typesense from source
-First, clone the git repository with `--recursive` option.
-
-### OS X
-
-Install the following Homebrew dependencies:
-
-```
-brew install cmake
-brew install openssl
-brew install h2o
-brew install rocksdb
-```
-
-Building libfor:
-
-```
-make -C external/libfor
-```
-
-Finally, build Typesense:
+Please ensure that you have `cmake` available on your system. All other dependencies are built automatically.
```
./build.sh
diff --git a/For.cmake b/cmake/For.cmake
similarity index 100%
rename from For.cmake
rename to cmake/For.cmake
diff --git a/GoogleTest.cmake b/cmake/GoogleTest.cmake
similarity index 69%
rename from GoogleTest.cmake
rename to cmake/GoogleTest.cmake
index b2d681fc..6b2af49e 100644
--- a/GoogleTest.cmake
+++ b/cmake/GoogleTest.cmake
@@ -6,16 +6,16 @@ set(GTEST_TAR_PATH ${CMAKE_SOURCE_DIR}/external/${GTEST_NAME}.tar.gz)
if(NOT EXISTS ${CMAKE_SOURCE_DIR}/external/${GTEST_NAME})
message(STATUS "Downloading and extracting Google Test...")
- file(DOWNLOAD https://github.com/google/googletest/archive/release-1.8.0.tar.gz ${GTEST_TAR_PATH})
+ file(DOWNLOAD https://github.com/google/googletest/archive/release-${GTEST_VERSION}.tar.gz ${GTEST_TAR_PATH})
execute_process(COMMAND ${CMAKE_COMMAND} -E tar xvzf ${GTEST_TAR_PATH} WORKING_DIRECTORY ${CMAKE_SOURCE_DIR}/external/)
endif()
-if(NOT EXISTS ${CMAKE_SOURCE_DIR}/external/googletest-release-1.8.0/googletest/build)
+if(NOT EXISTS ${CMAKE_SOURCE_DIR}/external/googletest-release-${GTEST_VERSION}/googletest/build)
message("Configuring Google Test...")
- file(MAKE_DIRECTORY ${CMAKE_SOURCE_DIR}/external/googletest-release-1.8.0/googletest/build)
+ file(MAKE_DIRECTORY ${CMAKE_SOURCE_DIR}/external/googletest-release-${GTEST_VERSION}/googletest/build)
execute_process(COMMAND ${CMAKE_COMMAND}
- "-H${CMAKE_SOURCE_DIR}/external/googletest-release-1.8.0/googletest"
- "-B${CMAKE_SOURCE_DIR}/external/googletest-release-1.8.0/googletest/build"
+ "-H${CMAKE_SOURCE_DIR}/external/googletest-release-${GTEST_VERSION}/googletest"
+ "-B${CMAKE_SOURCE_DIR}/external/googletest-release-${GTEST_VERSION}/googletest/build"
RESULT_VARIABLE
GOOGLETEST_CONFIGURE)
if(NOT GOOGLETEST_CONFIGURE EQUAL 0)
@@ -24,7 +24,7 @@ if(NOT EXISTS ${CMAKE_SOURCE_DIR}/external/googletest-release-1.8.0/googletest/b
message("Building Google Test locally...")
execute_process(COMMAND ${CMAKE_COMMAND} --build
- "${CMAKE_SOURCE_DIR}/external/googletest-release-1.8.0/googletest/build"
+ "${CMAKE_SOURCE_DIR}/external/googletest-release-${GTEST_VERSION}/googletest/build"
RESULT_VARIABLE
GOOGLETEST_BUILD)
if(NOT GOOGLETEST_BUILD EQUAL 0)
diff --git a/cmake/H2O.cmake b/cmake/H2O.cmake
new file mode 100644
index 00000000..30731351
--- /dev/null
+++ b/cmake/H2O.cmake
@@ -0,0 +1,34 @@
+# Download and build gtest
+
+set(H2O_VERSION 2.0.4)
+set(H2O_NAME h2o-${H2O_VERSION})
+set(H2O_TAR_PATH ${CMAKE_SOURCE_DIR}/external/${H2O_NAME}.tar.gz)
+
+if(NOT EXISTS ${CMAKE_SOURCE_DIR}/external/${H2O_NAME})
+ message(STATUS "Downloading and extracting ${H2O_NAME}...")
+ file(DOWNLOAD https://github.com/h2o/h2o/archive/v2.0.4.tar.gz ${H2O_TAR_PATH})
+ execute_process(COMMAND ${CMAKE_COMMAND} -E tar xvzf ${H2O_TAR_PATH} WORKING_DIRECTORY ${CMAKE_SOURCE_DIR}/external/)
+endif()
+
+if(NOT EXISTS ${CMAKE_SOURCE_DIR}/external/${H2O_NAME}/build)
+ message("Configuring ${H2O_NAME}...")
+ file(MAKE_DIRECTORY ${CMAKE_SOURCE_DIR}/external/${H2O_NAME}/build)
+ execute_process(COMMAND ${CMAKE_COMMAND}
+ "-DWITH_BUNDLED_SSL=on"
+ "-H${CMAKE_SOURCE_DIR}/external/${H2O_NAME}"
+ "-B${CMAKE_SOURCE_DIR}/external/${H2O_NAME}/build"
+ RESULT_VARIABLE
+ H2O_CONFIGURE)
+ if(NOT H2O_CONFIGURE EQUAL 0)
+ message(FATAL_ERROR "${H2O_NAME} configure failed!")
+ endif()
+
+ message("Building ${H2O_NAME} locally...")
+ execute_process(COMMAND ${CMAKE_COMMAND} --build
+ "${CMAKE_SOURCE_DIR}/external/${H2O_NAME}/build"
+ RESULT_VARIABLE
+ H2O_BUILD)
+ if(NOT H2O_BUILD EQUAL 0)
+ message(FATAL_ERROR "${H2O_NAME} build failed!")
+ endif()
+endif()
\ No newline at end of file
diff --git a/cmake/RocksDB.cmake b/cmake/RocksDB.cmake
new file mode 100644
index 00000000..4936e200
--- /dev/null
+++ b/cmake/RocksDB.cmake
@@ -0,0 +1,16 @@
+# Download and build RocksDB
+
+set(ROCKSDB_VERSION 4.11.2)
+set(ROCKSDB_NAME rocksdb-${ROCKSDB_VERSION})
+set(ROCKSDB_TAR_PATH ${CMAKE_SOURCE_DIR}/external/${ROCKSDB_NAME}.tar.gz)
+
+if(NOT EXISTS ${CMAKE_SOURCE_DIR}/external/${ROCKSDB_NAME})
+ message(STATUS "Downloading and extracting ${ROCKSDB_NAME}...")
+ file(DOWNLOAD https://github.com/facebook/rocksdb/archive/v${ROCKSDB_VERSION}.tar.gz ${ROCKSDB_TAR_PATH})
+ execute_process(COMMAND ${CMAKE_COMMAND} -E tar xvzf ${ROCKSDB_TAR_PATH} WORKING_DIRECTORY ${CMAKE_SOURCE_DIR}/external/)
+endif()
+
+if(NOT EXISTS ${CMAKE_SOURCE_DIR}/external/${ROCKSDB_NAME}/libfor.a)
+ message("Building ${ROCKSDB_NAME} locally...")
+ execute_process(COMMAND make "static_lib" WORKING_DIRECTORY ${CMAKE_SOURCE_DIR}/external/${ROCKSDB_NAME}/)
+endif()
\ No newline at end of file
diff --git a/include/forarray.h b/include/forarray.h
index 90121f82..d6821a49 100644
--- a/include/forarray.h
+++ b/include/forarray.h
@@ -3,6 +3,7 @@
#include <stdio.h>
#include <cstdlib>
#include <for.h>
+#include <cstring>
#include <limits>
#include <iostream>
@@ -46,7 +47,7 @@ public:
}
static inline uint32_t required_bits(const uint32_t v) {
- return v == 0 ? 0 : 32 - __builtin_clz(v);
+ return (uint32_t) (v == 0 ? 0 : 32 - __builtin_clz(v));
}
uint32_t inline sorted_append_size_required(uint32_t value, uint32_t new_length) {
diff --git a/src/art.cpp b/src/art.cpp
index 6178cc43..d19e06df 100644
--- a/src/art.cpp
+++ b/src/art.cpp
@@ -5,6 +5,9 @@
#include <emmintrin.h>
#include <assert.h>
#include <art.h>
+#include <functional>
+#include <chrono>
+#include <algorithm>
#include <iostream>
#include <limits>
#include <queue>
diff --git a/src/collection.cpp b/src/collection.cpp
index 4cb99438..c8a32c5c 100644
--- a/src/collection.cpp
+++ b/src/collection.cpp
@@ -2,6 +2,7 @@
#include <iostream>
#include <numeric>
+#include <chrono>
#include <topster.h>
#include <intersection.h>
#include <match_score.h>
diff --git a/src/main.cpp b/src/main.cpp
index 3f92b37c..aae50e08 100644
--- a/src/main.cpp
+++ b/src/main.cpp
@@ -3,6 +3,7 @@
#include <fstream>
#include <vector>
#include <numeric>
+#include <chrono>
#include <art.h>
#include <unordered_map>
#include "string_utils.h"