Open Source alternative to Algolia + Pinecone and an Easier-to-Use alternative to ElasticSearch ⚡ 🔍 ✨ Fast, typo tolerant, in-memory fuzzy Search Engine for building delightful search experiences
C++ algoliadatastoreelasticsearchenterprise-searchfacetingfull-text-searchfuzzy-searchgeosearchin-memoryinstantsearchmerchandisingpineconesearchsearch-enginesemantic-searchsimilarity-searchsite-searchsynonymstypo-tolerancevector-search
Handle normalization of unicode elegantly.
Kishore Nallanf8be8f4f6a2f
diff --git a/CMakeLists.txt b/CMakeLists.txtindex 9c82d152..11ae50f4 100644--- a/CMakeLists.txt+++ b/CMakeLists.txt@@ -35,6 +35,7 @@ FIND_PACKAGE(ZLIB REQUIRED)FIND_PACKAGE(CURL REQUIRED)include(cmake/For.cmake)+include(cmake/Miniutf.cmake)include(cmake/H2O.cmake)include(cmake/RocksDB.cmake)include(cmake/GoogleTest.cmake)@@ -46,12 +47,14 @@ include_directories(include)include_directories(${OPENSSL_INCLUDE_DIR})include_directories(${CURL_INCLUDE_DIR})include_directories(${DEP_ROOT_DIR}/${FOR_NAME})+include_directories(${DEP_ROOT_DIR}/${MINIUTF_NAME})include_directories(${DEP_ROOT_DIR}/${GTEST_NAME}/googletest/include)include_directories(${DEP_ROOT_DIR}/${H2O_NAME}/include)include_directories(${DEP_ROOT_DIR}/${ROCKSDB_NAME}/include)link_directories(${DEP_ROOT_DIR}/${GTEST_NAME}/googletest/build)link_directories(${DEP_ROOT_DIR}/${FOR_NAME})+link_directories(${DEP_ROOT_DIR}/${MINIUTF_NAME}/build)link_directories(${DEP_ROOT_DIR}/${H2O_NAME}/build)link_directories(${DEP_ROOT_DIR}/${ROCKSDB_NAME})@@ -83,7 +86,7 @@ if(NOT APPLE)list(APPEND ROCKSDB_LIBS rt)endif()-target_link_libraries(typesense-server h2o-evloop for pthread ${CURL_LIBRARIES} ${ROCKSDB_LIBS} ${OPENSSL_LIBRARIES} dl ${STD_LIB})-target_link_libraries(search for pthread h2o-evloop ${CURL_LIBRARIES} ${ROCKSDB_LIBS} ${OPENSSL_LIBRARIES} dl ${STD_LIB})-target_link_libraries(benchmark for pthread ${CURL_LIBRARIES} h2o-evloop ${ROCKSDB_LIBS} ${OPENSSL_LIBRARIES} dl ${STD_LIB})-target_link_libraries(typesense_test h2o-evloop ${OPENSSL_LIBRARIES} pthread for ${ROCKSDB_LIBS} gtest gtest_main dl ${STD_LIB})+target_link_libraries(typesense-server h2o-evloop for miniutf pthread ${CURL_LIBRARIES} ${ROCKSDB_LIBS} ${OPENSSL_LIBRARIES} dl ${STD_LIB})+target_link_libraries(search for miniutf pthread h2o-evloop ${CURL_LIBRARIES} ${ROCKSDB_LIBS} ${OPENSSL_LIBRARIES} dl ${STD_LIB})+target_link_libraries(benchmark for miniutf pthread ${CURL_LIBRARIES} h2o-evloop ${ROCKSDB_LIBS} ${OPENSSL_LIBRARIES} dl ${STD_LIB})+target_link_libraries(typesense_test h2o-evloop ${OPENSSL_LIBRARIES} pthread for miniutf ${ROCKSDB_LIBS} gtest gtest_main dl ${STD_LIB})diff --git a/TODO.md b/TODO.mdindex f681ed29..ad069658 100644--- a/TODO.md+++ b/TODO.md@@ -86,7 +86,7 @@- ~~handle hyphens (replace them)~~- ~~clean special chars before indexing~~- ~~Add docs/explanation around ranking calc~~-- UTF-8 normalization+- ~~UTF-8 normalization~~- NOT operator support- > INT32_MAX validation for float field- Proper loggingdiff --git a/cmake/Miniutf.cmake b/cmake/Miniutf.cmakenew file mode 100644index 00000000..5f584f18--- /dev/null+++ b/cmake/Miniutf.cmake@@ -0,0 +1,41 @@+# Download and build MINIUTF++set(MINIUTF_VERSION 8babf72ddc3e85d794af24520483040fcfbf85f8)+set(MINIUTF_NAME miniutf-${MINIUTF_VERSION})+set(MINIUTF_TAR_PATH ${DEP_ROOT_DIR}/${MINIUTF_NAME}.tar.gz)++if(NOT EXISTS ${MINIUTF_TAR_PATH})+ message(STATUS "Downloading Miniutf...")+ file(DOWNLOAD https://github.com/dropbox/miniutf/archive/${MINIUTF_VERSION}.tar.gz ${MINIUTF_TAR_PATH})+endif()++if(NOT EXISTS ${DEP_ROOT_DIR}/${MINIUTF_NAME})+ message(STATUS "Extracting Miniutf...")+ execute_process(COMMAND ${CMAKE_COMMAND} -E tar xzf ${MINIUTF_TAR_PATH} WORKING_DIRECTORY ${DEP_ROOT_DIR})+endif()++# Copy a custom CMakeLists.txt which we will use for the build+file(COPY ${CMAKE_SOURCE_DIR}/cmake/patches/miniutf/CMakeLists.txt DESTINATION+ ${DEP_ROOT_DIR}/${MINIUTF_NAME})++if(NOT EXISTS ${DEP_ROOT_DIR}/${MINIUTF_NAME}/build)+ message("Configuring Miniutf...")+ file(MAKE_DIRECTORY ${DEP_ROOT_DIR}/${MINIUTF_NAME}/build)+ execute_process(COMMAND ${CMAKE_COMMAND}+ "-H${DEP_ROOT_DIR}/${MINIUTF_NAME}"+ "-B${DEP_ROOT_DIR}/${MINIUTF_NAME}/build"+ RESULT_VARIABLE+ MINIUTF_CONFIGURE)+ if(NOT MINIUTF_CONFIGURE EQUAL 0)+ message(FATAL_ERROR "Miniutf Configure failed!")+ endif()++ message("Building Miniutf locally...")+ execute_process(COMMAND ${CMAKE_COMMAND} --build+ "${DEP_ROOT_DIR}/${MINIUTF_NAME}/build"+ RESULT_VARIABLE+ MINIUTF_BUILD)+ if(NOT MINIUTF_BUILD EQUAL 0)+ message(FATAL_ERROR "Miniutf build failed!")+ endif()+endif()\ No newline at end of filediff --git a/cmake/patches/miniutf/CMakeLists.txt b/cmake/patches/miniutf/CMakeLists.txtnew file mode 100644index 00000000..ebac5060--- /dev/null+++ b/cmake/patches/miniutf/CMakeLists.txt@@ -0,0 +1,10 @@+cmake_minimum_required(VERSION 2.8)+project(miniutf)++set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wall -Wextra -std=c++11 -O2")+set(CMAKE_CXX_FLAGS_DEBUG "${CMAKE_CXX_FLAGS_DEBUG} -Wall -Wextra -std=c++11 -O0")++FILE(GLOB SRC_FILES miniutf.cpp)++add_library(miniutf STATIC ${SRC_FILES})+target_link_libraries(miniutf)\ No newline at end of filediff --git a/include/string_utils.h b/include/string_utils.hindex baebbe28..5cfb74a6 100644--- a/include/string_utils.h+++ b/include/string_utils.h@@ -4,6 +4,7 @@#include <algorithm>#include <sstream>#include <ctype.h>+#include "miniutf.hpp"struct StringUtils {// Adapted from: http://stackoverflow.com/a/236180/131050@@ -126,7 +127,8 @@ struct StringUtils {return !std::isalnum(c) && (int)(c) >= 0;}), str.end());- std::transform(str.begin(), str.end(), str.begin(), ::tolower);+ //std::transform(str.begin(), str.end(), str.begin(), ::tolower);+ str = miniutf::lowercase(str);}/* https://stackoverflow.com/a/34571089/131050 */diff --git a/test/string_utils_test.cpp b/test/string_utils_test.cppindex ceac84dd..3664ce23 100644--- a/test/string_utils_test.cpp+++ b/test/string_utils_test.cpp@@ -14,8 +14,9 @@ TEST(StringUtilsTest, ShouldNormalizeString) {StringUtils::normalize(alphanum_specialchars);ASSERT_STREQ("aa12zzwr", alphanum_specialchars.c_str());- // retain non-ascii unicode characters+ // retain non-ascii unicode characters and should also lower case themstd::string alphanum_unicodechars = "abcÅà123";StringUtils::normalize(alphanum_unicodechars);- ASSERT_STREQ("abc\xC3\x85\xC3\xA0""123", alphanum_unicodechars.c_str());+ std::cout << alphanum_unicodechars << std::endl;+ ASSERT_STREQ("abcåà123", alphanum_unicodechars.c_str());}\ No newline at end of file