Open Source alternative to Algolia + Pinecone and an Easier-to-Use alternative to ElasticSearch ⚡ 🔍 ✨ Fast, typo tolerant, in-memory fuzzy Search Engine for building delightful search experiences
C++ algoliadatastoreelasticsearchenterprise-searchfacetingfull-text-searchfuzzy-searchgeosearchin-memoryinstantsearchmerchandisingpineconesearchsearch-enginesemantic-searchsimilarity-searchsite-searchsynonymstypo-tolerancevector-search
Normalize unicode characters using ICU.
Kishore Nallan55ac4feb08af
diff --git a/CMakeLists.txt b/CMakeLists.txtindex 87761f8b..ae06d4bc 100644--- a/CMakeLists.txt+++ b/CMakeLists.txt@@ -33,6 +33,7 @@ FIND_PACKAGE(OpenSSL 1.0.2 REQUIRED)FIND_PACKAGE(Snappy REQUIRED)FIND_PACKAGE(ZLIB REQUIRED)FIND_PACKAGE(CURL REQUIRED)+FIND_PACKAGE(ICU REQUIRED)include(cmake/For.cmake)include(cmake/Miniutf.cmake)@@ -47,6 +48,7 @@ FILE(GLOB SRC_FILES src/*.cpp)include_directories(include)include_directories(${OPENSSL_INCLUDE_DIR})include_directories(${CURL_INCLUDE_DIR})+include_directories(${ICU_INCLUDE_DIRS})include_directories(${DEP_ROOT_DIR}/${FOR_NAME})include_directories(${DEP_ROOT_DIR}/${MINIUTF_NAME})include_directories(${DEP_ROOT_DIR}/${GTEST_NAME}/googletest/include)@@ -92,8 +94,9 @@ endif()# Specifically pick the static version since the build generates both static and dynamic librariesset(G3LOGGER "g3logger.a")+set(ICU_ALL_LIBRARIES ${ICU_I18N_LIBRARIES} ${ICU_LIBRARIES} ${ICU_DATA_LIBRARIES})-target_link_libraries(typesense-server h2o-evloop for miniutf ${G3LOGGER} pthread ${CURL_LIBRARIES} ${ROCKSDB_LIBS} ${OPENSSL_LIBRARIES} dl ${STD_LIB})-target_link_libraries(search for miniutf ${G3LOGGER} pthread h2o-evloop ${CURL_LIBRARIES} ${ROCKSDB_LIBS} ${OPENSSL_LIBRARIES} dl ${STD_LIB})-target_link_libraries(benchmark for miniutf ${G3LOGGER} pthread ${CURL_LIBRARIES} h2o-evloop ${ROCKSDB_LIBS} ${OPENSSL_LIBRARIES} dl ${STD_LIB})-target_link_libraries(typesense_test h2o-evloop ${OPENSSL_LIBRARIES} pthread for miniutf ${G3LOGGER} ${ROCKSDB_LIBS} gtest gtest_main dl ${STD_LIB})+target_link_libraries(typesense-server h2o-evloop for miniutf ${ICU_ALL_LIBRARIES} ${G3LOGGER} pthread ${CURL_LIBRARIES} ${ROCKSDB_LIBS} ${OPENSSL_LIBRARIES} dl ${STD_LIB})+target_link_libraries(search for miniutf ${ICU_ALL_LIBRARIES} ${G3LOGGER} pthread h2o-evloop ${CURL_LIBRARIES} ${ROCKSDB_LIBS} ${OPENSSL_LIBRARIES} dl ${STD_LIB})+target_link_libraries(benchmark for miniutf ${ICU_ALL_LIBRARIES} ${G3LOGGER} pthread ${CURL_LIBRARIES} h2o-evloop ${ROCKSDB_LIBS} ${OPENSSL_LIBRARIES} dl ${STD_LIB})+target_link_libraries(typesense_test h2o-evloop ${ICU_ALL_LIBRARIES} ${OPENSSL_LIBRARIES} pthread for miniutf ${G3LOGGER} ${ROCKSDB_LIBS} gtest gtest_main dl ${STD_LIB})diff --git a/cmake/Modules/FindICU.cmake b/cmake/Modules/FindICU.cmakenew file mode 100644index 00000000..e5fe47b0--- /dev/null+++ b/cmake/Modules/FindICU.cmake@@ -0,0 +1,97 @@+# Source: https://github.com/facebook/hhvm/blob/master/CMake/FindICU.cmake+#+# Finds the International Components for Unicode (ICU) Library+#+# ICU_FOUND - True if ICU found.+# ICU_I18N_FOUND - True if ICU's internationalization library found.+# ICU_INCLUDE_DIRS - Directory to include to get ICU headers+# Note: always include ICU headers as, e.g.,+# unicode/utypes.h+# ICU_LIBRARIES - Libraries to link against for the common ICU+# ICU_I18N_LIBRARIES - Libraries to link against for ICU internationaliation+# (note: in addition to ICU_LIBRARIES)++# Look for the header file.+find_path(+ ICU_INCLUDE_DIR+ NAMES unicode/utypes.h+ DOC "Include directory for the ICU library")+mark_as_advanced(ICU_INCLUDE_DIR)++# Look for the library.+find_library(+ ICU_LIBRARY+ NAMES icuuc cygicuuc cygicuuc32+ DOC "Libraries to link against for the common parts of ICU")+mark_as_advanced(ICU_LIBRARY)++# Copy the results to the output variables.+if(ICU_INCLUDE_DIR AND ICU_LIBRARY)+ set(ICU_FOUND 1)+ set(ICU_LIBRARIES ${ICU_LIBRARY})+ set(ICU_INCLUDE_DIRS ${ICU_INCLUDE_DIR})++ set(ICU_VERSION 0)+ set(ICU_MAJOR_VERSION 0)+ set(ICU_MINOR_VERSION 0)+ if (EXISTS "${ICU_INCLUDE_DIR}/unicode/uvernum.h")+ FILE(READ "${ICU_INCLUDE_DIR}/unicode/uvernum.h" _ICU_VERSION_CONENTS)+ else()+ FILE(READ "${ICU_INCLUDE_DIR}/unicode/uversion.h" _ICU_VERSION_CONENTS)+ endif()++ STRING(REGEX REPLACE ".*#define U_ICU_VERSION_MAJOR_NUM ([0-9]+).*" "\\1" ICU_MAJOR_VERSION "${_ICU_VERSION_CONENTS}")+ STRING(REGEX REPLACE ".*#define U_ICU_VERSION_MINOR_NUM ([0-9]+).*" "\\1" ICU_MINOR_VERSION "${_ICU_VERSION_CONENTS}")++ set(ICU_VERSION "${ICU_MAJOR_VERSION}.${ICU_MINOR_VERSION}")++ # Look for the ICU internationalization libraries+ find_library(+ ICU_I18N_LIBRARY+ NAMES icuin icui18n cygicuin cygicuin32+ DOC "Libraries to link against for ICU internationalization")+ mark_as_advanced(ICU_I18N_LIBRARY)+ if (ICU_I18N_LIBRARY)+ set(ICU_I18N_FOUND 1)+ set(ICU_I18N_LIBRARIES ${ICU_I18N_LIBRARY})+ else (ICU_I18N_LIBRARY)+ set(ICU_I18N_FOUND 0)+ set(ICU_I18N_LIBRARIES)+ endif (ICU_I18N_LIBRARY)++ # Look for the ICU data libraries+ find_library(+ ICU_DATA_LIBRARY+ NAMES icudt icudata cygicudata cygicudata32+ DOC "Libraries to link against for ICU data")+ mark_as_advanced(ICU_DATA_LIBRARY)+ if (ICU_DATA_LIBRARY)+ set(ICU_DATA_FOUND 1)+ set(ICU_DATA_LIBRARIES ${ICU_DATA_LIBRARY})+ else (ICU_DATA_LIBRARY)+ set(ICU_DATA_FOUND 0)+ set(ICU_DATA_LIBRARIES)+ endif (ICU_DATA_LIBRARY)+else(ICU_INCLUDE_DIR AND ICU_LIBRARY)+ set(ICU_FOUND 0)+ set(ICU_I18N_FOUND 0)+ set(ICU_LIBRARIES)+ set(ICU_I18N_LIBRARIES)+ set(ICU_INCLUDE_DIRS)+ set(ICU_VERSION)+ set(ICU_MAJOR_VERSION)+ set(ICU_MINOR_VERSION)+endif(ICU_INCLUDE_DIR AND ICU_LIBRARY)++IF(ICU_FOUND)+ IF( NOT ICU_FIND_QUIETLY )+ MESSAGE( STATUS "Found ICU header files in ${ICU_INCLUDE_DIRS}")+ MESSAGE( STATUS "Found ICU libraries: ${ICU_LIBRARIES}")+ ENDIF( NOT ICU_FIND_QUIETLY )+ELSE(ICU_FOUND)+ IF(ICU_FIND_REQUIRED)+ MESSAGE( FATAL_ERROR "Could not find ICU" )+ ELSE(ICU_FIND_REQUIRED)+ MESSAGE( STATUS "Optional package ICU was not found" )+ ENDIF(ICU_FIND_REQUIRED)+ENDIF(ICU_FOUND)\ No newline at end of filediff --git a/docker/development.Dockerfile b/docker/development.Dockerfileindex 9b5c939d..0c6c8817 100644--- a/docker/development.Dockerfile+++ b/docker/development.Dockerfile@@ -35,5 +35,13 @@ ADD https://github.com/curl/curl/releases/download/curl-7_55_1/curl-7.55.1.tar.bRUN tar -C /opt -xf /opt/curl-7.55.1.tar.bz2RUN cd /opt/curl-7.55.1 && ./configure && make && make install+ADD https://ssl.icu-project.org/files/icu4c/61.1/icu4c-61_1-src.tgz /opt/+RUN tar -C /opt -xf /opt/icu4c-61_1-src.tgz+RUN cd /opt/icu/source && echo "#define U_DISABLE_RENAMING 1" >> common/unicode/uconfig.h && \+ echo "#define U_STATIC_IMPLEMENTATION 1" >> common/unicode/uconfig.h && \+ echo "#define U_USING_ICU_NAMESPACE 0" >> common/unicode/uconfig.h+RUN cd /opt/icu/source && ./runConfigureICU Linux --disable-samples --disable-tests --enable-static \+ --disable-shared --disable-renaming && make && make install+ENV CC /usr/local/gcc-6.4.0/bin/gccENV CXX /usr/local/gcc-6.4.0/bin/g++diff --git a/include/index.h b/include/index.hindex 87fcc26b..ec51aede 100644--- a/include/index.h+++ b/include/index.h@@ -13,6 +13,7 @@#include <json.hpp>#include <field.h>#include <option.h>+#include "string_utils.h"struct token_candidates {std::string token;@@ -68,6 +69,8 @@ private:spp::sparse_hash_map<std::string, spp::sparse_hash_map<uint32_t, number_t>*> sort_index;+ StringUtils string_utils;+static inline std::vector<art_leaf *> next_suggestion(const std::vector<token_candidates> &token_candidates_vec,long long int n);diff --git a/include/string_utils.h b/include/string_utils.hindex d69c9dc8..015f1efd 100644--- a/include/string_utils.h+++ b/include/string_utils.h@@ -1,12 +1,29 @@#pragma once+#define U_USING_ICU_NAMESPACE 0+#define U_STATIC_IMPLEMENTATION 1+//#define U_DISABLE_RENAMING 1+#include <string>#include <algorithm>#include <sstream>#include <ctype.h>#include "miniutf.hpp"+#include <unicode/translit.h>+#include <vector>struct StringUtils {+ UErrorCode status;+ icu::Transliterator* transliterator;++ StringUtils():status(U_ZERO_ERROR),+ transliterator(icu::Transliterator::createInstance("Latin-ASCII", UTRANS_FORWARD, status)) {+ }++ ~StringUtils() {+ delete transliterator;+ }+// Adapted from: http://stackoverflow.com/a/236180/131050static void split(const std::string& s, std::vector<std::string> & result, const std::string& delim, const bool keep_empty = false) {if (delim.empty()) {@@ -130,6 +147,8 @@ struct StringUtils {str = miniutf::lowercase(str);}+ void unicode_normalize(std::string& str) const;+/* https://stackoverflow.com/a/34571089/131050 */static std::string base64_encode(const std::string &in) {std::string out;diff --git a/src/index.cpp b/src/index.cppindex 12ce2177..6c09ee6c 100644--- a/src/index.cpp+++ b/src/index.cpp@@ -227,7 +227,7 @@ void Index::index_string_field(const std::string & text, const uint32_t score, aStringUtils::split(text, tokens, " ");for(uint32_t i=0; i<tokens.size(); i++) {auto & token = tokens[i];- transform(token.begin(), token.end(), token.begin(), tolower);+ string_utils.unicode_normalize(token);token_to_offsets[token].push_back(i);}}@@ -483,7 +483,7 @@ Option<uint32_t> Index::do_filtering(uint32_t** filter_ids_out, const std::vectofor(size_t i = 0; i < str_tokens.size(); i++) {std::string & str_token = str_tokens[i];- StringUtils::normalize(str_token);+ string_utils.unicode_normalize(str_token);art_leaf* leaf = (art_leaf *) art_search(t, (const unsigned char*) str_token.c_str(),str_token.length()+1);if(leaf == nullptr) {@@ -654,7 +654,7 @@ void Index::search_field(std::string & query, const std::string & field, uint32_}token_to_costs.push_back(all_costs);- StringUtils::normalize(tokens[token_index]);+ string_utils.unicode_normalize(tokens[token_index]);}// stores candidates for each token, i.e. i-th index would have all possible tokens with a cost of "c"@@ -1051,7 +1051,7 @@ Option<uint32_t> Index::remove(const uint32_t seq_id, nlohmann::json & document)int key_len;if(name_field.second.type == field_types::STRING_ARRAY || name_field.second.type == field_types::STRING) {- StringUtils::normalize(token);+ string_utils.unicode_normalize(token);key = (const unsigned char *) token.c_str();key_len = (int) (token.length() + 1);} else {diff --git a/src/string_utils.cpp b/src/string_utils.cppnew file mode 100644index 00000000..98180bd7--- /dev/null+++ b/src/string_utils.cpp@@ -0,0 +1,13 @@+#include "string_utils.h"++void StringUtils::unicode_normalize(std::string& str) const {+ // remove special chars within ASCII range+ str.erase(std::remove_if(str.begin(), str.end(), [](char c) {+ return !std::isalnum(c) && (int)(c) >= 0;+ }), str.end());++ UnicodeString u_str = UnicodeString::fromUTF8(str);+ transliterator->transliterate(u_str);+ str.clear();+ u_str.toLower().toUTF8String(str);+}\ No newline at end of filediff --git a/test/string_utils_test.cpp b/test/string_utils_test.cppindex a74638b1..25b244a1 100644--- a/test/string_utils_test.cpp+++ b/test/string_utils_test.cpp@@ -2,20 +2,26 @@#include "string_utils.h"TEST(StringUtilsTest, ShouldNormalizeString) {+ StringUtils string_utils;+std::string alphanum = "Aa12Zz";- StringUtils::normalize(alphanum);+ string_utils.unicode_normalize(alphanum);ASSERT_STREQ("aa12zz", alphanum.c_str());std::string alphanum_space = "Aa12Zz 12A";- StringUtils::normalize(alphanum_space);+ string_utils.unicode_normalize(alphanum_space);ASSERT_STREQ("aa12zz12a", alphanum_space.c_str());std::string alphanum_specialchars = "Aa12Zz@W-_?,.R";- StringUtils::normalize(alphanum_specialchars);+ string_utils.unicode_normalize(alphanum_specialchars);ASSERT_STREQ("aa12zzwr", alphanum_specialchars.c_str());- // retain non-ascii unicode characters and should also lower case them+ // transliterate unicode characters and should also lower case themstd::string alphanum_unicodechars = "abcÅà123";- StringUtils::normalize(alphanum_unicodechars);- ASSERT_STREQ("abcåà123", alphanum_unicodechars.c_str());+ string_utils.unicode_normalize(alphanum_unicodechars);+ ASSERT_STREQ("abcaa123", alphanum_unicodechars.c_str());++ std::string tamil_unicodechars = "தமிழ் நாடு";+ string_utils.unicode_normalize(tamil_unicodechars);+ ASSERT_STREQ("தமிழ்நாடு", tamil_unicodechars.c_str());}\ No newline at end of file