Skip to content

Open Source alternative to Algolia + Pinecone and an Easier-to-Use alternative to ElasticSearch ⚡ 🔍 ✨ Fast, typo tolerant, in-memory fuzzy Search Engine for building delightful search experiences

Normalize unicode characters using ICU.

Kishore Nallan55ac4feb08af
diff --git a/CMakeLists.txt b/CMakeLists.txt
index 87761f8b..ae06d4bc 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -33,6 +33,7 @@ FIND_PACKAGE(OpenSSL 1.0.2 REQUIRED)
FIND_PACKAGE(Snappy REQUIRED)
FIND_PACKAGE(ZLIB REQUIRED)
FIND_PACKAGE(CURL REQUIRED)
+FIND_PACKAGE(ICU REQUIRED)
include(cmake/For.cmake)
include(cmake/Miniutf.cmake)
@@ -47,6 +48,7 @@ FILE(GLOB SRC_FILES src/*.cpp)
include_directories(include)
include_directories(${OPENSSL_INCLUDE_DIR})
include_directories(${CURL_INCLUDE_DIR})
+include_directories(${ICU_INCLUDE_DIRS})
include_directories(${DEP_ROOT_DIR}/${FOR_NAME})
include_directories(${DEP_ROOT_DIR}/${MINIUTF_NAME})
include_directories(${DEP_ROOT_DIR}/${GTEST_NAME}/googletest/include)
@@ -92,8 +94,9 @@ endif()
# Specifically pick the static version since the build generates both static and dynamic libraries
set(G3LOGGER "g3logger.a")
+set(ICU_ALL_LIBRARIES ${ICU_I18N_LIBRARIES} ${ICU_LIBRARIES} ${ICU_DATA_LIBRARIES})
-target_link_libraries(typesense-server h2o-evloop for miniutf ${G3LOGGER} pthread ${CURL_LIBRARIES} ${ROCKSDB_LIBS} ${OPENSSL_LIBRARIES} dl ${STD_LIB})
-target_link_libraries(search for miniutf ${G3LOGGER} pthread h2o-evloop ${CURL_LIBRARIES} ${ROCKSDB_LIBS} ${OPENSSL_LIBRARIES} dl ${STD_LIB})
-target_link_libraries(benchmark for miniutf ${G3LOGGER} pthread ${CURL_LIBRARIES} h2o-evloop ${ROCKSDB_LIBS} ${OPENSSL_LIBRARIES} dl ${STD_LIB})
-target_link_libraries(typesense_test h2o-evloop ${OPENSSL_LIBRARIES} pthread for miniutf ${G3LOGGER} ${ROCKSDB_LIBS} gtest gtest_main dl ${STD_LIB})
+target_link_libraries(typesense-server h2o-evloop for miniutf ${ICU_ALL_LIBRARIES} ${G3LOGGER} pthread ${CURL_LIBRARIES} ${ROCKSDB_LIBS} ${OPENSSL_LIBRARIES} dl ${STD_LIB})
+target_link_libraries(search for miniutf ${ICU_ALL_LIBRARIES} ${G3LOGGER} pthread h2o-evloop ${CURL_LIBRARIES} ${ROCKSDB_LIBS} ${OPENSSL_LIBRARIES} dl ${STD_LIB})
+target_link_libraries(benchmark for miniutf ${ICU_ALL_LIBRARIES} ${G3LOGGER} pthread ${CURL_LIBRARIES} h2o-evloop ${ROCKSDB_LIBS} ${OPENSSL_LIBRARIES} dl ${STD_LIB})
+target_link_libraries(typesense_test h2o-evloop ${ICU_ALL_LIBRARIES} ${OPENSSL_LIBRARIES} pthread for miniutf ${G3LOGGER} ${ROCKSDB_LIBS} gtest gtest_main dl ${STD_LIB})
diff --git a/cmake/Modules/FindICU.cmake b/cmake/Modules/FindICU.cmake
new file mode 100644
index 00000000..e5fe47b0
--- /dev/null
+++ b/cmake/Modules/FindICU.cmake
@@ -0,0 +1,97 @@
+# Source: https://github.com/facebook/hhvm/blob/master/CMake/FindICU.cmake
+#
+# Finds the International Components for Unicode (ICU) Library
+#
+# ICU_FOUND - True if ICU found.
+# ICU_I18N_FOUND - True if ICU's internationalization library found.
+# ICU_INCLUDE_DIRS - Directory to include to get ICU headers
+# Note: always include ICU headers as, e.g.,
+# unicode/utypes.h
+# ICU_LIBRARIES - Libraries to link against for the common ICU
+# ICU_I18N_LIBRARIES - Libraries to link against for ICU internationaliation
+# (note: in addition to ICU_LIBRARIES)
+
+# Look for the header file.
+find_path(
+ ICU_INCLUDE_DIR
+ NAMES unicode/utypes.h
+ DOC "Include directory for the ICU library")
+mark_as_advanced(ICU_INCLUDE_DIR)
+
+# Look for the library.
+find_library(
+ ICU_LIBRARY
+ NAMES icuuc cygicuuc cygicuuc32
+ DOC "Libraries to link against for the common parts of ICU")
+mark_as_advanced(ICU_LIBRARY)
+
+# Copy the results to the output variables.
+if(ICU_INCLUDE_DIR AND ICU_LIBRARY)
+ set(ICU_FOUND 1)
+ set(ICU_LIBRARIES ${ICU_LIBRARY})
+ set(ICU_INCLUDE_DIRS ${ICU_INCLUDE_DIR})
+
+ set(ICU_VERSION 0)
+ set(ICU_MAJOR_VERSION 0)
+ set(ICU_MINOR_VERSION 0)
+ if (EXISTS "${ICU_INCLUDE_DIR}/unicode/uvernum.h")
+ FILE(READ "${ICU_INCLUDE_DIR}/unicode/uvernum.h" _ICU_VERSION_CONENTS)
+ else()
+ FILE(READ "${ICU_INCLUDE_DIR}/unicode/uversion.h" _ICU_VERSION_CONENTS)
+ endif()
+
+ STRING(REGEX REPLACE ".*#define U_ICU_VERSION_MAJOR_NUM ([0-9]+).*" "\\1" ICU_MAJOR_VERSION "${_ICU_VERSION_CONENTS}")
+ STRING(REGEX REPLACE ".*#define U_ICU_VERSION_MINOR_NUM ([0-9]+).*" "\\1" ICU_MINOR_VERSION "${_ICU_VERSION_CONENTS}")
+
+ set(ICU_VERSION "${ICU_MAJOR_VERSION}.${ICU_MINOR_VERSION}")
+
+ # Look for the ICU internationalization libraries
+ find_library(
+ ICU_I18N_LIBRARY
+ NAMES icuin icui18n cygicuin cygicuin32
+ DOC "Libraries to link against for ICU internationalization")
+ mark_as_advanced(ICU_I18N_LIBRARY)
+ if (ICU_I18N_LIBRARY)
+ set(ICU_I18N_FOUND 1)
+ set(ICU_I18N_LIBRARIES ${ICU_I18N_LIBRARY})
+ else (ICU_I18N_LIBRARY)
+ set(ICU_I18N_FOUND 0)
+ set(ICU_I18N_LIBRARIES)
+ endif (ICU_I18N_LIBRARY)
+
+ # Look for the ICU data libraries
+ find_library(
+ ICU_DATA_LIBRARY
+ NAMES icudt icudata cygicudata cygicudata32
+ DOC "Libraries to link against for ICU data")
+ mark_as_advanced(ICU_DATA_LIBRARY)
+ if (ICU_DATA_LIBRARY)
+ set(ICU_DATA_FOUND 1)
+ set(ICU_DATA_LIBRARIES ${ICU_DATA_LIBRARY})
+ else (ICU_DATA_LIBRARY)
+ set(ICU_DATA_FOUND 0)
+ set(ICU_DATA_LIBRARIES)
+ endif (ICU_DATA_LIBRARY)
+else(ICU_INCLUDE_DIR AND ICU_LIBRARY)
+ set(ICU_FOUND 0)
+ set(ICU_I18N_FOUND 0)
+ set(ICU_LIBRARIES)
+ set(ICU_I18N_LIBRARIES)
+ set(ICU_INCLUDE_DIRS)
+ set(ICU_VERSION)
+ set(ICU_MAJOR_VERSION)
+ set(ICU_MINOR_VERSION)
+endif(ICU_INCLUDE_DIR AND ICU_LIBRARY)
+
+IF(ICU_FOUND)
+ IF( NOT ICU_FIND_QUIETLY )
+ MESSAGE( STATUS "Found ICU header files in ${ICU_INCLUDE_DIRS}")
+ MESSAGE( STATUS "Found ICU libraries: ${ICU_LIBRARIES}")
+ ENDIF( NOT ICU_FIND_QUIETLY )
+ELSE(ICU_FOUND)
+ IF(ICU_FIND_REQUIRED)
+ MESSAGE( FATAL_ERROR "Could not find ICU" )
+ ELSE(ICU_FIND_REQUIRED)
+ MESSAGE( STATUS "Optional package ICU was not found" )
+ ENDIF(ICU_FIND_REQUIRED)
+ENDIF(ICU_FOUND)
\ No newline at end of file
diff --git a/docker/development.Dockerfile b/docker/development.Dockerfile
index 9b5c939d..0c6c8817 100644
--- a/docker/development.Dockerfile
+++ b/docker/development.Dockerfile
@@ -35,5 +35,13 @@ ADD https://github.com/curl/curl/releases/download/curl-7_55_1/curl-7.55.1.tar.b
RUN tar -C /opt -xf /opt/curl-7.55.1.tar.bz2
RUN cd /opt/curl-7.55.1 && ./configure && make && make install
+ADD https://ssl.icu-project.org/files/icu4c/61.1/icu4c-61_1-src.tgz /opt/
+RUN tar -C /opt -xf /opt/icu4c-61_1-src.tgz
+RUN cd /opt/icu/source && echo "#define U_DISABLE_RENAMING 1" >> common/unicode/uconfig.h && \
+ echo "#define U_STATIC_IMPLEMENTATION 1" >> common/unicode/uconfig.h && \
+ echo "#define U_USING_ICU_NAMESPACE 0" >> common/unicode/uconfig.h
+RUN cd /opt/icu/source && ./runConfigureICU Linux --disable-samples --disable-tests --enable-static \
+ --disable-shared --disable-renaming && make && make install
+
ENV CC /usr/local/gcc-6.4.0/bin/gcc
ENV CXX /usr/local/gcc-6.4.0/bin/g++
diff --git a/include/index.h b/include/index.h
index 87fcc26b..ec51aede 100644
--- a/include/index.h
+++ b/include/index.h
@@ -13,6 +13,7 @@
#include <json.hpp>
#include <field.h>
#include <option.h>
+#include "string_utils.h"
struct token_candidates {
std::string token;
@@ -68,6 +69,8 @@ private:
spp::sparse_hash_map<std::string, spp::sparse_hash_map<uint32_t, number_t>*> sort_index;
+ StringUtils string_utils;
+
static inline std::vector<art_leaf *> next_suggestion(const std::vector<token_candidates> &token_candidates_vec,
long long int n);
diff --git a/include/string_utils.h b/include/string_utils.h
index d69c9dc8..015f1efd 100644
--- a/include/string_utils.h
+++ b/include/string_utils.h
@@ -1,12 +1,29 @@
#pragma once
+#define U_USING_ICU_NAMESPACE 0
+#define U_STATIC_IMPLEMENTATION 1
+//#define U_DISABLE_RENAMING 1
+
#include <string>
#include <algorithm>
#include <sstream>
#include <ctype.h>
#include "miniutf.hpp"
+#include <unicode/translit.h>
+#include <vector>
struct StringUtils {
+ UErrorCode status;
+ icu::Transliterator* transliterator;
+
+ StringUtils():status(U_ZERO_ERROR),
+ transliterator(icu::Transliterator::createInstance("Latin-ASCII", UTRANS_FORWARD, status)) {
+ }
+
+ ~StringUtils() {
+ delete transliterator;
+ }
+
// Adapted from: http://stackoverflow.com/a/236180/131050
static void split(const std::string& s, std::vector<std::string> & result, const std::string& delim, const bool keep_empty = false) {
if (delim.empty()) {
@@ -130,6 +147,8 @@ struct StringUtils {
str = miniutf::lowercase(str);
}
+ void unicode_normalize(std::string& str) const;
+
/* https://stackoverflow.com/a/34571089/131050 */
static std::string base64_encode(const std::string &in) {
std::string out;
diff --git a/src/index.cpp b/src/index.cpp
index 12ce2177..6c09ee6c 100644
--- a/src/index.cpp
+++ b/src/index.cpp
@@ -227,7 +227,7 @@ void Index::index_string_field(const std::string & text, const uint32_t score, a
StringUtils::split(text, tokens, " ");
for(uint32_t i=0; i<tokens.size(); i++) {
auto & token = tokens[i];
- transform(token.begin(), token.end(), token.begin(), tolower);
+ string_utils.unicode_normalize(token);
token_to_offsets[token].push_back(i);
}
}
@@ -483,7 +483,7 @@ Option<uint32_t> Index::do_filtering(uint32_t** filter_ids_out, const std::vecto
for(size_t i = 0; i < str_tokens.size(); i++) {
std::string & str_token = str_tokens[i];
- StringUtils::normalize(str_token);
+ string_utils.unicode_normalize(str_token);
art_leaf* leaf = (art_leaf *) art_search(t, (const unsigned char*) str_token.c_str(),
str_token.length()+1);
if(leaf == nullptr) {
@@ -654,7 +654,7 @@ void Index::search_field(std::string & query, const std::string & field, uint32_
}
token_to_costs.push_back(all_costs);
- StringUtils::normalize(tokens[token_index]);
+ string_utils.unicode_normalize(tokens[token_index]);
}
// stores candidates for each token, i.e. i-th index would have all possible tokens with a cost of "c"
@@ -1051,7 +1051,7 @@ Option<uint32_t> Index::remove(const uint32_t seq_id, nlohmann::json & document)
int key_len;
if(name_field.second.type == field_types::STRING_ARRAY || name_field.second.type == field_types::STRING) {
- StringUtils::normalize(token);
+ string_utils.unicode_normalize(token);
key = (const unsigned char *) token.c_str();
key_len = (int) (token.length() + 1);
} else {
diff --git a/src/string_utils.cpp b/src/string_utils.cpp
new file mode 100644
index 00000000..98180bd7
--- /dev/null
+++ b/src/string_utils.cpp
@@ -0,0 +1,13 @@
+#include "string_utils.h"
+
+void StringUtils::unicode_normalize(std::string& str) const {
+ // remove special chars within ASCII range
+ str.erase(std::remove_if(str.begin(), str.end(), [](char c) {
+ return !std::isalnum(c) && (int)(c) >= 0;
+ }), str.end());
+
+ UnicodeString u_str = UnicodeString::fromUTF8(str);
+ transliterator->transliterate(u_str);
+ str.clear();
+ u_str.toLower().toUTF8String(str);
+}
\ No newline at end of file
diff --git a/test/string_utils_test.cpp b/test/string_utils_test.cpp
index a74638b1..25b244a1 100644
--- a/test/string_utils_test.cpp
+++ b/test/string_utils_test.cpp
@@ -2,20 +2,26 @@
#include "string_utils.h"
TEST(StringUtilsTest, ShouldNormalizeString) {
+ StringUtils string_utils;
+
std::string alphanum = "Aa12Zz";
- StringUtils::normalize(alphanum);
+ string_utils.unicode_normalize(alphanum);
ASSERT_STREQ("aa12zz", alphanum.c_str());
std::string alphanum_space = "Aa12Zz 12A";
- StringUtils::normalize(alphanum_space);
+ string_utils.unicode_normalize(alphanum_space);
ASSERT_STREQ("aa12zz12a", alphanum_space.c_str());
std::string alphanum_specialchars = "Aa12Zz@W-_?,.R";
- StringUtils::normalize(alphanum_specialchars);
+ string_utils.unicode_normalize(alphanum_specialchars);
ASSERT_STREQ("aa12zzwr", alphanum_specialchars.c_str());
- // retain non-ascii unicode characters and should also lower case them
+ // transliterate unicode characters and should also lower case them
std::string alphanum_unicodechars = "abcÅà123";
- StringUtils::normalize(alphanum_unicodechars);
- ASSERT_STREQ("abcåà123", alphanum_unicodechars.c_str());
+ string_utils.unicode_normalize(alphanum_unicodechars);
+ ASSERT_STREQ("abcaa123", alphanum_unicodechars.c_str());
+
+ std::string tamil_unicodechars = "தமிழ் நாடு";
+ string_utils.unicode_normalize(tamil_unicodechars);
+ ASSERT_STREQ("தமிழ்நாடு", tamil_unicodechars.c_str());
}
\ No newline at end of file