Open Source alternative to Algolia + Pinecone and an Easier-to-Use alternative to ElasticSearch ⚡ 🔍 ✨ Fast, typo tolerant, in-memory fuzzy Search Engine for building delightful search experiences
C++ algoliadatastoreelasticsearchenterprise-searchfacetingfull-text-searchfuzzy-searchgeosearchin-memoryinstantsearchmerchandisingpineconesearchsearch-enginesemantic-searchsimilarity-searchsite-searchsynonymstypo-tolerancevector-search
Improve unicode normalization.
Kishore Nallan3c98931c0ed6
diff --git a/CMakeLists.txt b/CMakeLists.txtindex 6d18acda..2a33953d 100644--- a/CMakeLists.txt+++ b/CMakeLists.txt@@ -45,6 +45,7 @@ include(cmake/RocksDB.cmake)include(cmake/GoogleTest.cmake)include(cmake/TestResources.cmake)include(cmake/g3log.cmake)+include(cmake/Iconv.cmake)FILE(GLOB SRC_FILES src/*.cpp)@@ -57,6 +58,7 @@ include_directories(${DEP_ROOT_DIR}/${GTEST_NAME}/googletest/include)include_directories(${DEP_ROOT_DIR}/${H2O_NAME}/include)include_directories(${DEP_ROOT_DIR}/${ROCKSDB_NAME}/include)include_directories(${DEP_ROOT_DIR}/${G3LOG_NAME}/src)+include_directories(${DEP_ROOT_DIR}/${ICONV_NAME}/include)# Write dependency include directories to a filefile(WRITE ${DEP_ROOT_DIR}/includes.txt "")@@ -70,6 +72,7 @@ link_directories(${DEP_ROOT_DIR}/${FOR_NAME})link_directories(${DEP_ROOT_DIR}/${H2O_NAME}/build)link_directories(${DEP_ROOT_DIR}/${ROCKSDB_NAME})link_directories(${DEP_ROOT_DIR}/${G3LOG_NAME}/build)+link_directories(${DEP_ROOT_DIR}/${ICONV_NAME}/lib/.libs)# Write dependency libraries to a filefile(WRITE ${DEP_ROOT_DIR}/libs.txt "")@@ -130,7 +133,7 @@ if(NOT APPLE)endif()set(ICU_ALL_LIBRARIES ${ICU_I18N_LIBRARIES} ${ICU_LIBRARIES} ${ICU_DATA_LIBRARIES})-set(CORE_LIBS h2o-evloop ${CURL_LIBRARIES} for ${ICU_ALL_LIBRARIES} ${G3LOGGER_LIBRARIES} pthread+set(CORE_LIBS h2o-evloop iconv ${CURL_LIBRARIES} for ${ICU_ALL_LIBRARIES} ${G3LOGGER_LIBRARIES} pthread${ROCKSDB_LIBS} ${OPENSSL_LIBRARIES} dl ${STD_LIB})target_link_libraries(typesense-core ${CORE_LIBS})diff --git a/cmake/Iconv.cmake b/cmake/Iconv.cmakenew file mode 100644index 00000000..e87c6b5e--- /dev/null+++ b/cmake/Iconv.cmake@@ -0,0 +1,31 @@+# Download and build libiconv++set(ICONV_VERSION 1.15)+set(ICONV_NAME libiconv-${ICONV_VERSION})+set(ICONV_TAR_PATH ${DEP_ROOT_DIR}/${ICONV_NAME}.tar.gz)++if(NOT EXISTS ${ICONV_TAR_PATH})+ message(STATUS "Downloading libconv...")+ file(DOWNLOAD https://ftp.gnu.org/pub/gnu/libiconv/libiconv-${ICONV_VERSION}.tar.gz ${ICONV_TAR_PATH})+endif()++if(NOT EXISTS ${DEP_ROOT_DIR}/${ICONV_NAME})+ message(STATUS "Extracting libconv...")+ execute_process(COMMAND ${CMAKE_COMMAND} -E tar xzf ${ICONV_TAR_PATH} WORKING_DIRECTORY ${DEP_ROOT_DIR})+endif()++if(NOT EXISTS ${DEP_ROOT_DIR}/${ICONV_NAME}/Makefile AND BUILD_DEPS STREQUAL "yes")+ message("Configuring libconv locally...")+ execute_process(COMMAND ./configure "--enable-static" WORKING_DIRECTORY ${DEP_ROOT_DIR}/${ICONV_NAME}/ RESULT_VARIABLE ICONV_CONFIGURE)+ if(NOT ICONV_CONFIGURE EQUAL 0)+ message(FATAL_ERROR "${ICONV_NAME} configure failed!")+ endif()+endif()++if(NOT EXISTS ${DEP_ROOT_DIR}/${ICONV_NAME}/lib/.libs/libiconv.a AND BUILD_DEPS STREQUAL "yes")+ message("Building libconv locally...")+ execute_process(COMMAND make WORKING_DIRECTORY ${DEP_ROOT_DIR}/${ICONV_NAME})+ if(NOT EXISTS ${DEP_ROOT_DIR}/${ICONV_NAME}/lib/.libs/libiconv.a)+ message(FATAL_ERROR "${ICONV_NAME} build failed!")+ endif()+endif()diff --git a/include/string_utils.h b/include/string_utils.hindex 919538c8..cd022add 100644--- a/include/string_utils.h+++ b/include/string_utils.h@@ -5,14 +5,17 @@#include <sstream>#include <ctype.h>#include <unicode/translit.h>+#include <iconv.h>#include <vector>struct StringUtils {UErrorCode status;//icu::Transliterator* transliterator;+ iconv_t cd;StringUtils(): status(U_ZERO_ERROR) {// transliterator(icu::Transliterator::createInstance("Latin-ASCII", UTRANS_FORWARD, status))+ cd = iconv_open("ascii//TRANSLIT", "UTF-8");}~StringUtils() {diff --git a/src/collection.cpp b/src/collection.cppindex 74bd502e..7d34d191 100644--- a/src/collection.cpp+++ b/src/collection.cpp@@ -610,7 +610,7 @@ Option<nlohmann::json> Collection::search(std::string query, const std::vector<sprune_document(document, include_fields, exclude_fields);wrapper_doc["document"] = document;//wrapper_doc["match_score"] = field_order_kv.match_score;- //wrapper_doc["seq_id"] = (uint32_t) field_order_kv.key;+ wrapper_doc["seq_id"] = (uint32_t) field_order_kv.key;result["hits"].push_back(wrapper_doc);}diff --git a/src/index.cpp b/src/index.cppindex 75d7fcd3..5c775d05 100644--- a/src/index.cpp+++ b/src/index.cpp@@ -732,7 +732,7 @@ void Index::search_field(const uint8_t & field_id, std::string & query, const stconst std::string token_cost_hash = token + std::to_string(costs[token_index]);std::vector<art_leaf*> leaves;- //LOG(INFO) << "\nSearching for: " << token << " - cost: " << costs[token_index];+ //LOG(INFO) << "\nSearching for field: " << field << ", token:" << token << " - cost: " << costs[token_index];if(token_cost_cache.count(token_cost_hash) != 0) {leaves = token_cost_cache[token_cost_hash];@@ -831,7 +831,7 @@ void Index::log_leaves(const int cost, const std::string &token, const std::vectLOG(INFO) << "Token: " << token << ", cost: " << cost;for(size_t i=0; i < leaves.size(); i++) {- printf("%.*s, ", leaves[i]->key_len, leaves[i]->key);+ printf("%.*s - %d, ", leaves[i]->key_len, leaves[i]->key, leaves[i]->values->ids.getLength());LOG(INFO) << "frequency: " << leaves[i]->values->ids.getLength() << ", max_score: " << leaves[i]->max_score;/*for(auto j=0; j<leaves[i]->values->ids.getLength(); j++) {LOG(INFO) << "id: " << leaves[i]->values->ids.at(j);diff --git a/src/string_utils.cpp b/src/string_utils.cppindex b416bcee..13e48bb4 100644--- a/src/string_utils.cpp+++ b/src/string_utils.cpp@@ -1,12 +1,36 @@#include "string_utils.h"+#include <iostream>++std::string lower_and_no_special_chars(const std::string & str) {+ std::stringstream ss;++ for(const auto c: str) {+ bool should_remove = (!std::isalnum(c) && (int)(c) >= 0);+ if(!should_remove) {+ ss << (char) std::tolower(c);+ }+ }++ return ss.str();+}void StringUtils::unicode_normalize(std::string& str) const {- // remove special chars within ASCII range- str.erase(std::remove_if(str.begin(), str.end(), [](char c) {- return !std::isalnum(c) && (int)(c) >= 0;- }), str.end());-- icu::UnicodeString u_str = icu::UnicodeString::fromUTF8(str);- str.clear();- u_str.toLower().toUTF8String(str);+ size_t outbuflen = str.length() * 2;+ char output[outbuflen];+ char *outptr = output;++ char *input = (char *) str.c_str();+ size_t insize = str.length();+ size_t outsize = outbuflen;++ iconv(cd, &input, &insize, &outptr, &outsize);+ size_t actual_size = outbuflen - outsize;++ if(actual_size == 0) {+ str.assign(lower_and_no_special_chars(str));+ return ;+ }++ std::string nstr = std::string(output, actual_size);+ str.assign(lower_and_no_special_chars(nstr));}\ No newline at end of filediff --git a/test/string_utils_test.cpp b/test/string_utils_test.cppindex 7e55189f..b7df56aa 100644--- a/test/string_utils_test.cpp+++ b/test/string_utils_test.cpp@@ -1,5 +1,7 @@#include <gtest/gtest.h>#include "string_utils.h"+#include <iconv.h>+#include <unicode/translit.h>TEST(StringUtilsTest, ShouldNormalizeString) {StringUtils string_utils;@@ -16,13 +18,17 @@ TEST(StringUtilsTest, ShouldNormalizeString) {string_utils.unicode_normalize(alphanum_specialchars);ASSERT_STREQ("aa12zzwr", alphanum_specialchars.c_str());- std::string alphanum_unicodechars = "abcÅà123";+ std::string alphanum_unicodechars = "abcÅà123ß";string_utils.unicode_normalize(alphanum_unicodechars);- ASSERT_STREQ("abcåà123", alphanum_unicodechars.c_str());+ ASSERT_STREQ("abcaa123ss", alphanum_unicodechars.c_str());std::string tamil_unicodechars = "தமிழ் நாடு";string_utils.unicode_normalize(tamil_unicodechars);ASSERT_STREQ("தமிழ்நாடு", tamil_unicodechars.c_str());++ std::string chinese_unicodechars = "你好吗";+ string_utils.unicode_normalize(chinese_unicodechars);+ ASSERT_STREQ("你好吗", chinese_unicodechars.c_str());}TEST(StringUtilsTest, ShouldJoinString) {@@ -33,4 +39,4 @@ TEST(StringUtilsTest, ShouldJoinString) {const std::string & joined_str2 = StringUtils::join(parts, "/", 2);ASSERT_STREQ("baz/bazinga", joined_str2.c_str());-}\ No newline at end of file+}