Skip to content

Open Source alternative to Algolia + Pinecone and an Easier-to-Use alternative to ElasticSearch ⚡ 🔍 ✨ Fast, typo tolerant, in-memory fuzzy Search Engine for building delightful search experiences

Handle normalization of unicode elegantly.

Kishore Nallanf8be8f4f6a2f
diff --git a/CMakeLists.txt b/CMakeLists.txt
index 9c82d152..11ae50f4 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -35,6 +35,7 @@ FIND_PACKAGE(ZLIB REQUIRED)
FIND_PACKAGE(CURL REQUIRED)
include(cmake/For.cmake)
+include(cmake/Miniutf.cmake)
include(cmake/H2O.cmake)
include(cmake/RocksDB.cmake)
include(cmake/GoogleTest.cmake)
@@ -46,12 +47,14 @@ include_directories(include)
include_directories(${OPENSSL_INCLUDE_DIR})
include_directories(${CURL_INCLUDE_DIR})
include_directories(${DEP_ROOT_DIR}/${FOR_NAME})
+include_directories(${DEP_ROOT_DIR}/${MINIUTF_NAME})
include_directories(${DEP_ROOT_DIR}/${GTEST_NAME}/googletest/include)
include_directories(${DEP_ROOT_DIR}/${H2O_NAME}/include)
include_directories(${DEP_ROOT_DIR}/${ROCKSDB_NAME}/include)
link_directories(${DEP_ROOT_DIR}/${GTEST_NAME}/googletest/build)
link_directories(${DEP_ROOT_DIR}/${FOR_NAME})
+link_directories(${DEP_ROOT_DIR}/${MINIUTF_NAME}/build)
link_directories(${DEP_ROOT_DIR}/${H2O_NAME}/build)
link_directories(${DEP_ROOT_DIR}/${ROCKSDB_NAME})
@@ -83,7 +86,7 @@ if(NOT APPLE)
list(APPEND ROCKSDB_LIBS rt)
endif()
-target_link_libraries(typesense-server h2o-evloop for pthread ${CURL_LIBRARIES} ${ROCKSDB_LIBS} ${OPENSSL_LIBRARIES} dl ${STD_LIB})
-target_link_libraries(search for pthread h2o-evloop ${CURL_LIBRARIES} ${ROCKSDB_LIBS} ${OPENSSL_LIBRARIES} dl ${STD_LIB})
-target_link_libraries(benchmark for pthread ${CURL_LIBRARIES} h2o-evloop ${ROCKSDB_LIBS} ${OPENSSL_LIBRARIES} dl ${STD_LIB})
-target_link_libraries(typesense_test h2o-evloop ${OPENSSL_LIBRARIES} pthread for ${ROCKSDB_LIBS} gtest gtest_main dl ${STD_LIB})
+target_link_libraries(typesense-server h2o-evloop for miniutf pthread ${CURL_LIBRARIES} ${ROCKSDB_LIBS} ${OPENSSL_LIBRARIES} dl ${STD_LIB})
+target_link_libraries(search for miniutf pthread h2o-evloop ${CURL_LIBRARIES} ${ROCKSDB_LIBS} ${OPENSSL_LIBRARIES} dl ${STD_LIB})
+target_link_libraries(benchmark for miniutf pthread ${CURL_LIBRARIES} h2o-evloop ${ROCKSDB_LIBS} ${OPENSSL_LIBRARIES} dl ${STD_LIB})
+target_link_libraries(typesense_test h2o-evloop ${OPENSSL_LIBRARIES} pthread for miniutf ${ROCKSDB_LIBS} gtest gtest_main dl ${STD_LIB})
diff --git a/TODO.md b/TODO.md
index f681ed29..ad069658 100644
--- a/TODO.md
+++ b/TODO.md
@@ -86,7 +86,7 @@
- ~~handle hyphens (replace them)~~
- ~~clean special chars before indexing~~
- ~~Add docs/explanation around ranking calc~~
-- UTF-8 normalization
+- ~~UTF-8 normalization~~
- NOT operator support
- > INT32_MAX validation for float field
- Proper logging
diff --git a/cmake/Miniutf.cmake b/cmake/Miniutf.cmake
new file mode 100644
index 00000000..5f584f18
--- /dev/null
+++ b/cmake/Miniutf.cmake
@@ -0,0 +1,41 @@
+# Download and build MINIUTF
+
+set(MINIUTF_VERSION 8babf72ddc3e85d794af24520483040fcfbf85f8)
+set(MINIUTF_NAME miniutf-${MINIUTF_VERSION})
+set(MINIUTF_TAR_PATH ${DEP_ROOT_DIR}/${MINIUTF_NAME}.tar.gz)
+
+if(NOT EXISTS ${MINIUTF_TAR_PATH})
+ message(STATUS "Downloading Miniutf...")
+ file(DOWNLOAD https://github.com/dropbox/miniutf/archive/${MINIUTF_VERSION}.tar.gz ${MINIUTF_TAR_PATH})
+endif()
+
+if(NOT EXISTS ${DEP_ROOT_DIR}/${MINIUTF_NAME})
+ message(STATUS "Extracting Miniutf...")
+ execute_process(COMMAND ${CMAKE_COMMAND} -E tar xzf ${MINIUTF_TAR_PATH} WORKING_DIRECTORY ${DEP_ROOT_DIR})
+endif()
+
+# Copy a custom CMakeLists.txt which we will use for the build
+file(COPY ${CMAKE_SOURCE_DIR}/cmake/patches/miniutf/CMakeLists.txt DESTINATION
+ ${DEP_ROOT_DIR}/${MINIUTF_NAME})
+
+if(NOT EXISTS ${DEP_ROOT_DIR}/${MINIUTF_NAME}/build)
+ message("Configuring Miniutf...")
+ file(MAKE_DIRECTORY ${DEP_ROOT_DIR}/${MINIUTF_NAME}/build)
+ execute_process(COMMAND ${CMAKE_COMMAND}
+ "-H${DEP_ROOT_DIR}/${MINIUTF_NAME}"
+ "-B${DEP_ROOT_DIR}/${MINIUTF_NAME}/build"
+ RESULT_VARIABLE
+ MINIUTF_CONFIGURE)
+ if(NOT MINIUTF_CONFIGURE EQUAL 0)
+ message(FATAL_ERROR "Miniutf Configure failed!")
+ endif()
+
+ message("Building Miniutf locally...")
+ execute_process(COMMAND ${CMAKE_COMMAND} --build
+ "${DEP_ROOT_DIR}/${MINIUTF_NAME}/build"
+ RESULT_VARIABLE
+ MINIUTF_BUILD)
+ if(NOT MINIUTF_BUILD EQUAL 0)
+ message(FATAL_ERROR "Miniutf build failed!")
+ endif()
+endif()
\ No newline at end of file
diff --git a/cmake/patches/miniutf/CMakeLists.txt b/cmake/patches/miniutf/CMakeLists.txt
new file mode 100644
index 00000000..ebac5060
--- /dev/null
+++ b/cmake/patches/miniutf/CMakeLists.txt
@@ -0,0 +1,10 @@
+cmake_minimum_required(VERSION 2.8)
+project(miniutf)
+
+set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wall -Wextra -std=c++11 -O2")
+set(CMAKE_CXX_FLAGS_DEBUG "${CMAKE_CXX_FLAGS_DEBUG} -Wall -Wextra -std=c++11 -O0")
+
+FILE(GLOB SRC_FILES miniutf.cpp)
+
+add_library(miniutf STATIC ${SRC_FILES})
+target_link_libraries(miniutf)
\ No newline at end of file
diff --git a/include/string_utils.h b/include/string_utils.h
index baebbe28..5cfb74a6 100644
--- a/include/string_utils.h
+++ b/include/string_utils.h
@@ -4,6 +4,7 @@
#include <algorithm>
#include <sstream>
#include <ctype.h>
+#include "miniutf.hpp"
struct StringUtils {
// Adapted from: http://stackoverflow.com/a/236180/131050
@@ -126,7 +127,8 @@ struct StringUtils {
return !std::isalnum(c) && (int)(c) >= 0;
}), str.end());
- std::transform(str.begin(), str.end(), str.begin(), ::tolower);
+ //std::transform(str.begin(), str.end(), str.begin(), ::tolower);
+ str = miniutf::lowercase(str);
}
/* https://stackoverflow.com/a/34571089/131050 */
diff --git a/test/string_utils_test.cpp b/test/string_utils_test.cpp
index ceac84dd..3664ce23 100644
--- a/test/string_utils_test.cpp
+++ b/test/string_utils_test.cpp
@@ -14,8 +14,9 @@ TEST(StringUtilsTest, ShouldNormalizeString) {
StringUtils::normalize(alphanum_specialchars);
ASSERT_STREQ("aa12zzwr", alphanum_specialchars.c_str());
- // retain non-ascii unicode characters
+ // retain non-ascii unicode characters and should also lower case them
std::string alphanum_unicodechars = "abcÅà123";
StringUtils::normalize(alphanum_unicodechars);
- ASSERT_STREQ("abc\xC3\x85\xC3\xA0""123", alphanum_unicodechars.c_str());
+ std::cout << alphanum_unicodechars << std::endl;
+ ASSERT_STREQ("abcåà123", alphanum_unicodechars.c_str());
}
\ No newline at end of file