Open Source alternative to Algolia + Pinecone and an Easier-to-Use alternative to ElasticSearch β‘ π β¨ Fast, typo tolerant, in-memory fuzzy Search Engine for building delightful search experiences
C++ algoliadatastoreelasticsearchenterprise-searchfacetingfull-text-searchfuzzy-searchgeosearchin-memoryinstantsearchmerchandisingpineconesearchsearch-enginesemantic-searchsimilarity-searchsite-searchsynonymstypo-tolerancevector-search
Integrate with Kakasi.
kishorenc3a926859678a
diff --git a/CMakeLists.txt b/CMakeLists.txtindex aaef720d..b5a71859 100644--- a/CMakeLists.txt+++ b/CMakeLists.txt@@ -56,6 +56,7 @@ include(cmake/TestResources.cmake)include(cmake/Iconv.cmake)include(cmake/Jemalloc.cmake)include(cmake/h3.cmake)+include(cmake/kakasi.cmake)FIND_PACKAGE(OpenSSL 1.1.1 REQUIRED)FIND_PACKAGE(Snappy REQUIRED)@@ -69,7 +70,7 @@ FIND_PACKAGE(glog REQUIRED)message("OpenSSL library: ${OPENSSL_LIBRARIES}")-FILE(GLOB SRC_FILES src/*.cpp)+FILE(GLOB SRC_FILES src/*.cpp ${DEP_ROOT_DIR}/${KAKASI_NAME}/data/*.cpp)FILE(GLOB TEST_FILES test/*.cpp)include_directories(include)@@ -86,6 +87,8 @@ include_directories(${DEP_ROOT_DIR}/${BRPC_NAME}/include)include_directories(${DEP_ROOT_DIR}/${BRAFT_NAME}/include)include_directories(${DEP_ROOT_DIR}/${JEMALLOC_NAME}/include/jemalloc)include_directories(${DEP_ROOT_DIR}/${H3_NAME}/build/src/h3lib/include)+include_directories(${DEP_ROOT_DIR}/${KAKASI_NAME}/build/include)+include_directories(${DEP_ROOT_DIR}/${KAKASI_NAME}/data)link_directories(/usr/local/lib)link_directories(${DEP_ROOT_DIR}/${GTEST_NAME}/googletest/build)@@ -95,6 +98,7 @@ link_directories(${DEP_ROOT_DIR}/${ROCKSDB_NAME})link_directories(${DEP_ROOT_DIR}/${ICONV_NAME}/lib/.libs)link_directories(${DEP_ROOT_DIR}/${JEMALLOC_NAME}/lib)link_directories(${DEP_ROOT_DIR}/${H3_NAME}/build/lib)+link_directories(${DEP_ROOT_DIR}/${KAKASI_NAME}/build/lib)set(JEMALLOC_ROOT_DIR "${DEP_ROOT_DIR}/${JEMALLOC_NAME}")FIND_PACKAGE(Jemalloc REQUIRED)@@ -143,7 +147,7 @@ else()endif()set(ICU_ALL_LIBRARIES ${ICU_I18N_LIBRARIES} ${ICU_LIBRARIES} ${ICU_DATA_LIBRARIES})-set(CORE_LIBS h2o-evloop braft brpc iconv ${ICU_ALL_LIBRARIES} ${CURL_LIBRARIES} for h3+set(CORE_LIBS kakasi h2o-evloop braft brpc iconv ${ICU_ALL_LIBRARIES} ${CURL_LIBRARIES} for h3${LevelDB_LIBRARIES} ${ROCKSDB_LIBS}glog ${GFLAGS_LIBRARIES} ${PROTOBUF_LIBRARIES} ${STACKTRACE_LIBS}${OPENSSL_LIBRARIES} ${ZLIB_LIBRARIES} ${JEMALLOC_LIBRARIES}diff --git a/cmake/kakasi.cmake b/cmake/kakasi.cmakenew file mode 100644index 00000000..1a5c2a3b--- /dev/null+++ b/cmake/kakasi.cmake@@ -0,0 +1,45 @@+# Download and build libfor++set(KAKASI_VERSION 9e0825a02c7ea5605e968f6208f769f7c49d6860)+set(KAKASI_NAME kakasi-${KAKASI_VERSION})+set(KAKASI_TAR_PATH ${DEP_ROOT_DIR}/${KAKASI_NAME}.tar.gz)++if(NOT EXISTS ${KAKASI_TAR_PATH})+ message(STATUS "Downloading kakasi...")+ file(DOWNLOAD https://github.com/typesense/kakasi/archive/${KAKASI_VERSION}.tar.gz ${KAKASI_TAR_PATH})+endif()++if(NOT EXISTS ${DEP_ROOT_DIR}/${KAKASI_NAME})+ message(STATUS "Extracting kakasi...")+ execute_process(COMMAND ${CMAKE_COMMAND} -E tar xzf ${KAKASI_TAR_PATH} WORKING_DIRECTORY ${DEP_ROOT_DIR})+endif()++if(NOT EXISTS ${DEP_ROOT_DIR}/${KAKASI_NAME}/build/lib/libkakasi.a AND BUILD_DEPS STREQUAL "yes")+ set(ENV{LIBRARY_PATH} "${DEP_ROOT_DIR}/${ICONV_NAME}/lib/.libs")+ set(ENV{CFLAGS} "-I${DEP_ROOT_DIR}/${ICONV_NAME}/include -g")+ set(ENV{LDFLAGS} "-g")++ message("Configuring kakasi...")+ execute_process(COMMAND ./configure+ "--prefix=${DEP_ROOT_DIR}/${KAKASI_NAME}/build"+ "--enable-shared=no"+ WORKING_DIRECTORY ${DEP_ROOT_DIR}/${KAKASI_NAME}/ RESULT_VARIABLE KAKASI_CONFIGURE)+ if(NOT KAKASI_CONFIGURE EQUAL 0)+ message(FATAL_ERROR "${KAKASI_NAME} configure failed!")+ endif()++ message("Building kakasi locally...")++ execute_process(COMMAND make WORKING_DIRECTORY ${DEP_ROOT_DIR}/${KAKASI_NAME}/+ RESULT_VARIABLE KAKASI_BUILD)+ if(NOT KAKASI_BUILD EQUAL 0)+ message(FATAL_ERROR "${KAKASI_NAME} build failed!")+ endif()++ message("Installing kakasi locally...")+ execute_process(COMMAND make install WORKING_DIRECTORY ${DEP_ROOT_DIR}/${KAKASI_NAME}/+ RESULT_VARIABLE KAKASI_INSTALL)+ if(NOT KAKASI_INSTALL EQUAL 0)+ message(FATAL_ERROR "${KAKASI_NAME} install failed!")+ endif()+endif()diff --git a/include/field.h b/include/field.hindex b48baca0..1dae7278 100644--- a/include/field.h+++ b/include/field.h@@ -300,9 +300,9 @@ struct field {field_json[fields::name].get<std::string>() + std::string("` should be a string."));}- if(!field_json[fields::locale].empty() || field_json[fields::locale] != "en" ||- field_json[fields::locale] != "ja" || field_json[fields::locale] != "ko" ||- field_json[fields::locale] != "zh" || field_json[fields::locale] != "th") {+ if(!field_json[fields::locale].get<std::string>().empty() && field_json[fields::locale] != "en" &&+ field_json[fields::locale] != "ja" && field_json[fields::locale] != "ko" &&+ field_json[fields::locale] != "zh" && field_json[fields::locale] != "th") {return Option<bool>(400, std::string("The `locale` value of the field `") +field_json[fields::name].get<std::string>() + std::string("` is not valid."));}@@ -321,25 +321,29 @@ struct field {field_json[fields::index] = true;}+ if(field_json.count(fields::locale) == 0) {+ field_json[fields::locale] = "";+ }+if(field_json.count(fields::geo_resolution) != 0) {- return Option<bool>(400, "Field `*` cannot contain a geo resolution.");+ return Option<bool>(400, "Field `.*` cannot contain a geo resolution.");}if(field_json[fields::optional] == false) {- return Option<bool>(400, "Field `*` must be an optional field.");+ return Option<bool>(400, "Field `.*` must be an optional field.");}if(field_json[fields::facet] == true) {- return Option<bool>(400, "Field `*` cannot be a facet field.");+ return Option<bool>(400, "Field `.*` cannot be a facet field.");}if(field_json[fields::index] == false) {- return Option<bool>(400, "Field `*` must be an index field.");+ return Option<bool>(400, "Field `.*` must be an index field.");}-field fallback_field(field_json["name"], field_json["type"], field_json["facet"],- field_json["optional"]);+ field_json["optional"], field_json[fields::index],+ DEFAULT_GEO_RESOLUTION, field_json[fields::locale]);if(fallback_field.has_valid_type()) {fallback_field_type = fallback_field.type;@@ -360,6 +364,10 @@ struct field {field_json[fields::index] = true;}+ if(field_json.count(fields::locale) == 0) {+ field_json[fields::locale] = "";+ }+bool is_dynamic = field_json[fields::name].get<std::string>().find(".*") != std::string::npos;if(field_json.count(fields::optional) == 0) {@@ -373,7 +381,8 @@ struct field {fields.emplace_back(field(field_json[fields::name], field_json[fields::type], field_json[fields::facet],- field_json[fields::optional], field_json[fields::index], field_json[fields::geo_resolution])+ field_json[fields::optional], field_json[fields::index],+ field_json[fields::geo_resolution], field_json[fields::locale]));}diff --git a/include/japanese_localizer.h b/include/japanese_localizer.hnew file mode 100644index 00000000..42f38571--- /dev/null+++ b/include/japanese_localizer.h@@ -0,0 +1,23 @@+#pragma once++#include <string>++class JapaneseLocalizer {+private:+ JapaneseLocalizer();++ ~JapaneseLocalizer() = default;++ static void write_data_file(const std::string& base64_data, const std::string& file_name);++public:++ static JapaneseLocalizer & get_instance() {+ static JapaneseLocalizer instance;+ return instance;+ }++ bool init();++ char* normalize(const std::string& text);+};diff --git a/include/tokenizer.h b/include/tokenizer.hindex 798855ba..1e19c188 100644--- a/include/tokenizer.h+++ b/include/tokenizer.h@@ -4,10 +4,12 @@#include <vector>#include <iconv.h>#include <unicode/brkiter.h>+#include "japanese_localizer.h"+#include "logger.h"class Tokenizer {private:- const std::string& text;+ std::string_view text;size_t i;const bool keep_separators;const bool normalize;@@ -27,14 +29,23 @@ private:icu::UnicodeString unicode_text;int32_t position = 0;int32_t prev_position = -1;+ char* normalized_text = nullptr;public:explicit Tokenizer(const std::string& input,bool keep_separators=true, bool normalize=true, bool no_op=false,const std::string& locale = ""):- text(input), i(0), keep_separators(keep_separators), normalize(normalize),+ i(0), keep_separators(keep_separators), normalize(normalize),no_op(no_op), locale(locale) {++ if(locale == "ja") {+ normalized_text = JapaneseLocalizer::get_instance().normalize(input);+ text = normalized_text;+ } else {+ text = input;+ }+cd = iconv_open("ASCII//TRANSLIT", "UTF-8");if(!input.empty() && (std::isalnum(text[0]) || (text[i] & ~0x7f) != 0)) {@@ -59,6 +70,7 @@ public:~Tokenizer() {iconv_close(cd);+ free(normalized_text);delete bi;}diff --git a/src/japanese_localizer.cpp b/src/japanese_localizer.cppnew file mode 100644index 00000000..5403e91e--- /dev/null+++ b/src/japanese_localizer.cpp@@ -0,0 +1,51 @@+#include "japanese_localizer.h"+#include "japanese_data.h"+#include "string_utils.h"+#include "logger.h"+#include <cstdlib>+#include <fstream>++extern "C" {+ #include "libkakasi.h"+}++void JapaneseLocalizer::write_data_file(const std::string &base64_data, const std::string &file_name) {+ const std::string& binary_str = StringUtils::base64_decode(base64_data);+ std::ofstream out(file_name, std::ios::out | std::ios::binary);+ out << binary_str;+ out.flush();+ out.close();+}++bool JapaneseLocalizer::init() {+ const std::string kanwa_data_file_path = "/tmp/kanwa.data";+ const std::string itaji_data_file_path = "/tmp/itaji.data";++ write_data_file(JA_DATA::kanwa_dict, kanwa_data_file_path);+ write_data_file(JA_DATA::itaji_dict, itaji_data_file_path);++ setenv("KANWADICTPATH", kanwa_data_file_path.c_str(), true);+ setenv("ITAIJIDICTPATH", itaji_data_file_path.c_str(), true);++ // initialize kakasi datastructures+ std::vector<std::string> arguments = {"./kakasi", "-JH", "-KH", "-s", "-iutf8", "-outf8"};+ std::vector<char*> argv;+ for (const auto& arg : arguments) {+ argv.push_back((char*)arg.data());+ }++ if (kakasi_getopt_argv(argv.size(), argv.data()) != 0) {+ LOG(ERROR) << "Kakasi initialization failed.";+ return false;+ }++ return true;+}++char* JapaneseLocalizer::normalize(const std::string& text) {+ return kakasi_do((char *)text.c_str());+}++JapaneseLocalizer::JapaneseLocalizer() {+ init();+}diff --git a/test/collection_all_fields_test.cpp b/test/collection_all_fields_test.cppindex 47285e4b..5ddc842f 100644--- a/test/collection_all_fields_test.cpp+++ b/test/collection_all_fields_test.cpp@@ -557,7 +557,7 @@ TEST_F(CollectionAllFieldsTest, JsonFieldsToFieldsConversion) {parse_op = field::json_fields_to_fields(fields_json, fallback_field_type, fields);ASSERT_FALSE(parse_op.ok());- ASSERT_EQ("Field `*` cannot contain a geo resolution.", parse_op.error());+ ASSERT_EQ("Field `.*` cannot contain a geo resolution.", parse_op.error());fields_json[0].erase(fields::geo_resolution);// reject when you try to set optional to false or facet to true@@ -565,14 +565,14 @@ TEST_F(CollectionAllFieldsTest, JsonFieldsToFieldsConversion) {parse_op = field::json_fields_to_fields(fields_json, fallback_field_type, fields);ASSERT_FALSE(parse_op.ok());- ASSERT_EQ("Field `*` must be an optional field.", parse_op.error());+ ASSERT_EQ("Field `.*` must be an optional field.", parse_op.error());fields_json[0][fields::optional] = true;fields_json[0][fields::facet] = true;parse_op = field::json_fields_to_fields(fields_json, fallback_field_type, fields);ASSERT_FALSE(parse_op.ok());- ASSERT_EQ("Field `*` cannot be a facet field.", parse_op.error());+ ASSERT_EQ("Field `.*` cannot be a facet field.", parse_op.error());fields_json[0][fields::facet] = false;@@ -594,6 +594,30 @@ TEST_F(CollectionAllFieldsTest, JsonFieldsToFieldsConversion) {ASSERT_TRUE(parse_op.ok());ASSERT_EQ("auto", fields[0].type);+ // try with locale on a regular field+ fields_json.clear();+ fields.clear();+ all_field[fields::type] = "string";+ all_field[fields::name] = "title";+ all_field[fields::locale] = "ja";+ fields_json.emplace_back(all_field);++ parse_op = field::json_fields_to_fields(fields_json, fallback_field_type, fields);+ ASSERT_TRUE(parse_op.ok());+ ASSERT_EQ("ja", fields[0].locale);++ // try with locale on fallback field+ fields_json.clear();+ fields.clear();+ all_field[fields::type] = "string";+ all_field[fields::name] = ".*";+ all_field[fields::locale] = "ko";+ fields_json.emplace_back(all_field);++ parse_op = field::json_fields_to_fields(fields_json, fallback_field_type, fields);+ ASSERT_TRUE(parse_op.ok());+ ASSERT_EQ("ko", fields[0].locale);+fields_json.clear();all_field[fields::name] = "loc";all_field[fields::type] = "geopoint";diff --git a/test/tokenizer_test.cpp b/test/tokenizer_test.cppindex 656be0de..5bf33df9 100644--- a/test/tokenizer_test.cpp+++ b/test/tokenizer_test.cpp@@ -206,6 +206,8 @@ TEST(TokenizerTest, ShouldTokenizeLocaleText) {ASSERT_EQ("Odd", tokens[0]);ASSERT_EQ("Thomas", tokens[1]);+ // korean+tokens.clear();Tokenizer("κ²½μΉμ§Β·μ°μ Β·ν곑", false, false, false, "ko").tokenize(tokens);ASSERT_EQ(3, tokens.size());@@ -219,4 +221,28 @@ TEST(TokenizerTest, ShouldTokenizeLocaleText) {ASSERT_EQ("μλ μνμ² λ", tokens[0]);ASSERT_EQ("999", tokens[1]);ASSERT_EQ("κ·Ήμ₯ν", tokens[2]);++ // japanese+ tokens.clear();+ Tokenizer("ιε±", false, false, false, "ja").tokenize(tokens);+ ASSERT_EQ(2, tokens.size());+ ASSERT_EQ("γ", tokens[0]);+ ASSERT_EQ("γγγ€", tokens[1]);++ tokens.clear();+ Tokenizer("γ’ιε±γ§γ", false, false, false, "ja").tokenize(tokens);+ ASSERT_EQ(5, tokens.size());+ ASSERT_EQ("γ", tokens[0]);+ ASSERT_EQ("γ", tokens[1]);+ ASSERT_EQ("γγγ€", tokens[2]);+ ASSERT_EQ("γ§", tokens[3]);+ ASSERT_EQ("γ", tokens[4]);++ tokens.clear();+ Tokenizer("ζ ζ°γͺη¬", false, false, false, "ja").tokenize(tokens);+ ASSERT_EQ(4, tokens.size());+ ASSERT_EQ("γγ", tokens[0]);+ ASSERT_EQ("γ ", tokens[1]);+ ASSERT_EQ("γͺ", tokens[2]);+ ASSERT_EQ("γγ¬", tokens[3]);}