Skip to content

Open Source alternative to Algolia + Pinecone and an Easier-to-Use alternative to ElasticSearch ⚑ πŸ” ✨ Fast, typo tolerant, in-memory fuzzy Search Engine for building delightful search experiences

Integrate with Kakasi.

kishorenc3a926859678a
diff --git a/CMakeLists.txt b/CMakeLists.txt
index aaef720d..b5a71859 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -56,6 +56,7 @@ include(cmake/TestResources.cmake)
include(cmake/Iconv.cmake)
include(cmake/Jemalloc.cmake)
include(cmake/h3.cmake)
+include(cmake/kakasi.cmake)
FIND_PACKAGE(OpenSSL 1.1.1 REQUIRED)
FIND_PACKAGE(Snappy REQUIRED)
@@ -69,7 +70,7 @@ FIND_PACKAGE(glog REQUIRED)
message("OpenSSL library: ${OPENSSL_LIBRARIES}")
-FILE(GLOB SRC_FILES src/*.cpp)
+FILE(GLOB SRC_FILES src/*.cpp ${DEP_ROOT_DIR}/${KAKASI_NAME}/data/*.cpp)
FILE(GLOB TEST_FILES test/*.cpp)
include_directories(include)
@@ -86,6 +87,8 @@ include_directories(${DEP_ROOT_DIR}/${BRPC_NAME}/include)
include_directories(${DEP_ROOT_DIR}/${BRAFT_NAME}/include)
include_directories(${DEP_ROOT_DIR}/${JEMALLOC_NAME}/include/jemalloc)
include_directories(${DEP_ROOT_DIR}/${H3_NAME}/build/src/h3lib/include)
+include_directories(${DEP_ROOT_DIR}/${KAKASI_NAME}/build/include)
+include_directories(${DEP_ROOT_DIR}/${KAKASI_NAME}/data)
link_directories(/usr/local/lib)
link_directories(${DEP_ROOT_DIR}/${GTEST_NAME}/googletest/build)
@@ -95,6 +98,7 @@ link_directories(${DEP_ROOT_DIR}/${ROCKSDB_NAME})
link_directories(${DEP_ROOT_DIR}/${ICONV_NAME}/lib/.libs)
link_directories(${DEP_ROOT_DIR}/${JEMALLOC_NAME}/lib)
link_directories(${DEP_ROOT_DIR}/${H3_NAME}/build/lib)
+link_directories(${DEP_ROOT_DIR}/${KAKASI_NAME}/build/lib)
set(JEMALLOC_ROOT_DIR "${DEP_ROOT_DIR}/${JEMALLOC_NAME}")
FIND_PACKAGE(Jemalloc REQUIRED)
@@ -143,7 +147,7 @@ else()
endif()
set(ICU_ALL_LIBRARIES ${ICU_I18N_LIBRARIES} ${ICU_LIBRARIES} ${ICU_DATA_LIBRARIES})
-set(CORE_LIBS h2o-evloop braft brpc iconv ${ICU_ALL_LIBRARIES} ${CURL_LIBRARIES} for h3
+set(CORE_LIBS kakasi h2o-evloop braft brpc iconv ${ICU_ALL_LIBRARIES} ${CURL_LIBRARIES} for h3
${LevelDB_LIBRARIES} ${ROCKSDB_LIBS}
glog ${GFLAGS_LIBRARIES} ${PROTOBUF_LIBRARIES} ${STACKTRACE_LIBS}
${OPENSSL_LIBRARIES} ${ZLIB_LIBRARIES} ${JEMALLOC_LIBRARIES}
diff --git a/cmake/kakasi.cmake b/cmake/kakasi.cmake
new file mode 100644
index 00000000..1a5c2a3b
--- /dev/null
+++ b/cmake/kakasi.cmake
@@ -0,0 +1,45 @@
+# Download and build libfor
+
+set(KAKASI_VERSION 9e0825a02c7ea5605e968f6208f769f7c49d6860)
+set(KAKASI_NAME kakasi-${KAKASI_VERSION})
+set(KAKASI_TAR_PATH ${DEP_ROOT_DIR}/${KAKASI_NAME}.tar.gz)
+
+if(NOT EXISTS ${KAKASI_TAR_PATH})
+ message(STATUS "Downloading kakasi...")
+ file(DOWNLOAD https://github.com/typesense/kakasi/archive/${KAKASI_VERSION}.tar.gz ${KAKASI_TAR_PATH})
+endif()
+
+if(NOT EXISTS ${DEP_ROOT_DIR}/${KAKASI_NAME})
+ message(STATUS "Extracting kakasi...")
+ execute_process(COMMAND ${CMAKE_COMMAND} -E tar xzf ${KAKASI_TAR_PATH} WORKING_DIRECTORY ${DEP_ROOT_DIR})
+endif()
+
+if(NOT EXISTS ${DEP_ROOT_DIR}/${KAKASI_NAME}/build/lib/libkakasi.a AND BUILD_DEPS STREQUAL "yes")
+ set(ENV{LIBRARY_PATH} "${DEP_ROOT_DIR}/${ICONV_NAME}/lib/.libs")
+ set(ENV{CFLAGS} "-I${DEP_ROOT_DIR}/${ICONV_NAME}/include -g")
+ set(ENV{LDFLAGS} "-g")
+
+ message("Configuring kakasi...")
+ execute_process(COMMAND ./configure
+ "--prefix=${DEP_ROOT_DIR}/${KAKASI_NAME}/build"
+ "--enable-shared=no"
+ WORKING_DIRECTORY ${DEP_ROOT_DIR}/${KAKASI_NAME}/ RESULT_VARIABLE KAKASI_CONFIGURE)
+ if(NOT KAKASI_CONFIGURE EQUAL 0)
+ message(FATAL_ERROR "${KAKASI_NAME} configure failed!")
+ endif()
+
+ message("Building kakasi locally...")
+
+ execute_process(COMMAND make WORKING_DIRECTORY ${DEP_ROOT_DIR}/${KAKASI_NAME}/
+ RESULT_VARIABLE KAKASI_BUILD)
+ if(NOT KAKASI_BUILD EQUAL 0)
+ message(FATAL_ERROR "${KAKASI_NAME} build failed!")
+ endif()
+
+ message("Installing kakasi locally...")
+ execute_process(COMMAND make install WORKING_DIRECTORY ${DEP_ROOT_DIR}/${KAKASI_NAME}/
+ RESULT_VARIABLE KAKASI_INSTALL)
+ if(NOT KAKASI_INSTALL EQUAL 0)
+ message(FATAL_ERROR "${KAKASI_NAME} install failed!")
+ endif()
+endif()
diff --git a/include/field.h b/include/field.h
index b48baca0..1dae7278 100644
--- a/include/field.h
+++ b/include/field.h
@@ -300,9 +300,9 @@ struct field {
field_json[fields::name].get<std::string>() + std::string("` should be a string."));
}
- if(!field_json[fields::locale].empty() || field_json[fields::locale] != "en" ||
- field_json[fields::locale] != "ja" || field_json[fields::locale] != "ko" ||
- field_json[fields::locale] != "zh" || field_json[fields::locale] != "th") {
+ if(!field_json[fields::locale].get<std::string>().empty() && field_json[fields::locale] != "en" &&
+ field_json[fields::locale] != "ja" && field_json[fields::locale] != "ko" &&
+ field_json[fields::locale] != "zh" && field_json[fields::locale] != "th") {
return Option<bool>(400, std::string("The `locale` value of the field `") +
field_json[fields::name].get<std::string>() + std::string("` is not valid."));
}
@@ -321,25 +321,29 @@ struct field {
field_json[fields::index] = true;
}
+ if(field_json.count(fields::locale) == 0) {
+ field_json[fields::locale] = "";
+ }
+
if(field_json.count(fields::geo_resolution) != 0) {
- return Option<bool>(400, "Field `*` cannot contain a geo resolution.");
+ return Option<bool>(400, "Field `.*` cannot contain a geo resolution.");
}
if(field_json[fields::optional] == false) {
- return Option<bool>(400, "Field `*` must be an optional field.");
+ return Option<bool>(400, "Field `.*` must be an optional field.");
}
if(field_json[fields::facet] == true) {
- return Option<bool>(400, "Field `*` cannot be a facet field.");
+ return Option<bool>(400, "Field `.*` cannot be a facet field.");
}
if(field_json[fields::index] == false) {
- return Option<bool>(400, "Field `*` must be an index field.");
+ return Option<bool>(400, "Field `.*` must be an index field.");
}
-
field fallback_field(field_json["name"], field_json["type"], field_json["facet"],
- field_json["optional"]);
+ field_json["optional"], field_json[fields::index],
+ DEFAULT_GEO_RESOLUTION, field_json[fields::locale]);
if(fallback_field.has_valid_type()) {
fallback_field_type = fallback_field.type;
@@ -360,6 +364,10 @@ struct field {
field_json[fields::index] = true;
}
+ if(field_json.count(fields::locale) == 0) {
+ field_json[fields::locale] = "";
+ }
+
bool is_dynamic = field_json[fields::name].get<std::string>().find(".*") != std::string::npos;
if(field_json.count(fields::optional) == 0) {
@@ -373,7 +381,8 @@ struct field {
fields.emplace_back(
field(field_json[fields::name], field_json[fields::type], field_json[fields::facet],
- field_json[fields::optional], field_json[fields::index], field_json[fields::geo_resolution])
+ field_json[fields::optional], field_json[fields::index],
+ field_json[fields::geo_resolution], field_json[fields::locale])
);
}
diff --git a/include/japanese_localizer.h b/include/japanese_localizer.h
new file mode 100644
index 00000000..42f38571
--- /dev/null
+++ b/include/japanese_localizer.h
@@ -0,0 +1,23 @@
+#pragma once
+
+#include <string>
+
+class JapaneseLocalizer {
+private:
+ JapaneseLocalizer();
+
+ ~JapaneseLocalizer() = default;
+
+ static void write_data_file(const std::string& base64_data, const std::string& file_name);
+
+public:
+
+ static JapaneseLocalizer & get_instance() {
+ static JapaneseLocalizer instance;
+ return instance;
+ }
+
+ bool init();
+
+ char* normalize(const std::string& text);
+};
diff --git a/include/tokenizer.h b/include/tokenizer.h
index 798855ba..1e19c188 100644
--- a/include/tokenizer.h
+++ b/include/tokenizer.h
@@ -4,10 +4,12 @@
#include <vector>
#include <iconv.h>
#include <unicode/brkiter.h>
+#include "japanese_localizer.h"
+#include "logger.h"
class Tokenizer {
private:
- const std::string& text;
+ std::string_view text;
size_t i;
const bool keep_separators;
const bool normalize;
@@ -27,14 +29,23 @@ private:
icu::UnicodeString unicode_text;
int32_t position = 0;
int32_t prev_position = -1;
+ char* normalized_text = nullptr;
public:
explicit Tokenizer(const std::string& input,
bool keep_separators=true, bool normalize=true, bool no_op=false,
const std::string& locale = ""):
- text(input), i(0), keep_separators(keep_separators), normalize(normalize),
+ i(0), keep_separators(keep_separators), normalize(normalize),
no_op(no_op), locale(locale) {
+
+ if(locale == "ja") {
+ normalized_text = JapaneseLocalizer::get_instance().normalize(input);
+ text = normalized_text;
+ } else {
+ text = input;
+ }
+
cd = iconv_open("ASCII//TRANSLIT", "UTF-8");
if(!input.empty() && (std::isalnum(text[0]) || (text[i] & ~0x7f) != 0)) {
@@ -59,6 +70,7 @@ public:
~Tokenizer() {
iconv_close(cd);
+ free(normalized_text);
delete bi;
}
diff --git a/src/japanese_localizer.cpp b/src/japanese_localizer.cpp
new file mode 100644
index 00000000..5403e91e
--- /dev/null
+++ b/src/japanese_localizer.cpp
@@ -0,0 +1,51 @@
+#include "japanese_localizer.h"
+#include "japanese_data.h"
+#include "string_utils.h"
+#include "logger.h"
+#include <cstdlib>
+#include <fstream>
+
+extern "C" {
+ #include "libkakasi.h"
+}
+
+void JapaneseLocalizer::write_data_file(const std::string &base64_data, const std::string &file_name) {
+ const std::string& binary_str = StringUtils::base64_decode(base64_data);
+ std::ofstream out(file_name, std::ios::out | std::ios::binary);
+ out << binary_str;
+ out.flush();
+ out.close();
+}
+
+bool JapaneseLocalizer::init() {
+ const std::string kanwa_data_file_path = "/tmp/kanwa.data";
+ const std::string itaji_data_file_path = "/tmp/itaji.data";
+
+ write_data_file(JA_DATA::kanwa_dict, kanwa_data_file_path);
+ write_data_file(JA_DATA::itaji_dict, itaji_data_file_path);
+
+ setenv("KANWADICTPATH", kanwa_data_file_path.c_str(), true);
+ setenv("ITAIJIDICTPATH", itaji_data_file_path.c_str(), true);
+
+ // initialize kakasi datastructures
+ std::vector<std::string> arguments = {"./kakasi", "-JH", "-KH", "-s", "-iutf8", "-outf8"};
+ std::vector<char*> argv;
+ for (const auto& arg : arguments) {
+ argv.push_back((char*)arg.data());
+ }
+
+ if (kakasi_getopt_argv(argv.size(), argv.data()) != 0) {
+ LOG(ERROR) << "Kakasi initialization failed.";
+ return false;
+ }
+
+ return true;
+}
+
+char* JapaneseLocalizer::normalize(const std::string& text) {
+ return kakasi_do((char *)text.c_str());
+}
+
+JapaneseLocalizer::JapaneseLocalizer() {
+ init();
+}
diff --git a/test/collection_all_fields_test.cpp b/test/collection_all_fields_test.cpp
index 47285e4b..5ddc842f 100644
--- a/test/collection_all_fields_test.cpp
+++ b/test/collection_all_fields_test.cpp
@@ -557,7 +557,7 @@ TEST_F(CollectionAllFieldsTest, JsonFieldsToFieldsConversion) {
parse_op = field::json_fields_to_fields(fields_json, fallback_field_type, fields);
ASSERT_FALSE(parse_op.ok());
- ASSERT_EQ("Field `*` cannot contain a geo resolution.", parse_op.error());
+ ASSERT_EQ("Field `.*` cannot contain a geo resolution.", parse_op.error());
fields_json[0].erase(fields::geo_resolution);
// reject when you try to set optional to false or facet to true
@@ -565,14 +565,14 @@ TEST_F(CollectionAllFieldsTest, JsonFieldsToFieldsConversion) {
parse_op = field::json_fields_to_fields(fields_json, fallback_field_type, fields);
ASSERT_FALSE(parse_op.ok());
- ASSERT_EQ("Field `*` must be an optional field.", parse_op.error());
+ ASSERT_EQ("Field `.*` must be an optional field.", parse_op.error());
fields_json[0][fields::optional] = true;
fields_json[0][fields::facet] = true;
parse_op = field::json_fields_to_fields(fields_json, fallback_field_type, fields);
ASSERT_FALSE(parse_op.ok());
- ASSERT_EQ("Field `*` cannot be a facet field.", parse_op.error());
+ ASSERT_EQ("Field `.*` cannot be a facet field.", parse_op.error());
fields_json[0][fields::facet] = false;
@@ -594,6 +594,30 @@ TEST_F(CollectionAllFieldsTest, JsonFieldsToFieldsConversion) {
ASSERT_TRUE(parse_op.ok());
ASSERT_EQ("auto", fields[0].type);
+ // try with locale on a regular field
+ fields_json.clear();
+ fields.clear();
+ all_field[fields::type] = "string";
+ all_field[fields::name] = "title";
+ all_field[fields::locale] = "ja";
+ fields_json.emplace_back(all_field);
+
+ parse_op = field::json_fields_to_fields(fields_json, fallback_field_type, fields);
+ ASSERT_TRUE(parse_op.ok());
+ ASSERT_EQ("ja", fields[0].locale);
+
+ // try with locale on fallback field
+ fields_json.clear();
+ fields.clear();
+ all_field[fields::type] = "string";
+ all_field[fields::name] = ".*";
+ all_field[fields::locale] = "ko";
+ fields_json.emplace_back(all_field);
+
+ parse_op = field::json_fields_to_fields(fields_json, fallback_field_type, fields);
+ ASSERT_TRUE(parse_op.ok());
+ ASSERT_EQ("ko", fields[0].locale);
+
fields_json.clear();
all_field[fields::name] = "loc";
all_field[fields::type] = "geopoint";
diff --git a/test/tokenizer_test.cpp b/test/tokenizer_test.cpp
index 656be0de..5bf33df9 100644
--- a/test/tokenizer_test.cpp
+++ b/test/tokenizer_test.cpp
@@ -206,6 +206,8 @@ TEST(TokenizerTest, ShouldTokenizeLocaleText) {
ASSERT_EQ("Odd", tokens[0]);
ASSERT_EQ("Thomas", tokens[1]);
+ // korean
+
tokens.clear();
Tokenizer("κ²½μŠΉμ§€Β·μ‚°μ•…Β·ν˜‘κ³‘", false, false, false, "ko").tokenize(tokens);
ASSERT_EQ(3, tokens.size());
@@ -219,4 +221,28 @@ TEST(TokenizerTest, ShouldTokenizeLocaleText) {
ASSERT_EQ("μ•ˆλ…•μ€ν•˜μ² λ„", tokens[0]);
ASSERT_EQ("999", tokens[1]);
ASSERT_EQ("κ·Ήμž₯판", tokens[2]);
+
+ // japanese
+ tokens.clear();
+ Tokenizer("ι€€ε±ˆ", false, false, false, "ja").tokenize(tokens);
+ ASSERT_EQ(2, tokens.size());
+ ASSERT_EQ("た", tokens[0]);
+ ASSERT_EQ("いく぀", tokens[1]);
+
+ tokens.clear();
+ Tokenizer("γ‚’ι€€ε±ˆγ§γ‚", false, false, false, "ja").tokenize(tokens);
+ ASSERT_EQ(5, tokens.size());
+ ASSERT_EQ("あ", tokens[0]);
+ ASSERT_EQ("た", tokens[1]);
+ ASSERT_EQ("いく぀", tokens[2]);
+ ASSERT_EQ("で", tokens[3]);
+ ASSERT_EQ("あ", tokens[4]);
+
+ tokens.clear();
+ Tokenizer("ζ€ ζƒ°γͺ犬", false, false, false, "ja").tokenize(tokens);
+ ASSERT_EQ(4, tokens.size());
+ ASSERT_EQ("γŸγ„", tokens[0]);
+ ASSERT_EQ("だ", tokens[1]);
+ ASSERT_EQ("γͺ", tokens[2]);
+ ASSERT_EQ("いぬ", tokens[3]);
}