Skip to content

Open Source alternative to Algolia + Pinecone and an Easier-to-Use alternative to ElasticSearch ⚡ 🔍 ✨ Fast, typo tolerant, in-memory fuzzy Search Engine for building delightful search experiences

Introduce field level locale.

kishorencdd72e2a78c5e
diff --git a/include/collection.h b/include/collection.h
index 256100fc..abe32c55 100644
--- a/include/collection.h
+++ b/include/collection.h
@@ -491,7 +491,8 @@ public:
bool is_exceeding_memory_threshold() const;
void parse_search_query(const std::string &query, std::vector<std::string>& q_include_tokens,
- std::vector<std::string>& q_exclude_tokens) const;
+ std::vector<std::string>& q_exclude_tokens,
+ const std::string& locale) const;
// PUBLIC OPERATIONS
diff --git a/include/field.h b/include/field.h
index 9b6f21e1..b48baca0 100644
--- a/include/field.h
+++ b/include/field.h
@@ -34,6 +34,7 @@ namespace fields {
static const std::string optional = "optional";
static const std::string index = "index";
static const std::string geo_resolution = "geo_resolution";
+ static const std::string locale = "locale";
}
static const uint8_t DEFAULT_GEO_RESOLUTION = 7;
@@ -48,9 +49,13 @@ struct field {
uint8_t geo_resolution;
+ std::string locale;
+
field(const std::string &name, const std::string &type, const bool facet, const bool optional = false,
- bool index = true, const uint8_t geo_resolution = DEFAULT_GEO_RESOLUTION) :
- name(name), type(type), facet(facet), optional(optional), index(index), geo_resolution(geo_resolution) {
+ bool index = true, const uint8_t geo_resolution = DEFAULT_GEO_RESOLUTION,
+ std::string locale = "") :
+ name(name), type(type), facet(facet), optional(optional), index(index),
+ geo_resolution(geo_resolution), locale(locale) {
}
@@ -199,6 +204,8 @@ struct field {
field_val[fields::geo_resolution] = field.geo_resolution;
}
+ field_val[fields::locale] = field.locale;
+
fields_json.push_back(field_val);
if(!field.has_valid_type()) {
@@ -287,6 +294,20 @@ struct field {
}
}
+ if(field_json.count(fields::locale) != 0){
+ if(!field_json.at(fields::locale).is_string()) {
+ return Option<bool>(400, std::string("The `locale` property of the field `") +
+ field_json[fields::name].get<std::string>() + std::string("` should be a string."));
+ }
+
+ if(!field_json[fields::locale].empty() || field_json[fields::locale] != "en" ||
+ field_json[fields::locale] != "ja" || field_json[fields::locale] != "ko" ||
+ field_json[fields::locale] != "zh" || field_json[fields::locale] != "th") {
+ return Option<bool>(400, std::string("The `locale` value of the field `") +
+ field_json[fields::name].get<std::string>() + std::string("` is not valid."));
+ }
+ }
+
if(field_json["name"] == ".*") {
if(field_json.count(fields::facet) == 0) {
field_json[fields::facet] = false;
diff --git a/include/index.h b/include/index.h
index fcacae77..ade60052 100644
--- a/include/index.h
+++ b/include/index.h
@@ -36,10 +36,14 @@ struct search_field_t {
size_t weight;
};
-struct search_args {
+struct query_tokens_t {
std::vector<std::string> q_include_tokens;
std::vector<std::string> q_exclude_tokens;
std::vector<std::vector<std::string>> q_synonyms;
+};
+
+struct search_args {
+ std::vector<query_tokens_t> field_query_tokens;
std::vector<search_field_t> search_fields;
std::vector<filter> filters;
std::vector<facet> facets;
@@ -70,9 +74,7 @@ struct search_args {
}
- search_args(std::vector<std::string> q_include_tokens,
- std::vector<std::string> q_exclude_tokens,
- std::vector<std::vector<std::string>> q_synonyms,
+ search_args(std::vector<query_tokens_t> field_query_tokens,
std::vector<search_field_t> search_fields, std::vector<filter> filters,
std::vector<facet> facets, std::map<size_t, std::map<size_t, uint32_t>> included_ids, std::vector<uint32_t> excluded_ids,
std::vector<sort_by> sort_fields_std, facet_query_t facet_query, int num_typos, size_t max_facet_values,
@@ -80,7 +82,7 @@ struct search_args {
size_t drop_tokens_threshold, size_t typo_tokens_threshold,
const std::vector<std::string>& group_by_fields, size_t group_limit,
const std::string& default_sorting_field):
- q_include_tokens(q_include_tokens), q_exclude_tokens(q_exclude_tokens), q_synonyms(q_synonyms),
+ field_query_tokens(field_query_tokens),
search_fields(search_fields), filters(filters), facets(facets),
included_ids(included_ids), excluded_ids(excluded_ids), sort_fields_std(sort_fields_std),
facet_query(facet_query), num_typos(num_typos), max_facet_values(max_facet_values), per_page(per_page),
@@ -344,9 +346,7 @@ public:
void run_search(search_args* search_params);
- void search(const std::vector<std::string> & q_include_tokens,
- const std::vector<std::string> & q_exclude_tokens,
- const std::vector<std::vector<std::string>>& q_synonyms,
+ void search(const std::vector<query_tokens_t>& field_query_tokens,
const std::vector<search_field_t> & search_fields,
const std::vector<filter> & filters, std::vector<facet> & facets,
facet_query_t & facet_query,
diff --git a/include/tokenizer.h b/include/tokenizer.h
index 985d59e8..798855ba 100644
--- a/include/tokenizer.h
+++ b/include/tokenizer.h
@@ -3,6 +3,7 @@
#include <string>
#include <vector>
#include <iconv.h>
+#include <unicode/brkiter.h>
class Tokenizer {
private:
@@ -21,11 +22,19 @@ private:
std::stringstream out;
+ std::string locale;
+ icu::BreakIterator* bi = nullptr;
+ icu::UnicodeString unicode_text;
+ int32_t position = 0;
+ int32_t prev_position = -1;
+
public:
explicit Tokenizer(const std::string& input,
- bool keep_separators=true, bool normalize=true, bool no_op=false):
- text(input), i(0), keep_separators(keep_separators), normalize(normalize), no_op(no_op) {
+ bool keep_separators=true, bool normalize=true, bool no_op=false,
+ const std::string& locale = ""):
+ text(input), i(0), keep_separators(keep_separators), normalize(normalize),
+ no_op(no_op), locale(locale) {
cd = iconv_open("ASCII//TRANSLIT", "UTF-8");
if(!input.empty() && (std::isalnum(text[0]) || (text[i] & ~0x7f) != 0)) {
@@ -34,10 +43,23 @@ public:
} else {
stream_mode = SEPARATORS;
}
+
+ if(!locale.empty() && locale != "en") {
+ UErrorCode status = U_ZERO_ERROR;
+ const icu::Locale& icu_locale = icu::Locale(locale.c_str());
+ bi = icu::BreakIterator::createWordInstance(icu_locale, status);
+
+ unicode_text = icu::UnicodeString::fromUTF8(text);
+ bi->setText(unicode_text);
+
+ position = bi->first();
+ prev_position = -1;
+ }
}
~Tokenizer() {
iconv_close(cd);
+ delete bi;
}
bool next(std::string& token, size_t& token_index);
diff --git a/src/collection.cpp b/src/collection.cpp
index 9a111b69..05268e26 100644
--- a/src/collection.cpp
+++ b/src/collection.cpp
@@ -819,15 +819,26 @@ Option<nlohmann::json> Collection::search(const std::string & query, const std::
spp::sparse_hash_set<uint64_t> groups_processed; // used to calculate total_found for grouped query
//LOG(INFO) << "Num indices used for querying: " << indices.size();
+ std::vector<query_tokens_t> field_query_tokens;
- std::vector<std::string> q_include_tokens;
- std::vector<std::string> q_exclude_tokens;
+ if(search_fields.size() == 0) {
+ // has to be a wildcard query
+ field_query_tokens.emplace_back(query_tokens_t{});
+ parse_search_query(query, field_query_tokens[0].q_include_tokens, field_query_tokens[0].q_exclude_tokens,"");
+ } else {
+ for(size_t i = 0; i < search_fields.size(); i++) {
+ const auto& search_field = search_fields[i];
+ field_query_tokens.emplace_back(query_tokens_t{});
- parse_search_query(query, q_include_tokens, q_exclude_tokens);
+ const std::string & field_locale = search_schema.at(search_field).locale;
+ parse_search_query(query, field_query_tokens[i].q_include_tokens, field_query_tokens[i].q_exclude_tokens,
+ field_locale);
- // get synonyms
- std::vector<std::vector<std::string>> q_synonyms;
- synonym_reduction(q_include_tokens, q_synonyms);
+ // get synonyms
+ std::vector<std::vector<std::string>> q_synonyms;
+ synonym_reduction(field_query_tokens[i].q_include_tokens, field_query_tokens[i].q_synonyms);
+ }
+ }
// search all indices
std::vector<search_args*> search_args_vec;
@@ -837,7 +848,7 @@ Option<nlohmann::json> Collection::search(const std::string & query, const std::
size_t index_id = 0;
for(Index* index: indices) {
- search_args* search_params = new search_args(q_include_tokens, q_exclude_tokens, q_synonyms, weighted_search_fields,
+ search_args* search_params = new search_args(field_query_tokens, weighted_search_fields,
filters, facets, index_to_included_ids[index_id],
index_to_excluded_ids[index_id],
sort_fields_std, facet_query, num_typos, max_facet_values, max_hits,
@@ -1217,13 +1228,14 @@ Option<nlohmann::json> Collection::search(const std::string & query, const std::
}
void Collection::parse_search_query(const std::string &query, std::vector<std::string>& q_include_tokens,
- std::vector<std::string>& q_exclude_tokens) const {
+ std::vector<std::string>& q_exclude_tokens,
+ const std::string& locale) const {
if(query == "*") {
q_exclude_tokens = {};
q_include_tokens = {query};
} else {
std::vector<std::string> tokens;
- Tokenizer(query, true, true).tokenize(tokens);
+ Tokenizer(query, true, true, false, locale).tokenize(tokens);
bool exclude_operator_prior = false;
for(const auto& token: tokens) {
@@ -1235,7 +1247,9 @@ void Collection::parse_search_query(const std::string &query, std::vector<std::s
exclude_operator_prior = true;
}
- if(!std::isalnum(token[0])) {
+ bool is_ascii = (token[0] & ~0x7f) == 0;
+
+ if(is_ascii && !std::isalnum(token[0])) {
continue;
}
@@ -1447,7 +1461,7 @@ void Collection::highlight_result(const field &search_field,
const Match& match = match_index.match;
const std::string& text = (search_field.type == field_types::STRING) ? document[search_field.name] : document[search_field.name][match_index.index];
- Tokenizer tokenizer(text, true, false);
+ Tokenizer tokenizer(text, true, false, false, search_field.locale);
std::string raw_token;
size_t raw_token_index = 0;
diff --git a/src/collection_manager.cpp b/src/collection_manager.cpp
index acf2482e..02bedb2b 100644
--- a/src/collection_manager.cpp
+++ b/src/collection_manager.cpp
@@ -36,8 +36,13 @@ Collection* CollectionManager::init_collection(const nlohmann::json & collection
field_obj[fields::geo_resolution] = size_t(DEFAULT_GEO_RESOLUTION);
}
+ if(field_obj.count(fields::locale) == 0) {
+ field_obj[fields::locale] = "";
+ }
+
fields.push_back({field_obj[fields::name], field_obj[fields::type], field_obj[fields::facet],
- field_obj[fields::optional], field_obj[fields::index], field_obj[fields::geo_resolution]});
+ field_obj[fields::optional], field_obj[fields::index],
+ field_obj[fields::geo_resolution], field_obj[fields::locale]});
}
std::string default_sorting_field = collection_meta[Collection::COLLECTION_DEFAULT_SORTING_FIELD_KEY].get<std::string>();
diff --git a/src/index.cpp b/src/index.cpp
index 2054838b..c3fe1b69 100644
--- a/src/index.cpp
+++ b/src/index.cpp
@@ -575,7 +575,7 @@ void Index::index_string_field(const std::string & text, const int64_t score, ar
uint32_t seq_id, bool is_facet, const field & a_field) {
std::unordered_map<std::string, std::vector<uint32_t>> token_to_offsets;
- Tokenizer tokenizer(text, false, true, !a_field.is_string());
+ Tokenizer tokenizer(text, false, true, !a_field.is_string(), a_field.locale);
std::string token;
size_t token_index = 0;
@@ -622,7 +622,7 @@ void Index::index_string_array_field(const std::vector<std::string> & strings, c
const std::string& str = strings[array_index];
std::set<std::string> token_set; // required to deal with repeating tokens
- Tokenizer tokenizer(str, false, true, !a_field.is_string());
+ Tokenizer tokenizer(str, false, true, !a_field.is_string(), a_field.locale);
std::string token;
size_t token_index = 0;
@@ -1246,7 +1246,7 @@ uint32_t Index::do_filtering(uint32_t** filter_ids_out, const std::vector<filter
// there could be multiple tokens in a filter value, which we have to treat as ANDs
// e.g. country: South Africa
- Tokenizer tokenizer(filter_value, false, true);
+ Tokenizer tokenizer(filter_value, false, true, false, f.locale);
std::string str_token;
size_t token_index = 0;
std::vector<std::string> str_tokens;
@@ -1420,8 +1420,7 @@ void Index::eq_str_filter_plain(const uint32_t *strt_ids, size_t strt_ids_size,
}
void Index::run_search(search_args* search_params) {
- search(search_params->q_include_tokens, search_params->q_exclude_tokens,
- search_params->q_synonyms,
+ search(search_params->field_query_tokens,
search_params->search_fields,
search_params->filters, search_params->facets, search_params->facet_query,
search_params->included_ids, search_params->excluded_ids,
@@ -1506,9 +1505,7 @@ void Index::concat_topster_ids(Topster* topster, spp::sparse_hash_map<uint64_t,
}
}
-void Index::search(const std::vector<std::string>& q_include_tokens,
- const std::vector<std::string>& q_exclude_tokens,
- const std::vector<std::vector<std::string>>& q_synonyms,
+void Index::search(const std::vector<query_tokens_t>& field_query_tokens,
const std::vector<search_field_t>& search_fields,
const std::vector<filter>& filters,
std::vector<facet>& facets, facet_query_t& facet_query,
@@ -1565,7 +1562,7 @@ void Index::search(const std::vector<std::string>& q_include_tokens,
// find documents that contain the excluded tokens to exclude them from results later
for(size_t i = 0; i < num_search_fields; i++) {
const std::string & field_name = search_fields[i].name;
- for(const std::string& exclude_token: q_exclude_tokens) {
+ for(const std::string& exclude_token: field_query_tokens[i].q_exclude_tokens) {
art_leaf* leaf = (art_leaf *) art_search(search_index.at(field_name),
(const unsigned char *) exclude_token.c_str(),
exclude_token.size() + 1);
@@ -1585,7 +1582,8 @@ void Index::search(const std::vector<std::string>& q_include_tokens,
std::vector<Topster*> ftopsters;
- if(!q_include_tokens.empty() && q_include_tokens[0] == "*") {
+ if (!field_query_tokens.empty() && !field_query_tokens[0].q_include_tokens.empty() &&
+ field_query_tokens[0].q_include_tokens[0] == "*") {
const uint8_t field_id = (uint8_t)(FIELD_LIMIT_NUM - 0);
const std::string& field = search_fields[0].name;
@@ -1629,7 +1627,7 @@ void Index::search(const std::vector<std::string>& q_include_tokens,
uint32_t token_bits = 255;
score_results(sort_fields_std, (uint16_t) searched_queries.size(), field_id, 0, topster, {},
groups_processed, filter_ids, filter_ids_length, group_limit, group_by_fields, token_bits);
- collate_included_ids(q_include_tokens, field, field_id, included_ids_map, curated_topster, searched_queries);
+ collate_included_ids(field_query_tokens[0].q_include_tokens, field, field_id, included_ids_map, curated_topster, searched_queries);
all_result_ids_len = filter_ids_length;
all_result_ids = filter_ids;
@@ -1638,45 +1636,45 @@ void Index::search(const std::vector<std::string>& q_include_tokens,
// In multi-field searches, a record can be matched across different fields, so we use this for aggregation
spp::sparse_hash_map<uint64_t, std::vector<KV*>> topster_ids;
- std::vector<token_t> q_include_pos_tokens;
- for(size_t i=0; i < q_include_tokens.size(); i++) {
- q_include_pos_tokens.push_back({i, q_include_tokens[i]});
- }
-
- std::vector<std::vector<token_t>> q_pos_synonyms;
- for(const auto& q_syn_vec: q_synonyms) {
- std::vector<token_t> q_pos_syn;
- for(size_t i=0; i < q_syn_vec.size(); i++) {
- q_pos_syn.push_back({i, q_syn_vec[i]});
- }
- q_pos_synonyms.emplace_back(q_pos_syn);
- }
-
//begin = std::chrono::high_resolution_clock::now();
// non-wildcard
for(size_t i = 0; i < num_search_fields; i++) {
+ std::vector<token_t> q_include_pos_tokens;
+ for(size_t j=0; j < field_query_tokens[i].q_include_tokens.size(); j++) {
+ q_include_pos_tokens.push_back({j, field_query_tokens[i].q_include_tokens[j]});
+ }
+
+ std::vector<std::vector<token_t>> q_pos_synonyms;
+ for(const auto& q_syn_vec: field_query_tokens[i].q_synonyms) {
+ std::vector<token_t> q_pos_syn;
+ for(size_t j=0; j < q_syn_vec.size(); j++) {
+ q_pos_syn.push_back({j, q_syn_vec[j]});
+ }
+ q_pos_synonyms.emplace_back(q_pos_syn);
+ }
+
// proceed to query search only when no filters are provided or when filtering produces results
if(filters.empty() || filter_ids_length > 0) {
const uint8_t field_id = (uint8_t)(FIELD_LIMIT_NUM - i);
- const std::string& field = search_fields[i].name;
+ const std::string& field_name = search_fields[i].name;
std::vector<token_t> query_tokens = q_include_pos_tokens;
std::vector<token_t> search_tokens = q_include_pos_tokens;
size_t num_tokens_dropped = 0;
- //LOG(INFO) << "searching field! " << field;
+ //LOG(INFO) << "searching field_name! " << field_name;
Topster* ftopster = new Topster(topster->MAX_SIZE, topster->distinct);
ftopsters.push_back(ftopster);
- // Don't waste additional cycles for single field searches
+ // Don't waste additional cycles for single field_name searches
Topster* actual_topster = (num_search_fields == 1) ? topster : ftopster;
- // tracks the number of results found for the current field
+ // tracks the number of results found for the current field_name
size_t field_num_results = 0;
search_field(field_id, query_tokens, search_tokens, exclude_token_ids, exclude_token_ids_size, num_tokens_dropped,
- field, filter_ids, filter_ids_length, curated_ids_sorted, facets, sort_fields_std,
+ field_name, filter_ids, filter_ids_length, curated_ids_sorted, facets, sort_fields_std,
num_typos, searched_queries, actual_topster, groups_processed, &all_result_ids, all_result_ids_len,
field_num_results, group_limit, group_by_fields, token_order, prefix,
drop_tokens_threshold, typo_tokens_threshold);
@@ -1688,14 +1686,14 @@ void Index::search(const std::vector<std::string>& q_include_tokens,
query_tokens = search_tokens = syn_tokens;
search_field(field_id, query_tokens, search_tokens, exclude_token_ids, exclude_token_ids_size, num_tokens_dropped,
- field, filter_ids, filter_ids_length, curated_ids_sorted, facets, sort_fields_std,
+ field_name, filter_ids, filter_ids_length, curated_ids_sorted, facets, sort_fields_std,
num_typos, searched_queries, actual_topster, groups_processed, &all_result_ids, all_result_ids_len,
field_num_results, group_limit, group_by_fields, token_order, prefix,
drop_tokens_threshold, typo_tokens_threshold);
}
concat_topster_ids(ftopster, topster_ids);
- collate_included_ids(q_include_tokens, field, field_id, included_ids_map, curated_topster, searched_queries);
+ collate_included_ids(field_query_tokens[i].q_include_tokens, field_name, field_id, included_ids_map, curated_topster, searched_queries);
//LOG(INFO) << "topster_ids.size: " << topster_ids.size();
}
}
@@ -1738,7 +1736,7 @@ void Index::search(const std::vector<std::string>& q_include_tokens,
int64_t field_typos = 255 - ((match_score >> 8) & 0xFF);
total_typos += (field_typos + 1) * weight;
- if(field_typos == 0 && tokens_found == q_include_tokens.size()) {
+ if(field_typos == 0 && tokens_found == field_query_tokens[i].q_include_tokens.size()) {
num_exact_matches++;
}
@@ -1755,11 +1753,11 @@ void Index::search(const std::vector<std::string>& q_include_tokens,
size_t words_present = 0;
- for(size_t token_index=0; token_index < q_include_tokens.size(); token_index++) {
- const auto& token = q_include_tokens[token_index];
+ for(size_t token_index=0; token_index < field_query_tokens[i].q_include_tokens.size(); token_index++) {
+ const auto& token = field_query_tokens[i].q_include_tokens[token_index];
std::vector<art_leaf*> leaves;
- const bool prefix_search = prefix && (token_index == q_include_tokens.size()-1);
+ const bool prefix_search = prefix && (token_index == field_query_tokens[i].q_include_tokens.size()-1);
const size_t token_len = prefix_search ? (int) token.length() : (int) token.length() + 1;
art_fuzzy_search(search_index.at(field), (const unsigned char *) token.c_str(), token_len,
0, 0, 1, token_order, prefix_search, nullptr, 0, leaves);
@@ -1791,7 +1789,7 @@ void Index::search(const std::vector<std::string>& q_include_tokens,
uint64_t field_typos = 255 - ((match_score >> 8) & 0xFF);
total_typos += (field_typos + 1) * weight;
- if(field_typos == 0 && tokens_found == q_include_tokens.size()) {
+ if(field_typos == 0 && tokens_found == field_query_tokens[i].q_include_tokens.size()) {
num_exact_matches++;
}
//LOG(INFO) << "seq_id: " << seq_id << ", total_typos: " << ((match_score >> 8) & 0xFF);
@@ -1865,7 +1863,11 @@ void Index::search_field(const uint8_t & field_id,
const token_ordering token_order, const bool prefix,
const size_t drop_tokens_threshold, const size_t typo_tokens_threshold) const {
- const size_t max_cost = (num_typos < 0 || num_typos > 2) ? 2 : num_typos;
+ size_t max_cost = (num_typos < 0 || num_typos > 2) ? 2 : num_typos;
+ if(search_schema.at(field).locale != "" && search_schema.at(field).locale != "en") {
+ // disable fuzzy trie traversal for non-english locales
+ max_cost = 0;
+ }
// To prevent us from doing ART search repeatedly as we iterate through possible corrections
spp::sparse_hash_map<std::string, std::vector<art_leaf*>> token_cost_cache;
diff --git a/src/tokenizer.cpp b/src/tokenizer.cpp
index ac99a2bb..71fcc26e 100644
--- a/src/tokenizer.cpp
+++ b/src/tokenizer.cpp
@@ -2,6 +2,39 @@
#include "tokenizer.h"
bool Tokenizer::next(std::string &token, size_t& token_index) {
+ if(!locale.empty() && locale != "en") {
+ while (position != icu::BreakIterator::DONE) {
+ //LOG(INFO) << "Position: " << position;
+ bool found_token = false;
+
+ if(prev_position != -1) {
+ std::string word;
+ size_t length = position - prev_position;
+ token = unicode_text.tempSubString(prev_position, length).toUTF8String(word);
+
+ if(!token.empty()) {
+ if (!keep_separators && !std::isalnum(token[0]) && (token[i] & ~0x7f) == 0) {
+ found_token = false;
+ } else if(locale == "ko" && token == "·") {
+ found_token = false;
+ } else {
+ found_token = true;
+ token_index = token_counter++;
+ }
+ }
+ }
+
+ prev_position = position;
+ position = bi->next();
+
+ if(found_token) {
+ return true;
+ }
+ }
+
+ return false;
+ }
+
if(no_op) {
if(i == text.size()) {
return false;
diff --git a/test/collection_locale_test.cpp b/test/collection_locale_test.cpp
new file mode 100644
index 00000000..6bde3362
--- /dev/null
+++ b/test/collection_locale_test.cpp
@@ -0,0 +1,131 @@
+#include <gtest/gtest.h>
+#include <collection.h>
+#include <collection_manager.h>
+
+class CollectionLocaleTest : public ::testing::Test {
+protected:
+ Store *store;
+ CollectionManager & collectionManager = CollectionManager::get_instance();
+
+ std::vector<std::string> query_fields;
+ std::vector<sort_by> sort_fields;
+
+ void setupCollection() {
+ std::string state_dir_path = "/tmp/typesense_test/collection_locale";
+ LOG(INFO) << "Truncating and creating: " << state_dir_path;
+ system(("rm -rf "+state_dir_path+" && mkdir -p "+state_dir_path).c_str());
+
+ store = new Store(state_dir_path);
+ collectionManager.init(store, 1.0, "auth_key");
+ collectionManager.load(8, 1000);
+ }
+
+ virtual void SetUp() {
+ setupCollection();
+ }
+
+ virtual void TearDown() {
+ collectionManager.dispose();
+ delete store;
+ }
+};
+
+TEST_F(CollectionLocaleTest, SearchAgainstThaiText) {
+ Collection *coll1;
+
+ std::vector<field> fields = {field("title", field_types::STRING, false, false, true, DEFAULT_GEO_RESOLUTION, "th"),
+ field("artist", field_types::STRING, false),
+ field("points", field_types::INT32, false),};
+
+ coll1 = collectionManager.get_collection("coll1").get();
+ if(coll1 == nullptr) {
+ coll1 = collectionManager.create_collection("coll1", 1, fields, "points").get();
+ }
+
+ std::vector<std::vector<std::string>> records = {
+ {"ลงที่นั่นโดยรถไฟ", "Dustin Kensrue"},
+ {"พกติดตัวเสมอ", "Gord Downie"},
+ {"พกไฟ\nเสมอ", "Dustin Kensrue"},
+ };
+
+ for(size_t i=0; i<records.size(); i++) {
+ nlohmann::json doc;
+
+ doc["id"] = std::to_string(i);
+ doc["title"] = records[i][0];
+ doc["artist"] = records[i][1];
+ doc["points"] = i;
+
+ ASSERT_TRUE(coll1->add(doc.dump()).ok());
+ }
+
+ auto results = coll1->search("ลงรถไฟ",
+ {"title"}, "", {}, {}, 0, 10, 1, FREQUENCY).get();
+
+ ASSERT_EQ(1, results["found"].get<size_t>());
+ ASSERT_EQ(1, results["hits"].size());
+ ASSERT_EQ("0", results["hits"][0]["document"]["id"].get<std::string>());
+ ASSERT_EQ("<mark>ลง</mark>ที่นั่นโดย<mark>รถไฟ</mark>", results["hits"][0]["highlights"][0]["snippet"].get<std::string>());
+
+ results = coll1->search("ลงรถไฟ downie",
+ {"title", "artist"}, "", {}, {}, 0, 10, 1, FREQUENCY).get();
+
+ ASSERT_EQ(2, results["found"].get<size_t>());
+ ASSERT_EQ(2, results["hits"].size());
+ ASSERT_EQ("0", results["hits"][0]["document"]["id"].get<std::string>());
+ ASSERT_EQ("<mark>ลง</mark>ที่นั่นโดย<mark>รถไฟ</mark>", results["hits"][0]["highlights"][0]["snippet"].get<std::string>());
+
+ ASSERT_EQ("1", results["hits"][1]["document"]["id"].get<std::string>());
+ ASSERT_EQ("Gord <mark>Downie</mark>", results["hits"][1]["highlights"][0]["snippet"].get<std::string>());
+
+ results = coll1->search("พกไฟ", {"title", "artist"}, "", {}, {}, 0, 10, 1, FREQUENCY).get();
+ ASSERT_EQ(1, results["found"].get<size_t>());
+ ASSERT_EQ(1, results["hits"].size());
+ ASSERT_EQ("2", results["hits"][0]["document"]["id"].get<std::string>());
+ ASSERT_EQ("<mark>พกไฟ</mark>\nเสมอ", results["hits"][0]["highlights"][0]["snippet"].get<std::string>());
+}
+
+TEST_F(CollectionLocaleTest, SearchAgainstKoreanText) {
+ Collection *coll1;
+
+ std::vector<field> fields = {field("title", field_types::STRING, false, false, true, DEFAULT_GEO_RESOLUTION, "th"),
+ field("artist", field_types::STRING, false),
+ field("points", field_types::INT32, false),};
+
+ coll1 = collectionManager.get_collection("coll1").get();
+ if(coll1 == nullptr) {
+ coll1 = collectionManager.create_collection("coll1", 1, fields, "points").get();
+ }
+
+ std::vector<std::vector<std::string>> records = {
+ {"경승지·산악·협곡", "Dustin Kensrue"},
+ {"안녕은하철도999극장판", "Gord Downie"},
+ };
+
+ for(size_t i=0; i<records.size(); i++) {
+ nlohmann::json doc;
+
+ doc["id"] = std::to_string(i);
+ doc["title"] = records[i][0];
+ doc["artist"] = records[i][1];
+ doc["points"] = i;
+
+ ASSERT_TRUE(coll1->add(doc.dump()).ok());
+ }
+
+ auto results = coll1->search("극장판",
+ {"title"}, "", {}, {}, 0, 10, 1, FREQUENCY).get();
+
+ ASSERT_EQ(1, results["found"].get<size_t>());
+ ASSERT_EQ(1, results["hits"].size());
+ ASSERT_EQ("1", results["hits"][0]["document"]["id"].get<std::string>());
+ ASSERT_EQ("안녕은하철도999<mark>극장판</mark>", results["hits"][0]["highlights"][0]["snippet"].get<std::string>());
+
+ results = coll1->search("산악",
+ {"title", "artist"}, "", {}, {}, 0, 10, 1, FREQUENCY).get();
+
+ ASSERT_EQ(1, results["found"].get<size_t>());
+ ASSERT_EQ(1, results["hits"].size());
+ ASSERT_EQ("0", results["hits"][0]["document"]["id"].get<std::string>());
+ ASSERT_EQ("경승지·<mark>산악</mark>·협곡", results["hits"][0]["highlights"][0]["snippet"].get<std::string>());
+}
diff --git a/test/collection_manager_test.cpp b/test/collection_manager_test.cpp
index 1669f137..e1b99cb9 100644
--- a/test/collection_manager_test.cpp
+++ b/test/collection_manager_test.cpp
@@ -24,7 +24,7 @@ protected:
collectionManager.load(8, 1000);
search_fields = {
- field("title", field_types::STRING, false),
+ field("title", field_types::STRING, false, false, true, DEFAULT_GEO_RESOLUTION, "en"),
field("starring", field_types::STRING, false),
field("cast", field_types::STRING_ARRAY, true, true),
field(".*_year", field_types::INT32, true, true),
@@ -88,12 +88,12 @@ TEST_F(CollectionManagerTest, CollectionCreation) {
// we already call `collection1->get_next_seq_id` above, which is side-effecting
ASSERT_EQ(1, StringUtils::deserialize_uint32_t(next_seq_id));
ASSERT_EQ("{\"created_at\":12345,\"default_sorting_field\":\"points\",\"fallback_field_type\":\"\","
- "\"fields\":[{\"facet\":false,\"name\":\"title\",\"optional\":false,\"type\":\"string\"},"
- "{\"facet\":false,\"name\":\"starring\",\"optional\":false,\"type\":\"string\"},"
- "{\"facet\":true,\"name\":\"cast\",\"optional\":true,\"type\":\"string[]\"},"
- "{\"facet\":true,\"name\":\".*_year\",\"optional\":true,\"type\":\"int32\"},"
- "{\"facet\":false,\"geo_resolution\":14,\"name\":\"location\",\"optional\":true,\"type\":\"geopoint\"},"
- "{\"facet\":false,\"name\":\"points\",\"optional\":false,\"type\":\"int32\"}],\"id\":0,"
+ "\"fields\":[{\"facet\":false,\"locale\":\"en\",\"name\":\"title\",\"optional\":false,\"type\":\"string\"},"
+ "{\"facet\":false,\"locale\":\"\",\"name\":\"starring\",\"optional\":false,\"type\":\"string\"},"
+ "{\"facet\":true,\"locale\":\"\",\"name\":\"cast\",\"optional\":true,\"type\":\"string[]\"},"
+ "{\"facet\":true,\"locale\":\"\",\"name\":\".*_year\",\"optional\":true,\"type\":\"int32\"},"
+ "{\"facet\":false,\"geo_resolution\":14,\"locale\":\"\",\"name\":\"location\",\"optional\":true,\"type\":\"geopoint\"},"
+ "{\"facet\":false,\"locale\":\"\",\"name\":\"points\",\"optional\":false,\"type\":\"int32\"}],\"id\":0,"
"\"name\":\"collection1\",\"num_memory_shards\":4}",
collection_meta_json);
ASSERT_EQ("1", next_collection_id);
@@ -232,6 +232,7 @@ TEST_F(CollectionManagerTest, RestoreRecordsOnRestart) {
std::vector<std::string> facets;
nlohmann::json results = collection1->search("thomas", search_fields, "", facets, sort_fields, 0, 10, 1, FREQUENCY, false).get();
+ LOG(INFO) << results;
ASSERT_EQ(4, results["hits"].size());
std::unordered_map<std::string, field> schema = collection1->get_schema();
diff --git a/test/tokenizer_test.cpp b/test/tokenizer_test.cpp
index 58123d4c..656be0de 100644
--- a/test/tokenizer_test.cpp
+++ b/test/tokenizer_test.cpp
@@ -1,5 +1,6 @@
#include <gtest/gtest.h>
#include "tokenizer.h"
+#include "logger.h"
TEST(TokenizerTest, ShouldTokenizeNormalizeDifferentStrings) {
const std::string withnewline = "Michael Jordan:\nWelcome, everybody. Welcome! ";
@@ -177,3 +178,45 @@ TEST(TokenizerTest, ShouldTokenizeIteratively) {
ASSERT_EQ(1, tokens.size());
ASSERT_STREQ("Michael Jordan:\n\nWelcome, everybody. Welcome!", tokens[0].c_str());
}
+
+TEST(TokenizerTest, ShouldTokenizeLocaleText) {
+ std::string str = "จิ้งจอกสีน้ำตาลด่วน";
+ std::vector<std::string> tokens;
+ Tokenizer(str, false, false, false, "th").tokenize(tokens);
+
+ ASSERT_EQ(4, tokens.size());
+ ASSERT_EQ("จิ้งจอก", tokens[0]);
+ ASSERT_EQ("สี", tokens[1]);
+ ASSERT_EQ("น้ำตาล", tokens[2]);
+ ASSERT_EQ("ด่วน", tokens[3]);
+
+ tokens.clear();
+ str = "น. วันที่ 31 มี.ค.";
+ Tokenizer(str, false, false, false, "th").tokenize(tokens);
+ ASSERT_EQ(5, tokens.size());
+ ASSERT_EQ("น", tokens[0]);
+ ASSERT_EQ("วัน", tokens[1]);
+ ASSERT_EQ("ที่", tokens[2]);
+ ASSERT_EQ("31", tokens[3]);
+ ASSERT_EQ("มี.ค", tokens[4]);
+
+ tokens.clear();
+ Tokenizer("Odd Thomas", false, false, false, "en").tokenize(tokens);
+ ASSERT_EQ(2, tokens.size());
+ ASSERT_EQ("Odd", tokens[0]);
+ ASSERT_EQ("Thomas", tokens[1]);
+
+ tokens.clear();
+ Tokenizer("경승지·산악·협곡", false, false, false, "ko").tokenize(tokens);
+ ASSERT_EQ(3, tokens.size());
+ ASSERT_EQ("경승지", tokens[0]);
+ ASSERT_EQ("산악", tokens[1]);
+ ASSERT_EQ("협곡", tokens[2]);
+
+ tokens.clear();
+ Tokenizer("안녕은하철도999극장판", false, false, false, "ko").tokenize(tokens);
+ ASSERT_EQ(3, tokens.size());
+ ASSERT_EQ("안녕은하철도", tokens[0]);
+ ASSERT_EQ("999", tokens[1]);
+ ASSERT_EQ("극장판", tokens[2]);
+}