Open Source alternative to Algolia + Pinecone and an Easier-to-Use alternative to ElasticSearch ⚡ 🔍 ✨ Fast, typo tolerant, in-memory fuzzy Search Engine for building delightful search experiences
C++ algoliadatastoreelasticsearchenterprise-searchfacetingfull-text-searchfuzzy-searchgeosearchin-memoryinstantsearchmerchandisingpineconesearchsearch-enginesemantic-searchsimilarity-searchsite-searchsynonymstypo-tolerancevector-search
Introduce field level locale.
kishorencdd72e2a78c5e
diff --git a/include/collection.h b/include/collection.hindex 256100fc..abe32c55 100644--- a/include/collection.h+++ b/include/collection.h@@ -491,7 +491,8 @@ public:bool is_exceeding_memory_threshold() const;void parse_search_query(const std::string &query, std::vector<std::string>& q_include_tokens,- std::vector<std::string>& q_exclude_tokens) const;+ std::vector<std::string>& q_exclude_tokens,+ const std::string& locale) const;// PUBLIC OPERATIONSdiff --git a/include/field.h b/include/field.hindex 9b6f21e1..b48baca0 100644--- a/include/field.h+++ b/include/field.h@@ -34,6 +34,7 @@ namespace fields {static const std::string optional = "optional";static const std::string index = "index";static const std::string geo_resolution = "geo_resolution";+ static const std::string locale = "locale";}static const uint8_t DEFAULT_GEO_RESOLUTION = 7;@@ -48,9 +49,13 @@ struct field {uint8_t geo_resolution;+ std::string locale;+field(const std::string &name, const std::string &type, const bool facet, const bool optional = false,- bool index = true, const uint8_t geo_resolution = DEFAULT_GEO_RESOLUTION) :- name(name), type(type), facet(facet), optional(optional), index(index), geo_resolution(geo_resolution) {+ bool index = true, const uint8_t geo_resolution = DEFAULT_GEO_RESOLUTION,+ std::string locale = "") :+ name(name), type(type), facet(facet), optional(optional), index(index),+ geo_resolution(geo_resolution), locale(locale) {}@@ -199,6 +204,8 @@ struct field {field_val[fields::geo_resolution] = field.geo_resolution;}+ field_val[fields::locale] = field.locale;+fields_json.push_back(field_val);if(!field.has_valid_type()) {@@ -287,6 +294,20 @@ struct field {}}+ if(field_json.count(fields::locale) != 0){+ if(!field_json.at(fields::locale).is_string()) {+ return Option<bool>(400, std::string("The `locale` property of the field `") ++ field_json[fields::name].get<std::string>() + std::string("` should be a string."));+ }++ if(!field_json[fields::locale].empty() || field_json[fields::locale] != "en" ||+ field_json[fields::locale] != "ja" || field_json[fields::locale] != "ko" ||+ field_json[fields::locale] != "zh" || field_json[fields::locale] != "th") {+ return Option<bool>(400, std::string("The `locale` value of the field `") ++ field_json[fields::name].get<std::string>() + std::string("` is not valid."));+ }+ }+if(field_json["name"] == ".*") {if(field_json.count(fields::facet) == 0) {field_json[fields::facet] = false;diff --git a/include/index.h b/include/index.hindex fcacae77..ade60052 100644--- a/include/index.h+++ b/include/index.h@@ -36,10 +36,14 @@ struct search_field_t {size_t weight;};-struct search_args {+struct query_tokens_t {std::vector<std::string> q_include_tokens;std::vector<std::string> q_exclude_tokens;std::vector<std::vector<std::string>> q_synonyms;+};++struct search_args {+ std::vector<query_tokens_t> field_query_tokens;std::vector<search_field_t> search_fields;std::vector<filter> filters;std::vector<facet> facets;@@ -70,9 +74,7 @@ struct search_args {}- search_args(std::vector<std::string> q_include_tokens,- std::vector<std::string> q_exclude_tokens,- std::vector<std::vector<std::string>> q_synonyms,+ search_args(std::vector<query_tokens_t> field_query_tokens,std::vector<search_field_t> search_fields, std::vector<filter> filters,std::vector<facet> facets, std::map<size_t, std::map<size_t, uint32_t>> included_ids, std::vector<uint32_t> excluded_ids,std::vector<sort_by> sort_fields_std, facet_query_t facet_query, int num_typos, size_t max_facet_values,@@ -80,7 +82,7 @@ struct search_args {size_t drop_tokens_threshold, size_t typo_tokens_threshold,const std::vector<std::string>& group_by_fields, size_t group_limit,const std::string& default_sorting_field):- q_include_tokens(q_include_tokens), q_exclude_tokens(q_exclude_tokens), q_synonyms(q_synonyms),+ field_query_tokens(field_query_tokens),search_fields(search_fields), filters(filters), facets(facets),included_ids(included_ids), excluded_ids(excluded_ids), sort_fields_std(sort_fields_std),facet_query(facet_query), num_typos(num_typos), max_facet_values(max_facet_values), per_page(per_page),@@ -344,9 +346,7 @@ public:void run_search(search_args* search_params);- void search(const std::vector<std::string> & q_include_tokens,- const std::vector<std::string> & q_exclude_tokens,- const std::vector<std::vector<std::string>>& q_synonyms,+ void search(const std::vector<query_tokens_t>& field_query_tokens,const std::vector<search_field_t> & search_fields,const std::vector<filter> & filters, std::vector<facet> & facets,facet_query_t & facet_query,diff --git a/include/tokenizer.h b/include/tokenizer.hindex 985d59e8..798855ba 100644--- a/include/tokenizer.h+++ b/include/tokenizer.h@@ -3,6 +3,7 @@#include <string>#include <vector>#include <iconv.h>+#include <unicode/brkiter.h>class Tokenizer {private:@@ -21,11 +22,19 @@ private:std::stringstream out;+ std::string locale;+ icu::BreakIterator* bi = nullptr;+ icu::UnicodeString unicode_text;+ int32_t position = 0;+ int32_t prev_position = -1;+public:explicit Tokenizer(const std::string& input,- bool keep_separators=true, bool normalize=true, bool no_op=false):- text(input), i(0), keep_separators(keep_separators), normalize(normalize), no_op(no_op) {+ bool keep_separators=true, bool normalize=true, bool no_op=false,+ const std::string& locale = ""):+ text(input), i(0), keep_separators(keep_separators), normalize(normalize),+ no_op(no_op), locale(locale) {cd = iconv_open("ASCII//TRANSLIT", "UTF-8");if(!input.empty() && (std::isalnum(text[0]) || (text[i] & ~0x7f) != 0)) {@@ -34,10 +43,23 @@ public:} else {stream_mode = SEPARATORS;}++ if(!locale.empty() && locale != "en") {+ UErrorCode status = U_ZERO_ERROR;+ const icu::Locale& icu_locale = icu::Locale(locale.c_str());+ bi = icu::BreakIterator::createWordInstance(icu_locale, status);++ unicode_text = icu::UnicodeString::fromUTF8(text);+ bi->setText(unicode_text);++ position = bi->first();+ prev_position = -1;+ }}~Tokenizer() {iconv_close(cd);+ delete bi;}bool next(std::string& token, size_t& token_index);diff --git a/src/collection.cpp b/src/collection.cppindex 9a111b69..05268e26 100644--- a/src/collection.cpp+++ b/src/collection.cpp@@ -819,15 +819,26 @@ Option<nlohmann::json> Collection::search(const std::string & query, const std::spp::sparse_hash_set<uint64_t> groups_processed; // used to calculate total_found for grouped query//LOG(INFO) << "Num indices used for querying: " << indices.size();+ std::vector<query_tokens_t> field_query_tokens;- std::vector<std::string> q_include_tokens;- std::vector<std::string> q_exclude_tokens;+ if(search_fields.size() == 0) {+ // has to be a wildcard query+ field_query_tokens.emplace_back(query_tokens_t{});+ parse_search_query(query, field_query_tokens[0].q_include_tokens, field_query_tokens[0].q_exclude_tokens,"");+ } else {+ for(size_t i = 0; i < search_fields.size(); i++) {+ const auto& search_field = search_fields[i];+ field_query_tokens.emplace_back(query_tokens_t{});- parse_search_query(query, q_include_tokens, q_exclude_tokens);+ const std::string & field_locale = search_schema.at(search_field).locale;+ parse_search_query(query, field_query_tokens[i].q_include_tokens, field_query_tokens[i].q_exclude_tokens,+ field_locale);- // get synonyms- std::vector<std::vector<std::string>> q_synonyms;- synonym_reduction(q_include_tokens, q_synonyms);+ // get synonyms+ std::vector<std::vector<std::string>> q_synonyms;+ synonym_reduction(field_query_tokens[i].q_include_tokens, field_query_tokens[i].q_synonyms);+ }+ }// search all indicesstd::vector<search_args*> search_args_vec;@@ -837,7 +848,7 @@ Option<nlohmann::json> Collection::search(const std::string & query, const std::size_t index_id = 0;for(Index* index: indices) {- search_args* search_params = new search_args(q_include_tokens, q_exclude_tokens, q_synonyms, weighted_search_fields,+ search_args* search_params = new search_args(field_query_tokens, weighted_search_fields,filters, facets, index_to_included_ids[index_id],index_to_excluded_ids[index_id],sort_fields_std, facet_query, num_typos, max_facet_values, max_hits,@@ -1217,13 +1228,14 @@ Option<nlohmann::json> Collection::search(const std::string & query, const std::}void Collection::parse_search_query(const std::string &query, std::vector<std::string>& q_include_tokens,- std::vector<std::string>& q_exclude_tokens) const {+ std::vector<std::string>& q_exclude_tokens,+ const std::string& locale) const {if(query == "*") {q_exclude_tokens = {};q_include_tokens = {query};} else {std::vector<std::string> tokens;- Tokenizer(query, true, true).tokenize(tokens);+ Tokenizer(query, true, true, false, locale).tokenize(tokens);bool exclude_operator_prior = false;for(const auto& token: tokens) {@@ -1235,7 +1247,9 @@ void Collection::parse_search_query(const std::string &query, std::vector<std::sexclude_operator_prior = true;}- if(!std::isalnum(token[0])) {+ bool is_ascii = (token[0] & ~0x7f) == 0;++ if(is_ascii && !std::isalnum(token[0])) {continue;}@@ -1447,7 +1461,7 @@ void Collection::highlight_result(const field &search_field,const Match& match = match_index.match;const std::string& text = (search_field.type == field_types::STRING) ? document[search_field.name] : document[search_field.name][match_index.index];- Tokenizer tokenizer(text, true, false);+ Tokenizer tokenizer(text, true, false, false, search_field.locale);std::string raw_token;size_t raw_token_index = 0;diff --git a/src/collection_manager.cpp b/src/collection_manager.cppindex acf2482e..02bedb2b 100644--- a/src/collection_manager.cpp+++ b/src/collection_manager.cpp@@ -36,8 +36,13 @@ Collection* CollectionManager::init_collection(const nlohmann::json & collectionfield_obj[fields::geo_resolution] = size_t(DEFAULT_GEO_RESOLUTION);}+ if(field_obj.count(fields::locale) == 0) {+ field_obj[fields::locale] = "";+ }+fields.push_back({field_obj[fields::name], field_obj[fields::type], field_obj[fields::facet],- field_obj[fields::optional], field_obj[fields::index], field_obj[fields::geo_resolution]});+ field_obj[fields::optional], field_obj[fields::index],+ field_obj[fields::geo_resolution], field_obj[fields::locale]});}std::string default_sorting_field = collection_meta[Collection::COLLECTION_DEFAULT_SORTING_FIELD_KEY].get<std::string>();diff --git a/src/index.cpp b/src/index.cppindex 2054838b..c3fe1b69 100644--- a/src/index.cpp+++ b/src/index.cpp@@ -575,7 +575,7 @@ void Index::index_string_field(const std::string & text, const int64_t score, aruint32_t seq_id, bool is_facet, const field & a_field) {std::unordered_map<std::string, std::vector<uint32_t>> token_to_offsets;- Tokenizer tokenizer(text, false, true, !a_field.is_string());+ Tokenizer tokenizer(text, false, true, !a_field.is_string(), a_field.locale);std::string token;size_t token_index = 0;@@ -622,7 +622,7 @@ void Index::index_string_array_field(const std::vector<std::string> & strings, cconst std::string& str = strings[array_index];std::set<std::string> token_set; // required to deal with repeating tokens- Tokenizer tokenizer(str, false, true, !a_field.is_string());+ Tokenizer tokenizer(str, false, true, !a_field.is_string(), a_field.locale);std::string token;size_t token_index = 0;@@ -1246,7 +1246,7 @@ uint32_t Index::do_filtering(uint32_t** filter_ids_out, const std::vector<filter// there could be multiple tokens in a filter value, which we have to treat as ANDs// e.g. country: South Africa- Tokenizer tokenizer(filter_value, false, true);+ Tokenizer tokenizer(filter_value, false, true, false, f.locale);std::string str_token;size_t token_index = 0;std::vector<std::string> str_tokens;@@ -1420,8 +1420,7 @@ void Index::eq_str_filter_plain(const uint32_t *strt_ids, size_t strt_ids_size,}void Index::run_search(search_args* search_params) {- search(search_params->q_include_tokens, search_params->q_exclude_tokens,- search_params->q_synonyms,+ search(search_params->field_query_tokens,search_params->search_fields,search_params->filters, search_params->facets, search_params->facet_query,search_params->included_ids, search_params->excluded_ids,@@ -1506,9 +1505,7 @@ void Index::concat_topster_ids(Topster* topster, spp::sparse_hash_map<uint64_t,}}-void Index::search(const std::vector<std::string>& q_include_tokens,- const std::vector<std::string>& q_exclude_tokens,- const std::vector<std::vector<std::string>>& q_synonyms,+void Index::search(const std::vector<query_tokens_t>& field_query_tokens,const std::vector<search_field_t>& search_fields,const std::vector<filter>& filters,std::vector<facet>& facets, facet_query_t& facet_query,@@ -1565,7 +1562,7 @@ void Index::search(const std::vector<std::string>& q_include_tokens,// find documents that contain the excluded tokens to exclude them from results laterfor(size_t i = 0; i < num_search_fields; i++) {const std::string & field_name = search_fields[i].name;- for(const std::string& exclude_token: q_exclude_tokens) {+ for(const std::string& exclude_token: field_query_tokens[i].q_exclude_tokens) {art_leaf* leaf = (art_leaf *) art_search(search_index.at(field_name),(const unsigned char *) exclude_token.c_str(),exclude_token.size() + 1);@@ -1585,7 +1582,8 @@ void Index::search(const std::vector<std::string>& q_include_tokens,std::vector<Topster*> ftopsters;- if(!q_include_tokens.empty() && q_include_tokens[0] == "*") {+ if (!field_query_tokens.empty() && !field_query_tokens[0].q_include_tokens.empty() &&+ field_query_tokens[0].q_include_tokens[0] == "*") {const uint8_t field_id = (uint8_t)(FIELD_LIMIT_NUM - 0);const std::string& field = search_fields[0].name;@@ -1629,7 +1627,7 @@ void Index::search(const std::vector<std::string>& q_include_tokens,uint32_t token_bits = 255;score_results(sort_fields_std, (uint16_t) searched_queries.size(), field_id, 0, topster, {},groups_processed, filter_ids, filter_ids_length, group_limit, group_by_fields, token_bits);- collate_included_ids(q_include_tokens, field, field_id, included_ids_map, curated_topster, searched_queries);+ collate_included_ids(field_query_tokens[0].q_include_tokens, field, field_id, included_ids_map, curated_topster, searched_queries);all_result_ids_len = filter_ids_length;all_result_ids = filter_ids;@@ -1638,45 +1636,45 @@ void Index::search(const std::vector<std::string>& q_include_tokens,// In multi-field searches, a record can be matched across different fields, so we use this for aggregationspp::sparse_hash_map<uint64_t, std::vector<KV*>> topster_ids;- std::vector<token_t> q_include_pos_tokens;- for(size_t i=0; i < q_include_tokens.size(); i++) {- q_include_pos_tokens.push_back({i, q_include_tokens[i]});- }-- std::vector<std::vector<token_t>> q_pos_synonyms;- for(const auto& q_syn_vec: q_synonyms) {- std::vector<token_t> q_pos_syn;- for(size_t i=0; i < q_syn_vec.size(); i++) {- q_pos_syn.push_back({i, q_syn_vec[i]});- }- q_pos_synonyms.emplace_back(q_pos_syn);- }-//begin = std::chrono::high_resolution_clock::now();// non-wildcardfor(size_t i = 0; i < num_search_fields; i++) {+ std::vector<token_t> q_include_pos_tokens;+ for(size_t j=0; j < field_query_tokens[i].q_include_tokens.size(); j++) {+ q_include_pos_tokens.push_back({j, field_query_tokens[i].q_include_tokens[j]});+ }++ std::vector<std::vector<token_t>> q_pos_synonyms;+ for(const auto& q_syn_vec: field_query_tokens[i].q_synonyms) {+ std::vector<token_t> q_pos_syn;+ for(size_t j=0; j < q_syn_vec.size(); j++) {+ q_pos_syn.push_back({j, q_syn_vec[j]});+ }+ q_pos_synonyms.emplace_back(q_pos_syn);+ }+// proceed to query search only when no filters are provided or when filtering produces resultsif(filters.empty() || filter_ids_length > 0) {const uint8_t field_id = (uint8_t)(FIELD_LIMIT_NUM - i);- const std::string& field = search_fields[i].name;+ const std::string& field_name = search_fields[i].name;std::vector<token_t> query_tokens = q_include_pos_tokens;std::vector<token_t> search_tokens = q_include_pos_tokens;size_t num_tokens_dropped = 0;- //LOG(INFO) << "searching field! " << field;+ //LOG(INFO) << "searching field_name! " << field_name;Topster* ftopster = new Topster(topster->MAX_SIZE, topster->distinct);ftopsters.push_back(ftopster);- // Don't waste additional cycles for single field searches+ // Don't waste additional cycles for single field_name searchesTopster* actual_topster = (num_search_fields == 1) ? topster : ftopster;- // tracks the number of results found for the current field+ // tracks the number of results found for the current field_namesize_t field_num_results = 0;search_field(field_id, query_tokens, search_tokens, exclude_token_ids, exclude_token_ids_size, num_tokens_dropped,- field, filter_ids, filter_ids_length, curated_ids_sorted, facets, sort_fields_std,+ field_name, filter_ids, filter_ids_length, curated_ids_sorted, facets, sort_fields_std,num_typos, searched_queries, actual_topster, groups_processed, &all_result_ids, all_result_ids_len,field_num_results, group_limit, group_by_fields, token_order, prefix,drop_tokens_threshold, typo_tokens_threshold);@@ -1688,14 +1686,14 @@ void Index::search(const std::vector<std::string>& q_include_tokens,query_tokens = search_tokens = syn_tokens;search_field(field_id, query_tokens, search_tokens, exclude_token_ids, exclude_token_ids_size, num_tokens_dropped,- field, filter_ids, filter_ids_length, curated_ids_sorted, facets, sort_fields_std,+ field_name, filter_ids, filter_ids_length, curated_ids_sorted, facets, sort_fields_std,num_typos, searched_queries, actual_topster, groups_processed, &all_result_ids, all_result_ids_len,field_num_results, group_limit, group_by_fields, token_order, prefix,drop_tokens_threshold, typo_tokens_threshold);}concat_topster_ids(ftopster, topster_ids);- collate_included_ids(q_include_tokens, field, field_id, included_ids_map, curated_topster, searched_queries);+ collate_included_ids(field_query_tokens[i].q_include_tokens, field_name, field_id, included_ids_map, curated_topster, searched_queries);//LOG(INFO) << "topster_ids.size: " << topster_ids.size();}}@@ -1738,7 +1736,7 @@ void Index::search(const std::vector<std::string>& q_include_tokens,int64_t field_typos = 255 - ((match_score >> 8) & 0xFF);total_typos += (field_typos + 1) * weight;- if(field_typos == 0 && tokens_found == q_include_tokens.size()) {+ if(field_typos == 0 && tokens_found == field_query_tokens[i].q_include_tokens.size()) {num_exact_matches++;}@@ -1755,11 +1753,11 @@ void Index::search(const std::vector<std::string>& q_include_tokens,size_t words_present = 0;- for(size_t token_index=0; token_index < q_include_tokens.size(); token_index++) {- const auto& token = q_include_tokens[token_index];+ for(size_t token_index=0; token_index < field_query_tokens[i].q_include_tokens.size(); token_index++) {+ const auto& token = field_query_tokens[i].q_include_tokens[token_index];std::vector<art_leaf*> leaves;- const bool prefix_search = prefix && (token_index == q_include_tokens.size()-1);+ const bool prefix_search = prefix && (token_index == field_query_tokens[i].q_include_tokens.size()-1);const size_t token_len = prefix_search ? (int) token.length() : (int) token.length() + 1;art_fuzzy_search(search_index.at(field), (const unsigned char *) token.c_str(), token_len,0, 0, 1, token_order, prefix_search, nullptr, 0, leaves);@@ -1791,7 +1789,7 @@ void Index::search(const std::vector<std::string>& q_include_tokens,uint64_t field_typos = 255 - ((match_score >> 8) & 0xFF);total_typos += (field_typos + 1) * weight;- if(field_typos == 0 && tokens_found == q_include_tokens.size()) {+ if(field_typos == 0 && tokens_found == field_query_tokens[i].q_include_tokens.size()) {num_exact_matches++;}//LOG(INFO) << "seq_id: " << seq_id << ", total_typos: " << ((match_score >> 8) & 0xFF);@@ -1865,7 +1863,11 @@ void Index::search_field(const uint8_t & field_id,const token_ordering token_order, const bool prefix,const size_t drop_tokens_threshold, const size_t typo_tokens_threshold) const {- const size_t max_cost = (num_typos < 0 || num_typos > 2) ? 2 : num_typos;+ size_t max_cost = (num_typos < 0 || num_typos > 2) ? 2 : num_typos;+ if(search_schema.at(field).locale != "" && search_schema.at(field).locale != "en") {+ // disable fuzzy trie traversal for non-english locales+ max_cost = 0;+ }// To prevent us from doing ART search repeatedly as we iterate through possible correctionsspp::sparse_hash_map<std::string, std::vector<art_leaf*>> token_cost_cache;diff --git a/src/tokenizer.cpp b/src/tokenizer.cppindex ac99a2bb..71fcc26e 100644--- a/src/tokenizer.cpp+++ b/src/tokenizer.cpp@@ -2,6 +2,39 @@#include "tokenizer.h"bool Tokenizer::next(std::string &token, size_t& token_index) {+ if(!locale.empty() && locale != "en") {+ while (position != icu::BreakIterator::DONE) {+ //LOG(INFO) << "Position: " << position;+ bool found_token = false;++ if(prev_position != -1) {+ std::string word;+ size_t length = position - prev_position;+ token = unicode_text.tempSubString(prev_position, length).toUTF8String(word);++ if(!token.empty()) {+ if (!keep_separators && !std::isalnum(token[0]) && (token[i] & ~0x7f) == 0) {+ found_token = false;+ } else if(locale == "ko" && token == "·") {+ found_token = false;+ } else {+ found_token = true;+ token_index = token_counter++;+ }+ }+ }++ prev_position = position;+ position = bi->next();++ if(found_token) {+ return true;+ }+ }++ return false;+ }+if(no_op) {if(i == text.size()) {return false;diff --git a/test/collection_locale_test.cpp b/test/collection_locale_test.cppnew file mode 100644index 00000000..6bde3362--- /dev/null+++ b/test/collection_locale_test.cpp@@ -0,0 +1,131 @@+#include <gtest/gtest.h>+#include <collection.h>+#include <collection_manager.h>++class CollectionLocaleTest : public ::testing::Test {+protected:+ Store *store;+ CollectionManager & collectionManager = CollectionManager::get_instance();++ std::vector<std::string> query_fields;+ std::vector<sort_by> sort_fields;++ void setupCollection() {+ std::string state_dir_path = "/tmp/typesense_test/collection_locale";+ LOG(INFO) << "Truncating and creating: " << state_dir_path;+ system(("rm -rf "+state_dir_path+" && mkdir -p "+state_dir_path).c_str());++ store = new Store(state_dir_path);+ collectionManager.init(store, 1.0, "auth_key");+ collectionManager.load(8, 1000);+ }++ virtual void SetUp() {+ setupCollection();+ }++ virtual void TearDown() {+ collectionManager.dispose();+ delete store;+ }+};++TEST_F(CollectionLocaleTest, SearchAgainstThaiText) {+ Collection *coll1;++ std::vector<field> fields = {field("title", field_types::STRING, false, false, true, DEFAULT_GEO_RESOLUTION, "th"),+ field("artist", field_types::STRING, false),+ field("points", field_types::INT32, false),};++ coll1 = collectionManager.get_collection("coll1").get();+ if(coll1 == nullptr) {+ coll1 = collectionManager.create_collection("coll1", 1, fields, "points").get();+ }++ std::vector<std::vector<std::string>> records = {+ {"ลงที่นั่นโดยรถไฟ", "Dustin Kensrue"},+ {"พกติดตัวเสมอ", "Gord Downie"},+ {"พกไฟ\nเสมอ", "Dustin Kensrue"},+ };++ for(size_t i=0; i<records.size(); i++) {+ nlohmann::json doc;++ doc["id"] = std::to_string(i);+ doc["title"] = records[i][0];+ doc["artist"] = records[i][1];+ doc["points"] = i;++ ASSERT_TRUE(coll1->add(doc.dump()).ok());+ }++ auto results = coll1->search("ลงรถไฟ",+ {"title"}, "", {}, {}, 0, 10, 1, FREQUENCY).get();++ ASSERT_EQ(1, results["found"].get<size_t>());+ ASSERT_EQ(1, results["hits"].size());+ ASSERT_EQ("0", results["hits"][0]["document"]["id"].get<std::string>());+ ASSERT_EQ("<mark>ลง</mark>ที่นั่นโดย<mark>รถไฟ</mark>", results["hits"][0]["highlights"][0]["snippet"].get<std::string>());++ results = coll1->search("ลงรถไฟ downie",+ {"title", "artist"}, "", {}, {}, 0, 10, 1, FREQUENCY).get();++ ASSERT_EQ(2, results["found"].get<size_t>());+ ASSERT_EQ(2, results["hits"].size());+ ASSERT_EQ("0", results["hits"][0]["document"]["id"].get<std::string>());+ ASSERT_EQ("<mark>ลง</mark>ที่นั่นโดย<mark>รถไฟ</mark>", results["hits"][0]["highlights"][0]["snippet"].get<std::string>());++ ASSERT_EQ("1", results["hits"][1]["document"]["id"].get<std::string>());+ ASSERT_EQ("Gord <mark>Downie</mark>", results["hits"][1]["highlights"][0]["snippet"].get<std::string>());++ results = coll1->search("พกไฟ", {"title", "artist"}, "", {}, {}, 0, 10, 1, FREQUENCY).get();+ ASSERT_EQ(1, results["found"].get<size_t>());+ ASSERT_EQ(1, results["hits"].size());+ ASSERT_EQ("2", results["hits"][0]["document"]["id"].get<std::string>());+ ASSERT_EQ("<mark>พกไฟ</mark>\nเสมอ", results["hits"][0]["highlights"][0]["snippet"].get<std::string>());+}++TEST_F(CollectionLocaleTest, SearchAgainstKoreanText) {+ Collection *coll1;++ std::vector<field> fields = {field("title", field_types::STRING, false, false, true, DEFAULT_GEO_RESOLUTION, "th"),+ field("artist", field_types::STRING, false),+ field("points", field_types::INT32, false),};++ coll1 = collectionManager.get_collection("coll1").get();+ if(coll1 == nullptr) {+ coll1 = collectionManager.create_collection("coll1", 1, fields, "points").get();+ }++ std::vector<std::vector<std::string>> records = {+ {"경승지·산악·협곡", "Dustin Kensrue"},+ {"안녕은하철도999극장판", "Gord Downie"},+ };++ for(size_t i=0; i<records.size(); i++) {+ nlohmann::json doc;++ doc["id"] = std::to_string(i);+ doc["title"] = records[i][0];+ doc["artist"] = records[i][1];+ doc["points"] = i;++ ASSERT_TRUE(coll1->add(doc.dump()).ok());+ }++ auto results = coll1->search("극장판",+ {"title"}, "", {}, {}, 0, 10, 1, FREQUENCY).get();++ ASSERT_EQ(1, results["found"].get<size_t>());+ ASSERT_EQ(1, results["hits"].size());+ ASSERT_EQ("1", results["hits"][0]["document"]["id"].get<std::string>());+ ASSERT_EQ("안녕은하철도999<mark>극장판</mark>", results["hits"][0]["highlights"][0]["snippet"].get<std::string>());++ results = coll1->search("산악",+ {"title", "artist"}, "", {}, {}, 0, 10, 1, FREQUENCY).get();++ ASSERT_EQ(1, results["found"].get<size_t>());+ ASSERT_EQ(1, results["hits"].size());+ ASSERT_EQ("0", results["hits"][0]["document"]["id"].get<std::string>());+ ASSERT_EQ("경승지·<mark>산악</mark>·협곡", results["hits"][0]["highlights"][0]["snippet"].get<std::string>());+}diff --git a/test/collection_manager_test.cpp b/test/collection_manager_test.cppindex 1669f137..e1b99cb9 100644--- a/test/collection_manager_test.cpp+++ b/test/collection_manager_test.cpp@@ -24,7 +24,7 @@ protected:collectionManager.load(8, 1000);search_fields = {- field("title", field_types::STRING, false),+ field("title", field_types::STRING, false, false, true, DEFAULT_GEO_RESOLUTION, "en"),field("starring", field_types::STRING, false),field("cast", field_types::STRING_ARRAY, true, true),field(".*_year", field_types::INT32, true, true),@@ -88,12 +88,12 @@ TEST_F(CollectionManagerTest, CollectionCreation) {// we already call `collection1->get_next_seq_id` above, which is side-effectingASSERT_EQ(1, StringUtils::deserialize_uint32_t(next_seq_id));ASSERT_EQ("{\"created_at\":12345,\"default_sorting_field\":\"points\",\"fallback_field_type\":\"\","- "\"fields\":[{\"facet\":false,\"name\":\"title\",\"optional\":false,\"type\":\"string\"},"- "{\"facet\":false,\"name\":\"starring\",\"optional\":false,\"type\":\"string\"},"- "{\"facet\":true,\"name\":\"cast\",\"optional\":true,\"type\":\"string[]\"},"- "{\"facet\":true,\"name\":\".*_year\",\"optional\":true,\"type\":\"int32\"},"- "{\"facet\":false,\"geo_resolution\":14,\"name\":\"location\",\"optional\":true,\"type\":\"geopoint\"},"- "{\"facet\":false,\"name\":\"points\",\"optional\":false,\"type\":\"int32\"}],\"id\":0,"+ "\"fields\":[{\"facet\":false,\"locale\":\"en\",\"name\":\"title\",\"optional\":false,\"type\":\"string\"},"+ "{\"facet\":false,\"locale\":\"\",\"name\":\"starring\",\"optional\":false,\"type\":\"string\"},"+ "{\"facet\":true,\"locale\":\"\",\"name\":\"cast\",\"optional\":true,\"type\":\"string[]\"},"+ "{\"facet\":true,\"locale\":\"\",\"name\":\".*_year\",\"optional\":true,\"type\":\"int32\"},"+ "{\"facet\":false,\"geo_resolution\":14,\"locale\":\"\",\"name\":\"location\",\"optional\":true,\"type\":\"geopoint\"},"+ "{\"facet\":false,\"locale\":\"\",\"name\":\"points\",\"optional\":false,\"type\":\"int32\"}],\"id\":0,""\"name\":\"collection1\",\"num_memory_shards\":4}",collection_meta_json);ASSERT_EQ("1", next_collection_id);@@ -232,6 +232,7 @@ TEST_F(CollectionManagerTest, RestoreRecordsOnRestart) {std::vector<std::string> facets;nlohmann::json results = collection1->search("thomas", search_fields, "", facets, sort_fields, 0, 10, 1, FREQUENCY, false).get();+ LOG(INFO) << results;ASSERT_EQ(4, results["hits"].size());std::unordered_map<std::string, field> schema = collection1->get_schema();diff --git a/test/tokenizer_test.cpp b/test/tokenizer_test.cppindex 58123d4c..656be0de 100644--- a/test/tokenizer_test.cpp+++ b/test/tokenizer_test.cpp@@ -1,5 +1,6 @@#include <gtest/gtest.h>#include "tokenizer.h"+#include "logger.h"TEST(TokenizerTest, ShouldTokenizeNormalizeDifferentStrings) {const std::string withnewline = "Michael Jordan:\nWelcome, everybody. Welcome! ";@@ -177,3 +178,45 @@ TEST(TokenizerTest, ShouldTokenizeIteratively) {ASSERT_EQ(1, tokens.size());ASSERT_STREQ("Michael Jordan:\n\nWelcome, everybody. Welcome!", tokens[0].c_str());}++TEST(TokenizerTest, ShouldTokenizeLocaleText) {+ std::string str = "จิ้งจอกสีน้ำตาลด่วน";+ std::vector<std::string> tokens;+ Tokenizer(str, false, false, false, "th").tokenize(tokens);++ ASSERT_EQ(4, tokens.size());+ ASSERT_EQ("จิ้งจอก", tokens[0]);+ ASSERT_EQ("สี", tokens[1]);+ ASSERT_EQ("น้ำตาล", tokens[2]);+ ASSERT_EQ("ด่วน", tokens[3]);++ tokens.clear();+ str = "น. วันที่ 31 มี.ค.";+ Tokenizer(str, false, false, false, "th").tokenize(tokens);+ ASSERT_EQ(5, tokens.size());+ ASSERT_EQ("น", tokens[0]);+ ASSERT_EQ("วัน", tokens[1]);+ ASSERT_EQ("ที่", tokens[2]);+ ASSERT_EQ("31", tokens[3]);+ ASSERT_EQ("มี.ค", tokens[4]);++ tokens.clear();+ Tokenizer("Odd Thomas", false, false, false, "en").tokenize(tokens);+ ASSERT_EQ(2, tokens.size());+ ASSERT_EQ("Odd", tokens[0]);+ ASSERT_EQ("Thomas", tokens[1]);++ tokens.clear();+ Tokenizer("경승지·산악·협곡", false, false, false, "ko").tokenize(tokens);+ ASSERT_EQ(3, tokens.size());+ ASSERT_EQ("경승지", tokens[0]);+ ASSERT_EQ("산악", tokens[1]);+ ASSERT_EQ("협곡", tokens[2]);++ tokens.clear();+ Tokenizer("안녕은하철도999극장판", false, false, false, "ko").tokenize(tokens);+ ASSERT_EQ(3, tokens.size());+ ASSERT_EQ("안녕은하철도", tokens[0]);+ ASSERT_EQ("999", tokens[1]);+ ASSERT_EQ("극장판", tokens[2]);+}