Open Source alternative to Algolia + Pinecone and an Easier-to-Use alternative to ElasticSearch ⚡ 🔍 ✨ Fast, typo tolerant, in-memory fuzzy Search Engine for building delightful search experiences
C++ algoliadatastoreelasticsearchenterprise-searchfacetingfull-text-searchfuzzy-searchgeosearchin-memoryinstantsearchmerchandisingpineconesearchsearch-enginesemantic-searchsimilarity-searchsite-searchsynonymstypo-tolerancevector-search
Refactor highlighting + tokenizer to simplify logic.
Kishore Nallanb3b47f565150
diff --git a/include/collection.h b/include/collection.hindex abe32c55..211f9eab 100644--- a/include/collection.h+++ b/include/collection.h@@ -235,13 +235,13 @@ struct synonym_t {}std::vector<std::string> tokens;- Tokenizer(synonym, false).tokenize(tokens);+ Tokenizer(synonym, true).tokenize(tokens);syn.synonyms.push_back(tokens);}if(synonym_json.count("root") != 0) {std::vector<std::string> tokens;- Tokenizer(synonym_json["root"], false).tokenize(tokens);+ Tokenizer(synonym_json["root"], true).tokenize(tokens);syn.root = tokens;}diff --git a/include/tokenizer.h b/include/tokenizer.hindex c5681e88..9d75e0e3 100644--- a/include/tokenizer.h+++ b/include/tokenizer.h@@ -11,7 +11,6 @@ class Tokenizer {private:std::string_view text;size_t i;- const bool keep_separators;const bool normalize;const bool no_op;@@ -22,29 +21,35 @@ private:static const size_t SEPARATE = 1;static const size_t SKIP = 2;- size_t prev_stream_mode;+ uint8_t index_symbols[256] = {};- std::stringstream out;+ std::string out;std::string locale;icu::BreakIterator* bi = nullptr;icu::UnicodeString unicode_text;int32_t position = 0;int32_t prev_position = -1;+ int32_t utf8_start_index = 0;char* normalized_text = nullptr;inline size_t get_stream_mode(char c) {- return std::isalnum(c) ? INDEX : (+ return (std::isalnum(c) || index_symbols[uint8_t(c)] == 1) ? INDEX : ((c == ' ' || c == '\n') ? SEPARATE : SKIP);}+ static inline bool is_ascii_char(char c) {+ return (c & ~0x7f) == 0;+ }+public:explicit Tokenizer(const std::string& input,- bool keep_separators=true, bool normalize=true, bool no_op=false,- const std::string& locale = ""):- i(0), keep_separators(keep_separators), normalize(normalize),+ bool normalize=true, bool no_op=false,+ const std::string& locale = "",+ const std::vector<char>& symbols_to_index = {}):+ i(0), normalize(normalize),no_op(no_op), locale(locale) {if(locale == "ja") {@@ -56,13 +61,6 @@ public:cd = iconv_open("ASCII//TRANSLIT", "UTF-8");- if(!text.empty() && (std::isalnum(text[0]) || (text[i] & ~0x7f) != 0)) {- // alphanum or non-ascii- prev_stream_mode = INDEX;- } else {- prev_stream_mode = SEPARATE;- }-if(!locale.empty() && locale != "en") {UErrorCode status = U_ZERO_ERROR;const icu::Locale& icu_locale = icu::Locale(locale.c_str());@@ -74,6 +72,10 @@ public:position = bi->first();prev_position = -1;}++ for(char c: symbols_to_index) {+ index_symbols[uint8_t(c)] = 1;+ }}~Tokenizer() {@@ -82,6 +84,8 @@ public:delete bi;}+ bool next(std::string& token, size_t& token_index, size_t& start_index, size_t& end_index);+bool next(std::string& token, size_t& token_index);void tokenize(std::vector<std::string>& tokens);diff --git a/src/collection.cpp b/src/collection.cppindex 5b83ab6a..17b29c39 100644--- a/src/collection.cpp+++ b/src/collection.cpp@@ -14,6 +14,7 @@#include <collection_manager.h>#include <h3api.h>#include <regex>+#include <list>#include "topster.h"#include "logger.h"@@ -1237,22 +1238,14 @@ void Collection::parse_search_query(const std::string &query, std::vector<std::sq_include_tokens = {query};} else {std::vector<std::string> tokens;- Tokenizer(query, true, true, false, locale).tokenize(tokens);- bool exclude_operator_prior = false;+ Tokenizer(query, true, false, locale, {'-'}).tokenize(tokens);- for(const auto& token: tokens) {- if(token.empty()) {- continue;- }+ bool exclude_operator_prior = false;- if(token == "-" || token == " -") {+ for(auto& token: tokens) {+ if(token[0] == '-') {exclude_operator_prior = true;- }-- bool is_ascii = (token[0] & ~0x7f) == 0;-- if(is_ascii && !std::isalnum(token[0])) {- continue;+ token = token.substr(1);}if(exclude_operator_prior) {@@ -1463,97 +1456,119 @@ void Collection::highlight_result(const field &search_field,const Match& match = match_index.match;const std::string& text = (search_field.type == field_types::STRING) ? document[search_field.name] : document[search_field.name][match_index.index];- Tokenizer tokenizer(text, true, false, false, search_field.locale);+ Tokenizer tokenizer(text, true, false, search_field.locale);++ // need an ordered map here to ensure that it is ordered by the key (start offset)+ std::map<size_t, size_t> token_offsets;- std::string raw_token;- size_t raw_token_index = 0;- int indexed_token_index = -1;size_t match_offset_index = 0;+ std::string raw_token;+ spp::sparse_hash_set<std::string> token_hits; // used to identify repeating tokens+ size_t raw_token_index = 0, tok_start = 0, tok_end = 0;- std::set<size_t> token_indices;- spp::sparse_hash_set<std::string> token_hits;- std::vector<std::string> raw_tokens;- std::unordered_map<size_t, size_t> indexed_to_raw;-- while(tokenizer.next(raw_token, raw_token_index)) {- if(!raw_token.empty() && (std::isalnum(raw_token[0]) || (raw_token[0] & ~0x7f) != 0)) {- // check for actual token (first char is NOT alphanum or ascii)- indexed_token_index++;- indexed_to_raw[indexed_token_index] = raw_token_index;- /*LOG(INFO) << "raw_token: " << raw_token << ", indexed_token_index: " << indexed_token_index- << ", raw_token_index: " << raw_token_index;*/- }+ // based on `highlight_affix_num_tokens`+ size_t snippet_start_offset = 0, snippet_end_offset = (text.empty() ? 0 : text.size() - 1);++ // window used to locate the starting offset for snippet on the text+ std::list<size_t> snippet_start_window;- if (match_offset_index < match.offsets.size() &&- match.offsets[match_offset_index].offset == indexed_token_index) {- std::string indexed_token;- Tokenizer(raw_token, true, true).tokenize(indexed_token);+ highlight.matched_tokens.emplace_back();+ std::vector<std::string>& matched_tokens = highlight.matched_tokens.back();- if(token_indices.count(indexed_token_index) == 0) {- // repetition could occur, for e.g. in the case of synonym constructed queries- token_indices.insert(indexed_token_index);- token_hits.insert(indexed_token);+ while(tokenizer.next(raw_token, raw_token_index, tok_start, tok_end)) {+ if(token_offsets.empty()) {+ if(snippet_start_window.size() == highlight_affix_num_tokens + 1) {+ snippet_start_window.pop_front();}- match_offset_index++;+ snippet_start_window.push_back(tok_start);}- raw_tokens.push_back(raw_token);- }+ if (token_hits.count(raw_token) != 0 ||+ (match_offset_index < match.offsets.size() &&+ match.offsets[match_offset_index].offset == raw_token_index)) {- size_t num_indexed_tokens = indexed_token_index + 1;- auto minmax = std::minmax_element(token_indices.begin(), token_indices.end());+ token_offsets.emplace(tok_start, tok_end);+ token_hits.insert(raw_token);- size_t prefix_length = highlight_affix_num_tokens;- size_t suffix_length = highlight_affix_num_tokens;+ // to skip over duplicate tokens in the query+ do {+ match_offset_index++;+ } while(match_offset_index < match.offsets.size() &&+ match.offsets[match_offset_index - 1].offset == match.offsets[match_offset_index].offset);- if(num_indexed_tokens == 0) {- continue;+ if(token_offsets.size() == 1) {+ snippet_start_offset = snippet_start_window.front();+ }+ }++ if(raw_token_index == match.offsets.back().offset + highlight_affix_num_tokens) {+ // register end of highlight snippet+ snippet_end_offset = tok_end;+ }++ if(raw_token_index > match.offsets.back().offset + highlight_affix_num_tokens &&+ raw_token_index == snippet_threshold - 1) {+ // since we have already crossed snippeting threshold, we can break now+ break;+ }}- // For longer strings, pick surrounding raw_tokens within `prefix_length` of min_index and max_index for snippet- const size_t start_index = (num_indexed_tokens <= snippet_threshold) ? 0 :- indexed_to_raw[std::max(0, (int)(*(minmax.first) - prefix_length))];+ if(token_offsets.empty()) {+ continue;+ }- const size_t end_index = (num_indexed_tokens <= snippet_threshold) ? raw_tokens.size() - 1 :- indexed_to_raw[std::min((int)num_indexed_tokens - 1, (int)(*(minmax.second) + suffix_length))];+ if(raw_token_index + 1 < snippet_threshold) {+ // fully highlight field whose token size is less than given snippeth threshold+ snippet_start_offset = 0;+ snippet_end_offset = text.size() - 1;+ }- std::stringstream snippet_stream;+ // with `token_index_offsets` we have a list of ranges to target for highlighting- highlight.matched_tokens.emplace_back();- std::vector<std::string>& matched_tokens = highlight.matched_tokens.back();- size_t snippet_index = start_index;+ auto offset_it = token_offsets.begin();+ std::stringstream highlighted_text;- while(snippet_index <= end_index) {- std::string normalized_token;- Tokenizer(raw_tokens[snippet_index], true, true).tokenize(normalized_token);+ for(size_t i = snippet_start_offset; i <= snippet_end_offset; i++) {+ if(offset_it != token_offsets.end()) {+ if (i == offset_it->first) {+ highlighted_text << highlight_start_tag;+ matched_tokens.push_back(text.substr(i, (offset_it->second - i) + 1));+ }- if(token_hits.count(normalized_token) != 0) {- snippet_stream << highlight_start_tag << raw_tokens[snippet_index] << highlight_end_tag;- matched_tokens.push_back(raw_tokens[snippet_index]);- } else {- snippet_stream << raw_tokens[snippet_index];+ if (i == offset_it->second) {+ highlighted_text << text[i];+ highlighted_text << highlight_end_tag;+ offset_it++;+ continue;+ }}- snippet_index++;+ highlighted_text << text[i];}- highlight.snippets.push_back(snippet_stream.str());+ highlight.snippets.push_back(highlighted_text.str());if(search_field.type == field_types::STRING_ARRAY) {highlight.indices.push_back(match_index.index);}if(highlighted_fully) {std::stringstream value_stream;- for(size_t value_index = 0; value_index < raw_tokens.size(); value_index++) {- std::string normalized_token;- Tokenizer(raw_tokens[value_index], true, true).tokenize(normalized_token);-- if(token_hits.count(normalized_token) != 0) {- value_stream << highlight_start_tag << raw_tokens[value_index] << highlight_end_tag;- } else {- value_stream << raw_tokens[value_index];+ offset_it = token_offsets.begin();++ for(size_t i = 0; i < text.size(); i++) {+ if(offset_it != token_offsets.end()) {+ if (i == offset_it->first) {+ value_stream << highlight_start_tag;+ } else if (i == offset_it->second) {+ value_stream << text[i];+ value_stream << highlight_end_tag;+ offset_it++;+ continue;+ }}++ value_stream << text[i];}highlight.values.push_back(value_stream.str());diff --git a/src/index.cpp b/src/index.cppindex 10f418a5..bd80ee3e 100644--- a/src/index.cpp+++ b/src/index.cpp@@ -533,7 +533,7 @@ void Index::index_string_field(const std::string & text, const int64_t score, aruint32_t seq_id, bool is_facet, const field & a_field) {std::unordered_map<std::string, std::vector<uint32_t>> token_to_offsets;- Tokenizer tokenizer(text, false, true, !a_field.is_string(), a_field.locale);+ Tokenizer tokenizer(text, true, !a_field.is_string(), a_field.locale);std::string token;size_t token_index = 0;@@ -580,7 +580,7 @@ void Index::index_string_array_field(const std::vector<std::string> & strings, cconst std::string& str = strings[array_index];std::set<std::string> token_set; // required to deal with repeating tokens- Tokenizer tokenizer(str, false, true, !a_field.is_string(), a_field.locale);+ Tokenizer tokenizer(str, true, !a_field.is_string(), a_field.locale);std::string token;size_t token_index = 0;@@ -707,7 +707,7 @@ void Index::do_facets(std::vector<facet> & facets, facet_query_t & facet_query,art_tree *t = search_index.at(facet_field.faceted_name());std::vector<std::string> query_tokens;- Tokenizer(facet_query.query, false, true, !facet_field.is_string()).tokenize(query_tokens);+ Tokenizer(facet_query.query, true, !facet_field.is_string()).tokenize(query_tokens);for (size_t qtoken_index = 0; qtoken_index < query_tokens.size(); qtoken_index++) {auto &q = query_tokens[qtoken_index];@@ -1204,7 +1204,7 @@ uint32_t Index::do_filtering(uint32_t** filter_ids_out, const std::vector<filter// there could be multiple tokens in a filter value, which we have to treat as ANDs// e.g. country: South Africa- Tokenizer tokenizer(filter_value, false, true, false, f.locale);+ Tokenizer tokenizer(filter_value, true, false, f.locale);std::string str_token;size_t token_index = 0;@@ -2447,11 +2447,11 @@ void Index::tokenize_string_field(const nlohmann::json& document, const field& sconst std::string& field_name = search_field.name;if(search_field.type == field_types::STRING) {- Tokenizer(document[field_name], false, true, false, locale).tokenize(tokens);+ Tokenizer(document[field_name], true, false, locale).tokenize(tokens);} else if(search_field.type == field_types::STRING_ARRAY) {const std::vector<std::string>& values = document[field_name].get<std::vector<std::string>>();for(const std::string & value: values) {- Tokenizer(value, false, true, false, locale).tokenize(tokens);+ Tokenizer(value, true, false, locale).tokenize(tokens);}}}diff --git a/src/tokenizer.cpp b/src/tokenizer.cppindex 9bb4b709..127a3708 100644--- a/src/tokenizer.cpp+++ b/src/tokenizer.cpp@@ -1,7 +1,7 @@#include <sstream>#include "tokenizer.h"-bool Tokenizer::next(std::string &token, size_t& token_index) {+bool Tokenizer::next(std::string &token, size_t& token_index, size_t& start_index, size_t& end_index) {if(no_op) {if(i == text.size()) {return false;@@ -9,6 +9,8 @@ bool Tokenizer::next(std::string &token, size_t& token_index) {token = text;i = text.size();+ start_index = 0;+ end_index = text.size() - 1;return true;}@@ -22,8 +24,11 @@ bool Tokenizer::next(std::string &token, size_t& token_index) {size_t length = position - prev_position;token = unicode_text.tempSubString(prev_position, length).toUTF8String(word);+ //LOG(INFO) << "token: " << token;+if(!token.empty()) {- if (!keep_separators && !std::isalnum(token[0]) && (token[i] & ~0x7f) == 0) {+ if (!std::isalnum(token[0]) && is_ascii_char(token[i])) {+ // ignore ascii symbolsfound_token = false;} else if(locale == "ko" && token == "·") {found_token = false;@@ -31,6 +36,10 @@ bool Tokenizer::next(std::string &token, size_t& token_index) {found_token = true;token_index = token_counter++;}++ start_index = utf8_start_index;+ end_index = utf8_start_index + token.size() - 1;+ utf8_start_index = end_index + 1;}}@@ -46,57 +55,40 @@ bool Tokenizer::next(std::string &token, size_t& token_index) {}while(i < text.size()) {- bool is_ascii = (text[i] & ~0x7f) == 0;- if(is_ascii) {+ if(is_ascii_char(text[i])) {size_t this_stream_mode = get_stream_mode(text[i]);- if(this_stream_mode == SKIP && !keep_separators) {+ if(this_stream_mode == SKIP) {i++;continue;}- if(this_stream_mode != prev_stream_mode) {- // We tokenize when `prev_stream_mode` changes- token = out.str();-- out.str(std::string());- if(normalize) {- out << char(std::tolower(text[i]));- } else {- out << text[i];- }- i++;-- if(prev_stream_mode == SEPARATE && !keep_separators) {- prev_stream_mode = this_stream_mode;+ if(this_stream_mode == SEPARATE) {+ if(out.empty()) {+ i++;continue;}+ token = out;+ out.clear();+token_index = token_counter++;- prev_stream_mode = this_stream_mode;+ end_index = i - 1;+ i++;return true;} else {- if(normalize) {- out << char(std::tolower(text[i]));- } else {- out << text[i];+ if(out.empty()) {+ start_index = i;}+ out += normalize ? char(std::tolower(text[i])) : text[i];i++;continue;}}- if(prev_stream_mode == SEPARATE) { // to detect first non-ascii character- // we will tokenize now and treat the following non-ascii chars as a different token- prev_stream_mode = INDEX;- token = out.str();- out.str(std::string());-- if(keep_separators) {- token_index = token_counter++;- return true;- }+ if(out.empty()) {+ start_index = i;}char inbuf[5];@@ -111,7 +103,7 @@ bool Tokenizer::next(std::string &token, size_t& token_index) {size_t insize = (p - &inbuf[0]);if(!normalize) {- out << inbuf;+ out += inbuf;continue;}@@ -127,38 +119,35 @@ bool Tokenizer::next(std::string &token, size_t& token_index) {if(errno == EILSEQ) {// symbol cannot be represented as ASCII, so write the original symbol- out << inbuf;+ out += inbuf;} else {for(size_t out_index=0; out_index<5; out_index++) {if(!normalize) {- out << outbuf[out_index];+ out += outbuf[out_index];continue;}- bool unicode_is_ascii = ((outbuf[out_index] & ~0x7f) == 0);+ bool unicode_is_ascii = is_ascii_char(outbuf[out_index]);bool keep_char = !unicode_is_ascii || std::isalnum(outbuf[out_index]);if(keep_char) {if(unicode_is_ascii && std::isalnum(outbuf[out_index])) {outbuf[out_index] = char(std::tolower(outbuf[out_index]));}- out << outbuf[out_index];+ out += outbuf[out_index];}}}}- token = out.str();- out.str(std::string());+ token = out;+ out.clear();+ end_index = i - 1;if(token.empty()) {return false;}- if(!std::isalnum(token[0]) && !keep_separators) {- return false;- }-token_index = token_counter++;return true;}@@ -176,3 +165,8 @@ void Tokenizer::tokenize(std::string& token) {size_t token_index;next(token, token_index);}++bool Tokenizer::next(std::string &token, size_t &token_index) {+ size_t start_index, end_index;+ return next(token, token_index, start_index, end_index);+}diff --git a/test/art_test.cpp b/test/art_test.cppindex 4af38758..61e7b057 100644--- a/test/art_test.cpp+++ b/test/art_test.cpp@@ -51,7 +51,7 @@ TEST(ArtTest, test_art_insert) {ASSERT_TRUE(art_size(&t) == line);line++;- delete document.offsets;+ delete [] document.offsets;}res = art_tree_destroy(&t);diff --git a/test/collection_locale_test.cpp b/test/collection_locale_test.cppindex 6bde3362..a180039e 100644--- a/test/collection_locale_test.cpp+++ b/test/collection_locale_test.cpp@@ -98,8 +98,8 @@ TEST_F(CollectionLocaleTest, SearchAgainstKoreanText) {}std::vector<std::vector<std::string>> records = {- {"경승지·산악·협곡", "Dustin Kensrue"},- {"안녕은하철도999극장판", "Gord Downie"},+ {"경승지·산악·협곡", "Dustin Kensrue"},+ {"안녕은하철도999극장판", "Gord Downie"},};for(size_t i=0; i<records.size(); i++) {@@ -129,3 +129,47 @@ TEST_F(CollectionLocaleTest, SearchAgainstKoreanText) {ASSERT_EQ("0", results["hits"][0]["document"]["id"].get<std::string>());ASSERT_EQ("경승지·<mark>산악</mark>·협곡", results["hits"][0]["highlights"][0]["snippet"].get<std::string>());}++TEST_F(CollectionLocaleTest, SearchAgainstKoreanTextContainingEnglishChars) {+ Collection *coll1;++ std::vector<field> fields = {field("title", field_types::STRING, false, false, true, DEFAULT_GEO_RESOLUTION, "th"),+ field("artist", field_types::STRING, false),+ field("points", field_types::INT32, false),};++ coll1 = collectionManager.get_collection("coll1").get();+ if(coll1 == nullptr) {+ coll1 = collectionManager.create_collection("coll1", 1, fields, "points").get();+ }++ std::vector<std::vector<std::string>> records = {+ {"개혁 등의 영향으로 11%나 위축됐다", "Dustin Kensrue"},+ };++ for(size_t i=0; i<records.size(); i++) {+ nlohmann::json doc;++ doc["id"] = std::to_string(i);+ doc["title"] = records[i][0];+ doc["artist"] = records[i][1];+ doc["points"] = i;++ ASSERT_TRUE(coll1->add(doc.dump()).ok());+ }++ auto results = coll1->search("위축됐다",+ {"title"}, "", {}, {}, 0, 10, 1, FREQUENCY).get();++ ASSERT_EQ(1, results["found"].get<size_t>());+ ASSERT_EQ(1, results["hits"].size());+ ASSERT_EQ("0", results["hits"][0]["document"]["id"].get<std::string>());+ ASSERT_EQ("개혁 등의 영향으로 11%나 <mark>위축됐다</mark>", results["hits"][0]["highlights"][0]["snippet"].get<std::string>());++ results = coll1->search("11%",+ {"title"}, "", {}, {}, 0, 10, 1, FREQUENCY).get();++ ASSERT_EQ(1, results["found"].get<size_t>());+ ASSERT_EQ(1, results["hits"].size());+ ASSERT_EQ("0", results["hits"][0]["document"]["id"].get<std::string>());+ ASSERT_EQ("개혁 등의 영향으로 <mark>11</mark>%나 위축됐다", results["hits"][0]["highlights"][0]["snippet"].get<std::string>());+}diff --git a/test/collection_synonyms_test.cpp b/test/collection_synonyms_test.cppindex 4b40cf6a..39f4a5ff 100644--- a/test/collection_synonyms_test.cpp+++ b/test/collection_synonyms_test.cpp@@ -359,8 +359,8 @@ TEST_F(CollectionSynonymsTest, MultiWaySynonym) {ASSERT_EQ(2, res["hits"].size());ASSERT_EQ(2, res["found"].get<uint32_t>());- ASSERT_STREQ("<mark>Samuel</mark> <mark>L</mark>. <mark>Jackson</mark>", res["hits"][0]["highlights"][0]["snippet"].get<std::string>().c_str());- ASSERT_STREQ("<mark>Samuel</mark> <mark>L</mark>. <mark>Jackson</mark>", res["hits"][1]["highlights"][0]["snippet"].get<std::string>().c_str());+ ASSERT_STREQ("<mark>Samuel</mark> <mark>L.</mark> <mark>Jackson</mark>", res["hits"][0]["highlights"][0]["snippet"].get<std::string>().c_str());+ ASSERT_STREQ("<mark>Samuel</mark> <mark>L.</mark> <mark>Jackson</mark>", res["hits"][1]["highlights"][0]["snippet"].get<std::string>().c_str());// for now we don't support synonyms on ANY prefixdiff --git a/test/collection_test.cpp b/test/collection_test.cppindex ef4ed5f9..cbe2ea2a 100644--- a/test/collection_test.cpp+++ b/test/collection_test.cpp@@ -2028,7 +2028,7 @@ TEST_F(CollectionTest, SearchLargeTextField) {ASSERT_EQ(1, results["hits"].size());- ASSERT_STREQ("non arcu id lectus <mark>accumsan</mark> venenatis at at justo",+ ASSERT_STREQ("non arcu id lectus <mark>accumsan</mark> venenatis at at justo.",results["hits"][0]["highlights"][0]["snippet"].get<std::string>().c_str());collectionManager.drop_collection("coll_large_text");@@ -3149,7 +3149,7 @@ TEST_F(CollectionTest, MultiFieldHighlighting) {results["hits"][0]["highlights"][0]["snippet"].get<std::string>());ASSERT_EQ("description", results["hits"][0]["highlights"][1]["field"].get<std::string>());- ASSERT_EQ("Easily replenish your cell phone with this wireless <mark>charger</mark>.",+ ASSERT_EQ("Easily replenish your cell phone with this wireless <mark>charger.</mark>",results["hits"][0]["highlights"][1]["snippet"].get<std::string>());collectionManager.drop_collection("coll1");@@ -3338,9 +3338,10 @@ TEST_F(CollectionTest, HighlightWithAccentedCharacters) {}std::vector<std::vector<std::string>> records = {- {"Mise à jour Timy depuis PC"},- {"Down There by the Train"},+ {"Mise T.J. à jour Timy depuis PC"},+ {"Down There by the T.r.a.i.n"},{"State Trooper"},+ {"The Google Nexus Q Is Baffling"},};for (size_t i = 0; i < records.size(); i++) {@@ -3358,13 +3359,36 @@ TEST_F(CollectionTest, HighlightWithAccentedCharacters) {ASSERT_EQ(1, results["found"].get<size_t>());ASSERT_EQ(1, results["hits"].size());- ASSERT_STREQ("Mise <mark>à</mark> <mark>jour</mark> Timy depuis PC",+ ASSERT_STREQ("Mise T.J. <mark>à</mark> <mark>jour</mark> Timy depuis PC",results["hits"][0]["highlights"][0]["snippet"].get<std::string>().c_str());ASSERT_EQ(2, results["hits"][0]["highlights"][0]["matched_tokens"].size());ASSERT_STREQ("à", results["hits"][0]["highlights"][0]["matched_tokens"][0].get<std::string>().c_str());ASSERT_STREQ("jour", results["hits"][0]["highlights"][0]["matched_tokens"][1].get<std::string>().c_str());+ results = coll1->search("by train", {"title"}, "", {}, {}, 0, 10, 1, FREQUENCY).get();++ ASSERT_EQ(1, results["found"].get<size_t>());+ ASSERT_EQ(1, results["hits"].size());++ ASSERT_STREQ("Down There <mark>by</mark> the <mark>T.r.a.i.n</mark>",+ results["hits"][0]["highlights"][0]["snippet"].get<std::string>().c_str());++ results = coll1->search("state trooper", {"title"}, "", {}, {}, 0, 10, 1, FREQUENCY).get();++ ASSERT_EQ(1, results["found"].get<size_t>());+ ASSERT_EQ(1, results["hits"].size());++ ASSERT_STREQ("<mark>State</mark> <mark>Trooper</mark>",+ results["hits"][0]["highlights"][0]["snippet"].get<std::string>().c_str());++ // test single character highlight++ results = coll1->search("q", {"title"}, "", {}, {}, 0, 10, 1, FREQUENCY).get();+ ASSERT_EQ(1, results["found"].get<size_t>());+ ASSERT_STREQ("The Google Nexus <mark>Q</mark> Is Baffling",+ results["hits"][0]["highlights"][0]["snippet"].get<std::string>().c_str());+collectionManager.drop_collection("coll1");}diff --git a/test/tokenizer_test.cpp b/test/tokenizer_test.cppindex a7430cd2..681d12b1 100644--- a/test/tokenizer_test.cpp+++ b/test/tokenizer_test.cpp@@ -3,9 +3,19 @@#include "logger.h"TEST(TokenizerTest, ShouldTokenizeNormalizeDifferentStrings) {- const std::string withnewline = "Michael Jordan:\nWelcome, everybody. Welcome! ";+ const std::string withaccent = "Mise T.J. à jour Timy depuis PC";std::vector<std::string> tokens;- Tokenizer(withnewline, false, true, false).tokenize(tokens);+ Tokenizer(withaccent, true, false).tokenize(tokens);++ std::vector<std::string> withaccent_tokens = {"mise", "tj", "a", "jour", "timy", "depuis", "pc"};+ ASSERT_EQ(withaccent_tokens.size(), tokens.size());+ for(size_t i = 0; i < withaccent_tokens.size(); i++) {+ ASSERT_EQ(withaccent_tokens[i], tokens[i]);+ }++ const std::string withnewline = "Michael Jordan:\nWelcome, everybody. Welcome! ";+ tokens.clear();+ Tokenizer(withnewline, true, false).tokenize(tokens);ASSERT_EQ(5, tokens.size());ASSERT_STREQ("michael", tokens[0].c_str());ASSERT_STREQ("jordan", tokens[1].c_str());@@ -15,16 +25,7 @@ TEST(TokenizerTest, ShouldTokenizeNormalizeDifferentStrings) {const std::string withspaces = " Michael Jordan ";tokens.clear();- Tokenizer(withspaces, true, true, false).tokenize(tokens);- ASSERT_EQ(5, tokens.size());- ASSERT_STREQ(" ", tokens[0].c_str());- ASSERT_STREQ("michael", tokens[1].c_str());- ASSERT_STREQ(" ", tokens[2].c_str());- ASSERT_STREQ("jordan", tokens[3].c_str());- ASSERT_STREQ(" ", tokens[4].c_str());-- tokens.clear();- Tokenizer(withspaces, false, true, false).tokenize(tokens);+ Tokenizer(withspaces, true, false).tokenize(tokens);ASSERT_EQ(2, tokens.size());ASSERT_STREQ("michael", tokens[0].c_str());ASSERT_STREQ("jordan", tokens[1].c_str());@@ -32,49 +33,42 @@ TEST(TokenizerTest, ShouldTokenizeNormalizeDifferentStrings) {// single tokenconst std::string single_token = "foobar";tokens.clear();- Tokenizer(single_token, false, false, false).tokenize(tokens);+ Tokenizer(single_token, false, false).tokenize(tokens);ASSERT_EQ(1, tokens.size());ASSERT_STREQ("foobar", tokens[0].c_str());// split tokensconst std::string split_tokens = "foo-bar-baz";tokens.clear();- Tokenizer(split_tokens, false, false, false).tokenize(tokens);+ Tokenizer(split_tokens, false, false).tokenize(tokens);ASSERT_EQ(1, tokens.size());ASSERT_STREQ("foobarbaz", tokens[0].c_str());tokens.clear();- Tokenizer(split_tokens, false, true, false).tokenize(tokens);+ Tokenizer(split_tokens, true, false).tokenize(tokens);ASSERT_EQ(1, tokens.size());ASSERT_STREQ("foobarbaz", tokens[0].c_str());// multiple spacesconst std::string multispace_tokens = "foo bar";tokens.clear();- Tokenizer(multispace_tokens, true, false, false).tokenize(tokens);- ASSERT_EQ(3, tokens.size());+ Tokenizer(multispace_tokens, false, false).tokenize(tokens);+ ASSERT_EQ(2, tokens.size());ASSERT_STREQ("foo", tokens[0].c_str());- ASSERT_STREQ(" ", tokens[1].c_str());- ASSERT_STREQ("bar", tokens[2].c_str());+ ASSERT_STREQ("bar", tokens[1].c_str());// special charsconst std::string specialchar_tokens = "https://www.amazon.com/s?k=phone&ref=nb_sb_noss_2";tokens.clear();- Tokenizer(specialchar_tokens, true, false, false).tokenize(tokens);- ASSERT_EQ(23, tokens.size());- ASSERT_STREQ("https", tokens[0].c_str());- ASSERT_STREQ("://", tokens[1].c_str());- ASSERT_STREQ("www", tokens[2].c_str());- ASSERT_STREQ(".", tokens[3].c_str());- ASSERT_STREQ("noss", tokens[20].c_str());- ASSERT_STREQ("_", tokens[21].c_str());- ASSERT_STREQ("2", tokens[22].c_str());+ Tokenizer(specialchar_tokens, false, false).tokenize(tokens);+ ASSERT_EQ(1, tokens.size());+ ASSERT_STREQ("httpswwwamazoncomskphonerefnbsbnoss2", tokens[0].c_str());// nooptokens.clear();const std::string withspecialchars = "Special ½¥ and தமிழ் 你好吗 abcÅà123ß12 here.";- Tokenizer(withspecialchars, false, true, true).tokenize(tokens);+ Tokenizer(withspecialchars, true, true).tokenize(tokens);ASSERT_EQ(1, tokens.size());ASSERT_STREQ(withspecialchars.c_str(), tokens[0].c_str());}@@ -82,35 +76,23 @@ TEST(TokenizerTest, ShouldTokenizeNormalizeDifferentStrings) {TEST(TokenizerTest, ShouldTokenizeNormalizeUnicodeStrings) {std::vector<std::string> tokens;- const std::string withspecialchars = "Special ½¥ and -தமிழ் 你2好吗 abcÅà123ß12 here.";+ const std::string withspecialchars = "Special ½¥ and -thenதமிழ், 你2好吗 abcÅà123ß12 verläßlich here.";tokens.clear();- Tokenizer(withspecialchars, false, true, false).tokenize(tokens);- ASSERT_EQ(7, tokens.size());+ Tokenizer(withspecialchars, true, false).tokenize(tokens);+ ASSERT_EQ(8, tokens.size());ASSERT_STREQ("special", tokens[0].c_str());ASSERT_STREQ("12yen", tokens[1].c_str());ASSERT_STREQ("and", tokens[2].c_str());- ASSERT_STREQ("தமிழ்", tokens[3].c_str());+ ASSERT_STREQ("thenதமிழ்", tokens[3].c_str());ASSERT_STREQ("你2好吗", tokens[4].c_str());ASSERT_STREQ("abcaa123ss12", tokens[5].c_str());- ASSERT_STREQ("here", tokens[6].c_str());+ ASSERT_STREQ("verlasslich", tokens[6].c_str());+ ASSERT_STREQ("here", tokens[7].c_str());- // when normalization is disabled and keep empty is enabled+ // when normalization is disabledconst std::string withoutnormalize = "Mise à, jour.";tokens.clear();- Tokenizer(withoutnormalize, true, false, false).tokenize(tokens);- ASSERT_EQ(7, tokens.size());- ASSERT_STREQ("Mise", tokens[0].c_str());- ASSERT_STREQ(" ", tokens[1].c_str());- ASSERT_STREQ("à", tokens[2].c_str());- ASSERT_STREQ(",", tokens[3].c_str());- ASSERT_STREQ(" ", tokens[4].c_str());- ASSERT_STREQ("jour", tokens[5].c_str());- ASSERT_STREQ(".", tokens[6].c_str());-- // when normalization and keep empty are disabled- const std::string withoutnormalizeandkeepempty = "Mise à jour.";- tokens.clear();- Tokenizer(withoutnormalizeandkeepempty, false, false, false).tokenize(tokens);+ Tokenizer(withoutnormalize, false, false).tokenize(tokens);ASSERT_EQ(3, tokens.size());ASSERT_STREQ("Mise", tokens[0].c_str());ASSERT_STREQ("à", tokens[1].c_str());@@ -119,12 +101,7 @@ TEST(TokenizerTest, ShouldTokenizeNormalizeUnicodeStrings) {// single accented word tokenizationstd::string singleword = "à";tokens.clear();- Tokenizer(singleword, false, true, false).tokenize(tokens);- ASSERT_EQ(1, tokens.size());- ASSERT_STREQ("a", tokens[0].c_str());-- tokens.clear();- Tokenizer(singleword, true, true, false).tokenize(tokens);+ Tokenizer(singleword, true, false).tokenize(tokens);ASSERT_EQ(1, tokens.size());ASSERT_STREQ("a", tokens[0].c_str());}@@ -132,7 +109,7 @@ TEST(TokenizerTest, ShouldTokenizeNormalizeUnicodeStrings) {TEST(TokenizerTest, ShouldTokenizeIteratively) {const std::string withnewline = "Michael Jordan:\n\nWelcome, everybody. Welcome!";std::vector<std::string> tokens;- Tokenizer tokenizer1(withnewline, true, true, false);+ Tokenizer tokenizer1(withnewline, true, false);std::string token;size_t token_index;@@ -141,23 +118,15 @@ TEST(TokenizerTest, ShouldTokenizeIteratively) {tokens.push_back(token);}- ASSERT_EQ(13, tokens.size());+ ASSERT_EQ(5, tokens.size());ASSERT_STREQ("michael", tokens[0].c_str());- ASSERT_STREQ(" ", tokens[1].c_str());- ASSERT_STREQ("jordan", tokens[2].c_str());- ASSERT_STREQ(":", tokens[3].c_str());- ASSERT_STREQ("\n\n", tokens[4].c_str());- ASSERT_STREQ("welcome", tokens[5].c_str());- ASSERT_STREQ(",", tokens[6].c_str());- ASSERT_STREQ(" ", tokens[7].c_str());- ASSERT_STREQ("everybody", tokens[8].c_str());- ASSERT_STREQ(".", tokens[9].c_str());- ASSERT_STREQ(" ", tokens[10].c_str());- ASSERT_STREQ("welcome", tokens[11].c_str());- ASSERT_STREQ("!", tokens[12].c_str());+ ASSERT_STREQ("jordan", tokens[1].c_str());+ ASSERT_STREQ("welcome", tokens[2].c_str());+ ASSERT_STREQ("everybody", tokens[3].c_str());+ ASSERT_STREQ("welcome", tokens[4].c_str());// check for index when separators are not kept- Tokenizer tokenizer2(withnewline, false, true, false);+ Tokenizer tokenizer2(withnewline, true, false);size_t expected_token_index = 0;std::vector<std::string> expected_tokens = {"michael", "jordan", "welcome", "everybody", "welcome"};while(tokenizer2.next(token, token_index)) {@@ -169,7 +138,7 @@ TEST(TokenizerTest, ShouldTokenizeIteratively) {// verbatim (no_op=true)tokens.clear();- Tokenizer tokenizer3(withnewline, true, false, true);+ Tokenizer tokenizer3(withnewline, false, true);while(tokenizer3.next(token, token_index)) {tokens.push_back(token);@@ -179,10 +148,23 @@ TEST(TokenizerTest, ShouldTokenizeIteratively) {ASSERT_STREQ("Michael Jordan:\n\nWelcome, everybody. Welcome!", tokens[0].c_str());}+TEST(TokenizerTest, ShouldTokenizeTextWithCustomSpecialChars) {+ std::vector<std::string> tokens;+ Tokenizer("and -some -more", true, false, "en", {'-'}).tokenize(tokens);+ ASSERT_EQ(3, tokens.size());+ ASSERT_EQ("and", tokens[0]);+ ASSERT_EQ("-some", tokens[1]);+ ASSERT_EQ("-more", tokens[2]);+}+TEST(TokenizerTest, ShouldTokenizeLocaleText) {+ std::string tstr = "ลงรถไฟ";+ std::vector<std::string> ttokens;+ Tokenizer(tstr, true, false, "th").tokenize(ttokens);+std::string str = "จิ้งจอกสีน้ำตาลด่วน";std::vector<std::string> tokens;- Tokenizer(str, false, false, false, "th").tokenize(tokens);+ Tokenizer(str, false, false, "th").tokenize(tokens);ASSERT_EQ(4, tokens.size());ASSERT_EQ("จิ้งจอก", tokens[0]);@@ -192,7 +174,7 @@ TEST(TokenizerTest, ShouldTokenizeLocaleText) {tokens.clear();str = "น. วันที่ 31 มี.ค.";- Tokenizer(str, false, false, false, "th").tokenize(tokens);+ Tokenizer(str, false, false, "th").tokenize(tokens);ASSERT_EQ(5, tokens.size());ASSERT_EQ("น", tokens[0]);ASSERT_EQ("วัน", tokens[1]);@@ -201,7 +183,7 @@ TEST(TokenizerTest, ShouldTokenizeLocaleText) {ASSERT_EQ("มี.ค", tokens[4]);tokens.clear();- Tokenizer("Odd Thomas", false, false, false, "en").tokenize(tokens);+ Tokenizer("Odd Thomas", false, false, "en").tokenize(tokens);ASSERT_EQ(2, tokens.size());ASSERT_EQ("Odd", tokens[0]);ASSERT_EQ("Thomas", tokens[1]);@@ -209,14 +191,14 @@ TEST(TokenizerTest, ShouldTokenizeLocaleText) {// koreantokens.clear();- Tokenizer("경승지·산악·협곡", false, false, false, "ko").tokenize(tokens);+ Tokenizer("경승지·산악·협곡", false, false, "ko").tokenize(tokens);ASSERT_EQ(3, tokens.size());ASSERT_EQ("경승지", tokens[0]);ASSERT_EQ("산악", tokens[1]);ASSERT_EQ("협곡", tokens[2]);tokens.clear();- Tokenizer("안녕은하철도999극장판", false, false, false, "ko").tokenize(tokens);+ Tokenizer("안녕은하철도999극장판", false, false, "ko").tokenize(tokens);ASSERT_EQ(3, tokens.size());ASSERT_EQ("안녕은하철도", tokens[0]);ASSERT_EQ("999", tokens[1]);@@ -224,13 +206,13 @@ TEST(TokenizerTest, ShouldTokenizeLocaleText) {// japanesetokens.clear();- Tokenizer("退屈", false, false, false, "ja").tokenize(tokens);+ Tokenizer("退屈", false, false, "ja").tokenize(tokens);ASSERT_EQ(2, tokens.size());ASSERT_EQ("た", tokens[0]);ASSERT_EQ("いくつ", tokens[1]);tokens.clear();- Tokenizer("ア退屈であ", false, false, false, "ja").tokenize(tokens);+ Tokenizer("ア退屈であ", false, false, "ja").tokenize(tokens);ASSERT_EQ(5, tokens.size());ASSERT_EQ("あ", tokens[0]);ASSERT_EQ("た", tokens[1]);@@ -239,7 +221,7 @@ TEST(TokenizerTest, ShouldTokenizeLocaleText) {ASSERT_EQ("あ", tokens[4]);tokens.clear();- Tokenizer("怠惰な犬", false, false, false, "ja").tokenize(tokens);+ Tokenizer("怠惰な犬", false, false, "ja").tokenize(tokens);ASSERT_EQ(4, tokens.size());ASSERT_EQ("たい", tokens[0]);ASSERT_EQ("だ", tokens[1]);