Open Source alternative to Algolia + Pinecone and an Easier-to-Use alternative to ElasticSearch ⚡ 🔍 ✨ Fast, typo tolerant, in-memory fuzzy Search Engine for building delightful search experiences
C++ algoliadatastoreelasticsearchenterprise-searchfacetingfull-text-searchfuzzy-searchgeosearchin-memoryinstantsearchmerchandisingpineconesearchsearch-enginesemantic-searchsimilarity-searchsite-searchsynonymstypo-tolerancevector-search
Fix highlighting of strings with special characters.
kishorencc2eec85277ac
diff --git a/include/index.h b/include/index.hindex 73e45f9e..fcacae77 100644--- a/include/index.h+++ b/include/index.h@@ -389,10 +389,10 @@ public:static void transform_for_180th_meridian(GeoCoord& point, double offset);- // the following methods are not synchronized because their parent calls are synchronized-art_leaf* get_token_leaf(const std::string & field_name, const unsigned char* token, uint32_t token_len);+ // the following methods are not synchronized because their parent calls are synchronized+uint32_t do_filtering(uint32_t** filter_ids_out, const std::vector<filter> & filters) const;static Option<uint32_t> validate_index_in_memory(nlohmann::json &document, uint32_t seq_id,diff --git a/include/match_score.h b/include/match_score.hindex 5224c9dd..c5d69733 100644--- a/include/match_score.h+++ b/include/match_score.h@@ -30,6 +30,10 @@ struct TokenOffset {bool operator>(const TokenOffset &a) const {return offset > a.offset;}++ bool operator<(const TokenOffset &a) const {+ return offset < a.offset;+ }};struct Match {diff --git a/include/tokenizer.h b/include/tokenizer.hindex 39c0b511..985d59e8 100644--- a/include/tokenizer.h+++ b/include/tokenizer.h@@ -8,19 +8,32 @@ class Tokenizer {private:const std::string& text;size_t i;- const bool keep_empty;+ const bool keep_separators;const bool normalize;const bool no_op;size_t token_counter = 0;iconv_t cd;+ static const size_t CHARS = 0;+ static const size_t SEPARATORS = 1;+ size_t stream_mode;++ std::stringstream out;+public:explicit Tokenizer(const std::string& input,- bool keep_empty=true, bool normalize=true, bool no_op=false):- text(input), i(0), keep_empty(keep_empty), normalize(normalize), no_op(no_op) {+ bool keep_separators=true, bool normalize=true, bool no_op=false):+ text(input), i(0), keep_separators(keep_separators), normalize(normalize), no_op(no_op) {cd = iconv_open("ASCII//TRANSLIT", "UTF-8");++ if(!input.empty() && (std::isalnum(text[0]) || (text[i] & ~0x7f) != 0)) {+ // alphanum or non-ascii+ stream_mode = CHARS;+ } else {+ stream_mode = SEPARATORS;+ }}~Tokenizer() {diff --git a/src/collection.cpp b/src/collection.cppindex a98e0e1a..96ae52c9 100644--- a/src/collection.cpp+++ b/src/collection.cpp@@ -1223,20 +1223,27 @@ void Collection::parse_search_query(const std::string &query, std::vector<std::sq_include_tokens = {query};} else {std::vector<std::string> tokens;- StringUtils::split(query, tokens, " ");-- for(std::string& token: tokens) {- if(token[0] == '-') {- std::string&& just_token = token.substr(1);- Tokenizer(just_token, false, true).tokenize(just_token);- if(!just_token.empty()) {- q_exclude_tokens.push_back(just_token);- }+ Tokenizer(query, true, true).tokenize(tokens);+ bool exclude_operator_prior = false;++ for(const auto& token: tokens) {+ if(token.empty()) {+ continue;+ }++ if(token == "-" || token == " -") {+ exclude_operator_prior = true;+ }++ if(!std::isalnum(token[0])) {+ continue;+ }++ if(exclude_operator_prior) {+ q_exclude_tokens.push_back(token);+ exclude_operator_prior = false;} else {- Tokenizer(token, false, true).tokenize(token);- if(!token.empty()) {- q_include_tokens.push_back(token);- }+ q_include_tokens.push_back(token);}}@@ -1383,7 +1390,9 @@ void Collection::highlight_result(const field &search_field,// is from the best matched field and need not be present in other fields of a document.Index* index = indices[field_order_kv->key % num_memory_shards];art_leaf *actual_leaf = index->get_token_leaf(search_field.name, &token_leaf->key[0], token_leaf->key_len);+if(actual_leaf != nullptr) {+ //LOG(INFO) << "field: " << search_field.name << ", key: " << actual_leaf->key;query_suggestion.push_back(actual_leaf);std::vector<uint16_t> positions;uint32_t doc_index = actual_leaf->values->ids.indexOf(field_order_kv->key);@@ -1432,66 +1441,84 @@ void Collection::highlight_result(const field &search_field,std::partial_sort(match_indices.begin(), match_indices.begin()+max_array_matches, match_indices.end());for(size_t index = 0; index < max_array_matches; index++) {- const match_index_t & match_index = match_indices[index];- const Match & match = match_index.match;+ std::sort(match_indices[index].match.offsets.begin(), match_indices[index].match.offsets.end());+ const auto& match_index = match_indices[index];+ const Match& match = match_index.match;- std::vector<std::string> tokens;+ const std::string& text = (search_field.type == field_types::STRING) ? document[search_field.name] : document[search_field.name][match_index.index];+ Tokenizer tokenizer(text, true, false);- if(search_field.type == field_types::STRING) {- Tokenizer(document[search_field.name], true, false).tokenize(tokens);- } else {- Tokenizer(document[search_field.name][match_index.index], true, false).tokenize(tokens);- }+ std::string raw_token;+ size_t raw_token_index = 0;+ int indexed_token_index = -1;+ size_t match_offset_index = 0;- std::vector<size_t> token_indices;+ std::set<size_t> token_indices;spp::sparse_hash_set<std::string> token_hits;+ std::vector<std::string> raw_tokens;+ std::unordered_map<size_t, size_t> indexed_to_raw;++ while(tokenizer.next(raw_token, raw_token_index)) {+ if(!raw_token.empty() && (std::isalnum(raw_token[0]) || (raw_token[0] & ~0x7f) != 0)) {+ // check for actual token (first char is NOT alphanum or ascii)+ indexed_token_index++;+ indexed_to_raw[indexed_token_index] = raw_token_index;+ /*LOG(INFO) << "raw_token: " << raw_token << ", indexed_token_index: " << indexed_token_index+ << ", raw_token_index: " << raw_token_index;*/+ }- for(size_t i = 0; i < match.offsets.size(); i++) {- if(match.offsets[i].offset != MAX_DISPLACEMENT) {- size_t token_index = (size_t)(match.offsets[i].offset);- token_indices.push_back(token_index);- if(token_index >= tokens.size()) {- LOG(ERROR) << "Highlight token index " << token_index << " is greater than length of store field.";- continue;+ if (match_offset_index < match.offsets.size() &&+ match.offsets[match_offset_index].offset == indexed_token_index) {+ std::string indexed_token;+ Tokenizer(raw_token, true, true).tokenize(indexed_token);++ if(token_indices.count(indexed_token_index) == 0) {+ // repetition could occur, for e.g. in the case of synonym constructed queries+ token_indices.insert(indexed_token_index);+ token_hits.insert(indexed_token);}- std::string token = tokens[token_index];- Tokenizer(token, true, true).tokenize(token);- token_hits.insert(token);+ match_offset_index++;}++ raw_tokens.push_back(raw_token);}+ size_t num_indexed_tokens = indexed_token_index + 1;auto minmax = std::minmax_element(token_indices.begin(), token_indices.end());size_t prefix_length = highlight_affix_num_tokens;- size_t suffix_length = highlight_affix_num_tokens + 1;+ size_t suffix_length = highlight_affix_num_tokens;- // For longer strings, pick surrounding tokens within `prefix_length` of min_index and max_index for snippet- const size_t start_index = (tokens.size() <= snippet_threshold) ? 0 :- std::max(0, (int)(*(minmax.first) - prefix_length));+ if(num_indexed_tokens == 0) {+ continue;+ }- const size_t end_index = (tokens.size() <= snippet_threshold) ? tokens.size() :- std::min((int)tokens.size(), (int)(*(minmax.second) + suffix_length));+ // For longer strings, pick surrounding raw_tokens within `prefix_length` of min_index and max_index for snippet+ const size_t start_index = (num_indexed_tokens <= snippet_threshold) ? 0 :+ indexed_to_raw[std::max(0, (int)(*(minmax.first) - prefix_length))];++ const size_t end_index = (num_indexed_tokens <= snippet_threshold) ? raw_tokens.size() - 1 :+ indexed_to_raw[std::min((int)num_indexed_tokens - 1, (int)(*(minmax.second) + suffix_length))];std::stringstream snippet_stream;highlight.matched_tokens.emplace_back();std::vector<std::string>& matched_tokens = highlight.matched_tokens.back();+ size_t snippet_index = start_index;- for(size_t snippet_index = start_index; snippet_index < end_index; snippet_index++) {- if(snippet_index != start_index) {- snippet_stream << " ";- }-- std::string token = tokens[snippet_index];- Tokenizer(token, true, true).tokenize(token);+ while(snippet_index <= end_index) {+ std::string normalized_token;+ Tokenizer(raw_tokens[snippet_index], true, true).tokenize(normalized_token);- if(token_hits.count(token) != 0) {- snippet_stream << highlight_start_tag << tokens[snippet_index] << highlight_end_tag;- matched_tokens.push_back(tokens[snippet_index]);+ if(token_hits.count(normalized_token) != 0) {+ snippet_stream << highlight_start_tag << raw_tokens[snippet_index] << highlight_end_tag;+ matched_tokens.push_back(raw_tokens[snippet_index]);} else {- snippet_stream << tokens[snippet_index];+ snippet_stream << raw_tokens[snippet_index];}++ snippet_index++;}highlight.snippets.push_back(snippet_stream.str());@@ -1501,18 +1528,14 @@ void Collection::highlight_result(const field &search_field,if(highlighted_fully) {std::stringstream value_stream;- for(size_t value_index = 0; value_index < tokens.size(); value_index++) {- if(value_index != 0) {- value_stream << " ";- }-- std::string token = tokens[value_index];- Tokenizer(token, true, true).tokenize(token);+ for(size_t value_index = 0; value_index < raw_tokens.size(); value_index++) {+ std::string normalized_token;+ Tokenizer(raw_tokens[value_index], true, true).tokenize(normalized_token);- if(token_hits.count(token) != 0) {- value_stream << highlight_start_tag << tokens[value_index] << highlight_end_tag;+ if(token_hits.count(normalized_token) != 0) {+ value_stream << highlight_start_tag << raw_tokens[value_index] << highlight_end_tag;} else {- value_stream << tokens[value_index];+ value_stream << raw_tokens[value_index];}}diff --git a/src/index.cpp b/src/index.cppindex c272eeec..8500dee5 100644--- a/src/index.cpp+++ b/src/index.cpp@@ -575,7 +575,7 @@ void Index::index_string_field(const std::string & text, const int64_t score, aruint32_t seq_id, bool is_facet, const field & a_field) {std::unordered_map<std::string, std::vector<uint32_t>> token_to_offsets;- Tokenizer tokenizer(text, true, true, !a_field.is_string());+ Tokenizer tokenizer(text, false, true, !a_field.is_string());std::string token;size_t token_index = 0;@@ -588,7 +588,6 @@ void Index::index_string_field(const std::string & text, const int64_t score, arif(is_facet) {uint64_t hash = facet_token_hash(a_field, token);- //facet_index_v2[seq_id][facet_id].push_back(hash);facet_hashes.push_back(hash);}@@ -623,7 +622,7 @@ void Index::index_string_array_field(const std::vector<std::string> & strings, cconst std::string& str = strings[array_index];std::set<std::string> token_set; // required to deal with repeating tokens- Tokenizer tokenizer(str, true, true, !a_field.is_string());+ Tokenizer tokenizer(str, false, true, !a_field.is_string());std::string token;size_t token_index = 0;@@ -2216,6 +2215,8 @@ void Index::populate_token_positions(const std::vector<art_leaf *>& query_suggesfor(size_t i = 0; i < query_suggestion.size(); i++) {const art_leaf* token_leaf = query_suggestion[i];uint32_t doc_index = leaf_to_indices[i][result_index];+ /*LOG(INFO) << "doc_id: " << token_leaf->values->ids.at(doc_index) << ", token_leaf->values->ids.getLength(): "+ << token_leaf->values->ids.getLength();*/// it's possible for a query token to not appear in a resulting documentif(doc_index == token_leaf->values->ids.getLength()) {@@ -2229,7 +2230,14 @@ void Index::populate_token_positions(const std::vector<art_leaf *>& query_sugges/*uint32_t* offsets = token_leaf->values->offsets.uncompress();for(size_t ii=0; ii < token_leaf->values->offsets.getLength(); ii++) {LOG(INFO) << "offset: " << offsets[ii];- }*/+ }++ uint32_t* offset_indices = token_leaf->values->offset_index.uncompress();+ for(size_t ii=0; ii < token_leaf->values->offset_index.getLength(); ii++) {+ LOG(INFO) << "offset index: " << offset_indices[ii];+ }++ LOG(INFO) << "token_leaf->values->offsets.getLength(): " << token_leaf->values->offsets.getLength();*/uint32_t start_offset = token_leaf->values->offset_index.at(doc_index);uint32_t end_offset = (doc_index == token_leaf->values->ids.getLength() - 1) ?@@ -2464,6 +2472,7 @@ void Index::tokenize_doc_field(const nlohmann::json& document, const field& sear}art_leaf* Index::get_token_leaf(const std::string & field_name, const unsigned char* token, uint32_t token_len) {+ std::shared_lock lock(mutex);const art_tree *t = search_index.at(field_name);return (art_leaf*) art_search(t, token, (int) token_len);}diff --git a/src/tokenizer.cpp b/src/tokenizer.cppindex 9dc86cd8..ac99a2bb 100644--- a/src/tokenizer.cpp+++ b/src/tokenizer.cpp@@ -2,62 +2,63 @@#include "tokenizer.h"bool Tokenizer::next(std::string &token, size_t& token_index) {- std::stringstream out;-- if(i >= text.size()) {- if(i == text.size() && !text.empty() && text.back() == ' ') {- token = "";- i++;- return true;+ if(no_op) {+ if(i == text.size()) {+ return false;}- return false;- }-- if(no_op) {token = text;i = text.size();return true;}while(i < text.size()) {- if((text[i] & ~0x7f) == 0 ) {- // ASCII character: split on space/newline or lowercase otherwise- if(std::isalnum(text[i])) {+ bool is_ascii = (text[i] & ~0x7f) == 0;+ if(is_ascii) {+ const size_t next_stream_mode = std::isalnum(text[i]) ? CHARS : SEPARATORS;++ if(next_stream_mode != stream_mode) {+ // We tokenize when `stream_mode` changes+ token = out.str();++ out.str(std::string());if(normalize) {out << char(std::tolower(text[i]));} else {out << text[i];}- } else {- bool is_space = text[i] == 32;- bool is_new_line = text[i] == 10;- bool is_whitespace = is_space || is_new_line;+ i++;- bool next_char_alphanum = (i != text.length() - 1) && std::isalnum(text[i + 1]);+ if(stream_mode == SEPARATORS && !keep_separators) {+ stream_mode = next_stream_mode;+ continue;+ }- if(!normalize && !is_whitespace && (i == text.length() - 1 || !next_char_alphanum)) {- // checking for next char ensures that `foo-bar` does not get split to `foo-`+ token_index = token_counter++;+ stream_mode = next_stream_mode;+ return true;+ } else {+ if(normalize) {+ out << char(std::tolower(text[i]));+ } else {out << text[i];}- if(is_whitespace || next_char_alphanum) {- // we split on space or on a special character whose next char is alphanumeric- token = out.str();- out.clear();- i++;+ i++;+ continue;+ }+ }- if(!keep_empty && token.empty()) {- continue;- }+ if(stream_mode == SEPARATORS) { // to detect first non-ascii character+ // we will tokenize now and treat the following non-ascii chars as a different token+ stream_mode = CHARS;+ token = out.str();+ out.str(std::string());- token_index = token_counter++;- return true;- }+ if(keep_separators) {+ token_index = token_counter++;+ return true;}-- i++;- continue;}char inbuf[5];@@ -90,18 +91,17 @@ bool Tokenizer::next(std::string &token, size_t& token_index) {// symbol cannot be represented as ASCII, so write the original symbolout << inbuf;} else {- // NOTE: outsize indicates bytes available AFTER current position so have to do <=for(size_t out_index=0; out_index<5; out_index++) {if(!normalize) {out << outbuf[out_index];continue;}- bool is_ascii = ((outbuf[out_index] & ~0x7f) == 0);- bool keep_char = !is_ascii || std::isalnum(outbuf[out_index]);+ bool unicode_is_ascii = ((outbuf[out_index] & ~0x7f) == 0);+ bool keep_char = !unicode_is_ascii || std::isalnum(outbuf[out_index]);if(keep_char) {- if(is_ascii && std::isalnum(outbuf[out_index])) {+ if(unicode_is_ascii && std::isalnum(outbuf[out_index])) {outbuf[out_index] = char(std::tolower(outbuf[out_index]));}out << outbuf[out_index];@@ -111,9 +111,13 @@ bool Tokenizer::next(std::string &token, size_t& token_index) {}token = out.str();- out.clear();+ out.str(std::string());++ if(token.empty()) {+ return false;+ }- if(!keep_empty && token.empty()) {+ if(!std::isalnum(token[0]) && !keep_separators) {return false;}diff --git a/test/collection_synonyms_test.cpp b/test/collection_synonyms_test.cppindex 322b0f9a..61b1c742 100644--- a/test/collection_synonyms_test.cpp+++ b/test/collection_synonyms_test.cpp@@ -359,8 +359,8 @@ TEST_F(CollectionSynonymsTest, MultiWaySynonym) {ASSERT_EQ(2, res["hits"].size());ASSERT_EQ(2, res["found"].get<uint32_t>());- ASSERT_STREQ("<mark>Samuel</mark> <mark>L.</mark> <mark>Jackson</mark>", res["hits"][0]["highlights"][0]["snippet"].get<std::string>().c_str());- ASSERT_STREQ("<mark>Samuel</mark> <mark>L.</mark> <mark>Jackson</mark>", res["hits"][1]["highlights"][0]["snippet"].get<std::string>().c_str());+ ASSERT_STREQ("<mark>Samuel</mark> <mark>L</mark>. <mark>Jackson</mark>", res["hits"][0]["highlights"][0]["snippet"].get<std::string>().c_str());+ ASSERT_STREQ("<mark>Samuel</mark> <mark>L</mark>. <mark>Jackson</mark>", res["hits"][1]["highlights"][0]["snippet"].get<std::string>().c_str());// for now we don't support synonyms on ANY prefixdiff --git a/test/collection_test.cpp b/test/collection_test.cppindex 871fc156..65879b1d 100644--- a/test/collection_test.cpp+++ b/test/collection_test.cpp@@ -164,7 +164,7 @@ TEST_F(CollectionTest, PhraseSearch) {13: score: 12, (single word match)*/- std::vector<std::string> ids = {"8", "1", "16", "17", "13"};+ std::vector<std::string> ids = {"8", "1", "17", "16", "13"};for(size_t i = 0; i < results["hits"].size(); i++) {nlohmann::json result = results["hits"].at(i);@@ -184,7 +184,7 @@ TEST_F(CollectionTest, PhraseSearch) {ASSERT_EQ(5, results["hits"].size());ASSERT_EQ(5, results["found"].get<uint32_t>());- ids = {"8", "1", "17", "16", "13"};+ ids = {"8", "17", "1", "16", "13"};for(size_t i = 0; i < results["hits"].size(); i++) {nlohmann::json result = results["hits"].at(i);@@ -200,7 +200,7 @@ TEST_F(CollectionTest, PhraseSearch) {ASSERT_EQ(3, results["request_params"]["per_page"].get<size_t>());- ids = {"8", "1", "16"};+ ids = {"8", "1", "17"};for(size_t i = 0; i < 3; i++) {nlohmann::json result = results["hits"].at(i);@@ -1958,7 +1958,7 @@ TEST_F(CollectionTest, SearchLargeTextField) {ASSERT_EQ(1, results["hits"].size());- ASSERT_STREQ("non arcu id lectus <mark>accumsan</mark> venenatis at at justo.",+ ASSERT_STREQ("non arcu id lectus <mark>accumsan</mark> venenatis at at justo",results["hits"][0]["highlights"][0]["snippet"].get<std::string>().c_str());collectionManager.drop_collection("coll_large_text");@@ -2141,7 +2141,7 @@ TEST_F(CollectionTest, SearchHighlightWithNewLine) {token_ordering::FREQUENCY, true, 10, spp::sparse_hash_set<std::string>(),spp::sparse_hash_set<std::string>(), 10, "", 30, 4, "", 40, {}, {}, {}, 0).get();- ASSERT_STREQ("Blah, blah <mark>Stark</mark> Industries",+ ASSERT_STREQ("Blah, blah\n<mark>Stark</mark> Industries",res["hits"][0]["highlights"][0]["snippet"].get<std::string>().c_str());ASSERT_STREQ("Stark", res["hits"][0]["highlights"][0]["matched_tokens"][0].get<std::string>().c_str());@@ -3184,7 +3184,7 @@ TEST_F(CollectionTest, SearchingForRecordsWithSpecialChars) {std::vector<std::vector<std::string>> records = {{"Amazon Home", "https://amazon.com/"},- {"Google Home", "https://google.com/"},+ {"Google Home", "https://google.com///"},{"Github Issue", "https://github.com/typesense/typesense/issues/241"},{"Amazon Search", "https://www.amazon.com/s?k=phone&ref=nb_sb_noss_2"},};@@ -3206,12 +3206,17 @@ TEST_F(CollectionTest, SearchingForRecordsWithSpecialChars) {ASSERT_EQ(1, results["found"].get<size_t>());ASSERT_STREQ("1", results["hits"][0]["document"]["id"].get<std::string>().c_str());+ ASSERT_EQ(2, results["hits"][0]["highlights"].size());+ ASSERT_EQ("<mark>Google</mark> Home", results["hits"][0]["highlights"][0]["snippet"].get<std::string>());+ ASSERT_EQ("https://<mark>google</mark>.com///", results["hits"][0]["highlights"][1]["snippet"].get<std::string>());+results = coll1->search("amazon.com",{"title", "url"}, "", {}, {}, 2, 10, 1, FREQUENCY).get();- ASSERT_EQ(2, results["found"].get<size_t>());+ ASSERT_EQ(3, results["found"].get<size_t>());ASSERT_STREQ("3", results["hits"][0]["document"]["id"].get<std::string>().c_str());ASSERT_STREQ("0", results["hits"][1]["document"]["id"].get<std::string>().c_str());+ ASSERT_STREQ("1", results["hits"][2]["document"]["id"].get<std::string>().c_str());results = coll1->search("typesense",{"title", "url"}, "", {}, {}, 2, 10, 1, FREQUENCY).get();@@ -3225,5 +3230,9 @@ TEST_F(CollectionTest, SearchingForRecordsWithSpecialChars) {ASSERT_EQ(1, results["found"].get<size_t>());ASSERT_STREQ("3", results["hits"][0]["document"]["id"].get<std::string>().c_str());+ ASSERT_EQ(1, results["hits"][0]["highlights"].size());+ ASSERT_EQ("https://www.amazon.com/s?k=phone&ref=<mark>nb</mark>_<mark>sb</mark>_<mark>noss</mark>_<mark>2</mark>",+ results["hits"][0]["highlights"][0]["snippet"].get<std::string>());+collectionManager.drop_collection("coll1");-}\ No newline at end of file+}diff --git a/test/tokenizer_test.cpp b/test/tokenizer_test.cppindex 3a4b1264..58123d4c 100644--- a/test/tokenizer_test.cpp+++ b/test/tokenizer_test.cpp@@ -4,25 +4,23 @@TEST(TokenizerTest, ShouldTokenizeNormalizeDifferentStrings) {const std::string withnewline = "Michael Jordan:\nWelcome, everybody. Welcome! ";std::vector<std::string> tokens;- Tokenizer(withnewline, true, true, false).tokenize(tokens);- ASSERT_EQ(6, tokens.size());+ Tokenizer(withnewline, false, true, false).tokenize(tokens);+ ASSERT_EQ(5, tokens.size());ASSERT_STREQ("michael", tokens[0].c_str());ASSERT_STREQ("jordan", tokens[1].c_str());ASSERT_STREQ("welcome", tokens[2].c_str());ASSERT_STREQ("everybody", tokens[3].c_str());ASSERT_STREQ("welcome", tokens[4].c_str());- ASSERT_STREQ("", tokens[5].c_str());const std::string withspaces = " Michael Jordan ";tokens.clear();Tokenizer(withspaces, true, true, false).tokenize(tokens);- ASSERT_EQ(6, tokens.size());- ASSERT_STREQ("", tokens[0].c_str());+ ASSERT_EQ(5, tokens.size());+ ASSERT_STREQ(" ", tokens[0].c_str());ASSERT_STREQ("michael", tokens[1].c_str());- ASSERT_STREQ("", tokens[2].c_str());+ ASSERT_STREQ(" ", tokens[2].c_str());ASSERT_STREQ("jordan", tokens[3].c_str());- ASSERT_STREQ("", tokens[4].c_str());- ASSERT_STREQ("", tokens[5].c_str());+ ASSERT_STREQ(" ", tokens[4].c_str());tokens.clear();Tokenizer(withspaces, false, true, false).tokenize(tokens);@@ -30,38 +28,6 @@ TEST(TokenizerTest, ShouldTokenizeNormalizeDifferentStrings) {ASSERT_STREQ("michael", tokens[0].c_str());ASSERT_STREQ("jordan", tokens[1].c_str());- const std::string withspecialchars = "Special ½¥ and தமிழ் 你好吗 abcÅà123ß12 here.";- tokens.clear();- Tokenizer(withspecialchars, false, true, false).tokenize(tokens);- ASSERT_EQ(7, tokens.size());- ASSERT_STREQ("special", tokens[0].c_str());- ASSERT_STREQ("12yen", tokens[1].c_str());- ASSERT_STREQ("and", tokens[2].c_str());- ASSERT_STREQ("தமிழ்", tokens[3].c_str());- ASSERT_STREQ("你好吗", tokens[4].c_str());- ASSERT_STREQ("abcaa123ss12", tokens[5].c_str());- ASSERT_STREQ("here", tokens[6].c_str());-- // when normalization is disabled and keep empty is enabled- const std::string withoutnormalize = "Mise à, jour.";- tokens.clear();- Tokenizer(withoutnormalize, true, false, false).tokenize(tokens);- ASSERT_EQ(5, tokens.size());- ASSERT_STREQ("Mise", tokens[0].c_str());- ASSERT_STREQ("", tokens[1].c_str());- ASSERT_STREQ("à,", tokens[2].c_str());- ASSERT_STREQ("", tokens[3].c_str());- ASSERT_STREQ("jour.", tokens[4].c_str());-- // when normalization and keep empty are disabled- const std::string withoutnormalizeandkeepempty = "Mise à jour.";- tokens.clear();- Tokenizer(withoutnormalizeandkeepempty, false, false, false).tokenize(tokens);- ASSERT_EQ(3, tokens.size());- ASSERT_STREQ("Mise", tokens[0].c_str());- ASSERT_STREQ("à", tokens[1].c_str());- ASSERT_STREQ("jour.", tokens[2].c_str());-// single tokenconst std::string single_token = "foobar";tokens.clear();@@ -89,22 +55,82 @@ TEST(TokenizerTest, ShouldTokenizeNormalizeDifferentStrings) {const std::string multispace_tokens = "foo bar";tokens.clear();Tokenizer(multispace_tokens, true, false, false).tokenize(tokens);- ASSERT_EQ(6, tokens.size());+ ASSERT_EQ(3, tokens.size());ASSERT_STREQ("foo", tokens[0].c_str());- ASSERT_STREQ("", tokens[1].c_str());- ASSERT_STREQ("", tokens[2].c_str());- ASSERT_STREQ("", tokens[3].c_str());- ASSERT_STREQ("", tokens[4].c_str());- ASSERT_STREQ("bar", tokens[5].c_str());+ ASSERT_STREQ(" ", tokens[1].c_str());+ ASSERT_STREQ("bar", tokens[2].c_str());++ // special chars+ const std::string specialchar_tokens = "https://www.amazon.com/s?k=phone&ref=nb_sb_noss_2";;+ tokens.clear();+ Tokenizer(specialchar_tokens, true, false, false).tokenize(tokens);+ ASSERT_EQ(23, tokens.size());+ ASSERT_STREQ("https", tokens[0].c_str());+ ASSERT_STREQ("://", tokens[1].c_str());+ ASSERT_STREQ("www", tokens[2].c_str());+ ASSERT_STREQ(".", tokens[3].c_str());+ ASSERT_STREQ("noss", tokens[20].c_str());+ ASSERT_STREQ("_", tokens[21].c_str());+ ASSERT_STREQ("2", tokens[22].c_str());// nooptokens.clear();+ const std::string withspecialchars = "Special ½¥ and தமிழ் 你好吗 abcÅà123ß12 here.";Tokenizer(withspecialchars, false, true, true).tokenize(tokens);ASSERT_EQ(1, tokens.size());ASSERT_STREQ(withspecialchars.c_str(), tokens[0].c_str());}+TEST(TokenizerTest, ShouldTokenizeNormalizeUnicodeStrings) {+ std::vector<std::string> tokens;++ const std::string withspecialchars = "Special ½¥ and -தமிழ் 你2好吗 abcÅà123ß12 here.";+ tokens.clear();+ Tokenizer(withspecialchars, false, true, false).tokenize(tokens);+ ASSERT_EQ(7, tokens.size());+ ASSERT_STREQ("special", tokens[0].c_str());+ ASSERT_STREQ("12yen", tokens[1].c_str());+ ASSERT_STREQ("and", tokens[2].c_str());+ ASSERT_STREQ("தமிழ்", tokens[3].c_str());+ ASSERT_STREQ("你2好吗", tokens[4].c_str());+ ASSERT_STREQ("abcaa123ss12", tokens[5].c_str());+ ASSERT_STREQ("here", tokens[6].c_str());++ // when normalization is disabled and keep empty is enabled+ const std::string withoutnormalize = "Mise à, jour.";+ tokens.clear();+ Tokenizer(withoutnormalize, true, false, false).tokenize(tokens);+ ASSERT_EQ(6, tokens.size());+ ASSERT_STREQ("Mise", tokens[0].c_str());+ ASSERT_STREQ(" ", tokens[1].c_str());+ ASSERT_STREQ("à", tokens[2].c_str());+ ASSERT_STREQ(", ", tokens[3].c_str());+ ASSERT_STREQ("jour", tokens[4].c_str());+ ASSERT_STREQ(".", tokens[5].c_str());++ // when normalization and keep empty are disabled+ const std::string withoutnormalizeandkeepempty = "Mise à jour.";+ tokens.clear();+ Tokenizer(withoutnormalizeandkeepempty, false, false, false).tokenize(tokens);+ ASSERT_EQ(3, tokens.size());+ ASSERT_STREQ("Mise", tokens[0].c_str());+ ASSERT_STREQ("à", tokens[1].c_str());+ ASSERT_STREQ("jour", tokens[2].c_str());++ // single accented word tokenization+ std::string singleword = "à";+ tokens.clear();+ Tokenizer(singleword, false, true, false).tokenize(tokens);+ ASSERT_EQ(1, tokens.size());+ ASSERT_STREQ("a", tokens[0].c_str());++ tokens.clear();+ Tokenizer(singleword, true, true, false).tokenize(tokens);+ ASSERT_EQ(1, tokens.size());+ ASSERT_STREQ("a", tokens[0].c_str());+}+TEST(TokenizerTest, ShouldTokenizeIteratively) {const std::string withnewline = "Michael Jordan:\n\nWelcome, everybody. Welcome!";std::vector<std::string> tokens;@@ -117,20 +143,34 @@ TEST(TokenizerTest, ShouldTokenizeIteratively) {tokens.push_back(token);}- ASSERT_EQ(6, tokens.size());+ ASSERT_EQ(10, tokens.size());ASSERT_STREQ("michael", tokens[0].c_str());- ASSERT_STREQ("jordan", tokens[1].c_str());- ASSERT_STREQ("", tokens[2].c_str());- ASSERT_STREQ("welcome", tokens[3].c_str());- ASSERT_STREQ("everybody", tokens[4].c_str());- ASSERT_STREQ("welcome", tokens[5].c_str());+ ASSERT_STREQ(" ", tokens[1].c_str());+ ASSERT_STREQ("jordan", tokens[2].c_str());+ ASSERT_STREQ(":\n\n", tokens[3].c_str());+ ASSERT_STREQ("welcome", tokens[4].c_str());+ ASSERT_STREQ(", ", tokens[5].c_str());+ ASSERT_STREQ("everybody", tokens[6].c_str());+ ASSERT_STREQ(". ", tokens[7].c_str());+ ASSERT_STREQ("welcome", tokens[8].c_str());+ ASSERT_STREQ("!", tokens[9].c_str());++ // check for index when separators are not kept+ Tokenizer tokenizer2(withnewline, false, true, false);+ size_t expected_token_index = 0;+ std::vector<std::string> expected_tokens = {"michael", "jordan", "welcome", "everybody", "welcome"};+ while(tokenizer2.next(token, token_index)) {+ ASSERT_EQ(expected_token_index, token_index);+ ASSERT_EQ(expected_tokens[expected_token_index], token);+ expected_token_index++;+ }// verbatim (no_op=true)tokens.clear();- Tokenizer tokenizer2(withnewline, true, false, true);+ Tokenizer tokenizer3(withnewline, true, false, true);- while(tokenizer2.next(token, token_index)) {+ while(tokenizer3.next(token, token_index)) {tokens.push_back(token);}