Skip to content

Open Source alternative to Algolia + Pinecone and an Easier-to-Use alternative to ElasticSearch ⚡ 🔍 ✨ Fast, typo tolerant, in-memory fuzzy Search Engine for building delightful search experiences

Refactor highlighting + tokenizer to simplify logic.

Kishore Nallanb3b47f565150
diff --git a/include/collection.h b/include/collection.h
index abe32c55..211f9eab 100644
--- a/include/collection.h
+++ b/include/collection.h
@@ -235,13 +235,13 @@ struct synonym_t {
}
std::vector<std::string> tokens;
- Tokenizer(synonym, false).tokenize(tokens);
+ Tokenizer(synonym, true).tokenize(tokens);
syn.synonyms.push_back(tokens);
}
if(synonym_json.count("root") != 0) {
std::vector<std::string> tokens;
- Tokenizer(synonym_json["root"], false).tokenize(tokens);
+ Tokenizer(synonym_json["root"], true).tokenize(tokens);
syn.root = tokens;
}
diff --git a/include/tokenizer.h b/include/tokenizer.h
index c5681e88..9d75e0e3 100644
--- a/include/tokenizer.h
+++ b/include/tokenizer.h
@@ -11,7 +11,6 @@ class Tokenizer {
private:
std::string_view text;
size_t i;
- const bool keep_separators;
const bool normalize;
const bool no_op;
@@ -22,29 +21,35 @@ private:
static const size_t SEPARATE = 1;
static const size_t SKIP = 2;
- size_t prev_stream_mode;
+ uint8_t index_symbols[256] = {};
- std::stringstream out;
+ std::string out;
std::string locale;
icu::BreakIterator* bi = nullptr;
icu::UnicodeString unicode_text;
int32_t position = 0;
int32_t prev_position = -1;
+ int32_t utf8_start_index = 0;
char* normalized_text = nullptr;
inline size_t get_stream_mode(char c) {
- return std::isalnum(c) ? INDEX : (
+ return (std::isalnum(c) || index_symbols[uint8_t(c)] == 1) ? INDEX : (
(c == ' ' || c == '\n') ? SEPARATE : SKIP
);
}
+ static inline bool is_ascii_char(char c) {
+ return (c & ~0x7f) == 0;
+ }
+
public:
explicit Tokenizer(const std::string& input,
- bool keep_separators=true, bool normalize=true, bool no_op=false,
- const std::string& locale = ""):
- i(0), keep_separators(keep_separators), normalize(normalize),
+ bool normalize=true, bool no_op=false,
+ const std::string& locale = "",
+ const std::vector<char>& symbols_to_index = {}):
+ i(0), normalize(normalize),
no_op(no_op), locale(locale) {
if(locale == "ja") {
@@ -56,13 +61,6 @@ public:
cd = iconv_open("ASCII//TRANSLIT", "UTF-8");
- if(!text.empty() && (std::isalnum(text[0]) || (text[i] & ~0x7f) != 0)) {
- // alphanum or non-ascii
- prev_stream_mode = INDEX;
- } else {
- prev_stream_mode = SEPARATE;
- }
-
if(!locale.empty() && locale != "en") {
UErrorCode status = U_ZERO_ERROR;
const icu::Locale& icu_locale = icu::Locale(locale.c_str());
@@ -74,6 +72,10 @@ public:
position = bi->first();
prev_position = -1;
}
+
+ for(char c: symbols_to_index) {
+ index_symbols[uint8_t(c)] = 1;
+ }
}
~Tokenizer() {
@@ -82,6 +84,8 @@ public:
delete bi;
}
+ bool next(std::string& token, size_t& token_index, size_t& start_index, size_t& end_index);
+
bool next(std::string& token, size_t& token_index);
void tokenize(std::vector<std::string>& tokens);
diff --git a/src/collection.cpp b/src/collection.cpp
index 5b83ab6a..17b29c39 100644
--- a/src/collection.cpp
+++ b/src/collection.cpp
@@ -14,6 +14,7 @@
#include <collection_manager.h>
#include <h3api.h>
#include <regex>
+#include <list>
#include "topster.h"
#include "logger.h"
@@ -1237,22 +1238,14 @@ void Collection::parse_search_query(const std::string &query, std::vector<std::s
q_include_tokens = {query};
} else {
std::vector<std::string> tokens;
- Tokenizer(query, true, true, false, locale).tokenize(tokens);
- bool exclude_operator_prior = false;
+ Tokenizer(query, true, false, locale, {'-'}).tokenize(tokens);
- for(const auto& token: tokens) {
- if(token.empty()) {
- continue;
- }
+ bool exclude_operator_prior = false;
- if(token == "-" || token == " -") {
+ for(auto& token: tokens) {
+ if(token[0] == '-') {
exclude_operator_prior = true;
- }
-
- bool is_ascii = (token[0] & ~0x7f) == 0;
-
- if(is_ascii && !std::isalnum(token[0])) {
- continue;
+ token = token.substr(1);
}
if(exclude_operator_prior) {
@@ -1463,97 +1456,119 @@ void Collection::highlight_result(const field &search_field,
const Match& match = match_index.match;
const std::string& text = (search_field.type == field_types::STRING) ? document[search_field.name] : document[search_field.name][match_index.index];
- Tokenizer tokenizer(text, true, false, false, search_field.locale);
+ Tokenizer tokenizer(text, true, false, search_field.locale);
+
+ // need an ordered map here to ensure that it is ordered by the key (start offset)
+ std::map<size_t, size_t> token_offsets;
- std::string raw_token;
- size_t raw_token_index = 0;
- int indexed_token_index = -1;
size_t match_offset_index = 0;
+ std::string raw_token;
+ spp::sparse_hash_set<std::string> token_hits; // used to identify repeating tokens
+ size_t raw_token_index = 0, tok_start = 0, tok_end = 0;
- std::set<size_t> token_indices;
- spp::sparse_hash_set<std::string> token_hits;
- std::vector<std::string> raw_tokens;
- std::unordered_map<size_t, size_t> indexed_to_raw;
-
- while(tokenizer.next(raw_token, raw_token_index)) {
- if(!raw_token.empty() && (std::isalnum(raw_token[0]) || (raw_token[0] & ~0x7f) != 0)) {
- // check for actual token (first char is NOT alphanum or ascii)
- indexed_token_index++;
- indexed_to_raw[indexed_token_index] = raw_token_index;
- /*LOG(INFO) << "raw_token: " << raw_token << ", indexed_token_index: " << indexed_token_index
- << ", raw_token_index: " << raw_token_index;*/
- }
+ // based on `highlight_affix_num_tokens`
+ size_t snippet_start_offset = 0, snippet_end_offset = (text.empty() ? 0 : text.size() - 1);
+
+ // window used to locate the starting offset for snippet on the text
+ std::list<size_t> snippet_start_window;
- if (match_offset_index < match.offsets.size() &&
- match.offsets[match_offset_index].offset == indexed_token_index) {
- std::string indexed_token;
- Tokenizer(raw_token, true, true).tokenize(indexed_token);
+ highlight.matched_tokens.emplace_back();
+ std::vector<std::string>& matched_tokens = highlight.matched_tokens.back();
- if(token_indices.count(indexed_token_index) == 0) {
- // repetition could occur, for e.g. in the case of synonym constructed queries
- token_indices.insert(indexed_token_index);
- token_hits.insert(indexed_token);
+ while(tokenizer.next(raw_token, raw_token_index, tok_start, tok_end)) {
+ if(token_offsets.empty()) {
+ if(snippet_start_window.size() == highlight_affix_num_tokens + 1) {
+ snippet_start_window.pop_front();
}
- match_offset_index++;
+ snippet_start_window.push_back(tok_start);
}
- raw_tokens.push_back(raw_token);
- }
+ if (token_hits.count(raw_token) != 0 ||
+ (match_offset_index < match.offsets.size() &&
+ match.offsets[match_offset_index].offset == raw_token_index)) {
- size_t num_indexed_tokens = indexed_token_index + 1;
- auto minmax = std::minmax_element(token_indices.begin(), token_indices.end());
+ token_offsets.emplace(tok_start, tok_end);
+ token_hits.insert(raw_token);
- size_t prefix_length = highlight_affix_num_tokens;
- size_t suffix_length = highlight_affix_num_tokens;
+ // to skip over duplicate tokens in the query
+ do {
+ match_offset_index++;
+ } while(match_offset_index < match.offsets.size() &&
+ match.offsets[match_offset_index - 1].offset == match.offsets[match_offset_index].offset);
- if(num_indexed_tokens == 0) {
- continue;
+ if(token_offsets.size() == 1) {
+ snippet_start_offset = snippet_start_window.front();
+ }
+ }
+
+ if(raw_token_index == match.offsets.back().offset + highlight_affix_num_tokens) {
+ // register end of highlight snippet
+ snippet_end_offset = tok_end;
+ }
+
+ if(raw_token_index > match.offsets.back().offset + highlight_affix_num_tokens &&
+ raw_token_index == snippet_threshold - 1) {
+ // since we have already crossed snippeting threshold, we can break now
+ break;
+ }
}
- // For longer strings, pick surrounding raw_tokens within `prefix_length` of min_index and max_index for snippet
- const size_t start_index = (num_indexed_tokens <= snippet_threshold) ? 0 :
- indexed_to_raw[std::max(0, (int)(*(minmax.first) - prefix_length))];
+ if(token_offsets.empty()) {
+ continue;
+ }
- const size_t end_index = (num_indexed_tokens <= snippet_threshold) ? raw_tokens.size() - 1 :
- indexed_to_raw[std::min((int)num_indexed_tokens - 1, (int)(*(minmax.second) + suffix_length))];
+ if(raw_token_index + 1 < snippet_threshold) {
+ // fully highlight field whose token size is less than given snippeth threshold
+ snippet_start_offset = 0;
+ snippet_end_offset = text.size() - 1;
+ }
- std::stringstream snippet_stream;
+ // with `token_index_offsets` we have a list of ranges to target for highlighting
- highlight.matched_tokens.emplace_back();
- std::vector<std::string>& matched_tokens = highlight.matched_tokens.back();
- size_t snippet_index = start_index;
+ auto offset_it = token_offsets.begin();
+ std::stringstream highlighted_text;
- while(snippet_index <= end_index) {
- std::string normalized_token;
- Tokenizer(raw_tokens[snippet_index], true, true).tokenize(normalized_token);
+ for(size_t i = snippet_start_offset; i <= snippet_end_offset; i++) {
+ if(offset_it != token_offsets.end()) {
+ if (i == offset_it->first) {
+ highlighted_text << highlight_start_tag;
+ matched_tokens.push_back(text.substr(i, (offset_it->second - i) + 1));
+ }
- if(token_hits.count(normalized_token) != 0) {
- snippet_stream << highlight_start_tag << raw_tokens[snippet_index] << highlight_end_tag;
- matched_tokens.push_back(raw_tokens[snippet_index]);
- } else {
- snippet_stream << raw_tokens[snippet_index];
+ if (i == offset_it->second) {
+ highlighted_text << text[i];
+ highlighted_text << highlight_end_tag;
+ offset_it++;
+ continue;
+ }
}
- snippet_index++;
+ highlighted_text << text[i];
}
- highlight.snippets.push_back(snippet_stream.str());
+ highlight.snippets.push_back(highlighted_text.str());
if(search_field.type == field_types::STRING_ARRAY) {
highlight.indices.push_back(match_index.index);
}
if(highlighted_fully) {
std::stringstream value_stream;
- for(size_t value_index = 0; value_index < raw_tokens.size(); value_index++) {
- std::string normalized_token;
- Tokenizer(raw_tokens[value_index], true, true).tokenize(normalized_token);
-
- if(token_hits.count(normalized_token) != 0) {
- value_stream << highlight_start_tag << raw_tokens[value_index] << highlight_end_tag;
- } else {
- value_stream << raw_tokens[value_index];
+ offset_it = token_offsets.begin();
+
+ for(size_t i = 0; i < text.size(); i++) {
+ if(offset_it != token_offsets.end()) {
+ if (i == offset_it->first) {
+ value_stream << highlight_start_tag;
+ } else if (i == offset_it->second) {
+ value_stream << text[i];
+ value_stream << highlight_end_tag;
+ offset_it++;
+ continue;
+ }
}
+
+ value_stream << text[i];
}
highlight.values.push_back(value_stream.str());
diff --git a/src/index.cpp b/src/index.cpp
index 10f418a5..bd80ee3e 100644
--- a/src/index.cpp
+++ b/src/index.cpp
@@ -533,7 +533,7 @@ void Index::index_string_field(const std::string & text, const int64_t score, ar
uint32_t seq_id, bool is_facet, const field & a_field) {
std::unordered_map<std::string, std::vector<uint32_t>> token_to_offsets;
- Tokenizer tokenizer(text, false, true, !a_field.is_string(), a_field.locale);
+ Tokenizer tokenizer(text, true, !a_field.is_string(), a_field.locale);
std::string token;
size_t token_index = 0;
@@ -580,7 +580,7 @@ void Index::index_string_array_field(const std::vector<std::string> & strings, c
const std::string& str = strings[array_index];
std::set<std::string> token_set; // required to deal with repeating tokens
- Tokenizer tokenizer(str, false, true, !a_field.is_string(), a_field.locale);
+ Tokenizer tokenizer(str, true, !a_field.is_string(), a_field.locale);
std::string token;
size_t token_index = 0;
@@ -707,7 +707,7 @@ void Index::do_facets(std::vector<facet> & facets, facet_query_t & facet_query,
art_tree *t = search_index.at(facet_field.faceted_name());
std::vector<std::string> query_tokens;
- Tokenizer(facet_query.query, false, true, !facet_field.is_string()).tokenize(query_tokens);
+ Tokenizer(facet_query.query, true, !facet_field.is_string()).tokenize(query_tokens);
for (size_t qtoken_index = 0; qtoken_index < query_tokens.size(); qtoken_index++) {
auto &q = query_tokens[qtoken_index];
@@ -1204,7 +1204,7 @@ uint32_t Index::do_filtering(uint32_t** filter_ids_out, const std::vector<filter
// there could be multiple tokens in a filter value, which we have to treat as ANDs
// e.g. country: South Africa
- Tokenizer tokenizer(filter_value, false, true, false, f.locale);
+ Tokenizer tokenizer(filter_value, true, false, f.locale);
std::string str_token;
size_t token_index = 0;
@@ -2447,11 +2447,11 @@ void Index::tokenize_string_field(const nlohmann::json& document, const field& s
const std::string& field_name = search_field.name;
if(search_field.type == field_types::STRING) {
- Tokenizer(document[field_name], false, true, false, locale).tokenize(tokens);
+ Tokenizer(document[field_name], true, false, locale).tokenize(tokens);
} else if(search_field.type == field_types::STRING_ARRAY) {
const std::vector<std::string>& values = document[field_name].get<std::vector<std::string>>();
for(const std::string & value: values) {
- Tokenizer(value, false, true, false, locale).tokenize(tokens);
+ Tokenizer(value, true, false, locale).tokenize(tokens);
}
}
}
diff --git a/src/tokenizer.cpp b/src/tokenizer.cpp
index 9bb4b709..127a3708 100644
--- a/src/tokenizer.cpp
+++ b/src/tokenizer.cpp
@@ -1,7 +1,7 @@
#include <sstream>
#include "tokenizer.h"
-bool Tokenizer::next(std::string &token, size_t& token_index) {
+bool Tokenizer::next(std::string &token, size_t& token_index, size_t& start_index, size_t& end_index) {
if(no_op) {
if(i == text.size()) {
return false;
@@ -9,6 +9,8 @@ bool Tokenizer::next(std::string &token, size_t& token_index) {
token = text;
i = text.size();
+ start_index = 0;
+ end_index = text.size() - 1;
return true;
}
@@ -22,8 +24,11 @@ bool Tokenizer::next(std::string &token, size_t& token_index) {
size_t length = position - prev_position;
token = unicode_text.tempSubString(prev_position, length).toUTF8String(word);
+ //LOG(INFO) << "token: " << token;
+
if(!token.empty()) {
- if (!keep_separators && !std::isalnum(token[0]) && (token[i] & ~0x7f) == 0) {
+ if (!std::isalnum(token[0]) && is_ascii_char(token[i])) {
+ // ignore ascii symbols
found_token = false;
} else if(locale == "ko" && token == "·") {
found_token = false;
@@ -31,6 +36,10 @@ bool Tokenizer::next(std::string &token, size_t& token_index) {
found_token = true;
token_index = token_counter++;
}
+
+ start_index = utf8_start_index;
+ end_index = utf8_start_index + token.size() - 1;
+ utf8_start_index = end_index + 1;
}
}
@@ -46,57 +55,40 @@ bool Tokenizer::next(std::string &token, size_t& token_index) {
}
while(i < text.size()) {
- bool is_ascii = (text[i] & ~0x7f) == 0;
- if(is_ascii) {
+ if(is_ascii_char(text[i])) {
size_t this_stream_mode = get_stream_mode(text[i]);
- if(this_stream_mode == SKIP && !keep_separators) {
+ if(this_stream_mode == SKIP) {
i++;
continue;
}
- if(this_stream_mode != prev_stream_mode) {
- // We tokenize when `prev_stream_mode` changes
- token = out.str();
-
- out.str(std::string());
- if(normalize) {
- out << char(std::tolower(text[i]));
- } else {
- out << text[i];
- }
- i++;
-
- if(prev_stream_mode == SEPARATE && !keep_separators) {
- prev_stream_mode = this_stream_mode;
+ if(this_stream_mode == SEPARATE) {
+ if(out.empty()) {
+ i++;
continue;
}
+ token = out;
+ out.clear();
+
token_index = token_counter++;
- prev_stream_mode = this_stream_mode;
+ end_index = i - 1;
+ i++;
return true;
} else {
- if(normalize) {
- out << char(std::tolower(text[i]));
- } else {
- out << text[i];
+ if(out.empty()) {
+ start_index = i;
}
+ out += normalize ? char(std::tolower(text[i])) : text[i];
i++;
continue;
}
}
- if(prev_stream_mode == SEPARATE) { // to detect first non-ascii character
- // we will tokenize now and treat the following non-ascii chars as a different token
- prev_stream_mode = INDEX;
- token = out.str();
- out.str(std::string());
-
- if(keep_separators) {
- token_index = token_counter++;
- return true;
- }
+ if(out.empty()) {
+ start_index = i;
}
char inbuf[5];
@@ -111,7 +103,7 @@ bool Tokenizer::next(std::string &token, size_t& token_index) {
size_t insize = (p - &inbuf[0]);
if(!normalize) {
- out << inbuf;
+ out += inbuf;
continue;
}
@@ -127,38 +119,35 @@ bool Tokenizer::next(std::string &token, size_t& token_index) {
if(errno == EILSEQ) {
// symbol cannot be represented as ASCII, so write the original symbol
- out << inbuf;
+ out += inbuf;
} else {
for(size_t out_index=0; out_index<5; out_index++) {
if(!normalize) {
- out << outbuf[out_index];
+ out += outbuf[out_index];
continue;
}
- bool unicode_is_ascii = ((outbuf[out_index] & ~0x7f) == 0);
+ bool unicode_is_ascii = is_ascii_char(outbuf[out_index]);
bool keep_char = !unicode_is_ascii || std::isalnum(outbuf[out_index]);
if(keep_char) {
if(unicode_is_ascii && std::isalnum(outbuf[out_index])) {
outbuf[out_index] = char(std::tolower(outbuf[out_index]));
}
- out << outbuf[out_index];
+ out += outbuf[out_index];
}
}
}
}
- token = out.str();
- out.str(std::string());
+ token = out;
+ out.clear();
+ end_index = i - 1;
if(token.empty()) {
return false;
}
- if(!std::isalnum(token[0]) && !keep_separators) {
- return false;
- }
-
token_index = token_counter++;
return true;
}
@@ -176,3 +165,8 @@ void Tokenizer::tokenize(std::string& token) {
size_t token_index;
next(token, token_index);
}
+
+bool Tokenizer::next(std::string &token, size_t &token_index) {
+ size_t start_index, end_index;
+ return next(token, token_index, start_index, end_index);
+}
diff --git a/test/art_test.cpp b/test/art_test.cpp
index 4af38758..61e7b057 100644
--- a/test/art_test.cpp
+++ b/test/art_test.cpp
@@ -51,7 +51,7 @@ TEST(ArtTest, test_art_insert) {
ASSERT_TRUE(art_size(&t) == line);
line++;
- delete document.offsets;
+ delete [] document.offsets;
}
res = art_tree_destroy(&t);
diff --git a/test/collection_locale_test.cpp b/test/collection_locale_test.cpp
index 6bde3362..a180039e 100644
--- a/test/collection_locale_test.cpp
+++ b/test/collection_locale_test.cpp
@@ -98,8 +98,8 @@ TEST_F(CollectionLocaleTest, SearchAgainstKoreanText) {
}
std::vector<std::vector<std::string>> records = {
- {"경승지·산악·협곡", "Dustin Kensrue"},
- {"안녕은하철도999극장판", "Gord Downie"},
+ {"경승지·산악·협곡", "Dustin Kensrue"},
+ {"안녕은하철도999극장판", "Gord Downie"},
};
for(size_t i=0; i<records.size(); i++) {
@@ -129,3 +129,47 @@ TEST_F(CollectionLocaleTest, SearchAgainstKoreanText) {
ASSERT_EQ("0", results["hits"][0]["document"]["id"].get<std::string>());
ASSERT_EQ("경승지·<mark>산악</mark>·협곡", results["hits"][0]["highlights"][0]["snippet"].get<std::string>());
}
+
+TEST_F(CollectionLocaleTest, SearchAgainstKoreanTextContainingEnglishChars) {
+ Collection *coll1;
+
+ std::vector<field> fields = {field("title", field_types::STRING, false, false, true, DEFAULT_GEO_RESOLUTION, "th"),
+ field("artist", field_types::STRING, false),
+ field("points", field_types::INT32, false),};
+
+ coll1 = collectionManager.get_collection("coll1").get();
+ if(coll1 == nullptr) {
+ coll1 = collectionManager.create_collection("coll1", 1, fields, "points").get();
+ }
+
+ std::vector<std::vector<std::string>> records = {
+ {"개혁 등의 영향으로 11%나 위축됐다", "Dustin Kensrue"},
+ };
+
+ for(size_t i=0; i<records.size(); i++) {
+ nlohmann::json doc;
+
+ doc["id"] = std::to_string(i);
+ doc["title"] = records[i][0];
+ doc["artist"] = records[i][1];
+ doc["points"] = i;
+
+ ASSERT_TRUE(coll1->add(doc.dump()).ok());
+ }
+
+ auto results = coll1->search("위축됐다",
+ {"title"}, "", {}, {}, 0, 10, 1, FREQUENCY).get();
+
+ ASSERT_EQ(1, results["found"].get<size_t>());
+ ASSERT_EQ(1, results["hits"].size());
+ ASSERT_EQ("0", results["hits"][0]["document"]["id"].get<std::string>());
+ ASSERT_EQ("개혁 등의 영향으로 11%나 <mark>위축됐다</mark>", results["hits"][0]["highlights"][0]["snippet"].get<std::string>());
+
+ results = coll1->search("11%",
+ {"title"}, "", {}, {}, 0, 10, 1, FREQUENCY).get();
+
+ ASSERT_EQ(1, results["found"].get<size_t>());
+ ASSERT_EQ(1, results["hits"].size());
+ ASSERT_EQ("0", results["hits"][0]["document"]["id"].get<std::string>());
+ ASSERT_EQ("개혁 등의 영향으로 <mark>11</mark>%나 위축됐다", results["hits"][0]["highlights"][0]["snippet"].get<std::string>());
+}
diff --git a/test/collection_synonyms_test.cpp b/test/collection_synonyms_test.cpp
index 4b40cf6a..39f4a5ff 100644
--- a/test/collection_synonyms_test.cpp
+++ b/test/collection_synonyms_test.cpp
@@ -359,8 +359,8 @@ TEST_F(CollectionSynonymsTest, MultiWaySynonym) {
ASSERT_EQ(2, res["hits"].size());
ASSERT_EQ(2, res["found"].get<uint32_t>());
- ASSERT_STREQ("<mark>Samuel</mark> <mark>L</mark>. <mark>Jackson</mark>", res["hits"][0]["highlights"][0]["snippet"].get<std::string>().c_str());
- ASSERT_STREQ("<mark>Samuel</mark> <mark>L</mark>. <mark>Jackson</mark>", res["hits"][1]["highlights"][0]["snippet"].get<std::string>().c_str());
+ ASSERT_STREQ("<mark>Samuel</mark> <mark>L.</mark> <mark>Jackson</mark>", res["hits"][0]["highlights"][0]["snippet"].get<std::string>().c_str());
+ ASSERT_STREQ("<mark>Samuel</mark> <mark>L.</mark> <mark>Jackson</mark>", res["hits"][1]["highlights"][0]["snippet"].get<std::string>().c_str());
// for now we don't support synonyms on ANY prefix
diff --git a/test/collection_test.cpp b/test/collection_test.cpp
index ef4ed5f9..cbe2ea2a 100644
--- a/test/collection_test.cpp
+++ b/test/collection_test.cpp
@@ -2028,7 +2028,7 @@ TEST_F(CollectionTest, SearchLargeTextField) {
ASSERT_EQ(1, results["hits"].size());
- ASSERT_STREQ("non arcu id lectus <mark>accumsan</mark> venenatis at at justo",
+ ASSERT_STREQ("non arcu id lectus <mark>accumsan</mark> venenatis at at justo.",
results["hits"][0]["highlights"][0]["snippet"].get<std::string>().c_str());
collectionManager.drop_collection("coll_large_text");
@@ -3149,7 +3149,7 @@ TEST_F(CollectionTest, MultiFieldHighlighting) {
results["hits"][0]["highlights"][0]["snippet"].get<std::string>());
ASSERT_EQ("description", results["hits"][0]["highlights"][1]["field"].get<std::string>());
- ASSERT_EQ("Easily replenish your cell phone with this wireless <mark>charger</mark>.",
+ ASSERT_EQ("Easily replenish your cell phone with this wireless <mark>charger.</mark>",
results["hits"][0]["highlights"][1]["snippet"].get<std::string>());
collectionManager.drop_collection("coll1");
@@ -3338,9 +3338,10 @@ TEST_F(CollectionTest, HighlightWithAccentedCharacters) {
}
std::vector<std::vector<std::string>> records = {
- {"Mise à jour Timy depuis PC"},
- {"Down There by the Train"},
+ {"Mise T.J. à jour Timy depuis PC"},
+ {"Down There by the T.r.a.i.n"},
{"State Trooper"},
+ {"The Google Nexus Q Is Baffling"},
};
for (size_t i = 0; i < records.size(); i++) {
@@ -3358,13 +3359,36 @@ TEST_F(CollectionTest, HighlightWithAccentedCharacters) {
ASSERT_EQ(1, results["found"].get<size_t>());
ASSERT_EQ(1, results["hits"].size());
- ASSERT_STREQ("Mise <mark>à</mark> <mark>jour</mark> Timy depuis PC",
+ ASSERT_STREQ("Mise T.J. <mark>à</mark> <mark>jour</mark> Timy depuis PC",
results["hits"][0]["highlights"][0]["snippet"].get<std::string>().c_str());
ASSERT_EQ(2, results["hits"][0]["highlights"][0]["matched_tokens"].size());
ASSERT_STREQ("à", results["hits"][0]["highlights"][0]["matched_tokens"][0].get<std::string>().c_str());
ASSERT_STREQ("jour", results["hits"][0]["highlights"][0]["matched_tokens"][1].get<std::string>().c_str());
+ results = coll1->search("by train", {"title"}, "", {}, {}, 0, 10, 1, FREQUENCY).get();
+
+ ASSERT_EQ(1, results["found"].get<size_t>());
+ ASSERT_EQ(1, results["hits"].size());
+
+ ASSERT_STREQ("Down There <mark>by</mark> the <mark>T.r.a.i.n</mark>",
+ results["hits"][0]["highlights"][0]["snippet"].get<std::string>().c_str());
+
+ results = coll1->search("state trooper", {"title"}, "", {}, {}, 0, 10, 1, FREQUENCY).get();
+
+ ASSERT_EQ(1, results["found"].get<size_t>());
+ ASSERT_EQ(1, results["hits"].size());
+
+ ASSERT_STREQ("<mark>State</mark> <mark>Trooper</mark>",
+ results["hits"][0]["highlights"][0]["snippet"].get<std::string>().c_str());
+
+ // test single character highlight
+
+ results = coll1->search("q", {"title"}, "", {}, {}, 0, 10, 1, FREQUENCY).get();
+ ASSERT_EQ(1, results["found"].get<size_t>());
+ ASSERT_STREQ("The Google Nexus <mark>Q</mark> Is Baffling",
+ results["hits"][0]["highlights"][0]["snippet"].get<std::string>().c_str());
+
collectionManager.drop_collection("coll1");
}
diff --git a/test/tokenizer_test.cpp b/test/tokenizer_test.cpp
index a7430cd2..681d12b1 100644
--- a/test/tokenizer_test.cpp
+++ b/test/tokenizer_test.cpp
@@ -3,9 +3,19 @@
#include "logger.h"
TEST(TokenizerTest, ShouldTokenizeNormalizeDifferentStrings) {
- const std::string withnewline = "Michael Jordan:\nWelcome, everybody. Welcome! ";
+ const std::string withaccent = "Mise T.J. à jour Timy depuis PC";
std::vector<std::string> tokens;
- Tokenizer(withnewline, false, true, false).tokenize(tokens);
+ Tokenizer(withaccent, true, false).tokenize(tokens);
+
+ std::vector<std::string> withaccent_tokens = {"mise", "tj", "a", "jour", "timy", "depuis", "pc"};
+ ASSERT_EQ(withaccent_tokens.size(), tokens.size());
+ for(size_t i = 0; i < withaccent_tokens.size(); i++) {
+ ASSERT_EQ(withaccent_tokens[i], tokens[i]);
+ }
+
+ const std::string withnewline = "Michael Jordan:\nWelcome, everybody. Welcome! ";
+ tokens.clear();
+ Tokenizer(withnewline, true, false).tokenize(tokens);
ASSERT_EQ(5, tokens.size());
ASSERT_STREQ("michael", tokens[0].c_str());
ASSERT_STREQ("jordan", tokens[1].c_str());
@@ -15,16 +25,7 @@ TEST(TokenizerTest, ShouldTokenizeNormalizeDifferentStrings) {
const std::string withspaces = " Michael Jordan ";
tokens.clear();
- Tokenizer(withspaces, true, true, false).tokenize(tokens);
- ASSERT_EQ(5, tokens.size());
- ASSERT_STREQ(" ", tokens[0].c_str());
- ASSERT_STREQ("michael", tokens[1].c_str());
- ASSERT_STREQ(" ", tokens[2].c_str());
- ASSERT_STREQ("jordan", tokens[3].c_str());
- ASSERT_STREQ(" ", tokens[4].c_str());
-
- tokens.clear();
- Tokenizer(withspaces, false, true, false).tokenize(tokens);
+ Tokenizer(withspaces, true, false).tokenize(tokens);
ASSERT_EQ(2, tokens.size());
ASSERT_STREQ("michael", tokens[0].c_str());
ASSERT_STREQ("jordan", tokens[1].c_str());
@@ -32,49 +33,42 @@ TEST(TokenizerTest, ShouldTokenizeNormalizeDifferentStrings) {
// single token
const std::string single_token = "foobar";
tokens.clear();
- Tokenizer(single_token, false, false, false).tokenize(tokens);
+ Tokenizer(single_token, false, false).tokenize(tokens);
ASSERT_EQ(1, tokens.size());
ASSERT_STREQ("foobar", tokens[0].c_str());
// split tokens
const std::string split_tokens = "foo-bar-baz";
tokens.clear();
- Tokenizer(split_tokens, false, false, false).tokenize(tokens);
+ Tokenizer(split_tokens, false, false).tokenize(tokens);
ASSERT_EQ(1, tokens.size());
ASSERT_STREQ("foobarbaz", tokens[0].c_str());
tokens.clear();
- Tokenizer(split_tokens, false, true, false).tokenize(tokens);
+ Tokenizer(split_tokens, true, false).tokenize(tokens);
ASSERT_EQ(1, tokens.size());
ASSERT_STREQ("foobarbaz", tokens[0].c_str());
// multiple spaces
const std::string multispace_tokens = "foo bar";
tokens.clear();
- Tokenizer(multispace_tokens, true, false, false).tokenize(tokens);
- ASSERT_EQ(3, tokens.size());
+ Tokenizer(multispace_tokens, false, false).tokenize(tokens);
+ ASSERT_EQ(2, tokens.size());
ASSERT_STREQ("foo", tokens[0].c_str());
- ASSERT_STREQ(" ", tokens[1].c_str());
- ASSERT_STREQ("bar", tokens[2].c_str());
+ ASSERT_STREQ("bar", tokens[1].c_str());
// special chars
const std::string specialchar_tokens = "https://www.amazon.com/s?k=phone&ref=nb_sb_noss_2";
tokens.clear();
- Tokenizer(specialchar_tokens, true, false, false).tokenize(tokens);
- ASSERT_EQ(23, tokens.size());
- ASSERT_STREQ("https", tokens[0].c_str());
- ASSERT_STREQ("://", tokens[1].c_str());
- ASSERT_STREQ("www", tokens[2].c_str());
- ASSERT_STREQ(".", tokens[3].c_str());
- ASSERT_STREQ("noss", tokens[20].c_str());
- ASSERT_STREQ("_", tokens[21].c_str());
- ASSERT_STREQ("2", tokens[22].c_str());
+ Tokenizer(specialchar_tokens, false, false).tokenize(tokens);
+ ASSERT_EQ(1, tokens.size());
+ ASSERT_STREQ("httpswwwamazoncomskphonerefnbsbnoss2", tokens[0].c_str());
// noop
tokens.clear();
const std::string withspecialchars = "Special ½¥ and தமிழ் 你好吗 abcÅà123ß12 here.";
- Tokenizer(withspecialchars, false, true, true).tokenize(tokens);
+ Tokenizer(withspecialchars, true, true).tokenize(tokens);
ASSERT_EQ(1, tokens.size());
ASSERT_STREQ(withspecialchars.c_str(), tokens[0].c_str());
}
@@ -82,35 +76,23 @@ TEST(TokenizerTest, ShouldTokenizeNormalizeDifferentStrings) {
TEST(TokenizerTest, ShouldTokenizeNormalizeUnicodeStrings) {
std::vector<std::string> tokens;
- const std::string withspecialchars = "Special ½¥ and -தமிழ் 你2好吗 abcÅà123ß12 here.";
+ const std::string withspecialchars = "Special ½¥ and -thenதமிழ், 你2好吗 abcÅà123ß12 verläßlich here.";
tokens.clear();
- Tokenizer(withspecialchars, false, true, false).tokenize(tokens);
- ASSERT_EQ(7, tokens.size());
+ Tokenizer(withspecialchars, true, false).tokenize(tokens);
+ ASSERT_EQ(8, tokens.size());
ASSERT_STREQ("special", tokens[0].c_str());
ASSERT_STREQ("12yen", tokens[1].c_str());
ASSERT_STREQ("and", tokens[2].c_str());
- ASSERT_STREQ("தமிழ்", tokens[3].c_str());
+ ASSERT_STREQ("thenதமிழ்", tokens[3].c_str());
ASSERT_STREQ("你2好吗", tokens[4].c_str());
ASSERT_STREQ("abcaa123ss12", tokens[5].c_str());
- ASSERT_STREQ("here", tokens[6].c_str());
+ ASSERT_STREQ("verlasslich", tokens[6].c_str());
+ ASSERT_STREQ("here", tokens[7].c_str());
- // when normalization is disabled and keep empty is enabled
+ // when normalization is disabled
const std::string withoutnormalize = "Mise à, jour.";
tokens.clear();
- Tokenizer(withoutnormalize, true, false, false).tokenize(tokens);
- ASSERT_EQ(7, tokens.size());
- ASSERT_STREQ("Mise", tokens[0].c_str());
- ASSERT_STREQ(" ", tokens[1].c_str());
- ASSERT_STREQ("à", tokens[2].c_str());
- ASSERT_STREQ(",", tokens[3].c_str());
- ASSERT_STREQ(" ", tokens[4].c_str());
- ASSERT_STREQ("jour", tokens[5].c_str());
- ASSERT_STREQ(".", tokens[6].c_str());
-
- // when normalization and keep empty are disabled
- const std::string withoutnormalizeandkeepempty = "Mise à jour.";
- tokens.clear();
- Tokenizer(withoutnormalizeandkeepempty, false, false, false).tokenize(tokens);
+ Tokenizer(withoutnormalize, false, false).tokenize(tokens);
ASSERT_EQ(3, tokens.size());
ASSERT_STREQ("Mise", tokens[0].c_str());
ASSERT_STREQ("à", tokens[1].c_str());
@@ -119,12 +101,7 @@ TEST(TokenizerTest, ShouldTokenizeNormalizeUnicodeStrings) {
// single accented word tokenization
std::string singleword = "à";
tokens.clear();
- Tokenizer(singleword, false, true, false).tokenize(tokens);
- ASSERT_EQ(1, tokens.size());
- ASSERT_STREQ("a", tokens[0].c_str());
-
- tokens.clear();
- Tokenizer(singleword, true, true, false).tokenize(tokens);
+ Tokenizer(singleword, true, false).tokenize(tokens);
ASSERT_EQ(1, tokens.size());
ASSERT_STREQ("a", tokens[0].c_str());
}
@@ -132,7 +109,7 @@ TEST(TokenizerTest, ShouldTokenizeNormalizeUnicodeStrings) {
TEST(TokenizerTest, ShouldTokenizeIteratively) {
const std::string withnewline = "Michael Jordan:\n\nWelcome, everybody. Welcome!";
std::vector<std::string> tokens;
- Tokenizer tokenizer1(withnewline, true, true, false);
+ Tokenizer tokenizer1(withnewline, true, false);
std::string token;
size_t token_index;
@@ -141,23 +118,15 @@ TEST(TokenizerTest, ShouldTokenizeIteratively) {
tokens.push_back(token);
}
- ASSERT_EQ(13, tokens.size());
+ ASSERT_EQ(5, tokens.size());
ASSERT_STREQ("michael", tokens[0].c_str());
- ASSERT_STREQ(" ", tokens[1].c_str());
- ASSERT_STREQ("jordan", tokens[2].c_str());
- ASSERT_STREQ(":", tokens[3].c_str());
- ASSERT_STREQ("\n\n", tokens[4].c_str());
- ASSERT_STREQ("welcome", tokens[5].c_str());
- ASSERT_STREQ(",", tokens[6].c_str());
- ASSERT_STREQ(" ", tokens[7].c_str());
- ASSERT_STREQ("everybody", tokens[8].c_str());
- ASSERT_STREQ(".", tokens[9].c_str());
- ASSERT_STREQ(" ", tokens[10].c_str());
- ASSERT_STREQ("welcome", tokens[11].c_str());
- ASSERT_STREQ("!", tokens[12].c_str());
+ ASSERT_STREQ("jordan", tokens[1].c_str());
+ ASSERT_STREQ("welcome", tokens[2].c_str());
+ ASSERT_STREQ("everybody", tokens[3].c_str());
+ ASSERT_STREQ("welcome", tokens[4].c_str());
// check for index when separators are not kept
- Tokenizer tokenizer2(withnewline, false, true, false);
+ Tokenizer tokenizer2(withnewline, true, false);
size_t expected_token_index = 0;
std::vector<std::string> expected_tokens = {"michael", "jordan", "welcome", "everybody", "welcome"};
while(tokenizer2.next(token, token_index)) {
@@ -169,7 +138,7 @@ TEST(TokenizerTest, ShouldTokenizeIteratively) {
// verbatim (no_op=true)
tokens.clear();
- Tokenizer tokenizer3(withnewline, true, false, true);
+ Tokenizer tokenizer3(withnewline, false, true);
while(tokenizer3.next(token, token_index)) {
tokens.push_back(token);
@@ -179,10 +148,23 @@ TEST(TokenizerTest, ShouldTokenizeIteratively) {
ASSERT_STREQ("Michael Jordan:\n\nWelcome, everybody. Welcome!", tokens[0].c_str());
}
+TEST(TokenizerTest, ShouldTokenizeTextWithCustomSpecialChars) {
+ std::vector<std::string> tokens;
+ Tokenizer("and -some -more", true, false, "en", {'-'}).tokenize(tokens);
+ ASSERT_EQ(3, tokens.size());
+ ASSERT_EQ("and", tokens[0]);
+ ASSERT_EQ("-some", tokens[1]);
+ ASSERT_EQ("-more", tokens[2]);
+}
+
TEST(TokenizerTest, ShouldTokenizeLocaleText) {
+ std::string tstr = "ลงรถไฟ";
+ std::vector<std::string> ttokens;
+ Tokenizer(tstr, true, false, "th").tokenize(ttokens);
+
std::string str = "จิ้งจอกสีน้ำตาลด่วน";
std::vector<std::string> tokens;
- Tokenizer(str, false, false, false, "th").tokenize(tokens);
+ Tokenizer(str, false, false, "th").tokenize(tokens);
ASSERT_EQ(4, tokens.size());
ASSERT_EQ("จิ้งจอก", tokens[0]);
@@ -192,7 +174,7 @@ TEST(TokenizerTest, ShouldTokenizeLocaleText) {
tokens.clear();
str = "น. วันที่ 31 มี.ค.";
- Tokenizer(str, false, false, false, "th").tokenize(tokens);
+ Tokenizer(str, false, false, "th").tokenize(tokens);
ASSERT_EQ(5, tokens.size());
ASSERT_EQ("น", tokens[0]);
ASSERT_EQ("วัน", tokens[1]);
@@ -201,7 +183,7 @@ TEST(TokenizerTest, ShouldTokenizeLocaleText) {
ASSERT_EQ("มี.ค", tokens[4]);
tokens.clear();
- Tokenizer("Odd Thomas", false, false, false, "en").tokenize(tokens);
+ Tokenizer("Odd Thomas", false, false, "en").tokenize(tokens);
ASSERT_EQ(2, tokens.size());
ASSERT_EQ("Odd", tokens[0]);
ASSERT_EQ("Thomas", tokens[1]);
@@ -209,14 +191,14 @@ TEST(TokenizerTest, ShouldTokenizeLocaleText) {
// korean
tokens.clear();
- Tokenizer("경승지·산악·협곡", false, false, false, "ko").tokenize(tokens);
+ Tokenizer("경승지·산악·협곡", false, false, "ko").tokenize(tokens);
ASSERT_EQ(3, tokens.size());
ASSERT_EQ("경승지", tokens[0]);
ASSERT_EQ("산악", tokens[1]);
ASSERT_EQ("협곡", tokens[2]);
tokens.clear();
- Tokenizer("안녕은하철도999극장판", false, false, false, "ko").tokenize(tokens);
+ Tokenizer("안녕은하철도999극장판", false, false, "ko").tokenize(tokens);
ASSERT_EQ(3, tokens.size());
ASSERT_EQ("안녕은하철도", tokens[0]);
ASSERT_EQ("999", tokens[1]);
@@ -224,13 +206,13 @@ TEST(TokenizerTest, ShouldTokenizeLocaleText) {
// japanese
tokens.clear();
- Tokenizer("退屈", false, false, false, "ja").tokenize(tokens);
+ Tokenizer("退屈", false, false, "ja").tokenize(tokens);
ASSERT_EQ(2, tokens.size());
ASSERT_EQ("た", tokens[0]);
ASSERT_EQ("いくつ", tokens[1]);
tokens.clear();
- Tokenizer("ア退屈であ", false, false, false, "ja").tokenize(tokens);
+ Tokenizer("ア退屈であ", false, false, "ja").tokenize(tokens);
ASSERT_EQ(5, tokens.size());
ASSERT_EQ("あ", tokens[0]);
ASSERT_EQ("た", tokens[1]);
@@ -239,7 +221,7 @@ TEST(TokenizerTest, ShouldTokenizeLocaleText) {
ASSERT_EQ("あ", tokens[4]);
tokens.clear();
- Tokenizer("怠惰な犬", false, false, false, "ja").tokenize(tokens);
+ Tokenizer("怠惰な犬", false, false, "ja").tokenize(tokens);
ASSERT_EQ(4, tokens.size());
ASSERT_EQ("たい", tokens[0]);
ASSERT_EQ("だ", tokens[1]);