Skip to content

Open Source alternative to Algolia + Pinecone and an Easier-to-Use alternative to ElasticSearch ⚡ 🔍 ✨ Fast, typo tolerant, in-memory fuzzy Search Engine for building delightful search experiences

Return matched tokens in highlight response structure.

kishorenc6c1455bc2f4f
diff --git a/include/collection.h b/include/collection.h
index 065ba6f8..0a3f17ce 100644
--- a/include/collection.h
+++ b/include/collection.h
@@ -106,6 +106,7 @@ private:
std::vector<std::string> values;
std::vector<size_t> indices;
uint64_t match_score;
+ std::vector<std::vector<std::string>> matched_tokens;
highlight_t() {
@@ -159,6 +160,8 @@ private:
const size_t snippet_threshold,
const size_t highlight_affix_num_tokens,
bool highlighted_fully,
+ const std::string& highlight_start_tag,
+ const std::string& highlight_end_tag,
highlight_t &highlight);
void remove_document(const nlohmann::json & document, const uint32_t seq_id, bool remove_from_store);
@@ -252,7 +255,9 @@ public:
const std::map<size_t, std::vector<std::string>>& pinned_hits={},
const std::vector<std::string>& hidden_hits={},
const std::vector<std::string>& group_by_fields={},
- const size_t group_limit = 0);
+ const size_t group_limit = 0,
+ const std::string& highlight_start_tag="<mark>",
+ const std::string& highlight_end_tag="</mark>");
Option<nlohmann::json> get(const std::string & id);
diff --git a/include/index.h b/include/index.h
index 9e7570b9..854605d1 100644
--- a/include/index.h
+++ b/include/index.h
@@ -139,8 +139,10 @@ private:
StringUtils string_utils;
- static inline std::vector<art_leaf *> next_suggestion(const std::vector<token_candidates> &token_candidates_vec,
- long long int n);
+ static inline void next_suggestion(const std::vector<token_candidates> &token_candidates_vec,
+ long long int n,
+ std::vector<art_leaf *>& actual_query_suggestion,
+ std::vector<art_leaf *>& query_suggestion);
void log_leaves(const int cost, const std::string &token, const std::vector<art_leaf *> &leaves) const;
diff --git a/include/string_utils.h b/include/string_utils.h
index 46baf767..b7bfb207 100644
--- a/include/string_utils.h
+++ b/include/string_utils.h
@@ -290,4 +290,6 @@ struct StringUtils {
static std::string randstring(size_t length);
static std::string hmac(const std::string& key, const std::string& msg);
+
+ static size_t unicode_length(const std::string& bytes);
};
\ No newline at end of file
diff --git a/src/collection.cpp b/src/collection.cpp
index 328eaaa3..c497271d 100644
--- a/src/collection.cpp
+++ b/src/collection.cpp
@@ -507,7 +507,9 @@ Option<nlohmann::json> Collection::search(const std::string & query, const std::
const std::map<size_t, std::vector<std::string>>& pinned_hits,
const std::vector<std::string>& hidden_hits,
const std::vector<std::string>& group_by_fields,
- const size_t group_limit) {
+ const size_t group_limit,
+ const std::string& highlight_start_tag,
+ const std::string& highlight_end_tag) {
if(query != "*" && search_fields.empty()) {
return Option<nlohmann::json>(400, "No search fields specified for the query.");
@@ -1080,7 +1082,7 @@ Option<nlohmann::json> Collection::search(const std::string & query, const std::
highlight_t highlight;
highlight_result(search_field, searched_queries, field_order_kv, document,
string_utils, snippet_threshold, highlight_affix_num_tokens,
- highlighted_fully, highlight);
+ highlighted_fully, highlight_start_tag, highlight_end_tag, highlight);
if(!highlight.snippets.empty()) {
highlights.push_back(highlight);
@@ -1096,13 +1098,14 @@ Option<nlohmann::json> Collection::search(const std::string & query, const std::
bool highlight_fully = (fields_highlighted_fully.find(highlight.field) != fields_highlighted_fully.end());
if(!highlight.indices.empty()) {
+ h_json["matched_tokens"] = highlight.matched_tokens;
h_json["indices"] = highlight.indices;
h_json["snippets"] = highlight.snippets;
if(highlight_fully) {
h_json["values"] = highlight.values;
}
-
} else {
+ h_json["matched_tokens"] = highlight.matched_tokens[0];
h_json["snippet"] = highlight.snippets[0];
if(highlight_fully) {
h_json["value"] = highlight.values[0];
@@ -1205,7 +1208,9 @@ Option<nlohmann::json> Collection::search(const std::string & query, const std::
// handle query token being larger than actual token (typo correction)
query_token_len = std::min(query_token_len, tokens[i].size());
const std::string & unmarked = tokens[i].substr(query_token_len, std::string::npos);
- highlightedss << "<mark>" + tokens[i].substr(0, query_token_len) + "</mark>" + unmarked;
+ highlightedss << highlight_start_tag +
+ tokens[i].substr(0, query_token_len) +
+ highlight_end_tag + unmarked;
} else {
highlightedss << tokens[i];
}
@@ -1354,10 +1359,12 @@ void Collection::highlight_result(const field &search_field,
const size_t snippet_threshold,
const size_t highlight_affix_num_tokens,
bool highlighted_fully,
+ const std::string& highlight_start_tag,
+ const std::string& highlight_end_tag,
highlight_t & highlight) {
std::vector<uint32_t*> leaf_to_indices;
- std::vector<art_leaf *> query_suggestion;
+ std::vector<art_leaf*> query_suggestion;
for (const art_leaf *token_leaf : searched_queries[field_order_kv->query_index]) {
// Must search for the token string fresh on that field for the given document since `token_leaf`
@@ -1418,9 +1425,9 @@ void Collection::highlight_result(const field &search_field,
std::vector<std::string> tokens;
if(search_field.type == field_types::STRING) {
- StringUtils::split(document[search_field.name], tokens, " ");
+ StringUtils::split(document[search_field.name], tokens, " ", true);
} else {
- StringUtils::split(document[search_field.name][match_index.index], tokens, " ");
+ StringUtils::split(document[search_field.name][match_index.index], tokens, " ", true);
}
std::vector<size_t> token_indices;
@@ -1445,7 +1452,7 @@ void Collection::highlight_result(const field &search_field,
size_t prefix_length = highlight_affix_num_tokens;
size_t suffix_length = highlight_affix_num_tokens + 1;
- // For longer strings, pick surrounding tokens within 4 tokens of min_index and max_index for the snippet
+ // For longer strings, pick surrounding tokens within `prefix_length` of min_index and max_index for snippet
const size_t start_index = (tokens.size() <= snippet_threshold) ? 0 :
std::max(0, (int)(*(minmax.first) - prefix_length));
@@ -1453,6 +1460,10 @@ void Collection::highlight_result(const field &search_field,
std::min((int)tokens.size(), (int)(*(minmax.second) + suffix_length));
std::stringstream snippet_stream;
+
+ highlight.matched_tokens.emplace_back();
+ std::vector<std::string>& matched_tokens = highlight.matched_tokens.back();
+
for(size_t snippet_index = start_index; snippet_index < end_index; snippet_index++) {
if(snippet_index != start_index) {
snippet_stream << " ";
@@ -1462,7 +1473,8 @@ void Collection::highlight_result(const field &search_field,
string_utils.unicode_normalize(token);
if(token_hits.count(token) != 0) {
- snippet_stream << "<mark>" + tokens[snippet_index] + "</mark>";
+ snippet_stream << highlight_start_tag << tokens[snippet_index] << highlight_end_tag;
+ matched_tokens.push_back(tokens[snippet_index]);
} else {
snippet_stream << tokens[snippet_index];
}
@@ -1484,7 +1496,7 @@ void Collection::highlight_result(const field &search_field,
string_utils.unicode_normalize(token);
if(token_hits.count(token) != 0) {
- value_stream << "<mark>" + tokens[value_index] + "</mark>";
+ value_stream << highlight_start_tag << tokens[value_index] << highlight_end_tag;
} else {
value_stream << tokens[value_index];
}
diff --git a/src/core_api.cpp b/src/core_api.cpp
index a0e2ee21..d0fa76aa 100644
--- a/src/core_api.cpp
+++ b/src/core_api.cpp
@@ -272,6 +272,9 @@ bool get_search(http_req & req, http_res & res) {
// list of fields which will be highlighted fully without snippeting
const char *HIGHLIGHT_FULL_FIELDS = "highlight_full_fields";
+ const char *HIGHLIGHT_START_TAG = "highlight_start_tag";
+ const char *HIGHLIGHT_END_TAG = "highlight_end_tag";
+
if(req.params.count(NUM_TYPOS) == 0) {
req.params[NUM_TYPOS] = "2";
}
@@ -313,6 +316,14 @@ bool get_search(http_req & req, http_res & res) {
req.params[HIGHLIGHT_FULL_FIELDS] = "";
}
+ if(req.params.count(HIGHLIGHT_START_TAG) == 0) {
+ req.params[HIGHLIGHT_START_TAG] = "<mark>";
+ }
+
+ if(req.params.count(HIGHLIGHT_END_TAG) == 0) {
+ req.params[HIGHLIGHT_END_TAG] = "</mark>";
+ }
+
if(req.params.count(PER_PAGE) == 0) {
if(req.params[FACET_QUERY].empty()) {
req.params[PER_PAGE] = "10";
diff --git a/src/index.cpp b/src/index.cpp
index 4efa5575..adb81c6b 100644
--- a/src/index.cpp
+++ b/src/index.cpp
@@ -557,13 +557,17 @@ uint64_t Index::facet_token_hash(const field & a_field, const std::string &token
void Index::index_string_field(const std::string & text, const int64_t score, art_tree *t,
uint32_t seq_id, int facet_id, const field & a_field) {
std::vector<std::string> tokens;
- StringUtils::split(text, tokens, " ");
+ StringUtils::split(text, tokens, " ", a_field.is_string());
std::unordered_map<std::string, std::vector<uint32_t>> token_to_offsets;
- for(uint32_t i=0; i<tokens.size(); i++) {
+ for(size_t i=0; i<tokens.size(); i++) {
auto & token = tokens[i];
+ if(token.empty()) {
+ continue;
+ }
+
if(a_field.is_string()) {
string_utils.unicode_normalize(token);
}
@@ -595,7 +599,7 @@ void Index::index_string_array_field(const std::vector<std::string> & strings, c
const std::string & str = strings[array_index];
std::vector<std::string> tokens;
std::string delim = " ";
- StringUtils::split(str, tokens, delim);
+ StringUtils::split(str, tokens, delim, a_field.is_string());
std::set<std::string> token_set; // required to deal with repeating tokens
@@ -603,6 +607,10 @@ void Index::index_string_array_field(const std::vector<std::string> & strings, c
for(size_t i=0; i<tokens.size(); i++) {
auto & token = tokens[i];
+ if(token.empty()) {
+ continue;
+ }
+
if(a_field.is_string()) {
string_utils.unicode_normalize(token);
}
@@ -903,7 +911,12 @@ void Index::search_candidates(const uint8_t & field_id, uint32_t* filter_ids, si
for(long long n=0; n<N && n<combination_limit; ++n) {
// every element in `query_suggestion` contains a token and its associated hits
- std::vector<art_leaf *> query_suggestion = next_suggestion(token_candidates_vec, n);
+ std::vector<art_leaf*> query_suggestion(token_candidates_vec.size());
+
+ // actual query suggestion preserves original order of tokens in query
+ std::vector<art_leaf*> actual_query_suggestion(token_candidates_vec.size());
+
+ next_suggestion(token_candidates_vec, n, actual_query_suggestion, query_suggestion);
/*LOG(INFO) << "n: " << n;
for(size_t i=0; i < query_suggestion.size(); i++) {
@@ -978,7 +991,7 @@ void Index::search_candidates(const uint8_t & field_id, uint32_t* filter_ids, si
delete[] result_ids;
}
- searched_queries.push_back(query_suggestion);
+ searched_queries.push_back(actual_query_suggestion);
//LOG(INFO) << "all_result_ids_len: " << all_result_ids_len << ", typo_tokens_threshold: " << typo_tokens_threshold;
if(all_result_ids_len >= typo_tokens_threshold) {
@@ -1854,14 +1867,15 @@ void Index::populate_token_positions(const std::vector<art_leaf *>& query_sugges
}
}
-inline std::vector<art_leaf *> Index::next_suggestion(const std::vector<token_candidates> &token_candidates_vec,
- long long int n) {
- std::vector<art_leaf*> query_suggestion(token_candidates_vec.size());
-
+inline void Index::next_suggestion(const std::vector<token_candidates> &token_candidates_vec,
+ long long int n,
+ std::vector<art_leaf *>& actual_query_suggestion,
+ std::vector<art_leaf *>& query_suggestion) {
// generate the next combination from `token_leaves` and store it in `query_suggestion`
ldiv_t q { n, 0 };
for(long long i = 0 ; i < (long long) token_candidates_vec.size(); i++) {
q = ldiv(q.quot, token_candidates_vec[i].candidates.size());
+ actual_query_suggestion[i] = token_candidates_vec[i].candidates[q.rem];
query_suggestion[i] = token_candidates_vec[i].candidates[q.rem];
}
@@ -1870,8 +1884,6 @@ inline std::vector<art_leaf *> Index::next_suggestion(const std::vector<token_ca
sort(query_suggestion.begin(), query_suggestion.end(), [](const art_leaf* left, const art_leaf* right) {
return left->values->ids.getLength() < right->values->ids.getLength();
});
-
- return query_suggestion;
}
void Index::remove_and_shift_offset_index(sorted_array& offset_index, const uint32_t* indices_sorted,
@@ -1989,11 +2001,11 @@ Option<uint32_t> Index::remove(const uint32_t seq_id, const nlohmann::json & doc
void Index::tokenize_doc_field(const nlohmann::json& document, const std::string& field_name, const field& search_field,
std::vector<std::string>& tokens) {
if(search_field.type == field_types::STRING) {
- StringUtils::split(document[field_name], tokens, " ");
+ StringUtils::split(document[field_name], tokens, " ", true);
} else if(search_field.type == field_types::STRING_ARRAY) {
const std::vector<std::string>& values = document[field_name].get<std::vector<std::string>>();
for(const std::string & value: values) {
- StringUtils::split(value, tokens, " ");
+ StringUtils::split(value, tokens, " ", true);
}
} else if(search_field.type == field_types::INT32) {
const int KEY_LEN = 8;
diff --git a/src/string_utils.cpp b/src/string_utils.cpp
index 6c1e8835..39246d0f 100644
--- a/src/string_utils.cpp
+++ b/src/string_utils.cpp
@@ -3,6 +3,7 @@
#include <openssl/evp.h>
#include <openssl/hmac.h>
#include <random>
+#include <codecvt>
std::string lower_and_no_special_chars(const std::string & str) {
std::stringstream ss;
@@ -29,6 +30,8 @@ void StringUtils::unicode_normalize(std::string & str) const {
for (char *s = &str[0]; *s;) {
char inbuf[5];
char *p = inbuf;
+
+ // group bytes to form a unicode representation
*p++ = *s++;
if ((*s & 0xC0) == 0x80) *p++ = *s++;
if ((*s & 0xC0) == 0x80) *p++ = *s++;
@@ -97,3 +100,8 @@ std::string StringUtils::str2hex(const std::string &str, bool capital) {
return hexstr;
}
+
+size_t StringUtils::unicode_length(const std::string& bytes) {
+ std::wstring_convert<std::codecvt_utf8<char32_t>, char32_t> utf8conv;
+ return utf8conv.from_bytes(bytes).size();
+}
diff --git a/test/collection_test.cpp b/test/collection_test.cpp
index 92c4ab6d..91d3fd8c 100644
--- a/test/collection_test.cpp
+++ b/test/collection_test.cpp
@@ -726,31 +726,43 @@ TEST_F(CollectionTest, ArrayStringFieldHighlight) {
ASSERT_STREQ(id.c_str(), result_id.c_str());
}
- ASSERT_EQ(2, results["hits"][0]["highlights"][0].size());
- ASSERT_STREQ(results["hits"][0]["highlights"][0]["field"].get<std::string>().c_str(), "title");
- ASSERT_STREQ(results["hits"][0]["highlights"][0]["snippet"].get<std::string>().c_str(), "The <mark>Truth</mark> About Forever");
+ ASSERT_EQ(3, results["hits"][0]["highlights"][0].size());
+ ASSERT_STREQ("title", results["hits"][0]["highlights"][0]["field"].get<std::string>().c_str());
+ ASSERT_STREQ("The <mark>Truth</mark> About Forever", results["hits"][0]["highlights"][0]["snippet"].get<std::string>().c_str());
+ ASSERT_EQ(1, results["hits"][0]["highlights"][0]["matched_tokens"].size());
+ ASSERT_STREQ("Truth", results["hits"][0]["highlights"][0]["matched_tokens"][0].get<std::string>().c_str());
- ASSERT_EQ(3, results["hits"][0]["highlights"][1].size());
+ ASSERT_EQ(4, results["hits"][0]["highlights"][1].size());
ASSERT_STREQ(results["hits"][0]["highlights"][1]["field"].get<std::string>().c_str(), "tags");
ASSERT_EQ(2, results["hits"][0]["highlights"][1]["snippets"].size());
ASSERT_STREQ("the <mark>truth</mark>", results["hits"][0]["highlights"][1]["snippets"][0].get<std::string>().c_str());
ASSERT_STREQ("<mark>truth</mark> about", results["hits"][0]["highlights"][1]["snippets"][1].get<std::string>().c_str());
+ ASSERT_EQ(2, results["hits"][0]["highlights"][1]["matched_tokens"].size());
+ ASSERT_STREQ("truth", results["hits"][0]["highlights"][1]["matched_tokens"][0][0].get<std::string>().c_str());
+ ASSERT_STREQ("truth", results["hits"][0]["highlights"][1]["matched_tokens"][1][0].get<std::string>().c_str());
+
ASSERT_EQ(2, results["hits"][0]["highlights"][1]["indices"].size());
ASSERT_EQ(0, results["hits"][0]["highlights"][1]["indices"][0]);
ASSERT_EQ(2, results["hits"][0]["highlights"][1]["indices"][1]);
- ASSERT_EQ(2, results["hits"][1]["highlights"][0].size());
- ASSERT_STREQ(results["hits"][1]["highlights"][0]["field"].get<std::string>().c_str(), "title");
- ASSERT_STREQ(results["hits"][1]["highlights"][0]["snippet"].get<std::string>().c_str(), "Plain <mark>Truth</mark>");
+ ASSERT_EQ(3, results["hits"][1]["highlights"][0].size());
+ ASSERT_STREQ("title", results["hits"][1]["highlights"][0]["field"].get<std::string>().c_str());
+ ASSERT_STREQ("Plain <mark>Truth</mark>", results["hits"][1]["highlights"][0]["snippet"].get<std::string>().c_str());
+ ASSERT_EQ(1, results["hits"][1]["highlights"][0]["matched_tokens"].size());
+ ASSERT_STREQ("Truth", results["hits"][1]["highlights"][0]["matched_tokens"][0].get<std::string>().c_str());
- ASSERT_EQ(3, results["hits"][1]["highlights"][1].size());
+ ASSERT_EQ(4, results["hits"][1]["highlights"][1].size());
ASSERT_STREQ(results["hits"][1]["highlights"][1]["field"].get<std::string>().c_str(), "tags");
ASSERT_EQ(2, results["hits"][1]["highlights"][1]["snippets"].size());
ASSERT_STREQ("<mark>truth</mark>", results["hits"][1]["highlights"][1]["snippets"][0].get<std::string>().c_str());
ASSERT_STREQ("plain <mark>truth</mark>", results["hits"][1]["highlights"][1]["snippets"][1].get<std::string>().c_str());
+ ASSERT_EQ(2, results["hits"][1]["highlights"][1]["matched_tokens"].size());
+ ASSERT_STREQ("truth", results["hits"][0]["highlights"][1]["matched_tokens"][0][0].get<std::string>().c_str());
+ ASSERT_STREQ("truth", results["hits"][0]["highlights"][1]["matched_tokens"][1][0].get<std::string>().c_str());
+
ASSERT_EQ(2, results["hits"][1]["highlights"][1]["indices"].size());
ASSERT_EQ(1, results["hits"][1]["highlights"][1]["indices"][0]);
ASSERT_EQ(2, results["hits"][1]["highlights"][1]["indices"][1]);
@@ -761,16 +773,23 @@ TEST_F(CollectionTest, ArrayStringFieldHighlight) {
ASSERT_EQ(1, results["hits"].size());
ASSERT_EQ(2, results["hits"][0]["highlights"].size());
- ASSERT_EQ(3, results["hits"][0]["highlights"][0].size());
+ ASSERT_EQ(4, results["hits"][0]["highlights"][0].size());
ASSERT_STREQ("tags", results["hits"][0]["highlights"][0]["field"].get<std::string>().c_str());
ASSERT_STREQ("<mark>amazing</mark> <mark>movie</mark>", results["hits"][0]["highlights"][0]["snippets"][0].get<std::string>().c_str());
ASSERT_EQ(1, results["hits"][0]["highlights"][0]["indices"].size());
ASSERT_EQ(0, results["hits"][0]["highlights"][0]["indices"][0]);
+ ASSERT_EQ(1, results["hits"][0]["highlights"][0]["matched_tokens"].size());
+ ASSERT_STREQ("amazing", results["hits"][0]["highlights"][0]["matched_tokens"][0][0].get<std::string>().c_str());
- ASSERT_EQ(2, results["hits"][0]["highlights"][1].size());
+ ASSERT_EQ(3, results["hits"][0]["highlights"][1].size());
ASSERT_STREQ(results["hits"][0]["highlights"][1]["field"].get<std::string>().c_str(), "title");
ASSERT_STREQ(results["hits"][0]["highlights"][1]["snippet"].get<std::string>().c_str(),
"<mark>Amazing</mark> Spiderman is <mark>amazing</mark>"); // should highlight duplicating tokens
+ LOG(INFO) << results["hits"][0]["highlights"][1];
+
+ ASSERT_EQ(2, results["hits"][0]["highlights"][1]["matched_tokens"].size());
+ ASSERT_STREQ("Amazing", results["hits"][0]["highlights"][1]["matched_tokens"][0].get<std::string>().c_str());
+ ASSERT_STREQ("amazing", results["hits"][0]["highlights"][1]["matched_tokens"][1].get<std::string>().c_str());
// when query tokens are not found in an array field they should be ignored
results = coll_array_text->search("winds", query_fields, "", facets, sort_fields, 0, 10, 1, FREQUENCY,
@@ -1940,6 +1959,63 @@ TEST_F(CollectionTest, SearchHighlightShouldFollowThreshold) {
collectionManager.drop_collection("coll1");
}
+TEST_F(CollectionTest, SearchHighlightShouldUseHighlightTags) {
+ Collection *coll1;
+
+ std::vector<field> fields = {field("title", field_types::STRING, true),
+ field("points", field_types::INT32, false)};
+
+ std::vector<sort_by> sort_fields = {sort_by("points", "DESC")};
+
+ coll1 = collectionManager.get_collection("coll1");
+ if (coll1 == nullptr) {
+ coll1 = collectionManager.create_collection("coll1", 4, fields, "points").get();
+ }
+
+ nlohmann::json doc;
+ doc["id"] = "100";
+ doc["title"] = "The quick brown fox jumped over the lazy fox. "; // adding some extra spaces
+ doc["points"] = 25;
+
+ auto add_op = coll1->add(doc.dump());
+ ASSERT_TRUE(add_op.ok());
+
+ // use non-default highlighting tags
+
+ auto res = coll1->search("lazy", {"title"}, "", {}, sort_fields, 0, 10, 1,
+ token_ordering::FREQUENCY, true, 10, spp::sparse_hash_set<std::string>(),
+ spp::sparse_hash_set<std::string>(), 10, "", 30, 4, "", 40, {}, {}, {}, 0,
+ "<em>", "</em>").get();
+
+ ASSERT_STREQ("The quick brown fox jumped over the <em>lazy</em> fox. ",
+ res["hits"][0]["highlights"][0]["snippet"].get<std::string>().c_str());
+
+ // empty tags
+
+ res = coll1->search("lazy", {"title"}, "", {}, sort_fields, 0, 10, 1,
+ token_ordering::FREQUENCY, true, 10, spp::sparse_hash_set<std::string>(),
+ spp::sparse_hash_set<std::string>(), 10, "", 30, 4, "", 40, {}, {}, {}, 0,
+ "", "").get();
+
+ ASSERT_STREQ("The quick brown fox jumped over the lazy fox. ",
+ res["hits"][0]["highlights"][0]["snippet"].get<std::string>().c_str());
+
+ // check for matched tokens
+
+ res = coll1->search("fox", {"title"}, "", {}, sort_fields, 0, 10, 1,
+ token_ordering::FREQUENCY, true, 10, spp::sparse_hash_set<std::string>(),
+ spp::sparse_hash_set<std::string>(), 10, "", 30, 4, "", 40, {}, {}, {}, 0).get();
+
+ ASSERT_EQ(2, res["hits"][0]["highlights"][0]["matched_tokens"].size());
+ ASSERT_STREQ("fox",
+ res["hits"][0]["highlights"][0]["matched_tokens"][0].get<std::string>().c_str());
+
+ ASSERT_STREQ("fox.",
+ res["hits"][0]["highlights"][0]["matched_tokens"][1].get<std::string>().c_str());
+
+ collectionManager.drop_collection("coll1");
+}
+
TEST_F(CollectionTest, UpdateDocument) {
Collection *coll1;
@@ -2201,7 +2277,7 @@ TEST_F(CollectionTest, SearchHighlightFieldFully) {
token_ordering::FREQUENCY, true, 10, spp::sparse_hash_set<std::string>(),
spp::sparse_hash_set<std::string>(), 10, "", 5, 5, "").get();
- ASSERT_EQ(2, res["hits"][0]["highlights"][0].size());
+ ASSERT_EQ(3, res["hits"][0]["highlights"][0].size());
// query multiple fields
res = coll1->search("lazy", {"title", "tags"}, "", {}, sort_fields, 0, 10, 1,
@@ -2212,6 +2288,9 @@ TEST_F(CollectionTest, SearchHighlightFieldFully) {
ASSERT_STREQ("The quick brown fox jumped over the <mark>lazy</mark> dog and ran straight to the forest to sleep.",
res["hits"][0]["highlights"][0]["value"].get<std::string>().c_str());
+ ASSERT_EQ(1, res["hits"][0]["highlights"][0]["matched_tokens"].size());
+ ASSERT_STREQ("lazy", res["hits"][0]["highlights"][0]["matched_tokens"][0].get<std::string>().c_str());
+
ASSERT_EQ(1, res["hits"][0]["highlights"][1]["values"][0].size());
ASSERT_STREQ("<mark>LAZY</mark>", res["hits"][0]["highlights"][1]["values"][0].get<std::string>().c_str());