Skip to content

Open Source alternative to Algolia + Pinecone and an Easier-to-Use alternative to ElasticSearch ⚡ 🔍 ✨ Fast, typo tolerant, in-memory fuzzy Search Engine for building delightful search experiences

Highlighting should include all search fields, and not just the best matched field.

Kishore Nallan0014e19ea685
diff --git a/include/collection.h b/include/collection.h
index f920b8b4..4e411549 100644
--- a/include/collection.h
+++ b/include/collection.h
@@ -19,6 +19,21 @@
class Collection {
private:
+ struct highlight_t {
+ std::string field;
+ std::string snippet;
+ uint64_t match_score;
+ int index;
+
+ highlight_t(): match_score(0), index(-1) {
+
+ }
+
+ bool operator<(const highlight_t& a) const {
+ return match_score > a.match_score;
+ }
+ };
+
std::string name;
uint32_t collection_id;
@@ -52,6 +67,10 @@ private:
Option<uint32_t> validate_index_in_memory(const nlohmann::json &document, uint32_t seq_id);
+ void highlight_result(const field &search_field, const std::vector<std::vector<art_leaf *>> &searched_queries,
+ const Topster<512>::KV &field_order_kv, const nlohmann::json &document,
+ StringUtils & string_utils, highlight_t &highlight);
+
public:
Collection() = delete;
diff --git a/include/index.h b/include/index.h
index 26cc9f42..11087a9a 100644
--- a/include/index.h
+++ b/include/index.h
@@ -147,6 +147,8 @@ public:
Option<uint32_t> remove(const uint32_t seq_id, nlohmann::json & document);
+ art_leaf* get_token_leaf(const std::string & field_name, const unsigned char* token, uint32_t token_len);
+
static void populate_token_positions(const std::vector<art_leaf *> &query_suggestion,
spp::sparse_hash_map<const art_leaf *, uint32_t *> &leaf_to_indices,
size_t result_index,
diff --git a/src/collection.cpp b/src/collection.cpp
index 300fcc0f..44bee62e 100644
--- a/src/collection.cpp
+++ b/src/collection.cpp
@@ -549,7 +549,6 @@ Option<nlohmann::json> Collection::search(std::string query, const std::vector<s
continue;
}
- nlohmann::json wrapper_doc;
nlohmann::json document;
try {
@@ -558,97 +557,32 @@ Option<nlohmann::json> Collection::search(std::string query, const std::vector<s
return Option<nlohmann::json>(500, "Error while parsing stored document.");
}
- // highlight query words in the result
- const std::string & field_name = search_fields[Index::FIELD_LIMIT_NUM - field_order_kv.field_id];
- field search_field = search_schema.at(field_name);
-
- if(query != "*" && (search_field.type == field_types::STRING || search_field.type == field_types::STRING_ARRAY)) {
-
- spp::sparse_hash_map<const art_leaf*, uint32_t*> leaf_to_indices;
- for (const art_leaf *token_leaf : searched_queries[field_order_kv.query_index]) {
- std::vector<uint16_t> positions;
- uint32_t doc_index = token_leaf->values->ids.indexOf(field_order_kv.key);
- uint32_t *indices = new uint32_t[1];
- indices[0] = doc_index;
- leaf_to_indices.emplace(token_leaf, indices);
- }
-
- // positions in the field of each token in the query
- std::vector<std::vector<std::vector<uint16_t>>> array_token_positions;
- Index::populate_token_positions(searched_queries[field_order_kv.query_index],
- leaf_to_indices, 0, array_token_positions);
-
- Match match;
- uint64_t match_score = 0;
- size_t matched_array_index = 0;
-
- for(size_t array_index = 0; array_index < array_token_positions.size(); array_index++) {
- const std::vector<std::vector<uint16_t>> & token_positions = array_token_positions[array_index];
-
- if(token_positions.empty()) {
- continue;
- }
-
- const Match & this_match = Match::match(field_order_kv.key, token_positions);
- uint64_t this_match_score = this_match.get_match_score(1, field_order_kv.field_id);
- if(this_match_score > match_score) {
- match_score = this_match_score;
- match = this_match;
- matched_array_index = array_index;
- }
- }
-
- std::vector<std::string> tokens;
- if(search_field.type == field_types::STRING) {
- StringUtils::split(document[field_name], tokens, " ");
- } else {
- StringUtils::split(document[field_name][matched_array_index], tokens, " ");
- }
-
- // unpack `match.offset_diffs` into `token_indices`
- std::vector<size_t> token_indices;
- size_t num_tokens_found = (size_t) match.offset_diffs[0];
- for(size_t i = 1; i <= num_tokens_found; i++) {
- if(match.offset_diffs[i] != std::numeric_limits<int8_t>::max()) {
- size_t token_index = (size_t)(match.start_offset + match.offset_diffs[i]);
- token_indices.push_back(token_index);
- }
- }
-
- auto minmax = std::minmax_element(token_indices.begin(), token_indices.end());
-
- // For longer strings, pick surrounding tokens within N tokens of min_index and max_index for the snippet
- const size_t start_index = (tokens.size() <= SNIPPET_STR_ABOVE_LEN) ? 0 :
- std::max(0, (int)(*(minmax.first)-5));
-
- const size_t end_index = (tokens.size() <= SNIPPET_STR_ABOVE_LEN) ? tokens.size() :
- std::min((int)tokens.size(), (int)(*(minmax.second)+5));
-
- for(const size_t token_index: token_indices) {
- tokens[token_index] = "<mark>" + tokens[token_index] + "</mark>";
- }
-
- std::stringstream snippet_stream;
- for(size_t snippet_index = start_index; snippet_index < end_index; snippet_index++) {
- if(snippet_index != start_index) {
- snippet_stream << " ";
- }
-
- snippet_stream << tokens[snippet_index];
+ nlohmann::json wrapper_doc;
+ wrapper_doc["highlights"] = nlohmann::json::array();
+ std::vector<highlight_t> highlights;
+ StringUtils string_utils;
+
+ for(const std::string & field_name: search_fields) {
+ field search_field = search_schema.at(field_name);
+ if(query != "*" && (search_field.type == field_types::STRING ||
+ search_field.type == field_types::STRING_ARRAY)) {
+ highlight_t highlight;
+ highlight_result(search_field, searched_queries, field_order_kv, document, string_utils, highlight);
+ highlights.push_back(highlight);
}
+ }
- wrapper_doc["highlight"] = nlohmann::json::object();
- wrapper_doc["highlight"]["field"] = field_name;
- wrapper_doc["highlight"]["snippet"] = snippet_stream.str();
+ std::sort(highlights.begin(), highlights.end());
- if(search_field.type == field_types::STRING_ARRAY) {
- wrapper_doc["highlight"]["index"] = matched_array_index;
+ for(const auto highlight: highlights) {
+ nlohmann::json h_json = nlohmann::json::object();
+ h_json["field"] = highlight.field;
+ h_json["snippet"] = highlight.snippet;
+ if(highlight.index != -1) {
+ h_json["index"] = highlight.index;
}
- for (auto it = leaf_to_indices.begin(); it != leaf_to_indices.end(); it++) {
- delete [] it->second;
- it->second = nullptr;
- }
+ wrapper_doc["highlights"].push_back(h_json);
}
prune_document(document, include_fields, exclude_fields);
@@ -695,6 +629,119 @@ Option<nlohmann::json> Collection::search(std::string query, const std::vector<s
return result;
}
+void Collection::highlight_result(const field &search_field,
+ const std::vector<std::vector<art_leaf *>> &searched_queries,
+ const Topster<512>::KV & field_order_kv, const nlohmann::json & document,
+ StringUtils & string_utils, highlight_t & highlight) {
+
+ spp::sparse_hash_map<const art_leaf*, uint32_t*> leaf_to_indices;
+ std::vector<art_leaf *> query_suggestion;
+
+ for (const art_leaf *token_leaf : searched_queries[field_order_kv.query_index]) {
+ // Must search for the token string fresh on that field for the given document since `token_leaf`
+ // is from the best matched field and need not be present in other fields of a document.
+ Index* index = indices[field_order_kv.key % num_indices];
+ art_leaf *actual_leaf = index->get_token_leaf(search_field.name, &token_leaf->key[0], token_leaf->key_len);
+ if(actual_leaf != NULL) {
+ query_suggestion.push_back(actual_leaf);
+ std::vector<uint16_t> positions;
+ uint32_t doc_index = actual_leaf->values->ids.indexOf(field_order_kv.key);
+ uint32_t *indices = new uint32_t[1];
+ indices[0] = doc_index;
+ leaf_to_indices.emplace(actual_leaf, indices);
+ }
+ }
+
+ // positions in the field of each token in the query
+ std::vector<std::vector<std::vector<uint16_t>>> array_token_positions;
+ Index::populate_token_positions(query_suggestion, leaf_to_indices, 0, array_token_positions);
+
+ if(array_token_positions.size() == 0) {
+ // none of the tokens from the query were found on this field
+ return ;
+ }
+
+ Match match;
+ uint64_t match_score = 0;
+ size_t matched_array_index = 0;
+
+ for(size_t array_index = 0; array_index < array_token_positions.size(); array_index++) {
+ const std::vector<std::vector<uint16_t>> & token_positions = array_token_positions[array_index];
+
+ if(token_positions.empty()) {
+ continue;
+ }
+
+ const Match & this_match = Match::match(field_order_kv.key, token_positions);
+ uint64_t this_match_score = this_match.get_match_score(1, field_order_kv.field_id);
+ if(this_match_score > match_score) {
+ match_score = this_match_score;
+ match = this_match;
+ matched_array_index = array_index;
+ }
+ }
+
+ std::vector<std::string> tokens;
+ if(search_field.type == field_types::STRING) {
+ StringUtils::split(document[search_field.name], tokens, " ");
+ } else {
+ StringUtils::split(document[search_field.name][matched_array_index], tokens, " ");
+ }
+
+ // unpack `match.offset_diffs` into `token_indices`
+ std::vector<size_t> token_indices;
+ spp::sparse_hash_set<std::string> token_hits;
+
+ size_t num_tokens_found = (size_t) match.offset_diffs[0];
+ for(size_t i = 1; i <= num_tokens_found; i++) {
+ if(match.offset_diffs[i] != std::numeric_limits<int8_t>::max()) {
+ size_t token_index = (size_t)(match.start_offset + match.offset_diffs[i]);
+ token_indices.push_back(token_index);
+ std::string token = tokens[token_index];
+ string_utils.unicode_normalize(token);
+ token_hits.insert(token);
+ }
+ }
+
+ auto minmax = std::minmax_element(token_indices.begin(), token_indices.end());
+
+ // For longer strings, pick surrounding tokens within N tokens of min_index and max_index for the snippet
+ const size_t start_index = (tokens.size() <= SNIPPET_STR_ABOVE_LEN) ? 0 :
+ std::max(0, (int)(*(minmax.first) - 5));
+
+ const size_t end_index = (tokens.size() <= SNIPPET_STR_ABOVE_LEN) ? tokens.size() :
+ std::min((int)tokens.size(), (int)(*(minmax.second) + 5));
+
+ std::stringstream snippet_stream;
+ for(size_t snippet_index = start_index; snippet_index < end_index; snippet_index++) {
+ if(snippet_index != start_index) {
+ snippet_stream << " ";
+ }
+
+ std::string token = tokens[snippet_index];
+ string_utils.unicode_normalize(token);
+
+ if(token_hits.count(token) != 0) {
+ snippet_stream << "<mark>" + tokens[snippet_index] + "</mark>";
+ } else {
+ snippet_stream << tokens[snippet_index];
+ }
+ }
+
+ highlight.field = search_field.name;
+ highlight.snippet = snippet_stream.str();
+ highlight.match_score = match_score;
+
+ if(search_field.type == field_types::STRING_ARRAY) {
+ highlight.index = matched_array_index;
+ }
+
+ for (auto it = leaf_to_indices.begin(); it != leaf_to_indices.end(); it++) {
+ delete [] it->second;
+ it->second = nullptr;
+ }
+}
+
Option<nlohmann::json> Collection::get(const std::string & id) {
std::string seq_id_str;
StoreStatus seq_id_status = store->get(get_doc_id_key(id), seq_id_str);
diff --git a/src/index.cpp b/src/index.cpp
index a6942610..75d7fcd3 100644
--- a/src/index.cpp
+++ b/src/index.cpp
@@ -1217,4 +1217,9 @@ Option<uint32_t> Index::remove(const uint32_t seq_id, nlohmann::json & document)
}
return Option<uint32_t>(seq_id);
+}
+
+art_leaf* Index::get_token_leaf(const std::string & field_name, const unsigned char* token, uint32_t token_len) {
+ const art_tree *t = search_index.at(field_name);
+ return (art_leaf*) art_search(t, token, (int) token_len);
}
\ No newline at end of file
diff --git a/test/array_text_documents.jsonl b/test/array_text_documents.jsonl
index 8cd27b6a..722bfbf7 100644
--- a/test/array_text_documents.jsonl
+++ b/test/array_text_documents.jsonl
@@ -1,3 +1,4 @@
{"title": "The Truth About Forever", "tags": ["the truth", "about forever", "truth about"], "points": 100}
{"title": "Plain Truth", "tags": ["plain", "truth", "plain truth"], "points": 40}
-{"title": "Temple of the Winds", "tags": ["temple", "of", "temple of"], "points": 87}
\ No newline at end of file
+{"title": "Temple of the Winds", "tags": ["temple", "of", "temple of"], "points": 87}
+{"title": "Amazing Spiderman is amazing", "tags": ["amazing movie", "spiderman", "really fun really"], "points": 90}
\ No newline at end of file
diff --git a/test/collection_test.cpp b/test/collection_test.cpp
index f6db59f5..1415949f 100644
--- a/test/collection_test.cpp
+++ b/test/collection_test.cpp
@@ -138,9 +138,9 @@ TEST_F(CollectionTest, ExactPhraseSearch) {
ASSERT_STREQ(id.c_str(), result_id.c_str());
}
- ASSERT_EQ(results["hits"][0]["highlight"].size(), (unsigned long) 2);
- ASSERT_STREQ(results["hits"][0]["highlight"]["field"].get<std::string>().c_str(), "title");
- ASSERT_STREQ(results["hits"][0]["highlight"]["snippet"].get<std::string>().c_str(),
+ ASSERT_EQ(results["hits"][0]["highlights"].size(), (unsigned long) 1);
+ ASSERT_STREQ(results["hits"][0]["highlights"][0]["field"].get<std::string>().c_str(), "title");
+ ASSERT_STREQ(results["hits"][0]["highlights"][0]["snippet"].get<std::string>().c_str(),
"What is the power requirement of a <mark>rocket</mark> <mark>launch</mark> these days?");
// Check ASC sort order
@@ -554,11 +554,11 @@ TEST_F(CollectionTest, ArrayStringFieldHighlight) {
ASSERT_STREQ(id.c_str(), result_id.c_str());
}
- ASSERT_EQ(results["hits"][0]["highlight"].size(), 3);
- ASSERT_STREQ(results["hits"][0]["highlight"]["field"].get<std::string>().c_str(), "tags");
- ASSERT_STREQ(results["hits"][0]["highlight"]["snippet"].get<std::string>().c_str(),
+ ASSERT_EQ(results["hits"][0]["highlights"].size(), 1);
+ ASSERT_STREQ(results["hits"][0]["highlights"][0]["field"].get<std::string>().c_str(), "tags");
+ ASSERT_STREQ(results["hits"][0]["highlights"][0]["snippet"].get<std::string>().c_str(),
"<mark>truth</mark> <mark>about</mark>");
- ASSERT_EQ(results["hits"][0]["highlight"]["index"].get<size_t>(), 2);
+ ASSERT_EQ(results["hits"][0]["highlights"][0]["index"].get<size_t>(), 2);
results = coll_array_text->search("forever truth", query_fields, "", facets, sort_fields, 0, 10, 1, FREQUENCY,
false, 0).get();
@@ -573,9 +573,9 @@ TEST_F(CollectionTest, ArrayStringFieldHighlight) {
ASSERT_STREQ(id.c_str(), result_id.c_str());
}
- ASSERT_STREQ(results["hits"][0]["highlight"]["field"].get<std::string>().c_str(), "tags");
- ASSERT_STREQ(results["hits"][0]["highlight"]["snippet"].get<std::string>().c_str(), "the <mark>truth</mark>");
- ASSERT_EQ(results["hits"][0]["highlight"]["index"].get<size_t>(), 0);
+ ASSERT_STREQ(results["hits"][0]["highlights"][0]["field"].get<std::string>().c_str(), "tags");
+ ASSERT_STREQ(results["hits"][0]["highlights"][0]["snippet"].get<std::string>().c_str(), "the <mark>truth</mark>");
+ ASSERT_EQ(results["hits"][0]["highlights"][0]["index"].get<size_t>(), 0);
results = coll_array_text->search("truth", query_fields, "", facets, sort_fields, 0, 10, 1, FREQUENCY,
false, 0).get();
@@ -594,6 +594,56 @@ TEST_F(CollectionTest, ArrayStringFieldHighlight) {
false, 0).get();
ASSERT_EQ(0, results["hits"].size());
+ query_fields = {"title", "tags"};
+ results = coll_array_text->search("truth", query_fields, "", facets, sort_fields, 0, 10, 1, FREQUENCY,
+ false, 0).get();
+ ASSERT_EQ(2, results["hits"].size());
+ ASSERT_EQ(2, results["hits"][0]["highlights"].size());
+
+ ids = {"0", "1"};
+
+ for (size_t i = 0; i < results["hits"].size(); i++) {
+ nlohmann::json result = results["hits"].at(i);
+ std::string result_id = result["document"]["id"];
+ std::string id = ids.at(i);
+ ASSERT_STREQ(id.c_str(), result_id.c_str());
+ }
+
+ ASSERT_EQ(2, results["hits"][0]["highlights"][0].size());
+ ASSERT_STREQ(results["hits"][0]["highlights"][0]["field"].get<std::string>().c_str(), "title");
+ ASSERT_STREQ(results["hits"][0]["highlights"][0]["snippet"].get<std::string>().c_str(), "The <mark>Truth</mark> About Forever");
+
+ ASSERT_EQ(3, results["hits"][0]["highlights"][1].size());
+ ASSERT_STREQ(results["hits"][0]["highlights"][1]["field"].get<std::string>().c_str(), "tags");
+ ASSERT_STREQ(results["hits"][0]["highlights"][1]["snippet"].get<std::string>().c_str(), "the <mark>truth</mark>");
+ ASSERT_EQ(results["hits"][0]["highlights"][1]["index"].get<size_t>(), 0);
+
+ ASSERT_EQ(2, results["hits"][1]["highlights"][0].size());
+ ASSERT_STREQ(results["hits"][1]["highlights"][0]["field"].get<std::string>().c_str(), "title");
+ ASSERT_STREQ(results["hits"][1]["highlights"][0]["snippet"].get<std::string>().c_str(), "Plain <mark>Truth</mark>");
+
+ ASSERT_EQ(3, results["hits"][1]["highlights"][1].size());
+ ASSERT_STREQ(results["hits"][1]["highlights"][1]["field"].get<std::string>().c_str(), "tags");
+ ASSERT_STREQ(results["hits"][1]["highlights"][1]["snippet"].get<std::string>().c_str(), "<mark>truth</mark>");
+ ASSERT_EQ(results["hits"][1]["highlights"][1]["index"].get<size_t>(), 1);
+
+ // highlight fields must be ordered based on match score
+ results = coll_array_text->search("amazing movie", query_fields, "", facets, sort_fields, 0, 10, 1, FREQUENCY,
+ false, 0).get();
+ ASSERT_EQ(1, results["hits"].size());
+ ASSERT_EQ(2, results["hits"][0]["highlights"].size());
+
+ ASSERT_EQ(3, results["hits"][0]["highlights"][0].size());
+ ASSERT_STREQ(results["hits"][0]["highlights"][0]["field"].get<std::string>().c_str(), "tags");
+ ASSERT_STREQ(results["hits"][0]["highlights"][0]["snippet"].get<std::string>().c_str(),
+ "<mark>amazing</mark> <mark>movie</mark>");
+ ASSERT_EQ(results["hits"][0]["highlights"][0]["index"].get<size_t>(), 0);
+
+ ASSERT_EQ(2, results["hits"][0]["highlights"][1].size());
+ ASSERT_STREQ(results["hits"][0]["highlights"][1]["field"].get<std::string>().c_str(), "title");
+ ASSERT_STREQ(results["hits"][0]["highlights"][1]["snippet"].get<std::string>().c_str(),
+ "<mark>Amazing</mark> Spiderman is <mark>amazing</mark>"); // should highlight duplicating tokens
+
collectionManager.drop_collection("coll_array_text");
}