Open Source alternative to Algolia + Pinecone and an Easier-to-Use alternative to ElasticSearch ⚡ 🔍 ✨ Fast, typo tolerant, in-memory fuzzy Search Engine for building delightful search experiences
C++ algoliadatastoreelasticsearchenterprise-searchfacetingfull-text-searchfuzzy-searchgeosearchin-memoryinstantsearchmerchandisingpineconesearchsearch-enginesemantic-searchsimilarity-searchsite-searchsynonymstypo-tolerancevector-search
Highlighting should include all search fields, and not just the best matched field.
Kishore Nallan0014e19ea685
diff --git a/include/collection.h b/include/collection.hindex f920b8b4..4e411549 100644--- a/include/collection.h+++ b/include/collection.h@@ -19,6 +19,21 @@class Collection {private:+ struct highlight_t {+ std::string field;+ std::string snippet;+ uint64_t match_score;+ int index;++ highlight_t(): match_score(0), index(-1) {++ }++ bool operator<(const highlight_t& a) const {+ return match_score > a.match_score;+ }+ };+std::string name;uint32_t collection_id;@@ -52,6 +67,10 @@ private:Option<uint32_t> validate_index_in_memory(const nlohmann::json &document, uint32_t seq_id);+ void highlight_result(const field &search_field, const std::vector<std::vector<art_leaf *>> &searched_queries,+ const Topster<512>::KV &field_order_kv, const nlohmann::json &document,+ StringUtils & string_utils, highlight_t &highlight);+public:Collection() = delete;diff --git a/include/index.h b/include/index.hindex 26cc9f42..11087a9a 100644--- a/include/index.h+++ b/include/index.h@@ -147,6 +147,8 @@ public:Option<uint32_t> remove(const uint32_t seq_id, nlohmann::json & document);+ art_leaf* get_token_leaf(const std::string & field_name, const unsigned char* token, uint32_t token_len);+static void populate_token_positions(const std::vector<art_leaf *> &query_suggestion,spp::sparse_hash_map<const art_leaf *, uint32_t *> &leaf_to_indices,size_t result_index,diff --git a/src/collection.cpp b/src/collection.cppindex 300fcc0f..44bee62e 100644--- a/src/collection.cpp+++ b/src/collection.cpp@@ -549,7 +549,6 @@ Option<nlohmann::json> Collection::search(std::string query, const std::vector<scontinue;}- nlohmann::json wrapper_doc;nlohmann::json document;try {@@ -558,97 +557,32 @@ Option<nlohmann::json> Collection::search(std::string query, const std::vector<sreturn Option<nlohmann::json>(500, "Error while parsing stored document.");}- // highlight query words in the result- const std::string & field_name = search_fields[Index::FIELD_LIMIT_NUM - field_order_kv.field_id];- field search_field = search_schema.at(field_name);-- if(query != "*" && (search_field.type == field_types::STRING || search_field.type == field_types::STRING_ARRAY)) {-- spp::sparse_hash_map<const art_leaf*, uint32_t*> leaf_to_indices;- for (const art_leaf *token_leaf : searched_queries[field_order_kv.query_index]) {- std::vector<uint16_t> positions;- uint32_t doc_index = token_leaf->values->ids.indexOf(field_order_kv.key);- uint32_t *indices = new uint32_t[1];- indices[0] = doc_index;- leaf_to_indices.emplace(token_leaf, indices);- }-- // positions in the field of each token in the query- std::vector<std::vector<std::vector<uint16_t>>> array_token_positions;- Index::populate_token_positions(searched_queries[field_order_kv.query_index],- leaf_to_indices, 0, array_token_positions);-- Match match;- uint64_t match_score = 0;- size_t matched_array_index = 0;-- for(size_t array_index = 0; array_index < array_token_positions.size(); array_index++) {- const std::vector<std::vector<uint16_t>> & token_positions = array_token_positions[array_index];-- if(token_positions.empty()) {- continue;- }-- const Match & this_match = Match::match(field_order_kv.key, token_positions);- uint64_t this_match_score = this_match.get_match_score(1, field_order_kv.field_id);- if(this_match_score > match_score) {- match_score = this_match_score;- match = this_match;- matched_array_index = array_index;- }- }-- std::vector<std::string> tokens;- if(search_field.type == field_types::STRING) {- StringUtils::split(document[field_name], tokens, " ");- } else {- StringUtils::split(document[field_name][matched_array_index], tokens, " ");- }-- // unpack `match.offset_diffs` into `token_indices`- std::vector<size_t> token_indices;- size_t num_tokens_found = (size_t) match.offset_diffs[0];- for(size_t i = 1; i <= num_tokens_found; i++) {- if(match.offset_diffs[i] != std::numeric_limits<int8_t>::max()) {- size_t token_index = (size_t)(match.start_offset + match.offset_diffs[i]);- token_indices.push_back(token_index);- }- }-- auto minmax = std::minmax_element(token_indices.begin(), token_indices.end());-- // For longer strings, pick surrounding tokens within N tokens of min_index and max_index for the snippet- const size_t start_index = (tokens.size() <= SNIPPET_STR_ABOVE_LEN) ? 0 :- std::max(0, (int)(*(minmax.first)-5));-- const size_t end_index = (tokens.size() <= SNIPPET_STR_ABOVE_LEN) ? tokens.size() :- std::min((int)tokens.size(), (int)(*(minmax.second)+5));-- for(const size_t token_index: token_indices) {- tokens[token_index] = "<mark>" + tokens[token_index] + "</mark>";- }-- std::stringstream snippet_stream;- for(size_t snippet_index = start_index; snippet_index < end_index; snippet_index++) {- if(snippet_index != start_index) {- snippet_stream << " ";- }-- snippet_stream << tokens[snippet_index];+ nlohmann::json wrapper_doc;+ wrapper_doc["highlights"] = nlohmann::json::array();+ std::vector<highlight_t> highlights;+ StringUtils string_utils;++ for(const std::string & field_name: search_fields) {+ field search_field = search_schema.at(field_name);+ if(query != "*" && (search_field.type == field_types::STRING ||+ search_field.type == field_types::STRING_ARRAY)) {+ highlight_t highlight;+ highlight_result(search_field, searched_queries, field_order_kv, document, string_utils, highlight);+ highlights.push_back(highlight);}+ }- wrapper_doc["highlight"] = nlohmann::json::object();- wrapper_doc["highlight"]["field"] = field_name;- wrapper_doc["highlight"]["snippet"] = snippet_stream.str();+ std::sort(highlights.begin(), highlights.end());- if(search_field.type == field_types::STRING_ARRAY) {- wrapper_doc["highlight"]["index"] = matched_array_index;+ for(const auto highlight: highlights) {+ nlohmann::json h_json = nlohmann::json::object();+ h_json["field"] = highlight.field;+ h_json["snippet"] = highlight.snippet;+ if(highlight.index != -1) {+ h_json["index"] = highlight.index;}- for (auto it = leaf_to_indices.begin(); it != leaf_to_indices.end(); it++) {- delete [] it->second;- it->second = nullptr;- }+ wrapper_doc["highlights"].push_back(h_json);}prune_document(document, include_fields, exclude_fields);@@ -695,6 +629,119 @@ Option<nlohmann::json> Collection::search(std::string query, const std::vector<sreturn result;}+void Collection::highlight_result(const field &search_field,+ const std::vector<std::vector<art_leaf *>> &searched_queries,+ const Topster<512>::KV & field_order_kv, const nlohmann::json & document,+ StringUtils & string_utils, highlight_t & highlight) {++ spp::sparse_hash_map<const art_leaf*, uint32_t*> leaf_to_indices;+ std::vector<art_leaf *> query_suggestion;++ for (const art_leaf *token_leaf : searched_queries[field_order_kv.query_index]) {+ // Must search for the token string fresh on that field for the given document since `token_leaf`+ // is from the best matched field and need not be present in other fields of a document.+ Index* index = indices[field_order_kv.key % num_indices];+ art_leaf *actual_leaf = index->get_token_leaf(search_field.name, &token_leaf->key[0], token_leaf->key_len);+ if(actual_leaf != NULL) {+ query_suggestion.push_back(actual_leaf);+ std::vector<uint16_t> positions;+ uint32_t doc_index = actual_leaf->values->ids.indexOf(field_order_kv.key);+ uint32_t *indices = new uint32_t[1];+ indices[0] = doc_index;+ leaf_to_indices.emplace(actual_leaf, indices);+ }+ }++ // positions in the field of each token in the query+ std::vector<std::vector<std::vector<uint16_t>>> array_token_positions;+ Index::populate_token_positions(query_suggestion, leaf_to_indices, 0, array_token_positions);++ if(array_token_positions.size() == 0) {+ // none of the tokens from the query were found on this field+ return ;+ }++ Match match;+ uint64_t match_score = 0;+ size_t matched_array_index = 0;++ for(size_t array_index = 0; array_index < array_token_positions.size(); array_index++) {+ const std::vector<std::vector<uint16_t>> & token_positions = array_token_positions[array_index];++ if(token_positions.empty()) {+ continue;+ }++ const Match & this_match = Match::match(field_order_kv.key, token_positions);+ uint64_t this_match_score = this_match.get_match_score(1, field_order_kv.field_id);+ if(this_match_score > match_score) {+ match_score = this_match_score;+ match = this_match;+ matched_array_index = array_index;+ }+ }++ std::vector<std::string> tokens;+ if(search_field.type == field_types::STRING) {+ StringUtils::split(document[search_field.name], tokens, " ");+ } else {+ StringUtils::split(document[search_field.name][matched_array_index], tokens, " ");+ }++ // unpack `match.offset_diffs` into `token_indices`+ std::vector<size_t> token_indices;+ spp::sparse_hash_set<std::string> token_hits;++ size_t num_tokens_found = (size_t) match.offset_diffs[0];+ for(size_t i = 1; i <= num_tokens_found; i++) {+ if(match.offset_diffs[i] != std::numeric_limits<int8_t>::max()) {+ size_t token_index = (size_t)(match.start_offset + match.offset_diffs[i]);+ token_indices.push_back(token_index);+ std::string token = tokens[token_index];+ string_utils.unicode_normalize(token);+ token_hits.insert(token);+ }+ }++ auto minmax = std::minmax_element(token_indices.begin(), token_indices.end());++ // For longer strings, pick surrounding tokens within N tokens of min_index and max_index for the snippet+ const size_t start_index = (tokens.size() <= SNIPPET_STR_ABOVE_LEN) ? 0 :+ std::max(0, (int)(*(minmax.first) - 5));++ const size_t end_index = (tokens.size() <= SNIPPET_STR_ABOVE_LEN) ? tokens.size() :+ std::min((int)tokens.size(), (int)(*(minmax.second) + 5));++ std::stringstream snippet_stream;+ for(size_t snippet_index = start_index; snippet_index < end_index; snippet_index++) {+ if(snippet_index != start_index) {+ snippet_stream << " ";+ }++ std::string token = tokens[snippet_index];+ string_utils.unicode_normalize(token);++ if(token_hits.count(token) != 0) {+ snippet_stream << "<mark>" + tokens[snippet_index] + "</mark>";+ } else {+ snippet_stream << tokens[snippet_index];+ }+ }++ highlight.field = search_field.name;+ highlight.snippet = snippet_stream.str();+ highlight.match_score = match_score;++ if(search_field.type == field_types::STRING_ARRAY) {+ highlight.index = matched_array_index;+ }++ for (auto it = leaf_to_indices.begin(); it != leaf_to_indices.end(); it++) {+ delete [] it->second;+ it->second = nullptr;+ }+}+Option<nlohmann::json> Collection::get(const std::string & id) {std::string seq_id_str;StoreStatus seq_id_status = store->get(get_doc_id_key(id), seq_id_str);diff --git a/src/index.cpp b/src/index.cppindex a6942610..75d7fcd3 100644--- a/src/index.cpp+++ b/src/index.cpp@@ -1217,4 +1217,9 @@ Option<uint32_t> Index::remove(const uint32_t seq_id, nlohmann::json & document)}return Option<uint32_t>(seq_id);+}++art_leaf* Index::get_token_leaf(const std::string & field_name, const unsigned char* token, uint32_t token_len) {+ const art_tree *t = search_index.at(field_name);+ return (art_leaf*) art_search(t, token, (int) token_len);}\ No newline at end of filediff --git a/test/array_text_documents.jsonl b/test/array_text_documents.jsonlindex 8cd27b6a..722bfbf7 100644--- a/test/array_text_documents.jsonl+++ b/test/array_text_documents.jsonl@@ -1,3 +1,4 @@{"title": "The Truth About Forever", "tags": ["the truth", "about forever", "truth about"], "points": 100}{"title": "Plain Truth", "tags": ["plain", "truth", "plain truth"], "points": 40}-{"title": "Temple of the Winds", "tags": ["temple", "of", "temple of"], "points": 87}\ No newline at end of file+{"title": "Temple of the Winds", "tags": ["temple", "of", "temple of"], "points": 87}+{"title": "Amazing Spiderman is amazing", "tags": ["amazing movie", "spiderman", "really fun really"], "points": 90}\ No newline at end of filediff --git a/test/collection_test.cpp b/test/collection_test.cppindex f6db59f5..1415949f 100644--- a/test/collection_test.cpp+++ b/test/collection_test.cpp@@ -138,9 +138,9 @@ TEST_F(CollectionTest, ExactPhraseSearch) {ASSERT_STREQ(id.c_str(), result_id.c_str());}- ASSERT_EQ(results["hits"][0]["highlight"].size(), (unsigned long) 2);- ASSERT_STREQ(results["hits"][0]["highlight"]["field"].get<std::string>().c_str(), "title");- ASSERT_STREQ(results["hits"][0]["highlight"]["snippet"].get<std::string>().c_str(),+ ASSERT_EQ(results["hits"][0]["highlights"].size(), (unsigned long) 1);+ ASSERT_STREQ(results["hits"][0]["highlights"][0]["field"].get<std::string>().c_str(), "title");+ ASSERT_STREQ(results["hits"][0]["highlights"][0]["snippet"].get<std::string>().c_str(),"What is the power requirement of a <mark>rocket</mark> <mark>launch</mark> these days?");// Check ASC sort order@@ -554,11 +554,11 @@ TEST_F(CollectionTest, ArrayStringFieldHighlight) {ASSERT_STREQ(id.c_str(), result_id.c_str());}- ASSERT_EQ(results["hits"][0]["highlight"].size(), 3);- ASSERT_STREQ(results["hits"][0]["highlight"]["field"].get<std::string>().c_str(), "tags");- ASSERT_STREQ(results["hits"][0]["highlight"]["snippet"].get<std::string>().c_str(),+ ASSERT_EQ(results["hits"][0]["highlights"].size(), 1);+ ASSERT_STREQ(results["hits"][0]["highlights"][0]["field"].get<std::string>().c_str(), "tags");+ ASSERT_STREQ(results["hits"][0]["highlights"][0]["snippet"].get<std::string>().c_str(),"<mark>truth</mark> <mark>about</mark>");- ASSERT_EQ(results["hits"][0]["highlight"]["index"].get<size_t>(), 2);+ ASSERT_EQ(results["hits"][0]["highlights"][0]["index"].get<size_t>(), 2);results = coll_array_text->search("forever truth", query_fields, "", facets, sort_fields, 0, 10, 1, FREQUENCY,false, 0).get();@@ -573,9 +573,9 @@ TEST_F(CollectionTest, ArrayStringFieldHighlight) {ASSERT_STREQ(id.c_str(), result_id.c_str());}- ASSERT_STREQ(results["hits"][0]["highlight"]["field"].get<std::string>().c_str(), "tags");- ASSERT_STREQ(results["hits"][0]["highlight"]["snippet"].get<std::string>().c_str(), "the <mark>truth</mark>");- ASSERT_EQ(results["hits"][0]["highlight"]["index"].get<size_t>(), 0);+ ASSERT_STREQ(results["hits"][0]["highlights"][0]["field"].get<std::string>().c_str(), "tags");+ ASSERT_STREQ(results["hits"][0]["highlights"][0]["snippet"].get<std::string>().c_str(), "the <mark>truth</mark>");+ ASSERT_EQ(results["hits"][0]["highlights"][0]["index"].get<size_t>(), 0);results = coll_array_text->search("truth", query_fields, "", facets, sort_fields, 0, 10, 1, FREQUENCY,false, 0).get();@@ -594,6 +594,56 @@ TEST_F(CollectionTest, ArrayStringFieldHighlight) {false, 0).get();ASSERT_EQ(0, results["hits"].size());+ query_fields = {"title", "tags"};+ results = coll_array_text->search("truth", query_fields, "", facets, sort_fields, 0, 10, 1, FREQUENCY,+ false, 0).get();+ ASSERT_EQ(2, results["hits"].size());+ ASSERT_EQ(2, results["hits"][0]["highlights"].size());++ ids = {"0", "1"};++ for (size_t i = 0; i < results["hits"].size(); i++) {+ nlohmann::json result = results["hits"].at(i);+ std::string result_id = result["document"]["id"];+ std::string id = ids.at(i);+ ASSERT_STREQ(id.c_str(), result_id.c_str());+ }++ ASSERT_EQ(2, results["hits"][0]["highlights"][0].size());+ ASSERT_STREQ(results["hits"][0]["highlights"][0]["field"].get<std::string>().c_str(), "title");+ ASSERT_STREQ(results["hits"][0]["highlights"][0]["snippet"].get<std::string>().c_str(), "The <mark>Truth</mark> About Forever");++ ASSERT_EQ(3, results["hits"][0]["highlights"][1].size());+ ASSERT_STREQ(results["hits"][0]["highlights"][1]["field"].get<std::string>().c_str(), "tags");+ ASSERT_STREQ(results["hits"][0]["highlights"][1]["snippet"].get<std::string>().c_str(), "the <mark>truth</mark>");+ ASSERT_EQ(results["hits"][0]["highlights"][1]["index"].get<size_t>(), 0);++ ASSERT_EQ(2, results["hits"][1]["highlights"][0].size());+ ASSERT_STREQ(results["hits"][1]["highlights"][0]["field"].get<std::string>().c_str(), "title");+ ASSERT_STREQ(results["hits"][1]["highlights"][0]["snippet"].get<std::string>().c_str(), "Plain <mark>Truth</mark>");++ ASSERT_EQ(3, results["hits"][1]["highlights"][1].size());+ ASSERT_STREQ(results["hits"][1]["highlights"][1]["field"].get<std::string>().c_str(), "tags");+ ASSERT_STREQ(results["hits"][1]["highlights"][1]["snippet"].get<std::string>().c_str(), "<mark>truth</mark>");+ ASSERT_EQ(results["hits"][1]["highlights"][1]["index"].get<size_t>(), 1);++ // highlight fields must be ordered based on match score+ results = coll_array_text->search("amazing movie", query_fields, "", facets, sort_fields, 0, 10, 1, FREQUENCY,+ false, 0).get();+ ASSERT_EQ(1, results["hits"].size());+ ASSERT_EQ(2, results["hits"][0]["highlights"].size());++ ASSERT_EQ(3, results["hits"][0]["highlights"][0].size());+ ASSERT_STREQ(results["hits"][0]["highlights"][0]["field"].get<std::string>().c_str(), "tags");+ ASSERT_STREQ(results["hits"][0]["highlights"][0]["snippet"].get<std::string>().c_str(),+ "<mark>amazing</mark> <mark>movie</mark>");+ ASSERT_EQ(results["hits"][0]["highlights"][0]["index"].get<size_t>(), 0);++ ASSERT_EQ(2, results["hits"][0]["highlights"][1].size());+ ASSERT_STREQ(results["hits"][0]["highlights"][1]["field"].get<std::string>().c_str(), "title");+ ASSERT_STREQ(results["hits"][0]["highlights"][1]["snippet"].get<std::string>().c_str(),+ "<mark>Amazing</mark> Spiderman is <mark>amazing</mark>"); // should highlight duplicating tokens+collectionManager.drop_collection("coll_array_text");}