Open Source alternative to Algolia + Pinecone and an Easier-to-Use alternative to ElasticSearch ⚡ 🔍 ✨ Fast, typo tolerant, in-memory fuzzy Search Engine for building delightful search experiences
C++ algoliadatastoreelasticsearchenterprise-searchfacetingfull-text-searchfuzzy-searchgeosearchin-memoryinstantsearchmerchandisingpineconesearchsearch-enginesemantic-searchsimilarity-searchsite-searchsynonymstypo-tolerancevector-search
Refactor highlighting logic.
Kishore Nallanf2cad66dee0f
diff --git a/include/collection.h b/include/collection.hindex 69046a86..16332cb5 100644--- a/include/collection.h+++ b/include/collection.h@@ -120,6 +120,17 @@ struct synonym_t {}};+struct highlight_field_t {+ std::string name;+ bool fully_highlighted;+ bool infix;++ highlight_field_t(const std::string& name, bool fully_highlighted, bool infix):+ name(name), fully_highlighted(fully_highlighted), infix(infix) {++ }+};+class Collection {private:@@ -449,5 +460,12 @@ public:const std::map<size_t, std::string>& prefix_start_offsets, size_t snippet_end_offset,std::vector<std::string>& matched_tokens, std::map<size_t, size_t>::iterator& offset_it,std::stringstream& highlighted_text, size_t snippet_start_offset) ;++ void process_highlight_fields(const std::vector<std::string>& search_fields,+ const spp::sparse_hash_set<std::string>& exclude_fields,+ const string& highlight_fields,+ const std::string& highlight_full_fields,+ const std::vector<infix_t>& infixes,+ std::vector<highlight_field_t>& highlight_items) const;};diff --git a/src/collection.cpp b/src/collection.cppindex 61034cea..61f4fa7c 100644--- a/src/collection.cpp+++ b/src/collection.cpp@@ -971,6 +971,7 @@ Option<nlohmann::json> Collection::search(const std::string & raw_query, const s//LOG(INFO) << "Num indices used for querying: " << indices.size();std::vector<query_tokens_t> field_query_tokens;+ std::vector<std::string> q_tokens; // used for auxillary highlightingif(search_fields.size() == 0) {// has to be a wildcard query@@ -989,6 +990,15 @@ Option<nlohmann::json> Collection::search(const std::string & raw_query, const sfield_query_tokens[i].q_phrases,field_locale, pre_segmented_query);+ if(i == 0) {+ q_tokens = field_query_tokens[i].q_include_tokens;+ for(auto& phrase: field_query_tokens[i].q_phrases) {+ for(auto& token: phrase) {+ q_tokens.push_back(token);+ }+ }+ }+// get synonymssynonym_reduction(field_query_tokens[i].q_include_tokens, field_query_tokens[i].q_synonyms);@@ -1136,6 +1146,15 @@ Option<nlohmann::json> Collection::search(const std::string & raw_query, const s// `end_result_index` could be -1 when max_hits is 0const long end_result_index = std::min((page * per_page), std::min(max_hits, result_group_kvs.size())) - 1;+ // handle which fields have to be highlighted++ std::vector<highlight_field_t> highlight_items;++ if(query != "*") {+ process_highlight_fields(search_fields, exclude_fields, highlight_fields, highlight_full_fields,+ infixes, highlight_items);+ }+nlohmann::json result = nlohmann::json::object();result["found"] = total_found;@@ -1171,67 +1190,21 @@ Option<nlohmann::json> Collection::search(const std::string & raw_query, const sstd::vector<highlight_t> highlights;StringUtils string_utils;- // find out if fields have to be highlighted fully- std::vector<std::string> fields_highlighted_fully_vec;- spp::sparse_hash_set<std::string> fields_highlighted_fully;- StringUtils::split(highlight_full_fields, fields_highlighted_fully_vec, ",");-- std::vector<std::string> fields_highlighted_vec;- std::vector<size_t> fields_highlighted_indices;- if(highlight_fields.empty()) {- for(size_t i = 0; i < search_fields.size(); i++) {- const auto& field_name = search_fields[i];- // should not pick excluded field for highlighting- if(exclude_fields.count(field_name) > 0) {- continue;- }-- fields_highlighted_vec.emplace_back(field_name);- fields_highlighted_indices.push_back(i);- }- } else {- if(query != "*") {- StringUtils::split(highlight_fields, fields_highlighted_vec, ",");- for(size_t i = 0; i < fields_highlighted_vec.size(); i++) {- fields_highlighted_indices.push_back(0);- }- }- }-- for(std::string & highlight_full_field: fields_highlighted_fully_vec) {- fields_highlighted_fully.emplace(highlight_full_field);- }-- for(size_t i = 0; i < fields_highlighted_vec.size(); i++) {- const std::string& field_name = fields_highlighted_vec[i];- std::vector<std::string> q_tokens = field_query_tokens[fields_highlighted_indices[i]].q_include_tokens;-- // we will also add phrase search tokens to the list- for(auto& phrase: field_query_tokens[fields_highlighted_indices[i]].q_phrases) {- for(auto& token: phrase) {- q_tokens.push_back(token);- }- }-+ for(auto& highlight_item: highlight_items) {+ const std::string& field_name = highlight_item.name;if(search_schema.count(field_name) == 0) {continue;}field search_field = search_schema.at(field_name);+if(query != "*" && (search_field.type == field_types::STRING ||search_field.type == field_types::STRING_ARRAY)) {- bool highlighted_fully = (fields_highlighted_fully.find(field_name) != fields_highlighted_fully.end());highlight_t highlight;- //LOG(INFO) << "Highlighting: " << document;- /*if(document["title"] == "Quantum Quest: A Cassini Space Odyssey") {- LOG(INFO) << "here!";- }*/highlight_result(raw_query, search_field, searched_queries, q_tokens, field_order_kv, document,string_utils, snippet_threshold, highlight_affix_num_tokens,- highlighted_fully, highlight_start_tag, highlight_end_tag, highlight);- //LOG(INFO) << "End";-+ highlight_item.fully_highlighted, highlight_start_tag, highlight_end_tag, highlight);if(!highlight.snippets.empty()) {highlights.push_back(highlight);}@@ -1243,19 +1216,18 @@ Option<nlohmann::json> Collection::search(const std::string & raw_query, const sfor(const auto & highlight: highlights) {nlohmann::json h_json = nlohmann::json::object();h_json["field"] = highlight.field;- bool highlight_fully = (fields_highlighted_fully.find(highlight.field) != fields_highlighted_fully.end());if(!highlight.indices.empty()) {h_json["matched_tokens"] = highlight.matched_tokens;h_json["indices"] = highlight.indices;h_json["snippets"] = highlight.snippets;- if(highlight_fully) {+ if(!highlight.values.empty()) {h_json["values"] = highlight.values;}} else {h_json["matched_tokens"] = highlight.matched_tokens[0];h_json["snippet"] = highlight.snippets[0];- if(highlight_fully) {+ if(!highlight.values.empty() && !highlight.values[0].empty()) {h_json["value"] = highlight.values[0];}}@@ -1479,6 +1451,56 @@ Option<nlohmann::json> Collection::search(const std::string & raw_query, const sreturn Option<nlohmann::json>(result);}+void Collection::process_highlight_fields(const std::vector<std::string>& search_fields,+ const spp::sparse_hash_set<std::string>& exclude_fields,+ const string& highlight_fields,+ const std::string& highlight_full_fields,+ const std::vector<infix_t>& infixes,+ std::vector<highlight_field_t>& highlight_items) const {++ // identify full highlight fields+ spp::sparse_hash_set<std::string> fields_highlighted_fully_set;+ std::vector<std::string> fields_highlighted_fully_vec;+ StringUtils::split(highlight_full_fields, fields_highlighted_fully_vec, ",");+ for(std::string & highlight_full_field: fields_highlighted_fully_vec) {+ fields_highlighted_fully_set.insert(highlight_full_field);+ }++ // identify infix enabled fields+ spp::sparse_hash_set<std::string> fields_infixed_set;++ for(size_t i = 0; i < search_fields.size(); i++) {+ const auto& field_name = search_fields[i];++ infix_t field_infix = (i < infixes.size()) ? infixes[i] : infixes[0];+ if(field_infix != off) {+ fields_infixed_set.insert(field_name);+ }+ }++ if(highlight_fields.empty()) {+ for(size_t i = 0; i < search_fields.size(); i++) {+ const auto& field_name = search_fields[i];+ if(exclude_fields.count(field_name) != 0) {+ // should not pick excluded field for highlighting (only for implicit highlighting)+ continue;+ }+ bool fully_highlighted = (fields_highlighted_fully_set.count(field_name) != 0);+ bool infixed = (fields_infixed_set.count(field_name) != 0);+ highlight_items.emplace_back(field_name, fully_highlighted, infixed);+ }+ } else {+ std::vector<std::string> highlight_field_names;+ StringUtils::split(highlight_fields, highlight_field_names, ",");++ for(size_t i = 0; i < highlight_field_names.size(); i++) {+ bool fully_highlighted = (fields_highlighted_fully_set.count(highlight_field_names[i]) != 0);+ bool infixed = (fields_infixed_set.count(highlight_field_names[i]) != 0);+ highlight_items.emplace_back(highlight_field_names[i], fully_highlighted, infixed);+ }+ }+}+void Collection::parse_search_query(const std::string &query, std::vector<std::string>& q_include_tokens,std::vector<std::vector<std::string>>& q_exclude_tokens,std::vector<std::vector<std::string>>& q_phrases,diff --git a/test/collection_specific_test.cpp b/test/collection_specific_test.cppindex 3e8b0758..fb5e6bce 100644--- a/test/collection_specific_test.cpp+++ b/test/collection_specific_test.cpp@@ -59,6 +59,7 @@ TEST_F(CollectionSpecificTest, SearchTextWithHyphen) {}TEST_F(CollectionSpecificTest, ExplicitHighlightFieldsConfig) {+ // Allow highlighting independent of query_by fieldsstd::vector<field> fields = {field("title", field_types::STRING, false),field("description", field_types::STRING, false),field("author", field_types::STRING, false),@@ -109,16 +110,35 @@ TEST_F(CollectionSpecificTest, ExplicitHighlightFieldsConfig) {ASSERT_EQ("description", results["hits"][0]["highlights"][0]["field"].get<std::string>());ASSERT_EQ("author", results["hits"][0]["highlights"][1]["field"].get<std::string>());- // query not matching field selected for highlighting+ // query_by not matching field selected for highlighting- results = coll1->search("pernell", {"title", "author"}, "", {}, {}, {2}, 10,+ results = coll1->search("fox", {"title", "author"}, "", {}, {}, {2}, 10,1, FREQUENCY, {false}, 1, spp::sparse_hash_set<std::string>(),{"description"}, 10, "", 30, 4, "", 1, {}, {}, {}, 0,"<mark>", "</mark>", {1,1}, 10000, true, false, true, "description").get();ASSERT_EQ(1, results["found"].get<size_t>());ASSERT_EQ(1, results["hits"].size());- ASSERT_EQ(0, results["hits"][0]["highlights"].size());+ ASSERT_EQ(1, results["hits"][0]["highlights"].size());+ ASSERT_EQ("description", results["hits"][0]["highlights"][0]["field"].get<std::string>());+ ASSERT_EQ("A story about a brown <mark>fox</mark> who was fast.", results["hits"][0]["highlights"][0]["snippet"].get<std::string>());+ ASSERT_FALSE(results["hits"][0]["highlights"][0].contains("value"));++ // query_by not matching field selected for full highlighting++ results = coll1->search("fox", {"title", "author"}, "", {}, {}, {2}, 10,+ 1, FREQUENCY, {false}, 1, spp::sparse_hash_set<std::string>(),+ {"description"}, 10, "", 30, 4, "description", 1, {}, {}, {}, 0,+ "<mark>", "</mark>", {1,1}, 10000, true, false, true, "description").get();++ ASSERT_EQ(1, results["found"].get<size_t>());+ ASSERT_EQ(1, results["hits"].size());+ ASSERT_EQ(1, results["hits"][0]["highlights"].size());++ ASSERT_EQ("description", results["hits"][0]["highlights"][0]["field"].get<std::string>());+ ASSERT_EQ("A story about a brown <mark>fox</mark> who was fast.", results["hits"][0]["highlights"][0]["snippet"].get<std::string>());+ ASSERT_TRUE(results["hits"][0]["highlights"][0].contains("value"));+ ASSERT_EQ("A story about a brown <mark>fox</mark> who was fast.", results["hits"][0]["highlights"][0]["value"].get<std::string>());// wildcard query with search field names