Open Source alternative to Algolia + Pinecone and an Easier-to-Use alternative to ElasticSearch ⚡ 🔍 ✨ Fast, typo tolerant, in-memory fuzzy Search Engine for building delightful search experiences
C++ algoliadatastoreelasticsearchenterprise-searchfacetingfull-text-searchfuzzy-searchgeosearchin-memoryinstantsearchmerchandisingpineconesearchsearch-enginesemantic-searchsimilarity-searchsite-searchsynonymstypo-tolerancevector-search
Return fully highlighted values for specified fields.
kishorenc63ecc841b50b
diff --git a/include/collection.h b/include/collection.hindex 71291a47..41daf0d6 100644--- a/include/collection.h+++ b/include/collection.h@@ -98,6 +98,7 @@ private:struct highlight_t {std::string field;std::vector<std::string> snippets;+ std::vector<std::string> values;std::vector<size_t> indices;uint64_t match_score;@@ -147,7 +148,9 @@ private:void highlight_result(const field &search_field, const std::vector<std::vector<art_leaf *>> &searched_queries,const KV &field_order_kv, const nlohmann::json &document,- StringUtils & string_utils, size_t snippet_threshold, highlight_t &highlight);+ StringUtils & string_utils, size_t snippet_threshold,+ bool highlighted_fully,+ highlight_t &highlight);void remove_document(nlohmann::json & document, const uint32_t seq_id, bool remove_from_store);@@ -223,7 +226,8 @@ public:const spp::sparse_hash_set<std::string> & exclude_fields = spp::sparse_hash_set<std::string>(),size_t max_facet_values=10, size_t max_hits=500,const std::string & simple_facet_query = "",- const size_t snippet_threshold = 30);+ const size_t snippet_threshold = 30,+ const std::string & highlight_full_fields = "");Option<nlohmann::json> get(const std::string & id);diff --git a/src/collection.cpp b/src/collection.cppindex 76295ef8..e01d9485 100644--- a/src/collection.cpp+++ b/src/collection.cpp@@ -312,7 +312,8 @@ Option<nlohmann::json> Collection::search(const std::string & query, const std::const spp::sparse_hash_set<std::string> & exclude_fields,const size_t max_facet_values, const size_t max_hits,const std::string & simple_facet_query,- const size_t snippet_threshold) {+ const size_t snippet_threshold,+ const std::string & highlight_full_fields) {std::vector<uint32_t> included_ids;std::vector<uint32_t> excluded_ids;@@ -694,13 +695,26 @@ Option<nlohmann::json> Collection::search(const std::string & query, const std::std::vector<highlight_t> highlights;StringUtils string_utils;+ // find out if fields have to be highlighted fully+ std::vector<std::string> fields_highlighted_fully_vec;+ spp::sparse_hash_set<std::string> fields_highlighted_fully;+ StringUtils::split(highlight_full_fields, fields_highlighted_fully_vec, ",");++ for(std::string & highlight_full_field: fields_highlighted_fully_vec) {+ StringUtils::trim(highlight_full_field);+ fields_highlighted_fully.emplace(highlight_full_field);+ }+for(const std::string & field_name: search_fields) {field search_field = search_schema.at(field_name);if(query != "*" && (search_field.type == field_types::STRING ||search_field.type == field_types::STRING_ARRAY)) {++ bool highlighted_fully = (fields_highlighted_fully.find(field_name) != fields_highlighted_fully.end());highlight_t highlight;highlight_result(search_field, searched_queries, field_order_kv, document,- string_utils, snippet_threshold, highlight);+ string_utils, snippet_threshold, highlighted_fully, highlight);+if(!highlight.snippets.empty()) {highlights.push_back(highlight);}@@ -712,11 +726,20 @@ Option<nlohmann::json> Collection::search(const std::string & query, const std::for(const auto & highlight: highlights) {nlohmann::json h_json = nlohmann::json::object();h_json["field"] = highlight.field;+ bool highlight_fully = (fields_highlighted_fully.find(highlight.field) != fields_highlighted_fully.end());+if(!highlight.indices.empty()) {h_json["indices"] = highlight.indices;h_json["snippets"] = highlight.snippets;+ if(highlight_fully) {+ h_json["values"] = highlight.values;+ }+} else {h_json["snippet"] = highlight.snippets[0];+ if(highlight_fully) {+ h_json["value"] = highlight.values[0];+ }}wrapper_doc["highlights"].push_back(h_json);@@ -869,7 +892,9 @@ void Collection::facet_value_to_string(const facet &a_facet, const facet_count_tvoid Collection::highlight_result(const field &search_field,const std::vector<std::vector<art_leaf *>> &searched_queries,const KV & field_order_kv, const nlohmann::json & document,- StringUtils & string_utils, size_t snippet_threshold, highlight_t & highlight) {+ StringUtils & string_utils, size_t snippet_threshold,+ bool highlighted_fully,+ highlight_t & highlight) {spp::sparse_hash_map<const art_leaf*, uint32_t*> leaf_to_indices;std::vector<art_leaf *> query_suggestion;@@ -972,10 +997,29 @@ void Collection::highlight_result(const field &search_field,}highlight.snippets.push_back(snippet_stream.str());-if(search_field.type == field_types::STRING_ARRAY) {highlight.indices.push_back(match_index.index);}++ if(highlighted_fully) {+ std::stringstream value_stream;+ for(size_t value_index = 0; value_index < tokens.size(); value_index++) {+ if(value_index != 0) {+ value_stream << " ";+ }++ std::string token = tokens[value_index];+ string_utils.unicode_normalize(token);++ if(token_hits.count(token) != 0) {+ value_stream << "<mark>" + tokens[value_index] + "</mark>";+ } else {+ value_stream << tokens[value_index];+ }+ }++ highlight.values.push_back(value_stream.str());+ }}highlight.field = search_field.name;diff --git a/src/core_api.cpp b/src/core_api.cppindex 557bde91..669ccc00 100644--- a/src/core_api.cpp+++ b/src/core_api.cpp@@ -188,6 +188,9 @@ void get_search(http_req & req, http_res & res) {// strings under this length will be fully highlighted, instead of showing a snippet of relevant portionconst char *SNIPPET_THRESHOLD = "snippet_threshold";+ // list of fields which will be highlighted fully without snippeting+ const char *HIGHLIGHT_FULL_FIELDS = "highlight_full_fields";+if(req.params.count(NUM_TYPOS) == 0) {req.params[NUM_TYPOS] = "2";}@@ -229,6 +232,10 @@ void get_search(http_req & req, http_res & res) {req.params[SNIPPET_THRESHOLD] = "30";}+ if(req.params.count(HIGHLIGHT_FULL_FIELDS) == 0) {+ req.params[HIGHLIGHT_FULL_FIELDS] = "";+ }+if(req.params.count(PER_PAGE) == 0) {req.params[PER_PAGE] = "10";}@@ -326,7 +333,9 @@ void get_search(http_req & req, http_res & res) {static_cast<size_t>(std::stoi(req.params[MAX_FACET_VALUES])),static_cast<size_t>(std::stoi(req.params[MAX_HITS])),req.params[FACET_QUERY],- static_cast<size_t>(std::stoi(req.params[SNIPPET_THRESHOLD])));+ static_cast<size_t>(std::stoi(req.params[SNIPPET_THRESHOLD])),+ req.params[HIGHLIGHT_FULL_FIELDS]+ );uint64_t timeMillis = std::chrono::duration_cast<std::chrono::milliseconds>(std::chrono::high_resolution_clock::now() - begin).count();diff --git a/test/collection_test.cpp b/test/collection_test.cppindex 981ebbd9..2c70ef6c 100644--- a/test/collection_test.cpp+++ b/test/collection_test.cpp@@ -2944,3 +2944,58 @@ TEST_F(CollectionTest, SearchHighlightShouldFollowThreshold) {collectionManager.drop_collection("coll1");}++TEST_F(CollectionTest, SearchHighlightFieldFully) {+ Collection *coll1;++ std::vector<field> fields = { field("title", field_types::STRING, true),+ field("tags", field_types::STRING_ARRAY, true),+ field("points", field_types::INT32, false)};++ std::vector<sort_by> sort_fields = {sort_by("points", "DESC")};++ coll1 = collectionManager.get_collection("coll1");+ if (coll1 == nullptr) {+ coll1 = collectionManager.create_collection("coll1", fields, "points").get();+ }++ nlohmann::json doc;+ doc["id"] = "100";+ doc["title"] = "The quick brown fox jumped over the lazy dog and ran straight to the forest to sleep.";+ doc["tags"] = {"NEWS", "LAZY"};+ doc["points"] = 25;++ auto add_op = coll1->add(doc.dump());+ ASSERT_TRUE(add_op.ok());++ // look for fully highlighted value in response++ auto res = coll1->search("lazy", {"title"}, "", {}, sort_fields, 0, 10, 1,+ token_ordering::FREQUENCY, true, 10, spp::sparse_hash_set<std::string>(),+ spp::sparse_hash_set<std::string>(), 10, 500, "", 5, "title").get();++ ASSERT_EQ(1, res["hits"][0]["highlights"].size());+ ASSERT_STREQ("The quick brown fox jumped over the <mark>lazy</mark> dog and ran straight to the forest to sleep.",+ res["hits"][0]["highlights"][0]["value"].get<std::string>().c_str());++ // should not return value key when highlight_full_fields is not specified+ res = coll1->search("lazy", {"title"}, "", {}, sort_fields, 0, 10, 1,+ token_ordering::FREQUENCY, true, 10, spp::sparse_hash_set<std::string>(),+ spp::sparse_hash_set<std::string>(), 10, 500, "", 5, "").get();++ ASSERT_EQ(2, res["hits"][0]["highlights"][0].size());++ // query multiple fields+ res = coll1->search("lazy", {"title", "tags"}, "", {}, sort_fields, 0, 10, 1,+ token_ordering::FREQUENCY, true, 10, spp::sparse_hash_set<std::string>(),+ spp::sparse_hash_set<std::string>(), 10, 500, "", 5, "title, tags").get();++ ASSERT_EQ(2, res["hits"][0]["highlights"].size());+ ASSERT_STREQ("The quick brown fox jumped over the <mark>lazy</mark> dog and ran straight to the forest to sleep.",+ res["hits"][0]["highlights"][0]["value"].get<std::string>().c_str());++ ASSERT_EQ(1, res["hits"][0]["highlights"][1]["values"][0].size());+ ASSERT_STREQ("<mark>LAZY</mark>", res["hits"][0]["highlights"][1]["values"][0].get<std::string>().c_str());++ collectionManager.drop_collection("coll1");+}