Skip to content

Open Source alternative to Algolia + Pinecone and an Easier-to-Use alternative to ElasticSearch ⚡ 🔍 ✨ Fast, typo tolerant, in-memory fuzzy Search Engine for building delightful search experiences

Return fully highlighted values for specified fields.

kishorenc63ecc841b50b
diff --git a/include/collection.h b/include/collection.h
index 71291a47..41daf0d6 100644
--- a/include/collection.h
+++ b/include/collection.h
@@ -98,6 +98,7 @@ private:
struct highlight_t {
std::string field;
std::vector<std::string> snippets;
+ std::vector<std::string> values;
std::vector<size_t> indices;
uint64_t match_score;
@@ -147,7 +148,9 @@ private:
void highlight_result(const field &search_field, const std::vector<std::vector<art_leaf *>> &searched_queries,
const KV &field_order_kv, const nlohmann::json &document,
- StringUtils & string_utils, size_t snippet_threshold, highlight_t &highlight);
+ StringUtils & string_utils, size_t snippet_threshold,
+ bool highlighted_fully,
+ highlight_t &highlight);
void remove_document(nlohmann::json & document, const uint32_t seq_id, bool remove_from_store);
@@ -223,7 +226,8 @@ public:
const spp::sparse_hash_set<std::string> & exclude_fields = spp::sparse_hash_set<std::string>(),
size_t max_facet_values=10, size_t max_hits=500,
const std::string & simple_facet_query = "",
- const size_t snippet_threshold = 30);
+ const size_t snippet_threshold = 30,
+ const std::string & highlight_full_fields = "");
Option<nlohmann::json> get(const std::string & id);
diff --git a/src/collection.cpp b/src/collection.cpp
index 76295ef8..e01d9485 100644
--- a/src/collection.cpp
+++ b/src/collection.cpp
@@ -312,7 +312,8 @@ Option<nlohmann::json> Collection::search(const std::string & query, const std::
const spp::sparse_hash_set<std::string> & exclude_fields,
const size_t max_facet_values, const size_t max_hits,
const std::string & simple_facet_query,
- const size_t snippet_threshold) {
+ const size_t snippet_threshold,
+ const std::string & highlight_full_fields) {
std::vector<uint32_t> included_ids;
std::vector<uint32_t> excluded_ids;
@@ -694,13 +695,26 @@ Option<nlohmann::json> Collection::search(const std::string & query, const std::
std::vector<highlight_t> highlights;
StringUtils string_utils;
+ // find out if fields have to be highlighted fully
+ std::vector<std::string> fields_highlighted_fully_vec;
+ spp::sparse_hash_set<std::string> fields_highlighted_fully;
+ StringUtils::split(highlight_full_fields, fields_highlighted_fully_vec, ",");
+
+ for(std::string & highlight_full_field: fields_highlighted_fully_vec) {
+ StringUtils::trim(highlight_full_field);
+ fields_highlighted_fully.emplace(highlight_full_field);
+ }
+
for(const std::string & field_name: search_fields) {
field search_field = search_schema.at(field_name);
if(query != "*" && (search_field.type == field_types::STRING ||
search_field.type == field_types::STRING_ARRAY)) {
+
+ bool highlighted_fully = (fields_highlighted_fully.find(field_name) != fields_highlighted_fully.end());
highlight_t highlight;
highlight_result(search_field, searched_queries, field_order_kv, document,
- string_utils, snippet_threshold, highlight);
+ string_utils, snippet_threshold, highlighted_fully, highlight);
+
if(!highlight.snippets.empty()) {
highlights.push_back(highlight);
}
@@ -712,11 +726,20 @@ Option<nlohmann::json> Collection::search(const std::string & query, const std::
for(const auto & highlight: highlights) {
nlohmann::json h_json = nlohmann::json::object();
h_json["field"] = highlight.field;
+ bool highlight_fully = (fields_highlighted_fully.find(highlight.field) != fields_highlighted_fully.end());
+
if(!highlight.indices.empty()) {
h_json["indices"] = highlight.indices;
h_json["snippets"] = highlight.snippets;
+ if(highlight_fully) {
+ h_json["values"] = highlight.values;
+ }
+
} else {
h_json["snippet"] = highlight.snippets[0];
+ if(highlight_fully) {
+ h_json["value"] = highlight.values[0];
+ }
}
wrapper_doc["highlights"].push_back(h_json);
@@ -869,7 +892,9 @@ void Collection::facet_value_to_string(const facet &a_facet, const facet_count_t
void Collection::highlight_result(const field &search_field,
const std::vector<std::vector<art_leaf *>> &searched_queries,
const KV & field_order_kv, const nlohmann::json & document,
- StringUtils & string_utils, size_t snippet_threshold, highlight_t & highlight) {
+ StringUtils & string_utils, size_t snippet_threshold,
+ bool highlighted_fully,
+ highlight_t & highlight) {
spp::sparse_hash_map<const art_leaf*, uint32_t*> leaf_to_indices;
std::vector<art_leaf *> query_suggestion;
@@ -972,10 +997,29 @@ void Collection::highlight_result(const field &search_field,
}
highlight.snippets.push_back(snippet_stream.str());
-
if(search_field.type == field_types::STRING_ARRAY) {
highlight.indices.push_back(match_index.index);
}
+
+ if(highlighted_fully) {
+ std::stringstream value_stream;
+ for(size_t value_index = 0; value_index < tokens.size(); value_index++) {
+ if(value_index != 0) {
+ value_stream << " ";
+ }
+
+ std::string token = tokens[value_index];
+ string_utils.unicode_normalize(token);
+
+ if(token_hits.count(token) != 0) {
+ value_stream << "<mark>" + tokens[value_index] + "</mark>";
+ } else {
+ value_stream << tokens[value_index];
+ }
+ }
+
+ highlight.values.push_back(value_stream.str());
+ }
}
highlight.field = search_field.name;
diff --git a/src/core_api.cpp b/src/core_api.cpp
index 557bde91..669ccc00 100644
--- a/src/core_api.cpp
+++ b/src/core_api.cpp
@@ -188,6 +188,9 @@ void get_search(http_req & req, http_res & res) {
// strings under this length will be fully highlighted, instead of showing a snippet of relevant portion
const char *SNIPPET_THRESHOLD = "snippet_threshold";
+ // list of fields which will be highlighted fully without snippeting
+ const char *HIGHLIGHT_FULL_FIELDS = "highlight_full_fields";
+
if(req.params.count(NUM_TYPOS) == 0) {
req.params[NUM_TYPOS] = "2";
}
@@ -229,6 +232,10 @@ void get_search(http_req & req, http_res & res) {
req.params[SNIPPET_THRESHOLD] = "30";
}
+ if(req.params.count(HIGHLIGHT_FULL_FIELDS) == 0) {
+ req.params[HIGHLIGHT_FULL_FIELDS] = "";
+ }
+
if(req.params.count(PER_PAGE) == 0) {
req.params[PER_PAGE] = "10";
}
@@ -326,7 +333,9 @@ void get_search(http_req & req, http_res & res) {
static_cast<size_t>(std::stoi(req.params[MAX_FACET_VALUES])),
static_cast<size_t>(std::stoi(req.params[MAX_HITS])),
req.params[FACET_QUERY],
- static_cast<size_t>(std::stoi(req.params[SNIPPET_THRESHOLD])));
+ static_cast<size_t>(std::stoi(req.params[SNIPPET_THRESHOLD])),
+ req.params[HIGHLIGHT_FULL_FIELDS]
+ );
uint64_t timeMillis = std::chrono::duration_cast<std::chrono::milliseconds>(
std::chrono::high_resolution_clock::now() - begin).count();
diff --git a/test/collection_test.cpp b/test/collection_test.cpp
index 981ebbd9..2c70ef6c 100644
--- a/test/collection_test.cpp
+++ b/test/collection_test.cpp
@@ -2944,3 +2944,58 @@ TEST_F(CollectionTest, SearchHighlightShouldFollowThreshold) {
collectionManager.drop_collection("coll1");
}
+
+TEST_F(CollectionTest, SearchHighlightFieldFully) {
+ Collection *coll1;
+
+ std::vector<field> fields = { field("title", field_types::STRING, true),
+ field("tags", field_types::STRING_ARRAY, true),
+ field("points", field_types::INT32, false)};
+
+ std::vector<sort_by> sort_fields = {sort_by("points", "DESC")};
+
+ coll1 = collectionManager.get_collection("coll1");
+ if (coll1 == nullptr) {
+ coll1 = collectionManager.create_collection("coll1", fields, "points").get();
+ }
+
+ nlohmann::json doc;
+ doc["id"] = "100";
+ doc["title"] = "The quick brown fox jumped over the lazy dog and ran straight to the forest to sleep.";
+ doc["tags"] = {"NEWS", "LAZY"};
+ doc["points"] = 25;
+
+ auto add_op = coll1->add(doc.dump());
+ ASSERT_TRUE(add_op.ok());
+
+ // look for fully highlighted value in response
+
+ auto res = coll1->search("lazy", {"title"}, "", {}, sort_fields, 0, 10, 1,
+ token_ordering::FREQUENCY, true, 10, spp::sparse_hash_set<std::string>(),
+ spp::sparse_hash_set<std::string>(), 10, 500, "", 5, "title").get();
+
+ ASSERT_EQ(1, res["hits"][0]["highlights"].size());
+ ASSERT_STREQ("The quick brown fox jumped over the <mark>lazy</mark> dog and ran straight to the forest to sleep.",
+ res["hits"][0]["highlights"][0]["value"].get<std::string>().c_str());
+
+ // should not return value key when highlight_full_fields is not specified
+ res = coll1->search("lazy", {"title"}, "", {}, sort_fields, 0, 10, 1,
+ token_ordering::FREQUENCY, true, 10, spp::sparse_hash_set<std::string>(),
+ spp::sparse_hash_set<std::string>(), 10, 500, "", 5, "").get();
+
+ ASSERT_EQ(2, res["hits"][0]["highlights"][0].size());
+
+ // query multiple fields
+ res = coll1->search("lazy", {"title", "tags"}, "", {}, sort_fields, 0, 10, 1,
+ token_ordering::FREQUENCY, true, 10, spp::sparse_hash_set<std::string>(),
+ spp::sparse_hash_set<std::string>(), 10, 500, "", 5, "title, tags").get();
+
+ ASSERT_EQ(2, res["hits"][0]["highlights"].size());
+ ASSERT_STREQ("The quick brown fox jumped over the <mark>lazy</mark> dog and ran straight to the forest to sleep.",
+ res["hits"][0]["highlights"][0]["value"].get<std::string>().c_str());
+
+ ASSERT_EQ(1, res["hits"][0]["highlights"][1]["values"][0].size());
+ ASSERT_STREQ("<mark>LAZY</mark>", res["hits"][0]["highlights"][1]["values"][0].get<std::string>().c_str());
+
+ collectionManager.drop_collection("coll1");
+}