Skip to content

Open Source alternative to Algolia + Pinecone and an Easier-to-Use alternative to ElasticSearch ⚡ 🔍 ✨ Fast, typo tolerant, in-memory fuzzy Search Engine for building delightful search experiences

Refactor highlighting logic.

Kishore Nallanf2cad66dee0f
diff --git a/include/collection.h b/include/collection.h
index 69046a86..16332cb5 100644
--- a/include/collection.h
+++ b/include/collection.h
@@ -120,6 +120,17 @@ struct synonym_t {
}
};
+struct highlight_field_t {
+ std::string name;
+ bool fully_highlighted;
+ bool infix;
+
+ highlight_field_t(const std::string& name, bool fully_highlighted, bool infix):
+ name(name), fully_highlighted(fully_highlighted), infix(infix) {
+
+ }
+};
+
class Collection {
private:
@@ -449,5 +460,12 @@ public:
const std::map<size_t, std::string>& prefix_start_offsets, size_t snippet_end_offset,
std::vector<std::string>& matched_tokens, std::map<size_t, size_t>::iterator& offset_it,
std::stringstream& highlighted_text, size_t snippet_start_offset) ;
+
+ void process_highlight_fields(const std::vector<std::string>& search_fields,
+ const spp::sparse_hash_set<std::string>& exclude_fields,
+ const string& highlight_fields,
+ const std::string& highlight_full_fields,
+ const std::vector<infix_t>& infixes,
+ std::vector<highlight_field_t>& highlight_items) const;
};
diff --git a/src/collection.cpp b/src/collection.cpp
index 61034cea..61f4fa7c 100644
--- a/src/collection.cpp
+++ b/src/collection.cpp
@@ -971,6 +971,7 @@ Option<nlohmann::json> Collection::search(const std::string & raw_query, const s
//LOG(INFO) << "Num indices used for querying: " << indices.size();
std::vector<query_tokens_t> field_query_tokens;
+ std::vector<std::string> q_tokens; // used for auxillary highlighting
if(search_fields.size() == 0) {
// has to be a wildcard query
@@ -989,6 +990,15 @@ Option<nlohmann::json> Collection::search(const std::string & raw_query, const s
field_query_tokens[i].q_phrases,
field_locale, pre_segmented_query);
+ if(i == 0) {
+ q_tokens = field_query_tokens[i].q_include_tokens;
+ for(auto& phrase: field_query_tokens[i].q_phrases) {
+ for(auto& token: phrase) {
+ q_tokens.push_back(token);
+ }
+ }
+ }
+
// get synonyms
synonym_reduction(field_query_tokens[i].q_include_tokens, field_query_tokens[i].q_synonyms);
@@ -1136,6 +1146,15 @@ Option<nlohmann::json> Collection::search(const std::string & raw_query, const s
// `end_result_index` could be -1 when max_hits is 0
const long end_result_index = std::min((page * per_page), std::min(max_hits, result_group_kvs.size())) - 1;
+ // handle which fields have to be highlighted
+
+ std::vector<highlight_field_t> highlight_items;
+
+ if(query != "*") {
+ process_highlight_fields(search_fields, exclude_fields, highlight_fields, highlight_full_fields,
+ infixes, highlight_items);
+ }
+
nlohmann::json result = nlohmann::json::object();
result["found"] = total_found;
@@ -1171,67 +1190,21 @@ Option<nlohmann::json> Collection::search(const std::string & raw_query, const s
std::vector<highlight_t> highlights;
StringUtils string_utils;
- // find out if fields have to be highlighted fully
- std::vector<std::string> fields_highlighted_fully_vec;
- spp::sparse_hash_set<std::string> fields_highlighted_fully;
- StringUtils::split(highlight_full_fields, fields_highlighted_fully_vec, ",");
-
- std::vector<std::string> fields_highlighted_vec;
- std::vector<size_t> fields_highlighted_indices;
- if(highlight_fields.empty()) {
- for(size_t i = 0; i < search_fields.size(); i++) {
- const auto& field_name = search_fields[i];
- // should not pick excluded field for highlighting
- if(exclude_fields.count(field_name) > 0) {
- continue;
- }
-
- fields_highlighted_vec.emplace_back(field_name);
- fields_highlighted_indices.push_back(i);
- }
- } else {
- if(query != "*") {
- StringUtils::split(highlight_fields, fields_highlighted_vec, ",");
- for(size_t i = 0; i < fields_highlighted_vec.size(); i++) {
- fields_highlighted_indices.push_back(0);
- }
- }
- }
-
- for(std::string & highlight_full_field: fields_highlighted_fully_vec) {
- fields_highlighted_fully.emplace(highlight_full_field);
- }
-
- for(size_t i = 0; i < fields_highlighted_vec.size(); i++) {
- const std::string& field_name = fields_highlighted_vec[i];
- std::vector<std::string> q_tokens = field_query_tokens[fields_highlighted_indices[i]].q_include_tokens;
-
- // we will also add phrase search tokens to the list
- for(auto& phrase: field_query_tokens[fields_highlighted_indices[i]].q_phrases) {
- for(auto& token: phrase) {
- q_tokens.push_back(token);
- }
- }
-
+ for(auto& highlight_item: highlight_items) {
+ const std::string& field_name = highlight_item.name;
if(search_schema.count(field_name) == 0) {
continue;
}
field search_field = search_schema.at(field_name);
+
if(query != "*" && (search_field.type == field_types::STRING ||
search_field.type == field_types::STRING_ARRAY)) {
- bool highlighted_fully = (fields_highlighted_fully.find(field_name) != fields_highlighted_fully.end());
highlight_t highlight;
- //LOG(INFO) << "Highlighting: " << document;
- /*if(document["title"] == "Quantum Quest: A Cassini Space Odyssey") {
- LOG(INFO) << "here!";
- }*/
highlight_result(raw_query, search_field, searched_queries, q_tokens, field_order_kv, document,
string_utils, snippet_threshold, highlight_affix_num_tokens,
- highlighted_fully, highlight_start_tag, highlight_end_tag, highlight);
- //LOG(INFO) << "End";
-
+ highlight_item.fully_highlighted, highlight_start_tag, highlight_end_tag, highlight);
if(!highlight.snippets.empty()) {
highlights.push_back(highlight);
}
@@ -1243,19 +1216,18 @@ Option<nlohmann::json> Collection::search(const std::string & raw_query, const s
for(const auto & highlight: highlights) {
nlohmann::json h_json = nlohmann::json::object();
h_json["field"] = highlight.field;
- bool highlight_fully = (fields_highlighted_fully.find(highlight.field) != fields_highlighted_fully.end());
if(!highlight.indices.empty()) {
h_json["matched_tokens"] = highlight.matched_tokens;
h_json["indices"] = highlight.indices;
h_json["snippets"] = highlight.snippets;
- if(highlight_fully) {
+ if(!highlight.values.empty()) {
h_json["values"] = highlight.values;
}
} else {
h_json["matched_tokens"] = highlight.matched_tokens[0];
h_json["snippet"] = highlight.snippets[0];
- if(highlight_fully) {
+ if(!highlight.values.empty() && !highlight.values[0].empty()) {
h_json["value"] = highlight.values[0];
}
}
@@ -1479,6 +1451,56 @@ Option<nlohmann::json> Collection::search(const std::string & raw_query, const s
return Option<nlohmann::json>(result);
}
+void Collection::process_highlight_fields(const std::vector<std::string>& search_fields,
+ const spp::sparse_hash_set<std::string>& exclude_fields,
+ const string& highlight_fields,
+ const std::string& highlight_full_fields,
+ const std::vector<infix_t>& infixes,
+ std::vector<highlight_field_t>& highlight_items) const {
+
+ // identify full highlight fields
+ spp::sparse_hash_set<std::string> fields_highlighted_fully_set;
+ std::vector<std::string> fields_highlighted_fully_vec;
+ StringUtils::split(highlight_full_fields, fields_highlighted_fully_vec, ",");
+ for(std::string & highlight_full_field: fields_highlighted_fully_vec) {
+ fields_highlighted_fully_set.insert(highlight_full_field);
+ }
+
+ // identify infix enabled fields
+ spp::sparse_hash_set<std::string> fields_infixed_set;
+
+ for(size_t i = 0; i < search_fields.size(); i++) {
+ const auto& field_name = search_fields[i];
+
+ infix_t field_infix = (i < infixes.size()) ? infixes[i] : infixes[0];
+ if(field_infix != off) {
+ fields_infixed_set.insert(field_name);
+ }
+ }
+
+ if(highlight_fields.empty()) {
+ for(size_t i = 0; i < search_fields.size(); i++) {
+ const auto& field_name = search_fields[i];
+ if(exclude_fields.count(field_name) != 0) {
+ // should not pick excluded field for highlighting (only for implicit highlighting)
+ continue;
+ }
+ bool fully_highlighted = (fields_highlighted_fully_set.count(field_name) != 0);
+ bool infixed = (fields_infixed_set.count(field_name) != 0);
+ highlight_items.emplace_back(field_name, fully_highlighted, infixed);
+ }
+ } else {
+ std::vector<std::string> highlight_field_names;
+ StringUtils::split(highlight_fields, highlight_field_names, ",");
+
+ for(size_t i = 0; i < highlight_field_names.size(); i++) {
+ bool fully_highlighted = (fields_highlighted_fully_set.count(highlight_field_names[i]) != 0);
+ bool infixed = (fields_infixed_set.count(highlight_field_names[i]) != 0);
+ highlight_items.emplace_back(highlight_field_names[i], fully_highlighted, infixed);
+ }
+ }
+}
+
void Collection::parse_search_query(const std::string &query, std::vector<std::string>& q_include_tokens,
std::vector<std::vector<std::string>>& q_exclude_tokens,
std::vector<std::vector<std::string>>& q_phrases,
diff --git a/test/collection_specific_test.cpp b/test/collection_specific_test.cpp
index 3e8b0758..fb5e6bce 100644
--- a/test/collection_specific_test.cpp
+++ b/test/collection_specific_test.cpp
@@ -59,6 +59,7 @@ TEST_F(CollectionSpecificTest, SearchTextWithHyphen) {
}
TEST_F(CollectionSpecificTest, ExplicitHighlightFieldsConfig) {
+ // Allow highlighting independent of query_by fields
std::vector<field> fields = {field("title", field_types::STRING, false),
field("description", field_types::STRING, false),
field("author", field_types::STRING, false),
@@ -109,16 +110,35 @@ TEST_F(CollectionSpecificTest, ExplicitHighlightFieldsConfig) {
ASSERT_EQ("description", results["hits"][0]["highlights"][0]["field"].get<std::string>());
ASSERT_EQ("author", results["hits"][0]["highlights"][1]["field"].get<std::string>());
- // query not matching field selected for highlighting
+ // query_by not matching field selected for highlighting
- results = coll1->search("pernell", {"title", "author"}, "", {}, {}, {2}, 10,
+ results = coll1->search("fox", {"title", "author"}, "", {}, {}, {2}, 10,
1, FREQUENCY, {false}, 1, spp::sparse_hash_set<std::string>(),
{"description"}, 10, "", 30, 4, "", 1, {}, {}, {}, 0,
"<mark>", "</mark>", {1,1}, 10000, true, false, true, "description").get();
ASSERT_EQ(1, results["found"].get<size_t>());
ASSERT_EQ(1, results["hits"].size());
- ASSERT_EQ(0, results["hits"][0]["highlights"].size());
+ ASSERT_EQ(1, results["hits"][0]["highlights"].size());
+ ASSERT_EQ("description", results["hits"][0]["highlights"][0]["field"].get<std::string>());
+ ASSERT_EQ("A story about a brown <mark>fox</mark> who was fast.", results["hits"][0]["highlights"][0]["snippet"].get<std::string>());
+ ASSERT_FALSE(results["hits"][0]["highlights"][0].contains("value"));
+
+ // query_by not matching field selected for full highlighting
+
+ results = coll1->search("fox", {"title", "author"}, "", {}, {}, {2}, 10,
+ 1, FREQUENCY, {false}, 1, spp::sparse_hash_set<std::string>(),
+ {"description"}, 10, "", 30, 4, "description", 1, {}, {}, {}, 0,
+ "<mark>", "</mark>", {1,1}, 10000, true, false, true, "description").get();
+
+ ASSERT_EQ(1, results["found"].get<size_t>());
+ ASSERT_EQ(1, results["hits"].size());
+ ASSERT_EQ(1, results["hits"][0]["highlights"].size());
+
+ ASSERT_EQ("description", results["hits"][0]["highlights"][0]["field"].get<std::string>());
+ ASSERT_EQ("A story about a brown <mark>fox</mark> who was fast.", results["hits"][0]["highlights"][0]["snippet"].get<std::string>());
+ ASSERT_TRUE(results["hits"][0]["highlights"][0].contains("value"));
+ ASSERT_EQ("A story about a brown <mark>fox</mark> who was fast.", results["hits"][0]["highlights"][0]["value"].get<std::string>());
// wildcard query with search field names