Skip to content

Open Source alternative to Algolia + Pinecone and an Easier-to-Use alternative to ElasticSearch ⚡ 🔍 ✨ Fast, typo tolerant, in-memory fuzzy Search Engine for building delightful search experiences

Highlight meta should follow nested doc structure.

Kishore Nallanc87e0ddf9ddb
diff --git a/include/collection.h b/include/collection.h
index 3f785124..297952ab 100644
--- a/include/collection.h
+++ b/include/collection.h
@@ -246,6 +246,7 @@ private:
template<class T>
static bool highlight_nested_field(const nlohmann::json& hdoc, nlohmann::json& hobj,
const nlohmann::json& fdoc, nlohmann::json& fobj,
+ const nlohmann::json& mdoc, nlohmann::json& mobj,
std::vector<std::string>& path_parts, size_t path_index, T func);
static Option<bool> resolve_field_type(field& new_field,
@@ -500,6 +501,7 @@ public:
template<class T>
bool Collection::highlight_nested_field(const nlohmann::json& hdoc, nlohmann::json& hobj,
const nlohmann::json& fdoc, nlohmann::json& fobj,
+ const nlohmann::json& mdoc, nlohmann::json& mobj,
std::vector<std::string>& path_parts, size_t path_index, T func) {
if(path_index == path_parts.size()) {
// end of path: guaranteed to be a string
@@ -507,7 +509,7 @@ bool Collection::highlight_nested_field(const nlohmann::json& hdoc, nlohmann::js
return false;
}
- func(hobj, fobj);
+ func(hobj, fobj, mobj);
}
const std::string& fragment = path_parts[path_index];
@@ -519,12 +521,15 @@ bool Collection::highlight_nested_field(const nlohmann::json& hdoc, nlohmann::js
for(size_t i = 0; i < it.value().size(); i++) {
auto& h_ele = it.value().at(i);
auto& f_ele = fobj.empty() ? fobj : fobj[fragment][i];
- resolved |= highlight_nested_field(hdoc, h_ele, fdoc, f_ele, path_parts, path_index + 1, func);
+ auto& m_ele = mobj.empty() ? mobj : mobj[fragment][i];
+ resolved |= highlight_nested_field(hdoc, h_ele, fdoc, f_ele, mdoc, m_ele,
+ path_parts, path_index + 1, func);
}
return resolved;
} else {
auto& f_ele = fobj.empty() ? fobj : fobj[fragment];
- return highlight_nested_field(hdoc, it.value(), fdoc, f_ele, path_parts, path_index + 1, func);
+ auto& m_ele = mobj.empty() ? mobj : mobj[fragment];
+ return highlight_nested_field(hdoc, it.value(), fdoc, f_ele, mdoc, m_ele, path_parts, path_index + 1, func);
}
} {
return false;
diff --git a/src/collection.cpp b/src/collection.cpp
index 68884fe1..eb05e596 100644
--- a/src/collection.cpp
+++ b/src/collection.cpp
@@ -1491,6 +1491,9 @@ Option<nlohmann::json> Collection::search(const std::string & raw_query,
if(!highlight_items.empty()) {
highlight_res["meta"] = nlohmann::json::object();
+ copy_highlight_doc(highlight_items, document, highlight_res["meta"]);
+ remove_flat_fields(highlight_res["meta"]);
+ highlight_res["meta"].erase("id");
highlight_res["snippet"] = nlohmann::json::object();
copy_highlight_doc(highlight_items, document, highlight_res["snippet"]);
@@ -1568,9 +1571,11 @@ Option<nlohmann::json> Collection::search(const std::string & raw_query,
// remove fields from highlight doc that were not highlighted
if(!hfield_names.empty()) {
+ prune_doc(highlight_res["meta"], hfield_names, tsl::htrie_set<char>(), "");
prune_doc(highlight_res["snippet"], hfield_names, tsl::htrie_set<char>(), "");
prune_doc(highlight_res["full"], h_full_field_names, tsl::htrie_set<char>(), "");
} else {
+ highlight_res["meta"] = nlohmann::json::object();
highlight_res["snippet"] = nlohmann::json::object();
highlight_res["full"] = nlohmann::json::object();
}
@@ -2415,7 +2420,8 @@ void Collection::highlight_result(const std::string& raw_query, const field &sea
}
highlight_nested_field(highlight_doc, highlight_doc, highlight_full_doc, highlight_full_doc,
- path_parts, 0, [&](nlohmann::json& h_obj, nlohmann::json& f_obj) {
+ highlight_meta, highlight_meta, path_parts, 0,
+ [&](nlohmann::json& h_obj, nlohmann::json& f_obj, nlohmann::json& m_obj) {
Match match;
match_index_t match_index(match, 0, 0);
int last_valid_offset_index = -1;
@@ -2437,9 +2443,11 @@ void Collection::highlight_result(const std::string& raw_query, const field &sea
if(!array_highlight.snippets.empty()) {
h_obj = array_highlight.snippets[0];
found_highlight = found_highlight || true;
+
+ m_obj = nlohmann::json::object();
for(auto& token_vec: array_highlight.matched_tokens) {
for(auto& token: token_vec) {
- matched_tokens.insert(token);
+ m_obj["matched_tokens"].push_back(token);
}
}
}
@@ -2450,13 +2458,6 @@ void Collection::highlight_result(const std::string& raw_query, const field &sea
}
});
- if(found_highlight) {
- highlight_meta[search_field.name] = nlohmann::json::object();
- for(auto it = matched_tokens.begin(); it != matched_tokens.end(); ++it) {
- highlight_meta[search_field.name]["matched_tokens"].push_back(it.key());
- }
- }
-
return ;
}
@@ -2570,9 +2571,19 @@ void Collection::highlight_result(const std::string& raw_query, const field &sea
StringUtils::split(search_field.name, parts, ".");
}
+ nlohmann::json* mval = highlight_meta.contains(parts[0]) ? &highlight_meta[parts[0]] : nullptr;
nlohmann::json* hval = highlight_doc.contains(parts[0]) ? &highlight_doc[parts[0]] : nullptr;
nlohmann::json* fval = highlight_full_doc.contains(parts[0]) ? &highlight_full_doc[parts[0]] : nullptr;
+ for(size_t i = 1; mval != nullptr && i < parts.size(); i++) {
+ const auto& part = parts[i];
+ if(mval->contains(part)) {
+ mval = &mval->at(part);
+ } else {
+ mval = nullptr;
+ }
+ }
+
for(size_t i = 1; hval != nullptr && i < parts.size(); i++) {
const auto& part = parts[i];
if(hval->contains(part)) {
@@ -2602,9 +2613,12 @@ void Collection::highlight_result(const std::string& raw_query, const field &sea
}
}
- highlight_meta[search_field.name] = nlohmann::json::object();
- for(auto it = matched_tokens.begin(); it != matched_tokens.end(); ++it) {
- highlight_meta[search_field.name]["matched_tokens"].push_back(it.key());
+ if(mval) {
+ *mval = nlohmann::json::object();
+ mval->emplace("matched_tokens", nlohmann::json::array());
+ for(auto it = matched_tokens.begin(); it != matched_tokens.end(); ++it) {
+ mval->at("matched_tokens").push_back(it.key());
+ }
}
}
diff --git a/test/collection_nested_fields_test.cpp b/test/collection_nested_fields_test.cpp
index 58950b71..69e6ca78 100644
--- a/test/collection_nested_fields_test.cpp
+++ b/test/collection_nested_fields_test.cpp
@@ -786,14 +786,14 @@ TEST_F(CollectionNestedFieldsTest, HighlightShouldHaveMeta) {
ASSERT_EQ("brown", matched_tokens[0]);
ASSERT_EQ("fox", matched_tokens[1]);
- ASSERT_EQ(2, results["hits"][0]["highlight"]["meta"]["details.names"]["matched_tokens"].size());
- matched_tokens = results["hits"][0]["highlight"]["meta"]["details.names"]["matched_tokens"].get<std::vector<std::string>>();
+ ASSERT_EQ(2, results["hits"][0]["highlight"]["meta"]["details"]["names"]["matched_tokens"].size());
+ matched_tokens = results["hits"][0]["highlight"]["meta"]["details"]["names"]["matched_tokens"].get<std::vector<std::string>>();
std::sort(matched_tokens.begin(), matched_tokens.end());
ASSERT_EQ("brown", matched_tokens[0]);
ASSERT_EQ("fox", matched_tokens[1]);
- ASSERT_EQ(1, results["hits"][0]["highlight"]["meta"]["locations.address.street"]["matched_tokens"].size());
- matched_tokens = results["hits"][0]["highlight"]["meta"]["locations.address.street"]["matched_tokens"].get<std::vector<std::string>>();
+ ASSERT_EQ(1, results["hits"][0]["highlight"]["meta"]["locations"][0]["address"]["street"]["matched_tokens"].size());
+ matched_tokens = results["hits"][0]["highlight"]["meta"]["locations"][0]["address"]["street"]["matched_tokens"].get<std::vector<std::string>>();
std::sort(matched_tokens.begin(), matched_tokens.end());
ASSERT_EQ("Brown", matched_tokens[0]);
@@ -804,9 +804,10 @@ TEST_F(CollectionNestedFieldsTest, HighlightShouldHaveMeta) {
20, {}, {}, {}, 0, "<mark>", "</mark>", {}, 1000, true, false, true,
"x").get();
- ASSERT_EQ(2, results["hits"][0]["highlight"].size());
+ ASSERT_EQ(3, results["hits"][0]["highlight"].size());
ASSERT_EQ(0, results["hits"][0]["highlight"]["snippet"].size());
ASSERT_EQ(0, results["hits"][0]["highlight"]["full"].size());
+ ASSERT_EQ(0, results["hits"][0]["highlight"]["meta"].size());
}
TEST_F(CollectionNestedFieldsTest, FieldsWithExplicitSchema) {
diff --git a/test/collection_specific_more_test.cpp b/test/collection_specific_more_test.cpp
index 6cc38afe..8fd8cd24 100644
--- a/test/collection_specific_more_test.cpp
+++ b/test/collection_specific_more_test.cpp
@@ -930,9 +930,9 @@ TEST_F(CollectionSpecificMoreTest, HighlightWithAccentedChars) {
ASSERT_EQ(1, results["hits"][0]["highlight"]["meta"]["title"]["matched_tokens"].size());
ASSERT_EQ("Rāp", results["hits"][0]["highlight"]["meta"]["title"]["matched_tokens"][0]);
- ASSERT_EQ(1, results["hits"][0]["highlight"]["meta"]["companies.title"].size());
- ASSERT_EQ(1, results["hits"][0]["highlight"]["meta"]["companies.title"]["matched_tokens"].size());
- ASSERT_EQ("Rāp", results["hits"][0]["highlight"]["meta"]["companies.title"]["matched_tokens"][0]);
+ ASSERT_EQ(1, results["hits"][0]["highlight"]["meta"]["companies"][0]["title"].size());
+ ASSERT_EQ(1, results["hits"][0]["highlight"]["meta"]["companies"][0]["title"]["matched_tokens"].size());
+ ASSERT_EQ("Rāp", results["hits"][0]["highlight"]["meta"]["companies"][0]["title"]["matched_tokens"][0]);
}
TEST_F(CollectionSpecificMoreTest, FieldWeightNormalization) {