Skip to content

Open Source alternative to Algolia + Pinecone and an Easier-to-Use alternative to ElasticSearch ⚡ 🔍 ✨ Fast, typo tolerant, in-memory fuzzy Search Engine for building delightful search experiences

Fix highlight meta structure.

Kishore Nallan5292956718a1
diff --git a/include/collection.h b/include/collection.h
index 297952ab..298dfd12 100644
--- a/include/collection.h
+++ b/include/collection.h
@@ -247,7 +247,8 @@ private:
static bool highlight_nested_field(const nlohmann::json& hdoc, nlohmann::json& hobj,
const nlohmann::json& fdoc, nlohmann::json& fobj,
const nlohmann::json& mdoc, nlohmann::json& mobj,
- std::vector<std::string>& path_parts, size_t path_index, T func);
+ std::vector<std::string>& path_parts, size_t path_index,
+ int arr_index, nlohmann::json& parent_mobj, T func);
static Option<bool> resolve_field_type(field& new_field,
nlohmann::detail::iter_impl<nlohmann::basic_json<>>& kv,
@@ -502,14 +503,16 @@ template<class T>
bool Collection::highlight_nested_field(const nlohmann::json& hdoc, nlohmann::json& hobj,
const nlohmann::json& fdoc, nlohmann::json& fobj,
const nlohmann::json& mdoc, nlohmann::json& mobj,
- std::vector<std::string>& path_parts, size_t path_index, T func) {
+ std::vector<std::string>& path_parts, size_t path_index,
+ int arr_index, nlohmann::json& parent_mobj,
+ T func) {
if(path_index == path_parts.size()) {
// end of path: guaranteed to be a string
if(!hobj.is_string()) {
return false;
}
- func(hobj, fobj, mobj);
+ func(hobj, fobj, mobj, arr_index, parent_mobj);
}
const std::string& fragment = path_parts[path_index];
@@ -518,18 +521,24 @@ bool Collection::highlight_nested_field(const nlohmann::json& hdoc, nlohmann::js
if(it != hobj.end()) {
if(it.value().is_array()) {
bool resolved = false;
+ auto& parent_mobj = mobj.empty() ? mobj : mobj[fragment];
+ nlohmann::json empty_obj;
+
for(size_t i = 0; i < it.value().size(); i++) {
auto& h_ele = it.value().at(i);
auto& f_ele = fobj.empty() ? fobj : fobj[fragment][i];
- auto& m_ele = mobj.empty() ? mobj : mobj[fragment][i];
+ // if value is a primitive, we don't want the array object getting destroyed
+ auto& m_ele = h_ele.is_object() && !mobj.empty() ? mobj[fragment][i] : empty_obj;
resolved |= highlight_nested_field(hdoc, h_ele, fdoc, f_ele, mdoc, m_ele,
- path_parts, path_index + 1, func);
+ path_parts, path_index + 1, i, parent_mobj, func);
}
return resolved;
} else {
auto& f_ele = fobj.empty() ? fobj : fobj[fragment];
auto& m_ele = mobj.empty() ? mobj : mobj[fragment];
- return highlight_nested_field(hdoc, it.value(), fdoc, f_ele, mdoc, m_ele, path_parts, path_index + 1, func);
+ auto& parent_mobj = mobj.empty() ? mobj : mobj[fragment];
+ return highlight_nested_field(hdoc, it.value(), fdoc, f_ele, mdoc, m_ele, path_parts, path_index + 1,
+ -1, parent_mobj, func);
}
} {
return false;
diff --git a/src/collection.cpp b/src/collection.cpp
index eb05e596..dec6f33e 100644
--- a/src/collection.cpp
+++ b/src/collection.cpp
@@ -2420,8 +2420,9 @@ void Collection::highlight_result(const std::string& raw_query, const field &sea
}
highlight_nested_field(highlight_doc, highlight_doc, highlight_full_doc, highlight_full_doc,
- highlight_meta, highlight_meta, path_parts, 0,
- [&](nlohmann::json& h_obj, nlohmann::json& f_obj, nlohmann::json& m_obj) {
+ highlight_meta, highlight_meta, path_parts, 0, -1, highlight_meta,
+ [&](nlohmann::json& h_obj, nlohmann::json& f_obj, nlohmann::json& m_obj,
+ int arr_index, nlohmann::json& parent_mobj) {
Match match;
match_index_t match_index(match, 0, 0);
int last_valid_offset_index = -1;
@@ -2444,12 +2445,20 @@ void Collection::highlight_result(const std::string& raw_query, const field &sea
h_obj = array_highlight.snippets[0];
found_highlight = found_highlight || true;
- m_obj = nlohmann::json::object();
+ nlohmann::json& ref_meta_obj = (arr_index >= 0) ? parent_mobj : m_obj;
+ if(!ref_meta_obj.is_object() || !ref_meta_obj.contains("matched_tokens")) {
+ ref_meta_obj = nlohmann::json::object();
+ }
+
for(auto& token_vec: array_highlight.matched_tokens) {
for(auto& token: token_vec) {
- m_obj["matched_tokens"].push_back(token);
+ ref_meta_obj["matched_tokens"].push_back(token);
}
}
+
+ if(arr_index >= 0) {
+ ref_meta_obj["matched_indices"].push_back(arr_index);
+ }
}
if(highlight_fully && !array_highlight.values.empty()) {
@@ -2619,6 +2628,13 @@ void Collection::highlight_result(const std::string& raw_query, const field &sea
for(auto it = matched_tokens.begin(); it != matched_tokens.end(); ++it) {
mval->at("matched_tokens").push_back(it.key());
}
+
+ if(hval->is_array()) {
+ mval->emplace("matched_indices", nlohmann::json::array());
+ for(auto highlight_index: highlight.indices) {
+ mval->at("matched_indices").push_back(highlight_index);
+ }
+ }
}
}
diff --git a/test/collection_nested_fields_test.cpp b/test/collection_nested_fields_test.cpp
index e1dc4c61..ad30313f 100644
--- a/test/collection_nested_fields_test.cpp
+++ b/test/collection_nested_fields_test.cpp
@@ -620,8 +620,42 @@ TEST_F(CollectionNestedFieldsTest, HighlightNestedFieldFully) {
]
})"_json;
+ auto highlight_meta_doc = R"({
+ "locations":[
+ {
+ "address":{
+ "city":"Beaverton",
+ "products":{
+ "matched_indices":[
+ 0
+ ],
+ "matched_tokens":[
+ "shoes"
+ ]
+ },
+ "street":"One Bowerman Drive"
+ }
+ },
+ {
+ "address":{
+ "city":"Thornhill",
+ "products":{
+ "matched_indices":[
+ 1
+ ],
+ "matched_tokens":[
+ "shoes"
+ ]
+ },
+ "street":"175 Commerce Drive"
+ }
+ }
+ ]
+ })"_json;
+
ASSERT_EQ(highlight_full_doc.dump(), results["hits"][0]["highlight"]["full"].dump());
ASSERT_EQ(highlight_full_doc.dump(), results["hits"][0]["highlight"]["snippet"].dump());
+ ASSERT_EQ(highlight_meta_doc.dump(), results["hits"][0]["highlight"]["meta"].dump());
// full highlighting only one of the 3 highlight fields
results = coll1->search("drive", {"company.names", "company_names", "locations.address"}, "", {}, sort_fields, {0}, 10, 1,
@@ -778,7 +812,7 @@ TEST_F(CollectionNestedFieldsTest, HighlightShouldHaveMeta) {
spp::sparse_hash_set<std::string>(), 10, "", 30, 4, "locations.address").get();
ASSERT_EQ(3, results["hits"][0]["highlight"]["meta"].size());
- ASSERT_EQ(1, results["hits"][0]["highlight"]["meta"]["company_names"].size());
+ ASSERT_EQ(2, results["hits"][0]["highlight"]["meta"]["company_names"].size());
ASSERT_EQ(2, results["hits"][0]["highlight"]["meta"]["company_names"]["matched_tokens"].size());
std::vector<std::string> matched_tokens = results["hits"][0]["highlight"]["meta"]["company_names"]["matched_tokens"].get<std::vector<std::string>>();
@@ -786,6 +820,11 @@ TEST_F(CollectionNestedFieldsTest, HighlightShouldHaveMeta) {
ASSERT_EQ("brown", matched_tokens[0]);
ASSERT_EQ("fox", matched_tokens[1]);
+ ASSERT_EQ(2, results["hits"][0]["highlight"]["meta"]["company_names"]["matched_indices"].size());
+ std::vector<size_t> matched_indices = results["hits"][0]["highlight"]["meta"]["company_names"]["matched_indices"].get<std::vector<size_t>>();
+ ASSERT_EQ(0, matched_indices[0]);
+ ASSERT_EQ(1, matched_indices[1]);
+
ASSERT_EQ(2, results["hits"][0]["highlight"]["meta"]["details"]["names"]["matched_tokens"].size());
matched_tokens = results["hits"][0]["highlight"]["meta"]["details"]["names"]["matched_tokens"].get<std::vector<std::string>>();
std::sort(matched_tokens.begin(), matched_tokens.end());
@@ -797,6 +836,20 @@ TEST_F(CollectionNestedFieldsTest, HighlightShouldHaveMeta) {
std::sort(matched_tokens.begin(), matched_tokens.end());
ASSERT_EQ("Brown", matched_tokens[0]);
+ // partial matched index match
+
+ results = coll1->search("fast", {"company_names"},
+ "", {}, sort_fields, {0}, 10, 1,
+ token_ordering::FREQUENCY, {true}, 10, spp::sparse_hash_set<std::string>(),
+ spp::sparse_hash_set<std::string>(), 10, "", 30, 4, "locations.address").get();
+
+ ASSERT_EQ(1, results["hits"][0]["highlight"]["meta"].size());
+ ASSERT_EQ(2, results["hits"][0]["highlight"]["meta"]["company_names"].size());
+
+ ASSERT_EQ(1, results["hits"][0]["highlight"]["meta"]["company_names"]["matched_indices"].size());
+ matched_indices = results["hits"][0]["highlight"]["meta"]["company_names"]["matched_indices"].get<std::vector<size_t>>();
+ ASSERT_EQ(1, matched_indices[0]);
+
// when no highlighting is enabled by setting unknown field for highlighting
results = coll1->search("brown fox", {"company_names", "details", "locations"}, "", {}, sort_fields, {0}, 10, 1,
token_ordering::FREQUENCY, {true}, 10, spp::sparse_hash_set<std::string>(),
@@ -1608,3 +1661,54 @@ TEST_F(CollectionNestedFieldsTest, NestedSchemaAutoAndFacet) {
ASSERT_TRUE(coll1->get_schema()["schools.name"].optional);
}
+
+TEST_F(CollectionNestedFieldsTest, HighlightArrayInsideArrayOfObj) {
+ nlohmann::json schema = R"({
+ "name": "coll1",
+ "enable_nested_fields": true,
+ "fields": [
+ {"name": "studies", "type": "auto"}
+ ]
+ })"_json;
+
+ auto op = collectionManager.create_collection(schema);
+ ASSERT_TRUE(op.ok());
+ Collection* coll1 = op.get();
+
+ auto doc1 = R"({
+ "id": "0",
+ "studies": [
+ {"name": "College 1", "tags": ["foo", "bar"]},
+ {"name": "College 1", "tags": ["alpha", "beta"]}
+ ]
+ })"_json;
+
+ ASSERT_TRUE(coll1->add(doc1.dump(), CREATE).ok());
+
+ auto results = coll1->search("beta", {"studies"}, "", {}, {}, {0}, 10, 1, FREQUENCY, {false}).get();
+ ASSERT_EQ(1, results["found"].get<size_t>());
+ ASSERT_EQ(1, results["hits"].size());
+
+ nlohmann::json highlight_meta_doc = R"({
+ "studies":[
+ {
+ "tags":[
+ "foo",
+ "bar"
+ ]
+ },
+ {
+ "tags":{
+ "matched_indices":[
+ 1
+ ],
+ "matched_tokens":[
+ "beta"
+ ]
+ }
+ }
+ ]
+ })"_json;
+
+ ASSERT_EQ(highlight_meta_doc.dump(), results["hits"][0]["highlight"]["meta"].dump());
+}