Open Source alternative to Algolia + Pinecone and an Easier-to-Use alternative to ElasticSearch ⚡ 🔍 ✨ Fast, typo tolerant, in-memory fuzzy Search Engine for building delightful search experiences
C++ algoliadatastoreelasticsearchenterprise-searchfacetingfull-text-searchfuzzy-searchgeosearchin-memoryinstantsearchmerchandisingpineconesearchsearch-enginesemantic-searchsimilarity-searchsite-searchsynonymstypo-tolerancevector-search
Object reference field. (#1911)
Harpreet Sangarbca83ef529ed
diff --git a/include/collection.h b/include/collection.hindex 8501c3d1..8ac8bc55 100644--- a/include/collection.h+++ b/include/collection.h@@ -467,7 +467,8 @@ public:static Option<bool> include_references(nlohmann::json& doc, const uint32_t& seq_id, Collection *const collection,const std::map<std::string, reference_filter_result_t>& reference_filter_results,- const std::vector<ref_include_exclude_fields>& ref_include_exclude_fields_vec);+ const std::vector<ref_include_exclude_fields>& ref_include_exclude_fields_vec,+ const nlohmann::json& original_doc);Option<bool> prune_doc_with_lock(nlohmann::json& doc, const tsl::htrie_set<char>& include_names,const tsl::htrie_set<char>& exclude_names,diff --git a/src/collection.cpp b/src/collection.cppindex 7e36814a..53447b38 100644--- a/src/collection.cpp+++ b/src/collection.cpp@@ -5448,6 +5448,11 @@ Option<bool> Collection::prune_ref_doc(nlohmann::json& doc,const tsl::htrie_set<char>& ref_exclude_fields_full,const bool& is_reference_array,const ref_include_exclude_fields& ref_include_exclude) {+ nlohmann::json original_doc;+ if (!ref_include_exclude.nested_join_includes.empty()) {+ original_doc = doc;+ }+auto const& ref_collection_name = ref_include_exclude.collection_name;auto& cm = CollectionManager::get_instance();auto ref_collection = cm.get_collection(ref_collection_name);@@ -5503,7 +5508,7 @@ Option<bool> Collection::prune_ref_doc(nlohmann::json& doc,ref_collection.get(),references.coll_to_references == nullptr ? refs :references.coll_to_references[0],- ref_include_exclude.nested_join_includes);+ ref_include_exclude.nested_join_includes, original_doc);if (!nested_include_exclude_op.ok()) {return nested_include_exclude_op;}@@ -5568,7 +5573,7 @@ Option<bool> Collection::prune_ref_doc(nlohmann::json& doc,ref_collection.get(),references.coll_to_references == nullptr ? refs :references.coll_to_references[i],- ref_include_exclude.nested_join_includes);+ ref_include_exclude.nested_join_includes, original_doc);if (!nested_include_exclude_op.ok()) {return nested_include_exclude_op;}@@ -5580,7 +5585,8 @@ Option<bool> Collection::prune_ref_doc(nlohmann::json& doc,Option<bool> Collection::include_references(nlohmann::json& doc, const uint32_t& seq_id, Collection *const collection,const std::map<std::string, reference_filter_result_t>& reference_filter_results,- const std::vector<ref_include_exclude_fields>& ref_include_exclude_fields_vec) {+ const std::vector<ref_include_exclude_fields>& ref_include_exclude_fields_vec,+ const nlohmann::json& original_doc) {for (auto const& ref_include_exclude: ref_include_exclude_fields_vec) {auto ref_collection_name = ref_include_exclude.collection_name;@@ -5656,13 +5662,26 @@ Option<bool> Collection::include_references(nlohmann::json& doc, const uint32_t&if (collection->object_reference_helper_fields.count(field_name) != 0) {std::vector<std::string> keys;StringUtils::split(field_name, keys, ".");- if (!doc.contains(keys[0])) {- return Option<bool>(400, "Could not find `" + keys[0] +- "` in the document to include the referenced document.");+ auto const& key = keys[0];++ if (!doc.contains(key)) {+ if (!original_doc.contains(key)) {+ return Option<bool>(400, "Could not find `" + key ++ "` key in the document to include the referenced document.");+ }++ // The key is excluded from the doc by the query, inserting empty object(s) so referenced doc can be+ // included in it.+ if (original_doc[key].is_array()) {+ doc[key] = nlohmann::json::array();+ doc[key].insert(doc[key].begin(), original_doc[key].size(), nlohmann::json::object());+ } else {+ doc[key] = nlohmann::json::object();+ }}- if (doc[keys[0]].is_array()) {- for (uint32_t i = 0; i < doc[keys[0]].size(); i++) {+ if (doc[key].is_array()) {+ for (uint32_t i = 0; i < doc[key].size(); i++) {uint32_t ref_doc_id;auto op = collection->get_object_array_related_id(field_name, seq_id, i, ref_doc_id);if (!op.ok()) {@@ -5674,7 +5693,7 @@ Option<bool> Collection::include_references(nlohmann::json& doc, const uint32_t&}reference_filter_result_t result(1, new uint32_t[1]{ref_doc_id});- prune_doc_op = prune_ref_doc(doc[keys[0]][i], result,+ prune_doc_op = prune_ref_doc(doc[key][i], result,ref_include_fields_full, ref_exclude_fields_full,false, ref_include_exclude);if (!prune_doc_op.ok()) {@@ -5688,7 +5707,7 @@ Option<bool> Collection::include_references(nlohmann::json& doc, const uint32_t&continue;}reference_filter_result_t result(ids.size(), &ids[0]);- prune_doc_op = prune_ref_doc(doc[keys[0]], result, ref_include_fields_full, ref_exclude_fields_full,+ prune_doc_op = prune_ref_doc(doc[key], result, ref_include_fields_full, ref_exclude_fields_full,collection->search_schema.at(field_name).is_array(), ref_include_exclude);result.docs = nullptr;}@@ -5764,6 +5783,11 @@ Option<bool> Collection::prune_doc(nlohmann::json& doc,const std::map<std::string, reference_filter_result_t>& reference_filter_results,Collection *const collection, const uint32_t& seq_id,const std::vector<ref_include_exclude_fields>& ref_include_exclude_fields_vec) {+ nlohmann::json original_doc;+ if (!ref_include_exclude_fields_vec.empty()) {+ original_doc = doc;+ }+// doc can only be an objectauto it = doc.begin();while(it != doc.end()) {@@ -5839,7 +5863,8 @@ Option<bool> Collection::prune_doc(nlohmann::json& doc,it++;}- return include_references(doc, seq_id, collection, reference_filter_results, ref_include_exclude_fields_vec);+ return include_references(doc, seq_id, collection, reference_filter_results, ref_include_exclude_fields_vec,+ original_doc);}Option<bool> Collection::validate_alter_payload(nlohmann::json& schema_changes,diff --git a/test/collection_join_test.cpp b/test/collection_join_test.cppindex d2a98e4e..827a07fc 100644--- a/test/collection_join_test.cpp+++ b/test/collection_join_test.cpp@@ -3913,8 +3913,8 @@ TEST_F(CollectionJoinTest, FilterByObjectReferenceField) {auto now_ts = std::chrono::duration_cast<std::chrono::microseconds>(std::chrono::system_clock::now().time_since_epoch()).count();- auto search_op_bool = collectionManager.do_search(req_params, embedded_params, json_res, now_ts);- ASSERT_TRUE(search_op_bool.ok());+ auto search_op = collectionManager.do_search(req_params, embedded_params, json_res, now_ts);+ ASSERT_TRUE(search_op.ok());auto res_obj = nlohmann::json::parse(json_res);ASSERT_EQ(2, res_obj["found"].get<size_t>());@@ -3931,14 +3931,39 @@ TEST_F(CollectionJoinTest, FilterByObjectReferenceField) {ASSERT_EQ("a", res_obj["hits"][1]["document"]["coll_id"]);ASSERT_EQ(0, res_obj["hits"][1]["document"]["object"].size());+ req_params = {+ {"collection", "coll1"},+ {"q", "*"},+ {"include_fields", "$Products(product_id)"},+ {"exclude_fields", "object"}+ };+ search_op = collectionManager.do_search(req_params, embedded_params, json_res, now_ts);+ ASSERT_TRUE(search_op.ok());++ res_obj = nlohmann::json::parse(json_res);+ ASSERT_EQ(2, res_obj["found"].get<size_t>());+ ASSERT_EQ(2, res_obj["hits"].size());+ ASSERT_EQ(3, res_obj["hits"][0]["document"].size());+ ASSERT_EQ("1", res_obj["hits"][0]["document"]["id"]);+ ASSERT_EQ(1, res_obj["hits"][0]["document"].count("object"));+ ASSERT_EQ(1, res_obj["hits"][0]["document"]["object"].size());+ ASSERT_EQ(1, res_obj["hits"][0]["document"]["object"].count("Products"));+ ASSERT_EQ(1, res_obj["hits"][0]["document"]["object"]["Products"].size());+ ASSERT_EQ(1, res_obj["hits"][0]["document"]["object"]["Products"].count("product_id"));+ ASSERT_EQ("product_c", res_obj["hits"][0]["document"]["object"]["Products"]["product_id"]);+ ASSERT_EQ(3, res_obj["hits"][1]["document"].size());+ ASSERT_EQ("0", res_obj["hits"][1]["document"]["id"]);+ ASSERT_EQ(1, res_obj["hits"][1]["document"].count("object"));+ ASSERT_EQ(0, res_obj["hits"][1]["document"]["object"].size());+req_params = {{"collection", "Products"},{"q", "*"},{"filter_by", "$coll1(id: *)"},{"include_fields", "$coll1(coll_id)"}};- search_op_bool = collectionManager.do_search(req_params, embedded_params, json_res, now_ts);- ASSERT_TRUE(search_op_bool.ok());+ search_op = collectionManager.do_search(req_params, embedded_params, json_res, now_ts);+ ASSERT_TRUE(search_op.ok());res_obj = nlohmann::json::parse(json_res);ASSERT_EQ(1, res_obj["found"].get<size_t>());@@ -3988,8 +4013,8 @@ TEST_F(CollectionJoinTest, FilterByObjectReferenceField) {{"q", "*"},{"include_fields", "$Products(product_id)"}};- search_op_bool = collectionManager.do_search(req_params, embedded_params, json_res, now_ts);- ASSERT_TRUE(search_op_bool.ok());+ search_op = collectionManager.do_search(req_params, embedded_params, json_res, now_ts);+ ASSERT_TRUE(search_op.ok());res_obj = nlohmann::json::parse(json_res);ASSERT_EQ(2, res_obj["found"].get<size_t>());@@ -4009,14 +4034,41 @@ TEST_F(CollectionJoinTest, FilterByObjectReferenceField) {ASSERT_EQ("a", res_obj["hits"][1]["document"]["coll_id"]);ASSERT_EQ(0, res_obj["hits"][1]["document"]["object"].size());+ req_params = {+ {"collection", "coll2"},+ {"q", "*"},+ {"include_fields", "$Products(product_id)"},+ {"exclude_fields", "object"}+ };+ search_op = collectionManager.do_search(req_params, embedded_params, json_res, now_ts);+ ASSERT_TRUE(search_op.ok());++ res_obj = nlohmann::json::parse(json_res);+ ASSERT_EQ(2, res_obj["found"].get<size_t>());+ ASSERT_EQ(2, res_obj["hits"].size());+ ASSERT_EQ(3, res_obj["hits"][0]["document"].size());+ ASSERT_EQ("1", res_obj["hits"][0]["document"]["id"]);+ ASSERT_EQ(1, res_obj["hits"][0]["document"].count("object"));+ ASSERT_EQ(1, res_obj["hits"][0]["document"]["object"].size());+ ASSERT_EQ(1, res_obj["hits"][0]["document"]["object"].count("Products"));+ ASSERT_EQ(2, res_obj["hits"][0]["document"]["object"]["Products"].size());+ ASSERT_EQ(1, res_obj["hits"][0]["document"]["object"]["Products"][0].count("product_id"));+ ASSERT_EQ("product_a", res_obj["hits"][0]["document"]["object"]["Products"][0]["product_id"]);+ ASSERT_EQ(1, res_obj["hits"][0]["document"]["object"]["Products"][1].count("product_id"));+ ASSERT_EQ("product_b", res_obj["hits"][0]["document"]["object"]["Products"][1]["product_id"]);+ ASSERT_EQ(3, res_obj["hits"][1]["document"].size());+ ASSERT_EQ("0", res_obj["hits"][1]["document"]["id"]);+ ASSERT_EQ(1, res_obj["hits"][1]["document"].count("object"));+ ASSERT_EQ(0, res_obj["hits"][1]["document"]["object"].size());+req_params = {{"collection", "Products"},{"q", "*"},{"filter_by", "$coll2(id: *)"},{"include_fields", "$coll2(coll_id)"}};- search_op_bool = collectionManager.do_search(req_params, embedded_params, json_res, now_ts);- ASSERT_TRUE(search_op_bool.ok());+ search_op = collectionManager.do_search(req_params, embedded_params, json_res, now_ts);+ ASSERT_TRUE(search_op.ok());res_obj = nlohmann::json::parse(json_res);ASSERT_EQ(2, res_obj["found"].get<size_t>());@@ -4071,8 +4123,8 @@ TEST_F(CollectionJoinTest, FilterByObjectReferenceField) {{"q", "*"},{"include_fields", "$Products(product_id)"}};- search_op_bool = collectionManager.do_search(req_params, embedded_params, json_res, now_ts);- ASSERT_TRUE(search_op_bool.ok());+ search_op = collectionManager.do_search(req_params, embedded_params, json_res, now_ts);+ ASSERT_TRUE(search_op.ok());res_obj = nlohmann::json::parse(json_res);ASSERT_EQ(2, res_obj["found"].get<size_t>());@@ -4098,8 +4150,8 @@ TEST_F(CollectionJoinTest, FilterByObjectReferenceField) {{"filter_by", "$coll3(id: *)"},{"include_fields", "$coll3(coll_id)"}};- search_op_bool = collectionManager.do_search(req_params, embedded_params, json_res, now_ts);- ASSERT_TRUE(search_op_bool.ok());+ search_op = collectionManager.do_search(req_params, embedded_params, json_res, now_ts);+ ASSERT_TRUE(search_op.ok());res_obj = nlohmann::json::parse(json_res);ASSERT_EQ(2, res_obj["found"].get<size_t>());@@ -4206,8 +4258,8 @@ TEST_F(CollectionJoinTest, FilterByObjectReferenceField) {{"q", "*"},{"include_fields", "$Portions(*, strategy:merge)"}};- search_op_bool = collectionManager.do_search(req_params, embedded_params, json_res, now_ts);- ASSERT_TRUE(search_op_bool.ok());+ search_op = collectionManager.do_search(req_params, embedded_params, json_res, now_ts);+ ASSERT_TRUE(search_op.ok());res_obj = nlohmann::json::parse(json_res);ASSERT_EQ(2, res_obj["found"].get<size_t>());@@ -4245,6 +4297,45 @@ TEST_F(CollectionJoinTest, FilterByObjectReferenceField) {ASSERT_EQ("g", res_obj["hits"][1]["document"]["portions"][0].at("unit"));ASSERT_EQ(10 , res_obj["hits"][1]["document"]["portions"][0].at("count"));+ req_params = {+ {"collection", "Foods"},+ {"q", "*"},+ {"include_fields", "$Portions(*, strategy:merge)"},+ {"exclude_fields", "portions"}+ };+ search_op = collectionManager.do_search(req_params, embedded_params, json_res, now_ts);+ ASSERT_TRUE(search_op.ok());++ res_obj = nlohmann::json::parse(json_res);+ ASSERT_EQ(2, res_obj["found"].get<size_t>());+ ASSERT_EQ(2, res_obj["hits"].size());+ ASSERT_EQ(3, res_obj["hits"][0]["document"].size());++ ASSERT_EQ("1", res_obj["hits"][0]["document"]["id"]);+ ASSERT_EQ(1, res_obj["hits"][0]["document"].count("portions"));+ ASSERT_EQ(3, res_obj["hits"][0]["document"]["portions"].size());++ ASSERT_EQ(4, res_obj["hits"][0]["document"]["portions"][0].size());+ ASSERT_EQ("portion_b", res_obj["hits"][0]["document"]["portions"][0].at("portion_id"));+ ASSERT_EQ(1 , res_obj["hits"][0]["document"]["portions"][0].at("quantity"));+ ASSERT_EQ("lt", res_obj["hits"][0]["document"]["portions"][0].at("unit"));++ ASSERT_EQ(0, res_obj["hits"][0]["document"]["portions"][1].size());++ ASSERT_EQ(4, res_obj["hits"][0]["document"]["portions"][2].size());+ ASSERT_EQ("portion_c", res_obj["hits"][0]["document"]["portions"][2].at("portion_id"));+ ASSERT_EQ(500 , res_obj["hits"][0]["document"]["portions"][2].at("quantity"));+ ASSERT_EQ("ml", res_obj["hits"][0]["document"]["portions"][2].at("unit"));++ ASSERT_EQ("0", res_obj["hits"][1]["document"]["id"]);+ ASSERT_EQ(1, res_obj["hits"][1]["document"].count("portions"));+ ASSERT_EQ(1, res_obj["hits"][1]["document"]["portions"].size());++ ASSERT_EQ(4, res_obj["hits"][1]["document"]["portions"][0].size());+ ASSERT_EQ("portion_a", res_obj["hits"][1]["document"]["portions"][0].at("portion_id"));+ ASSERT_EQ(500 , res_obj["hits"][1]["document"]["portions"][0].at("quantity"));+ ASSERT_EQ("g", res_obj["hits"][1]["document"]["portions"][0].at("unit"));+// recreate collection manager to ensure that it initializes `object_reference_helper_fields` correctly.collectionManager.dispose();delete store;@@ -4263,8 +4354,8 @@ TEST_F(CollectionJoinTest, FilterByObjectReferenceField) {{"q", "*"},{"include_fields", "$Portions(*, strategy:merge)"}};- search_op_bool = collectionManager.do_search(req_params, embedded_params, json_res, now_ts);- ASSERT_TRUE(search_op_bool.ok());+ search_op = collectionManager.do_search(req_params, embedded_params, json_res, now_ts);+ ASSERT_TRUE(search_op.ok());res_obj = nlohmann::json::parse(json_res);ASSERT_EQ(2, res_obj["found"].get<size_t>());@@ -4321,8 +4412,8 @@ TEST_F(CollectionJoinTest, FilterByObjectReferenceField) {{"q", "*"},{"include_fields", "$Portions(*, strategy:merge)"}};- search_op_bool = collectionManager.do_search(req_params, embedded_params, json_res, now_ts);- ASSERT_TRUE(search_op_bool.ok());+ search_op = collectionManager.do_search(req_params, embedded_params, json_res, now_ts);+ ASSERT_TRUE(search_op.ok());res_obj = nlohmann::json::parse(json_res);ASSERT_EQ(2, res_obj["found"].get<size_t>());