Open Source alternative to Algolia + Pinecone and an Easier-to-Use alternative to ElasticSearch ⚡ 🔍 ✨ Fast, typo tolerant, in-memory fuzzy Search Engine for building delightful search experiences
C++ algoliadatastoreelasticsearchenterprise-searchfacetingfull-text-searchfuzzy-searchgeosearchin-memoryinstantsearchmerchandisingpineconesearchsearch-enginesemantic-searchsimilarity-searchsite-searchsynonymstypo-tolerancevector-search
Add support for `nest` and `merge` reference include strategy.
Harpreet Sangarce6a5c2f3ff5
diff --git a/include/collection.h b/include/collection.hindex 792e1621..33a73d3c 100644--- a/include/collection.h+++ b/include/collection.h@@ -400,12 +400,14 @@ public:static void remove_flat_fields(nlohmann::json& document);static Option<bool> add_reference_fields(nlohmann::json& doc,+ const std::string& ref_collection_name,Collection *const ref_collection,const std::string& alias,const reference_filter_result_t& references,const tsl::htrie_set<char>& ref_include_fields_full,const tsl::htrie_set<char>& ref_exclude_fields_full,- const std::string& error_prefix, const bool& is_reference_array);+ const std::string& error_prefix, const bool& is_reference_array,+ const bool& nest_ref_doc);static Option<bool> prune_doc(nlohmann::json& doc, const tsl::htrie_set<char>& include_names,const tsl::htrie_set<char>& exclude_names, const std::string& parent_name = "",diff --git a/include/field.h b/include/field.hindex 0e55142f..274f59cd 100644--- a/include/field.h+++ b/include/field.h@@ -496,9 +496,16 @@ namespace sort_field_const {static const std::string vector_query = "_vector_query";}+namespace ref_include {+ static const std::string merge = "merge";+ static const std::string nest = "nest";+}+struct ref_include_fields {- std::string expression;+ std::string collection_name;+ std::string fields;std::string alias;+ bool nest_ref_doc = false;};struct hnsw_index_t;diff --git a/src/collection.cpp b/src/collection.cppindex 3464642e..ed423356 100644--- a/src/collection.cpp+++ b/src/collection.cpp@@ -4440,12 +4440,14 @@ void Collection::remove_flat_fields(nlohmann::json& document) {}Option<bool> Collection::add_reference_fields(nlohmann::json& doc,+ const std::string& ref_collection_name,Collection *const ref_collection,const std::string& alias,const reference_filter_result_t& references,const tsl::htrie_set<char>& ref_include_fields_full,const tsl::htrie_set<char>& ref_exclude_fields_full,- const std::string& error_prefix, const bool& is_reference_array) {+ const std::string& error_prefix, const bool& is_reference_array,+ const bool& nest_ref_doc) {// One-to-one relation.if (!is_reference_array && references.count == 1) {auto ref_doc_seq_id = references.docs[0];@@ -4463,15 +4465,19 @@ Option<bool> Collection::add_reference_fields(nlohmann::json& doc,return Option<bool>(prune_op.code(), error_prefix + prune_op.error());}- if (!alias.empty()) {- auto temp_doc = ref_doc;- ref_doc.clear();- for (const auto &item: temp_doc.items()) {- ref_doc[alias + item.key()] = item.value();+ if (nest_ref_doc && !ref_doc.empty()) {+ auto field_name = alias.empty() ? ref_collection_name : alias;+ doc[field_name] = ref_doc;+ } else {+ if (!alias.empty()) {+ auto temp_doc = ref_doc;+ ref_doc.clear();+ for (const auto &item: temp_doc.items()) {+ ref_doc[alias + item.key()] = item.value();+ }}+ doc.update(ref_doc);}-- doc.update(ref_doc);return Option<bool>(true);}@@ -4492,17 +4498,21 @@ Option<bool> Collection::add_reference_fields(nlohmann::json& doc,return Option<bool>(prune_op.code(), error_prefix + prune_op.error());}- if (!alias.empty()) {- auto temp_doc = ref_doc;- ref_doc.clear();- for (const auto &item: temp_doc.items()) {- ref_doc[alias + item.key()] = item.value();+ if (nest_ref_doc && !ref_doc.empty()) {+ auto field_name = alias.empty() ? ref_collection_name : alias;+ doc[field_name] += ref_doc;+ } else {+ if (!alias.empty()) {+ auto temp_doc = ref_doc;+ ref_doc.clear();+ for (const auto &item: temp_doc.items()) {+ ref_doc[alias + item.key()] = item.value();+ }+ }+ for (auto ref_doc_it = ref_doc.begin(); ref_doc_it != ref_doc.end(); ref_doc_it++) {+ // Add the values of ref_doc as JSON array into doc.+ doc[ref_doc_it.key()] += ref_doc_it.value();}- }-- for (auto ref_doc_it = ref_doc.begin(); ref_doc_it != ref_doc.end(); ref_doc_it++) {- // Add the values of ref_doc as JSON array into doc.- doc[ref_doc_it.key()] += ref_doc_it.value();}}@@ -4592,11 +4602,7 @@ Option<bool> Collection::prune_doc(nlohmann::json& doc,}for (auto const& ref_include: ref_includes) {- auto const& ref = ref_include.expression;- size_t parenthesis_index = ref.find('(');-- auto ref_collection_name = ref.substr(1, parenthesis_index - 1);- auto reference_fields = ref.substr(parenthesis_index + 1, ref.size() - parenthesis_index - 2);+ auto const& ref_collection_name = ref_include.collection_name;auto& cm = CollectionManager::get_instance();auto ref_collection = cm.get_collection(ref_collection_name);@@ -4631,12 +4637,12 @@ Option<bool> Collection::prune_doc(nlohmann::json& doc,}std::vector<std::string> ref_include_fields_vec, ref_exclude_fields_vec;- StringUtils::split(reference_fields, ref_include_fields_vec, ",");+ StringUtils::split(ref_include.fields, ref_include_fields_vec, ",");auto exclude_reference_it = exclude_names.equal_prefix_range("$" + ref_collection_name);if (exclude_reference_it.first != exclude_reference_it.second) {auto ref_exclude = exclude_reference_it.first.key();- parenthesis_index = ref_exclude.find('(');- reference_fields = ref_exclude.substr(parenthesis_index + 1, ref_exclude.size() - parenthesis_index - 2);+ auto parenthesis_index = ref_exclude.find('(');+ auto reference_fields = ref_exclude.substr(parenthesis_index + 1, ref_exclude.size() - parenthesis_index - 2);StringUtils::split(reference_fields, ref_exclude_fields_vec, ",");}@@ -4664,10 +4670,12 @@ Option<bool> Collection::prune_doc(nlohmann::json& doc,if (ref_collection->search_schema.count(field_name) == 0) {continue;}- add_reference_fields_op = add_reference_fields(doc, ref_collection.get(), ref_include.alias,+ add_reference_fields_op = add_reference_fields(doc, ref_include.collection_name,+ ref_collection.get(), ref_include.alias,reference_filter_results.at(ref_collection_name),ref_include_fields_full, ref_exclude_fields_full, error_prefix,- ref_collection->get_schema().at(field_name).is_array());+ ref_collection->get_schema().at(field_name).is_array(),+ ref_include.nest_ref_doc);} else if (doc_has_reference) {auto get_reference_field_op = ref_collection->get_referenced_in_field_with_lock(collection->name);if (!get_reference_field_op.ok()) {@@ -4687,9 +4695,11 @@ Option<bool> Collection::prune_doc(nlohmann::json& doc,result.count = ids.size();result.docs = &ids[0];- add_reference_fields_op = add_reference_fields(doc, ref_collection.get(), ref_include.alias, result,+ add_reference_fields_op = add_reference_fields(doc, ref_include.collection_name,+ ref_collection.get(), ref_include.alias, result,ref_include_fields_full, ref_exclude_fields_full, error_prefix,- collection->search_schema.at(field_name).is_array());+ collection->search_schema.at(field_name).is_array(),+ ref_include.nest_ref_doc);result.docs = nullptr;} else if (joined_coll_has_reference) {auto joined_collection = cm.get_collection(joined_coll_having_reference);@@ -4720,9 +4730,11 @@ Option<bool> Collection::prune_doc(nlohmann::json& doc,reference_filter_result_t result;result.count = ids.size();result.docs = &ids[0];- add_reference_fields_op = add_reference_fields(doc, ref_collection.get(), ref_include.alias, result,+ add_reference_fields_op = add_reference_fields(doc, ref_include.collection_name,+ ref_collection.get(), ref_include.alias, result,ref_include_fields_full, ref_exclude_fields_full, error_prefix,- joined_collection->get_schema().at(reference_field_name).is_array());+ joined_collection->get_schema().at(reference_field_name).is_array(),+ ref_include.nest_ref_doc);result.docs = nullptr;}diff --git a/src/collection_manager.cpp b/src/collection_manager.cppindex 7c44ac37..8cec4c92 100644--- a/src/collection_manager.cpp+++ b/src/collection_manager.cpp@@ -967,14 +967,30 @@ void initialize_ref_include_fields_vec(const std::string& filter_query, std::veccontinue;}+ // Format: $ref_collection_name(field_1, field_2: include_strategy) as ref_aliasauto as_pos = include_field_exp.find(" as ");- auto ref_include = include_field_exp.substr(0, as_pos),- alias = (as_pos == std::string::npos) ? "" :+ auto ref_include = include_field_exp.substr(0, as_pos);+ auto alias = (as_pos == std::string::npos) ? "" :include_field_exp.substr(as_pos + 4, include_field_exp.size() - (as_pos + 4));- // For an alias `foo`, we need append `foo.` to all the top level keys of reference doc.- ref_include_fields_vec.emplace_back(ref_include_fields{ref_include, alias.empty() ? alias :- StringUtils::trim(alias) + "."});+ auto parenthesis_index = ref_include.find('(');+ auto ref_collection_name = ref_include.substr(1, parenthesis_index - 1);+ auto ref_fields = ref_include.substr(parenthesis_index + 1, ref_include.size() - parenthesis_index - 2);++ auto nest_ref_doc = true;+ auto colon_pos = ref_fields.find(':');+ if (colon_pos != std::string::npos) {+ auto include_strategy = ref_fields.substr(colon_pos + 1, ref_fields.size() - colon_pos - 1);+ StringUtils::trim(include_strategy);+ nest_ref_doc = include_strategy == ref_include::nest;+ ref_fields = ref_fields.substr(0, colon_pos);+ }++ // For an alias `foo`,+ // In case of "merge" reference doc, we need append `foo.` to all the top level keys of reference doc.+ // In case of "nest" reference doc, `foo` becomes the key with reference doc as value.+ auto ref_alias = !alias.empty() ? (StringUtils::trim(alias) + (nest_ref_doc ? "" : ".")) : "";+ ref_include_fields_vec.emplace_back(ref_include_fields{ref_collection_name, ref_fields, ref_alias, nest_ref_doc});auto open_paren_pos = include_field_exp.find('(');if (open_paren_pos == std::string::npos) {@@ -993,7 +1009,7 @@ void initialize_ref_include_fields_vec(const std::string& filter_query, std::vec// Get all the fields of the referenced collection in the filter but not mentioned in include_fields.for (const auto &reference_collection_name: reference_collection_names) {- ref_include_fields_vec.emplace_back(ref_include_fields{"$" + reference_collection_name + "(*)", ""});+ ref_include_fields_vec.emplace_back(ref_include_fields{reference_collection_name, "", "", true});}// Since no field of the collection is mentioned in include_fields, get all the fields.diff --git a/test/collection_join_test.cpp b/test/collection_join_test.cppindex 0dedeea4..77d35afa 100644--- a/test/collection_join_test.cpp+++ b/test/collection_join_test.cpp@@ -915,6 +915,7 @@ TEST_F(CollectionJoinTest, FilterByReference_SingleMatch) {{"q", "Dan"},{"query_by", "customer_name"},{"filter_by", "$Products(rating:>3)"},+ {"include_fields", "$Products(*:merge)"},};search_op_bool = collectionManager.do_search(req_params, embedded_params, json_res, now_ts);@@ -930,6 +931,7 @@ TEST_F(CollectionJoinTest, FilterByReference_SingleMatch) {{"q", "Dan"},{"query_by", "customer_name"},{"filter_by", "$Products(id:*) && product_price:>100"},+ {"include_fields", "$Products(*:merge)"},};search_op_bool = collectionManager.do_search(req_params, embedded_params, json_res, now_ts);@@ -1678,24 +1680,55 @@ TEST_F(CollectionJoinTest, IncludeExcludeFieldsByReference) {ASSERT_EQ(1, res_obj["found"].get<size_t>());ASSERT_EQ(1, res_obj["hits"].size());// No fields are mentioned in `include_fields`, should include all fields of Products and Customers by default.- ASSERT_EQ(10, res_obj["hits"][0]["document"].size());+ ASSERT_EQ(7, res_obj["hits"][0]["document"].size());ASSERT_EQ(1, res_obj["hits"][0]["document"].count("id"));ASSERT_EQ(1, res_obj["hits"][0]["document"].count("product_id"));ASSERT_EQ(1, res_obj["hits"][0]["document"].count("product_name"));ASSERT_EQ(1, res_obj["hits"][0]["document"].count("product_description"));ASSERT_EQ(1, res_obj["hits"][0]["document"].count("embedding"));ASSERT_EQ(1, res_obj["hits"][0]["document"].count("rating"));- ASSERT_EQ(1, res_obj["hits"][0]["document"].count("customer_id"));- ASSERT_EQ(1, res_obj["hits"][0]["document"].count("customer_name"));- ASSERT_EQ(1, res_obj["hits"][0]["document"].count("product_price"));- ASSERT_EQ(1, res_obj["hits"][0]["document"].count("product_id_sequence_id"));+ // Default strategy of reference includes is nest. No alias was provided, collection name becomes the field name.+ ASSERT_EQ(6, res_obj["hits"][0]["document"]["Customers"].size());+ ASSERT_EQ(1, res_obj["hits"][0]["document"]["Customers"].count("customer_id"));+ ASSERT_EQ(1, res_obj["hits"][0]["document"]["Customers"].count("customer_name"));+ ASSERT_EQ(1, res_obj["hits"][0]["document"]["Customers"].count("id"));+ ASSERT_EQ(1, res_obj["hits"][0]["document"]["Customers"].count("product_id"));+ ASSERT_EQ(1, res_obj["hits"][0]["document"]["Customers"].count("product_id_sequence_id"));+ ASSERT_EQ(1, res_obj["hits"][0]["document"]["Customers"].count("product_price"));++ req_params = {+ {"collection", "Products"},+ {"q", "*"},+ {"query_by", "product_name"},+ {"filter_by", "$Customers(customer_id:=customer_a && product_price:<100)"},+ {"include_fields", "*, $Customers(*:merge) as Customers"}+ };+ search_op = collectionManager.do_search(req_params, embedded_params, json_res, now_ts);+ ASSERT_TRUE(search_op.ok());++ res_obj = nlohmann::json::parse(json_res);+ ASSERT_EQ(1, res_obj["found"].get<size_t>());+ ASSERT_EQ(1, res_obj["hits"].size());+ ASSERT_EQ(12, res_obj["hits"][0]["document"].size());+ ASSERT_EQ(1, res_obj["hits"][0]["document"].count("id"));+ ASSERT_EQ(1, res_obj["hits"][0]["document"].count("product_id"));+ ASSERT_EQ(1, res_obj["hits"][0]["document"].count("product_name"));+ ASSERT_EQ(1, res_obj["hits"][0]["document"].count("product_description"));+ ASSERT_EQ(1, res_obj["hits"][0]["document"].count("embedding"));+ ASSERT_EQ(1, res_obj["hits"][0]["document"].count("rating"));+ ASSERT_EQ(1, res_obj["hits"][0]["document"].count("Customers.customer_id"));+ ASSERT_EQ(1, res_obj["hits"][0]["document"].count("Customers.customer_name"));+ ASSERT_EQ(1, res_obj["hits"][0]["document"].count("Customers.id"));+ ASSERT_EQ(1, res_obj["hits"][0]["document"].count("Customers.product_id"));+ ASSERT_EQ(1, res_obj["hits"][0]["document"].count("Customers.product_id_sequence_id"));+ ASSERT_EQ(1, res_obj["hits"][0]["document"].count("Customers.product_price"));req_params = {{"collection", "Products"},{"q", "*"},{"query_by", "product_name"},{"filter_by", "$Customers(customer_id:=customer_a && product_price:<100)"},- {"include_fields", "$Customers(bar)"}+ {"include_fields", "$Customers(bar:merge)"}};search_op = collectionManager.do_search(req_params, embedded_params, json_res, now_ts);ASSERT_TRUE(search_op.ok());@@ -1717,7 +1750,7 @@ TEST_F(CollectionJoinTest, IncludeExcludeFieldsByReference) {{"q", "*"},{"query_by", "product_name"},{"filter_by", "$Customers(customer_id:=customer_a && product_price:<100)"},- {"include_fields", "$Customers(product_price)"}+ {"include_fields", "$Customers(product_price:merge)"}};search_op = collectionManager.do_search(req_params, embedded_params, json_res, now_ts);ASSERT_TRUE(search_op.ok());@@ -1734,7 +1767,7 @@ TEST_F(CollectionJoinTest, IncludeExcludeFieldsByReference) {{"q", "*"},{"query_by", "product_name"},{"filter_by", "$Customers(customer_id:=customer_a && product_price:<100)"},- {"include_fields", "$Customers(product_price, customer_id)"}+ {"include_fields", "$Customers(product_price, customer_id:merge)"}};search_op = collectionManager.do_search(req_params, embedded_params, json_res, now_ts);ASSERT_TRUE(search_op.ok());@@ -1753,7 +1786,7 @@ TEST_F(CollectionJoinTest, IncludeExcludeFieldsByReference) {{"q", "*"},{"query_by", "product_name"},{"filter_by", "$Customers(customer_id:=customer_a && product_price:<100)"},- {"include_fields", "*, $Customers(product_price, customer_id)"}+ {"include_fields", "*, $Customers(product_price, customer_id:merge)"}};search_op = collectionManager.do_search(req_params, embedded_params, json_res, now_ts);ASSERT_TRUE(search_op.ok());@@ -1769,7 +1802,7 @@ TEST_F(CollectionJoinTest, IncludeExcludeFieldsByReference) {{"q", "*"},{"query_by", "product_name"},{"filter_by", "$Customers(customer_id:=customer_a && product_price:<100)"},- {"include_fields", "$Customers(product*)"}+ {"include_fields", "$Customers(product*:merge)"}};search_op = collectionManager.do_search(req_params, embedded_params, json_res, now_ts);ASSERT_TRUE(search_op.ok());@@ -1787,7 +1820,7 @@ TEST_F(CollectionJoinTest, IncludeExcludeFieldsByReference) {{"q", "s"},{"query_by", "product_name"},{"filter_by", "$Customers(customer_id:=customer_a && product_price:<100)"},- {"include_fields", "$Customers(product*)"},+ {"include_fields", "$Customers(product*:merge)"},{"exclude_fields", "$Customers(product_id_sequence_id)"}};search_op = collectionManager.do_search(req_params, embedded_params, json_res, now_ts);@@ -1848,7 +1881,7 @@ TEST_F(CollectionJoinTest, IncludeExcludeFieldsByReference) {{"q", "*"},{"query_by", "product_name"},{"filter_by", "product_name:soap && $Customers(product_price:>100)"},- {"include_fields", "product_name, $Customers(product_price)"},+ {"include_fields", "product_name, $Customers(product_price:merge)"},{"exclude_fields", ""}};search_op = collectionManager.do_search(req_params, embedded_params, json_res, now_ts);@@ -1869,7 +1902,7 @@ TEST_F(CollectionJoinTest, IncludeExcludeFieldsByReference) {{"q", "soap"},{"query_by", "product_name"},{"filter_by", "$Customers(product_price: >0)"},- {"include_fields", "product_name, $Customers(customer_name, product_price)"},+ {"include_fields", "product_name, $Customers(customer_name, product_price:merge)"},{"exclude_fields", ""}};search_op = collectionManager.do_search(req_params, embedded_params, json_res, now_ts);@@ -1894,7 +1927,7 @@ TEST_F(CollectionJoinTest, IncludeExcludeFieldsByReference) {{"q", "natural products"},{"query_by", "embedding"},{"filter_by", "$Customers(customer_id:=customer_a && product_price:<100)"},- {"include_fields", "product_name, $Customers(product_price)"},+ {"include_fields", "product_name, $Customers(product_price:merge)"},{"exclude_fields", ""}};search_op = collectionManager.do_search(req_params, embedded_params, json_res, now_ts);@@ -1924,7 +1957,7 @@ TEST_F(CollectionJoinTest, IncludeExcludeFieldsByReference) {{"q", "*"},{"vector_query", "embedding:(" + vec_string + ", flat_search_cutoff: 0)"},{"filter_by", "$Customers(customer_id:=customer_a && product_price:<100)"},- {"include_fields", "product_name, $Customers(product_price)"},+ {"include_fields", "product_name, $Customers(product_price : merge)"},{"exclude_fields", ""}};search_op = collectionManager.do_search(req_params, embedded_params, json_res, now_ts);@@ -1944,7 +1977,7 @@ TEST_F(CollectionJoinTest, IncludeExcludeFieldsByReference) {{"q", "soap"},{"query_by", "product_name, embedding"},{"filter_by", "$Customers(customer_id:=customer_a && product_price:<100)"},- {"include_fields", "product_name, $Customers(product_price)"},+ {"include_fields", "product_name, $Customers(product_price: merge)"},{"exclude_fields", ""}};search_op = collectionManager.do_search(req_params, embedded_params, json_res, now_ts);@@ -1966,7 +1999,7 @@ TEST_F(CollectionJoinTest, IncludeExcludeFieldsByReference) {{"q", "natural products"},{"query_by", "product_name, embedding"},{"filter_by", "$Customers(customer_id:=customer_a && product_price:<100)"},- {"include_fields", "product_name, $Customers(product_price)"},+ {"include_fields", "product_name, $Customers(product_price :merge)"},{"exclude_fields", ""}};search_op = collectionManager.do_search(req_params, embedded_params, json_res, now_ts);@@ -1989,7 +2022,7 @@ TEST_F(CollectionJoinTest, IncludeExcludeFieldsByReference) {{"query_by", "product_name"},{"infix", "always"},{"filter_by", "$Customers(customer_id:=customer_a && product_price:<100)"},- {"include_fields", "product_name, $Customers(product_price)"},+ {"include_fields", "product_name, $Customers(product_price:merge)"},{"exclude_fields", ""}};search_op = collectionManager.do_search(req_params, embedded_params, json_res, now_ts);@@ -2009,7 +2042,7 @@ TEST_F(CollectionJoinTest, IncludeExcludeFieldsByReference) {{"q", "Dan"},{"query_by", "customer_name"},{"filter_by", "$Products(rating:>3)"},- {"include_fields", "$Products(product_name), product_price"}+ {"include_fields", "$Products(product_name:merge), product_price"}};search_op = collectionManager.do_search(req_params, embedded_params, json_res, now_ts);@@ -2030,7 +2063,7 @@ TEST_F(CollectionJoinTest, IncludeExcludeFieldsByReference) {{"q", "Joe"},{"query_by", "customer_name"},{"filter_by", "product_price:<100"},- {"include_fields", "$Products(product_name), product_price"}+ {"include_fields", "$Products(product_name: merge), product_price"}};search_op = collectionManager.do_search(req_params, embedded_params, json_res, now_ts);ASSERT_TRUE(search_op.ok());@@ -2050,7 +2083,28 @@ TEST_F(CollectionJoinTest, IncludeExcludeFieldsByReference) {{"q", "Joe"},{"query_by", "customer_name"},{"filter_by", "product_price:<100"},- {"include_fields", "$Products(product_name) as p, product_price"}+ // With merge, alias is prepended+ {"include_fields", "$Products(product_name:merge) as prod, product_price"}+ };+ search_op = collectionManager.do_search(req_params, embedded_params, json_res, now_ts);+ ASSERT_TRUE(search_op.ok());++ res_obj = nlohmann::json::parse(json_res);+ ASSERT_EQ(1, res_obj["found"].get<size_t>());+ ASSERT_EQ(1, res_obj["hits"].size());+ ASSERT_EQ(2, res_obj["hits"][0]["document"].size());+ ASSERT_EQ(1, res_obj["hits"][0]["document"].count("prod.product_name"));+ ASSERT_EQ("soap", res_obj["hits"][0]["document"].at("prod.product_name"));+ ASSERT_EQ(1, res_obj["hits"][0]["document"].count("product_price"));+ ASSERT_EQ(73.5, res_obj["hits"][0]["document"].at("product_price"));++ req_params = {+ {"collection", "Customers"},+ {"q", "Joe"},+ {"query_by", "customer_name"},+ {"filter_by", "product_price:<100"},+ // With nest, alias becomes the key+ {"include_fields", "$Products(product_name:nest) as prod, product_price"}};search_op = collectionManager.do_search(req_params, embedded_params, json_res, now_ts);ASSERT_TRUE(search_op.ok());@@ -2059,11 +2113,38 @@ TEST_F(CollectionJoinTest, IncludeExcludeFieldsByReference) {ASSERT_EQ(1, res_obj["found"].get<size_t>());ASSERT_EQ(1, res_obj["hits"].size());ASSERT_EQ(2, res_obj["hits"][0]["document"].size());- ASSERT_EQ(1, res_obj["hits"][0]["document"].count("p.product_name"));- ASSERT_EQ("soap", res_obj["hits"][0]["document"].at("p.product_name"));+ ASSERT_EQ(1, res_obj["hits"][0]["document"].count("prod"));+ ASSERT_EQ(1, res_obj["hits"][0]["document"]["prod"].count("product_name"));+ ASSERT_EQ("soap", res_obj["hits"][0]["document"]["prod"].at("product_name"));ASSERT_EQ(1, res_obj["hits"][0]["document"].count("product_price"));ASSERT_EQ(73.5, res_obj["hits"][0]["document"].at("product_price"));+ req_params = {+ {"collection", "Products"},+ {"q", "soap"},+ {"query_by", "product_name"},+ {"filter_by", "$Customers(id:*)"},+ // With nest, alias becomes the key+ {"include_fields", "$Customers(customer_name, product_price :nest) as CustomerPrices, product_name"}+ };+ search_op = collectionManager.do_search(req_params, embedded_params, json_res, now_ts);+ ASSERT_TRUE(search_op.ok());++ res_obj = nlohmann::json::parse(json_res);+ ASSERT_EQ(1, res_obj["found"].get<size_t>());+ ASSERT_EQ(1, res_obj["hits"].size());+ ASSERT_EQ(2, res_obj["hits"][0]["document"].size());+ ASSERT_EQ(1, res_obj["hits"][0]["document"].count("product_name"));+ ASSERT_EQ("soap", res_obj["hits"][0]["document"]["product_name"]);+ ASSERT_EQ(1, res_obj["hits"][0]["document"].count("CustomerPrices"));+ ASSERT_EQ(2, res_obj["hits"][0]["document"]["CustomerPrices"].size());++ ASSERT_EQ("Joe", res_obj["hits"][0]["document"]["CustomerPrices"].at(0)["customer_name"]);+ ASSERT_EQ(73.5, res_obj["hits"][0]["document"]["CustomerPrices"].at(0)["product_price"]);++ ASSERT_EQ("Dan", res_obj["hits"][0]["document"]["CustomerPrices"].at(1)["customer_name"]);+ ASSERT_EQ(140, res_obj["hits"][0]["document"]["CustomerPrices"].at(1)["product_price"]);+schema_json =R"({"name": "Users",@@ -2267,7 +2348,7 @@ TEST_F(CollectionJoinTest, IncludeExcludeFieldsByReference) {{"q", "R"},{"query_by", "user_name"},{"filter_by", "$Participants(org_id:=org_a) && $Links(repo_id:=repo_b)"},- {"include_fields", "user_id, user_name, $Repos(repo_content), $Organizations(name) as org"},+ {"include_fields", "user_id, user_name, $Repos(repo_content:merge), $Organizations(name:merge) as org"},{"exclude_fields", "$Participants(*), $Links(*), "}};search_op = collectionManager.do_search(req_params, embedded_params, json_res, now_ts);@@ -2341,7 +2422,7 @@ TEST_F(CollectionJoinTest, FilterByReferenceArrayField) {std::map<std::string, std::string> req_params = {{"collection", "songs"},{"q", "*"},- {"include_fields", "$genres(name) as genre"},+ {"include_fields", "$genres(name:merge) as genre"},{"exclude_fields", "genres_sequence_id"},};nlohmann::json embedded_params;@@ -2372,7 +2453,7 @@ TEST_F(CollectionJoinTest, FilterByReferenceArrayField) {{"collection", "genres"},{"q", "*"},{"filter_by", "$songs(id: *)"},- {"include_fields", "$songs(title) as song"},+ {"include_fields", "$songs(title:merge) as song"},};search_op_bool = collectionManager.do_search(req_params, embedded_params, json_res, now_ts);ASSERT_TRUE(search_op_bool.ok());@@ -2720,7 +2801,7 @@ TEST_F(CollectionJoinTest, SortByReference) {{"query_by", "product_name"},{"filter_by", "$Customers(customer_id:=customer_a)"},{"sort_by", "$Customers(product_price:asc)"},- {"include_fields", "product_id, $Customers(product_price)"},+ {"include_fields", "product_id, $Customers(product_price:merge)"},};search_op = collectionManager.do_search(req_params, embedded_params, json_res, now_ts);ASSERT_TRUE(search_op.ok());@@ -2739,7 +2820,7 @@ TEST_F(CollectionJoinTest, SortByReference) {{"query_by", "product_name"},{"filter_by", "$Customers(customer_id:=customer_a)"},{"sort_by", "$Customers(product_price:desc)"},- {"include_fields", "product_id, $Customers(product_price)"},+ {"include_fields", "product_id, $Customers(product_price:merge)"},};search_op = collectionManager.do_search(req_params, embedded_params, json_res, now_ts);ASSERT_TRUE(search_op.ok());@@ -2759,7 +2840,7 @@ TEST_F(CollectionJoinTest, SortByReference) {{"query_by", "product_name"},{"filter_by", "$Customers(customer_id:=customer_a)"},{"sort_by", "$Customers(product_id:asc)"},- {"include_fields", "product_id, $Customers(product_price)"},+ {"include_fields", "product_id, $Customers(product_price:merge)"},};search_op = collectionManager.do_search(req_params, embedded_params, json_res, now_ts);ASSERT_TRUE(search_op.ok());@@ -2779,7 +2860,7 @@ TEST_F(CollectionJoinTest, SortByReference) {{"query_by", "product_name"},{"filter_by", "$Customers(customer_id:=customer_a)"},{"sort_by", "$Customers(_eval(product_available:true):asc)"},- {"include_fields", "product_id, $Customers(product_price)"},+ {"include_fields", "product_id, $Customers(product_price:merge)"},};search_op = collectionManager.do_search(req_params, embedded_params, json_res, now_ts);ASSERT_TRUE(search_op.ok());@@ -2798,7 +2879,7 @@ TEST_F(CollectionJoinTest, SortByReference) {{"query_by", "product_name"},{"filter_by", "$Customers(customer_id:=customer_a)"},{"sort_by", "$Customers(_eval(product_available:true):desc)"},- {"include_fields", "product_id, $Customers(product_price)"},+ {"include_fields", "product_id, $Customers(product_price:merge)"},};search_op = collectionManager.do_search(req_params, embedded_params, json_res, now_ts);ASSERT_TRUE(search_op.ok());@@ -2818,7 +2899,7 @@ TEST_F(CollectionJoinTest, SortByReference) {{"query_by", "product_name"},{"filter_by", "$Customers(customer_id:=customer_a)"},{"sort_by", "$Customers(product_price:desc)"},- {"include_fields", "product_id, $Customers(product_price)"},+ {"include_fields", "product_id, $Customers(product_price:merge)"},};search_op = collectionManager.do_search(req_params, embedded_params, json_res, now_ts);ASSERT_TRUE(search_op.ok());@@ -2837,7 +2918,7 @@ TEST_F(CollectionJoinTest, SortByReference) {{"q", R"("our")"},{"query_by", "product_description"},{"filter_by", "$Customers(customer_id:=customer_a)"},- {"include_fields", "product_id, $Customers(product_price)"},+ {"include_fields", "product_id, $Customers(product_price:merge)"},{"sort_by", "$Customers(product_price:desc)"},};search_op = collectionManager.do_search(req_params, embedded_params, json_res, now_ts);@@ -2857,7 +2938,7 @@ TEST_F(CollectionJoinTest, SortByReference) {{"q", "natural products"},{"query_by", "embedding"},{"filter_by", "$Customers(customer_id:=customer_a)"},- {"include_fields", "product_id, $Customers(product_price)"},+ {"include_fields", "product_id, $Customers(product_price:merge)"},{"sort_by", "$Customers(product_price:desc)"},};search_op = collectionManager.do_search(req_params, embedded_params, json_res, now_ts);@@ -2891,7 +2972,7 @@ TEST_F(CollectionJoinTest, SortByReference) {{"q", "*"},{"vector_query", "embedding:(" + vec_string + ", flat_search_cutoff: 0)"},{"filter_by", "$Customers(customer_id:=customer_a)"},- {"include_fields", "product_id, $Customers(product_price)"},+ {"include_fields", "product_id, $Customers(product_price:merge)"},{"sort_by", "$Customers(product_price:desc)"},};search_op = collectionManager.do_search(req_params, embedded_params, json_res, now_ts);@@ -2915,7 +2996,7 @@ TEST_F(CollectionJoinTest, SortByReference) {{"q", "soap"},{"query_by", "product_name, embedding"},{"filter_by", "$Customers(customer_id:=customer_a)"},- {"include_fields", "product_id, $Customers(product_price)"},+ {"include_fields", "product_id, $Customers(product_price:merge)"},{"sort_by", "$Customers(product_price:desc)"},};search_op = collectionManager.do_search(req_params, embedded_params, json_res, now_ts);@@ -2941,7 +3022,7 @@ TEST_F(CollectionJoinTest, SortByReference) {{"q", "natural products"},{"query_by", "product_name, embedding"},{"filter_by", "$Customers(customer_id:=customer_a)"},- {"include_fields", "product_id, $Customers(product_price)"},+ {"include_fields", "product_id, $Customers(product_price:merge)"},{"sort_by", "$Customers(product_price:desc)"},};search_op = collectionManager.do_search(req_params, embedded_params, json_res, now_ts);@@ -2966,7 +3047,7 @@ TEST_F(CollectionJoinTest, SortByReference) {{"query_by", "product_name"},{"infix", "always"},{"filter_by", "$Customers(customer_id:=customer_a)"},- {"include_fields", "product_id, $Customers(product_price)"},+ {"include_fields", "product_id, $Customers(product_price:merge)"},{"sort_by", "$Customers(product_price:desc)"},};search_op = collectionManager.do_search(req_params, embedded_params, json_res, now_ts);@@ -2985,7 +3066,7 @@ TEST_F(CollectionJoinTest, SortByReference) {{"collection", "Customers"},{"q", "*"},{"filter_by", "customer_name:= [Joe, Dan] && product_price:<100"},- {"include_fields", "$Products(product_name), product_price"},+ {"include_fields", "$Products(product_name:merge), product_price"},{"sort_by", "$Products(product_name:desc)"},};search_op = collectionManager.do_search(req_params, embedded_params, json_res, now_ts);@@ -3004,7 +3085,7 @@ TEST_F(CollectionJoinTest, SortByReference) {{"collection", "Customers"},{"q", "*"},{"filter_by", "customer_name:= [Joe, Dan] && product_price:<100"},- {"include_fields", "$Products(product_name), product_price"},+ {"include_fields", "$Products(product_name:merge), product_price"},{"sort_by", "$Products(product_name:asc)"},};search_op = collectionManager.do_search(req_params, embedded_params, json_res, now_ts);@@ -3166,7 +3247,7 @@ TEST_F(CollectionJoinTest, SortByReference) {{"collection", "Users"},{"q", "*"},{"filter_by", "$Links(repo_id:=[repo_a, repo_d])"},- {"include_fields", "user_id, user_name, $Repos(repo_content, repo_stars), "},+ {"include_fields", "user_id, user_name, $Repos(repo_content, repo_stars:merge), "},{"exclude_fields", "$Links(*), "},{"sort_by", "$Repos(repo_stars: asc)"}};@@ -3196,7 +3277,7 @@ TEST_F(CollectionJoinTest, SortByReference) {{"collection", "Users"},{"q", "*"},{"filter_by", "$Links(repo_id:=[repo_a, repo_d])"},- {"include_fields", "user_id, user_name, $Repos(repo_content, repo_stars), "},+ {"include_fields", "user_id, user_name, $Repos(repo_content, repo_stars:merge), "},{"exclude_fields", "$Links(*), "},{"sort_by", "$Repos(repo_stars: desc)"}};