Open Source alternative to Algolia + Pinecone and an Easier-to-Use alternative to ElasticSearch ⚡ 🔍 ✨ Fast, typo tolerant, in-memory fuzzy Search Engine for building delightful search experiences
C++ algoliadatastoreelasticsearchenterprise-searchfacetingfull-text-searchfuzzy-searchgeosearchin-memoryinstantsearchmerchandisingpineconesearchsearch-enginesemantic-searchsimilarity-searchsite-searchsynonymstypo-tolerancevector-search
Support nested join.
Harpreet Sangar08c0a087a1d2
diff --git a/include/collection.h b/include/collection.hindex dfaa4f38..d4253f80 100644--- a/include/collection.h+++ b/include/collection.h@@ -438,22 +438,23 @@ public:static void remove_reference_helper_fields(nlohmann::json& document);- static Option<bool> include_references(nlohmann::json& doc,- const std::string& ref_collection_name,- Collection *const ref_collection,- const std::string& alias,- const reference_filter_result_t& references,- const tsl::htrie_set<char>& ref_include_fields_full,- const tsl::htrie_set<char>& ref_exclude_fields_full,- const std::string& error_prefix, const bool& is_reference_array,- const ref_include::strategy_enum& strategy);+ static Option<bool> prune_ref_doc(nlohmann::json& doc,+ const reference_filter_result_t& references,+ const tsl::htrie_set<char>& ref_include_fields_full,+ const tsl::htrie_set<char>& ref_exclude_fields_full,+ const bool& is_reference_array,+ const ref_include_exclude_fields& ref_include_exclude);++ static Option<bool> include_references(nlohmann::json& doc, const uint32_t& seq_id, Collection *const collection,+ const std::map<std::string, reference_filter_result_t>& reference_filter_results,+ const std::vector<ref_include_exclude_fields>& ref_include_exclude_fields_vec);static Option<bool> prune_doc(nlohmann::json& doc, const tsl::htrie_set<char>& include_names,const tsl::htrie_set<char>& exclude_names, const std::string& parent_name = "",size_t depth = 0,const std::map<std::string, reference_filter_result_t>& reference_filter_results = {},Collection *const collection = nullptr, const uint32_t& seq_id = 0,- const std::vector<ref_include_fields>& ref_include_fields_vec = {});+ const std::vector<ref_include_exclude_fields>& ref_include_exclude_fields_vec = {});const Index* _get_index() const;@@ -558,7 +559,7 @@ public:const size_t remote_embedding_num_tries = 2,const std::string& stopwords_set="",const std::vector<std::string>& facet_return_parent = {},- const std::vector<ref_include_fields>& ref_include_fields_vec = {},+ const std::vector<ref_include_exclude_fields>& ref_include_exclude_fields_vec = {},const std::string& drop_tokens_mode = "right_to_left",const bool prioritize_num_matching_fields = true,const bool group_missing_values = true,diff --git a/include/collection_manager.h b/include/collection_manager.hindex 889391f5..44484df1 100644--- a/include/collection_manager.h+++ b/include/collection_manager.h@@ -221,10 +221,11 @@ public:static void _get_reference_collection_names(const std::string& filter_query,ref_include_collection_names_t*& reference_collection_names);- // Separate out the reference includes into `ref_include_fields_vec`.- static Option<bool> _initialize_ref_include_fields_vec(const std::string& filter_query,- std::vector<std::string>& include_fields_vec,- std::vector<ref_include_fields>& ref_include_fields_vec);+ // Separate out the reference includes and excludes into `ref_include_exclude_fields_vec`.+ static Option<bool> _initialize_ref_include_exclude_fields_vec(const std::string& filter_query,+ std::vector<std::string>& include_fields_vec,+ std::vector<std::string>& exclude_fields_vec,+ std::vector<ref_include_exclude_fields>& ref_include_exclude_fields_vec);void add_referenced_in_backlog(const std::string& collection_name, reference_pair&& pair);diff --git a/include/field.h b/include/field.hindex b06e07c3..20c34427 100644--- a/include/field.h+++ b/include/field.h@@ -530,14 +530,15 @@ namespace ref_include {}}-struct ref_include_fields {+struct ref_include_exclude_fields {std::string collection_name;- std::string fields;+ std::string include_fields;+ std::string exclude_fields;std::string alias;ref_include::strategy_enum strategy = ref_include::nest;// In case we have nested join.- std::vector<ref_include_fields> nested_join_includes = {};+ std::vector<ref_include_exclude_fields> nested_join_includes = {};};struct hnsw_index_t;diff --git a/include/filter_result_iterator.h b/include/filter_result_iterator.hindex 77da1bd8..1e89b1b7 100644--- a/include/filter_result_iterator.h+++ b/include/filter_result_iterator.h@@ -14,8 +14,14 @@ struct filter_node_t;struct reference_filter_result_t {uint32_t count = 0;uint32_t* docs = nullptr;+ bool is_reference_array_field = true;- explicit reference_filter_result_t(uint32_t count = 0, uint32_t* docs = nullptr) : count(count), docs(docs) {}+ // In case of nested join, references can further have references.+ std::map<std::string, reference_filter_result_t>* coll_to_references = nullptr;++ explicit reference_filter_result_t(uint32_t count = 0, uint32_t* docs = nullptr,+ bool is_reference_array_field = true) : count(count), docs(docs),+ is_reference_array_field(is_reference_array_field) {}reference_filter_result_t(const reference_filter_result_t& obj) {if (&obj == this) {@@ -25,6 +31,9 @@ struct reference_filter_result_t {count = obj.count;docs = new uint32_t[count];memcpy(docs, obj.docs, count * sizeof(uint32_t));+ is_reference_array_field = obj.is_reference_array_field;++ copy_references(obj, *this);}reference_filter_result_t& operator=(const reference_filter_result_t& obj) noexcept {@@ -35,7 +44,9 @@ struct reference_filter_result_t {count = obj.count;docs = new uint32_t[count];memcpy(docs, obj.docs, count * sizeof(uint32_t));+ is_reference_array_field = obj.is_reference_array_field;+ copy_references(obj, *this);return *this;}@@ -46,26 +57,38 @@ struct reference_filter_result_t {count = obj.count;docs = obj.docs;+ coll_to_references = obj.coll_to_references;+ is_reference_array_field = obj.is_reference_array_field;+ // Set default values in obj.+ obj.count = 0;obj.docs = nullptr;+ obj.coll_to_references = nullptr;+ obj.is_reference_array_field = true;return *this;}~reference_filter_result_t() {delete[] docs;+ delete[] coll_to_references;}++ static void copy_references(const reference_filter_result_t& from, reference_filter_result_t& to);};struct single_filter_result_t {uint32_t seq_id = 0;// Collection name -> Reference filter resultstd::map<std::string, reference_filter_result_t> reference_filter_results = {};+ bool is_reference_array_field = true;single_filter_result_t() = default;- single_filter_result_t(uint32_t seq_id, std::map<std::string, reference_filter_result_t>&& reference_filter_results) :- seq_id(seq_id), reference_filter_results(std::move(reference_filter_results)) {}+ single_filter_result_t(uint32_t seq_id, std::map<std::string, reference_filter_result_t>&& reference_filter_results,+ bool is_reference_array_field = true) :+ seq_id(seq_id), reference_filter_results(std::move(reference_filter_results)),+ is_reference_array_field(is_reference_array_field) {}single_filter_result_t(const single_filter_result_t& obj) {if (&obj == this) {@@ -73,12 +96,52 @@ struct single_filter_result_t {}seq_id = obj.seq_id;+ is_reference_array_field = obj.is_reference_array_field;++ // Copy every collection's reference.+ for (const auto &item: obj.reference_filter_results) {+ auto& ref_coll_name = item.first;+ reference_filter_results[ref_coll_name] = item.second;+ }+ }++ single_filter_result_t(single_filter_result_t&& obj) {+ if (&obj == this) {+ return;+ }++ seq_id = obj.seq_id;+ is_reference_array_field = obj.is_reference_array_field;+ reference_filter_results = std::move(obj.reference_filter_results);+ }++ single_filter_result_t& operator=(const single_filter_result_t& obj) noexcept {+ if (&obj == this) {+ return *this;+ }++ seq_id = obj.seq_id;+ is_reference_array_field = obj.is_reference_array_field;// Copy every collection's reference.for (const auto &item: obj.reference_filter_results) {auto& ref_coll_name = item.first;reference_filter_results[ref_coll_name] = item.second;}++ return *this;+ }++ single_filter_result_t& operator=(single_filter_result_t&& obj) noexcept {+ if (&obj == this) {+ return *this;+ }++ seq_id = obj.seq_id;+ is_reference_array_field = obj.is_reference_array_field;+ reference_filter_results = std::move(obj.reference_filter_results);++ return *this;}};@@ -127,6 +190,8 @@ struct filter_result_t {docs = obj.docs;coll_to_references = obj.coll_to_references;+ // Set default values in obj.+ obj.count = 0;obj.docs = nullptr;obj.coll_to_references = nullptr;diff --git a/include/index.h b/include/index.hindex db881e87..c767d280 100644--- a/include/index.h+++ b/include/index.h@@ -786,12 +786,15 @@ public:filter_result_t& filter_result,const std::string& collection_name = "") const;-Option<bool> do_reference_filtering_with_lock(filter_node_t* const filter_tree_root,filter_result_t& filter_result,- const std::string& collection_name,+ const std::string& ref_collection_name,const std::string& reference_helper_field_name) const;+ Option<filter_result_t> do_filtering_with_reference_ids(const std::string& reference_helper_field_name,+ const std::string& ref_collection_name,+ filter_result_t&& ref_filter_result) const;+void refresh_schemas(const std::vector<field>& new_fields, const std::vector<field>& del_fields);// the following methods are not synchronized because their parent calls are synchronized or they are const/staticdiff --git a/include/string_utils.h b/include/string_utils.hindex 0568a29e..e3d834e6 100644--- a/include/string_utils.h+++ b/include/string_utils.h@@ -334,11 +334,11 @@ struct StringUtils {static Option<bool> tokenize_filter_query(const std::string& filter_query, std::queue<std::string>& tokens);- static Option<bool> split_include_fields(const std::string& include_fields, std::vector<std::string>& tokens);+ static Option<bool> split_include_exclude_fields(const std::string& include_exclude_fields,+ std::vector<std::string>& tokens);static size_t get_occurence_count(const std::string& str, char symbol);- static Option<bool> split_reference_include_fields(const std::string& include_fields,- size_t& index,- std::string& token);+ static Option<bool> split_reference_include_exclude_fields(const std::string& include_fields,+ size_t& index, std::string& token);};diff --git a/src/collection.cpp b/src/collection.cppindex 83131a6b..95563136 100644--- a/src/collection.cpp+++ b/src/collection.cpp@@ -1611,7 +1611,7 @@ Option<nlohmann::json> Collection::search(std::string raw_query,const size_t remote_embedding_num_tries,const std::string& stopwords_set,const std::vector<std::string>& facet_return_parent,- const std::vector<ref_include_fields>& ref_include_fields_vec,+ const std::vector<ref_include_exclude_fields>& ref_include_exclude_fields_vec,const std::string& drop_tokens_mode,const bool prioritize_num_matching_fields,const bool group_missing_values,@@ -2547,7 +2547,7 @@ Option<nlohmann::json> Collection::search(std::string raw_query,0,field_order_kv->reference_filter_results,const_cast<Collection *>(this), get_seq_id_from_key(seq_id_key),- ref_include_fields_vec);+ ref_include_exclude_fields_vec);if (!prune_op.ok()) {return Option<nlohmann::json>(prune_op.code(), prune_op.error());}@@ -4862,15 +4862,23 @@ void Collection::remove_reference_helper_fields(nlohmann::json& document) {}}-Option<bool> Collection::include_references(nlohmann::json& doc,- const std::string& ref_collection_name,- Collection *const ref_collection,- const std::string& alias,- const reference_filter_result_t& references,- const tsl::htrie_set<char>& ref_include_fields_full,- const tsl::htrie_set<char>& ref_exclude_fields_full,- const std::string& error_prefix, const bool& is_reference_array,- const ref_include::strategy_enum& strategy) {+Option<bool> Collection::prune_ref_doc(nlohmann::json& doc,+ const reference_filter_result_t& references,+ const tsl::htrie_set<char>& ref_include_fields_full,+ const tsl::htrie_set<char>& ref_exclude_fields_full,+ const bool& is_reference_array,+ const ref_include_exclude_fields& ref_include_exclude) {+ auto const& ref_collection_name = ref_include_exclude.collection_name;+ auto& cm = CollectionManager::get_instance();+ auto ref_collection = cm.get_collection(ref_collection_name);+ if (ref_collection == nullptr) {+ return Option<bool>(400, "Referenced collection `" + ref_collection_name + "` in `include_fields` not found.");+ }++ auto const& alias = ref_include_exclude.alias;+ auto const& strategy = ref_include_exclude.strategy;+ auto error_prefix = "Referenced collection `" + ref_collection_name + "`: ";+// One-to-one relation.if (strategy != ref_include::nest_array && !is_reference_array && references.count == 1) {auto ref_doc_seq_id = references.docs[0];@@ -4889,23 +4897,33 @@ Option<bool> Collection::include_references(nlohmann::json& doc,return Option<bool>(prune_op.code(), error_prefix + prune_op.error());}- if (ref_doc.empty()) {- return Option<bool>(true);+ auto const key = alias.empty() ? ref_collection_name : alias;+ auto const& nest_ref_doc = (strategy == ref_include::nest);+ if (!ref_doc.empty()) {+ if (nest_ref_doc) {+ doc[key] = ref_doc;+ } else {+ if (!alias.empty()) {+ auto temp_doc = ref_doc;+ ref_doc.clear();+ for (const auto &item: temp_doc.items()) {+ ref_doc[alias + item.key()] = item.value();+ }+ }+ doc.update(ref_doc);+ }}- if (strategy == ref_include::nest) {- auto key = alias.empty() ? ref_collection_name : alias;- doc[key] = ref_doc;- } else {- if (!alias.empty()) {- auto temp_doc = ref_doc;- ref_doc.clear();- for (const auto &item: temp_doc.items()) {- ref_doc[alias + item.key()] = item.value();- }+ // Include nested join references.+ if (!ref_include_exclude.nested_join_includes.empty() && !references.coll_to_references->empty()) {+ auto nested_include_exclude_op = include_references(nest_ref_doc ? doc[key] : doc, ref_doc_seq_id,+ ref_collection.get(), references.coll_to_references[0],+ ref_include_exclude.nested_join_includes);+ if (!nested_include_exclude_op.ok()) {+ return nested_include_exclude_op;}- doc.update(ref_doc);}+return Option<bool>(true);}@@ -4927,123 +4945,55 @@ Option<bool> Collection::include_references(nlohmann::json& doc,return Option<bool>(prune_op.code(), error_prefix + prune_op.error());}- if (ref_doc.empty()) {- continue;- }-- if (strategy == ref_include::nest || strategy == ref_include::nest_array) {- auto key = alias.empty() ? ref_collection_name : alias;- if (doc.contains(key) && !doc[key].is_array()) {- return Option<bool>(400, "Could not include the reference document of `" + ref_collection_name +- "` collection. Expected `" + key + "` to be an array. Try " +- (alias.empty() ? "adding an" : "renaming the") + " alias.");- }-- doc[key] += ref_doc;- } else {- for (auto ref_doc_it = ref_doc.begin(); ref_doc_it != ref_doc.end(); ref_doc_it++) {- auto const& ref_doc_key = ref_doc_it.key();- auto const& doc_key = alias + ref_doc_key;- if (doc.contains(doc_key) && !doc[doc_key].is_array()) {- return Option<bool>(400, "Could not include the value of `" + ref_doc_key +- "` key of the reference document of `" + ref_collection_name +- "` collection. Expected `" + doc_key + "` to be an array. Try " ++ std::string key;+ auto const& nest_ref_doc = (strategy == ref_include::nest || strategy == ref_include::nest_array);+ if (!ref_doc.empty()) {+ if (nest_ref_doc) {+ key = alias.empty() ? ref_collection_name : alias;+ if (doc.contains(key) && !doc[key].is_array()) {+ return Option<bool>(400, "Could not include the reference document of `" + ref_collection_name ++ "` collection. Expected `" + key + "` to be an array. Try " +(alias.empty() ? "adding an" : "renaming the") + " alias.");}- // Add the values of ref_doc as JSON array into doc.- doc[doc_key] += ref_doc_it.value();- }- }- }-- return Option<bool>(true);-}--Option<bool> Collection::prune_doc(nlohmann::json& doc,- const tsl::htrie_set<char>& include_names,- const tsl::htrie_set<char>& exclude_names,- const std::string& parent_name, size_t depth,- const std::map<std::string, reference_filter_result_t>& reference_filter_results,- Collection *const collection, const uint32_t& seq_id,- const std::vector<ref_include_fields>& ref_includes) {- // doc can only be an object- auto it = doc.begin();- while(it != doc.end()) {- std::string nested_name = parent_name + (parent_name.empty() ? it.key() : "." + it.key());-- //LOG(INFO) << "it.key(): " << it.key() << ", nested_name: " << nested_name;-- // use prefix lookup to prune non-matching sub-trees early- auto prefix_it = include_names.equal_prefix_range(nested_name);- if(!include_names.empty() && prefix_it.first == prefix_it.second) {- // prefix not found in allowed list of highlight field names, so can trim early- it = doc.erase(it);- continue ;- }-- if(exclude_names.count(nested_name) != 0) {- it = doc.erase(it);- continue ;- }-- if(exclude_names.empty() && !include_names.empty() && include_names.count(nested_name) != 0) {- // without exclusions, we can pick the sub-tree early if parent name is found in include names- it++;- continue;- }-- if(it.value().is_object()) {- bool is_orig_empty = it.value().empty();- prune_doc(it.value(), include_names, exclude_names, nested_name, depth+1);- if(!is_orig_empty && it.value().empty()) {- it = doc.erase(it);+ doc[key] += ref_doc;} else {- it++;- }-- continue;- }+ for (auto ref_doc_it = ref_doc.begin(); ref_doc_it != ref_doc.end(); ref_doc_it++) {+ auto const& ref_doc_key = ref_doc_it.key();+ key = alias + ref_doc_key;+ if (doc.contains(key) && !doc[key].is_array()) {+ return Option<bool>(400, "Could not include the value of `" + ref_doc_key ++ "` key of the reference document of `" + ref_collection_name ++ "` collection. Expected `" + key + "` to be an array. Try " ++ (alias.empty() ? "adding an" : "renaming the") + " alias.");+ }- else if(it.value().is_array()) {- bool orig_array_empty = it.value().empty();- bool primitive_array = true;- auto arr_it = it.value().begin();- while(arr_it != it.value().end()) {- // NOTE: we will not support array of array of nested objects- primitive_array = primitive_array && !arr_it.value().is_object();- if(arr_it.value().is_object()) {- bool orig_ele_empty = arr_it.value().empty();- prune_doc(arr_it.value(), include_names, exclude_names, nested_name, depth+1);- // don't remove empty array objects to help frontend+ // Add the values of ref_doc as JSON array into doc.+ doc[key] += ref_doc_it.value();}-- arr_it++;- }-- if(!orig_array_empty && it.value().empty()) {- // only drop field if array became empty because of pruning (and not empty already)- it = doc.erase(it);- continue;- }-- if(!primitive_array) {- it++;- continue;}}- if(!include_names.empty() && include_names.count(nested_name) == 0) {- // at this point, name should match fully, otherwise we should erase the value- it = doc.erase(it);- continue;+ // Include nested join references.+ if (!ref_include_exclude.nested_join_includes.empty() &&+ references.coll_to_references != nullptr && !references.coll_to_references->empty()) {+ auto nested_include_exclude_op = include_references(nest_ref_doc ? doc[key].at(i) : doc, ref_doc_seq_id,+ ref_collection.get(), references.coll_to_references[i],+ ref_include_exclude.nested_join_includes);+ if (!nested_include_exclude_op.ok()) {+ return nested_include_exclude_op;+ }}-- it++;}- for (auto const& ref_include: ref_includes) {- auto const& ref_collection_name = ref_include.collection_name;+ return Option<bool>(true);+}++Option<bool> Collection::include_references(nlohmann::json& doc, const uint32_t& seq_id, Collection *const collection,+ const std::map<std::string, reference_filter_result_t>& reference_filter_results,+ const std::vector<ref_include_exclude_fields>& ref_include_exclude_fields_vec) {+ for (auto const& ref_include_exclude: ref_include_exclude_fields_vec) {+ auto const& ref_collection_name = ref_include_exclude.collection_name;auto& cm = CollectionManager::get_instance();auto ref_collection = cm.get_collection(ref_collection_name);@@ -5052,8 +5002,8 @@ Option<bool> Collection::prune_doc(nlohmann::json& doc,}auto const joined_on_ref_collection = reference_filter_results.count(ref_collection_name) > 0,- has_filter_reference = (joined_on_ref_collection &&- reference_filter_results.at(ref_collection_name).count > 0);+ has_filter_reference = (joined_on_ref_collection &&+ reference_filter_results.at(ref_collection_name).count > 0);auto doc_has_reference = false, joined_coll_has_reference = false;// Reference include_by without join, check if doc itself contains the reference.@@ -5078,45 +5028,29 @@ Option<bool> Collection::prune_doc(nlohmann::json& doc,}std::vector<std::string> ref_include_fields_vec, ref_exclude_fields_vec;- StringUtils::split(ref_include.fields, ref_include_fields_vec, ",");- auto exclude_reference_it = exclude_names.equal_prefix_range("$" + ref_collection_name);- if (exclude_reference_it.first != exclude_reference_it.second) {- auto ref_exclude = exclude_reference_it.first.key();- auto parenthesis_index = ref_exclude.find('(');- auto reference_fields = ref_exclude.substr(parenthesis_index + 1, ref_exclude.size() - parenthesis_index - 2);- StringUtils::split(reference_fields, ref_exclude_fields_vec, ",");- }+ StringUtils::split(ref_include_exclude.include_fields, ref_include_fields_vec, ",");+ StringUtils::split(ref_include_exclude.exclude_fields, ref_exclude_fields_vec, ",");spp::sparse_hash_set<std::string> ref_include_fields, ref_exclude_fields;ref_include_fields.insert(ref_include_fields_vec.begin(), ref_include_fields_vec.end());ref_exclude_fields.insert(ref_exclude_fields_vec.begin(), ref_exclude_fields_vec.end());tsl::htrie_set<char> ref_include_fields_full, ref_exclude_fields_full;- auto include_exclude_op = ref_collection->populate_include_exclude_fields(ref_include_fields,- ref_exclude_fields,- ref_include_fields_full,- ref_exclude_fields_full);+ auto include_exclude_op = ref_collection->populate_include_exclude_fields_lk(ref_include_fields,+ ref_exclude_fields,+ ref_include_fields_full,+ ref_exclude_fields_full);auto error_prefix = "Referenced collection `" + ref_collection_name + "`: ";if (!include_exclude_op.ok()) {return Option<bool>(include_exclude_op.code(), error_prefix + include_exclude_op.error());}- Option<bool> include_references_op = Option<bool>(true);+ Option<bool> prune_doc_op = Option<bool>(true);+ auto const& ref_collection_alias = ref_include_exclude.alias;if (has_filter_reference) {- auto get_reference_field_op = collection->get_referenced_in_field(ref_collection_name);- if (!get_reference_field_op.ok()) {- continue;- }- auto const& field_name = get_reference_field_op.get();- if (ref_collection->search_schema.count(field_name) == 0) {- continue;- }- include_references_op = include_references(doc, ref_include.collection_name,- ref_collection.get(), ref_include.alias,- reference_filter_results.at(ref_collection_name),- ref_include_fields_full, ref_exclude_fields_full, error_prefix,- ref_collection->get_schema().at(field_name).is_array(),- ref_include.strategy);+ auto const& ref_filter_result = reference_filter_results.at(ref_collection_name);+ prune_doc_op = prune_ref_doc(doc, ref_filter_result, ref_include_fields_full, ref_exclude_fields_full,+ ref_filter_result.is_reference_array_field, ref_include_exclude);} else if (doc_has_reference) {auto get_reference_field_op = ref_collection->get_referenced_in_field_with_lock(collection->name);if (!get_reference_field_op.ok()) {@@ -5148,12 +5082,11 @@ Option<bool> Collection::prune_doc(nlohmann::json& doc,}reference_filter_result_t result(1, new uint32_t[1]{ref_doc_id});- include_references_op = include_references(doc[keys[0]][i], ref_include.collection_name,- ref_collection.get(), ref_include.alias, result,- ref_include_fields_full, ref_exclude_fields_full, error_prefix,- false, ref_include.strategy);- if (!include_references_op.ok()) {- return include_references_op;+ prune_doc_op = prune_ref_doc(doc[keys[0]][i], result,+ ref_include_fields_full, ref_exclude_fields_full,+ false, ref_include_exclude);+ if (!prune_doc_op.ok()) {+ return prune_doc_op;}}} else {@@ -5163,11 +5096,8 @@ Option<bool> Collection::prune_doc(nlohmann::json& doc,continue;}reference_filter_result_t result(ids.size(), &ids[0]);- include_references_op = include_references(doc[keys[0]], ref_include.collection_name,- ref_collection.get(), ref_include.alias, result,- ref_include_fields_full, ref_exclude_fields_full, error_prefix,- collection->search_schema.at(field_name).is_array(),- ref_include.strategy);+ prune_doc_op = prune_ref_doc(doc[keys[0]], result, ref_include_fields_full, ref_exclude_fields_full,+ collection->search_schema.at(field_name).is_array(), ref_include_exclude);result.docs = nullptr;}} else {@@ -5177,11 +5107,8 @@ Option<bool> Collection::prune_doc(nlohmann::json& doc,continue;}reference_filter_result_t result(ids.size(), &ids[0]);- include_references_op = include_references(doc, ref_include.collection_name,- ref_collection.get(), ref_include.alias, result,- ref_include_fields_full, ref_exclude_fields_full, error_prefix,- collection->search_schema.at(field_name).is_array(),- ref_include.strategy);+ prune_doc_op = prune_ref_doc(doc, result, ref_include_fields_full, ref_exclude_fields_full,+ collection->search_schema.at(field_name).is_array(), ref_include_exclude);result.docs = nullptr;}} else if (joined_coll_has_reference) {@@ -5213,22 +5140,105 @@ Option<bool> Collection::prune_doc(nlohmann::json& doc,reference_filter_result_t result;result.count = ids.size();result.docs = &ids[0];- include_references_op = include_references(doc, ref_include.collection_name,- ref_collection.get(), ref_include.alias, result,- ref_include_fields_full, ref_exclude_fields_full, error_prefix,- joined_collection->get_schema().at(reference_field_name).is_array(),- ref_include.strategy);+ prune_doc_op = prune_ref_doc(doc, result, ref_include_fields_full, ref_exclude_fields_full,+ joined_collection->get_schema().at(reference_field_name).is_array(),+ ref_include_exclude);result.docs = nullptr;}- if (!include_references_op.ok()) {- return include_references_op;+ if (!prune_doc_op.ok()) {+ return prune_doc_op;}}return Option<bool>(true);}+Option<bool> Collection::prune_doc(nlohmann::json& doc,+ const tsl::htrie_set<char>& include_names,+ const tsl::htrie_set<char>& exclude_names,+ const std::string& parent_name, size_t depth,+ const std::map<std::string, reference_filter_result_t>& reference_filter_results,+ Collection *const collection, const uint32_t& seq_id,+ const std::vector<ref_include_exclude_fields>& ref_include_exclude_fields_vec) {+ // doc can only be an object+ auto it = doc.begin();+ while(it != doc.end()) {+ std::string nested_name = parent_name + (parent_name.empty() ? it.key() : "." + it.key());++ //LOG(INFO) << "it.key(): " << it.key() << ", nested_name: " << nested_name;++ // use prefix lookup to prune non-matching sub-trees early+ auto prefix_it = include_names.equal_prefix_range(nested_name);+ if(!include_names.empty() && prefix_it.first == prefix_it.second) {+ // prefix not found in allowed list of highlight field names, so can trim early+ it = doc.erase(it);+ continue ;+ }++ if(exclude_names.count(nested_name) != 0) {+ it = doc.erase(it);+ continue ;+ }++ if(exclude_names.empty() && !include_names.empty() && include_names.count(nested_name) != 0) {+ // without exclusions, we can pick the sub-tree early if parent name is found in include names+ it++;+ continue;+ }++ if(it.value().is_object()) {+ bool is_orig_empty = it.value().empty();+ prune_doc(it.value(), include_names, exclude_names, nested_name, depth+1);+ if(!is_orig_empty && it.value().empty()) {+ it = doc.erase(it);+ } else {+ it++;+ }++ continue;+ }++ else if(it.value().is_array()) {+ bool orig_array_empty = it.value().empty();+ bool primitive_array = true;+ auto arr_it = it.value().begin();+ while(arr_it != it.value().end()) {+ // NOTE: we will not support array of array of nested objects+ primitive_array = primitive_array && !arr_it.value().is_object();+ if(arr_it.value().is_object()) {+ bool orig_ele_empty = arr_it.value().empty();+ prune_doc(arr_it.value(), include_names, exclude_names, nested_name, depth+1);+ // don't remove empty array objects to help frontend+ }++ arr_it++;+ }++ if(!orig_array_empty && it.value().empty()) {+ // only drop field if array became empty because of pruning (and not empty already)+ it = doc.erase(it);+ continue;+ }++ if(!primitive_array) {+ it++;+ continue;+ }+ }++ if(!include_names.empty() && include_names.count(nested_name) == 0) {+ // at this point, name should match fully, otherwise we should erase the value+ it = doc.erase(it);+ continue;+ }++ it++;+ }++ return include_references(doc, seq_id, collection, reference_filter_results, ref_include_exclude_fields_vec);+}+Option<bool> Collection::validate_alter_payload(nlohmann::json& schema_changes,std::vector<field>& addition_fields,std::vector<field>& reindex_fields,diff --git a/src/collection_manager.cpp b/src/collection_manager.cppindex 1758a69e..aa4184c9 100644--- a/src/collection_manager.cpp+++ b/src/collection_manager.cpp@@ -982,9 +982,61 @@ void CollectionManager::_get_reference_collection_names(const std::string& filte}}+Option<bool> parse_nested_exclude(const std::string& exclude_field_exp,+ std::unordered_map<std::string, std::string>& ref_excludes) {+ // Format: $ref_collection_name(field_1, field_2, $nested_ref_coll(nested_field_1))+ size_t index = 0;+ while (index < exclude_field_exp.size()) {+ auto parenthesis_index = exclude_field_exp.find('(');+ auto ref_collection_name = exclude_field_exp.substr(index + 1, parenthesis_index - index - 1);+ std::string ref_fields;++ index = parenthesis_index + 1;+ auto nested_exclude_pos = exclude_field_exp.find('$', parenthesis_index);+ auto closing_parenthesis_pos = exclude_field_exp.find(')', parenthesis_index);+ size_t comma_pos;+ if (nested_exclude_pos < closing_parenthesis_pos) {+ // Nested reference exclude.+ // "... $product_variants(title, $inventory(qty)) ..."+ do {+ ref_fields += exclude_field_exp.substr(index, nested_exclude_pos - index);+ StringUtils::trim(ref_fields);+ index = nested_exclude_pos;+ std::string nested_exclude_field_exp;+ auto split_op = StringUtils::split_reference_include_exclude_fields(exclude_field_exp, index,+ nested_exclude_field_exp);+ if (!split_op.ok()) {+ return split_op;+ }++ auto parse_op = parse_nested_exclude(nested_exclude_field_exp, ref_excludes);+ if (!parse_op.ok()) {+ return parse_op;+ }++ nested_exclude_pos = exclude_field_exp.find('$', index);+ closing_parenthesis_pos = exclude_field_exp.find(')', index);+ comma_pos = exclude_field_exp.find(',', index);+ index = std::min(closing_parenthesis_pos, comma_pos) + 1;+ } while (index < exclude_field_exp.size() && nested_exclude_pos < closing_parenthesis_pos);+ }++ // ... $inventory(qty) ...+ if (index < closing_parenthesis_pos) {+ ref_fields += exclude_field_exp.substr(index, closing_parenthesis_pos - index);+ }+ StringUtils::trim(ref_fields);++ ref_excludes[ref_collection_name] = ref_fields;+ index = closing_parenthesis_pos + 1;+ }++ return Option<bool>(true);+}+Option<bool> parse_nested_include(const std::string& include_field_exp,CollectionManager::ref_include_collection_names_t* const ref_include_coll_names,- std::vector<ref_include_fields>& ref_include_fields_vec) {+ std::vector<ref_include_exclude_fields>& ref_include_exclude_fields_vec) {// Format: $ref_collection_name(field_1, field_2, $nested_ref_coll(nested_field_1: nested_include_strategy) as nested_ref_alias: include_strategy) as ref_aliassize_t index = 0;while (index < include_field_exp.size()) {@@ -998,7 +1050,7 @@ Option<bool> parse_nested_include(const std::string& include_field_exp,auto closing_parenthesis_pos = include_field_exp.find(')', parenthesis_index);auto colon_pos = include_field_exp.find(':', index);size_t comma_pos;- std::vector<ref_include_fields> nested_ref_include_fields_vec;+ std::vector<ref_include_exclude_fields> nested_ref_include_exclude_fields_vec;if (nested_include_pos < closing_parenthesis_pos) {// Nested reference include.// "... $product_variants(title, $inventory(qty:merge) as inventory :nest) as variants ..."@@ -1007,15 +1059,15 @@ Option<bool> parse_nested_include(const std::string& include_field_exp,StringUtils::trim(ref_fields);index = nested_include_pos;std::string nested_include_field_exp;- auto split_op = StringUtils::split_reference_include_fields(include_field_exp, index,- nested_include_field_exp);+ auto split_op = StringUtils::split_reference_include_exclude_fields(include_field_exp, index,+ nested_include_field_exp);if (!split_op.ok()) {return split_op;}auto parse_op = parse_nested_include(nested_include_field_exp,ref_include_coll_names == nullptr ? nullptr : ref_include_coll_names->nested_include,- nested_ref_include_fields_vec);+ nested_ref_include_exclude_fields_vec);if (!parse_op.ok()) {return parse_op;}@@ -1062,11 +1114,11 @@ Option<bool> parse_nested_include(const std::string& include_field_exp,nest_ref_doc = strategy_enum == ref_include::nest || strategy_enum == ref_include::nest_array;ref_alias = !ref_alias.empty() ? (StringUtils::trim(ref_alias) + (nest_ref_doc ? "" : ".")) : "";- ref_include_fields_vec.emplace_back(ref_include_fields{ref_collection_name, ref_fields, ref_alias,- strategy_enum});- ref_include_fields_vec.back().nested_join_includes = std::move(nested_ref_include_fields_vec);+ ref_include_exclude_fields_vec.emplace_back(ref_include_exclude_fields{ref_collection_name, ref_fields, "",+ ref_alias, strategy_enum});+ ref_include_exclude_fields_vec.back().nested_join_includes = std::move(nested_ref_include_exclude_fields_vec);- // Referenced collection in filter_by is already mentioned in ref_include_fields.+ // Referenced collection in filter_by is already mentioned in include_fields.if (ref_include_coll_names != nullptr) {ref_include_coll_names->collection_names.erase(ref_collection_name);}@@ -1079,9 +1131,10 @@ Option<bool> parse_nested_include(const std::string& include_field_exp,return Option<bool>(true);}-Option<bool> CollectionManager::_initialize_ref_include_fields_vec(const std::string& filter_query,- std::vector<std::string>& include_fields_vec,- std::vector<ref_include_fields>& ref_include_fields_vec) {+Option<bool> CollectionManager::_initialize_ref_include_exclude_fields_vec(const std::string& filter_query,+ std::vector<std::string>& include_fields_vec,+ std::vector<std::string>& exclude_fields_vec,+ std::vector<ref_include_exclude_fields>& ref_include_exclude_fields_vec) {ref_include_collection_names_t* ref_include_coll_names = nullptr;CollectionManager::_get_reference_collection_names(filter_query, ref_include_coll_names);std::unique_ptr<CollectionManager::ref_include_collection_names_t> guard(ref_include_coll_names);@@ -1101,7 +1154,7 @@ Option<bool> CollectionManager::_initialize_ref_include_fields_vec(const std::st// Nested reference include.if (include_field_exp.find('$', 1) != std::string::npos) {- auto parse_op = parse_nested_include(include_field_exp, ref_include_coll_names, ref_include_fields_vec);+ auto parse_op = parse_nested_include(include_field_exp, ref_include_coll_names, ref_include_exclude_fields_vec);if (!parse_op.ok()) {return parse_op;}@@ -1139,46 +1192,77 @@ Option<bool> CollectionManager::_initialize_ref_include_fields_vec(const std::st// In case of "nest" reference doc, `foo` becomes the key with reference doc as value.auto const& nest_ref_doc = strategy_enum == ref_include::nest || strategy_enum == ref_include::nest_array;auto ref_alias = !alias.empty() ? (StringUtils::trim(alias) + (nest_ref_doc ? "" : ".")) : "";- ref_include_fields_vec.emplace_back(ref_include_fields{ref_collection_name, ref_fields, ref_alias,- strategy_enum});+ ref_include_exclude_fields_vec.emplace_back(ref_include_exclude_fields{ref_collection_name, ref_fields, "",+ ref_alias, strategy_enum});- auto open_paren_pos = include_field_exp.find('(');- if (open_paren_pos == std::string::npos) {- continue;- }-- auto reference_collection_name = include_field_exp.substr(1, open_paren_pos - 1);- StringUtils::trim(reference_collection_name);- if (reference_collection_name.empty()) {- continue;- }-- // Referenced collection in filter_by is already mentioned in ref_include_fields.+ // Referenced collection in filter_by is already mentioned in include_fields.if (ref_include_coll_names != nullptr) {- ref_include_coll_names->collection_names.erase(reference_collection_name);+ ref_include_coll_names->collection_names.erase(ref_collection_name);}}- // Get all the fields of the referenced collection in the filter but not mentioned in include_fields.- auto ref_includes = std::ref(ref_include_fields_vec);+ // Get all the fields of the referenced collection mentioned in the filter_by but not in include_fields.+ auto references = std::ref(ref_include_exclude_fields_vec);while (ref_include_coll_names != nullptr) {for (const auto &reference_collection_name: ref_include_coll_names->collection_names) {- ref_includes.get().emplace_back(ref_include_fields{reference_collection_name, "", "", ref_include::nest});+ references.get().emplace_back(ref_include_exclude_fields{reference_collection_name, "", "", ""});}ref_include_coll_names = ref_include_coll_names->nested_include;- if (ref_includes.get().empty()) {+ if (references.get().empty()) {break;}- ref_includes = std::ref(ref_includes.get().front().nested_join_includes);+ references = std::ref(references.get().front().nested_join_includes);+ }++ std::unordered_map<std::string, std::string> ref_excludes;+ std::vector<std::string> result_exclude_fields_vec;+ for (const auto& exclude_field_exp: exclude_fields_vec) {+ if (exclude_field_exp[0] != '$') {+ result_exclude_fields_vec.emplace_back(exclude_field_exp);+ continue;+ }++ // Nested reference exclude.+ if (exclude_field_exp.find('$', 1) != std::string::npos) {+ auto parse_op = parse_nested_exclude(exclude_field_exp, ref_excludes);+ if (!parse_op.ok()) {+ return parse_op;+ }+ continue;+ }++ // Format: $ref_collection_name(field_1, field_2)+ auto parenthesis_index = exclude_field_exp.find('(');+ auto ref_collection_name = exclude_field_exp.substr(1, parenthesis_index - 1);+ auto ref_fields = exclude_field_exp.substr(parenthesis_index + 1, exclude_field_exp.size() - parenthesis_index - 2);+ if (!ref_fields.empty()) {+ ref_excludes[ref_collection_name] = ref_fields;+ }+ }++ if (!ref_excludes.empty()) {+ references = std::ref(ref_include_exclude_fields_vec);+ while (!references.get().empty()) {+ for (auto& ref_include_exclude: references.get()) {+ if (ref_excludes.count(ref_include_exclude.collection_name) == 0) {+ continue;+ }++ ref_include_exclude.exclude_fields = ref_excludes[ref_include_exclude.collection_name];+ }++ references = std::ref(references.get().front().nested_join_includes);+ }}- // Since no field of the collection is mentioned in include_fields, get all the fields.+ // Since no field of the collection being searched is mentioned in include_fields, include all the fields.if (wildcard_include_all) {result_include_fields_vec.clear();}include_fields_vec = std::move(result_include_fields_vec);+ exclude_fields_vec = std::move(result_exclude_fields_vec);return Option<bool>(true);}@@ -1356,7 +1440,7 @@ Option<bool> CollectionManager::do_search(std::map<std::string, std::string>& restd::vector<std::string> include_fields_vec;std::vector<std::string> exclude_fields_vec;- std::vector<ref_include_fields> ref_include_fields_vec;+ std::vector<ref_include_exclude_fields> ref_include_exclude_fields_vec;spp::sparse_hash_set<std::string> include_fields;spp::sparse_hash_set<std::string> exclude_fields;@@ -1540,8 +1624,8 @@ Option<bool> CollectionManager::do_search(std::map<std::string, std::string>& reif(key == FACET_BY){StringUtils::split_facet(val, *find_str_list_it->second);}- else if(key == INCLUDE_FIELDS){- auto op = StringUtils::split_include_fields(val, *find_str_list_it->second);+ else if(key == INCLUDE_FIELDS || key == EXCLUDE_FIELDS){+ auto op = StringUtils::split_include_exclude_fields(val, *find_str_list_it->second);if (!op.ok()) {return op;}@@ -1566,7 +1650,8 @@ Option<bool> CollectionManager::do_search(std::map<std::string, std::string>& reper_page = 0;}- auto initialize_op = _initialize_ref_include_fields_vec(filter_query, include_fields_vec, ref_include_fields_vec);+ auto initialize_op = _initialize_ref_include_exclude_fields_vec(filter_query, include_fields_vec, exclude_fields_vec,+ ref_include_exclude_fields_vec);if (!initialize_op.ok()) {return initialize_op;}@@ -1662,7 +1747,7 @@ Option<bool> CollectionManager::do_search(std::map<std::string, std::string>& reremote_embedding_num_tries,stopwords_set,facet_return_parent,- ref_include_fields_vec,+ ref_include_exclude_fields_vec,drop_tokens_mode_str,prioritize_num_matching_fields,group_missing_values,diff --git a/src/filter_result_iterator.cpp b/src/filter_result_iterator.cppindex b3254aaf..44867fec 100644--- a/src/filter_result_iterator.cpp+++ b/src/filter_result_iterator.cpp@@ -14,23 +14,31 @@#include "posting.h"#include "collection_manager.h"-void filter_result_t::copy_references(const filter_result_t& from, filter_result_t& to) {- if (from.coll_to_references == nullptr) {+void copy_references_helper(const std::map<std::string, reference_filter_result_t>* from,+ std::map<std::string, reference_filter_result_t>*& to, const uint32_t& count) {+ if (from == nullptr) {return;}- auto const& count = from.count;- to.coll_to_references = new std::map<std::string, reference_filter_result_t>[count] {};+ to = new std::map<std::string, reference_filter_result_t>[count] {};for (uint32_t i = 0; i < count; i++) {- if (from.coll_to_references[i].empty()) {+ if (from[i].empty()) {continue;}- auto& ref = to.coll_to_references[i];- ref.insert(from.coll_to_references[i].begin(), from.coll_to_references[i].end());+ auto& ref = to[i];+ ref.insert(from[i].begin(), from[i].end());}}+void reference_filter_result_t::copy_references(const reference_filter_result_t& from, reference_filter_result_t& to) {+ return copy_references_helper(from.coll_to_references, to.coll_to_references, from.count);+}++void filter_result_t::copy_references(const filter_result_t& from, filter_result_t& to) {+ return copy_references_helper(from.coll_to_references, to.coll_to_references, from.count);+}+void filter_result_t::and_filter_results(const filter_result_t& a, const filter_result_t& b, filter_result_t& result) {auto lenA = a.count, lenB = b.count;if (lenA == 0 || lenB == 0) {@@ -660,26 +668,16 @@ void filter_result_iterator_t::init() {return;}- std::vector<std::string> values;- for (uint32_t i = 0; i < result.count; i++) {- values.push_back(std::to_string(result.docs[i]));- }-- filter filter_exp = {get_reference_field_op.get(), std::move(values),- std::vector<NUM_COMPARATOR>(result.count, EQUALS)};-- auto filter_tree_root = new filter_node_t(filter_exp);- std::unique_ptr<filter_node_t> filter_tree_root_guard(filter_tree_root);-- auto fit = filter_result_iterator_t(collection_name, index, filter_tree_root);- auto filter_init_op = fit.init_status();- if (!filter_init_op.ok()) {- status = Option<bool>(filter_init_op.code(), filter_init_op.error());+ auto const& reference_helper_field_name = get_reference_field_op.get();+ auto op = index->do_filtering_with_reference_ids(reference_helper_field_name, ref_collection_name,+ std::move(result));+ if (!op.ok()) {+ status = Option<bool>(op.code(), op.error());validity = invalid;return;}- filter_result = std::move(fit.filter_result);+ filter_result = op.get();}if (filter_result.count == 0) {@@ -1304,6 +1302,7 @@ int filter_result_iterator_t::is_valid(uint32_t id) {}seq_id = id;+ and_filter_iterators();return 1;} else {validity = (left_it->validity == valid || right_it->validity == valid) ? valid : invalid;@@ -1327,6 +1326,7 @@ int filter_result_iterator_t::is_valid(uint32_t id) {}seq_id = id;+ or_filter_iterators();return 1;}}diff --git a/src/index.cpp b/src/index.cppindex 0287f443..d2eb01b6 100644--- a/src/index.cpp+++ b/src/index.cpp@@ -1801,37 +1801,160 @@ Option<bool> Index::do_filtering_with_lock(filter_node_t* const filter_tree_rootreturn filter_init_op;}- filter_result.count = filter_result_iterator.to_filter_id_array(filter_result.docs);+ if (filter_result_iterator.reference.empty()) {+ filter_result.count = filter_result_iterator.to_filter_id_array(filter_result.docs);+ return Option(true);+ }+ filter_result_iterator.compute_result();+ if (filter_result_iterator.approx_filter_ids_length == 0) {+ return Option(true);+ }++ uint32_t count = filter_result_iterator.approx_filter_ids_length, dummy;+ auto ref_filter_result = new filter_result_t();+ std::unique_ptr<filter_result_t> ref_filter_result_guard(ref_filter_result);+ filter_result_iterator.get_n_ids(count, dummy, nullptr, 0, ref_filter_result);++ if (filter_result_iterator.validity == filter_result_iterator_t::timed_out) {+ return Option<bool>(true);+ }++ filter_result = std::move(*ref_filter_result);return Option(true);}+void aggregate_nested_references(single_filter_result_t *const reference_result,+ reference_filter_result_t& ref_filter_result) {+ // Add reference doc id in result.+ auto temp_docs = new uint32_t[ref_filter_result.count + 1];+ std::copy(ref_filter_result.docs, ref_filter_result.docs + ref_filter_result.count, temp_docs);+ temp_docs[ref_filter_result.count] = reference_result->seq_id;++ delete[] ref_filter_result.docs;+ ref_filter_result.docs = temp_docs;+ ref_filter_result.count++;+ ref_filter_result.is_reference_array_field = false;++ // Add references of the reference doc id in result.+ auto& references = ref_filter_result.coll_to_references;+ auto temp_references = new std::map<std::string, reference_filter_result_t>[ref_filter_result.count] {};+ for (uint32_t i = 0; i < ref_filter_result.count - 1; i++) {+ temp_references[i] = std::move(references[i]);+ }+ temp_references[ref_filter_result.count - 1] = std::move(reference_result->reference_filter_results);++ delete[] references;+ references = temp_references;+}+Option<bool> Index::do_reference_filtering_with_lock(filter_node_t* const filter_tree_root,filter_result_t& filter_result,- const std::string& collection_name,+ const std::string& ref_collection_name,const std::string& reference_helper_field_name) const {std::shared_lock lock(mutex);- auto filter_result_iterator = filter_result_iterator_t(collection_name, this, filter_tree_root,- search_begin_us, search_stop_us);- auto filter_init_op = filter_result_iterator.init_status();+ auto ref_filter_result_iterator = filter_result_iterator_t(ref_collection_name, this, filter_tree_root,+ search_begin_us, search_stop_us);+ auto filter_init_op = ref_filter_result_iterator.init_status();if (!filter_init_op.ok()) {return filter_init_op;}- uint32_t* reference_docs = nullptr;- uint32_t count = filter_result_iterator.to_filter_id_array(reference_docs);- std::unique_ptr<uint32_t[]> docs_guard(reference_docs);-- if (count == 0) {+ ref_filter_result_iterator.compute_result();+ if (ref_filter_result_iterator.approx_filter_ids_length == 0) {return Option(true);}- if (search_schema.at(reference_helper_field_name).is_singular()) {+ uint32_t count = ref_filter_result_iterator.approx_filter_ids_length, dummy;+ auto ref_filter_result = new filter_result_t();+ std::unique_ptr<filter_result_t> ref_filter_result_guard(ref_filter_result);+ ref_filter_result_iterator.get_n_ids(count, dummy, nullptr, 0, ref_filter_result);++ if (ref_filter_result_iterator.validity == filter_result_iterator_t::timed_out) {+ return Option<bool>(true);+ }++ uint32_t* reference_docs = ref_filter_result->docs;+ ref_filter_result->docs = nullptr;+ std::unique_ptr<uint32_t[]> docs_guard(reference_docs);++ auto const is_nested_join = !ref_filter_result_iterator.reference.empty();+ if (search_schema.at(reference_helper_field_name).is_singular()) { // Only one reference per doc.+ if (sort_index.count(reference_helper_field_name) == 0) {+ return Option<bool>(400, "`" + reference_helper_field_name + "` is not present in sort index.");+ }+ auto const& ref_index = *sort_index.at(reference_helper_field_name);++ if (is_nested_join) {+ // In case of nested join, we need to collect all the doc ids from the reference ids along with their references.+ std::vector<std::pair<uint32_t, single_filter_result_t*>> id_pairs;+ std::unordered_set<uint32_t> unique_doc_ids;++ for (uint32_t i = 0; i < count; i++) {+ auto& reference_doc_id = reference_docs[i];+ auto reference_doc_references = std::move(ref_filter_result->coll_to_references[i]);+ if (ref_index.count(reference_doc_id) == 0) { // Reference field might be optional.+ continue;+ }+ auto doc_id = ref_index.at(reference_doc_id);++ id_pairs.emplace_back(std::make_pair(doc_id, new single_filter_result_t(reference_doc_id,+ std::move(reference_doc_references),+ false)));+ unique_doc_ids.insert(doc_id);+ }++ if (id_pairs.empty()) {+ return Option(true);+ }++ std::sort(id_pairs.begin(), id_pairs.end(), [](auto const& left, auto const& right) {+ return left.first < right.first;+ });++ filter_result.count = unique_doc_ids.size();+ filter_result.docs = new uint32_t[unique_doc_ids.size()];+ filter_result.coll_to_references = new std::map<std::string, reference_filter_result_t>[unique_doc_ids.size()] {};++ reference_filter_result_t previous_doc_references;+ for (uint32_t i = 0, previous_doc = id_pairs[0].first + 1, result_index = 0; i < id_pairs.size(); i++) {+ auto const& current_doc = id_pairs[i].first;+ auto& reference_result = id_pairs[i].second;++ if (current_doc != previous_doc) {+ filter_result.docs[result_index] = current_doc;+ if (result_index > 0) {+ std::map<std::string, reference_filter_result_t> references;+ references[ref_collection_name] = std::move(previous_doc_references);+ filter_result.coll_to_references[result_index - 1] = std::move(references);+ }++ result_index++;+ previous_doc = current_doc;+ aggregate_nested_references(reference_result, previous_doc_references);+ } else {+ aggregate_nested_references(reference_result, previous_doc_references);+ }+ }++ if (previous_doc_references.count != 0) {+ std::map<std::string, reference_filter_result_t> references;+ references[ref_collection_name] = std::move(previous_doc_references);+ filter_result.coll_to_references[filter_result.count - 1] = std::move(references);+ }++ for (auto &item: id_pairs) {+ delete item.second;+ }++ return Option(true);+ }+// Collect all the doc ids from the reference ids.std::vector<std::pair<uint32_t, uint32_t>> id_pairs;std::unordered_set<uint32_t> unique_doc_ids;- auto const& ref_index = *sort_index.at(reference_helper_field_name);+for (uint32_t i = 0; i < count; i++) {auto& reference_doc_id = reference_docs[i];if (ref_index.count(reference_doc_id) == 0) { // Reference field might be optional.@@ -1839,10 +1962,14 @@ Option<bool> Index::do_reference_filtering_with_lock(filter_node_t* const filter}auto doc_id = ref_index.at(reference_doc_id);- id_pairs.emplace_back(std::pair(doc_id, reference_doc_id));+ id_pairs.emplace_back(std::make_pair(doc_id, reference_doc_id));unique_doc_ids.insert(doc_id);}+ if (id_pairs.empty()) {+ return Option(true);+ }+std::sort(id_pairs.begin(), id_pairs.end(), [](auto const& left, auto const& right) {return left.first < right.first;});@@ -1861,9 +1988,11 @@ Option<bool> Index::do_reference_filtering_with_lock(filter_node_t* const filterif (result_index > 0) {auto& reference_result = filter_result.coll_to_references[result_index - 1];- auto r = reference_filter_result_t(previous_doc_references.size(), new uint32_t[previous_doc_references.size()]);+ auto r = reference_filter_result_t(previous_doc_references.size(),+ new uint32_t[previous_doc_references.size()],+ false);std::copy(previous_doc_references.begin(), previous_doc_references.end(), r.docs);- reference_result[collection_name] = std::move(r);+ reference_result[ref_collection_name] = std::move(r);previous_doc_references.clear();}@@ -1879,41 +2008,317 @@ Option<bool> Index::do_reference_filtering_with_lock(filter_node_t* const filterif (!previous_doc_references.empty()) {auto& reference_result = filter_result.coll_to_references[filter_result.count - 1];- auto r = reference_filter_result_t(previous_doc_references.size(), new uint32_t[previous_doc_references.size()]);+ auto r = reference_filter_result_t(previous_doc_references.size(),+ new uint32_t[previous_doc_references.size()],+ false);std::copy(previous_doc_references.begin(), previous_doc_references.end(), r.docs);- reference_result[collection_name] = std::move(r);+ reference_result[ref_collection_name] = std::move(r);}return Option(true);}- size_t ids_len = 0;- uint32_t *ids = nullptr;+ // Multiple references per doc.+ if (reference_index.count(reference_helper_field_name) == 0) {+ return Option<bool>(400, "`" + reference_helper_field_name + "` is not present in reference index.");+ }auto& ref_index = *reference_index.at(reference_helper_field_name);++ if (is_nested_join) {+ // In case of nested join, we need to collect all the doc ids from the reference ids along with their references.+ std::vector<std::pair<uint32_t, single_filter_result_t*>> id_pairs;+ std::unordered_set<uint32_t> unique_doc_ids;++ for (uint32_t i = 0; i < count; i++) {+ auto& reference_doc_id = reference_docs[i];+ auto reference_doc_references = std::move(ref_filter_result->coll_to_references[i]);+ size_t doc_ids_len = 0;+ uint32_t* doc_ids = nullptr;++ ref_index.search(EQUALS, reference_doc_id, &doc_ids, doc_ids_len);++ for (size_t j = 0; j < doc_ids_len; j++) {+ auto doc_id = doc_ids[j];+ auto reference_doc_references_copy = reference_doc_references;+ id_pairs.emplace_back(std::make_pair(doc_id, new single_filter_result_t(reference_doc_id,+ std::move(reference_doc_references_copy),+ false)));+ unique_doc_ids.insert(doc_id);+ }+ delete[] doc_ids;+ }++ if (id_pairs.empty()) {+ return Option(true);+ }++ std::sort(id_pairs.begin(), id_pairs.end(), [](auto const& left, auto const& right) {+ return left.first < right.first;+ });++ filter_result.count = unique_doc_ids.size();+ filter_result.docs = new uint32_t[unique_doc_ids.size()];+ filter_result.coll_to_references = new std::map<std::string, reference_filter_result_t>[unique_doc_ids.size()] {};++ reference_filter_result_t previous_doc_references;+ for (uint32_t i = 0, previous_doc = id_pairs[0].first + 1, result_index = 0; i < id_pairs.size(); i++) {+ auto const& current_doc = id_pairs[i].first;+ auto& reference_result = id_pairs[i].second;++ if (current_doc != previous_doc) {+ filter_result.docs[result_index] = current_doc;+ if (result_index > 0) {+ std::map<std::string, reference_filter_result_t> references;+ references[ref_collection_name] = std::move(previous_doc_references);+ filter_result.coll_to_references[result_index - 1] = std::move(references);+ }++ result_index++;+ previous_doc = current_doc;+ aggregate_nested_references(reference_result, previous_doc_references);+ } else {+ aggregate_nested_references(reference_result, previous_doc_references);+ }+ }++ if (previous_doc_references.count != 0) {+ std::map<std::string, reference_filter_result_t> references;+ references[ref_collection_name] = std::move(previous_doc_references);+ filter_result.coll_to_references[filter_result.count - 1] = std::move(references);+ }++ for (auto &item: id_pairs) {+ delete item.second;+ }++ return Option<bool>(true);+ }++ std::vector<std::pair<uint32_t, uint32_t>> id_pairs;+ std::unordered_set<uint32_t> unique_doc_ids;+for (uint32_t i = 0; i < count; i++) {auto& reference_doc_id = reference_docs[i];- ref_index.search(EQUALS, reference_doc_id, &ids, ids_len);+ size_t doc_ids_len = 0;+ uint32_t* doc_ids = nullptr;++ ref_index.search(EQUALS, reference_doc_id, &doc_ids, doc_ids_len);++ for (size_t j = 0; j < doc_ids_len; j++) {+ auto doc_id = doc_ids[j];+ id_pairs.emplace_back(std::make_pair(doc_id, reference_doc_id));+ unique_doc_ids.insert(doc_id);+ }+ delete[] doc_ids;+ }++ if (id_pairs.empty()) {+ return Option(true);}- filter_result.count = ids_len;- filter_result.docs = new uint32_t[ids_len];- filter_result.coll_to_references = new std::map<std::string, reference_filter_result_t>[ids_len] {};+ std::sort(id_pairs.begin(), id_pairs.end(), [](auto const& left, auto const& right) {+ return left.first < right.first;+ });++ filter_result.count = unique_doc_ids.size();+ filter_result.docs = new uint32_t[unique_doc_ids.size()];+ filter_result.coll_to_references = new std::map<std::string, reference_filter_result_t>[unique_doc_ids.size()] {};- auto& num_index = *numerical_index.at(reference_helper_field_name);- for (size_t i = 0; i < ids_len; i++) {- filter_result.docs[i] = ids[i];+ std::vector<uint32_t> previous_doc_references;+ for (uint32_t i = 0, previous_doc = id_pairs[0].first + 1, result_index = 0; i < id_pairs.size(); i++) {+ auto const& current_doc = id_pairs[i].first;+ auto const& reference_doc_id = id_pairs[i].second;- reference_filter_result_t reference_result;- size_t len = 0;- num_index.search(EQUALS, ids[i], &reference_result.docs, len);- reference_result.count = len;- filter_result.coll_to_references[i][collection_name] = std::move(reference_result);+ if (current_doc != previous_doc) {+ filter_result.docs[result_index] = current_doc;+ if (result_index > 0) {+ auto& reference_result = filter_result.coll_to_references[result_index - 1];++ auto r = reference_filter_result_t(previous_doc_references.size(), new uint32_t[previous_doc_references.size()]);+ std::copy(previous_doc_references.begin(), previous_doc_references.end(), r.docs);+ reference_result[ref_collection_name] = std::move(r);++ previous_doc_references.clear();+ }++ result_index++;+ previous_doc = current_doc;+ previous_doc_references.push_back(reference_doc_id);+ } else {+ previous_doc_references.push_back(reference_doc_id);+ }+ }++ if (!previous_doc_references.empty()) {+ auto& reference_result = filter_result.coll_to_references[filter_result.count - 1];++ auto r = reference_filter_result_t(previous_doc_references.size(), new uint32_t[previous_doc_references.size()]);+ std::copy(previous_doc_references.begin(), previous_doc_references.end(), r.docs);+ reference_result[ref_collection_name] = std::move(r);}- delete [] ids;return Option(true);}+Option<filter_result_t> Index::do_filtering_with_reference_ids(const std::string& reference_helper_field_name,+ const std::string& ref_collection_name,+ filter_result_t&& ref_filter_result) const {+ filter_result_t filter_result;+ auto const& count = ref_filter_result.count;+ auto const& reference_docs = ref_filter_result.docs;+ auto const is_nested_join = ref_filter_result.coll_to_references != nullptr;++ if (count == 0) {+ return Option<filter_result_t>(filter_result);+ }++ if (numerical_index.count(reference_helper_field_name) == 0) {+ return Option<filter_result_t>(400, "`" + reference_helper_field_name + "` is not present in index.");+ }+ auto num_tree = numerical_index.at(reference_helper_field_name);++ if (is_nested_join) {+ // In case of nested join, we need to collect all the doc ids from the reference ids along with their references.+ std::vector<std::pair<uint32_t, single_filter_result_t*>> id_pairs;+ std::unordered_set<uint32_t> unique_doc_ids;++ for (uint32_t i = 0; i < count; i++) {+ auto& reference_doc_id = reference_docs[i];+ auto reference_doc_references = std::move(ref_filter_result.coll_to_references[i]);+ size_t doc_ids_len = 0;+ uint32_t* doc_ids = nullptr;++ num_tree->search(NUM_COMPARATOR::EQUALS, reference_doc_id, &doc_ids, doc_ids_len);++ for (size_t j = 0; j < doc_ids_len; j++) {+ auto doc_id = doc_ids[j];+ auto reference_doc_references_copy = reference_doc_references;+ id_pairs.emplace_back(std::make_pair(doc_id, new single_filter_result_t(reference_doc_id,+ std::move(reference_doc_references_copy),+ false)));+ unique_doc_ids.insert(doc_id);+ }++ delete[] doc_ids;+ }++ if (id_pairs.empty()) {+ return Option(filter_result);+ }++ std::sort(id_pairs.begin(), id_pairs.end(), [](auto const& left, auto const& right) {+ return left.first < right.first;+ });++ filter_result.count = unique_doc_ids.size();+ filter_result.docs = new uint32_t[unique_doc_ids.size()];+ filter_result.coll_to_references = new std::map<std::string, reference_filter_result_t>[unique_doc_ids.size()] {};++ reference_filter_result_t previous_doc_references;+ for (uint32_t i = 0, previous_doc = id_pairs[0].first + 1, result_index = 0; i < id_pairs.size(); i++) {+ auto const& current_doc = id_pairs[i].first;+ auto& reference_result = id_pairs[i].second;++ if (current_doc != previous_doc) {+ filter_result.docs[result_index] = current_doc;+ if (result_index > 0) {+ std::map<std::string, reference_filter_result_t> references;+ references[ref_collection_name] = std::move(previous_doc_references);+ filter_result.coll_to_references[result_index - 1] = std::move(references);+ }++ result_index++;+ previous_doc = current_doc;+ aggregate_nested_references(reference_result, previous_doc_references);+ } else {+ aggregate_nested_references(reference_result, previous_doc_references);+ }+ }++ if (previous_doc_references.count != 0) {+ std::map<std::string, reference_filter_result_t> references;+ references[ref_collection_name] = std::move(previous_doc_references);+ filter_result.coll_to_references[filter_result.count - 1] = std::move(references);+ }++ for (auto &item: id_pairs) {+ delete item.second;+ }++ return Option<filter_result_t>(filter_result);+ }++ // Collect all the doc ids from the reference ids.+ std::vector<std::pair<uint32_t, uint32_t>> id_pairs;+ std::unordered_set<uint32_t> unique_doc_ids;++ for (uint32_t i = 0; i < count; i++) {+ auto& reference_doc_id = reference_docs[i];+ size_t doc_ids_len = 0;+ uint32_t* doc_ids = nullptr;++ num_tree->search(NUM_COMPARATOR::EQUALS, reference_doc_id, &doc_ids, doc_ids_len);++ for (size_t j = 0; j < doc_ids_len; j++) {+ auto doc_id = doc_ids[j];+ id_pairs.emplace_back(std::make_pair(doc_id, reference_doc_id));+ unique_doc_ids.insert(doc_id);+ }+ delete[] doc_ids;+ }++ if (id_pairs.empty()) {+ return Option(filter_result);+ }++ std::sort(id_pairs.begin(), id_pairs.end(), [](auto const& left, auto const& right) {+ return left.first < right.first;+ });++ filter_result.count = unique_doc_ids.size();+ filter_result.docs = new uint32_t[unique_doc_ids.size()];+ filter_result.coll_to_references = new std::map<std::string, reference_filter_result_t>[unique_doc_ids.size()] {};++ std::vector<uint32_t> previous_doc_references;+ for (uint32_t i = 0, previous_doc = id_pairs[0].first + 1, result_index = 0; i < id_pairs.size(); i++) {+ auto const& current_doc = id_pairs[i].first;+ auto const& reference_doc_id = id_pairs[i].second;++ if (current_doc != previous_doc) {+ filter_result.docs[result_index] = current_doc;+ if (result_index > 0) {+ auto& reference_result = filter_result.coll_to_references[result_index - 1];++ auto r = reference_filter_result_t(previous_doc_references.size(),+ new uint32_t[previous_doc_references.size()],+ false);+ std::copy(previous_doc_references.begin(), previous_doc_references.end(), r.docs);+ reference_result[ref_collection_name] = std::move(r);++ previous_doc_references.clear();+ }++ result_index++;+ previous_doc = current_doc;+ previous_doc_references.push_back(reference_doc_id);+ } else {+ previous_doc_references.push_back(reference_doc_id);+ }+ }++ if (!previous_doc_references.empty()) {+ auto& reference_result = filter_result.coll_to_references[filter_result.count - 1];++ auto r = reference_filter_result_t(previous_doc_references.size(),+ new uint32_t[previous_doc_references.size()],+ false);+ std::copy(previous_doc_references.begin(), previous_doc_references.end(), r.docs);+ reference_result[ref_collection_name] = std::move(r);+ }++ return Option<filter_result_t>(filter_result);+}+Option<bool> Index::run_search(search_args* search_params, const std::string& collection_name,facet_index_type_t facet_index_type) {return search(search_params->field_query_tokens,diff --git a/src/string_utils.cpp b/src/string_utils.cppindex cd74e96a..781929ab 100644--- a/src/string_utils.cpp+++ b/src/string_utils.cpp@@ -492,24 +492,24 @@ Option<bool> StringUtils::tokenize_filter_query(const std::string& filter_query,return Option<bool>(true);}-Option<bool> StringUtils::split_reference_include_fields(const std::string& include_fields,- size_t& index,- std::string& token) {- auto ref_include_error = Option<bool>(400, "Invalid reference in include_fields, expected `$CollectionName(fieldA, ...)`.");- auto const& size = include_fields.size();+Option<bool> StringUtils::split_reference_include_exclude_fields(const std::string& include_exclude_fields,+ size_t& index, std::string& token) {+ auto ref_include_error = Option<bool>(400, "Invalid reference `" + include_exclude_fields + "` in include_fields/"+ "exclude_fields, expected `$CollectionName(fieldA, ...)`.");+ auto const& size = include_exclude_fields.size();size_t start_index = index;- while(++index < size && include_fields[index] != '(') {}+ while(++index < size && include_exclude_fields[index] != '(') {}if (index >= size) {return ref_include_error;}- // In case of nested join, the reference include field could have parenthesis inside it.+ // In case of nested join, the reference include/exclude field could have parenthesis inside it.int parenthesis_count = 1;while (++index < size && parenthesis_count > 0) {- if (include_fields[index] == '(') {+ if (include_exclude_fields[index] == '(') {parenthesis_count++;- } else if (include_fields[index] == ')') {+ } else if (include_exclude_fields[index] == ')') {parenthesis_count--;}}@@ -525,34 +525,35 @@ Option<bool> StringUtils::split_reference_include_fields(const std::string& incl// ...^// $ref_include( $nested_ref_include(foo :merge)as nest :merge ) as ref// ...^- auto closing_parenthesis_pos = include_fields.find(')', index);- auto comma_pos = include_fields.find(',', index);- auto colon_pos = include_fields.find(':', index);- auto alias_start_pos = include_fields.find(" as ", index);+ auto closing_parenthesis_pos = include_exclude_fields.find(')', index);+ auto comma_pos = include_exclude_fields.find(',', index);+ auto colon_pos = include_exclude_fields.find(':', index);+ auto alias_start_pos = include_exclude_fields.find(" as ", index);auto alias_end_pos = std::min(std::min(closing_parenthesis_pos, comma_pos), colon_pos);std::string alias;if (alias_start_pos != std::string::npos && alias_start_pos < alias_end_pos) {- alias = include_fields.substr(alias_start_pos, alias_end_pos - alias_start_pos);+ alias = include_exclude_fields.substr(alias_start_pos, alias_end_pos - alias_start_pos);}- token = include_fields.substr(start_index, index - start_index) + " " + trim(alias);+ token = include_exclude_fields.substr(start_index, index - start_index) + " " + trim(alias);trim(token);index = alias_end_pos;return Option<bool>(true);}-Option<bool> StringUtils::split_include_fields(const std::string& include_fields, std::vector<std::string>& tokens) {+Option<bool> StringUtils::split_include_exclude_fields(const std::string& include_exclude_fields,+ std::vector<std::string>& tokens) {std::string token;- auto const& size = include_fields.size();+ auto const& size = include_exclude_fields.size();for (size_t i = 0; i < size;) {- auto c = include_fields[i];+ auto c = include_exclude_fields[i];if (c == ' ') {i++;continue;- } else if (c == '$') { // Reference include+ } else if (c == '$') { // Reference include/excludestd::string ref_include_token;- auto split_op = split_reference_include_fields(include_fields, i, ref_include_token);+ auto split_op = split_reference_include_exclude_fields(include_exclude_fields, i, ref_include_token);if (!split_op.ok()) {return split_op;}@@ -561,8 +562,8 @@ Option<bool> StringUtils::split_include_fields(const std::string& include_fieldscontinue;}- auto comma_pos = include_fields.find(',', i);- token = include_fields.substr(i, (comma_pos == std::string::npos ? size : comma_pos) - i);+ auto comma_pos = include_exclude_fields.find(',', i);+ token = include_exclude_fields.substr(i, (comma_pos == std::string::npos ? size : comma_pos) - i);trim(token);if (!token.empty()) {tokens.push_back(token);diff --git a/test/collection_join_test.cpp b/test/collection_join_test.cppindex f254a00a..2fb3c103 100644--- a/test/collection_join_test.cpp+++ b/test/collection_join_test.cpp@@ -1827,6 +1827,559 @@ TEST_F(CollectionJoinTest, FilterByNReferences) {collectionManager.drop_collection("Links");}+TEST_F(CollectionJoinTest, FilterByNestedReferences) {+ auto schema_json =+ R"({+ "name": "Coll_A",+ "fields": [+ {"name": "title", "type": "string"}+ ]+ })"_json;+ std::vector<nlohmann::json> documents = {+ R"({+ "title": "coll_a_0"+ })"_json,+ R"({+ "title": "coll_a_1"+ })"_json+ };+ auto collection_create_op = collectionManager.create_collection(schema_json);+ ASSERT_TRUE(collection_create_op.ok());+ for (auto const &json: documents) {+ auto add_op = collection_create_op.get()->add(json.dump());+ if (!add_op.ok()) {+ LOG(INFO) << add_op.error();+ }+ ASSERT_TRUE(add_op.ok());+ }+ schema_json =+ R"({+ "name": "Coll_B",+ "fields": [+ {"name": "title", "type": "string"},+ {"name": "ref_coll_a", "type": "string", "reference": "Coll_A.id"}+ ]+ })"_json;+ documents = {+ R"({+ "title": "coll_b_0",+ "ref_coll_a": "1"+ })"_json,+ R"({+ "title": "coll_b_1",+ "ref_coll_a": "0"+ })"_json,+ R"({+ "title": "coll_b_2",+ "ref_coll_a": "0"+ })"_json+ };+ collection_create_op = collectionManager.create_collection(schema_json);+ ASSERT_TRUE(collection_create_op.ok());+ for (auto const &json: documents) {+ auto add_op = collection_create_op.get()->add(json.dump());+ if (!add_op.ok()) {+ LOG(INFO) << add_op.error();+ }+ ASSERT_TRUE(add_op.ok());+ }+ schema_json =+ R"({+ "name": "Coll_C",+ "fields": [+ {"name": "title", "type": "string"},+ {"name": "ref_coll_b", "type": "string[]", "reference": "Coll_B.id"}+ ]+ })"_json;+ documents = {+ R"({+ "title": "coll_c_0",+ "ref_coll_b": ["0"]+ })"_json,+ R"({+ "title": "coll_c_1",+ "ref_coll_b": ["1"]+ })"_json,+ R"({+ "title": "coll_c_2",+ "ref_coll_b": ["0", "1"]+ })"_json,+ R"({+ "title": "coll_c_3",+ "ref_coll_b": ["2"]+ })"_json+ };+ collection_create_op = collectionManager.create_collection(schema_json);+ ASSERT_TRUE(collection_create_op.ok());+ for (auto const &json: documents) {+ auto add_op = collection_create_op.get()->add(json.dump());+ if (!add_op.ok()) {+ LOG(INFO) << add_op.error();+ }+ ASSERT_TRUE(add_op.ok());+ }++ std::map<std::string, std::string> req_params = {+ {"collection", "Coll_A"},+ {"q", "*"},+ {"filter_by", "$Coll_B($Coll_C(id: [1, 3]))"},+ {"include_fields", "title, $Coll_B(title, $Coll_C(title))"}+ };+ nlohmann::json embedded_params;+ std::string json_res;+ auto now_ts = std::chrono::duration_cast<std::chrono::microseconds>(+ std::chrono::system_clock::now().time_since_epoch()).count();++ auto search_op = collectionManager.do_search(req_params, embedded_params, json_res, now_ts);+ ASSERT_TRUE(search_op.ok());++ nlohmann::json res_obj = nlohmann::json::parse(json_res);+ // coll_b_1 <- coll_c_1+ // coll_a_0 <+ // coll_b_2 <- coll_c_3+ ASSERT_EQ(1, res_obj["found"].get<size_t>());+ ASSERT_EQ(1, res_obj["hits"].size());+ ASSERT_EQ(2, res_obj["hits"][0]["document"].size());+ ASSERT_EQ("coll_a_0", res_obj["hits"][0]["document"]["title"]);+ ASSERT_EQ(2, res_obj["hits"][0]["document"]["Coll_B"].size());+ ASSERT_EQ("coll_b_1", res_obj["hits"][0]["document"]["Coll_B"][0]["title"]);+ ASSERT_EQ(1, res_obj["hits"][0]["document"]["Coll_B"][0]["Coll_C"].size());+ ASSERT_EQ("coll_c_1", res_obj["hits"][0]["document"]["Coll_B"][0]["Coll_C"][0]["title"]);+ ASSERT_EQ("coll_b_2", res_obj["hits"][0]["document"]["Coll_B"][1]["title"]);+ ASSERT_EQ(1, res_obj["hits"][0]["document"]["Coll_B"][1]["Coll_C"].size());+ ASSERT_EQ("coll_c_3", res_obj["hits"][0]["document"]["Coll_B"][1]["Coll_C"][0]["title"]);++ req_params = {+ {"collection", "Coll_A"},+ {"q", "*"},+ {"filter_by", "$Coll_B($Coll_C(id: != 0))"},+ {"include_fields", "title, $Coll_B(title, $Coll_C(title):nest_array)"}+ };+ search_op = collectionManager.do_search(req_params, embedded_params, json_res, now_ts);+ ASSERT_TRUE(search_op.ok());++ res_obj = nlohmann::json::parse(json_res);+ // coll_a_1 <- coll_b_0 <- coll_c_2+ //+ // coll_b_1 <- coll_c_1, coll_c_2+ // coll_a_0 <+ // coll_b_2 <- coll_c_3+ ASSERT_EQ(2, res_obj["found"].get<size_t>());+ ASSERT_EQ(2, res_obj["hits"].size());+ ASSERT_EQ(2, res_obj["hits"][0]["document"].size());+ ASSERT_EQ("coll_a_1", res_obj["hits"][0]["document"]["title"]);+ ASSERT_EQ(1, res_obj["hits"][0]["document"]["Coll_B"].size());+ ASSERT_EQ("coll_b_0", res_obj["hits"][0]["document"]["Coll_B"][0]["title"]);+ ASSERT_EQ(1, res_obj["hits"][0]["document"]["Coll_B"][0]["Coll_C"].size());+ ASSERT_EQ("coll_c_2", res_obj["hits"][0]["document"]["Coll_B"][0]["Coll_C"][0]["title"]);++ ASSERT_EQ("coll_a_0", res_obj["hits"][1]["document"]["title"]);+ ASSERT_EQ(2, res_obj["hits"][1]["document"]["Coll_B"].size());+ ASSERT_EQ("coll_b_1", res_obj["hits"][1]["document"]["Coll_B"][0]["title"]);+ ASSERT_EQ(2, res_obj["hits"][1]["document"]["Coll_B"][0]["Coll_C"].size());+ ASSERT_EQ("coll_c_1", res_obj["hits"][1]["document"]["Coll_B"][0]["Coll_C"][0]["title"]);+ ASSERT_EQ("coll_c_2", res_obj["hits"][1]["document"]["Coll_B"][0]["Coll_C"][1]["title"]);+ ASSERT_EQ("coll_b_2", res_obj["hits"][1]["document"]["Coll_B"][1]["title"]);+ ASSERT_EQ(1, res_obj["hits"][1]["document"]["Coll_B"][1]["Coll_C"].size());+ ASSERT_EQ("coll_c_3", res_obj["hits"][1]["document"]["Coll_B"][1]["Coll_C"][0]["title"]);++ req_params = {+ {"collection", "Coll_C"},+ {"q", "*"},+ {"filter_by", "$Coll_B($Coll_A(id: 0))"},+ {"include_fields", "title, $Coll_B(title, $Coll_A(title))"}+ };+ search_op = collectionManager.do_search(req_params, embedded_params, json_res, now_ts);+ ASSERT_TRUE(search_op.ok());++ res_obj = nlohmann::json::parse(json_res);+ // coll_c_3 -> coll_b_2 -> coll_a_0+ //+ // coll_c_2 -> coll_b_1 -> coll_a_0+ //+ // coll_c_1 -> coll_b_1 -> coll_a_0+ ASSERT_EQ(3, res_obj["found"].get<size_t>());+ ASSERT_EQ(3, res_obj["hits"].size());+ ASSERT_EQ(2, res_obj["hits"][0]["document"].size());+ ASSERT_EQ("coll_c_3", res_obj["hits"][0]["document"]["title"]);+ ASSERT_EQ(2, res_obj["hits"][0]["document"]["Coll_B"].size());+ ASSERT_EQ("coll_b_2", res_obj["hits"][0]["document"]["Coll_B"]["title"]);+ ASSERT_EQ(1, res_obj["hits"][0]["document"]["Coll_B"]["Coll_A"].size());+ ASSERT_EQ("coll_a_0", res_obj["hits"][0]["document"]["Coll_B"]["Coll_A"]["title"]);++ ASSERT_EQ(2, res_obj["hits"][1]["document"].size());+ ASSERT_EQ("coll_c_2", res_obj["hits"][1]["document"]["title"]);+ ASSERT_EQ(2, res_obj["hits"][1]["document"]["Coll_B"].size());+ ASSERT_EQ("coll_b_1", res_obj["hits"][1]["document"]["Coll_B"]["title"]);+ ASSERT_EQ(1, res_obj["hits"][1]["document"]["Coll_B"]["Coll_A"].size());+ ASSERT_EQ("coll_a_0", res_obj["hits"][1]["document"]["Coll_B"]["Coll_A"]["title"]);++ ASSERT_EQ(2, res_obj["hits"][2]["document"].size());+ ASSERT_EQ("coll_c_1", res_obj["hits"][2]["document"]["title"]);+ ASSERT_EQ(2, res_obj["hits"][2]["document"]["Coll_B"].size());+ ASSERT_EQ("coll_b_1", res_obj["hits"][2]["document"]["Coll_B"]["title"]);+ ASSERT_EQ(1, res_obj["hits"][2]["document"]["Coll_B"]["Coll_A"].size());+ ASSERT_EQ("coll_a_0", res_obj["hits"][2]["document"]["Coll_B"]["Coll_A"]["title"]);++ schema_json =+ R"({+ "name": "Coll_D",+ "fields": [+ {"name": "title", "type": "string"},+ {"name": "ref_coll_c", "type": "string[]", "reference": "Coll_C.id"}+ ]+ })"_json;+ documents = {+ R"({+ "title": "coll_d_0",+ "ref_coll_c": []+ })"_json,+ R"({+ "title": "coll_d_1",+ "ref_coll_c": ["1", "3"]+ })"_json,+ R"({+ "title": "coll_d_2",+ "ref_coll_c": ["2", "3"]+ })"_json,+ };+ collection_create_op = collectionManager.create_collection(schema_json);+ ASSERT_TRUE(collection_create_op.ok());+ for (auto const &json: documents) {+ auto add_op = collection_create_op.get()->add(json.dump());+ if (!add_op.ok()) {+ LOG(INFO) << add_op.error();+ }+ ASSERT_TRUE(add_op.ok());+ }++ req_params = {+ {"collection", "Coll_B"},+ {"q", "*"},+ {"filter_by", "$Coll_C($Coll_D(id: *))"},+ {"include_fields", "title, $Coll_C(title, $Coll_D(title:nest_array):nest_array)"}+ };+ search_op = collectionManager.do_search(req_params, embedded_params, json_res, now_ts);+ ASSERT_TRUE(search_op.ok());++ res_obj = nlohmann::json::parse(json_res);+ // coll_b_2 <- coll_c_3 <- coll_d_1, coll_d_2+ //+ // coll_c_1 <- coll_d_1+ // coll_b_1 <+ // coll_c_2 <- coll_d_2+ //+ // coll_b_0 <- coll_c_2 <- coll_d_2+ ASSERT_EQ(3, res_obj["found"].get<size_t>());+ ASSERT_EQ(3, res_obj["hits"].size());+ ASSERT_EQ(2, res_obj["hits"][0]["document"].size());+ ASSERT_EQ("coll_b_2", res_obj["hits"][0]["document"]["title"]);+ ASSERT_EQ(1, res_obj["hits"][0]["document"]["Coll_C"].size());+ ASSERT_EQ("coll_c_3", res_obj["hits"][0]["document"]["Coll_C"][0]["title"]);+ ASSERT_EQ(2, res_obj["hits"][0]["document"]["Coll_C"][0]["Coll_D"].size());+ ASSERT_EQ("coll_d_1", res_obj["hits"][0]["document"]["Coll_C"][0]["Coll_D"][0]["title"]);+ ASSERT_EQ("coll_d_2", res_obj["hits"][0]["document"]["Coll_C"][0]["Coll_D"][1]["title"]);++ ASSERT_EQ(2, res_obj["hits"][1]["document"].size());+ ASSERT_EQ("coll_b_1", res_obj["hits"][1]["document"]["title"]);+ ASSERT_EQ(2, res_obj["hits"][1]["document"]["Coll_C"].size());+ ASSERT_EQ("coll_c_1", res_obj["hits"][1]["document"]["Coll_C"][0]["title"]);+ ASSERT_EQ(1, res_obj["hits"][1]["document"]["Coll_C"][0]["Coll_D"].size());+ ASSERT_EQ("coll_d_1", res_obj["hits"][1]["document"]["Coll_C"][0]["Coll_D"][0]["title"]);+ ASSERT_EQ("coll_c_2", res_obj["hits"][1]["document"]["Coll_C"][1]["title"]);+ ASSERT_EQ(1, res_obj["hits"][1]["document"]["Coll_C"][1]["Coll_D"].size());+ ASSERT_EQ("coll_d_2", res_obj["hits"][1]["document"]["Coll_C"][1]["Coll_D"][0]["title"]);++ ASSERT_EQ(2, res_obj["hits"][2]["document"].size());+ ASSERT_EQ("coll_b_0", res_obj["hits"][2]["document"]["title"]);+ ASSERT_EQ(1, res_obj["hits"][2]["document"]["Coll_C"].size());+ ASSERT_EQ("coll_c_2", res_obj["hits"][2]["document"]["Coll_C"][0]["title"]);+ ASSERT_EQ(1, res_obj["hits"][2]["document"]["Coll_C"][0]["Coll_D"].size());+ ASSERT_EQ("coll_d_2", res_obj["hits"][2]["document"]["Coll_C"][0]["Coll_D"][0]["title"]);++ req_params = {+ {"collection", "Coll_D"},+ {"q", "*"},+ {"filter_by", "$Coll_C($Coll_B(id: [0, 1]))"},+ {"include_fields", "title, $Coll_C(title, $Coll_B(title:nest_array):nest_array)"}+ };+ search_op = collectionManager.do_search(req_params, embedded_params, json_res, now_ts);+ ASSERT_TRUE(search_op.ok());++ res_obj = nlohmann::json::parse(json_res);+ LOG(INFO) << res_obj.dump();+ // coll_d_2 -> coll_c_2 -> coll_b_0, coll_b_1+ //+ // coll_d_1 -> coll_c_1 -> coll_b_1+ ASSERT_EQ(2, res_obj["found"].get<size_t>());+ ASSERT_EQ(2, res_obj["hits"].size());+ ASSERT_EQ(2, res_obj["hits"][0]["document"].size());+ ASSERT_EQ("coll_d_2", res_obj["hits"][0]["document"]["title"]);+ ASSERT_EQ(1, res_obj["hits"][0]["document"]["Coll_C"].size());+ ASSERT_EQ("coll_c_2", res_obj["hits"][0]["document"]["Coll_C"][0]["title"]);+ ASSERT_EQ(2, res_obj["hits"][0]["document"]["Coll_C"][0]["Coll_B"].size());+ ASSERT_EQ("coll_b_0", res_obj["hits"][0]["document"]["Coll_C"][0]["Coll_B"][0]["title"]);+ ASSERT_EQ("coll_b_1", res_obj["hits"][0]["document"]["Coll_C"][0]["Coll_B"][1]["title"]);++ ASSERT_EQ(2, res_obj["hits"][1]["document"].size());+ ASSERT_EQ("coll_d_1", res_obj["hits"][1]["document"]["title"]);+ ASSERT_EQ(1, res_obj["hits"][1]["document"]["Coll_C"].size());+ ASSERT_EQ("coll_c_1", res_obj["hits"][1]["document"]["Coll_C"][0]["title"]);+ ASSERT_EQ(1, res_obj["hits"][1]["document"]["Coll_C"][0]["Coll_B"].size());+ ASSERT_EQ("coll_b_1", res_obj["hits"][1]["document"]["Coll_C"][0]["Coll_B"][0]["title"]);++ schema_json =+ R"({+ "name": "products",+ "fields": [+ {"name": "title", "type": "string"}+ ]+ })"_json;+ documents = {+ R"({+ "title": "shampoo"+ })"_json,+ R"({+ "title": "soap"+ })"_json+ };+ collection_create_op = collectionManager.create_collection(schema_json);+ ASSERT_TRUE(collection_create_op.ok());+ for (auto const &json: documents) {+ auto add_op = collection_create_op.get()->add(json.dump());+ if (!add_op.ok()) {+ LOG(INFO) << add_op.error();+ }+ ASSERT_TRUE(add_op.ok());+ }++ schema_json =+ R"({+ "name": "product_variants",+ "fields": [+ {"name": "title", "type": "string"},+ {"name": "product_id", "type": "string", "reference": "products.id"}+ ]+ })"_json;+ documents = {+ R"({+ "title": "panteen",+ "product_id": "0"+ })"_json,+ R"({+ "title": "loreal",+ "product_id": "0"+ })"_json,+ R"({+ "title": "pears",+ "product_id": "1"+ })"_json,+ R"({+ "title": "lifebuoy",+ "product_id": "1"+ })"_json+ };+ collection_create_op = collectionManager.create_collection(schema_json);+ ASSERT_TRUE(collection_create_op.ok());+ for (auto const &json: documents) {+ auto add_op = collection_create_op.get()->add(json.dump());+ if (!add_op.ok()) {+ LOG(INFO) << add_op.error();+ }+ ASSERT_TRUE(add_op.ok());+ }++ schema_json =+ R"({+ "name": "retailers",+ "fields": [+ {"name": "title", "type": "string"},+ {"name": "location", "type": "geopoint"}+ ]+ })"_json;+ documents = {+ R"({+ "title": "retailer 1",+ "location": [48.872576479306765, 2.332291112241466]+ })"_json,+ R"({+ "title": "retailer 2",+ "location": [48.888286721920934, 2.342340862419206]+ })"_json,+ R"({+ "title": "retailer 3",+ "location": [48.87538726829884, 2.296113163780903]+ })"_json+ };+ collection_create_op = collectionManager.create_collection(schema_json);+ ASSERT_TRUE(collection_create_op.ok());+ for (auto const &json: documents) {+ auto add_op = collection_create_op.get()->add(json.dump());+ if (!add_op.ok()) {+ LOG(INFO) << add_op.error();+ }+ ASSERT_TRUE(add_op.ok());+ }++ schema_json =+ R"({+ "name": "inventory",+ "fields": [+ {"name": "qty", "type": "int32"},+ {"name": "retailer_id", "type": "string", "reference": "retailers.id"},+ {"name": "product_variant_id", "type": "string", "reference": "product_variants.id"}+ ]+ })"_json;+ documents = {+ R"({+ "qty": "1",+ "retailer_id": "0",+ "product_variant_id": "0"+ })"_json,+ R"({+ "qty": "2",+ "retailer_id": "0",+ "product_variant_id": "1"+ })"_json,+ R"({+ "qty": "3",+ "retailer_id": "0",+ "product_variant_id": "2"+ })"_json,+ R"({+ "qty": "4",+ "retailer_id": "0",+ "product_variant_id": "3"+ })"_json,+ R"({+ "qty": "5",+ "retailer_id": "1",+ "product_variant_id": "0"+ })"_json,+ R"({+ "qty": "6",+ "retailer_id": "1",+ "product_variant_id": "1"+ })"_json,+ R"({+ "qty": "7",+ "retailer_id": "1",+ "product_variant_id": "2"+ })"_json,+ R"({+ "qty": "8",+ "retailer_id": "1",+ "product_variant_id": "3"+ })"_json,+ R"({+ "qty": "9",+ "retailer_id": "2",+ "product_variant_id": "0"+ })"_json,+ R"({+ "qty": "10",+ "retailer_id": "2",+ "product_variant_id": "1"+ })"_json,+ R"({+ "qty": "11",+ "retailer_id": "2",+ "product_variant_id": "2"+ })"_json,+ R"({+ "qty": "12",+ "retailer_id": "2",+ "product_variant_id": "3"+ })"_json,+ };+ collection_create_op = collectionManager.create_collection(schema_json);+ ASSERT_TRUE(collection_create_op.ok());+ for (auto const &json: documents) {+ auto add_op = collection_create_op.get()->add(json.dump());+ if (!add_op.ok()) {+ LOG(INFO) << add_op.error();+ }+ ASSERT_TRUE(add_op.ok());+ }++ req_params = {+ {"collection", "products"},+ {"q", "*"},+ {"filter_by", "$product_variants($inventory($retailers(location:(48.87538726829884, 2.296113163780903,1 km))))"},+ {"include_fields", "$product_variants(id,$inventory(qty,sku,$retailers(id,title)))"}+ };+ search_op = collectionManager.do_search(req_params, embedded_params, json_res, now_ts);+ ASSERT_TRUE(search_op.ok());++ res_obj = nlohmann::json::parse(json_res);+ ASSERT_EQ(2, res_obj["found"].get<size_t>());+ ASSERT_EQ(2, res_obj["hits"].size());+ ASSERT_EQ("1", res_obj["hits"][0]["document"]["id"]);+ ASSERT_EQ("soap", res_obj["hits"][0]["document"]["title"]);+ ASSERT_EQ(2, res_obj["hits"][0]["document"]["product_variants"].size());++ ASSERT_EQ("2", res_obj["hits"][0]["document"]["product_variants"][0]["id"]);+ ASSERT_EQ(2, res_obj["hits"][0]["document"]["product_variants"][0]["inventory"].size());+ ASSERT_EQ(11, res_obj["hits"][0]["document"]["product_variants"][0]["inventory"]["qty"]);+ ASSERT_EQ(2, res_obj["hits"][0]["document"]["product_variants"][0]["inventory"]["retailers"].size());+ ASSERT_EQ("2", res_obj["hits"][0]["document"]["product_variants"][0]["inventory"]["retailers"]["id"]);+ ASSERT_EQ("retailer 3", res_obj["hits"][0]["document"]["product_variants"][0]["inventory"]["retailers"]["title"]);++ ASSERT_EQ("3", res_obj["hits"][0]["document"]["product_variants"][1]["id"]);+ ASSERT_EQ(2, res_obj["hits"][0]["document"]["product_variants"][1]["inventory"].size());+ ASSERT_EQ(12, res_obj["hits"][0]["document"]["product_variants"][1]["inventory"]["qty"]);+ ASSERT_EQ(2, res_obj["hits"][0]["document"]["product_variants"][1]["inventory"]["retailers"].size());+ ASSERT_EQ("2", res_obj["hits"][0]["document"]["product_variants"][1]["inventory"]["retailers"]["id"]);+ ASSERT_EQ("retailer 3", res_obj["hits"][0]["document"]["product_variants"][1]["inventory"]["retailers"]["title"]);++ ASSERT_EQ("0", res_obj["hits"][1]["document"]["id"]);+ ASSERT_EQ("shampoo", res_obj["hits"][1]["document"]["title"]);+ ASSERT_EQ(2, res_obj["hits"][1]["document"]["product_variants"].size());++ ASSERT_EQ("0", res_obj["hits"][1]["document"]["product_variants"][0]["id"]);+ ASSERT_EQ(2, res_obj["hits"][1]["document"]["product_variants"][0]["inventory"].size());+ ASSERT_EQ(9, res_obj["hits"][1]["document"]["product_variants"][0]["inventory"]["qty"]);+ ASSERT_EQ(2, res_obj["hits"][1]["document"]["product_variants"][0]["inventory"]["retailers"].size());+ ASSERT_EQ("2", res_obj["hits"][1]["document"]["product_variants"][0]["inventory"]["retailers"]["id"]);+ ASSERT_EQ("retailer 3", res_obj["hits"][1]["document"]["product_variants"][0]["inventory"]["retailers"]["title"]);++ ASSERT_EQ("1", res_obj["hits"][1]["document"]["product_variants"][1]["id"]);+ ASSERT_EQ(2, res_obj["hits"][1]["document"]["product_variants"][1]["inventory"].size());+ ASSERT_EQ(10, res_obj["hits"][1]["document"]["product_variants"][1]["inventory"]["qty"]);+ ASSERT_EQ(2, res_obj["hits"][1]["document"]["product_variants"][1]["inventory"]["retailers"].size());+ ASSERT_EQ("2", res_obj["hits"][1]["document"]["product_variants"][1]["inventory"]["retailers"]["id"]);+ ASSERT_EQ("retailer 3", res_obj["hits"][1]["document"]["product_variants"][1]["inventory"]["retailers"]["title"]);++ req_params = {+ {"collection", "products"},+ {"q", "*"},+ {"filter_by", "$product_variants($inventory($retailers(id: [0, 1]) && qty: [4..5]))"},+ {"include_fields", "$product_variants(id,$inventory(qty,sku,$retailers(id,title)))"},+ {"exclude_fields", "$product_variants($inventory($retailers(id)))"}+ };+ search_op = collectionManager.do_search(req_params, embedded_params, json_res, now_ts);+ ASSERT_TRUE(search_op.ok());++ res_obj = nlohmann::json::parse(json_res);+ ASSERT_EQ(2, res_obj["found"].get<size_t>());+ ASSERT_EQ(2, res_obj["hits"].size());+ ASSERT_EQ("1", res_obj["hits"][0]["document"]["id"]);+ ASSERT_EQ("soap", res_obj["hits"][0]["document"]["title"]);+ ASSERT_EQ("3", res_obj["hits"][0]["document"]["product_variants"]["id"]);+ ASSERT_EQ(2, res_obj["hits"][0]["document"]["product_variants"]["inventory"].size());+ ASSERT_EQ(4, res_obj["hits"][0]["document"]["product_variants"]["inventory"]["qty"]);+ ASSERT_EQ(1, res_obj["hits"][0]["document"]["product_variants"]["inventory"]["retailers"].size());+ ASSERT_EQ("retailer 1", res_obj["hits"][0]["document"]["product_variants"]["inventory"]["retailers"]["title"]);++ ASSERT_EQ("0", res_obj["hits"][1]["document"]["id"]);+ ASSERT_EQ("shampoo", res_obj["hits"][1]["document"]["title"]);+ ASSERT_EQ("0", res_obj["hits"][1]["document"]["product_variants"]["id"]);+ ASSERT_EQ(2, res_obj["hits"][1]["document"]["product_variants"]["inventory"].size());+ ASSERT_EQ(5, res_obj["hits"][1]["document"]["product_variants"]["inventory"]["qty"]);+ ASSERT_EQ(1, res_obj["hits"][1]["document"]["product_variants"]["inventory"]["retailers"].size());+ ASSERT_EQ("retailer 2", res_obj["hits"][1]["document"]["product_variants"]["inventory"]["retailers"]["title"]);+}+TEST_F(CollectionJoinTest, IncludeExcludeFieldsByReference) {auto schema_json =R"({@@ -1926,12 +2479,14 @@ TEST_F(CollectionJoinTest, IncludeExcludeFieldsByReference) {auto search_op = collectionManager.do_search(req_params, embedded_params, json_res, now_ts);ASSERT_FALSE(search_op.ok());- ASSERT_EQ("Invalid reference in include_fields, expected `$CollectionName(fieldA, ...)`.", search_op.error());+ ASSERT_EQ("Invalid reference `$foo.bar` in include_fields/exclude_fields, expected `$CollectionName(fieldA, ...)`.",+ search_op.error());req_params["include_fields"] = "$foo(bar";search_op = collectionManager.do_search(req_params, embedded_params, json_res, now_ts);ASSERT_FALSE(search_op.ok());- ASSERT_EQ("Invalid reference in include_fields, expected `$CollectionName(fieldA, ...)`.", search_op.error());+ ASSERT_EQ("Invalid reference `$foo(bar` in include_fields/exclude_fields, expected `$CollectionName(fieldA, ...)`.",+ search_op.error());req_params["include_fields"] = "$foo(bar)";search_op = collectionManager.do_search(req_params, embedded_params, json_res, now_ts);diff --git a/test/collection_manager_test.cpp b/test/collection_manager_test.cppindex 4a7da68a..29373f2a 100644--- a/test/collection_manager_test.cpp+++ b/test/collection_manager_test.cpp@@ -1460,156 +1460,241 @@ TEST_F(CollectionManagerTest, GetReferenceCollectionNames) {ref_includes = nullptr;}-TEST_F(CollectionManagerTest, InitializeRefIncludeFields) {+TEST_F(CollectionManagerTest, InitializeRefIncludeExcludeFields) {std::string filter_query = "";- std::vector<std::string> include_fields_vec;- std::vector<ref_include_fields> ref_include_fields_vec;- auto initialize_op = CollectionManager::_initialize_ref_include_fields_vec(filter_query, include_fields_vec,- ref_include_fields_vec);+ std::vector<std::string> include_fields_vec, exclude_fields_vec;+ std::vector<ref_include_exclude_fields> ref_include_exclude_fields_vec;+ auto initialize_op = CollectionManager::_initialize_ref_include_exclude_fields_vec(filter_query, include_fields_vec,+ exclude_fields_vec,+ ref_include_exclude_fields_vec);ASSERT_TRUE(initialize_op.ok());- ASSERT_TRUE(ref_include_fields_vec.empty());+ ASSERT_TRUE(ref_include_exclude_fields_vec.empty());filter_query = "$foo(bar:baz)";- initialize_op = CollectionManager::_initialize_ref_include_fields_vec(filter_query, include_fields_vec,- ref_include_fields_vec);+ exclude_fields_vec = {"$foo(bar)"};+ initialize_op = CollectionManager::_initialize_ref_include_exclude_fields_vec(filter_query, include_fields_vec,+ exclude_fields_vec,+ ref_include_exclude_fields_vec);ASSERT_TRUE(initialize_op.ok());- ASSERT_EQ(1, ref_include_fields_vec.size());- ASSERT_EQ("foo", ref_include_fields_vec[0].collection_name);- ASSERT_TRUE(ref_include_fields_vec[0].fields.empty());- ASSERT_TRUE(ref_include_fields_vec[0].alias.empty());- ASSERT_EQ(ref_include::nest, ref_include_fields_vec[0].strategy);- ASSERT_TRUE(ref_include_fields_vec[0].nested_join_includes.empty());- ref_include_fields_vec.clear();+ ASSERT_EQ(1, ref_include_exclude_fields_vec.size());+ ASSERT_EQ("foo", ref_include_exclude_fields_vec[0].collection_name);+ ASSERT_TRUE(ref_include_exclude_fields_vec[0].include_fields.empty());+ ASSERT_EQ("bar", ref_include_exclude_fields_vec[0].exclude_fields);+ ASSERT_TRUE(ref_include_exclude_fields_vec[0].alias.empty());+ ASSERT_EQ(ref_include::nest, ref_include_exclude_fields_vec[0].strategy);+ ASSERT_TRUE(ref_include_exclude_fields_vec[0].nested_join_includes.empty());+ ref_include_exclude_fields_vec.clear();+ exclude_fields_vec.clear();filter_query = "";include_fields_vec = {"$Customers(product_price: foo) as customers"};- initialize_op = CollectionManager::_initialize_ref_include_fields_vec(filter_query, include_fields_vec,- ref_include_fields_vec);+ initialize_op = CollectionManager::_initialize_ref_include_exclude_fields_vec(filter_query, include_fields_vec,+ exclude_fields_vec,+ ref_include_exclude_fields_vec);ASSERT_FALSE(initialize_op.ok());ASSERT_EQ("Error parsing `$Customers(product_price: foo) as customers`: Unknown include strategy `foo`. ""Valid options are `merge`, `nest`, `nest_array`.", initialize_op.error());filter_query = "$Customers(customer_id:=customer_a && (product_price:>100 && product_price:<200))";include_fields_vec = {"$Customers(product_price: merge) as customers"};- initialize_op = CollectionManager::_initialize_ref_include_fields_vec(filter_query, include_fields_vec,- ref_include_fields_vec);+ initialize_op = CollectionManager::_initialize_ref_include_exclude_fields_vec(filter_query, include_fields_vec,+ exclude_fields_vec,+ ref_include_exclude_fields_vec);ASSERT_TRUE(initialize_op.ok());- ASSERT_EQ(1, ref_include_fields_vec.size());- ASSERT_EQ("Customers", ref_include_fields_vec[0].collection_name);- ASSERT_EQ("product_price", ref_include_fields_vec[0].fields);- ASSERT_EQ("customers.", ref_include_fields_vec[0].alias);- ASSERT_EQ(ref_include::merge, ref_include_fields_vec[0].strategy);- ASSERT_TRUE(ref_include_fields_vec[0].nested_join_includes.empty());- ref_include_fields_vec.clear();+ ASSERT_EQ(1, ref_include_exclude_fields_vec.size());+ ASSERT_EQ("Customers", ref_include_exclude_fields_vec[0].collection_name);+ ASSERT_EQ("product_price", ref_include_exclude_fields_vec[0].include_fields);+ ASSERT_EQ("customers.", ref_include_exclude_fields_vec[0].alias);+ ASSERT_EQ(ref_include::merge, ref_include_exclude_fields_vec[0].strategy);+ ASSERT_TRUE(ref_include_exclude_fields_vec[0].nested_join_includes.empty());+ ref_include_exclude_fields_vec.clear();filter_query = "$Customers(customer_id:=customer_a && (product_price:>100 && product_price:<200))";include_fields_vec = {"$Customers(product_price: nest_array) as customers"};- initialize_op = CollectionManager::_initialize_ref_include_fields_vec(filter_query, include_fields_vec,- ref_include_fields_vec);+ initialize_op = CollectionManager::_initialize_ref_include_exclude_fields_vec(filter_query, include_fields_vec,+ exclude_fields_vec,+ ref_include_exclude_fields_vec);ASSERT_TRUE(initialize_op.ok());- ASSERT_EQ(1, ref_include_fields_vec.size());- ASSERT_EQ("Customers", ref_include_fields_vec[0].collection_name);- ASSERT_EQ("product_price", ref_include_fields_vec[0].fields);- ASSERT_EQ("customers", ref_include_fields_vec[0].alias);- ASSERT_EQ(ref_include::nest_array, ref_include_fields_vec[0].strategy);- ASSERT_TRUE(ref_include_fields_vec[0].nested_join_includes.empty());- ref_include_fields_vec.clear();+ ASSERT_EQ(1, ref_include_exclude_fields_vec.size());+ ASSERT_EQ("Customers", ref_include_exclude_fields_vec[0].collection_name);+ ASSERT_EQ("product_price", ref_include_exclude_fields_vec[0].include_fields);+ ASSERT_EQ("customers", ref_include_exclude_fields_vec[0].alias);+ ASSERT_EQ(ref_include::nest_array, ref_include_exclude_fields_vec[0].strategy);+ ASSERT_TRUE(ref_include_exclude_fields_vec[0].nested_join_includes.empty());+ ref_include_exclude_fields_vec.clear();filter_query = "$product_variants( $inventory($retailers(location:(33.865,-118.375,100 km))))";include_fields_vec = {"$product_variants(id,$inventory(qty,sku,$retailers(id,title)))"};- initialize_op = CollectionManager::_initialize_ref_include_fields_vec(filter_query, include_fields_vec,- ref_include_fields_vec);+ initialize_op = CollectionManager::_initialize_ref_include_exclude_fields_vec(filter_query, include_fields_vec,+ exclude_fields_vec,+ ref_include_exclude_fields_vec);ASSERT_TRUE(initialize_op.ok());- ASSERT_EQ(1, ref_include_fields_vec.size());- ASSERT_EQ("product_variants", ref_include_fields_vec[0].collection_name);- ASSERT_EQ("id,", ref_include_fields_vec[0].fields);- ASSERT_TRUE(ref_include_fields_vec[0].alias.empty());- ASSERT_EQ(ref_include::nest, ref_include_fields_vec[0].strategy);-- auto nested_join_includes = ref_include_fields_vec[0].nested_join_includes;- ASSERT_EQ("inventory", nested_join_includes[0].collection_name);- ASSERT_EQ("qty,sku,", nested_join_includes[0].fields);- ASSERT_TRUE(nested_join_includes[0].alias.empty());- ASSERT_EQ(ref_include::nest, nested_join_includes[0].strategy);-- nested_join_includes = ref_include_fields_vec[0].nested_join_includes[0].nested_join_includes;- ASSERT_EQ("retailers", nested_join_includes[0].collection_name);- ASSERT_EQ("id,title", nested_join_includes[0].fields);- ASSERT_TRUE(nested_join_includes[0].alias.empty());- ASSERT_EQ(ref_include::nest, ref_include_fields_vec[0].strategy);- ref_include_fields_vec.clear();+ ASSERT_EQ(1, ref_include_exclude_fields_vec.size());+ ASSERT_EQ("product_variants", ref_include_exclude_fields_vec[0].collection_name);+ ASSERT_EQ("id,", ref_include_exclude_fields_vec[0].include_fields);+ ASSERT_TRUE(ref_include_exclude_fields_vec[0].alias.empty());+ ASSERT_EQ(ref_include::nest, ref_include_exclude_fields_vec[0].strategy);++ auto nested_include_excludes = ref_include_exclude_fields_vec[0].nested_join_includes;+ ASSERT_EQ("inventory", nested_include_excludes[0].collection_name);+ ASSERT_EQ("qty,sku,", nested_include_excludes[0].include_fields);+ ASSERT_TRUE(nested_include_excludes[0].alias.empty());+ ASSERT_EQ(ref_include::nest, nested_include_excludes[0].strategy);++ nested_include_excludes = ref_include_exclude_fields_vec[0].nested_join_includes[0].nested_join_includes;+ ASSERT_EQ("retailers", nested_include_excludes[0].collection_name);+ ASSERT_EQ("id,title", nested_include_excludes[0].include_fields);+ ASSERT_TRUE(nested_include_excludes[0].alias.empty());+ ASSERT_EQ(ref_include::nest, ref_include_exclude_fields_vec[0].strategy);+ ref_include_exclude_fields_vec.clear();filter_query = "$product_variants( $inventory($retailers(location:(33.865,-118.375,100 km))))";include_fields_vec = {"$product_variants(title, $inventory(qty:merge) as inventory: nest) as variants"};- initialize_op = CollectionManager::_initialize_ref_include_fields_vec(filter_query, include_fields_vec,- ref_include_fields_vec);+ initialize_op = CollectionManager::_initialize_ref_include_exclude_fields_vec(filter_query, include_fields_vec,+ exclude_fields_vec,+ ref_include_exclude_fields_vec);ASSERT_TRUE(initialize_op.ok());- ASSERT_EQ(1, ref_include_fields_vec.size());- ASSERT_EQ("product_variants", ref_include_fields_vec[0].collection_name);- ASSERT_EQ("title,", ref_include_fields_vec[0].fields);- ASSERT_EQ("variants", ref_include_fields_vec[0].alias);- ASSERT_EQ(ref_include::nest, ref_include_fields_vec[0].strategy);-- nested_join_includes = ref_include_fields_vec[0].nested_join_includes;- ASSERT_EQ("inventory", nested_join_includes[0].collection_name);- ASSERT_EQ("qty", nested_join_includes[0].fields);- ASSERT_EQ("inventory.", nested_join_includes[0].alias);- ASSERT_EQ(ref_include::merge, nested_join_includes[0].strategy);-- nested_join_includes = ref_include_fields_vec[0].nested_join_includes[0].nested_join_includes;- ASSERT_EQ("retailers", nested_join_includes[0].collection_name);- ASSERT_TRUE(nested_join_includes[0].fields.empty());- ASSERT_TRUE(nested_join_includes[0].alias.empty());- ASSERT_EQ(ref_include::nest, ref_include_fields_vec[0].strategy);- ref_include_fields_vec.clear();+ ASSERT_EQ(1, ref_include_exclude_fields_vec.size());+ ASSERT_EQ("product_variants", ref_include_exclude_fields_vec[0].collection_name);+ ASSERT_EQ("title,", ref_include_exclude_fields_vec[0].include_fields);+ ASSERT_EQ("variants", ref_include_exclude_fields_vec[0].alias);+ ASSERT_EQ(ref_include::nest, ref_include_exclude_fields_vec[0].strategy);++ nested_include_excludes = ref_include_exclude_fields_vec[0].nested_join_includes;+ ASSERT_EQ("inventory", nested_include_excludes[0].collection_name);+ ASSERT_EQ("qty", nested_include_excludes[0].include_fields);+ ASSERT_EQ("inventory.", nested_include_excludes[0].alias);+ ASSERT_EQ(ref_include::merge, nested_include_excludes[0].strategy);++ nested_include_excludes = ref_include_exclude_fields_vec[0].nested_join_includes[0].nested_join_includes;+ ASSERT_EQ("retailers", nested_include_excludes[0].collection_name);+ ASSERT_TRUE(nested_include_excludes[0].include_fields.empty());+ ASSERT_TRUE(nested_include_excludes[0].alias.empty());+ ASSERT_EQ(ref_include::nest, ref_include_exclude_fields_vec[0].strategy);+ ref_include_exclude_fields_vec.clear();filter_query = "$product_variants( $inventory(id:*) && $retailers(location:(33.865,-118.375,100 km)))";include_fields_vec = {"$product_variants(title, $inventory(qty:merge) as inventory,"" $retailers(title): merge) as variants"};- initialize_op = CollectionManager::_initialize_ref_include_fields_vec(filter_query, include_fields_vec,- ref_include_fields_vec);+ initialize_op = CollectionManager::_initialize_ref_include_exclude_fields_vec(filter_query, include_fields_vec,+ exclude_fields_vec,+ ref_include_exclude_fields_vec);ASSERT_TRUE(initialize_op.ok());- ASSERT_EQ(1, ref_include_fields_vec.size());- ASSERT_EQ("product_variants", ref_include_fields_vec[0].collection_name);- ASSERT_EQ("title,", ref_include_fields_vec[0].fields);- ASSERT_EQ("variants.", ref_include_fields_vec[0].alias);- ASSERT_EQ(ref_include::merge, ref_include_fields_vec[0].strategy);-- nested_join_includes = ref_include_fields_vec[0].nested_join_includes;- ASSERT_EQ("inventory", nested_join_includes[0].collection_name);- ASSERT_EQ("qty", nested_join_includes[0].fields);- ASSERT_EQ("inventory.", nested_join_includes[0].alias);- ASSERT_EQ(ref_include::merge, nested_join_includes[0].strategy);-- ASSERT_EQ("retailers", nested_join_includes[1].collection_name);- ASSERT_EQ("title", nested_join_includes[1].fields);- ASSERT_TRUE(nested_join_includes[1].alias.empty());- ASSERT_EQ(ref_include::nest, nested_join_includes[1].strategy);- ref_include_fields_vec.clear();+ ASSERT_EQ(1, ref_include_exclude_fields_vec.size());+ ASSERT_EQ("product_variants", ref_include_exclude_fields_vec[0].collection_name);+ ASSERT_EQ("title,", ref_include_exclude_fields_vec[0].include_fields);+ ASSERT_EQ("variants.", ref_include_exclude_fields_vec[0].alias);+ ASSERT_EQ(ref_include::merge, ref_include_exclude_fields_vec[0].strategy);++ nested_include_excludes = ref_include_exclude_fields_vec[0].nested_join_includes;+ ASSERT_EQ("inventory", nested_include_excludes[0].collection_name);+ ASSERT_EQ("qty", nested_include_excludes[0].include_fields);+ ASSERT_EQ("inventory.", nested_include_excludes[0].alias);+ ASSERT_EQ(ref_include::merge, nested_include_excludes[0].strategy);++ ASSERT_EQ("retailers", nested_include_excludes[1].collection_name);+ ASSERT_EQ("title", nested_include_excludes[1].include_fields);+ ASSERT_TRUE(nested_include_excludes[1].alias.empty());+ ASSERT_EQ(ref_include::nest, nested_include_excludes[1].strategy);+ ref_include_exclude_fields_vec.clear();filter_query = "$product_variants( $inventory(id:*) && $retailers(location:(33.865,-118.375,100 km)))";include_fields_vec = {"$product_variants(title, $inventory(qty:merge) as inventory, description,"" $retailers(title), foo: merge) as variants"};- initialize_op = CollectionManager::_initialize_ref_include_fields_vec(filter_query, include_fields_vec,- ref_include_fields_vec);+ initialize_op = CollectionManager::_initialize_ref_include_exclude_fields_vec(filter_query, include_fields_vec,+ exclude_fields_vec,+ ref_include_exclude_fields_vec);ASSERT_TRUE(initialize_op.ok());- ASSERT_EQ(1, ref_include_fields_vec.size());- ASSERT_EQ("product_variants", ref_include_fields_vec[0].collection_name);- ASSERT_EQ("title, description, foo", ref_include_fields_vec[0].fields);- ASSERT_EQ("variants.", ref_include_fields_vec[0].alias);- ASSERT_EQ(ref_include::merge, ref_include_fields_vec[0].strategy);-- nested_join_includes = ref_include_fields_vec[0].nested_join_includes;- ASSERT_EQ("inventory", nested_join_includes[0].collection_name);- ASSERT_EQ("qty", nested_join_includes[0].fields);- ASSERT_EQ("inventory.", nested_join_includes[0].alias);- ASSERT_EQ(ref_include::merge, nested_join_includes[0].strategy);-- ASSERT_EQ("retailers", nested_join_includes[1].collection_name);- ASSERT_EQ("title", nested_join_includes[1].fields);- ASSERT_TRUE(nested_join_includes[1].alias.empty());- ASSERT_EQ(ref_include::nest, nested_join_includes[1].strategy);- ref_include_fields_vec.clear();+ ASSERT_EQ(1, ref_include_exclude_fields_vec.size());+ ASSERT_EQ("product_variants", ref_include_exclude_fields_vec[0].collection_name);+ ASSERT_EQ("title, description, foo", ref_include_exclude_fields_vec[0].include_fields);+ ASSERT_EQ("variants.", ref_include_exclude_fields_vec[0].alias);+ ASSERT_EQ(ref_include::merge, ref_include_exclude_fields_vec[0].strategy);++ nested_include_excludes = ref_include_exclude_fields_vec[0].nested_join_includes;+ ASSERT_EQ("inventory", nested_include_excludes[0].collection_name);+ ASSERT_EQ("qty", nested_include_excludes[0].include_fields);+ ASSERT_EQ("inventory.", nested_include_excludes[0].alias);+ ASSERT_EQ(ref_include::merge, nested_include_excludes[0].strategy);++ ASSERT_EQ("retailers", nested_include_excludes[1].collection_name);+ ASSERT_EQ("title", nested_include_excludes[1].include_fields);+ ASSERT_TRUE(nested_include_excludes[1].alias.empty());+ ASSERT_EQ(ref_include::nest, nested_include_excludes[1].strategy);+ ref_include_exclude_fields_vec.clear();++ filter_query = "$Customers(customer_id:=customer_a && (product_price:>100 && product_price:<200))";+ include_fields_vec.clear();+ exclude_fields_vec = {"$Customers(product_price)"};+ initialize_op = CollectionManager::_initialize_ref_include_exclude_fields_vec(filter_query, include_fields_vec,+ exclude_fields_vec,+ ref_include_exclude_fields_vec);+ ASSERT_TRUE(initialize_op.ok());+ ASSERT_EQ(1, ref_include_exclude_fields_vec.size());+ ASSERT_EQ("Customers", ref_include_exclude_fields_vec[0].collection_name);+ ASSERT_TRUE(ref_include_exclude_fields_vec[0].include_fields.empty());+ ASSERT_EQ("product_price", ref_include_exclude_fields_vec[0].exclude_fields);+ ASSERT_TRUE(ref_include_exclude_fields_vec[0].alias.empty());+ ASSERT_EQ(ref_include::nest, ref_include_exclude_fields_vec[0].strategy);+ ASSERT_TRUE(ref_include_exclude_fields_vec[0].nested_join_includes.empty());+ ref_include_exclude_fields_vec.clear();++ filter_query = "$product_variants( $inventory(id:*) && $retailers(location:(33.865,-118.375,100 km)))";+ include_fields_vec.clear();+ exclude_fields_vec = {"$product_variants(title, $inventory(qty), description, $retailers(title), foo)"};+ initialize_op = CollectionManager::_initialize_ref_include_exclude_fields_vec(filter_query, include_fields_vec,+ exclude_fields_vec,+ ref_include_exclude_fields_vec);+ ASSERT_TRUE(initialize_op.ok());+ ASSERT_EQ(1, ref_include_exclude_fields_vec.size());+ ASSERT_EQ("product_variants", ref_include_exclude_fields_vec[0].collection_name);+ ASSERT_TRUE(ref_include_exclude_fields_vec[0].include_fields.empty());+ ASSERT_EQ("title, description, foo", ref_include_exclude_fields_vec[0].exclude_fields);+ ASSERT_TRUE(ref_include_exclude_fields_vec[0].alias.empty());+ ASSERT_EQ(ref_include::nest, ref_include_exclude_fields_vec[0].strategy);++ nested_include_excludes = ref_include_exclude_fields_vec[0].nested_join_includes;+ ASSERT_EQ("inventory", nested_include_excludes[0].collection_name);+ ASSERT_TRUE(nested_include_excludes[0].include_fields.empty());+ ASSERT_EQ("qty", nested_include_excludes[0].exclude_fields);+ ASSERT_TRUE(nested_include_excludes[0].alias.empty());+ ASSERT_EQ(ref_include::nest, nested_include_excludes[0].strategy);++ ASSERT_EQ("retailers", nested_include_excludes[1].collection_name);+ ASSERT_TRUE(nested_include_excludes[1].include_fields.empty());+ ASSERT_EQ("title", nested_include_excludes[1].exclude_fields);+ ASSERT_TRUE(nested_include_excludes[1].alias.empty());+ ASSERT_EQ(ref_include::nest, nested_include_excludes[1].strategy);+ ref_include_exclude_fields_vec.clear();++ filter_query = "$product_variants( $inventory($retailers(location:(33.865,-118.375,100 km))))";+ include_fields_vec = {"$product_variants(title, $inventory(qty:merge) as inventory: nest) as variants"};+ exclude_fields_vec = {"$product_variants(title, $inventory(qty, $retailers(title)))"};+ initialize_op = CollectionManager::_initialize_ref_include_exclude_fields_vec(filter_query, include_fields_vec,+ exclude_fields_vec,+ ref_include_exclude_fields_vec);+ ASSERT_TRUE(initialize_op.ok());+ ASSERT_EQ(1, ref_include_exclude_fields_vec.size());+ ASSERT_EQ("product_variants", ref_include_exclude_fields_vec[0].collection_name);+ ASSERT_EQ("title,", ref_include_exclude_fields_vec[0].include_fields);+ ASSERT_EQ("title,", ref_include_exclude_fields_vec[0].exclude_fields);+ ASSERT_EQ("variants", ref_include_exclude_fields_vec[0].alias);+ ASSERT_EQ(ref_include::nest, ref_include_exclude_fields_vec[0].strategy);++ nested_include_excludes = ref_include_exclude_fields_vec[0].nested_join_includes;+ ASSERT_EQ("inventory", nested_include_excludes[0].collection_name);+ ASSERT_EQ("qty", nested_include_excludes[0].include_fields);+ ASSERT_EQ("qty,", nested_include_excludes[0].exclude_fields);+ ASSERT_EQ("inventory.", nested_include_excludes[0].alias);+ ASSERT_EQ(ref_include::merge, nested_include_excludes[0].strategy);++ nested_include_excludes = ref_include_exclude_fields_vec[0].nested_join_includes[0].nested_join_includes;+ ASSERT_EQ("retailers", nested_include_excludes[0].collection_name);+ ASSERT_TRUE(nested_include_excludes[0].include_fields.empty());+ ASSERT_EQ("title", nested_include_excludes[0].exclude_fields);+ ASSERT_TRUE(nested_include_excludes[0].alias.empty());+ ASSERT_EQ(ref_include::nest, ref_include_exclude_fields_vec[0].strategy);+ ref_include_exclude_fields_vec.clear();}TEST_F(CollectionManagerTest, ReferencedInBacklog) {diff --git a/test/string_utils_test.cpp b/test/string_utils_test.cppindex 44e39bb9..1093343b 100644--- a/test/string_utils_test.cpp+++ b/test/string_utils_test.cpp@@ -398,9 +398,9 @@ TEST(StringUtilsTest, TokenizeFilterQuery) {tokenizeTestHelper(filter_query, tokenList);}-void splitIncludeTestHelper(const std::string& include_fields, const std::vector<std::string>& expected) {+void splitIncludeExcludeTestHelper(const std::string& include_exclude_fields, const std::vector<std::string>& expected) {std::vector<std::string> output;- auto tokenize_op = StringUtils::split_include_fields(include_fields, output);+ auto tokenize_op = StringUtils::split_include_exclude_fields(include_exclude_fields, output);ASSERT_TRUE(tokenize_op.ok());ASSERT_EQ(expected.size(), output.size());for (auto i = 0; i < output.size(); i++) {@@ -408,51 +408,73 @@ void splitIncludeTestHelper(const std::string& include_fields, const std::vector}}-TEST(StringUtilsTest, SplitIncludeFields) {+TEST(StringUtilsTest, SplitIncludeExcludeFields) {std::string include_fields;std::vector<std::string> tokens;include_fields = " id, title , count ";tokens = {"id", "title", "count"};- splitIncludeTestHelper(include_fields, tokens);+ splitIncludeExcludeTestHelper(include_fields, tokens);include_fields = "id, $Collection(title, pref*),count";tokens = {"id", "$Collection(title, pref*)", "count"};- splitIncludeTestHelper(include_fields, tokens);+ splitIncludeExcludeTestHelper(include_fields, tokens);include_fields = "id, $Collection(title, pref*), count, ";tokens = {"id", "$Collection(title, pref*)", "count"};- splitIncludeTestHelper(include_fields, tokens);+ splitIncludeExcludeTestHelper(include_fields, tokens);include_fields = "$Collection(title, pref*) as coll";tokens = {"$Collection(title, pref*) as coll"};- splitIncludeTestHelper(include_fields, tokens);+ splitIncludeExcludeTestHelper(include_fields, tokens);include_fields = "id, $Collection(title, pref*) as coll , count, ";tokens = {"id", "$Collection(title, pref*) as coll", "count"};- splitIncludeTestHelper(include_fields, tokens);+ splitIncludeExcludeTestHelper(include_fields, tokens);include_fields = "$Collection(title, pref*: merge) as coll";tokens = {"$Collection(title, pref*: merge) as coll"};- splitIncludeTestHelper(include_fields, tokens);+ splitIncludeExcludeTestHelper(include_fields, tokens);include_fields = "$product_variants(id,$inventory(qty,sku,$retailer(id,title: merge) as retailer_info)) as variants";tokens = {"$product_variants(id,$inventory(qty,sku,$retailer(id,title: merge) as retailer_info)) as variants"};- splitIncludeTestHelper(include_fields, tokens);+ splitIncludeExcludeTestHelper(include_fields, tokens);++ std::string exclude_fields = " id, title, $Collection(title), count,";+ tokens = {"id", "title", "$Collection(title)", "count"};+ splitIncludeExcludeTestHelper(exclude_fields, tokens);++ exclude_fields = " id, title , count, $Collection(title), $product_variants(id,$inventory(qty,sku,$retailer(id,title)))";+ tokens = {"id", "title", "count", "$Collection(title)", "$product_variants(id,$inventory(qty,sku,$retailer(id,title)))"};+ splitIncludeExcludeTestHelper(exclude_fields, tokens);}-TEST(StringUtilsTest, SplitReferenceIncludeFields) {+TEST(StringUtilsTest, SplitReferenceIncludeExcludeFields) {std::string include_fields = "$retailer(id,title: merge) as retailer_info:merge) as variants, foo", token;size_t index = 0;- auto tokenize_op = StringUtils::split_reference_include_fields(include_fields, index, token);+ auto tokenize_op = StringUtils::split_reference_include_exclude_fields(include_fields, index, token);ASSERT_TRUE(tokenize_op.ok());ASSERT_EQ("$retailer(id,title: merge) as retailer_info", token);ASSERT_EQ(":merge) as variants, foo", include_fields.substr(index));include_fields = "$inventory(qty,sku,$retailer(id,title: merge) as retailer_info) as inventory) as variants, foo";index = 0;- tokenize_op = StringUtils::split_reference_include_fields(include_fields, index, token);+ tokenize_op = StringUtils::split_reference_include_exclude_fields(include_fields, index, token);ASSERT_TRUE(tokenize_op.ok());ASSERT_EQ("$inventory(qty,sku,$retailer(id,title: merge) as retailer_info) as inventory", token);ASSERT_EQ(") as variants, foo", include_fields.substr(index));++ std::string exclude_fields = "$Collection(title), $product_variants(id,$inventory(qty,sku,$retailer(id,title)))";+ index = 0;+ tokenize_op = StringUtils::split_reference_include_exclude_fields(exclude_fields, index, token);+ ASSERT_TRUE(tokenize_op.ok());+ ASSERT_EQ("$Collection(title)", token);+ ASSERT_EQ(", $product_variants(id,$inventory(qty,sku,$retailer(id,title)))", exclude_fields.substr(index));++ exclude_fields = "$inventory(qty,sku,$retailer(id,title)), foo)";+ index = 0;+ tokenize_op = StringUtils::split_reference_include_exclude_fields(exclude_fields, index, token);+ ASSERT_TRUE(tokenize_op.ok());+ ASSERT_EQ("$inventory(qty,sku,$retailer(id,title))", token);+ ASSERT_EQ(", foo)", exclude_fields.substr(index));}