Skip to content

Open Source alternative to Algolia + Pinecone and an Easier-to-Use alternative to ElasticSearch ⚡ 🔍 ✨ Fast, typo tolerant, in-memory fuzzy Search Engine for building delightful search experiences

Support nested join.

Harpreet Sangar08c0a087a1d2
diff --git a/include/collection.h b/include/collection.h
index dfaa4f38..d4253f80 100644
--- a/include/collection.h
+++ b/include/collection.h
@@ -438,22 +438,23 @@ public:
static void remove_reference_helper_fields(nlohmann::json& document);
- static Option<bool> include_references(nlohmann::json& doc,
- const std::string& ref_collection_name,
- Collection *const ref_collection,
- const std::string& alias,
- const reference_filter_result_t& references,
- const tsl::htrie_set<char>& ref_include_fields_full,
- const tsl::htrie_set<char>& ref_exclude_fields_full,
- const std::string& error_prefix, const bool& is_reference_array,
- const ref_include::strategy_enum& strategy);
+ static Option<bool> prune_ref_doc(nlohmann::json& doc,
+ const reference_filter_result_t& references,
+ const tsl::htrie_set<char>& ref_include_fields_full,
+ const tsl::htrie_set<char>& ref_exclude_fields_full,
+ const bool& is_reference_array,
+ const ref_include_exclude_fields& ref_include_exclude);
+
+ static Option<bool> include_references(nlohmann::json& doc, const uint32_t& seq_id, Collection *const collection,
+ const std::map<std::string, reference_filter_result_t>& reference_filter_results,
+ const std::vector<ref_include_exclude_fields>& ref_include_exclude_fields_vec);
static Option<bool> prune_doc(nlohmann::json& doc, const tsl::htrie_set<char>& include_names,
const tsl::htrie_set<char>& exclude_names, const std::string& parent_name = "",
size_t depth = 0,
const std::map<std::string, reference_filter_result_t>& reference_filter_results = {},
Collection *const collection = nullptr, const uint32_t& seq_id = 0,
- const std::vector<ref_include_fields>& ref_include_fields_vec = {});
+ const std::vector<ref_include_exclude_fields>& ref_include_exclude_fields_vec = {});
const Index* _get_index() const;
@@ -558,7 +559,7 @@ public:
const size_t remote_embedding_num_tries = 2,
const std::string& stopwords_set="",
const std::vector<std::string>& facet_return_parent = {},
- const std::vector<ref_include_fields>& ref_include_fields_vec = {},
+ const std::vector<ref_include_exclude_fields>& ref_include_exclude_fields_vec = {},
const std::string& drop_tokens_mode = "right_to_left",
const bool prioritize_num_matching_fields = true,
const bool group_missing_values = true,
diff --git a/include/collection_manager.h b/include/collection_manager.h
index 889391f5..44484df1 100644
--- a/include/collection_manager.h
+++ b/include/collection_manager.h
@@ -221,10 +221,11 @@ public:
static void _get_reference_collection_names(const std::string& filter_query,
ref_include_collection_names_t*& reference_collection_names);
- // Separate out the reference includes into `ref_include_fields_vec`.
- static Option<bool> _initialize_ref_include_fields_vec(const std::string& filter_query,
- std::vector<std::string>& include_fields_vec,
- std::vector<ref_include_fields>& ref_include_fields_vec);
+ // Separate out the reference includes and excludes into `ref_include_exclude_fields_vec`.
+ static Option<bool> _initialize_ref_include_exclude_fields_vec(const std::string& filter_query,
+ std::vector<std::string>& include_fields_vec,
+ std::vector<std::string>& exclude_fields_vec,
+ std::vector<ref_include_exclude_fields>& ref_include_exclude_fields_vec);
void add_referenced_in_backlog(const std::string& collection_name, reference_pair&& pair);
diff --git a/include/field.h b/include/field.h
index b06e07c3..20c34427 100644
--- a/include/field.h
+++ b/include/field.h
@@ -530,14 +530,15 @@ namespace ref_include {
}
}
-struct ref_include_fields {
+struct ref_include_exclude_fields {
std::string collection_name;
- std::string fields;
+ std::string include_fields;
+ std::string exclude_fields;
std::string alias;
ref_include::strategy_enum strategy = ref_include::nest;
// In case we have nested join.
- std::vector<ref_include_fields> nested_join_includes = {};
+ std::vector<ref_include_exclude_fields> nested_join_includes = {};
};
struct hnsw_index_t;
diff --git a/include/filter_result_iterator.h b/include/filter_result_iterator.h
index 77da1bd8..1e89b1b7 100644
--- a/include/filter_result_iterator.h
+++ b/include/filter_result_iterator.h
@@ -14,8 +14,14 @@ struct filter_node_t;
struct reference_filter_result_t {
uint32_t count = 0;
uint32_t* docs = nullptr;
+ bool is_reference_array_field = true;
- explicit reference_filter_result_t(uint32_t count = 0, uint32_t* docs = nullptr) : count(count), docs(docs) {}
+ // In case of nested join, references can further have references.
+ std::map<std::string, reference_filter_result_t>* coll_to_references = nullptr;
+
+ explicit reference_filter_result_t(uint32_t count = 0, uint32_t* docs = nullptr,
+ bool is_reference_array_field = true) : count(count), docs(docs),
+ is_reference_array_field(is_reference_array_field) {}
reference_filter_result_t(const reference_filter_result_t& obj) {
if (&obj == this) {
@@ -25,6 +31,9 @@ struct reference_filter_result_t {
count = obj.count;
docs = new uint32_t[count];
memcpy(docs, obj.docs, count * sizeof(uint32_t));
+ is_reference_array_field = obj.is_reference_array_field;
+
+ copy_references(obj, *this);
}
reference_filter_result_t& operator=(const reference_filter_result_t& obj) noexcept {
@@ -35,7 +44,9 @@ struct reference_filter_result_t {
count = obj.count;
docs = new uint32_t[count];
memcpy(docs, obj.docs, count * sizeof(uint32_t));
+ is_reference_array_field = obj.is_reference_array_field;
+ copy_references(obj, *this);
return *this;
}
@@ -46,26 +57,38 @@ struct reference_filter_result_t {
count = obj.count;
docs = obj.docs;
+ coll_to_references = obj.coll_to_references;
+ is_reference_array_field = obj.is_reference_array_field;
+ // Set default values in obj.
+ obj.count = 0;
obj.docs = nullptr;
+ obj.coll_to_references = nullptr;
+ obj.is_reference_array_field = true;
return *this;
}
~reference_filter_result_t() {
delete[] docs;
+ delete[] coll_to_references;
}
+
+ static void copy_references(const reference_filter_result_t& from, reference_filter_result_t& to);
};
struct single_filter_result_t {
uint32_t seq_id = 0;
// Collection name -> Reference filter result
std::map<std::string, reference_filter_result_t> reference_filter_results = {};
+ bool is_reference_array_field = true;
single_filter_result_t() = default;
- single_filter_result_t(uint32_t seq_id, std::map<std::string, reference_filter_result_t>&& reference_filter_results) :
- seq_id(seq_id), reference_filter_results(std::move(reference_filter_results)) {}
+ single_filter_result_t(uint32_t seq_id, std::map<std::string, reference_filter_result_t>&& reference_filter_results,
+ bool is_reference_array_field = true) :
+ seq_id(seq_id), reference_filter_results(std::move(reference_filter_results)),
+ is_reference_array_field(is_reference_array_field) {}
single_filter_result_t(const single_filter_result_t& obj) {
if (&obj == this) {
@@ -73,12 +96,52 @@ struct single_filter_result_t {
}
seq_id = obj.seq_id;
+ is_reference_array_field = obj.is_reference_array_field;
+
+ // Copy every collection's reference.
+ for (const auto &item: obj.reference_filter_results) {
+ auto& ref_coll_name = item.first;
+ reference_filter_results[ref_coll_name] = item.second;
+ }
+ }
+
+ single_filter_result_t(single_filter_result_t&& obj) {
+ if (&obj == this) {
+ return;
+ }
+
+ seq_id = obj.seq_id;
+ is_reference_array_field = obj.is_reference_array_field;
+ reference_filter_results = std::move(obj.reference_filter_results);
+ }
+
+ single_filter_result_t& operator=(const single_filter_result_t& obj) noexcept {
+ if (&obj == this) {
+ return *this;
+ }
+
+ seq_id = obj.seq_id;
+ is_reference_array_field = obj.is_reference_array_field;
// Copy every collection's reference.
for (const auto &item: obj.reference_filter_results) {
auto& ref_coll_name = item.first;
reference_filter_results[ref_coll_name] = item.second;
}
+
+ return *this;
+ }
+
+ single_filter_result_t& operator=(single_filter_result_t&& obj) noexcept {
+ if (&obj == this) {
+ return *this;
+ }
+
+ seq_id = obj.seq_id;
+ is_reference_array_field = obj.is_reference_array_field;
+ reference_filter_results = std::move(obj.reference_filter_results);
+
+ return *this;
}
};
@@ -127,6 +190,8 @@ struct filter_result_t {
docs = obj.docs;
coll_to_references = obj.coll_to_references;
+ // Set default values in obj.
+ obj.count = 0;
obj.docs = nullptr;
obj.coll_to_references = nullptr;
diff --git a/include/index.h b/include/index.h
index db881e87..c767d280 100644
--- a/include/index.h
+++ b/include/index.h
@@ -786,12 +786,15 @@ public:
filter_result_t& filter_result,
const std::string& collection_name = "") const;
-
Option<bool> do_reference_filtering_with_lock(filter_node_t* const filter_tree_root,
filter_result_t& filter_result,
- const std::string& collection_name,
+ const std::string& ref_collection_name,
const std::string& reference_helper_field_name) const;
+ Option<filter_result_t> do_filtering_with_reference_ids(const std::string& reference_helper_field_name,
+ const std::string& ref_collection_name,
+ filter_result_t&& ref_filter_result) const;
+
void refresh_schemas(const std::vector<field>& new_fields, const std::vector<field>& del_fields);
// the following methods are not synchronized because their parent calls are synchronized or they are const/static
diff --git a/include/string_utils.h b/include/string_utils.h
index 0568a29e..e3d834e6 100644
--- a/include/string_utils.h
+++ b/include/string_utils.h
@@ -334,11 +334,11 @@ struct StringUtils {
static Option<bool> tokenize_filter_query(const std::string& filter_query, std::queue<std::string>& tokens);
- static Option<bool> split_include_fields(const std::string& include_fields, std::vector<std::string>& tokens);
+ static Option<bool> split_include_exclude_fields(const std::string& include_exclude_fields,
+ std::vector<std::string>& tokens);
static size_t get_occurence_count(const std::string& str, char symbol);
- static Option<bool> split_reference_include_fields(const std::string& include_fields,
- size_t& index,
- std::string& token);
+ static Option<bool> split_reference_include_exclude_fields(const std::string& include_fields,
+ size_t& index, std::string& token);
};
diff --git a/src/collection.cpp b/src/collection.cpp
index 83131a6b..95563136 100644
--- a/src/collection.cpp
+++ b/src/collection.cpp
@@ -1611,7 +1611,7 @@ Option<nlohmann::json> Collection::search(std::string raw_query,
const size_t remote_embedding_num_tries,
const std::string& stopwords_set,
const std::vector<std::string>& facet_return_parent,
- const std::vector<ref_include_fields>& ref_include_fields_vec,
+ const std::vector<ref_include_exclude_fields>& ref_include_exclude_fields_vec,
const std::string& drop_tokens_mode,
const bool prioritize_num_matching_fields,
const bool group_missing_values,
@@ -2547,7 +2547,7 @@ Option<nlohmann::json> Collection::search(std::string raw_query,
0,
field_order_kv->reference_filter_results,
const_cast<Collection *>(this), get_seq_id_from_key(seq_id_key),
- ref_include_fields_vec);
+ ref_include_exclude_fields_vec);
if (!prune_op.ok()) {
return Option<nlohmann::json>(prune_op.code(), prune_op.error());
}
@@ -4862,15 +4862,23 @@ void Collection::remove_reference_helper_fields(nlohmann::json& document) {
}
}
-Option<bool> Collection::include_references(nlohmann::json& doc,
- const std::string& ref_collection_name,
- Collection *const ref_collection,
- const std::string& alias,
- const reference_filter_result_t& references,
- const tsl::htrie_set<char>& ref_include_fields_full,
- const tsl::htrie_set<char>& ref_exclude_fields_full,
- const std::string& error_prefix, const bool& is_reference_array,
- const ref_include::strategy_enum& strategy) {
+Option<bool> Collection::prune_ref_doc(nlohmann::json& doc,
+ const reference_filter_result_t& references,
+ const tsl::htrie_set<char>& ref_include_fields_full,
+ const tsl::htrie_set<char>& ref_exclude_fields_full,
+ const bool& is_reference_array,
+ const ref_include_exclude_fields& ref_include_exclude) {
+ auto const& ref_collection_name = ref_include_exclude.collection_name;
+ auto& cm = CollectionManager::get_instance();
+ auto ref_collection = cm.get_collection(ref_collection_name);
+ if (ref_collection == nullptr) {
+ return Option<bool>(400, "Referenced collection `" + ref_collection_name + "` in `include_fields` not found.");
+ }
+
+ auto const& alias = ref_include_exclude.alias;
+ auto const& strategy = ref_include_exclude.strategy;
+ auto error_prefix = "Referenced collection `" + ref_collection_name + "`: ";
+
// One-to-one relation.
if (strategy != ref_include::nest_array && !is_reference_array && references.count == 1) {
auto ref_doc_seq_id = references.docs[0];
@@ -4889,23 +4897,33 @@ Option<bool> Collection::include_references(nlohmann::json& doc,
return Option<bool>(prune_op.code(), error_prefix + prune_op.error());
}
- if (ref_doc.empty()) {
- return Option<bool>(true);
+ auto const key = alias.empty() ? ref_collection_name : alias;
+ auto const& nest_ref_doc = (strategy == ref_include::nest);
+ if (!ref_doc.empty()) {
+ if (nest_ref_doc) {
+ doc[key] = ref_doc;
+ } else {
+ if (!alias.empty()) {
+ auto temp_doc = ref_doc;
+ ref_doc.clear();
+ for (const auto &item: temp_doc.items()) {
+ ref_doc[alias + item.key()] = item.value();
+ }
+ }
+ doc.update(ref_doc);
+ }
}
- if (strategy == ref_include::nest) {
- auto key = alias.empty() ? ref_collection_name : alias;
- doc[key] = ref_doc;
- } else {
- if (!alias.empty()) {
- auto temp_doc = ref_doc;
- ref_doc.clear();
- for (const auto &item: temp_doc.items()) {
- ref_doc[alias + item.key()] = item.value();
- }
+ // Include nested join references.
+ if (!ref_include_exclude.nested_join_includes.empty() && !references.coll_to_references->empty()) {
+ auto nested_include_exclude_op = include_references(nest_ref_doc ? doc[key] : doc, ref_doc_seq_id,
+ ref_collection.get(), references.coll_to_references[0],
+ ref_include_exclude.nested_join_includes);
+ if (!nested_include_exclude_op.ok()) {
+ return nested_include_exclude_op;
}
- doc.update(ref_doc);
}
+
return Option<bool>(true);
}
@@ -4927,123 +4945,55 @@ Option<bool> Collection::include_references(nlohmann::json& doc,
return Option<bool>(prune_op.code(), error_prefix + prune_op.error());
}
- if (ref_doc.empty()) {
- continue;
- }
-
- if (strategy == ref_include::nest || strategy == ref_include::nest_array) {
- auto key = alias.empty() ? ref_collection_name : alias;
- if (doc.contains(key) && !doc[key].is_array()) {
- return Option<bool>(400, "Could not include the reference document of `" + ref_collection_name +
- "` collection. Expected `" + key + "` to be an array. Try " +
- (alias.empty() ? "adding an" : "renaming the") + " alias.");
- }
-
- doc[key] += ref_doc;
- } else {
- for (auto ref_doc_it = ref_doc.begin(); ref_doc_it != ref_doc.end(); ref_doc_it++) {
- auto const& ref_doc_key = ref_doc_it.key();
- auto const& doc_key = alias + ref_doc_key;
- if (doc.contains(doc_key) && !doc[doc_key].is_array()) {
- return Option<bool>(400, "Could not include the value of `" + ref_doc_key +
- "` key of the reference document of `" + ref_collection_name +
- "` collection. Expected `" + doc_key + "` to be an array. Try " +
+ std::string key;
+ auto const& nest_ref_doc = (strategy == ref_include::nest || strategy == ref_include::nest_array);
+ if (!ref_doc.empty()) {
+ if (nest_ref_doc) {
+ key = alias.empty() ? ref_collection_name : alias;
+ if (doc.contains(key) && !doc[key].is_array()) {
+ return Option<bool>(400, "Could not include the reference document of `" + ref_collection_name +
+ "` collection. Expected `" + key + "` to be an array. Try " +
(alias.empty() ? "adding an" : "renaming the") + " alias.");
}
- // Add the values of ref_doc as JSON array into doc.
- doc[doc_key] += ref_doc_it.value();
- }
- }
- }
-
- return Option<bool>(true);
-}
-
-Option<bool> Collection::prune_doc(nlohmann::json& doc,
- const tsl::htrie_set<char>& include_names,
- const tsl::htrie_set<char>& exclude_names,
- const std::string& parent_name, size_t depth,
- const std::map<std::string, reference_filter_result_t>& reference_filter_results,
- Collection *const collection, const uint32_t& seq_id,
- const std::vector<ref_include_fields>& ref_includes) {
- // doc can only be an object
- auto it = doc.begin();
- while(it != doc.end()) {
- std::string nested_name = parent_name + (parent_name.empty() ? it.key() : "." + it.key());
-
- //LOG(INFO) << "it.key(): " << it.key() << ", nested_name: " << nested_name;
-
- // use prefix lookup to prune non-matching sub-trees early
- auto prefix_it = include_names.equal_prefix_range(nested_name);
- if(!include_names.empty() && prefix_it.first == prefix_it.second) {
- // prefix not found in allowed list of highlight field names, so can trim early
- it = doc.erase(it);
- continue ;
- }
-
- if(exclude_names.count(nested_name) != 0) {
- it = doc.erase(it);
- continue ;
- }
-
- if(exclude_names.empty() && !include_names.empty() && include_names.count(nested_name) != 0) {
- // without exclusions, we can pick the sub-tree early if parent name is found in include names
- it++;
- continue;
- }
-
- if(it.value().is_object()) {
- bool is_orig_empty = it.value().empty();
- prune_doc(it.value(), include_names, exclude_names, nested_name, depth+1);
- if(!is_orig_empty && it.value().empty()) {
- it = doc.erase(it);
+ doc[key] += ref_doc;
} else {
- it++;
- }
-
- continue;
- }
+ for (auto ref_doc_it = ref_doc.begin(); ref_doc_it != ref_doc.end(); ref_doc_it++) {
+ auto const& ref_doc_key = ref_doc_it.key();
+ key = alias + ref_doc_key;
+ if (doc.contains(key) && !doc[key].is_array()) {
+ return Option<bool>(400, "Could not include the value of `" + ref_doc_key +
+ "` key of the reference document of `" + ref_collection_name +
+ "` collection. Expected `" + key + "` to be an array. Try " +
+ (alias.empty() ? "adding an" : "renaming the") + " alias.");
+ }
- else if(it.value().is_array()) {
- bool orig_array_empty = it.value().empty();
- bool primitive_array = true;
- auto arr_it = it.value().begin();
- while(arr_it != it.value().end()) {
- // NOTE: we will not support array of array of nested objects
- primitive_array = primitive_array && !arr_it.value().is_object();
- if(arr_it.value().is_object()) {
- bool orig_ele_empty = arr_it.value().empty();
- prune_doc(arr_it.value(), include_names, exclude_names, nested_name, depth+1);
- // don't remove empty array objects to help frontend
+ // Add the values of ref_doc as JSON array into doc.
+ doc[key] += ref_doc_it.value();
}
-
- arr_it++;
- }
-
- if(!orig_array_empty && it.value().empty()) {
- // only drop field if array became empty because of pruning (and not empty already)
- it = doc.erase(it);
- continue;
- }
-
- if(!primitive_array) {
- it++;
- continue;
}
}
- if(!include_names.empty() && include_names.count(nested_name) == 0) {
- // at this point, name should match fully, otherwise we should erase the value
- it = doc.erase(it);
- continue;
+ // Include nested join references.
+ if (!ref_include_exclude.nested_join_includes.empty() &&
+ references.coll_to_references != nullptr && !references.coll_to_references->empty()) {
+ auto nested_include_exclude_op = include_references(nest_ref_doc ? doc[key].at(i) : doc, ref_doc_seq_id,
+ ref_collection.get(), references.coll_to_references[i],
+ ref_include_exclude.nested_join_includes);
+ if (!nested_include_exclude_op.ok()) {
+ return nested_include_exclude_op;
+ }
}
-
- it++;
}
- for (auto const& ref_include: ref_includes) {
- auto const& ref_collection_name = ref_include.collection_name;
+ return Option<bool>(true);
+}
+
+Option<bool> Collection::include_references(nlohmann::json& doc, const uint32_t& seq_id, Collection *const collection,
+ const std::map<std::string, reference_filter_result_t>& reference_filter_results,
+ const std::vector<ref_include_exclude_fields>& ref_include_exclude_fields_vec) {
+ for (auto const& ref_include_exclude: ref_include_exclude_fields_vec) {
+ auto const& ref_collection_name = ref_include_exclude.collection_name;
auto& cm = CollectionManager::get_instance();
auto ref_collection = cm.get_collection(ref_collection_name);
@@ -5052,8 +5002,8 @@ Option<bool> Collection::prune_doc(nlohmann::json& doc,
}
auto const joined_on_ref_collection = reference_filter_results.count(ref_collection_name) > 0,
- has_filter_reference = (joined_on_ref_collection &&
- reference_filter_results.at(ref_collection_name).count > 0);
+ has_filter_reference = (joined_on_ref_collection &&
+ reference_filter_results.at(ref_collection_name).count > 0);
auto doc_has_reference = false, joined_coll_has_reference = false;
// Reference include_by without join, check if doc itself contains the reference.
@@ -5078,45 +5028,29 @@ Option<bool> Collection::prune_doc(nlohmann::json& doc,
}
std::vector<std::string> ref_include_fields_vec, ref_exclude_fields_vec;
- StringUtils::split(ref_include.fields, ref_include_fields_vec, ",");
- auto exclude_reference_it = exclude_names.equal_prefix_range("$" + ref_collection_name);
- if (exclude_reference_it.first != exclude_reference_it.second) {
- auto ref_exclude = exclude_reference_it.first.key();
- auto parenthesis_index = ref_exclude.find('(');
- auto reference_fields = ref_exclude.substr(parenthesis_index + 1, ref_exclude.size() - parenthesis_index - 2);
- StringUtils::split(reference_fields, ref_exclude_fields_vec, ",");
- }
+ StringUtils::split(ref_include_exclude.include_fields, ref_include_fields_vec, ",");
+ StringUtils::split(ref_include_exclude.exclude_fields, ref_exclude_fields_vec, ",");
spp::sparse_hash_set<std::string> ref_include_fields, ref_exclude_fields;
ref_include_fields.insert(ref_include_fields_vec.begin(), ref_include_fields_vec.end());
ref_exclude_fields.insert(ref_exclude_fields_vec.begin(), ref_exclude_fields_vec.end());
tsl::htrie_set<char> ref_include_fields_full, ref_exclude_fields_full;
- auto include_exclude_op = ref_collection->populate_include_exclude_fields(ref_include_fields,
- ref_exclude_fields,
- ref_include_fields_full,
- ref_exclude_fields_full);
+ auto include_exclude_op = ref_collection->populate_include_exclude_fields_lk(ref_include_fields,
+ ref_exclude_fields,
+ ref_include_fields_full,
+ ref_exclude_fields_full);
auto error_prefix = "Referenced collection `" + ref_collection_name + "`: ";
if (!include_exclude_op.ok()) {
return Option<bool>(include_exclude_op.code(), error_prefix + include_exclude_op.error());
}
- Option<bool> include_references_op = Option<bool>(true);
+ Option<bool> prune_doc_op = Option<bool>(true);
+ auto const& ref_collection_alias = ref_include_exclude.alias;
if (has_filter_reference) {
- auto get_reference_field_op = collection->get_referenced_in_field(ref_collection_name);
- if (!get_reference_field_op.ok()) {
- continue;
- }
- auto const& field_name = get_reference_field_op.get();
- if (ref_collection->search_schema.count(field_name) == 0) {
- continue;
- }
- include_references_op = include_references(doc, ref_include.collection_name,
- ref_collection.get(), ref_include.alias,
- reference_filter_results.at(ref_collection_name),
- ref_include_fields_full, ref_exclude_fields_full, error_prefix,
- ref_collection->get_schema().at(field_name).is_array(),
- ref_include.strategy);
+ auto const& ref_filter_result = reference_filter_results.at(ref_collection_name);
+ prune_doc_op = prune_ref_doc(doc, ref_filter_result, ref_include_fields_full, ref_exclude_fields_full,
+ ref_filter_result.is_reference_array_field, ref_include_exclude);
} else if (doc_has_reference) {
auto get_reference_field_op = ref_collection->get_referenced_in_field_with_lock(collection->name);
if (!get_reference_field_op.ok()) {
@@ -5148,12 +5082,11 @@ Option<bool> Collection::prune_doc(nlohmann::json& doc,
}
reference_filter_result_t result(1, new uint32_t[1]{ref_doc_id});
- include_references_op = include_references(doc[keys[0]][i], ref_include.collection_name,
- ref_collection.get(), ref_include.alias, result,
- ref_include_fields_full, ref_exclude_fields_full, error_prefix,
- false, ref_include.strategy);
- if (!include_references_op.ok()) {
- return include_references_op;
+ prune_doc_op = prune_ref_doc(doc[keys[0]][i], result,
+ ref_include_fields_full, ref_exclude_fields_full,
+ false, ref_include_exclude);
+ if (!prune_doc_op.ok()) {
+ return prune_doc_op;
}
}
} else {
@@ -5163,11 +5096,8 @@ Option<bool> Collection::prune_doc(nlohmann::json& doc,
continue;
}
reference_filter_result_t result(ids.size(), &ids[0]);
- include_references_op = include_references(doc[keys[0]], ref_include.collection_name,
- ref_collection.get(), ref_include.alias, result,
- ref_include_fields_full, ref_exclude_fields_full, error_prefix,
- collection->search_schema.at(field_name).is_array(),
- ref_include.strategy);
+ prune_doc_op = prune_ref_doc(doc[keys[0]], result, ref_include_fields_full, ref_exclude_fields_full,
+ collection->search_schema.at(field_name).is_array(), ref_include_exclude);
result.docs = nullptr;
}
} else {
@@ -5177,11 +5107,8 @@ Option<bool> Collection::prune_doc(nlohmann::json& doc,
continue;
}
reference_filter_result_t result(ids.size(), &ids[0]);
- include_references_op = include_references(doc, ref_include.collection_name,
- ref_collection.get(), ref_include.alias, result,
- ref_include_fields_full, ref_exclude_fields_full, error_prefix,
- collection->search_schema.at(field_name).is_array(),
- ref_include.strategy);
+ prune_doc_op = prune_ref_doc(doc, result, ref_include_fields_full, ref_exclude_fields_full,
+ collection->search_schema.at(field_name).is_array(), ref_include_exclude);
result.docs = nullptr;
}
} else if (joined_coll_has_reference) {
@@ -5213,22 +5140,105 @@ Option<bool> Collection::prune_doc(nlohmann::json& doc,
reference_filter_result_t result;
result.count = ids.size();
result.docs = &ids[0];
- include_references_op = include_references(doc, ref_include.collection_name,
- ref_collection.get(), ref_include.alias, result,
- ref_include_fields_full, ref_exclude_fields_full, error_prefix,
- joined_collection->get_schema().at(reference_field_name).is_array(),
- ref_include.strategy);
+ prune_doc_op = prune_ref_doc(doc, result, ref_include_fields_full, ref_exclude_fields_full,
+ joined_collection->get_schema().at(reference_field_name).is_array(),
+ ref_include_exclude);
result.docs = nullptr;
}
- if (!include_references_op.ok()) {
- return include_references_op;
+ if (!prune_doc_op.ok()) {
+ return prune_doc_op;
}
}
return Option<bool>(true);
}
+Option<bool> Collection::prune_doc(nlohmann::json& doc,
+ const tsl::htrie_set<char>& include_names,
+ const tsl::htrie_set<char>& exclude_names,
+ const std::string& parent_name, size_t depth,
+ const std::map<std::string, reference_filter_result_t>& reference_filter_results,
+ Collection *const collection, const uint32_t& seq_id,
+ const std::vector<ref_include_exclude_fields>& ref_include_exclude_fields_vec) {
+ // doc can only be an object
+ auto it = doc.begin();
+ while(it != doc.end()) {
+ std::string nested_name = parent_name + (parent_name.empty() ? it.key() : "." + it.key());
+
+ //LOG(INFO) << "it.key(): " << it.key() << ", nested_name: " << nested_name;
+
+ // use prefix lookup to prune non-matching sub-trees early
+ auto prefix_it = include_names.equal_prefix_range(nested_name);
+ if(!include_names.empty() && prefix_it.first == prefix_it.second) {
+ // prefix not found in allowed list of highlight field names, so can trim early
+ it = doc.erase(it);
+ continue ;
+ }
+
+ if(exclude_names.count(nested_name) != 0) {
+ it = doc.erase(it);
+ continue ;
+ }
+
+ if(exclude_names.empty() && !include_names.empty() && include_names.count(nested_name) != 0) {
+ // without exclusions, we can pick the sub-tree early if parent name is found in include names
+ it++;
+ continue;
+ }
+
+ if(it.value().is_object()) {
+ bool is_orig_empty = it.value().empty();
+ prune_doc(it.value(), include_names, exclude_names, nested_name, depth+1);
+ if(!is_orig_empty && it.value().empty()) {
+ it = doc.erase(it);
+ } else {
+ it++;
+ }
+
+ continue;
+ }
+
+ else if(it.value().is_array()) {
+ bool orig_array_empty = it.value().empty();
+ bool primitive_array = true;
+ auto arr_it = it.value().begin();
+ while(arr_it != it.value().end()) {
+ // NOTE: we will not support array of array of nested objects
+ primitive_array = primitive_array && !arr_it.value().is_object();
+ if(arr_it.value().is_object()) {
+ bool orig_ele_empty = arr_it.value().empty();
+ prune_doc(arr_it.value(), include_names, exclude_names, nested_name, depth+1);
+ // don't remove empty array objects to help frontend
+ }
+
+ arr_it++;
+ }
+
+ if(!orig_array_empty && it.value().empty()) {
+ // only drop field if array became empty because of pruning (and not empty already)
+ it = doc.erase(it);
+ continue;
+ }
+
+ if(!primitive_array) {
+ it++;
+ continue;
+ }
+ }
+
+ if(!include_names.empty() && include_names.count(nested_name) == 0) {
+ // at this point, name should match fully, otherwise we should erase the value
+ it = doc.erase(it);
+ continue;
+ }
+
+ it++;
+ }
+
+ return include_references(doc, seq_id, collection, reference_filter_results, ref_include_exclude_fields_vec);
+}
+
Option<bool> Collection::validate_alter_payload(nlohmann::json& schema_changes,
std::vector<field>& addition_fields,
std::vector<field>& reindex_fields,
diff --git a/src/collection_manager.cpp b/src/collection_manager.cpp
index 1758a69e..aa4184c9 100644
--- a/src/collection_manager.cpp
+++ b/src/collection_manager.cpp
@@ -982,9 +982,61 @@ void CollectionManager::_get_reference_collection_names(const std::string& filte
}
}
+Option<bool> parse_nested_exclude(const std::string& exclude_field_exp,
+ std::unordered_map<std::string, std::string>& ref_excludes) {
+ // Format: $ref_collection_name(field_1, field_2, $nested_ref_coll(nested_field_1))
+ size_t index = 0;
+ while (index < exclude_field_exp.size()) {
+ auto parenthesis_index = exclude_field_exp.find('(');
+ auto ref_collection_name = exclude_field_exp.substr(index + 1, parenthesis_index - index - 1);
+ std::string ref_fields;
+
+ index = parenthesis_index + 1;
+ auto nested_exclude_pos = exclude_field_exp.find('$', parenthesis_index);
+ auto closing_parenthesis_pos = exclude_field_exp.find(')', parenthesis_index);
+ size_t comma_pos;
+ if (nested_exclude_pos < closing_parenthesis_pos) {
+ // Nested reference exclude.
+ // "... $product_variants(title, $inventory(qty)) ..."
+ do {
+ ref_fields += exclude_field_exp.substr(index, nested_exclude_pos - index);
+ StringUtils::trim(ref_fields);
+ index = nested_exclude_pos;
+ std::string nested_exclude_field_exp;
+ auto split_op = StringUtils::split_reference_include_exclude_fields(exclude_field_exp, index,
+ nested_exclude_field_exp);
+ if (!split_op.ok()) {
+ return split_op;
+ }
+
+ auto parse_op = parse_nested_exclude(nested_exclude_field_exp, ref_excludes);
+ if (!parse_op.ok()) {
+ return parse_op;
+ }
+
+ nested_exclude_pos = exclude_field_exp.find('$', index);
+ closing_parenthesis_pos = exclude_field_exp.find(')', index);
+ comma_pos = exclude_field_exp.find(',', index);
+ index = std::min(closing_parenthesis_pos, comma_pos) + 1;
+ } while (index < exclude_field_exp.size() && nested_exclude_pos < closing_parenthesis_pos);
+ }
+
+ // ... $inventory(qty) ...
+ if (index < closing_parenthesis_pos) {
+ ref_fields += exclude_field_exp.substr(index, closing_parenthesis_pos - index);
+ }
+ StringUtils::trim(ref_fields);
+
+ ref_excludes[ref_collection_name] = ref_fields;
+ index = closing_parenthesis_pos + 1;
+ }
+
+ return Option<bool>(true);
+}
+
Option<bool> parse_nested_include(const std::string& include_field_exp,
CollectionManager::ref_include_collection_names_t* const ref_include_coll_names,
- std::vector<ref_include_fields>& ref_include_fields_vec) {
+ std::vector<ref_include_exclude_fields>& ref_include_exclude_fields_vec) {
// Format: $ref_collection_name(field_1, field_2, $nested_ref_coll(nested_field_1: nested_include_strategy) as nested_ref_alias: include_strategy) as ref_alias
size_t index = 0;
while (index < include_field_exp.size()) {
@@ -998,7 +1050,7 @@ Option<bool> parse_nested_include(const std::string& include_field_exp,
auto closing_parenthesis_pos = include_field_exp.find(')', parenthesis_index);
auto colon_pos = include_field_exp.find(':', index);
size_t comma_pos;
- std::vector<ref_include_fields> nested_ref_include_fields_vec;
+ std::vector<ref_include_exclude_fields> nested_ref_include_exclude_fields_vec;
if (nested_include_pos < closing_parenthesis_pos) {
// Nested reference include.
// "... $product_variants(title, $inventory(qty:merge) as inventory :nest) as variants ..."
@@ -1007,15 +1059,15 @@ Option<bool> parse_nested_include(const std::string& include_field_exp,
StringUtils::trim(ref_fields);
index = nested_include_pos;
std::string nested_include_field_exp;
- auto split_op = StringUtils::split_reference_include_fields(include_field_exp, index,
- nested_include_field_exp);
+ auto split_op = StringUtils::split_reference_include_exclude_fields(include_field_exp, index,
+ nested_include_field_exp);
if (!split_op.ok()) {
return split_op;
}
auto parse_op = parse_nested_include(nested_include_field_exp,
ref_include_coll_names == nullptr ? nullptr : ref_include_coll_names->nested_include,
- nested_ref_include_fields_vec);
+ nested_ref_include_exclude_fields_vec);
if (!parse_op.ok()) {
return parse_op;
}
@@ -1062,11 +1114,11 @@ Option<bool> parse_nested_include(const std::string& include_field_exp,
nest_ref_doc = strategy_enum == ref_include::nest || strategy_enum == ref_include::nest_array;
ref_alias = !ref_alias.empty() ? (StringUtils::trim(ref_alias) + (nest_ref_doc ? "" : ".")) : "";
- ref_include_fields_vec.emplace_back(ref_include_fields{ref_collection_name, ref_fields, ref_alias,
- strategy_enum});
- ref_include_fields_vec.back().nested_join_includes = std::move(nested_ref_include_fields_vec);
+ ref_include_exclude_fields_vec.emplace_back(ref_include_exclude_fields{ref_collection_name, ref_fields, "",
+ ref_alias, strategy_enum});
+ ref_include_exclude_fields_vec.back().nested_join_includes = std::move(nested_ref_include_exclude_fields_vec);
- // Referenced collection in filter_by is already mentioned in ref_include_fields.
+ // Referenced collection in filter_by is already mentioned in include_fields.
if (ref_include_coll_names != nullptr) {
ref_include_coll_names->collection_names.erase(ref_collection_name);
}
@@ -1079,9 +1131,10 @@ Option<bool> parse_nested_include(const std::string& include_field_exp,
return Option<bool>(true);
}
-Option<bool> CollectionManager::_initialize_ref_include_fields_vec(const std::string& filter_query,
- std::vector<std::string>& include_fields_vec,
- std::vector<ref_include_fields>& ref_include_fields_vec) {
+Option<bool> CollectionManager::_initialize_ref_include_exclude_fields_vec(const std::string& filter_query,
+ std::vector<std::string>& include_fields_vec,
+ std::vector<std::string>& exclude_fields_vec,
+ std::vector<ref_include_exclude_fields>& ref_include_exclude_fields_vec) {
ref_include_collection_names_t* ref_include_coll_names = nullptr;
CollectionManager::_get_reference_collection_names(filter_query, ref_include_coll_names);
std::unique_ptr<CollectionManager::ref_include_collection_names_t> guard(ref_include_coll_names);
@@ -1101,7 +1154,7 @@ Option<bool> CollectionManager::_initialize_ref_include_fields_vec(const std::st
// Nested reference include.
if (include_field_exp.find('$', 1) != std::string::npos) {
- auto parse_op = parse_nested_include(include_field_exp, ref_include_coll_names, ref_include_fields_vec);
+ auto parse_op = parse_nested_include(include_field_exp, ref_include_coll_names, ref_include_exclude_fields_vec);
if (!parse_op.ok()) {
return parse_op;
}
@@ -1139,46 +1192,77 @@ Option<bool> CollectionManager::_initialize_ref_include_fields_vec(const std::st
// In case of "nest" reference doc, `foo` becomes the key with reference doc as value.
auto const& nest_ref_doc = strategy_enum == ref_include::nest || strategy_enum == ref_include::nest_array;
auto ref_alias = !alias.empty() ? (StringUtils::trim(alias) + (nest_ref_doc ? "" : ".")) : "";
- ref_include_fields_vec.emplace_back(ref_include_fields{ref_collection_name, ref_fields, ref_alias,
- strategy_enum});
+ ref_include_exclude_fields_vec.emplace_back(ref_include_exclude_fields{ref_collection_name, ref_fields, "",
+ ref_alias, strategy_enum});
- auto open_paren_pos = include_field_exp.find('(');
- if (open_paren_pos == std::string::npos) {
- continue;
- }
-
- auto reference_collection_name = include_field_exp.substr(1, open_paren_pos - 1);
- StringUtils::trim(reference_collection_name);
- if (reference_collection_name.empty()) {
- continue;
- }
-
- // Referenced collection in filter_by is already mentioned in ref_include_fields.
+ // Referenced collection in filter_by is already mentioned in include_fields.
if (ref_include_coll_names != nullptr) {
- ref_include_coll_names->collection_names.erase(reference_collection_name);
+ ref_include_coll_names->collection_names.erase(ref_collection_name);
}
}
- // Get all the fields of the referenced collection in the filter but not mentioned in include_fields.
- auto ref_includes = std::ref(ref_include_fields_vec);
+ // Get all the fields of the referenced collection mentioned in the filter_by but not in include_fields.
+ auto references = std::ref(ref_include_exclude_fields_vec);
while (ref_include_coll_names != nullptr) {
for (const auto &reference_collection_name: ref_include_coll_names->collection_names) {
- ref_includes.get().emplace_back(ref_include_fields{reference_collection_name, "", "", ref_include::nest});
+ references.get().emplace_back(ref_include_exclude_fields{reference_collection_name, "", "", ""});
}
ref_include_coll_names = ref_include_coll_names->nested_include;
- if (ref_includes.get().empty()) {
+ if (references.get().empty()) {
break;
}
- ref_includes = std::ref(ref_includes.get().front().nested_join_includes);
+ references = std::ref(references.get().front().nested_join_includes);
+ }
+
+ std::unordered_map<std::string, std::string> ref_excludes;
+ std::vector<std::string> result_exclude_fields_vec;
+ for (const auto& exclude_field_exp: exclude_fields_vec) {
+ if (exclude_field_exp[0] != '$') {
+ result_exclude_fields_vec.emplace_back(exclude_field_exp);
+ continue;
+ }
+
+ // Nested reference exclude.
+ if (exclude_field_exp.find('$', 1) != std::string::npos) {
+ auto parse_op = parse_nested_exclude(exclude_field_exp, ref_excludes);
+ if (!parse_op.ok()) {
+ return parse_op;
+ }
+ continue;
+ }
+
+ // Format: $ref_collection_name(field_1, field_2)
+ auto parenthesis_index = exclude_field_exp.find('(');
+ auto ref_collection_name = exclude_field_exp.substr(1, parenthesis_index - 1);
+ auto ref_fields = exclude_field_exp.substr(parenthesis_index + 1, exclude_field_exp.size() - parenthesis_index - 2);
+ if (!ref_fields.empty()) {
+ ref_excludes[ref_collection_name] = ref_fields;
+ }
+ }
+
+ if (!ref_excludes.empty()) {
+ references = std::ref(ref_include_exclude_fields_vec);
+ while (!references.get().empty()) {
+ for (auto& ref_include_exclude: references.get()) {
+ if (ref_excludes.count(ref_include_exclude.collection_name) == 0) {
+ continue;
+ }
+
+ ref_include_exclude.exclude_fields = ref_excludes[ref_include_exclude.collection_name];
+ }
+
+ references = std::ref(references.get().front().nested_join_includes);
+ }
}
- // Since no field of the collection is mentioned in include_fields, get all the fields.
+ // Since no field of the collection being searched is mentioned in include_fields, include all the fields.
if (wildcard_include_all) {
result_include_fields_vec.clear();
}
include_fields_vec = std::move(result_include_fields_vec);
+ exclude_fields_vec = std::move(result_exclude_fields_vec);
return Option<bool>(true);
}
@@ -1356,7 +1440,7 @@ Option<bool> CollectionManager::do_search(std::map<std::string, std::string>& re
std::vector<std::string> include_fields_vec;
std::vector<std::string> exclude_fields_vec;
- std::vector<ref_include_fields> ref_include_fields_vec;
+ std::vector<ref_include_exclude_fields> ref_include_exclude_fields_vec;
spp::sparse_hash_set<std::string> include_fields;
spp::sparse_hash_set<std::string> exclude_fields;
@@ -1540,8 +1624,8 @@ Option<bool> CollectionManager::do_search(std::map<std::string, std::string>& re
if(key == FACET_BY){
StringUtils::split_facet(val, *find_str_list_it->second);
}
- else if(key == INCLUDE_FIELDS){
- auto op = StringUtils::split_include_fields(val, *find_str_list_it->second);
+ else if(key == INCLUDE_FIELDS || key == EXCLUDE_FIELDS){
+ auto op = StringUtils::split_include_exclude_fields(val, *find_str_list_it->second);
if (!op.ok()) {
return op;
}
@@ -1566,7 +1650,8 @@ Option<bool> CollectionManager::do_search(std::map<std::string, std::string>& re
per_page = 0;
}
- auto initialize_op = _initialize_ref_include_fields_vec(filter_query, include_fields_vec, ref_include_fields_vec);
+ auto initialize_op = _initialize_ref_include_exclude_fields_vec(filter_query, include_fields_vec, exclude_fields_vec,
+ ref_include_exclude_fields_vec);
if (!initialize_op.ok()) {
return initialize_op;
}
@@ -1662,7 +1747,7 @@ Option<bool> CollectionManager::do_search(std::map<std::string, std::string>& re
remote_embedding_num_tries,
stopwords_set,
facet_return_parent,
- ref_include_fields_vec,
+ ref_include_exclude_fields_vec,
drop_tokens_mode_str,
prioritize_num_matching_fields,
group_missing_values,
diff --git a/src/filter_result_iterator.cpp b/src/filter_result_iterator.cpp
index b3254aaf..44867fec 100644
--- a/src/filter_result_iterator.cpp
+++ b/src/filter_result_iterator.cpp
@@ -14,23 +14,31 @@
#include "posting.h"
#include "collection_manager.h"
-void filter_result_t::copy_references(const filter_result_t& from, filter_result_t& to) {
- if (from.coll_to_references == nullptr) {
+void copy_references_helper(const std::map<std::string, reference_filter_result_t>* from,
+ std::map<std::string, reference_filter_result_t>*& to, const uint32_t& count) {
+ if (from == nullptr) {
return;
}
- auto const& count = from.count;
- to.coll_to_references = new std::map<std::string, reference_filter_result_t>[count] {};
+ to = new std::map<std::string, reference_filter_result_t>[count] {};
for (uint32_t i = 0; i < count; i++) {
- if (from.coll_to_references[i].empty()) {
+ if (from[i].empty()) {
continue;
}
- auto& ref = to.coll_to_references[i];
- ref.insert(from.coll_to_references[i].begin(), from.coll_to_references[i].end());
+ auto& ref = to[i];
+ ref.insert(from[i].begin(), from[i].end());
}
}
+void reference_filter_result_t::copy_references(const reference_filter_result_t& from, reference_filter_result_t& to) {
+ return copy_references_helper(from.coll_to_references, to.coll_to_references, from.count);
+}
+
+void filter_result_t::copy_references(const filter_result_t& from, filter_result_t& to) {
+ return copy_references_helper(from.coll_to_references, to.coll_to_references, from.count);
+}
+
void filter_result_t::and_filter_results(const filter_result_t& a, const filter_result_t& b, filter_result_t& result) {
auto lenA = a.count, lenB = b.count;
if (lenA == 0 || lenB == 0) {
@@ -660,26 +668,16 @@ void filter_result_iterator_t::init() {
return;
}
- std::vector<std::string> values;
- for (uint32_t i = 0; i < result.count; i++) {
- values.push_back(std::to_string(result.docs[i]));
- }
-
- filter filter_exp = {get_reference_field_op.get(), std::move(values),
- std::vector<NUM_COMPARATOR>(result.count, EQUALS)};
-
- auto filter_tree_root = new filter_node_t(filter_exp);
- std::unique_ptr<filter_node_t> filter_tree_root_guard(filter_tree_root);
-
- auto fit = filter_result_iterator_t(collection_name, index, filter_tree_root);
- auto filter_init_op = fit.init_status();
- if (!filter_init_op.ok()) {
- status = Option<bool>(filter_init_op.code(), filter_init_op.error());
+ auto const& reference_helper_field_name = get_reference_field_op.get();
+ auto op = index->do_filtering_with_reference_ids(reference_helper_field_name, ref_collection_name,
+ std::move(result));
+ if (!op.ok()) {
+ status = Option<bool>(op.code(), op.error());
validity = invalid;
return;
}
- filter_result = std::move(fit.filter_result);
+ filter_result = op.get();
}
if (filter_result.count == 0) {
@@ -1304,6 +1302,7 @@ int filter_result_iterator_t::is_valid(uint32_t id) {
}
seq_id = id;
+ and_filter_iterators();
return 1;
} else {
validity = (left_it->validity == valid || right_it->validity == valid) ? valid : invalid;
@@ -1327,6 +1326,7 @@ int filter_result_iterator_t::is_valid(uint32_t id) {
}
seq_id = id;
+ or_filter_iterators();
return 1;
}
}
diff --git a/src/index.cpp b/src/index.cpp
index 0287f443..d2eb01b6 100644
--- a/src/index.cpp
+++ b/src/index.cpp
@@ -1801,37 +1801,160 @@ Option<bool> Index::do_filtering_with_lock(filter_node_t* const filter_tree_root
return filter_init_op;
}
- filter_result.count = filter_result_iterator.to_filter_id_array(filter_result.docs);
+ if (filter_result_iterator.reference.empty()) {
+ filter_result.count = filter_result_iterator.to_filter_id_array(filter_result.docs);
+ return Option(true);
+ }
+ filter_result_iterator.compute_result();
+ if (filter_result_iterator.approx_filter_ids_length == 0) {
+ return Option(true);
+ }
+
+ uint32_t count = filter_result_iterator.approx_filter_ids_length, dummy;
+ auto ref_filter_result = new filter_result_t();
+ std::unique_ptr<filter_result_t> ref_filter_result_guard(ref_filter_result);
+ filter_result_iterator.get_n_ids(count, dummy, nullptr, 0, ref_filter_result);
+
+ if (filter_result_iterator.validity == filter_result_iterator_t::timed_out) {
+ return Option<bool>(true);
+ }
+
+ filter_result = std::move(*ref_filter_result);
return Option(true);
}
+void aggregate_nested_references(single_filter_result_t *const reference_result,
+ reference_filter_result_t& ref_filter_result) {
+ // Add reference doc id in result.
+ auto temp_docs = new uint32_t[ref_filter_result.count + 1];
+ std::copy(ref_filter_result.docs, ref_filter_result.docs + ref_filter_result.count, temp_docs);
+ temp_docs[ref_filter_result.count] = reference_result->seq_id;
+
+ delete[] ref_filter_result.docs;
+ ref_filter_result.docs = temp_docs;
+ ref_filter_result.count++;
+ ref_filter_result.is_reference_array_field = false;
+
+ // Add references of the reference doc id in result.
+ auto& references = ref_filter_result.coll_to_references;
+ auto temp_references = new std::map<std::string, reference_filter_result_t>[ref_filter_result.count] {};
+ for (uint32_t i = 0; i < ref_filter_result.count - 1; i++) {
+ temp_references[i] = std::move(references[i]);
+ }
+ temp_references[ref_filter_result.count - 1] = std::move(reference_result->reference_filter_results);
+
+ delete[] references;
+ references = temp_references;
+}
+
Option<bool> Index::do_reference_filtering_with_lock(filter_node_t* const filter_tree_root,
filter_result_t& filter_result,
- const std::string& collection_name,
+ const std::string& ref_collection_name,
const std::string& reference_helper_field_name) const {
std::shared_lock lock(mutex);
- auto filter_result_iterator = filter_result_iterator_t(collection_name, this, filter_tree_root,
- search_begin_us, search_stop_us);
- auto filter_init_op = filter_result_iterator.init_status();
+ auto ref_filter_result_iterator = filter_result_iterator_t(ref_collection_name, this, filter_tree_root,
+ search_begin_us, search_stop_us);
+ auto filter_init_op = ref_filter_result_iterator.init_status();
if (!filter_init_op.ok()) {
return filter_init_op;
}
- uint32_t* reference_docs = nullptr;
- uint32_t count = filter_result_iterator.to_filter_id_array(reference_docs);
- std::unique_ptr<uint32_t[]> docs_guard(reference_docs);
-
- if (count == 0) {
+ ref_filter_result_iterator.compute_result();
+ if (ref_filter_result_iterator.approx_filter_ids_length == 0) {
return Option(true);
}
- if (search_schema.at(reference_helper_field_name).is_singular()) {
+ uint32_t count = ref_filter_result_iterator.approx_filter_ids_length, dummy;
+ auto ref_filter_result = new filter_result_t();
+ std::unique_ptr<filter_result_t> ref_filter_result_guard(ref_filter_result);
+ ref_filter_result_iterator.get_n_ids(count, dummy, nullptr, 0, ref_filter_result);
+
+ if (ref_filter_result_iterator.validity == filter_result_iterator_t::timed_out) {
+ return Option<bool>(true);
+ }
+
+ uint32_t* reference_docs = ref_filter_result->docs;
+ ref_filter_result->docs = nullptr;
+ std::unique_ptr<uint32_t[]> docs_guard(reference_docs);
+
+ auto const is_nested_join = !ref_filter_result_iterator.reference.empty();
+ if (search_schema.at(reference_helper_field_name).is_singular()) { // Only one reference per doc.
+ if (sort_index.count(reference_helper_field_name) == 0) {
+ return Option<bool>(400, "`" + reference_helper_field_name + "` is not present in sort index.");
+ }
+ auto const& ref_index = *sort_index.at(reference_helper_field_name);
+
+ if (is_nested_join) {
+ // In case of nested join, we need to collect all the doc ids from the reference ids along with their references.
+ std::vector<std::pair<uint32_t, single_filter_result_t*>> id_pairs;
+ std::unordered_set<uint32_t> unique_doc_ids;
+
+ for (uint32_t i = 0; i < count; i++) {
+ auto& reference_doc_id = reference_docs[i];
+ auto reference_doc_references = std::move(ref_filter_result->coll_to_references[i]);
+ if (ref_index.count(reference_doc_id) == 0) { // Reference field might be optional.
+ continue;
+ }
+ auto doc_id = ref_index.at(reference_doc_id);
+
+ id_pairs.emplace_back(std::make_pair(doc_id, new single_filter_result_t(reference_doc_id,
+ std::move(reference_doc_references),
+ false)));
+ unique_doc_ids.insert(doc_id);
+ }
+
+ if (id_pairs.empty()) {
+ return Option(true);
+ }
+
+ std::sort(id_pairs.begin(), id_pairs.end(), [](auto const& left, auto const& right) {
+ return left.first < right.first;
+ });
+
+ filter_result.count = unique_doc_ids.size();
+ filter_result.docs = new uint32_t[unique_doc_ids.size()];
+ filter_result.coll_to_references = new std::map<std::string, reference_filter_result_t>[unique_doc_ids.size()] {};
+
+ reference_filter_result_t previous_doc_references;
+ for (uint32_t i = 0, previous_doc = id_pairs[0].first + 1, result_index = 0; i < id_pairs.size(); i++) {
+ auto const& current_doc = id_pairs[i].first;
+ auto& reference_result = id_pairs[i].second;
+
+ if (current_doc != previous_doc) {
+ filter_result.docs[result_index] = current_doc;
+ if (result_index > 0) {
+ std::map<std::string, reference_filter_result_t> references;
+ references[ref_collection_name] = std::move(previous_doc_references);
+ filter_result.coll_to_references[result_index - 1] = std::move(references);
+ }
+
+ result_index++;
+ previous_doc = current_doc;
+ aggregate_nested_references(reference_result, previous_doc_references);
+ } else {
+ aggregate_nested_references(reference_result, previous_doc_references);
+ }
+ }
+
+ if (previous_doc_references.count != 0) {
+ std::map<std::string, reference_filter_result_t> references;
+ references[ref_collection_name] = std::move(previous_doc_references);
+ filter_result.coll_to_references[filter_result.count - 1] = std::move(references);
+ }
+
+ for (auto &item: id_pairs) {
+ delete item.second;
+ }
+
+ return Option(true);
+ }
+
// Collect all the doc ids from the reference ids.
std::vector<std::pair<uint32_t, uint32_t>> id_pairs;
std::unordered_set<uint32_t> unique_doc_ids;
- auto const& ref_index = *sort_index.at(reference_helper_field_name);
+
for (uint32_t i = 0; i < count; i++) {
auto& reference_doc_id = reference_docs[i];
if (ref_index.count(reference_doc_id) == 0) { // Reference field might be optional.
@@ -1839,10 +1962,14 @@ Option<bool> Index::do_reference_filtering_with_lock(filter_node_t* const filter
}
auto doc_id = ref_index.at(reference_doc_id);
- id_pairs.emplace_back(std::pair(doc_id, reference_doc_id));
+ id_pairs.emplace_back(std::make_pair(doc_id, reference_doc_id));
unique_doc_ids.insert(doc_id);
}
+ if (id_pairs.empty()) {
+ return Option(true);
+ }
+
std::sort(id_pairs.begin(), id_pairs.end(), [](auto const& left, auto const& right) {
return left.first < right.first;
});
@@ -1861,9 +1988,11 @@ Option<bool> Index::do_reference_filtering_with_lock(filter_node_t* const filter
if (result_index > 0) {
auto& reference_result = filter_result.coll_to_references[result_index - 1];
- auto r = reference_filter_result_t(previous_doc_references.size(), new uint32_t[previous_doc_references.size()]);
+ auto r = reference_filter_result_t(previous_doc_references.size(),
+ new uint32_t[previous_doc_references.size()],
+ false);
std::copy(previous_doc_references.begin(), previous_doc_references.end(), r.docs);
- reference_result[collection_name] = std::move(r);
+ reference_result[ref_collection_name] = std::move(r);
previous_doc_references.clear();
}
@@ -1879,41 +2008,317 @@ Option<bool> Index::do_reference_filtering_with_lock(filter_node_t* const filter
if (!previous_doc_references.empty()) {
auto& reference_result = filter_result.coll_to_references[filter_result.count - 1];
- auto r = reference_filter_result_t(previous_doc_references.size(), new uint32_t[previous_doc_references.size()]);
+ auto r = reference_filter_result_t(previous_doc_references.size(),
+ new uint32_t[previous_doc_references.size()],
+ false);
std::copy(previous_doc_references.begin(), previous_doc_references.end(), r.docs);
- reference_result[collection_name] = std::move(r);
+ reference_result[ref_collection_name] = std::move(r);
}
return Option(true);
}
- size_t ids_len = 0;
- uint32_t *ids = nullptr;
+ // Multiple references per doc.
+ if (reference_index.count(reference_helper_field_name) == 0) {
+ return Option<bool>(400, "`" + reference_helper_field_name + "` is not present in reference index.");
+ }
auto& ref_index = *reference_index.at(reference_helper_field_name);
+
+ if (is_nested_join) {
+ // In case of nested join, we need to collect all the doc ids from the reference ids along with their references.
+ std::vector<std::pair<uint32_t, single_filter_result_t*>> id_pairs;
+ std::unordered_set<uint32_t> unique_doc_ids;
+
+ for (uint32_t i = 0; i < count; i++) {
+ auto& reference_doc_id = reference_docs[i];
+ auto reference_doc_references = std::move(ref_filter_result->coll_to_references[i]);
+ size_t doc_ids_len = 0;
+ uint32_t* doc_ids = nullptr;
+
+ ref_index.search(EQUALS, reference_doc_id, &doc_ids, doc_ids_len);
+
+ for (size_t j = 0; j < doc_ids_len; j++) {
+ auto doc_id = doc_ids[j];
+ auto reference_doc_references_copy = reference_doc_references;
+ id_pairs.emplace_back(std::make_pair(doc_id, new single_filter_result_t(reference_doc_id,
+ std::move(reference_doc_references_copy),
+ false)));
+ unique_doc_ids.insert(doc_id);
+ }
+ delete[] doc_ids;
+ }
+
+ if (id_pairs.empty()) {
+ return Option(true);
+ }
+
+ std::sort(id_pairs.begin(), id_pairs.end(), [](auto const& left, auto const& right) {
+ return left.first < right.first;
+ });
+
+ filter_result.count = unique_doc_ids.size();
+ filter_result.docs = new uint32_t[unique_doc_ids.size()];
+ filter_result.coll_to_references = new std::map<std::string, reference_filter_result_t>[unique_doc_ids.size()] {};
+
+ reference_filter_result_t previous_doc_references;
+ for (uint32_t i = 0, previous_doc = id_pairs[0].first + 1, result_index = 0; i < id_pairs.size(); i++) {
+ auto const& current_doc = id_pairs[i].first;
+ auto& reference_result = id_pairs[i].second;
+
+ if (current_doc != previous_doc) {
+ filter_result.docs[result_index] = current_doc;
+ if (result_index > 0) {
+ std::map<std::string, reference_filter_result_t> references;
+ references[ref_collection_name] = std::move(previous_doc_references);
+ filter_result.coll_to_references[result_index - 1] = std::move(references);
+ }
+
+ result_index++;
+ previous_doc = current_doc;
+ aggregate_nested_references(reference_result, previous_doc_references);
+ } else {
+ aggregate_nested_references(reference_result, previous_doc_references);
+ }
+ }
+
+ if (previous_doc_references.count != 0) {
+ std::map<std::string, reference_filter_result_t> references;
+ references[ref_collection_name] = std::move(previous_doc_references);
+ filter_result.coll_to_references[filter_result.count - 1] = std::move(references);
+ }
+
+ for (auto &item: id_pairs) {
+ delete item.second;
+ }
+
+ return Option<bool>(true);
+ }
+
+ std::vector<std::pair<uint32_t, uint32_t>> id_pairs;
+ std::unordered_set<uint32_t> unique_doc_ids;
+
for (uint32_t i = 0; i < count; i++) {
auto& reference_doc_id = reference_docs[i];
- ref_index.search(EQUALS, reference_doc_id, &ids, ids_len);
+ size_t doc_ids_len = 0;
+ uint32_t* doc_ids = nullptr;
+
+ ref_index.search(EQUALS, reference_doc_id, &doc_ids, doc_ids_len);
+
+ for (size_t j = 0; j < doc_ids_len; j++) {
+ auto doc_id = doc_ids[j];
+ id_pairs.emplace_back(std::make_pair(doc_id, reference_doc_id));
+ unique_doc_ids.insert(doc_id);
+ }
+ delete[] doc_ids;
+ }
+
+ if (id_pairs.empty()) {
+ return Option(true);
}
- filter_result.count = ids_len;
- filter_result.docs = new uint32_t[ids_len];
- filter_result.coll_to_references = new std::map<std::string, reference_filter_result_t>[ids_len] {};
+ std::sort(id_pairs.begin(), id_pairs.end(), [](auto const& left, auto const& right) {
+ return left.first < right.first;
+ });
+
+ filter_result.count = unique_doc_ids.size();
+ filter_result.docs = new uint32_t[unique_doc_ids.size()];
+ filter_result.coll_to_references = new std::map<std::string, reference_filter_result_t>[unique_doc_ids.size()] {};
- auto& num_index = *numerical_index.at(reference_helper_field_name);
- for (size_t i = 0; i < ids_len; i++) {
- filter_result.docs[i] = ids[i];
+ std::vector<uint32_t> previous_doc_references;
+ for (uint32_t i = 0, previous_doc = id_pairs[0].first + 1, result_index = 0; i < id_pairs.size(); i++) {
+ auto const& current_doc = id_pairs[i].first;
+ auto const& reference_doc_id = id_pairs[i].second;
- reference_filter_result_t reference_result;
- size_t len = 0;
- num_index.search(EQUALS, ids[i], &reference_result.docs, len);
- reference_result.count = len;
- filter_result.coll_to_references[i][collection_name] = std::move(reference_result);
+ if (current_doc != previous_doc) {
+ filter_result.docs[result_index] = current_doc;
+ if (result_index > 0) {
+ auto& reference_result = filter_result.coll_to_references[result_index - 1];
+
+ auto r = reference_filter_result_t(previous_doc_references.size(), new uint32_t[previous_doc_references.size()]);
+ std::copy(previous_doc_references.begin(), previous_doc_references.end(), r.docs);
+ reference_result[ref_collection_name] = std::move(r);
+
+ previous_doc_references.clear();
+ }
+
+ result_index++;
+ previous_doc = current_doc;
+ previous_doc_references.push_back(reference_doc_id);
+ } else {
+ previous_doc_references.push_back(reference_doc_id);
+ }
+ }
+
+ if (!previous_doc_references.empty()) {
+ auto& reference_result = filter_result.coll_to_references[filter_result.count - 1];
+
+ auto r = reference_filter_result_t(previous_doc_references.size(), new uint32_t[previous_doc_references.size()]);
+ std::copy(previous_doc_references.begin(), previous_doc_references.end(), r.docs);
+ reference_result[ref_collection_name] = std::move(r);
}
- delete [] ids;
return Option(true);
}
+Option<filter_result_t> Index::do_filtering_with_reference_ids(const std::string& reference_helper_field_name,
+ const std::string& ref_collection_name,
+ filter_result_t&& ref_filter_result) const {
+ filter_result_t filter_result;
+ auto const& count = ref_filter_result.count;
+ auto const& reference_docs = ref_filter_result.docs;
+ auto const is_nested_join = ref_filter_result.coll_to_references != nullptr;
+
+ if (count == 0) {
+ return Option<filter_result_t>(filter_result);
+ }
+
+ if (numerical_index.count(reference_helper_field_name) == 0) {
+ return Option<filter_result_t>(400, "`" + reference_helper_field_name + "` is not present in index.");
+ }
+ auto num_tree = numerical_index.at(reference_helper_field_name);
+
+ if (is_nested_join) {
+ // In case of nested join, we need to collect all the doc ids from the reference ids along with their references.
+ std::vector<std::pair<uint32_t, single_filter_result_t*>> id_pairs;
+ std::unordered_set<uint32_t> unique_doc_ids;
+
+ for (uint32_t i = 0; i < count; i++) {
+ auto& reference_doc_id = reference_docs[i];
+ auto reference_doc_references = std::move(ref_filter_result.coll_to_references[i]);
+ size_t doc_ids_len = 0;
+ uint32_t* doc_ids = nullptr;
+
+ num_tree->search(NUM_COMPARATOR::EQUALS, reference_doc_id, &doc_ids, doc_ids_len);
+
+ for (size_t j = 0; j < doc_ids_len; j++) {
+ auto doc_id = doc_ids[j];
+ auto reference_doc_references_copy = reference_doc_references;
+ id_pairs.emplace_back(std::make_pair(doc_id, new single_filter_result_t(reference_doc_id,
+ std::move(reference_doc_references_copy),
+ false)));
+ unique_doc_ids.insert(doc_id);
+ }
+
+ delete[] doc_ids;
+ }
+
+ if (id_pairs.empty()) {
+ return Option(filter_result);
+ }
+
+ std::sort(id_pairs.begin(), id_pairs.end(), [](auto const& left, auto const& right) {
+ return left.first < right.first;
+ });
+
+ filter_result.count = unique_doc_ids.size();
+ filter_result.docs = new uint32_t[unique_doc_ids.size()];
+ filter_result.coll_to_references = new std::map<std::string, reference_filter_result_t>[unique_doc_ids.size()] {};
+
+ reference_filter_result_t previous_doc_references;
+ for (uint32_t i = 0, previous_doc = id_pairs[0].first + 1, result_index = 0; i < id_pairs.size(); i++) {
+ auto const& current_doc = id_pairs[i].first;
+ auto& reference_result = id_pairs[i].second;
+
+ if (current_doc != previous_doc) {
+ filter_result.docs[result_index] = current_doc;
+ if (result_index > 0) {
+ std::map<std::string, reference_filter_result_t> references;
+ references[ref_collection_name] = std::move(previous_doc_references);
+ filter_result.coll_to_references[result_index - 1] = std::move(references);
+ }
+
+ result_index++;
+ previous_doc = current_doc;
+ aggregate_nested_references(reference_result, previous_doc_references);
+ } else {
+ aggregate_nested_references(reference_result, previous_doc_references);
+ }
+ }
+
+ if (previous_doc_references.count != 0) {
+ std::map<std::string, reference_filter_result_t> references;
+ references[ref_collection_name] = std::move(previous_doc_references);
+ filter_result.coll_to_references[filter_result.count - 1] = std::move(references);
+ }
+
+ for (auto &item: id_pairs) {
+ delete item.second;
+ }
+
+ return Option<filter_result_t>(filter_result);
+ }
+
+ // Collect all the doc ids from the reference ids.
+ std::vector<std::pair<uint32_t, uint32_t>> id_pairs;
+ std::unordered_set<uint32_t> unique_doc_ids;
+
+ for (uint32_t i = 0; i < count; i++) {
+ auto& reference_doc_id = reference_docs[i];
+ size_t doc_ids_len = 0;
+ uint32_t* doc_ids = nullptr;
+
+ num_tree->search(NUM_COMPARATOR::EQUALS, reference_doc_id, &doc_ids, doc_ids_len);
+
+ for (size_t j = 0; j < doc_ids_len; j++) {
+ auto doc_id = doc_ids[j];
+ id_pairs.emplace_back(std::make_pair(doc_id, reference_doc_id));
+ unique_doc_ids.insert(doc_id);
+ }
+ delete[] doc_ids;
+ }
+
+ if (id_pairs.empty()) {
+ return Option(filter_result);
+ }
+
+ std::sort(id_pairs.begin(), id_pairs.end(), [](auto const& left, auto const& right) {
+ return left.first < right.first;
+ });
+
+ filter_result.count = unique_doc_ids.size();
+ filter_result.docs = new uint32_t[unique_doc_ids.size()];
+ filter_result.coll_to_references = new std::map<std::string, reference_filter_result_t>[unique_doc_ids.size()] {};
+
+ std::vector<uint32_t> previous_doc_references;
+ for (uint32_t i = 0, previous_doc = id_pairs[0].first + 1, result_index = 0; i < id_pairs.size(); i++) {
+ auto const& current_doc = id_pairs[i].first;
+ auto const& reference_doc_id = id_pairs[i].second;
+
+ if (current_doc != previous_doc) {
+ filter_result.docs[result_index] = current_doc;
+ if (result_index > 0) {
+ auto& reference_result = filter_result.coll_to_references[result_index - 1];
+
+ auto r = reference_filter_result_t(previous_doc_references.size(),
+ new uint32_t[previous_doc_references.size()],
+ false);
+ std::copy(previous_doc_references.begin(), previous_doc_references.end(), r.docs);
+ reference_result[ref_collection_name] = std::move(r);
+
+ previous_doc_references.clear();
+ }
+
+ result_index++;
+ previous_doc = current_doc;
+ previous_doc_references.push_back(reference_doc_id);
+ } else {
+ previous_doc_references.push_back(reference_doc_id);
+ }
+ }
+
+ if (!previous_doc_references.empty()) {
+ auto& reference_result = filter_result.coll_to_references[filter_result.count - 1];
+
+ auto r = reference_filter_result_t(previous_doc_references.size(),
+ new uint32_t[previous_doc_references.size()],
+ false);
+ std::copy(previous_doc_references.begin(), previous_doc_references.end(), r.docs);
+ reference_result[ref_collection_name] = std::move(r);
+ }
+
+ return Option<filter_result_t>(filter_result);
+}
+
Option<bool> Index::run_search(search_args* search_params, const std::string& collection_name,
facet_index_type_t facet_index_type) {
return search(search_params->field_query_tokens,
diff --git a/src/string_utils.cpp b/src/string_utils.cpp
index cd74e96a..781929ab 100644
--- a/src/string_utils.cpp
+++ b/src/string_utils.cpp
@@ -492,24 +492,24 @@ Option<bool> StringUtils::tokenize_filter_query(const std::string& filter_query,
return Option<bool>(true);
}
-Option<bool> StringUtils::split_reference_include_fields(const std::string& include_fields,
- size_t& index,
- std::string& token) {
- auto ref_include_error = Option<bool>(400, "Invalid reference in include_fields, expected `$CollectionName(fieldA, ...)`.");
- auto const& size = include_fields.size();
+Option<bool> StringUtils::split_reference_include_exclude_fields(const std::string& include_exclude_fields,
+ size_t& index, std::string& token) {
+ auto ref_include_error = Option<bool>(400, "Invalid reference `" + include_exclude_fields + "` in include_fields/"
+ "exclude_fields, expected `$CollectionName(fieldA, ...)`.");
+ auto const& size = include_exclude_fields.size();
size_t start_index = index;
- while(++index < size && include_fields[index] != '(') {}
+ while(++index < size && include_exclude_fields[index] != '(') {}
if (index >= size) {
return ref_include_error;
}
- // In case of nested join, the reference include field could have parenthesis inside it.
+ // In case of nested join, the reference include/exclude field could have parenthesis inside it.
int parenthesis_count = 1;
while (++index < size && parenthesis_count > 0) {
- if (include_fields[index] == '(') {
+ if (include_exclude_fields[index] == '(') {
parenthesis_count++;
- } else if (include_fields[index] == ')') {
+ } else if (include_exclude_fields[index] == ')') {
parenthesis_count--;
}
}
@@ -525,34 +525,35 @@ Option<bool> StringUtils::split_reference_include_fields(const std::string& incl
// ...^
// $ref_include( $nested_ref_include(foo :merge)as nest :merge ) as ref
// ...^
- auto closing_parenthesis_pos = include_fields.find(')', index);
- auto comma_pos = include_fields.find(',', index);
- auto colon_pos = include_fields.find(':', index);
- auto alias_start_pos = include_fields.find(" as ", index);
+ auto closing_parenthesis_pos = include_exclude_fields.find(')', index);
+ auto comma_pos = include_exclude_fields.find(',', index);
+ auto colon_pos = include_exclude_fields.find(':', index);
+ auto alias_start_pos = include_exclude_fields.find(" as ", index);
auto alias_end_pos = std::min(std::min(closing_parenthesis_pos, comma_pos), colon_pos);
std::string alias;
if (alias_start_pos != std::string::npos && alias_start_pos < alias_end_pos) {
- alias = include_fields.substr(alias_start_pos, alias_end_pos - alias_start_pos);
+ alias = include_exclude_fields.substr(alias_start_pos, alias_end_pos - alias_start_pos);
}
- token = include_fields.substr(start_index, index - start_index) + " " + trim(alias);
+ token = include_exclude_fields.substr(start_index, index - start_index) + " " + trim(alias);
trim(token);
index = alias_end_pos;
return Option<bool>(true);
}
-Option<bool> StringUtils::split_include_fields(const std::string& include_fields, std::vector<std::string>& tokens) {
+Option<bool> StringUtils::split_include_exclude_fields(const std::string& include_exclude_fields,
+ std::vector<std::string>& tokens) {
std::string token;
- auto const& size = include_fields.size();
+ auto const& size = include_exclude_fields.size();
for (size_t i = 0; i < size;) {
- auto c = include_fields[i];
+ auto c = include_exclude_fields[i];
if (c == ' ') {
i++;
continue;
- } else if (c == '$') { // Reference include
+ } else if (c == '$') { // Reference include/exclude
std::string ref_include_token;
- auto split_op = split_reference_include_fields(include_fields, i, ref_include_token);
+ auto split_op = split_reference_include_exclude_fields(include_exclude_fields, i, ref_include_token);
if (!split_op.ok()) {
return split_op;
}
@@ -561,8 +562,8 @@ Option<bool> StringUtils::split_include_fields(const std::string& include_fields
continue;
}
- auto comma_pos = include_fields.find(',', i);
- token = include_fields.substr(i, (comma_pos == std::string::npos ? size : comma_pos) - i);
+ auto comma_pos = include_exclude_fields.find(',', i);
+ token = include_exclude_fields.substr(i, (comma_pos == std::string::npos ? size : comma_pos) - i);
trim(token);
if (!token.empty()) {
tokens.push_back(token);
diff --git a/test/collection_join_test.cpp b/test/collection_join_test.cpp
index f254a00a..2fb3c103 100644
--- a/test/collection_join_test.cpp
+++ b/test/collection_join_test.cpp
@@ -1827,6 +1827,559 @@ TEST_F(CollectionJoinTest, FilterByNReferences) {
collectionManager.drop_collection("Links");
}
+TEST_F(CollectionJoinTest, FilterByNestedReferences) {
+ auto schema_json =
+ R"({
+ "name": "Coll_A",
+ "fields": [
+ {"name": "title", "type": "string"}
+ ]
+ })"_json;
+ std::vector<nlohmann::json> documents = {
+ R"({
+ "title": "coll_a_0"
+ })"_json,
+ R"({
+ "title": "coll_a_1"
+ })"_json
+ };
+ auto collection_create_op = collectionManager.create_collection(schema_json);
+ ASSERT_TRUE(collection_create_op.ok());
+ for (auto const &json: documents) {
+ auto add_op = collection_create_op.get()->add(json.dump());
+ if (!add_op.ok()) {
+ LOG(INFO) << add_op.error();
+ }
+ ASSERT_TRUE(add_op.ok());
+ }
+ schema_json =
+ R"({
+ "name": "Coll_B",
+ "fields": [
+ {"name": "title", "type": "string"},
+ {"name": "ref_coll_a", "type": "string", "reference": "Coll_A.id"}
+ ]
+ })"_json;
+ documents = {
+ R"({
+ "title": "coll_b_0",
+ "ref_coll_a": "1"
+ })"_json,
+ R"({
+ "title": "coll_b_1",
+ "ref_coll_a": "0"
+ })"_json,
+ R"({
+ "title": "coll_b_2",
+ "ref_coll_a": "0"
+ })"_json
+ };
+ collection_create_op = collectionManager.create_collection(schema_json);
+ ASSERT_TRUE(collection_create_op.ok());
+ for (auto const &json: documents) {
+ auto add_op = collection_create_op.get()->add(json.dump());
+ if (!add_op.ok()) {
+ LOG(INFO) << add_op.error();
+ }
+ ASSERT_TRUE(add_op.ok());
+ }
+ schema_json =
+ R"({
+ "name": "Coll_C",
+ "fields": [
+ {"name": "title", "type": "string"},
+ {"name": "ref_coll_b", "type": "string[]", "reference": "Coll_B.id"}
+ ]
+ })"_json;
+ documents = {
+ R"({
+ "title": "coll_c_0",
+ "ref_coll_b": ["0"]
+ })"_json,
+ R"({
+ "title": "coll_c_1",
+ "ref_coll_b": ["1"]
+ })"_json,
+ R"({
+ "title": "coll_c_2",
+ "ref_coll_b": ["0", "1"]
+ })"_json,
+ R"({
+ "title": "coll_c_3",
+ "ref_coll_b": ["2"]
+ })"_json
+ };
+ collection_create_op = collectionManager.create_collection(schema_json);
+ ASSERT_TRUE(collection_create_op.ok());
+ for (auto const &json: documents) {
+ auto add_op = collection_create_op.get()->add(json.dump());
+ if (!add_op.ok()) {
+ LOG(INFO) << add_op.error();
+ }
+ ASSERT_TRUE(add_op.ok());
+ }
+
+ std::map<std::string, std::string> req_params = {
+ {"collection", "Coll_A"},
+ {"q", "*"},
+ {"filter_by", "$Coll_B($Coll_C(id: [1, 3]))"},
+ {"include_fields", "title, $Coll_B(title, $Coll_C(title))"}
+ };
+ nlohmann::json embedded_params;
+ std::string json_res;
+ auto now_ts = std::chrono::duration_cast<std::chrono::microseconds>(
+ std::chrono::system_clock::now().time_since_epoch()).count();
+
+ auto search_op = collectionManager.do_search(req_params, embedded_params, json_res, now_ts);
+ ASSERT_TRUE(search_op.ok());
+
+ nlohmann::json res_obj = nlohmann::json::parse(json_res);
+ // coll_b_1 <- coll_c_1
+ // coll_a_0 <
+ // coll_b_2 <- coll_c_3
+ ASSERT_EQ(1, res_obj["found"].get<size_t>());
+ ASSERT_EQ(1, res_obj["hits"].size());
+ ASSERT_EQ(2, res_obj["hits"][0]["document"].size());
+ ASSERT_EQ("coll_a_0", res_obj["hits"][0]["document"]["title"]);
+ ASSERT_EQ(2, res_obj["hits"][0]["document"]["Coll_B"].size());
+ ASSERT_EQ("coll_b_1", res_obj["hits"][0]["document"]["Coll_B"][0]["title"]);
+ ASSERT_EQ(1, res_obj["hits"][0]["document"]["Coll_B"][0]["Coll_C"].size());
+ ASSERT_EQ("coll_c_1", res_obj["hits"][0]["document"]["Coll_B"][0]["Coll_C"][0]["title"]);
+ ASSERT_EQ("coll_b_2", res_obj["hits"][0]["document"]["Coll_B"][1]["title"]);
+ ASSERT_EQ(1, res_obj["hits"][0]["document"]["Coll_B"][1]["Coll_C"].size());
+ ASSERT_EQ("coll_c_3", res_obj["hits"][0]["document"]["Coll_B"][1]["Coll_C"][0]["title"]);
+
+ req_params = {
+ {"collection", "Coll_A"},
+ {"q", "*"},
+ {"filter_by", "$Coll_B($Coll_C(id: != 0))"},
+ {"include_fields", "title, $Coll_B(title, $Coll_C(title):nest_array)"}
+ };
+ search_op = collectionManager.do_search(req_params, embedded_params, json_res, now_ts);
+ ASSERT_TRUE(search_op.ok());
+
+ res_obj = nlohmann::json::parse(json_res);
+ // coll_a_1 <- coll_b_0 <- coll_c_2
+ //
+ // coll_b_1 <- coll_c_1, coll_c_2
+ // coll_a_0 <
+ // coll_b_2 <- coll_c_3
+ ASSERT_EQ(2, res_obj["found"].get<size_t>());
+ ASSERT_EQ(2, res_obj["hits"].size());
+ ASSERT_EQ(2, res_obj["hits"][0]["document"].size());
+ ASSERT_EQ("coll_a_1", res_obj["hits"][0]["document"]["title"]);
+ ASSERT_EQ(1, res_obj["hits"][0]["document"]["Coll_B"].size());
+ ASSERT_EQ("coll_b_0", res_obj["hits"][0]["document"]["Coll_B"][0]["title"]);
+ ASSERT_EQ(1, res_obj["hits"][0]["document"]["Coll_B"][0]["Coll_C"].size());
+ ASSERT_EQ("coll_c_2", res_obj["hits"][0]["document"]["Coll_B"][0]["Coll_C"][0]["title"]);
+
+ ASSERT_EQ("coll_a_0", res_obj["hits"][1]["document"]["title"]);
+ ASSERT_EQ(2, res_obj["hits"][1]["document"]["Coll_B"].size());
+ ASSERT_EQ("coll_b_1", res_obj["hits"][1]["document"]["Coll_B"][0]["title"]);
+ ASSERT_EQ(2, res_obj["hits"][1]["document"]["Coll_B"][0]["Coll_C"].size());
+ ASSERT_EQ("coll_c_1", res_obj["hits"][1]["document"]["Coll_B"][0]["Coll_C"][0]["title"]);
+ ASSERT_EQ("coll_c_2", res_obj["hits"][1]["document"]["Coll_B"][0]["Coll_C"][1]["title"]);
+ ASSERT_EQ("coll_b_2", res_obj["hits"][1]["document"]["Coll_B"][1]["title"]);
+ ASSERT_EQ(1, res_obj["hits"][1]["document"]["Coll_B"][1]["Coll_C"].size());
+ ASSERT_EQ("coll_c_3", res_obj["hits"][1]["document"]["Coll_B"][1]["Coll_C"][0]["title"]);
+
+ req_params = {
+ {"collection", "Coll_C"},
+ {"q", "*"},
+ {"filter_by", "$Coll_B($Coll_A(id: 0))"},
+ {"include_fields", "title, $Coll_B(title, $Coll_A(title))"}
+ };
+ search_op = collectionManager.do_search(req_params, embedded_params, json_res, now_ts);
+ ASSERT_TRUE(search_op.ok());
+
+ res_obj = nlohmann::json::parse(json_res);
+ // coll_c_3 -> coll_b_2 -> coll_a_0
+ //
+ // coll_c_2 -> coll_b_1 -> coll_a_0
+ //
+ // coll_c_1 -> coll_b_1 -> coll_a_0
+ ASSERT_EQ(3, res_obj["found"].get<size_t>());
+ ASSERT_EQ(3, res_obj["hits"].size());
+ ASSERT_EQ(2, res_obj["hits"][0]["document"].size());
+ ASSERT_EQ("coll_c_3", res_obj["hits"][0]["document"]["title"]);
+ ASSERT_EQ(2, res_obj["hits"][0]["document"]["Coll_B"].size());
+ ASSERT_EQ("coll_b_2", res_obj["hits"][0]["document"]["Coll_B"]["title"]);
+ ASSERT_EQ(1, res_obj["hits"][0]["document"]["Coll_B"]["Coll_A"].size());
+ ASSERT_EQ("coll_a_0", res_obj["hits"][0]["document"]["Coll_B"]["Coll_A"]["title"]);
+
+ ASSERT_EQ(2, res_obj["hits"][1]["document"].size());
+ ASSERT_EQ("coll_c_2", res_obj["hits"][1]["document"]["title"]);
+ ASSERT_EQ(2, res_obj["hits"][1]["document"]["Coll_B"].size());
+ ASSERT_EQ("coll_b_1", res_obj["hits"][1]["document"]["Coll_B"]["title"]);
+ ASSERT_EQ(1, res_obj["hits"][1]["document"]["Coll_B"]["Coll_A"].size());
+ ASSERT_EQ("coll_a_0", res_obj["hits"][1]["document"]["Coll_B"]["Coll_A"]["title"]);
+
+ ASSERT_EQ(2, res_obj["hits"][2]["document"].size());
+ ASSERT_EQ("coll_c_1", res_obj["hits"][2]["document"]["title"]);
+ ASSERT_EQ(2, res_obj["hits"][2]["document"]["Coll_B"].size());
+ ASSERT_EQ("coll_b_1", res_obj["hits"][2]["document"]["Coll_B"]["title"]);
+ ASSERT_EQ(1, res_obj["hits"][2]["document"]["Coll_B"]["Coll_A"].size());
+ ASSERT_EQ("coll_a_0", res_obj["hits"][2]["document"]["Coll_B"]["Coll_A"]["title"]);
+
+ schema_json =
+ R"({
+ "name": "Coll_D",
+ "fields": [
+ {"name": "title", "type": "string"},
+ {"name": "ref_coll_c", "type": "string[]", "reference": "Coll_C.id"}
+ ]
+ })"_json;
+ documents = {
+ R"({
+ "title": "coll_d_0",
+ "ref_coll_c": []
+ })"_json,
+ R"({
+ "title": "coll_d_1",
+ "ref_coll_c": ["1", "3"]
+ })"_json,
+ R"({
+ "title": "coll_d_2",
+ "ref_coll_c": ["2", "3"]
+ })"_json,
+ };
+ collection_create_op = collectionManager.create_collection(schema_json);
+ ASSERT_TRUE(collection_create_op.ok());
+ for (auto const &json: documents) {
+ auto add_op = collection_create_op.get()->add(json.dump());
+ if (!add_op.ok()) {
+ LOG(INFO) << add_op.error();
+ }
+ ASSERT_TRUE(add_op.ok());
+ }
+
+ req_params = {
+ {"collection", "Coll_B"},
+ {"q", "*"},
+ {"filter_by", "$Coll_C($Coll_D(id: *))"},
+ {"include_fields", "title, $Coll_C(title, $Coll_D(title:nest_array):nest_array)"}
+ };
+ search_op = collectionManager.do_search(req_params, embedded_params, json_res, now_ts);
+ ASSERT_TRUE(search_op.ok());
+
+ res_obj = nlohmann::json::parse(json_res);
+ // coll_b_2 <- coll_c_3 <- coll_d_1, coll_d_2
+ //
+ // coll_c_1 <- coll_d_1
+ // coll_b_1 <
+ // coll_c_2 <- coll_d_2
+ //
+ // coll_b_0 <- coll_c_2 <- coll_d_2
+ ASSERT_EQ(3, res_obj["found"].get<size_t>());
+ ASSERT_EQ(3, res_obj["hits"].size());
+ ASSERT_EQ(2, res_obj["hits"][0]["document"].size());
+ ASSERT_EQ("coll_b_2", res_obj["hits"][0]["document"]["title"]);
+ ASSERT_EQ(1, res_obj["hits"][0]["document"]["Coll_C"].size());
+ ASSERT_EQ("coll_c_3", res_obj["hits"][0]["document"]["Coll_C"][0]["title"]);
+ ASSERT_EQ(2, res_obj["hits"][0]["document"]["Coll_C"][0]["Coll_D"].size());
+ ASSERT_EQ("coll_d_1", res_obj["hits"][0]["document"]["Coll_C"][0]["Coll_D"][0]["title"]);
+ ASSERT_EQ("coll_d_2", res_obj["hits"][0]["document"]["Coll_C"][0]["Coll_D"][1]["title"]);
+
+ ASSERT_EQ(2, res_obj["hits"][1]["document"].size());
+ ASSERT_EQ("coll_b_1", res_obj["hits"][1]["document"]["title"]);
+ ASSERT_EQ(2, res_obj["hits"][1]["document"]["Coll_C"].size());
+ ASSERT_EQ("coll_c_1", res_obj["hits"][1]["document"]["Coll_C"][0]["title"]);
+ ASSERT_EQ(1, res_obj["hits"][1]["document"]["Coll_C"][0]["Coll_D"].size());
+ ASSERT_EQ("coll_d_1", res_obj["hits"][1]["document"]["Coll_C"][0]["Coll_D"][0]["title"]);
+ ASSERT_EQ("coll_c_2", res_obj["hits"][1]["document"]["Coll_C"][1]["title"]);
+ ASSERT_EQ(1, res_obj["hits"][1]["document"]["Coll_C"][1]["Coll_D"].size());
+ ASSERT_EQ("coll_d_2", res_obj["hits"][1]["document"]["Coll_C"][1]["Coll_D"][0]["title"]);
+
+ ASSERT_EQ(2, res_obj["hits"][2]["document"].size());
+ ASSERT_EQ("coll_b_0", res_obj["hits"][2]["document"]["title"]);
+ ASSERT_EQ(1, res_obj["hits"][2]["document"]["Coll_C"].size());
+ ASSERT_EQ("coll_c_2", res_obj["hits"][2]["document"]["Coll_C"][0]["title"]);
+ ASSERT_EQ(1, res_obj["hits"][2]["document"]["Coll_C"][0]["Coll_D"].size());
+ ASSERT_EQ("coll_d_2", res_obj["hits"][2]["document"]["Coll_C"][0]["Coll_D"][0]["title"]);
+
+ req_params = {
+ {"collection", "Coll_D"},
+ {"q", "*"},
+ {"filter_by", "$Coll_C($Coll_B(id: [0, 1]))"},
+ {"include_fields", "title, $Coll_C(title, $Coll_B(title:nest_array):nest_array)"}
+ };
+ search_op = collectionManager.do_search(req_params, embedded_params, json_res, now_ts);
+ ASSERT_TRUE(search_op.ok());
+
+ res_obj = nlohmann::json::parse(json_res);
+ LOG(INFO) << res_obj.dump();
+ // coll_d_2 -> coll_c_2 -> coll_b_0, coll_b_1
+ //
+ // coll_d_1 -> coll_c_1 -> coll_b_1
+ ASSERT_EQ(2, res_obj["found"].get<size_t>());
+ ASSERT_EQ(2, res_obj["hits"].size());
+ ASSERT_EQ(2, res_obj["hits"][0]["document"].size());
+ ASSERT_EQ("coll_d_2", res_obj["hits"][0]["document"]["title"]);
+ ASSERT_EQ(1, res_obj["hits"][0]["document"]["Coll_C"].size());
+ ASSERT_EQ("coll_c_2", res_obj["hits"][0]["document"]["Coll_C"][0]["title"]);
+ ASSERT_EQ(2, res_obj["hits"][0]["document"]["Coll_C"][0]["Coll_B"].size());
+ ASSERT_EQ("coll_b_0", res_obj["hits"][0]["document"]["Coll_C"][0]["Coll_B"][0]["title"]);
+ ASSERT_EQ("coll_b_1", res_obj["hits"][0]["document"]["Coll_C"][0]["Coll_B"][1]["title"]);
+
+ ASSERT_EQ(2, res_obj["hits"][1]["document"].size());
+ ASSERT_EQ("coll_d_1", res_obj["hits"][1]["document"]["title"]);
+ ASSERT_EQ(1, res_obj["hits"][1]["document"]["Coll_C"].size());
+ ASSERT_EQ("coll_c_1", res_obj["hits"][1]["document"]["Coll_C"][0]["title"]);
+ ASSERT_EQ(1, res_obj["hits"][1]["document"]["Coll_C"][0]["Coll_B"].size());
+ ASSERT_EQ("coll_b_1", res_obj["hits"][1]["document"]["Coll_C"][0]["Coll_B"][0]["title"]);
+
+ schema_json =
+ R"({
+ "name": "products",
+ "fields": [
+ {"name": "title", "type": "string"}
+ ]
+ })"_json;
+ documents = {
+ R"({
+ "title": "shampoo"
+ })"_json,
+ R"({
+ "title": "soap"
+ })"_json
+ };
+ collection_create_op = collectionManager.create_collection(schema_json);
+ ASSERT_TRUE(collection_create_op.ok());
+ for (auto const &json: documents) {
+ auto add_op = collection_create_op.get()->add(json.dump());
+ if (!add_op.ok()) {
+ LOG(INFO) << add_op.error();
+ }
+ ASSERT_TRUE(add_op.ok());
+ }
+
+ schema_json =
+ R"({
+ "name": "product_variants",
+ "fields": [
+ {"name": "title", "type": "string"},
+ {"name": "product_id", "type": "string", "reference": "products.id"}
+ ]
+ })"_json;
+ documents = {
+ R"({
+ "title": "panteen",
+ "product_id": "0"
+ })"_json,
+ R"({
+ "title": "loreal",
+ "product_id": "0"
+ })"_json,
+ R"({
+ "title": "pears",
+ "product_id": "1"
+ })"_json,
+ R"({
+ "title": "lifebuoy",
+ "product_id": "1"
+ })"_json
+ };
+ collection_create_op = collectionManager.create_collection(schema_json);
+ ASSERT_TRUE(collection_create_op.ok());
+ for (auto const &json: documents) {
+ auto add_op = collection_create_op.get()->add(json.dump());
+ if (!add_op.ok()) {
+ LOG(INFO) << add_op.error();
+ }
+ ASSERT_TRUE(add_op.ok());
+ }
+
+ schema_json =
+ R"({
+ "name": "retailers",
+ "fields": [
+ {"name": "title", "type": "string"},
+ {"name": "location", "type": "geopoint"}
+ ]
+ })"_json;
+ documents = {
+ R"({
+ "title": "retailer 1",
+ "location": [48.872576479306765, 2.332291112241466]
+ })"_json,
+ R"({
+ "title": "retailer 2",
+ "location": [48.888286721920934, 2.342340862419206]
+ })"_json,
+ R"({
+ "title": "retailer 3",
+ "location": [48.87538726829884, 2.296113163780903]
+ })"_json
+ };
+ collection_create_op = collectionManager.create_collection(schema_json);
+ ASSERT_TRUE(collection_create_op.ok());
+ for (auto const &json: documents) {
+ auto add_op = collection_create_op.get()->add(json.dump());
+ if (!add_op.ok()) {
+ LOG(INFO) << add_op.error();
+ }
+ ASSERT_TRUE(add_op.ok());
+ }
+
+ schema_json =
+ R"({
+ "name": "inventory",
+ "fields": [
+ {"name": "qty", "type": "int32"},
+ {"name": "retailer_id", "type": "string", "reference": "retailers.id"},
+ {"name": "product_variant_id", "type": "string", "reference": "product_variants.id"}
+ ]
+ })"_json;
+ documents = {
+ R"({
+ "qty": "1",
+ "retailer_id": "0",
+ "product_variant_id": "0"
+ })"_json,
+ R"({
+ "qty": "2",
+ "retailer_id": "0",
+ "product_variant_id": "1"
+ })"_json,
+ R"({
+ "qty": "3",
+ "retailer_id": "0",
+ "product_variant_id": "2"
+ })"_json,
+ R"({
+ "qty": "4",
+ "retailer_id": "0",
+ "product_variant_id": "3"
+ })"_json,
+ R"({
+ "qty": "5",
+ "retailer_id": "1",
+ "product_variant_id": "0"
+ })"_json,
+ R"({
+ "qty": "6",
+ "retailer_id": "1",
+ "product_variant_id": "1"
+ })"_json,
+ R"({
+ "qty": "7",
+ "retailer_id": "1",
+ "product_variant_id": "2"
+ })"_json,
+ R"({
+ "qty": "8",
+ "retailer_id": "1",
+ "product_variant_id": "3"
+ })"_json,
+ R"({
+ "qty": "9",
+ "retailer_id": "2",
+ "product_variant_id": "0"
+ })"_json,
+ R"({
+ "qty": "10",
+ "retailer_id": "2",
+ "product_variant_id": "1"
+ })"_json,
+ R"({
+ "qty": "11",
+ "retailer_id": "2",
+ "product_variant_id": "2"
+ })"_json,
+ R"({
+ "qty": "12",
+ "retailer_id": "2",
+ "product_variant_id": "3"
+ })"_json,
+ };
+ collection_create_op = collectionManager.create_collection(schema_json);
+ ASSERT_TRUE(collection_create_op.ok());
+ for (auto const &json: documents) {
+ auto add_op = collection_create_op.get()->add(json.dump());
+ if (!add_op.ok()) {
+ LOG(INFO) << add_op.error();
+ }
+ ASSERT_TRUE(add_op.ok());
+ }
+
+ req_params = {
+ {"collection", "products"},
+ {"q", "*"},
+ {"filter_by", "$product_variants($inventory($retailers(location:(48.87538726829884, 2.296113163780903,1 km))))"},
+ {"include_fields", "$product_variants(id,$inventory(qty,sku,$retailers(id,title)))"}
+ };
+ search_op = collectionManager.do_search(req_params, embedded_params, json_res, now_ts);
+ ASSERT_TRUE(search_op.ok());
+
+ res_obj = nlohmann::json::parse(json_res);
+ ASSERT_EQ(2, res_obj["found"].get<size_t>());
+ ASSERT_EQ(2, res_obj["hits"].size());
+ ASSERT_EQ("1", res_obj["hits"][0]["document"]["id"]);
+ ASSERT_EQ("soap", res_obj["hits"][0]["document"]["title"]);
+ ASSERT_EQ(2, res_obj["hits"][0]["document"]["product_variants"].size());
+
+ ASSERT_EQ("2", res_obj["hits"][0]["document"]["product_variants"][0]["id"]);
+ ASSERT_EQ(2, res_obj["hits"][0]["document"]["product_variants"][0]["inventory"].size());
+ ASSERT_EQ(11, res_obj["hits"][0]["document"]["product_variants"][0]["inventory"]["qty"]);
+ ASSERT_EQ(2, res_obj["hits"][0]["document"]["product_variants"][0]["inventory"]["retailers"].size());
+ ASSERT_EQ("2", res_obj["hits"][0]["document"]["product_variants"][0]["inventory"]["retailers"]["id"]);
+ ASSERT_EQ("retailer 3", res_obj["hits"][0]["document"]["product_variants"][0]["inventory"]["retailers"]["title"]);
+
+ ASSERT_EQ("3", res_obj["hits"][0]["document"]["product_variants"][1]["id"]);
+ ASSERT_EQ(2, res_obj["hits"][0]["document"]["product_variants"][1]["inventory"].size());
+ ASSERT_EQ(12, res_obj["hits"][0]["document"]["product_variants"][1]["inventory"]["qty"]);
+ ASSERT_EQ(2, res_obj["hits"][0]["document"]["product_variants"][1]["inventory"]["retailers"].size());
+ ASSERT_EQ("2", res_obj["hits"][0]["document"]["product_variants"][1]["inventory"]["retailers"]["id"]);
+ ASSERT_EQ("retailer 3", res_obj["hits"][0]["document"]["product_variants"][1]["inventory"]["retailers"]["title"]);
+
+ ASSERT_EQ("0", res_obj["hits"][1]["document"]["id"]);
+ ASSERT_EQ("shampoo", res_obj["hits"][1]["document"]["title"]);
+ ASSERT_EQ(2, res_obj["hits"][1]["document"]["product_variants"].size());
+
+ ASSERT_EQ("0", res_obj["hits"][1]["document"]["product_variants"][0]["id"]);
+ ASSERT_EQ(2, res_obj["hits"][1]["document"]["product_variants"][0]["inventory"].size());
+ ASSERT_EQ(9, res_obj["hits"][1]["document"]["product_variants"][0]["inventory"]["qty"]);
+ ASSERT_EQ(2, res_obj["hits"][1]["document"]["product_variants"][0]["inventory"]["retailers"].size());
+ ASSERT_EQ("2", res_obj["hits"][1]["document"]["product_variants"][0]["inventory"]["retailers"]["id"]);
+ ASSERT_EQ("retailer 3", res_obj["hits"][1]["document"]["product_variants"][0]["inventory"]["retailers"]["title"]);
+
+ ASSERT_EQ("1", res_obj["hits"][1]["document"]["product_variants"][1]["id"]);
+ ASSERT_EQ(2, res_obj["hits"][1]["document"]["product_variants"][1]["inventory"].size());
+ ASSERT_EQ(10, res_obj["hits"][1]["document"]["product_variants"][1]["inventory"]["qty"]);
+ ASSERT_EQ(2, res_obj["hits"][1]["document"]["product_variants"][1]["inventory"]["retailers"].size());
+ ASSERT_EQ("2", res_obj["hits"][1]["document"]["product_variants"][1]["inventory"]["retailers"]["id"]);
+ ASSERT_EQ("retailer 3", res_obj["hits"][1]["document"]["product_variants"][1]["inventory"]["retailers"]["title"]);
+
+ req_params = {
+ {"collection", "products"},
+ {"q", "*"},
+ {"filter_by", "$product_variants($inventory($retailers(id: [0, 1]) && qty: [4..5]))"},
+ {"include_fields", "$product_variants(id,$inventory(qty,sku,$retailers(id,title)))"},
+ {"exclude_fields", "$product_variants($inventory($retailers(id)))"}
+ };
+ search_op = collectionManager.do_search(req_params, embedded_params, json_res, now_ts);
+ ASSERT_TRUE(search_op.ok());
+
+ res_obj = nlohmann::json::parse(json_res);
+ ASSERT_EQ(2, res_obj["found"].get<size_t>());
+ ASSERT_EQ(2, res_obj["hits"].size());
+ ASSERT_EQ("1", res_obj["hits"][0]["document"]["id"]);
+ ASSERT_EQ("soap", res_obj["hits"][0]["document"]["title"]);
+ ASSERT_EQ("3", res_obj["hits"][0]["document"]["product_variants"]["id"]);
+ ASSERT_EQ(2, res_obj["hits"][0]["document"]["product_variants"]["inventory"].size());
+ ASSERT_EQ(4, res_obj["hits"][0]["document"]["product_variants"]["inventory"]["qty"]);
+ ASSERT_EQ(1, res_obj["hits"][0]["document"]["product_variants"]["inventory"]["retailers"].size());
+ ASSERT_EQ("retailer 1", res_obj["hits"][0]["document"]["product_variants"]["inventory"]["retailers"]["title"]);
+
+ ASSERT_EQ("0", res_obj["hits"][1]["document"]["id"]);
+ ASSERT_EQ("shampoo", res_obj["hits"][1]["document"]["title"]);
+ ASSERT_EQ("0", res_obj["hits"][1]["document"]["product_variants"]["id"]);
+ ASSERT_EQ(2, res_obj["hits"][1]["document"]["product_variants"]["inventory"].size());
+ ASSERT_EQ(5, res_obj["hits"][1]["document"]["product_variants"]["inventory"]["qty"]);
+ ASSERT_EQ(1, res_obj["hits"][1]["document"]["product_variants"]["inventory"]["retailers"].size());
+ ASSERT_EQ("retailer 2", res_obj["hits"][1]["document"]["product_variants"]["inventory"]["retailers"]["title"]);
+}
+
TEST_F(CollectionJoinTest, IncludeExcludeFieldsByReference) {
auto schema_json =
R"({
@@ -1926,12 +2479,14 @@ TEST_F(CollectionJoinTest, IncludeExcludeFieldsByReference) {
auto search_op = collectionManager.do_search(req_params, embedded_params, json_res, now_ts);
ASSERT_FALSE(search_op.ok());
- ASSERT_EQ("Invalid reference in include_fields, expected `$CollectionName(fieldA, ...)`.", search_op.error());
+ ASSERT_EQ("Invalid reference `$foo.bar` in include_fields/exclude_fields, expected `$CollectionName(fieldA, ...)`.",
+ search_op.error());
req_params["include_fields"] = "$foo(bar";
search_op = collectionManager.do_search(req_params, embedded_params, json_res, now_ts);
ASSERT_FALSE(search_op.ok());
- ASSERT_EQ("Invalid reference in include_fields, expected `$CollectionName(fieldA, ...)`.", search_op.error());
+ ASSERT_EQ("Invalid reference `$foo(bar` in include_fields/exclude_fields, expected `$CollectionName(fieldA, ...)`.",
+ search_op.error());
req_params["include_fields"] = "$foo(bar)";
search_op = collectionManager.do_search(req_params, embedded_params, json_res, now_ts);
diff --git a/test/collection_manager_test.cpp b/test/collection_manager_test.cpp
index 4a7da68a..29373f2a 100644
--- a/test/collection_manager_test.cpp
+++ b/test/collection_manager_test.cpp
@@ -1460,156 +1460,241 @@ TEST_F(CollectionManagerTest, GetReferenceCollectionNames) {
ref_includes = nullptr;
}
-TEST_F(CollectionManagerTest, InitializeRefIncludeFields) {
+TEST_F(CollectionManagerTest, InitializeRefIncludeExcludeFields) {
std::string filter_query = "";
- std::vector<std::string> include_fields_vec;
- std::vector<ref_include_fields> ref_include_fields_vec;
- auto initialize_op = CollectionManager::_initialize_ref_include_fields_vec(filter_query, include_fields_vec,
- ref_include_fields_vec);
+ std::vector<std::string> include_fields_vec, exclude_fields_vec;
+ std::vector<ref_include_exclude_fields> ref_include_exclude_fields_vec;
+ auto initialize_op = CollectionManager::_initialize_ref_include_exclude_fields_vec(filter_query, include_fields_vec,
+ exclude_fields_vec,
+ ref_include_exclude_fields_vec);
ASSERT_TRUE(initialize_op.ok());
- ASSERT_TRUE(ref_include_fields_vec.empty());
+ ASSERT_TRUE(ref_include_exclude_fields_vec.empty());
filter_query = "$foo(bar:baz)";
- initialize_op = CollectionManager::_initialize_ref_include_fields_vec(filter_query, include_fields_vec,
- ref_include_fields_vec);
+ exclude_fields_vec = {"$foo(bar)"};
+ initialize_op = CollectionManager::_initialize_ref_include_exclude_fields_vec(filter_query, include_fields_vec,
+ exclude_fields_vec,
+ ref_include_exclude_fields_vec);
ASSERT_TRUE(initialize_op.ok());
- ASSERT_EQ(1, ref_include_fields_vec.size());
- ASSERT_EQ("foo", ref_include_fields_vec[0].collection_name);
- ASSERT_TRUE(ref_include_fields_vec[0].fields.empty());
- ASSERT_TRUE(ref_include_fields_vec[0].alias.empty());
- ASSERT_EQ(ref_include::nest, ref_include_fields_vec[0].strategy);
- ASSERT_TRUE(ref_include_fields_vec[0].nested_join_includes.empty());
- ref_include_fields_vec.clear();
+ ASSERT_EQ(1, ref_include_exclude_fields_vec.size());
+ ASSERT_EQ("foo", ref_include_exclude_fields_vec[0].collection_name);
+ ASSERT_TRUE(ref_include_exclude_fields_vec[0].include_fields.empty());
+ ASSERT_EQ("bar", ref_include_exclude_fields_vec[0].exclude_fields);
+ ASSERT_TRUE(ref_include_exclude_fields_vec[0].alias.empty());
+ ASSERT_EQ(ref_include::nest, ref_include_exclude_fields_vec[0].strategy);
+ ASSERT_TRUE(ref_include_exclude_fields_vec[0].nested_join_includes.empty());
+ ref_include_exclude_fields_vec.clear();
+ exclude_fields_vec.clear();
filter_query = "";
include_fields_vec = {"$Customers(product_price: foo) as customers"};
- initialize_op = CollectionManager::_initialize_ref_include_fields_vec(filter_query, include_fields_vec,
- ref_include_fields_vec);
+ initialize_op = CollectionManager::_initialize_ref_include_exclude_fields_vec(filter_query, include_fields_vec,
+ exclude_fields_vec,
+ ref_include_exclude_fields_vec);
ASSERT_FALSE(initialize_op.ok());
ASSERT_EQ("Error parsing `$Customers(product_price: foo) as customers`: Unknown include strategy `foo`. "
"Valid options are `merge`, `nest`, `nest_array`.", initialize_op.error());
filter_query = "$Customers(customer_id:=customer_a && (product_price:>100 && product_price:<200))";
include_fields_vec = {"$Customers(product_price: merge) as customers"};
- initialize_op = CollectionManager::_initialize_ref_include_fields_vec(filter_query, include_fields_vec,
- ref_include_fields_vec);
+ initialize_op = CollectionManager::_initialize_ref_include_exclude_fields_vec(filter_query, include_fields_vec,
+ exclude_fields_vec,
+ ref_include_exclude_fields_vec);
ASSERT_TRUE(initialize_op.ok());
- ASSERT_EQ(1, ref_include_fields_vec.size());
- ASSERT_EQ("Customers", ref_include_fields_vec[0].collection_name);
- ASSERT_EQ("product_price", ref_include_fields_vec[0].fields);
- ASSERT_EQ("customers.", ref_include_fields_vec[0].alias);
- ASSERT_EQ(ref_include::merge, ref_include_fields_vec[0].strategy);
- ASSERT_TRUE(ref_include_fields_vec[0].nested_join_includes.empty());
- ref_include_fields_vec.clear();
+ ASSERT_EQ(1, ref_include_exclude_fields_vec.size());
+ ASSERT_EQ("Customers", ref_include_exclude_fields_vec[0].collection_name);
+ ASSERT_EQ("product_price", ref_include_exclude_fields_vec[0].include_fields);
+ ASSERT_EQ("customers.", ref_include_exclude_fields_vec[0].alias);
+ ASSERT_EQ(ref_include::merge, ref_include_exclude_fields_vec[0].strategy);
+ ASSERT_TRUE(ref_include_exclude_fields_vec[0].nested_join_includes.empty());
+ ref_include_exclude_fields_vec.clear();
filter_query = "$Customers(customer_id:=customer_a && (product_price:>100 && product_price:<200))";
include_fields_vec = {"$Customers(product_price: nest_array) as customers"};
- initialize_op = CollectionManager::_initialize_ref_include_fields_vec(filter_query, include_fields_vec,
- ref_include_fields_vec);
+ initialize_op = CollectionManager::_initialize_ref_include_exclude_fields_vec(filter_query, include_fields_vec,
+ exclude_fields_vec,
+ ref_include_exclude_fields_vec);
ASSERT_TRUE(initialize_op.ok());
- ASSERT_EQ(1, ref_include_fields_vec.size());
- ASSERT_EQ("Customers", ref_include_fields_vec[0].collection_name);
- ASSERT_EQ("product_price", ref_include_fields_vec[0].fields);
- ASSERT_EQ("customers", ref_include_fields_vec[0].alias);
- ASSERT_EQ(ref_include::nest_array, ref_include_fields_vec[0].strategy);
- ASSERT_TRUE(ref_include_fields_vec[0].nested_join_includes.empty());
- ref_include_fields_vec.clear();
+ ASSERT_EQ(1, ref_include_exclude_fields_vec.size());
+ ASSERT_EQ("Customers", ref_include_exclude_fields_vec[0].collection_name);
+ ASSERT_EQ("product_price", ref_include_exclude_fields_vec[0].include_fields);
+ ASSERT_EQ("customers", ref_include_exclude_fields_vec[0].alias);
+ ASSERT_EQ(ref_include::nest_array, ref_include_exclude_fields_vec[0].strategy);
+ ASSERT_TRUE(ref_include_exclude_fields_vec[0].nested_join_includes.empty());
+ ref_include_exclude_fields_vec.clear();
filter_query = "$product_variants( $inventory($retailers(location:(33.865,-118.375,100 km))))";
include_fields_vec = {"$product_variants(id,$inventory(qty,sku,$retailers(id,title)))"};
- initialize_op = CollectionManager::_initialize_ref_include_fields_vec(filter_query, include_fields_vec,
- ref_include_fields_vec);
+ initialize_op = CollectionManager::_initialize_ref_include_exclude_fields_vec(filter_query, include_fields_vec,
+ exclude_fields_vec,
+ ref_include_exclude_fields_vec);
ASSERT_TRUE(initialize_op.ok());
- ASSERT_EQ(1, ref_include_fields_vec.size());
- ASSERT_EQ("product_variants", ref_include_fields_vec[0].collection_name);
- ASSERT_EQ("id,", ref_include_fields_vec[0].fields);
- ASSERT_TRUE(ref_include_fields_vec[0].alias.empty());
- ASSERT_EQ(ref_include::nest, ref_include_fields_vec[0].strategy);
-
- auto nested_join_includes = ref_include_fields_vec[0].nested_join_includes;
- ASSERT_EQ("inventory", nested_join_includes[0].collection_name);
- ASSERT_EQ("qty,sku,", nested_join_includes[0].fields);
- ASSERT_TRUE(nested_join_includes[0].alias.empty());
- ASSERT_EQ(ref_include::nest, nested_join_includes[0].strategy);
-
- nested_join_includes = ref_include_fields_vec[0].nested_join_includes[0].nested_join_includes;
- ASSERT_EQ("retailers", nested_join_includes[0].collection_name);
- ASSERT_EQ("id,title", nested_join_includes[0].fields);
- ASSERT_TRUE(nested_join_includes[0].alias.empty());
- ASSERT_EQ(ref_include::nest, ref_include_fields_vec[0].strategy);
- ref_include_fields_vec.clear();
+ ASSERT_EQ(1, ref_include_exclude_fields_vec.size());
+ ASSERT_EQ("product_variants", ref_include_exclude_fields_vec[0].collection_name);
+ ASSERT_EQ("id,", ref_include_exclude_fields_vec[0].include_fields);
+ ASSERT_TRUE(ref_include_exclude_fields_vec[0].alias.empty());
+ ASSERT_EQ(ref_include::nest, ref_include_exclude_fields_vec[0].strategy);
+
+ auto nested_include_excludes = ref_include_exclude_fields_vec[0].nested_join_includes;
+ ASSERT_EQ("inventory", nested_include_excludes[0].collection_name);
+ ASSERT_EQ("qty,sku,", nested_include_excludes[0].include_fields);
+ ASSERT_TRUE(nested_include_excludes[0].alias.empty());
+ ASSERT_EQ(ref_include::nest, nested_include_excludes[0].strategy);
+
+ nested_include_excludes = ref_include_exclude_fields_vec[0].nested_join_includes[0].nested_join_includes;
+ ASSERT_EQ("retailers", nested_include_excludes[0].collection_name);
+ ASSERT_EQ("id,title", nested_include_excludes[0].include_fields);
+ ASSERT_TRUE(nested_include_excludes[0].alias.empty());
+ ASSERT_EQ(ref_include::nest, ref_include_exclude_fields_vec[0].strategy);
+ ref_include_exclude_fields_vec.clear();
filter_query = "$product_variants( $inventory($retailers(location:(33.865,-118.375,100 km))))";
include_fields_vec = {"$product_variants(title, $inventory(qty:merge) as inventory: nest) as variants"};
- initialize_op = CollectionManager::_initialize_ref_include_fields_vec(filter_query, include_fields_vec,
- ref_include_fields_vec);
+ initialize_op = CollectionManager::_initialize_ref_include_exclude_fields_vec(filter_query, include_fields_vec,
+ exclude_fields_vec,
+ ref_include_exclude_fields_vec);
ASSERT_TRUE(initialize_op.ok());
- ASSERT_EQ(1, ref_include_fields_vec.size());
- ASSERT_EQ("product_variants", ref_include_fields_vec[0].collection_name);
- ASSERT_EQ("title,", ref_include_fields_vec[0].fields);
- ASSERT_EQ("variants", ref_include_fields_vec[0].alias);
- ASSERT_EQ(ref_include::nest, ref_include_fields_vec[0].strategy);
-
- nested_join_includes = ref_include_fields_vec[0].nested_join_includes;
- ASSERT_EQ("inventory", nested_join_includes[0].collection_name);
- ASSERT_EQ("qty", nested_join_includes[0].fields);
- ASSERT_EQ("inventory.", nested_join_includes[0].alias);
- ASSERT_EQ(ref_include::merge, nested_join_includes[0].strategy);
-
- nested_join_includes = ref_include_fields_vec[0].nested_join_includes[0].nested_join_includes;
- ASSERT_EQ("retailers", nested_join_includes[0].collection_name);
- ASSERT_TRUE(nested_join_includes[0].fields.empty());
- ASSERT_TRUE(nested_join_includes[0].alias.empty());
- ASSERT_EQ(ref_include::nest, ref_include_fields_vec[0].strategy);
- ref_include_fields_vec.clear();
+ ASSERT_EQ(1, ref_include_exclude_fields_vec.size());
+ ASSERT_EQ("product_variants", ref_include_exclude_fields_vec[0].collection_name);
+ ASSERT_EQ("title,", ref_include_exclude_fields_vec[0].include_fields);
+ ASSERT_EQ("variants", ref_include_exclude_fields_vec[0].alias);
+ ASSERT_EQ(ref_include::nest, ref_include_exclude_fields_vec[0].strategy);
+
+ nested_include_excludes = ref_include_exclude_fields_vec[0].nested_join_includes;
+ ASSERT_EQ("inventory", nested_include_excludes[0].collection_name);
+ ASSERT_EQ("qty", nested_include_excludes[0].include_fields);
+ ASSERT_EQ("inventory.", nested_include_excludes[0].alias);
+ ASSERT_EQ(ref_include::merge, nested_include_excludes[0].strategy);
+
+ nested_include_excludes = ref_include_exclude_fields_vec[0].nested_join_includes[0].nested_join_includes;
+ ASSERT_EQ("retailers", nested_include_excludes[0].collection_name);
+ ASSERT_TRUE(nested_include_excludes[0].include_fields.empty());
+ ASSERT_TRUE(nested_include_excludes[0].alias.empty());
+ ASSERT_EQ(ref_include::nest, ref_include_exclude_fields_vec[0].strategy);
+ ref_include_exclude_fields_vec.clear();
filter_query = "$product_variants( $inventory(id:*) && $retailers(location:(33.865,-118.375,100 km)))";
include_fields_vec = {"$product_variants(title, $inventory(qty:merge) as inventory,"
" $retailers(title): merge) as variants"};
- initialize_op = CollectionManager::_initialize_ref_include_fields_vec(filter_query, include_fields_vec,
- ref_include_fields_vec);
+ initialize_op = CollectionManager::_initialize_ref_include_exclude_fields_vec(filter_query, include_fields_vec,
+ exclude_fields_vec,
+ ref_include_exclude_fields_vec);
ASSERT_TRUE(initialize_op.ok());
- ASSERT_EQ(1, ref_include_fields_vec.size());
- ASSERT_EQ("product_variants", ref_include_fields_vec[0].collection_name);
- ASSERT_EQ("title,", ref_include_fields_vec[0].fields);
- ASSERT_EQ("variants.", ref_include_fields_vec[0].alias);
- ASSERT_EQ(ref_include::merge, ref_include_fields_vec[0].strategy);
-
- nested_join_includes = ref_include_fields_vec[0].nested_join_includes;
- ASSERT_EQ("inventory", nested_join_includes[0].collection_name);
- ASSERT_EQ("qty", nested_join_includes[0].fields);
- ASSERT_EQ("inventory.", nested_join_includes[0].alias);
- ASSERT_EQ(ref_include::merge, nested_join_includes[0].strategy);
-
- ASSERT_EQ("retailers", nested_join_includes[1].collection_name);
- ASSERT_EQ("title", nested_join_includes[1].fields);
- ASSERT_TRUE(nested_join_includes[1].alias.empty());
- ASSERT_EQ(ref_include::nest, nested_join_includes[1].strategy);
- ref_include_fields_vec.clear();
+ ASSERT_EQ(1, ref_include_exclude_fields_vec.size());
+ ASSERT_EQ("product_variants", ref_include_exclude_fields_vec[0].collection_name);
+ ASSERT_EQ("title,", ref_include_exclude_fields_vec[0].include_fields);
+ ASSERT_EQ("variants.", ref_include_exclude_fields_vec[0].alias);
+ ASSERT_EQ(ref_include::merge, ref_include_exclude_fields_vec[0].strategy);
+
+ nested_include_excludes = ref_include_exclude_fields_vec[0].nested_join_includes;
+ ASSERT_EQ("inventory", nested_include_excludes[0].collection_name);
+ ASSERT_EQ("qty", nested_include_excludes[0].include_fields);
+ ASSERT_EQ("inventory.", nested_include_excludes[0].alias);
+ ASSERT_EQ(ref_include::merge, nested_include_excludes[0].strategy);
+
+ ASSERT_EQ("retailers", nested_include_excludes[1].collection_name);
+ ASSERT_EQ("title", nested_include_excludes[1].include_fields);
+ ASSERT_TRUE(nested_include_excludes[1].alias.empty());
+ ASSERT_EQ(ref_include::nest, nested_include_excludes[1].strategy);
+ ref_include_exclude_fields_vec.clear();
filter_query = "$product_variants( $inventory(id:*) && $retailers(location:(33.865,-118.375,100 km)))";
include_fields_vec = {"$product_variants(title, $inventory(qty:merge) as inventory, description,"
" $retailers(title), foo: merge) as variants"};
- initialize_op = CollectionManager::_initialize_ref_include_fields_vec(filter_query, include_fields_vec,
- ref_include_fields_vec);
+ initialize_op = CollectionManager::_initialize_ref_include_exclude_fields_vec(filter_query, include_fields_vec,
+ exclude_fields_vec,
+ ref_include_exclude_fields_vec);
ASSERT_TRUE(initialize_op.ok());
- ASSERT_EQ(1, ref_include_fields_vec.size());
- ASSERT_EQ("product_variants", ref_include_fields_vec[0].collection_name);
- ASSERT_EQ("title, description, foo", ref_include_fields_vec[0].fields);
- ASSERT_EQ("variants.", ref_include_fields_vec[0].alias);
- ASSERT_EQ(ref_include::merge, ref_include_fields_vec[0].strategy);
-
- nested_join_includes = ref_include_fields_vec[0].nested_join_includes;
- ASSERT_EQ("inventory", nested_join_includes[0].collection_name);
- ASSERT_EQ("qty", nested_join_includes[0].fields);
- ASSERT_EQ("inventory.", nested_join_includes[0].alias);
- ASSERT_EQ(ref_include::merge, nested_join_includes[0].strategy);
-
- ASSERT_EQ("retailers", nested_join_includes[1].collection_name);
- ASSERT_EQ("title", nested_join_includes[1].fields);
- ASSERT_TRUE(nested_join_includes[1].alias.empty());
- ASSERT_EQ(ref_include::nest, nested_join_includes[1].strategy);
- ref_include_fields_vec.clear();
+ ASSERT_EQ(1, ref_include_exclude_fields_vec.size());
+ ASSERT_EQ("product_variants", ref_include_exclude_fields_vec[0].collection_name);
+ ASSERT_EQ("title, description, foo", ref_include_exclude_fields_vec[0].include_fields);
+ ASSERT_EQ("variants.", ref_include_exclude_fields_vec[0].alias);
+ ASSERT_EQ(ref_include::merge, ref_include_exclude_fields_vec[0].strategy);
+
+ nested_include_excludes = ref_include_exclude_fields_vec[0].nested_join_includes;
+ ASSERT_EQ("inventory", nested_include_excludes[0].collection_name);
+ ASSERT_EQ("qty", nested_include_excludes[0].include_fields);
+ ASSERT_EQ("inventory.", nested_include_excludes[0].alias);
+ ASSERT_EQ(ref_include::merge, nested_include_excludes[0].strategy);
+
+ ASSERT_EQ("retailers", nested_include_excludes[1].collection_name);
+ ASSERT_EQ("title", nested_include_excludes[1].include_fields);
+ ASSERT_TRUE(nested_include_excludes[1].alias.empty());
+ ASSERT_EQ(ref_include::nest, nested_include_excludes[1].strategy);
+ ref_include_exclude_fields_vec.clear();
+
+ filter_query = "$Customers(customer_id:=customer_a && (product_price:>100 && product_price:<200))";
+ include_fields_vec.clear();
+ exclude_fields_vec = {"$Customers(product_price)"};
+ initialize_op = CollectionManager::_initialize_ref_include_exclude_fields_vec(filter_query, include_fields_vec,
+ exclude_fields_vec,
+ ref_include_exclude_fields_vec);
+ ASSERT_TRUE(initialize_op.ok());
+ ASSERT_EQ(1, ref_include_exclude_fields_vec.size());
+ ASSERT_EQ("Customers", ref_include_exclude_fields_vec[0].collection_name);
+ ASSERT_TRUE(ref_include_exclude_fields_vec[0].include_fields.empty());
+ ASSERT_EQ("product_price", ref_include_exclude_fields_vec[0].exclude_fields);
+ ASSERT_TRUE(ref_include_exclude_fields_vec[0].alias.empty());
+ ASSERT_EQ(ref_include::nest, ref_include_exclude_fields_vec[0].strategy);
+ ASSERT_TRUE(ref_include_exclude_fields_vec[0].nested_join_includes.empty());
+ ref_include_exclude_fields_vec.clear();
+
+ filter_query = "$product_variants( $inventory(id:*) && $retailers(location:(33.865,-118.375,100 km)))";
+ include_fields_vec.clear();
+ exclude_fields_vec = {"$product_variants(title, $inventory(qty), description, $retailers(title), foo)"};
+ initialize_op = CollectionManager::_initialize_ref_include_exclude_fields_vec(filter_query, include_fields_vec,
+ exclude_fields_vec,
+ ref_include_exclude_fields_vec);
+ ASSERT_TRUE(initialize_op.ok());
+ ASSERT_EQ(1, ref_include_exclude_fields_vec.size());
+ ASSERT_EQ("product_variants", ref_include_exclude_fields_vec[0].collection_name);
+ ASSERT_TRUE(ref_include_exclude_fields_vec[0].include_fields.empty());
+ ASSERT_EQ("title, description, foo", ref_include_exclude_fields_vec[0].exclude_fields);
+ ASSERT_TRUE(ref_include_exclude_fields_vec[0].alias.empty());
+ ASSERT_EQ(ref_include::nest, ref_include_exclude_fields_vec[0].strategy);
+
+ nested_include_excludes = ref_include_exclude_fields_vec[0].nested_join_includes;
+ ASSERT_EQ("inventory", nested_include_excludes[0].collection_name);
+ ASSERT_TRUE(nested_include_excludes[0].include_fields.empty());
+ ASSERT_EQ("qty", nested_include_excludes[0].exclude_fields);
+ ASSERT_TRUE(nested_include_excludes[0].alias.empty());
+ ASSERT_EQ(ref_include::nest, nested_include_excludes[0].strategy);
+
+ ASSERT_EQ("retailers", nested_include_excludes[1].collection_name);
+ ASSERT_TRUE(nested_include_excludes[1].include_fields.empty());
+ ASSERT_EQ("title", nested_include_excludes[1].exclude_fields);
+ ASSERT_TRUE(nested_include_excludes[1].alias.empty());
+ ASSERT_EQ(ref_include::nest, nested_include_excludes[1].strategy);
+ ref_include_exclude_fields_vec.clear();
+
+ filter_query = "$product_variants( $inventory($retailers(location:(33.865,-118.375,100 km))))";
+ include_fields_vec = {"$product_variants(title, $inventory(qty:merge) as inventory: nest) as variants"};
+ exclude_fields_vec = {"$product_variants(title, $inventory(qty, $retailers(title)))"};
+ initialize_op = CollectionManager::_initialize_ref_include_exclude_fields_vec(filter_query, include_fields_vec,
+ exclude_fields_vec,
+ ref_include_exclude_fields_vec);
+ ASSERT_TRUE(initialize_op.ok());
+ ASSERT_EQ(1, ref_include_exclude_fields_vec.size());
+ ASSERT_EQ("product_variants", ref_include_exclude_fields_vec[0].collection_name);
+ ASSERT_EQ("title,", ref_include_exclude_fields_vec[0].include_fields);
+ ASSERT_EQ("title,", ref_include_exclude_fields_vec[0].exclude_fields);
+ ASSERT_EQ("variants", ref_include_exclude_fields_vec[0].alias);
+ ASSERT_EQ(ref_include::nest, ref_include_exclude_fields_vec[0].strategy);
+
+ nested_include_excludes = ref_include_exclude_fields_vec[0].nested_join_includes;
+ ASSERT_EQ("inventory", nested_include_excludes[0].collection_name);
+ ASSERT_EQ("qty", nested_include_excludes[0].include_fields);
+ ASSERT_EQ("qty,", nested_include_excludes[0].exclude_fields);
+ ASSERT_EQ("inventory.", nested_include_excludes[0].alias);
+ ASSERT_EQ(ref_include::merge, nested_include_excludes[0].strategy);
+
+ nested_include_excludes = ref_include_exclude_fields_vec[0].nested_join_includes[0].nested_join_includes;
+ ASSERT_EQ("retailers", nested_include_excludes[0].collection_name);
+ ASSERT_TRUE(nested_include_excludes[0].include_fields.empty());
+ ASSERT_EQ("title", nested_include_excludes[0].exclude_fields);
+ ASSERT_TRUE(nested_include_excludes[0].alias.empty());
+ ASSERT_EQ(ref_include::nest, ref_include_exclude_fields_vec[0].strategy);
+ ref_include_exclude_fields_vec.clear();
}
TEST_F(CollectionManagerTest, ReferencedInBacklog) {
diff --git a/test/string_utils_test.cpp b/test/string_utils_test.cpp
index 44e39bb9..1093343b 100644
--- a/test/string_utils_test.cpp
+++ b/test/string_utils_test.cpp
@@ -398,9 +398,9 @@ TEST(StringUtilsTest, TokenizeFilterQuery) {
tokenizeTestHelper(filter_query, tokenList);
}
-void splitIncludeTestHelper(const std::string& include_fields, const std::vector<std::string>& expected) {
+void splitIncludeExcludeTestHelper(const std::string& include_exclude_fields, const std::vector<std::string>& expected) {
std::vector<std::string> output;
- auto tokenize_op = StringUtils::split_include_fields(include_fields, output);
+ auto tokenize_op = StringUtils::split_include_exclude_fields(include_exclude_fields, output);
ASSERT_TRUE(tokenize_op.ok());
ASSERT_EQ(expected.size(), output.size());
for (auto i = 0; i < output.size(); i++) {
@@ -408,51 +408,73 @@ void splitIncludeTestHelper(const std::string& include_fields, const std::vector
}
}
-TEST(StringUtilsTest, SplitIncludeFields) {
+TEST(StringUtilsTest, SplitIncludeExcludeFields) {
std::string include_fields;
std::vector<std::string> tokens;
include_fields = " id, title , count ";
tokens = {"id", "title", "count"};
- splitIncludeTestHelper(include_fields, tokens);
+ splitIncludeExcludeTestHelper(include_fields, tokens);
include_fields = "id, $Collection(title, pref*),count";
tokens = {"id", "$Collection(title, pref*)", "count"};
- splitIncludeTestHelper(include_fields, tokens);
+ splitIncludeExcludeTestHelper(include_fields, tokens);
include_fields = "id, $Collection(title, pref*), count, ";
tokens = {"id", "$Collection(title, pref*)", "count"};
- splitIncludeTestHelper(include_fields, tokens);
+ splitIncludeExcludeTestHelper(include_fields, tokens);
include_fields = "$Collection(title, pref*) as coll";
tokens = {"$Collection(title, pref*) as coll"};
- splitIncludeTestHelper(include_fields, tokens);
+ splitIncludeExcludeTestHelper(include_fields, tokens);
include_fields = "id, $Collection(title, pref*) as coll , count, ";
tokens = {"id", "$Collection(title, pref*) as coll", "count"};
- splitIncludeTestHelper(include_fields, tokens);
+ splitIncludeExcludeTestHelper(include_fields, tokens);
include_fields = "$Collection(title, pref*: merge) as coll";
tokens = {"$Collection(title, pref*: merge) as coll"};
- splitIncludeTestHelper(include_fields, tokens);
+ splitIncludeExcludeTestHelper(include_fields, tokens);
include_fields = "$product_variants(id,$inventory(qty,sku,$retailer(id,title: merge) as retailer_info)) as variants";
tokens = {"$product_variants(id,$inventory(qty,sku,$retailer(id,title: merge) as retailer_info)) as variants"};
- splitIncludeTestHelper(include_fields, tokens);
+ splitIncludeExcludeTestHelper(include_fields, tokens);
+
+ std::string exclude_fields = " id, title, $Collection(title), count,";
+ tokens = {"id", "title", "$Collection(title)", "count"};
+ splitIncludeExcludeTestHelper(exclude_fields, tokens);
+
+ exclude_fields = " id, title , count, $Collection(title), $product_variants(id,$inventory(qty,sku,$retailer(id,title)))";
+ tokens = {"id", "title", "count", "$Collection(title)", "$product_variants(id,$inventory(qty,sku,$retailer(id,title)))"};
+ splitIncludeExcludeTestHelper(exclude_fields, tokens);
}
-TEST(StringUtilsTest, SplitReferenceIncludeFields) {
+TEST(StringUtilsTest, SplitReferenceIncludeExcludeFields) {
std::string include_fields = "$retailer(id,title: merge) as retailer_info:merge) as variants, foo", token;
size_t index = 0;
- auto tokenize_op = StringUtils::split_reference_include_fields(include_fields, index, token);
+ auto tokenize_op = StringUtils::split_reference_include_exclude_fields(include_fields, index, token);
ASSERT_TRUE(tokenize_op.ok());
ASSERT_EQ("$retailer(id,title: merge) as retailer_info", token);
ASSERT_EQ(":merge) as variants, foo", include_fields.substr(index));
include_fields = "$inventory(qty,sku,$retailer(id,title: merge) as retailer_info) as inventory) as variants, foo";
index = 0;
- tokenize_op = StringUtils::split_reference_include_fields(include_fields, index, token);
+ tokenize_op = StringUtils::split_reference_include_exclude_fields(include_fields, index, token);
ASSERT_TRUE(tokenize_op.ok());
ASSERT_EQ("$inventory(qty,sku,$retailer(id,title: merge) as retailer_info) as inventory", token);
ASSERT_EQ(") as variants, foo", include_fields.substr(index));
+
+ std::string exclude_fields = "$Collection(title), $product_variants(id,$inventory(qty,sku,$retailer(id,title)))";
+ index = 0;
+ tokenize_op = StringUtils::split_reference_include_exclude_fields(exclude_fields, index, token);
+ ASSERT_TRUE(tokenize_op.ok());
+ ASSERT_EQ("$Collection(title)", token);
+ ASSERT_EQ(", $product_variants(id,$inventory(qty,sku,$retailer(id,title)))", exclude_fields.substr(index));
+
+ exclude_fields = "$inventory(qty,sku,$retailer(id,title)), foo)";
+ index = 0;
+ tokenize_op = StringUtils::split_reference_include_exclude_fields(exclude_fields, index, token);
+ ASSERT_TRUE(tokenize_op.ok());
+ ASSERT_EQ("$inventory(qty,sku,$retailer(id,title))", token);
+ ASSERT_EQ(", foo)", exclude_fields.substr(index));
}