Skip to content

Open Source alternative to Algolia + Pinecone and an Easier-to-Use alternative to ElasticSearch ⚡ 🔍 ✨ Fast, typo tolerant, in-memory fuzzy Search Engine for building delightful search experiences

Allow joining export documents. (#1822)

Harpreet Sangar87e33e197066
diff --git a/include/collection.h b/include/collection.h
index 32ece94d..8ebd0e88 100644
--- a/include/collection.h
+++ b/include/collection.h
@@ -469,6 +469,12 @@ public:
const std::map<std::string, reference_filter_result_t>& reference_filter_results,
const std::vector<ref_include_exclude_fields>& ref_include_exclude_fields_vec);
+ Option<bool> prune_doc_with_lock(nlohmann::json& doc, const tsl::htrie_set<char>& include_names,
+ const tsl::htrie_set<char>& exclude_names,
+ const std::map<std::string, reference_filter_result_t>& reference_filter_results = {},
+ const uint32_t& seq_id = 0,
+ const std::vector<ref_include_exclude_fields>& ref_include_exclude_fields_vec = {});
+
static Option<bool> prune_doc(nlohmann::json& doc, const tsl::htrie_set<char>& include_names,
const tsl::htrie_set<char>& exclude_names, const std::string& parent_name = "",
size_t depth = 0,
diff --git a/include/collection_manager.h b/include/collection_manager.h
index 4a2b5c77..e572200a 100644
--- a/include/collection_manager.h
+++ b/include/collection_manager.h
@@ -228,10 +228,10 @@ public:
ref_include_collection_names_t*& reference_collection_names);
// Separate out the reference includes and excludes into `ref_include_exclude_fields_vec`.
- static Option<bool> _initialize_ref_include_exclude_fields_vec(const std::string& filter_query,
- std::vector<std::string>& include_fields_vec,
- std::vector<std::string>& exclude_fields_vec,
- std::vector<ref_include_exclude_fields>& ref_include_exclude_fields_vec);
+ static Option<bool> initialize_ref_include_exclude_fields_vec(const std::string& filter_query,
+ std::vector<std::string>& include_fields_vec,
+ std::vector<std::string>& exclude_fields_vec,
+ std::vector<ref_include_exclude_fields>& ref_include_exclude_fields_vec);
void add_referenced_in_backlog(const std::string& collection_name, reference_pair&& pair);
diff --git a/include/core_api_utils.h b/include/core_api_utils.h
index cc266576..480768bc 100644
--- a/include/core_api_utils.h
+++ b/include/core_api_utils.h
@@ -20,10 +20,11 @@ struct deletion_state_t: public req_state_t {
struct export_state_t: public req_state_t {
Collection* collection;
- std::vector<std::pair<size_t, uint32_t*>> index_ids;
- std::vector<size_t> offsets;
+ filter_result_t filter_result;
+ size_t offset = 0;
tsl::htrie_set<char> include_fields;
tsl::htrie_set<char> exclude_fields;
+ std::vector<ref_include_exclude_fields> ref_include_exclude_fields_vec;
size_t export_batch_size = 100;
std::string* res_body;
@@ -34,10 +35,6 @@ struct export_state_t: public req_state_t {
rocksdb::Slice* iter_upper_bound = nullptr;
~export_state_t() override {
- for(auto& kv: index_ids) {
- delete [] kv.second;
- }
-
delete iter_upper_bound;
delete it;
}
diff --git a/src/collection.cpp b/src/collection.cpp
index de209e15..ba521d9f 100644
--- a/src/collection.cpp
+++ b/src/collection.cpp
@@ -5584,6 +5584,17 @@ Option<bool> Collection::include_references(nlohmann::json& doc, const uint32_t&
return Option<bool>(true);
}
+Option<bool> Collection::prune_doc_with_lock(nlohmann::json& doc, const tsl::htrie_set<char>& include_names,
+ const tsl::htrie_set<char>& exclude_names,
+ const std::map<std::string, reference_filter_result_t>& reference_filter_results,
+ const uint32_t& seq_id,
+ const std::vector<ref_include_exclude_fields>& ref_include_exclude_fields_vec) {
+ std::shared_lock lock(mutex);
+
+ return prune_doc(doc, include_names, exclude_names, "", 0, reference_filter_results, this, seq_id,
+ ref_include_exclude_fields_vec);
+}
+
Option<bool> Collection::prune_doc(nlohmann::json& doc,
const tsl::htrie_set<char>& include_names,
const tsl::htrie_set<char>& exclude_names,
diff --git a/src/collection_manager.cpp b/src/collection_manager.cpp
index 5e189d46..2f6549a0 100644
--- a/src/collection_manager.cpp
+++ b/src/collection_manager.cpp
@@ -1373,10 +1373,10 @@ Option<bool> parse_nested_include(const std::string& include_field_exp,
return Option<bool>(true);
}
-Option<bool> CollectionManager::_initialize_ref_include_exclude_fields_vec(const std::string& filter_query,
- std::vector<std::string>& include_fields_vec,
- std::vector<std::string>& exclude_fields_vec,
- std::vector<ref_include_exclude_fields>& ref_include_exclude_fields_vec) {
+Option<bool> CollectionManager::initialize_ref_include_exclude_fields_vec(const std::string& filter_query,
+ std::vector<std::string>& include_fields_vec,
+ std::vector<std::string>& exclude_fields_vec,
+ std::vector<ref_include_exclude_fields>& ref_include_exclude_fields_vec) {
ref_include_collection_names_t* ref_include_coll_names = nullptr;
CollectionManager::_get_reference_collection_names(filter_query, ref_include_coll_names);
std::unique_ptr<CollectionManager::ref_include_collection_names_t> guard(ref_include_coll_names);
@@ -1933,8 +1933,8 @@ Option<bool> CollectionManager::do_search(std::map<std::string, std::string>& re
per_page = 0;
}
- auto initialize_op = _initialize_ref_include_exclude_fields_vec(filter_query, include_fields_vec, exclude_fields_vec,
- ref_include_exclude_fields_vec);
+ auto initialize_op = initialize_ref_include_exclude_fields_vec(filter_query, include_fields_vec, exclude_fields_vec,
+ ref_include_exclude_fields_vec);
if (!initialize_op.ok()) {
return initialize_op;
}
diff --git a/src/core_api.cpp b/src/core_api.cpp
index 45776b4b..ec2b7772 100644
--- a/src/core_api.cpp
+++ b/src/core_api.cpp
@@ -1039,30 +1039,62 @@ bool get_export_documents(const std::shared_ptr<http_req>& req, const std::share
if(req->data == nullptr) {
export_state = new export_state_t();
+ export_state->collection = collection.get();
// destruction of data is managed by req destructor
req->data = export_state;
- std::string simple_filter_query;
+ std::string filter_query;
+ std::vector<std::string> include_fields_vec;
+ std::vector<std::string> exclude_fields_vec;
spp::sparse_hash_set<std::string> exclude_fields;
spp::sparse_hash_set<std::string> include_fields;
if(req->params.count(FILTER_BY) != 0) {
- simple_filter_query = req->params[FILTER_BY];
+ filter_query = req->params[FILTER_BY];
}
if(req->params.count(INCLUDE_FIELDS) != 0) {
- std::vector<std::string> include_fields_vec;
- StringUtils::split(req->params[INCLUDE_FIELDS], include_fields_vec, ",");
- include_fields = spp::sparse_hash_set<std::string>(include_fields_vec.begin(), include_fields_vec.end());
+ auto op = StringUtils::split_include_exclude_fields(req->params[INCLUDE_FIELDS], include_fields_vec);
+ if (!op.ok()) {
+ res->set(op.code(), op.error());
+ req->last_chunk_aggregate = true;
+ res->final = true;
+ stream_response(req, res);
+ return false;
+ }
}
if(req->params.count(EXCLUDE_FIELDS) != 0) {
- std::vector<std::string> exclude_fields_vec;
- StringUtils::split(req->params[EXCLUDE_FIELDS], exclude_fields_vec, ",");
- exclude_fields = spp::sparse_hash_set<std::string>(exclude_fields_vec.begin(), exclude_fields_vec.end());
+ auto op = StringUtils::split_include_exclude_fields(req->params[EXCLUDE_FIELDS], exclude_fields_vec);
+ if (!op.ok()) {
+ res->set(op.code(), op.error());
+ req->last_chunk_aggregate = true;
+ res->final = true;
+ stream_response(req, res);
+ return false;
+ }
}
+ auto initialize_op = CollectionManager::initialize_ref_include_exclude_fields_vec(filter_query,
+ include_fields_vec,
+ exclude_fields_vec,
+ export_state->ref_include_exclude_fields_vec);
+ if (!initialize_op.ok()) {
+ res->set(initialize_op.code(), initialize_op.error());
+ req->last_chunk_aggregate = true;
+ res->final = true;
+ stream_response(req, res);
+ return false;
+ }
+
+ if (include_fields_vec.empty()) {
+ include_fields_vec.emplace_back("*");
+ }
+
+ include_fields.insert(include_fields_vec.begin(), include_fields_vec.end());
+ exclude_fields.insert(exclude_fields_vec.begin(), exclude_fields_vec.end());
+
collection->populate_include_exclude_fields_lk(include_fields, exclude_fields,
export_state->include_fields, export_state->exclude_fields);
@@ -1070,13 +1102,12 @@ bool get_export_documents(const std::shared_ptr<http_req>& req, const std::share
export_state->export_batch_size = std::stoul(req->params[BATCH_SIZE]);
}
- if(simple_filter_query.empty()) {
+ if(filter_query.empty()) {
export_state->iter_upper_bound_key = collection->get_seq_id_collection_prefix() + "`"; // cannot inline this
export_state->iter_upper_bound = new rocksdb::Slice(export_state->iter_upper_bound_key);
export_state->it = collectionManager.get_store()->scan(seq_id_prefix, export_state->iter_upper_bound);
} else {
- filter_result_t filter_result;
- auto filter_ids_op = collection->get_filter_ids(simple_filter_query, filter_result);
+ auto filter_ids_op = collection->get_filter_ids(filter_query, export_state->filter_result);
if(!filter_ids_op.ok()) {
res->set(filter_ids_op.code(), filter_ids_op.error());
@@ -1086,14 +1117,7 @@ bool get_export_documents(const std::shared_ptr<http_req>& req, const std::share
return false;
}
- export_state->index_ids.emplace_back(filter_result.count, filter_result.docs);
- filter_result.docs = nullptr;
-
- for(size_t i=0; i<export_state->index_ids.size(); i++) {
- export_state->offsets.push_back(0);
- }
export_state->res_body = &res->body;
- export_state->collection = collection.get();
}
} else {
export_state = dynamic_cast<export_state_t*>(req->data);
@@ -1105,11 +1129,34 @@ bool get_export_documents(const std::shared_ptr<http_req>& req, const std::share
std::string().swap(res->body);
while(it->Valid() && it->key().ToString().compare(0, seq_id_prefix.size(), seq_id_prefix) == 0) {
+ nlohmann::json doc = nlohmann::json::parse(it->value().ToString());
+ Collection::remove_flat_fields(doc);
+ Collection::remove_reference_helper_fields(doc);
+
if(export_state->include_fields.empty() && export_state->exclude_fields.empty()) {
- res->body += it->value().ToString();
+ res->body += doc.dump();
} else {
- nlohmann::json doc = nlohmann::json::parse(it->value().ToString());
- Collection::prune_doc(doc, export_state->include_fields, export_state->exclude_fields);
+ if (doc.count("id") == 0 || !doc.at("id").is_string()) {
+ res->set(500, "Could not find `id` field in the document: " + doc.dump());
+ req->last_chunk_aggregate = true;
+ res->final = true;
+ stream_response(req, res);
+ return false;
+ }
+
+ auto const& coll = export_state->collection;
+ auto const seq_id_op = coll->doc_id_to_seq_id_with_lock(doc.at("id"));
+ if (!seq_id_op.ok()) {
+ res->set(seq_id_op.code(), seq_id_op.error());
+ req->last_chunk_aggregate = true;
+ res->final = true;
+ stream_response(req, res);
+ return false;
+ }
+
+ std::map<std::string, reference_filter_result_t> references = {};
+ coll->prune_doc_with_lock(doc, export_state->include_fields, export_state->exclude_fields,
+ references, seq_id_op.get(), export_state->ref_include_exclude_fields_vec);
res->body += doc.dump();
}
diff --git a/src/core_api_utils.cpp b/src/core_api_utils.cpp
index b043ffc2..7d440dd5 100644
--- a/src/core_api_utils.cpp
+++ b/src/core_api_utils.cpp
@@ -49,48 +49,46 @@ Option<bool> stateful_export_docs(export_state_t* export_state, size_t batch_siz
export_state->res_body->clear();
- for(size_t i = 0; i < export_state->index_ids.size(); i++) {
- std::pair<size_t, uint32_t*>& size_ids = export_state->index_ids[i];
- size_t ids_len = size_ids.first;
- uint32_t* ids = size_ids.second;
-
- size_t start_index = export_state->offsets[i];
- size_t batched_len = std::min(ids_len, (start_index+batch_size));
-
- for(size_t j = start_index; j < batched_len; j++) {
- auto seq_id = ids[j];
- nlohmann::json doc;
- Option<bool> get_op = export_state->collection->get_document_from_store(seq_id, doc);
-
- if(get_op.ok()) {
- if(export_state->include_fields.empty() && export_state->exclude_fields.empty()) {
- export_state->res_body->append(doc.dump());
- } else {
- Collection::remove_flat_fields(doc);
- Collection::remove_reference_helper_fields(doc);
- Collection::prune_doc(doc, export_state->include_fields, export_state->exclude_fields);
- export_state->res_body->append(doc.dump());
- }
-
- export_state->res_body->append("\n");
+ auto const& filter_result = export_state->filter_result;
+ size_t ids_len = filter_result.count;
+ uint32_t* ids = filter_result.docs;
+
+ size_t start_index = export_state->offset;
+ size_t batched_len = std::min(ids_len, (start_index+batch_size));
+
+ for(size_t j = start_index; j < batched_len; j++) {
+ uint32_t seq_id = ids[j];
+ nlohmann::json doc;
+
+ auto const& coll = export_state->collection;
+ Option<bool> get_op = coll->get_document_from_store(seq_id, doc);
+ Collection::remove_flat_fields(doc);
+ Collection::remove_reference_helper_fields(doc);
+
+ if(get_op.ok()) {
+ if(export_state->include_fields.empty() && export_state->exclude_fields.empty()) {
+ export_state->res_body->append(doc.dump());
+ } else {
+ coll->prune_doc_with_lock(doc, export_state->include_fields, export_state->exclude_fields,
+ filter_result.coll_to_references[j], seq_id,
+ export_state->ref_include_exclude_fields_vec);
+ export_state->res_body->append(doc.dump());
}
- export_state->offsets[i]++;
- batch_count++;
+ export_state->res_body->append("\n");
+ }
- if(batch_count == batch_size) {
- goto END;
- }
+ export_state->offset++;
+ batch_count++;
+
+ if(batch_count == batch_size) {
+ goto END;
}
}
END:
- done = true;
- for(size_t i=0; i<export_state->index_ids.size(); i++) {
- size_t current_offset = export_state->offsets[i];
- done = done && (current_offset == export_state->index_ids[i].first);
- }
+ done = export_state->offset == export_state->filter_result.count;
if(done && !export_state->res_body->empty()) {
export_state->res_body->pop_back();
diff --git a/test/collection_manager_test.cpp b/test/collection_manager_test.cpp
index 64861318..a9ff40ac 100644
--- a/test/collection_manager_test.cpp
+++ b/test/collection_manager_test.cpp
@@ -1582,17 +1582,17 @@ TEST_F(CollectionManagerTest, InitializeRefIncludeExcludeFields) {
std::string filter_query = "";
std::vector<std::string> include_fields_vec, exclude_fields_vec;
std::vector<ref_include_exclude_fields> ref_include_exclude_fields_vec;
- auto initialize_op = CollectionManager::_initialize_ref_include_exclude_fields_vec(filter_query, include_fields_vec,
- exclude_fields_vec,
- ref_include_exclude_fields_vec);
+ auto initialize_op = CollectionManager::initialize_ref_include_exclude_fields_vec(filter_query, include_fields_vec,
+ exclude_fields_vec,
+ ref_include_exclude_fields_vec);
ASSERT_TRUE(initialize_op.ok());
ASSERT_TRUE(ref_include_exclude_fields_vec.empty());
filter_query = "$foo(bar:baz)";
exclude_fields_vec = {"$foo(bar)"};
- initialize_op = CollectionManager::_initialize_ref_include_exclude_fields_vec(filter_query, include_fields_vec,
- exclude_fields_vec,
- ref_include_exclude_fields_vec);
+ initialize_op = CollectionManager::initialize_ref_include_exclude_fields_vec(filter_query, include_fields_vec,
+ exclude_fields_vec,
+ ref_include_exclude_fields_vec);
ASSERT_TRUE(initialize_op.ok());
ASSERT_EQ(1, ref_include_exclude_fields_vec.size());
ASSERT_EQ("foo", ref_include_exclude_fields_vec[0].collection_name);
@@ -1606,25 +1606,25 @@ TEST_F(CollectionManagerTest, InitializeRefIncludeExcludeFields) {
filter_query = "";
include_fields_vec = {"$Customers(product_price, strategy: foo) as customers"};
- initialize_op = CollectionManager::_initialize_ref_include_exclude_fields_vec(filter_query, include_fields_vec,
- exclude_fields_vec,
- ref_include_exclude_fields_vec);
+ initialize_op = CollectionManager::initialize_ref_include_exclude_fields_vec(filter_query, include_fields_vec,
+ exclude_fields_vec,
+ ref_include_exclude_fields_vec);
ASSERT_FALSE(initialize_op.ok());
ASSERT_EQ("Error parsing `$Customers(product_price, strategy: foo) as customers`: Unknown include strategy `foo`. "
"Valid options are `merge`, `nest`, `nest_array`.", initialize_op.error());
include_fields_vec = {"$Customers(product_price, foo: bar) as customers"};
- initialize_op = CollectionManager::_initialize_ref_include_exclude_fields_vec(filter_query, include_fields_vec,
- exclude_fields_vec,
- ref_include_exclude_fields_vec);
+ initialize_op = CollectionManager::initialize_ref_include_exclude_fields_vec(filter_query, include_fields_vec,
+ exclude_fields_vec,
+ ref_include_exclude_fields_vec);
ASSERT_FALSE(initialize_op.ok());
ASSERT_EQ("Unknown reference `include_fields` parameter: `foo`.", initialize_op.error());
filter_query = "$Customers(customer_id:=customer_a && (product_price:>100 && product_price:<200))";
include_fields_vec = {"$Customers(product_price, strategy: merge) as customers"};
- initialize_op = CollectionManager::_initialize_ref_include_exclude_fields_vec(filter_query, include_fields_vec,
- exclude_fields_vec,
- ref_include_exclude_fields_vec);
+ initialize_op = CollectionManager::initialize_ref_include_exclude_fields_vec(filter_query, include_fields_vec,
+ exclude_fields_vec,
+ ref_include_exclude_fields_vec);
ASSERT_TRUE(initialize_op.ok());
ASSERT_EQ(1, ref_include_exclude_fields_vec.size());
ASSERT_EQ("Customers", ref_include_exclude_fields_vec[0].collection_name);
@@ -1636,9 +1636,9 @@ TEST_F(CollectionManagerTest, InitializeRefIncludeExcludeFields) {
filter_query = "$Customers(customer_id:=customer_a && (product_price:>100 && product_price:<200))";
include_fields_vec = {"$Customers(product_price, strategy: nest_array) as customers"};
- initialize_op = CollectionManager::_initialize_ref_include_exclude_fields_vec(filter_query, include_fields_vec,
- exclude_fields_vec,
- ref_include_exclude_fields_vec);
+ initialize_op = CollectionManager::initialize_ref_include_exclude_fields_vec(filter_query, include_fields_vec,
+ exclude_fields_vec,
+ ref_include_exclude_fields_vec);
ASSERT_TRUE(initialize_op.ok());
ASSERT_EQ(1, ref_include_exclude_fields_vec.size());
ASSERT_EQ("Customers", ref_include_exclude_fields_vec[0].collection_name);
@@ -1650,9 +1650,9 @@ TEST_F(CollectionManagerTest, InitializeRefIncludeExcludeFields) {
filter_query = "$product_variants( $inventory($retailers(location:(33.865,-118.375,100 km))))";
include_fields_vec = {"$product_variants(id,$inventory(qty,sku,$retailers(id,title)))"};
- initialize_op = CollectionManager::_initialize_ref_include_exclude_fields_vec(filter_query, include_fields_vec,
- exclude_fields_vec,
- ref_include_exclude_fields_vec);
+ initialize_op = CollectionManager::initialize_ref_include_exclude_fields_vec(filter_query, include_fields_vec,
+ exclude_fields_vec,
+ ref_include_exclude_fields_vec);
ASSERT_TRUE(initialize_op.ok());
ASSERT_EQ(1, ref_include_exclude_fields_vec.size());
ASSERT_EQ("product_variants", ref_include_exclude_fields_vec[0].collection_name);
@@ -1675,9 +1675,9 @@ TEST_F(CollectionManagerTest, InitializeRefIncludeExcludeFields) {
filter_query = "$product_variants( $inventory($retailers(location:(33.865,-118.375,100 km))))";
include_fields_vec = {"$product_variants(title, $inventory(qty, strategy:merge) as inventory, strategy: nest) as variants"};
- initialize_op = CollectionManager::_initialize_ref_include_exclude_fields_vec(filter_query, include_fields_vec,
- exclude_fields_vec,
- ref_include_exclude_fields_vec);
+ initialize_op = CollectionManager::initialize_ref_include_exclude_fields_vec(filter_query, include_fields_vec,
+ exclude_fields_vec,
+ ref_include_exclude_fields_vec);
ASSERT_TRUE(initialize_op.ok());
ASSERT_EQ(1, ref_include_exclude_fields_vec.size());
ASSERT_EQ("product_variants", ref_include_exclude_fields_vec[0].collection_name);
@@ -1701,9 +1701,9 @@ TEST_F(CollectionManagerTest, InitializeRefIncludeExcludeFields) {
filter_query = "$product_variants( $inventory(id:*) && $retailers(location:(33.865,-118.375,100 km)))";
include_fields_vec = {"$product_variants(title, $inventory(qty, strategy:merge) as inventory,"
" $retailers(title), strategy: merge) as variants"};
- initialize_op = CollectionManager::_initialize_ref_include_exclude_fields_vec(filter_query, include_fields_vec,
- exclude_fields_vec,
- ref_include_exclude_fields_vec);
+ initialize_op = CollectionManager::initialize_ref_include_exclude_fields_vec(filter_query, include_fields_vec,
+ exclude_fields_vec,
+ ref_include_exclude_fields_vec);
ASSERT_TRUE(initialize_op.ok());
ASSERT_EQ(1, ref_include_exclude_fields_vec.size());
ASSERT_EQ("product_variants", ref_include_exclude_fields_vec[0].collection_name);
@@ -1726,9 +1726,9 @@ TEST_F(CollectionManagerTest, InitializeRefIncludeExcludeFields) {
filter_query = "$product_variants( $inventory(id:*) && $retailers(location:(33.865,-118.375,100 km)))";
include_fields_vec = {"$product_variants(title, $inventory(qty, strategy:merge) as inventory, description,"
" $retailers(title), foo, strategy: merge) as variants"};
- initialize_op = CollectionManager::_initialize_ref_include_exclude_fields_vec(filter_query, include_fields_vec,
- exclude_fields_vec,
- ref_include_exclude_fields_vec);
+ initialize_op = CollectionManager::initialize_ref_include_exclude_fields_vec(filter_query, include_fields_vec,
+ exclude_fields_vec,
+ ref_include_exclude_fields_vec);
ASSERT_TRUE(initialize_op.ok());
ASSERT_EQ(1, ref_include_exclude_fields_vec.size());
ASSERT_EQ("product_variants", ref_include_exclude_fields_vec[0].collection_name);
@@ -1751,9 +1751,9 @@ TEST_F(CollectionManagerTest, InitializeRefIncludeExcludeFields) {
filter_query = "$Customers(customer_id:=customer_a && (product_price:>100 && product_price:<200))";
include_fields_vec.clear();
exclude_fields_vec = {"$Customers(product_price)"};
- initialize_op = CollectionManager::_initialize_ref_include_exclude_fields_vec(filter_query, include_fields_vec,
- exclude_fields_vec,
- ref_include_exclude_fields_vec);
+ initialize_op = CollectionManager::initialize_ref_include_exclude_fields_vec(filter_query, include_fields_vec,
+ exclude_fields_vec,
+ ref_include_exclude_fields_vec);
ASSERT_TRUE(initialize_op.ok());
ASSERT_EQ(1, ref_include_exclude_fields_vec.size());
ASSERT_EQ("Customers", ref_include_exclude_fields_vec[0].collection_name);
@@ -1767,9 +1767,9 @@ TEST_F(CollectionManagerTest, InitializeRefIncludeExcludeFields) {
filter_query = "$product_variants( $inventory(id:*) && $retailers(location:(33.865,-118.375,100 km)))";
include_fields_vec.clear();
exclude_fields_vec = {"$product_variants(title, $inventory(qty), description, $retailers(title), foo)"};
- initialize_op = CollectionManager::_initialize_ref_include_exclude_fields_vec(filter_query, include_fields_vec,
- exclude_fields_vec,
- ref_include_exclude_fields_vec);
+ initialize_op = CollectionManager::initialize_ref_include_exclude_fields_vec(filter_query, include_fields_vec,
+ exclude_fields_vec,
+ ref_include_exclude_fields_vec);
ASSERT_TRUE(initialize_op.ok());
ASSERT_EQ(1, ref_include_exclude_fields_vec.size());
ASSERT_EQ("product_variants", ref_include_exclude_fields_vec[0].collection_name);
@@ -1795,9 +1795,9 @@ TEST_F(CollectionManagerTest, InitializeRefIncludeExcludeFields) {
filter_query = "$product_variants( $inventory($retailers(location:(33.865,-118.375,100 km))))";
include_fields_vec = {"$product_variants(title, $inventory(qty, strategy:merge) as inventory, strategy: nest) as variants"};
exclude_fields_vec = {"$product_variants(title, $inventory(qty, $retailers(title)))"};
- initialize_op = CollectionManager::_initialize_ref_include_exclude_fields_vec(filter_query, include_fields_vec,
- exclude_fields_vec,
- ref_include_exclude_fields_vec);
+ initialize_op = CollectionManager::initialize_ref_include_exclude_fields_vec(filter_query, include_fields_vec,
+ exclude_fields_vec,
+ ref_include_exclude_fields_vec);
ASSERT_TRUE(initialize_op.ok());
ASSERT_EQ(1, ref_include_exclude_fields_vec.size());
ASSERT_EQ("product_variants", ref_include_exclude_fields_vec[0].collection_name);
diff --git a/test/core_api_utils_test.cpp b/test/core_api_utils_test.cpp
index f6624313..f24ffbcf 100644
--- a/test/core_api_utils_test.cpp
+++ b/test/core_api_utils_test.cpp
@@ -895,12 +895,7 @@ TEST_F(CoreAPIUtilsTest, ExportWithFilter) {
export_state_t export_state;
filter_result_t filter_result;
- coll1->get_filter_ids("points:>=0", filter_result);
- export_state.index_ids.emplace_back(filter_result.count, filter_result.docs);
- filter_result.docs = nullptr;
- for(size_t i=0; i<export_state.index_ids.size(); i++) {
- export_state.offsets.push_back(0);
- }
+ coll1->get_filter_ids("points:>=0", export_state.filter_result);
export_state.collection = coll1;
export_state.res_body = &res_body;
@@ -915,6 +910,125 @@ TEST_F(CoreAPIUtilsTest, ExportWithFilter) {
ASSERT_EQ('}', export_state.res_body->back());
}
+TEST_F(CoreAPIUtilsTest, ExportWithJoin) {
+ auto schema_json =
+ R"({
+ "name": "Products",
+ "fields": [
+ {"name": "product_id", "type": "string"},
+ {"name": "product_name", "type": "string"},
+ {"name": "product_description", "type": "string"},
+ {"name": "rating", "type": "int32"}
+ ]
+ })"_json;
+ std::vector<nlohmann::json> documents = {
+ R"({
+ "product_id": "product_a",
+ "product_name": "shampoo",
+ "product_description": "Our new moisturizing shampoo is perfect for those with dry or damaged hair.",
+ "rating": "2"
+ })"_json,
+ R"({
+ "product_id": "product_b",
+ "product_name": "soap",
+ "product_description": "Introducing our all-natural, organic soap bar made with essential oils and botanical ingredients.",
+ "rating": "4"
+ })"_json
+ };
+ auto collection_create_op = collectionManager.create_collection(schema_json);
+ ASSERT_TRUE(collection_create_op.ok());
+ for (auto const &json: documents) {
+ auto add_op = collection_create_op.get()->add(json.dump());
+ if (!add_op.ok()) {
+ LOG(INFO) << add_op.error();
+ }
+ ASSERT_TRUE(add_op.ok());
+ }
+
+ schema_json =
+ R"({
+ "name": "Customers",
+ "fields": [
+ {"name": "customer_id", "type": "string"},
+ {"name": "customer_name", "type": "string"},
+ {"name": "product_price", "type": "float"},
+ {"name": "product_id", "type": "string", "reference": "Products.product_id"}
+ ]
+ })"_json;
+ documents = {
+ R"({
+ "customer_id": "customer_a",
+ "customer_name": "Joe",
+ "product_price": 143,
+ "product_id": "product_a"
+ })"_json,
+ R"({
+ "customer_id": "customer_a",
+ "customer_name": "Joe",
+ "product_price": 73.5,
+ "product_id": "product_b"
+ })"_json,
+ R"({
+ "customer_id": "customer_b",
+ "customer_name": "Dan",
+ "product_price": 75,
+ "product_id": "product_a"
+ })"_json,
+ R"({
+ "customer_id": "customer_b",
+ "customer_name": "Dan",
+ "product_price": 140,
+ "product_id": "product_b"
+ })"_json
+ };
+ collection_create_op = collectionManager.create_collection(schema_json);
+ ASSERT_TRUE(collection_create_op.ok());
+ for (auto const &json: documents) {
+ auto add_op = collection_create_op.get()->add(json.dump());
+ if (!add_op.ok()) {
+ LOG(INFO) << add_op.error();
+ }
+ ASSERT_TRUE(add_op.ok());
+ }
+
+ schema_json =
+ R"({
+ "name": "Dummy",
+ "fields": [
+ {"name": "dummy_id", "type": "string"}
+ ]
+ })"_json;
+ collection_create_op = collectionManager.create_collection(schema_json);
+ ASSERT_TRUE(collection_create_op.ok());
+
+ bool done;
+ std::string res_body;
+
+ export_state_t export_state;
+ auto coll1 = collectionManager.get_collection_unsafe("Products");
+ coll1->get_filter_ids("$Customers(customer_id:customer_a)", export_state.filter_result);
+ export_state.collection = coll1;
+ export_state.res_body = &res_body;
+ export_state.include_fields.insert("product_name");
+ export_state.ref_include_exclude_fields_vec.emplace_back(ref_include_exclude_fields{"Customers", {"product_price"}, "",
+ "", ref_include::nest});
+
+ stateful_export_docs(&export_state, 1, done);
+ ASSERT_FALSE(done);
+ ASSERT_EQ('\n', export_state.res_body->back());
+ auto doc = nlohmann::json::parse(export_state.res_body->c_str());
+ ASSERT_EQ("shampoo", doc["product_name"]);
+ ASSERT_EQ(143, doc["Customers"]["product_price"]);
+
+ // should not have trailing newline character for the last line
+ stateful_export_docs(&export_state, 1, done);
+ ASSERT_TRUE(done);
+ ASSERT_EQ('}', export_state.res_body->back());
+ doc = nlohmann::json::parse(export_state.res_body->c_str());
+ ASSERT_EQ("soap", doc["product_name"]);
+ ASSERT_EQ(73.5, doc["Customers"]["product_price"]);
+}
+
TEST_F(CoreAPIUtilsTest, TestParseAPIKeyIPFromMetadata) {
// format <length of api key>:<api key><ip address>
std::string valid_metadata = "4:abcd127.0.0.1";