Open Source alternative to Algolia + Pinecone and an Easier-to-Use alternative to ElasticSearch ⚡ 🔍 ✨ Fast, typo tolerant, in-memory fuzzy Search Engine for building delightful search experiences
C++ algoliadatastoreelasticsearchenterprise-searchfacetingfull-text-searchfuzzy-searchgeosearchin-memoryinstantsearchmerchandisingpineconesearchsearch-enginesemantic-searchsimilarity-searchsite-searchsynonymstypo-tolerancevector-search
Allow joining export documents. (#1822)
Harpreet Sangar87e33e197066
diff --git a/include/collection.h b/include/collection.hindex 32ece94d..8ebd0e88 100644--- a/include/collection.h+++ b/include/collection.h@@ -469,6 +469,12 @@ public:const std::map<std::string, reference_filter_result_t>& reference_filter_results,const std::vector<ref_include_exclude_fields>& ref_include_exclude_fields_vec);+ Option<bool> prune_doc_with_lock(nlohmann::json& doc, const tsl::htrie_set<char>& include_names,+ const tsl::htrie_set<char>& exclude_names,+ const std::map<std::string, reference_filter_result_t>& reference_filter_results = {},+ const uint32_t& seq_id = 0,+ const std::vector<ref_include_exclude_fields>& ref_include_exclude_fields_vec = {});+static Option<bool> prune_doc(nlohmann::json& doc, const tsl::htrie_set<char>& include_names,const tsl::htrie_set<char>& exclude_names, const std::string& parent_name = "",size_t depth = 0,diff --git a/include/collection_manager.h b/include/collection_manager.hindex 4a2b5c77..e572200a 100644--- a/include/collection_manager.h+++ b/include/collection_manager.h@@ -228,10 +228,10 @@ public:ref_include_collection_names_t*& reference_collection_names);// Separate out the reference includes and excludes into `ref_include_exclude_fields_vec`.- static Option<bool> _initialize_ref_include_exclude_fields_vec(const std::string& filter_query,- std::vector<std::string>& include_fields_vec,- std::vector<std::string>& exclude_fields_vec,- std::vector<ref_include_exclude_fields>& ref_include_exclude_fields_vec);+ static Option<bool> initialize_ref_include_exclude_fields_vec(const std::string& filter_query,+ std::vector<std::string>& include_fields_vec,+ std::vector<std::string>& exclude_fields_vec,+ std::vector<ref_include_exclude_fields>& ref_include_exclude_fields_vec);void add_referenced_in_backlog(const std::string& collection_name, reference_pair&& pair);diff --git a/include/core_api_utils.h b/include/core_api_utils.hindex cc266576..480768bc 100644--- a/include/core_api_utils.h+++ b/include/core_api_utils.h@@ -20,10 +20,11 @@ struct deletion_state_t: public req_state_t {struct export_state_t: public req_state_t {Collection* collection;- std::vector<std::pair<size_t, uint32_t*>> index_ids;- std::vector<size_t> offsets;+ filter_result_t filter_result;+ size_t offset = 0;tsl::htrie_set<char> include_fields;tsl::htrie_set<char> exclude_fields;+ std::vector<ref_include_exclude_fields> ref_include_exclude_fields_vec;size_t export_batch_size = 100;std::string* res_body;@@ -34,10 +35,6 @@ struct export_state_t: public req_state_t {rocksdb::Slice* iter_upper_bound = nullptr;~export_state_t() override {- for(auto& kv: index_ids) {- delete [] kv.second;- }-delete iter_upper_bound;delete it;}diff --git a/src/collection.cpp b/src/collection.cppindex de209e15..ba521d9f 100644--- a/src/collection.cpp+++ b/src/collection.cpp@@ -5584,6 +5584,17 @@ Option<bool> Collection::include_references(nlohmann::json& doc, const uint32_t&return Option<bool>(true);}+Option<bool> Collection::prune_doc_with_lock(nlohmann::json& doc, const tsl::htrie_set<char>& include_names,+ const tsl::htrie_set<char>& exclude_names,+ const std::map<std::string, reference_filter_result_t>& reference_filter_results,+ const uint32_t& seq_id,+ const std::vector<ref_include_exclude_fields>& ref_include_exclude_fields_vec) {+ std::shared_lock lock(mutex);++ return prune_doc(doc, include_names, exclude_names, "", 0, reference_filter_results, this, seq_id,+ ref_include_exclude_fields_vec);+}+Option<bool> Collection::prune_doc(nlohmann::json& doc,const tsl::htrie_set<char>& include_names,const tsl::htrie_set<char>& exclude_names,diff --git a/src/collection_manager.cpp b/src/collection_manager.cppindex 5e189d46..2f6549a0 100644--- a/src/collection_manager.cpp+++ b/src/collection_manager.cpp@@ -1373,10 +1373,10 @@ Option<bool> parse_nested_include(const std::string& include_field_exp,return Option<bool>(true);}-Option<bool> CollectionManager::_initialize_ref_include_exclude_fields_vec(const std::string& filter_query,- std::vector<std::string>& include_fields_vec,- std::vector<std::string>& exclude_fields_vec,- std::vector<ref_include_exclude_fields>& ref_include_exclude_fields_vec) {+Option<bool> CollectionManager::initialize_ref_include_exclude_fields_vec(const std::string& filter_query,+ std::vector<std::string>& include_fields_vec,+ std::vector<std::string>& exclude_fields_vec,+ std::vector<ref_include_exclude_fields>& ref_include_exclude_fields_vec) {ref_include_collection_names_t* ref_include_coll_names = nullptr;CollectionManager::_get_reference_collection_names(filter_query, ref_include_coll_names);std::unique_ptr<CollectionManager::ref_include_collection_names_t> guard(ref_include_coll_names);@@ -1933,8 +1933,8 @@ Option<bool> CollectionManager::do_search(std::map<std::string, std::string>& reper_page = 0;}- auto initialize_op = _initialize_ref_include_exclude_fields_vec(filter_query, include_fields_vec, exclude_fields_vec,- ref_include_exclude_fields_vec);+ auto initialize_op = initialize_ref_include_exclude_fields_vec(filter_query, include_fields_vec, exclude_fields_vec,+ ref_include_exclude_fields_vec);if (!initialize_op.ok()) {return initialize_op;}diff --git a/src/core_api.cpp b/src/core_api.cppindex 45776b4b..ec2b7772 100644--- a/src/core_api.cpp+++ b/src/core_api.cpp@@ -1039,30 +1039,62 @@ bool get_export_documents(const std::shared_ptr<http_req>& req, const std::shareif(req->data == nullptr) {export_state = new export_state_t();+ export_state->collection = collection.get();// destruction of data is managed by req destructorreq->data = export_state;- std::string simple_filter_query;+ std::string filter_query;+ std::vector<std::string> include_fields_vec;+ std::vector<std::string> exclude_fields_vec;spp::sparse_hash_set<std::string> exclude_fields;spp::sparse_hash_set<std::string> include_fields;if(req->params.count(FILTER_BY) != 0) {- simple_filter_query = req->params[FILTER_BY];+ filter_query = req->params[FILTER_BY];}if(req->params.count(INCLUDE_FIELDS) != 0) {- std::vector<std::string> include_fields_vec;- StringUtils::split(req->params[INCLUDE_FIELDS], include_fields_vec, ",");- include_fields = spp::sparse_hash_set<std::string>(include_fields_vec.begin(), include_fields_vec.end());+ auto op = StringUtils::split_include_exclude_fields(req->params[INCLUDE_FIELDS], include_fields_vec);+ if (!op.ok()) {+ res->set(op.code(), op.error());+ req->last_chunk_aggregate = true;+ res->final = true;+ stream_response(req, res);+ return false;+ }}if(req->params.count(EXCLUDE_FIELDS) != 0) {- std::vector<std::string> exclude_fields_vec;- StringUtils::split(req->params[EXCLUDE_FIELDS], exclude_fields_vec, ",");- exclude_fields = spp::sparse_hash_set<std::string>(exclude_fields_vec.begin(), exclude_fields_vec.end());+ auto op = StringUtils::split_include_exclude_fields(req->params[EXCLUDE_FIELDS], exclude_fields_vec);+ if (!op.ok()) {+ res->set(op.code(), op.error());+ req->last_chunk_aggregate = true;+ res->final = true;+ stream_response(req, res);+ return false;+ }}+ auto initialize_op = CollectionManager::initialize_ref_include_exclude_fields_vec(filter_query,+ include_fields_vec,+ exclude_fields_vec,+ export_state->ref_include_exclude_fields_vec);+ if (!initialize_op.ok()) {+ res->set(initialize_op.code(), initialize_op.error());+ req->last_chunk_aggregate = true;+ res->final = true;+ stream_response(req, res);+ return false;+ }++ if (include_fields_vec.empty()) {+ include_fields_vec.emplace_back("*");+ }++ include_fields.insert(include_fields_vec.begin(), include_fields_vec.end());+ exclude_fields.insert(exclude_fields_vec.begin(), exclude_fields_vec.end());+collection->populate_include_exclude_fields_lk(include_fields, exclude_fields,export_state->include_fields, export_state->exclude_fields);@@ -1070,13 +1102,12 @@ bool get_export_documents(const std::shared_ptr<http_req>& req, const std::shareexport_state->export_batch_size = std::stoul(req->params[BATCH_SIZE]);}- if(simple_filter_query.empty()) {+ if(filter_query.empty()) {export_state->iter_upper_bound_key = collection->get_seq_id_collection_prefix() + "`"; // cannot inline thisexport_state->iter_upper_bound = new rocksdb::Slice(export_state->iter_upper_bound_key);export_state->it = collectionManager.get_store()->scan(seq_id_prefix, export_state->iter_upper_bound);} else {- filter_result_t filter_result;- auto filter_ids_op = collection->get_filter_ids(simple_filter_query, filter_result);+ auto filter_ids_op = collection->get_filter_ids(filter_query, export_state->filter_result);if(!filter_ids_op.ok()) {res->set(filter_ids_op.code(), filter_ids_op.error());@@ -1086,14 +1117,7 @@ bool get_export_documents(const std::shared_ptr<http_req>& req, const std::sharereturn false;}- export_state->index_ids.emplace_back(filter_result.count, filter_result.docs);- filter_result.docs = nullptr;-- for(size_t i=0; i<export_state->index_ids.size(); i++) {- export_state->offsets.push_back(0);- }export_state->res_body = &res->body;- export_state->collection = collection.get();}} else {export_state = dynamic_cast<export_state_t*>(req->data);@@ -1105,11 +1129,34 @@ bool get_export_documents(const std::shared_ptr<http_req>& req, const std::sharestd::string().swap(res->body);while(it->Valid() && it->key().ToString().compare(0, seq_id_prefix.size(), seq_id_prefix) == 0) {+ nlohmann::json doc = nlohmann::json::parse(it->value().ToString());+ Collection::remove_flat_fields(doc);+ Collection::remove_reference_helper_fields(doc);+if(export_state->include_fields.empty() && export_state->exclude_fields.empty()) {- res->body += it->value().ToString();+ res->body += doc.dump();} else {- nlohmann::json doc = nlohmann::json::parse(it->value().ToString());- Collection::prune_doc(doc, export_state->include_fields, export_state->exclude_fields);+ if (doc.count("id") == 0 || !doc.at("id").is_string()) {+ res->set(500, "Could not find `id` field in the document: " + doc.dump());+ req->last_chunk_aggregate = true;+ res->final = true;+ stream_response(req, res);+ return false;+ }++ auto const& coll = export_state->collection;+ auto const seq_id_op = coll->doc_id_to_seq_id_with_lock(doc.at("id"));+ if (!seq_id_op.ok()) {+ res->set(seq_id_op.code(), seq_id_op.error());+ req->last_chunk_aggregate = true;+ res->final = true;+ stream_response(req, res);+ return false;+ }++ std::map<std::string, reference_filter_result_t> references = {};+ coll->prune_doc_with_lock(doc, export_state->include_fields, export_state->exclude_fields,+ references, seq_id_op.get(), export_state->ref_include_exclude_fields_vec);res->body += doc.dump();}diff --git a/src/core_api_utils.cpp b/src/core_api_utils.cppindex b043ffc2..7d440dd5 100644--- a/src/core_api_utils.cpp+++ b/src/core_api_utils.cpp@@ -49,48 +49,46 @@ Option<bool> stateful_export_docs(export_state_t* export_state, size_t batch_sizexport_state->res_body->clear();- for(size_t i = 0; i < export_state->index_ids.size(); i++) {- std::pair<size_t, uint32_t*>& size_ids = export_state->index_ids[i];- size_t ids_len = size_ids.first;- uint32_t* ids = size_ids.second;-- size_t start_index = export_state->offsets[i];- size_t batched_len = std::min(ids_len, (start_index+batch_size));-- for(size_t j = start_index; j < batched_len; j++) {- auto seq_id = ids[j];- nlohmann::json doc;- Option<bool> get_op = export_state->collection->get_document_from_store(seq_id, doc);-- if(get_op.ok()) {- if(export_state->include_fields.empty() && export_state->exclude_fields.empty()) {- export_state->res_body->append(doc.dump());- } else {- Collection::remove_flat_fields(doc);- Collection::remove_reference_helper_fields(doc);- Collection::prune_doc(doc, export_state->include_fields, export_state->exclude_fields);- export_state->res_body->append(doc.dump());- }-- export_state->res_body->append("\n");+ auto const& filter_result = export_state->filter_result;+ size_t ids_len = filter_result.count;+ uint32_t* ids = filter_result.docs;++ size_t start_index = export_state->offset;+ size_t batched_len = std::min(ids_len, (start_index+batch_size));++ for(size_t j = start_index; j < batched_len; j++) {+ uint32_t seq_id = ids[j];+ nlohmann::json doc;++ auto const& coll = export_state->collection;+ Option<bool> get_op = coll->get_document_from_store(seq_id, doc);+ Collection::remove_flat_fields(doc);+ Collection::remove_reference_helper_fields(doc);++ if(get_op.ok()) {+ if(export_state->include_fields.empty() && export_state->exclude_fields.empty()) {+ export_state->res_body->append(doc.dump());+ } else {+ coll->prune_doc_with_lock(doc, export_state->include_fields, export_state->exclude_fields,+ filter_result.coll_to_references[j], seq_id,+ export_state->ref_include_exclude_fields_vec);+ export_state->res_body->append(doc.dump());}- export_state->offsets[i]++;- batch_count++;+ export_state->res_body->append("\n");+ }- if(batch_count == batch_size) {- goto END;- }+ export_state->offset++;+ batch_count++;++ if(batch_count == batch_size) {+ goto END;}}END:- done = true;- for(size_t i=0; i<export_state->index_ids.size(); i++) {- size_t current_offset = export_state->offsets[i];- done = done && (current_offset == export_state->index_ids[i].first);- }+ done = export_state->offset == export_state->filter_result.count;if(done && !export_state->res_body->empty()) {export_state->res_body->pop_back();diff --git a/test/collection_manager_test.cpp b/test/collection_manager_test.cppindex 64861318..a9ff40ac 100644--- a/test/collection_manager_test.cpp+++ b/test/collection_manager_test.cpp@@ -1582,17 +1582,17 @@ TEST_F(CollectionManagerTest, InitializeRefIncludeExcludeFields) {std::string filter_query = "";std::vector<std::string> include_fields_vec, exclude_fields_vec;std::vector<ref_include_exclude_fields> ref_include_exclude_fields_vec;- auto initialize_op = CollectionManager::_initialize_ref_include_exclude_fields_vec(filter_query, include_fields_vec,- exclude_fields_vec,- ref_include_exclude_fields_vec);+ auto initialize_op = CollectionManager::initialize_ref_include_exclude_fields_vec(filter_query, include_fields_vec,+ exclude_fields_vec,+ ref_include_exclude_fields_vec);ASSERT_TRUE(initialize_op.ok());ASSERT_TRUE(ref_include_exclude_fields_vec.empty());filter_query = "$foo(bar:baz)";exclude_fields_vec = {"$foo(bar)"};- initialize_op = CollectionManager::_initialize_ref_include_exclude_fields_vec(filter_query, include_fields_vec,- exclude_fields_vec,- ref_include_exclude_fields_vec);+ initialize_op = CollectionManager::initialize_ref_include_exclude_fields_vec(filter_query, include_fields_vec,+ exclude_fields_vec,+ ref_include_exclude_fields_vec);ASSERT_TRUE(initialize_op.ok());ASSERT_EQ(1, ref_include_exclude_fields_vec.size());ASSERT_EQ("foo", ref_include_exclude_fields_vec[0].collection_name);@@ -1606,25 +1606,25 @@ TEST_F(CollectionManagerTest, InitializeRefIncludeExcludeFields) {filter_query = "";include_fields_vec = {"$Customers(product_price, strategy: foo) as customers"};- initialize_op = CollectionManager::_initialize_ref_include_exclude_fields_vec(filter_query, include_fields_vec,- exclude_fields_vec,- ref_include_exclude_fields_vec);+ initialize_op = CollectionManager::initialize_ref_include_exclude_fields_vec(filter_query, include_fields_vec,+ exclude_fields_vec,+ ref_include_exclude_fields_vec);ASSERT_FALSE(initialize_op.ok());ASSERT_EQ("Error parsing `$Customers(product_price, strategy: foo) as customers`: Unknown include strategy `foo`. ""Valid options are `merge`, `nest`, `nest_array`.", initialize_op.error());include_fields_vec = {"$Customers(product_price, foo: bar) as customers"};- initialize_op = CollectionManager::_initialize_ref_include_exclude_fields_vec(filter_query, include_fields_vec,- exclude_fields_vec,- ref_include_exclude_fields_vec);+ initialize_op = CollectionManager::initialize_ref_include_exclude_fields_vec(filter_query, include_fields_vec,+ exclude_fields_vec,+ ref_include_exclude_fields_vec);ASSERT_FALSE(initialize_op.ok());ASSERT_EQ("Unknown reference `include_fields` parameter: `foo`.", initialize_op.error());filter_query = "$Customers(customer_id:=customer_a && (product_price:>100 && product_price:<200))";include_fields_vec = {"$Customers(product_price, strategy: merge) as customers"};- initialize_op = CollectionManager::_initialize_ref_include_exclude_fields_vec(filter_query, include_fields_vec,- exclude_fields_vec,- ref_include_exclude_fields_vec);+ initialize_op = CollectionManager::initialize_ref_include_exclude_fields_vec(filter_query, include_fields_vec,+ exclude_fields_vec,+ ref_include_exclude_fields_vec);ASSERT_TRUE(initialize_op.ok());ASSERT_EQ(1, ref_include_exclude_fields_vec.size());ASSERT_EQ("Customers", ref_include_exclude_fields_vec[0].collection_name);@@ -1636,9 +1636,9 @@ TEST_F(CollectionManagerTest, InitializeRefIncludeExcludeFields) {filter_query = "$Customers(customer_id:=customer_a && (product_price:>100 && product_price:<200))";include_fields_vec = {"$Customers(product_price, strategy: nest_array) as customers"};- initialize_op = CollectionManager::_initialize_ref_include_exclude_fields_vec(filter_query, include_fields_vec,- exclude_fields_vec,- ref_include_exclude_fields_vec);+ initialize_op = CollectionManager::initialize_ref_include_exclude_fields_vec(filter_query, include_fields_vec,+ exclude_fields_vec,+ ref_include_exclude_fields_vec);ASSERT_TRUE(initialize_op.ok());ASSERT_EQ(1, ref_include_exclude_fields_vec.size());ASSERT_EQ("Customers", ref_include_exclude_fields_vec[0].collection_name);@@ -1650,9 +1650,9 @@ TEST_F(CollectionManagerTest, InitializeRefIncludeExcludeFields) {filter_query = "$product_variants( $inventory($retailers(location:(33.865,-118.375,100 km))))";include_fields_vec = {"$product_variants(id,$inventory(qty,sku,$retailers(id,title)))"};- initialize_op = CollectionManager::_initialize_ref_include_exclude_fields_vec(filter_query, include_fields_vec,- exclude_fields_vec,- ref_include_exclude_fields_vec);+ initialize_op = CollectionManager::initialize_ref_include_exclude_fields_vec(filter_query, include_fields_vec,+ exclude_fields_vec,+ ref_include_exclude_fields_vec);ASSERT_TRUE(initialize_op.ok());ASSERT_EQ(1, ref_include_exclude_fields_vec.size());ASSERT_EQ("product_variants", ref_include_exclude_fields_vec[0].collection_name);@@ -1675,9 +1675,9 @@ TEST_F(CollectionManagerTest, InitializeRefIncludeExcludeFields) {filter_query = "$product_variants( $inventory($retailers(location:(33.865,-118.375,100 km))))";include_fields_vec = {"$product_variants(title, $inventory(qty, strategy:merge) as inventory, strategy: nest) as variants"};- initialize_op = CollectionManager::_initialize_ref_include_exclude_fields_vec(filter_query, include_fields_vec,- exclude_fields_vec,- ref_include_exclude_fields_vec);+ initialize_op = CollectionManager::initialize_ref_include_exclude_fields_vec(filter_query, include_fields_vec,+ exclude_fields_vec,+ ref_include_exclude_fields_vec);ASSERT_TRUE(initialize_op.ok());ASSERT_EQ(1, ref_include_exclude_fields_vec.size());ASSERT_EQ("product_variants", ref_include_exclude_fields_vec[0].collection_name);@@ -1701,9 +1701,9 @@ TEST_F(CollectionManagerTest, InitializeRefIncludeExcludeFields) {filter_query = "$product_variants( $inventory(id:*) && $retailers(location:(33.865,-118.375,100 km)))";include_fields_vec = {"$product_variants(title, $inventory(qty, strategy:merge) as inventory,"" $retailers(title), strategy: merge) as variants"};- initialize_op = CollectionManager::_initialize_ref_include_exclude_fields_vec(filter_query, include_fields_vec,- exclude_fields_vec,- ref_include_exclude_fields_vec);+ initialize_op = CollectionManager::initialize_ref_include_exclude_fields_vec(filter_query, include_fields_vec,+ exclude_fields_vec,+ ref_include_exclude_fields_vec);ASSERT_TRUE(initialize_op.ok());ASSERT_EQ(1, ref_include_exclude_fields_vec.size());ASSERT_EQ("product_variants", ref_include_exclude_fields_vec[0].collection_name);@@ -1726,9 +1726,9 @@ TEST_F(CollectionManagerTest, InitializeRefIncludeExcludeFields) {filter_query = "$product_variants( $inventory(id:*) && $retailers(location:(33.865,-118.375,100 km)))";include_fields_vec = {"$product_variants(title, $inventory(qty, strategy:merge) as inventory, description,"" $retailers(title), foo, strategy: merge) as variants"};- initialize_op = CollectionManager::_initialize_ref_include_exclude_fields_vec(filter_query, include_fields_vec,- exclude_fields_vec,- ref_include_exclude_fields_vec);+ initialize_op = CollectionManager::initialize_ref_include_exclude_fields_vec(filter_query, include_fields_vec,+ exclude_fields_vec,+ ref_include_exclude_fields_vec);ASSERT_TRUE(initialize_op.ok());ASSERT_EQ(1, ref_include_exclude_fields_vec.size());ASSERT_EQ("product_variants", ref_include_exclude_fields_vec[0].collection_name);@@ -1751,9 +1751,9 @@ TEST_F(CollectionManagerTest, InitializeRefIncludeExcludeFields) {filter_query = "$Customers(customer_id:=customer_a && (product_price:>100 && product_price:<200))";include_fields_vec.clear();exclude_fields_vec = {"$Customers(product_price)"};- initialize_op = CollectionManager::_initialize_ref_include_exclude_fields_vec(filter_query, include_fields_vec,- exclude_fields_vec,- ref_include_exclude_fields_vec);+ initialize_op = CollectionManager::initialize_ref_include_exclude_fields_vec(filter_query, include_fields_vec,+ exclude_fields_vec,+ ref_include_exclude_fields_vec);ASSERT_TRUE(initialize_op.ok());ASSERT_EQ(1, ref_include_exclude_fields_vec.size());ASSERT_EQ("Customers", ref_include_exclude_fields_vec[0].collection_name);@@ -1767,9 +1767,9 @@ TEST_F(CollectionManagerTest, InitializeRefIncludeExcludeFields) {filter_query = "$product_variants( $inventory(id:*) && $retailers(location:(33.865,-118.375,100 km)))";include_fields_vec.clear();exclude_fields_vec = {"$product_variants(title, $inventory(qty), description, $retailers(title), foo)"};- initialize_op = CollectionManager::_initialize_ref_include_exclude_fields_vec(filter_query, include_fields_vec,- exclude_fields_vec,- ref_include_exclude_fields_vec);+ initialize_op = CollectionManager::initialize_ref_include_exclude_fields_vec(filter_query, include_fields_vec,+ exclude_fields_vec,+ ref_include_exclude_fields_vec);ASSERT_TRUE(initialize_op.ok());ASSERT_EQ(1, ref_include_exclude_fields_vec.size());ASSERT_EQ("product_variants", ref_include_exclude_fields_vec[0].collection_name);@@ -1795,9 +1795,9 @@ TEST_F(CollectionManagerTest, InitializeRefIncludeExcludeFields) {filter_query = "$product_variants( $inventory($retailers(location:(33.865,-118.375,100 km))))";include_fields_vec = {"$product_variants(title, $inventory(qty, strategy:merge) as inventory, strategy: nest) as variants"};exclude_fields_vec = {"$product_variants(title, $inventory(qty, $retailers(title)))"};- initialize_op = CollectionManager::_initialize_ref_include_exclude_fields_vec(filter_query, include_fields_vec,- exclude_fields_vec,- ref_include_exclude_fields_vec);+ initialize_op = CollectionManager::initialize_ref_include_exclude_fields_vec(filter_query, include_fields_vec,+ exclude_fields_vec,+ ref_include_exclude_fields_vec);ASSERT_TRUE(initialize_op.ok());ASSERT_EQ(1, ref_include_exclude_fields_vec.size());ASSERT_EQ("product_variants", ref_include_exclude_fields_vec[0].collection_name);diff --git a/test/core_api_utils_test.cpp b/test/core_api_utils_test.cppindex f6624313..f24ffbcf 100644--- a/test/core_api_utils_test.cpp+++ b/test/core_api_utils_test.cpp@@ -895,12 +895,7 @@ TEST_F(CoreAPIUtilsTest, ExportWithFilter) {export_state_t export_state;filter_result_t filter_result;- coll1->get_filter_ids("points:>=0", filter_result);- export_state.index_ids.emplace_back(filter_result.count, filter_result.docs);- filter_result.docs = nullptr;- for(size_t i=0; i<export_state.index_ids.size(); i++) {- export_state.offsets.push_back(0);- }+ coll1->get_filter_ids("points:>=0", export_state.filter_result);export_state.collection = coll1;export_state.res_body = &res_body;@@ -915,6 +910,125 @@ TEST_F(CoreAPIUtilsTest, ExportWithFilter) {ASSERT_EQ('}', export_state.res_body->back());}+TEST_F(CoreAPIUtilsTest, ExportWithJoin) {+ auto schema_json =+ R"({+ "name": "Products",+ "fields": [+ {"name": "product_id", "type": "string"},+ {"name": "product_name", "type": "string"},+ {"name": "product_description", "type": "string"},+ {"name": "rating", "type": "int32"}+ ]+ })"_json;+ std::vector<nlohmann::json> documents = {+ R"({+ "product_id": "product_a",+ "product_name": "shampoo",+ "product_description": "Our new moisturizing shampoo is perfect for those with dry or damaged hair.",+ "rating": "2"+ })"_json,+ R"({+ "product_id": "product_b",+ "product_name": "soap",+ "product_description": "Introducing our all-natural, organic soap bar made with essential oils and botanical ingredients.",+ "rating": "4"+ })"_json+ };+ auto collection_create_op = collectionManager.create_collection(schema_json);+ ASSERT_TRUE(collection_create_op.ok());+ for (auto const &json: documents) {+ auto add_op = collection_create_op.get()->add(json.dump());+ if (!add_op.ok()) {+ LOG(INFO) << add_op.error();+ }+ ASSERT_TRUE(add_op.ok());+ }++ schema_json =+ R"({+ "name": "Customers",+ "fields": [+ {"name": "customer_id", "type": "string"},+ {"name": "customer_name", "type": "string"},+ {"name": "product_price", "type": "float"},+ {"name": "product_id", "type": "string", "reference": "Products.product_id"}+ ]+ })"_json;+ documents = {+ R"({+ "customer_id": "customer_a",+ "customer_name": "Joe",+ "product_price": 143,+ "product_id": "product_a"+ })"_json,+ R"({+ "customer_id": "customer_a",+ "customer_name": "Joe",+ "product_price": 73.5,+ "product_id": "product_b"+ })"_json,+ R"({+ "customer_id": "customer_b",+ "customer_name": "Dan",+ "product_price": 75,+ "product_id": "product_a"+ })"_json,+ R"({+ "customer_id": "customer_b",+ "customer_name": "Dan",+ "product_price": 140,+ "product_id": "product_b"+ })"_json+ };+ collection_create_op = collectionManager.create_collection(schema_json);+ ASSERT_TRUE(collection_create_op.ok());+ for (auto const &json: documents) {+ auto add_op = collection_create_op.get()->add(json.dump());+ if (!add_op.ok()) {+ LOG(INFO) << add_op.error();+ }+ ASSERT_TRUE(add_op.ok());+ }++ schema_json =+ R"({+ "name": "Dummy",+ "fields": [+ {"name": "dummy_id", "type": "string"}+ ]+ })"_json;+ collection_create_op = collectionManager.create_collection(schema_json);+ ASSERT_TRUE(collection_create_op.ok());++ bool done;+ std::string res_body;++ export_state_t export_state;+ auto coll1 = collectionManager.get_collection_unsafe("Products");+ coll1->get_filter_ids("$Customers(customer_id:customer_a)", export_state.filter_result);+ export_state.collection = coll1;+ export_state.res_body = &res_body;+ export_state.include_fields.insert("product_name");+ export_state.ref_include_exclude_fields_vec.emplace_back(ref_include_exclude_fields{"Customers", {"product_price"}, "",+ "", ref_include::nest});++ stateful_export_docs(&export_state, 1, done);+ ASSERT_FALSE(done);+ ASSERT_EQ('\n', export_state.res_body->back());+ auto doc = nlohmann::json::parse(export_state.res_body->c_str());+ ASSERT_EQ("shampoo", doc["product_name"]);+ ASSERT_EQ(143, doc["Customers"]["product_price"]);++ // should not have trailing newline character for the last line+ stateful_export_docs(&export_state, 1, done);+ ASSERT_TRUE(done);+ ASSERT_EQ('}', export_state.res_body->back());+ doc = nlohmann::json::parse(export_state.res_body->c_str());+ ASSERT_EQ("soap", doc["product_name"]);+ ASSERT_EQ(73.5, doc["Customers"]["product_price"]);+}+TEST_F(CoreAPIUtilsTest, TestParseAPIKeyIPFromMetadata) {// format <length of api key>:<api key><ip address>std::string valid_metadata = "4:abcd127.0.0.1";