Open Source alternative to Algolia + Pinecone and an Easier-to-Use alternative to ElasticSearch ⚡ 🔍 ✨ Fast, typo tolerant, in-memory fuzzy Search Engine for building delightful search experiences
C++ algoliadatastoreelasticsearchenterprise-searchfacetingfull-text-searchfuzzy-searchgeosearchin-memoryinstantsearchmerchandisingpineconesearchsearch-enginesemantic-searchsimilarity-searchsite-searchsynonymstypo-tolerancevector-search
Async reference field. (#1835)
Harpreet Sangarc3a4d2d4f479
diff --git a/include/collection.h b/include/collection.hindex 8ac8bc55..31e08f11 100644--- a/include/collection.h+++ b/include/collection.h@@ -20,6 +20,7 @@#include "tokenizer.h"#include "synonym_index.h"#include "vq_model_manager.h"+#include "join.h"struct doc_seq_id_t {uint32_t seq_id;@@ -39,17 +40,6 @@ struct highlight_field_t {}};-struct reference_pair {- std::string collection;- std::string field;-- reference_pair(std::string collection, std::string field) : collection(std::move(collection)), field(std::move(field)) {}-- bool operator < (const reference_pair& pair) const {- return collection < pair.collection;- }-};-class Collection {private:@@ -140,14 +130,17 @@ private:SynonymIndex* synonym_index;- /// "field name" -> reference_pair(referenced_collection_name, referenced_field_name)- spp::sparse_hash_map<std::string, reference_pair> reference_fields;+ /// "field name" -> reference_info(referenced_collection_name, referenced_field_name, is_async)+ spp::sparse_hash_map<std::string, reference_info_t> reference_fields;/// Contains the info where the current collection is referenced./// Useful to perform operations such as cascading delete./// collection_name -> field_namespp::sparse_hash_map<std::string, std::string> referenced_in;+ /// "field name" -> List of <collection, field> pairs where this collection is referenced and is marked as `async`.+ spp::sparse_hash_map<std::string, std::vector<reference_pair_t>> async_referenced_ins;+/// Reference helper fields that are part of an object. The reference doc of these fields will be included in the/// object rather than in the document.tsl::htrie_set<char> object_reference_helper_fields;@@ -220,7 +213,7 @@ private:bool is_update,std::vector<field>& new_fields,bool enable_nested_fields,- const spp::sparse_hash_map<std::string, reference_pair>& reference_fields,+ const spp::sparse_hash_map<std::string, reference_info_t>& reference_fields,tsl::htrie_set<char>& object_reference_helper_fields);static bool facet_count_compare(const facet_count_t& a, const facet_count_t& b) {@@ -331,13 +324,6 @@ private:Option<std::string> get_referenced_in_field(const std::string& collection_name) const;- Option<bool> get_related_ids(const std::string& ref_field_name, const uint32_t& seq_id,- std::vector<uint32_t>& result) const;-- Option<bool> get_object_array_related_id(const std::string& ref_field_name,- const uint32_t& seq_id, const uint32_t& object_index,- uint32_t& result) const;-void remove_embedding_field(const std::string& field_name);Option<bool> parse_and_validate_vector_query(const std::string& vector_query_str,@@ -375,6 +361,9 @@ public:static constexpr const char* COLLECTION_METADATA = "metadata";+ /// Value used when async_reference is true and a reference doc is not found.+ static constexpr int64_t reference_helper_sentinel_value = UINT32_MAX;+// methodsCollection() = delete;@@ -386,7 +375,9 @@ public:const std::vector<std::string>& symbols_to_index, const std::vector<std::string>& token_separators,const bool enable_nested_fields, std::shared_ptr<VQModel> vq_model = nullptr,spp::sparse_hash_map<std::string, std::string> referenced_in = spp::sparse_hash_map<std::string, std::string>(),- const nlohmann::json& metadata = {});+ const nlohmann::json& metadata = {},+ spp::sparse_hash_map<std::string, std::vector<reference_pair_t>> async_referenced_ins =+ spp::sparse_hash_map<std::string, std::vector<reference_pair_t>>());~Collection();@@ -434,11 +425,6 @@ public:void update_metadata(const nlohmann::json& meta);- static Option<bool> add_reference_helper_fields(nlohmann::json& document, const tsl::htrie_map<char, field>& schema,- const spp::sparse_hash_map<std::string, reference_pair>& reference_fields,- tsl::htrie_set<char>& object_reference_helper_fields,- const bool& is_update);-Option<doc_seq_id_t> to_doc(const std::string& json_str, nlohmann::json& document,const index_operation_t& operation,const DIRTY_VALUES dirty_values,@@ -458,18 +444,6 @@ public:static void remove_reference_helper_fields(nlohmann::json& document);- static Option<bool> prune_ref_doc(nlohmann::json& doc,- const reference_filter_result_t& references,- const tsl::htrie_set<char>& ref_include_fields_full,- const tsl::htrie_set<char>& ref_exclude_fields_full,- const bool& is_reference_array,- const ref_include_exclude_fields& ref_include_exclude);-- static Option<bool> include_references(nlohmann::json& doc, const uint32_t& seq_id, Collection *const collection,- const std::map<std::string, reference_filter_result_t>& reference_filter_results,- const std::vector<ref_include_exclude_fields>& ref_include_exclude_fields_vec,- const nlohmann::json& original_doc);-Option<bool> prune_doc_with_lock(nlohmann::json& doc, const tsl::htrie_set<char>& include_names,const tsl::htrie_set<char>& exclude_names,const std::map<std::string, reference_filter_result_t>& reference_filter_results = {},@@ -617,7 +591,7 @@ public:Option<nlohmann::json> get(const std::string & id) const;- void cascade_remove_docs(const std::string& ref_helper_field_name, const uint32_t& ref_seq_id,+ void cascade_remove_docs(const std::string& field_name, const uint32_t& ref_seq_id,const nlohmann::json& ref_doc, bool remove_from_store = true);Option<std::string> remove(const std::string & id, bool remove_from_store = true);@@ -666,7 +640,9 @@ public:SynonymIndex* get_synonym_index();- spp::sparse_hash_map<std::string, reference_pair> get_reference_fields();+ spp::sparse_hash_map<std::string, reference_info_t> get_reference_fields();++ spp::sparse_hash_map<std::string, std::vector<reference_pair_t>> get_async_referenced_ins();// highlight ops@@ -710,17 +686,20 @@ public:bool is_referenced_in(const std::string& collection_name) const;- void add_referenced_in(const reference_pair& pair);+ void add_referenced_ins(const std::set<reference_info_t>& ref_infos);- void add_referenced_ins(const std::set<reference_pair>& pairs);-- void add_referenced_in(const std::string& collection_name, const std::string& field_name);+ void add_referenced_in(const std::string& collection_name, const std::string& field_name,+ const bool& is_async, const std::string& referenced_field_name);Option<std::string> get_referenced_in_field_with_lock(const std::string& collection_name) const;Option<bool> get_related_ids_with_lock(const std::string& field_name, const uint32_t& seq_id,std::vector<uint32_t>& result) const;+ Option<bool> update_async_references_with_lock(const std::string& ref_coll_name, const std::string& filter,+ const std::set<std::string>& filter_values,+ const uint32_t ref_seq_id, const std::string& field_name);+Option<uint32_t> get_sort_index_value_with_lock(const std::string& field_name, const uint32_t& seq_id) const;static void hide_credential(nlohmann::json& json, const std::string& credential_name);@@ -732,6 +711,13 @@ public:void expand_search_query(const std::string& raw_query, size_t offset, size_t total, const search_args* search_params,const std::vector<std::vector<KV*>>& result_group_kvs,const std::vector<std::string>& raw_search_fields, std::string& first_q) const;++ Option<bool> get_object_array_related_id(const std::string& ref_field_name,+ const uint32_t& seq_id, const uint32_t& object_index,+ uint32_t& result) const;++ Option<bool> get_related_ids(const std::string& ref_field_name, const uint32_t& seq_id,+ std::vector<uint32_t>& result) const;};template<class T>diff --git a/include/collection_manager.h b/include/collection_manager.hindex 4d988dbf..a5d3d547 100644--- a/include/collection_manager.h+++ b/include/collection_manager.h@@ -79,7 +79,7 @@ private:std::atomic<bool>* quit;// All the references to a particular collection are stored until it is created.- std::map<std::string, std::set<reference_pair>> referenced_in_backlog;+ std::map<std::string, std::set<reference_info_t>> referenced_in_backlog;CollectionManager();@@ -125,13 +125,15 @@ public:const uint32_t collection_next_seq_id,Store* store,float max_memory_ratio,- spp::sparse_hash_map<std::string, std::string>& referenced_in);+ spp::sparse_hash_map<std::string, std::string>& referenced_in,+ spp::sparse_hash_map<std::string, std::vector<reference_pair_t>>& async_referenced_ins);static Option<bool> load_collection(const nlohmann::json& collection_meta,const size_t batch_size,const StoreStatus& next_coll_id_status,const std::atomic<bool>& quit,- spp::sparse_hash_map<std::string, std::string>& referenced_in);+ spp::sparse_hash_map<std::string, std::string>& referenced_in,+ spp::sparse_hash_map<std::string, std::vector<reference_pair_t>>& async_referenced_ins);Option<Collection*> clone_collection(const std::string& existing_name, const nlohmann::json& req_json);@@ -234,14 +236,15 @@ public:std::vector<std::string>& exclude_fields_vec,std::vector<ref_include_exclude_fields>& ref_include_exclude_fields_vec);- void add_referenced_in_backlog(const std::string& collection_name, reference_pair&& pair);+ void add_referenced_in_backlog(const std::string& collection_name, reference_info_t&& ref_info);- std::map<std::string, std::set<reference_pair>> _get_referenced_in_backlog() const;+ std::map<std::string, std::set<reference_info_t>> _get_referenced_in_backlog() const;void process_embedding_field_delete(const std::string& model_name);static void _populate_referenced_ins(const std::string& collection_meta_json,- std::map<std::string, spp::sparse_hash_map<std::string, std::string>>& referenced_ins);+ std::map<std::string, spp::sparse_hash_map<std::string, std::string>>& referenced_ins,+ std::map<std::string, spp::sparse_hash_map<std::string, std::vector<reference_pair_t>>>& async_referenced_ins);std::unordered_set<std::string> get_collection_references(const std::string& coll_name);diff --git a/include/field.h b/include/field.hindex 1ffe7a86..24588f33 100644--- a/include/field.h+++ b/include/field.h@@ -60,6 +60,7 @@ namespace fields {static const std::string num_dim = "num_dim";static const std::string vec_dist = "vec_dist";static const std::string reference = "reference";+ static const std::string async_reference = "async_reference";static const std::string embed = "embed";static const std::string from = "from";static const std::string model_name = "model_name";@@ -86,6 +87,14 @@ enum vector_distance_type_t {cosine};+struct reference_pair_t {+ std::string collection;+ std::string field;++ reference_pair_t(std::string collection, std::string field) : collection(std::move(collection)),+ field(std::move(field)) {}+};+struct field {std::string name;std::string type;@@ -112,6 +121,7 @@ struct field {static constexpr int VAL_UNKNOWN = 2;std::string reference; // Foo.bar (reference to bar field in Foo collection).+ bool is_async_reference = false;bool range_index;@@ -127,10 +137,13 @@ struct field {field(const std::string &name, const std::string &type, const bool facet, const bool optional = false,bool index = true, std::string locale = "", int sort = -1, int infix = -1, bool nested = false,int nested_array = 0, size_t num_dim = 0, vector_distance_type_t vec_dist = cosine,- std::string reference = "", const nlohmann::json& embed = nlohmann::json(), const bool range_index = false, const bool store = true, const bool stem = false, const nlohmann::json hnsw_params = nlohmann::json()) :+ std::string reference = "", const nlohmann::json& embed = nlohmann::json(), const bool range_index = false,+ const bool store = true, const bool stem = false, const nlohmann::json hnsw_params = nlohmann::json(),+ const bool async_reference = false) :name(name), type(type), facet(facet), optional(optional), index(index), locale(locale),nested(nested), nested_array(nested_array), num_dim(num_dim), vec_dist(vec_dist), reference(reference),- embed(embed), range_index(range_index), store(store), stem(stem), hnsw_params(hnsw_params) {+ embed(embed), range_index(range_index), store(store), stem(stem), hnsw_params(hnsw_params),+ is_async_reference(async_reference) {set_computed_defaults(sort, infix);diff --git a/include/index.h b/include/index.hindex c18a0127..522e0e7d 100644--- a/include/index.h+++ b/include/index.h@@ -569,6 +569,10 @@ private:const index_record* record,const std::vector<embedding_res_t>& embedding_results,size_t& count, const field& the_field);++ void update_async_references(const std::string& collection_name, const field& afield,+ std::vector<index_record>& iter_batch,+ const std::vector<reference_pair_t>& async_referenced_ins = {});public:// for limiting number of results on multiple candidates / query rewritesenum {TYPO_TOKENS_THRESHOLD = 1};@@ -740,9 +744,14 @@ public:const size_t remote_embedding_timeout_ms = 60000,const size_t remote_embedding_num_tries = 2, const bool generate_embeddings = true,const bool use_addition_fields = false,- const tsl::htrie_map<char, field>& addition_fields = tsl::htrie_map<char, field>());+ const tsl::htrie_map<char, field>& addition_fields = tsl::htrie_map<char, field>(),+ const std::string& collection_name = "",+ const spp::sparse_hash_map<std::string, std::vector<reference_pair_t>>& async_referenced_ins =+ spp::sparse_hash_map<std::string, std::vector<reference_pair_t>>());- void index_field_in_memory(const field& afield, std::vector<index_record>& iter_batch);+ void index_field_in_memory(const std::string& collection_name, const field& afield,+ std::vector<index_record>& iter_batch,+ const std::vector<reference_pair_t>& async_referenced_ins = {});template<class T>void iterate_and_index_numerical_field(std::vector<index_record>& iter_batch, const field& afield, T func);@@ -762,9 +771,9 @@ public:Option<bool> do_reference_filtering_with_lock(filter_node_t* const filter_tree_root,filter_result_t& filter_result,const std::string& ref_collection_name,- const std::string& reference_helper_field_name) const;+ const std::string& field_name) const;- Option<filter_result_t> do_filtering_with_reference_ids(const std::string& reference_helper_field_name,+ Option<filter_result_t> do_filtering_with_reference_ids(const std::string& field_name,const std::string& ref_collection_name,filter_result_t&& ref_filter_result) const;@@ -1037,7 +1046,7 @@ public:std::vector<uint32_t>& outside_seq_ids);Option<bool> get_related_ids(const std::string& collection_name,- const std::string& reference_helper_field_name,+ const std::string& field_name,const uint32_t& seq_id, std::vector<uint32_t>& result) const;Option<bool> get_object_array_related_id(const std::string& collection_name,diff --git a/include/join.h b/include/join.hnew file mode 100644index 00000000..5805a282--- /dev/null+++ b/include/join.h@@ -0,0 +1,46 @@+#pragma once++#include "option.h"+#include "json.hpp"+#include "tsl/htrie_map.h"+#include "field.h"+#include "tsl/htrie_set.h"+#include "filter_result_iterator.h"++struct reference_info_t {+ std::string collection;+ std::string field;+ bool is_async;++ std::string referenced_field_name;++ reference_info_t(std::string collection, std::string field, bool is_async, std::string referenced_field_name = "") :+ collection(std::move(collection)), field(std::move(field)), is_async(is_async),+ referenced_field_name(std::move(referenced_field_name)) {}++ bool operator < (const reference_info_t& pair) const {+ return collection < pair.collection;+ }+};++class Join {+public:++ static Option<bool> add_reference_helper_fields(nlohmann::json& document,+ const tsl::htrie_map<char, field>& schema,+ const spp::sparse_hash_map<std::string, reference_info_t>& reference_fields,+ tsl::htrie_set<char>& object_reference_helper_fields,+ const bool& is_update);++ static Option<bool> prune_ref_doc(nlohmann::json& doc,+ const reference_filter_result_t& references,+ const tsl::htrie_set<char>& ref_include_fields_full,+ const tsl::htrie_set<char>& ref_exclude_fields_full,+ const bool& is_reference_array,+ const ref_include_exclude_fields& ref_include_exclude);++ static Option<bool> include_references(nlohmann::json& doc, const uint32_t& seq_id, Collection *const collection,+ const std::map<std::string, reference_filter_result_t>& reference_filter_results,+ const std::vector<ref_include_exclude_fields>& ref_include_exclude_fields_vec,+ const nlohmann::json& original_doc);+};\ No newline at end of filediff --git a/src/collection.cpp b/src/collection.cppindex 4a7a134f..2e07d0d2 100644--- a/src/collection.cpp+++ b/src/collection.cpp@@ -24,6 +24,7 @@#include "conversation_manager.h"#include "conversation_model_manager.h"#include "field.h"+#include "join.h"const std::string override_t::MATCH_EXACT = "exact";const std::string override_t::MATCH_CONTAINS = "contains";@@ -54,7 +55,8 @@ Collection::Collection(const std::string& name, const uint32_t collection_id, coconst std::vector<std::string>& token_separators,const bool enable_nested_fields, std::shared_ptr<VQModel> vq_model,spp::sparse_hash_map<std::string, std::string> referenced_in,- const nlohmann::json& metadata) :+ const nlohmann::json& metadata,+ spp::sparse_hash_map<std::string, std::vector<reference_pair_t>> async_referenced_ins) :name(name), collection_id(collection_id), created_at(created_at),next_seq_id(next_seq_id), store(store),fields(fields), default_sorting_field(default_sorting_field), enable_nested_fields(enable_nested_fields),@@ -63,7 +65,7 @@ Collection::Collection(const std::string& name, const uint32_t collection_id, cosymbols_to_index(to_char_array(symbols_to_index)), token_separators(to_char_array(token_separators)),index(init_index()), vq_model(vq_model),referenced_in(std::move(referenced_in)),- metadata(metadata) {+ metadata(metadata), async_referenced_ins(std::move(async_referenced_ins)) {if (vq_model) {vq_model->inc_collection_ref_count();@@ -92,318 +94,141 @@ uint32_t Collection::get_next_seq_id() {return next_seq_id++;}-Option<bool> single_value_filter_query(nlohmann::json& document, const std::string& field_name,- const std::string& ref_field_type, std::string& filter_query) {- auto const& value = document[field_name];-- if (value.is_null()) {- return Option<bool>(422, "Field has `null` value.");- }-- if (value.is_string() && ref_field_type == field_types::STRING) {- filter_query[filter_query.size() - 1] = '=';- filter_query += (" " + value.get<std::string>());- } else if (value.is_number_integer() && (ref_field_type == field_types::INT64 ||- (ref_field_type == field_types::INT32 &&- StringUtils::is_int32_t(std::to_string(value.get<int64_t>()))))) {- filter_query += std::to_string(value.get<int64_t>());- } else {- return Option<bool>(400, "Field `" + field_name + "` must have `" + ref_field_type + "` value.");- }+inline std::string get_field_value(const nlohmann::json& doc, const std::string& field_name) {+ return doc[field_name].is_number_integer() ?+ std::to_string(doc[field_name].get<int64_t>()) :+ doc[field_name].is_string() ?+ doc[field_name].get<std::string>() :+ doc[field_name].dump();+}- return Option<bool>(true);+inline std::string get_array_field_value(const nlohmann::json& doc, const std::string& field_name, const size_t& index) {+ return doc[field_name][index].is_number_integer() ?+ std::to_string(doc[field_name][index].get<int64_t>()) :+ doc[field_name][index].is_string() ?+ doc[field_name][index].get<std::string>() :+ doc[field_name][index].dump();}-Option<bool> Collection::add_reference_helper_fields(nlohmann::json& document, const tsl::htrie_map<char, field>& schema,- const spp::sparse_hash_map<std::string, reference_pair>& reference_fields,- tsl::htrie_set<char>& object_reference_helper_fields,- const bool& is_update) {- tsl::htrie_set<char> flat_fields;- if (!reference_fields.empty() && document.contains(".flat")) {- for (const auto &item: document[".flat"].get<std::vector<std::string>>()) {- flat_fields.insert(item);- }- }-- // Add reference helper fields in the document.- for (auto const& pair: reference_fields) {- auto field_name = pair.first;- auto const reference_helper_field = field_name + fields::REFERENCE_HELPER_FIELD_SUFFIX;-- auto const& field = schema.at(field_name);- auto const& optional = field.optional;- // Strict checking for presence of non-optional reference field during indexing operation.- auto is_required = !is_update && !optional;- if (is_required && document.count(field_name) != 1) {- return Option<bool>(400, "Missing the required reference field `" + field_name- + "` in the document.");- } else if (document.count(field_name) != 1) {- if (is_update) {- document[fields::reference_helper_fields] += reference_helper_field;- }- continue;- }+Option<bool> Collection::update_async_references_with_lock(const std::string& ref_coll_name, const std::string& filter,+ const std::set<std::string>& filter_values,+ const uint32_t ref_seq_id, const std::string& field_name) {+ // Update reference helper field of the docs matching the filter.+ filter_result_t filter_result;+ get_filter_ids(filter, filter_result);- auto reference_pair = pair.second;- auto reference_collection_name = reference_pair.collection;- auto reference_field_name = reference_pair.field;- auto& cm = CollectionManager::get_instance();- auto ref_collection = cm.get_collection(reference_collection_name);- if (ref_collection == nullptr) {- return Option<bool>(400, "Referenced collection `" + reference_collection_name- + "` not found.");- }+ if (filter_result.count == 0) {+ return Option<bool>(true);+ }- bool is_object_reference_field = flat_fields.count(field_name) != 0;- std::string object_key;- bool is_object_array = false;- if (is_object_reference_field) {- object_reference_helper_fields.insert(reference_helper_field);+ field field;+ {+ std::shared_lock lock(mutex);- std::vector<std::string> tokens;- StringUtils::split(field_name, tokens, ".");- if (schema.count(tokens[0]) == 0) {- return Option<bool>(400, "Could not find `" + tokens[0] + "` object/object[] field in the schema.");- }- object_key = tokens[0];- is_object_array = schema.at(object_key).is_array();+ auto it = search_schema.find(field_name);+ if (it == search_schema.end()) {+ return Option<bool>(400, "Could not find field `" + field_name + "` in the schema.");}+ field = it.value();+ }- if (reference_field_name == "id") {- auto id_field_type_error_op = Option<bool>(400, "Field `" + field_name + "` must have string value.");- if (is_object_array) {- if (!document[field_name].is_array()) {- return Option<bool>(400, "Expected `" + field_name + "` to be an array.");- }-- document[reference_helper_field] = nlohmann::json::array();- document[fields::reference_helper_fields] += reference_helper_field;-- std::vector<std::string> keys;- StringUtils::split(field_name, keys, ".");- auto const& object_array = document[keys[0]];-- for (uint32_t i = 0; i < object_array.size(); i++) {- if (optional && object_array[i].count(keys[1]) == 0) {- continue;- } else if (object_array[i].count(keys[1]) == 0) {- return Option<bool>(400, "Object at index `" + std::to_string(i) + "` is missing `" + field_name + "`.");- } else if (!object_array[i].at(keys[1]).is_string()) {- return id_field_type_error_op;- }-- auto id = object_array[i].at(keys[1]).get<std::string>();- auto ref_doc_id_op = ref_collection->doc_id_to_seq_id_with_lock(id);- if (!ref_doc_id_op.ok()) {- return Option<bool>(400, "Referenced document having `id: " + id +- "` not found in the collection `" +- reference_collection_name + "`." );- }-- // Adding the index of the object along with referenced doc id to account for the scenario where a- // reference field of an object array might be optional and missing.- document[reference_helper_field] += nlohmann::json::array({i, ref_doc_id_op.get()});- }- } else if (document[field_name].is_array()) {- document[reference_helper_field] = nlohmann::json::array();- document[fields::reference_helper_fields] += reference_helper_field;-- for (const auto &item: document[field_name].items()) {- if (optional && item.value().is_null()) {- continue;- } else if (!item.value().is_string()) {- return id_field_type_error_op;- }-- auto id = item.value().get<std::string>();- auto ref_doc_id_op = ref_collection->doc_id_to_seq_id_with_lock(id);- if (!ref_doc_id_op.ok()) {- return Option<bool>(400, "Referenced document having `id: " + id +- "` not found in the collection `" +- reference_collection_name + "`." );- }+ std::vector<std::string> buffer;+ buffer.reserve(filter_result.count);- document[reference_helper_field] += ref_doc_id_op.get();- }- } else if (document[field_name].is_string()) {- auto id = document[field_name].get<std::string>();- auto ref_doc_id_op = ref_collection->doc_id_to_seq_id_with_lock(id);- if (!ref_doc_id_op.ok()) {- return Option<bool>(400, "Referenced document having `id: " + id +- "` not found in the collection `" +- reference_collection_name + "`." );- }+ for (uint32_t i = 0; i < filter_result.count; i++) {+ auto const& seq_id = filter_result.docs[i];- document[reference_helper_field] = ref_doc_id_op.get();- document[fields::reference_helper_fields] += reference_helper_field;- } else if (optional && document[field_name].is_null()) {- // Reference helper field should also be removed along with reference field.- if (is_update) {- document[reference_helper_field] = nullptr;- }+ nlohmann::json existing_document;+ auto get_doc_op = get_document_from_store(get_seq_id_key(seq_id), existing_document);+ if (!get_doc_op.ok()) {+ if (get_doc_op.code() == 404) {+ LOG(ERROR) << "`" << name << "` collection: Sequence ID `" << seq_id << "` exists, but document is missing.";continue;- } else {- return id_field_type_error_op;- }-- continue;- }-- if (ref_collection->get_schema().count(reference_field_name) == 0) {- return Option<bool>(400, "Referenced field `" + reference_field_name +- "` not found in the collection `" + reference_collection_name + "`.");- }-- auto const ref_field = ref_collection->get_schema().at(reference_field_name);- if (!ref_field.index) {- return Option<bool>(400, "Referenced field `" + reference_field_name +- "` in the collection `" + reference_collection_name + "` must be indexed.");- }-- // Create filter query from the value(s) in the reference field and get the reference doc id(s).- std::string filter_query = reference_field_name + ": ";- std::string ref_field_type = ref_field.is_string() ? field_types::STRING :- ref_field.is_int32() ? field_types::INT32 :- ref_field.is_int64() ? field_types::INT64 : field_types::NIL;-- if (ref_field_type == field_types::NIL) {- return Option<bool>(400, "Cannot add a reference to `" + reference_collection_name + "." + reference_field_name +- "` of type `" + ref_field.type + "`.");- }-- if (is_object_array) {- if (!document[field_name].is_array()) {- return Option<bool>(400, "Expected `" + field_name + "` to be an array.");}- document[reference_helper_field] = nlohmann::json::array();- document[fields::reference_helper_fields] += reference_helper_field;- nlohmann::json temp_doc; // To store singular values of `field_name` field.-- std::vector<std::string> keys;- StringUtils::split(field_name, keys, ".");- auto const& object_array = document[keys[0]];-- for (uint32_t i = 0; i < object_array.size(); i++) {- if (optional && object_array[i].count(keys[1]) == 0) {- continue;- } else if (object_array[i].count(keys[1]) == 0) {- return Option<bool>(400, "Object at index `" + std::to_string(i) + "` is missing `" + field_name + "`.");- }-- temp_doc[field_name] = object_array[i].at(keys[1]);- auto single_value_filter_query_op = single_value_filter_query(temp_doc, field_name, ref_field_type,- filter_query);- if (!single_value_filter_query_op.ok()) {- if (single_value_filter_query_op.code() == 422) {- continue;- }- return single_value_filter_query_op;- }-- filter_result_t filter_result;- auto filter_ids_op = ref_collection->get_filter_ids(filter_query, filter_result);- if (!filter_ids_op.ok()) {- return filter_ids_op;- }-- if (filter_result.count != 1) {- // Constraints similar to foreign key apply here. The reference match must be unique and not null.- return Option<bool>(400, filter_result.count < 1 ?- "Reference document having `" + filter_query + "` not found in the collection `"- + reference_collection_name + "`." :- "Multiple documents having `" + filter_query + "` found in the collection `" +- reference_collection_name + "`.");- }-- // Adding the index of the object along with referenced doc id to account for the scenario where a- // reference field of an object array might be optional and missing.- document[reference_helper_field] += nlohmann::json::array({i, filter_result.docs[0]});- filter_query = reference_field_name + ": ";- }+ LOG(ERROR) << "`" << name << "` collection: " << get_doc_op.error();continue;}-- if (document[field_name].is_array()) {- if (ref_field_type == field_types::STRING) {- filter_query[filter_query.size() - 1] = '=';- filter_query += " [";- } else {- filter_query += "[";- }- bool filter_values_added = false;- for (const auto &item: document[field_name].items()) {- auto const& item_value = item.value();- if (item_value.is_string() && ref_field_type == field_types::STRING) {- filter_query += item_value.get<std::string>();- filter_values_added = true;- } else if (item_value.is_number_integer() && (ref_field_type == field_types::INT64 ||- (ref_field_type == field_types::INT32 &&- StringUtils::is_int32_t(std::to_string(item_value.get<int64_t>()))))) {- filter_query += std::to_string(item_value.get<int64_t>());- filter_values_added = true;- } else if (optional && item_value.is_null()) {+ auto const id = existing_document["id"].get<std::string>();+ auto const reference_helper_field_name = field_name + fields::REFERENCE_HELPER_FIELD_SUFFIX;++ if (field.is_singular()) {+ // Referenced value must be unique.+ if (existing_document.contains(reference_helper_field_name) &&+ existing_document[reference_helper_field_name].is_number_integer()) {+ const int64_t existing_ref_seq_id = existing_document[reference_helper_field_name].get<int64_t>();+ if (existing_ref_seq_id != Collection::reference_helper_sentinel_value &&+ existing_ref_seq_id != ref_seq_id) {+ return Option<bool>(400, "Document `id: " + id + "` already has a reference to document `" +=+ std::to_string(existing_ref_seq_id) + "` of `" += ref_coll_name ++ "` collection, having reference value `" +=+ get_field_value(existing_document, field_name) + "`.");+ } else if (existing_ref_seq_id == ref_seq_id) {continue;- } else {- return Option<bool>(400, "Field `" + field_name + "` must only have `" + ref_field_type + "` values.");}- filter_query += ",";}- if (!filter_values_added) {- document[reference_helper_field] = nlohmann::json::array();- document[fields::reference_helper_fields] += reference_helper_field;- continue;- }- filter_query[filter_query.size() - 1] = ']';- } else if (field.is_array() && document[field_name].is_null()) {- document[reference_helper_field] = nlohmann::json::array();- document[fields::reference_helper_fields] += reference_helper_field;+ // Set reference helper field of all the docs that matched filter to `ref_seq_id`.+ nlohmann::json update_document;+ update_document["id"] = id;+ update_document[field_name] = existing_document[field_name];+ update_document[reference_helper_field_name] = ref_seq_id;- continue;+ buffer.push_back(update_document.dump());} else {- auto single_value_filter_query_op = single_value_filter_query(document, field_name, ref_field_type,- filter_query);- if (!single_value_filter_query_op.ok()) {- if (optional && single_value_filter_query_op.code() == 422) {- // Reference helper field should also be removed along with reference field.- if (is_update) {- document[reference_helper_field] = nullptr;- }+ if (!existing_document.contains(field_name) || !existing_document[field_name].is_array()) {+ return Option<bool>(400, "Expected document `id: " + id + "` to have `" += field_name + "` array field "+ "that is `" += get_field_value(existing_document, field_name) + "` instead.");+ } else if (!existing_document.contains(reference_helper_field_name) ||+ !existing_document[reference_helper_field_name].is_array()) {+ return Option<bool>(400, "Expected document `id: " + id + "` to have `" += reference_helper_field_name ++ "` array field that is `" += get_field_value(existing_document, field_name) ++ "` instead.");+ } else if (existing_document[field_name].size() != existing_document[reference_helper_field_name].size()) {+ return Option<bool>(400, "Expected document `id: " + id + "` to have equal count of elements in `" +=+ field_name + ": " += get_field_value(existing_document, field_name) ++ "` field and `" += reference_helper_field_name + ": " +=+ get_field_value(existing_document, reference_helper_field_name) + "` field.");+ }++ nlohmann::json update_document;+ update_document["id"] = id;+ update_document[field_name] = existing_document[field_name];+ update_document[reference_helper_field_name] = existing_document[reference_helper_field_name];++ auto should_update = false;+ for (uint32_t j = 0; j < existing_document[field_name].size(); j++) {+ auto const& ref_value = get_array_field_value(existing_document, field_name, j);+ if (filter_values.count(ref_value) == 0) {continue;}- return single_value_filter_query_op;- }- }-- filter_result_t filter_result;- auto filter_ids_op = ref_collection->get_filter_ids(filter_query, filter_result);- if (!filter_ids_op.ok()) {- return filter_ids_op;- }- if (document[field_name].is_array()) {- document[reference_helper_field] = nlohmann::json::array();- document[fields::reference_helper_fields] += reference_helper_field;+ const int64_t existing_ref_seq_id = existing_document[reference_helper_field_name][j].get<int64_t>();+ if (existing_ref_seq_id != Collection::reference_helper_sentinel_value &&+ existing_ref_seq_id != ref_seq_id) {+ return Option<bool>(400, "Document `id: " + id + "` at `" += field_name ++ "` reference array field and index `" + std::to_string(j) ++ "` already has a reference to document `" += std::to_string(existing_ref_seq_id) ++ "` of `" += ref_coll_name + "` collection, having reference value `" +=+ get_array_field_value(existing_document, field_name, j) + "`.");+ } else if (existing_ref_seq_id == ref_seq_id) {+ continue;+ }- for (uint32_t i = 0; i < filter_result.count; i++) {- document[reference_helper_field] += filter_result.docs[i];- }- } else {- if (filter_result.count != 1) {- // Constraints similar to foreign key apply here. The reference match must be unique and not null.- return Option<bool>(400, filter_result.count < 1 ?- "Reference document having `" + filter_query + "` not found in the collection `"- + reference_collection_name + "`." :- "Multiple documents having `" + filter_query + "` found in the collection `" +- reference_collection_name + "`.");+ should_update = true;+ // Set reference helper field to `ref_seq_id` at the index corresponding to where reference field has value.+ update_document[reference_helper_field_name][j] = ref_seq_id;}- document[reference_helper_field] = filter_result.docs[0];- document[fields::reference_helper_fields] += reference_helper_field;+ if (should_update) {+ buffer.push_back(update_document.dump());+ }}}+ nlohmann::json dummy;+ add_many(buffer, dummy, index_operation_t::UPDATE);+return Option<bool>(true);}@@ -644,10 +469,12 @@ nlohmann::json Collection::add_many(std::vector<std::string>& json_lines, nlohmabatch_doc_ids.insert(doc_id);+ std::unique_lock lock(mutex);+// if `fallback_field_type` or `dynamic_fields` is enabled, update schema first before indexing- if(!fallback_field_type.empty() || !dynamic_fields.empty() || !nested_fields.empty() || !reference_fields.empty()) {+ if(!fallback_field_type.empty() || !dynamic_fields.empty() || !nested_fields.empty() ||+ !reference_fields.empty() || !async_referenced_ins.empty()) {std::vector<field> new_fields;- std::unique_lock lock(mutex);Option<bool> new_fields_op = detect_new_fields(record.doc, dirty_values,search_schema, dynamic_fields,@@ -701,6 +528,7 @@ nlohmann::json Collection::add_many(std::vector<std::string>& json_lines, nlohmaremove_flat_fields(document);remove_reference_helper_fields(document);}+index_records.clear();batch_doc_ids.clear();}@@ -935,7 +763,9 @@ size_t Collection::batch_index_in_memory(std::vector<index_record>& index_recordstd::unique_lock lock(mutex);size_t num_indexed = Index::batch_memory_index(index, index_records, default_sorting_field,search_schema, embedding_fields, fallback_field_type,- token_separators, symbols_to_index, true, remote_embedding_batch_size, remote_embedding_timeout_ms, remote_embedding_num_tries, generate_embeddings);+ token_separators, symbols_to_index, true, remote_embedding_batch_size,+ remote_embedding_timeout_ms, remote_embedding_num_tries,generate_embeddings,+ false, tsl::htrie_map<char, field>(), name, async_referenced_ins);num_documents += num_indexed;return num_indexed;}@@ -4546,9 +4376,9 @@ void Collection::remove_document(nlohmann::json & document, const uint32_t seq_i}}-void Collection::cascade_remove_docs(const std::string& ref_helper_field_name, const uint32_t& ref_seq_id,+void Collection::cascade_remove_docs(const std::string& field_name, const uint32_t& ref_seq_id,const nlohmann::json& ref_doc, bool remove_from_store) {- auto field_name = ref_helper_field_name.substr(0, ref_helper_field_name.size() - fields::REFERENCE_HELPER_FIELD_SUFFIX.size());+ auto const ref_helper_field_name = field_name + fields::REFERENCE_HELPER_FIELD_SUFFIX;filter_result_t filter_result;get_filter_ids(ref_helper_field_name + ":" + std::to_string(ref_seq_id), filter_result);@@ -4569,6 +4399,9 @@ void Collection::cascade_remove_docs(const std::string& ref_helper_field_name, cis_field_optional = it.value().optional;}+ std::vector<std::string> buffer;+ buffer.reserve(filter_result.count);+if (is_field_singular) {// Delete all the docs where reference helper field has value `seq_id`.for (uint32_t i = 0; i < filter_result.count; i++) {@@ -4599,7 +4432,7 @@ void Collection::cascade_remove_docs(const std::string& ref_helper_field_name, cupdate_document["id"] = id;update_document[field_name] = nullptr;- add(update_document.dump(), index_operation_t::UPDATE, id);+ buffer.push_back(update_document.dump());} else {remove_document(existing_document, seq_id, remove_from_store);}@@ -4627,9 +4460,6 @@ void Collection::cascade_remove_docs(const std::string& ref_helper_field_name, creturn;}- std::vector<std::string> buffer;- buffer.reserve(filter_result.count);-// Delete all references to `seq_id` in the docs.for (uint32_t i = 0; i < filter_result.count; i++) {auto const& seq_id = filter_result.docs[i];@@ -4662,6 +4492,16 @@ void Collection::cascade_remove_docs(const std::string& ref_helper_field_name, cLOG(ERROR) << "`" << name << "` collection doc `" << existing_document.dump() <<"` at field `" << field_name << "` elements do not match the type of `" << ref_coll_name <<"` collection doc `"<< ref_doc.dump() << "` at field `" << ref_field_name << "`.";+ } else if (existing_document.count(ref_helper_field_name) == 0) {+ LOG(ERROR) << "`" << name << "` collection doc `" << existing_document.dump() << "` is missing `" <<+ ref_helper_field_name << "` field.";+ } else if (!existing_document.at(ref_helper_field_name).is_array()) {+ LOG(ERROR) << "`" << name << "` collection doc `" << existing_document.dump() << "` field `" <<+ ref_helper_field_name << "` is not an array.";+ } else if (existing_document[field_name].size() != existing_document[ref_helper_field_name].size()) {+ LOG(ERROR) << "`" << name << "` collection doc `" << existing_document.dump() << "` reference field `" <<+ field_name << "` values and its reference helper field `" << ref_helper_field_name <<+ "` values differ in count.";}// If there are more than one references present in this document, we cannot delete the whole doc. Only remove// `ref_seq_id` from reference helper field.@@ -4671,13 +4511,18 @@ void Collection::cascade_remove_docs(const std::string& ref_helper_field_name, cupdate_document[field_name] = nlohmann::json::array();auto removed_ref_value_found = false;- for (const auto& ref_value: existing_document.at(field_name)) {++ // We assume here that the value in reference field at a particular index corresponds to the value+ // present at the same index in the reference helper field.+ for (uint32_t j = 0; j < existing_document[field_name].size(); j++) {+ auto const& ref_value = existing_document[field_name][j];if (ref_value == ref_doc.at(ref_field_name)) {removed_ref_value_found = true;continue;}update_document[field_name] += ref_value;+ update_document[ref_helper_field_name] += existing_document[ref_helper_field_name][j];}if (removed_ref_value_found) {@@ -4703,10 +4548,10 @@ void Collection::cascade_remove_docs(const std::string& ref_helper_field_name, cremove_document(existing_document, seq_id, remove_from_store);}}-- nlohmann::json dummy;- add_many(buffer, dummy, index_operation_t::UPDATE);}++ nlohmann::json dummy;+ add_many(buffer, dummy, index_operation_t::UPDATE);}Option<std::string> Collection::remove(const std::string & id, const bool remove_from_store) {@@ -5129,11 +4974,16 @@ SynonymIndex* Collection::get_synonym_index() {return synonym_index;}-spp::sparse_hash_map<std::string, reference_pair> Collection::get_reference_fields() {+spp::sparse_hash_map<std::string, reference_info_t> Collection::get_reference_fields() {std::shared_lock lock(mutex);return reference_fields;}+spp::sparse_hash_map<std::string, std::vector<reference_pair_t>> Collection::get_async_referenced_ins() {+ std::shared_lock lock(mutex);+ return async_referenced_ins;+};+Option<bool> Collection::persist_collection_meta() {// first compact nested fields (to keep only parents of expanded children)field::compact_nested_fields(nested_fields);@@ -5444,329 +5294,6 @@ void Collection::remove_reference_helper_fields(nlohmann::json& document) {}}-Option<bool> Collection::prune_ref_doc(nlohmann::json& doc,- const reference_filter_result_t& references,- const tsl::htrie_set<char>& ref_include_fields_full,- const tsl::htrie_set<char>& ref_exclude_fields_full,- const bool& is_reference_array,- const ref_include_exclude_fields& ref_include_exclude) {- nlohmann::json original_doc;- if (!ref_include_exclude.nested_join_includes.empty()) {- original_doc = doc;- }-- auto const& ref_collection_name = ref_include_exclude.collection_name;- auto& cm = CollectionManager::get_instance();- auto ref_collection = cm.get_collection(ref_collection_name);- if (ref_collection == nullptr) {- return Option<bool>(400, "Referenced collection `" + ref_collection_name + "` in `include_fields` not found.");- }-- auto const& alias = ref_include_exclude.alias;- auto const& strategy = ref_include_exclude.strategy;- auto error_prefix = "Referenced collection `" + ref_collection_name + "`: ";-- // One-to-one relation.- if (strategy != ref_include::nest_array && !is_reference_array && references.count == 1) {- auto ref_doc_seq_id = references.docs[0];-- nlohmann::json ref_doc;- auto get_doc_op = ref_collection->get_document_from_store(ref_doc_seq_id, ref_doc);- if (!get_doc_op.ok()) {- return Option<bool>(get_doc_op.code(), error_prefix + get_doc_op.error());- }-- remove_flat_fields(ref_doc);- remove_reference_helper_fields(ref_doc);-- auto prune_op = prune_doc(ref_doc, ref_include_fields_full, ref_exclude_fields_full);- if (!prune_op.ok()) {- return Option<bool>(prune_op.code(), error_prefix + prune_op.error());- }-- auto const key = alias.empty() ? ref_collection_name : alias;- auto const& nest_ref_doc = (strategy == ref_include::nest);- if (!ref_doc.empty()) {- if (nest_ref_doc) {- doc[key] = ref_doc;- } else {- if (!alias.empty()) {- auto temp_doc = ref_doc;- ref_doc.clear();- for (const auto &item: temp_doc.items()) {- ref_doc[alias + item.key()] = item.value();- }- }- doc.update(ref_doc);- }- }-- // Include nested join references.- if (!ref_include_exclude.nested_join_includes.empty()) {- // Passing empty references in case the nested include collection is not joined, but it still can be included- // if we have a reference to it.- std::map<std::string, reference_filter_result_t> refs;- auto nested_include_exclude_op = include_references(nest_ref_doc ? doc[key] : doc, ref_doc_seq_id,- ref_collection.get(),- references.coll_to_references == nullptr ? refs :- references.coll_to_references[0],- ref_include_exclude.nested_join_includes, original_doc);- if (!nested_include_exclude_op.ok()) {- return nested_include_exclude_op;- }- }-- return Option<bool>(true);- }-- // One-to-many relation.- for (uint32_t i = 0; i < references.count; i++) {- auto ref_doc_seq_id = references.docs[i];-- nlohmann::json ref_doc;- auto get_doc_op = ref_collection->get_document_from_store(ref_doc_seq_id, ref_doc);- if (!get_doc_op.ok()) {- return Option<bool>(get_doc_op.code(), error_prefix + get_doc_op.error());- }-- remove_flat_fields(ref_doc);- remove_reference_helper_fields(ref_doc);-- auto prune_op = prune_doc(ref_doc, ref_include_fields_full, ref_exclude_fields_full);- if (!prune_op.ok()) {- return Option<bool>(prune_op.code(), error_prefix + prune_op.error());- }-- std::string key;- auto const& nest_ref_doc = (strategy == ref_include::nest || strategy == ref_include::nest_array);- if (!ref_doc.empty()) {- if (nest_ref_doc) {- key = alias.empty() ? ref_collection_name : alias;- if (doc.contains(key) && !doc[key].is_array()) {- return Option<bool>(400, "Could not include the reference document of `" + ref_collection_name +- "` collection. Expected `" + key + "` to be an array. Try " +- (alias.empty() ? "adding an" : "renaming the") + " alias.");- }-- doc[key] += ref_doc;- } else {- for (auto ref_doc_it = ref_doc.begin(); ref_doc_it != ref_doc.end(); ref_doc_it++) {- auto const& ref_doc_key = ref_doc_it.key();- key = alias + ref_doc_key;- if (doc.contains(key) && !doc[key].is_array()) {- return Option<bool>(400, "Could not include the value of `" + ref_doc_key +- "` key of the reference document of `" + ref_collection_name +- "` collection. Expected `" + key + "` to be an array. Try " +- (alias.empty() ? "adding an" : "renaming the") + " alias.");- }-- // Add the values of ref_doc as JSON array into doc.- doc[key] += ref_doc_it.value();- }- }- }-- // Include nested join references.- if (!ref_include_exclude.nested_join_includes.empty()) {- // Passing empty references in case the nested include collection is not joined, but it still can be included- // if we have a reference to it.- std::map<std::string, reference_filter_result_t> refs;- auto nested_include_exclude_op = include_references(nest_ref_doc ? doc[key].at(i) : doc, ref_doc_seq_id,- ref_collection.get(),- references.coll_to_references == nullptr ? refs :- references.coll_to_references[i],- ref_include_exclude.nested_join_includes, original_doc);- if (!nested_include_exclude_op.ok()) {- return nested_include_exclude_op;- }- }- }-- return Option<bool>(true);-}--Option<bool> Collection::include_references(nlohmann::json& doc, const uint32_t& seq_id, Collection *const collection,- const std::map<std::string, reference_filter_result_t>& reference_filter_results,- const std::vector<ref_include_exclude_fields>& ref_include_exclude_fields_vec,- const nlohmann::json& original_doc) {- for (auto const& ref_include_exclude: ref_include_exclude_fields_vec) {- auto ref_collection_name = ref_include_exclude.collection_name;-- auto& cm = CollectionManager::get_instance();- auto ref_collection = cm.get_collection(ref_collection_name);- if (ref_collection == nullptr) {- return Option<bool>(400, "Referenced collection `" + ref_collection_name + "` in `include_fields` not found.");- }- // `CollectionManager::get_collection` accounts for collection alias being used and provides pointer to the- // original collection.- ref_collection_name = ref_collection->name;-- auto const joined_on_ref_collection = reference_filter_results.count(ref_collection_name) > 0,- has_filter_reference = (joined_on_ref_collection &&- reference_filter_results.at(ref_collection_name).count > 0);- auto doc_has_reference = false, joined_coll_has_reference = false;-- // Reference include_by without join, check if doc itself contains the reference.- if (!joined_on_ref_collection && collection != nullptr) {- doc_has_reference = ref_collection->is_referenced_in(collection->name);- }-- std::string joined_coll_having_reference;- // Check if the joined collection has a reference.- if (!joined_on_ref_collection && !doc_has_reference) {- for (const auto &reference_filter_result: reference_filter_results) {- joined_coll_has_reference = ref_collection->is_referenced_in(reference_filter_result.first);- if (joined_coll_has_reference) {- joined_coll_having_reference = reference_filter_result.first;- break;- }- }- }-- if (!has_filter_reference && !doc_has_reference && !joined_coll_has_reference) {- continue;- }-- std::vector<std::string> ref_include_fields_vec, ref_exclude_fields_vec;- StringUtils::split(ref_include_exclude.include_fields, ref_include_fields_vec, ",");- StringUtils::split(ref_include_exclude.exclude_fields, ref_exclude_fields_vec, ",");-- spp::sparse_hash_set<std::string> ref_include_fields, ref_exclude_fields;- ref_include_fields.insert(ref_include_fields_vec.begin(), ref_include_fields_vec.end());- ref_exclude_fields.insert(ref_exclude_fields_vec.begin(), ref_exclude_fields_vec.end());-- tsl::htrie_set<char> ref_include_fields_full, ref_exclude_fields_full;- auto include_exclude_op = ref_collection->populate_include_exclude_fields_lk(ref_include_fields,- ref_exclude_fields,- ref_include_fields_full,- ref_exclude_fields_full);- auto error_prefix = "Referenced collection `" + ref_collection_name + "`: ";- if (!include_exclude_op.ok()) {- return Option<bool>(include_exclude_op.code(), error_prefix + include_exclude_op.error());- }-- Option<bool> prune_doc_op = Option<bool>(true);- auto const& ref_collection_alias = ref_include_exclude.alias;- if (has_filter_reference) {- auto const& ref_filter_result = reference_filter_results.at(ref_collection_name);- prune_doc_op = prune_ref_doc(doc, ref_filter_result, ref_include_fields_full, ref_exclude_fields_full,- ref_filter_result.is_reference_array_field, ref_include_exclude);- } else if (doc_has_reference) {- auto get_reference_field_op = ref_collection->get_referenced_in_field_with_lock(collection->name);- if (!get_reference_field_op.ok()) {- continue;- }- auto const& field_name = get_reference_field_op.get();- if (collection->search_schema.count(field_name) == 0) {- continue;- }-- if (collection->object_reference_helper_fields.count(field_name) != 0) {- std::vector<std::string> keys;- StringUtils::split(field_name, keys, ".");- auto const& key = keys[0];-- if (!doc.contains(key)) {- if (!original_doc.contains(key)) {- return Option<bool>(400, "Could not find `" + key +- "` key in the document to include the referenced document.");- }-- // The key is excluded from the doc by the query, inserting empty object(s) so referenced doc can be- // included in it.- if (original_doc[key].is_array()) {- doc[key] = nlohmann::json::array();- doc[key].insert(doc[key].begin(), original_doc[key].size(), nlohmann::json::object());- } else {- doc[key] = nlohmann::json::object();- }- }-- if (doc[key].is_array()) {- for (uint32_t i = 0; i < doc[key].size(); i++) {- uint32_t ref_doc_id;- auto op = collection->get_object_array_related_id(field_name, seq_id, i, ref_doc_id);- if (!op.ok()) {- if (op.code() == 404) { // field_name is not indexed.- break;- } else { // No reference found for this object.- continue;- }- }-- reference_filter_result_t result(1, new uint32_t[1]{ref_doc_id});- prune_doc_op = prune_ref_doc(doc[key][i], result,- ref_include_fields_full, ref_exclude_fields_full,- false, ref_include_exclude);- if (!prune_doc_op.ok()) {- return prune_doc_op;- }- }- } else {- std::vector<uint32_t> ids;- auto get_references_op = collection->get_related_ids(field_name, seq_id, ids);- if (!get_references_op.ok()) {- continue;- }- reference_filter_result_t result(ids.size(), &ids[0]);- prune_doc_op = prune_ref_doc(doc[key], result, ref_include_fields_full, ref_exclude_fields_full,- collection->search_schema.at(field_name).is_array(), ref_include_exclude);- result.docs = nullptr;- }- } else {- std::vector<uint32_t> ids;- auto get_references_op = collection->get_related_ids(field_name, seq_id, ids);- if (!get_references_op.ok()) {- continue;- }- reference_filter_result_t result(ids.size(), &ids[0]);- prune_doc_op = prune_ref_doc(doc, result, ref_include_fields_full, ref_exclude_fields_full,- collection->search_schema.at(field_name).is_array(), ref_include_exclude);- result.docs = nullptr;- }- } else if (joined_coll_has_reference) {- auto joined_collection = cm.get_collection(joined_coll_having_reference);- if (joined_collection == nullptr) {- continue;- }-- auto reference_field_name_op = ref_collection->get_referenced_in_field_with_lock(joined_coll_having_reference);- if (!reference_field_name_op.ok() || joined_collection->get_schema().count(reference_field_name_op.get()) == 0) {- continue;- }-- auto const& reference_field_name = reference_field_name_op.get();- auto const& reference_filter_result = reference_filter_results.at(joined_coll_having_reference);- auto const& count = reference_filter_result.count;- std::vector<uint32_t> ids;- ids.reserve(count);- for (uint32_t i = 0; i < count; i++) {- joined_collection->get_related_ids_with_lock(reference_field_name, reference_filter_result.docs[i], ids);- }- if (ids.empty()) {- continue;- }-- gfx::timsort(ids.begin(), ids.end());- ids.erase(unique(ids.begin(), ids.end()), ids.end());-- reference_filter_result_t result;- result.count = ids.size();- result.docs = &ids[0];- prune_doc_op = prune_ref_doc(doc, result, ref_include_fields_full, ref_exclude_fields_full,- joined_collection->get_schema().at(reference_field_name).is_array(),- ref_include_exclude);- result.docs = nullptr;- }-- if (!prune_doc_op.ok()) {- return prune_doc_op;- }- }-- return Option<bool>(true);-}-Option<bool> Collection::prune_doc_with_lock(nlohmann::json& doc, const tsl::htrie_set<char>& include_names,const tsl::htrie_set<char>& exclude_names,const std::map<std::string, reference_filter_result_t>& reference_filter_results,@@ -5865,8 +5392,8 @@ Option<bool> Collection::prune_doc(nlohmann::json& doc,it++;}- return include_references(doc, seq_id, collection, reference_filter_results, ref_include_exclude_fields_vec,- original_doc);+ return Join::include_references(doc, seq_id, collection, reference_filter_results, ref_include_exclude_fields_vec,+ original_doc);}Option<bool> Collection::validate_alter_payload(nlohmann::json& schema_changes,@@ -6296,7 +5823,7 @@ Option<bool> Collection::detect_new_fields(nlohmann::json& document,bool is_update,std::vector<field>& new_fields,const bool enable_nested_fields,- const spp::sparse_hash_map<std::string, reference_pair>& reference_fields,+ const spp::sparse_hash_map<std::string, reference_info_t>& reference_fields,tsl::htrie_set<char>& object_reference_helper_fields) {auto kv = document.begin();@@ -6381,8 +5908,8 @@ Option<bool> Collection::detect_new_fields(nlohmann::json& document,}}- auto add_reference_helper_fields_op = add_reference_helper_fields(document, schema, reference_fields,- object_reference_helper_fields, is_update);+ auto add_reference_helper_fields_op = Join::add_reference_helper_fields(document, schema, reference_fields,+ object_reference_helper_fields, is_update);if (!add_reference_helper_fields_op.ok()) {return add_reference_helper_fields_op;}@@ -6424,17 +5951,17 @@ Index* Collection::init_index() {// the original collection.ref_coll_name = ref_coll->name;- // Passing reference helper field helps perform operation on doc_id instead of field value.- ref_coll->add_referenced_in(name, field.name + fields::REFERENCE_HELPER_FIELD_SUFFIX);+ ref_coll->add_referenced_in(name, field.name, field.is_async_reference, ref_field_name);} else {// Reference collection has not been created yet.collectionManager.add_referenced_in_backlog(ref_coll_name,- reference_pair{name, field.name + fields::REFERENCE_HELPER_FIELD_SUFFIX});+ reference_info_t{name, field.name, field.is_async_reference,+ ref_field_name});}- reference_fields.emplace(field.name, reference_pair(ref_coll_name, ref_field_name));+ reference_fields.emplace(field.name, reference_info_t(ref_coll_name, ref_field_name, field.is_async_reference));if (field.nested) {- object_reference_helper_fields.insert(field.name + fields::REFERENCE_HELPER_FIELD_SUFFIX);+ object_reference_helper_fields.insert(field.name);}}}@@ -6911,20 +6438,40 @@ bool Collection::is_referenced_in(const std::string& collection_name) const {return referenced_in.count(collection_name) > 0;}-void Collection::add_referenced_in(const reference_pair& pair) {- return add_referenced_in(pair.collection, pair.field);-}--void Collection::add_referenced_ins(const std::set<reference_pair>& pairs) {+void Collection::add_referenced_ins(const std::set<reference_info_t>& ref_infos) {std::shared_lock lock(mutex);- for (const auto &pair: pairs) {- referenced_in.emplace(pair.collection, pair.field);+ for (const auto &ref_info: ref_infos) {+ auto const& referenced_field_name = ref_info.referenced_field_name;++ auto it = search_schema.find(referenced_field_name);+ if (referenced_field_name != "id" && it == search_schema.end()) {+ LOG(ERROR) << "Field `" << referenced_field_name << "` not found in the collection `" << name <<+ "` which is referenced in `" << ref_info.collection << "." << ref_info.field + "`.";+ continue;+ }++ referenced_in.emplace(ref_info.collection, ref_info.field);+ if (ref_info.is_async) {+ async_referenced_ins[referenced_field_name].emplace_back(ref_info.collection, ref_info.field);+ }}}-void Collection::add_referenced_in(const std::string& collection_name, const std::string& field_name) {+void Collection::add_referenced_in(const std::string& collection_name, const std::string& field_name,+ const bool& is_async, const std::string& referenced_field_name) {std::shared_lock lock(mutex);++ auto it = search_schema.find(referenced_field_name);+ if (referenced_field_name != "id" && it == search_schema.end()) {+ LOG(ERROR) << "Field `" << referenced_field_name << "` not found in the collection `" << name <<+ "` which is referenced in `" << collection_name << "." << field_name + "`.";+ return;+ }+referenced_in.emplace(collection_name, field_name);+ if (is_async) {+ async_referenced_ins[referenced_field_name].emplace_back(collection_name, field_name);+ }}Option<std::string> Collection::get_referenced_in_field_with_lock(const std::string& collection_name) const {diff --git a/src/collection_manager.cpp b/src/collection_manager.cppindex 0f332652..1b152d0e 100644--- a/src/collection_manager.cpp+++ b/src/collection_manager.cpp@@ -22,7 +22,8 @@ Collection* CollectionManager::init_collection(const nlohmann::json & collectionconst uint32_t collection_next_seq_id,Store* store,float max_memory_ratio,- spp::sparse_hash_map<std::string, std::string>& referenced_in) {+ spp::sparse_hash_map<std::string, std::string>& referenced_in,+ spp::sparse_hash_map<std::string, std::vector<reference_pair_t>>& async_referenced_ins) {std::string this_collection_name = collection_meta[Collection::COLLECTION_NAME_KEY].get<std::string>();std::vector<field> fields;@@ -201,7 +202,7 @@ Collection* CollectionManager::init_collection(const nlohmann::json & collectionsymbols_to_index,token_separators,enable_nested_fields, model, std::move(referenced_in),- metadata);+ metadata, std::move(async_referenced_ins));return collection;}@@ -238,7 +239,8 @@ void CollectionManager::init(Store *store, const float max_memory_ratio, const s}void CollectionManager::_populate_referenced_ins(const std::string& collection_meta_json,- std::map<std::string, spp::sparse_hash_map<std::string, std::string>>& referenced_ins) {+ std::map<std::string, spp::sparse_hash_map<std::string, std::string>>& referenced_ins,+ std::map<std::string, spp::sparse_hash_map<std::string, std::vector<reference_pair_t>>>& async_referenced_ins) {auto const& obj = nlohmann::json::parse(collection_meta_json, nullptr, false);if (!obj.is_discarded() && obj.is_object() && obj.contains("name") && obj["name"].is_string() &&@@ -249,10 +251,18 @@ void CollectionManager::_populate_referenced_ins(const std::string& collection_mif (!field.contains("name") || !field.contains("reference")) {continue;}- auto field_name = std::string(field["name"]) + fields::REFERENCE_HELPER_FIELD_SUFFIX;+ auto field_name = std::string(field["name"]);++ auto const& reference = field["reference"].get<std::string>();std::vector<std::string> split_result;- StringUtils::split(field["reference"], split_result, ".");- auto ref_coll_name = split_result.front();+ StringUtils::split(reference, split_result, ".");+ if (split_result.size() < 2) {+ LOG(ERROR) << "Invalid reference `" << reference << "`.";+ continue;+ }++ auto ref_coll_name = split_result[0];+ auto ref_field_name = reference.substr(ref_coll_name.size() + 1);// Resolves alias if used in schema.auto actual_ref_coll_it = CollectionManager::get_instance().collection_symlinks.find(ref_coll_name);@@ -264,6 +274,11 @@ void CollectionManager::_populate_referenced_ins(const std::string& collection_m}referenced_ins[ref_coll_name].emplace(collection_name, field_name);++ if (field.contains(fields::async_reference) &&+ field[fields::async_reference].is_boolean() && field[fields::async_reference].get<bool>()) {+ async_referenced_ins[ref_coll_name][ref_field_name].emplace_back(collection_name, field_name);+ }}}}@@ -321,8 +336,10 @@ Option<bool> CollectionManager::load(const size_t collection_batch_size, const s// Collection name -> Ref collection name -> Ref field namestd::map<std::string, spp::sparse_hash_map<std::string, std::string>> referenced_ins;+ // Collection name -> field name -> {Ref collection name, Ref field name}+ std::map<std::string, spp::sparse_hash_map<std::string, std::vector<reference_pair_t>>> async_referenced_ins;for (const auto &collection_meta_json: collection_meta_jsons) {- _populate_referenced_ins(collection_meta_json, referenced_ins);+ _populate_referenced_ins(collection_meta_json, referenced_ins, async_referenced_ins);}size_t num_processed = 0;@@ -343,7 +360,7 @@ Option<bool> CollectionManager::load(const size_t collection_batch_size, const sauto captured_store = store;loading_pool.enqueue([captured_store, num_collections, collection_meta, document_batch_size,&m_process, &cv_process, &num_processed, &next_coll_id_status, quit = quit,- &referenced_ins, collection_name]() {+ &referenced_ins, &async_referenced_ins, collection_name]() {spp::sparse_hash_map<std::string, std::string> referenced_in;auto const& it = referenced_ins.find(collection_name);@@ -351,9 +368,15 @@ Option<bool> CollectionManager::load(const size_t collection_batch_size, const sreferenced_in = it->second;}+ spp::sparse_hash_map<std::string, std::vector<reference_pair_t>> async_referenced_in;+ auto const& async_it = async_referenced_ins.find(collection_name);+ if (async_it != async_referenced_ins.end()) {+ async_referenced_in = async_it->second;+ }+//auto begin = std::chrono::high_resolution_clock::now();Option<bool> res = load_collection(collection_meta, document_batch_size, next_coll_id_status, *quit,- referenced_in);+ referenced_in, async_referenced_in);/*long long int timeMillis =std::chrono::duration_cast<std::chrono::milliseconds>(std::chrono::high_resolution_clock::now() - begin).count();LOG(INFO) << "Time taken for indexing: " << timeMillis << "ms";*/@@ -457,6 +480,7 @@ void CollectionManager::dispose() {collections.clear();collection_symlinks.clear();preset_configs.clear();+ referenced_in_backlog.clear();store->close();}@@ -554,10 +578,10 @@ Option<Collection*> CollectionManager::create_collection(const std::string& nameadd_to_collections(new_collection);lock.lock();-- if (referenced_in_backlog.count(name) > 0) {- new_collection->add_referenced_ins(referenced_in_backlog.at(name));- referenced_in_backlog.erase(name);+ auto it = referenced_in_backlog.find(name);+ if (it != referenced_in_backlog.end()) {+ new_collection->add_referenced_ins(it->second);+ referenced_in_backlog.erase(it);}return Option<Collection*>(new_collection);@@ -2286,7 +2310,8 @@ Option<bool> CollectionManager::load_collection(const nlohmann::json &collectionconst size_t batch_size,const StoreStatus& next_coll_id_status,const std::atomic<bool>& quit,- spp::sparse_hash_map<std::string, std::string>& referenced_in) {+ spp::sparse_hash_map<std::string, std::string>& referenced_in,+ spp::sparse_hash_map<std::string, std::vector<reference_pair_t>>& async_referenced_ins) {auto& cm = CollectionManager::get_instance();@@ -2332,7 +2357,8 @@ Option<bool> CollectionManager::load_collection(const nlohmann::json &collection}}- Collection* collection = init_collection(collection_meta, collection_next_seq_id, cm.store, 1.0f, referenced_in);+ Collection* collection = init_collection(collection_meta, collection_next_seq_id, cm.store, 1.0f, referenced_in,+ async_referenced_ins);LOG(INFO) << "Loading collection " << collection->get_name();@@ -2564,12 +2590,12 @@ Option<Collection*> CollectionManager::clone_collection(const string& existing_nreturn Option<Collection*>(new_coll);}-void CollectionManager::add_referenced_in_backlog(const std::string& collection_name, reference_pair&& pair) {+void CollectionManager::add_referenced_in_backlog(const std::string& collection_name, reference_info_t&& ref_info) {std::shared_lock lock(mutex);- referenced_in_backlog[collection_name].insert(pair);+ referenced_in_backlog[collection_name].insert(ref_info);}-std::map<std::string, std::set<reference_pair>> CollectionManager::_get_referenced_in_backlog() const {+std::map<std::string, std::set<reference_info_t>> CollectionManager::_get_referenced_in_backlog() const {std::shared_lock lock(mutex);return referenced_in_backlog;}diff --git a/src/field.cpp b/src/field.cppindex 9bbb3f34..8c98c11d 100644--- a/src/field.cpp+++ b/src/field.cpp@@ -81,6 +81,18 @@ Option<bool> field::json_field_to_field(bool enable_nested_fields, nlohmann::jsofield_json[fields::reference] = "";}+ if (field_json.count(fields::async_reference) == 0) {+ field_json[fields::async_reference] = false;+ } else if (!field_json.at(fields::async_reference).is_boolean()) {+ return Option<bool>(400, std::string("The `async_reference` property of the field `") ++ field_json[fields::name].get<std::string>() + std::string("` should be a boolean."));+ } else if (field_json[fields::async_reference].get<bool>() &&+ field_json[fields::reference].get<std::string>().empty()) {+ return Option<bool>(400, std::string("The `async_reference` property of the field `") ++ field_json[fields::name].get<std::string>() + std::string("` is only applicable if "+ "`reference` is specified."));+ }+if(field_json.count(fields::stem) != 0) {if(!field_json.at(fields::stem).is_boolean()) {return Option<bool>(400, std::string("The `stem` property of the field `") +@@ -411,7 +423,8 @@ Option<bool> field::json_field_to_field(bool enable_nested_fields, nlohmann::jsofield_json[fields::sort], field_json[fields::infix], field_json[fields::nested],field_json[fields::nested_array], field_json[fields::num_dim], vec_dist,field_json[fields::reference], field_json[fields::embed], field_json[fields::range_index],- field_json[fields::store], field_json[fields::stem], field_json[fields::hnsw_params])+ field_json[fields::store], field_json[fields::stem], field_json[fields::hnsw_params],+ field_json[fields::async_reference]));if (!field_json[fields::reference].get<std::string>().empty()) {@@ -812,6 +825,7 @@ Option<bool> field::fields_to_json_fields(const std::vector<field>& fields, consif (!field.reference.empty()) {field_val[fields::reference] = field.reference;+ field_val[fields::async_reference] = field.is_async_reference;}fields_json.push_back(field_val);diff --git a/src/filter_result_iterator.cpp b/src/filter_result_iterator.cppindex 936332a0..0f3c8cd1 100644--- a/src/filter_result_iterator.cpp+++ b/src/filter_result_iterator.cpp@@ -837,8 +837,8 @@ void filter_result_iterator_t::init() {return;}- auto const& reference_helper_field_name = get_reference_field_op.get();- auto op = index->do_filtering_with_reference_ids(reference_helper_field_name, ref_collection_name,+ auto const& ref_field_name = get_reference_field_op.get();+ auto op = index->do_filtering_with_reference_ids(ref_field_name, ref_collection_name,std::move(result));if (!op.ok()) {status = Option<bool>(op.code(), op.error());diff --git a/src/index.cpp b/src/index.cppindex 6af2dbb6..3a05a8e1 100644--- a/src/index.cpp+++ b/src/index.cpp@@ -565,7 +565,9 @@ batch_memory_index(Index *index,const bool do_validation, const size_t remote_embedding_batch_size,const size_t remote_embedding_timeout_ms, const size_t remote_embedding_num_tries,const bool generate_embeddings,- const bool use_addition_fields, const tsl::htrie_map<char, field>& addition_fields) {+ const bool use_addition_fields, const tsl::htrie_map<char, field>& addition_fields,+ const std::string& collection_name,+ const spp::sparse_hash_map<std::string, std::vector<reference_pair_t>>& async_referenced_ins) {const size_t concurrency = 4;const size_t num_threads = std::min(concurrency, iter_batch.size());const size_t window_size = (num_threads == 0) ? 0 :@@ -649,8 +651,14 @@ batch_memory_index(Index *index,const field& f = (field_name == "id") ?field("id", field_types::STRING, false) : indexable_schema.at(field_name);+ std::vector<reference_pair_t> async_references;+ auto it = async_referenced_ins.find(field_name);+ if (it != async_referenced_ins.end()) {+ async_references = it->second;+ }+try {- index->index_field_in_memory(f, iter_batch);+ index->index_field_in_memory(collection_name, f, iter_batch, async_references);} catch(std::exception& e) {LOG(ERROR) << "Unhandled Typesense error: " << e.what();for(auto& record: iter_batch) {@@ -672,7 +680,9 @@ batch_memory_index(Index *index,return num_indexed;}-void Index::index_field_in_memory(const field& afield, std::vector<index_record>& iter_batch) {+void Index::index_field_in_memory(const std::string& collection_name, const field& afield,+ std::vector<index_record>& iter_batch,+ const std::vector<reference_pair_t>& async_referenced_ins) {// indexes a given field of all documents in the batchif(afield.name == "id") {@@ -687,6 +697,9 @@ void Index::index_field_in_memory(const field& afield, std::vector<index_record>}}+ if (!async_referenced_ins.empty()) {+ update_async_references(collection_name, afield, iter_batch, async_referenced_ins);+ }return;}@@ -1150,6 +1163,127 @@ void Index::index_field_in_memory(const field& afield, std::vector<index_record>}}}++ if (!async_referenced_ins.empty()) {+ update_async_references(collection_name, afield, iter_batch, async_referenced_ins);+ }+}++void Index::update_async_references(const std::string& collection_name, const field& afield,+ std::vector<index_record>& iter_batch,+ const std::vector<reference_pair_t>& async_referenced_ins) {+ for (auto& record: iter_batch) {+ if (!record.indexed.ok() || record.is_update) {+ continue;+ }+ auto const& document = record.doc;+ auto const& is_update = record.is_update;+ auto const& seq_id = record.seq_id;++ for (const auto& pair: async_referenced_ins) {+ auto const& reference_collection_name = pair.collection;+ auto const& reference_field_name = pair.field;++ auto& cm = CollectionManager::get_instance();+ auto ref_coll = cm.get_collection(reference_collection_name);+ if (ref_coll == nullptr) {+ record.index_failure(400, "Collection `" + reference_collection_name + "` with async_reference to the"+ " collection `" += collection_name + "` not found.");+ continue;+ }++ auto const& ref_fields = ref_coll->get_reference_fields();+ auto const ref_field_it = ref_fields.find(reference_field_name);+ if (ref_field_it == ref_fields.end()) {+ record.index_failure(400, "Field `" + reference_field_name + "` not found in the ref schema of `" +=+ reference_collection_name + "` having async_reference to `" += collection_name ++ "` collection.");+ continue;+ }++ if (ref_field_it->second.collection != collection_name) {+ record.index_failure(400, "`" + reference_collection_name + "." += reference_field_name ++ "` does not have a reference to `" += collection_name + "` collection.");+ continue;+ }++ auto const& ref_schema = ref_coll->get_schema();+ if (ref_schema.count(reference_field_name) == 0) {+ record.index_failure(400, "Field `" + reference_field_name + "` not found in the schema of `" +=+ reference_collection_name + "` having async_reference to `" +=+ collection_name + "` collection.");+ continue;+ }++ auto const& field_name = ref_field_it->second.field;+ if (field_name != "id" && search_schema.count(field_name) == 0) {+ record.index_failure(400, "Field `" + field_name + "`, referenced by `" += reference_collection_name ++ "." += reference_field_name + "`, not found in `" += collection_name ++ "` collection.");+ continue;+ }++ auto const& optional = field_name != "id" && search_schema.at(field_name).optional;+ auto is_required = !is_update && !optional;+ if (is_required && document.count(field_name) != 1) {+ record.index_failure(400, "Missing the required field `" + field_name + "` in the document.");+ continue;+ } else if (document.count(field_name) != 1) {+ continue;+ }++ // After collecting the value(s) present in the field referenced by the other collection(ref_coll), we will add+ // this document's seq_id as a reference where the value(s) match.+ std::string ref_filter_value;+ std::set<std::string> values;+ if (document.at(field_name).is_array()) {+ ref_filter_value = "[";++ for (auto const& value: document[field_name]) {+ if (value.is_number_integer()) {+ auto const& v = std::to_string(value.get<int64_t>());+ ref_filter_value += v;+ values.insert(v);+ } else if (value.is_string()) {+ auto const& v = value.get<std::string>();+ ref_filter_value += v;+ values.insert(v);+ } else {+ record.index_failure(400, "Field `" + field_name + "` must only have string/int32/int64 values.");+ continue;+ }+ ref_filter_value += ",";+ }+ ref_filter_value[ref_filter_value.size() - 1] = ']';+ } else {+ auto const& value = document[field_name];+ if (value.is_number_integer()) {+ auto const& v = std::to_string(value.get<int64_t>());+ ref_filter_value += v;+ values.insert(v);+ } else if (value.is_string()) {+ auto const& v = value.get<std::string>();+ ref_filter_value += v;+ values.insert(v);+ } else {+ record.index_failure(400, "Field `" + field_name + "` must only have string/int32/int64 values.");+ continue;+ }+ }++ if (values.empty()) {+ continue;+ }++ auto const ref_filter = reference_field_name + ":= " += ref_filter_value;+ auto update_op = ref_coll->update_async_references_with_lock(collection_name, ref_filter, values, seq_id,+ reference_field_name);+ if (!update_op.ok()) {+ record.index_failure(400, "Error while updating async reference field `" + reference_field_name ++ "` of collection `" += reference_collection_name + "`: " += update_op.error());+ }+ }+ }}void Index::tokenize_string(const std::string& text, const field& a_field,@@ -1727,7 +1861,7 @@ void aggregate_nested_references(single_filter_result_t *const reference_result,Option<bool> Index::do_reference_filtering_with_lock(filter_node_t* const filter_tree_root,filter_result_t& filter_result,const std::string& ref_collection_name,- const std::string& reference_helper_field_name) const {+ const std::string& field_name) const {std::shared_lock lock(mutex);auto ref_filter_result_iterator = filter_result_iterator_t(ref_collection_name, this, filter_tree_root,@@ -1755,6 +1889,7 @@ Option<bool> Index::do_reference_filtering_with_lock(filter_node_t* const filterref_filter_result->docs = nullptr;std::unique_ptr<uint32_t[]> docs_guard(reference_docs);+ auto const reference_helper_field_name = field_name + fields::REFERENCE_HELPER_FIELD_SUFFIX;auto const is_nested_join = !ref_filter_result_iterator.reference.empty();if (search_schema.at(reference_helper_field_name).is_singular()) { // Only one reference per doc.if (sort_index.count(reference_helper_field_name) == 0) {@@ -1838,6 +1973,10 @@ Option<bool> Index::do_reference_filtering_with_lock(filter_node_t* const filter}auto doc_id = ref_index.at(reference_doc_id);+ if (doc_id == Collection::reference_helper_sentinel_value) {+ continue;+ }+id_pairs.emplace_back(std::make_pair(doc_id, reference_doc_id));unique_doc_ids.insert(doc_id);}@@ -2036,7 +2175,7 @@ Option<bool> Index::do_reference_filtering_with_lock(filter_node_t* const filterreturn Option(true);}-Option<filter_result_t> Index::do_filtering_with_reference_ids(const std::string& reference_helper_field_name,+Option<filter_result_t> Index::do_filtering_with_reference_ids(const std::string& field_name,const std::string& ref_collection_name,filter_result_t&& ref_filter_result) const {filter_result_t filter_result;@@ -2048,6 +2187,7 @@ Option<filter_result_t> Index::do_filtering_with_reference_ids(const std::stringreturn Option<filter_result_t>(filter_result);}+ auto const reference_helper_field_name = field_name + fields::REFERENCE_HELPER_FIELD_SUFFIX;if (numerical_index.count(reference_helper_field_name) == 0) {return Option<filter_result_t>(400, "`" + reference_helper_field_name + "` is not present in index.");}@@ -4831,12 +4971,14 @@ Option<bool> Index::ref_compute_sort_scores(const sort_by& sort_field, const uinreturn Option<bool>(get_reference_field_op.code(), get_reference_field_op.error());}auto const& field_name = get_reference_field_op.get();- if (sort_index.count(field_name) == 0) {- return Option<bool>(400, "Could not find `" + field_name + "` in sort_index.");- } else if (sort_index.at(field_name)->count(seq_id) == 0) {+ auto const reference_helper_field_name = field_name + fields::REFERENCE_HELPER_FIELD_SUFFIX;++ if (sort_index.count(reference_helper_field_name) == 0) {+ return Option<bool>(400, "Could not find `" + reference_helper_field_name + "` in sort_index.");+ } else if (sort_index.at(reference_helper_field_name)->count(seq_id) == 0) {reference_found = false;} else {- ref_seq_id = sort_index.at(field_name)->at(seq_id);+ ref_seq_id = sort_index.at(reference_helper_field_name)->at(seq_id);}}// Joined collection has a reference@@ -7646,28 +7788,39 @@ int64_t Index::reference_string_sort_score(const string &field_name, const uint3Option<bool> Index::get_related_ids(const std::string& collection_name, const string& field_name,const uint32_t& seq_id, std::vector<uint32_t>& result) const {std::shared_lock lock(mutex);- if (search_schema.count(field_name) == 0) {- return Option<bool>(400, "Could not find `" + field_name + "` in the collection `" + collection_name + "`.");++ auto const reference_helper_field_name = field_name + fields::REFERENCE_HELPER_FIELD_SUFFIX;+ if (search_schema.count(reference_helper_field_name) == 0) {+ return Option<bool>(400, "Could not find `" + reference_helper_field_name + "` in the collection `" + collection_name + "`.");}- auto const no_match_op = Option<bool>(400, "Could not find `" + field_name + "` value for doc `" +- std::to_string(seq_id) + "`.");- if (search_schema.at(field_name).is_singular()) {- if (sort_index.count(field_name) == 0 || sort_index.at(field_name)->count(seq_id) == 0) {+ auto const no_match_op = Option<bool>(400, "Could not find `" + reference_helper_field_name + "` value for doc `" ++ std::to_string(seq_id) + "`.");+ if (search_schema.at(reference_helper_field_name).is_singular()) {+ if (sort_index.count(reference_helper_field_name) == 0) {+ return no_match_op;+ }++ auto const& ref_index = sort_index.at(reference_helper_field_name);+ auto const it = ref_index->find(seq_id);+ if (it == ref_index->end()) {return no_match_op;}- result.emplace_back(sort_index.at(field_name)->at(seq_id));+ const uint32_t id = it->second;+ if (id != Collection::reference_helper_sentinel_value) {+ result.emplace_back(id);+ }return Option<bool>(true);}- if (reference_index.count(field_name) == 0) {+ if (reference_index.count(reference_helper_field_name) == 0) {return no_match_op;}size_t ids_len = 0;uint32_t* ids = nullptr;- reference_index.at(field_name)->search(EQUALS, seq_id, &ids, ids_len);+ reference_index.at(reference_helper_field_name)->search(EQUALS, seq_id, &ids, ids_len);if (ids_len == 0) {return no_match_op;}@@ -7700,17 +7853,21 @@ Option<uint32_t> Index::get_sort_index_value_with_lock(const std::string& collecconst std::string& field_name,const uint32_t& seq_id) const {std::shared_lock lock(mutex);- if (search_schema.count(field_name) == 0) {- return Option<uint32_t>(400, "Could not find `" + field_name + "` in the collection `" + collection_name + "`.");- } else if (search_schema.at(field_name).is_array()) {- return Option<uint32_t>(400, "Cannot sort on `" + field_name + "` in the collection, `" + collection_name +- "` is `" + search_schema.at(field_name).type + "`.");- } else if (sort_index.count(field_name) == 0 || sort_index.at(field_name)->count(seq_id) == 0) {- return Option<uint32_t>(404, "Could not find `" + field_name + "` value for doc `" +++ auto const reference_helper_field_name = field_name + fields::REFERENCE_HELPER_FIELD_SUFFIX;+ if (search_schema.count(reference_helper_field_name) == 0) {+ return Option<uint32_t>(400, "Could not find `" + reference_helper_field_name + "` in the collection `" ++ collection_name + "`.");+ } else if (search_schema.at(reference_helper_field_name).is_array()) {+ return Option<uint32_t>(400, "Cannot sort on `" + reference_helper_field_name + "` in the collection, `" ++ collection_name + "` is `" + search_schema.at(reference_helper_field_name).type + "`.");+ } else if (sort_index.count(reference_helper_field_name) == 0 ||+ sort_index.at(reference_helper_field_name)->count(seq_id) == 0) {+ return Option<uint32_t>(404, "Could not find `" + reference_helper_field_name + "` value for doc `" +std::to_string(seq_id) + "`.");;}- return Option<uint32_t>(sort_index.at(field_name)->at(seq_id));+ return Option<uint32_t>(sort_index.at(reference_helper_field_name)->at(seq_id));}float Index::get_distance(const string& geo_field_name, const uint32_t& seq_id,diff --git a/src/join.cpp b/src/join.cppnew file mode 100644index 00000000..7379d973--- /dev/null+++ b/src/join.cpp@@ -0,0 +1,688 @@+#include "join.h"++#include <collection_manager.h>+#include "collection.h"+#include "logger.h"+#include <timsort.hpp>++Option<bool> single_value_filter_query(nlohmann::json& document, const std::string& field_name,+ const std::string& ref_field_type, std::string& filter_query) {+ auto const& value = document[field_name];++ if (value.is_null()) {+ return Option<bool>(422, "Field `" + field_name + "` has `null` value.");+ }++ if (value.is_string() && ref_field_type == field_types::STRING) {+ filter_query += value.get<std::string>();+ } else if (value.is_number_integer() && (ref_field_type == field_types::INT64 ||+ (ref_field_type == field_types::INT32 &&+ StringUtils::is_int32_t(std::to_string(value.get<int64_t>()))))) {+ filter_query += std::to_string(value.get<int64_t>());+ } else {+ return Option<bool>(400, "Field `" + field_name + "` must have `" + ref_field_type + "` value.");+ }++ return Option<bool>(true);+}++Option<bool> Join::add_reference_helper_fields(nlohmann::json& document,+ const tsl::htrie_map<char, field>& schema,+ const spp::sparse_hash_map<std::string, reference_info_t>& reference_fields,+ tsl::htrie_set<char>& object_reference_helper_fields,+ const bool& is_update) {+ tsl::htrie_set<char> flat_fields;+ if (!reference_fields.empty() && document.contains(".flat")) {+ for (const auto &item: document[".flat"].get<std::vector<std::string>>()) {+ flat_fields.insert(item);+ }+ }++ // Add reference helper fields in the document.+ for (auto const& pair: reference_fields) {+ auto field_name = pair.first;+ auto const reference_helper_field = field_name + fields::REFERENCE_HELPER_FIELD_SUFFIX;++ auto const& field = schema.at(field_name);+ auto const& optional = field.optional;+ auto const& is_async_reference = field.is_async_reference;+ // Strict checking for presence of non-optional reference field during indexing operation.+ auto is_required = !is_update && !optional;+ if (is_required && document.count(field_name) != 1) {+ return Option<bool>(400, "Missing the required reference field `" + field_name+ + "` in the document.");+ } else if (document.count(field_name) != 1) {+ if (is_update) {+ document[fields::reference_helper_fields] += reference_helper_field;+ }+ continue;+ }++ auto reference_pair = pair.second;+ auto reference_collection_name = reference_pair.collection;+ auto reference_field_name = reference_pair.field;+ auto& cm = CollectionManager::get_instance();+ auto ref_collection = cm.get_collection(reference_collection_name);++ if (is_update && document.contains(reference_helper_field) &&+ (!document[field_name].is_array() || document[field_name].size() == document[reference_helper_field].size())) {+ // No need to look up the reference collection since reference helper field is already populated.+ // Saves needless computation in cases where references are known beforehand. For example, when cascade+ // deleting the related docs.+ document[fields::reference_helper_fields] += reference_helper_field;+ continue;+ }++ if (ref_collection == nullptr && is_async_reference) {+ document[fields::reference_helper_fields] += reference_helper_field;+ if (document[field_name].is_array()) {+ document[reference_helper_field] = nlohmann::json::array();+ // Having the same number of values makes it easier to update the references in the future.+ document[reference_helper_field].insert(document[reference_helper_field].begin(),+ document[field_name].size(),+ Collection::reference_helper_sentinel_value);+ } else {+ document[reference_helper_field] = Collection::reference_helper_sentinel_value;+ }++ continue;+ } else if (ref_collection == nullptr) {+ return Option<bool>(400, "Referenced collection `" + reference_collection_name+ + "` not found.");+ }++ bool is_object_reference_field = flat_fields.count(field_name) != 0;+ std::string object_key;+ bool is_object_array = false;+ if (is_object_reference_field) {+ object_reference_helper_fields.insert(reference_helper_field);++ std::vector<std::string> tokens;+ StringUtils::split(field_name, tokens, ".");+ if (schema.count(tokens[0]) == 0) {+ return Option<bool>(400, "Could not find `" + tokens[0] + "` object/object[] field in the schema.");+ }+ object_key = tokens[0];+ is_object_array = schema.at(object_key).is_array();+ }++ if (reference_field_name == "id") {+ auto id_field_type_error_op = Option<bool>(400, "Field `" + field_name + "` must have string value.");+ if (is_object_array) {+ if (!document[field_name].is_array()) {+ return Option<bool>(400, "Expected `" + field_name + "` to be an array.");+ }++ document[reference_helper_field] = nlohmann::json::array();+ document[fields::reference_helper_fields] += reference_helper_field;++ std::vector<std::string> keys;+ StringUtils::split(field_name, keys, ".");+ auto const& object_array = document[keys[0]];++ for (uint32_t i = 0; i < object_array.size(); i++) {+ if (optional && object_array[i].count(keys[1]) == 0) {+ continue;+ } else if (object_array[i].count(keys[1]) == 0) {+ return Option<bool>(400, "Object at index `" + std::to_string(i) + "` is missing `" + field_name + "`.");+ } else if (!object_array[i].at(keys[1]).is_string()) {+ return id_field_type_error_op;+ }++ auto id = object_array[i].at(keys[1]).get<std::string>();+ auto ref_doc_id_op = ref_collection->doc_id_to_seq_id_with_lock(id);+ if (!ref_doc_id_op.ok() && is_async_reference) {+ auto const& value = nlohmann::json::array({i, Collection::reference_helper_sentinel_value});+ document[reference_helper_field] += value;+ } else if (!ref_doc_id_op.ok()) {+ return Option<bool>(400, "Referenced document having `id: " + id ++ "` not found in the collection `" +=+ reference_collection_name + "`." );+ } else {+ // Adding the index of the object along with referenced doc id to account for the scenario where a+ // reference field of an object array might be optional and missing.+ document[reference_helper_field] += nlohmann::json::array({i, ref_doc_id_op.get()});+ }+ }+ } else if (document[field_name].is_array()) {+ document[reference_helper_field] = nlohmann::json::array();+ document[fields::reference_helper_fields] += reference_helper_field;++ for (const auto &item: document[field_name].items()) {+ if (optional && item.value().is_null()) {+ continue;+ } else if (!item.value().is_string()) {+ return id_field_type_error_op;+ }++ auto id = item.value().get<std::string>();+ auto ref_doc_id_op = ref_collection->doc_id_to_seq_id_with_lock(id);+ if (!ref_doc_id_op.ok() && is_async_reference) {+ document[reference_helper_field] += Collection::reference_helper_sentinel_value;+ } else if (!ref_doc_id_op.ok()) {+ return Option<bool>(400, "Referenced document having `id: " + id ++ "` not found in the collection `" +=+ reference_collection_name + "`." );+ } else {+ document[reference_helper_field] += ref_doc_id_op.get();+ }+ }+ } else if (document[field_name].is_string()) {+ document[fields::reference_helper_fields] += reference_helper_field;++ auto id = document[field_name].get<std::string>();+ auto ref_doc_id_op = ref_collection->doc_id_to_seq_id_with_lock(id);+ if (!ref_doc_id_op.ok() && is_async_reference) {+ document[reference_helper_field] = Collection::reference_helper_sentinel_value;+ } else if (!ref_doc_id_op.ok()) {+ return Option<bool>(400, "Referenced document having `id: " + id ++ "` not found in the collection `" +=+ reference_collection_name + "`." );+ } else {+ document[reference_helper_field] = ref_doc_id_op.get();+ }+ } else if (optional && document[field_name].is_null()) {+ // Reference helper field should also be removed along with reference field.+ if (is_update) {+ document[reference_helper_field] = nullptr;+ }+ continue;+ } else {+ return id_field_type_error_op;+ }++ continue;+ }++ if (ref_collection->get_schema().count(reference_field_name) == 0) {+ return Option<bool>(400, "Referenced field `" + reference_field_name ++ "` not found in the collection `" += reference_collection_name + "`.");+ }++ auto const ref_field = ref_collection->get_schema().at(reference_field_name);+ if (!ref_field.index) {+ return Option<bool>(400, "Referenced field `" + reference_field_name ++ "` in the collection `" += reference_collection_name + "` must be indexed.");+ }++ std::string ref_field_type = ref_field.is_string() ? field_types::STRING :+ ref_field.is_int32() ? field_types::INT32 :+ ref_field.is_int64() ? field_types::INT64 : field_types::NIL;++ if (ref_field_type == field_types::NIL) {+ return Option<bool>(400, "Cannot add a reference to `" + reference_collection_name + "." += reference_field_name ++ "` of type `" += ref_field.type + "`.");+ }++ if (is_object_array) {+ if (!document[field_name].is_array()) {+ return Option<bool>(400, "Expected `" + field_name + "` to be an array.");+ }++ document[reference_helper_field] = nlohmann::json::array();+ document[fields::reference_helper_fields] += reference_helper_field;+ nlohmann::json temp_doc; // To store singular values of `field_name` field.++ std::vector<std::string> keys;+ StringUtils::split(field_name, keys, ".");+ auto const& object_array = document[keys[0]];++ for (uint32_t i = 0; i < object_array.size(); i++) {+ if (optional && object_array[i].count(keys[1]) == 0) {+ continue;+ } else if (object_array[i].count(keys[1]) == 0) {+ return Option<bool>(400, "Object at index `" + std::to_string(i) + "` is missing `" + field_name + "`.");+ }++ temp_doc[field_name] = object_array[i].at(keys[1]);+ std::string filter_query = reference_field_name + ":= ";++ auto single_value_filter_query_op = single_value_filter_query(temp_doc, field_name, ref_field_type,+ filter_query);+ if (!single_value_filter_query_op.ok()) {+ if (optional && single_value_filter_query_op.code() == 422) {+ continue;+ }+ return Option<bool>(400, single_value_filter_query_op.error());+ }++ filter_result_t filter_result;+ auto filter_ids_op = ref_collection->get_filter_ids(filter_query, filter_result);+ if (!filter_ids_op.ok()) {+ return filter_ids_op;+ }++ if (filter_result.count == 0 && is_async_reference) {+ document[reference_helper_field] += nlohmann::json::array({i, Collection::reference_helper_sentinel_value});+ } else if (filter_result.count != 1) {+ // Constraints similar to foreign key apply here. The reference match must be unique and not null.+ return Option<bool>(400, filter_result.count < 1 ?+ "Reference document having `" + filter_query + "` not found in the collection `"+ += reference_collection_name + "`." :+ "Multiple documents having `" + filter_query + "` found in the collection `" +=+ reference_collection_name + "`.");+ } else {+ // Adding the index of the object along with referenced doc id to account for the scenario where a+ // reference field of an object array might be optional and missing.+ document[reference_helper_field] += nlohmann::json::array({i, filter_result.docs[0]});+ }+ }+ continue;+ }++ auto const is_reference_array_field = field.is_array();+ std::vector<std::string> filter_values;+ if (is_reference_array_field) {+ if (document[field_name].is_null()) {+ document[reference_helper_field] = nlohmann::json::array();+ document[fields::reference_helper_fields] += reference_helper_field;++ continue;+ } else if (!document[field_name].is_array()) {+ return Option<bool>(400, "Expected `" + field_name + "` to be an array.");+ }++ for (const auto &item: document[field_name].items()) {+ auto const& item_value = item.value();+ if (item_value.is_string() && ref_field_type == field_types::STRING) {+ filter_values.emplace_back(item_value.get<std::string>());+ } else if (item_value.is_number_integer() && (ref_field_type == field_types::INT64 ||+ (ref_field_type == field_types::INT32 &&+ StringUtils::is_int32_t(std::to_string(item_value.get<int64_t>()))))) {+ filter_values.emplace_back(std::to_string(item_value.get<int64_t>()));+ } else {+ return Option<bool>(400, "Field `" + field_name + "` must only have `" += ref_field_type + "` values.");+ }+ }++ document[reference_helper_field] = nlohmann::json::array();+ document[fields::reference_helper_fields] += reference_helper_field;++ if (filter_values.empty()) {+ continue;+ }+ } else {+ std::string value;+ auto single_value_filter_query_op = single_value_filter_query(document, field_name, ref_field_type, value);+ if (!single_value_filter_query_op.ok()) {+ if (optional && single_value_filter_query_op.code() == 422) {+ // Reference helper field should also be removed along with reference field.+ if (is_update) {+ document[reference_helper_field] = nullptr;+ }+ continue;+ }+ return Option<bool>(400, single_value_filter_query_op.error());+ }++ filter_values.emplace_back(value);+ document[fields::reference_helper_fields] += reference_helper_field;+ }++ for (const auto& filter_value: filter_values) {+ std::string filter_query = reference_field_name + (field.is_string() ? ":= " : ": ") += filter_value;+ filter_result_t filter_result;+ auto filter_ids_op = ref_collection->get_filter_ids(filter_query, filter_result);+ if (!filter_ids_op.ok()) {+ return filter_ids_op;+ }++ if (filter_result.count == 0 && is_async_reference) {+ if (is_reference_array_field) {+ document[reference_helper_field] += Collection::reference_helper_sentinel_value;+ } else {+ document[reference_helper_field] = Collection::reference_helper_sentinel_value;+ }+ } else if (filter_result.count != 1) {+ // Constraints similar to foreign key apply here. The reference match must be unique and not null.+ return Option<bool>(400, filter_result.count < 1 ?+ "Reference document having `" + filter_query + "` not found in the collection `"+ += reference_collection_name + "`." :+ "Multiple documents having `" + filter_query + "` found in the collection `" +=+ reference_collection_name + "`.");+ } else {+ if (is_reference_array_field) {+ document[reference_helper_field] += filter_result.docs[0];+ } else {+ document[reference_helper_field] = filter_result.docs[0];+ }+ }+ }+ }++ return Option<bool>(true);+}++Option<bool> Join::prune_ref_doc(nlohmann::json& doc,+ const reference_filter_result_t& references,+ const tsl::htrie_set<char>& ref_include_fields_full,+ const tsl::htrie_set<char>& ref_exclude_fields_full,+ const bool& is_reference_array,+ const ref_include_exclude_fields& ref_include_exclude) {+ nlohmann::json original_doc;+ if (!ref_include_exclude.nested_join_includes.empty()) {+ original_doc = doc;+ }++ auto const& ref_collection_name = ref_include_exclude.collection_name;+ auto& cm = CollectionManager::get_instance();+ auto ref_collection = cm.get_collection(ref_collection_name);+ if (ref_collection == nullptr) {+ return Option<bool>(400, "Referenced collection `" + ref_collection_name + "` in `include_fields` not found.");+ }++ auto const& alias = ref_include_exclude.alias;+ auto const& strategy = ref_include_exclude.strategy;+ auto error_prefix = "Referenced collection `" + ref_collection_name + "`: ";++ // One-to-one relation.+ if (strategy != ref_include::nest_array && !is_reference_array && references.count == 1) {+ auto ref_doc_seq_id = references.docs[0];++ nlohmann::json ref_doc;+ auto get_doc_op = ref_collection->get_document_from_store(ref_doc_seq_id, ref_doc);+ if (!get_doc_op.ok()) {+ if (ref_doc_seq_id == Collection::reference_helper_sentinel_value) {+ return Option<bool>(true);+ }+ return Option<bool>(get_doc_op.code(), error_prefix + get_doc_op.error());+ }++ Collection::remove_flat_fields(ref_doc);+ Collection::remove_reference_helper_fields(ref_doc);++ auto prune_op = Collection::prune_doc(ref_doc, ref_include_fields_full, ref_exclude_fields_full);+ if (!prune_op.ok()) {+ return Option<bool>(prune_op.code(), error_prefix + prune_op.error());+ }++ auto const key = alias.empty() ? ref_collection_name : alias;+ auto const& nest_ref_doc = (strategy == ref_include::nest);+ if (!ref_doc.empty()) {+ if (nest_ref_doc) {+ doc[key] = ref_doc;+ } else {+ if (!alias.empty()) {+ auto temp_doc = ref_doc;+ ref_doc.clear();+ for (const auto &item: temp_doc.items()) {+ ref_doc[alias + item.key()] = item.value();+ }+ }+ doc.update(ref_doc);+ }+ }++ // Include nested join references.+ if (!ref_include_exclude.nested_join_includes.empty()) {+ // Passing empty references in case the nested include collection is not joined, but it still can be included+ // if we have a reference to it.+ std::map<std::string, reference_filter_result_t> refs;+ auto nested_include_exclude_op = include_references(nest_ref_doc ? doc[key] : doc, ref_doc_seq_id,+ ref_collection.get(),+ references.coll_to_references == nullptr ? refs :+ references.coll_to_references[0],+ ref_include_exclude.nested_join_includes, original_doc);+ if (!nested_include_exclude_op.ok()) {+ return nested_include_exclude_op;+ }+ }++ return Option<bool>(true);+ }++ // One-to-many relation.+ for (uint32_t i = 0; i < references.count; i++) {+ auto ref_doc_seq_id = references.docs[i];++ nlohmann::json ref_doc;+ std::string key;+ auto const& nest_ref_doc = (strategy == ref_include::nest || strategy == ref_include::nest_array);++ auto get_doc_op = ref_collection->get_document_from_store(ref_doc_seq_id, ref_doc);+ if (!get_doc_op.ok()) {+ // Referenced document is not yet indexed.+ if (ref_doc_seq_id == Collection::reference_helper_sentinel_value) {+ continue;+ }+ return Option<bool>(get_doc_op.code(), error_prefix + get_doc_op.error());+ }++ Collection::remove_flat_fields(ref_doc);+ Collection::remove_reference_helper_fields(ref_doc);++ auto prune_op = Collection::prune_doc(ref_doc, ref_include_fields_full, ref_exclude_fields_full);+ if (!prune_op.ok()) {+ return Option<bool>(prune_op.code(), error_prefix + prune_op.error());+ }++ if (!ref_doc.empty()) {+ if (nest_ref_doc) {+ key = alias.empty() ? ref_collection_name : alias;+ if (doc.contains(key) && !doc[key].is_array()) {+ return Option<bool>(400, "Could not include the reference document of `" + ref_collection_name ++ "` collection. Expected `" += key + "` to be an array. Try " ++ (alias.empty() ? "adding an" : "renaming the") + " alias.");+ }++ doc[key] += ref_doc;+ } else {+ for (auto ref_doc_it = ref_doc.begin(); ref_doc_it != ref_doc.end(); ref_doc_it++) {+ auto const& ref_doc_key = ref_doc_it.key();+ key = alias + ref_doc_key;+ if (doc.contains(key) && !doc[key].is_array()) {+ return Option<bool>(400, "Could not include the value of `" + ref_doc_key ++ "` key of the reference document of `" += ref_collection_name ++ "` collection. Expected `" += key + "` to be an array. Try " ++ (alias.empty() ? "adding an" : "renaming the") + " alias.");+ }++ // Add the values of ref_doc as JSON array into doc.+ doc[key] += ref_doc_it.value();+ }+ }+ }++ // Include nested join references.+ if (!ref_include_exclude.nested_join_includes.empty()) {+ // Passing empty references in case the nested include collection is not joined, but it still can be included+ // if we have a reference to it.+ std::map<std::string, reference_filter_result_t> refs;+ auto nested_include_exclude_op = include_references(nest_ref_doc ? doc[key].at(i) : doc, ref_doc_seq_id,+ ref_collection.get(),+ references.coll_to_references == nullptr ? refs :+ references.coll_to_references[i],+ ref_include_exclude.nested_join_includes, original_doc);+ if (!nested_include_exclude_op.ok()) {+ return nested_include_exclude_op;+ }+ }+ }++ return Option<bool>(true);+}++Option<bool> Join::include_references(nlohmann::json& doc, const uint32_t& seq_id, Collection *const collection,+ const std::map<std::string, reference_filter_result_t>& reference_filter_results,+ const std::vector<ref_include_exclude_fields>& ref_include_exclude_fields_vec,+ const nlohmann::json& original_doc) {+ for (auto const& ref_include_exclude: ref_include_exclude_fields_vec) {+ auto ref_collection_name = ref_include_exclude.collection_name;++ auto& cm = CollectionManager::get_instance();+ auto ref_collection = cm.get_collection(ref_collection_name);+ if (ref_collection == nullptr) {+ return Option<bool>(400, "Referenced collection `" + ref_collection_name + "` in `include_fields` not found.");+ }+ // `CollectionManager::get_collection` accounts for collection alias being used and provides pointer to the+ // original collection.+ ref_collection_name = ref_collection->get_name();++ auto const joined_on_ref_collection = reference_filter_results.count(ref_collection_name) > 0,+ has_filter_reference = (joined_on_ref_collection &&+ reference_filter_results.at(ref_collection_name).count > 0);+ auto doc_has_reference = false, joined_coll_has_reference = false;++ // Reference include_by without join, check if doc itself contains the reference.+ if (!joined_on_ref_collection && collection != nullptr) {+ doc_has_reference = ref_collection->is_referenced_in(collection->get_name());+ }++ std::string joined_coll_having_reference;+ // Check if the joined collection has a reference.+ if (!joined_on_ref_collection && !doc_has_reference) {+ for (const auto &reference_filter_result: reference_filter_results) {+ joined_coll_has_reference = ref_collection->is_referenced_in(reference_filter_result.first);+ if (joined_coll_has_reference) {+ joined_coll_having_reference = reference_filter_result.first;+ break;+ }+ }+ }++ if (!has_filter_reference && !doc_has_reference && !joined_coll_has_reference) {+ continue;+ }++ std::vector<std::string> ref_include_fields_vec, ref_exclude_fields_vec;+ StringUtils::split(ref_include_exclude.include_fields, ref_include_fields_vec, ",");+ StringUtils::split(ref_include_exclude.exclude_fields, ref_exclude_fields_vec, ",");++ spp::sparse_hash_set<std::string> ref_include_fields, ref_exclude_fields;+ ref_include_fields.insert(ref_include_fields_vec.begin(), ref_include_fields_vec.end());+ ref_exclude_fields.insert(ref_exclude_fields_vec.begin(), ref_exclude_fields_vec.end());++ tsl::htrie_set<char> ref_include_fields_full, ref_exclude_fields_full;+ auto include_exclude_op = ref_collection->populate_include_exclude_fields_lk(ref_include_fields,+ ref_exclude_fields,+ ref_include_fields_full,+ ref_exclude_fields_full);+ auto error_prefix = "Referenced collection `" + ref_collection_name + "`: ";+ if (!include_exclude_op.ok()) {+ return Option<bool>(include_exclude_op.code(), error_prefix + include_exclude_op.error());+ }++ Option<bool> prune_doc_op = Option<bool>(true);+ auto const& ref_collection_alias = ref_include_exclude.alias;+ if (has_filter_reference) {+ auto const& ref_filter_result = reference_filter_results.at(ref_collection_name);+ prune_doc_op = prune_ref_doc(doc, ref_filter_result, ref_include_fields_full, ref_exclude_fields_full,+ ref_filter_result.is_reference_array_field, ref_include_exclude);+ } else if (doc_has_reference) {+ auto get_reference_field_op = ref_collection->get_referenced_in_field_with_lock(collection->get_name());+ if (!get_reference_field_op.ok()) {+ continue;+ }+ auto const& field_name = get_reference_field_op.get();+ auto const& reference_helper_field_name = field_name + fields::REFERENCE_HELPER_FIELD_SUFFIX;+ if (collection->get_schema().count(reference_helper_field_name) == 0) {+ continue;+ }++ if (collection->get_object_reference_helper_fields().count(field_name) != 0) {+ std::vector<std::string> keys;+ StringUtils::split(field_name, keys, ".");+ auto const& key = keys[0];++ if (!doc.contains(key)) {+ if (!original_doc.contains(key)) {+ return Option<bool>(400, "Could not find `" + key ++ "` key in the document to include the referenced document.");+ }++ // The key is excluded from the doc by the query, inserting empty object(s) so referenced doc can be+ // included in it.+ if (original_doc[key].is_array()) {+ doc[key] = nlohmann::json::array();+ doc[key].insert(doc[key].begin(), original_doc[key].size(), nlohmann::json::object());+ } else {+ doc[key] = nlohmann::json::object();+ }+ }++ if (doc[key].is_array()) {+ for (uint32_t i = 0; i < doc[key].size(); i++) {+ uint32_t ref_doc_id;+ auto op = collection->get_object_array_related_id(reference_helper_field_name, seq_id, i, ref_doc_id);+ if (!op.ok()) {+ if (op.code() == 404) { // field_name is not indexed.+ break;+ } else { // No reference found for this object.+ continue;+ }+ }++ reference_filter_result_t result(1, new uint32_t[1]{ref_doc_id});+ prune_doc_op = prune_ref_doc(doc[key][i], result,+ ref_include_fields_full, ref_exclude_fields_full,+ false, ref_include_exclude);+ if (!prune_doc_op.ok()) {+ return prune_doc_op;+ }+ }+ } else {+ std::vector<uint32_t> ids;+ auto get_references_op = collection->get_related_ids(field_name, seq_id, ids);+ if (!get_references_op.ok()) {+ LOG(ERROR) << "Error while getting related ids: " + get_references_op.error();+ continue;+ }+ reference_filter_result_t result(ids.size(), &ids[0]);+ prune_doc_op = prune_ref_doc(doc[key], result, ref_include_fields_full, ref_exclude_fields_full,+ collection->get_schema().at(field_name).is_array(), ref_include_exclude);+ result.docs = nullptr;+ }+ } else {+ std::vector<uint32_t> ids;+ auto get_references_op = collection->get_related_ids(field_name, seq_id, ids);+ if (!get_references_op.ok()) {+ LOG(ERROR) << "Error while getting related ids: " + get_references_op.error();+ continue;+ }+ reference_filter_result_t result(ids.size(), &ids[0]);+ prune_doc_op = prune_ref_doc(doc, result, ref_include_fields_full, ref_exclude_fields_full,+ collection->get_schema().at(field_name).is_array(), ref_include_exclude);+ result.docs = nullptr;+ }+ } else if (joined_coll_has_reference) {+ auto joined_collection = cm.get_collection(joined_coll_having_reference);+ if (joined_collection == nullptr) {+ continue;+ }++ auto reference_field_name_op = ref_collection->get_referenced_in_field_with_lock(joined_coll_having_reference);+ if (!reference_field_name_op.ok() || joined_collection->get_schema().count(reference_field_name_op.get()) == 0) {+ continue;+ }++ auto const& reference_field_name = reference_field_name_op.get();+ auto const& reference_filter_result = reference_filter_results.at(joined_coll_having_reference);+ auto const& count = reference_filter_result.count;+ std::vector<uint32_t> ids;+ ids.reserve(count);+ for (uint32_t i = 0; i < count; i++) {+ joined_collection->get_related_ids_with_lock(reference_field_name, reference_filter_result.docs[i], ids);+ }+ if (ids.empty()) {+ continue;+ }++ gfx::timsort(ids.begin(), ids.end());+ ids.erase(unique(ids.begin(), ids.end()), ids.end());++ reference_filter_result_t result;+ result.count = ids.size();+ result.docs = &ids[0];+ prune_doc_op = prune_ref_doc(doc, result, ref_include_fields_full, ref_exclude_fields_full,+ joined_collection->get_schema().at(reference_field_name).is_array(),+ ref_include_exclude);+ result.docs = nullptr;+ }++ if (!prune_doc_op.ok()) {+ return prune_doc_op;+ }+ }++ return Option<bool>(true);+}\ No newline at end of filediff --git a/test/collection_join_test.cpp b/test/collection_join_test.cppindex b8292a81..625511d1 100644--- a/test/collection_join_test.cpp+++ b/test/collection_join_test.cpp@@ -525,7 +525,14 @@ TEST_F(CollectionJoinTest, IndexDocumentHavingReferenceField) {})"_json;add_doc_op = coll2->add(doc_json.dump());ASSERT_FALSE(add_doc_op.ok());- ASSERT_EQ("Field `ref_string_array_field` must be an array of string.", add_doc_op.error());+ ASSERT_EQ("Field `ref_string_array_field` must only have `string` values.", add_doc_op.error());++ doc_json = R"({+ "ref_string_array_field": ["foo"]+ })"_json;+ add_doc_op = coll2->add(doc_json.dump());+ ASSERT_FALSE(add_doc_op.ok());+ ASSERT_EQ("Reference document having `string_array_field:= foo` not found in the collection `coll1`.", add_doc_op.error());collectionManager.drop_collection("coll2");temp_json = schema_json;@@ -556,29 +563,16 @@ TEST_F(CollectionJoinTest, IndexDocumentHavingReferenceField) {ASSERT_EQ(0, doc.count("ref_string_field_sequence_id"));ASSERT_EQ(0, doc.count(".ref"));- doc_json = R"({- "ref_string_array_field": ["foo"]- })"_json;- add_doc_op = coll2->add(doc_json.dump());- ASSERT_TRUE(add_doc_op.ok());-result = coll2->search("*", {}, "", {}, {}, {0}).get();ASSERT_EQ(0, result["hits"][0]["document"]["ref_string_array_field_sequence_id"].size());- doc = coll2->get("2").get();- ASSERT_EQ(1, doc.count("ref_string_array_field_sequence_id"));- ASSERT_EQ(0, doc["ref_string_array_field_sequence_id"].size());- ASSERT_EQ(1, doc.count(".ref"));- ASSERT_EQ(1, doc[".ref"].size());- ASSERT_EQ("ref_string_array_field_sequence_id", doc[".ref"][0]);-doc_json = R"({- "ref_string_array_field": ["b", "foo"]+ "ref_string_array_field": ["b"]})"_json;add_doc_op = coll2->add(doc_json.dump());ASSERT_TRUE(add_doc_op.ok());- doc = coll2->get("3").get();+ doc = coll2->get("2").get();ASSERT_EQ(1, doc.count("ref_string_array_field_sequence_id"));ASSERT_EQ(1, doc["ref_string_array_field_sequence_id"].size());ASSERT_EQ(0, doc["ref_string_array_field_sequence_id"][0]);@@ -589,7 +583,7 @@ TEST_F(CollectionJoinTest, IndexDocumentHavingReferenceField) {add_doc_op = coll2->add(doc_json.dump());ASSERT_TRUE(add_doc_op.ok());- doc = coll2->get("4").get();+ doc = coll2->get("3").get();ASSERT_EQ(1, doc.count("ref_string_array_field_sequence_id"));ASSERT_EQ(2, doc["ref_string_array_field_sequence_id"].size());ASSERT_EQ(0, doc["ref_string_array_field_sequence_id"][0]);@@ -665,6 +659,13 @@ TEST_F(CollectionJoinTest, IndexDocumentHavingReferenceField) {ASSERT_FALSE(add_doc_op.ok());ASSERT_EQ("Field `ref_int32_array_field` must only have `int32` values.", add_doc_op.error());+ doc_json = R"({+ "ref_int32_array_field": [1]+ })"_json;+ add_doc_op = coll2->add(doc_json.dump());+ ASSERT_FALSE(add_doc_op.ok());+ ASSERT_EQ("Reference document having `int32_array_field: 1` not found in the collection `coll1`.", add_doc_op.error());+collectionManager.drop_collection("coll2");temp_json = schema_json;collection_create_op = collectionManager.create_collection(temp_json);@@ -685,26 +686,13 @@ TEST_F(CollectionJoinTest, IndexDocumentHavingReferenceField) {ASSERT_EQ("ref_int32_field_sequence_id", doc[".ref"][0]);doc_json = R"({- "ref_int32_array_field": [1]+ "ref_int32_array_field": [2]})"_json;add_doc_op = coll2->add(doc_json.dump());ASSERT_TRUE(add_doc_op.ok());doc = coll2->get("1").get();ASSERT_EQ(1, doc.count("ref_int32_array_field_sequence_id"));- ASSERT_EQ(0, doc["ref_int32_array_field_sequence_id"].size());- ASSERT_EQ(1, doc.count(".ref"));- ASSERT_EQ(1, doc[".ref"].size());- ASSERT_EQ("ref_int32_array_field_sequence_id", doc[".ref"][0]);-- doc_json = R"({- "ref_int32_array_field": [1, 2]- })"_json;- add_doc_op = coll2->add(doc_json.dump());- ASSERT_TRUE(add_doc_op.ok());-- doc = coll2->get("2").get();- ASSERT_EQ(1, doc.count("ref_int32_array_field_sequence_id"));ASSERT_EQ(1, doc["ref_int32_array_field_sequence_id"].size());ASSERT_EQ(3, doc["ref_int32_array_field_sequence_id"][0]);@@ -714,7 +702,7 @@ TEST_F(CollectionJoinTest, IndexDocumentHavingReferenceField) {add_doc_op = coll2->add(doc_json.dump());ASSERT_TRUE(add_doc_op.ok());- doc = coll2->get("3").get();+ doc = coll2->get("2").get();ASSERT_EQ(1, doc.count("ref_int32_array_field_sequence_id"));ASSERT_EQ(2, doc["ref_int32_array_field_sequence_id"].size());ASSERT_EQ(3, doc["ref_int32_array_field_sequence_id"][0]);@@ -726,7 +714,7 @@ TEST_F(CollectionJoinTest, IndexDocumentHavingReferenceField) {add_doc_op = coll2->add(doc_json.dump());ASSERT_TRUE(add_doc_op.ok());- doc = coll2->get("4").get();+ doc = coll2->get("3").get();ASSERT_EQ(1, doc.count("ref_int32_array_field_sequence_id"));ASSERT_EQ(1, doc["ref_int32_array_field_sequence_id"].size());ASSERT_EQ(3, doc["ref_int32_array_field_sequence_id"][0]);@@ -794,6 +782,13 @@ TEST_F(CollectionJoinTest, IndexDocumentHavingReferenceField) {ASSERT_FALSE(add_doc_op.ok());ASSERT_EQ("Field `ref_int64_array_field` must only have `int64` values.", add_doc_op.error());+ doc_json = R"({+ "ref_int64_array_field": [1]+ })"_json;+ add_doc_op = coll2->add(doc_json.dump());+ ASSERT_FALSE(add_doc_op.ok());+ ASSERT_EQ("Reference document having `int64_array_field: 1` not found in the collection `coll1`.", add_doc_op.error());+collectionManager.drop_collection("coll2");temp_json = schema_json;collection_create_op = collectionManager.create_collection(temp_json);@@ -814,26 +809,13 @@ TEST_F(CollectionJoinTest, IndexDocumentHavingReferenceField) {ASSERT_EQ("ref_int64_field_sequence_id", doc[".ref"][0]);doc_json = R"({- "ref_int64_array_field": [1]+ "ref_int64_array_field": [2]})"_json;add_doc_op = coll2->add(doc_json.dump());ASSERT_TRUE(add_doc_op.ok());doc = coll2->get("1").get();ASSERT_EQ(1, doc.count("ref_int64_array_field_sequence_id"));- ASSERT_EQ(0, doc["ref_int64_array_field_sequence_id"].size());- ASSERT_EQ(1, doc.count(".ref"));- ASSERT_EQ(1, doc[".ref"].size());- ASSERT_EQ("ref_int64_array_field_sequence_id", doc[".ref"][0]);-- doc_json = R"({- "ref_int64_array_field": [1, 2]- })"_json;- add_doc_op = coll2->add(doc_json.dump());- ASSERT_TRUE(add_doc_op.ok());-- doc = coll2->get("2").get();- ASSERT_EQ(1, doc.count("ref_int64_array_field_sequence_id"));ASSERT_EQ(1, doc["ref_int64_array_field_sequence_id"].size());ASSERT_EQ(6, doc["ref_int64_array_field_sequence_id"][0]);@@ -843,7 +825,7 @@ TEST_F(CollectionJoinTest, IndexDocumentHavingReferenceField) {add_doc_op = coll2->add(doc_json.dump());ASSERT_TRUE(add_doc_op.ok());- doc = coll2->get("3").get();+ doc = coll2->get("2").get();ASSERT_EQ(1, doc.count("ref_int64_array_field_sequence_id"));ASSERT_EQ(2, doc["ref_int64_array_field_sequence_id"].size());ASSERT_EQ(6, doc["ref_int64_array_field_sequence_id"][0]);@@ -855,7 +837,7 @@ TEST_F(CollectionJoinTest, IndexDocumentHavingReferenceField) {add_doc_op = coll2->add(doc_json.dump());ASSERT_TRUE(add_doc_op.ok());- doc = coll2->get("4").get();+ doc = coll2->get("3").get();ASSERT_EQ(1, doc.count("ref_int64_array_field_sequence_id"));ASSERT_EQ(1, doc["ref_int64_array_field_sequence_id"].size());ASSERT_EQ(6, doc["ref_int64_array_field_sequence_id"][0]);@@ -933,6 +915,15 @@ TEST_F(CollectionJoinTest, IndexDocumentHavingReferenceField) {ASSERT_FALSE(add_doc_op.ok());ASSERT_EQ("Reference document having `string_array_field:= foo` not found in the collection `coll1`.", add_doc_op.error());+ doc_json = R"({+ "object": {+ "ref_array_field": ["foo"]+ }+ })"_json;+ add_doc_op = coll2->add(doc_json.dump());+ ASSERT_FALSE(add_doc_op.ok());+ ASSERT_EQ("Reference document having `string_array_field:= foo` not found in the collection `coll1`.", add_doc_op.error());+collectionManager.drop_collection("coll2");temp_json = schema_json;collection_create_op = collectionManager.create_collection(temp_json);@@ -957,7 +948,7 @@ TEST_F(CollectionJoinTest, IndexDocumentHavingReferenceField) {doc_json = R"({"object": {- "ref_array_field": ["foo"]+ "ref_array_field": ["b"]}})"_json;add_doc_op = coll2->add(doc_json.dump());@@ -965,21 +956,6 @@ TEST_F(CollectionJoinTest, IndexDocumentHavingReferenceField) {doc = coll2->get("1").get();ASSERT_EQ(1, doc.count("object.ref_array_field_sequence_id"));- ASSERT_EQ(0, doc["object.ref_array_field_sequence_id"].size());- ASSERT_EQ(1, doc.count(".ref"));- ASSERT_EQ(1, doc[".ref"].size());- ASSERT_EQ("object.ref_array_field_sequence_id", doc[".ref"][0]);-- doc_json = R"({- "object": {- "ref_array_field": ["b", "foo"]- }- })"_json;- add_doc_op = coll2->add(doc_json.dump());- ASSERT_TRUE(add_doc_op.ok());-- doc = coll2->get("2").get();- ASSERT_EQ(1, doc.count("object.ref_array_field_sequence_id"));ASSERT_EQ(1, doc["object.ref_array_field_sequence_id"].size());ASSERT_EQ(0, doc["object.ref_array_field_sequence_id"][0]);@@ -996,7 +972,7 @@ TEST_F(CollectionJoinTest, IndexDocumentHavingReferenceField) {add_doc_op = coll2->add(doc_json.dump());ASSERT_TRUE(add_doc_op.ok());- doc = coll2->get("3").get();+ doc = coll2->get("2").get();ASSERT_EQ(1, doc.count("object_array.ref_array_field_sequence_id"));ASSERT_EQ(2, doc["object_array.ref_array_field_sequence_id"].size());ASSERT_EQ(2, doc["object_array.ref_array_field_sequence_id"][0].size());@@ -1076,6 +1052,644 @@ TEST_F(CollectionJoinTest, IndexDocumentHavingReferenceField) {ASSERT_EQ("Cannot add a reference to `coll1.object_array_field` of type `object[]`.", add_doc_op.error());}+TEST_F(CollectionJoinTest, IndexDocumentHavingAsyncReferenceField) {+ auto schema_json =+ R"({+ "name": "Customers",+ "fields": [+ {"name": "customer_id", "type": "string"},+ {"name": "customer_name", "type": "string"},+ {"name": "product_price", "type": "float"},+ {"name": "product_id", "type": "string", "reference": "Products.product_id", "async_reference": true}+ ]+ })"_json;+ std::vector<nlohmann::json> documents = {+ R"({+ "customer_id": "customer_a",+ "customer_name": "Joe",+ "product_price": 143,+ "product_id": "product_a"+ })"_json,+ R"({+ "customer_id": "customer_a",+ "customer_name": "Joe",+ "product_price": 73.5,+ "product_id": "product_b"+ })"_json,+ R"({+ "customer_id": "customer_b",+ "customer_name": "Dan",+ "product_price": 75,+ "product_id": "product_a"+ })"_json+ };++ auto collection_create_op = collectionManager.create_collection(schema_json);+ ASSERT_TRUE(collection_create_op.ok());+ for (auto const &json: documents) {+ auto add_op = collection_create_op.get()->add(json.dump());+ if (!add_op.ok()) {+ LOG(INFO) << add_op.error();+ }+ ASSERT_TRUE(add_op.ok());+ }++ for (auto i = 0; i < 3; i++) {+ auto const doc_id = std::to_string(i);+ auto doc = collection_create_op.get()->get(doc_id).get();+ ASSERT_EQ(doc_id, doc["id"]);++ ASSERT_EQ(1, doc.count(".ref"));+ ASSERT_EQ(1, doc[".ref"].size());+ ASSERT_EQ("product_id_sequence_id", doc[".ref"][0]);++ ASSERT_EQ(1, doc.count("product_id_sequence_id"));+ // Referenced documents don't exist yet, so dummy value is present in the reference helper field.+ ASSERT_EQ(UINT32_MAX, doc["product_id_sequence_id"]);+ }++ schema_json =+ R"({+ "name": "coll1",+ "fields": [+ {"name": "coll_id", "type": "string"},+ {+ "name": "object.reference",+ "type": "string",+ "reference": "Products.product_id",+ "optional": true,+ "async_reference": true+ },+ {"name": "object", "type": "object"}+ ],+ "enable_nested_fields": true+ })"_json;+ documents = {+ R"({+ "coll_id": "a",+ "object": {}+ })"_json,+ R"({+ "coll_id": "b",+ "object": {+ "reference": "product_b"+ }+ })"_json,+ R"({+ "coll_id": "c",+ "object": {+ "reference": "product_a"+ }+ })"_json+ };++ collection_create_op = collectionManager.create_collection(schema_json);+ ASSERT_TRUE(collection_create_op.ok());+ for (auto const &json: documents) {+ auto add_op = collection_create_op.get()->add(json.dump());+ if (!add_op.ok()) {+ LOG(INFO) << add_op.error();+ }+ ASSERT_TRUE(add_op.ok());+ }++ for (auto i = 0; i < 3; i++) {+ auto const doc_id = std::to_string(i);+ auto doc = collection_create_op.get()->get(doc_id).get();+ ASSERT_EQ(doc_id, doc["id"]);++ if (i == 0) {+ ASSERT_EQ(0, doc.count(".ref"));+ ASSERT_EQ(0, doc.count("object.reference_sequence_id"));+ continue;+ }++ ASSERT_EQ(1, doc.count(".ref"));+ ASSERT_EQ(1, doc[".ref"].size());+ ASSERT_EQ("object.reference_sequence_id", doc[".ref"][0]);++ ASSERT_EQ(1, doc.count("object.reference_sequence_id"));+ // Referenced documents don't exist yet, so dummy value is present in the reference helper field.+ ASSERT_EQ(UINT32_MAX, doc["object.reference_sequence_id"]);+ }++ schema_json =+ R"({+ "name": "Products",+ "fields": [+ {"name": "product_id", "type": "string"},+ {"name": "product_name", "type": "string"},+ {"name": "product_description", "type": "string"},+ {"name": "rating", "type": "int32"}+ ]+ })"_json;+ documents = {+ R"({+ "product_id": "product_a",+ "product_name": "shampoo",+ "product_description": "Our new moisturizing shampoo is perfect for those with dry or damaged hair.",+ "rating": "2"+ })"_json,+ R"({+ "product_id": "product_c",+ "product_name": "comb",+ "product_description": "Experience the natural elegance and gentle care of our handcrafted wooden combs – because your hair deserves the best.",+ "rating": "3"+ })"_json+ };+ collection_create_op = collectionManager.create_collection(schema_json);+ ASSERT_TRUE(collection_create_op.ok());+ for (auto const &json: documents) {+ auto add_op = collection_create_op.get()->add(json.dump());+ if (!add_op.ok()) {+ LOG(INFO) << add_op.error();+ }+ ASSERT_TRUE(add_op.ok());+ }++ std::map<std::string, std::string> req_params = {+ {"collection", "Products"},+ {"q", "*"},+ {"query_by", "product_name"},+ {"filter_by", "id:* || $Customers(id:*)"},+ {"include_fields", "$Customers(id, strategy:nest_array) as Customers"}+ };+ nlohmann::json embedded_params;+ std::string json_res;+ auto now_ts = std::chrono::duration_cast<std::chrono::microseconds>(+ std::chrono::system_clock::now().time_since_epoch()).count();++ auto search_op = collectionManager.do_search(req_params, embedded_params, json_res, now_ts);+ ASSERT_TRUE(search_op.ok());++ auto res_obj = nlohmann::json::parse(json_res);+ ASSERT_EQ(2, res_obj["found"].get<size_t>());+ ASSERT_EQ(2, res_obj["hits"].size());+ ASSERT_EQ("1", res_obj["hits"][0]["document"]["id"]);+ ASSERT_EQ(0, res_obj["hits"][0]["document"].count("Customers"));++ ASSERT_EQ("0", res_obj["hits"][1]["document"]["id"]);+ ASSERT_EQ(1, res_obj["hits"][1]["document"].count("Customers"));+ ASSERT_EQ(2, res_obj["hits"][1]["document"]["Customers"].size());+ ASSERT_EQ("0", res_obj["hits"][1]["document"]["Customers"][0]["id"]);+ ASSERT_EQ("2", res_obj["hits"][1]["document"]["Customers"][1]["id"]);++ req_params = {+ {"collection", "coll1"},+ {"q", "*"},+ {"include_fields", "$Products(product_id)"}+ };++ search_op = collectionManager.do_search(req_params, embedded_params, json_res, now_ts);+ ASSERT_TRUE(search_op.ok());++ res_obj = nlohmann::json::parse(json_res);+ ASSERT_EQ(3, res_obj["found"].get<size_t>());+ ASSERT_EQ(3, res_obj["hits"].size());++ ASSERT_EQ("2", res_obj["hits"][0]["document"]["id"]);+ ASSERT_EQ(1, res_obj["hits"][0]["document"]["object"].count("Products"));+ ASSERT_EQ("product_a", res_obj["hits"][0]["document"]["object"]["Products"]["product_id"]);++ ASSERT_EQ("1", res_obj["hits"][1]["document"]["id"]);+ ASSERT_EQ(0, res_obj["hits"][1]["document"]["object"].count("Products"));+ ASSERT_EQ(1, res_obj["hits"][1]["document"]["object"].count("reference"));+ ASSERT_EQ("product_b", res_obj["hits"][1]["document"]["object"]["reference"]);+ ASSERT_EQ(0, res_obj["hits"][1]["document"].count("Products"));++ ASSERT_EQ("0", res_obj["hits"][2]["document"]["id"]);+ ASSERT_EQ(0, res_obj["hits"][2]["document"].count("Products"));+ ASSERT_EQ(0, res_obj["hits"][2]["document"]["object"].count("reference"));++ auto doc_json = R"({+ "product_id": "product_b",+ "product_name": "soap",+ "product_description": "Introducing our all-natural, organic soap bar made with essential oils and botanical ingredients.",+ "rating": "4"+ })"_json;+ auto add_doc_op = collection_create_op.get()->add(doc_json.dump());+ ASSERT_TRUE(add_doc_op.ok());++ req_params = {+ {"collection", "Products"},+ {"q", "*"},+ {"query_by", "product_name"},+ {"filter_by", "id:* || $Customers(id:*)"},+ {"include_fields", "$Customers(id, strategy:nest_array) as Customers"}+ };++ search_op = collectionManager.do_search(req_params, embedded_params, json_res, now_ts);+ ASSERT_TRUE(search_op.ok());++ res_obj = nlohmann::json::parse(json_res);+ ASSERT_EQ(3, res_obj["found"].get<size_t>());+ ASSERT_EQ(3, res_obj["hits"].size());++ ASSERT_EQ("2", res_obj["hits"][0]["document"]["id"]);+ ASSERT_EQ(1, res_obj["hits"][0]["document"].count("Customers"));+ ASSERT_EQ(1, res_obj["hits"][0]["document"]["Customers"].size());+ ASSERT_EQ("1", res_obj["hits"][0]["document"]["Customers"][0]["id"]);++ ASSERT_EQ("1", res_obj["hits"][1]["document"]["id"]);+ ASSERT_EQ(0, res_obj["hits"][1]["document"].count("Customers"));++ ASSERT_EQ("0", res_obj["hits"][2]["document"]["id"]);+ ASSERT_EQ(1, res_obj["hits"][2]["document"].count("Customers"));+ ASSERT_EQ(2, res_obj["hits"][2]["document"]["Customers"].size());+ ASSERT_EQ("0", res_obj["hits"][2]["document"]["Customers"][0]["id"]);+ ASSERT_EQ("2", res_obj["hits"][2]["document"]["Customers"][1]["id"]);++ {+ auto const& customers = collectionManager.get_collection_unsafe("Customers");++ doc_json = R"({+ "customer_id": "customer_b",+ "customer_name": "Dan",+ "product_price": 140,+ "product_id": "product_b"+ })"_json;+ add_doc_op = customers->add(doc_json.dump());+ ASSERT_TRUE(add_doc_op.ok());++ auto doc = customers->get("3").get();+ ASSERT_EQ("3", doc["id"]);++ ASSERT_EQ(1, doc.count(".ref"));+ ASSERT_EQ(1, doc[".ref"].size());+ ASSERT_EQ("product_id_sequence_id", doc[".ref"][0]);++ ASSERT_EQ(1, doc.count("product_id_sequence_id"));+ // When referenced document is already present, reference helper field should be initialized to its seq_id.+ ASSERT_EQ(2, doc["product_id_sequence_id"]);+ }++ req_params = {+ {"collection", "Products"},+ {"q", "*"},+ {"query_by", "product_name"},+ {"filter_by", "id:* || $Customers(id:*)"},+ {"include_fields", "$Customers(id, strategy:nest_array) as Customers"}+ };++ search_op = collectionManager.do_search(req_params, embedded_params, json_res, now_ts);+ ASSERT_TRUE(search_op.ok());++ res_obj = nlohmann::json::parse(json_res);+ ASSERT_EQ(3, res_obj["found"].get<size_t>());+ ASSERT_EQ(3, res_obj["hits"].size());++ ASSERT_EQ("2", res_obj["hits"][0]["document"]["id"]);+ ASSERT_EQ(1, res_obj["hits"][0]["document"].count("Customers"));+ ASSERT_EQ(2, res_obj["hits"][0]["document"]["Customers"].size());+ ASSERT_EQ("1", res_obj["hits"][0]["document"]["Customers"][0]["id"]);+ ASSERT_EQ("3", res_obj["hits"][0]["document"]["Customers"][1]["id"]);++ ASSERT_EQ("1", res_obj["hits"][1]["document"]["id"]);+ ASSERT_EQ(0, res_obj["hits"][1]["document"].count("Customers"));++ ASSERT_EQ("0", res_obj["hits"][2]["document"]["id"]);+ ASSERT_EQ(1, res_obj["hits"][2]["document"].count("Customers"));+ ASSERT_EQ(2, res_obj["hits"][2]["document"]["Customers"].size());+ ASSERT_EQ("0", res_obj["hits"][2]["document"]["Customers"][0]["id"]);+ ASSERT_EQ("2", res_obj["hits"][2]["document"]["Customers"][1]["id"]);++ {+ auto const& coll1 = collectionManager.get_collection_unsafe("coll1");++ doc_json = R"({+ "coll_id": "d",+ "object": {+ "reference": "product_d"+ }+ })"_json;+ add_doc_op = coll1->add(doc_json.dump());+ ASSERT_TRUE(add_doc_op.ok());++ auto doc = coll1->get("3").get();+ ASSERT_EQ("3", doc["id"]);++ ASSERT_EQ(1, doc.count(".ref"));+ ASSERT_EQ(1, doc[".ref"].size());+ ASSERT_EQ("object.reference_sequence_id", doc[".ref"][0]);++ ASSERT_EQ(1, doc.count("object.reference_sequence_id"));+ // product_d doesn't exist yet, so dummy value is present in the reference helper field.+ ASSERT_EQ(UINT32_MAX, doc["object.reference_sequence_id"]);++ doc_json = R"({+ "product_id": "product_d",+ "product_name": "hair oil",+ "product_description": "Revitalize your hair with our nourishing hair oil – nature's secret to lustrous, healthy locks.",+ "rating": "foo"+ })"_json;+ add_doc_op = collection_create_op.get()->add(doc_json.dump());+ ASSERT_FALSE(add_doc_op.ok());+ ASSERT_EQ("Field `rating` must be an int32.", add_doc_op.error());++ doc = coll1->get("3").get();+ ASSERT_EQ("3", doc["id"]);++ ASSERT_EQ(1, doc.count(".ref"));+ ASSERT_EQ(1, doc[".ref"].size());+ ASSERT_EQ("object.reference_sequence_id", doc[".ref"][0]);++ ASSERT_EQ(1, doc.count("object.reference_sequence_id"));+ // product_d was not indexed, reference helper field should remain unchanged.+ ASSERT_EQ(UINT32_MAX, doc["object.reference_sequence_id"]);++ doc_json = R"({+ "product_id": "product_a",+ "product_name": "hair oil",+ "product_description": "Revitalize your hair with our nourishing hair oil – nature's secret to lustrous, healthy locks.",+ "rating": "4"+ })"_json;+ add_doc_op = collection_create_op.get()->add(doc_json.dump());+ ASSERT_FALSE(add_doc_op.ok());+ // Singular reference field can only reference one document.+ ASSERT_EQ("Error while updating async reference field `object.reference` of collection `coll1`: "+ "Document `id: 2` already has a reference to document `0` of `Products` collection, "+ "having reference value `product_a`.", add_doc_op.error());++ doc = coll1->get("2").get();+ ASSERT_EQ("2", doc["id"]);++ ASSERT_EQ(1, doc.count(".ref"));+ ASSERT_EQ(1, doc[".ref"].size());+ ASSERT_EQ("object.reference_sequence_id", doc[".ref"][0]);++ ASSERT_EQ(1, doc.count("object.reference_sequence_id"));+ // product_a already existed, reference helper field should remain unchanged.+ ASSERT_EQ(0, doc["object.reference_sequence_id"]);++ doc_json = R"({+ "product_id": "product_d",+ "product_name": "hair oil",+ "product_description": "Revitalize your hair with our nourishing hair oil – nature's secret to lustrous, healthy locks.",+ "rating": "4"+ })"_json;+ add_doc_op = collection_create_op.get()->add(doc_json.dump());+ ASSERT_TRUE(add_doc_op.ok());++ doc = coll1->get("3").get();+ ASSERT_EQ("3", doc["id"]);++ ASSERT_EQ(1, doc.count(".ref"));+ ASSERT_EQ(1, doc[".ref"].size());+ ASSERT_EQ("object.reference_sequence_id", doc[".ref"][0]);++ ASSERT_EQ(1, doc.count("object.reference_sequence_id"));+ ASSERT_EQ(5, doc["object.reference_sequence_id"]);+ }++ schema_json =+ R"({+ "name": "songs",+ "fields": [+ { "name": "title", "type": "string" },+ { "name": "genres", "type": "string[]", "reference": "genres.id", "async_reference": true}+ ]+ })"_json;+ documents = {+ R"({"title":"Dil De Rani", "genres":[]})"_json,+ R"({"title":"Corduroy", "genres":["1"]})"_json,+ };+ collection_create_op = collectionManager.create_collection(schema_json);+ ASSERT_TRUE(collection_create_op.ok());+ for (auto const &json: documents) {+ auto add_op = collection_create_op.get()->add(json.dump());+ if (!add_op.ok()) {+ LOG(INFO) << add_op.error();+ }+ ASSERT_TRUE(add_op.ok());+ }++ {+ auto doc = collection_create_op.get()->get("0").get();+ ASSERT_EQ("0", doc["id"]);+ ASSERT_EQ(1, doc.count(".ref"));+ ASSERT_EQ(1, doc[".ref"].size());+ ASSERT_EQ("genres_sequence_id", doc[".ref"][0]);++ ASSERT_EQ(1, doc.count("genres_sequence_id"));+ ASSERT_TRUE(doc["genres"].size() == doc["genres_sequence_id"].size());+ ASSERT_EQ(0, doc["genres_sequence_id"].size());++ doc = collection_create_op.get()->get("1").get();+ ASSERT_EQ("1", doc["id"]);+ ASSERT_EQ(1, doc.count(".ref"));+ ASSERT_EQ(1, doc[".ref"].size());+ ASSERT_EQ("genres_sequence_id", doc[".ref"][0]);++ ASSERT_EQ(1, doc.count("genres_sequence_id"));+ ASSERT_TRUE(doc["genres"].size() == doc["genres_sequence_id"].size());+ ASSERT_EQ(1, doc["genres_sequence_id"].size());+ ASSERT_EQ(UINT32_MAX, doc["genres_sequence_id"][0]);+ }++ schema_json =+ R"({+ "name": "genres",+ "fields": [+ { "name": "id", "type": "string" },+ { "name": "name", "type": "string" }+ ]+ })"_json;+ documents = {+ R"({"id":"0","name":"Grunge"})"_json,+ R"({"id":"1","name":"Arena rock"})"_json+ };+ collection_create_op = collectionManager.create_collection(schema_json);+ ASSERT_TRUE(collection_create_op.ok());+ for (auto const &json: documents) {+ auto add_op = collection_create_op.get()->add(json.dump());+ if (!add_op.ok()) {+ LOG(INFO) << add_op.error();+ }+ ASSERT_TRUE(add_op.ok());+ }++ req_params = {+ {"collection", "songs"},+ {"q", "*"},+ {"include_fields", "$genres(name, strategy:nest) as genre"}+ };++ search_op = collectionManager.do_search(req_params, embedded_params, json_res, now_ts);+ ASSERT_TRUE(search_op.ok());++ res_obj = nlohmann::json::parse(json_res);+ ASSERT_EQ(2, res_obj["found"].get<size_t>());+ ASSERT_EQ(2, res_obj["hits"].size());++ ASSERT_EQ("Corduroy", res_obj["hits"][0]["document"]["title"].get<std::string>());+ ASSERT_EQ(1, res_obj["hits"][0]["document"]["genre"].size());+ ASSERT_EQ("Arena rock", res_obj["hits"][0]["document"]["genre"][0]["name"]);++ ASSERT_EQ("Dil De Rani", res_obj["hits"][1]["document"]["title"].get<std::string>());+ ASSERT_EQ(0, res_obj["hits"][1]["document"]["genre"].size());++ {+ auto const& songs_coll = collectionManager.get_collection_unsafe("songs");++ doc_json = R"({"title":"Achilles Last Stand", "genres":["3","0","2"]})"_json;+ add_doc_op = songs_coll->add(doc_json.dump());+ ASSERT_TRUE(add_doc_op.ok());++ auto doc = songs_coll->get("2").get();+ ASSERT_EQ("2", doc["id"]);+ ASSERT_EQ(1, doc.count(".ref"));+ ASSERT_EQ(1, doc[".ref"].size());+ ASSERT_EQ("genres_sequence_id", doc[".ref"][0]);++ ASSERT_EQ(1, doc.count("genres_sequence_id"));+ ASSERT_TRUE(doc["genres"].size() == doc["genres_sequence_id"].size());+ ASSERT_EQ(3, doc["genres_sequence_id"].size());++ ASSERT_EQ("3", doc["genres"][0]);+ ASSERT_EQ(UINT32_MAX, doc["genres_sequence_id"][0]);+ ASSERT_EQ("0", doc["genres"][1]);+ ASSERT_EQ(0, doc["genres_sequence_id"][1]);++ ASSERT_EQ("2", doc["genres"][2]);+ ASSERT_EQ(UINT32_MAX, doc["genres_sequence_id"][2]);++ auto remove_op = collection_create_op.get()->remove("0");+ ASSERT_TRUE(remove_op.ok());++ doc = songs_coll->get("2").get();+ ASSERT_EQ("2", doc["id"]);+ ASSERT_EQ(1, doc.count(".ref"));+ ASSERT_EQ(1, doc[".ref"].size());+ ASSERT_EQ("genres_sequence_id", doc[".ref"][0]);++ ASSERT_EQ(1, doc.count("genres_sequence_id"));+ ASSERT_TRUE(doc["genres"].size() == doc["genres_sequence_id"].size());+ ASSERT_EQ(2, doc["genres_sequence_id"].size());+ ASSERT_EQ("3", doc["genres"][0]);+ ASSERT_EQ(UINT32_MAX, doc["genres_sequence_id"][0]);++ ASSERT_EQ("2", doc["genres"][1]);+ ASSERT_EQ(UINT32_MAX, doc["genres_sequence_id"][1]);++ doc_json = R"({"id":"2","name":"Blues"})"_json;+ add_doc_op = collection_create_op.get()->add(doc_json.dump());+ ASSERT_TRUE(add_doc_op.ok());++ doc = songs_coll->get("2").get();+ ASSERT_EQ("2", doc["id"]);+ ASSERT_EQ(1, doc.count(".ref"));+ ASSERT_EQ(1, doc[".ref"].size());+ ASSERT_EQ("genres_sequence_id", doc[".ref"][0]);++ ASSERT_EQ(1, doc.count("genres_sequence_id"));+ ASSERT_TRUE(doc["genres"].size() == doc["genres_sequence_id"].size());+ ASSERT_EQ(2, doc["genres_sequence_id"].size());+ ASSERT_EQ("3", doc["genres"][0]);+ ASSERT_EQ(UINT32_MAX, doc["genres_sequence_id"][0]);++ ASSERT_EQ("2", doc["genres"][1]);+ ASSERT_EQ(2, doc["genres_sequence_id"][1]);+ }++ req_params = {+ {"collection", "songs"},+ {"q", "*"},+ {"include_fields", "$genres(name, strategy:nest) as genre"}+ };++ search_op = collectionManager.do_search(req_params, embedded_params, json_res, now_ts);+ ASSERT_TRUE(search_op.ok());++ res_obj = nlohmann::json::parse(json_res);+ ASSERT_EQ(3, res_obj["found"].get<size_t>());+ ASSERT_EQ(3, res_obj["hits"].size());++ ASSERT_EQ("Achilles Last Stand", res_obj["hits"][0]["document"]["title"].get<std::string>());+ ASSERT_EQ(1, res_obj["hits"][0]["document"]["genre"].size());+ ASSERT_EQ("Blues", res_obj["hits"][0]["document"]["genre"][0]["name"]);++ ASSERT_EQ("Corduroy", res_obj["hits"][1]["document"]["title"].get<std::string>());+ ASSERT_EQ(1, res_obj["hits"][1]["document"]["genre"].size());+ ASSERT_EQ("Arena rock", res_obj["hits"][1]["document"]["genre"][0]["name"]);++ ASSERT_EQ("Dil De Rani", res_obj["hits"][2]["document"]["title"].get<std::string>());+ ASSERT_EQ(0, res_obj["hits"][2]["document"]["genre"].size());++ collectionManager.dispose();+ delete store;++ store = new Store(state_dir_path);+ collectionManager.init(store, 1.0, "auth_key", quit);+ auto load_op = collectionManager.load(8, 1000);++ if(!load_op.ok()) {+ LOG(ERROR) << load_op.error();+ }+ ASSERT_TRUE(load_op.ok());++ req_params = {+ {"collection", "songs"},+ {"q", "*"},+ {"include_fields", "$genres(name, strategy:nest) as genre"}+ };++ search_op = collectionManager.do_search(req_params, embedded_params, json_res, now_ts);+ ASSERT_TRUE(search_op.ok());++ res_obj = nlohmann::json::parse(json_res);+ ASSERT_EQ(3, res_obj["found"].get<size_t>());+ ASSERT_EQ(3, res_obj["hits"].size());++ ASSERT_EQ("Achilles Last Stand", res_obj["hits"][0]["document"]["title"].get<std::string>());+ ASSERT_EQ(1, res_obj["hits"][0]["document"]["genre"].size());+ ASSERT_EQ("Blues", res_obj["hits"][0]["document"]["genre"][0]["name"]);++ ASSERT_EQ("Corduroy", res_obj["hits"][1]["document"]["title"].get<std::string>());+ ASSERT_EQ(1, res_obj["hits"][1]["document"]["genre"].size());+ ASSERT_EQ("Arena rock", res_obj["hits"][1]["document"]["genre"][0]["name"]);++ ASSERT_EQ("Dil De Rani", res_obj["hits"][2]["document"]["title"].get<std::string>());+ ASSERT_EQ(0, res_obj["hits"][2]["document"]["genre"].size());++ {+ auto const& songs_coll = collectionManager.get_collection_unsafe("songs");+ auto doc = songs_coll->get("2").get();+ ASSERT_EQ("2", doc["id"]);+ ASSERT_EQ(1, doc.count(".ref"));+ ASSERT_EQ(1, doc[".ref"].size());+ ASSERT_EQ("genres_sequence_id", doc[".ref"][0]);++ ASSERT_EQ(1, doc.count("genres_sequence_id"));+ ASSERT_TRUE(doc["genres"].size() == doc["genres_sequence_id"].size());+ ASSERT_EQ(2, doc["genres_sequence_id"].size());+ ASSERT_EQ("3", doc["genres"][0]);+ ASSERT_EQ(UINT32_MAX, doc["genres_sequence_id"][0]);++ ASSERT_EQ("2", doc["genres"][1]);+ ASSERT_EQ(2, doc["genres_sequence_id"][1]);++ auto const& genres_coll = collectionManager.get_collection_unsafe("genres");+ doc_json = R"({"id":"3","name":"Metal"})"_json;+ add_doc_op = genres_coll->add(doc_json.dump());+ ASSERT_TRUE(add_doc_op.ok());++ doc = songs_coll->get("2").get();+ ASSERT_EQ("2", doc["id"]);+ ASSERT_EQ(1, doc.count(".ref"));+ ASSERT_EQ(1, doc[".ref"].size());+ ASSERT_EQ("genres_sequence_id", doc[".ref"][0]);++ ASSERT_EQ(1, doc.count("genres_sequence_id"));+ ASSERT_TRUE(doc["genres"].size() == doc["genres_sequence_id"].size());+ ASSERT_EQ(2, doc["genres_sequence_id"].size());+ ASSERT_EQ("3", doc["genres"][0]);+ ASSERT_EQ(3, doc["genres_sequence_id"][0]);++ ASSERT_EQ("2", doc["genres"][1]);+ ASSERT_EQ(2, doc["genres_sequence_id"][1]);+ }+}+TEST_F(CollectionJoinTest, UpdateDocumentHavingReferenceField) {auto schema_json =R"({@@ -2573,6 +3187,7 @@ TEST_F(CollectionJoinTest, FilterByNestedReferences) {ASSERT_EQ(1, res_obj["hits"][0]["document"]["Coll_B"].size());ASSERT_EQ("coll_b_3", res_obj["hits"][0]["document"]["Coll_B"][0]["title"]);+ ASSERT_EQ(1, res_obj["hits"][0]["document"]["Coll_B"][0].count("Coll_A"));ASSERT_EQ(1, res_obj["hits"][0]["document"]["Coll_B"][0]["Coll_A"].size());ASSERT_EQ("coll_a_0", res_obj["hits"][0]["document"]["Coll_B"][0]["Coll_A"]["title"]);diff --git a/test/collection_manager_test.cpp b/test/collection_manager_test.cppindex 8d0cef5e..02b2505a 100644--- a/test/collection_manager_test.cpp+++ b/test/collection_manager_test.cpp@@ -43,7 +43,7 @@ protected:{"name": "points", "type": "int32"},{"name": "person", "type": "object", "optional": true},{"name": "vec", "type": "float[]", "num_dim": 128, "optional": true},- {"name": "product_id", "type": "string", "reference": "Products.product_id", "optional": true}+ {"name": "product_id", "type": "string", "reference": "Products.product_id", "optional": true, "async_reference": true}],"default_sorting_field": "points","symbols_to_index":["+"],@@ -255,6 +255,7 @@ TEST_F(CollectionManagerTest, CollectionCreation) {"stem":false},{+ "async_reference":true,"facet":false,"index":true,"infix":false,@@ -340,7 +341,10 @@ TEST_F(CollectionManagerTest, ShouldInitCollection) {"\"string\", \"facet\": false}], \"default_sorting_field\": \"foo\"}");spp::sparse_hash_map<std::string, std::string> referenced_in;- Collection *collection = collectionManager.init_collection(collection_meta1, 100, store, 1.0f, referenced_in);+ spp::sparse_hash_map<std::string, std::vector<reference_pair_t>> async_referenced_ins;++ Collection *collection = collectionManager.init_collection(collection_meta1, 100, store, 1.0f, referenced_in,+ async_referenced_ins);ASSERT_EQ("foobar", collection->get_name());ASSERT_EQ(100, collection->get_collection_id());ASSERT_EQ(1, collection->get_fields().size());@@ -362,7 +366,8 @@ TEST_F(CollectionManagerTest, ShouldInitCollection) {"\"symbols_to_index\": [\"+\"], \"token_separators\": [\"-\"]}");- collection = collectionManager.init_collection(collection_meta2, 100, store, 1.0f, referenced_in);+ collection = collectionManager.init_collection(collection_meta2, 100, store, 1.0f, referenced_in,+ async_referenced_ins);ASSERT_EQ(12345, collection->get_created_at());std::vector<char> expected_symbols = {'+'};@@ -500,6 +505,25 @@ TEST_F(CollectionManagerTest, RestoreRecordsOnRestart) {tsl::htrie_map<char, field> schema = collection1->get_schema();ASSERT_EQ(schema.count("product_id_sequence_id"), 1);+ auto products_schema_json =+ R"({+ "name": "Products",+ "fields": [+ {"name": "product_id", "type": "string"},+ {"name": "product_name", "type": "string"},+ {"name": "product_description", "type": "string"}+ ]+ })"_json;+ auto const& collection_create_op = collectionManager.create_collection(products_schema_json);+ ASSERT_TRUE(collection_create_op.ok());++ auto async_ref_fields = collection_create_op.get()->get_async_referenced_ins();+ ASSERT_EQ(1, async_ref_fields.size());+ ASSERT_EQ(1, async_ref_fields.count("product_id"));+ ASSERT_EQ(1, async_ref_fields["product_id"].size());+ ASSERT_EQ("collection1", async_ref_fields["product_id"][0].collection);+ ASSERT_EQ("product_id", async_ref_fields["product_id"][0].field);+// recreate collection manager to ensure that it restores the records from the disk backed storecollectionManager.dispose();delete store;@@ -573,6 +597,13 @@ TEST_F(CollectionManagerTest, RestoreRecordsOnRestart) {results = collection1->search("thomas", search_fields, "", facets, sort_fields, {0}, 10, 1, FREQUENCY, {false}).get();ASSERT_EQ(4, results["hits"].size());++ async_ref_fields = collectionManager.get_collection("Products").get()->get_async_referenced_ins();+ ASSERT_EQ(1, async_ref_fields.size());+ ASSERT_EQ(1, async_ref_fields.count("product_id"));+ ASSERT_EQ(1, async_ref_fields["product_id"].size());+ ASSERT_EQ("collection1", async_ref_fields["product_id"][0].collection);+ ASSERT_EQ("product_id", async_ref_fields["product_id"][0].field);}TEST_F(CollectionManagerTest, VerifyEmbeddedParametersOfScopedAPIKey) {@@ -1852,7 +1883,7 @@ TEST_F(CollectionManagerTest, ReferencedInBacklog) {auto const& references = referenced_ins_backlog.at("Products");ASSERT_EQ(1, references.size());ASSERT_EQ("collection1", references.cbegin()->collection);- ASSERT_EQ("product_id_sequence_id", references.cbegin()->field);+ ASSERT_EQ("product_id", references.cbegin()->field);auto schema_json =R"({@@ -1871,7 +1902,7 @@ TEST_F(CollectionManagerTest, ReferencedInBacklog) {auto get_reference_field_op = create_op.get()->get_referenced_in_field_with_lock("collection1");ASSERT_TRUE(get_reference_field_op.ok());- ASSERT_EQ("product_id_sequence_id", get_reference_field_op.get());+ ASSERT_EQ("product_id", get_reference_field_op.get());get_reference_field_op = create_op.get()->get_referenced_in_field_with_lock("foo");ASSERT_FALSE(get_reference_field_op.ok());@@ -2071,7 +2102,8 @@ TEST_F(CollectionManagerTest, PopulateReferencedIns) {"name": "B","fields": [{"name": "b_id", "type": "string"},- {"name": "b_ref", "type": "string", "reference": "A.a_id"}+ {"name": "a_ref", "type": "string", "reference": "A.a_id"},+ {"name": "c_ref", "type": "string", "reference": "C.c_id", "async_reference": true}]})"_json.dump(),R"({@@ -2082,16 +2114,29 @@ TEST_F(CollectionManagerTest, PopulateReferencedIns) {})"_json.dump(),};std::map<std::string, spp::sparse_hash_map<std::string, std::string>> referenced_ins;+ std::map<std::string, spp::sparse_hash_map<std::string, std::vector<reference_pair_t>>> async_referenced_ins;for (const auto &collection_meta_json: collection_meta_jsons) {- CollectionManager::_populate_referenced_ins(collection_meta_json, referenced_ins);+ CollectionManager::_populate_referenced_ins(collection_meta_json, referenced_ins, async_referenced_ins);}- ASSERT_EQ(1, referenced_ins.size());+ ASSERT_EQ(2, referenced_ins.size());ASSERT_EQ(1, referenced_ins.count("A"));ASSERT_EQ(1, referenced_ins["A"].size());ASSERT_EQ(1, referenced_ins["A"].count("B"));- ASSERT_EQ("b_ref_sequence_id", referenced_ins["A"]["B"]);+ ASSERT_EQ("a_ref", referenced_ins["A"]["B"]);++ ASSERT_EQ(1, referenced_ins.count("C"));+ ASSERT_EQ(1, referenced_ins["C"].size());+ ASSERT_EQ(1, referenced_ins["C"].count("B"));+ ASSERT_EQ("c_ref", referenced_ins["C"]["B"]);++ ASSERT_EQ(1, async_referenced_ins.count("C"));+ ASSERT_EQ(1, async_referenced_ins["C"].size());+ ASSERT_EQ(1, async_referenced_ins["C"].count("c_id"));+ ASSERT_EQ(1, async_referenced_ins["C"]["c_id"].size());+ ASSERT_EQ("B", async_referenced_ins["C"]["c_id"][0].collection);+ ASSERT_EQ("c_ref", async_referenced_ins["C"]["c_id"][0].field);}TEST_F(CollectionManagerTest, CollectionPagination) {