Open Source alternative to Algolia + Pinecone and an Easier-to-Use alternative to ElasticSearch ⚡ 🔍 ✨ Fast, typo tolerant, in-memory fuzzy Search Engine for building delightful search experiences
C++ algoliadatastoreelasticsearchenterprise-searchfacetingfull-text-searchfuzzy-searchgeosearchin-memoryinstantsearchmerchandisingpineconesearchsearch-enginesemantic-searchsimilarity-searchsite-searchsynonymstypo-tolerancevector-search
Add `reference_index` for reference array field.
Harpreet Sangar4ffbe3aad191
diff --git a/include/collection.h b/include/collection.hindex a34f6a46..3b2bbdd2 100644--- a/include/collection.h+++ b/include/collection.h@@ -287,7 +287,10 @@ private:tsl::htrie_set<char>& include_fields_full,tsl::htrie_set<char>& exclude_fields_full) const;- Option<uint32_t> get_reference_doc_id(const std::string& ref_collection_name, const uint32_t& seq_id) const;+ Option<std::string> get_referenced_in_field(const std::string& collection_name) const;++ Option<bool> get_related_ids(const std::string& ref_collection_name, const uint32_t& seq_id,+ std::vector<uint32_t>& result) const;static void hide_credential(nlohmann::json& json, const std::string& credential_name);@@ -320,8 +323,6 @@ public:static constexpr const char* COLLECTION_SYMBOLS_TO_INDEX = "symbols_to_index";static constexpr const char* COLLECTION_SEPARATORS = "token_separators";- static constexpr const char* REFERENCE_HELPER_FIELD_SUFFIX = "_sequence_id";-// methodsCollection() = delete;@@ -398,7 +399,7 @@ public:const reference_filter_result_t& references,const tsl::htrie_set<char>& ref_include_fields_full,const tsl::htrie_set<char>& ref_exclude_fields_full,- const std::string& error_prefix);+ const std::string& error_prefix, const bool& is_reference_array);static Option<bool> prune_doc(nlohmann::json& doc, const tsl::htrie_set<char>& include_names,const tsl::htrie_set<char>& exclude_names, const std::string& parent_name = "",@@ -614,9 +615,12 @@ public:void add_referenced_in(const std::string& collection_name, const std::string& field_name);- Option<std::string> get_reference_field(const std::string& collection_name) const;+ Option<std::string> get_referenced_in_field_with_lock(const std::string& collection_name) const;++ Option<bool> get_related_ids_with_lock(const std::string& field_name, const uint32_t& seq_id,+ std::vector<uint32_t>& result) const;- Option<uint32_t> get_sort_indexed_field_value(const std::string& field_name, const uint32_t& seq_id) const;+ Option<uint32_t> get_sort_index_value_with_lock(const std::string& field_name, const uint32_t& seq_id) const;friend class filter_result_iterator_t;};diff --git a/include/field.h b/include/field.hindex c92b1ff4..a2612f53 100644--- a/include/field.h+++ b/include/field.h@@ -66,6 +66,8 @@ namespace fields {static const std::string query_prefix = "query_prefix";static const std::string api_key = "api_key";static const std::string model_config = "model_config";++ static const std::string REFERENCE_HELPER_FIELD_SUFFIX = "_sequence_id";}enum vector_distance_type_t {@@ -100,6 +102,8 @@ struct field {bool range_index;+ bool is_reference_helper = false;+field() {}field(const std::string &name, const std::string &type, const bool facet, const bool optional = false,@@ -111,6 +115,9 @@ struct field {embed(embed), range_index(range_index) {set_computed_defaults(sort, infix);++ auto const suffix = std::string(fields::REFERENCE_HELPER_FIELD_SUFFIX);+ is_reference_helper = name.size() > suffix.size() && name.substr(name.size() - suffix.size()) == suffix;}void set_computed_defaults(int sort, int infix) {diff --git a/include/index.h b/include/index.hindex 2dbc88d6..823c8e43 100644--- a/include/index.h+++ b/include/index.h@@ -343,6 +343,10 @@ private:spp::sparse_hash_map<std::string, num_tree_t*> numerical_index;+ // reference_helper_field => (seq_id => ref_seq_ids)+ // Only used when the reference field is an array type otherwise sort_index is used.+ spp::sparse_hash_map<std::string, num_tree_t*> reference_index;+spp::sparse_hash_map<std::string, NumericTrie*> range_index;spp::sparse_hash_map<std::string, NumericTrie*> geo_range_index;@@ -779,8 +783,6 @@ public:int64_t reference_string_sort_score(const std::string& field_name, const uint32_t& seq_id) const;- Option<uint32_t> get_sort_indexed_field_value(const std::string& field_name, const uint32_t& seq_id) const;-static void remove_matched_tokens(std::vector<std::string>& tokens, const std::set<std::string>& rule_token_set) ;void compute_facet_infos(const std::vector<facet>& facets, facet_query_t& facet_query,@@ -1000,7 +1002,12 @@ public:Option<bool> seq_ids_outside_top_k(const std::string& field_name, size_t k,std::vector<uint32_t>& outside_seq_ids);- Option<uint32_t> get_reference_doc_id_with_lock(const std::string& reference_helper_field_name,+ Option<bool> get_related_ids(const std::string& collection_name,+ const std::string& reference_helper_field_name,+ const uint32_t& seq_id, std::vector<uint32_t>& result) const;++ Option<uint32_t> get_sort_index_value_with_lock(const std::string& collection_name,+ const std::string& field_name,const uint32_t& seq_id) const;friend class filter_result_iterator_t;diff --git a/src/collection.cpp b/src/collection.cppindex af4ae784..112e322c 100644--- a/src/collection.cpp+++ b/src/collection.cpp@@ -12,6 +12,7 @@#include <regex>#include <list>#include <posting.h>+#include <timsort.hpp>#include "validator.h"#include "topster.h"#include "logger.h"@@ -91,6 +92,7 @@ Option<bool> Collection::add_reference_helper_fields(nlohmann::json& document) {if (reference_field_name == "id") {auto id_field_type_error_op = Option<bool>(400, "Field `" + field_name + "` must have string value.");if (document[field_name].is_array()) {+ document[field_name + fields::REFERENCE_HELPER_FIELD_SUFFIX] = nlohmann::json::array();for (const auto &item: document[field_name].items()) {if (!item.value().is_string()) {return id_field_type_error_op;@@ -104,7 +106,7 @@ Option<bool> Collection::add_reference_helper_fields(nlohmann::json& document) {reference_collection_name + "`." );}- document[field_name + REFERENCE_HELPER_FIELD_SUFFIX] += ref_doc_id_op.get();+ document[field_name + fields::REFERENCE_HELPER_FIELD_SUFFIX] += ref_doc_id_op.get();}} else if (document[field_name].is_string()) {auto id = document[field_name].get<std::string>();@@ -115,7 +117,7 @@ Option<bool> Collection::add_reference_helper_fields(nlohmann::json& document) {reference_collection_name + "`." );}- document[field_name + REFERENCE_HELPER_FIELD_SUFFIX] = ref_doc_id_op.get();+ document[field_name + fields::REFERENCE_HELPER_FIELD_SUFFIX] = ref_doc_id_op.get();} else {return id_field_type_error_op;}@@ -185,13 +187,9 @@ Option<bool> Collection::add_reference_helper_fields(nlohmann::json& document) {}if (document[field_name].is_array()) {- if (filter_result.count == 0) {- return Option<bool>(400, "No reference document matching `" + filter_query + "` found in the collection `"- + reference_collection_name + "`.");- }-+ document[field_name + fields::REFERENCE_HELPER_FIELD_SUFFIX] = nlohmann::json::array();for (uint32_t i = 0; i < filter_result.count; i++) {- document[field_name + REFERENCE_HELPER_FIELD_SUFFIX] += filter_result.docs[i];+ document[field_name + fields::REFERENCE_HELPER_FIELD_SUFFIX] += filter_result.docs[i];}} else {if (filter_result.count != 1) {@@ -203,7 +201,7 @@ Option<bool> Collection::add_reference_helper_fields(nlohmann::json& document) {reference_collection_name + "`.");}- document[field_name + REFERENCE_HELPER_FIELD_SUFFIX] = filter_result.docs[0];+ document[field_name + fields::REFERENCE_HELPER_FIELD_SUFFIX] = filter_result.docs[0];}}@@ -3132,8 +3130,9 @@ Option<bool> Collection::get_filter_ids(const std::string& filter_query, filter_return index->do_filtering_with_lock(filter_tree_root, filter_result, name);}-Option<uint32_t> Collection::get_reference_doc_id(const std::string& ref_field_name, const uint32_t& seq_id) const {- return index->get_reference_doc_id_with_lock(ref_field_name, seq_id);+Option<bool> Collection::get_related_ids(const std::string& ref_field_name, const uint32_t& seq_id,+ std::vector<uint32_t>& result) const {+ return index->get_related_ids(name, ref_field_name, seq_id, result);}Option<bool> Collection::get_reference_filter_ids(const std::string & filter_query,@@ -4533,9 +4532,9 @@ Option<bool> Collection::add_reference_fields(nlohmann::json& doc,const reference_filter_result_t& references,const tsl::htrie_set<char>& ref_include_fields_full,const tsl::htrie_set<char>& ref_exclude_fields_full,- const std::string& error_prefix) {+ const std::string& error_prefix, const bool& is_reference_array) {// One-to-one relation.- if (references.count == 1) {+ if (!is_reference_array && references.count == 1) {auto ref_doc_seq_id = references.docs[0];nlohmann::json ref_doc;@@ -4744,54 +4743,74 @@ Option<bool> Collection::prune_doc(nlohmann::json& doc,Option<bool> add_reference_fields_op = Option<bool>(true);if (has_filter_reference) {+ auto get_reference_field_op = collection->get_referenced_in_field(ref_collection_name);+ if (!get_reference_field_op.ok()) {+ continue;+ }+ auto const& field_name = get_reference_field_op.get();+ if (ref_collection->search_schema.count(field_name) == 0) {+ continue;+ }add_reference_fields_op = add_reference_fields(doc, ref_collection.get(), ref_include.alias,reference_filter_results.at(ref_collection_name),- ref_include_fields_full, ref_exclude_fields_full,- error_prefix);+ ref_include_fields_full, ref_exclude_fields_full, error_prefix,+ ref_collection->get_schema().at(field_name).is_array());} else if (doc_has_reference) {- auto get_reference_field_op = ref_collection->get_reference_field(collection->name);+ auto get_reference_field_op = ref_collection->get_referenced_in_field_with_lock(collection->name);if (!get_reference_field_op.ok()) {continue;}auto const& field_name = get_reference_field_op.get();- auto get_reference_doc_id_op = collection->get_reference_doc_id(field_name, seq_id);- if (!get_reference_doc_id_op.ok()) {+ if (collection->search_schema.count(field_name) == 0) {+ continue;+ }++ reference_filter_result_t result;+ std::vector<uint32_t> ids;+ auto get_references_op = collection->get_related_ids(field_name, seq_id, ids);+ if (!get_references_op.ok()) {continue;}+ result.count = ids.size();+ result.docs = &ids[0];- reference_filter_result_t r{1, new uint32[1]{get_reference_doc_id_op.get()}};- add_reference_fields_op = add_reference_fields(doc, ref_collection.get(), ref_include.alias, r,- ref_include_fields_full, ref_exclude_fields_full,- error_prefix);+ add_reference_fields_op = add_reference_fields(doc, ref_collection.get(), ref_include.alias, result,+ ref_include_fields_full, ref_exclude_fields_full, error_prefix,+ collection->search_schema.at(field_name).is_array());+ result.docs = nullptr;} else if (joined_coll_has_reference) {auto joined_collection = cm.get_collection(joined_coll_having_reference);if (joined_collection == nullptr) {continue;}- auto reference_field_name_op = ref_collection->get_reference_field(joined_coll_having_reference);- if (!reference_field_name_op.ok()) {+ auto reference_field_name_op = ref_collection->get_referenced_in_field_with_lock(joined_coll_having_reference);+ if (!reference_field_name_op.ok() || joined_collection->get_schema().count(reference_field_name_op.get()) == 0) {continue;}auto const& reference_field_name = reference_field_name_op.get();auto const& reference_filter_result = reference_filter_results.at(joined_coll_having_reference);auto const& count = reference_filter_result.count;- reference_filter_result_t r{count, new uint32[count]};-+ std::vector<uint32_t> ids;+ ids.reserve(count);for (uint32_t i = 0; i < count; i++) {- auto op = joined_collection->get_sort_indexed_field_value(reference_field_name,- reference_filter_result.docs[i]);- if (!op.ok()) {- return Option<bool>(op.code(), error_prefix + op.error());- }-- r.docs[i] = op.get();+ joined_collection->get_related_ids_with_lock(reference_field_name, reference_filter_result.docs[i], ids);+ }+ if (ids.empty()) {+ continue;}- add_reference_fields_op = add_reference_fields(doc, ref_collection.get(), ref_include.alias, r,- ref_include_fields_full, ref_exclude_fields_full,- error_prefix);+ gfx::timsort(ids.begin(), ids.end());+ ids.erase(unique(ids.begin(), ids.end()), ids.end());++ reference_filter_result_t result;+ result.count = ids.size();+ result.docs = &ids[0];+ add_reference_fields_op = add_reference_fields(doc, ref_collection.get(), ref_include.alias, result,+ ref_include_fields_full, ref_exclude_fields_full, error_prefix,+ joined_collection->get_schema().at(reference_field_name).is_array());+ result.docs = nullptr;}if (!add_reference_fields_op.ok()) {@@ -5345,11 +5364,11 @@ Index* Collection::init_index() {auto ref_coll = collectionManager.get_collection(ref_coll_name);if (ref_coll != nullptr) {// Passing reference helper field helps perform operation on doc_id instead of field value.- ref_coll->add_referenced_in(name, field.name + REFERENCE_HELPER_FIELD_SUFFIX);+ ref_coll->add_referenced_in(name, field.name + fields::REFERENCE_HELPER_FIELD_SUFFIX);} else {// Reference collection has not been created yet.collectionManager.add_referenced_in_backlog(ref_coll_name,- reference_pair{name, field.name + REFERENCE_HELPER_FIELD_SUFFIX});+ reference_pair{name, field.name + fields::REFERENCE_HELPER_FIELD_SUFFIX});}}}@@ -5804,9 +5823,12 @@ void Collection::add_referenced_in(const std::string& collection_name, const stdreferenced_in.emplace(collection_name, field_name);}-Option<std::string> Collection::get_reference_field(const std::string& collection_name) const {+Option<std::string> Collection::get_referenced_in_field_with_lock(const std::string& collection_name) const {std::shared_lock lock(mutex);+ return get_referenced_in_field(collection_name);+}+Option<std::string> Collection::get_referenced_in_field(const std::string& collection_name) const {if (referenced_in.count(collection_name) == 0) {return Option<std::string>(400, "Could not find any field in `" + name + "` referencing the collection `"+ collection_name + "`.");@@ -5815,9 +5837,16 @@ Option<std::string> Collection::get_reference_field(const std::string& collectioreturn Option<std::string>(referenced_in.at(collection_name));}-Option<uint32_t> Collection::get_sort_indexed_field_value(const std::string& field_name, const uint32_t& seq_id) const {+Option<bool> Collection::get_related_ids_with_lock(const std::string& field_name, const uint32_t& seq_id,+ std::vector<uint32_t>& result) const {+ std::shared_lock lock(mutex);+ return index->get_related_ids(name, field_name, seq_id, result);+}++Option<uint32_t> Collection::get_sort_index_value_with_lock(const std::string& field_name,+ const uint32_t& seq_id) const {std::shared_lock lock(mutex);- return index->get_sort_indexed_field_value(field_name, seq_id);+ return index->get_sort_index_value_with_lock(name, field_name, seq_id);}void Collection::remove_embedding_field(const std::string& field_name) {diff --git a/src/collection_manager.cpp b/src/collection_manager.cppindex 6129c63c..7c44ac37 100644--- a/src/collection_manager.cpp+++ b/src/collection_manager.cpp@@ -263,7 +263,7 @@ Option<bool> CollectionManager::load(const size_t collection_batch_size, const sif (referenced_ins.count(ref_coll_name) == 0) {referenced_ins[ref_coll_name] = {};}- auto const field_name = item.first + Collection::REFERENCE_HELPER_FIELD_SUFFIX;+ auto const field_name = item.first + fields::REFERENCE_HELPER_FIELD_SUFFIX;referenced_ins.at(ref_coll_name).insert(reference_pair{collection_name, field_name});}}diff --git a/src/field.cpp b/src/field.cppindex 894fbb4e..2ee6bc37 100644--- a/src/field.cpp+++ b/src/field.cpp@@ -328,7 +328,7 @@ Option<bool> field::json_field_to_field(bool enable_nested_fields, nlohmann::jso// Add a reference helper field in the schema. It stores the doc id of the document it references to reduce the// computation while searching.the_fields.emplace_back(- field(field_json[fields::name].get<std::string>() + Collection::REFERENCE_HELPER_FIELD_SUFFIX,+ field(field_json[fields::name].get<std::string>() + fields::REFERENCE_HELPER_FIELD_SUFFIX,field_types::is_array(field_json[fields::type].get<std::string>()) ? field_types::INT64_ARRAY : field_types::INT64,false, field_json[fields::optional], true));diff --git a/src/filter_result_iterator.cpp b/src/filter_result_iterator.cppindex ed3bbfcd..e1c14489 100644--- a/src/filter_result_iterator.cpp+++ b/src/filter_result_iterator.cpp@@ -647,7 +647,7 @@ void filter_result_iterator_t::init() {return;}- auto get_reference_field_op = ref_collection->get_reference_field(collection_name);+ auto get_reference_field_op = ref_collection->get_referenced_in_field_with_lock(collection_name);if (!get_reference_field_op.ok()) {status = Option<bool>(get_reference_field_op.code(), get_reference_field_op.error());is_valid = false;diff --git a/src/index.cpp b/src/index.cppindex 4f1877fc..04987fc7 100644--- a/src/index.cpp+++ b/src/index.cpp@@ -124,6 +124,11 @@ Index::Index(const std::string& name, const uint32_t collection_id, const Store*infix_index.emplace(a_field.name, infix_sets);}++ if (a_field.is_reference_helper && a_field.is_array()) {+ auto num_tree = new num_tree_t;+ reference_index.emplace(a_field.name, num_tree);+ }}num_documents = 0;@@ -202,6 +207,13 @@ Index::~Index() {for(auto& vec_index_kv: vector_index) {delete vec_index_kv.second;}++ for(auto & name_tree: reference_index) {+ delete name_tree.second;+ name_tree.second = nullptr;+ }++ reference_index.clear();}int64_t Index::get_points_from_doc(const nlohmann::json &document, const std::string & default_sorting_field) {@@ -992,7 +1004,8 @@ void Index::index_field_in_memory(const field& afield, std::vector<index_record>// all other numerical arraysauto num_tree = afield.range_index ? nullptr : numerical_index.at(afield.name);auto trie = afield.range_index ? range_index.at(afield.name) : nullptr;- iterate_and_index_numerical_field(iter_batch, afield, [&afield, num_tree, trie]+ auto reference = afield.is_reference_helper ? reference_index.at(afield.name) : nullptr;+ iterate_and_index_numerical_field(iter_batch, afield, [&afield, num_tree, trie, reference](const index_record& record, uint32_t seq_id) {for(size_t arr_i = 0; arr_i < record.doc[afield.name].size(); arr_i++) {const auto& arr_value = record.doc[afield.name][arr_i];@@ -1013,6 +1026,9 @@ void Index::index_field_in_memory(const field& afield, std::vector<index_record>} else {num_tree->insert(value, seq_id);}+ if (reference != nullptr) {+ reference->insert(seq_id, value);+ }}else if(afield.type == field_types::FLOAT_ARRAY) {@@ -1762,59 +1778,87 @@ Option<bool> Index::do_reference_filtering_with_lock(filter_node_t* const filterreturn Option(true);}- // Collect all the doc ids from the reference ids.- std::vector<std::pair<uint32_t, uint32_t>> id_pairs;- std::unordered_set<uint32_t> unique_doc_ids;- auto const& ref_index = *sort_index.at(reference_helper_field_name);- for (uint32_t i = 0; i < count; i++) {- auto& reference_doc_id = reference_docs[i];- auto doc_id = ref_index.at(reference_doc_id);+ if (search_schema.at(reference_helper_field_name).is_singular()) {+ // Collect all the doc ids from the reference ids.+ std::vector<std::pair<uint32_t, uint32_t>> id_pairs;+ std::unordered_set<uint32_t> unique_doc_ids;+ auto const& ref_index = *sort_index.at(reference_helper_field_name);+ for (uint32_t i = 0; i < count; i++) {+ auto& reference_doc_id = reference_docs[i];+ auto doc_id = ref_index.at(reference_doc_id);- id_pairs.emplace_back(std::pair(doc_id, reference_doc_id));- unique_doc_ids.insert(doc_id);- }+ id_pairs.emplace_back(std::pair(doc_id, reference_doc_id));+ unique_doc_ids.insert(doc_id);+ }- std::sort(id_pairs.begin(), id_pairs.end(), [](auto const& left, auto const& right) {- return left.first < right.first;- });+ std::sort(id_pairs.begin(), id_pairs.end(), [](auto const& left, auto const& right) {+ return left.first < right.first;+ });++ filter_result.count = unique_doc_ids.size();+ filter_result.docs = new uint32_t[unique_doc_ids.size()];+ filter_result.coll_to_references = new std::map<std::string, reference_filter_result_t>[unique_doc_ids.size()] {};- filter_result.count = unique_doc_ids.size();- filter_result.docs = new uint32_t[unique_doc_ids.size()];- filter_result.coll_to_references = new std::map<std::string, reference_filter_result_t>[unique_doc_ids.size()] {};+ std::vector<uint32_t> previous_doc_references;+ for (uint32_t i = 0, previous_doc = id_pairs[0].first + 1, result_index = 0; i < id_pairs.size(); i++) {+ auto const& current_doc = id_pairs[i].first;+ auto const& reference_doc_id = id_pairs[i].second;- std::vector<uint32_t> previous_doc_references;- for (uint32_t i = 0, previous_doc = id_pairs[0].first + 1, result_index = 0; i < id_pairs.size(); i++) {- auto const& current_doc = id_pairs[i].first;- auto const& reference_doc_id = id_pairs[i].second;+ if (current_doc != previous_doc) {+ filter_result.docs[result_index] = current_doc;+ if (result_index > 0) {+ auto& reference_result = filter_result.coll_to_references[result_index - 1];- if (current_doc != previous_doc) {- filter_result.docs[result_index] = current_doc;- if (result_index > 0) {- auto& reference_result = filter_result.coll_to_references[result_index - 1];+ auto r = reference_filter_result_t(previous_doc_references.size(), new uint32_t[previous_doc_references.size()]);+ std::copy(previous_doc_references.begin(), previous_doc_references.end(), r.docs);+ reference_result[collection_name] = std::move(r);- auto r = reference_filter_result_t(previous_doc_references.size(), new uint32_t[previous_doc_references.size()]);- std::copy(previous_doc_references.begin(), previous_doc_references.end(), r.docs);- reference_result[collection_name] = std::move(r);+ previous_doc_references.clear();+ }- previous_doc_references.clear();+ result_index++;+ previous_doc = current_doc;+ previous_doc_references.push_back(reference_doc_id);+ } else {+ previous_doc_references.push_back(reference_doc_id);}+ }- result_index++;- previous_doc = current_doc;- previous_doc_references.push_back(reference_doc_id);- } else {- previous_doc_references.push_back(reference_doc_id);+ if (!previous_doc_references.empty()) {+ auto& reference_result = filter_result.coll_to_references[filter_result.count - 1];++ auto r = reference_filter_result_t(previous_doc_references.size(), new uint32_t[previous_doc_references.size()]);+ std::copy(previous_doc_references.begin(), previous_doc_references.end(), r.docs);+ reference_result[collection_name] = std::move(r);}++ return Option(true);}- if (!previous_doc_references.empty()) {- auto& reference_result = filter_result.coll_to_references[filter_result.count - 1];+ size_t ids_len = 0;+ uint32_t *ids = nullptr;+ auto& ref_index = *reference_index.at(reference_helper_field_name);+ for (uint32_t i = 0; i < count; i++) {+ auto& reference_doc_id = reference_docs[i];+ ref_index.search(EQUALS, reference_doc_id, &ids, ids_len);+ }++ filter_result.count = ids_len;+ filter_result.docs = new uint32_t[ids_len];+ filter_result.coll_to_references = new std::map<std::string, reference_filter_result_t>[ids_len] {};++ auto& num_index = *numerical_index.at(reference_helper_field_name);+ for (size_t i = 0; i < ids_len; i++) {+ filter_result.docs[i] = ids[i];- auto r = reference_filter_result_t(previous_doc_references.size(), new uint32_t[previous_doc_references.size()]);- std::copy(previous_doc_references.begin(), previous_doc_references.end(), r.docs);- reference_result[collection_name] = std::move(r);+ reference_filter_result_t reference_result;+ size_t len = 0;+ num_index.search(EQUALS, ids[i], &reference_result.docs, len);+ reference_result.count = len;+ filter_result.coll_to_references[i][collection_name] = std::move(reference_result);}+ delete [] ids;return Option(true);}@@ -4198,7 +4242,7 @@ Option<bool> Index::compute_sort_scores(const std::vector<sort_by>& sort_fields,// Current collection has a reference.if (ref_collection->is_referenced_in(collection_name)) {- auto get_reference_field_op = ref_collection->get_reference_field(collection_name);+ auto get_reference_field_op = ref_collection->get_referenced_in_field_with_lock(collection_name);if (!get_reference_field_op.ok()) {return Option<bool>(get_reference_field_op.code(), get_reference_field_op.error());}@@ -4229,7 +4273,7 @@ Option<bool> Index::compute_sort_scores(const std::vector<sort_by>& sort_fields,"` in `sort_by` not found.");}- auto reference_field_name_op = ref_collection->get_reference_field(joined_coll_having_reference);+ auto reference_field_name_op = ref_collection->get_referenced_in_field_with_lock(joined_coll_having_reference);if (!reference_field_name_op.ok()) {return Option<bool>(reference_field_name_op.code(), reference_field_name_op.error());}@@ -4239,8 +4283,8 @@ Option<bool> Index::compute_sort_scores(const std::vector<sort_by>& sort_fields,auto const& count = reference.count;if (count == 1) {- auto op = joined_collection->get_sort_indexed_field_value(reference_field_name,- reference.docs[0]);+ auto op = joined_collection->get_sort_index_value_with_lock(reference_field_name,+ reference.docs[0]);if (!op.ok()) {return Option<bool>(op.code(), op.error());}@@ -4361,7 +4405,7 @@ Option<bool> Index::compute_sort_scores(const std::vector<sort_by>& sort_fields,// Current collection has a reference.if (ref_collection->is_referenced_in(collection_name)) {- auto get_reference_field_op = ref_collection->get_reference_field(collection_name);+ auto get_reference_field_op = ref_collection->get_referenced_in_field_with_lock(collection_name);if (!get_reference_field_op.ok()) {return Option<bool>(get_reference_field_op.code(), get_reference_field_op.error());}@@ -4392,7 +4436,7 @@ Option<bool> Index::compute_sort_scores(const std::vector<sort_by>& sort_fields,"` in `sort_by` not found.");}- auto reference_field_name_op = ref_collection->get_reference_field(joined_coll_having_reference);+ auto reference_field_name_op = ref_collection->get_referenced_in_field_with_lock(joined_coll_having_reference);if (!reference_field_name_op.ok()) {return Option<bool>(reference_field_name_op.code(), reference_field_name_op.error());}@@ -4402,8 +4446,8 @@ Option<bool> Index::compute_sort_scores(const std::vector<sort_by>& sort_fields,auto const& count = reference.count;if (count == 1) {- auto op = joined_collection->get_sort_indexed_field_value(reference_field_name,- reference.docs[0]);+ auto op = joined_collection->get_sort_index_value_with_lock(reference_field_name,+ reference.docs[0]);if (!op.ok()) {return Option<bool>(op.code(), op.error());}@@ -4520,7 +4564,7 @@ Option<bool> Index::compute_sort_scores(const std::vector<sort_by>& sort_fields,// Current collection has a reference.if (ref_collection->is_referenced_in(collection_name)) {- auto get_reference_field_op = ref_collection->get_reference_field(collection_name);+ auto get_reference_field_op = ref_collection->get_referenced_in_field_with_lock(collection_name);if (!get_reference_field_op.ok()) {return Option<bool>(get_reference_field_op.code(), get_reference_field_op.error());}@@ -4551,7 +4595,7 @@ Option<bool> Index::compute_sort_scores(const std::vector<sort_by>& sort_fields,"` in `sort_by` not found.");}- auto reference_field_name_op = ref_collection->get_reference_field(joined_coll_having_reference);+ auto reference_field_name_op = ref_collection->get_referenced_in_field_with_lock(joined_coll_having_reference);if (!reference_field_name_op.ok()) {return Option<bool>(reference_field_name_op.code(), reference_field_name_op.error());}@@ -4561,8 +4605,8 @@ Option<bool> Index::compute_sort_scores(const std::vector<sort_by>& sort_fields,auto const& count = reference.count;if (count == 1) {- auto op = joined_collection->get_sort_indexed_field_value(reference_field_name,- reference.docs[0]);+ auto op = joined_collection->get_sort_index_value_with_lock(reference_field_name,+ reference.docs[0]);if (!op.ok()) {return Option<bool>(op.code(), op.error());}@@ -6957,27 +7001,59 @@ void Index::batch_embed_fields(std::vector<index_record*>& records,}}-Option<uint32_t> Index::get_reference_doc_id_with_lock(const string& reference_helper_field_name,- const uint32_t& seq_id) const {+int64_t Index::reference_string_sort_score(const string &field_name, const uint32_t &seq_id) const {std::shared_lock lock(mutex);- if (sort_index.count(reference_helper_field_name) == 0 ||- sort_index.at(reference_helper_field_name)->count(seq_id) == 0) {- return Option<uint32_t>(400, "Could not find a reference for doc " + std::to_string(seq_id));- }-- return Option<uint32_t>(sort_index.at(reference_helper_field_name)->at(seq_id));+ return str_sort_index.at(field_name)->rank(seq_id);}-int64_t Index::reference_string_sort_score(const string &field_name, const uint32_t &seq_id) const {+Option<bool> Index::get_related_ids(const std::string& collection_name, const string& field_name,+ const uint32_t& seq_id, std::vector<uint32_t>& result) const {std::shared_lock lock(mutex);- return str_sort_index.at(field_name)->rank(seq_id);+ if (search_schema.count(field_name) == 0) {+ return Option<bool>(400, "Could not find `" + field_name + "` in the collection `" + collection_name + "`.");+ }++ auto const no_match_op = Option<bool>(400, "Could not find `" + field_name + "` value for doc `" ++ std::to_string(seq_id) + "`.");+ if (search_schema.at(field_name).is_singular()) {+ if (sort_index.count(field_name) == 0 || sort_index.at(field_name)->count(seq_id) == 0) {+ return no_match_op;+ }++ result.emplace_back(sort_index.at(field_name)->at(seq_id));+ return Option<bool>(true);+ }++ if (reference_index.count(field_name) == 0) {+ return no_match_op;+ }++ size_t ids_len = 0;+ uint32_t* ids = nullptr;+ reference_index.at(field_name)->search(EQUALS, seq_id, &ids, ids_len);+ if (ids_len == 0) {+ return no_match_op;+ }++ for (uint32_t i = 0; i < ids_len; i++) {+ result.emplace_back(ids[i]);+ }+ delete [] ids;+ return Option<bool>(true);}-Option<uint32_t> Index::get_sort_indexed_field_value(const string& field_name, const uint32_t& seq_id) const {+Option<uint32_t> Index::get_sort_index_value_with_lock(const std::string& collection_name,+ const std::string& field_name,+ const uint32_t& seq_id) const {std::shared_lock lock(mutex);- if (sort_index.count(field_name) == 0 || sort_index.at(field_name)->count(seq_id) == 0) {- return Option<uint32_t>(400, "Could not find `" + field_name + "` value for doc `" + std::to_string(seq_id)- + "`.");+ if (search_schema.count(field_name) == 0) {+ return Option<uint32_t>(400, "Could not find `" + field_name + "` in the collection `" + collection_name + "`.");+ } else if (search_schema.at(field_name).is_array()) {+ return Option<uint32_t>(400, "Cannot sort on `" + field_name + "` in the collection, `" + collection_name ++ "` is `" + search_schema.at(field_name).type + "`.");+ } else if (sort_index.count(field_name) == 0 || sort_index.at(field_name)->count(seq_id) == 0) {+ return Option<uint32_t>(400, "Could not find `" + field_name + "` value for doc `" ++ std::to_string(seq_id) + "`.");;}return Option<uint32_t>(sort_index.at(field_name)->at(seq_id));diff --git a/test/collection_join_test.cpp b/test/collection_join_test.cppindex 2007d8f8..0dedeea4 100644--- a/test/collection_join_test.cpp+++ b/test/collection_join_test.cpp@@ -445,8 +445,10 @@ TEST_F(CollectionJoinTest, IndexDocumentHavingReferenceField) {"ref_string_array_field": ["a", "d"]})"_json;add_doc_op = coll2->add(doc_json.dump());- ASSERT_FALSE(add_doc_op.ok());- ASSERT_EQ("No reference document matching `string_array_field:= [a,d]` found in the collection `coll1`.", add_doc_op.error());+ ASSERT_TRUE(add_doc_op.ok());++ result = coll2->search("*", {}, "", {}, {}, {0}).get();+ ASSERT_EQ(0, result["hits"][0]["document"]["ref_string_array_field_sequence_id"].size());doc_json = R"({"ref_string_array_field": ["b", "foo"]@@ -535,25 +537,27 @@ TEST_F(CollectionJoinTest, IndexDocumentHavingReferenceField) {ASSERT_EQ("Field `ref_int32_array_field` must only have `int32` values.", add_doc_op.error());doc_json = R"({- "ref_int32_array_field": [1]+ "ref_int32_array_field": [1, -2147483649]})"_json;add_doc_op = coll2->add(doc_json.dump());ASSERT_FALSE(add_doc_op.ok());- ASSERT_EQ("No reference document matching `int32_array_field: [1]` found in the collection `coll1`.", add_doc_op.error());+ ASSERT_EQ("Field `ref_int32_array_field` must only have `int32` values.", add_doc_op.error());doc_json = R"({- "ref_int32_array_field": [1, -2147483649]+ "ref_int32_array_field": [1, 2147483648]})"_json;add_doc_op = coll2->add(doc_json.dump());ASSERT_FALSE(add_doc_op.ok());ASSERT_EQ("Field `ref_int32_array_field` must only have `int32` values.", add_doc_op.error());doc_json = R"({- "ref_int32_array_field": [1, 2147483648]+ "ref_int32_array_field": [1]})"_json;add_doc_op = coll2->add(doc_json.dump());- ASSERT_FALSE(add_doc_op.ok());- ASSERT_EQ("Field `ref_int32_array_field` must only have `int32` values.", add_doc_op.error());+ ASSERT_TRUE(add_doc_op.ok());++ result = coll2->search("*", {}, "", {}, {}, {0}).get();+ ASSERT_EQ(0, result["hits"][0]["document"]["ref_int32_array_field_sequence_id"].size());doc_json = R"({"ref_int32_array_field": [1, 2]@@ -645,25 +649,27 @@ TEST_F(CollectionJoinTest, IndexDocumentHavingReferenceField) {ASSERT_EQ("Field `ref_int64_array_field` must only have `int64` values.", add_doc_op.error());doc_json = R"({- "ref_int64_array_field": [1]+ "ref_int64_array_field": [1, -9223372036854775809]})"_json;add_doc_op = coll2->add(doc_json.dump());ASSERT_FALSE(add_doc_op.ok());- ASSERT_EQ("No reference document matching `int64_array_field: [1]` found in the collection `coll1`.", add_doc_op.error());+ ASSERT_EQ("Field `ref_int64_array_field` must only have `int64` values.", add_doc_op.error());doc_json = R"({- "ref_int64_array_field": [1, -9223372036854775809]+ "ref_int64_array_field": [1, 1.5]})"_json;add_doc_op = coll2->add(doc_json.dump());ASSERT_FALSE(add_doc_op.ok());ASSERT_EQ("Field `ref_int64_array_field` must only have `int64` values.", add_doc_op.error());doc_json = R"({- "ref_int64_array_field": [1, 1.5]+ "ref_int64_array_field": [1]})"_json;add_doc_op = coll2->add(doc_json.dump());- ASSERT_FALSE(add_doc_op.ok());- ASSERT_EQ("Field `ref_int64_array_field` must only have `int64` values.", add_doc_op.error());+ ASSERT_TRUE(add_doc_op.ok());++ result = coll2->search("*", {}, "", {}, {}, {0}).get();+ ASSERT_EQ(0, result["hits"][0]["document"]["ref_int64_array_field_sequence_id"].size());doc_json = R"({"ref_int64_array_field": [1, 2]@@ -917,13 +923,13 @@ TEST_F(CollectionJoinTest, FilterByReference_SingleMatch) {auto res_obj = nlohmann::json::parse(json_res);ASSERT_EQ(1, res_obj["found"].get<size_t>());ASSERT_EQ(1, res_obj["hits"].size());- ASSERT_EQ("soap", result["hits"][0]["document"]["product_name"].get<std::string>());+ ASSERT_EQ("soap", res_obj["hits"][0]["document"]["product_name"].get<std::string>());req_params = {{"collection", "Customers"},{"q", "Dan"},{"query_by", "customer_name"},- {"filter_by", "$Products(id:*) && product_price:<100"},+ {"filter_by", "$Products(id:*) && product_price:>100"},};search_op_bool = collectionManager.do_search(req_params, embedded_params, json_res, now_ts);@@ -932,7 +938,7 @@ TEST_F(CollectionJoinTest, FilterByReference_SingleMatch) {res_obj = nlohmann::json::parse(json_res);ASSERT_EQ(1, res_obj["found"].get<size_t>());ASSERT_EQ(1, res_obj["hits"].size());- ASSERT_EQ("soap", result["hits"][0]["document"]["product_name"].get<std::string>());+ ASSERT_EQ("soap", res_obj["hits"][0]["document"]["product_name"].get<std::string>());collectionManager.drop_collection("Customers");collectionManager.drop_collection("Products");@@ -2285,6 +2291,109 @@ TEST_F(CollectionJoinTest, IncludeExcludeFieldsByReference) {ASSERT_EQ("sense", res_obj["hits"][0]["document"]["org.name"].at("last"));}+TEST_F(CollectionJoinTest, FilterByReferenceArrayField) {+ auto schema_json =+ R"({+ "name": "genres",+ "fields": [+ { "name": "id", "type": "string" },+ { "name": "name", "type": "string" }+ ]+ })"_json;+ std::vector<nlohmann::json> documents = {+ R"({"id":"0","name":"Grunge"})"_json,+ R"({"id":"1","name":"Arena rock"})"_json,+ R"({"id":"2","name":"Blues"})"_json+ };+ auto collection_create_op = collectionManager.create_collection(schema_json);+ ASSERT_TRUE(collection_create_op.ok());+ for (auto const &json: documents) {+ auto add_op = collection_create_op.get()->add(json.dump());+ if (!add_op.ok()) {+ LOG(INFO) << add_op.error();+ }+ ASSERT_TRUE(add_op.ok());+ }++ schema_json =+ R"({+ "name": "songs",+ "fields": [+ { "name": "title", "type": "string" },+ { "name": "genres", "type": "string[]", "reference": "genres.id"}+ ]+ })"_json;+ documents = {+ R"({"title":"Dil De Rani", "genres":[]})"_json,+ R"({"title":"Corduroy", "genres":["0"]})"_json,+ R"({"title":"Achilles Last Stand", "genres":["1","2"]})"_json+ };+ collection_create_op = collectionManager.create_collection(schema_json);+ ASSERT_TRUE(collection_create_op.ok());+ for (auto const &json: documents) {+ auto add_op = collection_create_op.get()->add(json.dump());+ if (!add_op.ok()) {+ LOG(INFO) << add_op.error();+ }+ ASSERT_TRUE(add_op.ok());+ }++ std::map<std::string, std::string> req_params = {+ {"collection", "songs"},+ {"q", "*"},+ {"include_fields", "$genres(name) as genre"},+ {"exclude_fields", "genres_sequence_id"},+ };+ nlohmann::json embedded_params;+ std::string json_res;+ auto now_ts = std::chrono::duration_cast<std::chrono::microseconds>(+ std::chrono::system_clock::now().time_since_epoch()).count();++ auto search_op_bool = collectionManager.do_search(req_params, embedded_params, json_res, now_ts);+ ASSERT_TRUE(search_op_bool.ok());++ auto res_obj = nlohmann::json::parse(json_res);+ ASSERT_EQ(3, res_obj["found"].get<size_t>());+ ASSERT_EQ(3, res_obj["hits"].size());++ ASSERT_EQ("Achilles Last Stand", res_obj["hits"][0]["document"]["title"].get<std::string>());+ ASSERT_EQ(2, res_obj["hits"][0]["document"]["genre.name"].size());+ ASSERT_EQ("Arena rock", res_obj["hits"][0]["document"]["genre.name"][0]);+ ASSERT_EQ("Blues", res_obj["hits"][0]["document"]["genre.name"][1]);++ ASSERT_EQ("Corduroy", res_obj["hits"][1]["document"]["title"].get<std::string>());+ ASSERT_EQ(1, res_obj["hits"][1]["document"]["genre.name"].size());+ ASSERT_EQ("Grunge", res_obj["hits"][1]["document"]["genre.name"][0]);++ ASSERT_EQ("Dil De Rani", res_obj["hits"][2]["document"]["title"].get<std::string>());+ ASSERT_EQ(0, res_obj["hits"][2]["document"]["genre.name"].size());++ req_params = {+ {"collection", "genres"},+ {"q", "*"},+ {"filter_by", "$songs(id: *)"},+ {"include_fields", "$songs(title) as song"},+ };+ search_op_bool = collectionManager.do_search(req_params, embedded_params, json_res, now_ts);+ ASSERT_TRUE(search_op_bool.ok());++ res_obj = nlohmann::json::parse(json_res);+ ASSERT_EQ(3, res_obj["found"].get<size_t>());+ ASSERT_EQ(3, res_obj["hits"].size());++ ASSERT_EQ("Blues", res_obj["hits"][0]["document"]["name"].get<std::string>());+ ASSERT_EQ(1, res_obj["hits"][0]["document"]["song.title"].size());+ ASSERT_EQ("Achilles Last Stand", res_obj["hits"][0]["document"]["song.title"][0]);++ ASSERT_EQ("Arena rock", res_obj["hits"][1]["document"]["name"].get<std::string>());+ ASSERT_EQ(1, res_obj["hits"][1]["document"]["song.title"].size());+ ASSERT_EQ("Achilles Last Stand", res_obj["hits"][1]["document"]["song.title"][0]);++ ASSERT_EQ("Grunge", res_obj["hits"][2]["document"]["name"].get<std::string>());+ ASSERT_EQ(1, res_obj["hits"][2]["document"]["song.title"].size());+ ASSERT_EQ("Corduroy", res_obj["hits"][2]["document"]["song.title"][0]);+}+TEST_F(CollectionJoinTest, CascadeDeletion) {auto schema_json =R"({diff --git a/test/collection_manager_test.cpp b/test/collection_manager_test.cppindex 81b92106..89e89986 100644--- a/test/collection_manager_test.cpp+++ b/test/collection_manager_test.cpp@@ -1436,11 +1436,11 @@ TEST_F(CollectionManagerTest, ReferencedInBacklog) {referenced_ins_backlog = collectionManager._get_referenced_in_backlog();ASSERT_EQ(0, referenced_ins_backlog.count("Products"));- auto get_reference_field_op = create_op.get()->get_reference_field("collection1");+ auto get_reference_field_op = create_op.get()->get_referenced_in_field_with_lock("collection1");ASSERT_TRUE(get_reference_field_op.ok());ASSERT_EQ("product_id_sequence_id", get_reference_field_op.get());- get_reference_field_op = create_op.get()->get_reference_field("foo");+ get_reference_field_op = create_op.get()->get_referenced_in_field_with_lock("foo");ASSERT_FALSE(get_reference_field_op.ok());ASSERT_EQ("Could not find any field in `Products` referencing the collection `foo`.", get_reference_field_op.error());}\ No newline at end of file