Skip to content

Open Source alternative to Algolia + Pinecone and an Easier-to-Use alternative to ElasticSearch ⚡ 🔍 ✨ Fast, typo tolerant, in-memory fuzzy Search Engine for building delightful search experiences

Add `reference_index` for reference array field.

Harpreet Sangar4ffbe3aad191
diff --git a/include/collection.h b/include/collection.h
index a34f6a46..3b2bbdd2 100644
--- a/include/collection.h
+++ b/include/collection.h
@@ -287,7 +287,10 @@ private:
tsl::htrie_set<char>& include_fields_full,
tsl::htrie_set<char>& exclude_fields_full) const;
- Option<uint32_t> get_reference_doc_id(const std::string& ref_collection_name, const uint32_t& seq_id) const;
+ Option<std::string> get_referenced_in_field(const std::string& collection_name) const;
+
+ Option<bool> get_related_ids(const std::string& ref_collection_name, const uint32_t& seq_id,
+ std::vector<uint32_t>& result) const;
static void hide_credential(nlohmann::json& json, const std::string& credential_name);
@@ -320,8 +323,6 @@ public:
static constexpr const char* COLLECTION_SYMBOLS_TO_INDEX = "symbols_to_index";
static constexpr const char* COLLECTION_SEPARATORS = "token_separators";
- static constexpr const char* REFERENCE_HELPER_FIELD_SUFFIX = "_sequence_id";
-
// methods
Collection() = delete;
@@ -398,7 +399,7 @@ public:
const reference_filter_result_t& references,
const tsl::htrie_set<char>& ref_include_fields_full,
const tsl::htrie_set<char>& ref_exclude_fields_full,
- const std::string& error_prefix);
+ const std::string& error_prefix, const bool& is_reference_array);
static Option<bool> prune_doc(nlohmann::json& doc, const tsl::htrie_set<char>& include_names,
const tsl::htrie_set<char>& exclude_names, const std::string& parent_name = "",
@@ -614,9 +615,12 @@ public:
void add_referenced_in(const std::string& collection_name, const std::string& field_name);
- Option<std::string> get_reference_field(const std::string& collection_name) const;
+ Option<std::string> get_referenced_in_field_with_lock(const std::string& collection_name) const;
+
+ Option<bool> get_related_ids_with_lock(const std::string& field_name, const uint32_t& seq_id,
+ std::vector<uint32_t>& result) const;
- Option<uint32_t> get_sort_indexed_field_value(const std::string& field_name, const uint32_t& seq_id) const;
+ Option<uint32_t> get_sort_index_value_with_lock(const std::string& field_name, const uint32_t& seq_id) const;
friend class filter_result_iterator_t;
};
diff --git a/include/field.h b/include/field.h
index c92b1ff4..a2612f53 100644
--- a/include/field.h
+++ b/include/field.h
@@ -66,6 +66,8 @@ namespace fields {
static const std::string query_prefix = "query_prefix";
static const std::string api_key = "api_key";
static const std::string model_config = "model_config";
+
+ static const std::string REFERENCE_HELPER_FIELD_SUFFIX = "_sequence_id";
}
enum vector_distance_type_t {
@@ -100,6 +102,8 @@ struct field {
bool range_index;
+ bool is_reference_helper = false;
+
field() {}
field(const std::string &name, const std::string &type, const bool facet, const bool optional = false,
@@ -111,6 +115,9 @@ struct field {
embed(embed), range_index(range_index) {
set_computed_defaults(sort, infix);
+
+ auto const suffix = std::string(fields::REFERENCE_HELPER_FIELD_SUFFIX);
+ is_reference_helper = name.size() > suffix.size() && name.substr(name.size() - suffix.size()) == suffix;
}
void set_computed_defaults(int sort, int infix) {
diff --git a/include/index.h b/include/index.h
index 2dbc88d6..823c8e43 100644
--- a/include/index.h
+++ b/include/index.h
@@ -343,6 +343,10 @@ private:
spp::sparse_hash_map<std::string, num_tree_t*> numerical_index;
+ // reference_helper_field => (seq_id => ref_seq_ids)
+ // Only used when the reference field is an array type otherwise sort_index is used.
+ spp::sparse_hash_map<std::string, num_tree_t*> reference_index;
+
spp::sparse_hash_map<std::string, NumericTrie*> range_index;
spp::sparse_hash_map<std::string, NumericTrie*> geo_range_index;
@@ -779,8 +783,6 @@ public:
int64_t reference_string_sort_score(const std::string& field_name, const uint32_t& seq_id) const;
- Option<uint32_t> get_sort_indexed_field_value(const std::string& field_name, const uint32_t& seq_id) const;
-
static void remove_matched_tokens(std::vector<std::string>& tokens, const std::set<std::string>& rule_token_set) ;
void compute_facet_infos(const std::vector<facet>& facets, facet_query_t& facet_query,
@@ -1000,7 +1002,12 @@ public:
Option<bool> seq_ids_outside_top_k(const std::string& field_name, size_t k,
std::vector<uint32_t>& outside_seq_ids);
- Option<uint32_t> get_reference_doc_id_with_lock(const std::string& reference_helper_field_name,
+ Option<bool> get_related_ids(const std::string& collection_name,
+ const std::string& reference_helper_field_name,
+ const uint32_t& seq_id, std::vector<uint32_t>& result) const;
+
+ Option<uint32_t> get_sort_index_value_with_lock(const std::string& collection_name,
+ const std::string& field_name,
const uint32_t& seq_id) const;
friend class filter_result_iterator_t;
diff --git a/src/collection.cpp b/src/collection.cpp
index af4ae784..112e322c 100644
--- a/src/collection.cpp
+++ b/src/collection.cpp
@@ -12,6 +12,7 @@
#include <regex>
#include <list>
#include <posting.h>
+#include <timsort.hpp>
#include "validator.h"
#include "topster.h"
#include "logger.h"
@@ -91,6 +92,7 @@ Option<bool> Collection::add_reference_helper_fields(nlohmann::json& document) {
if (reference_field_name == "id") {
auto id_field_type_error_op = Option<bool>(400, "Field `" + field_name + "` must have string value.");
if (document[field_name].is_array()) {
+ document[field_name + fields::REFERENCE_HELPER_FIELD_SUFFIX] = nlohmann::json::array();
for (const auto &item: document[field_name].items()) {
if (!item.value().is_string()) {
return id_field_type_error_op;
@@ -104,7 +106,7 @@ Option<bool> Collection::add_reference_helper_fields(nlohmann::json& document) {
reference_collection_name + "`." );
}
- document[field_name + REFERENCE_HELPER_FIELD_SUFFIX] += ref_doc_id_op.get();
+ document[field_name + fields::REFERENCE_HELPER_FIELD_SUFFIX] += ref_doc_id_op.get();
}
} else if (document[field_name].is_string()) {
auto id = document[field_name].get<std::string>();
@@ -115,7 +117,7 @@ Option<bool> Collection::add_reference_helper_fields(nlohmann::json& document) {
reference_collection_name + "`." );
}
- document[field_name + REFERENCE_HELPER_FIELD_SUFFIX] = ref_doc_id_op.get();
+ document[field_name + fields::REFERENCE_HELPER_FIELD_SUFFIX] = ref_doc_id_op.get();
} else {
return id_field_type_error_op;
}
@@ -185,13 +187,9 @@ Option<bool> Collection::add_reference_helper_fields(nlohmann::json& document) {
}
if (document[field_name].is_array()) {
- if (filter_result.count == 0) {
- return Option<bool>(400, "No reference document matching `" + filter_query + "` found in the collection `"
- + reference_collection_name + "`.");
- }
-
+ document[field_name + fields::REFERENCE_HELPER_FIELD_SUFFIX] = nlohmann::json::array();
for (uint32_t i = 0; i < filter_result.count; i++) {
- document[field_name + REFERENCE_HELPER_FIELD_SUFFIX] += filter_result.docs[i];
+ document[field_name + fields::REFERENCE_HELPER_FIELD_SUFFIX] += filter_result.docs[i];
}
} else {
if (filter_result.count != 1) {
@@ -203,7 +201,7 @@ Option<bool> Collection::add_reference_helper_fields(nlohmann::json& document) {
reference_collection_name + "`.");
}
- document[field_name + REFERENCE_HELPER_FIELD_SUFFIX] = filter_result.docs[0];
+ document[field_name + fields::REFERENCE_HELPER_FIELD_SUFFIX] = filter_result.docs[0];
}
}
@@ -3132,8 +3130,9 @@ Option<bool> Collection::get_filter_ids(const std::string& filter_query, filter_
return index->do_filtering_with_lock(filter_tree_root, filter_result, name);
}
-Option<uint32_t> Collection::get_reference_doc_id(const std::string& ref_field_name, const uint32_t& seq_id) const {
- return index->get_reference_doc_id_with_lock(ref_field_name, seq_id);
+Option<bool> Collection::get_related_ids(const std::string& ref_field_name, const uint32_t& seq_id,
+ std::vector<uint32_t>& result) const {
+ return index->get_related_ids(name, ref_field_name, seq_id, result);
}
Option<bool> Collection::get_reference_filter_ids(const std::string & filter_query,
@@ -4533,9 +4532,9 @@ Option<bool> Collection::add_reference_fields(nlohmann::json& doc,
const reference_filter_result_t& references,
const tsl::htrie_set<char>& ref_include_fields_full,
const tsl::htrie_set<char>& ref_exclude_fields_full,
- const std::string& error_prefix) {
+ const std::string& error_prefix, const bool& is_reference_array) {
// One-to-one relation.
- if (references.count == 1) {
+ if (!is_reference_array && references.count == 1) {
auto ref_doc_seq_id = references.docs[0];
nlohmann::json ref_doc;
@@ -4744,54 +4743,74 @@ Option<bool> Collection::prune_doc(nlohmann::json& doc,
Option<bool> add_reference_fields_op = Option<bool>(true);
if (has_filter_reference) {
+ auto get_reference_field_op = collection->get_referenced_in_field(ref_collection_name);
+ if (!get_reference_field_op.ok()) {
+ continue;
+ }
+ auto const& field_name = get_reference_field_op.get();
+ if (ref_collection->search_schema.count(field_name) == 0) {
+ continue;
+ }
add_reference_fields_op = add_reference_fields(doc, ref_collection.get(), ref_include.alias,
reference_filter_results.at(ref_collection_name),
- ref_include_fields_full, ref_exclude_fields_full,
- error_prefix);
+ ref_include_fields_full, ref_exclude_fields_full, error_prefix,
+ ref_collection->get_schema().at(field_name).is_array());
} else if (doc_has_reference) {
- auto get_reference_field_op = ref_collection->get_reference_field(collection->name);
+ auto get_reference_field_op = ref_collection->get_referenced_in_field_with_lock(collection->name);
if (!get_reference_field_op.ok()) {
continue;
}
auto const& field_name = get_reference_field_op.get();
- auto get_reference_doc_id_op = collection->get_reference_doc_id(field_name, seq_id);
- if (!get_reference_doc_id_op.ok()) {
+ if (collection->search_schema.count(field_name) == 0) {
+ continue;
+ }
+
+ reference_filter_result_t result;
+ std::vector<uint32_t> ids;
+ auto get_references_op = collection->get_related_ids(field_name, seq_id, ids);
+ if (!get_references_op.ok()) {
continue;
}
+ result.count = ids.size();
+ result.docs = &ids[0];
- reference_filter_result_t r{1, new uint32[1]{get_reference_doc_id_op.get()}};
- add_reference_fields_op = add_reference_fields(doc, ref_collection.get(), ref_include.alias, r,
- ref_include_fields_full, ref_exclude_fields_full,
- error_prefix);
+ add_reference_fields_op = add_reference_fields(doc, ref_collection.get(), ref_include.alias, result,
+ ref_include_fields_full, ref_exclude_fields_full, error_prefix,
+ collection->search_schema.at(field_name).is_array());
+ result.docs = nullptr;
} else if (joined_coll_has_reference) {
auto joined_collection = cm.get_collection(joined_coll_having_reference);
if (joined_collection == nullptr) {
continue;
}
- auto reference_field_name_op = ref_collection->get_reference_field(joined_coll_having_reference);
- if (!reference_field_name_op.ok()) {
+ auto reference_field_name_op = ref_collection->get_referenced_in_field_with_lock(joined_coll_having_reference);
+ if (!reference_field_name_op.ok() || joined_collection->get_schema().count(reference_field_name_op.get()) == 0) {
continue;
}
auto const& reference_field_name = reference_field_name_op.get();
auto const& reference_filter_result = reference_filter_results.at(joined_coll_having_reference);
auto const& count = reference_filter_result.count;
- reference_filter_result_t r{count, new uint32[count]};
-
+ std::vector<uint32_t> ids;
+ ids.reserve(count);
for (uint32_t i = 0; i < count; i++) {
- auto op = joined_collection->get_sort_indexed_field_value(reference_field_name,
- reference_filter_result.docs[i]);
- if (!op.ok()) {
- return Option<bool>(op.code(), error_prefix + op.error());
- }
-
- r.docs[i] = op.get();
+ joined_collection->get_related_ids_with_lock(reference_field_name, reference_filter_result.docs[i], ids);
+ }
+ if (ids.empty()) {
+ continue;
}
- add_reference_fields_op = add_reference_fields(doc, ref_collection.get(), ref_include.alias, r,
- ref_include_fields_full, ref_exclude_fields_full,
- error_prefix);
+ gfx::timsort(ids.begin(), ids.end());
+ ids.erase(unique(ids.begin(), ids.end()), ids.end());
+
+ reference_filter_result_t result;
+ result.count = ids.size();
+ result.docs = &ids[0];
+ add_reference_fields_op = add_reference_fields(doc, ref_collection.get(), ref_include.alias, result,
+ ref_include_fields_full, ref_exclude_fields_full, error_prefix,
+ joined_collection->get_schema().at(reference_field_name).is_array());
+ result.docs = nullptr;
}
if (!add_reference_fields_op.ok()) {
@@ -5345,11 +5364,11 @@ Index* Collection::init_index() {
auto ref_coll = collectionManager.get_collection(ref_coll_name);
if (ref_coll != nullptr) {
// Passing reference helper field helps perform operation on doc_id instead of field value.
- ref_coll->add_referenced_in(name, field.name + REFERENCE_HELPER_FIELD_SUFFIX);
+ ref_coll->add_referenced_in(name, field.name + fields::REFERENCE_HELPER_FIELD_SUFFIX);
} else {
// Reference collection has not been created yet.
collectionManager.add_referenced_in_backlog(ref_coll_name,
- reference_pair{name, field.name + REFERENCE_HELPER_FIELD_SUFFIX});
+ reference_pair{name, field.name + fields::REFERENCE_HELPER_FIELD_SUFFIX});
}
}
}
@@ -5804,9 +5823,12 @@ void Collection::add_referenced_in(const std::string& collection_name, const std
referenced_in.emplace(collection_name, field_name);
}
-Option<std::string> Collection::get_reference_field(const std::string& collection_name) const {
+Option<std::string> Collection::get_referenced_in_field_with_lock(const std::string& collection_name) const {
std::shared_lock lock(mutex);
+ return get_referenced_in_field(collection_name);
+}
+Option<std::string> Collection::get_referenced_in_field(const std::string& collection_name) const {
if (referenced_in.count(collection_name) == 0) {
return Option<std::string>(400, "Could not find any field in `" + name + "` referencing the collection `"
+ collection_name + "`.");
@@ -5815,9 +5837,16 @@ Option<std::string> Collection::get_reference_field(const std::string& collectio
return Option<std::string>(referenced_in.at(collection_name));
}
-Option<uint32_t> Collection::get_sort_indexed_field_value(const std::string& field_name, const uint32_t& seq_id) const {
+Option<bool> Collection::get_related_ids_with_lock(const std::string& field_name, const uint32_t& seq_id,
+ std::vector<uint32_t>& result) const {
+ std::shared_lock lock(mutex);
+ return index->get_related_ids(name, field_name, seq_id, result);
+}
+
+Option<uint32_t> Collection::get_sort_index_value_with_lock(const std::string& field_name,
+ const uint32_t& seq_id) const {
std::shared_lock lock(mutex);
- return index->get_sort_indexed_field_value(field_name, seq_id);
+ return index->get_sort_index_value_with_lock(name, field_name, seq_id);
}
void Collection::remove_embedding_field(const std::string& field_name) {
diff --git a/src/collection_manager.cpp b/src/collection_manager.cpp
index 6129c63c..7c44ac37 100644
--- a/src/collection_manager.cpp
+++ b/src/collection_manager.cpp
@@ -263,7 +263,7 @@ Option<bool> CollectionManager::load(const size_t collection_batch_size, const s
if (referenced_ins.count(ref_coll_name) == 0) {
referenced_ins[ref_coll_name] = {};
}
- auto const field_name = item.first + Collection::REFERENCE_HELPER_FIELD_SUFFIX;
+ auto const field_name = item.first + fields::REFERENCE_HELPER_FIELD_SUFFIX;
referenced_ins.at(ref_coll_name).insert(reference_pair{collection_name, field_name});
}
}
diff --git a/src/field.cpp b/src/field.cpp
index 894fbb4e..2ee6bc37 100644
--- a/src/field.cpp
+++ b/src/field.cpp
@@ -328,7 +328,7 @@ Option<bool> field::json_field_to_field(bool enable_nested_fields, nlohmann::jso
// Add a reference helper field in the schema. It stores the doc id of the document it references to reduce the
// computation while searching.
the_fields.emplace_back(
- field(field_json[fields::name].get<std::string>() + Collection::REFERENCE_HELPER_FIELD_SUFFIX,
+ field(field_json[fields::name].get<std::string>() + fields::REFERENCE_HELPER_FIELD_SUFFIX,
field_types::is_array(field_json[fields::type].get<std::string>()) ? field_types::INT64_ARRAY : field_types::INT64,
false, field_json[fields::optional], true)
);
diff --git a/src/filter_result_iterator.cpp b/src/filter_result_iterator.cpp
index ed3bbfcd..e1c14489 100644
--- a/src/filter_result_iterator.cpp
+++ b/src/filter_result_iterator.cpp
@@ -647,7 +647,7 @@ void filter_result_iterator_t::init() {
return;
}
- auto get_reference_field_op = ref_collection->get_reference_field(collection_name);
+ auto get_reference_field_op = ref_collection->get_referenced_in_field_with_lock(collection_name);
if (!get_reference_field_op.ok()) {
status = Option<bool>(get_reference_field_op.code(), get_reference_field_op.error());
is_valid = false;
diff --git a/src/index.cpp b/src/index.cpp
index 4f1877fc..04987fc7 100644
--- a/src/index.cpp
+++ b/src/index.cpp
@@ -124,6 +124,11 @@ Index::Index(const std::string& name, const uint32_t collection_id, const Store*
infix_index.emplace(a_field.name, infix_sets);
}
+
+ if (a_field.is_reference_helper && a_field.is_array()) {
+ auto num_tree = new num_tree_t;
+ reference_index.emplace(a_field.name, num_tree);
+ }
}
num_documents = 0;
@@ -202,6 +207,13 @@ Index::~Index() {
for(auto& vec_index_kv: vector_index) {
delete vec_index_kv.second;
}
+
+ for(auto & name_tree: reference_index) {
+ delete name_tree.second;
+ name_tree.second = nullptr;
+ }
+
+ reference_index.clear();
}
int64_t Index::get_points_from_doc(const nlohmann::json &document, const std::string & default_sorting_field) {
@@ -992,7 +1004,8 @@ void Index::index_field_in_memory(const field& afield, std::vector<index_record>
// all other numerical arrays
auto num_tree = afield.range_index ? nullptr : numerical_index.at(afield.name);
auto trie = afield.range_index ? range_index.at(afield.name) : nullptr;
- iterate_and_index_numerical_field(iter_batch, afield, [&afield, num_tree, trie]
+ auto reference = afield.is_reference_helper ? reference_index.at(afield.name) : nullptr;
+ iterate_and_index_numerical_field(iter_batch, afield, [&afield, num_tree, trie, reference]
(const index_record& record, uint32_t seq_id) {
for(size_t arr_i = 0; arr_i < record.doc[afield.name].size(); arr_i++) {
const auto& arr_value = record.doc[afield.name][arr_i];
@@ -1013,6 +1026,9 @@ void Index::index_field_in_memory(const field& afield, std::vector<index_record>
} else {
num_tree->insert(value, seq_id);
}
+ if (reference != nullptr) {
+ reference->insert(seq_id, value);
+ }
}
else if(afield.type == field_types::FLOAT_ARRAY) {
@@ -1762,59 +1778,87 @@ Option<bool> Index::do_reference_filtering_with_lock(filter_node_t* const filter
return Option(true);
}
- // Collect all the doc ids from the reference ids.
- std::vector<std::pair<uint32_t, uint32_t>> id_pairs;
- std::unordered_set<uint32_t> unique_doc_ids;
- auto const& ref_index = *sort_index.at(reference_helper_field_name);
- for (uint32_t i = 0; i < count; i++) {
- auto& reference_doc_id = reference_docs[i];
- auto doc_id = ref_index.at(reference_doc_id);
+ if (search_schema.at(reference_helper_field_name).is_singular()) {
+ // Collect all the doc ids from the reference ids.
+ std::vector<std::pair<uint32_t, uint32_t>> id_pairs;
+ std::unordered_set<uint32_t> unique_doc_ids;
+ auto const& ref_index = *sort_index.at(reference_helper_field_name);
+ for (uint32_t i = 0; i < count; i++) {
+ auto& reference_doc_id = reference_docs[i];
+ auto doc_id = ref_index.at(reference_doc_id);
- id_pairs.emplace_back(std::pair(doc_id, reference_doc_id));
- unique_doc_ids.insert(doc_id);
- }
+ id_pairs.emplace_back(std::pair(doc_id, reference_doc_id));
+ unique_doc_ids.insert(doc_id);
+ }
- std::sort(id_pairs.begin(), id_pairs.end(), [](auto const& left, auto const& right) {
- return left.first < right.first;
- });
+ std::sort(id_pairs.begin(), id_pairs.end(), [](auto const& left, auto const& right) {
+ return left.first < right.first;
+ });
+
+ filter_result.count = unique_doc_ids.size();
+ filter_result.docs = new uint32_t[unique_doc_ids.size()];
+ filter_result.coll_to_references = new std::map<std::string, reference_filter_result_t>[unique_doc_ids.size()] {};
- filter_result.count = unique_doc_ids.size();
- filter_result.docs = new uint32_t[unique_doc_ids.size()];
- filter_result.coll_to_references = new std::map<std::string, reference_filter_result_t>[unique_doc_ids.size()] {};
+ std::vector<uint32_t> previous_doc_references;
+ for (uint32_t i = 0, previous_doc = id_pairs[0].first + 1, result_index = 0; i < id_pairs.size(); i++) {
+ auto const& current_doc = id_pairs[i].first;
+ auto const& reference_doc_id = id_pairs[i].second;
- std::vector<uint32_t> previous_doc_references;
- for (uint32_t i = 0, previous_doc = id_pairs[0].first + 1, result_index = 0; i < id_pairs.size(); i++) {
- auto const& current_doc = id_pairs[i].first;
- auto const& reference_doc_id = id_pairs[i].second;
+ if (current_doc != previous_doc) {
+ filter_result.docs[result_index] = current_doc;
+ if (result_index > 0) {
+ auto& reference_result = filter_result.coll_to_references[result_index - 1];
- if (current_doc != previous_doc) {
- filter_result.docs[result_index] = current_doc;
- if (result_index > 0) {
- auto& reference_result = filter_result.coll_to_references[result_index - 1];
+ auto r = reference_filter_result_t(previous_doc_references.size(), new uint32_t[previous_doc_references.size()]);
+ std::copy(previous_doc_references.begin(), previous_doc_references.end(), r.docs);
+ reference_result[collection_name] = std::move(r);
- auto r = reference_filter_result_t(previous_doc_references.size(), new uint32_t[previous_doc_references.size()]);
- std::copy(previous_doc_references.begin(), previous_doc_references.end(), r.docs);
- reference_result[collection_name] = std::move(r);
+ previous_doc_references.clear();
+ }
- previous_doc_references.clear();
+ result_index++;
+ previous_doc = current_doc;
+ previous_doc_references.push_back(reference_doc_id);
+ } else {
+ previous_doc_references.push_back(reference_doc_id);
}
+ }
- result_index++;
- previous_doc = current_doc;
- previous_doc_references.push_back(reference_doc_id);
- } else {
- previous_doc_references.push_back(reference_doc_id);
+ if (!previous_doc_references.empty()) {
+ auto& reference_result = filter_result.coll_to_references[filter_result.count - 1];
+
+ auto r = reference_filter_result_t(previous_doc_references.size(), new uint32_t[previous_doc_references.size()]);
+ std::copy(previous_doc_references.begin(), previous_doc_references.end(), r.docs);
+ reference_result[collection_name] = std::move(r);
}
+
+ return Option(true);
}
- if (!previous_doc_references.empty()) {
- auto& reference_result = filter_result.coll_to_references[filter_result.count - 1];
+ size_t ids_len = 0;
+ uint32_t *ids = nullptr;
+ auto& ref_index = *reference_index.at(reference_helper_field_name);
+ for (uint32_t i = 0; i < count; i++) {
+ auto& reference_doc_id = reference_docs[i];
+ ref_index.search(EQUALS, reference_doc_id, &ids, ids_len);
+ }
+
+ filter_result.count = ids_len;
+ filter_result.docs = new uint32_t[ids_len];
+ filter_result.coll_to_references = new std::map<std::string, reference_filter_result_t>[ids_len] {};
+
+ auto& num_index = *numerical_index.at(reference_helper_field_name);
+ for (size_t i = 0; i < ids_len; i++) {
+ filter_result.docs[i] = ids[i];
- auto r = reference_filter_result_t(previous_doc_references.size(), new uint32_t[previous_doc_references.size()]);
- std::copy(previous_doc_references.begin(), previous_doc_references.end(), r.docs);
- reference_result[collection_name] = std::move(r);
+ reference_filter_result_t reference_result;
+ size_t len = 0;
+ num_index.search(EQUALS, ids[i], &reference_result.docs, len);
+ reference_result.count = len;
+ filter_result.coll_to_references[i][collection_name] = std::move(reference_result);
}
+ delete [] ids;
return Option(true);
}
@@ -4198,7 +4242,7 @@ Option<bool> Index::compute_sort_scores(const std::vector<sort_by>& sort_fields,
// Current collection has a reference.
if (ref_collection->is_referenced_in(collection_name)) {
- auto get_reference_field_op = ref_collection->get_reference_field(collection_name);
+ auto get_reference_field_op = ref_collection->get_referenced_in_field_with_lock(collection_name);
if (!get_reference_field_op.ok()) {
return Option<bool>(get_reference_field_op.code(), get_reference_field_op.error());
}
@@ -4229,7 +4273,7 @@ Option<bool> Index::compute_sort_scores(const std::vector<sort_by>& sort_fields,
"` in `sort_by` not found.");
}
- auto reference_field_name_op = ref_collection->get_reference_field(joined_coll_having_reference);
+ auto reference_field_name_op = ref_collection->get_referenced_in_field_with_lock(joined_coll_having_reference);
if (!reference_field_name_op.ok()) {
return Option<bool>(reference_field_name_op.code(), reference_field_name_op.error());
}
@@ -4239,8 +4283,8 @@ Option<bool> Index::compute_sort_scores(const std::vector<sort_by>& sort_fields,
auto const& count = reference.count;
if (count == 1) {
- auto op = joined_collection->get_sort_indexed_field_value(reference_field_name,
- reference.docs[0]);
+ auto op = joined_collection->get_sort_index_value_with_lock(reference_field_name,
+ reference.docs[0]);
if (!op.ok()) {
return Option<bool>(op.code(), op.error());
}
@@ -4361,7 +4405,7 @@ Option<bool> Index::compute_sort_scores(const std::vector<sort_by>& sort_fields,
// Current collection has a reference.
if (ref_collection->is_referenced_in(collection_name)) {
- auto get_reference_field_op = ref_collection->get_reference_field(collection_name);
+ auto get_reference_field_op = ref_collection->get_referenced_in_field_with_lock(collection_name);
if (!get_reference_field_op.ok()) {
return Option<bool>(get_reference_field_op.code(), get_reference_field_op.error());
}
@@ -4392,7 +4436,7 @@ Option<bool> Index::compute_sort_scores(const std::vector<sort_by>& sort_fields,
"` in `sort_by` not found.");
}
- auto reference_field_name_op = ref_collection->get_reference_field(joined_coll_having_reference);
+ auto reference_field_name_op = ref_collection->get_referenced_in_field_with_lock(joined_coll_having_reference);
if (!reference_field_name_op.ok()) {
return Option<bool>(reference_field_name_op.code(), reference_field_name_op.error());
}
@@ -4402,8 +4446,8 @@ Option<bool> Index::compute_sort_scores(const std::vector<sort_by>& sort_fields,
auto const& count = reference.count;
if (count == 1) {
- auto op = joined_collection->get_sort_indexed_field_value(reference_field_name,
- reference.docs[0]);
+ auto op = joined_collection->get_sort_index_value_with_lock(reference_field_name,
+ reference.docs[0]);
if (!op.ok()) {
return Option<bool>(op.code(), op.error());
}
@@ -4520,7 +4564,7 @@ Option<bool> Index::compute_sort_scores(const std::vector<sort_by>& sort_fields,
// Current collection has a reference.
if (ref_collection->is_referenced_in(collection_name)) {
- auto get_reference_field_op = ref_collection->get_reference_field(collection_name);
+ auto get_reference_field_op = ref_collection->get_referenced_in_field_with_lock(collection_name);
if (!get_reference_field_op.ok()) {
return Option<bool>(get_reference_field_op.code(), get_reference_field_op.error());
}
@@ -4551,7 +4595,7 @@ Option<bool> Index::compute_sort_scores(const std::vector<sort_by>& sort_fields,
"` in `sort_by` not found.");
}
- auto reference_field_name_op = ref_collection->get_reference_field(joined_coll_having_reference);
+ auto reference_field_name_op = ref_collection->get_referenced_in_field_with_lock(joined_coll_having_reference);
if (!reference_field_name_op.ok()) {
return Option<bool>(reference_field_name_op.code(), reference_field_name_op.error());
}
@@ -4561,8 +4605,8 @@ Option<bool> Index::compute_sort_scores(const std::vector<sort_by>& sort_fields,
auto const& count = reference.count;
if (count == 1) {
- auto op = joined_collection->get_sort_indexed_field_value(reference_field_name,
- reference.docs[0]);
+ auto op = joined_collection->get_sort_index_value_with_lock(reference_field_name,
+ reference.docs[0]);
if (!op.ok()) {
return Option<bool>(op.code(), op.error());
}
@@ -6957,27 +7001,59 @@ void Index::batch_embed_fields(std::vector<index_record*>& records,
}
}
-Option<uint32_t> Index::get_reference_doc_id_with_lock(const string& reference_helper_field_name,
- const uint32_t& seq_id) const {
+int64_t Index::reference_string_sort_score(const string &field_name, const uint32_t &seq_id) const {
std::shared_lock lock(mutex);
- if (sort_index.count(reference_helper_field_name) == 0 ||
- sort_index.at(reference_helper_field_name)->count(seq_id) == 0) {
- return Option<uint32_t>(400, "Could not find a reference for doc " + std::to_string(seq_id));
- }
-
- return Option<uint32_t>(sort_index.at(reference_helper_field_name)->at(seq_id));
+ return str_sort_index.at(field_name)->rank(seq_id);
}
-int64_t Index::reference_string_sort_score(const string &field_name, const uint32_t &seq_id) const {
+Option<bool> Index::get_related_ids(const std::string& collection_name, const string& field_name,
+ const uint32_t& seq_id, std::vector<uint32_t>& result) const {
std::shared_lock lock(mutex);
- return str_sort_index.at(field_name)->rank(seq_id);
+ if (search_schema.count(field_name) == 0) {
+ return Option<bool>(400, "Could not find `" + field_name + "` in the collection `" + collection_name + "`.");
+ }
+
+ auto const no_match_op = Option<bool>(400, "Could not find `" + field_name + "` value for doc `" +
+ std::to_string(seq_id) + "`.");
+ if (search_schema.at(field_name).is_singular()) {
+ if (sort_index.count(field_name) == 0 || sort_index.at(field_name)->count(seq_id) == 0) {
+ return no_match_op;
+ }
+
+ result.emplace_back(sort_index.at(field_name)->at(seq_id));
+ return Option<bool>(true);
+ }
+
+ if (reference_index.count(field_name) == 0) {
+ return no_match_op;
+ }
+
+ size_t ids_len = 0;
+ uint32_t* ids = nullptr;
+ reference_index.at(field_name)->search(EQUALS, seq_id, &ids, ids_len);
+ if (ids_len == 0) {
+ return no_match_op;
+ }
+
+ for (uint32_t i = 0; i < ids_len; i++) {
+ result.emplace_back(ids[i]);
+ }
+ delete [] ids;
+ return Option<bool>(true);
}
-Option<uint32_t> Index::get_sort_indexed_field_value(const string& field_name, const uint32_t& seq_id) const {
+Option<uint32_t> Index::get_sort_index_value_with_lock(const std::string& collection_name,
+ const std::string& field_name,
+ const uint32_t& seq_id) const {
std::shared_lock lock(mutex);
- if (sort_index.count(field_name) == 0 || sort_index.at(field_name)->count(seq_id) == 0) {
- return Option<uint32_t>(400, "Could not find `" + field_name + "` value for doc `" + std::to_string(seq_id)
- + "`.");
+ if (search_schema.count(field_name) == 0) {
+ return Option<uint32_t>(400, "Could not find `" + field_name + "` in the collection `" + collection_name + "`.");
+ } else if (search_schema.at(field_name).is_array()) {
+ return Option<uint32_t>(400, "Cannot sort on `" + field_name + "` in the collection, `" + collection_name +
+ "` is `" + search_schema.at(field_name).type + "`.");
+ } else if (sort_index.count(field_name) == 0 || sort_index.at(field_name)->count(seq_id) == 0) {
+ return Option<uint32_t>(400, "Could not find `" + field_name + "` value for doc `" +
+ std::to_string(seq_id) + "`.");;
}
return Option<uint32_t>(sort_index.at(field_name)->at(seq_id));
diff --git a/test/collection_join_test.cpp b/test/collection_join_test.cpp
index 2007d8f8..0dedeea4 100644
--- a/test/collection_join_test.cpp
+++ b/test/collection_join_test.cpp
@@ -445,8 +445,10 @@ TEST_F(CollectionJoinTest, IndexDocumentHavingReferenceField) {
"ref_string_array_field": ["a", "d"]
})"_json;
add_doc_op = coll2->add(doc_json.dump());
- ASSERT_FALSE(add_doc_op.ok());
- ASSERT_EQ("No reference document matching `string_array_field:= [a,d]` found in the collection `coll1`.", add_doc_op.error());
+ ASSERT_TRUE(add_doc_op.ok());
+
+ result = coll2->search("*", {}, "", {}, {}, {0}).get();
+ ASSERT_EQ(0, result["hits"][0]["document"]["ref_string_array_field_sequence_id"].size());
doc_json = R"({
"ref_string_array_field": ["b", "foo"]
@@ -535,25 +537,27 @@ TEST_F(CollectionJoinTest, IndexDocumentHavingReferenceField) {
ASSERT_EQ("Field `ref_int32_array_field` must only have `int32` values.", add_doc_op.error());
doc_json = R"({
- "ref_int32_array_field": [1]
+ "ref_int32_array_field": [1, -2147483649]
})"_json;
add_doc_op = coll2->add(doc_json.dump());
ASSERT_FALSE(add_doc_op.ok());
- ASSERT_EQ("No reference document matching `int32_array_field: [1]` found in the collection `coll1`.", add_doc_op.error());
+ ASSERT_EQ("Field `ref_int32_array_field` must only have `int32` values.", add_doc_op.error());
doc_json = R"({
- "ref_int32_array_field": [1, -2147483649]
+ "ref_int32_array_field": [1, 2147483648]
})"_json;
add_doc_op = coll2->add(doc_json.dump());
ASSERT_FALSE(add_doc_op.ok());
ASSERT_EQ("Field `ref_int32_array_field` must only have `int32` values.", add_doc_op.error());
doc_json = R"({
- "ref_int32_array_field": [1, 2147483648]
+ "ref_int32_array_field": [1]
})"_json;
add_doc_op = coll2->add(doc_json.dump());
- ASSERT_FALSE(add_doc_op.ok());
- ASSERT_EQ("Field `ref_int32_array_field` must only have `int32` values.", add_doc_op.error());
+ ASSERT_TRUE(add_doc_op.ok());
+
+ result = coll2->search("*", {}, "", {}, {}, {0}).get();
+ ASSERT_EQ(0, result["hits"][0]["document"]["ref_int32_array_field_sequence_id"].size());
doc_json = R"({
"ref_int32_array_field": [1, 2]
@@ -645,25 +649,27 @@ TEST_F(CollectionJoinTest, IndexDocumentHavingReferenceField) {
ASSERT_EQ("Field `ref_int64_array_field` must only have `int64` values.", add_doc_op.error());
doc_json = R"({
- "ref_int64_array_field": [1]
+ "ref_int64_array_field": [1, -9223372036854775809]
})"_json;
add_doc_op = coll2->add(doc_json.dump());
ASSERT_FALSE(add_doc_op.ok());
- ASSERT_EQ("No reference document matching `int64_array_field: [1]` found in the collection `coll1`.", add_doc_op.error());
+ ASSERT_EQ("Field `ref_int64_array_field` must only have `int64` values.", add_doc_op.error());
doc_json = R"({
- "ref_int64_array_field": [1, -9223372036854775809]
+ "ref_int64_array_field": [1, 1.5]
})"_json;
add_doc_op = coll2->add(doc_json.dump());
ASSERT_FALSE(add_doc_op.ok());
ASSERT_EQ("Field `ref_int64_array_field` must only have `int64` values.", add_doc_op.error());
doc_json = R"({
- "ref_int64_array_field": [1, 1.5]
+ "ref_int64_array_field": [1]
})"_json;
add_doc_op = coll2->add(doc_json.dump());
- ASSERT_FALSE(add_doc_op.ok());
- ASSERT_EQ("Field `ref_int64_array_field` must only have `int64` values.", add_doc_op.error());
+ ASSERT_TRUE(add_doc_op.ok());
+
+ result = coll2->search("*", {}, "", {}, {}, {0}).get();
+ ASSERT_EQ(0, result["hits"][0]["document"]["ref_int64_array_field_sequence_id"].size());
doc_json = R"({
"ref_int64_array_field": [1, 2]
@@ -917,13 +923,13 @@ TEST_F(CollectionJoinTest, FilterByReference_SingleMatch) {
auto res_obj = nlohmann::json::parse(json_res);
ASSERT_EQ(1, res_obj["found"].get<size_t>());
ASSERT_EQ(1, res_obj["hits"].size());
- ASSERT_EQ("soap", result["hits"][0]["document"]["product_name"].get<std::string>());
+ ASSERT_EQ("soap", res_obj["hits"][0]["document"]["product_name"].get<std::string>());
req_params = {
{"collection", "Customers"},
{"q", "Dan"},
{"query_by", "customer_name"},
- {"filter_by", "$Products(id:*) && product_price:<100"},
+ {"filter_by", "$Products(id:*) && product_price:>100"},
};
search_op_bool = collectionManager.do_search(req_params, embedded_params, json_res, now_ts);
@@ -932,7 +938,7 @@ TEST_F(CollectionJoinTest, FilterByReference_SingleMatch) {
res_obj = nlohmann::json::parse(json_res);
ASSERT_EQ(1, res_obj["found"].get<size_t>());
ASSERT_EQ(1, res_obj["hits"].size());
- ASSERT_EQ("soap", result["hits"][0]["document"]["product_name"].get<std::string>());
+ ASSERT_EQ("soap", res_obj["hits"][0]["document"]["product_name"].get<std::string>());
collectionManager.drop_collection("Customers");
collectionManager.drop_collection("Products");
@@ -2285,6 +2291,109 @@ TEST_F(CollectionJoinTest, IncludeExcludeFieldsByReference) {
ASSERT_EQ("sense", res_obj["hits"][0]["document"]["org.name"].at("last"));
}
+TEST_F(CollectionJoinTest, FilterByReferenceArrayField) {
+ auto schema_json =
+ R"({
+ "name": "genres",
+ "fields": [
+ { "name": "id", "type": "string" },
+ { "name": "name", "type": "string" }
+ ]
+ })"_json;
+ std::vector<nlohmann::json> documents = {
+ R"({"id":"0","name":"Grunge"})"_json,
+ R"({"id":"1","name":"Arena rock"})"_json,
+ R"({"id":"2","name":"Blues"})"_json
+ };
+ auto collection_create_op = collectionManager.create_collection(schema_json);
+ ASSERT_TRUE(collection_create_op.ok());
+ for (auto const &json: documents) {
+ auto add_op = collection_create_op.get()->add(json.dump());
+ if (!add_op.ok()) {
+ LOG(INFO) << add_op.error();
+ }
+ ASSERT_TRUE(add_op.ok());
+ }
+
+ schema_json =
+ R"({
+ "name": "songs",
+ "fields": [
+ { "name": "title", "type": "string" },
+ { "name": "genres", "type": "string[]", "reference": "genres.id"}
+ ]
+ })"_json;
+ documents = {
+ R"({"title":"Dil De Rani", "genres":[]})"_json,
+ R"({"title":"Corduroy", "genres":["0"]})"_json,
+ R"({"title":"Achilles Last Stand", "genres":["1","2"]})"_json
+ };
+ collection_create_op = collectionManager.create_collection(schema_json);
+ ASSERT_TRUE(collection_create_op.ok());
+ for (auto const &json: documents) {
+ auto add_op = collection_create_op.get()->add(json.dump());
+ if (!add_op.ok()) {
+ LOG(INFO) << add_op.error();
+ }
+ ASSERT_TRUE(add_op.ok());
+ }
+
+ std::map<std::string, std::string> req_params = {
+ {"collection", "songs"},
+ {"q", "*"},
+ {"include_fields", "$genres(name) as genre"},
+ {"exclude_fields", "genres_sequence_id"},
+ };
+ nlohmann::json embedded_params;
+ std::string json_res;
+ auto now_ts = std::chrono::duration_cast<std::chrono::microseconds>(
+ std::chrono::system_clock::now().time_since_epoch()).count();
+
+ auto search_op_bool = collectionManager.do_search(req_params, embedded_params, json_res, now_ts);
+ ASSERT_TRUE(search_op_bool.ok());
+
+ auto res_obj = nlohmann::json::parse(json_res);
+ ASSERT_EQ(3, res_obj["found"].get<size_t>());
+ ASSERT_EQ(3, res_obj["hits"].size());
+
+ ASSERT_EQ("Achilles Last Stand", res_obj["hits"][0]["document"]["title"].get<std::string>());
+ ASSERT_EQ(2, res_obj["hits"][0]["document"]["genre.name"].size());
+ ASSERT_EQ("Arena rock", res_obj["hits"][0]["document"]["genre.name"][0]);
+ ASSERT_EQ("Blues", res_obj["hits"][0]["document"]["genre.name"][1]);
+
+ ASSERT_EQ("Corduroy", res_obj["hits"][1]["document"]["title"].get<std::string>());
+ ASSERT_EQ(1, res_obj["hits"][1]["document"]["genre.name"].size());
+ ASSERT_EQ("Grunge", res_obj["hits"][1]["document"]["genre.name"][0]);
+
+ ASSERT_EQ("Dil De Rani", res_obj["hits"][2]["document"]["title"].get<std::string>());
+ ASSERT_EQ(0, res_obj["hits"][2]["document"]["genre.name"].size());
+
+ req_params = {
+ {"collection", "genres"},
+ {"q", "*"},
+ {"filter_by", "$songs(id: *)"},
+ {"include_fields", "$songs(title) as song"},
+ };
+ search_op_bool = collectionManager.do_search(req_params, embedded_params, json_res, now_ts);
+ ASSERT_TRUE(search_op_bool.ok());
+
+ res_obj = nlohmann::json::parse(json_res);
+ ASSERT_EQ(3, res_obj["found"].get<size_t>());
+ ASSERT_EQ(3, res_obj["hits"].size());
+
+ ASSERT_EQ("Blues", res_obj["hits"][0]["document"]["name"].get<std::string>());
+ ASSERT_EQ(1, res_obj["hits"][0]["document"]["song.title"].size());
+ ASSERT_EQ("Achilles Last Stand", res_obj["hits"][0]["document"]["song.title"][0]);
+
+ ASSERT_EQ("Arena rock", res_obj["hits"][1]["document"]["name"].get<std::string>());
+ ASSERT_EQ(1, res_obj["hits"][1]["document"]["song.title"].size());
+ ASSERT_EQ("Achilles Last Stand", res_obj["hits"][1]["document"]["song.title"][0]);
+
+ ASSERT_EQ("Grunge", res_obj["hits"][2]["document"]["name"].get<std::string>());
+ ASSERT_EQ(1, res_obj["hits"][2]["document"]["song.title"].size());
+ ASSERT_EQ("Corduroy", res_obj["hits"][2]["document"]["song.title"][0]);
+}
+
TEST_F(CollectionJoinTest, CascadeDeletion) {
auto schema_json =
R"({
diff --git a/test/collection_manager_test.cpp b/test/collection_manager_test.cpp
index 81b92106..89e89986 100644
--- a/test/collection_manager_test.cpp
+++ b/test/collection_manager_test.cpp
@@ -1436,11 +1436,11 @@ TEST_F(CollectionManagerTest, ReferencedInBacklog) {
referenced_ins_backlog = collectionManager._get_referenced_in_backlog();
ASSERT_EQ(0, referenced_ins_backlog.count("Products"));
- auto get_reference_field_op = create_op.get()->get_reference_field("collection1");
+ auto get_reference_field_op = create_op.get()->get_referenced_in_field_with_lock("collection1");
ASSERT_TRUE(get_reference_field_op.ok());
ASSERT_EQ("product_id_sequence_id", get_reference_field_op.get());
- get_reference_field_op = create_op.get()->get_reference_field("foo");
+ get_reference_field_op = create_op.get()->get_referenced_in_field_with_lock("foo");
ASSERT_FALSE(get_reference_field_op.ok());
ASSERT_EQ("Could not find any field in `Products` referencing the collection `foo`.", get_reference_field_op.error());
}
\ No newline at end of file