Skip to content

Open Source alternative to Algolia + Pinecone and an Easier-to-Use alternative to ElasticSearch ⚡ 🔍 ✨ Fast, typo tolerant, in-memory fuzzy Search Engine for building delightful search experiences

Include reference fields without join.

Harpreet Sangar710ee61c873f
diff --git a/include/collection.h b/include/collection.h
index 692e4142..391a4bac 100644
--- a/include/collection.h
+++ b/include/collection.h
@@ -276,10 +276,10 @@ private:
const spp::sparse_hash_set<std::string>& exclude_fields,
tsl::htrie_set<char>& include_fields_full,
tsl::htrie_set<char>& exclude_fields_full) const;
-
+ Option<uint32_t> get_reference_doc_id(const std::string& ref_collection_name, const uint32_t& seq_id) const;
- Option<std::string> get_reference_field(const std::string & collection_name) const;
+ Option<std::string> get_reference_field(const std::string& ref_collection_name) const;
static void hide_credential(nlohmann::json& json, const std::string& credential_name);
@@ -377,8 +377,10 @@ public:
static void remove_flat_fields(nlohmann::json& document);
static Option<bool> prune_doc(nlohmann::json& doc, const tsl::htrie_set<char>& include_names,
- const tsl::htrie_set<char>& exclude_names, const std::string& parent_name = "", size_t depth = 0,
- const std::map<std::string, reference_filter_result_t>& reference_filter_results = {});
+ const tsl::htrie_set<char>& exclude_names, const std::string& parent_name = "",
+ size_t depth = 0,
+ const std::map<std::string, reference_filter_result_t>& reference_filter_results = {},
+ Collection *const collection = nullptr, const uint32_t& seq_id = 0);
const Index* _get_index() const;
diff --git a/include/index.h b/include/index.h
index e9d20cda..bc961ec2 100644
--- a/include/index.h
+++ b/include/index.h
@@ -482,24 +482,8 @@ private:
const size_t num_search_fields,
std::vector<size_t>& popular_field_ids);
- void numeric_not_equals_filter(num_tree_t* const num_tree,
- const int64_t value,
- const uint32_t& context_ids_length,
- uint32_t* const& context_ids,
- size_t& ids_len,
- uint32_t*& ids) const;
-
bool field_is_indexed(const std::string& field_name) const;
- void aproximate_numerical_match(num_tree_t* const num_tree,
- const NUM_COMPARATOR& comparator,
- const int64_t& value,
- const int64_t& range_end_value,
- uint32_t& filter_ids_length) const;
-
- void insert_doc(const int64_t score, art_tree *t, uint32_t seq_id,
- const std::unordered_map<std::string, std::vector<uint32_t>> &token_to_offsets) const;
-
static void tokenize_string(const std::string& text,
const field& a_field,
const std::vector<char>& symbols_to_index,
@@ -941,6 +925,9 @@ public:
Option<bool> seq_ids_outside_top_k(const std::string& field_name, size_t k,
std::vector<uint32_t>& outside_seq_ids);
+ Option<uint32_t> get_reference_doc_id_with_lock(const std::string& reference_helper_field_name,
+ const uint32_t& seq_id) const;
+
friend class filter_result_iterator_t;
};
diff --git a/src/collection.cpp b/src/collection.cpp
index 99bfacf0..a0f6e341 100644
--- a/src/collection.cpp
+++ b/src/collection.cpp
@@ -1945,7 +1945,8 @@ Option<nlohmann::json> Collection::search(std::string raw_query,
exclude_fields_full,
"",
0,
- field_order_kv->reference_filter_results);
+ field_order_kv->reference_filter_results,
+ const_cast<Collection *>(this), get_seq_id_from_key(seq_id_key));
if (!prune_op.ok()) {
return Option<nlohmann::json>(prune_op.code(), prune_op.error());
}
@@ -2697,11 +2698,21 @@ Option<bool> Collection::get_filter_ids(const std::string& filter_query, filter_
return index->do_filtering_with_lock(filter_tree_root, filter_result, name);
}
-Option<std::string> Collection::get_reference_field(const std::string & collection_name) const {
+Option<uint32_t> Collection::get_reference_doc_id(const std::string& ref_collection_name, const uint32_t& seq_id) const {
+ auto get_reference_field_op = get_reference_field(ref_collection_name);
+ if (!get_reference_field_op.ok()) {
+ return Option<uint32_t>(get_reference_field_op.code(), get_reference_field_op.error());
+ }
+
+ auto field_name = get_reference_field_op.get() + REFERENCE_HELPER_FIELD_SUFFIX;
+ return index->get_reference_doc_id_with_lock(field_name, seq_id);
+}
+
+Option<std::string> Collection::get_reference_field(const std::string& ref_collection_name) const {
std::string reference_field_name;
for (auto const& pair: reference_fields) {
auto reference_pair = pair.second;
- if (reference_pair.collection == collection_name) {
+ if (reference_pair.collection == ref_collection_name) {
reference_field_name = pair.first;
break;
}
@@ -2709,7 +2720,7 @@ Option<std::string> Collection::get_reference_field(const std::string & collecti
if (reference_field_name.empty()) {
return Option<std::string>(400, "Could not find any field in `" + name + "` referencing the collection `"
- + collection_name + "`.");
+ + ref_collection_name + "`.");
}
return Option(reference_field_name);
@@ -4044,7 +4055,8 @@ Option<bool> Collection::prune_doc(nlohmann::json& doc,
const tsl::htrie_set<char>& include_names,
const tsl::htrie_set<char>& exclude_names,
const std::string& parent_name, size_t depth,
- const std::map<std::string, reference_filter_result_t>& reference_filter_results) {
+ const std::map<std::string, reference_filter_result_t>& reference_filter_results,
+ Collection *const collection, const uint32_t& seq_id) {
// doc can only be an object
auto it = doc.begin();
while(it != doc.end()) {
@@ -4120,12 +4132,8 @@ Option<bool> Collection::prune_doc(nlohmann::json& doc,
it++;
}
- if (reference_filter_results.empty()) {
- return Option<bool>(true);
- }
-
- auto include_reference_it = include_names.equal_prefix_range("$");
- for (auto reference = include_reference_it.first; reference != include_reference_it.second; reference++) {
+ auto include_reference_it_pair = include_names.equal_prefix_range("$");
+ for (auto reference = include_reference_it_pair.first; reference != include_reference_it_pair.second; reference++) {
auto ref = reference.key();
size_t parenthesis_index = ref.find('(');
@@ -4162,9 +4170,36 @@ Option<bool> Collection::prune_doc(nlohmann::json& doc,
return Option<bool>(include_exclude_op.code(), error_prefix + include_exclude_op.error());
}
- if (reference_filter_results.count(ref_collection_name) == 0 ||
- reference_filter_results.at(ref_collection_name).count == 0) {
- // doc has no references.
+ bool has_filter_reference = reference_filter_results.count(ref_collection_name) > 0;
+ if (!has_filter_reference) {
+ if (collection == nullptr) {
+ continue;
+ }
+
+ // Reference include_by without join, check if doc itself contains the reference.
+ auto get_reference_doc_id_op = collection->get_reference_doc_id(ref_collection_name, seq_id);
+ if (!get_reference_doc_id_op.ok()) {
+ continue;
+ }
+
+ auto ref_doc_seq_id = get_reference_doc_id_op.get();
+
+ nlohmann::json ref_doc;
+ auto get_doc_op = ref_collection->get_document_from_store(ref_doc_seq_id, ref_doc);
+ if (!get_doc_op.ok()) {
+ return Option<bool>(get_doc_op.code(), error_prefix + get_doc_op.error());
+ }
+
+ auto prune_op = prune_doc(ref_doc, ref_include_fields_full, ref_exclude_fields_full);
+ if (!prune_op.ok()) {
+ return Option<bool>(prune_op.code(), error_prefix + prune_op.error());
+ }
+
+ doc.update(ref_doc);
+ continue;
+ }
+
+ if (has_filter_reference && reference_filter_results.at(ref_collection_name).count == 0) {
continue;
}
diff --git a/src/index.cpp b/src/index.cpp
index de07a27a..c61a2ff1 100644
--- a/src/index.cpp
+++ b/src/index.cpp
@@ -1656,34 +1656,6 @@ bool Index::field_is_indexed(const std::string& field_name) const {
geo_range_index.count(field_name) != 0;
}
-void Index::aproximate_numerical_match(num_tree_t* const num_tree,
- const NUM_COMPARATOR& comparator,
- const int64_t& value,
- const int64_t& range_end_value,
- uint32_t& filter_ids_length) const {
- if (comparator == RANGE_INCLUSIVE) {
- num_tree->approx_range_inclusive_search_count(value, range_end_value, filter_ids_length);
- return;
- }
-
- if (comparator == NOT_EQUALS) {
- uint32_t to_exclude_ids_len = 0;
- num_tree->approx_search_count(EQUALS, value, to_exclude_ids_len);
-
- if (to_exclude_ids_len == 0) {
- filter_ids_length += seq_ids->num_ids();
- } else if (to_exclude_ids_len >= seq_ids->num_ids()) {
- filter_ids_length += 0;
- } else {
- filter_ids_length += (seq_ids->num_ids() - to_exclude_ids_len);
- }
-
- return;
- }
-
- num_tree->approx_search_count(comparator, value, filter_ids_length);
-}
-
Option<bool> Index::do_filtering_with_lock(filter_node_t* const filter_tree_root,
filter_result_t& filter_result,
const std::string& collection_name) const {
@@ -6150,6 +6122,17 @@ void Index::batch_embed_fields(std::vector<index_record*>& records,
}
}
+Option<uint32_t> Index::get_reference_doc_id_with_lock(const string& reference_helper_field_name,
+ const uint32_t& seq_id) const {
+ std::shared_lock lock(mutex);
+ if (sort_index.count(reference_helper_field_name) == 0 ||
+ sort_index.at(reference_helper_field_name)->count(seq_id) == 0) {
+ return Option<uint32_t>(400, "Could not find a reference for doc " + std::to_string(seq_id));
+ }
+
+ return Option<uint32_t>(sort_index.at(reference_helper_field_name)->at(seq_id));
+}
+
/*
// https://stackoverflow.com/questions/924171/geo-fencing-point-inside-outside-polygon
// NOTE: polygon and point should have been transformed with `transform_for_180th_meridian`
diff --git a/test/collection_join_test.cpp b/test/collection_join_test.cpp
index a93c0d08..13d8b2a6 100644
--- a/test/collection_join_test.cpp
+++ b/test/collection_join_test.cpp
@@ -1427,6 +1427,26 @@ TEST_F(CollectionJoinTest, IncludeExcludeFieldsByReference_SingleMatch) {
ASSERT_EQ("soap", res_obj["hits"][0]["document"].at("product_name"));
ASSERT_EQ(1, res_obj["hits"][0]["document"].count("product_price"));
ASSERT_EQ(73.5, res_obj["hits"][0]["document"].at("product_price"));
+
+ // Reference include_by without join
+ req_params = {
+ {"collection", "Customers"},
+ {"q", "Joe"},
+ {"query_by", "customer_name"},
+ {"filter_by", "product_price:<100"},
+ {"include_fields", "$Products(product_name), product_price"}
+ };
+ search_op = collectionManager.do_search(req_params, embedded_params, json_res, now_ts);
+ ASSERT_TRUE(search_op.ok());
+
+ res_obj = nlohmann::json::parse(json_res);
+ ASSERT_EQ(1, res_obj["found"].get<size_t>());
+ ASSERT_EQ(1, res_obj["hits"].size());
+ ASSERT_EQ(2, res_obj["hits"][0]["document"].size());
+ ASSERT_EQ(1, res_obj["hits"][0]["document"].count("product_name"));
+ ASSERT_EQ("soap", res_obj["hits"][0]["document"].at("product_name"));
+ ASSERT_EQ(1, res_obj["hits"][0]["document"].count("product_price"));
+ ASSERT_EQ(73.5, res_obj["hits"][0]["document"].at("product_price"));
}
TEST_F(CollectionJoinTest, CascadeDeletion) {