Skip to content

Open Source alternative to Algolia + Pinecone and an Easier-to-Use alternative to ElasticSearch ⚡ 🔍 ✨ Fast, typo tolerant, in-memory fuzzy Search Engine for building delightful search experiences

Handle reference filter during approximation.

Harpreet Sangar84787510c8b9
diff --git a/include/collection.h b/include/collection.h
index 8d77cfde..4e08d4da 100644
--- a/include/collection.h
+++ b/include/collection.h
@@ -457,6 +457,9 @@ public:
Option<bool> get_filter_ids(const std::string & filter_query, filter_result_t& filter_result) const;
+ Option<bool> get_approximate_reference_filter_ids(const std::string& filter_query,
+ uint32_t& filter_ids_length) const;
+
Option<bool> get_reference_filter_ids(const std::string& filter_query,
filter_result_t& filter_result,
const std::string& collection_name) const;
diff --git a/include/index.h b/include/index.h
index 0ce10daf..f7b5f3fe 100644
--- a/include/index.h
+++ b/include/index.h
@@ -486,6 +486,10 @@ private:
const int64_t& range_end_value,
uint32_t& filter_ids_length) const;
+ Option<bool> approximate_filter_ids(const filter& a_filter,
+ uint32_t& filter_ids_length,
+ const std::string& collection_name) const;
+
Option<bool> rearranging_recursive_filter(filter_node_t* const filter_tree_root,
filter_result_t& result,
const std::string& collection_name = "") const;
@@ -702,6 +706,9 @@ public:
const std::string& collection_name,
const std::string& reference_helper_field_name) const;
+ Option<bool> get_approximate_reference_filter_ids_with_lock(filter_node_t* const filter_tree_root,
+ uint32_t& filter_ids_length) const;
+
void refresh_schemas(const std::vector<field>& new_fields, const std::vector<field>& del_fields);
// the following methods are not synchronized because their parent calls are synchronized or they are const/static
diff --git a/src/collection.cpp b/src/collection.cpp
index af35b70d..9d1c0635 100644
--- a/src/collection.cpp
+++ b/src/collection.cpp
@@ -2513,10 +2513,9 @@ Option<bool> Collection::get_filter_ids(const std::string& filter_query, filter_
return filter_op;
}
- index->do_filtering_with_lock(filter_tree_root, filter_result, name);
+ std::unique_ptr<filter_node_t> filter_tree_root_guard(filter_tree_root);
- delete filter_tree_root;
- return Option<bool>(true);
+ return index->do_filtering_with_lock(filter_tree_root, filter_result, name);
}
Option<std::string> Collection::get_reference_field(const std::string & collection_name) const {
@@ -2537,6 +2536,23 @@ Option<std::string> Collection::get_reference_field(const std::string & collecti
return Option(reference_field_name);
}
+Option<bool> Collection::get_approximate_reference_filter_ids(const std::string& filter_query,
+ uint32_t& filter_ids_length) const {
+ std::shared_lock lock(mutex);
+
+ const std::string doc_id_prefix = std::to_string(collection_id) + "_" + DOC_ID_PREFIX + "_";
+ filter_node_t* filter_tree_root = nullptr;
+ Option<bool> parse_op = filter::parse_filter_query(filter_query, search_schema,
+ store, doc_id_prefix, filter_tree_root);
+ if(!parse_op.ok()) {
+ return parse_op;
+ }
+
+ std::unique_ptr<filter_node_t> filter_tree_root_guard(filter_tree_root);
+
+ return index->get_approximate_reference_filter_ids_with_lock(filter_tree_root, filter_ids_length);
+}
+
Option<bool> Collection::get_reference_filter_ids(const std::string & filter_query,
filter_result_t& filter_result,
const std::string & collection_name) const {
@@ -2555,15 +2571,11 @@ Option<bool> Collection::get_reference_filter_ids(const std::string & filter_que
return parse_op;
}
+ std::unique_ptr<filter_node_t> filter_tree_root_guard(filter_tree_root);
+
// Reference helper field has the sequence id of other collection's documents.
auto field_name = reference_field_op.get() + REFERENCE_HELPER_FIELD_SUFFIX;
- auto filter_op = index->do_reference_filtering_with_lock(filter_tree_root, filter_result, name, field_name);
- if (!filter_op.ok()) {
- return filter_op;
- }
-
- delete filter_tree_root;
- return Option<bool>(true);
+ return index->do_reference_filtering_with_lock(filter_tree_root, filter_result, name, field_name);
}
Option<bool> Collection::validate_reference_filter(const std::string& filter_query) const {
diff --git a/src/index.cpp b/src/index.cpp
index e88badc0..b704356d 100644
--- a/src/index.cpp
+++ b/src/index.cpp
@@ -1952,45 +1952,19 @@ void Index::aproximate_numerical_match(num_tree_t* const num_tree,
num_tree->approx_search_count(comparator, value, filter_ids_length);
}
-Option<bool> Index::rearrange_filter_tree(filter_node_t* const root,
- uint32_t& filter_ids_length,
- const std::string& collection_name) const {
- if (root == nullptr) {
- return Option(true);
- }
-
- if (root->isOperator) {
- uint32_t l_filter_ids_length = 0;
- if (root->left != nullptr) {
- auto rearrange_op = rearrange_filter_tree(root->left, l_filter_ids_length, collection_name);
- if (!rearrange_op.ok()) {
- return rearrange_op;
- }
- }
-
- uint32_t r_filter_ids_length = 0;
- if (root->right != nullptr) {
- auto rearrange_op = rearrange_filter_tree(root->right, r_filter_ids_length, collection_name);
- if (!rearrange_op.ok()) {
- return rearrange_op;
- }
- }
-
- if (root->filter_operator == AND) {
- filter_ids_length = std::min(l_filter_ids_length, r_filter_ids_length);
- } else {
- filter_ids_length = l_filter_ids_length + r_filter_ids_length;
- }
-
- if (l_filter_ids_length > r_filter_ids_length) {
- std::swap(root->left, root->right);
+Option<bool> Index::approximate_filter_ids(const filter& a_filter,
+ uint32_t& filter_ids_length,
+ const std::string& collection_name) const {
+ if (!a_filter.referenced_collection_name.empty()) {
+ auto& cm = CollectionManager::get_instance();
+ auto collection = cm.get_collection(a_filter.referenced_collection_name);
+ if (collection == nullptr) {
+ return Option<bool>(400, "Referenced collection `" + a_filter.referenced_collection_name + "` not found.");
}
- return Option(true);
+ return collection->get_approximate_reference_filter_ids(a_filter.field_name, filter_ids_length);
}
- auto a_filter = root->filter_exp;
-
if (a_filter.field_name == "id") {
filter_ids_length = a_filter.values.size();
return Option(true);
@@ -2062,7 +2036,7 @@ Option<bool> Index::rearrange_filter_tree(filter_node_t* const root,
while (tokenizer.next(str_token, token_index)) {
auto const leaf = (art_leaf *) art_search(t, (const unsigned char*) str_token.c_str(),
- str_token.length()+1);
+ str_token.length()+1);
if (leaf == nullptr) {
continue;
}
@@ -2080,6 +2054,47 @@ Option<bool> Index::rearrange_filter_tree(filter_node_t* const root,
return Option(true);
}
+Option<bool> Index::rearrange_filter_tree(filter_node_t* const root,
+ uint32_t& filter_ids_length,
+ const std::string& collection_name) const {
+ if (root == nullptr) {
+ return Option(true);
+ }
+
+ if (root->isOperator) {
+ uint32_t l_filter_ids_length = 0;
+ if (root->left != nullptr) {
+ auto rearrange_op = rearrange_filter_tree(root->left, l_filter_ids_length, collection_name);
+ if (!rearrange_op.ok()) {
+ return rearrange_op;
+ }
+ }
+
+ uint32_t r_filter_ids_length = 0;
+ if (root->right != nullptr) {
+ auto rearrange_op = rearrange_filter_tree(root->right, r_filter_ids_length, collection_name);
+ if (!rearrange_op.ok()) {
+ return rearrange_op;
+ }
+ }
+
+ if (root->filter_operator == AND) {
+ filter_ids_length = std::min(l_filter_ids_length, r_filter_ids_length);
+ } else {
+ filter_ids_length = l_filter_ids_length + r_filter_ids_length;
+ }
+
+ if (l_filter_ids_length > r_filter_ids_length) {
+ std::swap(root->left, root->right);
+ }
+
+ return Option(true);
+ }
+
+ approximate_filter_ids(root->filter_exp, filter_ids_length, collection_name);
+ return Option(true);
+}
+
Option<bool> Index::rearranging_recursive_filter(filter_node_t* const filter_tree_root,
filter_result_t& result,
const std::string& collection_name) const {
@@ -2233,6 +2248,13 @@ Option<bool> Index::do_reference_filtering_with_lock(filter_node_t* const filter
return Option(true);
}
+Option<bool> Index::get_approximate_reference_filter_ids_with_lock(filter_node_t* const filter_tree_root,
+ uint32_t& filter_ids_length) const {
+ std::shared_lock lock(mutex);
+
+ return rearrange_filter_tree(filter_tree_root, filter_ids_length);
+}
+
Option<bool> Index::run_search(search_args* search_params, const std::string& collection_name) {
return search(search_params->field_query_tokens,
search_params->search_fields,