Skip to content

Open Source alternative to Algolia + Pinecone and an Easier-to-Use alternative to ElasticSearch ⚡ 🔍 ✨ Fast, typo tolerant, in-memory fuzzy Search Engine for building delightful search experiences

Filtering shouldn't timeout during `Export`, `Delete by query`, `Update by query`, `Cascade deletion of references`, and `Updating async references`. (#1942)

Harpreet Sangarab8dde022d04
diff --git a/include/collection.h b/include/collection.h
index 31e08f11..d3df0a54 100644
--- a/include/collection.h
+++ b/include/collection.h
@@ -583,7 +583,8 @@ public:
bool enable_lazy_filter = false,
bool enable_typos_for_alpha_numerical_tokens = true) const;
- Option<bool> get_filter_ids(const std::string & filter_query, filter_result_t& filter_result) const;
+ Option<bool> get_filter_ids(const std::string & filter_query, filter_result_t& filter_result,
+ const bool& should_timeout = true) const;
Option<bool> get_reference_filter_ids(const std::string& filter_query,
filter_result_t& filter_result,
diff --git a/include/index.h b/include/index.h
index 9e02b0c9..b9cfe25c 100644
--- a/include/index.h
+++ b/include/index.h
@@ -766,7 +766,8 @@ public:
Option<bool> do_filtering_with_lock(filter_node_t* const filter_tree_root,
filter_result_t& filter_result,
- const std::string& collection_name = "") const;
+ const std::string& collection_name = "",
+ const bool& should_timeout = true) const;
Option<bool> do_reference_filtering_with_lock(filter_node_t* const filter_tree_root,
filter_result_t& filter_result,
diff --git a/src/collection.cpp b/src/collection.cpp
index 735aeec8..40d01b37 100644
--- a/src/collection.cpp
+++ b/src/collection.cpp
@@ -115,7 +115,7 @@ Option<bool> Collection::update_async_references_with_lock(const std::string& re
const uint32_t ref_seq_id, const std::string& field_name) {
// Update reference helper field of the docs matching the filter.
filter_result_t filter_result;
- get_filter_ids(filter, filter_result);
+ get_filter_ids(filter, filter_result, false);
if (filter_result.count == 0) {
return Option<bool>(true);
@@ -602,7 +602,7 @@ Option<nlohmann::json> Collection::update_matching_filter(const std::string& fil
delete it;
} else {
filter_result_t filter_result;
- auto filter_ids_op = get_filter_ids(_filter_query, filter_result);
+ auto filter_ids_op = get_filter_ids(_filter_query, filter_result, false);
if(!filter_ids_op.ok()) {
return Option<nlohmann::json>(filter_ids_op.code(), filter_ids_op.error());
}
@@ -3564,7 +3564,8 @@ void Collection::populate_result_kvs(Topster *topster, std::vector<std::vector<K
}
}
-Option<bool> Collection::get_filter_ids(const std::string& filter_query, filter_result_t& filter_result) const {
+Option<bool> Collection::get_filter_ids(const std::string& filter_query, filter_result_t& filter_result,
+ const bool& should_timeout) const {
std::shared_lock lock(mutex);
const std::string doc_id_prefix = std::to_string(collection_id) + "_" + DOC_ID_PREFIX + "_";
@@ -3577,7 +3578,7 @@ Option<bool> Collection::get_filter_ids(const std::string& filter_query, filter_
return filter_op;
}
- return index->do_filtering_with_lock(filter_tree_root, filter_result, name);
+ return index->do_filtering_with_lock(filter_tree_root, filter_result, name, should_timeout);
}
Option<bool> Collection::get_related_ids(const std::string& ref_field_name, const uint32_t& seq_id,
@@ -4386,7 +4387,7 @@ void Collection::cascade_remove_docs(const std::string& field_name, const uint32
auto const ref_helper_field_name = field_name + fields::REFERENCE_HELPER_FIELD_SUFFIX;
filter_result_t filter_result;
- get_filter_ids(ref_helper_field_name + ":" + std::to_string(ref_seq_id), filter_result);
+ get_filter_ids(ref_helper_field_name + ":" + std::to_string(ref_seq_id), filter_result, false);
if (filter_result.count == 0) {
return;
diff --git a/src/core_api.cpp b/src/core_api.cpp
index b455ca3b..89d5de80 100644
--- a/src/core_api.cpp
+++ b/src/core_api.cpp
@@ -1113,7 +1113,7 @@ bool get_export_documents(const std::shared_ptr<http_req>& req, const std::share
export_state->iter_upper_bound = new rocksdb::Slice(export_state->iter_upper_bound_key);
export_state->it = collectionManager.get_store()->scan(seq_id_prefix, export_state->iter_upper_bound);
} else {
- auto filter_ids_op = collection->get_filter_ids(filter_query, export_state->filter_result);
+ auto filter_ids_op = collection->get_filter_ids(filter_query, export_state->filter_result, false);
if(!filter_ids_op.ok()) {
res->set(filter_ids_op.code(), filter_ids_op.error());
@@ -1750,9 +1750,8 @@ bool del_remove_documents(const std::shared_ptr<http_req>& req, const std::share
// destruction of data is managed by req destructor
req->data = deletion_state;
- search_stop_us = UINT64_MAX; // Filtering shouldn't timeout during delete operation.
filter_result_t filter_result;
- auto filter_ids_op = collection->get_filter_ids(simple_filter_query, filter_result);
+ auto filter_ids_op = collection->get_filter_ids(simple_filter_query, filter_result, false);
if(!filter_ids_op.ok()) {
res->set(filter_ids_op.code(), filter_ids_op.error());
diff --git a/src/index.cpp b/src/index.cpp
index cb7c6306..778f6a47 100644
--- a/src/index.cpp
+++ b/src/index.cpp
@@ -1801,11 +1801,12 @@ bool Index::field_is_indexed(const std::string& field_name) const {
Option<bool> Index::do_filtering_with_lock(filter_node_t* const filter_tree_root,
filter_result_t& filter_result,
- const std::string& collection_name) const {
+ const std::string& collection_name,
+ const bool& should_timeout) const {
std::shared_lock lock(mutex);
auto filter_result_iterator = filter_result_iterator_t(collection_name, this, filter_tree_root,
- search_begin_us, search_stop_us);
+ search_begin_us, should_timeout ? search_stop_us : UINT64_MAX);
auto filter_init_op = filter_result_iterator.init_status();
if (!filter_init_op.ok()) {
return filter_init_op;