Open Source alternative to Algolia + Pinecone and an Easier-to-Use alternative to ElasticSearch ⚡ 🔍 ✨ Fast, typo tolerant, in-memory fuzzy Search Engine for building delightful search experiences
C++ algoliadatastoreelasticsearchenterprise-searchfacetingfull-text-searchfuzzy-searchgeosearchin-memoryinstantsearchmerchandisingpineconesearchsearch-enginesemantic-searchsimilarity-searchsite-searchsynonymstypo-tolerancevector-search
Filtering shouldn't timeout during `Export`, `Delete by query`, `Update by query`, `Cascade deletion of references`, and `Updating async references`. (#1942)
Harpreet Sangarab8dde022d04
diff --git a/include/collection.h b/include/collection.hindex 31e08f11..d3df0a54 100644--- a/include/collection.h+++ b/include/collection.h@@ -583,7 +583,8 @@ public:bool enable_lazy_filter = false,bool enable_typos_for_alpha_numerical_tokens = true) const;- Option<bool> get_filter_ids(const std::string & filter_query, filter_result_t& filter_result) const;+ Option<bool> get_filter_ids(const std::string & filter_query, filter_result_t& filter_result,+ const bool& should_timeout = true) const;Option<bool> get_reference_filter_ids(const std::string& filter_query,filter_result_t& filter_result,diff --git a/include/index.h b/include/index.hindex 9e02b0c9..b9cfe25c 100644--- a/include/index.h+++ b/include/index.h@@ -766,7 +766,8 @@ public:Option<bool> do_filtering_with_lock(filter_node_t* const filter_tree_root,filter_result_t& filter_result,- const std::string& collection_name = "") const;+ const std::string& collection_name = "",+ const bool& should_timeout = true) const;Option<bool> do_reference_filtering_with_lock(filter_node_t* const filter_tree_root,filter_result_t& filter_result,diff --git a/src/collection.cpp b/src/collection.cppindex 735aeec8..40d01b37 100644--- a/src/collection.cpp+++ b/src/collection.cpp@@ -115,7 +115,7 @@ Option<bool> Collection::update_async_references_with_lock(const std::string& reconst uint32_t ref_seq_id, const std::string& field_name) {// Update reference helper field of the docs matching the filter.filter_result_t filter_result;- get_filter_ids(filter, filter_result);+ get_filter_ids(filter, filter_result, false);if (filter_result.count == 0) {return Option<bool>(true);@@ -602,7 +602,7 @@ Option<nlohmann::json> Collection::update_matching_filter(const std::string& fildelete it;} else {filter_result_t filter_result;- auto filter_ids_op = get_filter_ids(_filter_query, filter_result);+ auto filter_ids_op = get_filter_ids(_filter_query, filter_result, false);if(!filter_ids_op.ok()) {return Option<nlohmann::json>(filter_ids_op.code(), filter_ids_op.error());}@@ -3564,7 +3564,8 @@ void Collection::populate_result_kvs(Topster *topster, std::vector<std::vector<K}}-Option<bool> Collection::get_filter_ids(const std::string& filter_query, filter_result_t& filter_result) const {+Option<bool> Collection::get_filter_ids(const std::string& filter_query, filter_result_t& filter_result,+ const bool& should_timeout) const {std::shared_lock lock(mutex);const std::string doc_id_prefix = std::to_string(collection_id) + "_" + DOC_ID_PREFIX + "_";@@ -3577,7 +3578,7 @@ Option<bool> Collection::get_filter_ids(const std::string& filter_query, filter_return filter_op;}- return index->do_filtering_with_lock(filter_tree_root, filter_result, name);+ return index->do_filtering_with_lock(filter_tree_root, filter_result, name, should_timeout);}Option<bool> Collection::get_related_ids(const std::string& ref_field_name, const uint32_t& seq_id,@@ -4386,7 +4387,7 @@ void Collection::cascade_remove_docs(const std::string& field_name, const uint32auto const ref_helper_field_name = field_name + fields::REFERENCE_HELPER_FIELD_SUFFIX;filter_result_t filter_result;- get_filter_ids(ref_helper_field_name + ":" + std::to_string(ref_seq_id), filter_result);+ get_filter_ids(ref_helper_field_name + ":" + std::to_string(ref_seq_id), filter_result, false);if (filter_result.count == 0) {return;diff --git a/src/core_api.cpp b/src/core_api.cppindex b455ca3b..89d5de80 100644--- a/src/core_api.cpp+++ b/src/core_api.cpp@@ -1113,7 +1113,7 @@ bool get_export_documents(const std::shared_ptr<http_req>& req, const std::shareexport_state->iter_upper_bound = new rocksdb::Slice(export_state->iter_upper_bound_key);export_state->it = collectionManager.get_store()->scan(seq_id_prefix, export_state->iter_upper_bound);} else {- auto filter_ids_op = collection->get_filter_ids(filter_query, export_state->filter_result);+ auto filter_ids_op = collection->get_filter_ids(filter_query, export_state->filter_result, false);if(!filter_ids_op.ok()) {res->set(filter_ids_op.code(), filter_ids_op.error());@@ -1750,9 +1750,8 @@ bool del_remove_documents(const std::shared_ptr<http_req>& req, const std::share// destruction of data is managed by req destructorreq->data = deletion_state;- search_stop_us = UINT64_MAX; // Filtering shouldn't timeout during delete operation.filter_result_t filter_result;- auto filter_ids_op = collection->get_filter_ids(simple_filter_query, filter_result);+ auto filter_ids_op = collection->get_filter_ids(simple_filter_query, filter_result, false);if(!filter_ids_op.ok()) {res->set(filter_ids_op.code(), filter_ids_op.error());diff --git a/src/index.cpp b/src/index.cppindex cb7c6306..778f6a47 100644--- a/src/index.cpp+++ b/src/index.cpp@@ -1801,11 +1801,12 @@ bool Index::field_is_indexed(const std::string& field_name) const {Option<bool> Index::do_filtering_with_lock(filter_node_t* const filter_tree_root,filter_result_t& filter_result,- const std::string& collection_name) const {+ const std::string& collection_name,+ const bool& should_timeout) const {std::shared_lock lock(mutex);auto filter_result_iterator = filter_result_iterator_t(collection_name, this, filter_tree_root,- search_begin_us, search_stop_us);+ search_begin_us, should_timeout ? search_stop_us : UINT64_MAX);auto filter_init_op = filter_result_iterator.init_status();if (!filter_init_op.ok()) {return filter_init_op;