Open Source alternative to Algolia + Pinecone and an Easier-to-Use alternative to ElasticSearch ⚡ 🔍 ✨ Fast, typo tolerant, in-memory fuzzy Search Engine for building delightful search experiences
C++ algoliadatastoreelasticsearchenterprise-searchfacetingfull-text-searchfuzzy-searchgeosearchin-memoryinstantsearchmerchandisingpineconesearchsearch-enginesemantic-searchsimilarity-searchsite-searchsynonymstypo-tolerancevector-search
Handle reference filter during approximation.
Harpreet Sangar84787510c8b9
diff --git a/include/collection.h b/include/collection.hindex 8d77cfde..4e08d4da 100644--- a/include/collection.h+++ b/include/collection.h@@ -457,6 +457,9 @@ public:Option<bool> get_filter_ids(const std::string & filter_query, filter_result_t& filter_result) const;+ Option<bool> get_approximate_reference_filter_ids(const std::string& filter_query,+ uint32_t& filter_ids_length) const;+Option<bool> get_reference_filter_ids(const std::string& filter_query,filter_result_t& filter_result,const std::string& collection_name) const;diff --git a/include/index.h b/include/index.hindex 0ce10daf..f7b5f3fe 100644--- a/include/index.h+++ b/include/index.h@@ -486,6 +486,10 @@ private:const int64_t& range_end_value,uint32_t& filter_ids_length) const;+ Option<bool> approximate_filter_ids(const filter& a_filter,+ uint32_t& filter_ids_length,+ const std::string& collection_name) const;+Option<bool> rearranging_recursive_filter(filter_node_t* const filter_tree_root,filter_result_t& result,const std::string& collection_name = "") const;@@ -702,6 +706,9 @@ public:const std::string& collection_name,const std::string& reference_helper_field_name) const;+ Option<bool> get_approximate_reference_filter_ids_with_lock(filter_node_t* const filter_tree_root,+ uint32_t& filter_ids_length) const;+void refresh_schemas(const std::vector<field>& new_fields, const std::vector<field>& del_fields);// the following methods are not synchronized because their parent calls are synchronized or they are const/staticdiff --git a/src/collection.cpp b/src/collection.cppindex af35b70d..9d1c0635 100644--- a/src/collection.cpp+++ b/src/collection.cpp@@ -2513,10 +2513,9 @@ Option<bool> Collection::get_filter_ids(const std::string& filter_query, filter_return filter_op;}- index->do_filtering_with_lock(filter_tree_root, filter_result, name);+ std::unique_ptr<filter_node_t> filter_tree_root_guard(filter_tree_root);- delete filter_tree_root;- return Option<bool>(true);+ return index->do_filtering_with_lock(filter_tree_root, filter_result, name);}Option<std::string> Collection::get_reference_field(const std::string & collection_name) const {@@ -2537,6 +2536,23 @@ Option<std::string> Collection::get_reference_field(const std::string & collectireturn Option(reference_field_name);}+Option<bool> Collection::get_approximate_reference_filter_ids(const std::string& filter_query,+ uint32_t& filter_ids_length) const {+ std::shared_lock lock(mutex);++ const std::string doc_id_prefix = std::to_string(collection_id) + "_" + DOC_ID_PREFIX + "_";+ filter_node_t* filter_tree_root = nullptr;+ Option<bool> parse_op = filter::parse_filter_query(filter_query, search_schema,+ store, doc_id_prefix, filter_tree_root);+ if(!parse_op.ok()) {+ return parse_op;+ }++ std::unique_ptr<filter_node_t> filter_tree_root_guard(filter_tree_root);++ return index->get_approximate_reference_filter_ids_with_lock(filter_tree_root, filter_ids_length);+}+Option<bool> Collection::get_reference_filter_ids(const std::string & filter_query,filter_result_t& filter_result,const std::string & collection_name) const {@@ -2555,15 +2571,11 @@ Option<bool> Collection::get_reference_filter_ids(const std::string & filter_quereturn parse_op;}+ std::unique_ptr<filter_node_t> filter_tree_root_guard(filter_tree_root);+// Reference helper field has the sequence id of other collection's documents.auto field_name = reference_field_op.get() + REFERENCE_HELPER_FIELD_SUFFIX;- auto filter_op = index->do_reference_filtering_with_lock(filter_tree_root, filter_result, name, field_name);- if (!filter_op.ok()) {- return filter_op;- }-- delete filter_tree_root;- return Option<bool>(true);+ return index->do_reference_filtering_with_lock(filter_tree_root, filter_result, name, field_name);}Option<bool> Collection::validate_reference_filter(const std::string& filter_query) const {diff --git a/src/index.cpp b/src/index.cppindex e88badc0..b704356d 100644--- a/src/index.cpp+++ b/src/index.cpp@@ -1952,45 +1952,19 @@ void Index::aproximate_numerical_match(num_tree_t* const num_tree,num_tree->approx_search_count(comparator, value, filter_ids_length);}-Option<bool> Index::rearrange_filter_tree(filter_node_t* const root,- uint32_t& filter_ids_length,- const std::string& collection_name) const {- if (root == nullptr) {- return Option(true);- }-- if (root->isOperator) {- uint32_t l_filter_ids_length = 0;- if (root->left != nullptr) {- auto rearrange_op = rearrange_filter_tree(root->left, l_filter_ids_length, collection_name);- if (!rearrange_op.ok()) {- return rearrange_op;- }- }-- uint32_t r_filter_ids_length = 0;- if (root->right != nullptr) {- auto rearrange_op = rearrange_filter_tree(root->right, r_filter_ids_length, collection_name);- if (!rearrange_op.ok()) {- return rearrange_op;- }- }-- if (root->filter_operator == AND) {- filter_ids_length = std::min(l_filter_ids_length, r_filter_ids_length);- } else {- filter_ids_length = l_filter_ids_length + r_filter_ids_length;- }-- if (l_filter_ids_length > r_filter_ids_length) {- std::swap(root->left, root->right);+Option<bool> Index::approximate_filter_ids(const filter& a_filter,+ uint32_t& filter_ids_length,+ const std::string& collection_name) const {+ if (!a_filter.referenced_collection_name.empty()) {+ auto& cm = CollectionManager::get_instance();+ auto collection = cm.get_collection(a_filter.referenced_collection_name);+ if (collection == nullptr) {+ return Option<bool>(400, "Referenced collection `" + a_filter.referenced_collection_name + "` not found.");}- return Option(true);+ return collection->get_approximate_reference_filter_ids(a_filter.field_name, filter_ids_length);}- auto a_filter = root->filter_exp;-if (a_filter.field_name == "id") {filter_ids_length = a_filter.values.size();return Option(true);@@ -2062,7 +2036,7 @@ Option<bool> Index::rearrange_filter_tree(filter_node_t* const root,while (tokenizer.next(str_token, token_index)) {auto const leaf = (art_leaf *) art_search(t, (const unsigned char*) str_token.c_str(),- str_token.length()+1);+ str_token.length()+1);if (leaf == nullptr) {continue;}@@ -2080,6 +2054,47 @@ Option<bool> Index::rearrange_filter_tree(filter_node_t* const root,return Option(true);}+Option<bool> Index::rearrange_filter_tree(filter_node_t* const root,+ uint32_t& filter_ids_length,+ const std::string& collection_name) const {+ if (root == nullptr) {+ return Option(true);+ }++ if (root->isOperator) {+ uint32_t l_filter_ids_length = 0;+ if (root->left != nullptr) {+ auto rearrange_op = rearrange_filter_tree(root->left, l_filter_ids_length, collection_name);+ if (!rearrange_op.ok()) {+ return rearrange_op;+ }+ }++ uint32_t r_filter_ids_length = 0;+ if (root->right != nullptr) {+ auto rearrange_op = rearrange_filter_tree(root->right, r_filter_ids_length, collection_name);+ if (!rearrange_op.ok()) {+ return rearrange_op;+ }+ }++ if (root->filter_operator == AND) {+ filter_ids_length = std::min(l_filter_ids_length, r_filter_ids_length);+ } else {+ filter_ids_length = l_filter_ids_length + r_filter_ids_length;+ }++ if (l_filter_ids_length > r_filter_ids_length) {+ std::swap(root->left, root->right);+ }++ return Option(true);+ }++ approximate_filter_ids(root->filter_exp, filter_ids_length, collection_name);+ return Option(true);+}+Option<bool> Index::rearranging_recursive_filter(filter_node_t* const filter_tree_root,filter_result_t& result,const std::string& collection_name) const {@@ -2233,6 +2248,13 @@ Option<bool> Index::do_reference_filtering_with_lock(filter_node_t* const filterreturn Option(true);}+Option<bool> Index::get_approximate_reference_filter_ids_with_lock(filter_node_t* const filter_tree_root,+ uint32_t& filter_ids_length) const {+ std::shared_lock lock(mutex);++ return rearrange_filter_tree(filter_tree_root, filter_ids_length);+}+Option<bool> Index::run_search(search_args* search_params, const std::string& collection_name) {return search(search_params->field_query_tokens,search_params->search_fields,