Open Source alternative to Algolia + Pinecone and an Easier-to-Use alternative to ElasticSearch ⚡ 🔍 ✨ Fast, typo tolerant, in-memory fuzzy Search Engine for building delightful search experiences
C++ algoliadatastoreelasticsearchenterprise-searchfacetingfull-text-searchfuzzy-searchgeosearchin-memoryinstantsearchmerchandisingpineconesearchsearch-enginesemantic-searchsimilarity-searchsite-searchsynonymstypo-tolerancevector-search
Refactor `rearranging_recursive_filter`.
Harpreet Sangarda1b327749d4
diff --git a/include/field.h b/include/field.hindex e189648d..6f5fe485 100644--- a/include/field.h+++ b/include/field.h@@ -542,7 +542,6 @@ struct filter_node_t {bool isOperator;filter_node_t* left;filter_node_t* right;- std::pair<uint32_t, uint32_t*> match_index_ids = {0, nullptr};filter_tree_metrics* metrics = nullptr;filter_node_t(filter filter_exp)@@ -561,7 +560,6 @@ struct filter_node_t {~filter_node_t() {delete metrics;- delete[] match_index_ids.second;delete left;delete right;}diff --git a/include/index.h b/include/index.hindex e0875935..78288464 100644--- a/include/index.h+++ b/include/index.h@@ -484,7 +484,10 @@ private:uint32_t*& ids,size_t& ids_len) const;- void do_filtering(filter_node_t* const root, const std::string& collection_name) const;+ void do_filtering(uint32_t*& filter_ids,+ uint32_t& filter_ids_length,+ filter_node_t* const root,+ const std::string& collection_name) const;void rearranging_recursive_filter (uint32_t*& filter_ids,uint32_t& filter_ids_length,@@ -501,9 +504,9 @@ private:filter_node_t* const filter_tree_root,const std::string& collection_name = "") const;- void get_filter_matches(filter_node_t* const root,- std::vector<std::pair<uint32_t,- filter_node_t*>>& vec,+ void rearrange_filter_tree(uint32_t*& filter_ids,+ uint32_t& filter_ids_length,+ filter_node_t* const root,const std::string& collection_name) const;void insert_doc(const int64_t score, art_tree *t, uint32_t seq_id,diff --git a/src/index.cpp b/src/index.cppindex a14b5274..cb8b5562 100644--- a/src/index.cpp+++ b/src/index.cpp@@ -1617,7 +1617,10 @@ void Index::numeric_not_equals_filter(num_tree_t* const num_tree,ids = out;}-void Index::do_filtering(filter_node_t* const root, const std::string& collection_name) const {+void Index::do_filtering(uint32_t*& filter_ids,+ uint32_t& filter_ids_length,+ filter_node_t* const root,+ const std::string& collection_name) const {// auto begin = std::chrono::high_resolution_clock::now();const filter a_filter = root->filter_exp;@@ -1627,12 +1630,17 @@ void Index::do_filtering(filter_node_t* const root, const std::string& collectioauto& cm = CollectionManager::get_instance();auto collection = cm.get_collection(a_filter.referenced_collection_name);+ std::pair<uint32_t, uint32_t*> reference_index_ids;auto op = collection->get_reference_filter_ids(a_filter.field_name,collection_name,- root->match_index_ids);+ reference_index_ids);if (!op.ok()) {return;}++ filter_ids_length = reference_index_ids.first;+ filter_ids = reference_index_ids.second;+return;}@@ -1645,9 +1653,17 @@ void Index::do_filtering(filter_node_t* const root, const std::string& collectiostd::sort(result_ids.begin(), result_ids.end());- root->match_index_ids.second = new uint32[result_ids.size()];- std::copy(result_ids.begin(), result_ids.end(), root->match_index_ids.second);- root->match_index_ids.first = result_ids.size();+ if (filter_ids_length == 0) {+ filter_ids = new uint32[result_ids.size()];+ std::copy(result_ids.begin(), result_ids.end(), filter_ids);+ filter_ids_length = result_ids.size();+ } else {+ uint32_t* filtered_results = nullptr;+ filter_ids_length = ArrayUtils::and_scalar(filter_ids, filter_ids_length, &result_ids[0],+ result_ids.size(), &filtered_results);+ delete[] filter_ids;+ filter_ids = filtered_results;+ }return;}@@ -1947,8 +1963,8 @@ void Index::do_filtering(filter_node_t* const root, const std::string& collectioresult_ids_len = to_include_ids_len;}- root->match_index_ids.first = result_ids_len;- root->match_index_ids.second = result_ids;+ filter_ids = result_ids;+ filter_ids_length = result_ids_len;/*long long int timeMillis =std::chrono::duration_cast<std::chrono::milliseconds>(std::chrono::high_resolution_clock::now()@@ -1957,100 +1973,51 @@ void Index::do_filtering(filter_node_t* const root, const std::string& collectioLOG(INFO) << "Time taken for filtering: " << timeMillis << "ms";*/}-void Index::get_filter_matches(filter_node_t* const root,- std::vector<std::pair<uint32_t,- filter_node_t*>>& vec,- const std::string& collection_name) const {+void Index::rearrange_filter_tree(uint32_t*& filter_ids,+ uint32_t& filter_ids_length,+ filter_node_t* const root,+ const std::string& collection_name) const {if (root == nullptr) {return;}if (root->isOperator) {- if (root->filter_operator == AND) {- get_filter_matches(root->left, vec, collection_name);- get_filter_matches(root->right, vec, collection_name);- } else {- uint32_t *l_filter_ids = nullptr;- uint32_t l_filter_ids_length = 0;- if (root->left != nullptr) {- rearranging_recursive_filter(l_filter_ids, l_filter_ids_length, root->left, collection_name);- }-- uint32_t *r_filter_ids = nullptr;- uint32_t r_filter_ids_length = 0;- if (root->right != nullptr) {- rearranging_recursive_filter(r_filter_ids, r_filter_ids_length, root->right, collection_name);- }-- root->match_index_ids.first = ArrayUtils::or_scalar(- l_filter_ids, l_filter_ids_length, r_filter_ids,- r_filter_ids_length, &(root->match_index_ids.second));-- delete[] l_filter_ids;- delete[] r_filter_ids;-- vec.emplace_back(root->match_index_ids.first, root);- }-- return;- }-- do_filtering(root, collection_name);- vec.emplace_back(root->match_index_ids.first, root);-}--void evaluate_rearranged_filter_tree(uint32_t*& filter_ids,- uint32_t& filter_ids_length,- filter_node_t* const root,- std::vector<std::pair<uint32_t, filter_node_t*>>& vec,- size_t& index) {- if (root == nullptr) {- return;- }-- if (root->isOperator && root->filter_operator == AND) {uint32_t* l_filter_ids = nullptr;uint32_t l_filter_ids_length = 0;if (root->left != nullptr) {- evaluate_rearranged_filter_tree(l_filter_ids, l_filter_ids_length, root->left, vec, index);+ rearrange_filter_tree(l_filter_ids, l_filter_ids_length,root->left, collection_name);}uint32_t* r_filter_ids = nullptr;uint32_t r_filter_ids_length = 0;if (root->right != nullptr) {- evaluate_rearranged_filter_tree(r_filter_ids, r_filter_ids_length, root->right, vec, index);+ rearrange_filter_tree(r_filter_ids, r_filter_ids_length, root->right, collection_name);}- root->match_index_ids.first = ArrayUtils::and_scalar(- l_filter_ids, l_filter_ids_length, r_filter_ids,- r_filter_ids_length, &(root->match_index_ids.second));+ if (root->filter_operator == AND) {+ filter_ids_length = std::min(l_filter_ids_length, r_filter_ids_length);+ } else {+ filter_ids_length = l_filter_ids_length + r_filter_ids_length;+ }- filter_ids_length = root->match_index_ids.first;- filter_ids = root->match_index_ids.second;- } else {- filter_ids_length = vec[index].first;- filter_ids = vec[index].second->match_index_ids.second;- index++;+ if (l_filter_ids_length > r_filter_ids_length) {+ std::swap(root->left, root->right);+ }++ delete[] l_filter_ids;+ delete[] r_filter_ids;+ return;}++ do_filtering(filter_ids, filter_ids_length, root, collection_name);}void Index::rearranging_recursive_filter(uint32_t*& filter_ids,uint32_t& filter_ids_length,filter_node_t* const root,const std::string& collection_name) const {- std::vector<std::pair<uint32_t, filter_node_t*>> vec;- get_filter_matches(root, vec, collection_name);-- std::sort(vec.begin(), vec.end(),- [](const std::pair<uint32_t, filter_node_t*>& lhs, const std::pair<uint32_t, filter_node_t*>& rhs) {- return lhs.first < rhs.first;- });-- size_t index = 0;- evaluate_rearranged_filter_tree(filter_ids, filter_ids_length, root, vec, index);-- // To disable deletion of filter_ids when filter tree is destructed.- root->match_index_ids.second = nullptr;+ rearrange_filter_tree(filter_ids, filter_ids_length, root, collection_name);+ recursive_filter(filter_ids, filter_ids_length, root, collection_name);}void Index::recursive_filter(uint32_t*& filter_ids,@@ -2092,12 +2059,7 @@ void Index::recursive_filter(uint32_t*& filter_ids,return;}- do_filtering(root, collection_name);- filter_ids_length = root->match_index_ids.first;- filter_ids = root->match_index_ids.second;-- // Prevents double deletion. We'll be deleting this array upstream and when the filter tree is destructed.- root->match_index_ids.second = nullptr;+ do_filtering(filter_ids, filter_ids_length, root, collection_name);}void Index::adaptive_filter(uint32_t*& filter_ids,