Skip to content

Open Source alternative to Algolia + Pinecone and an Easier-to-Use alternative to ElasticSearch ⚡ 🔍 ✨ Fast, typo tolerant, in-memory fuzzy Search Engine for building delightful search experiences

Refactor `rearranging_recursive_filter`.

Harpreet Sangarda1b327749d4
diff --git a/include/field.h b/include/field.h
index e189648d..6f5fe485 100644
--- a/include/field.h
+++ b/include/field.h
@@ -542,7 +542,6 @@ struct filter_node_t {
bool isOperator;
filter_node_t* left;
filter_node_t* right;
- std::pair<uint32_t, uint32_t*> match_index_ids = {0, nullptr};
filter_tree_metrics* metrics = nullptr;
filter_node_t(filter filter_exp)
@@ -561,7 +560,6 @@ struct filter_node_t {
~filter_node_t() {
delete metrics;
- delete[] match_index_ids.second;
delete left;
delete right;
}
diff --git a/include/index.h b/include/index.h
index e0875935..78288464 100644
--- a/include/index.h
+++ b/include/index.h
@@ -484,7 +484,10 @@ private:
uint32_t*& ids,
size_t& ids_len) const;
- void do_filtering(filter_node_t* const root, const std::string& collection_name) const;
+ void do_filtering(uint32_t*& filter_ids,
+ uint32_t& filter_ids_length,
+ filter_node_t* const root,
+ const std::string& collection_name) const;
void rearranging_recursive_filter (uint32_t*& filter_ids,
uint32_t& filter_ids_length,
@@ -501,9 +504,9 @@ private:
filter_node_t* const filter_tree_root,
const std::string& collection_name = "") const;
- void get_filter_matches(filter_node_t* const root,
- std::vector<std::pair<uint32_t,
- filter_node_t*>>& vec,
+ void rearrange_filter_tree(uint32_t*& filter_ids,
+ uint32_t& filter_ids_length,
+ filter_node_t* const root,
const std::string& collection_name) const;
void insert_doc(const int64_t score, art_tree *t, uint32_t seq_id,
diff --git a/src/index.cpp b/src/index.cpp
index a14b5274..cb8b5562 100644
--- a/src/index.cpp
+++ b/src/index.cpp
@@ -1617,7 +1617,10 @@ void Index::numeric_not_equals_filter(num_tree_t* const num_tree,
ids = out;
}
-void Index::do_filtering(filter_node_t* const root, const std::string& collection_name) const {
+void Index::do_filtering(uint32_t*& filter_ids,
+ uint32_t& filter_ids_length,
+ filter_node_t* const root,
+ const std::string& collection_name) const {
// auto begin = std::chrono::high_resolution_clock::now();
const filter a_filter = root->filter_exp;
@@ -1627,12 +1630,17 @@ void Index::do_filtering(filter_node_t* const root, const std::string& collectio
auto& cm = CollectionManager::get_instance();
auto collection = cm.get_collection(a_filter.referenced_collection_name);
+ std::pair<uint32_t, uint32_t*> reference_index_ids;
auto op = collection->get_reference_filter_ids(a_filter.field_name,
collection_name,
- root->match_index_ids);
+ reference_index_ids);
if (!op.ok()) {
return;
}
+
+ filter_ids_length = reference_index_ids.first;
+ filter_ids = reference_index_ids.second;
+
return;
}
@@ -1645,9 +1653,17 @@ void Index::do_filtering(filter_node_t* const root, const std::string& collectio
std::sort(result_ids.begin(), result_ids.end());
- root->match_index_ids.second = new uint32[result_ids.size()];
- std::copy(result_ids.begin(), result_ids.end(), root->match_index_ids.second);
- root->match_index_ids.first = result_ids.size();
+ if (filter_ids_length == 0) {
+ filter_ids = new uint32[result_ids.size()];
+ std::copy(result_ids.begin(), result_ids.end(), filter_ids);
+ filter_ids_length = result_ids.size();
+ } else {
+ uint32_t* filtered_results = nullptr;
+ filter_ids_length = ArrayUtils::and_scalar(filter_ids, filter_ids_length, &result_ids[0],
+ result_ids.size(), &filtered_results);
+ delete[] filter_ids;
+ filter_ids = filtered_results;
+ }
return;
}
@@ -1947,8 +1963,8 @@ void Index::do_filtering(filter_node_t* const root, const std::string& collectio
result_ids_len = to_include_ids_len;
}
- root->match_index_ids.first = result_ids_len;
- root->match_index_ids.second = result_ids;
+ filter_ids = result_ids;
+ filter_ids_length = result_ids_len;
/*long long int timeMillis =
std::chrono::duration_cast<std::chrono::milliseconds>(std::chrono::high_resolution_clock::now()
@@ -1957,100 +1973,51 @@ void Index::do_filtering(filter_node_t* const root, const std::string& collectio
LOG(INFO) << "Time taken for filtering: " << timeMillis << "ms";*/
}
-void Index::get_filter_matches(filter_node_t* const root,
- std::vector<std::pair<uint32_t,
- filter_node_t*>>& vec,
- const std::string& collection_name) const {
+void Index::rearrange_filter_tree(uint32_t*& filter_ids,
+ uint32_t& filter_ids_length,
+ filter_node_t* const root,
+ const std::string& collection_name) const {
if (root == nullptr) {
return;
}
if (root->isOperator) {
- if (root->filter_operator == AND) {
- get_filter_matches(root->left, vec, collection_name);
- get_filter_matches(root->right, vec, collection_name);
- } else {
- uint32_t *l_filter_ids = nullptr;
- uint32_t l_filter_ids_length = 0;
- if (root->left != nullptr) {
- rearranging_recursive_filter(l_filter_ids, l_filter_ids_length, root->left, collection_name);
- }
-
- uint32_t *r_filter_ids = nullptr;
- uint32_t r_filter_ids_length = 0;
- if (root->right != nullptr) {
- rearranging_recursive_filter(r_filter_ids, r_filter_ids_length, root->right, collection_name);
- }
-
- root->match_index_ids.first = ArrayUtils::or_scalar(
- l_filter_ids, l_filter_ids_length, r_filter_ids,
- r_filter_ids_length, &(root->match_index_ids.second));
-
- delete[] l_filter_ids;
- delete[] r_filter_ids;
-
- vec.emplace_back(root->match_index_ids.first, root);
- }
-
- return;
- }
-
- do_filtering(root, collection_name);
- vec.emplace_back(root->match_index_ids.first, root);
-}
-
-void evaluate_rearranged_filter_tree(uint32_t*& filter_ids,
- uint32_t& filter_ids_length,
- filter_node_t* const root,
- std::vector<std::pair<uint32_t, filter_node_t*>>& vec,
- size_t& index) {
- if (root == nullptr) {
- return;
- }
-
- if (root->isOperator && root->filter_operator == AND) {
uint32_t* l_filter_ids = nullptr;
uint32_t l_filter_ids_length = 0;
if (root->left != nullptr) {
- evaluate_rearranged_filter_tree(l_filter_ids, l_filter_ids_length, root->left, vec, index);
+ rearrange_filter_tree(l_filter_ids, l_filter_ids_length,root->left, collection_name);
}
uint32_t* r_filter_ids = nullptr;
uint32_t r_filter_ids_length = 0;
if (root->right != nullptr) {
- evaluate_rearranged_filter_tree(r_filter_ids, r_filter_ids_length, root->right, vec, index);
+ rearrange_filter_tree(r_filter_ids, r_filter_ids_length, root->right, collection_name);
}
- root->match_index_ids.first = ArrayUtils::and_scalar(
- l_filter_ids, l_filter_ids_length, r_filter_ids,
- r_filter_ids_length, &(root->match_index_ids.second));
+ if (root->filter_operator == AND) {
+ filter_ids_length = std::min(l_filter_ids_length, r_filter_ids_length);
+ } else {
+ filter_ids_length = l_filter_ids_length + r_filter_ids_length;
+ }
- filter_ids_length = root->match_index_ids.first;
- filter_ids = root->match_index_ids.second;
- } else {
- filter_ids_length = vec[index].first;
- filter_ids = vec[index].second->match_index_ids.second;
- index++;
+ if (l_filter_ids_length > r_filter_ids_length) {
+ std::swap(root->left, root->right);
+ }
+
+ delete[] l_filter_ids;
+ delete[] r_filter_ids;
+ return;
}
+
+ do_filtering(filter_ids, filter_ids_length, root, collection_name);
}
void Index::rearranging_recursive_filter(uint32_t*& filter_ids,
uint32_t& filter_ids_length,
filter_node_t* const root,
const std::string& collection_name) const {
- std::vector<std::pair<uint32_t, filter_node_t*>> vec;
- get_filter_matches(root, vec, collection_name);
-
- std::sort(vec.begin(), vec.end(),
- [](const std::pair<uint32_t, filter_node_t*>& lhs, const std::pair<uint32_t, filter_node_t*>& rhs) {
- return lhs.first < rhs.first;
- });
-
- size_t index = 0;
- evaluate_rearranged_filter_tree(filter_ids, filter_ids_length, root, vec, index);
-
- // To disable deletion of filter_ids when filter tree is destructed.
- root->match_index_ids.second = nullptr;
+ rearrange_filter_tree(filter_ids, filter_ids_length, root, collection_name);
+ recursive_filter(filter_ids, filter_ids_length, root, collection_name);
}
void Index::recursive_filter(uint32_t*& filter_ids,
@@ -2092,12 +2059,7 @@ void Index::recursive_filter(uint32_t*& filter_ids,
return;
}
- do_filtering(root, collection_name);
- filter_ids_length = root->match_index_ids.first;
- filter_ids = root->match_index_ids.second;
-
- // Prevents double deletion. We'll be deleting this array upstream and when the filter tree is destructed.
- root->match_index_ids.second = nullptr;
+ do_filtering(filter_ids, filter_ids_length, root, collection_name);
}
void Index::adaptive_filter(uint32_t*& filter_ids,