Skip to content

Open Source alternative to Algolia + Pinecone and an Easier-to-Use alternative to ElasticSearch ⚡ 🔍 ✨ Fast, typo tolerant, in-memory fuzzy Search Engine for building delightful search experiences

Add `Index::adaptive_filter`.

Harpreet Sangar6c19c95af6d9
diff --git a/include/field.h b/include/field.h
index 34d0c6e7..e189648d 100644
--- a/include/field.h
+++ b/include/field.h
@@ -530,13 +530,20 @@ struct filter {
filter_node_t*& root);
};
+struct filter_tree_metrics {
+ int filter_exp_count;
+ int and_operator_count;
+ int or_operator_count;
+};
+
struct filter_node_t {
filter filter_exp;
FILTER_OPERATOR filter_operator;
bool isOperator;
filter_node_t* left;
filter_node_t* right;
- std::pair<uint32_t, uint32_t*> match_index_ids;
+ std::pair<uint32_t, uint32_t*> match_index_ids = {0, nullptr};
+ filter_tree_metrics* metrics = nullptr;
filter_node_t(filter filter_exp)
: filter_exp(std::move(filter_exp)),
@@ -553,6 +560,7 @@ struct filter_node_t {
right(right) {}
~filter_node_t() {
+ delete metrics;
delete[] match_index_ids.second;
delete left;
delete right;
diff --git a/include/index.h b/include/index.h
index f8690e99..db16a24c 100644
--- a/include/index.h
+++ b/include/index.h
@@ -493,6 +493,11 @@ private:
filter_node_t* const root,
const bool enable_short_circuit = false) const;
+ void adaptive_filter(uint32_t*& filter_ids,
+ uint32_t& filter_ids_length,
+ filter_node_t* const filter_tree_root,
+ const bool enable_short_circuit = false) const;
+
void get_filter_matches(filter_node_t* const root, std::vector<std::pair<uint32_t, filter_node_t*>>& vec) const;
void insert_doc(const int64_t score, art_tree *t, uint32_t seq_id,
diff --git a/src/field.cpp b/src/field.cpp
index 36ab953e..e5aa527c 100644
--- a/src/field.cpp
+++ b/src/field.cpp
@@ -383,7 +383,9 @@ Option<bool> toFilter(const std::string expression,
Option<bool> toParseTree(std::queue<std::string>& postfix, filter_node_t*& root,
const tsl::htrie_map<char, field>& search_schema,
const Store* store,
- const std::string& doc_id_prefix) {
+ const std::string& doc_id_prefix,
+ int& and_operator_count,
+ int& or_operator_count) {
std::stack<filter_node_t*> nodeStack;
while (!postfix.empty()) {
@@ -406,6 +408,7 @@ Option<bool> toParseTree(std::queue<std::string>& postfix, filter_node_t*& root,
auto operandA = nodeStack.top();
nodeStack.pop();
+ expression == "&&" ? and_operator_count++ : or_operator_count++;
filter_node = new filter_node_t(expression == "&&" ? AND : OR, operandA, operandB);
} else {
filter filter_exp;
@@ -478,11 +481,22 @@ Option<bool> filter::parse_filter_query(const std::string& filter_query,
return toPostfix_op;
}
- Option<bool> toParseTree_op = toParseTree(postfix, root, search_schema, store, doc_id_prefix);
+ int postfix_size = (int) postfix.size(), and_operator_count = 0, or_operator_count = 0;
+ Option<bool> toParseTree_op = toParseTree(postfix,
+ root,
+ search_schema,
+ store,
+ doc_id_prefix,
+ and_operator_count,
+ or_operator_count);
if (!toParseTree_op.ok()) {
return toParseTree_op;
}
+ root->metrics = new filter_tree_metrics{static_cast<int>(postfix_size - (and_operator_count + or_operator_count)),
+ and_operator_count,
+ or_operator_count};
+
return Option<bool>(true);
}
diff --git a/src/index.cpp b/src/index.cpp
index 202e35d5..c4095454 100644
--- a/src/index.cpp
+++ b/src/index.cpp
@@ -2013,13 +2013,13 @@ void Index::get_filter_matches(filter_node_t* const root, std::vector<std::pair<
uint32_t* l_filter_ids = nullptr;
uint32_t l_filter_ids_length = 0;
if (root->left != nullptr) {
- recursive_filter(l_filter_ids, l_filter_ids_length, root->left);
+ rearranging_recursive_filter(l_filter_ids, l_filter_ids_length, root->left);
}
uint32_t* r_filter_ids = nullptr;
uint32_t r_filter_ids_length = 0;
if (root->right != nullptr) {
- recursive_filter(r_filter_ids, r_filter_ids_length, root->right);
+ rearranging_recursive_filter(r_filter_ids, r_filter_ids_length, root->right);
}
root->match_index_ids.first = ArrayUtils::or_scalar(
@@ -2102,12 +2102,14 @@ void Index::recursive_filter(uint32_t*& filter_ids,
recursive_filter(l_filter_ids, l_filter_ids_length, root->left,
enable_short_circuit);
}
+
uint32_t* r_filter_ids = nullptr;
uint32_t r_filter_ids_length = 0;
if (root->right != nullptr) {
recursive_filter(r_filter_ids, r_filter_ids_length, root->right,
enable_short_circuit);
}
+
if (root->isOperator) {
uint32_t* filtered_results = nullptr;
if (root->filter_operator == AND) {
@@ -2134,18 +2136,37 @@ void Index::recursive_filter(uint32_t*& filter_ids,
}
}
+void Index::adaptive_filter(uint32_t*& filter_ids,
+ uint32_t& filter_ids_length,
+ filter_node_t* const filter_tree_root,
+ const bool enable_short_circuit) const {
+ if (filter_tree_root == nullptr) {
+ return;
+ }
+
+ if (filter_tree_root->metrics != nullptr &&
+ (*filter_tree_root->metrics).filter_exp_count > 2 &&
+ (*filter_tree_root->metrics).and_operator_count > 0 &&
+ // If there are more || in the filter tree than &&, we'll not gain much by rearranging the filter tree.
+ ((float) (*filter_tree_root->metrics).or_operator_count / (float) (*filter_tree_root->metrics).and_operator_count < 0.5)) {
+ rearranging_recursive_filter(filter_ids, filter_ids_length, filter_tree_root);
+ } else {
+ recursive_filter(filter_ids, filter_ids_length, filter_tree_root, false);
+ }
+}
+
void Index::do_filtering_with_lock(uint32_t*& filter_ids,
uint32_t& filter_ids_length,
filter_node_t* filter_tree_root) const {
std::shared_lock lock(mutex);
- recursive_filter(filter_ids, filter_ids_length, filter_tree_root, false);
+ adaptive_filter(filter_ids, filter_ids_length, filter_tree_root, false);
}
void Index::do_reference_filtering_with_lock(std::pair<uint32_t, uint32_t*>& reference_index_ids,
filter_node_t* filter_tree_root,
const std::string& reference_field_name) const {
std::shared_lock lock(mutex);
- recursive_filter(reference_index_ids.second, reference_index_ids.first, filter_tree_root, false);
+ adaptive_filter(reference_index_ids.second, reference_index_ids.first, filter_tree_root, false);
std::vector<uint32> vector;
vector.reserve(reference_index_ids.first);
@@ -2653,7 +2674,7 @@ void Index::search(std::vector<query_tokens_t>& field_query_tokens, const std::v
std::shared_lock lock(mutex);
- recursive_filter(filter_ids, filter_ids_length, filter_tree_root, true);
+ adaptive_filter(filter_ids, filter_ids_length, filter_tree_root, true);
if (filter_tree_root != nullptr && filter_ids_length == 0) {
delete [] filter_ids;
@@ -4749,7 +4770,7 @@ void Index::populate_sort_mapping(int* sort_order, std::vector<size_t>& geopoint
field_values[i] = &seq_id_sentinel_value;
} else if (sort_fields_std[i].name == sort_field_const::eval) {
field_values[i] = &eval_sentinel_value;
- recursive_filter(sort_fields_std[i].eval.ids, sort_fields_std[i].eval.size, sort_fields_std[i].eval.filter_tree_root, true);
+ adaptive_filter(sort_fields_std[i].eval.ids, sort_fields_std[i].eval.size, sort_fields_std[i].eval.filter_tree_root, true);
} else if (search_schema.count(sort_fields_std[i].name) != 0 && search_schema.at(sort_fields_std[i].name).sort) {
if (search_schema.at(sort_fields_std[i].name).type == field_types::GEOPOINT_ARRAY) {
geopoint_indices.push_back(i);
diff --git a/test/collection_filtering_test.cpp b/test/collection_filtering_test.cpp
index d253bf4e..2039c5e7 100644
--- a/test/collection_filtering_test.cpp
+++ b/test/collection_filtering_test.cpp
@@ -2590,5 +2590,17 @@ TEST_F(CollectionFilteringTest, ComplexFilterQuery) {
ASSERT_STREQ(id.c_str(), result_id.c_str());
}
+// results = coll->search("Jeremy", {"name"}, "years:>2000 && ((age:<30 && rating:>5) || (age:>50 && rating:<5))",
+// {}, sort_fields_desc, {0}, 10, 1, FREQUENCY, {false}).get();
+// ASSERT_EQ(1, results["hits"].size());
+//
+// ids = {"2"};
+// for (size_t i = 0; i < results["hits"].size(); i++) {
+// nlohmann::json result = results["hits"].at(i);
+// std::string result_id = result["document"]["id"];
+// std::string id = ids.at(i);
+// ASSERT_STREQ(id.c_str(), result_id.c_str());
+// }
+
collectionManager.drop_collection("ComplexFilterQueryCollection");
}
\ No newline at end of file