Open Source alternative to Algolia + Pinecone and an Easier-to-Use alternative to ElasticSearch ⚡ 🔍 ✨ Fast, typo tolerant, in-memory fuzzy Search Engine for building delightful search experiences
C++ algoliadatastoreelasticsearchenterprise-searchfacetingfull-text-searchfuzzy-searchgeosearchin-memoryinstantsearchmerchandisingpineconesearchsearch-enginesemantic-searchsimilarity-searchsite-searchsynonymstypo-tolerancevector-search
Add `Index::adaptive_filter`.
Harpreet Sangara3d79062adbb
diff --git a/include/field.h b/include/field.hindex 34d0c6e7..e189648d 100644--- a/include/field.h+++ b/include/field.h@@ -530,13 +530,20 @@ struct filter {filter_node_t*& root);};+struct filter_tree_metrics {+ int filter_exp_count;+ int and_operator_count;+ int or_operator_count;+};+struct filter_node_t {filter filter_exp;FILTER_OPERATOR filter_operator;bool isOperator;filter_node_t* left;filter_node_t* right;- std::pair<uint32_t, uint32_t*> match_index_ids;+ std::pair<uint32_t, uint32_t*> match_index_ids = {0, nullptr};+ filter_tree_metrics* metrics = nullptr;filter_node_t(filter filter_exp): filter_exp(std::move(filter_exp)),@@ -553,6 +560,7 @@ struct filter_node_t {right(right) {}~filter_node_t() {+ delete metrics;delete[] match_index_ids.second;delete left;delete right;diff --git a/include/index.h b/include/index.hindex f8690e99..db16a24c 100644--- a/include/index.h+++ b/include/index.h@@ -493,6 +493,11 @@ private:filter_node_t* const root,const bool enable_short_circuit = false) const;+ void adaptive_filter(uint32_t*& filter_ids,+ uint32_t& filter_ids_length,+ filter_node_t* const filter_tree_root,+ const bool enable_short_circuit = false) const;+void get_filter_matches(filter_node_t* const root, std::vector<std::pair<uint32_t, filter_node_t*>>& vec) const;void insert_doc(const int64_t score, art_tree *t, uint32_t seq_id,diff --git a/src/field.cpp b/src/field.cppindex 57e40e37..62875592 100644--- a/src/field.cpp+++ b/src/field.cpp@@ -383,7 +383,9 @@ Option<bool> toFilter(const std::string expression,Option<bool> toParseTree(std::queue<std::string>& postfix, filter_node_t*& root,const tsl::htrie_map<char, field>& search_schema,const Store* store,- const std::string& doc_id_prefix) {+ const std::string& doc_id_prefix,+ int& and_operator_count,+ int& or_operator_count) {std::stack<filter_node_t*> nodeStack;while (!postfix.empty()) {@@ -406,6 +408,7 @@ Option<bool> toParseTree(std::queue<std::string>& postfix, filter_node_t*& root,auto operandA = nodeStack.top();nodeStack.pop();+ expression == "&&" ? and_operator_count++ : or_operator_count++;filter_node = new filter_node_t(expression == "&&" ? AND : OR, operandA, operandB);} else {filter filter_exp;@@ -478,11 +481,22 @@ Option<bool> filter::parse_filter_query(const std::string& filter_query,return toPostfix_op;}- Option<bool> toParseTree_op = toParseTree(postfix, root, search_schema, store, doc_id_prefix);+ int postfix_size = (int) postfix.size(), and_operator_count = 0, or_operator_count = 0;+ Option<bool> toParseTree_op = toParseTree(postfix,+ root,+ search_schema,+ store,+ doc_id_prefix,+ and_operator_count,+ or_operator_count);if (!toParseTree_op.ok()) {return toParseTree_op;}+ root->metrics = new filter_tree_metrics{static_cast<int>(postfix_size - (and_operator_count + or_operator_count)),+ and_operator_count,+ or_operator_count};+return Option<bool>(true);}diff --git a/src/index.cpp b/src/index.cppindex 054a31f7..9e41b70e 100644--- a/src/index.cpp+++ b/src/index.cpp@@ -2031,13 +2031,13 @@ void Index::get_filter_matches(filter_node_t* const root, std::vector<std::pair<uint32_t* l_filter_ids = nullptr;uint32_t l_filter_ids_length = 0;if (root->left != nullptr) {- recursive_filter(l_filter_ids, l_filter_ids_length, root->left);+ rearranging_recursive_filter(l_filter_ids, l_filter_ids_length, root->left);}uint32_t* r_filter_ids = nullptr;uint32_t r_filter_ids_length = 0;if (root->right != nullptr) {- recursive_filter(r_filter_ids, r_filter_ids_length, root->right);+ rearranging_recursive_filter(r_filter_ids, r_filter_ids_length, root->right);}root->match_index_ids.first = ArrayUtils::or_scalar(@@ -2120,12 +2120,14 @@ void Index::recursive_filter(uint32_t*& filter_ids,recursive_filter(l_filter_ids, l_filter_ids_length, root->left,enable_short_circuit);}+uint32_t* r_filter_ids = nullptr;uint32_t r_filter_ids_length = 0;if (root->right != nullptr) {recursive_filter(r_filter_ids, r_filter_ids_length, root->right,enable_short_circuit);}+if (root->isOperator) {uint32_t* filtered_results = nullptr;if (root->filter_operator == AND) {@@ -2152,18 +2154,37 @@ void Index::recursive_filter(uint32_t*& filter_ids,}}+void Index::adaptive_filter(uint32_t*& filter_ids,+ uint32_t& filter_ids_length,+ filter_node_t* const filter_tree_root,+ const bool enable_short_circuit) const {+ if (filter_tree_root == nullptr) {+ return;+ }++ if (filter_tree_root->metrics != nullptr &&+ (*filter_tree_root->metrics).filter_exp_count > 2 &&+ (*filter_tree_root->metrics).and_operator_count > 0 &&+ // If there are more || in the filter tree than &&, we'll not gain much by rearranging the filter tree.+ ((float) (*filter_tree_root->metrics).or_operator_count / (float) (*filter_tree_root->metrics).and_operator_count < 0.5)) {+ rearranging_recursive_filter(filter_ids, filter_ids_length, filter_tree_root);+ } else {+ recursive_filter(filter_ids, filter_ids_length, filter_tree_root, false);+ }+}+void Index::do_filtering_with_lock(uint32_t*& filter_ids,uint32_t& filter_ids_length,filter_node_t* filter_tree_root) const {std::shared_lock lock(mutex);- recursive_filter(filter_ids, filter_ids_length, filter_tree_root, false);+ adaptive_filter(filter_ids, filter_ids_length, filter_tree_root, false);}void Index::do_reference_filtering_with_lock(std::pair<uint32_t, uint32_t*>& reference_index_ids,filter_node_t* filter_tree_root,const std::string& reference_field_name) const {std::shared_lock lock(mutex);- recursive_filter(reference_index_ids.second, reference_index_ids.first, filter_tree_root, false);+ adaptive_filter(reference_index_ids.second, reference_index_ids.first, filter_tree_root, false);std::vector<uint32> vector;vector.reserve(reference_index_ids.first);@@ -2671,7 +2692,7 @@ void Index::search(std::vector<query_tokens_t>& field_query_tokens, const std::vstd::shared_lock lock(mutex);- recursive_filter(filter_ids, filter_ids_length, filter_tree_root, true);+ adaptive_filter(filter_ids, filter_ids_length, filter_tree_root, true);if (filter_tree_root != nullptr && filter_ids_length == 0) {delete [] filter_ids;@@ -4767,7 +4788,7 @@ void Index::populate_sort_mapping(int* sort_order, std::vector<size_t>& geopointfield_values[i] = &seq_id_sentinel_value;} else if (sort_fields_std[i].name == sort_field_const::eval) {field_values[i] = &eval_sentinel_value;- recursive_filter(sort_fields_std[i].eval.ids, sort_fields_std[i].eval.size, sort_fields_std[i].eval.filter_tree_root, true);+ adaptive_filter(sort_fields_std[i].eval.ids, sort_fields_std[i].eval.size, sort_fields_std[i].eval.filter_tree_root, true);} else if (search_schema.count(sort_fields_std[i].name) != 0 && search_schema.at(sort_fields_std[i].name).sort) {if (search_schema.at(sort_fields_std[i].name).type == field_types::GEOPOINT_ARRAY) {geopoint_indices.push_back(i);diff --git a/test/collection_filtering_test.cpp b/test/collection_filtering_test.cppindex d253bf4e..2039c5e7 100644--- a/test/collection_filtering_test.cpp+++ b/test/collection_filtering_test.cpp@@ -2590,5 +2590,17 @@ TEST_F(CollectionFilteringTest, ComplexFilterQuery) {ASSERT_STREQ(id.c_str(), result_id.c_str());}+// results = coll->search("Jeremy", {"name"}, "years:>2000 && ((age:<30 && rating:>5) || (age:>50 && rating:<5))",+// {}, sort_fields_desc, {0}, 10, 1, FREQUENCY, {false}).get();+// ASSERT_EQ(1, results["hits"].size());+//+// ids = {"2"};+// for (size_t i = 0; i < results["hits"].size(); i++) {+// nlohmann::json result = results["hits"].at(i);+// std::string result_id = result["document"]["id"];+// std::string id = ids.at(i);+// ASSERT_STREQ(id.c_str(), result_id.c_str());+// }+collectionManager.drop_collection("ComplexFilterQueryCollection");}\ No newline at end of file