Skip to content

Open Source alternative to Algolia + Pinecone and an Easier-to-Use alternative to ElasticSearch ⚡ 🔍 ✨ Fast, typo tolerant, in-memory fuzzy Search Engine for building delightful search experiences

Optimize filter iterator initialization. (#1626)

Harpreet Sangaracf28000062c
diff --git a/include/filter_result_iterator.h b/include/filter_result_iterator.h
index 04af6e69..26028074 100644
--- a/include/filter_result_iterator.h
+++ b/include/filter_result_iterator.h
@@ -375,4 +375,12 @@ public:
[[nodiscard]] bool _get_is_filter_result_initialized() const {
return is_filter_result_initialized;
}
+
+ [[nodiscard]] filter_result_iterator_t* _get_left_it() const {
+ return left_it;
+ }
+
+ [[nodiscard]] filter_result_iterator_t* _get_right_it() const {
+ return right_it;
+ }
};
diff --git a/src/filter_result_iterator.cpp b/src/filter_result_iterator.cpp
index 16b80755..759a2ee3 100644
--- a/src/filter_result_iterator.cpp
+++ b/src/filter_result_iterator.cpp
@@ -1506,7 +1506,9 @@ int filter_result_iterator_t::is_valid(uint32_t id) {
}
Option<bool> filter_result_iterator_t::init_status() {
- if (filter_node != nullptr && filter_node->isOperator) {
+ if (is_filter_result_initialized) {
+ return status;
+ } else if (filter_node != nullptr && filter_node->isOperator) {
auto left_status = left_it->init_status();
return !left_status.ok() ? left_status : right_it->init_status();
@@ -1779,6 +1781,15 @@ filter_result_iterator_t::filter_result_iterator_t(const std::string& collection
// Generate the iterator tree and then initialize each node.
if (filter_node->isOperator) {
left_it = new filter_result_iterator_t(collection_name, index, filter_node->left);
+ // If left subtree of && operator is invalid, we don't have to evaluate its right subtree.
+ if (filter_node->filter_operator == AND && left_it->validity == invalid) {
+ validity = invalid;
+ is_filter_result_initialized = true;
+ delete left_it;
+ left_it = nullptr;
+ return;
+ }
+
right_it = new filter_result_iterator_t(collection_name, index, filter_node->right);
}
diff --git a/test/filter_test.cpp b/test/filter_test.cpp
index a6a7a5b2..f65a6c3b 100644
--- a/test/filter_test.cpp
+++ b/test/filter_test.cpp
@@ -863,3 +863,78 @@ TEST_F(FilterTest, FilterTreeIteratorTimeout) {
ASSERT_EQ(count, result->count); // With `override_timeout` true, we should get result.
delete result;
}
+
+TEST_F(FilterTest, FilterTreeInitialization) {
+ nlohmann::json schema =
+ R"({
+ "name": "Collection",
+ "fields": [
+ {"name": "name", "type": "string"},
+ {"name": "age", "type": "int32"},
+ {"name": "years", "type": "int32[]"},
+ {"name": "rating", "type": "float"},
+ {"name": "tags", "type": "string[]"}
+ ]
+ })"_json;
+
+ Collection* coll = collectionManager.create_collection(schema).get();
+
+ std::ifstream infile(std::string(ROOT_DIR)+"test/numeric_array_documents.jsonl");
+ std::string json_line;
+ while (std::getline(infile, json_line)) {
+ auto add_op = coll->add(json_line);
+ ASSERT_TRUE(add_op.ok());
+ }
+ infile.close();
+
+ const std::string doc_id_prefix = std::to_string(coll->get_collection_id()) + "_" + Collection::DOC_ID_PREFIX + "_";
+ filter_node_t* filter_tree_root = nullptr;
+
+ Option<bool> filter_op = filter::parse_filter_query("age: 0 && (rating: >0 && years: 2016)", coll->get_schema(), store, doc_id_prefix,
+ filter_tree_root);
+ ASSERT_TRUE(filter_op.ok());
+
+ auto iter_left_subtree_0_matches = filter_result_iterator_t(coll->get_name(), coll->_get_index(), filter_tree_root);
+
+ ASSERT_TRUE(iter_left_subtree_0_matches.init_status().ok());
+ ASSERT_EQ(filter_result_iterator_t::invalid, iter_left_subtree_0_matches.validity);
+ ASSERT_EQ(0, iter_left_subtree_0_matches.approx_filter_ids_length);
+ ASSERT_TRUE(iter_left_subtree_0_matches._get_is_filter_result_initialized());
+ ASSERT_EQ(nullptr, iter_left_subtree_0_matches._get_left_it());
+ ASSERT_EQ(nullptr, iter_left_subtree_0_matches._get_right_it());
+
+ delete filter_tree_root;
+ filter_tree_root = nullptr;
+
+ filter_op = filter::parse_filter_query("(rating: >0 && years: 2016) && age: 0", coll->get_schema(), store, doc_id_prefix,
+ filter_tree_root);
+ ASSERT_TRUE(filter_op.ok());
+
+ auto iter_right_subtree_0_matches = filter_result_iterator_t(coll->get_name(), coll->_get_index(), filter_tree_root);
+
+ ASSERT_TRUE(iter_right_subtree_0_matches.init_status().ok());
+ ASSERT_EQ(filter_result_iterator_t::invalid, iter_right_subtree_0_matches.validity);
+ ASSERT_EQ(0, iter_right_subtree_0_matches.approx_filter_ids_length);
+ ASSERT_FALSE(iter_right_subtree_0_matches._get_is_filter_result_initialized());
+ ASSERT_NE(nullptr, iter_right_subtree_0_matches._get_left_it());
+ ASSERT_NE(nullptr, iter_right_subtree_0_matches._get_right_it());
+
+ delete filter_tree_root;
+ filter_tree_root = nullptr;
+
+ filter_op = filter::parse_filter_query("(age: 0 && rating: >0) || (age: 0 && rating: >0)", coll->get_schema(), store, doc_id_prefix,
+ filter_tree_root);
+ ASSERT_TRUE(filter_op.ok());
+
+ auto iter_inner_subtree_0_matches = filter_result_iterator_t(coll->get_name(), coll->_get_index(), filter_tree_root);
+
+ ASSERT_TRUE(iter_inner_subtree_0_matches.init_status().ok());
+ ASSERT_EQ(filter_result_iterator_t::invalid, iter_inner_subtree_0_matches.validity);
+ ASSERT_EQ(0, iter_inner_subtree_0_matches.approx_filter_ids_length);
+ ASSERT_FALSE(iter_inner_subtree_0_matches._get_is_filter_result_initialized());
+ ASSERT_NE(nullptr, iter_inner_subtree_0_matches._get_left_it());
+ ASSERT_NE(nullptr, iter_inner_subtree_0_matches._get_right_it());
+
+ delete filter_tree_root;
+ filter_tree_root = nullptr;
+}