Skip to content

Open Source alternative to Algolia + Pinecone and an Easier-to-Use alternative to ElasticSearch ⚡ 🔍 ✨ Fast, typo tolerant, in-memory fuzzy Search Engine for building delightful search experiences

Compute filter result into an array. (#1533)

Harpreet Sangar56a69d984472
diff --git a/include/filter_result_iterator.h b/include/filter_result_iterator.h
index fc92ceb4..823243ac 100644
--- a/include/filter_result_iterator.h
+++ b/include/filter_result_iterator.h
@@ -213,10 +213,10 @@ struct filter_result_t {
#ifdef TEST_BUILD
constexpr uint16_t function_call_modulo = 10;
- constexpr uint16_t not_equals_filter_ids_threshold = 3;
+ constexpr uint16_t string_filter_ids_threshold = 3;
#else
constexpr uint16_t function_call_modulo = 16'384;
- constexpr uint16_t not_equals_filter_ids_threshold = 20'000;
+ constexpr uint16_t string_filter_ids_threshold = 20'000;
#endif
struct filter_result_iterator_timeout_info {
diff --git a/src/filter_result_iterator.cpp b/src/filter_result_iterator.cpp
index a823ffaf..a5f4267a 100644
--- a/src/filter_result_iterator.cpp
+++ b/src/filter_result_iterator.cpp
@@ -1164,12 +1164,21 @@ void filter_result_iterator_t::init() {
approx_filter_ids_length += approx_filter_value_match;
}
- if (a_filter.apply_not_equals && approx_filter_ids_length < not_equals_filter_ids_threshold) {
+ if (a_filter.values.size() > 10) {
+ compute_result();
+ return;
+ }
+
+ if (a_filter.apply_not_equals &&
+ index->seq_ids->num_ids() - approx_filter_ids_length < string_filter_ids_threshold) {
// Since there are very few matches, and we have to apply not equals, iteration will be inefficient.
compute_result();
return;
} else if (a_filter.apply_not_equals) {
all_seq_ids_iter = index->seq_ids->new_iterator();
+ } else if (approx_filter_ids_length < string_filter_ids_threshold) {
+ compute_result();
+ return;
}
get_string_filter_first_match(f.is_array());
@@ -1480,7 +1489,8 @@ void filter_result_iterator_t::reset(const bool& override_timeout) {
}
}
- if (a_filter.apply_not_equals && approx_filter_ids_length >= not_equals_filter_ids_threshold) {
+ if (a_filter.apply_not_equals &&
+ (index->seq_ids->num_ids() - approx_filter_ids_length) >= string_filter_ids_threshold) {
all_seq_ids_iter = index->seq_ids->new_iterator();
}
diff --git a/src/index.cpp b/src/index.cpp
index cef54ef7..f351aa44 100644
--- a/src/index.cpp
+++ b/src/index.cpp
@@ -2870,6 +2870,18 @@ Option<bool> Index::search(std::vector<query_tokens_t>& field_query_tokens, cons
return Option(true);
}
+#ifdef TEST_BUILD
+
+ if (filter_result_iterator->approx_filter_ids_length > 20) {
+ filter_result_iterator->compute_result();
+ }
+#else
+
+ if (filter_result_iterator->approx_filter_ids_length < 25'000) {
+ filter_result_iterator->compute_result();
+ }
+#endif
+
size_t fetch_size = offset + per_page;
std::set<uint32_t> curated_ids;
diff --git a/test/collection_filtering_test.cpp b/test/collection_filtering_test.cpp
index dc7e49c9..02901799 100644
--- a/test/collection_filtering_test.cpp
+++ b/test/collection_filtering_test.cpp
@@ -2291,3 +2291,40 @@ TEST_F(CollectionFilteringTest, NonIndexedFiltering) {
ASSERT_FALSE(search_op.ok());
ASSERT_EQ("Cannot filter on non-indexed field `non_index`.", search_op.error());
}
+
+TEST_F(CollectionFilteringTest, ComputeFilterResult) {
+ Collection *coll1;
+
+ std::vector<field> fields = {field("title", field_types::STRING, false),
+ field("points", field_types::INT32, false),};
+
+ coll1 = collectionManager.get_collection("coll1").get();
+ if(coll1 == nullptr) {
+ coll1 = collectionManager.create_collection("coll1", 1, fields, "points").get();
+ }
+
+ for(size_t i=0; i<50; i++) {
+ nlohmann::json doc;
+
+ doc["title"] = i < 10 ? "foo" : "bar";
+ doc["points"] = i;
+
+ ASSERT_TRUE(coll1->add(doc.dump()).ok());
+ }
+
+ auto res_op = coll1->search("*", {}, "title: foo",
+ {}, {}, {0}, 10, 1, FREQUENCY, {true});
+
+ auto results = res_op.get();
+
+ ASSERT_EQ(10, results["found"]);
+
+ res_op = coll1->search("*", {}, "title: bar",
+ {}, {}, {0}, 10, 1, FREQUENCY, {true});
+
+ results = res_op.get();
+
+ ASSERT_EQ(40, results["found"]);
+
+ collectionManager.drop_collection("coll1");
+}
diff --git a/test/filter_test.cpp b/test/filter_test.cpp
index 64a3d3ab..b56358ae 100644
--- a/test/filter_test.cpp
+++ b/test/filter_test.cpp
@@ -529,6 +529,80 @@ TEST_F(FilterTest, FilterTreeIterator) {
ASSERT_EQ(filter_result_iterator_t::valid, iter_add_phrase_ids_test->validity);
ASSERT_EQ(6, iter_add_phrase_ids_test->seq_id);
+ delete filter_tree_root;
+
+ filter_tree_root = nullptr;
+ filter_op = filter::parse_filter_query("tags: [gold]", coll->get_schema(), store, doc_id_prefix,
+ filter_tree_root);
+ ASSERT_TRUE(filter_op.ok());
+
+ auto iter_string_multi_value_test = filter_result_iterator_t(coll->get_name(), coll->_get_index(), filter_tree_root);
+ ASSERT_TRUE(iter_string_multi_value_test.init_status().ok());
+ ASSERT_FALSE(iter_string_multi_value_test._get_is_filter_result_initialized());
+
+ expected = {0, 2, 4};
+ for (auto const& i : expected) {
+ ASSERT_EQ(filter_result_iterator_t::valid, iter_string_multi_value_test.validity);
+ ASSERT_EQ(i, iter_string_multi_value_test.seq_id);
+ iter_string_multi_value_test.next();
+ }
+ ASSERT_EQ(filter_result_iterator_t::invalid, iter_string_multi_value_test.validity);
+ delete filter_tree_root;
+
+ filter_tree_root = nullptr;
+ filter_op = filter::parse_filter_query("tags: [gold, gold, gold, gold, gold, gold, gold, gold, gold, gold, gold]",
+ coll->get_schema(), store, doc_id_prefix,
+ filter_tree_root);
+ ASSERT_TRUE(filter_op.ok());
+
+ auto iter_string_multi_value_test_2 = filter_result_iterator_t(coll->get_name(), coll->_get_index(), filter_tree_root);
+ ASSERT_TRUE(iter_string_multi_value_test_2.init_status().ok());
+ ASSERT_TRUE(iter_string_multi_value_test_2._get_is_filter_result_initialized());
+
+ expected = {0, 2, 4};
+ for (auto const& i : expected) {
+ ASSERT_EQ(filter_result_iterator_t::valid, iter_string_multi_value_test_2.validity);
+ ASSERT_EQ(i, iter_string_multi_value_test_2.seq_id);
+ iter_string_multi_value_test_2.next();
+ }
+ ASSERT_EQ(filter_result_iterator_t::invalid, iter_string_multi_value_test_2.validity);
+ delete filter_tree_root;
+
+ filter_tree_root = nullptr;
+ filter_op = filter::parse_filter_query("tags:= bronze", coll->get_schema(), store, doc_id_prefix,
+ filter_tree_root);
+ ASSERT_TRUE(filter_op.ok());
+
+ auto iter_string_equals_test = filter_result_iterator_t(coll->get_name(), coll->_get_index(), filter_tree_root);
+ ASSERT_TRUE(iter_string_equals_test.init_status().ok());
+ ASSERT_TRUE(iter_string_equals_test._get_is_filter_result_initialized());
+
+ expected = {2, 4};
+ for (auto const& i : expected) {
+ ASSERT_EQ(filter_result_iterator_t::valid, iter_string_equals_test.validity);
+ ASSERT_EQ(i, iter_string_equals_test.seq_id);
+ iter_string_equals_test.next();
+ }
+ ASSERT_EQ(filter_result_iterator_t::invalid, iter_string_equals_test.validity);
+
+ delete filter_tree_root;
+
+ filter_tree_root = nullptr;
+ filter_op = filter::parse_filter_query("tags: gold", coll->get_schema(), store, doc_id_prefix,
+ filter_tree_root);
+ ASSERT_TRUE(filter_op.ok());
+
+ auto iter_string_equals_test_2 = filter_result_iterator_t(coll->get_name(), coll->_get_index(), filter_tree_root);
+ ASSERT_TRUE(iter_string_equals_test_2.init_status().ok());
+ ASSERT_FALSE(iter_string_equals_test_2._get_is_filter_result_initialized());
+
+ expected = {0, 2, 4};
+ for (auto const& i : expected) {
+ ASSERT_EQ(filter_result_iterator_t::valid, iter_string_equals_test_2.validity);
+ ASSERT_EQ(i, iter_string_equals_test_2.seq_id);
+ iter_string_equals_test_2.next();
+ }
+ ASSERT_EQ(filter_result_iterator_t::invalid, iter_string_equals_test_2.validity);
delete filter_tree_root;
@@ -552,15 +626,14 @@ TEST_F(FilterTest, FilterTreeIterator) {
delete filter_tree_root;
filter_tree_root = nullptr;
- filter_op = filter::parse_filter_query("tags: != bronze", coll->get_schema(), store, doc_id_prefix,
+ filter_op = filter::parse_filter_query("tags: != [gold, silver]", coll->get_schema(), store, doc_id_prefix,
filter_tree_root);
ASSERT_TRUE(filter_op.ok());
-
auto iter_string_not_equals_test_2 = filter_result_iterator_t(coll->get_name(), coll->_get_index(), filter_tree_root);
ASSERT_TRUE(iter_string_not_equals_test_2.init_status().ok());
ASSERT_TRUE(iter_string_not_equals_test_2._get_is_filter_result_initialized());
- expected = {0, 1, 3, 5, 6};
+ expected = {1, 5, 6};
for (auto const& i : expected) {
ASSERT_EQ(filter_result_iterator_t::valid, iter_string_not_equals_test_2.validity);
ASSERT_EQ(i, iter_string_not_equals_test_2.seq_id);