Open Source alternative to Algolia + Pinecone and an Easier-to-Use alternative to ElasticSearch ⚡ 🔍 ✨ Fast, typo tolerant, in-memory fuzzy Search Engine for building delightful search experiences
C++ algoliadatastoreelasticsearchenterprise-searchfacetingfull-text-searchfuzzy-searchgeosearchin-memoryinstantsearchmerchandisingpineconesearchsearch-enginesemantic-searchsimilarity-searchsite-searchsynonymstypo-tolerancevector-search
Compute filter result into an array. (#1533)
Harpreet Sangar56a69d984472
diff --git a/include/filter_result_iterator.h b/include/filter_result_iterator.hindex fc92ceb4..823243ac 100644--- a/include/filter_result_iterator.h+++ b/include/filter_result_iterator.h@@ -213,10 +213,10 @@ struct filter_result_t {#ifdef TEST_BUILDconstexpr uint16_t function_call_modulo = 10;- constexpr uint16_t not_equals_filter_ids_threshold = 3;+ constexpr uint16_t string_filter_ids_threshold = 3;#elseconstexpr uint16_t function_call_modulo = 16'384;- constexpr uint16_t not_equals_filter_ids_threshold = 20'000;+ constexpr uint16_t string_filter_ids_threshold = 20'000;#endifstruct filter_result_iterator_timeout_info {diff --git a/src/filter_result_iterator.cpp b/src/filter_result_iterator.cppindex a823ffaf..a5f4267a 100644--- a/src/filter_result_iterator.cpp+++ b/src/filter_result_iterator.cpp@@ -1164,12 +1164,21 @@ void filter_result_iterator_t::init() {approx_filter_ids_length += approx_filter_value_match;}- if (a_filter.apply_not_equals && approx_filter_ids_length < not_equals_filter_ids_threshold) {+ if (a_filter.values.size() > 10) {+ compute_result();+ return;+ }++ if (a_filter.apply_not_equals &&+ index->seq_ids->num_ids() - approx_filter_ids_length < string_filter_ids_threshold) {// Since there are very few matches, and we have to apply not equals, iteration will be inefficient.compute_result();return;} else if (a_filter.apply_not_equals) {all_seq_ids_iter = index->seq_ids->new_iterator();+ } else if (approx_filter_ids_length < string_filter_ids_threshold) {+ compute_result();+ return;}get_string_filter_first_match(f.is_array());@@ -1480,7 +1489,8 @@ void filter_result_iterator_t::reset(const bool& override_timeout) {}}- if (a_filter.apply_not_equals && approx_filter_ids_length >= not_equals_filter_ids_threshold) {+ if (a_filter.apply_not_equals &&+ (index->seq_ids->num_ids() - approx_filter_ids_length) >= string_filter_ids_threshold) {all_seq_ids_iter = index->seq_ids->new_iterator();}diff --git a/src/index.cpp b/src/index.cppindex cef54ef7..f351aa44 100644--- a/src/index.cpp+++ b/src/index.cpp@@ -2870,6 +2870,18 @@ Option<bool> Index::search(std::vector<query_tokens_t>& field_query_tokens, consreturn Option(true);}+#ifdef TEST_BUILD++ if (filter_result_iterator->approx_filter_ids_length > 20) {+ filter_result_iterator->compute_result();+ }+#else++ if (filter_result_iterator->approx_filter_ids_length < 25'000) {+ filter_result_iterator->compute_result();+ }+#endif+size_t fetch_size = offset + per_page;std::set<uint32_t> curated_ids;diff --git a/test/collection_filtering_test.cpp b/test/collection_filtering_test.cppindex dc7e49c9..02901799 100644--- a/test/collection_filtering_test.cpp+++ b/test/collection_filtering_test.cpp@@ -2291,3 +2291,40 @@ TEST_F(CollectionFilteringTest, NonIndexedFiltering) {ASSERT_FALSE(search_op.ok());ASSERT_EQ("Cannot filter on non-indexed field `non_index`.", search_op.error());}++TEST_F(CollectionFilteringTest, ComputeFilterResult) {+ Collection *coll1;++ std::vector<field> fields = {field("title", field_types::STRING, false),+ field("points", field_types::INT32, false),};++ coll1 = collectionManager.get_collection("coll1").get();+ if(coll1 == nullptr) {+ coll1 = collectionManager.create_collection("coll1", 1, fields, "points").get();+ }++ for(size_t i=0; i<50; i++) {+ nlohmann::json doc;++ doc["title"] = i < 10 ? "foo" : "bar";+ doc["points"] = i;++ ASSERT_TRUE(coll1->add(doc.dump()).ok());+ }++ auto res_op = coll1->search("*", {}, "title: foo",+ {}, {}, {0}, 10, 1, FREQUENCY, {true});++ auto results = res_op.get();++ ASSERT_EQ(10, results["found"]);++ res_op = coll1->search("*", {}, "title: bar",+ {}, {}, {0}, 10, 1, FREQUENCY, {true});++ results = res_op.get();++ ASSERT_EQ(40, results["found"]);++ collectionManager.drop_collection("coll1");+}diff --git a/test/filter_test.cpp b/test/filter_test.cppindex 64a3d3ab..b56358ae 100644--- a/test/filter_test.cpp+++ b/test/filter_test.cpp@@ -529,6 +529,80 @@ TEST_F(FilterTest, FilterTreeIterator) {ASSERT_EQ(filter_result_iterator_t::valid, iter_add_phrase_ids_test->validity);ASSERT_EQ(6, iter_add_phrase_ids_test->seq_id);+ delete filter_tree_root;++ filter_tree_root = nullptr;+ filter_op = filter::parse_filter_query("tags: [gold]", coll->get_schema(), store, doc_id_prefix,+ filter_tree_root);+ ASSERT_TRUE(filter_op.ok());++ auto iter_string_multi_value_test = filter_result_iterator_t(coll->get_name(), coll->_get_index(), filter_tree_root);+ ASSERT_TRUE(iter_string_multi_value_test.init_status().ok());+ ASSERT_FALSE(iter_string_multi_value_test._get_is_filter_result_initialized());++ expected = {0, 2, 4};+ for (auto const& i : expected) {+ ASSERT_EQ(filter_result_iterator_t::valid, iter_string_multi_value_test.validity);+ ASSERT_EQ(i, iter_string_multi_value_test.seq_id);+ iter_string_multi_value_test.next();+ }+ ASSERT_EQ(filter_result_iterator_t::invalid, iter_string_multi_value_test.validity);+ delete filter_tree_root;++ filter_tree_root = nullptr;+ filter_op = filter::parse_filter_query("tags: [gold, gold, gold, gold, gold, gold, gold, gold, gold, gold, gold]",+ coll->get_schema(), store, doc_id_prefix,+ filter_tree_root);+ ASSERT_TRUE(filter_op.ok());++ auto iter_string_multi_value_test_2 = filter_result_iterator_t(coll->get_name(), coll->_get_index(), filter_tree_root);+ ASSERT_TRUE(iter_string_multi_value_test_2.init_status().ok());+ ASSERT_TRUE(iter_string_multi_value_test_2._get_is_filter_result_initialized());++ expected = {0, 2, 4};+ for (auto const& i : expected) {+ ASSERT_EQ(filter_result_iterator_t::valid, iter_string_multi_value_test_2.validity);+ ASSERT_EQ(i, iter_string_multi_value_test_2.seq_id);+ iter_string_multi_value_test_2.next();+ }+ ASSERT_EQ(filter_result_iterator_t::invalid, iter_string_multi_value_test_2.validity);+ delete filter_tree_root;++ filter_tree_root = nullptr;+ filter_op = filter::parse_filter_query("tags:= bronze", coll->get_schema(), store, doc_id_prefix,+ filter_tree_root);+ ASSERT_TRUE(filter_op.ok());++ auto iter_string_equals_test = filter_result_iterator_t(coll->get_name(), coll->_get_index(), filter_tree_root);+ ASSERT_TRUE(iter_string_equals_test.init_status().ok());+ ASSERT_TRUE(iter_string_equals_test._get_is_filter_result_initialized());++ expected = {2, 4};+ for (auto const& i : expected) {+ ASSERT_EQ(filter_result_iterator_t::valid, iter_string_equals_test.validity);+ ASSERT_EQ(i, iter_string_equals_test.seq_id);+ iter_string_equals_test.next();+ }+ ASSERT_EQ(filter_result_iterator_t::invalid, iter_string_equals_test.validity);++ delete filter_tree_root;++ filter_tree_root = nullptr;+ filter_op = filter::parse_filter_query("tags: gold", coll->get_schema(), store, doc_id_prefix,+ filter_tree_root);+ ASSERT_TRUE(filter_op.ok());++ auto iter_string_equals_test_2 = filter_result_iterator_t(coll->get_name(), coll->_get_index(), filter_tree_root);+ ASSERT_TRUE(iter_string_equals_test_2.init_status().ok());+ ASSERT_FALSE(iter_string_equals_test_2._get_is_filter_result_initialized());++ expected = {0, 2, 4};+ for (auto const& i : expected) {+ ASSERT_EQ(filter_result_iterator_t::valid, iter_string_equals_test_2.validity);+ ASSERT_EQ(i, iter_string_equals_test_2.seq_id);+ iter_string_equals_test_2.next();+ }+ ASSERT_EQ(filter_result_iterator_t::invalid, iter_string_equals_test_2.validity);delete filter_tree_root;@@ -552,15 +626,14 @@ TEST_F(FilterTest, FilterTreeIterator) {delete filter_tree_root;filter_tree_root = nullptr;- filter_op = filter::parse_filter_query("tags: != bronze", coll->get_schema(), store, doc_id_prefix,+ filter_op = filter::parse_filter_query("tags: != [gold, silver]", coll->get_schema(), store, doc_id_prefix,filter_tree_root);ASSERT_TRUE(filter_op.ok());-auto iter_string_not_equals_test_2 = filter_result_iterator_t(coll->get_name(), coll->_get_index(), filter_tree_root);ASSERT_TRUE(iter_string_not_equals_test_2.init_status().ok());ASSERT_TRUE(iter_string_not_equals_test_2._get_is_filter_result_initialized());- expected = {0, 1, 3, 5, 6};+ expected = {1, 5, 6};for (auto const& i : expected) {ASSERT_EQ(filter_result_iterator_t::valid, iter_string_not_equals_test_2.validity);ASSERT_EQ(i, iter_string_not_equals_test_2.seq_id);