Open Source alternative to Algolia + Pinecone and an Easier-to-Use alternative to ElasticSearch ⚡ 🔍 ✨ Fast, typo tolerant, in-memory fuzzy Search Engine for building delightful search experiences
C++ algoliadatastoreelasticsearchenterprise-searchfacetingfull-text-searchfuzzy-searchgeosearchin-memoryinstantsearchmerchandisingpineconesearchsearch-enginesemantic-searchsimilarity-searchsite-searchsynonymstypo-tolerancevector-search
String not equals filter logic refactoring. (#1528)
Harpreet Sangar7e9cb789a73b
diff --git a/include/filter_result_iterator.h b/include/filter_result_iterator.hindex f7febbc9..fc92ceb4 100644--- a/include/filter_result_iterator.h+++ b/include/filter_result_iterator.h@@ -7,6 +7,7 @@#include <memory>#include "option.h"#include "posting_list.h"+#include "id_list.h"class Index;struct filter_node_t;@@ -212,8 +213,10 @@ struct filter_result_t {#ifdef TEST_BUILDconstexpr uint16_t function_call_modulo = 10;+ constexpr uint16_t not_equals_filter_ids_threshold = 3;#else- constexpr uint16_t function_call_modulo = 16384;+ constexpr uint16_t function_call_modulo = 16'384;+ constexpr uint16_t not_equals_filter_ids_threshold = 20'000;#endifstruct filter_result_iterator_timeout_info {@@ -248,6 +251,12 @@ private:std::vector<std::vector<posting_list_t::iterator_t>> posting_list_iterators;std::vector<posting_list_t*> expanded_plists;+ /// Used in case of a not equals string filter.+ /// The iterative logic to find not equals match is to return the ids that occur in between the equals match. This+ /// might lead to returning some ids that are deleted. So we use this iterator to check and return only the ids that+ /// exist in `index->seq_ids`.+ id_list_t::iterator_t all_seq_ids_iter = id_list_t::iterator_t(nullptr, nullptr, nullptr, false);+bool delete_filter_node = false;std::unique_ptr<filter_result_iterator_timeout_info> timeout_info;@@ -356,4 +365,8 @@ public:static void add_phrase_ids(filter_result_iterator_t*& filter_result_iterator,uint32_t* phrase_result_ids, const uint32_t& phrase_result_count);++ bool _get_is_filter_result_initialized() const {+ return is_filter_result_initialized;+ }};diff --git a/include/id_list.h b/include/id_list.hindex f5e2f3ba..176272d8 100644--- a/include/id_list.h+++ b/include/id_list.h@@ -48,6 +48,7 @@ public:explicit iterator_t(block_t* start, block_t* end, std::map<last_id_t, block_t*>* id_block_map, bool reverse);iterator_t(iterator_t&& rhs) noexcept;~iterator_t();+ iterator_t& operator=(iterator_t&& obj) noexcept;[[nodiscard]] bool valid() const;void next();void previous();diff --git a/src/filter_result_iterator.cpp b/src/filter_result_iterator.cppindex 00d9800f..a823ffaf 100644--- a/src/filter_result_iterator.cpp+++ b/src/filter_result_iterator.cpp@@ -479,31 +479,32 @@ void filter_result_iterator_t::next() {if (f.is_string()) {if (filter_node->filter_exp.apply_not_equals) {- if (++seq_id < result_index) {- return;- }-- uint32_t previous_match;do {- previous_match = seq_id;- advance_string_filter_token_iterators();- get_string_filter_next_match(f.is_array());- } while (validity && previous_match + 1 == seq_id);+ if (++seq_id >= result_index) {+ uint32_t previous_match;+ do {+ previous_match = seq_id;+ advance_string_filter_token_iterators();+ get_string_filter_next_match(f.is_array());+ } while (validity && previous_match + 1 == seq_id);++ if (!validity) {+ // We've reached the end of the index, no possible matches pending.+ if (previous_match >= index->seq_ids->last_id()) {+ return;+ }- if (!validity) {- // We've reached the end of the index, no possible matches pending.- if (previous_match >= index->seq_ids->last_id()) {- return;+ // (previous_match, last_doc_id] are a match for not equals.+ validity = valid;+ result_index = index->seq_ids->last_id() + 1;+ seq_id = previous_match + 1;+ } else {+ result_index = seq_id;+ seq_id = previous_match + 1;+ }}-- validity = valid;- result_index = index->seq_ids->last_id() + 1;- seq_id = previous_match + 1;- return;- }-- result_index = seq_id;- seq_id = previous_match + 1;+ all_seq_ids_iter.skip_to(seq_id);+ } while (all_seq_ids_iter.valid() && all_seq_ids_iter.id() != seq_id); // Deleted id should not be considered a match.return;}@@ -558,6 +559,11 @@ void filter_result_iterator_t::get_string_filter_first_match(const bool& field_ivalidity = valid;seq_id = 0;result_index = index->seq_ids->last_id() + 1;++ all_seq_ids_iter.skip_to(seq_id);+ if (all_seq_ids_iter.valid() && all_seq_ids_iter.id() != seq_id) { // Deleted id should not be considered a match.+ next();+ }return;}@@ -565,6 +571,11 @@ void filter_result_iterator_t::get_string_filter_first_match(const bool& field_iif (seq_id > 0) {result_index = seq_id;seq_id = 0;++ all_seq_ids_iter.skip_to(seq_id);+ if (all_seq_ids_iter.valid() && all_seq_ids_iter.id() != seq_id) { // Deleted id should not be considered a match.+ next();+ }return;}@@ -582,14 +593,25 @@ void filter_result_iterator_t::get_string_filter_first_match(const bool& field_ireturn;}+ // (previous_match, last_doc_id] are a match for not equals.validity = valid;result_index = index->seq_ids->last_id() + 1;seq_id = previous_match + 1;++ all_seq_ids_iter.skip_to(seq_id);+ if (all_seq_ids_iter.valid() && all_seq_ids_iter.id() != seq_id) { // Deleted id should not be considered a match.+ next();+ }return;}result_index = seq_id;seq_id = previous_match + 1;++ all_seq_ids_iter.skip_to(seq_id);+ if (all_seq_ids_iter.valid() && all_seq_ids_iter.id() != seq_id) { // Deleted id should not be considered a match.+ next();+ }}}@@ -1106,6 +1128,7 @@ void filter_result_iterator_t::init() {std::string str_token;size_t token_index = 0;std::vector<std::string> str_tokens;+ auto approx_filter_value_match = UINT32_MAX;while (tokenizer.next(str_token, token_index)) {if (str_token.size() > 100) {@@ -1119,7 +1142,8 @@ void filter_result_iterator_t::init() {continue;}- approx_filter_ids_length += posting_t::num_ids(leaf->values);+ // Tokens of a filter value get AND.+ approx_filter_value_match = std::min(posting_t::num_ids(leaf->values), approx_filter_value_match);raw_posting_lists.push_back(leaf->values);}@@ -1135,10 +1159,17 @@ void filter_result_iterator_t::init() {for (auto const& plist: plists) {posting_list_iterators.back().push_back(plist->new_iterator());}++ // Multiple filter values get OR.+ approx_filter_ids_length += approx_filter_value_match;}- if (a_filter.apply_not_equals && approx_filter_ids_length == 0) {- approx_filter_ids_length = index->seq_ids->num_ids();+ if (a_filter.apply_not_equals && approx_filter_ids_length < not_equals_filter_ids_threshold) {+ // Since there are very few matches, and we have to apply not equals, iteration will be inefficient.+ compute_result();+ return;+ } else if (a_filter.apply_not_equals) {+ all_seq_ids_iter = index->seq_ids->new_iterator();}get_string_filter_first_match(f.is_array());@@ -1449,6 +1480,10 @@ void filter_result_iterator_t::reset(const bool& override_timeout) {}}+ if (a_filter.apply_not_equals && approx_filter_ids_length >= not_equals_filter_ids_threshold) {+ all_seq_ids_iter = index->seq_ids->new_iterator();+ }+get_string_filter_first_match(f.is_array());return;}diff --git a/src/id_list.cpp b/src/id_list.cppindex 0710873e..f86ed229 100644--- a/src/id_list.cpp+++ b/src/id_list.cpp@@ -126,6 +126,26 @@ id_list_t::iterator_t::~iterator_t() {ids = nullptr;}+id_list_t::iterator_t& id_list_t::iterator_t::operator=(id_list_t::iterator_t&& obj) noexcept {+ if (&obj == this) {+ return *this;+ }++ delete [] ids;+ ids = obj.ids;+ obj.ids = nullptr;++ curr_block = obj.curr_block;+ curr_index = obj.curr_index;++ end_block = obj.end_block;+ id_block_map = obj.id_block_map;++ reverse = obj.reverse;++ return *this;+}+id_list_t::iterator_t::iterator_t(iterator_t&& rhs) noexcept {curr_block = rhs.curr_block;curr_index = rhs.curr_index;diff --git a/test/filter_test.cpp b/test/filter_test.cppindex 4537ae73..64a3d3ab 100644--- a/test/filter_test.cpp+++ b/test/filter_test.cpp@@ -531,6 +531,85 @@ TEST_F(FilterTest, FilterTreeIterator) {ASSERT_EQ(6, iter_add_phrase_ids_test->seq_id);delete filter_tree_root;++ filter_tree_root = nullptr;+ filter_op = filter::parse_filter_query("tags: != gold", coll->get_schema(), store, doc_id_prefix,+ filter_tree_root);+ ASSERT_TRUE(filter_op.ok());++ auto iter_string_not_equals_test = filter_result_iterator_t(coll->get_name(), coll->_get_index(), filter_tree_root);+ ASSERT_TRUE(iter_string_not_equals_test.init_status().ok());+ ASSERT_FALSE(iter_string_not_equals_test._get_is_filter_result_initialized());++ expected = {1, 3, 5, 6};+ for (auto const& i : expected) {+ ASSERT_EQ(filter_result_iterator_t::valid, iter_string_not_equals_test.validity);+ ASSERT_EQ(i, iter_string_not_equals_test.seq_id);+ iter_string_not_equals_test.next();+ }+ ASSERT_EQ(filter_result_iterator_t::invalid, iter_string_not_equals_test.validity);++ delete filter_tree_root;++ filter_tree_root = nullptr;+ filter_op = filter::parse_filter_query("tags: != bronze", coll->get_schema(), store, doc_id_prefix,+ filter_tree_root);+ ASSERT_TRUE(filter_op.ok());++ auto iter_string_not_equals_test_2 = filter_result_iterator_t(coll->get_name(), coll->_get_index(), filter_tree_root);+ ASSERT_TRUE(iter_string_not_equals_test_2.init_status().ok());+ ASSERT_TRUE(iter_string_not_equals_test_2._get_is_filter_result_initialized());++ expected = {0, 1, 3, 5, 6};+ for (auto const& i : expected) {+ ASSERT_EQ(filter_result_iterator_t::valid, iter_string_not_equals_test_2.validity);+ ASSERT_EQ(i, iter_string_not_equals_test_2.seq_id);+ iter_string_not_equals_test_2.next();+ }+ ASSERT_EQ(filter_result_iterator_t::invalid, iter_string_not_equals_test_2.validity);++ coll->remove("0");+ coll->remove("2");++ doc = R"({+ "name": "James Rowdy",+ "age": 16,+ "years": [2022],+ "rating": 2.03,+ "tags": ["FINE PLATINUM"]+ })"_json;+ add_op = coll->add(doc.dump());+ ASSERT_TRUE(add_op.ok());++ delete filter_tree_root;++ filter_tree_root = nullptr;+ filter_op = filter::parse_filter_query("name: != James Rowdy", coll->get_schema(), store, doc_id_prefix,+ filter_tree_root);+ ASSERT_TRUE(filter_op.ok());++ auto iter_string_not_equals_test_3 = filter_result_iterator_t(coll->get_name(), coll->_get_index(), filter_tree_root);+ ASSERT_TRUE(iter_string_not_equals_test_3.init_status().ok());+ ASSERT_FALSE(iter_string_not_equals_test_3._get_is_filter_result_initialized());++ expected = {1, 3, 4};+ for (auto const& i : expected) {+ ASSERT_EQ(filter_result_iterator_t::valid, iter_string_not_equals_test_3.validity);+ ASSERT_EQ(i, iter_string_not_equals_test_3.seq_id);+ iter_string_not_equals_test_3.next();+ }+ ASSERT_EQ(filter_result_iterator_t::invalid, iter_string_not_equals_test_3.validity);++ iter_string_not_equals_test_3.reset();++ expected = {1, 3, 4};+ for (auto const& i : expected) {+ ASSERT_EQ(filter_result_iterator_t::valid, iter_string_not_equals_test_3.validity);+ ASSERT_EQ(i, iter_string_not_equals_test_3.seq_id);+ iter_string_not_equals_test_3.next();+ }+ ASSERT_EQ(filter_result_iterator_t::invalid, iter_string_not_equals_test_3.validity);+ delete filter_tree_root;}TEST_F(FilterTest, FilterTreeIteratorTimeout) {