Skip to content

Open Source alternative to Algolia + Pinecone and an Easier-to-Use alternative to ElasticSearch ⚡ 🔍 ✨ Fast, typo tolerant, in-memory fuzzy Search Engine for building delightful search experiences

String not equals filter logic refactoring. (#1528)

Harpreet Sangar7e9cb789a73b
diff --git a/include/filter_result_iterator.h b/include/filter_result_iterator.h
index f7febbc9..fc92ceb4 100644
--- a/include/filter_result_iterator.h
+++ b/include/filter_result_iterator.h
@@ -7,6 +7,7 @@
#include <memory>
#include "option.h"
#include "posting_list.h"
+#include "id_list.h"
class Index;
struct filter_node_t;
@@ -212,8 +213,10 @@ struct filter_result_t {
#ifdef TEST_BUILD
constexpr uint16_t function_call_modulo = 10;
+ constexpr uint16_t not_equals_filter_ids_threshold = 3;
#else
- constexpr uint16_t function_call_modulo = 16384;
+ constexpr uint16_t function_call_modulo = 16'384;
+ constexpr uint16_t not_equals_filter_ids_threshold = 20'000;
#endif
struct filter_result_iterator_timeout_info {
@@ -248,6 +251,12 @@ private:
std::vector<std::vector<posting_list_t::iterator_t>> posting_list_iterators;
std::vector<posting_list_t*> expanded_plists;
+ /// Used in case of a not equals string filter.
+ /// The iterative logic to find not equals match is to return the ids that occur in between the equals match. This
+ /// might lead to returning some ids that are deleted. So we use this iterator to check and return only the ids that
+ /// exist in `index->seq_ids`.
+ id_list_t::iterator_t all_seq_ids_iter = id_list_t::iterator_t(nullptr, nullptr, nullptr, false);
+
bool delete_filter_node = false;
std::unique_ptr<filter_result_iterator_timeout_info> timeout_info;
@@ -356,4 +365,8 @@ public:
static void add_phrase_ids(filter_result_iterator_t*& filter_result_iterator,
uint32_t* phrase_result_ids, const uint32_t& phrase_result_count);
+
+ bool _get_is_filter_result_initialized() const {
+ return is_filter_result_initialized;
+ }
};
diff --git a/include/id_list.h b/include/id_list.h
index f5e2f3ba..176272d8 100644
--- a/include/id_list.h
+++ b/include/id_list.h
@@ -48,6 +48,7 @@ public:
explicit iterator_t(block_t* start, block_t* end, std::map<last_id_t, block_t*>* id_block_map, bool reverse);
iterator_t(iterator_t&& rhs) noexcept;
~iterator_t();
+ iterator_t& operator=(iterator_t&& obj) noexcept;
[[nodiscard]] bool valid() const;
void next();
void previous();
diff --git a/src/filter_result_iterator.cpp b/src/filter_result_iterator.cpp
index 00d9800f..a823ffaf 100644
--- a/src/filter_result_iterator.cpp
+++ b/src/filter_result_iterator.cpp
@@ -479,31 +479,32 @@ void filter_result_iterator_t::next() {
if (f.is_string()) {
if (filter_node->filter_exp.apply_not_equals) {
- if (++seq_id < result_index) {
- return;
- }
-
- uint32_t previous_match;
do {
- previous_match = seq_id;
- advance_string_filter_token_iterators();
- get_string_filter_next_match(f.is_array());
- } while (validity && previous_match + 1 == seq_id);
+ if (++seq_id >= result_index) {
+ uint32_t previous_match;
+ do {
+ previous_match = seq_id;
+ advance_string_filter_token_iterators();
+ get_string_filter_next_match(f.is_array());
+ } while (validity && previous_match + 1 == seq_id);
+
+ if (!validity) {
+ // We've reached the end of the index, no possible matches pending.
+ if (previous_match >= index->seq_ids->last_id()) {
+ return;
+ }
- if (!validity) {
- // We've reached the end of the index, no possible matches pending.
- if (previous_match >= index->seq_ids->last_id()) {
- return;
+ // (previous_match, last_doc_id] are a match for not equals.
+ validity = valid;
+ result_index = index->seq_ids->last_id() + 1;
+ seq_id = previous_match + 1;
+ } else {
+ result_index = seq_id;
+ seq_id = previous_match + 1;
+ }
}
-
- validity = valid;
- result_index = index->seq_ids->last_id() + 1;
- seq_id = previous_match + 1;
- return;
- }
-
- result_index = seq_id;
- seq_id = previous_match + 1;
+ all_seq_ids_iter.skip_to(seq_id);
+ } while (all_seq_ids_iter.valid() && all_seq_ids_iter.id() != seq_id); // Deleted id should not be considered a match.
return;
}
@@ -558,6 +559,11 @@ void filter_result_iterator_t::get_string_filter_first_match(const bool& field_i
validity = valid;
seq_id = 0;
result_index = index->seq_ids->last_id() + 1;
+
+ all_seq_ids_iter.skip_to(seq_id);
+ if (all_seq_ids_iter.valid() && all_seq_ids_iter.id() != seq_id) { // Deleted id should not be considered a match.
+ next();
+ }
return;
}
@@ -565,6 +571,11 @@ void filter_result_iterator_t::get_string_filter_first_match(const bool& field_i
if (seq_id > 0) {
result_index = seq_id;
seq_id = 0;
+
+ all_seq_ids_iter.skip_to(seq_id);
+ if (all_seq_ids_iter.valid() && all_seq_ids_iter.id() != seq_id) { // Deleted id should not be considered a match.
+ next();
+ }
return;
}
@@ -582,14 +593,25 @@ void filter_result_iterator_t::get_string_filter_first_match(const bool& field_i
return;
}
+ // (previous_match, last_doc_id] are a match for not equals.
validity = valid;
result_index = index->seq_ids->last_id() + 1;
seq_id = previous_match + 1;
+
+ all_seq_ids_iter.skip_to(seq_id);
+ if (all_seq_ids_iter.valid() && all_seq_ids_iter.id() != seq_id) { // Deleted id should not be considered a match.
+ next();
+ }
return;
}
result_index = seq_id;
seq_id = previous_match + 1;
+
+ all_seq_ids_iter.skip_to(seq_id);
+ if (all_seq_ids_iter.valid() && all_seq_ids_iter.id() != seq_id) { // Deleted id should not be considered a match.
+ next();
+ }
}
}
@@ -1106,6 +1128,7 @@ void filter_result_iterator_t::init() {
std::string str_token;
size_t token_index = 0;
std::vector<std::string> str_tokens;
+ auto approx_filter_value_match = UINT32_MAX;
while (tokenizer.next(str_token, token_index)) {
if (str_token.size() > 100) {
@@ -1119,7 +1142,8 @@ void filter_result_iterator_t::init() {
continue;
}
- approx_filter_ids_length += posting_t::num_ids(leaf->values);
+ // Tokens of a filter value get AND.
+ approx_filter_value_match = std::min(posting_t::num_ids(leaf->values), approx_filter_value_match);
raw_posting_lists.push_back(leaf->values);
}
@@ -1135,10 +1159,17 @@ void filter_result_iterator_t::init() {
for (auto const& plist: plists) {
posting_list_iterators.back().push_back(plist->new_iterator());
}
+
+ // Multiple filter values get OR.
+ approx_filter_ids_length += approx_filter_value_match;
}
- if (a_filter.apply_not_equals && approx_filter_ids_length == 0) {
- approx_filter_ids_length = index->seq_ids->num_ids();
+ if (a_filter.apply_not_equals && approx_filter_ids_length < not_equals_filter_ids_threshold) {
+ // Since there are very few matches, and we have to apply not equals, iteration will be inefficient.
+ compute_result();
+ return;
+ } else if (a_filter.apply_not_equals) {
+ all_seq_ids_iter = index->seq_ids->new_iterator();
}
get_string_filter_first_match(f.is_array());
@@ -1449,6 +1480,10 @@ void filter_result_iterator_t::reset(const bool& override_timeout) {
}
}
+ if (a_filter.apply_not_equals && approx_filter_ids_length >= not_equals_filter_ids_threshold) {
+ all_seq_ids_iter = index->seq_ids->new_iterator();
+ }
+
get_string_filter_first_match(f.is_array());
return;
}
diff --git a/src/id_list.cpp b/src/id_list.cpp
index 0710873e..f86ed229 100644
--- a/src/id_list.cpp
+++ b/src/id_list.cpp
@@ -126,6 +126,26 @@ id_list_t::iterator_t::~iterator_t() {
ids = nullptr;
}
+id_list_t::iterator_t& id_list_t::iterator_t::operator=(id_list_t::iterator_t&& obj) noexcept {
+ if (&obj == this) {
+ return *this;
+ }
+
+ delete [] ids;
+ ids = obj.ids;
+ obj.ids = nullptr;
+
+ curr_block = obj.curr_block;
+ curr_index = obj.curr_index;
+
+ end_block = obj.end_block;
+ id_block_map = obj.id_block_map;
+
+ reverse = obj.reverse;
+
+ return *this;
+}
+
id_list_t::iterator_t::iterator_t(iterator_t&& rhs) noexcept {
curr_block = rhs.curr_block;
curr_index = rhs.curr_index;
diff --git a/test/filter_test.cpp b/test/filter_test.cpp
index 4537ae73..64a3d3ab 100644
--- a/test/filter_test.cpp
+++ b/test/filter_test.cpp
@@ -531,6 +531,85 @@ TEST_F(FilterTest, FilterTreeIterator) {
ASSERT_EQ(6, iter_add_phrase_ids_test->seq_id);
delete filter_tree_root;
+
+ filter_tree_root = nullptr;
+ filter_op = filter::parse_filter_query("tags: != gold", coll->get_schema(), store, doc_id_prefix,
+ filter_tree_root);
+ ASSERT_TRUE(filter_op.ok());
+
+ auto iter_string_not_equals_test = filter_result_iterator_t(coll->get_name(), coll->_get_index(), filter_tree_root);
+ ASSERT_TRUE(iter_string_not_equals_test.init_status().ok());
+ ASSERT_FALSE(iter_string_not_equals_test._get_is_filter_result_initialized());
+
+ expected = {1, 3, 5, 6};
+ for (auto const& i : expected) {
+ ASSERT_EQ(filter_result_iterator_t::valid, iter_string_not_equals_test.validity);
+ ASSERT_EQ(i, iter_string_not_equals_test.seq_id);
+ iter_string_not_equals_test.next();
+ }
+ ASSERT_EQ(filter_result_iterator_t::invalid, iter_string_not_equals_test.validity);
+
+ delete filter_tree_root;
+
+ filter_tree_root = nullptr;
+ filter_op = filter::parse_filter_query("tags: != bronze", coll->get_schema(), store, doc_id_prefix,
+ filter_tree_root);
+ ASSERT_TRUE(filter_op.ok());
+
+ auto iter_string_not_equals_test_2 = filter_result_iterator_t(coll->get_name(), coll->_get_index(), filter_tree_root);
+ ASSERT_TRUE(iter_string_not_equals_test_2.init_status().ok());
+ ASSERT_TRUE(iter_string_not_equals_test_2._get_is_filter_result_initialized());
+
+ expected = {0, 1, 3, 5, 6};
+ for (auto const& i : expected) {
+ ASSERT_EQ(filter_result_iterator_t::valid, iter_string_not_equals_test_2.validity);
+ ASSERT_EQ(i, iter_string_not_equals_test_2.seq_id);
+ iter_string_not_equals_test_2.next();
+ }
+ ASSERT_EQ(filter_result_iterator_t::invalid, iter_string_not_equals_test_2.validity);
+
+ coll->remove("0");
+ coll->remove("2");
+
+ doc = R"({
+ "name": "James Rowdy",
+ "age": 16,
+ "years": [2022],
+ "rating": 2.03,
+ "tags": ["FINE PLATINUM"]
+ })"_json;
+ add_op = coll->add(doc.dump());
+ ASSERT_TRUE(add_op.ok());
+
+ delete filter_tree_root;
+
+ filter_tree_root = nullptr;
+ filter_op = filter::parse_filter_query("name: != James Rowdy", coll->get_schema(), store, doc_id_prefix,
+ filter_tree_root);
+ ASSERT_TRUE(filter_op.ok());
+
+ auto iter_string_not_equals_test_3 = filter_result_iterator_t(coll->get_name(), coll->_get_index(), filter_tree_root);
+ ASSERT_TRUE(iter_string_not_equals_test_3.init_status().ok());
+ ASSERT_FALSE(iter_string_not_equals_test_3._get_is_filter_result_initialized());
+
+ expected = {1, 3, 4};
+ for (auto const& i : expected) {
+ ASSERT_EQ(filter_result_iterator_t::valid, iter_string_not_equals_test_3.validity);
+ ASSERT_EQ(i, iter_string_not_equals_test_3.seq_id);
+ iter_string_not_equals_test_3.next();
+ }
+ ASSERT_EQ(filter_result_iterator_t::invalid, iter_string_not_equals_test_3.validity);
+
+ iter_string_not_equals_test_3.reset();
+
+ expected = {1, 3, 4};
+ for (auto const& i : expected) {
+ ASSERT_EQ(filter_result_iterator_t::valid, iter_string_not_equals_test_3.validity);
+ ASSERT_EQ(i, iter_string_not_equals_test_3.seq_id);
+ iter_string_not_equals_test_3.next();
+ }
+ ASSERT_EQ(filter_result_iterator_t::invalid, iter_string_not_equals_test_3.validity);
+ delete filter_tree_root;
}
TEST_F(FilterTest, FilterTreeIteratorTimeout) {