Skip to content

Open Source alternative to Algolia + Pinecone and an Easier-to-Use alternative to ElasticSearch ⚡ 🔍 ✨ Fast, typo tolerant, in-memory fuzzy Search Engine for building delightful search experiences

num_tree iterator (#1538)

Harpreet Sangar54492fafdc5f
diff --git a/include/art.h b/include/art.h
index 92e043e3..7fedbbea 100644
--- a/include/art.h
+++ b/include/art.h
@@ -7,6 +7,7 @@
#include "array.h"
#include "sorted_array.h"
#include "filter_result_iterator.h"
+#include "filter.h"
#define IGNORE_PRINTF 1
@@ -146,17 +147,6 @@ enum token_ordering {
MAX_SCORE
};
-enum NUM_COMPARATOR {
- LESS_THAN,
- LESS_THAN_EQUALS,
- EQUALS,
- NOT_EQUALS,
- CONTAINS,
- GREATER_THAN,
- GREATER_THAN_EQUALS,
- RANGE_INCLUSIVE
-};
-
/**
* Initializes an ART tree
* @return 0 on success.
diff --git a/include/filter.h b/include/filter.h
index 84dd793b..e6071eba 100644
--- a/include/filter.h
+++ b/include/filter.h
@@ -3,10 +3,20 @@
#include <string>
#include <map>
#include <tsl/htrie_map.h>
-#include <art.h>
#include <json.hpp>
#include "store.h"
+enum NUM_COMPARATOR {
+ LESS_THAN,
+ LESS_THAN_EQUALS,
+ EQUALS,
+ NOT_EQUALS,
+ CONTAINS,
+ GREATER_THAN,
+ GREATER_THAN_EQUALS,
+ RANGE_INCLUSIVE
+};
+
enum FILTER_OPERATOR {
AND,
OR
diff --git a/include/filter_result_iterator.h b/include/filter_result_iterator.h
index f36ae750..04af6e69 100644
--- a/include/filter_result_iterator.h
+++ b/include/filter_result_iterator.h
@@ -5,6 +5,7 @@
#include <utility>
#include <vector>
#include <memory>
+#include "num_tree.h"
#include "option.h"
#include "posting_list.h"
#include "id_list.h"
@@ -214,9 +215,11 @@ struct filter_result_t {
#ifdef TEST_BUILD
constexpr uint16_t function_call_modulo = 10;
constexpr uint16_t string_filter_ids_threshold = 3;
+ constexpr uint16_t bool_filter_ids_threshold = 3;
#else
constexpr uint16_t function_call_modulo = 16'384;
constexpr uint16_t string_filter_ids_threshold = 20'000;
+ constexpr uint16_t bool_filter_ids_threshold = 20'000;
#endif
struct filter_result_iterator_timeout_info {
@@ -257,6 +260,9 @@ private:
/// exist in `index->seq_ids`.
id_list_t::iterator_t all_seq_ids_iter = id_list_t::iterator_t(nullptr, nullptr, nullptr, false);
+ /// Used in case of a single boolean filter matching more than `bool_filter_ids_threshold` ids.
+ num_tree_t::iterator_t bool_iterator = num_tree_t::iterator_t(nullptr, NUM_COMPARATOR::EQUALS, 0);
+
bool delete_filter_node = false;
std::unique_ptr<filter_result_iterator_timeout_info> timeout_info;
@@ -322,7 +328,7 @@ public:
Option<bool> init_status();
/// Recursively computes the result of each node and stores the final result in the root node.
- void compute_string_components();
+ void compute_iterators();
/// Returns a tri-state:
/// 0: id is not valid
@@ -366,7 +372,7 @@ public:
static void add_phrase_ids(filter_result_iterator_t*& filter_result_iterator,
uint32_t* phrase_result_ids, const uint32_t& phrase_result_count);
- bool _get_is_filter_result_initialized() const {
+ [[nodiscard]] bool _get_is_filter_result_initialized() const {
return is_filter_result_initialized;
}
};
diff --git a/include/num_tree.h b/include/num_tree.h
index 5315922a..b0b1a951 100644
--- a/include/num_tree.h
+++ b/include/num_tree.h
@@ -4,8 +4,8 @@
#include "sparsepp.h"
#include "sorted_array.h"
#include "array_utils.h"
-#include "art.h"
#include "ids_t.h"
+#include "filter.h"
class num_tree_t {
private:
@@ -42,7 +42,7 @@ public:
void search(NUM_COMPARATOR comparator, int64_t value, uint32_t** ids, size_t& ids_len);
- void approx_search_count(NUM_COMPARATOR comparator, int64_t value, uint32_t& ids_len);
+ uint32_t approx_search_count(NUM_COMPARATOR comparator, int64_t value);
void remove(uint64_t value, uint32_t id);
@@ -57,4 +57,48 @@ public:
uint32_t*& result_ids) const;
std::pair<int64_t, int64_t> get_min_max(const uint32_t* result_ids, size_t result_ids_len);
+
+ class iterator_t {
+ /// If true, `id_list_array` is initialized otherwise `id_list_iterator` is.
+ bool is_compact_id_list = true;
+
+ uint32_t index = 0;
+ uint32_t id_list_array_len = 0;
+ uint32_t* id_list_array = nullptr;
+
+ id_list_t* id_list = nullptr;
+ id_list_t::iterator_t id_list_iterator = id_list_t::iterator_t(nullptr, nullptr, nullptr, false);
+
+ public:
+
+ uint32_t seq_id = 0;
+ bool is_valid = true;
+
+ /// Holds the upper-bound of the number of seq ids this iterator would match.
+ /// Useful in a scenario where we need to differentiate between iterator not matching any document v/s iterator
+ /// reaching it's end. (is_valid would be false in both these cases)
+ uint32_t approx_filter_ids_length = 0;
+
+ explicit iterator_t(num_tree_t* num_tree, NUM_COMPARATOR comparator, int64_t value);
+
+ ~iterator_t();
+
+ iterator_t& operator=(iterator_t&& obj) noexcept;
+
+ /// Returns a tri-state:
+ /// 0: id is not valid
+ /// 1: id is valid
+ /// -1: end of iterator
+ [[nodiscard]] int is_id_valid(uint32_t id);
+
+ /// Advances the iterator to get the next seq_id. The iterator may become invalid during this operation.
+ void next();
+
+ /// Advances the iterator until the seq_id reaches or just overshoots id. The iterator may become invalid during
+ /// this operation.
+ void skip_to(uint32_t id);
+
+ /// Returns to the initial state of the iterator.
+ void reset();
+ };
};
\ No newline at end of file
diff --git a/src/filter_result_iterator.cpp b/src/filter_result_iterator.cpp
index 4ff0e76c..c8b50144 100644
--- a/src/filter_result_iterator.cpp
+++ b/src/filter_result_iterator.cpp
@@ -429,7 +429,7 @@ void filter_result_iterator_t::next() {
return;
}
- // No need to traverse iterator tree if there's only one filter or compute_string_components() has been called.
+ // No need to traverse iterator tree if there's only one filter or compute_iterators() has been called.
if (is_filter_result_initialized) {
if (++result_index >= filter_result.count) {
validity = invalid;
@@ -477,7 +477,16 @@ void filter_result_iterator_t::next() {
field f = index->search_schema.at(a_filter.field_name);
- if (f.is_string()) {
+ if (f.is_bool()) {
+ bool_iterator.next();
+ if (!bool_iterator.is_valid) {
+ validity = invalid;
+ return;
+ }
+
+ seq_id = bool_iterator.seq_id;
+ return;
+ } else if (f.is_string()) {
if (filter_node->filter_exp.apply_not_equals) {
do {
if (++seq_id >= result_index) {
@@ -629,7 +638,7 @@ void filter_result_iterator_t::init() {
approx_filter_ids_length = std::max(left_it->approx_filter_ids_length, right_it->approx_filter_ids_length);
}
- // Rearranging the subtree in hope to reduce computation if/when compute_string_components() is called.
+ // Rearranging the subtree in hope to reduce computation if/when compute_iterators() is called.
if (left_it->approx_filter_ids_length > right_it->approx_filter_ids_length) {
std::swap(left_it, right_it);
}
@@ -932,6 +941,20 @@ void filter_result_iterator_t::init() {
} else {
auto num_tree = index->numerical_index.at(a_filter.field_name);
+ // For a boolean filter like `in_stock: true` that could match a large number of ids, we use bool_iterator.
+ if (a_filter.values.size() == 1 && a_filter.comparators[0] == EQUALS && !a_filter.apply_not_equals &&
+ num_tree->approx_search_count(EQUALS, (a_filter.values[0] == "1" ? 1 : 0)) > bool_filter_ids_threshold) {
+ bool_iterator = num_tree_t::iterator_t(num_tree, EQUALS, (a_filter.values[0] == "1" ? 1 : 0));
+ if (!bool_iterator.is_valid) {
+ validity = invalid;
+ return;
+ }
+
+ seq_id = bool_iterator.seq_id;
+ approx_filter_ids_length = bool_iterator.approx_filter_ids_length;
+ return;
+ }
+
size_t value_index = 0;
for (const std::string& filter_value : a_filter.values) {
int64_t bool_int64 = (filter_value == "1") ? 1 : 0;
@@ -1165,19 +1188,19 @@ void filter_result_iterator_t::init() {
}
if (a_filter.values.size() > 10) {
- compute_string_components();
+ compute_iterators();
return;
}
if (a_filter.apply_not_equals &&
index->seq_ids->num_ids() - approx_filter_ids_length < string_filter_ids_threshold) {
// Since there are very few matches, and we have to apply not equals, iteration will be inefficient.
- compute_string_components();
+ compute_iterators();
return;
} else if (a_filter.apply_not_equals) {
all_seq_ids_iter = index->seq_ids->new_iterator();
} else if (approx_filter_ids_length < string_filter_ids_threshold) {
- compute_string_components();
+ compute_iterators();
return;
}
@@ -1191,7 +1214,7 @@ void filter_result_iterator_t::skip_to(uint32_t id, const bool& override_timeout
return;
}
- // No need to traverse iterator tree if there's only one filter or compute_string_components() has been called.
+ // No need to traverse iterator tree if there's only one filter or compute_iterators() has been called.
if (is_filter_result_initialized) {
ArrayUtils::skip_index_to_id(result_index, filter_result.docs, filter_result.count, id);
@@ -1233,7 +1256,16 @@ void filter_result_iterator_t::skip_to(uint32_t id, const bool& override_timeout
field f = index->search_schema.at(a_filter.field_name);
- if (f.is_string()) {
+ if (f.is_bool()) {
+ bool_iterator.skip_to(id);
+ if (!bool_iterator.is_valid) {
+ validity = invalid;
+ return;
+ }
+
+ seq_id = bool_iterator.seq_id;
+ return;
+ } else if (f.is_string()) {
if (filter_node->filter_exp.apply_not_equals) {
if (id < seq_id) {
return;
@@ -1306,7 +1338,7 @@ int filter_result_iterator_t::is_valid(uint32_t id) {
return -1;
}
- // No need to traverse iterator tree if there's only one filter or compute_string_components() has been called.
+ // No need to traverse iterator tree if there's only one filter or compute_iterators() has been called.
if (is_filter_result_initialized) {
skip_to(id);
return validity ? (seq_id == id ? 1 : 0) : -1;
@@ -1436,7 +1468,7 @@ void filter_result_iterator_t::reset(const bool& override_timeout) {
return;
}
- // No need to traverse iterator tree if there's only one filter or compute_string_components() has been called.
+ // No need to traverse iterator tree if there's only one filter or compute_iterators() has been called.
if (is_filter_result_initialized) {
if (filter_result.count == 0) {
validity = invalid;
@@ -1479,7 +1511,18 @@ void filter_result_iterator_t::reset(const bool& override_timeout) {
field f = index->search_schema.at(a_filter.field_name);
- if (f.is_string()) {
+ if (f.is_bool()) {
+ auto num_tree = index->numerical_index.at(a_filter.field_name);
+ bool_iterator = num_tree_t::iterator_t(num_tree, EQUALS, (a_filter.values[0] == "1" ? 1 : 0));
+ if (!bool_iterator.is_valid) {
+ validity = invalid;
+ return;
+ }
+
+ seq_id = bool_iterator.seq_id;
+ validity = valid;
+ return;
+ } else if (f.is_string()) {
for (uint32_t i = 0; i < posting_lists.size(); i++) {
auto const& plists = posting_lists[i];
@@ -1589,7 +1632,7 @@ void filter_result_iterator_t::and_scalar(const uint32_t* A, const uint32_t& len
}
if (!is_filter_result_initialized) {
- compute_string_components();
+ compute_iterators();
}
std::vector<uint32_t> match_indexes;
@@ -1755,7 +1798,7 @@ void filter_result_iterator_t::get_n_ids(const uint32_t& n,
return get_n_ids(n, result, override_timeout);
}
- // This method is only called in Index::search_wildcard after filter_result_iterator_t::compute_string_components.
+ // This method is only called in Index::search_wildcard after filter_result_iterator_t::compute_iterators.
if (!is_filter_result_initialized) {
return;
}
@@ -1824,7 +1867,7 @@ filter_result_iterator_t::filter_result_iterator_t(uint32_t* ids, const uint32_t
if (validity) {
seq_id = filter_result.docs[result_index];
is_filter_result_initialized = true;
- filter_node = new filter_node_t({"dummy", {}, {}});
+ filter_node = new filter_node_t(filter{"dummy", {}, {}});
delete_filter_node = true;
if (search_stop != UINT64_MAX) {
@@ -1857,7 +1900,7 @@ void filter_result_iterator_t::add_phrase_ids(filter_result_iterator_t*& fit,
fit = root_iterator;
}
-void filter_result_iterator_t::compute_string_components() {
+void filter_result_iterator_t::compute_iterators() {
if (filter_node == nullptr) {
validity = invalid;
is_filter_result_initialized = false;
@@ -1873,8 +1916,8 @@ void filter_result_iterator_t::compute_string_components() {
}
if (filter_node->isOperator) {
- left_it->compute_string_components();
- right_it->compute_string_components();
+ left_it->compute_iterators();
+ right_it->compute_iterators();
if (filter_node->filter_operator == AND) {
filter_result_t::and_filter_results(left_it->filter_result, right_it->filter_result, filter_result);
@@ -1903,67 +1946,86 @@ void filter_result_iterator_t::compute_string_components() {
return;
}
- // Only string field filter needs to be evaluated.
- if (index->search_index.count(filter_node->filter_exp.field_name) == 0) {
+ if (index->search_schema.count(filter_node->filter_exp.field_name) == 0) {
return;
}
- // Resetting posting_list_iterators.
- for (uint32_t i = 0; i < posting_lists.size(); i++) {
- auto const& plists = posting_lists[i];
+ const filter a_filter = filter_node->filter_exp;
+ field f = index->search_schema.at(a_filter.field_name);
- posting_list_iterators[i].clear();
- for (auto const& plist: plists) {
- posting_list_iterators[i].push_back(plist->new_iterator());
+ if (f.is_bool()) {
+ auto num_tree = index->numerical_index.at(a_filter.field_name);
+
+ int64_t bool_int64 = (a_filter.values[0] == "1") ? 1 : 0;
+ size_t result_size = 0;
+ num_tree->search(a_filter.comparators[0], bool_int64, &filter_result.docs, result_size);
+ filter_result.count = result_size;
+ } else if (f.is_string()) {
+ // Resetting posting_list_iterators.
+ for (uint32_t i = 0; i < posting_lists.size(); i++) {
+ auto const& plists = posting_lists[i];
+
+ posting_list_iterators[i].clear();
+ for (auto const& plist: plists) {
+ posting_list_iterators[i].push_back(plist->new_iterator());
+ }
}
- }
- auto const& a_filter = filter_node->filter_exp;
- auto const& f = index->search_schema.at(a_filter.field_name);
+ uint32_t* or_ids = nullptr;
+ size_t or_ids_size = 0;
- uint32_t* or_ids = nullptr;
- size_t or_ids_size = 0;
+ // aggregates IDs across array of filter values and reduces excessive ORing
+ std::vector<uint32_t> f_id_buff;
- // aggregates IDs across array of filter values and reduces excessive ORing
- std::vector<uint32_t> f_id_buff;
+ for (uint32_t i = 0; i < posting_lists.size(); i++) {
+ auto& p_list = posting_lists[i];
+ if (a_filter.comparators[0] == EQUALS || a_filter.comparators[0] == NOT_EQUALS) {
+ // needs intersection + exact matching (unlike CONTAINS)
+ std::vector<uint32_t> result_id_vec;
+ posting_list_t::intersect(p_list, result_id_vec);
- for (uint32_t i = 0; i < posting_lists.size(); i++) {
- auto& p_list = posting_lists[i];
- if (a_filter.comparators[0] == EQUALS || a_filter.comparators[0] == NOT_EQUALS) {
- // needs intersection + exact matching (unlike CONTAINS)
- std::vector<uint32_t> result_id_vec;
- posting_list_t::intersect(p_list, result_id_vec);
+ if (result_id_vec.empty()) {
+ continue;
+ }
- if (result_id_vec.empty()) {
- continue;
- }
+ // need to do exact match
+ uint32_t* exact_str_ids = new uint32_t[result_id_vec.size()];
+ size_t exact_str_ids_size = 0;
+ std::unique_ptr<uint32_t[]> exact_str_ids_guard(exact_str_ids);
- // need to do exact match
- uint32_t* exact_str_ids = new uint32_t[result_id_vec.size()];
- size_t exact_str_ids_size = 0;
- std::unique_ptr<uint32_t[]> exact_str_ids_guard(exact_str_ids);
+ posting_list_t::get_exact_matches(posting_list_iterators[i], f.is_array(),
+ result_id_vec.data(), result_id_vec.size(),
+ exact_str_ids, exact_str_ids_size);
- posting_list_t::get_exact_matches(posting_list_iterators[i], f.is_array(),
- result_id_vec.data(), result_id_vec.size(),
- exact_str_ids, exact_str_ids_size);
+ if (exact_str_ids_size == 0) {
+ continue;
+ }
- if (exact_str_ids_size == 0) {
- continue;
+ for (size_t ei = 0; ei < exact_str_ids_size; ei++) {
+ f_id_buff.push_back(exact_str_ids[ei]);
+ }
+ } else {
+ // CONTAINS
+ size_t before_size = f_id_buff.size();
+ posting_list_t::intersect(p_list, f_id_buff);
+ if (f_id_buff.size() == before_size) {
+ continue;
+ }
}
- for (size_t ei = 0; ei < exact_str_ids_size; ei++) {
- f_id_buff.push_back(exact_str_ids[ei]);
- }
- } else {
- // CONTAINS
- size_t before_size = f_id_buff.size();
- posting_list_t::intersect(p_list, f_id_buff);
- if (f_id_buff.size() == before_size) {
- continue;
+ if (f_id_buff.size() > 100000 || a_filter.values.size() == 1) {
+ gfx::timsort(f_id_buff.begin(), f_id_buff.end());
+ f_id_buff.erase(std::unique( f_id_buff.begin(), f_id_buff.end() ), f_id_buff.end());
+
+ uint32_t* out = nullptr;
+ or_ids_size = ArrayUtils::or_scalar(or_ids, or_ids_size, f_id_buff.data(), f_id_buff.size(), &out);
+ delete[] or_ids;
+ or_ids = out;
+ std::vector<uint32_t>().swap(f_id_buff); // clears out memory
}
}
- if (f_id_buff.size() > 100000 || a_filter.values.size() == 1) {
+ if (!f_id_buff.empty()) {
gfx::timsort(f_id_buff.begin(), f_id_buff.end());
f_id_buff.erase(std::unique( f_id_buff.begin(), f_id_buff.end() ), f_id_buff.end());
@@ -1973,24 +2035,13 @@ void filter_result_iterator_t::compute_string_components() {
or_ids = out;
std::vector<uint32_t>().swap(f_id_buff); // clears out memory
}
- }
- if (!f_id_buff.empty()) {
- gfx::timsort(f_id_buff.begin(), f_id_buff.end());
- f_id_buff.erase(std::unique( f_id_buff.begin(), f_id_buff.end() ), f_id_buff.end());
-
- uint32_t* out = nullptr;
- or_ids_size = ArrayUtils::or_scalar(or_ids, or_ids_size, f_id_buff.data(), f_id_buff.size(), &out);
- delete[] or_ids;
- or_ids = out;
- std::vector<uint32_t>().swap(f_id_buff); // clears out memory
- }
+ filter_result.docs = or_ids;
+ filter_result.count = or_ids_size;
- filter_result.docs = or_ids;
- filter_result.count = or_ids_size;
-
- if (a_filter.apply_not_equals) {
- apply_not_equals(index->seq_ids->uncompress(), index->seq_ids->num_ids(), filter_result.docs, filter_result.count);
+ if (a_filter.apply_not_equals) {
+ apply_not_equals(index->seq_ids->uncompress(), index->seq_ids->num_ids(), filter_result.docs, filter_result.count);
+ }
}
if (filter_result.count == 0) {
diff --git a/src/index.cpp b/src/index.cpp
index 06fb67fa..3acfa7ed 100644
--- a/src/index.cpp
+++ b/src/index.cpp
@@ -1820,7 +1820,7 @@ Option<bool> Index::do_filtering_with_lock(filter_node_t* const filter_tree_root
return filter_init_op;
}
- filter_result_iterator.compute_string_components();
+ filter_result_iterator.compute_iterators();
if (filter_result_iterator.approx_filter_ids_length == 0) {
return Option(true);
}
@@ -1880,7 +1880,7 @@ Option<bool> Index::do_reference_filtering_with_lock(filter_node_t* const filter
return filter_init_op;
}
- ref_filter_result_iterator.compute_string_components();
+ ref_filter_result_iterator.compute_iterators();
if (ref_filter_result_iterator.approx_filter_ids_length == 0) {
return Option(true);
}
@@ -2873,12 +2873,12 @@ Option<bool> Index::search(std::vector<query_tokens_t>& field_query_tokens, cons
#ifdef TEST_BUILD
if (filter_result_iterator->approx_filter_ids_length > 20) {
- filter_result_iterator->compute_string_components();
+ filter_result_iterator->compute_iterators();
}
#else
if (filter_result_iterator->approx_filter_ids_length < 25'000) {
- filter_result_iterator->compute_string_components();
+ filter_result_iterator->compute_iterators();
}
#endif
@@ -5930,7 +5930,7 @@ Option<bool> Index::search_wildcard(filter_node_t const* const& filter_tree_root
const std::vector<size_t>& geopoint_indices,
const std::string& collection_name) const {
- filter_result_iterator->compute_string_components();
+ filter_result_iterator->compute_iterators();
auto const& approx_filter_ids_length = filter_result_iterator->approx_filter_ids_length;
uint32_t token_bits = 0;
diff --git a/src/num_tree.cpp b/src/num_tree.cpp
index 3e142212..8e41511d 100644
--- a/src/num_tree.cpp
+++ b/src/num_tree.cpp
@@ -187,11 +187,12 @@ void num_tree_t::search(NUM_COMPARATOR comparator, int64_t value, uint32_t** ids
}
}
-void num_tree_t::approx_search_count(NUM_COMPARATOR comparator, int64_t value, uint32_t& ids_len) {
+uint32_t num_tree_t::approx_search_count(NUM_COMPARATOR comparator, int64_t value) {
if (int64map.empty()) {
- return;
+ return 0;
}
+ uint32_t ids_len = 0;
if (comparator == EQUALS) {
const auto& it = int64map.find(value);
if (it != int64map.end()) {
@@ -203,7 +204,7 @@ void num_tree_t::approx_search_count(NUM_COMPARATOR comparator, int64_t value, u
auto iter_ge_value = int64map.lower_bound(value);
if (iter_ge_value == int64map.end()) {
- return;
+ return 0;
}
if (comparator == GREATER_THAN && iter_ge_value->first == value) {
@@ -233,6 +234,8 @@ void num_tree_t::approx_search_count(NUM_COMPARATOR comparator, int64_t value, u
ids_len += val_ids;
}
}
+
+ return ids_len;
}
void num_tree_t::remove(uint64_t value, uint32_t id) {
@@ -371,3 +374,145 @@ num_tree_t::~num_tree_t() {
ids_t::destroy_list(kv.second);
}
}
+
+num_tree_t::iterator_t::iterator_t(num_tree_t* num_tree, NUM_COMPARATOR comparator, int64_t value) {
+ if (num_tree == nullptr || num_tree->int64map.empty() || comparator != EQUALS) {
+ is_valid = false;
+ return;
+ }
+
+ const auto& it = num_tree->int64map.find(value);
+ if (it == num_tree->int64map.end()) {
+ is_valid = false;
+ return;
+ }
+
+ auto obj = it->second;
+ is_compact_id_list = IS_COMPACT_IDS(obj);
+ if (is_compact_id_list) {
+ id_list_array_len = ids_t::num_ids(obj);
+ id_list_array = ids_t::uncompress(obj);
+ approx_filter_ids_length = id_list_array_len;
+
+ is_valid = id_list_array_len > index;
+ if (is_valid) {
+ seq_id = id_list_array[index];
+ }
+ } else {
+ id_list = (id_list_t*)(obj);
+ id_list_iterator = id_list->new_iterator();
+ approx_filter_ids_length = id_list->num_ids();
+
+ is_valid = id_list_iterator.valid();
+ if (is_valid) {
+ seq_id = id_list_iterator.id();
+ }
+ }
+}
+
+int num_tree_t::iterator_t::is_id_valid(uint32_t id) {
+ if (!is_valid) {
+ return -1;
+ }
+
+ skip_to(id);
+ return is_valid ? (seq_id == id) : -1;
+}
+
+void num_tree_t::iterator_t::next() {
+ if (!is_valid) {
+ return;
+ }
+
+ if (is_compact_id_list) {
+ if (++index >= id_list_array_len) {
+ is_valid = false;
+ return;
+ }
+
+ seq_id = id_list_array[index];
+ } else {
+ id_list_iterator.next();
+
+ if (!id_list_iterator.valid()) {
+ is_valid = false;
+ return;
+ }
+ seq_id = id_list_iterator.id();
+ }
+}
+
+void num_tree_t::iterator_t::skip_to(uint32_t id) {
+ if (!is_valid) {
+ return;
+ }
+
+ if (is_compact_id_list) {
+ ArrayUtils::skip_index_to_id(index, id_list_array, id_list_array_len, id);
+
+ if (index >= id_list_array_len) {
+ is_valid = false;
+ return;
+ }
+ seq_id = id_list_array[index];
+ } else {
+ id_list_iterator.skip_to(id);
+
+ if (!id_list_iterator.valid()) {
+ is_valid = false;
+ return;
+ }
+ seq_id = id_list_iterator.id();
+ }
+}
+
+void num_tree_t::iterator_t::reset() {
+ if (is_compact_id_list) {
+ index = 0;
+ is_valid = index < id_list_array_len;
+ if (is_valid) {
+ seq_id = id_list_array[index];
+ }
+ } else {
+ id_list_iterator = id_list->new_iterator();
+ is_valid = id_list_iterator.valid();
+ if (is_valid) {
+ seq_id = id_list_iterator.id();
+ }
+ }
+}
+
+num_tree_t::iterator_t::~iterator_t() {
+ if (is_compact_id_list) {
+ delete[] id_list_array;
+ }
+}
+
+num_tree_t::iterator_t& num_tree_t::iterator_t::operator=(num_tree_t::iterator_t&& obj) noexcept {
+ if (&obj == this) {
+ return *this;
+ }
+
+ if (is_compact_id_list) {
+ delete[] id_list_array;
+ }
+
+ if (obj.is_compact_id_list) {
+ is_compact_id_list = true;
+ id_list_array_len = obj.id_list_array_len;
+ id_list_array = obj.id_list_array;
+ index = obj.index;
+
+ obj.id_list_array = nullptr;
+ } else {
+ is_compact_id_list = false;
+ id_list = obj.id_list;
+ id_list_iterator = id_list->new_iterator();
+ id_list_iterator.skip_to(obj.id_list_iterator.id());
+ }
+
+ approx_filter_ids_length = obj.approx_filter_ids_length;
+ is_valid = obj.is_valid;
+ seq_id = obj.seq_id;
+ return *this;
+}
diff --git a/test/collection_filtering_test.cpp b/test/collection_filtering_test.cpp
index 9c2182ae..41ee554c 100644
--- a/test/collection_filtering_test.cpp
+++ b/test/collection_filtering_test.cpp
@@ -1674,6 +1674,18 @@ TEST_F(CollectionFilteringTest, QueryBoolFields) {
ASSERT_STREQ(id.c_str(), result_id.c_str());
}
+ results = coll_bool->search("*", query_fields, "popular:true", facets, sort_fields, {0}, 10, 1, FREQUENCY, {false}).get();
+ ASSERT_EQ(7, results["hits"].size());
+
+ ids = {"1", "0", "3", "5", "6", "7", "4"};
+
+ for(size_t i = 0; i < results["hits"].size(); i++) {
+ nlohmann::json result = results["hits"].at(i);
+ std::string result_id = result["document"]["id"];
+ std::string id = ids.at(i);
+ ASSERT_EQ(id, result_id);
+ }
+
// alternative `:=` syntax
results = coll_bool->search("the", query_fields, "popular:=true", facets, sort_fields, {0}, 10, 1, FREQUENCY, {false}).get();
ASSERT_EQ(3, results["hits"].size());
diff --git a/test/filter_test.cpp b/test/filter_test.cpp
index b56358ae..e5419838 100644
--- a/test/filter_test.cpp
+++ b/test/filter_test.cpp
@@ -683,6 +683,87 @@ TEST_F(FilterTest, FilterTreeIterator) {
}
ASSERT_EQ(filter_result_iterator_t::invalid, iter_string_not_equals_test_3.validity);
delete filter_tree_root;
+
+ Collection *bool_coll;
+
+ std::vector<field> fields = {field("title", field_types::STRING, false),
+ field("in_stock", field_types::BOOL, false),
+ field("points", field_types::INT32, false),};
+
+ bool_coll = collectionManager.get_collection("bool_coll").get();
+ if(bool_coll == nullptr) {
+ bool_coll = collectionManager.create_collection("bool_coll", 1, fields, "points").get();
+ }
+
+ for(size_t i=0; i<10; i++) {
+ nlohmann::json bool_doc;
+
+ bool_doc["title"] = "title_" + std::to_string(i);
+ bool_doc["in_stock"] = (i < 5 || i % 2) ? "true" : "false";
+ bool_doc["points"] = i;
+
+ ASSERT_TRUE(bool_coll->add(bool_doc.dump()).ok());
+ }
+
+ filter_tree_root = nullptr;
+ filter_op = filter::parse_filter_query("in_stock: false", bool_coll->get_schema(), store, doc_id_prefix,
+ filter_tree_root);
+ ASSERT_TRUE(filter_op.ok());
+
+ auto iter_boolean_test = filter_result_iterator_t(bool_coll->get_name(), bool_coll->_get_index(), filter_tree_root);
+ ASSERT_TRUE(iter_boolean_test.init_status().ok());
+ ASSERT_TRUE(iter_boolean_test._get_is_filter_result_initialized());
+ ASSERT_EQ(2, iter_boolean_test.approx_filter_ids_length);
+
+ expected = {6, 8};
+ for (auto const& i : expected) {
+ ASSERT_EQ(filter_result_iterator_t::valid, iter_boolean_test.validity);
+ ASSERT_EQ(i, iter_boolean_test.seq_id);
+ iter_boolean_test.next();
+ }
+ ASSERT_EQ(filter_result_iterator_t::invalid, iter_boolean_test.validity);
+ delete filter_tree_root;
+
+ filter_tree_root = nullptr;
+ filter_op = filter::parse_filter_query("in_stock: true", bool_coll->get_schema(), store, doc_id_prefix,
+ filter_tree_root);
+ ASSERT_TRUE(filter_op.ok());
+
+ auto iter_boolean_test_2 = filter_result_iterator_t(bool_coll->get_name(), bool_coll->_get_index(), filter_tree_root);
+ ASSERT_TRUE(iter_boolean_test_2.init_status().ok());
+ ASSERT_FALSE(iter_boolean_test_2._get_is_filter_result_initialized());
+ ASSERT_EQ(8, iter_boolean_test_2.approx_filter_ids_length);
+
+ expected = {0, 1, 2, 3, 4, 5, 7, 9};
+ for (auto const& i : expected) {
+ ASSERT_EQ(filter_result_iterator_t::valid, iter_boolean_test_2.validity);
+ ASSERT_EQ(i, iter_boolean_test_2.seq_id);
+ iter_boolean_test_2.next();
+ }
+ ASSERT_EQ(filter_result_iterator_t::invalid, iter_boolean_test_2.validity);
+
+ iter_boolean_test_2.reset();
+
+ expected = {0, 1, 2, 3, 4, 5, 7, 9};
+ for (auto const& i : expected) {
+ ASSERT_EQ(filter_result_iterator_t::valid, iter_boolean_test_2.validity);
+ ASSERT_EQ(i, iter_boolean_test_2.seq_id);
+ iter_boolean_test_2.next();
+ }
+ ASSERT_EQ(filter_result_iterator_t::invalid, iter_boolean_test_2.validity);
+
+ iter_boolean_test_2.reset();
+ iter_boolean_test_2.skip_to(6);
+ ASSERT_EQ(filter_result_iterator_t::valid, iter_boolean_test_2.validity);
+ ASSERT_EQ(7, iter_boolean_test_2.seq_id);
+
+ ASSERT_EQ(0, iter_boolean_test_2.is_valid(8));
+ ASSERT_EQ(filter_result_iterator_t::valid, iter_boolean_test_2.validity);
+ ASSERT_EQ(9, iter_boolean_test_2.seq_id);
+
+ ASSERT_EQ(-1, iter_boolean_test_2.is_valid(10));
+ ASSERT_EQ(filter_result_iterator_t::invalid, iter_boolean_test_2.validity);
+ delete filter_tree_root;
}
TEST_F(FilterTest, FilterTreeIteratorTimeout) {
diff --git a/test/num_tree_test.cpp b/test/num_tree_test.cpp
index b44f5aa5..755aefef 100644
--- a/test/num_tree_test.cpp
+++ b/test/num_tree_test.cpp
@@ -72,3 +72,99 @@ TEST(NumTreeTest, EraseFullList) {
tree.search(NUM_COMPARATOR::EQUALS, 0, &ids, ids_len);
ASSERT_EQ(nullptr, ids);
}
+
+TEST(NumTreeTest, Iterator) {
+ num_tree_t compact_tree;
+ compact_tree.insert(-1200, 0);
+ compact_tree.insert(-1750, 1);
+ compact_tree.insert(0, 2);
+ compact_tree.insert(100, 3);
+ compact_tree.insert(2000, 4);
+
+ compact_tree.insert(-1200, 5);
+ compact_tree.insert(100, 6);
+
+ auto iterator = num_tree_t::iterator_t(&compact_tree, EQUALS, 1);
+ ASSERT_FALSE(iterator.is_valid);
+ iterator.reset();
+ ASSERT_FALSE(iterator.is_valid);
+
+ iterator = num_tree_t::iterator_t(&compact_tree, GREATER_THAN_EQUALS, 0);
+ ASSERT_FALSE(iterator.is_valid);
+ iterator.reset();
+ ASSERT_FALSE(iterator.is_valid);
+
+ iterator = num_tree_t::iterator_t(&compact_tree, EQUALS, 0);
+ std::vector<uint32_t> expected_ids = {2};
+
+ for (const auto& id: expected_ids) {
+ ASSERT_TRUE(iterator.is_valid);
+ ASSERT_EQ(id, iterator.seq_id);
+ iterator.next();
+ }
+ ASSERT_FALSE(iterator.is_valid);
+ iterator.reset();
+ for (const auto& id: expected_ids) {
+ ASSERT_TRUE(iterator.is_valid);
+ ASSERT_EQ(id, iterator.seq_id);
+ iterator.next();
+ }
+ ASSERT_FALSE(iterator.is_valid);
+
+ iterator = num_tree_t::iterator_t(&compact_tree, EQUALS, -1200);
+ expected_ids = {0, 5};
+
+ for (const auto& id: expected_ids) {
+ ASSERT_TRUE(iterator.is_valid);
+ ASSERT_EQ(id, iterator.seq_id);
+ iterator.next();
+ }
+ ASSERT_FALSE(iterator.is_valid);
+ iterator.reset();
+ for (const auto& id: expected_ids) {
+ ASSERT_TRUE(iterator.is_valid);
+ ASSERT_EQ(id, iterator.seq_id);
+ iterator.next();
+ }
+ ASSERT_FALSE(iterator.is_valid);
+
+ iterator.reset();
+ iterator.skip_to(1);
+ ASSERT_TRUE(iterator.is_valid);
+ ASSERT_EQ(5, iterator.seq_id);
+
+ iterator.skip_to(10);
+ ASSERT_FALSE(iterator.is_valid);
+
+ num_tree_t tree;
+ for (uint32_t i = 0; i < 100; i++) {
+ tree.insert(1, i);
+ }
+
+ iterator = num_tree_t::iterator_t(&tree, EQUALS, 1);
+ expected_ids = {};
+ for (uint32_t i = 0; i < 100; i++) {
+ expected_ids.push_back(i);
+ }
+
+ for (const auto& id: expected_ids) {
+ ASSERT_TRUE(iterator.is_valid);
+ ASSERT_EQ(id, iterator.seq_id);
+ iterator.next();
+ }
+ ASSERT_FALSE(iterator.is_valid);
+ iterator.reset();
+ for (const auto& id: expected_ids) {
+ ASSERT_TRUE(iterator.is_valid);
+ ASSERT_EQ(id, iterator.seq_id);
+ iterator.next();
+ }
+ ASSERT_FALSE(iterator.is_valid);
+
+ iterator.reset();
+ iterator.skip_to(50);
+ ASSERT_TRUE(iterator.is_valid);
+ ASSERT_EQ(50, iterator.seq_id);
+ iterator.skip_to(100);
+ ASSERT_FALSE(iterator.is_valid);
+}