Open Source alternative to Algolia + Pinecone and an Easier-to-Use alternative to ElasticSearch ⚡ 🔍 ✨ Fast, typo tolerant, in-memory fuzzy Search Engine for building delightful search experiences
C++ algoliadatastoreelasticsearchenterprise-searchfacetingfull-text-searchfuzzy-searchgeosearchin-memoryinstantsearchmerchandisingpineconesearchsearch-enginesemantic-searchsimilarity-searchsite-searchsynonymstypo-tolerancevector-search
num_tree iterator (#1538)
Harpreet Sangar54492fafdc5f
diff --git a/include/art.h b/include/art.hindex 92e043e3..7fedbbea 100644--- a/include/art.h+++ b/include/art.h@@ -7,6 +7,7 @@#include "array.h"#include "sorted_array.h"#include "filter_result_iterator.h"+#include "filter.h"#define IGNORE_PRINTF 1@@ -146,17 +147,6 @@ enum token_ordering {MAX_SCORE};-enum NUM_COMPARATOR {- LESS_THAN,- LESS_THAN_EQUALS,- EQUALS,- NOT_EQUALS,- CONTAINS,- GREATER_THAN,- GREATER_THAN_EQUALS,- RANGE_INCLUSIVE-};-/*** Initializes an ART tree* @return 0 on success.diff --git a/include/filter.h b/include/filter.hindex 84dd793b..e6071eba 100644--- a/include/filter.h+++ b/include/filter.h@@ -3,10 +3,20 @@#include <string>#include <map>#include <tsl/htrie_map.h>-#include <art.h>#include <json.hpp>#include "store.h"+enum NUM_COMPARATOR {+ LESS_THAN,+ LESS_THAN_EQUALS,+ EQUALS,+ NOT_EQUALS,+ CONTAINS,+ GREATER_THAN,+ GREATER_THAN_EQUALS,+ RANGE_INCLUSIVE+};+enum FILTER_OPERATOR {AND,ORdiff --git a/include/filter_result_iterator.h b/include/filter_result_iterator.hindex f36ae750..04af6e69 100644--- a/include/filter_result_iterator.h+++ b/include/filter_result_iterator.h@@ -5,6 +5,7 @@#include <utility>#include <vector>#include <memory>+#include "num_tree.h"#include "option.h"#include "posting_list.h"#include "id_list.h"@@ -214,9 +215,11 @@ struct filter_result_t {#ifdef TEST_BUILDconstexpr uint16_t function_call_modulo = 10;constexpr uint16_t string_filter_ids_threshold = 3;+ constexpr uint16_t bool_filter_ids_threshold = 3;#elseconstexpr uint16_t function_call_modulo = 16'384;constexpr uint16_t string_filter_ids_threshold = 20'000;+ constexpr uint16_t bool_filter_ids_threshold = 20'000;#endifstruct filter_result_iterator_timeout_info {@@ -257,6 +260,9 @@ private:/// exist in `index->seq_ids`.id_list_t::iterator_t all_seq_ids_iter = id_list_t::iterator_t(nullptr, nullptr, nullptr, false);+ /// Used in case of a single boolean filter matching more than `bool_filter_ids_threshold` ids.+ num_tree_t::iterator_t bool_iterator = num_tree_t::iterator_t(nullptr, NUM_COMPARATOR::EQUALS, 0);+bool delete_filter_node = false;std::unique_ptr<filter_result_iterator_timeout_info> timeout_info;@@ -322,7 +328,7 @@ public:Option<bool> init_status();/// Recursively computes the result of each node and stores the final result in the root node.- void compute_string_components();+ void compute_iterators();/// Returns a tri-state:/// 0: id is not valid@@ -366,7 +372,7 @@ public:static void add_phrase_ids(filter_result_iterator_t*& filter_result_iterator,uint32_t* phrase_result_ids, const uint32_t& phrase_result_count);- bool _get_is_filter_result_initialized() const {+ [[nodiscard]] bool _get_is_filter_result_initialized() const {return is_filter_result_initialized;}};diff --git a/include/num_tree.h b/include/num_tree.hindex 5315922a..b0b1a951 100644--- a/include/num_tree.h+++ b/include/num_tree.h@@ -4,8 +4,8 @@#include "sparsepp.h"#include "sorted_array.h"#include "array_utils.h"-#include "art.h"#include "ids_t.h"+#include "filter.h"class num_tree_t {private:@@ -42,7 +42,7 @@ public:void search(NUM_COMPARATOR comparator, int64_t value, uint32_t** ids, size_t& ids_len);- void approx_search_count(NUM_COMPARATOR comparator, int64_t value, uint32_t& ids_len);+ uint32_t approx_search_count(NUM_COMPARATOR comparator, int64_t value);void remove(uint64_t value, uint32_t id);@@ -57,4 +57,48 @@ public:uint32_t*& result_ids) const;std::pair<int64_t, int64_t> get_min_max(const uint32_t* result_ids, size_t result_ids_len);++ class iterator_t {+ /// If true, `id_list_array` is initialized otherwise `id_list_iterator` is.+ bool is_compact_id_list = true;++ uint32_t index = 0;+ uint32_t id_list_array_len = 0;+ uint32_t* id_list_array = nullptr;++ id_list_t* id_list = nullptr;+ id_list_t::iterator_t id_list_iterator = id_list_t::iterator_t(nullptr, nullptr, nullptr, false);++ public:++ uint32_t seq_id = 0;+ bool is_valid = true;++ /// Holds the upper-bound of the number of seq ids this iterator would match.+ /// Useful in a scenario where we need to differentiate between iterator not matching any document v/s iterator+ /// reaching it's end. (is_valid would be false in both these cases)+ uint32_t approx_filter_ids_length = 0;++ explicit iterator_t(num_tree_t* num_tree, NUM_COMPARATOR comparator, int64_t value);++ ~iterator_t();++ iterator_t& operator=(iterator_t&& obj) noexcept;++ /// Returns a tri-state:+ /// 0: id is not valid+ /// 1: id is valid+ /// -1: end of iterator+ [[nodiscard]] int is_id_valid(uint32_t id);++ /// Advances the iterator to get the next seq_id. The iterator may become invalid during this operation.+ void next();++ /// Advances the iterator until the seq_id reaches or just overshoots id. The iterator may become invalid during+ /// this operation.+ void skip_to(uint32_t id);++ /// Returns to the initial state of the iterator.+ void reset();+ };};\ No newline at end of filediff --git a/src/filter_result_iterator.cpp b/src/filter_result_iterator.cppindex 4ff0e76c..c8b50144 100644--- a/src/filter_result_iterator.cpp+++ b/src/filter_result_iterator.cpp@@ -429,7 +429,7 @@ void filter_result_iterator_t::next() {return;}- // No need to traverse iterator tree if there's only one filter or compute_string_components() has been called.+ // No need to traverse iterator tree if there's only one filter or compute_iterators() has been called.if (is_filter_result_initialized) {if (++result_index >= filter_result.count) {validity = invalid;@@ -477,7 +477,16 @@ void filter_result_iterator_t::next() {field f = index->search_schema.at(a_filter.field_name);- if (f.is_string()) {+ if (f.is_bool()) {+ bool_iterator.next();+ if (!bool_iterator.is_valid) {+ validity = invalid;+ return;+ }++ seq_id = bool_iterator.seq_id;+ return;+ } else if (f.is_string()) {if (filter_node->filter_exp.apply_not_equals) {do {if (++seq_id >= result_index) {@@ -629,7 +638,7 @@ void filter_result_iterator_t::init() {approx_filter_ids_length = std::max(left_it->approx_filter_ids_length, right_it->approx_filter_ids_length);}- // Rearranging the subtree in hope to reduce computation if/when compute_string_components() is called.+ // Rearranging the subtree in hope to reduce computation if/when compute_iterators() is called.if (left_it->approx_filter_ids_length > right_it->approx_filter_ids_length) {std::swap(left_it, right_it);}@@ -932,6 +941,20 @@ void filter_result_iterator_t::init() {} else {auto num_tree = index->numerical_index.at(a_filter.field_name);+ // For a boolean filter like `in_stock: true` that could match a large number of ids, we use bool_iterator.+ if (a_filter.values.size() == 1 && a_filter.comparators[0] == EQUALS && !a_filter.apply_not_equals &&+ num_tree->approx_search_count(EQUALS, (a_filter.values[0] == "1" ? 1 : 0)) > bool_filter_ids_threshold) {+ bool_iterator = num_tree_t::iterator_t(num_tree, EQUALS, (a_filter.values[0] == "1" ? 1 : 0));+ if (!bool_iterator.is_valid) {+ validity = invalid;+ return;+ }++ seq_id = bool_iterator.seq_id;+ approx_filter_ids_length = bool_iterator.approx_filter_ids_length;+ return;+ }+size_t value_index = 0;for (const std::string& filter_value : a_filter.values) {int64_t bool_int64 = (filter_value == "1") ? 1 : 0;@@ -1165,19 +1188,19 @@ void filter_result_iterator_t::init() {}if (a_filter.values.size() > 10) {- compute_string_components();+ compute_iterators();return;}if (a_filter.apply_not_equals &&index->seq_ids->num_ids() - approx_filter_ids_length < string_filter_ids_threshold) {// Since there are very few matches, and we have to apply not equals, iteration will be inefficient.- compute_string_components();+ compute_iterators();return;} else if (a_filter.apply_not_equals) {all_seq_ids_iter = index->seq_ids->new_iterator();} else if (approx_filter_ids_length < string_filter_ids_threshold) {- compute_string_components();+ compute_iterators();return;}@@ -1191,7 +1214,7 @@ void filter_result_iterator_t::skip_to(uint32_t id, const bool& override_timeoutreturn;}- // No need to traverse iterator tree if there's only one filter or compute_string_components() has been called.+ // No need to traverse iterator tree if there's only one filter or compute_iterators() has been called.if (is_filter_result_initialized) {ArrayUtils::skip_index_to_id(result_index, filter_result.docs, filter_result.count, id);@@ -1233,7 +1256,16 @@ void filter_result_iterator_t::skip_to(uint32_t id, const bool& override_timeoutfield f = index->search_schema.at(a_filter.field_name);- if (f.is_string()) {+ if (f.is_bool()) {+ bool_iterator.skip_to(id);+ if (!bool_iterator.is_valid) {+ validity = invalid;+ return;+ }++ seq_id = bool_iterator.seq_id;+ return;+ } else if (f.is_string()) {if (filter_node->filter_exp.apply_not_equals) {if (id < seq_id) {return;@@ -1306,7 +1338,7 @@ int filter_result_iterator_t::is_valid(uint32_t id) {return -1;}- // No need to traverse iterator tree if there's only one filter or compute_string_components() has been called.+ // No need to traverse iterator tree if there's only one filter or compute_iterators() has been called.if (is_filter_result_initialized) {skip_to(id);return validity ? (seq_id == id ? 1 : 0) : -1;@@ -1436,7 +1468,7 @@ void filter_result_iterator_t::reset(const bool& override_timeout) {return;}- // No need to traverse iterator tree if there's only one filter or compute_string_components() has been called.+ // No need to traverse iterator tree if there's only one filter or compute_iterators() has been called.if (is_filter_result_initialized) {if (filter_result.count == 0) {validity = invalid;@@ -1479,7 +1511,18 @@ void filter_result_iterator_t::reset(const bool& override_timeout) {field f = index->search_schema.at(a_filter.field_name);- if (f.is_string()) {+ if (f.is_bool()) {+ auto num_tree = index->numerical_index.at(a_filter.field_name);+ bool_iterator = num_tree_t::iterator_t(num_tree, EQUALS, (a_filter.values[0] == "1" ? 1 : 0));+ if (!bool_iterator.is_valid) {+ validity = invalid;+ return;+ }++ seq_id = bool_iterator.seq_id;+ validity = valid;+ return;+ } else if (f.is_string()) {for (uint32_t i = 0; i < posting_lists.size(); i++) {auto const& plists = posting_lists[i];@@ -1589,7 +1632,7 @@ void filter_result_iterator_t::and_scalar(const uint32_t* A, const uint32_t& len}if (!is_filter_result_initialized) {- compute_string_components();+ compute_iterators();}std::vector<uint32_t> match_indexes;@@ -1755,7 +1798,7 @@ void filter_result_iterator_t::get_n_ids(const uint32_t& n,return get_n_ids(n, result, override_timeout);}- // This method is only called in Index::search_wildcard after filter_result_iterator_t::compute_string_components.+ // This method is only called in Index::search_wildcard after filter_result_iterator_t::compute_iterators.if (!is_filter_result_initialized) {return;}@@ -1824,7 +1867,7 @@ filter_result_iterator_t::filter_result_iterator_t(uint32_t* ids, const uint32_tif (validity) {seq_id = filter_result.docs[result_index];is_filter_result_initialized = true;- filter_node = new filter_node_t({"dummy", {}, {}});+ filter_node = new filter_node_t(filter{"dummy", {}, {}});delete_filter_node = true;if (search_stop != UINT64_MAX) {@@ -1857,7 +1900,7 @@ void filter_result_iterator_t::add_phrase_ids(filter_result_iterator_t*& fit,fit = root_iterator;}-void filter_result_iterator_t::compute_string_components() {+void filter_result_iterator_t::compute_iterators() {if (filter_node == nullptr) {validity = invalid;is_filter_result_initialized = false;@@ -1873,8 +1916,8 @@ void filter_result_iterator_t::compute_string_components() {}if (filter_node->isOperator) {- left_it->compute_string_components();- right_it->compute_string_components();+ left_it->compute_iterators();+ right_it->compute_iterators();if (filter_node->filter_operator == AND) {filter_result_t::and_filter_results(left_it->filter_result, right_it->filter_result, filter_result);@@ -1903,67 +1946,86 @@ void filter_result_iterator_t::compute_string_components() {return;}- // Only string field filter needs to be evaluated.- if (index->search_index.count(filter_node->filter_exp.field_name) == 0) {+ if (index->search_schema.count(filter_node->filter_exp.field_name) == 0) {return;}- // Resetting posting_list_iterators.- for (uint32_t i = 0; i < posting_lists.size(); i++) {- auto const& plists = posting_lists[i];+ const filter a_filter = filter_node->filter_exp;+ field f = index->search_schema.at(a_filter.field_name);- posting_list_iterators[i].clear();- for (auto const& plist: plists) {- posting_list_iterators[i].push_back(plist->new_iterator());+ if (f.is_bool()) {+ auto num_tree = index->numerical_index.at(a_filter.field_name);++ int64_t bool_int64 = (a_filter.values[0] == "1") ? 1 : 0;+ size_t result_size = 0;+ num_tree->search(a_filter.comparators[0], bool_int64, &filter_result.docs, result_size);+ filter_result.count = result_size;+ } else if (f.is_string()) {+ // Resetting posting_list_iterators.+ for (uint32_t i = 0; i < posting_lists.size(); i++) {+ auto const& plists = posting_lists[i];++ posting_list_iterators[i].clear();+ for (auto const& plist: plists) {+ posting_list_iterators[i].push_back(plist->new_iterator());+ }}- }- auto const& a_filter = filter_node->filter_exp;- auto const& f = index->search_schema.at(a_filter.field_name);+ uint32_t* or_ids = nullptr;+ size_t or_ids_size = 0;- uint32_t* or_ids = nullptr;- size_t or_ids_size = 0;+ // aggregates IDs across array of filter values and reduces excessive ORing+ std::vector<uint32_t> f_id_buff;- // aggregates IDs across array of filter values and reduces excessive ORing- std::vector<uint32_t> f_id_buff;+ for (uint32_t i = 0; i < posting_lists.size(); i++) {+ auto& p_list = posting_lists[i];+ if (a_filter.comparators[0] == EQUALS || a_filter.comparators[0] == NOT_EQUALS) {+ // needs intersection + exact matching (unlike CONTAINS)+ std::vector<uint32_t> result_id_vec;+ posting_list_t::intersect(p_list, result_id_vec);- for (uint32_t i = 0; i < posting_lists.size(); i++) {- auto& p_list = posting_lists[i];- if (a_filter.comparators[0] == EQUALS || a_filter.comparators[0] == NOT_EQUALS) {- // needs intersection + exact matching (unlike CONTAINS)- std::vector<uint32_t> result_id_vec;- posting_list_t::intersect(p_list, result_id_vec);+ if (result_id_vec.empty()) {+ continue;+ }- if (result_id_vec.empty()) {- continue;- }+ // need to do exact match+ uint32_t* exact_str_ids = new uint32_t[result_id_vec.size()];+ size_t exact_str_ids_size = 0;+ std::unique_ptr<uint32_t[]> exact_str_ids_guard(exact_str_ids);- // need to do exact match- uint32_t* exact_str_ids = new uint32_t[result_id_vec.size()];- size_t exact_str_ids_size = 0;- std::unique_ptr<uint32_t[]> exact_str_ids_guard(exact_str_ids);+ posting_list_t::get_exact_matches(posting_list_iterators[i], f.is_array(),+ result_id_vec.data(), result_id_vec.size(),+ exact_str_ids, exact_str_ids_size);- posting_list_t::get_exact_matches(posting_list_iterators[i], f.is_array(),- result_id_vec.data(), result_id_vec.size(),- exact_str_ids, exact_str_ids_size);+ if (exact_str_ids_size == 0) {+ continue;+ }- if (exact_str_ids_size == 0) {- continue;+ for (size_t ei = 0; ei < exact_str_ids_size; ei++) {+ f_id_buff.push_back(exact_str_ids[ei]);+ }+ } else {+ // CONTAINS+ size_t before_size = f_id_buff.size();+ posting_list_t::intersect(p_list, f_id_buff);+ if (f_id_buff.size() == before_size) {+ continue;+ }}- for (size_t ei = 0; ei < exact_str_ids_size; ei++) {- f_id_buff.push_back(exact_str_ids[ei]);- }- } else {- // CONTAINS- size_t before_size = f_id_buff.size();- posting_list_t::intersect(p_list, f_id_buff);- if (f_id_buff.size() == before_size) {- continue;+ if (f_id_buff.size() > 100000 || a_filter.values.size() == 1) {+ gfx::timsort(f_id_buff.begin(), f_id_buff.end());+ f_id_buff.erase(std::unique( f_id_buff.begin(), f_id_buff.end() ), f_id_buff.end());++ uint32_t* out = nullptr;+ or_ids_size = ArrayUtils::or_scalar(or_ids, or_ids_size, f_id_buff.data(), f_id_buff.size(), &out);+ delete[] or_ids;+ or_ids = out;+ std::vector<uint32_t>().swap(f_id_buff); // clears out memory}}- if (f_id_buff.size() > 100000 || a_filter.values.size() == 1) {+ if (!f_id_buff.empty()) {gfx::timsort(f_id_buff.begin(), f_id_buff.end());f_id_buff.erase(std::unique( f_id_buff.begin(), f_id_buff.end() ), f_id_buff.end());@@ -1973,24 +2035,13 @@ void filter_result_iterator_t::compute_string_components() {or_ids = out;std::vector<uint32_t>().swap(f_id_buff); // clears out memory}- }- if (!f_id_buff.empty()) {- gfx::timsort(f_id_buff.begin(), f_id_buff.end());- f_id_buff.erase(std::unique( f_id_buff.begin(), f_id_buff.end() ), f_id_buff.end());-- uint32_t* out = nullptr;- or_ids_size = ArrayUtils::or_scalar(or_ids, or_ids_size, f_id_buff.data(), f_id_buff.size(), &out);- delete[] or_ids;- or_ids = out;- std::vector<uint32_t>().swap(f_id_buff); // clears out memory- }+ filter_result.docs = or_ids;+ filter_result.count = or_ids_size;- filter_result.docs = or_ids;- filter_result.count = or_ids_size;-- if (a_filter.apply_not_equals) {- apply_not_equals(index->seq_ids->uncompress(), index->seq_ids->num_ids(), filter_result.docs, filter_result.count);+ if (a_filter.apply_not_equals) {+ apply_not_equals(index->seq_ids->uncompress(), index->seq_ids->num_ids(), filter_result.docs, filter_result.count);+ }}if (filter_result.count == 0) {diff --git a/src/index.cpp b/src/index.cppindex 06fb67fa..3acfa7ed 100644--- a/src/index.cpp+++ b/src/index.cpp@@ -1820,7 +1820,7 @@ Option<bool> Index::do_filtering_with_lock(filter_node_t* const filter_tree_rootreturn filter_init_op;}- filter_result_iterator.compute_string_components();+ filter_result_iterator.compute_iterators();if (filter_result_iterator.approx_filter_ids_length == 0) {return Option(true);}@@ -1880,7 +1880,7 @@ Option<bool> Index::do_reference_filtering_with_lock(filter_node_t* const filterreturn filter_init_op;}- ref_filter_result_iterator.compute_string_components();+ ref_filter_result_iterator.compute_iterators();if (ref_filter_result_iterator.approx_filter_ids_length == 0) {return Option(true);}@@ -2873,12 +2873,12 @@ Option<bool> Index::search(std::vector<query_tokens_t>& field_query_tokens, cons#ifdef TEST_BUILDif (filter_result_iterator->approx_filter_ids_length > 20) {- filter_result_iterator->compute_string_components();+ filter_result_iterator->compute_iterators();}#elseif (filter_result_iterator->approx_filter_ids_length < 25'000) {- filter_result_iterator->compute_string_components();+ filter_result_iterator->compute_iterators();}#endif@@ -5930,7 +5930,7 @@ Option<bool> Index::search_wildcard(filter_node_t const* const& filter_tree_rootconst std::vector<size_t>& geopoint_indices,const std::string& collection_name) const {- filter_result_iterator->compute_string_components();+ filter_result_iterator->compute_iterators();auto const& approx_filter_ids_length = filter_result_iterator->approx_filter_ids_length;uint32_t token_bits = 0;diff --git a/src/num_tree.cpp b/src/num_tree.cppindex 3e142212..8e41511d 100644--- a/src/num_tree.cpp+++ b/src/num_tree.cpp@@ -187,11 +187,12 @@ void num_tree_t::search(NUM_COMPARATOR comparator, int64_t value, uint32_t** ids}}-void num_tree_t::approx_search_count(NUM_COMPARATOR comparator, int64_t value, uint32_t& ids_len) {+uint32_t num_tree_t::approx_search_count(NUM_COMPARATOR comparator, int64_t value) {if (int64map.empty()) {- return;+ return 0;}+ uint32_t ids_len = 0;if (comparator == EQUALS) {const auto& it = int64map.find(value);if (it != int64map.end()) {@@ -203,7 +204,7 @@ void num_tree_t::approx_search_count(NUM_COMPARATOR comparator, int64_t value, uauto iter_ge_value = int64map.lower_bound(value);if (iter_ge_value == int64map.end()) {- return;+ return 0;}if (comparator == GREATER_THAN && iter_ge_value->first == value) {@@ -233,6 +234,8 @@ void num_tree_t::approx_search_count(NUM_COMPARATOR comparator, int64_t value, uids_len += val_ids;}}++ return ids_len;}void num_tree_t::remove(uint64_t value, uint32_t id) {@@ -371,3 +374,145 @@ num_tree_t::~num_tree_t() {ids_t::destroy_list(kv.second);}}++num_tree_t::iterator_t::iterator_t(num_tree_t* num_tree, NUM_COMPARATOR comparator, int64_t value) {+ if (num_tree == nullptr || num_tree->int64map.empty() || comparator != EQUALS) {+ is_valid = false;+ return;+ }++ const auto& it = num_tree->int64map.find(value);+ if (it == num_tree->int64map.end()) {+ is_valid = false;+ return;+ }++ auto obj = it->second;+ is_compact_id_list = IS_COMPACT_IDS(obj);+ if (is_compact_id_list) {+ id_list_array_len = ids_t::num_ids(obj);+ id_list_array = ids_t::uncompress(obj);+ approx_filter_ids_length = id_list_array_len;++ is_valid = id_list_array_len > index;+ if (is_valid) {+ seq_id = id_list_array[index];+ }+ } else {+ id_list = (id_list_t*)(obj);+ id_list_iterator = id_list->new_iterator();+ approx_filter_ids_length = id_list->num_ids();++ is_valid = id_list_iterator.valid();+ if (is_valid) {+ seq_id = id_list_iterator.id();+ }+ }+}++int num_tree_t::iterator_t::is_id_valid(uint32_t id) {+ if (!is_valid) {+ return -1;+ }++ skip_to(id);+ return is_valid ? (seq_id == id) : -1;+}++void num_tree_t::iterator_t::next() {+ if (!is_valid) {+ return;+ }++ if (is_compact_id_list) {+ if (++index >= id_list_array_len) {+ is_valid = false;+ return;+ }++ seq_id = id_list_array[index];+ } else {+ id_list_iterator.next();++ if (!id_list_iterator.valid()) {+ is_valid = false;+ return;+ }+ seq_id = id_list_iterator.id();+ }+}++void num_tree_t::iterator_t::skip_to(uint32_t id) {+ if (!is_valid) {+ return;+ }++ if (is_compact_id_list) {+ ArrayUtils::skip_index_to_id(index, id_list_array, id_list_array_len, id);++ if (index >= id_list_array_len) {+ is_valid = false;+ return;+ }+ seq_id = id_list_array[index];+ } else {+ id_list_iterator.skip_to(id);++ if (!id_list_iterator.valid()) {+ is_valid = false;+ return;+ }+ seq_id = id_list_iterator.id();+ }+}++void num_tree_t::iterator_t::reset() {+ if (is_compact_id_list) {+ index = 0;+ is_valid = index < id_list_array_len;+ if (is_valid) {+ seq_id = id_list_array[index];+ }+ } else {+ id_list_iterator = id_list->new_iterator();+ is_valid = id_list_iterator.valid();+ if (is_valid) {+ seq_id = id_list_iterator.id();+ }+ }+}++num_tree_t::iterator_t::~iterator_t() {+ if (is_compact_id_list) {+ delete[] id_list_array;+ }+}++num_tree_t::iterator_t& num_tree_t::iterator_t::operator=(num_tree_t::iterator_t&& obj) noexcept {+ if (&obj == this) {+ return *this;+ }++ if (is_compact_id_list) {+ delete[] id_list_array;+ }++ if (obj.is_compact_id_list) {+ is_compact_id_list = true;+ id_list_array_len = obj.id_list_array_len;+ id_list_array = obj.id_list_array;+ index = obj.index;++ obj.id_list_array = nullptr;+ } else {+ is_compact_id_list = false;+ id_list = obj.id_list;+ id_list_iterator = id_list->new_iterator();+ id_list_iterator.skip_to(obj.id_list_iterator.id());+ }++ approx_filter_ids_length = obj.approx_filter_ids_length;+ is_valid = obj.is_valid;+ seq_id = obj.seq_id;+ return *this;+}diff --git a/test/collection_filtering_test.cpp b/test/collection_filtering_test.cppindex 9c2182ae..41ee554c 100644--- a/test/collection_filtering_test.cpp+++ b/test/collection_filtering_test.cpp@@ -1674,6 +1674,18 @@ TEST_F(CollectionFilteringTest, QueryBoolFields) {ASSERT_STREQ(id.c_str(), result_id.c_str());}+ results = coll_bool->search("*", query_fields, "popular:true", facets, sort_fields, {0}, 10, 1, FREQUENCY, {false}).get();+ ASSERT_EQ(7, results["hits"].size());++ ids = {"1", "0", "3", "5", "6", "7", "4"};++ for(size_t i = 0; i < results["hits"].size(); i++) {+ nlohmann::json result = results["hits"].at(i);+ std::string result_id = result["document"]["id"];+ std::string id = ids.at(i);+ ASSERT_EQ(id, result_id);+ }+// alternative `:=` syntaxresults = coll_bool->search("the", query_fields, "popular:=true", facets, sort_fields, {0}, 10, 1, FREQUENCY, {false}).get();ASSERT_EQ(3, results["hits"].size());diff --git a/test/filter_test.cpp b/test/filter_test.cppindex b56358ae..e5419838 100644--- a/test/filter_test.cpp+++ b/test/filter_test.cpp@@ -683,6 +683,87 @@ TEST_F(FilterTest, FilterTreeIterator) {}ASSERT_EQ(filter_result_iterator_t::invalid, iter_string_not_equals_test_3.validity);delete filter_tree_root;++ Collection *bool_coll;++ std::vector<field> fields = {field("title", field_types::STRING, false),+ field("in_stock", field_types::BOOL, false),+ field("points", field_types::INT32, false),};++ bool_coll = collectionManager.get_collection("bool_coll").get();+ if(bool_coll == nullptr) {+ bool_coll = collectionManager.create_collection("bool_coll", 1, fields, "points").get();+ }++ for(size_t i=0; i<10; i++) {+ nlohmann::json bool_doc;++ bool_doc["title"] = "title_" + std::to_string(i);+ bool_doc["in_stock"] = (i < 5 || i % 2) ? "true" : "false";+ bool_doc["points"] = i;++ ASSERT_TRUE(bool_coll->add(bool_doc.dump()).ok());+ }++ filter_tree_root = nullptr;+ filter_op = filter::parse_filter_query("in_stock: false", bool_coll->get_schema(), store, doc_id_prefix,+ filter_tree_root);+ ASSERT_TRUE(filter_op.ok());++ auto iter_boolean_test = filter_result_iterator_t(bool_coll->get_name(), bool_coll->_get_index(), filter_tree_root);+ ASSERT_TRUE(iter_boolean_test.init_status().ok());+ ASSERT_TRUE(iter_boolean_test._get_is_filter_result_initialized());+ ASSERT_EQ(2, iter_boolean_test.approx_filter_ids_length);++ expected = {6, 8};+ for (auto const& i : expected) {+ ASSERT_EQ(filter_result_iterator_t::valid, iter_boolean_test.validity);+ ASSERT_EQ(i, iter_boolean_test.seq_id);+ iter_boolean_test.next();+ }+ ASSERT_EQ(filter_result_iterator_t::invalid, iter_boolean_test.validity);+ delete filter_tree_root;++ filter_tree_root = nullptr;+ filter_op = filter::parse_filter_query("in_stock: true", bool_coll->get_schema(), store, doc_id_prefix,+ filter_tree_root);+ ASSERT_TRUE(filter_op.ok());++ auto iter_boolean_test_2 = filter_result_iterator_t(bool_coll->get_name(), bool_coll->_get_index(), filter_tree_root);+ ASSERT_TRUE(iter_boolean_test_2.init_status().ok());+ ASSERT_FALSE(iter_boolean_test_2._get_is_filter_result_initialized());+ ASSERT_EQ(8, iter_boolean_test_2.approx_filter_ids_length);++ expected = {0, 1, 2, 3, 4, 5, 7, 9};+ for (auto const& i : expected) {+ ASSERT_EQ(filter_result_iterator_t::valid, iter_boolean_test_2.validity);+ ASSERT_EQ(i, iter_boolean_test_2.seq_id);+ iter_boolean_test_2.next();+ }+ ASSERT_EQ(filter_result_iterator_t::invalid, iter_boolean_test_2.validity);++ iter_boolean_test_2.reset();++ expected = {0, 1, 2, 3, 4, 5, 7, 9};+ for (auto const& i : expected) {+ ASSERT_EQ(filter_result_iterator_t::valid, iter_boolean_test_2.validity);+ ASSERT_EQ(i, iter_boolean_test_2.seq_id);+ iter_boolean_test_2.next();+ }+ ASSERT_EQ(filter_result_iterator_t::invalid, iter_boolean_test_2.validity);++ iter_boolean_test_2.reset();+ iter_boolean_test_2.skip_to(6);+ ASSERT_EQ(filter_result_iterator_t::valid, iter_boolean_test_2.validity);+ ASSERT_EQ(7, iter_boolean_test_2.seq_id);++ ASSERT_EQ(0, iter_boolean_test_2.is_valid(8));+ ASSERT_EQ(filter_result_iterator_t::valid, iter_boolean_test_2.validity);+ ASSERT_EQ(9, iter_boolean_test_2.seq_id);++ ASSERT_EQ(-1, iter_boolean_test_2.is_valid(10));+ ASSERT_EQ(filter_result_iterator_t::invalid, iter_boolean_test_2.validity);+ delete filter_tree_root;}TEST_F(FilterTest, FilterTreeIteratorTimeout) {diff --git a/test/num_tree_test.cpp b/test/num_tree_test.cppindex b44f5aa5..755aefef 100644--- a/test/num_tree_test.cpp+++ b/test/num_tree_test.cpp@@ -72,3 +72,99 @@ TEST(NumTreeTest, EraseFullList) {tree.search(NUM_COMPARATOR::EQUALS, 0, &ids, ids_len);ASSERT_EQ(nullptr, ids);}++TEST(NumTreeTest, Iterator) {+ num_tree_t compact_tree;+ compact_tree.insert(-1200, 0);+ compact_tree.insert(-1750, 1);+ compact_tree.insert(0, 2);+ compact_tree.insert(100, 3);+ compact_tree.insert(2000, 4);++ compact_tree.insert(-1200, 5);+ compact_tree.insert(100, 6);++ auto iterator = num_tree_t::iterator_t(&compact_tree, EQUALS, 1);+ ASSERT_FALSE(iterator.is_valid);+ iterator.reset();+ ASSERT_FALSE(iterator.is_valid);++ iterator = num_tree_t::iterator_t(&compact_tree, GREATER_THAN_EQUALS, 0);+ ASSERT_FALSE(iterator.is_valid);+ iterator.reset();+ ASSERT_FALSE(iterator.is_valid);++ iterator = num_tree_t::iterator_t(&compact_tree, EQUALS, 0);+ std::vector<uint32_t> expected_ids = {2};++ for (const auto& id: expected_ids) {+ ASSERT_TRUE(iterator.is_valid);+ ASSERT_EQ(id, iterator.seq_id);+ iterator.next();+ }+ ASSERT_FALSE(iterator.is_valid);+ iterator.reset();+ for (const auto& id: expected_ids) {+ ASSERT_TRUE(iterator.is_valid);+ ASSERT_EQ(id, iterator.seq_id);+ iterator.next();+ }+ ASSERT_FALSE(iterator.is_valid);++ iterator = num_tree_t::iterator_t(&compact_tree, EQUALS, -1200);+ expected_ids = {0, 5};++ for (const auto& id: expected_ids) {+ ASSERT_TRUE(iterator.is_valid);+ ASSERT_EQ(id, iterator.seq_id);+ iterator.next();+ }+ ASSERT_FALSE(iterator.is_valid);+ iterator.reset();+ for (const auto& id: expected_ids) {+ ASSERT_TRUE(iterator.is_valid);+ ASSERT_EQ(id, iterator.seq_id);+ iterator.next();+ }+ ASSERT_FALSE(iterator.is_valid);++ iterator.reset();+ iterator.skip_to(1);+ ASSERT_TRUE(iterator.is_valid);+ ASSERT_EQ(5, iterator.seq_id);++ iterator.skip_to(10);+ ASSERT_FALSE(iterator.is_valid);++ num_tree_t tree;+ for (uint32_t i = 0; i < 100; i++) {+ tree.insert(1, i);+ }++ iterator = num_tree_t::iterator_t(&tree, EQUALS, 1);+ expected_ids = {};+ for (uint32_t i = 0; i < 100; i++) {+ expected_ids.push_back(i);+ }++ for (const auto& id: expected_ids) {+ ASSERT_TRUE(iterator.is_valid);+ ASSERT_EQ(id, iterator.seq_id);+ iterator.next();+ }+ ASSERT_FALSE(iterator.is_valid);+ iterator.reset();+ for (const auto& id: expected_ids) {+ ASSERT_TRUE(iterator.is_valid);+ ASSERT_EQ(id, iterator.seq_id);+ iterator.next();+ }+ ASSERT_FALSE(iterator.is_valid);++ iterator.reset();+ iterator.skip_to(50);+ ASSERT_TRUE(iterator.is_valid);+ ASSERT_EQ(50, iterator.seq_id);+ iterator.skip_to(100);+ ASSERT_FALSE(iterator.is_valid);+}